From a628d56e250eed8596c5c0a5e06a97cd46ff029d Mon Sep 17 00:00:00 2001 From: shockrah Date: Wed, 5 May 2021 15:50:39 -0700 Subject: [PATCH 01/27] - Removing dotenv from required dependancies + Leveraging serde_json to use a config.json This lets us use the same config for both the json-api and the rtc-server without adding dependancies --- json-api/.env | 16 ++++++++-- json-api/Cargo.lock | 70 +++----------------------------------------- json-api/Cargo.toml | 2 -- json-api/config.json | 18 ++++++++++++ json-api/src/main.rs | 51 ++++++++++++++++++++++++++------ 5 files changed, 78 insertions(+), 79 deletions(-) create mode 100644 json-api/config.json diff --git a/json-api/.env b/json-api/.env index 04d1e15..17d799e 100644 --- a/json-api/.env +++ b/json-api/.env @@ -1,4 +1,11 @@ +# Example .env file +# In theory this file can be used for _both_ the rtc-server and the json-api server + +# This field is required by the json-api binary to actually find the database DATABASE_URL=mysql://freechat_dev:password@localhost:3306/freechat +# The fields below are unused but are here in case you're confused as to the +# structure of the url above +# It is perfectly safe to delete these DATABASE_NAME=freechat DATABASE_PASS=password DATABASE_USER=freechat_dev @@ -11,9 +18,14 @@ HMAC_PATH=hmac.secret WSS_HMAC_PATH=wss-hmac.secret -# Server meta things +# Public name that your server has SERVER_NAME="Freechat Dev Server" +# Public description of what youre server is about SERVER_DESCRIPTION="Server for sick development things" -# NOTE: most clients shouldn't expect these to end with a slash + +# These are the actual url's which are sent to/used by clients to communicate with +# your instance. If you have a reverse proxy setup then you can configure it so that +# the port numbers are not in the URL(suggested) PUBLIC_URL=http://localhost:4536 PUBLIC_WS_URL=ws://localhost:5648 + diff --git a/json-api/Cargo.lock b/json-api/Cargo.lock index 6fa65be..bea1027 100644 --- a/json-api/Cargo.lock +++ b/json-api/Cargo.lock @@ -12,15 +12,6 @@ version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "739f4a8db6605981345c5654f3a85b056ce52f37a39d34da03f25bf2151ea16e" -[[package]] -name = "aho-corasick" -version = "0.6.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81ce3d38065e618af2d7b77e10c5ad9a069859b4be3c2250f674af3840d9c8a5" -dependencies = [ - "memchr", -] - [[package]] name = "aho-corasick" version = "0.7.15" @@ -272,15 +263,6 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" -[[package]] -name = "dotenv" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "400b347fe65ccfbd8f545c9d9a75d04b0caf23fec49aaa838a9a05398f94c019" -dependencies = [ - "regex 0.2.11", -] - [[package]] name = "flate2" version = "1.0.20" @@ -606,7 +588,6 @@ dependencies = [ "bcrypt", "clap", "db", - "dotenv", "futures", "getrandom 0.1.16", "hyper", @@ -800,7 +781,7 @@ dependencies = [ "num-bigint 0.3.2", "num-traits", "rand 0.8.3", - "regex 1.4.5", + "regex", "rust_decimal", "serde", "serde_json", @@ -967,7 +948,7 @@ checksum = "fd56cbd21fea48d0c440b41cd69c589faacade08c992d9a54e471b79d0fd13eb" dependencies = [ "base64 0.13.0", "once_cell", - "regex 1.4.5", + "regex", ] [[package]] @@ -1146,37 +1127,15 @@ dependencies = [ "bitflags", ] -[[package]] -name = "regex" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9329abc99e39129fcceabd24cf5d85b4671ef7c29c50e972bc5afe32438ec384" -dependencies = [ - "aho-corasick 0.6.10", - "memchr", - "regex-syntax 0.5.6", - "thread_local", - "utf8-ranges", -] - [[package]] name = "regex" version = "1.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "957056ecddbeba1b26965114e191d2e8589ce74db242b6ea25fc4062427a5c19" dependencies = [ - "aho-corasick 0.7.15", + "aho-corasick", "memchr", - "regex-syntax 0.6.23", -] - -[[package]] -name = "regex-syntax" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d707a4fa2637f2dca2ef9fd02225ec7661fe01a53623c1e6515b6916511f7a7" -dependencies = [ - "ucd-util", + "regex-syntax", ] [[package]] @@ -1524,15 +1483,6 @@ dependencies = [ "syn", ] -[[package]] -name = "thread_local" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6b53e329000edc2b34dbe8545fd20e55a333362d0a321909685a19bd28c3f1b" -dependencies = [ - "lazy_static", -] - [[package]] name = "time" version = "0.1.43" @@ -1757,12 +1707,6 @@ version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "879f6906492a7cd215bfa4cf595b600146ccfac0c79bcbd1f3000162af5e8b06" -[[package]] -name = "ucd-util" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c85f514e095d348c279b1e5cd76795082cf15bd59b93207832abe0b1d8fed236" - [[package]] name = "unicode-bidi" version = "0.3.5" @@ -1817,12 +1761,6 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05e42f7c18b8f902290b009cde6d651262f956c98bc51bca4cd1d511c9cd85c7" -[[package]] -name = "utf8-ranges" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4ae116fef2b7fea257ed6440d3cfcff7f190865f170cdad00bb6465bf18ecba" - [[package]] name = "uuid" version = "0.8.2" diff --git a/json-api/Cargo.toml b/json-api/Cargo.toml index 5d7c03e..b8b2b29 100644 --- a/json-api/Cargo.toml +++ b/json-api/Cargo.toml @@ -18,8 +18,6 @@ futures = "0.3" url = "2.2.1" -dotenv = "0.9.0" - # Crypto things getrandom = "0.1" bcrypt = "0.8" diff --git a/json-api/config.json b/json-api/config.json new file mode 100644 index 0000000..f200e31 --- /dev/null +++ b/json-api/config.json @@ -0,0 +1,18 @@ +{ + "#": "The fields below are required and can not be empty", + "#": "All other fields are ignored completely", + + "database_url": "mysql://freechat_dev:password@localhost:3306/freechat", + "hmac_path": "hmac.secret", + "wss_hmac_path": "wss-hmac.secret", + "name": "Freechat Server", + "description": "Chatting server", + "public_url": "http://localhost:4536", + "public_ws_url": "ws://localhost:5648", + + "tags": [ + "Freechat", + "Libre software", + "Chatting" + ] +} diff --git a/json-api/src/main.rs b/json-api/src/main.rs index cd5b611..a1e1a38 100644 --- a/json-api/src/main.rs +++ b/json-api/src/main.rs @@ -1,6 +1,5 @@ extern crate db; extern crate clap; -extern crate dotenv; extern crate getrandom; extern crate bcrypt; extern crate base64; @@ -23,8 +22,8 @@ use hyper::{ HeaderMap }; use mysql_async::Pool; +use serde::Deserialize; -use dotenv::dotenv; use clap::{Arg, App}; use auth::AuthReason; @@ -193,14 +192,51 @@ async fn attempt_owner_creation(name: &str) { let _ = p.disconnect().await; } +fn init_config() -> Result<(), Box> { + #[derive(Deserialize, Debug)] + struct RequiredFields { + pub database_url: String, + pub hmac_path: String, + pub wss_hmac_path: String, + pub name: String, + pub description: Option, + pub public_url: String, + pub public_ws_url: String, + pub tags: Option> + } + use std::fs::File; + use std::io::BufReader; + + let file = File::open("config.json")?; + let reader = BufReader::new(file); + let fields: RequiredFields = serde_json::from_reader(reader)?; + + // Now we can setup each environment variable for this process from config.json + // Note that we only have to do this once since all of these are read from + // lazy statics so the cost is very minimal + set_var("DATABASE_URL", fields.database_url); + + set_var("HMAC_PATH", fields.hmac_path); + set_var("WSS_HMAC_PATH", fields.wss_hmac_path); + + set_var("SERVER_NAME", fields.name); + + set_var("SERVER_DESCRIPTION", fields.description.unwrap_or("".into())); + + set_var("PUBLIC_URL", fields.public_url); + set_var("PUBLIC_WS_URL", fields.public_ws_url); + + Ok(()) +} + #[tokio::main] async fn main() -> Result<(), u16>{ - let mut main_ret: u16 = 0; - let d_result = dotenv(); + let mut main_ret: u16 = 0; let d_result = init_config(); // check for a database_url before the override we get from the cmd line - if let Err(_d) = d_result { + if let Err(d) = d_result { + eprintln!("Config error: {}", d); if let Err(_e) = env::var("DATABASE_URL") { main_ret |= CONFIG_ERR; } @@ -243,9 +279,6 @@ async fn main() -> Result<(), u16>{ .get_matches(); - if let Some(db_url) = args.value_of("db-url") { - set_var("DATABASE_URL", db_url); - } // safe because we have a default value set in code let port = args.value_of("port").unwrap().to_string(); @@ -255,7 +288,7 @@ async fn main() -> Result<(), u16>{ attempt_owner_creation(owner_name).await; } - // This check overrides the value set in the .env since this + // Here we override some of the config.json variables if let Some(hmac) = args.value_of("hmac") { std::env::set_var("HMAC_PATH", hmac); } From 181b1cadc4bd05c62b9762b78205645a1aa7912c Mon Sep 17 00:00:00 2001 From: shockrah Date: Thu, 6 May 2021 13:29:32 -0700 Subject: [PATCH 02/27] + Exposing serde_json to db-lib lyaer This dep was already there but just being used (as we're using serde). Now we can use it at the db layer which makes neighbor struct creation easier * Cargo locks are being updated to reflect new dependancy changes These changes actually reduce the dependancy count overall + Adding a very simple table to hold all the neighbors that an instance may want to reference At some point we may want to support vanity join urls and this would be a good place to reference those There would also need to be a way for admins to add/edit/remove vanity urls but that's for another patch --- json-api/Cargo.lock | 191 +++++++++--------- json-api/db/Cargo.toml | 1 + json-api/db/src/lib.rs | 9 + json-api/db/src/neighbors.rs | 19 ++ .../2021-05-04-200859_neighbors/down.sql | 1 + .../2021-05-04-200859_neighbors/up.sql | 10 + 6 files changed, 136 insertions(+), 95 deletions(-) create mode 100644 json-api/db/src/neighbors.rs create mode 100644 json-api/migrations/2021-05-04-200859_neighbors/down.sql create mode 100644 json-api/migrations/2021-05-04-200859_neighbors/up.sql diff --git a/json-api/Cargo.lock b/json-api/Cargo.lock index bea1027..7ded578 100644 --- a/json-api/Cargo.lock +++ b/json-api/Cargo.lock @@ -14,9 +14,9 @@ checksum = "739f4a8db6605981345c5654f3a85b056ce52f37a39d34da03f25bf2151ea16e" [[package]] name = "aho-corasick" -version = "0.7.15" +version = "0.7.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5" +checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" dependencies = [ "memchr", ] @@ -203,9 +203,9 @@ dependencies = [ [[package]] name = "const_fn" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "076a6803b0dacd6a88cfe64deba628b01533ff5ef265687e6938280c1afd0a28" +checksum = "402da840495de3f976eaefc3485b7f5eb5b0bf9761f9a47be27fe975b3b8c2ec" [[package]] name = "core-foundation" @@ -224,10 +224,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea221b5284a47e40033bf9b66f35f984ec0ea2931eb03505246cd27a963f981b" [[package]] -name = "cpuid-bool" -version = "0.1.2" +name = "cpufeatures" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634" +checksum = "5cd5a7748210e7ec1a9696610b1015e6e31fbf58f77a160801f124bd1c36592a" [[package]] name = "crc32fast" @@ -245,7 +245,8 @@ dependencies = [ "mysql_async", "rand 0.8.3", "serde", - "tokio 1.4.0", + "serde_json", + "tokio 1.5.0", ] [[package]] @@ -309,9 +310,9 @@ dependencies = [ [[package]] name = "futures" -version = "0.3.13" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f55667319111d593ba876406af7c409c0ebb44dc4be6132a783ccf163ea14c1" +checksum = "a9d5813545e459ad3ca1bff9915e9ad7f1a47dc6a91b627ce321d5863b7dd253" dependencies = [ "futures-channel", "futures-core", @@ -324,9 +325,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.13" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c2dd2df839b57db9ab69c2c9d8f3e8c81984781937fe2807dc6dcf3b2ad2939" +checksum = "ce79c6a52a299137a6013061e0cf0e688fce5d7f1bc60125f520912fdb29ec25" dependencies = [ "futures-core", "futures-sink", @@ -334,15 +335,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.13" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15496a72fabf0e62bdc3df11a59a3787429221dd0710ba8ef163d6f7a9112c94" +checksum = "098cd1c6dda6ca01650f1a37a794245eb73181d0d4d4e955e2f3c37db7af1815" [[package]] name = "futures-executor" -version = "0.3.13" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891a4b7b96d84d5940084b2a37632dd65deeae662c114ceaa2c879629c9c0ad1" +checksum = "10f6cb7042eda00f0049b1d2080aa4b93442997ee507eb3828e8bd7577f94c9d" dependencies = [ "futures-core", "futures-task", @@ -351,15 +352,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.13" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71c2c65c57704c32f5241c1223167c2c3294fd34ac020c807ddbe6db287ba59" +checksum = "365a1a1fb30ea1c03a830fdb2158f5236833ac81fa0ad12fe35b29cddc35cb04" [[package]] name = "futures-macro" -version = "0.3.13" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea405816a5139fb39af82c2beb921d52143f556038378d6db21183a5c37fbfb7" +checksum = "668c6733a182cd7deb4f1de7ba3bf2120823835b3bcfbeacf7d2c4a773c1bb8b" dependencies = [ "proc-macro-hack", "proc-macro2", @@ -369,21 +370,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.13" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85754d98985841b7d4f5e8e6fbfa4a4ac847916893ec511a2917ccd8525b8bb3" +checksum = "5c5629433c555de3d82861a7a4e3794a4c40040390907cfbfd7143a92a426c23" [[package]] name = "futures-task" -version = "0.3.13" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa189ef211c15ee602667a6fcfe1c1fd9e07d42250d2156382820fba33c9df80" +checksum = "ba7aa51095076f3ba6d9a1f702f74bd05ec65f555d70d2033d55ba8d69f581bc" [[package]] name = "futures-util" -version = "0.3.13" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1812c7ab8aedf8d6f2701a43e1243acdbcc2b36ab26e2ad421eb99ac963d96d1" +checksum = "3c144ad54d60f23927f0a6b6d816e4271278b64f005ad65e4e35291d2de9c025" dependencies = [ "futures-channel", "futures-core", @@ -433,9 +434,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc018e188373e2777d0ef2467ebff62a08e66c3f5857b23c8fbec3018210dc00" +checksum = "825343c4eef0b63f541f8903f395dc5beb362a979b5799a84062527ef1e37726" dependencies = [ "bytes 1.0.1", "fnv", @@ -445,7 +446,7 @@ dependencies = [ "http", "indexmap", "slab", - "tokio 1.4.0", + "tokio 1.5.0", "tokio-util", "tracing", ] @@ -492,21 +493,21 @@ dependencies = [ [[package]] name = "httparse" -version = "1.3.6" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc35c995b9d93ec174cf9a27d425c7892722101e14993cd227fdb51d70cf9589" +checksum = "4a1ce40d6fc9764887c2fdc7305c3dcc429ba11ff981c1509416afd5697e4437" [[package]] name = "httpdate" -version = "0.3.2" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "494b4d60369511e7dea41cf646832512a94e542f68bb9c49e54518e0f468eb47" +checksum = "05842d0d43232b23ccb7060ecb0f0626922c21f30012e97b767b30afd4a5d4b9" [[package]] name = "hyper" -version = "0.14.5" +version = "0.14.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bf09f61b52cfcf4c00de50df88ae423d6c02354e385a86341133b5338630ad1" +checksum = "1e5f105c494081baa3bf9e200b279e27ec1623895cd504c7dbef8d0b080fcf54" dependencies = [ "bytes 1.0.1", "futures-channel", @@ -520,7 +521,7 @@ dependencies = [ "itoa", "pin-project", "socket2", - "tokio 1.4.0", + "tokio 1.5.0", "tower-service", "tracing", "want", @@ -528,9 +529,9 @@ dependencies = [ [[package]] name = "idna" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89829a5d69c23d348314a7ac337fe39173b61149a9864deabd260983aed48c21" +checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" dependencies = [ "matches", "unicode-bidi", @@ -597,7 +598,7 @@ dependencies = [ "rand 0.7.3", "serde", "serde_json", - "tokio 1.4.0", + "tokio 1.5.0", "tokio-test", "tokio-tungstenite", "url", @@ -625,9 +626,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "lexical" -version = "5.2.1" +version = "5.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a11b9b772d42a9cf7ff6742d4695054070432742b19cecea07e7c4eca8173d25" +checksum = "f404a90a744e32e8be729034fc33b90cf2a56418fbf594d69aa3c0214ad414e5" dependencies = [ "cfg-if 1.0.0", "lexical-core", @@ -635,9 +636,9 @@ dependencies = [ [[package]] name = "lexical-core" -version = "0.7.5" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21f866863575d0e1d654fbeeabdc927292fdf862873dc3c96c6f753357e13374" +checksum = "6607c62aa161d23d17a9072cc5da0be67cdfc89d3afb1e8d9c842bebc2525ffe" dependencies = [ "arrayvec", "bitflags", @@ -648,15 +649,15 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.93" +version = "0.2.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9385f66bf6105b241aa65a61cb923ef20efc665cb9f9bb50ac2f0c4b7f378d41" +checksum = "18794a8ad5b29321f790b55d93dfba91e125cb1a9edbd4f8e3150acc771c1a5e" [[package]] name = "libz-sys" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "602113192b08db8f38796c4e85c39e960c145965140e918018bcde1952429655" +checksum = "de5435b8549c16d423ed0c03dbaafe57cf6c3344744f1242520d59c9d8ecec66" dependencies = [ "cc", "pkg-config", @@ -665,9 +666,9 @@ dependencies = [ [[package]] name = "lock_api" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a3c91c24eae6777794bb1997ad98bbb87daf92890acab859f7eaa4320333176" +checksum = "0382880606dff6d15c9476c416d18690b72742aa7b605bb6dd6ec9030fbf07eb" dependencies = [ "scopeguard", ] @@ -698,9 +699,9 @@ checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" [[package]] name = "memchr" -version = "2.3.4" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" +checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc" [[package]] name = "miniz_oxide" @@ -755,7 +756,7 @@ dependencies = [ "serde", "serde_json", "thiserror", - "tokio 1.4.0", + "tokio 1.5.0", "tokio-native-tls", "tokio-util", "twox-hash", @@ -884,9 +885,9 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "openssl" -version = "0.10.33" +version = "0.10.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a61075b62a23fef5a29815de7536d940aa35ce96d18ce0cc5076272db678a577" +checksum = "6d7830286ad6a3973c0f1d9b73738f69c76b739301d0229c4b96501695cbe4c8" dependencies = [ "bitflags", "cfg-if 1.0.0", @@ -904,9 +905,9 @@ checksum = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de" [[package]] name = "openssl-sys" -version = "0.9.61" +version = "0.9.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "313752393519e876837e09e1fa183ddef0be7735868dced3196f4472d536277f" +checksum = "fa52160d45fa2e7608d504b7c3a3355afed615e6d8b627a74458634ba21b69bd" dependencies = [ "autocfg", "cc", @@ -959,18 +960,18 @@ checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" [[package]] name = "pin-project" -version = "1.0.6" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc174859768806e91ae575187ada95c91a29e96a98dc5d2cd9a1fed039501ba6" +checksum = "c7509cc106041c40a4518d2af7a61530e1eed0e6285296a3d8c5472806ccc4a4" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.0.6" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a490329918e856ed1b083f244e3bfe2d8c4f336407e4ea9e1a9f479ff09049e5" +checksum = "48c950132583b500556b1efd71d45b319029f2b71518d979fcc208e16b42426f" dependencies = [ "proc-macro2", "quote", @@ -1120,18 +1121,18 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.2.5" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94341e4e44e24f6b591b59e47a8a027df12e008d73fd5672dbea9cc22f4507d9" +checksum = "742739e41cd49414de871ea5e549afb7e2a3ac77b589bcbebe8c82fab37147fc" dependencies = [ "bitflags", ] [[package]] name = "regex" -version = "1.4.5" +version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "957056ecddbeba1b26965114e191d2e8589ce74db242b6ea25fc4062427a5c19" +checksum = "ce5f1ceb7f74abbce32601642fcf8e8508a8a8991e0621c7d750295b9095702b" dependencies = [ "aho-corasick", "memchr", @@ -1140,9 +1141,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.23" +version = "0.6.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d5f089152e60f62d28b835fbff2cd2e8dc0baf1ac13343bef92ab7eed84548" +checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" [[package]] name = "remove_dir_all" @@ -1170,9 +1171,9 @@ dependencies = [ [[package]] name = "rust_decimal" -version = "1.10.3" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc7f5b8840fb1f83869a3e1dfd06d93db79ea05311ac5b42b8337d3371caa4f1" +checksum = "72e80c4f9a71b5949e283189c3c3ae35fedddecfe112e75c9d58751c36605b62" dependencies = [ "arrayvec", "num-traits", @@ -1281,13 +1282,13 @@ dependencies = [ [[package]] name = "sha-1" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfebf75d25bd900fd1e7d11501efab59bc846dbc76196839663e6637bba9f25f" +checksum = "b659df5fc3ce22274daac600ffb845300bd2125bcfaec047823075afdab81c00" dependencies = [ "block-buffer", "cfg-if 1.0.0", - "cpuid-bool", + "cpufeatures", "digest", "opaque-debug", ] @@ -1300,13 +1301,13 @@ checksum = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d" [[package]] name = "sha2" -version = "0.9.3" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa827a14b29ab7f44778d14a88d3cb76e949c45083f7dbfa507d0cb699dc12de" +checksum = "d8f6b75b17576b792bef0db1bcc4b8b8bcdf9506744cf34b974195487af6cff2" dependencies = [ "block-buffer", "cfg-if 1.0.0", - "cpuid-bool", + "cpufeatures", "digest", "opaque-debug", ] @@ -1333,9 +1334,9 @@ dependencies = [ [[package]] name = "slab" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" +checksum = "f173ac3d1a7e3b28003f40de0b5ce7fe2710f9b9dc3fc38664cebee46b3b6527" [[package]] name = "smallvec" @@ -1431,9 +1432,9 @@ checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" [[package]] name = "syn" -version = "1.0.69" +version = "1.0.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48fe99c6bd8b1cc636890bcc071842de909d902c81ac7dab53ba33c421ab8ffb" +checksum = "a1e8cdbefb79a9a5a65e0db8b47b723ee907b7c7f8496c76a1770b5c310bab82" dependencies = [ "proc-macro2", "quote", @@ -1561,9 +1562,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "134af885d758d645f0f0505c9a8b3f9bf8a348fd822e112ab5248138348f1722" +checksum = "83f0c8e7c0addab50b663055baf787d0af7f413a46e6e7fb9559a4e4db7137a5" dependencies = [ "autocfg", "bytes 1.0.1", @@ -1597,7 +1598,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" dependencies = [ "native-tls", - "tokio 1.4.0", + "tokio 1.5.0", ] [[package]] @@ -1620,22 +1621,22 @@ dependencies = [ "futures-util", "log", "pin-project", - "tokio 1.4.0", + "tokio 1.5.0", "tungstenite", ] [[package]] name = "tokio-util" -version = "0.6.5" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5143d049e85af7fbc36f5454d990e62c2df705b3589f123b71f441b6b59f443f" +checksum = "940a12c99365c31ea8dd9ba04ec1be183ffe4920102bb7122c2f515437601e8e" dependencies = [ "bytes 1.0.1", "futures-core", "futures-sink", "log", "pin-project-lite 0.2.6", - "tokio 1.4.0", + "tokio 1.5.0", ] [[package]] @@ -1646,9 +1647,9 @@ checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" [[package]] name = "tracing" -version = "0.1.25" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01ebdc2bb4498ab1ab5f5b73c5803825e60199229ccba0698170e3be0e7f959f" +checksum = "09adeb8c97449311ccd28a427f96fb563e7fd31aabf994189879d9da2394b89d" dependencies = [ "cfg-if 1.0.0", "pin-project-lite 0.2.6", @@ -1657,9 +1658,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f50de3927f93d202783f4513cda820ab47ef17f624b03c096e86ef00c67e6b5f" +checksum = "a9ff14f98b1a4b289c6248a023c1c2fa1491062964e9fed67ab29c4e4da4a052" dependencies = [ "lazy_static", ] @@ -1733,9 +1734,9 @@ checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" [[package]] name = "unicode-xid" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" +checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" [[package]] name = "untrusted" @@ -1757,9 +1758,9 @@ dependencies = [ [[package]] name = "utf-8" -version = "0.7.5" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05e42f7c18b8f902290b009cde6d651262f956c98bc51bca4cd1d511c9cd85c7" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" [[package]] name = "uuid" @@ -1772,9 +1773,9 @@ dependencies = [ [[package]] name = "vcpkg" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b00bca6106a5e23f3eee943593759b7fcddb00554332e856d990c893966879fb" +checksum = "cbdbff6266a24120518560b5dc983096efb98462e51d0d68169895b237be3e5d" [[package]] name = "vec_map" diff --git a/json-api/db/Cargo.toml b/json-api/db/Cargo.toml index 5d05bf3..1a9522c 100644 --- a/json-api/db/Cargo.toml +++ b/json-api/db/Cargo.toml @@ -10,5 +10,6 @@ edition = "2018" mysql_async = "0.27" serde = { version = "1.0.117", features = [ "derive" ] } +serde_json = "1.0" tokio = { version = "1", features = ["fs", "io-util"] } rand = "0.8.3" diff --git a/json-api/db/src/lib.rs b/json-api/db/src/lib.rs index d0503b2..aef5d38 100644 --- a/json-api/db/src/lib.rs +++ b/json-api/db/src/lib.rs @@ -5,6 +5,7 @@ pub mod common; pub mod invites; pub mod channels; pub mod messages; +pub mod neighbors; use std::vec::Vec; @@ -90,3 +91,11 @@ pub struct PublicMember { pub permissions: UBigInt, } +#[derive(Serialize)] +pub struct Neighbor { + pub name: String, + pub description: Option, + pub tags: Vec, + pub url: String, + pub wsurl: Option, +} diff --git a/json-api/db/src/neighbors.rs b/json-api/db/src/neighbors.rs new file mode 100644 index 0000000..d211cb1 --- /dev/null +++ b/json-api/db/src/neighbors.rs @@ -0,0 +1,19 @@ +use crate::Neighbor; +use mysql_async::Result as SqlResult; +use mysql_async::{Pool, prelude::Queryable}; +use serde_json; + +impl Neighbor { + pub async fn get_all(p: &Pool) -> SqlResult> { + let mut conn = p.get_conn().await?; + let q = "SELECT name, description, tags, url, wsurl FROM neighbors"; + type Types = (String, Option, String, String, Option); + let set: Vec = conn.exec_map(q, (), |(name, description, tags, url, wsurl): Types| { + let json: Vec = serde_json::from_str(tags.as_str()).unwrap_or(vec![]); + Neighbor { + name, description, tags: json, url, wsurl + } + }).await?; + Ok(set) + } +} diff --git a/json-api/migrations/2021-05-04-200859_neighbors/down.sql b/json-api/migrations/2021-05-04-200859_neighbors/down.sql new file mode 100644 index 0000000..337e79d --- /dev/null +++ b/json-api/migrations/2021-05-04-200859_neighbors/down.sql @@ -0,0 +1 @@ +DROP TABLE `neighbors`; \ No newline at end of file diff --git a/json-api/migrations/2021-05-04-200859_neighbors/up.sql b/json-api/migrations/2021-05-04-200859_neighbors/up.sql new file mode 100644 index 0000000..c612dcd --- /dev/null +++ b/json-api/migrations/2021-05-04-200859_neighbors/up.sql @@ -0,0 +1,10 @@ +-- everything else is nullable due to +-- A) Flexibility +-- B) They're part of optional features(such as the wsurl which not all servers may have) +CREATE TABLE IF NOT EXISTS `neighbors`( + `url` VARCHAR(512) UNIQUE NOT NULL, + `wsurl` VARCHAR(512), + `name` VARCHAR(512) NOT NULL, + `description` VARCHAR(1024), + `tags` VARCHAR(1024) +); From 34115477ab222bfe72558e74d9444e5734a56d4f Mon Sep 17 00:00:00 2001 From: shockrah Date: Thu, 6 May 2021 17:03:24 -0700 Subject: [PATCH 03/27] + Mostly adding quality of life changes in this patch with the mock client These changes will make it easier to read through verbose logs with some tactful colors (yellow) --- json-api/client-tests/config.py | 10 ++++++++++ json-api/client-tests/request.py | 19 ++++++++++--------- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/json-api/client-tests/config.py b/json-api/client-tests/config.py index bb768c8..118556c 100644 --- a/json-api/client-tests/config.py +++ b/json-api/client-tests/config.py @@ -8,6 +8,15 @@ class Server: self.wsurl = meta.get('wsurl') self.serv_name = meta.get('name') + def __str__(self) -> str: + fields = { + 'url': self.url, + 'wsurl': self.wsurl, + 'name': self.serv_name + } + return str(fields) + + class Admin: def __init__(self, user: dict, server: dict): self.id = user.get('id') @@ -33,6 +42,7 @@ def create_admin() -> Admin : raw = json.loads(proc.stdout) user = raw.get('user') server = raw.get('server') + print(f'Raw data to use as user: {raw}') if user is None or server is None: return None else: diff --git a/json-api/client-tests/request.py b/json-api/client-tests/request.py index 45b3152..529aa0d 100644 --- a/json-api/client-tests/request.py +++ b/json-api/client-tests/request.py @@ -4,6 +4,7 @@ import requests NC = '\033[0m' RED = '\033[1;31m' GREEN = '\033[1;32m' +YELLOW = '\033[1;33m' class Request: @@ -56,23 +57,23 @@ class Request: print(abstract) print('\t', self.method, ' ', self.url) if len(self.headers) != 0: - print('\tRequest-Headers ', self.headers) + print(YELLOW + '\tRequest-Headers ' + NC, self.headers) if len(self.qs) != 0: - print('\tQuery-Dictionary ', self.qs) + print(YELLOW + '\tQuery-Dictionary' + NC, self.qs) if self.body is not None: - print('\tRequest-Body ', str(self.body)) + print(YELLOW + '\tRequest-Body' + NC, str(self.body)) if self.verbose: - print(f'\tResponse-Status {self.response.status_code}') - print(f'\tResponse-Headers {self.response.headers}') - print(f'\tResponse-Text {self.response.text}') + print(f'\t{YELLOW}Response-Status{NC} {self.response.status_code}') + print(f'\t{YELLOW}Response-Headers{NC} {self.response.headers}') + print(f'\t{YELLOW}Response-Text{NC} {self.response.text}') else: print(f'{GREEN}Pass{NC} {self.method} {self.url}') if self.verbose: - print(f'\tResponse-Status {self.response.status_code}') - print(f'\tResponse-Headers {self.response.headers}') - print(f'\tResponse-Text {self.response.text}') + print(f'\t{YELLOW}Response-Status{NC} {self.response.status_code}') + print(f'\t{YELLOW}Response-Headers{NC} {self.response.headers}') + print(f'\t{YELLOW}Response-Text{NC} {self.response.text}') From 9a22713080178d86f9de9810c2a8bfa957598cac Mon Sep 17 00:00:00 2001 From: shockrah Date: Thu, 6 May 2021 17:12:49 -0700 Subject: [PATCH 04/27] + Testing /invite/join + Testing /meta + Testing /neighbors/list --- json-api/client-tests/config.py | 6 ++++-- json-api/client-tests/main.py | 9 ++++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/json-api/client-tests/config.py b/json-api/client-tests/config.py index 118556c..28cc25f 100644 --- a/json-api/client-tests/config.py +++ b/json-api/client-tests/config.py @@ -1,5 +1,6 @@ import subprocess import os +import sys import json class Server: @@ -42,11 +43,12 @@ def create_admin() -> Admin : raw = json.loads(proc.stdout) user = raw.get('user') server = raw.get('server') - print(f'Raw data to use as user: {raw}') if user is None or server is None: + print(f'User/Server Data was not serializable => raw', file=sys.stderr) return None else: return Admin(user, server) - except: + except Exception as e: + print(f'General exception caught in parsing => {e}', file=sys.stderr) return None diff --git a/json-api/client-tests/main.py b/json-api/client-tests/main.py index 8002455..8b35026 100644 --- a/json-api/client-tests/main.py +++ b/json-api/client-tests/main.py @@ -7,7 +7,8 @@ RESPONSES = [] VOICE_CHAN = 1 TEXT_CHAN = 2 -def login() -> (Request, str): +def login(admin: Admin) -> (Request, str): + print(f'Provided admin account {admin.server}') req = Request( admin.server.url + '/login', 'post', @@ -89,7 +90,7 @@ if __name__ == '__main__': # First a quick sanity check for login # add this after we fire the generic tests - login_req, jwt = login() + login_req, jwt = login(admin) if jwt is None: print('Unable to /login - stopping now to avoid pointless failure') req.show_response() @@ -118,7 +119,9 @@ if __name__ == '__main__': req(admin, 'get', '/members/me', {}, 200), req(admin, 'get', '/members/get_online', {}, 200), req(admin, 'post', '/members/me/nickname', {'nick': f'randy-{time()}'}, 200), - req(admin , 'get', '/invite/join', {'code': 123}, 404) + req(admin , 'get', '/invite/join', {'code': 123}, 404), + req(admin , "get", "/meta", {}, 200), + req(admin, 'get', '/neighbors/list', {}, 200, verbose=True) ]) # add this after fire the generic tests From c850d42ce150baad71209c761fba503abc3127a8 Mon Sep 17 00:00:00 2001 From: shockrah Date: Sat, 8 May 2021 01:29:44 -0700 Subject: [PATCH 05/27] + Jwt tables - SEE NOTE ! wat - because have to do maintain permissions on a per request level we have to do this check for permissions at what is basically every level, this does mean we have to hit the database for a lot of routes however there is a check that requests go through in order to avoid hitting the database whenever possible + rng field in claims now has real purpose It's purpose is to act as a validator field in the jwt table. By verifying rng fields we no longer have to store whole jwt's --- json-api/db/src/jwt.rs | 19 ++++++ json-api/db/src/lib.rs | 1 + .../migrations/2021-05-07-201858_jwt/down.sql | 1 + .../migrations/2021-05-07-201858_jwt/up.sql | 5 ++ json-api/src/auth.rs | 60 ++++++++++++------- json-api/src/routes.rs | 9 +++ 6 files changed, 73 insertions(+), 22 deletions(-) create mode 100644 json-api/db/src/jwt.rs create mode 100644 json-api/migrations/2021-05-07-201858_jwt/down.sql create mode 100644 json-api/migrations/2021-05-07-201858_jwt/up.sql diff --git a/json-api/db/src/jwt.rs b/json-api/db/src/jwt.rs new file mode 100644 index 0000000..26efeaa --- /dev/null +++ b/json-api/db/src/jwt.rs @@ -0,0 +1,19 @@ +use mysql_async::{Pool, params, Result, prelude::Queryable}; + +pub async fn listed(p: &Pool, id: u64, given_rng_value: &str) -> Result { + let mut conn = p.get_conn().await?; + let q = "SELECT rng FROM jwt WHERE id = :id"; + let row: Option = conn.exec_first(q, params!{"id" => id}).await?; + if let Some(value) = row { + Ok(value == given_rng_value) + } else{ + Ok(false) + } +} + +pub async fn insert(p: &Pool, id: u64, given_rng_value: &str) -> Result<()> { + let mut conn = p.get_conn().await?; + let q = "INSERT INTO jwt (id, rng) VALUES (:id, :rng)"; + conn.exec_drop(q, params!{"id" => id, "rng" => given_rng_value}).await?; + Ok(()) +} diff --git a/json-api/db/src/lib.rs b/json-api/db/src/lib.rs index aef5d38..564d421 100644 --- a/json-api/db/src/lib.rs +++ b/json-api/db/src/lib.rs @@ -6,6 +6,7 @@ pub mod invites; pub mod channels; pub mod messages; pub mod neighbors; +pub mod jwt; use std::vec::Vec; diff --git a/json-api/migrations/2021-05-07-201858_jwt/down.sql b/json-api/migrations/2021-05-07-201858_jwt/down.sql new file mode 100644 index 0000000..df15255 --- /dev/null +++ b/json-api/migrations/2021-05-07-201858_jwt/down.sql @@ -0,0 +1 @@ +DROP TABLE `jwt`; diff --git a/json-api/migrations/2021-05-07-201858_jwt/up.sql b/json-api/migrations/2021-05-07-201858_jwt/up.sql new file mode 100644 index 0000000..e91e054 --- /dev/null +++ b/json-api/migrations/2021-05-07-201858_jwt/up.sql @@ -0,0 +1,5 @@ +CREATE TABLE IF NOT EXISTS `jwt` ( + `id` BIGINT UNSIGNED NOT NULL, + `rng` VARCHAR(48) NOT NULL, + PRIMARY KEY(`id`) +); diff --git a/json-api/src/auth.rs b/json-api/src/auth.rs index 3060b92..40fda73 100644 --- a/json-api/src/auth.rs +++ b/json-api/src/auth.rs @@ -6,7 +6,6 @@ use std::collections::HashMap; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use crate::routes; -use crate::qs_param; use db::{Response, Member}; @@ -26,15 +25,16 @@ lazy_static! { } #[derive(Debug, Serialize, Deserialize)] -struct Claim { - sub: db::UBigInt, // user id - exp: db::BigInt, // expiry date - nbf: i64, - cookie: String, // unique cookie value +pub struct Claim { + sub: u64,// user id + exp: i64, // expiry date + nbf: i64, // "valid-start" time for the claim + prm: u64, // user permissions + rng: String, // Cheesy way of helping reduce the amount } impl Claim { - pub fn new(id: db::UBigInt) -> Claim { + pub fn new(id: db::UBigInt, prm: u64) -> Claim { // JWT's expire every 48 hours let now = SystemTime::now(); @@ -52,20 +52,20 @@ impl Claim { sub: id, exp, nbf, - cookie: generate_cookie() + prm, + rng: generate_cookie() } } } - // used when we create a new users for the first time #[derive(Debug)] pub enum AuthReason { - Good, //passed regular check + Good(Claim), //passed regular check OpenAuth, // route does not require auth NoKey, // key missing BadKey, // key is bad - LoginValid, // used only to access the login route which is also our refresh + LoginValid(Member), // used only to access the login route which is also our refresh ServerIssue(String) // for well 500's } @@ -81,7 +81,7 @@ fn valid_secret(given_pass: &str, hash: &str) -> bool { } } -fn valid_perms(member: Member, path: &str) -> bool { +fn valid_perms(member: &Member, path: &str) -> bool { use crate::perms; // if there are perms on the current path make sure the user has them if let Some(p) = perms::get_perm_mask(path) { @@ -120,7 +120,13 @@ pub fn encrypt_secret(raw: &str) -> BcryptResult { } -async fn valid_jwt(token: &str) -> AuthReason { +async fn valid_jwt(pool: &Pool, path: &str, token: &str) -> AuthReason { + /* + * This function does a database check because of the requirement of always + * enforcing permissions + * TODO: Not all routes actually require a permissions check so we should + * try to only do that DB call when absolutely required + */ use jsonwebtoken::{ decode, DecodingKey, Validation, Algorithm @@ -136,7 +142,18 @@ async fn valid_jwt(token: &str) -> AuthReason { let active = now < decoded.claims.exp; if active { - AuthReason::Good + if routes::requires_perms(path) { + match db::jwt::listed(pool, decoded.claims.sub, decoded.claims.rng.as_str()).await { + Ok(listed) => if listed { + AuthReason::Good(decoded.claims) + } else { + AuthReason::BadKey + } + _ => AuthReason::BadKey + } + } else { + AuthReason::Good(decoded.claims) + } } else { AuthReason::BadKey } @@ -182,7 +199,7 @@ pub async fn wall_entry<'path, 'pool, 'params>( if let Some(jwt) = jwt { // get the headers here - return valid_jwt(jwt).await; + return valid_jwt(pool, path, jwt).await; } if let Some((id, secret)) = login_params_from_qs(params) { // Last chance we might be hitting the /login route so we have to do the heavy auth flow @@ -194,8 +211,8 @@ pub async fn wall_entry<'path, 'pool, 'params>( match Member::get(pool, id).await { Ok(response) => match response { Response::Row(user) => { - if valid_secret(secret, &user.secret) && valid_perms(user, path){ - AuthReason::LoginValid + if valid_secret(secret, &user.secret) && valid_perms(&user, path){ + AuthReason::LoginValid(user) } else { AuthReason::BadKey @@ -215,7 +232,7 @@ pub async fn wall_entry<'path, 'pool, 'params>( } } -pub async fn login_get_jwt(response: &mut hyper::Response, params: HashMap) { +pub async fn login_get_jwt(response: &mut hyper::Response, user: Member) { // Login data has already been validated at this point // Required data such as 'id' and 'secret' are there and validated use jsonwebtoken::{ @@ -225,10 +242,9 @@ pub async fn login_get_jwt(response: &mut hyper::Response, params: use hyper::header::HeaderValue; use crate::http; - let id = qs_param!(params, "id", u64).unwrap(); + let id = user.id; - - let claim = Claim::new(id); + let claim = Claim::new(id, user.permissions); let header = Header::new(Algorithm::HS512); let encoded = encode( &header, @@ -259,7 +275,7 @@ mod auth_tests { #[test] fn verify_jwt() { - let claim = super::Claim::new(123); // example claim that we send out + let claim = super::Claim::new(123, 456); // example claim that we send out let header = Header::new(Algorithm::HS512); // header that basically all clients get let encoded = encode( &header, diff --git a/json-api/src/routes.rs b/json-api/src/routes.rs index e2bfb6e..9f8f672 100644 --- a/json-api/src/routes.rs +++ b/json-api/src/routes.rs @@ -24,7 +24,16 @@ pub const SELF_UPDATE_NICKNAME: Rstr= "/members/me/nickname"; pub const SET_PERMS_BY_ADMIN: Rstr = "/admin/setpermisions"; // @requires perms::ADMIN pub const SET_NEW_ADMIN: Rstr = "/owner/newadmin"; // @requiers: owner perms +// Server -> Server Routes +pub const GET_NEIGHBORS: Rstr = "/neighbors/list"; // @requires: none @note must be a member for this list + pub fn is_open(path: &str) -> bool { return path.starts_with("/join") || path.starts_with("/meta"); } +pub fn requires_perms(path: &str) -> bool { + return match path { + AUTH_LOGIN | META | CHANNELS_LIST | GET_MYSELF | GET_NEIGHBORS => false, + _ => true + } +} From 1c366611d9850c3ff061d8b5c9f1d7808411e735 Mon Sep 17 00:00:00 2001 From: shockrah Date: Sat, 8 May 2021 01:59:31 -0700 Subject: [PATCH 06/27] + Server tags are now stored in json format(still in the environment variable mind you Lazy static initialization still allocates once per run so the above is jank but fine(mostly) [] Should probably come up with a cleaner way of passing this data around The above is very low on the priority list of things to do + Now injecting permisions + id via the claim structure which is passed to the route dispatcher ! For now this is unused further down the decision tree however I'm going to sprinkle in its incorporation as it lets us avoid string conversions which is p nice --- json-api/src/main.rs | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/json-api/src/main.rs b/json-api/src/main.rs index a1e1a38..66588b1 100644 --- a/json-api/src/main.rs +++ b/json-api/src/main.rs @@ -58,7 +58,8 @@ async fn route_dispatcher( path: &str, body: Body, params: HashMap, - headers: HeaderMap) { + headers: HeaderMap, + claims: Option/* Faster id/perms access from here */) { const GET: &Method = &Method::GET; const POST: &Method = &Method::POST; @@ -87,6 +88,8 @@ async fn route_dispatcher( (POST, routes::SET_NEW_ADMIN) => admin::new_admin(pool, resp, params).await, /* META ROUTE */ (GET, routes::META) => meta::server_meta(resp).await, + /* Federated Routes */ + (GET, routes::GET_NEIGHBORS) => meta::server_neighbors(pool, resp).await, _ => { println!("[HTTP]\tNOT FOUND: {}: {}", meth, path); *resp.status_mut() = StatusCode::NOT_FOUND @@ -109,15 +112,18 @@ async fn main_responder(request: Request) -> Result, hyper: None }; - if let Some(params) = params_opt { - match auth::wall_entry(path, &DB_POOL, ¶ms).await { - OpenAuth | Good => { - // route dispatch has its own more comprehensive logging - route_dispatcher(&DB_POOL, &mut response, &method, path, body, params, headers).await; + if let Some(qs) = params_opt { + match auth::wall_entry(path, &DB_POOL, &qs).await { + OpenAuth => { + route_dispatcher(&DB_POOL, &mut response, &method, path, body, qs, headers, None).await; }, - LoginValid => { + // Only with typical routes do we have to inject the permissions + Good(claim) => { + route_dispatcher(&DB_POOL, &mut response, &method, path, body, qs, headers, Some(claim)).await; + }, + LoginValid(member) => { println!("[HTTP] POST /login"); - auth::login_get_jwt(&mut response, params).await; + auth::login_get_jwt(&DB_POOL, &mut response, member).await; }, NoKey | BadKey => { *response.status_mut() = StatusCode::UNAUTHORIZED; @@ -225,6 +231,13 @@ fn init_config() -> Result<(), Box> { set_var("PUBLIC_URL", fields.public_url); set_var("PUBLIC_WS_URL", fields.public_ws_url); + // NOTE: the debug print actually prints out what looks to be valid json + // so we're just going to leverage that for passing this bs around + let tags_json = match fields.tags { + Some(t) => format!("{:?}", t), + None => format!("[]") + }; + set_var("SERVER_TAGS", tags_json); Ok(()) } From cdb956a85ca2e98f7aeed6e2a557ace05782953a Mon Sep 17 00:00:00 2001 From: shockrah Date: Sat, 8 May 2021 02:03:58 -0700 Subject: [PATCH 07/27] Minor changes lumped together * claim.rng is now seeded from 16 bytes + passing Member to auth::login_get_jwt * Better error loggin on jwt::insert call + Moar meta config-caching fixes Basically lazy static sucks and we have to start accessing meta::basic_config though a proxy function which forces the initialization on startup ! We should probably init this prior to listening for connections to avoid connection issues + New /neighbors/list endpoint which has already passed required checks --- json-api/src/auth.rs | 14 +++++++--- json-api/src/meta.rs | 61 +++++++++++++++++++++++++++++++++++++------- 2 files changed, 62 insertions(+), 13 deletions(-) diff --git a/json-api/src/auth.rs b/json-api/src/auth.rs index 40fda73..7bd20a4 100644 --- a/json-api/src/auth.rs +++ b/json-api/src/auth.rs @@ -1,6 +1,7 @@ use serde::{Serialize, Deserialize}; use bcrypt::{self, BcryptResult}; use mysql_async::Pool; +use hyper::StatusCode; use std::collections::HashMap; use std::time::{Duration, SystemTime, UNIX_EPOCH}; @@ -111,7 +112,7 @@ pub fn generate_secret() -> String { } pub fn generate_cookie() -> String { - return rng_secret(32) + return rng_secret(16) } pub fn encrypt_secret(raw: &str) -> BcryptResult { @@ -198,7 +199,6 @@ pub async fn wall_entry<'path, 'pool, 'params>( } if let Some(jwt) = jwt { - // get the headers here return valid_jwt(pool, path, jwt).await; } if let Some((id, secret)) = login_params_from_qs(params) { @@ -232,7 +232,7 @@ pub async fn wall_entry<'path, 'pool, 'params>( } } -pub async fn login_get_jwt(response: &mut hyper::Response, user: Member) { +pub async fn login_get_jwt(p: &Pool, response: &mut hyper::Response, user: Member) { // Login data has already been validated at this point // Required data such as 'id' and 'secret' are there and validated use jsonwebtoken::{ @@ -254,7 +254,13 @@ pub async fn login_get_jwt(response: &mut hyper::Response, user: Me response.headers_mut().insert("Content-Type", HeaderValue::from_static("application/json")); - http::set_json_body(response, serde_json::json!({"jwt": encoded})); + match db::jwt::insert(p, id, &claim.rng).await { + Ok(_) => http::set_json_body(response, serde_json::json!({"jwt": encoded})), + Err(e) => { + eprintln!("/login 500 {}", e); + *response.status_mut() = StatusCode::INTERNAL_SERVER_ERROR; + } + } } diff --git a/json-api/src/meta.rs b/json-api/src/meta.rs index b21bfeb..0f14cbd 100644 --- a/json-api/src/meta.rs +++ b/json-api/src/meta.rs @@ -1,28 +1,71 @@ // Basic handler for getting meta data about the server use std::env::var; +use std::collections::HashMap; +use crate::http::set_json_body; -use hyper::{Response, Body}; -use serde_json::to_string; -use serde::Serialize; +use mysql_async::Pool; +use hyper::{Response, Body, StatusCode}; +use serde_json::{json, to_string}; +use serde::{Serialize, Deserialize}; +use lazy_static::lazy_static; - -#[derive( Serialize)] +#[derive(Debug, Serialize)] pub struct Config { pub name: String, pub description: String, pub url: String, pub wsurl: String, + pub tags: Vec +} + +lazy_static! { + // NOTE: this object must be access by proxy through get_config() + #[derive(Deserialize, Serialize)] + static ref BASIC_CONFIG: Config = { + let tags = var("SERVER_TAGS").unwrap(); + let tags: Vec = match serde_json::from_str(tags.as_str()).expect("Unable to parse tags") { + Some(tags) => tags, + None => vec![] + }; + + let cfg = Config { + name: var("SERVER_NAME").unwrap_or("No name".into()), + description: var("SERVER_DESCRIPTION").unwrap_or("No description".into()), + url: var("PUBLIC_URL").unwrap(), + wsurl: var("PUBLIC_WS_URL").unwrap(), + tags + }; + cfg + }; } pub fn get_config() -> Config { + // We have to do this (for now) because lazy_static silently hides the actual fields + // we care about Config { - name: var("SERVER_NAME").unwrap_or("No name".into()), - description: var("SERVER_DESCRIPTION").unwrap_or("No description".into()), - url: var("PUBLIC_URL").unwrap(), - wsurl: var("PUBLIC_WS_URL").unwrap() + name: BASIC_CONFIG.name.clone(), + description: BASIC_CONFIG.description.clone(), + url: BASIC_CONFIG.url.clone(), + wsurl: BASIC_CONFIG.wsurl.clone(), + tags: BASIC_CONFIG.tags.clone() } } pub async fn server_meta(response: &mut Response) { + // NOTE: This route _is_ open but we should do something to rate limit the + // number of requests we service as it could be ripe for abuse *response.body_mut() = Body::from(to_string(&get_config()).unwrap()); } + +pub async fn server_neighbors(p: &Pool, response: &mut Response) { + // This method refers to what servers have been added as **related** by the admins + // It returns a list of servers meta objects which converted to JSON + match db::Neighbor::get_all(p).await { + Ok(neighbors) => set_json_body(response, json!({"neighbors": neighbors})), + Err(e) => { + *response.status_mut() = StatusCode::INTERNAL_SERVER_ERROR; + eprintln!("500 /neighbors/list {}", e); + } + } +} + From e8348918c4c0df51c324ae898dcf4080ba53a77f Mon Sep 17 00:00:00 2001 From: shockrah Date: Sat, 8 May 2021 02:07:32 -0700 Subject: [PATCH 08/27] + More qol like better __str__ methods everywhere + More error handling in case shit goes wrong Basically handling more cases where some initial test could result in the whole script exiting with code(1) Not really that big of a deal since most tests after _that_ point will fail anyway but the fix has revealed issues in the auth code magically so I'm keeping up with the new idea that initial tests should have every resultant case validated to avoid weird behavior > good code results in good results who would have guessed --- json-api/client-tests/config.py | 10 +++++++++- json-api/client-tests/main.py | 7 +++++-- json-api/client-tests/request.py | 3 +++ 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/json-api/client-tests/config.py b/json-api/client-tests/config.py index 28cc25f..44c965d 100644 --- a/json-api/client-tests/config.py +++ b/json-api/client-tests/config.py @@ -30,7 +30,15 @@ class Admin: self.server = Server(server) def __str__(self) -> str: - return f'{self.name}#{self.id}' + acc = { + 'id': self.id, + 'name': self.name, + 'permissions': self.permissions, + 'secret': self.secret, + 'jwt': self.jwt, + } + container = {'user': acc, 'server': str(server)} + return str(container) def create_admin() -> Admin : CARGO_BIN = os.getenv('CARGO_BIN') diff --git a/json-api/client-tests/main.py b/json-api/client-tests/main.py index 8b35026..189a58b 100644 --- a/json-api/client-tests/main.py +++ b/json-api/client-tests/main.py @@ -8,7 +8,7 @@ VOICE_CHAN = 1 TEXT_CHAN = 2 def login(admin: Admin) -> (Request, str): - print(f'Provided admin account {admin.server}') + print(f'Provided admin account {admin}') req = Request( admin.server.url + '/login', 'post', @@ -68,9 +68,12 @@ def make_and_receive_invite(admin: Admin) -> (Request, Request): make_inv_req.fire() if make_inv_req.response is None: return (make_inv_req, user_inv_req) + elif make_inv_req.response.status_code >= 400: + print('Params used ', make_inv_req) + return (make_inv_req, None) # Setup to fire the second request, .fire() is safe to blindly use at this point - print('Response text from /join ', make_inv_req.response.text) + print('Response text from /join ', make_inv_req.response.status_code, ' ', make_inv_req.response.text) new_invite_body = make_inv_req.response.json()['invite'] code = new_invite_body['id'] user_inv_req = req(admin , 'get' , '/join', {'code': code}, 200, verbose=True) diff --git a/json-api/client-tests/request.py b/json-api/client-tests/request.py index 529aa0d..89d34a3 100644 --- a/json-api/client-tests/request.py +++ b/json-api/client-tests/request.py @@ -22,6 +22,9 @@ class Request: self.verbose = verbose + def __str__(self): + return str(self.qs) + @property def passing(self): if self.response is None: From a79195076d2575cb57a0ab017170f12d32541f4c Mon Sep 17 00:00:00 2001 From: shockrah Date: Sun, 9 May 2021 23:08:55 -0700 Subject: [PATCH 09/27] * Changin public_url & public_ws_url to shorter url & wsurl respectively There's also some miscellaneous changes that don't fit anywhere in this patch (mostly dev qol of roadmap updates) --- json-api/Makefile | 3 +++ json-api/client-tests/config.py | 4 ++-- json-api/config.json | 4 ++-- roadmap.md | 2 +- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/json-api/Makefile b/json-api/Makefile index 4c8f1e8..199a3b9 100644 --- a/json-api/Makefile +++ b/json-api/Makefile @@ -7,5 +7,8 @@ dep: run: cargo run --features rtc --release -- -s +test: + cd ../ && bash scripts/run-api-tests.sh + clean: cargo clean diff --git a/json-api/client-tests/config.py b/json-api/client-tests/config.py index 44c965d..73ce845 100644 --- a/json-api/client-tests/config.py +++ b/json-api/client-tests/config.py @@ -37,7 +37,7 @@ class Admin: 'secret': self.secret, 'jwt': self.jwt, } - container = {'user': acc, 'server': str(server)} + container = {'user': acc, 'server': str(self.server)} return str(container) def create_admin() -> Admin : @@ -57,6 +57,6 @@ def create_admin() -> Admin : else: return Admin(user, server) except Exception as e: - print(f'General exception caught in parsing => {e}', file=sys.stderr) + print(f'[create_admin] General exception caught in parsing => {e}', file=sys.stderr) return None diff --git a/json-api/config.json b/json-api/config.json index f200e31..e5e15db 100644 --- a/json-api/config.json +++ b/json-api/config.json @@ -7,8 +7,8 @@ "wss_hmac_path": "wss-hmac.secret", "name": "Freechat Server", "description": "Chatting server", - "public_url": "http://localhost:4536", - "public_ws_url": "ws://localhost:5648", + "url": "http://localhost:4536", + "wsurl": "ws://localhost:5648", "tags": [ "Freechat", diff --git a/roadmap.md b/roadmap.md index 099e594..d21c3fd 100644 --- a/roadmap.md +++ b/roadmap.md @@ -39,7 +39,7 @@ I'd like to be able to support people that make good software and give them a pl * More server meta endpoint support: Some basic server metadata is stored in environment vars but we can probably store these somewhere a bit smarter idk. -Basically this work has to be put in so that we can further decentralize Freechat Ecosystem + * This is now actively being worked with /neighbor/list and /neighbor/add now being added to the supported routes. Further testing is required, as well as relevant documentation. At least this should be done within the next few days as I find time for it among other projects. * RTC Permissions: Unprivileged users shouldn't receive messaegs if they don't have the required permissions From c443b9bb071e86b5a9c5251e61999d5b6781e00e Mon Sep 17 00:00:00 2001 From: shockrah Date: Sun, 9 May 2021 23:11:51 -0700 Subject: [PATCH 10/27] + More tests for the /neighbor/add and /neighbor/list routes These are baseline tests however a new /neighbor/remove & /neighbor/edit route should be edited before I call this done on the roadmap. Also some more intense testing around these current routes is required. Mostly because the expectation that JSON is being sent to us in /neighbor/add It could be worth the effort to send this data as json in the body ! Currently parameters are sent via the query string is in line with how most routes behave For this route is just feels weird dealing with al the issues with json in the query string --- json-api/client-tests/main.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/json-api/client-tests/main.py b/json-api/client-tests/main.py index 189a58b..4c092f5 100644 --- a/json-api/client-tests/main.py +++ b/json-api/client-tests/main.py @@ -124,7 +124,9 @@ if __name__ == '__main__': req(admin, 'post', '/members/me/nickname', {'nick': f'randy-{time()}'}, 200), req(admin , 'get', '/invite/join', {'code': 123}, 404), req(admin , "get", "/meta", {}, 200), - req(admin, 'get', '/neighbors/list', {}, 200, verbose=True) + req(admin, 'get', '/neighbor/list', {}, 200), + req(admin,'post', '/neighbor/add', {'name':'name','url':'url','wsurl':'wsurl','description':'asdf','tags':'["red","blue"]'}, 200), + req(admin, 'get', '/neighbor/list', {}, 200, verbose=True) ]) # add this after fire the generic tests From adc5b261e88ad3f47f85ff3fb3217f5e785e3772 Mon Sep 17 00:00:00 2001 From: shockrah Date: Sun, 9 May 2021 23:14:02 -0700 Subject: [PATCH 11/27] + ADD_NEIGHBOR route now supported in API backend ! Requiring a special event in the RTC server docs for this change --- json-api/db/src/neighbors.rs | 43 ++++++++++++++++++++++++------------ json-api/src/main.rs | 21 +++++++++--------- json-api/src/meta.rs | 27 ++++++++++++++++++++++ json-api/src/routes.rs | 4 +++- 4 files changed, 69 insertions(+), 26 deletions(-) diff --git a/json-api/db/src/neighbors.rs b/json-api/db/src/neighbors.rs index d211cb1..a922d06 100644 --- a/json-api/db/src/neighbors.rs +++ b/json-api/db/src/neighbors.rs @@ -1,19 +1,34 @@ use crate::Neighbor; use mysql_async::Result as SqlResult; -use mysql_async::{Pool, prelude::Queryable}; +use mysql_async::{params, Pool, prelude::Queryable}; use serde_json; -impl Neighbor { - pub async fn get_all(p: &Pool) -> SqlResult> { - let mut conn = p.get_conn().await?; - let q = "SELECT name, description, tags, url, wsurl FROM neighbors"; - type Types = (String, Option, String, String, Option); - let set: Vec = conn.exec_map(q, (), |(name, description, tags, url, wsurl): Types| { - let json: Vec = serde_json::from_str(tags.as_str()).unwrap_or(vec![]); - Neighbor { - name, description, tags: json, url, wsurl - } - }).await?; - Ok(set) - } +pub async fn get_all(p: &Pool) -> SqlResult> { + let mut conn = p.get_conn().await?; + let q = "SELECT name, description, tags, url, wsurl FROM neighbors"; + type Types = (String, Option, String, String, Option); + let set: Vec = conn.exec_map(q, (), |(name, description, tags, url, wsurl): Types| { + let json: Vec = serde_json::from_str(tags.as_str()).unwrap_or(vec![]); + Neighbor { + name, description, tags: json, url, wsurl + } + }).await?; + Ok(set) +} + +pub async fn add_neighbor(p: &Pool, url: &str, wsurl: &str, name: &str, desc: &str, tags:&str) -> SqlResult<()> { + // Note we assume that the tags field has been verified as proper valid json prior to + // writing it to the db + let mut conn = p.get_conn().await?; + let q = "INSERT INTO neighbors(url, wsurl, name, description, tags) + VALUES(:url, :wsurl, :name, :description, :tags)"; + let sqlparams = params!{ + "url"=> url, + "wsurl"=>wsurl, + "name"=>name, + "desc"=>desc, + "tags"=>tags + }; + conn.exec_drop(q, sqlparams).await?; + Ok(()) } diff --git a/json-api/src/main.rs b/json-api/src/main.rs index 66588b1..199eb5a 100644 --- a/json-api/src/main.rs +++ b/json-api/src/main.rs @@ -90,6 +90,7 @@ async fn route_dispatcher( (GET, routes::META) => meta::server_meta(resp).await, /* Federated Routes */ (GET, routes::GET_NEIGHBORS) => meta::server_neighbors(pool, resp).await, + (POST, routes::ADD_NEIGHBOR) => meta::add_neighbor(pool, resp, params).await, _ => { println!("[HTTP]\tNOT FOUND: {}: {}", meth, path); *resp.status_mut() = StatusCode::NOT_FOUND @@ -206,8 +207,8 @@ fn init_config() -> Result<(), Box> { pub wss_hmac_path: String, pub name: String, pub description: Option, - pub public_url: String, - pub public_ws_url: String, + pub url: String, + pub wsurl: String, pub tags: Option> } use std::fs::File; @@ -229,15 +230,13 @@ fn init_config() -> Result<(), Box> { set_var("SERVER_DESCRIPTION", fields.description.unwrap_or("".into())); - set_var("PUBLIC_URL", fields.public_url); - set_var("PUBLIC_WS_URL", fields.public_ws_url); - // NOTE: the debug print actually prints out what looks to be valid json - // so we're just going to leverage that for passing this bs around - let tags_json = match fields.tags { - Some(t) => format!("{:?}", t), - None => format!("[]") - }; - set_var("SERVER_TAGS", tags_json); + set_var("PUBLIC_URL", fields.url); + set_var("PUBLIC_WS_URL", fields.wsurl); + + // Mega cheesy way of forcing config initialization + if meta::get_config().tags.len() == 0 { + eprintln!("[API] [WARN] No tags have been set", ); + } Ok(()) } diff --git a/json-api/src/meta.rs b/json-api/src/meta.rs index 0f14cbd..b5df7d8 100644 --- a/json-api/src/meta.rs +++ b/json-api/src/meta.rs @@ -69,3 +69,30 @@ pub async fn server_neighbors(p: &Pool, response: &mut Response) { } } + +pub async fn add_neighbor(p: &Pool, response: &mut Response, params: HashMap) { + let url = params.get("url"); + let wsurl = params.get("wsurl"); + let name = params.get("name"); + let desc = params.get("description"); + let tags = params.get("tags"); + + // If any parameter is missing then fail away quickly + if url.is_none() || wsurl.is_none() || name.is_none() || desc.is_none() || tags.is_none() { + *response.status_mut() = StatusCode::BAD_REQUEST; + return; + } + // Safe unwrap because of the check above + let url = url.unwrap(); + let wsurl = wsurl.unwrap(); + let name = name.unwrap(); + let desc = desc.unwrap(); + let tags = tags.unwrap(); + match db::neighbors::add_neighbor(p, url, wsurl, name, desc, tags).await { + Ok(()) => {}, + Err(e) => { + eprintln!("{}", e); + *response.status_mut() = StatusCode::INTERNAL_SERVER_ERROR; + } + } +} diff --git a/json-api/src/routes.rs b/json-api/src/routes.rs index 9f8f672..d391ffe 100644 --- a/json-api/src/routes.rs +++ b/json-api/src/routes.rs @@ -25,7 +25,8 @@ pub const SET_PERMS_BY_ADMIN: Rstr = "/admin/setpermisions"; // @requires per pub const SET_NEW_ADMIN: Rstr = "/owner/newadmin"; // @requiers: owner perms // Server -> Server Routes -pub const GET_NEIGHBORS: Rstr = "/neighbors/list"; // @requires: none @note must be a member for this list +pub const GET_NEIGHBORS: Rstr = "/neighbor/list"; // @requires: none @note must be a member for this list +pub const ADD_NEIGHBOR: Rstr = "/neighbor/add"; // @requires: perm::add_neighbor pub fn is_open(path: &str) -> bool { return path.starts_with("/join") || path.starts_with("/meta"); @@ -33,6 +34,7 @@ pub fn is_open(path: &str) -> bool { pub fn requires_perms(path: &str) -> bool { return match path { + /* These routes _don't_ require any permissions */ AUTH_LOGIN | META | CHANNELS_LIST | GET_MYSELF | GET_NEIGHBORS => false, _ => true } From b3c27b86cecfb1ccc852c6a7454ad46c256ccee8 Mon Sep 17 00:00:00 2001 From: shockrah Date: Sun, 9 May 2021 23:15:09 -0700 Subject: [PATCH 12/27] * Fixing some json serialization/parsing weirdness in the meta::BASIC_CONFIG initialization This is part of a bigger change to stop using environment variables around as state as its mega cheese --- json-api/src/meta.rs | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/json-api/src/meta.rs b/json-api/src/meta.rs index b5df7d8..96f1d6e 100644 --- a/json-api/src/meta.rs +++ b/json-api/src/meta.rs @@ -1,15 +1,14 @@ // Basic handler for getting meta data about the server -use std::env::var; use std::collections::HashMap; use crate::http::set_json_body; use mysql_async::Pool; use hyper::{Response, Body, StatusCode}; -use serde_json::{json, to_string}; +use serde_json::{json, to_string, Result as JsonResult}; use serde::{Serialize, Deserialize}; use lazy_static::lazy_static; -#[derive(Debug, Serialize)] +#[derive(Debug, Deserialize, Serialize)] pub struct Config { pub name: String, pub description: String, @@ -22,20 +21,20 @@ lazy_static! { // NOTE: this object must be access by proxy through get_config() #[derive(Deserialize, Serialize)] static ref BASIC_CONFIG: Config = { - let tags = var("SERVER_TAGS").unwrap(); - let tags: Vec = match serde_json::from_str(tags.as_str()).expect("Unable to parse tags") { - Some(tags) => tags, - None => vec![] - }; + use std::fs::File; + use std::io::BufReader; + match File::open("config.json") { + Ok(file) => { + let reader = BufReader::new(file); + let rr: JsonResult = serde_json::from_reader(reader); + match rr { + Ok(meta) => meta, + Err(e) => panic!("{}", e) + } - let cfg = Config { - name: var("SERVER_NAME").unwrap_or("No name".into()), - description: var("SERVER_DESCRIPTION").unwrap_or("No description".into()), - url: var("PUBLIC_URL").unwrap(), - wsurl: var("PUBLIC_WS_URL").unwrap(), - tags - }; - cfg + }, + Err(e) => panic!("{}", e) + } }; } @@ -60,7 +59,7 @@ pub async fn server_meta(response: &mut Response) { pub async fn server_neighbors(p: &Pool, response: &mut Response) { // This method refers to what servers have been added as **related** by the admins // It returns a list of servers meta objects which converted to JSON - match db::Neighbor::get_all(p).await { + match db::neighbors::get_all(p).await { Ok(neighbors) => set_json_body(response, json!({"neighbors": neighbors})), Err(e) => { *response.status_mut() = StatusCode::INTERNAL_SERVER_ERROR; From ee5d9fb248978b148ed8f51d9ede5508304c62d4 Mon Sep 17 00:00:00 2001 From: shockrah Date: Sun, 9 May 2021 23:16:43 -0700 Subject: [PATCH 13/27] + Better formatting in hyper compact code + Adding ADD_NEIGHBOR permissions flag --- json-api/src/perms.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/json-api/src/perms.rs b/json-api/src/perms.rs index 0541e17..ea98e12 100644 --- a/json-api/src/perms.rs +++ b/json-api/src/perms.rs @@ -13,6 +13,7 @@ pub const _ADMIN: u64 = 1 << 62; // can make other admins but can't really touch // ADMIN PERMS pub const CREATE_CHANNEL:u64 = 64; pub const DELETE_CHANNEL:u64 = 128; +pub const ADD_NEIGHBOR:u64 = 256; // BELOW ARE COLLECTIVE PERMISSION SETS pub const OWNER: u64 = std::u64::MAX; @@ -21,16 +22,22 @@ pub const ADMIN_PERMS: u64 = !(std::u64::MAX & OWNER); // filter the only perm a pub fn get_perm_mask(path: &str) -> Option { use crate::routes::{ + self, INVITE_CREATE, CHANNELS_LIST, CHANNELS_CREATE, CHANNELS_DELETE, MESSAGE_SEND, }; match path { INVITE_CREATE => Some(CREATE_TMP_INVITES), + CHANNELS_LIST => None, CHANNELS_CREATE => Some(CREATE_CHANNEL), + CHANNELS_DELETE => Some(DELETE_CHANNEL), + MESSAGE_SEND => Some(SEND_MESSAGES), + + routes::ADD_NEIGHBOR => Some(ADD_NEIGHBOR), _ => Some(0) } } @@ -39,4 +46,4 @@ pub fn get_perm_mask(path: &str) -> Option { pub fn has_perm(mask: u64, target: u64) -> bool { // Check if the given mask contains the target permissions mask return (mask & target) == target; -} \ No newline at end of file +} From f1b1581588a43d08667b85511e009f23f1b9f0b9 Mon Sep 17 00:00:00 2001 From: shockrah Date: Tue, 11 May 2021 13:45:41 -0700 Subject: [PATCH 14/27] + db::neighbors::get and db::neighbors::update It should be noted at this point we're considering url's as the "primary key" even though its kinda weird to use them that way. Even though a varchar primary key doesn't really scale well I'm considering it fine for now as the neighbors table is controlled by admins(by hand) and hopefully won't need to scale. --- json-api/db/src/lib.rs | 4 ++-- json-api/db/src/neighbors.rs | 42 +++++++++++++++++++++++++++++++++++- 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/json-api/db/src/lib.rs b/json-api/db/src/lib.rs index 564d421..e8848ec 100644 --- a/json-api/db/src/lib.rs +++ b/json-api/db/src/lib.rs @@ -1,4 +1,4 @@ -use serde::Serialize; +use serde::{Serialize, Deserialize}; pub mod member; pub mod common; @@ -92,7 +92,7 @@ pub struct PublicMember { pub permissions: UBigInt, } -#[derive(Serialize)] +#[derive(Deserialize, Serialize)] pub struct Neighbor { pub name: String, pub description: Option, diff --git a/json-api/db/src/neighbors.rs b/json-api/db/src/neighbors.rs index a922d06..4dd111e 100644 --- a/json-api/db/src/neighbors.rs +++ b/json-api/db/src/neighbors.rs @@ -21,7 +21,7 @@ pub async fn add_neighbor(p: &Pool, url: &str, wsurl: &str, name: &str, desc: &s // writing it to the db let mut conn = p.get_conn().await?; let q = "INSERT INTO neighbors(url, wsurl, name, description, tags) - VALUES(:url, :wsurl, :name, :description, :tags)"; + VALUES(:url, :wsurl, :name, :desc, :tags)"; let sqlparams = params!{ "url"=> url, "wsurl"=>wsurl, @@ -32,3 +32,43 @@ pub async fn add_neighbor(p: &Pool, url: &str, wsurl: &str, name: &str, desc: &s conn.exec_drop(q, sqlparams).await?; Ok(()) } + +pub async fn get(p: &Pool, url: &str) -> SqlResult> { + let mut conn = p.get_conn().await?; + let q = "SELECT wsurl, name, description, tags FROM neighbors + WHERE url = :url"; + + type Fields = (Option, String, Option, String); + let row: Option = conn.exec_first(q, params!{"url" => url}).await?; + if let Some(fields) = row { + let tags: Vec = serde_json::from_str(fields.3.as_str()).unwrap_or(vec![]); + let n = Neighbor { + name: fields.1, + wsurl: fields.0, + description: fields.2, + tags, + url: url.into(), + }; + return Ok(Some(n)); + } + Ok(None) +} + +pub async fn update(p: &Pool, url: &str, new: Neighbor) -> SqlResult<()> { + let mut conn = p.get_conn().await?; + let q = "UPDATE neighbors + SET name = :nm, description = :desc, tags = :tags, wsurl = :ws, url = :newurl + WHERE url = :url"; + + let sqlparams = params!{ + "url" => url, + "nm" => new.name, + "desc" => new.description, + "tags" => serde_json::to_string(&new.tags).unwrap(), + "ws" => new.wsurl, + "newurl" => new.url + }; + + conn.exec_drop(q, sqlparams).await?; + Ok(()) +} From b810a5abbab2ca37e1269df58be99b10a5da4650 Mon Sep 17 00:00:00 2001 From: shockrah Date: Tue, 11 May 2021 13:52:07 -0700 Subject: [PATCH 15/27] * Fixing message body unwrap to safer unwrap_or unwrap had a chance to panic where as the unwrap_or now defaults to an empty Bytes object which lets the code after it fail with an HTTP 400. This is preferrable to a 500 error as there really is no error, just bad/unpollable input from the end user. --- json-api/src/messages.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/json-api/src/messages.rs b/json-api/src/messages.rs index 2eef93c..2fd53cd 100644 --- a/json-api/src/messages.rs +++ b/json-api/src/messages.rs @@ -1,6 +1,7 @@ use mysql_async::Pool; use hyper::{Response, Body, HeaderMap, StatusCode}; use hyper::body::to_bytes; +use hyper::body::Bytes; use serde_json::json; use std::collections::HashMap; @@ -94,7 +95,7 @@ pub async fn send_message(pool: &Pool, response: &mut Response, body: Body let channel_id = qs_param!(params, "channel_id", u64); // Black magic - let body_bytes: &[u8] = &to_bytes(body).await.unwrap(); // yolo + let body_bytes: &[u8] = &to_bytes(body).await.unwrap_or(Bytes::new()); let content = String::from_utf8_lossy(body_bytes); // 400 on empty bodies or missing channel id's @@ -164,3 +165,4 @@ pub async fn recent_messages(pool: &Pool, response: &mut Response, params: } else { } } + From 45540ddd2586a3f720fdd19edf3cddd9861b1e27 Mon Sep 17 00:00:00 2001 From: shockrah Date: Tue, 11 May 2021 15:22:56 -0700 Subject: [PATCH 16/27] * Better failure logging in failed requests self.passing property is a much more comprehensive way of checking for passing tests Also this uses the colors to dump to stdout --- json-api/client-tests/request.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/json-api/client-tests/request.py b/json-api/client-tests/request.py index 89d34a3..91362a7 100644 --- a/json-api/client-tests/request.py +++ b/json-api/client-tests/request.py @@ -49,8 +49,8 @@ class Request: def show_response(self): - if self.response is None: - print('Response := None') + if not self.passing: + print(RED + 'Fail' + NC + ' ' + self.url) return real_code = self.response.status_code From b4c55b72eaed94bbe1f8cbb6136e5704c55e374c Mon Sep 17 00:00:00 2001 From: shockrah Date: Tue, 11 May 2021 17:14:22 -0700 Subject: [PATCH 17/27] - Removing unused .env file diesel still requires this however the --database-url flag is there to alleviate this issue + make test now cleans up fake png images from the cwd This is more for comfort reall --- json-api/.env | 31 ------------------------------- json-api/Makefile | 1 + 2 files changed, 1 insertion(+), 31 deletions(-) delete mode 100644 json-api/.env diff --git a/json-api/.env b/json-api/.env deleted file mode 100644 index 17d799e..0000000 --- a/json-api/.env +++ /dev/null @@ -1,31 +0,0 @@ -# Example .env file -# In theory this file can be used for _both_ the rtc-server and the json-api server - -# This field is required by the json-api binary to actually find the database -DATABASE_URL=mysql://freechat_dev:password@localhost:3306/freechat -# The fields below are unused but are here in case you're confused as to the -# structure of the url above -# It is perfectly safe to delete these -DATABASE_NAME=freechat -DATABASE_PASS=password -DATABASE_USER=freechat_dev -DATABASE_HOST=localhost -DATABASE_PORT=3306 - -# Note that these should literally never point to the same file -# that completely breaks the web socket's permissions+authentication model -HMAC_PATH=hmac.secret -WSS_HMAC_PATH=wss-hmac.secret - - -# Public name that your server has -SERVER_NAME="Freechat Dev Server" -# Public description of what youre server is about -SERVER_DESCRIPTION="Server for sick development things" - -# These are the actual url's which are sent to/used by clients to communicate with -# your instance. If you have a reverse proxy setup then you can configure it so that -# the port numbers are not in the URL(suggested) -PUBLIC_URL=http://localhost:4536 -PUBLIC_WS_URL=ws://localhost:5648 - diff --git a/json-api/Makefile b/json-api/Makefile index 199a3b9..9ab722e 100644 --- a/json-api/Makefile +++ b/json-api/Makefile @@ -9,6 +9,7 @@ run: test: cd ../ && bash scripts/run-api-tests.sh + rm -f *.png clean: cargo clean From 664b837f130b425a4525b26fe1e8023e99741898 Mon Sep 17 00:00:00 2001 From: shockrah Date: Tue, 11 May 2021 17:16:50 -0700 Subject: [PATCH 18/27] + More tests for /neighbor routes ! put /neighbor/update is failing due to a failure to parse the body correctly on thebackend Further investigation is required + Kinda minor but I'm also adding the ability to `put` things now + Also an if statement to avoid failing on checkf for NoneType responses --- json-api/client-tests/main.py | 24 ++++++++++++++++++++---- json-api/client-tests/request.py | 12 ++++++++---- 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/json-api/client-tests/main.py b/json-api/client-tests/main.py index 4c092f5..a3f7470 100644 --- a/json-api/client-tests/main.py +++ b/json-api/client-tests/main.py @@ -2,6 +2,7 @@ from time import time from request import Request from config import create_admin, Admin from config import Server +from json import dumps as to_json RESPONSES = [] VOICE_CHAN = 1 @@ -73,7 +74,6 @@ def make_and_receive_invite(admin: Admin) -> (Request, Request): return (make_inv_req, None) # Setup to fire the second request, .fire() is safe to blindly use at this point - print('Response text from /join ', make_inv_req.response.status_code, ' ', make_inv_req.response.text) new_invite_body = make_inv_req.response.json()['invite'] code = new_invite_body['id'] user_inv_req = req(admin , 'get' , '/join', {'code': code}, 200, verbose=True) @@ -108,6 +108,19 @@ if __name__ == '__main__': # Invite test # Container for most/all the generic requests we want to fire off + now = time() + tmp_neighbor = { + 'name': str(now), + 'wsurl': 'wsurl', + 'url': str(now), + 'description': 'asdf', + 'tags': ['red','blue'] + } + # This neighbor is used to "update" the tmp one above in /neighbor tests + updated_neighbor = tmp_neighbor.copy() + updated_neighbor['name'] = 'new' + print('Updated neighbor as json ', to_json(updated_neighbor)) + requests.extend([ req(fake_user, 'get', '/channels/list', {}, 401), req(admin, 'post', '/channels/list', {'kind': TEXT_CHAN}, 404), @@ -121,12 +134,14 @@ if __name__ == '__main__': req(admin, 'get', '/message/recent', {'channel_id': 123, 'limit': 20}, 404), req(admin, 'get', '/members/me', {}, 200), req(admin, 'get', '/members/get_online', {}, 200), - req(admin, 'post', '/members/me/nickname', {'nick': f'randy-{time()}'}, 200), + req(admin, 'post', '/members/me/nickname', {'nick': f'randy-{now}'}, 200), req(admin , 'get', '/invite/join', {'code': 123}, 404), req(admin , "get", "/meta", {}, 200), req(admin, 'get', '/neighbor/list', {}, 200), - req(admin,'post', '/neighbor/add', {'name':'name','url':'url','wsurl':'wsurl','description':'asdf','tags':'["red","blue"]'}, 200), - req(admin, 'get', '/neighbor/list', {}, 200, verbose=True) + req(admin,'post', '/neighbor/add', {}, 200, body=to_json(tmp_neighbor)), + req(admin, 'get', '/neighbor/list', {}, 200, verbose=True), + req(admin, 'put', '/neighbor/update', {'url':str(now)}, 200, body=to_json(updated_neighbor)), + req(admin, 'get', '/neighbor/list', {}, 200, verbose=True), ]) # add this after fire the generic tests @@ -150,6 +165,7 @@ if __name__ == '__main__': r.show_response() if r.passing: pass_count += 1 + req_count = len(requests) print(f'Passing {pass_count}/{req_count}') diff --git a/json-api/client-tests/request.py b/json-api/client-tests/request.py index 91362a7..1979d47 100644 --- a/json-api/client-tests/request.py +++ b/json-api/client-tests/request.py @@ -40,6 +40,8 @@ class Request: self.response = requests.post(self.url, headers=self.headers, params=self.qs, data=self.body) elif self.method == 'delete': self.response = requests.delete(self.url, headers=self.headers, params=self.qs) + elif self.method == 'put': + self.response = requests.put(self.url, headers=self.headers, params=self.qs) return self.response except Exception as e: @@ -49,13 +51,15 @@ class Request: def show_response(self): - if not self.passing: - print(RED + 'Fail' + NC + ' ' + self.url) - return - real_code = self.response.status_code + real_code = None + if self.response is not None: + real_code = self.response.status_code + if self.hope != real_code: abstract = RED + 'Fail ' + NC + 'Expected ' + str(self.hope) + ' Got ' + str(real_code) + if self.response is None: + print('\tNo Response to show for') print(abstract) print('\t', self.method, ' ', self.url) From 666894af0e920c978ef49fd2aae45aaf4cb12b2c Mon Sep 17 00:00:00 2001 From: shockrah Date: Tue, 11 May 2021 17:18:17 -0700 Subject: [PATCH 19/27] + Debug to Neighbor struct * add_neighbor now takes a moved Neighbor param instead of the fields individually This should make it a easier for both front & backend to parse what's happening. It also let's us leveraege serde_json a bit more --- json-api/db/src/lib.rs | 2 +- json-api/db/src/neighbors.rs | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/json-api/db/src/lib.rs b/json-api/db/src/lib.rs index e8848ec..0de94d3 100644 --- a/json-api/db/src/lib.rs +++ b/json-api/db/src/lib.rs @@ -92,7 +92,7 @@ pub struct PublicMember { pub permissions: UBigInt, } -#[derive(Deserialize, Serialize)] +#[derive(Debug, Deserialize, Serialize)] pub struct Neighbor { pub name: String, pub description: Option, diff --git a/json-api/db/src/neighbors.rs b/json-api/db/src/neighbors.rs index 4dd111e..9804251 100644 --- a/json-api/db/src/neighbors.rs +++ b/json-api/db/src/neighbors.rs @@ -16,17 +16,18 @@ pub async fn get_all(p: &Pool) -> SqlResult> { Ok(set) } -pub async fn add_neighbor(p: &Pool, url: &str, wsurl: &str, name: &str, desc: &str, tags:&str) -> SqlResult<()> { +pub async fn add_neighbor(p: &Pool, new: Neighbor) -> SqlResult<()> { // Note we assume that the tags field has been verified as proper valid json prior to // writing it to the db let mut conn = p.get_conn().await?; let q = "INSERT INTO neighbors(url, wsurl, name, description, tags) VALUES(:url, :wsurl, :name, :desc, :tags)"; + let tags = serde_json::to_string(&new.tags).unwrap_or("[]".to_string()); let sqlparams = params!{ - "url"=> url, - "wsurl"=>wsurl, - "name"=>name, - "desc"=>desc, + "url"=> new.url, + "wsurl"=>new.wsurl, + "name"=>new.name, + "desc"=>new.description, "tags"=>tags }; conn.exec_drop(q, sqlparams).await?; From 1ca17ec6e0c63078285b48e35d0afa9887c2dd01 Mon Sep 17 00:00:00 2001 From: shockrah Date: Tue, 11 May 2021 17:23:15 -0700 Subject: [PATCH 20/27] * /neighbor/add handler now uses body for neighbor structure data This chage is basically required due to the ridiculuous number of parameters that needed to be passed. Also more POC for showing how serde_json's result type can be used to safely parse http bodies --- json-api/src/meta.rs | 40 ++++++++++++++++++---------------------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/json-api/src/meta.rs b/json-api/src/meta.rs index 96f1d6e..54571c5 100644 --- a/json-api/src/meta.rs +++ b/json-api/src/meta.rs @@ -1,9 +1,12 @@ // Basic handler for getting meta data about the server use std::collections::HashMap; use crate::http::set_json_body; +use db::Neighbor; use mysql_async::Pool; use hyper::{Response, Body, StatusCode}; +use hyper::body::to_bytes; +use hyper::body::Bytes; use serde_json::{json, to_string, Result as JsonResult}; use serde::{Serialize, Deserialize}; use lazy_static::lazy_static; @@ -69,28 +72,21 @@ pub async fn server_neighbors(p: &Pool, response: &mut Response) { } -pub async fn add_neighbor(p: &Pool, response: &mut Response, params: HashMap) { - let url = params.get("url"); - let wsurl = params.get("wsurl"); - let name = params.get("name"); - let desc = params.get("description"); - let tags = params.get("tags"); - - // If any parameter is missing then fail away quickly - if url.is_none() || wsurl.is_none() || name.is_none() || desc.is_none() || tags.is_none() { - *response.status_mut() = StatusCode::BAD_REQUEST; - return; - } - // Safe unwrap because of the check above - let url = url.unwrap(); - let wsurl = wsurl.unwrap(); - let name = name.unwrap(); - let desc = desc.unwrap(); - let tags = tags.unwrap(); - match db::neighbors::add_neighbor(p, url, wsurl, name, desc, tags).await { - Ok(()) => {}, - Err(e) => { - eprintln!("{}", e); +pub async fn add_neighbor(p: &Pool, response: &mut Response, body: Body) { + let body: Bytes = to_bytes(body).await.unwrap_or(Bytes::new()); + let json: JsonResult = serde_json::from_slice(&body); + if let Ok(neighbor) = json { + // Before we try adding the new neighbor, make sure there isn't already + // an entry with the same url + if let Ok(row) = db::neighbors::get(p, &neighbor.url).await { + match row.is_some() { + true => *response.status_mut() = StatusCode::CONFLICT, + false => if let Err(e) = db::neighbors::add_neighbor(p, neighbor).await { + eprintln!("{}", e); + } + }; + } else { + eprintln!(); *response.status_mut() = StatusCode::INTERNAL_SERVER_ERROR; } } From a941165ea576c86ea6ed3423f2a87aaca811fee6 Mon Sep 17 00:00:00 2001 From: shockrah Date: Tue, 11 May 2021 17:25:51 -0700 Subject: [PATCH 21/27] + adding /neighbor/update route to dispatch ! This route needs way more testing as its currently failing the prelim test listed right now ! Some more misc changes in testing/mod.rs that aren't imporant at all to anyone It's just an extra comment --- json-api/src/main.rs | 13 +++++++------ json-api/src/meta.rs | 25 +++++++++++++++++++++++++ json-api/src/routes.rs | 1 + json-api/src/testing/mod.rs | 4 ++-- 4 files changed, 35 insertions(+), 8 deletions(-) diff --git a/json-api/src/main.rs b/json-api/src/main.rs index 199eb5a..2ff3e16 100644 --- a/json-api/src/main.rs +++ b/json-api/src/main.rs @@ -58,12 +58,12 @@ async fn route_dispatcher( path: &str, body: Body, params: HashMap, - headers: HeaderMap, - claims: Option/* Faster id/perms access from here */) { + headers: HeaderMap) { const GET: &Method = &Method::GET; const POST: &Method = &Method::POST; const DELETE: &Method = &Method::DELETE; + const PUT: &Method = &Method::PUT; println!("[HTTP] {}: {}", meth, path); match (meth, path) { /* INVITES */ @@ -90,7 +90,8 @@ async fn route_dispatcher( (GET, routes::META) => meta::server_meta(resp).await, /* Federated Routes */ (GET, routes::GET_NEIGHBORS) => meta::server_neighbors(pool, resp).await, - (POST, routes::ADD_NEIGHBOR) => meta::add_neighbor(pool, resp, params).await, + (POST, routes::ADD_NEIGHBOR) => meta::add_neighbor(pool, resp, body).await, + (PUT, routes::UPDATE_NEIGHBOR) => meta::update_neighbor(pool, resp, params, body).await, _ => { println!("[HTTP]\tNOT FOUND: {}: {}", meth, path); *resp.status_mut() = StatusCode::NOT_FOUND @@ -116,11 +117,11 @@ async fn main_responder(request: Request) -> Result, hyper: if let Some(qs) = params_opt { match auth::wall_entry(path, &DB_POOL, &qs).await { OpenAuth => { - route_dispatcher(&DB_POOL, &mut response, &method, path, body, qs, headers, None).await; + route_dispatcher(&DB_POOL, &mut response, &method, path, body, qs, headers).await; }, // Only with typical routes do we have to inject the permissions - Good(claim) => { - route_dispatcher(&DB_POOL, &mut response, &method, path, body, qs, headers, Some(claim)).await; + Good(_claim/* TODO: start moving route handlers to use claims.id for faster access */) => { + route_dispatcher(&DB_POOL, &mut response, &method, path, body, qs, headers).await; }, LoginValid(member) => { println!("[HTTP] POST /login"); diff --git a/json-api/src/meta.rs b/json-api/src/meta.rs index 54571c5..3f12b8c 100644 --- a/json-api/src/meta.rs +++ b/json-api/src/meta.rs @@ -89,5 +89,30 @@ pub async fn add_neighbor(p: &Pool, response: &mut Response, body: Body) { eprintln!(); *response.status_mut() = StatusCode::INTERNAL_SERVER_ERROR; } + } else { + *response.status_mut() = StatusCode::BAD_REQUEST; + } +} + +pub async fn update_neighbor(p: &Pool, response: &mut Response, params: HashMap, body: Body) { + // First collect the target url from the map and try to parse the body + let target = params.get("url"); + let body: Bytes = to_bytes(body).await.unwrap_or(Bytes::new()); + let s: String = String::from_utf8_lossy(&body).to_string(); + let json: JsonResult = serde_json::from_str(&s); + + println!("\tjson result: {:?}", json); + println!("\tbody {}", s); + // Verify query string parameter **and** body before attempting to reach database + match (target, json) { + (Some(target_url), Ok(neighbor)) => { + // Only return a 500 if something happened on db-lib's end + if let Err(e) = db::neighbors::update(p, target_url, neighbor).await { + *response.status_mut() = StatusCode::INTERNAL_SERVER_ERROR; + eprintln!("/neighbor/update [DB-LIB] {}", e); + } + // Nothing to do on success 200 is already set by hyper + }, + _ => *response.status_mut() = StatusCode::BAD_REQUEST } } diff --git a/json-api/src/routes.rs b/json-api/src/routes.rs index d391ffe..b0670e1 100644 --- a/json-api/src/routes.rs +++ b/json-api/src/routes.rs @@ -27,6 +27,7 @@ pub const SET_NEW_ADMIN: Rstr = "/owner/newadmin"; // @requiers: ow // Server -> Server Routes pub const GET_NEIGHBORS: Rstr = "/neighbor/list"; // @requires: none @note must be a member for this list pub const ADD_NEIGHBOR: Rstr = "/neighbor/add"; // @requires: perm::add_neighbor +pub const UPDATE_NEIGHBOR: Rstr = "/neighbor/update"; // @requires perms::add_neighbor @url(unique) pub fn is_open(path: &str) -> bool { return path.starts_with("/join") || path.starts_with("/meta"); diff --git a/json-api/src/testing/mod.rs b/json-api/src/testing/mod.rs index 38e4837..340b9de 100644 --- a/json-api/src/testing/mod.rs +++ b/json-api/src/testing/mod.rs @@ -3,10 +3,10 @@ #[cfg(test)] pub fn get_pool() -> mysql_async::Pool { - use dotenv::dotenv; + // NOTE: this assumes that DATABASE_URL has been set in the environment + // prior to running use mysql_async::Pool; - dotenv().ok(); return Pool::new(&std::env::var("DATABASE_URL").unwrap()) } From a0d60c05683d689249ee297ab49e273b7bec791f Mon Sep 17 00:00:00 2001 From: shockrah Date: Tue, 11 May 2021 17:27:04 -0700 Subject: [PATCH 22/27] - Pointless permissions check /message/send handler Auth module literally does this for us see: auth::valid_perms --- json-api/src/messages.rs | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/json-api/src/messages.rs b/json-api/src/messages.rs index 2fd53cd..e38e2dc 100644 --- a/json-api/src/messages.rs +++ b/json-api/src/messages.rs @@ -63,7 +63,6 @@ pub async fn send_message(pool: &Pool, response: &mut Response, body: Body * TODO: more features here because send_message is a large handler */ use db::Response::*; - use db::Member; // NOTE: auth module guarantees that id will be present so the unwrap is safe let uid = qs_param!(params, "id", u64).unwrap(); @@ -77,21 +76,6 @@ pub async fn send_message(pool: &Pool, response: &mut Response, body: Body None => None }; - let permissions = match Member::get(pool, uid).await { - Ok(dbresponse) => match dbresponse { - Row(user) => user.permissions, - _ => 0 - }, - Err(e) => { - eprintln!("[DB-SQL] {}",e ); - 0 - } - }; - if perms::has_perm(permissions, perms::SEND_MESSAGES) == false { - *response.status_mut() = StatusCode::BAD_REQUEST; - return; - } - let channel_id = qs_param!(params, "channel_id", u64); // Black magic From 8bff61ab7111fde6066305e18cb9acc69e8896b6 Mon Sep 17 00:00:00 2001 From: shockrah Date: Wed, 12 May 2021 13:02:42 -0700 Subject: [PATCH 23/27] * Fixed issue where put requests weren't firing with the optional body parameter This also "fixes" the /neighbor/update route conveniently enough, which had much better behavior than expected before. - Remvoing some fluff from debugging --- json-api/client-tests/request.py | 2 +- json-api/src/messages.rs | 1 - json-api/src/meta.rs | 2 -- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/json-api/client-tests/request.py b/json-api/client-tests/request.py index 1979d47..17c3c21 100644 --- a/json-api/client-tests/request.py +++ b/json-api/client-tests/request.py @@ -41,7 +41,7 @@ class Request: elif self.method == 'delete': self.response = requests.delete(self.url, headers=self.headers, params=self.qs) elif self.method == 'put': - self.response = requests.put(self.url, headers=self.headers, params=self.qs) + self.response = requests.put(self.url, headers=self.headers, params=self.qs, data=self.body) return self.response except Exception as e: diff --git a/json-api/src/messages.rs b/json-api/src/messages.rs index e38e2dc..1edcd6a 100644 --- a/json-api/src/messages.rs +++ b/json-api/src/messages.rs @@ -7,7 +7,6 @@ use serde_json::json; use std::collections::HashMap; use crate::http::set_json_body; -use crate::perms; use crate::qs_param; use db::Message; diff --git a/json-api/src/meta.rs b/json-api/src/meta.rs index 3f12b8c..cb6bd42 100644 --- a/json-api/src/meta.rs +++ b/json-api/src/meta.rs @@ -101,8 +101,6 @@ pub async fn update_neighbor(p: &Pool, response: &mut Response, params: Ha let s: String = String::from_utf8_lossy(&body).to_string(); let json: JsonResult = serde_json::from_str(&s); - println!("\tjson result: {:?}", json); - println!("\tbody {}", s); // Verify query string parameter **and** body before attempting to reach database match (target, json) { (Some(target_url), Ok(neighbor)) => { From e94c72033243619fab5326d946e41d0ab3d7f85f Mon Sep 17 00:00:00 2001 From: shockrah Date: Wed, 12 May 2021 13:05:52 -0700 Subject: [PATCH 24/27] + More comprehensive testing on /neighbor/update The cases that actually matter should be covered now so I'm confident with this endpoint's behavior ! A preliminary db::neighbors::get is done to cover the 404 case Basically if a bad url is used in the request we should 404 the update as there won't be anything relevant to update --- json-api/client-tests/main.py | 1 + json-api/src/meta.rs | 19 ++++++++++++++----- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/json-api/client-tests/main.py b/json-api/client-tests/main.py index a3f7470..88ce3da 100644 --- a/json-api/client-tests/main.py +++ b/json-api/client-tests/main.py @@ -141,6 +141,7 @@ if __name__ == '__main__': req(admin,'post', '/neighbor/add', {}, 200, body=to_json(tmp_neighbor)), req(admin, 'get', '/neighbor/list', {}, 200, verbose=True), req(admin, 'put', '/neighbor/update', {'url':str(now)}, 200, body=to_json(updated_neighbor)), + req(admin, 'put', '/neighbor/update', {'url':'fake'}, 404, body=to_json(updated_neighbor)), req(admin, 'get', '/neighbor/list', {}, 200, verbose=True), ]) diff --git a/json-api/src/meta.rs b/json-api/src/meta.rs index cb6bd42..51e3f22 100644 --- a/json-api/src/meta.rs +++ b/json-api/src/meta.rs @@ -104,12 +104,21 @@ pub async fn update_neighbor(p: &Pool, response: &mut Response, params: Ha // Verify query string parameter **and** body before attempting to reach database match (target, json) { (Some(target_url), Ok(neighbor)) => { - // Only return a 500 if something happened on db-lib's end - if let Err(e) = db::neighbors::update(p, target_url, neighbor).await { - *response.status_mut() = StatusCode::INTERNAL_SERVER_ERROR; - eprintln!("/neighbor/update [DB-LIB] {}", e); + match db::neighbors::get(p, target_url).await { + Ok(row) => if row.is_some() { + if let Err(e) = db::neighbors::update(p, target_url, neighbor).await { + *response.status_mut() = StatusCode::INTERNAL_SERVER_ERROR; + eprintln!("/neighbor/update [DB-LIB] {}", e); + } + // Nothing to do on success 200 is already set by hyper + } else{ + *response.status_mut() = StatusCode::NOT_FOUND; + }, + Err(e) => { + eprintln!("/neighbor/update {}", e); + *response.status_mut() = StatusCode::INTERNAL_SERVER_ERROR; + } } - // Nothing to do on success 200 is already set by hyper }, _ => *response.status_mut() = StatusCode::BAD_REQUEST } From fbcf16b735310f3bf09e102db976586ddb24d1aa Mon Sep 17 00:00:00 2001 From: shockrah Date: Wed, 12 May 2021 14:07:38 -0700 Subject: [PATCH 25/27] + Adding /neighbor/remove route api code and db-lib code This patch has not yet been tested but will be in the coming patches Sumamary: we're just doing a regular delete on the neighbor's table based off the url field --- json-api/db/src/neighbors.rs | 7 +++++++ json-api/src/main.rs | 1 + json-api/src/meta.rs | 13 +++++++++++++ json-api/src/routes.rs | 1 + 4 files changed, 22 insertions(+) diff --git a/json-api/db/src/neighbors.rs b/json-api/db/src/neighbors.rs index 9804251..d75c3b3 100644 --- a/json-api/db/src/neighbors.rs +++ b/json-api/db/src/neighbors.rs @@ -73,3 +73,10 @@ pub async fn update(p: &Pool, url: &str, new: Neighbor) -> SqlResult<()> { conn.exec_drop(q, sqlparams).await?; Ok(()) } + +pub async fn remove(p: &Pool, url: &str) -> SqlResult<()> { + let mut conn = p.get_conn().await?; + let q = "DELETE FROM neighbors WHERE url = :url"; + conn.exec_drop(q, params!{"url" => url}).await?; + Ok(()) +} diff --git a/json-api/src/main.rs b/json-api/src/main.rs index 2ff3e16..343b22e 100644 --- a/json-api/src/main.rs +++ b/json-api/src/main.rs @@ -92,6 +92,7 @@ async fn route_dispatcher( (GET, routes::GET_NEIGHBORS) => meta::server_neighbors(pool, resp).await, (POST, routes::ADD_NEIGHBOR) => meta::add_neighbor(pool, resp, body).await, (PUT, routes::UPDATE_NEIGHBOR) => meta::update_neighbor(pool, resp, params, body).await, + (DELETE, routes::REMOVE_NEIGHBOR) => meta::remove_neighbor(pool, resp, params).await, _ => { println!("[HTTP]\tNOT FOUND: {}: {}", meth, path); *resp.status_mut() = StatusCode::NOT_FOUND diff --git a/json-api/src/meta.rs b/json-api/src/meta.rs index 51e3f22..fb9ebd4 100644 --- a/json-api/src/meta.rs +++ b/json-api/src/meta.rs @@ -123,3 +123,16 @@ pub async fn update_neighbor(p: &Pool, response: &mut Response, params: Ha _ => *response.status_mut() = StatusCode::BAD_REQUEST } } + +pub async fn remove_neighbor(p: &Pool, response: &mut Response, params: HashMap) { + match params.get("url") { + Some(url) => { + // As usual 500 on server errors otherwise there's nothing to do + if let Err(e) = db::neighbors::remove(p, url).await { + eprintln!("[HTTP] [DB-LIB] /neighbor/remove {}", e); + *response.status_mut() = StatusCode::INTERNAL_SERVER_ERROR; + } + }, + None => *response.status_mut() = StatusCode::BAD_REQUEST + } +} diff --git a/json-api/src/routes.rs b/json-api/src/routes.rs index b0670e1..8f4b17a 100644 --- a/json-api/src/routes.rs +++ b/json-api/src/routes.rs @@ -28,6 +28,7 @@ pub const SET_NEW_ADMIN: Rstr = "/owner/newadmin"; // @requiers: ow pub const GET_NEIGHBORS: Rstr = "/neighbor/list"; // @requires: none @note must be a member for this list pub const ADD_NEIGHBOR: Rstr = "/neighbor/add"; // @requires: perm::add_neighbor pub const UPDATE_NEIGHBOR: Rstr = "/neighbor/update"; // @requires perms::add_neighbor @url(unique) +pub const REMOVE_NEIGHBOR: Rstr = "/neighbor/delete"; // @requires perms::add_neighbor @url(unique) pub fn is_open(path: &str) -> bool { return path.starts_with("/join") || path.starts_with("/meta"); From 24b5f11b130ed2ff0412f6e4c1c126353764f350 Mon Sep 17 00:00:00 2001 From: shockrah Date: Wed, 12 May 2021 14:14:30 -0700 Subject: [PATCH 26/27] * Better logging in meta module(more logging tags) --- json-api/src/meta.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/json-api/src/meta.rs b/json-api/src/meta.rs index fb9ebd4..37e5926 100644 --- a/json-api/src/meta.rs +++ b/json-api/src/meta.rs @@ -32,7 +32,7 @@ lazy_static! { let rr: JsonResult = serde_json::from_reader(reader); match rr { Ok(meta) => meta, - Err(e) => panic!("{}", e) + Err(e) => panic!("[HTTP] [FATAL-INIT] {}", e) } }, @@ -82,7 +82,7 @@ pub async fn add_neighbor(p: &Pool, response: &mut Response, body: Body) { match row.is_some() { true => *response.status_mut() = StatusCode::CONFLICT, false => if let Err(e) = db::neighbors::add_neighbor(p, neighbor).await { - eprintln!("{}", e); + eprintln!("[HTTP] [DB-LIB] {}", e); } }; } else { @@ -108,14 +108,14 @@ pub async fn update_neighbor(p: &Pool, response: &mut Response, params: Ha Ok(row) => if row.is_some() { if let Err(e) = db::neighbors::update(p, target_url, neighbor).await { *response.status_mut() = StatusCode::INTERNAL_SERVER_ERROR; - eprintln!("/neighbor/update [DB-LIB] {}", e); + eprintln!("[HTTP] [DB-LIB] /neighbor/update [DB-LIB] {}", e); } // Nothing to do on success 200 is already set by hyper } else{ *response.status_mut() = StatusCode::NOT_FOUND; }, Err(e) => { - eprintln!("/neighbor/update {}", e); + eprintln!("[HTTP] [DB-LIB] /neighbor/update {}", e); *response.status_mut() = StatusCode::INTERNAL_SERVER_ERROR; } } From c25e8047cf87cf1bf13bec481f16e15ffdaeeff2 Mon Sep 17 00:00:00 2001 From: shockrah Date: Wed, 12 May 2021 14:22:14 -0700 Subject: [PATCH 27/27] + Finishing up more tests for the /neighbors routes For now this sub api is at an mvp stage so it should be fine to move onto new features after this point for now. --- json-api/client-tests/main.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/json-api/client-tests/main.py b/json-api/client-tests/main.py index 88ce3da..8e2ff7f 100644 --- a/json-api/client-tests/main.py +++ b/json-api/client-tests/main.py @@ -142,6 +142,9 @@ if __name__ == '__main__': req(admin, 'get', '/neighbor/list', {}, 200, verbose=True), req(admin, 'put', '/neighbor/update', {'url':str(now)}, 200, body=to_json(updated_neighbor)), req(admin, 'put', '/neighbor/update', {'url':'fake'}, 404, body=to_json(updated_neighbor)), + req(admin, 'get', '/neighbor/list', {}, 200), + req(admin, 'delete', '/neighbor/delete', {'url':'fake'}, 200), + req(admin, 'delete', '/neighbor/delete', {'url': str(now)}, 200), req(admin, 'get', '/neighbor/list', {}, 200, verbose=True), ])