From 38ff0edd397ab87c5d4e1b1d6b743040d4b51bad Mon Sep 17 00:00:00 2001 From: shockrah Date: Wed, 21 Apr 2021 17:22:16 -0700 Subject: [PATCH] ! Massive test suite overhaul see details below ! Problem: the old test suite was extremely inflexible This meant that making new tests took way too much time. + This new rework makes the new client's backend much thinner and less "magical" With less magic going on we can pass way more data more easily to the actual http-request engine making the convenience wrapper over top it much more flexible Translating old tests to the new engine might take a while but for now the old client codebase is completely deprecated and will no longer be used+updated --- json-api/client-tests/client.py | 67 +++++++++---------- json-api/client-tests/config.py | 42 ++++++++++++ json-api/client-tests/main.py | 104 ++++++++++++++++++++++++++++++ json-api/client-tests/request.py | 59 +++++++++++++++++ json-api/client-tests/web/http.py | 24 ++++--- 5 files changed, 251 insertions(+), 45 deletions(-) create mode 100644 json-api/client-tests/config.py create mode 100644 json-api/client-tests/main.py create mode 100644 json-api/client-tests/request.py diff --git a/json-api/client-tests/client.py b/json-api/client-tests/client.py index 47c8f6f..db8d2b4 100644 --- a/json-api/client-tests/client.py +++ b/json-api/client-tests/client.py @@ -1,6 +1,7 @@ import time import subprocess, os, sys import json, requests +from json import JSONDecodeError from web import http @@ -31,9 +32,10 @@ class Worker: proc = subprocess.run(f'cargo run --release -- -c dev-test'.split(), text=True, capture_output=True) try: return json.loads(proc.stdout) - except: + except JSONDecodeError as err: import sys print('TESTBOT UNABLE TO LOAD JSON DATA FROM -c flag', file=sys.stderr) + print(err) exit(1) def auth(self, auth_opt: str): @@ -69,13 +71,16 @@ class Worker: # 'body': # show response body or not (optional # } - # not 'try'ing these as misconfigurations are to be cleaned up before testing + # not 'try'ing these as misconfigurations are completely fatal anyway method, path, opts = test['init'] auth = test['auth'] hope = test['hope'] if 'body' in test: - self.request(method, path, auth, opts, hope, show_body=test['body']) + # Only checking for headers if a body is present as we really only use + # headers with the body + head = test['headers'] if 'headers' in test else None + self.request(method, path, auth, opts, hope, show_body=test['body'], headers=head) else: self.request(method, path, auth, opts, hope) @@ -108,7 +113,7 @@ class Worker: - def request(self, method: str, path: str, auth: str, opts: dict, expectation: int, show_body=False): + def request(self, method: str, path: str, auth: str, opts: dict, expectation: int, show_body=False, headers={}): assert(path[0] == '/') # First make sure we add in the correct auth params that are requested @@ -116,7 +121,7 @@ class Worker: # Build the request and store it in our structure url = self.domain + path - req = http.Request(method, url, opts) + req = http.Request(method, url, opts, headers=headers) r_id = time.time() resp = req.make(expectation) @@ -146,15 +151,12 @@ def run(worker: Worker): {'init': ['post', '/channels/list', {}], 'auth': jwt, 'hope': 404}, {'init': ['post', '/channels/create', {'name': str(new_channel_name), 'kind': TEXT_CHAN, 'description': 'asdf'}], 'auth': jwt, 'hope': 200}, - # Just a regular test no saving for this one {'init': ['post', '/channels/create', {'name': str(new_channel_name+1), 'kind': TEXT_CHAN,}], 'auth': jwt, 'hope': 200}, {'init': ['post', '/channels/create', {}], 'auth': jwt, 'hope': 400}, {'init': ['post', '/channels/create', {'name': 123, 'kind': 'adsf'}], 'auth': jwt, 'hope': 400}, - # save this and compare its results to the previous - {'init': ['get', '/channels/list', {}], 'auth': jwt, 'hope': 200}, - {'init': ['get', '/channels/list', {'random-param': 123}], 'auth': jwt, 'hope': 200}, + {'init': ['get', '/channels/list', {'kind': TEXT_CHAN}], 'auth': jwt, 'hope': 200}, ] for test in channel_tests: @@ -171,10 +173,26 @@ def run(worker: Worker): message_tests = [ # bs message spam - {'init': ['post', '/message/send', {'type': 'text', 'channel_id': chan_d['id'], 'content': 'bs content'}], 'auth': jwt, 'hope': 200, 'body': True}, - {'init': ['post', '/message/send', {'type': 'text', 'channel_id': chan_d['id'], 'content': 'bs content'}], 'auth': jwt, 'hope': 200}, - {'init': ['post', '/message/send', {'type': 'text', 'channel_id': chan_d['id'], 'content': 'bs content'}], 'auth': jwt, 'hope': 200}, - {'init': ['post', '/message/send', {'type': 'text', 'channel_id': chan_d['id'], 'content': 'bs content'}], 'auth': jwt, 'hope': 200}, + { + 'init': ['post', '/message/send', {'type': 'text', 'channel_id': chan_d['id'], 'content': 'bs content'}], + 'headers': {'content-type': 'text/plain'}, + 'auth': jwt, 'hope': 200, 'body': False + }, + { + 'init': ['post', '/message/send', {'type': 'text', 'channel_id': chan_d['id'], 'content': 'bs content'}], + 'headers': {'content-type': 'text/plain'}, + 'auth': jwt, 'hope': 200 + }, + { + 'init': ['post', '/message/send', {'type': 'text', 'channel_id': chan_d['id'], 'content': 'bs content'}], + 'headers': {'content-type': 'text/plain'}, + 'auth': jwt, 'hope': 200 + }, + { + 'init': ['post', '/message/send', {'type': 'text', 'channel_id': chan_d['id'], 'content': 'bs content'}], + 'headers': {'content-type': 'text/plain'}, + 'auth': jwt, 'hope': 200 + }, # can we get them back tho? { @@ -201,29 +219,6 @@ def run(worker: Worker): ], 'auth': jwt, 'hope': 400 }, - # two tests that follow the rules - { - 'init': [ - 'get', '/message/from_id', {'start': 1, 'channel_id': 3} - ], - 'auth': jwt, 'hope': 200, 'body': True - }, - { - 'init': [ - 'get', '/message/from_id', {'start':1, 'channel_id':3, 'limit':2} - ], - 'auth': jwt, 'hope': 200, 'body': True - }, - { - # Channel doesn't exist so empty vector is result - 'init': [ 'get', '/message/from_id', {'start': 1, 'channel_id':9} ], - 'auth': jwt, 'hope': 404 - }, - { - # Channel id doesn't refer to a real channel - 'init': [ 'get', '/message/from_id', {'start': 5, 'channel_id':3} ], - 'auth': jwt, 'hope': 404 - }, ] for test in message_tests: diff --git a/json-api/client-tests/config.py b/json-api/client-tests/config.py new file mode 100644 index 0000000..bb768c8 --- /dev/null +++ b/json-api/client-tests/config.py @@ -0,0 +1,42 @@ +import subprocess +import os +import json + +class Server: + def __init__(self, meta: dict): + self.url = meta.get('url') + self.wsurl = meta.get('wsurl') + self.serv_name = meta.get('name') + +class Admin: + def __init__(self, user: dict, server: dict): + self.id = user.get('id') + self.name = user.get('name') + self.permissions = user.get('permissions') + self.secret = user.get('secret') + self.status = user.get('status') + self.jwt = None + + self.server = Server(server) + + def __str__(self) -> str: + return f'{self.name}#{self.id}' + +def create_admin() -> Admin : + CARGO_BIN = os.getenv('CARGO_BIN') + + proc = subprocess.run( + f'cargo run --release -- -c python-tester'.split(), + text=True, capture_output=True + ) + try: + raw = json.loads(proc.stdout) + user = raw.get('user') + server = raw.get('server') + if user is None or server is None: + return None + else: + return Admin(user, server) + except: + return None + diff --git a/json-api/client-tests/main.py b/json-api/client-tests/main.py new file mode 100644 index 0000000..07ad223 --- /dev/null +++ b/json-api/client-tests/main.py @@ -0,0 +1,104 @@ +from time import time +from request import Request +from config import create_admin, Admin +from config import Server + +RESPONSES = [] +VOICE_CHAN = 1 +TEXT_CHAN = 2 + +def login() -> (Request, str): + req = Request( + admin.server.url + '/login', + 'post', + {'id':admin.id, 'secret': admin.secret}, + {}, + 200 + ) + response = req.fire() + jwt = response.json().get('jwt') + return (req, jwt) + +def make_channel(url: str, id: int, jwt: str) -> (Request, int): + # making a text channel + channel_name = str(time()) + qs = {'id': id, 'jwt': jwt, 'kind': 2, 'name': channel_name} + req = Request(url + '/channels/create', 'post', qs, {}, 200) + resp = req.fire() + try: + channel = resp.json().get('channel') + chan_id = 0 if channel is None else channel.get('id') + return (req, chan_id) + except Exception as e: + print(e) + print(f'Actual value: {resp.text}: {resp.status_code}') + return (req, 0) + +def bs_admin(server: Server) -> Admin: + user_ = { + 'id': 123, + 'jwt': None, + 'secret': 'no thanks' + } + server_ = { + 'url': server.url, + 'wsurl': server.wsurl, + 'name': server.serv_name + } + return Admin(user_, server_) + + +def std_request(admin: Admin, method: str, path: str, qs: dict, hope: int, headers: dict={}, body=None) -> Request: + url = admin.server.url + path + # Copy over the additional params into the query string + params = {'id': admin.id, 'jwt': admin.jwt} + for key in qs: + params[key] = qs[key] + + return Request(url, method, params, headers, hope, body) + +if __name__ == '__main__': + admin = create_admin() + if admin is None: + print('Unable to parse/create admin account') + exit(1) + fake_user = bs_admin(admin.server) + + requests = [] + + # First a quick sanity check for login + # add this after we fire the generic tests + login_req, jwt = login() + if jwt is None: + print('Unable to /login - stopping now to avoid pointless failure') + exit(1) + + admin.jwt = jwt + + # add this after we fire the generic tests + mk_chan_sanity, chan_id = make_channel(admin.server.url, admin.id, admin.jwt) + mk_chan_to_delete, del_chan_id = make_channel(admin.server.url, admin.id, admin.jwt) + + # Container for most/all the generic requests we want to fire off + requests.extend([ + std_request(fake_user, 'get', '/channels/list', {}, 401), + std_request(admin, 'post', '/channels/list', {'kind': TEXT_CHAN}, 404), + std_request(admin, 'get' , '/channels/list', {'kind': TEXT_CHAN}, 200), + std_request(admin , 'delete', '/channels/delete', {'channel_id':del_chan_id}, 200), + std_request(admin, 'post', '/message/send', {'channel_id': chan_id},200,{'content-type':'text/plain'}, 'asdf'), + std_request(admin, 'post', '/message/send', {'channel_id': 123}, 400, {'content-type': 'text/plain'}, 'asdf'), + std_request(admin , 'post', '/message/send', {'channel_id': chan_id}, 200, {'content-type': 'image/png'}, 'asdf'), + std_request(admin , 'post', '/message/send', {'channel_id': 123}, 400, {'content-type': 'image/png'}, 'asdf'), + ]) + + for req in requests: + req.fire() + + # Prepend the sanity checks we did early on + requests.insert(0, login_req) + requests.insert(0, mk_chan_sanity) + requests.insert(0, mk_chan_to_delete) + for req in requests: + req.show_response() + + diff --git a/json-api/client-tests/request.py b/json-api/client-tests/request.py new file mode 100644 index 0000000..0602acf --- /dev/null +++ b/json-api/client-tests/request.py @@ -0,0 +1,59 @@ +from requests import Session +import requests + +NC = '\033[0m' +RED = '\033[1;31m' +GREEN = '\033[1;32m' + +class Request: + + def __init__(self, url: str, method: str, qs: dict, headers: dict, hope: int, body=None): + self.method = method + self.url = url + # query string parameters are appended to the url which is why we do this + self.qs = qs + self.headers = headers + self.body = body + + self.response = None + + self.hope = hope + + + def fire(self) -> requests.Response: + try: + if self.method == 'get': + self.response = requests.get(self.url, headers=self.headers, params=self.qs) + elif self.method == 'post': + self.response = requests.post(self.url, headers=self.headers, params=self.qs, data=self.body) + elif self.method == 'delete': + self.response = requests.delete(self.url, headers=self.headers, params=self.qs) + + return self.response + except: + return None + + + + def show_response(self): + if self.response is None: + print('Response := None') + return + + real_code = self.response.status_code + if self.hope != real_code: + abstract = RED + 'Fail ' + NC + 'Expected ' + str(self.hope) + ' Got ' + str(real_code) + + print(abstract) + print('\t', self.method, ' ', self.url) + if len(self.headers) != 0: + print('\tHeaders ', self.headers) + + if len(self.qs) != 0: + print('\tQuery Dictionary ', self.qs) + + if self.body is not None: + print('\tBody ', str(self.body)) + else: + print(f'{GREEN}Pass{NC} {self.method} {self.url}') + diff --git a/json-api/client-tests/web/http.py b/json-api/client-tests/web/http.py index 097445e..becf142 100644 --- a/json-api/client-tests/web/http.py +++ b/json-api/client-tests/web/http.py @@ -20,9 +20,11 @@ class Response: Response is wrapper for reading + extracting information we care about Primarily created by Requests that get `make`'d. ''' - def __init__(self, method: str, url: str, body: str, code: int, expected: int, out=sys.stdout, color=True, - truncate_long_body=True, p=None): + def __init__(self, method: str, url: str, body: str, code: int, expected: int, headers=None, + out=sys.stdout, color=True, truncate_long_body=True, p=None): + + self.headers = headers self.method = method self.url = url self.base_url = self.url[:self.url.find('?')] @@ -77,11 +79,13 @@ class Response: if self.code != self.expected: fail = f'Failed {self.method.upper()} {self.base_url}\t{self.code} expected {self.expected}\n' content = f'\tRaw params: {self.raw_params[0]}\n\tParsed Params: {self.raw_params[1]}' + headers = f'\tHeaders: {self.headers}' if self.color: fail = self._color_failing(fail) self.__write_msg(fail) self.__write_msg(content) + print(headers) self._log_body() else: msg = f'Passing: {self.method} {self.base_url}' @@ -112,11 +116,12 @@ class Response: return f'{self.code} => {self.body}' class Request: - def __init__(self, method: str, url: str, params: dict): + def __init__(self, method: str, url: str, params: dict, headers: dict): assert(method in ['get', 'post', 'delete']) self.method = method self.url = url + self.headers = headers if 'content' in params: self.body = params['content'] else: @@ -146,14 +151,15 @@ class Request: url = self.url + self.query_string raw_params = (self.query_string, self.qs_dict) if method == 'get': - resp = requests.get(url, verify=False) - return Response('get', url, resp.text, resp.status_code, hope, p=raw_params) + resp = requests.get(url, verify=False, headers=self.headers) + return Response('get', url, resp.text, resp.status_code, hope, p=raw_params, headers=self.headers) elif method == 'post': - resp = requests.post(url, data=self.body, verify=False) - return Response('post', url, resp.text, resp.status_code, hope, p=raw_params) + print(self.headers) + resp = requests.post(url, data=self.body, verify=False, headers=self.headers) + return Response('post', url, resp.text, resp.status_code, hope, p=raw_params, headers=self.headers) elif method == 'delete': - resp = requests.delete(url, verify=False) - return Response('delete', url, resp.text, resp.status_code, hope, p=raw_params) + resp = requests.delete(url, verify=False, headers=self.headers) + return Response('delete', url, resp.text, resp.status_code, hope, p=raw_params, headers=self.headers) else: raise RequestError('Invalid method passed')