! Massive test suite overhaul see details below !

Problem: the old test suite was extremely inflexible
This meant that making new tests took way too much time.

+ This new rework makes the new client's backend much thinner and less "magical"
With less magic going on we can pass way more data more easily to the actual
http-request engine making the convenience wrapper over top it much more flexible

Translating old tests to the new engine might take a while but for now the old
client codebase is completely deprecated and will no longer be used+updated

json-api/client-tests/client.py

@@ -1,6 +1,7 @@
 import time
 import subprocess, os, sys
 import json, requests
+from json import JSONDecodeError
 from web import http
 
 
@@ -31,9 +32,10 @@ class Worker:
         proc = subprocess.run(f'cargo run --release -- -c dev-test'.split(), text=True, capture_output=True)
         try: 
             return json.loads(proc.stdout)
-        except:
+        except JSONDecodeError as err:
             import sys
             print('TESTBOT UNABLE TO LOAD JSON DATA FROM -c flag', file=sys.stderr)
+            print(err)
             exit(1)
 
     def auth(self, auth_opt: str):
@@ -69,13 +71,16 @@ class Worker:
         #   'body': <bool> # show response body or not (optional
         # }
 
-        # not 'try'ing these as misconfigurations are to be cleaned up before testing
+        # not 'try'ing these as misconfigurations are completely fatal anyway
         method, path, opts = test['init']
         auth = test['auth']
         hope = test['hope']
 
         if 'body' in test:
-            self.request(method, path, auth, opts, hope, show_body=test['body'])
+            # Only checking for headers if a body is present as we really only use 
+            # headers with the body
+            head = test['headers'] if 'headers' in test else None
+            self.request(method, path, auth, opts, hope, show_body=test['body'], headers=head)
         else:
             self.request(method, path, auth, opts, hope)
 
@@ -108,7 +113,7 @@ class Worker:
 
 
 
-    def request(self, method: str, path: str, auth: str, opts: dict, expectation: int, show_body=False):
+    def request(self, method: str, path: str, auth: str, opts: dict, expectation: int, show_body=False, headers={}):
         assert(path[0] ==  '/')
 
         # First make sure we add in the correct auth params that are requested
@@ -116,7 +121,7 @@ class Worker:
 
         # Build the request and store it in our structure
         url = self.domain + path
-        req = http.Request(method, url, opts)
+        req = http.Request(method, url, opts, headers=headers)
         r_id = time.time()
 
         resp = req.make(expectation)
@@ -146,15 +151,12 @@ def run(worker: Worker):
         {'init': ['post', '/channels/list', {}], 'auth': jwt, 'hope': 404},
 
         {'init': ['post', '/channels/create', {'name': str(new_channel_name), 'kind': TEXT_CHAN, 'description': 'asdf'}], 'auth': jwt, 'hope': 200},
-        # Just a regular test no saving for this one
         {'init': ['post', '/channels/create', {'name': str(new_channel_name+1), 'kind': TEXT_CHAN,}], 'auth': jwt, 'hope': 200},
 
         {'init': ['post', '/channels/create', {}], 'auth': jwt, 'hope': 400},
         {'init': ['post', '/channels/create', {'name': 123, 'kind': 'adsf'}], 'auth': jwt, 'hope': 400},
 
-        # save this and compare its results to the previous
+        {'init': ['get', '/channels/list', {'kind': TEXT_CHAN}], 'auth': jwt, 'hope': 200},
-        {'init': ['get', '/channels/list', {}], 'auth': jwt, 'hope': 200},
-        {'init': ['get', '/channels/list', {'random-param': 123}], 'auth': jwt, 'hope': 200},
     ]
 
     for test in channel_tests:
@@ -171,10 +173,26 @@ def run(worker: Worker):
 
     message_tests = [
         # bs message spam
-        {'init': ['post', '/message/send', {'type': 'text', 'channel_id': chan_d['id'], 'content': 'bs content'}], 'auth': jwt, 'hope': 200, 'body': True},
+        {
-        {'init': ['post', '/message/send', {'type': 'text', 'channel_id': chan_d['id'], 'content': 'bs content'}], 'auth': jwt, 'hope': 200},
+            'init': ['post', '/message/send', {'type': 'text', 'channel_id': chan_d['id'], 'content': 'bs content'}], 
-        {'init': ['post', '/message/send', {'type': 'text', 'channel_id': chan_d['id'], 'content': 'bs content'}], 'auth': jwt, 'hope': 200},
+            'headers': {'content-type': 'text/plain'},
-        {'init': ['post', '/message/send', {'type': 'text', 'channel_id': chan_d['id'], 'content': 'bs content'}], 'auth': jwt, 'hope': 200},
+            'auth': jwt, 'hope': 200, 'body': False
+        },
+        {
+            'init': ['post', '/message/send', {'type': 'text', 'channel_id': chan_d['id'], 'content': 'bs content'}], 
+            'headers': {'content-type': 'text/plain'},
+            'auth': jwt, 'hope': 200
+        },
+        {
+            'init': ['post', '/message/send', {'type': 'text', 'channel_id': chan_d['id'], 'content': 'bs content'}], 
+            'headers': {'content-type': 'text/plain'},
+            'auth': jwt, 'hope': 200
+        },
+        {
+            'init': ['post', '/message/send', {'type': 'text', 'channel_id': chan_d['id'], 'content': 'bs content'}], 
+            'headers': {'content-type': 'text/plain'},
+            'auth': jwt, 'hope': 200
+         },
 
         # can we get them back tho?
         {
@@ -201,29 +219,6 @@ def run(worker: Worker):
             ], 
             'auth': jwt, 'hope': 400
         },
-        # two tests that follow the rules
-        {
-            'init': [
-                'get', '/message/from_id', {'start': 1, 'channel_id': 3}
-            ],
-            'auth': jwt, 'hope': 200, 'body': True
-        },
-        {
-            'init': [
-                'get', '/message/from_id', {'start':1, 'channel_id':3, 'limit':2}
-            ],
-            'auth': jwt, 'hope': 200, 'body': True
-        },
-        {
-            # Channel doesn't exist so empty vector is result
-            'init': [ 'get', '/message/from_id', {'start': 1, 'channel_id':9} ],
-            'auth': jwt, 'hope': 404
-        },
-        {
-             # Channel id doesn't refer to a real channel
-            'init': [ 'get', '/message/from_id', {'start': 5, 'channel_id':3} ],
-            'auth': jwt, 'hope': 404
-        },
     ]
 
     for test in message_tests:

json-api/client-tests/config.py

@@ -0,0 +1,42 @@
+import subprocess
+import os
+import json
+
+class Server:
+    def __init__(self, meta: dict):
+        self.url = meta.get('url')
+        self.wsurl = meta.get('wsurl')
+        self.serv_name = meta.get('name')
+
+class Admin:
+    def __init__(self, user: dict, server: dict):
+        self.id = user.get('id')
+        self.name = user.get('name')
+        self.permissions = user.get('permissions')
+        self.secret = user.get('secret')
+        self.status = user.get('status')
+        self.jwt = None
+
+        self.server = Server(server)
+
+    def __str__(self) -> str:
+        return f'{self.name}#{self.id}'
+
+def create_admin() -> Admin :
+    CARGO_BIN = os.getenv('CARGO_BIN')
+
+    proc = subprocess.run(
+        f'cargo run --release -- -c python-tester'.split(),
+        text=True, capture_output=True
+    )
+    try:
+        raw = json.loads(proc.stdout)
+        user = raw.get('user')
+        server = raw.get('server')
+        if user is None or server is None:
+            return None
+        else:
+            return Admin(user, server)
+    except:
+        return None
+

json-api/client-tests/main.py

@@ -0,0 +1,104 @@
+from time import time
+from request import Request
+from config import create_admin, Admin
+from config import Server
+
+RESPONSES = []
+VOICE_CHAN = 1
+TEXT_CHAN = 2
+
+def login() -> (Request, str):
+    req = Request(
+        admin.server.url + '/login', 
+        'post', 
+        {'id':admin.id, 'secret': admin.secret},
+        {},
+        200
+    )
+    response = req.fire()
+    jwt = response.json().get('jwt')
+    return (req, jwt)
+
+def make_channel(url: str, id: int, jwt: str) -> (Request, int):
+    # making a text channel
+    channel_name = str(time())
+    qs = {'id': id, 'jwt': jwt, 'kind': 2, 'name': channel_name} 
+    req = Request(url + '/channels/create', 'post', qs, {}, 200)
+    resp = req.fire()
+    try:
+        channel = resp.json().get('channel')
+        chan_id = 0 if channel is None else channel.get('id')
+        return (req, chan_id)
+    except Exception as e:
+        print(e)
+        print(f'Actual value: {resp.text}: {resp.status_code}')
+        return (req, 0)
+
+def bs_admin(server: Server) -> Admin:
+    user_ =  {
+        'id': 123,
+        'jwt': None,
+        'secret': 'no thanks'
+    }
+    server_ = {
+        'url': server.url,
+        'wsurl': server.wsurl,
+        'name': server.serv_name
+    }
+    return Admin(user_, server_)
+
+
+def std_request(admin: Admin, method: str, path: str, qs: dict, hope: int, headers: dict={}, body=None) -> Request:
+    url = admin.server.url + path
+    # Copy over the additional params into the query string
+    params = {'id': admin.id, 'jwt': admin.jwt}
+    for key in qs:
+        params[key] = qs[key]
+
+    return Request(url, method, params, headers, hope, body)
+
+if __name__ == '__main__':
+    admin = create_admin()
+    if admin is None:
+        print('Unable to parse/create admin account')
+        exit(1)
+    fake_user = bs_admin(admin.server)
+
+    requests = []
+
+    # First a quick sanity check for login
+    # add this after we fire the generic tests
+    login_req, jwt = login()
+    if jwt is None:
+        print('Unable to /login - stopping now to avoid pointless failure')
+        exit(1)
+
+    admin.jwt = jwt
+
+    # add this after we fire the generic tests
+    mk_chan_sanity, chan_id = make_channel(admin.server.url, admin.id, admin.jwt)
+    mk_chan_to_delete, del_chan_id = make_channel(admin.server.url, admin.id, admin.jwt)
+
+    # Container for most/all the generic requests we want to fire off
+    requests.extend([
+        std_request(fake_user, 'get', '/channels/list', {}, 401),
+        std_request(admin, 'post', '/channels/list', {'kind': TEXT_CHAN}, 404),
+        std_request(admin, 'get' , '/channels/list', {'kind': TEXT_CHAN}, 200),
+        std_request(admin , 'delete', '/channels/delete', {'channel_id':del_chan_id}, 200),
+        std_request(admin, 'post', '/message/send', {'channel_id': chan_id},200,{'content-type':'text/plain'}, 'asdf'),
+        std_request(admin, 'post', '/message/send', {'channel_id': 123}, 400, {'content-type': 'text/plain'}, 'asdf'),
+        std_request(admin , 'post', '/message/send', {'channel_id': chan_id}, 200, {'content-type': 'image/png'}, 'asdf'),
+        std_request(admin , 'post', '/message/send', {'channel_id': 123}, 400, {'content-type': 'image/png'}, 'asdf'),
+    ])
+
+    for req in requests:
+        req.fire()
+
+    # Prepend the sanity checks we did early on
+    requests.insert(0, login_req)
+    requests.insert(0, mk_chan_sanity)
+    requests.insert(0, mk_chan_to_delete)
+    for req in requests:
+        req.show_response()
+        
+

json-api/client-tests/request.py

@@ -0,0 +1,59 @@
+from requests import Session
+import requests
+
+NC = '\033[0m'
+RED = '\033[1;31m'
+GREEN = '\033[1;32m'
+
+class Request:
+
+    def __init__(self, url: str, method: str, qs: dict, headers: dict, hope: int, body=None):
+        self.method = method
+        self.url = url
+        # query string parameters are appended to the url which is why  we do this
+        self.qs = qs
+        self.headers = headers
+        self.body = body
+
+        self.response = None
+
+        self.hope = hope
+
+
+    def fire(self) -> requests.Response:
+        try:
+            if self.method == 'get':
+                self.response = requests.get(self.url, headers=self.headers, params=self.qs)
+            elif self.method == 'post':
+                self.response = requests.post(self.url, headers=self.headers, params=self.qs, data=self.body)
+            elif self.method == 'delete':
+                self.response = requests.delete(self.url, headers=self.headers, params=self.qs)
+            
+            return self.response
+        except:
+            return None
+
+        
+
+    def show_response(self):
+        if self.response is None:
+            print('Response := None')
+            return 
+
+        real_code = self.response.status_code
+        if self.hope != real_code:
+            abstract = RED + 'Fail ' + NC + 'Expected ' + str(self.hope) + ' Got ' + str(real_code)
+
+            print(abstract)
+            print('\t', self.method, ' ', self.url)
+            if len(self.headers) != 0:
+                print('\tHeaders ', self.headers)
+
+            if len(self.qs) != 0:
+                print('\tQuery Dictionary ', self.qs)
+
+            if self.body is not None:
+                print('\tBody ', str(self.body))
+        else:
+            print(f'{GREEN}Pass{NC} {self.method} {self.url}')
+

json-api/client-tests/web/http.py

@@ -20,9 +20,11 @@ class Response:
     Response is wrapper for reading + extracting information we care about 
     Primarily created by Requests that get `make`'d.
     '''
-    def __init__(self, method: str, url: str, body: str, code: int, expected: int, out=sys.stdout, color=True,
+    def __init__(self, method: str, url: str, body: str, code: int, expected: int, headers=None,
-                 truncate_long_body=True, p=None):
+                 out=sys.stdout, color=True, truncate_long_body=True, p=None):
 
+
+        self.headers = headers
         self.method = method
         self.url = url
         self.base_url = self.url[:self.url.find('?')]
@@ -77,11 +79,13 @@ class Response:
         if self.code != self.expected:
             fail = f'Failed {self.method.upper()} {self.base_url}\t{self.code} expected {self.expected}\n'
             content = f'\tRaw params: {self.raw_params[0]}\n\tParsed Params: {self.raw_params[1]}'
+            headers = f'\tHeaders: {self.headers}'
             if self.color:
                 fail = self._color_failing(fail)
 
             self.__write_msg(fail)
             self.__write_msg(content)
+            print(headers)
             self._log_body()
         else:
             msg = f'Passing: {self.method} {self.base_url}'
@@ -112,11 +116,12 @@ class Response:
         return f'{self.code} => {self.body}'
 
 class Request: 
-    def __init__(self, method: str, url: str, params: dict):
+    def __init__(self, method: str, url: str, params: dict, headers: dict):
         assert(method in ['get', 'post', 'delete'])
 
         self.method = method
         self.url    = url
+        self.headers = headers
         if 'content' in params:
             self.body = params['content']
         else:
@@ -146,14 +151,15 @@ class Request:
         url = self.url + self.query_string
         raw_params = (self.query_string, self.qs_dict)
         if method == 'get':
-            resp = requests.get(url, verify=False)
+            resp = requests.get(url, verify=False, headers=self.headers)
-            return Response('get', url, resp.text, resp.status_code, hope, p=raw_params)
+            return Response('get', url, resp.text, resp.status_code, hope, p=raw_params, headers=self.headers)
         elif method == 'post':
-            resp = requests.post(url, data=self.body, verify=False)
+            print(self.headers)
-            return Response('post', url, resp.text, resp.status_code, hope, p=raw_params)
+            resp = requests.post(url, data=self.body, verify=False, headers=self.headers)
+            return Response('post', url, resp.text, resp.status_code, hope, p=raw_params, headers=self.headers)
         elif method == 'delete':
-            resp = requests.delete(url, verify=False)
+            resp = requests.delete(url, verify=False, headers=self.headers)
-            return Response('delete', url, resp.text, resp.status_code, hope, p=raw_params)
+            return Response('delete', url, resp.text, resp.status_code, hope, p=raw_params, headers=self.headers)
 
         else:
             raise RequestError('Invalid method passed')