! Massive test suite overhaul see details below !
Problem: the old test suite was extremely inflexible This meant that making new tests took way too much time. + This new rework makes the new client's backend much thinner and less "magical" With less magic going on we can pass way more data more easily to the actual http-request engine making the convenience wrapper over top it much more flexible Translating old tests to the new engine might take a while but for now the old client codebase is completely deprecated and will no longer be used+updated
This commit is contained in:
shockrah
parent
c4d7eb9111
commit
38ff0edd39
@ -1,6 +1,7 @@
|
||||
import time
|
||||
import subprocess, os, sys
|
||||
import json, requests
|
||||
from json import JSONDecodeError
|
||||
from web import http
|
||||
|
||||
|
||||
@ -31,9 +32,10 @@ class Worker:
|
||||
proc = subprocess.run(f'cargo run --release -- -c dev-test'.split(), text=True, capture_output=True)
|
||||
try:
|
||||
return json.loads(proc.stdout)
|
||||
except:
|
||||
except JSONDecodeError as err:
|
||||
import sys
|
||||
print('TESTBOT UNABLE TO LOAD JSON DATA FROM -c flag', file=sys.stderr)
|
||||
print(err)
|
||||
exit(1)
|
||||
|
||||
def auth(self, auth_opt: str):
|
||||
@ -69,13 +71,16 @@ class Worker:
|
||||
# 'body': <bool> # show response body or not (optional
|
||||
# }
|
||||
|
||||
# not 'try'ing these as misconfigurations are to be cleaned up before testing
|
||||
# not 'try'ing these as misconfigurations are completely fatal anyway
|
||||
method, path, opts = test['init']
|
||||
auth = test['auth']
|
||||
hope = test['hope']
|
||||
|
||||
if 'body' in test:
|
||||
self.request(method, path, auth, opts, hope, show_body=test['body'])
|
||||
# Only checking for headers if a body is present as we really only use
|
||||
# headers with the body
|
||||
head = test['headers'] if 'headers' in test else None
|
||||
self.request(method, path, auth, opts, hope, show_body=test['body'], headers=head)
|
||||
else:
|
||||
self.request(method, path, auth, opts, hope)
|
||||
|
||||
@ -108,7 +113,7 @@ class Worker:
|
||||
|
||||
|
||||
|
||||
def request(self, method: str, path: str, auth: str, opts: dict, expectation: int, show_body=False):
|
||||
def request(self, method: str, path: str, auth: str, opts: dict, expectation: int, show_body=False, headers={}):
|
||||
assert(path[0] == '/')
|
||||
|
||||
# First make sure we add in the correct auth params that are requested
|
||||
@ -116,7 +121,7 @@ class Worker:
|
||||
|
||||
# Build the request and store it in our structure
|
||||
url = self.domain + path
|
||||
req = http.Request(method, url, opts)
|
||||
req = http.Request(method, url, opts, headers=headers)
|
||||
r_id = time.time()
|
||||
|
||||
resp = req.make(expectation)
|
||||
@ -146,15 +151,12 @@ def run(worker: Worker):
|
||||
{'init': ['post', '/channels/list', {}], 'auth': jwt, 'hope': 404},
|
||||
|
||||
{'init': ['post', '/channels/create', {'name': str(new_channel_name), 'kind': TEXT_CHAN, 'description': 'asdf'}], 'auth': jwt, 'hope': 200},
|
||||
# Just a regular test no saving for this one
|
||||
{'init': ['post', '/channels/create', {'name': str(new_channel_name+1), 'kind': TEXT_CHAN,}], 'auth': jwt, 'hope': 200},
|
||||
|
||||
{'init': ['post', '/channels/create', {}], 'auth': jwt, 'hope': 400},
|
||||
{'init': ['post', '/channels/create', {'name': 123, 'kind': 'adsf'}], 'auth': jwt, 'hope': 400},
|
||||
|
||||
# save this and compare its results to the previous
|
||||
{'init': ['get', '/channels/list', {}], 'auth': jwt, 'hope': 200},
|
||||
{'init': ['get', '/channels/list', {'random-param': 123}], 'auth': jwt, 'hope': 200},
|
||||
{'init': ['get', '/channels/list', {'kind': TEXT_CHAN}], 'auth': jwt, 'hope': 200},
|
||||
]
|
||||
|
||||
for test in channel_tests:
|
||||
@ -171,10 +173,26 @@ def run(worker: Worker):
|
||||
|
||||
message_tests = [
|
||||
# bs message spam
|
||||
{'init': ['post', '/message/send', {'type': 'text', 'channel_id': chan_d['id'], 'content': 'bs content'}], 'auth': jwt, 'hope': 200, 'body': True},
|
||||
{'init': ['post', '/message/send', {'type': 'text', 'channel_id': chan_d['id'], 'content': 'bs content'}], 'auth': jwt, 'hope': 200},
|
||||
{'init': ['post', '/message/send', {'type': 'text', 'channel_id': chan_d['id'], 'content': 'bs content'}], 'auth': jwt, 'hope': 200},
|
||||
{'init': ['post', '/message/send', {'type': 'text', 'channel_id': chan_d['id'], 'content': 'bs content'}], 'auth': jwt, 'hope': 200},
|
||||
{
|
||||
'init': ['post', '/message/send', {'type': 'text', 'channel_id': chan_d['id'], 'content': 'bs content'}],
|
||||
'headers': {'content-type': 'text/plain'},
|
||||
'auth': jwt, 'hope': 200, 'body': False
|
||||
},
|
||||
{
|
||||
'init': ['post', '/message/send', {'type': 'text', 'channel_id': chan_d['id'], 'content': 'bs content'}],
|
||||
'headers': {'content-type': 'text/plain'},
|
||||
'auth': jwt, 'hope': 200
|
||||
},
|
||||
{
|
||||
'init': ['post', '/message/send', {'type': 'text', 'channel_id': chan_d['id'], 'content': 'bs content'}],
|
||||
'headers': {'content-type': 'text/plain'},
|
||||
'auth': jwt, 'hope': 200
|
||||
},
|
||||
{
|
||||
'init': ['post', '/message/send', {'type': 'text', 'channel_id': chan_d['id'], 'content': 'bs content'}],
|
||||
'headers': {'content-type': 'text/plain'},
|
||||
'auth': jwt, 'hope': 200
|
||||
},
|
||||
|
||||
# can we get them back tho?
|
||||
{
|
||||
@ -201,29 +219,6 @@ def run(worker: Worker):
|
||||
],
|
||||
'auth': jwt, 'hope': 400
|
||||
},
|
||||
# two tests that follow the rules
|
||||
{
|
||||
'init': [
|
||||
'get', '/message/from_id', {'start': 1, 'channel_id': 3}
|
||||
],
|
||||
'auth': jwt, 'hope': 200, 'body': True
|
||||
},
|
||||
{
|
||||
'init': [
|
||||
'get', '/message/from_id', {'start':1, 'channel_id':3, 'limit':2}
|
||||
],
|
||||
'auth': jwt, 'hope': 200, 'body': True
|
||||
},
|
||||
{
|
||||
# Channel doesn't exist so empty vector is result
|
||||
'init': [ 'get', '/message/from_id', {'start': 1, 'channel_id':9} ],
|
||||
'auth': jwt, 'hope': 404
|
||||
},
|
||||
{
|
||||
# Channel id doesn't refer to a real channel
|
||||
'init': [ 'get', '/message/from_id', {'start': 5, 'channel_id':3} ],
|
||||
'auth': jwt, 'hope': 404
|
||||
},
|
||||
]
|
||||
|
||||
for test in message_tests:
|
||||
|
||||
42
json-api/client-tests/config.py
Normal file
42
json-api/client-tests/config.py
Normal file
@ -0,0 +1,42 @@
|
||||
import subprocess
|
||||
import os
|
||||
import json
|
||||
|
||||
class Server:
|
||||
def __init__(self, meta: dict):
|
||||
self.url = meta.get('url')
|
||||
self.wsurl = meta.get('wsurl')
|
||||
self.serv_name = meta.get('name')
|
||||
|
||||
class Admin:
|
||||
def __init__(self, user: dict, server: dict):
|
||||
self.id = user.get('id')
|
||||
self.name = user.get('name')
|
||||
self.permissions = user.get('permissions')
|
||||
self.secret = user.get('secret')
|
||||
self.status = user.get('status')
|
||||
self.jwt = None
|
||||
|
||||
self.server = Server(server)
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f'{self.name}#{self.id}'
|
||||
|
||||
def create_admin() -> Admin :
|
||||
CARGO_BIN = os.getenv('CARGO_BIN')
|
||||
|
||||
proc = subprocess.run(
|
||||
f'cargo run --release -- -c python-tester'.split(),
|
||||
text=True, capture_output=True
|
||||
)
|
||||
try:
|
||||
raw = json.loads(proc.stdout)
|
||||
user = raw.get('user')
|
||||
server = raw.get('server')
|
||||
if user is None or server is None:
|
||||
return None
|
||||
else:
|
||||
return Admin(user, server)
|
||||
except:
|
||||
return None
|
||||
|
||||
104
json-api/client-tests/main.py
Normal file
104
json-api/client-tests/main.py
Normal file
@ -0,0 +1,104 @@
|
||||
from time import time
|
||||
from request import Request
|
||||
from config import create_admin, Admin
|
||||
from config import Server
|
||||
|
||||
RESPONSES = []
|
||||
VOICE_CHAN = 1
|
||||
TEXT_CHAN = 2
|
||||
|
||||
def login() -> (Request, str):
|
||||
req = Request(
|
||||
admin.server.url + '/login',
|
||||
'post',
|
||||
{'id':admin.id, 'secret': admin.secret},
|
||||
{},
|
||||
200
|
||||
)
|
||||
response = req.fire()
|
||||
jwt = response.json().get('jwt')
|
||||
return (req, jwt)
|
||||
|
||||
def make_channel(url: str, id: int, jwt: str) -> (Request, int):
|
||||
# making a text channel
|
||||
channel_name = str(time())
|
||||
qs = {'id': id, 'jwt': jwt, 'kind': 2, 'name': channel_name}
|
||||
req = Request(url + '/channels/create', 'post', qs, {}, 200)
|
||||
resp = req.fire()
|
||||
try:
|
||||
channel = resp.json().get('channel')
|
||||
chan_id = 0 if channel is None else channel.get('id')
|
||||
return (req, chan_id)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
print(f'Actual value: {resp.text}: {resp.status_code}')
|
||||
return (req, 0)
|
||||
|
||||
def bs_admin(server: Server) -> Admin:
|
||||
user_ = {
|
||||
'id': 123,
|
||||
'jwt': None,
|
||||
'secret': 'no thanks'
|
||||
}
|
||||
server_ = {
|
||||
'url': server.url,
|
||||
'wsurl': server.wsurl,
|
||||
'name': server.serv_name
|
||||
}
|
||||
return Admin(user_, server_)
|
||||
|
||||
|
||||
def std_request(admin: Admin, method: str, path: str, qs: dict, hope: int, headers: dict={}, body=None) -> Request:
|
||||
url = admin.server.url + path
|
||||
# Copy over the additional params into the query string
|
||||
params = {'id': admin.id, 'jwt': admin.jwt}
|
||||
for key in qs:
|
||||
params[key] = qs[key]
|
||||
|
||||
return Request(url, method, params, headers, hope, body)
|
||||
|
||||
if __name__ == '__main__':
|
||||
admin = create_admin()
|
||||
if admin is None:
|
||||
print('Unable to parse/create admin account')
|
||||
exit(1)
|
||||
fake_user = bs_admin(admin.server)
|
||||
|
||||
requests = []
|
||||
|
||||
# First a quick sanity check for login
|
||||
# add this after we fire the generic tests
|
||||
login_req, jwt = login()
|
||||
if jwt is None:
|
||||
print('Unable to /login - stopping now to avoid pointless failure')
|
||||
exit(1)
|
||||
|
||||
admin.jwt = jwt
|
||||
|
||||
# add this after we fire the generic tests
|
||||
mk_chan_sanity, chan_id = make_channel(admin.server.url, admin.id, admin.jwt)
|
||||
mk_chan_to_delete, del_chan_id = make_channel(admin.server.url, admin.id, admin.jwt)
|
||||
|
||||
# Container for most/all the generic requests we want to fire off
|
||||
requests.extend([
|
||||
std_request(fake_user, 'get', '/channels/list', {}, 401),
|
||||
std_request(admin, 'post', '/channels/list', {'kind': TEXT_CHAN}, 404),
|
||||
std_request(admin, 'get' , '/channels/list', {'kind': TEXT_CHAN}, 200),
|
||||
std_request(admin , 'delete', '/channels/delete', {'channel_id':del_chan_id}, 200),
|
||||
std_request(admin, 'post', '/message/send', {'channel_id': chan_id},200,{'content-type':'text/plain'}, 'asdf'),
|
||||
std_request(admin, 'post', '/message/send', {'channel_id': 123}, 400, {'content-type': 'text/plain'}, 'asdf'),
|
||||
std_request(admin , 'post', '/message/send', {'channel_id': chan_id}, 200, {'content-type': 'image/png'}, 'asdf'),
|
||||
std_request(admin , 'post', '/message/send', {'channel_id': 123}, 400, {'content-type': 'image/png'}, 'asdf'),
|
||||
])
|
||||
|
||||
for req in requests:
|
||||
req.fire()
|
||||
|
||||
# Prepend the sanity checks we did early on
|
||||
requests.insert(0, login_req)
|
||||
requests.insert(0, mk_chan_sanity)
|
||||
requests.insert(0, mk_chan_to_delete)
|
||||
for req in requests:
|
||||
req.show_response()
|
||||
|
||||
|
||||
59
json-api/client-tests/request.py
Normal file
59
json-api/client-tests/request.py
Normal file
@ -0,0 +1,59 @@
|
||||
from requests import Session
|
||||
import requests
|
||||
|
||||
NC = '\033[0m'
|
||||
RED = '\033[1;31m'
|
||||
GREEN = '\033[1;32m'
|
||||
|
||||
class Request:
|
||||
|
||||
def __init__(self, url: str, method: str, qs: dict, headers: dict, hope: int, body=None):
|
||||
self.method = method
|
||||
self.url = url
|
||||
# query string parameters are appended to the url which is why we do this
|
||||
self.qs = qs
|
||||
self.headers = headers
|
||||
self.body = body
|
||||
|
||||
self.response = None
|
||||
|
||||
self.hope = hope
|
||||
|
||||
|
||||
def fire(self) -> requests.Response:
|
||||
try:
|
||||
if self.method == 'get':
|
||||
self.response = requests.get(self.url, headers=self.headers, params=self.qs)
|
||||
elif self.method == 'post':
|
||||
self.response = requests.post(self.url, headers=self.headers, params=self.qs, data=self.body)
|
||||
elif self.method == 'delete':
|
||||
self.response = requests.delete(self.url, headers=self.headers, params=self.qs)
|
||||
|
||||
return self.response
|
||||
except:
|
||||
return None
|
||||
|
||||
|
||||
|
||||
def show_response(self):
|
||||
if self.response is None:
|
||||
print('Response := None')
|
||||
return
|
||||
|
||||
real_code = self.response.status_code
|
||||
if self.hope != real_code:
|
||||
abstract = RED + 'Fail ' + NC + 'Expected ' + str(self.hope) + ' Got ' + str(real_code)
|
||||
|
||||
print(abstract)
|
||||
print('\t', self.method, ' ', self.url)
|
||||
if len(self.headers) != 0:
|
||||
print('\tHeaders ', self.headers)
|
||||
|
||||
if len(self.qs) != 0:
|
||||
print('\tQuery Dictionary ', self.qs)
|
||||
|
||||
if self.body is not None:
|
||||
print('\tBody ', str(self.body))
|
||||
else:
|
||||
print(f'{GREEN}Pass{NC} {self.method} {self.url}')
|
||||
|
||||
@ -20,9 +20,11 @@ class Response:
|
||||
Response is wrapper for reading + extracting information we care about
|
||||
Primarily created by Requests that get `make`'d.
|
||||
'''
|
||||
def __init__(self, method: str, url: str, body: str, code: int, expected: int, out=sys.stdout, color=True,
|
||||
truncate_long_body=True, p=None):
|
||||
def __init__(self, method: str, url: str, body: str, code: int, expected: int, headers=None,
|
||||
out=sys.stdout, color=True, truncate_long_body=True, p=None):
|
||||
|
||||
|
||||
self.headers = headers
|
||||
self.method = method
|
||||
self.url = url
|
||||
self.base_url = self.url[:self.url.find('?')]
|
||||
@ -77,11 +79,13 @@ class Response:
|
||||
if self.code != self.expected:
|
||||
fail = f'Failed {self.method.upper()} {self.base_url}\t{self.code} expected {self.expected}\n'
|
||||
content = f'\tRaw params: {self.raw_params[0]}\n\tParsed Params: {self.raw_params[1]}'
|
||||
headers = f'\tHeaders: {self.headers}'
|
||||
if self.color:
|
||||
fail = self._color_failing(fail)
|
||||
|
||||
self.__write_msg(fail)
|
||||
self.__write_msg(content)
|
||||
print(headers)
|
||||
self._log_body()
|
||||
else:
|
||||
msg = f'Passing: {self.method} {self.base_url}'
|
||||
@ -112,11 +116,12 @@ class Response:
|
||||
return f'{self.code} => {self.body}'
|
||||
|
||||
class Request:
|
||||
def __init__(self, method: str, url: str, params: dict):
|
||||
def __init__(self, method: str, url: str, params: dict, headers: dict):
|
||||
assert(method in ['get', 'post', 'delete'])
|
||||
|
||||
self.method = method
|
||||
self.url = url
|
||||
self.headers = headers
|
||||
if 'content' in params:
|
||||
self.body = params['content']
|
||||
else:
|
||||
@ -146,14 +151,15 @@ class Request:
|
||||
url = self.url + self.query_string
|
||||
raw_params = (self.query_string, self.qs_dict)
|
||||
if method == 'get':
|
||||
resp = requests.get(url, verify=False)
|
||||
return Response('get', url, resp.text, resp.status_code, hope, p=raw_params)
|
||||
resp = requests.get(url, verify=False, headers=self.headers)
|
||||
return Response('get', url, resp.text, resp.status_code, hope, p=raw_params, headers=self.headers)
|
||||
elif method == 'post':
|
||||
resp = requests.post(url, data=self.body, verify=False)
|
||||
return Response('post', url, resp.text, resp.status_code, hope, p=raw_params)
|
||||
print(self.headers)
|
||||
resp = requests.post(url, data=self.body, verify=False, headers=self.headers)
|
||||
return Response('post', url, resp.text, resp.status_code, hope, p=raw_params, headers=self.headers)
|
||||
elif method == 'delete':
|
||||
resp = requests.delete(url, verify=False)
|
||||
return Response('delete', url, resp.text, resp.status_code, hope, p=raw_params)
|
||||
resp = requests.delete(url, verify=False, headers=self.headers)
|
||||
return Response('delete', url, resp.text, resp.status_code, hope, p=raw_params, headers=self.headers)
|
||||
|
||||
else:
|
||||
raise RequestError('Invalid method passed')
|
||||
|
||||
Loading…
Reference in New Issue
Block a user