This is an automated email from the ASF dual-hosted git repository. dragon pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/trafficserver.git
The following commit(s) were added to refs/heads/master by this push: new 42b896e Tests wait until microserver is ready 42b896e is described below commit 42b896e810b892f92d5a179c96a6f9ee3826db4e Author: Derek Dagit <der...@oath.com> AuthorDate: Wed Apr 4 15:48:29 2018 +0000 Tests wait until microserver is ready --- tests/gold_tests/autest-site/microserver.test.ext | 71 +++++++---- tests/gold_tests/continuations/double.test.py | 4 +- tests/gold_tests/transaction/txn.test.py | 6 +- tests/tools/microServer/README.md | 48 +++++--- tests/tools/microServer/uWServer.py | 130 +++++++++------------ tests/tools/sessionvalidation/response.py | 15 ++- tests/tools/sessionvalidation/sessionvalidation.py | 10 +- 7 files changed, 159 insertions(+), 125 deletions(-) diff --git a/tests/gold_tests/autest-site/microserver.test.ext b/tests/gold_tests/autest-site/microserver.test.ext index f4a2685..a52e9b8 100644 --- a/tests/gold_tests/autest-site/microserver.test.ext +++ b/tests/gold_tests/autest-site/microserver.test.ext @@ -16,8 +16,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +from autest.api import AddWhenFunction from ports import get_port import json +import socket +import ssl +import time def addMethod(self, testName, request_header, functionName): @@ -33,22 +37,6 @@ def httpObject(self, header, data): r["body"] = data return r -# addResponse adds customized response with respect to request_header. request_header and response_header are both dictionaries - - -def addResponse(self, filename, testName, request_header, response_header): - - txn = dict() - txn["timestamp"] = "" - txn["uuid"] = testName - txn["request"] = request_header - txn["response"] = response_header - - addTransactionToSession(txn, filename) - absFilepath = os.path.abspath(filename) - self.Setup.CopyAs(absFilepath, self.Variables.DataDir) - return - def getHeaderFieldVal(request_header, field): requestline = request_header["headers"].split("\r\n")[0] @@ -79,7 +67,6 @@ def addResponse(self, filename, request_header, response_header): path_ = url_part[1].split("/", 1)[1] kpath = "" - #print("Format of lookup key",self.Variables.lookup_key) argsList = [] keyslist = self.Variables.lookup_key.split("}") @@ -124,7 +111,7 @@ def addTransactionToSession(txn, JFile): if jsondata == None: jsondata = dict() - jsondata["version"] = '0.1' + jsondata["version"] = '0.2' jsondata["timestamp"] = "1234567890.098" jsondata["encoding"] = "url_encoded" jsondata["txns"] = list() @@ -144,6 +131,36 @@ def makeHeader(self, requestString, **kwargs): return headerStr +def uServerUpAndRunning(host, port, isSsl): + plain_sock = socket.socket(socket.AF_INET) + sock = ssl.wrap_socket(plain_sock) if isSsl else plain_sock + try: + sock.connect((host, port)) + except ConnectionRefusedError: + return False + + sock.sendall("GET /ruok HTTP/1.1\r\nHost: {}\r\n\r\n".format(host).encode()) + decoded_output='' + while True: + output = sock.recv(4096) # suggested bufsize from docs.python.org + if len(output) <= 0: + break + else: + decoded_output+=output.decode() + sock.close() + sock = None + + expected_response="HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Length: 4\r\n\r\nimok" + if decoded_output == expected_response: + return True + raise RuntimeError('\n'.join([ + 'Got invalid response from microserver:', + '----', + decoded_output, + '----'])) +AddWhenFunction(uServerUpAndRunning) + + def MakeOriginServer(obj, name, port=False, ip=False, delay=False, ssl=False, lookup_key='{PATH}', mode='test', options={}): server_path = os.path.join(obj.Variables.AtsTestToolsDir, 'microServer/uWServer.py') data_dir = os.path.join(obj.RunDirectory, name) @@ -160,16 +177,28 @@ def MakeOriginServer(obj, name, port=False, ip=False, delay=False, ssl=False, lo for flag, value in options.items(): command += " {} {}".format(flag, value) - # create process p.Command = command p.Setup.MakeDir(data_dir) p.Variables.DataDir = data_dir p.Variables.lookup_key = lookup_key - p.Ready = When.PortOpen(port, ip) - p.ReturnCode = Any(None, 0) AddMethodToInstance(p, addResponse) AddMethodToInstance(p, addTransactionToSession) + # Set up health check. + addResponse(p, "healthcheck.json", { + "headers": "GET /ruok HTTP/1.1\r\nHost: {}\r\n\r\n".format(ip), + "timestamp": "1469733493.993", + "body": "" + }, { + "headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", + "timestamp": "1469733493.993", + "body": "imok", + "options": "skipHooks" + }) + + p.Ready = When.uServerUpAndRunning(ip, port, ssl) + p.ReturnCode = Any(None, 0) + return p diff --git a/tests/gold_tests/continuations/double.test.py b/tests/gold_tests/continuations/double.test.py index 6b95134..fccd584 100644 --- a/tests/gold_tests/continuations/double.test.py +++ b/tests/gold_tests/continuations/double.test.py @@ -32,7 +32,7 @@ ts = Test.MakeATSProcess("ts", command="traffic_manager") server = Test.MakeOriginServer("server") Test.testName = "" -request_header = {"headers": "GET / HTTP/1.1\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +request_header = {"headers": "GET / HTTP/1.1\r\nHost: double.test\r\n\r\n", "timestamp": "1469733493.993", "body": ""} # expected response from the origin server response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} @@ -47,7 +47,7 @@ ts.Disk.records_config.update({ 'proxy.config.cache.enable_read_while_writer' : 0 }) ts.Disk.remap_config.AddLine( - 'map http://127.0.0.1:{0} http://127.0.0.1:{1}'.format(ts.Variables.port, server.Variables.Port) + 'map http://double.test:{0} http://127.0.0.1:{1}'.format(ts.Variables.port, server.Variables.Port) ) numberOfRequests = randint(1000, 1500) diff --git a/tests/gold_tests/transaction/txn.test.py b/tests/gold_tests/transaction/txn.test.py index b285409..6f78fc2 100644 --- a/tests/gold_tests/transaction/txn.test.py +++ b/tests/gold_tests/transaction/txn.test.py @@ -33,7 +33,7 @@ ts = Test.MakeATSProcess("ts", command="traffic_manager") server = Test.MakeOriginServer("server") Test.testName = "" -request_header = {"headers": "GET / HTTP/1.1\r\n\r\n", +request_header = {"headers": "GET / HTTP/1.1\r\nHost: txn.test\r\n\r\n", "timestamp": "1469733493.993", "body": ""} # expected response from the origin server response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", @@ -52,7 +52,7 @@ ts.Disk.records_config.update({ }) ts.Disk.remap_config.AddLine( - 'map http://127.0.0.1:{0} http://127.0.0.1:{1}'.format( + 'map http://txn.test:{0} http://127.0.0.1:{1}'.format( ts.Variables.port, server.Variables.Port) ) @@ -60,7 +60,7 @@ numberOfRequests = randint(1000, 1500) # Make a *ton* of calls to the proxy! tr = Test.AddTestRun() -tr.Processes.Default.Command = 'ab -n {0} -c 10 http://127.0.0.1:{1}/;sleep 5'.format( +tr.Processes.Default.Command = 'ab -n {0} -c 10 -X 127.0.0.1:{1} http://txn.test/;sleep 5'.format( numberOfRequests, ts.Variables.port) tr.Processes.Default.ReturnCode = 0 # time delay as proxy.config.http.wait_for_cache could be broken diff --git a/tests/tools/microServer/README.md b/tests/tools/microServer/README.md index 0a8fa65..a20ea85 100644 --- a/tests/tools/microServer/README.md +++ b/tests/tools/microServer/README.md @@ -4,20 +4,7 @@ uWServer uWServer is a mock HTTP server that takes predefined set of sessions for serving response to HTTP requests. Each session includes one or more transactions. A transaction is composed of an HTTP request and an HTTP response. uWServer accepts session data in JSON fromat only. -Example session : -``` - {"version": "0.1", - "txns": [ - {"request": {"headers": "GET /path1\r\n Host: example.com \r\n\r\n", "timestamp": "1522783378", "body": "Apache Traffic Server"}, - "response": {"headers": "HTTP/1.1\r\n Server: microserver\r\nContent-Length:100 \r\n\r\n", "timestamp": "1522783378", "body": ""}, - "uuid": "1"}, - {"request": {"headers": "GET /path2\r\n\r\n", "timestamp": "1522783378", "body": "Apache Traffic Server"}, - "response": {"headers": "HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n", "timestamp": "1522783378", "body": "Apache Traffic Server"}, - "uuid": "2"} - ], - "timestamp": "1522783378", - "encoding": ""} -``` + Command: ---------------- @@ -26,6 +13,37 @@ Command: Options: ----------- -To see the options please run `python3.5 uWServer.py -h` +To see the options please run `python3.5 uWServer.py --help` + +Session Definitions: +-------------------- + +Example session: + +``` +{ + "encoding": "url_encoded", + "version": "0.2", + "txns": [ + { + "response": { + "headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", + "body": "", + "timestamp": "1469733493.993" + }, + "request": { + "headers": "GET / HTTP/1.1\r\nHost: www.example.test\r\n\r\n", + "body": "", + "timestamp": "1469733493.993" + }, + "uuid": "", + "timestamp": "" + } + ], + "timestamp": "1234567890.098" +} +``` +Each session should be in its own file, and any number of files may be created to define sessions. +The `response` map may include an `options` string, which is a comma-delimited list of options to be enabled. Currently the only option supported is `skipHooks`, which will ignore any hooks created for the matching request/response pair. See **Options**. diff --git a/tests/tools/microServer/uWServer.py b/tests/tools/microServer/uWServer.py index 78c8e81..0edbb73 100644 --- a/tests/tools/microServer/uWServer.py +++ b/tests/tools/microServer/uWServer.py @@ -36,7 +36,7 @@ import importlib.util import time test_mode_enabled = True lookup_key_ = "{PATH}" -__version__ = "1.0" +__version__ = "1.1" sys.path.append( @@ -160,10 +160,6 @@ class MyHandler(BaseHTTPRequestHandler): def handleExpect100Continue(self, contentLength, chunked=False): print("....expect", contentLength) self.wfile.write(bytes('HTTP/1.1 100 Continue\r\n\r\n', 'UTF-8')) - # self.send_response(HTTPStatus.CONTINUE) - # self.send_header('Server','blablabla') - #self.send_header('Connection', 'keep-alive') - # self.end_headers() if(not chunked): message = self.rfile.read(contentLength) else: @@ -232,7 +228,6 @@ class MyHandler(BaseHTTPRequestHandler): def send_response(self, code, message=None): ''' Override `send_response()`'s tacking on of server and date header lines. ''' - # self.log_request(code) self.send_response_only(code, message) def createDummyBodywithLength(self, numberOfbytes): @@ -258,14 +253,11 @@ class MyHandler(BaseHTTPRequestHandler): # print("==========================================>",size) size = int(size, 16) while size > 0: - #print("reading bytes",raw_size) chunk = self.rfile.read(size + 2) # 2 for reading /r/n - #print("cuhnk: ",chunk) raw_data += chunk raw_size = self.rfile.readline(65537) size = str(raw_size, 'UTF-8').rstrip('\r\n') size = int(size, 16) - #print("full chunk",raw_data) chunk = self.rfile.readline(65537) # read the extra blank newline \r\n after the last chunk def send_header(self, keyword, value): @@ -287,20 +279,19 @@ class MyHandler(BaseHTTPRequestHandler): The request should be stored in self.raw_requestline; the results are in self.command, self.path, self.request_version and - self.headers. + self.headers. Any matching response is in self.response. Return True for success, False for failure; on failure, an error is sent back. """ - global count, test_mode_enabled + global count, test_mode_enabled, G_replay_dict self.command = None # set in case of error on the first line self.request_version = version = self.default_request_version self.close_connection = True requestline = str(self.raw_requestline, 'UTF-8') - # print("request",requestline) requestline = requestline.rstrip('\r\n') self.requestline = requestline @@ -308,8 +299,14 @@ class MyHandler(BaseHTTPRequestHandler): try: self.headers = http.client.parse_headers(self.rfile, _class=self.MessageClass) - self.server.hook_set.invoke(HookSet.ReadRequestHook, self.headers) + if test_mode_enabled: + key = self.getLookupKey(self.requestline) + else: + key, __ = cgi.parse_header(self.headers.get('Content-MD5')) + self.resp = G_replay_dict[key] if key in G_replay_dict else None + if self.resp is None or 'skipHooks' not in self.resp.getOptions(): + self.server.hook_set.invoke(HookSet.ReadRequestHook, self.headers) # read message body if self.headers.get('Content-Length') != None: bodysize = int(self.headers.get('Content-Length')) @@ -396,22 +393,19 @@ class MyHandler(BaseHTTPRequestHandler): global G_replay_dict, test_mode_enabled if test_mode_enabled: time.sleep(time_delay) - request_hash = self.getLookupKey(self.requestline) - else: - request_hash, __ = cgi.parse_header(self.headers.get('Content-MD5')) - # print("key:",request_hash) + try: response_string = None chunkedResponse = False - if request_hash not in G_replay_dict: + if self.resp is None: self.send_response(404) self.send_header('Server', 'MicroServer') self.send_header('Connection', 'close') self.end_headers() + return else: - resp = G_replay_dict[request_hash] - headers = resp.getHeaders().split('\r\n') + headers = self.resp.getHeaders().split('\r\n') # set status codes status_code = self.get_response_code(headers[0]) @@ -431,7 +425,7 @@ class MyHandler(BaseHTTPRequestHandler): lengthSTR = header.split(':')[1] length = lengthSTR.strip(' ') if test_mode_enabled: # the length of the body is given priority in test mode rather than the value in Content-Length. But in replay mode Content-Length gets the priority - if not (resp and resp.getBody()): # Don't attach content-length yet if body is present in the response specified by tester + if not (self.resp.getBody()): # Don't attach content-length yet if body is present in the response specified by tester self.send_header('Content-Length', str(length)) else: self.send_header('Content-Length', str(length)) @@ -446,13 +440,12 @@ class MyHandler(BaseHTTPRequestHandler): header_parts = header.split(':', 1) header_field = str(header_parts[0].strip()) header_field_val = str(header_parts[1].strip()) - # print("{0} === >{1}".format(header_field, header_field_val)) self.send_header(header_field, header_field_val) # End for if test_mode_enabled: - if resp and resp.getBody(): - length = len(bytes(resp.getBody(), 'UTF-8')) - response_string = resp.getBody() + if self.resp.getBody(): + length = len(bytes(self.resp.getBody(), 'UTF-8')) + response_string = self.resp.getBody() self.send_header('Content-Length', str(length)) self.end_headers() @@ -460,7 +453,6 @@ class MyHandler(BaseHTTPRequestHandler): self.writeChunkedData() elif response_string != None and response_string != '': self.wfile.write(bytes(response_string, 'UTF-8')) - return except: e = sys.exc_info() print("Error", e, self.headers) @@ -469,59 +461,46 @@ class MyHandler(BaseHTTPRequestHandler): self.end_headers() def do_HEAD(self): - global G_replay_dict, test_mode_enabled - if test_mode_enabled: - request_hash = self.getLookupKey(self.requestline) - else: - request_hash, __ = cgi.parse_header(self.headers.get('Content-MD5')) - - if request_hash not in G_replay_dict: + if self.resp is None: self.send_response(404) self.send_header('Connection', 'close') self.end_headers() + return - else: - resp = G_replay_dict[request_hash] - headers = resp.getHeaders().split('\r\n') - - # set status codes - status_code = self.get_response_code(headers[0]) - self.send_response(status_code) - - # set headers - for header in headers[1:]: # skip first one b/c it's response code - if header == '': - continue - elif 'Content-Length' in header: - self.send_header('Content-Length', '0') - continue - - header_parts = header.split(':', 1) - header_field = str(header_parts[0].strip()) - header_field_val = str(header_parts[1].strip()) - #print("{0} === >{1}".format(header_field, header_field_val)) - self.send_header(header_field, header_field_val) + headers = self.resp.getHeaders().split('\r\n') - self.end_headers() + # set status codes + status_code = self.get_response_code(headers[0]) + self.send_response(status_code) + + # set headers + for header in headers[1:]: # skip first one b/c it's response code + if header == '': + continue + elif 'Content-Length' in header: + self.send_header('Content-Length', '0') + continue + + header_parts = header.split(':', 1) + header_field = str(header_parts[0].strip()) + header_field_val = str(header_parts[1].strip()) + self.send_header(header_field, header_field_val) + + self.end_headers() def do_POST(self): response_string = None chunkedResponse = False - global G_replay_dict, test_mode_enabled - if test_mode_enabled: - request_hash = self.getLookupKey(self.requestline) - else: - request_hash, __ = cgi.parse_header(self.headers.get('Content-MD5')) + global test_mode_enabled try: - if request_hash not in G_replay_dict: + if self.resp is None: self.send_response(404) self.send_header('Connection', 'close') self.end_headers() - resp = None + return else: - resp = G_replay_dict[request_hash] - resp_headers = resp.getHeaders().split('\r\n') + resp_headers = self.resp.getHeaders().split('\r\n') # set status codes status_code = self.get_response_code(resp_headers[0]) #print("response code",status_code) @@ -543,7 +522,7 @@ class MyHandler(BaseHTTPRequestHandler): lengthSTR = header.split(':')[1] length = lengthSTR.strip(' ') if test_mode_enabled: # the length of the body is given priority in test mode rather than the value in Content-Length. But in replay mode Content-Length gets the priority - if not (resp and resp.getBody()): # Don't attach content-length yet if body is present in the response specified by tester + if not (self.resp.getBody()): # Don't attach content-length yet if body is present in the response specified by tester self.send_header('Content-Length', str(length)) else: self.send_header('Content-Length', str(length)) @@ -562,9 +541,9 @@ class MyHandler(BaseHTTPRequestHandler): self.send_header(header_field, header_field_val) # End for loop if test_mode_enabled: - if resp and resp.getBody(): - length = len(bytes(resp.getBody(), 'UTF-8')) - response_string = resp.getBody() + if self.resp.getBody(): + length = len(bytes(self.resp.getBody(), 'UTF-8')) + response_string = self.resp.getBody() self.send_header('Content-Length', str(length)) self.end_headers() @@ -572,7 +551,6 @@ class MyHandler(BaseHTTPRequestHandler): self.writeChunkedData() elif response_string != None and response_string != '': self.wfile.write(bytes(response_string, 'UTF-8')) - return except: e = sys.exc_info() print("Error", e, self.headers) @@ -608,9 +586,9 @@ def _path(exists, arg): def _bool(arg): opt_true_values = set(['y', 'yes', 'true', 't', '1', 'on', 'all']) - opt_false_values = set(['n', 'no', 'false', 'f', '0', 'off', 'none']) + opt_false_values = set(['n', 'no', 'false', 'f', '0', 'off', 'none', None]) - tmp = arg.lower() + tmp = arg.lower() if arg is not None else None if tmp in opt_true_values: return True elif tmp in opt_false_values: @@ -618,7 +596,13 @@ def _bool(arg): else: msg = 'Invalid value Boolean value : "{0}"\n Valid options are {0}'.format(arg, opt_true_values | opt_false_values) - raise argparse.ArgumentTypeError(msg) + raise ValueError(msg) + +def _argparse_bool(): + try: + _bool + except ValueError as ve: + raise argparse.ArgumentTypeError(ve) def main(): @@ -671,7 +655,7 @@ def main(): default="ssl/server.crt", help="certificate") parser.add_argument("--clientverify", "-cverify", - type=bool, + type=_argparse_bool, default=False, help="verify client cert") parser.add_argument("--load", diff --git a/tests/tools/sessionvalidation/response.py b/tests/tools/sessionvalidation/response.py index b8438e2..faa5f97 100644 --- a/tests/tools/sessionvalidation/response.py +++ b/tests/tools/sessionvalidation/response.py @@ -16,6 +16,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import re + class Response(object): ''' Response encapsulates a single request from the UA ''' @@ -29,12 +31,19 @@ class Response(object): def getBody(self): return self._body + def getOptions(self): + return self._options + def __repr__(self): - return "<Response: {{'timestamp': {0}, 'headers': {1}, 'body': {2}}}>".format( - self._timestamp, self._headers, self._body + return "<Response: {{'timestamp': {0}, 'headers': {1}, 'body': {2}, 'options': {3}}}>".format( + self._timestamp, self._headers, self._body, self._options ) - def __init__(self, timestamp, headers, body): + def __init__(self, timestamp, headers, body, options_string): self._timestamp = timestamp self._headers = headers self._body = body + if options_string: + self._options = re.compile(r'\s*,\s*').split(options_string) + else: + self._options = list() diff --git a/tests/tools/sessionvalidation/sessionvalidation.py b/tests/tools/sessionvalidation/sessionvalidation.py index 5c926ea..d38dcfb 100644 --- a/tests/tools/sessionvalidation/sessionvalidation.py +++ b/tests/tools/sessionvalidation/sessionvalidation.py @@ -75,7 +75,6 @@ class SessionValidator(object): session_version = sesh['version'] session_txns = list() for txn in sesh['txns']: - # print("PERSIA____________________________________________________________",txn) # create transaction Request object txn_request = txn['request'] @@ -88,12 +87,12 @@ class SessionValidator(object): txn_response_body = '' if 'body' in txn_response: txn_response_body = txn_response['body'] - txn_response_obj = response.Response(txn_response['timestamp'], txn_response['headers'], txn_response_body) + txn_response_obj = response.Response(txn_response['timestamp'], txn_response['headers'], txn_response_body, + txn_response.get('options')) # create Transaction object txn_obj = transaction.Transaction(txn_request_obj, txn_response_obj, txn['uuid']) session_txns.append(txn_obj) - # print(txn_request['timestamp']) session_obj = session.Session(fname, session_version, session_timestamp, session_txns) except KeyError as e: @@ -197,11 +196,6 @@ class SessionValidator(object): _verbose_print("transaction request Host header doesn't have specified host") retval = False - # reject if the host is localhost (since ATS seems to ignore remap rules for localhost requests) - if "127.0.0.1" in txn_req.getHeaders() or "localhost" in txn_req.getHeaders(): - _verbose_print("transaction request Host is localhost, we must reject because ATS ignores remap rules for localhost requests") - retval = False - # now validate response if not txn_resp: _verbose_print("no transaction response") -- To stop receiving notification emails like this one, please contact dra...@apache.org.