This is an automated email from the ASF dual-hosted git repository. cmcfarlen pushed a commit to branch 10.2.x in repository https://gitbox.apache.org/repos/asf/trafficserver.git
commit 7d95f45b63d29bf054529cf32e38444c62315ac5 Author: Mo Chen <[email protected]> AuthorDate: Thu Jun 25 11:59:38 2026 -0500 tests: add TLS gold tests (#13306) Increase autest coverage with the following tests: - TLS renegotiation, rejected by default and allowed when configured - the TLS record-size clamp, checked against on-the-wire record sizes - a failed outbound TLS origin connection surfaced as a 5xx - a TLS origin that resets the connection mid-request-body (cherry picked from commit 3ce2acfbc6cf287e49638c4b054f48711c09a7e7) --- AGENTS.md | 7 + tests/gold_tests/tls/tls_flow_control.test.py | 116 +++++++++++++ tests/gold_tests/tls/tls_flow_control_client.py | 100 +++++++++++ .../gold_tests/tls/tls_origin_open_failed.test.py | 78 +++++++++ tests/gold_tests/tls/tls_origin_post_abort.test.py | 114 +++++++++++++ tests/gold_tests/tls/tls_post_abort_client.py | 127 ++++++++++++++ tests/gold_tests/tls/tls_post_abort_origin.py | 92 +++++++++++ tests/gold_tests/tls/tls_record_size.test.py | 117 +++++++++++++ tests/gold_tests/tls/tls_record_size_client.py | 155 +++++++++++++++++ tests/gold_tests/tls/tls_reload_under_load.test.py | 100 +++++++++++ .../gold_tests/tls/tls_reload_under_load_client.py | 183 +++++++++++++++++++++ tests/gold_tests/tls/tls_renegotiation.test.py | 124 ++++++++++++++ .../tls/tls_renegotiation_allowed.test.py | 145 ++++++++++++++++ .../tls/tls_renegotiation_allowed_client.py | 132 +++++++++++++++ tests/gold_tests/tls/tls_renegotiation_client.py | 67 ++++++++ 15 files changed, 1657 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index 75308cecdf..b686484d67 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -123,6 +123,13 @@ regularly). using the Proxy Verifier format. This is simpler, more maintainable, and parseable by tools. +**Python conventions for test and helper scripts:** +- Launch Python helpers with `{sys.executable}` rather than a hardcoded `python3`, + so the test runs under the same interpreter the harness uses. +- Prefer f-strings over `str.format()` when building command lines, config lines, + and `Testers` expressions. +- Add type annotations to helper functions. + **For complete details on writing autests, see:** - `doc/developer-guide/testing/autests.en.rst` - Comprehensive guide to autest - Proxy Verifier format: https://github.com/yahoo/proxy-verifier diff --git a/tests/gold_tests/tls/tls_flow_control.test.py b/tests/gold_tests/tls/tls_flow_control.test.py new file mode 100644 index 0000000000..f1da922182 --- /dev/null +++ b/tests/gold_tests/tls/tls_flow_control.test.py @@ -0,0 +1,116 @@ +''' +Exercise HTTP tunnel flow control (proxy.config.http.flow_control) on a TLS +client connection: a slow reader makes ATS throttle the origin and unthrottle as +the buffered data drains. The full response body must still be delivered. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys + +Test.Summary = __doc__ + + +class TestTlsFlowControl: + '''Verify a slow TLS reader under flow control still receives the whole body.''' + + # Comfortably larger than the water marks so the tunnel throttles and has to + # unthrottle many times over the transfer. + _body_len: int = 8 * 1024 * 1024 + _high_water: int = 64 * 1024 + _low_water: int = 32 * 1024 + + _server_counter: int = 0 + _ts_counter: int = 0 + + def __init__(self) -> None: + '''Declare the test Processes.''' + self._server = self._configure_server() + self._ts = self._configure_trafficserver() + + def _configure_server(self) -> 'Process': + '''Configure the origin server with a large response body. + + :return: The origin server Process. + ''' + server = Test.MakeOriginServer(f'server-{TestTlsFlowControl._server_counter}') + TestTlsFlowControl._server_counter += 1 + + request_header = {"headers": "GET /obj HTTP/1.1\r\nHost: ex.test\r\n\r\n", "timestamp": "1469733493.993", "body": ""} + response_header = { + "headers": + "HTTP/1.1 200 OK\r\nServer: microserver\r\nConnection: close\r\n" + f"Content-Length: {TestTlsFlowControl._body_len}\r\n\r\n", + "timestamp": "1469733493.993", + "body": "x" * TestTlsFlowControl._body_len + } + server.addResponse("sessionlog.json", request_header, response_header) + return server + + def _configure_trafficserver(self) -> 'Process': + '''Configure Traffic Server with HTTP flow control enabled over TLS. + + :return: The Traffic Server Process. + ''' + ts = Test.MakeATSProcess(f'ts-{TestTlsFlowControl._ts_counter}', enable_tls=True, enable_cache=False) + TestTlsFlowControl._ts_counter += 1 + + ts.addDefaultSSLFiles() + ts.Disk.ssl_multicert_yaml.AddLines( + """ +ssl_multicert: + - dest_ip: "*" + ssl_cert_name: server.pem + ssl_key_name: server.key +""".split("\n")) + ts.Disk.remap_config.AddLine(f'map / http://127.0.0.1:{self._server.Variables.Port}') + ts.Disk.records_config.update( + { + 'proxy.config.ssl.server.cert.path': f'{ts.Variables.SSLDir}', + 'proxy.config.ssl.server.private_key.path': f'{ts.Variables.SSLDir}', + # Small thresholds on a network sink: the tunnel reenable fires before + # the socket write, so the unthrottle depends on the buffer draining. + 'proxy.config.http.flow_control.enabled': 1, + 'proxy.config.http.flow_control.high_water': TestTlsFlowControl._high_water, + 'proxy.config.http.flow_control.low_water': TestTlsFlowControl._low_water, + }) + + ts.Disk.traffic_out.Content = Testers.ExcludesExpression( + "received signal|failed assertion", "ATS must not crash under TLS flow control") + ts.Disk.traffic_out.Content += Testers.ExcludesExpression( + "AddressSanitizer|use-after-free|runtime error:", "no memory-safety error under TLS flow control") + return ts + + def run(self) -> None: + '''Configure and run the TestRun.''' + tr = Test.AddTestRun("slow TLS reader under flow control must receive the whole body") + tr.Processes.Default.StartBefore(self._server) + tr.Processes.Default.StartBefore(self._ts) + tr.Processes.Default.Command = ( + f'{sys.executable} {os.path.join(Test.TestDirectory, "tls_flow_control_client.py")} ' + f'-p {self._ts.Variables.ssl_port} --host ex.test --path /obj ' + f'--expect-bytes {TestTlsFlowControl._body_len}') + tr.Processes.Default.ReturnCode = 0 + tr.Processes.Default.Streams.All = Testers.ContainsExpression( + "RESULT=PASS", "the full body must be delivered under flow control") + tr.Processes.Default.Streams.All += Testers.ContainsExpression( + f"BODY_BYTES={TestTlsFlowControl._body_len}", "every body byte must arrive (no flow-control stall)") + tr.StillRunningAfter = self._ts + tr.StillRunningAfter = self._server + + +TestTlsFlowControl().run() diff --git a/tests/gold_tests/tls/tls_flow_control_client.py b/tests/gold_tests/tls/tls_flow_control_client.py new file mode 100644 index 0000000000..66565c1912 --- /dev/null +++ b/tests/gold_tests/tls/tls_flow_control_client.py @@ -0,0 +1,100 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Download a large object from ATS over TLS while reading the response body +slowly, to exercise HTTP tunnel flow control (proxy.config.http.flow_control) +on a TLS client connection. + +A slow reader keeps ATS's client-side write buffer full, so the tunnel +repeatedly throttles the origin read and must unthrottle as the buffered data +drains to the client. The whole body must still arrive; a flow-control stall +(no unthrottle) shows up as a short or timed-out read. This path had no prior +gold coverage, and the layered TLS VConnection drives the unthrottle through its +demand-driven write rather than the inherited write-buffer-empty trap, so it is +worth guarding directly. +""" + +import argparse +import socket +import ssl +import sys +import time + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("-p", "--port", type=int, required=True, help="ATS TLS port") + parser.add_argument("--host", default="ex.test", help="Host header / SNI") + parser.add_argument("--path", default="/obj", help="request path") + parser.add_argument("--expect-bytes", type=int, required=True, help="expected body length") + parser.add_argument("--read-size", type=int, default=16 * 1024, help="bytes per recv") + parser.add_argument("--read-delay", type=float, default=0.002, help="sleep between recvs (s)") + parser.add_argument("--recv-timeout", type=float, default=15.0, help="per-recv socket timeout (s)") + parser.add_argument("--deadline", type=float, default=120.0, help="overall wall-clock budget (s)") + args = parser.parse_args() + + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + + deadline = time.monotonic() + args.deadline + body_received = 0 + error = "" + try: + with socket.create_connection(("127.0.0.1", args.port), timeout=args.recv_timeout) as sock: + with ctx.wrap_socket(sock, server_hostname=args.host) as ssock: + ssock.settimeout(args.recv_timeout) + request = (f"GET {args.path} HTTP/1.1\r\nHost: {args.host}\r\n" + "Connection: close\r\n\r\n").encode() + ssock.sendall(request) + + # Read until we have the full header block, keeping any body bytes + # that arrive in the same recv. + buf = b"" + while b"\r\n\r\n" not in buf: + if time.monotonic() > deadline: + raise TimeoutError("deadline reached reading response headers") + chunk = ssock.recv(args.read_size) + if not chunk: + raise ConnectionError("connection closed before headers complete") + buf += chunk + header_blob, _, leftover = buf.partition(b"\r\n\r\n") + body_received = len(leftover) + + # Drain the body slowly so ATS's client-side write buffer stays full + # and the tunnel relies on the buffer-empty unthrottle. + while body_received < args.expect_bytes: + if time.monotonic() > deadline: + raise TimeoutError(f"deadline reached after {body_received} body bytes") + time.sleep(args.read_delay) + chunk = ssock.recv(args.read_size) + if not chunk: + break + body_received += len(chunk) + except Exception as exc: # noqa: BLE001 - any failure is a test signal + error = f"{type(exc).__name__}: {exc}" + + passed = error == "" and body_received == args.expect_bytes + + print(f"BODY_BYTES={body_received}") + print(f"EXPECT_BYTES={args.expect_bytes}") + if error: + print(f"ERROR={error}") + print(f"RESULT={'PASS' if passed else 'FAIL'}") + return 0 if passed else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/gold_tests/tls/tls_origin_open_failed.test.py b/tests/gold_tests/tls/tls_origin_open_failed.test.py new file mode 100644 index 0000000000..52092aadf0 --- /dev/null +++ b/tests/gold_tests/tls/tls_origin_open_failed.test.py @@ -0,0 +1,78 @@ +''' +Verify that a failed outbound TLS origin connection is surfaced to the client as +an error (5xx) without crashing ATS. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import ports + +Test.Summary = __doc__ + + +class TestOriginOpenFailed: + '''Verify a failed outbound TLS connect is surfaced as a 5xx, not a crash.''' + + _ts_counter: int = 0 + + def __init__(self) -> None: + '''Declare the test Processes.''' + self._ts = self._configure_trafficserver() + + def _configure_trafficserver(self) -> 'Process': + '''Configure Traffic Server with an https origin whose connect fails. + + :return: The Traffic Server Process. + ''' + ts = Test.MakeATSProcess(f'ts-{TestOriginOpenFailed._ts_counter}') + TestOriginOpenFailed._ts_counter += 1 + + # Reserve a port for a TLS origin; its liveness is irrelevant since the + # source bind below fails before any connect is attempted. + ports.get_port(ts, 'origin_port') + + ts.Disk.remap_config.AddLine(f'map http://dead.test/ https://127.0.0.1:{ts.Variables.origin_port}/') + ts.Disk.records_config.update( + { + # Bind every outbound connection to a non-local source address + # (RFC 5737 documentation range) so bind() fails synchronously with + # EADDRNOTAVAIL, i.e. the connect fails before any TCP/TLS exchange + # rather than via a later, routable connect that would be refused + # asynchronously. + 'proxy.config.outgoing_ip_to_bind': '192.0.2.1', + 'proxy.config.http.connect_attempts_max_retries': 1, + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'http|ssl', + }) + + # Tearing down the failed outbound TLS connect must not crash ATS. + ts.Disk.traffic_out.Content = Testers.ExcludesExpression( + "received signal|failed assertion", "ATS must not crash on a failed outbound TLS connect") + return ts + + def run(self) -> None: + '''Configure and run the TestRun.''' + tr = Test.AddTestRun("a failed outbound TLS connect is surfaced cleanly") + tr.Processes.Default.StartBefore(self._ts) + tr.MakeCurlCommand( + f'-s -o /dev/null -w "%{{http_code}}" -H "Host: dead.test" http://127.0.0.1:{self._ts.Variables.port}/', ts=self._ts) + tr.Processes.Default.ReturnCode = 0 + # A failed origin connection is surfaced as a 5xx (502 Bad Gateway). + tr.Processes.Default.Streams.stdout = Testers.ContainsExpression("50[02]", "a failed origin connect yields a 5xx") + tr.StillRunningAfter = self._ts + + +TestOriginOpenFailed().run() diff --git a/tests/gold_tests/tls/tls_origin_post_abort.test.py b/tests/gold_tests/tls/tls_origin_post_abort.test.py new file mode 100644 index 0000000000..4d989a26ef --- /dev/null +++ b/tests/gold_tests/tls/tls_origin_post_abort.test.py @@ -0,0 +1,114 @@ +''' +When a TLS origin resets the connection while ATS is still sending it a POST +request body, ATS must fail the transaction promptly -- the transport error must +reach the request-body tunnel right away as VC_EVENT_ERROR, not sit until the +outbound inactivity timeout rescues it. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys + +Test.Summary = __doc__ + + +class TestOriginPostAbort: + '''Verify a TLS origin RST mid-POST-body fails the transaction promptly.''' + + _ts_counter: int = 0 + _origin_counter: int = 0 + + def __init__(self) -> None: + '''Declare the test Processes.''' + Test.GetTcpPort("origin_port") + self._ts = self._configure_trafficserver() + self._origin = self._configure_origin() + + def _configure_trafficserver(self) -> 'Process': + '''Configure Traffic Server with a TLS origin and a generous rescue timeout. + + :return: The Traffic Server Process. + ''' + ts = Test.MakeATSProcess(f'ts-{TestOriginPostAbort._ts_counter}', enable_cache=False) + TestOriginPostAbort._ts_counter += 1 + + ts.Disk.records_config.update( + { + 'proxy.config.url_remap.remap_required': 1, + 'proxy.config.http.connect_attempts_max_retries': 0, + 'proxy.config.http.connect_attempts_timeout': 15, + # The rescue timeout: a prompt failure must beat this by a wide margin. + 'proxy.config.http.transaction_no_activity_timeout_out': 15, + 'proxy.config.http.transaction_no_activity_timeout_in': 30, + # Keep the kernel send buffer small so the request body cannot be + # absorbed by the kernel: ATS must still be mid-send at the RST. + 'proxy.config.net.sock_send_buffer_size_out': 65536, + 'proxy.config.ssl.client.verify.server.policy': 'DISABLED', + 'proxy.config.diags.debug.enabled': 0, + 'proxy.config.diags.debug.tags': 'http|ssl|ssl_io', + }) + ts.Disk.remap_config.AddLine(f'map /post https://127.0.0.1:{Test.Variables.origin_port}') + + ts.Disk.traffic_out.Content = Testers.ExcludesExpression( + 'received signal|failed assertion', 'ATS must not crash handling the mid-body origin abort') + return ts + + def _configure_origin(self) -> 'Process': + '''Configure a raw-socket TLS origin that resets mid-request-body. + + The scenario needs a TLS origin that accepts the request body and then RSTs + the connection mid-body (SO_LINGER 0). Proxy Verifier can only close cleanly + (close_notify), so it cannot express a mid-body reset; hence the small + raw-socket origin helper rather than an ATSReplayTest. + + :return: The origin server Process. + ''' + origin = Test.Processes.Process( + f'origin-{TestOriginPostAbort._origin_counter}', + f'{sys.executable} {os.path.join(Test.TestDirectory, "tls_post_abort_origin.py")} ' + f'-p {Test.Variables.origin_port} ' + f'-c {os.path.join(Test.Variables.AtsTestToolsDir, "ssl", "server.pem")} -d 1.0') + TestOriginPostAbort._origin_counter += 1 + + # These markers prove the reset path was actually exercised: the origin + # completed the TLS handshake, received the forwarded request headers, and + # then reset the connection mid-body. Without them the client could pass on + # any prompt response -- e.g. a broken remap or a never-started origin that + # yields a fast 5xx without the body ever reaching a resetting origin. + origin.Streams.stdout = Testers.ContainsExpression( + 'request headers received', 'the origin must receive the forwarded request headers') + origin.Streams.stdout += Testers.ContainsExpression( + 'connection reset sent', 'the origin must reset the connection mid-body') + return origin + + def run(self) -> None: + '''Configure and run the TestRun.''' + tr = Test.AddTestRun("POST to a TLS origin that resets mid-body must fail promptly") + tr.Processes.Default.StartBefore(self._ts) + tr.Processes.Default.StartBefore(self._origin, ready=When.PortOpen(Test.Variables.origin_port)) + tr.Processes.Default.Command = ( + f'{sys.executable} {os.path.join(Test.TestDirectory, "tls_post_abort_client.py")} ' + f'-p {self._ts.Variables.port} -t 8') + tr.Processes.Default.ReturnCode = 0 + tr.Processes.Default.Streams.All += Testers.ContainsExpression( + 'PASS: transaction failed promptly', 'the POST must fail promptly on the origin RST') + tr.Processes.Default.Streams.All += Testers.ContainsExpression( + 'status-code: 5', 'ATS must surface the mid-body origin reset as a 5xx') + tr.StillRunningAfter = self._ts + + +TestOriginPostAbort().run() diff --git a/tests/gold_tests/tls/tls_post_abort_client.py b/tests/gold_tests/tls/tls_post_abort_client.py new file mode 100644 index 0000000000..afd02385a1 --- /dev/null +++ b/tests/gold_tests/tls/tls_post_abort_client.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +''' +POST a large request body through ATS and measure how long ATS takes to fail +the transaction after the origin aborts the connection mid-body. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import select +import socket +import sys +import time + + +def parse_status_code(status_line: str) -> 'int | None': + '''Return the numeric HTTP status from a status line, or None if absent.''' + fields = status_line.split() + if len(fields) >= 2 and fields[0].startswith('HTTP/') and fields[1].isdigit(): + return int(fields[1]) + return None + + +def main() -> int: + parser = argparse.ArgumentParser(description='POST through ATS; time how fast the transaction fails.') + parser.add_argument('-p', '--proxy-port', type=int, required=True, help='ATS HTTP port') + parser.add_argument('-s', '--size', type=int, default=64 * 1024 * 1024, help='request body size in bytes') + parser.add_argument( + '-t', '--threshold', type=float, default=8.0, help='maximum seconds from request start until ATS fails the transaction') + parser.add_argument('--timeout', type=float, default=60.0, help='give up entirely after this many seconds') + parser.add_argument( + '--pause-after', type=int, default=0, help='stop sending body bytes after this many (0: keep sending until done)') + args = parser.parse_args() + + start = time.monotonic() + sock = socket.create_connection(('127.0.0.1', args.proxy_port), timeout=10) + request = (f'POST /post HTTP/1.1\r\n' + f'Host: origin\r\n' + f'Content-Length: {args.size}\r\n' + f'Connection: close\r\n' + f'\r\n').encode() + sock.sendall(request) + sock.setblocking(False) + + chunk = b'x' * 65536 + sent = 0 + response = b'' + outcome = None + while time.monotonic() - start < args.timeout: + want_write = sent < args.size and not (args.pause_after and sent >= args.pause_after) + readable, writable, _ = select.select([sock], [sock] if want_write else [], [], 1.0) + if readable: + try: + data = sock.recv(65536) + except (ConnectionResetError, BrokenPipeError, OSError) as e: + outcome = f'connection error while reading: {e.__class__.__name__}' + break + if not data: + outcome = 'connection closed by ATS' + break + response += data + if b'\r\n\r\n' in response: + outcome = 'response received' + break + if writable: + try: + sent += sock.send(chunk[:min(len(chunk), args.size - sent)]) + except (ConnectionResetError, BrokenPipeError, OSError) as e: + outcome = f'connection error while sending: {e.__class__.__name__}' + break + if sent >= args.size: + outcome = 'entire body sent' + break + + elapsed = time.monotonic() - start + if outcome is None: + outcome = 'gave up waiting' + status_line = response.split(b'\r\n', 1)[0].decode(errors='replace') if response else '<none>' + status_code = parse_status_code(status_line) + print(f'outcome: {outcome}') + print(f'status-line: {status_line}') + print(f'status-code: {status_code if status_code is not None else "<none>"}') + print(f'body-bytes-sent: {sent}') + print(f'elapsed: {elapsed:.3f}') + + # The transaction must actually have failed, not completed. + if outcome in ('entire body sent', 'gave up waiting'): + print('FAIL: transaction did not fail at all') + return 2 + # The client must have started streaming the request body; zero bytes sent + # would mean ATS failed at the request headers, not the mid-body path. (That + # the body reached a resetting origin is proven by the origin-side markers the + # test asserts, not by this client-side count.) + if sent <= 0: + print('FAIL: ATS failed before any request body was sent; mid-body path not exercised') + return 1 + # A prompt failure must beat the inactivity-timeout rescue by a wide margin. + if elapsed > args.threshold: + print(f'FAIL: transaction failed only after {elapsed:.3f}s (threshold {args.threshold}s)') + return 1 + # The upstream reset must be surfaced as a 5xx, not masked as success and not + # a bare connection drop with no status at all. + if status_code is None: + print('FAIL: ATS dropped the connection without surfacing a response status') + return 1 + if not 500 <= status_code <= 599: + print(f'FAIL: ATS surfaced status {status_code}; expected a 5xx for the origin reset') + return 1 + print('PASS: transaction failed promptly') + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tests/gold_tests/tls/tls_post_abort_origin.py b/tests/gold_tests/tls/tls_post_abort_origin.py new file mode 100644 index 0000000000..8300358a31 --- /dev/null +++ b/tests/gold_tests/tls/tls_post_abort_origin.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +''' +A TLS origin that accepts one connection, reads the request headers, then +stops reading and aborts the connection with a TCP RST while the proxy is +still sending the request body. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import socket +import ssl +import struct +import sys +import time + + +def main() -> int: + parser = argparse.ArgumentParser(description='TLS origin that RSTs mid-request-body.') + parser.add_argument('-p', '--port', type=int, required=True, help='port to listen on') + parser.add_argument('-c', '--cert', required=True, help='PEM file with certificate and key') + parser.add_argument('-d', '--delay', type=float, default=1.0, help='seconds to wait after the headers before the RST') + args = parser.parse_args() + + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(args.cert) + + listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + # Keep the receive window small so the proxy cannot park the whole request + # body in kernel buffers: it must still be mid-send when the RST arrives. + listener.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 65536) + listener.bind(('127.0.0.1', args.port)) + listener.listen(5) + print(f'listening on {args.port}', flush=True) + + # Accept until a connection completes the TLS handshake and delivers the + # request headers: the test harness probes the port (a bare TCP connect + # then close) to detect readiness, and that probe must not consume the + # one real connection. + tls = None + while tls is None: + conn, addr = listener.accept() + print(f'accepted connection from {addr}', flush=True) + try: + tls = context.wrap_socket(conn, server_side=True) + except (ssl.SSLError, OSError) as e: + print(f'TLS handshake failed (readiness probe?): {e}', flush=True) + conn.close() + tls = None + continue + print('TLS handshake complete', flush=True) + data = b'' + while b'\r\n\r\n' not in data: + chunk = tls.recv(4096) + if not chunk: + break + data += chunk + if b'\r\n\r\n' not in data: + print('peer closed before request headers arrived', flush=True) + tls.close() + tls = None + continue + print('request headers received; not reading the body', flush=True) + + # Let the proxy get deep into sending the request body, then abort. With + # SO_LINGER(0) the close discards buffered data and sends an RST; the + # unread body bytes in the receive queue force an RST as well. Do not + # unwrap()/shutdown the TLS layer: this is an abortive transport-level + # close, not a clean close_notify. + time.sleep(args.delay) + tls.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0)) + tls.close() + print('connection reset sent', flush=True) + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tests/gold_tests/tls/tls_record_size.test.py b/tests/gold_tests/tls/tls_record_size.test.py new file mode 100644 index 0000000000..b40c33a6fa --- /dev/null +++ b/tests/gold_tests/tls/tls_record_size.test.py @@ -0,0 +1,117 @@ +''' +Exercise the TLS record-size clamp (proxy.config.ssl.max_record_size > 0): on a +large TLS download the body must arrive intact and every application-data record +on the wire must be clamped to the configured size. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# NOTE: only the positive (fixed-clamp) branch of the record-sizing logic is +# covered here. The documented dynamic mode (max_record_size == -1) cannot be +# enabled through records.yaml because the record's validity check is [0-16383], +# which rejects -1; that inconsistency is pre-existing, so the dynamic branch +# stays uncovered by design. + +import os +import sys + +Test.Summary = __doc__ + + +class TestRecordSizeClamp: + '''Verify max_record_size clamps every record of a large TLS download.''' + + # Comfortably larger than the clamp so many records pass through it. + _body_len: int = 1024 * 1024 + _max_record: int = 4096 + + _server_counter: int = 0 + _ts_counter: int = 0 + + def __init__(self) -> None: + '''Declare the test Processes.''' + self._server = self._configure_server() + self._ts = self._configure_trafficserver() + + def _configure_server(self) -> 'Process': + '''Configure the origin server with a large response body. + + :return: The origin server Process. + ''' + server = Test.MakeOriginServer(f'server-{TestRecordSizeClamp._server_counter}') + TestRecordSizeClamp._server_counter += 1 + + body = "x" * TestRecordSizeClamp._body_len + request_header = {"headers": "GET /obj HTTP/1.1\r\nHost: ex.test\r\n\r\n", "timestamp": "1469733493.993", "body": ""} + response_header = { + "headers": + "HTTP/1.1 200 OK\r\nServer: microserver\r\nConnection: close\r\n" + f"Cache-Control: max-age=3600\r\nContent-Length: {TestRecordSizeClamp._body_len}\r\n\r\n", + "timestamp": "1469733493.993", + "body": body + } + server.addResponse("sessionlog.json", request_header, response_header) + return server + + def _configure_trafficserver(self) -> 'Process': + '''Configure Traffic Server with a positive max_record_size clamp. + + :return: The Traffic Server Process. + ''' + ts = Test.MakeATSProcess(f'ts-{TestRecordSizeClamp._ts_counter}', enable_tls=True) + TestRecordSizeClamp._ts_counter += 1 + + ts.addDefaultSSLFiles() + ts.Disk.ssl_multicert_yaml.AddLines( + """ +ssl_multicert: + - dest_ip: "*" + ssl_cert_name: server.pem + ssl_key_name: server.key +""".split("\n")) + ts.Disk.remap_config.AddLine(f'map / http://127.0.0.1:{self._server.Variables.Port}') + ts.Disk.records_config.update( + { + 'proxy.config.ssl.server.cert.path': f'{ts.Variables.SSLDir}', + 'proxy.config.ssl.server.private_key.path': f'{ts.Variables.SSLDir}', + # Positive cap -> the write path clamps each TLS record to this many bytes. + 'proxy.config.ssl.max_record_size': TestRecordSizeClamp._max_record, + }) + return ts + + def run(self) -> None: + '''Configure and run the TestRun. + + The client downloads the object and measures the TLS records on the wire, + asserting both that the body is intact and that no application-data record + exceeds the configured clamp. + ''' + tr = Test.AddTestRun("max_record_size>0 clamps records on a large TLS download") + tr.Processes.Default.StartBefore(self._server) + tr.Processes.Default.StartBefore(self._ts) + tr.Processes.Default.Command = ( + f'{sys.executable} {os.path.join(Test.TestDirectory, "tls_record_size_client.py")} ' + f'-p {self._ts.Variables.ssl_port} --host ex.test --path /obj ' + f'--max-record {TestRecordSizeClamp._max_record} --expect-bytes {TestRecordSizeClamp._body_len}') + tr.Processes.Default.ReturnCode = 0 + tr.Processes.Default.Streams.All += Testers.ContainsExpression( + "PASS: every application-data record is within the configured clamp", + "every TLS record must be clamped to the configured size") + tr.StillRunningAfter = self._ts + tr.StillRunningAfter = self._server + + +TestRecordSizeClamp().run() diff --git a/tests/gold_tests/tls/tls_record_size_client.py b/tests/gold_tests/tls/tls_record_size_client.py new file mode 100644 index 0000000000..a169944e5c --- /dev/null +++ b/tests/gold_tests/tls/tls_record_size_client.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +''' +Download an object from ATS over TLS and inspect the TLS records on the wire: +confirm the body arrives intact AND that every application-data record is no +larger than the configured proxy.config.ssl.max_record_size (plus AEAD overhead). + +A MemoryBIO drives the handshake so the raw ciphertext stream is visible; the +5-byte TLS record headers (type, version, length) are in cleartext, so record +sizes can be measured without decrypting. TLS 1.2 is pinned so that handshake +messages are their own record type (22) and only real application data is type 23, +and the cipher is restricted to AEAD (GCM) suites so the per-record overhead is a +small fixed constant rather than the variable IV/MAC/padding of a CBC suite. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import socket +import ssl +import sys +from collections.abc import Iterator + +TLS_APPLICATION_DATA = 23 +# A clamped plaintext record becomes ciphertext of plaintext + AEAD overhead +# (TLS1.2 GCM: 8-byte explicit nonce + 16-byte tag = 24 bytes; the cipher is pinned +# to AEAD below). 256 is a generous ceiling over that, far below an unclamped ~16 KB +# record, so the clamp check stays decisive and cannot be tripped by the larger, +# variable expansion of a CBC suite. +RECORD_OVERHEAD = 256 + + +def iter_record_lengths(buf: bytes | bytearray) -> Iterator[tuple[int, int]]: + '''Yield (content_type, record_length) for each complete TLS record in buf.''' + i, n = 0, len(buf) + while i + 5 <= n: + content_type = buf[i] + length = (buf[i + 3] << 8) | buf[i + 4] + if i + 5 + length > n: + break # truncated trailing record + yield content_type, length + i += 5 + length + + +def main() -> int: + parser = argparse.ArgumentParser(description='Measure ATS TLS record sizes on a download.') + parser.add_argument('-p', '--port', type=int, required=True, help='ATS TLS port') + parser.add_argument('--host', default='ex.test', help='Host header / SNI') + parser.add_argument('--path', default='/obj', help='request path') + parser.add_argument('--max-record', type=int, required=True, help='configured proxy.config.ssl.max_record_size') + parser.add_argument('--expect-bytes', type=int, required=True, help='expected response body length') + args = parser.parse_args() + + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + ctx.minimum_version = ssl.TLSVersion.TLSv1_2 + ctx.maximum_version = ssl.TLSVersion.TLSv1_2 + # Pin AEAD (GCM) suites: their per-record overhead is a small fixed 24 bytes, so a + # clamped record stays well within max_record + RECORD_OVERHEAD. A negotiated CBC + # suite could expand a record by IV + MAC + up to 255 bytes of padding and trip the + # clamp check spuriously. SECLEVEL=0 keeps the small test cert usable. + ctx.set_ciphers( + 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:' + 'ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:' + 'AES128-GCM-SHA256:AES256-GCM-SHA384@SECLEVEL=0') + + incoming, outgoing = ssl.MemoryBIO(), ssl.MemoryBIO() + tls = ctx.wrap_bio(incoming, outgoing, server_hostname=args.host) + + sock = socket.create_connection(('127.0.0.1', args.port), timeout=30) + sock.settimeout(30) + raw = bytearray() # every ciphertext byte the server sends, in order + + def flush() -> None: + data = outgoing.read() + if data: + sock.sendall(data) + + def feed() -> bytes: + chunk = sock.recv(65536) + if chunk: + raw.extend(chunk) + incoming.write(chunk) + return chunk + + while True: + try: + tls.do_handshake() + break + except ssl.SSLWantReadError: + flush() + if not feed(): + print('FAIL: server closed during handshake') + return 2 + flush() + + request = (f'GET {args.path} HTTP/1.1\r\n' + f'Host: {args.host}\r\n' + f'Connection: close\r\n\r\n').encode() + tls.write(request) + flush() + + response = bytearray() + while True: + try: + data = tls.read(65536) + if not data: + break # clean close_notify + response.extend(data) + except ssl.SSLWantReadError: + flush() + if not feed(): + break # socket closed by ATS + except ssl.SSLEOFError: + break + + separator = response.find(b'\r\n\r\n') + body_len = len(response) - (separator + 4) if separator >= 0 else -1 + + app_lengths = [length for content_type, length in iter_record_lengths(raw) if content_type == TLS_APPLICATION_DATA] + max_record = max(app_lengths) if app_lengths else 0 + limit = args.max_record + RECORD_OVERHEAD + + print( + f'app_data_records={len(app_lengths)} max_record_len={max_record} ' + f'limit={limit} body_len={body_len} expect={args.expect_bytes}') + + if body_len != args.expect_bytes: + print(f'FAIL: body length {body_len} != expected {args.expect_bytes}') + return 1 + if len(app_lengths) < 2: + print('FAIL: too few application-data records to judge clamping') + return 1 + if max_record > limit: + print(f'FAIL: an application-data record ({max_record}) exceeds the clamp + overhead ({limit})') + return 1 + print('PASS: every application-data record is within the configured clamp') + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tests/gold_tests/tls/tls_reload_under_load.test.py b/tests/gold_tests/tls/tls_reload_under_load.test.py new file mode 100644 index 0000000000..1828da760d --- /dev/null +++ b/tests/gold_tests/tls/tls_reload_under_load.test.py @@ -0,0 +1,100 @@ +''' +Existing cert/SNI reload tests reload while the server is idle. This one drives +continuous concurrent TLS handshakes and reloads ssl_multicert.yaml on top of +them, stressing the SSL/BIO ownership boundary of the layered TLS VConnection. +The swapped-in certificate must take effect, every handshake must succeed, and +ATS must not crash. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys + +Test.Summary = __doc__ + + +class TestTlsReloadUnderLoad: + '''Reload TLS certs while concurrent handshakes are in flight; serving must not break.''' + + _ts_counter: int = 0 + + def __init__(self) -> None: + '''Declare the test Processes.''' + self._ts = self._configure_trafficserver() + + def _configure_trafficserver(self) -> 'Process': + '''Configure Traffic Server with a swappable live certificate. + + :return: The Traffic Server Process. + ''' + ts = Test.MakeATSProcess(f'ts-{TestTlsReloadUnderLoad._ts_counter}', enable_tls=True, enable_cache=False) + TestTlsReloadUnderLoad._ts_counter += 1 + + # signed-bar is the live cert; signed2-bar shares its key and is copied over + # the live cert at reload time. Distinct fingerprints let the client prove the + # swap took effect under load. + ts.addSSLfile("ssl/signed-bar.pem") + ts.addSSLfile("ssl/signed-bar.key") + ts.addSSLfile("ssl/signed2-bar.pem") + + ts.Disk.ssl_multicert_yaml.AddLines( + """ +ssl_multicert: + - dest_ip: "*" + ssl_cert_name: signed-bar.pem + ssl_key_name: signed-bar.key +""".split("\n")) + ts.Disk.records_config.update( + { + 'proxy.config.ssl.server.cert.path': f'{ts.Variables.SSLDir}', + 'proxy.config.ssl.server.private_key.path': f'{ts.Variables.SSLDir}', + 'proxy.config.exec_thread.autoconfig.scale': 1.0, + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'ssl', + }) + + # The reload must actually have run (otherwise the test would be vacuous). + ts.Disk.diags_log.Content = Testers.ContainsExpression( + "ssl_multicert.yaml finished loading", "the cert configuration must reload while load is in flight") + # The reload-under-load must not crash or trip an assertion / sanitizer. + ts.Disk.traffic_out.Content = Testers.ExcludesExpression( + "received signal|failed assertion", "ATS must not crash reloading certs under load") + ts.Disk.traffic_out.Content += Testers.ExcludesExpression( + "AddressSanitizer|use-after-free|runtime error:", "no memory-safety error reloading certs under load") + return ts + + def run(self) -> None: + '''Configure and run the TestRun.''' + tr = Test.AddTestRun("reload certs while TLS handshakes are in flight") + tr.Processes.Default.StartBefore(self._ts) + tr.Processes.Default.Command = ( + f'{sys.executable} {os.path.join(Test.TestDirectory, "tls_reload_under_load_client.py")} ' + f'-p {self._ts.Variables.ssl_port} --sni bar.com --ssldir {self._ts.Variables.SSLDir} ' + f'--live-cert signed-bar.pem ' + f'--v2-cert {os.path.join(self._ts.Variables.SSLDir, "signed2-bar.pem")} ' + f'--reloads 3 --duration 8 --concurrency 4') + # traffic_ctl (invoked by the client to reload) needs the runroot environment. + tr.Processes.Default.Env = self._ts.Env + tr.Processes.Default.ReturnCode = 0 + tr.Processes.Default.Streams.All = Testers.ContainsExpression("RESULT=PASS", "load + reload run must pass") + tr.Processes.Default.Streams.All += Testers.ContainsExpression("FAILURES=0", "no handshake may fail across the reload") + tr.Processes.Default.Streams.All += Testers.ContainsExpression( + "CERT_CHANGED=1", "the swapped-in certificate must be served after the reload") + tr.StillRunningAfter = self._ts + + +TestTlsReloadUnderLoad().run() diff --git a/tests/gold_tests/tls/tls_reload_under_load_client.py b/tests/gold_tests/tls/tls_reload_under_load_client.py new file mode 100644 index 0000000000..c4ed0577a3 --- /dev/null +++ b/tests/gold_tests/tls/tls_reload_under_load_client.py @@ -0,0 +1,183 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Drive continuous concurrent TLS handshakes against ATS while a certificate +reload (traffic_ctl config reload) happens mid-flight. + +This stresses the SSL/BIO ownership boundary of the layered TLS VConnection: the +SSL context list is rebuilt while inbound handshakes and request/response writes +are in flight. A correct server keeps serving (no failed handshakes, no crash) +and the swapped-in certificate takes effect. + +The script is self-contained: worker threads hammer handshakes for the whole run +while the main thread swaps the cert file and reloads ATS at a known offset, so +the reload provably overlaps in-flight handshakes. +""" + +import argparse +import collections +import hashlib +import os +import shutil +import socket +import ssl +import subprocess +import sys +import threading +import time + + +def make_context() -> ssl.SSLContext: + # We validate handshake mechanics and cert identity (by fingerprint), not the + # trust chain, so disable verification to keep the client independent of which + # signer is currently installed. + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + return ctx + + +class Stats: + + def __init__(self) -> None: + self.lock = threading.Lock() + self.ok = 0 + self.fail = 0 + self.errors: collections.Counter = collections.Counter() + self.pre_fps: collections.Counter = collections.Counter() + self.post_fps: collections.Counter = collections.Counter() + + +def worker( + args: argparse.Namespace, stats: Stats, stop: threading.Event, reload_started: threading.Event, + reload_settled: threading.Event) -> None: + ctx = make_context() + while not stop.is_set(): + try: + with socket.create_connection((args.host, args.port), timeout=args.timeout) as sock: + with ctx.wrap_socket(sock, server_hostname=args.sni) as ssock: + der = ssock.getpeercert(binary_form=True) + fp = hashlib.sha256(der).hexdigest()[:16] if der else "none" + request = (f"GET / HTTP/1.1\r\nHost: {args.sni}\r\n" + "Connection: close\r\n\r\n").encode() + ssock.sendall(request) + resp = ssock.recv(256) + if not resp.startswith(b"HTTP/1."): + with stats.lock: + stats.fail += 1 + stats.errors["bad_response"] += 1 + continue + with stats.lock: + stats.ok += 1 + # Bucket the served cert by phase. Ignore the transition window so + # pre==v1 and post==v2 cleanly, regardless of reload latency. + if not reload_started.is_set(): + stats.pre_fps[fp] += 1 + elif reload_settled.is_set(): + stats.post_fps[fp] += 1 + except Exception as exc: # noqa: BLE001 - any failure is a test signal + with stats.lock: + stats.fail += 1 + stats.errors[type(exc).__name__] += 1 + + +def do_reload(env_traffic_ctl: str) -> None: + subprocess.run([env_traffic_ctl, "config", "reload"], check=True, timeout=30) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--host", default="127.0.0.1") + parser.add_argument("-p", "--port", type=int, required=True) + parser.add_argument("--sni", default="bar.com") + parser.add_argument("--ssldir", required=True, help="dir holding the live cert that gets swapped") + parser.add_argument("--live-cert", default="signed-bar.pem", help="cert filename inside --ssldir") + parser.add_argument("--v2-cert", required=True, help="source cert to copy over the live cert at reload") + parser.add_argument("--traffic-ctl", default="traffic_ctl") + parser.add_argument("--duration", type=float, default=8.0) + parser.add_argument("--concurrency", type=int, default=4) + parser.add_argument("--timeout", type=float, default=5.0) + parser.add_argument("--reload-at", type=float, default=2.0) + parser.add_argument("--reloads", type=int, default=3) + parser.add_argument("--reload-spacing", type=float, default=0.7) + parser.add_argument("--settle", type=float, default=0.8, help="grace after last reload before sampling post cert") + parser.add_argument("--min-ok", type=int, default=20) + args = parser.parse_args() + + stats = Stats() + stop = threading.Event() + reload_started = threading.Event() + reload_settled = threading.Event() + + workers = [ + threading.Thread(target=worker, args=(args, stats, stop, reload_started, reload_settled), daemon=True) + for _ in range(args.concurrency) + ] + start = time.monotonic() + for t in workers: + t.start() + + # Let load ramp up so the reload lands on top of in-flight handshakes. + time.sleep(args.reload_at) + + live_path = os.path.join(args.ssldir, args.live_cert) + reloads_done = 0 + reload_error = "" + try: + # Swap to v2 once; the first reload installs it. Further reloads keep + # rebuilding the SSL store under sustained load. + shutil.copyfile(args.v2_cert, live_path) + os.utime(live_path, None) + reload_started.set() + for i in range(args.reloads): + do_reload(args.traffic_ctl) + reloads_done += 1 + if i != args.reloads - 1: + time.sleep(args.reload_spacing) + except Exception as exc: # noqa: BLE001 + reload_error = f"{type(exc).__name__}: {exc}" + + time.sleep(args.settle) + reload_settled.set() + + elapsed = time.monotonic() - start + if args.duration > elapsed: + time.sleep(args.duration - elapsed) + stop.set() + for t in workers: + t.join(timeout=args.timeout + 2) + + pre_fp = stats.pre_fps.most_common(1)[0][0] if stats.pre_fps else "none" + post_fp = stats.post_fps.most_common(1)[0][0] if stats.post_fps else "none" + cert_changed = int(pre_fp != "none" and post_fp != "none" and pre_fp != post_fp) + + passed = ( + reload_error == "" and reloads_done == args.reloads and stats.fail == 0 and stats.ok >= args.min_ok and cert_changed == 1) + + print(f"HANDSHAKES_OK={stats.ok}") + print(f"FAILURES={stats.fail}") + print(f"RELOADS_DONE={reloads_done}") + print(f"CERT_CHANGED={cert_changed}") + print(f"PRE_FP={pre_fp} POST_FP={post_fp}") + if stats.errors: + print(f"ERROR_BREAKDOWN={dict(stats.errors)}") + if reload_error: + print(f"RELOAD_ERROR={reload_error}") + print(f"RESULT={'PASS' if passed else 'FAIL'}") + return 0 if passed else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/gold_tests/tls/tls_renegotiation.test.py b/tests/gold_tests/tls/tls_renegotiation.test.py new file mode 100644 index 0000000000..70fecc14df --- /dev/null +++ b/tests/gold_tests/tls/tls_renegotiation.test.py @@ -0,0 +1,124 @@ +''' +With client renegotiation disallowed (the default), a TLSv1.2 client that asks to +renegotiate must have its connection refused without taking down the server. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys + +Test.Summary = __doc__ + +# Renegotiation only exists in TLS 1.2 and earlier. +Test.SkipUnless(Condition.HasOpenSSLVersion("1.1.1"), Condition.HasLegacyTLSSupport()) + + +class TestRenegotiationRefused: + '''Verify a refused client-initiated renegotiation does not crash ATS.''' + + _server_counter: int = 0 + _ts_counter: int = 0 + + def __init__(self) -> None: + '''Declare the test Processes.''' + self._server = self._configure_server() + self._ts = self._configure_trafficserver() + + def _configure_server(self) -> 'Process': + '''Configure the origin server. + + :return: The origin server Process. + ''' + server = Test.MakeOriginServer(f'server-{TestRenegotiationRefused._server_counter}') + TestRenegotiationRefused._server_counter += 1 + + request_header = {"headers": "GET / HTTP/1.1\r\nHost: example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} + response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": "ok"} + server.addResponse("sessionlog.json", request_header, response_header) + return server + + def _configure_trafficserver(self) -> 'Process': + '''Configure Traffic Server to refuse client renegotiation (the default). + + :return: The Traffic Server Process. + ''' + ts = Test.MakeATSProcess(f'ts-{TestRenegotiationRefused._ts_counter}', enable_tls=True) + TestRenegotiationRefused._ts_counter += 1 + + ts.addSSLfile("ssl/server.pem") + ts.addSSLfile("ssl/server.key") + ts.Disk.ssl_multicert_yaml.AddLines( + """ +ssl_multicert: + - dest_ip: "*" + ssl_cert_name: server.pem + ssl_key_name: server.key +""".split("\n")) + ts.Disk.records_config.update( + { + 'proxy.config.ssl.server.cert.path': f'{ts.Variables.SSLDir}', + 'proxy.config.ssl.server.private_key.path': f'{ts.Variables.SSLDir}', + # The secure default: never honor a client-initiated renegotiation. + 'proxy.config.ssl.allow_client_renegotiation': 0, + # Renegotiation requires a sub-TLS1.3 protocol. + 'proxy.config.ssl.TLSv1_3.enabled': 0, + 'proxy.config.ssl.TLSv1_2': 1, + # So the renegotiation-detection log line below is emitted. + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'ssl_load', + }) + ts.Disk.remap_config.AddLine(f'map / http://127.0.0.1:{self._server.Variables.Port}') + + # The refused renegotiation must not abort the process. + ts.Disk.traffic_out.Content = Testers.ExcludesExpression( + "received signal|failed assertion", "ATS must refuse the renegotiation without crashing") + # ...and it must actually reach the renegotiation-detection path (otherwise + # a no-crash pass could mean the client never managed to renegotiate at all). + ts.Disk.traffic_out.Content += Testers.ContainsExpression( + "trying to renegotiate from the client", "ATS must detect the client-initiated renegotiation") + return ts + + def run(self) -> None: + '''Configure and run the TestRuns.''' + # Complete a TLSv1.2 handshake, then ask to renegotiate. ATS must detect + # and refuse the renegotiation without aborting the process. + tr = Test.AddTestRun("client renegotiation is refused without crashing ATS") + tr.Processes.Default.StartBefore(self._server) + tr.Processes.Default.StartBefore(self._ts) + tr.Processes.Default.Command = ( + f'{sys.executable} {os.path.join(Test.TestDirectory, "tls_renegotiation_client.py")} ' + f'-p {self._ts.Variables.ssl_port} -s example.com') + tr.Processes.Default.ReturnCode = 0 + tr.StillRunningAfter = self._ts + tr.StillRunningAfter = self._server + + # ATS survived, so a normal request still succeeds. + tr = Test.AddTestRun("server still serves requests after the refused renegotiation") + tr.MakeCurlCommand( + ( + "-v --http1.1 --tls-max 1.2 --tlsv1.2 --ciphers DEFAULT@SECLEVEL=0 -k " + f"--resolve 'example.com:{self._ts.Variables.ssl_port}:127.0.0.1' " + f"https://example.com:{self._ts.Variables.ssl_port}/";), + ts=self._ts) + tr.Processes.Default.ReturnCode = 0 + # curl -v writes the response status line to stderr. + tr.Processes.Default.Streams.stderr = Testers.ContainsExpression( + "HTTP/1.1 200 OK", "request after renegotiation should succeed") + tr.StillRunningAfter = self._ts + + +TestRenegotiationRefused().run() diff --git a/tests/gold_tests/tls/tls_renegotiation_allowed.test.py b/tests/gold_tests/tls/tls_renegotiation_allowed.test.py new file mode 100644 index 0000000000..a5f58ff358 --- /dev/null +++ b/tests/gold_tests/tls/tls_renegotiation_allowed.test.py @@ -0,0 +1,145 @@ +''' +With client renegotiation allowed, a TLSv1.2 client that asks to renegotiate must +get a prompt answer from ATS (the renegotiation either completes or is refused +with an alert) rather than hanging. ATS must flush the SSL protocol output the +renegotiation generates; otherwise the answer is never sent and the client +stalls. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys + +from ports import get_port + +Test.Summary = __doc__ + +# Renegotiation only exists in TLS 1.2 and earlier. +Test.SkipUnless(Condition.HasOpenSSLVersion("1.1.1"), Condition.HasLegacyTLSSupport()) + + +class TestRenegotiationAllowed: + '''Verify an allowed client renegotiation is answered promptly, not stranded.''' + + _server_counter: int = 0 + _ts_counter: int = 0 + + def __init__(self) -> None: + '''Declare the test Processes.''' + self._server = self._configure_server() + self._ts = self._configure_trafficserver() + + def _configure_server(self) -> 'Process': + '''Configure a plain HTTP/1.0 origin that closes each connection. + + A python http.server origin answers each request on its own connection and + closes it, so ATS's client-facing TLS write face is fully drained between + requests. When the renegotiation ClientHello then arrives there is no + in-flight transport write to coincidentally carry the renegotiation answer + out -- exactly the condition under which a proxy that fails to flush its own + SSL protocol output would strand the answer and hang the client. (The + Proxy-Verifier microserver keeps the response connection in a state that + masks the stall, so it is unsuitable for this regression.) + + :return: The origin server Process. + ''' + server = Test.Processes.Process(f'server-{TestRenegotiationAllowed._server_counter}') + TestRenegotiationAllowed._server_counter += 1 + + origin_dir = os.path.join(Test.RunDirectory, "origin") + os.makedirs(origin_dir, exist_ok=True) + with open(os.path.join(origin_dir, "index.html"), "w") as f: + f.write("ok\n") + + origin_port = get_port(server, "http_port") + server.Command = f'{sys.executable} -m http.server {origin_port} --bind 127.0.0.1 --directory {origin_dir}' + server.Ready = When.PortOpenv4(origin_port) + # The server is a long-running process the harness terminates at the end of + # the test, so its exit status is whatever the signal leaves behind. + server.ReturnCode = Any(None, 0, -2, -15) + return server + + def _configure_trafficserver(self) -> 'Process': + '''Configure Traffic Server to honor client renegotiation. + + :return: The Traffic Server Process. + ''' + ts = Test.MakeATSProcess(f'ts-{TestRenegotiationAllowed._ts_counter}', enable_tls=True) + TestRenegotiationAllowed._ts_counter += 1 + + ts.addSSLfile("ssl/server.pem") + ts.addSSLfile("ssl/server.key") + ts.Disk.ssl_multicert_yaml.AddLines( + """ +ssl_multicert: + - dest_ip: "*" + ssl_cert_name: server.pem + ssl_key_name: server.key +""".split("\n")) + ts.Disk.records_config.update( + { + 'proxy.config.ssl.server.cert.path': f'{ts.Variables.SSLDir}', + 'proxy.config.ssl.server.private_key.path': f'{ts.Variables.SSLDir}', + # The operator opts in: honor a client-initiated renegotiation. + 'proxy.config.ssl.allow_client_renegotiation': 1, + # Renegotiation requires a sub-TLS1.3 protocol. + 'proxy.config.ssl.TLSv1_3.enabled': 0, + 'proxy.config.ssl.TLSv1_2': 1, + }) + ts.Disk.remap_config.AddLine(f'map / http://127.0.0.1:{self._server.Variables.http_port}') + + # The renegotiation must not abort the process. + ts.Disk.traffic_out.Content = Testers.ExcludesExpression( + "received signal|failed assertion", "ATS must service the renegotiation without crashing") + return ts + + def run(self) -> None: + '''Configure and run the TestRuns.''' + # Complete a TLSv1.2 handshake and one request, let the response drain, ask + # to renegotiate, then drive a second request. ATS must answer the + # renegotiation promptly; the helper reports RENEGOTIATION-STALLED (and + # exits non-zero) if the client hangs waiting for the answer. + tr = Test.AddTestRun("client renegotiation is answered promptly") + tr.Processes.Default.StartBefore(self._server) + tr.Processes.Default.StartBefore(self._ts) + tr.Processes.Default.Command = ( + f'{sys.executable} {os.path.join(Test.TestDirectory, "tls_renegotiation_allowed_client.py")} ' + f'-p {self._ts.Variables.ssl_port} -s example.com') + tr.Processes.Default.ReturnCode = 0 + tr.Processes.Default.Streams.stdout = Testers.ContainsExpression( + "RENEGOTIATION-COMPLETED|RENEGOTIATION-REFUSED-PROMPTLY", "the renegotiation must be answered, not stranded") + tr.Processes.Default.Streams.stdout += Testers.ExcludesExpression( + "RENEGOTIATION-STALLED", "the client must not hang waiting for the renegotiation answer") + tr.StillRunningAfter = self._ts + tr.StillRunningAfter = self._server + + # ATS survived and keeps serving, so a normal request still succeeds. + tr = Test.AddTestRun("server still serves requests after the renegotiation") + tr.MakeCurlCommand( + ( + "-v --http1.1 --tls-max 1.2 --tlsv1.2 --ciphers DEFAULT@SECLEVEL=0 -k " + f"--resolve 'example.com:{self._ts.Variables.ssl_port}:127.0.0.1' " + f"https://example.com:{self._ts.Variables.ssl_port}/";), + ts=self._ts) + tr.Processes.Default.ReturnCode = 0 + # curl -v writes the response status line to stderr. + tr.Processes.Default.Streams.stderr = Testers.ContainsExpression( + "HTTP/1.1 200 OK", "request after renegotiation should succeed") + tr.StillRunningAfter = self._ts + + +TestRenegotiationAllowed().run() diff --git a/tests/gold_tests/tls/tls_renegotiation_allowed_client.py b/tests/gold_tests/tls/tls_renegotiation_allowed_client.py new file mode 100644 index 0000000000..ea4711be04 --- /dev/null +++ b/tests/gold_tests/tls/tls_renegotiation_allowed_client.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +''' +Drive an OpenSSL s_client connection that completes a TLSv1.2 handshake and +then requests a client-initiated renegotiation, with ATS configured to allow +client renegotiation. Verifies that ATS answers the renegotiation attempt +promptly -- either by completing it (libraries that honor it) or by +delivering its refusal alert (OpenSSL 3.x never honors a client +renegotiation unless SSL_OP_ALLOW_CLIENT_RENEGOTIATION is set) -- instead of +stranding the TLS response in its write buffer and leaving the client +hanging. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import shlex +import subprocess +import sys +import threading +import time + + +class OutputCollector: + '''Accumulate a subprocess's merged output from a reader thread.''' + + def __init__(self, stream): + self._stream = stream + self._lock = threading.Lock() + self._data = b'' + self._thread = threading.Thread(target=self._read, daemon=True) + self._thread.start() + + def _read(self): + while True: + chunk = self._stream.read1(4096) + if not chunk: + break + with self._lock: + self._data += chunk + + def data(self) -> bytes: + with self._lock: + return self._data + + +def wait_for(predicate, timeout: float) -> bool: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if predicate(): + return True + time.sleep(0.1) + return predicate() + + +def main() -> int: + parser = argparse.ArgumentParser(description='Trigger a TLSv1.2 client renegotiation against ATS and require a prompt answer.') + parser.add_argument('-p', '--ats-port', type=int, dest='ats_port', required=True, help='ATS TLS port number') + parser.add_argument('-s', '--server-name', type=str, dest='sni', default='example.com', help='SNI server name') + args = parser.parse_args() + + # -tls1_2 forces a protocol that supports renegotiation (TLS 1.3 has none). + # No -quiet: it suppresses s_client's interpretation of the "R" command, so + # the renegotiation request would otherwise be sent as plain application data. + cmd = shlex.split( + f'openssl s_client -connect 127.0.0.1:{args.ats_port} -tls1_2 -servername {args.sni} -cipher DEFAULT@SECLEVEL=0') + proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + output = OutputCollector(proc.stdout) + verdict = 'RENEGOTIATION-STALLED' + + try: + request = b'GET / HTTP/1.1\r\nHost: ' + args.sni.encode() + b'\r\n\r\n' + proc.stdin.write(request) + proc.stdin.flush() + if not wait_for(lambda: b'HTTP/1.1 200 OK' in output.data(), 10): + print('NO-FIRST-RESPONSE: the request before the renegotiation never completed') + proc.kill() + return 1 + + # Let the first response fully drain so the proxy's TLS write face goes + # idle: that is exactly when the renegotiation answer has no in-flight + # transport write to ride out on, so a layered proxy that fails to flush + # SSL protocol output on its own strands the answer and the client hangs. + time.sleep(2) + + # s_client interprets a line containing just "R" as a request to + # renegotiate the session. + proc.stdin.write(b'R\n') + proc.stdin.flush() + time.sleep(1) + # A second request provokes the client into resolving the + # renegotiation it started: it either completes it and sends the + # request, or it reads the server's refusal alert and exits. + try: + proc.stdin.write(request) + proc.stdin.flush() + except BrokenPipeError: + # The client already exited on the refusal alert. + pass + + def renegotiation_answered(): + return output.data().count(b'HTTP/1.1 200 OK') >= 2 or proc.poll() is not None + + if wait_for(renegotiation_answered, 10): + if output.data().count(b'HTTP/1.1 200 OK') >= 2: + verdict = 'RENEGOTIATION-COMPLETED' + else: + verdict = 'RENEGOTIATION-REFUSED-PROMPTLY' + finally: + if proc.poll() is None: + proc.kill() + proc.wait() + + sys.stdout.write(output.data().decode('utf-8', errors='replace')) + print(verdict) + return 0 if verdict != 'RENEGOTIATION-STALLED' else 1 + + +if __name__ == '__main__': + exit(main()) diff --git a/tests/gold_tests/tls/tls_renegotiation_client.py b/tests/gold_tests/tls/tls_renegotiation_client.py new file mode 100644 index 0000000000..46af21aef0 --- /dev/null +++ b/tests/gold_tests/tls/tls_renegotiation_client.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +''' +Drive an OpenSSL s_client connection that completes a TLSv1.2 handshake and +then requests a client-initiated renegotiation. Used to verify that Traffic +Server refuses the renegotiation without crashing. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import shlex +import subprocess +import sys +import time + + +def main() -> None: + parser = argparse.ArgumentParser(description='Trigger a TLSv1.2 client renegotiation against ATS.') + parser.add_argument('-p', '--ats-port', type=int, dest='ats_port', required=True, help='ATS TLS port number') + parser.add_argument('-s', '--server-name', type=str, dest='sni', default='example.com', help='SNI server name') + args = parser.parse_args() + + # -tls1_2 forces a protocol that supports renegotiation (TLS 1.3 has none). + # No -quiet: it suppresses s_client's interpretation of the "R" command, so + # the renegotiation request would otherwise be sent as plain application data. + cmd = shlex.split( + f'openssl s_client -connect 127.0.0.1:{args.ats_port} -tls1_2 -servername {args.sni} -cipher DEFAULT@SECLEVEL=0') + proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + + # Complete the handshake and one request, then send "R" on its own line: + # s_client interprets that as a request to renegotiate the session. + try: + proc.stdin.write(b'GET / HTTP/1.1\r\nHost: ' + args.sni.encode() + b'\r\n\r\n') + proc.stdin.flush() + time.sleep(2) + proc.stdin.write(b'R\n') + proc.stdin.flush() + time.sleep(2) + proc.stdin.write(b'GET / HTTP/1.1\r\nHost: ' + args.sni.encode() + b'\r\n\r\n') + proc.stdin.flush() + out, _ = proc.communicate(timeout=10) + except (subprocess.TimeoutExpired, BrokenPipeError): + proc.kill() + out, _ = proc.communicate() + + sys.stdout.write(out.decode('utf-8', errors='replace')) + # The client's own exit status is irrelevant: ATS legitimately tears down + # the connection on the refused renegotiation. The test asserts on ATS + # staying alive, not on this process. + exit(0) + + +if __name__ == '__main__': + main()
