This is an automated email from the ASF dual-hosted git repository. cmcfarlen pushed a commit to branch 10.2.x in repository https://gitbox.apache.org/repos/asf/trafficserver.git
commit f122d34c03f167a6569132121e1c5fa9da157e3a Author: Brian Neradt <[email protected]> AuthorDate: Thu Feb 19 12:24:13 2026 -0600 Add proxy.process.http.429_responses metric (#12878) Add a new HTTP stat to track 429 (Too Many Requests) responses sent to clients. This is useful for monitoring rate-limiting behavior, as the rate_limit plugin defaults to returning 429 when limits are exceeded. The metric follows the same pattern as existing per-status-code counters (400-416) across HttpConfig, HttpTransact, traffic_top, and traffic_logstats. Also enhance the ATSReplayTest framework with a metric_checks feature that allows replay YAML files to declare expected metric values. After traffic completes, follow-up test runs automatically verify the metrics via traffic_ctl. This required creating the ATS process on the Test object (rather than a TestRun) when post-traffic verification is needed, so that ATS persists across test runs. (cherry picked from commit 4c21c453192e86d72e721d77764d91f30158746e) --- .../statistics/core/http-response-code.en.rst | 3 + doc/developer-guide/testing/autests.en.rst | 22 ++++++ include/proxy/http/HttpConfig.h | 1 + src/proxy/http/HttpConfig.cc | 1 + src/proxy/http/HttpTransact.cc | 3 + src/traffic_logstats/logstats.cc | 5 ++ src/traffic_logstats/tests/logstats.json | 3 + src/traffic_logstats/tests/logstats.summary | 1 + src/traffic_top/stats.h | 1 + tests/gold_tests/autest-site/ats_replay.test.ext | 44 ++++++++++- .../statistics/metric_response_429.test.py | 20 +++++ .../replay/metric_response_429.replay.yaml | 85 ++++++++++++++++++++++ 12 files changed, 188 insertions(+), 1 deletion(-) diff --git a/doc/admin-guide/monitoring/statistics/core/http-response-code.en.rst b/doc/admin-guide/monitoring/statistics/core/http-response-code.en.rst index 5efeada65c..56904f284c 100644 --- a/doc/admin-guide/monitoring/statistics/core/http-response-code.en.rst +++ b/doc/admin-guide/monitoring/statistics/core/http-response-code.en.rst @@ -136,6 +136,9 @@ code means. .. ts:stat:: global proxy.process.http.416_responses integer :type: counter +.. ts:stat:: global proxy.process.http.429_responses integer + :type: counter + .. ts:stat:: global proxy.process.http.4xx_responses integer :type: counter diff --git a/doc/developer-guide/testing/autests.en.rst b/doc/developer-guide/testing/autests.en.rst index 64cf40fe4f..04a02e5712 100644 --- a/doc/developer-guide/testing/autests.en.rst +++ b/doc/developer-guide/testing/autests.en.rst @@ -251,6 +251,11 @@ YAML node). Here is an example: - expression: "Unwanted message in diags.log" description: "Verify this does NOT appear in diags.log" + # Optional: Verify ATS metric values after traffic completes. + metric_checks: + - metric: "proxy.process.http.200_responses" + value: 1 + # Traffic specification using Proxy Verifier format # client-request and server-response generate request and response traffic # toward the ATS proxy. @@ -353,6 +358,7 @@ The ``autest`` section configures the test environment: - **remap_config**: List of remap rules (string or dict format) - **copy_to_config_dir**: List of files/directories to copy to ATS config directory - **log_validation**: Log validation rules for ``traffic_out`` and ``diags_log`` + - **metric_checks**: List of metric name/value pairs to verify after traffic completes Log Validation ~~~~~~~~~~~~~~ @@ -375,6 +381,22 @@ The ``log_validation`` section allows you to verify the contents of - expression: "Plugin initialized" description: "Verify plugin loaded" +Metric Verification +~~~~~~~~~~~~~~~~~~~~ + +The ``metric_checks`` section allows you to verify |TS| metric values after all +traffic in the test has completed. Each entry specifies a metric name and its +expected value. After the traffic test run, ``ATSReplayTest`` automatically adds +follow-up test runs that use ``traffic_ctl metric get`` to verify each metric. + +.. code-block:: yaml + + metric_checks: + - metric: "proxy.process.http.429_responses" + value: 1 + - metric: "proxy.process.http.200_responses" + value: 2 + Sessions and Transactions ~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/include/proxy/http/HttpConfig.h b/include/proxy/http/HttpConfig.h index 29cabb50a6..bf797f7a9c 100644 --- a/include/proxy/http/HttpConfig.h +++ b/include/proxy/http/HttpConfig.h @@ -285,6 +285,7 @@ struct HttpStatsBlock { Metrics::Counter::AtomicType *response_status_414_count; Metrics::Counter::AtomicType *response_status_415_count; Metrics::Counter::AtomicType *response_status_416_count; + Metrics::Counter::AtomicType *response_status_429_count; Metrics::Counter::AtomicType *response_status_4xx_count; Metrics::Counter::AtomicType *response_status_500_count; Metrics::Counter::AtomicType *response_status_501_count; diff --git a/src/proxy/http/HttpConfig.cc b/src/proxy/http/HttpConfig.cc index 7875a79f3b..7a06f4c972 100644 --- a/src/proxy/http/HttpConfig.cc +++ b/src/proxy/http/HttpConfig.cc @@ -436,6 +436,7 @@ register_stat_callbacks() http_rsb.response_status_414_count = Metrics::Counter::createPtr("proxy.process.http.414_responses"); http_rsb.response_status_415_count = Metrics::Counter::createPtr("proxy.process.http.415_responses"); http_rsb.response_status_416_count = Metrics::Counter::createPtr("proxy.process.http.416_responses"); + http_rsb.response_status_429_count = Metrics::Counter::createPtr("proxy.process.http.429_responses"); http_rsb.response_status_4xx_count = Metrics::Counter::createPtr("proxy.process.http.4xx_responses"); http_rsb.response_status_500_count = Metrics::Counter::createPtr("proxy.process.http.500_responses"); http_rsb.response_status_501_count = Metrics::Counter::createPtr("proxy.process.http.501_responses"); diff --git a/src/proxy/http/HttpTransact.cc b/src/proxy/http/HttpTransact.cc index a30849e698..27c090d616 100644 --- a/src/proxy/http/HttpTransact.cc +++ b/src/proxy/http/HttpTransact.cc @@ -8756,6 +8756,9 @@ HttpTransact::client_result_stat(State *s, ink_hrtime total_time, ink_hrtime req case 416: Metrics::Counter::increment(http_rsb.response_status_416_count); break; + case 429: + Metrics::Counter::increment(http_rsb.response_status_429_count); + break; case 500: Metrics::Counter::increment(http_rsb.response_status_500_count); break; diff --git a/src/traffic_logstats/logstats.cc b/src/traffic_logstats/logstats.cc index fec49eb02b..f9099b8c9c 100644 --- a/src/traffic_logstats/logstats.cc +++ b/src/traffic_logstats/logstats.cc @@ -209,6 +209,7 @@ struct OriginStats { StatsCounter c_415; StatsCounter c_416; StatsCounter c_417; + StatsCounter c_429; StatsCounter c_4xx; StatsCounter c_500; StatsCounter c_501; @@ -1083,6 +1084,9 @@ update_codes(OriginStats *stat, int code, int size) case 417: update_counter(stat->codes.c_417, size); break; + case 429: + update_counter(stat->codes.c_429, size); + break; // 500's case 500: @@ -2116,6 +2120,7 @@ print_detail_stats(const OriginStats *stat, bool json, bool concise) format_line(json ? "status.415" : "415 Unsupported Media Type", stat->codes.c_415, stat->total, json, concise); format_line(json ? "status.416" : "416 Req Range Not Satisfiable", stat->codes.c_416, stat->total, json, concise); format_line(json ? "status.417" : "417 Expectation Failed", stat->codes.c_417, stat->total, json, concise); + format_line(json ? "status.429" : "429 Too Many Requests", stat->codes.c_429, stat->total, json, concise); format_line(json ? "status.4xx" : "4xx Total", stat->codes.c_4xx, stat->total, json, concise); if (!json) { diff --git a/src/traffic_logstats/tests/logstats.json b/src/traffic_logstats/tests/logstats.json index f12731142b..15c5893cfa 100644 --- a/src/traffic_logstats/tests/logstats.json +++ b/src/traffic_logstats/tests/logstats.json @@ -53,6 +53,7 @@ "status.415" : { "req": "0", "req_pct": "0.00", "bytes": "0", "bytes_pct": "0.00" }, "status.416" : { "req": "0", "req_pct": "0.00", "bytes": "0", "bytes_pct": "0.00" }, "status.417" : { "req": "0", "req_pct": "0.00", "bytes": "0", "bytes_pct": "0.00" }, + "status.429" : { "req": "0", "req_pct": "0.00", "bytes": "0", "bytes_pct": "0.00" }, "status.4xx" : { "req": "0", "req_pct": "0.00", "bytes": "0", "bytes_pct": "0.00" }, "status.500" : { "req": "0", "req_pct": "0.00", "bytes": "0", "bytes_pct": "0.00" }, "status.501" : { "req": "0", "req_pct": "0.00", "bytes": "0", "bytes_pct": "0.00" }, @@ -181,6 +182,7 @@ "status.415" : { "req": "0", "req_pct": "0.00", "bytes": "0", "bytes_pct": "0.00" }, "status.416" : { "req": "0", "req_pct": "0.00", "bytes": "0", "bytes_pct": "0.00" }, "status.417" : { "req": "0", "req_pct": "0.00", "bytes": "0", "bytes_pct": "0.00" }, + "status.429" : { "req": "0", "req_pct": "0.00", "bytes": "0", "bytes_pct": "0.00" }, "status.4xx" : { "req": "0", "req_pct": "0.00", "bytes": "0", "bytes_pct": "0.00" }, "status.500" : { "req": "0", "req_pct": "0.00", "bytes": "0", "bytes_pct": "0.00" }, "status.501" : { "req": "0", "req_pct": "0.00", "bytes": "0", "bytes_pct": "0.00" }, @@ -309,6 +311,7 @@ "status.415" : { "req": "0", "req_pct": "0.00", "bytes": "0", "bytes_pct": "0.00" }, "status.416" : { "req": "0", "req_pct": "0.00", "bytes": "0", "bytes_pct": "0.00" }, "status.417" : { "req": "0", "req_pct": "0.00", "bytes": "0", "bytes_pct": "0.00" }, + "status.429" : { "req": "0", "req_pct": "0.00", "bytes": "0", "bytes_pct": "0.00" }, "status.4xx" : { "req": "0", "req_pct": "0.00", "bytes": "0", "bytes_pct": "0.00" }, "status.500" : { "req": "0", "req_pct": "0.00", "bytes": "0", "bytes_pct": "0.00" }, "status.501" : { "req": "0", "req_pct": "0.00", "bytes": "0", "bytes_pct": "0.00" }, diff --git a/src/traffic_logstats/tests/logstats.summary b/src/traffic_logstats/tests/logstats.summary index 111812f9a4..9a297455d0 100644 --- a/src/traffic_logstats/tests/logstats.summary +++ b/src/traffic_logstats/tests/logstats.summary @@ -66,6 +66,7 @@ HTTP return codes Count Percent Bytes Percent 415 Unsupported Media Type 0 0.00% 0.00KB 0.00% 416 Req Range Not Satisfiable 0 0.00% 0.00KB 0.00% 417 Expectation Failed 0 0.00% 0.00KB 0.00% +429 Too Many Requests 0 0.00% 0.00KB 0.00% 4xx Total 0 0.00% 0.00KB 0.00% 500 Internal Server Error 0 0.00% 0.00KB 0.00% diff --git a/src/traffic_top/stats.h b/src/traffic_top/stats.h index 035151eb5e..fa0aae1b69 100644 --- a/src/traffic_top/stats.h +++ b/src/traffic_top/stats.h @@ -231,6 +231,7 @@ public: lookup_table.insert(make_pair("414", LookupItem("414", "proxy.process.http.414_responses", 5))); lookup_table.insert(make_pair("415", LookupItem("415", "proxy.process.http.415_responses", 5))); lookup_table.insert(make_pair("416", LookupItem("416", "proxy.process.http.416_responses", 5))); + lookup_table.insert(make_pair("429", LookupItem("429", "proxy.process.http.429_responses", 5))); lookup_table.insert(make_pair("4xx", LookupItem("4xx", "proxy.process.http.4xx_responses", 5))); lookup_table.insert(make_pair("500", LookupItem("500", "proxy.process.http.500_responses", 5))); lookup_table.insert(make_pair("501", LookupItem("501", "proxy.process.http.501_responses", 5))); diff --git a/tests/gold_tests/autest-site/ats_replay.test.ext b/tests/gold_tests/autest-site/ats_replay.test.ext index c041c42770..b731f8a5ea 100644 --- a/tests/gold_tests/autest-site/ats_replay.test.ext +++ b/tests/gold_tests/autest-site/ats_replay.test.ext @@ -19,6 +19,7 @@ Implement general-purpose ATS test extensions using proxy verifier replay files. from typing import Optional import os +import re import yaml @@ -126,6 +127,24 @@ def configure_ats(obj: 'TestRun', server: 'Process', ats_config: dict, dns: Opti return ts +def _requires_persistent_ats(ats_config: dict) -> bool: + '''Determine whether ATS must span multiple test runs. + + Some features, such as metric verification, require additional test runs + using the ATS process after traffic completes. In those cases the ATS + process must be owned by Test rather than a TestRun so it persists across + test runs. + + :param ats_config: The autest.ats configuration from the replay file. + :returns: True if ATS must span multiple test runs. + ''' + + # This will become more complicated over time. For example, it would be nice + # to add transaction log support which would require a longer-lived ATS + # process. + return bool(ats_config.get('metric_checks')) + + def ATSReplayTest(obj, replay_file: str): '''Create a TestRun that configures ATS and runs HTTP traffic using the replay file. @@ -195,7 +214,10 @@ def ATSReplayTest(obj, replay_file: str): raise ValueError(f"Replay file {replay_file} does not contain 'autest.ats' section") ats_config = autest_config['ats'] enable_tls = ats_config.get('enable_tls', False) - ts = configure_ats(tr, server=server, ats_config=ats_config, dns=dns) + metric_checks = ats_config.get('metric_checks', []) + + ats_owner = obj if _requires_persistent_ats(ats_config) else tr + ts = configure_ats(ats_owner, server=server, ats_config=ats_config, dns=dns) # Proxy Verifier Client configuration. if not 'client' in autest_config: @@ -230,6 +252,26 @@ def ATSReplayTest(obj, replay_file: str): ts.StartBefore(server) client.StartBefore(ts) + # Metric verification test runs. + if metric_checks: + tr.StillRunningAfter = ts + + wait_tr = obj.AddTestRun('Wait for stats to propagate') + wait_tr.Processes.Default.Command = 'sleep 2' + wait_tr.Processes.Default.ReturnCode = 0 + wait_tr.StillRunningAfter = ts + + for check in metric_checks: + metric_name = check['metric'] + expected_value = check['value'] + check_tr = obj.AddTestRun(f'Verify {metric_name} == {expected_value}') + check_tr.Processes.Default.Command = f'traffic_ctl metric get {metric_name}' + check_tr.Processes.Default.Env = ts.Env + check_tr.Processes.Default.ReturnCode = 0 + check_tr.Processes.Default.Streams.All = Testers.ContainsExpression( + f'^{re.escape(metric_name)}\\s+{expected_value}$', f'{metric_name} should be {expected_value}') + check_tr.StillRunningAfter = ts + return tr diff --git a/tests/gold_tests/statistics/metric_response_429.test.py b/tests/gold_tests/statistics/metric_response_429.test.py new file mode 100644 index 0000000000..ae4e5e5838 --- /dev/null +++ b/tests/gold_tests/statistics/metric_response_429.test.py @@ -0,0 +1,20 @@ +'''Verify the proxy.process.http.429_responses metric.''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = __doc__ + +Test.ATSReplayTest(replay_file="replay/metric_response_429.replay.yaml") diff --git a/tests/gold_tests/statistics/replay/metric_response_429.replay.yaml b/tests/gold_tests/statistics/replay/metric_response_429.replay.yaml new file mode 100644 index 0000000000..b326bdac06 --- /dev/null +++ b/tests/gold_tests/statistics/replay/metric_response_429.replay.yaml @@ -0,0 +1,85 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +meta: + version: "1.0" + +autest: + description: 'Verify proxy.process.http.429_responses metric is incremented' + + dns: + name: 'dns' + + server: + name: 'server' + + client: + name: 'client' + + ats: + name: 'ts' + + process_config: + enable_cache: false + + remap_config: + - from: "http://example.com/"; + to: "http://backend.example.com:{SERVER_HTTP_PORT}/"; + + metric_checks: + - metric: "proxy.process.http.429_responses" + value: 1 + +sessions: +- transactions: + + - client-request: + method: GET + url: /rate-limited + version: '1.1' + headers: + fields: + - [Host, example.com] + - [uuid, 429-response] + + server-response: + status: 429 + reason: Too Many Requests + headers: + fields: + - [Content-Length, "0"] + + proxy-response: + status: 429 + + - client-request: + method: GET + url: /ok + version: '1.1' + headers: + fields: + - [Host, example.com] + - [uuid, 200-response] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [Content-Length, "0"] + + proxy-response: + status: 200
