This is an automated email from the ASF dual-hosted git repository. cmcfarlen pushed a commit to branch 10.0.x in repository https://gitbox.apache.org/repos/asf/trafficserver.git
commit 766fb35065e5a18efd0fbc6a87868acc2acfc9ee Author: Chris McFarlen <[email protected]> AuthorDate: Tue Apr 30 12:57:14 2024 -0500 Restore derived metrics using new metrics (#11297) * Restore derived metrics using new metrics * lock derived metrics during update * lines,comment * format * docs (cherry picked from commit dc3e578cd2d6134c148ead10f60e64fac22f1fa4) --- include/tsutil/Metrics.h | 30 ++++++++++++++ src/iocore/eventsystem/RecProcess.cc | 4 ++ src/proxy/http/HttpConfig.cc | 42 ++++++++++++++++++++ src/tsutil/Metrics.cc | 75 +++++++++++++++++++++++++++++++++++ src/tsutil/unit_tests/test_Metrics.cc | 34 ++++++++++++++++ 5 files changed, 185 insertions(+) diff --git a/include/tsutil/Metrics.h b/include/tsutil/Metrics.h index c8a0917442..80da30fd71 100644 --- a/include/tsutil/Metrics.h +++ b/include/tsutil/Metrics.h @@ -31,6 +31,7 @@ #include <cstdint> #include <string> #include <string_view> +#include <variant> #include "swoc/MemSpan.h" @@ -506,6 +507,35 @@ public: }; // class Counter + /** + * Derive metrics by summing a set of other metrics. + * + */ + class Derived + { + public: + struct DerivedMetricSpec { + using MetricSpec = std::variant<Metrics::AtomicType *, Metrics::IdType, std::string_view>; + std::string_view derived_name; + std::initializer_list<MetricSpec> derived_from; + }; + + /** + * Create new metrics derived from existing metrics. + * + * This function will create new metrics from a list of existing metrics. The existing metric can + * be specified by name, id or a pointer to the metric. + */ + static void derive(const std::initializer_list<DerivedMetricSpec> &metrics); + + /** + * Update derived metrics. + * + * This static function should be called periodically to update derived metrics. + */ + static void update_derived(); + }; + }; // class Metrics } // namespace ts diff --git a/src/iocore/eventsystem/RecProcess.cc b/src/iocore/eventsystem/RecProcess.cc index 2cc282181e..8bc467e656 100644 --- a/src/iocore/eventsystem/RecProcess.cc +++ b/src/iocore/eventsystem/RecProcess.cc @@ -23,6 +23,7 @@ #include "tscore/ink_platform.h" #include "tscore/EventNotify.h" +#include "tsutil/Metrics.h" #include "iocore/eventsystem/Tasks.h" @@ -90,6 +91,9 @@ struct raw_stat_sync_cont : public Continuation { RecExecRawStatSyncCbs(); Dbg(dbg_ctl_statsproc, "raw_stat_sync_cont() processed"); + // This needs to be called periodically even after the old metrics sync is removed + ts::Metrics::Derived::update_derived(); + return EVENT_CONT; } }; diff --git a/src/proxy/http/HttpConfig.cc b/src/proxy/http/HttpConfig.cc index e3cb777546..170be374fc 100644 --- a/src/proxy/http/HttpConfig.cc +++ b/src/proxy/http/HttpConfig.cc @@ -544,6 +544,48 @@ register_stat_callbacks() Metrics::Counter::createPtr("proxy.process.http.user_agent_response_header_total_size"); http_rsb.websocket_current_active_client_connections = Metrics::Gauge::createPtr("proxy.process.http.websocket.current_active_client_connections"); + + Metrics::Derived::derive({ + // Total bytes of client request body + headers + {"proxy.process.http.user_agent_total_request_bytes", + {http_rsb.user_agent_request_document_total_size, http_rsb.user_agent_request_header_total_size} }, + // Total bytes of client response body + headers + {"proxy.process.http.user_agent_total_response_bytes", + {http_rsb.user_agent_response_document_total_size, http_rsb.user_agent_response_header_total_size} }, + // Total bytes of origin server request body + headers + {"proxy.process.http.origin_server_total_request_bytes", + {http_rsb.origin_server_request_document_total_size, http_rsb.origin_server_request_header_total_size} }, + // Total bytes of origin server response body + headers + {"proxy.process.http.origin_server_total_response_bytes", + {http_rsb.origin_server_response_document_total_size, http_rsb.origin_server_response_header_total_size} }, + // Total bytes of client request and response (total traffic to and from clients) + {"proxy.process.user_agent_total_bytes", + {"proxy.process.http.user_agent_total_request_bytes", "proxy.process.http.user_agent_total_response_bytes"} }, + // Total bytes of origin/parent request and response + {"proxy.process.origin_server_total_bytes", + {"proxy.process.http.origin_server_total_request_bytes", "proxy.process.http.origin_server_total_response_bytes", + http_rsb.parent_proxy_request_total_bytes, http_rsb.parent_proxy_response_total_bytes} }, + // Total requests which are cache hits + {"proxy.process.cache_total_hits", + {http_rsb.cache_hit_fresh, http_rsb.cache_hit_reval, http_rsb.cache_hit_ims, http_rsb.cache_hit_stale_served} }, + // Total requests which are cache misses + {"proxy.process.cache_total_misses", + {http_rsb.cache_miss_cold, http_rsb.cache_miss_changed, http_rsb.cache_miss_client_no_cache, http_rsb.cache_miss_ims, + http_rsb.cache_miss_uncacheable} }, + // Total of all server connections (sum of origins and parent connections) + {"proxy.process.current_server_connections", {http_rsb.current_server_connections, http_rsb.current_parent_proxy_connections} }, + // Total requests, both hits and misses (this is slightly superfluous, but assures correct percentage calculations) + {"proxy.process.cache_total_requests", {"proxy.process.cache_total_hits", "proxy.process.cache_total_misses"} }, + // Total cache requests bytes which are cache hits + {"proxy.process.cache_total_hits_bytes", + {http_rsb.tcp_hit_user_agent_bytes, http_rsb.tcp_refresh_hit_user_agent_bytes, http_rsb.tcp_ims_hit_user_agent_bytes} }, + // Total cache requests bytes which are cache misses + {"proxy.process.cache_total_misses_bytes", + {http_rsb.tcp_miss_user_agent_bytes, http_rsb.tcp_expired_miss_user_agent_bytes, http_rsb.tcp_refresh_miss_user_agent_bytes, + http_rsb.tcp_ims_miss_user_agent_bytes} }, + // Total request bytes, both hits and misses + {"proxy.process.cache_total_bytes", {"proxy.process.cache_total_hits_bytes", "proxy.process.cache_total_misses_bytes"}} + }); } static bool diff --git a/src/tsutil/Metrics.cc b/src/tsutil/Metrics.cc index 47eb108b81..bcc5c79ef6 100644 --- a/src/tsutil/Metrics.cc +++ b/src/tsutil/Metrics.cc @@ -23,6 +23,9 @@ #include "tsutil/Assert.h" #include <memory> +#include <mutex> +#include <variant> +#include <vector> #include "tsutil/Metrics.h" namespace ts @@ -201,4 +204,76 @@ Metrics::iterator::next() _it = _makeId(blob, offset); } +namespace details +{ + struct DerivedMetric { + Metrics::IdType metric; + std::vector<Metrics::AtomicType *> derived_from; + }; + + struct DerivativeMetrics { + std::vector<DerivedMetric> metrics; + std::mutex metrics_lock; + + void + update() + { + auto &instance = Metrics::instance(); + std::lock_guard l(metrics_lock); + + for (auto &m : metrics) { + int64_t sum = 0; + + for (auto d : m.derived_from) { + sum += d->load(); + } + instance[m.metric].store(sum); + } + } + + void + push_back(const DerivedMetric &m) + { + std::lock_guard l(metrics_lock); + metrics.push_back(std::move(m)); + } + + static DerivativeMetrics & + instance() + { + static DerivativeMetrics theDerivedMetrics; + return theDerivedMetrics; + } + }; + +} // namespace details + +void +Metrics::Derived::derive(const std::initializer_list<Metrics::Derived::DerivedMetricSpec> &metrics) +{ + auto &instance = Metrics::instance(); + + for (auto &m : metrics) { + details::DerivedMetric dm{}; + dm.metric = instance._create(m.derived_name); + + for (auto &d : m.derived_from) { + if (std::holds_alternative<Metrics::AtomicType *>(d)) { + dm.derived_from.push_back(std::get<Metrics::AtomicType *>(d)); + } else if (std::holds_alternative<Metrics::IdType>(d)) { + dm.derived_from.push_back(instance.lookup(std::get<Metrics::IdType>(d))); + } else if (std::holds_alternative<std::string_view>(d)) { + dm.derived_from.push_back(instance.lookup(instance.lookup(std::get<std::string_view>(d)))); + } + } + details::DerivativeMetrics::instance().push_back(dm); + } +} + +void +Metrics::Derived::update_derived() +{ + details::DerivativeMetrics::instance().update(); +} + } // namespace ts diff --git a/src/tsutil/unit_tests/test_Metrics.cc b/src/tsutil/unit_tests/test_Metrics.cc index c5a736f2fb..551381a0a4 100644 --- a/src/tsutil/unit_tests/test_Metrics.cc +++ b/src/tsutil/unit_tests/test_Metrics.cc @@ -98,4 +98,38 @@ TEST_CASE("Metrics", "[libtsapi][Metrics]") REQUIRE(mid == fmid); } + + SECTION("derived") + { + auto a = Metrics::Counter::createPtr("m-a"); + auto b = Metrics::Counter::createPtr("m-b"); + auto c = Metrics::Counter::createPtr("m-c"); + auto d = Metrics::Counter::createPtr("m-d"); + auto e = Metrics::Counter::createPtr("m-e"); + ts::Metrics::Derived::derive({ + {"derived-a-c", {a, b, c} }, // test using ptr + {"derived-cd", {m.lookup("m-c"), "m-d"}}, // using IdType and string + {"derived-ce", {"derived-cd", "m-e"} } // using another derived + }); + + auto derived = m.lookup("derived-a-c"); + auto derivedcd = m.lookup("derived-cd"); + auto derivedce = m.lookup("derived-ce"); + REQUIRE(derived != ts::Metrics::NOT_FOUND); + + REQUIRE(m[derived].load() == 0); + + a->increment(1); + b->increment(1); + b->increment(1); + c->increment(1); + d->increment(4); + e->increment(5); + + ts::Metrics::Derived::update_derived(); + + REQUIRE(m[derived].load() == 4); + REQUIRE(m[derivedcd].load() == 5); + REQUIRE(m[derivedce].load() == 10); + } }
