This is an automated email from the ASF dual-hosted git repository. wkaras pushed a commit to branch tunnel_stats in repository https://gitbox.apache.org/repos/asf/trafficserver.git
commit e0f71fddfd831c57840365ef2210804d66699632 Author: Walt Karas <[email protected]> AuthorDate: Fri Feb 10 05:22:24 2023 +0000 Add 14 metrics for TCP connections created for tunnels. Add current and total metrics for TCP connetions towards clients for blind TCP tunnels, and TLS tunnel, forward, and partial blind tunnel SNI-based tunnels. Add current and total metrics for TCP connetions towards servers, for blind TCP tunnels and TLS tunnels. Only partial blind tunnel SNI-based tunnels are counted as TLS tunnels on the outgoing side, because they are only SNI-based tunnels where ATS termitates the TLS connection form the client and originates a new one towards the server. --- .../statistics/core/http-connection.en.rst | 25 ++++++- .../monitoring/statistics/core/ssl.en.rst | 60 ++++++++++++++++ iocore/eventsystem/I_VConnection.h | 8 ++- iocore/net/Net.cc | 42 +++++++++-- iocore/net/P_Net.h | 14 ++++ iocore/net/P_SSLNetVConnection.h | 3 + iocore/net/P_UnixNetVConnection.h | 18 +++++ iocore/net/SSLNetVConnection.cc | 70 ++++++++++++++++++ iocore/net/UnixNetVConnection.cc | 54 ++++++++++++++ proxy/ProxyTransaction.cc | 8 +++ proxy/ProxyTransaction.h | 2 + proxy/http/HttpSM.cc | 6 ++ tests/gold_tests/connect/connect.test.py | 37 +++++++++- tests/gold_tests/connect/gold/metrics.gold | 21 ++++++ tests/gold_tests/remap/gold/remap-ws-metrics.gold | 21 ++++++ tests/gold_tests/remap/remap_ws.test.py | 34 +++++++++ .../tls/gold/tls-partial-blind-tunnel-metrics.gold | 21 ++++++ .../tls/gold/tls-tunnel-forward-metrics.gold | 21 ++++++ tests/gold_tests/tls/gold/tls-tunnel-metrics.gold | 14 ++++ .../tls/tls_partial_blind_tunnel.test.py | 32 +++++++++ tests/gold_tests/tls/tls_tunnel.test.py | 25 +++++++ tests/gold_tests/tls/tls_tunnel_forward.test.py | 32 +++++++++ tests/tools/stdout_wait | 82 ++++++++++++++++++++++ 23 files changed, 640 insertions(+), 10 deletions(-) diff --git a/doc/admin-guide/monitoring/statistics/core/http-connection.en.rst b/doc/admin-guide/monitoring/statistics/core/http-connection.en.rst index c11d5c7bf2..3672f0022f 100644 --- a/doc/admin-guide/monitoring/statistics/core/http-connection.en.rst +++ b/doc/admin-guide/monitoring/statistics/core/http-connection.en.rst @@ -164,10 +164,33 @@ HTTP Connection Counts the number of times current parent or next parent was detected +.. ts:stat:: global proxy.process.tunnel.total_client_connections_blind_tcp integer + :type: counter + + Total number of non-TLS TCP connections for tunnels where the far end is the client + initiated with an HTTP request (such as a CONNECT or WebSocket request). + +.. ts:stat:: global proxy.process.tunnel.current_client_connections_blind_tcp integer + :type: counter + + Current number of non-TLS TCP connections for tunnels where the far end is the client + initiated with an HTTP request (such as a CONNECT or WebSocket request). + +.. ts:stat:: global proxy.process.tunnel.total_server_connections_blind_tcp integer + :type: counter + + Total number of TCP connections for tunnels where the far end is the server, + except for those counted by ``proxy.process.tunnel.total_server_connections_tls`` + +.. ts:stat:: global proxy.process.tunnel.current_server_connections_blind_tcp integer + :type: counter + + Current number of TCP connections for tunnels where the far end is the server, + except for those counted by ``proxy.process.tunnel.current_server_connections_tls`` + HTTP/2 ------ - .. ts:stat:: global proxy.process.http2.total_client_connections integer :type: counter diff --git a/doc/admin-guide/monitoring/statistics/core/ssl.en.rst b/doc/admin-guide/monitoring/statistics/core/ssl.en.rst index e18bae11c6..59eaa918a5 100644 --- a/doc/admin-guide/monitoring/statistics/core/ssl.en.rst +++ b/doc/admin-guide/monitoring/statistics/core/ssl.en.rst @@ -238,6 +238,66 @@ SSL/TLS A gauge of current active SNI Routing Tunnels. +.. ts:stat:: global proxy.process.tunnel.total_client_connections_tls_tunnel integer + :type: counter + + Total number of TCP connections for TLS tunnels where the far end is the client + created based on a ``tunnel_route`` key in a table in the :file:`sni.yaml` file. + +.. ts:stat:: global proxy.process.tunnel.current_client_connections_tls_tunnel integer + :type: counter + + Current number of TCP connections for TLS tunnels where the far end is the client + created based on a ``tunnel_route`` key in a table in the :file:`sni.yaml` file. + +.. ts:stat:: global proxy.process.tunnel.total_client_connections_tls_forward integer + :type: counter + + Total number of TCP connections for TLS tunnels where the far end is the client + created based on a ``forward_route`` key in a table in the :file:`sni.yaml` file. + +.. ts:stat:: global proxy.process.tunnel.current_client_connections_tls_forward integer + :type: counter + + Current number of TCP connections for TLS tunnels where the far end is the client + created based on a ``forward_route`` key in a table in the :file:`sni.yaml` file. + +.. ts:stat:: global proxy.process.tunnel.total_client_connections_tls_partial_blind integer + :type: counter + + Total number of TCP connections for TLS tunnels where the far end is the client + created based on a ``partial_blind_route`` key in a table in the :file:`sni.yaml` file. + +.. ts:stat:: global proxy.process.tunnel.current_client_connections_tls_partial_blind integer + :type: counter + + Current number of TCP connections for TLS tunnels where the far end is the client + created based on a ``partial_blind_route`` key in a table in the :file:`sni.yaml` file. + +.. ts:stat:: global proxy.process.tunnel.total_client_connections_tls_http integer + :type: counter + + Total number of TLS connections for tunnels where the far end is the client + initiated with an HTTP request. + +.. ts:stat:: global proxy.process.tunnel.current_client_connections_tls_http integer + :type: counter + + Current number of TLS connections for tunnels where the far end is the client + initiated with an HTTP request. + +.. ts:stat:: global proxy.process.tunnel.total_server_connections_tls integer + :type: counter + + Total number of TCP connections for TLS tunnels where the far end is the server + created based on a ``partial_blind_route`` key in a table in the :file:`sni.yaml` file. + +.. ts:stat:: global proxy.process.tunnel.current_server_connections_tls integer + :type: counter + + Current number of TCP connections for TLS tunnels where the far end is the server + created based on a ``partial_blind_route`` key in a table in the :file:`sni.yaml` file. + .. _pre-warming-tls-tunnel-stats: Pre-warming TLS Tunnel diff --git a/iocore/eventsystem/I_VConnection.h b/iocore/eventsystem/I_VConnection.h index c60915e78a..c656c040af 100644 --- a/iocore/eventsystem/I_VConnection.h +++ b/iocore/eventsystem/I_VConnection.h @@ -362,7 +362,13 @@ public: return false; } -public: + // This function should be called when the VConnection is a tunnel endpoint. By default, a VConnection does not care if it + // is a tunnel endpoint. + virtual void + make_tunnel_endpoint() + { + } + /** The error code from the last error. diff --git a/iocore/net/Net.cc b/iocore/net/Net.cc index 5d3e64fe40..5bc393f8a5 100644 --- a/iocore/net/Net.cc +++ b/iocore/net/Net.cc @@ -98,13 +98,27 @@ register_net_stats() }; const std::pair<const char *, Net_Stats> non_persistent[] = { - {"proxy.process.net.accepts_currently_open", net_accepts_currently_open_stat }, - {"proxy.process.net.connections_currently_open", net_connections_currently_open_stat }, - {"proxy.process.net.default_inactivity_timeout_applied", default_inactivity_timeout_applied_stat}, - {"proxy.process.net.default_inactivity_timeout_count", default_inactivity_timeout_count_stat }, - {"proxy.process.net.dynamic_keep_alive_timeout_in_count", keep_alive_queue_timeout_count_stat }, - {"proxy.process.net.dynamic_keep_alive_timeout_in_total", keep_alive_queue_timeout_total_stat }, - {"proxy.process.socks.connections_currently_open", socks_connections_currently_open_stat }, + {"proxy.process.net.accepts_currently_open", net_accepts_currently_open_stat }, + {"proxy.process.net.connections_currently_open", net_connections_currently_open_stat }, + {"proxy.process.net.default_inactivity_timeout_applied", default_inactivity_timeout_applied_stat }, + {"proxy.process.net.default_inactivity_timeout_count", default_inactivity_timeout_count_stat }, + {"proxy.process.net.dynamic_keep_alive_timeout_in_count", keep_alive_queue_timeout_count_stat }, + {"proxy.process.net.dynamic_keep_alive_timeout_in_total", keep_alive_queue_timeout_total_stat }, + {"proxy.process.socks.connections_currently_open", socks_connections_currently_open_stat }, + {"proxy.process.tunnel.total_client_connections_blind_tcp", tunnel_total_client_connections_blind_tcp_stat }, + {"proxy.process.tunnel.current_client_connections_blind_tcp", tunnel_current_client_connections_blind_tcp_stat }, + {"proxy.process.tunnel.total_server_connections_blind_tcp", tunnel_total_server_connections_blind_tcp_stat }, + {"proxy.process.tunnel.current_server_connections_blind_tcp", tunnel_current_server_connections_blind_tcp_stat }, + {"proxy.process.tunnel.total_client_connections_tls_tunnel", tunnel_total_client_connections_tls_tunnel_stat }, + {"proxy.process.tunnel.current_client_connections_tls_tunnel", tunnel_current_client_connections_tls_tunnel_stat }, + {"proxy.process.tunnel.total_client_connections_tls_forward", tunnel_total_client_connections_tls_forward_stat }, + {"proxy.process.tunnel.current_client_connections_tls_forward", tunnel_current_client_connections_tls_forward_stat }, + {"proxy.process.tunnel.total_client_connections_tls_partial_blind", tunnel_total_client_connections_tls_partial_blind_stat }, + {"proxy.process.tunnel.current_client_connections_tls_partial_blind", tunnel_current_client_connections_tls_partial_blind_stat}, + {"proxy.process.tunnel.total_client_connections_tls_http", tunnel_total_client_connections_tls_http_stat }, + {"proxy.process.tunnel.current_client_connections_tls_http", tunnel_current_client_connections_tls_http_stat }, + {"proxy.process.tunnel.total_server_connections_tls", tunnel_total_server_connections_tls_stat }, + {"proxy.process.tunnel.current_server_connections_tls", tunnel_current_server_connections_tls_stat }, }; for (auto &p : persistent) { @@ -129,6 +143,20 @@ register_net_stats() NET_CLEAR_DYN_STAT(keep_alive_queue_timeout_count_stat); NET_CLEAR_DYN_STAT(default_inactivity_timeout_count_stat); NET_CLEAR_DYN_STAT(default_inactivity_timeout_applied_stat); + NET_CLEAR_DYN_STAT(tunnel_total_client_connections_blind_tcp_stat); + NET_CLEAR_DYN_STAT(tunnel_current_client_connections_blind_tcp_stat); + NET_CLEAR_DYN_STAT(tunnel_total_server_connections_blind_tcp_stat); + NET_CLEAR_DYN_STAT(tunnel_current_server_connections_blind_tcp_stat); + NET_CLEAR_DYN_STAT(tunnel_total_client_connections_tls_tunnel_stat); + NET_CLEAR_DYN_STAT(tunnel_current_client_connections_tls_tunnel_stat); + NET_CLEAR_DYN_STAT(tunnel_total_client_connections_tls_forward_stat); + NET_CLEAR_DYN_STAT(tunnel_current_client_connections_tls_forward_stat); + NET_CLEAR_DYN_STAT(tunnel_total_client_connections_tls_partial_blind_stat); + NET_CLEAR_DYN_STAT(tunnel_current_client_connections_tls_partial_blind_stat); + NET_CLEAR_DYN_STAT(tunnel_total_client_connections_tls_http_stat); + NET_CLEAR_DYN_STAT(tunnel_current_client_connections_tls_http_stat); + NET_CLEAR_DYN_STAT(tunnel_total_server_connections_tls_stat); + NET_CLEAR_DYN_STAT(tunnel_current_server_connections_tls_stat); RecRegisterRawStat(net_rsb, RECT_PROCESS, "proxy.process.tcp.total_accepts", RECD_INT, RECP_NON_PERSISTENT, static_cast<int>(net_tcp_accept_stat), RecRawStatSyncSum); diff --git a/iocore/net/P_Net.h b/iocore/net/P_Net.h index 09c2fa1251..96d6b9e962 100644 --- a/iocore/net/P_Net.h +++ b/iocore/net/P_Net.h @@ -57,6 +57,20 @@ enum Net_Stats { net_connections_throttled_in_stat, net_connections_throttled_out_stat, net_requests_max_throttled_in_stat, + tunnel_total_client_connections_blind_tcp_stat, + tunnel_current_client_connections_blind_tcp_stat, + tunnel_total_server_connections_blind_tcp_stat, + tunnel_current_server_connections_blind_tcp_stat, + tunnel_total_client_connections_tls_tunnel_stat, + tunnel_current_client_connections_tls_tunnel_stat, + tunnel_total_server_connections_tls_stat, + tunnel_current_server_connections_tls_stat, + tunnel_total_client_connections_tls_forward_stat, + tunnel_current_client_connections_tls_forward_stat, + tunnel_total_client_connections_tls_partial_blind_stat, + tunnel_current_client_connections_tls_partial_blind_stat, + tunnel_total_client_connections_tls_http_stat, + tunnel_current_client_connections_tls_http_stat, Net_Stat_Count }; diff --git a/iocore/net/P_SSLNetVConnection.h b/iocore/net/P_SSLNetVConnection.h index e2f1dcca60..917e4e0edd 100644 --- a/iocore/net/P_SSLNetVConnection.h +++ b/iocore/net/P_SSLNetVConnection.h @@ -479,6 +479,9 @@ private: ssl_error_t _ssl_write_buffer(const void *buf, int64_t nbytes, int64_t &nwritten); ssl_error_t _ssl_connect(); ssl_error_t _ssl_accept(); + + void _in_context_tunnel() override; + void _out_context_tunnel() override; }; typedef int (SSLNetVConnection::*SSLNetVConnHandler)(int, void *); diff --git a/iocore/net/P_UnixNetVConnection.h b/iocore/net/P_UnixNetVConnection.h index fbc9747a85..d299bd71d2 100644 --- a/iocore/net/P_UnixNetVConnection.h +++ b/iocore/net/P_UnixNetVConnection.h @@ -44,6 +44,8 @@ struct PollDescriptor; enum tcp_congestion_control_t { CLIENT_SIDE, SERVER_SIDE }; +// WARNING: many or most of the member functions of UnixNetVConnection should only be used when it is instantiated +// directly. They should not be used when UnixNetVConnection is a base class. class UnixNetVConnection : public NetVConnection, public NetEvent { public: @@ -225,9 +227,25 @@ public: friend void write_to_net_io(NetHandler *, UnixNetVConnection *, EThread *); + // set_context() should be called before calling this member function. + void make_tunnel_endpoint() override; + + bool + is_tunnel_endpoint() const + { + return _is_tunnel_endpoint; + } + private: virtual void *_prepareForMigration(); virtual NetProcessor *_getNetProcessor(); + bool _is_tunnel_endpoint{false}; + + // Called by make_tunnel_endpiont() when the far end of the TCP connection is the active/client end. + virtual void _in_context_tunnel(); + + // Called by make_tunnel_endpiont() when the far end of the TCP connection is the passive/server end. + virtual void _out_context_tunnel(); }; extern ClassAllocator<UnixNetVConnection> netVCAllocator; diff --git a/iocore/net/SSLNetVConnection.cc b/iocore/net/SSLNetVConnection.cc index bcb6b49396..56f1465341 100644 --- a/iocore/net/SSLNetVConnection.cc +++ b/iocore/net/SSLNetVConnection.cc @@ -996,6 +996,35 @@ SSLNetVConnection::free(EThread *t) } con.close(); + if (is_tunnel_endpoint()) { + ink_assert(get_context() != NET_VCONNECTION_UNSET); + + int c; + + if (get_context() == NET_VCONNECTION_IN) { + switch (get_tunnel_type()) { + case SNIRoutingType::BLIND: + c = tunnel_current_client_connections_tls_tunnel_stat; + break; + case SNIRoutingType::FORWARD: + c = tunnel_current_client_connections_tls_forward_stat; + break; + case SNIRoutingType::PARTIAL_BLIND: + c = tunnel_current_client_connections_tls_partial_blind_stat; + break; + default: + c = tunnel_current_client_connections_tls_http_stat; + break; + } + } else { // NET_VCONNECTION_OUT + // Never a tunnel type for out (to server) context. + ink_assert(get_tunnel_type() == SNIRoutingType::NONE); + + c = tunnel_current_server_connections_tls_stat; + } + NET_DECREMENT_DYN_STAT(c); + } + #if TS_HAS_TLS_EARLY_DATA if (_early_data_reader != nullptr) { _early_data_reader->dealloc(); @@ -1908,6 +1937,47 @@ SSLNetVConnection::populate(Connection &con, Continuation *c, void *arg) return EVENT_DONE; } +void +SSLNetVConnection::_in_context_tunnel() +{ + ink_assert(get_context() == NET_VCONNECTION_IN); + + int t, c; + + switch (get_tunnel_type()) { + case SNIRoutingType::BLIND: + t = tunnel_total_client_connections_tls_tunnel_stat; + c = tunnel_current_client_connections_tls_tunnel_stat; + break; + case SNIRoutingType::FORWARD: + t = tunnel_total_client_connections_tls_forward_stat; + c = tunnel_current_client_connections_tls_forward_stat; + break; + case SNIRoutingType::PARTIAL_BLIND: + t = tunnel_total_client_connections_tls_partial_blind_stat; + c = tunnel_current_client_connections_tls_partial_blind_stat; + break; + default: + t = tunnel_total_client_connections_tls_http_stat; + c = tunnel_current_client_connections_tls_http_stat; + break; + } + NET_INCREMENT_DYN_STAT(t); + NET_INCREMENT_DYN_STAT(c); +} + +void +SSLNetVConnection::_out_context_tunnel() +{ + ink_assert(get_context() == NET_VCONNECTION_OUT); + + // Never a tunnel type for out (to server) context. + ink_assert(get_tunnel_type() == SNIRoutingType::NONE); + + NET_INCREMENT_DYN_STAT(tunnel_total_server_connections_tls_stat); + NET_INCREMENT_DYN_STAT(tunnel_current_server_connections_tls_stat); +} + void SSLNetVConnection::increment_ssl_version_metric(int version) const { diff --git a/iocore/net/UnixNetVConnection.cc b/iocore/net/UnixNetVConnection.cc index 842a8b98b5..fde54b60e1 100644 --- a/iocore/net/UnixNetVConnection.cc +++ b/iocore/net/UnixNetVConnection.cc @@ -1288,6 +1288,8 @@ UnixNetVConnection::clear() void UnixNetVConnection::free(EThread *t) { + Debug("iocore_net", "Entering UnixNetVConnection::free()"); + ink_release_assert(t == this_ethread()); // close socket fd @@ -1296,6 +1298,23 @@ UnixNetVConnection::free(EThread *t) } con.close(); + if (is_tunnel_endpoint()) { + Debug("iocore_net", "Freeing UnixNetVConnection that is tunnel endpoint"); + + int c; + switch (get_context()) { + case NET_VCONNECTION_IN: + c = tunnel_current_client_connections_blind_tcp_stat; + break; + case NET_VCONNECTION_OUT: + c = tunnel_current_server_connections_blind_tcp_stat; + break; + default: + ink_release_assert(false); + } + NET_DECREMENT_DYN_STAT(c); + } + clear(); SET_CONTINUATION_HANDLER(this, &UnixNetVConnection::startEvent); ink_assert(con.fd == NO_FD); @@ -1495,3 +1514,38 @@ UnixNetVConnection::set_tcp_congestion_control(int side) return -1; #endif } + +void +UnixNetVConnection::make_tunnel_endpoint() +{ + Debug("iocore_net", "Entering UnixNetVConnection::make_tunnel_endpoint()"); + + ink_assert(!_is_tunnel_endpoint); + + _is_tunnel_endpoint = true; + + switch (get_context()) { + case NET_VCONNECTION_IN: + _in_context_tunnel(); + break; + case NET_VCONNECTION_OUT: + _out_context_tunnel(); + break; + default: + ink_release_assert(false); + } +} + +void +UnixNetVConnection::_in_context_tunnel() +{ + NET_INCREMENT_DYN_STAT(tunnel_total_client_connections_blind_tcp_stat); + NET_INCREMENT_DYN_STAT(tunnel_current_client_connections_blind_tcp_stat); +} + +void +UnixNetVConnection::_out_context_tunnel() +{ + NET_INCREMENT_DYN_STAT(tunnel_total_server_connections_blind_tcp_stat); + NET_INCREMENT_DYN_STAT(tunnel_current_server_connections_blind_tcp_stat); +} diff --git a/proxy/ProxyTransaction.cc b/proxy/ProxyTransaction.cc index 0198cf8361..4b7b4ca1ae 100644 --- a/proxy/ProxyTransaction.cc +++ b/proxy/ProxyTransaction.cc @@ -276,3 +276,11 @@ void ProxyTransaction::set_close_connection(HTTPHdr &hdr) const { } + +void +ProxyTransaction::make_tunnel_endpoint() +{ + auto nvc = get_netvc(); + ink_assert(nvc != nullptr); + nvc->make_tunnel_endpoint(); +} diff --git a/proxy/ProxyTransaction.h b/proxy/ProxyTransaction.h index 3a8193b5b3..ff6e60ea7c 100644 --- a/proxy/ProxyTransaction.h +++ b/proxy/ProxyTransaction.h @@ -136,6 +136,8 @@ public: bool support_sni() const; + void make_tunnel_endpoint() override; + /// Variables // HttpSessionAccept::Options upstream_outbound_options; // overwritable copy of options diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc index c0087fadef..8205a7a4de 100644 --- a/proxy/http/HttpSM.cc +++ b/proxy/http/HttpSM.cc @@ -7280,6 +7280,9 @@ HttpSM::setup_push_transfer_to_cache() void HttpSM::setup_blind_tunnel(bool send_response_hdr, IOBufferReader *initial) { + ink_assert(server_entry->vc != nullptr); + ink_assert(ua_entry->vc != nullptr); + HttpTunnelConsumer *c_ua; HttpTunnelConsumer *c_os; HttpTunnelProducer *p_ua; @@ -7337,6 +7340,9 @@ HttpSM::setup_blind_tunnel(bool send_response_hdr, IOBufferReader *initial) c_os = tunnel.add_consumer(server_entry->vc, ua_entry->vc, &HttpSM::tunnel_handler_ssl_consumer, HT_HTTP_SERVER, "http server - tunnel"); + ua_entry->vc->make_tunnel_endpoint(); + server_entry->vc->make_tunnel_endpoint(); + // Make the tunnel aware that the entries are bi-directional tunnel.chain(c_os, p_os); tunnel.chain(c_ua, p_ua); diff --git a/tests/gold_tests/connect/connect.test.py b/tests/gold_tests/connect/connect.test.py index 95b25958a7..c521f5a264 100644 --- a/tests/gold_tests/connect/connect.test.py +++ b/tests/gold_tests/connect/connect.test.py @@ -135,7 +135,7 @@ class ConnectViaPVTest: self.ts.Disk.records_config.update({ 'proxy.config.diags.debug.enabled': 1, - 'proxy.config.diags.debug.tags': 'http', + 'proxy.config.diags.debug.tags': 'http|iocore_net|rec', 'proxy.config.http.server_ports': f"{self.ts.Variables.port}", 'proxy.config.http.connect_ports': f"{self.server.Variables.http_port}", }) @@ -160,8 +160,43 @@ class ConnectViaPVTest: tr.StillRunningAfter = self.server tr.StillRunningAfter = self.ts + def __testMetrics(self): + tr = Test.AddTestRun("Reload config") + tr.Processes.Default.Command = ( + f"{Test.Variables.AtsTestToolsDir}/stdout_wait" + + " 'traffic_ctl metric get" + + " proxy.process.http.total_incoming_connections" + + " proxy.process.http.total_client_connections" + + " proxy.process.http.total_client_connections_ipv4" + + " proxy.process.http.total_client_connections_ipv6" + + " proxy.process.http.total_server_connections" + + " proxy.process.http2.total_client_connections" + + " proxy.process.http.connect_requests" + + " proxy.process.tunnel.total_client_connections_blind_tcp" + + " proxy.process.tunnel.current_client_connections_blind_tcp" + + " proxy.process.tunnel.total_server_connections_blind_tcp" + + " proxy.process.tunnel.current_server_connections_blind_tcp" + + " proxy.process.tunnel.total_client_connections_tls_tunnel" + + " proxy.process.tunnel.current_client_connections_tls_tunnel" + + " proxy.process.tunnel.total_client_connections_tls_forward" + + " proxy.process.tunnel.current_client_connections_tls_forward" + + " proxy.process.tunnel.total_client_connections_tls_partial_blind" + + " proxy.process.tunnel.current_client_connections_tls_partial_blind" + + " proxy.process.tunnel.total_client_connections_tls_http" + + " proxy.process.tunnel.current_client_connections_tls_http" + + " proxy.process.tunnel.total_server_connections_tls" + + " proxy.process.tunnel.current_server_connections_tls'" + + f" {Test.TestDirectory}/gold/metrics.gold" + ) + # Need to copy over the environment so traffic_ctl knows where to find the unix domain socket + tr.Processes.Default.Env = self.ts.Env + tr.Processes.Default.ReturnCode = 0 + tr.StillRunningAfter = self.server + tr.StillRunningAfter = self.ts + def run(self): self.runTraffic() + self.__testMetrics() ConnectViaPVTest().run() diff --git a/tests/gold_tests/connect/gold/metrics.gold b/tests/gold_tests/connect/gold/metrics.gold new file mode 100644 index 0000000000..bdbda6443c --- /dev/null +++ b/tests/gold_tests/connect/gold/metrics.gold @@ -0,0 +1,21 @@ +proxy.process.http.total_incoming_connections 1 +proxy.process.http.total_client_connections 1 +proxy.process.http.total_client_connections_ipv4 1 +proxy.process.http.total_client_connections_ipv6 0 +proxy.process.http.total_server_connections 0 +proxy.process.http2.total_client_connections 0 +proxy.process.http.connect_requests 1 +proxy.process.tunnel.total_client_connections_blind_tcp 1 +proxy.process.tunnel.current_client_connections_blind_tcp 0 +proxy.process.tunnel.total_server_connections_blind_tcp 1 +proxy.process.tunnel.current_server_connections_blind_tcp 0 +proxy.process.tunnel.total_client_connections_tls_tunnel 0 +proxy.process.tunnel.current_client_connections_tls_tunnel 0 +proxy.process.tunnel.total_client_connections_tls_forward 0 +proxy.process.tunnel.current_client_connections_tls_forward 0 +proxy.process.tunnel.total_client_connections_tls_partial_blind 0 +proxy.process.tunnel.current_client_connections_tls_partial_blind 0 +proxy.process.tunnel.total_client_connections_tls_http 0 +proxy.process.tunnel.current_client_connections_tls_http 0 +proxy.process.tunnel.total_server_connections_tls 0 +proxy.process.tunnel.current_server_connections_tls 0 diff --git a/tests/gold_tests/remap/gold/remap-ws-metrics.gold b/tests/gold_tests/remap/gold/remap-ws-metrics.gold new file mode 100644 index 0000000000..cf39c20a91 --- /dev/null +++ b/tests/gold_tests/remap/gold/remap-ws-metrics.gold @@ -0,0 +1,21 @@ +proxy.process.http.total_incoming_connections 3 +proxy.process.http.total_client_connections 3 +proxy.process.http.total_client_connections_ipv4 3 +proxy.process.http.total_client_connections_ipv6 0 +proxy.process.http.total_server_connections 2 +proxy.process.http2.total_client_connections 0 +proxy.process.http.connect_requests 0 +proxy.process.tunnel.total_client_connections_blind_tcp 1 +proxy.process.tunnel.current_client_connections_blind_tcp 0 +proxy.process.tunnel.total_server_connections_blind_tcp 2 +proxy.process.tunnel.current_server_connections_blind_tcp 0 +proxy.process.tunnel.total_client_connections_tls_tunnel 0 +proxy.process.tunnel.current_client_connections_tls_tunnel 0 +proxy.process.tunnel.total_client_connections_tls_forward 0 +proxy.process.tunnel.current_client_connections_tls_forward 0 +proxy.process.tunnel.total_client_connections_tls_partial_blind 0 +proxy.process.tunnel.current_client_connections_tls_partial_blind 0 +proxy.process.tunnel.total_client_connections_tls_http 1 +proxy.process.tunnel.current_client_connections_tls_http 0 +proxy.process.tunnel.total_server_connections_tls 0 +proxy.process.tunnel.current_server_connections_tls 0 diff --git a/tests/gold_tests/remap/remap_ws.test.py b/tests/gold_tests/remap/remap_ws.test.py index f87f797824..e1b96d3f78 100644 --- a/tests/gold_tests/remap/remap_ws.test.py +++ b/tests/gold_tests/remap/remap_ws.test.py @@ -77,3 +77,37 @@ tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.stderr = "gold/remap-ws-upgrade-400.gold" tr.StillRunningAfter = server tr.StillRunningAfter = ts + +# Test metrics +tr = Test.AddTestRun() +tr.Processes.Default.Command = ( + f"{Test.Variables.AtsTestToolsDir}/stdout_wait" + + " 'traffic_ctl metric get" + + " proxy.process.http.total_incoming_connections" + + " proxy.process.http.total_client_connections" + + " proxy.process.http.total_client_connections_ipv4" + + " proxy.process.http.total_client_connections_ipv6" + + " proxy.process.http.total_server_connections" + + " proxy.process.http2.total_client_connections" + + " proxy.process.http.connect_requests" + + " proxy.process.tunnel.total_client_connections_blind_tcp" + + " proxy.process.tunnel.current_client_connections_blind_tcp" + + " proxy.process.tunnel.total_server_connections_blind_tcp" + + " proxy.process.tunnel.current_server_connections_blind_tcp" + + " proxy.process.tunnel.total_client_connections_tls_tunnel" + + " proxy.process.tunnel.current_client_connections_tls_tunnel" + + " proxy.process.tunnel.total_client_connections_tls_forward" + + " proxy.process.tunnel.current_client_connections_tls_forward" + + " proxy.process.tunnel.total_client_connections_tls_partial_blind" + + " proxy.process.tunnel.current_client_connections_tls_partial_blind" + + " proxy.process.tunnel.total_client_connections_tls_http" + + " proxy.process.tunnel.current_client_connections_tls_http" + + " proxy.process.tunnel.total_server_connections_tls" + + " proxy.process.tunnel.current_server_connections_tls'" + + f" {Test.TestDirectory}/gold/remap-ws-metrics.gold" +) +# Need to copy over the environment so traffic_ctl knows where to find the unix domain socket +tr.Processes.Default.Env = ts.Env +tr.Processes.Default.ReturnCode = 0 +tr.StillRunningAfter = server +tr.StillRunningAfter = ts diff --git a/tests/gold_tests/tls/gold/tls-partial-blind-tunnel-metrics.gold b/tests/gold_tests/tls/gold/tls-partial-blind-tunnel-metrics.gold new file mode 100644 index 0000000000..fc8da18b10 --- /dev/null +++ b/tests/gold_tests/tls/gold/tls-partial-blind-tunnel-metrics.gold @@ -0,0 +1,21 @@ +proxy.process.http.total_incoming_connections 1 +proxy.process.http.total_client_connections 1 +proxy.process.http.total_client_connections_ipv4 1 +proxy.process.http.total_client_connections_ipv6 0 +proxy.process.http.total_server_connections 0 +proxy.process.http2.total_client_connections 0 +proxy.process.http.connect_requests 1 +proxy.process.tunnel.total_client_connections_blind_tcp 0 +proxy.process.tunnel.current_client_connections_blind_tcp 0 +proxy.process.tunnel.total_server_connections_blind_tcp 0 +proxy.process.tunnel.current_server_connections_blind_tcp 0 +proxy.process.tunnel.total_client_connections_tls_tunnel 0 +proxy.process.tunnel.current_client_connections_tls_tunnel 0 +proxy.process.tunnel.total_client_connections_tls_forward 0 +proxy.process.tunnel.current_client_connections_tls_forward 0 +proxy.process.tunnel.total_client_connections_tls_partial_blind 1 +proxy.process.tunnel.current_client_connections_tls_partial_blind 0 +proxy.process.tunnel.total_client_connections_tls_http 0 +proxy.process.tunnel.current_client_connections_tls_http 0 +proxy.process.tunnel.total_server_connections_tls 1 +proxy.process.tunnel.current_server_connections_tls 0 diff --git a/tests/gold_tests/tls/gold/tls-tunnel-forward-metrics.gold b/tests/gold_tests/tls/gold/tls-tunnel-forward-metrics.gold new file mode 100644 index 0000000000..025d86c556 --- /dev/null +++ b/tests/gold_tests/tls/gold/tls-tunnel-forward-metrics.gold @@ -0,0 +1,21 @@ +proxy.process.http.total_incoming_connections 3 +proxy.process.http.total_client_connections 3 +proxy.process.http.total_client_connections_ipv4 3 +proxy.process.http.total_client_connections_ipv6 0 +proxy.process.http.total_server_connections 0 +proxy.process.http2.total_client_connections 0 +proxy.process.http.connect_requests 3 +proxy.process.tunnel.total_client_connections_blind_tcp 0 +proxy.process.tunnel.current_client_connections_blind_tcp 0 +proxy.process.tunnel.total_server_connections_blind_tcp 3 +proxy.process.tunnel.current_server_connections_blind_tcp 0 +proxy.process.tunnel.total_client_connections_tls_tunnel 1 +proxy.process.tunnel.current_client_connections_tls_tunnel 0 +proxy.process.tunnel.total_client_connections_tls_forward 2 +proxy.process.tunnel.current_client_connections_tls_forward 0 +proxy.process.tunnel.total_client_connections_tls_partial_blind 0 +proxy.process.tunnel.current_client_connections_tls_partial_blind 0 +proxy.process.tunnel.total_client_connections_tls_http 0 +proxy.process.tunnel.current_client_connections_tls_http 0 +proxy.process.tunnel.total_server_connections_tls 0 +proxy.process.tunnel.current_server_connections_tls 0 diff --git a/tests/gold_tests/tls/gold/tls-tunnel-metrics.gold b/tests/gold_tests/tls/gold/tls-tunnel-metrics.gold new file mode 100644 index 0000000000..31a3eedcec --- /dev/null +++ b/tests/gold_tests/tls/gold/tls-tunnel-metrics.gold @@ -0,0 +1,14 @@ +proxy.process.tunnel.total_client_connections_blind_tcp 0 +proxy.process.tunnel.current_client_connections_blind_tcp 0 +proxy.process.tunnel.total_server_connections_blind_tcp 8 +proxy.process.tunnel.current_server_connections_blind_tcp 0 +proxy.process.tunnel.total_client_connections_tls_tunnel 8 +proxy.process.tunnel.current_client_connections_tls_tunnel 0 +proxy.process.tunnel.total_client_connections_tls_forward 0 +proxy.process.tunnel.current_client_connections_tls_forward 0 +proxy.process.tunnel.total_client_connections_tls_partial_blind 0 +proxy.process.tunnel.current_client_connections_tls_partial_blind 0 +proxy.process.tunnel.total_client_connections_tls_http 0 +proxy.process.tunnel.current_client_connections_tls_http 0 +proxy.process.tunnel.total_server_connections_tls 0 +proxy.process.tunnel.current_server_connections_tls 0 diff --git a/tests/gold_tests/tls/tls_partial_blind_tunnel.test.py b/tests/gold_tests/tls/tls_partial_blind_tunnel.test.py index 021eae8885..44f14fca33 100644 --- a/tests/gold_tests/tls/tls_partial_blind_tunnel.test.py +++ b/tests/gold_tests/tls/tls_partial_blind_tunnel.test.py @@ -74,3 +74,35 @@ tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Not Found on Acc "Should not try to remap on Traffic Server") tr.Processes.Default.Streams.All += Testers.ContainsExpression("HTTP/1.1 200 OK", "Should get a successful response") tr.Processes.Default.Streams.All += Testers.ContainsExpression("ok bar", "Body is expected") + +tr = Test.AddTestRun("Test Metrics") +tr.Processes.Default.Command = ( + f"{Test.Variables.AtsTestToolsDir}/stdout_wait" + + " 'traffic_ctl metric get" + + " proxy.process.http.total_incoming_connections" + + " proxy.process.http.total_client_connections" + + " proxy.process.http.total_client_connections_ipv4" + + " proxy.process.http.total_client_connections_ipv6" + + " proxy.process.http.total_server_connections" + + " proxy.process.http2.total_client_connections" + + " proxy.process.http.connect_requests" + + " proxy.process.tunnel.total_client_connections_blind_tcp" + + " proxy.process.tunnel.current_client_connections_blind_tcp" + + " proxy.process.tunnel.total_server_connections_blind_tcp" + + " proxy.process.tunnel.current_server_connections_blind_tcp" + + " proxy.process.tunnel.total_client_connections_tls_tunnel" + + " proxy.process.tunnel.current_client_connections_tls_tunnel" + + " proxy.process.tunnel.total_client_connections_tls_forward" + + " proxy.process.tunnel.current_client_connections_tls_forward" + + " proxy.process.tunnel.total_client_connections_tls_partial_blind" + + " proxy.process.tunnel.current_client_connections_tls_partial_blind" + + " proxy.process.tunnel.total_client_connections_tls_http" + + " proxy.process.tunnel.current_client_connections_tls_http" + + " proxy.process.tunnel.total_server_connections_tls" + + " proxy.process.tunnel.current_server_connections_tls'" + + f" {Test.TestDirectory}/gold/tls-partial-blind-tunnel-metrics.gold" +) +# Need to copy over the environment so traffic_ctl knows where to find the unix domain socket +tr.Processes.Default.Env = ts.Env +tr.Processes.Default.ReturnCode = 0 +tr.StillRunningAfter = ts diff --git a/tests/gold_tests/tls/tls_tunnel.test.py b/tests/gold_tests/tls/tls_tunnel.test.py index fd14c3fe77..9127e96e2a 100644 --- a/tests/gold_tests/tls/tls_tunnel.test.py +++ b/tests/gold_tests/tls/tls_tunnel.test.py @@ -322,3 +322,28 @@ tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Could Not Connec tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Not Found on Accelerato", "Terminates on on Traffic Server") tr.Processes.Default.Streams.All += Testers.ExcludesExpression("ATS", "Terminate on Traffic Server") tr.Processes.Default.Streams.All += Testers.ContainsExpression("bar ok", "Should get a response from bar") + +tr = Test.AddTestRun("Test Metrics") +tr.Processes.Default.Command = ( + f"{Test.Variables.AtsTestToolsDir}/stdout_wait" + + " 'traffic_ctl metric get" + + " proxy.process.tunnel.total_client_connections_blind_tcp" + + " proxy.process.tunnel.current_client_connections_blind_tcp" + + " proxy.process.tunnel.total_server_connections_blind_tcp" + + " proxy.process.tunnel.current_server_connections_blind_tcp" + + " proxy.process.tunnel.total_client_connections_tls_tunnel" + + " proxy.process.tunnel.current_client_connections_tls_tunnel" + + " proxy.process.tunnel.total_client_connections_tls_forward" + + " proxy.process.tunnel.current_client_connections_tls_forward" + + " proxy.process.tunnel.total_client_connections_tls_partial_blind" + + " proxy.process.tunnel.current_client_connections_tls_partial_blind" + + " proxy.process.tunnel.total_client_connections_tls_http" + + " proxy.process.tunnel.current_client_connections_tls_http" + + " proxy.process.tunnel.total_server_connections_tls" + + " proxy.process.tunnel.current_server_connections_tls'" + + f" {Test.TestDirectory}/gold/tls-tunnel-metrics.gold" +) +# Need to copy over the environment so traffic_ctl knows where to find the unix domain socket +tr.Processes.Default.Env = ts.Env +tr.Processes.Default.ReturnCode = 0 +tr.StillRunningAfter = ts diff --git a/tests/gold_tests/tls/tls_tunnel_forward.test.py b/tests/gold_tests/tls/tls_tunnel_forward.test.py index d82b92a07b..bd2417eaf8 100644 --- a/tests/gold_tests/tls/tls_tunnel_forward.test.py +++ b/tests/gold_tests/tls/tls_tunnel_forward.test.py @@ -122,3 +122,35 @@ tr3.Processes.Default.Streams.All += Testers.ExcludesExpression( tr3.Processes.Default.Streams.All += Testers.ContainsExpression("CN=foo.com", "Should TLS terminate on Traffic Server") tr3.Processes.Default.Streams.All += Testers.ContainsExpression("HTTP/1.1 200 OK", "Should get a successful response") tr3.Processes.Default.Streams.All += Testers.ContainsExpression("ok random", "Body is expected") + +tr = Test.AddTestRun("Test Metrics") +tr.Processes.Default.Command = ( + f"{Test.Variables.AtsTestToolsDir}/stdout_wait" + + " 'traffic_ctl metric get" + + " proxy.process.http.total_incoming_connections" + + " proxy.process.http.total_client_connections" + + " proxy.process.http.total_client_connections_ipv4" + + " proxy.process.http.total_client_connections_ipv6" + + " proxy.process.http.total_server_connections" + + " proxy.process.http2.total_client_connections" + + " proxy.process.http.connect_requests" + + " proxy.process.tunnel.total_client_connections_blind_tcp" + + " proxy.process.tunnel.current_client_connections_blind_tcp" + + " proxy.process.tunnel.total_server_connections_blind_tcp" + + " proxy.process.tunnel.current_server_connections_blind_tcp" + + " proxy.process.tunnel.total_client_connections_tls_tunnel" + + " proxy.process.tunnel.current_client_connections_tls_tunnel" + + " proxy.process.tunnel.total_client_connections_tls_forward" + + " proxy.process.tunnel.current_client_connections_tls_forward" + + " proxy.process.tunnel.total_client_connections_tls_partial_blind" + + " proxy.process.tunnel.current_client_connections_tls_partial_blind" + + " proxy.process.tunnel.total_client_connections_tls_http" + + " proxy.process.tunnel.current_client_connections_tls_http" + + " proxy.process.tunnel.total_server_connections_tls" + + " proxy.process.tunnel.current_server_connections_tls'" + + f" {Test.TestDirectory}/gold/tls-tunnel-forward-metrics.gold" +) +# Need to copy over the environment so traffic_ctl knows where to find the unix domain socket +tr.Processes.Default.Env = ts.Env +tr.Processes.Default.ReturnCode = 0 +tr.StillRunningAfter = ts diff --git a/tests/tools/stdout_wait b/tests/tools/stdout_wait new file mode 100755 index 0000000000..059eb1fdf2 --- /dev/null +++ b/tests/tools/stdout_wait @@ -0,0 +1,82 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# A bash script to repeatedly run a command until its standard output matches a gold file. +# +# usage: +# stdout_wait [ MAX-WAIT [ POST-WAIT ] ] COMMAND GOLD-FILE-SPEC +# +# (COMMAND may contain white space.) +# +# MAX-WAIT is the maximum number of seconds to wait for the condition. If it is omitted, it defaults to 60. +# +# POST-WAIT is the number of seconds to wait after the condition is true. If it is omitted, it defaults to 0. +# +# The script exits with status 0 when the command's standard output matches the gold file. It exits with status 1 if the +# output never matche the gold file, and the maximum wait has expired. + +WAIT=60 +POST_WAIT=0 + +USAGE="usage: stdout_wait [ MAX-WAIT [ POST-WAIT ] ] COMMAND GOLD-FILE-SPEC" + +if [[ "$1" = "" ]] ; then + echo $USAGE >&2 + exit 1 +fi + +X=$( echo "$1" | sed 's/x/yy/g' | sed 's/[^0-9]/x/g' ) +if [[ "$X" = "$1" ]] ; then + WAIT=$1 + shift + if [[ "$1" = "" ]] ; then + echo $USAGE >&2 + exit 1 + fi + X=$( echo "$1" | sed 's/x/yy/g' | sed 's/[^0-9]/x/g' ) + if [[ "$X" = "$1" ]] ; then + POST_WAIT=$1 + shift + fi +fi + +if [[ "$1" = "" ]] ; then + echo $USAGE >&2 + exit 1 +fi + +if [[ ! -f "$2" ]] ; then + echo $USAGE >&2 + exit 1 +fi + +while (( WAIT > 0 )) +do + if $1 | diff - $2 >| /dev/null 2> /dev/null + then + if (( POST_WAIT > 0 )) + then + sleep $POST_WAIT + fi + exit 0 + fi + sleep 1 + let WAIT=WAIT-1 +done +$1 | diff - $2 +exit 1
