This is an automated email from the ASF dual-hosted git repository. bcall pushed a commit to branch 8.1.x in repository https://gitbox.apache.org/repos/asf/trafficserver.git
The following commit(s) were added to refs/heads/8.1.x by this push: new c4cb0526d6 APIs to get the h2 error codes and a plugin to use them (#10572) c4cb0526d6 is described below commit c4cb0526d65460d1db0f5e01f48411766652928b Author: Bryan Call <bc...@apache.org> AuthorDate: Mon Oct 9 10:29:03 2023 -0700 APIs to get the h2 error codes and a plugin to use them (#10572) --- doc/admin-guide/plugins/block_errors.en.rst | 69 +++++ doc/admin-guide/plugins/index.en.rst | 4 + include/ts/ts.h | 36 +++ include/tscore/ink_inet.h | 26 ++ plugins/Makefile.am | 1 + plugins/experimental/block_errors/Makefile.inc | 20 ++ plugins/experimental/block_errors/block_errors.cc | 318 ++++++++++++++++++++++ src/traffic_server/InkAPI.cc | 50 ++++ 8 files changed, 524 insertions(+) diff --git a/doc/admin-guide/plugins/block_errors.en.rst b/doc/admin-guide/plugins/block_errors.en.rst new file mode 100644 index 0000000000..36b51c451f --- /dev/null +++ b/doc/admin-guide/plugins/block_errors.en.rst @@ -0,0 +1,69 @@ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + + .. include:: ../../common.defs + +.. _admin-plugins-block_errors: + +Block Errors Plugin +******************* + +Description +=========== +The `block_errors` plugin blocks connections for clients that have too many HTTP/2 errors on the server. + +The plugin tracks users based on their IP address and blocks them for a configurable amount of time. +The existing connection that experience errors and is over the error limit will be closed. The plugin also supports on the fly configuration changes using the `traffic_ctl` command. + + +Configuration +============= + +To enable the `block_errors` plugin, insert the following line in :file:`plugin.config`: + + block_errors.so + +Additional configuration options are available and can be set in :file:`plugin.config`: + + block_errors.so <error limit> <timeout> <enable> + +- ``error limit``: The number of errors allowed before blocking the client. Default: 1000 (per minute) +- ``timeout``: The time in minutes to block the client. Default: 4 (minutes) +- ``enable``: Enable (1) or disable (0) the plugin. Default: 1 (enabled) + +Example Configuration +===================== + + block_errors.so 1000 4 0 1 + +Run Time Configuration +====================== +The plugin can be configured at run time using the `traffic_ctl` command. The following commands are available: + +- ``block_errors.error_limit``: Set the error limit. Takes a single argument, the number of errors allowed before blocking the client. +- ``block_errors.timeout``: Set the block timeout. Takes a single argument, the number of minutes to block the client. +- ``block_errors.enable``: Enable or disable the plugin. Takes a single argument, 0 to disable, 1 to enable. + +Example Run Time Configuration +============================== + + traffic_ctl plugin msg block_errors.error_limit 10000 + + traffic_ctl plugin msg block_errors.timeout 10 + + traffic_ctl plugin msg block_errors.enable 1 diff --git a/doc/admin-guide/plugins/index.en.rst b/doc/admin-guide/plugins/index.en.rst index 7c408387a2..73a1e56f0c 100644 --- a/doc/admin-guide/plugins/index.en.rst +++ b/doc/admin-guide/plugins/index.en.rst @@ -150,6 +150,7 @@ directory of the |TS| source tree. Experimental plugins can be compiled by passi Access Control <access_control.en> Balancer <balancer.en> Buffer Upload <buffer_upload.en> + Block Errors <block_errors.en> Cache Fill <cache_fill.en> Certifier <certifier.en> Collapsed-Forwarding <collapsed_forwarding.en> @@ -181,6 +182,9 @@ directory of the |TS| source tree. Experimental plugins can be compiled by passi :doc:`Buffer Upload <buffer_upload.en>` Buffers POST data before connecting to the Origin server. +:doc:`Block Errors <block_errors.en>` + Blocks or downgrades new connections when the server receives too many errors from an IP address. + :doc:`Certifier <certifier.en>` Manages and/or generates certificates for incoming HTTPS requests. diff --git a/include/ts/ts.h b/include/ts/ts.h index 6b39abf925..6dee5d05d5 100644 --- a/include/ts/ts.h +++ b/include/ts/ts.h @@ -2468,6 +2468,42 @@ tsapi TSReturnCode TSRemapToUrlGet(TSHttpTxn txnp, TSMLoc *urlLocp); */ tsapi TSIOBufferReader TSHttpTxnPostBufferReaderGet(TSHttpTxn txnp); +/** + * @brief Get the client error received from the transaction + * + * @param txnp The transaction where the error code is stored + * @param error_class Either session/connection or stream/transaction error + * @param error_code Error code received from the client + */ +void TSHttpTxnClientReceivedErrorGet(TSHttpTxn txnp, uint32_t *error_class, uint64_t *error_code); + +/** + * @brief Get the client error sent from the transaction + * + * @param txnp The transaction where the error code is stored + * @param error_class Either session/connection or stream/transaction error + * @param error_code Error code sent to the client + */ +void TSHttpTxnClientSentErrorGet(TSHttpTxn txnp, uint32_t *error_class, uint64_t *error_code); + +/** + * @brief Get the server error received from the transaction + * + * @param txnp The transaction where the error code is stored + * @param error_class Either session/connection or stream/transaction error + * @param error_code Error code sent from the server + */ +void TSHttpTxnServerReceivedErrorGet(TSHttpTxn txnp, uint32_t *error_class, uint64_t *error_code); + +/** + * @brief Get the server error sent from the transaction + * + * @param txnp The transaction where the error code is stored + * @param error_class Either session/connection or stream/transaction error + * @param error_code Error code sent to the server + */ +void TSHttpTxnServerSentErrorGet(TSHttpTxn txnp, uint32_t *error_class, uint64_t *error_code); + #ifdef __cplusplus } #endif /* __cplusplus */ diff --git a/include/tscore/ink_inet.h b/include/tscore/ink_inet.h index 4cbf14c1d4..437b139df6 100644 --- a/include/tscore/ink_inet.h +++ b/include/tscore/ink_inet.h @@ -1240,6 +1240,7 @@ struct IpAddr { - Else: 0. */ uint32_t hash() const; + uint64_t hash64() const; /** The hashing function embedded in a functor. @see hash @@ -1434,6 +1435,18 @@ IpAddr::hash() const return zret; } +inline uint64_t +IpAddr::hash64() const +{ + uint64_t zret = 0; + if (this->isIp4()) { + zret = ntohl(_addr._ip4); + } else if (this->isIp6()) { + zret = _addr._u64[0] ^ _addr._u64[1]; + } + return zret; +} + /// Write IP @a addr to storage @a dst. /// @return @s dst. sockaddr *ats_ip_set(sockaddr *dst, ///< Destination storage. @@ -1559,3 +1572,16 @@ bwformat(BufferWriter &w, BWFSpec const &spec, IpEndpoint const &addr) return bwformat(w, spec, &addr.sa); } } // namespace ts + +namespace std +{ +/// Standard hash support for @a IPAddr. +template <> struct hash<IpAddr> { + size_t + operator()(IpAddr const &addr) const + { + return addr.hash64(); + } +}; + +} // namespace std diff --git a/plugins/Makefile.am b/plugins/Makefile.am index 8dca2f2c60..91fecd50be 100644 --- a/plugins/Makefile.am +++ b/plugins/Makefile.am @@ -55,6 +55,7 @@ if BUILD_EXPERIMENTAL_PLUGINS include experimental/access_control/Makefile.inc include experimental/acme/Makefile.inc include experimental/balancer/Makefile.inc +include experimental/block_errors/Makefile.inc include experimental/buffer_upload/Makefile.inc include experimental/cache_range_requests/Makefile.inc include experimental/certifier/Makefile.inc diff --git a/plugins/experimental/block_errors/Makefile.inc b/plugins/experimental/block_errors/Makefile.inc new file mode 100644 index 0000000000..3659b6a1ee --- /dev/null +++ b/plugins/experimental/block_errors/Makefile.inc @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +pkglib_LTLIBRARIES += experimental/block_errors/block_errors.la + +experimental_block_errors_block_errors_la_SOURCES = \ + experimental/block_errors/block_errors.cc diff --git a/plugins/experimental/block_errors/block_errors.cc b/plugins/experimental/block_errors/block_errors.cc new file mode 100644 index 0000000000..221b77e6e5 --- /dev/null +++ b/plugins/experimental/block_errors/block_errors.cc @@ -0,0 +1,318 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include <ts/ts.h> +#include <unordered_map> +#include <limits> +#include <tscore/ink_inet.h> +#include <tscore/BufferWriter.h> +#include <unistd.h> +#include <arpa/inet.h> +#include <shared_mutex> +#include <cinttypes> +#include <string_view> +#include <mutex> + +#define PLUGIN_NAME "block_errors" +#define PLUGIN_NAME_CLEAN "block_clean" +static uint32_t RESET_LIMIT = 1000; +static uint32_t TIMEOUT_CYCLES = 4; +static int StatCountBlocks = -1; +static const bool shutdown_connection = true; +static bool enabled = true; + +//------------------------------------------------------------------------- +static int +msg_hook(TSCont *contp, TSEvent event, void *edata) +{ + TSPluginMsg *msg = static_cast<TSPluginMsg *>(edata); + std::string_view tag(static_cast<const char *>(msg->tag)); + std::string_view data(static_cast<const char *>(msg->data)); + + TSDebug(PLUGIN_NAME, "msg_hook: tag=%s data=%s", tag.data(), data.data()); + + if (tag == "block_errors.enabled") { + enabled = static_cast<bool>(atoi(data.data())); + } else if (tag == "block_errors.limit") { + RESET_LIMIT = atoi(data.data()); + } else if (tag == "block_errors.cycles") { + TIMEOUT_CYCLES = atoi(data.data()); + } else { + TSDebug(PLUGIN_NAME, "msg_hook: unknown message tag '%s'", tag.data()); + TSError("block_errors: unknown message tag '%s'", tag.data()); + } + + TSDebug(PLUGIN_NAME, "reset limit: %d per minute, timeout limit: %d minutes, shutdown connection: %d enabled: %d", RESET_LIMIT, + TIMEOUT_CYCLES, shutdown_connection, enabled); + + return 0; +} + +//------------------------------------------------------------------------- +// convert a sockaddr to a string +std::string & +ipaddr_to_string(const IpAddr &ip, std::string &address) +{ + ts::LocalBufferWriter<128> writer; + writer.print("{}", ip); + address = writer.view(); + + return address; +} + +//------------------------------------------------------------------------- +struct IPTableItem { + uint32_t _count = 1; + uint32_t _cycles = 0; +}; + +//------------------------------------------------------------------------- +class IPTable +{ +public: + IPTable() = default; + + uint32_t + increment(IpAddr const &ip) + { + std::unique_lock lock(_mutex); + auto item = _table.find(ip); + if (item == _table.end()) { + _table.insert(std::make_pair(ip, IPTableItem())); + return 1; + } else { + ++item->second._count; + uint32_t tmp_count = item->second._count; + return tmp_count; + } + } + + uint32_t + getCount(IpAddr const &ip) + { + std::shared_lock lock(_mutex); + auto item = _table.find(ip); + if (item == _table.end()) { + return 0; + } else { + uint32_t tmp_count = item->second._count; + return tmp_count; + } + } + + void + clean() + { + std::string address; + std::unique_lock lock(_mutex); + for (auto item = _table.begin(); item != _table.end();) { + if (item->second._count <= RESET_LIMIT || item->second._cycles >= TIMEOUT_CYCLES) { + // remove the item if the count is below the limit or the timeout has expired + TSDebug(PLUGIN_NAME_CLEAN, "ip=%s count=%d removing", ipaddr_to_string(item->first, address).c_str(), item->second._count); + item = _table.erase(item); + } else { + // increment the timeout cycles if the count is above the limit + if (item->second._cycles == 0) { + // log only once per ip address per timeout period + TSError("block_errors: blocking or downgrading ip=%s for %d minutes, reset count=%d", + ipaddr_to_string(item->first, address).c_str(), TIMEOUT_CYCLES, item->second._count); + TSStatIntIncrement(StatCountBlocks, 1); + } + ++item->second._cycles; + TSDebug(PLUGIN_NAME_CLEAN, "ip=%s count=%d incrementing cycles=%d", ipaddr_to_string(item->first, address).c_str(), + item->second._count, item->second._cycles); + ++item; + } + } + } + +private: + std::unordered_map<IpAddr, IPTableItem> _table; + std::shared_mutex _mutex; +}; + +IPTable ip_table; + +//------------------------------------------------------------------------- +static int +handle_start_hook(TSCont *contp, TSEvent event, void *edata) +{ + TSDebug(PLUGIN_NAME, "handle_start_hook"); + auto vconn = static_cast<TSVConn>(edata); + + if (enabled == false) { + TSDebug(PLUGIN_NAME, "plugin disabled"); + TSVConnReenable(vconn); + return 0; + } + + // only handle ssl connections + if (TSVConnIsSsl(vconn) == 0) { + TSDebug(PLUGIN_NAME, "not a ssl connection"); + TSVConnReenable(vconn); + return 0; + } + + // get the ip address + const sockaddr *addr = TSNetVConnRemoteAddrGet(vconn); + if (addr == nullptr) { + // only needed for 8.1x, 9.2.x and 10.x always returns an address + TSDebug(PLUGIN_NAME, "unable to get remote address"); + TSVConnReenable(vconn); + return 0; + } + IpAddr ipaddr(addr); + + // get the count for the ip address + uint32_t count = ip_table.getCount(ipaddr); + TSDebug(PLUGIN_NAME, "count=%d", count); + + // if the count is over the limit, shutdown or downgrade the connection + if (count > RESET_LIMIT) { + std::string address; + if (shutdown_connection == true) { + // shutdown the connection + TSDebug(PLUGIN_NAME, "ip=%s count=%d is over the limit, shutdown connection on start", + ipaddr_to_string(ipaddr, address).c_str(), count); + int fd = TSVConnFdGet(vconn); + shutdown(fd, SHUT_RDWR); + char buffer[4096]; + while (read(fd, buffer, sizeof(buffer)) > 0) { + // drain the connection + } + } else { + // downgrade the connection + TSDebug(PLUGIN_NAME, "ip=%s count=%d is over the limit, downgrading connection", ipaddr_to_string(ipaddr, address).c_str(), + count); + // TSVConnProtocolDisable(vconn, TS_ALPN_PROTOCOL_HTTP_2_0); + } + } + + TSVConnReenable(vconn); + return 0; +} + +//------------------------------------------------------------------------- +struct Errors { + uint32_t cls = 0; // class of error + uint64_t code = 0; // error code +}; + +//------------------------------------------------------------------------- +static int +handle_close_hook(TSCont *contp, TSEvent event, void *edata) +{ + TSDebug(PLUGIN_NAME, "handle_close_hook"); + auto txnp = static_cast<TSHttpTxn>(edata); + + if (enabled == false) { + TSDebug(PLUGIN_NAME, "plugin disabled"); + TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE); + return 0; + } + + // get the errors from the state machine + Errors transaction; + Errors session; + TSHttpTxnClientReceivedErrorGet(txnp, &transaction.cls, &transaction.code); + TSHttpTxnClientSentErrorGet(txnp, &session.cls, &session.code); + + // debug if we have an error + if (transaction.cls != 0 || session.cls != 0 || transaction.code != 0 || session.code != 0) { + TSDebug(PLUGIN_NAME, "transaction error class=%d code=%" PRIu64 " session error class=%d code=%" PRIu64, transaction.cls, + transaction.code, session.cls, session.code); + } + + // count the error if there is a transaction error CANCEL or a session error ENHANCE_YOUR_CALM + // https://www.rfc-editor.org/rfc/rfc9113.html#name-error-codes + if ((transaction.cls == 2 && transaction.code == 8) || (session.cls == 1 && session.code == 11)) { + TSHttpSsn ssn = TSHttpTxnSsnGet(txnp); + TSVConn vconn = TSHttpSsnClientVConnGet(ssn); + if (vconn == nullptr) { + // only needed for 8.1x, 9.2.x and 10.x always returns a vconn + TSDebug(PLUGIN_NAME, "unable to get the vconn"); + TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE); + return 0; + } + const sockaddr *addr = TSNetVConnRemoteAddrGet(vconn); + IpAddr ipaddr(addr); + uint32_t count = ip_table.increment(ipaddr); + if (count > RESET_LIMIT) { + std::string address; + TSDebug(PLUGIN_NAME, "ip=%s count=%d is over the limit, shutdown connection on close", + ipaddr_to_string(ipaddr, address).c_str(), count); + int fd = TSVConnFdGet(vconn); + shutdown(fd, SHUT_RDWR); + } + } + + TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE); + return 0; +} + +//------------------------------------------------------------------------- +static int +clean_table(TSCont *contp, TSEvent event, void *edata) +{ + ip_table.clean(); + return 0; +} + +//------------------------------------------------------------------------- +void +TSPluginInit(int argc, const char *argv[]) +{ + TSDebug(PLUGIN_NAME, "TSPluginInit"); + + // register the plugin + TSPluginRegistrationInfo info; + info.plugin_name = "block_errors"; + info.vendor_name = "Apache Software Foundation"; + info.support_email = "d...@trafficserver.apache.org"; + + if (TSPluginRegister(&info) != TS_SUCCESS) { + TSError("Plugin registration failed"); + } + + // set the reset and timeout values + if (argc == 4) { + RESET_LIMIT = atoi(argv[1]); + TIMEOUT_CYCLES = atoi(argv[2]); + enabled = static_cast<bool>(atoi(argv[3])); + } else if (argc > 1 && argc < 4) { + TSDebug(PLUGIN_NAME, + "block_errors: invalid number of arguments, using the defaults - usage: block_errors.so <reset limit> <timeout " + "cycles> <shutdown connection> <enabled>"); + TSError("block_errors: invalid number of arguments, using the defaults - usage: block_errors.so <reset limit> <timeout cycles> " + "<shutdown connection> <enabled>"); + } + + TSDebug(PLUGIN_NAME, "reset limit: %d per minute, timeout limit: %d minutes, shutdown connection: %d enabled: %d", RESET_LIMIT, + TIMEOUT_CYCLES, shutdown_connection, enabled); + + // create a stat counter + StatCountBlocks = TSStatCreate("block_errors.count", TS_RECORDDATATYPE_INT, TS_STAT_NON_PERSISTENT, TS_STAT_SYNC_COUNT); + + // register the hooks + TSHttpHookAdd(TS_VCONN_START_HOOK, TSContCreate(reinterpret_cast<TSEventFunc>(handle_start_hook), nullptr)); + TSHttpHookAdd(TS_HTTP_TXN_CLOSE_HOOK, TSContCreate(reinterpret_cast<TSEventFunc>(handle_close_hook), nullptr)); + TSLifecycleHookAdd(TS_LIFECYCLE_MSG_HOOK, TSContCreate(reinterpret_cast<TSEventFunc>(msg_hook), nullptr)); + + // schedule cleanup on task thread every 60 seconds + TSContScheduleEvery(TSContCreate(reinterpret_cast<TSEventFunc>(clean_table), TSMutexCreate()), 60 * 1000, TS_THREAD_POOL_TASK); +} diff --git a/src/traffic_server/InkAPI.cc b/src/traffic_server/InkAPI.cc index 2d878123ea..691ff9dc45 100644 --- a/src/traffic_server/InkAPI.cc +++ b/src/traffic_server/InkAPI.cc @@ -7716,6 +7716,56 @@ TSHttpTxnIsInternal(TSHttpTxn txnp) return TSHttpSsnIsInternal(TSHttpTxnSsnGet(txnp)); } +static void +txn_error_get(TSHttpTxn txnp, bool client, bool sent, uint32_t &error_class, uint64_t &error_code) +{ + sdk_assert(sdk_sanity_check_txn(txnp) == TS_SUCCESS); + HttpSM *sm = reinterpret_cast<HttpSM *>(txnp); + HttpTransact::ConnectionAttributes *connection_attributes = nullptr; + + if (client == true) { + // client + connection_attributes = &sm->t_state.client_info; + } else { + // server + connection_attributes = &sm->t_state.server_info; + } + + if (sent == true) { + // sent + error_code = connection_attributes->tx_error_code.code; + error_class = static_cast<uint32_t>(connection_attributes->tx_error_code.cls); + } else { + // received + error_code = connection_attributes->rx_error_code.code; + error_class = static_cast<uint32_t>(connection_attributes->rx_error_code.cls); + } +} + +void +TSHttpTxnClientReceivedErrorGet(TSHttpTxn txnp, uint32_t *error_class, uint64_t *error_code) +{ + txn_error_get(txnp, true, false, *error_class, *error_code); +} + +void +TSHttpTxnClientSentErrorGet(TSHttpTxn txnp, uint32_t *error_class, uint64_t *error_code) +{ + txn_error_get(txnp, true, true, *error_class, *error_code); +} + +void +TSHttpTxnServerReceivedErrorGet(TSHttpTxn txnp, uint32_t *error_class, uint64_t *error_code) +{ + txn_error_get(txnp, false, false, *error_class, *error_code); +} + +void +TSHttpTxnServerSentErrorGet(TSHttpTxn txnp, uint32_t *error_class, uint64_t *error_code) +{ + txn_error_get(txnp, false, true, *error_class, *error_code); +} + void TSHttpTxnServerPush(TSHttpTxn txnp, const char *url, int url_len) {