This is an automated email from the ASF dual-hosted git repository. bcall pushed a commit to branch 7.0.x in repository https://git-dual.apache.org/repos/asf/trafficserver.git
commit c1fa856c3b9f2d172117ac404667faa0ac920363 Author: Bryan Call <[email protected]> AuthorDate: Wed Sep 28 21:17:43 2016 -0700 TS-4909: Throttling based on resident memory (cherry picked from commit cbd5cc1f9229235df44f8fcf1262eb491a48f2de) --- iocore/net/P_UnixNet.h | 2 +- iocore/net/P_UnixNetProcessor.h | 1 - iocore/net/UnixNet.cc | 1 + iocore/net/UnixNetAccept.cc | 54 ++++++----------------------------------- mgmt/RecordsConfig.cc | 2 ++ proxy/Main.cc | 50 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 62 insertions(+), 48 deletions(-) diff --git a/iocore/net/P_UnixNet.h b/iocore/net/P_UnixNet.h index b4fbc54..b045145 100644 --- a/iocore/net/P_UnixNet.h +++ b/iocore/net/P_UnixNet.h @@ -124,6 +124,7 @@ extern ink_hrtime last_throttle_warning; extern ink_hrtime last_shedding_warning; extern ink_hrtime emergency_throttle_time; extern int net_connections_throttle; +extern bool net_memory_throttle; extern int fds_throttle; extern int fds_limit; extern ink_hrtime last_transient_accept_error; @@ -141,7 +142,6 @@ extern int http_accept_port_number; #define TRANSIENT_ACCEPT_ERROR_MESSAGE_EVERY HRTIME_HOURS(24) // also the 'throttle connect headroom' -#define THROTTLE_AT_ONCE 5 #define EMERGENCY_THROTTLE 16 #define HYPER_EMERGENCY_THROTTLE 6 diff --git a/iocore/net/P_UnixNetProcessor.h b/iocore/net/P_UnixNetProcessor.h index bf24b80..7c388f4 100644 --- a/iocore/net/P_UnixNetProcessor.h +++ b/iocore/net/P_UnixNetProcessor.h @@ -45,7 +45,6 @@ public: virtual int start(int number_of_net_threads, size_t stacksize); - char *throttle_error_message; Event *accept_thread_event; // offsets for per thread data structures diff --git a/iocore/net/UnixNet.cc b/iocore/net/UnixNet.cc index 2b3964e..2b7afec 100644 --- a/iocore/net/UnixNet.cc +++ b/iocore/net/UnixNet.cc @@ -27,6 +27,7 @@ ink_hrtime last_throttle_warning; ink_hrtime last_shedding_warning; ink_hrtime emergency_throttle_time; int net_connections_throttle; +bool net_memory_throttle = false; int fds_throttle; int fds_limit = 8000; ink_hrtime last_transient_accept_error; diff --git a/iocore/net/UnixNetAccept.cc b/iocore/net/UnixNetAccept.cc index 901cc6b..441275c 100644 --- a/iocore/net/UnixNetAccept.cc +++ b/iocore/net/UnixNetAccept.cc @@ -39,39 +39,6 @@ safe_delay(int msec) } // -// Send the throttling message to up to THROTTLE_AT_ONCE connections, -// delaying to let some of the current connections complete -// -static int -send_throttle_message(NetAccept *na) -{ - struct pollfd afd; - Connection con[100]; - char dummy_read_request[4096]; - - afd.fd = na->server.fd; - afd.events = POLLIN; - - int n = 0; - while (check_net_throttle(ACCEPT, Thread::get_hrtime()) && n < THROTTLE_AT_ONCE - 1 && (socketManager.poll(&afd, 1, 0) > 0)) { - int res = 0; - if ((res = na->server.accept(&con[n])) < 0) - return res; - n++; - } - safe_delay(net_throttle_delay / 2); - int i = 0; - for (i = 0; i < n; i++) { - socketManager.read(con[i].fd, dummy_read_request, 4096); - socketManager.write(con[i].fd, unix_netProcessor.throttle_error_message, strlen(unix_netProcessor.throttle_error_message)); - } - safe_delay(net_throttle_delay / 2); - for (i = 0; i < n; i++) - con[i].close(); - return 0; -} - -// // General case network connection accept code // int @@ -252,20 +219,7 @@ NetAccept::do_blocking_accept(EThread *t) do { ink_hrtime now = Thread::get_hrtime(); - // Throttle accepts - - while (!backdoor && check_net_throttle(ACCEPT, now)) { - check_throttle_warning(); - if (!unix_netProcessor.throttle_error_message) { - safe_delay(net_throttle_delay); - } else if (send_throttle_message(this) < 0) { - goto Lerror; - } - now = Thread::get_hrtime(); - } - if ((res = server.accept(&con)) < 0) { - Lerror: int seriousness = accept_error_seriousness(res); if (seriousness >= 0) { // not so bad if (!seriousness) // bad enough to warn about @@ -281,6 +235,14 @@ NetAccept::do_blocking_accept(EThread *t) return -1; } + // Throttle accepts + if (!backdoor && (check_net_throttle(ACCEPT, now) || net_memory_throttle)) { + Debug("net_accept", "Too many connections or too much memory used, throttling"); + check_throttle_warning(); + con.close(); + continue; + } + // Use 'NULL' to Bypass thread allocator vc = (UnixNetVConnection *)this->getNetProcessor()->allocate_vc(NULL); if (unlikely(!vc || shutdown_event_system == true)) { diff --git a/mgmt/RecordsConfig.cc b/mgmt/RecordsConfig.cc index e0b2e71..e3a917f 100644 --- a/mgmt/RecordsConfig.cc +++ b/mgmt/RecordsConfig.cc @@ -110,6 +110,8 @@ static const RecordElement RecordsConfig[] = //# 0 = disable {RECT_CONFIG, "proxy.config.res_track_memory", RECD_INT, "0", RECU_NULL, RR_REQUIRED, RECC_NULL, "[0-2]", RECA_NULL} , + {RECT_CONFIG, "proxy.config.memory.max_usage", RECD_INT, "0", RECU_RESTART_TS, RR_NULL, RECC_STR, "^-?[0-9]+$", RECA_NULL} + , //############################################################################## //# Traffic Server system settings //############################################################################## diff --git a/proxy/Main.cc b/proxy/Main.cc index 125558d..1b20586 100644 --- a/proxy/Main.cc +++ b/proxy/Main.cc @@ -346,6 +346,55 @@ public: } }; +class MemoryLimit : public Continuation +{ +public: + MemoryLimit() : Continuation(new_ProxyMutex()), _memory_limit(0) { SET_HANDLER(&MemoryLimit::periodic); } + ~MemoryLimit() { mutex = NULL; } + int + periodic(int event, Event *e) + { + if (event == EVENT_IMMEDIATE) { + // rescheduled from periodic to immediate event + // this is the indication to terminate + delete this; + return EVENT_DONE; + } + if (_memory_limit == 0) { + // first time it has been run + _memory_limit = REC_ConfigReadInteger("proxy.config.memory.max_usage"); + _memory_limit = _memory_limit >> 10; // divide by 1024 + } + if (_memory_limit > 0) { + if (getrusage(RUSAGE_SELF, &_usage) == 0) { + Debug("server", "memory usage - ru_maxrss: %ld memory limit: %" PRId64, _usage.ru_maxrss, _memory_limit); + if (_usage.ru_maxrss > _memory_limit) { + if (net_memory_throttle == false) { + net_memory_throttle = true; + Debug("server", "memory usage exceeded limit - ru_maxrss: %ld memory limit: %" PRId64, _usage.ru_maxrss, _memory_limit); + } + } else { + if (net_memory_throttle == true) { + net_memory_throttle = false; + Debug("server", "memory usage under limit - ru_maxrss: %ld memory limit: %" PRId64, _usage.ru_maxrss, _memory_limit); + } + } + } + } else { + // this feature has not be enabled + Debug("server", "limiting connections based on memory usage has been disabled"); + e->cancel(); + delete this; + return EVENT_DONE; + } + return EVENT_CONT; + } + +private: + int64_t _memory_limit; + struct rusage _usage; +}; + static int init_memory_tracker(const char *config_var, RecDataT /* type ATS_UNUSED */, RecData data, void * /* cookie ATS_UNUSED */) { @@ -1727,6 +1776,7 @@ main(int /* argc ATS_UNUSED */, const char **argv) eventProcessor.schedule_every(new SignalContinuation, HRTIME_MSECOND * 500, ET_CALL); eventProcessor.schedule_every(new DiagsLogContinuation, HRTIME_SECOND, ET_TASK); + eventProcessor.schedule_every(new MemoryLimit, HRTIME_SECOND, ET_TASK); REC_RegisterConfigUpdateFunc("proxy.config.dump_mem_info_frequency", init_memory_tracker, NULL); init_memory_tracker(NULL, RECD_NULL, RecData(), NULL); -- To stop receiving notification emails like this one, please contact "[email protected]" <[email protected]>.
