TS-3233: allow traffic draining before restart Add the --wait option to traffic_line. This option delays the corresponsing bounce or restart until the number of active client connections drops to the threshold given by the proxy.config.restart.active_client_threshold configuration variable.
Project: http://git-wip-us.apache.org/repos/asf/trafficserver/repo Commit: http://git-wip-us.apache.org/repos/asf/trafficserver/commit/c7554f26 Tree: http://git-wip-us.apache.org/repos/asf/trafficserver/tree/c7554f26 Diff: http://git-wip-us.apache.org/repos/asf/trafficserver/diff/c7554f26 Branch: refs/heads/master Commit: c7554f2636c593fdcb16287eea10adf7c4e73eae Parents: 13053ee Author: James Peach <[email protected]> Authored: Wed Nov 19 20:09:11 2014 -0800 Committer: James Peach <[email protected]> Committed: Tue Dec 9 14:12:26 2014 -0800 ---------------------------------------------------------------------- cmd/traffic_line/traffic_line.cc | 12 ++++-- cmd/traffic_manager/traffic_manager.cc | 45 +++++++++++++++++++- doc/reference/commands/traffic_line.en.rst | 9 ++++ .../configuration/records.config.en.rst | 6 +++ mgmt/LocalManager.cc | 29 ++++++------- mgmt/LocalManager.h | 13 +++++- mgmt/RecordsConfig.cc | 2 + mgmt/api/CoreAPI.cc | 15 ++++--- mgmt/api/CoreAPI.h | 4 +- mgmt/api/CoreAPIRemote.cc | 12 +++--- mgmt/api/INKMgmtAPI.cc | 12 +++--- mgmt/api/TSControlMain.cc | 16 +++---- mgmt/api/include/mgmtapi.h | 14 ++++-- mgmt/cluster/ClusterCom.cc | 4 +- mgmt/utils/MgmtLocalCleanup.cc | 2 +- 15 files changed, 133 insertions(+), 62 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/trafficserver/blob/c7554f26/cmd/traffic_line/traffic_line.cc ---------------------------------------------------------------------- diff --git a/cmd/traffic_line/traffic_line.cc b/cmd/traffic_line/traffic_line.cc index f998730..3826829 100644 --- a/cmd/traffic_line/traffic_line.cc +++ b/cmd/traffic_line/traffic_line.cc @@ -51,23 +51,26 @@ static char StorageCmdOffline[1024]; static int ShowAlarms; static int ShowStatus; static int ShowBacktrace; +static int DrainTraffic; static char ClearAlarms[1024]; static TSMgmtError handleArgInvocation() { + unsigned restart = DrainTraffic ? TS_RESTART_OPT_DRAIN : TS_RESTART_OPT_NONE; + if (ReRead == 1) { return TSReconfigure(); } else if (ShutdownMgmtCluster == 1) { - return TSRestart(true); + return TSRestart(restart | TS_RESTART_OPT_CLUSTER); } else if (ShutdownMgmtLocal == 1) { - return TSRestart(false); + return TSRestart(restart); } else if (Shutdown == 1) { return TSProxyStateSet(TS_PROXY_OFF, TS_CACHE_CLEAR_OFF); } else if (BounceCluster == 1) { - return TSBounce(true); + return TSBounce(restart | TS_RESTART_OPT_CLUSTER); } else if (BounceLocal == 1) { - return TSBounce(false); + return TSBounce(restart); } else if (Startup == 1) { return TSProxyStateSet(TS_PROXY_ON, TS_CACHE_CLEAR_OFF); } else if (ClearCluster == 1) { @@ -355,6 +358,7 @@ main(int /* argc ATS_UNUSED */, char **argv) {"clear_alarms", '-', "Clear specified, or all, alarms", "S1024", &ClearAlarms, NULL, NULL}, {"status", '-', "Show proxy server status", "F", &ShowStatus, NULL, NULL}, {"backtrace", '-', "Show proxy stack backtrace", "F", &ShowBacktrace, NULL, NULL}, + {"drain", '-', "Wait for client connections to drain before restarting", "F", &DrainTraffic, NULL, NULL}, HELP_ARGUMENT_DESCRIPTION(), VERSION_ARGUMENT_DESCRIPTION() }; http://git-wip-us.apache.org/repos/asf/trafficserver/blob/c7554f26/cmd/traffic_manager/traffic_manager.cc ---------------------------------------------------------------------- diff --git a/cmd/traffic_manager/traffic_manager.cc b/cmd/traffic_manager/traffic_manager.cc index d7a37b5..b08cf82 100644 --- a/cmd/traffic_manager/traffic_manager.cc +++ b/cmd/traffic_manager/traffic_manager.cc @@ -96,6 +96,24 @@ static void SignalAlrmHandler(int sig); static volatile int sigHupNotifier = 0; static void SigChldHandler(int sig); +static bool +is_server_idle() +{ + RecInt active = 0; + RecInt threshold = 0; + + if (RecGetRecordInt("proxy.config.restart.active_client_threshold", &threshold) != REC_ERR_OKAY) { + return false; + } + + if (RecGetRecordInt("proxy.process.http.current_active_client_connections", &active) != REC_ERR_OKAY) { + return false; + } + + Debug("lm", "%" PRId64 " active clients, threshold is %" PRId64, active, threshold); + return active <= threshold; +} + static void check_lockfile() { @@ -726,9 +744,32 @@ main(int argc, char **argv) statProcessor->processStat(); } - if (lmgmt->mgmt_shutdown_outstanding == true) { - lmgmt->mgmtShutdown(true); + if (lmgmt->mgmt_shutdown_outstanding != MGMT_PENDING_NONE) { + Debug("lm", "pending shutdown %d", lmgmt->mgmt_shutdown_outstanding); + } + switch (lmgmt->mgmt_shutdown_outstanding) { + case MGMT_PENDING_RESTART: + lmgmt->mgmtShutdown(); _exit(0); + break; + case MGMT_PENDING_IDLE_RESTART: + if (is_server_idle()) { + lmgmt->mgmtShutdown(); + _exit(0); + } + break; + case MGMT_PENDING_BOUNCE: + lmgmt->processBounce(); + lmgmt->mgmt_shutdown_outstanding = MGMT_PENDING_NONE; + break; + case MGMT_PENDING_IDLE_BOUNCE: + if (is_server_idle()) { + lmgmt->processBounce(); + lmgmt->mgmt_shutdown_outstanding = MGMT_PENDING_NONE; + } + break; + default: + break; } if (lmgmt->run_proxy && !lmgmt->processRunning()) { /* Make sure we still have a proxy up */ http://git-wip-us.apache.org/repos/asf/trafficserver/blob/c7554f26/doc/reference/commands/traffic_line.en.rst ---------------------------------------------------------------------- diff --git a/doc/reference/commands/traffic_line.en.rst b/doc/reference/commands/traffic_line.en.rst index f0bbd0a..8157921 100644 --- a/doc/reference/commands/traffic_line.en.rst +++ b/doc/reference/commands/traffic_line.en.rst @@ -57,6 +57,15 @@ Options Clears accumulated statistics on the local node. +.. option:: --drain + + This option modifies the behavior of :option:`traffic_line -b` + and :option:`traffic_line -L` such that :program:`traffic_server` + is not shut down until the number of active client connections + drops to the number given by the + :ts:cv:`proxy.config.restart.active_client_threshold` configuration + variable. + .. option:: -h, --help Print usage information and exit. http://git-wip-us.apache.org/repos/asf/trafficserver/blob/c7554f26/doc/reference/configuration/records.config.en.rst ---------------------------------------------------------------------- diff --git a/doc/reference/configuration/records.config.en.rst b/doc/reference/configuration/records.config.en.rst index c8c445a..2caeb4b 100644 --- a/doc/reference/configuration/records.config.en.rst +++ b/doc/reference/configuration/records.config.en.rst @@ -266,6 +266,12 @@ Value Effect before :program:`traffic_server` drops privilege. If this variable is set to ``NULL``, no helper will be spawned. +.. ts:cv:: CONFIG proxy.config.restart.active_client_threshold INT 0 + :reloadable: + + This setting specifies the number of active client connections + for use by :option:`traffic_line --drain`. + Network ======= http://git-wip-us.apache.org/repos/asf/trafficserver/blob/c7554f26/mgmt/LocalManager.cc ---------------------------------------------------------------------- diff --git a/mgmt/LocalManager.cc b/mgmt/LocalManager.cc index fc9f148..bb54405 100644 --- a/mgmt/LocalManager.cc +++ b/mgmt/LocalManager.cc @@ -54,27 +54,22 @@ LocalManager::mgmtCleanup() void -LocalManager::mgmtShutdown(bool mainThread) +LocalManager::mgmtShutdown() { - if (mainThread) { - mgmt_log("[LocalManager::mgmtShutdown] Executing shutdown request.\n"); - processShutdown(mainThread); - // WCCP TBD: Send a shutdown message to routers. + mgmt_log("[LocalManager::mgmtShutdown] Executing shutdown request.\n"); + processShutdown(true); + // WCCP TBD: Send a shutdown message to routers. - if (processRunning()) { - waitpid(watched_process_pid, NULL, 0); + if (processRunning()) { + waitpid(watched_process_pid, NULL, 0); #if defined(linux) - /* Avert race condition, wait for the thread to complete, - before getting one more restart process */ - /* Workaround for bugid INKqa10060 */ - mgmt_sleep_msec(1); + /* Avert race condition, wait for the thread to complete, + before getting one more restart process */ + /* Workaround for bugid INKqa10060 */ + mgmt_sleep_msec(1); #endif - } - mgmtCleanup(); - } else { - mgmt_shutdown_outstanding = true; } - return; + mgmtCleanup(); } @@ -207,7 +202,7 @@ LocalManager::LocalManager(bool proxy_on) proxy_launch_count = 0; manager_started_at = time(NULL); proxy_launch_outstanding = false; - mgmt_shutdown_outstanding = false; + mgmt_shutdown_outstanding = MGMT_PENDING_NONE; proxy_running = 0; RecSetRecordInt("proxy.node.proxy_running", 0); mgmt_sync_key = REC_readInteger("proxy.config.lm.sem_id", &found); http://git-wip-us.apache.org/repos/asf/trafficserver/blob/c7554f26/mgmt/LocalManager.h ---------------------------------------------------------------------- diff --git a/mgmt/LocalManager.h b/mgmt/LocalManager.h index aaf6cad..c7e9260 100644 --- a/mgmt/LocalManager.h +++ b/mgmt/LocalManager.h @@ -45,6 +45,15 @@ class FileManager; class ClusterCom; class VMap; +enum ManagementPendingOperation +{ + MGMT_PENDING_NONE, // Do nothing + MGMT_PENDING_RESTART, // Restart TS and TM + MGMT_PENDING_BOUNCE, // Restart TS + MGMT_PENDING_IDLE_RESTART, // Restart TS and TM when TS is idle + MGMT_PENDING_IDLE_BOUNCE // Restart TS when TS is idle +}; + class LocalManager: public BaseManager { public: @@ -72,7 +81,7 @@ public: void closeProxyPorts(); void mgmtCleanup(); - void mgmtShutdown(bool mainThread = false); + void mgmtShutdown(); void processShutdown(bool mainThread = false); void processRestart(); void processBounce(); @@ -98,7 +107,7 @@ public: volatile time_t proxy_started_at; volatile int proxy_launch_count; volatile bool proxy_launch_outstanding; - volatile bool mgmt_shutdown_outstanding; + volatile ManagementPendingOperation mgmt_shutdown_outstanding; volatile int proxy_running; HttpProxyPort::Group m_proxy_ports; // Local inbound addresses to bind, if set. http://git-wip-us.apache.org/repos/asf/trafficserver/blob/c7554f26/mgmt/RecordsConfig.cc ---------------------------------------------------------------------- diff --git a/mgmt/RecordsConfig.cc b/mgmt/RecordsConfig.cc index bb8f738..722c518 100644 --- a/mgmt/RecordsConfig.cc +++ b/mgmt/RecordsConfig.cc @@ -126,6 +126,8 @@ static const RecordElement RecordsConfig[] = , {RECT_CONFIG, "proxy.config.user_name", RECD_STRING, "nobody", RECU_NULL, RR_NULL, RECC_NULL, NULL, RECA_NULL} , + {RECT_CONFIG, "proxy.config.restart.active_client_threshold", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_INT, NULL, RECA_NULL} + , //############################################################################## //# http://git-wip-us.apache.org/repos/asf/trafficserver/blob/c7554f26/mgmt/api/CoreAPI.cc ---------------------------------------------------------------------- diff --git a/mgmt/api/CoreAPI.cc b/mgmt/api/CoreAPI.cc index 33351c5..35509ef 100644 --- a/mgmt/api/CoreAPI.cc +++ b/mgmt/api/CoreAPI.cc @@ -425,13 +425,14 @@ Reconfigure() * restart Traffic Manager!! */ TSMgmtError -Restart(bool cluster) +Restart(unsigned options) { - if (cluster) { // Enqueue an event to restart the proxies across the cluster + if (options & TS_RESTART_OPT_CLUSTER) { + // Enqueue an event to restart the proxies across the cluster // this will kill TM completely;traffic_cop will restart TM/TS lmgmt->ccom->sendClusterMessage(CLUSTER_MSG_SHUTDOWN_MANAGER); - } else { // just bounce local proxy - lmgmt->mgmtShutdown(); + } else { + lmgmt->mgmt_shutdown_outstanding = (options & TS_RESTART_OPT_DRAIN) ? MGMT_PENDING_IDLE_RESTART : MGMT_PENDING_RESTART; } return TS_ERR_OKAY; @@ -443,12 +444,12 @@ Restart(bool cluster) * Bounces traffic_server process(es). */ TSMgmtError -Bounce(bool cluster) +Bounce(unsigned options) { - if (cluster) { + if (options & TS_RESTART_OPT_CLUSTER) { lmgmt->ccom->sendClusterMessage(CLUSTER_MSG_BOUNCE_PROCESS); } else { - lmgmt->processBounce(); + lmgmt->mgmt_shutdown_outstanding = (options & TS_RESTART_OPT_DRAIN) ? MGMT_PENDING_IDLE_BOUNCE : MGMT_PENDING_BOUNCE; } return TS_ERR_OKAY; http://git-wip-us.apache.org/repos/asf/trafficserver/blob/c7554f26/mgmt/api/CoreAPI.h ---------------------------------------------------------------------- diff --git a/mgmt/api/CoreAPI.h b/mgmt/api/CoreAPI.h index 93ef6ec..b7a434f 100644 --- a/mgmt/api/CoreAPI.h +++ b/mgmt/api/CoreAPI.h @@ -48,8 +48,8 @@ TSMgmtError ProxyStateSet(TSProxyStateT state, TSCacheClearT clear); TSMgmtError ServerBacktrace(unsigned options, char ** trace); TSMgmtError Reconfigure(); // TS reread config files -TSMgmtError Restart(bool cluster); //restart TM -TSMgmtError Bounce(bool cluster); //restart traffic_server +TSMgmtError Restart(unsigned options); //restart TM +TSMgmtError Bounce(unsigned options); //restart traffic_server TSMgmtError StorageDeviceCmdOffline(const char * dev); // Storage device operation. /*************************************************************************** http://git-wip-us.apache.org/repos/asf/trafficserver/blob/c7554f26/mgmt/api/CoreAPIRemote.cc ---------------------------------------------------------------------- diff --git a/mgmt/api/CoreAPIRemote.cc b/mgmt/api/CoreAPIRemote.cc index 531659a..8ddc45b 100644 --- a/mgmt/api/CoreAPIRemote.cc +++ b/mgmt/api/CoreAPIRemote.cc @@ -408,13 +408,13 @@ Reconfigure() * so keep trying to reconnect until successful or for MAX_CONN_TRIES */ TSMgmtError -Restart(bool cluster) +Restart(unsigned options) { TSMgmtError ret; MgmtMarshallInt optype = RESTART; - MgmtMarshallInt bval = cluster ? 1 : 0; + MgmtMarshallInt oval = options; - ret = MGMTAPI_SEND_MESSAGE(main_socket_fd, RESTART, &optype, &bval); + ret = MGMTAPI_SEND_MESSAGE(main_socket_fd, RESTART, &optype, &oval); if (ret != TS_ERR_OKAY) { return ret; } @@ -434,13 +434,13 @@ Restart(bool cluster) * Restart the traffic_server process(es) only. */ TSMgmtError -Bounce(bool cluster) +Bounce(unsigned options) { TSMgmtError ret; MgmtMarshallInt optype = BOUNCE; - MgmtMarshallInt bval = cluster ? 1 : 0; + MgmtMarshallInt oval = options; - ret = MGMTAPI_SEND_MESSAGE(main_socket_fd, BOUNCE, &optype, &bval); + ret = MGMTAPI_SEND_MESSAGE(main_socket_fd, BOUNCE, &optype, &oval); if (ret != TS_ERR_OKAY) { return ret; } http://git-wip-us.apache.org/repos/asf/trafficserver/blob/c7554f26/mgmt/api/INKMgmtAPI.cc ---------------------------------------------------------------------- diff --git a/mgmt/api/INKMgmtAPI.cc b/mgmt/api/INKMgmtAPI.cc index dbfbe6b..ed40eba 100644 --- a/mgmt/api/INKMgmtAPI.cc +++ b/mgmt/api/INKMgmtAPI.cc @@ -1781,13 +1781,13 @@ TSReconfigure() } /* TSRestart: restarts Traffic Server - * Input: cluster - local or cluster-wide + * Input: options - bitmask of TSRestartOptionT * Output: TSMgmtError */ tsapi TSMgmtError -TSRestart(bool cluster) +TSRestart(unsigned options) { - return Restart(cluster); + return Restart(options); } /* TSActionDo: based on TSActionNeedT, will take appropriate action @@ -1818,13 +1818,13 @@ TSActionDo(TSActionNeedT action) } /* TSBouncer: restarts the traffic_server process(es) - * Input: cluster - local or cluster-wide + * Input: options - bitmask of TSRestartOptionT * Output: TSMgmtError */ tsapi TSMgmtError -TSBounce(bool cluster) +TSBounce(unsigned options) { - return Bounce(cluster); + return Bounce(options); } tsapi TSMgmtError http://git-wip-us.apache.org/repos/asf/trafficserver/blob/c7554f26/mgmt/api/TSControlMain.cc ---------------------------------------------------------------------- diff --git a/mgmt/api/TSControlMain.cc b/mgmt/api/TSControlMain.cc index 7810445..2b694cc 100644 --- a/mgmt/api/TSControlMain.cc +++ b/mgmt/api/TSControlMain.cc @@ -620,18 +620,16 @@ static TSMgmtError handle_restart(int fd, void * req, size_t reqlen) { MgmtMarshallInt optype; - MgmtMarshallInt cluster; + MgmtMarshallInt options; MgmtMarshallInt err; - err = recv_mgmt_request(req, reqlen, RESTART, &optype, &cluster); + err = recv_mgmt_request(req, reqlen, RESTART, &optype, &options); if (err == TS_ERR_OKAY) { - bool bounce = (optype == BOUNCE); - - // cluster == 0 means no cluster - if (bounce) - err = Bounce(0 != cluster); - else - err = Restart(0 != cluster); + switch (optype) { + case BOUNCE: err = Bounce(options); break; + case RESTART: err = Restart(options); break; + default: err = TS_ERR_PARAMS; break; + } } return send_mgmt_response(fd, RESTART, &err); http://git-wip-us.apache.org/repos/asf/trafficserver/blob/c7554f26/mgmt/api/include/mgmtapi.h ---------------------------------------------------------------------- diff --git a/mgmt/api/include/mgmtapi.h b/mgmt/api/include/mgmtapi.h index 0cb38ee..d7790a5 100644 --- a/mgmt/api/include/mgmtapi.h +++ b/mgmt/api/include/mgmtapi.h @@ -406,6 +406,12 @@ extern "C" TS_MGMT_OPT_NO_SOCK_TESTS /* No socket test thread */ } TSInitOptionT; + typedef enum + { + TS_RESTART_OPT_NONE = 0x0, + TS_RESTART_OPT_CLUSTER = 0x01, /* Restart across the cluster */ + TS_RESTART_OPT_DRAIN = 0x02, /* Wait for traffic to drain before restarting. */ + } TSRestartOptionT; /*************************************************************************** * Structures @@ -967,10 +973,10 @@ extern "C" tsapi TSMgmtError TSReconfigure(); /* TSRestart: restarts Traffic Manager and Traffic Server - * Input: cluster - local or cluster-wide + * Input: options - bitmask of TSRestartOptionT * Output: TSMgmtError */ - tsapi TSMgmtError TSRestart(bool cluster); + tsapi TSMgmtError TSRestart(unsigned options); /* TSActionDo: based on TSActionNeedT, will take appropriate action * Input: action - action that needs to be taken @@ -979,10 +985,10 @@ extern "C" tsapi TSMgmtError TSActionDo(TSActionNeedT action); /* TSBounce: restart the traffic_server process(es). - * Input: cluster - local or cluster-wide + * Input: options - bitmask of TSRestartOptionT * Output TSMgmtError */ - tsapi TSMgmtError TSBounce(bool cluster); + tsapi TSMgmtError TSBounce(unsigned options); /* TSStorageDeviceOp: Request an operation on a storage device. * @arg dev Target device, specified by path to device. http://git-wip-us.apache.org/repos/asf/trafficserver/blob/c7554f26/mgmt/cluster/ClusterCom.cc ---------------------------------------------------------------------- diff --git a/mgmt/cluster/ClusterCom.cc b/mgmt/cluster/ClusterCom.cc index 6293a37..7abad11 100644 --- a/mgmt/cluster/ClusterCom.cc +++ b/mgmt/cluster/ClusterCom.cc @@ -312,7 +312,7 @@ drainIncomingChannel(void *arg) delete buff; } else if (strstr(message, "cmd: shutdown_manager")) { mgmt_log("[ClusterCom::drainIncomingChannel] Received manager shutdown request\n"); - lmgmt->mgmtShutdown(); + lmgmt->mgmt_shutdown_outstanding = MGMT_PENDING_RESTART; } else if (strstr(message, "cmd: shutdown_process")) { mgmt_log("[ClusterCom::drainIncomingChannel] Received process shutdown request\n"); lmgmt->processShutdown(); @@ -1847,7 +1847,7 @@ ClusterCom::sendClusterMessage(int msg_type, const char *args) switch (msg_type) { case CLUSTER_MSG_SHUTDOWN_MANAGER: - lmgmt->mgmtShutdown(); + lmgmt->mgmt_shutdown_outstanding = MGMT_PENDING_RESTART; break; case CLUSTER_MSG_SHUTDOWN_PROCESS: lmgmt->processShutdown(); http://git-wip-us.apache.org/repos/asf/trafficserver/blob/c7554f26/mgmt/utils/MgmtLocalCleanup.cc ---------------------------------------------------------------------- diff --git a/mgmt/utils/MgmtLocalCleanup.cc b/mgmt/utils/MgmtLocalCleanup.cc index be9c39c..44fa531 100644 --- a/mgmt/utils/MgmtLocalCleanup.cc +++ b/mgmt/utils/MgmtLocalCleanup.cc @@ -28,6 +28,6 @@ void mgmt_cleanup() { if (lmgmt != NULL) { - lmgmt->mgmtShutdown(true); + lmgmt->mgmtShutdown(); } }
