This is an automated email from the ASF dual-hosted git repository. jrushford pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/trafficserver.git
The following commit(s) were added to refs/heads/master by this push: new a028e7b Change HostStatus to use only one stat per host. The host stat is now a string value containing all time and reason data so that it may be restored from persistent store when ATS is restarted. a028e7b is described below commit a028e7b9b22d4dcca87544b93f079ff6a4165c91 Author: John Rushford <jrushf...@apache.org> AuthorDate: Thu May 2 13:39:50 2019 +0000 Change HostStatus to use only one stat per host. The host stat is now a string value containing all time and reason data so that it may be restored from persistent store when ATS is restarted. --- doc/appendices/command-line/traffic_ctl.en.rst | 40 ++- iocore/cache/test/stub.cc | 4 +- proxy/HostStatus.h | 140 +++++++-- proxy/ParentSelection.cc | 54 ++-- src/traffic_ctl/host.cc | 58 ++-- src/traffic_server/HostStatus.cc | 404 ++++++++++++++++++------- 6 files changed, 510 insertions(+), 190 deletions(-) diff --git a/doc/appendices/command-line/traffic_ctl.en.rst b/doc/appendices/command-line/traffic_ctl.en.rst index 6539eda..653598b 100644 --- a/doc/appendices/command-line/traffic_ctl.en.rst +++ b/doc/appendices/command-line/traffic_ctl.en.rst @@ -258,27 +258,53 @@ traffic_ctl host Get the current status of the hosts used in parent.config as a next hop in a multi-tiered cache hierarchy. The value 0 or 1 is returned indicating that the host is marked as down '0' or marked as up '1'. If a host is marked as down, it will not be used as the next hop parent, another host marked as up will be chosen. .. program:: traffic_ctl host -.. option:: down --time seconds --reason 'manual|active|local' HOSTNAME [HOSTNAME ...] +.. option:: down --time seconds --reason 'active|local|manual' HOSTNAME [HOSTNAME ...] Marks the listed hosts as down so that they will not be chosen as a next hop parent. If the --time option is included, the host is marked down for the specified number of seconds after which the host will automatically be marked up. 0 seconds marks the host down indefinitely until marked up manually and is the default. A reason tag may be used - when marking a host down. Valid values are 'manual', 'active', or 'local', 'manual' is - used as the default. The tags are used to indicate wehter the host was marked down - manually or by an 'active' or 'local' health check. There are three reason tag - metrics for each host that may be viewed to see the reason a host was marked down. + when marking a host down. Valid values are 'manual', 'active', and 'local', 'manual' + is used as the default if no reason is specified. The tags are used to indicate wether the host + was marked down manually or by an 'active' or 'local' health check. 'self_detect' indicates + that a parent entry in parent.config was marked down because the entry refers to the + local host so, it is automatically marked down to prevent requests from looping. A host is + not marked up until all reason codes are cleared by marking up the host for the specified + reason code. + + A stat is created for each host, with a the host fqdn and is prefixed with the string + `proxy.process.host_status` with a string value. The string value is a + serialized representation of the Host status struct showing all current data ie, reasons, + marked down times, and down time for each host. The stats may be viewed using the + `traffic_ctl metric` command or through the `stats_over_http` endpoint. .. program:: traffic_ctl host -.. option:: up --reason 'manual|active|local' HOSTNAME [HOSTNAME ...] +.. option:: up --reason 'active|local|manual' HOSTNAME [HOSTNAME ...] Marks the listed hosts as up so that they will be available for use as a next hop parent. By default, the 'manual' reason tag is used when marking up a host. Use the --reason - tag to mark the host reason stat as up using one of 'manual', 'active', or 'local'. + tag to mark the host reason code as up using one of 'manual', 'active', or 'local'. + The 'self_detect' is an internal reason code used by parent selection to mark down + a parent when it is identified as itself and `proxy.config.http.parent_proxy.self_detect' + is set to the default of 2. 'self_detect' down cannot be set or unset with traffic_ctl Examples ======== +Mark down a host with `traffic_ctl` and view the associated host stats:: + +$ traffic_ctl host down cdn-cache-02.foo.com --reason manual + +$ /opt/trafficserver/bin/traffic_ctl metric match host_status +proxy.process.host_status.cdn-cache-01.foo.com HOST_STATUS_DOWN,ACTIVE:UP:0:0,LOCAL:UP:0:0,MANUAL:DOWN:1556896844:0,SELF_DETECT:UP:0 +proxy.process.host_status.cdn-cache-02.foo.com HOST_STATUS_UP,ACTIVE:UP:0:0,LOCAL:UP:0:0,MANUAL:UP:0:0,SELF_DETECT:UP:0 +proxy.process.host_status.cdn-cache-origin-01.foo.com HOST_STATUS_UP,ACTIVE:UP:0:0,LOCAL:UP:0:0,MANUAL:UP:0:0,SELF_DETECT:UP:0 + +In the example above, 'cdn-cache-01.foo.com' is unavailable, `HOST_STATUS_DOWN` and was marked down +for the `manual` reason, `MANUAL:DOWN:1556896844:0`, at the time indicated by the UNIX time stamp +`1556896844`. To make the host available, one would have to clear the `manual` reason using:: +`traffic_ctl host up cdn-cache-01.foo.com --reason manual` + Configure Traffic Server to insert ``Via`` header in the response to the client:: diff --git a/iocore/cache/test/stub.cc b/iocore/cache/test/stub.cc index d6710ec..56a5da2 100644 --- a/iocore/cache/test/stub.cc +++ b/iocore/cache/test/stub.cc @@ -149,7 +149,7 @@ ts::svtoi(TextView src, TextView *out, int base) } void -HostStatus::setHostStatus(const char *name, HostStatus_t status, const unsigned int down_time, const char *reason) +HostStatus::setHostStatus(const char *name, HostStatus_t status, const unsigned int down_time, const unsigned int reason) { } @@ -160,7 +160,7 @@ HostStatus::getHostStatus(const char *name) } void -HostStatus::createHostStat(const char *name) +HostStatus::createHostStat(const char *name, const char *data) { } diff --git a/proxy/HostStatus.h b/proxy/HostStatus.h index f349b90..2fdc33c 100644 --- a/proxy/HostStatus.h +++ b/proxy/HostStatus.h @@ -32,30 +32,38 @@ #include <ctime> #include <string> +#include <sstream> #include "tscore/ink_rwlock.h" #include "records/P_RecProcess.h" #include <unordered_map> +// host_status stats prefix. +static const std::string stat_prefix = "proxy.process.host_status."; + enum HostStatus_t { HOST_STATUS_INIT, HOST_STATUS_DOWN, HOST_STATUS_UP, }; -struct HostStatRec_t { - HostStatus_t status; - time_t marked_down; // the time that this host was marked down. - unsigned int down_time; // number of seconds that the host should be down, 0 is indefinately -}; +static const constexpr char *HostStatusNames[3] = {"HOST_STATUS_INIT", "HOST_STATUS_DOWN", "HOST_STATUS_UP"}; +static const constexpr char *ReasonStatus[2] = {"UP", "DOWN"}; + +struct Reason { + static constexpr const unsigned int ACTIVE = 0x1; + static constexpr const unsigned int LOCAL = 0x2; + static constexpr const unsigned int MANUAL = 0x4; + static constexpr const unsigned int SELF_DETECT = 0x8; + static constexpr const unsigned int ALL = 0xf; -struct Reasons { - static constexpr const char *ACTIVE = "active"; - static constexpr const char *LOCAL = "local"; - static constexpr const char *MANUAL = "manual"; - static constexpr const char *SELF_DETECT = "self_detect"; + static constexpr const char *ACTIVE_REASON = "active"; + static constexpr const char *LOCAL_REASON = "local"; + static constexpr const char *MANUAL_REASON = "manual"; + static constexpr const char *SELF_DETECT_REASON = "self_detect"; + static constexpr const char *ALL_REASON = "all"; - static constexpr const char *reasons[4] = {ACTIVE, LOCAL, MANUAL, SELF_DETECT}; + static constexpr const char *reasons[3] = {ACTIVE_REASON, LOCAL_REASON, MANUAL_REASON}; static bool validReason(const char *reason) @@ -67,9 +75,102 @@ struct Reasons { } return false; } + + static unsigned int + getReason(const char *reason_str) + { + if (strcmp(reason_str, ACTIVE_REASON) == 0) { + return ACTIVE; + } else if (strcmp(reason_str, LOCAL_REASON) == 0) { + return LOCAL; + } else if (strcmp(reason_str, MANUAL_REASON) == 0) { + return MANUAL; + } else if (strcmp(reason_str, SELF_DETECT_REASON) == 0) { + return SELF_DETECT; + } else if (strcmp(reason_str, ALL_REASON) == 0) { + return ALL; + } + // default is MANUAL + return MANUAL; + } }; -static const std::string stat_prefix = "proxy.process.host_status."; +// host status POD +struct HostStatRec { + HostStatus_t status; + unsigned int reasons; + // time the host was marked down for a given reason. + time_t active_marked_down; + time_t local_marked_down; + time_t manual_marked_down; + time_t self_detect_marked_down; + // number of seconds that the host should be marked down for a given reason. + unsigned int active_down_time; + unsigned int local_down_time; + unsigned int manual_down_time; + + HostStatRec(); + HostStatRec(std::string str); + HostStatRec(const HostStatRec &src) + { + status = src.status; + reasons = src.reasons; + active_marked_down = src.active_marked_down; + active_down_time = src.active_down_time; + local_marked_down = src.local_marked_down; + local_down_time = src.local_down_time; + manual_marked_down = src.manual_marked_down; + manual_down_time = src.manual_down_time; + self_detect_marked_down = src.self_detect_marked_down; + } + ~HostStatRec() {} + + // serialize this HostStatusRec + std::stringstream & + operator<<(std::stringstream &os) + { + unsigned int r = getReasonState(Reason::ACTIVE); + os << HostStatusNames[status]; + os << ",ACTIVE:" << ReasonStatus[r] << ":" << active_marked_down << ":" << active_down_time; + r = getReasonState(Reason::LOCAL); + os << ",LOCAL:" << ReasonStatus[r] << ":" << local_marked_down << ":" << local_down_time; + r = getReasonState(Reason::MANUAL); + os << ",MANUAL:" << ReasonStatus[r] << ":" << manual_marked_down << ":" << manual_down_time; + r = getReasonState(Reason::SELF_DETECT); + os << ",SELF_DETECT:" << ReasonStatus[r] << ":" << self_detect_marked_down; + + return os; + } + + // serialize a HostStatRec + friend std::stringstream & + operator<<(std::stringstream &os, HostStatRec &hs) + { + unsigned int r = hs.getReasonState(Reason::ACTIVE); + os << HostStatusNames[hs.status]; + os << ",ACTIVE:" << ReasonStatus[r] << ":" << hs.active_marked_down << ":" << hs.active_down_time; + r = hs.getReasonState(Reason::LOCAL); + os << ",LOCAL:" << ReasonStatus[r] << ":" << hs.local_marked_down << ":" << hs.local_down_time; + r = hs.getReasonState(Reason::MANUAL); + os << ",MANUAL:" << ReasonStatus[r] << ":" << hs.manual_marked_down << ":" << hs.manual_down_time; + r = hs.getReasonState(Reason::SELF_DETECT); + os << ",SELF_DETECT:" << ReasonStatus[r] << ":" << hs.self_detect_marked_down; + + return os; + } + + inline unsigned int + getReasonState(unsigned int reason) + { + unsigned int r = 0; + if (reasons == 0) { + r = 0; + } else if (reasons & reason) { + r = 1; + } + return r; + } +}; /** * Singleton placeholder for next hop status. @@ -83,23 +184,20 @@ struct HostStatus { static HostStatus instance; return instance; } - void setHostStatus(const char *name, const HostStatus_t status, const unsigned int down_time, const char *reason); + void setHostStatus(const char *name, const HostStatus_t status, const unsigned int down_time, const unsigned int reason); HostStatus_t getHostStatus(const char *name); - void createHostStat(const char *name); + void createHostStat(const char *name, const char *data = nullptr); void loadHostStatusFromStats(); - int getHostStatId(const char *name); + void loadRecord(std::string &name, HostStatRec &h); + RecErrT getHostStat(std::string &stat_name, char *buf, unsigned int buf_len); private: - int next_stat_id = 1; HostStatus(); HostStatus(const HostStatus &obj) = delete; HostStatus &operator=(HostStatus const &) = delete; - // next hop status, key is hostname or ip string, data is bool (available). - std::unordered_map<std::string, HostStatRec_t *> hosts_statuses; - // next hop stat ids, key is hostname or ip string, data is int stat id. - std::unordered_map<std::string, int> hosts_stats_ids; + // next hop status, key is hostname or ip string, data is HostStatRec + std::unordered_map<std::string, HostStatRec *> hosts_statuses; ink_rwlock host_status_rwlock; - ink_rwlock host_statids_rwlock; }; diff --git a/proxy/ParentSelection.cc b/proxy/ParentSelection.cc index 6d3d0e5..fd3fef3 100644 --- a/proxy/ParentSelection.cc +++ b/proxy/ParentSelection.cc @@ -372,7 +372,7 @@ ParentRecord::PreProcessParents(const char *val, const int line_num, char *buf, continue; } else { Debug("parent_select", "token: %s, matches this machine. Marking down self from parent list at line %d", fqdn, line_num); - hs.setHostStatus(fqdn, HostStatus_t::HOST_STATUS_DOWN, 0, Reasons::SELF_DETECT); + hs.setHostStatus(fqdn, HostStatus_t::HOST_STATUS_DOWN, 0, Reason::SELF_DETECT); } } } else { @@ -384,7 +384,7 @@ ParentRecord::PreProcessParents(const char *val, const int line_num, char *buf, } else { Debug("parent_select", "token: %s, matches this machine. Marking down self from parent list at line %d", token, line_num); - hs.setHostStatus(token, HostStatus_t::HOST_STATUS_DOWN, 0, Reasons::SELF_DETECT); + hs.setHostStatus(token, HostStatus_t::HOST_STATUS_DOWN, 0, Reason::SELF_DETECT); } } } @@ -516,7 +516,7 @@ ParentRecord::ProcessParents(char *val, bool isPrimary) this->parents[i].name = this->parents[i].hash_string; } if (hs.getHostStatus(this->parents[i].hostname) == HostStatus_t::HOST_STATUS_INIT) { - hs.setHostStatus(this->parents[i].hostname, HOST_STATUS_UP, 0, Reasons::MANUAL); + hs.setHostStatus(this->parents[i].hostname, HOST_STATUS_UP, 0, Reason::MANUAL); } } else { memcpy(this->secondary_parents[i].hostname, current, tmp - current); @@ -534,7 +534,7 @@ ParentRecord::ProcessParents(char *val, bool isPrimary) this->secondary_parents[i].name = this->secondary_parents[i].hash_string; } if (hs.getHostStatus(this->secondary_parents[i].hostname) == HostStatus_t::HOST_STATUS_INIT) { - hs.setHostStatus(this->secondary_parents[i].hostname, HOST_STATUS_UP, 0, Reasons::MANUAL); + hs.setHostStatus(this->secondary_parents[i].hostname, HOST_STATUS_UP, 0, Reason::MANUAL); } } tmp3 = nullptr; @@ -1451,7 +1451,7 @@ EXCLUSIVE_REGRESSION_TEST(PARENTSELECTION)(RegressionTest * /* t ATS_UNUSED */, // Test 184 // mark fuzzy down with HostStatus API. HostStatus &_st = HostStatus::instance(); - _st.setHostStatus("fuzzy", HOST_STATUS_DOWN, 0, Reasons::MANUAL); + _st.setHostStatus("fuzzy", HOST_STATUS_DOWN, 0, Reason::MANUAL); ST(184); REINIT; @@ -1462,7 +1462,7 @@ EXCLUSIVE_REGRESSION_TEST(PARENTSELECTION)(RegressionTest * /* t ATS_UNUSED */, // Test 185 // mark fluffy down and expect furry to be chosen - _st.setHostStatus("fluffy", HOST_STATUS_DOWN, 0, Reasons::MANUAL); + _st.setHostStatus("fluffy", HOST_STATUS_DOWN, 0, Reason::MANUAL); ST(185); REINIT; @@ -1473,9 +1473,9 @@ EXCLUSIVE_REGRESSION_TEST(PARENTSELECTION)(RegressionTest * /* t ATS_UNUSED */, // Test 186 // mark furry and frisky down, fuzzy up and expect fuzzy to be chosen - _st.setHostStatus("furry", HOST_STATUS_DOWN, 0, Reasons::MANUAL); - _st.setHostStatus("frisky", HOST_STATUS_DOWN, 0, Reasons::MANUAL); - _st.setHostStatus("fuzzy", HOST_STATUS_UP, 0, Reasons::MANUAL); + _st.setHostStatus("furry", HOST_STATUS_DOWN, 0, Reason::MANUAL); + _st.setHostStatus("frisky", HOST_STATUS_DOWN, 0, Reason::MANUAL); + _st.setHostStatus("fuzzy", HOST_STATUS_UP, 0, Reason::MANUAL); ST(186); REINIT; @@ -1493,10 +1493,10 @@ EXCLUSIVE_REGRESSION_TEST(PARENTSELECTION)(RegressionTest * /* t ATS_UNUSED */, REBUILD; // mark all up. - _st.setHostStatus("furry", HOST_STATUS_UP, 0, Reasons::MANUAL); - _st.setHostStatus("fluffy", HOST_STATUS_UP, 0, Reasons::MANUAL); - _st.setHostStatus("frisky", HOST_STATUS_UP, 0, Reasons::MANUAL); - _st.setHostStatus("fuzzy", HOST_STATUS_UP, 0, Reasons::MANUAL); + _st.setHostStatus("furry", HOST_STATUS_UP, 0, Reason::MANUAL); + _st.setHostStatus("fluffy", HOST_STATUS_UP, 0, Reason::MANUAL); + _st.setHostStatus("frisky", HOST_STATUS_UP, 0, Reason::MANUAL); + _st.setHostStatus("fuzzy", HOST_STATUS_UP, 0, Reason::MANUAL); REINIT; br(request, "i.am.rabbit.net"); @@ -1506,7 +1506,7 @@ EXCLUSIVE_REGRESSION_TEST(PARENTSELECTION)(RegressionTest * /* t ATS_UNUSED */, // Test 188 // mark fuzzy down and expect fluffy. - _st.setHostStatus("fuzzy", HOST_STATUS_DOWN, 0, Reasons::MANUAL); + _st.setHostStatus("fuzzy", HOST_STATUS_DOWN, 0, Reason::MANUAL); ST(188); REINIT; @@ -1517,7 +1517,7 @@ EXCLUSIVE_REGRESSION_TEST(PARENTSELECTION)(RegressionTest * /* t ATS_UNUSED */, // Test 189 // mark fuzzy back up and expect fuzzy. - _st.setHostStatus("fuzzy", HOST_STATUS_UP, 0, Reasons::MANUAL); + _st.setHostStatus("fuzzy", HOST_STATUS_UP, 0, Reason::MANUAL); ST(189); REINIT; @@ -1533,7 +1533,7 @@ EXCLUSIVE_REGRESSION_TEST(PARENTSELECTION)(RegressionTest * /* t ATS_UNUSED */, // because the host status is set to down. params->markParentDown(result, fail_threshold, retry_time); // set host status down - _st.setHostStatus("fuzzy", HOST_STATUS_DOWN, 0, Reasons::MANUAL); + _st.setHostStatus("fuzzy", HOST_STATUS_DOWN, 0, Reason::MANUAL); // sleep long enough so that fuzzy is retryable sleep(params->policy.ParentRetryTime + 1); ST(190); @@ -1544,7 +1544,7 @@ EXCLUSIVE_REGRESSION_TEST(PARENTSELECTION)(RegressionTest * /* t ATS_UNUSED */, // now set the host staus on fuzzy to up and it should now // be retried. - _st.setHostStatus("fuzzy", HOST_STATUS_UP, 0, Reasons::MANUAL); + _st.setHostStatus("fuzzy", HOST_STATUS_UP, 0, Reason::MANUAL); ST(191); REINIT; br(request, "i.am.rabbit.net"); @@ -1557,10 +1557,10 @@ EXCLUSIVE_REGRESSION_TEST(PARENTSELECTION)(RegressionTest * /* t ATS_UNUSED */, T("dest_domain=rabbit.net parent=fuzzy:80,fluffy:80,furry:80,frisky:80 round_robin=false go_direct=true\n"); REBUILD; // mark all up. - _st.setHostStatus("fuzzy", HOST_STATUS_UP, 0, Reasons::MANUAL); - _st.setHostStatus("fluffy", HOST_STATUS_UP, 0, Reasons::MANUAL); - _st.setHostStatus("furry", HOST_STATUS_UP, 0, Reasons::MANUAL); - _st.setHostStatus("frisky", HOST_STATUS_UP, 0, Reasons::MANUAL); + _st.setHostStatus("fuzzy", HOST_STATUS_UP, 0, Reason::MANUAL); + _st.setHostStatus("fluffy", HOST_STATUS_UP, 0, Reason::MANUAL); + _st.setHostStatus("furry", HOST_STATUS_UP, 0, Reason::MANUAL); + _st.setHostStatus("frisky", HOST_STATUS_UP, 0, Reason::MANUAL); // fuzzy should be chosen. sleep(1); REINIT; @@ -1575,7 +1575,7 @@ EXCLUSIVE_REGRESSION_TEST(PARENTSELECTION)(RegressionTest * /* t ATS_UNUSED */, sleep(params->policy.ParentRetryTime + 1); // since the host status is down even though fuzzy is // retryable, fluffy should be chosen - _st.setHostStatus("fuzzy", HOST_STATUS_DOWN, 0, Reasons::MANUAL); + _st.setHostStatus("fuzzy", HOST_STATUS_DOWN, 0, Reason::MANUAL); REINIT; br(request, "i.am.rabbit.net"); FP; @@ -1585,7 +1585,7 @@ EXCLUSIVE_REGRESSION_TEST(PARENTSELECTION)(RegressionTest * /* t ATS_UNUSED */, // set the host status for fuzzy back up and since its // retryable fuzzy should be chosen ST(194); - _st.setHostStatus("fuzzy", HOST_STATUS_UP, 0, Reasons::MANUAL); + _st.setHostStatus("fuzzy", HOST_STATUS_UP, 0, Reason::MANUAL); REINIT; br(request, "i.am.rabbit.net"); FP; @@ -1706,7 +1706,7 @@ EXCLUSIVE_REGRESSION_TEST(PARENTSELECTION)(RegressionTest * /* t ATS_UNUSED */, T("dest_domain=rabbit.net parent=fuzzy:80|1.0;fluffy:80|1.0 secondary_parent=furry:80|1.0;frisky:80|1.0 " "round_robin=consistent_hash go_direct=false secondary_mode=3\n"); REBUILD; - _st.setHostStatus("fuzzy", HOST_STATUS_DOWN, 0, Reasons::MANUAL); + _st.setHostStatus("fuzzy", HOST_STATUS_DOWN, 0, Reason::MANUAL); REINIT; br(request, "i.am.rabbit.net"); FP; @@ -1777,15 +1777,15 @@ EXCLUSIVE_REGRESSION_TEST(PARENTSELECTION)(RegressionTest * /* t ATS_UNUSED */, "round_robin=consistent_hash go_direct=false\n"); REBUILD; REINIT; - _st.setHostStatus("curly", HOST_STATUS_DOWN, 0, Reasons::MANUAL); + _st.setHostStatus("curly", HOST_STATUS_DOWN, 0, Reason::MANUAL); br(request, "i.am.stooges.net"); FP; RE(verify(result, PARENT_SPECIFIED, "carol", 80), 211); // cleanup, allow changes to be persisted to records.snap // so that subsequent test runs do not fail unexpectedly. - _st.setHostStatus("curly", HOST_STATUS_UP, 0, Reasons::MANUAL); - _st.setHostStatus("fuzzy", HOST_STATUS_UP, 0, Reasons::MANUAL); + _st.setHostStatus("curly", HOST_STATUS_UP, 0, Reason::MANUAL); + _st.setHostStatus("fuzzy", HOST_STATUS_UP, 0, Reason::MANUAL); sleep(2); delete request; diff --git a/src/traffic_ctl/host.cc b/src/traffic_ctl/host.cc index 2b23bc9..1834104 100644 --- a/src/traffic_ctl/host.cc +++ b/src/traffic_ctl/host.cc @@ -33,18 +33,15 @@ CtrlEngine::status_get() TSMgmtError error; std::string str = stat_prefix + it; - for (const char *_reason_tag : Reasons::reasons) { - std::string _stat = str + "_" + _reason_tag; - error = record.fetch(_stat.c_str()); - if (error != TS_ERR_OKAY) { - CtrlMgmtError(error, "failed to fetch %s", it.c_str()); - status_code = CTRL_EX_ERROR; - return; - } - - if (REC_TYPE_IS_STAT(record.rclass())) { - std::cout << record.name() << ' ' << CtrlMgmtRecordValue(record).c_str() << std::endl; - } + error = record.fetch(str.c_str()); + if (error != TS_ERR_OKAY) { + CtrlMgmtError(error, "failed to fetch %s", it.c_str()); + status_code = CTRL_EX_ERROR; + return; + } + + if (REC_TYPE_IS_STAT(record.rclass())) { + std::cout << record.name() << ' ' << CtrlMgmtRecordValue(record).c_str() << std::endl; } } } @@ -52,21 +49,34 @@ CtrlEngine::status_get() void CtrlEngine::status_down() { - int down_time = 0; - std::string reason = arguments.get("reason").value(); + const char *usage = "traffic_ctl host down --reason 'active | local | manual' --time seconds host ...."; + unsigned int down_time = 0; + std::string reason = arguments.get("reason").value(); + std::string down = arguments.get("time").value(); // if reason is not set, set it to manual (default) if (reason.empty()) { - reason = Reasons::MANUAL; + reason = Reason::MANUAL; + } + if (!down.empty()) { + down_time = atoi(down.c_str()); } - if (!Reasons::validReason(reason.c_str())) { + if (!Reason::validReason(reason.c_str())) { fprintf(stderr, "\nInvalid reason: '%s'\n\n", reason.c_str()); - parser.help_message(); + fprintf(stderr, "Usage: %s\n\n", usage); + status_code = CTRL_EX_ERROR; + return; } TSMgmtError error = TS_ERR_OKAY; for (const auto &it : arguments.get("down")) { + if (strncmp(it.c_str(), "--", 2) == 0) { + fprintf(stderr, "\nInvalid option: %s\n", it.c_str()); + fprintf(stderr, "Usage: %s\n\n", usage); + status_code = CTRL_EX_ERROR; + return; + } error = TSHostStatusSetDown(it.c_str(), down_time, reason.c_str()); if (error != TS_ERR_OKAY) { CtrlMgmtError(error, "failed to set %s", it.c_str()); @@ -78,21 +88,29 @@ CtrlEngine::status_down() void CtrlEngine::status_up() { + const char *usage = "traffic_ctl host up --reason 'active | local | manual' host ...."; std::string reason = arguments.get("reason").value(); // if reason is not set, set it to manual (default) if (reason.empty()) { - reason = Reasons::MANUAL; + reason = Reason::MANUAL; } - if (!Reasons::validReason(reason.c_str())) { + if (!Reason::validReason(reason.c_str())) { fprintf(stderr, "\nInvalid reason: '%s'\n\n", reason.c_str()); - parser.help_message(); + fprintf(stderr, "Usage: %s\n\n", usage); + status_code = CTRL_EX_ERROR; + return; } TSMgmtError error; for (const auto &it : arguments.get("up")) { error = TSHostStatusSetUp(it.c_str(), 0, reason.c_str()); + if (strncmp("--", it.c_str(), 2) == 0) { + fprintf(stderr, "\nInvalid option: %s\n", it.c_str()); + fprintf(stderr, "Usage: %s\n\n", usage); + status_code = CTRL_EX_ERROR; + } if (error != TS_ERR_OKAY) { CtrlMgmtError(error, "failed to set %s", it.c_str()); status_code = CTRL_EX_ERROR; diff --git a/src/traffic_server/HostStatus.cc b/src/traffic_server/HostStatus.cc index 0facb4c..ac208f7 100644 --- a/src/traffic_server/HostStatus.cc +++ b/src/traffic_server/HostStatus.cc @@ -25,16 +25,10 @@ static RecRawStatBlock *host_status_rsb = nullptr; -static void -getStatName(std::string &stat_name, const char *name, const char *reason) +inline void +getStatName(std::string &stat_name, const char *name) { - stat_name = stat_prefix + name + "_"; - - if (reason == nullptr) { - stat_name += Reasons::MANUAL; - } else { - stat_name += reason; - } + stat_name = stat_prefix + name; } static void @@ -43,22 +37,27 @@ mgmt_host_status_up_callback(ts::MemSpan span) MgmtInt op; MgmtMarshallString name; MgmtMarshallInt down_time; - MgmtMarshallString reason; - std::string reason_stat; + MgmtMarshallString reason_str; + std::string stat_name; + char buf[1024] = {0}; char *data = static_cast<char *>(span.data()); auto len = span.size(); static const MgmtMarshallType fields[] = {MGMT_MARSHALL_INT, MGMT_MARSHALL_STRING, MGMT_MARSHALL_STRING, MGMT_MARSHALL_INT}; Debug("host_statuses", "%s:%s:%d - data: %s, len: %ld\n", __FILE__, __func__, __LINE__, data, len); - if (mgmt_message_parse(data, len, fields, countof(fields), &op, &name, &reason, &down_time) == -1) { + if (mgmt_message_parse(data, len, fields, countof(fields), &op, &name, &reason_str, &down_time) == -1) { Error("Plugin message - RPC parsing error - message discarded."); } - Debug("host_statuses", "op: %ld, name: %s, down_time: %d, reason: %s", static_cast<long>(op), name, static_cast<int>(down_time), - reason); + Debug("host_statuses", "op: %ld, name: %s, down_time: %d, reason_str: %s", static_cast<long>(op), name, + static_cast<int>(down_time), reason_str); + + unsigned int reason = Reason::getReason(reason_str); + + getStatName(stat_name, name); if (data != nullptr) { Debug("host_statuses", "marking up server %s", data); HostStatus &hs = HostStatus::instance(); - if (hs.getHostStatId(reason_stat.c_str()) == -1) { + if (hs.getHostStat(stat_name, buf, 1024) == REC_ERR_FAIL) { hs.createHostStat(name); } hs.setHostStatus(name, HostStatus_t::HOST_STATUS_UP, down_time, reason); @@ -71,61 +70,142 @@ mgmt_host_status_down_callback(ts::MemSpan span) MgmtInt op; MgmtMarshallString name; MgmtMarshallInt down_time; - MgmtMarshallString reason; - std::string reason_stat; + MgmtMarshallString reason_str; + std::string stat_name; char *data = static_cast<char *>(span.data()); + char buf[1024] = {0}; auto len = span.size(); static const MgmtMarshallType fields[] = {MGMT_MARSHALL_INT, MGMT_MARSHALL_STRING, MGMT_MARSHALL_STRING, MGMT_MARSHALL_INT}; Debug("host_statuses", "%s:%s:%d - data: %s, len: %ld\n", __FILE__, __func__, __LINE__, data, len); - if (mgmt_message_parse(data, len, fields, countof(fields), &op, &name, &reason, &down_time) == -1) { + if (mgmt_message_parse(data, len, fields, countof(fields), &op, &name, &reason_str, &down_time) == -1) { Error("Plugin message - RPC parsing error - message discarded."); } - Debug("host_statuses", "op: %ld, name: %s, down_time: %d, reason: %s", static_cast<long>(op), name, static_cast<int>(down_time), - reason); + Debug("host_statuses", "op: %ld, name: %s, down_time: %d, reason_str: %s", static_cast<long>(op), name, + static_cast<int>(down_time), reason_str); + + unsigned int reason = Reason::getReason(reason_str); if (data != nullptr) { Debug("host_statuses", "marking down server %s", name); HostStatus &hs = HostStatus::instance(); - if (hs.getHostStatId(reason_stat.c_str()) == -1) { + if (hs.getHostStat(stat_name, buf, 1024) == REC_ERR_FAIL) { hs.createHostStat(name); } hs.setHostStatus(name, HostStatus_t::HOST_STATUS_DOWN, down_time, reason); } } +HostStatRec::HostStatRec() + : status(HOST_STATUS_UP), + reasons(0), + active_marked_down(0), + local_marked_down(0), + manual_marked_down(0), + self_detect_marked_down(0), + active_down_time(0), + local_down_time(0), + manual_down_time(0){}; + +HostStatRec::HostStatRec(std::string str) +{ + std::vector<std::string> v1; + std::stringstream ss1(str); + + reasons = 0; + + // parse the csv strings from the stat record value string. + while (ss1.good()) { + char b1[64]; + ss1.getline(b1, 64, ','); + v1.push_back(b1); + } + + // v1 contains 5 strings. + ink_assert(v1.size() == 5); + + // set the status and reasons fields. + for (unsigned int i = 0; i < v1.size(); i++) { + if (i == 0) { // set the status field + if (v1.at(i).compare("HOST_STATUS_UP") == 0) { + status = HOST_STATUS_UP; + } else if (v1.at(i).compare("HOST_STATUS_DOWN") == 0) { + status = HOST_STATUS_DOWN; + } + } else { // parse and set remaining reason fields. + std::vector<std::string> v2; + v2.clear(); + std::stringstream ss2(v1.at(i)); + while (ss2.good()) { + char b2[64]; + ss2.getline(b2, 64, ':'); + v2.push_back(b2); + } + // v2 contains 4 strings. + ink_assert(v2.size() == 3 || v2.size() == 4); + + if (v2.at(0).compare("ACTIVE") == 0) { + if (v2.at(1).compare("DOWN") == 0) { + reasons |= Reason::ACTIVE; + } else if (reasons & Reason::ACTIVE) { + reasons ^= Reason::ACTIVE; + } + active_marked_down = atoi(v2.at(2).c_str()); + active_down_time = atoi(v2.at(3).c_str()); + } + if (v2.at(0).compare("LOCAL") == 0) { + if (v2.at(1).compare("DOWN") == 0) { + reasons |= Reason::LOCAL; + } else if (reasons & Reason::LOCAL) { + reasons ^= Reason::LOCAL; + } + local_marked_down = atoi(v2.at(2).c_str()); + local_down_time = atoi(v2.at(3).c_str()); + } + if (v2.at(0).compare("MANUAL") == 0) { + if (v2.at(1).compare("DOWN") == 0) { + reasons |= Reason::MANUAL; + } else if (reasons & Reason::MANUAL) { + reasons ^= Reason::MANUAL; + } + manual_marked_down = atoi(v2.at(2).c_str()); + manual_down_time = atoi(v2.at(3).c_str()); + } + if (v2.at(0).compare("SELF_DETECT") == 0) { + if (v2.at(1).compare("DOWN") == 0) { + reasons |= Reason::SELF_DETECT; + } else if (reasons & Reason::SELF_DETECT) { + reasons ^= Reason::SELF_DETECT; + } + self_detect_marked_down = atoi(v2.at(2).c_str()); + } + } + } +} + static void handle_record_read(const RecRecord *rec, void *edata) { HostStatus &hs = HostStatus::instance(); std::string hostname; - std::string reason; if (rec) { + Debug("host_statuses", "name: %s", rec->name); + // parse the hostname from the stat name char *s = const_cast<char *>(rec->name); // 1st move the pointer past the stat prefix. - s += strlen(stat_prefix.c_str()); + s += stat_prefix.length(); hostname = s; - // parse the reason from the stat name. - reason = hostname.substr(hostname.find('_')); - reason.erase(0, 1); - // erase the reason tag - hostname.erase(hostname.find('_')); - - // if the data loaded from stats indicates that the host was down, - // then update the state so that the host remains down until - // specifically marked up using traffic_ctl. - if (rec->data.rec_int == 0 && Reasons::validReason(reason.c_str())) { - hs.setHostStatus(hostname.c_str(), HOST_STATUS_DOWN, 0, reason.c_str()); - } + hs.createHostStat(hostname.c_str(), rec->data.rec_string); + HostStatRec h(rec->data.rec_string); + hs.loadRecord(hostname, h); } } HostStatus::HostStatus() { ink_rwlock_init(&host_status_rwlock); - ink_rwlock_init(&host_statids_rwlock); pmgmt->registerMgmtCallback(MGMT_EVENT_HOST_STATUS_UP, &mgmt_host_status_up_callback); pmgmt->registerMgmtCallback(MGMT_EVENT_HOST_STATUS_DOWN, &mgmt_host_status_down_callback); host_status_rsb = RecAllocateRawStatBlock((int)TS_MAX_API_STATS); @@ -136,64 +216,142 @@ HostStatus::~HostStatus() for (auto &&it : hosts_statuses) { ats_free(it.second); } - // release host_stats_ids hash and the read and writer locks. + // release the read and writer locks. ink_rwlock_destroy(&host_status_rwlock); - ink_rwlock_destroy(&host_statids_rwlock); } void HostStatus::loadHostStatusFromStats() { - if (RecLookupMatchingRecords(RECT_ALL, stat_prefix.c_str(), handle_record_read, nullptr) != REC_ERR_OKAY) { + if (RecLookupMatchingRecords(RECT_PROCESS, stat_prefix.c_str(), handle_record_read, nullptr) != REC_ERR_OKAY) { Error("[HostStatus] - While loading HostStatus stats, there was an Error reading HostStatus stats."); } } void -HostStatus::setHostStatus(const char *name, HostStatus_t status, const unsigned int down_time, const char *reason) +HostStatus::loadRecord(std::string &name, HostStatRec &h) +{ + HostStatRec *host_stat = nullptr; + Debug("host_statuses", "loading host status record for %s", name.c_str()); + ink_rwlock_wrlock(&host_status_rwlock); + { + if (auto it = hosts_statuses.find(name.c_str()); it != hosts_statuses.end()) { + host_stat = it->second; + } else { + host_stat = static_cast<HostStatRec *>(ats_malloc(sizeof(HostStatRec))); + *host_stat = h; + hosts_statuses.emplace(name, host_stat); + } + } + ink_rwlock_unlock(&host_status_rwlock); + + *host_stat = h; +} + +void +HostStatus::setHostStatus(const char *name, HostStatus_t status, const unsigned int down_time, const unsigned int reason) { - std::string reason_stat; + std::string stat_name; + char buf[1024] = {0}; - getStatName(reason_stat, name, reason); + getStatName(stat_name, name); - if (getHostStatId(reason_stat.c_str()) == -1) { + if (getHostStat(stat_name, buf, 1024) == REC_ERR_FAIL) { createHostStat(name); } - int stat_id = getHostStatId(reason_stat.c_str()); + RecErrT result = getHostStat(stat_name, buf, 1024); - // update the stats - if (stat_id != -1) { + // update / insert status. + // using the hash table pointer to store the HostStatus_t value. + HostStatRec *host_stat = nullptr; + ink_rwlock_wrlock(&host_status_rwlock); + { + if (auto it = hosts_statuses.find(name); it != hosts_statuses.end()) { + host_stat = it->second; + } else { + host_stat = static_cast<HostStatRec *>(ats_malloc(sizeof(HostStatRec))); + bzero(host_stat, sizeof(HostStatRec)); + hosts_statuses.emplace(name, host_stat); + } + if (reason & Reason::ACTIVE) { + Debug("host_statuses", "for host %s set status: %s, Reason:ACTIVE", name, HostStatusNames[status]); + if (status == HostStatus_t::HOST_STATUS_DOWN) { + host_stat->active_marked_down = time(0); + host_stat->active_down_time = down_time; + host_stat->reasons |= Reason::ACTIVE; + } else { + host_stat->active_marked_down = 0; + host_stat->active_down_time = 0; + if (host_stat->reasons & Reason::ACTIVE) { + host_stat->reasons ^= Reason::ACTIVE; + } + } + } + if (reason & Reason::LOCAL) { + Debug("host_statuses", "for host %s set status: %s, Reason:LOCAL", name, HostStatusNames[status]); + if (status == HostStatus_t::HOST_STATUS_DOWN) { + host_stat->local_marked_down = time(0); + host_stat->local_down_time = down_time; + host_stat->reasons |= Reason::LOCAL; + } else { + host_stat->local_marked_down = 0; + host_stat->local_down_time = 0; + if (host_stat->reasons & Reason::LOCAL) { + host_stat->reasons ^= Reason::LOCAL; + } + } + } + if (reason & Reason::MANUAL) { + Debug("host_statuses", "for host %s set status: %s, Reason:MANUAL", name, HostStatusNames[status]); + if (status == HostStatus_t::HOST_STATUS_DOWN) { + host_stat->manual_marked_down = time(0); + host_stat->manual_down_time = down_time; + host_stat->reasons |= Reason::MANUAL; + } else { + host_stat->manual_marked_down = 0; + host_stat->manual_down_time = 0; + if (host_stat->reasons & Reason::MANUAL) { + host_stat->reasons ^= Reason::MANUAL; + } + } + } + if (reason & Reason::SELF_DETECT) { + Debug("host_statuses", "for host %s set status: %s, Reason:SELF_DETECT", name, HostStatusNames[status]); + if (status == HostStatus_t::HOST_STATUS_DOWN) { + host_stat->self_detect_marked_down = time(0); + host_stat->reasons |= Reason::SELF_DETECT; + } else { + host_stat->self_detect_marked_down = 0; + if (host_stat->reasons & Reason::SELF_DETECT) { + host_stat->reasons ^= Reason::SELF_DETECT; + } + } + } if (status == HostStatus_t::HOST_STATUS_UP) { - Debug("host_statuses", "set status up for : name: %s, status: %d, reason_stat: %s", name, status, reason_stat.c_str()); - RecSetRawStatCount(host_status_rsb, stat_id, 1); - RecSetRawStatSum(host_status_rsb, stat_id, 1); + if (host_stat->reasons == 0) { + host_stat->status = HostStatus_t::HOST_STATUS_UP; + } + Debug("host_statuses", "reasons: %d, status: %s", host_stat->reasons, HostStatusNames[host_stat->status]); } else { - Debug("host_statuses", "set status down for : name: %s, status: %d, reason_stat: %s", name, status, reason_stat.c_str()); - RecSetRawStatCount(host_status_rsb, stat_id, 0); - RecSetRawStatSum(host_status_rsb, stat_id, 0); + host_stat->status = status; + Debug("host_statuses", "reasons: %d, status: %s", host_stat->reasons, HostStatusNames[host_stat->status]); } } - Debug("host_statuses", "name: %s, status: %d", name, status); + ink_rwlock_unlock(&host_status_rwlock); - // update / insert status. - // using the hash table pointer to store the HostStatus_t value. - HostStatRec_t *host_stat = nullptr; - ink_rwlock_wrlock(&host_status_rwlock); - if (auto it = hosts_statuses.find(name); it != hosts_statuses.end()) { - host_stat = it->second; - } else { - host_stat = static_cast<HostStatRec_t *>(ats_malloc(sizeof(HostStatRec_t))); - hosts_statuses.emplace(name, host_stat); - } - host_stat->status = status; - host_stat->down_time = down_time; - if (status == HostStatus_t::HOST_STATUS_DOWN) { - host_stat->marked_down = time(0); - } else { - host_stat->marked_down = 0; + // update the stats + if (result == REC_ERR_OKAY) { + std::stringstream status_rec; + status_rec << *host_stat; + RecSetRecordString(stat_name.c_str(), const_cast<char *>(status_rec.str().c_str()), REC_SOURCE_EXPLICIT, true, false); + if (status == HostStatus_t::HOST_STATUS_UP) { + Debug("host_statuses", "set status up for name: %s, status: %d, stat_name: %s", name, status, stat_name.c_str()); + } else { + Debug("host_statuses", "set status down for name: %s, status: %d, stat_name: %s", name, status, stat_name.c_str()); + } } - ink_rwlock_unlock(&host_status_rwlock); + Debug("host_statuses", "name: %s, status: %d", name, status); // log it. if (status == HostStatus_t::HOST_STATUS_DOWN) { @@ -206,65 +364,85 @@ HostStatus::setHostStatus(const char *name, HostStatus_t status, const unsigned HostStatus_t HostStatus::getHostStatus(const char *name) { - HostStatRec_t *_status = 0; - time_t now = time(0); + HostStatRec *_status = 0; + time_t now = time(0); + bool lookup = false; // the hash table value pointer has the HostStatus_t value. ink_rwlock_rdlock(&host_status_rwlock); - auto it = hosts_statuses.find(name); - bool lookup = it != hosts_statuses.end(); - if (lookup) { - _status = it->second; + { + auto it = hosts_statuses.find(name); + lookup = it != hosts_statuses.end(); + if (lookup) { + _status = it->second; + } } ink_rwlock_unlock(&host_status_rwlock); // if the host was marked down and it's down_time has elapsed, mark it up. - if (lookup && _status->status == HostStatus_t::HOST_STATUS_DOWN && _status->down_time > 0) { - if ((_status->down_time + _status->marked_down) < now) { - Debug("host_statuses", "name: %s, now: %ld, down_time: %d, marked_down: %ld", name, now, _status->down_time, - _status->marked_down); - setHostStatus(name, HostStatus_t::HOST_STATUS_UP, 0, nullptr); + if (lookup && _status->status == HostStatus_t::HOST_STATUS_DOWN) { + unsigned int reasons = _status->reasons; + if ((_status->reasons & Reason::ACTIVE) && _status->active_down_time > 0) { + if ((_status->active_down_time + _status->active_marked_down) < now) { + Debug("host_statuses", "name: %s, now: %ld, down_time: %d, marked_down: %ld, reason: %s", name, now, + _status->active_down_time, _status->active_marked_down, Reason::ACTIVE_REASON); + setHostStatus(name, HostStatus_t::HOST_STATUS_UP, 0, Reason::ACTIVE); + reasons ^= Reason::ACTIVE; + } + } + if ((_status->reasons & Reason::LOCAL) && _status->local_down_time > 0) { + if ((_status->local_down_time + _status->local_marked_down) < now) { + Debug("host_statuses", "name: %s, now: %ld, down_time: %d, marked_down: %ld, reason: %s", name, now, + _status->local_down_time, _status->local_marked_down, Reason::LOCAL_REASON); + setHostStatus(name, HostStatus_t::HOST_STATUS_UP, 0, Reason::LOCAL); + reasons ^= Reason::LOCAL; + } + } + if ((_status->reasons & Reason::MANUAL) && _status->manual_down_time > 0) { + if ((_status->manual_down_time + _status->manual_marked_down) < now) { + Debug("host_statuses", "name: %s, now: %ld, down_time: %d, marked_down: %ld, reason: %s", name, now, + _status->manual_down_time, _status->manual_marked_down, Reason::MANUAL_REASON); + setHostStatus(name, HostStatus_t::HOST_STATUS_UP, 0, Reason::MANUAL); + reasons ^= Reason::MANUAL; + } + } + if (reasons == 0) { return HostStatus_t::HOST_STATUS_UP; + } else { + return HostStatus_t::HOST_STATUS_DOWN; } } - return lookup ? static_cast<HostStatus_t>(_status->status) : HostStatus_t::HOST_STATUS_INIT; + // didn't find this host in host status db, create the record + if (!lookup) { + createHostStat(name); + } + + return lookup ? static_cast<HostStatus_t>(_status->status) : HostStatus_t::HOST_STATUS_UP; } void -HostStatus::createHostStat(const char *name) +HostStatus::createHostStat(const char *name, const char *data) { - ink_rwlock_wrlock(&host_statids_rwlock); - { - for (const char *i : Reasons::reasons) { - std::string reason_stat; - getStatName(reason_stat, name, i); - if (hosts_stats_ids.find(reason_stat) == hosts_stats_ids.end()) { - RecRegisterRawStat(host_status_rsb, RECT_PROCESS, (reason_stat).c_str(), RECD_INT, RECP_PERSISTENT, (int)next_stat_id, - RecRawStatSyncSum); - RecSetRawStatCount(host_status_rsb, next_stat_id, 1); - RecSetRawStatSum(host_status_rsb, next_stat_id, 1); - - hosts_stats_ids.emplace(reason_stat, next_stat_id); - - Debug("host_statuses", "stat name: %s, id: %d", reason_stat.c_str(), next_stat_id); - next_stat_id++; - } - } - } - ink_rwlock_unlock(&host_statids_rwlock); -} + char buf[1024] = {0}; + HostStatRec r; -int -HostStatus::getHostStatId(const char *stat_name) -{ - int _id = -1; + std::string stat_name; + std::stringstream status_rec; + if (data != nullptr) { + HostStatRec h(data); + r = h; + } + status_rec << r; + getStatName(stat_name, name); - ink_rwlock_rdlock(&host_statids_rwlock); - if (auto it = hosts_stats_ids.find(stat_name); it != hosts_stats_ids.end()) { - _id = it->second; + if (getHostStat(stat_name, buf, 1024) == REC_ERR_FAIL) { + RecRegisterStatString(RECT_PROCESS, stat_name.c_str(), const_cast<char *>(status_rec.str().c_str()), RECP_PERSISTENT); + Debug("host_statuses", "stat name: %s, data: %s", stat_name.c_str(), status_rec.str().c_str()); } - ink_rwlock_unlock(&host_statids_rwlock); - Debug("host_statuses", "name: %s, id: %d", stat_name, static_cast<int>(_id)); +} - return _id; +RecErrT +HostStatus::getHostStat(std::string &stat_name, char *buf, unsigned int buf_len) +{ + return RecGetRecordString(stat_name.c_str(), buf, buf_len, true); }