This is an automated email from the ASF dual-hosted git repository.
amc pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/trafficserver.git
commit 35476688cd314041263915f5376d6222b1984be0
Author: dyrock <zeyu...@gmail.com>
AuthorDate: Wed Oct 24 13:59:13 2018 -0500
Tested locally
---
doc/admin-guide/files/logging.yaml.en.rst | 2 +
doc/admin-guide/files/records.config.en.rst | 30 ++++++--
doc/admin-guide/logging/rotation.en.rst | 10 ++-
mgmt/RecordsConfig.cc | 6 ++
proxy/logging/LogConfig.cc | 112 ++++++++++++++++++++--------
proxy/logging/LogConfig.h | 84 ++++++++++++++++++---
proxy/logging/YamlLogConfig.cc | 26 ++++++-
7 files changed, 217 insertions(+), 53 deletions(-)
diff --git a/doc/admin-guide/files/logging.yaml.en.rst
b/doc/admin-guide/files/logging.yaml.en.rst
index 670a492..10bc404 100644
--- a/doc/admin-guide/files/logging.yaml.en.rst
+++ b/doc/admin-guide/files/logging.yaml.en.rst
@@ -313,6 +313,8 @@ rolling_offset_hr number Specifies an hour (from
0 to 23) at which log
:ts:cv:`proxy.config.log.rolling_offset_hr`.
rolling_size_mb number Size, in megabytes, at which log files are
rolled.
+rolling_min_count number Specifies the minimum number of rolled logs
to
+ keep.
filters array of The optional list of filter objects which
filters restrict the individual events logged. The
array
may only contain one accept filter.
diff --git a/doc/admin-guide/files/records.config.en.rst
b/doc/admin-guide/files/records.config.en.rst
index cc0c46b..c773e97 100644
--- a/doc/admin-guide/files/records.config.en.rst
+++ b/doc/admin-guide/files/records.config.en.rst
@@ -272,6 +272,12 @@ System Variables
Specifies at what size to roll the output log at.
+.. ts:cv:: CONFIG proxy.config.output.logfile.rolling_min_count INT 0
+ :reloadable:
+
+ Specifies the minimum count of rolled output logs to keep. This value will
be used to decide the
+ order of auto-deletion (if enabled). A default value of 0 means
auto-deletion will try to keep
+ output logs as much as possible.
Thread Variables
----------------
@@ -2950,6 +2956,13 @@ Logging Configuration
The size, in megabytes, that log files must reach before rolling takes
place.
The minimum value for this setting is ``10``.
+.. ts:cv:: CONFIG proxy.config.log.rolling_min_count INT 0
+ :reloadable:
+
+ Specifies the minimum count of rolled (event) logs to keep. This value will
be used to decide the
+ order of auto-deletion (if enabled). A default value of 0 means
auto-deletion will try to keep
+ logs as much as possible. This value can be and should be overridden in
logging.yaml.
+
.. ts:cv:: CONFIG proxy.config.log.auto_delete_rolled_files INT 1
:reloadable:
@@ -3112,6 +3125,13 @@ Diagnostic Logging Configuration
Specifies at what size to roll the diagnostics log at.
+.. ts:cv:: CONFIG proxy.config.diags.logfile.rolling_min_count INT 0
+ :reloadable:
+
+ Specifies the minimum count of rolled diagnostic logs to keep. This value
will be used to decide the
+ order of auto-deletion (if enabled). A default value of 0 means
auto-deletion will try to keep
+ diagnostic logs as much as possible.
+
Reverse Proxy
=============
@@ -3457,11 +3477,11 @@ Client-Related Configuration
You can override this global setting on a per domain basis in the
ssl_servername.yaml file using the :ref:`verify_server_policy
attribute<override-verify-server-policy>`.
-:code:`DISABLED`
+:code:`DISABLED`
Server Certificate will not be verified
-:code:`PERMISSIVE`
+:code:`PERMISSIVE`
Certificate will be verified and the connection will not be established if
verification fails.
-:code:`ENFORCED`
+:code:`ENFORCED`
The provided certificate will be verified and the connection will be
established irrespective of the verification result. If verification fails the
name of the server will be logged.
.. ts:cv:: CONFIG proxy.config.ssl.client.verify.server.properties STRING ALL
@@ -3494,7 +3514,7 @@ Client-Related Configuration
:0: Server Certificate will not be verified
:1: Certificate will be verified and the connection will not be established
if verification fail
- :2: The provided certificate will be verified and the connection will be
established
+ :2: The provided certificate will be verified and the connection will be
established
.. ts:cv:: CONFIG proxy.config.ssl.client.cert.filename STRING NULL
:reloadable:
@@ -3930,7 +3950,7 @@ Sockets
Reduce the number of worker threads (net-threads)
Reduce the number of disk (AIO) threads
- Make sure accept threads are enabled
+ Make sure accept threads are enabled
The relevant configurations for this are::
diff --git a/doc/admin-guide/logging/rotation.en.rst
b/doc/admin-guide/logging/rotation.en.rst
index 94f25e6..5e01242 100644
--- a/doc/admin-guide/logging/rotation.en.rst
+++ b/doc/admin-guide/logging/rotation.en.rst
@@ -167,6 +167,11 @@ they reach a certain size, adjust the following settings in
CONFIG proxy.config.log.rolling_interval_sec INT 21600
+#. Set the minimum number of rolled files with
+ :ts:cv:`proxy.config.log.rolling_min_count`. ::
+
+ CONFIG proxy.config.log.rolling_min_count INT 0
+
#. Run the command :option:`traffic_ctl config reload` to apply the
configuration
changes.
@@ -190,8 +195,9 @@ following actions:
- If the autodelete option is enabled, then |TS| identifies previously-rolled
log files (log files with the ``.old`` extension). It starts deleting files
- one by one, beginning with the oldest file, until it emerges from the low
- state. |TS| logs a record of all deleted files in the system error log.
+ one by one, beginning with the oldest file with largest ratio to the minimum
+ rolling count, until it emerges from the low state. |TS| logs a record of
all
+ deleted files in the system error log.
- If the autodelete option is disabled or there are not enough old log files
to delete for the system to emerge from its low space state, then |TS|
diff --git a/mgmt/RecordsConfig.cc b/mgmt/RecordsConfig.cc
index a03324f..b28e330 100644
--- a/mgmt/RecordsConfig.cc
+++ b/mgmt/RecordsConfig.cc
@@ -95,6 +95,8 @@ static const RecordElement RecordsConfig[] =
,
{RECT_CONFIG, "proxy.config.output.logfile.rolling_size_mb", RECD_INT,
"100", RECU_DYNAMIC, RR_NULL, RECC_STR, "^0*[1-9][0-9]*$", RECA_NULL}
,
+ {RECT_CONFIG, "proxy.config.output.logfile.rolling_min_count", RECD_INT,
"0", RECU_DYNAMIC, RR_NULL, RECC_STR, "^0*[1-9][0-9]*$", RECA_NULL}
+ ,
//# 0 = disable
{RECT_CONFIG, "proxy.config.res_track_memory", RECD_INT, "0",
RECU_RESTART_TS, RR_REQUIRED, RECC_INT, "[0-2]", RECA_NULL}
,
@@ -230,6 +232,8 @@ static const RecordElement RecordsConfig[] =
,
{RECT_CONFIG, "proxy.config.diags.logfile.rolling_size_mb", RECD_INT, "10",
RECU_DYNAMIC, RR_NULL, RECC_STR, "^0*[1-9][0-9]*$", RECA_NULL}
,
+ {RECT_CONFIG, "proxy.config.diags.logfile.rolling_min_count", RECD_INT, "0",
RECU_DYNAMIC, RR_NULL, RECC_STR, "^0*[1-9][0-9]*$", RECA_NULL}
+ ,
//##############################################################################
//#
@@ -1044,6 +1048,8 @@ static const RecordElement RecordsConfig[] =
,
{RECT_CONFIG, "proxy.config.log.rolling_size_mb", RECD_INT, "10",
RECU_DYNAMIC, RR_NULL, RECC_STR, "^0*[1-9][0-9]*$", RECA_NULL}
,
+ {RECT_CONFIG, "proxy.config.log.rolling_min_count", RECD_INT, "0",
RECU_DYNAMIC, RR_NULL, RECC_STR, "^0*[1-9][0-9]*$", RECA_NULL}
+ ,
{RECT_CONFIG, "proxy.config.log.auto_delete_rolled_files", RECD_INT, "1",
RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-1]", RECA_NULL}
,
{RECT_CONFIG, "proxy.config.log.sampling_frequency", RECD_INT, "1",
RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL}
diff --git a/proxy/logging/LogConfig.cc b/proxy/logging/LogConfig.cc
index ce29674..7176855 100644
--- a/proxy/logging/LogConfig.cc
+++ b/proxy/logging/LogConfig.cc
@@ -63,6 +63,7 @@
#define DISK_IS_ACTUAL_LOW_MESSAGE "Access logging to local log directory
suspended - partition space is low."
#define PARTITION_HEADROOM_MB 10
+#define DIAGS_LOG_FILENAME "diags.log"
void
LogConfig::setup_default_values()
@@ -220,8 +221,8 @@ LogConfig::read_configuration_variables()
rolling_interval_sec =
(int)REC_ConfigReadInteger("proxy.config.log.rolling_interval_sec");
rolling_offset_hr =
(int)REC_ConfigReadInteger("proxy.config.log.rolling_offset_hr");
rolling_size_mb =
(int)REC_ConfigReadInteger("proxy.config.log.rolling_size_mb");
-
- val = (int)REC_ConfigReadInteger("proxy.config.log.rolling_enabled");
+ rolling_min_count =
(int)REC_ConfigReadInteger("proxy.config.log.rolling_min_count");
+ val =
(int)REC_ConfigReadInteger("proxy.config.log.rolling_enabled");
if (LogRollingEnabledIsValid(val)) {
rolling_enabled = (Log::RollingEnabledValues)val;
} else {
@@ -232,6 +233,23 @@ LogConfig::read_configuration_variables()
val =
(int)REC_ConfigReadInteger("proxy.config.log.auto_delete_rolled_files");
auto_delete_rolled_files = (val > 0);
+ // Read in min_count control values for auto deletion
+ if (auto_delete_rolled_files) {
+ // For diagnostic logs
+ val =
(int)REC_ConfigReadInteger("proxy.config.diags.logfile.rolling_min_count");
+ val = ((val == 0) ? INT_MAX : val);
+ deleting_info.insert(new LogDeletingInfo(DIAGS_LOG_FILENAME, val));
+
+ // For traffic.out
+ ats_scoped_str name(REC_ConfigReadString("proxy.config.output.logfile"));
+ val =
(int)REC_ConfigReadInteger("proxy.config.output.logfile.rolling_min_count");
+ val = ((val == 0) ? INT_MAX : val);
+ if (name) {
+ deleting_info.insert(new LogDeletingInfo(name.get(), val));
+ } else {
+ deleting_info.insert(new LogDeletingInfo("traffic.out", val));
+ }
+ }
// PERFORMANCE
val = (int)REC_ConfigReadInteger("proxy.config.log.sampling_frequency");
if (val > 0) {
@@ -697,12 +715,6 @@ LogConfig::space_to_write(int64_t bytes_to_write) const
periodic space check.
-------------------------------------------------------------------------*/
-static int
-delete_candidate_compare(const LogDeleteCandidate *a, const LogDeleteCandidate
*b)
-{
- return ((int)(a->mtime - b->mtime));
-}
-
void
LogConfig::update_space_used()
{
@@ -712,9 +724,7 @@ LogConfig::update_space_used()
return;
}
- static const int MAX_CANDIDATES = 128;
- LogDeleteCandidate candidates[MAX_CANDIDATES];
- int i, victim, candidate_count;
+ int candidate_count;
int64_t total_space_used, partition_space_left;
char path[MAXPATHLEN];
int sret;
@@ -758,6 +768,9 @@ LogConfig::update_space_used()
total_space_used = 0LL;
candidate_count = 0;
+ for (auto it = deleting_info.begin(); it != deleting_info.end(); it++) {
+ Debug("%s", it->name.c_str());
+ }
while ((entry = readdir(ld))) {
snprintf(path, MAXPATHLEN, "%s/%s", logfile_dir, entry->d_name);
@@ -765,13 +778,21 @@ LogConfig::update_space_used()
if (sret != -1 && S_ISREG(sbuf.st_mode)) {
total_space_used += (int64_t)sbuf.st_size;
- if (auto_delete_rolled_files && LogFile::rolled_logfile(entry->d_name)
&& candidate_count < MAX_CANDIDATES) {
+ if (auto_delete_rolled_files && LogFile::rolled_logfile(entry->d_name)) {
//
- // then add this entry to the candidate list
+ // then check if the candidate belongs to any given log type
//
- candidates[candidate_count].name = ats_strdup(path);
- candidates[candidate_count].size = (int64_t)sbuf.st_size;
- candidates[candidate_count].mtime = sbuf.st_mtime;
+ ts::TextView type_name(entry->d_name, strlen(entry->d_name));
+ auto suffix = type_name;
+ type_name.remove_suffix(suffix.remove_prefix(suffix.find('.') +
1).remove_prefix(suffix.find('.')).size());
+ auto iter = deleting_info.find(type_name);
+ if (iter == deleting_info.end()) {
+ // We won't delete the log if its name doesn't match any give type.
+ break;
+ }
+
+ auto &candidates = iter->candidates;
+ candidates.push_back(LogDeleteCandidate(path, (int64_t)sbuf.st_size,
sbuf.st_mtime));
candidate_count++;
}
}
@@ -818,36 +839,61 @@ LogConfig::update_space_used()
if (candidate_count > 0 && !space_to_write(headroom)) {
Debug("logspace", "headroom reached, trying to clear space ...");
Debug("logspace", "sorting %d delete candidates ...", candidate_count);
- qsort(candidates, candidate_count, sizeof(LogDeleteCandidate), (int
(*)(const void *, const void *))delete_candidate_compare);
- for (victim = 0; victim < candidate_count; victim++) {
+ deleting_info.apply([](LogDeletingInfo &info) {
+ std::sort(info.candidates.begin(), info.candidates.end(),
+ [](LogDeleteCandidate const &a, LogDeleteCandidate const &b) {
return a.mtime > b.mtime; });
+ });
+
+ while (candidate_count > 0) {
if (space_to_write(headroom + log_buffer_size)) {
Debug("logspace", "low water mark reached; stop deleting");
break;
}
- Debug("logspace", "auto-deleting %s", candidates[victim].name);
-
- if (unlink(candidates[victim].name) < 0) {
- Note("Traffic Server was Unable to auto-delete rolled "
- "logfile %s: %s.",
- candidates[victim].name, strerror(errno));
+ // Select the group with biggest ratio
+ auto target =
+ std::max_element(deleting_info.begin(), deleting_info.end(),
[](LogDeletingInfo const &A, LogDeletingInfo const &B) {
+ double diff =
+ static_cast<double>(A.candidates.size()) / A.min_count -
static_cast<double>(B.candidates.size()) / B.min_count;
+ return diff < 0.0;
+ });
+
+ auto &candidates = target->candidates;
+
+ // Check if any candidate exists
+ if (candidates.empty()) {
+ // This shouldn't be triggered unless min_count are configured wrong
or extra non-log files occupy the directory
+ Debug("logspace", "No more victims for log type %s. Check your
rolling_min_count settings and logging directory.",
+ target->name.c_str());
} else {
- Debug("logspace",
- "The rolled logfile, %s, was auto-deleted; "
- "%" PRId64 " bytes were reclaimed.",
- candidates[victim].name, candidates[victim].size);
- m_space_used -= candidates[victim].size;
- m_partition_space_left += candidates[victim].size;
+ auto &victim = candidates.back();
+ Debug("logspace", "auto-deleting %s", victim.name.c_str());
+
+ if (unlink(victim.name.c_str()) < 0) {
+ Note("Traffic Server was Unable to auto-delete rolled "
+ "logfile %s: %s.",
+ victim.name.c_str(), strerror(errno));
+ } else {
+ Debug("logspace",
+ "The rolled logfile, %s, was auto-deleted; "
+ "%" PRId64 " bytes were reclaimed.",
+ victim.name.c_str(), victim.size);
+
+ // Update after successful unlink;
+ m_space_used -= victim.size;
+ m_partition_space_left += victim.size;
+ }
+ // Update total candidates and remove victim
+ --candidate_count;
+ candidates.pop_back();
}
}
}
//
// Clean up the candidate array
//
- for (i = 0; i < candidate_count; i++) {
- ats_free(candidates[i].name);
- }
+ deleting_info.apply([](LogDeletingInfo &info) { info.clear(); });
//
// Now that we've updated the m_space_used value, see if we need to
diff --git a/proxy/logging/LogConfig.h b/proxy/logging/LogConfig.h
index 1920aa4..d8efcf3 100644
--- a/proxy/logging/LogConfig.h
+++ b/proxy/logging/LogConfig.h
@@ -23,6 +23,10 @@
#pragma once
+#include <string_view>
+#include <string>
+
+#include "tscore/IntrusiveHashMap.h"
#include "tscore/ink_platform.h"
#include "records/P_RecProcess.h"
#include "ProxyConfig.h"
@@ -75,6 +79,73 @@ struct dirent;
struct LogCollationAccept;
/*-------------------------------------------------------------------------
+ LogDeleteCandidate, LogDeletingInfo&Descriptor
+ -------------------------------------------------------------------------*/
+
+struct LogDeleteCandidate {
+ std::string name;
+ int64_t size;
+ time_t mtime;
+
+ LogDeleteCandidate(char *p_name, int64_t st_size, time_t st_time) :
name(p_name), size(st_size), mtime(st_time) {}
+};
+
+struct LogDeletingInfo {
+ std::string name;
+ int min_count;
+ std::vector<LogDeleteCandidate> candidates;
+
+ LogDeletingInfo *_next{nullptr};
+ LogDeletingInfo *_prev{nullptr};
+
+ LogDeletingInfo(const char *type, int limit) : name(type), min_count(limit)
{}
+ LogDeletingInfo(std::string_view type, int limit) : name(type),
min_count(limit) {}
+
+ void
+ clear()
+ {
+ candidates.clear();
+ }
+};
+
+struct LogDeletingInfoDescriptor {
+ using key_type = std::string_view;
+ using value_type = LogDeletingInfo;
+
+ static key_type
+ key_of(value_type *value)
+ {
+ return value->name;
+ }
+
+ static bool
+ equal(key_type const &lhs, key_type const &rhs)
+ {
+ return lhs == rhs;
+ }
+
+ static value_type *&
+ next_ptr(value_type *value)
+ {
+ return value->_next;
+ }
+
+ static value_type *&
+ prev_ptr(value_type *value)
+ {
+ return value->_prev;
+ }
+
+ static constexpr std::hash<std::string_view> hasher{};
+
+ static auto
+ hash_of(key_type s) -> decltype(hasher(s))
+ {
+ return hasher(s);
+ }
+};
+
+/*-------------------------------------------------------------------------
this object keeps the state of the logging configuraion variables. upon
construction, the log configuration file is read and the logging
variables are initialized.
@@ -188,8 +259,11 @@ public:
int rolling_interval_sec;
int rolling_offset_hr;
int rolling_size_mb;
+ int rolling_min_count;
bool auto_delete_rolled_files;
+ IntrusiveHashMap<LogDeletingInfoDescriptor> deleting_info;
+
int sampling_frequency;
int file_stat_frequency;
int space_used_frequency;
@@ -227,13 +301,3 @@ private:
LogConfig(const LogConfig &) = delete;
LogConfig &operator=(const LogConfig &) = delete;
};
-
-/*-------------------------------------------------------------------------
- LogDeleteCandidate
- -------------------------------------------------------------------------*/
-
-struct LogDeleteCandidate {
- time_t mtime;
- char *name;
- int64_t size;
-};
diff --git a/proxy/logging/YamlLogConfig.cc b/proxy/logging/YamlLogConfig.cc
index 07bfcfb..b3fd5bc 100644
--- a/proxy/logging/YamlLogConfig.cc
+++ b/proxy/logging/YamlLogConfig.cc
@@ -102,8 +102,8 @@ YamlLogConfig::loadLogConfig(const char *cfgFilename)
TsEnumDescriptor ROLLING_MODE = {{{"none", 0}, {"time", 1}, {"size", 2},
{"both", 3}, {"any", 4}}};
std::set<std::string> valid_log_object_keys = {
- "filename", "format", "mode", "header",
"rolling_enabled", "rolling_interval_sec",
- "rolling_offset_hr", "rolling_size_mb", "filters", "collation_hosts"};
+ "filename", "format", "mode", "header",
"rolling_enabled", "rolling_interval_sec",
+ "rolling_offset_hr", "rolling_size_mb", "filters", "collation_hosts",
"min_count"};
LogObject *
YamlLogConfig::decodeLogObject(const YAML::Node &node)
@@ -149,6 +149,7 @@ YamlLogConfig::decodeLogObject(const YAML::Node &node)
int obj_rolling_interval_sec = cfg->rolling_interval_sec;
int obj_rolling_offset_hr = cfg->rolling_offset_hr;
int obj_rolling_size_mb = cfg->rolling_size_mb;
+ int obj_min_count = cfg->rolling_min_count;
if (node["rolling_enabled"]) {
auto value = node["rolling_enabled"].as<std::string>();
@@ -166,7 +167,9 @@ YamlLogConfig::decodeLogObject(const YAML::Node &node)
if (node["rolling_size_mb"]) {
obj_rolling_size_mb = node["rolling_size_mb"].as<int>();
}
-
+ if (node["min_count"]) {
+ obj_min_count = node["min_count"].as<int>();
+ }
if (!LogRollingEnabledIsValid(obj_rolling_enabled)) {
Warning("Invalid log rolling value '%d' in log object",
obj_rolling_enabled);
}
@@ -175,6 +178,23 @@ YamlLogConfig::decodeLogObject(const YAML::Node &node)
(Log::RollingEnabledValues)obj_rolling_enabled,
Log::config->collation_preproc_threads,
obj_rolling_interval_sec,
obj_rolling_offset_hr, obj_rolling_size_mb);
+ // Generate LogDeletingInfo entry for later use
+ std::string ext;
+ switch (file_type) {
+ case LOG_FILE_ASCII:
+ ext = LOG_FILE_ASCII_OBJECT_FILENAME_EXTENSION;
+ break;
+ case LOG_FILE_PIPE:
+ ext = LOG_FILE_PIPE_OBJECT_FILENAME_EXTENSION;
+ break;
+ case LOG_FILE_BINARY:
+ ext = LOG_FILE_BINARY_OBJECT_FILENAME_EXTENSION;
+ break;
+ default:
+ break;
+ }
+ cfg->deleting_info.insert(new LogDeletingInfo(filename + ext,
((obj_min_count == 0) ? INT_MAX : obj_min_count)));
+
// filters
auto filters = node["filters"];
if (!filters) {
