Repository: incubator-impala Updated Branches: refs/heads/master 5eef1bf82 -> 218ed9215
IMPALA-4431: Add audit event log control mechanism to prevent disk overflow There is no limit to the number of audit event log files. When audit event log is enabled the growing number of log files will fill up the disk space. This patch adds checking and rotation mechanism on audit event log files to prevent file number out of control which causes disk overflow. Change-Id: I8c3229cbdb6275f969c15258c9ccab6efeb24368 Reviewed-on: http://gerrit.cloudera.org:8080/4971 Reviewed-by: Tim Armstrong <[email protected]> Tested-by: Impala Public Jenkins Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/218ed921 Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/218ed921 Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/218ed921 Branch: refs/heads/master Commit: 218ed9215648427dc08ece675e2a90088b71aa6c Parents: 5eef1bf Author: davidxdh <[email protected]> Authored: Sun Nov 6 16:27:32 2016 +0800 Committer: Impala Public Jenkins <[email protected]> Committed: Wed Dec 28 05:25:38 2016 +0000 ---------------------------------------------------------------------- be/src/common/init.cc | 6 ++++++ be/src/common/logging.cc | 16 ++++++++++++++++ be/src/common/logging.h | 9 +++++++-- be/src/service/impala-server.cc | 2 +- be/src/service/impala-server.h | 3 +++ 5 files changed, 33 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/218ed921/be/src/common/init.cc ---------------------------------------------------------------------- diff --git a/be/src/common/init.cc b/be/src/common/init.cc index 210a642..a1e734b 100644 --- a/be/src/common/init.cc +++ b/be/src/common/init.cc @@ -65,6 +65,10 @@ DEFINE_int32(max_log_files, 10, "Maximum number of log files to retain per sever "level. The most recent log files are retained. If set to 0, all log files are " "retained."); +DEFINE_int32(max_audit_event_log_files, 0, "Maximum number of audit event log files " + "to retain. The most recent audit event log files are retained. If set to 0, " + "all audit event log files are retained."); + DEFINE_int64(pause_monitor_sleep_time_ms, 500, "Sleep time in milliseconds for " "pause monitor thread."); @@ -142,6 +146,8 @@ static scoped_ptr<impala::Thread> pause_monitor; // Check for log rotation in every interval of the maintenance thread impala::CheckAndRotateLogFiles(FLAGS_max_log_files); + // Check for audit event log rotation in every interval of the maintenance thread + impala::CheckAndRotateAuditEventLogFiles(FLAGS_max_audit_event_log_files); } } http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/218ed921/be/src/common/logging.cc ---------------------------------------------------------------------- diff --git a/be/src/common/logging.cc b/be/src/common/logging.cc index a8430c3..4ef1aeb 100644 --- a/be/src/common/logging.cc +++ b/be/src/common/logging.cc @@ -32,6 +32,7 @@ #include <stdio.h> #include "common/logging.h" +#include "service/impala-server.h" #include "util/error-util.h" #include "util/logging-support.h" #include "util/redactor.h" @@ -47,6 +48,8 @@ DEFINE_bool(redirect_stdout_stderr, true, DECLARE_string(redaction_rules_file); +DECLARE_string(audit_event_log_dir); + using boost::uuids::random_generator; bool logging_initialized = false; @@ -170,3 +173,16 @@ void impala::CheckAndRotateLogFiles(int max_log_files) { impala::LoggingSupport::DeleteOldLogs(fname, max_log_files); } } + +void impala::CheckAndRotateAuditEventLogFiles(int max_log_files) { + // Return if audit event logging is disabled + if (FLAGS_audit_event_log_dir.empty()) return; + // Ignore bad input or disable log rotation + if (max_log_files <= 0) return; + // Check audit event log files + // Build glob pattern for input e.g. /tmp/impala_audit_event_log_1.0-* + string fname = strings::Substitute( + "$0/$1*", FLAGS_audit_event_log_dir, ImpalaServer::AUDIT_EVENT_LOG_FILE_PREFIX); + + impala::LoggingSupport::DeleteOldLogs(fname, max_log_files); +} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/218ed921/be/src/common/logging.h ---------------------------------------------------------------------- diff --git a/be/src/common/logging.h b/be/src/common/logging.h index 03a8294..057954b 100644 --- a/be/src/common/logging.h +++ b/be/src/common/logging.h @@ -82,9 +82,14 @@ void ShutdownLogging(); /// Writes all command-line flags to the log at level INFO. void LogCommandLineFlags(); -/// Helper function that checks for the number of logfiles in the log directory and removes -/// the oldest ones given an upper bound of number of logfiles to keep. +/// Helper function that checks for the number of logfiles in the log directory and +/// removes the oldest ones given an upper bound of number of logfiles to keep. void CheckAndRotateLogFiles(int max_log_files); + +/// Helper function that checks for the number of audit event logfiles in the log +/// directory and removes the oldest ones given an upper bound of number of audit event +/// logfiles to keep. +void CheckAndRotateAuditEventLogFiles(int max_log_files); } #endif // IR_COMPILE http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/218ed921/be/src/service/impala-server.cc ---------------------------------------------------------------------- diff --git a/be/src/service/impala-server.cc b/be/src/service/impala-server.cc index a5f176b..e159fa2 100644 --- a/be/src/service/impala-server.cc +++ b/be/src/service/impala-server.cc @@ -193,7 +193,7 @@ namespace impala { // relative to UTC. The same time zone change was made for the audit log, but the // version was kept at 1.0 because there is no known consumer of the timestamp. const string PROFILE_LOG_FILE_PREFIX = "impala_profile_log_1.1-"; -const string AUDIT_EVENT_LOG_FILE_PREFIX = "impala_audit_event_log_1.0-"; +const string ImpalaServer::AUDIT_EVENT_LOG_FILE_PREFIX = "impala_audit_event_log_1.0-"; const string LINEAGE_LOG_FILE_PREFIX = "impala_lineage_log_1.0-"; const uint32_t MAX_CANCELLATION_QUEUE_SIZE = 65536; http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/218ed921/be/src/service/impala-server.h ---------------------------------------------------------------------- diff --git a/be/src/service/impala-server.h b/be/src/service/impala-server.h index e2f9c67..0697357 100644 --- a/be/src/service/impala-server.h +++ b/be/src/service/impala-server.h @@ -270,6 +270,9 @@ class ImpalaServer : public ImpalaServiceIf, public ImpalaHiveServer2ServiceIf, /// Returns true if lineage logging is enabled, false otherwise. bool IsLineageLoggingEnabled(); + /// The prefix of audit event log filename. + static const string AUDIT_EVENT_LOG_FILE_PREFIX; + private: friend class ChildQuery; friend class ImpalaHttpHandler;
