This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 94a7b44540 [Improvement](log) add config to controll compression of fe
log & fe audit log (#22865)
94a7b44540 is described below
commit 94a7b445401f737da8007e86d617d5873412eff3
Author: Yulei-Yang <[email protected]>
AuthorDate: Fri Aug 11 14:08:08 2023 +0800
[Improvement](log) add config to controll compression of fe log & fe audit
log (#22865)
fe log is large for a busy doris cluster, if you want to preserve some
historical logs, it cost too much disk space.
enable compression is a good way to save space.
and a gzip compressed text file can be viewed without decompression.
---
docs/en/docs/admin-manual/config/fe-config.md | 12 ++++++++++++
docs/zh-CN/docs/admin-manual/config/fe-config.md | 12 ++++++++++++
.../src/main/java/org/apache/doris/common/Config.java | 7 +++++++
.../src/main/java/org/apache/doris/common/Log4jConfig.java | 11 ++++++++---
4 files changed, 39 insertions(+), 3 deletions(-)
diff --git a/docs/en/docs/admin-manual/config/fe-config.md
b/docs/en/docs/admin-manual/config/fe-config.md
index 6f45d9b1bb..a3eb2eade7 100644
--- a/docs/en/docs/admin-manual/config/fe-config.md
+++ b/docs/en/docs/admin-manual/config/fe-config.md
@@ -1699,6 +1699,12 @@ Default：SIZE-MB-1024
The size of the log split, split a log file every 1 G
+#### `sys_log_enable_compress`
+
+Default: false
+
+If true, will compress fe.log & fe.warn.log by gzip
+
#### `audit_log_dir`
Default：DORIS_HOME_DIR + "/log"
@@ -1744,6 +1750,12 @@ support format:
- 60m 60 min
- 120s 120 seconds
+#### `audit_log_enable_compress`
+
+Default: false
+
+If true, will compress fe.audit.log by gzip
+
### Storage
#### `min_replication_num_per_tablet`
diff --git a/docs/zh-CN/docs/admin-manual/config/fe-config.md
b/docs/zh-CN/docs/admin-manual/config/fe-config.md
index 7c33cdfd96..e8974ba6e2 100644
--- a/docs/zh-CN/docs/admin-manual/config/fe-config.md
+++ b/docs/zh-CN/docs/admin-manual/config/fe-config.md
@@ -1700,6 +1700,12 @@ sys_log_dir:
日志拆分的大小，每1G拆分一个日志文件
+#### `sys_log_enable_compress`
+
+默认值：false
+
+控制是否压缩fe log, 包括fe.log 及 fe.warn.log。如果开启，则使用gzip算法进行压缩。
+
#### `audit_log_dir`
默认值：DorisFE.DORIS_HOME_DIR + "/log"
@@ -1745,6 +1751,12 @@ HOUR: log前缀是：yyyyMMddHH
- 60m 60 分钟
- 120s 120 秒
+#### `audit_log_enable_compress`
+
+默认值：false
+
+控制是否压缩 fe.audit.log。如果开启，则使用gzip算法进行压缩。
+
### 存储
#### `min_replication_num_per_tablet`
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 268c2136f8..53b00dcefa 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -58,6 +58,9 @@ public class Config extends ConfigBase {
* 10h 10 hours
* 60m 60 mins
* 120s 120 seconds
+ *
+ * sys_log_enable_compress:
+ * default is false. if true, will compress fe.log & fe.warn.log by
gzip
*/
@ConfField(description = {"FE 日志文件的存放路径，用于存放 fe.log。",
"The path of the FE log file, used to store fe.log"})
@@ -93,6 +96,8 @@ public class Config extends ConfigBase {
"The maximum survival time of the FE log file. After exceeding
this time, the log file will be deleted. "
+ "Supported formats include: 7d, 10h, 60m, 120s"})
public static String sys_log_delete_age = "7d";
+ @ConfField(description = {"是否压缩 FE 的历史日志", "enable compression for FE log
file"})
+ public static boolean sys_log_enable_compress = false;
@ConfField(description = {"FE 审计日志文件的存放路径，用于存放 fe.audit.log。",
"The path of the FE audit log file, used to store fe.audit.log"})
@@ -118,6 +123,8 @@ public class Config extends ConfigBase {
+ "After exceeding this time, the log file will be
deleted. "
+ "Supported formats include: 7d, 10h, 60m, 120s"})
public static String audit_log_delete_age = "30d";
+ @ConfField(description = {"是否压缩 FE 的 Audit 日志", "enable compression for FE
audit log file"})
+ public static boolean audit_log_enable_compress = false;
@ConfField(description = {"插件的安装目录", "The installation directory of the
plugin"})
public static String plugin_dir = System.getenv("DORIS_HOME") + "/plugins";
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java
b/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java
index b4300ca426..f65d154a2c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java
@@ -47,7 +47,7 @@ public class Log4jConfig extends XmlConfiguration {
+ " <Pattern>%d{yyyy-MM-dd HH:mm:ss,SSS} %p
(%t|%tid)<!--REPLACED BY LOG FORMAT-->%m%n</Pattern>\n"
+ " </PatternLayout>\n"
+ " </Console>"
- + " <RollingFile name=\"Sys\"
fileName=\"${sys_log_dir}/fe.log\"
filePattern=\"${sys_log_dir}/fe.log.${sys_file_pattern}-%i\"
immediateFlush=\"${immediate_flush_flag}\">\n"
+ + " <RollingFile name=\"Sys\"
fileName=\"${sys_log_dir}/fe.log\"
filePattern=\"${sys_log_dir}/fe.log.${sys_file_pattern}-%i${sys_file_postfix}\"
immediateFlush=\"${immediate_flush_flag}\">\n"
+ " <PatternLayout charset=\"UTF-8\">\n"
+ " <Pattern>%d{yyyy-MM-dd HH:mm:ss,SSS} %p
(%t|%tid)<!--REPLACED BY LOG FORMAT-->%m%n</Pattern>\n"
+ " </PatternLayout>\n"
@@ -62,7 +62,7 @@ public class Log4jConfig extends XmlConfiguration {
+ " </Delete>\n"
+ " </DefaultRolloverStrategy>\n"
+ " </RollingFile>\n"
- + " <RollingFile name=\"SysWF\"
fileName=\"${sys_log_dir}/fe.warn.log\"
filePattern=\"${sys_log_dir}/fe.warn.log.${sys_file_pattern}-%i\"
immediateFlush=\"${immediate_flush_flag}\">\n"
+ + " <RollingFile name=\"SysWF\"
fileName=\"${sys_log_dir}/fe.warn.log\"
filePattern=\"${sys_log_dir}/fe.warn.log.${sys_file_pattern}-%i${sys_file_postfix}\"
immediateFlush=\"${immediate_flush_flag}\">\n"
+ " <PatternLayout charset=\"UTF-8\">\n"
+ " <Pattern>%d{yyyy-MM-dd HH:mm:ss,SSS} %p
(%t|%tid)<!--REPLACED BY LOG FORMAT-->%m%n</Pattern>\n"
+ " </PatternLayout>\n"
@@ -77,7 +77,7 @@ public class Log4jConfig extends XmlConfiguration {
+ " </Delete>\n"
+ " </DefaultRolloverStrategy>\n"
+ " </RollingFile>\n"
- + " <RollingFile name=\"Auditfile\"
fileName=\"${audit_log_dir}/fe.audit.log\"
filePattern=\"${audit_log_dir}/fe.audit.log.${audit_file_pattern}-%i\">\n"
+ + " <RollingFile name=\"Auditfile\"
fileName=\"${audit_log_dir}/fe.audit.log\"
filePattern=\"${audit_log_dir}/fe.audit.log.${audit_file_pattern}-%i${audit_file_postfix}\">\n"
+ " <PatternLayout charset=\"UTF-8\">\n"
+ " <Pattern>%d{yyyy-MM-dd HH:mm:ss,SSS} [%c{1}]
%m%n</Pattern>\n"
+ " </PatternLayout>\n"
@@ -132,6 +132,7 @@ public class Log4jConfig extends XmlConfiguration {
String sysLogDir = Config.sys_log_dir;
String sysRollNum = String.valueOf(Config.sys_log_roll_num);
String sysDeleteAge = String.valueOf(Config.sys_log_delete_age);
+ boolean compressSysLog = Config.sys_log_enable_compress;
if (!(sysLogLevel.equalsIgnoreCase("INFO")
|| sysLogLevel.equalsIgnoreCase("WARN")
@@ -162,6 +163,7 @@ public class Log4jConfig extends XmlConfiguration {
String auditRollNum = String.valueOf(Config.audit_log_roll_num);
String auditRollMaxSize = String.valueOf(Config.log_roll_size_mb);
String auditDeleteAge = String.valueOf(Config.audit_log_delete_age);
+ boolean compressAuditLog = Config.audit_log_enable_compress;
if (Config.audit_log_roll_interval.equals("HOUR")) {
auditLogRollPattern = "%d{yyyyMMddHH}";
} else if (Config.audit_log_roll_interval.equals("DAY")) {
@@ -197,6 +199,7 @@ public class Log4jConfig extends XmlConfiguration {
newXmlConfTemplate = newXmlConfTemplate.replaceAll("<!--REPLACED
BY Console Logger-->",
consoleLogger.toString());
}
+
Map<String, String> properties = Maps.newHashMap();
properties.put("sys_log_dir", sysLogDir);
properties.put("sys_file_pattern", sysLogRollPattern);
@@ -204,6 +207,7 @@ public class Log4jConfig extends XmlConfiguration {
properties.put("sys_roll_num", sysRollNum);
properties.put("sys_log_delete_age", sysDeleteAge);
properties.put("sys_log_level", sysLogLevel);
+ properties.put("sys_file_postfix", compressSysLog ? ".gz" : "");
properties.put("audit_log_dir", auditLogDir);
properties.put("audit_file_pattern", auditLogRollPattern);
@@ -212,6 +216,7 @@ public class Log4jConfig extends XmlConfiguration {
properties.put("audit_log_delete_age", auditDeleteAge);
properties.put("include_location_flag",
sysLogMode.equalsIgnoreCase("NORMAL") ? "true" : "false");
properties.put("immediate_flush_flag",
sysLogMode.equalsIgnoreCase("ASYNC") ? "false" : "true");
+ properties.put("audit_file_postfix", compressAuditLog ? ".gz" : "");
strSub = new StrSubstitutor(new Interpolator(properties));
newXmlConfTemplate = strSub.replace(newXmlConfTemplate);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
