This is an automated email from the ASF dual-hosted git repository. yiguolei pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/doris.git
commit 14de0974a5f8ad0a59cb8238f5e7d82bca521cca Author: Mingyu Chen <[email protected]> AuthorDate: Thu May 30 11:05:30 2024 +0800 [opt](audit) add timeout for audit log load and modify the label format (#35535) ## Proposed changes 1. Add a new global variable: `audit_plugin_load_timeout`, default is 600 sec Avoid using default stream load, which is 4 hours, too long for audit log load. 2. Modify the label of audit log load Add millisecond to avoid same label from different FE. --- .../apache/doris/plugin/audit/AuditStreamLoader.java | 17 ++++++++++++----- .../main/java/org/apache/doris/qe/GlobalVariable.java | 4 ++++ 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/plugin/audit/AuditStreamLoader.java b/fe/fe-core/src/main/java/org/apache/doris/plugin/audit/AuditStreamLoader.java index 72b046f015a..b29b3dfbe3b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/plugin/audit/AuditStreamLoader.java +++ b/fe/fe-core/src/main/java/org/apache/doris/plugin/audit/AuditStreamLoader.java @@ -20,6 +20,7 @@ package org.apache.doris.plugin.audit; import org.apache.doris.catalog.InternalSchema; import org.apache.doris.common.Config; import org.apache.doris.common.FeConstants; +import org.apache.doris.qe.GlobalVariable; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -61,6 +62,7 @@ public class AuditStreamLoader { conn.addRequestProperty("Expect", "100-continue"); conn.addRequestProperty("Content-Type", "text/plain; charset=UTF-8"); conn.addRequestProperty("label", label); + conn.setRequestProperty("timeout", String.valueOf(GlobalVariable.auditPluginLoadTimeoutS)); conn.addRequestProperty("max_filter_ratio", "1.0"); conn.addRequestProperty("columns", InternalSchema.AUDIT_SCHEMA.stream().map(c -> c.getName()).collect( @@ -105,11 +107,7 @@ public class AuditStreamLoader { } public LoadResponse loadBatch(StringBuilder sb, String clusterToken) { - Calendar calendar = Calendar.getInstance(); - String label = String.format("_log_%s%02d%02d_%02d%02d%02d_%s", - calendar.get(Calendar.YEAR), calendar.get(Calendar.MONTH) + 1, calendar.get(Calendar.DAY_OF_MONTH), - calendar.get(Calendar.HOUR_OF_DAY), calendar.get(Calendar.MINUTE), calendar.get(Calendar.SECOND), - feIdentity); + String label = genLabel(); HttpURLConnection feConn = null; HttpURLConnection beConn = null; @@ -159,6 +157,15 @@ public class AuditStreamLoader { } } + private String genLabel() { + Calendar calendar = Calendar.getInstance(); + return String.format("_log_%s%02d%02d_%02d%02d%02d_%s_%s", + calendar.get(Calendar.YEAR), calendar.get(Calendar.MONTH) + 1, calendar.get(Calendar.DAY_OF_MONTH), + calendar.get(Calendar.HOUR_OF_DAY), calendar.get(Calendar.MINUTE), calendar.get(Calendar.SECOND), + calendar.get(Calendar.MILLISECOND), + feIdentity); + } + public static class LoadResponse { public int status; public String respMsg; diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java index e34c4474848..14d8c35ff72 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java @@ -55,6 +55,7 @@ public final class GlobalVariable { public static final String AUDIT_PLUGIN_MAX_BATCH_BYTES = "audit_plugin_max_batch_bytes"; public static final String AUDIT_PLUGIN_MAX_BATCH_INTERVAL_SEC = "audit_plugin_max_batch_interval_sec"; public static final String AUDIT_PLUGIN_MAX_SQL_LENGTH = "audit_plugin_max_sql_length"; + public static final String AUDIT_PLUGIN_LOAD_TIMEOUT = "audit_plugin_load_timeout"; public static final String ENABLE_GET_ROW_COUNT_FROM_FILE_LIST = "enable_get_row_count_from_file_list"; public static final String READ_ONLY = "read_only"; @@ -131,6 +132,9 @@ public final class GlobalVariable { @VariableMgr.VarAttr(name = AUDIT_PLUGIN_MAX_SQL_LENGTH, flag = VariableMgr.GLOBAL) public static int auditPluginMaxSqlLength = 4096; + @VariableMgr.VarAttr(name = AUDIT_PLUGIN_LOAD_TIMEOUT, flag = VariableMgr.GLOBAL) + public static int auditPluginLoadTimeoutS = 600; + @VariableMgr.VarAttr(name = ENABLE_GET_ROW_COUNT_FROM_FILE_LIST, flag = VariableMgr.GLOBAL, description = { "针对外表，是否允许根据文件列表估算表行数。获取文件列表可能是一个耗时的操作，" --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
