This is an automated email from the ASF dual-hosted git repository. morningman pushed a commit to branch branch-1.2-lts in repository https://gitbox.apache.org/repos/asf/doris.git
commit 204ca0b2f3565fb99414ee2089d6c861ae29ed22 Author: Mingyu Chen <[email protected]> AuthorDate: Wed Feb 8 16:38:04 2023 +0800 [improvement](fuzzy) print fuzzy session variable in FE audit log (#16493) * [improvement](fuzzy) print fuzzy session variable in FE audit log --- .../java/org/apache/doris/plugin/AuditEvent.java | 7 ++++ .../java/org/apache/doris/qe/ConnectProcessor.java | 3 +- .../java/org/apache/doris/qe/SessionVariable.java | 40 ++++++++++++++++++---- .../main/java/org/apache/doris/qe/VariableMgr.java | 3 ++ 4 files changed, 46 insertions(+), 7 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/plugin/AuditEvent.java b/fe/fe-core/src/main/java/org/apache/doris/plugin/AuditEvent.java index 864a3e8029..597d156ea0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/plugin/AuditEvent.java +++ b/fe/fe-core/src/main/java/org/apache/doris/plugin/AuditEvent.java @@ -92,6 +92,8 @@ public class AuditEvent { public String sqlDigest = ""; @AuditField(value = "TraceId") public String traceId = ""; + @AuditField(value = "FuzzyVariables") + public String fuzzyVariables = ""; public static class AuditEventBuilder { @@ -214,6 +216,11 @@ public class AuditEvent { return this; } + public AuditEventBuilder setFuzzyVariables(String variables) { + auditEvent.fuzzyVariables = variables; + return this; + } + public AuditEvent build() { return this.auditEvent; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java index 493126af54..f6878ab0f2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java @@ -190,7 +190,8 @@ public class ConnectProcessor { .setReturnRows(ctx.getReturnRows()) .setStmtId(ctx.getStmtId()) .setQueryId(ctx.queryId() == null ? "NaN" : DebugUtil.printId(ctx.queryId())) - .setTraceId(spanContext.isValid() ? spanContext.getTraceId() : ""); + .setTraceId(spanContext.isValid() ? spanContext.getTraceId() : "") + .setFuzzyVariables(ctx.getSessionVariable().printFuzzyVariables()); if (ctx.getState().isQuery()) { MetricRepo.COUNTER_QUERY_ALL.increase(1L); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index db99c90e26..53b9e5f534 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -25,6 +25,7 @@ import org.apache.doris.qe.VariableMgr.VarAttr; import org.apache.doris.thrift.TQueryOptions; import org.apache.doris.thrift.TResourceLimit; +import com.google.common.base.Joiner; import com.google.common.base.Strings; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -388,7 +389,7 @@ public class SessionVariable implements Serializable, Writable { @VariableMgr.VarAttr(name = BATCH_SIZE) public int batchSize = 4064; - @VariableMgr.VarAttr(name = DISABLE_STREAMING_PREAGGREGATIONS) + @VariableMgr.VarAttr(name = DISABLE_STREAMING_PREAGGREGATIONS, fuzzy = true) public boolean disableStreamPreaggregations = false; @VariableMgr.VarAttr(name = DISABLE_COLOCATE_PLAN) @@ -407,7 +408,7 @@ public class SessionVariable implements Serializable, Writable { * the parallel exec instance num for one Fragment in one BE * 1 means disable this feature */ - @VariableMgr.VarAttr(name = PARALLEL_FRAGMENT_EXEC_INSTANCE_NUM) + @VariableMgr.VarAttr(name = PARALLEL_FRAGMENT_EXEC_INSTANCE_NUM, fuzzy = true) public int parallelExecInstanceNum = 1; @VariableMgr.VarAttr(name = ENABLE_INSERT_STRICT, needForward = true) @@ -495,7 +496,7 @@ public class SessionVariable implements Serializable, Writable { @VariableMgr.VarAttr(name = ENABLE_COST_BASED_JOIN_REORDER) private boolean enableJoinReorderBasedCost = false; - @VariableMgr.VarAttr(name = ENABLE_FOLD_CONSTANT_BY_BE) + @VariableMgr.VarAttr(name = ENABLE_FOLD_CONSTANT_BY_BE, fuzzy = true) private boolean enableFoldConstantByBe = false; @VariableMgr.VarAttr(name = RUNTIME_FILTER_MODE) @@ -554,7 +555,7 @@ public class SessionVariable implements Serializable, Writable { @VariableMgr.VarAttr(name = NEREIDS_STAR_SCHEMA_SUPPORT) private boolean nereidsStarSchemaSupport = true; - @VariableMgr.VarAttr(name = REWRITE_OR_TO_IN_PREDICATE_THRESHOLD) + @VariableMgr.VarAttr(name = REWRITE_OR_TO_IN_PREDICATE_THRESHOLD, fuzzy = true) private int rewriteOrToInPredicateThreshold = 2; @VariableMgr.VarAttr(name = NEREIDS_CBO_PENALTY_FACTOR) @@ -594,7 +595,7 @@ public class SessionVariable implements Serializable, Writable { @VariableMgr.VarAttr(name = ENABLE_FUNCTION_PUSHDOWN) public boolean enableFunctionPushdown; - @VariableMgr.VarAttr(name = ENABLE_LOCAL_EXCHANGE) + @VariableMgr.VarAttr(name = ENABLE_LOCAL_EXCHANGE, fuzzy = true) public boolean enableLocalExchange = true; /** @@ -644,7 +645,7 @@ public class SessionVariable implements Serializable, Writable { public boolean internalSession = false; // Use partitioned hash join if build side row count >= the threshold . 0 - the threshold is not set. - @VariableMgr.VarAttr(name = PARTITIONED_HASH_JOIN_ROWS_THRESHOLD) + @VariableMgr.VarAttr(name = PARTITIONED_HASH_JOIN_ROWS_THRESHOLD, fuzzy = true) public int partitionedHashJoinRowsThreshold = 0; @VariableMgr.VarAttr(name = ENABLE_SHARE_HASH_TABLE_FOR_BROADCAST_JOIN) @@ -669,6 +670,33 @@ public class SessionVariable implements Serializable, Writable { this.disableStreamPreaggregations = random.nextBoolean(); this.partitionedHashJoinRowsThreshold = random.nextBoolean() ? 8 : 1048576; this.enableShareHashTableForBroadcastJoin = random.nextBoolean(); + int randomInt = random.nextInt(4); + if (randomInt % 2 == 0) { + this.rewriteOrToInPredicateThreshold = 100000; + } else { + this.rewriteOrToInPredicateThreshold = 2; + } + } + + public String printFuzzyVariables() { + if (!Config.use_fuzzy_session_variable) { + return ""; + } + List<String> res = Lists.newArrayList(); + for (Field field : SessionVariable.class.getDeclaredFields()) { + VarAttr attr = field.getAnnotation(VarAttr.class); + if (attr == null || !attr.fuzzy()) { + continue; + } + field.setAccessible(true); + try { + Object val = field.get(this); + res.add(attr.name() + "=" + val.toString()); + } catch (IllegalAccessException e) { + LOG.warn("failed to get fuzzy session variable {}", attr.name(), e); + } + } + return Joiner.on(",").join(res); } public String getBlockEncryptionMode() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java b/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java index d6c224e68c..cd633dd9fb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java @@ -518,6 +518,9 @@ public class VariableMgr { // Set to true if the variables need to be forwarded along with forward statement. boolean needForward() default false; + + // Set to true if this variable is fuzzy + boolean fuzzy() default false; } private static class VarContext { --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
