This is an automated email from the ASF dual-hosted git repository. kxiao pushed a commit to branch branch-2.0 in repository https://gitbox.apache.org/repos/asf/doris.git
commit 8754f629a6c0dcd90634b8e46c881b602f5dd3bc Author: Mryange <[email protected]> AuthorDate: Thu Aug 3 13:12:32 2023 +0800 [feature](executor) using max_instance_num to limit automatically instance (#22521) --- docs/en/docs/admin-manual/config/fe-config.md | 9 --------- .../query-acceleration/pipeline-execution-engine.md | 1 + docs/zh-CN/docs/admin-manual/config/fe-config.md | 8 -------- .../query-acceleration/pipeline-execution-engine.md | 2 +- .../src/main/java/org/apache/doris/common/Config.java | 7 ------- .../main/java/org/apache/doris/analysis/SetVar.java | 18 +----------------- .../main/java/org/apache/doris/qe/SessionVariable.java | 7 ++++++- 7 files changed, 9 insertions(+), 43 deletions(-) diff --git a/docs/en/docs/admin-manual/config/fe-config.md b/docs/en/docs/admin-manual/config/fe-config.md index 3820d72fc4..224cb25a1f 100644 --- a/docs/en/docs/admin-manual/config/fe-config.md +++ b/docs/en/docs/admin-manual/config/fe-config.md @@ -2723,15 +2723,6 @@ MasterOnly: false Controls whether to enable query hit statistics. The default is false. -#### `max_instance_num` - -<version since="dev"></version> - -Default: 128 - -This is used to limit the setting of "parallel_fragment_exec_instance_num". -"parallel_fragment_exec_instance_num" cannot be set higher than "max_instance_num". - #### `div_precision_increment` <version since="dev"></version> diff --git a/docs/en/docs/query-acceleration/pipeline-execution-engine.md b/docs/en/docs/query-acceleration/pipeline-execution-engine.md index d32ca42217..8d3a2697c1 100644 --- a/docs/en/docs/query-acceleration/pipeline-execution-engine.md +++ b/docs/en/docs/query-acceleration/pipeline-execution-engine.md @@ -77,3 +77,4 @@ If the user upgrades from a lower version, the default value will be the paralle ``` set parallel_pipeline_task_num = 0; ``` +You can limit the automatically configured concurrency by setting "max_instance_num."（The default value is 64) diff --git a/docs/zh-CN/docs/admin-manual/config/fe-config.md b/docs/zh-CN/docs/admin-manual/config/fe-config.md index 9913de60d9..b93ca46774 100644 --- a/docs/zh-CN/docs/admin-manual/config/fe-config.md +++ b/docs/zh-CN/docs/admin-manual/config/fe-config.md @@ -2714,14 +2714,6 @@ show data （其他用法：HELP SHOW DATA） 这个参数主要用于避免因 external catalog 无法访问、信息过多等原因导致的查询 `information_schema` 超时的问题。 -#### `max_instance_num` - -<version since="dev"></version> - -默认值：128 - -用于限制parallel_fragment_exec_instance_num的设置，set parallel_fragment_exec_instance_num不能超过max_instance_num - #### `enable_query_hit_stats` <version since="dev"></version> diff --git a/docs/zh-CN/docs/query-acceleration/pipeline-execution-engine.md b/docs/zh-CN/docs/query-acceleration/pipeline-execution-engine.md index 0ce0f65448..3953d9c8d8 100644 --- a/docs/zh-CN/docs/query-acceleration/pipeline-execution-engine.md +++ b/docs/zh-CN/docs/query-acceleration/pipeline-execution-engine.md @@ -77,4 +77,4 @@ set enable_pipeline_engine = true; ``` set parallel_pipeline_task_num = 0; ``` - +可以通过设置max_instance_num来限制自动设置的并发数(默认为64) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index c5d8765128..1a6456fefa 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -1957,13 +1957,6 @@ public class Config extends ConfigBase { "Now default set to true, not support create complex type(array/struct/map) nested complex type " + "when we create table, only support array type nested array"}) public static boolean disable_nested_complex_type = true; - /* - * "max_instance_num" is used to set the maximum concurrency. When the value set - * by "parallel_fragment_exec_instance_num" is greater than "max_instance_num", - * an error will be reported. - */ - @ConfField(mutable = true) - public static int max_instance_num = 128; /* * This variable indicates the number of digits by which to increase the scale diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SetVar.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SetVar.java index a0d61248a5..6142feec89 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SetVar.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SetVar.java @@ -19,7 +19,6 @@ package org.apache.doris.analysis; import org.apache.doris.catalog.Env; import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.Config; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.common.UserException; @@ -179,22 +178,7 @@ public class SetVar { this.value = new StringLiteral(TimeUtils.checkTimeZoneValidAndStandardize(getResult().getStringValue())); this.result = (LiteralExpr) this.value; } - if (getVariable().equalsIgnoreCase(SessionVariable.PARALLEL_FRAGMENT_EXEC_INSTANCE_NUM)) { - int instanceNum = Integer.parseInt(getResult().getStringValue()); - if (instanceNum > Config.max_instance_num) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_VALUE_FOR_VAR, - SessionVariable.PARALLEL_FRAGMENT_EXEC_INSTANCE_NUM, - instanceNum + "(Should not be set to more than " + Config.max_instance_num + ")"); - } - } - if (getVariable().equalsIgnoreCase(SessionVariable.PARALLEL_PIPELINE_TASK_NUM)) { - int instanceNum = Integer.parseInt(getValue().getStringValue()); - if (instanceNum > Config.max_instance_num) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_VALUE_FOR_VAR, - SessionVariable.PARALLEL_PIPELINE_TASK_NUM, - instanceNum + "(Should not be set to more than " + Config.max_instance_num + ")"); - } - } + if (getVariable().equalsIgnoreCase(SessionVariable.EXEC_MEM_LIMIT)) { this.value = new StringLiteral(Long.toString(ParseUtil.analyzeDataVolumn(getResult().getStringValue()))); this.result = (LiteralExpr) this.value; diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 739f94cba7..4b4a4eba9d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -106,6 +106,7 @@ public class SessionVariable implements Serializable, Writable { public static final String ENABLE_BUCKET_SHUFFLE_JOIN = "enable_bucket_shuffle_join"; public static final String PARALLEL_FRAGMENT_EXEC_INSTANCE_NUM = "parallel_fragment_exec_instance_num"; public static final String PARALLEL_PIPELINE_TASK_NUM = "parallel_pipeline_task_num"; + public static final String MAX_INSTANCE_NUM = "max_instance_num"; public static final String ENABLE_INSERT_STRICT = "enable_insert_strict"; public static final String ENABLE_SPILLING = "enable_spilling"; public static final String ENABLE_EXCHANGE_NODE_PARALLEL_MERGE = "enable_exchange_node_parallel_merge"; @@ -563,6 +564,9 @@ public class SessionVariable implements Serializable, Writable { @VariableMgr.VarAttr(name = PARALLEL_PIPELINE_TASK_NUM, fuzzy = true) public int parallelPipelineTaskNum = 0; + @VariableMgr.VarAttr(name = MAX_INSTANCE_NUM) + public int maxInstanceNum = 64; + @VariableMgr.VarAttr(name = ENABLE_INSERT_STRICT, needForward = true) public boolean enableInsertStrict = true; @@ -1428,7 +1432,8 @@ public class SessionVariable implements Serializable, Writable { public int getParallelExecInstanceNum() { if (enablePipelineEngine && parallelPipelineTaskNum == 0) { int size = Env.getCurrentSystemInfo().getMinPipelineExecutorSize(); - return (size + 1) / 2; + int autoInstance = (size + 1) / 2; + return Math.min(autoInstance, maxInstanceNum); } else if (enablePipelineEngine) { return parallelPipelineTaskNum; } else { --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
