This is an automated email from the ASF dual-hosted git repository. yiguolei pushed a commit to branch selectdb-doris-2.0-ikas in repository https://gitbox.apache.org/repos/asf/doris.git
commit 4d70881dca7c084f5c7cad3d68f7beec983039f3 Author: yuanyuan8983 <[email protected]> AuthorDate: Fri May 19 14:18:47 2023 +0800 potfire sql force limit --- .../doris/nereids/jobs/executor/Rewriter.java | 2 + .../org/apache/doris/nereids/rules/RuleType.java | 2 + .../nereids/rules/rewrite/ConvertPagingToFull.java | 55 ++++++++++++++++++++++ .../java/org/apache/doris/qe/SessionVariable.java | 15 ++++++ .../java/org/apache/doris/qe/StmtExecutor.java | 19 +++++++- 5 files changed, 92 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java index 56b313faf8c..c27671e7dbe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java @@ -46,6 +46,7 @@ import org.apache.doris.nereids.rules.rewrite.CollectFilterAboveConsumer; import org.apache.doris.nereids.rules.rewrite.CollectProjectAboveConsumer; import org.apache.doris.nereids.rules.rewrite.ColumnPruning; import org.apache.doris.nereids.rules.rewrite.ConvertInnerOrCrossJoin; +import org.apache.doris.nereids.rules.rewrite.ConvertPagingToFull; import org.apache.doris.nereids.rules.rewrite.CountDistinctRewrite; import org.apache.doris.nereids.rules.rewrite.CountLiteralToCountStar; import org.apache.doris.nereids.rules.rewrite.CreatePartitionTopNFromWindow; @@ -217,6 +218,7 @@ public class Rewriter extends AbstractBatchJobExecutor { ), topic("Eliminate optimization", bottomUp( + new ConvertPagingToFull(), new EliminateLimit(), new EliminateFilter(), new EliminateAggregate(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java index 14a23aa13af..36361781c03 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java @@ -282,6 +282,8 @@ public enum RuleType { COLLECT_PROJECT_ABOVE_FILTER_CONSUMER(RuleTypeClass.REWRITE), LEADING_JOIN(RuleTypeClass.REWRITE), + CONVERT_PAGING_TO_FULL(RuleTypeClass.REWRITE), + REWRITE_SENTINEL(RuleTypeClass.REWRITE), // topn opts diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ConvertPagingToFull.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ConvertPagingToFull.java new file mode 100644 index 00000000000..017815a56ec --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ConvertPagingToFull.java @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite; + +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.trees.plans.algebra.Sort; +import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation; +import org.apache.doris.nereids.trees.plans.logical.LogicalLimit; +import org.apache.doris.qe.ConnectContext; + +/** + * convert paging to full if enable rewrite limit + * If a statement has limit, but not have order by, then this statement is from spotfire + * spotfire will add select xxx (xxx) as xxx limit offset,length. + */ +public class ConvertPagingToFull extends OneRewriteRuleFactory { + @Override + public Rule build() { + return logicalResultSink(logicalLimit()) + .thenApply(ctx -> { + LogicalLimit<?> logicalLimit = ctx.root.child(); + if (!ctx.connectContext.getSessionVariable().getEnableRewriteLimit() + || logicalLimit.child() instanceof Sort + || ctx.connectContext.getSessionVariable().getMaxExecutionTimeMS() < 0) { + return null; + } + if (logicalLimit.getOffset() == 0) { + return ctx.root.withChildren(logicalLimit.child()); + } else { + // The second package has a offset > 0, just return empty, because we already + // return the full data in the first package. + return ctx.root.withChildren(new LogicalEmptyRelation( + ConnectContext.get().getStatementContext().getNextRelationId(), + logicalLimit.getOutput())); + } + }) + .toRule(RuleType.CONVERT_PAGING_TO_FULL); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index efcfec7570e..4af3fef704d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -69,10 +69,14 @@ public class SessionVariable implements Serializable, Writable { public static final String EXEC_MEM_LIMIT = "exec_mem_limit"; public static final String SCAN_QUEUE_MEM_LIMIT = "scan_queue_mem_limit"; public static final String QUERY_TIMEOUT = "query_timeout"; +<<<<<<< HEAD public static final String ANALYZE_TIMEOUT = "analyze_timeout"; +======= +>>>>>>> 29b4448635 (Fixed spotfire sql force limit) public static final String MAX_EXECUTION_TIME = "max_execution_time"; public static final String INSERT_TIMEOUT = "insert_timeout"; + public static final String ENABLE_REWRITE_LIMIT = "enable_rewrite_limit"; public static final String ENABLE_PROFILE = "enable_profile"; public static final String SQL_MODE = "sql_mode"; public static final String WORKLOAD_VARIABLE = "workload_group"; @@ -519,6 +523,8 @@ public class SessionVariable implements Serializable, Writable { @VariableMgr.VarAttr(name = INSERT_TIMEOUT) public int insertTimeoutS = 14400; + @VariableMgr.VarAttr(name = ENABLE_REWRITE_LIMIT) + public boolean enableRewriteLimit = false; // if true, need report to coordinator when plan fragment execute successfully. @VariableMgr.VarAttr(name = ENABLE_PROFILE, needForward = true) public boolean enableProfile = false; @@ -1560,10 +1566,15 @@ public class SessionVariable implements Serializable, Writable { return queryTimeoutS; } + public int getAnalyzeTimeoutS() { return analyzeTimeoutS; } + public boolean getEnableRewriteLimit() { + return this.enableRewriteLimit; + } + public void setEnableTwoPhaseReadOpt(boolean enable) { enableTwoPhaseReadOpt = enable; } @@ -1785,6 +1796,10 @@ public class SessionVariable implements Serializable, Writable { this.workloadGroup = workloadGroup; } + public void setEnableRewriteLimit(boolean enableRewriteLimit) { + this.enableRewriteLimit = enableRewriteLimit; + } + public String getResourceGroup() { return resourceGroup; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index 8f0a6b7b5a6..6e1cf3c1459 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -687,7 +687,24 @@ public class StmtExecutor { // IOException: talk with client failed. public void executeByLegacy(TUniqueId queryId) throws Exception { context.setStartTime(); - + SessionVariable sessionVariable2 = context.getSessionVariable(); + // If a statement has limit, but not have order by, then this statement is from spotfire + // spotfire will add select xxx (xxx) as xxx limit offset,length. + if (parsedStmt instanceof SelectStmt && sessionVariable2.getMaxExecutionTimeMS() > 0 + && sessionVariable2.getEnableRewriteLimit()) { + SelectStmt selectStmt = (SelectStmt) parsedStmt; + // If has limit and not have order by, this is from BI spotfire because limit without order + // by clause is unstable, user should not write this query. + if (selectStmt.hasLimitClause() && selectStmt.getOrderByElements() == null) { + if (selectStmt.getOffset() == 0) { + // This is the first batch, then we treat it as a total data, just remote limit clause + selectStmt.removeLimitElement(); + } else { + // This is the second batch, then just treat it as empty, to indicate finished pull data + selectStmt.setLimit(0); + } + } + } profile.getSummaryProfile().setQueryBeginTime(); context.setStmtId(STMT_ID_GENERATOR.incrementAndGet()); context.setQueryId(queryId); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
