This is an automated email from the ASF dual-hosted git repository.
huajianlan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 2e0559b0323 [enhancement](nereids) Speedup partition pruner (#38191)
2e0559b0323 is described below
commit 2e0559b0323a70235aa0ee1795b1ab7126b0c115
Author: 924060929 <[email protected]>
AuthorDate: Tue Jul 23 12:18:50 2024 +0800
[enhancement](nereids) Speedup partition pruner (#38191)
1. fast return when partition predicate is true/false/null
2. fast compute table's hash code
3. fast merge two ranges when equals
---
fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java | 10 ++++++----
.../src/main/java/org/apache/doris/catalog/OlapTable.java | 4 +---
.../src/main/java/org/apache/doris/common/TreeNode.java | 2 +-
.../rules/expression/rules/OneRangePartitionEvaluator.java | 4 ++++
.../nereids/rules/expression/rules/PartitionPruner.java | 12 +++++++++---
.../nereids/trees/expressions/literal/DateV2Literal.java | 11 ++++++++---
.../src/main/java/org/apache/doris/qe/ConnectContext.java | 3 ++-
.../src/main/java/org/apache/doris/qe/SessionVariable.java | 11 ++++++-----
8 files changed, 37 insertions(+), 20 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
index 89932e6ca08..6e9e713df9d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
@@ -57,6 +57,7 @@ import com.google.common.base.MoreObjects;
import com.google.common.base.Objects;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
+import com.google.common.base.Suppliers;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.gson.annotations.SerializedName;
@@ -74,6 +75,7 @@ import java.util.ListIterator;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
+import java.util.function.Supplier;
import java.util.stream.Collectors;
/**
@@ -277,7 +279,7 @@ public abstract class Expr extends TreeNode<Expr>
implements ParseNode, Cloneabl
protected Function fn;
// Cached value of IsConstant(), set during analyze() and valid if
isAnalyzed_ is true.
- private boolean isConstant;
+ private Supplier<Boolean> isConstant = Suppliers.memoize(() -> false);
// Flag to indicate whether to wrap this expr's toSql() in parenthesis.
Set by parser.
// Needed for properly capturing expr precedences in the SQL string.
@@ -460,7 +462,7 @@ public abstract class Expr extends TreeNode<Expr>
implements ParseNode, Cloneabl
Preconditions.checkState(!isAnalyzed);
// We need to compute the const-ness as the last step, since analysis
may change
// the result, e.g. by resolving function.
- isConstant = isConstantImpl();
+ isConstant = Suppliers.memoize(this::isConstantImpl);
isAnalyzed = true;
}
@@ -1353,7 +1355,7 @@ public abstract class Expr extends TreeNode<Expr>
implements ParseNode, Cloneabl
*/
public final boolean isConstant() {
if (isAnalyzed) {
- return isConstant;
+ return isConstant.get();
}
return isConstantImpl();
}
@@ -2540,7 +2542,7 @@ public abstract class Expr extends TreeNode<Expr>
implements ParseNode, Cloneabl
// In this case, agg output must be materialized whether
outer query block required or not.
if (f.getFunctionName().getFunction().equals("count")) {
for (Expr expr : funcExpr.children) {
- if (expr.isConstant && !(expr instanceof
LiteralExpr)) {
+ if (expr.isConstant() && !(expr instanceof
LiteralExpr)) {
return true;
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index 13f15335055..e1e3faaaa9c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -1986,9 +1986,7 @@ public class OlapTable extends Table implements
MTMVRelatedTableIf, GsonPostProc
@Override
public int hashCode() {
- return Objects.hash(state, indexIdToMeta, indexNameToId, keysType,
partitionInfo, idToPartition,
- nameToPartition, defaultDistributionInfo, tempPartitions,
bfColumns, bfFpp, colocateGroup,
- hasSequenceCol, sequenceType, indexes, baseIndexId,
tableProperty);
+ return (int) baseIndexId;
}
public Column getBaseColumn(String columnName) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/TreeNode.java
b/fe/fe-core/src/main/java/org/apache/doris/common/TreeNode.java
index b254dd9a1d0..8e14d6e10ae 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/TreeNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/TreeNode.java
@@ -33,7 +33,7 @@ import java.util.List;
*/
public class TreeNode<NodeType extends TreeNode<NodeType>> {
@SerializedName("children")
- protected ArrayList<NodeType> children = Lists.newArrayList();
+ protected ArrayList<NodeType> children = Lists.newArrayListWithCapacity(2);
public NodeType getChild(int i) {
return hasChild(i) ? children.get(i) : null;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java
index 4b7b940c9f6..8b01d2b8c67 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java
@@ -571,6 +571,10 @@ public class OneRangePartitionEvaluator
Map<Slot, ColumnRange> leftRanges = left.columnRanges;
Map<Slot, ColumnRange> rightRanges = right.columnRanges;
+ if (leftRanges.equals(rightRanges)) {
+ return new EvaluateRangeResult(originResult, leftRanges,
ImmutableList.of(left, right));
+ }
+
Set<Slot> slots = ImmutableSet.<Slot>builder()
.addAll(leftRanges.keySet())
.addAll(rightRanges.keySet())
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java
index b65c0d2ec55..b0b45077dcc 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java
@@ -33,6 +33,7 @@ import
org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral;
import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
import
org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter;
import org.apache.doris.nereids.types.DateTimeType;
+import org.apache.doris.nereids.util.Utils;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableList.Builder;
@@ -125,14 +126,19 @@ public class PartitionPruner extends
DefaultExpressionRewriter<Void> {
"partitionPruningExpandThreshold",
10, sessionVariable ->
sessionVariable.partitionPruningExpandThreshold);
+ partitionPredicate = OrToIn.INSTANCE.rewriteTree(
+ partitionPredicate, new
ExpressionRewriteContext(cascadesContext));
+ if (BooleanLiteral.TRUE.equals(partitionPredicate)) {
+ return Utils.fastToImmutableList(idToPartitions.keySet());
+ } else if (Boolean.FALSE.equals(partitionPredicate) ||
partitionPredicate.isNullLiteral()) {
+ return ImmutableList.of();
+ }
+
List<OnePartitionEvaluator> evaluators =
Lists.newArrayListWithCapacity(idToPartitions.size());
for (Entry<Long, PartitionItem> kv : idToPartitions.entrySet()) {
evaluators.add(toPartitionEvaluator(
kv.getKey(), kv.getValue(), partitionSlots,
cascadesContext, expandThreshold));
}
-
- partitionPredicate = OrToIn.INSTANCE.rewriteTree(
- partitionPredicate, new
ExpressionRewriteContext(cascadesContext));
PartitionPruner partitionPruner = new PartitionPruner(evaluators,
partitionPredicate);
//TODO: we keep default partition because it's too hard to prune it,
we return false in canPrune().
return partitionPruner.prune();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateV2Literal.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateV2Literal.java
index d51a0eccc82..1534d551850 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateV2Literal.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateV2Literal.java
@@ -17,7 +17,6 @@
package org.apache.doris.nereids.trees.expressions.literal;
-import org.apache.doris.analysis.LiteralExpr;
import org.apache.doris.catalog.Type;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.trees.expressions.Expression;
@@ -25,12 +24,18 @@ import
org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.DateTimeV2Type;
import org.apache.doris.nereids.types.DateV2Type;
+import com.google.common.base.Suppliers;
+
import java.time.LocalDateTime;
+import java.util.function.Supplier;
/**
* date v2 literal for nereids
*/
public class DateV2Literal extends DateLiteral {
+ private final Supplier<org.apache.doris.analysis.DateLiteral>
legacyLiteral = Suppliers.memoize(() ->
+ new org.apache.doris.analysis.DateLiteral(year, month, day,
Type.DATEV2)
+ );
public DateV2Literal(String s) throws AnalysisException {
super(DateV2Type.INSTANCE, s);
@@ -41,8 +46,8 @@ public class DateV2Literal extends DateLiteral {
}
@Override
- public LiteralExpr toLegacyLiteral() {
- return new org.apache.doris.analysis.DateLiteral(year, month, day,
Type.DATEV2);
+ public org.apache.doris.analysis.DateLiteral toLegacyLiteral() {
+ return legacyLiteral.get();
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java
index e6d90fdd66d..b5f8c8d9be5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java
@@ -76,6 +76,7 @@ import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
+import io.netty.util.concurrent.FastThreadLocal;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.json.JSONObject;
@@ -99,7 +100,7 @@ import javax.annotation.Nullable;
// Use `volatile` to make the reference change atomic.
public class ConnectContext {
private static final Logger LOG =
LogManager.getLogger(ConnectContext.class);
- protected static ThreadLocal<ConnectContext> threadLocalInfo = new
ThreadLocal<>();
+ protected static FastThreadLocal<ConnectContext> threadLocalInfo = new
FastThreadLocal<>();
private static final String SSL_PROTOCOL = "TLS";
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index a7248119177..12455e31c76 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -3250,20 +3250,21 @@ public class SessionVariable implements Serializable,
Writable {
public Set<String> getDisableNereidsRuleNames() {
String checkPrivilege = RuleType.CHECK_PRIVILEGES.name();
String checkRowPolicy = RuleType.CHECK_ROW_POLICY.name();
- return Arrays.stream(disableNereidsRules.split(",[\\s]*"))
- .map(rule -> rule.toUpperCase(Locale.ROOT))
- .filter(rule -> !StringUtils.equalsIgnoreCase(rule,
checkPrivilege)
+ return Arrays.stream(disableNereidsRules.split(","))
+ .map(rule -> rule.trim().toUpperCase(Locale.ROOT))
+ .filter(rule -> !rule.isEmpty()
+ && !StringUtils.equalsIgnoreCase(rule, checkPrivilege)
&& !StringUtils.equalsIgnoreCase(rule, checkRowPolicy))
.collect(ImmutableSet.toImmutableSet());
}
public BitSet getDisableNereidsRules() {
BitSet bitSet = new BitSet();
- for (String ruleName : disableNereidsRules.split(",[\\s]*")) {
+ for (String ruleName : disableNereidsRules.split(",")) {
+ ruleName = ruleName.trim().toUpperCase(Locale.ROOT);
if (ruleName.isEmpty()) {
continue;
}
- ruleName = ruleName.toUpperCase(Locale.ROOT);
RuleType ruleType = RuleType.valueOf(ruleName);
if (ruleType == RuleType.CHECK_PRIVILEGES || ruleType ==
RuleType.CHECK_ROW_POLICY) {
continue;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
