This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-1.1-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-1.1-lts by this push:
new c61d8bbb21 [fix](planner) infer predicate could generate predicates in
another scope (#13877)
c61d8bbb21 is described below
commit c61d8bbb21fd70f23a60d39d0ddd4bcec1628d38
Author: morrySnow <[email protected]>
AuthorDate: Wed Nov 2 08:39:58 2022 +0800
[fix](planner) infer predicate could generate predicates in another scope
(#13877)
* [fix](planner) infer predicate could generate predicates in another scope
---
.../java/org/apache/doris/analysis/Analyzer.java | 147 ++++++++++++---------
.../suites/correctness/test_infer_predicate.groovy | 114 ++++++++++++++++
2 files changed, 199 insertions(+), 62 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java
index c0c71289ea..274170693e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java
@@ -202,6 +202,59 @@ public class Analyzer {
return globalState.autoBroadcastJoinThreshold;
}
+ private static class InferPredicateState {
+ // map from two table tuple ids to JoinOperator between two tables.
+ // NOTE: first tupleId's position in front of the second tupleId.
+ public final Map<Pair<TupleId, TupleId>, JoinOperator>
anyTwoTalesJoinOperator = Maps.newHashMap();
+
+ // slotEqSlotExpr: Record existing and infer equivalent connections
+ private final List<Expr> onSlotEqSlotExpr = new ArrayList<>();
+
+ // slotEqSlotDeDuplication: De-Duplication for slotEqSlotExpr
+ private final Set<Pair<Expr, Expr>> onSlotEqSlotDeDuplication =
Sets.newHashSet();
+
+ // slotToLiteralExpr: Record existing and infer expr which slot and
literal are equal
+ private final List<Expr> onSlotToLiteralExpr = new ArrayList<>();
+
+ // slotToLiteralDeDuplication: De-Duplication for slotToLiteralExpr
+ private final Set<Pair<Expr, Expr>> onSlotToLiteralDeDuplication =
Sets.newHashSet();
+
+ // inExpr: Recoud existing and infer expr which in predicate
+ private final List<Expr> onInExpr = new ArrayList<>();
+
+ // inExprDeDuplication: De-Duplication for inExpr
+ private final Set<Expr> onInDeDuplication = Sets.newHashSet();
+
+ // isNullExpr: Record existing and infer not null predicate
+ private final List<Expr> onIsNullExpr = new ArrayList<>();
+
+ //isNullDeDuplication: De-Duplication for isNullExpr
+ private final Set<Expr> onIsNullDeDuplication = Sets.newHashSet();
+
+ // slotToLiteralDeDuplication: De-Duplication for slotToLiteralExpr.
Contain on and where.
+ private final Set<Pair<Expr, Expr>> globalSlotToLiteralDeDuplication =
Sets.newHashSet();
+
+ // inExprDeDuplication: De-Duplication for inExpr. Contain on and where
+ private final Set<Expr> globalInDeDuplication = Sets.newHashSet();
+
+ public InferPredicateState() {
+ }
+
+ public InferPredicateState(InferPredicateState that) {
+ anyTwoTalesJoinOperator.putAll(that.anyTwoTalesJoinOperator);
+ onSlotEqSlotExpr.addAll(that.onSlotEqSlotExpr);
+ onSlotEqSlotDeDuplication.addAll(that.onSlotEqSlotDeDuplication);
+ onSlotToLiteralExpr.addAll(that.onSlotToLiteralExpr);
+
onSlotToLiteralDeDuplication.addAll(that.onSlotToLiteralDeDuplication);
+ onInExpr.addAll(that.onInExpr);
+ onInDeDuplication.addAll(that.onInDeDuplication);
+ onIsNullExpr.addAll(that.onIsNullExpr);
+ onIsNullDeDuplication.addAll(that.onIsNullDeDuplication);
+
globalSlotToLiteralDeDuplication.addAll(that.globalSlotToLiteralDeDuplication);
+ globalInDeDuplication.addAll(that.globalInDeDuplication);
+ }
+ }
+
// state shared between all objects of an Analyzer tree
// TODO: Many maps here contain properties about tuples, e.g., whether
// a tuple is outer/semi joined, etc. Remove the maps in favor of making
@@ -282,40 +335,6 @@ public class Analyzer {
// TODO chenhao16, to save conjuncts, which children are constant
public final Map<TupleId, Set<Expr>> constantConjunct =
Maps.newHashMap();
- // map from two table tuple ids to JoinOperator between two tables.
- // NOTE: first tupleId's position in front of the second tupleId.
- public final Map<Pair<TupleId, TupleId>, JoinOperator>
anyTwoTalesJoinOperator = Maps.newHashMap();
-
- // slotEqSlotExpr: Record existing and infer equivalent connections
- private final List<Expr> onSlotEqSlotExpr = new ArrayList<>();
-
- // slotEqSlotDeDuplication: De-Duplication for slotEqSlotExpr
- private final Set<Pair<Expr, Expr>> onSlotEqSlotDeDuplication =
Sets.newHashSet();
-
- // slotToLiteralExpr: Record existing and infer expr which slot and
literal are equal
- private final List<Expr> onSlotToLiteralExpr = new ArrayList<>();
-
- // slotToLiteralDeDuplication: De-Duplication for slotToLiteralExpr
- private final Set<Pair<Expr, Expr>> onSlotToLiteralDeDuplication =
Sets.newHashSet();
-
- // inExpr: Recoud existing and infer expr which in predicate
- private final List<Expr> onInExpr = new ArrayList<>();
-
- // inExprDeDuplication: De-Duplication for inExpr
- private final Set<Expr> onInDeDuplication = Sets.newHashSet();
-
- // isNullExpr: Record existing and infer not null predicate
- private final List<Expr> onIsNullExpr = new ArrayList<>();
-
- //isNullDeDuplication: De-Duplication for isNullExpr
- private final Set<Expr> onIsNullDeDuplication = Sets.newHashSet();
-
- // slotToLiteralDeDuplication: De-Duplication for slotToLiteralExpr.
Contain on and where.
- private final Set<Pair<Expr, Expr>> globalSlotToLiteralDeDuplication =
Sets.newHashSet();
-
- // inExprDeDuplication: De-Duplication for inExpr. Contain on and where
- private final Set<Expr> globalInDeDuplication = Sets.newHashSet();
-
// map from slot id to the analyzer/block in which it was registered
private final Map<SlotId, Analyzer> blockBySlot = Maps.newHashMap();
@@ -384,6 +403,8 @@ public class Analyzer {
private final GlobalState globalState;
+ private final InferPredicateState inferPredicateState;
+
// An analyzer stores analysis state for a single select block. A select
block can be
// a top level select statement, or an inline view select block.
// ancestors contains the Analyzers of the enclosing select blocks of
'this'
@@ -418,6 +439,7 @@ public class Analyzer {
public Analyzer(Catalog catalog, ConnectContext context) {
ancestors = Lists.newArrayList();
globalState = new GlobalState(catalog, context);
+ inferPredicateState = new InferPredicateState();
}
/**
@@ -427,7 +449,7 @@ public class Analyzer {
* @param parentAnalyzer the analyzer of the enclosing select block
*/
public Analyzer(Analyzer parentAnalyzer) {
- this(parentAnalyzer, parentAnalyzer.globalState);
+ this(parentAnalyzer, parentAnalyzer.globalState,
parentAnalyzer.inferPredicateState);
if (parentAnalyzer.isSubquery) {
this.isSubquery = true;
}
@@ -436,10 +458,11 @@ public class Analyzer {
/**
* Analyzer constructor for nested select block with the specified global
state.
*/
- private Analyzer(Analyzer parentAnalyzer, GlobalState globalState) {
+ private Analyzer(Analyzer parentAnalyzer, GlobalState globalState,
InferPredicateState inferPredicateState) {
ancestors = Lists.newArrayList(parentAnalyzer);
ancestors.addAll(parentAnalyzer.ancestors);
this.globalState = globalState;
+ this.inferPredicateState = new
InferPredicateState(inferPredicateState);
}
/**
@@ -448,7 +471,7 @@ public class Analyzer {
*/
public static Analyzer createWithNewGlobalState(Analyzer parentAnalyzer) {
GlobalState globalState = new
GlobalState(parentAnalyzer.globalState.catalog, parentAnalyzer.getContext());
- return new Analyzer(parentAnalyzer, globalState);
+ return new Analyzer(parentAnalyzer, globalState, new
InferPredicateState());
}
public void setIsExplain() { globalState.isExplain = true; }
@@ -962,47 +985,47 @@ public class Analyzer {
if (joinOperator == null) {
joinOperator = JoinOperator.INNER_JOIN;
}
- globalState.anyTwoTalesJoinOperator.put(tids, joinOperator);
+ inferPredicateState.anyTwoTalesJoinOperator.put(tids, joinOperator);
}
public void registerOnSlotEqSlotExpr(Expr expr) {
- globalState.onSlotEqSlotExpr.add(expr);
+ inferPredicateState.onSlotEqSlotExpr.add(expr);
}
public void registerOnSlotEqSlotDeDuplication(Pair<Expr, Expr> pair) {
- globalState.onSlotEqSlotDeDuplication.add(pair);
+ inferPredicateState.onSlotEqSlotDeDuplication.add(pair);
}
public void registerOnSlotToLiteralExpr(Expr expr) {
- globalState.onSlotToLiteralExpr.add(expr);
+ inferPredicateState.onSlotToLiteralExpr.add(expr);
}
- public void registerOnSlotToLiteralDeDuplication(Pair<Expr,Expr> pair) {
- globalState.onSlotToLiteralDeDuplication.add(pair);
+ public void registerOnSlotToLiteralDeDuplication(Pair<Expr, Expr> pair) {
+ inferPredicateState.onSlotToLiteralDeDuplication.add(pair);
}
public void registerInExpr(Expr expr) {
- globalState.onInExpr.add(expr);
+ inferPredicateState.onInExpr.add(expr);
}
public void registerInDeDuplication(Expr expr) {
- globalState.onInDeDuplication.add(expr);
+ inferPredicateState.onInDeDuplication.add(expr);
}
public void registerOnIsNullExpr(Expr expr) {
- globalState.onIsNullExpr.add(expr);
+ inferPredicateState.onIsNullExpr.add(expr);
}
public void registerOnIsNullDeDuplication(Expr expr) {
- globalState.onIsNullDeDuplication.add(expr);
+ inferPredicateState.onIsNullDeDuplication.add(expr);
}
public void registerGlobalSlotToLiteralDeDuplication(Pair<Expr, Expr>
pair) {
- globalState.globalSlotToLiteralDeDuplication.add(pair);
+ inferPredicateState.globalSlotToLiteralDeDuplication.add(pair);
}
public void registerGlobalInDeDuplication(Expr expr) {
- globalState.globalInDeDuplication.add(expr);
+ inferPredicateState.globalInDeDuplication.add(expr);
}
public void registerConjunct(Expr e, TupleId tupleId) throws
AnalysisException {
@@ -1330,11 +1353,11 @@ public class Analyzer {
* Return JoinOperator between two tables
*/
public JoinOperator getAnyTwoTablesJoinOp(Pair<TupleId, TupleId> tids) {
- return globalState.anyTwoTalesJoinOperator.get(tids);
+ return inferPredicateState.anyTwoTalesJoinOperator.get(tids);
}
public boolean isContainTupleIds(Pair<TupleId, TupleId> tids) {
- return globalState.anyTwoTalesJoinOperator.containsKey(tids);
+ return inferPredicateState.anyTwoTalesJoinOperator.containsKey(tids);
}
public boolean isWhereClauseConjunct(Expr e) {
@@ -1427,43 +1450,43 @@ public class Analyzer {
}
public List<Expr> getOnSlotEqSlotExpr() {
- return new ArrayList<>(globalState.onSlotEqSlotExpr);
+ return new ArrayList<>(inferPredicateState.onSlotEqSlotExpr);
}
- public Set<Pair<Expr,Expr>> getOnSlotEqSlotDeDuplication() {
- return Sets.newHashSet(globalState.onSlotEqSlotDeDuplication);
+ public Set<Pair<Expr, Expr>> getOnSlotEqSlotDeDuplication() {
+ return Sets.newHashSet(inferPredicateState.onSlotEqSlotDeDuplication);
}
public List<Expr> getOnSlotToLiteralExpr() {
- return new ArrayList<>(globalState.onSlotToLiteralExpr);
+ return new ArrayList<>(inferPredicateState.onSlotToLiteralExpr);
}
public Set<Pair<Expr, Expr>> getOnSlotToLiteralDeDuplication() {
- return Sets.newHashSet(globalState.onSlotToLiteralDeDuplication);
+ return
Sets.newHashSet(inferPredicateState.onSlotToLiteralDeDuplication);
}
public List<Expr> getInExpr() {
- return new ArrayList<>(globalState.onInExpr);
+ return new ArrayList<>(inferPredicateState.onInExpr);
}
public Set<Expr> getInDeDuplication() {
- return Sets.newHashSet(globalState.onInDeDuplication);
+ return Sets.newHashSet(inferPredicateState.onInDeDuplication);
}
public List<Expr> getOnIsNullExpr() {
- return new ArrayList<>(globalState.onIsNullExpr);
+ return new ArrayList<>(inferPredicateState.onIsNullExpr);
}
public Set<Expr> getOnIsNullDeDuplication() {
- return Sets.newHashSet(globalState.onIsNullDeDuplication);
+ return Sets.newHashSet(inferPredicateState.onIsNullDeDuplication);
}
public Set<Pair<Expr, Expr>> getGlobalSlotToLiteralDeDuplication() {
- return Sets.newHashSet(globalState.globalSlotToLiteralDeDuplication);
+ return
Sets.newHashSet(inferPredicateState.globalSlotToLiteralDeDuplication);
}
public Set<Expr> getGlobalInDeDuplication() {
- return Sets.newHashSet(globalState.globalInDeDuplication);
+ return Sets.newHashSet(inferPredicateState.globalInDeDuplication);
}
/**
diff --git a/regression-test/suites/correctness/test_infer_predicate.groovy
b/regression-test/suites/correctness/test_infer_predicate.groovy
new file mode 100644
index 0000000000..24ff31ac5b
--- /dev/null
+++ b/regression-test/suites/correctness/test_infer_predicate.groovy
@@ -0,0 +1,114 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+ // or more contributor license agreements. See the NOTICE file
+ // distributed with this work for additional information
+ // regarding copyright ownership. The ASF licenses this file
+ // to you under the Apache License, Version 2.0 (the
+ // "License"); you may not use this file except in compliance
+ // with the License. You may obtain a copy of the License at
+ //
+ // http://www.apache.org/licenses/LICENSE-2.0
+ //
+ // Unless required by applicable law or agreed to in writing,
+ // software distributed under the License is distributed on an
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ // KIND, either express or implied. See the License for the
+ // specific language governing permissions and limitations
+ // under the License.
+
+suite("test_infer_predicate") {
+ sql """ DROP TABLE IF EXISTS infer_predicate_t1 """
+ sql """ DROP TABLE IF EXISTS infer_predicate_t2 """
+ sql """ DROP VIEW IF EXISTS infer_predicate_v1 """
+ sql """ DROP TABLE IF EXISTS infer_predicate_t4 """
+ sql """ DROP TABLE IF EXISTS infer_predicate_t5 """
+ sql"""
+ CREATE TABLE infer_predicate_t2 (
+ a varchar(1) NULL COMMENT "",
+ x varchar(1) NULL COMMENT "",
+ c varchar(1) NULL COMMENT ""
+ ) ENGINE=OLAP
+ UNIQUE KEY(a)
+ DISTRIBUTED BY HASH(a) BUCKETS 8
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default:1",
+ "in_memory" = "false",
+ "storage_format" = "V2"
+ );
+ """
+
+
+ sql"""
+ CREATE TABLE infer_predicate_t5 (
+ x varchar(5) NULL COMMENT "",
+ d varchar(3) NULL COMMENT ""
+ ) ENGINE=OLAP
+ UNIQUE KEY(x)
+ DISTRIBUTED BY HASH(x) BUCKETS 8
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default:1",
+ "in_memory" = "false",
+ "storage_format" = "V2"
+ );
+ """
+
+ sql"""
+ CREATE TABLE infer_predicate_t4 (
+ a varchar(1) NULL COMMENT "",
+ e varchar(1) NULL COMMENT ""
+ ) ENGINE=OLAP
+ UNIQUE KEY(a)
+ DISTRIBUTED BY HASH(a) BUCKETS 8
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default:1",
+ "in_memory" = "false",
+ "storage_format" = "V2"
+ );
+ """
+
+
+ sql"""
+ CREATE TABLE infer_predicate_t1 (
+ k1 varchar(20) NULL COMMENT "",
+ dt date NULL COMMENT ""
+ ) ENGINE=OLAP
+ UNIQUE KEY(k1)
+ DISTRIBUTED BY HASH(k1) BUCKETS 6
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default:1",
+ "in_memory" = "false",
+ "storage_format" = "V2"
+ );
+ """
+
+ sql"""
+ CREATE VIEW infer_predicate_v1
+ AS
+ SELECT k1
+ FROM infer_predicate_t1
+ WHERE dt = (
+ SELECT max(dt)
+ FROM infer_predicate_t1
+ );
+ """
+
+ try {
+ sql """
+ select
+ f1.k1
+ from (
+ select
+ infer_predicate_t2.c as k1
+ from infer_predicate_t5
+ inner join infer_predicate_t2 on
infer_predicate_t2.x=infer_predicate_t5.x
+ inner join infer_predicate_t4 on
infer_predicate_t2.a=infer_predicate_t4.a and infer_predicate_t2.x in ('ZCR',
'ZDR')
+ ) f1
+ left join infer_predicate_v1 on f1.k1=infer_predicate_v1.k1;
+ """
+ } finally {
+ sql """ DROP TABLE IF EXISTS infer_predicate_t1 """
+ sql """ DROP TABLE IF EXISTS infer_predicate_t2 """
+ sql """ DROP VIEW IF EXISTS infer_predicate_v1 """
+ sql """ DROP TABLE IF EXISTS infer_predicate_t4 """
+ sql """ DROP TABLE IF EXISTS infer_predicate_t5 """
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
