This is an automated email from the ASF dual-hosted git repository.
panxiaolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new c8ad62a3cd [Enchancement](materialized-view) enchance materialized
view where clause match (#18179)
c8ad62a3cd is described below
commit c8ad62a3cda378c40ff4163f08de0c470ea73c32
Author: Pxl <[email protected]>
AuthorDate: Thu Mar 30 13:02:21 2023 +0800
[Enchancement](materialized-view) enchance materialized view where clause
match (#18179)
enchance materialized view where clause match
---
.../apache/doris/analysis/CompoundPredicate.java | 27 ++++++++++++++++++++++
.../main/java/org/apache/doris/analysis/Expr.java | 14 +++++++++++
.../doris/planner/MaterializedViewSelector.java | 4 ++--
.../org/apache/doris/planner/OlapScanNode.java | 11 +++++----
.../apache/doris/planner/SingleNodePlanner.java | 7 ++----
.../java/org/apache/doris/planner/PlannerTest.java | 2 +-
.../org/apache/doris/planner/QueryPlanTest.java | 16 ++++++-------
.../java/org/apache/doris/policy/PolicyTest.java | 11 ++++-----
regression-test/data/mv_p0/where/k123/k123.out | 4 ++++
.../suites/mv_p0/where/k123/k123.groovy | 6 +++++
10 files changed, 76 insertions(+), 26 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/CompoundPredicate.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/CompoundPredicate.java
index d711114db4..b98b982eb6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CompoundPredicate.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CompoundPredicate.java
@@ -277,4 +277,31 @@ public class CompoundPredicate extends Predicate {
public String toString() {
return toSqlImpl();
}
+
+ @Override
+ public boolean containsSubPredicate(Expr subExpr) throws AnalysisException
{
+ if (op.equals(Operator.AND)) {
+ for (Expr child : children) {
+ if (child.containsSubPredicate(subExpr)) {
+ return true;
+ }
+ }
+ }
+ return super.containsSubPredicate(subExpr);
+ }
+
+ @Override
+ public Expr replaceSubPredicate(Expr subExpr) throws AnalysisException {
+ if (op.equals(Operator.AND)) {
+ Expr lhs = children.get(0);
+ Expr rhs = children.get(1);
+ if (lhs.replaceSubPredicate(subExpr) == null) {
+ return rhs;
+ }
+ if (rhs.replaceSubPredicate(subExpr) == null) {
+ return lhs;
+ }
+ }
+ return this;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
index c39616fc6d..a2d03c50ae 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
@@ -2226,6 +2226,20 @@ public abstract class Expr extends TreeNode<Expr>
implements ParseNode, Cloneabl
return true;
}
+ public boolean containsSubPredicate(Expr subExpr) throws AnalysisException
{
+ if (toSqlWithoutTbl().equals(subExpr.toSqlWithoutTbl())) {
+ return true;
+ }
+ return false;
+ }
+
+ public Expr replaceSubPredicate(Expr subExpr) throws AnalysisException {
+ if (toSqlWithoutTbl().equals(subExpr.toSqlWithoutTbl())) {
+ return null;
+ }
+ return this;
+ }
+
protected Type[] getActualArgTypes(Type[] originType) {
return Arrays.stream(originType).map(
(Type type) -> {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/MaterializedViewSelector.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/MaterializedViewSelector.java
index 0996ff30b2..139abb0ba4 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/planner/MaterializedViewSelector.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/planner/MaterializedViewSelector.java
@@ -325,8 +325,8 @@ public class MaterializedViewSelector {
}
if (entry.getValue().getWhereClause() != null) {
- if (selectStmt.getOriginalWhereClause() == null ||
!entry.getValue().getWhereClause().toSqlWithoutTbl()
-
.equals(selectStmt.getOriginalWhereClause().toSqlWithoutTbl())) {
+ if (selectStmt.getOriginalWhereClause() == null ||
!selectStmt.getOriginalWhereClause()
+
.containsSubPredicate(entry.getValue().getWhereClause())) {
iterator.remove();
}
continue;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
index f07adf882c..cfc42fd8ff 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
@@ -333,8 +333,11 @@ public class OlapScanNode extends ScanNode {
return selectedIndexId;
}
- public void ignoreConjuncts() {
- vconjunct = null;
+ public void ignoreConjuncts(Expr whereExpr) throws AnalysisException {
+ if (whereExpr == null) {
+ return;
+ }
+ vconjunct = vconjunct.replaceSubPredicate(whereExpr);
}
/**
@@ -1103,8 +1106,8 @@ public class OlapScanNode extends ScanNode {
if (useTopnOpt) {
output.append(prefix).append("TOPN OPT\n");
}
- if (!conjuncts.isEmpty()) {
- output.append(prefix).append("PREDICATES:
").append(getExplainString(conjuncts)).append("\n");
+ if (vconjunct != null) {
+ output.append(prefix).append("PREDICATES:
").append(vconjunct.toSql()).append("\n");
}
if (!runtimeFilters.isEmpty()) {
output.append(prefix).append("runtime filters: ");
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
index 1883cc9094..9b7e716b9d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
@@ -1363,12 +1363,9 @@ public class SingleNodePlanner {
} else {
try {
// mv index have where clause, so where expr on scan
node is unused.
- Expr whereExpr = olapScanNode.getOlapTable()
+
olapScanNode.ignoreConjuncts(olapScanNode.getOlapTable()
.getIndexMetaByIndexId(bestIndexInfo.getBestIndexId())
- .getWhereClause();
- if (whereExpr != null) {
- olapScanNode.ignoreConjuncts();
- }
+ .getWhereClause());
// if the new selected index id is different from the
old one, scan node will be
// updated.
diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/PlannerTest.java
b/fe/fe-core/src/test/java/org/apache/doris/planner/PlannerTest.java
index 730aab5674..97812cb0d0 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/planner/PlannerTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/planner/PlannerTest.java
@@ -455,7 +455,7 @@ public class PlannerTest extends TestWithFeService {
stmtExecutor.execute();
Planner planner = stmtExecutor.planner();
String plan = planner.getExplainString(new ExplainOptions(false,
false));
- Assertions.assertTrue(plan.contains("PREDICATES: `k1` = 1, `k2` =
1\n"));
+ Assertions.assertTrue(plan.contains("PREDICATES: `k1` = 1 AND `k2` =
1\n"));
}
@Test
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java
b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java
index 206456e188..a057325d10 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java
@@ -1654,7 +1654,7 @@ public class QueryPlanTest extends TestWithFeService {
//default format
String sql = "select * from test1 where from_unixtime(query_time) >
'2021-03-02 10:01:28'";
String explainString = getSQLPlanOrErrorMsg("EXPLAIN " + sql);
- Assert.assertTrue(explainString.contains("PREDICATES: `query_time` <=
253402271999, `query_time` > 1614650488"));
+ Assert.assertTrue(explainString.contains("PREDICATES: `query_time` <=
253402271999 AND `query_time` > 1614650488"));
}
@Test
@@ -1829,7 +1829,7 @@ public class QueryPlanTest extends TestWithFeService {
// (false or expr1) and (false or expr2) ==> expr1 and expr2
String sql9 = "select * from test.test1 where (-2=2 or query_time=2)
and (-2=2 or stmt_id=2);";
String explainString9 = getSQLPlanOrErrorMsg("EXPLAIN " + sql9);
- Assert.assertTrue(explainString9.contains("PREDICATES: `query_time` =
2, `stmt_id` = 2"));
+ Assert.assertTrue(explainString9.contains("PREDICATES: `query_time` =
2 AND `stmt_id` = 2"));
// false or (expr and true) ==> expr
String sql10 = "select * from test.test1 where (2=-2) OR (query_time=0
AND 1=1);";
@@ -1865,7 +1865,7 @@ public class QueryPlanTest extends TestWithFeService {
Assert.assertTrue(explainStr.contains("PREDICATES: `date` >=
'2021-10-07',"
+ " `date` <= '2021-10-11'"));
} else {
- Assert.assertTrue(explainStr.contains("PREDICATES: `date` >=
'2021-10-07 00:00:00',"
+ Assert.assertTrue(explainStr.contains("PREDICATES: `date` >=
'2021-10-07 00:00:00' AND"
+ " `date` <= '2021-10-11 00:00:00'"));
}
}
@@ -2210,15 +2210,15 @@ public class QueryPlanTest extends TestWithFeService {
sql = "SELECT * from test1 where (query_time = 1 or query_time = 2)
and query_time in (3, 4)";
explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext,
"EXPLAIN " + sql);
- Assert.assertTrue(explainString.contains("PREDICATES: `query_time` IN
(1, 2), `query_time` IN (3, 4)\n"));
+ Assert.assertTrue(explainString.contains("PREDICATES: `query_time` IN
(1, 2) AND `query_time` IN (3, 4)\n"));
sql = "SELECT * from test1 where (query_time = 1 or query_time = 2 or
scan_bytes = 2) and scan_bytes in (2, 3)";
explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext,
"EXPLAIN " + sql);
- Assert.assertTrue(explainString.contains("PREDICATES: `query_time` IN
(1, 2) OR `scan_bytes` = 2, `scan_bytes` IN (2, 3)\n"));
+ Assert.assertTrue(explainString.contains("PREDICATES: `query_time` IN
(1, 2) OR `scan_bytes` = 2 AND `scan_bytes` IN (2, 3)\n"));
sql = "SELECT * from test1 where (query_time = 1 or query_time = 2)
and (scan_bytes = 2 or scan_bytes = 3)";
explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext,
"EXPLAIN " + sql);
- Assert.assertTrue(explainString.contains("PREDICATES: `query_time` IN
(1, 2), `scan_bytes` IN (2, 3)\n"));
+ Assert.assertTrue(explainString.contains("PREDICATES: `query_time` IN
(1, 2) AND `scan_bytes` IN (2, 3)\n"));
sql = "SELECT * from test1 where query_time = 1 or query_time = 2 or
query_time = 3 or query_time = 1";
explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext,
"EXPLAIN " + sql);
@@ -2236,7 +2236,7 @@ public class QueryPlanTest extends TestWithFeService {
sql = "SELECT * from test1 where (query_time = 1 or query_time = 2)
and query_time in (3, 4)";
explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext,
"EXPLAIN " + sql);
- Assert.assertTrue(explainString.contains("PREDICATES: `query_time` IN
(1, 2), `query_time` IN (3, 4)\n"));
+ Assert.assertTrue(explainString.contains("PREDICATES: `query_time` IN
(1, 2) AND `query_time` IN (3, 4)\n"));
//test we can handle `!=` and `not in`
sql = "select * from test1 where (query_time = 1 or query_time = 2 or
query_time!= 3 or query_time not in (5, 6))";
@@ -2269,7 +2269,7 @@ public class QueryPlanTest extends TestWithFeService {
sql = "select * from test1 where (stmt_id=1 and state='a') or
(stmt_id=2 and state='b')";
explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext,
"EXPLAIN " + sql);
Assert.assertTrue(explainString.contains(
- "PREDICATES: `state` IN ('a', 'b'), `stmt_id` IN (1, 2),"
+ "PREDICATES: `state` IN ('a', 'b') AND `stmt_id` IN (1, 2) AND"
+ " `stmt_id` = 1 AND `state` = 'a' OR `stmt_id` = 2
AND `state` = 'b'\n"
));
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/policy/PolicyTest.java
b/fe/fe-core/src/test/java/org/apache/doris/policy/PolicyTest.java
index 71084fb429..b53bbe46ef 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/policy/PolicyTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/policy/PolicyTest.java
@@ -172,7 +172,7 @@ public class PolicyTest extends TestWithFeService {
createPolicy("CREATE ROW POLICY test_row_policy4 ON test.table1 AS
PERMISSIVE TO test_policy USING (k2 = 1)");
String queryStr = "EXPLAIN select * from test.table1";
String explainString = getSQLPlanOrErrorMsg(queryStr);
- Assertions.assertTrue(explainString.contains("`k1` = 1, `k2` = 1, `k2`
= 2 OR `k2` = 1"));
+ Assertions.assertTrue(explainString.contains("`k1` = 1 AND `k2` = 1
AND `k2` = 2 OR `k2` = 1"));
dropPolicy("DROP ROW POLICY test_row_policy1 ON test.table1");
dropPolicy("DROP ROW POLICY test_row_policy2 ON test.table1");
dropPolicy("DROP ROW POLICY test_row_policy3 ON test.table1");
@@ -184,14 +184,13 @@ public class PolicyTest extends TestWithFeService {
createPolicy("CREATE ROW POLICY test_row_policy1 ON test.table1 AS
RESTRICTIVE TO test_policy USING (k1 = 1)");
createPolicy("CREATE ROW POLICY test_row_policy2 ON test.table1 AS
RESTRICTIVE TO test_policy USING (k2 = 1)");
String joinSql = "select * from table1 join table2 on
table1.k1=table2.k1";
- System.out.println(getSQLPlanOrErrorMsg(joinSql));
-
Assertions.assertTrue(getSQLPlanOrErrorMsg(joinSql).contains("PREDICATES: `k1`
= 1, `k2` = 1"));
+
Assertions.assertTrue(getSQLPlanOrErrorMsg(joinSql).contains("PREDICATES: `k1`
= 1 AND `k2` = 1"));
String unionSql = "select * from table1 union select * from table2";
-
Assertions.assertTrue(getSQLPlanOrErrorMsg(unionSql).contains("PREDICATES: `k1`
= 1, `k2` = 1"));
+
Assertions.assertTrue(getSQLPlanOrErrorMsg(unionSql).contains("PREDICATES: `k1`
= 1 AND `k2` = 1"));
String subQuerySql = "select * from table2 where k1 in (select k1 from
table1)";
-
Assertions.assertTrue(getSQLPlanOrErrorMsg(subQuerySql).contains("PREDICATES:
`k1` = 1, `k2` = 1"));
+
Assertions.assertTrue(getSQLPlanOrErrorMsg(subQuerySql).contains("PREDICATES:
`k1` = 1 AND `k2` = 1"));
String aliasSql = "select * from table1 t1 join table2 t2 on
t1.k1=t2.k1";
-
Assertions.assertTrue(getSQLPlanOrErrorMsg(aliasSql).contains("PREDICATES:
`t1`.`k1` = 1, `t1`.`k2` = 1"));
+
Assertions.assertTrue(getSQLPlanOrErrorMsg(aliasSql).contains("PREDICATES:
`t1`.`k1` = 1 AND `t1`.`k2` = 1"));
dropPolicy("DROP ROW POLICY test_row_policy1 ON test.table1");
dropPolicy("DROP ROW POLICY test_row_policy2 ON test.table1");
}
diff --git a/regression-test/data/mv_p0/where/k123/k123.out
b/regression-test/data/mv_p0/where/k123/k123.out
index fb9659afad..55bd6c538c 100644
--- a/regression-test/data/mv_p0/where/k123/k123.out
+++ b/regression-test/data/mv_p0/where/k123/k123.out
@@ -35,3 +35,7 @@
1 2
1 2
+-- !select_mv --
+2 4
+2 4
+
diff --git a/regression-test/suites/mv_p0/where/k123/k123.groovy
b/regression-test/suites/mv_p0/where/k123/k123.groovy
index d80c59b1e2..09201fe31c 100644
--- a/regression-test/suites/mv_p0/where/k123/k123.groovy
+++ b/regression-test/suites/mv_p0/where/k123/k123.groovy
@@ -80,4 +80,10 @@ suite ("k123p") {
contains "(d_table)"
}
qt_select_mv "select k1,k2+k3 from d_table where k4 = 'a' order by k1;"
+
+ explain {
+ sql("""select k1,k2+k3 from d_table where k1 = 2 and k4 = "b";""")
+ contains "(k123p4w)"
+ }
+ qt_select_mv """select k1,k2+k3 from d_table where k1 = 2 and k4 = "b"
order by k1;"""
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
