This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 2e651bbc9a [fix](nereids) fix some planner bugs (#21533)
2e651bbc9a is described below
commit 2e651bbc9a88de1ca99060a15281711660137d37
Author: starocean999 <[email protected]>
AuthorDate: Thu Jul 6 16:13:37 2023 +0800
[fix](nereids) fix some planner bugs (#21533)
1. allow cast boolean as date like type in nereids, the result is null
2. PruneOlapScanTablet rule can prune tablet even if a mv index is selected.
3. constant conjunct should not be pushed through agg node in old planner
---
.../java/org/apache/doris/analysis/Analyzer.java | 6 +-
.../nereids/rules/expression/check/CheckCast.java | 12 ++--
.../nereids/rules/rewrite/PruneOlapScanTablet.java | 12 ++--
.../doris/planner/HashDistributionPruner.java | 12 +++-
.../org/apache/doris/planner/OlapScanNode.java | 3 +-
.../doris/planner/HashDistributionPrunerTest.java | 2 +-
.../org/apache/doris/planner/OlapScanNodeTest.java | 6 +-
.../data/correctness_p0/test_constant_having.out | 5 ++
.../correctness_p0/test_constant_having.groovy | 74 ++++++++++++++++++++++
.../suites/nereids_p0/datatype/test_cast.groovy | 20 ++++++
.../suites/nereids_p0/test_prune_tablet_mv.groovy | 45 +++++++++++++
11 files changed, 177 insertions(+), 20 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java
index 0faa84d2e4..33aa53412b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java
@@ -40,6 +40,7 @@ import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.IdGenerator;
import org.apache.doris.common.Pair;
import org.apache.doris.common.util.TimeUtils;
+import org.apache.doris.planner.AggregationNode;
import org.apache.doris.planner.PlanNode;
import org.apache.doris.planner.RuntimeFilter;
import org.apache.doris.qe.ConnectContext;
@@ -2416,10 +2417,11 @@ public class Analyzer {
* Wrapper around getUnassignedConjuncts(List<TupleId> tupleIds).
*/
public List<Expr> getUnassignedConjuncts(PlanNode node) {
- // constant conjuncts should be push down to all leaf node.
+ // constant conjuncts should be push down to all leaf node except agg
node.
+ // (see getPredicatesBoundedByGroupbysSourceExpr method)
// so we need remove constant conjuncts when expr is not a leaf node.
List<Expr> unassigned = getUnassignedConjuncts(node.getTblRefIds());
- if (!node.getChildren().isEmpty()) {
+ if (!node.getChildren().isEmpty() && !(node instanceof
AggregationNode)) {
unassigned = unassigned.stream()
.filter(e -> !e.isConstant()).collect(Collectors.toList());
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/check/CheckCast.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/check/CheckCast.java
index cc64baeb34..361b5f38f5 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/check/CheckCast.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/check/CheckCast.java
@@ -59,11 +59,10 @@ public class CheckCast extends
AbstractExpressionRewriteRule {
/**
* forbid this original and target type
- * 1. boolean to date, datev2, datetime, datetimev2
- * 2. original type is object type
- * 3. target type is object type
- * 4. original type is same with target type
- * 5. target type is null type
+ * 1. original type is object type
+ * 2. target type is object type
+ * 3. original type is same with target type
+ * 4. target type is null type
*/
private boolean checkPrimitiveType(DataType originalType, DataType
targetType) {
if (!originalType.isPrimitive() || !targetType.isPrimitive()) {
@@ -72,9 +71,6 @@ public class CheckCast extends AbstractExpressionRewriteRule {
if (originalType.equals(targetType)) {
return false;
}
- if (originalType.isBooleanType() && targetType.isDateLikeType()) {
- return false;
- }
if (originalType.isNullType()) {
return true;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTablet.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTablet.java
index 1cf718ff48..1b26f6b3f8 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTablet.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTablet.java
@@ -58,8 +58,11 @@ public class PruneOlapScanTablet extends
OneRewriteRuleFactory {
for (Long id : olapScan.getSelectedPartitionIds()) {
Partition partition = table.getPartition(id);
MaterializedIndex index =
partition.getIndex(olapScan.getSelectedIndexId());
-
selectedTabletIdsBuilder.addAll(getSelectedTabletIds(filter.getConjuncts(),
- index, partition.getDistributionInfo()));
+ selectedTabletIdsBuilder
+
.addAll(getSelectedTabletIds(filter.getConjuncts(), index,
+ olapScan.getSelectedIndexId() ==
olapScan.getTable()
+ .getBaseIndexId(),
+ partition.getDistributionInfo()));
}
List<Long> selectedTabletIds =
selectedTabletIdsBuilder.build();
if (new HashSet(selectedTabletIds).equals(new
HashSet(olapScan.getSelectedTabletIds()))) {
@@ -70,7 +73,7 @@ public class PruneOlapScanTablet extends
OneRewriteRuleFactory {
}
private Collection<Long> getSelectedTabletIds(Set<Expression> expressions,
- MaterializedIndex index, DistributionInfo info) {
+ MaterializedIndex index, boolean isBaseIndexSelected,
DistributionInfo info) {
if (info.getType() != DistributionInfoType.HASH) {
return index.getTabletIdsInOrder();
}
@@ -81,7 +84,8 @@ public class PruneOlapScanTablet extends
OneRewriteRuleFactory {
return new HashDistributionPruner(index.getTabletIdsInOrder(),
hashInfo.getDistributionColumns(),
filterMap,
- hashInfo.getBucketNum()
+ hashInfo.getBucketNum(),
+ isBaseIndexSelected
).prune();
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/HashDistributionPruner.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/HashDistributionPruner.java
index 7314fe0564..2a60e4029a 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/planner/HashDistributionPruner.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/planner/HashDistributionPruner.java
@@ -17,6 +17,7 @@
package org.apache.doris.planner;
+import org.apache.doris.analysis.CreateMaterializedViewStmt;
import org.apache.doris.analysis.InPredicate;
import org.apache.doris.analysis.LiteralExpr;
import org.apache.doris.analysis.SlotRef;
@@ -58,12 +59,15 @@ public class HashDistributionPruner implements
DistributionPruner {
private Map<String, PartitionColumnFilter> distributionColumnFilters;
private int hashMod;
+ private boolean isBaseIndexSelected;
+
public HashDistributionPruner(List<Long> bucketsList, List<Column> columns,
- Map<String, PartitionColumnFilter> filters, int
hashMod) {
+ Map<String, PartitionColumnFilter> filters, int
hashMod, boolean isBaseIndexSelected) {
this.bucketsList = bucketsList;
this.distributionColumns = columns;
this.distributionColumnFilters = filters;
this.hashMod = hashMod;
+ this.isBaseIndexSelected = isBaseIndexSelected;
}
// columnId: which column to compute
@@ -75,7 +79,11 @@ public class HashDistributionPruner implements
DistributionPruner {
return Lists.newArrayList(bucketsList.get((int) ((hashValue &
0xffffffff) % hashMod)));
}
Column keyColumn = distributionColumns.get(columnId);
- PartitionColumnFilter filter =
distributionColumnFilters.get(keyColumn.getName());
+ String columnName = isBaseIndexSelected ? keyColumn.getName()
+ :
org.apache.doris.nereids.rules.rewrite.mv.AbstractSelectMaterializedIndexRule
+ .normalizeName(
+
CreateMaterializedViewStmt.mvColumnBuilder(keyColumn.getName()));
+ PartitionColumnFilter filter =
distributionColumnFilters.get(columnName);
if (null == filter) {
// no filter in this column, no partition Key
// return all subPartition
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
index b520e058e1..1fcac274ec 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
@@ -692,7 +692,8 @@ public class OlapScanNode extends ScanNode {
distributionPruner = new
HashDistributionPruner(table.getTabletIdsInOrder(),
info.getDistributionColumns(),
columnFilters,
- info.getBucketNum());
+ info.getBucketNum(),
+ getSelectedIndexId() == olapTable.getBaseIndexId());
return distributionPruner.prune();
}
case RANDOM: {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/planner/HashDistributionPrunerTest.java
b/fe/fe-core/src/test/java/org/apache/doris/planner/HashDistributionPrunerTest.java
index 7f3df304e3..2da47314d6 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/planner/HashDistributionPrunerTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/planner/HashDistributionPrunerTest.java
@@ -91,7 +91,7 @@ public class HashDistributionPrunerTest {
filters.put("channel", channelFilter);
filters.put("shop_type", shopTypeFilter);
- HashDistributionPruner pruner = new HashDistributionPruner(tabletIds,
columns, filters, tabletIds.size());
+ HashDistributionPruner pruner = new HashDistributionPruner(tabletIds,
columns, filters, tabletIds.size(), true);
Collection<Long> results = pruner.prune();
// 20 = 1 * 5 * 2 * 2 * 1 (element num of each filter)
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/planner/OlapScanNodeTest.java
b/fe/fe-core/src/test/java/org/apache/doris/planner/OlapScanNodeTest.java
index 22e6cd74f2..d8d7854355 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/planner/OlapScanNodeTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/planner/OlapScanNodeTest.java
@@ -69,7 +69,8 @@ public class OlapScanNodeTest {
partitions,
columns,
filterMap,
- 3);
+ 3,
+ true);
Collection<Long> ids = partitionPruner.prune();
Assert.assertEquals(ids.size(), 1);
@@ -112,7 +113,8 @@ public class OlapScanNodeTest {
partitions,
columns,
filterMap,
- 3);
+ 3,
+ true);
Collection<Long> ids = partitionPruner.prune();
Assert.assertEquals(ids.size(), 3);
diff --git a/regression-test/data/correctness_p0/test_constant_having.out
b/regression-test/data/correctness_p0/test_constant_having.out
new file mode 100644
index 0000000000..b0e2754c2a
--- /dev/null
+++ b/regression-test/data/correctness_p0/test_constant_having.out
@@ -0,0 +1,5 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !sql1 --
+
+-- !sql2 --
+
diff --git a/regression-test/suites/correctness_p0/test_constant_having.groovy
b/regression-test/suites/correctness_p0/test_constant_having.groovy
new file mode 100644
index 0000000000..a21b4c0f74
--- /dev/null
+++ b/regression-test/suites/correctness_p0/test_constant_having.groovy
@@ -0,0 +1,74 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_constant_having") {
+ sql """
+ drop table if exists test_constant_having_t0;
+ """
+
+ sql """
+ CREATE TABLE test_constant_having_t0(c0 VARCHAR(19) NOT NULL)
DISTRIBUTED BY HASH (c0) PROPERTIES ("replication_num" = "1");
+ """
+
+ sql """
+ INSERT INTO test_constant_having_t0 (c0) VALUES (1),(2);
+ """
+
+ sql """ set enable_nereids_planner=false;"""
+ qt_sql1 """
+ SELECT
+ CAST(DATE '1970-12-16' AS FLOAT),
+ 9998895.0,
+ 0.946221655
+ FROM
+ test_constant_having_t0
+ GROUP BY
+ test_constant_having_t0.c0
+ HAVING
+ (
+ NOT (
+ CAST(false AS DATETIME) NOT IN (CAST(-994966193 AS
DATETIME))
+ )
+ )
+ ORDER BY
+ 1;
+ """
+
+ sql """ set enable_nereids_planner=true;"""
+ qt_sql2 """
+ SELECT
+ CAST(DATE '1970-12-16' AS FLOAT),
+ 9998895.0,
+ 0.946221655
+ FROM
+ test_constant_having_t0
+ GROUP BY
+ test_constant_having_t0.c0
+ HAVING
+ (
+ NOT (
+ CAST(false AS DATETIME) NOT IN (CAST(-994966193 AS
DATETIME))
+ )
+ )
+ ORDER BY
+ 1;
+ """
+
+ sql """
+ drop table if exists test_constant_having_t0;
+ """
+}
diff --git a/regression-test/suites/nereids_p0/datatype/test_cast.groovy
b/regression-test/suites/nereids_p0/datatype/test_cast.groovy
index f9b6ee6e35..29691297ef 100644
--- a/regression-test/suites/nereids_p0/datatype/test_cast.groovy
+++ b/regression-test/suites/nereids_p0/datatype/test_cast.groovy
@@ -74,4 +74,24 @@ suite("test_cast") {
sql "select * from ${tbl} where case when k0 = 101 then 'true' else 1
end"
result([[101]])
}
+
+ test {
+ sql "select cast(false as datetime);"
+ result([[null]])
+ }
+
+ test {
+ sql "select cast(true as datetime);"
+ result([[null]])
+ }
+
+ test {
+ sql "select cast(false as date);"
+ result([[null]])
+ }
+
+ test {
+ sql "select cast(true as date);"
+ result([[null]])
+ }
}
\ No newline at end of file
diff --git a/regression-test/suites/nereids_p0/test_prune_tablet_mv.groovy
b/regression-test/suites/nereids_p0/test_prune_tablet_mv.groovy
new file mode 100644
index 0000000000..2a5c1349ac
--- /dev/null
+++ b/regression-test/suites/nereids_p0/test_prune_tablet_mv.groovy
@@ -0,0 +1,45 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_prune_tablet_mv") {
+ sql "SET enable_nereids_planner=true"
+ sql "SET enable_fallback_to_original_planner=false"
+
+ sql "DROP TABLE IF EXISTS test_prune_tablet_t2"
+ sql """
+ create table test_prune_tablet_t2(id int, c1 boolean) distributed
by hash(id) BUCKETS 16 properties('replication_num'='1');
+ """
+
+ createMV( """
+ CREATE
+ MATERIALIZED VIEW mv_t2 AS
+ SELECT c1,
+ id
+ FROM test_prune_tablet_t2
+ ORDER BY c1,
+ id;
+ """)
+ sql "insert into test_prune_tablet_t2
values(1,0),(2,0),(3,0),(4,0),(5,0),(6,0),(7,0);"
+
+ explain {
+ sql("select * from test_prune_tablet_t2 where id = 3;")
+ contains "mv_t2"
+ contains "tablets=1/16"
+ }
+
+ sql "DROP TABLE IF EXISTS test_prune_tablet_t2"
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
