This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new b0387acf969 [fix](nereids) runtime filter prune skip filter with
invisible column (#28010) (#28521)
b0387acf969 is described below
commit b0387acf969566982dbaadd1ad093652c3460949
Author: minghong <[email protected]>
AuthorDate: Sun Dec 17 20:54:52 2023 +0800
[fix](nereids) runtime filter prune skip filter with invisible column
(#28010) (#28521)
if a conjunct only contains invisible column, this conjunct should not be
used in runtime filter pruner
(cherry picked from commit a0fee4c96ecd8ef229c2ba3dc407dc9873fe8b23)
---
.../processor/post/RuntimeFilterPruner.java | 19 +++++-
.../runtime_filter/rf_prune.groovy | 77 ++++++++++++++++++++++
2 files changed, 95 insertions(+), 1 deletion(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruner.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruner.java
index c9cc43d0c29..2bf4a907fa5 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruner.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruner.java
@@ -22,6 +22,7 @@ import org.apache.doris.nereids.trees.expressions.EqualTo;
import org.apache.doris.nereids.trees.expressions.ExprId;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.plans.AbstractPlan;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalAssertNumRows;
@@ -100,6 +101,16 @@ public class RuntimeFilterPruner extends PlanPostProcessor
{
return join;
}
+ private boolean isVisibleColumn(Slot slot) {
+ if (slot instanceof SlotReference) {
+ SlotReference slotReference = (SlotReference) slot;
+ if (slotReference.getColumn().isPresent()) {
+ return slotReference.getColumn().get().isVisible();
+ }
+ }
+ return true;
+ }
+
@Override
public PhysicalProject visitPhysicalProject(PhysicalProject<? extends
Plan> project, CascadesContext context) {
project.child().accept(this, context);
@@ -112,7 +123,13 @@ public class RuntimeFilterPruner extends PlanPostProcessor
{
@Override
public PhysicalFilter visitPhysicalFilter(PhysicalFilter<? extends Plan>
filter, CascadesContext context) {
filter.child().accept(this, context);
- context.getRuntimeFilterContext().addEffectiveSrcNode(filter);
+ boolean visibleFilter = filter.getExpressions().stream()
+ .flatMap(expression -> expression.getInputSlots().stream())
+ .anyMatch(slot -> isVisibleColumn(slot));
+ if (visibleFilter) {
+ // skip filters like: __DORIS_DELETE_SIGN__ = 0
+ context.getRuntimeFilterContext().addEffectiveSrcNode(filter);
+ }
return filter;
}
diff --git
a/regression-test/suites/nereids_rules_p0/runtime_filter/rf_prune.groovy
b/regression-test/suites/nereids_rules_p0/runtime_filter/rf_prune.groovy
new file mode 100644
index 00000000000..a4cee458a18
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/runtime_filter/rf_prune.groovy
@@ -0,0 +1,77 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("rf_prune") {
+ String db = context.config.getDbNameByFile(context.file)
+ sql "use ${db}"
+ sql "SET enable_nereids_planner=true"
+ sql "SET enable_fallback_to_original_planner=false"
+ sql "set enable_runtime_filter_prune=true;"
+ sql "set disable_join_reorder=true"
+ sql "drop table if exists A"
+ sql """
+ CREATE TABLE A (
+ `id` INT NULL,
+ `msg` TEXT NULL
+ ) ENGINE=OLAP
+ unique KEY(`id`)
+ COMMENT 'OLAP'
+ DISTRIBUTED BY HASH(`id`) BUCKETS 10
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "min_load_replica_num" = "-1",
+ "is_being_synced" = "false",
+ "storage_format" = "V2",
+ "light_schema_change" = "true",
+ "disable_auto_compaction" = "false",
+ "enable_single_replica_compaction" = "false",
+ "group_commit_interval_ms" = "10000"
+ );
+ """
+ sql "drop table if exists B"
+ sql """
+ CREATE TABLE B (
+ `id` INT NULL,
+ `msg` TEXT NULL
+ ) ENGINE=OLAP
+ unique KEY(`id`)
+ COMMENT 'OLAP'
+ DISTRIBUTED BY HASH(`id`) BUCKETS 10
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "min_load_replica_num" = "-1",
+ "is_being_synced" = "false",
+ "storage_format" = "V2",
+ "light_schema_change" = "true",
+ "disable_auto_compaction" = "false",
+ "enable_single_replica_compaction" = "false",
+ "group_commit_interval_ms" = "10000"
+ );
+ """
+
+ explain{
+ sql " shape plan select * from A join B on A.id=B.id;"
+ notContains "build RFs"
+ }
+
+ explain{
+ sql " shape plan select * from A join B on A.id=B.id where B.msg='xyz';"
+ contains "PhysicalOlapScan[A] apply RFs: RF0"
+ }
+
+
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
