This is an automated email from the ASF dual-hosted git repository. dataroaring pushed a commit to branch branch-3.0 in repository https://gitbox.apache.org/repos/asf/doris.git
commit c23b9297ffd2067ae6c26f7ad6c65bcd98dae7a0 Author: zhangstar333 <[email protected]> AuthorDate: Mon May 27 14:07:24 2024 +0800 [improve](nereids) support pushdown count when scan project without any slot ref (#35162) select 66 from baseall_dup; could use pushAggOp=COUNT, so no need to scan real data --- .../org/apache/doris/nereids/rules/RuleType.java | 1 + .../rules/implementation/AggregateStrategies.java | 41 ++++++++++++++++++++++ .../eliminate_distinct_constant.out | 8 ++--- .../push_down_filter_other_condition.out | 2 +- .../explain/test_pushdown_explain.groovy | 8 +++++ 5 files changed, 55 insertions(+), 5 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java index fc625d56514..59746099781 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java @@ -432,6 +432,7 @@ public enum RuleType { STORAGE_LAYER_AGGREGATE_WITH_PROJECT(RuleTypeClass.IMPLEMENTATION), STORAGE_LAYER_AGGREGATE_WITHOUT_PROJECT_FOR_FILE_SCAN(RuleTypeClass.IMPLEMENTATION), STORAGE_LAYER_AGGREGATE_WITH_PROJECT_FOR_FILE_SCAN(RuleTypeClass.IMPLEMENTATION), + STORAGE_LAYER_WITH_PROJECT_NO_SLOT_REF(RuleTypeClass.IMPLEMENTATION), STORAGE_LAYER_AGGREGATE_MINMAX_ON_UNIQUE(RuleTypeClass.IMPLEMENTATION), STORAGE_LAYER_AGGREGATE_MINMAX_ON_UNIQUE_WITHOUT_PROJECT(RuleTypeClass.IMPLEMENTATION), COUNT_ON_INDEX(RuleTypeClass.IMPLEMENTATION), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java index e1095df7bab..937515ad8b5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java @@ -215,6 +215,16 @@ public class AggregateStrategies implements ImplementationRuleFactory { .when(agg -> agg.isNormalized() && enablePushDownNoGroupAgg()) .thenApply(ctx -> storageLayerAggregate(ctx.root, null, ctx.root.child(), ctx.cascadesContext)) ), + RuleType.STORAGE_LAYER_WITH_PROJECT_NO_SLOT_REF.build( + logicalProject( + logicalOlapScan() + ) + .thenApply(ctx -> { + LogicalProject<LogicalOlapScan> project = ctx.root; + LogicalOlapScan olapScan = project.child(); + return pushDownCountWithoutSlotRef(project, olapScan, ctx.cascadesContext); + }) + ), RuleType.STORAGE_LAYER_AGGREGATE_WITH_PROJECT.build( logicalAggregate( logicalProject( @@ -306,6 +316,37 @@ public class AggregateStrategies implements ImplementationRuleFactory { ); } + /* + * select 66 from baseall_dup; could use pushAggOp=COUNT to not scan real data. + */ + private LogicalProject<? extends Plan> pushDownCountWithoutSlotRef( + LogicalProject<? extends Plan> project, + LogicalOlapScan logicalScan, + CascadesContext cascadesContext) { + final LogicalProject<? extends Plan> canNotPush = project; + if (!enablePushDownNoGroupAgg()) { + return canNotPush; + } + if (logicalScan != null) { + KeysType keysType = logicalScan.getTable().getKeysType(); + if (keysType != KeysType.DUP_KEYS) { + return canNotPush; + } + } + for (Expression e : project.getProjects()) { + if (e.anyMatch(SlotReference.class::isInstance)) { + return canNotPush; + } + } + PhysicalOlapScan physicalOlapScan + = (PhysicalOlapScan) new LogicalOlapScanToPhysicalOlapScan() + .build() + .transform(logicalScan, cascadesContext) + .get(0); + return project.withChildren(ImmutableList.of(new PhysicalStorageLayerAggregate( + physicalOlapScan, PushDownAggOp.COUNT))); + } + private boolean enablePushDownMinMaxOnUnique() { ConnectContext connectContext = ConnectContext.get(); return connectContext != null && connectContext.getSessionVariable().isEnablePushDownMinMaxOnUnique(); diff --git a/regression-test/data/nereids_rules_p0/eliminate_distinct_constant/eliminate_distinct_constant.out b/regression-test/data/nereids_rules_p0/eliminate_distinct_constant/eliminate_distinct_constant.out index f5ed6baa505..dd35a0e8e31 100644 --- a/regression-test/data/nereids_rules_p0/eliminate_distinct_constant/eliminate_distinct_constant.out +++ b/regression-test/data/nereids_rules_p0/eliminate_distinct_constant/eliminate_distinct_constant.out @@ -4,28 +4,28 @@ PhysicalResultSink --PhysicalLimit[GLOBAL] ----PhysicalLimit[LOCAL] ------PhysicalProject ---------PhysicalOlapScan[t1] +--------PhysicalStorageLayerAggregate[t1] -- !basic_2 -- PhysicalResultSink --PhysicalLimit[GLOBAL] ----PhysicalLimit[LOCAL] ------PhysicalProject ---------PhysicalOlapScan[t2] +--------PhysicalStorageLayerAggregate[t2] -- !basic_3 -- PhysicalResultSink --PhysicalLimit[GLOBAL] ----PhysicalLimit[LOCAL] ------PhysicalProject ---------PhysicalOlapScan[t3] +--------PhysicalStorageLayerAggregate[t3] -- !basic_4 -- PhysicalResultSink --PhysicalLimit[GLOBAL] ----PhysicalLimit[LOCAL] ------PhysicalProject ---------PhysicalOlapScan[t4] +--------PhysicalStorageLayerAggregate[t4] -- !basic_1 -- 1 2 3 diff --git a/regression-test/data/nereids_rules_p0/filter_push_down/push_down_filter_other_condition.out b/regression-test/data/nereids_rules_p0/filter_push_down/push_down_filter_other_condition.out index 056949b11f5..65f95f83ec0 100644 --- a/regression-test/data/nereids_rules_p0/filter_push_down/push_down_filter_other_condition.out +++ b/regression-test/data/nereids_rules_p0/filter_push_down/push_down_filter_other_condition.out @@ -241,5 +241,5 @@ PhysicalResultSink ------PhysicalOlapScan[t1] ----PhysicalLimit[GLOBAL] ------PhysicalLimit[LOCAL] ---------PhysicalOlapScan[t2] +--------PhysicalStorageLayerAggregate[t2] diff --git a/regression-test/suites/nereids_p0/explain/test_pushdown_explain.groovy b/regression-test/suites/nereids_p0/explain/test_pushdown_explain.groovy index f7360963e55..1e691f2be9f 100644 --- a/regression-test/suites/nereids_p0/explain/test_pushdown_explain.groovy +++ b/regression-test/suites/nereids_p0/explain/test_pushdown_explain.groovy @@ -65,6 +65,14 @@ suite("test_pushdown_explain") { sql("select count(cast(lo_orderkey as bigint)) from test_lineorder;") contains "pushAggOp=COUNT" } + explain { + sql("select 66 from test_lineorder;") + contains "pushAggOp=COUNT" + } + explain { + sql("select lo_orderkey from test_lineorder;") + contains "pushAggOp=NONE" + } sql "DROP TABLE IF EXISTS table_unique0" sql """ --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
