This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 374c75938db [Fix](nereids) remove duplicate expr in grouping set
(#32834)
374c75938db is described below
commit 374c75938dba4539ccd08695b365acf75222397a
Author: feiniaofeiafei <[email protected]>
AuthorDate: Fri Mar 29 10:33:33 2024 +0800
[Fix](nereids) remove duplicate expr in grouping set (#32834)
cherry-pick #32290
---
.../nereids/rules/analysis/NormalizeRepeat.java | 11 +++++++
.../doris/nereids/trees/plans/algebra/Repeat.java | 3 --
.../nereids/trees/plans/logical/LogicalRepeat.java | 4 +++
.../remove_duplicate_expr_in_grouping_set.out | 19 +++++++++++
.../remove_duplicate_expr_in_grouping_set.groovy | 37 ++++++++++++++++++++++
5 files changed, 71 insertions(+), 3 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/NormalizeRepeat.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/NormalizeRepeat.java
index 3c8c2b36999..2ccdba9d5e9 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/NormalizeRepeat.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/NormalizeRepeat.java
@@ -78,12 +78,23 @@ public class NormalizeRepeat extends OneAnalysisRuleFactory
{
return RuleType.NORMALIZE_REPEAT.build(
logicalRepeat(any()).when(LogicalRepeat::canBindVirtualSlot).then(repeat -> {
checkRepeatLegality(repeat);
+ repeat = removeDuplicateColumns(repeat);
// add virtual slot, LogicalAggregate and LogicalProject for
normalize
return normalizeRepeat(repeat);
})
);
}
+ private LogicalRepeat<Plan> removeDuplicateColumns(LogicalRepeat<Plan>
repeat) {
+ List<List<Expression>> groupingSets = repeat.getGroupingSets();
+ ImmutableList.Builder<List<Expression>> builder =
ImmutableList.builder();
+ for (List<Expression> sets : groupingSets) {
+ List<Expression> newList =
ImmutableList.copyOf(ImmutableSet.copyOf(sets));
+ builder.add(newList);
+ }
+ return repeat.withGroupSets(builder.build());
+ }
+
private void checkRepeatLegality(LogicalRepeat<Plan> repeat) {
checkIfAggFuncSlotInGroupingSets(repeat);
checkGroupingSetsSize(repeat);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Repeat.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Repeat.java
index 29258b6bbab..e729f2a7cb3 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Repeat.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Repeat.java
@@ -175,9 +175,6 @@ public interface Repeat<CHILD_PLAN extends Plan> extends
Aggregate<CHILD_PLAN> {
if (index == null) {
throw new AnalysisException("Can not find grouping set
expression in output: " + expression);
}
- if (groupingSetIndex.contains(index)) {
- throw new AnalysisException("expression duplicate in
grouping set: " + expression);
- }
groupingSetIndex.add(index);
}
groupingSetsIndex.add(groupingSetIndex);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java
index 61fc8584d40..8fee9add02b 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java
@@ -154,6 +154,10 @@ public class LogicalRepeat<CHILD_TYPE extends Plan>
extends LogicalUnary<CHILD_T
children.get(0));
}
+ public LogicalRepeat<CHILD_TYPE> withGroupSets(List<List<Expression>>
groupingSets) {
+ return new LogicalRepeat<>(groupingSets, outputExpressions, child());
+ }
+
public LogicalRepeat<CHILD_TYPE>
withGroupSetsAndOutput(List<List<Expression>> groupingSets,
List<NamedExpression> outputExpressionList) {
return new LogicalRepeat<>(groupingSets, outputExpressionList,
child());
diff --git
a/regression-test/data/nereids_rules_p0/grouping_sets/remove_duplicate_expr_in_grouping_set.out
b/regression-test/data/nereids_rules_p0/grouping_sets/remove_duplicate_expr_in_grouping_set.out
new file mode 100644
index 00000000000..5956255d7bd
--- /dev/null
+++
b/regression-test/data/nereids_rules_p0/grouping_sets/remove_duplicate_expr_in_grouping_set.out
@@ -0,0 +1,19 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !test_col --
+\N 6
+1 20
+2 8
+3 20
+4 2
+5 26
+7 1
+
+-- !test_expr --
+\N 6
+2 20
+3 8
+4 20
+5 2
+6 26
+8 1
+
diff --git
a/regression-test/suites/nereids_rules_p0/grouping_sets/remove_duplicate_expr_in_grouping_set.groovy
b/regression-test/suites/nereids_rules_p0/grouping_sets/remove_duplicate_expr_in_grouping_set.groovy
new file mode 100644
index 00000000000..b9a10fe21d2
--- /dev/null
+++
b/regression-test/suites/nereids_rules_p0/grouping_sets/remove_duplicate_expr_in_grouping_set.groovy
@@ -0,0 +1,37 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+suite("remove_duplicate_expr_in_grouping_set") {
+ sql "SET enable_nereids_planner=true"
+ sql "SET enable_fallback_to_original_planner=false"
+ sql """
+ DROP TABLE IF EXISTS mal_test1
+ """
+
+ sql """
+ create table mal_test1(pk int, a int, b int) distributed by hash(pk)
buckets 10
+ properties('replication_num' = '1');
+ """
+
+ sql """
+ insert into mal_test1
values(2,1,3),(1,1,2),(3,5,6),(6,null,6),(4,5,6),(2,1,4),(2,3,5),(1,1,4)
+
,(3,5,6),(3,5,null),(6,7,1),(2,1,7),(2,4,2),(2,3,9),(1,3,6),(3,5,8),(3,2,8);
+ """
+ sql "sync"
+ qt_test_col "select a, sum(b) from mal_test1 group by grouping sets((a,a))
order by 1,2"
+ qt_test_expr "select a+1,sum(b) from mal_test1 group by grouping
sets((a+1,a+1)) order by 1,2"
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
