This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new 6acf3dad10b branch-4.1: [fix](nereids) Fix DISTINCT aggregate planning
#65368 #65664 #68386 (#70198)
6acf3dad10b is described below
commit 6acf3dad10bd59d4f645ae998734a22199b90013
Author: yujun <[email protected]>
AuthorDate: Tue Nov 10 13:52:16 2026 +0800
branch-4.1: [fix](nereids) Fix DISTINCT aggregate planning #65368 #65664
#68386 (#70198)
cherry-pick: #65368
cherry-pick: #65664
cherry-pick: #68386
---------
Co-authored-by: feiniaofeiafei <[email protected]>
Co-authored-by: Mryange <[email protected]>
---
.../nereids/rules/analysis/NormalizeAggregate.java | 5 +-
.../nereids/rules/analysis/NormalizeRepeat.java | 3 +
.../rules/expression/rules/MedianConvert.java | 2 +-
.../rules/implementation/SplitAggMultiPhase.java | 2 +-
.../SplitAggMultiPhaseWithoutGbyKey.java | 2 +-
.../nereids/rules/rewrite/CheckMultiDistinct.java | 50 +--
.../rules/rewrite/DistinctAggStrategySelector.java | 16 +-
.../rules/rewrite/DistinctAggregateRewriter.java | 67 +++-
.../rules/rewrite/SplitMultiDistinctStrategy.java | 6 +-
.../expressions/functions/agg/CollectList.java | 15 +
.../functions/agg/ExponentialMovingAverage.java | 6 +
.../expressions/functions/agg/GroupBitmapXor.java | 11 +-
.../expressions/functions/agg/GroupConcat.java | 8 +
.../trees/expressions/functions/agg/Histogram.java | 5 +
.../trees/expressions/functions/agg/MapAgg.java | 2 +-
.../trees/expressions/functions/agg/MapAggV2.java | 2 +-
.../functions/agg/MultiDistinctGroupConcat.java | 8 +
.../expressions/functions/agg/Percentile.java | 5 +
.../functions/agg/PercentileApprox.java | 5 +
.../functions/agg/PercentileApproxWeighted.java | 5 +
.../expressions/functions/agg/PercentileArray.java | 5 +
.../functions/agg/PercentileReservoir.java | 5 +
.../trees/expressions/functions/agg/TopN.java | 5 +
.../trees/expressions/functions/agg/TopNArray.java | 5 +
.../expressions/functions/agg/TopNWeighted.java | 5 +
.../apache/doris/nereids/util/AggregateUtils.java | 31 +-
.../rewrite/DistinctAggregateRewriterTest.java | 17 +-
.../agg_function/agg_distinct_function.out | 393 +++++++++++++++++++++
.../agg_strategy/distinct_agg_rewriter.out | 9 +
.../distinct_agg_strategy_selector.out | 15 +
.../test_convert_median_to_percentile.out | 3 +
.../agg_function/agg_distinct_function.groovy | 212 +++++++++++
.../nereids_p0/aggregate/agg_group_concat.groovy | 9 +
.../agg_strategy/distinct_agg_rewriter.groovy | 23 +-
.../distinct_agg_strategy_selector.groovy | 43 +++
.../test_convert_median_to_percentile.groovy | 13 +
.../query_p0/group_concat/test_group_concat.groovy | 2 +-
37 files changed, 925 insertions(+), 95 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/NormalizeAggregate.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/NormalizeAggregate.java
index 2cc864192fb..9b46201c7b1 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/NormalizeAggregate.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/NormalizeAggregate.java
@@ -40,7 +40,6 @@ import
org.apache.doris.nereids.trees.expressions.WindowExpression;
import
org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction;
import org.apache.doris.nereids.trees.expressions.functions.agg.AnyValue;
import org.apache.doris.nereids.trees.expressions.functions.generator.Unnest;
-import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
@@ -181,7 +180,7 @@ public class NormalizeAggregate implements
RewriteRuleFactory, NormalizeToSlot {
for (Expression arg : aggFunc.children()) {
// should not push down literal under aggregate
// e.g. group_concat(distinct xxx, ','), the ',' literal
show stay in aggregate
- if (arg instanceof Literal) {
+ if (arg.isConstant()) {
continue;
}
if (arg.containsType(SubqueryExpr.class,
WindowExpression.class, Unnest.class,
@@ -199,7 +198,7 @@ public class NormalizeAggregate implements
RewriteRuleFactory, NormalizeToSlot {
for (Expression arg : aggFunc.children()) {
// should not push down literal under aggregate
// e.g. group_concat(distinct xxx, ','), the ',' literal
show stay in aggregate
- if (arg instanceof Literal) {
+ if (arg.isConstant()) {
continue;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/NormalizeRepeat.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/NormalizeRepeat.java
index 8016bf6f7e7..bbabf96f7f1 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/NormalizeRepeat.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/NormalizeRepeat.java
@@ -241,6 +241,9 @@ public class NormalizeRepeat extends OneAnalysisRuleFactory
{
ImmutableSet.Builder<Expression> argumentsOfAggregateFunctionBuilder =
ImmutableSet.builder();
for (AggregateFunction function : aggregateFunctions) {
for (Expression arg : function.getArguments()) {
+ if (arg.isConstant()) {
+ continue;
+ }
if (arg instanceof OrderExpression) {
argumentsOfAggregateFunctionBuilder.add(arg.child(0));
} else {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/MedianConvert.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/MedianConvert.java
index e6e295ef98c..d8144bf2508 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/MedianConvert.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/MedianConvert.java
@@ -39,7 +39,7 @@ public class MedianConvert implements
ExpressionPatternRuleFactory {
public List<ExpressionPatternMatcher<? extends Expression>> buildRules() {
return ImmutableList.of(
matchesType(Median.class).then(median ->
- new Percentile(median.child(0), DoubleLiteral.of(0.5))
+ new Percentile(median.isDistinct(), median.child(0),
DoubleLiteral.of(0.5))
).toRule(ExpressionRuleType.MEDIAN_CONVERT)
);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/SplitAggMultiPhase.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/SplitAggMultiPhase.java
index 9a5b9ad412a..390c3af6e04 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/SplitAggMultiPhase.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/SplitAggMultiPhase.java
@@ -76,7 +76,7 @@ public class SplitAggMultiPhase extends SplitAggBaseRule
implements Implementati
return ImmutableList.of(
logicalAggregate()
.when(agg -> !agg.getGroupByExpressions().isEmpty())
- .when(agg -> agg.getDistinctArguments().size() == 1 ||
agg.distinctFuncNum() == 1)
+ .when(agg ->
AggregateUtils.distinctArgumentGroupCountUpToTwo(agg) == 1)
.thenApplyMulti(this::rewrite)
.toRule(RuleType.SPLIT_AGG_MULTI_PHASE)
);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/SplitAggMultiPhaseWithoutGbyKey.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/SplitAggMultiPhaseWithoutGbyKey.java
index 294777a900a..be7666544ee 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/SplitAggMultiPhaseWithoutGbyKey.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/SplitAggMultiPhaseWithoutGbyKey.java
@@ -75,7 +75,7 @@ public class SplitAggMultiPhaseWithoutGbyKey extends
SplitAggBaseRule implements
return ImmutableList.of(
logicalAggregate()
.when(agg -> agg.getGroupByExpressions().isEmpty())
- .when(agg -> agg.getDistinctArguments().size() == 1 ||
agg.distinctFuncNum() == 1)
+ .when(agg ->
AggregateUtils.distinctArgumentGroupCountUpToTwo(agg) == 1)
.thenApplyMulti(ctx -> rewrite(ctx.root,
ctx.cascadesContext))
.toRule(RuleType.SPLIT_AGG_MULTI_PHASE_WITHOUT_GBY_KEY)
);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CheckMultiDistinct.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CheckMultiDistinct.java
index d9c953444ba..0f23dcd1a7d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CheckMultiDistinct.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CheckMultiDistinct.java
@@ -20,20 +20,12 @@ package org.apache.doris.nereids.rules.rewrite;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
-import org.apache.doris.nereids.trees.expressions.OrderExpression;
-import
org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction;
-import
org.apache.doris.nereids.trees.expressions.functions.agg.SupportMultiDistinct;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
+import org.apache.doris.nereids.util.AggregateUtils;
/**
- * If there are multiple distinct aggregate functions that cannot
- * be transformed into multi_distinct, an error is reported.
- * The following functions can be transformed into multi_distinct:
- * - count -> MULTI_DISTINCT_COUNT
- * - sum -> MULTI_DISTINCT_SUM
- * - avg -> MULTI_DISTINCT_AVG
- * - group_concat -> MULTI_DISTINCT_GROUP_CONCAT
+ * Check that previous aggregate rewrite rules have already handled multiple
distinct argument groups.
*/
public class CheckMultiDistinct extends OneRewriteRuleFactory {
@Override
@@ -42,43 +34,9 @@ public class CheckMultiDistinct extends
OneRewriteRuleFactory {
}
private LogicalAggregate checkDistinct(LogicalAggregate<? extends Plan>
aggregate) {
- if (aggregate.getDistinctArguments().size() > 1) {
-
- for (AggregateFunction func : aggregate.getAggregateFunctions()) {
- if (func.isDistinct() && !(func instanceof
SupportMultiDistinct)) {
- throw new AnalysisException(func.toString() + " can't
support multi distinct.");
- }
- }
- }
-
- boolean distinctMultiColumns = false;
- for (AggregateFunction func : aggregate.getAggregateFunctions()) {
- if (!func.isDistinct()) {
- continue;
- }
- if (func.arity() <= 1) {
- continue;
- }
- for (int i = 1; i < func.arity(); i++) {
- if (!func.child(i).getInputSlots().isEmpty() &&
!(func.child(i) instanceof OrderExpression)) {
- // think about group_concat(distinct col_1, ',')
- distinctMultiColumns = true;
- break;
- }
- }
- if (distinctMultiColumns) {
- break;
- }
- }
-
- long distinctFunctionNum = 0;
- for (AggregateFunction aggregateFunction :
aggregate.getAggregateFunctions()) {
- distinctFunctionNum += aggregateFunction.isDistinct() ? 1 : 0;
- }
-
- if (distinctMultiColumns && distinctFunctionNum > 1) {
+ if (AggregateUtils.distinctArgumentGroupCountUpToTwo(aggregate) > 1) {
throw new AnalysisException(
- "The query contains multi count distinct or sum distinct,
each can't have multi columns");
+ "Multiple DISTINCT aggregate functions with different
argument lists are not supported.");
}
return aggregate;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DistinctAggStrategySelector.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DistinctAggStrategySelector.java
index de83915c843..1ff8efc0341 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DistinctAggStrategySelector.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DistinctAggStrategySelector.java
@@ -103,11 +103,12 @@ public class DistinctAggStrategySelector extends
DefaultPlanRewriter<DistinctSel
// not process：
// count(distinct a,b);
// count(distinct a), sum(distinct a);
+ // count(distinct a,b), count(distinct b,a);
// count(distinct a)
// process:
// count(distinct a,b), count(distinct a,c)
// count(distinct a), sum(distinct b)
- if (agg.distinctFuncNum() < 2 || agg.getDistinctArguments().size() <
2) {
+ if (AggregateUtils.distinctArgumentGroupCountUpToTwo(agg) <= 1) {
return agg;
}
if (shouldUseMultiDistinct(agg)) {
@@ -118,19 +119,20 @@ public class DistinctAggStrategySelector extends
DefaultPlanRewriter<DistinctSel
}
private boolean shouldUseMultiDistinct(LogicalAggregate<? extends Plan>
agg) {
- boolean mustUseCte =
AggregateUtils.containsCountDistinctMultiExpr(agg);
+ boolean mustUseCte =
AggregateUtils.containsNotSupportMultiDistinctFunction(agg);
boolean mustUseMulti = agg.getSourceRepeat().isPresent();
if (mustUseCte && mustUseMulti) {
throw new AnalysisException(
- "Unsupported query: GROUPING SETS/ROLLUP/CUBE cannot be
used with a combination of "
- + "multi-column COUNT(DISTINCT) and other COUNT(DISTINCT)
expressions.\n\n"
+ "Unsupported query: GROUPING SETS/ROLLUP/CUBE cannot be
used with multiple DISTINCT "
+ + "argument groups when any DISTINCT aggregate does not
support multi-distinct execution.\n\n"
+ "Unsupported scenarios:\n"
+ + "• ARRAY_AGG(DISTINCT a) with ARRAY_AGG(DISTINCT b) +
GROUPING\n"
+ "• COUNT(DISTINCT a, b) with COUNT(DISTINCT a) +
GROUPING\n"
+ "• COUNT(DISTINCT a, b) with COUNT(DISTINCT a, c) +
GROUPING\n\n"
+ "Supported scenarios:\n"
- + "• Single COUNT(DISTINCT a, b) + GROUPING\n"
- + "• Multiple COUNT(DISTINCT single_column) + "
- + "GROUPING (e.g., COUNT(DISTINCT a), COUNT(DISTINCT b))");
+ + "• A single DISTINCT argument group with GROUPING\n"
+ + "• Multiple DISTINCT argument groups when all DISTINCT
aggregates support "
+ + "multi-distinct execution (e.g., COUNT(DISTINCT a),
COUNT(DISTINCT b))");
}
if (mustUseCte) {
return false;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DistinctAggregateRewriter.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DistinctAggregateRewriter.java
index 832641add2d..b6eb9470615 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DistinctAggregateRewriter.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DistinctAggregateRewriter.java
@@ -21,6 +21,7 @@ import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.DistributionInfo;
import org.apache.doris.catalog.HashDistributionInfo;
import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.rules.rewrite.StatsDerive.DeriveContext;
@@ -83,6 +84,13 @@ public class DistinctAggregateRewriter implements
RewriteRuleFactory {
private static final Set<Class<? extends AggregateFunction>>
supportSplitOtherFunctions = ImmutableSet.of(
Sum.class, Min.class, Max.class, Count.class, Sum0.class,
AnyValue.class);
+ enum Strategy {
+ SPLIT_IN_REWRITE, MULTI_STRATEGY, SPLIT_IN_CASCADES
+ }
+
+ private static final String errorString = "Unsupported query:
GROUP_CONCAT(DISTINCT ... ORDER BY ...)"
+ + " cannot be used together with a multi-argument COUNT(DISTINCT
...).";
+
@Override
public List<Rule> buildRules() {
return ImmutableList.of(
@@ -92,25 +100,41 @@ public class DistinctAggregateRewriter implements
RewriteRuleFactory {
.thenApply(ctx -> rewrite(ctx.root,
ctx.connectContext))
.toRule(RuleType.DISTINCT_AGGREGATE_SPLIT),
logicalAggregate()
- .when(agg -> agg.getGroupByExpressions().isEmpty()
- && agg.mustUseMultiDistinctAgg() &&
!AggregateUtils.containsCountDistinctMultiExpr(agg))
- .then(this::convertToMultiDistinct)
+ .when(agg -> agg.getGroupByExpressions().isEmpty() &&
agg.mustUseMultiDistinctAgg())
+ .then(agg -> {
+ // count(distinct a,b) cannot use multi_distinct
+ if
(AggregateUtils.containsCountDistinctMultiExpr(agg)) {
+ throw new AnalysisException(errorString);
+ }
+ return convertToMultiDistinct(agg);
+ })
.toRule(RuleType.PROCESS_SCALAR_AGG_MUST_USE_MULTI_DISTINCT)
);
}
@VisibleForTesting
- boolean shouldUseMultiDistinct(LogicalAggregate<? extends Plan> aggregate)
{
+ Strategy chooseStrategy(LogicalAggregate<? extends Plan> aggregate) {
// count(distinct a,b) cannot use multi_distinct
- if (AggregateUtils.containsCountDistinctMultiExpr(aggregate)) {
- return false;
- }
- if (aggregate.mustUseMultiDistinctAgg()) {
- return true;
+ boolean mustSplit =
AggregateUtils.containsCountDistinctMultiExpr(aggregate);
+ boolean mustUseMulti = aggregate.mustUseMultiDistinctAgg();
+ if (mustSplit && mustUseMulti) {
+ throw new AnalysisException(errorString);
}
ConnectContext ctx = ConnectContext.get();
+ if (mustSplit) {
+ if (ctx.getSessionVariable().aggPhase == 3 ||
ctx.getSessionVariable().aggPhase == 4) {
+ return Strategy.SPLIT_IN_CASCADES;
+ }
+ return Strategy.SPLIT_IN_REWRITE;
+ }
+ if (mustUseMulti) {
+ return Strategy.MULTI_STRATEGY;
+ }
if (ctx.getSessionVariable().aggPhase == 1 ||
ctx.getSessionVariable().aggPhase == 2) {
- return true;
+ return Strategy.MULTI_STRATEGY;
+ }
+ if (ctx.getSessionVariable().aggPhase == 3 ||
ctx.getSessionVariable().aggPhase == 4) {
+ return Strategy.SPLIT_IN_CASCADES;
}
if (aggregate.getStats() == null || aggregate.child().getStats() ==
null) {
StatsDerive derive = new StatsDerive(false);
@@ -120,12 +144,12 @@ public class DistinctAggregateRewriter implements
RewriteRuleFactory {
Statistics aggChildStats = aggregate.child().getStats();
Set<Expression> dstArgs = aggregate.getDistinctArguments();
if (isDistinctKeySatisfyDistribution(aggregate)) {
- return false;
+ return Strategy.SPLIT_IN_REWRITE;
}
// has unknown statistics, split to bottom and top agg
if
(AggregateUtils.hasUnknownStatistics(aggregate.getGroupByExpressions(),
aggChildStats)
|| AggregateUtils.hasUnknownStatistics(dstArgs,
aggChildStats)) {
- return true;
+ return Strategy.MULTI_STRATEGY;
}
double gbyNdv = aggStats.getRowCount();
@@ -138,8 +162,12 @@ public class DistinctAggregateRewriter implements
RewriteRuleFactory {
double inputRows = aggChildStats.getRowCount();
// group by key ndv is low, distinct key ndv is high, multi_distinct
is better
// otherwise split to bottom and top agg
- return gbyNdv < inputRows * AggregateUtils.LOW_CARDINALITY_THRESHOLD
- && dstNdv > inputRows *
AggregateUtils.HIGH_CARDINALITY_THRESHOLD;
+ if (gbyNdv < inputRows * AggregateUtils.LOW_CARDINALITY_THRESHOLD
+ && dstNdv > inputRows *
AggregateUtils.HIGH_CARDINALITY_THRESHOLD) {
+ return Strategy.MULTI_STRATEGY;
+ } else {
+ return Strategy.SPLIT_IN_REWRITE;
+ }
}
private boolean isDistinctKeySatisfyDistribution(LogicalAggregate<?
extends Plan> aggregate) {
@@ -249,14 +277,15 @@ public class DistinctAggregateRewriter implements
RewriteRuleFactory {
if (aggregate.distinctFuncNum() == 0) {
return null;
}
- if (ctx.getSessionVariable().aggPhase == 3 ||
ctx.getSessionVariable().aggPhase == 4) {
- return null;
- }
- if (shouldUseMultiDistinct(aggregate)) {
+ Strategy strategy = chooseStrategy(aggregate);
+ if (strategy == Strategy.MULTI_STRATEGY) {
return convertToMultiDistinct(aggregate);
- } else {
+ } else if (strategy == Strategy.SPLIT_IN_REWRITE) {
return splitDistinctAgg(aggregate);
+ } else if (strategy == Strategy.SPLIT_IN_CASCADES) {
+ return null;
}
+ return null;
}
private Plan convertToMultiDistinct(LogicalAggregate<? extends Plan>
aggregate) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SplitMultiDistinctStrategy.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SplitMultiDistinctStrategy.java
index c9585d269aa..f05be0a1d11 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SplitMultiDistinctStrategy.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SplitMultiDistinctStrategy.java
@@ -43,6 +43,7 @@ import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
@@ -166,10 +167,11 @@ public class SplitMultiDistinctStrategy {
List<List<Alias>> aliases, List<Alias> otherAggFuncs) {
Map<Set<Expression>, List<Alias>> distinctArgToAliasList = new
LinkedHashMap<>();
for (NamedExpression namedExpression : agg.getOutputExpressions()) {
- if (!(namedExpression instanceof Alias) ||
!(namedExpression.child(0) instanceof AggregateFunction)) {
+ if (!(namedExpression instanceof Alias) ||
!namedExpression.containsType(AggregateFunction.class)) {
continue;
}
- AggregateFunction aggFunc = (AggregateFunction)
namedExpression.child(0);
+ Optional<Expression> op = namedExpression.collectFirst(e -> e
instanceof AggregateFunction);
+ AggregateFunction aggFunc = (AggregateFunction) op.get();
if (aggFunc.isDistinct()) {
distinctArgToAliasList.computeIfAbsent(ImmutableSet.copyOf(aggFunc.getDistinctArguments()),
k -> new ArrayList<>()).add((Alias) namedExpression);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/CollectList.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/CollectList.java
index 6866e23a22e..6c447a22b6d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/CollectList.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/CollectList.java
@@ -18,6 +18,7 @@
package org.apache.doris.nereids.trees.expressions.functions.agg;
import org.apache.doris.catalog.FunctionSignature;
+import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.trees.expressions.Expression;
import
org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.literal.ArrayLiteral;
@@ -102,4 +103,18 @@ public class CollectList extends
NotNullableAggregateFunction
public Expression resultForEmptyInput() {
return new ArrayLiteral(new ArrayList<>(), this.getDataType());
}
+
+ @Override
+ public List<Expression> getDistinctArguments() {
+ return distinct ? ImmutableList.of(getArgument(0)) :
ImmutableList.of();
+ }
+
+ @Override
+ public void checkLegalityBeforeTypeCoercion() {
+ if (arity() == 2 && !getArgument(1).isConstant()) {
+ throw new AnalysisException(
+ "collect_list requires second parameter must be a
constant: "
+ + this.toSql());
+ }
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/ExponentialMovingAverage.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/ExponentialMovingAverage.java
index caeb3b82dc7..7ceda4cf912 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/ExponentialMovingAverage.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/ExponentialMovingAverage.java
@@ -119,4 +119,10 @@ public class ExponentialMovingAverage extends
NullableAggregateFunction
public List<FunctionSignature> getSignatures() {
return SIGNATURES;
}
+
+ @Override
+ public List<Expression> getDistinctArguments() {
+ return distinct ? ImmutableList.of(getArgument(1), getArgument(2)) :
ImmutableList.of();
+ }
+
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/GroupBitmapXor.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/GroupBitmapXor.java
index 202a469fdd4..f276f935048 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/GroupBitmapXor.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/GroupBitmapXor.java
@@ -18,6 +18,7 @@
package org.apache.doris.nereids.trees.expressions.functions.agg;
import org.apache.doris.catalog.FunctionSignature;
+import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.trees.expressions.Expression;
import
org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression;
@@ -55,8 +56,7 @@ public class GroupBitmapXor extends NullableAggregateFunction
}
private GroupBitmapXor(boolean distinct, boolean alwaysNullable,
Expression arg0) {
- /* distinct is meaningless to bitmap */
- super("group_bitmap_xor", false, alwaysNullable, arg0);
+ super("group_bitmap_xor", distinct, alwaysNullable, arg0);
}
/** constructor for withChildren and reuse signature */
@@ -92,4 +92,11 @@ public class GroupBitmapXor extends NullableAggregateFunction
public List<FunctionSignature> getSignatures() {
return SIGNATURES;
}
+
+ @Override
+ public void checkLegalityBeforeTypeCoercion() {
+ if (distinct) {
+ throw new AnalysisException("group_bitmap_xor does not support
DISTINCT");
+ }
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/GroupConcat.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/GroupConcat.java
index 8003d687380..1d1205aea8c 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/GroupConcat.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/GroupConcat.java
@@ -201,4 +201,12 @@ public class GroupConcat extends NullableAggregateFunction
}
return firstOrderExrIndex;
}
+
+ @Override
+ public void checkLegalityBeforeTypeCoercion() {
+ if (nonOrderArguments == 2 && !getArgument(1).isConstant()) {
+ throw new AnalysisException(
+ "group_concat requires separator must be a constant : " +
this.toSql());
+ }
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Histogram.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Histogram.java
index 26e02959243..01486ff2bd6 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Histogram.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Histogram.java
@@ -113,4 +113,9 @@ public class Histogram extends NotNullableAggregateFunction
public Expression resultForEmptyInput() {
return new VarcharLiteral("{\"num_buckets\":0,\"buckets\":[]}");
}
+
+ @Override
+ public List<Expression> getDistinctArguments() {
+ return distinct ? ImmutableList.of(getArgument(0)) :
ImmutableList.of();
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/MapAgg.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/MapAgg.java
index c611ced6fb4..9e57a17c560 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/MapAgg.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/MapAgg.java
@@ -54,7 +54,7 @@ public class MapAgg extends NotNullableAggregateFunction
/**
* constructor with 2 arguments.
*/
- private MapAgg(boolean distinct, Expression arg0, Expression arg1) {
+ public MapAgg(boolean distinct, Expression arg0, Expression arg1) {
super("map_agg_v1", distinct, arg0, arg1);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/MapAggV2.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/MapAggV2.java
index de099933eea..22093016601 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/MapAggV2.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/MapAggV2.java
@@ -54,7 +54,7 @@ public class MapAggV2 extends NotNullableAggregateFunction
/**
* constructor with 2 arguments.
*/
- private MapAggV2(boolean distinct, Expression arg0, Expression arg1) {
+ public MapAggV2(boolean distinct, Expression arg0, Expression arg1) {
super("map_agg_v2", distinct, arg0, arg1);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/MultiDistinctGroupConcat.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/MultiDistinctGroupConcat.java
index 5fc78899a72..8f1f6debe80 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/MultiDistinctGroupConcat.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/MultiDistinctGroupConcat.java
@@ -140,4 +140,12 @@ public class MultiDistinctGroupConcat extends
NullableAggregateFunction
}
return firstOrderExrIndex;
}
+
+ @Override
+ public void checkLegalityBeforeTypeCoercion() {
+ if (nonOrderArguments == 2 && !getArgument(1).isConstant()) {
+ throw new AnalysisException(
+ "multi_distinct_group_concat requires separator must be a
constant : " + this.toSql());
+ }
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Percentile.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Percentile.java
index 41d85424f8e..16a2821a541 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Percentile.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Percentile.java
@@ -107,4 +107,9 @@ public class Percentile extends NullableAggregateFunction
public List<FunctionSignature> getSignatures() {
return SIGNATURES;
}
+
+ @Override
+ public List<Expression> getDistinctArguments() {
+ return distinct ? ImmutableList.of(getArgument(0)) :
ImmutableList.of();
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/PercentileApprox.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/PercentileApprox.java
index d889f588df6..3e0229a1234 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/PercentileApprox.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/PercentileApprox.java
@@ -121,4 +121,9 @@ public class PercentileApprox extends
NullableAggregateFunction
public List<FunctionSignature> getSignatures() {
return SIGNATURES;
}
+
+ @Override
+ public List<Expression> getDistinctArguments() {
+ return distinct ? ImmutableList.of(getArgument(0)) :
ImmutableList.of();
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/PercentileApproxWeighted.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/PercentileApproxWeighted.java
index 86e3f40c781..2eae91d865b 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/PercentileApproxWeighted.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/PercentileApproxWeighted.java
@@ -125,4 +125,9 @@ public class PercentileApproxWeighted extends
NullableAggregateFunction
public List<FunctionSignature> getSignatures() {
return SIGNATURES;
}
+
+ @Override
+ public List<Expression> getDistinctArguments() {
+ return distinct ? ImmutableList.of(getArgument(0), getArgument(1)) :
ImmutableList.of();
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/PercentileArray.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/PercentileArray.java
index 560d1003383..89e8dded9a5 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/PercentileArray.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/PercentileArray.java
@@ -111,4 +111,9 @@ public class PercentileArray extends
NotNullableAggregateFunction
public Expression resultForEmptyInput() {
return new ArrayLiteral(new ArrayList<>(), this.getDataType());
}
+
+ @Override
+ public List<Expression> getDistinctArguments() {
+ return distinct ? ImmutableList.of(getArgument(0)) :
ImmutableList.of();
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/PercentileReservoir.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/PercentileReservoir.java
index 7c6f7bb6652..5e91e794064 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/PercentileReservoir.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/PercentileReservoir.java
@@ -107,4 +107,9 @@ public class PercentileReservoir extends
NullableAggregateFunction
public List<FunctionSignature> getSignatures() {
return SIGNATURES;
}
+
+ @Override
+ public List<Expression> getDistinctArguments() {
+ return distinct ? ImmutableList.of(getArgument(0)) :
ImmutableList.of();
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/TopN.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/TopN.java
index b223b130cc0..f03bbedf343 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/TopN.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/TopN.java
@@ -134,4 +134,9 @@ public class TopN extends NullableAggregateFunction
public List<FunctionSignature> getSignatures() {
return SIGNATURES;
}
+
+ @Override
+ public List<Expression> getDistinctArguments() {
+ return distinct ? ImmutableList.of(getArgument(0)) :
ImmutableList.of();
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/TopNArray.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/TopNArray.java
index bc370be9619..840f520b876 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/TopNArray.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/TopNArray.java
@@ -137,4 +137,9 @@ public class TopNArray extends NullableAggregateFunction
public List<FunctionSignature> getSignatures() {
return SIGNATURES;
}
+
+ @Override
+ public List<Expression> getDistinctArguments() {
+ return distinct ? ImmutableList.of(getArgument(0)) :
ImmutableList.of();
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/TopNWeighted.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/TopNWeighted.java
index 9297e7f1ae0..9d3152685a4 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/TopNWeighted.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/TopNWeighted.java
@@ -219,4 +219,9 @@ public class TopNWeighted extends NullableAggregateFunction
+ this.toSql());
}
}
+
+ @Override
+ public List<Expression> getDistinctArguments() {
+ return distinct ? ImmutableList.of(getArgument(0), getArgument(1)) :
ImmutableList.of();
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/AggregateUtils.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/AggregateUtils.java
index 067bd734467..6e5e0068408 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/AggregateUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/AggregateUtils.java
@@ -29,6 +29,7 @@ import
org.apache.doris.nereids.trees.expressions.functions.agg.SupportMultiDist
import org.apache.doris.nereids.trees.expressions.functions.scalar.If;
import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.algebra.Aggregate;
import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.statistics.ColumnStatistic;
@@ -134,6 +135,34 @@ public class AggregateUtils {
expr instanceof Count && ((Count) expr).isDistinct() &&
expr.arity() > 1);
}
+ /** e.g. Aggregation with avg(distinct a)(not support multiDistinct) or
count(distinct a,b) will return true*/
+ public static boolean
containsNotSupportMultiDistinctFunction(LogicalAggregate<? extends Plan>
aggregate) {
+ return ExpressionUtils.deapAnyMatch(aggregate.getOutputExpressions(),
expr -> {
+ if (expr instanceof AggregateFunction && ((AggregateFunction)
expr).isDistinct()) {
+ return !(expr instanceof SupportMultiDistinct)
+ || expr instanceof Count && ((Count)
expr).isDistinct() && expr.arity() > 1;
+ }
+ return false;
+ });
+ }
+
+ /** count agg function distinct group, up to 2*/
+ public static int distinctArgumentGroupCountUpToTwo(Aggregate<? extends
Plan> aggregate) {
+ Set<Expression> distinctArgumentGroup = null;
+ for (AggregateFunction aggregateFunction :
aggregate.getAggregateFunctions()) {
+ if (!aggregateFunction.isDistinct()) {
+ continue;
+ }
+ Set<Expression> currentGroup =
ImmutableSet.copyOf(aggregateFunction.getDistinctArguments());
+ if (distinctArgumentGroup == null) {
+ distinctArgumentGroup = currentGroup;
+ } else if (!distinctArgumentGroup.equals(currentGroup)) {
+ return 2;
+ }
+ }
+ return distinctArgumentGroup == null ? 0 : 1;
+ }
+
/**getAllKeySet*/
public static Set<NamedExpression> getAllKeySet(LogicalAggregate<? extends
Plan> aggregate) {
Set<NamedExpression> distinctArguments =
getDistinctNamedExpr(aggregate);
@@ -156,7 +185,7 @@ public class AggregateUtils {
public static Set<NamedExpression> getDistinctNamedExpr(LogicalAggregate<?
extends Plan> aggregate) {
return aggregate.getAggregateFunctions().stream()
.filter(AggregateFunction::isDistinct)
- .flatMap(aggFunc -> aggFunc.getArguments().stream())
+ .flatMap(aggFunc -> aggFunc.getDistinctArguments().stream())
.filter(NamedExpression.class::isInstance)
.map(NamedExpression.class::cast)
.collect(ImmutableSet.toImmutableSet());
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/DistinctAggregateRewriterTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/DistinctAggregateRewriterTest.java
index 7b9e957b742..dac2395c17f 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/DistinctAggregateRewriterTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/DistinctAggregateRewriterTest.java
@@ -20,6 +20,7 @@ package org.apache.doris.nereids.rules.rewrite;
import org.apache.doris.catalog.DistributionInfo;
import org.apache.doris.catalog.HashDistributionInfo;
import
org.apache.doris.nereids.rules.analysis.LogicalSubQueryAliasToLogicalProject;
+import
org.apache.doris.nereids.rules.rewrite.DistinctAggregateRewriter.Strategy;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import
org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction;
import org.apache.doris.nereids.trees.expressions.functions.agg.Count;
@@ -94,8 +95,8 @@ public class DistinctAggregateRewriterTest extends
TestWithFeService implements
private void applyMock() {
new MockUp<DistinctAggregateRewriter>() {
@Mock
- boolean shouldUseMultiDistinct(LogicalAggregate<? extends Plan>
aggregate) {
- return false;
+ Strategy chooseStrategy(LogicalAggregate<? extends Plan>
aggregate) {
+ return Strategy.SPLIT_IN_REWRITE;
}
};
}
@@ -249,7 +250,7 @@ public class DistinctAggregateRewriterTest extends
TestWithFeService implements
}
@Test
- void testShouldUseMultiDistinctWithoutStatsSatisfyDistribution() throws
Exception {
+ void testChooseStrategyWithoutStatsSatisfyDistribution() throws Exception {
DistinctAggregateRewriter rewriter =
DistinctAggregateRewriter.INSTANCE;
LogicalAggregate<? extends Plan> aggregate = getLogicalAggregate(
"select bb, count(distinct aa) from "
@@ -265,11 +266,11 @@ public class DistinctAggregateRewriterTest extends
TestWithFeService implements
((AbstractPlan) child).setStatistics(new Statistics(10000, colStats));
aggregate.setStatistics(new Statistics(100, ImmutableMap.of()));
- Assertions.assertFalse(rewriter.shouldUseMultiDistinct(aggregate));
+ Assertions.assertEquals(Strategy.SPLIT_IN_REWRITE,
rewriter.chooseStrategy(aggregate));
}
@Test
- void testShouldUseMultiDistinctWithStatsSelected() throws Exception {
+ void testChooseStrategyWithStatsSelected() throws Exception {
DistinctAggregateRewriter rewriter = new DistinctAggregateRewriter();
LogicalAggregate<? extends Plan> aggregate = getLogicalAggregate(
"select b, count(distinct a) from test.distinct_agg_split_t
group by b"
@@ -283,11 +284,11 @@ public class DistinctAggregateRewriterTest extends
TestWithFeService implements
((AbstractPlan) child).setStatistics(new Statistics(100000, colStats));
aggregate.setStatistics(new Statistics(240, ImmutableMap.of()));
- Assertions.assertFalse(rewriter.shouldUseMultiDistinct(aggregate));
+ Assertions.assertEquals(Strategy.SPLIT_IN_REWRITE,
rewriter.chooseStrategy(aggregate));
}
@Test
- void testShouldUseMultiDistinctWithPartitionTable() {
+ void testChooseStrategyWithPartitionTable() {
DistinctAggregateRewriter rewriter =
DistinctAggregateRewriter.INSTANCE;
LogicalAggregate<? extends Plan> aggregate = getLogicalAggregate(
"select count(distinct record_id) from sales_records group by
sale_date;"
@@ -301,7 +302,7 @@ public class DistinctAggregateRewriterTest extends
TestWithFeService implements
((AbstractPlan) child).setStatistics(new Statistics(10000, colStats));
aggregate.setStatistics(new Statistics(100, ImmutableMap.of()));
- Assertions.assertTrue(rewriter.shouldUseMultiDistinct(aggregate));
+ Assertions.assertEquals(Strategy.MULTI_STRATEGY,
rewriter.chooseStrategy(aggregate));
}
@Test
diff --git
a/regression-test/data/nereids_function_p0/agg_function/agg_distinct_function.out
b/regression-test/data/nereids_function_p0/agg_function/agg_distinct_function.out
new file mode 100644
index 00000000000..e3c944e1365
--- /dev/null
+++
b/regression-test/data/nereids_function_p0/agg_function/agg_distinct_function.out
@@ -0,0 +1,393 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !array_agg --
+[1, 2, 3, 4, 5]
+
+-- !array_agg_group --
+1 [1, 2, 3]
+2 [1, 4, 5]
+
+-- !avg --
+3.0
+
+-- !avg_group --
+1 2.0
+2 3.333333333333333
+
+-- !avg_weighted --
+3.666666666666667
+
+-- !avg_weighted_group --
+1 2.333333333333333
+2 4.2
+
+-- !collect_list --
+[1, 2, 3, 4, 5]
+
+-- !collect_list_group --
+1 [1, 2, 3]
+2 [1, 4, 5]
+
+-- !collect_set --
+[1, 2, 3, 4, 5]
+
+-- !collect_set_group --
+1 [1, 2, 3]
+2 [1, 4, 5]
+
+-- !corr_welford --
+1.0
+
+-- !corr_welford_group --
+1 1.0
+2 1.0
+
+-- !corr --
+1.0
+
+-- !corr_group --
+1 1.0
+2 1.0
+
+-- !covar --
+20.0
+
+-- !covar_group --
+1 6.666666666666664
+2 28.88888888888889
+
+-- !covar_samp --
+25.0
+
+-- !covar_samp_group --
+1 10.0
+2 43.33333333333334
+
+-- !group_bit_and --
+0
+
+-- !group_bit_and_group --
+1 0
+2 0
+
+-- !group_bit_or --
+7
+
+-- !group_bit_or_group --
+1 3
+2 5
+
+-- !group_bit_xor --
+1
+
+-- !group_bit_xor_group --
+1 0
+2 0
+
+-- !group_concat --
+a,b,c,d,e
+
+-- !group_concat_group --
+1 a,b,c
+2 a,d,e
+
+-- !histogram --
+{"num_buckets":5,"buckets":[{"lower":"1","upper":"1","ndv":1,"count":1,"pre_sum":0},{"lower":"2","upper":"2","ndv":1,"count":1,"pre_sum":1},{"lower":"3","upper":"3","ndv":1,"count":1,"pre_sum":2},{"lower":"4","upper":"4","ndv":1,"count":1,"pre_sum":3},{"lower":"5","upper":"5","ndv":1,"count":1,"pre_sum":4}]}
+
+-- !histogram_group --
+1
{"num_buckets":3,"buckets":[{"lower":"1","upper":"1","ndv":1,"count":1,"pre_sum":0},{"lower":"2","upper":"2","ndv":1,"count":1,"pre_sum":1},{"lower":"3","upper":"3","ndv":1,"count":1,"pre_sum":2}]}
+2
{"num_buckets":3,"buckets":[{"lower":"1","upper":"1","ndv":1,"count":1,"pre_sum":0},{"lower":"4","upper":"4","ndv":1,"count":1,"pre_sum":1},{"lower":"5","upper":"5","ndv":1,"count":1,"pre_sum":2}]}
+
+-- !hll_raw_agg --
+5
+
+-- !hll_raw_agg_group --
+1 3
+2 3
+
+-- !hll_union_agg --
+5
+
+-- !hll_union_agg_group --
+1 3
+2 3
+
+-- !kurt --
+-1.3
+
+-- !kurt_group --
+1 -1.5
+2 -1.499999999999995
+
+-- !map_agg_group --
+[1, 2, 3] ["a", "b", "c"]
+[1, 4, 5] ["a", "d", "e"]
+
+-- !map_agg --
+[1, 2, 3, 4, 5] ["a", "b", "c", "d", "e"]
+
+-- !max_by --
+e
+
+-- !max_by_group --
+1 c
+2 e
+
+-- !min_by --
+a
+
+-- !min_by_group --
+1 a
+2 a
+
+-- !percentile --
+3.0
+
+-- !percentile_group --
+1 2.0
+2 4.0
+
+-- !percentile_approx --
+3.0
+
+-- !percentile_approx_group --
+1 2.0
+2 4.0
+
+-- !percentile_array --
+[2, 3, 4]
+
+-- !percentile_array_group --
+1 [1.5, 2, 2.5]
+2 [2.5, 4, 4.5]
+
+-- !percentile_approx_weighted --
+3.857142925262451
+
+-- !percentile_approx_weighted_group --
+1 2.400000095367432
+2 4.44444465637207
+
+-- !topn --
+{"e":1,"d":1,"c":1}
+
+-- !topn_group --
+1 {"c":1,"b":1,"a":1}
+2 {"e":1,"d":1,"a":1}
+
+-- !topn_array --
+["e", "d", "c"]
+
+-- !topn_array_group --
+1 ["c", "b", "a"]
+2 ["e", "d", "a"]
+
+-- !topn_weighted --
+["e", "d", "c"]
+
+-- !topn_weighted_group --
+1 ["c", "b", "a"]
+2 ["e", "d", "a"]
+
+-- !two_distinct_functions --
+[1, 2, 3, 4, 5] [10, 20, 30, 40, 50]
+
+-- !two_distinct_functions_group --
+1 [1, 2, 3] [10, 20, 30]
+2 [1, 4, 5] [10, 40, 50]
+
+-- !two_avg --
+3.0 30.0
+
+-- !two_avg_group --
+1 2.0 20.0
+2 3.333333333333333 33.33333333333334
+
+-- !two_avg_weighted --
+3.666666666666667 36.66666666666666
+
+-- !two_avg_weighted_group --
+1 2.333333333333333 23.33333333333333
+2 4.2 42.0
+
+-- !two_collect_list --
+[1, 2, 3, 4, 5] [10, 20, 30, 40, 50]
+
+-- !two_collect_list_group --
+1 [1, 2, 3] [10, 20, 30]
+2 [1, 4, 5] [10, 40, 50]
+
+-- !two_collect_set --
+[1, 2, 3, 4, 5] [10, 20, 30, 40, 50]
+
+-- !two_collect_set_group --
+1 [1, 2, 3] [10, 20, 30]
+2 [1, 4, 5] [10, 40, 50]
+
+-- !two_corr_welford --
+0.9999999999999999 1.0
+
+-- !two_corr_welford_group --
+1 1.0 1.0
+2 1.0 1.0
+
+-- !two_corr --
+1.0 1.0
+
+-- !two_corr_group --
+1 1.0 1.0
+2 1.0 1.0
+
+-- !two_covar --
+20.0 2.0
+
+-- !two_covar_group --
+1 6.666666666666664 0.666666666666667
+2 28.88888888888889 2.888888888888889
+
+-- !two_covar_samp --
+25.0 2.5
+
+-- !two_covar_samp_group --
+1 10.0 1.0
+2 43.33333333333334 4.333333333333332
+
+-- !two_group_bit_and --
+0 0
+
+-- !two_group_bit_and_group --
+1 0 0
+2 0 0
+
+-- !two_group_bit_or --
+7 62
+
+-- !two_group_bit_or_group --
+1 3 30
+2 5 58
+
+-- !two_group_bit_xor --
+1 26
+
+-- !two_group_bit_xor_group --
+1 0 0
+2 0 16
+
+-- !two_group_concat --
+a,b,c,d,e 1,2,3,4,5
+
+-- !two_group_concat_group --
+1 a,b,c 1,2,3
+2 a,d,e 1,4,5
+
+-- !two_histogram --
+{"num_buckets":5,"buckets":[{"lower":"1","upper":"1","ndv":1,"count":1,"pre_sum":0},{"lower":"2","upper":"2","ndv":1,"count":1,"pre_sum":1},{"lower":"3","upper":"3","ndv":1,"count":1,"pre_sum":2},{"lower":"4","upper":"4","ndv":1,"count":1,"pre_sum":3},{"lower":"5","upper":"5","ndv":1,"count":1,"pre_sum":4}]}
{"num_buckets":5,"buckets":[{"lower":"10","upper":"10","ndv":1,"count":1,"pre_sum":0},{"lower":"20","upper":"20","ndv":1,"count":1,"pre_sum":1},{"lower":"30","upper":"30","ndv":1,"co
[...]
+
+-- !two_histogram_group --
+1
{"num_buckets":3,"buckets":[{"lower":"1","upper":"1","ndv":1,"count":1,"pre_sum":0},{"lower":"2","upper":"2","ndv":1,"count":1,"pre_sum":1},{"lower":"3","upper":"3","ndv":1,"count":1,"pre_sum":2}]}
{"num_buckets":3,"buckets":[{"lower":"10","upper":"10","ndv":1,"count":1,"pre_sum":0},{"lower":"20","upper":"20","ndv":1,"count":1,"pre_sum":1},{"lower":"30","upper":"30","ndv":1,"count":1,"pre_sum":2}]}
+2
{"num_buckets":3,"buckets":[{"lower":"1","upper":"1","ndv":1,"count":1,"pre_sum":0},{"lower":"4","upper":"4","ndv":1,"count":1,"pre_sum":1},{"lower":"5","upper":"5","ndv":1,"count":1,"pre_sum":2}]}
{"num_buckets":3,"buckets":[{"lower":"10","upper":"10","ndv":1,"count":1,"pre_sum":0},{"lower":"40","upper":"40","ndv":1,"count":1,"pre_sum":1},{"lower":"50","upper":"50","ndv":1,"count":1,"pre_sum":2}]}
+
+-- !two_hll_raw_agg --
+5 5
+
+-- !two_hll_raw_agg_group --
+1 3 3
+2 3 3
+
+-- !two_hll_union_agg --
+5 5
+
+-- !two_hll_union_agg_group --
+1 3 3
+2 3 3
+
+-- !two_kurt --
+-1.3 -1.3
+
+-- !two_kurt_group --
+1 -1.5 -1.5
+2 -1.499999999999995 -1.499999999999998
+
+-- !two_map_agg --
+[1, 2, 3, 4, 5] ["a", "b", "c", "d", "e"] [10, 20, 30, 40, 50]
["a", "b", "c", "d", "e"]
+
+-- !two_map_agg_group --
+1 [1, 2, 3] ["a", "b", "c"] [10, 20, 30] ["a", "b", "c"]
+2 [1, 4, 5] ["a", "d", "e"] [10, 40, 50] ["a", "d", "e"]
+
+-- !two_max_by --
+e e
+
+-- !two_max_by_group --
+1 c c
+2 e e
+
+-- !two_min_by --
+a a
+
+-- !two_min_by_group --
+1 a a
+2 a a
+
+-- !two_percentile --
+3.0 30.0
+
+-- !two_percentile_group --
+1 2.0 20.0
+2 4.0 40.0
+
+-- !two_percentile_approx --
+3.0 30.0
+
+-- !two_percentile_approx_group --
+1 2.0 20.0
+2 4.0 40.0
+
+-- !two_percentile_array --
+[2, 3, 4] [20, 30, 40]
+
+-- !two_percentile_array_group --
+1 [1.5, 2, 2.5] [15, 20, 25]
+2 [2.5, 4, 4.5] [25, 40, 45]
+
+-- !two_percentile_approx_weighted --
+3.857142925262451 38.57143020629883
+
+-- !two_percentile_approx_weighted_group --
+1 2.400000095367432 24.0
+2 4.44444465637207 44.44444274902344
+
+-- !two_topn --
+{"e":1,"d":1,"c":1} {"5":1,"4":1,"3":1}
+
+-- !two_topn_group --
+1 {"c":1,"b":1,"a":1} {"3":1,"2":1,"1":1}
+2 {"e":1,"d":1,"a":1} {"5":1,"4":1,"1":1}
+
+-- !two_topn_array --
+["e", "d", "c"] ["5", "4", "3"]
+
+-- !two_topn_array_group --
+1 ["c", "b", "a"] ["3", "2", "1"]
+2 ["e", "d", "a"] ["5", "4", "1"]
+
+-- !two_topn_weighted --
+["e", "d", "c"] ["5", "4", "3"]
+
+-- !two_topn_weighted_group --
+1 ["c", "b", "a"] ["3", "2", "1"]
+2 ["e", "d", "a"] ["5", "4", "1"]
+
+-- !percentile_res --
+2.0 40.0
+
+-- !percentile_res_group --
+1.5 25.0
+2.5 45.0
+
+-- !ema --
+2.501954556443252 0.8055193463643526
+
+-- !ema_group --
+1.500977039337158 0.4611636182575578
+2.501953125000455 0.7772022820853798
+
diff --git
a/regression-test/data/nereids_rules_p0/agg_strategy/distinct_agg_rewriter.out
b/regression-test/data/nereids_rules_p0/agg_strategy/distinct_agg_rewriter.out
index 18088719ce3..d051d2efe37 100644
---
a/regression-test/data/nereids_rules_p0/agg_strategy/distinct_agg_rewriter.out
+++
b/regression-test/data/nereids_rules_p0/agg_strategy/distinct_agg_rewriter.out
@@ -38,3 +38,12 @@ PhysicalResultSink
--------hashAgg[LOCAL]
----------PhysicalOlapScan[t1000_2]
+-- !agg_phase3 --
+PhysicalResultSink
+--PhysicalDistribute[DistributionSpecGather]
+----hashAgg[DISTINCT_GLOBAL]
+------hashAgg[GLOBAL]
+--------PhysicalDistribute[DistributionSpecHash]
+----------hashAgg[LOCAL]
+------------PhysicalTVFRelation
+
diff --git
a/regression-test/data/nereids_rules_p0/agg_strategy/distinct_agg_strategy_selector.out
b/regression-test/data/nereids_rules_p0/agg_strategy/distinct_agg_strategy_selector.out
index 05ef347e89b..09ac8a07824 100644
---
a/regression-test/data/nereids_rules_p0/agg_strategy/distinct_agg_strategy_selector.out
+++
b/regression-test/data/nereids_rules_p0/agg_strategy/distinct_agg_strategy_selector.out
@@ -110,3 +110,18 @@ PhysicalResultSink
--------hashAgg[LOCAL]
----------PhysicalOlapScan[t1000]
+-- !count_distinct_group --
+5 5
+
+-- !count_distinct_group_with_gby --
+1 1
+1 1
+1 1
+1 1
+1 1
+1 1
+1 1
+1 1
+1 1
+1 1
+
diff --git
a/regression-test/data/nereids_rules_p0/expression/test_convert_median_to_percentile.out
b/regression-test/data/nereids_rules_p0/expression/test_convert_median_to_percentile.out
index 3cccc3d6fb0..43e5deec56b 100644
---
a/regression-test/data/nereids_rules_p0/expression/test_convert_median_to_percentile.out
+++
b/regression-test/data/nereids_rules_p0/expression/test_convert_median_to_percentile.out
@@ -19,3 +19,6 @@
-- !select_6 --
2001 125.5
+-- !median_distinct --
+5.5 1
+
diff --git
a/regression-test/suites/nereids_function_p0/agg_function/agg_distinct_function.groovy
b/regression-test/suites/nereids_function_p0/agg_function/agg_distinct_function.groovy
new file mode 100644
index 00000000000..bd555aa6fac
--- /dev/null
+++
b/regression-test/suites/nereids_function_p0/agg_function/agg_distinct_function.groovy
@@ -0,0 +1,212 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("agg_distinct_function") {
+ sql "drop table if exists agg_distinct_function"
+ sql """
+ create table agg_distinct_function (
+ k int,
+ a int,
+ b int,
+ s varchar(10),
+ w int
+ ) engine=olap
+ duplicate key(k)
+ distributed by hash(k) buckets 1
+ properties ("replication_num" = "1")
+ """
+
+ // The duplicate rows make DISTINCT observable. Every query is run both as
a
+ // scalar aggregation and as a grouped aggregation.
+ sql """
+ insert into agg_distinct_function values
+ (1, 1, 10, 'a', 1), (1, 1, 10, 'a', 1),
+ (1, 2, 20, 'b', 2), (1, 3, 30, 'c', 3),
+ (2, 1, 10, 'a', 1), (2, 4, 40, 'd', 4),
+ (2, 4, 40, 'd', 4), (2, 5, 50, 'e', 5)
+ """
+
+ qt_array_agg "select array_sort(array_agg(distinct a)) from
agg_distinct_function"
+ order_qt_array_agg_group "select k, array_sort(array_agg(distinct a)) from
agg_distinct_function group by k order by k"
+ qt_avg "select avg(distinct a) from agg_distinct_function"
+ order_qt_avg_group "select k, avg(distinct a) from agg_distinct_function
group by k order by k"
+ qt_avg_weighted "select avg_weighted(distinct a, w) from
agg_distinct_function"
+ order_qt_avg_weighted_group "select k, avg_weighted(distinct a, w) from
agg_distinct_function group by k order by k"
+ qt_collect_list "select array_sort(collect_list(distinct a)) from
agg_distinct_function"
+ order_qt_collect_list_group "select k, array_sort(collect_list(distinct
a)) from agg_distinct_function group by k order by k"
+ qt_collect_set "select array_sort(collect_set(distinct a)) from
agg_distinct_function"
+ order_qt_collect_set_group "select k, array_sort(collect_set(distinct a))
from agg_distinct_function group by k order by k"
+ qt_corr_welford "select corr_welford(distinct a, b) from
agg_distinct_function"
+ order_qt_corr_welford_group "select k, corr_welford(distinct a, b) from
agg_distinct_function group by k order by k"
+ qt_corr "select corr(distinct a, b) from agg_distinct_function"
+ order_qt_corr_group "select k, corr(distinct a, b) from
agg_distinct_function group by k order by k"
+ qt_covar "select covar(distinct a, b) from agg_distinct_function"
+ order_qt_covar_group "select k, covar(distinct a, b) from
agg_distinct_function group by k order by k"
+ qt_covar_samp "select covar_samp(distinct a, b) from agg_distinct_function"
+ order_qt_covar_samp_group "select k, covar_samp(distinct a, b) from
agg_distinct_function group by k order by k"
+ qt_group_bit_and "select group_bit_and(distinct a) from
agg_distinct_function"
+ order_qt_group_bit_and_group "select k, group_bit_and(distinct a) from
agg_distinct_function group by k order by k"
+ qt_group_bit_or "select group_bit_or(distinct a) from
agg_distinct_function"
+ order_qt_group_bit_or_group "select k, group_bit_or(distinct a) from
agg_distinct_function group by k order by k"
+ qt_group_bit_xor "select group_bit_xor(distinct a) from
agg_distinct_function"
+ order_qt_group_bit_xor_group "select k, group_bit_xor(distinct a) from
agg_distinct_function group by k order by k"
+// qt_group_bitmap_xor "select bitmap_to_string(group_bitmap_xor(distinct
bitmap_hash(a))) from agg_distinct_function"
+// order_qt_group_bitmap_xor_group "select k,
bitmap_to_string(group_bitmap_xor(distinct bitmap_hash(a))) from
agg_distinct_function group by k order by k"
+ qt_group_concat "select group_concat(distinct s order by s) from
agg_distinct_function"
+ order_qt_group_concat_group "select k, group_concat(distinct s order by s)
from agg_distinct_function group by k order by k"
+ qt_histogram "select histogram(distinct a) from agg_distinct_function"
+ order_qt_histogram_group "select k, histogram(distinct a) from
agg_distinct_function group by k order by k"
+ qt_hll_raw_agg "select hll_cardinality(hll_raw_agg(distinct hll_hash(a)))
from agg_distinct_function"
+ order_qt_hll_raw_agg_group "select k, hll_cardinality(hll_raw_agg(distinct
hll_hash(a))) from agg_distinct_function group by k order by k"
+ qt_hll_union_agg "select hll_union_agg(distinct hll_hash(a)) from
agg_distinct_function"
+ order_qt_hll_union_agg_group "select k, hll_union_agg(distinct
hll_hash(a)) from agg_distinct_function group by k order by k"
+ qt_kurt "select kurt(distinct a) from agg_distinct_function"
+ order_qt_kurt_group "select k, kurt(distinct a) from agg_distinct_function
group by k order by k"
+ order_qt_map_agg_group """
+ SELECT
+ array_sort(map_keys(m)) AS sorted_keys,
+ array_sortby(map_values(m), map_keys(m)) AS values_by_sorted_key
+ FROM (
+ SELECT map_agg(DISTINCT a, s) AS m
+ FROM agg_distinct_function
+ group by k
+ ) t;
+ """
+ order_qt_map_agg """
+ SELECT
+ array_sort(map_keys(m)) AS sorted_keys,
+ array_sortby(map_values(m), map_keys(m)) AS values_by_sorted_key
+ FROM (
+ SELECT map_agg(DISTINCT a, s) AS m
+ FROM agg_distinct_function
+ ) t;
+ """
+ qt_max_by "select max_by(distinct s, a) from agg_distinct_function"
+ order_qt_max_by_group "select k, max_by(distinct s, a) from
agg_distinct_function group by k order by k"
+ qt_min_by "select min_by(distinct s, a) from agg_distinct_function"
+ order_qt_min_by_group "select k, min_by(distinct s, a) from
agg_distinct_function group by k order by k"
+ qt_percentile "select percentile(distinct a, 0.5) from
agg_distinct_function"
+ order_qt_percentile_group "select k, percentile(distinct a, 0.5) from
agg_distinct_function group by k order by k"
+ qt_percentile_approx "select percentile_approx(distinct a, 0.5) from
agg_distinct_function"
+ order_qt_percentile_approx_group "select k, percentile_approx(distinct a,
0.5) from agg_distinct_function group by k order by k"
+ qt_percentile_array "select percentile_array(distinct a, [0.25, 0.5,
0.75]) from agg_distinct_function"
+ order_qt_percentile_array_group "select k, percentile_array(distinct a,
[0.25, 0.5, 0.75]) from agg_distinct_function group by k order by k"
+ qt_percentile_approx_weighted "select percentile_approx_weighted(distinct
a, w, 0.5) from agg_distinct_function"
+ order_qt_percentile_approx_weighted_group "select k,
percentile_approx_weighted(distinct a, w, 0.5) from agg_distinct_function group
by k order by k"
+ qt_topn "select topn(distinct s, 3) from agg_distinct_function"
+ order_qt_topn_group "select k, topn(distinct s, 3) from
agg_distinct_function group by k order by k"
+ qt_topn_array "select topn_array(distinct s, 3) from agg_distinct_function"
+ order_qt_topn_array_group "select k, topn_array(distinct s, 3) from
agg_distinct_function group by k order by k"
+ qt_topn_weighted "select topn_weighted(distinct s, w, 3) from
agg_distinct_function"
+ order_qt_topn_weighted_group "select k, topn_weighted(distinct s, w, 3)
from agg_distinct_function group by k order by k"
+ sql "select count_by_enum(distinct s) from agg_distinct_function"
+ sql "select k, count_by_enum(distinct s) from agg_distinct_function group
by k order by k"
+
+ // Each pair uses different DISTINCT argument groups and exercises the CTE
split path.
+ qt_two_distinct_functions "select array_sort(array_agg(distinct a)),
array_sort(array_agg(distinct b)) from agg_distinct_function"
+ order_qt_two_distinct_functions_group "select k,
array_sort(array_agg(distinct a)), array_sort(array_agg(distinct b)) from
agg_distinct_function group by k order by k"
+ qt_two_avg "select avg(distinct a), avg(distinct b) from
agg_distinct_function"
+ order_qt_two_avg_group "select k, avg(distinct a), avg(distinct b) from
agg_distinct_function group by k order by k"
+ qt_two_avg_weighted "select avg_weighted(distinct a, w),
avg_weighted(distinct b, w) from agg_distinct_function"
+ order_qt_two_avg_weighted_group "select k, avg_weighted(distinct a, w),
avg_weighted(distinct b, w) from agg_distinct_function group by k order by k"
+ qt_two_collect_list "select array_sort(collect_list(distinct a)),
array_sort(collect_list(distinct b)) from agg_distinct_function"
+ order_qt_two_collect_list_group "select k,
array_sort(collect_list(distinct a)), array_sort(collect_list(distinct b)) from
agg_distinct_function group by k order by k"
+ qt_two_collect_set "select array_sort(collect_set(distinct a)),
array_sort(collect_set(distinct b)) from agg_distinct_function"
+ order_qt_two_collect_set_group "select k, array_sort(collect_set(distinct
a)), array_sort(collect_set(distinct b)) from agg_distinct_function group by k
order by k"
+ qt_two_corr_welford "select corr_welford(distinct a, b),
corr_welford(distinct a, w) from agg_distinct_function"
+ order_qt_two_corr_welford_group "select k, corr_welford(distinct a, b),
corr_welford(distinct a, w) from agg_distinct_function group by k order by k"
+ qt_two_corr "select corr(distinct a, b), corr(distinct a, w) from
agg_distinct_function"
+ order_qt_two_corr_group "select k, corr(distinct a, b), corr(distinct a,
w) from agg_distinct_function group by k order by k"
+ qt_two_covar "select covar(distinct a, b), covar(distinct a, w) from
agg_distinct_function"
+ order_qt_two_covar_group "select k, covar(distinct a, b), covar(distinct
a, w) from agg_distinct_function group by k order by k"
+ qt_two_covar_samp "select covar_samp(distinct a, b), covar_samp(distinct
a, w) from agg_distinct_function"
+ order_qt_two_covar_samp_group "select k, covar_samp(distinct a, b),
covar_samp(distinct a, w) from agg_distinct_function group by k order by k"
+ qt_two_group_bit_and "select group_bit_and(distinct a),
group_bit_and(distinct b) from agg_distinct_function"
+ order_qt_two_group_bit_and_group "select k, group_bit_and(distinct a),
group_bit_and(distinct b) from agg_distinct_function group by k order by k"
+ qt_two_group_bit_or "select group_bit_or(distinct a),
group_bit_or(distinct b) from agg_distinct_function"
+ order_qt_two_group_bit_or_group "select k, group_bit_or(distinct a),
group_bit_or(distinct b) from agg_distinct_function group by k order by k"
+ qt_two_group_bit_xor "select group_bit_xor(distinct a),
group_bit_xor(distinct b) from agg_distinct_function"
+ order_qt_two_group_bit_xor_group "select k, group_bit_xor(distinct a),
group_bit_xor(distinct b) from agg_distinct_function group by k order by k"
+// qt_two_group_bitmap_xor "select
bitmap_to_string(group_bitmap_xor(distinct bitmap_hash(a))),
bitmap_to_string(group_bitmap_xor(distinct bitmap_hash(b))) from
agg_distinct_function"
+// order_qt_two_group_bitmap_xor_group "select k,
bitmap_to_string(group_bitmap_xor(distinct bitmap_hash(a))),
bitmap_to_string(group_bitmap_xor(distinct bitmap_hash(b))) from
agg_distinct_function group by k order by k"
+ qt_two_group_concat "select group_concat(distinct s order by s),
group_concat(distinct cast(a as string) order by cast(a as string)) from
agg_distinct_function"
+ order_qt_two_group_concat_group "select k, group_concat(distinct s order
by s), group_concat(distinct cast(a as string) order by cast(a as string)) from
agg_distinct_function group by k order by k"
+ qt_two_histogram "select histogram(distinct a), histogram(distinct b) from
agg_distinct_function"
+ order_qt_two_histogram_group "select k, histogram(distinct a),
histogram(distinct b) from agg_distinct_function group by k order by k"
+ qt_two_hll_raw_agg "select hll_cardinality(hll_raw_agg(distinct
hll_hash(a))), hll_cardinality(hll_raw_agg(distinct hll_hash(b))) from
agg_distinct_function"
+ order_qt_two_hll_raw_agg_group "select k,
hll_cardinality(hll_raw_agg(distinct hll_hash(a))),
hll_cardinality(hll_raw_agg(distinct hll_hash(b))) from agg_distinct_function
group by k order by k"
+ qt_two_hll_union_agg "select hll_union_agg(distinct hll_hash(a)),
hll_union_agg(distinct hll_hash(b)) from agg_distinct_function"
+ order_qt_two_hll_union_agg_group "select k, hll_union_agg(distinct
hll_hash(a)), hll_union_agg(distinct hll_hash(b)) from agg_distinct_function
group by k order by k"
+ qt_two_kurt "select kurt(distinct a), kurt(distinct b) from
agg_distinct_function"
+ order_qt_two_kurt_group "select k, kurt(distinct a), kurt(distinct b) from
agg_distinct_function group by k order by k"
+ qt_two_map_agg """
+ select array_sort(map_keys(a_map)), array_sortby(map_values(a_map),
map_keys(a_map)),
+ array_sort(map_keys(b_map)), array_sortby(map_values(b_map),
map_keys(b_map))
+ from (
+ select map_agg(distinct a, s) as a_map, map_agg(distinct b, s) as
b_map
+ from agg_distinct_function
+ ) t
+ """
+ order_qt_two_map_agg_group """
+ select k, array_sort(map_keys(a_map)), array_sortby(map_values(a_map),
map_keys(a_map)),
+ array_sort(map_keys(b_map)), array_sortby(map_values(b_map),
map_keys(b_map))
+ from (
+ select k, map_agg(distinct a, s) as a_map, map_agg(distinct b, s)
as b_map
+ from agg_distinct_function
+ group by k
+ ) t
+ order by k
+ """
+ qt_two_max_by "select max_by(distinct s, a), max_by(distinct s, b) from
agg_distinct_function"
+ order_qt_two_max_by_group "select k, max_by(distinct s, a),
max_by(distinct s, b) from agg_distinct_function group by k order by k"
+ qt_two_min_by "select min_by(distinct s, a), min_by(distinct s, b) from
agg_distinct_function"
+ order_qt_two_min_by_group "select k, min_by(distinct s, a),
min_by(distinct s, b) from agg_distinct_function group by k order by k"
+ qt_two_percentile "select percentile(distinct a, 0.5), percentile(distinct
b, 0.5) from agg_distinct_function"
+ order_qt_two_percentile_group "select k, percentile(distinct a, 0.5),
percentile(distinct b, 0.5) from agg_distinct_function group by k order by k"
+ qt_two_percentile_approx "select percentile_approx(distinct a, 0.5),
percentile_approx(distinct b, 0.5) from agg_distinct_function"
+ order_qt_two_percentile_approx_group "select k, percentile_approx(distinct
a, 0.5), percentile_approx(distinct b, 0.5) from agg_distinct_function group by
k order by k"
+ qt_two_percentile_array "select percentile_array(distinct a, [0.25, 0.5,
0.75]), percentile_array(distinct b, [0.25, 0.5, 0.75]) from
agg_distinct_function"
+ order_qt_two_percentile_array_group "select k, percentile_array(distinct
a, [0.25, 0.5, 0.75]), percentile_array(distinct b, [0.25, 0.5, 0.75]) from
agg_distinct_function group by k order by k"
+ qt_two_percentile_approx_weighted "select
percentile_approx_weighted(distinct a, w, 0.5),
percentile_approx_weighted(distinct b, w, 0.5) from agg_distinct_function"
+ order_qt_two_percentile_approx_weighted_group "select k,
percentile_approx_weighted(distinct a, w, 0.5),
percentile_approx_weighted(distinct b, w, 0.5) from agg_distinct_function group
by k order by k"
+ qt_two_topn "select topn(distinct s, 3), topn(distinct cast(a as string),
3) from agg_distinct_function"
+ order_qt_two_topn_group "select k, topn(distinct s, 3), topn(distinct
cast(a as string), 3) from agg_distinct_function group by k order by k"
+ qt_two_topn_array "select topn_array(distinct s, 3), topn_array(distinct
cast(a as string), 3) from agg_distinct_function"
+ order_qt_two_topn_array_group "select k, topn_array(distinct s, 3),
topn_array(distinct cast(a as string), 3) from agg_distinct_function group by k
order by k"
+ qt_two_topn_weighted "select topn_weighted(distinct s, w, 3),
topn_weighted(distinct cast(a as string), w, 3) from agg_distinct_function"
+ order_qt_two_topn_weighted_group "select k, topn_weighted(distinct s, w,
3), topn_weighted(distinct cast(a as string), w, 3) from agg_distinct_function
group by k order by k"
+ sql "select count_by_enum(distinct s), count_by_enum(distinct cast(a as
string)) from agg_distinct_function"
+ sql "select k, count_by_enum(distinct s), count_by_enum(distinct cast(a as
string)) from agg_distinct_function group by k order by k"
+ order_qt_percentile_res """select percentile_reservoir(distinct a, 0.25),
percentile_reservoir(distinct b, 0.75) from agg_distinct_function"""
+ order_qt_percentile_res_group """select percentile_reservoir(distinct a,
0.25), percentile_reservoir(distinct b, 0.75) from agg_distinct_function group
by k"""
+ order_qt_ema """select exponential_moving_average(distinct 1.0, w,
b),exponential_moving_average(distinct 5.0, a, b) from agg_distinct_function"""
+ order_qt_ema_group """select exponential_moving_average(distinct 1.0, w,
b),exponential_moving_average(distinct 5.0, a, b) from agg_distinct_function
group by k"""
+
+ sql """
+ select percentile_reservoir(distinct a, 0.25),
percentile_reservoir(distinct a, 0.75)
+ from agg_distinct_function group by rollup(k)
+ """
+ sql """
+ select exponential_moving_average(distinct 1.0, a,
b),exponential_moving_average(distinct 5.0, a, b)
+ from agg_distinct_function group by rollup(k)
+ """
+
+ test {
+ sql "select bitmap_to_string(group_bitmap_xor(distinct
bitmap_hash(a))) from agg_distinct_function"
+ exception "group_bitmap_xor does not support DISTINCT"
+ }
+}
diff --git
a/regression-test/suites/nereids_p0/aggregate/agg_group_concat.groovy
b/regression-test/suites/nereids_p0/aggregate/agg_group_concat.groovy
index cdf5d5292fa..6fc8130b67f 100644
--- a/regression-test/suites/nereids_p0/aggregate/agg_group_concat.groovy
+++ b/regression-test/suites/nereids_p0/aggregate/agg_group_concat.groovy
@@ -133,4 +133,13 @@ suite("agg_group_concat") {
insert into test_distinct_multi
values(1,2,3,'abc','2024-01-02'),(1,2,4,'abc','2024-01-03'),(2,2,4,'abcd','2024-01-02'),(1,2,3,'abcd','2024-01-04'),(1,2,4,'eee','2024-02-02'),(2,2,4,'abc','2024-01-02');
"""
qt_test "select group_concat( distinct d order by d) from
test_distinct_multi order by 1; "
+
+ test {
+ sql "select multi_distinct_GROUP_CONCAT(a, c) from test_distinct_multi
group by b; "
+ exception "separator must be a constant"
+ }
+ test {
+ sql "select GROUP_CONCAT(a, c order by c) from test_distinct_multi
group by b; "
+ exception "separator must be a constant"
+ }
}
diff --git
a/regression-test/suites/nereids_rules_p0/agg_strategy/distinct_agg_rewriter.groovy
b/regression-test/suites/nereids_rules_p0/agg_strategy/distinct_agg_rewriter.groovy
index 953c233c0d5..10d3d8185ca 100644
---
a/regression-test/suites/nereids_rules_p0/agg_strategy/distinct_agg_rewriter.groovy
+++
b/regression-test/suites/nereids_rules_p0/agg_strategy/distinct_agg_rewriter.groovy
@@ -33,4 +33,25 @@ suite("distinct_agg_rewriter") {
select count(distinct d_200) from t1000_2 group by b_5;"""
qt_use_multi_distinct """explain shape plan
select count(distinct d_200) from t1000_2 group by a_1;"""
-}
\ No newline at end of file
+
+ sql "set agg_phase=4"
+ sql "select group_concat(distinct dst_key1 order by dst_key2) from
t_gbykey_10_dstkey_10_1000_id group by gby_key"
+ test {
+ sql """select number % 2, count(distinct cast(number as varchar),
cast(number as varchar)),
+ group_concat(distinct cast(number as varchar) order by number + 1)
from numbers('number'='10') group by number % 2"""
+ exception "Unsupported query"
+ }
+
+ test {
+ sql """select count(distinct cast(number as varchar), cast(number as
varchar)),
+ group_concat(distinct cast(number as varchar) order by number + 1)
from numbers('number'='10')"""
+ exception "Unsupported query"
+ }
+
+ // expect to have 3 HashAgg instead of 4 HashAgg
+ sql "set agg_phase=3"
+ qt_agg_phase3 """explain shape plan
+ select number % 2, count(distinct number)
+ from numbers('number'='10')
+ group by number % 2;"""
+}
diff --git
a/regression-test/suites/nereids_rules_p0/agg_strategy/distinct_agg_strategy_selector.groovy
b/regression-test/suites/nereids_rules_p0/agg_strategy/distinct_agg_strategy_selector.groovy
index a565f55a0bf..23717e8491e 100644
---
a/regression-test/suites/nereids_rules_p0/agg_strategy/distinct_agg_strategy_selector.groovy
+++
b/regression-test/suites/nereids_rules_p0/agg_strategy/distinct_agg_strategy_selector.groovy
@@ -52,4 +52,47 @@ suite("distinct_agg_strategy_selector") {
contains "multi_distinct_count"
}
sql "set multi_distinct_strategy=0 "
+
+ // test
+ order_qt_count_distinct_group "select count(distinct a_1,b_5),
count(distinct b_5,a_1) from t1000;"
+ order_qt_count_distinct_group_with_gby "select count(distinct a_1,b_5),
count(distinct b_5,a_1) from t1000 group by c_10;"
+
+ multi_sql """
+ SET enable_nereids_planner = true;
+
+ DROP VIEW IF EXISTS v_distinct_guard_repro;
+ DROP TABLE IF EXISTS t_distinct_guard_repro;
+
+ CREATE TABLE t_distinct_guard_repro (
+ id INT,
+ a DECIMAL(20, 2),
+ b INT
+ )
+ ENGINE = OLAP
+ DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES (
+ "replication_num" = "1"
+ );
+
+ INSERT INTO t_distinct_guard_repro VALUES
+ (1, 1.00, 10),
+ (2, 1.00, 10),
+ (3, 2.00, 20),
+ (4, 3.00, 30);
+ """
+ sql "SET enable_decimal256 = false;"
+
+ sql """CREATE VIEW v_distinct_guard_repro AS
+ SELECT
+ SUM(DISTINCT a) AS distinct_sum_a,
+ ARRAY_AGG(DISTINCT b) AS distinct_b_values
+ FROM t_distinct_guard_repro;"""
+
+ sql "SET enable_decimal256 = true;"
+
+ sql """SELECT
+ distinct_sum_a,
+ array_sort(distinct_b_values)
+ FROM v_distinct_guard_repro;"""
}
\ No newline at end of file
diff --git
a/regression-test/suites/nereids_rules_p0/expression/test_convert_median_to_percentile.groovy
b/regression-test/suites/nereids_rules_p0/expression/test_convert_median_to_percentile.groovy
index 2d8cf3a8b59..11c03bb37cd 100644
---
a/regression-test/suites/nereids_rules_p0/expression/test_convert_median_to_percentile.groovy
+++
b/regression-test/suites/nereids_rules_p0/expression/test_convert_median_to_percentile.groovy
@@ -73,6 +73,19 @@ suite("test_convert_median_to_percentile") {
qt_select_5 "${sql5}"
qt_select_6 "${sql6}"
+ sql "DROP TABLE IF EXISTS median_distinct"
+ sql """
+ CREATE TABLE median_distinct (
+ id INT,
+ value INT
+ )
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES ("replication_num" = "1")
+ """
+ sql "INSERT INTO median_distinct VALUES (1, 1), (2, 1), (3, 10)"
+ order_qt_median_distinct "SELECT median(DISTINCT value), median(value)
FROM median_distinct"
+ sql "DROP TABLE IF EXISTS median_distinct"
+
sql "DROP TABLE if exists sales"
sql "DROP DATABASE if exists test_convert_median_to_percentile"
}
diff --git
a/regression-test/suites/query_p0/group_concat/test_group_concat.groovy
b/regression-test/suites/query_p0/group_concat/test_group_concat.groovy
index a60514c719f..eeb3cb0cff7 100644
--- a/regression-test/suites/query_p0/group_concat/test_group_concat.groovy
+++ b/regression-test/suites/query_p0/group_concat/test_group_concat.groovy
@@ -91,7 +91,7 @@ suite("test_group_concat", "query,p0,arrow_flight_sql") {
// test SPLIT_MULTI_DISTINCT could work right with can not be banned
aggregation
qt_select_13 """
select
- group_concat( distinct b1, cast(b2 as varchar)), group_concat(
distinct b3, '?')
+ group_concat( distinct b1, ','), group_concat( distinct b3,
'?')
from
table_group_concat
group by
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
