This is an automated email from the ASF dual-hosted git repository.
starocean999 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 27352afdf6 [fix](fe)support multi distinct group_concat (#17237)
27352afdf6 is described below
commit 27352afdf6e14c96cb3c10c526a5fb7b8ae7895f
Author: starocean999 <[email protected]>
AuthorDate: Thu Mar 2 17:53:13 2023 +0800
[fix](fe)support multi distinct group_concat (#17237)
* [fix](fe)support multi distinct group_concat
* update based on comments
---
.../aggregate_function_simple_factory.cpp | 2 +-
.../main/java/org/apache/doris/analysis/Expr.java | 9 ++
.../apache/doris/analysis/FunctionCallExpr.java | 5 +
.../java/org/apache/doris/analysis/SelectStmt.java | 15 ++-
.../apache/doris/catalog/AggregateFunction.java | 7 +-
.../java/org/apache/doris/catalog/FunctionSet.java | 20 ++++
.../rules/implementation/AggregateStrategies.java | 3 +
.../expressions/functions/agg/GroupConcat.java | 24 +++++
...upConcat.java => MultiDistinctGroupConcat.java} | 105 ++++++++++-----------
.../visitor/AggregateFunctionVisitor.java | 5 +
.../nereids_p0/group_concat/test_group_concat.out | 65 ++-----------
.../query_p0/group_concat/test_group_concat.out | 64 ++-----------
.../group_concat/test_group_concat.groovy | 60 ++++++++----
.../query_p0/group_concat/test_group_concat.groovy | 60 ++++++++----
14 files changed, 228 insertions(+), 216 deletions(-)
diff --git
a/be/src/vec/aggregate_functions/aggregate_function_simple_factory.cpp
b/be/src/vec/aggregate_functions/aggregate_function_simple_factory.cpp
index d42ae77c41..400c34ae4d 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_simple_factory.cpp
+++ b/be/src/vec/aggregate_functions/aggregate_function_simple_factory.cpp
@@ -69,6 +69,7 @@ AggregateFunctionSimpleFactory&
AggregateFunctionSimpleFactory::instance() {
register_aggregate_function_uniq(instance);
register_aggregate_function_bit(instance);
register_aggregate_function_bitmap(instance);
+ register_aggregate_function_group_concat(instance);
register_aggregate_function_combinator_distinct(instance);
register_aggregate_function_reader_load(
instance); // register aggregate function for agg reader
@@ -76,7 +77,6 @@ AggregateFunctionSimpleFactory&
AggregateFunctionSimpleFactory::instance() {
register_aggregate_function_stddev_variance_pop(instance);
register_aggregate_function_topn(instance);
register_aggregate_function_approx_count_distinct(instance);
- register_aggregate_function_group_concat(instance);
register_aggregate_function_percentile(instance);
register_aggregate_function_percentile_approx(instance);
register_aggregate_function_window_funnel(instance);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
index 0e019871e5..dc32900bbc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
@@ -2100,6 +2100,15 @@ public abstract class Expr extends TreeNode<Expr>
implements ParseNode, Cloneabl
return hasNullableChild();
}
}
+ if (fn.functionName().equalsIgnoreCase("group_concat")) {
+ int size = Math.min(fn.getNumArgs(), children.size());
+ for (int i = 0; i < size; ++i) {
+ if (children.get(i).isNullable()) {
+ return true;
+ }
+ }
+ return false;
+ }
return true;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
index 1d90d181bc..9e101275a0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
@@ -719,6 +719,11 @@ public class FunctionCallExpr extends Expr {
}
}
+ if (fnParams.isDistinct() && !orderByElements.isEmpty()) {
+ throw new AnalysisException(
+ "group_concat don't support using distinct with order
by together: " + this.toSql());
+ }
+
return;
}
if (fnName.getFunction().equalsIgnoreCase("field")) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
index 1261126a72..9382c01edf 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
@@ -1321,9 +1321,9 @@ public class SelectStmt extends QueryStmt {
// i) There is no GROUP-BY clause, and
// ii) Other DISTINCT aggregates are present.
ExprSubstitutionMap countAllMap = createCountAllMap(aggExprs,
analyzer);
- final ExprSubstitutionMap multiCountOrSumDistinctMap =
- createSumOrCountMultiDistinctSMap(aggExprs, analyzer);
- countAllMap = ExprSubstitutionMap.compose(multiCountOrSumDistinctMap,
countAllMap, analyzer);
+ final ExprSubstitutionMap multiDistinctAggMap =
+ createMultiDistinctAggSMap(aggExprs, analyzer);
+ countAllMap = ExprSubstitutionMap.compose(multiDistinctAggMap,
countAllMap, analyzer);
List<Expr> substitutedAggs =
Expr.substituteList(aggExprs, countAllMap, analyzer, false);
// the resultExprs must substitute in the same way as aggExprs
@@ -1530,10 +1530,10 @@ public class SelectStmt extends QueryStmt {
}
/**
- * Build smap count_distinct->multi_count_distinct
sum_distinct->multi_count_distinct
+ * Build smap like: count_distinct->multi_count_distinct
sum_distinct->multi_count_distinct
* assumes that select list and having clause have been analyzed.
*/
- private ExprSubstitutionMap createSumOrCountMultiDistinctSMap(
+ private ExprSubstitutionMap createMultiDistinctAggSMap(
ArrayList<FunctionCallExpr> aggExprs, Analyzer analyzer) throws
AnalysisException {
final List<FunctionCallExpr> distinctExprs = Lists.newArrayList();
for (FunctionCallExpr aggExpr : aggExprs) {
@@ -1565,6 +1565,11 @@ public class SelectStmt extends QueryStmt {
final FunctionCallExpr countExpr = new
FunctionCallExpr("MULTI_DISTINCT_COUNT",
new FunctionParams(inputExpr.isDistinct(),
countInputExpr));
replaceExpr = new
ArithmeticExpr(ArithmeticExpr.Operator.DIVIDE, sumExpr, countExpr);
+ } else if (functionName.equalsIgnoreCase("GROUP_CONCAT")) {
+ final List<Expr> groupConcatInputExprs =
inputExpr.getChildren();
+ replaceExpr = new FunctionCallExpr(new
FunctionName("MULTI_DISTINCT_GROUP_CONCAT"),
+ new FunctionParams(inputExpr.isDistinct(),
groupConcatInputExprs),
+ inputExpr.getOrderByElements());
} else {
throw new AnalysisException(inputExpr.getFnName() + " can't
support multi distinct.");
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/AggregateFunction.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/AggregateFunction.java
index 2ec55fc8bf..84667a5897 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/AggregateFunction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/AggregateFunction.java
@@ -58,6 +58,9 @@ public class AggregateFunction extends Function {
public static ImmutableSet<String>
ALWAYS_NULLABLE_AGGREGATE_FUNCTION_NAME_SET =
ImmutableSet.of("stddev_samp", "variance_samp", "var_samp",
"percentile_approx");
+ public static ImmutableSet<String> CUSTOM_AGGREGATE_FUNCTION_NAME_SET =
+ ImmutableSet.of("group_concat");
+
public static ImmutableSet<String>
SUPPORT_ORDER_BY_AGGREGATE_FUNCTION_NAME_SET = ImmutableSet.of("group_concat");
// Set if different from retType_, null otherwise.
@@ -169,7 +172,9 @@ public class AggregateFunction extends Function {
AggregateFunction.NOT_NULLABLE_AGGREGATE_FUNCTION_NAME_SET.contains(fnName.getFunction())
? NullableMode.ALWAYS_NOT_NULLABLE :
AggregateFunction.ALWAYS_NULLABLE_AGGREGATE_FUNCTION_NAME_SET.contains(fnName.getFunction())
- ? NullableMode.ALWAYS_NULLABLE :
NullableMode.DEPEND_ON_ARGUMENT);
+ ? NullableMode.ALWAYS_NULLABLE :
+
AggregateFunction.CUSTOM_AGGREGATE_FUNCTION_NAME_SET.contains(fnName.getFunction())
+ ? NullableMode.CUSTOM :
NullableMode.DEPEND_ON_ARGUMENT);
setLocation(location);
this.intermediateType = (intermediateType.equals(retType)) ? null :
intermediateType;
this.updateFnSymbol = updateFnSymbol;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java
index f344a3c721..0a4bed2789 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java
@@ -2748,9 +2748,29 @@ public class FunctionSet<T> {
// Group_concat(string) vectorized
addBuiltin(AggregateFunction.createBuiltin("group_concat",
Lists.<Type>newArrayList(Type.VARCHAR), Type.VARCHAR,
Type.VARCHAR, initNullString, "", "", "", "", false, true,
false, true));
+
addBuiltin(AggregateFunction.createBuiltin("multi_distinct_group_concat",
Lists.<Type>newArrayList(Type.VARCHAR), Type.VARCHAR,
+ Type.VARCHAR, initNullString, "", "", "", "", false, true,
false, true));
+ addBuiltin(AggregateFunction.createBuiltin("group_concat",
Lists.<Type>newArrayList(Type.CHAR), Type.CHAR,
+ Type.CHAR, initNullString, "", "", "", "", false, true, false,
true));
+
addBuiltin(AggregateFunction.createBuiltin("multi_distinct_group_concat",
Lists.<Type>newArrayList(Type.CHAR), Type.CHAR,
+ Type.CHAR, initNullString, "", "", "", "", false, true, false,
true));
+ addBuiltin(AggregateFunction.createBuiltin("group_concat",
Lists.<Type>newArrayList(Type.STRING), Type.STRING,
+ Type.STRING, initNullString, "", "", "", "", false, true,
false, true));
+
addBuiltin(AggregateFunction.createBuiltin("multi_distinct_group_concat",
Lists.<Type>newArrayList(Type.STRING), Type.STRING,
+ Type.STRING, initNullString, "", "", "", "", false, true,
false, true));
// Group_concat(string, string) vectorized
addBuiltin(AggregateFunction.createBuiltin("group_concat",
Lists.<Type>newArrayList(Type.VARCHAR, Type.VARCHAR),
Type.VARCHAR, Type.VARCHAR, initNullString, "", "", "", "",
false, true, false, true));
+
addBuiltin(AggregateFunction.createBuiltin("multi_distinct_group_concat",
Lists.<Type>newArrayList(Type.VARCHAR, Type.VARCHAR),
+ Type.VARCHAR, Type.VARCHAR, initNullString, "", "", "", "",
false, true, false, true));
+ addBuiltin(AggregateFunction.createBuiltin("group_concat",
Lists.<Type>newArrayList(Type.CHAR, Type.CHAR),
+ Type.CHAR, Type.CHAR, initNullString, "", "", "", "", false,
true, false, true));
+
addBuiltin(AggregateFunction.createBuiltin("multi_distinct_group_concat",
Lists.<Type>newArrayList(Type.CHAR, Type.CHAR),
+ Type.CHAR, Type.CHAR, initNullString, "", "", "", "", false,
true, false, true));
+ addBuiltin(AggregateFunction.createBuiltin("group_concat",
Lists.<Type>newArrayList(Type.STRING, Type.STRING),
+ Type.STRING, Type.STRING, initNullString, "", "", "", "",
false, true, false, true));
+
addBuiltin(AggregateFunction.createBuiltin("multi_distinct_group_concat",
Lists.<Type>newArrayList(Type.STRING, Type.STRING),
+ Type.STRING, Type.STRING, initNullString, "", "", "", "",
false, true, false, true));
// analytic functions
// Rank
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java
index 5cf69e5f63..f56bec8d21 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java
@@ -43,6 +43,7 @@ import
org.apache.doris.nereids.trees.expressions.SlotReference;
import
org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction;
import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateParam;
import org.apache.doris.nereids.trees.expressions.functions.agg.Count;
+import org.apache.doris.nereids.trees.expressions.functions.agg.GroupConcat;
import
org.apache.doris.nereids.trees.expressions.functions.agg.MultiDistinctCount;
import
org.apache.doris.nereids.trees.expressions.functions.agg.MultiDistinctSum;
import org.apache.doris.nereids.trees.expressions.functions.agg.Sum;
@@ -1183,6 +1184,8 @@ public class AggregateStrategies implements
ImplementationRuleFactory {
function.getArguments().subList(1,
function.arity()).toArray(new Expression[0]));
} else if (function instanceof Sum && function.isDistinct()) {
return new MultiDistinctSum(function.getArgument(0));
+ } else if (function instanceof GroupConcat && function.isDistinct()) {
+ return ((GroupConcat) function).convertToMultiDistinct();
}
return function;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/GroupConcat.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/GroupConcat.java
index 20e6c29c62..68f94433be 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/GroupConcat.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/GroupConcat.java
@@ -55,6 +55,7 @@ public class GroupConcat extends NullableAggregateFunction
public GroupConcat(boolean distinct, boolean alwaysNullable, Expression
arg, OrderExpression... orders) {
super("group_concat", distinct, alwaysNullable,
ExpressionUtils.mergeArguments(arg, orders));
this.nonOrderArguments = 1;
+ checkArguments();
}
/**
@@ -78,6 +79,7 @@ public class GroupConcat extends NullableAggregateFunction
Expression arg0, Expression arg1, OrderExpression... orders) {
super("group_concat", distinct, alwaysNullable,
ExpressionUtils.mergeArguments(arg0, arg1, orders));
this.nonOrderArguments = 2;
+ checkArguments();
}
/**
@@ -100,6 +102,13 @@ public class GroupConcat extends NullableAggregateFunction
public GroupConcat(boolean distinct, boolean alwaysNullable, int
nonOrderArguments, List<Expression> args) {
super("group_concat", distinct, alwaysNullable, args);
this.nonOrderArguments = nonOrderArguments;
+ checkArguments();
+ }
+
+ @Override
+ public boolean nullable() {
+ return alwaysNullable || children().stream()
+ .anyMatch(expression -> !(expression instanceof
OrderExpression) && expression.nullable());
}
@Override
@@ -164,4 +173,19 @@ public class GroupConcat extends NullableAggregateFunction
public List<FunctionSignature> getSignatures() {
return SIGNATURES;
}
+
+ public MultiDistinctGroupConcat convertToMultiDistinct() {
+ Preconditions.checkArgument(distinct,
+ "can't convert to multi_distinct_group_concat because there is
no distinct args");
+ return new MultiDistinctGroupConcat(alwaysNullable, nonOrderArguments,
children);
+ }
+
+ // TODO: because of current be's limitation, we have to thow exception for
now
+ // remove this after be support new method of multi distinct functions
+ private void checkArguments() {
+ if (isDistinct() && children().stream().anyMatch(expression ->
expression instanceof OrderExpression)) {
+ throw new AnalysisException(
+ "group_concat don't support using distinct with order by
together");
+ }
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/GroupConcat.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/MultiDistinctGroupConcat.java
similarity index 51%
copy from
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/GroupConcat.java
copy to
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/MultiDistinctGroupConcat.java
index 20e6c29c62..737e895906 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/GroupConcat.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/MultiDistinctGroupConcat.java
@@ -23,7 +23,8 @@ import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.OrderExpression;
import
org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
-import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.nereids.types.CharType;
+import org.apache.doris.nereids.types.StringType;
import org.apache.doris.nereids.types.VarcharType;
import org.apache.doris.nereids.types.coercion.AnyDataType;
import org.apache.doris.nereids.util.ExpressionUtils;
@@ -33,102 +34,90 @@ import com.google.common.collect.ImmutableList;
import java.util.List;
-/**
- * AggregateFunction 'group_concat'. This class is generated by
GenerateFunction.
- */
-public class GroupConcat extends NullableAggregateFunction
+/** MultiDistinctGroupConcat */
+public class MultiDistinctGroupConcat extends NullableAggregateFunction
implements ExplicitlyCastableSignature {
public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT).args(VarcharType.SYSTEM_DEFAULT),
- FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT)
- .varArgs(VarcharType.SYSTEM_DEFAULT, AnyDataType.INSTANCE),
- FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT)
- .varArgs(VarcharType.SYSTEM_DEFAULT,
VarcharType.SYSTEM_DEFAULT, AnyDataType.INSTANCE)
- );
+
FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT).varArgs(VarcharType.SYSTEM_DEFAULT,
+ AnyDataType.INSTANCE),
+
FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT).varArgs(VarcharType.SYSTEM_DEFAULT,
+ VarcharType.SYSTEM_DEFAULT, AnyDataType.INSTANCE),
+
+
FunctionSignature.ret(StringType.INSTANCE).args(StringType.INSTANCE),
+
FunctionSignature.ret(StringType.INSTANCE).varArgs(StringType.INSTANCE,
+ AnyDataType.INSTANCE),
+
FunctionSignature.ret(StringType.INSTANCE).varArgs(StringType.INSTANCE,
+ StringType.INSTANCE, AnyDataType.INSTANCE),
+
+
FunctionSignature.ret(CharType.SYSTEM_DEFAULT).args(CharType.SYSTEM_DEFAULT),
+
FunctionSignature.ret(CharType.SYSTEM_DEFAULT).varArgs(CharType.SYSTEM_DEFAULT,
+ AnyDataType.INSTANCE),
+
FunctionSignature.ret(CharType.SYSTEM_DEFAULT).varArgs(CharType.SYSTEM_DEFAULT,
+ CharType.SYSTEM_DEFAULT, AnyDataType.INSTANCE));
private final int nonOrderArguments;
/**
* constructor with 1 argument.
*/
- public GroupConcat(boolean distinct, boolean alwaysNullable, Expression
arg, OrderExpression... orders) {
- super("group_concat", distinct, alwaysNullable,
ExpressionUtils.mergeArguments(arg, orders));
+ public MultiDistinctGroupConcat(boolean alwaysNullable, Expression arg,
+ OrderExpression... orders) {
+ super("multi_distinct_group_concat", true, alwaysNullable,
+ ExpressionUtils.mergeArguments(arg, orders));
this.nonOrderArguments = 1;
}
/**
* constructor with 1 argument.
*/
- public GroupConcat(boolean distinct, Expression arg, OrderExpression...
orders) {
- this(distinct, false, arg, orders);
- }
-
- /**
- * constructor with 1 argument, use for function search.
- */
- public GroupConcat(Expression arg, OrderExpression... orders) {
+ public MultiDistinctGroupConcat(Expression arg, OrderExpression... orders)
{
this(false, arg, orders);
}
/**
* constructor with 2 arguments.
*/
- public GroupConcat(boolean distinct, boolean alwaysNullable,
- Expression arg0, Expression arg1, OrderExpression... orders) {
- super("group_concat", distinct, alwaysNullable,
ExpressionUtils.mergeArguments(arg0, arg1, orders));
+ public MultiDistinctGroupConcat(boolean alwaysNullable, Expression arg0,
+ Expression arg1, OrderExpression... orders) {
+ super("multi_distinct_group_concat", true, alwaysNullable,
+ ExpressionUtils.mergeArguments(arg0, arg1, orders));
this.nonOrderArguments = 2;
}
/**
* constructor with 2 arguments.
*/
- public GroupConcat(boolean distinct, Expression arg0, Expression arg1,
OrderExpression... orders) {
- this(distinct, false, arg0, arg1, orders);
- }
-
- /**
- * constructor with 2 arguments, use for function search.
- */
- public GroupConcat(Expression arg0, Expression arg1, OrderExpression...
orders) {
+ public MultiDistinctGroupConcat(Expression arg0, Expression arg1,
OrderExpression... orders) {
this(false, arg0, arg1, orders);
}
/**
* constructor for always nullable.
*/
- public GroupConcat(boolean distinct, boolean alwaysNullable, int
nonOrderArguments, List<Expression> args) {
- super("group_concat", distinct, alwaysNullable, args);
+ public MultiDistinctGroupConcat(boolean alwaysNullable, int
nonOrderArguments,
+ List<Expression> args) {
+ super("multi_distinct_group_concat", true, alwaysNullable, args);
this.nonOrderArguments = nonOrderArguments;
}
@Override
- public void checkLegalityBeforeTypeCoercion() {
- DataType typeOrArg0 = getArgumentType(0);
- if (!typeOrArg0.isStringLikeType() && !typeOrArg0.isNullType()) {
- throw new AnalysisException(
- "group_concat requires first parameter to be of type
STRING: " + this.toSql());
- }
-
- if (nonOrderArguments == 2) {
- DataType typeOrArg1 = getArgumentType(1);
- if (!typeOrArg1.isStringLikeType() && !typeOrArg1.isNullType()) {
- throw new AnalysisException(
- "group_concat requires second parameter to be of type
STRING: " + this.toSql());
- }
- }
+ public boolean nullable() {
+ return alwaysNullable || children().stream()
+ .anyMatch(expression -> !(expression instanceof
OrderExpression) && expression.nullable());
}
@Override
- public GroupConcat withAlwaysNullable(boolean alwaysNullable) {
- return new GroupConcat(distinct, alwaysNullable, nonOrderArguments,
children);
+ public MultiDistinctGroupConcat withAlwaysNullable(boolean alwaysNullable)
{
+ return new MultiDistinctGroupConcat(alwaysNullable, nonOrderArguments,
children);
}
/**
* withDistinctAndChildren.
*/
@Override
- public GroupConcat withDistinctAndChildren(boolean distinct,
List<Expression> children) {
+ public MultiDistinctGroupConcat withDistinctAndChildren(boolean distinct,
List<Expression> children) {
Preconditions.checkArgument(children().size() >= 1);
boolean foundOrderExpr = false;
int firstOrderExrIndex = 0;
@@ -139,25 +128,27 @@ public class GroupConcat extends NullableAggregateFunction
} else if (!foundOrderExpr) {
firstOrderExrIndex++;
} else {
- throw new AnalysisException("invalid group_concat parameters:
" + children);
+ throw new AnalysisException(
+ "invalid multi_distinct_group_concat parameters: " +
children);
}
}
List<OrderExpression> orders = (List)
children.subList(firstOrderExrIndex, children.size());
if (firstOrderExrIndex == 1) {
- return new GroupConcat(distinct, alwaysNullable,
- children.get(0), orders.toArray(new OrderExpression[0]));
+ return new MultiDistinctGroupConcat(alwaysNullable,
children.get(0),
+ orders.toArray(new OrderExpression[0]));
} else if (firstOrderExrIndex == 2) {
- return new GroupConcat(distinct, alwaysNullable,
- children.get(0), children.get(1), orders.toArray(new
OrderExpression[0]));
+ return new MultiDistinctGroupConcat(alwaysNullable,
children.get(0),
+ children.get(1), orders.toArray(new OrderExpression[0]));
} else {
- throw new AnalysisException("group_concat requires one or two
parameters: " + children);
+ throw new AnalysisException(
+ "multi_distinct_group_concat requires one or two
parameters: " + children);
}
}
@Override
public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
- return visitor.visitGroupConcat(this, context);
+ return visitor.visitMultiDistinctGroupConcat(this, context);
}
@Override
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/AggregateFunctionVisitor.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/AggregateFunctionVisitor.java
index 9c1b46e292..fe5e660b7f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/AggregateFunctionVisitor.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/AggregateFunctionVisitor.java
@@ -42,6 +42,7 @@ import
org.apache.doris.nereids.trees.expressions.functions.agg.MaxBy;
import org.apache.doris.nereids.trees.expressions.functions.agg.Min;
import org.apache.doris.nereids.trees.expressions.functions.agg.MinBy;
import
org.apache.doris.nereids.trees.expressions.functions.agg.MultiDistinctCount;
+import
org.apache.doris.nereids.trees.expressions.functions.agg.MultiDistinctGroupConcat;
import
org.apache.doris.nereids.trees.expressions.functions.agg.MultiDistinctSum;
import org.apache.doris.nereids.trees.expressions.functions.agg.Ndv;
import
org.apache.doris.nereids.trees.expressions.functions.agg.NullableAggregateFunction;
@@ -118,6 +119,10 @@ public interface AggregateFunctionVisitor<R, C> {
return visitAggregateFunction(multiDistinctCount, context);
}
+ default R visitMultiDistinctGroupConcat(MultiDistinctGroupConcat
multiDistinctGroupConcat, C context) {
+ return visitAggregateFunction(multiDistinctGroupConcat, context);
+ }
+
default R visitMultiDistinctSum(MultiDistinctSum multiDistinctSum, C
context) {
return visitAggregateFunction(multiDistinctSum, context);
}
diff --git a/regression-test/data/nereids_p0/group_concat/test_group_concat.out
b/regression-test/data/nereids_p0/group_concat/test_group_concat.out
index 9569bd5b32..c00188a6cf 100644
--- a/regression-test/data/nereids_p0/group_concat/test_group_concat.out
+++ b/regression-test/data/nereids_p0/group_concat/test_group_concat.out
@@ -26,65 +26,12 @@ false
2147483647 255:1991:32767:32767
-- !select --
-\N \N
-103 255
-1001 1989, 1986
-1002 1989, 32767
-3021 1991, 32767, 1992
-5014 1985, 1991
-25699 1989
-2147483647 255, 1991, 32767, 32767
-
--- !select --
-\N \N
-103 255
-1001 1989:1986
-1002 1989:32767
-3021 1991:32767:1992
-5014 1985:1991
-25699 1989
-2147483647 255:1991:32767:32767
-
--- !select --
-\N \N
-103 255
-1001 1986, 1989
-1002 1989, 32767
-3021 1991, 1992, 32767
-5014 1985, 1991
-25699 1989
-2147483647 255, 1991, 32767, 32767
-
--- !select --
-\N \N
-103 255
-1001 1986:1989
-1002 1989:32767
-3021 1991:1992:32767
-5014 1985:1991
-25699 1989
-2147483647 255:1991:32767:32767
-
--- !select --
-\N \N
-103 255
-1001 1989, 1986
-1002 1989, 32767
-3021 1991, 32767, 1992
-5014 1985, 1991
-25699 1989
-2147483647 255, 1991, 32767, 32767
+12 false, false, false, false, false, false, false, false, true, true,
true, true, true, true, true
--- !select --
-\N \N
-103 255
-1001 1989:1986
-1002 1989:32767
-3021 1991:32767:1992
-5014 1985:1991
-25699 1989
-2147483647 255:1991:32767:32767
+-- !select_10 --
+1 2
--- !select --
-12 false, false, false, false, false, false, false, false, true, true,
true, true, true, true, true
+-- !select_11 --
+1 2
+1 2
diff --git a/regression-test/data/query_p0/group_concat/test_group_concat.out
b/regression-test/data/query_p0/group_concat/test_group_concat.out
index 38f6f95ca3..2b121d104f 100644
--- a/regression-test/data/query_p0/group_concat/test_group_concat.out
+++ b/regression-test/data/query_p0/group_concat/test_group_concat.out
@@ -25,66 +25,16 @@ false
25699 1989
2147483647 255:1991:32767:32767
--- !select --
-\N \N
-103 255
-1001 1989, 1986
-1002 1989, 32767
-3021 1991, 32767, 1992
-5014 1985, 1991
-25699 1989
-2147483647 255, 1991, 32767, 32767
-
--- !select --
-\N \N
-103 255
-1001 1989:1986
-1002 1989:32767
-3021 1991:32767:1992
-5014 1985:1991
-25699 1989
-2147483647 255:1991:32767:32767
-
-- !select --
12 false, false, false, false, false, false, false, false, true, true,
true, true, true, true, true
--- !select --
-\N \N
-103 255
-1001 1986, 1989
-1002 1989, 32767
-3021 1991, 1992, 32767
-5014 1985, 1991
-25699 1989
-2147483647 255, 1991, 32767, 32767
+-- !select_10 --
+1 2
--- !select --
-\N \N
-103 255
-1001 1986:1989
-1002 1989:32767
-3021 1991:1992:32767
-5014 1985:1991
-25699 1989
-2147483647 255:1991:32767:32767
-
--- !select --
-\N \N
-103 255
-1001 1989, 1986
-1002 1989, 32767
-3021 1991, 32767, 1992
-5014 1985, 1991
-25699 1989
-2147483647 255, 1991, 32767, 32767
+-- !select_11 --
+1 2
+1 2
--- !select --
-\N \N
-103 255
-1001 1989:1986
-1002 1989:32767
-3021 1991:32767:1992
-5014 1985:1991
-25699 1989
-2147483647 255:1991:32767:32767
+-- !select_11 --
+1 2
diff --git
a/regression-test/suites/nereids_p0/group_concat/test_group_concat.groovy
b/regression-test/suites/nereids_p0/group_concat/test_group_concat.groovy
index 8d48f86b2c..682cf05d99 100644
--- a/regression-test/suites/nereids_p0/group_concat/test_group_concat.groovy
+++ b/regression-test/suites/nereids_p0/group_concat/test_group_concat.groovy
@@ -34,28 +34,54 @@ suite("test_group_concat") {
qt_select """
SELECT abs(k3), group_concat(cast(abs(k2) as varchar), ":"
order by abs(k2), k1) FROM test_query_db.baseall group by abs(k3) order by
abs(k3)
"""
- qt_select """
- SELECT abs(k3), group_concat(distinct cast(abs(k2) as char)
order by abs(k1), k2) FROM test_query_db.baseall group by abs(k3) order by
abs(k3);
- """
- qt_select """
- SELECT abs(k3), group_concat(distinct cast(abs(k2) as char),
":" order by abs(k1), k2) FROM test_query_db.baseall group by abs(k3) order by
abs(k3);
- """
+ test {
+ sql"""SELECT abs(k3), group_concat(distinct cast(abs(k2) as char)
order by abs(k1), k2) FROM test_query_db.baseall group by abs(k3) order by
abs(k3);"""
+ check{result, exception, startTime, endTime ->
+ assertTrue(exception != null)
+ logger.info(exception.message)
+ }
+ }
- qt_select """
- SELECT abs(k3), group_concat(cast(abs(k2) as varchar) order by
abs(k2), k1) FROM test_query_db.baseall group by abs(k3) order by abs(k3)
- """
+ test {
+ sql"""SELECT abs(k3), group_concat(distinct cast(abs(k2) as char), ":"
order by abs(k1), k2) FROM test_query_db.baseall group by abs(k3) order by
abs(k3);"""
+ check{result, exception, startTime, endTime ->
+ assertTrue(exception != null)
+ logger.info(exception.message)
+ }
+ }
qt_select """
- SELECT abs(k3), group_concat(cast(abs(k2) as varchar), ":"
order by abs(k2), k1) FROM test_query_db.baseall group by abs(k3) order by
abs(k3)
- """
- qt_select """
- SELECT abs(k3), group_concat(distinct cast(abs(k2) as char)
order by abs(k1), k2) FROM test_query_db.baseall group by abs(k3) order by
abs(k3);
+ SELECT count(distinct k7), group_concat(k6 order by k6) FROM
test_query_db.baseall;
"""
- qt_select """
- SELECT abs(k3), group_concat(distinct cast(abs(k2) as char),
":" order by abs(k1), k2) FROM test_query_db.baseall group by abs(k3) order by
abs(k3);
+
+ sql """drop table if exists table_group_concat;"""
+
+ sql """create table table_group_concat ( b1 varchar(10) not null, b2 int
not null, b3 varchar(10) not null )
+ ENGINE=OLAP
+ DISTRIBUTED BY HASH(b3) BUCKETS 4
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "in_memory" = "false",
+ "storage_format" = "V2"
+ );
+ """
+
+ sql """insert into table_group_concat values('1', 1, '2'),('1', 1,
'2'),('1', 2, '2');"""
+
+ qt_select_10 """
+ select
+ group_concat( distinct b1 ), group_concat( distinct b3 )
+ from
+ table_group_concat;
"""
- qt_select """
- SELECT count(distinct k7), group_concat(k6 order by k6) FROM
test_query_db.baseall;
+
+ qt_select_11 """
+ select
+ group_concat( distinct b1 ), group_concat( distinct b3 )
+ from
+ table_group_concat
+ group by
+ b2;
"""
}
diff --git
a/regression-test/suites/query_p0/group_concat/test_group_concat.groovy
b/regression-test/suites/query_p0/group_concat/test_group_concat.groovy
index e423f4f4d1..8b2739af0f 100644
--- a/regression-test/suites/query_p0/group_concat/test_group_concat.groovy
+++ b/regression-test/suites/query_p0/group_concat/test_group_concat.groovy
@@ -31,31 +31,53 @@ suite("test_group_concat") {
qt_select """
SELECT abs(k3), group_concat(cast(abs(k2) as varchar), ":"
order by abs(k2), k1) FROM test_query_db.baseall group by abs(k3) order by
abs(k3)
"""
- qt_select """
- SELECT abs(k3), group_concat(distinct cast(abs(k2) as char)
order by abs(k1), k2) FROM test_query_db.baseall group by abs(k3) order by
abs(k3);
- """
- qt_select """
- SELECT abs(k3), group_concat(distinct cast(abs(k2) as char),
":" order by abs(k1), k2) FROM test_query_db.baseall group by abs(k3) order by
abs(k3);
- """
+ test {
+ sql"""SELECT abs(k3), group_concat(distinct cast(abs(k2) as char)
order by abs(k1), k2) FROM test_query_db.baseall group by abs(k3) order by
abs(k3);"""
+ check{result, exception, startTime, endTime ->
+ assertTrue(exception != null)
+ logger.info(exception.message)
+ }
+ }
+
+ test {
+ sql"""SELECT abs(k3), group_concat(distinct cast(abs(k2) as char), ":"
order by abs(k1), k2) FROM test_query_db.baseall group by abs(k3) order by
abs(k3);"""
+ check{result, exception, startTime, endTime ->
+ assertTrue(exception != null)
+ logger.info(exception.message)
+ }
+ }
+
qt_select """
SELECT count(distinct k7), group_concat(k6 order by k6) FROM
test_query_db.baseall;
"""
- sql "set enable_nereids_planner=true"
- sql "set enable_vectorized_engine=true"
- sql "set enable_fallback_to_original_planner=false"
+ sql """drop table if exists table_group_concat;"""
- qt_select """
- SELECT abs(k3), group_concat(cast(abs(k2) as varchar) order by
abs(k2), k1) FROM test_query_db.baseall group by abs(k3) order by abs(k3)
- """
+ sql """create table table_group_concat ( b1 varchar(10) not null, b2 int
not null, b3 varchar(10) not null )
+ ENGINE=OLAP
+ DISTRIBUTED BY HASH(b3) BUCKETS 4
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "in_memory" = "false",
+ "storage_format" = "V2"
+ );
+ """
- qt_select """
- SELECT abs(k3), group_concat(cast(abs(k2) as varchar), ":"
order by abs(k2), k1) FROM test_query_db.baseall group by abs(k3) order by
abs(k3)
- """
- qt_select """
- SELECT abs(k3), group_concat(distinct cast(abs(k2) as char)
order by abs(k1), k2) FROM test_query_db.baseall group by abs(k3) order by
abs(k3);
+ sql """insert into table_group_concat values('1', 1, '2'),('1', 1,
'2'),('1', 2, '2');"""
+
+ qt_select_10 """
+ select
+ group_concat( distinct b1 ), group_concat( distinct b3 )
+ from
+ table_group_concat;
"""
- qt_select """
- SELECT abs(k3), group_concat(distinct cast(abs(k2) as char),
":" order by abs(k1), k2) FROM test_query_db.baseall group by abs(k3) order by
abs(k3);
+
+ qt_select_11 """
+ select
+ group_concat( distinct b1 ), group_concat( distinct b3 )
+ from
+ table_group_concat
+ group by
+ b2;
"""
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
