This is an automated email from the ASF dual-hosted git repository.
zhli pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new b5657d319 [VL] Enable collect_set, min, max for complex types (#6629)
b5657d319 is described below
commit b5657d3190db75f07f4480b82bc7bc325480cdf8
Author: Zhen Li <[email protected]>
AuthorDate: Tue Jul 30 13:46:06 2024 +0800
[VL] Enable collect_set, min, max for complex types (#6629)
[VL] Enable collect_set, min, max for complex types.
---
.../gluten/execution/VeloxAggregateFunctionsSuite.scala | 12 ++++--------
cpp/velox/substrait/SubstraitParser.cc | 1 -
cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc | 9 ---------
3 files changed, 4 insertions(+), 18 deletions(-)
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxAggregateFunctionsSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxAggregateFunctionsSuite.scala
index 270334011..fb1f05426 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxAggregateFunctionsSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxAggregateFunctionsSuite.scala
@@ -22,7 +22,6 @@ import
org.apache.gluten.extension.columnar.validator.FallbackInjects
import org.apache.spark.SparkConf
import org.apache.spark.sql.catalyst.expressions.aggregate.{Final, Partial}
import org.apache.spark.sql.execution.aggregate.BaseAggregateExec
-import org.apache.spark.sql.functions._
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types._
@@ -1141,13 +1140,10 @@ abstract class VeloxAggregateFunctionsSuite extends
VeloxWholeStageTransformerSu
StructField("lastUpdated", LongType, true),
StructField("version", LongType, true))),
true)))
- val df = spark.read.schema(jsonSchema).json(Seq(jsonStr).toDS)
- df.select(collect_set(col("txn"))).collect
-
- df.select(min(col("txn"))).collect
-
- df.select(max(col("txn"))).collect
-
+
spark.read.schema(jsonSchema).json(Seq(jsonStr).toDS).createOrReplaceTempView("t1")
+ runQueryAndCompare("select collect_set(txn), min(txn), max(txn) from t1") {
+ checkGlutenOperatorMatch[HashAggregateExecTransformer]
+ }
}
test("drop redundant partial sort which has pre-project when offload
sortAgg") {
diff --git a/cpp/velox/substrait/SubstraitParser.cc
b/cpp/velox/substrait/SubstraitParser.cc
index b842914ca..6eb62f854 100644
--- a/cpp/velox/substrait/SubstraitParser.cc
+++ b/cpp/velox/substrait/SubstraitParser.cc
@@ -395,7 +395,6 @@ std::unordered_map<std::string, std::string>
SubstraitParser::substraitVeloxFunc
{"xxhash64", "xxhash64_with_seed"},
{"modulus", "remainder"},
{"date_format", "format_datetime"},
- {"collect_set", "set_agg"},
{"negative", "unaryminus"},
{"get_array_item", "get"}};
diff --git a/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc
b/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc
index c229ca84f..c18d26598 100644
--- a/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc
+++ b/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc
@@ -1048,15 +1048,6 @@ bool
SubstraitToVeloxPlanValidator::validateAggRelFunctionType(const ::substrait
LOG_VALIDATION_MSG("Validation failed for function " + funcName + "
resolve type in AggregateRel.");
return false;
}
- static const std::unordered_set<std::string>
notSupportComplexTypeAggFuncs = {"set_agg", "min", "max"};
- if (notSupportComplexTypeAggFuncs.find(baseFuncName) !=
notSupportComplexTypeAggFuncs.end() &&
- exec::isRawInput(funcStep)) {
- auto type = binder.tryResolveType(signature->argumentTypes()[0]);
- if (type->isArray() || type->isMap() || type->isRow()) {
- LOG_VALIDATION_MSG("Validation failed for function " +
baseFuncName + " complex type is not supported.");
- return false;
- }
- }
resolved = true;
break;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
