This is an automated email from the ASF dual-hosted git repository.
chengchengjin pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 1a6c689181 [GLUTEN-10963][VL] Fix the validation for merge_extract
companion function (#10976)
1a6c689181 is described below
commit 1a6c68918176b97cd4467e7bdd93c6aa7a21d0aa
Author: Rui Mo <[email protected]>
AuthorDate: Thu Oct 30 03:34:52 2025 +0800
[GLUTEN-10963][VL] Fix the validation for merge_extract companion function
(#10976)
---
.../execution/VeloxAggregateFunctionsSuite.scala | 2 +-
cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc | 18 ++++++++++++++++--
.../apache/gluten/utils/velox/VeloxTestSettings.scala | 2 --
.../apache/gluten/utils/velox/VeloxTestSettings.scala | 2 --
.../apache/gluten/utils/velox/VeloxTestSettings.scala | 2 --
.../apache/gluten/utils/velox/VeloxTestSettings.scala | 2 --
6 files changed, 17 insertions(+), 11 deletions(-)
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxAggregateFunctionsSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxAggregateFunctionsSuite.scala
index 5719931569..1ad6677172 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxAggregateFunctionsSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxAggregateFunctionsSuite.scala
@@ -116,7 +116,7 @@ abstract class VeloxAggregateFunctionsSuite extends
VeloxWholeStageTransformerSu
getExecutedPlan(df).count(
plan => {
plan.isInstanceOf[HashAggregateExecTransformer]
- }) == 3)
+ }) == 4)
}
}
}
diff --git a/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc
b/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc
index 27a81b1fe4..4b39af51c6 100644
--- a/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc
+++ b/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc
@@ -1167,8 +1167,22 @@ bool
SubstraitToVeloxPlanValidator::validateAggRelFunctionType(const ::substrait
for (const auto& signature : signaturesOpt.value()) {
exec::SignatureBinder binder(*signature, types);
if (binder.tryBind()) {
- auto resolveType = binder.tryResolveType(
- exec::isPartialOutput(funcStep) ? signature->intermediateType() :
signature->returnType());
+ TypePtr resolveType = nullptr;
+ try {
+ resolveType = binder.tryResolveType(
+ exec::isPartialOutput(funcStep) ? signature->intermediateType()
: signature->returnType());
+ } catch (const VeloxException& e) {
+ if (!exec::isPartialOutput(funcStep) &&
funcName.find("merge_extract") != std::string::npos) {
+ // For the merge_extract companion function, result
+ // types may not always be inferable from the intermediate types.
As a
+ // result, an exception might be thrown during the type resolution
process. More
+ // details can be found in
+ //
https://github.com/facebookincubator/velox/pull/11999#issuecomment-3274577979
+ // and https://github.com/facebookincubator/velox/issues/12830.
+ return true;
+ }
+ }
+
if (resolveType == nullptr) {
LOG_VALIDATION_MSG("Validation failed for function " + funcName + "
resolve type in AggregateRel.");
return false;
diff --git
a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index e1db913720..584905e493 100644
---
a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -139,8 +139,6 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude("SPARK-22271: mean overflows and returns null for some decimal
variables")
// Rewrite this test since it checks the physical operator which is
changed in Gluten
.exclude("SPARK-27439: Explain result should match collected result after
view change")
- // https://github.com/apache/incubator-gluten/issues/10963
- .exclude("SPARK-35955: Aggregate avg should not return wrong results for
decimal overflow")
enableSuite[GlutenDataFrameNaFunctionsSuite]
.exclude(
diff --git
a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index cf147e46da..e036f539e8 100644
---
a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -721,8 +721,6 @@ class VeloxTestSettings extends BackendTestSettings {
.excludeGlutenTest("describe")
// Rewrite this test since it checks the physical operator which is
changed in Gluten
.exclude("SPARK-27439: Explain result should match collected result after
view change")
- // https://github.com/apache/incubator-gluten/issues/10963
- .exclude("SPARK-35955: Aggregate avg should not return wrong results for
decimal overflow")
enableSuite[GlutenDataFrameTimeWindowingSuite]
enableSuite[GlutenDataFrameTungstenSuite]
enableSuite[GlutenDataFrameWindowFunctionsSuite]
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index cfed0f90c9..d351e707ca 100644
---
a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -770,8 +770,6 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude("SPARK-41048: Improve output partitioning and ordering with AQE
cache")
// Rewrite this test since it checks the physical operator which is
changed in Gluten
.exclude("SPARK-27439: Explain result should match collected result after
view change")
- // https://github.com/apache/incubator-gluten/issues/10963
- .exclude("SPARK-35955: Aggregate avg should not return wrong results for
decimal overflow")
enableSuite[GlutenDataFrameTimeWindowingSuite]
enableSuite[GlutenDataFrameTungstenSuite]
enableSuite[GlutenDataFrameWindowFunctionsSuite]
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 2b5c0ab78c..d21e39fafd 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -790,8 +790,6 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude("SPARK-41048: Improve output partitioning and ordering with AQE
cache")
// Rewrite this test since it checks the physical operator which is
changed in Gluten
.exclude("SPARK-27439: Explain result should match collected result after
view change")
- // https://github.com/apache/incubator-gluten/issues/10963
- .exclude("SPARK-35955: Aggregate avg should not return wrong results for
decimal overflow")
enableSuite[GlutenDataFrameTimeWindowingSuite]
enableSuite[GlutenDataFrameTungstenSuite]
enableSuite[GlutenDataFrameWindowFunctionsSuite]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
