This is an automated email from the ASF dual-hosted git repository.
yuanzhou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 98e58e7c9 [GLUTEN-5341][VL] Fix and enable part uts of spark 3.5
(#5379)
98e58e7c9 is described below
commit 98e58e7c96ef8f4c11d2a5a60ab7d83eecc1d280
Author: 高阳阳 <[email protected]>
AuthorDate: Mon Apr 15 11:16:01 2024 +0800
[GLUTEN-5341][VL] Fix and enable part uts of spark 3.5 (#5379)
---
.../org/apache/gluten/expression/ExpressionConverter.scala | 5 +++++
.../org/apache/gluten/utils/velox/VeloxTestSettings.scala | 11 -----------
2 files changed, 5 insertions(+), 11 deletions(-)
diff --git
a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala
b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala
index 66cac9b0d..d70bb8fed 100644
---
a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala
+++
b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala
@@ -431,6 +431,11 @@ object ExpressionConverter extends SQLConfHelper with
Logging {
sha2
)
case size: Size =>
+ if (size.legacySizeOfNull != SQLConf.get.legacySizeOfNull) {
+ throw new GlutenNotSupportException(
+ "The value of legacySizeOfNull field of size is " +
+ "not equals to legacySizeOfNull of SQLConf, this case is not
supported yet")
+ }
BackendsApiManager.getSparkPlanExecApiInstance.genSizeExpressionTransformer(
substraitExprName,
replaceWithExpressionTransformerInternal(size.child, attributeSeq,
expressionsMap),
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 5396a7c5e..a82e73ad4 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -966,10 +966,6 @@ class VeloxTestSettings extends BackendTestSettings {
enableSuite[GlutenDataFrameAsOfJoinSuite]
enableSuite[GlutenDataFrameComplexTypeSuite]
enableSuite[GlutenDataFrameFunctionsSuite]
- // Disabled as failed in Spark3.5.
- .exclude("array_size function")
- .exclude("cardinality function")
- .exclude("SPARK-40470: array_zip should return field names in
GetArrayStructFields")
// blocked by Velox-5768
.exclude("aggregate function - array for primitive type containing null")
.exclude("aggregate function - array for non-primitive type")
@@ -1042,8 +1038,6 @@ class VeloxTestSettings extends BackendTestSettings {
enableSuite[GlutenDataFrameWriterV2Suite]
enableSuite[GlutenDatasetAggregatorSuite]
enableSuite[GlutenDatasetCacheSuite]
- // Disable for Spark3.5.
- // .exclude("SPARK-27739 Save stats from optimized plan")
enableSuite[GlutenDatasetOptimizationSuite]
enableSuite[GlutenDatasetPrimitiveSuite]
enableSuite[GlutenDatasetSerializerRegistratorSuite]
@@ -1162,14 +1156,9 @@ class VeloxTestSettings extends BackendTestSettings {
"SPARK-27619: When spark.sql.legacy.allowHashOnMapType is true, hash can
be used on Maptype")
enableSuite[GlutenSQLQueryTestSuite]
enableSuite[GlutenStatisticsCollectionSuite]
- // Disable for Spark3.5.
- .exclude("statistics collection of a table with zero column")
// The output byte size of Velox is different
.exclude("SPARK-33687: analyze all tables in a specific database")
enableSuite[GlutenSubquerySuite]
- // Disable for Spark3.5.
- .exclude("SPARK-40615: Check unsupported data type when decorrelating
subqueries")
- .exclude("SPARK-40618: Regression test for merging subquery bug with
nested subqueries")
.excludeByPrefix(
"SPARK-26893" // Rewrite this test because it checks Spark's physical
operators.
)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
