This is an automated email from the ASF dual-hosted git repository.
zhouyuan pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new e5c5dfec81 [GLUTEN-12837][TEST] Migrate Spark 3.3-only test coverage
to Spark 3.4+ (#12840)
e5c5dfec81 is described below
commit e5c5dfec812555e697442137fcddc1ae1f571ab6
Author: YangJie <[email protected]>
AuthorDate: Mon Aug 24 16:21:55 2026 +0800
[GLUTEN-12837][TEST] Migrate Spark 3.3-only test coverage to Spark 3.4+
(#12840)
---
.../utils/clickhouse/ClickHouseTestSettings.scala | 4 --
.../gluten/utils/velox/VeloxTestSettings.scala | 1 -
.../gluten/utils/velox/VeloxTestSettings.scala | 4 ++
.../spark/sql/GlutenDataFrameFunctionsSuite.scala | 82 ++++++++++++++++++++++
.../spark/sql/GlutenDateFunctionsSuite.scala | 18 +++++
.../sql/GlutenDynamicPartitionPruningSuite.scala | 35 +++++++++
.../expressions/GlutenLiteralExpressionSuite.scala | 38 +++++++++-
.../expressions/GlutenMathExpressionsSuite.scala | 20 ++++++
.../ClickHouseAdaptiveQueryExecSuite.scala | 55 +++++++++++++++
.../hive/execution/GlutenHiveSQLQuerySuite.scala | 20 ++++++
.../gluten/utils/velox/VeloxTestSettings.scala | 4 ++
.../spark/sql/GlutenDataFrameFunctionsSuite.scala | 82 ++++++++++++++++++++++
.../spark/sql/GlutenDateFunctionsSuite.scala | 18 +++++
.../sql/GlutenDynamicPartitionPruningSuite.scala | 35 +++++++++
.../expressions/GlutenLiteralExpressionSuite.scala | 38 +++++++++-
.../expressions/GlutenMathExpressionsSuite.scala | 20 ++++++
.../ClickHouseAdaptiveQueryExecSuite.scala | 55 +++++++++++++++
.../hive/execution/GlutenHiveSQLQuerySuite.scala | 20 ++++++
.../gluten/utils/velox/VeloxTestSettings.scala | 4 ++
.../spark/sql/GlutenDataFrameFunctionsSuite.scala | 81 +++++++++++++++++++++
.../spark/sql/GlutenDateFunctionsSuite.scala | 18 +++++
.../sql/GlutenDynamicPartitionPruningSuite.scala | 35 +++++++++
.../expressions/GlutenLiteralExpressionSuite.scala | 38 +++++++++-
.../expressions/GlutenMathExpressionsSuite.scala | 20 ++++++
.../ClickHouseAdaptiveQueryExecSuite.scala | 55 +++++++++++++++
.../hive/execution/GlutenHiveSQLQuerySuite.scala | 20 ++++++
.../gluten/utils/velox/VeloxTestSettings.scala | 4 ++
.../spark/sql/GlutenDataFrameFunctionsSuite.scala | 81 +++++++++++++++++++++
.../spark/sql/GlutenDateFunctionsSuite.scala | 18 +++++
.../sql/GlutenDynamicPartitionPruningSuite.scala | 35 +++++++++
.../expressions/GlutenLiteralExpressionSuite.scala | 38 +++++++++-
.../expressions/GlutenMathExpressionsSuite.scala | 20 ++++++
.../ClickHouseAdaptiveQueryExecSuite.scala | 55 +++++++++++++++
.../hive/execution/GlutenHiveSQLQuerySuite.scala | 20 ++++++
.../org/apache/spark/sql/GlutenImplicitsTest.scala | 74 ++++++++++++++-----
35 files changed, 1140 insertions(+), 25 deletions(-)
diff --git
a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
index b0b767cd91..d9dcdde5b7 100644
---
a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
+++
b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
@@ -1875,10 +1875,6 @@ class ClickHouseTestSettings extends BackendTestSettings
{
.exclude(
"SELECT structFieldSimple.key, arrayFieldSimple[1] FROM tableWithSchema
a where int_Field=1")
.exclude("SELECT structFieldComplex.Value.`value_(2)` FROM
tableWithSchema")
- enableSuite[GlutenImplicitsTest]
- .excludeGlutenTest("fallbackSummary with shuffle")
- .excludeGlutenTest("fallbackSummary with cache")
- .excludeGlutenTest("fallbackSummary with cached data and shuffle")
enableSuite[GlutenSparkSessionExtensionSuite]
.includeGlutenTest("customColumnarOp")
enableSuite[GlutenHiveSQLQueryCHSuite]
diff --git
a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 0ba7d7796c..fb42198e2d 100644
---
a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -907,7 +907,6 @@ class VeloxTestSettings extends BackendTestSettings {
enableSuite[GlutenHashUDAQueryWithControlledFallbackSuite]
enableSuite[GlutenSQLQuerySuiteAE]
enableSuite[GlutenWindowQuerySuite]
- enableSuite[GlutenImplicitsTest]
enableSuite[GlutenCollapseProjectExecTransformerSuite]
enableSuite[GlutenSparkSessionExtensionSuite]
.includeGlutenTest("customColumnarOp")
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index e7cda00f71..e0d2e3ee5e 100644
---
a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -826,10 +826,14 @@ class VeloxTestSettings extends BackendTestSettings {
enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOn]
enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOnDisableScan]
enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOffDisableScan]
+ enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOffWSCGOnDisableProject]
+ enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOffWSCGOffDisableProject]
enableSuite[GlutenDynamicPartitionPruningV2SuiteAEOff]
enableSuite[GlutenDynamicPartitionPruningV2SuiteAEOn]
enableSuite[GlutenDynamicPartitionPruningV2SuiteAEOnDisableScan]
enableSuite[GlutenDynamicPartitionPruningV2SuiteAEOffDisableScan]
+ enableSuite[GlutenDynamicPartitionPruningV2SuiteAEOffWSCGOnDisableProject]
+ enableSuite[GlutenDynamicPartitionPruningV2SuiteAEOffWSCGOffDisableProject]
enableSuite[GlutenExpressionsSchemaSuite]
enableSuite[GlutenExtraStrategiesSuite]
enableSuite[GlutenFileBasedDataSourceSuite]
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDataFrameFunctionsSuite.scala
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDataFrameFunctionsSuite.scala
index 1725b98e75..ebe1fc50fc 100644
---
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDataFrameFunctionsSuite.scala
+++
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDataFrameFunctionsSuite.scala
@@ -111,4 +111,86 @@ class GlutenDataFrameFunctionsSuite extends
DataFrameFunctionsSuite with GlutenS
checkAnswer(df1.selectExpr("array_insert(a, -6, c)"), Seq(Row(Seq(3,
null, 3, 2, 5, 1, 2))))
}
}
+
+ testGluten("flatten function") {
+ // Test cases with a primitive type
+ val intDF = Seq(
+ (Seq(Seq(1, 2, 3), Seq(4, 5), Seq(6))),
+ (Seq(Seq(1, 2))),
+ (Seq(Seq(1), Seq.empty)),
+ (Seq(Seq.empty, Seq(1)))
+ ).toDF("i")
+
+ val intDFResult = Seq(Row(Seq(1, 2, 3, 4, 5, 6)), Row(Seq(1, 2)),
Row(Seq(1)), Row(Seq(1)))
+
+ def testInt(): Unit = {
+ checkAnswer(intDF.select(flatten($"i")), intDFResult)
+ checkAnswer(intDF.selectExpr("flatten(i)"), intDFResult)
+ }
+
+ // Test with local relation, the Project will be evaluated without codegen
+ testInt()
+ // Test with cached relation, the Project will be evaluated with codegen
+ intDF.cache()
+ testInt()
+
+ // Test cases with non-primitive types
+ val strDF = Seq(
+ (Seq(Seq("a", "b"), Seq("c"), Seq("d", "e", "f"))),
+ (Seq(Seq("a", "b"))),
+ (Seq(Seq("a", null), Seq(null, "b"), Seq(null, null))),
+ (Seq(Seq("a"), Seq.empty)),
+ (Seq(Seq.empty, Seq("a")))
+ ).toDF("s")
+
+ val strDFResult = Seq(
+ Row(Seq("a", "b", "c", "d", "e", "f")),
+ Row(Seq("a", "b")),
+ Row(Seq("a", null, null, "b", null, null)),
+ Row(Seq("a")),
+ Row(Seq("a")))
+
+ def testString(): Unit = {
+ checkAnswer(strDF.select(flatten($"s")), strDFResult)
+ checkAnswer(strDF.selectExpr("flatten(s)"), strDFResult)
+ }
+
+ // Test with local relation, the Project will be evaluated without codegen
+ testString()
+ // Test with cached relation, the Project will be evaluated with codegen
+ strDF.cache()
+ testString()
+
+ val arrDF = Seq((1, "a", Seq(1, 2, 3))).toDF("i", "s", "arr")
+
+ def testArray(): Unit = {
+ checkAnswer(
+ arrDF.selectExpr("flatten(array(arr, array(null, 5), array(6,
null)))"),
+ Seq(Row(Seq(1, 2, 3, null, 5, 6, null))))
+ checkAnswer(
+ arrDF.selectExpr("flatten(array(array(arr, arr), array(arr)))"),
+ Seq(Row(Seq(Seq(1, 2, 3), Seq(1, 2, 3), Seq(1, 2, 3)))))
+ }
+
+ // Test with local relation, the Project will be evaluated without codegen
+ testArray()
+ // Test with cached relation, the Project will be evaluated with codegen
+ arrDF.cache()
+ testArray()
+
+ // Error test cases
+ val oneRowDF = Seq((1, "a", Seq(1, 2, 3))).toDF("i", "s", "arr")
+ intercept[AnalysisException] {
+ oneRowDF.select(flatten($"arr"))
+ }
+ intercept[AnalysisException] {
+ oneRowDF.select(flatten($"i"))
+ }
+ intercept[AnalysisException] {
+ oneRowDF.select(flatten($"s"))
+ }
+ intercept[AnalysisException] {
+ oneRowDF.selectExpr("flatten(null)")
+ }
+ }
}
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
index d66f26d7c0..37c3afef66 100644
---
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
+++
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
@@ -282,4 +282,22 @@ class GlutenDateFunctionsSuite extends DateFunctionsSuite
with GlutenSQLTestsTra
val df1 = Seq(x1, x2).toDF("x")
checkAnswer(df1.select(to_date(col("x"))), Row(Date.valueOf("2016-02-29"))
:: Row(null) :: Nil)
}
+
+ testGluten("date_from_unix_date") {
+ // -100000 and 200000 are outside ClickHouse's native Date32 range
+ // [1900-01-01, 2299-12-31]. They guard against implementations that clamp
to that
+ // range (e.g. mapping to CH toDate32), which would silently diverge from
Spark.
+ val df = Seq(Some(0), Some(1000), Some(-100000), Some(200000),
None).toDF("unix_date")
+ val expected = Seq(
+ Row(Date.valueOf("1970-01-01")),
+ Row(Date.valueOf("1972-09-27")),
+ Row(Date.valueOf("1696-03-17")),
+ Row(Date.valueOf("2517-08-01")),
+ Row(null))
+
+ // Go through expr() so the same case works on every supported version:
the Scala
+ // functions API does not expose date_from_unix_date before Spark 3.5.
+ checkAnswer(df.select(expr("date_from_unix_date(unix_date)")), expected)
+ checkAnswer(df.selectExpr("date_from_unix_date(unix_date)"), expected)
+ }
}
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDynamicPartitionPruningSuite.scala
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDynamicPartitionPruningSuite.scala
index 740d52d0b1..3b2be3c72e 100644
---
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDynamicPartitionPruningSuite.scala
+++
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDynamicPartitionPruningSuite.scala
@@ -828,3 +828,38 @@ class GlutenDynamicPartitionPruningV2SuiteAEOffDisableScan
super.sparkConf.set(GlutenConfig.COLUMNAR_BATCHSCAN_ENABLED.key, "false")
}
}
+
+// Test DPP with project disabled by user for some reason, which can also mock
the situation
+// that project is not transformable.
+class GlutenDynamicPartitionPruningV1SuiteAEOffWSCGOnDisableProject
+ extends GlutenDynamicPartitionPruningV2SuiteAEOff {
+ override def sparkConf: SparkConf = {
+ super.sparkConf.set(GlutenConfig.COLUMNAR_PROJECT_ENABLED.key, "false")
+ }
+}
+
+// Same as above except whole-stage codegen is off.
+class GlutenDynamicPartitionPruningV1SuiteAEOffWSCGOffDisableProject
+ extends GlutenDynamicPartitionPruningV2SuiteAEOff {
+ override def sparkConf: SparkConf = {
+ super.sparkConf
+ .set(GlutenConfig.COLUMNAR_PROJECT_ENABLED.key, "false")
+ .set(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key, "false")
+ }
+}
+
+class GlutenDynamicPartitionPruningV2SuiteAEOffWSCGOnDisableProject
+ extends GlutenDynamicPartitionPruningV2SuiteAEOff {
+ override def sparkConf: SparkConf = {
+ super.sparkConf.set(GlutenConfig.COLUMNAR_PROJECT_ENABLED.key, "false")
+ }
+}
+
+class GlutenDynamicPartitionPruningV2SuiteAEOffWSCGOffDisableProject
+ extends GlutenDynamicPartitionPruningV2SuiteAEOff {
+ override def sparkConf: SparkConf = {
+ super.sparkConf
+ .set(GlutenConfig.COLUMNAR_PROJECT_ENABLED.key, "false")
+ .set(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key, "false")
+ }
+}
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenLiteralExpressionSuite.scala
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenLiteralExpressionSuite.scala
index 556d185af0..f81ef0b6ff 100644
---
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenLiteralExpressionSuite.scala
+++
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenLiteralExpressionSuite.scala
@@ -17,5 +17,41 @@
package org.apache.spark.sql.catalyst.expressions
import org.apache.spark.sql.GlutenTestsTrait
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.catalyst.util.DateTimeUtils
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.types._
+import org.apache.spark.unsafe.types.CalendarInterval
-class GlutenLiteralExpressionSuite extends LiteralExpressionSuite with
GlutenTestsTrait {}
+import java.nio.charset.StandardCharsets
+import java.time.{Instant, LocalDate}
+
+class GlutenLiteralExpressionSuite extends LiteralExpressionSuite with
GlutenTestsTrait {
+ testGluten("default") {
+ checkEvaluation(Literal.default(BooleanType), false)
+ checkEvaluation(Literal.default(ByteType), 0.toByte)
+ checkEvaluation(Literal.default(ShortType), 0.toShort)
+ checkEvaluation(Literal.default(IntegerType), 0)
+ checkEvaluation(Literal.default(LongType), 0L)
+ checkEvaluation(Literal.default(FloatType), 0.0f)
+ checkEvaluation(Literal.default(DoubleType), 0.0)
+ checkEvaluation(Literal.default(StringType), "")
+ checkEvaluation(Literal.default(BinaryType),
"".getBytes(StandardCharsets.UTF_8))
+ checkEvaluation(Literal.default(DecimalType.USER_DEFAULT), Decimal(0))
+ checkEvaluation(Literal.default(DecimalType.SYSTEM_DEFAULT), Decimal(0))
+ withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> "false") {
+ checkEvaluation(Literal.default(DateType), DateTimeUtils.toJavaDate(0))
+ checkEvaluation(Literal.default(TimestampType),
DateTimeUtils.toJavaTimestamp(0L))
+ }
+ withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> "true") {
+ checkEvaluation(Literal.default(DateType), LocalDate.ofEpochDay(0))
+ checkEvaluation(Literal.default(TimestampType), Instant.ofEpochSecond(0))
+ }
+ checkEvaluation(Literal.default(CalendarIntervalType), new
CalendarInterval(0, 0, 0L))
+ checkEvaluation(Literal.default(YearMonthIntervalType()), 0)
+ checkEvaluation(Literal.default(DayTimeIntervalType()), 0L)
+ checkEvaluation(Literal.default(ArrayType(StringType)), Array())
+ checkEvaluation(Literal.default(MapType(IntegerType, StringType)), Map())
+ checkEvaluation(Literal.default(StructType(StructField("a", StringType) ::
Nil)), Row(""))
+ }
+}
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenMathExpressionsSuite.scala
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenMathExpressionsSuite.scala
index c755a475ed..b793896de2 100644
---
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenMathExpressionsSuite.scala
+++
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenMathExpressionsSuite.scala
@@ -20,6 +20,8 @@ import org.apache.spark.sql.GlutenTestsTrait
import org.apache.spark.sql.catalyst.dsl.expressions._
import org.apache.spark.sql.types._
+import java.nio.charset.StandardCharsets
+
class GlutenMathExpressionsSuite extends MathExpressionsSuite with
GlutenTestsTrait {
testGluten("round/bround/floor/ceil") {
val scales = -6 to 6
@@ -274,4 +276,22 @@ class GlutenMathExpressionsSuite extends
MathExpressionsSuite with GlutenTestsTr
checkEvaluation(checkDataTypeAndCast(RoundCeil(Literal(3.1411),
Literal(-3))), Decimal(1000))
checkEvaluation(checkDataTypeAndCast(RoundCeil(Literal(135.135),
Literal(-2))), Decimal(200))
}
+
+ testGluten("unhex") {
+ checkEvaluation(Unhex(Literal.create(null, StringType)), null)
+ checkEvaluation(Unhex(Literal("737472696E67")),
"string".getBytes(StandardCharsets.UTF_8))
+ checkEvaluation(Unhex(Literal("")), new Array[Byte](0))
+ checkEvaluation(Unhex(Literal("F")), Array[Byte](15))
+ checkEvaluation(Unhex(Literal("ff")), Array[Byte](-1))
+
+// checkEvaluation(Unhex(Literal("GG")), null)
+ checkEvaluation(Unhex(Literal("123")), Array[Byte](1, 35))
+ checkEvaluation(Unhex(Literal("12345")), Array[Byte](1, 35, 69))
+ // scalastyle:off
+ // Turn off scala style for non-ascii chars
+ checkEvaluation(Unhex(Literal("E4B889E9878DE79A84")),
"三重的".getBytes(StandardCharsets.UTF_8))
+// checkEvaluation(Unhex(Literal("三重的")), null)
+ // scalastyle:on
+ checkConsistencyBetweenInterpretedAndCodegen((e: Expression) => Unhex(e),
StringType)
+ }
}
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/adaptive/clickhouse/ClickHouseAdaptiveQueryExecSuite.scala
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/adaptive/clickhouse/ClickHouseAdaptiveQueryExecSuite.scala
index 3c4e7bc5e1..35729ac030 100644
---
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/adaptive/clickhouse/ClickHouseAdaptiveQueryExecSuite.scala
+++
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/adaptive/clickhouse/ClickHouseAdaptiveQueryExecSuite.scala
@@ -1549,4 +1549,59 @@ class ClickHouseAdaptiveQueryExecSuite extends
AdaptiveQueryExecSuite with Glute
}
}
}
+
+ // TODO: re-enable this test after #12845.
findTopLevelSortMergeJoinTransform matches the Velox
+ // SortMergeJoinExecTransformer, which ClickHouse never produces, so it
collects nothing here and
+ // the case fails on CH-3.5. The expected counts are also still unverified
on ClickHouse.
+ ignoreGluten("SPARK-37652: optimize skewed join through union") {
+ withSQLConf(
+ SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
+ SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1",
+ SQLConf.SKEW_JOIN_SKEWED_PARTITION_THRESHOLD.key -> "100",
+ SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key -> "100"
+ ) {
+ withTempView("skewData1", "skewData2") {
+ spark
+ .range(0, 1000, 1, 10)
+ .selectExpr("id % 3 as key1", "id as value1")
+ .createOrReplaceTempView("skewData1")
+ spark
+ .range(0, 1000, 1, 10)
+ .selectExpr("id % 1 as key2", "id as value2")
+ .createOrReplaceTempView("skewData2")
+
+ def checkSkewJoin(query: String, joinNums: Int, optimizeSkewJoinNums:
Int): Unit = {
+ val (_, innerAdaptivePlan) = runAdaptiveAndVerifyResult(query)
+ val joins = findTopLevelSortMergeJoinTransform(innerAdaptivePlan)
+ val optimizeSkewJoins = joins.filter(_.isSkewJoin)
+ assert(joins.size == joinNums && optimizeSkewJoins.size ==
optimizeSkewJoinNums)
+ }
+
+ // skewJoin union skewJoin
+ checkSkewJoin(
+ "SELECT key1 FROM skewData1 JOIN skewData2 ON key1 = key2 " +
+ "UNION ALL SELECT key2 FROM skewData1 JOIN skewData2 ON key1 =
key2",
+ 2,
+ 2)
+
+ // skewJoin union aggregate
+ checkSkewJoin(
+ "SELECT key1 FROM skewData1 JOIN skewData2 ON key1 = key2 " +
+ "UNION ALL SELECT key2 FROM skewData2 GROUP BY key2",
+ 1,
+ 1)
+
+ // skewJoin1 union (skewJoin2 join aggregate)
+ // skewJoin2 will lead to extra shuffles, but skew1 cannot be optimized
+ checkSkewJoin(
+ "SELECT key1 FROM skewData1 JOIN skewData2 ON key1 = key2 UNION ALL
" +
+ "SELECT key1 from (SELECT key1 FROM skewData1 JOIN skewData2 ON
key1 = key2) tmp1 " +
+ "JOIN (SELECT key2 FROM skewData2 GROUP BY key2) tmp2 ON key1 =
key2",
+ 3,
+ 0
+ )
+ }
+ }
+ }
+
}
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQuerySuite.scala
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQuerySuite.scala
index 010e55c346..8be19c216a 100644
---
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQuerySuite.scala
+++
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQuerySuite.scala
@@ -171,4 +171,24 @@ class GlutenHiveSQLQuerySuite extends
GlutenHiveSQLQuerySuiteBase {
}
}
}
+
+ testGluten("avoid unnecessary filter binding for subfield during scan") {
+ withSQLConf(
+ "spark.sql.hive.convertMetastoreParquet" -> "false") {
+ sql("DROP TABLE IF EXISTS test_subfield")
+ sql(
+ "CREATE TABLE test_subfield (name STRING, favorite_color STRING," +
+ " label STRUCT<label_1:STRING, label_2:STRING>) USING hive
OPTIONS(fileFormat 'parquet')")
+ sql(
+ "INSERT INTO test_subfield VALUES('test_1', 'red',
named_struct('label_1', 'label-a'," +
+ "'label_2', 'label-b'))")
+ val df = spark.sql("select * from test_subfield where name='test_1'")
+ checkAnswer(df, Seq(Row("test_1", "red", Row("label-a", "label-b"))))
+ checkOperatorMatch[HiveTableScanExecTransformer](df)
+ }
+ spark.sessionState.catalog.dropTable(
+ TableIdentifier("test_subfield"),
+ ignoreIfNotExists = true,
+ purge = false)
+ }
}
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 7fa7fa8dd6..c753c17ed0 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -791,10 +791,14 @@ class VeloxTestSettings extends BackendTestSettings {
enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOn]
enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOnDisableScan]
enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOffDisableScan]
+ enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOffWSCGOnDisableProject]
+ enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOffWSCGOffDisableProject]
enableSuite[GlutenDynamicPartitionPruningV2SuiteAEOff]
enableSuite[GlutenDynamicPartitionPruningV2SuiteAEOn]
enableSuite[GlutenDynamicPartitionPruningV2SuiteAEOnDisableScan]
enableSuite[GlutenDynamicPartitionPruningV2SuiteAEOffDisableScan]
+ enableSuite[GlutenDynamicPartitionPruningV2SuiteAEOffWSCGOnDisableProject]
+ enableSuite[GlutenDynamicPartitionPruningV2SuiteAEOffWSCGOffDisableProject]
enableSuite[GlutenExpressionsSchemaSuite]
enableSuite[GlutenExtraStrategiesSuite]
enableSuite[GlutenFileBasedDataSourceSuite]
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenDataFrameFunctionsSuite.scala
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenDataFrameFunctionsSuite.scala
index 2b0b40790a..e64f760ab5 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenDataFrameFunctionsSuite.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenDataFrameFunctionsSuite.scala
@@ -49,4 +49,86 @@ class GlutenDataFrameFunctionsSuite extends
DataFrameFunctionsSuite with GlutenS
false
)
}
+
+ testGluten("flatten function") {
+ // Test cases with a primitive type
+ val intDF = Seq(
+ (Seq(Seq(1, 2, 3), Seq(4, 5), Seq(6))),
+ (Seq(Seq(1, 2))),
+ (Seq(Seq(1), Seq.empty)),
+ (Seq(Seq.empty, Seq(1)))
+ ).toDF("i")
+
+ val intDFResult = Seq(Row(Seq(1, 2, 3, 4, 5, 6)), Row(Seq(1, 2)),
Row(Seq(1)), Row(Seq(1)))
+
+ def testInt(): Unit = {
+ checkAnswer(intDF.select(flatten($"i")), intDFResult)
+ checkAnswer(intDF.selectExpr("flatten(i)"), intDFResult)
+ }
+
+ // Test with local relation, the Project will be evaluated without codegen
+ testInt()
+ // Test with cached relation, the Project will be evaluated with codegen
+ intDF.cache()
+ testInt()
+
+ // Test cases with non-primitive types
+ val strDF = Seq(
+ (Seq(Seq("a", "b"), Seq("c"), Seq("d", "e", "f"))),
+ (Seq(Seq("a", "b"))),
+ (Seq(Seq("a", null), Seq(null, "b"), Seq(null, null))),
+ (Seq(Seq("a"), Seq.empty)),
+ (Seq(Seq.empty, Seq("a")))
+ ).toDF("s")
+
+ val strDFResult = Seq(
+ Row(Seq("a", "b", "c", "d", "e", "f")),
+ Row(Seq("a", "b")),
+ Row(Seq("a", null, null, "b", null, null)),
+ Row(Seq("a")),
+ Row(Seq("a")))
+
+ def testString(): Unit = {
+ checkAnswer(strDF.select(flatten($"s")), strDFResult)
+ checkAnswer(strDF.selectExpr("flatten(s)"), strDFResult)
+ }
+
+ // Test with local relation, the Project will be evaluated without codegen
+ testString()
+ // Test with cached relation, the Project will be evaluated with codegen
+ strDF.cache()
+ testString()
+
+ val arrDF = Seq((1, "a", Seq(1, 2, 3))).toDF("i", "s", "arr")
+
+ def testArray(): Unit = {
+ checkAnswer(
+ arrDF.selectExpr("flatten(array(arr, array(null, 5), array(6,
null)))"),
+ Seq(Row(Seq(1, 2, 3, null, 5, 6, null))))
+ checkAnswer(
+ arrDF.selectExpr("flatten(array(array(arr, arr), array(arr)))"),
+ Seq(Row(Seq(Seq(1, 2, 3), Seq(1, 2, 3), Seq(1, 2, 3)))))
+ }
+
+ // Test with local relation, the Project will be evaluated without codegen
+ testArray()
+ // Test with cached relation, the Project will be evaluated with codegen
+ arrDF.cache()
+ testArray()
+
+ // Error test cases
+ val oneRowDF = Seq((1, "a", Seq(1, 2, 3))).toDF("i", "s", "arr")
+ intercept[AnalysisException] {
+ oneRowDF.select(flatten($"arr"))
+ }
+ intercept[AnalysisException] {
+ oneRowDF.select(flatten($"i"))
+ }
+ intercept[AnalysisException] {
+ oneRowDF.select(flatten($"s"))
+ }
+ intercept[AnalysisException] {
+ oneRowDF.selectExpr("flatten(null)")
+ }
+ }
}
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
index 082f06641b..7390d3ff5a 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
@@ -297,4 +297,22 @@ class GlutenDateFunctionsSuite extends DateFunctionsSuite
with GlutenSQLTestsTra
val df1 = Seq(x1, x2).toDF("x")
checkAnswer(df1.select(to_date(col("x"))), Row(Date.valueOf("2016-02-29"))
:: Row(null) :: Nil)
}
+
+ testGluten("date_from_unix_date") {
+ // -100000 and 200000 are outside ClickHouse's native Date32 range
+ // [1900-01-01, 2299-12-31]. They guard against implementations that clamp
to that
+ // range (e.g. mapping to CH toDate32), which would silently diverge from
Spark.
+ val df = Seq(Some(0), Some(1000), Some(-100000), Some(200000),
None).toDF("unix_date")
+ val expected = Seq(
+ Row(Date.valueOf("1970-01-01")),
+ Row(Date.valueOf("1972-09-27")),
+ Row(Date.valueOf("1696-03-17")),
+ Row(Date.valueOf("2517-08-01")),
+ Row(null))
+
+ // Go through expr() so the same case works on every supported version:
the Scala
+ // functions API does not expose date_from_unix_date before Spark 3.5.
+ checkAnswer(df.select(expr("date_from_unix_date(unix_date)")), expected)
+ checkAnswer(df.selectExpr("date_from_unix_date(unix_date)"), expected)
+ }
}
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenDynamicPartitionPruningSuite.scala
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenDynamicPartitionPruningSuite.scala
index 4c1dcd2a09..ab34d46f67 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenDynamicPartitionPruningSuite.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenDynamicPartitionPruningSuite.scala
@@ -768,3 +768,38 @@ class GlutenDynamicPartitionPruningV2SuiteAEOffDisableScan
super.sparkConf.set(GlutenConfig.COLUMNAR_BATCHSCAN_ENABLED.key, "false")
}
}
+
+// Test DPP with project disabled by user for some reason, which can also mock
the situation
+// that project is not transformable.
+class GlutenDynamicPartitionPruningV1SuiteAEOffWSCGOnDisableProject
+ extends GlutenDynamicPartitionPruningV2SuiteAEOff {
+ override def sparkConf: SparkConf = {
+ super.sparkConf.set(GlutenConfig.COLUMNAR_PROJECT_ENABLED.key, "false")
+ }
+}
+
+// Same as above except whole-stage codegen is off.
+class GlutenDynamicPartitionPruningV1SuiteAEOffWSCGOffDisableProject
+ extends GlutenDynamicPartitionPruningV2SuiteAEOff {
+ override def sparkConf: SparkConf = {
+ super.sparkConf
+ .set(GlutenConfig.COLUMNAR_PROJECT_ENABLED.key, "false")
+ .set(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key, "false")
+ }
+}
+
+class GlutenDynamicPartitionPruningV2SuiteAEOffWSCGOnDisableProject
+ extends GlutenDynamicPartitionPruningV2SuiteAEOff {
+ override def sparkConf: SparkConf = {
+ super.sparkConf.set(GlutenConfig.COLUMNAR_PROJECT_ENABLED.key, "false")
+ }
+}
+
+class GlutenDynamicPartitionPruningV2SuiteAEOffWSCGOffDisableProject
+ extends GlutenDynamicPartitionPruningV2SuiteAEOff {
+ override def sparkConf: SparkConf = {
+ super.sparkConf
+ .set(GlutenConfig.COLUMNAR_PROJECT_ENABLED.key, "false")
+ .set(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key, "false")
+ }
+}
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenLiteralExpressionSuite.scala
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenLiteralExpressionSuite.scala
index 556d185af0..f81ef0b6ff 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenLiteralExpressionSuite.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenLiteralExpressionSuite.scala
@@ -17,5 +17,41 @@
package org.apache.spark.sql.catalyst.expressions
import org.apache.spark.sql.GlutenTestsTrait
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.catalyst.util.DateTimeUtils
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.types._
+import org.apache.spark.unsafe.types.CalendarInterval
-class GlutenLiteralExpressionSuite extends LiteralExpressionSuite with
GlutenTestsTrait {}
+import java.nio.charset.StandardCharsets
+import java.time.{Instant, LocalDate}
+
+class GlutenLiteralExpressionSuite extends LiteralExpressionSuite with
GlutenTestsTrait {
+ testGluten("default") {
+ checkEvaluation(Literal.default(BooleanType), false)
+ checkEvaluation(Literal.default(ByteType), 0.toByte)
+ checkEvaluation(Literal.default(ShortType), 0.toShort)
+ checkEvaluation(Literal.default(IntegerType), 0)
+ checkEvaluation(Literal.default(LongType), 0L)
+ checkEvaluation(Literal.default(FloatType), 0.0f)
+ checkEvaluation(Literal.default(DoubleType), 0.0)
+ checkEvaluation(Literal.default(StringType), "")
+ checkEvaluation(Literal.default(BinaryType),
"".getBytes(StandardCharsets.UTF_8))
+ checkEvaluation(Literal.default(DecimalType.USER_DEFAULT), Decimal(0))
+ checkEvaluation(Literal.default(DecimalType.SYSTEM_DEFAULT), Decimal(0))
+ withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> "false") {
+ checkEvaluation(Literal.default(DateType), DateTimeUtils.toJavaDate(0))
+ checkEvaluation(Literal.default(TimestampType),
DateTimeUtils.toJavaTimestamp(0L))
+ }
+ withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> "true") {
+ checkEvaluation(Literal.default(DateType), LocalDate.ofEpochDay(0))
+ checkEvaluation(Literal.default(TimestampType), Instant.ofEpochSecond(0))
+ }
+ checkEvaluation(Literal.default(CalendarIntervalType), new
CalendarInterval(0, 0, 0L))
+ checkEvaluation(Literal.default(YearMonthIntervalType()), 0)
+ checkEvaluation(Literal.default(DayTimeIntervalType()), 0L)
+ checkEvaluation(Literal.default(ArrayType(StringType)), Array())
+ checkEvaluation(Literal.default(MapType(IntegerType, StringType)), Map())
+ checkEvaluation(Literal.default(StructType(StructField("a", StringType) ::
Nil)), Row(""))
+ }
+}
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenMathExpressionsSuite.scala
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenMathExpressionsSuite.scala
index b4459df420..2e329264d8 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenMathExpressionsSuite.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenMathExpressionsSuite.scala
@@ -20,6 +20,8 @@ import org.apache.spark.sql.GlutenTestsTrait
import org.apache.spark.sql.catalyst.dsl.expressions._
import org.apache.spark.sql.types._
+import java.nio.charset.StandardCharsets
+
class GlutenMathExpressionsSuite extends MathExpressionsSuite with
GlutenTestsTrait {
testGluten("round/bround/floor/ceil") {
val scales = -6 to 6
@@ -277,4 +279,22 @@ class GlutenMathExpressionsSuite extends
MathExpressionsSuite with GlutenTestsTr
checkEvaluation(checkDataTypeAndCast(RoundCeil(Literal(3.1411),
Literal(-3))), Decimal(1000))
checkEvaluation(checkDataTypeAndCast(RoundCeil(Literal(135.135),
Literal(-2))), Decimal(200))
}
+
+ testGluten("unhex") {
+ checkEvaluation(Unhex(Literal.create(null, StringType)), null)
+ checkEvaluation(Unhex(Literal("737472696E67")),
"string".getBytes(StandardCharsets.UTF_8))
+ checkEvaluation(Unhex(Literal("")), new Array[Byte](0))
+ checkEvaluation(Unhex(Literal("F")), Array[Byte](15))
+ checkEvaluation(Unhex(Literal("ff")), Array[Byte](-1))
+
+// checkEvaluation(Unhex(Literal("GG")), null)
+ checkEvaluation(Unhex(Literal("123")), Array[Byte](1, 35))
+ checkEvaluation(Unhex(Literal("12345")), Array[Byte](1, 35, 69))
+ // scalastyle:off
+ // Turn off scala style for non-ascii chars
+ checkEvaluation(Unhex(Literal("E4B889E9878DE79A84")),
"三重的".getBytes(StandardCharsets.UTF_8))
+// checkEvaluation(Unhex(Literal("三重的")), null)
+ // scalastyle:on
+ checkConsistencyBetweenInterpretedAndCodegen((e: Expression) => Unhex(e),
StringType)
+ }
}
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/adaptive/clickhouse/ClickHouseAdaptiveQueryExecSuite.scala
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/adaptive/clickhouse/ClickHouseAdaptiveQueryExecSuite.scala
index 9fcc6677b9..a682f65546 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/adaptive/clickhouse/ClickHouseAdaptiveQueryExecSuite.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/adaptive/clickhouse/ClickHouseAdaptiveQueryExecSuite.scala
@@ -1572,4 +1572,59 @@ class ClickHouseAdaptiveQueryExecSuite extends
AdaptiveQueryExecSuite with Glute
}
}
}
+
+ // TODO: re-enable this test after #12845.
findTopLevelSortMergeJoinTransform matches the Velox
+ // SortMergeJoinExecTransformer, which ClickHouse never produces, so it
collects nothing here and
+ // the case fails on CH-3.5. The expected counts are also still unverified
on ClickHouse.
+ ignoreGluten("SPARK-37652: optimize skewed join through union") {
+ withSQLConf(
+ SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
+ SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1",
+ SQLConf.SKEW_JOIN_SKEWED_PARTITION_THRESHOLD.key -> "100",
+ SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key -> "100"
+ ) {
+ withTempView("skewData1", "skewData2") {
+ spark
+ .range(0, 1000, 1, 10)
+ .selectExpr("id % 3 as key1", "id as value1")
+ .createOrReplaceTempView("skewData1")
+ spark
+ .range(0, 1000, 1, 10)
+ .selectExpr("id % 1 as key2", "id as value2")
+ .createOrReplaceTempView("skewData2")
+
+ def checkSkewJoin(query: String, joinNums: Int, optimizeSkewJoinNums:
Int): Unit = {
+ val (_, innerAdaptivePlan) = runAdaptiveAndVerifyResult(query)
+ val joins = findTopLevelSortMergeJoinTransform(innerAdaptivePlan)
+ val optimizeSkewJoins = joins.filter(_.isSkewJoin)
+ assert(joins.size == joinNums && optimizeSkewJoins.size ==
optimizeSkewJoinNums)
+ }
+
+ // skewJoin union skewJoin
+ checkSkewJoin(
+ "SELECT key1 FROM skewData1 JOIN skewData2 ON key1 = key2 " +
+ "UNION ALL SELECT key2 FROM skewData1 JOIN skewData2 ON key1 =
key2",
+ 2,
+ 2)
+
+ // skewJoin union aggregate
+ checkSkewJoin(
+ "SELECT key1 FROM skewData1 JOIN skewData2 ON key1 = key2 " +
+ "UNION ALL SELECT key2 FROM skewData2 GROUP BY key2",
+ 1,
+ 1)
+
+ // skewJoin1 union (skewJoin2 join aggregate)
+ // skewJoin2 will lead to extra shuffles, but skew1 cannot be optimized
+ checkSkewJoin(
+ "SELECT key1 FROM skewData1 JOIN skewData2 ON key1 = key2 UNION ALL
" +
+ "SELECT key1 from (SELECT key1 FROM skewData1 JOIN skewData2 ON
key1 = key2) tmp1 " +
+ "JOIN (SELECT key2 FROM skewData2 GROUP BY key2) tmp2 ON key1 =
key2",
+ 3,
+ 0
+ )
+ }
+ }
+ }
+
}
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQuerySuite.scala
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQuerySuite.scala
index 5308ee7ede..ddd61613ee 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQuerySuite.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQuerySuite.scala
@@ -241,4 +241,24 @@ class GlutenHiveSQLQuerySuite extends
GlutenHiveSQLQuerySuiteBase {
}
}
}
+
+ testGluten("avoid unnecessary filter binding for subfield during scan") {
+ withSQLConf(
+ "spark.sql.hive.convertMetastoreParquet" -> "false") {
+ sql("DROP TABLE IF EXISTS test_subfield")
+ sql(
+ "CREATE TABLE test_subfield (name STRING, favorite_color STRING," +
+ " label STRUCT<label_1:STRING, label_2:STRING>) USING hive
OPTIONS(fileFormat 'parquet')")
+ sql(
+ "INSERT INTO test_subfield VALUES('test_1', 'red',
named_struct('label_1', 'label-a'," +
+ "'label_2', 'label-b'))")
+ val df = spark.sql("select * from test_subfield where name='test_1'")
+ checkAnswer(df, Seq(Row("test_1", "red", Row("label-a", "label-b"))))
+ checkOperatorMatch[HiveTableScanExecTransformer](df)
+ }
+ spark.sessionState.catalog.dropTable(
+ TableIdentifier("test_subfield"),
+ ignoreIfNotExists = true,
+ purge = false)
+ }
}
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 7cbe7fea4e..c0b965d6e8 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -1071,10 +1071,14 @@ class VeloxTestSettings extends BackendTestSettings {
enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOn]
enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOnDisableScan]
enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOffDisableScan]
+ enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOffWSCGOnDisableProject]
+ enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOffWSCGOffDisableProject]
enableSuite[GlutenDynamicPartitionPruningV2SuiteAEOff]
enableSuite[GlutenDynamicPartitionPruningV2SuiteAEOn]
enableSuite[GlutenDynamicPartitionPruningV2SuiteAEOnDisableScan]
enableSuite[GlutenDynamicPartitionPruningV2SuiteAEOffDisableScan]
+ enableSuite[GlutenDynamicPartitionPruningV2SuiteAEOffWSCGOnDisableProject]
+ enableSuite[GlutenDynamicPartitionPruningV2SuiteAEOffWSCGOffDisableProject]
enableSuite[GlutenExpressionsSchemaSuite]
enableSuite[GlutenExtraStrategiesSuite]
enableSuite[GlutenFileBasedDataSourceSuite]
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenDataFrameFunctionsSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenDataFrameFunctionsSuite.scala
index 49f6052b20..0d2caf6100 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenDataFrameFunctionsSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenDataFrameFunctionsSuite.scala
@@ -278,4 +278,85 @@ class GlutenDataFrameFunctionsSuite extends
DataFrameFunctionsSuite with GlutenS
testMapOfPrimitiveTypesCombination()
}
+ testGluten("flatten function") {
+ // Test cases with a primitive type
+ val intDF = Seq(
+ (Seq(Seq(1, 2, 3), Seq(4, 5), Seq(6))),
+ (Seq(Seq(1, 2))),
+ (Seq(Seq(1), Seq.empty)),
+ (Seq(Seq.empty, Seq(1)))
+ ).toDF("i")
+
+ val intDFResult = Seq(Row(Seq(1, 2, 3, 4, 5, 6)), Row(Seq(1, 2)),
Row(Seq(1)), Row(Seq(1)))
+
+ def testInt(): Unit = {
+ checkAnswer(intDF.select(flatten($"i")), intDFResult)
+ checkAnswer(intDF.selectExpr("flatten(i)"), intDFResult)
+ }
+
+ // Test with local relation, the Project will be evaluated without codegen
+ testInt()
+ // Test with cached relation, the Project will be evaluated with codegen
+ intDF.cache()
+ testInt()
+
+ // Test cases with non-primitive types
+ val strDF = Seq(
+ (Seq(Seq("a", "b"), Seq("c"), Seq("d", "e", "f"))),
+ (Seq(Seq("a", "b"))),
+ (Seq(Seq("a", null), Seq(null, "b"), Seq(null, null))),
+ (Seq(Seq("a"), Seq.empty)),
+ (Seq(Seq.empty, Seq("a")))
+ ).toDF("s")
+
+ val strDFResult = Seq(
+ Row(Seq("a", "b", "c", "d", "e", "f")),
+ Row(Seq("a", "b")),
+ Row(Seq("a", null, null, "b", null, null)),
+ Row(Seq("a")),
+ Row(Seq("a")))
+
+ def testString(): Unit = {
+ checkAnswer(strDF.select(flatten($"s")), strDFResult)
+ checkAnswer(strDF.selectExpr("flatten(s)"), strDFResult)
+ }
+
+ // Test with local relation, the Project will be evaluated without codegen
+ testString()
+ // Test with cached relation, the Project will be evaluated with codegen
+ strDF.cache()
+ testString()
+
+ val arrDF = Seq((1, "a", Seq(1, 2, 3))).toDF("i", "s", "arr")
+
+ def testArray(): Unit = {
+ checkAnswer(
+ arrDF.selectExpr("flatten(array(arr, array(null, 5), array(6,
null)))"),
+ Seq(Row(Seq(1, 2, 3, null, 5, 6, null))))
+ checkAnswer(
+ arrDF.selectExpr("flatten(array(array(arr, arr), array(arr)))"),
+ Seq(Row(Seq(Seq(1, 2, 3), Seq(1, 2, 3), Seq(1, 2, 3)))))
+ }
+
+ // Test with local relation, the Project will be evaluated without codegen
+ testArray()
+ // Test with cached relation, the Project will be evaluated with codegen
+ arrDF.cache()
+ testArray()
+
+ // Error test cases
+ val oneRowDF = Seq((1, "a", Seq(1, 2, 3))).toDF("i", "s", "arr")
+ intercept[AnalysisException] {
+ oneRowDF.select(flatten($"arr"))
+ }
+ intercept[AnalysisException] {
+ oneRowDF.select(flatten($"i"))
+ }
+ intercept[AnalysisException] {
+ oneRowDF.select(flatten($"s"))
+ }
+ intercept[AnalysisException] {
+ oneRowDF.selectExpr("flatten(null)")
+ }
+ }
}
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
index 082f06641b..7390d3ff5a 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
@@ -297,4 +297,22 @@ class GlutenDateFunctionsSuite extends DateFunctionsSuite
with GlutenSQLTestsTra
val df1 = Seq(x1, x2).toDF("x")
checkAnswer(df1.select(to_date(col("x"))), Row(Date.valueOf("2016-02-29"))
:: Row(null) :: Nil)
}
+
+ testGluten("date_from_unix_date") {
+ // -100000 and 200000 are outside ClickHouse's native Date32 range
+ // [1900-01-01, 2299-12-31]. They guard against implementations that clamp
to that
+ // range (e.g. mapping to CH toDate32), which would silently diverge from
Spark.
+ val df = Seq(Some(0), Some(1000), Some(-100000), Some(200000),
None).toDF("unix_date")
+ val expected = Seq(
+ Row(Date.valueOf("1970-01-01")),
+ Row(Date.valueOf("1972-09-27")),
+ Row(Date.valueOf("1696-03-17")),
+ Row(Date.valueOf("2517-08-01")),
+ Row(null))
+
+ // Go through expr() so the same case works on every supported version:
the Scala
+ // functions API does not expose date_from_unix_date before Spark 3.5.
+ checkAnswer(df.select(expr("date_from_unix_date(unix_date)")), expected)
+ checkAnswer(df.selectExpr("date_from_unix_date(unix_date)"), expected)
+ }
}
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenDynamicPartitionPruningSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenDynamicPartitionPruningSuite.scala
index dc96c09bc2..9f4e18fb82 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenDynamicPartitionPruningSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenDynamicPartitionPruningSuite.scala
@@ -709,3 +709,38 @@ class GlutenDynamicPartitionPruningV2SuiteAEOffDisableScan
super.sparkConf.set(GlutenConfig.COLUMNAR_BATCHSCAN_ENABLED.key, "false")
}
}
+
+// Test DPP with project disabled by user for some reason, which can also mock
the situation
+// that project is not transformable.
+class GlutenDynamicPartitionPruningV1SuiteAEOffWSCGOnDisableProject
+ extends GlutenDynamicPartitionPruningV2SuiteAEOff {
+ override def sparkConf: SparkConf = {
+ super.sparkConf.set(GlutenConfig.COLUMNAR_PROJECT_ENABLED.key, "false")
+ }
+}
+
+// Same as above except whole-stage codegen is off.
+class GlutenDynamicPartitionPruningV1SuiteAEOffWSCGOffDisableProject
+ extends GlutenDynamicPartitionPruningV2SuiteAEOff {
+ override def sparkConf: SparkConf = {
+ super.sparkConf
+ .set(GlutenConfig.COLUMNAR_PROJECT_ENABLED.key, "false")
+ .set(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key, "false")
+ }
+}
+
+class GlutenDynamicPartitionPruningV2SuiteAEOffWSCGOnDisableProject
+ extends GlutenDynamicPartitionPruningV2SuiteAEOff {
+ override def sparkConf: SparkConf = {
+ super.sparkConf.set(GlutenConfig.COLUMNAR_PROJECT_ENABLED.key, "false")
+ }
+}
+
+class GlutenDynamicPartitionPruningV2SuiteAEOffWSCGOffDisableProject
+ extends GlutenDynamicPartitionPruningV2SuiteAEOff {
+ override def sparkConf: SparkConf = {
+ super.sparkConf
+ .set(GlutenConfig.COLUMNAR_PROJECT_ENABLED.key, "false")
+ .set(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key, "false")
+ }
+}
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenLiteralExpressionSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenLiteralExpressionSuite.scala
index 556d185af0..f81ef0b6ff 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenLiteralExpressionSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenLiteralExpressionSuite.scala
@@ -17,5 +17,41 @@
package org.apache.spark.sql.catalyst.expressions
import org.apache.spark.sql.GlutenTestsTrait
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.catalyst.util.DateTimeUtils
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.types._
+import org.apache.spark.unsafe.types.CalendarInterval
-class GlutenLiteralExpressionSuite extends LiteralExpressionSuite with
GlutenTestsTrait {}
+import java.nio.charset.StandardCharsets
+import java.time.{Instant, LocalDate}
+
+class GlutenLiteralExpressionSuite extends LiteralExpressionSuite with
GlutenTestsTrait {
+ testGluten("default") {
+ checkEvaluation(Literal.default(BooleanType), false)
+ checkEvaluation(Literal.default(ByteType), 0.toByte)
+ checkEvaluation(Literal.default(ShortType), 0.toShort)
+ checkEvaluation(Literal.default(IntegerType), 0)
+ checkEvaluation(Literal.default(LongType), 0L)
+ checkEvaluation(Literal.default(FloatType), 0.0f)
+ checkEvaluation(Literal.default(DoubleType), 0.0)
+ checkEvaluation(Literal.default(StringType), "")
+ checkEvaluation(Literal.default(BinaryType),
"".getBytes(StandardCharsets.UTF_8))
+ checkEvaluation(Literal.default(DecimalType.USER_DEFAULT), Decimal(0))
+ checkEvaluation(Literal.default(DecimalType.SYSTEM_DEFAULT), Decimal(0))
+ withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> "false") {
+ checkEvaluation(Literal.default(DateType), DateTimeUtils.toJavaDate(0))
+ checkEvaluation(Literal.default(TimestampType),
DateTimeUtils.toJavaTimestamp(0L))
+ }
+ withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> "true") {
+ checkEvaluation(Literal.default(DateType), LocalDate.ofEpochDay(0))
+ checkEvaluation(Literal.default(TimestampType), Instant.ofEpochSecond(0))
+ }
+ checkEvaluation(Literal.default(CalendarIntervalType), new
CalendarInterval(0, 0, 0L))
+ checkEvaluation(Literal.default(YearMonthIntervalType()), 0)
+ checkEvaluation(Literal.default(DayTimeIntervalType()), 0L)
+ checkEvaluation(Literal.default(ArrayType(StringType)), Array())
+ checkEvaluation(Literal.default(MapType(IntegerType, StringType)), Map())
+ checkEvaluation(Literal.default(StructType(StructField("a", StringType) ::
Nil)), Row(""))
+ }
+}
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenMathExpressionsSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenMathExpressionsSuite.scala
index 1c47e1b2f3..db1be819ad 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenMathExpressionsSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenMathExpressionsSuite.scala
@@ -20,6 +20,8 @@ import org.apache.spark.sql.GlutenExpressionOffloadTracker
import org.apache.spark.sql.catalyst.dsl.expressions._
import org.apache.spark.sql.types._
+import java.nio.charset.StandardCharsets
+
class GlutenMathExpressionsSuite extends MathExpressionsSuite with
GlutenExpressionOffloadTracker {
override protected def offloadCategory: String = "math"
testGluten("round/bround/floor/ceil") {
@@ -278,4 +280,22 @@ class GlutenMathExpressionsSuite extends
MathExpressionsSuite with GlutenExpress
checkEvaluation(checkDataTypeAndCast(RoundCeil(Literal(3.1411),
Literal(-3))), Decimal(1000))
checkEvaluation(checkDataTypeAndCast(RoundCeil(Literal(135.135),
Literal(-2))), Decimal(200))
}
+
+ testGluten("unhex") {
+ checkEvaluation(Unhex(Literal.create(null, StringType)), null)
+ checkEvaluation(Unhex(Literal("737472696E67")),
"string".getBytes(StandardCharsets.UTF_8))
+ checkEvaluation(Unhex(Literal("")), new Array[Byte](0))
+ checkEvaluation(Unhex(Literal("F")), Array[Byte](15))
+ checkEvaluation(Unhex(Literal("ff")), Array[Byte](-1))
+
+// checkEvaluation(Unhex(Literal("GG")), null)
+ checkEvaluation(Unhex(Literal("123")), Array[Byte](1, 35))
+ checkEvaluation(Unhex(Literal("12345")), Array[Byte](1, 35, 69))
+ // scalastyle:off
+ // Turn off scala style for non-ascii chars
+ checkEvaluation(Unhex(Literal("E4B889E9878DE79A84")),
"三重的".getBytes(StandardCharsets.UTF_8))
+// checkEvaluation(Unhex(Literal("三重的")), null)
+ // scalastyle:on
+ checkConsistencyBetweenInterpretedAndCodegen((e: Expression) => Unhex(e),
StringType)
+ }
}
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/adaptive/clickhouse/ClickHouseAdaptiveQueryExecSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/adaptive/clickhouse/ClickHouseAdaptiveQueryExecSuite.scala
index 903c7ccd13..babab4b660 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/adaptive/clickhouse/ClickHouseAdaptiveQueryExecSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/adaptive/clickhouse/ClickHouseAdaptiveQueryExecSuite.scala
@@ -1566,4 +1566,59 @@ class ClickHouseAdaptiveQueryExecSuite extends
AdaptiveQueryExecSuite with Glute
}
}
}
+
+ // TODO: re-enable this test after #12845.
findTopLevelSortMergeJoinTransform matches the Velox
+ // SortMergeJoinExecTransformer, which ClickHouse never produces, so it
collects nothing here and
+ // the case fails on CH-3.5. The expected counts are also still unverified
on ClickHouse.
+ ignoreGluten("SPARK-37652: optimize skewed join through union") {
+ withSQLConf(
+ SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
+ SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1",
+ SQLConf.SKEW_JOIN_SKEWED_PARTITION_THRESHOLD.key -> "100",
+ SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key -> "100"
+ ) {
+ withTempView("skewData1", "skewData2") {
+ spark
+ .range(0, 1000, 1, 10)
+ .selectExpr("id % 3 as key1", "id as value1")
+ .createOrReplaceTempView("skewData1")
+ spark
+ .range(0, 1000, 1, 10)
+ .selectExpr("id % 1 as key2", "id as value2")
+ .createOrReplaceTempView("skewData2")
+
+ def checkSkewJoin(query: String, joinNums: Int, optimizeSkewJoinNums:
Int): Unit = {
+ val (_, innerAdaptivePlan) = runAdaptiveAndVerifyResult(query)
+ val joins = findTopLevelSortMergeJoinTransform(innerAdaptivePlan)
+ val optimizeSkewJoins = joins.filter(_.isSkewJoin)
+ assert(joins.size == joinNums && optimizeSkewJoins.size ==
optimizeSkewJoinNums)
+ }
+
+ // skewJoin union skewJoin
+ checkSkewJoin(
+ "SELECT key1 FROM skewData1 JOIN skewData2 ON key1 = key2 " +
+ "UNION ALL SELECT key2 FROM skewData1 JOIN skewData2 ON key1 =
key2",
+ 2,
+ 2)
+
+ // skewJoin union aggregate
+ checkSkewJoin(
+ "SELECT key1 FROM skewData1 JOIN skewData2 ON key1 = key2 " +
+ "UNION ALL SELECT key2 FROM skewData2 GROUP BY key2",
+ 1,
+ 1)
+
+ // skewJoin1 union (skewJoin2 join aggregate)
+ // skewJoin2 will lead to extra shuffles, but skew1 cannot be optimized
+ checkSkewJoin(
+ "SELECT key1 FROM skewData1 JOIN skewData2 ON key1 = key2 UNION ALL
" +
+ "SELECT key1 from (SELECT key1 FROM skewData1 JOIN skewData2 ON
key1 = key2) tmp1 " +
+ "JOIN (SELECT key2 FROM skewData2 GROUP BY key2) tmp2 ON key1 =
key2",
+ 3,
+ 0
+ )
+ }
+ }
+ }
+
}
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQuerySuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQuerySuite.scala
index 5308ee7ede..ddd61613ee 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQuerySuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQuerySuite.scala
@@ -241,4 +241,24 @@ class GlutenHiveSQLQuerySuite extends
GlutenHiveSQLQuerySuiteBase {
}
}
}
+
+ testGluten("avoid unnecessary filter binding for subfield during scan") {
+ withSQLConf(
+ "spark.sql.hive.convertMetastoreParquet" -> "false") {
+ sql("DROP TABLE IF EXISTS test_subfield")
+ sql(
+ "CREATE TABLE test_subfield (name STRING, favorite_color STRING," +
+ " label STRUCT<label_1:STRING, label_2:STRING>) USING hive
OPTIONS(fileFormat 'parquet')")
+ sql(
+ "INSERT INTO test_subfield VALUES('test_1', 'red',
named_struct('label_1', 'label-a'," +
+ "'label_2', 'label-b'))")
+ val df = spark.sql("select * from test_subfield where name='test_1'")
+ checkAnswer(df, Seq(Row("test_1", "red", Row("label-a", "label-b"))))
+ checkOperatorMatch[HiveTableScanExecTransformer](df)
+ }
+ spark.sessionState.catalog.dropTable(
+ TableIdentifier("test_subfield"),
+ ignoreIfNotExists = true,
+ purge = false)
+ }
}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 61bd44f554..ee88614222 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -1059,10 +1059,14 @@ class VeloxTestSettings extends BackendTestSettings {
enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOn]
enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOnDisableScan]
enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOffDisableScan]
+ enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOffWSCGOnDisableProject]
+ enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOffWSCGOffDisableProject]
enableSuite[GlutenDynamicPartitionPruningV2SuiteAEOff]
enableSuite[GlutenDynamicPartitionPruningV2SuiteAEOn]
enableSuite[GlutenDynamicPartitionPruningV2SuiteAEOnDisableScan]
enableSuite[GlutenDynamicPartitionPruningV2SuiteAEOffDisableScan]
+ enableSuite[GlutenDynamicPartitionPruningV2SuiteAEOffWSCGOnDisableProject]
+ enableSuite[GlutenDynamicPartitionPruningV2SuiteAEOffWSCGOffDisableProject]
enableSuite[GlutenExpressionsSchemaSuite]
enableSuite[GlutenExtraStrategiesSuite]
enableSuite[GlutenFileBasedDataSourceSuite]
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenDataFrameFunctionsSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenDataFrameFunctionsSuite.scala
index 49f6052b20..0d2caf6100 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenDataFrameFunctionsSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenDataFrameFunctionsSuite.scala
@@ -278,4 +278,85 @@ class GlutenDataFrameFunctionsSuite extends
DataFrameFunctionsSuite with GlutenS
testMapOfPrimitiveTypesCombination()
}
+ testGluten("flatten function") {
+ // Test cases with a primitive type
+ val intDF = Seq(
+ (Seq(Seq(1, 2, 3), Seq(4, 5), Seq(6))),
+ (Seq(Seq(1, 2))),
+ (Seq(Seq(1), Seq.empty)),
+ (Seq(Seq.empty, Seq(1)))
+ ).toDF("i")
+
+ val intDFResult = Seq(Row(Seq(1, 2, 3, 4, 5, 6)), Row(Seq(1, 2)),
Row(Seq(1)), Row(Seq(1)))
+
+ def testInt(): Unit = {
+ checkAnswer(intDF.select(flatten($"i")), intDFResult)
+ checkAnswer(intDF.selectExpr("flatten(i)"), intDFResult)
+ }
+
+ // Test with local relation, the Project will be evaluated without codegen
+ testInt()
+ // Test with cached relation, the Project will be evaluated with codegen
+ intDF.cache()
+ testInt()
+
+ // Test cases with non-primitive types
+ val strDF = Seq(
+ (Seq(Seq("a", "b"), Seq("c"), Seq("d", "e", "f"))),
+ (Seq(Seq("a", "b"))),
+ (Seq(Seq("a", null), Seq(null, "b"), Seq(null, null))),
+ (Seq(Seq("a"), Seq.empty)),
+ (Seq(Seq.empty, Seq("a")))
+ ).toDF("s")
+
+ val strDFResult = Seq(
+ Row(Seq("a", "b", "c", "d", "e", "f")),
+ Row(Seq("a", "b")),
+ Row(Seq("a", null, null, "b", null, null)),
+ Row(Seq("a")),
+ Row(Seq("a")))
+
+ def testString(): Unit = {
+ checkAnswer(strDF.select(flatten($"s")), strDFResult)
+ checkAnswer(strDF.selectExpr("flatten(s)"), strDFResult)
+ }
+
+ // Test with local relation, the Project will be evaluated without codegen
+ testString()
+ // Test with cached relation, the Project will be evaluated with codegen
+ strDF.cache()
+ testString()
+
+ val arrDF = Seq((1, "a", Seq(1, 2, 3))).toDF("i", "s", "arr")
+
+ def testArray(): Unit = {
+ checkAnswer(
+ arrDF.selectExpr("flatten(array(arr, array(null, 5), array(6,
null)))"),
+ Seq(Row(Seq(1, 2, 3, null, 5, 6, null))))
+ checkAnswer(
+ arrDF.selectExpr("flatten(array(array(arr, arr), array(arr)))"),
+ Seq(Row(Seq(Seq(1, 2, 3), Seq(1, 2, 3), Seq(1, 2, 3)))))
+ }
+
+ // Test with local relation, the Project will be evaluated without codegen
+ testArray()
+ // Test with cached relation, the Project will be evaluated with codegen
+ arrDF.cache()
+ testArray()
+
+ // Error test cases
+ val oneRowDF = Seq((1, "a", Seq(1, 2, 3))).toDF("i", "s", "arr")
+ intercept[AnalysisException] {
+ oneRowDF.select(flatten($"arr"))
+ }
+ intercept[AnalysisException] {
+ oneRowDF.select(flatten($"i"))
+ }
+ intercept[AnalysisException] {
+ oneRowDF.select(flatten($"s"))
+ }
+ intercept[AnalysisException] {
+ oneRowDF.selectExpr("flatten(null)")
+ }
+ }
}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
index 082f06641b..7390d3ff5a 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
@@ -297,4 +297,22 @@ class GlutenDateFunctionsSuite extends DateFunctionsSuite
with GlutenSQLTestsTra
val df1 = Seq(x1, x2).toDF("x")
checkAnswer(df1.select(to_date(col("x"))), Row(Date.valueOf("2016-02-29"))
:: Row(null) :: Nil)
}
+
+ testGluten("date_from_unix_date") {
+ // -100000 and 200000 are outside ClickHouse's native Date32 range
+ // [1900-01-01, 2299-12-31]. They guard against implementations that clamp
to that
+ // range (e.g. mapping to CH toDate32), which would silently diverge from
Spark.
+ val df = Seq(Some(0), Some(1000), Some(-100000), Some(200000),
None).toDF("unix_date")
+ val expected = Seq(
+ Row(Date.valueOf("1970-01-01")),
+ Row(Date.valueOf("1972-09-27")),
+ Row(Date.valueOf("1696-03-17")),
+ Row(Date.valueOf("2517-08-01")),
+ Row(null))
+
+ // Go through expr() so the same case works on every supported version:
the Scala
+ // functions API does not expose date_from_unix_date before Spark 3.5.
+ checkAnswer(df.select(expr("date_from_unix_date(unix_date)")), expected)
+ checkAnswer(df.selectExpr("date_from_unix_date(unix_date)"), expected)
+ }
}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenDynamicPartitionPruningSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenDynamicPartitionPruningSuite.scala
index d2222aa8a6..2a1a2c55fe 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenDynamicPartitionPruningSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenDynamicPartitionPruningSuite.scala
@@ -711,3 +711,38 @@ class GlutenDynamicPartitionPruningV2SuiteAEOffDisableScan
super.sparkConf.set(GlutenConfig.COLUMNAR_BATCHSCAN_ENABLED.key, "false")
}
}
+
+// Test DPP with project disabled by user for some reason, which can also mock
the situation
+// that project is not transformable.
+class GlutenDynamicPartitionPruningV1SuiteAEOffWSCGOnDisableProject
+ extends GlutenDynamicPartitionPruningV2SuiteAEOff {
+ override def sparkConf: SparkConf = {
+ super.sparkConf.set(GlutenConfig.COLUMNAR_PROJECT_ENABLED.key, "false")
+ }
+}
+
+// Same as above except whole-stage codegen is off.
+class GlutenDynamicPartitionPruningV1SuiteAEOffWSCGOffDisableProject
+ extends GlutenDynamicPartitionPruningV2SuiteAEOff {
+ override def sparkConf: SparkConf = {
+ super.sparkConf
+ .set(GlutenConfig.COLUMNAR_PROJECT_ENABLED.key, "false")
+ .set(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key, "false")
+ }
+}
+
+class GlutenDynamicPartitionPruningV2SuiteAEOffWSCGOnDisableProject
+ extends GlutenDynamicPartitionPruningV2SuiteAEOff {
+ override def sparkConf: SparkConf = {
+ super.sparkConf.set(GlutenConfig.COLUMNAR_PROJECT_ENABLED.key, "false")
+ }
+}
+
+class GlutenDynamicPartitionPruningV2SuiteAEOffWSCGOffDisableProject
+ extends GlutenDynamicPartitionPruningV2SuiteAEOff {
+ override def sparkConf: SparkConf = {
+ super.sparkConf
+ .set(GlutenConfig.COLUMNAR_PROJECT_ENABLED.key, "false")
+ .set(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key, "false")
+ }
+}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenLiteralExpressionSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenLiteralExpressionSuite.scala
index 556d185af0..f81ef0b6ff 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenLiteralExpressionSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenLiteralExpressionSuite.scala
@@ -17,5 +17,41 @@
package org.apache.spark.sql.catalyst.expressions
import org.apache.spark.sql.GlutenTestsTrait
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.catalyst.util.DateTimeUtils
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.types._
+import org.apache.spark.unsafe.types.CalendarInterval
-class GlutenLiteralExpressionSuite extends LiteralExpressionSuite with
GlutenTestsTrait {}
+import java.nio.charset.StandardCharsets
+import java.time.{Instant, LocalDate}
+
+class GlutenLiteralExpressionSuite extends LiteralExpressionSuite with
GlutenTestsTrait {
+ testGluten("default") {
+ checkEvaluation(Literal.default(BooleanType), false)
+ checkEvaluation(Literal.default(ByteType), 0.toByte)
+ checkEvaluation(Literal.default(ShortType), 0.toShort)
+ checkEvaluation(Literal.default(IntegerType), 0)
+ checkEvaluation(Literal.default(LongType), 0L)
+ checkEvaluation(Literal.default(FloatType), 0.0f)
+ checkEvaluation(Literal.default(DoubleType), 0.0)
+ checkEvaluation(Literal.default(StringType), "")
+ checkEvaluation(Literal.default(BinaryType),
"".getBytes(StandardCharsets.UTF_8))
+ checkEvaluation(Literal.default(DecimalType.USER_DEFAULT), Decimal(0))
+ checkEvaluation(Literal.default(DecimalType.SYSTEM_DEFAULT), Decimal(0))
+ withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> "false") {
+ checkEvaluation(Literal.default(DateType), DateTimeUtils.toJavaDate(0))
+ checkEvaluation(Literal.default(TimestampType),
DateTimeUtils.toJavaTimestamp(0L))
+ }
+ withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> "true") {
+ checkEvaluation(Literal.default(DateType), LocalDate.ofEpochDay(0))
+ checkEvaluation(Literal.default(TimestampType), Instant.ofEpochSecond(0))
+ }
+ checkEvaluation(Literal.default(CalendarIntervalType), new
CalendarInterval(0, 0, 0L))
+ checkEvaluation(Literal.default(YearMonthIntervalType()), 0)
+ checkEvaluation(Literal.default(DayTimeIntervalType()), 0L)
+ checkEvaluation(Literal.default(ArrayType(StringType)), Array())
+ checkEvaluation(Literal.default(MapType(IntegerType, StringType)), Map())
+ checkEvaluation(Literal.default(StructType(StructField("a", StringType) ::
Nil)), Row(""))
+ }
+}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenMathExpressionsSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenMathExpressionsSuite.scala
index ac0296c3f0..9d4d1fe848 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenMathExpressionsSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenMathExpressionsSuite.scala
@@ -21,6 +21,8 @@ import org.apache.spark.sql.catalyst.dsl.expressions._
import org.apache.spark.sql.shim.GlutenTestsTrait
import org.apache.spark.sql.types._
+import java.nio.charset.StandardCharsets
+
class GlutenMathExpressionsSuite
extends MathExpressionsSuite
with GlutenExpressionOffloadTracker
@@ -282,4 +284,22 @@ class GlutenMathExpressionsSuite
checkEvaluation(checkDataTypeAndCast(RoundCeil(Literal(3.1411),
Literal(-3))), Decimal(1000))
checkEvaluation(checkDataTypeAndCast(RoundCeil(Literal(135.135),
Literal(-2))), Decimal(200))
}
+
+ testGluten("unhex") {
+ checkEvaluation(Unhex(Literal.create(null, StringType)), null)
+ checkEvaluation(Unhex(Literal("737472696E67")),
"string".getBytes(StandardCharsets.UTF_8))
+ checkEvaluation(Unhex(Literal("")), new Array[Byte](0))
+ checkEvaluation(Unhex(Literal("F")), Array[Byte](15))
+ checkEvaluation(Unhex(Literal("ff")), Array[Byte](-1))
+
+// checkEvaluation(Unhex(Literal("GG")), null)
+ checkEvaluation(Unhex(Literal("123")), Array[Byte](1, 35))
+ checkEvaluation(Unhex(Literal("12345")), Array[Byte](1, 35, 69))
+ // scalastyle:off
+ // Turn off scala style for non-ascii chars
+ checkEvaluation(Unhex(Literal("E4B889E9878DE79A84")),
"三重的".getBytes(StandardCharsets.UTF_8))
+// checkEvaluation(Unhex(Literal("三重的")), null)
+ // scalastyle:on
+ checkConsistencyBetweenInterpretedAndCodegen((e: Expression) => Unhex(e),
StringType)
+ }
}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/adaptive/clickhouse/ClickHouseAdaptiveQueryExecSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/adaptive/clickhouse/ClickHouseAdaptiveQueryExecSuite.scala
index 903c7ccd13..babab4b660 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/adaptive/clickhouse/ClickHouseAdaptiveQueryExecSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/adaptive/clickhouse/ClickHouseAdaptiveQueryExecSuite.scala
@@ -1566,4 +1566,59 @@ class ClickHouseAdaptiveQueryExecSuite extends
AdaptiveQueryExecSuite with Glute
}
}
}
+
+ // TODO: re-enable this test after #12845.
findTopLevelSortMergeJoinTransform matches the Velox
+ // SortMergeJoinExecTransformer, which ClickHouse never produces, so it
collects nothing here and
+ // the case fails on CH-3.5. The expected counts are also still unverified
on ClickHouse.
+ ignoreGluten("SPARK-37652: optimize skewed join through union") {
+ withSQLConf(
+ SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
+ SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1",
+ SQLConf.SKEW_JOIN_SKEWED_PARTITION_THRESHOLD.key -> "100",
+ SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key -> "100"
+ ) {
+ withTempView("skewData1", "skewData2") {
+ spark
+ .range(0, 1000, 1, 10)
+ .selectExpr("id % 3 as key1", "id as value1")
+ .createOrReplaceTempView("skewData1")
+ spark
+ .range(0, 1000, 1, 10)
+ .selectExpr("id % 1 as key2", "id as value2")
+ .createOrReplaceTempView("skewData2")
+
+ def checkSkewJoin(query: String, joinNums: Int, optimizeSkewJoinNums:
Int): Unit = {
+ val (_, innerAdaptivePlan) = runAdaptiveAndVerifyResult(query)
+ val joins = findTopLevelSortMergeJoinTransform(innerAdaptivePlan)
+ val optimizeSkewJoins = joins.filter(_.isSkewJoin)
+ assert(joins.size == joinNums && optimizeSkewJoins.size ==
optimizeSkewJoinNums)
+ }
+
+ // skewJoin union skewJoin
+ checkSkewJoin(
+ "SELECT key1 FROM skewData1 JOIN skewData2 ON key1 = key2 " +
+ "UNION ALL SELECT key2 FROM skewData1 JOIN skewData2 ON key1 =
key2",
+ 2,
+ 2)
+
+ // skewJoin union aggregate
+ checkSkewJoin(
+ "SELECT key1 FROM skewData1 JOIN skewData2 ON key1 = key2 " +
+ "UNION ALL SELECT key2 FROM skewData2 GROUP BY key2",
+ 1,
+ 1)
+
+ // skewJoin1 union (skewJoin2 join aggregate)
+ // skewJoin2 will lead to extra shuffles, but skew1 cannot be optimized
+ checkSkewJoin(
+ "SELECT key1 FROM skewData1 JOIN skewData2 ON key1 = key2 UNION ALL
" +
+ "SELECT key1 from (SELECT key1 FROM skewData1 JOIN skewData2 ON
key1 = key2) tmp1 " +
+ "JOIN (SELECT key2 FROM skewData2 GROUP BY key2) tmp2 ON key1 =
key2",
+ 3,
+ 0
+ )
+ }
+ }
+ }
+
}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQuerySuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQuerySuite.scala
index 5308ee7ede..ddd61613ee 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQuerySuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQuerySuite.scala
@@ -241,4 +241,24 @@ class GlutenHiveSQLQuerySuite extends
GlutenHiveSQLQuerySuiteBase {
}
}
}
+
+ testGluten("avoid unnecessary filter binding for subfield during scan") {
+ withSQLConf(
+ "spark.sql.hive.convertMetastoreParquet" -> "false") {
+ sql("DROP TABLE IF EXISTS test_subfield")
+ sql(
+ "CREATE TABLE test_subfield (name STRING, favorite_color STRING," +
+ " label STRUCT<label_1:STRING, label_2:STRING>) USING hive
OPTIONS(fileFormat 'parquet')")
+ sql(
+ "INSERT INTO test_subfield VALUES('test_1', 'red',
named_struct('label_1', 'label-a'," +
+ "'label_2', 'label-b'))")
+ val df = spark.sql("select * from test_subfield where name='test_1'")
+ checkAnswer(df, Seq(Row("test_1", "red", Row("label-a", "label-b"))))
+ checkOperatorMatch[HiveTableScanExecTransformer](df)
+ }
+ spark.sessionState.catalog.dropTable(
+ TableIdentifier("test_subfield"),
+ ignoreIfNotExists = true,
+ purge = false)
+ }
}
diff --git
a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenImplicitsTest.scala
b/gluten-ut/test/src/test/scala/org/apache/spark/sql/GlutenImplicitsTest.scala
similarity index 55%
rename from
gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenImplicitsTest.scala
rename to
gluten-ut/test/src/test/scala/org/apache/spark/sql/GlutenImplicitsTest.scala
index 6de15f434a..07f6e48b3e 100644
---
a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenImplicitsTest.scala
+++
b/gluten-ut/test/src/test/scala/org/apache/spark/sql/GlutenImplicitsTest.scala
@@ -17,13 +17,39 @@
package org.apache.spark.sql
import org.apache.gluten.config.GlutenConfig
+import org.apache.gluten.utils.BackendTestUtils
import org.apache.spark.SparkConf
import org.apache.spark.sql.execution.GlutenImplicits._
import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.util.Utils
-class GlutenImplicitsTest extends GlutenSQLTestsBaseTrait {
- sys.props.put(GlutenConfig.COLUMNAR_TABLE_CACHE_ENABLED.key, "true")
+class GlutenImplicitsTest extends GlutenQueryTest with SharedSparkSession {
+
+ // Keep the warehouse under target/ so that `mvn clean` removes it. Left to
itself,
+ // SharedSparkSession resolves StaticSQLConf.WAREHOUSE_PATH against the
fork's working
+ // directory, which is the module basedir, and a run interrupted between
beforeAll and afterAll
+ // would leave t1 behind in a place no build step cleans up. Appending the
class name keeps
+ // this suite's warehouse separate from the other suites in this module. The
path mirrors what
+ // GlutenTestsBaseTrait hands to the per-version suites, inlined rather than
inherited because
+ // mixing that trait in would route every case through BackendTestSettings,
where this suite is
+ // no longer registered.
+ private val warehouse: String =
+ getClass.getResource("/").getPath +
"unit-tests-working-home/spark-warehouse/" +
+ getClass.getCanonicalName
+
+ override protected def sparkConf: SparkConf = {
+ // Reuse the session conf every other Gluten SQL suite runs with, since
the node counts
+ // asserted below depend on it.
+ GlutenSQLTestsBaseTrait
+ .nativeSparkConf(super.sparkConf, warehouse)
+ .set("spark.sql.shuffle.partitions", "5")
+ // Three cases below assert node counts for a cached relation, which
needs the table cache
+ // offloaded. That is the default, but set it explicitly so the
expectations do not change
+ // silently if the default ever flips.
+ .set(GlutenConfig.COLUMNAR_TABLE_CACHE_ENABLED.key, "true")
+ }
override protected def beforeAll(): Unit = {
super.beforeAll()
@@ -36,8 +62,16 @@ class GlutenImplicitsTest extends GlutenSQLTestsBaseTrait {
}
override protected def afterAll(): Unit = {
- spark.sql("drop table t1")
- super.afterAll()
+ // super.afterAll() stops the session, so it has to run even if the drop
fails: this module
+ // runs every suite in one fork, and a leaked SparkContext would take the
later suites with
+ // it. tryWithSafeFinally rather than a bare finally, because both halves
throw for the same
+ // reason (a stopped SparkContext fails the drop and catalog.reset()
alike) and a bare finally
+ // would replace the original exception with the one from super.afterAll().
+ Utils.tryWithSafeFinally {
+ spark.sql("DROP TABLE IF EXISTS t1")
+ } {
+ super.afterAll()
+ }
}
override protected def afterEach(): Unit = {
@@ -45,11 +79,6 @@ class GlutenImplicitsTest extends GlutenSQLTestsBaseTrait {
super.afterEach()
}
- override def sparkConf: SparkConf = {
- super.sparkConf
- .set("spark.sql.shuffle.partitions", "5")
- }
-
private def withAQEEnabledAndDisabled(f: => Unit): Unit = {
withSQLConf(
SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
@@ -67,7 +96,13 @@ class GlutenImplicitsTest extends GlutenSQLTestsBaseTrait {
}
}
- testGluten("fallbackSummary with query") {
+ // ClickHouse reports different Gluten node counts for the cases that
involve a shuffle or a
+ // cached relation. spark33's ClickHouseTestSettings excluded exactly these
three without
+ // recording what CH actually reports, so they stay Velox-only until someone
reads the real
+ // counts off a ClickHouse run. Follows GlutenCheckOverflowTransformerSuite
in this module.
+ private def assumeVeloxOnly(): Unit =
assume(BackendTestUtils.isVeloxBackendLoaded())
+
+ test("fallbackSummary with query") {
withAQEEnabledAndDisabled {
val df = spark.table("t1").filter(_.getLong(0) > 0)
assert(df.fallbackSummary().numGlutenNodes == 1, df.fallbackSummary())
@@ -78,7 +113,8 @@ class GlutenImplicitsTest extends GlutenSQLTestsBaseTrait {
}
}
- testGluten("fallbackSummary with shuffle") {
+ test("fallbackSummary with shuffle") {
+ assumeVeloxOnly()
withAQEEnabledAndDisabled {
val df = spark.sql("SELECT c2 FROM t1 group by c2").filter(_.getLong(0)
> 0)
assert(df.fallbackSummary().numGlutenNodes == 6, df.fallbackSummary())
@@ -89,7 +125,7 @@ class GlutenImplicitsTest extends GlutenSQLTestsBaseTrait {
}
}
- testGluten("fallbackSummary with set command") {
+ test("fallbackSummary with set command") {
withAQEEnabledAndDisabled {
val df = spark.sql("set k=v")
assert(df.fallbackSummary().numGlutenNodes == 0, df.fallbackSummary())
@@ -97,17 +133,22 @@ class GlutenImplicitsTest extends GlutenSQLTestsBaseTrait {
}
}
- testGluten("fallbackSummary with data write command") {
+ test("fallbackSummary with data write command") {
withAQEEnabledAndDisabled {
withTable("tmp") {
val df = spark.sql("create table tmp using parquet as select * from
t1")
- assert(df.fallbackSummary().numGlutenNodes == 1, df.fallbackSummary())
+ // Spark 3.3 counts one Gluten node here. Since 3.4 the CTAS is
executed as an
+ // ExecutedCommandExec, which collectFallbackNodes walks past without
counting anything,
+ // so the summary reports neither a Gluten node nor a fallback node.
+ val expectedGlutenNodes = if (isSparkVersionGE("3.4")) 0 else 1
+ assert(df.fallbackSummary().numGlutenNodes == expectedGlutenNodes,
df.fallbackSummary())
assert(df.fallbackSummary().numFallbackNodes == 0,
df.fallbackSummary())
}
}
}
- testGluten("fallbackSummary with cache") {
+ test("fallbackSummary with cache") {
+ assumeVeloxOnly()
withAQEEnabledAndDisabled {
val df = spark.table("t1").cache().filter(_.getLong(0) > 0)
assert(df.fallbackSummary().numGlutenNodes == 2, df.fallbackSummary())
@@ -118,7 +159,8 @@ class GlutenImplicitsTest extends GlutenSQLTestsBaseTrait {
}
}
- testGluten("fallbackSummary with cached data and shuffle") {
+ test("fallbackSummary with cached data and shuffle") {
+ assumeVeloxOnly()
withAQEEnabledAndDisabled {
val df = spark.sql("select * from t1").filter(_.getLong(0) >
0).cache.repartition()
assert(df.fallbackSummary().numGlutenNodes == 7, df.fallbackSummary())
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
