This is an automated email from the ASF dual-hosted git repository.
jackylee-ch pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 8409bdac5d [GLUTEN-13003][TEST] Replace constant-true
testWithMinSparkVersion calls with test (#13004)
8409bdac5d is described below
commit 8409bdac5df7faabb7ee4e54a53306736958643e
Author: YangJie <[email protected]>
AuthorDate: Mon Sep 21 01:35:43 2026 -0400
[GLUTEN-13003][TEST] Replace constant-true testWithMinSparkVersion calls
with test (#13004)
---
.../execution/BoltAggregateFunctionsSuite.scala | 8 ++---
.../gluten/execution/BoltColumnarCacheSuite.scala | 4 +--
.../gluten/execution/BoltHashJoinSuite.scala | 12 ++-----
.../execution/BoltStringFunctionsSuite.scala | 4 +--
.../gluten/execution/MiscOperatorSuite.scala | 2 +-
.../functions/DateFunctionsValidateSuite.scala | 4 +--
.../functions/JsonFunctionsValidateSuite.scala | 22 ++++++------
.../functions/MathFunctionsValidateSuite.scala | 6 ++--
.../functions/ScalarFunctionsValidateSuite.scala | 24 ++++++-------
.../execution/iceberg/ClickHouseIcebergSuite.scala | 4 +--
.../gluten/execution/VeloxIcebergSuite.scala | 2 +-
.../apache/gluten/execution/FallbackSuite.scala | 2 +-
.../gluten/execution/MiscOperatorSuite.scala | 2 +-
.../execution/VeloxAggregateFunctionsSuite.scala | 6 ++--
.../VeloxParquetDataTypeValidationSuite.scala | 6 ++--
.../gluten/expression/UDFPartialProjectSuite.scala | 2 +-
.../functions/DateFunctionsValidateSuite.scala | 2 +-
.../functions/JsonFunctionsValidateSuite.scala | 22 ++++++------
.../functions/MathFunctionsValidateSuite.scala | 2 +-
.../functions/ScalarFunctionsValidateSuite.scala | 33 +++++++++--------
.../org/apache/gluten/execution/DeltaSuite.scala | 41 +++++++---------------
.../org/apache/gluten/execution/IcebergSuite.scala | 12 +++----
.../expressions/GlutenExpressionMappingSuite.scala | 4 +--
23 files changed, 97 insertions(+), 129 deletions(-)
diff --git
a/backends-bolt/src/test/scala/org/apache/gluten/execution/BoltAggregateFunctionsSuite.scala
b/backends-bolt/src/test/scala/org/apache/gluten/execution/BoltAggregateFunctionsSuite.scala
index b7abefe81a..6e04cbdc87 100644
---
a/backends-bolt/src/test/scala/org/apache/gluten/execution/BoltAggregateFunctionsSuite.scala
+++
b/backends-bolt/src/test/scala/org/apache/gluten/execution/BoltAggregateFunctionsSuite.scala
@@ -389,7 +389,7 @@ abstract class BoltAggregateFunctionsSuite extends
BoltWholeStageTransformerSuit
}
}
- testWithMinSparkVersion("regr_r2", "3.3") {
+ test("regr_r2") {
runQueryAndCompare("""
|select regr_r2(l_partkey, l_suppkey) from lineitem;
|""".stripMargin) {
@@ -408,7 +408,7 @@ abstract class BoltAggregateFunctionsSuite extends
BoltWholeStageTransformerSuit
}
}
- testWithMinSparkVersion("regr_slope", "3.4") {
+ test("regr_slope") {
runQueryAndCompare("""
|select regr_slope(l_partkey, l_suppkey) from
lineitem;
|""".stripMargin) {
@@ -427,7 +427,7 @@ abstract class BoltAggregateFunctionsSuite extends
BoltWholeStageTransformerSuit
}
}
- testWithMinSparkVersion("regr_intercept", "3.4") {
+ test("regr_intercept") {
runQueryAndCompare("""
|select regr_intercept(l_partkey, l_suppkey) from
lineitem;
|""".stripMargin) {
@@ -446,7 +446,7 @@ abstract class BoltAggregateFunctionsSuite extends
BoltWholeStageTransformerSuit
}
}
- testWithMinSparkVersion("regr_sxy regr_sxx regr_syy", "3.4") {
+ test("regr_sxy regr_sxx regr_syy") {
runQueryAndCompare("""
|select regr_sxy(l_quantity, l_tax) from lineitem;
|""".stripMargin) {
diff --git
a/backends-bolt/src/test/scala/org/apache/gluten/execution/BoltColumnarCacheSuite.scala
b/backends-bolt/src/test/scala/org/apache/gluten/execution/BoltColumnarCacheSuite.scala
index 7d35c757b4..23542a323c 100644
---
a/backends-bolt/src/test/scala/org/apache/gluten/execution/BoltColumnarCacheSuite.scala
+++
b/backends-bolt/src/test/scala/org/apache/gluten/execution/BoltColumnarCacheSuite.scala
@@ -87,7 +87,7 @@ class BoltColumnarCacheSuite extends
BoltWholeStageTransformerSuite with Adaptiv
}
}
- testWithMinSparkVersion("input row", "3.2") {
+ test("input row") {
withTable("t") {
sql("CREATE TABLE t USING json AS SELECT * FROM values(1, 'a', (2, 'b'),
(3, 'c'))")
runQueryAndCompare("SELECT * FROM t", cache = true) {
@@ -110,7 +110,7 @@ class BoltColumnarCacheSuite extends
BoltWholeStageTransformerSuite with Adaptiv
}
// See issue https://github.com/apache/incubator-gluten/issues/8497.
- testWithMinSparkVersion("Input fallen back vanilla Spark columnar scan",
"3.3") {
+ test("Input fallen back vanilla Spark columnar scan") {
def withId(id: Int): Metadata =
new MetadataBuilder().putLong("parquet.field.id", id).build()
diff --git
a/backends-bolt/src/test/scala/org/apache/gluten/execution/BoltHashJoinSuite.scala
b/backends-bolt/src/test/scala/org/apache/gluten/execution/BoltHashJoinSuite.scala
index 75d7f01c55..1c09903661 100644
---
a/backends-bolt/src/test/scala/org/apache/gluten/execution/BoltHashJoinSuite.scala
+++
b/backends-bolt/src/test/scala/org/apache/gluten/execution/BoltHashJoinSuite.scala
@@ -74,7 +74,7 @@ class BoltHashJoinSuite extends
BoltWholeStageTransformerSuite {
}
}
- testWithMinSparkVersion("generate hash join plan - v2", "3.2") {
+ test("generate hash join plan - v2") {
withSQLConf(
("spark.sql.autoBroadcastJoinThreshold", "-1"),
("spark.sql.adaptive.enabled", "false"),
@@ -92,9 +92,7 @@ class BoltHashJoinSuite extends
BoltWholeStageTransformerSuite {
// The computing is combined into one single whole stage transformer.
val wholeStages = plan.collect { case wst: WholeStageTransformer => wst }
- if (SparkShimLoader.getSparkVersion.startsWith("3.2.")) {
- assert(wholeStages.length == 1)
- } else if (SparkShimLoader.getSparkVersion.startsWith("3.5.")) {
+ if (SparkShimLoader.getSparkVersion.startsWith("3.5.")) {
assert(wholeStages.length == 5)
} else {
assert(wholeStages.length == 3)
@@ -107,11 +105,7 @@ class BoltHashJoinSuite extends
BoltWholeStageTransformerSuite {
case _: ShuffledHashJoinExecTransformer => 1
}.getOrElse(0)
}.sum
- if (SparkShimLoader.getSparkVersion.startsWith("3.2.")) {
- assert(countSHJ == 1)
- } else {
- assert(countSHJ == 2)
- }
+ assert(countSHJ == 2)
}
}
diff --git
a/backends-bolt/src/test/scala/org/apache/gluten/execution/BoltStringFunctionsSuite.scala
b/backends-bolt/src/test/scala/org/apache/gluten/execution/BoltStringFunctionsSuite.scala
index b5a0f56e31..d518431196 100644
---
a/backends-bolt/src/test/scala/org/apache/gluten/execution/BoltStringFunctionsSuite.scala
+++
b/backends-bolt/src/test/scala/org/apache/gluten/execution/BoltStringFunctionsSuite.scala
@@ -418,7 +418,7 @@ class BoltStringFunctionsSuite extends
BoltWholeStageTransformerSuite {
s"from $LINEITEM_TABLE where l_comment rlike '%$$##@@#&&' limit
$LENGTH") { _ => }
}
- testWithMinSparkVersion("ilike", "3.3") {
+ test("ilike") {
runQueryAndCompare(
s"select l_orderkey, l_comment, ilike(l_comment, 'a*') " +
s"from $LINEITEM_TABLE limit
$LENGTH")(checkGlutenPlan[ProjectExecTransformer])
@@ -544,7 +544,7 @@ class BoltStringFunctionsSuite extends
BoltWholeStageTransformerSuite {
s"from $LINEITEM_TABLE limit 5") { _ => }
}
- testWithMinSparkVersion("split", "3.4") {
+ test("split") {
runQueryAndCompare(
s"select l_orderkey, l_comment, split(l_comment, '') " +
s"from $LINEITEM_TABLE limit 5") {
diff --git
a/backends-bolt/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala
b/backends-bolt/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala
index cd2626b2e7..d48e27e77a 100644
---
a/backends-bolt/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala
+++
b/backends-bolt/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala
@@ -314,7 +314,7 @@ class MiscOperatorSuite extends
BoltWholeStageTransformerSuite with AdaptiveSpar
checkLengthAndPlan(df, 5)
}
- testWithMinSparkVersion("coalesce validation", "3.4") {
+ test("coalesce validation") {
withTempPath {
path =>
val data = "2019-09-09 01:02:03.456789"
diff --git
a/backends-bolt/src/test/scala/org/apache/gluten/functions/DateFunctionsValidateSuite.scala
b/backends-bolt/src/test/scala/org/apache/gluten/functions/DateFunctionsValidateSuite.scala
index 2a7ad3c270..6270e5d9d2 100644
---
a/backends-bolt/src/test/scala/org/apache/gluten/functions/DateFunctionsValidateSuite.scala
+++
b/backends-bolt/src/test/scala/org/apache/gluten/functions/DateFunctionsValidateSuite.scala
@@ -262,7 +262,7 @@ class DateFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
- testWithMinSparkVersion("timestampadd", "3.3") {
+ test("timestampadd") {
withTempPath {
path =>
val ts = Timestamp.valueOf("2020-02-29 00:00:00.500")
@@ -276,7 +276,7 @@ class DateFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
- testWithMinSparkVersion("timestampdiff", "3.3") {
+ test("timestampdiff") {
withTempPath {
path =>
val t1 = Timestamp.valueOf("2020-03-01 00:00:00.500")
diff --git
a/backends-bolt/src/test/scala/org/apache/gluten/functions/JsonFunctionsValidateSuite.scala
b/backends-bolt/src/test/scala/org/apache/gluten/functions/JsonFunctionsValidateSuite.scala
index 55d53c2c83..ee5147b374 100644
---
a/backends-bolt/src/test/scala/org/apache/gluten/functions/JsonFunctionsValidateSuite.scala
+++
b/backends-bolt/src/test/scala/org/apache/gluten/functions/JsonFunctionsValidateSuite.scala
@@ -81,7 +81,7 @@ class JsonFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
- testWithMinSparkVersion("from_json function bool", "3.4") {
+ test("from_json function bool") {
withTempPath {
path =>
Seq[String](
@@ -103,7 +103,7 @@ class JsonFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
- testWithMinSparkVersion("from_json function small int", "3.4") {
+ test("from_json function small int") {
withTempPath {
path =>
Seq[String](
@@ -125,7 +125,7 @@ class JsonFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
- testWithMinSparkVersion("from_json function int", "3.4") {
+ test("from_json function int") {
withTempPath {
path =>
Seq[String](
@@ -147,7 +147,7 @@ class JsonFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
- testWithMinSparkVersion("from_json function big int", "3.4") {
+ test("from_json function big int") {
withTempPath {
path =>
Seq[String](
@@ -169,7 +169,7 @@ class JsonFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
- testWithMinSparkVersion("from_json function float", "3.4") {
+ test("from_json function float") {
withTempPath {
path =>
Seq[String](
@@ -193,7 +193,7 @@ class JsonFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
- testWithMinSparkVersion("from_json function double", "3.4") {
+ test("from_json function double") {
withTempPath {
path =>
Seq[String](
@@ -217,7 +217,7 @@ class JsonFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
- testWithMinSparkVersion("from_json function string", "3.4") {
+ test("from_json function string") {
withTempPath {
path =>
Seq[String](
@@ -239,7 +239,7 @@ class JsonFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
- testWithMinSparkVersion("from_json function array", "3.4") {
+ test("from_json function array") {
withTempPath {
path =>
Seq[String](
@@ -259,7 +259,7 @@ class JsonFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
- testWithMinSparkVersion("from_json function map", "3.4") {
+ test("from_json function map") {
withTempPath {
path =>
Seq[String](
@@ -281,7 +281,7 @@ class JsonFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
- testWithMinSparkVersion("from_json function row", "3.4") {
+ test("from_json function row") {
withTempPath {
path =>
Seq[String](
@@ -321,7 +321,7 @@ class JsonFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
- testWithMinSparkVersion("from_json function duplicate key", "3.4") {
+ test("from_json function duplicate key") {
withTempPath {
path =>
Seq[String](
diff --git
a/backends-bolt/src/test/scala/org/apache/gluten/functions/MathFunctionsValidateSuite.scala
b/backends-bolt/src/test/scala/org/apache/gluten/functions/MathFunctionsValidateSuite.scala
index 5ed31c776e..b2271b1b41 100644
---
a/backends-bolt/src/test/scala/org/apache/gluten/functions/MathFunctionsValidateSuite.scala
+++
b/backends-bolt/src/test/scala/org/apache/gluten/functions/MathFunctionsValidateSuite.scala
@@ -49,7 +49,7 @@ class MathFunctionsValidateSuiteAnsiOn extends
FunctionsValidateSuite {
}
}
- testWithMinSparkVersion("try_multiply", "3.3") {
+ test("try_multiply") {
runQueryAndCompare(
"select try_multiply(2147483647, cast(l_orderkey as int)), " +
"try_multiply(-2147483648, cast(l_orderkey as int)) from lineitem") {
@@ -57,7 +57,7 @@ class MathFunctionsValidateSuiteAnsiOn extends
FunctionsValidateSuite {
}
}
- testWithMinSparkVersion("try_subtract", "3.3") {
+ test("try_subtract") {
runQueryAndCompare(
"select try_subtract(2147483647, cast(l_orderkey as int)), " +
"try_subtract(-2147483648, cast(l_orderkey as int)) from lineitem") {
@@ -363,7 +363,7 @@ class MathFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
- testWithMinSparkVersion("width_bucket", "3.4") {
+ test("width_bucket") {
withTempPath {
path =>
Seq[(Integer, Integer, Integer, Integer)](
diff --git
a/backends-bolt/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
b/backends-bolt/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
index 026b4d9da6..61c79e0c84 100644
---
a/backends-bolt/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
+++
b/backends-bolt/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
@@ -51,7 +51,7 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
- testWithMinSparkVersion("array_append - INT", "3.4") {
+ test("array_append - INT") {
withTempPath {
path =>
Seq[(Array[Int], Int)](
@@ -76,7 +76,7 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
- testWithMinSparkVersion("array_append - STRING", "3.4") {
+ test("array_append - STRING") {
withTempPath {
path =>
Seq[(Array[String], String)](
@@ -124,7 +124,7 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
- testWithMinSparkVersion("array_compact", "3.4") {
+ test("array_compact") {
withTempPath {
path =>
Seq[Array[String]](
@@ -145,7 +145,7 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
- testWithMinSparkVersion("null input for array_size", "3.3") {
+ test("null input for array_size") {
withTempPath {
path =>
Seq[Array[Int]](
@@ -321,7 +321,7 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
- testWithMinSparkVersion("map_contains_key", "3.3") {
+ test("map_contains_key") {
withTempPath {
path =>
Seq(
@@ -629,7 +629,7 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
- testWithMinSparkVersion("url_decode", "3.4") {
+ test("url_decode") {
withTempPath {
path =>
Seq("https%3A%2F%2Fspark.apache.org")
@@ -643,7 +643,7 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
- testWithMinSparkVersion("url_encode", "3.4") {
+ test("url_encode") {
withTempPath {
path =>
Seq("https://spark.apache.org";)
@@ -680,7 +680,7 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
- testWithMinSparkVersion("mask", "3.4") {
+ test("mask") {
runQueryAndCompare("SELECT mask(c_comment) FROM customer limit 50") {
checkGlutenPlan[ProjectExecTransformer]
}
@@ -883,7 +883,7 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
- testWithMinSparkVersion("get", "3.4") {
+ test("get") {
withTempPath {
path =>
Seq[Seq[Integer]](Seq(1, null, 5, 4), Seq(5, -1, 8, 9, -7, 2),
Seq.empty, null)
@@ -1044,7 +1044,7 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
- testWithMinSparkVersion("array insert", "3.4") {
+ test("array insert") {
withTempPath {
path =>
Seq[Seq[Integer]](Seq(1, null, 5, 4), Seq(5, -1, 8, 9, -7, 2),
Seq.empty, null)
@@ -1091,7 +1091,7 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
- testWithMinSparkVersion("try_cast", "3.4") {
+ test("try_cast") {
withTempView("try_cast_table") {
withTempPath {
path =>
@@ -1475,7 +1475,7 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
- testWithMinSparkVersion("equal_null", "3.4") {
+ test("equal_null") {
Seq[(Integer, Integer)]().toDF("a", "b")
withTempPath {
path =>
diff --git
a/backends-clickhouse/src-iceberg/test/scala/org/apache/gluten/execution/iceberg/ClickHouseIcebergSuite.scala
b/backends-clickhouse/src-iceberg/test/scala/org/apache/gluten/execution/iceberg/ClickHouseIcebergSuite.scala
index 12446c8759..568c67fec6 100644
---
a/backends-clickhouse/src-iceberg/test/scala/org/apache/gluten/execution/iceberg/ClickHouseIcebergSuite.scala
+++
b/backends-clickhouse/src-iceberg/test/scala/org/apache/gluten/execution/iceberg/ClickHouseIcebergSuite.scala
@@ -292,9 +292,7 @@ class ClickHouseIcebergSuite extends
GlutenClickHouseWholeStageTransformerSuite
}
}
- testWithMinSparkVersion(
- "iceberg bucketed join partition value not exists partial cluster",
- "3.4") {
+ test("iceberg bucketed join partition value not exists partial cluster") {
val leftTable = "p_str_tb"
val rightTable = "p_int_tb"
withTable(leftTable, rightTable) {
diff --git
a/backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/VeloxIcebergSuite.scala
b/backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/VeloxIcebergSuite.scala
index c956029392..10617a6f07 100644
---
a/backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/VeloxIcebergSuite.scala
+++
b/backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/VeloxIcebergSuite.scala
@@ -53,7 +53,7 @@ class VeloxIcebergSuite extends IcebergSuite {
}
}
- testWithMinSparkVersion("iceberg v3 initial default for an added column",
"3.4") {
+ test("iceberg v3 initial default for an added column") {
withTable("iceberg_v3_initial_default") {
withSQLConf(GlutenConfig.GLUTEN_ENABLED.key -> "false") {
spark.sql("""
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/execution/FallbackSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/execution/FallbackSuite.scala
index 889e639464..575324162b 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/execution/FallbackSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/execution/FallbackSuite.scala
@@ -310,7 +310,7 @@ class FallbackSuite extends VeloxWholeStageTransformerSuite
with AdaptiveSparkPl
}
}
- testWithMinSparkVersion("fallback with index based schema evolution", "3.4")
{
+ test("fallback with index based schema evolution") {
val query = "SELECT c2 FROM test"
Seq("parquet", "orc").foreach {
format =>
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala
index 7b43ad18e5..adca290e19 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala
@@ -315,7 +315,7 @@ class MiscOperatorSuite extends
VeloxWholeStageTransformerSuite with AdaptiveSpa
checkLengthAndPlan(df, 5)
}
- testWithMinSparkVersion("coalesce validation", "3.4") {
+ test("coalesce validation") {
withTempPath {
path =>
val data = "2019-09-09 01:02:03.456789"
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxAggregateFunctionsSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxAggregateFunctionsSuite.scala
index 20d2cb04f4..6157f356bf 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxAggregateFunctionsSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxAggregateFunctionsSuite.scala
@@ -418,7 +418,7 @@ abstract class VeloxAggregateFunctionsSuite extends
VeloxWholeStageTransformerSu
}
}
- testWithMinSparkVersion("regr_slope", "3.4") {
+ test("regr_slope") {
runQueryAndCompare("""
|select regr_slope(l_partkey, l_suppkey) from
lineitem;
|""".stripMargin) {
@@ -437,7 +437,7 @@ abstract class VeloxAggregateFunctionsSuite extends
VeloxWholeStageTransformerSu
}
}
- testWithMinSparkVersion("regr_intercept", "3.4") {
+ test("regr_intercept") {
runQueryAndCompare("""
|select regr_intercept(l_partkey, l_suppkey) from
lineitem;
|""".stripMargin) {
@@ -456,7 +456,7 @@ abstract class VeloxAggregateFunctionsSuite extends
VeloxWholeStageTransformerSu
}
}
- testWithMinSparkVersion("regr_sxy regr_sxx regr_syy", "3.4") {
+ test("regr_sxy regr_sxx regr_syy") {
runQueryAndCompare("""
|select regr_sxy(l_quantity, l_tax) from lineitem;
|""".stripMargin) {
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxParquetDataTypeValidationSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxParquetDataTypeValidationSuite.scala
index 1408b4c651..d06ca9f84b 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxParquetDataTypeValidationSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxParquetDataTypeValidationSuite.scala
@@ -461,7 +461,7 @@ class VeloxParquetDataTypeValidationSuite extends
VeloxWholeStageTransformerSuit
}
}
- testWithMinSparkVersion("TimestampNTZ type scan", "3.4") {
+ test("TimestampNTZ type scan") {
withTempDir {
dir =>
val path = new File(dir, "ntz_data").toURI.getPath
@@ -475,9 +475,7 @@ class VeloxParquetDataTypeValidationSuite extends
VeloxWholeStageTransformerSuit
}
}
- testWithMinSparkVersion(
- "Schema validation for TimestampNTZ respects enableTimestampNtzValidation",
- "3.4") {
+ test("Schema validation for TimestampNTZ respects
enableTimestampNtzValidation") {
val ntzType = spark.sql("SELECT
TIMESTAMP_NTZ'2024-01-01'").schema.head.dataType
Seq("true", "false").foreach {
enabled =>
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/expression/UDFPartialProjectSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/expression/UDFPartialProjectSuite.scala
index 41cd9ac186..71b60aeefa 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/expression/UDFPartialProjectSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/expression/UDFPartialProjectSuite.scala
@@ -104,7 +104,7 @@ class UDFPartialProjectSuite extends
WholeStageTransformerSuite {
}
}
- testWithMinSparkVersion("test plus_one in nested project lists", "3.4") {
+ test("test plus_one in nested project lists") {
val sql = """
|select plus_one(col1) as col2, l_partkey from (
| select plus_one(l_orderkey) as col1, l_partkey from lineitem
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/functions/DateFunctionsValidateSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/functions/DateFunctionsValidateSuite.scala
index 9eaea0c498..604c455b99 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/functions/DateFunctionsValidateSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/functions/DateFunctionsValidateSuite.scala
@@ -569,7 +569,7 @@ class DateFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
- testWithMinSparkVersion("read as timestamp_ntz", "3.4") {
+ test("read as timestamp_ntz") {
val inputs: Seq[String] = Seq(
"1970-01-01",
"1970-01-01 00:00:00-02:00",
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/functions/JsonFunctionsValidateSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/functions/JsonFunctionsValidateSuite.scala
index 75f5f2fd43..d7566f5e93 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/functions/JsonFunctionsValidateSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/functions/JsonFunctionsValidateSuite.scala
@@ -81,7 +81,7 @@ class JsonFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
- testWithMinSparkVersion("from_json function bool", "3.4") {
+ test("from_json function bool") {
withTempPath {
path =>
Seq[String](
@@ -103,7 +103,7 @@ class JsonFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
- testWithMinSparkVersion("from_json function small int", "3.4") {
+ test("from_json function small int") {
withTempPath {
path =>
Seq[String](
@@ -125,7 +125,7 @@ class JsonFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
- testWithMinSparkVersion("from_json function int", "3.4") {
+ test("from_json function int") {
withTempPath {
path =>
Seq[String](
@@ -147,7 +147,7 @@ class JsonFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
- testWithMinSparkVersion("from_json function big int", "3.4") {
+ test("from_json function big int") {
withTempPath {
path =>
Seq[String](
@@ -169,7 +169,7 @@ class JsonFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
- testWithMinSparkVersion("from_json function float", "3.4") {
+ test("from_json function float") {
withTempPath {
path =>
Seq[String](
@@ -193,7 +193,7 @@ class JsonFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
- testWithMinSparkVersion("from_json function double", "3.4") {
+ test("from_json function double") {
withTempPath {
path =>
Seq[String](
@@ -217,7 +217,7 @@ class JsonFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
- testWithMinSparkVersion("from_json function string", "3.4") {
+ test("from_json function string") {
withTempPath {
path =>
Seq[String](
@@ -239,7 +239,7 @@ class JsonFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
- testWithMinSparkVersion("from_json function array", "3.4") {
+ test("from_json function array") {
withTempPath {
path =>
Seq[String](
@@ -259,7 +259,7 @@ class JsonFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
- testWithMinSparkVersion("from_json function map", "3.4") {
+ test("from_json function map") {
withTempPath {
path =>
Seq[String](
@@ -281,7 +281,7 @@ class JsonFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
- testWithMinSparkVersion("from_json function row", "3.4") {
+ test("from_json function row") {
withTempPath {
path =>
Seq[String](
@@ -321,7 +321,7 @@ class JsonFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
- testWithMinSparkVersion("from_json function duplicate key", "3.4") {
+ test("from_json function duplicate key") {
withTempPath {
path =>
Seq[String](
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/functions/MathFunctionsValidateSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/functions/MathFunctionsValidateSuite.scala
index 9de39c42eb..dae50a7296 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/functions/MathFunctionsValidateSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/functions/MathFunctionsValidateSuite.scala
@@ -415,7 +415,7 @@ class MathFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
- testWithMinSparkVersion("width_bucket", "3.4") {
+ test("width_bucket") {
withTempPath {
path =>
Seq[(Integer, Integer, Integer, Integer)](
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
index 78051d4539..59ea4c0b64 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
@@ -52,7 +52,7 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
- testWithMinSparkVersion("array_append - INT", "3.4") {
+ test("array_append - INT") {
withTempPath {
path =>
Seq[(Array[Int], Int)](
@@ -77,7 +77,7 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
- testWithMinSparkVersion("array_append - STRING", "3.4") {
+ test("array_append - STRING") {
withTempPath {
path =>
Seq[(Array[String], String)](
@@ -125,7 +125,7 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
- testWithMinSparkVersion("array_compact", "3.4") {
+ test("array_compact") {
withTempPath {
path =>
Seq[Array[String]](
@@ -658,7 +658,7 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
- testWithMinSparkVersion("url_decode", "3.4") {
+ test("url_decode") {
withTempPath {
path =>
Seq("https%3A%2F%2Fspark.apache.org")
@@ -672,7 +672,7 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
- testWithMinSparkVersion("url_encode", "3.4") {
+ test("url_encode") {
withTempPath {
path =>
Seq("https://spark.apache.org";)
@@ -688,8 +688,7 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
// Add test suite for CharVarcharCodegenUtils functions.
// A ProjectExecTransformer is expected to be constructed after expr support.
- // We currently test below functions with Spark v3.4
- testWithMinSparkVersion("charTypeWriteSideCheck", "3.4") {
+ test("charTypeWriteSideCheck") {
withTable("src", "dest") {
sql("create table src(id string) USING PARQUET")
@@ -702,7 +701,7 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
- testWithMinSparkVersion("varcharTypeWriteSideCheck", "3.4") {
+ test("varcharTypeWriteSideCheck") {
withTable("src", "dest") {
sql("create table src(id string) USING PARQUET")
@@ -715,7 +714,7 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
- testWithMinSparkVersion("readSidePadding", "3.4") {
+ test("readSidePadding") {
withTable("src", "dest") {
sql("create table tgt(id char(3)) USING PARQUET")
@@ -750,7 +749,7 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
- testWithMinSparkVersion("regexp_instr", "3.4") {
+ test("regexp_instr") {
// Two-argument form.
runQueryAndCompare("SELECT regexp_instr(c_comment, '\\w+') FROM customer
limit 50") {
checkGlutenPlan[ProjectExecTransformer]
@@ -815,7 +814,7 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
- testWithMinSparkVersion("mask", "3.4") {
+ test("mask") {
runQueryAndCompare("SELECT mask(c_comment) FROM customer limit 50") {
checkGlutenPlan[ProjectExecTransformer]
}
@@ -1018,7 +1017,7 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
- testWithMinSparkVersion("get", "3.4") {
+ test("get") {
withTempPath {
path =>
Seq[Seq[Integer]](Seq(1, null, 5, 4), Seq(5, -1, 8, 9, -7, 2),
Seq.empty, null)
@@ -1223,7 +1222,7 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
- testWithMinSparkVersion("array insert", "3.4") {
+ test("array insert") {
withTempPath {
path =>
Seq[Seq[Integer]](Seq(1, null, 5, 4), Seq(5, -1, 8, 9, -7, 2),
Seq.empty, null)
@@ -1270,7 +1269,7 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
- testWithMinSparkVersion("try_cast", "3.4") {
+ test("try_cast") {
withTempView("try_cast_table") {
withTempPath {
path =>
@@ -1654,7 +1653,7 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
- testWithMinSparkVersion("equal_null", "3.4") {
+ test("equal_null") {
Seq[(Integer, Integer)]().toDF("a", "b")
withTempPath {
path =>
@@ -1715,7 +1714,7 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
- testWithMinSparkVersion("localtimestamp with validation enabled", "3.4") {
+ test("localtimestamp with validation enabled") {
// localtimestamp() is folded to a TimestampNTZType literal by
ComputeCurrentTime. With
// validation enabled, any Project whose output contains TimestampNTZ
falls back to JVM.
// The Project falls back at the top of the plan (above the one
VeloxColumnarToRow), so
@@ -1729,7 +1728,7 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
- testWithMinSparkVersion("localtimestamp with validation disabled", "3.4") {
+ test("localtimestamp with validation disabled") {
// With validation disabled, scans on TimestampNTZ columns are allowed
natively. For
// localtimestamp(), the expression constant-folds to a TimestampNTZType
literal in the
// Project; Gluten only permits native Projects when NTZ appears in
Hour(ntz_col), so
diff --git
a/gluten-delta/src/test/scala/org/apache/gluten/execution/DeltaSuite.scala
b/gluten-delta/src/test/scala/org/apache/gluten/execution/DeltaSuite.scala
index d5fd0bfb7e..87915b7b77 100644
--- a/gluten-delta/src/test/scala/org/apache/gluten/execution/DeltaSuite.scala
+++ b/gluten-delta/src/test/scala/org/apache/gluten/execution/DeltaSuite.scala
@@ -461,7 +461,7 @@ abstract class DeltaSuite extends
WholeStageTransformerSuite {
}
}
- testWithMinSparkVersion("delta: change data feed read with deletion
vectors", "3.4") {
+ test("delta: change data feed read with deletion vectors") {
withTable("delta_cdf_dv") {
spark.sql(s"""
|create table delta_cdf_dv (id int, name string) using delta
@@ -605,7 +605,7 @@ abstract class DeltaSuite extends
WholeStageTransformerSuite {
}
}
- testWithMinSparkVersion("deletion vector", "3.4") {
+ test("deletion vector") {
withTempPath {
p =>
import testImplicits._
@@ -632,7 +632,7 @@ abstract class DeltaSuite extends
WholeStageTransformerSuite {
}
}
- testWithMinSparkVersion("deletion vector on partitioned table", "3.4") {
+ test("deletion vector on partitioned table") {
withTempPath {
p =>
import testImplicits._
@@ -671,7 +671,7 @@ abstract class DeltaSuite extends
WholeStageTransformerSuite {
}
}
- testWithMinSparkVersion("deletion vector on shallow-cloned table", "3.4") {
+ test("deletion vector on shallow-cloned table") {
withTable("dv_clone_source", "dv_clone_target") {
import testImplicits._
// Shallow clone is the case the old data-file walk-up got wrong. The
clone's AddFile paths
@@ -840,10 +840,7 @@ abstract class DeltaSuite extends
WholeStageTransformerSuite {
}
}
- // TIMESTAMP_NTZ was introduced in Spark 3.4 / Delta 2.4
- testWithMinSparkVersion(
- "delta: create table with TIMESTAMP_NTZ and return correct results",
- "3.4") {
+ test("delta: create table with TIMESTAMP_NTZ and return correct results") {
withTable("delta_ntz") {
spark.sql("CREATE TABLE delta_ntz(c1 STRING, c2 TIMESTAMP, c3
TIMESTAMP_NTZ) USING DELTA")
spark.sql("""INSERT INTO delta_ntz VALUES
@@ -858,9 +855,7 @@ abstract class DeltaSuite extends
WholeStageTransformerSuite {
}
}
- testWithMinSparkVersion(
- "delta: TIMESTAMP_NTZ as partition column should fallback and return
correct results",
- "3.4") {
+ test("delta: TIMESTAMP_NTZ as partition column should fallback and return
correct results") {
withTable("delta_ntz_part") {
spark.sql("""CREATE TABLE delta_ntz_part(c1 STRING, c2 TIMESTAMP, c3
TIMESTAMP_NTZ)
|USING DELTA PARTITIONED BY (c3)""".stripMargin)
@@ -886,9 +881,7 @@ abstract class DeltaSuite extends
WholeStageTransformerSuite {
}
}
- testWithMinSparkVersion(
- "delta: filter on TIMESTAMP_NTZ column should fallback and return correct
results",
- "3.4") {
+ test("delta: filter on TIMESTAMP_NTZ column should fallback and return
correct results") {
withTable("delta_ntz_filter") {
spark.sql("CREATE TABLE delta_ntz_filter(id INT, ts TIMESTAMP_NTZ) USING
DELTA")
spark.sql("""INSERT INTO delta_ntz_filter VALUES
@@ -902,9 +895,7 @@ abstract class DeltaSuite extends
WholeStageTransformerSuite {
}
}
- testWithMinSparkVersion(
- "merge with column mapping handles struct field metadata correctly",
- "3.4") {
+ test("merge with column mapping handles struct field metadata correctly") {
withTable("merge_struct_source", "merge_struct_target") {
spark.sql("""
|CREATE TABLE merge_struct_target(
@@ -941,9 +932,7 @@ abstract class DeltaSuite extends
WholeStageTransformerSuite {
}
}
- testWithMinSparkVersion(
- "merge with column mapping handles array-of-struct field metadata
correctly",
- "3.4") {
+ test("merge with column mapping handles array-of-struct field metadata
correctly") {
withTable("merge_arraystruct_source", "merge_arraystruct_target") {
spark.sql("""
|CREATE TABLE merge_arraystruct_target(
@@ -976,9 +965,7 @@ abstract class DeltaSuite extends
WholeStageTransformerSuite {
}
}
- testWithMinSparkVersion(
- "merge with column mapping handles map-of-struct field metadata correctly",
- "3.4") {
+ test("merge with column mapping handles map-of-struct field metadata
correctly") {
withTable("merge_mapstruct_source", "merge_mapstruct_target") {
spark.sql("""
|CREATE TABLE merge_mapstruct_target(
@@ -1011,9 +998,7 @@ abstract class DeltaSuite extends
WholeStageTransformerSuite {
}
}
- testWithMinSparkVersion(
- "merge with column mapping handles nested struct-within-struct field
metadata correctly",
- "3.4") {
+ test("merge with column mapping handles nested struct-within-struct field
metadata correctly") {
withTable("merge_nestedstruct_source", "merge_nestedstruct_target") {
spark.sql("""
|CREATE TABLE merge_nestedstruct_target(
@@ -1044,9 +1029,7 @@ abstract class DeltaSuite extends
WholeStageTransformerSuite {
}
}
- testWithMinSparkVersion(
- "merge with column mapping handles array with null struct elements
correctly",
- "3.4") {
+ test("merge with column mapping handles array with null struct elements
correctly") {
withTable("merge_arraynull_source", "merge_arraynull_target") {
spark.sql("""
|CREATE TABLE merge_arraynull_target(
diff --git
a/gluten-iceberg/src/test/scala/org/apache/gluten/execution/IcebergSuite.scala
b/gluten-iceberg/src/test/scala/org/apache/gluten/execution/IcebergSuite.scala
index 5a194fc9a9..4b97852a5a 100644
---
a/gluten-iceberg/src/test/scala/org/apache/gluten/execution/IcebergSuite.scala
+++
b/gluten-iceberg/src/test/scala/org/apache/gluten/execution/IcebergSuite.scala
@@ -155,7 +155,7 @@ abstract class IcebergSuite extends
WholeStageTransformerSuite {
}
}
- testWithMinSparkVersion("iceberg bucketed join", "3.4") {
+ test("iceberg bucketed join") {
val leftTable = "p_str_tb"
val rightTable = "p_int_tb"
withTable(leftTable, rightTable) {
@@ -229,7 +229,7 @@ abstract class IcebergSuite extends
WholeStageTransformerSuite {
}
}
- testWithMinSparkVersion("iceberg bucketed join with partition", "3.4") {
+ test("iceberg bucketed join with partition") {
val leftTable = "p_str_tb"
val rightTable = "p_int_tb"
withTable(leftTable, rightTable) {
@@ -303,7 +303,7 @@ abstract class IcebergSuite extends
WholeStageTransformerSuite {
}
}
- testWithMinSparkVersion("iceberg bucketed join partition value not exists",
"3.4") {
+ test("iceberg bucketed join partition value not exists") {
val leftTable = "p_str_tb"
val rightTable = "p_int_tb"
withTable(leftTable, rightTable) {
@@ -378,9 +378,7 @@ abstract class IcebergSuite extends
WholeStageTransformerSuite {
}
}
- testWithMinSparkVersion(
- "iceberg bucketed join partition value not exists partial cluster",
- "3.4") {
+ test("iceberg bucketed join partition value not exists partial cluster") {
val leftTable = "p_str_tb"
val rightTable = "p_int_tb"
withTable(leftTable, rightTable) {
@@ -455,7 +453,7 @@ abstract class IcebergSuite extends
WholeStageTransformerSuite {
}
}
- testWithMinSparkVersion("iceberg bucketed join with partition filter",
"3.4") {
+ test("iceberg bucketed join with partition filter") {
val leftTable = "p_str_tb"
val rightTable = "p_int_tb"
withTable(leftTable, rightTable) {
diff --git
a/gluten-ut/test/src/test/scala/org/apache/gluten/expressions/GlutenExpressionMappingSuite.scala
b/gluten-ut/test/src/test/scala/org/apache/gluten/expressions/GlutenExpressionMappingSuite.scala
index 244b060d00..cc8bb5309e 100644
---
a/gluten-ut/test/src/test/scala/org/apache/gluten/expressions/GlutenExpressionMappingSuite.scala
+++
b/gluten-ut/test/src/test/scala/org/apache/gluten/expressions/GlutenExpressionMappingSuite.scala
@@ -94,9 +94,7 @@ class GlutenExpressionMappingSuite
}
}
- testWithMinSparkVersion(
- "GLUTEN-7213: Check fallback reason with CheckOverflowInTableInsert",
- "3.4") {
+ test("GLUTEN-7213: Check fallback reason with CheckOverflowInTableInsert") {
withTable("t1", "t2") {
sql("create table t1 (a float) using parquet")
sql("insert into t1 values(1.1)")
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
