This is an automated email from the ASF dual-hosted git repository.
philo-he pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 3801653fe5 [GLUTEN-10134][VL] Add runQueryAndCheck helper for ANSI
mode tests (#13184)
3801653fe5 is described below
commit 3801653fe50c9946ea83a1dec7155d02fea64a07
Author: rdtr <[email protected]>
AuthorDate: Tue Oct 13 23:23:25 2026 -0700
[GLUTEN-10134][VL] Add runQueryAndCheck helper for ANSI mode tests (#13184)
---
.../apache/gluten/execution/FallbackSuite.scala | 40 +--
.../gluten/execution/MiscOperatorSuite.scala | 21 +-
.../gluten/expression/UDFPartialProjectSuite.scala | 5 +-
.../functions/DateFunctionsValidateSuite.scala | 2 +-
.../functions/MathFunctionsValidateSuite.scala | 4 +-
.../functions/ScalarFunctionsValidateSuite.scala | 280 +++++++++++----------
.../execution/GlutenQueryComparisonTest.scala | 30 +++
7 files changed, 220 insertions(+), 162 deletions(-)
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/execution/FallbackSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/execution/FallbackSuite.scala
index 575324162b..b9279e126f 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/execution/FallbackSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/execution/FallbackSuite.scala
@@ -375,25 +375,29 @@ class FallbackSuite extends
VeloxWholeStageTransformerSuite with AdaptiveSparkPl
}
test("fallback when nested loop join has unsupported expression") {
- GlutenSuiteUtils.withFallbackEventListener(spark.sparkContext) {
- events =>
- val df = spark.sql("""
- |select tmp1.c1, tmp1.c2 from tmp1
- |left join tmp2 on (
- | tmp1.c1 = regexp_extract(tmp2.c1,
'(?<=@)[^.]+(?=\.)', 0)
- | or tmp2.c1 > 10
- |)
- |""".stripMargin)
- df.collect()
- GlutenSuiteUtils.waitUntilEmpty(spark.sparkContext)
+ // The join condition casts strings such as '' to bigint, which fails in
ANSI mode. This test
+ // checks the fallback reason, so run it with ANSI mode disabled.
+ withSQLConf(SQLConf.ANSI_ENABLED.key -> "false") {
+ GlutenSuiteUtils.withFallbackEventListener(spark.sparkContext) {
+ events =>
+ val df = spark.sql("""
+ |select tmp1.c1, tmp1.c2 from tmp1
+ |left join tmp2 on (
+ | tmp1.c1 = regexp_extract(tmp2.c1,
'(?<=@)[^.]+(?=\.)', 0)
+ | or tmp2.c1 > 10
+ |)
+ |""".stripMargin)
+ df.collect()
+ GlutenSuiteUtils.waitUntilEmpty(spark.sparkContext)
- val nestedLoopJoin = find(df.queryExecution.executedPlan) {
- _.isInstanceOf[BroadcastNestedLoopJoinExec]
- }
- assert(nestedLoopJoin.isDefined)
- val fallbackReasons = events.flatMap(_.fallbackNodeToReason.values)
- assert(fallbackReasons.nonEmpty)
- assert(fallbackReasons.forall(_.contains("regexp_extract due to
Pattern")))
+ val nestedLoopJoin = find(df.queryExecution.executedPlan) {
+ _.isInstanceOf[BroadcastNestedLoopJoinExec]
+ }
+ assert(nestedLoopJoin.isDefined)
+ val fallbackReasons = events.flatMap(_.fallbackNodeToReason.values)
+ assert(fallbackReasons.nonEmpty)
+ assert(fallbackReasons.forall(_.contains("regexp_extract due to
Pattern")))
+ }
}
}
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala
index 0eb44db73a..da2c0a569a 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala
@@ -1664,12 +1664,17 @@ class MiscOperatorSuite extends
VeloxWholeStageTransformerSuite with AdaptiveSpa
}
test("Support multi-children count with row construct") {
- runQueryAndCompare(
- """
- |select l_orderkey, count(distinct l_partkey, l_comment),
corr(l_partkey, l_partkey+1)
- |from lineitem group by l_orderkey
- |""".stripMargin
- )(df => checkFallbackOperators(df, 0))
+ // Spark before 4.3 (SPARK-58213) throws DIVIDE_BY_ZERO for corr on a
column with zero variance
+ // in ANSI mode, while Velox returns NULL as Spark 4.3 does. Run with ANSI
mode disabled for
+ // now, see https://github.com/apache/gluten/issues/13255.
+ withSQLConf(SQLConf.ANSI_ENABLED.key -> "false") {
+ runQueryAndCompare(
+ """
+ |select l_orderkey, count(distinct l_partkey, l_comment),
corr(l_partkey, l_partkey+1)
+ |from lineitem group by l_orderkey
+ |""".stripMargin
+ )(df => checkFallbackOperators(df, 0))
+ }
}
test("Remainder with non-foldable right side") {
@@ -1680,7 +1685,7 @@ class MiscOperatorSuite extends
VeloxWholeStageTransformerSuite with AdaptiveSpa
|""".stripMargin)
spark.sql("INSERT INTO TABLE remainder VALUES(0, null)")
- runQueryAndCompare("SELECT c1 % c2 FROM remainder")(df =>
checkFallbackOperators(df, 0))
+ runQueryAndCheck("SELECT c1 % c2 FROM remainder")(df =>
checkFallbackOperators(df, 0))
}
}
@@ -1917,7 +1922,7 @@ class MiscOperatorSuite extends
VeloxWholeStageTransformerSuite with AdaptiveSpa
Seq("2023-01-01", "2023-01-02", "-1", "-111-01-01")
.toDF("dateColumn")
.createOrReplaceTempView("view")
- runQueryAndCompare("SELECT cast(dateColumn as date) from view") {
+ runQueryAndCheck("SELECT cast(dateColumn as date) from view") {
checkGlutenPlan[ProjectExecTransformer]
}
}
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/expression/UDFPartialProjectSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/expression/UDFPartialProjectSuite.scala
index a8287f7e0a..1fd47bcef5 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/expression/UDFPartialProjectSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/expression/UDFPartialProjectSuite.scala
@@ -155,8 +155,9 @@ class UDFPartialProjectSuite extends
WholeStageTransformerSuite {
}
test("udf in agg simple") {
- runQueryAndCompare("""select sum(hash(plus_one(l_extendedprice)) +
hash(l_orderkey) ) as revenue
- | from lineitem""".stripMargin) {
+ runQueryAndCheck(
+ """select sum(hash(plus_one(l_extendedprice)) + hash(l_orderkey) ) as
revenue
+ | from lineitem""".stripMargin) {
checkGlutenPlan[ColumnarPartialProjectExec]
}
}
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/functions/DateFunctionsValidateSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/functions/DateFunctionsValidateSuite.scala
index 420da5db67..72798f58d0 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/functions/DateFunctionsValidateSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/functions/DateFunctionsValidateSuite.scala
@@ -169,7 +169,7 @@ class DateFunctionsValidateSuite extends
FunctionsValidateSuite {
}
test("make_date") {
- runQueryAndCompare(
+ runQueryAndCheck(
"select make_date(2025, 2, 7), make_date(2024, 11, null),
make_date(2024, 11, 50)") {
checkGlutenPlan[ProjectExecTransformer]
}
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/functions/MathFunctionsValidateSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/functions/MathFunctionsValidateSuite.scala
index dae50a7296..13f3f4feea 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/functions/MathFunctionsValidateSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/functions/MathFunctionsValidateSuite.scala
@@ -451,7 +451,7 @@ class MathFunctionsValidateSuite extends
FunctionsValidateSuite {
Seq("true", "false").foreach {
enabled =>
withSQLConf("spark.sql.decimalOperations.allowPrecisionLoss" ->
enabled) {
- runQueryAndCompare("SELECT a - b, a + b, a * b, a / b FROM t") {
+ runQueryAndCheck("SELECT a - b, a + b, a * b, a / b FROM t") {
checkGlutenPlan[ProjectExecTransformer]
}
}
@@ -598,7 +598,7 @@ class MathFunctionsValidateSuite extends
FunctionsValidateSuite {
// Query under the opposite setting -- Gluten must use the captured
context, not SQLConf.
withSQLConf("spark.sql.decimalOperations.allowPrecisionLoss" -> "true") {
- runQueryAndCompare("SELECT * FROM v") {
+ runQueryAndCheck("SELECT * FROM v") {
checkGlutenPlan[ProjectExecTransformer]
}
}
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
index 266ae43290..458ca01b8c 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
@@ -395,7 +395,7 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("map_tbl")
- runQueryAndCompare(
+ runQueryAndCheck(
"select map_zip_with(m1, m2, (k, v1, v2) -> k == v1 + v2) from
map_tbl") {
checkGlutenPlan[ProjectExecTransformer]
}
@@ -497,7 +497,7 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
}
test("isnan") {
- runQueryAndCompare(
+ runQueryAndCheck(
"SELECT isnan(l_orderkey), isnan(cast('NaN' as double)),
isnan(0.0F/0.0F)" +
" from lineitem limit 1") {
checkGlutenPlan[ProjectExecTransformer]
@@ -507,15 +507,15 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
test("conditional") {
Seq("nanvl", "nullif", "nvl").foreach {
func =>
- runQueryAndCompare(s"""
- |SELECT
- | $func(cast('nan' as float), 1f),
- | $func(l_orderkey, cast('null' as double)),
- | $func(cast('null' as double), l_orderkey),
- | $func(l_orderkey, l_orderkey / 0.0d),
- | $func(cast('nan' as float), l_orderkey)
- | from lineitem limit 1
- |""".stripMargin) {
+ runQueryAndCheck(s"""
+ |SELECT
+ | $func(cast('nan' as float), 1f),
+ | $func(l_orderkey, cast('null' as double)),
+ | $func(cast('null' as double), l_orderkey),
+ | $func(l_orderkey, l_orderkey / 0.0d),
+ | $func(cast('nan' as float), l_orderkey)
+ | from lineitem limit 1
+ |""".stripMargin) {
checkGlutenPlan[ProjectExecTransformer]
}
}
@@ -524,15 +524,15 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
test("nvl2") {
Seq("null", "l_orderkey").foreach {
expr =>
- runQueryAndCompare(s"""
- |SELECT
- | nvl2($expr, cast('nan' as float), 1f),
- | nvl2($expr, l_orderkey, cast('null' as
double)),
- | nvl2($expr, cast('null' as double),
l_orderkey),
- | nvl2($expr, l_orderkey, l_orderkey / 0.0d),
- | nvl2($expr, cast('nan' as float), l_orderkey)
- | from lineitem limit 1
- |""".stripMargin) {
+ runQueryAndCheck(s"""
+ |SELECT
+ | nvl2($expr, cast('nan' as float), 1f),
+ | nvl2($expr, l_orderkey, cast('null' as double)),
+ | nvl2($expr, cast('null' as double), l_orderkey),
+ | nvl2($expr, l_orderkey, l_orderkey / 0.0d),
+ | nvl2($expr, cast('nan' as float), l_orderkey)
+ | from lineitem limit 1
+ |""".stripMargin) {
checkGlutenPlan[ProjectExecTransformer]
}
}
@@ -864,7 +864,7 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
}
test("bitwise_and") {
- runQueryAndCompare(
+ runQueryAndCheck(
"select cast(l_orderkey as tinyint) & cast(l_partkey as tinyint)," +
" cast(l_orderkey as int) & cast(l_partkey as int), l_orderkey &
l_partkey" +
" from lineitem") {
@@ -873,7 +873,7 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
}
test("bitwise_not") {
- runQueryAndCompare(
+ runQueryAndCheck(
"select ~(cast(l_orderkey as tinyint)), ~(cast(l_orderkey as int)),
~l_orderkey" +
" from lineitem") {
checkGlutenPlan[ProjectExecTransformer]
@@ -881,7 +881,7 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
}
test("bitwise_or") {
- runQueryAndCompare(
+ runQueryAndCheck(
"select cast(l_orderkey as tinyint) | cast(l_partkey as tinyint)," +
" cast(l_orderkey as int) | cast(l_partkey as int), l_orderkey |
l_partkey" +
" from lineitem") {
@@ -890,7 +890,7 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
}
test("bitwise_xor") {
- runQueryAndCompare(
+ runQueryAndCheck(
"select cast(l_orderkey as tinyint) ^ cast(l_partkey as tinyint)," +
" cast(l_orderkey as int) ^ cast(l_partkey as int), l_orderkey ^
l_partkey" +
" from lineitem") {
@@ -1353,94 +1353,100 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
.write
.parquet(path.getCanonicalPath)
spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("cast_table")
- runQueryAndCompare("select cast(str as bigint) from cast_table") {
+ runQueryAndCheck("select cast(str as bigint) from cast_table") {
checkGlutenPlan[ProjectExecTransformer]
}
- runQueryAndCompare("select cast(str as double) from cast_table") {
+ runQueryAndCheck("select cast(str as double) from cast_table") {
checkGlutenPlan[ProjectExecTransformer]
}
}
}
- runQueryAndCompare("select cast(' 123 ' AS int)") {
+ runQueryAndCheck("select cast(' 123 ' AS int)") {
checkGlutenPlan[ProjectExecTransformer]
}
- runQueryAndCompare("select cast('2147483648' AS int)") {
+ runQueryAndCheck("select cast('2147483648' AS int)") {
checkGlutenPlan[ProjectExecTransformer]
}
- runQueryAndCompare("select cast('12a34' AS int)") {
+ runQueryAndCheck("select cast('12a34' AS int)") {
checkGlutenPlan[ProjectExecTransformer]
}
- runQueryAndCompare("select cast('2023-08-21 ' AS date)") {
+ runQueryAndCheck("select cast('2023-08-21 ' AS date)") {
checkGlutenPlan[ProjectExecTransformer]
}
- runQueryAndCompare("select cast(' true' AS boolean)") {
+ runQueryAndCheck("select cast(' true' AS boolean)") {
checkGlutenPlan[ProjectExecTransformer]
}
- runQueryAndCompare("select cast('null' AS int)") {
+ runQueryAndCheck("select cast('null' AS int)") {
checkGlutenPlan[ProjectExecTransformer]
}
- runQueryAndCompare("select cast('on' AS BOOLEAN)") {
+ runQueryAndCheck("select cast('on' AS BOOLEAN)") {
checkGlutenPlan[ProjectExecTransformer]
}
- runQueryAndCompare("select cast(128 AS DECIMAL(2, 0))") {
+ runQueryAndCheck("select cast(128 AS DECIMAL(2, 0))") {
checkGlutenPlan[ProjectExecTransformer]
}
- runQueryAndCompare("select cast(128 AS TINYINT)") {
+ runQueryAndCheck("select cast(128 AS TINYINT)") {
checkGlutenPlan[ProjectExecTransformer]
}
- runQueryAndCompare("select cast(9223372036854775807 AS int)") {
+ runQueryAndCheck("select cast(9223372036854775807 AS int)") {
checkGlutenPlan[ProjectExecTransformer]
}
- runQueryAndCompare("select cast('123.0' AS INT)") {
+ runQueryAndCheck("select cast('123.0' AS INT)") {
checkGlutenPlan[ProjectExecTransformer]
}
// Cast Array as Array[String]
- runQueryAndCompare("select cast(array(1, null) AS array<string>)") {
+ runQueryAndCheck("select cast(array(1, null) AS array<string>)") {
checkGlutenPlan[ProjectExecTransformer]
}
- runQueryAndCompare("select cast(array(1L, null) AS array<string>)") {
+ runQueryAndCheck("select cast(array(1L, null) AS array<string>)") {
checkGlutenPlan[ProjectExecTransformer]
}
- runQueryAndCompare("select cast(array(1.1d, null) AS array<string>)") {
+ runQueryAndCheck("select cast(array(1.1d, null) AS array<string>)") {
checkGlutenPlan[ProjectExecTransformer]
}
- runQueryAndCompare("select cast(array(false, null) AS array<string>)") {
+ runQueryAndCheck("select cast(array(false, null) AS array<string>)") {
checkGlutenPlan[ProjectExecTransformer]
}
- runQueryAndCompare("select cast(array(date'2024-01-01') AS
array<string>)") {
+ runQueryAndCheck("select cast(array(date'2024-01-01') AS array<string>)") {
checkGlutenPlan[ProjectExecTransformer]
}
- runQueryAndCompare("select cast(array(timestamp'2024-01-01 12:00:00') AS
array<string>)") {
+ runQueryAndCheck(
+ "select cast(array(timestamp'2024-01-01 12:00:00') AS array<string>)") {
checkGlutenPlan[ProjectExecTransformer]
}
// Cast Array[String] to Array
- runQueryAndCompare("select cast(array('123', '-98', 'abc', null) AS
array<tinyint>)") {
+ runQueryAndCheck(
+ "select cast(array('123', '-98', 'abc', null) AS array<tinyint>)") {
checkGlutenPlan[ProjectExecTransformer]
}
- runQueryAndCompare("select cast(array('123', '-98', 'abc', null) AS
array<smallint>)") {
+ runQueryAndCheck(
+ "select cast(array('123', '-98', 'abc', null) AS array<smallint>)") {
checkGlutenPlan[ProjectExecTransformer]
}
- runQueryAndCompare("select cast(array('123', '-98', 'abc', null) AS
array<int>)") {
+ runQueryAndCheck("select cast(array('123', '-98', 'abc', null) AS
array<int>)") {
checkGlutenPlan[ProjectExecTransformer]
}
- runQueryAndCompare("select cast(array('123', '-98', 'abc', null) AS
array<bigint>)") {
+ runQueryAndCheck("select cast(array('123', '-98', 'abc', null) AS
array<bigint>)") {
checkGlutenPlan[ProjectExecTransformer]
}
- runQueryAndCompare("select cast(array('123e-2', '-234.548', 'xyz', null)
AS array<float>)") {
+ runQueryAndCheck(
+ "select cast(array('123e-2', '-234.548', 'xyz', null) AS array<float>)")
{
checkGlutenPlan[ProjectExecTransformer]
}
- runQueryAndCompare("select cast(array('123e-2', '-234.548', 'xyz', null)
AS array<double>)") {
+ runQueryAndCheck(
+ "select cast(array('123e-2', '-234.548', 'xyz', null) AS
array<double>)") {
checkGlutenPlan[ProjectExecTransformer]
}
- runQueryAndCompare("""
- |select
- | cast(
- | array('2023-01-01 12:00:00', '2023-01-02
12:00:00', 'def', null)
- | AS array<timestamp>)
- |""".stripMargin) {
+ runQueryAndCheck(
+ """
+ |select
+ | cast(
+ | array('2023-01-01 12:00:00', '2023-01-02 12:00:00', 'def', null)
+ | AS array<timestamp>)
+ |""".stripMargin) {
checkGlutenPlan[ProjectExecTransformer]
}
- runQueryAndCompare(
+ runQueryAndCheck(
"select cast(array('2024-01-01', '2024-01-02', 'uvw', null) AS
array<date>)") {
checkGlutenPlan[ProjectExecTransformer]
}
@@ -1453,27 +1459,27 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
.write
.parquet(path.getCanonicalPath)
spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("cast_table")
- runQueryAndCompare("select cast(c1 as string) from cast_table") {
+ runQueryAndCheck("select cast(c1 as string) from cast_table") {
checkGlutenPlan[ProjectExecTransformer]
}
}
}
- runQueryAndCompare("select cast(array(1, 2) AS string)") {
+ runQueryAndCheck("select cast(array(1, 2) AS string)") {
checkGlutenPlan[ProjectExecTransformer]
}
- runQueryAndCompare("select cast(array(1L, null) AS string)") {
+ runQueryAndCheck("select cast(array(1L, null) AS string)") {
checkGlutenPlan[ProjectExecTransformer]
}
- runQueryAndCompare("select cast(array(1.1d, null) AS string)") {
+ runQueryAndCheck("select cast(array(1.1d, null) AS string)") {
checkGlutenPlan[ProjectExecTransformer]
}
- runQueryAndCompare("select cast(array(false, null) AS string)") {
+ runQueryAndCheck("select cast(array(false, null) AS string)") {
checkGlutenPlan[ProjectExecTransformer]
}
- runQueryAndCompare("select cast(array(date'2024-01-01') AS string)") {
+ runQueryAndCheck("select cast(array(date'2024-01-01') AS string)") {
checkGlutenPlan[ProjectExecTransformer]
}
- runQueryAndCompare("select cast(array(timestamp'2024-01-01 12:00:00') AS
string)") {
+ runQueryAndCheck("select cast(array(timestamp'2024-01-01 12:00:00') AS
string)") {
checkGlutenPlan[ProjectExecTransformer]
}
// Cast Map
@@ -1488,19 +1494,24 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("byte_map_tbl")
- runQueryAndCompare("select cast(c1 as map<tinyint, double>) from
byte_map_tbl") {
+ runQueryAndCheck(
+ "select cast(c1 as map<tinyint, double>) from byte_map_tbl") {
checkGlutenPlan[ProjectExecTransformer]
}
- runQueryAndCompare("select cast(c1 as map<double, tinyint>) from
byte_map_tbl") {
+ runQueryAndCheck(
+ "select cast(c1 as map<double, tinyint>) from byte_map_tbl") {
checkGlutenPlan[ProjectExecTransformer]
}
- runQueryAndCompare("select cast(c1 as map<tinyint, string>) from
byte_map_tbl") {
+ runQueryAndCheck(
+ "select cast(c1 as map<tinyint, string>) from byte_map_tbl") {
checkGlutenPlan[ProjectExecTransformer]
}
- runQueryAndCompare("select cast(c1 as map<string, tinyint>) from
byte_map_tbl") {
+ runQueryAndCheck(
+ "select cast(c1 as map<string, tinyint>) from byte_map_tbl") {
checkGlutenPlan[ProjectExecTransformer]
}
- runQueryAndCompare("select cast(c1 as map<tinyint, boolean>) from
byte_map_tbl") {
+ runQueryAndCheck(
+ "select cast(c1 as map<tinyint, boolean>) from byte_map_tbl") {
checkGlutenPlan[ProjectExecTransformer]
}
}
@@ -1520,19 +1531,24 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("small_int_map_tbl")
- runQueryAndCompare("select cast(c1 as map<smallint, double>) from
small_int_map_tbl") {
+ runQueryAndCheck(
+ "select cast(c1 as map<smallint, double>) from small_int_map_tbl")
{
checkGlutenPlan[ProjectExecTransformer]
}
- runQueryAndCompare("select cast(c1 as map<double, smallint>) from
small_int_map_tbl") {
+ runQueryAndCheck(
+ "select cast(c1 as map<double, smallint>) from small_int_map_tbl")
{
checkGlutenPlan[ProjectExecTransformer]
}
- runQueryAndCompare("select cast(c1 as map<smallint, string>) from
small_int_map_tbl") {
+ runQueryAndCheck(
+ "select cast(c1 as map<smallint, string>) from small_int_map_tbl")
{
checkGlutenPlan[ProjectExecTransformer]
}
- runQueryAndCompare("select cast(c1 as map<string, smallint>) from
small_int_map_tbl") {
+ runQueryAndCheck(
+ "select cast(c1 as map<string, smallint>) from small_int_map_tbl")
{
checkGlutenPlan[ProjectExecTransformer]
}
- runQueryAndCompare("select cast(c1 as map<smallint, boolean>) from
small_int_map_tbl") {
+ runQueryAndCheck(
+ "select cast(c1 as map<smallint, boolean>) from
small_int_map_tbl") {
checkGlutenPlan[ProjectExecTransformer]
}
}
@@ -1548,10 +1564,10 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("int_map_tbl")
- runQueryAndCompare("select cast(c1 as map<double, int>) from
int_map_tbl") {
+ runQueryAndCheck("select cast(c1 as map<double, int>) from
int_map_tbl") {
checkGlutenPlan[ProjectExecTransformer]
}
- runQueryAndCompare("select cast(c1 as map<string, string>) from
int_map_tbl") {
+ runQueryAndCheck("select cast(c1 as map<string, string>) from
int_map_tbl") {
checkGlutenPlan[ProjectExecTransformer]
}
}
@@ -1567,31 +1583,33 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("float_map_tbl")
- runQueryAndCompare("select cast(c1 as map<float, float>) from
float_map_tbl") {
+ runQueryAndCheck("select cast(c1 as map<float, float>) from
float_map_tbl") {
checkGlutenPlan[ProjectExecTransformer]
}
- runQueryAndCompare("select cast(c1 as map<string, string>) from
float_map_tbl") {
+ runQueryAndCheck(
+ "select cast(c1 as map<string, string>) from float_map_tbl") {
checkGlutenPlan[ProjectExecTransformer]
}
}
}
- runQueryAndCompare("""
- |select
- | cast(
- | map(
- | timestamp '2023-01-01 12:00:00', '2023-01-01
13:00:00',
- | timestamp '2023-01-02 12:00:00', 'xyz')
- | as map<string, timestamp>)
- |""".stripMargin) {
+ runQueryAndCheck("""
+ |select
+ | cast(
+ | map(
+ | timestamp '2023-01-01 12:00:00', '2023-01-01
13:00:00',
+ | timestamp '2023-01-02 12:00:00', 'xyz')
+ | as map<string, timestamp>)
+ |""".stripMargin) {
checkGlutenPlan[ProjectExecTransformer]
}
- runQueryAndCompare("""
- |select
- | cast(
- | map(date '2024-01-01', '2024-01-02', date
'2024-02-01', 'xyz')
- | as map<string, date>)
- |""".stripMargin) {
+ runQueryAndCheck(
+ """
+ |select
+ | cast(
+ | map(date '2024-01-01', '2024-01-02', date '2024-02-01', 'xyz')
+ | as map<string, date>)
+ |""".stripMargin) {
checkGlutenPlan[ProjectExecTransformer]
}
// Cast struct
@@ -1624,52 +1642,52 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("struct_tbl")
- runQueryAndCompare("""
- |select
- | cast(
- | c1 as
- | struct<
- | a: array<bigint>,
- | b: map<smallint, int>,
- | c: struct<x: string, y: string,
z:boolean>>)
- |from struct_tbl
- |""".stripMargin) {
+ runQueryAndCheck("""
+ |select
+ | cast(
+ | c1 as
+ | struct<
+ | a: array<bigint>,
+ | b: map<smallint, int>,
+ | c: struct<x: string, y: string,
z:boolean>>)
+ |from struct_tbl
+ |""".stripMargin) {
checkGlutenPlan[ProjectExecTransformer]
}
- runQueryAndCompare("""
- |select
- | cast(
- | c1 as
- | struct<
- | a: array<double>,
- | b: map<int, bigint>,
- | c: struct<x: int, y: boolean,
z:string>>)
- |from struct_tbl
- |""".stripMargin) {
+ runQueryAndCheck("""
+ |select
+ | cast(
+ | c1 as
+ | struct<
+ | a: array<double>,
+ | b: map<int, bigint>,
+ | c: struct<x: int, y: boolean, z:string>>)
+ |from struct_tbl
+ |""".stripMargin) {
checkGlutenPlan[ProjectExecTransformer]
}
- runQueryAndCompare("""
- |select
- | cast(
- | c1 as
- | struct<
- | a: array<timestamp>,
- | b: map<bigint, boolean>,
- | c: struct<x: tinyint, y: smallint,
z:double>>)
- |from struct_tbl
- |""".stripMargin) {
+ runQueryAndCheck("""
+ |select
+ | cast(
+ | c1 as
+ | struct<
+ | a: array<timestamp>,
+ | b: map<bigint, boolean>,
+ | c: struct<x: tinyint, y: smallint,
z:double>>)
+ |from struct_tbl
+ |""".stripMargin) {
checkGlutenPlan[ProjectExecTransformer]
}
- runQueryAndCompare("""
- |select
- | cast(
- | c1 as
- | struct<
- | a: array<date>,
- | b: map<string, double>,
- | c: struct<x: int, y: bigint, z:float>>)
- |from struct_tbl
- |""".stripMargin) {
+ runQueryAndCheck("""
+ |select
+ | cast(
+ | c1 as
+ | struct<
+ | a: array<date>,
+ | b: map<string, double>,
+ | c: struct<x: int, y: bigint, z:float>>)
+ |from struct_tbl
+ |""".stripMargin) {
checkGlutenPlan[ProjectExecTransformer]
}
}
diff --git
a/gluten-substrait/src/test/scala/org/apache/gluten/execution/GlutenQueryComparisonTest.scala
b/gluten-substrait/src/test/scala/org/apache/gluten/execution/GlutenQueryComparisonTest.scala
index e175b314bd..b13852b572 100644
---
a/gluten-substrait/src/test/scala/org/apache/gluten/execution/GlutenQueryComparisonTest.scala
+++
b/gluten-substrait/src/test/scala/org/apache/gluten/execution/GlutenQueryComparisonTest.scala
@@ -28,6 +28,8 @@ import org.apache.spark.sql.{DataFrame, GlutenQueryTest, Row}
import java.util.concurrent.atomic.AtomicBoolean
+import scala.util.Try
+
abstract class GlutenQueryComparisonTest extends GlutenQueryTest {
private val isFallbackCheckDisabled0 = new AtomicBoolean(false)
@@ -72,6 +74,34 @@ abstract class GlutenQueryComparisonTest extends
GlutenQueryTest {
cache)
}
+ /**
+ * Like `runQueryAndCompare`, but in ANSI mode also accepts a query that
fails on vanilla Spark,
+ * which happens when ANSI mode rejects its input (for example, a malformed
string cast to a
+ * number). In that case, checks that Gluten fails too.
+ */
+ protected def runQueryAndCheck(
+ sqlStr: String,
+ compareResult: Boolean = true,
+ noFallBack: Boolean = true,
+ cache: Boolean = false)(customCheck: DataFrame => Unit): Unit = {
+ if (spark.sessionState.conf.ansiEnabled) {
+ // withSQLConf returns Unit before Spark 4.0, so the result is captured
in a var.
+ var vanillaFailed = false
+ withSQLConf(vanillaSparkConfs(): _*) {
+ vanillaFailed = Try(spark.sql(sqlStr).collect()).isFailure
+ }
+ if (vanillaFailed) {
+ // TODO: Also compare the error class once native errors are
translated into Spark
+ // exceptions.
+ withClue(s"Vanilla Spark failed but Gluten did not for query:
$sqlStr\n") {
+ intercept[Exception](spark.sql(sqlStr).collect())
+ }
+ return
+ }
+ }
+ runQueryAndCompare(sqlStr, compareResult, noFallBack, cache)(customCheck)
+ }
+
protected def compareResultsAgainstVanillaSpark(
sql: String,
compareResult: Boolean = true,
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
