This is an automated email from the ASF dual-hosted git repository.
philo-he pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 5d9bced5d4 [GLUTEN-12157][VL] Fix silently-skipped math/scalar test
suites and add Velox native tests for sin, tan, tanh, radians, ln (#12158)
5d9bced5d4 is described below
commit 5d9bced5d4c80115e4adfa5c82948615a5a31277
Author: BRIJ RAJ KISHORE <[email protected]>
AuthorDate: Thu Jul 9 11:04:02 2026 +0530
[GLUTEN-12157][VL] Fix silently-skipped math/scalar test suites and add
Velox native tests for sin, tan, tanh, radians, ln (#12158)
---
.../functions/MathFunctionsValidateSuite.scala | 35 ++++++++
.../functions/ScalarFunctionsValidateSuite.scala | 99 ++++++++++------------
2 files changed, 79 insertions(+), 55 deletions(-)
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/functions/MathFunctionsValidateSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/functions/MathFunctionsValidateSuite.scala
index 7d3907ca31..81a9ad5cdb 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/functions/MathFunctionsValidateSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/functions/MathFunctionsValidateSuite.scala
@@ -248,6 +248,17 @@ class MathFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
+ test("ln") {
+ runQueryAndCompare("SELECT ln(l_orderkey) from lineitem limit 1") {
+ checkGlutenPlan[ProjectExecTransformer]
+ }
+ // Verify null semantics: ln(0) and ln(-1) must return null, not
-Infinity/NaN
+ compareResultsAgainstVanillaSpark(
+ "SELECT ln(0), ln(-1), ln(cast(null as double))",
+ true,
+ { _ => })
+ }
+
test("log") {
runQueryAndCompare("SELECT log(10, l_orderkey) from lineitem limit 1") {
checkGlutenPlan[ProjectExecTransformer]
@@ -295,6 +306,12 @@ class MathFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
+ test("radians") {
+ runQueryAndCompare("SELECT radians(l_orderkey) from lineitem limit 1") {
+ checkGlutenPlan[ProjectExecTransformer]
+ }
+ }
+
test("rint") {
withTempPath {
path =>
@@ -332,6 +349,24 @@ class MathFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
+ test("sin") {
+ runQueryAndCompare("SELECT sin(l_orderkey) from lineitem limit 1") {
+ checkGlutenPlan[ProjectExecTransformer]
+ }
+ }
+
+ test("tan") {
+ runQueryAndCompare("SELECT tan(l_orderkey) from lineitem limit 1") {
+ checkGlutenPlan[ProjectExecTransformer]
+ }
+ }
+
+ test("tanh") {
+ runQueryAndCompare("SELECT tanh(l_orderkey) from lineitem limit 1") {
+ checkGlutenPlan[ProjectExecTransformer]
+ }
+ }
+
test("try_add") {
runQueryAndCompare(
"select try_add(cast(l_orderkey as int), 1), try_add(cast(l_orderkey as
int), 2147483647)" +
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
index 74b2933a67..d3afacce53 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
@@ -26,7 +26,8 @@ import org.apache.spark.sql.execution.ProjectExec
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types._
-abstract class ScalarFunctionsValidateSuite extends FunctionsValidateSuite {
+class ScalarFunctionsValidateSuite extends FunctionsValidateSuite {
+
disableFallbackCheck
import testImplicits._
@@ -227,23 +228,23 @@ abstract class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
sql("INSERT INTO t1 VALUES(1, NOW())")
runQueryAndCompare("SELECT c1, HOUR(c2) FROM t1 LIMIT 1")(df =>
checkFallbackOperators(df, 0))
}
+ }
- test("MINUTE") {
- withTable("t1") {
- sql("create table t1 (c1 int, c2 timestamp) USING PARQUET")
- sql("INSERT INTO t1 VALUES(1, NOW())")
- runQueryAndCompare("SELECT c1, MINUTE(c2) FROM t1 LIMIT 1")(
- df => checkFallbackOperators(df, 0))
- }
+ test("MINUTE") {
+ withTable("t1") {
+ sql("create table t1 (c1 int, c2 timestamp) USING PARQUET")
+ sql("INSERT INTO t1 VALUES(1, NOW())")
+ runQueryAndCompare("SELECT c1, MINUTE(c2) FROM t1 LIMIT 1")(
+ df => checkFallbackOperators(df, 0))
}
+ }
- test("SECOND") {
- withTable("t1") {
- sql("create table t1 (c1 int, c2 timestamp) USING PARQUET")
- sql("INSERT INTO t1 VALUES(1, NOW())")
- runQueryAndCompare("SELECT c1, SECOND(c2) FROM t1 LIMIT 1")(
- df => checkFallbackOperators(df, 0))
- }
+ test("SECOND") {
+ withTable("t1") {
+ sql("create table t1 (c1 int, c2 timestamp) USING PARQUET")
+ sql("INSERT INTO t1 VALUES(1, NOW())")
+ runQueryAndCompare("SELECT c1, SECOND(c2) FROM t1 LIMIT 1")(
+ df => checkFallbackOperators(df, 0))
}
}
@@ -1593,61 +1594,49 @@ abstract class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
test("current_timestamp") {
- withSQLConf(
- "spark.sql.optimizer.excludedRules" ->
- "org.apache.spark.sql.catalyst.optimizer.ConstantFolding") {
- runQueryAndCompare("SELECT l_orderkey, current_timestamp() from lineitem
limit 1") {
- df =>
- val optimizedPlan = df.queryExecution.optimizedPlan.toString()
- assert(
- optimizedPlan.contains("CurrentTimestamp"),
- s"Expected CurrentTimestamp in plan when ConstantFolding is
disabled, " +
- s"but got: $optimizedPlan"
- )
- checkGlutenPlan[ProjectExecTransformer](df)
- }
+ // current_timestamp() is folded to a wall-clock literal by
ComputeCurrentTime (an always-on
+ // optimizer batch that excludedRules cannot suppress). Each run captures
a different instant,
+ // so result comparison across two executions always fails. Skip result
comparison but still
+ // assert that the Project offloads natively.
+ runQueryAndCompare(
+ "SELECT l_orderkey, current_timestamp() from lineitem limit 1",
+ compareResult = false) {
+ checkGlutenPlan[ProjectExecTransformer]
}
}
test("now") {
- withSQLConf(
- "spark.sql.optimizer.excludedRules" ->
- "org.apache.spark.sql.catalyst.optimizer.ConstantFolding") {
- runQueryAndCompare("SELECT l_orderkey, now() from lineitem limit 1") {
- df =>
- val optimizedPlan = df.queryExecution.optimizedPlan.toString()
- assert(
- optimizedPlan.contains("Now"),
- s"Expected Now in plan when ConstantFolding is disabled, but got:
$optimizedPlan"
- )
- checkGlutenPlan[ProjectExecTransformer](df)
- }
+ // now() is an alias for current_timestamp() -- same constant-folding
behaviour.
+ runQueryAndCompare(
+ "SELECT l_orderkey, now() from lineitem limit 1",
+ compareResult = false) {
+ checkGlutenPlan[ProjectExecTransformer]
}
}
testWithMinSparkVersion("localtimestamp with validation enabled", "3.4") {
- // With validation enabled (default), localtimestamp should fallback to
Spark
- // because it returns TimestampNTZType
-
withSQLConf("spark.gluten.sql.columnar.backend.velox.enableTimestampNtzValidation"
-> "true") {
+ // localtimestamp() is folded to a TimestampNTZType literal by
ComputeCurrentTime. With
+ // validation enabled, any Project whose output contains TimestampNTZ
falls back to JVM.
+ // The Project falls back at the top of the plan (above the one
VeloxColumnarToRow), so
+ // the extra-transition count is 0 (1 ColumnarToRow - 1 baseline = 0).
+ withSQLConf(
+ "spark.gluten.sql.columnar.backend.velox.enableTimestampNtzValidation"
-> "true"
+ ) {
val df = spark.sql("SELECT l_orderkey, localtimestamp() from lineitem
limit 1")
- // Should fallback to Spark execution due to TimestampNTZ validation
- checkFallbackOperators(df, 1)
+ checkFallbackOperators(df, 0)
df.collect()
}
}
testWithMinSparkVersion("localtimestamp with validation disabled", "3.4") {
- // With validation disabled, localtimestamp can use native execution
- // This allows developers to test TimestampNTZ support
-
withSQLConf("spark.gluten.sql.columnar.backend.velox.enableTimestampNtzValidation"
-> "false") {
+ // With validation disabled, scans on TimestampNTZ columns are allowed
natively. For
+ // localtimestamp(), the expression constant-folds to a TimestampNTZType
literal in the
+ // Project; Gluten only permits native Projects when NTZ appears in
Hour(ntz_col), so
+ // this Project still falls back -- same extra-transition count as the
validation-enabled case.
+ withSQLConf(
+ "spark.gluten.sql.columnar.backend.velox.enableTimestampNtzValidation"
-> "false"
+ ) {
val df = spark.sql("SELECT l_orderkey, localtimestamp() from lineitem
limit 1")
- val optimizedPlan = df.queryExecution.optimizedPlan.toString()
- assert(
- !optimizedPlan.contains("LocalTimestamp"),
- s"Expected LocalTimestamp to be folded to a literal, but got:
$optimizedPlan"
- )
- // Should use native execution when validation is disabled
- checkGlutenPlan[ProjectExecTransformer](df)
checkFallbackOperators(df, 0)
df.collect()
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
