This is an automated email from the ASF dual-hosted git repository.
zhouyuan pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 712976be25 [VL] Enforce format_number restrictions on the JVM side
(#13137)
712976be25 is described below
commit 712976be25025cbb916defd27d0a218e78a73388
Author: Anshul Baliga <[email protected]>
AuthorDate: Tue Sep 29 00:15:49 2026 +0530
[VL] Enforce format_number restrictions on the JVM side (#13137)
---
.../backendsapi/velox/VeloxSparkPlanExecApi.scala | 24 +++++++++++++++
.../functions/ScalarFunctionsValidateSuite.scala | 35 ++++++++++++++++++----
.../gluten/backendsapi/SparkPlanExecApi.scala | 7 +++++
.../gluten/expression/ExpressionConverter.scala | 5 ++++
4 files changed, 65 insertions(+), 6 deletions(-)
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
index 383fce140a..b862107075 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
@@ -1151,6 +1151,30 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi
with Logging {
GenericExpressionTransformer(substraitExprName, children, expr)
}
+ override def genFormatNumberTransformer(
+ substraitExprName: String,
+ children: Seq[ExpressionTransformer],
+ expr: Expression): ExpressionTransformer = {
+ // Velox registers format_number only for integral and floating-point
input with an integer
+ // number of decimal places. Reject the other Spark forms here so the
fallback reason names
+ // the documented restriction instead of a generic native validation
failure.
+ expr.children.head.dataType match {
+ case _: DecimalType =>
+ GlutenExceptionUtil.throwsNotFullySupported(
+ ExpressionNames.FORMAT_NUMBER,
+ FormatNumberRestrictions.NOT_SUPPORT_DECIMAL_INPUT)
+ case _ =>
+ }
+ expr.children(1).dataType match {
+ case _: StringType =>
+ GlutenExceptionUtil.throwsNotFullySupported(
+ ExpressionNames.FORMAT_NUMBER,
+ FormatNumberRestrictions.NOT_SUPPORT_STRING_FORMAT)
+ case _ =>
+ }
+ GenericExpressionTransformer(substraitExprName, children, expr)
+ }
+
/** Generate an expression transformer to transform JsonToStructs to
Substrait. */
override def genFromJsonTransformer(
substraitExprName: String,
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
index 59ea4c0b64..266ae43290 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
@@ -18,9 +18,11 @@ package org.apache.gluten.functions
import org.apache.gluten.config.GlutenConfig
import org.apache.gluten.execution.{BatchScanExecTransformer,
FilterExecTransformer, ProjectExecTransformer}
+import org.apache.gluten.expression.FormatNumberRestrictions
+import org.apache.gluten.extension.columnar.FallbackTags
import org.apache.spark.SparkException
-import org.apache.spark.sql.Row
+import org.apache.spark.sql.{DataFrame, Row}
import org.apache.spark.sql.catalyst.optimizer.NullPropagation
import org.apache.spark.sql.execution.ProjectExec
import org.apache.spark.sql.internal.SQLConf
@@ -32,6 +34,18 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
import testImplicits._
+ // Collects the fallback reasons recorded on the executed plan.
GlutenFallbackReporter moves the
+ // tag from the physical node to its logical link, so read both places like
the reporter does.
+ private def fallbackReasons(df: DataFrame): Seq[String] = {
+ getExecutedPlan(df).flatMap {
+ p =>
+ FallbackTags
+ .getOption(p)
+ .orElse(p.logicalLink.flatMap(FallbackTags.getOption))
+ .map(_.reason())
+ }
+ }
+
// Test "SELECT ..." without a from clause.
test("isnull") {
runQueryAndCompare("SELECT
isnull(1)")(checkGlutenPlan[ProjectExecTransformer])
@@ -801,16 +815,25 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
"SELECT format_number(cast(l_discount as double), 3) FROM lineitem limit
50") {
checkGlutenPlan[ProjectExecTransformer]
}
- // Velox format_number only supports
tinyint/smallint/integer/bigint/float/double.
- // Decimal input has no matching signature, so it must fall back to
vanilla Spark.
+ // Velox format_number only supports
tinyint/smallint/integer/bigint/float/double. Decimal
+ // input is rejected on the JVM side, so it falls back to vanilla Spark
and the fallback
+ // reason names the documented restriction instead of a generic native
validation failure.
runQueryAndCompare("SELECT format_number(l_quantity, 1) FROM lineitem
limit 50") {
- checkSparkPlan[ProjectExec]
+ df =>
+ checkSparkPlan[ProjectExec](df)
+ assert(
+ fallbackReasons(df).exists(
+ _.contains(FormatNumberRestrictions.NOT_SUPPORT_DECIMAL_INPUT)))
}
// Velox only implements the integer decimal-places form. The
string-format form
- // (e.g. '#,###.##') has no matching signature, so it must fall back to
vanilla Spark.
+ // (e.g. '#,###.##') is rejected on the JVM side in the same way.
runQueryAndCompare(
"SELECT format_number(cast(l_quantity as double), '#,###.##') FROM
lineitem limit 50") {
- checkSparkPlan[ProjectExec]
+ df =>
+ checkSparkPlan[ProjectExec](df)
+ assert(
+ fallbackReasons(df).exists(
+ _.contains(FormatNumberRestrictions.NOT_SUPPORT_STRING_FORMAT)))
}
}
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala
index 394ee6afd7..916c31c2ed 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala
@@ -182,6 +182,13 @@ trait SparkPlanExecApi {
GenericExpressionTransformer(substraitExprName, children, expr)
}
+ def genFormatNumberTransformer(
+ substraitExprName: String,
+ children: Seq[ExpressionTransformer],
+ expr: Expression): ExpressionTransformer = {
+ GenericExpressionTransformer(substraitExprName, children, expr)
+ }
+
def genFromJsonTransformer(
substraitExprName: String,
children: Seq[ExpressionTransformer],
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala
index f5595437bf..7827055969 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala
@@ -647,6 +647,11 @@ object ExpressionConverter extends SQLConfHelper with
Logging {
substraitExprName,
expr.children.map(replaceWithExpressionTransformer0(_, attributeSeq,
expressionsMap)),
expr)
+ case _: FormatNumber =>
+
BackendsApiManager.getSparkPlanExecApiInstance.genFormatNumberTransformer(
+ substraitExprName,
+ expr.children.map(replaceWithExpressionTransformer0(_, attributeSeq,
expressionsMap)),
+ expr)
case c: CheckOverflow =>
CheckOverflowTransformer(
substraitExprName,
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
