This is an automated email from the ASF dual-hosted git repository.

weiting-chen pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git


The following commit(s) were added to refs/heads/main by this push:
     new 493bcf4730 [VL] Add native Spark format_string and printf support 
(#13223)
493bcf4730 is described below

commit 493bcf473045a16b79899b00baf8e2d60cf5a46f
Author: Minni Mittal <[email protected]>
AuthorDate: Tue Nov 10 09:59:42 2026 +0530

    [VL] Add native Spark format_string and printf support (#13223)
    
    * [VL] Add native Spark format_string and printf support
    
    Validate format_string patterns and argument types against the subset
    implemented by Velox, falling back to Spark for unsupported or unsafe cases.
    Add native and fallback coverage for format_string and printf.
    
    Co-authored-by: Copilot <[email protected]>
    Copilot-Session: 309a2429-55f4-46e7-9229-4c2be58b1598
    
    * [VL] Document format width validation limit
    
    Explain that the validator's width limit follows Velox's per-specifier
    output cap.
    
    Co-authored-by: Copilot <[email protected]>
    Copilot-Session: 309a2429-55f4-46e7-9229-4c2be58b1598
    
    * [VL] Cover printf fallback validation
    
    ## Summary
    Exercise the existing compatibility-subset and malformed-format fallback
    cases through both format_string and printf.
    
    ## Prompting Intent
    Address the missing printf fallback coverage identified in PR #13223,
    keeping the follow-up limited to tests.
    
    ## Linked Sources
    - PR: https://github.com/apache/gluten/pull/13223
    - Review: https://github.com/apache/gluten/pull/13223#discussion_r4268439755
    
    ## Rationale
    Share the same cases across both SQL names instead of duplicating tests.
    Preserve plan-only assertions for unsafe or malformed formats so invalid
    queries are never executed during fallback checks.
    
    Co-authored-by: Copilot <[email protected]>
    Copilot-Session: 1ee80ef9-575d-4e21-8139-b5951e735292
    
    * [VL] Address format string review feedback
    
    ## Summary
    Compare executable fallback cases against vanilla Spark, separate maximum
    width planning coverage from bounded runtime padding checks, and clarify
    format_string and printf validation.
    
    ## Prompting Intent
    Address the outstanding review findings on PR #13223 while preserving the
    supported formatting subset and avoiding execution of malformed formats.
    
    ## Linked Sources
    - PR: https://github.com/apache/gluten/pull/13223
    - Alias validation: 
https://github.com/apache/gluten/pull/13223#discussion_r4268439600
    - Explicit import reference: 
https://github.com/apache/gluten/pull/13223#discussion_r4292446998
    - Fallback parity: 
https://github.com/apache/gluten/pull/13223#discussion_r4292447082
    - Width coverage: 
https://github.com/apache/gluten/pull/13223#discussion_r4292447127
    
    ## Rationale
    Reuse the existing query-comparison harness to check both result parity
    and fallback planning. Exercise the maximum supported width without
    allocating its output and compare complete 4096-character padded results
    for both SQL names. Keep malformed and over-limit cases planning-only.
    Qualify the parser buffer reference and document shared Catalyst alias
    resolution without changing validator behavior.
    
    Co-authored-by: Copilot <[email protected]>
    Copilot-Session: 1ee80ef9-575d-4e21-8139-b5951e735292
    
    ---------
    
    Co-authored-by: Copilot <[email protected]>
    Co-authored-by: minni31 <[email protected]>
    Copilot-Session: 309a2429-55f4-46e7-9229-4c2be58b1598
    Copilot-Session: 1ee80ef9-575d-4e21-8139-b5951e735292
---
 .../backendsapi/velox/VeloxValidatorApi.scala      | 158 +++++++++++++-
 .../gluten/execution/VeloxFormatStringSuite.scala  | 237 +++++++++++++++++++++
 2 files changed, 392 insertions(+), 3 deletions(-)

diff --git 
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxValidatorApi.scala
 
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxValidatorApi.scala
index e6f7a0fd92..5089e4104a 100644
--- 
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxValidatorApi.scala
+++ 
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxValidatorApi.scala
@@ -27,11 +27,12 @@ import org.apache.gluten.substrait.plan.PlanNode
 import org.apache.gluten.validate.NativePlanValidationInfo
 import org.apache.gluten.vectorized.NativePlanEvaluator
 
-import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression}
+import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression, 
FormatString, Literal}
 import org.apache.spark.sql.catalyst.plans.physical.Partitioning
 import org.apache.spark.sql.execution.SparkPlan
 import org.apache.spark.sql.types._
 import org.apache.spark.task.TaskResources
+import org.apache.spark.unsafe.types.UTF8String
 
 import io.substrait.proto.SimpleExtensionDeclaration
 
@@ -42,8 +43,14 @@ class VeloxValidatorApi extends ValidatorApi {
   import VeloxValidatorApi._
 
   /** For velox backend, key validation is on native side. */
-  override def doExprValidate(substraitExprName: String, expr: Expression): 
Boolean =
-    true
+  override def doExprValidate(substraitExprName: String, expr: Expression): 
Boolean = {
+    expr match {
+      // Both format_string and printf resolve to Spark's FormatString 
expression.
+      case formatString: FormatString =>
+        supportsFormatString(formatString)
+      case _ => true
+    }
+  }
 
   override def doNativeValidateWithFailureReason(plan: PlanNode): 
ValidationResult = {
     TaskResources.runUnsafe {
@@ -104,6 +111,151 @@ class VeloxValidatorApi extends ValidatorApi {
 }
 
 object VeloxValidatorApi {
+  private val nativeFormatFlags = Set('-', '+', ' ', '0')
+  // Match Velox's per-specifier output limit to prevent excessive allocations.
+  private val maxNativeFormatWidth = 1 << 20
+
+  private def supportsFormatString(formatString: FormatString): Boolean = {
+    val format = formatString.children.headOption match {
+      case Some(Literal(value: UTF8String, dataType))
+          if dataType.isInstanceOf[StringType] =>
+        value.toString
+      case _ =>
+        return false
+    }
+    val conversions = parseNativeFormatString(format).getOrElse {
+      return false
+    }
+    conversions.length <= formatString.children.length - 1 &&
+    conversions
+      .zip(formatString.children.tail)
+      .forall {
+        case (conversion, argument) =>
+          supportsNativeFormatType(conversion, argument.dataType)
+      }
+  }
+
+  private def parseNativeFormatString(format: String): Option[Seq[Char]] = {
+    val conversions = scala.collection.mutable.ArrayBuffer.empty[Char]
+    var index = 0
+    while (index < format.length) {
+      if (format.charAt(index) != '%') {
+        index += 1
+      } else if (index + 1 < format.length && format.charAt(index + 1) == '%') 
{
+        index += 2
+      } else {
+        index += 1
+        val flags = scala.collection.mutable.Set.empty[Char]
+        while (index < format.length && 
nativeFormatFlags.contains(format.charAt(index))) {
+          if (!flags.add(format.charAt(index))) {
+            return None
+          }
+          index += 1
+        }
+        val widthStart = index
+        while (
+          index < format.length && format.charAt(index) >= '0' &&
+          format.charAt(index) <= '9'
+        ) {
+          index += 1
+        }
+        val width =
+          if (widthStart == index) None
+          else parseBoundedFormatNumber(format, widthStart, index)
+        if (widthStart != index && width.isEmpty) {
+          return None
+        }
+        var precision: Option[Int] = None
+        if (index < format.length && format.charAt(index) == '.') {
+          index += 1
+          val precisionStart = index
+          while (
+            index < format.length && format.charAt(index) >= '0' &&
+            format.charAt(index) <= '9'
+          ) {
+            index += 1
+          }
+          if (precisionStart == index) {
+            return None
+          }
+          precision = parseBoundedFormatNumber(format, precisionStart, index)
+          if (precision.isEmpty) {
+            return None
+          }
+        }
+        if (index >= format.length) {
+          return None
+        }
+        val conversion = format.charAt(index)
+        index += 1
+        if (!isNativeFormatSpecifierSupported(conversion, flags.toSet, width, 
precision)) {
+          return None
+        }
+        conversions += conversion
+      }
+    }
+    Some(conversions.toSeq)
+  }
+
+  private def parseBoundedFormatNumber(format: String, start: Int, end: Int): 
Option[Int] = {
+    var value = 0
+    var index = start
+    while (index < end) {
+      val digit = format.charAt(index) - '0'
+      if (value > (maxNativeFormatWidth - digit) / 10) {
+        return None
+      }
+      value = value * 10 + digit
+      index += 1
+    }
+    Some(value)
+  }
+
+  private def isNativeFormatSpecifierSupported(
+      conversion: Char,
+      flags: Set[Char],
+      width: Option[Int],
+      precision: Option[Int]): Boolean = {
+    if ((flags.contains('-') || flags.contains('0')) && width.isEmpty) {
+      return false
+    }
+    if (
+      (flags.contains('-') && flags.contains('0')) ||
+      (flags.contains('+') && flags.contains(' '))
+    ) {
+      return false
+    }
+    conversion match {
+      case 's' => flags.isEmpty && width.isEmpty && precision.isEmpty
+      case 'd' => precision.isEmpty
+      case 'o' | 'x' | 'X' =>
+        precision.isEmpty && !flags.contains('+') && !flags.contains(' ')
+      case _ => false
+    }
+  }
+
+  private def supportsNativeFormatType(conversion: Char, dataType: DataType): 
Boolean = {
+    if (dataType == NullType) {
+      true
+    } else {
+      conversion match {
+        case 's' =>
+          dataType.isInstanceOf[StringType] ||
+          dataType == BooleanType ||
+          dataType == ByteType ||
+          dataType == ShortType ||
+          dataType == IntegerType ||
+          dataType == LongType
+        case 'd' | 'o' | 'x' | 'X' =>
+          dataType == ByteType ||
+          dataType == ShortType ||
+          dataType == IntegerType ||
+          dataType == LongType
+        case _ => false
+      }
+    }
+  }
+
   private def isPrimitiveType(dataType: DataType): Boolean = {
     val enableTimestampNtzValidation = 
VeloxConfig.get.enableTimestampNtzValidation
     dataType match {
diff --git 
a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxFormatStringSuite.scala
 
b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxFormatStringSuite.scala
new file mode 100644
index 0000000000..93b5926c4f
--- /dev/null
+++ 
b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxFormatStringSuite.scala
@@ -0,0 +1,237 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.execution
+
+import org.apache.gluten.test.FallbackUtil
+
+import org.apache.spark.sql.{DataFrame, Row}
+
+class VeloxFormatStringSuite extends VeloxWholeStageTransformerSuite {
+
+  override protected val resourcePath: String = "/tpch-data-parquet"
+  override protected val fileFormat: String = "parquet"
+
+  private def withFormatValues(f: => Unit): Unit = {
+    withTempView("format_values") {
+      withTempPath {
+        path =>
+          spark.range(0, 3).write.parquet(path.getAbsolutePath)
+          
spark.read.parquet(path.getAbsolutePath).createOrReplaceTempView("format_values")
+          f
+      }
+    }
+  }
+
+  private def assertFallback(dataFrame: DataFrame): Unit = {
+    assert(
+      FallbackUtil.hasFallback(dataFrame.queryExecution.executedPlan),
+      s"Expected format_string/printf fallback: 
${dataFrame.queryExecution.executedPlan}")
+  }
+
+  private def assertFallbackWithoutExecution(sql: String): Unit = {
+    val executedPlan = spark.sql(sql).queryExecution.executedPlan
+    assert(
+      FallbackUtil.hasFallback(executedPlan),
+      s"Expected format_string/printf fallback: $executedPlan")
+  }
+
+  private def assertNative(dataFrame: DataFrame): Unit = {
+    val executedPlan = dataFrame.queryExecution.executedPlan
+    assert(
+      !FallbackUtil.hasFallback(executedPlan),
+      s"Expected native format_string/printf execution: $executedPlan")
+    assert(
+      executedPlan.find(_.isInstanceOf[ProjectExecTransformer]).isDefined,
+      s"Expected a native project for format_string/printf: $executedPlan")
+  }
+
+  test("format_string and printf share native integer formatting") {
+    withFormatValues {
+      runQueryAndCompare("""
+                           |SELECT
+                           |  format_string('id=%04d', CAST(id AS INT)),
+                           |  printf('id=%04d', CAST(id AS INT))
+                           |FROM format_values
+                           |""".stripMargin)(assertNative)
+    }
+  }
+
+  test("format_string offloads verified integral conversions") {
+    withFormatValues {
+      runQueryAndCompare("""
+                           |SELECT format_string(
+                           |  '%d|%05d|%o|%x|%X',
+                           |  CAST(id AS INT),
+                           |  CAST(id AS INT),
+                           |  CAST(id AS INT),
+                           |  CAST(id AS INT),
+                           |  CAST(id AS INT))
+                           |FROM format_values
+                           |""".stripMargin)(assertNative)
+    }
+  }
+
+  test("format_string offloads strings, booleans, integrals, and nulls with 
percent-s") {
+    withFormatValues {
+      runQueryAndCompare("""
+                           |SELECT format_string(
+                           |  '%s|%s|%s|%s',
+                           |  CAST(id AS STRING),
+                           |  id % 2 = 0,
+                           |  id,
+                           |  CASE WHEN id = 1 THEN NULL ELSE CAST(id AS 
STRING) END)
+                           |FROM format_values
+                           |""".stripMargin)(assertNative)
+    }
+  }
+
+  test("format_string offloads padded nullable BIGINT values") {
+    withFormatValues {
+      runQueryAndCompare("""
+                           |SELECT format_string(
+                           |  '%+05d|%05X|%-5d',
+                           |  CASE WHEN id = 1 THEN CAST(NULL AS BIGINT) ELSE 
id END,
+                           |  CASE WHEN id = 1 THEN CAST(NULL AS BIGINT) ELSE 
id END,
+                           |  CASE WHEN id = 1 THEN CAST(NULL AS BIGINT) ELSE 
id END)
+                           |FROM format_values
+                           |""".stripMargin)(assertNative)
+    }
+  }
+
+  Seq("format_string", "printf").foreach {
+    functionName =>
+      test(s"$functionName falls back outside the verified compatibility 
subset") {
+        withFormatValues {
+          Seq(
+            "CASE WHEN id = 0 THEN '%d' ELSE '%x' END, id",
+            "'%b', id = 0",
+            "'%c', CAST(id + 65 AS INT)",
+            "'%g', CAST(id AS DOUBLE) + 0.25D",
+            "'%.2f', CAST(id AS DOUBLE) + 0.25D",
+            "'%.3e', CAST(id AS DOUBLE) + 0.25D",
+            "'%.3E', CAST(id AS DOUBLE) + 0.25D",
+            "'%1$s', CAST(id AS STRING)",
+            "'%10s', CAST(id AS STRING)",
+            "'%s', CAST(id AS DOUBLE)",
+            "'%f', CAST(id AS DOUBLE) + 0.25D"
+          ).foreach {
+            arguments =>
+              runQueryAndCompare(
+                s"SELECT $functionName($arguments) FROM format_values",
+                noFallBack = false)(assertFallback)
+          }
+        }
+      }
+  }
+
+  test("format_string handles null arguments and uppercase integral 
conversions") {
+    withFormatValues {
+      val dataFrame =
+        runQueryAndCompare("""
+                             |SELECT format_string(
+                             |  '%d|%X',
+                             |  CASE WHEN id = 1 THEN NULL ELSE CAST(id AS 
INT) END,
+                             |  CASE WHEN id = 1 THEN NULL ELSE CAST(id AS 
INT) END)
+                             |FROM format_values
+                             |""".stripMargin)(assertNative)
+      checkAnswer(dataFrame, Seq(Row("0|0"), Row("null|NULL"), Row("2|2")))
+    }
+  }
+
+  Seq("format_string", "printf").foreach {
+    functionName =>
+      test(s"$functionName plans the maximum native width without execution") {
+        withFormatValues {
+          assertNative(spark.sql(s"""
+                                    |SELECT $functionName('%1048576d', CAST(id 
AS INT))
+                                    |FROM format_values
+                                    |WHERE id = 0
+                                    |""".stripMargin))
+        }
+      }
+
+      test(s"$functionName executes wide native padding") {
+        withFormatValues {
+          val dataFrame =
+            runQueryAndCompare(s"""
+                                  |SELECT $functionName('%4096d', CAST(id AS 
INT))
+                                  |FROM format_values
+                                  |WHERE id = 0
+                                  |""".stripMargin)(assertNative)
+          checkAnswer(dataFrame, Row(" " * 4095 + "0"))
+        }
+      }
+  }
+
+  Seq("format_string", "printf").foreach {
+    functionName =>
+      test(s"$functionName rejects malformed and unsafe literals before native 
execution") {
+        withFormatValues {
+          Seq(
+            "%--5d",
+            "%-d",
+            "%0d",
+            "%-05d",
+            "%+ d",
+            "%.d",
+            "%1048577d",
+            "%",
+            "%d %d"
+          ).foreach {
+            format =>
+              assertFallbackWithoutExecution(
+                s"SELECT $functionName('$format', CAST(id AS INT)) FROM 
format_values")
+          }
+        }
+      }
+  }
+
+  test("format_string replaces malformed UTF-8 in string arguments") {
+    withFormatValues {
+      val dataFrame =
+        runQueryAndCompare("""
+                             |SELECT id, hex(format_string(
+                             |  '%s',
+                             |  CAST(unhex(
+                             |    CASE id
+                             |      WHEN 0 THEN 'FF'
+                             |      WHEN 1 THEN 'E282'
+                             |      ELSE 'E08080'
+                             |    END) AS STRING)))
+                             |FROM format_values
+                             |""".stripMargin)(assertNative)
+      checkAnswer(
+        dataFrame,
+        Seq(Row(0L, "EFBFBD"), Row(1L, "EFBFBD"), Row(2L, 
"EFBFBDEFBFBDEFBFBD")))
+    }
+  }
+
+  test("format_string replaces malformed UTF-8 in a constant pattern") {
+    withFormatValues {
+      val dataFrame =
+        runQueryAndCompare("""
+                             |SELECT id, hex(format_string(
+                             |  CAST(unhex('FF2573') AS STRING),
+                             |  concat('v', id)))
+                             |FROM format_values
+                             |""".stripMargin)(assertNative)
+      checkAnswer(
+        dataFrame,
+        Seq(Row(0L, "EFBFBD7630"), Row(1L, "EFBFBD7631"), Row(2L, 
"EFBFBD7632")))
+    }
+  }
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to