This is an automated email from the ASF dual-hosted git repository.
weiting-chen pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 493bcf4730 [VL] Add native Spark format_string and printf support
(#13223)
493bcf4730 is described below
commit 493bcf473045a16b79899b00baf8e2d60cf5a46f
Author: Minni Mittal <[email protected]>
AuthorDate: Tue Nov 10 09:59:42 2026 +0530
[VL] Add native Spark format_string and printf support (#13223)
* [VL] Add native Spark format_string and printf support
Validate format_string patterns and argument types against the subset
implemented by Velox, falling back to Spark for unsupported or unsafe cases.
Add native and fallback coverage for format_string and printf.
Co-authored-by: Copilot <[email protected]>
Copilot-Session: 309a2429-55f4-46e7-9229-4c2be58b1598
* [VL] Document format width validation limit
Explain that the validator's width limit follows Velox's per-specifier
output cap.
Co-authored-by: Copilot <[email protected]>
Copilot-Session: 309a2429-55f4-46e7-9229-4c2be58b1598
* [VL] Cover printf fallback validation
## Summary
Exercise the existing compatibility-subset and malformed-format fallback
cases through both format_string and printf.
## Prompting Intent
Address the missing printf fallback coverage identified in PR #13223,
keeping the follow-up limited to tests.
## Linked Sources
- PR: https://github.com/apache/gluten/pull/13223
- Review: https://github.com/apache/gluten/pull/13223#discussion_r4268439755
## Rationale
Share the same cases across both SQL names instead of duplicating tests.
Preserve plan-only assertions for unsafe or malformed formats so invalid
queries are never executed during fallback checks.
Co-authored-by: Copilot <[email protected]>
Copilot-Session: 1ee80ef9-575d-4e21-8139-b5951e735292
* [VL] Address format string review feedback
## Summary
Compare executable fallback cases against vanilla Spark, separate maximum
width planning coverage from bounded runtime padding checks, and clarify
format_string and printf validation.
## Prompting Intent
Address the outstanding review findings on PR #13223 while preserving the
supported formatting subset and avoiding execution of malformed formats.
## Linked Sources
- PR: https://github.com/apache/gluten/pull/13223
- Alias validation:
https://github.com/apache/gluten/pull/13223#discussion_r4268439600
- Explicit import reference:
https://github.com/apache/gluten/pull/13223#discussion_r4292446998
- Fallback parity:
https://github.com/apache/gluten/pull/13223#discussion_r4292447082
- Width coverage:
https://github.com/apache/gluten/pull/13223#discussion_r4292447127
## Rationale
Reuse the existing query-comparison harness to check both result parity
and fallback planning. Exercise the maximum supported width without
allocating its output and compare complete 4096-character padded results
for both SQL names. Keep malformed and over-limit cases planning-only.
Qualify the parser buffer reference and document shared Catalyst alias
resolution without changing validator behavior.
Co-authored-by: Copilot <[email protected]>
Copilot-Session: 1ee80ef9-575d-4e21-8139-b5951e735292
---------
Co-authored-by: Copilot <[email protected]>
Co-authored-by: minni31 <[email protected]>
Copilot-Session: 309a2429-55f4-46e7-9229-4c2be58b1598
Copilot-Session: 1ee80ef9-575d-4e21-8139-b5951e735292
---
.../backendsapi/velox/VeloxValidatorApi.scala | 158 +++++++++++++-
.../gluten/execution/VeloxFormatStringSuite.scala | 237 +++++++++++++++++++++
2 files changed, 392 insertions(+), 3 deletions(-)
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxValidatorApi.scala
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxValidatorApi.scala
index e6f7a0fd92..5089e4104a 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxValidatorApi.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxValidatorApi.scala
@@ -27,11 +27,12 @@ import org.apache.gluten.substrait.plan.PlanNode
import org.apache.gluten.validate.NativePlanValidationInfo
import org.apache.gluten.vectorized.NativePlanEvaluator
-import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression}
+import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression,
FormatString, Literal}
import org.apache.spark.sql.catalyst.plans.physical.Partitioning
import org.apache.spark.sql.execution.SparkPlan
import org.apache.spark.sql.types._
import org.apache.spark.task.TaskResources
+import org.apache.spark.unsafe.types.UTF8String
import io.substrait.proto.SimpleExtensionDeclaration
@@ -42,8 +43,14 @@ class VeloxValidatorApi extends ValidatorApi {
import VeloxValidatorApi._
/** For velox backend, key validation is on native side. */
- override def doExprValidate(substraitExprName: String, expr: Expression):
Boolean =
- true
+ override def doExprValidate(substraitExprName: String, expr: Expression):
Boolean = {
+ expr match {
+ // Both format_string and printf resolve to Spark's FormatString
expression.
+ case formatString: FormatString =>
+ supportsFormatString(formatString)
+ case _ => true
+ }
+ }
override def doNativeValidateWithFailureReason(plan: PlanNode):
ValidationResult = {
TaskResources.runUnsafe {
@@ -104,6 +111,151 @@ class VeloxValidatorApi extends ValidatorApi {
}
object VeloxValidatorApi {
+ private val nativeFormatFlags = Set('-', '+', ' ', '0')
+ // Match Velox's per-specifier output limit to prevent excessive allocations.
+ private val maxNativeFormatWidth = 1 << 20
+
+ private def supportsFormatString(formatString: FormatString): Boolean = {
+ val format = formatString.children.headOption match {
+ case Some(Literal(value: UTF8String, dataType))
+ if dataType.isInstanceOf[StringType] =>
+ value.toString
+ case _ =>
+ return false
+ }
+ val conversions = parseNativeFormatString(format).getOrElse {
+ return false
+ }
+ conversions.length <= formatString.children.length - 1 &&
+ conversions
+ .zip(formatString.children.tail)
+ .forall {
+ case (conversion, argument) =>
+ supportsNativeFormatType(conversion, argument.dataType)
+ }
+ }
+
+ private def parseNativeFormatString(format: String): Option[Seq[Char]] = {
+ val conversions = scala.collection.mutable.ArrayBuffer.empty[Char]
+ var index = 0
+ while (index < format.length) {
+ if (format.charAt(index) != '%') {
+ index += 1
+ } else if (index + 1 < format.length && format.charAt(index + 1) == '%')
{
+ index += 2
+ } else {
+ index += 1
+ val flags = scala.collection.mutable.Set.empty[Char]
+ while (index < format.length &&
nativeFormatFlags.contains(format.charAt(index))) {
+ if (!flags.add(format.charAt(index))) {
+ return None
+ }
+ index += 1
+ }
+ val widthStart = index
+ while (
+ index < format.length && format.charAt(index) >= '0' &&
+ format.charAt(index) <= '9'
+ ) {
+ index += 1
+ }
+ val width =
+ if (widthStart == index) None
+ else parseBoundedFormatNumber(format, widthStart, index)
+ if (widthStart != index && width.isEmpty) {
+ return None
+ }
+ var precision: Option[Int] = None
+ if (index < format.length && format.charAt(index) == '.') {
+ index += 1
+ val precisionStart = index
+ while (
+ index < format.length && format.charAt(index) >= '0' &&
+ format.charAt(index) <= '9'
+ ) {
+ index += 1
+ }
+ if (precisionStart == index) {
+ return None
+ }
+ precision = parseBoundedFormatNumber(format, precisionStart, index)
+ if (precision.isEmpty) {
+ return None
+ }
+ }
+ if (index >= format.length) {
+ return None
+ }
+ val conversion = format.charAt(index)
+ index += 1
+ if (!isNativeFormatSpecifierSupported(conversion, flags.toSet, width,
precision)) {
+ return None
+ }
+ conversions += conversion
+ }
+ }
+ Some(conversions.toSeq)
+ }
+
+ private def parseBoundedFormatNumber(format: String, start: Int, end: Int):
Option[Int] = {
+ var value = 0
+ var index = start
+ while (index < end) {
+ val digit = format.charAt(index) - '0'
+ if (value > (maxNativeFormatWidth - digit) / 10) {
+ return None
+ }
+ value = value * 10 + digit
+ index += 1
+ }
+ Some(value)
+ }
+
+ private def isNativeFormatSpecifierSupported(
+ conversion: Char,
+ flags: Set[Char],
+ width: Option[Int],
+ precision: Option[Int]): Boolean = {
+ if ((flags.contains('-') || flags.contains('0')) && width.isEmpty) {
+ return false
+ }
+ if (
+ (flags.contains('-') && flags.contains('0')) ||
+ (flags.contains('+') && flags.contains(' '))
+ ) {
+ return false
+ }
+ conversion match {
+ case 's' => flags.isEmpty && width.isEmpty && precision.isEmpty
+ case 'd' => precision.isEmpty
+ case 'o' | 'x' | 'X' =>
+ precision.isEmpty && !flags.contains('+') && !flags.contains(' ')
+ case _ => false
+ }
+ }
+
+ private def supportsNativeFormatType(conversion: Char, dataType: DataType):
Boolean = {
+ if (dataType == NullType) {
+ true
+ } else {
+ conversion match {
+ case 's' =>
+ dataType.isInstanceOf[StringType] ||
+ dataType == BooleanType ||
+ dataType == ByteType ||
+ dataType == ShortType ||
+ dataType == IntegerType ||
+ dataType == LongType
+ case 'd' | 'o' | 'x' | 'X' =>
+ dataType == ByteType ||
+ dataType == ShortType ||
+ dataType == IntegerType ||
+ dataType == LongType
+ case _ => false
+ }
+ }
+ }
+
private def isPrimitiveType(dataType: DataType): Boolean = {
val enableTimestampNtzValidation =
VeloxConfig.get.enableTimestampNtzValidation
dataType match {
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxFormatStringSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxFormatStringSuite.scala
new file mode 100644
index 0000000000..93b5926c4f
--- /dev/null
+++
b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxFormatStringSuite.scala
@@ -0,0 +1,237 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.execution
+
+import org.apache.gluten.test.FallbackUtil
+
+import org.apache.spark.sql.{DataFrame, Row}
+
+class VeloxFormatStringSuite extends VeloxWholeStageTransformerSuite {
+
+ override protected val resourcePath: String = "/tpch-data-parquet"
+ override protected val fileFormat: String = "parquet"
+
+ private def withFormatValues(f: => Unit): Unit = {
+ withTempView("format_values") {
+ withTempPath {
+ path =>
+ spark.range(0, 3).write.parquet(path.getAbsolutePath)
+
spark.read.parquet(path.getAbsolutePath).createOrReplaceTempView("format_values")
+ f
+ }
+ }
+ }
+
+ private def assertFallback(dataFrame: DataFrame): Unit = {
+ assert(
+ FallbackUtil.hasFallback(dataFrame.queryExecution.executedPlan),
+ s"Expected format_string/printf fallback:
${dataFrame.queryExecution.executedPlan}")
+ }
+
+ private def assertFallbackWithoutExecution(sql: String): Unit = {
+ val executedPlan = spark.sql(sql).queryExecution.executedPlan
+ assert(
+ FallbackUtil.hasFallback(executedPlan),
+ s"Expected format_string/printf fallback: $executedPlan")
+ }
+
+ private def assertNative(dataFrame: DataFrame): Unit = {
+ val executedPlan = dataFrame.queryExecution.executedPlan
+ assert(
+ !FallbackUtil.hasFallback(executedPlan),
+ s"Expected native format_string/printf execution: $executedPlan")
+ assert(
+ executedPlan.find(_.isInstanceOf[ProjectExecTransformer]).isDefined,
+ s"Expected a native project for format_string/printf: $executedPlan")
+ }
+
+ test("format_string and printf share native integer formatting") {
+ withFormatValues {
+ runQueryAndCompare("""
+ |SELECT
+ | format_string('id=%04d', CAST(id AS INT)),
+ | printf('id=%04d', CAST(id AS INT))
+ |FROM format_values
+ |""".stripMargin)(assertNative)
+ }
+ }
+
+ test("format_string offloads verified integral conversions") {
+ withFormatValues {
+ runQueryAndCompare("""
+ |SELECT format_string(
+ | '%d|%05d|%o|%x|%X',
+ | CAST(id AS INT),
+ | CAST(id AS INT),
+ | CAST(id AS INT),
+ | CAST(id AS INT),
+ | CAST(id AS INT))
+ |FROM format_values
+ |""".stripMargin)(assertNative)
+ }
+ }
+
+ test("format_string offloads strings, booleans, integrals, and nulls with
percent-s") {
+ withFormatValues {
+ runQueryAndCompare("""
+ |SELECT format_string(
+ | '%s|%s|%s|%s',
+ | CAST(id AS STRING),
+ | id % 2 = 0,
+ | id,
+ | CASE WHEN id = 1 THEN NULL ELSE CAST(id AS
STRING) END)
+ |FROM format_values
+ |""".stripMargin)(assertNative)
+ }
+ }
+
+ test("format_string offloads padded nullable BIGINT values") {
+ withFormatValues {
+ runQueryAndCompare("""
+ |SELECT format_string(
+ | '%+05d|%05X|%-5d',
+ | CASE WHEN id = 1 THEN CAST(NULL AS BIGINT) ELSE
id END,
+ | CASE WHEN id = 1 THEN CAST(NULL AS BIGINT) ELSE
id END,
+ | CASE WHEN id = 1 THEN CAST(NULL AS BIGINT) ELSE
id END)
+ |FROM format_values
+ |""".stripMargin)(assertNative)
+ }
+ }
+
+ Seq("format_string", "printf").foreach {
+ functionName =>
+ test(s"$functionName falls back outside the verified compatibility
subset") {
+ withFormatValues {
+ Seq(
+ "CASE WHEN id = 0 THEN '%d' ELSE '%x' END, id",
+ "'%b', id = 0",
+ "'%c', CAST(id + 65 AS INT)",
+ "'%g', CAST(id AS DOUBLE) + 0.25D",
+ "'%.2f', CAST(id AS DOUBLE) + 0.25D",
+ "'%.3e', CAST(id AS DOUBLE) + 0.25D",
+ "'%.3E', CAST(id AS DOUBLE) + 0.25D",
+ "'%1$s', CAST(id AS STRING)",
+ "'%10s', CAST(id AS STRING)",
+ "'%s', CAST(id AS DOUBLE)",
+ "'%f', CAST(id AS DOUBLE) + 0.25D"
+ ).foreach {
+ arguments =>
+ runQueryAndCompare(
+ s"SELECT $functionName($arguments) FROM format_values",
+ noFallBack = false)(assertFallback)
+ }
+ }
+ }
+ }
+
+ test("format_string handles null arguments and uppercase integral
conversions") {
+ withFormatValues {
+ val dataFrame =
+ runQueryAndCompare("""
+ |SELECT format_string(
+ | '%d|%X',
+ | CASE WHEN id = 1 THEN NULL ELSE CAST(id AS
INT) END,
+ | CASE WHEN id = 1 THEN NULL ELSE CAST(id AS
INT) END)
+ |FROM format_values
+ |""".stripMargin)(assertNative)
+ checkAnswer(dataFrame, Seq(Row("0|0"), Row("null|NULL"), Row("2|2")))
+ }
+ }
+
+ Seq("format_string", "printf").foreach {
+ functionName =>
+ test(s"$functionName plans the maximum native width without execution") {
+ withFormatValues {
+ assertNative(spark.sql(s"""
+ |SELECT $functionName('%1048576d', CAST(id
AS INT))
+ |FROM format_values
+ |WHERE id = 0
+ |""".stripMargin))
+ }
+ }
+
+ test(s"$functionName executes wide native padding") {
+ withFormatValues {
+ val dataFrame =
+ runQueryAndCompare(s"""
+ |SELECT $functionName('%4096d', CAST(id AS
INT))
+ |FROM format_values
+ |WHERE id = 0
+ |""".stripMargin)(assertNative)
+ checkAnswer(dataFrame, Row(" " * 4095 + "0"))
+ }
+ }
+ }
+
+ Seq("format_string", "printf").foreach {
+ functionName =>
+ test(s"$functionName rejects malformed and unsafe literals before native
execution") {
+ withFormatValues {
+ Seq(
+ "%--5d",
+ "%-d",
+ "%0d",
+ "%-05d",
+ "%+ d",
+ "%.d",
+ "%1048577d",
+ "%",
+ "%d %d"
+ ).foreach {
+ format =>
+ assertFallbackWithoutExecution(
+ s"SELECT $functionName('$format', CAST(id AS INT)) FROM
format_values")
+ }
+ }
+ }
+ }
+
+ test("format_string replaces malformed UTF-8 in string arguments") {
+ withFormatValues {
+ val dataFrame =
+ runQueryAndCompare("""
+ |SELECT id, hex(format_string(
+ | '%s',
+ | CAST(unhex(
+ | CASE id
+ | WHEN 0 THEN 'FF'
+ | WHEN 1 THEN 'E282'
+ | ELSE 'E08080'
+ | END) AS STRING)))
+ |FROM format_values
+ |""".stripMargin)(assertNative)
+ checkAnswer(
+ dataFrame,
+ Seq(Row(0L, "EFBFBD"), Row(1L, "EFBFBD"), Row(2L,
"EFBFBDEFBFBDEFBFBD")))
+ }
+ }
+
+ test("format_string replaces malformed UTF-8 in a constant pattern") {
+ withFormatValues {
+ val dataFrame =
+ runQueryAndCompare("""
+ |SELECT id, hex(format_string(
+ | CAST(unhex('FF2573') AS STRING),
+ | concat('v', id)))
+ |FROM format_values
+ |""".stripMargin)(assertNative)
+ checkAnswer(
+ dataFrame,
+ Seq(Row(0L, "EFBFBD7630"), Row(1L, "EFBFBD7631"), Row(2L,
"EFBFBD7632")))
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]