This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new bc5eb3ddbf64 [SPARK-60635][SQL] Isolate mutable number-format parsers 
in expression copies
bc5eb3ddbf64 is described below

commit bc5eb3ddbf64dadb1f65cf135dc182f5b806a596
Author: Linhong Liu <[email protected]>
AuthorDate: Thu Nov 5 06:49:21 2026 +0900

    [SPARK-60635][SQL] Isolate mutable number-format parsers in expression 
copies
    
    ### What changes were proposed in this pull request?
    
    Mark `ToNumberBase` and numeric `ToCharacter` as stateful so fresh 
evaluator copies create separate cached `ToNumberParser` instances.
    
    ### Why are the changes needed?
    
    `ToNumberParser` mutates its working state during parsing and formatting. 
Concurrent queries derived from the same DataFrame can share an analyzed 
`to_number`, `try_to_number`, or numeric `to_char` expression. With literal 
inputs, `ConstantFolding` evaluates that expression during optimization. 
Without the stateful marker, the requested fresh copy can return the shared 
expression, allowing optimizer threads to use the same mutable parser. 
Interleaved parser state can produce an incorr [...]
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. Fixes timing-dependent incorrect results or query failures when 
concurrent queries evaluate shared number-format expressions during constant 
folding.
    
    ### How was this patch tested?
    
    Added regression tests for fresh expression identity, separate parser 
references in generated code, conversion results, and NULL inputs for all three 
expressions. Separate tests assert that literal inputs and their fresh copies 
are foldable and that `ConstantFolding` replaces them with the expected 
literals. Attribute inputs and their fresh copies must remain nonfoldable and 
their projections must remain unchanged by `ConstantFolding`.
    
    These tests verify parser isolation and constant-folding behavior; they do 
not reproduce the timing-dependent concurrent-query race.
    
    All 79 tests in `StringExpressionsSuite` passed with SBT, including the six 
parser-isolation and constant-folding cases. Relevant Scala style checks passed:
    
    ```bash
    build/sbt 'catalyst/testOnly *StringExpressionsSuite'
    ```
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Codex CLI 0.164.0 (GPT-6)
    
    Closes #59882 from linhongliu-db/fix-number-parser-copy-isolation.
    
    Authored-by: Linhong Liu <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 .../expressions/numberFormatExpressions.scala      |  6 +++
 .../expressions/StringExpressionsSuite.scala       | 53 +++++++++++++++++++++-
 2 files changed, 58 insertions(+), 1 deletion(-)

diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/numberFormatExpressions.scala
 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/numberFormatExpressions.scala
index 77b9ae3ea5aa..8d9dae56c3b8 100644
--- 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/numberFormatExpressions.scala
+++ 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/numberFormatExpressions.scala
@@ -33,6 +33,9 @@ import org.apache.spark.unsafe.types.UTF8String
 abstract class ToNumberBase(left: Expression, right: Expression, errorOnFail: 
Boolean)
   extends BinaryExpression with Serializable with ImplicitCastInputTypes {
   override def nullIntolerant: Boolean = true
+  // ToNumberParser mutates its working state during parsing/formatting. 
Stateful fresh
+  // copies give each evaluator its own parser instead of sharing one across 
threads.
+  override def stateful: Boolean = true
 
   private lazy val numberFormatter = {
     val value = right.eval()
@@ -302,6 +305,9 @@ object ToCharacterBuilder extends ExpressionBuilder {
 case class ToCharacter(left: Expression, right: Expression)
   extends BinaryExpression with ImplicitCastInputTypes with 
DefaultStringProducingExpression {
   override def nullIntolerant: Boolean = true
+  // ToNumberParser mutates its working state during parsing/formatting. 
Stateful fresh
+  // copies give each evaluator its own parser instead of sharing one across 
threads.
+  override def stateful: Boolean = true
 
   private lazy val numberFormatter = {
     val value = right.eval()
diff --git 
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala
 
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala
index f3d479887ab2..2ac0d601ae0c 100644
--- 
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala
+++ 
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala
@@ -26,9 +26,12 @@ import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
 import 
org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{DataTypeMismatch, 
InvalidFormat}
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.expressions.Cast._
-import 
org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection
+import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, 
GenerateUnsafeProjection}
+import org.apache.spark.sql.catalyst.optimizer.ConstantFolding
+import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, 
OneRowRelation, Project}
 import org.apache.spark.sql.catalyst.util.CharsetProvider
 import org.apache.spark.sql.catalyst.util.CollationFactory
+import org.apache.spark.sql.catalyst.util.ToNumberParser
 import org.apache.spark.sql.errors.QueryExecutionErrors.toSQLId
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.internal.types.StringTypeWithCollation
@@ -1640,6 +1643,54 @@ class StringExpressionsSuite extends SparkFunSuite with 
ExpressionEvalHelper {
       Literal.create(null, IntegerType), Literal.create(null, IntegerType)), 
null)
   }
 
+  for ((expression, expected) <- Seq(
+      (ToNumber(Literal("12"), Literal("99")), Decimal(12)),
+      (TryToNumber(Literal("12"), Literal("99")), Decimal(12)),
+      (ToCharacter(Literal(Decimal(12)), Literal("99")), "12"))) {
+    test(s"number format evaluator copies isolate parser state: 
${expression.prettyName}") {
+      checkEvaluation(expression, expected)
+      val first = expression.freshCopyIfContainsStatefulExpression()
+      val second = expression.freshCopyIfContainsStatefulExpression()
+      assert(first ne expression)
+      assert(second ne first)
+      checkEvaluation(first, expected)
+      checkEvaluation(second, expected)
+      val contexts = Seq(first, second).map { copy =>
+        val context = new CodegenContext
+        copy.genCode(context)
+        context.references.collect { case parser: ToNumberParser => parser 
}.toSeq
+      }
+      assert(contexts.forall(_.size == 1))
+      assert(contexts.head.head ne contexts.last.head)
+      checkEvaluation(expression.withNewChildren(Seq(
+        Literal.create(null, expression.children.head.dataType), 
Literal("99"))), null)
+    }
+
+    test(s"number format expressions preserve constant folding: 
${expression.prettyName}") {
+      val copies = Seq(expression,
+        expression.freshCopyIfContainsStatefulExpression(),
+        expression.freshCopyIfContainsStatefulExpression())
+      copies.foreach { copy =>
+        assert(copy.foldable)
+        val project = Project(Seq(Alias(copy, "result")()), OneRowRelation())
+        val optimized = ConstantFolding(project).asInstanceOf[Project]
+        assert(optimized.projectList.head.asInstanceOf[Alias].child ==
+          Literal.create(expected, expression.dataType))
+      }
+
+      val input = AttributeReference("input", 
expression.children.head.dataType)()
+      val nonFoldable = expression.withNewChildren(Seq(input, Literal("99")))
+      val nonFoldableCopies = Seq(nonFoldable,
+        nonFoldable.freshCopyIfContainsStatefulExpression(),
+        nonFoldable.freshCopyIfContainsStatefulExpression())
+      nonFoldableCopies.foreach { copy =>
+        assert(!copy.foldable)
+        val project = Project(Seq(Alias(copy, "result")()), 
LocalRelation(input))
+        assert(ConstantFolding(project) === project)
+      }
+    }
+  }
+
   test("ToNumber: positive tests") {
     Seq(
       ("$345", "S$999,099.99") -> Decimal(345),


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to