This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-4.x by this push:
     new cf41670ecec5 [SPARK-60634][SQL] Give copied UUID expressions 
independent random generators
cf41670ecec5 is described below

commit cf41670ecec576bf2c3cfe24d6297857bff6b594
Author: Linhong Liu <[email protected]>
AuthorDate: Thu Nov 5 07:08:31 2026 +0900

    [SPARK-60634][SQL] Give copied UUID expressions independent random 
generators
    
    ### What changes were proposed in this pull request?
    
    Give each fresh copy of `Uuid` its own random generator while retaining the 
seed. Preserve UUID arguments to classic Dataset generators and Hive UDTFs by 
evaluating the children that Spark has initialized for the partition. Cache an 
array of those child references and fill the output argument array directly, 
retaining the previous per-row allocation shape.
    
    ### Why are the changes needed?
    
    The default leaf-expression copy hook returns the same instance even though 
`Uuid` is stateful. Separate evaluators can therefore advance or reinitialize 
one another's UUID sequence.
    
    The two generators currently build nested interpreted projections without 
initializing them. A real UUID copy in such a projection would be 
uninitialized. Evaluating the generator's already initialized children 
preserves existing generator behavior, including NULL handling and Scala/Hive 
type conversion.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. Fresh evaluator copies produce independent seeded UUID sequences. 
Existing UUID arguments to `Dataset.explode` and Hive UDTFs retain their 
partition-specific values and continue to succeed. Evaluating the initialized 
generator children also allows `rand()` and `monotonically_increasing_id()` 
arguments to these generators to succeed; previously, those arguments failed 
because the nested projection copied them without initializing the copies.
    
    ### How was this patch tested?
    
    Expression tests cover distinct copies, interleaved evaluation, and 
partition reinitialization. End-to-end Dataset and Hive UDTF regressions cover 
seeds 0, 42, and `Long.MinValue`, two partitions, whole-stage code generation 
on and off, and NULL and structured input conversion. Hive coverage includes 
SELECT-list and LATERAL VIEW calls.
    
    `MiscExpressionsSuite`: 10 tests passed. The 12 `DataFrameSuite` tests 
matching `explode` and all 59 `HiveUDFSuite` tests passed with SBT. All six new 
Dataset cases and all six new Hive cases reproduced the initialization failure 
before the generator repair. Relevant Scala style checks passed.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Codex CLI 0.164.0 (GPT-6)
    
    Closes #59881 from linhongliu-db/fix-uuid-copy-isolation.
    
    Authored-by: Linhong Liu <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
    (cherry picked from commit 4e58a0f5752ac58a0e6cda8c283dfc45ac817bcc)
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 .../sql/catalyst/expressions/generators.scala      | 18 ++++++++++++----
 .../spark/sql/catalyst/expressions/misc.scala      |  5 +++++
 .../expressions/MiscExpressionsSuite.scala         | 25 ++++++++++++++++++++++
 .../org/apache/spark/sql/DataFrameSuite.scala      | 19 ++++++++++++++++
 .../scala/org/apache/spark/sql/hive/hiveUDFs.scala | 15 +++++++++++--
 .../spark/sql/hive/execution/HiveUDFSuite.scala    | 25 +++++++++++++++++++++-
 6 files changed, 100 insertions(+), 7 deletions(-)

diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala
 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala
index 3f2e838c1903..7a19855ba3f0 100644
--- 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala
+++ 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala
@@ -105,11 +105,10 @@ case class UserDefinedGenerator(
     children: Seq[Expression])
   extends Generator with CodegenFallback {
 
-  @transient private[this] var inputRow: InterpretedProjection = _
   @transient private[this] var convertToScala: (InternalRow) => Row = _
+  @transient private lazy val inputExpressions = children.toArray
 
   private def initializeConverters(): Unit = {
-    inputRow = new InterpretedProjection(children)
     convertToScala = {
       val inputSchema = StructType(children.map { e =>
         StructField(e.simpleString(SQLConf.get.maxToStringFields), e.dataType, 
nullable = true)
@@ -119,11 +118,22 @@ case class UserDefinedGenerator(
   }
 
   override def eval(input: InternalRow): IterableOnce[InternalRow] = {
-    if (inputRow == null) {
+    if (convertToScala == null) {
       initializeConverters()
     }
+    // GenerateExec initializes these children for the partition. A projection 
would fresh-copy
+    // stateful expressions without initializing the copies. Fill the final 
array directly to
+    // avoid creating an intermediate collection for each row.
+    val expressions = inputExpressions
+    val outputArray = new Array[Any](expressions.length)
+    var i = 0
+    while (i < expressions.length) {
+      outputArray(i) = expressions(i).eval(input)
+      i += 1
+    }
+    val inputRow = new GenericInternalRow(outputArray)
     // Convert the objects into Scala Type before calling function, we need 
schema to support UDT
-    function(convertToScala(inputRow(input)))
+    function(convertToScala(inputRow))
   }
 
   override def toString: String = 
s"UserDefinedGenerator(${children.mkString(",")})"
diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala
 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala
index 61e04b9f761c..6fd653789f10 100644
--- 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala
+++ 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala
@@ -318,6 +318,11 @@ case class Uuid(randomSeed: Option[Long] = None) extends 
LeafExpression with Non
 
   override def stateful: Boolean = true
 
+  // Stateful fresh copies use this hook even for leaves. The default leaf 
hook returns this,
+  // which would share randomGenerator across evaluators; copy preserves the 
seed without
+  // sharing the initialized generator.
+  override def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): 
Expression = copy()
+
   @transient private[this] var randomGenerator: RandomUUIDGenerator = _
 
   override protected def initializeInternal(partitionIndex: Int): Unit =
diff --git 
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/MiscExpressionsSuite.scala
 
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/MiscExpressionsSuite.scala
index a586e33afdd2..8bddd43c69da 100644
--- 
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/MiscExpressionsSuite.scala
+++ 
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/MiscExpressionsSuite.scala
@@ -94,6 +94,31 @@ class MiscExpressionsSuite extends SparkFunSuite with 
ExpressionEvalHelper {
       evaluateWithUnsafeProjection(new Uuid(seed3)))
   }
 
+  Seq(0L, 42L, Long.MinValue).foreach { seed =>
+    test(s"uuid fresh copies own independent generators for seed $seed") {
+      val original = Uuid(Some(seed))
+      val first = 
original.freshCopyIfContainsStatefulExpression().asInstanceOf[Uuid]
+      val second = 
original.freshCopyIfContainsStatefulExpression().asInstanceOf[Uuid]
+      assert(first ne original)
+      assert(second ne original)
+      assert(first ne second)
+      assert(first.randomSeed == original.randomSeed)
+      original.initialize(0)
+      first.initialize(0)
+      second.initialize(0)
+      for (_ <- 0 until 20) {
+        val expected = original.eval()
+        assert(first.eval() == expected)
+        assert(second.eval() == expected)
+      }
+      first.initialize(1)
+      assert(second.eval() == original.eval())
+      val otherPartition = Uuid(Some(seed))
+      otherPartition.initialize(1)
+      assert(first.eval() == otherPartition.eval())
+    }
+  }
+
   test("PrintToStderr") {
     val inputExpr = Literal(1)
     val systemErr = System.err
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala 
b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
index d2ba7f17ee96..0edb2861b2d3 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
@@ -36,6 +36,7 @@ import org.apache.spark.sql.catalyst.expressions.{Attribute, 
AttributeReference,
 import org.apache.spark.sql.catalyst.optimizer.ConvertToLocalRelation
 import org.apache.spark.sql.catalyst.parser.ParseException
 import org.apache.spark.sql.catalyst.plans.logical.{Filter, LeafNode, 
LocalRelation, LogicalPlan, OneRowRelation}
+import org.apache.spark.sql.catalyst.util.RandomUUIDGenerator
 import org.apache.spark.sql.connector.FakeV2Provider
 import org.apache.spark.sql.execution.{FilterExec, LogicalRDD, QueryExecution, 
SortExec, WholeStageCodegenExec}
 import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
@@ -111,6 +112,24 @@ class DataFrameSuite extends SharedSparkSession
     )
   }
 
+  for (seed <- Seq(0L, 42L, Long.MinValue); codegen <- Seq(true, false)) {
+    test(s"SPARK-60634: Dataset.explode initializes UUID arguments ($seed, 
$codegen)") {
+      withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> codegen.toString) {
+        val result = spark.range(0, 6, 1, 2).explode(
+          expr(s"uuid($seed)"), lit(null).cast(StringType), struct(lit(7))) { 
row: Row =>
+          Seq((row.getString(0), row.isNullAt(1), row.getStruct(2).getInt(0)))
+        }
+        val expected = (0 until 2).flatMap { partition =>
+          val generator = RandomUUIDGenerator(seed + partition)
+          (0 until 3).map { offset =>
+            Row(partition * 3L + offset, generator.getNextUUID().toString, 
true, 7)
+          }
+        }
+        checkAnswer(result.select("id", "_1", "_2", "_3"), expected)
+      }
+    }
+  }
+
   test("explode") {
     val df = Seq((1, "a b c"), (2, "a b"), (3, "a")).toDF("number", "letters")
     val df2 =
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala 
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala
index 7472341eabb2..93fd721bc390 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala
@@ -260,11 +260,22 @@ private[hive] case class HiveGenericUDTF(
   private lazy val unwrapper = unwrapperFor(outputInspector, elementSchema)
 
   @transient
-  private lazy val inputProjection = new InterpretedProjection(children)
+  private lazy val inputExpressions = children.toArray
 
   override def eval(input: InternalRow): IterableOnce[InternalRow] = {
     outputInspector // Make sure initialized.
-    function.process(wrap(inputProjection(input), wrappers, udtInput, 
inputDataTypes))
+    // GenerateExec has initialized these children for the partition. A 
projection would create
+    // fresh stateful copies without initializing them. Fill the final array 
directly to avoid
+    // creating an intermediate collection for each row.
+    val expressions = inputExpressions
+    val outputArray = new Array[Any](expressions.length)
+    var i = 0
+    while (i < expressions.length) {
+      outputArray(i) = expressions(i).eval(input)
+      i += 1
+    }
+    val inputRow = new GenericInternalRow(outputArray)
+    function.process(wrap(inputRow, wrappers, udtInput, inputDataTypes))
     collector.collectRows()
   }
 
diff --git 
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
 
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
index 5b5474512f29..54824bb479c3 100644
--- 
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
+++ 
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
@@ -44,7 +44,7 @@ import org.apache.spark.sql.{AnalysisException, QueryTest, 
Row}
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.{ApplyFunctionExpression, 
AttributeReference, BindReferences, CodegenObjectFactoryMode, Literal}
 import org.apache.spark.sql.catalyst.plans.logical.{Filter, Project}
-import org.apache.spark.sql.catalyst.util.{DateTimeUtils, GenericArrayData}
+import org.apache.spark.sql.catalyst.util.{DateTimeUtils, GenericArrayData, 
RandomUUIDGenerator}
 import org.apache.spark.sql.connector.catalog.functions.{ScalarFunction, 
SimpleFunction}
 import org.apache.spark.sql.execution.WholeStageCodegenExec
 import org.apache.spark.sql.functions.{call_function, max}
@@ -74,6 +74,29 @@ class HiveUDFSuite extends QueryTest with TestHiveSingleton {
 
   import spark.udf
 
+  for (seed <- Seq(0L, 42L, Long.MinValue); codegen <- Seq(true, false)) {
+    test(s"SPARK-60634: Hive UDTFs initialize UUID arguments ($seed, 
$codegen)") {
+      withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> codegen.toString) {
+        withUserDefinedFunction("hive_uuid_explode" -> true) {
+          sql(s"CREATE TEMPORARY FUNCTION hive_uuid_explode AS " +
+            s"'${classOf[GenericUDTFExplode].getName}'")
+          val expected = (0 until 2).flatMap { partition =>
+            val generator = RandomUUIDGenerator(seed + partition)
+            (0 until 3).flatMap { offset =>
+              val id = partition * 3L + offset
+              Seq(Row(id, generator.getNextUUID().toString), Row(id, null), 
Row(id, id.toString))
+            }
+          }
+          val argument = s"array(uuid($seed), CAST(NULL AS STRING), CAST(id AS 
STRING))"
+          checkAnswer(sql(s"SELECT id, hive_uuid_explode($argument) AS value " 
+
+            "FROM range(0, 6, 1, 2)"), expected)
+          checkAnswer(sql(s"SELECT id, value FROM range(0, 6, 1, 2) " +
+            s"LATERAL VIEW hive_uuid_explode($argument) v AS value"), expected)
+        }
+      }
+    }
+  }
+
   test("spark sql udf test that returns a struct") {
     udf.register("getStruct", (_: Int) => Fields(1, 2, 3, 4, 5))
     assert(sql(


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to