This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new cf41670ecec5 [SPARK-60634][SQL] Give copied UUID expressions
independent random generators
cf41670ecec5 is described below
commit cf41670ecec576bf2c3cfe24d6297857bff6b594
Author: Linhong Liu <[email protected]>
AuthorDate: Thu Nov 5 07:08:31 2026 +0900
[SPARK-60634][SQL] Give copied UUID expressions independent random
generators
### What changes were proposed in this pull request?
Give each fresh copy of `Uuid` its own random generator while retaining the
seed. Preserve UUID arguments to classic Dataset generators and Hive UDTFs by
evaluating the children that Spark has initialized for the partition. Cache an
array of those child references and fill the output argument array directly,
retaining the previous per-row allocation shape.
### Why are the changes needed?
The default leaf-expression copy hook returns the same instance even though
`Uuid` is stateful. Separate evaluators can therefore advance or reinitialize
one another's UUID sequence.
The two generators currently build nested interpreted projections without
initializing them. A real UUID copy in such a projection would be
uninitialized. Evaluating the generator's already initialized children
preserves existing generator behavior, including NULL handling and Scala/Hive
type conversion.
### Does this PR introduce _any_ user-facing change?
Yes. Fresh evaluator copies produce independent seeded UUID sequences.
Existing UUID arguments to `Dataset.explode` and Hive UDTFs retain their
partition-specific values and continue to succeed. Evaluating the initialized
generator children also allows `rand()` and `monotonically_increasing_id()`
arguments to these generators to succeed; previously, those arguments failed
because the nested projection copied them without initializing the copies.
### How was this patch tested?
Expression tests cover distinct copies, interleaved evaluation, and
partition reinitialization. End-to-end Dataset and Hive UDTF regressions cover
seeds 0, 42, and `Long.MinValue`, two partitions, whole-stage code generation
on and off, and NULL and structured input conversion. Hive coverage includes
SELECT-list and LATERAL VIEW calls.
`MiscExpressionsSuite`: 10 tests passed. The 12 `DataFrameSuite` tests
matching `explode` and all 59 `HiveUDFSuite` tests passed with SBT. All six new
Dataset cases and all six new Hive cases reproduced the initialization failure
before the generator repair. Relevant Scala style checks passed.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Codex CLI 0.164.0 (GPT-6)
Closes #59881 from linhongliu-db/fix-uuid-copy-isolation.
Authored-by: Linhong Liu <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
(cherry picked from commit 4e58a0f5752ac58a0e6cda8c283dfc45ac817bcc)
Signed-off-by: Hyukjin Kwon <[email protected]>
---
.../sql/catalyst/expressions/generators.scala | 18 ++++++++++++----
.../spark/sql/catalyst/expressions/misc.scala | 5 +++++
.../expressions/MiscExpressionsSuite.scala | 25 ++++++++++++++++++++++
.../org/apache/spark/sql/DataFrameSuite.scala | 19 ++++++++++++++++
.../scala/org/apache/spark/sql/hive/hiveUDFs.scala | 15 +++++++++++--
.../spark/sql/hive/execution/HiveUDFSuite.scala | 25 +++++++++++++++++++++-
6 files changed, 100 insertions(+), 7 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala
index 3f2e838c1903..7a19855ba3f0 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala
@@ -105,11 +105,10 @@ case class UserDefinedGenerator(
children: Seq[Expression])
extends Generator with CodegenFallback {
- @transient private[this] var inputRow: InterpretedProjection = _
@transient private[this] var convertToScala: (InternalRow) => Row = _
+ @transient private lazy val inputExpressions = children.toArray
private def initializeConverters(): Unit = {
- inputRow = new InterpretedProjection(children)
convertToScala = {
val inputSchema = StructType(children.map { e =>
StructField(e.simpleString(SQLConf.get.maxToStringFields), e.dataType,
nullable = true)
@@ -119,11 +118,22 @@ case class UserDefinedGenerator(
}
override def eval(input: InternalRow): IterableOnce[InternalRow] = {
- if (inputRow == null) {
+ if (convertToScala == null) {
initializeConverters()
}
+ // GenerateExec initializes these children for the partition. A projection
would fresh-copy
+ // stateful expressions without initializing the copies. Fill the final
array directly to
+ // avoid creating an intermediate collection for each row.
+ val expressions = inputExpressions
+ val outputArray = new Array[Any](expressions.length)
+ var i = 0
+ while (i < expressions.length) {
+ outputArray(i) = expressions(i).eval(input)
+ i += 1
+ }
+ val inputRow = new GenericInternalRow(outputArray)
// Convert the objects into Scala Type before calling function, we need
schema to support UDT
- function(convertToScala(inputRow(input)))
+ function(convertToScala(inputRow))
}
override def toString: String =
s"UserDefinedGenerator(${children.mkString(",")})"
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala
index 61e04b9f761c..6fd653789f10 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala
@@ -318,6 +318,11 @@ case class Uuid(randomSeed: Option[Long] = None) extends
LeafExpression with Non
override def stateful: Boolean = true
+ // Stateful fresh copies use this hook even for leaves. The default leaf
hook returns this,
+ // which would share randomGenerator across evaluators; copy preserves the
seed without
+ // sharing the initialized generator.
+ override def withNewChildrenInternal(newChildren: IndexedSeq[Expression]):
Expression = copy()
+
@transient private[this] var randomGenerator: RandomUUIDGenerator = _
override protected def initializeInternal(partitionIndex: Int): Unit =
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/MiscExpressionsSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/MiscExpressionsSuite.scala
index a586e33afdd2..8bddd43c69da 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/MiscExpressionsSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/MiscExpressionsSuite.scala
@@ -94,6 +94,31 @@ class MiscExpressionsSuite extends SparkFunSuite with
ExpressionEvalHelper {
evaluateWithUnsafeProjection(new Uuid(seed3)))
}
+ Seq(0L, 42L, Long.MinValue).foreach { seed =>
+ test(s"uuid fresh copies own independent generators for seed $seed") {
+ val original = Uuid(Some(seed))
+ val first =
original.freshCopyIfContainsStatefulExpression().asInstanceOf[Uuid]
+ val second =
original.freshCopyIfContainsStatefulExpression().asInstanceOf[Uuid]
+ assert(first ne original)
+ assert(second ne original)
+ assert(first ne second)
+ assert(first.randomSeed == original.randomSeed)
+ original.initialize(0)
+ first.initialize(0)
+ second.initialize(0)
+ for (_ <- 0 until 20) {
+ val expected = original.eval()
+ assert(first.eval() == expected)
+ assert(second.eval() == expected)
+ }
+ first.initialize(1)
+ assert(second.eval() == original.eval())
+ val otherPartition = Uuid(Some(seed))
+ otherPartition.initialize(1)
+ assert(first.eval() == otherPartition.eval())
+ }
+ }
+
test("PrintToStderr") {
val inputExpr = Literal(1)
val systemErr = System.err
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
index d2ba7f17ee96..0edb2861b2d3 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
@@ -36,6 +36,7 @@ import org.apache.spark.sql.catalyst.expressions.{Attribute,
AttributeReference,
import org.apache.spark.sql.catalyst.optimizer.ConvertToLocalRelation
import org.apache.spark.sql.catalyst.parser.ParseException
import org.apache.spark.sql.catalyst.plans.logical.{Filter, LeafNode,
LocalRelation, LogicalPlan, OneRowRelation}
+import org.apache.spark.sql.catalyst.util.RandomUUIDGenerator
import org.apache.spark.sql.connector.FakeV2Provider
import org.apache.spark.sql.execution.{FilterExec, LogicalRDD, QueryExecution,
SortExec, WholeStageCodegenExec}
import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
@@ -111,6 +112,24 @@ class DataFrameSuite extends SharedSparkSession
)
}
+ for (seed <- Seq(0L, 42L, Long.MinValue); codegen <- Seq(true, false)) {
+ test(s"SPARK-60634: Dataset.explode initializes UUID arguments ($seed,
$codegen)") {
+ withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> codegen.toString) {
+ val result = spark.range(0, 6, 1, 2).explode(
+ expr(s"uuid($seed)"), lit(null).cast(StringType), struct(lit(7))) {
row: Row =>
+ Seq((row.getString(0), row.isNullAt(1), row.getStruct(2).getInt(0)))
+ }
+ val expected = (0 until 2).flatMap { partition =>
+ val generator = RandomUUIDGenerator(seed + partition)
+ (0 until 3).map { offset =>
+ Row(partition * 3L + offset, generator.getNextUUID().toString,
true, 7)
+ }
+ }
+ checkAnswer(result.select("id", "_1", "_2", "_3"), expected)
+ }
+ }
+ }
+
test("explode") {
val df = Seq((1, "a b c"), (2, "a b"), (3, "a")).toDF("number", "letters")
val df2 =
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala
index 7472341eabb2..93fd721bc390 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala
@@ -260,11 +260,22 @@ private[hive] case class HiveGenericUDTF(
private lazy val unwrapper = unwrapperFor(outputInspector, elementSchema)
@transient
- private lazy val inputProjection = new InterpretedProjection(children)
+ private lazy val inputExpressions = children.toArray
override def eval(input: InternalRow): IterableOnce[InternalRow] = {
outputInspector // Make sure initialized.
- function.process(wrap(inputProjection(input), wrappers, udtInput,
inputDataTypes))
+ // GenerateExec has initialized these children for the partition. A
projection would create
+ // fresh stateful copies without initializing them. Fill the final array
directly to avoid
+ // creating an intermediate collection for each row.
+ val expressions = inputExpressions
+ val outputArray = new Array[Any](expressions.length)
+ var i = 0
+ while (i < expressions.length) {
+ outputArray(i) = expressions(i).eval(input)
+ i += 1
+ }
+ val inputRow = new GenericInternalRow(outputArray)
+ function.process(wrap(inputRow, wrappers, udtInput, inputDataTypes))
collector.collectRows()
}
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
index 5b5474512f29..54824bb479c3 100644
---
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
@@ -44,7 +44,7 @@ import org.apache.spark.sql.{AnalysisException, QueryTest,
Row}
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions.{ApplyFunctionExpression,
AttributeReference, BindReferences, CodegenObjectFactoryMode, Literal}
import org.apache.spark.sql.catalyst.plans.logical.{Filter, Project}
-import org.apache.spark.sql.catalyst.util.{DateTimeUtils, GenericArrayData}
+import org.apache.spark.sql.catalyst.util.{DateTimeUtils, GenericArrayData,
RandomUUIDGenerator}
import org.apache.spark.sql.connector.catalog.functions.{ScalarFunction,
SimpleFunction}
import org.apache.spark.sql.execution.WholeStageCodegenExec
import org.apache.spark.sql.functions.{call_function, max}
@@ -74,6 +74,29 @@ class HiveUDFSuite extends QueryTest with TestHiveSingleton {
import spark.udf
+ for (seed <- Seq(0L, 42L, Long.MinValue); codegen <- Seq(true, false)) {
+ test(s"SPARK-60634: Hive UDTFs initialize UUID arguments ($seed,
$codegen)") {
+ withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> codegen.toString) {
+ withUserDefinedFunction("hive_uuid_explode" -> true) {
+ sql(s"CREATE TEMPORARY FUNCTION hive_uuid_explode AS " +
+ s"'${classOf[GenericUDTFExplode].getName}'")
+ val expected = (0 until 2).flatMap { partition =>
+ val generator = RandomUUIDGenerator(seed + partition)
+ (0 until 3).flatMap { offset =>
+ val id = partition * 3L + offset
+ Seq(Row(id, generator.getNextUUID().toString), Row(id, null),
Row(id, id.toString))
+ }
+ }
+ val argument = s"array(uuid($seed), CAST(NULL AS STRING), CAST(id AS
STRING))"
+ checkAnswer(sql(s"SELECT id, hive_uuid_explode($argument) AS value "
+
+ "FROM range(0, 6, 1, 2)"), expected)
+ checkAnswer(sql(s"SELECT id, value FROM range(0, 6, 1, 2) " +
+ s"LATERAL VIEW hive_uuid_explode($argument) v AS value"), expected)
+ }
+ }
+ }
+ }
+
test("spark sql udf test that returns a struct") {
udf.register("getStruct", (_: Int) => Fields(1, 2, 3, 4, 5))
assert(sql(
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]