This is an automated email from the ASF dual-hosted git repository.
sunchao pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new f7c391791d40 [SPARK-60179][SQL] Treat a cast to STRING as time-zone
dependent when the source type contains a nested TIMESTAMP_LTZ
f7c391791d40 is described below
commit f7c391791d4093929532b8813900b86c47fafad1
Author: hu hengrui <[email protected]>
AuthorDate: Fri Oct 23 08:24:19 2026 -0700
[SPARK-60179][SQL] Treat a cast to STRING as time-zone dependent when the
source type contains a nested TIMESTAMP_LTZ
### What changes were proposed in this pull request?
`Cast.needsTimeZone` incorrectly treats complex-to-STRING casts containing
nested LTZ timestamps as time-zone independent, even though `castToString` uses
the session time zone.
This PR fixes the classification by recursively checking the source type,
including Python UDTs. TIMESTAMP_NTZ, DATE and TIME remain time-zone
independent.
### Why are the changes needed?
The incorrect classification can cause unresolved time-zone access,
incorrect expression equivalence, and wrong results when cached queries are
reused across session time zones.
### Does this PR introduce any user-facing change?
Yes. Complex-to-STRING casts now correctly preserve time-zone semantics,
preventing incorrect cross-time-zone cache reuse.
### How was this patch tested?
- Added tests for nested types, Python UDTs, expression resolution,
semantic equality, and cross-time-zone cache reuse.
- Mutation testing confirmed that the affected cache tests fail without the
fix, while the scalar TIMESTAMP control still passes.
- All 276 Cast tests and 107 CachedTableSuite tests pass.
### Was this patch authored or co-authored using generative AI tooling?
Yes. Co-authored-by: Claude Code (Claude Opus 5)
Closes #59394 from hhr293/SPARK-60179.
Authored-by: hu hengrui <[email protected]>
Signed-off-by: Chao Sun <[email protected]>
(cherry picked from commit 11e2e6c3700f97aa42a001067e1872bcde7b03e4)
Signed-off-by: Chao Sun <[email protected]>
---
.../spark/sql/catalyst/expressions/Cast.scala | 22 +++++++++
.../sql/catalyst/expressions/CastSuiteBase.scala | 53 ++++++++++++++++++++++
.../org/apache/spark/sql/CachedTableSuite.scala | 31 +++++++++++++
3 files changed, 106 insertions(+)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
index d5f628afc6b5..c41bcbf76ad3 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
@@ -380,6 +380,8 @@ object Cast extends QueryErrorsBase {
* (micro and nanosecond), TIME -> TIMESTAMP_NTZ (whose date fields come
from CURRENT_DATE), and
* both directions of TIME <-> TIMESTAMP_LTZ (the LTZ value is an absolute
instant, so extracting
* its time-of-day and attaching CURRENT_DATE to a TIME both depend on the
session time zone).
+ * A cast to STRING is zone-dependent whenever one of those timestamps
appears anywhere in the
+ * source type, including under ARRAY/MAP/STRUCT.
*/
def needsTimeZone(from: DataType, to: DataType): Boolean = (from, to) match {
case (VariantType, _) => true
@@ -429,6 +431,26 @@ object Cast extends QueryErrorsBase {
case (fromField, toField) =>
needsTimeZone(fromField.dataType, toField.dataType)
}
+ // `castToString` recurses into ARRAY/MAP/STRUCT and formats each LTZ leaf
with `zoneId`, while
+ // the complex-to-complex cases above only match a target of the same
shape.
+ case (from, _: StringType) => from.existsRecursively(needsTimeZoneToString)
+ case _ => false
+ }
+
+ /**
+ * Whether this type node reads the session time zone when rendered as a
string.
+ * ARRAY/MAP/STRUCT nodes return false; the caller uses `existsRecursively`
to inspect
+ * nested types, while Python UDTs recursively check their `sqlType`.
+ *
+ * Only the LTZ families carry an absolute instant that has to be localized;
+ * TIMESTAMP_NTZ, DATE and TIME render on a fixed grid, and a nested VARIANT
renders
+ * through `VariantVal.toString` at UTC (only a top-level VARIANT reads the
zone,
+ * via the `(VariantType, _)` arm).
+ */
+ private def needsTimeZoneToString(dt: DataType): Boolean = dt match {
+ case TimestampType | _: TimestampLTZNanosType => true
+ // A Python UDT renders through its `sqlType`; a Scala UDT through its
`stringifyValue`.
+ case pudt: PythonUserDefinedType =>
pudt.sqlType.existsRecursively(needsTimeZoneToString)
case _ => false
}
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuiteBase.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuiteBase.scala
index a99d1f5e4047..c02acc157b32 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuiteBase.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuiteBase.scala
@@ -2698,6 +2698,59 @@ abstract class CastSuiteBase extends SparkFunSuite with
ExpressionEvalHelper {
assert(!Cast.isTimeToTimestampLTZ(TimestampType, TimestampType))
}
+ test("SPARK-60179: needsTimeZone covers complex types collapsing to a
string") {
+ // An LTZ leaf is rendered in the session time zone at any nesting depth.
+ Seq(
+ ArrayType(TimestampType),
+ MapType(IntegerType, TimestampType),
+ MapType(TimestampType, IntegerType),
+ new StructType().add("t", TimestampType),
+ new StructType().add("i", IntegerType).add("t",
TimestampLTZNanosType(9)),
+ ArrayType(ArrayType(TimestampType)),
+ ArrayType(new StructType().add("t", TimestampType)),
+ new PythonUserDefinedType(ArrayType(TimestampType), "pyClass",
"serializedClass"))
+ .foreach { from =>
+ assert(Cast.needsTimeZone(from, StringType), s"$from -> STRING")
+ // One level further out, the ARRAY/ARRAY case recurses into the pair
above.
+ assert(Cast.needsTimeZone(ArrayType(from), ArrayType(StringType)),
+ s"ARRAY<$from> -> ARRAY<STRING>")
+ }
+ // TIMESTAMP_NTZ, DATE and TIME render on a fixed grid, and a nested
VARIANT renders at UTC.
+ Seq(
+ ArrayType(TimestampNTZType),
+ MapType(IntegerType, DateType),
+ new StructType().add("t", TimestampNTZNanosType(9)).add("d", DateType),
+ ArrayType(ArrayType(TimeType(TimeType.MAX_PRECISION))),
+ ArrayType(VariantType),
+ MapType(IntegerType, VariantType),
+ new StructType().add("v", VariantType),
+ new PythonUserDefinedType(ArrayType(TimestampNTZType), "pyClass",
"serializedClass"))
+ .foreach { from =>
+ assert(!Cast.needsTimeZone(from, StringType), s"$from -> STRING")
+ assert(!Cast.needsTimeZone(ArrayType(from), ArrayType(StringType)),
+ s"ARRAY<$from> -> ARRAY<STRING>")
+ }
+ // A top-level VARIANT still reads the zone, element-wise casts included.
+ assert(Cast.needsTimeZone(VariantType, StringType))
+ assert(Cast.needsTimeZone(ArrayType(VariantType), ArrayType(StringType)))
+ }
+
+ test("SPARK-60179: a complex-to-string cast needs a time zone to be
resolved") {
+ val arrayOfTimestamp = Literal.create(Seq(0L), ArrayType(TimestampType))
+ // Without a time zone the cast cannot be evaluated: the leaf formatter
reads `zoneId`.
+ assert(!Cast(arrayOfTimestamp, StringType).resolved)
+ assert(Cast(arrayOfTimestamp, StringType, UTC_OPT).resolved)
+ // `canonicalized` may no longer drop the zone, so two zones are not
semantically equal.
+ assert(!Cast(arrayOfTimestamp, StringType, UTC_OPT)
+ .semanticEquals(Cast(arrayOfTimestamp, StringType, Option(LA.getId))))
+ checkEvaluation(Cast(arrayOfTimestamp, StringType, UTC_OPT), "[1970-01-01
00:00:00]")
+ checkEvaluation(
+ Cast(arrayOfTimestamp, StringType, Option(LA.getId)), "[1969-12-31
16:00:00]")
+ // The NTZ control renders on a fixed grid and stays resolvable without a
zone.
+ val arrayOfTimestampNTZ = Literal.create(Seq(0L),
ArrayType(TimestampNTZType))
+ assert(Cast(arrayOfTimestampNTZ, StringType).resolved)
+ }
+
test("cast timestamp_ntz to time") {
// Per ANSI rule 15.d the time-of-day fields are extracted and truncated
to the target
// precision; the operation is deterministic and time-zone independent.
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
index 015e71d39937..e228bd9ea67e 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
@@ -2853,6 +2853,37 @@ class CachedTableSuite extends SharedSparkSession
}
}
+ test("SPARK-60179: cached plan is not reused across session time zones for a
" +
+ "complex-to-string cast") {
+ withTempView("t") {
+ sql("""SELECT array(timestamp_micros(0)) AS ats,
+ | array(array(timestamp_micros(0))) AS aats,
+ | timestamp_micros(0) AS ts""".stripMargin)
+ .createOrReplaceTempView("t")
+ // All three columns hold the same instant, and rendering one as a
string localizes it. The
+ // scalar column is the control: it was already classified as
zone-dependent.
+ Seq(
+ ("ats", "[1970-01-01 00:00:00]", "[1969-12-31 16:00:00]"),
+ ("aats", "[[1970-01-01 00:00:00]]", "[[1969-12-31 16:00:00]]"),
+ ("ts", "1970-01-01 00:00:00", "1969-12-31 16:00:00")).foreach {
+ case (column, inUTC, inLA) =>
+ val query = s"SELECT CAST($column AS STRING) AS v FROM t"
+ withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> "UTC") {
+ val cached = sql(query)
+ cached.persist()
+ try {
+ checkAnswer(cached, Row(inUTC))
+ withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key ->
"America/Los_Angeles") {
+ checkAnswer(sql(query), Row(inLA))
+ }
+ } finally {
+ cached.unpersist()
+ }
+ }
+ }
+ }
+ }
+
private def cacheManager = spark.sharedState.cacheManager
private def pinTable(
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]