This is an automated email from the ASF dual-hosted git repository.

sunchao pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-4.x by this push:
     new f7c391791d40 [SPARK-60179][SQL] Treat a cast to STRING as time-zone 
dependent when the source type contains a nested TIMESTAMP_LTZ
f7c391791d40 is described below

commit f7c391791d4093929532b8813900b86c47fafad1
Author: hu hengrui <[email protected]>
AuthorDate: Fri Oct 23 08:24:19 2026 -0700

    [SPARK-60179][SQL] Treat a cast to STRING as time-zone dependent when the 
source type contains a nested TIMESTAMP_LTZ
    
    ### What changes were proposed in this pull request?
    
    `Cast.needsTimeZone` incorrectly treats complex-to-STRING casts containing 
nested LTZ timestamps as time-zone independent, even though `castToString` uses 
the session time zone.
    
    This PR fixes the classification by recursively checking the source type, 
including Python UDTs. TIMESTAMP_NTZ, DATE and TIME remain time-zone 
independent.
    
    ### Why are the changes needed?
    
    The incorrect classification can cause unresolved time-zone access, 
incorrect expression equivalence, and wrong results when cached queries are 
reused across session time zones.
    
    ### Does this PR introduce any user-facing change?
    
    Yes. Complex-to-STRING casts now correctly preserve time-zone semantics, 
preventing incorrect cross-time-zone cache reuse.
    
    ### How was this patch tested?
    
    - Added tests for nested types, Python UDTs, expression resolution, 
semantic equality, and cross-time-zone cache reuse.
    - Mutation testing confirmed that the affected cache tests fail without the 
fix, while the scalar TIMESTAMP control still passes.
    - All 276 Cast tests and 107 CachedTableSuite tests pass.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Yes. Co-authored-by: Claude Code (Claude Opus 5)
    
    Closes #59394 from hhr293/SPARK-60179.
    
    Authored-by: hu hengrui <[email protected]>
    Signed-off-by: Chao Sun <[email protected]>
    (cherry picked from commit 11e2e6c3700f97aa42a001067e1872bcde7b03e4)
    Signed-off-by: Chao Sun <[email protected]>
---
 .../spark/sql/catalyst/expressions/Cast.scala      | 22 +++++++++
 .../sql/catalyst/expressions/CastSuiteBase.scala   | 53 ++++++++++++++++++++++
 .../org/apache/spark/sql/CachedTableSuite.scala    | 31 +++++++++++++
 3 files changed, 106 insertions(+)

diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
index d5f628afc6b5..c41bcbf76ad3 100644
--- 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
+++ 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
@@ -380,6 +380,8 @@ object Cast extends QueryErrorsBase {
    * (micro and nanosecond), TIME -> TIMESTAMP_NTZ (whose date fields come 
from CURRENT_DATE), and
    * both directions of TIME <-> TIMESTAMP_LTZ (the LTZ value is an absolute 
instant, so extracting
    * its time-of-day and attaching CURRENT_DATE to a TIME both depend on the 
session time zone).
+   * A cast to STRING is zone-dependent whenever one of those timestamps 
appears anywhere in the
+   * source type, including under ARRAY/MAP/STRUCT.
    */
   def needsTimeZone(from: DataType, to: DataType): Boolean = (from, to) match {
     case (VariantType, _) => true
@@ -429,6 +431,26 @@ object Cast extends QueryErrorsBase {
           case (fromField, toField) =>
             needsTimeZone(fromField.dataType, toField.dataType)
         }
+    // `castToString` recurses into ARRAY/MAP/STRUCT and formats each LTZ leaf 
with `zoneId`, while
+    // the complex-to-complex cases above only match a target of the same 
shape.
+    case (from, _: StringType) => from.existsRecursively(needsTimeZoneToString)
+    case _ => false
+  }
+
+  /**
+   * Whether this type node reads the session time zone when rendered as a 
string.
+   * ARRAY/MAP/STRUCT nodes return false; the caller uses `existsRecursively` 
to inspect
+   * nested types, while Python UDTs recursively check their `sqlType`.
+   *
+   * Only the LTZ families carry an absolute instant that has to be localized;
+   * TIMESTAMP_NTZ, DATE and TIME render on a fixed grid, and a nested VARIANT 
renders
+   * through `VariantVal.toString` at UTC (only a top-level VARIANT reads the 
zone,
+   * via the `(VariantType, _)` arm).
+   */
+  private def needsTimeZoneToString(dt: DataType): Boolean = dt match {
+    case TimestampType | _: TimestampLTZNanosType => true
+    // A Python UDT renders through its `sqlType`; a Scala UDT through its 
`stringifyValue`.
+    case pudt: PythonUserDefinedType => 
pudt.sqlType.existsRecursively(needsTimeZoneToString)
     case _ => false
   }
 
diff --git 
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuiteBase.scala
 
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuiteBase.scala
index a99d1f5e4047..c02acc157b32 100644
--- 
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuiteBase.scala
+++ 
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuiteBase.scala
@@ -2698,6 +2698,59 @@ abstract class CastSuiteBase extends SparkFunSuite with 
ExpressionEvalHelper {
     assert(!Cast.isTimeToTimestampLTZ(TimestampType, TimestampType))
   }
 
+  test("SPARK-60179: needsTimeZone covers complex types collapsing to a 
string") {
+    // An LTZ leaf is rendered in the session time zone at any nesting depth.
+    Seq(
+      ArrayType(TimestampType),
+      MapType(IntegerType, TimestampType),
+      MapType(TimestampType, IntegerType),
+      new StructType().add("t", TimestampType),
+      new StructType().add("i", IntegerType).add("t", 
TimestampLTZNanosType(9)),
+      ArrayType(ArrayType(TimestampType)),
+      ArrayType(new StructType().add("t", TimestampType)),
+      new PythonUserDefinedType(ArrayType(TimestampType), "pyClass", 
"serializedClass"))
+      .foreach { from =>
+      assert(Cast.needsTimeZone(from, StringType), s"$from -> STRING")
+      // One level further out, the ARRAY/ARRAY case recurses into the pair 
above.
+      assert(Cast.needsTimeZone(ArrayType(from), ArrayType(StringType)),
+        s"ARRAY<$from> -> ARRAY<STRING>")
+    }
+    // TIMESTAMP_NTZ, DATE and TIME render on a fixed grid, and a nested 
VARIANT renders at UTC.
+    Seq(
+      ArrayType(TimestampNTZType),
+      MapType(IntegerType, DateType),
+      new StructType().add("t", TimestampNTZNanosType(9)).add("d", DateType),
+      ArrayType(ArrayType(TimeType(TimeType.MAX_PRECISION))),
+      ArrayType(VariantType),
+      MapType(IntegerType, VariantType),
+      new StructType().add("v", VariantType),
+      new PythonUserDefinedType(ArrayType(TimestampNTZType), "pyClass", 
"serializedClass"))
+      .foreach { from =>
+      assert(!Cast.needsTimeZone(from, StringType), s"$from -> STRING")
+      assert(!Cast.needsTimeZone(ArrayType(from), ArrayType(StringType)),
+        s"ARRAY<$from> -> ARRAY<STRING>")
+    }
+    // A top-level VARIANT still reads the zone, element-wise casts included.
+    assert(Cast.needsTimeZone(VariantType, StringType))
+    assert(Cast.needsTimeZone(ArrayType(VariantType), ArrayType(StringType)))
+  }
+
+  test("SPARK-60179: a complex-to-string cast needs a time zone to be 
resolved") {
+    val arrayOfTimestamp = Literal.create(Seq(0L), ArrayType(TimestampType))
+    // Without a time zone the cast cannot be evaluated: the leaf formatter 
reads `zoneId`.
+    assert(!Cast(arrayOfTimestamp, StringType).resolved)
+    assert(Cast(arrayOfTimestamp, StringType, UTC_OPT).resolved)
+    // `canonicalized` may no longer drop the zone, so two zones are not 
semantically equal.
+    assert(!Cast(arrayOfTimestamp, StringType, UTC_OPT)
+      .semanticEquals(Cast(arrayOfTimestamp, StringType, Option(LA.getId))))
+    checkEvaluation(Cast(arrayOfTimestamp, StringType, UTC_OPT), "[1970-01-01 
00:00:00]")
+    checkEvaluation(
+      Cast(arrayOfTimestamp, StringType, Option(LA.getId)), "[1969-12-31 
16:00:00]")
+    // The NTZ control renders on a fixed grid and stays resolvable without a 
zone.
+    val arrayOfTimestampNTZ = Literal.create(Seq(0L), 
ArrayType(TimestampNTZType))
+    assert(Cast(arrayOfTimestampNTZ, StringType).resolved)
+  }
+
   test("cast timestamp_ntz to time") {
     // Per ANSI rule 15.d the time-of-day fields are extracted and truncated 
to the target
     // precision; the operation is deterministic and time-zone independent.
diff --git 
a/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala 
b/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
index 015e71d39937..e228bd9ea67e 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
@@ -2853,6 +2853,37 @@ class CachedTableSuite extends SharedSparkSession
     }
   }
 
+  test("SPARK-60179: cached plan is not reused across session time zones for a 
" +
+    "complex-to-string cast") {
+    withTempView("t") {
+      sql("""SELECT array(timestamp_micros(0)) AS ats,
+            |       array(array(timestamp_micros(0))) AS aats,
+            |       timestamp_micros(0) AS ts""".stripMargin)
+        .createOrReplaceTempView("t")
+      // All three columns hold the same instant, and rendering one as a 
string localizes it. The
+      // scalar column is the control: it was already classified as 
zone-dependent.
+      Seq(
+        ("ats", "[1970-01-01 00:00:00]", "[1969-12-31 16:00:00]"),
+        ("aats", "[[1970-01-01 00:00:00]]", "[[1969-12-31 16:00:00]]"),
+        ("ts", "1970-01-01 00:00:00", "1969-12-31 16:00:00")).foreach {
+        case (column, inUTC, inLA) =>
+          val query = s"SELECT CAST($column AS STRING) AS v FROM t"
+          withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> "UTC") {
+            val cached = sql(query)
+            cached.persist()
+            try {
+              checkAnswer(cached, Row(inUTC))
+              withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> 
"America/Los_Angeles") {
+                checkAnswer(sql(query), Row(inLA))
+              }
+            } finally {
+              cached.unpersist()
+            }
+          }
+      }
+    }
+  }
+
   private def cacheManager = spark.sharedState.cacheManager
 
   private def pinTable(


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to