This is an automated email from the ASF dual-hosted git repository.
viirya pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 5ceafeda5c51 [SPARK-58414][SQL][TEST] Add e2e coverage for nanosecond
timestamps nested in complex types in the Arrow cache
5ceafeda5c51 is described below
commit 5ceafeda5c512457d012fef81cf5b5154c41b2c9
Author: Liang-Chi Hsieh <[email protected]>
AuthorDate: Wed Jul 29 07:52:25 2026 -0700
[SPARK-58414][SQL][TEST] Add e2e coverage for nanosecond timestamps nested
in complex types in the Arrow cache
### What changes were proposed in this pull request?
Add a test to `ArrowCachedBatchSerializerSuite` that round-trips nanosecond
timestamps nested inside an array, a struct field, and a map value through the
Arrow cache, using a value outside the int64 epoch-nanos window (year 3000 at
nanosecond precision) that the standard interchange encoding cannot represent,
alongside an in-window value.
### Why are the changes needed?
The suite covers top-level nanosecond timestamps and nested
`CalendarInterval`, but had no round trip for nanosecond timestamps nested in
complex types. The nested path relies on recursion in two places:
`ArrowWriter`'s field writers dispatch on `(type, vector)` recursively on the
write side, and on the read side every container accessor in
`ArrowColumnVector` (struct children, `ArrayAccessor`'s data vector,
`MapAccessor`'s keys/values) wraps its element vector through the constructor
[...]
### Does this PR introduce _any_ user-facing change?
No, test-only.
### How was this patch tested?
The new test; full `ArrowCachedBatchSerializerSuite` and
`ArrowCachedBatchKryoRegistrationSuite` pass (76 tests).
### Was this patch authored or co-authored using generative AI tooling?
Yes, this pull request and its description were written by Claude Code.
Closes #57619 from viirya/arrow-cache-nested-nanos-test.
Authored-by: Liang-Chi Hsieh <[email protected]>
Signed-off-by: Liang-Chi Hsieh <[email protected]>
---
.../columnar/ArrowCachedBatchSerializerSuite.scala | 48 ++++++++++++++++++++++
1 file changed, 48 insertions(+)
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/ArrowCachedBatchSerializerSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/ArrowCachedBatchSerializerSuite.scala
index 621fc4d48130..07672ad3e5cc 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/ArrowCachedBatchSerializerSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/ArrowCachedBatchSerializerSuite.scala
@@ -2276,6 +2276,54 @@ class ArrowCachedBatchSerializerSuite extends QueryTest
with SharedSparkSession
}
}
+ test("nanosecond timestamps round-trip inside complex types") {
+ // Nested nanosecond timestamps go through the same recursive machinery as
top-level ones:
+ // ArrowWriter's field writers dispatch recursively on (type, vector), and
every container
+ // accessor in ArrowColumnVector wraps its element vector through the
constructor that runs
+ // the tagged-struct recognizers -- so the lossless struct representation
must round-trip at
+ // any nesting depth, including values outside the int64 epoch-nanos
window (~1677-2262)
+ // that the standard interchange encoding cannot represent.
+ val outOfWindow = java.time.LocalDateTime.of(3000, 1, 6, 12, 30, 45,
123456789)
+ val inWindow = java.time.LocalDateTime.of(2025, 1, 6, 12, 30, 45,
987654321)
+ val nanosType = TimestampNTZNanosType(9)
+
+ val arrayDf = singlePartDf(
+ Seq(Seq(outOfWindow, inWindow)), ArrayType(nanosType)).cache()
+ try {
+ assert(arrayDf.count() == 1)
+ val read = arrayDf.collect().head.getSeq[java.time.LocalDateTime](0)
+ assert(read == Seq(outOfWindow, inWindow),
+ s"expected nested nanos to round-trip through an array, got: $read")
+ } finally {
+ arrayDf.unpersist()
+ InMemoryRelation.clearSerializer()
+ }
+
+ val structDf = singlePartDf(
+ Seq(Row(outOfWindow)), StructType(Seq(StructField("ts",
nanosType)))).cache()
+ try {
+ assert(structDf.count() == 1)
+ val read =
structDf.collect().head.getStruct(0).getAs[java.time.LocalDateTime](0)
+ assert(read == outOfWindow,
+ s"expected nested nanos to round-trip through a struct, got: $read")
+ } finally {
+ structDf.unpersist()
+ InMemoryRelation.clearSerializer()
+ }
+
+ val mapDf = singlePartDf(
+ Seq(Map(1 -> outOfWindow)), MapType(IntegerType, nanosType)).cache()
+ try {
+ assert(mapDf.count() == 1)
+ val read = mapDf.collect().head.getMap[Int, java.time.LocalDateTime](0)
+ assert(read == Map(1 -> outOfWindow),
+ s"expected nested nanos to round-trip through a map value, got: $read")
+ } finally {
+ mapDf.unpersist()
+ InMemoryRelation.clearSerializer()
+ }
+ }
+
test("nonpositive maxRecordsPerBatch caches all rows in a single batch") {
// A nonpositive maxRecordsPerBatch means unlimited; without the `<= 0`
guard the write
// iterator would emit zero-row batches forever instead of finishing.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]