dongjoon-hyun commented on code in PR #57619:
URL: https://github.com/apache/spark/pull/57619#discussion_r3675461763
##########
sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/ArrowCachedBatchSerializerSuite.scala:
##########
@@ -2276,6 +2276,54 @@ class ArrowCachedBatchSerializerSuite extends QueryTest
with SharedSparkSession
}
}
+ test("nanosecond timestamps round-trip inside complex types") {
+ // Nested nanosecond timestamps go through the same recursive machinery as
top-level ones:
+ // ArrowWriter's field writers dispatch recursively on (type, vector), and
every container
+ // accessor in ArrowColumnVector wraps its element vector through the
constructor that runs
+ // the tagged-struct recognizers -- so the lossless struct representation
must round-trip at
+ // any nesting depth, including values outside the int64 epoch-nanos
window (~1677-2262)
+ // that the standard interchange encoding cannot represent.
+ val outOfWindow = java.time.LocalDateTime.of(3000, 1, 6, 12, 30, 45,
123456789)
+ val inWindow = java.time.LocalDateTime.of(2025, 1, 6, 12, 30, 45,
987654321)
+ val nanosType = TimestampNTZNanosType(9)
+
+ val arrayDf = singlePartDf(
+ Seq(Seq(outOfWindow, inWindow)), ArrayType(nanosType)).cache()
+ try {
+ assert(arrayDf.count() == 1)
+ val read = arrayDf.collect().head.getSeq[java.time.LocalDateTime](0)
+ assert(read == Seq(outOfWindow, inWindow),
+ s"expected nested nanos to round-trip through an array, got: $read")
+ } finally {
+ arrayDf.unpersist()
+ InMemoryRelation.clearSerializer()
+ }
+
+ val structDf = singlePartDf(
+ Seq(Row(outOfWindow)), StructType(Seq(StructField("ts",
nanosType)))).cache()
+ try {
+ assert(structDf.count() == 1)
+ val read =
structDf.collect().head.getStruct(0).getAs[java.time.LocalDateTime](0)
Review Comment:
```suggestion
val read = structDf.collect().head.getStruct(0).getAs[LocalDateTime](0)
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
