This is an automated email from the ASF dual-hosted git repository.
dongjoon-hyun pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new 0d0c544f4867 [SPARK-58414][SQL][TEST][FOLLOWUP] Use the imported
LocalDateTime instead of the fully qualified name
0d0c544f4867 is described below
commit 0d0c544f4867f96bc823e9531f4d193d2bb1c4ba
Author: Liang-Chi Hsieh <[email protected]>
AuthorDate: Wed Jul 29 08:59:50 2026 -0700
[SPARK-58414][SQL][TEST][FOLLOWUP] Use the imported LocalDateTime instead
of the fully qualified name
### What changes were proposed in this pull request?
Two cleanups to the test added by #57619, both from dongjoon-hyun's
post-merge reviews:
1. Replace the fully qualified `java.time.LocalDateTime` references with
the bare `LocalDateTime`, which the suite already imports.
2. Run the nested nanos round trip under both
`spark.sql.inMemoryColumnarStorage.enableVectorizedReader` settings via the
same `Seq(false, true)` loop the neighboring round-trip tests use, since the
suite's default pins the conf to `false`.
### Why are the changes needed?
Addresses the post-merge reviews on #57619.
### Does this PR introduce _any_ user-facing change?
No, test-only.
### How was this patch tested?
The affected test passes under both conf values.
### Was this patch authored or co-authored using generative AI tooling?
Yes, this pull request and its description were written by Claude Code.
Closes #57631 from viirya/arrow-cache-nested-nanos-followup.
Authored-by: Liang-Chi Hsieh <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
(cherry picked from commit a06f5910cb25c0b83a68d9bf8ad239fe6b7bbde1)
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../columnar/ArrowCachedBatchSerializerSuite.scala | 75 ++++++++++++----------
1 file changed, 40 insertions(+), 35 deletions(-)
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/ArrowCachedBatchSerializerSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/ArrowCachedBatchSerializerSuite.scala
index 07672ad3e5cc..73c2b15daf2f 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/ArrowCachedBatchSerializerSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/ArrowCachedBatchSerializerSuite.scala
@@ -2282,45 +2282,50 @@ class ArrowCachedBatchSerializerSuite extends QueryTest
with SharedSparkSession
// accessor in ArrowColumnVector wraps its element vector through the
constructor that runs
// the tagged-struct recognizers -- so the lossless struct representation
must round-trip at
// any nesting depth, including values outside the int64 epoch-nanos
window (~1677-2262)
- // that the standard interchange encoding cannot represent.
- val outOfWindow = java.time.LocalDateTime.of(3000, 1, 6, 12, 30, 45,
123456789)
- val inWindow = java.time.LocalDateTime.of(2025, 1, 6, 12, 30, 45,
987654321)
+ // that the standard interchange encoding cannot represent. Like the
neighboring round-trip
+ // tests, run under both vectorized-reader settings.
+ val outOfWindow = LocalDateTime.of(3000, 1, 6, 12, 30, 45, 123456789)
+ val inWindow = LocalDateTime.of(2025, 1, 6, 12, 30, 45, 987654321)
val nanosType = TimestampNTZNanosType(9)
- val arrayDf = singlePartDf(
- Seq(Seq(outOfWindow, inWindow)), ArrayType(nanosType)).cache()
- try {
- assert(arrayDf.count() == 1)
- val read = arrayDf.collect().head.getSeq[java.time.LocalDateTime](0)
- assert(read == Seq(outOfWindow, inWindow),
- s"expected nested nanos to round-trip through an array, got: $read")
- } finally {
- arrayDf.unpersist()
- InMemoryRelation.clearSerializer()
- }
+ Seq(false, true).foreach { vectorized =>
+ withSQLConf(SQLConf.CACHE_VECTORIZED_READER_ENABLED.key ->
vectorized.toString) {
+ val arrayDf = singlePartDf(
+ Seq(Seq(outOfWindow, inWindow)), ArrayType(nanosType)).cache()
+ try {
+ assert(arrayDf.count() == 1)
+ val read = arrayDf.collect().head.getSeq[LocalDateTime](0)
+ assert(read == Seq(outOfWindow, inWindow),
+ s"expected nested nanos to round-trip through an array, got:
$read")
+ } finally {
+ arrayDf.unpersist()
+ InMemoryRelation.clearSerializer()
+ }
- val structDf = singlePartDf(
- Seq(Row(outOfWindow)), StructType(Seq(StructField("ts",
nanosType)))).cache()
- try {
- assert(structDf.count() == 1)
- val read =
structDf.collect().head.getStruct(0).getAs[java.time.LocalDateTime](0)
- assert(read == outOfWindow,
- s"expected nested nanos to round-trip through a struct, got: $read")
- } finally {
- structDf.unpersist()
- InMemoryRelation.clearSerializer()
- }
+ val structDf = singlePartDf(
+ Seq(Row(outOfWindow)), StructType(Seq(StructField("ts",
nanosType)))).cache()
+ try {
+ assert(structDf.count() == 1)
+ val read =
structDf.collect().head.getStruct(0).getAs[LocalDateTime](0)
+ assert(read == outOfWindow,
+ s"expected nested nanos to round-trip through a struct, got:
$read")
+ } finally {
+ structDf.unpersist()
+ InMemoryRelation.clearSerializer()
+ }
- val mapDf = singlePartDf(
- Seq(Map(1 -> outOfWindow)), MapType(IntegerType, nanosType)).cache()
- try {
- assert(mapDf.count() == 1)
- val read = mapDf.collect().head.getMap[Int, java.time.LocalDateTime](0)
- assert(read == Map(1 -> outOfWindow),
- s"expected nested nanos to round-trip through a map value, got: $read")
- } finally {
- mapDf.unpersist()
- InMemoryRelation.clearSerializer()
+ val mapDf = singlePartDf(
+ Seq(Map(1 -> outOfWindow)), MapType(IntegerType, nanosType)).cache()
+ try {
+ assert(mapDf.count() == 1)
+ val read = mapDf.collect().head.getMap[Int, LocalDateTime](0)
+ assert(read == Map(1 -> outOfWindow),
+ s"expected nested nanos to round-trip through a map value, got:
$read")
+ } finally {
+ mapDf.unpersist()
+ InMemoryRelation.clearSerializer()
+ }
+ }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]