uros-b commented on code in PR #56739:
URL: https://github.com/apache/spark/pull/56739#discussion_r3472376334
##########
sql/catalyst/src/test/scala/org/apache/spark/sql/util/ArrowUtilsSuite.scala:
##########
@@ -86,6 +87,73 @@ class ArrowUtilsSuite extends SparkFunSuite {
roundtripWithTz(LA.getId)
}
+ test("timestamp nanos") {
+ // NTZ is zone-independent (null Arrow timezone); precision preserved via
field metadata.
+ Seq(7, 8, 9).foreach { p =>
+ val schema = new StructType().add("value", TimestampNTZNanosType(p))
+ val arrowSchema = ArrowUtils.toArrowSchema(schema, null, true, false)
+ val fieldType =
arrowSchema.findField("value").getType.asInstanceOf[ArrowType.Timestamp]
+ assert(fieldType.getUnit === TimeUnit.NANOSECOND)
+ assert(fieldType.getTimezone === null)
+ assert(ArrowUtils.fromArrowSchema(arrowSchema) === schema)
+ }
+
+ // LTZ is zone-aware: it requires a non-null session time zone; precision
preserved.
+ def roundtripLtz(timeZoneId: String): Unit = {
+ Seq(7, 8, 9).foreach { p =>
+ val schema = new StructType().add("value", TimestampLTZNanosType(p))
+ val arrowSchema = ArrowUtils.toArrowSchema(schema, timeZoneId, true,
false)
+ val fieldType =
arrowSchema.findField("value").getType.asInstanceOf[ArrowType.Timestamp]
+ assert(fieldType.getUnit === TimeUnit.NANOSECOND)
+ assert(fieldType.getTimezone === timeZoneId)
+ assert(ArrowUtils.fromArrowSchema(arrowSchema) === schema)
+ }
+ }
+ roundtripLtz(ZoneId.systemDefault().getId)
+ roundtripLtz("Asia/Tokyo")
+ roundtripLtz("UTC")
+ roundtripLtz(LA.getId)
+
+ // LTZ without a time zone is an error, mirroring TimestampType.
+ checkError(
+ exception = intercept[SparkException] {
+ ArrowUtils.toArrowSchema(
+ new StructType().add("value", TimestampLTZNanosType(9)), null, true,
false)
+ },
+ condition = "INTERNAL_ERROR",
+ parameters = Map("message" -> "Missing timezoneId where it is
mandatory."))
+
+ // Fallback: a nanosecond Arrow timestamp without precision metadata maps
to canonical p=9.
+ def nanosField(timeZoneId: String): Field = new Field(
+ "value",
+ new FieldType(true, new ArrowType.Timestamp(TimeUnit.NANOSECOND,
timeZoneId), null, null),
+ java.util.Collections.emptyList[Field]())
+ assert(ArrowUtils.fromArrowField(nanosField(null)) ===
TimestampNTZNanosType(9))
+ assert(ArrowUtils.fromArrowField(nanosField("UTC")) ===
TimestampLTZNanosType(9))
+
+ // Fallback also covers a present-but-invalid precision key (out of [7, 9]
or non-numeric):
+ // the value is unusable, so the type maps to the canonical p=9 just like
the no-metadata case.
+ def nanosFieldWithPrecision(timeZoneId: String, precision: String): Field
= new Field(
+ "value",
+ new FieldType(
+ true,
+ new ArrowType.Timestamp(TimeUnit.NANOSECOND, timeZoneId),
+ null,
+ java.util.Collections.singletonMap("SPARK::timestampNanos::precision",
precision)),
+ java.util.Collections.emptyList[Field]())
+ assert(
+ ArrowUtils.fromArrowField(nanosFieldWithPrecision(null, "5")) ===
TimestampNTZNanosType(9))
+ assert(
+ ArrowUtils.fromArrowField(nanosFieldWithPrecision("UTC", "x")) ===
TimestampLTZNanosType(9))
+
+ // The precision metadata key does not leak into the reconstructed column
Metadata.
+ val md = new MetadataBuilder().putString("city", "beijing").build()
+ val schemaWithMeta =
+ new StructType().add("value", TimestampNTZNanosType(7), nullable = true,
md)
+ assert(ArrowUtils.fromArrowSchema(
+ ArrowUtils.toArrowSchema(schemaWithMeta, null, true, false)) ===
schemaWithMeta)
+ }
+
Review Comment:
Note: no end-to-end ArrowConvertersSuite / toArrowBatchRdd case. An e2e
batch test would be nice once Connect wiring lands.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
