Github user cloud-fan commented on a diff in the pull request:
https://github.com/apache/spark/pull/21935#discussion_r207095392
--- Diff:
external/avro/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala ---
@@ -93,7 +94,13 @@ class AvroSerializer(rootCatalystType: DataType,
rootAvroType: Schema, nullable:
case DateType =>
(getter, ordinal) => getter.getInt(ordinal) *
DateTimeUtils.MILLIS_PER_DAY
case TimestampType =>
- (getter, ordinal) => getter.getLong(ordinal) / 1000
+ (getter, ordinal) => avroType.getLogicalType match {
+ case _: TimestampMillis => getter.getLong(ordinal) / 1000
+ case _: TimestampMicros => getter.getLong(ordinal)
+ // For backward compatibility, if the Avro type is Long and it
is not logical type,
+ // output the timestamp value as with millisecond precision.
+ case null => getter.getLong(ordinal) / 1000
--- End diff --
ditto, add a default case.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
