gengliangwang commented on a change in pull request #35379:
URL: https://github.com/apache/spark/pull/35379#discussion_r809611761
##########
File path:
external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSerdeSuite.scala
##########
@@ -49,6 +56,160 @@ class AvroSerdeSuite extends SparkFunSuite {
}
}
+ test("Serialize DecimalType to Avro FIXED with logical type decimal") {
+ withFieldMatchType { fieldMatch =>
+ val structType = StructType(
+ Seq(
+ StructField("javaBigDecimal", DecimalType(6, 2), nullable = false),
+ StructField("sparkDecimal", DecimalType(6, 2), nullable = false)))
+
+ val fixedSchema = Schema.createFixed("fixed_name", "doc", "namespace",
32)
+ val logicalType = LogicalTypes.decimal(6, 2)
+ val fieldSchema = logicalType.addToSchema(fixedSchema)
+
+ val avroSchema = Schema.createRecord(
+ "name",
+ "doc",
+ "space",
+ true,
+ Seq(
+ new Schema.Field("javaBigDecimal", fieldSchema, "",
null.asInstanceOf[AnyVal]),
+ new Schema.Field("sparkDecimal", fieldSchema, "",
null.asInstanceOf[AnyVal])).asJava)
+
+ val serializer = Serializer.create(structType, avroSchema, fieldMatch)
+
+ val input = InternalRow(new java.math.BigDecimal("1000.12"),
Decimal("1000.12"))
+
+ val grec = serializer.serialize(input).asInstanceOf[GenericRecord]
+ val javaDecimal = grec.get("javaBigDecimal").asInstanceOf[GenericFixed]
+ val sparkDecimal = grec.get("sparkDecimal").asInstanceOf[GenericFixed]
+
+ assert(javaDecimal === sparkDecimal)
+ assert(
+ new DecimalConversion().fromFixed(sparkDecimal, fixedSchema,
logicalType) ===
+ new java.math.BigDecimal("1000.12"))
+ }
+ }
+
+ test("Serialize DecimalType to Avro BYTES with logical type decimal") {
+ withFieldMatchType { fieldMatch =>
+ val structType = StructType(
+ Seq(
+ StructField("javaBigDecimal", DecimalType(6, 2), nullable = false),
+ StructField("sparkDecimal", DecimalType(6, 2), nullable = false)))
+
+ val bytesSchema = Schema.create(BYTES)
+ val logicalType = LogicalTypes.decimal(6, 2)
+ val fieldSchema = logicalType.addToSchema(bytesSchema)
+
+ val avroSchema = Schema.createRecord(
+ "name",
+ "doc",
+ "space",
+ true,
+ Seq(
+ new Schema.Field("javaBigDecimal", fieldSchema, "",
null.asInstanceOf[AnyVal]),
+ new Schema.Field("sparkDecimal", fieldSchema, "",
null.asInstanceOf[AnyVal])).asJava)
+
+ val serializer = Serializer.create(structType, avroSchema, fieldMatch)
+
+ val input = InternalRow(new java.math.BigDecimal("1000.12"),
Decimal("1000.12"))
+
+ val grec = serializer.serialize(input).asInstanceOf[GenericRecord]
+ val javaDecimal = grec.get("javaBigDecimal").asInstanceOf[ByteBuffer]
+ val sparkDecimal = grec.get("sparkDecimal").asInstanceOf[ByteBuffer]
+
+ assert(javaDecimal === sparkDecimal)
+ assert(
+ new DecimalConversion().fromBytes(sparkDecimal, bytesSchema,
logicalType) ===
+ new java.math.BigDecimal("1000.12"))
+ }
+ }
+
+ test("Serialize DateType to Avro INT") {
+ withFieldMatchType { fieldMatch =>
+ val structType = StructType(
+ Seq(
+ StructField("javaSqlDate", DateType, nullable = false),
+ StructField("java8TimeDate", DateType, nullable = false)))
+
+ val dateSchema = Schema.create(INT)
+
+ val avroSchema = Schema.createRecord(
+ "name",
+ "doc",
+ "space",
+ true,
+ Seq(
+ new Schema.Field("javaSqlDate", dateSchema, "",
null.asInstanceOf[AnyVal]),
+ new Schema.Field("java8TimeDate", dateSchema, "",
null.asInstanceOf[AnyVal])).asJava)
+
+ val serializer = Serializer.create(structType, avroSchema, fieldMatch)
+
+ val input = InternalRow(
+ new java.sql.Date(1643121231000L),
+
Instant.ofEpochMilli(1643121231000L).atZone(ZoneId.of("UTC")).toLocalDate())
+
+ val grec = serializer.serialize(input).asInstanceOf[GenericRecord]
+ val javaSqlDate = grec.get("javaSqlDate").asInstanceOf[Int]
+ val java8TimeDate = grec.get("java8TimeDate").asInstanceOf[Int]
+
+ assert(javaSqlDate === java8TimeDate)
+ assert(javaSqlDate === 19017) // 19017 is 25 January 2022
+ }
+ }
+
+ test(s"""
Review comment:
There is a bug in the test case
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
