dongjoon-hyun commented on code in PR #45895:
URL: https://github.com/apache/spark/pull/45895#discussion_r1554650108
##########
connector/avro/src/test/scala/org/apache/spark/sql/avro/AvroLogicalTypeSuite.scala:
##########
@@ -82,6 +82,35 @@ abstract class AvroLogicalTypeSuite extends QueryTest with
SharedSparkSession {
}
}
+ test("custom logical type registration test") {
+ val avroTypeJson =
+ s"""
+ |{
+ | "type": "record",
+ | "name": "Entry",
+ | "fields": [
+ | {
+ | "name": "test_col",
+ | "type": [
+ | "null",
+ | {
+ | "type": "long",
+ | "logicalType": "custom-decimal",
+ | "precision": 38,
+ | "scale": 9
+ | }
+ | ],
+ | "default": null
+ | }
+ | ]
+ |}
+ |
+ """.stripMargin
+
+ val df = spark.read.format("avro").option("avroSchema",
avroTypeJson).load()
+ assert(df.schema.fields(0).dataType == DecimalType(38, 9))
Review Comment:
According to the PR description, this should fail. However, this succeeds
already.
```
$ git diff master
diff --git
a/connector/avro/src/test/scala/org/apache/spark/sql/avro/AvroLogicalTypeSuite.scala
b/connector/avro/src/test/scala/org/apache/spark/sql/avro/AvroLogicalTypeSuite.scala
index 0a6ebb9e0a7..f54cb28f6ce 100644
---
a/connector/avro/src/test/scala/org/apache/spark/sql/avro/AvroLogicalTypeSuite.scala
+++
b/connector/avro/src/test/scala/org/apache/spark/sql/avro/AvroLogicalTypeSuite.scala
@@ -82,6 +82,35 @@ abstract class AvroLogicalTypeSuite extends QueryTest
with SharedSparkSession {
}
}
+ test("custom logical type registration test") {
+ val avroTypeJson =
+ s"""
+ |{
+ | "type": "record",
+ | "name": "Entry",
+ | "fields": [
+ | {
+ | "name": "test_col",
+ | "type": [
+ | "null",
+ | {
+ | "type": "long",
+ | "logicalType": "custom-decimal",
+ | "precision": 38,
+ | "scale": 9
+ | }
+ | ],
+ | "default": null
+ | }
+ | ]
+ |}
+ |
+ """.stripMargin
+
+ val df = spark.read.format("avro").option("avroSchema",
avroTypeJson).load()
+ assert(df.schema.fields(0).dataType == DecimalType(38, 9))
+ }
+
// scalastyle:off line.size.limit
val timestampSchema = s"""
{
$ build/sbt "avro/testOnly *Avro*LogicalTypeSuite" -Pavro
...
[info] AvroV1LogicalTypeSuite:
[info] - Logical type: date (2 seconds, 227 milliseconds)
[info] - custom logical type registration test (8 milliseconds)
[info] - Logical type: timestamp_millis (258 milliseconds)
[info] - Logical type: timestamp_micros (228 milliseconds)
[info] - Logical type: local_timestamp_millis (232 milliseconds)
[info] - Logical type: local_timestamp_micros (192 milliseconds)
[info] - Logical type: user specified output schema with different timestamp
types (160 milliseconds)
[info] - Logical type: user specified output schema with different timestamp
ntz types (203 milliseconds)
[info] - Read Long type as Timestamp (73 milliseconds)
[info] - Read Long type as Timestamp without time zone (54 milliseconds)
[info] - Logical type: user specified read schema (92 milliseconds)
[info] - Logical type: Decimal (579 milliseconds)
[info] - Logical type: write Decimal with BYTES type (169 milliseconds)
[info] - Logical type: Decimal with too large precision (93 milliseconds)
[info] - SPARK-43901: LogicalType: Custom Decimal for Long Type (46
milliseconds)
[info] - SPARK-43901: LogicalType: Decimal for Long Type Exception Cases
(328 milliseconds)
[info] AvroV2LogicalTypeSuite:
[info] - Logical type: date (375 milliseconds)
[info] - custom logical type registration test (3 milliseconds)
[info] - Logical type: timestamp_millis (252 milliseconds)
[info] - Logical type: timestamp_micros (247 milliseconds)
[info] - Logical type: local_timestamp_millis (227 milliseconds)
[info] - Logical type: local_timestamp_micros (403 milliseconds)
[info] - Logical type: user specified output schema with different timestamp
types (236 milliseconds)
[info] - Logical type: user specified output schema with different timestamp
ntz types (158 milliseconds)
[info] - Read Long type as Timestamp (75 milliseconds)
[info] - Read Long type as Timestamp without time zone (112 milliseconds)
[info] - Logical type: user specified read schema (114 milliseconds)
[info] - Logical type: Decimal (298 milliseconds)
[info] - Logical type: write Decimal with BYTES type (220 milliseconds)
[info] - Logical type: Decimal with too large precision (67 milliseconds)
[info] - SPARK-43901: LogicalType: Custom Decimal for Long Type (63
milliseconds)
[info] - SPARK-43901: LogicalType: Decimal for Long Type Exception Cases
(395 milliseconds)
[info] Run completed in 9 seconds, 471 milliseconds.
[info] Total number of tests run: 32
[info] Suites: completed 2, aborted 0
[info] Tests: succeeded 32, failed 0, canceled 0, ignored 0, pending 0
[info] All tests passed.
[success] Total time: 26 s, completed Apr 6, 2024, 11:17:15 AM
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
