wayneguow commented on code in PR #47651:
URL: https://github.com/apache/spark/pull/47651#discussion_r1717355345
##########
connector/avro/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala:
##########
@@ -386,14 +386,46 @@ private[sql] class AvroDeserializer(
case (LONG, _: DayTimeIntervalType) => (updater, ordinal, value) =>
updater.setLong(ordinal, value.asInstanceOf[Long])
- case (LONG, _: DecimalType) => (updater, ordinal, value) =>
- val d = avroType.getLogicalType.asInstanceOf[CustomDecimal]
- updater.setDecimal(ordinal, Decimal(value.asInstanceOf[Long],
d.precision, d.scale))
+ case (INT, dt: DecimalType) => avroType.getLogicalType match {
+ case null =>
+ if (!isDecimalTypeMatched(INT, dt)) {
+ throw new IncompatibleSchemaException(incompatibleMsg)
+ }
+ (updater, ordinal, value) =>
+ updater.setDecimal(ordinal, Decimal(value.asInstanceOf[Int],
dt.precision, dt.scale))
+ case _: CustomDecimal => (updater, ordinal, value) =>
+ val d = avroType.getLogicalType.asInstanceOf[CustomDecimal]
+ updater.setDecimal(ordinal, Decimal(value.asInstanceOf[Int],
d.precision, d.scale))
+ case other => throw new IncompatibleSchemaException(errorPrefix +
+ s"Avro logical type $other cannot be converted to SQL type
${DecimalType.simpleString}.")
+ }
+
+ case (LONG, dt: DecimalType) => avroType.getLogicalType match {
+ case null =>
+ if (!isDecimalTypeMatched(LONG, dt)) {
+ throw new IncompatibleSchemaException(incompatibleMsg)
+ }
+ (updater, ordinal, value) =>
+ updater.setDecimal(ordinal, Decimal(value.asInstanceOf[Long],
dt.precision, dt.scale))
+ case _: CustomDecimal => (updater, ordinal, value) =>
+ val d = avroType.getLogicalType.asInstanceOf[CustomDecimal]
+ updater.setDecimal(ordinal, Decimal(value.asInstanceOf[Long],
d.precision, d.scale))
+ case other => throw new IncompatibleSchemaException(errorPrefix +
+ s"Avro logical type $other cannot be converted to SQL type
${DecimalType.simpleString}.")
+ }
case _ => throw new IncompatibleSchemaException(incompatibleMsg)
}
}
+ private def isDecimalTypeMatched(avroType: Schema.Type, dt: DecimalType):
Boolean = {
+ avroType match {
+ case INT => dt.precision >= DecimalType.IntDecimal.precision && dt.scale
== 0
Review Comment:
@cloud-fan What I want to discuss more about is here. Currently, I have
relatively strict limitations on the precision and scale of `DecimalType`.
Do we need to consider allowing scale greater than 0 and precision that does
not need to be completely greater than `IntDecimal` or `LongDecimal`? For
example:
Is it reasonable to use `Decimal (7,2)` for the value `123456`
- Decimal(7, 2) => 123456.00
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
