wayneguow commented on code in PR #47651:
URL: https://github.com/apache/spark/pull/47651#discussion_r1717771836
##########
connector/avro/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala:
##########
@@ -386,14 +386,46 @@ private[sql] class AvroDeserializer(
case (LONG, _: DayTimeIntervalType) => (updater, ordinal, value) =>
updater.setLong(ordinal, value.asInstanceOf[Long])
- case (LONG, _: DecimalType) => (updater, ordinal, value) =>
- val d = avroType.getLogicalType.asInstanceOf[CustomDecimal]
- updater.setDecimal(ordinal, Decimal(value.asInstanceOf[Long],
d.precision, d.scale))
+ case (INT, dt: DecimalType) => avroType.getLogicalType match {
+ case null =>
+ if (!isDecimalTypeMatched(INT, dt)) {
+ throw new IncompatibleSchemaException(incompatibleMsg)
+ }
+ (updater, ordinal, value) =>
+ updater.setDecimal(ordinal, Decimal(value.asInstanceOf[Int],
dt.precision, dt.scale))
+ case _: CustomDecimal => (updater, ordinal, value) =>
+ val d = avroType.getLogicalType.asInstanceOf[CustomDecimal]
+ updater.setDecimal(ordinal, Decimal(value.asInstanceOf[Int],
d.precision, d.scale))
+ case other => throw new IncompatibleSchemaException(errorPrefix +
+ s"Avro logical type $other cannot be converted to SQL type
${DecimalType.simpleString}.")
+ }
+
+ case (LONG, dt: DecimalType) => avroType.getLogicalType match {
+ case null =>
+ if (!isDecimalTypeMatched(LONG, dt)) {
+ throw new IncompatibleSchemaException(incompatibleMsg)
+ }
+ (updater, ordinal, value) =>
+ updater.setDecimal(ordinal, Decimal(value.asInstanceOf[Long],
dt.precision, dt.scale))
+ case _: CustomDecimal => (updater, ordinal, value) =>
+ val d = avroType.getLogicalType.asInstanceOf[CustomDecimal]
+ updater.setDecimal(ordinal, Decimal(value.asInstanceOf[Long],
d.precision, d.scale))
+ case other => throw new IncompatibleSchemaException(errorPrefix +
+ s"Avro logical type $other cannot be converted to SQL type
${DecimalType.simpleString}.")
+ }
case _ => throw new IncompatibleSchemaException(incompatibleMsg)
}
}
+ private def isDecimalTypeMatched(avroType: Schema.Type, dt: DecimalType):
Boolean = {
+ avroType match {
+ case INT => dt.precision >= DecimalType.IntDecimal.precision && dt.scale
== 0
Review Comment:
We didn't know the length of all value about int fields. However, this
method is used when actually parsing data.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
