cloud-fan commented on code in PR #44513:
URL: https://github.com/apache/spark/pull/44513#discussion_r1444305951
##########
sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/ParquetVectorUpdaterFactory.java:
##########
@@ -1358,6 +1368,188 @@ public void decodeSingleDictionaryId(
}
}
+ private abstract static class DecimalUpdater implements ParquetVectorUpdater
{
+
+ private final DecimalType sparkType;
+
+ DecimalUpdater(DecimalType sparkType) {
+ this.sparkType = sparkType;
+ }
+
+ @Override
+ public void readValues(
+ int total,
+ int offset,
+ WritableColumnVector values,
+ VectorizedValuesReader valuesReader) {
+ for (int i = 0; i < total; i++) {
+ readValue(offset + i, values, valuesReader);
+ }
+ }
+
+ protected void writeDecimal(int offset, WritableColumnVector values,
BigDecimal decimal) {
+ BigDecimal scaledDecimal = decimal.setScale(sparkType.scale(),
RoundingMode.UNNECESSARY);
+ if (DecimalType.is32BitDecimalType(sparkType)) {
+ values.putInt(offset, scaledDecimal.unscaledValue().intValue());
+ } else if (DecimalType.is64BitDecimalType(sparkType)) {
+ values.putLong(offset, scaledDecimal.unscaledValue().longValue());
+ } else {
+ values.putByteArray(offset,
scaledDecimal.unscaledValue().toByteArray());
+ }
+ }
+ }
+
+ private static class IntegerToDecimalUpdater extends DecimalUpdater {
+ private final int parquetScale;
+
+ IntegerToDecimalUpdater(ColumnDescriptor descriptor, DecimalType
sparkType) {
+ super(sparkType);
+ LogicalTypeAnnotation typeAnnotation =
+ descriptor.getPrimitiveType().getLogicalTypeAnnotation();
+ this.parquetScale = ((DecimalLogicalTypeAnnotation)
typeAnnotation).getScale();
+ }
+
+ @Override
+ public void skipValues(int total, VectorizedValuesReader valuesReader) {
+ valuesReader.skipIntegers(total);
Review Comment:
```suggestion
valuesReader.skipIntegers(total);
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
