(hudi) branch master updated: [HUDI-7443] Fix decimal conversion with legacy bytes type (#10756)

[danny0405]

This is an automated email from the ASF dual-hosted git repository.

danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git


The following commit(s) were added to refs/heads/master by this push:
     new 20e234884cd [HUDI-7443] Fix decimal conversion with legacy bytes type 
(#10756)
20e234884cd is described below

commit 20e234884cdeb3731083b69f20cdd4fc67e63a9f
Author: stream2000 <stream2...@apache.org>
AuthorDate: Tue Feb 27 12:52:03 2024 +0800

    [HUDI-7443] Fix decimal conversion with legacy bytes type (#10756)
---
 .../org/apache/hudi/hadoop/utils/HiveAvroSerializer.java |  6 +++++-
 .../hadoop/utils/HoodieRealtimeRecordReaderUtils.java    | 16 +++++++++++-----
 2 files changed, 16 insertions(+), 6 deletions(-)

diff --git 
a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HiveAvroSerializer.java
 
b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HiveAvroSerializer.java
index 5f33844d60c..22116283d12 100644
--- 
a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HiveAvroSerializer.java
+++ 
b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HiveAvroSerializer.java
@@ -289,7 +289,11 @@ public class HiveAvroSerializer {
         HiveDecimal dec = (HiveDecimal) 
fieldOI.getPrimitiveJavaObject(structFieldData);
         LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) 
schema.getLogicalType();
         BigDecimal bd = new 
BigDecimal(dec.toString()).setScale(decimal.getScale());
-        return HoodieAvroUtils.DECIMAL_CONVERSION.toFixed(bd, schema, decimal);
+        if (schema.getType() == Schema.Type.BYTES) {
+          return HoodieAvroUtils.DECIMAL_CONVERSION.toBytes(bd, schema, 
decimal);
+        } else {
+          return HoodieAvroUtils.DECIMAL_CONVERSION.toFixed(bd, schema, 
decimal);
+        }
       case CHAR:
         HiveChar ch = (HiveChar) 
fieldOI.getPrimitiveJavaObject(structFieldData);
         return new Utf8(ch.getStrippedValue());
diff --git 
a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieRealtimeRecordReaderUtils.java
 
b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieRealtimeRecordReaderUtils.java
index ecab6b2e11c..6d98f0c8f52 100644
--- 
a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieRealtimeRecordReaderUtils.java
+++ 
b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieRealtimeRecordReaderUtils.java
@@ -167,6 +167,9 @@ public class HoodieRealtimeRecordReaderUtils {
       case STRING:
         return new Text(value.toString());
       case BYTES:
+        if (schema.getLogicalType() != null && 
schema.getLogicalType().getName().equals("decimal")) {
+          return toHiveDecimalWritable(((ByteBuffer) value).array(), schema);
+        }
         return new BytesWritable(((ByteBuffer) value).array());
       case INT:
         if (schema.getLogicalType() != null && 
schema.getLogicalType().getName().equals("date")) {
@@ -248,11 +251,7 @@ public class HoodieRealtimeRecordReaderUtils {
         }
       case FIXED:
         if (schema.getLogicalType() != null && 
schema.getLogicalType().getName().equals("decimal")) {
-          LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) 
LogicalTypes.fromSchema(schema);
-          HiveDecimalWritable writable = new 
HiveDecimalWritable(((GenericFixed) value).bytes(),
-              decimal.getScale());
-          return HiveDecimalUtils.enforcePrecisionScale(writable,
-              new DecimalTypeInfo(decimal.getPrecision(), decimal.getScale()));
+          return toHiveDecimalWritable(((GenericFixed) value).bytes(), schema);
         }
         return new BytesWritable(((GenericFixed) value).bytes());
       default:
@@ -319,4 +318,11 @@ public class HoodieRealtimeRecordReaderUtils {
     }
     return appendFieldsToSchema(schema, newFields);
   }
+
+  private static HiveDecimalWritable toHiveDecimalWritable(byte[] bytes, 
Schema schema) {
+    LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) 
LogicalTypes.fromSchema(schema);
+    HiveDecimalWritable writable = new HiveDecimalWritable(bytes, 
decimal.getScale());
+    return HiveDecimalUtils.enforcePrecisionScale(writable,
+        new DecimalTypeInfo(decimal.getPrecision(), decimal.getScale()));
+  }
 }