the-other-tim-brown commented on code in PR #13882:
URL: https://github.com/apache/hudi/pull/13882#discussion_r2345640239
##########
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowParquetWriteSupport.java:
##########
@@ -73,6 +178,173 @@ public void add(UTF8String recordKey) {
bloomFilterWriteSupport.addKey(recordKey));
}
+ @FunctionalInterface
+ private interface ValueWriter {
+ void write(SpecializedGetters row, int ordinal);
+ }
+
+ private void consumeMessage(Runnable writer) {
+ recordConsumer.startMessage();
+ writer.run();
+ recordConsumer.endMessage();
+ }
+
+ private void consumeGroup(Runnable writer) {
+ recordConsumer.startGroup();
+ writer.run();
+ recordConsumer.endGroup();
+ }
+
+ private void consumeField(String field, int index, Runnable writer) {
+ recordConsumer.startField(field, index);
+ writer.run();
+ recordConsumer.endField(field, index);
+ }
+
+ private void writeFields(InternalRow row, Schema schema, ValueWriter[]
fieldWriters) {
+ for (int i = 0; i < fieldWriters.length; i++) {
+ int index = i;
+ if (!row.isNullAt(i)) {
+ Schema.Field field = schema.getFields().get(index);
+ consumeField(field.name(), index, () -> fieldWriters[index].write(row,
index));
+ }
+ }
+ }
+
+ private ValueWriter makeWriter(Schema avroSchema, DataType dataType) {
+ Schema resolvedSchema = resolveNullableSchema(avroSchema);
+ Schema.Type type = resolvedSchema.getType();
+ LogicalType logicalType = resolvedSchema.getLogicalType();
+ switch (type) {
+ case BOOLEAN:
+ return (row, ordinal) ->
recordConsumer.addBoolean(row.getBoolean(ordinal));
+ case INT:
+ if (logicalType != null) {
+ if (logicalType.getName().equals(LogicalTypes.date().getName())) {
+ return (row, ordinal) -> recordConsumer.addInteger((Integer)
dateRebaseFunction.apply(row.getInt(ordinal)));
+ }
+ }
+ return (row, ordinal) ->
recordConsumer.addInteger(row.getInt(ordinal));
+ case LONG:
+ if (logicalType != null) {
+ if
(logicalType.getName().equals(LogicalTypes.timestampMillis().getName())) {
+ return (row, ordinal) ->
recordConsumer.addLong(DateTimeUtils.microsToMillis((long)
timestampRebaseFunction.apply(row.getLong(ordinal))));
+ } else if
(logicalType.getName().equals(LogicalTypes.timestampMicros().getName())) {
+ return (row, ordinal) -> recordConsumer.addLong((long)
timestampRebaseFunction.apply(row.getLong(ordinal)));
+ } else if
(logicalType.getName().equals(LogicalTypes.localTimestampMicros().getName())) {
+ return (row, ordinal) ->
recordConsumer.addLong(row.getLong(ordinal));
+ } else if
(logicalType.getName().equals(LogicalTypes.localTimestampMillis().getName()) &&
dataType instanceof TimestampNTZType) {
+ // only go through conversion if spark data type is
TimestampNTZType
+ return (row, ordinal) ->
recordConsumer.addLong(DateTimeUtils.microsToMillis(row.getLong(ordinal)));
+ }
+ }
+ return (row, ordinal) -> recordConsumer.addLong(row.getLong(ordinal));
+ case FLOAT:
+ return (row, ordinal) ->
recordConsumer.addFloat(row.getFloat(ordinal));
+ case DOUBLE:
+ return (row, ordinal) ->
recordConsumer.addDouble(row.getDouble(ordinal));
+ case STRING:
+ case ENUM:
+ return (row, ordinal) -> recordConsumer.addBinary(
+ Binary.fromReusedByteArray(row.getUTF8String(ordinal).getBytes()));
+ case BYTES:
+ case FIXED:
+ if (logicalType != null && logicalType.getName().equals("decimal")) {
+ return (row, ordinal) -> {
+ int precision = ((LogicalTypes.Decimal)
logicalType).getPrecision();
+ int scale = ((LogicalTypes.Decimal) logicalType).getScale();
+ long unscaled = row.getDecimal(ordinal, precision,
scale).toUnscaledLong();
+ int i = 0;
+ int numBytes = Decimal.minBytesForPrecision()[precision];
+ int shift = 8 * (numBytes - 1);
+ while (i < numBytes) {
+ decimalBuffer[i] = (byte) ((unscaled >> shift) & 0xFF);
+ i += 1;
+ shift -= 8;
+ }
+ recordConsumer.addBinary(Binary.fromReusedByteArray(decimalBuffer,
0, numBytes));
+ };
+ }
+ return (row, ordinal) -> recordConsumer.addBinary(
+ Binary.fromReusedByteArray(row.getBinary(ordinal)));
+ case RECORD:
+ ValueWriter[] fieldWriters = getFieldWriters(((StructType) dataType),
resolvedSchema);
+ return (row, ordinal) ->
+ consumeGroup(() -> writeFields(row.getStruct(ordinal,
resolvedSchema.getFields().size()), resolvedSchema, fieldWriters));
+ case ARRAY:
+ ValueWriter elementWriter =
makeWriter(resolvedSchema.getElementType(), ((ArrayType)
dataType).elementType());
+ if (!writeLegacyListFormat) {
+ return threeLevelArrayWriter("list", "element", elementWriter);
+ } else if (resolvedSchema.getElementType().isNullable()) {
+ return threeLevelArrayWriter("bag", "array", elementWriter);
+ } else {
+ return twoLevelArrayWriter("array", elementWriter);
+ }
+ case MAP:
+ ValueWriter keyWriter = makeWriter(MAP_KEY_SCHEMA,
DataTypes.StringType);
+ ValueWriter valueWriter = makeWriter(resolvedSchema.getValueType(),
((MapType) dataType).valueType());
+ return (row, ordinal) -> {
+ MapData mapData = row.getMap(ordinal);
+ ArrayData keyArray = mapData.keyArray();
+ ArrayData valueArray = mapData.valueArray();
+ consumeGroup(() -> {
+ if (mapData.numElements() > 0) {
+ consumeField(MAP_REPEATED_NAME, 0, () -> {
+ for (int i = 0; i < mapData.numElements(); i++) {
+ int index = i;
+ consumeGroup(() -> {
+ if (!keyArray.isNullAt(index)) {
Review Comment:
It does not based on my understanding, removing this
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
