the-other-tim-brown commented on code in PR #13882:
URL: https://github.com/apache/hudi/pull/13882#discussion_r2346902291
##########
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowParquetWriteSupport.java:
##########
@@ -73,6 +189,180 @@ public void add(UTF8String recordKey) {
bloomFilterWriteSupport.addKey(recordKey));
}
+ @FunctionalInterface
+ private interface ValueWriter {
+ void write(SpecializedGetters row, int ordinal);
+ }
+
+ private void consumeMessage(Runnable writer) {
+ recordConsumer.startMessage();
+ writer.run();
+ recordConsumer.endMessage();
+ }
+
+ private void consumeGroup(Runnable writer) {
+ recordConsumer.startGroup();
+ writer.run();
+ recordConsumer.endGroup();
+ }
+
+ private void consumeField(String field, int index, Runnable writer) {
+ recordConsumer.startField(field, index);
+ writer.run();
+ recordConsumer.endField(field, index);
+ }
+
+ private void writeFields(InternalRow row, StructType schema, ValueWriter[]
fieldWriters) {
+ for (int i = 0; i < row.numFields(); i++) {
+ int index = i;
+ if (!row.isNullAt(i)) {
+ StructField field = schema.fields()[i];
+ consumeField(field.name(), index, () -> fieldWriters[index].write(row,
index));
+ }
+ }
+ }
+
+ private ValueWriter makeWriter(Schema avroSchema, DataType dataType) {
+ Schema resolvedSchema = avroSchema == null ? null :
resolveNullableSchema(avroSchema);
+ LogicalType logicalType = resolvedSchema != null ?
resolvedSchema.getLogicalType() : null;
+
+ if (dataType == DataTypes.BooleanType) {
+ return (row, ordinal) ->
recordConsumer.addBoolean(row.getBoolean(ordinal));
+ } else if (dataType == DataTypes.DateType) {
+ return (row, ordinal) -> recordConsumer.addInteger((Integer)
dateRebaseFunction.apply(row.getInt(ordinal)));
+ } else if (dataType == DataTypes.ShortType) {
+ return (row, ordinal) ->
recordConsumer.addInteger(row.getShort(ordinal));
+ } else if (dataType == DataTypes.IntegerType) {
+ return (row, ordinal) -> recordConsumer.addInteger(row.getInt(ordinal));
+ } else if (dataType == DataTypes.LongType) {
+ return (row, ordinal) -> recordConsumer.addLong(row.getLong(ordinal));
+ } else if (dataType == DataTypes.TimestampType) {
+ if (logicalType == null ||
logicalType.getName().equals(LogicalTypes.timestampMicros().getName())) {
+ return (row, ordinal) -> recordConsumer.addLong((long)
timestampRebaseFunction.apply(row.getLong(ordinal)));
+ } else if
(logicalType.getName().equals(LogicalTypes.timestampMillis().getName())) {
+ return (row, ordinal) ->
recordConsumer.addLong(DateTimeUtils.microsToMillis((long)
timestampRebaseFunction.apply(row.getLong(ordinal))));
+ } else {
+ throw new UnsupportedOperationException("Unsupported timestamp type: "
+ logicalType);
Review Comment:
Updated
##########
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowParquetWriteSupport.java:
##########
@@ -73,6 +189,180 @@ public void add(UTF8String recordKey) {
bloomFilterWriteSupport.addKey(recordKey));
}
+ @FunctionalInterface
+ private interface ValueWriter {
+ void write(SpecializedGetters row, int ordinal);
+ }
+
+ private void consumeMessage(Runnable writer) {
+ recordConsumer.startMessage();
+ writer.run();
+ recordConsumer.endMessage();
+ }
+
+ private void consumeGroup(Runnable writer) {
+ recordConsumer.startGroup();
+ writer.run();
+ recordConsumer.endGroup();
+ }
+
+ private void consumeField(String field, int index, Runnable writer) {
+ recordConsumer.startField(field, index);
+ writer.run();
+ recordConsumer.endField(field, index);
+ }
+
+ private void writeFields(InternalRow row, StructType schema, ValueWriter[]
fieldWriters) {
+ for (int i = 0; i < row.numFields(); i++) {
+ int index = i;
+ if (!row.isNullAt(i)) {
+ StructField field = schema.fields()[i];
+ consumeField(field.name(), index, () -> fieldWriters[index].write(row,
index));
+ }
+ }
+ }
+
+ private ValueWriter makeWriter(Schema avroSchema, DataType dataType) {
+ Schema resolvedSchema = avroSchema == null ? null :
resolveNullableSchema(avroSchema);
+ LogicalType logicalType = resolvedSchema != null ?
resolvedSchema.getLogicalType() : null;
+
+ if (dataType == DataTypes.BooleanType) {
+ return (row, ordinal) ->
recordConsumer.addBoolean(row.getBoolean(ordinal));
+ } else if (dataType == DataTypes.DateType) {
+ return (row, ordinal) -> recordConsumer.addInteger((Integer)
dateRebaseFunction.apply(row.getInt(ordinal)));
+ } else if (dataType == DataTypes.ShortType) {
+ return (row, ordinal) ->
recordConsumer.addInteger(row.getShort(ordinal));
+ } else if (dataType == DataTypes.IntegerType) {
+ return (row, ordinal) -> recordConsumer.addInteger(row.getInt(ordinal));
+ } else if (dataType == DataTypes.LongType) {
+ return (row, ordinal) -> recordConsumer.addLong(row.getLong(ordinal));
+ } else if (dataType == DataTypes.TimestampType) {
+ if (logicalType == null ||
logicalType.getName().equals(LogicalTypes.timestampMicros().getName())) {
+ return (row, ordinal) -> recordConsumer.addLong((long)
timestampRebaseFunction.apply(row.getLong(ordinal)));
+ } else if
(logicalType.getName().equals(LogicalTypes.timestampMillis().getName())) {
+ return (row, ordinal) ->
recordConsumer.addLong(DateTimeUtils.microsToMillis((long)
timestampRebaseFunction.apply(row.getLong(ordinal))));
+ } else {
+ throw new UnsupportedOperationException("Unsupported timestamp type: "
+ logicalType);
+ }
+ } else if
(SparkAdapterSupport$.MODULE$.sparkAdapter().isTimestampNTZType(dataType)) {
+ if (logicalType == null ||
logicalType.getName().equals(LogicalTypes.localTimestampMicros().getName())) {
+ return (row, ordinal) -> recordConsumer.addLong(row.getLong(ordinal));
+ } else if
(logicalType.getName().equals(LogicalTypes.localTimestampMillis().getName())) {
+ return (row, ordinal) ->
recordConsumer.addLong(DateTimeUtils.microsToMillis(row.getLong(ordinal)));
+ } else {
+ throw new UnsupportedOperationException("Unsupported timestamp type: "
+ logicalType);
Review Comment:
Updated
##########
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowParquetWriteSupport.java:
##########
@@ -73,6 +189,180 @@ public void add(UTF8String recordKey) {
bloomFilterWriteSupport.addKey(recordKey));
}
+ @FunctionalInterface
+ private interface ValueWriter {
+ void write(SpecializedGetters row, int ordinal);
+ }
+
+ private void consumeMessage(Runnable writer) {
+ recordConsumer.startMessage();
+ writer.run();
+ recordConsumer.endMessage();
+ }
+
+ private void consumeGroup(Runnable writer) {
+ recordConsumer.startGroup();
+ writer.run();
+ recordConsumer.endGroup();
+ }
+
+ private void consumeField(String field, int index, Runnable writer) {
+ recordConsumer.startField(field, index);
+ writer.run();
+ recordConsumer.endField(field, index);
+ }
+
+ private void writeFields(InternalRow row, StructType schema, ValueWriter[]
fieldWriters) {
+ for (int i = 0; i < row.numFields(); i++) {
+ int index = i;
+ if (!row.isNullAt(i)) {
+ StructField field = schema.fields()[i];
+ consumeField(field.name(), index, () -> fieldWriters[index].write(row,
index));
+ }
+ }
+ }
+
+ private ValueWriter makeWriter(Schema avroSchema, DataType dataType) {
+ Schema resolvedSchema = avroSchema == null ? null :
resolveNullableSchema(avroSchema);
+ LogicalType logicalType = resolvedSchema != null ?
resolvedSchema.getLogicalType() : null;
+
+ if (dataType == DataTypes.BooleanType) {
+ return (row, ordinal) ->
recordConsumer.addBoolean(row.getBoolean(ordinal));
+ } else if (dataType == DataTypes.DateType) {
+ return (row, ordinal) -> recordConsumer.addInteger((Integer)
dateRebaseFunction.apply(row.getInt(ordinal)));
+ } else if (dataType == DataTypes.ShortType) {
+ return (row, ordinal) ->
recordConsumer.addInteger(row.getShort(ordinal));
+ } else if (dataType == DataTypes.IntegerType) {
+ return (row, ordinal) -> recordConsumer.addInteger(row.getInt(ordinal));
+ } else if (dataType == DataTypes.LongType) {
+ return (row, ordinal) -> recordConsumer.addLong(row.getLong(ordinal));
+ } else if (dataType == DataTypes.TimestampType) {
+ if (logicalType == null ||
logicalType.getName().equals(LogicalTypes.timestampMicros().getName())) {
+ return (row, ordinal) -> recordConsumer.addLong((long)
timestampRebaseFunction.apply(row.getLong(ordinal)));
+ } else if
(logicalType.getName().equals(LogicalTypes.timestampMillis().getName())) {
+ return (row, ordinal) ->
recordConsumer.addLong(DateTimeUtils.microsToMillis((long)
timestampRebaseFunction.apply(row.getLong(ordinal))));
+ } else {
+ throw new UnsupportedOperationException("Unsupported timestamp type: "
+ logicalType);
+ }
+ } else if
(SparkAdapterSupport$.MODULE$.sparkAdapter().isTimestampNTZType(dataType)) {
+ if (logicalType == null ||
logicalType.getName().equals(LogicalTypes.localTimestampMicros().getName())) {
+ return (row, ordinal) -> recordConsumer.addLong(row.getLong(ordinal));
+ } else if
(logicalType.getName().equals(LogicalTypes.localTimestampMillis().getName())) {
+ return (row, ordinal) ->
recordConsumer.addLong(DateTimeUtils.microsToMillis(row.getLong(ordinal)));
+ } else {
+ throw new UnsupportedOperationException("Unsupported timestamp type: "
+ logicalType);
+ }
+ } else if (dataType == DataTypes.FloatType) {
+ return (row, ordinal) -> recordConsumer.addFloat(row.getFloat(ordinal));
+ } else if (dataType == DataTypes.DoubleType) {
+ return (row, ordinal) ->
recordConsumer.addDouble(row.getDouble(ordinal));
+ } else if (dataType == DataTypes.StringType) {
+ return (row, ordinal) -> recordConsumer.addBinary(
+ Binary.fromReusedByteArray(row.getUTF8String(ordinal).getBytes()));
+ } else if (dataType == DataTypes.BinaryType) {
+ return (row, ordinal) -> recordConsumer.addBinary(
+ Binary.fromReusedByteArray(row.getBinary(ordinal)));
+ } else if (dataType instanceof DecimalType) {
+ return (row, ordinal) -> {
+ int precision = ((DecimalType) dataType).precision();
+ int scale = ((DecimalType) dataType).scale();
+ byte[] bytes = row.getDecimal(ordinal, precision,
scale).toJavaBigDecimal().unscaledValue().toByteArray();
Review Comment:
Added a check
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
