This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
from 412f4273da6 [HUDI-9207] Spark Insert Overwrite Support Row Writer
(#13014)
add 9352576357d [HUDI-9144] Avro elimination with RowData for Flink MOR
table writer (#12967)
No new revisions were added by this update.
Summary of changes:
.../org/apache/hudi/io/HoodieAppendHandle.java | 80 ++--
.../hudi/client/FlinkRowDataWriteClient.java | 75 ++++
.../apache/hudi/client/HoodieFlinkWriteClient.java | 73 +++-
.../hudi/client/model/HoodieFlinkRecord.java | 181 ++++++++++
.../io/log/block/HoodieFlinkParquetDataBlock.java | 105 ++++++
.../hudi/io/storage/row/HoodieFlinkIOFactory.java | 26 +-
.../io/storage/row/HoodieRowDataFileWriter.java | 16 +-
.../row/HoodieRowDataFileWriterFactory.java | 48 ++-
.../HoodieRowDataParquetOutputStreamWriter.java | 102 ++++++
.../row/HoodieRowDataParquetWriteSupport.java | 4 +-
.../io/storage/row/RowDataParquetWriteSupport.java | 11 +-
.../storage/row/parquet/ParquetRowDataWriter.java | 37 +-
.../row/parquet/ParquetSchemaConverter.java | 26 +-
.../hudi/io/v2/FlinkRecordSizeEstimator.java | 23 +-
.../hudi/io/v2/FlinkRowDataHandleFactory.java | 108 ++++++
.../apache/hudi/io/v2/RowDataLogWriteHandle.java | 238 ++++++++++++
.../hudi/table/ExplicitWriteHandleTable.java | 19 +
.../hudi/table/HoodieFlinkCopyOnWriteTable.java | 21 ++
.../hudi/table/HoodieFlinkMergeOnReadTable.java | 16 +
.../commit/BaseFlinkCommitActionExecutor.java | 21 +-
.../RowDataUpsertDeltaCommitActionExecutor.java | 86 +++++
.../org/apache/hudi/util/AvroSchemaConverter.java | 9 +-
.../hudi/util/MutableIteratorWrapperIterator.java | 67 ++++
.../apache/hudi/util/OrderingValueExtractor.java | 13 +-
.../row/parquet/TestParquetSchemaConverter.java | 2 +-
.../hudi/common/config/HoodieStorageConfig.java | 13 +
.../org/apache/hudi/common/model/HoodieRecord.java | 2 +-
.../table/log/block/HoodieParquetDataBlock.java | 12 +-
.../apache/hudi/common/util/FileFormatUtils.java | 20 +
.../io/storage/ColumnRangeMetadataProvider.java | 15 +-
.../apache/hudi/io/storage/HoodieFileWriter.java | 8 +
.../apache/hudi/configuration/FlinkOptions.java | 14 +
.../apache/hudi/configuration/OptionsResolver.java | 16 +
...nction.java => RowDataStreamWriteFunction.java} | 401 ++++++++++-----------
.../org/apache/hudi/sink/StreamWriteFunction.java | 78 +---
.../org/apache/hudi/sink/StreamWriteOperator.java | 4 +-
.../sink/bucket/BucketStreamWriteFunction.java | 2 -
.../sink/bucket/BucketStreamWriteOperator.java | 3 +-
....java => RowDataBucketStreamWriteFunction.java} | 14 +-
.../hudi/sink/buffer/BufferSizeDetector.java | 70 ++++
.../hudi/sink/buffer/HeapMemorySegmentPool.java | 71 ++++
.../hudi/sink/buffer/MemorySegmentPoolFactory.java | 44 +++
.../org/apache/hudi/sink/buffer/RowDataBucket.java | 91 +++++
.../apache/hudi/sink/buffer/TotalSizeTracer.java | 62 ++++
.../hudi/sink/bulk/BulkInsertWriterHelper.java | 40 +-
.../hudi/sink/clustering/ClusteringOperator.java | 3 +-
.../exception/MemoryPagesExhaustedException.java | 16 +-
.../org/apache/hudi/sink/utils/BufferUtils.java | 53 +++
.../hudi/sink/utils/NaturalOrderKeyComputer.java | 60 +++
.../sink/utils/NaturalOrderRecordComparator.java} | 27 +-
.../java/org/apache/hudi/util/DataTypeUtils.java | 38 ++
.../org/apache/hudi/util/FlinkWriteClients.java | 1 +
.../org/apache/hudi/sink/TestWriteCopyOnWrite.java | 20 +-
.../hudi/sink/bulk/TestBulkInsertWriteHelper.java | 3 +-
.../utils/BucketStreamWriteFunctionWrapper.java | 23 +-
.../sink/utils/StreamWriteFunctionWrapper.java | 18 +-
.../org/apache/hudi/sink/utils/TestWriteBase.java | 7 +
.../apache/hudi/table/ITTestHoodieDataSource.java | 59 +++
.../apache/hudi/table/ITTestSchemaEvolution.java | 4 +
.../test/java/org/apache/hudi/utils/TestData.java | 2 +-
.../table/format/cow/ParquetSplitReaderUtil.java | 42 ++-
.../table/format/cow/ParquetSplitReaderUtil.java | 42 ++-
.../table/format/cow/ParquetSplitReaderUtil.java | 42 ++-
.../org/apache/hudi/common/util/HFileUtils.java | 12 +
.../java/org/apache/hudi/common/util/OrcUtils.java | 15 +-
.../org/apache/hudi/common/util/ParquetUtils.java | 41 ++-
.../hudi/io/hadoop/HoodieHadoopIOFactory.java | 9 +
67 files changed, 2402 insertions(+), 522 deletions(-)
create mode 100644
hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/FlinkRowDataWriteClient.java
create mode 100644
hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/model/HoodieFlinkRecord.java
create mode 100644
hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/log/block/HoodieFlinkParquetDataBlock.java
copy
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/bootstrap/IndexRecord.java
=>
hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieFlinkIOFactory.java
(56%)
create mode 100644
hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataParquetOutputStreamWriter.java
copy
hudi-common/src/main/java/org/apache/hudi/exception/HoodieRecordMissingException.java
=>
hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/v2/FlinkRecordSizeEstimator.java
(60%)
create mode 100644
hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/v2/FlinkRowDataHandleFactory.java
create mode 100644
hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/v2/RowDataLogWriteHandle.java
create mode 100644
hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/action/commit/delta/RowDataUpsertDeltaCommitActionExecutor.java
rename {hudi-flink-datasource/hudi-flink =>
hudi-client/hudi-flink-client}/src/main/java/org/apache/hudi/util/AvroSchemaConverter.java
(98%)
create mode 100644
hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/MutableIteratorWrapperIterator.java
copy
hudi-common/src/main/java/org/apache/hudi/common/function/SerializableSupplier.java
=>
hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/OrderingValueExtractor.java
(77%)
copy
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileReader.java
=>
hudi-common/src/main/java/org/apache/hudi/io/storage/ColumnRangeMetadataProvider.java
(67%)
copy
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/{StreamWriteFunction.java
=> RowDataStreamWriteFunction.java} (55%)
copy
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/bucket/{BucketStreamWriteFunction.java
=> RowDataBucketStreamWriteFunction.java} (93%)
create mode 100644
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/buffer/BufferSizeDetector.java
create mode 100644
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/buffer/HeapMemorySegmentPool.java
create mode 100644
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/buffer/MemorySegmentPoolFactory.java
create mode 100644
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/buffer/RowDataBucket.java
create mode 100644
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/buffer/TotalSizeTracer.java
copy
hudi-utilities/src/main/java/org/apache/hudi/utilities/exception/HoodieReadFromSourceException.java
=>
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/exception/MemoryPagesExhaustedException.java
(71%)
create mode 100644
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/utils/BufferUtils.java
create mode 100644
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/utils/NaturalOrderKeyComputer.java
copy
hudi-flink-datasource/hudi-flink/src/{test/java/org/apache/hudi/sink/utils/ScalaCollector.java
=> main/java/org/apache/hudi/sink/utils/NaturalOrderRecordComparator.java}
(66%)
