This is an automated email from the ASF dual-hosted git repository.
yihua pushed a change to branch branch-0.x
in repository https://gitbox.apache.org/repos/asf/hudi.git
from 0b75508a22e [HUDI-7501] Use source profile for S3 and GCS sources
(#10861)
new db7136f5bc3 [HUDI-7523] Add HOODIE_SPARK_DATASOURCE_OPTIONS to be used
in HoodieIncrSource (#10900)
new 6a9b1d98c6b [HUDI-7743] Improve StoragePath usages (#11189)
new 2cf7f882213 [HUDI-7744] Introduce IOFactory and a config to set the
factory (#11192)
new 6ab41caca3b [HUDI-7750] Move HoodieLogFormatWriter class to
hoodie-hadoop-common module (#11207)
new 9480e6d7aa2 remove a few classes from hudi-common (#11209)
new cb2d27d2836 [HUDI-7589] Add API to create HoodieStorage in
HoodieIOFactory (#11208)
new acebbb846f0 [HUDI-7549] Reverting spurious log block deduction with
LogRecordReader (#10922)
new 4d3f0d5bd4e [HUDI-7617] Fix issues for bulk insert user defined
partitioner in StreamSync (#11014)
new 35cc12c09c5 [HUDI-7535] Add metrics for sourceParallelism and Refresh
profile in S3/GCS (#10918)
new d8ae9fa8c3d [HUDI-7749] Bump Spark version 3.3.1 to 3.3.4 (#11198)
new e130575622c [HUDI-7712] Fixing RLI initialization to account for file
slices instead of just base files while initializing (#11153)
new 23a58e65ddb [HUDI-7624] Fixing index tagging duration (#11035)
new 5ab6c63d261 [HUDI-7752] Abstract serializeRecords for log writing
(#11210)
new 6d5e9695959 [HUDI-7429] Fixing average record size estimation for
delta commits (#10763)
new cf048333f5d [HUDI-7759] Remove Hadoop dependencies in hudi-common
module (#11220)
new 13ad6baa110 [HUDI-7532] Include only compaction instants for
lastCompaction in getDeltaCommitsSinceLatestCompaction (#10915)
The 16 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../main/java/org/apache/hudi/cli/HoodieCLI.java | 4 +-
.../hudi/cli/commands/ArchivedCommitsCommand.java | 19 +-
.../hudi/cli/commands/CompactionCommand.java | 2 +-
.../hudi/cli/commands/FileSystemViewCommand.java | 2 +-
.../hudi/cli/commands/HoodieLogFileCommand.java | 2 +-
.../apache/hudi/cli/commands/RepairsCommand.java | 15 +-
.../org/apache/hudi/cli/commands/StatsCommand.java | 2 +-
.../org/apache/hudi/cli/commands/TableCommand.java | 11 +-
.../apache/hudi/cli/commands/TimelineCommand.java | 4 +-
.../java/org/apache/hudi/cli/utils/CommitUtil.java | 2 +-
.../hudi/cli/commands/TestRepairsCommand.java | 6 +-
.../apache/hudi/cli/commands/TestTableCommand.java | 10 +-
.../cli/commands/TestUpgradeDowngradeCommand.java | 4 +-
.../hudi/cli/integ/ITTestSavepointsCommand.java | 6 +-
.../apache/hudi/client/BaseHoodieWriteClient.java | 3 -
.../hudi/client/heartbeat/HeartbeatUtils.java | 2 +-
.../client/heartbeat/HoodieHeartbeatClient.java | 4 +-
.../org/apache/hudi/config/HoodieWriteConfig.java | 11 +-
.../org/apache/hudi/index/HoodieIndexUtils.java | 7 +-
.../index/bucket/ConsistentBucketIndexUtils.java | 10 +-
.../org/apache/hudi/io/HoodieAppendHandle.java | 28 +--
.../java/org/apache/hudi/io/HoodieIOHandle.java | 4 +-
.../hudi/io/HoodieKeyLocationFetchHandle.java | 8 +-
.../org/apache/hudi/io/HoodieKeyLookupHandle.java | 3 +-
.../java/org/apache/hudi/io/HoodieMergeHandle.java | 7 +-
.../org/apache/hudi/io/HoodieMergedReadHandle.java | 14 +-
.../java/org/apache/hudi/io/HoodieReadHandle.java | 11 +-
.../java/org/apache/hudi/io/HoodieWriteHandle.java | 2 +-
.../metadata/HoodieBackedTableMetadataWriter.java | 109 ++++++++++--
.../org/apache/hudi/metrics/HoodieMetrics.java | 20 +++
.../apache/hudi/table/BulkInsertPartitioner.java | 7 +
.../java/org/apache/hudi/table/HoodieTable.java | 4 +-
.../hudi/table/action/HoodieWriteMetadata.java | 12 ++
.../hudi/table/action/commit/BaseWriteHelper.java | 6 -
.../table/action/commit/HoodieMergeHelper.java | 14 +-
.../table/action/index/RunIndexActionExecutor.java | 3 +-
.../rollback/ListingBasedRollbackStrategy.java | 4 +-
.../hudi/table/upgrade/UpgradeDowngrade.java | 6 +-
.../table/upgrade/ZeroToOneUpgradeHandler.java | 2 +-
.../org/apache/hudi/DummyTaskContextSupplier.java | 5 -
.../hudi/client/utils/TestCommitMetadataUtils.java | 4 +-
.../org/apache/hudi/metrics/TestHoodieMetrics.java | 8 +
.../hudi/table/TestBulkInsertPartitioner.java | 20 ---
.../GenericRecordValidationTestUtils.java | 7 +-
.../hudi/client/FlinkTaskContextSupplier.java | 5 -
.../java/org/apache/hudi/io/FlinkAppendHandle.java | 4 -
.../apache/hudi/io/FlinkWriteHandleFactory.java | 4 +-
.../io/storage/row/HoodieRowDataCreateHandle.java | 7 +-
.../row/HoodieRowDataFileWriterFactory.java | 7 +-
.../FlinkHoodieBackedTableMetadataWriter.java | 7 +
.../run/strategy/JavaExecutionStrategy.java | 10 +-
.../client/common/JavaTaskContextSupplier.java | 6 -
.../JavaCustomColumnsSortPartitioner.java | 10 +-
.../org/apache/hudi/table/HoodieJavaTable.java | 5 +-
.../table/action/commit/JavaUpsertPartitioner.java | 2 +-
.../client/TestHoodieJavaWriteClientInsert.java | 6 +-
.../hudi/client/TestJavaHoodieBackedMetadata.java | 24 +--
.../TestHoodieJavaClientOnCopyOnWriteStorage.java | 6 +-
.../commit/TestJavaCopyOnWriteActionExecutor.java | 6 +-
.../testutils/HoodieJavaClientTestHarness.java | 15 +-
.../apache/hudi/client/SparkRDDWriteClient.java | 5 +-
.../hudi/client/SparkTaskContextSupplier.java | 5 -
.../MultipleSparkJobExecutionStrategy.java | 12 +-
.../strategy/SingleSparkJobExecutionStrategy.java | 7 +-
.../RDDCustomColumnsSortPartitioner.java | 16 +-
.../index/bloom/HoodieFileProbingFunction.java | 3 +-
.../io/storage/HoodieSparkFileReaderFactory.java | 21 ++-
.../io/storage/HoodieSparkFileWriterFactory.java | 20 ++-
.../hudi/io/storage/HoodieSparkIOFactory.java | 53 ++++++
.../hudi/io/storage/HoodieSparkParquetReader.java | 18 +-
.../row/HoodieInternalRowFileWriterFactory.java | 3 +-
.../SparkHoodieBackedTableMetadataWriter.java | 8 +
.../org/apache/hudi/table/HoodieSparkTable.java | 5 +-
.../bootstrap/ParquetBootstrapMetadataHandler.java | 6 +-
.../action/commit/AverageRecordSizeUtils.java | 91 ++++++++++
.../commit/BaseSparkCommitActionExecutor.java | 5 +
.../table/action/commit/UpsertPartitioner.java | 41 +----
.../SparkUpsertDeltaCommitActionExecutor.java | 2 +-
.../java/org/apache/hudi/client/TestMultiFS.java | 10 +-
.../hudi/client/TestTableSchemaEvolution.java | 4 +-
.../hudi/client/TestUpdateSchemaEvolution.java | 4 +-
.../functional/TestHoodieBackedMetadata.java | 30 ++--
.../functional/TestHoodieBackedTableMetadata.java | 7 +-
.../TestHoodieClientOnCopyOnWriteStorage.java | 16 +-
.../TestBulkInsertInternalPartitioner.java | 7 +-
.../org/apache/hudi/io/TestHoodieMergeHandle.java | 8 +-
.../java/org/apache/hudi/table/TestCleaner.java | 2 +-
.../hudi/table/TestHoodieMergeOnReadTable.java | 6 +-
.../action/commit/TestAverageRecordSizeUtils.java | 195 +++++++++++++++++++++
.../commit/TestCopyOnWriteActionExecutor.java | 8 +-
.../table/action/commit/TestUpsertPartitioner.java | 5 +-
.../table/action/compact/TestInlineCompaction.java | 6 +-
.../TestCopyOnWriteRollbackActionExecutor.java | 2 +-
...dieSparkMergeOnReadTableInsertUpdateDelete.java | 4 +-
.../TestHoodieSparkMergeOnReadTableRollback.java | 10 +-
.../hudi/table/upgrade/TestUpgradeDowngrade.java | 16 +-
.../hudi/testutils/FunctionalTestHarness.java | 3 +-
.../hudi/testutils/HoodieClientTestBase.java | 2 +-
.../hudi/testutils/HoodieClientTestUtils.java | 5 +-
.../SparkClientFunctionalTestHarness.java | 4 +-
hudi-common/pom.xml | 18 --
.../hudi/avro/HoodieBloomFilterWriteSupport.java | 5 +-
.../hudi/common/config/HoodieStorageConfig.java | 8 +
.../common/engine/LocalTaskContextSupplier.java | 5 -
.../hudi/common/engine/TaskContextSupplier.java | 5 -
.../java/org/apache/hudi/common/fs/FSUtils.java | 28 +--
.../common/heartbeat/HoodieHeartbeatUtils.java | 2 +-
.../hudi/common/model/HoodiePartitionMetadata.java | 6 +-
.../hudi/common/table/HoodieTableConfig.java | 8 +-
.../hudi/common/table/HoodieTableMetaClient.java | 19 +-
.../hudi/common/table/TableSchemaResolver.java | 6 +-
.../table/log/AbstractHoodieLogRecordReader.java | 172 +-----------------
.../hudi/common/table/log/HoodieLogFormat.java | 8 +-
.../common/table/log/block/HoodieDataBlock.java | 6 +-
.../table/log/block/HoodieHFileDataBlock.java | 127 +++-----------
.../table/log/block/HoodieParquetDataBlock.java | 60 ++-----
.../table/timeline/HoodieActiveTimeline.java | 4 +-
.../table/timeline/HoodieDefaultTimeline.java | 11 +-
.../view/HoodieTablePreCommitFileSystemView.java | 2 +-
.../apache/hudi/common/util/CompactionUtils.java | 3 +-
.../org/apache/hudi/common/util/ConfigUtils.java | 17 +-
.../{BaseFileUtils.java => FileFormatUtils.java} | 38 ++--
.../io/FileBasedInternalSchemaStorageManager.java | 5 +-
.../hudi/io/storage/HoodieFileReaderFactory.java | 69 +++-----
.../hudi/io/storage/HoodieFileWriterFactory.java | 53 ++----
.../apache/hudi/io/storage/HoodieIOFactory.java | 65 +++++++
.../metadata/FileSystemBackedTableMetadata.java | 2 +-
.../hudi/metadata/HoodieBackedTableMetadata.java | 10 +-
.../hudi/metadata/HoodieTableMetadataUtil.java | 23 ++-
.../secondary/index/SecondaryIndexManager.java | 7 +-
.../apache/hudi/storage/HoodieStorageUtils.java | 49 +-----
.../view/TestPriorityBasedFileSystemView.java | 2 +-
.../common/testutils/HoodieTestDataGenerator.java | 12 ++
.../hudi/common/testutils/HoodieTestUtils.java | 13 +-
.../apache/hudi/common/util/TestConfigUtils.java | 66 +++++--
.../hudi/sink/bootstrap/BootstrapOperator.java | 7 +-
.../hudi/sink/clustering/ClusteringOperator.java | 11 +-
.../java/org/apache/hudi/util/StreamerUtil.java | 2 +-
.../sink/TestStreamWriteOperatorCoordinator.java | 9 +-
.../hudi/sink/bucket/ITTestBucketStreamWrite.java | 2 +-
.../hudi/table/catalog/HoodieCatalogTestUtils.java | 7 +
.../hudi/table/catalog/TestHoodieCatalog.java | 4 +-
.../hudi/table/catalog/TestHoodieHiveCatalog.java | 15 +-
.../apache/hudi/table/format/TestInputFormat.java | 7 +-
.../test/java/org/apache/hudi/utils/TestUtils.java | 17 +-
.../apache/hudi/avro/HoodieAvroWriteSupport.java | 16 +-
.../common/config/DFSPropertiesConfiguration.java | 2 +-
.../common/table/log/HoodieLogFormatWriter.java | 17 +-
.../org/apache/hudi/common/util/HFileUtils.java | 128 +++++++++++++-
.../java/org/apache/hudi/common/util/OrcUtils.java | 11 +-
.../hudi/common/util/ParquetReaderIterator.java | 0
.../org/apache/hudi/common/util/ParquetUtils.java | 53 +++++-
.../org/apache/hudi/hadoop/fs/HadoopFSUtils.java | 28 ++-
.../io/hadoop/HoodieAvroFileReaderFactory.java | 46 ++---
.../io/hadoop/HoodieAvroFileWriterFactory.java | 61 +++----
.../apache/hudi/io/hadoop/HoodieAvroOrcReader.java | 6 +-
.../apache/hudi/io/hadoop/HoodieAvroOrcWriter.java | 3 +-
.../hudi/io/hadoop/HoodieAvroParquetReader.java | 6 +-
.../hudi/io/storage/HoodieHadoopIOFactory.java | 98 +++++++++++
.../hudi/io/storage/HoodieParquetConfig.java | 0
.../hudi/storage/hadoop/HoodieHadoopStorage.java | 26 +--
.../common/bootstrap/index/TestBootstrapIndex.java | 3 +-
.../fs/TestFSUtilsWithRetryWrapperEnable.java | 14 +-
.../common/fs/TestHoodieWrapperFileSystem.java | 4 +-
.../common/functional/TestHoodieLogFormat.java | 126 +------------
.../TestHoodieLogFormatAppendFailure.java | 4 +-
.../hudi/common/table/TestHoodieTableConfig.java | 26 +--
.../common/table/TestHoodieTableMetaClient.java | 10 +-
.../hudi/common/table/TestTimelineUtils.java | 12 +-
.../table/timeline/TestHoodieActiveTimeline.java | 48 +++--
.../hudi/common/util/TestCompactionUtils.java | 58 ++++++
.../apache/hudi/common/util/TestHFileUtils.java | 59 +++++++
.../common/util/TestParquetReaderIterator.java | 0
.../io/hadoop/TestHoodieAvroFileReaderFactory.java | 14 +-
.../hudi/io/hadoop/TestHoodieOrcReaderWriter.java | 8 +-
.../hadoop/HoodieCopyOnWriteTableInputFormat.java | 4 +-
.../hudi/hadoop/HoodieHFileRecordReader.java | 13 +-
.../hudi/hadoop/HoodieROTablePathFilter.java | 8 +-
.../apache/hudi/hadoop/SchemaEvolutionContext.java | 9 +-
.../HoodieMergeOnReadTableInputFormat.java | 3 +-
.../hudi/hadoop/utils/HoodieInputFormatUtils.java | 8 +-
.../utils/HoodieRealtimeRecordReaderUtils.java | 12 +-
.../hive/TestHoodieCombineHiveInputFormat.java | 4 +-
.../TestHoodieMergeOnReadSnapshotReader.java | 7 +-
.../TestHoodieMergeOnReadTableInputFormat.java | 4 +-
.../realtime/TestHoodieRealtimeRecordReader.java | 4 +-
.../hudi/hadoop/testutils/InputFormatTestUtil.java | 13 +-
.../reader/DFSHoodieDatasetInputReader.java | 9 +-
.../org/apache/hudi/storage/HoodieStorage.java | 6 +
.../main/java/org/apache/hudi/DataSourceUtils.java | 2 +-
.../main/scala/org/apache/hudi/DefaultSource.scala | 6 +-
.../scala/org/apache/hudi/HoodieBaseRelation.scala | 17 +-
.../org/apache/spark/sql/hudi/DedupeSparkJob.scala | 15 +-
.../org/apache/spark/sql/hudi/SparkHelpers.scala | 8 +-
.../procedures/ExportInstantsProcedure.scala | 3 +-
.../RepairAddpartitionmetaProcedure.scala | 2 +-
.../RepairMigratePartitionMetaProcedure.scala | 4 +-
.../RepairOverwriteHoodiePropsProcedure.scala | 5 +-
.../ShowHoodieLogFileRecordsProcedure.scala | 2 +-
.../StatsWriteAmplificationProcedure.scala | 2 +-
.../procedures/ValidateHoodieSyncProcedure.scala | 2 +-
.../src/test/java/HoodieJavaStreamingApp.java | 7 +-
.../org/apache/hudi/ColumnStatsIndexHelper.java | 4 +-
.../hudi/functional/RecordLevelIndexTestBase.scala | 21 ++-
.../hudi/functional/TestMORDataSourceStorage.scala | 6 +-
.../hudi/functional/TestRecordLevelIndex.scala | 78 ++++++++-
.../hudi/functional/TestStructuredStreaming.scala | 2 +-
.../functional/cdc/TestCDCDataFrameSuite.scala | 26 +--
.../apache/spark/sql/hudi/common/TestSqlConf.scala | 6 +-
.../hudi/procedure/TestClusteringProcedure.scala | 8 +-
.../hudi/procedure/TestCompactionProcedure.scala | 2 +-
.../sql/hudi/procedure/TestRepairsProcedure.scala | 8 +-
.../org/apache/hudi/hive/TestHiveSyncTool.java | 3 +-
.../apache/hudi/hive/testutils/HiveTestUtil.java | 4 +-
.../MarkerBasedEarlyConflictDetectionRunnable.java | 2 +-
.../utilities/HoodieMetadataTableValidator.java | 13 +-
.../utilities/config/HoodieIncrSourceConfig.java | 8 +
.../hudi/utilities/deltastreamer/DeltaSync.java | 4 +-
.../deltastreamer/HoodieDeltaStreamer.java | 6 +-
.../ingestion/HoodieIngestionMetrics.java | 4 +
.../sources/GcsEventsHoodieIncrSource.java | 24 ++-
.../hudi/utilities/sources/HoodieIncrSource.java | 17 +-
.../apache/hudi/utilities/sources/KafkaSource.java | 5 +-
.../sources/S3EventsHoodieIncrSource.java | 11 +-
.../sources/helpers/CloudDataFetcher.java | 12 +-
.../utilities/streamer/HoodieStreamerMetrics.java | 13 ++
.../apache/hudi/utilities/streamer/StreamSync.java | 7 +-
.../deltastreamer/HoodieDeltaStreamerTestBase.java | 4 +-
.../deltastreamer/TestHoodieDeltaStreamer.java | 57 +++++-
.../TestHoodieDeltaStreamerWithMultiWriter.java | 7 +-
.../utilities/sources/BaseTestKafkaSource.java | 4 +
.../sources/TestGcsEventsHoodieIncrSource.java | 15 +-
.../utilities/sources/TestHoodieIncrSource.java | 39 ++++-
.../sources/TestS3EventsHoodieIncrSource.java | 17 +-
.../streamer/TestStreamSyncUnitTests.java | 6 +-
.../utilities/testutils/UtilitiesTestBase.java | 4 +-
pom.xml | 2 +-
237 files changed, 2209 insertions(+), 1554 deletions(-)
create mode 100644
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkIOFactory.java
create mode 100644
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/AverageRecordSizeUtils.java
create mode 100644
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/commit/TestAverageRecordSizeUtils.java
rename
hudi-common/src/main/java/org/apache/hudi/common/util/{BaseFileUtils.java =>
FileFormatUtils.java} (87%)
create mode 100644
hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieIOFactory.java
rename {hudi-common =>
hudi-hadoop-common}/src/main/java/org/apache/hudi/avro/HoodieAvroWriteSupport.java
(82%)
rename {hudi-common =>
hudi-hadoop-common}/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatWriter.java
(96%)
rename {hudi-common =>
hudi-hadoop-common}/src/main/java/org/apache/hudi/common/util/ParquetReaderIterator.java
(100%)
create mode 100644
hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/HoodieHadoopIOFactory.java
rename {hudi-common =>
hudi-hadoop-common}/src/main/java/org/apache/hudi/io/storage/HoodieParquetConfig.java
(100%)
create mode 100644
hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestHFileUtils.java
rename {hudi-common =>
hudi-hadoop-common}/src/test/java/org/apache/hudi/common/util/TestParquetReaderIterator.java
(100%)
