This is an automated email from the ASF dual-hosted git repository.
yihua pushed a change to branch release-0.15.0
in repository https://gitbox.apache.org/repos/asf/hudi.git
from 0de2e8018ff [HUDI-7786] Fix roaring bitmap dependency in
hudi-integ-test-bundle (#11283)
new b71da938ac5 [HUDI-7785] Keep public APIs in utilities module the same
as before HoodieStorage abstraction (#11280)
new 670f13102f0 [HUDI-7775] Remove unused APIs in HoodieStorage (#11281)
new b350a26cabf [HUDI-7788] Fixing exception handling in
AverageRecordSizeUtils (#11290)
new 77b1ded0209 [HUDI-7776] Simplify HoodieStorage instance fetching
(#11292)
new 2e763c9ec8e [HUDI-7792] Bump h2 from 1.4.200 to 2.2.220 (#11296)
new 72dd5183ba5 [HUDI-7790] Revert changes in DFSPathSelector and
UtilHelpers.readConfig (#11294)
new b6078994449 [HUDI-7794] Bump org.apache.hive:hive-service from 2.3.1
to 2.3.4 (#11298)
The 7 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../hudi/cli/commands/ArchivedCommitsCommand.java | 2 +-
.../apache/hudi/cli/commands/MetadataCommand.java | 22 ++---
.../apache/hudi/cli/commands/RepairsCommand.java | 2 +-
.../org/apache/hudi/cli/commands/SparkMain.java | 3 +-
.../org/apache/hudi/client/BaseHoodieClient.java | 2 +-
.../apache/hudi/client/HoodieTimelineArchiver.java | 2 +-
.../hudi/client/heartbeat/HeartbeatUtils.java | 5 +-
.../hudi/client/transaction/lock/LockManager.java | 3 +-
.../lock/metrics/HoodieLockMetrics.java | 14 +--
.../org/apache/hudi/index/HoodieIndexUtils.java | 7 +-
.../hudi/index/bloom/HoodieGlobalBloomIndex.java | 2 +-
.../index/bucket/ConsistentBucketIndexUtils.java | 6 +-
.../hudi/index/simple/HoodieGlobalSimpleIndex.java | 2 +-
.../org/apache/hudi/io/HoodieCreateHandle.java | 2 +-
.../hudi/io/HoodieKeyLocationFetchHandle.java | 4 +-
.../org/apache/hudi/io/HoodieKeyLookupHandle.java | 2 +-
.../java/org/apache/hudi/io/HoodieMergeHandle.java | 4 +-
.../java/org/apache/hudi/io/HoodieReadHandle.java | 8 +-
.../java/org/apache/hudi/io/HoodieWriteHandle.java | 2 +-
.../metadata/HoodieBackedTableMetadataWriter.java | 8 +-
.../org/apache/hudi/metrics/HoodieMetrics.java | 6 +-
.../java/org/apache/hudi/table/HoodieTable.java | 10 +-
.../table/action/clean/CleanActionExecutor.java | 6 +-
.../hudi/table/action/clean/CleanPlanner.java | 4 +-
.../PartitionAwareClusteringPlanStrategy.java | 3 +-
.../action/commit/BaseCommitActionExecutor.java | 2 +-
.../table/action/commit/HoodieMergeHelper.java | 13 +--
.../compact/RunCompactionActionExecutor.java | 2 +-
.../BaseHoodieCompactionPlanGenerator.java | 2 +-
.../index/RecordBasedIndexingCatchupTask.java | 3 +-
.../table/action/index/RunIndexActionExecutor.java | 6 +-
.../action/index/ScheduleIndexActionExecutor.java | 2 +-
.../action/restore/BaseRestoreActionExecutor.java | 2 +-
.../rollback/BaseRollbackActionExecutor.java | 4 +-
.../rollback/ListingBasedRollbackStrategy.java | 2 +-
.../action/savepoint/SavepointActionExecutor.java | 3 +-
.../hudi/table/marker/DirectWriteMarkers.java | 2 +-
.../table/marker/MarkerBasedRollbackUtils.java | 2 +-
.../hudi/avro/TestHoodieAvroParquetWriter.java | 16 ++--
.../hudi/client/utils/TestCommitMetadataUtils.java | 2 +-
.../hudi/metrics/TestHoodieConsoleMetrics.java | 4 +-
.../hudi/metrics/TestHoodieGraphiteMetrics.java | 4 +-
.../apache/hudi/metrics/TestHoodieJmxMetrics.java | 4 +-
.../org/apache/hudi/metrics/TestHoodieMetrics.java | 4 +-
.../datadog/TestDatadogMetricsReporter.java | 10 +-
.../org/apache/hudi/metrics/m3/TestM3Metrics.java | 21 ++---
.../metrics/prometheus/TestPrometheusReporter.java | 6 +-
.../prometheus/TestPushGateWayReporter.java | 6 +-
.../GenericRecordValidationTestUtils.java | 7 +-
.../apache/hudi/client/HoodieFlinkWriteClient.java | 2 +-
.../io/storage/row/HoodieRowDataCreateHandle.java | 7 +-
.../FlinkHoodieBackedTableMetadataWriter.java | 2 +-
.../FlinkDeletePartitionCommitActionExecutor.java | 2 +-
.../index/bloom/TestFlinkHoodieBloomIndex.java | 3 +-
.../run/strategy/JavaExecutionStrategy.java | 7 +-
.../JavaHoodieBackedTableMetadataWriter.java | 2 +-
...vaInsertOverwriteTableCommitActionExecutor.java | 2 +-
.../client/TestHoodieJavaWriteClientInsert.java | 8 +-
.../hudi/client/TestJavaHoodieBackedMetadata.java | 10 +-
.../TestHoodieJavaClientOnCopyOnWriteStorage.java | 2 +-
.../commit/TestJavaCopyOnWriteActionExecutor.java | 14 +--
.../testutils/HoodieJavaClientTestHarness.java | 11 ++-
.../MultipleSparkJobExecutionStrategy.java | 14 ++-
.../strategy/SingleSparkJobExecutionStrategy.java | 3 +-
.../client/validator/SparkPreCommitValidator.java | 2 +-
.../index/bloom/HoodieFileProbingFunction.java | 3 +-
.../io/storage/HoodieSparkFileReaderFactory.java | 18 ++--
.../io/storage/HoodieSparkFileWriterFactory.java | 11 ++-
.../hudi/io/storage/HoodieSparkIOFactory.java | 15 +--
.../hudi/io/storage/HoodieSparkParquetReader.java | 32 +++----
.../hudi/io/storage/row/HoodieRowCreateHandle.java | 4 +-
.../SparkHoodieBackedTableMetadataWriter.java | 2 +-
.../bootstrap/ParquetBootstrapMetadataHandler.java | 2 +-
.../action/commit/AverageRecordSizeUtils.java | 5 +-
.../SparkDeletePartitionCommitActionExecutor.java | 2 +-
...rkInsertOverwriteTableCommitActionExecutor.java | 3 +-
.../org/apache/hudi/client/TestClientRollback.java | 4 +-
.../hudi/client/TestUpdateSchemaEvolution.java | 2 +-
.../functional/TestExternalPathHandling.java | 3 +-
.../functional/TestHoodieBackedMetadata.java | 40 ++++----
.../functional/TestHoodieBackedTableMetadata.java | 11 ++-
.../TestHoodieClientOnCopyOnWriteStorage.java | 30 +++---
.../TestRemoteFileSystemViewWithMetadataTable.java | 2 +-
.../hudi/index/bloom/TestHoodieBloomIndex.java | 2 +-
.../storage/TestHoodieAvroFileWriterFactory.java | 8 +-
.../action/commit/TestAverageRecordSizeUtils.java | 20 ++++
.../commit/TestCopyOnWriteActionExecutor.java | 6 +-
.../table/functional/TestHoodieSparkRollback.java | 7 +-
.../hudi/table/marker/TestDirectWriteMarkers.java | 3 +-
.../TestTimelineServerBasedWriteMarkers.java | 3 +-
.../hudi/table/upgrade/TestUpgradeDowngrade.java | 6 +-
.../hudi/testutils/FunctionalTestHarness.java | 3 +-
.../testutils/HoodieSparkClientTestHarness.java | 4 +-
.../org/apache/hudi/BaseHoodieTableFileIndex.java | 7 +-
.../hudi/common/config/HoodieStorageConfig.java | 12 ++-
.../java/org/apache/hudi/common/fs/FSUtils.java | 12 ++-
.../hudi/common/model/HoodieCommitMetadata.java | 16 ++--
.../hudi/common/model/HoodiePartitionMetadata.java | 2 +-
.../hudi/common/table/HoodieTableMetaClient.java | 7 +-
.../hudi/common/table/TableSchemaResolver.java | 4 +-
.../hudi/common/table/log/HoodieLogFileReader.java | 7 +-
.../table/log/block/HoodieAvroDataBlock.java | 4 +-
.../common/table/log/block/HoodieCommandBlock.java | 4 +-
.../common/table/log/block/HoodieCorruptBlock.java | 4 +-
.../common/table/log/block/HoodieDataBlock.java | 8 +-
.../common/table/log/block/HoodieDeleteBlock.java | 4 +-
.../table/log/block/HoodieHFileDataBlock.java | 20 ++--
.../common/table/log/block/HoodieLogBlock.java | 14 +--
.../table/log/block/HoodieParquetDataBlock.java | 12 ++-
.../table/timeline/HoodieActiveTimeline.java | 8 +-
.../common/table/view/FileSystemViewManager.java | 2 +-
.../apache/hudi/common/util/FileFormatUtils.java | 101 ++++++++++----------
.../hudi/common/util/InternalSchemaCache.java | 16 ++--
.../io/FileBasedInternalSchemaStorageManager.java | 15 +--
.../hudi/io/storage/HoodieFileReaderFactory.java | 7 +-
.../hudi/io/storage/HoodieFileWriterFactory.java | 16 ++--
.../apache/hudi/io/storage/HoodieIOFactory.java | 13 ++-
.../io/storage/HoodieNativeAvroHFileReader.java | 13 +--
.../hudi/metadata/AbstractHoodieTableMetadata.java | 7 +-
.../apache/hudi/metadata/BaseTableMetadata.java | 25 +++--
.../metadata/FileSystemBackedTableMetadata.java | 29 +++---
.../hudi/metadata/HoodieBackedTableMetadata.java | 17 +++-
.../metadata/HoodieMetadataFileSystemView.java | 5 +-
.../hudi/metadata/HoodieMetadataMetrics.java | 6 +-
.../hudi/metadata/HoodieMetadataPayload.java | 11 ---
.../apache/hudi/metadata/HoodieTableMetadata.java | 24 +++--
.../hudi/metadata/HoodieTableMetadataUtil.java | 23 ++---
.../main/java/org/apache/hudi/metrics/Metrics.java | 16 ++--
.../apache/hudi/storage/HoodieStorageUtils.java | 13 ++-
.../hudi/common/testutils/HoodieTestUtils.java | 6 ++
.../HoodieBackedTestDelayedTableMetadata.java | 4 +-
.../hudi/sink/bootstrap/BootstrapOperator.java | 6 +-
.../hudi/sink/clustering/ClusteringOperator.java | 6 +-
.../sink/partitioner/profile/WriteProfiles.java | 5 +-
.../java/org/apache/hudi/source/FileIndex.java | 9 +-
.../hudi/source/stats/ColumnStatsIndices.java | 4 +-
.../hudi/table/format/InternalSchemaManager.java | 6 +-
.../hudi/table/format/cdc/CdcInputFormat.java | 5 +-
.../sink/TestStreamWriteOperatorCoordinator.java | 2 +-
.../sink/compact/ITTestHoodieFlinkCompactor.java | 2 +-
.../common/config/DFSPropertiesConfiguration.java | 3 +-
.../common/table/log/HoodieLogFormatWriter.java | 2 +-
.../org/apache/hudi/common/util/HFileUtils.java | 29 +++---
.../java/org/apache/hudi/common/util/OrcUtils.java | 62 ++++++-------
.../org/apache/hudi/common/util/ParquetUtils.java | 103 ++++++++++-----------
.../io/hadoop/HoodieAvroFileReaderFactory.java | 19 ++--
.../io/hadoop/HoodieAvroFileWriterFactory.java | 16 ++--
.../apache/hudi/io/hadoop/HoodieAvroOrcReader.java | 22 ++---
.../hudi/io/hadoop/HoodieAvroParquetReader.java | 19 ++--
.../{storage => hadoop}/HoodieHadoopIOFactory.java | 24 ++---
.../hudi/storage/hadoop/HoodieHadoopStorage.java | 41 ++++----
.../common/functional/TestHoodieLogFormat.java | 4 +-
.../hudi/common/table/TestTableSchemaResolver.java | 7 +-
.../table/log/block/TestHoodieDeleteBlock.java | 2 +-
.../util/TestDFSPropertiesConfiguration.java | 6 +-
.../apache/hudi/common/util/TestParquetUtils.java | 14 +--
.../io/hadoop/TestHoodieAvroFileReaderFactory.java | 13 ++-
.../hadoop/TestHoodieHBaseHFileReaderWriter.java | 17 ++--
.../io/hadoop/TestHoodieHFileReaderWriter.java | 18 ++--
.../io/hadoop/TestHoodieHFileReaderWriterBase.java | 42 ++++-----
.../hudi/io/hadoop/TestHoodieOrcReaderWriter.java | 13 +--
.../hudi/io/hadoop/TestHoodieReaderWriterBase.java | 43 +++++----
.../TestFileSystemBackedTableMetadata.java | 12 +--
.../hudi/metadata/TestHoodieTableMetadataUtil.java | 2 +-
.../hudi/hadoop/HoodieHFileRecordReader.java | 3 +-
.../hudi/hadoop/HoodieROTablePathFilter.java | 5 +-
.../hudi/hadoop/utils/HoodieInputFormatUtils.java | 5 +-
.../utils/HoodieRealtimeRecordReaderUtils.java | 4 +-
.../hudi/integ/testsuite/HoodieTestSuiteJob.java | 3 +-
.../SparkDataSourceContinuousIngestTool.java | 3 +-
.../helpers/DFSTestSuitePathSelector.java | 41 ++++----
.../reader/DFSHoodieDatasetInputReader.java | 4 +-
.../org/apache/hudi/storage/HoodieStorage.java | 52 +++++------
.../hudi/io/storage/TestHoodieStorageBase.java | 9 --
hudi-platform-service/hudi-metaserver/pom.xml | 2 +-
...lkInsertOverwriteTableCommitActionExecutor.java | 4 +-
.../org/apache/hudi/ColumnStatsIndexSupport.scala | 2 +-
.../scala/org/apache/hudi/HoodieBaseRelation.scala | 7 +-
.../org/apache/hudi/HoodieSparkSqlWriter.scala | 18 ++--
.../src/main/scala/org/apache/hudi/Iterators.scala | 2 +-
.../org/apache/hudi/RecordLevelIndexSupport.scala | 4 +-
.../sql/catalyst/catalog/HoodieCatalogTable.scala | 2 +-
.../spark/sql/hudi/HoodieSqlCommonUtils.scala | 20 ++--
.../hudi/command/RepairHoodieTableCommand.scala | 16 +++-
.../sql/hudi/streaming/HoodieStreamSource.scala | 3 +-
.../org/apache/spark/sql/hudi/SparkHelpers.scala | 14 +--
.../RepairMigratePartitionMetaProcedure.scala | 3 +-
.../procedures/ShowInvalidParquetProcedure.scala | 6 +-
.../ShowMetadataTableFilesProcedure.scala | 4 +-
.../ShowMetadataTablePartitionsProcedure.scala | 7 +-
.../ShowMetadataTableStatsProcedure.scala | 4 +-
.../ValidateMetadataTableFilesProcedure.scala | 5 +-
.../org/apache/hudi/ColumnStatsIndexHelper.java | 18 ++--
.../org/apache/hudi/functional/TestBootstrap.java | 14 +--
.../TestSparkConsistentBucketClustering.java | 4 +-
.../row/TestHoodieInternalRowParquetWriter.java | 4 +-
.../org/apache/hudi/TestHoodieFileIndex.scala | 1 +
.../apache/hudi/functional/TestCOWDataSource.scala | 8 +-
.../hudi/functional/TestColumnStatsIndex.scala | 8 +-
.../TestMetadataTableWithSparkDataSource.scala | 13 ++-
.../hudi/dml/TestPartialUpdateForMergeInto.scala | 5 +
.../Spark30LegacyHoodieParquetFileFormat.scala | 4 +-
.../Spark31LegacyHoodieParquetFileFormat.scala | 4 +-
.../Spark32LegacyHoodieParquetFileFormat.scala | 4 +-
.../Spark33LegacyHoodieParquetFileFormat.scala | 4 +-
.../Spark34LegacyHoodieParquetFileFormat.scala | 5 +-
.../Spark35LegacyHoodieParquetFileFormat.scala | 4 +-
.../apache/hudi/sync/common/HoodieSyncClient.java | 1 +
.../hudi/sync/common/util/ManifestFileWriter.java | 2 +-
.../apache/hudi/utilities/HDFSParquetImporter.java | 3 +-
.../org/apache/hudi/utilities/HoodieCleaner.java | 3 +-
.../apache/hudi/utilities/HoodieClusteringJob.java | 4 +-
.../org/apache/hudi/utilities/HoodieCompactor.java | 4 +-
.../hudi/utilities/HoodieDataTableValidator.java | 5 +-
.../hudi/utilities/HoodieDropPartitionsTool.java | 4 +-
.../org/apache/hudi/utilities/HoodieIndexer.java | 4 +-
.../utilities/HoodieMetadataTableValidator.java | 18 ++--
.../apache/hudi/utilities/HoodieRepairTool.java | 4 +-
.../hudi/utilities/HoodieSnapshotCopier.java | 3 +-
.../hudi/utilities/HoodieSnapshotExporter.java | 7 +-
.../org/apache/hudi/utilities/TableSizeStats.java | 7 +-
.../org/apache/hudi/utilities/UtilHelpers.java | 8 +-
.../hudi/utilities/deltastreamer/DeltaSync.java | 3 +-
.../deltastreamer/HoodieDeltaStreamer.java | 5 +-
.../deltastreamer/HoodieDeltaStreamerMetrics.java | 10 +-
.../ingestion/HoodieIngestionMetrics.java | 12 +--
.../hudi/utilities/perf/TimelineServerPerf.java | 5 +-
.../utilities/sources/helpers/DFSPathSelector.java | 54 +++++------
.../sources/helpers/DatePartitionPathSelector.java | 46 ++++-----
.../utilities/streamer/BaseErrorTableWriter.java | 13 ++-
.../hudi/utilities/streamer/ErrorTableUtils.java | 5 +-
.../streamer/HoodieMultiTableStreamer.java | 5 +-
.../hudi/utilities/streamer/HoodieStreamer.java | 46 +++++----
.../utilities/streamer/HoodieStreamerMetrics.java | 12 +--
.../apache/hudi/utilities/streamer/StreamSync.java | 15 ++-
.../apache/hudi/utilities/TestHoodieIndexer.java | 4 +-
.../TestHoodieMetadataTableValidator.java | 2 +-
.../deltastreamer/TestHoodieDeltaStreamer.java | 4 +-
...TestHoodieDeltaStreamerSchemaEvolutionBase.java | 6 +-
.../utilities/sources/TestJsonKafkaSource.java | 2 +-
.../helpers/TestDFSPathSelectorCommonMethods.java | 19 ++--
.../helpers/TestDatePartitionPathSelector.java | 5 +-
.../utilities/streamer/TestErrorTableUtils.java | 70 ++++++++++++++
packaging/hudi-flink-bundle/pom.xml | 2 +-
packaging/hudi-metaserver-server-bundle/pom.xml | 2 +-
pom.xml | 2 +-
246 files changed, 1309 insertions(+), 1163 deletions(-)
rename hudi-hadoop-common/src/main/java/org/apache/hudi/io/{storage =>
hadoop}/HoodieHadoopIOFactory.java (81%)
create mode 100644
hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestErrorTableUtils.java
