This is an automated email from the ASF dual-hosted git repository.
yihua pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
from 4c7ac6112da [MINOR] Fix UT error in HUDI-6941 with stage task numbers
(#10554)
add ef70de2bba7 [HUDI-7335] Create hudi-hadoop-common for hadoop-specific
implementation (#10564)
No new revisions were added by this update.
Summary of changes:
.../hudi/aws/sync/AwsGlueCatalogSyncTool.java | 4 +-
.../main/java/org/apache/hudi/cli/HoodieCLI.java | 4 +-
.../hudi/cli/commands/ArchivedCommitsCommand.java | 10 +-
.../apache/hudi/cli/commands/ExportCommand.java | 7 +-
.../hudi/cli/commands/HoodieLogFileCommand.java | 2 +-
.../org/apache/hudi/cli/commands/SparkMain.java | 8 +-
.../java/org/apache/hudi/cli/utils/SparkUtil.java | 4 +-
.../hudi/cli/commands/TestCleansCommand.java | 4 +-
.../hudi/cli/commands/TestCompactionCommand.java | 8 +-
.../apache/hudi/cli/commands/TestDiffCommand.java | 4 +-
.../cli/commands/TestHoodieLogFileCommand.java | 4 +-
.../hudi/cli/commands/TestRepairsCommand.java | 3 +-
.../HoodieTestCommitMetadataGenerator.java | 3 +-
.../org/apache/hudi/client/BaseHoodieClient.java | 4 +-
.../client/embedded/EmbeddedTimelineService.java | 4 +-
.../lock/FileSystemBasedLockProvider.java | 6 +-
.../index/bucket/ConsistentBucketIndexUtils.java | 2 +-
.../metadata/HoodieBackedTableMetadataWriter.java | 4 +-
.../main/java/org/apache/hudi/metrics/Metrics.java | 4 +-
.../java/org/apache/hudi/table/HoodieTable.java | 4 +-
.../rollback/ListingBasedRollbackStrategy.java | 2 +-
.../hudi/table/marker/DirectWriteMarkers.java | 3 +-
.../SimpleDirectMarkerBasedDetectionStrategy.java | 2 +-
...nsactionDirectMarkerBasedDetectionStrategy.java | 2 +-
.../hudi/table/marker/WriteMarkersFactory.java | 6 +-
.../table/upgrade/TwoToOneDowngradeHandler.java | 3 +-
.../org/apache/hudi/HoodieTestCommitGenerator.java | 5 +-
.../FileSystemBasedLockProviderTestClass.java | 4 +-
.../hudi/table/marker/TestWriteMarkersFactory.java | 2 +-
.../testutils/HoodieFlinkClientTestHarness.java | 4 +-
.../testutils/HoodieJavaClientTestHarness.java | 5 +-
.../apache/hudi/client/SparkRDDWriteClient.java | 2 +-
.../hudi/client/utils/SparkPartitionUtils.java | 2 +-
.../hudi/io/storage/row/HoodieRowCreateHandle.java | 2 +-
.../SparkBootstrapCommitActionExecutor.java | 4 +-
.../hudi/BaseSparkInternalRowReaderContext.java | 4 +-
.../scala/org/apache/hudi/HoodieSparkUtils.scala | 9 +-
.../java/org/apache/hudi/client/TestMultiFS.java | 6 +-
.../hudi/client/TestUpdateSchemaEvolution.java | 4 +-
.../fs/TestHoodieSerializableFileStatus.java | 2 +-
.../apache/hudi/io/TestHoodieTimelineArchiver.java | 2 +-
.../apache/hudi/table/TestConsistencyGuard.java | 2 +-
.../table/action/compact/TestHoodieCompactor.java | 4 +-
.../hudi/table/marker/TestDirectWriteMarkers.java | 4 +-
.../TestTimelineServerBasedWriteMarkers.java | 4 +-
.../hudi/testutils/HoodieClientTestUtils.java | 4 +-
.../testutils/HoodieSparkClientTestHarness.java | 3 +-
.../SparkClientFunctionalTestHarness.java | 3 +-
hudi-common/pom.xml | 6 ++
.../org/apache/hudi/BaseHoodieTableFileIndex.java | 4 +-
.../common/config/DFSPropertiesConfiguration.java | 6 +-
.../DirectMarkerBasedDetectionStrategy.java | 2 +-
.../java/org/apache/hudi/common/fs/FSUtils.java | 72 +++----------
.../hudi/common/fs/FailSafeConsistencyGuard.java | 13 +--
.../hudi/common/fs/OptimisticConsistencyGuard.java | 2 +
.../org/apache/hudi/common/model/BaseFile.java | 2 +-
.../apache/hudi/common/model/HoodieBaseFile.java | 4 +-
.../hudi/common/model/HoodieCommitMetadata.java | 3 +-
.../apache/hudi/common/model/HoodieLogFile.java | 2 +-
.../hudi/common/table/HoodieTableMetaClient.java | 16 +--
.../table/log/AbstractHoodieLogRecordReader.java | 2 +-
.../table/log/BaseHoodieLogRecordReader.java | 2 +-
.../hudi/common/table/log/HoodieLogFileReader.java | 8 +-
.../hudi/common/table/log/LogReaderUtils.java | 4 +-
.../table/log/block/HoodieHFileDataBlock.java | 5 +-
.../common/table/timeline/dto/FilePathDTO.java | 2 +-
.../view/HoodieTablePreCommitFileSystemView.java | 2 +-
.../hudi/common/util/InternalSchemaCache.java | 4 +-
.../java/org/apache/hudi/common/util/OrcUtils.java | 4 +-
.../org/apache/hudi/common/util/ParquetUtils.java | 8 +-
.../io/FileBasedInternalSchemaStorageManager.java | 4 +-
.../hudi/io/storage/HoodieAvroHFileReader.java | 4 +-
.../hudi/io/storage/HoodieAvroHFileWriter.java | 2 +-
.../hudi/io/storage/HoodieAvroOrcWriter.java | 2 +-
.../hudi/io/storage/HoodieBaseParquetWriter.java | 10 +-
.../hudi/metadata/AbstractHoodieTableMetadata.java | 7 +-
.../metadata/FileSystemBackedTableMetadata.java | 5 +-
.../hudi/metadata/HoodieMetadataPayload.java | 4 +-
.../hudi/metadata/HoodieTableMetadataUtil.java | 7 +-
.../org/apache/hudi/common/fs/TestFSUtils.java | 5 +-
.../fs/TestFSUtilsWithRetryWrapperEnable.java | 13 ++-
.../common/fs/TestHoodieWrapperFileSystem.java | 5 +-
.../apache/hudi/common/fs/TestStorageSchemes.java | 3 +
.../common/functional/TestHoodieLogFormat.java | 25 ++---
.../table/timeline/TestHoodieActiveTimeline.java | 4 +-
.../table/view/TestHoodieTableFileSystemView.java | 2 +-
.../common/testutils/HoodieTestDataGenerator.java | 10 +-
.../hudi/common/testutils/HoodieTestUtils.java | 2 +-
.../testutils/reader/HoodieTestReaderContext.java | 4 +-
.../util/TestDFSPropertiesConfiguration.java | 4 +-
.../apache/hudi/common/util/TestMarkerUtils.java | 6 +-
.../io/storage/TestHoodieHFileReaderWriter.java | 6 +-
.../java/HoodieJavaWriteClientExample.java | 4 +-
.../examples/spark/HoodieWriteClientExample.java | 4 +-
.../hudi/schema/FilebasedSchemaProvider.java | 6 +-
.../apache/hudi/sink/meta/CkpMetadataFactory.java | 4 +-
.../sink/partitioner/profile/WriteProfiles.java | 4 +-
.../apache/hudi/table/catalog/HoodieCatalog.java | 4 +-
.../hudi/table/catalog/HoodieHiveCatalog.java | 4 +-
.../hudi/table/catalog/TableOptionProperties.java | 6 +-
.../apache/hudi/table/format/FilePathUtils.java | 6 +-
.../org/apache/hudi/table/format/FormatUtils.java | 8 +-
.../hudi/table/format/cdc/CdcInputFormat.java | 4 +-
.../table/format/cow/CopyOnWriteInputFormat.java | 12 +--
.../main/java/org/apache/hudi/util/ClientIds.java | 6 +-
.../java/org/apache/hudi/util/StreamerUtil.java | 7 +-
.../apache/hudi/util/ViewStorageProperties.java | 6 +-
.../sink/TestStreamWriteOperatorCoordinator.java | 6 +-
.../hudi/sink/bucket/ITTestBucketStreamWrite.java | 4 +-
.../bucket/ITTestConsistentBucketStreamWrite.java | 4 +-
.../sink/compact/ITTestHoodieFlinkCompactor.java | 2 +-
.../org/apache/hudi/sink/utils/TestWriteBase.java | 4 +-
.../hudi/table/catalog/TestHoodieHiveCatalog.java | 6 +-
.../org/apache/hudi/utils/TestStreamerUtil.java | 4 +-
.../pom.xml | 113 ++++++++-------------
.../hudi/hadoop}/fs/BoundedFsDataInputStream.java | 16 +--
.../org/apache/hudi/hadoop/fs}/CachingPath.java | 15 +--
.../apache/hudi/hadoop}/fs/ConsistencyGuard.java | 15 +--
.../org/apache/hudi/hadoop/fs/HadoopFSUtils.java | 85 ++++++++++++++++
.../hadoop}/fs/HoodieRetryWrapperFileSystem.java | 15 +--
.../hadoop}/fs/HoodieSerializableFileStatus.java | 15 +--
.../hudi/hadoop}/fs/HoodieWrapperFileSystem.java | 28 ++---
.../hudi/hadoop}/fs/NoOpConsistencyGuard.java | 15 +--
.../hadoop}/fs/SchemeAwareFSDataInputStream.java | 15 +--
.../apache/hudi/hadoop/fs}/SerializablePath.java | 15 +--
.../hadoop}/fs/SizeAwareFSDataOutputStream.java | 15 +--
.../hudi/hadoop}/fs/TimedFSDataInputStream.java | 15 +--
.../realtime/HoodieMergeOnReadSnapshotReader.java | 4 +-
.../realtime/RealtimeCompactedRecordReader.java | 4 +-
.../apache/hudi/hadoop/realtime/RealtimeSplit.java | 2 +-
.../realtime/RealtimeUnmergedRecordReader.java | 4 +-
.../TestHoodieMergeOnReadSnapshotReader.java | 2 +-
.../realtime/TestHoodieRealtimeRecordReader.java | 3 +-
.../hudi/integ/testsuite/HoodieTestSuiteJob.java | 8 +-
.../SparkDataSourceContinuousIngestTool.java | 4 +-
.../dag/nodes/ValidateAsyncOperations.java | 4 +-
.../integ/testsuite/generator/DeltaGenerator.java | 4 +-
.../testsuite/reader/DFSDeltaInputReader.java | 5 +-
.../testsuite/writer/AvroFileDeltaInputWriter.java | 12 ++-
.../nodes/spark/sql/SparkSqlCreateTableNode.scala | 4 +-
.../TestDFSHoodieTestSuiteWriterAdapter.java | 4 +-
.../integ/testsuite/TestFileDeltaInputWriter.java | 5 +-
.../reader/TestDFSAvroDeltaInputReader.java | 5 +-
.../org/apache/hudi/common/metrics/Counter.java | 0
.../apache/hudi/common/metrics/LocalRegistry.java | 0
.../org/apache/hudi/common/metrics/Metric.java | 0
.../org/apache/hudi/common/metrics/Registry.java | 0
.../org/apache/hudi/common/util/HoodieTimer.java | 0
.../apache/hudi/common/util/ReflectionUtils.java | 0
.../org/apache/hudi/common/util/RetryHelper.java | 13 +--
.../org/apache/hudi/common/util/StringUtils.java | 13 +--
.../apache/hudi/common/util/ValidationUtils.java | 0
.../org/apache/hudi/storage}/StorageSchemes.java | 15 +--
.../writers/KafkaConnectTransactionServices.java | 4 +-
.../main/scala/org/apache/hudi/DefaultSource.scala | 6 +-
.../scala/org/apache/hudi/FileSystemRelation.scala | 2 +-
.../scala/org/apache/hudi/HoodieBaseRelation.scala | 24 +++--
.../src/main/scala/org/apache/hudi/Iterators.scala | 7 +-
.../datasources/HoodieMultipleBaseFileFormat.scala | 2 +-
...odieFileGroupReaderBasedParquetFileFormat.scala | 2 +-
.../spark/sql/hudi/HoodieSqlCommonUtils.scala | 2 +-
.../AlterHoodieTableAddPartitionCommand.scala | 2 +-
.../sql/hudi/command/DropHoodieTableCommand.scala | 4 +-
.../hudi/command/TruncateHoodieTableCommand.scala | 4 +-
.../apache/hudi/cli/HDFSParquetImporterUtils.java | 4 +-
.../org/apache/spark/sql/hudi/DedupeSparkJob.scala | 2 +-
.../procedures/ExportInstantsProcedure.scala | 8 +-
.../RepairAddpartitionmetaProcedure.scala | 2 +-
.../procedures/RepairDeduplicateProcedure.scala | 8 +-
.../RepairOverwriteHoodiePropsProcedure.scala | 8 +-
.../command/procedures/RunBootstrapProcedure.scala | 7 +-
.../ShowColumnStatsOverlapProcedure.scala | 2 +-
.../procedures/ShowFileSystemViewProcedure.scala | 9 +-
.../procedures/ShowFsPathDetailProcedure.scala | 6 +-
.../ShowHoodieLogFileMetadataProcedure.scala | 2 +-
.../procedures/ShowInvalidParquetProcedure.scala | 4 +-
.../procedures/StatsFileSizeProcedure.scala | 2 +-
.../org/apache/hudi/functional/TestBootstrap.java | 3 +-
.../read/TestHoodieFileGroupReaderOnSpark.scala | 2 +-
.../apache/hudi/functional/TestCOWDataSource.scala | 2 +-
.../hudi/functional/TestCOWDataSourceStorage.scala | 8 +-
.../functional/TestColumnStatsIndexWithSQL.scala | 2 +-
.../functional/TestDataSourceForBootstrap.scala | 6 +-
.../hudi/functional/TestMORDataSourceStorage.scala | 8 +-
.../hudi/functional/TestSparkDataSource.scala | 8 +-
.../hudi/functional/TestSparkSqlCoreFlow.scala | 9 +-
.../spark/sql/hudi/HoodieSparkSqlTestBase.scala | 6 +-
.../org/apache/spark/sql/hudi/TestDropTable.scala | 10 +-
.../apache/spark/sql/hudi/TestMergeIntoTable.scala | 6 +-
.../procedure/TestHdfsParquetImportProcedure.scala | 9 +-
.../sql/hudi/procedure/TestRepairsProcedure.scala | 7 +-
.../TestShowInvalidParquetProcedure.scala | 6 +-
.../org/apache/hudi/hive/ddl/HMSDDLExecutor.java | 2 +-
.../hudi/hive/ddl/QueryBasedDDLExecutor.java | 2 +-
.../apache/hudi/sync/common/HoodieSyncClient.java | 2 +-
.../apache/hudi/sync/common/HoodieSyncConfig.java | 4 +-
.../hudi/sync/common/util/TestSyncUtilHelpers.java | 4 +-
.../hudi/timeline/service/TimelineService.java | 8 +-
.../apache/hudi/utilities/HDFSParquetImporter.java | 4 +-
.../hudi/utilities/HoodieCompactionAdminTool.java | 4 +-
.../org/apache/hudi/utilities/HoodieCompactor.java | 4 +-
.../hudi/utilities/HoodieDropPartitionsTool.java | 4 +-
.../utilities/HoodieMetadataTableValidator.java | 2 +-
.../apache/hudi/utilities/HoodieRepairTool.java | 9 +-
.../hudi/utilities/HoodieSnapshotCopier.java | 7 +-
.../hudi/utilities/HoodieSnapshotExporter.java | 15 +--
.../org/apache/hudi/utilities/TableSizeStats.java | 4 +-
.../hudi/utilities/perf/TimelineServerPerf.java | 5 +-
.../utilities/schema/FilebasedSchemaProvider.java | 4 +-
.../hudi/utilities/sources/HiveIncrPullSource.java | 4 +-
.../hudi/utilities/sources/SqlFileBasedSource.java | 4 +-
.../helpers/CloudObjectsSelectorCommon.java | 4 +-
.../utilities/sources/helpers/DFSPathSelector.java | 4 +-
.../streamer/HoodieMultiTableStreamer.java | 4 +-
.../hudi/utilities/streamer/HoodieStreamer.java | 6 +-
.../utilities/streamer/SparkSampleWritesUtils.java | 8 +-
.../apache/hudi/utilities/streamer/StreamSync.java | 4 +-
.../transform/SqlFileBasedTransformer.java | 4 +-
.../deltastreamer/TestHoodieDeltaStreamer.java | 6 +-
.../functional/TestHoodieSnapshotCopier.java | 3 +-
.../functional/TestHoodieSnapshotExporter.java | 4 +-
.../sources/helpers/TestSanitizationUtils.java | 4 +-
packaging/hudi-aws-bundle/pom.xml | 1 +
packaging/hudi-datahub-sync-bundle/pom.xml | 1 +
packaging/hudi-flink-bundle/pom.xml | 1 +
packaging/hudi-gcp-bundle/pom.xml | 1 +
packaging/hudi-hadoop-mr-bundle/pom.xml | 1 +
packaging/hudi-hive-sync-bundle/pom.xml | 1 +
packaging/hudi-integ-test-bundle/pom.xml | 1 +
packaging/hudi-kafka-connect-bundle/pom.xml | 1 +
packaging/hudi-presto-bundle/pom.xml | 1 +
packaging/hudi-spark-bundle/pom.xml | 1 +
packaging/hudi-timeline-server-bundle/pom.xml | 1 +
packaging/hudi-trino-bundle/pom.xml | 1 +
packaging/hudi-utilities-bundle/pom.xml | 1 +
packaging/hudi-utilities-slim-bundle/pom.xml | 1 +
pom.xml | 1 +
237 files changed, 817 insertions(+), 662 deletions(-)
copy {hudi-sync/hudi-datahub-sync => hudi-hadoop-common}/pom.xml (60%)
rename {hudi-common/src/main/java/org/apache/hudi/common =>
hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop}/fs/BoundedFsDataInputStream.java
(81%)
rename {hudi-common/src/main/java/org/apache/hudi/hadoop =>
hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop/fs}/CachingPath.java
(93%)
rename {hudi-common/src/main/java/org/apache/hudi/common =>
hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop}/fs/ConsistencyGuard.java
(85%)
create mode 100644
hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop/fs/HadoopFSUtils.java
rename {hudi-common/src/main/java/org/apache/hudi/common =>
hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop}/fs/HoodieRetryWrapperFileSystem.java
(97%)
rename {hudi-common/src/main/java/org/apache/hudi/common =>
hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop}/fs/HoodieSerializableFileStatus.java
(90%)
rename {hudi-common/src/main/java/org/apache/hudi/common =>
hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop}/fs/HoodieWrapperFileSystem.java
(97%)
rename {hudi-common/src/main/java/org/apache/hudi/common =>
hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop}/fs/NoOpConsistencyGuard.java
(71%)
rename {hudi-common/src/main/java/org/apache/hudi/common =>
hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop}/fs/SchemeAwareFSDataInputStream.java
(75%)
rename {hudi-common/src/main/java/org/apache/hudi/hadoop =>
hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop/fs}/SerializablePath.java
(78%)
rename {hudi-common/src/main/java/org/apache/hudi/common =>
hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop}/fs/SizeAwareFSDataOutputStream.java
(86%)
rename {hudi-common/src/main/java/org/apache/hudi/common =>
hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop}/fs/TimedFSDataInputStream.java
(86%)
rename {hudi-common =>
hudi-io}/src/main/java/org/apache/hudi/common/metrics/Counter.java (100%)
rename {hudi-common =>
hudi-io}/src/main/java/org/apache/hudi/common/metrics/LocalRegistry.java (100%)
rename {hudi-common =>
hudi-io}/src/main/java/org/apache/hudi/common/metrics/Metric.java (100%)
rename {hudi-common =>
hudi-io}/src/main/java/org/apache/hudi/common/metrics/Registry.java (100%)
rename {hudi-common =>
hudi-io}/src/main/java/org/apache/hudi/common/util/HoodieTimer.java (100%)
rename {hudi-common =>
hudi-io}/src/main/java/org/apache/hudi/common/util/ReflectionUtils.java (100%)
rename {hudi-common =>
hudi-io}/src/main/java/org/apache/hudi/common/util/RetryHelper.java (92%)
rename {hudi-common =>
hudi-io}/src/main/java/org/apache/hudi/common/util/StringUtils.java (93%)
rename {hudi-common =>
hudi-io}/src/main/java/org/apache/hudi/common/util/ValidationUtils.java (100%)
rename {hudi-common/src/main/java/org/apache/hudi/common/fs =>
hudi-io/src/main/java/org/apache/hudi/storage}/StorageSchemes.java (90%)
