This is an automated email from the ASF dual-hosted git repository.
vinoth pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git.
from c3e9243 [MINOR] Add maven profile to support skipping shade sources
jars (#2358)
new 298808b [HUDI-842] Implementation of HUDI RFC-15.
new 4e64226 [HUDI-1450] Use metadata table for listing in
HoodieROTablePathFilter (apache#2326)
new 2bd4a68 [HUDI-1469] Faster initialization of metadata table using
parallelized listing. (#2343)
new 4b94529 [HUDI-1325] [RFC-15] Merge updates of unsynced instants to
metadata table (apache#2342)
new 1a0579c [HUDI-1312] [RFC-15] Support for metadata listing for
snapshot queries through Hive/SparkSQL (#2366)
new 31e674e [HUDI-1504] Allow log files generated during restore/rollback
to be synced as well
The 6 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../apache/hudi/cli/commands/MetadataCommand.java | 226 ++++++
.../apache/hudi/client/AbstractHoodieClient.java | 5 +
.../hudi/client/AbstractHoodieWriteClient.java | 23 +-
.../apache/hudi/client/AsyncCleanerService.java | 7 +-
.../apache/hudi/config/HoodieMetricsConfig.java | 8 +
.../org/apache/hudi/config/HoodieWriteConfig.java | 61 ++
.../metadata/HoodieBackedTableMetadataWriter.java | 459 +++++++++++
.../hudi/metadata/HoodieTableMetadataWriter.java | 27 +-
.../java/org/apache/hudi/table/HoodieTable.java | 33 +-
.../hudi/table/HoodieTimelineArchiveLog.java | 14 +
.../hudi/table/action/clean/CleanPlanner.java | 6 +-
.../PartitionAwareClusteringPlanStrategy.java | 4 +-
.../AbstractMarkerBasedRollbackStrategy.java | 13 +-
.../hudi/table/action/rollback/RollbackUtils.java | 9 +-
.../action/savepoint/SavepointActionExecutor.java | 22 +-
.../apache/hudi/client/HoodieFlinkWriteClient.java | 4 +-
.../FlinkCopyOnWriteRollbackActionExecutor.java | 4 +-
.../table/upgrade/ZeroToOneUpgradeHandler.java | 2 +-
.../apache/hudi/client/HoodieJavaWriteClient.java | 8 +-
.../apache/hudi/client/SparkRDDWriteClient.java | 51 +-
.../index/bloom/SparkHoodieGlobalBloomIndex.java | 2 +-
.../index/simple/SparkHoodieGlobalSimpleIndex.java | 3 +-
.../SparkHoodieBackedTableMetadataWriter.java | 183 +++++
.../apache/hudi/metrics/DistributedRegistry.java | 107 +++
...rkInsertOverwriteTableCommitActionExecutor.java | 3 +-
.../HoodieSparkMergeOnReadTableCompactor.java | 2 +-
.../SparkCopyOnWriteRollbackActionExecutor.java | 4 +-
.../table/upgrade/ZeroToOneUpgradeHandler.java | 2 +-
.../org/apache/hudi/client/TestClientRollback.java | 4 +-
.../hudi/client/TestCompactionAdminClient.java | 6 +
.../hudi/metadata/TestHoodieBackedMetadata.java | 904 +++++++++++++++++++++
.../hudi/table/upgrade/TestUpgradeDowngrade.java | 6 +
.../hudi/testutils/HoodieClientTestHarness.java | 7 +-
hudi-common/pom.xml | 1 +
hudi-common/src/main/avro/HoodieMetadata.avsc | 56 ++
.../src/main/avro/HoodieRollbackMetadata.avsc | 10 +-
.../hudi/common/config/HoodieMetadataConfig.java | 151 ++++
.../java/org/apache/hudi/common/fs/FSUtils.java | 36 +-
.../hudi/common/fs/HoodieWrapperFileSystem.java | 297 ++++---
.../common/fs/SizeAwareFSDataOutputStream.java | 19 +-
.../hudi/common/fs/TimedFSDataInputStream.java | 79 ++
.../metrics/{Registry.java => LocalRegistry.java} | 64 +-
.../org/apache/hudi/common/metrics/Registry.java | 110 +--
.../table/log/AbstractHoodieLogRecordScanner.java | 16 +-
.../hudi/common/table/log/HoodieLogFileReader.java | 5 +-
.../table/log/HoodieMergedLogRecordScanner.java | 49 +-
.../table/log/block/HoodieHFileDataBlock.java | 3 +
.../table/timeline/HoodieActiveTimeline.java | 16 +-
.../table/timeline/TimelineMetadataUtils.java | 10 +-
.../table/view/AbstractTableFileSystemView.java | 12 +-
.../common/table/view/FileSystemViewManager.java | 17 +
.../table/view/HoodieMetadataException.java} | 13 +-
.../hudi/exception/TableNotFoundException.java | 14 +-
.../apache/hudi/metadata/BaseTableMetadata.java | 293 +++++++
.../metadata/FileSystemBackedTableMetadata.java | 69 ++
.../hudi/metadata/HoodieBackedTableMetadata.java | 248 ++++++
.../metadata/HoodieMetadataFileSystemView.java | 63 ++
.../HoodieMetadataMergedInstantRecordScanner.java | 115 +++
.../HoodieMetadataMergedLogRecordScanner.java | 75 ++
.../hudi/metadata/HoodieMetadataMetrics.java | 147 ++++
.../hudi/metadata/HoodieMetadataPayload.java | 233 ++++++
.../apache/hudi/metadata/HoodieTableMetadata.java | 93 +++
.../hudi/metadata/HoodieTableMetadataUtil.java | 318 ++++++++
.../hudi/metadata/MetadataPartitionType.java | 16 +-
.../apache/hudi/hadoop/HoodieHFileInputFormat.java | 12 +-
.../hudi/hadoop/HoodieParquetInputFormat.java | 13 +-
.../hudi/hadoop/HoodieROTablePathFilter.java | 17 +-
.../hudi/hadoop/utils/HoodieInputFormatUtils.java | 53 +-
.../utils/HoodieRealtimeInputFormatUtils.java | 20 +-
.../reader/DFSHoodieDatasetInputReader.java | 4 +-
.../org/apache/hudi/HoodieSparkSqlWriter.scala | 6 +-
.../scala/org/apache/hudi/HoodieWriterUtils.scala | 7 +
.../java/org/apache/hudi/client/TestBootstrap.java | 19 +-
.../apache/hudi/functional/TestCOWDataSource.scala | 17 +-
.../java/org/apache/hudi/dla/DLASyncConfig.java | 12 +
.../java/org/apache/hudi/dla/HoodieDLAClient.java | 3 +-
.../java/org/apache/hudi/hive/HiveSyncConfig.java | 48 +-
.../org/apache/hudi/hive/HoodieHiveClient.java | 2 +-
.../hudi/sync/common/AbstractSyncHoodieClient.java | 11 +-
.../hudi/utilities/HoodieSnapshotCopier.java | 16 +-
.../hudi/utilities/HoodieSnapshotExporter.java | 2 +-
.../hudi/utilities/perf/TimelineServerPerf.java | 10 +-
.../functional/TestHoodieSnapshotCopier.java | 8 +-
83 files changed, 4732 insertions(+), 415 deletions(-)
create mode 100644
hudi-cli/src/main/java/org/apache/hudi/cli/commands/MetadataCommand.java
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
copy
hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/clean/CleanPlanMigrator.java
=>
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataWriter.java
(52%)
create mode 100644
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieBackedTableMetadataWriter.java
create mode 100644
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metrics/DistributedRegistry.java
create mode 100644
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/metadata/TestHoodieBackedMetadata.java
create mode 100644 hudi-common/src/main/avro/HoodieMetadata.avsc
create mode 100644
hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java
create mode 100644
hudi-common/src/main/java/org/apache/hudi/common/fs/TimedFSDataInputStream.java
copy hudi-common/src/main/java/org/apache/hudi/common/metrics/{Registry.java
=> LocalRegistry.java} (58%)
copy
hudi-common/src/main/java/org/apache/hudi/{exception/HoodieIndexException.java
=> common/table/view/HoodieMetadataException.java} (78%)
create mode 100644
hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java
create mode 100644
hudi-common/src/main/java/org/apache/hudi/metadata/FileSystemBackedTableMetadata.java
create mode 100644
hudi-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadata.java
create mode 100644
hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataFileSystemView.java
create mode 100644
hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMergedInstantRecordScanner.java
create mode 100644
hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMergedLogRecordScanner.java
create mode 100644
hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMetrics.java
create mode 100644
hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataPayload.java
create mode 100644
hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadata.java
create mode 100644
hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java
copy
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/exception/HoodieClusteringException.java
=>
hudi-common/src/main/java/org/apache/hudi/metadata/MetadataPartitionType.java
(75%)
