This is an automated email from the ASF dual-hosted git repository.
vhs pushed a change to branch release-1.0.2
in repository https://gitbox.apache.org/repos/asf/hudi.git
from c05acf387e8 [MINOR] Update release version to reflect published
version 1.0.1
add 0f13d027fce [HUDI-8953] Update bundle validation workflow on a release
candidate (#12755)
add f5f09c33368 [HUDI-8767] Add bundle validation for release artifacts in
Maven central (#12488)
add 4197ab85150 [HUDI-8950] Explicitly set base path in sync config #12754
add 9c85c2c12a2 [HUDI-8278] Add log message about different primary keys
for HoodieTableFactory (#12766)
add e47f0bb8f8a [HUDI-8951] Disabled flaky
`TestHoodieDeltaStreamer::testHoodieIndexerExecutionAfterClustering` (#12769)
add 9411bb6b7b9 [MINOR] Fix code issues reported by SonarQube (#12767)
add a2e945ef2ae [MINOR] Move field ordering lookup outside of loop for
proto conversion (#12271)
add e7b28c6a8ea [HUDI-8966] Claim RFC-88 for New
Schema/DataType/Expression Abstractions (#12791)
add 39c91dc26ce [HUDI-8958] Enable record index validation by default
(#12785)
add 5eb23e95b56 [MINOR] Fix jenkins compile issue (#12810)
add 64bb2c33a4c [HUDI-8989] Fixing Compaction scheduling for tbl v6
(#12805)
add 31ca2cbbc7f [MINOR] Updating doap for release 1.0.1 (#12816)
add c8c9abb899d [HUDI-8569] Fix Insert overwrite / update MOR with global
index does not work (#12819)
add 7216fcf4488 [HUDI-8967] Add a config to fail job if duplicate data
files detected during reconcileAgainstMarkers (#12792)
add c4ba42182f3 [HUDI-8965] Fix Hoodie Hive Sync Tool to Throw Exceptions
on Sync Failures (#12790)
add 41edbfc2974 [HUDI-8959] HoodieMetadataTableValidator can skip
validation if data table is not found (#12775)
add fc7b3c49559 [HUDI-8998] Improve handling of zero scale decimals in
MercifulJsonConverter (#12822)
add 0387e8605ac [HUDI-8922] Avoid scheduling existing rollbacks (#12715)
add 151fbddc34f [HUDI-8941] Stop spark context with proper exit code in
HoodieStreamer (#12742)
add de87824f7f7 [HUDI-8798] Validate Metadata Table Enabled during
Archival (#12549)
add 339079bd957 [HUDI-5612] Add support for using metadata table with
spillable map based file system views (#12110)
add f7737e4b846 [HUDI-8885] Improve the efficiency of initializing the
metadata table and table services (#12665)
add a3dbee527ff [HUDI-8971] Fixing metadata metrics to honor metrics
prefix (#12797)
add a3fd66d1d48 [HUDI-8992] Initial changes to allow us to use streams
instead of byte arrays (#12814)
add bc5cac6c4e9 [HUDI-9009] Fix potential race condition when listing
files by leveraging exception type #12834
add a57f19791cb [HUDI-8701] prevent lock metrics failure from failing
process (#12458)
add 34dcd5068bc [MINOR] Register more classes with Kryo (#12268)
add eb96dec4068 [HUDI-8972] Fixing heart beats for failed writes in
HoodieStreamer (#12802)
add 35b2f578759 [MINOR] Fix skew in clustering operator (#12765)
add f8cdfe08257 [MINOR] Rebalance Azure CI jobs (2025-02-13) (#12840)
add 1b7304b1b7b [HUDI-8985] Fix avg record size estimator to avoid parsing
all commit metadata of active timeline (#12803)
add 21d0ed46ef8 [HUDI-7596] Enable Jacoco code coverage report across
multiple modules (#12842)
add 632cd5b5f68 [HUDI-8962] Adding more logs during indexing (#12787)
add 9da6a935e12 [HUDI-7222] Fix Scala style check (#12837)
add 2800d550c47 [HUDI-9038] Enable Codecov tracking and comment on code
coverage (#12845)
add 32172275203 [MINOR] Rebalance Azure CI jobs (2025-02-14) (#12844)
add 659c8f8a5f5 [HUDI-9039] Run do init table transaction only when
required (#12847)
add 0e7d23d15ae [HUDI-7624] Fixing source read and index tagging duration
(#12789)
add b521d5ca721 [HUDI-8882] Support Compaction/Rollback/Clean Timeline
Instant Metrics in HoodieMetrics (#12681)
add 8de749ade2b [HUDI-8126] Persist sourceRdd to optimise writeStatus DAG
for error table (#11844)
add 90a41f8d96b [HUDI-8883] Add ability to configure
HoodieCompactionPlanGenerator using reflection (#12664)
add d844ceb55c6 [HUDI-9045] Spillable Map operational improvements (#12850)
add 26690f58e83 [HUDI-8442] Reduce timeline loads while rolling back
failed writes (#12164)
add fc5f16ddede [HUDI-9016] Fix the HoodieCompactor schedule parameter
alias name #12836
add 4818545e18a [MINOR] Increase timeout of hudi-utilities tests in Azure
CI (#12862)
add c2ea6423ec6 [HUDI-8988] Implement retry logic for all HTTP calls to
timeline server including remote file system view and markers (#12804)
add cf9e9aac7d2 [HUDI-9059] Fix ordering of getInstantTimes in
CompletionTimeQueryViewV2 (#12858)
add b4ece02e561 [HUDI-9060] Remove validations for clustering metadata
(#12860)
add 8362c715b67 [HUDI-8126] Support proto messages for spark kryo
serializer excluding DynamicMessages (#12052)
add 5649baae374 [HUDI-8786] IntelliJ automatic code cleanup tool (#12524)
add 92eae7d71d7 [HUDI-8868] Use external spillable map for
cachedAllInputFileSlices in BaseHoodieTableFileIndex (#12647)
add b1721e19897 [HUDI-8126] Use union to parallelize data and error table
writes (#12813)
add 4a5e405dcdf [HUDI-8501] Improve SizeAwareDataInputStream to implement
idempotent (#12231)
add b45fb40a4d3 [HUDI-8378] Fix Avro schema deserializer failing with
schema evolution (#12111)
add c376d7207bf [HUDI-8955] Resolve Kafka beginning offsets with retention
to prevent OffsetOutOfRange exception (#12762)
add 17bfb1cc830 [MINOR] Add StorageSchemes for Aliyun Apsara File Storage
for HDFS (#12872)
add 71a30ca0ef5 [HUDI-9070] Fix Hudi cli bundle script to use
HUDI_CONF_DIR environment variable (#12876)
add cd61b4e0496 [HUDI-8901] Fix Timeline Server to process requests from
multiple storage lakes (#12696)
add 1310170aae9 [hudi-9041] Send commit ack event when reusing current
instant (#12849)
add 2f25a0a25f0 [HUDI-9061] JSON to AVRO schema converter (#12864)
add 456256f99f5 [HUDI-9071] MDT validator can configure if log truncation
applies (#12877)
add 935e5105712 [HUDI-9040] Set the correct table path when renaming
tables (#12848)
add 36dabc6ce7a [HUDI-8954] Reconstruct writer schema while clustering by
spark row writer (#12758)
add 83cdf4c7cdc [HUDI-9072] Support decimal in JsonKafkaSource (#12879)
add d5e783a9839 [HUDI-8992] Fix serde issues in commit metadata (#12829)
add 91c92dc686f [HUDI-8940] Fix Bloom Index Partitioner to distribute keys
uniformly across partitions (#12741)
add 21d945fb756 [HUDI-8577] Use config default for query type in default
source in Spark (#12435)
add 642927e58e2 [HUDI-7985] Add more test cases around timestamp and
decimal formats in Json Avro converter (#11629)
add 3a9fa5ec7f7 [HUDI-9065] Adding new filters to ArchivedTimelineV1
(#12869)
add ff2a76a57f7 [HUDI-8486] Enforce data type match for required columns
in Spark SQL MERGE INTO (#12798)
add a341dc77095 [HUDI-8486] Add column mismatch test coverage (#12800)
add 2532445d5c1 [HUDI-9086] Disabling failing test to unblock master
(#12890)
add fb044e89198 [HUDI-9093] Fix the HoodieClusteringJob duplicate
parameter alias (#12897)
add ec5cc0bf976 [HUDI-8992] Deprecate all byte array usage in metadata
deserialization path (#12826)
add a97a89fb5c9 [HUDI-9079] Log the exception message properly to handle
the 'default' partition value migration steps (#12911)
add 222363166c5 [HUDI-9092] Deprecate byte array of serialization of
instants (#12900)
add 6b2520d055e [HUDI-8826] Extend MIT partial update test (#12915)
add 3c45727564d [HUDI-8745] Add tests for record index and secondary index
with insert dups policy (#12914)
add 9f99365b4e2 [HUDI-9067] Fixing num spark tasks for clean action
(#12874)
add c32ac40e4fe [HUDI-5387] Add bundle validation for hudi-cli-bundle
(#12882)
add 53dd14fbd93 [HUDI-8219] Add concurrent schema evolution conflict
detection (#12781)
add 757d01c31e0 [HUDI-9030] Compatibility fixes for table version 6 with
1.0 (#12888)
add 5e6ff97260f [MINOR] Fix generating file id with wrong bucket index
(#12917)
add 499ae7dbc29 [HUDI-9115] Deprecate Hudi CLI script (#12916)
add 36e41591dec [MINOR] Add retries and logs to merging Jacoco execution
data files (#12921)
add 99b0c6bf5ad [HUDI-9057] Fixing ClassNotFound issue w/
ProtoBufSchemaProvider (#12913)
add ae3d58ee588 [HUDI-9086] Re-enable flaky tests and fixing spark context
not shutting down (#12909)
add c93c32c76ed [MINOR] Skip merging Jacoco execution data file from Azure
Job 1 FT (#12942)
add 78e22fa8559 [MINOR] Fix cleaner config compatibility with older
releases (#12943)
add 021053cae17 [HUDI-8768] Support bloom filter options when creating
expr index using bloom filter (#12919)
add 9cb777f1fd6 [HUDI-9125] Pass compaction/merge related props to
HoodieBaseFileGroupRecordBuffer (#12925)
add 8bdde9b3e1c [HUDI-9132] Avoid empty string row key for delete and
update operations (#12929)
add 9f7a584120a [HUDI-9068] Fix cloudwatch metrics (#12873)
add c9b4bf09f7f [HUDI-9141] Handle the case that dbName is empty in
HoodieFileIndex (#12885)
add 3cf7daa70bd [HUDI-9152] Improve read/write/compaction performance by
reusing avro schema (#12949)
add 444a47ed423 [MINOR] Remove storage instance variable from
HoodieIngestionMetrics (#12937)
add 6b8ba985798 [HUDI-7803] Fix bundle validation on Flink 1.18 (#12959)
add 5e18470cc94 Fix hadoop deps from hudi-common (#12965)
add 152976492aa [MINOR] Improve error logs on mdt validation job (#12973)
add 563e7d02b91 [HUDI-9166] Static cleanup method that doesn't hold
references to DiskMap instances (#12956)
add d11b67400cf [HUDI-9166] Introduce schema pruning for delete-record
(#12961)
add 2b77eae3d7e [HUDI-9173] Fix issue with inflight compaction and global
index lookup (#12976)
add 1d63f5a0b06 [HUDI-9186] Remove tableState from
HoodieFileGroupReaderBasedParquetFileFormat constructor (#12981)
add a00fa5f7afc [HUDI-9168] Refactor hudi-client-common to not import
hudi-aws (#12969)
add b4b9c74b75a [HUDI-8345] Delete partition stats index for a partition
that is deleted (#12953)
add 450ec195922 [HUDI-7375] Enable
testLogReaderWithDifferentVersionsOfDeleteBlocks (#12668)
add d268f8c77d6 [HUDI-9175] Remove the unnecessary MDT metadata check for
col_stats index config update (#12977)
add 982340a223c [HUDI-9127] Fixing completion time generation to honor
timezone from table config (#12926)
add 951e55c5293 [HUDI-9170] Fixing schema projection with file group
reader (#12970)
add 69081732a12 [HUDI-9083] Fixing flakiness with multi writer test
(#12987)
add 87dfa0225ed [HUDI-9013] Add backwards compatible MDT writer support
and reader support with tbl v6 (#12948)
add 72287ca4674 [HUDI-9120] Fix merge mode inference for table version 6
in file group reader (#12991)
add 93f798e319f [HUDI-9120] Remove HUDI-9130 code changes relating to FGR
(#12935)
add bdd946c3c4f [HUDI-9198] Support rate limit for append mode (#12999)
add 89f79c8c190 [HUDI-9207] Spark Insert Overwrite Support Row Writer
(#13014)
add 350050461ff [HUDI-9022] Handle records with custom delete markers in
FG reader (#12843)
add b13299de034 [HUDI-9215] Set partitionColumnsWithKeyGenerator based on
table version (#13025)
add 12e2ff01010 [HUDI-9211] Fix bug with config in DataHubSyncTool (#13018)
add aed1b81cf21 [HUDI-9216] Fallback to spark.catalog.currentDatabase if
tableConfig.databaseName is null or empty (#13026)
add 022cbbec0eb [HUDI-9220] Cannot find write operation type if run inline
log compaction (#13029)
add b57dbfa2c8e [HUDI-7037] Fix colstats reading for Decimal field (#12993)
add 6c7f09d0516 [HUDI-9212] Fix metadata compaction failure post downgrade
(#13022)
add d2971a22c96 [MINOR] Fix flaky test testCreateNewInstantTimes (#13032)
add af3219be16f [HUDI-9216] Ignore validation of empty databaseName in
HoodieTableMetaClient (#13033)
add c6b5af2b1cf [MINOR] Follow up HUDI-8803, optimize vectorized reader by
cache 'batchIdxField' (#13023)
add b2af5e40b82 [HUDI-9088] Fix unnecessary scanning of target table in
MERGE INTO on Spark (#12934)
add 311fff14a6d [MINOR] Use configutils to handle defaults for all zk lock
provider configs (#13039)
add 8c2822849e6 [HUDI-9227] Fix bulk insert overwrite after a failed
insert overwrite (#13041)
add 480192df898 [HUDI-9233] Handle empty table edge case for col stats
initialization (#13045)
add d5db06e9d20 [HUDI-9167] Remove redundant classes in
hudi-utilities-slim-bundle (#12962)
add c51e5b43800 [HUDI-9238] Fix bug with HudiIncrSource when source is
table version 6 (#13048)
add d51851a842b [HUDI-9239] Fix the bug of Spark Cache not releasing
cleanly (#13050)
add 3788a024c68 [HUDI-9133] Fallback to legacy schema fetching in case of
IllegalAccessError from schema registry client (#13030)
add 80dedd4e0ab [HUDI-9231] Show complete DAG for one single query in
spark web ui while inserting into hudi table (#13044)
add 872e35432ef [HUDI-9236] Handle markers for log files in table version
6 (#13007)
add 5ea5d4cf5ce [HUDI-8409] Fixing merge mode config during upgrade and
downgrade from version 7 to 8 and back (#13046)
add fc77caeff3f [HUDI-8635] Support numWrites metric for compaction
(#13047)
add 0370b2177a1 [MINOR]Fix typo and Add implementation class name in
interface method of HoodieRecordMerger (#13059)
add ffa7706d5dc [HUDI-9206] Support reading inflight instants with
HoodieLogRecordReader (#13010)
add ffbd0f5f022 [HUDI-9252] BaseHoodieWriteClient should only check for
upgrade (#13073)
add 6bce0272e3a [MINOR] Remove warning around table version six (#13080)
add 5135d2f5dd1 [MINOR] Fixing master for build failure (#13085)
add 5a3f33e2f25 [MINOR] Fix NumberFormatException while updating metrics
for MDT in table version 6 (#13056)
add 6e1a7b29cee [HUDI-9255] Fix inferring correct merge behavior for few
scenarios (#13079)
add 8fce95c1b53 [HUDI-9259] Fixing marker reconciliation for failures
during deleting additional files (#13088)
add 56fef2ec15d [HUDI-9258] Disable partial update when global index is
used (#13086)
add 3ef8ba3adb9 [HUDI-9247] Flip default value of reuse of TimeGenerator
instance (#13077)
add 8f3afe7a6fd [MINOR] Upgrade jetty version to 9.4.57.v20241219 to fix
CVE-2024-8184 (#13058)
add d0adc56df2c [MINOR] Update javadocs in MergeIntoHoodieTableCommand
(#13093)
add 666b3218496 [MINOR] Fix logging issue with
EightToSevenDowngradeHandler (#13096)
add 02b484c7625 [HUDI-9155] Add scaffolding for lock provider using
conditional writes (#12954)
add fd7149b914a [HUDI-9156] Add lock provider heartbeat manager (#12958)
add 6a9233d9652 [HUDI-9258] Disable partial update for CUSTOM merge mode
(#13092)
No new revisions were added by this update.
Summary of changes:
.codecov.yml | 15 +-
.github/workflows/bot.yml | 2 +-
...alidation.yml => maven_artifact_validation.yml} | 26 +-
.github/workflows/release_candidate_validation.yml | 14 +-
.gitignore | 4 +-
azure-pipelines-20230430.yml | 368 ++++++---
doap_HUDI.rdf | 7 +
hudi-aws/pom.xml | 5 +
.../cloudwatch/CloudWatchMetricsReporter.java | 3 +-
.../cloudwatch/CloudWatchReporter.java | 15 +-
.../hudi/aws/sync/AWSGlueCatalogSyncClient.java | 9 +-
...amoDBBasedImplicitPartitionKeyLockProvider.java | 2 +-
.../java/org/apache/hudi/aws/utils/S3Utils.java | 27 -
.../cloudwatch/TestCloudWatchMetricsReporter.java | 61 +-
.../cloudwatch/TestCloudWatchReporter.java | 30 +-
.../org/apache/hudi/aws/utils/TestS3Utils.java | 59 --
hudi-cli/hudi-cli.sh | 31 -
.../apache/hudi/cli/commands/CleansCommand.java | 10 +-
.../apache/hudi/cli/commands/CommitsCommand.java | 64 +-
.../hudi/cli/commands/CompactionCommand.java | 24 +-
.../org/apache/hudi/cli/commands/DiffCommand.java | 18 +-
.../apache/hudi/cli/commands/ExportCommand.java | 10 +-
.../hudi/cli/commands/FileSystemViewCommand.java | 5 +-
.../apache/hudi/cli/commands/MetadataCommand.java | 10 +-
.../apache/hudi/cli/commands/RestoresCommand.java | 12 +-
.../apache/hudi/cli/commands/RollbacksCommand.java | 9 +-
.../org/apache/hudi/cli/commands/StatsCommand.java | 9 +-
.../apache/hudi/cli/commands/TimelineCommand.java | 15 +-
.../java/org/apache/hudi/cli/utils/CLIUtils.java | 6 +-
.../java/org/apache/hudi/cli/utils/CommitUtil.java | 5 +-
.../apache/hudi/cli/utils/InputStreamConsumer.java | 2 +-
.../hudi/cli/utils/SparkTempViewProvider.java | 4 +-
.../hudi/cli/commands/TestCleansCommand.java | 4 +-
.../hudi/cli/commands/TestCompactionCommand.java | 10 +-
.../cli/commands/TestFileSystemViewCommand.java | 4 +-
.../hudi/cli/commands/TestRestoresCommand.java | 7 +-
.../hudi/cli/commands/TestRollbacksCommand.java | 7 +-
.../HoodieTestCommitMetadataGenerator.java | 11 +-
.../cli/testutils/HoodieTestCommitUtilities.java | 4 +-
hudi-client/hudi-client-common/pom.xml | 12 +-
.../org/apache/hudi/async/AsyncCompactService.java | 2 +-
.../util/HoodieWriteCommitCallbackUtil.java | 4 +-
.../hudi/client/BaseHoodieTableServiceClient.java | 68 +-
.../apache/hudi/client/BaseHoodieWriteClient.java | 67 +-
.../apache/hudi/client/CommitMetadataResolver.java | 48 ++
.../hudi/client/CommitMetadataResolverFactory.java | 43 +
.../apache/hudi/client/CompactionAdminClient.java | 20 +-
.../hudi/client/HoodieColumnStatsIndexUtils.java | 47 +-
.../client/MarkerBasedCommitMetadataResolver.java | 257 ++++++
.../NoOpCommitMetadataResolver.java} | 33 +-
.../java/org/apache/hudi/client/WriteStatus.java | 16 +-
.../client/embedded/EmbeddedTimelineService.java | 7 +-
.../timeline/versioning/v1/TimelineArchiverV1.java | 2 +-
.../timeline/versioning/v2/LSMTimelineWriter.java | 5 +-
.../timeline/versioning/v2/TimelineArchiverV2.java | 2 +-
...ConcurrentSchemaEvolutionTableSchemaGetter.java | 276 +++++++
.../SchemaConflictResolutionStrategy.java | 76 ++
.../SimpleSchemaConflictResolutionStrategy.java | 176 +++++
.../lock/BaseZookeeperBasedLockProvider.java | 27 +-
.../lock/FileSystemBasedLockProvider.java | 4 +-
.../hudi/client/transaction/lock/StorageLock.java | 63 ++
...ZookeeperBasedImplicitBasePathLockProvider.java | 2 +-
.../lock/metrics/HoodieLockMetrics.java | 22 +-
.../transaction/lock/models/HeartbeatManager.java | 49 ++
.../transaction/lock/models/LockGetResult.java} | 23 +-
.../lock/models/LockProviderHeartbeatManager.java | 351 ++++++++
.../transaction/lock/models/LockUpdateResult.java} | 21 +-
.../transaction/lock/models/StorageLockData.java | 71 ++
.../transaction/lock/models/StorageLockFile.java | 133 ++++
.../utils/LegacyArchivedMetaEntryReader.java | 9 +-
.../hudi/client/utils/MetadataTableUtils.java | 45 --
.../apache/hudi/client/utils/TransactionUtils.java | 37 +-
.../org/apache/hudi/config/HoodieCleanConfig.java | 15 +-
.../apache/hudi/config/HoodieCompactionConfig.java | 12 +-
.../apache/hudi/config/HoodieErrorTableConfig.java | 11 +
.../org/apache/hudi/config/HoodieIndexConfig.java | 16 +-
.../apache/hudi/config/HoodiePayloadConfig.java | 2 +-
.../org/apache/hudi/config/HoodieWriteConfig.java | 85 +-
.../hudi/estimator/AverageRecordSizeEstimator.java | 133 ++++
.../apache/hudi/estimator/RecordSizeEstimator.java | 28 +-
.../hudi/estimator/RecordSizeEstimatorFactory.java | 45 ++
.../org/apache/hudi/index/HoodieIndexUtils.java | 2 +-
.../hudi/index/bloom/BloomIndexFileInfo.java | 10 +-
.../apache/hudi/index/bloom/HoodieBloomIndex.java | 1 +
.../index/bucket/ConsistentBucketIndexUtils.java | 7 +-
.../hbase/DefaultHBaseQPSResourceAllocator.java | 2 +-
.../java/org/apache/hudi/io/HoodieCDCLogger.java | 5 +-
.../java/org/apache/hudi/io/HoodieMergeHandle.java | 5 +-
.../org/apache/hudi/io/HoodieMergedReadHandle.java | 4 +-
.../java/org/apache/hudi/io/HoodieWriteHandle.java | 10 +-
.../metadata/HoodieBackedTableMetadataWriter.java | 156 ++--
...ieBackedTableMetadataWriterTableVersionSix.java | 343 ++++++++
.../hudi/metadata/HoodieMetadataWriteUtils.java | 2 +
.../hudi/metadata/HoodieTableMetadataWriter.java | 15 +-
.../org/apache/hudi/metrics/HoodieMetrics.java | 123 ++-
.../java/org/apache/hudi/table/HoodieTable.java | 62 +-
.../org/apache/hudi/table/WorkloadProfile.java | 12 +-
.../java/org/apache/hudi/table/WorkloadStat.java | 12 +-
.../action/BaseTableServicePlanActionExecutor.java | 3 +-
.../table/action/clean/CleanActionExecutor.java | 18 +-
.../action/clean/CleanPlanActionExecutor.java | 37 +-
.../hudi/table/action/clean/CleanPlanner.java | 37 +-
.../cluster/ClusteringPlanActionExecutor.java | 20 +-
.../cluster/strategy/ClusteringPlanStrategy.java | 16 +-
.../action/commit/BaseCommitActionExecutor.java | 46 +-
.../hudi/table/action/commit/BaseWriteHelper.java | 5 +-
.../hudi/table/action/commit/BucketInfo.java | 10 +-
.../table/action/commit/HoodieDeleteHelper.java | 5 +-
.../hudi/table/action/commit/InsertBucket.java | 8 +-
.../apache/hudi/table/action/commit/SmallFile.java | 8 +-
.../hudi/table/action/compact/CompactHelpers.java | 19 +-
.../compact/RunCompactionActionExecutor.java | 2 +-
.../compact/ScheduleCompactionActionExecutor.java | 56 +-
.../BaseHoodieCompactionPlanGenerator.java | 4 +
.../strategy/CompositeCompactionStrategy.java | 2 +-
.../action/index/AbstractIndexingCatchupTask.java | 7 +-
.../index/RecordBasedIndexingCatchupTask.java | 4 +-
.../table/action/index/RunIndexActionExecutor.java | 7 +-
.../action/index/ScheduleIndexActionExecutor.java | 7 +-
.../index/WriteStatBasedIndexingCatchupTask.java | 4 +-
.../action/restore/BaseRestoreActionExecutor.java | 2 +-
.../hudi/table/action/restore/RestoreUtils.java | 4 +-
.../rollback/BaseRollbackActionExecutor.java | 16 +-
.../rollback/BaseRollbackPlanActionExecutor.java | 8 +-
.../rollback/ListingBasedRollbackStrategy.java | 105 ++-
.../rollback/MarkerBasedRollbackStrategy.java | 39 +-
.../action/rollback/RestorePlanActionExecutor.java | 8 +-
...BaseRollbackHelper.java => RollbackHelper.java} | 13 +-
.../rollback/RollbackHelperFactory.java} | 22 +-
.../table/action/rollback/RollbackHelperV1.java | 389 +++++++++
.../hudi/table/action/rollback/RollbackUtils.java | 4 +-
.../action/savepoint/SavepointActionExecutor.java | 18 +-
.../marker/AppendMarkerHandler.java} | 23 +-
.../hudi/table/marker/DirectWriteMarkers.java | 60 +-
.../hudi/table/marker/DirectWriteMarkersV1.java | 103 +++
.../marker/TimelineServerBasedWriteMarkers.java | 48 +-
.../marker/TimelineServerBasedWriteMarkersV1.java | 74 ++
.../org/apache/hudi/table/marker/WriteMarkers.java | 19 +
.../hudi/table/marker/WriteMarkersFactory.java | 31 +-
.../org/apache/hudi/table/repair/RepairUtils.java | 15 +-
.../hudi/table/upgrade/DowngradeHandler.java | 7 +-
.../upgrade/EightToSevenDowngradeHandler.java | 70 +-
.../table/upgrade/FiveToFourDowngradeHandler.java | 6 +-
.../table/upgrade/FourToFiveUpgradeHandler.java | 22 +-
.../table/upgrade/FourToThreeDowngradeHandler.java | 6 +-
.../table/upgrade/OneToZeroDowngradeHandler.java | 5 +-
.../table/upgrade/SevenToEightUpgradeHandler.java | 73 +-
.../table/upgrade/SevenToSixDowngradeHandler.java | 6 +-
.../table/upgrade/SixToFiveDowngradeHandler.java | 7 +-
.../table/upgrade/ThreeToTwoDowngradeHandler.java | 6 +-
.../table/upgrade/TwoToOneDowngradeHandler.java | 5 +-
.../hudi/table/upgrade/UpgradeDowngrade.java | 70 +-
.../hudi/table/upgrade/UpgradeDowngradeUtils.java | 2 +-
.../table/upgrade/ZeroToOneUpgradeHandler.java | 4 +-
.../org/apache/hudi/util/HttpRequestClient.java | 2 +-
.../org/apache/hudi/HoodieTestCommitGenerator.java | 12 +-
.../client/TestBaseHoodieTableServiceClient.java | 265 +++++++
.../hudi/client/TestBaseHoodieWriteClient.java | 221 ++++++
.../embedded/TestEmbeddedTimelineService.java | 14 +-
.../timeline/TestCompletionTimeQueryView.java | 28 +-
...ConcurrentSchemaEvolutionTableSchemaGetter.java | 625 +++++++++++++++
.../TestConflictResolutionStrategyUtil.java | 4 +-
.../client/transaction/TestHoodieLockMetrics.java | 70 ++
...TestSimpleSchemaConflictResolutionStrategy.java | 274 +++++++
.../lock/models/StorageLockFileTest.java | 165 ++++
.../models/TestLockProviderHeartbeatManager.java | 481 +++++++++++
.../client/utils/TestCommitMetadataResolver.java | 242 ++++++
...edTimeline.java => TestArchivedTimelineV2.java} | 14 +-
.../common/testutils/HoodieMetadataTestTable.java | 7 +-
.../hudi/config/TestHoodieErrorTableConfig.java | 36 +-
.../apache/hudi/config/TestHoodieWriteConfig.java | 14 +-
.../estimator/TestAverageRecordSizeEstimator.java | 275 +++++++
.../estimator/TestRecordSizeEstimatorFactory.java | 72 ++
.../TestHoodieBackedTableMetadataWriter.java | 111 +++
.../org/apache/hudi/metrics/TestHoodieMetrics.java | 86 +-
.../org/apache/hudi/table/TestBaseHoodieTable.java | 202 +++++
.../org/apache/hudi/table/TestHoodieTable.java | 132 ++++
.../apache/hudi/table/action/TestCleanPlanner.java | 99 +--
.../action/clean/TestCleanPlanActionExecutor.java | 159 ++++
.../TestScheduleCompactionActionExecutor.java | 80 ++
.../rollback/TestMarkerBasedRollbackStrategy.java | 17 +-
...RollbackHelper.java => TestRollbackHelper.java} | 6 +-
.../hudi/table/marker/TestWriteMarkersFactory.java | 71 +-
.../upgrade/TestEightToSevenDowngradeHandler.java | 45 +-
.../upgrade/TestSevenToEightUpgradeHandler.java | 60 +-
.../hudi/utils/HoodieWriterClientTestHarness.java | 28 +-
.../hudi/utils/TestMetadataConversionUtils.java | 53 +-
.../hudi/client/FlinkTaskContextSupplier.java | 12 +-
.../hudi/client/HoodieFlinkTableServiceClient.java | 13 +-
.../apache/hudi/client/HoodieFlinkWriteClient.java | 6 +-
...inkSizeBasedClusteringPlanStrategyRecently.java | 3 +-
.../hudi/execution/ExplicitWriteHandler.java | 2 +-
.../row/parquet/ParquetSchemaConverter.java | 6 +-
.../FlinkHoodieBackedTableMetadataWriter.java | 6 +-
.../commit/BaseFlinkCommitActionExecutor.java | 9 +-
.../FlinkDeletePartitionCommitActionExecutor.java | 3 +-
.../commit/FlinkInsertCommitActionExecutor.java | 2 +-
.../commit/FlinkUpsertCommitActionExecutor.java | 2 +-
.../table/upgrade/FlinkUpgradeDowngradeHelper.java | 3 +-
.../hudi/client/HoodieJavaTableServiceClient.java | 19 -
.../apache/hudi/client/HoodieJavaWriteClient.java | 3 +-
.../client/common/JavaTaskContextSupplier.java | 10 +
.../bulkinsert/JavaGlobalSortPartitioner.java | 14 +-
.../commit/BaseJavaCommitActionExecutor.java | 7 +-
.../commit/JavaInsertCommitActionExecutor.java | 2 +-
.../commit/JavaUpsertCommitActionExecutor.java | 2 +-
.../table/action/commit/JavaUpsertPartitioner.java | 13 +-
.../hudi/client/TestJavaHoodieBackedMetadata.java | 16 +-
.../testutils/HoodieJavaClientTestHarness.java | 19 +-
hudi-client/hudi-spark-client/pom.xml | 14 +
.../apache/hudi/client/HoodieSparkCompactor.java | 2 +-
.../org/apache/hudi/client/SparkRDDReadClient.java | 2 +-
.../hudi/client/SparkRDDTableServiceClient.java | 19 -
.../apache/hudi/client/SparkRDDWriteClient.java | 9 +-
.../hudi/client/SparkTaskContextSupplier.java | 10 +
.../MultipleSparkJobExecutionStrategy.java | 17 +-
...onsistentBucketClusteringExecutionStrategy.java | 3 +-
.../SparkSingleFileSortExecutionStrategy.java | 3 +-
.../SparkSortAndSizeExecutionStrategy.java | 3 +-
.../client/common/HoodieSparkEngineContext.java | 4 +-
.../client/utils/SparkMetadataWriterUtils.java | 32 +-
.../client/validator/SparkPreCommitValidator.java | 8 +-
.../org/apache/hudi/data/HoodieJavaPairRDD.java | 8 +
.../bulkinsert/BulkInsertMapFunction.java | 14 +-
.../bulkinsert/GlobalSortPartitioner.java | 6 +-
.../bulkinsert/RDDPartitionSortPartitioner.java | 9 +-
.../bloom/BucketizedBloomCheckPartitioner.java | 2 +-
.../index/bloom/SparkHoodieBloomIndexHelper.java | 21 +
.../expression/ExpressionIndexSparkFunctions.java | 10 +-
.../expression/HoodieSparkExpressionIndex.java | 15 +-
.../hudi/index/hbase/SparkHoodieHBaseIndex.java | 37 +-
...HoodieSparkFileGroupReaderBasedMergeHandle.java | 21 +-
.../hudi/io/storage/HoodieSparkParquetReader.java | 5 +-
.../apache/hudi/merge/SparkRecordMergingUtils.java | 5 +-
...eBackedTableMetadataWriterTableVersionSix.java} | 103 +--
.../hudi/metadata/SparkMetadataWriterFactory.java | 55 ++
.../apache/hudi/metrics/DistributedRegistry.java | 2 +-
.../apache/hudi/sort/SpaceCurveSortingHelper.java | 22 +-
.../org/apache/hudi/table/HoodieSparkTable.java | 6 +-
.../action/commit/AverageRecordSizeUtils.java | 91 ---
.../commit/BaseSparkCommitActionExecutor.java | 27 +-
.../action/commit/SparkBucketIndexPartitioner.java | 2 +-
.../SparkDeletePartitionCommitActionExecutor.java | 6 +-
.../table/action/commit/UpsertPartitioner.java | 14 +-
.../hudi/HoodieDatasetBulkInsertHelper.scala | 47 +-
.../SparkFileFormatInternalRowReaderContext.scala | 6 +-
.../org/apache/hudi/util/SparkKeyGenUtils.scala | 6 +-
.../apache/spark/HoodieSparkKryoRegistrar.scala | 11 +-
.../spark/sql/HoodieCatalystExpressionUtils.scala | 1 -
.../apache/spark/sql/HoodieInternalRowUtils.scala | 5 +
.../org/apache/spark/sql/hudi/SparkAdapter.scala | 11 +
.../hudi/client/TestCompactionAdminClient.java | 10 +-
.../java/org/apache/hudi/client/TestSavepoint.java | 11 +-
.../hudi/client/TestSparkRDDWriteClient.java | 36 +
.../common/TestHoodieSparkEngineContext.java | 57 ++
.../functional/TestExternalPathHandling.java | 12 +-
.../functional/TestHoodieBackedTableMetadata.java | 87 +-
.../TestHoodieClientOnCopyOnWriteStorage.java | 5 +-
.../functional/TestHoodieFileSystemViews.java | 299 +++++++
.../hudi/client/functional/TestHoodieIndex.java | 12 +-
.../client/functional/TestHoodieMetadataBase.java | 3 +-
.../TestSavepointRestoreCopyOnWrite.java | 2 +-
.../apache/hudi/data/TestHoodieJavaPairRDD.java | 110 +++
.../hudi/index/bloom/TestHoodieBloomIndex.java | 58 +-
.../expression/TestHoodieSparkExpressionIndex.java | 90 ++-
.../apache/hudi/io/TestHoodieTimelineArchiver.java | 37 +-
.../java/org/apache/hudi/table/TestCleaner.java | 11 +-
.../apache/hudi/table/TestHoodieSparkTable.java | 136 ++++
.../clean/TestCleanerInsertAndCleanByCommits.java | 4 +-
.../clean/TestCleanerInsertAndCleanByVersions.java | 6 +-
.../action/commit/TestAverageRecordSizeUtils.java | 216 -----
.../table/action/commit/TestUpsertPartitioner.java | 92 ---
.../table/action/compact/CompactionTestBase.java | 6 +-
.../table/functional/TestCleanActionExecutor.java | 8 +-
.../table/functional/TestCleanPlanExecutor.java | 3 +-
.../TestMarkerBasedRollbackStrategy.java | 27 +-
.../hudi/table/marker/TestDirectWriteMarkers.java | 73 +-
.../table/marker/TestDirectWriteMarkersV1.java | 35 +-
.../TestTimelineServerBasedWriteMarkers.java | 128 ++-
.../TestTimelineServerBasedWriteMarkersV1.java | 52 ++
.../hudi/table/marker/TestWriteMarkersBase.java | 73 +-
.../hudi/table/upgrade/TestUpgradeDowngrade.java | 39 +
.../hudi/testutils/HoodieCleanerTestBase.java | 7 +-
.../hudi/testutils/HoodieClientTestUtils.java | 12 +-
.../testutils/HoodieSparkClientTestHarness.java | 2 +-
.../SparkClientFunctionalTestHarness.java | 11 +-
hudi-common/pom.xml | 11 +-
.../src/main/avro/HoodieCommitMetadata.avsc | 70 ++
.../src/main/avro/HoodieReplaceCommitMetadata.avsc | 5 +
.../org/apache/hudi/BaseHoodieTableFileIndex.java | 33 +-
.../java/org/apache/hudi/avro/AvroSchemaCache.java | 47 ++
.../AvroSchemaComparatorForSchemaEvolution.java | 343 ++++++++
.../apache/hudi/avro/AvroSchemaCompatibility.java | 4 +-
.../java/org/apache/hudi/avro/HoodieAvroUtils.java | 30 +-
.../apache/hudi/avro/MercifulJsonConverter.java | 3 +-
.../processors/DecimalLogicalTypeProcessor.java | 37 +-
.../org/apache/hudi/avro/processors/Parser.java | 2 +-
.../org/apache/hudi/common/HoodieJsonPayload.java | 2 +-
.../org/apache/hudi/common/bloom/HashFunction.java | 6 +-
.../hudi/common/bootstrap/FileStatusUtils.java | 2 +-
.../index/hfile/HFileBootstrapIndexReader.java | 6 +-
.../hudi/common/config/HoodieCommonConfig.java | 14 +
.../hudi/common/config/HoodieStorageConfig.java | 2 +-
.../common/config/HoodieTimeGeneratorConfig.java | 2 +-
.../org/apache/hudi/common/data/HoodieData.java | 3 +-
.../hudi/common/data/HoodieListPairData.java | 27 +
.../apache/hudi/common/data/HoodiePairData.java | 12 +
.../hudi/common/engine/HoodieEngineContext.java | 2 +-
.../hudi/common/engine/HoodieReaderContext.java | 33 +-
.../common/engine/LocalTaskContextSupplier.java | 9 +
.../hudi/common/engine/TaskContextSupplier.java | 7 +
.../hudi/common/fs/ConsistencyGuardConfig.java | 10 +-
.../java/org/apache/hudi/common/fs/FSUtils.java | 13 +-
.../hudi/common/fs/SizeAwareDataInputStream.java | 14 +-
.../hudi/common/fs/SizeAwareDataOutputStream.java | 4 +-
.../org/apache/hudi/common/model/ActionType.java | 15 +-
.../hudi/common/model/ConsistentHashingNode.java | 8 +-
.../common/model/DefaultHoodieRecordPayload.java | 2 +-
.../org/apache/hudi/common/model/FileSlice.java | 12 +-
.../hudi/common/model/HoodieCommitMetadata.java | 14 +-
.../apache/hudi/common/model/HoodieFileGroup.java | 30 +-
.../org/apache/hudi/common/model/HoodieKey.java | 8 +-
.../hudi/common/model/HoodiePartitionMetadata.java | 5 +-
.../org/apache/hudi/common/model/HoodieRecord.java | 10 +-
.../common/model/HoodieRecordGlobalLocation.java | 12 +-
.../hudi/common/model/HoodieRecordLocation.java | 10 +-
.../hudi/common/model/HoodieRecordMerger.java | 38 +-
.../common/model/HoodieReplaceCommitMetadata.java | 32 +-
.../java/org/apache/hudi/common/model/IOType.java | 1 +
.../model/OverwriteWithLatestAvroPayload.java | 2 +-
.../common/model/PartialUpdateAvroPayload.java | 2 +-
.../hudi/common/model/RewriteAvroPayload.java | 2 +-
.../hudi/common/model/WriteOperationType.java | 4 +-
.../CustomSerializer.java} | 16 +-
.../DefaultSerializer.java} | 23 +-
.../serialization/HoodieFileGroupSerializer.java | 88 +++
.../serialization/HoodieFileSliceSerializer.java | 33 +-
.../hudi/common/table/HoodieTableConfig.java | 8 +-
.../hudi/common/table/HoodieTableMetaClient.java | 43 +-
.../hudi/common/table/TableSchemaResolver.java | 171 +---
.../hudi/common/table/cdc/HoodieCDCExtractor.java | 2 +-
.../common/table/cdc/HoodieCDCInferenceCase.java | 2 +-
.../table/log/AbstractHoodieLogRecordScanner.java | 6 +-
.../table/log/BaseHoodieLogRecordReader.java | 29 +-
.../hudi/common/table/log/HoodieLogFileReader.java | 2 +-
.../table/log/HoodieLogFormatReverseReader.java | 2 +-
.../table/log/HoodieMergedLogRecordReader.java | 29 +-
.../table/log/HoodieMergedLogRecordScanner.java | 15 +-
.../common/table/log/block/HoodieDataBlock.java | 13 +-
.../common/table/log/block/HoodieDeleteBlock.java | 4 +-
.../table/log/block/HoodieParquetDataBlock.java | 5 +-
.../hudi/common/table/marker/MarkerOperation.java | 2 +
...dler.java => FileGroupReaderSchemaHandler.java} | 147 +++-
...ecordBuffer.java => FileGroupRecordBuffer.java} | 76 +-
.../common/table/read/HoodieFileGroupReader.java | 59 +-
.../table/read/HoodieFileGroupRecordBuffer.java | 5 +
...fer.java => KeyBasedFileGroupRecordBuffer.java} | 47 +-
...ava => PositionBasedFileGroupRecordBuffer.java} | 74 +-
...andler.java => PositionBasedSchemaHandler.java} | 18 +-
...fer.java => UnmergedFileGroupRecordBuffer.java} | 9 +-
.../table/timeline/ArchivedTimelineLoader.java | 6 +-
.../common/table/timeline/BaseHoodieTimeline.java | 149 ++--
.../common/table/timeline/CommitMetadataSerDe.java | 26 +-
.../table/timeline/CompletionTimeQueryView.java | 20 +-
.../table/timeline/HoodieActiveTimeline.java | 194 ++---
.../table/timeline/HoodieArchivedTimeline.java | 49 +-
.../common/table/timeline/HoodieInstantReader.java | 55 ++
.../table/timeline/HoodieInstantTimeGenerator.java | 10 +-
.../hudi/common/table/timeline/HoodieTimeline.java | 220 +++++-
.../table/timeline/MetadataConversionUtils.java | 295 +++----
.../common/table/timeline/TimelineFactory.java | 5 +-
.../table/timeline/TimelineMetadataUtils.java | 138 +---
.../hudi/common/table/timeline/TimelineUtils.java | 108 ++-
.../common/table/timeline/dto/TimelineDTO.java | 6 +-
...enerator.java => DefaultInstantComparator.java} | 4 +-
.../timeline/versioning/v1/ActiveTimelineV1.java | 236 +++---
.../versioning/v1/ArchivedTimelineLoaderV1.java | 29 +-
.../timeline/versioning/v1/ArchivedTimelineV1.java | 172 +++-
.../timeline/versioning/v1/BaseTimelineV1.java | 26 +-
.../versioning/v1/CommitMetadataSerDeV1.java | 62 +-
.../versioning/v1/CompletionTimeQueryViewV1.java | 2 +-
.../timeline/versioning/v1/TimelineV1Factory.java | 9 +-
.../timeline/versioning/v2/ActiveTimelineV2.java | 250 +++---
.../timeline/versioning/v2/ArchivedTimelineV2.java | 38 +-
.../timeline/versioning/v2/BaseTimelineV2.java | 26 +-
.../versioning/v2/CommitMetadataSerDeV2.java | 90 ++-
.../versioning/v2/CompletionTimeQueryViewV2.java | 3 +-
.../timeline/versioning/v2/TimelineV2Factory.java | 9 +-
.../table/view/AbstractTableFileSystemView.java | 108 +--
.../common/table/view/FileSystemViewManager.java | 97 ++-
.../table/view/HoodieFileGroupSizeEstimator.java | 44 ++
.../table/view/HoodieTableFileSystemView.java | 28 +-
.../view/HoodieTablePreCommitFileSystemView.java | 10 +-
.../IncrementalTimelineSyncFileSystemView.java | 18 +-
.../hudi/common/table/view/NoOpTableMetadata.java | 148 ++++
.../table/view/PriorityBasedFileSystemView.java | 14 +-
.../view/RemoteHoodieTableFileSystemView.java | 56 +-
.../table/view/RocksDbBasedFileSystemView.java | 35 +-
.../view/SpillableMapBasedFileSystemView.java | 113 ++-
.../org/apache/hudi/common/util/CleanerUtils.java | 49 +-
.../apache/hudi/common/util/ClusteringUtils.java | 62 +-
.../org/apache/hudi/common/util/CommitUtils.java | 12 +-
.../apache/hudi/common/util/CompactionUtils.java | 23 +-
.../org/apache/hudi/common/util/ConfigUtils.java | 24 +-
.../common/util/HoodieCommonKryoRegistrar.java | 6 +-
.../hudi/common/util/InternalSchemaCache.java | 43 +-
...oSchemaCache.java => LocalAvroSchemaCache.java} | 14 +-
...DefaultSizeEstimator.java => LogicalClock.java} | 16 +-
.../org/apache/hudi/common/util/MarkerUtils.java | 7 +-
.../{VisibleForTesting.java => SystemClock.java} | 14 +-
.../common/util/collection/BitCaskDiskMap.java | 49 +-
.../hudi/common/util/collection/DiskMap.java | 77 +-
.../util/collection/ExternalSpillableMap.java | 35 +-
.../common/util/collection/LazyFileIterable.java | 17 +-
.../apache/hudi/common/util/collection/Pair.java | 2 +-
.../hudi/common/util/collection/RocksDBDAO.java | 55 +-
.../common/util/collection/RocksDbDiskMap.java | 12 +-
.../apache/hudi/common/util/collection/Triple.java | 4 +-
.../apache/hudi/common/util/hash/JenkinsHash.java | 12 +-
.../apache/hudi/common/util/hash/MurmurHash.java | 8 +-
.../common/util/queue/DisruptorMessageQueue.java | 2 +-
.../hudi/config/metrics/HoodieMetricsConfig.java | 5 +
.../metrics/HoodieMetricsPrometheusConfig.java | 2 +-
... HoodieDuplicateDataFileDetectedException.java} | 9 +-
...=> HoodieSchemaEvolutionConflictException.java} | 15 +-
.../org/apache/hudi/expression/Predicates.java | 6 +-
.../index/expression/HoodieExpressionIndex.java | 19 +
.../hudi/index/secondary/HoodieSecondaryIndex.java | 1 -
.../internal/schema/InternalSchemaBuilder.java | 1 -
.../org/apache/hudi/internal/schema/Types.java | 3 +-
.../schema/action/InternalSchemaChangeApplier.java | 4 +-
.../hudi/internal/schema/action/TableChange.java | 4 +-
.../io/FileBasedInternalSchemaStorageManager.java | 6 +-
.../internal/schema/utils/InternalSchemaUtils.java | 6 +-
.../hudi/internal/schema/utils/SerDeHelper.java | 4 +-
.../metadata/FileSystemBackedTableMetadata.java | 21 +
.../hudi/metadata/HoodieBackedTableMetadata.java | 53 +-
.../metadata/HoodieMetadataFileSystemView.java | 123 ---
.../hudi/metadata/HoodieMetadataMetrics.java | 8 +-
.../apache/hudi/metadata/HoodieTableMetadata.java | 7 +
.../hudi/metadata/HoodieTableMetadataUtil.java | 146 ++--
.../hudi/metadata/MetadataPartitionType.java | 5 +-
.../SecondaryIndexRecordGenerationUtils.java | 3 +-
.../hudi/metrics/MetricsGraphiteReporter.java | 4 +-
.../hudi/metrics/MetricsReporterFactory.java | 2 +-
.../custom/CustomizableMetricsReporter.java | 4 +-
.../hudi/metrics/datadog/DatadogReporter.java | 2 +-
.../hudi/timeline/TimelineServiceClient.java | 76 ++
.../hudi/timeline/TimelineServiceClientBase.java | 140 ++++
.../hudi/avro/MercifulJsonConverterTestBase.java | 215 ++++-
...TestAvroSchemaComparatorForSchemaEvolution.java | 499 ++++++++++++
.../org/apache/hudi/avro/TestHoodieAvroUtils.java | 71 +-
.../hudi/avro/TestMercifulJsonConverter.java | 149 +++-
.../common/data/TestHoodieListDataPairData.java | 34 +
.../common/fs/TestSizeAwareDataInputStream.java | 115 +++
.../common/model/TestHoodieCommitMetadata.java | 82 +-
.../model/TestHoodieReplaceCommitMetadata.java | 1 -
.../serialization/TestDefaultSerializer.java | 52 ++
.../TestHoodieFileGroupSerializer.java | 82 ++
.../TestHoodieFileSliceSerializer.java | 65 ++
.../hudi/common/table/log/TestLogFileIterator.java | 5 +-
.../table/read/TestFileGroupRecordBuffer.java | 368 +++++++++
.../table/read/TestHoodieFileGroupReaderBase.java | 11 +-
.../read/TestHoodieFileGroupRecordBuffer.java | 56 --
.../versioning/BaseTestCommitMetadataSerDe.java | 338 ++++++++
.../versioning/v1/TestCommitMetadataSerDeV1.java} | 31 +-
.../versioning/v2/TestCommitMetadataSerDeV2.java | 72 ++
.../view/TestHoodieFileGroupSizeEstimator.java | 62 ++
.../view/TestPriorityBasedFileSystemView.java | 103 +--
.../hudi/common/testutils/CompactionTestUtils.java | 3 +-
.../hudi/common/testutils/FileCreateUtils.java | 184 +++--
.../common/testutils/FileCreateUtilsLegacy.java | 64 +-
.../common/testutils/HoodieTestDataGenerator.java | 79 +-
.../hudi/common/testutils/HoodieTestUtils.java | 25 +-
.../reader/HoodieFileGroupReaderTestUtils.java | 27 +-
.../apache/hudi/common/util/TestCleanerUtils.java | 59 ++
.../apache/hudi/common/util/TestConfigUtils.java | 17 +
.../apache/hudi/common/util/TestHoodieTimer.java | 25 +-
...emaCache.java => TestLocalAvroSchemaCache.java} | 38 +-
.../apache/hudi/common/util/TestSystemClock.java | 34 +-
.../hudi/metrics/TestMetricsReporterFactory.java | 38 +-
.../hudi/timeline/TestTimelineServiceClient.java | 223 ++++++
.../avro-schema-evo/schema-allshapes-A.txt | 80 ++
.../avro-schema-evo/schema-allshapes-B.txt | 123 +++
...e.avsc => decimal-logical-type-zero-scale.avsc} | 2 +-
.../quickstart/utils/QuickstartConfigurations.java | 14 +-
.../examples/quickstart/utils/SchemaBuilder.java | 4 +-
.../spark/HoodieSparkBootstrapExample.java | 2 +-
.../examples/spark/HoodieWriteClientExample.java | 6 +-
.../hudi/sink/StreamWriteOperatorCoordinator.java | 5 +
.../AppendWriteFunctionWithRateLimit.java} | 19 +-
.../AppendWriteFunctions.java} | 19 +-
.../hudi/sink/append/AppendWriteOperator.java | 2 +-
.../bucket/ConsistentBucketAssignFunction.java | 3 +-
.../apache/hudi/sink/bulk/sort/SortOperator.java | 2 +-
.../hudi/sink/clustering/ClusteringPlanEvent.java | 17 +
.../sink/clustering/ClusteringPlanOperator.java | 18 +-
.../FlinkConsistentBucketUpdateStrategy.java | 4 +-
.../sink/partitioner/profile/WriteProfiles.java | 5 +-
.../hudi/sink/transform/ChainedTransformer.java | 2 +-
.../RowDataToHoodieFunctionWithRateLimit.java | 2 +-
.../java/org/apache/hudi/sink/utils/Pipelines.java | 8 +-
.../apache/hudi/source/ExpressionEvaluators.java | 4 +-
.../org/apache/hudi/table/HoodieTableFactory.java | 3 +
.../org/apache/hudi/table/HoodieTableSource.java | 12 +-
.../org/apache/hudi/table/format/FormatUtils.java | 11 +-
.../hudi/table/format/cdc/CdcInputFormat.java | 4 +-
.../hudi/table/lookup/HoodieLookupFunction.java | 2 +-
.../org/apache/hudi/util/FlinkWriteClients.java | 3 +-
.../apache/hudi/util/RowDataToAvroConverters.java | 4 +-
.../org/apache/hudi/sink/TestWriteCopyOnWrite.java | 24 +
.../hudi/sink/TestWriterWithPartitionTTl.java | 3 +-
.../hudi/sink/bucket/ITTestBucketStreamWrite.java | 4 +-
.../apache/hudi/sink/compact/TestCustomSerDe.java | 19 +-
.../utils/BucketStreamWriteFunctionWrapper.java | 6 +
.../hudi/sink/utils/BulkInsertFunctionWrapper.java | 6 +
.../hudi/sink/utils/InsertFunctionWrapper.java | 15 +
.../hudi/sink/utils/MockSubtaskGateway.java} | 35 +-
.../sink/utils/StreamWriteFunctionWrapper.java | 6 +
.../hudi/sink/utils/TestFunctionWrapper.java | 15 +-
.../org/apache/hudi/sink/utils/TestWriteBase.java | 13 +
.../hudi/source/TestIncrementalInputSplits.java | 16 +-
.../org/apache/hudi/utils/TestClusteringUtil.java | 15 +-
.../org/apache/hudi/utils/TestCompactionUtil.java | 11 +-
.../org/apache/hudi/utils/TestStreamerUtil.java | 4 +-
.../test/java/org/apache/hudi/utils/TestUtils.java | 6 +-
.../apache/hudi/adapter/ExecutionAttemptUtil.java} | 11 +-
.../apache/hudi/adapter/ExecutionAttemptUtil.java} | 11 +-
.../apache/hudi/adapter/ExecutionAttemptUtil.java} | 11 +-
.../apache/hudi/adapter/ExecutionAttemptUtil.java} | 11 +-
.../format/cow/vector/HeapMapColumnVector.java | 4 +-
...odieCatalogs.java => ExecutionAttemptUtil.java} | 11 +-
.../format/cow/vector/HeapMapColumnVector.java | 4 +-
.../apache/hudi/adapter/ExecutionAttemptUtil.java} | 11 +-
.../format/cow/vector/HeapMapColumnVector.java | 4 +-
.../apache/hudi/adapter/ExecutionAttemptUtil.java} | 11 +-
.../hfile/HBaseHFileBootstrapIndexReader.java | 6 +-
.../common/config/DFSPropertiesConfiguration.java | 2 +-
.../org/apache/hudi/common/util/AvroOrcUtils.java | 18 +-
.../apache/hudi/common/util/HadoopConfigUtils.java | 3 +-
.../hudi/hadoop/fs/BoundedFsDataInputStream.java | 4 +-
.../hadoop/fs/HoodieRetryWrapperFileSystem.java | 13 +-
.../hadoop/fs/HoodieSerializableFileStatus.java | 22 +-
.../hudi/hadoop/fs/HoodieWrapperFileSystem.java | 4 +-
.../hudi/hadoop/fs/inline/InMemoryFileSystem.java | 3 +-
.../hudi/io/hadoop/HoodieAvroHFileWriter.java | 4 +-
.../apache/hudi/io/hadoop/HoodieAvroOrcWriter.java | 2 +-
.../hudi/io/hadoop/HoodieBaseParquetWriter.java | 4 +-
.../index/TestBaseHoodieTableFileIndex.java | 86 ++
.../org/apache/hudi/common/fs/TestFSUtils.java | 32 +
.../common/fs/TestHoodieWrapperFileSystem.java | 5 +-
.../common/functional/TestHoodieLogFormat.java | 8 +-
.../hudi/common/table/TestHoodieTableConfig.java | 8 +-
.../common/table/TestHoodieTableMetaClient.java | 59 +-
.../hudi/common/table/TestTableSchemaResolver.java | 413 +---------
.../hudi/common/table/TestTimelineUtils.java | 53 +-
.../TestHoodieFileGroupReaderInflightCommit.java | 112 +++
.../table/timeline/TestArchivedTimelineV1.java | 749 ++++++++++++++++++
.../table/timeline/TestHoodieActiveTimeline.java | 156 +++-
.../table/timeline/TestWaitBasedTimeGenerator.java | 24 +-
.../view/TestHoodieTableFSViewWithClustering.java | 12 +-
.../table/view/TestHoodieTableFileSystemView.java | 206 +++--
.../table/view/TestIncrementalFSViewSync.java | 28 +-
.../TestRocksDBBasedIncrementalFSViewSync.java | 6 +-
.../table/view/TestRocksDbBasedFileSystemView.java | 6 +-
.../view/TestSpillableMapBasedFileSystemView.java | 6 +-
...TestSpillableMapBasedIncrementalFSViewSync.java | 6 +-
.../common/testutils/HoodieCommonTestHarness.java | 83 +-
.../hudi/common/testutils/HoodieTestTable.java | 163 ++--
.../reader/HoodieFileGroupReaderTestHarness.java | 9 +-
.../hudi/common/util/TestClusteringUtils.java | 34 +-
.../apache/hudi/common/util/TestCommitUtils.java | 19 +-
.../common/util/collection/TestBitCaskDiskMap.java | 213 ++---
.../util/collection/TestExternalSpillableMap.java | 20 +-
.../util/collection/TestRocksDbBasedMap.java | 28 +-
.../common/util/collection/TestRocksDbDiskMap.java | 225 +++---
.../hudi/hadoop/HiveHoodieReaderContext.java | 10 +
.../HoodieFileGroupReaderBasedRecordReader.java | 2 +-
.../hudi/hadoop/HoodieHFileRecordReader.java | 4 +-
.../hudi/hadoop/HoodieROTablePathFilter.java | 8 +-
.../org/apache/hudi/hadoop/InputPathHandler.java | 2 +-
.../hadoop/hive/HoodieCombineHiveInputFormat.java | 11 +-
.../hive/HoodieCombineRealtimeFileSplit.java | 6 +-
.../realtime/AbstractRealtimeRecordReader.java | 12 +-
.../HoodieCombineRealtimeRecordReader.java | 6 +-
.../realtime/HoodieMergeOnReadSnapshotReader.java | 5 +-
.../apache/hudi/hadoop/utils/HoodieHiveUtils.java | 2 +-
.../hudi/hadoop/utils/HoodieInputFormatUtils.java | 11 +-
.../utils/HoodieRealtimeInputFormatUtils.java | 3 +-
.../hudi/hadoop/TestHoodieHFileInputFormat.java | 23 +-
.../hudi/hadoop/TestHoodieParquetInputFormat.java | 23 +-
.../realtime/TestHoodieRealtimeRecordReader.java | 27 +-
.../hudi/integ/testsuite/HoodieTestSuiteJob.java | 5 +-
.../dag/nodes/BaseValidateDatasetNode.java | 5 +-
.../testsuite/dag/nodes/ScheduleCompactNode.java | 3 +-
.../reader/DFSHoodieDatasetInputReader.java | 4 +-
.../testsuite/dag/nodes/SparkBulkInsertNode.scala | 4 +-
.../testsuite/dag/nodes/SparkDeleteNode.scala | 5 +-
.../dag/nodes/SparkDeletePartitionNode.scala | 1 +
.../testsuite/dag/nodes/SparkInsertNode.scala | 5 +-
.../testsuite/dag/nodes/SparkUpsertNode.scala | 1 +
.../dag/nodes/spark/sql/BaseSparkSqlNode.scala | 3 +-
.../nodes/spark/sql/SparkSqlCreateTableNode.scala | 3 +-
.../dag/nodes/spark/sql/SparkSqlDeleteNode.scala | 3 +-
.../dag/nodes/spark/sql/SparkSqlMergeNode.scala | 3 +-
.../dag/nodes/spark/sql/SparkSqlUpdateNode.scala | 3 +-
.../spark/sql/SparkSqlValidateDatasetNode.scala | 3 +-
.../StructuredStreamingSinkTestWriter.scala | 5 +-
.../hudi/integ/testsuite/utils/SparkSqlUtils.scala | 8 +-
.../org/apache/hudi/integ/ITTestHoodieDemo.java | 2 +-
.../integ/command/ITTestHoodieSyncCommand.java | 2 +-
.../apache/hudi/common/util/ComparableVersion.java | 2 +-
.../org/apache/hudi/common/util/FileIOUtils.java | 11 +-
.../org/apache/hudi/common/util/HoodieTimer.java | 11 +
.../org/apache/hudi/common/util/StringUtils.java | 6 +-
.../org/apache/hudi/io/hfile/HFileBlockType.java | 2 +-
.../apache/hudi/storage/HoodieInstantWriter.java | 34 +-
.../org/apache/hudi/storage/HoodieStorage.java | 20 +-
.../org/apache/hudi/storage/StorageSchemes.java | 23 +-
.../apache/hudi/common/util/TestStringUtils.java | 4 +
.../hudi/io/storage/TestHoodieStorageBase.java | 3 +-
hudi-kafka-connect/pom.xml | 5 +-
.../connect/kafka/KafkaConnectControlAgent.java | 2 +-
.../hudi/connect/utils/KafkaConnectUtils.java | 7 +-
.../connect/writers/BufferedConnectWriter.java | 5 +-
.../timeline/HoodieMetaserverBasedTimeline.java | 19 +-
.../table/view/HoodieMetaserverFileSystemView.java | 11 +-
.../client/HoodieMetaserverClientImp.java | 4 +-
.../main/java/org/apache/hudi/DataSourceUtils.java | 2 +-
.../BaseDatasetBulkInsertCommitActionExecutor.java | 1 +
...setBulkInsertOverwriteCommitActionExecutor.java | 2 +-
...eamerDatasetBulkInsertCommitActionExecutor.java | 2 +-
.../hudi/internal/BaseWriterCommitMessage.java | 2 +-
.../internal/DataSourceInternalWriterHelper.java | 2 +-
.../main/java/org/apache/hudi/sql/InsertMode.java | 2 +-
.../apache/hudi/AutoRecordKeyGenerationUtils.scala | 1 +
.../org/apache/hudi/BloomFiltersIndexSupport.scala | 1 +
.../scala/org/apache/hudi/BucketIndexSupport.scala | 3 +-
.../org/apache/hudi/ColumnStatsIndexSupport.scala | 22 +-
.../scala/org/apache/hudi/DataSourceOptions.scala | 2 +-
.../main/scala/org/apache/hudi/DefaultSource.scala | 9 +-
.../main/scala/org/apache/hudi/EmptyRelation.scala | 2 +-
.../org/apache/hudi/ExpressionIndexSupport.scala | 7 +-
.../scala/org/apache/hudi/FileSystemRelation.scala | 3 +-
.../scala/org/apache/hudi/HoodieBaseRelation.scala | 7 +-
.../org/apache/hudi/HoodieBootstrapMORRDD.scala | 4 +-
.../apache/hudi/HoodieBootstrapMORRelation.scala | 1 +
.../scala/org/apache/hudi/HoodieBootstrapRDD.scala | 2 +-
.../org/apache/hudi/HoodieBootstrapRelation.scala | 2 +-
.../scala/org/apache/hudi/HoodieCLIUtils.scala | 2 +
.../org/apache/hudi/HoodieCreateRecordUtils.scala | 12 +-
.../scala/org/apache/hudi/HoodieFileIndex.scala | 31 +-
.../hudi/HoodieHadoopFsRelationFactory.scala | 22 +-
.../org/apache/hudi/HoodieMergeOnReadRDDV1.scala | 6 +-
.../hudi/HoodiePartitionCDCFileGroupMapping.scala | 1 +
.../hudi/HoodiePartitionFileSliceMapping.scala | 1 +
.../scala/org/apache/hudi/HoodieSchemaUtils.scala | 3 +-
.../org/apache/hudi/HoodieSparkSqlWriter.scala | 22 +-
.../scala/org/apache/hudi/HoodieWriterUtils.scala | 46 +-
.../org/apache/hudi/IncrementalRelationV1.scala | 12 +-
.../org/apache/hudi/IncrementalRelationV2.scala | 8 +-
.../src/main/scala/org/apache/hudi/Iterators.scala | 5 +-
.../hudi/MergeOnReadIncrementalRelationV1.scala | 4 +-
.../apache/hudi/PartitionStatsIndexSupport.scala | 11 +-
.../org/apache/hudi/RecordLevelIndexSupport.scala | 3 +-
.../org/apache/hudi/SecondaryIndexSupport.scala | 2 +-
.../org/apache/hudi/SparkBaseIndexSupport.scala | 11 +-
.../apache/hudi/SparkHoodieTableFileIndex.scala | 7 +-
.../scala/org/apache/hudi/TimelineRelation.scala | 3 +-
.../org/apache/hudi/cdc/CDCFileGroupIterator.scala | 16 +-
.../scala/org/apache/hudi/cdc/CDCRelation.scala | 4 +-
.../datasources/HoodieInMemoryFileIndex.scala | 5 +-
.../sql/catalyst/catalog/HoodieCatalogTable.scala | 6 +-
.../datasources/HoodieMultipleBaseFileFormat.scala | 7 +-
...odieFileGroupReaderBasedParquetFileFormat.scala | 26 +-
.../parquet/LegacyHoodieParquetFileFormat.scala | 3 +-
.../parquet/SparkParquetReaderBase.scala | 5 +-
.../apache/spark/sql/hudi/DataSkippingUtils.scala | 1 +
.../spark/sql/hudi/HoodieSqlCommonUtils.scala | 7 +-
.../spark/sql/hudi/ProvidesHoodieConfig.scala | 61 +-
.../AlterHoodieTableAddColumnsCommand.scala | 8 +-
.../AlterHoodieTableAddPartitionCommand.scala | 1 +
.../AlterHoodieTableChangeColumnCommand.scala | 3 +-
.../AlterHoodieTableDropPartitionCommand.scala | 1 +
.../command/AlterHoodieTableRenameCommand.scala | 4 +-
.../hudi/command/CreateHoodieTableCommand.scala | 5 +-
.../command/CreateHoodieTableLikeCommand.scala | 1 +
.../sql/hudi/command/DropHoodieTableCommand.scala | 1 +
.../sql/hudi/command/MergeIntoKeyGenerator.scala | 3 +-
.../hudi/command/RepairHoodieTableCommand.scala | 3 +-
.../command/ShowHoodieTablePartitionsCommand.scala | 1 +
.../spark/sql/hudi/command/SqlKeyGenerator.scala | 4 +-
.../hudi/command/TruncateHoodieTableCommand.scala | 1 +
.../sql/hudi/streaming/HoodieMetadataLog.scala | 1 +
.../sql/hudi/streaming/HoodieStreamSourceV1.scala | 9 +-
.../sql/hudi/streaming/HoodieStreamSourceV2.scala | 5 +-
.../java/org/apache/hudi/TestDataSourceUtils.java | 1 +
.../org/apache/hudi/TestHoodieWriterUtils.java | 46 ++
.../hudi/testutils/SparkDatasetTestUtils.java | 7 +-
.../org/apache/hudi/TestDataSourceOptions.scala | 1 +
.../apache/hudi/TestRecordLevelIndexSupport.scala | 1 +
.../apache/hudi/TestSecondaryIndexSupport.scala | 1 +
.../cdc/TestInternalRowToJsonStringConverter.scala | 1 +
.../hudi/schema/TestHoodieSparkSchemaUtils.scala | 1 +
.../datasources/TestHoodieInMemoryFileIndex.scala | 3 +-
...stSparkFileFormatInternalRowReaderContext.scala | 2 +-
.../spark/sql/hive/TestHiveClientUtils.scala | 5 +-
.../spark/sql/hudi/TestHoodieFileIndex.scala | 22 +-
.../spark/sql/hudi/TestProvidesHoodieConfig.scala | 5 +-
.../spark/sql/hudi/TestSparkConfigUtils.scala | 1 +
.../main/java/org/apache/hudi/QuickstartUtils.java | 31 +-
.../apache/hudi/cli/BootstrapExecutorUtils.java | 4 +-
.../apache/hudi/cli/HDFSParquetImporterUtils.java | 4 +-
.../spark/sql/hudi/analysis/HoodieAnalysis.scala | 15 +-
.../analysis/HoodiePruneFileSourcePartitions.scala | 6 +-
.../hudi/command/CallProcedureHoodieCommand.scala | 2 +-
.../hudi/command/CompactionHoodiePathCommand.scala | 2 +-
.../command/CompactionHoodieTableCommand.scala | 2 +-
.../command/CompactionShowHoodiePathCommand.scala | 2 +-
.../command/CompactionShowHoodieTableCommand.scala | 2 +-
.../command/CreateHoodieTableAsSelectCommand.scala | 16 +-
.../hudi/command/DeleteHoodieTableCommand.scala | 41 +-
.../spark/sql/hudi/command/IndexCommands.scala | 5 +-
.../command/InsertIntoHoodieTableCommand.scala | 27 +-
.../hudi/command/MergeIntoHoodieTableCommand.scala | 348 ++++++--
.../hudi/command/UpdateHoodieTableCommand.scala | 6 +-
.../hudi/command/payload/ExpressionPayload.scala | 4 +-
.../procedures/CommitsCompareProcedure.scala | 4 +-
.../procedures/CreateMetadataTableProcedure.scala | 4 +-
.../procedures/DropPartitionProcedure.scala | 3 +-
.../procedures/ExportInstantsProcedure.scala | 10 +-
.../procedures/InitMetadataTableProcedure.scala | 4 +-
.../RepairCorruptedCleanFilesProcedure.scala | 5 +-
.../procedures/RepairDeduplicateProcedure.scala | 2 +-
.../command/procedures/RunBootstrapProcedure.scala | 3 +-
.../procedures/RunClusteringProcedure.scala | 8 +-
.../procedures/RunCompactionProcedure.scala | 2 +-
.../RunRollbackInflightTableServiceProcedure.scala | 1 +
.../procedures/ShowArchivedCommitsProcedure.scala | 7 +-
.../procedures/ShowClusteringProcedure.scala | 3 +-
.../ShowColumnStatsOverlapProcedure.scala | 21 +-
.../ShowCommitExtraMetadataProcedure.scala | 10 +-
.../procedures/ShowCommitFilesProcedure.scala | 9 +-
.../procedures/ShowCommitPartitionsProcedure.scala | 13 +-
.../procedures/ShowCommitWriteStatsProcedure.scala | 13 +-
.../command/procedures/ShowCommitsProcedure.scala | 7 +-
.../procedures/ShowFileStatusProcedure.scala | 17 +-
.../procedures/ShowFileSystemViewProcedure.scala | 16 +-
.../ShowHoodieLogFileMetadataProcedure.scala | 2 +-
.../ShowHoodieLogFileRecordsProcedure.scala | 4 +-
.../procedures/ShowInvalidParquetProcedure.scala | 10 +-
.../ShowMetadataTableColumnStatsProcedure.scala | 26 +-
.../procedures/ShowRollbacksProcedure.scala | 12 +-
.../StatsWriteAmplificationProcedure.scala | 5 +-
.../procedures/TruncateTableProcedure.scala | 1 +
.../procedures/UpgradeOrDowngradeProcedure.scala | 2 +-
.../procedures/ValidateHoodieSyncProcedure.scala | 7 +-
.../spark/sql/parser/HoodieCommonSqlParser.scala | 6 +-
.../hudi/TestHoodieMergeHandleWithSparkMerger.java | 2 +-
...=> TestPositionBasedFileGroupRecordBuffer.java} | 12 +-
.../hudi/client/TestHoodieClientMultiWriter.java | 294 ++++++-
.../functional/TestHoodieBackedMetadata.java | 137 +++-
.../TestMetadataUtilRLIandSIRecordGeneration.java | 23 +-
.../TestRemoteFileSystemViewWithMetadataTable.java | 12 +-
.../TestSavepointRestoreMergeOnRead.java | 2 +-
.../TestColStatsRecordWithMetadataRecord.java | 10 +
.../TestHoodieDatasetBulkInsertHelper.java | 2 +-
.../functional/TestSparkClusteringCornerCases.java | 80 ++
.../apache/hudi/functional/TestWriteClient.java | 3 +-
.../hudi/table/TestHoodieMergeOnReadTable.java | 29 +-
.../table/action/compact/TestAsyncCompaction.java | 42 +-
.../table/action/compact/TestHoodieCompactor.java | 26 +-
.../TestMergeOnReadRollbackActionExecutor.java | 13 +-
...arkCopyOnWriteTableRollbackTableVersionSix.java | 63 ++
.../TestHoodieSparkMergeOnReadTableCompaction.java | 275 ++++++-
.../TestHoodieSparkMergeOnReadTableRollback.java | 258 ++++--
.../table/functional/TestHoodieSparkRollback.java | 307 +++++++
.../apache/hudi/testutils/DataSourceTestUtils.java | 33 +-
.../src/test/resources/exampleEvolvedSchema.txt | 5 +
.../src/test/resources/exampleSchema.txt | 5 +
.../src/test/resources/sql-statements.sql | 16 +-
.../TestIncrementalQueryWithArchivedInstants.scala | 10 +-
.../apache/hudi/HoodieSparkWriterTestBase.scala | 2 +-
.../org/apache/hudi/TestAutoKeyGenForSQL.scala | 1 +
.../org/apache/hudi/TestAvroConversionUtils.scala | 3 +-
.../hudi/TestAvroSchemaResolutionSupport.scala | 2 +-
.../org/apache/hudi/TestDataSkippingUtils.scala | 4 +-
.../org/apache/hudi/TestDataSourceDefaults.scala | 2 +-
.../hudi/TestGenericRecordAndRowConsistency.scala | 2 +-
.../apache/hudi/TestHoodieDataSourceHelper.scala | 6 +-
.../org/apache/hudi/TestHoodieFileIndex.scala | 14 +-
.../org/apache/hudi/TestHoodieSparkSqlWriter.scala | 55 +-
.../TestHoodieSparkSqlWriterPartitionTTL.scala | 3 +-
.../apache/hudi/TestHoodieSparkSqlWriterUtc.scala | 2 +-
.../org/apache/hudi/TestHoodieSparkUtils.scala | 2 +-
.../org/apache/hudi/TestInsertDedupPolicy.scala | 26 +-
.../hudi/TestMultipleTableVersionWriting.scala | 1 +
.../hudi/TestParquetReaderCompatibility.scala | 4 +-
.../org/apache/hudi/TestSparkFilterHelper.scala | 2 +-
.../hudi/TestTableSchemaResolverWithSparkSQL.scala | 2 +-
.../model/TestHoodieRecordSerialization.scala | 8 +-
.../read/TestHoodieFileGroupReaderOnSpark.scala | 145 +++-
.../hudi/functional/ColumnStatIndexTestBase.scala | 114 ++-
.../hudi/functional/HoodieStatsIndexTestBase.scala | 5 +-
.../functional/PartitionStatsIndexTestBase.scala | 1 +
.../hudi/functional/RecordLevelIndexTestBase.scala | 7 +-
.../TestAutoGenerationOfRecordKeys.scala | 12 +-
.../hudi/functional/TestBasicSchemaEvolution.scala | 6 +-
.../functional/TestBloomFiltersIndexSupport.scala | 16 +-
.../hudi/functional/TestBucketIndexSupport.scala | 6 +-
.../apache/hudi/functional/TestCOWDataSource.scala | 119 +--
.../hudi/functional/TestColumnStatsIndex.scala | 184 +++--
.../functional/TestColumnStatsIndexWithSQL.scala | 18 +-
.../functional/TestDataSourceForBootstrap.scala | 2 +
.../apache/hudi/functional/TestEmptyCommit.scala | 4 +-
.../hudi/functional/TestHoodieActiveTimeline.scala | 9 +-
.../TestHoodieMultipleBaseFileFormat.scala | 9 +-
.../TestIncrementalReadWithFullTableScan.scala | 7 +-
.../hudi/functional/TestLayoutOptimization.scala | 6 +-
.../apache/hudi/functional/TestMORDataSource.scala | 133 ++--
.../hudi/functional/TestMORDataSourceStorage.scala | 7 +-
.../hudi/functional/TestMetadataRecordIndex.scala | 6 +-
.../TestMetadataTableWithSparkDataSource.scala | 4 +-
.../hudi/functional/TestMetricsReporter.scala | 4 +-
.../functional/TestPartialUpdateAvroPayload.scala | 4 +-
.../hudi/functional/TestPartitionStatsIndex.scala | 26 +-
.../TestPartitionStatsIndexWithSql.scala | 25 +-
.../functional/TestPartitionStatsPruning.scala | 1 +
.../hudi/functional/TestRecordLevelIndex.scala | 43 +-
.../TestRecordLevelIndexTableVersionSix.scala | 16 +-
.../functional/TestRecordLevelIndexWithSQL.scala | 10 +-
.../functional/TestSecondaryIndexPruning.scala | 27 +-
.../hudi/functional/TestSevenToEightUpgrade.scala | 221 +++++-
.../functional/TestSixToFiveDowngradeHandler.scala | 8 +-
.../hudi/functional/TestSparkDataSource.scala | 135 +++-
.../hudi/functional/TestSparkSqlCoreFlow.scala | 6 +-
.../TestSparkSqlWithCustomKeyGenerator.scala | 3 +-
.../TestSparkSqlWithTimestampKeyGenerator.scala | 1 +
...TestStreamSourceReadByStateTransitionTime.scala | 1 -
.../hudi/functional/TestStreamingSource.scala | 3 +-
.../hudi/functional/TestTimeTravelQuery.scala | 11 +-
.../functional/TestWriteTableVersionConfig.scala | 6 +-
.../hudi/functional/cdc/HoodieCDCTestBase.scala | 16 +-
.../functional/cdc/TestCDCDataFrameSuite.scala | 11 +-
.../functional/cdc/TestCDCStreamingSuite.scala | 4 +-
.../org/apache/hudi/util/JavaConversions.scala | 2 +-
.../spark/hudi/benchmark/HoodieBenchmark.scala | 1 -
.../execution/benchmark/AvroSerDerBenchmark.scala | 3 +-
.../benchmark/BoundInMemoryExecutorBenchmark.scala | 2 +-
.../benchmark/CowTableReadBenchmark.scala | 2 +-
.../benchmark/LSMTimelineReadBenchmark.scala | 10 +-
.../TestHoodiePruneFileSourcePartitions.scala | 4 +-
.../analysis/TestMergeIntoHoodieTableCommand.scala | 108 +++
.../sql/hudi/common/HoodieSparkSqlTestBase.scala | 78 +-
.../hudi/common/TestHoodieInternalRowUtils.scala | 6 +-
.../apache/spark/sql/hudi/common/TestSqlConf.scala | 5 +-
.../hudi/common/TestTableColumnTypeMismatch.scala | 879 +++++++++++++++++++++
.../apache/spark/sql/hudi/ddl/TestAlterTable.scala | 83 +-
.../sql/hudi/ddl/TestAlterTableDropPartition.scala | 13 +-
.../sql/hudi/{dml => ddl}/TestDropTable.scala | 28 +-
.../sql/hudi/{dml => ddl}/TestRepairTable.scala | 15 +-
.../sql/hudi/{dml => ddl}/TestShowPartitions.scala | 28 +-
.../sql/hudi/{dml => ddl}/TestTruncateTable.scala | 15 +-
.../spark/sql/hudi/dml/TestDeleteFromTable.scala | 1 -
.../spark/sql/hudi/dml/TestDeleteTable.scala | 50 ++
.../spark/sql/hudi/dml/TestInsertTable.scala | 102 ++-
.../sql/hudi/dml/TestMergeIntoLogOnlyTable.scala | 4 +-
.../spark/sql/hudi/dml/TestMergeIntoTable.scala | 413 +++++-----
.../spark/sql/hudi/dml/TestMergeIntoTable2.scala | 208 +++--
.../TestMergeIntoTableWithNonRecordKeyField.scala | 6 +-
.../hudi/dml/TestMergeModeCommitTimeOrdering.scala | 3 +
.../hudi/dml/TestMergeModeEventTimeOrdering.scala | 5 +-
.../hudi/dml/TestPartialUpdateForMergeInto.scala | 159 +++-
.../spark/sql/hudi/dml/TestUpdateTable.scala | 54 +-
.../hudi/{dml => feature}/TestCDCForSparkSQL.scala | 28 +-
.../{dml => feature}/TestCompactionTable.scala | 28 +-
.../{dml => feature}/TestDataSkippingQuery.scala | 2 +-
.../TestQueryMergeOnReadOptimizedTable.scala | 28 +-
.../index/TestExpressionIndex.scala | 65 +-
.../sql/hudi/feature/index/TestGlobalIndex.scala | 96 +++
.../index/TestIndexSyntax.scala | 3 +-
.../index/TestSecondaryIndex.scala | 15 +-
.../sql/hudi/procedure/TestCallCommandParser.scala | 3 +-
.../hudi/procedure/TestClusteringProcedure.scala | 11 +-
.../procedure/TestDropPartitionProcedure.scala | 14 +-
.../procedure/TestHdfsParquetImportProcedure.scala | 2 +
...tRunRollbackInflightTableServiceProcedure.scala | 1 +
.../procedure/TestShowFileStatusProcedure.scala | 5 +-
.../TestShowInvalidParquetProcedure.scala | 6 +-
.../sql/hudi/procedure/TestTTLProcedure.scala | 2 +-
.../procedure/TestTruncateTableProcedure.scala | 7 +-
.../TestUpgradeOrDowngradeProcedure.scala | 5 +-
.../Spark3HoodieVectorizedParquetRecordReader.java | 13 +-
.../scala/org/apache/hudi/Spark3RowSerDe.scala | 1 +
.../spark/sql/HoodieSpark3CatalogUtils.scala | 1 +
.../spark/sql/HoodieSpark3CatalystPlanUtils.scala | 1 +
.../spark/sql/adapter/BaseSpark3Adapter.scala | 8 +-
.../HoodieFileSystemViewTableValuedFunction.scala | 1 +
.../sql/catalyst/plans/logcal/HoodieQuery.scala | 1 +
.../logcal/HoodieTimelineTableValuedFunction.scala | 1 +
.../datasources/Spark3ParsePartitionUtil.scala | 3 +-
.../Spark3ParquetSchemaEvolutionUtils.scala | 11 +-
.../sql/hudi/analysis/HoodieSpark3Analysis.scala | 2 +-
.../spark/sql/hudi/catalog/BasicStagedTable.scala | 1 +
.../spark/sql/hudi/catalog/HoodieCatalog.scala | 12 +-
.../sql/hudi/catalog/HoodieInternalV2Table.scala | 5 +-
.../spark/sql/hudi/command/AlterTableCommand.scala | 15 +-
.../sql/HoodieSpark33CatalystExpressionUtils.scala | 2 +-
.../spark/sql/HoodieSpark33CatalystPlanUtils.scala | 2 +-
.../apache/spark/sql/adapter/Spark3_3Adapter.scala | 6 +
.../apache/spark/sql/avro/AvroDeserializer.scala | 10 +-
.../org/apache/spark/sql/avro/AvroSerializer.scala | 4 +-
.../Spark33LegacyHoodieParquetFileFormat.scala | 12 +-
.../hudi/Spark33ResolveHudiAlterTableCommand.scala | 1 +
.../sql/hudi/analysis/HoodieSpark33Analysis.scala | 2 +-
.../HoodieSpark3_3ExtendedSqlAstBuilder.scala | 15 +-
.../parser/HoodieSpark3_3ExtendedSqlParser.scala | 6 +-
.../sql/HoodieSpark34CatalystExpressionUtils.scala | 2 +-
.../spark/sql/HoodieSpark34CatalystPlanUtils.scala | 2 +-
.../apache/spark/sql/adapter/Spark3_4Adapter.scala | 14 +-
.../apache/spark/sql/avro/AvroDeserializer.scala | 11 +-
.../org/apache/spark/sql/avro/AvroSerializer.scala | 5 +-
.../org/apache/spark/sql/avro/AvroUtils.scala | 9 +-
.../datasources/Spark34NestedSchemaPruning.scala | 1 +
.../Spark34LegacyHoodieParquetFileFormat.scala | 11 +-
.../hudi/Spark34ResolveHudiAlterTableCommand.scala | 1 +
.../sql/hudi/analysis/HoodieSpark34Analysis.scala | 4 +-
.../HoodieSpark3_4ExtendedSqlAstBuilder.scala | 19 +-
.../parser/HoodieSpark3_4ExtendedSqlParser.scala | 9 +-
.../sql/HoodieSpark35CatalystExpressionUtils.scala | 2 +-
.../apache/spark/sql/adapter/Spark3_5Adapter.scala | 8 +-
.../apache/spark/sql/avro/AvroDeserializer.scala | 11 +-
.../org/apache/spark/sql/avro/AvroSerializer.scala | 5 +-
.../org/apache/spark/sql/avro/AvroUtils.scala | 9 +-
.../HoodieSpark35PartitionedFileUtils.scala | 1 +
.../datasources/Spark35NestedSchemaPruning.scala | 1 +
.../parquet/Spark35DataSourceUtils.scala | 2 +-
.../Spark35LegacyHoodieParquetFileFormat.scala | 11 +-
.../datasources/parquet/Spark35ParquetReader.scala | 2 +-
.../hudi/Spark35ResolveHudiAlterTableCommand.scala | 1 +
.../HoodieSpark3_5ExtendedSqlAstBuilder.scala | 17 +-
.../parser/HoodieSpark3_5ExtendedSqlParser.scala | 9 +-
.../HoodieBulkInsertInternalWriterTestBase.java | 4 +-
.../TestHoodieBulkInsertDataInternalWriter.java | 14 +-
.../TestHoodieDataSourceInternalBatchWrite.java | 6 +-
hudi-sync/hudi-datahub-sync/pom.xml | 15 +
.../hudi/sync/datahub/DataHubTableProperties.java | 4 +-
.../sync/datahub/TestDataHubTableProperties.java | 77 ++
.../apache/hudi/hive/util/ColumnNameXLator.java | 6 +-
.../org/apache/hudi/hive/util/HiveSchemaUtil.java | 4 +-
.../org/apache/hudi/hive/TestHiveSyncTool.java | 50 ++
.../hudi/hive/testutils/HiveTestCluster.java | 8 +-
.../apache/hudi/hive/testutils/HiveTestUtil.java | 28 +-
.../org/apache/hudi/hive/SchemaDifference.java | 6 +-
.../apache/hudi/sync/common/HoodieSyncConfig.java | 2 +-
.../hudi/sync/common/HoodieSyncException.java | 2 +-
.../sync/common/metrics/HoodieMetaSyncMetrics.java | 2 +-
.../hudi/sync/common/util/SyncUtilHelpers.java | 2 +
hudi-tests-common/pom.xml | 5 +
.../hudi/timeline/service/RequestHandler.java | 23 +-
.../hudi/timeline/service/TimelineService.java | 21 +-
.../timeline/service/handlers/BaseFileHandler.java | 6 +-
.../service/handlers/FileSliceHandler.java | 6 +-
.../hudi/timeline/service/handlers/Handler.java | 13 +-
.../service/handlers/InstantStateHandler.java | 10 +-
.../timeline/service/handlers/MarkerHandler.java | 23 +-
.../timeline/service/handlers/TimelineHandler.java | 6 +-
.../AsyncTimelineServerBasedDetectionStrategy.java | 2 +-
.../MarkerBasedEarlyConflictDetectionRunnable.java | 14 +-
.../service/handlers/marker/MarkerDirState.java | 2 +-
.../timeline/service/MockHoodieHadoopStorage.java | 92 +++
.../hudi/timeline/service/TestRequestHandler.java | 147 ++++
.../hudi/timeline/service/TestTimelineService.java | 7 +-
.../service/TimelineServiceTestHarness.java | 192 +++++
.../TestRemoteHoodieTableFileSystemView.java | 100 +--
hudi-utilities/pom.xml | 27 +-
.../apache/hudi/utilities/HDFSParquetImporter.java | 12 +-
.../org/apache/hudi/utilities/HoodieCleaner.java | 7 +-
.../apache/hudi/utilities/HoodieClusteringJob.java | 2 +-
.../org/apache/hudi/utilities/HoodieCompactor.java | 4 +-
.../hudi/utilities/HoodieDataTableValidator.java | 4 +-
.../hudi/utilities/HoodieDropPartitionsTool.java | 6 +-
.../org/apache/hudi/utilities/HoodieIndexer.java | 2 +-
.../utilities/HoodieMetadataTableValidator.java | 121 +--
.../apache/hudi/utilities/HoodieRepairTool.java | 2 +-
.../hudi/utilities/HoodieSnapshotCopier.java | 4 +-
.../hudi/utilities/HoodieSnapshotExporter.java | 8 +-
.../org/apache/hudi/utilities/HoodieTTLJob.java | 7 +-
.../org/apache/hudi/utilities/TableSizeStats.java | 4 +-
.../utilities/callback/SourceCommitCallback.java | 6 +
.../kafka/HoodieWriteCommitKafkaCallback.java | 6 +-
...heckpointFromAnotherHoodieTimelineProvider.java | 5 +-
.../checkpointing/KafkaConnectHdfsProvider.java | 2 +-
.../hudi/utilities/config/KafkaSourceConfig.java | 10 +
.../ingestion/HoodieIngestionMetrics.java | 14 +-
.../hudi/utilities/perf/TimelineServerPerf.java | 4 +-
.../utilities/schema/FilebasedSchemaProvider.java | 60 +-
.../utilities/schema/JdbcbasedSchemaProvider.java | 2 +-
.../utilities/schema/SchemaRegistryProvider.java | 62 ++
.../converter/JsonToAvroSchemaConverter.java | 464 ++++++++---
.../converter/JsonToAvroSchemaConverterConfig.java | 51 ++
.../hudi/utilities/sources/AvroKafkaSource.java | 13 +-
.../hudi/utilities/sources/HoodieIncrSource.java | 9 +-
.../hudi/utilities/sources/JsonKafkaSource.java | 81 +-
.../apache/hudi/utilities/sources/KafkaSource.java | 8 +-
.../hudi/utilities/sources/ProtoKafkaSource.java | 7 +
.../org/apache/hudi/utilities/sources/Source.java | 41 +
.../utilities/sources/debezium/DebeziumSource.java | 8 +-
.../sources/helpers/DatePartitionPathSelector.java | 2 +-
.../utilities/sources/helpers/KafkaOffsetGen.java | 138 +++-
.../utilities/sources/helpers/KafkaSourceUtil.java | 50 ++
.../sources/helpers/ProtoConversionUtil.java | 3 +-
.../utilities/sources/helpers/QueryRunner.java | 9 +-
.../sources/helpers/gcs/MessageValidity.java | 2 +-
.../utilities/streamer/BaseErrorTableWriter.java | 5 +
.../hudi/utilities/streamer/BootstrapExecutor.java | 7 +-
.../streamer/HoodieMultiTableStreamer.java | 27 +-
.../hudi/utilities/streamer/HoodieStreamer.java | 17 +-
.../utilities/streamer/HoodieStreamerMetrics.java | 2 +-
.../utilities/streamer/HoodieStreamerUtils.java | 8 +
.../utilities/streamer/SourceFormatAdapter.java | 17 +-
.../utilities/streamer/SparkSampleWritesUtils.java | 4 +-
.../apache/hudi/utilities/streamer/StreamSync.java | 300 ++++---
.../streamer/StreamerCheckpointUtils.java | 5 +-
.../apache/hudi/utilities/TestHoodieIndexer.java | 15 +-
.../TestHoodieMetadataTableValidator.java | 212 ++++-
.../deltastreamer/HoodieDeltaStreamerTestBase.java | 41 +-
.../deltastreamer/TestHoodieDeltaStreamer.java | 210 ++++-
...TestHoodieDeltaStreamerErrorTableWriteFlow.java | 133 ++++
...TestHoodieDeltaStreamerSchemaEvolutionBase.java | 48 +-
...oodieDeltaStreamerSchemaEvolutionExtensive.java | 83 +-
.../TestHoodieDeltaStreamerWithMultiWriter.java | 11 +-
.../schema/TestFilebasedSchemaProvider.java | 35 +-
.../schema/TestSchemaRegistryProvider.java | 24 +
.../converter/TestJsonToAvroSchemaConverter.java | 59 +-
.../utilities/sources/BaseTestKafkaSource.java | 13 +
.../utilities/sources/TestAvroKafkaSource.java | 35 +
.../utilities/sources/TestHoodieIncrSource.java | 277 +++++--
.../utilities/sources/TestJsonKafkaSource.java | 201 ++++-
.../utilities/sources/TestProtoKafkaSource.java | 21 +-
.../hudi/utilities/sources/TestS3EventsSource.java | 41 +-
.../sources/helpers/TestKafkaOffsetGen.java | 260 +++++-
.../sources/helpers/TestKafkaSourceUtil.java | 66 ++
.../helpers/TestMercifulJsonToRowConverter.java | 129 ++-
.../streamer/TestStreamerCheckpointUtils.java | 6 +-
.../AbstractCloudObjectsSourceTestBase.java | 3 +-
.../json/array-with-item-type-union/expected.json | 78 ++
.../json/array-with-item-type-union/input.json | 93 +++
.../json/complex-json-union-types/expected.json | 66 ++
.../json/complex-json-union-types/input.json | 42 +
.../expected.json | 13 +-
.../input.json | 14 +-
.../expected_no_stripping_quotes.json | 138 ++++
.../expected_stripping_quotes.json | 138 ++++
.../json/not-null-default-value-schema/input.json | 67 ++
.../json/schema-repeating-names/expected.json | 153 ++++
.../json/schema-repeating-names/input.json | 124 +++
...e.avsc => evolved-test-with-default-value.avsc} | 5 +
..._uber.avsc => source_uber_encoded_decimal.avsc} | 35 +-
.../source_uber_encoded_decimal.json | 54 ++
packaging/bundle-validation/README.md | 23 +-
packaging/bundle-validation/ci_run.sh | 31 +-
packaging/bundle-validation/cli/commands.txt | 5 +
packaging/bundle-validation/service/read.scala | 2 -
packaging/bundle-validation/service/write.scala | 2 -
packaging/bundle-validation/validate.sh | 81 +-
packaging/hudi-cli-bundle/hudi-cli-with-bundle.sh | 8 +-
packaging/hudi-integ-test-bundle/pom.xml | 1 +
packaging/hudi-spark-bundle/pom.xml | 10 +-
packaging/hudi-utilities-bundle/pom.xml | 4 +
packaging/hudi-utilities-slim-bundle/pom.xml | 39 +-
pom.xml | 133 +++-
release/release_guide.md | 12 +-
rfc/README.md | 3 +-
scripts/jacoco/README.md | 110 +++
.../jacoco/download_jacoco.sh | 7 +-
.../generate_jacoco_coverage_report.sh} | 9 +-
.../merge_jacoco_exec_files.sh} | 26 +-
.../jacoco/merge_jacoco_job_files.sh | 10 +-
style/scalastyle.xml | 8 +-
1080 files changed, 29982 insertions(+), 8933 deletions(-)
copy .github/workflows/{release_candidate_validation.yml =>
maven_artifact_validation.yml} (80%)
rename {hudi-client/hudi-client-common/src/main/java/org/apache/hudi =>
hudi-aws/src/main/java/org/apache/hudi/aws}/metrics/cloudwatch/CloudWatchMetricsReporter.java
(96%)
rename hudi-aws/src/main/java/org/apache/hudi/aws/{ =>
metrics}/cloudwatch/CloudWatchReporter.java (94%)
delete mode 100644
hudi-aws/src/main/java/org/apache/hudi/aws/utils/S3Utils.java
rename {hudi-client/hudi-client-common/src/test/java/org/apache/hudi =>
hudi-aws/src/test/java/org/apache/hudi/aws}/metrics/cloudwatch/TestCloudWatchMetricsReporter.java
(53%)
rename hudi-aws/src/test/java/org/apache/hudi/aws/{ =>
metrics}/cloudwatch/TestCloudWatchReporter.java (87%)
delete mode 100644
hudi-aws/src/test/java/org/apache/hudi/aws/utils/TestS3Utils.java
delete mode 100755 hudi-cli/hudi-cli.sh
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/CommitMetadataResolver.java
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/CommitMetadataResolverFactory.java
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/MarkerBasedCommitMetadataResolver.java
copy
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/{table/upgrade/SupportsUpgradeDowngrade.java
=> client/NoOpCommitMetadataResolver.java} (55%)
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/ConcurrentSchemaEvolutionTableSchemaGetter.java
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/SchemaConflictResolutionStrategy.java
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/SimpleSchemaConflictResolutionStrategy.java
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/StorageLock.java
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/models/HeartbeatManager.java
copy
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/{exception/HoodieCompactionException.java
=> client/transaction/lock/models/LockGetResult.java} (66%)
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/models/LockProviderHeartbeatManager.java
copy
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/{exception/HoodieClusteringException.java
=> client/transaction/lock/models/LockUpdateResult.java} (63%)
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/models/StorageLockData.java
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/models/StorageLockFile.java
delete mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/utils/MetadataTableUtils.java
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/estimator/AverageRecordSizeEstimator.java
copy
hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamContext.java
=>
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/estimator/RecordSizeEstimator.java
(58%)
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/estimator/RecordSizeEstimatorFactory.java
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriterTableVersionSix.java
rename
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/{BaseRollbackHelper.java
=> RollbackHelper.java} (96%)
copy
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/{upgrade/FiveToFourDowngradeHandler.java
=> action/rollback/RollbackHelperFactory.java} (60%)
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/RollbackHelperV1.java
copy
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/{client/timeline/HoodieTimelineArchiver.java
=> table/marker/AppendMarkerHandler.java} (58%)
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/DirectWriteMarkersV1.java
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/TimelineServerBasedWriteMarkersV1.java
create mode 100644
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/TestBaseHoodieTableServiceClient.java
create mode 100644
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/TestBaseHoodieWriteClient.java
create mode 100644
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/TestConcurrentSchemaEvolutionTableSchemaGetter.java
create mode 100644
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/TestHoodieLockMetrics.java
create mode 100644
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/TestSimpleSchemaConflictResolutionStrategy.java
create mode 100644
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/lock/models/StorageLockFileTest.java
create mode 100644
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/lock/models/TestLockProviderHeartbeatManager.java
create mode 100644
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/utils/TestCommitMetadataResolver.java
rename
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/common/table/timeline/{TestHoodieArchivedTimeline.java
=> TestArchivedTimelineV2.java} (87%)
copy hudi-io/src/test/java/org/apache/hudi/io/util/TestHFileUtils.java =>
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/config/TestHoodieErrorTableConfig.java
(51%)
create mode 100644
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/estimator/TestAverageRecordSizeEstimator.java
create mode 100644
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/estimator/TestRecordSizeEstimatorFactory.java
create mode 100644
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metadata/TestHoodieBackedTableMetadataWriter.java
create mode 100644
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/TestBaseHoodieTable.java
create mode 100644
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/TestHoodieTable.java
create mode 100644
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/clean/TestCleanPlanActionExecutor.java
create mode 100644
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/compact/TestScheduleCompactionActionExecutor.java
rename
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/rollback/{TestBaseRollbackHelper.java
=> TestRollbackHelper.java} (98%)
copy
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/{SparkHoodieBackedTableMetadataWriter.java
=> SparkHoodieBackedTableMetadataWriterTableVersionSix.java} (51%)
create mode 100644
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkMetadataWriterFactory.java
delete mode 100644
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/AverageRecordSizeUtils.java
create mode 100644
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieFileSystemViews.java
create mode 100644
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/data/TestHoodieJavaPairRDD.java
create mode 100644
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestHoodieSparkTable.java
delete mode 100644
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/commit/TestAverageRecordSizeUtils.java
copy
hudi-io/src/main/java/org/apache/hudi/io/ByteArraySeekableDataInputStream.java
=>
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/marker/TestDirectWriteMarkersV1.java
(58%)
create mode 100644
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/marker/TestTimelineServerBasedWriteMarkersV1.java
create mode 100644
hudi-common/src/main/java/org/apache/hudi/avro/AvroSchemaCache.java
create mode 100644
hudi-common/src/main/java/org/apache/hudi/avro/AvroSchemaComparatorForSchemaEvolution.java
copy
hudi-common/src/main/java/org/apache/hudi/common/{function/SerializableSupplier.java
=> serialization/CustomSerializer.java} (74%)
copy
hudi-common/src/main/java/org/apache/hudi/common/{table/timeline/CommitMetadataSerDe.java
=> serialization/DefaultSerializer.java} (61%)
create mode 100644
hudi-common/src/main/java/org/apache/hudi/common/serialization/HoodieFileGroupSerializer.java
copy
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/PartitionRegexBasedCompactionStrategy.java
=>
hudi-common/src/main/java/org/apache/hudi/common/serialization/HoodieFileSliceSerializer.java
(50%)
rename
hudi-common/src/main/java/org/apache/hudi/common/table/read/{HoodieFileGroupReaderSchemaHandler.java
=> FileGroupReaderSchemaHandler.java} (57%)
rename
hudi-common/src/main/java/org/apache/hudi/common/table/read/{HoodieBaseFileGroupRecordBuffer.java
=> FileGroupRecordBuffer.java} (91%)
rename
hudi-common/src/main/java/org/apache/hudi/common/table/read/{HoodieKeyBasedFileGroupRecordBuffer.java
=> KeyBasedFileGroupRecordBuffer.java} (75%)
rename
hudi-common/src/main/java/org/apache/hudi/common/table/read/{HoodiePositionBasedFileGroupRecordBuffer.java
=> PositionBasedFileGroupRecordBuffer.java} (82%)
rename
hudi-common/src/main/java/org/apache/hudi/common/table/read/{HoodiePositionBasedSchemaHandler.java
=> PositionBasedSchemaHandler.java} (83%)
rename
hudi-common/src/main/java/org/apache/hudi/common/table/read/{HoodieUnmergedFileGroupRecordBuffer.java
=> UnmergedFileGroupRecordBuffer.java} (94%)
create mode 100644
hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieInstantReader.java
copy
hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/{DefaultInstantGenerator.java
=> DefaultInstantComparator.java} (91%)
create mode 100644
hudi-common/src/main/java/org/apache/hudi/common/table/view/HoodieFileGroupSizeEstimator.java
create mode 100644
hudi-common/src/main/java/org/apache/hudi/common/table/view/NoOpTableMetadata.java
rename
hudi-common/src/main/java/org/apache/hudi/common/util/{AvroSchemaCache.java =>
LocalAvroSchemaCache.java} (81%)
copy
hudi-common/src/main/java/org/apache/hudi/common/util/{DefaultSizeEstimator.java
=> LogicalClock.java} (70%)
copy
hudi-common/src/main/java/org/apache/hudi/common/util/{VisibleForTesting.java
=> SystemClock.java} (77%)
copy
hudi-common/src/main/java/org/apache/hudi/exception/{CorruptedLogFileException.java
=> HoodieDuplicateDataFileDetectedException.java} (79%)
copy
hudi-common/src/main/java/org/apache/hudi/exception/{HoodieWriteConflictException.java
=> HoodieSchemaEvolutionConflictException.java} (71%)
delete mode 100644
hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataFileSystemView.java
create mode 100644
hudi-common/src/main/java/org/apache/hudi/timeline/TimelineServiceClient.java
create mode 100644
hudi-common/src/main/java/org/apache/hudi/timeline/TimelineServiceClientBase.java
create mode 100644
hudi-common/src/test/java/org/apache/hudi/avro/TestAvroSchemaComparatorForSchemaEvolution.java
create mode 100644
hudi-common/src/test/java/org/apache/hudi/common/fs/TestSizeAwareDataInputStream.java
create mode 100644
hudi-common/src/test/java/org/apache/hudi/common/serialization/TestDefaultSerializer.java
create mode 100644
hudi-common/src/test/java/org/apache/hudi/common/serialization/TestHoodieFileGroupSerializer.java
create mode 100644
hudi-common/src/test/java/org/apache/hudi/common/serialization/TestHoodieFileSliceSerializer.java
create mode 100644
hudi-common/src/test/java/org/apache/hudi/common/table/read/TestFileGroupRecordBuffer.java
delete mode 100644
hudi-common/src/test/java/org/apache/hudi/common/table/read/TestHoodieFileGroupRecordBuffer.java
create mode 100644
hudi-common/src/test/java/org/apache/hudi/common/table/timeline/versioning/BaseTestCommitMetadataSerDe.java
copy
hudi-common/src/{main/java/org/apache/hudi/common/engine/LocalTaskContextSupplier.java
=>
test/java/org/apache/hudi/common/table/timeline/versioning/v1/TestCommitMetadataSerDeV1.java}
(58%)
create mode 100644
hudi-common/src/test/java/org/apache/hudi/common/table/timeline/versioning/v2/TestCommitMetadataSerDeV2.java
create mode 100644
hudi-common/src/test/java/org/apache/hudi/common/table/view/TestHoodieFileGroupSizeEstimator.java
create mode 100644
hudi-common/src/test/java/org/apache/hudi/common/util/TestCleanerUtils.java
copy
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/index/hbase/TestHBaseIndexUsage.java
=> hudi-common/src/test/java/org/apache/hudi/common/util/TestHoodieTimer.java
(64%)
rename
hudi-common/src/test/java/org/apache/hudi/common/util/{TestAvroSchemaCache.java
=> TestLocalAvroSchemaCache.java} (50%)
copy hudi-cli/src/test/java/org/apache/hudi/cli/TestSparkUtil.java =>
hudi-common/src/test/java/org/apache/hudi/common/util/TestSystemClock.java (55%)
rename {hudi-client/hudi-client-common =>
hudi-common}/src/test/java/org/apache/hudi/metrics/TestMetricsReporterFactory.java
(75%)
create mode 100644
hudi-common/src/test/java/org/apache/hudi/timeline/TestTimelineServiceClient.java
create mode 100644
hudi-common/src/test/resources/avro-schema-evo/schema-allshapes-A.txt
create mode 100644
hudi-common/src/test/resources/avro-schema-evo/schema-allshapes-B.txt
copy hudi-common/src/test/resources/{decimal-logical-type.avsc =>
decimal-logical-type-zero-scale.avsc} (95%)
copy
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/{transform/RowDataToHoodieFunctionWithRateLimit.java
=> append/AppendWriteFunctionWithRateLimit.java} (75%)
copy
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/{transform/RowDataToHoodieFunctions.java
=> append/AppendWriteFunctions.java} (62%)
copy
hudi-flink-datasource/{hudi-flink1.17.x/src/main/java/org/apache/hudi/adapter/OperatorCoordinatorAdapter.java
=>
hudi-flink/src/test/java/org/apache/hudi/sink/utils/MockSubtaskGateway.java}
(55%)
copy
hudi-flink-datasource/{hudi-flink1.18.x/src/test/java/org/apache/hudi/adapter/TestHoodieCatalogs.java
=>
hudi-flink1.14.x/src/test/java/org/apache/hudi/adapter/ExecutionAttemptUtil.java}
(77%)
copy
hudi-flink-datasource/{hudi-flink1.18.x/src/test/java/org/apache/hudi/adapter/TestHoodieCatalogs.java
=>
hudi-flink1.15.x/src/test/java/org/apache/hudi/adapter/ExecutionAttemptUtil.java}
(77%)
copy
hudi-flink-datasource/{hudi-flink1.18.x/src/test/java/org/apache/hudi/adapter/TestHoodieCatalogs.java
=>
hudi-flink1.16.x/src/test/java/org/apache/hudi/adapter/ExecutionAttemptUtil.java}
(77%)
copy
hudi-flink-datasource/{hudi-flink1.18.x/src/test/java/org/apache/hudi/adapter/TestHoodieCatalogs.java
=>
hudi-flink1.17.x/src/test/java/org/apache/hudi/adapter/ExecutionAttemptUtil.java}
(77%)
copy
hudi-flink-datasource/hudi-flink1.18.x/src/test/java/org/apache/hudi/adapter/{TestHoodieCatalogs.java
=> ExecutionAttemptUtil.java} (77%)
copy
hudi-flink-datasource/{hudi-flink1.18.x/src/test/java/org/apache/hudi/adapter/TestHoodieCatalogs.java
=>
hudi-flink1.19.x/src/test/java/org/apache/hudi/adapter/ExecutionAttemptUtil.java}
(77%)
copy
hudi-flink-datasource/{hudi-flink1.18.x/src/test/java/org/apache/hudi/adapter/TestHoodieCatalogs.java
=>
hudi-flink1.20.x/src/test/java/org/apache/hudi/adapter/ExecutionAttemptUtil.java}
(77%)
create mode 100644
hudi-hadoop-common/src/test/java/org/apache/hudi/common/bootstrap/index/TestBaseHoodieTableFileIndex.java
create mode 100644
hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/read/TestHoodieFileGroupReaderInflightCommit.java
create mode 100644
hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/timeline/TestArchivedTimelineV1.java
copy
hudi-cli/src/test/java/org/apache/hudi/cli/testutils/HoodieCLIIntegrationTestBase.java
=> hudi-io/src/main/java/org/apache/hudi/storage/HoodieInstantWriter.java (59%)
create mode 100644
hudi-spark-datasource/hudi-spark-common/src/test/java/org/apache/hudi/TestHoodieWriterUtils.java
copy
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/optimize/TestHilbertCurveUtils.java
=>
hudi-spark-datasource/hudi-spark-common/src/test/scala/org/apache/spark/sql/hudi/TestHoodieFileIndex.scala
(63%)
rename
hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/{TestHoodiePositionBasedFileGroupRecordBuffer.java
=> TestPositionBasedFileGroupRecordBuffer.java} (96%)
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestSparkClusteringCornerCases.java
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/functional/TestHoodieSparkCopyOnWriteTableRollbackTableVersionSix.java
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/functional/TestHoodieSparkRollback.java
copy
hudi-aws/src/main/java/org/apache/hudi/aws/sync/util/GluePartitionFilterGenerator.java
=>
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestRecordLevelIndexTableVersionSix.scala
(67%)
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/analysis/TestMergeIntoHoodieTableCommand.scala
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/common/TestTableColumnTypeMismatch.scala
rename
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/{dml
=> ddl}/TestDropTable.scala (93%)
rename
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/{dml
=> ddl}/TestRepairTable.scala (94%)
rename
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/{dml
=> ddl}/TestShowPartitions.scala (91%)
rename
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/{dml
=> ddl}/TestTruncateTable.scala (92%)
rename
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/{dml
=> feature}/TestCDCForSparkSQL.scala (93%)
rename
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/{dml
=> feature}/TestCompactionTable.scala (90%)
rename
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/{dml
=> feature}/TestDataSkippingQuery.scala (99%)
rename
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/{dml
=> feature}/TestQueryMergeOnReadOptimizedTable.scala (75%)
rename
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/{command
=> feature}/index/TestExpressionIndex.scala (97%)
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/index/TestGlobalIndex.scala
rename
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/{command
=> feature}/index/TestIndexSyntax.scala (99%)
rename
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/{command
=> feature}/index/TestSecondaryIndex.scala (98%)
create mode 100644
hudi-sync/hudi-datahub-sync/src/test/java/org/apache/hudi/sync/datahub/TestDataHubTableProperties.java
create mode 100644
hudi-timeline-service/src/test/java/org/apache/hudi/timeline/service/MockHoodieHadoopStorage.java
create mode 100644
hudi-timeline-service/src/test/java/org/apache/hudi/timeline/service/TestRequestHandler.java
create mode 100644
hudi-timeline-service/src/test/java/org/apache/hudi/timeline/service/TimelineServiceTestHarness.java
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/converter/JsonToAvroSchemaConverterConfig.java
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/KafkaSourceUtil.java
create mode 100644
hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamerErrorTableWriteFlow.java
create mode 100644
hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/helpers/TestKafkaSourceUtil.java
create mode 100644
hudi-utilities/src/test/resources/schema-provider/json/array-with-item-type-union/expected.json
create mode 100644
hudi-utilities/src/test/resources/schema-provider/json/array-with-item-type-union/input.json
create mode 100644
hudi-utilities/src/test/resources/schema-provider/json/complex-json-union-types/expected.json
create mode 100644
hudi-utilities/src/test/resources/schema-provider/json/complex-json-union-types/input.json
copy
hudi-utilities/src/test/resources/schema-provider/json/{multiple-properties =>
kafka-decimal-simple}/expected.json (61%)
copy
hudi-utilities/src/test/resources/schema-provider/json/{multiple-properties =>
kafka-decimal-simple}/input.json (55%)
create mode 100644
hudi-utilities/src/test/resources/schema-provider/json/not-null-default-value-schema/expected_no_stripping_quotes.json
create mode 100644
hudi-utilities/src/test/resources/schema-provider/json/not-null-default-value-schema/expected_stripping_quotes.json
create mode 100644
hudi-utilities/src/test/resources/schema-provider/json/not-null-default-value-schema/input.json
create mode 100644
hudi-utilities/src/test/resources/schema-provider/json/schema-repeating-names/expected.json
create mode 100644
hudi-utilities/src/test/resources/schema-provider/json/schema-repeating-names/input.json
copy
hudi-utilities/src/test/resources/schema/{simple-test-with-default-value.avsc
=> evolved-test-with-default-value.avsc} (92%)
copy hudi-utilities/src/test/resources/streamer-config/{source_uber.avsc =>
source_uber_encoded_decimal.avsc} (66%)
create mode 100644
hudi-utilities/src/test/resources/streamer-config/source_uber_encoded_decimal.json
create mode 100644 packaging/bundle-validation/cli/commands.txt
create mode 100644 scripts/jacoco/README.md
copy docker/hoodie/hadoop/trinocoordinator/etc/node.properties =>
scripts/jacoco/download_jacoco.sh (82%)
mode change 100644 => 100755
copy scripts/{release/validate_source_rat.sh =>
jacoco/generate_jacoco_coverage_report.sh} (77%)
copy scripts/{release/create_source_directory.sh =>
jacoco/merge_jacoco_exec_files.sh} (64%)
copy docker/hoodie/hadoop/trinocoordinator/etc/node.properties =>
scripts/jacoco/merge_jacoco_job_files.sh (78%)
mode change 100644 => 100755
