This is an automated email from the ASF dual-hosted git repository.
vhs pushed a change to branch release-1.0.2
in repository https://gitbox.apache.org/repos/asf/hudi.git
from 1bdba3600a7 [HUDI-8949] Relaxing checkpoint validation for
HoodieStreamerSources (#12752)
new 3b4132b9066 [HUDI-8953] Update bundle validation workflow on a release
candidate (#12755)
new ad0197caf99 [HUDI-8767] Add bundle validation for release artifacts in
Maven central (#12488)
new ebd6282b367 [HUDI-8950] Explicitly set base path in sync config #12754
new 3e48496d958 [HUDI-8278] Add log message about different primary keys
for HoodieTableFactory (#12766)
new e0f5d40cfa8 [HUDI-8951] Disabled flaky
`TestHoodieDeltaStreamer::testHoodieIndexerExecutionAfterClustering` (#12769)
new 4cee31fef1e [MINOR] Fix code issues reported by SonarQube (#12767)
new 5c50ee152ee [MINOR] Move field ordering lookup outside of loop for
proto conversion (#12271)
new 4ba44acb201 [HUDI-8966] Claim RFC-88 for New
Schema/DataType/Expression Abstractions (#12791)
new e7de74a6105 [HUDI-8958] Enable record index validation by default
(#12785)
new 9ec170eb31e [MINOR] Fix jenkins compile issue (#12810)
new 4ae869e691b [HUDI-8989] Fixing Compaction scheduling for tbl v6
(#12805)
new b399bd29a54 [MINOR] Updating doap for release 1.0.1 (#12816)
new 9e480538b5a [HUDI-8569] Fix Insert overwrite / update MOR with global
index does not work (#12819)
new a21dcc34827 [HUDI-8967] Add a config to fail job if duplicate data
files detected during reconcileAgainstMarkers (#12792)
new 34632cc2d33 [HUDI-8965] Fix Hoodie Hive Sync Tool to Throw Exceptions
on Sync Failures (#12790)
new 4fa61c721f0 [HUDI-8959] HoodieMetadataTableValidator can skip
validation if data table is not found (#12775)
new 38b9d1eeaf8 [HUDI-8998] Improve handling of zero scale decimals in
MercifulJsonConverter (#12822)
new f2a653a7753 [HUDI-8922] Avoid scheduling existing rollbacks (#12715)
new 0216728a2b3 [HUDI-8941] Stop spark context with proper exit code in
HoodieStreamer (#12742)
new cc1d0976546 [HUDI-8798] Validate Metadata Table Enabled during
Archival (#12549)
new a09ab5bfe23 [HUDI-5612] Add support for using metadata table with
spillable map based file system views (#12110)
new c0746351f5f [HUDI-8885] Improve the efficiency of initializing the
metadata table and table services (#12665)
new 07ab36f5197 [HUDI-8971] Fixing metadata metrics to honor metrics
prefix (#12797)
new 0c223ec6c66 [HUDI-8992] Initial changes to allow us to use streams
instead of byte arrays (#12814)
new 0783c9fc2c2 [HUDI-9009] Fix potential race condition when listing
files by leveraging exception type #12834
new 750fa4f72fa [HUDI-8701] prevent lock metrics failure from failing
process (#12458)
new fb4046cd44f [MINOR] Register more classes with Kryo (#12268)
new 0e5c4075e1b [HUDI-8972] Fixing heart beats for failed writes in
HoodieStreamer (#12802)
new ebb7bfac7dc [MINOR] Fix skew in clustering operator (#12765)
new 16dddb7ba4e [MINOR] Rebalance Azure CI jobs (2025-02-13) (#12840)
new 4e1b1f8690e [HUDI-8985] Fix avg record size estimator to avoid parsing
all commit metadata of active timeline (#12803)
new 2d94433e9ae [HUDI-7596] Enable Jacoco code coverage report across
multiple modules (#12842)
new 763d71bc14e [HUDI-8962] Adding more logs during indexing (#12787)
new 6110f9deab7 [HUDI-7222] Fix Scala style check (#12837)
new 9f4599bc1b2 [HUDI-9038] Enable Codecov tracking and comment on code
coverage (#12845)
new 7d470db3a45 [MINOR] Rebalance Azure CI jobs (2025-02-14) (#12844)
new b7d498b2153 [HUDI-9039] Run do init table transaction only when
required (#12847)
new f14f95c538e [HUDI-7624] Fixing source read and index tagging duration
(#12789)
new 055c1cfda6f [HUDI-8882] Support Compaction/Rollback/Clean Timeline
Instant Metrics in HoodieMetrics (#12681)
new ee0c41114a0 [HUDI-8126] Persist sourceRdd to optimise writeStatus DAG
for error table (#11844)
new 7a891f0c1f8 [HUDI-8883] Add ability to configure
HoodieCompactionPlanGenerator using reflection (#12664)
new 79fd3e5de0f [HUDI-9045] Spillable Map operational improvements (#12850)
new 05800716c4f [HUDI-8442] Reduce timeline loads while rolling back
failed writes (#12164)
new 8a6c150f50b [HUDI-9016] Fix the HoodieCompactor schedule parameter
alias name #12836
new a3401684080 [MINOR] Increase timeout of hudi-utilities tests in Azure
CI (#12862)
new 77920ab959c [HUDI-8988] Implement retry logic for all HTTP calls to
timeline server including remote file system view and markers (#12804)
new 855e0b91835 [HUDI-9059] Fix ordering of getInstantTimes in
CompletionTimeQueryViewV2 (#12858)
new d6634da8c61 [HUDI-9060] Remove validations for clustering metadata
(#12860)
new 916527f45a5 [HUDI-8126] Support proto messages for spark kryo
serializer excluding DynamicMessages (#12052)
new 2921d87c429 [HUDI-8786] IntelliJ automatic code cleanup tool (#12524)
new 73e12c06b4e [HUDI-8868] Use external spillable map for
cachedAllInputFileSlices in BaseHoodieTableFileIndex (#12647)
new a1f74834285 [HUDI-8126] Use union to parallelize data and error table
writes (#12813)
new b2a739a4ebf [HUDI-8501] Improve SizeAwareDataInputStream to implement
idempotent (#12231)
new f49f102272e [HUDI-8378] Fix Avro schema deserializer failing with
schema evolution (#12111)
new 5856027c2d6 [HUDI-8955] Resolve Kafka beginning offsets with retention
to prevent OffsetOutOfRange exception (#12762)
new e9e4417c9ed [MINOR] Add StorageSchemes for Aliyun Apsara File Storage
for HDFS (#12872)
new d2724ed5806 [HUDI-9070] Fix Hudi cli bundle script to use
HUDI_CONF_DIR environment variable (#12876)
new 873821724e6 [HUDI-8901] Fix Timeline Server to process requests from
multiple storage lakes (#12696)
new d07c6b703d4 [hudi-9041] Send commit ack event when reusing current
instant (#12849)
new cd4e24ad87a [HUDI-9061] JSON to AVRO schema converter (#12864)
new ebccf2f2df3 [HUDI-9071] MDT validator can configure if log truncation
applies (#12877)
new c26fd1bee8f [HUDI-9040] Set the correct table path when renaming
tables (#12848)
new 05a4b5051cd [HUDI-8954] Reconstruct writer schema while clustering by
spark row writer (#12758)
new 1b06059e9cc [HUDI-9072] Support decimal in JsonKafkaSource (#12879)
new fdd6fe2917e [HUDI-8992] Fix serde issues in commit metadata (#12829)
new f184bce41a5 [HUDI-8940] Fix Bloom Index Partitioner to distribute keys
uniformly across partitions (#12741)
new 3fecddf6e4e [HUDI-8577] Use config default for query type in default
source in Spark (#12435)
new 5fccbd3973c [HUDI-7985] Add more test cases around timestamp and
decimal formats in Json Avro converter (#11629)
new 571217e1115 [HUDI-9065] Adding new filters to ArchivedTimelineV1
(#12869)
new 9d091248cab [HUDI-8486] Enforce data type match for required columns
in Spark SQL MERGE INTO (#12798)
new c8b2500172e [HUDI-8486] Add column mismatch test coverage (#12800)
new 0587ccd841b [HUDI-9086] Disabling failing test to unblock master
(#12890)
new 23c3440cd4b [HUDI-9093] Fix the HoodieClusteringJob duplicate
parameter alias (#12897)
new ade674b66b1 [HUDI-8992] Deprecate all byte array usage in metadata
deserialization path (#12826)
new 87ad953c5ee [HUDI-9079] Log the exception message properly to handle
the 'default' partition value migration steps (#12911)
new e98b4e1bbcd [HUDI-9092] Deprecate byte array of serialization of
instants (#12900)
new 4da5f94f4e0 [HUDI-8826] Extend MIT partial update test (#12915)
new 6ad5afc8188 [HUDI-8745] Add tests for record index and secondary index
with insert dups policy (#12914)
new 963c30f8eae [HUDI-9067] Fixing num spark tasks for clean action
(#12874)
new d04a9019a19 [HUDI-5387] Add bundle validation for hudi-cli-bundle
(#12882)
new da8d035774d [HUDI-8219] Add concurrent schema evolution conflict
detection (#12781)
new a15b932b996 [HUDI-9030] Compatibility fixes for table version 6 with
1.0 (#12888)
new 7cc283f8fde [MINOR] Fix generating file id with wrong bucket index
(#12917)
new c9f228f90f9 [HUDI-9115] Deprecate Hudi CLI script (#12916)
new 91ce8b7e945 [MINOR] Add retries and logs to merging Jacoco execution
data files (#12921)
new bf6e58cf618 [HUDI-9057] Fixing ClassNotFound issue w/
ProtoBufSchemaProvider (#12913)
new d53523e34a6 [HUDI-9086] Re-enable flaky tests and fixing spark context
not shutting down (#12909)
new 388f0e76880 [MINOR] Skip merging Jacoco execution data file from Azure
Job 1 FT (#12942)
new b93c4c0bc30 [MINOR] Fix cleaner config compatibility with older
releases (#12943)
new 7e5ed58811f [HUDI-8768] Support bloom filter options when creating
expr index using bloom filter (#12919)
new b72659f7dc5 [HUDI-9125] Pass compaction/merge related props to
HoodieBaseFileGroupRecordBuffer (#12925)
new 9dc7fa75027 [HUDI-9132] Avoid empty string row key for delete and
update operations (#12929)
new 05a308399e4 [HUDI-9068] Fix cloudwatch metrics (#12873)
new e51d39220e3 [HUDI-9141] Handle the case that dbName is empty in
HoodieFileIndex (#12885)
new b423ba5ab2b [HUDI-9152] Improve read/write/compaction performance by
reusing avro schema (#12949)
new f36a125511a [MINOR] Remove storage instance variable from
HoodieIngestionMetrics (#12937)
new debd313c9cf [HUDI-7803] Fix bundle validation on Flink 1.18 (#12959)
new db6f98741e4 Fix hadoop deps from hudi-common (#12965)
new 9def3bfb71e [MINOR] Improve error logs on mdt validation job (#12973)
new 7a734e37f47 [HUDI-9166] Static cleanup method that doesn't hold
references to DiskMap instances (#12956)
new 888776e4d03 [HUDI-9166] Introduce schema pruning for delete-record
(#12961)
new 323076cddce [HUDI-9173] Fix issue with inflight compaction and global
index lookup (#12976)
new ea15c41d835 [HUDI-9186] Remove tableState from
HoodieFileGroupReaderBasedParquetFileFormat constructor (#12981)
new 54cbfebf609 [HUDI-9168] Refactor hudi-client-common to not import
hudi-aws (#12969)
new 48b39b6df92 [HUDI-8345] Delete partition stats index for a partition
that is deleted (#12953)
new 5f4058109b8 [HUDI-7375] Enable
testLogReaderWithDifferentVersionsOfDeleteBlocks (#12668)
new ef639e514c3 [HUDI-9175] Remove the unnecessary MDT metadata check for
col_stats index config update (#12977)
new 1f7c5fca059 [HUDI-9127] Fixing completion time generation to honor
timezone from table config (#12926)
new 9f0438f8ff0 [HUDI-9170] Fixing schema projection with file group
reader (#12970)
new ee4454e356c [HUDI-9083] Fixing flakiness with multi writer test
(#12987)
new f5c2cca374a [HUDI-9013] Add backwards compatible MDT writer support
and reader support with tbl v6 (#12948)
new 8e5a7b5225b [HUDI-9120] Fix merge mode inference for table version 6
in file group reader (#12991)
new 6e73206901a [HUDI-9120] Remove HUDI-9130 code changes relating to FGR
(#12935)
new ea58f229719 [HUDI-9198] Support rate limit for append mode (#12999)
new 41438c009ee [HUDI-9207] Spark Insert Overwrite Support Row Writer
(#13014)
new a16bf51c159 [HUDI-9022] Handle records with custom delete markers in
FG reader (#12843)
new 67f72273f61 [HUDI-9215] Set partitionColumnsWithKeyGenerator based on
table version (#13025)
new c3725999f0f [HUDI-9211] Fix bug with config in DataHubSyncTool (#13018)
new b526ac5bc2c [HUDI-9216] Fallback to spark.catalog.currentDatabase if
tableConfig.databaseName is null or empty (#13026)
new f0484f33c57 [HUDI-9220] Cannot find write operation type if run inline
log compaction (#13029)
new 6b4a823eedc [HUDI-7037] Fix colstats reading for Decimal field (#12993)
new ac6e03cfe93 [HUDI-9212] Fix metadata compaction failure post downgrade
(#13022)
new 34ba8984dff [MINOR] Fix flaky test testCreateNewInstantTimes (#13032)
new fe85e957554 [HUDI-9216] Ignore validation of empty databaseName in
HoodieTableMetaClient (#13033)
new 3722417a51b [MINOR] Follow up HUDI-8803, optimize vectorized reader by
cache 'batchIdxField' (#13023)
new 8fc35f09008 [HUDI-9088] Fix unnecessary scanning of target table in
MERGE INTO on Spark (#12934)
new 96c1e9a5c66 [MINOR] Use configutils to handle defaults for all zk lock
provider configs (#13039)
new 6166ad8728a [HUDI-9227] Fix bulk insert overwrite after a failed
insert overwrite (#13041)
new 2fbef041444 [HUDI-9233] Handle empty table edge case for col stats
initialization (#13045)
new 1e414a76d76 [HUDI-9167] Remove redundant classes in
hudi-utilities-slim-bundle (#12962)
new 384a150bd16 [HUDI-9238] Fix bug with HudiIncrSource when source is
table version 6 (#13048)
new 66a6bc7ffff [HUDI-9239] Fix the bug of Spark Cache not releasing
cleanly (#13050)
new 6d2c80d2289 [HUDI-9133] Fallback to legacy schema fetching in case of
IllegalAccessError from schema registry client (#13030)
new 6337cfaffa3 [HUDI-9231] Show complete DAG for one single query in
spark web ui while inserting into hudi table (#13044)
new e53ecfe2687 [HUDI-9236] Handle markers for log files in table version
6 (#13007)
new f9055b4771d [HUDI-8409] Fixing merge mode config during upgrade and
downgrade from version 7 to 8 and back (#13046)
new 17a7dff6c82 [HUDI-8635] Support numWrites metric for compaction
(#13047)
new be5ddaed783 [MINOR]Fix typo and Add implementation class name in
interface method of HoodieRecordMerger (#13059)
new bd36831d71c [HUDI-9206] Support reading inflight instants with
HoodieLogRecordReader (#13010)
new 0484bc42f39 [HUDI-9252] BaseHoodieWriteClient should only check for
upgrade (#13073)
new 730098367ac [MINOR] Remove warning around table version six (#13080)
new 9057002fcf9 [MINOR] Fixing master for build failure (#13085)
new c5565dba756 [MINOR] Fix NumberFormatException while updating metrics
for MDT in table version 6 (#13056)
new 7ee57a9d11b [HUDI-9255] Fix inferring correct merge behavior for few
scenarios (#13079)
new 4548e1c6ee4 [HUDI-9259] Fixing marker reconciliation for failures
during deleting additional files (#13088)
new 22627ee373f [HUDI-9258] Disable partial update when global index is
used (#13086)
new 6d9a3b66f6f [HUDI-9247] Flip default value of reuse of TimeGenerator
instance (#13077)
new f74ece430d2 [MINOR] Upgrade jetty version to 9.4.57.v20241219 to fix
CVE-2024-8184 (#13058)
new e32003857d2 [MINOR] Update javadocs in MergeIntoHoodieTableCommand
(#13093)
new 6491bc75886 [MINOR] Fix logging issue with
EightToSevenDowngradeHandler (#13096)
new f098a431d3f [HUDI-9155] Add scaffolding for lock provider using
conditional writes (#12954)
new a128a55ab22 [HUDI-9156] Add lock provider heartbeat manager (#12958)
new d6deed86f41 [HUDI-9258] Disable partial update for CUSTOM merge mode
(#13092)
The 153 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.codecov.yml | 15 +-
.github/workflows/bot.yml | 2 +-
...alidation.yml => maven_artifact_validation.yml} | 26 +-
.github/workflows/release_candidate_validation.yml | 14 +-
.gitignore | 4 +-
azure-pipelines-20230430.yml | 368 ++++++---
doap_HUDI.rdf | 7 +
hudi-aws/pom.xml | 5 +
.../cloudwatch/CloudWatchMetricsReporter.java | 3 +-
.../cloudwatch/CloudWatchReporter.java | 15 +-
.../hudi/aws/sync/AWSGlueCatalogSyncClient.java | 9 +-
...amoDBBasedImplicitPartitionKeyLockProvider.java | 2 +-
.../java/org/apache/hudi/aws/utils/S3Utils.java | 27 -
.../cloudwatch/TestCloudWatchMetricsReporter.java | 61 +-
.../cloudwatch/TestCloudWatchReporter.java | 30 +-
.../org/apache/hudi/aws/utils/TestS3Utils.java | 59 --
hudi-cli/hudi-cli.sh | 31 -
.../apache/hudi/cli/commands/CleansCommand.java | 10 +-
.../apache/hudi/cli/commands/CommitsCommand.java | 64 +-
.../hudi/cli/commands/CompactionCommand.java | 24 +-
.../org/apache/hudi/cli/commands/DiffCommand.java | 18 +-
.../apache/hudi/cli/commands/ExportCommand.java | 10 +-
.../hudi/cli/commands/FileSystemViewCommand.java | 5 +-
.../apache/hudi/cli/commands/MetadataCommand.java | 10 +-
.../apache/hudi/cli/commands/RestoresCommand.java | 12 +-
.../apache/hudi/cli/commands/RollbacksCommand.java | 9 +-
.../org/apache/hudi/cli/commands/StatsCommand.java | 9 +-
.../apache/hudi/cli/commands/TimelineCommand.java | 15 +-
.../java/org/apache/hudi/cli/utils/CLIUtils.java | 6 +-
.../java/org/apache/hudi/cli/utils/CommitUtil.java | 5 +-
.../apache/hudi/cli/utils/InputStreamConsumer.java | 2 +-
.../hudi/cli/utils/SparkTempViewProvider.java | 4 +-
.../hudi/cli/commands/TestCleansCommand.java | 4 +-
.../hudi/cli/commands/TestCompactionCommand.java | 10 +-
.../cli/commands/TestFileSystemViewCommand.java | 4 +-
.../hudi/cli/commands/TestRestoresCommand.java | 7 +-
.../hudi/cli/commands/TestRollbacksCommand.java | 7 +-
.../HoodieTestCommitMetadataGenerator.java | 11 +-
.../cli/testutils/HoodieTestCommitUtilities.java | 4 +-
hudi-client/hudi-client-common/pom.xml | 12 +-
.../org/apache/hudi/async/AsyncCompactService.java | 2 +-
.../util/HoodieWriteCommitCallbackUtil.java | 4 +-
.../hudi/client/BaseHoodieTableServiceClient.java | 68 +-
.../apache/hudi/client/BaseHoodieWriteClient.java | 67 +-
.../apache/hudi/client/CommitMetadataResolver.java | 48 ++
.../hudi/client/CommitMetadataResolverFactory.java | 43 +
.../apache/hudi/client/CompactionAdminClient.java | 20 +-
.../hudi/client/HoodieColumnStatsIndexUtils.java | 47 +-
.../client/MarkerBasedCommitMetadataResolver.java | 257 ++++++
.../NoOpCommitMetadataResolver.java} | 33 +-
.../java/org/apache/hudi/client/WriteStatus.java | 16 +-
.../client/embedded/EmbeddedTimelineService.java | 7 +-
.../timeline/versioning/v1/TimelineArchiverV1.java | 2 +-
.../timeline/versioning/v2/LSMTimelineWriter.java | 5 +-
.../timeline/versioning/v2/TimelineArchiverV2.java | 2 +-
...ConcurrentSchemaEvolutionTableSchemaGetter.java | 276 +++++++
.../SchemaConflictResolutionStrategy.java | 76 ++
.../SimpleSchemaConflictResolutionStrategy.java | 176 +++++
.../lock/BaseZookeeperBasedLockProvider.java | 27 +-
.../lock/FileSystemBasedLockProvider.java | 4 +-
.../hudi/client/transaction/lock/StorageLock.java | 63 ++
...ZookeeperBasedImplicitBasePathLockProvider.java | 2 +-
.../lock/metrics/HoodieLockMetrics.java | 22 +-
.../transaction/lock/models/HeartbeatManager.java | 49 ++
.../transaction/lock/models/LockGetResult.java} | 23 +-
.../lock/models/LockProviderHeartbeatManager.java | 351 ++++++++
.../transaction/lock/models/LockUpdateResult.java} | 21 +-
.../transaction/lock/models/StorageLockData.java | 71 ++
.../transaction/lock/models/StorageLockFile.java | 133 ++++
.../utils/LegacyArchivedMetaEntryReader.java | 9 +-
.../hudi/client/utils/MetadataTableUtils.java | 45 --
.../apache/hudi/client/utils/TransactionUtils.java | 37 +-
.../org/apache/hudi/config/HoodieCleanConfig.java | 15 +-
.../apache/hudi/config/HoodieCompactionConfig.java | 12 +-
.../apache/hudi/config/HoodieErrorTableConfig.java | 11 +
.../org/apache/hudi/config/HoodieIndexConfig.java | 16 +-
.../apache/hudi/config/HoodiePayloadConfig.java | 2 +-
.../org/apache/hudi/config/HoodieWriteConfig.java | 85 +-
.../hudi/estimator/AverageRecordSizeEstimator.java | 133 ++++
.../apache/hudi/estimator/RecordSizeEstimator.java | 28 +-
.../hudi/estimator/RecordSizeEstimatorFactory.java | 45 ++
.../org/apache/hudi/index/HoodieIndexUtils.java | 2 +-
.../hudi/index/bloom/BloomIndexFileInfo.java | 10 +-
.../apache/hudi/index/bloom/HoodieBloomIndex.java | 1 +
.../index/bucket/ConsistentBucketIndexUtils.java | 7 +-
.../hbase/DefaultHBaseQPSResourceAllocator.java | 2 +-
.../java/org/apache/hudi/io/HoodieCDCLogger.java | 5 +-
.../java/org/apache/hudi/io/HoodieMergeHandle.java | 5 +-
.../org/apache/hudi/io/HoodieMergedReadHandle.java | 4 +-
.../java/org/apache/hudi/io/HoodieWriteHandle.java | 10 +-
.../metadata/HoodieBackedTableMetadataWriter.java | 156 ++--
...ieBackedTableMetadataWriterTableVersionSix.java | 343 ++++++++
.../hudi/metadata/HoodieMetadataWriteUtils.java | 2 +
.../hudi/metadata/HoodieTableMetadataWriter.java | 15 +-
.../org/apache/hudi/metrics/HoodieMetrics.java | 123 ++-
.../java/org/apache/hudi/table/HoodieTable.java | 62 +-
.../org/apache/hudi/table/WorkloadProfile.java | 12 +-
.../java/org/apache/hudi/table/WorkloadStat.java | 12 +-
.../action/BaseTableServicePlanActionExecutor.java | 3 +-
.../table/action/clean/CleanActionExecutor.java | 18 +-
.../action/clean/CleanPlanActionExecutor.java | 37 +-
.../hudi/table/action/clean/CleanPlanner.java | 37 +-
.../cluster/ClusteringPlanActionExecutor.java | 20 +-
.../cluster/strategy/ClusteringPlanStrategy.java | 16 +-
.../action/commit/BaseCommitActionExecutor.java | 46 +-
.../hudi/table/action/commit/BaseWriteHelper.java | 5 +-
.../hudi/table/action/commit/BucketInfo.java | 10 +-
.../table/action/commit/HoodieDeleteHelper.java | 5 +-
.../hudi/table/action/commit/InsertBucket.java | 8 +-
.../apache/hudi/table/action/commit/SmallFile.java | 8 +-
.../hudi/table/action/compact/CompactHelpers.java | 19 +-
.../compact/RunCompactionActionExecutor.java | 2 +-
.../compact/ScheduleCompactionActionExecutor.java | 56 +-
.../BaseHoodieCompactionPlanGenerator.java | 4 +
.../strategy/CompositeCompactionStrategy.java | 2 +-
.../action/index/AbstractIndexingCatchupTask.java | 7 +-
.../index/RecordBasedIndexingCatchupTask.java | 4 +-
.../table/action/index/RunIndexActionExecutor.java | 7 +-
.../action/index/ScheduleIndexActionExecutor.java | 7 +-
.../index/WriteStatBasedIndexingCatchupTask.java | 4 +-
.../action/restore/BaseRestoreActionExecutor.java | 2 +-
.../hudi/table/action/restore/RestoreUtils.java | 4 +-
.../rollback/BaseRollbackActionExecutor.java | 16 +-
.../rollback/BaseRollbackPlanActionExecutor.java | 8 +-
.../rollback/ListingBasedRollbackStrategy.java | 105 ++-
.../rollback/MarkerBasedRollbackStrategy.java | 39 +-
.../action/rollback/RestorePlanActionExecutor.java | 8 +-
...BaseRollbackHelper.java => RollbackHelper.java} | 13 +-
.../rollback/RollbackHelperFactory.java} | 22 +-
.../table/action/rollback/RollbackHelperV1.java | 389 +++++++++
.../hudi/table/action/rollback/RollbackUtils.java | 4 +-
.../action/savepoint/SavepointActionExecutor.java | 18 +-
.../marker/AppendMarkerHandler.java} | 23 +-
.../hudi/table/marker/DirectWriteMarkers.java | 60 +-
.../hudi/table/marker/DirectWriteMarkersV1.java | 103 +++
.../marker/TimelineServerBasedWriteMarkers.java | 48 +-
.../marker/TimelineServerBasedWriteMarkersV1.java | 74 ++
.../org/apache/hudi/table/marker/WriteMarkers.java | 19 +
.../hudi/table/marker/WriteMarkersFactory.java | 31 +-
.../org/apache/hudi/table/repair/RepairUtils.java | 15 +-
.../hudi/table/upgrade/DowngradeHandler.java | 7 +-
.../upgrade/EightToSevenDowngradeHandler.java | 70 +-
.../table/upgrade/FiveToFourDowngradeHandler.java | 6 +-
.../table/upgrade/FourToFiveUpgradeHandler.java | 22 +-
.../table/upgrade/FourToThreeDowngradeHandler.java | 6 +-
.../table/upgrade/OneToZeroDowngradeHandler.java | 5 +-
.../table/upgrade/SevenToEightUpgradeHandler.java | 73 +-
.../table/upgrade/SevenToSixDowngradeHandler.java | 6 +-
.../table/upgrade/SixToFiveDowngradeHandler.java | 7 +-
.../table/upgrade/ThreeToTwoDowngradeHandler.java | 6 +-
.../table/upgrade/TwoToOneDowngradeHandler.java | 5 +-
.../hudi/table/upgrade/UpgradeDowngrade.java | 70 +-
.../hudi/table/upgrade/UpgradeDowngradeUtils.java | 2 +-
.../table/upgrade/ZeroToOneUpgradeHandler.java | 4 +-
.../org/apache/hudi/util/HttpRequestClient.java | 2 +-
.../org/apache/hudi/HoodieTestCommitGenerator.java | 12 +-
.../client/TestBaseHoodieTableServiceClient.java | 265 +++++++
.../hudi/client/TestBaseHoodieWriteClient.java | 221 ++++++
.../embedded/TestEmbeddedTimelineService.java | 14 +-
.../timeline/TestCompletionTimeQueryView.java | 28 +-
...ConcurrentSchemaEvolutionTableSchemaGetter.java | 625 +++++++++++++++
.../TestConflictResolutionStrategyUtil.java | 4 +-
.../client/transaction/TestHoodieLockMetrics.java | 70 ++
...TestSimpleSchemaConflictResolutionStrategy.java | 274 +++++++
.../lock/models/StorageLockFileTest.java | 165 ++++
.../models/TestLockProviderHeartbeatManager.java | 481 +++++++++++
.../client/utils/TestCommitMetadataResolver.java | 242 ++++++
...edTimeline.java => TestArchivedTimelineV2.java} | 14 +-
.../common/testutils/HoodieMetadataTestTable.java | 7 +-
.../hudi/config/TestHoodieErrorTableConfig.java | 36 +-
.../apache/hudi/config/TestHoodieWriteConfig.java | 14 +-
.../estimator/TestAverageRecordSizeEstimator.java | 275 +++++++
.../estimator/TestRecordSizeEstimatorFactory.java | 72 ++
.../TestHoodieBackedTableMetadataWriter.java | 111 +++
.../org/apache/hudi/metrics/TestHoodieMetrics.java | 86 +-
.../org/apache/hudi/table/TestBaseHoodieTable.java | 202 +++++
.../org/apache/hudi/table/TestHoodieTable.java | 132 ++++
.../apache/hudi/table/action/TestCleanPlanner.java | 99 +--
.../action/clean/TestCleanPlanActionExecutor.java | 159 ++++
.../TestScheduleCompactionActionExecutor.java | 80 ++
.../rollback/TestMarkerBasedRollbackStrategy.java | 17 +-
...RollbackHelper.java => TestRollbackHelper.java} | 6 +-
.../hudi/table/marker/TestWriteMarkersFactory.java | 71 +-
.../upgrade/TestEightToSevenDowngradeHandler.java | 45 +-
.../upgrade/TestSevenToEightUpgradeHandler.java | 60 +-
.../hudi/utils/HoodieWriterClientTestHarness.java | 28 +-
.../hudi/utils/TestMetadataConversionUtils.java | 53 +-
.../hudi/client/FlinkTaskContextSupplier.java | 12 +-
.../hudi/client/HoodieFlinkTableServiceClient.java | 13 +-
.../apache/hudi/client/HoodieFlinkWriteClient.java | 6 +-
...inkSizeBasedClusteringPlanStrategyRecently.java | 3 +-
.../hudi/execution/ExplicitWriteHandler.java | 2 +-
.../row/parquet/ParquetSchemaConverter.java | 6 +-
.../FlinkHoodieBackedTableMetadataWriter.java | 6 +-
.../commit/BaseFlinkCommitActionExecutor.java | 9 +-
.../FlinkDeletePartitionCommitActionExecutor.java | 3 +-
.../commit/FlinkInsertCommitActionExecutor.java | 2 +-
.../commit/FlinkUpsertCommitActionExecutor.java | 2 +-
.../table/upgrade/FlinkUpgradeDowngradeHelper.java | 3 +-
.../hudi/client/HoodieJavaTableServiceClient.java | 19 -
.../apache/hudi/client/HoodieJavaWriteClient.java | 3 +-
.../client/common/JavaTaskContextSupplier.java | 10 +
.../bulkinsert/JavaGlobalSortPartitioner.java | 14 +-
.../commit/BaseJavaCommitActionExecutor.java | 7 +-
.../commit/JavaInsertCommitActionExecutor.java | 2 +-
.../commit/JavaUpsertCommitActionExecutor.java | 2 +-
.../table/action/commit/JavaUpsertPartitioner.java | 13 +-
.../hudi/client/TestJavaHoodieBackedMetadata.java | 16 +-
.../testutils/HoodieJavaClientTestHarness.java | 19 +-
hudi-client/hudi-spark-client/pom.xml | 14 +
.../apache/hudi/client/HoodieSparkCompactor.java | 2 +-
.../org/apache/hudi/client/SparkRDDReadClient.java | 2 +-
.../hudi/client/SparkRDDTableServiceClient.java | 19 -
.../apache/hudi/client/SparkRDDWriteClient.java | 9 +-
.../hudi/client/SparkTaskContextSupplier.java | 10 +
.../MultipleSparkJobExecutionStrategy.java | 17 +-
...onsistentBucketClusteringExecutionStrategy.java | 3 +-
.../SparkSingleFileSortExecutionStrategy.java | 3 +-
.../SparkSortAndSizeExecutionStrategy.java | 3 +-
.../client/common/HoodieSparkEngineContext.java | 4 +-
.../client/utils/SparkMetadataWriterUtils.java | 32 +-
.../client/validator/SparkPreCommitValidator.java | 8 +-
.../org/apache/hudi/data/HoodieJavaPairRDD.java | 8 +
.../bulkinsert/BulkInsertMapFunction.java | 14 +-
.../bulkinsert/GlobalSortPartitioner.java | 6 +-
.../bulkinsert/RDDPartitionSortPartitioner.java | 9 +-
.../bloom/BucketizedBloomCheckPartitioner.java | 2 +-
.../index/bloom/SparkHoodieBloomIndexHelper.java | 21 +
.../expression/ExpressionIndexSparkFunctions.java | 10 +-
.../expression/HoodieSparkExpressionIndex.java | 15 +-
.../hudi/index/hbase/SparkHoodieHBaseIndex.java | 37 +-
...HoodieSparkFileGroupReaderBasedMergeHandle.java | 21 +-
.../hudi/io/storage/HoodieSparkParquetReader.java | 5 +-
.../apache/hudi/merge/SparkRecordMergingUtils.java | 5 +-
...eBackedTableMetadataWriterTableVersionSix.java} | 103 +--
.../hudi/metadata/SparkMetadataWriterFactory.java | 55 ++
.../apache/hudi/metrics/DistributedRegistry.java | 2 +-
.../apache/hudi/sort/SpaceCurveSortingHelper.java | 22 +-
.../org/apache/hudi/table/HoodieSparkTable.java | 6 +-
.../action/commit/AverageRecordSizeUtils.java | 91 ---
.../commit/BaseSparkCommitActionExecutor.java | 27 +-
.../action/commit/SparkBucketIndexPartitioner.java | 2 +-
.../SparkDeletePartitionCommitActionExecutor.java | 6 +-
.../table/action/commit/UpsertPartitioner.java | 14 +-
.../hudi/HoodieDatasetBulkInsertHelper.scala | 47 +-
.../SparkFileFormatInternalRowReaderContext.scala | 6 +-
.../org/apache/hudi/util/SparkKeyGenUtils.scala | 6 +-
.../apache/spark/HoodieSparkKryoRegistrar.scala | 11 +-
.../spark/sql/HoodieCatalystExpressionUtils.scala | 1 -
.../apache/spark/sql/HoodieInternalRowUtils.scala | 5 +
.../org/apache/spark/sql/hudi/SparkAdapter.scala | 11 +
.../hudi/client/TestCompactionAdminClient.java | 10 +-
.../java/org/apache/hudi/client/TestSavepoint.java | 11 +-
.../hudi/client/TestSparkRDDWriteClient.java | 36 +
.../common/TestHoodieSparkEngineContext.java | 57 ++
.../functional/TestExternalPathHandling.java | 12 +-
.../functional/TestHoodieBackedTableMetadata.java | 87 +-
.../TestHoodieClientOnCopyOnWriteStorage.java | 5 +-
.../functional/TestHoodieFileSystemViews.java | 299 +++++++
.../hudi/client/functional/TestHoodieIndex.java | 12 +-
.../client/functional/TestHoodieMetadataBase.java | 3 +-
.../TestSavepointRestoreCopyOnWrite.java | 2 +-
.../apache/hudi/data/TestHoodieJavaPairRDD.java | 110 +++
.../hudi/index/bloom/TestHoodieBloomIndex.java | 58 +-
.../expression/TestHoodieSparkExpressionIndex.java | 90 ++-
.../apache/hudi/io/TestHoodieTimelineArchiver.java | 37 +-
.../java/org/apache/hudi/table/TestCleaner.java | 11 +-
.../apache/hudi/table/TestHoodieSparkTable.java | 136 ++++
.../clean/TestCleanerInsertAndCleanByCommits.java | 4 +-
.../clean/TestCleanerInsertAndCleanByVersions.java | 6 +-
.../action/commit/TestAverageRecordSizeUtils.java | 216 -----
.../table/action/commit/TestUpsertPartitioner.java | 92 ---
.../table/action/compact/CompactionTestBase.java | 6 +-
.../table/functional/TestCleanActionExecutor.java | 8 +-
.../table/functional/TestCleanPlanExecutor.java | 3 +-
.../TestMarkerBasedRollbackStrategy.java | 27 +-
.../hudi/table/marker/TestDirectWriteMarkers.java | 73 +-
.../table/marker/TestDirectWriteMarkersV1.java | 35 +-
.../TestTimelineServerBasedWriteMarkers.java | 128 ++-
.../TestTimelineServerBasedWriteMarkersV1.java | 52 ++
.../hudi/table/marker/TestWriteMarkersBase.java | 73 +-
.../hudi/table/upgrade/TestUpgradeDowngrade.java | 39 +
.../hudi/testutils/HoodieCleanerTestBase.java | 7 +-
.../hudi/testutils/HoodieClientTestUtils.java | 12 +-
.../testutils/HoodieSparkClientTestHarness.java | 2 +-
.../SparkClientFunctionalTestHarness.java | 11 +-
hudi-common/pom.xml | 11 +-
.../src/main/avro/HoodieCommitMetadata.avsc | 70 ++
.../src/main/avro/HoodieReplaceCommitMetadata.avsc | 5 +
.../org/apache/hudi/BaseHoodieTableFileIndex.java | 33 +-
.../java/org/apache/hudi/avro/AvroSchemaCache.java | 47 ++
.../AvroSchemaComparatorForSchemaEvolution.java | 343 ++++++++
.../apache/hudi/avro/AvroSchemaCompatibility.java | 4 +-
.../java/org/apache/hudi/avro/HoodieAvroUtils.java | 30 +-
.../apache/hudi/avro/MercifulJsonConverter.java | 3 +-
.../processors/DecimalLogicalTypeProcessor.java | 37 +-
.../org/apache/hudi/avro/processors/Parser.java | 2 +-
.../org/apache/hudi/common/HoodieJsonPayload.java | 2 +-
.../org/apache/hudi/common/bloom/HashFunction.java | 6 +-
.../hudi/common/bootstrap/FileStatusUtils.java | 2 +-
.../index/hfile/HFileBootstrapIndexReader.java | 6 +-
.../hudi/common/config/HoodieCommonConfig.java | 14 +
.../hudi/common/config/HoodieStorageConfig.java | 2 +-
.../common/config/HoodieTimeGeneratorConfig.java | 2 +-
.../org/apache/hudi/common/data/HoodieData.java | 3 +-
.../hudi/common/data/HoodieListPairData.java | 27 +
.../apache/hudi/common/data/HoodiePairData.java | 12 +
.../hudi/common/engine/HoodieEngineContext.java | 2 +-
.../hudi/common/engine/HoodieReaderContext.java | 33 +-
.../common/engine/LocalTaskContextSupplier.java | 9 +
.../hudi/common/engine/TaskContextSupplier.java | 7 +
.../hudi/common/fs/ConsistencyGuardConfig.java | 10 +-
.../java/org/apache/hudi/common/fs/FSUtils.java | 13 +-
.../hudi/common/fs/SizeAwareDataInputStream.java | 14 +-
.../hudi/common/fs/SizeAwareDataOutputStream.java | 4 +-
.../org/apache/hudi/common/model/ActionType.java | 15 +-
.../hudi/common/model/ConsistentHashingNode.java | 8 +-
.../common/model/DefaultHoodieRecordPayload.java | 2 +-
.../org/apache/hudi/common/model/FileSlice.java | 12 +-
.../hudi/common/model/HoodieCommitMetadata.java | 14 +-
.../apache/hudi/common/model/HoodieFileGroup.java | 30 +-
.../org/apache/hudi/common/model/HoodieKey.java | 8 +-
.../hudi/common/model/HoodiePartitionMetadata.java | 5 +-
.../org/apache/hudi/common/model/HoodieRecord.java | 10 +-
.../common/model/HoodieRecordGlobalLocation.java | 12 +-
.../hudi/common/model/HoodieRecordLocation.java | 10 +-
.../hudi/common/model/HoodieRecordMerger.java | 38 +-
.../common/model/HoodieReplaceCommitMetadata.java | 32 +-
.../java/org/apache/hudi/common/model/IOType.java | 1 +
.../model/OverwriteWithLatestAvroPayload.java | 2 +-
.../common/model/PartialUpdateAvroPayload.java | 2 +-
.../hudi/common/model/RewriteAvroPayload.java | 2 +-
.../hudi/common/model/WriteOperationType.java | 4 +-
.../CustomSerializer.java} | 16 +-
.../DefaultSerializer.java} | 23 +-
.../serialization/HoodieFileGroupSerializer.java | 88 +++
.../serialization/HoodieFileSliceSerializer.java | 33 +-
.../hudi/common/table/HoodieTableConfig.java | 8 +-
.../hudi/common/table/HoodieTableMetaClient.java | 43 +-
.../hudi/common/table/TableSchemaResolver.java | 171 +---
.../hudi/common/table/cdc/HoodieCDCExtractor.java | 2 +-
.../common/table/cdc/HoodieCDCInferenceCase.java | 2 +-
.../table/log/AbstractHoodieLogRecordScanner.java | 6 +-
.../table/log/BaseHoodieLogRecordReader.java | 29 +-
.../hudi/common/table/log/HoodieLogFileReader.java | 2 +-
.../table/log/HoodieLogFormatReverseReader.java | 2 +-
.../table/log/HoodieMergedLogRecordReader.java | 29 +-
.../table/log/HoodieMergedLogRecordScanner.java | 15 +-
.../common/table/log/block/HoodieDataBlock.java | 13 +-
.../common/table/log/block/HoodieDeleteBlock.java | 4 +-
.../table/log/block/HoodieParquetDataBlock.java | 5 +-
.../hudi/common/table/marker/MarkerOperation.java | 2 +
...dler.java => FileGroupReaderSchemaHandler.java} | 147 +++-
...ecordBuffer.java => FileGroupRecordBuffer.java} | 76 +-
.../common/table/read/HoodieFileGroupReader.java | 59 +-
.../table/read/HoodieFileGroupRecordBuffer.java | 5 +
...fer.java => KeyBasedFileGroupRecordBuffer.java} | 47 +-
...ava => PositionBasedFileGroupRecordBuffer.java} | 74 +-
...andler.java => PositionBasedSchemaHandler.java} | 18 +-
...fer.java => UnmergedFileGroupRecordBuffer.java} | 9 +-
.../table/timeline/ArchivedTimelineLoader.java | 6 +-
.../common/table/timeline/BaseHoodieTimeline.java | 149 ++--
.../common/table/timeline/CommitMetadataSerDe.java | 26 +-
.../table/timeline/CompletionTimeQueryView.java | 20 +-
.../table/timeline/HoodieActiveTimeline.java | 194 ++---
.../table/timeline/HoodieArchivedTimeline.java | 49 +-
.../common/table/timeline/HoodieInstantReader.java | 55 ++
.../table/timeline/HoodieInstantTimeGenerator.java | 10 +-
.../hudi/common/table/timeline/HoodieTimeline.java | 220 +++++-
.../table/timeline/MetadataConversionUtils.java | 295 +++----
.../common/table/timeline/TimelineFactory.java | 5 +-
.../table/timeline/TimelineMetadataUtils.java | 138 +---
.../hudi/common/table/timeline/TimelineUtils.java | 108 ++-
.../common/table/timeline/dto/TimelineDTO.java | 6 +-
...enerator.java => DefaultInstantComparator.java} | 4 +-
.../timeline/versioning/v1/ActiveTimelineV1.java | 236 +++---
.../versioning/v1/ArchivedTimelineLoaderV1.java | 29 +-
.../timeline/versioning/v1/ArchivedTimelineV1.java | 172 +++-
.../timeline/versioning/v1/BaseTimelineV1.java | 26 +-
.../versioning/v1/CommitMetadataSerDeV1.java | 62 +-
.../versioning/v1/CompletionTimeQueryViewV1.java | 2 +-
.../timeline/versioning/v1/TimelineV1Factory.java | 9 +-
.../timeline/versioning/v2/ActiveTimelineV2.java | 250 +++---
.../timeline/versioning/v2/ArchivedTimelineV2.java | 38 +-
.../timeline/versioning/v2/BaseTimelineV2.java | 26 +-
.../versioning/v2/CommitMetadataSerDeV2.java | 90 ++-
.../versioning/v2/CompletionTimeQueryViewV2.java | 3 +-
.../timeline/versioning/v2/TimelineV2Factory.java | 9 +-
.../table/view/AbstractTableFileSystemView.java | 108 +--
.../common/table/view/FileSystemViewManager.java | 97 ++-
.../table/view/HoodieFileGroupSizeEstimator.java | 44 ++
.../table/view/HoodieTableFileSystemView.java | 28 +-
.../view/HoodieTablePreCommitFileSystemView.java | 10 +-
.../IncrementalTimelineSyncFileSystemView.java | 18 +-
.../hudi/common/table/view/NoOpTableMetadata.java | 148 ++++
.../table/view/PriorityBasedFileSystemView.java | 14 +-
.../view/RemoteHoodieTableFileSystemView.java | 56 +-
.../table/view/RocksDbBasedFileSystemView.java | 35 +-
.../view/SpillableMapBasedFileSystemView.java | 113 ++-
.../org/apache/hudi/common/util/CleanerUtils.java | 49 +-
.../apache/hudi/common/util/ClusteringUtils.java | 62 +-
.../org/apache/hudi/common/util/CommitUtils.java | 12 +-
.../apache/hudi/common/util/CompactionUtils.java | 23 +-
.../org/apache/hudi/common/util/ConfigUtils.java | 24 +-
.../common/util/HoodieCommonKryoRegistrar.java | 6 +-
.../hudi/common/util/InternalSchemaCache.java | 43 +-
...oSchemaCache.java => LocalAvroSchemaCache.java} | 14 +-
...DefaultSizeEstimator.java => LogicalClock.java} | 16 +-
.../org/apache/hudi/common/util/MarkerUtils.java | 7 +-
.../{VisibleForTesting.java => SystemClock.java} | 14 +-
.../common/util/collection/BitCaskDiskMap.java | 49 +-
.../hudi/common/util/collection/DiskMap.java | 77 +-
.../util/collection/ExternalSpillableMap.java | 35 +-
.../common/util/collection/LazyFileIterable.java | 17 +-
.../apache/hudi/common/util/collection/Pair.java | 2 +-
.../hudi/common/util/collection/RocksDBDAO.java | 55 +-
.../common/util/collection/RocksDbDiskMap.java | 12 +-
.../apache/hudi/common/util/collection/Triple.java | 4 +-
.../apache/hudi/common/util/hash/JenkinsHash.java | 12 +-
.../apache/hudi/common/util/hash/MurmurHash.java | 8 +-
.../common/util/queue/DisruptorMessageQueue.java | 2 +-
.../hudi/config/metrics/HoodieMetricsConfig.java | 5 +
.../metrics/HoodieMetricsPrometheusConfig.java | 2 +-
... HoodieDuplicateDataFileDetectedException.java} | 9 +-
...=> HoodieSchemaEvolutionConflictException.java} | 15 +-
.../org/apache/hudi/expression/Predicates.java | 6 +-
.../index/expression/HoodieExpressionIndex.java | 19 +
.../hudi/index/secondary/HoodieSecondaryIndex.java | 1 -
.../internal/schema/InternalSchemaBuilder.java | 1 -
.../org/apache/hudi/internal/schema/Types.java | 3 +-
.../schema/action/InternalSchemaChangeApplier.java | 4 +-
.../hudi/internal/schema/action/TableChange.java | 4 +-
.../io/FileBasedInternalSchemaStorageManager.java | 6 +-
.../internal/schema/utils/InternalSchemaUtils.java | 6 +-
.../hudi/internal/schema/utils/SerDeHelper.java | 4 +-
.../metadata/FileSystemBackedTableMetadata.java | 21 +
.../hudi/metadata/HoodieBackedTableMetadata.java | 53 +-
.../metadata/HoodieMetadataFileSystemView.java | 123 ---
.../hudi/metadata/HoodieMetadataMetrics.java | 8 +-
.../apache/hudi/metadata/HoodieTableMetadata.java | 7 +
.../hudi/metadata/HoodieTableMetadataUtil.java | 146 ++--
.../hudi/metadata/MetadataPartitionType.java | 5 +-
.../SecondaryIndexRecordGenerationUtils.java | 3 +-
.../hudi/metrics/MetricsGraphiteReporter.java | 4 +-
.../hudi/metrics/MetricsReporterFactory.java | 2 +-
.../custom/CustomizableMetricsReporter.java | 4 +-
.../hudi/metrics/datadog/DatadogReporter.java | 2 +-
.../hudi/timeline/TimelineServiceClient.java | 76 ++
.../hudi/timeline/TimelineServiceClientBase.java | 140 ++++
.../hudi/avro/MercifulJsonConverterTestBase.java | 215 ++++-
...TestAvroSchemaComparatorForSchemaEvolution.java | 499 ++++++++++++
.../org/apache/hudi/avro/TestHoodieAvroUtils.java | 71 +-
.../hudi/avro/TestMercifulJsonConverter.java | 149 +++-
.../common/data/TestHoodieListDataPairData.java | 34 +
.../common/fs/TestSizeAwareDataInputStream.java | 115 +++
.../common/model/TestHoodieCommitMetadata.java | 82 +-
.../model/TestHoodieReplaceCommitMetadata.java | 1 -
.../serialization/TestDefaultSerializer.java | 52 ++
.../TestHoodieFileGroupSerializer.java | 82 ++
.../TestHoodieFileSliceSerializer.java | 65 ++
.../hudi/common/table/log/TestLogFileIterator.java | 5 +-
.../table/read/TestFileGroupRecordBuffer.java | 368 +++++++++
.../table/read/TestHoodieFileGroupReaderBase.java | 11 +-
.../read/TestHoodieFileGroupRecordBuffer.java | 56 --
.../versioning/BaseTestCommitMetadataSerDe.java | 338 ++++++++
.../versioning/v1/TestCommitMetadataSerDeV1.java} | 31 +-
.../versioning/v2/TestCommitMetadataSerDeV2.java | 72 ++
.../view/TestHoodieFileGroupSizeEstimator.java | 62 ++
.../view/TestPriorityBasedFileSystemView.java | 103 +--
.../hudi/common/testutils/CompactionTestUtils.java | 3 +-
.../hudi/common/testutils/FileCreateUtils.java | 184 +++--
.../common/testutils/FileCreateUtilsLegacy.java | 64 +-
.../common/testutils/HoodieTestDataGenerator.java | 79 +-
.../hudi/common/testutils/HoodieTestUtils.java | 25 +-
.../reader/HoodieFileGroupReaderTestUtils.java | 27 +-
.../apache/hudi/common/util/TestCleanerUtils.java | 59 ++
.../apache/hudi/common/util/TestConfigUtils.java | 17 +
.../apache/hudi/common/util/TestHoodieTimer.java | 25 +-
...emaCache.java => TestLocalAvroSchemaCache.java} | 38 +-
.../apache/hudi/common/util/TestSystemClock.java | 34 +-
.../hudi/metrics/TestMetricsReporterFactory.java | 38 +-
.../hudi/timeline/TestTimelineServiceClient.java | 223 ++++++
.../avro-schema-evo/schema-allshapes-A.txt | 80 ++
.../avro-schema-evo/schema-allshapes-B.txt | 123 +++
...e.avsc => decimal-logical-type-zero-scale.avsc} | 2 +-
.../quickstart/utils/QuickstartConfigurations.java | 14 +-
.../examples/quickstart/utils/SchemaBuilder.java | 4 +-
.../spark/HoodieSparkBootstrapExample.java | 2 +-
.../examples/spark/HoodieWriteClientExample.java | 6 +-
.../hudi/sink/StreamWriteOperatorCoordinator.java | 5 +
.../AppendWriteFunctionWithRateLimit.java} | 19 +-
.../AppendWriteFunctions.java} | 19 +-
.../hudi/sink/append/AppendWriteOperator.java | 2 +-
.../bucket/ConsistentBucketAssignFunction.java | 3 +-
.../apache/hudi/sink/bulk/sort/SortOperator.java | 2 +-
.../hudi/sink/clustering/ClusteringPlanEvent.java | 17 +
.../sink/clustering/ClusteringPlanOperator.java | 18 +-
.../FlinkConsistentBucketUpdateStrategy.java | 4 +-
.../sink/partitioner/profile/WriteProfiles.java | 5 +-
.../hudi/sink/transform/ChainedTransformer.java | 2 +-
.../RowDataToHoodieFunctionWithRateLimit.java | 2 +-
.../java/org/apache/hudi/sink/utils/Pipelines.java | 8 +-
.../apache/hudi/source/ExpressionEvaluators.java | 4 +-
.../org/apache/hudi/table/HoodieTableFactory.java | 3 +
.../org/apache/hudi/table/HoodieTableSource.java | 12 +-
.../org/apache/hudi/table/format/FormatUtils.java | 11 +-
.../hudi/table/format/cdc/CdcInputFormat.java | 4 +-
.../hudi/table/lookup/HoodieLookupFunction.java | 2 +-
.../org/apache/hudi/util/FlinkWriteClients.java | 3 +-
.../apache/hudi/util/RowDataToAvroConverters.java | 4 +-
.../org/apache/hudi/sink/TestWriteCopyOnWrite.java | 24 +
.../hudi/sink/TestWriterWithPartitionTTl.java | 3 +-
.../hudi/sink/bucket/ITTestBucketStreamWrite.java | 4 +-
.../apache/hudi/sink/compact/TestCustomSerDe.java | 19 +-
.../utils/BucketStreamWriteFunctionWrapper.java | 6 +
.../hudi/sink/utils/BulkInsertFunctionWrapper.java | 6 +
.../hudi/sink/utils/InsertFunctionWrapper.java | 15 +
.../hudi/sink/utils/MockSubtaskGateway.java} | 35 +-
.../sink/utils/StreamWriteFunctionWrapper.java | 6 +
.../hudi/sink/utils/TestFunctionWrapper.java | 15 +-
.../org/apache/hudi/sink/utils/TestWriteBase.java | 13 +
.../hudi/source/TestIncrementalInputSplits.java | 16 +-
.../org/apache/hudi/utils/TestClusteringUtil.java | 15 +-
.../org/apache/hudi/utils/TestCompactionUtil.java | 11 +-
.../org/apache/hudi/utils/TestStreamerUtil.java | 4 +-
.../test/java/org/apache/hudi/utils/TestUtils.java | 6 +-
.../apache/hudi/adapter/ExecutionAttemptUtil.java} | 11 +-
.../apache/hudi/adapter/ExecutionAttemptUtil.java} | 11 +-
.../apache/hudi/adapter/ExecutionAttemptUtil.java} | 11 +-
.../apache/hudi/adapter/ExecutionAttemptUtil.java} | 11 +-
.../format/cow/vector/HeapMapColumnVector.java | 4 +-
...odieCatalogs.java => ExecutionAttemptUtil.java} | 11 +-
.../format/cow/vector/HeapMapColumnVector.java | 4 +-
.../apache/hudi/adapter/ExecutionAttemptUtil.java} | 11 +-
.../format/cow/vector/HeapMapColumnVector.java | 4 +-
.../apache/hudi/adapter/ExecutionAttemptUtil.java} | 11 +-
.../hfile/HBaseHFileBootstrapIndexReader.java | 6 +-
.../common/config/DFSPropertiesConfiguration.java | 2 +-
.../org/apache/hudi/common/util/AvroOrcUtils.java | 18 +-
.../apache/hudi/common/util/HadoopConfigUtils.java | 3 +-
.../hudi/hadoop/fs/BoundedFsDataInputStream.java | 4 +-
.../hadoop/fs/HoodieRetryWrapperFileSystem.java | 13 +-
.../hadoop/fs/HoodieSerializableFileStatus.java | 22 +-
.../hudi/hadoop/fs/HoodieWrapperFileSystem.java | 4 +-
.../hudi/hadoop/fs/inline/InMemoryFileSystem.java | 3 +-
.../hudi/io/hadoop/HoodieAvroHFileWriter.java | 4 +-
.../apache/hudi/io/hadoop/HoodieAvroOrcWriter.java | 2 +-
.../hudi/io/hadoop/HoodieBaseParquetWriter.java | 4 +-
.../index/TestBaseHoodieTableFileIndex.java | 86 ++
.../org/apache/hudi/common/fs/TestFSUtils.java | 32 +
.../common/fs/TestHoodieWrapperFileSystem.java | 5 +-
.../common/functional/TestHoodieLogFormat.java | 8 +-
.../hudi/common/table/TestHoodieTableConfig.java | 8 +-
.../common/table/TestHoodieTableMetaClient.java | 59 +-
.../hudi/common/table/TestTableSchemaResolver.java | 413 +---------
.../hudi/common/table/TestTimelineUtils.java | 53 +-
.../TestHoodieFileGroupReaderInflightCommit.java | 112 +++
.../table/timeline/TestArchivedTimelineV1.java | 749 ++++++++++++++++++
.../table/timeline/TestHoodieActiveTimeline.java | 156 +++-
.../table/timeline/TestWaitBasedTimeGenerator.java | 24 +-
.../view/TestHoodieTableFSViewWithClustering.java | 12 +-
.../table/view/TestHoodieTableFileSystemView.java | 206 +++--
.../table/view/TestIncrementalFSViewSync.java | 28 +-
.../TestRocksDBBasedIncrementalFSViewSync.java | 6 +-
.../table/view/TestRocksDbBasedFileSystemView.java | 6 +-
.../view/TestSpillableMapBasedFileSystemView.java | 6 +-
...TestSpillableMapBasedIncrementalFSViewSync.java | 6 +-
.../common/testutils/HoodieCommonTestHarness.java | 83 +-
.../hudi/common/testutils/HoodieTestTable.java | 163 ++--
.../reader/HoodieFileGroupReaderTestHarness.java | 9 +-
.../hudi/common/util/TestClusteringUtils.java | 34 +-
.../apache/hudi/common/util/TestCommitUtils.java | 19 +-
.../common/util/collection/TestBitCaskDiskMap.java | 213 ++---
.../util/collection/TestExternalSpillableMap.java | 20 +-
.../util/collection/TestRocksDbBasedMap.java | 28 +-
.../common/util/collection/TestRocksDbDiskMap.java | 225 +++---
.../hudi/hadoop/HiveHoodieReaderContext.java | 10 +
.../HoodieFileGroupReaderBasedRecordReader.java | 2 +-
.../hudi/hadoop/HoodieHFileRecordReader.java | 4 +-
.../hudi/hadoop/HoodieROTablePathFilter.java | 8 +-
.../org/apache/hudi/hadoop/InputPathHandler.java | 2 +-
.../hadoop/hive/HoodieCombineHiveInputFormat.java | 11 +-
.../hive/HoodieCombineRealtimeFileSplit.java | 6 +-
.../realtime/AbstractRealtimeRecordReader.java | 12 +-
.../HoodieCombineRealtimeRecordReader.java | 6 +-
.../realtime/HoodieMergeOnReadSnapshotReader.java | 5 +-
.../apache/hudi/hadoop/utils/HoodieHiveUtils.java | 2 +-
.../hudi/hadoop/utils/HoodieInputFormatUtils.java | 11 +-
.../utils/HoodieRealtimeInputFormatUtils.java | 3 +-
.../hudi/hadoop/TestHoodieHFileInputFormat.java | 23 +-
.../hudi/hadoop/TestHoodieParquetInputFormat.java | 23 +-
.../realtime/TestHoodieRealtimeRecordReader.java | 27 +-
.../hudi/integ/testsuite/HoodieTestSuiteJob.java | 5 +-
.../dag/nodes/BaseValidateDatasetNode.java | 5 +-
.../testsuite/dag/nodes/ScheduleCompactNode.java | 3 +-
.../reader/DFSHoodieDatasetInputReader.java | 4 +-
.../testsuite/dag/nodes/SparkBulkInsertNode.scala | 4 +-
.../testsuite/dag/nodes/SparkDeleteNode.scala | 5 +-
.../dag/nodes/SparkDeletePartitionNode.scala | 1 +
.../testsuite/dag/nodes/SparkInsertNode.scala | 5 +-
.../testsuite/dag/nodes/SparkUpsertNode.scala | 1 +
.../dag/nodes/spark/sql/BaseSparkSqlNode.scala | 3 +-
.../nodes/spark/sql/SparkSqlCreateTableNode.scala | 3 +-
.../dag/nodes/spark/sql/SparkSqlDeleteNode.scala | 3 +-
.../dag/nodes/spark/sql/SparkSqlMergeNode.scala | 3 +-
.../dag/nodes/spark/sql/SparkSqlUpdateNode.scala | 3 +-
.../spark/sql/SparkSqlValidateDatasetNode.scala | 3 +-
.../StructuredStreamingSinkTestWriter.scala | 5 +-
.../hudi/integ/testsuite/utils/SparkSqlUtils.scala | 8 +-
.../org/apache/hudi/integ/ITTestHoodieDemo.java | 2 +-
.../integ/command/ITTestHoodieSyncCommand.java | 2 +-
.../apache/hudi/common/util/ComparableVersion.java | 2 +-
.../org/apache/hudi/common/util/FileIOUtils.java | 11 +-
.../org/apache/hudi/common/util/HoodieTimer.java | 11 +
.../org/apache/hudi/common/util/StringUtils.java | 6 +-
.../org/apache/hudi/io/hfile/HFileBlockType.java | 2 +-
.../apache/hudi/storage/HoodieInstantWriter.java | 34 +-
.../org/apache/hudi/storage/HoodieStorage.java | 20 +-
.../org/apache/hudi/storage/StorageSchemes.java | 23 +-
.../apache/hudi/common/util/TestStringUtils.java | 4 +
.../hudi/io/storage/TestHoodieStorageBase.java | 3 +-
hudi-kafka-connect/pom.xml | 5 +-
.../connect/kafka/KafkaConnectControlAgent.java | 2 +-
.../hudi/connect/utils/KafkaConnectUtils.java | 7 +-
.../connect/writers/BufferedConnectWriter.java | 5 +-
.../timeline/HoodieMetaserverBasedTimeline.java | 19 +-
.../table/view/HoodieMetaserverFileSystemView.java | 11 +-
.../client/HoodieMetaserverClientImp.java | 4 +-
.../main/java/org/apache/hudi/DataSourceUtils.java | 2 +-
.../BaseDatasetBulkInsertCommitActionExecutor.java | 1 +
...setBulkInsertOverwriteCommitActionExecutor.java | 2 +-
...eamerDatasetBulkInsertCommitActionExecutor.java | 2 +-
.../hudi/internal/BaseWriterCommitMessage.java | 2 +-
.../internal/DataSourceInternalWriterHelper.java | 2 +-
.../main/java/org/apache/hudi/sql/InsertMode.java | 2 +-
.../apache/hudi/AutoRecordKeyGenerationUtils.scala | 1 +
.../org/apache/hudi/BloomFiltersIndexSupport.scala | 1 +
.../scala/org/apache/hudi/BucketIndexSupport.scala | 3 +-
.../org/apache/hudi/ColumnStatsIndexSupport.scala | 22 +-
.../scala/org/apache/hudi/DataSourceOptions.scala | 2 +-
.../main/scala/org/apache/hudi/DefaultSource.scala | 9 +-
.../main/scala/org/apache/hudi/EmptyRelation.scala | 2 +-
.../org/apache/hudi/ExpressionIndexSupport.scala | 7 +-
.../scala/org/apache/hudi/FileSystemRelation.scala | 3 +-
.../scala/org/apache/hudi/HoodieBaseRelation.scala | 7 +-
.../org/apache/hudi/HoodieBootstrapMORRDD.scala | 4 +-
.../apache/hudi/HoodieBootstrapMORRelation.scala | 1 +
.../scala/org/apache/hudi/HoodieBootstrapRDD.scala | 2 +-
.../org/apache/hudi/HoodieBootstrapRelation.scala | 2 +-
.../scala/org/apache/hudi/HoodieCLIUtils.scala | 2 +
.../org/apache/hudi/HoodieCreateRecordUtils.scala | 12 +-
.../scala/org/apache/hudi/HoodieFileIndex.scala | 31 +-
.../hudi/HoodieHadoopFsRelationFactory.scala | 22 +-
.../org/apache/hudi/HoodieMergeOnReadRDDV1.scala | 6 +-
.../hudi/HoodiePartitionCDCFileGroupMapping.scala | 1 +
.../hudi/HoodiePartitionFileSliceMapping.scala | 1 +
.../scala/org/apache/hudi/HoodieSchemaUtils.scala | 3 +-
.../org/apache/hudi/HoodieSparkSqlWriter.scala | 22 +-
.../scala/org/apache/hudi/HoodieWriterUtils.scala | 46 +-
.../org/apache/hudi/IncrementalRelationV1.scala | 12 +-
.../org/apache/hudi/IncrementalRelationV2.scala | 8 +-
.../src/main/scala/org/apache/hudi/Iterators.scala | 5 +-
.../hudi/MergeOnReadIncrementalRelationV1.scala | 4 +-
.../apache/hudi/PartitionStatsIndexSupport.scala | 11 +-
.../org/apache/hudi/RecordLevelIndexSupport.scala | 3 +-
.../org/apache/hudi/SecondaryIndexSupport.scala | 2 +-
.../org/apache/hudi/SparkBaseIndexSupport.scala | 11 +-
.../apache/hudi/SparkHoodieTableFileIndex.scala | 7 +-
.../scala/org/apache/hudi/TimelineRelation.scala | 3 +-
.../org/apache/hudi/cdc/CDCFileGroupIterator.scala | 16 +-
.../scala/org/apache/hudi/cdc/CDCRelation.scala | 4 +-
.../datasources/HoodieInMemoryFileIndex.scala | 5 +-
.../sql/catalyst/catalog/HoodieCatalogTable.scala | 6 +-
.../datasources/HoodieMultipleBaseFileFormat.scala | 7 +-
...odieFileGroupReaderBasedParquetFileFormat.scala | 26 +-
.../parquet/LegacyHoodieParquetFileFormat.scala | 3 +-
.../parquet/SparkParquetReaderBase.scala | 5 +-
.../apache/spark/sql/hudi/DataSkippingUtils.scala | 1 +
.../spark/sql/hudi/HoodieSqlCommonUtils.scala | 7 +-
.../spark/sql/hudi/ProvidesHoodieConfig.scala | 61 +-
.../AlterHoodieTableAddColumnsCommand.scala | 8 +-
.../AlterHoodieTableAddPartitionCommand.scala | 1 +
.../AlterHoodieTableChangeColumnCommand.scala | 3 +-
.../AlterHoodieTableDropPartitionCommand.scala | 1 +
.../command/AlterHoodieTableRenameCommand.scala | 4 +-
.../hudi/command/CreateHoodieTableCommand.scala | 5 +-
.../command/CreateHoodieTableLikeCommand.scala | 1 +
.../sql/hudi/command/DropHoodieTableCommand.scala | 1 +
.../sql/hudi/command/MergeIntoKeyGenerator.scala | 3 +-
.../hudi/command/RepairHoodieTableCommand.scala | 3 +-
.../command/ShowHoodieTablePartitionsCommand.scala | 1 +
.../spark/sql/hudi/command/SqlKeyGenerator.scala | 4 +-
.../hudi/command/TruncateHoodieTableCommand.scala | 1 +
.../sql/hudi/streaming/HoodieMetadataLog.scala | 1 +
.../sql/hudi/streaming/HoodieStreamSourceV1.scala | 9 +-
.../sql/hudi/streaming/HoodieStreamSourceV2.scala | 5 +-
.../java/org/apache/hudi/TestDataSourceUtils.java | 1 +
.../org/apache/hudi/TestHoodieWriterUtils.java | 46 ++
.../hudi/testutils/SparkDatasetTestUtils.java | 7 +-
.../org/apache/hudi/TestDataSourceOptions.scala | 1 +
.../apache/hudi/TestRecordLevelIndexSupport.scala | 1 +
.../apache/hudi/TestSecondaryIndexSupport.scala | 1 +
.../cdc/TestInternalRowToJsonStringConverter.scala | 1 +
.../hudi/schema/TestHoodieSparkSchemaUtils.scala | 1 +
.../datasources/TestHoodieInMemoryFileIndex.scala | 3 +-
...stSparkFileFormatInternalRowReaderContext.scala | 2 +-
.../spark/sql/hive/TestHiveClientUtils.scala | 5 +-
.../spark/sql/hudi/TestHoodieFileIndex.scala | 22 +-
.../spark/sql/hudi/TestProvidesHoodieConfig.scala | 5 +-
.../spark/sql/hudi/TestSparkConfigUtils.scala | 1 +
.../main/java/org/apache/hudi/QuickstartUtils.java | 31 +-
.../apache/hudi/cli/BootstrapExecutorUtils.java | 4 +-
.../apache/hudi/cli/HDFSParquetImporterUtils.java | 4 +-
.../spark/sql/hudi/analysis/HoodieAnalysis.scala | 15 +-
.../analysis/HoodiePruneFileSourcePartitions.scala | 6 +-
.../hudi/command/CallProcedureHoodieCommand.scala | 2 +-
.../hudi/command/CompactionHoodiePathCommand.scala | 2 +-
.../command/CompactionHoodieTableCommand.scala | 2 +-
.../command/CompactionShowHoodiePathCommand.scala | 2 +-
.../command/CompactionShowHoodieTableCommand.scala | 2 +-
.../command/CreateHoodieTableAsSelectCommand.scala | 16 +-
.../hudi/command/DeleteHoodieTableCommand.scala | 41 +-
.../spark/sql/hudi/command/IndexCommands.scala | 5 +-
.../command/InsertIntoHoodieTableCommand.scala | 27 +-
.../hudi/command/MergeIntoHoodieTableCommand.scala | 348 ++++++--
.../hudi/command/UpdateHoodieTableCommand.scala | 6 +-
.../hudi/command/payload/ExpressionPayload.scala | 4 +-
.../procedures/CommitsCompareProcedure.scala | 4 +-
.../procedures/CreateMetadataTableProcedure.scala | 4 +-
.../procedures/DropPartitionProcedure.scala | 3 +-
.../procedures/ExportInstantsProcedure.scala | 10 +-
.../procedures/InitMetadataTableProcedure.scala | 4 +-
.../RepairCorruptedCleanFilesProcedure.scala | 5 +-
.../procedures/RepairDeduplicateProcedure.scala | 2 +-
.../command/procedures/RunBootstrapProcedure.scala | 3 +-
.../procedures/RunClusteringProcedure.scala | 8 +-
.../procedures/RunCompactionProcedure.scala | 2 +-
.../RunRollbackInflightTableServiceProcedure.scala | 1 +
.../procedures/ShowArchivedCommitsProcedure.scala | 7 +-
.../procedures/ShowClusteringProcedure.scala | 3 +-
.../ShowColumnStatsOverlapProcedure.scala | 21 +-
.../ShowCommitExtraMetadataProcedure.scala | 10 +-
.../procedures/ShowCommitFilesProcedure.scala | 9 +-
.../procedures/ShowCommitPartitionsProcedure.scala | 13 +-
.../procedures/ShowCommitWriteStatsProcedure.scala | 13 +-
.../command/procedures/ShowCommitsProcedure.scala | 7 +-
.../procedures/ShowFileStatusProcedure.scala | 17 +-
.../procedures/ShowFileSystemViewProcedure.scala | 16 +-
.../ShowHoodieLogFileMetadataProcedure.scala | 2 +-
.../ShowHoodieLogFileRecordsProcedure.scala | 4 +-
.../procedures/ShowInvalidParquetProcedure.scala | 10 +-
.../ShowMetadataTableColumnStatsProcedure.scala | 26 +-
.../procedures/ShowRollbacksProcedure.scala | 12 +-
.../StatsWriteAmplificationProcedure.scala | 5 +-
.../procedures/TruncateTableProcedure.scala | 1 +
.../procedures/UpgradeOrDowngradeProcedure.scala | 2 +-
.../procedures/ValidateHoodieSyncProcedure.scala | 7 +-
.../spark/sql/parser/HoodieCommonSqlParser.scala | 6 +-
.../hudi/TestHoodieMergeHandleWithSparkMerger.java | 2 +-
...=> TestPositionBasedFileGroupRecordBuffer.java} | 12 +-
.../hudi/client/TestHoodieClientMultiWriter.java | 294 ++++++-
.../functional/TestHoodieBackedMetadata.java | 137 +++-
.../TestMetadataUtilRLIandSIRecordGeneration.java | 23 +-
.../TestRemoteFileSystemViewWithMetadataTable.java | 12 +-
.../TestSavepointRestoreMergeOnRead.java | 2 +-
.../TestColStatsRecordWithMetadataRecord.java | 10 +
.../TestHoodieDatasetBulkInsertHelper.java | 2 +-
.../functional/TestSparkClusteringCornerCases.java | 80 ++
.../apache/hudi/functional/TestWriteClient.java | 3 +-
.../hudi/table/TestHoodieMergeOnReadTable.java | 29 +-
.../table/action/compact/TestAsyncCompaction.java | 42 +-
.../table/action/compact/TestHoodieCompactor.java | 26 +-
.../TestMergeOnReadRollbackActionExecutor.java | 13 +-
...arkCopyOnWriteTableRollbackTableVersionSix.java | 63 ++
.../TestHoodieSparkMergeOnReadTableCompaction.java | 275 ++++++-
.../TestHoodieSparkMergeOnReadTableRollback.java | 258 ++++--
.../table/functional/TestHoodieSparkRollback.java | 307 +++++++
.../apache/hudi/testutils/DataSourceTestUtils.java | 33 +-
.../src/test/resources/exampleEvolvedSchema.txt | 5 +
.../src/test/resources/exampleSchema.txt | 5 +
.../src/test/resources/sql-statements.sql | 16 +-
.../TestIncrementalQueryWithArchivedInstants.scala | 10 +-
.../apache/hudi/HoodieSparkWriterTestBase.scala | 2 +-
.../org/apache/hudi/TestAutoKeyGenForSQL.scala | 1 +
.../org/apache/hudi/TestAvroConversionUtils.scala | 3 +-
.../hudi/TestAvroSchemaResolutionSupport.scala | 2 +-
.../org/apache/hudi/TestDataSkippingUtils.scala | 4 +-
.../org/apache/hudi/TestDataSourceDefaults.scala | 2 +-
.../hudi/TestGenericRecordAndRowConsistency.scala | 2 +-
.../apache/hudi/TestHoodieDataSourceHelper.scala | 6 +-
.../org/apache/hudi/TestHoodieFileIndex.scala | 14 +-
.../org/apache/hudi/TestHoodieSparkSqlWriter.scala | 55 +-
.../TestHoodieSparkSqlWriterPartitionTTL.scala | 3 +-
.../apache/hudi/TestHoodieSparkSqlWriterUtc.scala | 2 +-
.../org/apache/hudi/TestHoodieSparkUtils.scala | 2 +-
.../org/apache/hudi/TestInsertDedupPolicy.scala | 26 +-
.../hudi/TestMultipleTableVersionWriting.scala | 1 +
.../hudi/TestParquetReaderCompatibility.scala | 4 +-
.../org/apache/hudi/TestSparkFilterHelper.scala | 2 +-
.../hudi/TestTableSchemaResolverWithSparkSQL.scala | 2 +-
.../model/TestHoodieRecordSerialization.scala | 8 +-
.../read/TestHoodieFileGroupReaderOnSpark.scala | 145 +++-
.../hudi/functional/ColumnStatIndexTestBase.scala | 114 ++-
.../hudi/functional/HoodieStatsIndexTestBase.scala | 5 +-
.../functional/PartitionStatsIndexTestBase.scala | 1 +
.../hudi/functional/RecordLevelIndexTestBase.scala | 7 +-
.../TestAutoGenerationOfRecordKeys.scala | 12 +-
.../hudi/functional/TestBasicSchemaEvolution.scala | 6 +-
.../functional/TestBloomFiltersIndexSupport.scala | 16 +-
.../hudi/functional/TestBucketIndexSupport.scala | 6 +-
.../apache/hudi/functional/TestCOWDataSource.scala | 119 +--
.../hudi/functional/TestColumnStatsIndex.scala | 184 +++--
.../functional/TestColumnStatsIndexWithSQL.scala | 18 +-
.../functional/TestDataSourceForBootstrap.scala | 2 +
.../apache/hudi/functional/TestEmptyCommit.scala | 4 +-
.../hudi/functional/TestHoodieActiveTimeline.scala | 9 +-
.../TestHoodieMultipleBaseFileFormat.scala | 9 +-
.../TestIncrementalReadWithFullTableScan.scala | 7 +-
.../hudi/functional/TestLayoutOptimization.scala | 6 +-
.../apache/hudi/functional/TestMORDataSource.scala | 133 ++--
.../hudi/functional/TestMORDataSourceStorage.scala | 7 +-
.../hudi/functional/TestMetadataRecordIndex.scala | 6 +-
.../TestMetadataTableWithSparkDataSource.scala | 4 +-
.../hudi/functional/TestMetricsReporter.scala | 4 +-
.../functional/TestPartialUpdateAvroPayload.scala | 4 +-
.../hudi/functional/TestPartitionStatsIndex.scala | 26 +-
.../TestPartitionStatsIndexWithSql.scala | 25 +-
.../functional/TestPartitionStatsPruning.scala | 1 +
.../hudi/functional/TestRecordLevelIndex.scala | 43 +-
.../TestRecordLevelIndexTableVersionSix.scala | 16 +-
.../functional/TestRecordLevelIndexWithSQL.scala | 10 +-
.../functional/TestSecondaryIndexPruning.scala | 27 +-
.../hudi/functional/TestSevenToEightUpgrade.scala | 221 +++++-
.../functional/TestSixToFiveDowngradeHandler.scala | 8 +-
.../hudi/functional/TestSparkDataSource.scala | 135 +++-
.../hudi/functional/TestSparkSqlCoreFlow.scala | 6 +-
.../TestSparkSqlWithCustomKeyGenerator.scala | 3 +-
.../TestSparkSqlWithTimestampKeyGenerator.scala | 1 +
...TestStreamSourceReadByStateTransitionTime.scala | 1 -
.../hudi/functional/TestStreamingSource.scala | 3 +-
.../hudi/functional/TestTimeTravelQuery.scala | 11 +-
.../functional/TestWriteTableVersionConfig.scala | 6 +-
.../hudi/functional/cdc/HoodieCDCTestBase.scala | 16 +-
.../functional/cdc/TestCDCDataFrameSuite.scala | 11 +-
.../functional/cdc/TestCDCStreamingSuite.scala | 4 +-
.../org/apache/hudi/util/JavaConversions.scala | 2 +-
.../spark/hudi/benchmark/HoodieBenchmark.scala | 1 -
.../execution/benchmark/AvroSerDerBenchmark.scala | 3 +-
.../benchmark/BoundInMemoryExecutorBenchmark.scala | 2 +-
.../benchmark/CowTableReadBenchmark.scala | 2 +-
.../benchmark/LSMTimelineReadBenchmark.scala | 10 +-
.../TestHoodiePruneFileSourcePartitions.scala | 4 +-
.../analysis/TestMergeIntoHoodieTableCommand.scala | 108 +++
.../sql/hudi/common/HoodieSparkSqlTestBase.scala | 78 +-
.../hudi/common/TestHoodieInternalRowUtils.scala | 6 +-
.../apache/spark/sql/hudi/common/TestSqlConf.scala | 5 +-
.../hudi/common/TestTableColumnTypeMismatch.scala | 879 +++++++++++++++++++++
.../apache/spark/sql/hudi/ddl/TestAlterTable.scala | 83 +-
.../sql/hudi/ddl/TestAlterTableDropPartition.scala | 13 +-
.../sql/hudi/{dml => ddl}/TestDropTable.scala | 28 +-
.../sql/hudi/{dml => ddl}/TestRepairTable.scala | 15 +-
.../sql/hudi/{dml => ddl}/TestShowPartitions.scala | 28 +-
.../sql/hudi/{dml => ddl}/TestTruncateTable.scala | 15 +-
.../spark/sql/hudi/dml/TestDeleteFromTable.scala | 1 -
.../spark/sql/hudi/dml/TestDeleteTable.scala | 50 ++
.../spark/sql/hudi/dml/TestInsertTable.scala | 102 ++-
.../sql/hudi/dml/TestMergeIntoLogOnlyTable.scala | 4 +-
.../spark/sql/hudi/dml/TestMergeIntoTable.scala | 413 +++++-----
.../spark/sql/hudi/dml/TestMergeIntoTable2.scala | 208 +++--
.../TestMergeIntoTableWithNonRecordKeyField.scala | 6 +-
.../hudi/dml/TestMergeModeCommitTimeOrdering.scala | 3 +
.../hudi/dml/TestMergeModeEventTimeOrdering.scala | 5 +-
.../hudi/dml/TestPartialUpdateForMergeInto.scala | 159 +++-
.../spark/sql/hudi/dml/TestUpdateTable.scala | 54 +-
.../hudi/{dml => feature}/TestCDCForSparkSQL.scala | 28 +-
.../{dml => feature}/TestCompactionTable.scala | 28 +-
.../{dml => feature}/TestDataSkippingQuery.scala | 2 +-
.../TestQueryMergeOnReadOptimizedTable.scala | 28 +-
.../index/TestExpressionIndex.scala | 65 +-
.../sql/hudi/feature/index/TestGlobalIndex.scala | 96 +++
.../index/TestIndexSyntax.scala | 3 +-
.../index/TestSecondaryIndex.scala | 15 +-
.../sql/hudi/procedure/TestCallCommandParser.scala | 3 +-
.../hudi/procedure/TestClusteringProcedure.scala | 11 +-
.../procedure/TestDropPartitionProcedure.scala | 14 +-
.../procedure/TestHdfsParquetImportProcedure.scala | 2 +
...tRunRollbackInflightTableServiceProcedure.scala | 1 +
.../procedure/TestShowFileStatusProcedure.scala | 5 +-
.../TestShowInvalidParquetProcedure.scala | 6 +-
.../sql/hudi/procedure/TestTTLProcedure.scala | 2 +-
.../procedure/TestTruncateTableProcedure.scala | 7 +-
.../TestUpgradeOrDowngradeProcedure.scala | 5 +-
.../Spark3HoodieVectorizedParquetRecordReader.java | 13 +-
.../scala/org/apache/hudi/Spark3RowSerDe.scala | 1 +
.../spark/sql/HoodieSpark3CatalogUtils.scala | 1 +
.../spark/sql/HoodieSpark3CatalystPlanUtils.scala | 1 +
.../spark/sql/adapter/BaseSpark3Adapter.scala | 8 +-
.../HoodieFileSystemViewTableValuedFunction.scala | 1 +
.../sql/catalyst/plans/logcal/HoodieQuery.scala | 1 +
.../logcal/HoodieTimelineTableValuedFunction.scala | 1 +
.../datasources/Spark3ParsePartitionUtil.scala | 3 +-
.../Spark3ParquetSchemaEvolutionUtils.scala | 11 +-
.../sql/hudi/analysis/HoodieSpark3Analysis.scala | 2 +-
.../spark/sql/hudi/catalog/BasicStagedTable.scala | 1 +
.../spark/sql/hudi/catalog/HoodieCatalog.scala | 12 +-
.../sql/hudi/catalog/HoodieInternalV2Table.scala | 5 +-
.../spark/sql/hudi/command/AlterTableCommand.scala | 15 +-
.../sql/HoodieSpark33CatalystExpressionUtils.scala | 2 +-
.../spark/sql/HoodieSpark33CatalystPlanUtils.scala | 2 +-
.../apache/spark/sql/adapter/Spark3_3Adapter.scala | 6 +
.../apache/spark/sql/avro/AvroDeserializer.scala | 10 +-
.../org/apache/spark/sql/avro/AvroSerializer.scala | 4 +-
.../Spark33LegacyHoodieParquetFileFormat.scala | 12 +-
.../hudi/Spark33ResolveHudiAlterTableCommand.scala | 1 +
.../sql/hudi/analysis/HoodieSpark33Analysis.scala | 2 +-
.../HoodieSpark3_3ExtendedSqlAstBuilder.scala | 15 +-
.../parser/HoodieSpark3_3ExtendedSqlParser.scala | 6 +-
.../sql/HoodieSpark34CatalystExpressionUtils.scala | 2 +-
.../spark/sql/HoodieSpark34CatalystPlanUtils.scala | 2 +-
.../apache/spark/sql/adapter/Spark3_4Adapter.scala | 14 +-
.../apache/spark/sql/avro/AvroDeserializer.scala | 11 +-
.../org/apache/spark/sql/avro/AvroSerializer.scala | 5 +-
.../org/apache/spark/sql/avro/AvroUtils.scala | 9 +-
.../datasources/Spark34NestedSchemaPruning.scala | 1 +
.../Spark34LegacyHoodieParquetFileFormat.scala | 11 +-
.../hudi/Spark34ResolveHudiAlterTableCommand.scala | 1 +
.../sql/hudi/analysis/HoodieSpark34Analysis.scala | 4 +-
.../HoodieSpark3_4ExtendedSqlAstBuilder.scala | 19 +-
.../parser/HoodieSpark3_4ExtendedSqlParser.scala | 9 +-
.../sql/HoodieSpark35CatalystExpressionUtils.scala | 2 +-
.../apache/spark/sql/adapter/Spark3_5Adapter.scala | 8 +-
.../apache/spark/sql/avro/AvroDeserializer.scala | 11 +-
.../org/apache/spark/sql/avro/AvroSerializer.scala | 5 +-
.../org/apache/spark/sql/avro/AvroUtils.scala | 9 +-
.../HoodieSpark35PartitionedFileUtils.scala | 1 +
.../datasources/Spark35NestedSchemaPruning.scala | 1 +
.../parquet/Spark35DataSourceUtils.scala | 2 +-
.../Spark35LegacyHoodieParquetFileFormat.scala | 11 +-
.../datasources/parquet/Spark35ParquetReader.scala | 2 +-
.../hudi/Spark35ResolveHudiAlterTableCommand.scala | 1 +
.../HoodieSpark3_5ExtendedSqlAstBuilder.scala | 17 +-
.../parser/HoodieSpark3_5ExtendedSqlParser.scala | 9 +-
.../HoodieBulkInsertInternalWriterTestBase.java | 4 +-
.../TestHoodieBulkInsertDataInternalWriter.java | 14 +-
.../TestHoodieDataSourceInternalBatchWrite.java | 6 +-
hudi-sync/hudi-datahub-sync/pom.xml | 15 +
.../hudi/sync/datahub/DataHubTableProperties.java | 4 +-
.../sync/datahub/TestDataHubTableProperties.java | 77 ++
.../apache/hudi/hive/util/ColumnNameXLator.java | 6 +-
.../org/apache/hudi/hive/util/HiveSchemaUtil.java | 4 +-
.../org/apache/hudi/hive/TestHiveSyncTool.java | 50 ++
.../hudi/hive/testutils/HiveTestCluster.java | 8 +-
.../apache/hudi/hive/testutils/HiveTestUtil.java | 28 +-
.../org/apache/hudi/hive/SchemaDifference.java | 6 +-
.../apache/hudi/sync/common/HoodieSyncConfig.java | 2 +-
.../hudi/sync/common/HoodieSyncException.java | 2 +-
.../sync/common/metrics/HoodieMetaSyncMetrics.java | 2 +-
.../hudi/sync/common/util/SyncUtilHelpers.java | 2 +
hudi-tests-common/pom.xml | 5 +
.../hudi/timeline/service/RequestHandler.java | 23 +-
.../hudi/timeline/service/TimelineService.java | 21 +-
.../timeline/service/handlers/BaseFileHandler.java | 6 +-
.../service/handlers/FileSliceHandler.java | 6 +-
.../hudi/timeline/service/handlers/Handler.java | 13 +-
.../service/handlers/InstantStateHandler.java | 10 +-
.../timeline/service/handlers/MarkerHandler.java | 23 +-
.../timeline/service/handlers/TimelineHandler.java | 6 +-
.../AsyncTimelineServerBasedDetectionStrategy.java | 2 +-
.../MarkerBasedEarlyConflictDetectionRunnable.java | 14 +-
.../service/handlers/marker/MarkerDirState.java | 2 +-
.../timeline/service/MockHoodieHadoopStorage.java | 92 +++
.../hudi/timeline/service/TestRequestHandler.java | 147 ++++
.../hudi/timeline/service/TestTimelineService.java | 7 +-
.../service/TimelineServiceTestHarness.java | 192 +++++
.../TestRemoteHoodieTableFileSystemView.java | 100 +--
hudi-utilities/pom.xml | 27 +-
.../apache/hudi/utilities/HDFSParquetImporter.java | 12 +-
.../org/apache/hudi/utilities/HoodieCleaner.java | 7 +-
.../apache/hudi/utilities/HoodieClusteringJob.java | 2 +-
.../org/apache/hudi/utilities/HoodieCompactor.java | 4 +-
.../hudi/utilities/HoodieDataTableValidator.java | 4 +-
.../hudi/utilities/HoodieDropPartitionsTool.java | 6 +-
.../org/apache/hudi/utilities/HoodieIndexer.java | 2 +-
.../utilities/HoodieMetadataTableValidator.java | 121 +--
.../apache/hudi/utilities/HoodieRepairTool.java | 2 +-
.../hudi/utilities/HoodieSnapshotCopier.java | 4 +-
.../hudi/utilities/HoodieSnapshotExporter.java | 8 +-
.../org/apache/hudi/utilities/HoodieTTLJob.java | 7 +-
.../org/apache/hudi/utilities/TableSizeStats.java | 4 +-
.../utilities/callback/SourceCommitCallback.java | 6 +
.../kafka/HoodieWriteCommitKafkaCallback.java | 6 +-
...heckpointFromAnotherHoodieTimelineProvider.java | 5 +-
.../checkpointing/KafkaConnectHdfsProvider.java | 2 +-
.../hudi/utilities/config/KafkaSourceConfig.java | 10 +
.../ingestion/HoodieIngestionMetrics.java | 14 +-
.../hudi/utilities/perf/TimelineServerPerf.java | 4 +-
.../utilities/schema/FilebasedSchemaProvider.java | 60 +-
.../utilities/schema/JdbcbasedSchemaProvider.java | 2 +-
.../utilities/schema/SchemaRegistryProvider.java | 62 ++
.../converter/JsonToAvroSchemaConverter.java | 464 ++++++++---
.../converter/JsonToAvroSchemaConverterConfig.java | 51 ++
.../hudi/utilities/sources/AvroKafkaSource.java | 13 +-
.../hudi/utilities/sources/HoodieIncrSource.java | 9 +-
.../hudi/utilities/sources/JsonKafkaSource.java | 81 +-
.../apache/hudi/utilities/sources/KafkaSource.java | 8 +-
.../hudi/utilities/sources/ProtoKafkaSource.java | 7 +
.../org/apache/hudi/utilities/sources/Source.java | 41 +
.../utilities/sources/debezium/DebeziumSource.java | 8 +-
.../sources/helpers/DatePartitionPathSelector.java | 2 +-
.../utilities/sources/helpers/KafkaOffsetGen.java | 138 +++-
.../utilities/sources/helpers/KafkaSourceUtil.java | 50 ++
.../sources/helpers/ProtoConversionUtil.java | 3 +-
.../utilities/sources/helpers/QueryRunner.java | 9 +-
.../sources/helpers/gcs/MessageValidity.java | 2 +-
.../utilities/streamer/BaseErrorTableWriter.java | 5 +
.../hudi/utilities/streamer/BootstrapExecutor.java | 7 +-
.../streamer/HoodieMultiTableStreamer.java | 27 +-
.../hudi/utilities/streamer/HoodieStreamer.java | 17 +-
.../utilities/streamer/HoodieStreamerMetrics.java | 2 +-
.../utilities/streamer/HoodieStreamerUtils.java | 8 +
.../utilities/streamer/SourceFormatAdapter.java | 17 +-
.../utilities/streamer/SparkSampleWritesUtils.java | 4 +-
.../apache/hudi/utilities/streamer/StreamSync.java | 300 ++++---
.../streamer/StreamerCheckpointUtils.java | 5 +-
.../apache/hudi/utilities/TestHoodieIndexer.java | 15 +-
.../TestHoodieMetadataTableValidator.java | 212 ++++-
.../deltastreamer/HoodieDeltaStreamerTestBase.java | 41 +-
.../deltastreamer/TestHoodieDeltaStreamer.java | 210 ++++-
...TestHoodieDeltaStreamerErrorTableWriteFlow.java | 133 ++++
...TestHoodieDeltaStreamerSchemaEvolutionBase.java | 48 +-
...oodieDeltaStreamerSchemaEvolutionExtensive.java | 83 +-
.../TestHoodieDeltaStreamerWithMultiWriter.java | 11 +-
.../schema/TestFilebasedSchemaProvider.java | 35 +-
.../schema/TestSchemaRegistryProvider.java | 24 +
.../converter/TestJsonToAvroSchemaConverter.java | 59 +-
.../utilities/sources/BaseTestKafkaSource.java | 13 +
.../utilities/sources/TestAvroKafkaSource.java | 35 +
.../utilities/sources/TestHoodieIncrSource.java | 277 +++++--
.../utilities/sources/TestJsonKafkaSource.java | 201 ++++-
.../utilities/sources/TestProtoKafkaSource.java | 21 +-
.../hudi/utilities/sources/TestS3EventsSource.java | 41 +-
.../sources/helpers/TestKafkaOffsetGen.java | 260 +++++-
.../sources/helpers/TestKafkaSourceUtil.java | 66 ++
.../helpers/TestMercifulJsonToRowConverter.java | 129 ++-
.../streamer/TestStreamerCheckpointUtils.java | 6 +-
.../AbstractCloudObjectsSourceTestBase.java | 3 +-
.../json/array-with-item-type-union/expected.json | 78 ++
.../json/array-with-item-type-union/input.json | 93 +++
.../json/complex-json-union-types/expected.json | 66 ++
.../json/complex-json-union-types/input.json | 42 +
.../expected.json | 13 +-
.../input.json | 14 +-
.../expected_no_stripping_quotes.json | 138 ++++
.../expected_stripping_quotes.json | 138 ++++
.../json/not-null-default-value-schema/input.json | 67 ++
.../json/schema-repeating-names/expected.json | 153 ++++
.../json/schema-repeating-names/input.json | 124 +++
...e.avsc => evolved-test-with-default-value.avsc} | 5 +
..._uber.avsc => source_uber_encoded_decimal.avsc} | 35 +-
.../source_uber_encoded_decimal.json | 54 ++
packaging/bundle-validation/README.md | 23 +-
packaging/bundle-validation/ci_run.sh | 31 +-
packaging/bundle-validation/cli/commands.txt | 5 +
packaging/bundle-validation/service/read.scala | 2 -
packaging/bundle-validation/service/write.scala | 2 -
packaging/bundle-validation/validate.sh | 81 +-
packaging/hudi-cli-bundle/hudi-cli-with-bundle.sh | 8 +-
packaging/hudi-integ-test-bundle/pom.xml | 1 +
packaging/hudi-spark-bundle/pom.xml | 10 +-
packaging/hudi-utilities-bundle/pom.xml | 4 +
packaging/hudi-utilities-slim-bundle/pom.xml | 39 +-
pom.xml | 133 +++-
release/release_guide.md | 12 +-
rfc/README.md | 3 +-
scripts/jacoco/README.md | 110 +++
.../jacoco/download_jacoco.sh | 7 +-
.../generate_jacoco_coverage_report.sh} | 9 +-
.../merge_jacoco_exec_files.sh} | 26 +-
.../jacoco/merge_jacoco_job_files.sh | 10 +-
style/scalastyle.xml | 8 +-
1080 files changed, 29982 insertions(+), 8933 deletions(-)
copy .github/workflows/{release_candidate_validation.yml =>
maven_artifact_validation.yml} (80%)
rename {hudi-client/hudi-client-common/src/main/java/org/apache/hudi =>
hudi-aws/src/main/java/org/apache/hudi/aws}/metrics/cloudwatch/CloudWatchMetricsReporter.java
(96%)
rename hudi-aws/src/main/java/org/apache/hudi/aws/{ =>
metrics}/cloudwatch/CloudWatchReporter.java (94%)
delete mode 100644
hudi-aws/src/main/java/org/apache/hudi/aws/utils/S3Utils.java
rename {hudi-client/hudi-client-common/src/test/java/org/apache/hudi =>
hudi-aws/src/test/java/org/apache/hudi/aws}/metrics/cloudwatch/TestCloudWatchMetricsReporter.java
(53%)
rename hudi-aws/src/test/java/org/apache/hudi/aws/{ =>
metrics}/cloudwatch/TestCloudWatchReporter.java (87%)
delete mode 100644
hudi-aws/src/test/java/org/apache/hudi/aws/utils/TestS3Utils.java
delete mode 100755 hudi-cli/hudi-cli.sh
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/CommitMetadataResolver.java
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/CommitMetadataResolverFactory.java
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/MarkerBasedCommitMetadataResolver.java
copy
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/{table/upgrade/SupportsUpgradeDowngrade.java
=> client/NoOpCommitMetadataResolver.java} (55%)
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/ConcurrentSchemaEvolutionTableSchemaGetter.java
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/SchemaConflictResolutionStrategy.java
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/SimpleSchemaConflictResolutionStrategy.java
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/StorageLock.java
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/models/HeartbeatManager.java
copy
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/{exception/HoodieCompactionException.java
=> client/transaction/lock/models/LockGetResult.java} (66%)
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/models/LockProviderHeartbeatManager.java
copy
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/{exception/HoodieClusteringException.java
=> client/transaction/lock/models/LockUpdateResult.java} (63%)
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/models/StorageLockData.java
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/models/StorageLockFile.java
delete mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/utils/MetadataTableUtils.java
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/estimator/AverageRecordSizeEstimator.java
copy
hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamContext.java
=>
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/estimator/RecordSizeEstimator.java
(58%)
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/estimator/RecordSizeEstimatorFactory.java
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriterTableVersionSix.java
rename
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/{BaseRollbackHelper.java
=> RollbackHelper.java} (96%)
copy
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/{upgrade/FiveToFourDowngradeHandler.java
=> action/rollback/RollbackHelperFactory.java} (60%)
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/RollbackHelperV1.java
copy
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/{client/timeline/HoodieTimelineArchiver.java
=> table/marker/AppendMarkerHandler.java} (58%)
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/DirectWriteMarkersV1.java
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/TimelineServerBasedWriteMarkersV1.java
create mode 100644
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/TestBaseHoodieTableServiceClient.java
create mode 100644
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/TestBaseHoodieWriteClient.java
create mode 100644
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/TestConcurrentSchemaEvolutionTableSchemaGetter.java
create mode 100644
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/TestHoodieLockMetrics.java
create mode 100644
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/TestSimpleSchemaConflictResolutionStrategy.java
create mode 100644
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/lock/models/StorageLockFileTest.java
create mode 100644
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/lock/models/TestLockProviderHeartbeatManager.java
create mode 100644
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/utils/TestCommitMetadataResolver.java
rename
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/common/table/timeline/{TestHoodieArchivedTimeline.java
=> TestArchivedTimelineV2.java} (87%)
copy hudi-io/src/test/java/org/apache/hudi/io/util/TestHFileUtils.java =>
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/config/TestHoodieErrorTableConfig.java
(51%)
create mode 100644
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/estimator/TestAverageRecordSizeEstimator.java
create mode 100644
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/estimator/TestRecordSizeEstimatorFactory.java
create mode 100644
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metadata/TestHoodieBackedTableMetadataWriter.java
create mode 100644
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/TestBaseHoodieTable.java
create mode 100644
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/TestHoodieTable.java
create mode 100644
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/clean/TestCleanPlanActionExecutor.java
create mode 100644
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/compact/TestScheduleCompactionActionExecutor.java
rename
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/rollback/{TestBaseRollbackHelper.java
=> TestRollbackHelper.java} (98%)
copy
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/{SparkHoodieBackedTableMetadataWriter.java
=> SparkHoodieBackedTableMetadataWriterTableVersionSix.java} (51%)
create mode 100644
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkMetadataWriterFactory.java
delete mode 100644
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/AverageRecordSizeUtils.java
create mode 100644
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieFileSystemViews.java
create mode 100644
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/data/TestHoodieJavaPairRDD.java
create mode 100644
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestHoodieSparkTable.java
delete mode 100644
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/commit/TestAverageRecordSizeUtils.java
copy
hudi-io/src/main/java/org/apache/hudi/io/ByteArraySeekableDataInputStream.java
=>
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/marker/TestDirectWriteMarkersV1.java
(58%)
create mode 100644
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/marker/TestTimelineServerBasedWriteMarkersV1.java
create mode 100644
hudi-common/src/main/java/org/apache/hudi/avro/AvroSchemaCache.java
create mode 100644
hudi-common/src/main/java/org/apache/hudi/avro/AvroSchemaComparatorForSchemaEvolution.java
copy
hudi-common/src/main/java/org/apache/hudi/common/{function/SerializableSupplier.java
=> serialization/CustomSerializer.java} (74%)
copy
hudi-common/src/main/java/org/apache/hudi/common/{table/timeline/CommitMetadataSerDe.java
=> serialization/DefaultSerializer.java} (61%)
create mode 100644
hudi-common/src/main/java/org/apache/hudi/common/serialization/HoodieFileGroupSerializer.java
copy
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/PartitionRegexBasedCompactionStrategy.java
=>
hudi-common/src/main/java/org/apache/hudi/common/serialization/HoodieFileSliceSerializer.java
(50%)
rename
hudi-common/src/main/java/org/apache/hudi/common/table/read/{HoodieFileGroupReaderSchemaHandler.java
=> FileGroupReaderSchemaHandler.java} (57%)
rename
hudi-common/src/main/java/org/apache/hudi/common/table/read/{HoodieBaseFileGroupRecordBuffer.java
=> FileGroupRecordBuffer.java} (91%)
rename
hudi-common/src/main/java/org/apache/hudi/common/table/read/{HoodieKeyBasedFileGroupRecordBuffer.java
=> KeyBasedFileGroupRecordBuffer.java} (75%)
rename
hudi-common/src/main/java/org/apache/hudi/common/table/read/{HoodiePositionBasedFileGroupRecordBuffer.java
=> PositionBasedFileGroupRecordBuffer.java} (82%)
rename
hudi-common/src/main/java/org/apache/hudi/common/table/read/{HoodiePositionBasedSchemaHandler.java
=> PositionBasedSchemaHandler.java} (83%)
rename
hudi-common/src/main/java/org/apache/hudi/common/table/read/{HoodieUnmergedFileGroupRecordBuffer.java
=> UnmergedFileGroupRecordBuffer.java} (94%)
create mode 100644
hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieInstantReader.java
copy
hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/{DefaultInstantGenerator.java
=> DefaultInstantComparator.java} (91%)
create mode 100644
hudi-common/src/main/java/org/apache/hudi/common/table/view/HoodieFileGroupSizeEstimator.java
create mode 100644
hudi-common/src/main/java/org/apache/hudi/common/table/view/NoOpTableMetadata.java
rename
hudi-common/src/main/java/org/apache/hudi/common/util/{AvroSchemaCache.java =>
LocalAvroSchemaCache.java} (81%)
copy
hudi-common/src/main/java/org/apache/hudi/common/util/{DefaultSizeEstimator.java
=> LogicalClock.java} (70%)
copy
hudi-common/src/main/java/org/apache/hudi/common/util/{VisibleForTesting.java
=> SystemClock.java} (77%)
copy
hudi-common/src/main/java/org/apache/hudi/exception/{CorruptedLogFileException.java
=> HoodieDuplicateDataFileDetectedException.java} (79%)
copy
hudi-common/src/main/java/org/apache/hudi/exception/{HoodieWriteConflictException.java
=> HoodieSchemaEvolutionConflictException.java} (71%)
delete mode 100644
hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataFileSystemView.java
create mode 100644
hudi-common/src/main/java/org/apache/hudi/timeline/TimelineServiceClient.java
create mode 100644
hudi-common/src/main/java/org/apache/hudi/timeline/TimelineServiceClientBase.java
create mode 100644
hudi-common/src/test/java/org/apache/hudi/avro/TestAvroSchemaComparatorForSchemaEvolution.java
create mode 100644
hudi-common/src/test/java/org/apache/hudi/common/fs/TestSizeAwareDataInputStream.java
create mode 100644
hudi-common/src/test/java/org/apache/hudi/common/serialization/TestDefaultSerializer.java
create mode 100644
hudi-common/src/test/java/org/apache/hudi/common/serialization/TestHoodieFileGroupSerializer.java
create mode 100644
hudi-common/src/test/java/org/apache/hudi/common/serialization/TestHoodieFileSliceSerializer.java
create mode 100644
hudi-common/src/test/java/org/apache/hudi/common/table/read/TestFileGroupRecordBuffer.java
delete mode 100644
hudi-common/src/test/java/org/apache/hudi/common/table/read/TestHoodieFileGroupRecordBuffer.java
create mode 100644
hudi-common/src/test/java/org/apache/hudi/common/table/timeline/versioning/BaseTestCommitMetadataSerDe.java
copy
hudi-common/src/{main/java/org/apache/hudi/common/engine/LocalTaskContextSupplier.java
=>
test/java/org/apache/hudi/common/table/timeline/versioning/v1/TestCommitMetadataSerDeV1.java}
(58%)
create mode 100644
hudi-common/src/test/java/org/apache/hudi/common/table/timeline/versioning/v2/TestCommitMetadataSerDeV2.java
create mode 100644
hudi-common/src/test/java/org/apache/hudi/common/table/view/TestHoodieFileGroupSizeEstimator.java
create mode 100644
hudi-common/src/test/java/org/apache/hudi/common/util/TestCleanerUtils.java
copy
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/index/hbase/TestHBaseIndexUsage.java
=> hudi-common/src/test/java/org/apache/hudi/common/util/TestHoodieTimer.java
(64%)
rename
hudi-common/src/test/java/org/apache/hudi/common/util/{TestAvroSchemaCache.java
=> TestLocalAvroSchemaCache.java} (50%)
copy hudi-cli/src/test/java/org/apache/hudi/cli/TestSparkUtil.java =>
hudi-common/src/test/java/org/apache/hudi/common/util/TestSystemClock.java (55%)
rename {hudi-client/hudi-client-common =>
hudi-common}/src/test/java/org/apache/hudi/metrics/TestMetricsReporterFactory.java
(75%)
create mode 100644
hudi-common/src/test/java/org/apache/hudi/timeline/TestTimelineServiceClient.java
create mode 100644
hudi-common/src/test/resources/avro-schema-evo/schema-allshapes-A.txt
create mode 100644
hudi-common/src/test/resources/avro-schema-evo/schema-allshapes-B.txt
copy hudi-common/src/test/resources/{decimal-logical-type.avsc =>
decimal-logical-type-zero-scale.avsc} (95%)
copy
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/{transform/RowDataToHoodieFunctionWithRateLimit.java
=> append/AppendWriteFunctionWithRateLimit.java} (75%)
copy
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/{transform/RowDataToHoodieFunctions.java
=> append/AppendWriteFunctions.java} (62%)
copy
hudi-flink-datasource/{hudi-flink1.17.x/src/main/java/org/apache/hudi/adapter/OperatorCoordinatorAdapter.java
=>
hudi-flink/src/test/java/org/apache/hudi/sink/utils/MockSubtaskGateway.java}
(55%)
copy
hudi-flink-datasource/{hudi-flink1.18.x/src/test/java/org/apache/hudi/adapter/TestHoodieCatalogs.java
=>
hudi-flink1.14.x/src/test/java/org/apache/hudi/adapter/ExecutionAttemptUtil.java}
(77%)
copy
hudi-flink-datasource/{hudi-flink1.18.x/src/test/java/org/apache/hudi/adapter/TestHoodieCatalogs.java
=>
hudi-flink1.15.x/src/test/java/org/apache/hudi/adapter/ExecutionAttemptUtil.java}
(77%)
copy
hudi-flink-datasource/{hudi-flink1.18.x/src/test/java/org/apache/hudi/adapter/TestHoodieCatalogs.java
=>
hudi-flink1.16.x/src/test/java/org/apache/hudi/adapter/ExecutionAttemptUtil.java}
(77%)
copy
hudi-flink-datasource/{hudi-flink1.18.x/src/test/java/org/apache/hudi/adapter/TestHoodieCatalogs.java
=>
hudi-flink1.17.x/src/test/java/org/apache/hudi/adapter/ExecutionAttemptUtil.java}
(77%)
copy
hudi-flink-datasource/hudi-flink1.18.x/src/test/java/org/apache/hudi/adapter/{TestHoodieCatalogs.java
=> ExecutionAttemptUtil.java} (77%)
copy
hudi-flink-datasource/{hudi-flink1.18.x/src/test/java/org/apache/hudi/adapter/TestHoodieCatalogs.java
=>
hudi-flink1.19.x/src/test/java/org/apache/hudi/adapter/ExecutionAttemptUtil.java}
(77%)
copy
hudi-flink-datasource/{hudi-flink1.18.x/src/test/java/org/apache/hudi/adapter/TestHoodieCatalogs.java
=>
hudi-flink1.20.x/src/test/java/org/apache/hudi/adapter/ExecutionAttemptUtil.java}
(77%)
create mode 100644
hudi-hadoop-common/src/test/java/org/apache/hudi/common/bootstrap/index/TestBaseHoodieTableFileIndex.java
create mode 100644
hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/read/TestHoodieFileGroupReaderInflightCommit.java
create mode 100644
hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/timeline/TestArchivedTimelineV1.java
copy
hudi-cli/src/test/java/org/apache/hudi/cli/testutils/HoodieCLIIntegrationTestBase.java
=> hudi-io/src/main/java/org/apache/hudi/storage/HoodieInstantWriter.java (59%)
create mode 100644
hudi-spark-datasource/hudi-spark-common/src/test/java/org/apache/hudi/TestHoodieWriterUtils.java
copy
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/optimize/TestHilbertCurveUtils.java
=>
hudi-spark-datasource/hudi-spark-common/src/test/scala/org/apache/spark/sql/hudi/TestHoodieFileIndex.scala
(63%)
rename
hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/{TestHoodiePositionBasedFileGroupRecordBuffer.java
=> TestPositionBasedFileGroupRecordBuffer.java} (96%)
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestSparkClusteringCornerCases.java
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/functional/TestHoodieSparkCopyOnWriteTableRollbackTableVersionSix.java
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/functional/TestHoodieSparkRollback.java
copy
hudi-aws/src/main/java/org/apache/hudi/aws/sync/util/GluePartitionFilterGenerator.java
=>
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestRecordLevelIndexTableVersionSix.scala
(67%)
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/analysis/TestMergeIntoHoodieTableCommand.scala
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/common/TestTableColumnTypeMismatch.scala
rename
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/{dml
=> ddl}/TestDropTable.scala (93%)
rename
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/{dml
=> ddl}/TestRepairTable.scala (94%)
rename
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/{dml
=> ddl}/TestShowPartitions.scala (91%)
rename
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/{dml
=> ddl}/TestTruncateTable.scala (92%)
rename
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/{dml
=> feature}/TestCDCForSparkSQL.scala (93%)
rename
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/{dml
=> feature}/TestCompactionTable.scala (90%)
rename
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/{dml
=> feature}/TestDataSkippingQuery.scala (99%)
rename
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/{dml
=> feature}/TestQueryMergeOnReadOptimizedTable.scala (75%)
rename
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/{command
=> feature}/index/TestExpressionIndex.scala (97%)
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/index/TestGlobalIndex.scala
rename
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/{command
=> feature}/index/TestIndexSyntax.scala (99%)
rename
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/{command
=> feature}/index/TestSecondaryIndex.scala (98%)
create mode 100644
hudi-sync/hudi-datahub-sync/src/test/java/org/apache/hudi/sync/datahub/TestDataHubTableProperties.java
create mode 100644
hudi-timeline-service/src/test/java/org/apache/hudi/timeline/service/MockHoodieHadoopStorage.java
create mode 100644
hudi-timeline-service/src/test/java/org/apache/hudi/timeline/service/TestRequestHandler.java
create mode 100644
hudi-timeline-service/src/test/java/org/apache/hudi/timeline/service/TimelineServiceTestHarness.java
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/converter/JsonToAvroSchemaConverterConfig.java
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/KafkaSourceUtil.java
create mode 100644
hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamerErrorTableWriteFlow.java
create mode 100644
hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/helpers/TestKafkaSourceUtil.java
create mode 100644
hudi-utilities/src/test/resources/schema-provider/json/array-with-item-type-union/expected.json
create mode 100644
hudi-utilities/src/test/resources/schema-provider/json/array-with-item-type-union/input.json
create mode 100644
hudi-utilities/src/test/resources/schema-provider/json/complex-json-union-types/expected.json
create mode 100644
hudi-utilities/src/test/resources/schema-provider/json/complex-json-union-types/input.json
copy
hudi-utilities/src/test/resources/schema-provider/json/{multiple-properties =>
kafka-decimal-simple}/expected.json (61%)
copy
hudi-utilities/src/test/resources/schema-provider/json/{multiple-properties =>
kafka-decimal-simple}/input.json (55%)
create mode 100644
hudi-utilities/src/test/resources/schema-provider/json/not-null-default-value-schema/expected_no_stripping_quotes.json
create mode 100644
hudi-utilities/src/test/resources/schema-provider/json/not-null-default-value-schema/expected_stripping_quotes.json
create mode 100644
hudi-utilities/src/test/resources/schema-provider/json/not-null-default-value-schema/input.json
create mode 100644
hudi-utilities/src/test/resources/schema-provider/json/schema-repeating-names/expected.json
create mode 100644
hudi-utilities/src/test/resources/schema-provider/json/schema-repeating-names/input.json
copy
hudi-utilities/src/test/resources/schema/{simple-test-with-default-value.avsc
=> evolved-test-with-default-value.avsc} (92%)
copy hudi-utilities/src/test/resources/streamer-config/{source_uber.avsc =>
source_uber_encoded_decimal.avsc} (66%)
create mode 100644
hudi-utilities/src/test/resources/streamer-config/source_uber_encoded_decimal.json
create mode 100644 packaging/bundle-validation/cli/commands.txt
create mode 100644 scripts/jacoco/README.md
copy docker/hoodie/hadoop/trinocoordinator/etc/node.properties =>
scripts/jacoco/download_jacoco.sh (82%)
mode change 100644 => 100755
copy scripts/{release/validate_source_rat.sh =>
jacoco/generate_jacoco_coverage_report.sh} (77%)
copy scripts/{release/create_source_directory.sh =>
jacoco/merge_jacoco_exec_files.sh} (64%)
copy docker/hoodie/hadoop/trinocoordinator/etc/node.properties =>
scripts/jacoco/merge_jacoco_job_files.sh (78%)
mode change 100644 => 100755
