This is an automated email from the ASF dual-hosted git repository.
yuzhaojing pushed a change to branch release-feature-rfc46
in repository https://gitbox.apache.org/repos/asf/hudi.git
omit bb208e9861 [HUDI-3350][HUDI-3351] Support HoodieMerge API and Spark
engine-specific HoodieRecord (#5627)
omit 17d179ceeb [HUDI-4344] fix usage of HoodieDataBlock#getRecordIterator
(#6005)
omit 9d4b3bf5b4 [HUDI-4292][RFC-46] Update doc to align with the Record
Merge API changes (#5927)
omit c0c3c5ea06 [MINOR] Fix type casting in TestHoodieHFileReaderWriter
omit 0126868d96 [HUDI-3378][HUDI-3379][HUDI-3381] Migrate usage of
HoodieRecordPayload and raw Avro payload to HoodieRecord (#5522)
add b228a2788a [HUDI-4399][RFC-57] Protobuf support in DeltaStreamer
(#6111)
add b6b515917c [HUDI-4703] use the historical schema to response time
travel query (#6499)
add 71fc8b2fbe [HUDI-4549] Remove avro from hudi-hive-sync-bundle and
hudi-aws-bundle (#6472)
add c50b6346b5 [HUDI-4482] remove guava and use caffeine instead for cache
(#6240)
add ac9ce85334 [HUDI-4483] Fix checkstyle in integ-test module (#6523)
add 71b8174058 [HUDI-4340] fix not parsable text DateTimeParseException by
addng a method parseDateFromInstantTimeSafely for parsing timestamp when output
metrics (#6000)
add 7c9ceb6370 [DOCS] Add docs about
javax.security.auth.login.LoginException when starting Hudi Sink Connector
(#6255)
add a3481efdf0 [HUDI-4327] Fixing flaky deltastreamer test
(testCleanerDeleteReplacedDataWithArchive) (#6533)
add 439afa5582 [HUDI-4730] Fix batch job cannot clean old commits files
(#6515)
add 30e36f35eb [HUDI-4740] Add metadata fields for hive catalog
#createTable (#6541)
add d7a52400a8 [HUDI-4695] Fixing flaky
TestInlineCompaction#testCompactionRetryOnFailureBasedOnTime (#6534)
add 570989dc40 [HUDI-4193] change protoc version to unblock hudi
compilation on m1 mac (#6535)
add f7038e0195 [HUDI-4438] Fix flaky
TestCopyOnWriteActionExecutor#testPartitionMetafileFormat (#6546)
add dd096b087f [MINOR] Fix typo in HoodieArchivalConfig (#6542)
add 3e99ca71ce [HUDI-4582] Support batch synchronization of partition to
HMS to avoid timeout (#6347)
add 5d9db864f1 [HUDI-4742] Fix AWS Glue partition's location is wrong when
updatePartition (#6545)
add f5de4e434b [HUDI-4418] Add support for ProtoKafkaSource (#6135)
add 98c3d88b2f [HUDI-4642] Adding support to hudi-cli to repair deprecated
partition (#6438)
add 82d41f4b87 [HUDI-4751] Fix owner instants for transaction manager api
callers (#6549)
add 24dd00724c [HUDI-4739] Wrong value returned when key's length equals 1
(#6539)
add edbd7fd6cc [HUDI-4528] Add diff tool to compare commit metadata (#6485)
add d2c46fb62a [HUDI-4648] Support rename partition through CLI (#6569)
add 0f76d69e94 [HUDI-4775] Fixing incremental source for MOR table (#6587)
add 2ffa1f74ea [HUDI-4694] Print testcase running time for CI jobs (#6586)
add af78567925 [RFC] Claim RFC-62 for Diagnostic Reporter (#6599)
add 7e9ad05317 [minor] following HUDI-4739, fix the extraction for simple
record keys (#6594)
add 2590d2abf1 [HUDI-4619] Add a remote request retry mechanism for
'Remotehoodietablefilesystemview'. (#6393)
add 57e6223cad [HUDI-4720] Fix HoodieInternalRow return wrong num of
fields when source not contains meta fields (#6500)
add 993b3f1f3d [HUDI-4389] Make HoodieStreamingSink idempotent (#6098)
add 5fcec2fdc4 [MINOR] Remove redundant braces (#6604)
add 454f7d37b8 [HUDI-4618] Separate log word for CommitUitls class (#6392)
add 323f19685c [HUDI-4776] Fix merge into use unresolved assignment (#6589)
add 27c7efb4ef [HUDI-4795] Fix KryoException when bulk insert into a not
bucket index hudi table
add d2d1cb8b28 [HUDI-4615] Return checkpoint as null for empty data from
events queue. (#6387)
add 8ffcb2fc94 [HUDI-4782] Support TIMESTAMP_LTZ type for flink (#6607)
add dbb044b751 [HUDI-4731] Shutdown CloudWatch reporter when query
completes (#6468)
add e8aee84c7c [HUDI-4793] Fixing ScalaTest tests to properly respect
Log4j2 configs (#6617)
add adf36093d2 [HUDI-4766] Strengthen flink clustering job (#6566)
add dcb55b7019 [HUDI-4797] fix merge into table for source table with
different column order (#6620)
add e1da06fa70 [MINOR] Typo fix for kryo in flink-bundle (#6639)
add 13eb892081 [HUDI-4811] Fix the checkstyle of hudi flink (#6633)
add 4af60dcfba [HUDI-4465] Optimizing file-listing sequence of Metadata
Table (#6016)
add 228b6fe40f [HUDI-4807] Use base table instant for metadata
initialization (#6629)
add e87a92d603 [HUDI-3453] Fix HoodieBackedTableMetadata concurrent
reading issue (#5091)
add d0830c81cf [HUDI-4518] Add unit test for reentrant lock in diff
lockProvider (#6624)
add c92ea8a8bb [HUDI-4810] Fixing Hudi bundles requiring log4j2 on the
classpath (#6631)
add e788eb2dec [HUDI-4826] Update RemoteHoodieTableFileSystemView to allow
base path in UTF-8 (#6544)
add a311965816 [HUDI-4763] Allow hoodie read client to choose index (#6506)
add 70a724a9c3 [DOCS] Fix Slack invite link in README.md (#6648)
add 6badae46f0 [HUDI-3558] Consistent bucket index: bucket resizing
(split&merge) & concurrent write during resizing (#4958)
add cda31c65c3 [MINOR] Add dev setup and spark 3.3 profile to readme
(#6656)
add 8dea9cf89d [HUDI-4831] Fix
AWSDmsAvroPayload#getInsertValue,combineAndGetUpdateValue to invoke correct api
(#6637)
add ddb1c1aff6 [HUDI-4806] Use Avro version from the root pom for Flink
bundle (#6628)
add dc5ec0ce0f [HUDI-4833] Add Postgres Schema Name to Postgres Debezium
Source (#6616)
add 3cfe4855c2 [HUDI-4825] Remove redundant fields in serialized commit
metadata in JSON (#6646)
add a942d9d1ba [MINOR] Insert should call validateInsertSchema in
HoodieFlinkWriteClient (#5919)
add 1b2179269e [HUDI-3879] Suppress exceptions that are not fatal in
HoodieMetadataTableValidator (#5344)
add 03c49cae52 [HUDI-3998] Fix getCommitsSinceLastCleaning failed when
async cleaning (#5478)
add ddda7fad91 [HUDI-3994] - Added support for initializing DeltaStreamer
without a defined Spark Master (#5630)
add c7c64a7f68 [HUDI-4628] Hudi-flink support
GLOBAL_BLOOM，GLOBAL_SIMPLE，BUCKET index type (#6406)
add 45ec376777 [HUDI-4814] Schedules new clustering plan based on latest
clustering instant (#6574)
add 39dba79640 [HUDI-4817] Delete markers after full-record bootstrap
operation (#6667)
add 8c296e0356 [HUDI-4691] Cleaning up duplicated classes in Spark 3.3
module (#6550)
add 1f2e72e06e [HUDI-4752] Add dedup support for MOR table in cli (#6608)
add 35d03e9a1b [HUDI-4837] Stop sleeping where it is not necessary after
the success (#6270)
add 7ab59b9a0c [HUDI-4843] Delete the useless timer in
BaseRollbackActionExecutor (#6671)
add 851c6e12db [HUDI-4780] hoodie.logfile.max.size It does not take
effect, causing the log file to be too large (#6602)
add a1dedf3d59 [HUDI-4844] Skip partition value resolving when the field
does not exists for MergeOnReadInputFormat#getReader (#6678)
add c22568ee28 [MINOR] Fix the Spark job status description for
metadata-only bootstrap operation (#6666)
add a3921a845f [HUDI-3403] Ensure keygen props are set for bootstrap
(#6645)
add 39f562f4fa [HUDI-4193] Upgrade Protobuf to 3.21.5 (#5784)
add 488f58d770 [HUDI-4785] Fix partition discovery in bootstrap operation
(#6673)
add 22d6019559 [HUDI-4706] Fix InternalSchemaChangeApplier#applyAddChange
error to add nest type (#6486)
add 6e31b7cef4 [HUDI-4851] Fixing CSI not handling `InSet` operator
properly (#6685)
add bf64e60d31 [HUDI-4796] MetricsReporter stop bug (#6619)
add c2b72306bd [HUDI-3861] update tblp 'path' when rename table (#5320)
add f70678f435 [HUDI-4853] Get field by name for
OverwriteNonDefaultsWithLatestAvroPayload to avoid schema mismatch (#6689)
add 3faddb7da0 [HUDI-4813] Fix infer keygen not work in sparksql side
issue (#6634)
add 36fe10a0b2 [HUDI-4856] Missing option for HoodieCatalogFactory (#6693)
add 21cbfce617 [HUDI-4864] Fix AWSDmsAvroPayload#combineAndGetUpdateValue
when using MOR snapshot query after delete operations with test (#6688)
add f1caa3a2f9 [HUDI-4841] Fix sort idempotency issue (#6669)
add d95caa9300 [HUDI-4865] Optimize HoodieAvroUtils#isMetadataField to use
O(1) complexity (#6702)
add fd7bb181cf [HUDI-4736] Fix inflight clean action preventing clean
service to continue when multiple cleans are not allowed (#6536)
add cd2ea2a10b [HUDI-4842] Support compaction strategy based on delta log
file num (#6670)
add eaa2f8ed3b [HUDI-4282] Repair IOException in CHDFS when check block
corrupted in HoodieLogFileReader (#6031)
add 7e83f61309 [HUDI-4757] Create pyspark examples (#6672)
add 05adfa2930 [HUDI-3959] Rename class name for spark rdd reader (#5409)
add 8dfdd15f89 [HUDI-4828] Fix the extraction of record keys which may be
cut out (#6650)
add 5cbd1d83ba [HUDI-4873] Report number of messages to be processed via
metrics (#6271)
new 53a7f41c8f [HUDI-3378][HUDI-3379][HUDI-3381] Migrate usage of
HoodieRecordPayload and raw Avro payload to HoodieRecord (#5522)
new 8f22658196 [MINOR] Fix type casting in TestHoodieHFileReaderWriter
new 219f171f54 [HUDI-4292][RFC-46] Update doc to align with the Record
Merge API changes (#5927)
new def9dd6561 [HUDI-4344] fix usage of HoodieDataBlock#getRecordIterator
(#6005)
new d0300c2c2c [HUDI-3350][HUDI-3351] Support HoodieMerge API and Spark
engine-specific HoodieRecord (#5627)
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (bb208e9861)
\
N -- N -- N refs/heads/release-feature-rfc46 (d0300c2c2c)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
README.md | 13 +-
azure-pipelines.yml | 14 +-
docker/demo/config/dfs-source.properties | 4 +
hudi-aws/pom.xml | 6 +
.../hudi/aws/sync/AWSGlueCatalogSyncClient.java | 2 +-
hudi-cli/pom.xml | 12 +-
.../apache/hudi/cli/HoodieTableHeaderFields.java | 32 ++
.../apache/hudi/cli/commands/CommitsCommand.java | 229 ++++----
.../hudi/cli/commands/CompactionCommand.java | 52 +-
.../org/apache/hudi/cli/commands/DiffCommand.java | 184 +++++++
.../cli/commands/HoodieSyncValidateCommand.java | 5 +-
.../apache/hudi/cli/commands/RepairsCommand.java | 57 ++
.../apache/hudi/cli/commands/RollbacksCommand.java | 34 ++
.../org/apache/hudi/cli/commands/SparkMain.java | 92 +++-
.../java/org/apache/hudi/cli/utils/CommitUtil.java | 4 +-
.../scala/org/apache/hudi/cli/DedupeSparkJob.scala | 6 +-
.../hudi/cli/commands/TestCommitsCommand.java | 87 ++--
.../hudi/cli/commands/TestCompactionCommand.java | 2 +-
.../apache/hudi/cli/commands/TestDiffCommand.java | 147 ++++++
.../hudi/cli/commands/TestRepairsCommand.java | 132 +++++
.../cli/functional/CLIFunctionalTestHarness.java | 4 +-
.../hudi/cli/integ/ITTestRepairsCommand.java | 117 +++--
.../HoodieTestCommitMetadataGenerator.java | 32 +-
hudi-client/hudi-client-common/pom.xml | 6 +
.../apache/hudi/client/BaseHoodieWriteClient.java | 38 +-
.../client/embedded/EmbeddedTimelineService.java | 5 +
.../transaction/lock/InProcessLockProvider.java | 2 +-
.../apache/hudi/config/HoodieArchivalConfig.java | 10 +-
.../apache/hudi/config/HoodieBootstrapConfig.java | 20 +-
.../apache/hudi/config/HoodieClusteringConfig.java | 43 +-
.../apache/hudi/config/HoodieCompactionConfig.java | 11 +
.../org/apache/hudi/config/HoodieIndexConfig.java | 54 ++
.../org/apache/hudi/config/HoodieWriteConfig.java | 41 +-
.../java/org/apache/hudi/index/HoodieIndex.java | 12 +
.../index/bucket/BucketIndexLocationMapper.java | 4 +-
.../index/bucket/ConsistentBucketIdentifier.java | 100 +++-
.../hudi/index/bucket/HoodieBucketIndex.java | 6 +-
.../hudi/index/bucket/HoodieSimpleBucketIndex.java | 4 +-
.../org/apache/hudi/io/HoodieAppendHandle.java | 6 +
.../java/org/apache/hudi/keygen/KeyGenUtils.java | 27 +-
.../metadata/HoodieBackedTableMetadataWriter.java | 6 +-
.../hudi/metrics/ConsoleMetricsReporter.java | 15 +-
.../hudi/metrics/InMemoryMetricsReporter.java | 7 -
.../apache/hudi/metrics/JmxMetricsReporter.java | 7 -
.../main/java/org/apache/hudi/metrics/Metrics.java | 16 +-
.../hudi/metrics/MetricsGraphiteReporter.java | 6 -
.../org/apache/hudi/metrics/MetricsReporter.java | 4 -
.../cloudwatch/CloudWatchMetricsReporter.java | 6 -
.../metrics/datadog/DatadogMetricsReporter.java | 6 -
.../metrics/prometheus/PrometheusReporter.java | 13 +-
.../prometheus/PushGatewayMetricsReporter.java | 11 +-
.../table/action/clean/CleanActionExecutor.java | 7 +-
.../action/clean/CleanPlanActionExecutor.java | 17 +-
.../hudi/table/action/clean/CleanPlanner.java | 11 +
.../cluster/ClusteringPlanActionExecutor.java | 17 +-
.../cluster/strategy/ClusteringPlanStrategy.java | 7 +
.../PartitionAwareClusteringPlanStrategy.java | 4 +
.../action/cluster/strategy/UpdateStrategy.java | 12 +-
.../table/action/commit/BaseBulkInsertHelper.java | 2 +
.../action/commit/BaseCommitActionExecutor.java | 2 +-
...java => LogFileNumBasedCompactionStrategy.java} | 25 +-
.../action/index/ScheduleIndexActionExecutor.java | 2 +-
.../action/restore/BaseRestoreActionExecutor.java | 12 +-
.../rollback/BaseRollbackActionExecutor.java | 6 +-
.../storage/HoodieConsistentBucketLayout.java | 5 +-
.../transaction/TestInProcessLockProvider.java | 41 ++
.../apache/hudi/config/TestHoodieWriteConfig.java | 29 +-
.../hudi/index/bucket/TestBucketIdentifier.java | 21 +-
.../bucket/TestConsistentBucketIdIdentifier.java | 165 +++++-
.../org/apache/hudi/keygen/TestKeyGenUtils.java | 24 +-
.../hudi/metrics/TestMetricsReporterFactory.java | 6 -
.../hudi/utils/TestMetadataConversionUtils.java | 7 +-
hudi-client/hudi-flink-client/pom.xml | 573 +++++++++++----------
.../hudi/client/FlinkTaskContextSupplier.java | 2 +-
.../apache/hudi/client/HoodieFlinkWriteClient.java | 10 +-
.../FlinkSizeBasedClusteringPlanStrategy.java | 23 +-
.../client/common/HoodieFlinkEngineContext.java | 7 +-
.../hudi/execution/FlinkLazyInsertIterable.java | 5 +
.../apache/hudi/index/FlinkHoodieIndexFactory.java | 9 +
.../java/org/apache/hudi/io/MiniBatchHandle.java | 3 +-
.../io/storage/row/HoodieRowDataParquetWriter.java | 2 +-
.../row/parquet/ParquetSchemaConverter.java | 5 +-
.../FlinkHoodieBackedTableMetadataWriter.java | 3 +
.../hudi/table/ExplicitWriteHandleTable.java | 22 +-
.../hudi/table/HoodieFlinkCopyOnWriteTable.java | 23 +-
.../hudi/table/HoodieFlinkMergeOnReadTable.java | 3 +
.../org/apache/hudi/table/HoodieFlinkTable.java | 3 +
.../commit/FlinkDeleteCommitActionExecutor.java | 3 +
.../table/action/commit/FlinkDeleteHelper.java | 3 +
.../commit/FlinkInsertCommitActionExecutor.java | 3 +
.../FlinkInsertOverwriteCommitActionExecutor.java | 3 +
...nkInsertOverwriteTableCommitActionExecutor.java | 3 +
.../FlinkInsertPreppedCommitActionExecutor.java | 3 +
.../hudi/table/action/commit/FlinkMergeHelper.java | 3 +
.../commit/FlinkUpsertCommitActionExecutor.java | 3 +
.../FlinkUpsertPreppedCommitActionExecutor.java | 3 +
.../delta/BaseFlinkDeltaCommitActionExecutor.java | 3 +
.../FlinkUpsertDeltaCommitActionExecutor.java | 3 +
...linkUpsertPreppedDeltaCommitActionExecutor.java | 3 +
.../common/TestHoodieFlinkEngineContext.java | 2 +-
.../index/bloom/TestFlinkHoodieBloomIndex.java | 13 +-
.../testutils/HoodieFlinkClientTestHarness.java | 9 +-
.../testutils/HoodieFlinkWriteableTestTable.java | 5 +-
hudi-client/hudi-java-client/pom.xml | 6 +
.../JavaSizeBasedClusteringPlanStrategy.java | 15 +-
.../src/test/resources/testDataGeneratorSchema.txt | 3 +-
hudi-client/hudi-spark-client/pom.xml | 12 +-
.../org/apache/hudi/client/HoodieReadClient.java | 189 +------
...odieReadClient.java => SparkRDDReadClient.java} | 22 +-
.../apache/hudi/client/SparkRDDWriteClient.java | 22 +-
...parkConsistentBucketClusteringPlanStrategy.java | 322 ++++++++++++
.../strategy/SparkSingleFileSortPlanStrategy.java | 11 +-
.../SparkSizeBasedClusteringPlanStrategy.java | 15 +-
.../MultipleSparkJobExecutionStrategy.java | 6 +-
...onsistentBucketClusteringExecutionStrategy.java | 79 +++
.../SparkSingleFileSortExecutionStrategy.java | 3 +-
.../SparkSortAndSizeExecutionStrategy.java | 3 +-
.../update/strategy/BaseSparkUpdateStrategy.java | 7 +-
.../update/strategy/SparkAllowUpdateStrategy.java | 10 +-
...arkConsistentBucketDuplicateUpdateStrategy.java | 150 ++++++
.../update/strategy/SparkRejectUpdateStrategy.java | 9 +-
.../hudi/client/model/HoodieInternalRow.java | 6 +-
.../BulkInsertInternalPartitionerFactory.java | 11 +
.../bulkinsert/RDDBucketIndexPartitioner.java} | 14 +-
.../bulkinsert/RDDConsistentBucketPartitioner.java | 277 ++++++++++
.../bucket/HoodieSparkConsistentBucketIndex.java | 96 +++-
.../apache/hudi/keygen/BuiltinKeyGenerator.java | 1 -
.../org/apache/hudi/keygen/SimpleKeyGenerator.java | 7 +
.../SparkBootstrapCommitActionExecutor.java | 78 ++-
.../SparkBootstrapDeltaCommitActionExecutor.java | 12 +-
.../commit/BaseSparkCommitActionExecutor.java | 44 +-
.../table/action/commit/SparkBulkInsertHelper.java | 16 +-
.../commit/SparkInsertOverwritePartitioner.java | 1 +
.../BaseSparkDeltaCommitActionExecutor.java | 3 +-
.../org/apache/spark/sql/HoodieCatalogUtils.scala | 10 +-
.../spark/sql/HoodieCatalystPlansUtils.scala | 2 +-
.../datasources/SparkParsePartitionUtil.scala | 12 +-
.../org/apache/spark/sql/hudi/SparkAdapter.scala | 14 +-
.../apache/hudi/client/TestHoodieReadClient.java | 8 +-
...parkConsistentBucketClusteringPlanStrategy.java | 190 +++++++
.../functional/TestConsistentBucketIndex.java | 138 +++--
.../functional/TestHoodieBackedTableMetadata.java | 59 +++
.../TestSparkConsistentBucketClustering.java | 292 +++++++++++
.../hudi/client/model/TestHoodieInternalRow.java | 22 +
.../apache/hudi/index/TestHoodieIndexConfigs.java | 2 +-
.../index/bucket/TestHoodieSimpleBucketIndex.java | 2 +-
.../java/org/apache/hudi/table/TestCleaner.java | 66 ++-
.../hudi/table/TestHoodieMergeOnReadTable.java | 30 +-
.../commit/TestCopyOnWriteActionExecutor.java | 2 +-
.../table/action/compact/CompactionTestBase.java | 4 +-
.../table/action/compact/TestAsyncCompaction.java | 20 +-
.../table/action/compact/TestInlineCompaction.java | 26 +-
.../strategy/TestHoodieCompactionStrategy.java | 54 +-
.../TestHoodieSparkMergeOnReadTableCompaction.java | 19 +-
...HoodieSparkMergeOnReadTableIncrementalRead.java | 2 +-
.../TestHoodieSparkMergeOnReadTableRollback.java | 9 +-
.../hudi/testutils/FunctionalTestHarness.java | 4 +-
.../hudi/testutils/HoodieClientTestHarness.java | 21 +-
.../hudi/testutils/HoodieClientTestUtils.java | 4 +-
.../hudi/testutils/HoodieMergeOnReadTestUtils.java | 45 +-
.../SparkClientFunctionalTestHarness.java | 16 +-
.../src/test/resources/testDataGeneratorSchema.txt | 3 +-
hudi-common/pom.xml | 18 +-
hudi-common/src/main/avro/HoodieCleanMetadata.avsc | 1 +
hudi-common/src/main/avro/HoodieCleanerPlan.avsc | 5 +
.../src/main/avro/HoodieClusteringGroup.avsc | 10 +
.../org/apache/hudi/BaseHoodieTableFileIndex.java | 87 +++-
.../java/org/apache/hudi/avro/HoodieAvroUtils.java | 12 +-
.../org/apache/hudi/common/HoodieCleanStat.java | 26 +-
.../hudi/common/fs/BoundedFsDataInputStream.java | 68 +++
.../java/org/apache/hudi/common/fs/FSUtils.java | 21 +-
.../common/fs/SizeAwareFSDataOutputStream.java | 2 +-
.../hudi/common/model/AWSDmsAvroPayload.java | 15 +-
.../org/apache/hudi/common/model/BaseFile.java | 9 +
.../hudi/common/model/ConsistentHashingNode.java | 27 +-
.../org/apache/hudi/common/model/FileSlice.java | 5 +
.../model/HoodieConsistentHashingMetadata.java | 24 +
.../org/apache/hudi/common/model/HoodieRecord.java | 5 +-
.../OverwriteNonDefaultsWithLatestAvroPayload.java | 2 +-
.../common/model/debezium/DebeziumConstants.java | 2 +
.../hudi/common/table/HoodieTableMetaClient.java | 12 +-
.../hudi/common/table/TableSchemaResolver.java | 57 +-
.../hudi/common/table/log/HoodieLogFileReader.java | 6 +-
.../table/log/block/HoodieHFileDataBlock.java | 26 +-
.../table/timeline/HoodieActiveTimeline.java | 41 ++
.../table/timeline/HoodieArchivedTimeline.java | 2 +-
.../table/timeline/HoodieDefaultTimeline.java | 15 +
.../table/timeline/HoodieInstantTimeGenerator.java | 16 +-
.../hudi/common/table/timeline/TimelineUtils.java | 10 +-
.../clean/CleanMetadataV1MigrationHandler.java | 1 +
.../clean/CleanMetadataV2MigrationHandler.java | 7 +-
.../clean/CleanPlanV1MigrationHandler.java | 10 +-
.../clean/CleanPlanV2MigrationHandler.java | 4 +-
.../table/view/AbstractTableFileSystemView.java | 13 +-
.../common/table/view/FileSystemViewManager.java | 3 +-
.../table/view/FileSystemViewStorageConfig.java | 76 +++
.../view/RemoteHoodieTableFileSystemView.java | 59 ++-
.../org/apache/hudi/common/util/CleanerUtils.java | 6 +-
.../apache/hudi/common/util/CollectionUtils.java | 9 +
.../org/apache/hudi/common/util/CommitUtils.java | 29 +-
.../common/util/HoodieRecordSizeEstimator.java | 4 +-
.../org/apache/hudi/common/util/JsonUtils.java | 6 +
.../org/apache/hudi/common/util/RetryHelper.java | 52 +-
.../java/org/apache/hudi/hadoop/CachingPath.java | 59 ++-
.../org/apache/hudi/hadoop/SerializablePath.java | 9 +-
.../schema/action/InternalSchemaChangeApplier.java | 3 +-
.../internal/schema/action/TableChangesHelper.java | 5 +
.../hudi/internal/schema/utils/SerDeHelper.java | 4 +-
.../apache/hudi/io/storage/HoodieHFileUtils.java | 4 +-
.../apache/hudi/metadata/BaseTableMetadata.java | 135 ++---
.../hudi/metadata/HoodieBackedTableMetadata.java | 27 +-
.../hudi/metadata/HoodieMetadataPayload.java | 18 +-
.../apache/hudi/metadata/HoodieTableMetadata.java | 21 +-
.../org/apache/hudi/avro/TestHoodieAvroUtils.java | 6 +-
.../hudi/common/model/TestAWSDmsAvroPayload.java | 35 +-
.../common/model/TestHoodieCommitMetadata.java | 29 ++
.../model/TestHoodieReplaceCommitMetadata.java | 47 ++
...tOverwriteNonDefaultsWithLatestAvroPayload.java | 59 ++-
.../hudi/common/table/TestTableSchemaResolver.java | 12 +-
.../hudi/common/table/TestTimelineUtils.java | 5 +-
.../table/timeline/TestHoodieActiveTimeline.java | 23 +-
.../table/view/TestIncrementalFSViewSync.java | 4 +-
.../hudi/common/testutils/FileCreateUtils.java | 14 +-
.../common/testutils/HoodieTestDataGenerator.java | 3 +-
.../hudi/common/testutils/HoodieTestTable.java | 13 +-
.../hudi/common/testutils/HoodieTestUtils.java | 35 +-
.../internal/schema/action/TestTableChanges.java | 86 ++++
hudi-examples/hudi-examples-java/pom.xml | 7 +
hudi-examples/hudi-examples-spark/pom.xml | 25 +-
.../quickstart/TestHoodieSparkQuickstart.java | 5 +-
.../src/test/python/HoodiePySparkQuickstart.py | 266 ++++++++++
.../hudi-examples-spark/src/test/python/README.md | 42 ++
hudi-flink-datasource/hudi-flink/pom.xml | 14 +
.../apache/hudi/configuration/FlinkOptions.java | 14 +-
.../hudi/configuration/HadoopConfigurations.java | 2 +-
.../java/org/apache/hudi/sink/CleanFunction.java | 16 +-
.../hudi/sink/clustering/ClusteringCommitSink.java | 7 +
.../hudi/sink/clustering/ClusteringOperator.java | 5 +
.../sink/clustering/FlinkClusteringConfig.java | 61 ++-
.../sink/clustering/HoodieFlinkClusteringJob.java | 60 ++-
.../hudi/sink/compact/CompactionPlanOperator.java | 2 +-
.../hudi/sink/compact/HoodieFlinkCompactor.java | 16 +
.../java/org/apache/hudi/sink/utils/Pipelines.java | 20 +-
.../hudi/source/stats/ExpressionEvaluator.java | 1 +
.../apache/hudi/streamer/FlinkStreamerConfig.java | 4 +-
.../org/apache/hudi/table/HoodieTableSource.java | 2 +-
.../apache/hudi/table/catalog/CatalogOptions.java | 24 +
.../apache/hudi/table/catalog/HiveSchemaUtils.java | 34 +-
.../hudi/table/catalog/HoodieCatalogFactory.java | 10 +-
.../hudi/table/catalog/HoodieHiveCatalog.java | 14 +-
.../table/format/cow/CopyOnWriteInputFormat.java | 19 +-
.../table/format/mor/MergeOnReadInputFormat.java | 8 +-
.../org/apache/hudi/util/AvroSchemaConverter.java | 33 +-
.../apache/hudi/util/AvroToRowDataConverters.java | 3 +
.../java/org/apache/hudi/util/ClusteringUtil.java | 2 +-
.../java/org/apache/hudi/util/CompactionUtil.java | 2 +-
.../java/org/apache/hudi/util/DataTypeUtils.java | 15 +-
.../hudi/util/FlinkStateBackendConverter.java | 6 +-
.../java/org/apache/hudi/util/HoodiePipeline.java | 62 +--
.../apache/hudi/util/RowDataToAvroConverters.java | 6 +-
.../java/org/apache/hudi/util/StreamerUtil.java | 13 +-
.../apache/hudi/sink/ITTestDataStreamWrite.java | 4 +-
.../org/apache/hudi/sink/TestWriteMergeOnRead.java | 52 +-
.../sink/compact/ITTestHoodieFlinkCompactor.java | 2 +-
.../sink/compact/TestCompactionPlanStrategy.java | 39 +-
.../hudi/source/stats/TestExpressionEvaluator.java | 2 +-
.../apache/hudi/table/ITTestHoodieDataSource.java | 36 +-
.../hudi/table/catalog/HoodieCatalogTestUtils.java | 8 +-
.../table/catalog/TestHoodieCatalogFactory.java | 28 +-
.../hudi/table/catalog/TestHoodieHiveCatalog.java | 35 +-
.../table/format/cow/TestBlockLocationSort.java | 52 ++
.../apache/hudi/utils/TestAvroSchemaConverter.java | 38 ++
.../test/java/org/apache/hudi/utils/TestData.java | 12 +-
.../hudi-flink/src/test/resources/hive-site.xml | 16 +-
.../test-catalog-factory-conf/hive-site.xml | 10 +-
hudi-flink-datasource/hudi-flink1.13.x/pom.xml | 15 +
.../table/format/cow/ParquetSplitReaderUtil.java | 6 +-
hudi-flink-datasource/hudi-flink1.14.x/pom.xml | 15 +
.../table/format/cow/ParquetSplitReaderUtil.java | 6 +-
hudi-flink-datasource/hudi-flink1.15.x/pom.xml | 15 +
.../table/format/cow/ParquetSplitReaderUtil.java | 6 +-
.../format/cow/vector/HeapMapColumnVector.java | 2 +-
hudi-gcp/pom.xml | 6 +
hudi-hadoop-mr/pom.xml | 6 +
.../hudi/hadoop/testutils/InputFormatTestUtil.java | 5 +-
hudi-integ-test/pom.xml | 4 +-
.../testsuite/HoodieContinousTestSuiteWriter.java | 2 -
.../testsuite/HoodieInlineTestSuiteWriter.java | 8 -
.../testsuite/HoodieMultiWriterTestSuiteJob.java | 3 +-
.../integ/testsuite/HoodieTestSuiteWriter.java | 4 +-
.../SparkDataSourceContinuousIngestTool.java | 1 -
.../testsuite/configuration/DFSDeltaConfig.java | 2 +-
.../apache/hudi/integ/testsuite/dag/DagUtils.java | 28 +-
.../integ/testsuite/dag/nodes/BaseQueryNode.java | 3 +-
.../dag/nodes/BaseValidateDatasetNode.java | 24 +-
.../integ/testsuite/dag/nodes/HiveQueryNode.java | 3 +-
.../integ/testsuite/dag/nodes/HiveSyncNode.java | 1 -
.../integ/testsuite/dag/nodes/PrestoQueryNode.java | 3 +-
.../integ/testsuite/dag/nodes/TrinoQueryNode.java | 5 +-
.../dag/nodes/ValidateAsyncOperations.java | 11 +-
.../testsuite/dag/scheduler/DagScheduler.java | 1 -
.../GenericRecordFullPayloadGenerator.java | 6 +-
.../testsuite/reader/DFSAvroDeltaInputReader.java | 8 +-
.../reader/DFSHoodieDatasetInputReader.java | 1 -
hudi-kafka-connect/README.md | 25 +
hudi-kafka-connect/pom.xml | 22 -
hudi-spark-datasource/hudi-spark-common/pom.xml | 12 +-
.../main/java/org/apache/hudi/DataSourceUtils.java | 4 +-
.../scala/org/apache/hudi/DataSourceOptions.scala | 10 +-
.../main/scala/org/apache/hudi/DefaultSource.scala | 2 +
.../scala/org/apache/hudi/HoodieBaseRelation.scala | 12 +-
.../org/apache/hudi/HoodieBootstrapRelation.scala | 7 +-
.../org/apache/hudi/HoodieSparkSqlWriter.scala | 19 +-
.../org/apache/hudi/HoodieStreamingSink.scala | 147 ++++--
.../apache/hudi/SparkHoodieTableFileIndex.scala | 28 +-
.../sql/catalyst/catalog/HoodieCatalogTable.scala | 9 +-
.../apache/spark/sql/hudi/DataSkippingUtils.scala | 20 +-
.../spark/sql/hudi/HoodieSqlCommonUtils.scala | 2 +-
.../command/AlterHoodieTableRenameCommand.scala | 11 +-
.../sql/hudi/command/payload/SqlTypedRecord.scala | 28 +-
hudi-spark-datasource/hudi-spark/pom.xml | 57 +-
hudi-spark-datasource/hudi-spark/run_hoodie_app.sh | 2 +-
.../hudi-spark/run_hoodie_generate_app.sh | 2 +-
.../hudi-spark/run_hoodie_streaming_app.sh | 2 +-
.../main/java/org/apache/hudi/QuickstartUtils.java | 36 +-
.../SparkFullBootstrapDataProviderBase.java | 4 +-
.../org/apache/spark/sql/hudi/DedupeSparkJob.scala | 2 +-
.../spark/sql/hudi/analysis/HoodieAnalysis.scala | 20 +-
.../hudi/command/MergeIntoHoodieTableCommand.scala | 5 +-
.../hudi/command/payload/ExpressionPayload.scala | 33 +-
.../hudi/command/procedures/HoodieProcedures.scala | 114 ++--
.../procedures/RunClusteringProcedure.scala | 4 +-
.../procedures/ShowBootstrapMappingProcedure.scala | 1 -
.../procedures/ShowFileSystemViewProcedure.scala | 12 +-
.../procedures/ShowFsPathDetailProcedure.scala | 3 +-
.../ShowHoodieLogFileMetadataProcedure.scala | 9 +-
.../ShowHoodieLogFileRecordsProcedure.scala | 5 +-
.../procedures/StatsFileSizeProcedure.scala | 5 +-
.../StatsWriteAmplificationProcedure.scala | 3 +-
.../procedures/ValidateHoodieSyncProcedure.scala | 2 +-
.../src/test/java/HoodieJavaStreamingApp.java | 4 +-
.../org/apache/hudi/functional/TestBootstrap.java | 12 +-
.../org/apache/hudi/TestDataSkippingUtils.scala | 72 ++-
.../org/apache/hudi/TestHoodieSparkSqlWriter.scala | 20 +-
.../apache/hudi/functional/TestCOWDataSource.scala | 28 +-
.../functional/TestDataSourceForBootstrap.scala | 59 ++-
.../hudi/functional/TestStructuredStreaming.scala | 4 +-
.../hudi/functional/TestTimeTravelQuery.scala | 96 +++-
.../org/apache/spark/sql/hudi/TestAlterTable.scala | 78 ++-
.../apache/spark/sql/hudi/TestCreateTable.scala | 74 ++-
.../apache/spark/sql/hudi/TestMergeIntoTable.scala | 37 +-
.../spark/sql/hudi/TestMergeIntoTable2.scala | 37 ++
.../spark/sql/hudi/TestTimeTravelTable.scala | 59 +++
.../sql/hudi/procedure/TestCallCommandParser.scala | 14 +-
hudi-spark-datasource/hudi-spark2/pom.xml | 6 -
.../apache/spark/sql/adapter/Spark2Adapter.scala | 6 +-
.../apache/spark/sql/avro/AvroDeserializer.scala | 2 +-
.../org/apache/spark/sql/avro/AvroSerializer.scala | 2 +-
.../datasources/Spark2ParsePartitionUtil.scala | 14 +-
hudi-spark-datasource/hudi-spark3-common/pom.xml | 5 -
.../apache/hudi/spark3/internal/ReflectUtil.java | 4 +-
.../spark/sql/HoodieSpark3CatalogUtils.scala | 45 ++
.../spark/sql/HoodieSpark3CatalystPlanUtils.scala | 4 +-
.../spark/sql/adapter/BaseSpark3Adapter.scala | 8 +-
.../plans/logical/HoodieUnaryLikeSham.scala | 20 +-
.../sql/connector/catalog/HoodieIdentifier.scala | 0
.../datasources/Spark3ParsePartitionUtil.scala | 48 +-
hudi-spark-datasource/hudi-spark3.1.x/pom.xml | 5 -
...park31HoodieVectorizedParquetRecordReader.java} | 4 +-
...org.apache.spark.sql.sources.DataSourceRegister | 2 +-
...aultSource.scala => Spark31DefaultSource.scala} | 2 +-
.../spark/sql/HoodieSpark31CatalogUtils.scala} | 15 +-
.../apache/spark/sql/adapter/Spark3_1Adapter.scala | 4 +-
.../sql/avro/HoodieSpark3_1AvroDeserializer.scala | 9 +-
.../parquet/Spark31HoodieParquetFileFormat.scala | 2 +-
...a => Spark31ResolveHudiAlterTableCommand.scala} | 24 +-
...and312.scala => Spark31AlterTableCommand.scala} | 20 +-
hudi-spark-datasource/hudi-spark3.2.x/pom.xml | 30 +-
...org.apache.spark.sql.sources.DataSourceRegister | 2 +-
.../org/apache/hudi/Spark32HoodieFileScanRDD.scala | 3 -
.../spark/sql/HoodieSpark32CatalogUtils.scala | 19 +-
.../apache/spark/sql/adapter/Spark3_2Adapter.scala | 14 +-
.../parquet/Spark32HoodieParquetFileFormat.scala | 505 ------------------
.../hudi/Spark32ResolveHudiAlterTableCommand.scala | 1 -
.../sql/hudi/catalog/HoodieInternalV2Table.scala | 129 -----
.../pom.xml | 28 +-
...32PlusHoodieVectorizedParquetRecordReader.java} | 4 +-
.../apache/hudi/Spark32PlusDefaultSource.scala} | 2 +-
.../parquet/Spark32PlusDataSourceUtils.scala} | 2 +-
.../Spark32PlusHoodieParquetFileFormat.scala} | 18 +-
.../sql/hudi/analysis/HoodieSpark3Analysis.scala | 46 +-
.../spark/sql/hudi/catalog/BasicStagedTable.scala | 20 +-
.../spark/sql/hudi/catalog/HoodieCatalog.scala | 25 +-
.../sql/hudi/catalog/HoodieInternalV2Table.scala | 2 +-
.../spark/sql/hudi/catalog/HoodieStagedTable.scala | 17 +-
.../spark/sql/hudi/catalog/TableCreationMode.java | 46 +-
.../spark/sql/hudi/command/AlterTableCommand.scala | 0
.../sql/hudi}/logical/TimeTravelRelation.scala | 0
hudi-spark-datasource/hudi-spark3.3.x/pom.xml | 13 +-
...Spark33HoodieVectorizedParquetRecordReader.java | 191 -------
...org.apache.spark.sql.sources.DataSourceRegister | 2 +-
.../org/apache/hudi/Spark33HoodieFileScanRDD.scala | 5 +-
.../org/apache/hudi/Spark3DefaultSource.scala | 47 --
.../spark/sql/HoodieSpark33CatalogUtils.scala} | 15 +-
.../apache/spark/sql/adapter/Spark3_3Adapter.scala | 8 +-
.../plans/logical/TimeTravelRelation.scala | 33 --
.../sql/connector/catalog/HoodieIdentifier.scala | 43 --
.../parquet/Spark33DataSourceUtils.scala | 77 ---
.../hudi/Spark33ResolveHudiAlterTableCommand.scala | 1 -
.../sql/hudi/analysis/HoodieSpark3Analysis.scala | 222 --------
.../spark/sql/hudi/catalog/BasicStagedTable.scala | 57 --
.../spark/sql/hudi/catalog/HoodieCatalog.scala | 367 -------------
.../spark/sql/hudi/catalog/HoodieStagedTable.scala | 98 ----
.../spark/sql/hudi/catalog/TableCreationMode.java | 23 -
.../spark/sql/hudi/command/AlterTableCommand.scala | 347 -------------
.../test/resources/log4j-surefire-quiet.properties | 30 --
.../src/test/resources/log4j-surefire.properties | 31 --
hudi-sync/hudi-adb-sync/pom.xml | 11 +-
hudi-sync/hudi-datahub-sync/pom.xml | 7 -
hudi-sync/hudi-hive-sync/pom.xml | 15 +-
.../java/org/apache/hudi/hive/HiveSyncConfig.java | 7 +
.../org/apache/hudi/hive/ddl/HMSDDLExecutor.java | 32 +-
.../org/apache/hudi/hive/ddl/JDBCExecutor.java | 3 -
.../hudi/hive/ddl/QueryBasedDDLExecutor.java | 3 -
.../TestHiveMetastoreBasedLockProvider.java | 22 +
hudi-sync/hudi-sync-common/pom.xml | 9 +
hudi-tests-common/pom.xml | 20 +-
.../src/main/resources/log4j2.component.properties | 2 +-
hudi-timeline-service/pom.xml | 13 +-
.../hudi/timeline/service/RequestHandler.java | 38 +-
.../TestRemoteHoodieTableFileSystemView.java | 46 ++
hudi-utilities/pom.xml | 68 +--
.../utilities/HoodieMetadataTableValidator.java | 28 +-
.../org/apache/hudi/utilities/UtilHelpers.java | 18 +
.../hudi/utilities/deltastreamer/DeltaSync.java | 10 +-
.../deltastreamer/HoodieDeltaStreamer.java | 17 +-
.../deltastreamer/HoodieDeltaStreamerMetrics.java | 6 +
.../HoodieMultiTableDeltaStreamer.java | 6 +-
.../deltastreamer/SourceFormatAdapter.java | 37 +-
.../schema/ProtoClassBasedSchemaProvider.java | 78 +++
.../hudi/utilities/sources/AvroKafkaSource.java | 43 +-
.../hudi/utilities/sources/JsonKafkaSource.java | 43 +-
.../apache/hudi/utilities/sources/KafkaSource.java | 79 +++
.../hudi/utilities/sources/ProtoKafkaSource.java | 105 ++++
.../org/apache/hudi/utilities/sources/Source.java | 2 +-
.../sources/debezium/PostgresDebeziumSource.java | 2 +
.../utilities/sources/helpers/AvroConvertor.java | 6 +
.../sources/helpers/IncrSourceHelper.java | 2 +-
.../utilities/sources/helpers/KafkaOffsetGen.java | 4 +-
.../sources/helpers/ProtoConversionUtil.java | 359 +++++++++++++
.../sources/helpers/S3EventsMetaSelector.java | 4 +-
.../apache/hudi/utilities/TestHoodieIndexer.java | 4 +-
.../hudi/utilities/TestHoodieRepairTool.java | 4 +-
.../functional/TestHoodieDeltaStreamer.java | 45 +-
.../schema/TestProtoClassBasedSchemaProvider.java | 61 +++
...onKafkaSource.java => BaseTestKafkaSource.java} | 217 ++------
.../utilities/sources/TestHoodieIncrSource.java | 43 +-
.../utilities/sources/TestJsonKafkaSource.java | 267 +---------
.../sources/TestJsonKafkaSourcePostProcessor.java | 50 +-
.../utilities/sources/TestProtoKafkaSource.java | 216 ++++++++
.../sources/helpers/TestProtoConversionUtil.java | 367 +++++++++++++
.../sources/helpers/TestS3EventsMetaSelector.java | 24 +
.../resources/schema-provider/proto/sample.proto | 77 +++
.../proto/sample_schema_flattened.txt | 1 +
.../schema-provider/proto/sample_schema_nested.txt | 1 +
packaging/hudi-aws-bundle/pom.xml | 19 +-
packaging/hudi-flink-bundle/pom.xml | 5 +-
packaging/hudi-hadoop-mr-bundle/pom.xml | 1 -
packaging/hudi-hive-sync-bundle/pom.xml | 13 -
packaging/hudi-integ-test-bundle/pom.xml | 123 ++---
packaging/hudi-kafka-connect-bundle/pom.xml | 14 -
packaging/hudi-spark-bundle/pom.xml | 52 +-
packaging/hudi-timeline-server-bundle/pom.xml | 7 -
packaging/hudi-utilities-bundle/pom.xml | 54 +-
packaging/hudi-utilities-slim-bundle/pom.xml | 12 -
pom.xml | 150 +++++-
rfc/README.md | 1 +
rfc/rfc-57/rfc-57.md | 111 ++++
style/scalastyle.xml | 7 +-
479 files changed, 9770 insertions(+), 5640 deletions(-)
create mode 100644
hudi-cli/src/main/java/org/apache/hudi/cli/commands/DiffCommand.java
create mode 100644
hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestDiffCommand.java
copy
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/{UnBoundedCompactionStrategy.java
=> LogFileNumBasedCompactionStrategy.java} (51%)
copy
hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieConsistentHashingMetadata.java
=>
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/keygen/TestKeyGenUtils.java
(51%)
copy
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/{HoodieReadClient.java
=> SparkRDDReadClient.java} (90%)
create mode 100644
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/plan/strategy/SparkConsistentBucketClusteringPlanStrategy.java
create mode 100644
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/SparkConsistentBucketClusteringExecutionStrategy.java
create mode 100644
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/update/strategy/SparkConsistentBucketDuplicateUpdateStrategy.java
copy
hudi-client/{hudi-client-common/src/main/java/org/apache/hudi/execution/bulkinsert/BulkInsertSortMode.java
=>
hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/RDDBucketIndexPartitioner.java}
(70%)
create mode 100644
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/RDDConsistentBucketPartitioner.java
copy
hudi-flink-datasource/hudi-flink1.13.x/src/test/java/org/apache/hudi/adapter/StateInitializationContextAdapter.java
=>
hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/HoodieCatalogUtils.scala
(76%)
create mode 100644
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/clustering/plan/strategy/TestSparkConsistentBucketClusteringPlanStrategy.java
create mode 100644
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestSparkConsistentBucketClustering.java
create mode 100644
hudi-common/src/main/java/org/apache/hudi/common/fs/BoundedFsDataInputStream.java
create mode 100644
hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieReplaceCommitMetadata.java
create mode 100644
hudi-examples/hudi-examples-spark/src/test/python/HoodiePySparkQuickstart.py
create mode 100644 hudi-examples/hudi-examples-spark/src/test/python/README.md
create mode 100644
hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/format/cow/TestBlockLocationSort.java
create mode 100644
hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/HoodieSpark3CatalogUtils.scala
copy hudi-cli/src/main/java/org/apache/hudi/cli/Main.java =>
hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/HoodieUnaryLikeSham.scala
(64%)
rename hudi-spark-datasource/{hudi-spark3.2.x =>
hudi-spark3-common}/src/main/scala/org/apache/spark/sql/connector/catalog/HoodieIdentifier.scala
(100%)
rename
hudi-spark-datasource/hudi-spark3.1.x/src/main/java/org/apache/spark/sql/execution/datasources/parquet/{Spark312HoodieVectorizedParquetRecordReader.java
=> Spark31HoodieVectorizedParquetRecordReader.java} (97%)
rename
hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/hudi/{Spark3xDefaultSource.scala
=> Spark31DefaultSource.scala} (93%)
copy
hudi-spark-datasource/{hudi-spark2/src/main/scala/org/apache/spark/sql/avro/HoodieSpark2_4AvroSerializer.scala
=>
hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/HoodieSpark31CatalogUtils.scala}
(65%)
rename
hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/hudi/{Spark312ResolveHudiAlterTableCommand.scala
=> Spark31ResolveHudiAlterTableCommand.scala} (93%)
rename
hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/hudi/command/{AlterTableCommand312.scala
=> Spark31AlterTableCommand.scala} (94%)
copy
hudi-flink-datasource/hudi-flink1.14.x/src/test/java/org/apache/hudi/adapter/StateInitializationContextAdapter.java
=>
hudi-spark-datasource/hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/HoodieSpark32CatalogUtils.scala
(65%)
delete mode 100644
hudi-spark-datasource/hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark32HoodieParquetFileFormat.scala
delete mode 100644
hudi-spark-datasource/hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/hudi/catalog/HoodieInternalV2Table.scala
copy hudi-spark-datasource/{hudi-spark3-common =>
hudi-spark3.2plus-common}/pom.xml (90%)
rename
hudi-spark-datasource/{hudi-spark3.2.x/src/main/java/org/apache/spark/sql/execution/datasources/parquet/Spark32HoodieVectorizedParquetRecordReader.java
=>
hudi-spark3.2plus-common/src/main/java/org/apache/spark/sql/execution/datasources/parquet/Spark32PlusHoodieVectorizedParquetRecordReader.java}
(97%)
rename
hudi-spark-datasource/{hudi-spark3.2.x/src/main/scala/org/apache/hudi/Spark3DefaultSource.scala
=>
hudi-spark3.2plus-common/src/main/scala/org/apache/hudi/Spark32PlusDefaultSource.scala}
(94%)
rename
hudi-spark-datasource/{hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark32DataSourceUtils.scala
=>
hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark32PlusDataSourceUtils.scala}
(98%)
rename
hudi-spark-datasource/{hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark33HoodieParquetFileFormat.scala
=>
hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark32PlusHoodieParquetFileFormat.scala}
(96%)
rename hudi-spark-datasource/{hudi-spark3.2.x =>
hudi-spark3.2plus-common}/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSpark3Analysis.scala
(80%)
rename hudi-spark-datasource/{hudi-spark3.2.x =>
hudi-spark3.2plus-common}/src/main/scala/org/apache/spark/sql/hudi/catalog/BasicStagedTable.scala
(76%)
rename hudi-spark-datasource/{hudi-spark3.2.x =>
hudi-spark3.2plus-common}/src/main/scala/org/apache/spark/sql/hudi/catalog/HoodieCatalog.scala
(94%)
rename hudi-spark-datasource/{hudi-spark3.3.x =>
hudi-spark3.2plus-common}/src/main/scala/org/apache/spark/sql/hudi/catalog/HoodieInternalV2Table.scala
(97%)
rename hudi-spark-datasource/{hudi-spark3.2.x =>
hudi-spark3.2plus-common}/src/main/scala/org/apache/spark/sql/hudi/catalog/HoodieStagedTable.scala
(85%)
rename hudi-spark-datasource/{hudi-spark3.2.x =>
hudi-spark3.2plus-common}/src/main/scala/org/apache/spark/sql/hudi/catalog/TableCreationMode.java
(97%)
rename hudi-spark-datasource/{hudi-spark3.2.x =>
hudi-spark3.2plus-common}/src/main/scala/org/apache/spark/sql/hudi/command/AlterTableCommand.scala
(100%)
rename
hudi-spark-datasource/{hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/catalyst/plans
=>
hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/hudi}/logical/TimeTravelRelation.scala
(100%)
delete mode 100644
hudi-spark-datasource/hudi-spark3.3.x/src/main/java/org/apache/spark/sql/execution/datasources/parquet/Spark33HoodieVectorizedParquetRecordReader.java
delete mode 100644
hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/hudi/Spark3DefaultSource.scala
copy
hudi-spark-datasource/{hudi-spark2/src/main/scala/org/apache/spark/sql/avro/HoodieSpark2_4AvroSerializer.scala
=>
hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/HoodieSpark33CatalogUtils.scala}
(64%)
delete mode 100644
hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/TimeTravelRelation.scala
delete mode 100644
hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/connector/catalog/HoodieIdentifier.scala
delete mode 100644
hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark33DataSourceUtils.scala
delete mode 100644
hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSpark3Analysis.scala
delete mode 100644
hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/hudi/catalog/BasicStagedTable.scala
delete mode 100644
hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/hudi/catalog/HoodieCatalog.scala
delete mode 100644
hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/hudi/catalog/HoodieStagedTable.scala
delete mode 100644
hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/hudi/catalog/TableCreationMode.java
delete mode 100644
hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/hudi/command/AlterTableCommand.scala
delete mode 100644
hudi-spark-datasource/hudi-spark3.3.x/src/test/resources/log4j-surefire-quiet.properties
delete mode 100644
hudi-spark-datasource/hudi-spark3.3.x/src/test/resources/log4j-surefire.properties
copy
hudi-utilities/src/test/resources/delta-streamer-config/clusteringjob.properties
=> hudi-tests-common/src/main/resources/log4j2.component.properties (94%)
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/ProtoClassBasedSchemaProvider.java
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/KafkaSource.java
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/ProtoKafkaSource.java
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/ProtoConversionUtil.java
create mode 100644
hudi-utilities/src/test/java/org/apache/hudi/utilities/schema/TestProtoClassBasedSchemaProvider.java
copy
hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/{TestJsonKafkaSource.java
=> BaseTestKafkaSource.java} (50%)
create mode 100644
hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestProtoKafkaSource.java
create mode 100644
hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/helpers/TestProtoConversionUtil.java
create mode 100644
hudi-utilities/src/test/resources/schema-provider/proto/sample.proto
create mode 100644
hudi-utilities/src/test/resources/schema-provider/proto/sample_schema_flattened.txt
create mode 100644
hudi-utilities/src/test/resources/schema-provider/proto/sample_schema_nested.txt
create mode 100644 rfc/rfc-57/rfc-57.md
