This is an automated email from the ASF dual-hosted git repository.
yuzhaojing pushed a change to branch release-feature-rfc46
in repository https://gitbox.apache.org/repos/asf/hudi.git
omit 41392e119f [RFC-46][HUDI-4414] Update the RFC-46 doc to fix comments
feedback (#6132)
omit bd45932e04 [HUDI-4301] [HUDI-3384][HUDI-3385] Spark specific file
reader/writer.(#5629)
omit d0300c2c2c [HUDI-3350][HUDI-3351] Support HoodieMerge API and Spark
engine-specific HoodieRecord (#5627)
omit def9dd6561 [HUDI-4344] fix usage of HoodieDataBlock#getRecordIterator
(#6005)
omit 219f171f54 [HUDI-4292][RFC-46] Update doc to align with the Record
Merge API changes (#5927)
omit 8f22658196 [MINOR] Fix type casting in TestHoodieHFileReaderWriter
omit 53a7f41c8f [HUDI-3378][HUDI-3379][HUDI-3381] Migrate usage of
HoodieRecordPayload and raw Avro payload to HoodieRecord (#5522)
add 5a28f7f153 [HUDI-4870] Improve compaction config description (#6706)
add 3160c89d6d [HUDI-3304] Support partial update payload (#4676)
add 3a5bddaca8 [HUDI-4808] Fix HoodieSimpleBucketIndex not consider bucket
num in lo… (#6630)
add c0eae6dfde [HUDI-4485] Bump spring shell to 2.1.1 in CLI (#6489)
add 1b2792cefa [minor] following 3304, some code refactoring (#6713)
add 8962946996 [HUDI-4832] Fix drop partition meta sync (#6662)
add 7f9e43be58 [HUDI-4810] Fix log4j imports to use bridge API (#6710)
add b5dc4312a4 [HUDI-4877] Fix
org.apache.hudi.index.bucket.TestHoodieSimpleBucketIndex#testTagLocation not
work correct issue (#6717)
add e03c0388d9 [HUDI-4326] add updateTableSerDeInfo for HiveSyncTool
(#5920)
add 6b7157536d [MINOR] fix indent to make build pass (#6721)
add e8c26c5639 [HUDI-3478] Implement CDC Write in Spark (#6697)
add 61ed29271b [HUDI-4326] Fix hive sync serde properties (#6722)
add 2506c0fd2c [HUDI-4875] Fix NoSuchTableException when dropping
temporary view after applied HoodieSparkSessionExtension in Spark 3.2 (#6709)
add 6dbe2960f2 [DOCS] Improve the quick start guide for Kafka Connect Sink
(#6708)
add 30f489e424 [HUDI-4729] Fix file group pending compaction cannot be
queried when query _ro table (#6516)
add 598d0694d2 [HUDI-3983] Fix ClassNotFoundException when using
hudi-spark-bundle to write table with hbase index (#6715)
add 84b05c8473 [HUDI-4758] Add validations to java spark examples (#6615)
add cbf9b83ca6 [HUDI-4792] Batch clean files to delete (#6580)
add 5e624698f7 [HUDI-4363] Support Clustering row writer to improve
performance (#6046)
add 90d678c0f7 [HUDI-3478][HUDI-4887] Use Avro as the format of persisted
cdc data (#6734)
add 0558b6cfd9 [HUDI-4851] Fixing handling of `UTF8String` w/in `InSet`
operator (#6739)
add a9e0823398 [HUDI-3901] Correct the description of hoodie.index.type
(#6749)
add 4c0c1c7a6b [MINOR] Add .mvn directory to gitignore (#6746)
add 1380075a2d [HUDI-4901] Add avro.version to Flink profiles (#6757)
add a47dec71c3 [HUDI-4559] Support hiveSync command based on Call Produce
Command (#6322)
add efd1d79fe2 [HUDI-4883] Supporting delete savepoint for MOR (#6744)
add efe553b327 [HUDI-4897] Refactor the merge handle in CDC mode (#6740)
add 092375fc1f [HUDI-3523] Introduce AddColumnSchemaPostProcessor to
support add columns to the end of a schema (#5031)
add 9bab1063f4 Revert "[HUDI-3523] Introduce AddColumnSchemaPostProcessor
to support add columns to the end of a schema (#5031)" (#6768)
add 01352ca716 [HUDI-3523] Introduce AddPrimitiveColumnSchemaPostProcessor
to support add new primitive column to the end of a schema (#6769)
add 08dbc24049 [HUDI-4903] Fix TestHoodieLogFormat`s minor typo (#6762)
add 377a20d708 [MINOR] Drastically reducing concurrency level (to avoid CI
flakiness) (#6754)
add f34a2d733f Update HoodieIndex.java
add b512a4a3ff Merge pull request #6779 from wongxingjun/master
add c133fd9145 [HUDI-4906] Fix the local tests for hudi-flink (#6763)
add 4276494c46 [HUDI-4899] Fixing compatibility w/ Spark 3.2.2 (#6755)
add fd876a8568 [HUDI-4892] Fix hudi-spark3-bundle (#6735)
add 097e5a8687 [MINOR] Fix a few typos in HoodieIndex (#6784)
add 7cc614c604 [HUDI-4412] Fix multi writer INSERT_OVERWRITE NPE bug
(#6130)
add 60b62fc25c [MINOR] retain avro's namespace (#6783)
add 78528f6a5f [MINOR] Simple logging fix in LockManager (#6765)
add b873d8db31 [HUDI-4433] hudi-cli repair deduplicate not working with
non-partitioned dataset (#6349)
add fa6990bea1 [RFC-51][HUDI-3478] Update RFC: CDC support (#6256)
add 79b3e2b899 [HUDI-4915] improve avro serializer/deserializer (#6788)
add df69aa75db [HUDI-3478] Implement CDC Read in Spark (#6727)
add 8bf2f5738b [HUDI-4830] Fix testNoGlobalConfFileConfigured when add
hudi-defaults.conf in default dir (#6652)
add 6377b776b7 [HUDI-4760] Fixing repeated trigger of data file creations
w/ clustering (#6561)
add c54735cc87 [HUDI-4914] Managed memory weight should be set when sort
clustering is enabled (#6792)
add 3cae591e95 [HUDI-4910] Fix unknown variable or type "Cast" (#6778)
add 5c80acf497 [HUDI-4918] Fix bugs about when trying to show the non
-existing key from env, NullPointException occurs. (#6794)
add 5e58421f1b [HUDI-4718] Add Kerberos kinit command support. (#6719)
add 238628a9eb [HUDI-4902] Set default partitioner for SIMPLE BUCKET index
(#6759)
add 76b9354ec4 [MINOR] Update PR template with documentation update (#6748)
add db03e1fa70 [HUDI-4904] Add support for unraveling proto schemas in
ProtoClassBasedSchemaProvider (#6761)
add 28e3db103f [HUDI-4907] Prevent single commit multi instant issue
(#6766)
add a03a62e16d [HUDI-4923] Fix flaky
TestHoodieReadClient.testReadFilterExistAfterBulkInsertPrepped (#6801)
add a0e960b775 [HUDI-4848] Fixing repair deprecated partition tool (#6731)
add be9b4195ea [HUDI-4913] Fix HoodieSnapshotExporter for writing to a
different S3 bucket or FS (#6785)
add 96ce4b2f8e [HUDI-4453] Fix schema to include partition columns in
bootstrap operation (#6676)
add 9966b2c556 [HUDI-2780] Fix the issue of Mor log skipping complete
blocks when reading data (#4015)
add c733079734 [HUDI-4924] Auto-tune dedup parallelism (#6802)
add f13962281c [HUDI-4687] Avoid setAccessible which breaks strong
encapsulation (#6657)
add 79941fa6f3 [MINOR] fixing validate async operations to poll completed
clean instances (#6814)
add 5e4ab4f16e [HUDI-4734] Deltastreamer table config change validation
(#6753)
add 09d31b59b4 [HUDI-4934] Revert batch clean files (#6813)
add 58fe71d2b3 [HUDI-4722] Added locking metrics for Hudi (#6502)
add 0a39fce9fc [HUDI-4936] Fix `as.of.instant` not recognized as hoodie
config (#5616)
add 71c91c1804 [HUDI-4861] Relaxing `MERGE INTO` constraints to permit
limited casting operations w/in matched-on conditions (#6820)
add 986b63104d [HUDI-4885] Adding org.apache.avro to hudi-hive-sync bundle
(#6729)
add bfb4946f39 [HUDI-4951] Fix incorrect use of Long.getLong() (#6828)
add a65c3b798b [MINOR] Use base path URI in ITTestDataStreamWrite (#6826)
add 6b48c32f71 [HUDI-4308] READ_OPTIMIZED read mode will temporary loss of
data when compaction (#6664)
add b662cf6789 [HUDI-4237] Fixing empty partition-values being sync'd to
HMS (#6821)
add 15ca7a3060 [HUDI-4925] Should Force to use ExpressionPayload in
MergeIntoTableCommand (#6355)
add c73b3bdf7c [HUDI-4850] Add incremental source from GCS to Hudi (#6665)
add 45a699b83d [HUDI-4957] Shade JOL in bundles to fix
NoClassDefFoundError:GraphLayout (#6839)
add 9bd2a5220a [HUDI-4718] Add Kerberos kdestroy command support (#6810)
add f3d4ce919d [HUDI-4916] Implement change log feed for Flink (#6840)
add f171496de2 [HUDI-4769] Option read.streaming.skip_compaction skips
delta commit (#6848)
add 222592409a [HUDI-4949] optimize cdc read to avoid the problem of
reusing buffer underlying the Row (#6805)
add 41b397c104 [HUDI-4966] Add a partition extractor to handle partition
values with slashes (#6851)
add 30e89d6438 [MINOR] Fix testUpdateRejectForClustering (#6852)
add 48715d54c1 [HUDI-4962] Move cloud dependencies to cloud modules (#6846)
add 48e5bb0fed [HOTFIX] Fix source release validate script (#6865)
add 9f5d16529d [HUDI-4980] Calculate avg record size using commit only
(#6864)
add 067cc24d88 Revert "[HUDI-4915] improve avro serializer/deserializer
(#6788)" (#6809)
add fb4f026580 [HUDI-4970] Update kafka-connect readme and refactor
HoodieConfig#create (#6857)
add 280194d3b6 Enhancing README for multi-writer tests (#6870)
new 8a9dcf599b [HUDI-3378][HUDI-3379][HUDI-3381] Migrate usage of
HoodieRecordPayload and raw Avro payload to HoodieRecord (#5522)
new e7e0830cf6 [MINOR] Fix type casting in TestHoodieHFileReaderWriter
new a61e9db209 [HUDI-4292][RFC-46] Update doc to align with the Record
Merge API changes (#5927)
new 60edf301d7 [HUDI-4344] fix usage of HoodieDataBlock#getRecordIterator
(#6005)
new b2f5723ec0 [HUDI-3350][HUDI-3351] Support HoodieMerge API and Spark
engine-specific HoodieRecord (#5627)
new 5b0cdeb615 [HUDI-4301] [HUDI-3384][HUDI-3385] Spark specific file
reader/writer.(#5629)
new 517ca54fe2 [RFC-46][HUDI-4414] Update the RFC-46 doc to fix comments
feedback (#6132)
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (41392e119f)
\
N -- N -- N refs/heads/release-feature-rfc46 (517ca54fe2)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 7 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/PULL_REQUEST_TEMPLATE.md | 9 +
.gitignore | 1 +
docker/demo/compaction-bootstrap.commands | 30 +-
docker/demo/compaction.commands | 30 +-
docker/demo/sync-validate.commands | 30 +-
hudi-aws/pom.xml | 20 +-
hudi-cli/hudi-cli.sh | 4 +-
hudi-cli/pom.xml | 44 +-
.../java/org/apache/hudi/cli/HoodiePrompt.java | 24 +-
.../org/apache/hudi/cli/HoodieSplashScreen.java | 72 --
.../src/main/java/org/apache/hudi/cli/Main.java | 7 +-
.../hudi/cli/commands/ArchivedCommitsCommand.java | 52 +-
.../apache/hudi/cli/commands/BootstrapCommand.java | 74 +--
.../apache/hudi/cli/commands/CleansCommand.java | 55 +-
.../hudi/cli/commands/ClusteringCommand.java | 70 +-
.../apache/hudi/cli/commands/CommitsCommand.java | 145 +++--
.../hudi/cli/commands/CompactionCommand.java | 244 ++++---
.../org/apache/hudi/cli/commands/DiffCommand.java | 55 +-
.../apache/hudi/cli/commands/ExportCommand.java | 36 +-
.../hudi/cli/commands/FileSystemViewCommand.java | 96 ++-
.../cli/commands/HDFSParquetImportCommand.java | 49 +-
.../hudi/cli/commands/HoodieLogFileCommand.java | 55 +-
.../cli/commands/HoodieSyncValidateCommand.java | 28 +-
.../commands/KerberosAuthenticationCommand.java | 83 +++
.../apache/hudi/cli/commands/MarkersCommand.java | 23 +-
.../apache/hudi/cli/commands/MetadataCommand.java | 52 +-
.../apache/hudi/cli/commands/RepairsCommand.java | 102 +--
.../apache/hudi/cli/commands/RollbacksCommand.java | 51 +-
.../hudi/cli/commands/SavepointsCommand.java | 55 +-
.../apache/hudi/cli/commands/SparkEnvCommand.java | 24 +-
.../org/apache/hudi/cli/commands/SparkMain.java | 28 +-
.../org/apache/hudi/cli/commands/StatsCommand.java | 52 +-
.../org/apache/hudi/cli/commands/TableCommand.java | 73 ++-
.../apache/hudi/cli/commands/TempViewCommand.java | 22 +-
.../apache/hudi/cli/commands/TimelineCommand.java | 102 ++-
.../cli/commands/UpgradeOrDowngradeCommand.java | 34 +-
.../org/apache/hudi/cli/commands/UtilsCommand.java | 16 +-
.../apache/hudi/cli/utils/InputStreamConsumer.java | 8 +-
.../hudi/cli/utils/SparkTempViewProvider.java | 25 +-
.../META-INF/spring/spring-shell-plugin.xml | 28 -
hudi-cli/src/main/resources/application.yml | 23 +
hudi-cli/src/main/resources/banner.txt | 14 +
hudi-cli/src/main/resources/log4j2.properties | 38 ++
.../scala/org/apache/hudi/cli/DedupeSparkJob.scala | 6 +-
.../cli/commands/TestArchivedCommitsCommand.java | 29 +-
.../hudi/cli/commands/TestCleansCommand.java | 21 +-
.../hudi/cli/commands/TestCommitsCommand.java | 70 +-
.../hudi/cli/commands/TestCompactionCommand.java | 26 +-
.../apache/hudi/cli/commands/TestDiffCommand.java | 14 +-
.../cli/commands/TestFileSystemViewCommand.java | 33 +-
.../cli/commands/TestHoodieLogFileCommand.java | 29 +-
.../hudi/cli/commands/TestRepairsCommand.java | 35 +-
.../hudi/cli/commands/TestRollbacksCommand.java | 21 +-
.../hudi/cli/commands/TestSavepointsCommand.java | 19 +-
.../hudi/cli/commands/TestSparkEnvCommand.java | 21 +-
.../apache/hudi/cli/commands/TestStatsCommand.java | 21 +-
.../apache/hudi/cli/commands/TestTableCommand.java | 50 +-
.../hudi/cli/commands/TestTempViewCommand.java | 26 +-
.../cli/commands/TestUpgradeDowngradeCommand.java | 6 +-
.../apache/hudi/cli/commands/TestUtilsCommand.java | 33 +-
.../cli/functional/CLIFunctionalTestHarness.java | 15 +-
.../hudi/cli/integ/ITTestBootstrapCommand.java | 31 +-
.../hudi/cli/integ/ITTestClusteringCommand.java | 41 +-
.../hudi/cli/integ/ITTestCommitsCommand.java | 38 +-
.../hudi/cli/integ/ITTestCompactionCommand.java | 76 ++-
.../cli/integ/ITTestHDFSParquetImportCommand.java | 28 +-
.../hudi/cli/integ/ITTestMarkersCommand.java | 15 +-
.../hudi/cli/integ/ITTestRepairsCommand.java | 92 ++-
.../hudi/cli/integ/ITTestSavepointsCommand.java | 40 +-
.../testutils/HoodieCLIIntegrationTestHarness.java | 22 -
.../hudi/cli/testutils/MockCommandLineInput.java | 57 ++
.../cli/testutils/ShellEvaluationResultUtil.java | 21 +-
.../http/HoodieWriteCommitHttpCallbackClient.java | 4 +-
.../apache/hudi/client/BaseHoodieWriteClient.java | 4 +-
.../hudi/client/HoodieInternalWriteStatus.java | 17 +
.../client/transaction/ConcurrentOperation.java | 5 +-
.../hudi/client/transaction/lock/LockManager.java | 18 +-
.../lock/metrics/HoodieLockMetrics.java | 110 ++++
.../apache/hudi/config/HoodieCompactionConfig.java | 8 +-
.../org/apache/hudi/config/HoodieIndexConfig.java | 3 +-
.../org/apache/hudi/config/HoodieLayoutConfig.java | 11 +-
.../org/apache/hudi/config/HoodieWriteConfig.java | 24 +-
.../hudi/config/metrics/HoodieMetricsConfig.java | 16 +
.../hudi/execution/HoodieLazyInsertIterable.java | 2 +-
.../java/org/apache/hudi/index/HoodieIndex.java | 8 +-
.../org/apache/hudi/index/HoodieIndexUtils.java | 21 +
.../hudi/index/bucket/HoodieSimpleBucketIndex.java | 10 +-
.../org/apache/hudi/io/HoodieAppendHandle.java | 18 -
.../java/org/apache/hudi/io/HoodieCDCLogger.java | 256 ++++++++
.../java/org/apache/hudi/io/HoodieMergeHandle.java | 11 +-
.../apache/hudi/io/HoodieMergeHandleFactory.java | 100 +++
.../hudi/io/HoodieMergeHandleWithChangeLog.java | 108 +++
.../io/HoodieSortedMergeHandleWithChangeLog.java | 62 ++
.../java/org/apache/hudi/io/HoodieWriteHandle.java | 47 ++
.../org/apache/hudi/metrics/HoodieMetrics.java | 40 ++
.../hudi/metrics/datadog/DatadogHttpClient.java | 4 +-
.../hudi/metrics/datadog/DatadogReporter.java | 4 +-
.../hudi/table/action/clean/CleanPlanner.java | 25 +-
.../action/commit/BaseCommitActionExecutor.java | 3 +-
.../hudi/table/action/commit/BaseWriteHelper.java | 4 +-
.../table/action/commit/HoodieWriteHelper.java | 11 +-
.../table/action/savepoint/SavepointHelpers.java | 4 -
...urrentFileWritesConflictResolutionStrategy.java | 49 ++
.../apache/hudi/config/TestHoodieWriteConfig.java | 17 +
.../hudi/testutils/HoodieWriteableTestTable.java | 23 +-
.../apache/hudi/client/HoodieFlinkWriteClient.java | 50 +-
.../FlinkSizeBasedClusteringPlanStrategy.java | 3 +-
.../java/org/apache/hudi/io/FlinkAppendHandle.java | 6 -
.../FlinkMergeAndReplaceHandleWithChangeLog.java | 87 +++
.../hudi/io/FlinkMergeHandleWithChangeLog.java | 90 +++
.../apache/hudi/io/FlinkWriteHandleFactory.java | 282 ++++++++
.../java/org/apache/hudi/io/MiniBatchHandle.java | 8 -
.../hudi/table/HoodieFlinkCopyOnWriteTable.java | 11 +-
.../hudi/table/action/commit/FlinkWriteHelper.java | 6 +-
.../hudi/table/HoodieJavaCopyOnWriteTable.java | 11 +-
.../commit/BaseJavaCommitActionExecutor.java | 20 +-
.../hudi/table/action/commit/JavaWriteHelper.java | 8 +-
hudi-client/hudi-spark-client/pom.xml | 1 +
.../apache/hudi/client/SparkRDDWriteClient.java | 35 +-
.../HoodieSparkBootstrapSchemaProvider.java | 33 +-
...parkConsistentBucketClusteringPlanStrategy.java | 4 +-
.../MultipleSparkJobExecutionStrategy.java | 187 +++++-
...onsistentBucketClusteringExecutionStrategy.java | 18 +-
.../SparkSingleFileSortExecutionStrategy.java | 44 +-
.../SparkSortAndSizeExecutionStrategy.java | 38 +-
...arkConsistentBucketDuplicateUpdateStrategy.java | 4 +-
.../java/org/apache/hudi/data/HoodieJavaRDD.java | 5 +
.../bulkinsert/RDDConsistentBucketPartitioner.java | 4 +-
.../bulkinsert/RDDSpatialCurveSortPartitioner.java | 47 +-
.../bulkinsert/RowSpatialCurveSortPartitioner.java | 42 ++
.../SpatialCurveSortPartitionerBase.java | 83 +++
.../hudi/io/storage/row/HoodieRowCreateHandle.java | 28 +-
.../hudi/table/HoodieSparkCopyOnWriteTable.java | 11 +-
.../commit/BaseSparkCommitActionExecutor.java | 14 +-
.../BulkInsertDataInternalWriterHelper.java | 17 +-
.../table/action/commit/UpsertPartitioner.java | 16 +-
.../hudi/HoodieDatasetBulkInsertHelper.scala | 61 +-
.../scala/org/apache/hudi/HoodieSparkUtils.scala | 1 +
.../scala/org/apache/hudi/HoodieUnsafeRDD.scala | 0
.../org/apache/hudi/SparkAdapterSupport.scala | 8 +-
.../spark/sql/HoodieCatalystExpressionUtils.scala | 27 +-
.../org/apache/spark/sql/HoodieUnsafeUtils.scala | 1 -
.../org/apache/spark/sql/hudi/SparkAdapter.scala | 14 +-
.../apache/hudi/client/TestHoodieReadClient.java | 15 +-
.../functional/TestHoodieBackedTableMetadata.java | 2 +-
.../TestHoodieClientOnCopyOnWriteStorage.java | 74 ++-
.../org/apache/hudi/data/TestHoodieJavaRDD.java | 27 +-
.../index/bucket/TestHoodieSimpleBucketIndex.java | 25 +-
.../TestHoodieSparkMergeOnReadTableCompaction.java | 14 +-
.../TestHoodieSparkMergeOnReadTableRollback.java | 3 +
.../hudi/table/upgrade/TestUpgradeDowngrade.java | 6 +-
.../hudi/testutils/HoodieMergeOnReadTestUtils.java | 4 +-
.../testutils/HoodieSparkWriteableTestTable.java | 11 +
hudi-common/pom.xml | 5 +
.../org/apache/hudi/BaseHoodieTableFileIndex.java | 46 +-
.../java/org/apache/hudi/avro/AvroSchemaUtils.java | 8 +-
.../hudi/common/config/HoodieCommonConfig.java | 5 +
.../apache/hudi/common/config/HoodieConfig.java | 13 +-
.../hudi/common/config/SerializableSchema.java | 6 +-
.../org/apache/hudi/common/data/HoodieData.java | 9 +-
.../apache/hudi/common/data/HoodieListData.java | 5 +
.../java/org/apache/hudi/common/fs/FSUtils.java | 6 +-
.../org/apache/hudi/common/model/FileSlice.java | 10 +
.../hudi/common/model/HoodieCommitMetadata.java | 43 ++
.../apache/hudi/common/model/HoodieFileGroup.java | 4 +
.../org/apache/hudi/common/model/HoodieRecord.java | 2 +
.../hudi/common/model/HoodieRecordPayload.java | 14 +
.../apache/hudi/common/model/HoodieWriteStat.java | 28 +
.../OverwriteNonDefaultsWithLatestAvroPayload.java | 43 +-
.../common/model/PartialUpdateAvroPayload.java | 227 +++++++
.../hudi/common/model/WriteOperationType.java | 14 +
.../hudi/common/table/HoodieTableConfig.java | 31 +-
.../hudi/common/table/HoodieTableMetaClient.java | 24 +
.../hudi/common/table/cdc/HoodieCDCExtractor.java | 349 ++++++++++
.../hudi/common/table/cdc/HoodieCDCFileSplit.java | 111 ++++
.../hudi/common/table/cdc/HoodieCDCInferCase.java | 78 +++
.../hudi/common/table/cdc/HoodieCDCOperation.java | 40 +-
.../cdc/HoodieCDCSupplementalLoggingMode.java | 62 ++
.../hudi/common/table/cdc/HoodieCDCUtils.java | 149 +++++
.../table/log/AbstractHoodieLogRecordReader.java | 9 +
.../table/log/HoodieCDCLogRecordIterator.java | 88 +++
.../hudi/common/table/log/HoodieLogFileReader.java | 19 +-
.../table/log/HoodieMergedLogRecordScanner.java | 8 +-
.../table/log/HoodieUnMergedLogRecordScanner.java | 6 +-
.../table/log/block/HoodieAvroDataBlock.java | 3 +
.../common/table/log/block/HoodieCDCDataBlock.java | 62 ++
.../common/table/log/block/HoodieLogBlock.java | 3 +-
.../hudi/common/table/timeline/TimelineUtils.java | 26 +-
.../table/view/AbstractTableFileSystemView.java | 112 +++-
.../table/view/RocksDbBasedFileSystemView.java | 2 +-
.../apache/hudi/common/util/CollectionUtils.java | 10 +-
.../hudi/common/util/ObjectSizeCalculator.java | 321 +--------
.../convert/AvroInternalSchemaConverter.java | 31 +-
.../schema/utils/AvroSchemaEvolutionUtils.java | 4 +-
.../hudi/metadata/HoodieTableMetadataUtil.java | 22 +-
hudi-common/src/main/resources/hbase-site.xml | 7 -
.../hudi/common/data/TestHoodieListData.java | 8 +
.../common/functional/TestHoodieLogFormat.java | 201 ++++--
.../common/model/TestPartialUpdateAvroPayload.java | 210 ++++++
.../hudi/common/table/TestTimelineUtils.java | 4 +-
.../common/testutils/HoodieTestDataGenerator.java | 1 +
.../hudi/common/testutils/HoodieTestTable.java | 6 +
.../util/TestDFSPropertiesConfiguration.java | 12 +-
.../hudi/common/util/TestObjectSizeCalculator.java | 102 +++
.../common/HoodieExampleDataGenerator.java | 59 +-
hudi-examples/hudi-examples-flink/pom.xml | 6 +
hudi-examples/hudi-examples-spark/pom.xml | 7 +
.../examples/quickstart/HoodieSparkQuickstart.java | 67 +-
.../apache/hudi/configuration/FlinkOptions.java | 22 +
.../apache/hudi/configuration/OptionsResolver.java | 9 +
.../org/apache/hudi/sink/StreamWriteFunction.java | 4 +-
.../hudi/sink/StreamWriteOperatorCoordinator.java | 2 +-
.../hudi/sink/clustering/ClusteringOperator.java | 39 +-
.../org/apache/hudi/sink/meta/CkpMetadata.java | 6 +-
.../java/org/apache/hudi/sink/utils/Pipelines.java | 10 +-
.../apache/hudi/source/IncrementalInputSplits.java | 164 +++--
.../hudi/source/StreamReadMonitoringFunction.java | 8 +-
.../org/apache/hudi/table/HoodieTableSource.java | 34 +-
.../org/apache/hudi/table/format/FormatUtils.java | 20 +
.../hudi/table/format/cdc/CdcInputFormat.java | 723 +++++++++++++++++++++
.../hudi/table/format/cdc/CdcInputSplit.java | 61 ++
.../table/format/mor/MergeOnReadInputFormat.java | 166 +++--
.../table/format/mor/MergeOnReadInputSplit.java | 2 +-
.../org/apache/hudi/util/RowDataProjection.java | 1 +
.../java/org/apache/hudi/util/StreamerUtil.java | 4 +
.../apache/hudi/sink/ITTestDataStreamWrite.java | 28 +-
.../org/apache/hudi/sink/meta/TestCkpMetadata.java | 3 +-
.../apache/hudi/table/ITTestHoodieDataSource.java | 36 +
.../apache/hudi/table/format/TestInputFormat.java | 161 ++++-
.../test/java/org/apache/hudi/utils/TestData.java | 34 +-
hudi-gcp/pom.xml | 12 +
.../hudi/hadoop/HoodieROTablePathFilter.java | 15 +-
hudi-integ-test/README.md | 52 +-
.../dag/nodes/ValidateAsyncOperations.java | 2 +-
.../org/apache/hudi/integ/ITTestHoodieDemo.java | 4 +-
.../org/apache/hudi/integ/ITTestHoodieSanity.java | 4 +-
.../integ/command/ITTestHoodieSyncCommand.java | 4 +-
hudi-kafka-connect/README.md | 26 +-
hudi-spark-datasource/hudi-spark-common/pom.xml | 1 +
.../org/apache/hudi/BaseFileOnlyRelation.scala | 1 -
.../scala/org/apache/hudi/DataSourceOptions.scala | 29 +-
.../main/scala/org/apache/hudi/DefaultSource.scala | 136 ++--
.../main/scala/org/apache/hudi/EmptyRelation.scala | 18 +-
.../org/apache/hudi/HoodieBootstrapRelation.scala | 2 +-
.../org/apache/hudi/HoodieDataSourceHelper.scala | 17 +
.../org/apache/hudi/HoodieMergeOnReadRDD.scala | 391 +----------
.../org/apache/hudi/HoodieSparkSqlWriter.scala | 9 +-
.../scala/org/apache/hudi/HoodieWriterUtils.scala | 2 +-
.../scala/org/apache/hudi/LogFileIterator.scala | 358 ++++++++++
.../scala/org/apache/hudi/SafeAvroProjection.scala | 73 +++
.../scala/org/apache/hudi/cdc/CDCRelation.scala | 210 ++++++
.../scala/org/apache/hudi/cdc/HoodieCDCRDD.scala | 626 ++++++++++++++++++
.../sql/catalyst/catalog/HoodieCatalogTable.scala | 8 +-
.../apache/spark/sql/hudi/DataSkippingUtils.scala | 12 +-
.../spark/sql/hudi/HoodieSqlCommonUtils.scala | 12 +-
.../spark/sql/hudi/ProvidesHoodieConfig.scala | 39 +-
.../spark/sql/hudi/command/SqlKeyGenerator.scala | 131 ++--
.../sql/hudi/streaming/HoodieStreamSource.scala | 75 ++-
.../spark/sql/hudi/analysis/HoodieAnalysis.scala | 4 +-
.../hudi/command/MergeIntoHoodieTableCommand.scala | 91 ++-
.../hudi/command/payload/ExpressionCodeGen.scala | 5 +-
.../hudi/command/procedures/BaseProcedure.scala | 34 +-
.../command/procedures/HiveSyncProcedure.scala | 107 +++
.../hudi/command/procedures/HoodieProcedures.scala | 1 +
.../TestHoodieSparkMergeOnReadTableClustering.java | 55 +-
.../org/apache/hudi/TestDataSkippingUtils.scala | 29 +-
.../org/apache/hudi/TestHoodieSparkSqlWriter.scala | 2 +-
.../org/apache/hudi/TestHoodieSparkUtils.scala | 6 +-
.../apache/hudi/functional/TestCOWDataSource.scala | 17 +-
.../hudi/functional/TestLayoutOptimization.scala | 37 +-
.../apache/hudi/functional/TestMORDataSource.scala | 118 +++-
.../hudi/functional/cdc/HoodieCDCTestBase.scala | 202 ++++++
.../functional/cdc/TestCDCDataFrameSuite.scala | 545 ++++++++++++++++
.../functional/cdc/TestCDCStreamingSuite.scala | 214 ++++++
.../apache/spark/sql/hudi/TestCDCForSparkSQL.scala | 255 ++++++++
.../org/apache/spark/sql/hudi/TestDropTable.scala | 23 +-
.../apache/spark/sql/hudi/TestMergeIntoTable.scala | 82 ++-
.../spark/sql/hudi/TestMergeIntoTable2.scala | 78 +++
.../hudi/TestQueryMergeOnReadOptimizedTable.scala | 82 +++
.../sql/hudi/procedure/TestRepairsProcedure.scala | 35 +-
.../TestUpgradeOrDowngradeProcedure.scala | 5 +-
.../HoodieBulkInsertDataInternalWriter.java | 1 +
.../sql/HoodieSpark2CatalystExpressionUtils.scala | 12 +
.../apache/spark/sql/adapter/Spark2Adapter.scala | 15 +-
.../catalyst/encoders/DummyExpressionHolder.scala | 16 +-
.../HoodieBulkInsertDataInternalWriter.java | 2 +-
.../spark/sql/HoodieSpark3CatalystPlanUtils.scala | 12 +-
.../spark/sql/adapter/BaseSpark3Adapter.scala | 18 +-
.../sql/HoodieSpark31CatalystExpressionUtils.scala | 13 +-
.../spark/sql/HoodieSpark31CatalystPlanUtils.scala | 5 +-
.../datasources/Spark31NestedSchemaPruning.scala | 13 +-
.../sql/HoodieSpark32CatalystExpressionUtils.scala | 15 +-
.../spark/sql/HoodieSpark32CatalystPlanUtils.scala | 21 +-
.../datasources/Spark32NestedSchemaPruning.scala | 13 +-
.../sql/HoodieSpark33CatalystExpressionUtils.scala | 15 +-
.../spark/sql/HoodieSpark33CatalystPlanUtils.scala | 6 +-
.../datasources/Spark33NestedSchemaPruning.scala | 7 +-
.../java/org/apache/hudi/sync/adb/AdbSyncTool.java | 2 +-
.../java/org/apache/hudi/hive/HiveSyncTool.java | 15 +-
.../org/apache/hudi/hive/HoodieHiveSyncClient.java | 38 +-
.../hive/SinglePartPartitionValueExtractor.java | 40 ++
.../org/apache/hudi/hive/TestHiveSyncTool.java | 41 +-
.../hudi/hive/TestPartitionValueExtractor.java | 12 +
.../apache/hudi/hive/testutils/HiveTestUtil.java | 2 +-
.../hudi/sync/common/HoodieMetaSyncOperations.java | 7 +
.../apache/hudi/sync/common/HoodieSyncClient.java | 41 +-
.../apache/hudi/sync/common/HoodieSyncConfig.java | 11 +-
.../hudi/sync/common/TestHoodieSyncConfig.java | 9 +-
hudi-utilities/pom.xml | 21 +-
.../hudi/utilities/HoodieSnapshotExporter.java | 64 +-
.../org/apache/hudi/utilities/UtilHelpers.java | 2 +-
.../deltastreamer/HoodieDeltaStreamer.java | 5 +
.../schema/ProtoClassBasedSchemaProvider.java | 36 +-
.../ChainedSchemaPostProcessor.java | 3 +-
.../DeleteSupportSchemaPostProcessor.java | 3 +-
.../DropColumnSchemaPostProcessor.java | 3 +-
.../add/AddPrimitiveColumnSchemaPostProcessor.java | 97 +++
.../add/BaseSchemaPostProcessorConfig.java | 53 ++
.../sources/GcsEventsHoodieIncrSource.java | 199 ++++++
.../hudi/utilities/sources/GcsEventsSource.java | 213 ++++++
.../hudi/utilities/sources/HoodieIncrSource.java | 8 +-
.../hudi/utilities/sources/ProtoKafkaSource.java | 4 +-
.../helpers/CloudObjectsSelectorCommon.java | 118 ++++
.../sources/helpers/CloudStoreIngestionConfig.java | 96 +++
.../helpers/IncrSourceCloudStorageHelper.java | 81 +++
.../sources/helpers/IncrSourceHelper.java | 27 +
.../sources/helpers/ProtoConversionUtil.java | 111 +++-
.../sources/helpers/gcs/FileDataFetcher.java | 54 ++
.../sources/helpers/gcs/FilePathsFetcher.java | 114 ++++
.../sources/helpers/gcs/GcsIngestionConfig.java | 34 +-
.../sources/helpers/gcs/MessageBatch.java | 28 +-
.../sources/helpers/gcs/MessageValidity.java | 55 ++
.../sources/helpers/gcs/MetadataMessage.java | 101 +++
.../sources/helpers/gcs/PubsubMessagesFetcher.java | 115 ++++
.../utilities/sources/helpers/gcs/QueryInfo.java | 94 +++
.../hudi/utilities/TestSchemaPostProcessor.java | 56 +-
.../functional/TestHoodieDeltaStreamer.java | 78 ++-
.../schema/TestProtoClassBasedSchemaProvider.java | 50 +-
.../sources/TestGcsEventsHoodieIncrSource.java | 289 ++++++++
.../utilities/sources/TestGcsEventsSource.java | 261 ++++++++
.../utilities/sources/TestProtoKafkaSource.java | 4 +-
.../sources/helpers/TestProtoConversionUtil.java | 585 ++++++++++-------
.../delta-streamer-config/gcs-metadata.avsc | 108 +++
.../parent_schema_recursive_default_limit.avsc | 160 +++++
.../proto/parent_schema_recursive_depth_2.avsc | 115 ++++
.../schema-provider/proto/recursive.proto | 27 +-
.../proto/sample_schema_flattened.avsc | 310 +++++++++
.../proto/sample_schema_flattened.txt | 1 -
.../proto/sample_schema_nested.avsc | 409 ++++++++++++
.../schema-provider/proto/sample_schema_nested.txt | 1 -
packaging/hudi-aws-bundle/pom.xml | 5 +
packaging/hudi-datahub-sync-bundle/pom.xml | 5 +
packaging/hudi-flink-bundle/pom.xml | 8 +-
packaging/hudi-gcp-bundle/pom.xml | 5 +
packaging/hudi-hadoop-mr-bundle/pom.xml | 5 +
packaging/hudi-hive-sync-bundle/pom.xml | 14 +
packaging/hudi-integ-test-bundle/pom.xml | 5 +
packaging/hudi-kafka-connect-bundle/pom.xml | 5 +
packaging/hudi-presto-bundle/pom.xml | 5 +
packaging/hudi-spark-bundle/pom.xml | 8 +
packaging/hudi-timeline-server-bundle/pom.xml | 5 +
packaging/hudi-trino-bundle/pom.xml | 5 +
packaging/hudi-utilities-bundle/pom.xml | 5 +
packaging/hudi-utilities-slim-bundle/pom.xml | 5 +
pom.xml | 54 +-
rfc/rfc-51/{points.jpg => code-paths.jpg} | Bin
rfc/rfc-51/{arch.jpg => logic-flows.jpg} | Bin
rfc/rfc-51/rfc-51.md | 217 ++++---
...io-definition.jpg => scenario-illustration.jpg} | Bin
scripts/release/validate_staged_release.sh | 20 +-
370 files changed, 15472 insertions(+), 3652 deletions(-)
delete mode 100644
hudi-cli/src/main/java/org/apache/hudi/cli/HoodieSplashScreen.java
create mode 100644
hudi-cli/src/main/java/org/apache/hudi/cli/commands/KerberosAuthenticationCommand.java
delete mode 100644
hudi-cli/src/main/resources/META-INF/spring/spring-shell-plugin.xml
create mode 100644 hudi-cli/src/main/resources/application.yml
create mode 100644 hudi-cli/src/main/resources/banner.txt
create mode 100644 hudi-cli/src/main/resources/log4j2.properties
create mode 100644
hudi-cli/src/test/java/org/apache/hudi/cli/testutils/MockCommandLineInput.java
copy
hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/HoodieSpark31CatalystPlanUtils.scala
=>
hudi-cli/src/test/java/org/apache/hudi/cli/testutils/ShellEvaluationResultUtil.java
(58%)
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/metrics/HoodieLockMetrics.java
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCDCLogger.java
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieMergeHandleFactory.java
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieMergeHandleWithChangeLog.java
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieSortedMergeHandleWithChangeLog.java
create mode 100644
hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/FlinkMergeAndReplaceHandleWithChangeLog.java
create mode 100644
hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/FlinkMergeHandleWithChangeLog.java
create mode 100644
hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/FlinkWriteHandleFactory.java
create mode 100644
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/RowSpatialCurveSortPartitioner.java
create mode 100644
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/SpatialCurveSortPartitionerBase.java
rename
{hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/internal
=>
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit}/BulkInsertDataInternalWriterHelper.java
(90%)
rename {hudi-spark-datasource/hudi-spark-common =>
hudi-client/hudi-spark-client}/src/main/scala/org/apache/hudi/HoodieDatasetBulkInsertHelper.scala
(74%)
rename {hudi-spark-datasource/hudi-spark-common =>
hudi-client/hudi-spark-client}/src/main/scala/org/apache/hudi/HoodieUnsafeRDD.scala
(100%)
rename {hudi-spark-datasource/hudi-spark-common =>
hudi-client/hudi-spark-client}/src/main/scala/org/apache/spark/sql/HoodieUnsafeUtils.scala
(99%)
copy
hudi-cli/src/test/java/org/apache/hudi/cli/functional/CLIFunctionalTestSuite.java
=>
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/data/TestHoodieJavaRDD.java
(55%)
create mode 100644
hudi-common/src/main/java/org/apache/hudi/common/model/PartialUpdateAvroPayload.java
create mode 100644
hudi-common/src/main/java/org/apache/hudi/common/table/cdc/HoodieCDCExtractor.java
create mode 100644
hudi-common/src/main/java/org/apache/hudi/common/table/cdc/HoodieCDCFileSplit.java
create mode 100644
hudi-common/src/main/java/org/apache/hudi/common/table/cdc/HoodieCDCInferCase.java
copy
hudi-cli/src/main/java/org/apache/hudi/cli/HoodieHistoryFileNameProvider.java
=>
hudi-common/src/main/java/org/apache/hudi/common/table/cdc/HoodieCDCOperation.java
(55%)
create mode 100644
hudi-common/src/main/java/org/apache/hudi/common/table/cdc/HoodieCDCSupplementalLoggingMode.java
create mode 100644
hudi-common/src/main/java/org/apache/hudi/common/table/cdc/HoodieCDCUtils.java
create mode 100644
hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieCDCLogRecordIterator.java
create mode 100644
hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieCDCDataBlock.java
create mode 100644
hudi-common/src/test/java/org/apache/hudi/common/model/TestPartialUpdateAvroPayload.java
create mode 100644
hudi-common/src/test/java/org/apache/hudi/common/util/TestObjectSizeCalculator.java
create mode 100644
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/cdc/CdcInputFormat.java
create mode 100644
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/cdc/CdcInputSplit.java
create mode 100644
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/LogFileIterator.scala
create mode 100644
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/SafeAvroProjection.scala
create mode 100644
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/cdc/CDCRelation.scala
create mode 100644
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/cdc/HoodieCDCRDD.scala
create mode 100644
hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/HiveSyncProcedure.scala
rename {hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table =>
hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi}/functional/TestHoodieSparkMergeOnReadTableClustering.java
(87%)
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/cdc/HoodieCDCTestBase.scala
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/cdc/TestCDCDataFrameSuite.scala
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/cdc/TestCDCStreamingSuite.scala
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestCDCForSparkSQL.scala
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestQueryMergeOnReadOptimizedTable.scala
rename
hudi-cli/src/test/java/org/apache/hudi/cli/functional/CLIFunctionalTestSuite.java
=>
hudi-spark-datasource/hudi-spark2/src/main/scala/org/apache/spark/sql/catalyst/encoders/DummyExpressionHolder.scala
(66%)
create mode 100644
hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/SinglePartPartitionValueExtractor.java
rename hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/{ =>
postprocessor}/ChainedSchemaPostProcessor.java (93%)
rename hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/{ =>
postprocessor}/DeleteSupportSchemaPostProcessor.java (95%)
rename hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/{ =>
postprocessor}/DropColumnSchemaPostProcessor.java (96%)
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/postprocessor/add/AddPrimitiveColumnSchemaPostProcessor.java
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/postprocessor/add/BaseSchemaPostProcessorConfig.java
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/GcsEventsHoodieIncrSource.java
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/GcsEventsSource.java
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/CloudObjectsSelectorCommon.java
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/CloudStoreIngestionConfig.java
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/IncrSourceCloudStorageHelper.java
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/gcs/FileDataFetcher.java
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/gcs/FilePathsFetcher.java
copy
hudi-cli/src/main/java/org/apache/hudi/cli/HoodieHistoryFileNameProvider.java
=>
hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/gcs/GcsIngestionConfig.java
(52%)
rename
hudi-cli/src/main/java/org/apache/hudi/cli/HoodieHistoryFileNameProvider.java
=>
hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/gcs/MessageBatch.java
(59%)
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/gcs/MessageValidity.java
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/gcs/MetadataMessage.java
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/gcs/PubsubMessagesFetcher.java
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/gcs/QueryInfo.java
create mode 100644
hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestGcsEventsHoodieIncrSource.java
create mode 100644
hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestGcsEventsSource.java
create mode 100644
hudi-utilities/src/test/resources/delta-streamer-config/gcs-metadata.avsc
create mode 100644
hudi-utilities/src/test/resources/schema-provider/proto/parent_schema_recursive_default_limit.avsc
create mode 100644
hudi-utilities/src/test/resources/schema-provider/proto/parent_schema_recursive_depth_2.avsc
copy hudi-cli/src/main/java/org/apache/hudi/cli/Main.java =>
hudi-utilities/src/test/resources/schema-provider/proto/recursive.proto (67%)
create mode 100644
hudi-utilities/src/test/resources/schema-provider/proto/sample_schema_flattened.avsc
delete mode 100644
hudi-utilities/src/test/resources/schema-provider/proto/sample_schema_flattened.txt
create mode 100644
hudi-utilities/src/test/resources/schema-provider/proto/sample_schema_nested.avsc
delete mode 100644
hudi-utilities/src/test/resources/schema-provider/proto/sample_schema_nested.txt
rename rfc/rfc-51/{points.jpg => code-paths.jpg} (100%)
rename rfc/rfc-51/{arch.jpg => logic-flows.jpg} (100%)
rename rfc/rfc-51/{scenario-definition.jpg => scenario-illustration.jpg} (100%)
