This is an automated email from the ASF dual-hosted git repository.
yuzhaojing pushed a change to branch release-feature-rfc46
in repository https://gitbox.apache.org/repos/asf/hudi.git
omit af53430947 test
omit 276787aa2f [HUDI-3350][HUDI-3351] Support HoodieMerge API and Spark
engine-specific HoodieRecord (#5627)
omit 9c361b7954 [HUDI-4344] fix usage of HoodieDataBlock#getRecordIterator
(#6005)
omit 22976d05a4 [HUDI-4292][RFC-46] Update doc to align with the Record
Merge API changes (#5927)
omit 28306ff797 [MINOR] Fix type casting in TestHoodieHFileReaderWriter
omit a0969fc451 [HUDI-3378][HUDI-3379][HUDI-3381] Migrate usage of
HoodieRecordPayload and raw Avro payload to HoodieRecord (#5522)
add ed823f1c6f [HUDI-4320] Make sure
`HoodieStorageConfig.PARQUET_WRITE_LEGACY_FORMAT_ENABLED` could be specified by
the writer (#5970)
add fd7d25ab63 [HUDI-1176] Upgrade hudi to log4j2 (#5366)
add e3eb14ad2d [HUDI-4334] close SparkRDDWriteClient after usage in
Create/Delete/RollbackSavepointsProcedure (#5994)
add 637660b7aa [HUDI-1575] Claim RFC-56: Early Conflict Detection For
Multi-writer (#6002)
add e71f04768e [MINOR] Make CLI 'commit rollback' using
rollbackUsingMarkers false as default (#5174)
add 03a94d9ff5 [HUDI-4331] Allow loading external config file from class
loader (#5987)
add 3948b8935a [HUDI-4336] Fix records overwritten bug with binary primary
key (#5996)
add 6a01f7029c [MINOR] Following #2070, Fix BindException when running
tests on shared machines. (#5951)
add cdaaa3c4c7 [HUDI-4346] Fix params not update
BULKINSERT_ARE_PARTITIONER_RECORDS_SORTED (#5999)
add 8547899a39 [HUDI-4285] add ByteBuffer#rewind after ByteBuffer#get in
AvroDeseria… (#5907)
add 397fd30142 [HUDI-3984] Remove mandatory check of partiton path for cli
command (#5458)
add 62a0c962ac [HUDI-3634] Could read empty or partial
HoodieCommitMetaData in downstream if using HDFS (#5048)
add bdf73b2650 [HUDI-3953]Flink Hudi module should support low-level
source and sink api (#5445)
add 47792a3186 [HUDI-4353] Column stats data skipping for flink (#6026)
add c00ea84985 [HUDI-3505] Add call procedure for
UpgradeOrDowngradeCommand (#6012)
add c0e1587966 [HUDI-3730] Improve meta sync class design and hierarchies
(#5854)
add e0954040a9 [HUDI-3511] Add call procedure for MetadataCommand (#6018)
add c091e4cc30 [HUDI-3730] Add ConfigTool#toMap UT (#6035)
add 6187622178 [MINOR] Improve variable names (#6039)
add 45fdcf68a1 [HUDI-3116]Add a new HoodieDropPartitionsTool to let users
drop table partitions through a standalone job. (#4459)
add fbda4ad5bd [HUDI-4360] Fix HoodieDropPartitionsTool based on
refactored meta sync (#6043)
add 23c9c5c296 [HUDI-3836] Improve the way of fetching metadata partitions
from table (#5286)
add 8570c3aab4 [HUDI-4359] Support show_fs_path_detail command on Call
Produce Command (#6042)
add 3670e82af5 [HUDI-4356] Fix the error when sync hive in CTAS (#6029)
add b18c32379f [HUDI-4219] Merge Into when update expression "col=s.col+2"
on precombine cause exception (#5828)
add 7eeaff9ee0 [HUDI-4357] Support flink 1.15.x (#6050)
add e74ad324c3 [HUDI-4152] Flink offline compaction support compacting
multi compaction plan at once (#5677)
add 5673819736 [HUDI-4309] fix spark32 repartition error (#6033)
add c744848c59 [HUDI-4366] Synchronous cleaning for flink bounded source
(#6051)
add a998586396 [minor] following 4152, refactor the clazz about plan
selection strategy (#6060)
add f20acb8dc3 [HUDI-4367] Support copyToTable on call (#6054)
add fc8d96246a [HUDI-4335] Bug fixes in AWSGlueCatalogSyncClient post
schema evolution. (#5995)
add b686c07407 [HUDI-4276] Reconcile schema-inject null values for missing
fields and add new fields (#6017)
add 6566fc6625 [HUDI-3500] Add call procedure for RepairsCommand (#6053)
add 126b88b48d [HUDI-2150] Rename/Restructure configs for better
modularity (#6061)
add 10aec07fd2 [MINOR] Bump xalan from 2.7.1 to 2.7.2 (#6062)
add 046044c83d [HUDI-4324] Remove use_jdbc config from hudi sync (#6072)
add 63f95ab801 [HUDI-3730][RFC-55] Improve hudi-sync classes design and
simplify configs (#5695)
add 51244eba82 [HUDI-4323] Make database table names optional in sync tool
(#6073)
add a270eeeef9 [MINOR] Update RFCs status (#6078)
add 994c561488 [HUDI-4298] When reading the mor table with
QUERY_TYPE_SNAPSHOT,Unabl… (#5937)
add ee956b8951 [HUDI-4379] Bump Flink versions to 1.14.5 and 1.15.1 (#6080)
add e70a427956 [HUDI-4391] Incremental read from archived commits for
flink (#6096)
add aaccc63ad5 [RFC-51] [HUDI-3478] Hudi to support Change-Data-Capture
(#5436)
add 05606708fa [HUDI-4393] Add marker file for target file when flink
merge handle rolls over (#6103)
add 4898ea52f7 [HUDI-4399][RFC-57] Claim RFC 57 for DeltaStreamer proto
support (#6112)
add b781b31045 [HUDI-4397] Flink Inline Cluster and Compact plan
distribute strategy changed from rebalance to hash to avoid potential multiple
threads accessing the same file (#6106)
add 726e8e3590 [MINOR] Disable TestHiveSyncGlobalCommitTool (#6119)
add 0faa562b6f [HUDI-4403] Fix the end input metadata for bounded source
(#6116)
add 6aec9d754f [HUDI-4408] Reuse old rollover file as base file for flink
merge handle (#6120)
add 80368a049d [HUDI-3503] Add call procedure for CleanCommand (#6065)
add 4bda6afe0b [HUDI-4249] Fixing in-memory `HoodieData` implementation to
operate lazily (#5855)
add ded197800a [HUDI-4170] Make user can use hoodie.datasource.read.paths
to read necessary files (#5722)
add 3964c476e0 Fix file group count issue with metadata partitions (#5892)
add 9282611bae [HUDI-4098] Support HMS for flink HudiCatalog (#6082)
add 1959b843b7 [HUDI-4409] Improve LockManager wait logic when catch
exception (#6122)
add 382d19e85b [HUDI-4065] Add FileBasedLockProvider (#6071)
add 6c3578069e [HUDI-4416] Default database path for hoodie hive catalog
(#6136)
add e3675fe9b0 [HUDI-4372] Enable matadata table by default for flink
(#6066)
add 2b828ccb98 [HUDI-4401] Skip HBase version check (#6114)
add 348519f3cd [HUDI-4427] Add a computed column IT test (#6150)
add 2394c62973 [HUDI-4146][RFC-55] Update config changes proposal (#6162)
add de37774e12 [HUDI-3896] Porting Nested Schema Pruning optimization for
Hudi's custom Relations (#5428)
add c7fe3fd01d [HUDI-3764] Allow loading external configs while querying
Hudi tables with Spark (#4915)
add a33bdd32e3 [HUDI-3993] Replacing UDF in Bulk Insert w/ RDD
transformation (#5470)
add 2bf7920bd9 [MINOR] Add logger for HoodieCopyOnWriteTableInputFormat
(#6161)
add 473be87aa5 Disable EmrFS file metadata caching and EMR Spark's data
prefetcher feature
add f52b93fd10 Merge pull request #6154 from
rahil-c/rahil-c/disable-emrSpark-properties
add 50cdb867c7 [HUDI-4400] Fix missing bloom filters in metadata table in
non-partitioned table (#6113)
add 2e0dd29714 [HUDI-4204] Fixing NPE with row writer path and with OCC
(#5850)
add 36e656aa77 [HUDI-4247] Upgrading protocol buffers version for presto
bundle (#5852)
add 8e0b47e360 [MINOR] Fix result missing information issue in
commits_compare Procedure (#6165)
add 51b5783161 [HUDI-4404] Fix insert into dynamic partition write
misalignment (#6124)
add 41653fc708 [MINOR] Fallback to default for hive-style partitioning,
url-encoding configs (#6175)
add d5c904e10e [MINOR] Fix CI issue with TestHiveSyncTool (#6110)
add eea4a692c0 [HUDI-4039] Make sure all builtin `KeyGenerator`s properly
implement Spark specific APIs (#5523)
add 716dd3512b [MINOR] Disable Flink compactor IT test (#6189)
add 6b84384022 Revert "[MINOR] Fix CI issue with TestHiveSyncTool (#6110)"
(#6192)
add 39f2a06c85 [HUDI-3979] Optimize out mandatory columns when no merging
is performed (#5430)
add a36762a862 [HUDI-4303] Use Hive sentinel value as partition default to
avoid type caste issues (#5954)
add d5c7c79d87 Revert "[HUDI-4324] Remove use_jdbc config from hudi sync
(#6072)" (#6160)
add af10a97e7a [HUDI-4435] Fix Avro field not found issue introduced by
Avro 1.10 (#6155)
add 340c3dbbe1 [HUDI-4437] Fix test conflicts by clearing file system
cache (#6123)
add a5348cc685 [HUDI-4436] Invalidate cached table in Spark after write
(#6159)
add 859157ec01 [MINOR] Fix Call Procedure code style (#6186)
add f0e843249c [MINOR] Bump CI timeout to 150m (#6198)
add f1f0109ab8 [HUDI-4440] Treat boostrapped table as non-partitioned in
HudiFileIndex if partition column is missing from schema (#6163)
add da28e38fe3 [HUDI-4071] Make NONE sort mode as default for bulk insert
(#6195)
add 2d745057ea [HUDI-4420] Fixing table schema delineation on
partition/data schema for Spark relations (#5708)
add a0ffd05b77 [HUDI-4448] Remove the latest commit refresh for timeline
server (#6179)
add ba11082282 [HUDI-4450] Revert the checkpoint abort notification (#6181)
add 1a5a9f7f03 [HUDI-4439] Fix Amazon CloudWatch reporter for metadata
enabled tables (#6164)
add a54c963543 [HUDI-4348] fix merge into sql data quality in concurrent
scene (#6020)
add 1a910fd473 [HUDI-3510] Add sync validate procedure (#6200)
add 2a08a65f71 [MINOR] Fix typos in Spark client related classes (#6204)
add 76a28daeb0 [HUDI-4456] Close FileSystem in
SparkClientFunctionalTestHarness (#6201)
add f6e7227ed5 [MINOR] Only log stdout output for non-zero exit from
commands in IT (#6199)
add b513232449 [HUDI-4458] Add a converter cache for flink
ColumnStatsIndices (#6205)
add 1fda9ee9bb [HUDI-4071] Match ROLLBACK_USING_MARKERS_ENABLE in sql as
datasource (#6206)
add 71c2c3102b [HUDI-4455] Improve test classes for TestHiveSyncTool
(#6202)
add eee6a02f77 [HUDI-4456] Clean up test resources (#6203)
add 6e7ac45735 [HUDI-3884] Support archival beyond savepoint commits
(#5837)
add e7c8df7e8b [HUDI-4250][HUDI-4202] Optimize performance of Column Stats
Index reading in Data Skipping (#5746)
add 74d7b4d751 [HUDI-4471] Relocate AWSDmsAvroPayload class to hudi-common
add 1ea1e659c2 [HUDI-4474] Infer metasync configs (#6217)
add e5faf2cc84 [HUDI-4210] Create custom hbase index to solve data skew
issue on hbase regions (#5797)
add 717f159bfd [HUDI-3730] Keep metasync configs backward compatible
(#6221)
add 924c30c7ea [HUDI-4469] Flip reuse flag to true in
HoodieBackedTableMetadata to improve file listing (#6214)
add cdaec5a8da [HUDI-4186] Support Hudi with Spark 3.3.0 (#5943)
add 51599af281 [HUDI-4126] Disable file splits for Bootstrap real time
queries (via InputFormat) (#6219)
add 0a5ce000bf [HUDI-4490] Make AWSDmsAvroPayload class backwards
compatible (#6229)
add 07eedd3ef6 [HUDI-4484] Add default lock config options for flink
metadata table (#6222)
add ea1fbc71ec [HUDI-4494] keep the fields' order when data is written out
of order (#6233)
add 70b5cf6dab [MINOR] Minor changes around Spark 3.3 support (#6231)
add cfd0c1ee34 [HUDI-4081][HUDI-4472] Addressing Spark SQL vs Spark DS
performance gap (#6213)
add c39e88dcf0 [HUDI-4495] Fix handling of S3 paths incompatible with java
URI standards (#6237)
add ce4330d62b [HUDI-4499] Tweak default retry times for flink metadata
table lock (#6238)
add 765dd2eae6 [HUDI-4221] Optimzing getAllPartitionPaths (#6234)
add 9335e4e487 Moving to 0.13.0-SNAPSHOT on master branch.
add a1cf401350 [HUDI-4504] Disable metadata table by default for flink
(#6241)
add e04b3188e2 [HUDI-4505] Returns instead of throws if lock file exists
for FileSystemBasedLockProvider (#6242)
add 0f703a7e15 [HUDI-4507] Improve file name extraction logic in metadata
utils (#6250)
add c9725899c3 [MINOR] Fix convertPathWithScheme tests (#6251)
add c72d895fed [MINOR] Add license header (#6247)
add 2996292f2b [HUDI-4025] Add Presto and Trino query node to validate
queries (#5578)
add 251b63ef0b [HUDI-4518] Free lock if allocated but not acquired (#6272)
add 84d195d4d4 [HUDI-4510] Repair config "hive_sync.metastore.uris" in
flink sql hive schema sync is not effective (#6257)
add 5cb51220fe [HUDI-3848] Fixing minor bug in listing based rollback
request generation (#6244)
add 4cb36befd5 [HUDI-3378][HUDI-3379][HUDI-3381] Migrate usage of
HoodieRecordPayload and raw Avro payload to HoodieRecord (#5522)
add 798dfd9aa7 [MINOR] Fix type casting in TestHoodieHFileReaderWriter
add 2d6b966307 [HUDI-4292][RFC-46] Update doc to align with the Record
Merge API changes (#5927)
add 34dd504721 [HUDI-4344] fix usage of HoodieDataBlock#getRecordIterator
(#6005)
add ec486855e5 [HUDI-3350][HUDI-3351] Support HoodieMerge API and Spark
engine-specific HoodieRecord (#5627)
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (af53430947)
\
N -- N -- N refs/heads/release-feature-rfc46 (ec486855e5)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.github/workflows/bot.yml | 5 +-
azure-pipelines.yml | 10 +-
.../test-suite/deltastreamer-hive-sync-presto.yaml | 36 +-
docker/demo/sparksql-incremental.commands | 18 +-
docker/hoodie/hadoop/base/pom.xml | 2 +-
docker/hoodie/hadoop/base_java11/pom.xml | 2 +-
docker/hoodie/hadoop/datanode/pom.xml | 2 +-
docker/hoodie/hadoop/historyserver/pom.xml | 2 +-
docker/hoodie/hadoop/hive_base/pom.xml | 2 +-
docker/hoodie/hadoop/namenode/pom.xml | 2 +-
docker/hoodie/hadoop/pom.xml | 2 +-
docker/hoodie/hadoop/prestobase/pom.xml | 2 +-
docker/hoodie/hadoop/spark_base/pom.xml | 2 +-
docker/hoodie/hadoop/sparkadhoc/pom.xml | 2 +-
docker/hoodie/hadoop/sparkmaster/pom.xml | 2 +-
docker/hoodie/hadoop/sparkworker/pom.xml | 2 +-
docker/hoodie/hadoop/trinobase/pom.xml | 2 +-
docker/hoodie/hadoop/trinocoordinator/pom.xml | 2 +-
docker/hoodie/hadoop/trinoworker/pom.xml | 2 +-
hudi-aws/pom.xml | 16 +-
.../hudi/aws/sync/AWSGlueCatalogSyncClient.java | 96 +--
.../hudi/aws/sync/AwsGlueCatalogSyncTool.java | 35 +-
hudi-cli/pom.xml | 14 +-
.../apache/hudi/cli/commands/CommitsCommand.java | 2 +-
.../hudi/cli/commands/CompactionCommand.java | 2 +-
.../hudi/cli/commands/FileSystemViewCommand.java | 2 +-
.../cli/commands/HDFSParquetImportCommand.java | 2 +-
...Command.java => HoodieSyncValidateCommand.java} | 2 +-
.../apache/hudi/cli/commands/MetadataCommand.java | 10 +-
.../org/apache/hudi/cli/commands/SparkMain.java | 4 +-
.../cli/commands/TestArchivedCommitsCommand.java | 6 +-
.../hudi/cli/commands/TestCommitsCommand.java | 9 +-
.../hudi/cli/commands/TestCompactionCommand.java | 6 +-
.../cli/commands/TestFileSystemViewCommand.java | 121 ++-
hudi-client/hudi-client-common/pom.xml | 16 +-
.../http/HoodieWriteCommitHttpCallbackClient.java | 4 +-
.../apache/hudi/client/BaseHoodieWriteClient.java | 19 +-
.../hudi/client/HoodieInternalWriteStatus.java | 4 +
.../apache/hudi/client/HoodieTimelineArchiver.java | 55 +-
.../client/embedded/EmbeddedTimelineService.java | 3 +-
.../client/transaction/TransactionManager.java | 4 +
.../lock/FileSystemBasedLockProvider.java | 157 ++++
.../hudi/client/transaction/lock/LockManager.java | 5 +
.../apache/hudi/config/HoodieArchivalConfig.java | 209 +++++
.../org/apache/hudi/config/HoodieCleanConfig.java | 297 +++++++
.../apache/hudi/config/HoodieCompactionConfig.java | 397 +--------
.../apache/hudi/config/HoodieHBaseIndexConfig.java | 4 +
.../org/apache/hudi/config/HoodieLockConfig.java | 19 +-
.../apache/hudi/config/HoodiePayloadConfig.java | 20 +
.../apache/hudi/config/HoodieStorageConfig.java | 31 +-
.../org/apache/hudi/config/HoodieWriteConfig.java | 125 ++-
.../apache/hudi/index/bloom/HoodieBloomIndex.java | 4 +-
.../bloom/ListBasedHoodieBloomIndexHelper.java | 3 +-
.../hudi/index/bucket/HoodieBucketIndex.java | 3 +-
.../org/apache/hudi/io/HoodieKeyLookupHandle.java | 3 +-
.../hudi/keygen/ComplexAvroKeyGenerator.java | 2 +-
.../apache/hudi/keygen/CustomAvroKeyGenerator.java | 4 +-
.../hudi/keygen/GlobalAvroDeleteKeyGenerator.java | 2 +-
.../java/org/apache/hudi/keygen/KeyGenUtils.java | 1 +
.../keygen/NonpartitionedAvroKeyGenerator.java | 4 +-
.../apache/hudi/keygen/SimpleAvroKeyGenerator.java | 2 +-
.../keygen/TimestampBasedAvroKeyGenerator.java | 4 +-
.../metadata/HoodieBackedTableMetadataWriter.java | 28 +-
.../hudi/metrics/datadog/DatadogHttpClient.java | 4 +-
.../hudi/metrics/datadog/DatadogReporter.java | 4 +-
.../java/org/apache/hudi/table/HoodieTable.java | 13 +-
.../hudi/table/action/clean/CleanPlanner.java | 6 +-
.../table/action/commit/HoodieMergeHelper.java | 2 +-
.../table/action/index/RunIndexActionExecutor.java | 5 +-
.../rollback/ListingBasedRollbackStrategy.java | 6 +-
.../hudi/callback/http/TestCallbackHttpClient.java | 38 +-
.../client/transaction/TestTransactionManager.java | 8 +-
.../apache/hudi/config/TestHoodieWriteConfig.java | 14 +-
.../metrics/datadog/TestDatadogHttpClient.java | 41 +-
.../hudi/metrics/datadog/TestDatadogReporter.java | 25 +-
hudi-client/hudi-flink-client/pom.xml | 16 +-
.../apache/hudi/client/HoodieFlinkWriteClient.java | 63 +-
.../client/common/HoodieFlinkEngineContext.java | 6 +-
.../org/apache/hudi/index/FlinkHoodieIndex.java | 8 +-
.../apache/hudi/io/FlinkMergeAndReplaceHandle.java | 2 +-
.../java/org/apache/hudi/io/FlinkMergeHandle.java | 28 +-
.../FlinkHoodieBackedTableMetadataWriter.java | 3 +-
.../org/apache/hudi/table/HoodieFlinkTable.java | 4 +-
.../table/action/commit/FlinkDeleteHelper.java | 5 +-
.../hudi/table/action/commit/FlinkWriteHelper.java | 5 +-
.../index/bloom/TestFlinkHoodieBloomIndex.java | 11 +-
.../testutils/HoodieFlinkClientTestHarness.java | 7 +-
hudi-client/hudi-java-client/pom.xml | 4 +-
.../apache/hudi/client/HoodieJavaWriteClient.java | 5 +-
.../run/strategy/JavaExecutionStrategy.java | 4 +-
.../client/common/HoodieJavaEngineContext.java | 6 +-
.../org/apache/hudi/index/JavaHoodieIndex.java | 8 +-
.../org/apache/hudi/table/HoodieJavaTable.java | 4 +-
.../commit/BaseJavaCommitActionExecutor.java | 8 +-
.../hudi/table/action/commit/JavaDeleteHelper.java | 5 +-
.../hudi/table/action/commit/JavaWriteHelper.java | 5 +-
.../testutils/HoodieJavaClientTestHarness.java | 2 +-
hudi-client/hudi-spark-client/pom.xml | 4 +-
.../HoodieSparkBootstrapSchemaProvider.java | 18 +-
.../strategy/SingleSparkJobExecutionStrategy.java | 2 +-
.../hudi/client/model/HoodieInternalRow.java | 269 +++---
.../client/utils/SparkInternalSchemaConverter.java | 4 +-
.../hudi/client/utils/SparkValidatorUtils.java | 2 +-
.../org/apache/hudi/data/HoodieJavaPairRDD.java | 17 +-
.../java/org/apache/hudi/data/HoodieJavaRDD.java | 32 +-
.../index/bloom/SparkHoodieBloomIndexHelper.java | 3 +-
.../hbase/RebalancedSparkHoodieHBaseIndex.java | 19 +-
.../hudi/index/hbase/SparkHoodieHBaseIndex.java | 10 +-
.../storage/row/HoodieInternalRowFileWriter.java | 3 +-
.../row/HoodieInternalRowFileWriterFactory.java | 65 +-
.../row/HoodieInternalRowParquetWriter.java | 3 +-
.../hudi/io/storage/row/HoodieRowCreateHandle.java | 153 ++--
.../HoodieRowCreateHandleWithoutMetaFields.java | 64 --
.../storage/row/HoodieRowParquetWriteSupport.java | 41 +-
.../apache/hudi/keygen/BuiltinKeyGenerator.java | 592 +++++++++++--
.../apache/hudi/keygen/ComplexKeyGenerator.java | 33 +-
.../org/apache/hudi/keygen/CustomKeyGenerator.java | 45 +-
.../hudi/keygen/GlobalDeleteKeyGenerator.java | 23 +-
.../hudi/keygen/NonpartitionedKeyGenerator.java | 34 +-
.../apache/hudi/keygen/RowKeyGeneratorHelper.java | 288 -------
.../org/apache/hudi/keygen/SimpleKeyGenerator.java | 50 +-
.../hudi/keygen/SparkKeyGeneratorInterface.java | 51 +-
.../hudi/keygen/TimestampBasedKeyGenerator.java | 54 +-
.../bootstrap/BaseBootstrapMetadataHandler.java | 13 +-
.../java/org/apache/hudi/util/DataTypeUtils.java | 110 +--
.../scala/org/apache/hudi/HoodieSparkUtils.scala | 82 +-
.../org/apache/hudi/SparkAdapterSupport.scala | 4 +-
.../org/apache/hudi/unsafe/UTF8StringBuilder.java | 100 +++
.../scala/org/apache/hudi/util/JFunction.scala | 32 +-
.../spark/sql/HoodieCatalystPlansUtils.scala | 98 +++
.../apache/spark/sql/HoodieUnsafeRowUtils.scala | 122 +++
.../org/apache/spark/sql/hudi/SparkAdapter.scala | 118 ++-
.../org/apache/hudi/client/TestClientRollback.java | 12 +-
.../hudi/client/TestFileBasedLockProvider.java | 135 +++
.../hudi/client/TestHoodieClientMultiWriter.java | 117 ++-
.../hudi/client/TestTableSchemaEvolution.java | 8 +-
.../functional/TestHoodieBackedMetadata.java | 158 ++--
.../functional/TestHoodieBackedTableMetadata.java | 28 +-
.../TestHoodieClientOnCopyOnWriteStorage.java | 34 +-
.../hudi/client/functional/TestHoodieIndex.java | 3 +-
.../client/functional/TestHoodieMetadataBase.java | 22 +-
.../functional/TestHoodieMetadataBootstrap.java | 8 +-
.../hudi/client/model/TestHoodieInternalRow.java | 35 +-
.../hudi/index/bloom/TestHoodieBloomIndex.java | 203 ++++-
.../index/hbase/TestSparkHoodieHBaseIndex.java | 8 +-
.../apache/hudi/io/TestHoodieTimelineArchiver.java | 168 +++-
.../row/TestHoodieInternalRowParquetWriter.java | 3 +-
.../io/storage/row/TestHoodieRowCreateHandle.java | 46 +-
.../java/org/apache/hudi/table/TestCleaner.java | 43 +-
.../TestCopyOnWriteRollbackActionExecutor.java | 67 ++
.../table/functional/TestCleanPlanExecutor.java | 75 +-
...dieSparkCopyOnWriteTableArchiveWithReplace.java | 8 +-
.../TestHoodieSparkMergeOnReadTableRollback.java | 10 +-
.../hudi/testutils/HoodieClientTestBase.java | 9 +-
.../hudi/testutils/HoodieClientTestHarness.java | 4 +-
.../SparkClientFunctionalTestHarness.java | 15 +-
.../hudi/testutils/SparkDatasetTestUtils.java | 1 +
.../hudi/keygen/TestRowGeneratorHelper.scala | 102 ---
.../spark/sql/TestHoodieUnsafeRowUtils.scala | 167 ++++
hudi-client/pom.xml | 2 +-
hudi-common/pom.xml | 2 +-
.../org/apache/hudi/BaseHoodieTableFileIndex.java | 12 +-
.../java/org/apache/hudi/avro/HoodieAvroUtils.java | 47 +-
.../org/apache/hudi/common/bloom/BloomFilter.java | 9 +-
.../bloom/HoodieDynamicBoundedBloomFilter.java | 7 +-
.../hudi/common/bloom/SimpleBloomFilter.java | 9 +-
.../common/config/DFSPropertiesConfiguration.java | 22 +-
.../hudi/common/config/HoodieCommonConfig.java | 7 +
.../apache/hudi/common/config/HoodieConfig.java | 15 +
.../hudi/common/config/HoodieMetadataConfig.java | 30 +-
.../hudi/common/config/LockConfiguration.java | 2 +
.../apache/hudi/common/config/TypedProperties.java | 6 +
.../hudi/common/data/HoodieBaseListData.java | 72 ++
.../org/apache/hudi/common/data/HoodieData.java | 167 ++--
.../org/apache/hudi/common/data/HoodieList.java | 178 ----
.../apache/hudi/common/data/HoodieListData.java | 182 ++++
.../hudi/common/data/HoodieListPairData.java | 201 +++++
.../org/apache/hudi/common/data/HoodieMapPair.java | 168 ----
.../apache/hudi/common/data/HoodiePairData.java | 70 +-
.../common/engine/HoodieLocalEngineContext.java | 6 +-
.../java/org/apache/hudi/common/fs/FSUtils.java | 18 +
.../hudi/common/fs/HoodieWrapperFileSystem.java | 71 +-
.../hudi/common/model}/AWSDmsAvroPayload.java | 6 +-
.../apache/hudi/common/model/HoodieAvroRecord.java | 2 +-
.../hudi/common/model/HoodieAvroRecordMerge.java | 2 +-
.../apache/hudi/common/model/HoodieFileGroup.java | 20 +-
.../org/apache/hudi/common/model/HoodieRecord.java | 4 +
.../hudi/common/table/HoodieTableConfig.java | 14 +-
.../hudi/common/table/HoodieTableMetaClient.java | 8 +-
.../hudi/common/table/TableSchemaResolver.java | 27 +
.../table/log/AbstractHoodieLogRecordReader.java | 4 +-
.../apache/hudi/common/table/log/InstantRange.java | 120 ++-
.../table/log/block/HoodieAvroDataBlock.java | 6 +-
.../table/timeline/HoodieActiveTimeline.java | 31 +-
.../table/timeline/HoodieDefaultTimeline.java | 36 +-
.../hudi/common/table/timeline/HoodieTimeline.java | 9 +
.../view/RemoteHoodieTableFileSystemView.java | 2 -
.../apache/hudi/common/util/CollectionUtils.java | 26 +-
.../java/org/apache/hudi/common/util/Either.java | 93 +++
.../org/apache/hudi/common/util/HoodieTimer.java | 24 +-
.../org/apache/hudi/common/util/ParquetUtils.java | 2 +-
.../hudi/common/util/PartitionPathEncodeUtils.java | 2 +-
.../apache/hudi/common/util/ReflectionUtils.java | 6 +-
.../org/apache/hudi/common/util/StringUtils.java | 22 +-
.../org/apache/hudi/common/util/TypeUtils.java | 12 +
.../hudi/exception/HoodieCatalogException.java | 27 +-
.../schema/action/InternalSchemaMerger.java | 10 +-
.../schema/utils/AvroSchemaEvolutionUtils.java | 74 +-
.../internal/schema/utils/InternalSchemaUtils.java | 7 +-
.../org/apache/hudi/keygen/BaseKeyGenerator.java | 13 +-
.../java/org/apache/hudi/keygen/KeyGenerator.java | 2 +-
.../metadata/FileSystemBackedTableMetadata.java | 52 +-
.../hudi/metadata/HoodieBackedTableMetadata.java | 43 +-
.../metadata/HoodieMetadataFileSystemView.java | 2 +-
.../hudi/metadata/HoodieMetadataPayload.java | 2 +-
.../apache/hudi/metadata/HoodieTableMetadata.java | 3 +-
.../hudi/metadata/HoodieTableMetadataUtil.java | 33 +-
.../hudi/metadata/MetadataPartitionType.java | 2 +-
hudi-common/src/main/resources/hbase-site.xml | 9 +-
.../org/apache/hudi/avro/TestHoodieAvroUtils.java | 30 +
...TestHoodieList.java => TestHoodieListData.java} | 21 +-
...apPair.java => TestHoodieListDataPairData.java} | 80 +-
.../org/apache/hudi/common/fs/TestFSUtils.java | 12 +-
.../apache/hudi/common/fs/TestStorageSchemes.java | 22 +
.../hudi/common/model}/TestAWSDmsAvroPayload.java | 6 +-
.../hudi/common/model/TestHoodieFileGroup.java | 26 +
.../table/timeline/TestHoodieActiveTimeline.java | 95 +++
.../view/TestPriorityBasedFileSystemView.java | 2 +-
.../hudi/common/testutils/FileCreateUtils.java | 15 +
.../common/testutils/HoodieTestDataGenerator.java | 25 +-
.../hudi/common/testutils/HoodieTestTable.java | 28 +-
.../hudi/common/testutils/HoodieTestUtils.java | 4 +
.../hudi/common/testutils/MockHoodieTimeline.java | 6 +
.../testutils/minicluster/HdfsTestService.java | 4 +-
.../testutils/minicluster/MiniClusterUtil.java | 4 +-
.../apache/hudi/common/util/TestParquetUtils.java | 2 +-
.../apache/hudi/common/util/TestStringUtils.java | 24 +
.../schema/utils/TestAvroSchemaEvolutionUtils.java | 78 +-
hudi-examples/hudi-examples-common/pom.xml | 2 +-
hudi-examples/hudi-examples-flink/pom.xml | 24 +-
.../quickstart/source/ContinuousFileSource.java | 5 +-
.../quickstart/TestHoodieFlinkQuickstart.java | 2 +
hudi-examples/hudi-examples-java/pom.xml | 2 +-
.../java/HoodieJavaWriteClientExample.java | 4 +-
hudi-examples/hudi-examples-spark/pom.xml | 8 +-
.../examples/common/HoodieExampleSparkUtils.java | 1 +
.../examples/spark/HoodieWriteClientExample.java | 4 +-
hudi-examples/pom.xml | 2 +-
hudi-flink-datasource/hudi-flink/pom.xml | 64 +-
.../apache/hudi/configuration/FlinkOptions.java | 12 +-
.../hudi/configuration/HadoopConfigurations.java | 4 +-
.../apache/hudi/configuration/OptionsResolver.java | 25 +
.../java/org/apache/hudi/sink/CleanFunction.java | 4 +-
.../org/apache/hudi/sink/StreamWriteFunction.java | 1 +
.../hudi/sink/StreamWriteOperatorCoordinator.java | 56 +-
.../hudi/sink/append/AppendWriteFunction.java | 1 +
.../org/apache/hudi/sink/bulk/RowDataKeyGen.java | 9 +-
.../sink/common/AbstractStreamWriteFunction.java | 16 +-
.../hudi/sink/compact/CompactionCommitSink.java | 4 +-
.../sink/compact/CompactionPlanSourceFunction.java | 31 +-
.../hudi/sink/compact/FlinkCompactionConfig.java | 18 +-
.../hudi/sink/compact/HoodieFlinkCompactor.java | 93 ++-
.../compact/strategy/CompactionPlanStrategies.java | 74 ++
.../compact/strategy/CompactionPlanStrategy.java | 23 +-
.../org/apache/hudi/sink/meta/CkpMetadata.java | 8 +-
.../sink/partitioner/profile/WriteProfiles.java | 42 +
.../apache/hudi/sink/utils/HiveSyncContext.java | 104 ++-
.../apache/hudi/sink/utils/NonThrownExecutor.java | 4 +
.../java/org/apache/hudi/sink/utils/Pipelines.java | 24 +-
.../java/org/apache/hudi/source/FileIndex.java | 122 ++-
.../apache/hudi/source/IncrementalInputSplits.java | 194 ++++-
.../hudi/source/StreamReadMonitoringFunction.java | 3 +
.../hudi/source/stats/ColumnStatsIndices.java | 387 +++++++++
.../hudi/source/stats/ExpressionEvaluator.java | 552 +++++++++++++
.../apache/hudi/streamer/FlinkStreamerConfig.java | 3 +-
.../org/apache/hudi/table/HoodieTableSink.java | 6 +-
.../org/apache/hudi/table/HoodieTableSource.java | 79 +-
.../apache/hudi/table/catalog/CatalogOptions.java | 18 +
.../apache/hudi/table/catalog/HiveSchemaUtils.java | 207 +++++
.../hudi/table/catalog/HoodieCatalogFactory.java | 38 +-
.../hudi/table/catalog/HoodieCatalogUtil.java | 96 +++
.../hudi/table/catalog/HoodieHiveCatalog.java | 911 +++++++++++++++++++++
.../hudi/table/catalog/TableOptionProperties.java | 64 ++
.../table/catalog/TypeInfoLogicalTypeVisitor.java | 198 +++++
.../apache/hudi/table/format/FilePathUtils.java | 8 +
.../table/format/cow/CopyOnWriteInputFormat.java | 8 +-
.../table/format/mor/MergeOnReadInputFormat.java | 10 +-
.../org/apache/hudi/util/AvroSchemaConverter.java | 37 +-
.../apache/hudi/util/AvroToRowDataConverters.java | 4 +-
.../java/org/apache/hudi/util/DataTypeUtils.java | 62 ++
.../java/org/apache/hudi/util/ExpressionUtils.java | 180 ++++
.../java/org/apache/hudi/util/HoodiePipeline.java | 270 ++++++
.../org/apache/hudi/util/RowDataProjection.java | 8 +
.../java/org/apache/hudi/util/StreamerUtil.java | 102 ++-
.../apache/hudi/util/ViewStorageProperties.java | 2 +-
.../apache/hudi/sink/ITTestDataStreamWrite.java | 143 +++-
.../org/apache/hudi/sink/TestWriteCopyOnWrite.java | 6 +-
.../org/apache/hudi/sink/TestWriteMergeOnRead.java | 57 ++
.../apache/hudi/sink/bulk/TestRowDataKeyGen.java | 21 +-
.../sink/compact/ITTestHoodieFlinkCompactor.java | 125 ++-
.../sink/compact/TestCompactionPlanStrategy.java | 150 ++++
.../hudi/sink/utils/TestHiveSyncContext.java | 16 +-
.../org/apache/hudi/sink/utils/TestWriteBase.java | 19 +-
.../java/org/apache/hudi/source/TestFileIndex.java | 30 +-
.../apache/hudi/source/TestStreamReadOperator.java | 8 +-
.../hudi/source/stats/TestColumnStatsIndices.java | 98 +++
.../hudi/source/stats/TestExpressionEvaluator.java | 374 +++++++++
.../apache/hudi/table/ITTestHoodieDataSource.java | 143 +++-
.../apache/hudi/table/TestHoodieTableSource.java | 18 +-
.../hudi/table/catalog/HoodieCatalogTestUtils.java | 67 ++
.../table/catalog/TestHoodieCatalogFactory.java | 98 +++
.../hudi/table/catalog/TestHoodieHiveCatalog.java | 183 +++++
.../apache/hudi/table/format/TestInputFormat.java | 102 +++
.../apache/hudi/utils/TestAvroSchemaConverter.java | 53 ++
.../org/apache/hudi/utils/TestCompactionUtil.java | 6 +
.../org/apache/hudi/utils/TestConfigurations.java | 3 +
.../test/java/org/apache/hudi/utils/TestData.java | 145 +++-
.../test/java/org/apache/hudi/utils/TestUtils.java | 10 +-
.../hudi/utils/TestViewStorageProperties.java | 2 +-
.../hudi/utils/source/ContinuousFileSource.java | 5 +-
.../hudi-flink/src/test/resources/hive-site.xml | 30 +
.../test-catalog-factory-conf/hive-site.xml | 26 +
hudi-flink-datasource/hudi-flink1.13.x/pom.xml | 27 +-
.../adapter/DataStreamScanProviderAdapter.java | 50 +-
.../adapter/DataStreamSinkProviderAdapter.java | 50 +-
.../main/java/org/apache/hudi/adapter/Utils.java | 12 +
.../table/format/cow/ParquetSplitReaderUtil.java | 0
.../table/format/cow/vector/HeapArrayVector.java | 0
.../format/cow/vector/HeapMapColumnVector.java | 0
.../format/cow/vector/HeapRowColumnVector.java | 0
.../format/cow/vector/ParquetDecimalVector.java | 0
.../cow/vector/reader/AbstractColumnReader.java | 0
.../cow/vector/reader/ArrayColumnReader.java | 0
.../vector/reader/BaseVectorizedColumnReader.java | 0
.../vector/reader/FixedLenBytesColumnReader.java | 0
.../vector/reader/Int64TimestampColumnReader.java | 0
.../format/cow/vector/reader/MapColumnReader.java | 0
.../reader/ParquetColumnarRowSplitReader.java | 0
.../cow/vector/reader/ParquetDataColumnReader.java | 0
.../reader/ParquetDataColumnReaderFactory.java | 0
.../format/cow/vector/reader/RowColumnReader.java | 0
.../format/cow/vector/reader/RunLengthDecoder.java | 6 +-
hudi-flink-datasource/hudi-flink1.14.x/pom.xml | 33 +-
.../adapter/DataStreamScanProviderAdapter.java | 50 +-
.../adapter/DataStreamSinkProviderAdapter.java | 50 +-
.../main/java/org/apache/hudi/adapter/Utils.java | 12 +
.../table/format/cow/ParquetSplitReaderUtil.java | 0
.../table/format/cow/vector/HeapArrayVector.java | 0
.../format/cow/vector/HeapMapColumnVector.java | 0
.../format/cow/vector/HeapRowColumnVector.java | 0
.../format/cow/vector/ParquetDecimalVector.java | 0
.../cow/vector/reader/AbstractColumnReader.java | 0
.../cow/vector/reader/ArrayColumnReader.java | 0
.../vector/reader/BaseVectorizedColumnReader.java | 0
.../vector/reader/FixedLenBytesColumnReader.java | 0
.../vector/reader/Int64TimestampColumnReader.java | 0
.../format/cow/vector/reader/MapColumnReader.java | 0
.../reader/ParquetColumnarRowSplitReader.java | 0
.../cow/vector/reader/ParquetDataColumnReader.java | 0
.../reader/ParquetDataColumnReaderFactory.java | 0
.../format/cow/vector/reader/RowColumnReader.java | 0
.../format/cow/vector/reader/RunLengthDecoder.java | 6 +-
.../{hudi-flink1.14.x => hudi-flink1.15.x}/pom.xml | 49 +-
.../adapter/AbstractStreamOperatorAdapter.java | 0
.../AbstractStreamOperatorFactoryAdapter.java | 0
.../adapter/DataStreamScanProviderAdapter.java | 21 +-
.../adapter/DataStreamSinkProviderAdapter.java | 20 +-
.../hudi/adapter/MailboxExecutorAdapter.java | 0
.../apache/hudi/adapter/RateLimiterAdapter.java | 0
.../main/java/org/apache/hudi/adapter/Utils.java | 14 +
.../table/format/cow/ParquetSplitReaderUtil.java | 28 +-
.../table/format/cow/vector/HeapArrayVector.java | 10 +-
.../format/cow/vector/HeapMapColumnVector.java | 10 +-
.../format/cow/vector/HeapRowColumnVector.java | 10 +-
.../format/cow/vector/ParquetDecimalVector.java | 6 +-
.../cow/vector/reader/AbstractColumnReader.java | 4 +-
.../cow/vector/reader/ArrayColumnReader.java | 22 +-
.../vector/reader/BaseVectorizedColumnReader.java | 2 +-
.../vector/reader/FixedLenBytesColumnReader.java | 6 +-
.../vector/reader/Int64TimestampColumnReader.java | 4 +-
.../format/cow/vector/reader/MapColumnReader.java | 4 +-
.../reader/ParquetColumnarRowSplitReader.java | 8 +-
.../cow/vector/reader/ParquetDataColumnReader.java | 0
.../reader/ParquetDataColumnReaderFactory.java | 0
.../format/cow/vector/reader/RowColumnReader.java | 2 +-
.../format/cow/vector/reader/RunLengthDecoder.java | 10 +-
.../org/apache/hudi/adapter/OutputAdapter.java | 0
.../adapter/StateInitializationContextAdapter.java | 24 +-
.../adapter/StreamingRuntimeContextAdapter.java | 0
.../org/apache/hudi/adapter/TestTableEnvs.java | 0
hudi-flink-datasource/pom.xml | 5 +-
hudi-gcp/pom.xml | 14 +-
.../hudi/gcp/bigquery/BigQuerySyncConfig.java | 196 ++---
.../apache/hudi/gcp/bigquery/BigQuerySyncTool.java | 67 +-
.../gcp/bigquery/HoodieBigQuerySyncClient.java | 109 +--
.../hudi/gcp/bigquery/TestBigQuerySyncConfig.java | 92 +--
hudi-hadoop-mr/pom.xml | 2 +-
.../hadoop/HoodieCopyOnWriteTableInputFormat.java | 4 +
.../hudi/hadoop/realtime/HoodieRealtimePath.java | 2 +-
.../utils/HoodieRealtimeInputFormatUtils.java | 2 +-
.../utils/HoodieRealtimeRecordReaderUtils.java | 10 +-
.../TestHoodieCopyOnWriteTableInputFormat.java | 60 ++
.../apache/hudi/hadoop/TestInputPathHandler.java | 2 +-
.../TestHoodieMergeOnReadTableInputFormat.java | 68 ++
.../realtime/TestHoodieRealtimeRecordReader.java | 17 +
hudi-integ-test/pom.xml | 24 +-
.../hudi/integ/testsuite/HoodieTestSuiteJob.java | 23 +
.../integ/testsuite/HoodieTestSuiteWriter.java | 6 +-
.../integ/testsuite/configuration/DeltaConfig.java | 54 +-
.../apache/hudi/integ/testsuite/dag/DagUtils.java | 114 ++-
.../integ/testsuite/dag/nodes/BaseQueryNode.java | 62 ++
.../integ/testsuite/dag/nodes/HiveQueryNode.java | 50 +-
.../integ/testsuite/dag/nodes/PrestoQueryNode.java | 60 ++
.../integ/testsuite/dag/nodes/TrinoQueryNode.java | 60 ++
.../integ/testsuite/generator/DeltaGenerator.java | 2 +-
.../testsuite/helpers/HiveServiceProvider.java | 16 +-
.../java/org/apache/hudi/integ/ITTestBase.java | 1 +
.../testsuite/job/TestHoodieTestSuiteJob.java | 14 +-
.../src/test/resources/unit-test-cow-dag.yaml | 22 +-
hudi-kafka-connect/pom.xml | 18 +-
.../hudi/connect/utils/KafkaConnectUtils.java | 9 +-
.../writers/KafkaConnectWriterProvider.java | 19 +-
hudi-spark-datasource/README.md | 17 +-
hudi-spark-datasource/hudi-spark-common/pom.xml | 4 +-
.../main/java/org/apache/hudi/DataSourceUtils.java | 109 +--
.../apache/hudi/HoodieDatasetBulkInsertHelper.java | 189 -----
.../java/org/apache/hudi/SparkRowWriteHelper.java | 72 --
.../BulkInsertDataInternalWriterHelper.java | 118 +--
.../internal/DataSourceInternalWriterHelper.java | 1 +
.../org/apache/hudi/BaseFileOnlyRelation.scala | 89 +-
.../org/apache/hudi/ColumnStatsIndexSupport.scala | 481 +++++++----
.../scala/org/apache/hudi/DataSourceOptions.scala | 100 +--
.../main/scala/org/apache/hudi/DefaultSource.scala | 21 +-
.../scala/org/apache/hudi/HoodieBaseRelation.scala | 426 +++++++---
.../org/apache/hudi/HoodieCatalystUtils.scala | 65 ++
.../org/apache/hudi/HoodieDataSourceHelper.scala | 37 +-
.../hudi/HoodieDatasetBulkInsertHelper.scala | 156 ++++
.../scala/org/apache/hudi/HoodieDatasetUtils.scala | 45 -
.../scala/org/apache/hudi/HoodieFileIndex.scala | 117 +--
.../org/apache/hudi/HoodieMergeOnReadRDD.scala | 258 +++---
.../org/apache/hudi/HoodieSparkSqlWriter.scala | 131 ++-
.../scala/org/apache/hudi/HoodieWriterUtils.scala | 23 +-
.../hudi/MergeOnReadIncrementalRelation.scala | 55 +-
.../apache/hudi/MergeOnReadSnapshotRelation.scala | 201 ++++-
.../apache/hudi/SparkHoodieTableFileIndex.scala | 23 +-
...nsafeRDDUtils.scala => HoodieUnsafeUtils.scala} | 45 +-
.../apache/spark/sql/avro/SchemaConverters.scala | 41 +-
.../apache/spark/sql/hudi/DataSkippingUtils.scala | 2 +-
.../spark/sql/hudi/HoodieSqlCommonUtils.scala | 6 +-
.../spark/sql/hudi/ProvidesHoodieConfig.scala | 87 +-
.../sql/hudi/command/payload/SqlTypedRecord.scala | 11 +-
hudi-spark-datasource/hudi-spark/pom.xml | 18 +-
.../apache/hudi/cli/BootstrapExecutorUtils.java | 28 +-
.../org/apache/hudi/payload/AWSDmsAvroPayload.java | 57 +-
.../org/apache/spark/sql/hudi/DeDupeType.scala | 19 +-
.../org/apache/spark/sql/hudi/DedupeSparkJob.scala | 245 ++++++
.../sql/hudi/HoodieSparkSessionExtension.scala | 15 +-
.../org/apache/spark/sql/hudi/HoodieSqlUtils.scala | 2 +-
.../org/apache/spark/sql/hudi/SparkHelpers.scala | 134 +++
.../spark/sql/hudi/analysis/HoodieAnalysis.scala | 58 +-
.../command/CreateHoodieTableAsSelectCommand.scala | 13 +-
.../hudi/command/DeleteHoodieTableCommand.scala | 6 +-
.../command/InsertIntoHoodieTableCommand.scala | 244 +++---
.../hudi/command/MergeIntoHoodieTableCommand.scala | 50 +-
.../hudi/command/procedures/BaseProcedure.scala | 1 -
.../procedures/CommitsCompareProcedure.scala | 10 +-
.../command/procedures/CopyToTableProcedure.scala | 128 +++
.../procedures/CreateMetadataTableProcedure.scala | 80 ++
...cedure.scala => CreateSavepointProcedure.scala} | 14 +-
.../command/procedures/DeleteMarkerProcedure.scala | 2 +-
...re.scala => DeleteMetadataTableProcedure.scala} | 55 +-
...cedure.scala => DeleteSavepointProcedure.scala} | 12 +-
.../procedures/ExportInstantsProcedure.scala | 2 +-
.../procedures/HdfsParquetImportProcedure.scala | 14 +-
.../hudi/command/procedures/HoodieProcedures.scala | 24 +-
.../procedures/InitMetadataTableProcedure.scala | 84 ++
.../RepairAddpartitionmetaProcedure.scala | 89 ++
.../RepairCorruptedCleanFilesProcedure.scala | 86 ++
.../procedures/RepairDeduplicateProcedure.scala | 86 ++
.../RepairMigratePartitionMetaProcedure.scala | 112 +++
.../RepairOverwriteHoodiePropsProcedure.scala | 89 ++
.../RollbackToInstantTimeProcedure.scala | 2 +
...re.scala => RollbackToSavepointProcedure.scala} | 12 +-
.../command/procedures/RunBootstrapProcedure.scala | 30 +-
.../command/procedures/RunCleanProcedure.scala | 95 +++
.../procedures/RunClusteringProcedure.scala | 2 +-
.../procedures/ShowArchivedCommitsProcedure.scala | 6 +-
.../procedures/ShowBootstrapMappingProcedure.scala | 10 +-
.../procedures/ShowCommitFilesProcedure.scala | 2 +-
.../command/procedures/ShowCommitsProcedure.scala | 2 +-
.../procedures/ShowFileSystemViewProcedure.scala | 2 +-
.../procedures/ShowFsPathDetailProcedure.scala | 112 +++
.../ShowMetadataTableFilesProcedure.scala | 88 ++
.../ShowMetadataTablePartitionsProcedure.scala | 81 ++
...scala => ShowMetadataTableStatsProcedure.scala} | 63 +-
.../procedures/StatsFileSizeProcedure.scala | 4 +-
.../procedures/UpgradeOrDowngradeProcedure.scala | 107 +++
.../procedures/ValidateHoodieSyncProcedure.scala | 208 +++++
.../ValidateMetadataTableFilesProcedure.scala | 147 ++++
.../spark/sql/parser/HoodieCommonSqlParser.scala | 8 +
.../hudi-spark/src/test/java/HoodieJavaApp.java | 18 +-
.../src/test/java/HoodieJavaGenerateApp.java | 18 +-
.../src/test/java/HoodieJavaStreamingApp.java | 18 +-
.../java/org/apache/hudi/TestDataSourceUtils.java | 91 +-
.../TestHoodieDatasetBulkInsertHelper.java | 98 ++-
.../hudi/keygen/TestComplexKeyGenerator.java | 22 +-
.../apache/hudi/keygen/TestCustomKeyGenerator.java | 43 +-
.../keygen/TestGlobalDeleteRecordGenerator.java | 2 -
.../keygen/TestNonpartitionedKeyGenerator.java | 5 +-
.../apache/hudi/keygen/TestSimpleKeyGenerator.java | 33 +-
.../keygen/TestTimestampBasedKeyGenerator.java | 15 +-
.../apache/hudi/testutils/DataSourceTestUtils.java | 14 +-
.../hudi/testutils/KeyGeneratorTestUtilities.java | 2 +-
.../src/test/resources/table-config.properties | 0
.../org/apache/hudi/ScalaAssertionSupport.scala | 30 +-
.../org/apache/hudi/TestAvroConversionUtils.scala | 57 +-
.../org/apache/hudi/TestDataSkippingUtils.scala | 2 +-
.../org/apache/hudi/TestDataSourceDefaults.scala | 716 ++++++++--------
.../hudi/TestGenericRecordAndRowConsistency.scala | 3 +-
.../org/apache/hudi/TestHoodieFileIndex.scala | 10 +-
.../org/apache/hudi/TestHoodieRelations.scala | 54 ++
.../org/apache/hudi/TestHoodieSparkSqlWriter.scala | 16 +-
.../org/apache/hudi/TestHoodieSparkUtils.scala | 87 +-
.../apache/hudi/functional/TestCOWDataSource.scala | 43 +-
.../hudi/functional/TestColumnStatsIndex.scala | 168 ++--
.../functional/TestDataSourceForBootstrap.scala | 42 +-
.../apache/hudi/functional/TestMORDataSource.scala | 152 +++-
.../functional/TestParquetColumnProjection.scala | 9 +-
.../org/apache/spark/sql/avro/TestAvroSerDe.scala | 61 ++
.../spark/sql/avro/TestSchemaConverters.scala | 40 +
.../spark/sql/hudi/HoodieSparkSqlTestBase.scala | 23 +-
.../sql/hudi/TestAlterTableDropPartition.scala | 11 +-
.../apache/spark/sql/hudi/TestDeleteTable.scala | 39 +-
.../apache/spark/sql/hudi/TestInsertTable.scala | 209 ++++-
.../apache/spark/sql/hudi/TestMergeIntoTable.scala | 183 ++++-
.../hudi/TestNestedSchemaPruningOptimization.scala | 122 +++
.../apache/spark/sql/hudi/TestShowPartitions.scala | 42 +-
.../org/apache/spark/sql/hudi/TestSpark3DDL.scala | 97 ++-
.../org/apache/spark/sql/hudi/TestSqlConf.scala | 37 +-
.../spark/sql/hudi/TestTimeTravelTable.scala | 44 +
.../apache/spark/sql/hudi/TestUpdateTable.scala | 28 +-
.../hudi/procedure/TestBootstrapProcedure.scala | 12 +-
.../sql/hudi/procedure/TestCallCommandParser.scala | 11 +-
.../sql/hudi/procedure/TestCallProcedure.scala | 4 +-
.../sql/hudi/procedure/TestCleanProcedure.scala | 64 ++
.../sql/hudi/procedure/TestCommitsProcedure.scala | 4 +-
.../hudi/procedure/TestCopyToTableProcedure.scala | 353 ++++++++
.../procedure/TestExportInstantsProcedure.scala | 2 +-
.../procedure/TestHdfsParquetImportProcedure.scala | 20 +-
.../sql/hudi/procedure/TestMetadataProcedure.scala | 262 ++++++
.../sql/hudi/procedure/TestRepairsProcedure.scala | 507 ++++++++++++
.../hudi/procedure/TestSavepointsProcedure.scala | 18 +-
...e.scala => TestShowFsPathDetailProcedure.scala} | 12 +-
.../sql/hudi/procedure/TestStatsProcedure.scala | 6 +-
.../TestUpgradeOrDowngradeProcedure.scala | 97 +++
hudi-spark-datasource/hudi-spark2-common/pom.xml | 2 +-
hudi-spark-datasource/hudi-spark2/pom.xml | 4 +-
.../org/apache/hudi/internal/DefaultSource.java | 4 +-
.../org/apache/hudi/Spark2HoodieFileScanRDD.scala} | 13 +-
.../spark/sql/HoodieSpark2CatalystPlanUtils.scala | 81 ++
.../apache/spark/sql/adapter/Spark2Adapter.scala | 110 ++-
.../apache/spark/sql/avro/AvroDeserializer.scala | 2 +
hudi-spark-datasource/hudi-spark3-common/pom.xml | 2 +-
.../apache/hudi/spark3/internal/DefaultSource.java | 4 +-
.../apache/hudi/spark3/internal/ReflectUtil.java | 2 +-
.../spark/sql/HoodieSpark3CatalystPlanUtils.scala | 85 ++
.../apache/spark/sql/HoodieSpark3SqlUtils.scala | 45 -
.../spark/sql/adapter/BaseSpark3Adapter.scala | 103 +--
hudi-spark-datasource/hudi-spark3.1.x/pom.xml | 4 +-
.../apache/hudi/Spark31HoodieFileScanRDD.scala} | 13 +-
... => HoodieSpark31CatalystExpressionUtils.scala} | 2 +-
.../spark/sql/HoodieSpark31CatalystPlanUtils.scala | 18 +-
.../apache/spark/sql/adapter/Spark3_1Adapter.scala | 42 +-
.../apache/spark/sql/avro/AvroDeserializer.scala | 2 +
.../datasources/Spark31NestedSchemaPruning.scala | 197 +++++
... => Spark312ResolveHudiAlterTableCommand.scala} | 2 +-
...scala => HoodieSpark3_1ExtendedSqlParser.scala} | 2 +-
.../{hudi-spark3 => hudi-spark3.2.x}/pom.xml | 12 +-
.../src/main/antlr4/imports/SqlBase.g4 | 0
.../apache/hudi/spark/sql/parser/HoodieSqlBase.g4 | 0
...Spark32HoodieVectorizedParquetRecordReader.java | 0
...org.apache.spark.sql.sources.DataSourceRegister | 0
.../apache/hudi/Spark32HoodieFileScanRDD.scala} | 13 +-
.../org/apache/hudi/Spark3DefaultSource.scala | 0
.../HoodieSpark32CatalystExpressionUtils.scala} | 2 +-
.../spark/sql/HoodieSpark32CatalystPlanUtils.scala | 27 +-
.../apache/spark/sql/adapter/Spark3_2Adapter.scala | 57 +-
.../apache/spark/sql/avro/AvroDeserializer.scala | 2 +
.../org/apache/spark/sql/avro/AvroSerializer.scala | 0
.../org/apache/spark/sql/avro/AvroUtils.scala | 0
.../sql/avro/HoodieSpark3_2AvroDeserializer.scala | 0
.../sql/avro/HoodieSpark3_2AvroSerializer.scala | 0
.../plans/logical/TimeTravelRelation.scala | 0
.../sql/connector/catalog/HoodieIdentifier.scala | 0
.../datasources/Spark32NestedSchemaPruning.scala | 197 +++++
.../parquet/Spark32DataSourceUtils.scala | 0
.../parquet/Spark32HoodieParquetFileFormat.scala | 0
.../Spark32ResolveHudiAlterTableCommand.scala} | 2 +-
.../sql/hudi/analysis/HoodieSpark3Analysis.scala | 0
.../spark/sql/hudi/catalog/BasicStagedTable.scala | 0
.../spark/sql/hudi/catalog/HoodieCatalog.scala | 30 +-
.../sql/hudi/catalog/HoodieInternalV2Table.scala | 0
.../spark/sql/hudi/catalog/HoodieStagedTable.scala | 0
.../spark/sql/hudi/catalog/TableCreationMode.java | 0
.../spark/sql/hudi/command/AlterTableCommand.scala | 0
.../HoodieSpark3_2ExtendedSqlAstBuilder.scala | 2 +-
.../parser/HoodieSpark3_2ExtendedSqlParser.scala | 32 +-
.../TestHoodieBulkInsertDataInternalWriter.java | 0
.../TestHoodieDataSourceInternalBatchWrite.java | 0
.../hudi/spark3/internal/TestReflectUtil.java | 0
.../test/resources/log4j-surefire-quiet.properties | 0
.../src/test/resources/log4j-surefire.properties | 0
.../{hudi-spark3 => hudi-spark3.3.x}/pom.xml | 75 +-
.../src/main/antlr4/imports/SqlBase.g4 | 0
.../apache/hudi/spark/sql/parser/HoodieSqlBase.g4 | 0
...park33HoodieVectorizedParquetRecordReader.java} | 4 +-
...org.apache.spark.sql.sources.DataSourceRegister | 0
.../apache/hudi/Spark33HoodieFileScanRDD.scala} | 13 +-
.../org/apache/hudi/Spark3DefaultSource.scala | 0
.../HoodieSpark33CatalystExpressionUtils.scala} | 6 +-
.../spark/sql/HoodieSpark33CatalystPlanUtils.scala | 27 +-
.../apache/spark/sql/adapter/Spark3_3Adapter.scala | 80 ++
.../apache/spark/sql/avro/AvroDeserializer.scala | 173 ++--
.../org/apache/spark/sql/avro/AvroSerializer.scala | 151 ++--
.../org/apache/spark/sql/avro/AvroUtils.scala | 228 ++++++
.../sql/avro/HoodieSpark3_3AvroDeserializer.scala} | 2 +-
.../sql/avro/HoodieSpark3_3AvroSerializer.scala} | 2 +-
.../plans/logical/TimeTravelRelation.scala | 0
.../sql/connector/catalog/HoodieIdentifier.scala | 0
.../datasources/Spark33NestedSchemaPruning.scala | 195 +++++
.../parquet/Spark33DataSourceUtils.scala} | 2 +-
.../parquet/Spark33HoodieParquetFileFormat.scala} | 18 +-
.../Spark33ResolveHudiAlterTableCommand.scala} | 4 +-
.../sql/hudi/analysis/HoodieSpark3Analysis.scala | 16 +-
.../spark/sql/hudi/catalog/BasicStagedTable.scala | 0
.../spark/sql/hudi/catalog/HoodieCatalog.scala | 28 +-
.../sql/hudi/catalog/HoodieInternalV2Table.scala | 0
.../spark/sql/hudi/catalog/HoodieStagedTable.scala | 0
.../spark/sql/hudi/catalog/TableCreationMode.java | 0
.../spark/sql/hudi/command/AlterTableCommand.scala | 0
.../HoodieSpark3_3ExtendedSqlAstBuilder.scala} | 29 +-
.../parser/HoodieSpark3_3ExtendedSqlParser.scala} | 40 +-
.../TestHoodieBulkInsertDataInternalWriter.java | 0
.../TestHoodieDataSourceInternalBatchWrite.java | 0
.../hudi/spark3/internal/TestReflectUtil.java | 0
.../test/resources/log4j-surefire-quiet.properties | 0
.../src/test/resources/log4j-surefire.properties | 0
hudi-spark-datasource/pom.xml | 2 +-
hudi-sync/hudi-adb-sync/pom.xml | 14 +-
.../hudi/sync/adb/AbstractAdbSyncHoodieClient.java | 128 ---
.../org/apache/hudi/sync/adb/AdbSyncConfig.java | 223 +++--
.../java/org/apache/hudi/sync/adb/AdbSyncTool.java | 169 ++--
.../apache/hudi/sync/adb/HoodieAdbJdbcClient.java | 160 ++--
.../apache/hudi/sync/adb/TestAdbSyncConfig.java | 91 +-
hudi-sync/hudi-datahub-sync/pom.xml | 14 +-
.../hudi/sync/datahub/DataHubSyncClient.java | 83 +-
.../apache/hudi/sync/datahub/DataHubSyncTool.java | 42 +-
.../sync/datahub/config/DataHubSyncConfig.java | 70 +-
.../config/HoodieDataHubDatasetIdentifier.java | 13 +-
.../sync/datahub/config/TestDataHubSyncConfig.java | 26 +-
hudi-sync/hudi-hive-sync/pom.xml | 14 +-
.../hudi/hive/AbstractHiveSyncHoodieClient.java | 142 ----
.../hive/HiveStylePartitionValueExtractor.java | 2 +
.../java/org/apache/hudi/hive/HiveSyncConfig.java | 368 +++------
.../org/apache/hudi/hive/HiveSyncConfigHolder.java | 128 +++
.../java/org/apache/hudi/hive/HiveSyncTool.java | 244 +++---
...ieHiveClient.java => HoodieHiveSyncClient.java} | 161 ++--
.../hudi/hive/MultiPartKeysValueExtractor.java | 14 +-
.../apache/hudi/hive/NonPartitionedExtractor.java | 15 +-
.../SlashEncodedDayPartitionValueExtractor.java | 15 +-
.../SlashEncodedHourPartitionValueExtractor.java | 15 +-
.../java/org/apache/hudi/hive/ddl/DDLExecutor.java | 4 +-
.../org/apache/hudi/hive/ddl/HMSDDLExecutor.java | 83 +-
.../apache/hudi/hive/ddl/HiveQueryDDLExecutor.java | 34 +-
.../org/apache/hudi/hive/ddl/JDBCExecutor.java | 35 +-
.../hudi/hive/ddl/QueryBasedDDLExecutor.java | 64 +-
.../hive/replication/GlobalHiveSyncConfig.java | 56 +-
.../hudi/hive/replication/GlobalHiveSyncTool.java | 40 +-
...Config.java => HiveSyncGlobalCommitParams.java} | 63 +-
.../hive/replication/HiveSyncGlobalCommitTool.java | 48 +-
.../hive/replication/ReplicationStateSync.java | 24 +-
.../lock}/HiveMetastoreBasedLockProvider.java | 17 +-
.../apache/hudi/hive/util/HivePartitionUtil.java | 30 +-
.../org/apache/hudi/hive/util/HiveSchemaUtil.java | 28 +-
.../hudi/hive/TestHiveSyncGlobalCommitTool.java | 128 ---
.../org/apache/hudi/hive/TestHiveSyncTool.java | 295 +++----
.../TestHiveMetastoreBasedLockProvider.java | 2 +-
.../replication/TestHiveSyncGlobalCommitTool.java | 154 ++++
.../testutils/HiveSyncFunctionalTestHarness.java | 54 +-
.../{TestCluster.java => HiveTestCluster.java} | 64 +-
.../hudi/hive/testutils/HiveTestService.java | 167 ++--
.../apache/hudi/hive/testutils/HiveTestUtil.java | 43 +-
hudi-sync/hudi-sync-common/pom.xml | 2 +-
.../hudi/sync/common/AbstractSyncHoodieClient.java | 276 -------
.../hudi/sync/common/HoodieMetaSyncOperations.java | 196 +++++
.../apache/hudi/sync/common/HoodieSyncClient.java | 161 ++++
.../apache/hudi/sync/common/HoodieSyncConfig.java | 211 ++---
.../apache/hudi/sync/common/HoodieSyncTool.java | 62 ++
.../apache/hudi/sync/common/model/FieldSchema.java | 83 ++
.../hudi/sync/common/model/PartitionEvent.java | 50 ++
.../common/model}/PartitionValueExtractor.java | 15 +-
.../apache/hudi/sync/common/util/ConfigUtils.java | 20 +-
.../SparkDataSourceTableUtils.java} | 52 +-
.../hudi/sync/common/util/SyncUtilHelpers.java | 65 +-
.../hudi/sync/common/TestHoodieSyncConfig.java | 131 +++
.../hudi/sync/common/util/TestConfigUtils.java | 68 ++
.../hudi/sync/common/util/TestSyncUtilHelpers.java | 75 +-
hudi-sync/pom.xml | 2 +-
hudi-timeline-service/pom.xml | 14 +-
.../hudi/timeline/service/RequestHandler.java | 23 +-
.../hudi/timeline/service/TimelineService.java | 10 -
hudi-utilities/pom.xml | 14 +-
.../hudi/utilities/HoodieDropPartitionsTool.java | 400 +++++++++
.../org/apache/hudi/utilities/HoodieIndexer.java | 36 +-
.../utilities/deltastreamer/BootstrapExecutor.java | 15 +-
.../hudi/utilities/deltastreamer/DeltaSync.java | 12 +-
.../utilities/transform/AWSDmsTransformer.java | 2 +-
.../hudi/utilities/TestHiveIncrementalPuller.java | 48 +-
.../apache/hudi/utilities/TestHoodieIndexer.java | 95 ++-
.../functional/HoodieDeltaStreamerTestBase.java | 29 +-
.../TestAWSDatabaseMigrationServiceSource.java | 2 +-
.../functional/TestHoodieDeltaStreamer.java | 37 +-
.../TestHoodieDeltaStreamerWithMultiWriter.java | 18 +-
.../utilities/sources/TestHoodieIncrSource.java | 7 +-
.../utilities/testutils/UtilitiesTestBase.java | 61 +-
packaging/hudi-aws-bundle/pom.xml | 2 +-
packaging/hudi-datahub-sync-bundle/pom.xml | 2 +-
packaging/hudi-flink-bundle/pom.xml | 14 +-
packaging/hudi-gcp-bundle/pom.xml | 2 +-
packaging/hudi-hadoop-mr-bundle/pom.xml | 2 +-
packaging/hudi-hive-sync-bundle/pom.xml | 2 +-
packaging/hudi-integ-test-bundle/pom.xml | 16 +-
packaging/hudi-kafka-connect-bundle/pom.xml | 16 +-
packaging/hudi-presto-bundle/pom.xml | 4 +-
packaging/hudi-spark-bundle/pom.xml | 2 +-
packaging/hudi-timeline-server-bundle/pom.xml | 14 +-
packaging/hudi-trino-bundle/pom.xml | 2 +-
packaging/hudi-utilities-bundle/pom.xml | 2 +-
packaging/hudi-utilities-slim-bundle/pom.xml | 2 +-
pom.xml | 161 +++-
rfc/README.md | 18 +-
rfc/rfc-51/arch.jpg | Bin 0 -> 166734 bytes
rfc/rfc-51/points.jpg | Bin 0 -> 250122 bytes
rfc/rfc-51/query_cdc_on_mor.jpg | Bin 0 -> 170119 bytes
rfc/rfc-51/read_cdc_log_file.jpg | Bin 0 -> 210750 bytes
rfc/rfc-51/rfc-51.md | 281 +++++++
rfc/rfc-51/scenario-definition.jpg | Bin 0 -> 175342 bytes
rfc/rfc-55/hudi-sync-class-diagram.png | Bin 0 -> 418052 bytes
rfc/rfc-55/hudi-sync-flows.png | Bin 0 -> 139211 bytes
rfc/rfc-55/rfc-55.md | 173 ++++
750 files changed, 25608 insertions(+), 9710 deletions(-)
copy hudi-integ-test/src/test/resources/unit-test-cow-dag.yaml =>
docker/demo/config/test-suite/deltastreamer-hive-sync-presto.yaml (72%)
rename
hudi-cli/src/main/java/org/apache/hudi/cli/commands/{HoodieSyncCommand.java =>
HoodieSyncValidateCommand.java} (98%)
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/FileSystemBasedLockProvider.java
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieArchivalConfig.java
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieCleanConfig.java
copy
hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/NonPartitionedExtractor.java
=>
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/index/hbase/RebalancedSparkHoodieHBaseIndex.java
(55%)
delete mode 100644
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowCreateHandleWithoutMetaFields.java
delete mode 100644
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/RowKeyGeneratorHelper.java
create mode 100644
hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/unsafe/UTF8StringBuilder.java
create mode 100644
hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/HoodieCatalystPlansUtils.scala
create mode 100644
hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/HoodieUnsafeRowUtils.scala
create mode 100644
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestFileBasedLockProvider.java
delete mode 100644
hudi-client/hudi-spark-client/src/test/scala/org/apache/hudi/keygen/TestRowGeneratorHelper.scala
create mode 100644
hudi-client/hudi-spark-client/src/test/scala/org/apache/spark/sql/TestHoodieUnsafeRowUtils.scala
create mode 100644
hudi-common/src/main/java/org/apache/hudi/common/data/HoodieBaseListData.java
delete mode 100644
hudi-common/src/main/java/org/apache/hudi/common/data/HoodieList.java
create mode 100644
hudi-common/src/main/java/org/apache/hudi/common/data/HoodieListData.java
create mode 100644
hudi-common/src/main/java/org/apache/hudi/common/data/HoodieListPairData.java
delete mode 100644
hudi-common/src/main/java/org/apache/hudi/common/data/HoodieMapPair.java
copy {hudi-spark-datasource/hudi-spark/src/main/java/org/apache/hudi/payload
=>
hudi-common/src/main/java/org/apache/hudi/common/model}/AWSDmsAvroPayload.java
(97%)
create mode 100644
hudi-common/src/main/java/org/apache/hudi/common/util/Either.java
copy
hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/NonPartitionedExtractor.java
=>
hudi-common/src/main/java/org/apache/hudi/exception/HoodieCatalogException.java
(65%)
rename
hudi-common/src/test/java/org/apache/hudi/common/data/{TestHoodieList.java =>
TestHoodieListData.java} (64%)
rename
hudi-common/src/test/java/org/apache/hudi/common/data/{TestHoodieMapPair.java
=> TestHoodieListDataPairData.java} (75%)
rename {hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/payload
=>
hudi-common/src/test/java/org/apache/hudi/common/model}/TestAWSDmsAvroPayload.java
(97%)
create mode 100644
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/compact/strategy/CompactionPlanStrategies.java
copy
hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/NonPartitionedExtractor.java
=>
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/compact/strategy/CompactionPlanStrategy.java
(60%)
create mode 100644
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/stats/ColumnStatsIndices.java
create mode 100644
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/stats/ExpressionEvaluator.java
create mode 100644
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/catalog/HiveSchemaUtils.java
create mode 100644
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/catalog/HoodieCatalogUtil.java
create mode 100644
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/catalog/HoodieHiveCatalog.java
create mode 100644
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/catalog/TypeInfoLogicalTypeVisitor.java
create mode 100644
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/ExpressionUtils.java
create mode 100644
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/HoodiePipeline.java
create mode 100644
hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/compact/TestCompactionPlanStrategy.java
create mode 100644
hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/stats/TestColumnStatsIndices.java
create mode 100644
hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/stats/TestExpressionEvaluator.java
create mode 100644
hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/catalog/HoodieCatalogTestUtils.java
create mode 100644
hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/catalog/TestHoodieCatalogFactory.java
create mode 100644
hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/catalog/TestHoodieHiveCatalog.java
create mode 100644
hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestAvroSchemaConverter.java
create mode 100644
hudi-flink-datasource/hudi-flink/src/test/resources/hive-site.xml
create mode 100644
hudi-flink-datasource/hudi-flink/src/test/resources/test-catalog-factory-conf/hive-site.xml
copy
hudi-spark-datasource/hudi-spark3/src/main/scala/org/apache/spark/sql/hudi/catalog/TableCreationMode.java
=>
hudi-flink-datasource/hudi-flink1.13.x/src/main/java/org/apache/hudi/adapter/DataStreamScanProviderAdapter.java
(72%)
copy
hudi-spark-datasource/hudi-spark3/src/main/scala/org/apache/spark/sql/hudi/catalog/TableCreationMode.java
=>
hudi-flink-datasource/hudi-flink1.13.x/src/main/java/org/apache/hudi/adapter/DataStreamSinkProviderAdapter.java
(72%)
copy hudi-flink-datasource/{hudi-flink =>
hudi-flink1.13.x}/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java
(100%)
copy hudi-flink-datasource/{hudi-flink =>
hudi-flink1.13.x}/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayVector.java
(100%)
copy hudi-flink-datasource/{hudi-flink =>
hudi-flink1.13.x}/src/main/java/org/apache/hudi/table/format/cow/vector/HeapMapColumnVector.java
(100%)
copy hudi-flink-datasource/{hudi-flink =>
hudi-flink1.13.x}/src/main/java/org/apache/hudi/table/format/cow/vector/HeapRowColumnVector.java
(100%)
copy hudi-flink-datasource/{hudi-flink =>
hudi-flink1.13.x}/src/main/java/org/apache/hudi/table/format/cow/vector/ParquetDecimalVector.java
(100%)
copy hudi-flink-datasource/{hudi-flink =>
hudi-flink1.13.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/AbstractColumnReader.java
(100%)
copy hudi-flink-datasource/{hudi-flink =>
hudi-flink1.13.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayColumnReader.java
(100%)
copy hudi-flink-datasource/{hudi-flink =>
hudi-flink1.13.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/BaseVectorizedColumnReader.java
(100%)
copy hudi-flink-datasource/{hudi-flink =>
hudi-flink1.13.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/FixedLenBytesColumnReader.java
(100%)
copy hudi-flink-datasource/{hudi-flink =>
hudi-flink1.13.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/Int64TimestampColumnReader.java
(100%)
copy hudi-flink-datasource/{hudi-flink =>
hudi-flink1.13.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/MapColumnReader.java
(100%)
copy hudi-flink-datasource/{hudi-flink =>
hudi-flink1.13.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java
(100%)
copy hudi-flink-datasource/{hudi-flink =>
hudi-flink1.13.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetDataColumnReader.java
(100%)
copy hudi-flink-datasource/{hudi-flink =>
hudi-flink1.13.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetDataColumnReaderFactory.java
(100%)
copy hudi-flink-datasource/{hudi-flink =>
hudi-flink1.13.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/RowColumnReader.java
(100%)
copy hudi-flink-datasource/{hudi-flink =>
hudi-flink1.13.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/RunLengthDecoder.java
(98%)
copy
hudi-spark-datasource/hudi-spark3/src/main/scala/org/apache/spark/sql/hudi/catalog/TableCreationMode.java
=>
hudi-flink-datasource/hudi-flink1.14.x/src/main/java/org/apache/hudi/adapter/DataStreamScanProviderAdapter.java
(72%)
copy
hudi-spark-datasource/hudi-spark3/src/main/scala/org/apache/spark/sql/hudi/catalog/TableCreationMode.java
=>
hudi-flink-datasource/hudi-flink1.14.x/src/main/java/org/apache/hudi/adapter/DataStreamSinkProviderAdapter.java
(72%)
copy hudi-flink-datasource/{hudi-flink =>
hudi-flink1.14.x}/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java
(100%)
copy hudi-flink-datasource/{hudi-flink =>
hudi-flink1.14.x}/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayVector.java
(100%)
copy hudi-flink-datasource/{hudi-flink =>
hudi-flink1.14.x}/src/main/java/org/apache/hudi/table/format/cow/vector/HeapMapColumnVector.java
(100%)
copy hudi-flink-datasource/{hudi-flink =>
hudi-flink1.14.x}/src/main/java/org/apache/hudi/table/format/cow/vector/HeapRowColumnVector.java
(100%)
copy hudi-flink-datasource/{hudi-flink =>
hudi-flink1.14.x}/src/main/java/org/apache/hudi/table/format/cow/vector/ParquetDecimalVector.java
(100%)
copy hudi-flink-datasource/{hudi-flink =>
hudi-flink1.14.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/AbstractColumnReader.java
(100%)
copy hudi-flink-datasource/{hudi-flink =>
hudi-flink1.14.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayColumnReader.java
(100%)
copy hudi-flink-datasource/{hudi-flink =>
hudi-flink1.14.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/BaseVectorizedColumnReader.java
(100%)
copy hudi-flink-datasource/{hudi-flink =>
hudi-flink1.14.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/FixedLenBytesColumnReader.java
(100%)
copy hudi-flink-datasource/{hudi-flink =>
hudi-flink1.14.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/Int64TimestampColumnReader.java
(100%)
copy hudi-flink-datasource/{hudi-flink =>
hudi-flink1.14.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/MapColumnReader.java
(100%)
copy hudi-flink-datasource/{hudi-flink =>
hudi-flink1.14.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java
(100%)
copy hudi-flink-datasource/{hudi-flink =>
hudi-flink1.14.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetDataColumnReader.java
(100%)
copy hudi-flink-datasource/{hudi-flink =>
hudi-flink1.14.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetDataColumnReaderFactory.java
(100%)
copy hudi-flink-datasource/{hudi-flink =>
hudi-flink1.14.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/RowColumnReader.java
(100%)
copy hudi-flink-datasource/{hudi-flink =>
hudi-flink1.14.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/RunLengthDecoder.java
(98%)
copy hudi-flink-datasource/{hudi-flink1.14.x => hudi-flink1.15.x}/pom.xml (75%)
copy hudi-flink-datasource/{hudi-flink1.14.x =>
hudi-flink1.15.x}/src/main/java/org/apache/hudi/adapter/AbstractStreamOperatorAdapter.java
(100%)
copy hudi-flink-datasource/{hudi-flink1.14.x =>
hudi-flink1.15.x}/src/main/java/org/apache/hudi/adapter/AbstractStreamOperatorFactoryAdapter.java
(100%)
copy
hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/NonPartitionedExtractor.java
=>
hudi-flink-datasource/hudi-flink1.15.x/src/main/java/org/apache/hudi/adapter/DataStreamScanProviderAdapter.java
(51%)
copy
hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/NonPartitionedExtractor.java
=>
hudi-flink-datasource/hudi-flink1.15.x/src/main/java/org/apache/hudi/adapter/DataStreamSinkProviderAdapter.java
(51%)
copy hudi-flink-datasource/{hudi-flink1.14.x =>
hudi-flink1.15.x}/src/main/java/org/apache/hudi/adapter/MailboxExecutorAdapter.java
(100%)
copy hudi-flink-datasource/{hudi-flink1.14.x =>
hudi-flink1.15.x}/src/main/java/org/apache/hudi/adapter/RateLimiterAdapter.java
(100%)
copy hudi-flink-datasource/{hudi-flink1.14.x =>
hudi-flink1.15.x}/src/main/java/org/apache/hudi/adapter/Utils.java (75%)
rename hudi-flink-datasource/{hudi-flink =>
hudi-flink1.15.x}/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java
(95%)
rename hudi-flink-datasource/{hudi-flink =>
hudi-flink1.15.x}/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayVector.java
(83%)
rename hudi-flink-datasource/{hudi-flink =>
hudi-flink1.15.x}/src/main/java/org/apache/hudi/table/format/cow/vector/HeapMapColumnVector.java
(85%)
rename hudi-flink-datasource/{hudi-flink =>
hudi-flink1.15.x}/src/main/java/org/apache/hudi/table/format/cow/vector/HeapRowColumnVector.java
(79%)
rename hudi-flink-datasource/{hudi-flink =>
hudi-flink1.15.x}/src/main/java/org/apache/hudi/table/format/cow/vector/ParquetDecimalVector.java
(89%)
rename hudi-flink-datasource/{hudi-flink =>
hudi-flink1.15.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/AbstractColumnReader.java
(98%)
rename hudi-flink-datasource/{hudi-flink =>
hudi-flink1.15.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayColumnReader.java
(95%)
rename hudi-flink-datasource/{hudi-flink =>
hudi-flink1.15.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/BaseVectorizedColumnReader.java
(99%)
rename hudi-flink-datasource/{hudi-flink =>
hudi-flink1.15.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/FixedLenBytesColumnReader.java
(92%)
rename hudi-flink-datasource/{hudi-flink =>
hudi-flink1.15.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/Int64TimestampColumnReader.java
(96%)
rename hudi-flink-datasource/{hudi-flink =>
hudi-flink1.15.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/MapColumnReader.java
(95%)
rename hudi-flink-datasource/{hudi-flink =>
hudi-flink1.15.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java
(97%)
rename hudi-flink-datasource/{hudi-flink =>
hudi-flink1.15.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetDataColumnReader.java
(100%)
rename hudi-flink-datasource/{hudi-flink =>
hudi-flink1.15.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetDataColumnReaderFactory.java
(100%)
rename hudi-flink-datasource/{hudi-flink =>
hudi-flink1.15.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/RowColumnReader.java
(95%)
rename hudi-flink-datasource/{hudi-flink =>
hudi-flink1.15.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/RunLengthDecoder.java
(96%)
copy hudi-flink-datasource/{hudi-flink1.14.x =>
hudi-flink1.15.x}/src/test/java/org/apache/hudi/adapter/OutputAdapter.java
(100%)
copy
hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/util/JFunction.scala
=>
hudi-flink-datasource/hudi-flink1.15.x/src/test/java/org/apache/hudi/adapter/StateInitializationContextAdapter.java
(67%)
copy hudi-flink-datasource/{hudi-flink1.14.x =>
hudi-flink1.15.x}/src/test/java/org/apache/hudi/adapter/StreamingRuntimeContextAdapter.java
(100%)
copy hudi-flink-datasource/{hudi-flink1.14.x =>
hudi-flink1.15.x}/src/test/java/org/apache/hudi/adapter/TestTableEnvs.java
(100%)
create mode 100644
hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestHoodieCopyOnWriteTableInputFormat.java
create mode 100644
hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/realtime/TestHoodieMergeOnReadTableInputFormat.java
create mode 100644
hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/nodes/BaseQueryNode.java
create mode 100644
hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/nodes/PrestoQueryNode.java
create mode 100644
hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/nodes/TrinoQueryNode.java
delete mode 100644
hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/HoodieDatasetBulkInsertHelper.java
delete mode 100644
hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/SparkRowWriteHelper.java
create mode 100644
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieCatalystUtils.scala
create mode 100644
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieDatasetBulkInsertHelper.scala
delete mode 100644
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieDatasetUtils.scala
rename
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/{HoodieUnsafeRDDUtils.scala
=> HoodieUnsafeUtils.scala} (50%)
copy hudi-common/src/main/java/org/apache/hudi/TypeUtils.java =>
hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/DeDupeType.scala
(66%)
create mode 100644
hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/DedupeSparkJob.scala
create mode 100644
hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/SparkHelpers.scala
create mode 100644
hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/CopyToTableProcedure.scala
create mode 100644
hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/CreateMetadataTableProcedure.scala
rename
hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/{CreateSavepointsProcedure.scala
=> CreateSavepointProcedure.scala} (88%)
copy
hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/{DeleteMarkerProcedure.scala
=> DeleteMetadataTableProcedure.scala} (51%)
rename
hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/{DeleteSavepointsProcedure.scala
=> DeleteSavepointProcedure.scala} (89%)
create mode 100644
hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/InitMetadataTableProcedure.scala
create mode 100644
hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairAddpartitionmetaProcedure.scala
create mode 100644
hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairCorruptedCleanFilesProcedure.scala
create mode 100644
hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairDeduplicateProcedure.scala
create mode 100644
hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairMigratePartitionMetaProcedure.scala
create mode 100644
hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairOverwriteHoodiePropsProcedure.scala
rename
hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/{RollbackSavepointsProcedure.scala
=> RollbackToSavepointProcedure.scala} (89%)
create mode 100644
hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RunCleanProcedure.scala
create mode 100644
hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowFsPathDetailProcedure.scala
create mode 100644
hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowMetadataTableFilesProcedure.scala
create mode 100644
hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowMetadataTablePartitionsProcedure.scala
copy
hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/{DeleteMarkerProcedure.scala
=> ShowMetadataTableStatsProcedure.scala} (50%)
create mode 100644
hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/UpgradeOrDowngradeProcedure.scala
create mode 100644
hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ValidateHoodieSyncProcedure.scala
create mode 100644
hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ValidateMetadataTableFilesProcedure.scala
copy {hudi-cli =>
hudi-spark-datasource/hudi-spark}/src/test/resources/table-config.properties
(100%)
copy hudi-common/src/main/java/org/apache/hudi/TypeUtils.java =>
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/ScalaAssertionSupport.scala
(56%)
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieRelations.scala
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/avro/TestAvroSerDe.scala
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/avro/TestSchemaConverters.scala
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestNestedSchemaPruningOptimization.scala
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestCleanProcedure.scala
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestCopyToTableProcedure.scala
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestMetadataProcedure.scala
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestRepairsProcedure.scala
copy
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/{TestExportInstantsProcedure.scala
=> TestShowFsPathDetailProcedure.scala} (76%)
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestUpgradeOrDowngradeProcedure.scala
copy
hudi-spark-datasource/{hudi-spark-common/src/main/scala/org/apache/hudi/HoodieFileScanRDD.scala
=> hudi-spark2/src/main/scala/org/apache/hudi/Spark2HoodieFileScanRDD.scala}
(71%)
create mode 100644
hudi-spark-datasource/hudi-spark2/src/main/scala/org/apache/spark/sql/HoodieSpark2CatalystPlanUtils.scala
create mode 100644
hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/HoodieSpark3CatalystPlanUtils.scala
delete mode 100644
hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/HoodieSpark3SqlUtils.scala
copy
hudi-spark-datasource/{hudi-spark-common/src/main/scala/org/apache/hudi/HoodieFileScanRDD.scala
=>
hudi-spark3.1.x/src/main/scala/org/apache/hudi/Spark31HoodieFileScanRDD.scala}
(71%)
rename
hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/{HoodieSpark3_1CatalystExpressionUtils.scala
=> HoodieSpark31CatalystExpressionUtils.scala} (98%)
copy
hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/NonPartitionedExtractor.java
=>
hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/HoodieSpark31CatalystPlanUtils.scala
(61%)
create mode 100644
hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/Spark31NestedSchemaPruning.scala
rename
hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/hudi/{ResolveHudiAlterTableCommand312.scala
=> Spark312ResolveHudiAlterTableCommand.scala} (99%)
rename
hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/parser/{HoodieSpark312ExtendedSqlParser.scala
=> HoodieSpark3_1ExtendedSqlParser.scala} (95%)
copy hudi-spark-datasource/{hudi-spark3 => hudi-spark3.2.x}/pom.xml (96%)
copy hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/main/antlr4/imports/SqlBase.g4 (100%)
copy hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/main/antlr4/org/apache/hudi/spark/sql/parser/HoodieSqlBase.g4
(100%)
copy hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/main/java/org/apache/spark/sql/execution/datasources/parquet/Spark32HoodieVectorizedParquetRecordReader.java
(100%)
copy hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
(100%)
copy
hudi-spark-datasource/{hudi-spark-common/src/main/scala/org/apache/hudi/HoodieFileScanRDD.scala
=>
hudi-spark3.2.x/src/main/scala/org/apache/hudi/Spark32HoodieFileScanRDD.scala}
(71%)
copy hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/main/scala/org/apache/hudi/Spark3DefaultSource.scala (100%)
copy
hudi-spark-datasource/{hudi-spark3/src/main/scala/org/apache/spark/sql/HoodieSpark3_2CatalystExpressionUtils.scala
=>
hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/HoodieSpark32CatalystExpressionUtils.scala}
(98%)
copy hudi-common/src/main/java/org/apache/hudi/TypeUtils.java =>
hudi-spark-datasource/hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/HoodieSpark32CatalystPlanUtils.scala
(55%)
rename hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/adapter/Spark3_2Adapter.scala
(54%)
copy hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
(99%)
copy hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala
(100%)
rename hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala (100%)
copy hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_2AvroDeserializer.scala
(100%)
copy hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_2AvroSerializer.scala
(100%)
copy hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/TimeTravelRelation.scala
(100%)
copy hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/connector/catalog/HoodieIdentifier.scala
(100%)
create mode 100644
hudi-spark-datasource/hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/execution/datasources/Spark32NestedSchemaPruning.scala
copy hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark32DataSourceUtils.scala
(100%)
copy hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark32HoodieParquetFileFormat.scala
(100%)
copy
hudi-spark-datasource/{hudi-spark3/src/main/scala/org/apache/spark/sql/hudi/ResolveHudiAlterTableCommandSpark32.scala
=>
hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/hudi/Spark32ResolveHudiAlterTableCommand.scala}
(98%)
copy hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSpark3Analysis.scala
(100%)
copy hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/hudi/catalog/BasicStagedTable.scala
(100%)
copy hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/hudi/catalog/HoodieCatalog.scala
(93%)
copy hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/hudi/catalog/HoodieInternalV2Table.scala
(100%)
copy hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/hudi/catalog/HoodieStagedTable.scala
(100%)
copy hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/hudi/catalog/TableCreationMode.java
(100%)
copy hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/hudi/command/AlterTableCommand.scala
(100%)
copy hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_2ExtendedSqlAstBuilder.scala
(99%)
copy hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_2ExtendedSqlParser.scala
(88%)
copy hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/test/java/org/apache/hudi/spark3/internal/TestHoodieBulkInsertDataInternalWriter.java
(100%)
copy hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/test/java/org/apache/hudi/spark3/internal/TestHoodieDataSourceInternalBatchWrite.java
(100%)
copy hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/test/java/org/apache/hudi/spark3/internal/TestReflectUtil.java
(100%)
rename hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/test/resources/log4j-surefire-quiet.properties (100%)
rename hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/test/resources/log4j-surefire.properties (100%)
rename hudi-spark-datasource/{hudi-spark3 => hudi-spark3.3.x}/pom.xml (90%)
rename hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.3.x}/src/main/antlr4/imports/SqlBase.g4 (100%)
rename hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.3.x}/src/main/antlr4/org/apache/hudi/spark/sql/parser/HoodieSqlBase.g4
(100%)
rename
hudi-spark-datasource/{hudi-spark3/src/main/java/org/apache/spark/sql/execution/datasources/parquet/Spark32HoodieVectorizedParquetRecordReader.java
=>
hudi-spark3.3.x/src/main/java/org/apache/spark/sql/execution/datasources/parquet/Spark33HoodieVectorizedParquetRecordReader.java}
(98%)
rename hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.3.x}/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
(100%)
rename
hudi-spark-datasource/{hudi-spark-common/src/main/scala/org/apache/hudi/HoodieFileScanRDD.scala
=>
hudi-spark3.3.x/src/main/scala/org/apache/hudi/Spark33HoodieFileScanRDD.scala}
(67%)
rename hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.3.x}/src/main/scala/org/apache/hudi/Spark3DefaultSource.scala (100%)
rename
hudi-spark-datasource/{hudi-spark3/src/main/scala/org/apache/spark/sql/HoodieSpark3_2CatalystExpressionUtils.scala
=>
hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/HoodieSpark33CatalystExpressionUtils.scala}
(94%)
rename hudi-common/src/main/java/org/apache/hudi/TypeUtils.java =>
hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/HoodieSpark33CatalystPlanUtils.scala
(55%)
create mode 100644
hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/adapter/Spark3_3Adapter.scala
rename hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.3.x}/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
(82%)
rename hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.3.x}/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala
(73%)
create mode 100644
hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala
rename
hudi-spark-datasource/{hudi-spark3/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_2AvroDeserializer.scala
=>
hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_3AvroDeserializer.scala}
(95%)
rename
hudi-spark-datasource/{hudi-spark3/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_2AvroSerializer.scala
=>
hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_3AvroSerializer.scala}
(94%)
rename hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.3.x}/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/TimeTravelRelation.scala
(100%)
rename hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.3.x}/src/main/scala/org/apache/spark/sql/connector/catalog/HoodieIdentifier.scala
(100%)
create mode 100644
hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/Spark33NestedSchemaPruning.scala
rename
hudi-spark-datasource/{hudi-spark3/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark32DataSourceUtils.scala
=>
hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark33DataSourceUtils.scala}
(99%)
rename
hudi-spark-datasource/{hudi-spark3/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark32HoodieParquetFileFormat.scala
=>
hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark33HoodieParquetFileFormat.scala}
(97%)
rename
hudi-spark-datasource/{hudi-spark3/src/main/scala/org/apache/spark/sql/hudi/ResolveHudiAlterTableCommandSpark32.scala
=>
hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/hudi/Spark33ResolveHudiAlterTableCommand.scala}
(95%)
rename hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.3.x}/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSpark3Analysis.scala
(94%)
rename hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.3.x}/src/main/scala/org/apache/spark/sql/hudi/catalog/BasicStagedTable.scala
(100%)
rename hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.3.x}/src/main/scala/org/apache/spark/sql/hudi/catalog/HoodieCatalog.scala
(93%)
rename hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.3.x}/src/main/scala/org/apache/spark/sql/hudi/catalog/HoodieInternalV2Table.scala
(100%)
rename hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.3.x}/src/main/scala/org/apache/spark/sql/hudi/catalog/HoodieStagedTable.scala
(100%)
rename hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.3.x}/src/main/scala/org/apache/spark/sql/hudi/catalog/TableCreationMode.java
(100%)
rename hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.3.x}/src/main/scala/org/apache/spark/sql/hudi/command/AlterTableCommand.scala
(100%)
rename
hudi-spark-datasource/{hudi-spark3/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_2ExtendedSqlAstBuilder.scala
=>
hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_3ExtendedSqlAstBuilder.scala}
(99%)
rename
hudi-spark-datasource/{hudi-spark3/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_2ExtendedSqlParser.scala
=>
hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_3ExtendedSqlParser.scala}
(84%)
rename hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.3.x}/src/test/java/org/apache/hudi/spark3/internal/TestHoodieBulkInsertDataInternalWriter.java
(100%)
rename hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.3.x}/src/test/java/org/apache/hudi/spark3/internal/TestHoodieDataSourceInternalBatchWrite.java
(100%)
rename hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.3.x}/src/test/java/org/apache/hudi/spark3/internal/TestReflectUtil.java
(100%)
copy {hudi-common =>
hudi-spark-datasource/hudi-spark3.3.x}/src/test/resources/log4j-surefire-quiet.properties
(100%)
copy {hudi-client/hudi-client-common =>
hudi-spark-datasource/hudi-spark3.3.x}/src/test/resources/log4j-surefire.properties
(100%)
delete mode 100644
hudi-sync/hudi-adb-sync/src/main/java/org/apache/hudi/sync/adb/AbstractAdbSyncHoodieClient.java
delete mode 100644
hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/AbstractHiveSyncHoodieClient.java
create mode 100644
hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncConfigHolder.java
rename
hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/{HoodieHiveClient.java
=> HoodieHiveSyncClient.java} (64%)
rename
hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/replication/{HiveSyncGlobalCommitConfig.java
=> HiveSyncGlobalCommitParams.java} (65%)
rename hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/{ =>
transaction/lock}/HiveMetastoreBasedLockProvider.java (95%)
delete mode 100644
hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncGlobalCommitTool.java
create mode 100644
hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/replication/TestHiveSyncGlobalCommitTool.java
rename
hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/{TestCluster.java
=> HiveTestCluster.java} (86%)
delete mode 100644
hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/AbstractSyncHoodieClient.java
create mode 100644
hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/HoodieMetaSyncOperations.java
create mode 100644
hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/HoodieSyncClient.java
create mode 100644
hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/HoodieSyncTool.java
create mode 100644
hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/model/FieldSchema.java
create mode 100644
hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/model/PartitionEvent.java
rename hudi-sync/{hudi-hive-sync/src/main/java/org/apache/hudi/hive =>
hudi-sync-common/src/main/java/org/apache/hudi/sync/common/model}/PartitionValueExtractor.java
(69%)
rename
hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/{AbstractSyncTool.java
=> util/SparkDataSourceTableUtils.java} (69%)
create mode 100644
hudi-sync/hudi-sync-common/src/test/java/org/apache/hudi/sync/common/TestHoodieSyncConfig.java
create mode 100644
hudi-sync/hudi-sync-common/src/test/java/org/apache/hudi/sync/common/util/TestConfigUtils.java
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieDropPartitionsTool.java
create mode 100644 rfc/rfc-51/arch.jpg
create mode 100644 rfc/rfc-51/points.jpg
create mode 100644 rfc/rfc-51/query_cdc_on_mor.jpg
create mode 100644 rfc/rfc-51/read_cdc_log_file.jpg
create mode 100644 rfc/rfc-51/rfc-51.md
create mode 100644 rfc/rfc-51/scenario-definition.jpg
create mode 100644 rfc/rfc-55/hudi-sync-class-diagram.png
create mode 100644 rfc/rfc-55/hudi-sync-flows.png
create mode 100644 rfc/rfc-55/rfc-55.md
