This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch dependabot/maven/xalan-xalan-2.7.3
in repository https://gitbox.apache.org/repos/asf/hudi.git
discard 6878ff14acf Bump xalan from 2.7.2 to 2.7.3
add 66e838c8f18 [MINOR] Remove redundant advanced config marking (#8600)
add e6ee5a83a13 [HUDI-6185] Too many logs in the ExternalSpillableMap
(#8649)
add 21fedff40bf [HUDI-6174] Fixing flaky tests in HoodieDeltastreamer
(#8646)
add ec9cb3c4464 [HUDI-6095] Refactor the judgment condition of
WorkloadProfile (#8491)
add 20938c30b16 [HUDI-6120] Add some notion for
fetchAllLogsMergedFileSlice (#8529)
add 79412ed2a63 [HUDI-6186] Fix lock identity in InProcessLockProvider
(#8658)
add 6ef00d147c1 [HUDI-5816] List all partitions as the fallback mechanism
in Hive and Glue Sync (#8388)
add 6cdc1f583ef [HUDI-5895] Remove bootstrap key generator configs (#8557)
add aa3ef4ba97c [MINOR] Use try with resource to close stream (#8596)
add c1c80eeec05 [MINOR] Add script to build bundle validation image with
Spark 3.3.2 (#8655)
add 545a26222da [HUDI-6147] Deltastreamer finish failed compaction before
ingestion (#8589)
add 45b79362cc4 [HUDI-6117] Parallelize the initial creation of file
groups for a new MDT partition. (#8527)
add 76a3e0ea863 [MINOR][DOC][hudi-metaserver] Fix typos in README.md
(#8536)
add a71d3e49fe6 [MINOR] Claim RFC-69 for Hudi 1.x (#8671)
add 8db07a8b75f [HUDI-6040] Stop writing and reading compaction plans from
.aux folder (#8385)
add a7942003b9a [MINOR] Fix RunBootstrapProcedure doesn't has database
default value (#8660)
add a34067826c2 [HUDI-5514][HUDI-5574][HUDI-5604][HUDI-5535] Adding auto
generation of record keys support to Hudi/Spark (#8107)
add 11078f7b28f [HUDI-6047] Clustering operation on consistent hashing
index resulting in duplicate data (#8503)
add 93794743d71 [HUDI-6196] Keep compatibility for old version archival
instants without ACTION_STATE field (#8607)
add 92e52dca1ae [HUDI-6170] Use correct zone id while calculating
earliestTimeToRetain (#8631)
add 692d367f3ce [HUDI-6007] Add log files to savepoint metadata (#8364)
add 15cd0522d71 [HUDI-5308] Hive3 query returns null when the where clause
has a partition field (#7355)
add 31900cd77b7 [HUDI-6189] Reuse the `timeline` supplied in predefined
variable or parameter (#8664)
add 5bab66498c8 [HUDI-6122] Unify call procedure options (#8537)
add 65172d3d66a [HUDI-5868] Make hudi-spark compatible against Spark 3.3.2
(#8082)
add 9963b50ee17 [HUDI-6105] Support partial insert in MERGE INTO command
(#8597)
add e331141232c [MINOR] Added docs on gotchas when using
PartialUpdateAvroPayload (#8579)
add 2c2abaf14bd [HUDI-6180] Use ConfigProperty for Timestamp keygen
configs (#8643)
add b497ef1a3f0 [HUDI-4630] Add transformer capability to individual feeds
in MultiTableDeltaStreamer (#8399)
add 9d0b292022a [MINOR] Migrate azure-pipelines.yml with notes (#8694)
add 19d1094a73b [HUDI-6195] Test-cover different payload classes when use
HoodieMergedReadHandle (#8675)
add 378ea65a338 [HUDI-6142] Refactor the code related to creating
user-defined index (#8582)
add 90c40d48b06 [HUDI-6204] Add bundle validation on Spark 3.3.2 (#8692)
add 3df303eb770 [MINOR] Prevent timeline server from being reused in
ITTestSchemaEvolution (#8698)
add 81b4acad51d [HUDI-6106] Spark offline compaction/Clustering Job will
do clean like Flink job (#8505)
add a8b1fa33ead [HUDI-4630] Fix hudi-utilities test's transformer
misconfig (#8705)
add 67ae0c8e7e4 [HUDI-5710] Load all partitions in advance for clean when
MDT is enabled (#7865)
add b3f2c753e4b [MINOR] Update
docker-compose_hadoop284_hive233_spark244_mac_aarch64.yml (#8701)
add 42b517d9666 [MINOR] Update
docker-compose_hadoop284_hive233_spark244.yml (#8702)
add 3f231d789b2 [MINOR] Remove unused imports in Spark adapters (#8707)
add 1fc4d1c5db7 [HUDI-6209] Move test deps to tests-common (#8708)
add 2e2459ed865 Do not combine records if write operation is Upsert and
COMBINE_BEFORE_UPSERT is disabled (#7998)
add 09963100c93 [HUDI-6199] Fix deletes with custom payload implementation
(#8690)
add 398918c66e7 [MINOR] Optimizing schema validation in Metadata table
(#7619)
add d2b411ad192 [HUDI-6210] Failed to add fields in BUCKET index table
(#8710)
add 2370615e9bb [HUDI-6216] Get table type and payload class from table
config in flink table factory when not provided (#8720)
add 3e788ba8b33 [HUDI-6134] Prevent clean run concurrently in flink (#8568)
add 3dba6e488a1 [HUDI-6218] Support instant_time/commit_time of savepoint
procedure optional (#8722)
add dc2f67c6a23 [HUDI-6104] Clean deleted partition with clean policy
(#8506)
add 647ad0f4a0b [HUDI-6222] ParquetSchemaConverter shoud always convert
the Map key type as not nullable (#8728)
add efc30f636c6 [HUDI-915] Add missing partititonpath to records COW
(#8666)
add 014168b948a [HUDI-6211] Fix reading of schema-evolved complex columns
for Flink (#8711)
add a5fde6b5c22 [HUDI-6203] Add support to standalone utility tool to
fetch file size stats for a given table w/ optional partition filters (#8645)
add a241e130c08 [HUDI-6227] Improve logging in
ReflectionUtils.hasConstructor (#8732)
add 42173629067 [HUDI-6229] HoodieInternalWriteStatus marks failure with
totalErrorRecords increment (#8739)
add 8e61a9f242e [HUDI-6107] Fix java.lang.IllegalArgumentException for
bootstrap (#8640)
add 213940a3b63 [MINOR] Change scalastyle ReturnChecker lever (#8731)
add 720c9d76c1d [HUDI-5675] fix lazy clean schedule rollback on completed
instant (#7826)
add 423102b69ad [MINOR] HoodieAvroUtils supports enum => conversion
rewrite (#8738)
add 9ef7bd8a675 [HUDI-5394] Fix tests for RowCustomColumnsSortPartitioner
(#8741)
add cfa02f2dd99 [HUDI-6228] Re-enable tests that were flaky before (#8733)
add 0b87e143cfe [HUDI-6115] Adding hardening checks for transformer output
schema for quarantine enabled/disabled (#8520)
add 0d55c9d4a93 [HUDI-6208] Fix jetty conflicts in the packaging process
(#8706)
add c229ccf57ce [HUDI-6233] Fix table client conf in AlterTableCommand
(#8747)
add 4c980f3178a [HUDI-5238] Fix `HoodieMergeHandle` shutdown sequence
(#7245)
add 11aee9695ec [HUDI-6212] Hudi Spark 3.0.x integration (#8714)
add bcfd3545e60 [HUDI-6236] Write hive_style_partitioning_enable to table
config in DeltaStreamer entrypoints (#8752)
add 4a4ac9e133c [HUDI-6241] HIVE_SYNC_TABLE_STRATEGY in
HiveSyncConfigHolder Documentation fix (#8765)
add e11acad84dd [MINOR] Remove unused properties from
BootstrapPartitionPathTranslator (#8766)
add 84b3dd33edc [HUDI-6239] Fix clustering pool scheduler conf not take
effect bug (#8763)
add a64ccc777f3 [HUDI-6240] Adding default value as CORRECTED for rebase
modes in write and read for avro (#8764)
add 6534a7f2c35 [HUDI-5923] Fix deletion of metadata table via cli (#8167)
add 35aba7cb430 [HUDI-5520] Limit MDT deltacommits when data table has
pending action (#8772)
add bf5715df326 [HUDI-5980] Add tests to guard against repeated dag
trigger using spark event listeners (#8641)
add 5bcbdada0d0 [HUDI-6244] Add bundle validation images for Spark 3.0 and
3.4 (#8767)
add e9cf0443815 [HUDI-6237] Fix call stats_file_sizes failure error due to
empty globRegex for partitioned tables (#8755)
add bec544a0163 [HUDI-3775] Allow for offline compaction of MOR tables via
spark streaming (#7632)
add b74e6ad2eb9 [HUDI-6247] Add bundle validation for release candidates
(#8779)
add c3403ff5e09 [HUDI-5584] Metasync update props when changed (#8775)
add 2b3cfa8a744 [HUDI-6245] Automatically downgrade table version of
metadata table (#8771)
add dcc1a07dcfe [MINOR] Optimize code style (#8751)
add a5bd50c067f [MINOR] Disable schema validation in master (#8781)
add 0735dea6d8e [HUDI-4944] Do not decode URI twice in HoodieBootstrapRDD
(#8618)
add 72fffddb695 [HUDI-6232] Add option to skip table archival in glue sync
client (#8744)
add f04f9597840 [MINOR] Avoid synchronized block in HoodieLockMetrics if
key is present in cache (#8778)
add bce768bd241 [HUDI-6213] Parallelize deletion of files during rollback.
(#8717)
add ed1fc6e7a93 [HUDI-6197] Fix use CONTAINER_ID to judge hudi is running
on yarn (#8689)
add 29bca0085f3 [MINOR] Fix some typos and delete unused parameter (#8642)
add e2adc502a2a [HUDI-6190] Adjust description in the
HoodieTableFactory.checkRecordKey exception (#8688)
add 2f6d24f53d9 [HUDI-6235] Update and Delete statements for Flink (#8749)
add b28eeafea68 [MINOR] Check the return value from delete during rollback
and finalize to ensure the files actually got deleted (#8606)
add d8d563ad99c [MINOR] When a clean operation fails do not continue and
throw the exception (#8602)
add 9d58ee4b1f1 [HUDI-5994] Bucket index supports bulk insert row writer
(#8776)
add ea547e5681a [HUDI-6219] Ensure consistency between Spark catalog
schema and Hudi schema (#8725)
add 6b24ab16801 [HUDI-6201] Use concurrent map when possible in
filesystemview (#8782)
add c859ea4cd23 Revert "[MINOR] Check the return value from delete during
rollback and finalize to ensure the files actually got deleted (#8606)" (#8803)
add 516c3d59404 [HUDI-3088] Use Spark 3.2 as default Spark version (#8445)
add df64d647fa3 [HUDI-1517] create marker file for every log file (#4913)
add 83ea5016478 [HUDI-5517][FOLLOW-UP] Refine API names and ensure time
travel won't affect by stateTransitionTime (#8762)
add 1725b0915ad [HUDI-6041] add `options` input to Bootstrap Procedure for
passing hudi properties (#8387)
add 59786113fae [HUDI-5998] Speed up reads from bootstrapped tables in
spark (#8303)
add 98b3c6d3bcd Updating DOAP file for 0.13.1 (#8814)
add c9448f962c7 [HUDI-6267] Init deletedFileResult with false to avoid NPE
(#8812)
add ee92ca6c1bf [HUDI-6273] Fix default maven build and README (#8822)
add 078c949466c [HUDI-6272] Fix POM to properly skip tests (#8823)
add 1e3cdb66aae [HUDI-5189] Make HiveAvroSerializer compatible with hive3
(#7173)
add 31192242cea [HUDI-6261] Make deltastreamer exceptions more descriptive
(#8638)
add 464b9753b85 [HUDI-6260] Fix the MDT compaction reader with the instant
range filtering (#8797)
add 22ea5d094f5 [Minor] Hotfix in HoodieSparkEngineContext with delete log
(#8811)
add cf4b92959bd [HUDI-5724] Fix merger api usage with more UTs (#8736)
add 4b34681068c [MINOR] Make isErrorTableWriterEnabled premitive type and
improve comments (#8819)
add a7e0f4c0f9f [HUDI-5321] Prepend partition path for custom partitioner
(#8831)
add 182a2109372 [HUDI-5193] Improve test coverage for Spark DataSource
write flows (#7179)
add 372fcd8784f [HUDI-6254] Allow using absolute path in
ManifestFileWriter (#8787)
add d2e9865d0c8 [HUDI-6257] Upgrade table version partition path check
should consider hive style partitioning (#8794)
add 1fc8b1f41f4 [HUDI-6269] Ensure configs of one transformer are not
passed to another transformer (#8817)
add 0fb84943de8 [HUDI-6268] HoodieFlinkCompactor read parameter by
specifying options param (#8815)
add 91623814132 [HUDI-6259] Support configuration hot update for
deltastreamer continuous mode (#8807)
add 48d71266418 [MINOR] Make bootstrap operator invoke MDT's
getAllPartitions method (#8806)
add 51d28fd75f5 [HUDI-6264] Assert that bootstrap base path is different
than path (#8808)
add 4193f40c4d9 [HUDI-4944] Use mor solution to uri filenotfound problem
(#8809)
add 116a17bb8d5 [MINOR] Remove unnecessary code in DeltaSync#syncOnce
(#8836)
add 41e1e9a4fda [MINOR] Ensure metrics prefix does not contain any dot.
(#8599)
add 5420387869c [HUDI-4790][RFC-68] a more effective HoodieMergeHandler
for COW table with parquet (#6612)
add 7cf0c9571a5 [MINOR] Fix typo about `recordMergerStrategyId` (#8846)
add 6f5aaf6586d [HUDI-5988] Add a param, Implement a full partition sync
operation wh… (#8301)
add efa141f121e [HUDI-6275] Fix POM for building bundle jars of Spark 2
(#8825)
add 87740003822 [HUDI-6152] Fixed the check for older timestamps with
second granularity during index tagLocation. (#8605)
add 9c7d856656f [HUDI-6277][UBER] Clustering enhancements (#8829)
add 5b220703567 [HUDI-6060] Added a config to backup instants before
deletion during rollbacks and restores. (#8430)
add c6dadd4cb5d [HUDI-6287] Fix Memory Leak in
RealtimeCompactedRecordReader (#8839)
add 6671d1a2460 [HUDI-6294] Fix log classname in
SqlQuerySingleResultPreCommitValidator (#8853)
add 399b46c4942 [HUDI-6290] Fix Flink MDT compaction strategy (#8850)
add 00d50e91abe [HUDI-6293] Make HoodieFlinkCompactor's parallelism of
compact_task more reasonable (#8854)
add 87434e6d574 [HUDI-6303] Bump flink version to 1.16.2 and 1.17.1 (#8861)
add e8ca0d4121a [HUDI-6303] Bump flink version to 1.16.2 and 1.17.1 (#8861)
add 3f9c45fdfa9 [HUDI-6258] support olap engine query mor table in table
name without ro/rt suffix (#8795)
add 32adbe4dfb2 [HUDI-6256] Fix the data table archiving and MDT cleaning
config conf… (#8792)
add b36e7c45990 [HUDI-6293] Make HoodieClusteringJob's parallelism of
clustering_task more reasonable (#8866)
add 43e025a96af [HUDI-6309] Fix bug when hive queries Array type (#8873)
add 88e3221f126 [HUDI-6307] Sync TIMESTAMP_MILLIS to hive (#8867)
add 56086d6e99c [MINOR] Fix checkstyle + removed unnecessary index in #add
(#8883)
add d7fb993a4a0 [MINOR] Enabling codecoverage for master branch (#8889)
add 2294c52c36e [MINOR] Reformat HoodieAvroParquetReader (#8888)
add f60afd5376f [MINOR] Use spaces instead of tabs (#8895)
add 7ae8da02d12 [HUDI-6200] Enhancements to the MDT for improving
performance of larger indexes. (#8684)
add 80e0b557ffe [HUDI-6310]
CreateHoodieTableCommand::createHiveDataSourceTable arguments refactor (#8874)
add 593181397e2 [HUDI-5352] Fix `LocalDate` serialization in colstats
(#8840)
add f1c8049f81a [HUDI-6342] Fixing flaky Continuous mode multi writer
tests (#8910)
add 78094566e4b [HUDI-6182] Hive sync use state transient time to avoid
losing partit… (#8745)
add 195ae3a9a23 [HUDI-6334] Integrate logcompaction table service to
metadata table and provides various bugfixes to metadata table (#8900)
add 27c1edf97c3 [HUDI-6182] Fix the hive sync compatibility and improve
timeline check (#8917)
add c849866d6c7 [HUDI-3545] [UBER] Make HoodieAvroWriteSupport class
configurable (#8590)
add 96ca7aac693 [HUDI-6198] Support Hudi on Spark 3.4.0 (#8885)
add b5e98d5763e Bump xalan from 2.7.2 to 2.7.3
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (6878ff14acf)
\
N -- N -- N refs/heads/dependabot/maven/xalan-xalan-2.7.3
(b5e98d5763e)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.codecov.yml | 2 +-
.github/workflows/bot.yml | 142 +-
README.md | 42 +-
...e-pipelines.yml => azure-pipelines-20230430.yml | 55 +-
doap_HUDI.rdf | 5 +
.../docker-compose_hadoop284_hive233_spark244.yml | 1 +
...pose_hadoop284_hive233_spark244_mac_aarch64.yml | 1 +
hudi-aws/pom.xml | 42 -
.../hudi/aws/sync/AWSGlueCatalogSyncClient.java | 50 +-
.../hudi/config/GlueCatalogSyncClientConfig.java | 42 +
.../org/apache/hudi/config/HoodieAWSConfig.java | 7 +
hudi-cli/pom.xml | 100 +-
.../apache/hudi/cli/commands/BootstrapCommand.java | 4 +
.../apache/hudi/cli/commands/MetadataCommand.java | 13 +-
.../apache/hudi/cli/commands/RepairsCommand.java | 4 +-
.../org/apache/hudi/cli/commands/SparkMain.java | 8 +-
.../org/apache/hudi/cli/commands/TableCommand.java | 4 +-
.../hudi/cli/commands/TestMetadataCommand.java | 106 +
.../hudi/cli/integ/ITTestMarkersCommand.java | 4 +-
hudi-client/hudi-client-common/pom.xml | 108 +-
.../hudi/client/BaseHoodieTableServiceClient.java | 41 +-
.../apache/hudi/client/BaseHoodieWriteClient.java | 79 +-
.../hudi/client/HoodieInternalWriteStatus.java | 5 +-
.../apache/hudi/client/HoodieTimelineArchiver.java | 60 +-
.../BootstrapPartitionPathTranslator.java | 6 +-
...> DecodedBootstrapPartitionPathTranslator.java} | 14 +-
.../IdentityBootstrapPartitionPathTranslator.java | 6 +-
.../transaction/lock/InProcessLockProvider.java | 1 -
.../lock/metrics/HoodieLockMetrics.java | 12 +-
.../hudi/client/utils/ClosableMergingIterator.java | 41 +-
.../apache/hudi/client/utils/MergingIterator.java | 22 +-
.../hudi/client/utils/MetadataTableUtils.java | 44 +
.../apache/hudi/config/HoodieBootstrapConfig.java | 37 +-
.../apache/hudi/config/HoodieCompactionConfig.java | 15 +-
.../org/apache/hudi/config/HoodieWriteConfig.java | 63 +-
.../exception/HoodieUpgradeDowngradeException.java | 6 +-
.../hudi/execution/HoodieLazyInsertIterable.java | 8 +-
.../org/apache/hudi/index/HoodieIndexUtils.java | 37 +-
.../apache/hudi/index/bucket/BucketIdentifier.java | 4 +
.../org/apache/hudi/io/HoodieAppendHandle.java | 35 +-
.../org/apache/hudi/io/HoodieCreateHandle.java | 6 +
.../java/org/apache/hudi/io/HoodieMergeHandle.java | 31 +-
.../apache/hudi/io/HoodieMergeHandleFactory.java | 5 +
.../org/apache/hudi/io/HoodieMergedReadHandle.java | 4 +-
.../java/org/apache/hudi/io/HoodieWriteHandle.java | 44 +
.../AutoRecordGenWrapperAvroKeyGenerator.java | 83 +
.../hudi/keygen/ComplexAvroKeyGenerator.java | 5 +-
.../apache/hudi/keygen/CustomAvroKeyGenerator.java | 10 +-
.../hudi/keygen/GlobalAvroDeleteKeyGenerator.java | 6 +-
.../java/org/apache/hudi/keygen/KeyGenUtils.java | 79 +-
.../keygen/NonpartitionedAvroKeyGenerator.java | 7 +-
.../apache/hudi/keygen/SimpleAvroKeyGenerator.java | 11 +-
.../keygen/TimestampBasedAvroKeyGenerator.java | 24 +-
.../factory/HoodieAvroKeyGeneratorFactory.java | 28 +-
.../keygen/parser/BaseHoodieDateTimeParser.java | 10 +-
.../hudi/keygen/parser/HoodieDateTimeParser.java | 37 +-
.../metadata/HoodieBackedTableMetadataWriter.java | 962 +++---
.../hudi/metadata/HoodieMetadataWriteUtils.java | 193 ++
.../hudi/metadata/HoodieTableMetadataWriter.java | 47 +-
.../org/apache/hudi/metrics/HoodieMetrics.java | 11 +
.../apache/hudi/table/BulkInsertPartitioner.java | 30 +-
.../java/org/apache/hudi/table/HoodieTable.java | 40 +-
.../hudi/table/action/BaseActionExecutor.java | 3 +-
.../action/bootstrap/BootstrapRecordConsumer.java | 1 +
.../table/action/clean/CleanActionExecutor.java | 10 +-
.../hudi/table/action/clean/CleanPlanner.java | 43 +-
.../action/commit/BaseCommitActionExecutor.java | 4 -
.../hudi/table/action/commit/BaseMergeHelper.java | 1 +
.../table/action/commit/HoodieDeleteHelper.java | 13 +-
.../table/action/commit/HoodieMergeHelper.java | 48 +-
.../hudi/table/action/compact/CompactHelpers.java | 21 +
.../hudi/table/action/compact/HoodieCompactor.java | 12 +-
.../compact/RunCompactionActionExecutor.java | 1 +
.../BaseHoodieCompactionPlanGenerator.java | 12 +-
.../HoodieLogCompactionPlanGenerator.java | 16 +-
.../table/action/index/RunIndexActionExecutor.java | 2 +-
.../action/index/ScheduleIndexActionExecutor.java | 10 -
.../rollback/BaseRollbackActionExecutor.java | 39 +
.../rollback/ListingBasedRollbackStrategy.java | 32 +-
.../rollback/MarkerBasedRollbackStrategy.java | 96 +-
.../action/savepoint/SavepointActionExecutor.java | 46 +-
.../hudi/table/marker/DirectWriteMarkers.java | 4 +-
.../marker/TimelineServerBasedWriteMarkers.java | 8 +-
.../org/apache/hudi/table/marker/WriteMarkers.java | 46 +-
.../table/upgrade/FiveToSixUpgradeHandler.java | 70 +
.../table/upgrade/FourToFiveUpgradeHandler.java | 23 +-
.../table/upgrade/SixToFiveDowngradeHandler.java | 57 +
.../hudi/table/upgrade/UpgradeDowngrade.java | 31 +-
.../hudi/avro/TestHoodieAvroParquetWriter.java | 3 +-
.../transaction/TestInProcessLockProvider.java | 117 +
.../common/testutils/HoodieMetadataTestTable.java | 13 +-
.../io/storage/TestHoodieHFileReaderWriter.java | 8 +-
.../io/storage/TestHoodieReaderWriterBase.java | 39 +-
.../org/apache/hudi/keygen/TestKeyGenUtils.java | 13 +-
...estCreateAvroKeyGeneratorByTypeWithFactory.java | 13 +
.../hudi/table/TestBulkInsertPartitioner.java | 59 +
.../hudi/testutils/HoodieWriteableTestTable.java | 3 +-
.../providers/HoodieMetaClientProvider.java | 14 +
hudi-client/hudi-flink-client/pom.xml | 54 +-
.../hudi/client/HoodieFlinkTableServiceClient.java | 23 +-
.../apache/hudi/client/HoodieFlinkWriteClient.java | 2 +-
.../hudi/execution/FlinkLazyInsertIterable.java | 14 +-
.../apache/hudi/index/FlinkHoodieIndexFactory.java | 7 +-
.../java/org/apache/hudi/io/FlinkAppendHandle.java | 27 +-
.../io/storage/row/HoodieRowDataCreateHandle.java | 2 +-
.../row/parquet/ParquetSchemaConverter.java | 2 +-
.../FlinkHoodieBackedTableMetadataWriter.java | 59 +-
.../commit/BaseFlinkCommitActionExecutor.java | 5 -
.../FlinkDeletePartitionCommitActionExecutor.java | 2 +-
.../row/parquet/TestParquetSchemaConverter.java | 2 +-
hudi-client/hudi-java-client/pom.xml | 78 +-
.../run/strategy/JavaExecutionStrategy.java | 3 +-
.../hudi/execution/JavaLazyInsertIterable.java | 13 +-
.../JavaCustomColumnsSortPartitioner.java | 9 +-
.../apache/hudi/index/JavaHoodieIndexFactory.java | 7 +-
.../commit/BaseJavaCommitActionExecutor.java | 13 +-
.../TestJavaBulkInsertInternalPartitioner.java | 7 +-
hudi-client/hudi-spark-client/pom.xml | 57 +-
.../hudi/client/SparkRDDTableServiceClient.java | 43 +-
.../apache/hudi/client/SparkRDDWriteClient.java | 17 +-
.../MultipleSparkJobExecutionStrategy.java | 22 +-
.../client/common/HoodieSparkEngineContext.java | 3 -
.../hudi/client/utils/SparkPartitionUtils.java | 52 +
.../hudi/client/utils/SparkValidatorUtils.java | 6 +-
.../client/validator/SparkPreCommitValidator.java | 24 +-
.../SqlQueryEqualityPreCommitValidator.java | 6 +-
.../SqlQueryInequalityPreCommitValidator.java | 6 +-
.../SqlQuerySingleResultPreCommitValidator.java | 6 +-
.../BucketBulkInsertPartitionerWithRows.java | 81 +
.../RDDCustomColumnsSortPartitioner.java | 15 +-
.../RowCustomColumnsSortPartitioner.java | 19 +-
.../SpatialCurveSortPartitionerBase.java | 7 +-
.../apache/hudi/index/SparkHoodieIndexFactory.java | 8 +-
.../bucket/HoodieSparkConsistentBucketIndex.java | 154 +-
.../io/storage/HoodieSparkFileReaderFactory.java | 5 +
.../hudi/io/storage/row/HoodieRowCreateHandle.java | 2 +-
.../keygen/AutoRecordGenWrapperKeyGenerator.java | 108 +
.../apache/hudi/keygen/ComplexKeyGenerator.java | 5 +-
.../org/apache/hudi/keygen/CustomKeyGenerator.java | 20 +-
.../hudi/keygen/GlobalDeleteKeyGenerator.java | 4 +-
.../hudi/keygen/NonpartitionedKeyGenerator.java | 10 +-
.../org/apache/hudi/keygen/SimpleKeyGenerator.java | 15 +-
.../hudi/keygen/TimestampBasedKeyGenerator.java | 5 +-
.../factory/HoodieSparkKeyGeneratorFactory.java | 15 +-
.../SparkHoodieBackedTableMetadataWriter.java | 69 +-
.../SparkHoodieMetadataBulkInsertPartitioner.java | 111 +
.../hudi/table/HoodieSparkCopyOnWriteTable.java | 9 +
.../bootstrap/OrcBootstrapMetadataHandler.java | 23 +-
.../bootstrap/ParquetBootstrapMetadataHandler.java | 22 +-
.../SparkBootstrapCommitActionExecutor.java | 5 +-
.../commit/BaseSparkCommitActionExecutor.java | 32 +-
.../BucketBulkInsertDataInternalWriterHelper.java | 124 +
.../commit/BulkInsertDataInternalWriterHelper.java | 44 +-
.../org/apache/hudi/AvroConversionUtils.scala | 21 +-
.../hudi/HoodieDatasetBulkInsertHelper.scala | 24 +-
.../scala/org/apache/hudi/HoodieSparkUtils.scala | 47 +-
.../org/apache/hudi/SparkAdapterSupport.scala | 8 +-
.../scala/org/apache/hudi/SparkJdbcUtils.scala | 64 +
.../org/apache/hudi/util/SparkKeyGenUtils.scala | 41 +-
.../spark/sql/HoodieCatalystPlansUtils.scala | 25 +
.../org/apache/spark/sql/HoodieSchemaUtils.scala | 28 +-
.../apache/spark/sql/HoodieUnsafeRowUtils.scala | 4 +
.../HoodieSparkPartitionedFileUtils.scala | 77 +
.../org/apache/spark/sql/hudi/SparkAdapter.scala | 16 +-
.../hudi/client/TestHoodieClientMultiWriter.java | 90 +-
.../hudi/client/TestHoodieInternalWriteStatus.java | 2 +
.../java/org/apache/hudi/client/TestSavepoint.java | 34 +-
...stSparkHoodieMetadataBulkInsertPartitioner.java | 109 +
...DataValidationCheckForLogCompactionActions.java | 2 +-
.../functional/TestHoodieBackedMetadata.java | 120 +-
.../functional/TestHoodieBackedTableMetadata.java | 6 +-
.../TestHoodieClientOnCopyOnWriteStorage.java | 7 +-
.../TestHoodieClientOnMergeOnReadStorage.java | 26 +-
.../hudi/client/functional/TestHoodieIndex.java | 61 +
.../client/functional/TestHoodieMetadataBase.java | 109 +-
.../TestSparkConsistentBucketClustering.java | 60 +-
.../TestBoundedInMemoryExecutorInSpark.java | 15 +-
.../hudi/execution/TestBoundedInMemoryQueue.java | 16 +-
.../hudi/execution/TestDisruptorMessageQueue.java | 21 +-
.../TestBulkInsertInternalPartitioner.java | 19 +-
.../TestBulkInsertInternalPartitionerForRows.java | 40 +-
.../index/hbase/TestSparkHoodieHBaseIndex.java | 1 +
.../apache/hudi/io/TestHoodieMergedReadHandle.java | 213 +-
.../apache/hudi/io/TestHoodieTimelineArchiver.java | 10 +-
.../io/storage/row/TestHoodieRowCreateHandle.java | 1 -
.../java/org/apache/hudi/table/TestCleaner.java | 33 +-
.../hudi/table/TestHoodieMergeOnReadTable.java | 4 +-
.../table/action/compact/TestInlineCompaction.java | 2 +-
.../TestCopyOnWriteRollbackActionExecutor.java | 84 +-
.../TestMergeOnReadRollbackActionExecutor.java | 23 +-
.../table/functional/TestCleanPlanExecutor.java | 72 +-
...dieSparkMergeOnReadTableInsertUpdateDelete.java | 82 +-
.../TestHoodieSparkMergeOnReadTableRollback.java | 13 -
.../TestMarkerBasedRollbackStrategy.java | 59 +-
.../hudi/table/upgrade/TestUpgradeDowngrade.java | 198 +-
.../hudi/testutils/HoodieClientTestHarness.java | 6 +-
.../hudi/testutils/HoodieMergeOnReadTestUtils.java | 2 +-
hudi-common/pom.xml | 61 +-
.../java/org/apache/hudi/avro/HoodieAvroUtils.java | 3 +
.../apache/hudi/avro/HoodieAvroWriteSupport.java | 8 +-
.../apache/hudi/common/config/ConfigGroups.java | 5 +
.../hudi/common/config/HoodieMetadataConfig.java | 39 +
.../hudi/common/config/HoodieStorageConfig.java | 14 +
.../common/config/TimestampKeyGeneratorConfig.java | 99 +
.../java/org/apache/hudi/common/fs/FSUtils.java | 30 +-
.../hudi/common/model/AWSDmsAvroPayload.java | 11 +-
.../apache/hudi/common/model/BaseAvroPayload.java | 2 +-
.../model/HoodieConsistentHashingMetadata.java | 1 +
.../hudi/common/model/HoodiePartitionMetadata.java | 2 +-
.../hudi/common/model/HoodieTimelineTimeZone.java | 15 +-
.../common/model/PartialUpdateAvroPayload.java | 27 +
.../hudi/common/model/WriteOperationType.java | 15 +
.../debezium/AbstractDebeziumAvroPayload.java | 18 +-
.../hudi/common/table/HoodieTableConfig.java | 118 +-
.../hudi/common/table/HoodieTableVersion.java | 4 +-
.../table/log/HoodieLogFileWriteCallback.java | 24 +-
.../hudi/common/table/log/HoodieLogFormat.java | 15 +-
.../common/table/log/HoodieLogFormatWriter.java | 22 +-
.../table/log/HoodieMergedLogRecordScanner.java | 3 +-
.../apache/hudi/common/table/log/InstantRange.java | 29 +-
.../table/timeline/HoodieActiveTimeline.java | 37 +-
.../table/timeline/HoodieArchivedTimeline.java | 16 +-
.../table/timeline/HoodieDefaultTimeline.java | 23 +-
.../hudi/common/table/timeline/HoodieInstant.java | 10 +-
.../table/timeline/HoodieInstantTimeGenerator.java | 2 +-
.../hudi/common/table/timeline/HoodieTimeline.java | 4 +-
.../hudi/common/table/timeline/TimelineUtils.java | 33 +-
.../table/view/AbstractTableFileSystemView.java | 50 +-
.../common/table/view/FileSystemViewManager.java | 37 +-
.../table/view/HoodieTableFileSystemView.java | 16 +-
.../IncrementalTimelineSyncFileSystemView.java | 10 +-
.../table/view/PriorityBasedFileSystemView.java | 16 +
.../view/RemoteHoodieTableFileSystemView.java | 48 +
.../common/table/view/TableFileSystemView.java | 19 +
.../org/apache/hudi/common/util/FileIOUtils.java | 7 +-
.../org/apache/hudi/common/util/FutureUtils.java | 44 +-
.../org/apache/hudi/common/util/HoodieTimer.java | 9 +-
.../org/apache/hudi/common/util/JsonUtils.java | 8 +
.../java/org/apache/hudi/common/util/MapUtils.java | 4 +
.../apache/hudi/common/util/ReflectionUtils.java | 29 +-
.../common/util/collection/ClosableIterator.java | 2 -
.../util/collection/ExternalSpillableMap.java | 30 +-
.../util/queue/BaseHoodieQueueBasedExecutor.java | 38 +-
.../common/util/queue/BoundedInMemoryQueue.java | 5 +
.../common/util/queue/DisruptorMessageQueue.java | 8 +
.../hudi/common/util/queue/HoodieMessageQueue.java | 2 +
.../HoodieIncrementalPathNotFoundException.java} | 20 +-
.../HoodieLogCompactException.java} | 19 +-
.../hudi/exception/HoodieMetaSyncException.java | 19 +-
.../io/storage/HoodieAvroFileWriterFactory.java | 22 +-
.../hudi/io/storage/HoodieBootstrapFileReader.java | 37 +-
.../io/storage/HoodieBootstrapRecordIterator.java | 82 +
.../org/apache/hudi/keygen/BaseKeyGenerator.java | 1 +
.../hudi/keygen/constant/KeyGeneratorOptions.java | 41 +-
.../hudi/metadata/HoodieBackedTableMetadata.java | 83 +-
.../metadata/HoodieMetadataLogRecordReader.java | 22 +-
.../hudi/metadata/HoodieMetadataMetrics.java | 3 +
.../apache/hudi/metadata/HoodieTableMetadata.java | 7 +
.../hudi/metadata/HoodieTableMetadataUtil.java | 370 ++-
.../hudi/metadata/MetadataPartitionType.java | 13 -
.../common/functional/TestHoodieLogFormat.java | 4 +-
.../hudi/common/model/TestAWSDmsAvroPayload.java | 14 +-
.../common/model/TestPartialUpdateAvroPayload.java | 91 +-
.../debezium/TestMySqlDebeziumAvroPayload.java | 3 +
.../debezium/TestPostgresDebeziumAvroPayload.java | 3 +
.../hudi/common/table/TestTimelineUtils.java | 39 +-
.../table/timeline/TestHoodieActiveTimeline.java | 10 +-
.../common/table/timeline/TestHoodieInstant.java | 9 +
.../hudi/common/testutils/FileCreateUtils.java | 58 +-
.../HoodieAdaptablePayloadDataGenerator.java | 279 ++
.../hudi/common/testutils/HoodieTestTable.java | 58 +
.../hudi/common/testutils/HoodieTestUtils.java | 19 +-
.../hudi/common/testutils/RawTripTestPayload.java | 23 +
.../minicluster/ZookeeperTestService.java | 1 +
.../apache/hudi/common/util/TestFileIOUtils.java | 22 +
.../org/apache/hudi/common/util/TestMapUtils.java | 69 +
.../apache/hudi/common/util/TestParquetUtils.java | 3 +-
.../schema/utils/TestAvroSchemaEvolutionUtils.java | 15 +-
.../io/storage/TestHoodieBaseParquetWriter.java | 3 +-
.../src/test/resources/adaptable-payload.avsc | 82 +
hudi-examples/bin/hudi-delta-streamer | 4 +-
hudi-examples/hudi-examples-common/pom.xml | 27 +-
hudi-examples/hudi-examples-flink/pom.xml | 27 -
hudi-examples/hudi-examples-java/pom.xml | 26 -
hudi-examples/hudi-examples-spark/pom.xml | 33 -
.../spark/HoodieSparkBootstrapExample.java | 20 +-
hudi-flink-datasource/hudi-flink/pom.xml | 41 -
.../java/org/apache/hudi/sink/CleanFunction.java | 11 +-
.../org/apache/hudi/sink/StreamWriteFunction.java | 1 +
.../hudi/sink/bootstrap/BootstrapOperator.java | 4 +-
.../hudi/sink/clustering/ClusteringCommitSink.java | 2 +-
.../sink/clustering/FlinkClusteringConfig.java | 25 +-
.../sink/clustering/HoodieFlinkClusteringJob.java | 17 +-
.../hudi/sink/compact/CompactionCommitSink.java | 2 +-
.../hudi/sink/compact/FlinkCompactionConfig.java | 29 +-
.../hudi/sink/compact/HoodieFlinkCompactor.java | 18 +-
.../java/org/apache/hudi/source/FileIndex.java | 2 +-
.../org/apache/hudi/table/HoodieTableFactory.java | 39 +-
.../org/apache/hudi/table/HoodieTableSink.java | 24 +-
.../java/org/apache/hudi/util/CompactionUtil.java | 22 -
.../apache/hudi/util/DataModificationInfos.java | 24 +-
.../java/org/apache/hudi/util/StreamerUtil.java | 10 +
.../sink/TestStreamWriteOperatorCoordinator.java | 10 +-
.../hudi/sink/bucket/ITTestBucketStreamWrite.java | 27 +-
.../sink/compact/ITTestHoodieFlinkCompactor.java | 84 +
.../apache/hudi/table/ITTestHoodieDataSource.java | 62 +-
.../apache/hudi/table/ITTestSchemaEvolution.java | 203 +-
.../apache/hudi/table/TestHoodieTableFactory.java | 21 +-
.../org/apache/hudi/utils/TestConfigurations.java | 8 +
.../test/java/org/apache/hudi/utils/TestData.java | 39 +-
hudi-flink-datasource/hudi-flink1.13.x/pom.xml | 41 -
.../adapter/SupportsRowLevelDeleteAdapter.java | 22 +-
.../adapter/SupportsRowLevelUpdateAdapter.java | 26 +-
.../table/format/cow/ParquetSplitReaderUtil.java | 56 +-
.../cow/vector/reader/EmptyColumnReader.java | 41 +
hudi-flink-datasource/hudi-flink1.14.x/pom.xml | 41 -
.../adapter/SupportsRowLevelDeleteAdapter.java | 22 +-
.../adapter/SupportsRowLevelUpdateAdapter.java | 26 +-
.../table/format/cow/ParquetSplitReaderUtil.java | 56 +-
.../cow/vector/reader/EmptyColumnReader.java | 41 +
hudi-flink-datasource/hudi-flink1.15.x/pom.xml | 41 -
.../adapter/SupportsRowLevelDeleteAdapter.java | 22 +-
.../adapter/SupportsRowLevelUpdateAdapter.java | 26 +-
.../table/format/cow/ParquetSplitReaderUtil.java | 56 +-
.../cow/vector/reader/EmptyColumnReader.java | 42 +
hudi-flink-datasource/hudi-flink1.16.x/pom.xml | 41 -
.../adapter/SupportsRowLevelDeleteAdapter.java | 22 +-
.../adapter/SupportsRowLevelUpdateAdapter.java | 26 +-
.../table/format/cow/ParquetSplitReaderUtil.java | 56 +-
.../cow/vector/reader/EmptyColumnReader.java | 42 +
hudi-flink-datasource/hudi-flink1.17.x/pom.xml | 41 -
.../adapter/SupportsRowLevelDeleteAdapter.java | 28 +-
.../adapter/SupportsRowLevelUpdateAdapter.java | 45 +
.../table/format/cow/ParquetSplitReaderUtil.java | 56 +-
.../cow/vector/reader/EmptyColumnReader.java | 42 +
hudi-gcp/pom.xml | 41 -
.../apache/hudi/gcp/bigquery/BigQuerySyncTool.java | 2 +-
hudi-hadoop-mr/pom.xml | 46 +-
.../hudi/hadoop/HoodieColumnProjectionUtils.java | 21 +
.../hudi/hadoop/HoodieParquetInputFormat.java | 44 +-
.../hudi/hadoop/avro/HoodieAvroParquetReader.java | 104 +
.../HoodieTimestampAwareParquetInputFormat.java | 44 +
.../realtime/AbstractRealtimeRecordReader.java | 9 +
.../realtime/HoodieHFileRealtimeInputFormat.java | 2 +-
.../realtime/HoodieParquetRealtimeInputFormat.java | 22 +-
.../realtime/HoodieRealtimeRecordReader.java | 13 +-
.../realtime/RealtimeCompactedRecordReader.java | 2 +-
.../realtime/RealtimeUnmergedRecordReader.java | 2 +-
.../hudi/hadoop/utils/HiveAvroSerializer.java | 24 +-
.../apache/hudi/hadoop/utils/HoodieHiveUtils.java | 43 +
.../hudi/hadoop/utils/HoodieInputFormatUtils.java | 10 +
.../utils/HoodieRealtimeInputFormatUtils.java | 28 +-
.../utils/HoodieRealtimeRecordReaderUtils.java | 44 +-
.../apache/hudi/hadoop/utils/shims/Hive2Shim.java | 56 +
.../apache/hudi/hadoop/utils/shims/Hive3Shim.java | 125 +
.../apache/hudi/hadoop/utils/shims/HiveShim.java | 22 +-
.../apache/hudi/hadoop/utils/shims/HiveShims.java | 19 +-
.../hudi/hadoop/TestHoodieParquetInputFormat.java | 66 +
.../hudi/hadoop/testutils/InputFormatTestUtil.java | 2 +-
.../hudi/hadoop/utils/TestHiveAvroSerializer.java | 25 +-
.../src/test/resources/test_timetype.avsc | 50 +-
hudi-integ-test/pom.xml | 154 +-
.../testsuite/HoodieDeltaStreamerWrapper.java | 4 +-
.../org/apache/hudi/integ/ITTestHoodieDemo.java | 4 +-
.../org/apache/hudi/integ/ITTestHoodieSanity.java | 19 +
hudi-kafka-connect/pom.xml | 51 +-
hudi-platform-service/hudi-metaserver/README.md | 8 +-
hudi-platform-service/hudi-metaserver/pom.xml | 40 -
hudi-spark-datasource/hudi-spark-common/pom.xml | 42 -
.../internal/DataSourceInternalWriterHelper.java | 9 +-
.../apache/hudi/AutoRecordKeyGenerationUtils.scala | 50 +
.../org/apache/hudi/BaseFileOnlyRelation.scala | 13 +-
.../scala/org/apache/hudi/DataSourceOptions.scala | 15 +-
.../main/scala/org/apache/hudi/DefaultSource.scala | 24 +-
.../scala/org/apache/hudi/HoodieBaseRelation.scala | 18 +-
.../org/apache/hudi/HoodieBootstrapRelation.scala | 42 +-
.../scala/org/apache/hudi/HoodieCLIUtils.scala | 8 +
.../org/apache/hudi/HoodieDataSourceHelper.scala | 9 +-
.../scala/org/apache/hudi/HoodieFileIndex.scala | 40 +-
.../org/apache/hudi/HoodieSparkSqlWriter.scala | 105 +-
.../org/apache/hudi/HoodieStreamingSink.scala | 7 +-
.../scala/org/apache/hudi/HoodieWriterUtils.scala | 7 +-
.../org/apache/hudi/IncrementalRelation.scala | 34 +-
.../src/main/scala/org/apache/hudi/Iterators.scala | 19 +-
.../hudi/MergeOnReadIncrementalRelation.scala | 4 +-
.../apache/hudi/MergeOnReadSnapshotRelation.scala | 29 +-
.../apache/hudi/SparkHoodieTableFileIndex.scala | 92 +-
.../scala/org/apache/hudi/cdc/HoodieCDCRDD.scala | 16 +-
.../apache/spark/sql/avro/SchemaConverters.scala | 2 +-
.../sql/catalyst/catalog/HoodieCatalogTable.scala | 35 +-
.../parquet/HoodieParquetFileFormat.scala | 13 +-
.../apache/spark/sql/hudi/HoodieOptionConfig.scala | 9 +-
.../spark/sql/hudi/ProvidesHoodieConfig.scala | 8 +-
.../AlterHoodieTableAddColumnsCommand.scala | 40 +-
.../hudi/command/CreateHoodieTableCommand.scala | 5 +-
.../spark/sql/hudi/command/SqlKeyGenerator.scala | 14 +-
.../sql/hudi/streaming/HoodieStreamSource.scala | 2 +-
.../HoodieBulkInsertInternalWriterTestBase.java | 1 -
hudi-spark-datasource/hudi-spark/pom.xml | 78 -
hudi-spark-datasource/hudi-spark/run_hoodie_app.sh | 2 +-
.../hudi-spark/run_hoodie_generate_app.sh | 2 +-
.../hudi-spark/run_hoodie_streaming_app.sh | 2 +-
.../apache/hudi/cli/BootstrapExecutorUtils.java | 8 +-
.../org/apache/spark/sql/hudi/SparkHelpers.scala | 6 +-
.../spark/sql/hudi/analysis/HoodieAnalysis.scala | 38 +-
.../hudi/command/MergeIntoHoodieTableCommand.scala | 39 +-
.../hudi/command/payload/ExpressionPayload.scala | 4 +-
.../procedures/ArchiveCommitsProcedure.scala | 6 +-
.../hudi/command/procedures/BaseProcedure.scala | 48 +-
.../procedures/CommitsCompareProcedure.scala | 4 +-
.../command/procedures/CopyToTableProcedure.scala | 4 +-
.../hudi/command/procedures/CopyToTempView.scala | 4 +-
.../procedures/CreateMetadataTableProcedure.scala | 2 +-
.../procedures/CreateSavepointProcedure.scala | 19 +-
.../command/procedures/DeleteMarkerProcedure.scala | 4 +-
.../procedures/DeleteMetadataTableProcedure.scala | 2 +-
.../procedures/DeleteSavepointProcedure.scala | 12 +-
.../procedures/ExportInstantsProcedure.scala | 4 +-
.../procedures/HdfsParquetImportProcedure.scala | 14 +-
.../hudi/command/procedures/HelpProcedure.scala | 4 +-
.../command/procedures/HiveSyncProcedure.scala | 7 +-
.../procedures/InitMetadataTableProcedure.scala | 2 +-
.../command/procedures/ProcedureParameter.scala | 7 +-
.../RepairAddpartitionmetaProcedure.scala | 2 +-
.../RepairCorruptedCleanFilesProcedure.scala | 2 +-
.../procedures/RepairDeduplicateProcedure.scala | 6 +-
.../RepairMigratePartitionMetaProcedure.scala | 2 +-
.../RepairOverwriteHoodiePropsProcedure.scala | 4 +-
.../RollbackToInstantTimeProcedure.scala | 4 +-
.../procedures/RollbackToSavepointProcedure.scala | 12 +-
.../command/procedures/RunBootstrapProcedure.scala | 44 +-
.../command/procedures/RunCleanProcedure.scala | 48 +-
.../procedures/RunClusteringProcedure.scala | 36 +-
.../procedures/RunCompactionProcedure.scala | 15 +-
.../procedures/ShowArchivedCommitsProcedure.scala | 2 +-
.../procedures/ShowBootstrapMappingProcedure.scala | 2 +-
.../ShowBootstrapPartitionsProcedure.scala | 2 +-
.../procedures/ShowClusteringProcedure.scala | 4 +-
.../ShowCommitExtraMetadataProcedure.scala | 6 +-
.../procedures/ShowCommitFilesProcedure.scala | 4 +-
.../procedures/ShowCommitPartitionsProcedure.scala | 4 +-
.../procedures/ShowCommitWriteStatsProcedure.scala | 4 +-
.../command/procedures/ShowCommitsProcedure.scala | 2 +-
.../procedures/ShowCompactionProcedure.scala | 4 +-
.../procedures/ShowFileSystemViewProcedure.scala | 6 +-
.../procedures/ShowFsPathDetailProcedure.scala | 2 +-
.../ShowHoodieLogFileMetadataProcedure.scala | 4 +-
.../ShowHoodieLogFileRecordsProcedure.scala | 4 +-
.../procedures/ShowInvalidParquetProcedure.scala | 2 +-
.../ShowMetadataTableFilesProcedure.scala | 2 +-
.../ShowMetadataTablePartitionsProcedure.scala | 2 +-
.../ShowMetadataTableStatsProcedure.scala | 2 +-
.../procedures/ShowRollbacksProcedure.scala | 6 +-
.../procedures/ShowSavepointsProcedure.scala | 4 +-
.../procedures/ShowTablePropertiesProcedure.scala | 4 +-
.../procedures/StatsFileSizeProcedure.scala | 28 +-
.../StatsWriteAmplificationProcedure.scala | 2 +-
.../procedures/UpgradeOrDowngradeProcedure.scala | 4 +-
.../procedures/ValidateHoodieSyncProcedure.scala | 10 +-
.../ValidateMetadataTableFilesProcedure.scala | 2 +-
.../java/org/apache/hudi/TestDataSourceUtils.java | 2 +-
.../org/apache/hudi/functional/TestBootstrap.java | 27 +-
.../apache/hudi/functional/TestBootstrapRead.java | 303 ++
.../TestGlobalIndexEnableUpdatePartitions.java | 225 ++
.../TestHoodieDatasetBulkInsertHelper.java | 24 +-
.../apache/hudi/functional/TestOrcBootstrap.java | 41 +-
.../hudi/keygen/TestComplexKeyGenerator.java | 6 +-
.../apache/hudi/keygen/TestCustomKeyGenerator.java | 26 +-
.../keygen/TestGlobalDeleteRecordGenerator.java | 6 +-
.../keygen/TestNonpartitionedKeyGenerator.java | 6 +-
.../apache/hudi/keygen/TestSimpleKeyGenerator.java | 6 +-
.../keygen/TestTimestampBasedKeyGenerator.java | 39 +-
.../TestCreateKeyGeneratorByTypeWithFactory.java | 13 +
.../org/apache/hudi/TestDataSourceDefaults.scala | 10 +-
.../org/apache/hudi/TestHoodieFileIndex.scala | 8 +-
.../org/apache/hudi/TestHoodieSparkSqlWriter.scala | 89 +-
.../model/TestHoodieRecordSerialization.scala | 17 +-
.../apache/hudi/functional/CommonOptionUtils.scala | 56 +
.../TestAutoGenerationOfRecordKeys.scala | 264 ++
.../apache/hudi/functional/TestCOWDataSource.scala | 115 +-
.../hudi/functional/TestCOWDataSourceStorage.scala | 8 +-
.../hudi/functional/TestColumnStatsIndex.scala | 7 +-
.../functional/TestDataSourceForBootstrap.scala | 13 +-
.../TestIncrementalReadByStateTransitionTime.scala | 2 +-
.../TestIncrementalReadWithFullTableScan.scala | 3 +-
.../apache/hudi/functional/TestMORDataSource.scala | 10 +-
.../hudi/functional/TestMORDataSourceStorage.scala | 90 +-
.../TestMetadataTableWithSparkDataSource.scala | 2 +-
.../functional/TestParquetColumnProjection.scala | 1 -
.../hudi/functional/TestSparkDataSource.scala | 383 +++
.../TestSparkDataSourceDAGExecution.scala | 192 ++
.../hudi/functional/TestStreamingSource.scala | 4 +
.../hudi/functional/TestStructuredStreaming.scala | 37 +-
.../org/apache/spark/sql/avro/TestAvroSerDe.scala | 1 +
.../org/apache/spark/sql/hudi/TestAlterTable.scala | 36 +
.../sql/hudi/TestAlterTableDropPartition.scala | 9 +-
.../apache/spark/sql/hudi/TestCreateTable.scala | 23 +-
.../spark/sql/hudi/TestHoodieOptionConfig.scala | 10 -
.../apache/spark/sql/hudi/TestInsertTable.scala | 120 +-
.../apache/spark/sql/hudi/TestMergeIntoTable.scala | 148 +-
.../hudi/TestNestedSchemaPruningOptimization.scala | 4 +-
.../sql/hudi/command/index/TestIndexSyntax.scala | 17 +-
.../procedure/TestArchiveCommitsProcedure.scala | 5 +-
.../hudi/procedure/TestBootstrapProcedure.scala | 69 +
.../sql/hudi/procedure/TestCallProcedure.scala | 2 +-
.../sql/hudi/procedure/TestCleanProcedure.scala | 269 +-
.../sql/hudi/procedure/TestCommitsProcedure.scala | 8 +-
.../hudi/procedure/TestCompactionProcedure.scala | 45 +
.../procedure/TestCopyToTempViewProcedure.scala | 15 +-
.../hudi/procedure/TestSavepointsProcedure.scala | 154 +-
.../sql/hudi/procedure/TestStatsProcedure.scala | 93 +-
hudi-spark-datasource/hudi-spark2-common/pom.xml | 41 +-
hudi-spark-datasource/hudi-spark2/pom.xml | 64 -
.../spark/sql/HoodieSpark2CatalystPlanUtils.scala | 12 +-
.../apache/spark/sql/HoodieSpark2SchemaUtils.scala | 20 +-
.../apache/spark/sql/adapter/Spark2Adapter.scala | 12 +-
.../catalyst/analysis/HoodieSpark2Analysis.scala | 4 +
.../HoodieSpark2PartitionedFileUtils.scala | 43 +
hudi-spark-datasource/hudi-spark3-common/pom.xml | 13 +-
.../HoodieBulkInsertDataInternalWriter.java | 7 +-
.../sql/HoodieSpark3CatalystExpressionUtils.scala | 25 +-
.../spark/sql/adapter/BaseSpark3Adapter.scala | 45 +-
.../TestHoodieBulkInsertDataInternalWriter.java | 13 +-
.../TestHoodieDataSourceInternalBatchWrite.java | 25 +-
.../hudi/spark3/internal/TestReflectUtil.java | 8 +-
.../{hudi-spark3.1.x => hudi-spark3.0.x}/pom.xml | 19 +-
...park30HoodieVectorizedParquetRecordReader.java} | 10 +-
...org.apache.spark.sql.sources.DataSourceRegister | 19 +
.../org/apache/hudi/Spark30DefaultSource.scala} | 15 +-
.../org/apache/hudi/Spark30HoodieFileScanRDD.scala | 30 +-
.../spark/sql/HoodieSpark30CatalogUtils.scala} | 18 +-
.../HoodieSpark30CatalystExpressionUtils.scala} | 67 +-
.../sql/HoodieSpark30CatalystPlanUtils.scala} | 14 +-
.../spark/sql/HoodieSpark30SchemaUtils.scala | 20 +-
.../spark/sql/adapter/Spark3_0Adapter.scala} | 83 +-
.../apache/spark/sql/avro/AvroDeserializer.scala | 13 +-
.../org/apache/spark/sql/avro/AvroSerializer.scala | 13 +-
.../org/apache/spark/sql/avro/AvroUtils.scala | 115 +
.../sql/avro/HoodieSpark3_0AvroDeserializer.scala} | 6 +-
.../sql/avro/HoodieSpark3_0AvroSerializer.scala} | 10 +-
.../sql/catalyst/plans/logical/HStatments.scala | 0
.../HoodieSpark30PartitionedFileUtils.scala | 43 +
.../datasources/Spark30NestedSchemaPruning.scala | 272 ++
.../parquet/Spark30HoodieParquetFileFormat.scala} | 271 +-
.../hudi/Spark30ResolveHudiAlterTableCommand.scala | 297 ++
.../hudi/command/Spark30AlterTableCommand.scala} | 22 +-
.../sql/parser/HoodieSpark30SqlAstBuilder.scala | 125 +
.../parser/HoodieSpark3_0ExtendedSqlParser.scala} | 26 +-
.../spark/sql/vectorized/ColumnarUtils.scala | 0
hudi-spark-datasource/hudi-spark3.1.x/pom.xml | 13 +-
.../sql/HoodieSpark31CatalystExpressionUtils.scala | 13 +-
.../spark/sql/HoodieSpark31CatalystPlanUtils.scala | 12 +-
.../spark/sql/HoodieSpark31SchemaUtils.scala | 20 +-
.../apache/spark/sql/adapter/Spark3_1Adapter.scala | 58 +-
.../apache/spark/sql/avro/AvroDeserializer.scala | 3 +-
.../org/apache/spark/sql/avro/AvroSerializer.scala | 2 +-
.../sql/avro/HoodieSpark3_1AvroDeserializer.scala | 3 +-
.../HoodieSpark31PartitionedFileUtils.scala | 43 +
.../hudi/command/Spark31AlterTableCommand.scala | 20 +-
hudi-spark-datasource/hudi-spark3.2.x/pom.xml | 46 +-
.../sql/HoodieSpark32CatalystExpressionUtils.scala | 13 +-
.../spark/sql/HoodieSpark32CatalystPlanUtils.scala | 16 +-
.../spark/sql/HoodieSpark32SchemaUtils.scala | 20 +-
.../apache/spark/sql/adapter/Spark3_2Adapter.scala | 52 +-
.../org/apache/spark/sql/avro/AvroSerializer.scala | 3 +-
.../sql/avro/HoodieSpark3_2AvroDeserializer.scala | 3 +-
.../HoodieSpark32PartitionedFileUtils.scala | 43 +
.../parquet/Spark32HoodieParquetFileFormat.scala} | 16 +-
.../hudi-spark3.2plus-common/pom.xml | 13 +-
...k32PlusHoodieVectorizedParquetRecordReader.java | 7 +-
.../spark/sql/hudi/command/AlterTableCommand.scala | 20 +-
hudi-spark-datasource/hudi-spark3.3.x/pom.xml | 42 +-
.../sql/HoodieSpark33CatalystExpressionUtils.scala | 13 +-
.../spark/sql/HoodieSpark33CatalystPlanUtils.scala | 15 +-
.../spark/sql/HoodieSpark33SchemaUtils.scala | 20 +-
.../apache/spark/sql/adapter/Spark3_3Adapter.scala | 51 +-
.../org/apache/spark/sql/avro/AvroSerializer.scala | 3 +-
.../sql/avro/HoodieSpark3_3AvroDeserializer.scala | 3 +-
.../HoodieSpark33PartitionedFileUtils.scala | 43 +
.../parquet/Spark33HoodieParquetFileFormat.scala} | 49 +-
.../{hudi-spark3.3.x => hudi-spark3.4.x}/pom.xml | 31 +-
.../src/main/antlr4/imports/SqlBase.g4 | 0
.../apache/hudi/spark/sql/parser/HoodieSqlBase.g4 | 0
...org.apache.spark.sql.sources.DataSourceRegister | 0
.../org/apache/hudi/Spark34HoodieFileScanRDD.scala | 36 +
.../spark/sql/HoodieSpark34CatalogUtils.scala} | 17 +-
.../HoodieSpark34CatalystExpressionUtils.scala} | 25 +-
.../sql/HoodieSpark34CatalystPlanUtils.scala} | 17 +-
.../spark/sql/HoodieSpark34SchemaUtils.scala | 20 +-
.../spark/sql/adapter/Spark3_4Adapter.scala} | 75 +-
.../apache/spark/sql/avro/AvroDeserializer.scala | 0
.../org/apache/spark/sql/avro/AvroSerializer.scala | 13 +-
.../org/apache/spark/sql/avro/AvroUtils.scala | 0
.../sql/avro/HoodieSpark3_4AvroDeserializer.scala} | 5 +-
.../sql/avro/HoodieSpark3_4AvroSerializer.scala} | 10 +-
.../HoodieSpark34PartitionedFileUtils.scala | 44 +
.../datasources/Spark34NestedSchemaPruning.scala | 198 ++
.../parquet/Spark34HoodieParquetFileFormat.scala} | 53 +-
.../hudi/Spark34ResolveHudiAlterTableCommand.scala | 71 +
.../HoodieSpark3_4ExtendedSqlAstBuilder.scala | 3355 ++++++++++++++++++++
.../parser/HoodieSpark3_4ExtendedSqlParser.scala | 197 ++
.../TestHoodieBulkInsertDataInternalWriter.java | 0
.../TestHoodieDataSourceInternalBatchWrite.java | 0
.../hudi/spark3/internal/TestReflectUtil.java | 0
hudi-sync/hudi-adb-sync/pom.xml | 43 +-
.../java/org/apache/hudi/sync/adb/AdbSyncTool.java | 2 +-
.../apache/hudi/sync/adb/HoodieAdbJdbcClient.java | 2 +-
hudi-sync/hudi-datahub-sync/pom.xml | 44 +-
.../hudi/sync/datahub/DataHubSyncClient.java | 3 +-
hudi-sync/hudi-hive-sync/pom.xml | 92 +-
.../org/apache/hudi/hive/HiveSyncConfigHolder.java | 2 +-
.../java/org/apache/hudi/hive/HiveSyncTool.java | 273 +-
.../org/apache/hudi/hive/HoodieHiveSyncClient.java | 100 +-
.../org/apache/hudi/hive/util/HiveSchemaUtil.java | 2 +-
.../org/apache/hudi/hive/TestHiveSyncTool.java | 231 +-
.../hudi/hive/testutils/HiveTestCluster.java | 3 +-
.../apache/hudi/hive/testutils/HiveTestUtil.java | 67 +-
.../apache/hudi/hive/util/TestHiveSchemaUtil.java | 16 +-
hudi-sync/hudi-sync-common/pom.xml | 51 +-
.../hudi/sync/common/HoodieMetaSyncOperations.java | 27 +-
.../apache/hudi/sync/common/HoodieSyncClient.java | 114 +-
.../apache/hudi/sync/common/HoodieSyncConfig.java | 23 +
.../hudi/sync/common/util/ManifestFileWriter.java | 8 +-
.../hudi/sync/common/util/SyncUtilHelpers.java | 23 +-
.../sync/common/util/TestManifestFileWriter.java | 25 +-
.../hudi/sync/common/util/TestSyncUtilHelpers.java | 4 +-
hudi-tests-common/pom.xml | 149 +-
hudi-timeline-service/pom.xml | 80 +-
.../hudi/timeline/service/RequestHandler.java | 24 +
.../service/handlers/FileSliceHandler.java | 22 +
hudi-utilities/pom.xml | 108 +-
.../apache/hudi/utilities/HoodieClusteringJob.java | 31 +-
.../org/apache/hudi/utilities/HoodieCompactor.java | 10 +
.../org/apache/hudi/utilities/HoodieIndexer.java | 15 +-
.../org/apache/hudi/utilities/TableSizeStats.java | 466 +++
.../org/apache/hudi/utilities/UtilHelpers.java | 48 +-
.../utilities/deltastreamer/BootstrapExecutor.java | 11 +-
.../ConfigurationHotUpdateStrategy.java | 26 +-
.../ConfigurationHotUpdateStrategyUtils.java | 47 +
.../hudi/utilities/deltastreamer/DeltaSync.java | 183 +-
.../utilities/deltastreamer/ErrorTableUtils.java | 35 +-
.../deltastreamer/HoodieDeltaStreamer.java | 34 +
.../HoodieMultiTableDeltaStreamer.java | 10 +
.../deltastreamer/SchedulerConfGenerator.java | 26 +-
.../HoodieDeltaStreamerWriteException.java | 19 +-
.../exception/HoodieReadFromSourceException.java | 19 +-
.../exception/HoodieSchemaFetchException.java | 20 +-
.../exception/HoodieTransformException.java | 21 +-
.../HoodieTransformExecutionException.java | 19 +-
.../exception/HoodieTransformPlanException.java | 18 +-
.../utilities/schema/FilebasedSchemaProvider.java | 4 +-
.../hudi/utilities/schema/HiveSchemaProvider.java | 6 +-
.../utilities/schema/JdbcbasedSchemaProvider.java | 8 +-
.../utilities/schema/KafkaOffsetPostProcessor.java | 22 +-
.../schema/ProtoClassBasedSchemaProvider.java | 9 +-
.../utilities/schema/RowBasedSchemaProvider.java | 3 +-
.../utilities/schema/SchemaRegistryProvider.java | 56 +-
.../hudi/utilities/sources/AvroKafkaSource.java | 9 +-
.../hudi/utilities/sources/GcsEventsSource.java | 13 +-
.../hudi/utilities/sources/HiveIncrPullSource.java | 6 +-
.../apache/hudi/utilities/sources/JdbcSource.java | 3 +-
.../hudi/utilities/sources/ProtoKafkaSource.java | 8 +-
.../hudi/utilities/sources/PulsarSource.java | 5 +-
.../utilities/sources/debezium/DebeziumSource.java | 11 +-
.../utilities/sources/helpers/AvroConvertor.java | 45 +-
.../utilities/transform/ChainedTransformer.java | 38 +-
.../ErrorTableAwareChainedTransformer.java | 58 +
.../utilities/transform/FlatteningTransformer.java | 20 +-
.../transform/SqlFileBasedTransformer.java | 7 +-
.../transform/SqlQueryBasedTransformer.java | 26 +-
.../hudi/utilities/TestHiveIncrementalPuller.java | 2 +
.../apache/hudi/utilities/TestHoodieIndexer.java | 8 +-
.../deltastreamer/HoodieDeltaStreamerTestBase.java | 40 +-
.../MockConfigurationHotUpdateStrategy.java | 53 +
.../deltastreamer/TestHoodieDeltaStreamer.java | 204 +-
.../TestHoodieDeltaStreamerDAGExecution.java | 121 +
.../TestHoodieDeltaStreamerWithMultiWriter.java | 17 +-
.../TestHoodieMultiTableDeltaStreamer.java | 15 +
.../deltastreamer/TestSchedulerConfGenerator.java | 26 +
.../utilities/deltastreamer/TestTransformer.java | 7 +
.../multisync/TestMultipleMetaSync.java | 6 +-
.../functional/TestChainedTransformer.java | 3 +-
.../TestErrorTableAwareChainedTransformer.java | 144 +
.../offlinejob/HoodieOfflineJobTestBase.java | 126 +
.../offlinejob/TestHoodieClusteringJob.java | 116 +
.../offlinejob/TestHoodieCompactorJob.java | 124 +
.../utilities/sources/TestAvroKafkaSource.java | 4 +-
.../utilities/sources/TestJsonKafkaSource.java | 28 +-
.../debezium/TestAbstractDebeziumSource.java | 3 +-
.../sources/helpers/TestKafkaOffsetGen.java | 2 +-
.../utilities/testutils/UtilitiesTestBase.java | 16 +-
.../transform/TestSqlFileBasedTransformer.java | 4 +-
.../short_trip_uber_config.properties | 1 +
packaging/bundle-validation/Dockerfile | 2 +
packaging/bundle-validation/README.md | 18 +-
.../base/build_flink1146hive313spark302.sh | 26 +
.../base/build_flink1153hive313spark332.sh | 26 +
.../base/build_flink1170hive313spark332.sh | 27 +
.../base/build_flink1170hive313spark340.sh | 27 +
packaging/bundle-validation/ci_run.sh | 116 +-
.../bundle-validation/conf/hive-site.hive2.xml | 51 +
packaging/bundle-validation/validate.sh | 1 +
packaging/hudi-flink-bundle/pom.xml | 5 +-
packaging/hudi-hadoop-mr-bundle/pom.xml | 13 +
packaging/hudi-hive-sync-bundle/pom.xml | 3 +
packaging/hudi-integ-test-bundle/pom.xml | 5 +-
packaging/hudi-kafka-connect-bundle/pom.xml | 13 +
packaging/hudi-spark-bundle/pom.xml | 3 +
packaging/hudi-timeline-server-bundle/pom.xml | 9 +
packaging/hudi-utilities-bundle/pom.xml | 3 +
pom.xml | 279 +-
rfc/README.md | 1 +
rfc/rfc-68/1.jpg | Bin 0 -> 17762 bytes
rfc/rfc-68/2.jpg | Bin 0 -> 48766 bytes
rfc/rfc-68/3.jpg | Bin 0 -> 18669 bytes
rfc/rfc-68/4.jpg | Bin 0 -> 25297 bytes
rfc/rfc-68/rfc-68.md | 100 +
style/scalastyle.xml | 2 +-
718 files changed, 21520 insertions(+), 7148 deletions(-)
rename azure-pipelines.yml => azure-pipelines-20230430.yml (81%)
create mode 100644
hudi-aws/src/main/java/org/apache/hudi/config/GlueCatalogSyncClientConfig.java
create mode 100644
hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestMetadataCommand.java
copy
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/bootstrap/translator/{IdentityBootstrapPartitionPathTranslator.java
=> DecodedBootstrapPartitionPathTranslator.java} (74%)
copy
hudi-common/src/main/java/org/apache/hudi/common/util/collection/ClosableIterator.java
=>
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/utils/ClosableMergingIterator.java
(54%)
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/utils/MetadataTableUtils.java
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/AutoRecordGenWrapperAvroKeyGenerator.java
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataWriteUtils.java
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/FiveToSixUpgradeHandler.java
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/SixToFiveDowngradeHandler.java
create mode 100644
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/TestBulkInsertPartitioner.java
create mode 100644
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/utils/SparkPartitionUtils.java
create mode 100644
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/BucketBulkInsertPartitionerWithRows.java
create mode 100644
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/AutoRecordGenWrapperKeyGenerator.java
create mode 100644
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieMetadataBulkInsertPartitioner.java
create mode 100644
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/BucketBulkInsertDataInternalWriterHelper.java
create mode 100644
hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/SparkJdbcUtils.scala
copy hudi-common/src/main/java/org/apache/hudi/common/util/MapUtils.java =>
hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/HoodieSchemaUtils.scala
(54%)
create mode 100644
hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/execution/datasources/HoodieSparkPartitionedFileUtils.scala
create mode 100644
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestSparkHoodieMetadataBulkInsertPartitioner.java
create mode 100644
hudi-common/src/main/java/org/apache/hudi/common/config/TimestampKeyGeneratorConfig.java
copy
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/bootstrap/translator/IdentityBootstrapPartitionPathTranslator.java
=>
hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFileWriteCallback.java
(59%)
copy hudi-common/src/main/java/org/apache/hudi/{common/util/MapUtils.java =>
exception/HoodieIncrementalPathNotFoundException.java} (60%)
copy
hudi-common/src/main/java/org/apache/hudi/{common/model/HoodieTimelineTimeZone.java
=> exception/HoodieLogCompactException.java} (73%)
copy
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/exception/HoodieUpgradeDowngradeException.java
=>
hudi-common/src/main/java/org/apache/hudi/exception/HoodieMetaSyncException.java
(64%)
create mode 100644
hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieBootstrapRecordIterator.java
create mode 100644
hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieAdaptablePayloadDataGenerator.java
create mode 100644
hudi-common/src/test/java/org/apache/hudi/common/util/TestMapUtils.java
create mode 100644 hudi-common/src/test/resources/adaptable-payload.avsc
copy
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/bootstrap/translator/IdentityBootstrapPartitionPathTranslator.java
=>
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/DataModificationInfos.java
(52%)
copy
hudi-common/src/main/java/org/apache/hudi/common/model/HoodieTimelineTimeZone.java
=>
hudi-flink-datasource/hudi-flink1.13.x/src/main/java/org/apache/hudi/adapter/SupportsRowLevelDeleteAdapter.java
(65%)
copy
hudi-common/src/main/java/org/apache/hudi/common/model/HoodieTimelineTimeZone.java
=>
hudi-flink-datasource/hudi-flink1.13.x/src/main/java/org/apache/hudi/adapter/SupportsRowLevelUpdateAdapter.java
(60%)
create mode 100644
hudi-flink-datasource/hudi-flink1.13.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/EmptyColumnReader.java
copy
hudi-common/src/main/java/org/apache/hudi/common/model/HoodieTimelineTimeZone.java
=>
hudi-flink-datasource/hudi-flink1.14.x/src/main/java/org/apache/hudi/adapter/SupportsRowLevelDeleteAdapter.java
(65%)
copy
hudi-common/src/main/java/org/apache/hudi/common/model/HoodieTimelineTimeZone.java
=>
hudi-flink-datasource/hudi-flink1.14.x/src/main/java/org/apache/hudi/adapter/SupportsRowLevelUpdateAdapter.java
(60%)
create mode 100644
hudi-flink-datasource/hudi-flink1.14.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/EmptyColumnReader.java
copy
hudi-common/src/main/java/org/apache/hudi/common/model/HoodieTimelineTimeZone.java
=>
hudi-flink-datasource/hudi-flink1.15.x/src/main/java/org/apache/hudi/adapter/SupportsRowLevelDeleteAdapter.java
(65%)
copy
hudi-common/src/main/java/org/apache/hudi/common/model/HoodieTimelineTimeZone.java
=>
hudi-flink-datasource/hudi-flink1.15.x/src/main/java/org/apache/hudi/adapter/SupportsRowLevelUpdateAdapter.java
(60%)
create mode 100644
hudi-flink-datasource/hudi-flink1.15.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/EmptyColumnReader.java
copy
hudi-common/src/main/java/org/apache/hudi/common/model/HoodieTimelineTimeZone.java
=>
hudi-flink-datasource/hudi-flink1.16.x/src/main/java/org/apache/hudi/adapter/SupportsRowLevelDeleteAdapter.java
(65%)
copy
hudi-common/src/main/java/org/apache/hudi/common/model/HoodieTimelineTimeZone.java
=>
hudi-flink-datasource/hudi-flink1.16.x/src/main/java/org/apache/hudi/adapter/SupportsRowLevelUpdateAdapter.java
(60%)
create mode 100644
hudi-flink-datasource/hudi-flink1.16.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/EmptyColumnReader.java
copy
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/bootstrap/translator/IdentityBootstrapPartitionPathTranslator.java
=>
hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/adapter/SupportsRowLevelDeleteAdapter.java
(50%)
create mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/adapter/SupportsRowLevelUpdateAdapter.java
create mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/EmptyColumnReader.java
create mode 100644
hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/avro/HoodieAvroParquetReader.java
create mode 100644
hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/avro/HoodieTimestampAwareParquetInputFormat.java
create mode 100644
hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/shims/Hive2Shim.java
create mode 100644
hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/shims/Hive3Shim.java
copy
hudi-common/src/main/java/org/apache/hudi/common/model/HoodieTimelineTimeZone.java
=>
hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/shims/HiveShim.java
(70%)
copy
hudi-common/src/main/java/org/apache/hudi/common/model/HoodieTimelineTimeZone.java
=>
hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/shims/HiveShims.java
(73%)
copy
hudi-common/src/main/java/org/apache/hudi/common/model/HoodieTimelineTimeZone.java
=> hudi-hadoop-mr/src/test/resources/test_timetype.avsc (58%)
create mode 100644
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/AutoRecordKeyGenerationUtils.scala
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestBootstrapRead.java
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestGlobalIndexEnableUpdatePartitions.java
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/CommonOptionUtils.scala
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestAutoGenerationOfRecordKeys.scala
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSparkDataSource.scala
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSparkDataSourceDAGExecution.scala
copy hudi-common/src/main/java/org/apache/hudi/common/util/MapUtils.java =>
hudi-spark-datasource/hudi-spark2/src/main/scala/org/apache/spark/sql/HoodieSpark2SchemaUtils.scala
(63%)
create mode 100644
hudi-spark-datasource/hudi-spark2/src/main/scala/org/apache/spark/sql/execution/datasources/HoodieSpark2PartitionedFileUtils.scala
copy hudi-spark-datasource/{hudi-spark3.2.x =>
hudi-spark3-common}/src/test/java/org/apache/hudi/spark3/internal/TestHoodieBulkInsertDataInternalWriter.java
(94%)
copy hudi-spark-datasource/{hudi-spark3.2.x =>
hudi-spark3-common}/src/test/java/org/apache/hudi/spark3/internal/TestHoodieDataSourceInternalBatchWrite.java
(95%)
copy hudi-spark-datasource/{hudi-spark3.2.x =>
hudi-spark3-common}/src/test/java/org/apache/hudi/spark3/internal/TestReflectUtil.java
(92%)
copy hudi-spark-datasource/{hudi-spark3.1.x => hudi-spark3.0.x}/pom.xml (94%)
copy
hudi-spark-datasource/{hudi-spark3.2plus-common/src/main/java/org/apache/spark/sql/execution/datasources/parquet/Spark32PlusHoodieVectorizedParquetRecordReader.java
=>
hudi-spark3.0.x/src/main/java/org/apache/spark/sql/execution/datasources/parquet/Spark30HoodieVectorizedParquetRecordReader.java}
(94%)
create mode 100644
hudi-spark-datasource/hudi-spark3.0.x/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
copy
hudi-spark-datasource/{hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_3AvroDeserializer.scala
=> hudi-spark3.0.x/src/main/scala/org/apache/hudi/Spark30DefaultSource.scala}
(61%)
copy
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/bootstrap/translator/BootstrapPartitionPathTranslator.java
=>
hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/hudi/Spark30HoodieFileScanRDD.scala
(51%)
copy
hudi-spark-datasource/{hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_3AvroDeserializer.scala
=>
hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/HoodieSpark30CatalogUtils.scala}
(61%)
copy
hudi-spark-datasource/{hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/HoodieSpark31CatalystExpressionUtils.scala
=>
hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/HoodieSpark30CatalystExpressionUtils.scala}
(57%)
copy
hudi-spark-datasource/{hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/HoodieSpark31CatalystPlanUtils.scala
=>
hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/HoodieSpark30CatalystPlanUtils.scala}
(82%)
copy hudi-common/src/main/java/org/apache/hudi/common/util/MapUtils.java =>
hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/HoodieSpark30SchemaUtils.scala
(63%)
copy
hudi-spark-datasource/{hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/adapter/Spark3_1Adapter.scala
=>
hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/adapter/Spark3_0Adapter.scala}
(55%)
copy hudi-spark-datasource/{hudi-spark3.1.x =>
hudi-spark3.0.x}/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
(98%)
copy hudi-spark-datasource/{hudi-spark3.1.x =>
hudi-spark3.0.x}/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala
(97%)
create mode 100644
hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala
copy
hudi-spark-datasource/{hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_1AvroDeserializer.scala
=>
hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_0AvroDeserializer.scala}
(90%)
copy
hudi-spark-datasource/{hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_3AvroDeserializer.scala
=>
hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_0AvroSerializer.scala}
(70%)
copy hudi-spark-datasource/{hudi-spark3.1.x =>
hudi-spark3.0.x}/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/HStatments.scala
(100%)
create mode 100644
hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/execution/datasources/HoodieSpark30PartitionedFileUtils.scala
create mode 100644
hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/execution/datasources/Spark30NestedSchemaPruning.scala
copy
hudi-spark-datasource/{hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark32PlusHoodieParquetFileFormat.scala
=>
hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark30HoodieParquetFileFormat.scala}
(62%)
create mode 100644
hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/hudi/Spark30ResolveHudiAlterTableCommand.scala
copy
hudi-spark-datasource/{hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/hudi/command/Spark31AlterTableCommand.scala
=>
hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/hudi/command/Spark30AlterTableCommand.scala}
(96%)
create mode 100644
hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark30SqlAstBuilder.scala
copy
hudi-spark-datasource/{hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_1AvroDeserializer.scala
=>
hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_0ExtendedSqlParser.scala}
(51%)
copy hudi-spark-datasource/{hudi-spark3.1.x =>
hudi-spark3.0.x}/src/main/scala/org/apache/spark/sql/vectorized/ColumnarUtils.scala
(100%)
copy hudi-common/src/main/java/org/apache/hudi/common/util/MapUtils.java =>
hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/HoodieSpark31SchemaUtils.scala
(63%)
create mode 100644
hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/HoodieSpark31PartitionedFileUtils.scala
copy hudi-common/src/main/java/org/apache/hudi/common/util/MapUtils.java =>
hudi-spark-datasource/hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/HoodieSpark32SchemaUtils.scala
(63%)
create mode 100644
hudi-spark-datasource/hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/execution/datasources/HoodieSpark32PartitionedFileUtils.scala
copy
hudi-spark-datasource/{hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark32PlusHoodieParquetFileFormat.scala
=>
hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark32HoodieParquetFileFormat.scala}
(97%)
copy hudi-common/src/main/java/org/apache/hudi/common/util/MapUtils.java =>
hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/HoodieSpark33SchemaUtils.scala
(63%)
create mode 100644
hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/HoodieSpark33PartitionedFileUtils.scala
copy
hudi-spark-datasource/{hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark32PlusHoodieParquetFileFormat.scala
=>
hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark33HoodieParquetFileFormat.scala}
(93%)
copy hudi-spark-datasource/{hudi-spark3.3.x => hudi-spark3.4.x}/pom.xml (92%)
copy hudi-spark-datasource/{hudi-spark3.2.x =>
hudi-spark3.4.x}/src/main/antlr4/imports/SqlBase.g4 (100%)
copy hudi-spark-datasource/{hudi-spark3.2.x =>
hudi-spark3.4.x}/src/main/antlr4/org/apache/hudi/spark/sql/parser/HoodieSqlBase.g4
(100%)
copy hudi-spark-datasource/{hudi-spark3.2.x =>
hudi-spark3.4.x}/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
(100%)
create mode 100644
hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/hudi/Spark34HoodieFileScanRDD.scala
copy
hudi-spark-datasource/{hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_3AvroDeserializer.scala
=>
hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/HoodieSpark34CatalogUtils.scala}
(61%)
copy
hudi-spark-datasource/{hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/HoodieSpark33CatalystExpressionUtils.scala
=>
hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/HoodieSpark34CatalystExpressionUtils.scala}
(80%)
copy
hudi-spark-datasource/{hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/HoodieSpark33CatalystPlanUtils.scala
=>
hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/HoodieSpark34CatalystPlanUtils.scala}
(84%)
copy hudi-common/src/main/java/org/apache/hudi/common/util/MapUtils.java =>
hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/HoodieSpark34SchemaUtils.scala
(64%)
copy
hudi-spark-datasource/{hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/adapter/Spark3_3Adapter.scala
=>
hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/adapter/Spark3_4Adapter.scala}
(52%)
copy hudi-spark-datasource/{hudi-spark3.3.x =>
hudi-spark3.4.x}/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
(100%)
copy hudi-spark-datasource/{hudi-spark3.3.x =>
hudi-spark3.4.x}/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala
(98%)
copy hudi-spark-datasource/{hudi-spark3.3.x =>
hudi-spark3.4.x}/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala (100%)
copy
hudi-spark-datasource/{hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_3AvroDeserializer.scala
=>
hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_4AvroDeserializer.scala}
(83%)
copy
hudi-spark-datasource/{hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_3AvroDeserializer.scala
=>
hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_4AvroSerializer.scala}
(70%)
create mode 100644
hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/execution/datasources/HoodieSpark34PartitionedFileUtils.scala
create mode 100644
hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/execution/datasources/Spark34NestedSchemaPruning.scala
rename
hudi-spark-datasource/{hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark32PlusHoodieParquetFileFormat.scala
=>
hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark34HoodieParquetFileFormat.scala}
(92%)
create mode 100644
hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/hudi/Spark34ResolveHudiAlterTableCommand.scala
create mode 100644
hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_4ExtendedSqlAstBuilder.scala
create mode 100644
hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_4ExtendedSqlParser.scala
rename hudi-spark-datasource/{hudi-spark3.2.x =>
hudi-spark3.4.x}/src/test/java/org/apache/hudi/spark3/internal/TestHoodieBulkInsertDataInternalWriter.java
(100%)
rename hudi-spark-datasource/{hudi-spark3.2.x =>
hudi-spark3.4.x}/src/test/java/org/apache/hudi/spark3/internal/TestHoodieDataSourceInternalBatchWrite.java
(100%)
rename hudi-spark-datasource/{hudi-spark3.2.x =>
hudi-spark3.4.x}/src/test/java/org/apache/hudi/spark3/internal/TestReflectUtil.java
(100%)
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/TableSizeStats.java
copy
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/bootstrap/translator/BootstrapPartitionPathTranslator.java
=>
hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/ConfigurationHotUpdateStrategy.java
(54%)
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/ConfigurationHotUpdateStrategyUtils.java
copy
hudi-common/src/main/java/org/apache/hudi/common/model/HoodieTimelineTimeZone.java
=>
hudi-utilities/src/main/java/org/apache/hudi/utilities/exception/HoodieDeltaStreamerWriteException.java
(73%)
copy
hudi-common/src/main/java/org/apache/hudi/common/model/HoodieTimelineTimeZone.java
=>
hudi-utilities/src/main/java/org/apache/hudi/utilities/exception/HoodieReadFromSourceException.java
(72%)
copy
hudi-common/src/main/java/org/apache/hudi/common/model/HoodieTimelineTimeZone.java
=>
hudi-utilities/src/main/java/org/apache/hudi/utilities/exception/HoodieSchemaFetchException.java
(70%)
copy
hudi-common/src/main/java/org/apache/hudi/common/model/HoodieTimelineTimeZone.java
=>
hudi-utilities/src/main/java/org/apache/hudi/utilities/exception/HoodieTransformException.java
(70%)
copy
hudi-common/src/main/java/org/apache/hudi/common/model/HoodieTimelineTimeZone.java
=>
hudi-utilities/src/main/java/org/apache/hudi/utilities/exception/HoodieTransformExecutionException.java
(73%)
copy
hudi-common/src/main/java/org/apache/hudi/common/model/HoodieTimelineTimeZone.java
=>
hudi-utilities/src/main/java/org/apache/hudi/utilities/exception/HoodieTransformPlanException.java
(71%)
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/ErrorTableAwareChainedTransformer.java
create mode 100644
hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/MockConfigurationHotUpdateStrategy.java
create mode 100644
hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamerDAGExecution.java
create mode 100644
hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestErrorTableAwareChainedTransformer.java
create mode 100644
hudi-utilities/src/test/java/org/apache/hudi/utilities/offlinejob/HoodieOfflineJobTestBase.java
create mode 100644
hudi-utilities/src/test/java/org/apache/hudi/utilities/offlinejob/TestHoodieClusteringJob.java
create mode 100644
hudi-utilities/src/test/java/org/apache/hudi/utilities/offlinejob/TestHoodieCompactorJob.java
create mode 100755
packaging/bundle-validation/base/build_flink1146hive313spark302.sh
create mode 100755
packaging/bundle-validation/base/build_flink1153hive313spark332.sh
create mode 100755
packaging/bundle-validation/base/build_flink1170hive313spark332.sh
create mode 100755
packaging/bundle-validation/base/build_flink1170hive313spark340.sh
create mode 100644 packaging/bundle-validation/conf/hive-site.hive2.xml
create mode 100644 rfc/rfc-68/1.jpg
create mode 100644 rfc/rfc-68/2.jpg
create mode 100644 rfc/rfc-68/3.jpg
create mode 100644 rfc/rfc-68/4.jpg
create mode 100644 rfc/rfc-68/rfc-68.md
