This is an automated email from the ASF dual-hosted git repository.
voonhous pushed a change to branch variant-intro-shredded-avro-support
in repository https://gitbox.apache.org/repos/asf/hudi.git
discard f7c5fe312360 test(variant): gate shredded variant read-back to Spark
4.1+
discard c14ae042263f refactor(variant): address write-support and test review
nits
discard 72fcf98e659c fix(spark): do not mutate session SQLConf for variant
allow-reading-shredded
omit cebbf13eb2b1 refactor(variant): collapse parallel shredded-variant
arrays into a map
omit c96da1205047 test(variant): cover read-then-reshred guard in
HoodieAvroWriteSupport
omit 44f3b123a59c refactor(variant): guard read-then-reshred path, address
review nits
omit e3e63c98ca0e Add todo
omit f4b6b5112643 fix: build Avro writer Parquet schema from effective
shredded schema
omit eb694ef3161a feat(spark): set variant allow-reading-shredded flag in
Spark 4.1 parquet reader
omit c6abcaf6c704 test(spark): cover variant allow-reading-shredded
precedence resolution
omit 7e9413c27400 fix: fail fast when variant shredding requires a provider
but none is available
omit 45babcb31cf3 fix(spark): respect explicit
spark.sql.variant.allowReadingShredded in precedence
omit ab1eb72db828 fix: avoid mutating shared config Properties when
injecting shredding provider
omit ae4fab2e5ef9 test: use GroupType.containsField instead of try/catch on
getFieldIndex
omit b2025f8fb454 refactor: reduce surface of test-only variant
force-shredding config
omit 64ee9e049ee0 refactor: move VariantShreddingProvider to hudi-common
omit a35bc88b313e fix(spark): avoid Spark 4.0-only SQLConf field in
cross-version module
omit b20951c68388 feat(schema): Add read + write support for shredded for
AVRO
add cb14ca727583 refactor(core): Unify record key/index key splitting and
extraction (#18842)
add 9cddb237be14 fix(streamer): Use checkpoint V1 for non-incremental
streamer sources (#18896)
add 6f81c51fae1d fix(flink): Use the execution mode without rocksdb cache
by default … (#18894)
add 091caadfac40 feat: add lance format for Flink MOR table (#18911)
add 5182bf851569 feat(flink): support remote partitioner for simple bucket
index (#18897)
add 77f5851a5d53 feat(spark): add Spark 4.2 support (#18621)
add 9ca18678b8f0 chore: [MINOR] Update DOAP with 0.14.2 Release (#18924)
add c5e3cd522692 chore: remove the CI validation for Flink1.17 (#18928)
add aac975c9ff34 feat(flink): Support reading VECTOR columns from Parquet
and Avro format (#18723)
add f4816ce82109 fix(flink): avoid repeated timeline reload for unchanged
lookup table commits (#18930)
add eb863d84e42b chore: drop support for Flink 1.17 (#18920)
add 782bf654ee4c fix(common): Close log writer output stream on append
failure (#18909)
add 348e7f13a7fb feat(flink): Add validation to reject multiple writers
for flink RLI writes (#18946)
add 1fd2c3671a80 fix(metrics): NPE handling when hudi metrics is disabled
(#18947)
add 86d1650d0f19 feat(flink): Backport Flink 2.1 Dremel nested Parquet
reader to hudi-flink1.20 (#18953)
add 89332241ea0b fix(clustering): retain missing partitions in
selected/regex incremental scheduling (#18945)
add 21121acf3561 fix(flink): Close write client properly in
DefaultCleanHandler (#18940)
add 17b396ebc87f feat(mdt): Support writing bloom filter for hfile log
block (#18936)
add f1c6fe4d0932 feat(flink): Backport Flink 2.1 Dremel nested Parquet
reader to hudi-flink2.1.x (#18960)
add 7f3676be3979 feat(flink): Backport Flink 2.1 Dremel nested Parquet
reader to hudi-… (#18959)
add 466ae49c7767 chore: clean and fix the rfc readme (#18970)
add 84c1f591481d docs: RFC-106 - Record Level and Secondary Index Support
for Flink Writers (#17610)
add 91f3d227264d perf(clean): Avoid extra getPathInfo RPC per file during
clean execution (#18963)
add 173e76eb3ba8 [HUDI] Add input records caching to
HoodieGlobalSimpleIndex (#18921)
add 182428f803d0 fix(spark): catch HoodieSchemaNotFoundException in 3-arg
DefaultSource.createRelation (#18977)
add 612e327b51fb refactor: Add Lombok annotations to hudi-common module
(part 7) (#18944)
add 1f20fd42e2b9 fix(flink): relocate org.apache.flink.dropwizard (#18982)
add a032431c81aa feat: add more metrics for delta streamer (#18085)
add cde86ffd64a4 feat: Add HUDI version and engine properties to commit
metadata (#18183)
add fedf24ffc933 Fix NPE in getInputFileSlices when RO path filter returns
empty partition (#18639)
add 0915f7776be2 fix: Isolate classloader-aware parallel execution in
HoodiePreCommitValidatorEngineContext (#18585)
add 6428cb258366 perf(spark): Resolve drop-partition-columns projection
once per writer instead of per row (#18972)
add 97c03f75c737 fix(spark): strip _hoodie_* meta columns from CDC
before/after images (#18948)
add 27daf3df0757 perf(spark): Parse bucket index hash-field config once
instead of per… (#18979)
add 73ff4ea3aa4f perf(metadata): Parse RLI instant time once per batch
instead of per … (#18965)
add b75a5acc8d10 test(trino): de-flake TestHudi*FileOperations by
disabling async table statistics (#18995)
add 7c934f10f89b feat(spark): Reintroduce Apache Spark 3.3 integration
support (#18956)
add 097dd4a4c8df perf(metadata): Avoid per-record enum-array clone and
string parse when materializing MDT records (#18997)
add 1a8c8ad1f851 [MINOR] Disable codecov-commenter PR comment (comment:
false) (#18994)
add 9c66b47b474e docs: Claim RFC-107 Dynamic Partitioned Cache for Flink
upsert (#18998)
add 7eaa1c3630d2 [MINOR] Cap UT_FT_10 Azure install to -T 2 to avoid flaky
compiler heap OOM (#19008)
add dcfe9d403ecd refactor(metadata): Replace misused stream reduce with a
plain for-loop (#18532)
add e1193c385be6 perf(io): Derive log file size from AppendResult on
append-handle close (#19002)
add b140d249b6cc refactor: Add Lombok annotations to hudi-common module
(part 8) (#18957)
add f3ccf1b3c31b [MINOR] Wait for ZK connection in lock provider to
de-flake direct-marker detection test (#19014)
add e853440bac04 fix(flink): fix the mor small file record size estimation
(#18991)
add b5879f95ad14 perf(kafka-connect): reuse AvroConvertor across records
in the connect writer (#19015)
add a4f0e261b950 fix(spark): Propagate merge configs to file group reader
during clustering (#19007)
add 8389464e1613 perf(kafka-connect): memoize file id per partition path
in the connect writer (#19016)
add 8c2eacffb160 perf(kafka-connect): use a pre-sized ArrayList when
flushing buffered records (#19017)
add 177d34a6300d feat(schema): Add read + write support for shredded for
AVRO
add b6e6b8b92f80 fix(spark): avoid Spark 4.0-only SQLConf field in
cross-version module
add 6c06bd4d6fed refactor: move VariantShreddingProvider to hudi-common
add 7a654bf063ba refactor: reduce surface of test-only variant
force-shredding config
add c51c0ff38df3 test: use GroupType.containsField instead of try/catch on
getFieldIndex
add 029d2db3b36f fix: avoid mutating shared config Properties when
injecting shredding provider
add 4236e8bed291 fix(spark): respect explicit
spark.sql.variant.allowReadingShredded in precedence
add 3d3b86447ab3 fix: fail fast when variant shredding requires a provider
but none is available
add e26067bc5b65 test(spark): cover variant allow-reading-shredded
precedence resolution
add 834268b16328 feat(spark): set variant allow-reading-shredded flag in
Spark 4.1 parquet reader
add 6ee5ce900df7 fix: build Avro writer Parquet schema from effective
shredded schema
add e5799ab0aa6c Add todo
add 0b678be5efec refactor(variant): guard read-then-reshred path, address
review nits
add 9e03705290e6 test(variant): cover read-then-reshred guard in
HoodieAvroWriteSupport
add 75d17910735d refactor(variant): collapse parallel shredded-variant
arrays into a map
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (f7c5fe312360)
\
N -- N -- N refs/heads/variant-intro-shredded-avro-support
(75d17910735d)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.asf.yaml | 16 +-
.codecov.yml | 8 +-
.github/workflows/bot.yml | 60 +-
.github/workflows/maven_artifact_validation.yml | 6 +-
.github/workflows/release_candidate_validation.yml | 6 +-
README.md | 6 +-
azure-pipelines-20230430.yml | 8 +-
doap_HUDI.rdf | 7 +
.../org/apache/hudi/client/BaseHoodieClient.java | 4 +
.../hudi/client/BaseHoodieTableServiceClient.java | 2 +
.../apache/hudi/client/BaseHoodieWriteClient.java | 1 +
.../hudi/client/CommitMetadataProperties.java | 145 ++++
.../client/embedded/EmbeddedTimelineService.java | 22 +-
.../lock/BaseZookeeperBasedLockProvider.java | 27 +-
.../org/apache/hudi/config/HoodieIndexConfig.java | 11 +-
.../org/apache/hudi/config/HoodieWriteConfig.java | 4 +
.../apache/hudi/index/bucket/BucketIdentifier.java | 2 +-
.../hudi/index/bucket/HoodieBucketIndex.java | 4 +-
.../hudi/index/simple/HoodieGlobalSimpleIndex.java | 9 +-
.../org/apache/hudi/io/HoodieAppendHandle.java | 31 +-
.../apache/hudi/keygen/CustomAvroKeyGenerator.java | 6 +-
.../java/org/apache/hudi/keygen/KeyGenUtils.java | 26 +-
.../metadata/HoodieBackedTableMetadataWriter.java | 6 +-
.../hudi/metadata/HoodieMetadataWriteUtils.java | 11 +-
.../apache/hudi/metadata/RecordIndexMapper.java | 9 +-
.../org/apache/hudi/metrics/HoodieMetrics.java | 11 +-
.../table/BucketIndexBulkInsertPartitioner.java | 4 +-
.../table/action/clean/CleanActionExecutor.java | 45 +-
.../strategy/ClusteringExecutionStrategy.java | 6 +-
.../cluster/strategy/ClusteringPlanStrategy.java | 17 +
.../PartitionAwareClusteringPlanStrategy.java | 12 +-
.../hudi/client/TestBaseHoodieWriteClient.java | 5 +-
.../hudi/client/TestCommitMetadataProperties.java | 170 +++++
.../embedded/TestEmbeddedTimelineService.java | 24 +-
.../TestZookeeperBasedLockProvider.java | 21 +
.../apache/hudi/config/TestHoodieWriteConfig.java | 20 +
.../org/apache/hudi/keygen/TestKeyGenUtils.java | 21 +
.../metadata/TestHoodieMetadataWriteUtils.java | 18 +
.../org/apache/hudi/metrics/TestHoodieMetrics.java | 316 +++++++++
.../TestPartitionAwareClusteringPlanStrategy.java | 34 +-
.../apache/hudi/util/AvroToRowDataConverters.java | 102 ++-
.../org/apache/hudi/util/RowDataQueryContexts.java | 4 +-
.../apache/hudi/util/VectorConversionUtils.java | 310 +++++++++
.../hudi/util/TestAvroToRowDataConverters.java | 123 ++++
.../client/common/HoodieJavaEngineContext.java | 25 +
...SparkJobConsistentHashingExecutionStrategy.java | 9 +-
...arkConsistentBucketDuplicateUpdateStrategy.java | 4 +-
.../client/common/HoodieSparkEngineContext.java | 14 +
.../hudi/client/utils/SparkValidatorUtils.java | 38 +-
.../client/validator/SparkPreCommitValidator.java | 12 +
...ntBucketIndexBulkInsertPartitionerWithRows.java | 9 +-
.../hudi/io/storage/HoodieSparkParquetReader.java | 3 +-
.../hudi/io/storage/VectorConversionUtils.java | 59 +-
.../org/apache/hudi/keygen/CustomKeyGenerator.java | 8 +-
.../BucketBulkInsertDataInternalWriterHelper.java | 11 +-
.../commit/BulkInsertDataInternalWriterHelper.java | 64 +-
...ntBucketBulkInsertDataInternalWriterHelper.java | 2 +-
.../action/commit/SparkBucketIndexPartitioner.java | 9 +-
.../SparkPartitionBucketIndexPartitioner.java | 9 +-
.../scala/org/apache/hudi/HoodieSparkUtils.scala | 4 +
.../org/apache/hudi/SparkAdapterSupport.scala | 8 +-
.../SparkFileFormatInternalRowReaderContext.scala | 5 +-
.../apache/spark/sql/BucketPartitionUtils.scala | 6 +-
...erBasedDetectionStrategyWithZKLockProvider.java | 29 +-
.../hudi/client/utils/TestSparkValidatorUtils.java | 209 ++++++
.../validator/TestSparkPreCommitValidator.java | 143 ++++
.../hudi/common/table/log/TestLogReaderUtils.java | 58 ++
.../table/functional/TestCleanActionExecutor.java | 20 +-
hudi-common/pom.xml | 4 +-
.../org/apache/hudi/BaseHoodieTableFileIndex.java | 11 +-
.../main/java/org/apache/hudi/HoodieVersion.java | 43 +-
.../hudi/common/config/HoodieStorageConfig.java | 12 +
.../engine/ExecutorServiceBasedEngineContext.java | 290 ++++++++
.../hudi/common/engine/HoodieEngineContext.java | 16 +
.../apache/hudi/common/model/HoodieFileFormat.java | 6 +-
.../hudi/common/table/HoodieTableConfig.java | 56 +-
.../hudi/common/table/HoodieTableMetaClient.java | 159 ++---
.../hudi/common/table/HoodieTableVersion.java | 24 +-
.../hudi/common/table/TableSchemaResolver.java | 12 +-
.../hudi/common/table/cdc/HoodieCDCExtractor.java | 6 +-
.../hudi/common/table/cdc/HoodieCDCFileSplit.java | 24 +-
.../hudi/common/table/cdc/HoodieCDCOperation.java | 14 +-
.../hudi/common/table/checkpoint/Checkpoint.java | 21 +-
.../common/table/checkpoint/CheckpointUtils.java | 32 +-
.../table/log/AbstractHoodieLogRecordScanner.java | 92 +--
.../table/log/BaseHoodieLogRecordReader.java | 103 +--
.../apache/hudi/common/table/log/FullKeySpec.java | 15 +-
.../hudi/common/table/log/HoodieLogFileReader.java | 26 +-
.../hudi/common/table/log/HoodieLogFormat.java | 12 +-
.../common/table/log/HoodieLogFormatReader.java | 8 +-
.../table/log/HoodieMergedLogRecordReader.java | 20 +-
.../table/log/HoodieMergedLogRecordScanner.java | 20 +-
.../apache/hudi/common/table/log/InstantRange.java | 17 +-
.../common/table/log/block/HoodieCommandBlock.java | 7 +-
.../common/table/log/block/HoodieDataBlock.java | 15 +-
.../common/table/log/block/HoodieDeleteBlock.java | 6 +-
.../table/log/block/HoodieHFileDataBlock.java | 23 +-
.../common/table/log/block/HoodieLogBlock.java | 94 +--
.../hudi/common/table/read/BufferedRecord.java | 47 +-
.../table/read/BufferedRecordMergerFactory.java | 7 +-
.../hudi/common/table/read/DeleteContext.java | 22 +-
.../table/read/IncrementalQueryAnalyzer.java | 70 +-
.../buffer/DefaultFileGroupRecordBufferLoader.java | 8 +-
.../table/read/buffer/FileGroupRecordBuffer.java | 19 +-
.../buffer/PositionBasedFileGroupRecordBuffer.java | 15 +-
.../common/table/timeline/BaseHoodieTimeline.java | 13 +-
.../hudi/common/table/timeline/HoodieInstant.java | 33 +-
.../hudi/common/table/timeline/LSMTimeline.java | 11 +-
.../table/timeline/MetadataConversionUtils.java | 11 +-
.../common/table/timeline/TimeGeneratorBase.java | 12 +-
.../common/table/timeline/TimelineDiffHelper.java | 46 +-
.../hudi/common/table/timeline/TimelineLayout.java | 58 +-
.../hudi/common/table/timeline/TimelineUtils.java | 13 +-
.../timeline/versioning/TimelineLayoutVersion.java | 8 +-
.../timeline/versioning/v1/ActiveTimelineV1.java | 45 +-
.../versioning/v1/ArchivedTimelineLoaderV1.java | 10 +-
.../timeline/versioning/v1/ArchivedTimelineV1.java | 11 +-
.../versioning/v1/CommitMetadataSerDeV1.java | 5 +-
.../versioning/v1/CompletionTimeQueryViewV1.java | 9 +-
.../timeline/versioning/v2/ActiveTimelineV2.java | 48 +-
.../versioning/v2/CompletionTimeQueryViewV2.java | 7 +-
.../table/view/AbstractTableFileSystemView.java | 52 +-
.../common/table/view/FileSystemViewManager.java | 27 +-
.../table/view/HoodieTableFileSystemView.java | 15 +-
.../IncrementalTimelineSyncFileSystemView.java | 56 +-
.../table/view/PriorityBasedFileSystemView.java | 25 +-
.../view/RemoteHoodieTableFileSystemView.java | 6 +-
.../table/view/RocksDbBasedFileSystemView.java | 50 +-
.../view/SpillableMapBasedFileSystemView.java | 8 +-
.../org/apache/hudi/common/util/ConfigUtils.java | 2 +
.../org/apache/hudi/common/util/HFileUtils.java | 49 ++
.../apache/hudi/common/util/HoodieVectorUtils.java | 110 +++
.../org/apache/hudi/exception/ExceptionUtil.java | 18 +-
.../hudi/io/storage/HoodieFileReaderFactory.java | 4 +-
.../hudi/io/storage/HoodieFileWriterFactory.java | 2 +-
.../io/storage/HoodieNativeAvroHFileReader.java | 4 +-
.../hudi/metadata/BaseFileRecordParsingUtils.java | 3 +-
.../hudi/metadata/HoodieMetadataPayload.java | 42 +-
.../hudi/metadata/HoodieTableMetadataUtil.java | 58 +-
.../hudi/metadata/MetadataPartitionType.java | 20 +-
.../apache/hudi/BaseHoodieTableFileIndexTest.java | 69 ++
.../common/config/TestHoodieStorageConfig.java | 14 +
.../table/checkpoint/TestCheckpointUtils.java | 66 +-
.../apache/hudi/common/util/TestConfigUtils.java | 6 +-
.../apache/hudi/common/util/TestHFileUtils.java | 91 +++
.../apache/hudi/exception/TestExceptionUtil.java | 20 +
.../hudi/metadata/TestHoodieTableMetadataUtil.java | 80 +++
.../hudi/examples/common/RandomJsonSource.java | 4 +-
.../hudi/examples/common/TestRandomJsonSource.java | 76 ++
.../apache/hudi/configuration/FlinkOptions.java | 1 +
.../apache/hudi/configuration/OptionsResolver.java | 82 ++-
.../sink/bucket/BucketStreamWriteFunction.java | 13 +
.../bucket/ConsistentBucketAssignFunction.java | 2 +-
.../ConsistentBucketStreamWriteFunction.java | 4 +-
.../sink/compact/handler/DefaultCleanHandler.java | 2 +-
.../sink/partitioner/BucketAssignFunction.java | 8 +-
.../BucketIndexPartitionerFactory.java} | 28 +-
.../partitioner/BucketIndexRemotePartitioner.java | 112 +++
.../partitioner/GlobalRecordIndexPartitioner.java | 7 +
.../partitioner/MinibatchBucketAssignFunction.java | 8 +-
.../sink/partitioner/RecordIndexPartitioner.java | 27 +-
.../index/GlobalRecordLevelIndexBackend.java | 45 +-
.../hudi/sink/partitioner/index/IndexRowUtils.java | 4 +-
.../sink/partitioner/index/IndexWriteFunction.java | 4 +-
.../partitioner/profile/DeltaWriteProfile.java | 36 +-
.../sink/partitioner/profile/WriteProfile.java | 60 +-
.../java/org/apache/hudi/sink/utils/Pipelines.java | 13 +-
.../enumerator/AbstractHoodieSplitEnumerator.java | 2 +-
.../function/HoodieCdcSplitReaderFunction.java | 8 +-
.../source/split/HoodieSourceSplitSerializer.java | 12 +-
.../hudi/source/stats/ColumnStatsSchemas.java | 7 +-
.../apache/hudi/source/stats/FileStatsIndex.java | 4 +-
.../org/apache/hudi/table/HoodieTableFactory.java | 13 +-
.../org/apache/hudi/table/HoodieTableSource.java | 20 +-
.../hudi/table/catalog/HoodieHiveCatalog.java | 2 +-
.../table/format/FlinkRowDataReaderContext.java | 3 +-
.../org/apache/hudi/table/format/FormatUtils.java | 28 +
.../table/format/HoodieRowDataParquetReader.java | 13 +-
.../apache/hudi/table/format/cdc/CdcIterators.java | 2 +-
.../table/format/cow/CopyOnWriteInputFormat.java | 36 +-
.../table/format/mor/MergeOnReadInputFormat.java | 5 +
.../hudi/table/lookup/HoodieLookupFunction.java | 13 +-
.../java/org/apache/hudi/util/DataTypeUtils.java | 46 ++
.../org/apache/hudi/util/FlinkWriteClients.java | 52 +-
.../java/org/apache/hudi/util/StreamerUtil.java | 9 +-
.../hudi/configuration/TestOptionsResolver.java | 61 +-
.../hudi/sink/bucket/ITTestBucketStreamWrite.java | 71 ++
.../compact/handler/TestDefaultCleanHandler.java} | 29 +-
.../hudi/sink/partitioner/TestBucketAssigner.java | 105 ++-
.../TestBucketIndexPartitionerFactory.java | 60 ++
.../TestBucketIndexRemotePartitioner.java | 98 +++
.../TestMinibatchBucketAssignFunction.java | 80 +++
.../partitioner/TestRecordIndexPartitioner.java | 1 +
.../index/TestGlobalRecordLevelIndexBackend.java | 5 +-
.../hudi/source/TestIncrementalInputSplits.java | 4 +-
.../source/TestStreamReadMonitoringFunction.java | 4 +-
.../TestHoodieContinuousSplitEnumerator.java | 4 +-
.../hudi/source/split/TestHoodieSourceSplit.java | 14 +-
.../split/TestHoodieSourceSplitSerializer.java | 76 +-
.../hudi/table/ITTestDynamicBucketStreamWrite.java | 7 +-
.../apache/hudi/table/ITTestHoodieDataSource.java | 78 ++-
.../hudi/table/ITTestSchemaEvolutionBySQL.java | 7 -
.../ITTestVectorCrossEngineCompatibility.java | 74 +-
.../apache/hudi/table/TestHoodieTableFactory.java | 54 +-
.../hudi/table/catalog/TestHoodieHiveCatalog.java | 2 +-
.../apache/hudi/table/format/TestInputFormat.java | 119 +++-
.../table/lookup/TestHoodieLookupFunction.java | 161 +++++
.../org/apache/hudi/util/TestDataTypeUtils.java | 84 +++
.../apache/hudi/utils/TestFlinkWriteClients.java | 70 ++
.../test/java/org/apache/hudi/utils/TestSQL.java | 5 +
.../org/apache/hudi/utils/TestStreamerUtil.java | 39 +-
.../test/java/org/apache/hudi/utils/TestUtils.java | 6 +
hudi-flink-datasource/hudi-flink1.17.x/pom.xml | 174 -----
.../org/apache/flink/types/variant/Variant.java | 24 -
.../hudi/adapter/AbstractRichFunctionAdapter.java | 27 -
.../hudi/adapter/CollectionInputFormatAdapter.java | 34 -
.../adapter/DataStreamScanProviderAdapter.java | 34 -
.../adapter/DataStreamSinkProviderAdapter.java | 37 -
.../org/apache/hudi/adapter/DataTypeAdapter.java | 64 --
.../apache/hudi/adapter/HiveCatalogConstants.java | 52 --
.../adapter/InputFormatSourceFunctionAdapter.java | 32 -
.../hudi/adapter/KeyedProcessFunctionAdapter.java | 27 -
.../apache/hudi/adapter/MaskingOutputAdapter.java | 67 --
.../hudi/adapter/ProcessFunctionAdapter.java | 31 -
.../hudi/adapter/RichSinkFunctionAdapter.java | 27 -
.../hudi/adapter/RichSourceFunctionAdapter.java | 27 -
.../apache/hudi/adapter/SinkFunctionAdapter.java | 27 -
.../hudi/adapter/SinkFunctionProviderAdapter.java | 28 -
.../apache/hudi/adapter/SourceFunctionAdapter.java | 27 -
.../adapter/SupportsPreWriteTopologyAdapter.java | 37 -
.../hudi/adapter/TableFunctionProviderAdapter.java | 33 -
.../hudi/adapter/TypeInformationAdapter.java | 35 -
.../main/java/org/apache/hudi/adapter/Utils.java | 179 -----
.../adapter/YieldingOperatorFactoryAdapter.java | 27 -
.../table/format/cow/ParquetSplitReaderUtil.java | 595 ----------------
.../table/format/cow/vector/HeapArrayVector.java | 66 --
.../table/format/cow/vector/HeapDecimalVector.java | 39 --
.../format/cow/vector/HeapMapColumnVector.java | 59 --
.../format/cow/vector/HeapRowColumnVector.java | 54 --
.../format/cow/vector/ParquetDecimalVector.java | 54 --
.../cow/vector/reader/AbstractColumnReader.java | 325 ---------
.../cow/vector/reader/ArrayColumnReader.java | 473 -------------
.../vector/reader/BaseVectorizedColumnReader.java | 311 ---------
.../cow/vector/reader/EmptyColumnReader.java | 42 --
.../vector/reader/FixedLenBytesColumnReader.java | 84 ---
.../vector/reader/Int64TimestampColumnReader.java | 119 ----
.../format/cow/vector/reader/MapColumnReader.java | 76 --
.../reader/ParquetColumnarRowSplitReader.java | 399 -----------
.../cow/vector/reader/ParquetDataColumnReader.java | 199 ------
.../reader/ParquetDataColumnReaderFactory.java | 301 --------
.../format/cow/vector/reader/RowColumnReader.java | 63 --
.../format/cow/vector/reader/RunLengthDecoder.java | 304 --------
.../java/org/apache/hudi/utils/CatalogUtils.java | 36 -
.../java/org/apache/hudi/utils/DataTypeUtils.java | 33 -
.../org/apache/hudi/utils/RuntimeContextUtils.java | 56 --
.../org/apache/hudi/utils/SourceContextUtils.java | 48 --
.../org/apache/hudi/utils/StateTtlConfigUtils.java | 31 -
.../java/org/apache/hudi/utils/StreamerUtils.java | 53 --
.../apache/hudi/adapter/CollectOutputAdapter.java | 73 --
.../hudi/adapter/DataTypeAdapterTestUtils.java | 28 -
.../apache/hudi/adapter/ExecutionAttemptUtil.java | 31 -
.../hudi/adapter/KeyedStateStoreAdapter.java | 24 -
.../hudi/adapter/OperatorStateStoreAdapter.java | 24 -
.../apache/hudi/adapter/TestHoodieCatalogs.java | 30 -
.../org/apache/hudi/adapter/TestStreamConfigs.java | 35 -
.../org/apache/hudi/sink/utils/MockTaskInfo.java | 59 --
.../table/format/cow/ParquetSplitReaderUtil.java | 12 +-
.../cow/vector/reader/NestedColumnReader.java | 8 +-
.../apache/hudi/adapter/TestHoodieCatalogs.java | 30 -
.../table/format/cow/ParquetSplitReaderUtil.java | 12 +-
.../cow/vector/reader/NestedColumnReader.java | 8 +-
.../apache/hudi/adapter/TestHoodieCatalogs.java | 30 -
.../table/format/cow/ParquetSplitReaderUtil.java | 695 +++++++++++--------
.../table/format/cow/utils/BooleanArrayList.java | 0
.../hudi/table/format/cow/utils/IntArrayList.java | 0
.../hudi/table/format/cow/utils/LongArrayList.java | 0
.../table/format/cow/utils/NestedPositionUtil.java | 0
.../format/cow/vector/ColumnarGroupArrayData.java | 179 -----
.../format/cow/vector/ColumnarGroupMapData.java | 63 --
.../format/cow/vector/ColumnarGroupRowData.java | 138 ----
.../cow/vector/HeapArrayGroupColumnVector.java | 53 --
.../table/format/cow/vector/HeapArrayVector.java | 31 +
.../format/cow/vector/HeapMapColumnVector.java | 74 +-
.../format/cow/vector/HeapRowColumnVector.java | 15 +
.../format/cow/vector/ParquetDecimalVector.java | 197 +++++-
.../cow/vector/position/CollectionPosition.java | 0
.../cow/vector/position/LevelDelegation.java | 0
.../format/cow/vector/position/RowPosition.java | 0
.../cow/vector/reader/ArrayColumnReader.java | 473 -------------
.../format/cow/vector/reader/ArrayGroupReader.java | 45 --
.../format/cow/vector/reader/MapColumnReader.java | 55 --
.../cow/vector/reader/NestedColumnReader.java | 8 +-
.../vector/reader/NestedPrimitiveColumnReader.java | 0
.../reader/ParquetColumnarRowSplitReader.java | 34 +-
.../reader/ParquetDataColumnReaderFactory.java | 112 ++-
.../format/cow/vector/reader/RowColumnReader.java | 63 --
.../table/format/cow/vector/type/ParquetField.java | 0
.../format/cow/vector/type/ParquetGroupField.java | 0
.../cow/vector/type/ParquetPrimitiveField.java | 0
.../apache/hudi/adapter/TestHoodieCatalogs.java | 30 -
.../cow/vector/TestHeapColumnVectorAccessors.java | 0
.../cow/vector/TestParquetDecimalVector.java | 0
.../reader/TestParquetDataColumnReaderFactory.java | 0
.../cow/vector/type/TestParquetGroupField.java | 0
.../table/format/cow/ParquetSplitReaderUtil.java | 693 +++++++++++--------
.../table/format/cow/utils/BooleanArrayList.java | 0
.../hudi/table/format/cow/utils/IntArrayList.java | 0
.../hudi/table/format/cow/utils/LongArrayList.java | 0
.../table/format/cow/utils/NestedPositionUtil.java | 0
.../format/cow/vector/ColumnarGroupArrayData.java | 179 -----
.../format/cow/vector/ColumnarGroupMapData.java | 63 --
.../format/cow/vector/ColumnarGroupRowData.java | 138 ----
.../cow/vector/HeapArrayGroupColumnVector.java | 53 --
.../table/format/cow/vector/HeapArrayVector.java | 31 +
.../format/cow/vector/HeapMapColumnVector.java | 74 +-
.../format/cow/vector/HeapRowColumnVector.java | 15 +
.../format/cow/vector/ParquetDecimalVector.java | 197 +++++-
.../cow/vector/position/CollectionPosition.java | 0
.../cow/vector/position/LevelDelegation.java | 0
.../format/cow/vector/position/RowPosition.java | 0
.../cow/vector/reader/ArrayColumnReader.java | 473 -------------
.../format/cow/vector/reader/ArrayGroupReader.java | 45 --
.../format/cow/vector/reader/MapColumnReader.java | 55 --
.../cow/vector/reader/NestedColumnReader.java | 8 +-
.../vector/reader/NestedPrimitiveColumnReader.java | 0
.../reader/ParquetColumnarRowSplitReader.java | 34 +-
.../reader/ParquetDataColumnReaderFactory.java | 112 ++-
.../format/cow/vector/reader/RowColumnReader.java | 63 --
.../table/format/cow/vector/type/ParquetField.java | 0
.../format/cow/vector/type/ParquetGroupField.java | 0
.../cow/vector/type/ParquetPrimitiveField.java | 0
.../apache/hudi/adapter/TestHoodieCatalogs.java | 30 -
.../cow/vector/TestHeapColumnVectorAccessors.java | 0
.../cow/vector/TestParquetDecimalVector.java | 0
.../reader/TestParquetDataColumnReaderFactory.java | 0
.../cow/vector/type/TestParquetGroupField.java | 0
.../table/format/cow/ParquetSplitReaderUtil.java | 763 ++++++++++++---------
.../table/format/cow/utils/BooleanArrayList.java | 0
.../hudi/table/format/cow/utils/IntArrayList.java | 0
.../hudi/table/format/cow/utils/LongArrayList.java | 0
.../table/format/cow/utils/NestedPositionUtil.java | 0
.../format/cow/vector/ColumnarGroupArrayData.java | 185 -----
.../format/cow/vector/ColumnarGroupMapData.java | 63 --
.../format/cow/vector/ColumnarGroupRowData.java | 144 ----
.../cow/vector/HeapArrayGroupColumnVector.java | 53 --
.../table/format/cow/vector/HeapArrayVector.java | 35 +-
.../format/cow/vector/HeapMapColumnVector.java | 74 +-
.../format/cow/vector/HeapRowColumnVector.java | 15 +
.../format/cow/vector/ParquetDecimalVector.java | 197 +++++-
.../cow/vector/position/CollectionPosition.java | 0
.../cow/vector/position/LevelDelegation.java | 0
.../format/cow/vector/position/RowPosition.java | 0
.../cow/vector/reader/ArrayColumnReader.java | 473 -------------
.../format/cow/vector/reader/ArrayGroupReader.java | 45 --
.../format/cow/vector/reader/MapColumnReader.java | 55 --
.../cow/vector/reader/NestedColumnReader.java | 11 +-
.../vector/reader/NestedPrimitiveColumnReader.java | 0
.../reader/ParquetColumnarRowSplitReader.java | 34 +-
.../reader/ParquetDataColumnReaderFactory.java | 121 +++-
.../format/cow/vector/reader/RowColumnReader.java | 63 --
.../table/format/cow/vector/type/ParquetField.java | 0
.../format/cow/vector/type/ParquetGroupField.java | 0
.../cow/vector/type/ParquetPrimitiveField.java | 0
.../apache/hudi/adapter/TestHoodieCatalogs.java | 30 -
.../cow/vector/TestHeapColumnVectorAccessors.java | 0
.../cow/vector/TestParquetDecimalVector.java | 0
.../reader/TestParquetDataColumnReaderFactory.java | 0
.../cow/vector/type/TestParquetGroupField.java | 0
.../apache/hudi/avro/HoodieAvroWriteSupport.java | 104 +--
.../common/table/log/HoodieLogFormatWriter.java | 184 +++--
.../hadoop/HoodieAvroFileReaderFactory.java | 20 +-
.../hadoop/HoodieAvroFileWriterFactory.java | 25 +-
.../io/storage/hadoop/HoodieAvroHFileWriter.java | 20 +-
.../TestExecutorServiceBasedEngineContext.java | 112 +++
.../engine/TestHoodieLocalEngineContext.java | 12 +-
.../table/log/TestHoodieLogFormatWriter.java | 180 +++++
.../io/hadoop/TestHoodieAvroFileReaderFactory.java | 66 ++
.../io/hadoop/TestHoodieHFileReaderWriter.java | 79 ++-
.../hudi/hadoop/HiveHoodieReaderContext.java | 2 +-
.../helpers/DFSTestSuitePathSelector.java | 7 +-
.../connect/writers/AbstractConnectWriter.java | 25 +-
.../connect/writers/BufferedConnectWriter.java | 5 +-
hudi-spark-datasource/README.md | 6 +-
.../scala/org/apache/hudi/BucketIndexSupport.scala | 4 +-
.../main/scala/org/apache/hudi/DefaultSource.scala | 16 +-
.../hudi/HoodieHadoopFsRelationFactory.scala | 31 +-
.../scala/org/apache/hudi/SparkFilterHelper.scala | 2 +-
.../org/apache/hudi/cdc/CDCFileGroupIterator.scala | 22 +-
.../sql/hudi/streaming/HoodieStreamSourceV2.scala | 2 +-
.../hudi/TestHoodieHadoopFsRelationFactory.scala | 58 ++
.../spark/sql/hudi/analysis/HoodieAnalysis.scala | 38 +-
.../action/cluster/TestIncrementalClustering.java | 4 +-
.../org/apache/hudi/TestHoodieSparkUtils.scala | 28 +-
.../apache/hudi/functional/TestCOWDataSource.scala | 49 +-
.../apache/hudi/functional/TestMORDataSource.scala | 13 +-
.../functional/cdc/TestCDCDataFrameSuite.scala | 61 ++
.../sql/hudi/dml/others/TestMergeIntoTable.scala | 11 +-
.../sql/hudi/dml/others/TestMergeIntoTable2.scala | 11 +-
.../sql/hudi/dml/schema/TestVariantDataType.scala | 22 +-
.../procedure/TestClusteringWithCustomMerger.scala | 133 ++++
.../hudi/command/DeleteHoodieTableCommand.scala | 5 +-
.../{hudi-spark3.4.x => hudi-spark3.3.x}/pom.xml | 59 +-
.../src/main/antlr4/imports/SqlBase.g4 | 0
.../apache/hudi/spark/sql/parser/HoodieSqlBase.g4 | 0
.../apache/hudi/Spark33HoodieFileScanRDD.scala} | 2 +-
.../HoodieSpark33CatalystExpressionUtils.scala} | 20 +-
.../sql/HoodieSpark33CatalystPlanUtils.scala} | 24 +-
.../spark/sql/HoodieSpark33SchemaUtils.scala} | 6 +-
.../spark/sql/adapter/Spark3_3Adapter.scala} | 46 +-
.../apache/spark/sql/avro/AvroDeserializer.scala | 0
.../org/apache/spark/sql/avro/AvroSerializer.scala | 0
.../org/apache/spark/sql/avro/AvroUtils.scala | 2 +-
.../sql/avro/HoodieSpark3_3AvroDeserializer.scala} | 2 +-
.../sql/avro/HoodieSpark3_3AvroSerializer.scala} | 2 +-
.../HoodieSpark33PartitionedFileUtils.scala} | 17 +-
.../datasources/Spark33NestedSchemaPruning.scala} | 2 +-
.../datasources/orc/Spark33OrcReader.scala} | 24 +-
.../parquet/Spark33DataSourceUtils.scala} | 2 +-
.../Spark33LegacyHoodieParquetFileFormat.scala} | 77 +--
.../parquet/Spark33ParquetReader.scala} | 59 +-
.../Spark33ResolveHudiAlterTableCommand.scala} | 4 +-
.../sql/hudi/analysis/HoodieSpark33Analysis.scala} | 2 +-
.../Spark33HoodiePruneFileSourcePartitions.scala} | 34 +-
.../HoodieSpark3_3ExtendedSqlAstBuilder.scala} | 30 +-
.../parser/HoodieSpark3_3ExtendedSqlParser.scala} | 22 +-
.../hudi/TestHoodieStreamingSinkConstants.scala | 0
.../hudi/command/DeleteHoodieTableCommand.scala | 5 +-
.../datasources/parquet/Spark40ParquetReader.scala | 7 +
.../datasources/parquet/Spark41ParquetReader.scala | 7 +
.../{hudi-spark4.1.x => hudi-spark4.2.x}/pom.xml | 16 +-
.../src/main/antlr4/imports/SqlBase.g4 | 0
.../apache/hudi/spark/sql/parser/HoodieSqlBase.g4 | 0
.../apache/hudi/Spark42HoodieFileScanRDD.scala} | 2 +-
...park42HoodiePartitionCDCFileGroupMapping.scala} | 4 +-
.../Spark42HoodiePartitionFileSliceMapping.scala} | 4 +-
.../hudi/Spark42HoodiePartitionValues.scala} | 4 +-
.../client/model/Spark42HoodieInternalRow.scala} | 4 +-
.../HoodieSpark42CatalystExpressionUtils.scala} | 2 +-
.../sql/HoodieSpark42CatalystPlanUtils.scala} | 2 +-
.../spark/sql/HoodieSpark42SchemaUtils.scala} | 2 +-
.../spark/sql/adapter/Spark4_2Adapter.scala} | 44 +-
.../apache/spark/sql/avro/AvroDeserializer.scala | 6 +-
.../org/apache/spark/sql/avro/AvroSerializer.scala | 0
.../org/apache/spark/sql/avro/AvroUtils.scala | 0
.../sql/avro/HoodieSpark4_2AvroDeserializer.scala} | 2 +-
.../sql/avro/HoodieSpark4_2AvroSerializer.scala} | 2 +-
.../HoodieSpark42PartitionedFileUtils.scala} | 2 +-
.../datasources/Spark42NestedSchemaPruning.scala} | 2 +-
.../datasources/orc/Spark42OrcReader.scala} | 8 +-
.../parquet/Spark42DataSourceUtils.scala} | 2 +-
.../Spark42LegacyHoodieParquetFileFormat.scala} | 6 +-
.../parquet/Spark42ParquetReader.scala} | 8 +-
.../Spark42ResolveHudiAlterTableCommand.scala} | 2 +-
.../sql/hudi/analysis/HoodieSpark42Analysis.scala} | 10 +-
.../HoodieSpark4_2ExtendedSqlAstBuilder.scala} | 4 +-
.../parser/HoodieSpark4_2ExtendedSqlParser.scala} | 4 +-
.../hudi/TestHoodieStreamingSinkConstants.scala | 0
.../hudi/TestSpark4_2AvroLogicalTypeBytes.scala} | 16 +-
.../sync/common/metrics/HoodieMetaSyncMetrics.java | 3 +
.../hudi/TestHudiAlluxioCacheFileOperations.java | 48 +-
.../hudi/TestHudiMemoryCacheFileOperations.java | 48 +-
.../plugin/hudi/TestHudiNoCacheFileOperations.java | 48 +-
.../ingestion/HoodieIngestionMetrics.java | 4 +
.../hudi/utilities/sources/GcsEventsSource.java | 12 +-
.../hudi/utilities/sources/HiveIncrPullSource.java | 11 +-
.../hudi/utilities/sources/HoodieIncrSource.java | 4 +-
.../apache/hudi/utilities/sources/InputBatch.java | 9 -
.../apache/hudi/utilities/sources/JdbcSource.java | 8 +-
.../apache/hudi/utilities/sources/KafkaSource.java | 7 +-
.../hudi/utilities/sources/KinesisSource.java | 5 +-
.../hudi/utilities/sources/PulsarSource.java | 4 +-
.../org/apache/hudi/utilities/sources/Source.java | 53 +-
.../hudi/utilities/sources/SqlFileBasedSource.java | 10 +-
.../apache/hudi/utilities/sources/SqlSource.java | 10 +-
.../utilities/sources/debezium/DebeziumSource.java | 5 +-
.../utilities/sources/helpers/DFSPathSelector.java | 6 +-
.../sources/helpers/DatePartitionPathSelector.java | 6 +-
.../sources/helpers/S3EventsMetaSelector.java | 9 +-
.../hudi/utilities/streamer/HoodieStreamer.java | 4 +-
.../utilities/streamer/HoodieStreamerMetrics.java | 15 +
.../apache/hudi/utilities/streamer/StreamSync.java | 15 +-
.../streamer/StreamerCheckpointUtils.java | 26 +-
.../deltastreamer/HoodieDeltaStreamerTestBase.java | 19 +-
.../deltastreamer/TestHoodieDeltaStreamer.java | 488 ++++++-------
...estHoodieDeltaStreamerSchemaEvolutionQuick.java | 9 +-
.../TestHoodieDeltaStreamerWithMultiWriter.java | 5 +-
.../deltastreamer/TestSourceFormatAdapter.java | 6 +-
.../multisync/TestMultipleMetaSync.java | 13 +-
.../hudi/utilities/sources/TestDataSource.java | 4 +-
.../utilities/sources/TestGcsEventsSource.java | 3 +-
.../utilities/sources/TestHiveIncrPullSource.java | 99 +++
.../hudi/utilities/sources/TestInputBatch.java | 6 +-
.../hudi/utilities/sources/TestJdbcSource.java | 5 +-
.../utilities/sources/TestSqlFileBasedSource.java | 13 +-
.../hudi/utilities/sources/TestSqlSource.java | 15 +-
.../TestStreamerSourceCheckpointVersion.java | 351 ++++++++++
.../helpers/TestDFSPathSelectorCommonMethods.java | 15 +
.../TestMercifulJsonToRowConverterBase.java | 56 +-
.../streamer/TestHoodieIncrSourceE2E.java | 15 +-
.../TestHoodieIncrSourceE2EAutoUpgrade.java | 13 +-
.../streamer/TestHoodieStreamerMetrics.java | 47 ++
.../TestParquetDfsCheckpointFormatOnV6.java | 128 ++++
.../hudi/utilities/streamer/TestStreamSync.java | 64 +-
.../streamer/TestStreamerCheckpointUtils.java | 52 ++
.../sources/DistributedTestDataSource.java | 6 +-
.../checkpoint-v6/parquet-dfs-v1-fixture.zip | Bin 0 -> 103450 bytes
packaging/bundle-validation/Dockerfile | 2 +
...ark351.sh => build_flink1181hive313spark334.sh} | 12 +-
...ild_flink1200hive313spark420previewscala213.sh} | 8 +-
packaging/bundle-validation/ci_run.sh | 42 +-
packaging/bundle-validation/run_docker_java17.sh | 48 +-
packaging/bundle-validation/validate.sh | 6 +-
packaging/hudi-flink-bundle/pom.xml | 9 +
pom.xml | 141 +++-
rfc/README.md | 230 ++++---
rfc/rfc-106/index-compaction-flow.png | Bin 0 -> 329243 bytes
rfc/rfc-106/index-write-flow.png | Bin 0 -> 375082 bytes
rfc/rfc-106/rfc-106.md | 260 +++++++
rfc/rfc-106/rli-access-pattern.png | Bin 0 -> 96912 bytes
scripts/release/deploy_staging_jars.sh | 5 +-
scripts/release/deploy_staging_jars_java17.sh | 8 +-
scripts/release/validate_staged_bundles.sh | 6 +-
522 files changed, 11529 insertions(+), 12383 deletions(-)
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/CommitMetadataProperties.java
create mode 100644
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/TestCommitMetadataProperties.java
create mode 100644
hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/VectorConversionUtils.java
create mode 100644
hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/util/TestAvroToRowDataConverters.java
create mode 100644
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/validator/TestSparkPreCommitValidator.java
create mode 100644
hudi-common/src/main/java/org/apache/hudi/common/engine/ExecutorServiceBasedEngineContext.java
create mode 100644
hudi-common/src/main/java/org/apache/hudi/common/util/HoodieVectorUtils.java
create mode 100644
hudi-examples/hudi-examples-spark/src/test/java/org/apache/hudi/examples/common/TestRandomJsonSource.java
copy
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/{source/split/assign/HoodieSplitAssigners.java
=> sink/partitioner/BucketIndexPartitionerFactory.java} (53%)
create mode 100644
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/BucketIndexRemotePartitioner.java
rename
hudi-flink-datasource/{hudi-flink1.17.x/src/main/java/org/apache/hudi/adapter/SinkAdapter.java
=>
hudi-flink/src/test/java/org/apache/hudi/sink/compact/handler/TestDefaultCleanHandler.java}
(56%)
create mode 100644
hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/partitioner/TestBucketIndexPartitionerFactory.java
create mode 100644
hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/partitioner/TestBucketIndexRemotePartitioner.java
create mode 100644
hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/lookup/TestHoodieLookupFunction.java
create mode 100644
hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/util/TestDataTypeUtils.java
delete mode 100644 hudi-flink-datasource/hudi-flink1.17.x/pom.xml
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/flink/types/variant/Variant.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/adapter/AbstractRichFunctionAdapter.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/adapter/CollectionInputFormatAdapter.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/adapter/DataStreamScanProviderAdapter.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/adapter/DataStreamSinkProviderAdapter.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/adapter/HiveCatalogConstants.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/adapter/InputFormatSourceFunctionAdapter.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/adapter/KeyedProcessFunctionAdapter.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/adapter/MaskingOutputAdapter.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/adapter/ProcessFunctionAdapter.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/adapter/RichSinkFunctionAdapter.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/adapter/RichSourceFunctionAdapter.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/adapter/SinkFunctionAdapter.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/adapter/SinkFunctionProviderAdapter.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/adapter/SourceFunctionAdapter.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/adapter/SupportsPreWriteTopologyAdapter.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/adapter/TableFunctionProviderAdapter.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/adapter/TypeInformationAdapter.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/adapter/Utils.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/adapter/YieldingOperatorFactoryAdapter.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayVector.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapDecimalVector.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapMapColumnVector.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapRowColumnVector.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/table/format/cow/vector/ParquetDecimalVector.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/AbstractColumnReader.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayColumnReader.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/BaseVectorizedColumnReader.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/EmptyColumnReader.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/FixedLenBytesColumnReader.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/Int64TimestampColumnReader.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/MapColumnReader.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetDataColumnReader.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetDataColumnReaderFactory.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/RowColumnReader.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/RunLengthDecoder.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/utils/CatalogUtils.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/utils/DataTypeUtils.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/utils/RuntimeContextUtils.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/utils/SourceContextUtils.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/utils/StateTtlConfigUtils.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/utils/StreamerUtils.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/test/java/org/apache/hudi/adapter/CollectOutputAdapter.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/test/java/org/apache/hudi/adapter/DataTypeAdapterTestUtils.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/test/java/org/apache/hudi/adapter/ExecutionAttemptUtil.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/test/java/org/apache/hudi/adapter/KeyedStateStoreAdapter.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/test/java/org/apache/hudi/adapter/OperatorStateStoreAdapter.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/test/java/org/apache/hudi/adapter/TestHoodieCatalogs.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/test/java/org/apache/hudi/adapter/TestStreamConfigs.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/test/java/org/apache/hudi/sink/utils/MockTaskInfo.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.18.x/src/test/java/org/apache/hudi/adapter/TestHoodieCatalogs.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.19.x/src/test/java/org/apache/hudi/adapter/TestHoodieCatalogs.java
copy hudi-flink-datasource/{hudi-flink1.18.x =>
hudi-flink1.20.x}/src/main/java/org/apache/hudi/table/format/cow/utils/BooleanArrayList.java
(100%)
copy hudi-flink-datasource/{hudi-flink1.18.x =>
hudi-flink1.20.x}/src/main/java/org/apache/hudi/table/format/cow/utils/IntArrayList.java
(100%)
copy hudi-flink-datasource/{hudi-flink1.18.x =>
hudi-flink1.20.x}/src/main/java/org/apache/hudi/table/format/cow/utils/LongArrayList.java
(100%)
copy hudi-flink-datasource/{hudi-flink1.18.x =>
hudi-flink1.20.x}/src/main/java/org/apache/hudi/table/format/cow/utils/NestedPositionUtil.java
(100%)
delete mode 100644
hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupArrayData.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupMapData.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupRowData.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayGroupColumnVector.java
copy hudi-flink-datasource/{hudi-flink1.18.x =>
hudi-flink1.20.x}/src/main/java/org/apache/hudi/table/format/cow/vector/position/CollectionPosition.java
(100%)
copy hudi-flink-datasource/{hudi-flink1.18.x =>
hudi-flink1.20.x}/src/main/java/org/apache/hudi/table/format/cow/vector/position/LevelDelegation.java
(100%)
copy hudi-flink-datasource/{hudi-flink1.18.x =>
hudi-flink1.20.x}/src/main/java/org/apache/hudi/table/format/cow/vector/position/RowPosition.java
(100%)
delete mode 100644
hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayColumnReader.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayGroupReader.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/MapColumnReader.java
copy hudi-flink-datasource/{hudi-flink1.19.x =>
hudi-flink1.20.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java
(95%)
copy hudi-flink-datasource/{hudi-flink1.18.x =>
hudi-flink1.20.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedPrimitiveColumnReader.java
(100%)
delete mode 100644
hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/RowColumnReader.java
copy hudi-flink-datasource/{hudi-flink1.18.x =>
hudi-flink1.20.x}/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetField.java
(100%)
copy hudi-flink-datasource/{hudi-flink1.18.x =>
hudi-flink1.20.x}/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetGroupField.java
(100%)
copy hudi-flink-datasource/{hudi-flink1.18.x =>
hudi-flink1.20.x}/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetPrimitiveField.java
(100%)
delete mode 100644
hudi-flink-datasource/hudi-flink1.20.x/src/test/java/org/apache/hudi/adapter/TestHoodieCatalogs.java
copy hudi-flink-datasource/{hudi-flink1.18.x =>
hudi-flink1.20.x}/src/test/java/org/apache/hudi/table/format/cow/vector/TestHeapColumnVectorAccessors.java
(100%)
copy hudi-flink-datasource/{hudi-flink1.18.x =>
hudi-flink1.20.x}/src/test/java/org/apache/hudi/table/format/cow/vector/TestParquetDecimalVector.java
(100%)
copy hudi-flink-datasource/{hudi-flink1.18.x =>
hudi-flink1.20.x}/src/test/java/org/apache/hudi/table/format/cow/vector/reader/TestParquetDataColumnReaderFactory.java
(100%)
copy hudi-flink-datasource/{hudi-flink1.18.x =>
hudi-flink1.20.x}/src/test/java/org/apache/hudi/table/format/cow/vector/type/TestParquetGroupField.java
(100%)
copy hudi-flink-datasource/{hudi-flink1.18.x =>
hudi-flink2.0.x}/src/main/java/org/apache/hudi/table/format/cow/utils/BooleanArrayList.java
(100%)
copy hudi-flink-datasource/{hudi-flink1.18.x =>
hudi-flink2.0.x}/src/main/java/org/apache/hudi/table/format/cow/utils/IntArrayList.java
(100%)
copy hudi-flink-datasource/{hudi-flink1.18.x =>
hudi-flink2.0.x}/src/main/java/org/apache/hudi/table/format/cow/utils/LongArrayList.java
(100%)
copy hudi-flink-datasource/{hudi-flink1.18.x =>
hudi-flink2.0.x}/src/main/java/org/apache/hudi/table/format/cow/utils/NestedPositionUtil.java
(100%)
delete mode 100644
hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupArrayData.java
delete mode 100644
hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupMapData.java
delete mode 100644
hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupRowData.java
delete mode 100644
hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayGroupColumnVector.java
copy hudi-flink-datasource/{hudi-flink1.18.x =>
hudi-flink2.0.x}/src/main/java/org/apache/hudi/table/format/cow/vector/position/CollectionPosition.java
(100%)
copy hudi-flink-datasource/{hudi-flink1.18.x =>
hudi-flink2.0.x}/src/main/java/org/apache/hudi/table/format/cow/vector/position/LevelDelegation.java
(100%)
copy hudi-flink-datasource/{hudi-flink1.18.x =>
hudi-flink2.0.x}/src/main/java/org/apache/hudi/table/format/cow/vector/position/RowPosition.java
(100%)
delete mode 100644
hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayColumnReader.java
delete mode 100644
hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayGroupReader.java
delete mode 100644
hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/MapColumnReader.java
copy hudi-flink-datasource/{hudi-flink1.19.x =>
hudi-flink2.0.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java
(95%)
copy hudi-flink-datasource/{hudi-flink1.18.x =>
hudi-flink2.0.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedPrimitiveColumnReader.java
(100%)
delete mode 100644
hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/RowColumnReader.java
copy hudi-flink-datasource/{hudi-flink1.18.x =>
hudi-flink2.0.x}/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetField.java
(100%)
copy hudi-flink-datasource/{hudi-flink1.18.x =>
hudi-flink2.0.x}/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetGroupField.java
(100%)
copy hudi-flink-datasource/{hudi-flink1.18.x =>
hudi-flink2.0.x}/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetPrimitiveField.java
(100%)
delete mode 100644
hudi-flink-datasource/hudi-flink2.0.x/src/test/java/org/apache/hudi/adapter/TestHoodieCatalogs.java
copy hudi-flink-datasource/{hudi-flink1.18.x =>
hudi-flink2.0.x}/src/test/java/org/apache/hudi/table/format/cow/vector/TestHeapColumnVectorAccessors.java
(100%)
copy hudi-flink-datasource/{hudi-flink1.18.x =>
hudi-flink2.0.x}/src/test/java/org/apache/hudi/table/format/cow/vector/TestParquetDecimalVector.java
(100%)
copy hudi-flink-datasource/{hudi-flink1.18.x =>
hudi-flink2.0.x}/src/test/java/org/apache/hudi/table/format/cow/vector/reader/TestParquetDataColumnReaderFactory.java
(100%)
copy hudi-flink-datasource/{hudi-flink1.18.x =>
hudi-flink2.0.x}/src/test/java/org/apache/hudi/table/format/cow/vector/type/TestParquetGroupField.java
(100%)
copy hudi-flink-datasource/{hudi-flink1.18.x =>
hudi-flink2.1.x}/src/main/java/org/apache/hudi/table/format/cow/utils/BooleanArrayList.java
(100%)
copy hudi-flink-datasource/{hudi-flink1.18.x =>
hudi-flink2.1.x}/src/main/java/org/apache/hudi/table/format/cow/utils/IntArrayList.java
(100%)
copy hudi-flink-datasource/{hudi-flink1.18.x =>
hudi-flink2.1.x}/src/main/java/org/apache/hudi/table/format/cow/utils/LongArrayList.java
(100%)
copy hudi-flink-datasource/{hudi-flink1.18.x =>
hudi-flink2.1.x}/src/main/java/org/apache/hudi/table/format/cow/utils/NestedPositionUtil.java
(100%)
delete mode 100644
hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupArrayData.java
delete mode 100644
hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupMapData.java
delete mode 100644
hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupRowData.java
delete mode 100644
hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayGroupColumnVector.java
copy hudi-flink-datasource/{hudi-flink1.18.x =>
hudi-flink2.1.x}/src/main/java/org/apache/hudi/table/format/cow/vector/position/CollectionPosition.java
(100%)
copy hudi-flink-datasource/{hudi-flink1.18.x =>
hudi-flink2.1.x}/src/main/java/org/apache/hudi/table/format/cow/vector/position/LevelDelegation.java
(100%)
copy hudi-flink-datasource/{hudi-flink1.18.x =>
hudi-flink2.1.x}/src/main/java/org/apache/hudi/table/format/cow/vector/position/RowPosition.java
(100%)
delete mode 100644
hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayColumnReader.java
delete mode 100644
hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayGroupReader.java
delete mode 100644
hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/MapColumnReader.java
copy hudi-flink-datasource/{hudi-flink1.19.x =>
hudi-flink2.1.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java
(94%)
copy hudi-flink-datasource/{hudi-flink1.18.x =>
hudi-flink2.1.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedPrimitiveColumnReader.java
(100%)
delete mode 100644
hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/RowColumnReader.java
copy hudi-flink-datasource/{hudi-flink1.18.x =>
hudi-flink2.1.x}/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetField.java
(100%)
copy hudi-flink-datasource/{hudi-flink1.18.x =>
hudi-flink2.1.x}/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetGroupField.java
(100%)
copy hudi-flink-datasource/{hudi-flink1.18.x =>
hudi-flink2.1.x}/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetPrimitiveField.java
(100%)
delete mode 100644
hudi-flink-datasource/hudi-flink2.1.x/src/test/java/org/apache/hudi/adapter/TestHoodieCatalogs.java
copy hudi-flink-datasource/{hudi-flink1.18.x =>
hudi-flink2.1.x}/src/test/java/org/apache/hudi/table/format/cow/vector/TestHeapColumnVectorAccessors.java
(100%)
copy hudi-flink-datasource/{hudi-flink1.18.x =>
hudi-flink2.1.x}/src/test/java/org/apache/hudi/table/format/cow/vector/TestParquetDecimalVector.java
(100%)
copy hudi-flink-datasource/{hudi-flink1.18.x =>
hudi-flink2.1.x}/src/test/java/org/apache/hudi/table/format/cow/vector/reader/TestParquetDataColumnReaderFactory.java
(100%)
copy hudi-flink-datasource/{hudi-flink1.18.x =>
hudi-flink2.1.x}/src/test/java/org/apache/hudi/table/format/cow/vector/type/TestParquetGroupField.java
(100%)
create mode 100644
hudi-hadoop-common/src/test/java/org/apache/hudi/common/engine/TestExecutorServiceBasedEngineContext.java
create mode 100644
hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/log/TestHoodieLogFormatWriter.java
create mode 100644
hudi-spark-datasource/hudi-spark-common/src/test/scala/org/apache/hudi/TestHoodieHadoopFsRelationFactory.scala
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestClusteringWithCustomMerger.scala
copy hudi-spark-datasource/{hudi-spark3.4.x => hudi-spark3.3.x}/pom.xml (89%)
copy hudi-spark-datasource/{hudi-spark3.4.x =>
hudi-spark3.3.x}/src/main/antlr4/imports/SqlBase.g4 (100%)
copy hudi-spark-datasource/{hudi-spark3.4.x =>
hudi-spark3.3.x}/src/main/antlr4/org/apache/hudi/spark/sql/parser/HoodieSqlBase.g4
(100%)
copy
hudi-spark-datasource/{hudi-spark3.5.x/src/main/scala/org/apache/hudi/Spark35HoodieFileScanRDD.scala
=>
hudi-spark3.3.x/src/main/scala/org/apache/hudi/Spark33HoodieFileScanRDD.scala}
(96%)
copy
hudi-spark-datasource/{hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/HoodieSpark34CatalystExpressionUtils.scala
=>
hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/HoodieSpark33CatalystExpressionUtils.scala}
(88%)
copy
hudi-spark-datasource/{hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/HoodieSpark35CatalystPlanUtils.scala
=>
hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/HoodieSpark33CatalystPlanUtils.scala}
(88%)
copy
hudi-spark-datasource/{hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/HoodieSpark34SchemaUtils.scala
=>
hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/HoodieSpark33SchemaUtils.scala}
(90%)
copy
hudi-spark-datasource/{hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/adapter/Spark3_4Adapter.scala
=>
hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/adapter/Spark3_3Adapter.scala}
(87%)
copy hudi-spark-datasource/{hudi-spark3.4.x =>
hudi-spark3.3.x}/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
(100%)
copy hudi-spark-datasource/{hudi-spark3.4.x =>
hudi-spark3.3.x}/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala
(100%)
copy hudi-spark-datasource/{hudi-spark3.4.x =>
hudi-spark3.3.x}/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala (99%)
copy
hudi-spark-datasource/{hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_4AvroDeserializer.scala
=>
hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_3AvroDeserializer.scala}
(95%)
copy
hudi-spark-datasource/{hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_4AvroSerializer.scala
=>
hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_3AvroSerializer.scala}
(94%)
copy
hudi-spark-datasource/{hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/execution/datasources/HoodieSpark34PartitionedFileUtils.scala
=>
hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/HoodieSpark33PartitionedFileUtils.scala}
(78%)
copy
hudi-spark-datasource/{hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/execution/datasources/Spark34NestedSchemaPruning.scala
=>
hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/Spark33NestedSchemaPruning.scala}
(97%)
copy
hudi-spark-datasource/{hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/execution/datasources/orc/Spark34OrcReader.scala
=>
hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/orc/Spark33OrcReader.scala}
(86%)
copy
hudi-spark-datasource/{hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark34DataSourceUtils.scala
=>
hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark33DataSourceUtils.scala}
(99%)
copy
hudi-spark-datasource/{hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark34LegacyHoodieParquetFileFormat.scala
=>
hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark33LegacyHoodieParquetFileFormat.scala}
(88%)
copy
hudi-spark-datasource/{hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark34ParquetReader.scala
=>
hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark33ParquetReader.scala}
(88%)
copy
hudi-spark-datasource/{hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/hudi/Spark34ResolveHudiAlterTableCommand.scala
=>
hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/hudi/Spark33ResolveHudiAlterTableCommand.scala}
(96%)
copy
hudi-spark-datasource/{hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSpark34Analysis.scala
=>
hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSpark33Analysis.scala}
(97%)
copy
hudi-spark-datasource/{hudi-spark4-common/src/main/scala/org/apache/spark/sql/hudi/analysis/Spark4HoodiePruneFileSourcePartitions.scala
=>
hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/hudi/analysis/Spark33HoodiePruneFileSourcePartitions.scala}
(79%)
copy
hudi-spark-datasource/{hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_4ExtendedSqlAstBuilder.scala
=>
hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_3ExtendedSqlAstBuilder.scala}
(99%)
copy
hudi-spark-datasource/{hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_5ExtendedSqlParser.scala
=>
hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_3ExtendedSqlParser.scala}
(92%)
copy hudi-spark-datasource/{hudi-spark3.4.x =>
hudi-spark3.3.x}/src/test/scala/org/apache/hudi/TestHoodieStreamingSinkConstants.scala
(100%)
copy hudi-spark-datasource/{hudi-spark4.1.x => hudi-spark4.2.x}/pom.xml (95%)
copy hudi-spark-datasource/{hudi-spark4.1.x =>
hudi-spark4.2.x}/src/main/antlr4/imports/SqlBase.g4 (100%)
copy hudi-spark-datasource/{hudi-spark3.4.x =>
hudi-spark4.2.x}/src/main/antlr4/org/apache/hudi/spark/sql/parser/HoodieSqlBase.g4
(100%)
copy
hudi-spark-datasource/{hudi-spark3.5.x/src/main/scala/org/apache/hudi/Spark35HoodieFileScanRDD.scala
=>
hudi-spark4.2.x/src/main/scala/org/apache/hudi/Spark42HoodieFileScanRDD.scala}
(96%)
copy
hudi-spark-datasource/{hudi-spark4.1.x/src/main/scala/org/apache/hudi/Spark41HoodiePartitionCDCFileGroupMapping.scala
=>
hudi-spark4.2.x/src/main/scala/org/apache/hudi/Spark42HoodiePartitionCDCFileGroupMapping.scala}
(90%)
copy
hudi-spark-datasource/{hudi-spark4.1.x/src/main/scala/org/apache/hudi/Spark41HoodiePartitionFileSliceMapping.scala
=>
hudi-spark4.2.x/src/main/scala/org/apache/hudi/Spark42HoodiePartitionFileSliceMapping.scala}
(91%)
copy
hudi-spark-datasource/{hudi-spark4.1.x/src/main/scala/org/apache/hudi/Spark41HoodiePartitionValues.scala
=>
hudi-spark4.2.x/src/main/scala/org/apache/hudi/Spark42HoodiePartitionValues.scala}
(96%)
copy
hudi-spark-datasource/{hudi-spark4.1.x/src/main/scala/org/apache/hudi/client/model/Spark41HoodieInternalRow.scala
=>
hudi-spark4.2.x/src/main/scala/org/apache/hudi/client/model/Spark42HoodieInternalRow.scala}
(96%)
copy
hudi-spark-datasource/{hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/HoodieSpark41CatalystExpressionUtils.scala
=>
hudi-spark4.2.x/src/main/scala/org/apache/spark/sql/HoodieSpark42CatalystExpressionUtils.scala}
(98%)
copy
hudi-spark-datasource/{hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/HoodieSpark41CatalystPlanUtils.scala
=>
hudi-spark4.2.x/src/main/scala/org/apache/spark/sql/HoodieSpark42CatalystPlanUtils.scala}
(98%)
copy
hudi-spark-datasource/{hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/HoodieSpark41SchemaUtils.scala
=>
hudi-spark4.2.x/src/main/scala/org/apache/spark/sql/HoodieSpark42SchemaUtils.scala}
(97%)
copy
hudi-spark-datasource/{hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/adapter/Spark4_1Adapter.scala
=>
hudi-spark4.2.x/src/main/scala/org/apache/spark/sql/adapter/Spark4_2Adapter.scala}
(91%)
copy hudi-spark-datasource/{hudi-spark4.1.x =>
hudi-spark4.2.x}/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
(99%)
copy hudi-spark-datasource/{hudi-spark4.1.x =>
hudi-spark4.2.x}/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala
(100%)
copy hudi-spark-datasource/{hudi-spark3.4.x =>
hudi-spark4.2.x}/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala (100%)
copy
hudi-spark-datasource/{hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/avro/HoodieSpark4_1AvroDeserializer.scala
=>
hudi-spark4.2.x/src/main/scala/org/apache/spark/sql/avro/HoodieSpark4_2AvroDeserializer.scala}
(95%)
copy
hudi-spark-datasource/{hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_4AvroSerializer.scala
=>
hudi-spark4.2.x/src/main/scala/org/apache/spark/sql/avro/HoodieSpark4_2AvroSerializer.scala}
(94%)
copy
hudi-spark-datasource/{hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/HoodieSpark41PartitionedFileUtils.scala
=>
hudi-spark4.2.x/src/main/scala/org/apache/spark/sql/execution/datasources/HoodieSpark42PartitionedFileUtils.scala}
(97%)
copy
hudi-spark-datasource/{hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/Spark41NestedSchemaPruning.scala
=>
hudi-spark4.2.x/src/main/scala/org/apache/spark/sql/execution/datasources/Spark42NestedSchemaPruning.scala}
(97%)
copy
hudi-spark-datasource/{hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/execution/datasources/orc/Spark35OrcReader.scala
=>
hudi-spark4.2.x/src/main/scala/org/apache/spark/sql/execution/datasources/orc/Spark42OrcReader.scala}
(95%)
copy
hudi-spark-datasource/{hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark41DataSourceUtils.scala
=>
hudi-spark4.2.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark42DataSourceUtils.scala}
(99%)
copy
hudi-spark-datasource/{hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark41LegacyHoodieParquetFileFormat.scala
=>
hudi-spark4.2.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark42LegacyHoodieParquetFileFormat.scala}
(99%)
copy
hudi-spark-datasource/{hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark41ParquetReader.scala
=>
hudi-spark4.2.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark42ParquetReader.scala}
(99%)
copy
hudi-spark-datasource/{hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/hudi/Spark41ResolveHudiAlterTableCommand.scala
=>
hudi-spark4.2.x/src/main/scala/org/apache/spark/sql/hudi/Spark42ResolveHudiAlterTableCommand.scala}
(98%)
copy
hudi-spark-datasource/{hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSpark41Analysis.scala
=>
hudi-spark4.2.x/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSpark42Analysis.scala}
(96%)
copy
hudi-spark-datasource/{hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark4_1ExtendedSqlAstBuilder.scala
=>
hudi-spark4.2.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark4_2ExtendedSqlAstBuilder.scala}
(99%)
copy
hudi-spark-datasource/{hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark4_1ExtendedSqlParser.scala
=>
hudi-spark4.2.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark4_2ExtendedSqlParser.scala}
(98%)
copy hudi-spark-datasource/{hudi-spark4.1.x =>
hudi-spark4.2.x}/src/test/scala/org/apache/hudi/TestHoodieStreamingSinkConstants.scala
(100%)
copy
hudi-spark-datasource/{hudi-spark4.1.x/src/test/scala/org/apache/hudi/TestSpark4_1AvroLogicalTypeBytes.scala
=>
hudi-spark4.2.x/src/test/scala/org/apache/hudi/TestSpark4_2AvroLogicalTypeBytes.scala}
(93%)
create mode 100644
hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestHiveIncrPullSource.java
create mode 100644
hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestStreamerSourceCheckpointVersion.java
create mode 100644
hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestParquetDfsCheckpointFormatOnV6.java
create mode 100644
hudi-utilities/src/test/resources/checkpoint-v6/parquet-dfs-v1-fixture.zip
rename packaging/bundle-validation/base/{build_flink1170hive313spark351.sh =>
build_flink1181hive313spark334.sh} (71%)
copy
packaging/bundle-validation/base/{build_flink1200hive313spark411scala213.sh =>
build_flink1200hive313spark420previewscala213.sh} (80%)
create mode 100644 rfc/rfc-106/index-compaction-flow.png
create mode 100644 rfc/rfc-106/index-write-flow.png
create mode 100644 rfc/rfc-106/rfc-106.md
create mode 100644 rfc/rfc-106/rli-access-pattern.png
