This is an automated email from the ASF dual-hosted git repository.
yihua pushed a change to branch branch-0.x
in repository https://gitbox.apache.org/repos/asf/hudi.git
discard 1ca8cc7dc55 [HUDI-7207] Sequentially delete complete instant files in
archival to prevent inconsistency during data reads (#10711)
new 7b5b6c79cee Move version to 0.15.0-SNAPSHOT
new 6f25f414abd [HUDI-6825] Use UTF_8 to encode String to byte array in
all places (#9634)
new 232255ed473 [HUDI-6826] Port BloomFilter related classes from Hadoop
library to remove dependency (#9649)
new d0e98e163bd [HUDI-6850] Add tests and docs for ported Bloom Filter
classes (#9700)
new ec91bbcfb44 [MINOR] Update cleaner docs (#9716)
new 3998ef60dfb [MINOR] Move hoodie hfile/orc reader/writer test cases
from hudi-client-common to hudi-common (#9103)
new 82bd7658f10 [MINOR] Mark advanced configs and fix since version (#9757)
new 52c42f86a48 [HUDI-53] Update RFC-8 for Metadata based Record Index
(#9775)
new 69d09981827 [MINOR] Update DOAP with 0.14.0 Release (#9803)
new 1911c27d6c4 [HUDI-7016] Fix bundling of RoaringBitmap dependency
(#9963)
new 7acc41e7646 [HUDI-6993] Support Flink 1.18 (#9949)
new 8fc4135fe5e [HUDI-7082] Add Flink 1.14 and Spark 3.13 docker image
script (#10066)
new c0720077785 [HUDI-7016] Fix bundling of RoaringBitmap in
hudi-utilities-bundle (#10083)
new ae80cbd8175 [HUDI-6806] Support Spark 3.5.0 (#9717)
new 1605c2832c6 [HUDI-7113] Update release scripts and docs for Spark 3.5
support (#10123)
new 149ca9a2e33 [HUDI-7072] Remove support for Flink 1.13 (#10052)
new d1366d83aea [MINOR] Add Hopsworks File System to StorageSchemes
(#10141)
new 008320ca375 [HUDI-7207] Sequentially delete complete instant files in
archival to prevent inconsistency during data reads (#10711)
new af3f258ebac [HUDI-4699] Claiming RFC for auto record key generation
(#10357)
new 50119d28644 [HUDI-4699] Adding RFC for auto record key generation
(#10365)
new 155a66c13de [HUDI-7190] Fix nested columns vectorized read for
spark33+ legacy formats (#10265)
new e1625b1d91f [HUDI-7213] When using wrong tabe.type value in hudi
catalog happends npe (#10300)
new a8ef9d40206 [HUDI-7242] Avoid unnecessary bigquery table update when
using sync tool (#10374)
new 353d281e19b [MINOR] Merge logs into check instant file of
HoodieActiveTimeline.transitionPendingState (#10392)
new 5faefcd01fa [MINOR] DataStream need in closeure in
FileSystemBasedLockProvider (#10411)
new 1be74478d9c [HUDI-7249] Disable mor compaction scheduling when using
append mode (#10388)
new 94a162a4059 [HUDI-7268] HoodieFlinkStreamer should disable compaction
in pipeline with append mode (#10430)
new acace8f799f [HUDI-7260] Fix call repair_overwrite_hoodie_props failure
error due to specify hoodie.properties path (#10413)
new 2601a0e1044 [MINOR] Fix ArchivalUtils Logger named (#10436)
new 595d23029d3 [HUDI-7198] Create nested node path if does not exist for
zookeeper. (#10438)
new 37ff8fee231 [HUDI-7271] Copy a conf in ClusteringOperator to avoid
configuration leak (#10441)
new 91d7983504c [MINOR] Updating doap file for 0.14.1 release (#10439)
new 60b073fea4c [HUDI-7266] Add clustering metric for flink (#10420)
new 6ffc817a1e9 [MINOR] Disable flaky test (#10449)
new ef1ccce6774 [HUDI-7279] make sampling rate configurable for
BOUNDED_IN_MEMORY executor type (#10459)
new fc587b374f9 [HUDI-5973] Fixing refreshing of schemas in HoodieStreamer
continuous mode (#10261)
new b712666384e [MINOR] Fix unit tests (#10362)
new d1dd4a4ebb2 [HUDI-7284] Stream sync doesn't differentiate replace
commits (#10467)
new c0e59e95f57 [HUDI-7241] Avoid always broadcast HUDI relation if not
using HoodieSparkSessionExtension (#10373)
new 26df317e778 [MINOR] Fix usages of orElse (#10435)
new fcd6cd96210 [MINOR] Avoid resource leaks (#10345)
new cdefb4b7473 [HUDI-7288] Fix ArrayIndexOutOfBoundsException when
upgrade nonPartitionedTable created by 0.10/0.11 HUDI version (#10482)
new ef7f5237f90 [MINOR] Turning on publishing of test results to Azure
Devops (#10477)
new 635d0c6d507 [MINOR] Parallelized the check for existence of files in
IncrementalRelation. (#10480)
new 8546cbfddce [HUDI-7282] Avoid verification failure due to append
writing of the cow table with cluster configuration when the index is bucket.
(#10475)
new 744befe952b [HUDI-6902] Use mvnw command for hadoo-mr test (#10474)
new 36eeb94b264 [HUDI-6902] Give minimum memory for unit tests (#10469)
new da6a49061d6 [HUDI-7278] make bloom filter skippable for CPU saving
(#10457)
new 7d97216703b [HUDI-7293] Incremental read of insert table using
rebalance strategy (#10490)
new 2b2e1a0a19a [HUDI-7286] Flink get hudi index type ignore case
sensitive (#10476)
new 0de5f076524 [HUDI-6092] Set the timeout for the forked JVM (#10496)
new d414b6033a2 [MINOR] Clean default Hadoop configuration values in tests
(#10495)
new 9ddcfb166f0 [HUDI-7300] Merge schema in ParuqetDFSSource (#10199)
new 5bc160bf0a7 [MINOR] Fix eager rollback mdt ut (#10506)
new 8048c9988eb [HUDI-7296] Reduce CI Time by Minimizing Duplicate Code
Coverage in Tests (#10492)
new 7c13eb3e1c5 [HUDI-7246] Fix Data Skipping Issue: No Results When Query
Conditions Involve Both Columns with and without Column Stats (#10389)
new 23372705171 [HUDI-7170] Implement HFile reader independent of HBase
(#10241)
new a508d54e132 [HUDI-6902] Fix a unit test (#10513)
new 3facb0a2584 [HUDI-6902] Shutdown metric hooks properly (#10520)
new e8f34c3ecd5 [HUDI-7305] Fix cast exception for byte/short/float
partitioned field (#10518)
new 975ba221571 [HUDI-7297] Fix ambiguous error message when field type
defined in schema mismatches that in parquet file (#10497)
new cefc5300145 [HUDI-7309] Disable constructing AND & OR filter
predicates when filter pushing down for any of its operand's logical type for
is unsupported in ExpressionPredicates::toParquetPredicate (#10524)
new 0705849cf1f [HUDI-7284] Fix cluster stream sync check (#10501)
new 4361432dc63 [HUDI-7314] Hudi Create table support index type check
(#10536)
new ccb59939d37 [HUDI-7277] Fix `hoodie.bulkinsert.shuffle.parallelism`
not activated with no-partitioned table (#10532)
new 38525de1763 [MINOR] Added descriptive exception if column present in
required avro schema does not exist in hudi table (#10527)
new e5cabe6f168 [HUDI-7315] Disable constructing NOT filter predicate when
pushing down its wrapped filter unsupported, as its operand's primitive value
is incomparable (#10537)
new c9cdc2a04fa [HUDI-7317] FlinkTableFactory snatifyCheck should contains
index type (#10541)
new 288898e0058 [HUDI-7303] Fix date field type unexpectedly convert to
Long when using date comparison operator (#10517)
new 1554908a2fd [MINOR] Reduce UT spark-datasource test times (#10547)
new 1b37ee267ea [HUDI-7237] Hudi Streamer: Handle edge case with null
schema, minor cleanups (#10342)
new cef039f6cda [HUDI-7316] AbstractHoodieLogRecordReader should accept
HoodieTableMetaClient in order to reduce occurences of executors making file
listing calls when reloading active timeline (#10540)
new 492daf0272f [HUDI-7311] Add implicit literal type conversion before
filter push down (#10531)
new 126010b803f [HUDI-7228] Fix eager closure of log reader input streams
with log record reader (#10340)
new 9002a02a2d8 [HUDI-7298] Write bad records to error table in more cases
instead of failing stream (#10500)
new 31adbb92fe1 [HUDI-7323] Use a schema supplier instead of a static
value (#10549)
new 6f27d81c169 [HUDI-7327] remove meta cols from incoming schema in
stream sync (#10556)
new 54a3b674594 [HUDI-6230] Handle aws glue partition index (#8743)
new e76f2e84ebd [MINOR] add logger to CompactionPlanOperator &
ClusteringPlanOperator (#10562)
new 6dd4beaed63 [HUDI-7308] LockManager::unlock should not call
updateLockHeldTimerMetrics if lockDurationTimer has not been started (#10523)
new 86e3ca6d9bd [HUDI-7335] Create hudi-hadoop-common for hadoop-specific
implementation (#10564)
new b5200bfed28 [HUDI-7351] Fix missing implementation for glue metastore
schema retrieval (#10572)
new 005c7584958 [HUDI-7336] Introduce new HoodieStorage abstraction
(#10567)
new e00e2d7e896 [HUDI-7342] Use BaseFileUtils to hide format-specific
logic in HoodiePartitionMetadata (#10568)
new a05834462c4 [HUDI-7218] Integrate new HFile reader with file reader
factory (#10330)
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (1ca8cc7dc55)
\
N -- N -- N refs/heads/branch-0.x (a05834462c4)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 85 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/bot.yml | 43 +-
LICENSE | 15 +-
README.md | 21 +-
azure-pipelines-20230430.yml | 40 +-
doap_HUDI.rdf | 10 +
docker/hoodie/hadoop/base/pom.xml | 2 +-
docker/hoodie/hadoop/base_java11/pom.xml | 2 +-
docker/hoodie/hadoop/datanode/pom.xml | 2 +-
docker/hoodie/hadoop/historyserver/pom.xml | 2 +-
docker/hoodie/hadoop/hive_base/pom.xml | 2 +-
docker/hoodie/hadoop/namenode/pom.xml | 2 +-
docker/hoodie/hadoop/pom.xml | 2 +-
docker/hoodie/hadoop/prestobase/pom.xml | 2 +-
docker/hoodie/hadoop/spark_base/pom.xml | 2 +-
docker/hoodie/hadoop/sparkadhoc/pom.xml | 2 +-
docker/hoodie/hadoop/sparkmaster/pom.xml | 2 +-
docker/hoodie/hadoop/sparkworker/pom.xml | 2 +-
docker/hoodie/hadoop/trinobase/pom.xml | 2 +-
docker/hoodie/hadoop/trinocoordinator/pom.xml | 2 +-
docker/hoodie/hadoop/trinoworker/pom.xml | 2 +-
hudi-aws/pom.xml | 4 +-
.../hudi/aws/sync/AWSGlueCatalogSyncClient.java | 145 +-
.../hudi/aws/sync/AwsGlueCatalogSyncTool.java | 4 +-
.../hudi/config/GlueCatalogSyncClientConfig.java | 19 +
hudi-cli/pom.xml | 2 +-
.../main/java/org/apache/hudi/cli/HoodieCLI.java | 4 +-
.../hudi/cli/commands/ArchivedCommitsCommand.java | 10 +-
.../apache/hudi/cli/commands/ExportCommand.java | 7 +-
.../hudi/cli/commands/HoodieLogFileCommand.java | 2 +-
.../org/apache/hudi/cli/commands/SparkMain.java | 8 +-
.../org/apache/hudi/cli/commands/TableCommand.java | 3 +-
.../java/org/apache/hudi/cli/utils/SparkUtil.java | 4 +-
.../hudi/cli/commands/TestCleansCommand.java | 4 +-
.../hudi/cli/commands/TestCompactionCommand.java | 8 +-
.../apache/hudi/cli/commands/TestDiffCommand.java | 4 +-
.../cli/commands/TestHoodieLogFileCommand.java | 4 +-
.../hudi/cli/commands/TestRepairsCommand.java | 3 +-
.../cli/integ/ITTestHDFSParquetImportCommand.java | 10 +-
.../HoodieTestCommitMetadataGenerator.java | 7 +-
hudi-client/hudi-client-common/pom.xml | 4 +-
.../org/apache/hudi/client/BaseHoodieClient.java | 4 +-
.../hudi/client/BaseHoodieTableServiceClient.java | 9 +-
.../apache/hudi/client/BaseHoodieWriteClient.java | 8 +-
.../apache/hudi/client/CompactionAdminClient.java | 2 +-
.../apache/hudi/client/HoodieTimelineArchiver.java | 4 +-
.../client/embedded/EmbeddedTimelineService.java | 4 +-
.../lock/FileSystemBasedLockProvider.java | 14 +-
.../hudi/client/transaction/lock/LockManager.java | 7 +-
.../lock/ZookeeperBasedLockProvider.java | 42 +
.../apache/hudi/client/utils/ArchivalUtils.java | 3 +-
.../apache/hudi/client/utils/TransactionUtils.java | 2 +-
.../org/apache/hudi/config/HoodieCleanConfig.java | 43 +-
.../apache/hudi/config/HoodieCompactionConfig.java | 2 +-
.../apache/hudi/config/HoodieErrorTableConfig.java | 6 +
.../org/apache/hudi/config/HoodieWriteConfig.java | 39 +-
.../org/apache/hudi/index/HoodieIndexUtils.java | 3 +-
.../index/bucket/ConsistentBucketIndexUtils.java | 5 +-
.../org/apache/hudi/io/HoodieAppendHandle.java | 4 +-
.../java/org/apache/hudi/io/HoodieMergeHandle.java | 3 +-
.../org/apache/hudi/io/HoodieMergedReadHandle.java | 1 +
.../java/org/apache/hudi/io/HoodieReadHandle.java | 8 +-
.../metadata/HoodieBackedTableMetadataWriter.java | 12 +-
.../main/java/org/apache/hudi/metrics/Metrics.java | 39 +-
.../java/org/apache/hudi/table/HoodieTable.java | 10 +-
.../action/commit/BaseCommitActionExecutor.java | 4 +-
.../table/action/commit/HoodieMergeHelper.java | 4 +-
.../hudi/table/action/compact/CompactHelpers.java | 7 +-
.../hudi/table/action/compact/HoodieCompactor.java | 1 +
.../HoodieLogCompactionPlanGenerator.java | 1 +
.../rollback/ListingBasedRollbackStrategy.java | 2 +-
.../action/savepoint/SavepointActionExecutor.java | 2 +-
.../hudi/table/marker/DirectWriteMarkers.java | 3 +-
.../SimpleDirectMarkerBasedDetectionStrategy.java | 2 +-
...nsactionDirectMarkerBasedDetectionStrategy.java | 2 +-
.../hudi/table/marker/WriteMarkersFactory.java | 6 +-
.../table/upgrade/FourToFiveUpgradeHandler.java | 2 +-
.../table/upgrade/TwoToOneDowngradeHandler.java | 3 +-
.../java/org/apache/hudi/util/ExecutorFactory.java | 4 +-
.../org/apache/hudi/HoodieTestCommitGenerator.java | 9 +-
.../FileSystemBasedLockProviderTestClass.java | 4 +-
.../InProcessLockProviderWithRuntimeError.java | 33 +-
.../client/transaction/TestTransactionManager.java | 27 +-
.../io/storage/TestHoodieHFileReaderWriter.java | 479 ---
.../hudi/table/marker/TestWriteMarkersFactory.java | 2 +-
.../GenericRecordValidationTestUtils.java | 26 +-
hudi-client/hudi-flink-client/pom.xml | 4 +-
.../hudi/client/HoodieFlinkTableServiceClient.java | 7 +-
.../row/HoodieRowDataParquetWriteSupport.java | 4 +-
.../commit/BaseFlinkCommitActionExecutor.java | 5 +-
.../testutils/HoodieFlinkClientTestHarness.java | 4 +-
hudi-client/hudi-java-client/pom.xml | 4 +-
.../run/strategy/JavaExecutionStrategy.java | 6 +-
.../commit/BaseJavaCommitActionExecutor.java | 5 +-
.../table/action/commit/JavaBulkInsertHelper.java | 2 +-
.../hudi/client/TestJavaHoodieBackedMetadata.java | 27 +-
.../testutils/HoodieJavaClientTestHarness.java | 47 +-
.../hudi/testutils/TestHoodieMetadataBase.java | 2 +-
hudi-client/hudi-spark-client/pom.xml | 4 +-
.../apache/hudi/client/SparkRDDWriteClient.java | 2 +-
.../MultipleSparkJobExecutionStrategy.java | 9 +-
.../strategy/SingleSparkJobExecutionStrategy.java | 5 +-
.../client/utils/SparkInternalSchemaConverter.java | 14 +-
.../hudi/client/utils/SparkPartitionUtils.java | 2 +-
.../io/storage/HoodieSparkFileReaderFactory.java | 5 +-
.../io/storage/HoodieSparkFileWriterFactory.java | 3 +-
.../hudi/io/storage/row/HoodieRowCreateHandle.java | 2 +-
.../bootstrap/ParquetBootstrapMetadataHandler.java | 2 +-
.../SparkBootstrapCommitActionExecutor.java | 8 +-
.../commit/BaseSparkCommitActionExecutor.java | 6 +-
.../table/action/commit/SparkBulkInsertHelper.java | 2 +-
.../SparkInsertOverwriteCommitActionExecutor.java | 2 +-
.../org/apache/hudi/AvroConversionUtils.scala | 22 +-
.../hudi/HoodieDatasetBulkInsertHelper.scala | 29 +-
.../scala/org/apache/hudi/HoodieSparkUtils.scala | 32 +-
.../org/apache/hudi/SparkAdapterSupport.scala | 4 +-
.../scala/org/apache/spark/sql/DataFrameUtil.scala | 6 +-
.../spark/sql/HoodieCatalystExpressionUtils.scala | 16 +-
.../org/apache/spark/sql/HoodieSchemaUtils.scala | 9 +
.../org/apache/spark/sql/HoodieUnsafeUtils.scala | 13 +-
.../HoodieSparkPartitionedFileUtils.scala | 20 +-
.../org/apache/spark/sql/hudi/SparkAdapter.scala | 5 +-
.../java/org/apache/hudi/client/TestMultiFS.java | 6 +-
.../hudi/client/TestUpdateSchemaEvolution.java | 4 +-
.../functional/TestHoodieBackedMetadata.java | 19 +-
.../functional/TestHoodieBackedTableMetadata.java | 12 +-
.../TestHoodieClientOnMergeOnReadStorage.java | 2 +
.../fs/TestHoodieSerializableFileStatus.java | 2 +-
.../apache/hudi/io/TestHoodieTimelineArchiver.java | 8 +-
.../storage/TestHoodieAvroFileWriterFactory.java | 31 +
.../apache/hudi/table/TestConsistencyGuard.java | 2 +-
.../table/action/commit/TestUpsertPartitioner.java | 14 +-
.../table/action/compact/TestHoodieCompactor.java | 4 +-
.../hudi/table/marker/TestDirectWriteMarkers.java | 4 +-
.../TestTimelineServerBasedWriteMarkers.java | 4 +-
.../hudi/testutils/HoodieCleanerTestBase.java | 4 +-
.../hudi/testutils/HoodieClientTestUtils.java | 59 +-
.../testutils/HoodieSparkClientTestHarness.java | 12 +-
.../SparkClientFunctionalTestHarness.java | 4 +-
hudi-client/pom.xml | 2 +-
hudi-common/pom.xml | 22 +-
.../org/apache/hudi/BaseHoodieTableFileIndex.java | 8 +-
.../apache/hudi/avro/GenericAvroSerializer.java | 6 +-
.../java/org/apache/hudi/avro/HoodieAvroUtils.java | 58 +-
.../apache/hudi/avro/HoodieAvroWriteSupport.java | 4 +-
.../apache/hudi/avro/MercifulJsonConverter.java | 4 +-
.../org/apache/hudi/common/HoodieJsonPayload.java | 4 +-
.../hudi/common/bloom/BloomFilterFactory.java | 21 +-
.../org/apache/hudi/common/bloom/HashFunction.java | 132 +
.../bloom/HoodieDynamicBoundedBloomFilter.java | 41 +-
.../hudi/common/bloom/InternalBloomFilter.java | 248 ++
.../common/bloom/InternalDynamicBloomFilter.java | 33 +-
.../apache/hudi/common/bloom/InternalFilter.java | 58 +-
.../java/org/apache/hudi/common/bloom/Key.java | 176 +
.../hudi/common/bloom/SimpleBloomFilter.java | 52 +-
.../bootstrap/index/HFileBootstrapIndex.java | 246 +-
.../apache/hudi/common/config/ConfigGroups.java | 1 +
.../common/config/DFSPropertiesConfiguration.java | 6 +-
.../hudi/common/config/HoodieCommonConfig.java | 1 +
.../apache/hudi/common/config/HoodieConfig.java | 2 +-
.../hudi/common/config/HoodieMetadataConfig.java | 5 +-
.../hudi/common/config/HoodieReaderConfig.java | 24 +-
.../hudi/common/config/HoodieStorageConfig.java | 11 +
.../DirectMarkerBasedDetectionStrategy.java | 2 +-
.../java/org/apache/hudi/common/fs/FSUtils.java | 72 +-
.../hudi/common/fs/FailSafeConsistencyGuard.java | 13 +-
.../hudi/common/fs/OptimisticConsistencyGuard.java | 2 +
.../org/apache/hudi/common/model/BaseFile.java | 2 +-
.../apache/hudi/common/model/HoodieBaseFile.java | 4 +-
.../hudi/common/model/HoodieCommitMetadata.java | 3 +-
.../model/HoodieConsistentHashingMetadata.java | 6 +-
.../apache/hudi/common/model/HoodieLogFile.java | 2 +-
.../hudi/common/model/HoodiePartitionMetadata.java | 41 +-
.../hudi/common/table/HoodieTableConfig.java | 4 +-
.../hudi/common/table/HoodieTableMetaClient.java | 16 +-
.../hudi/common/table/TableSchemaResolver.java | 12 +-
.../table/log/AbstractHoodieLogRecordReader.java | 13 +-
.../hudi/common/table/log/HoodieLogFileReader.java | 26 +-
.../common/table/log/HoodieLogFormatReader.java | 8 +-
.../common/table/log/HoodieLogFormatWriter.java | 3 +-
.../table/log/HoodieMergedLogRecordScanner.java | 17 +-
.../table/log/HoodieUnMergedLogRecordScanner.java | 17 +-
.../hudi/common/table/log/LogReaderUtils.java | 4 +-
.../table/log/block/HoodieAvroDataBlock.java | 3 +-
.../common/table/log/block/HoodieDataBlock.java | 6 +-
.../table/log/block/HoodieHFileDataBlock.java | 54 +-
.../common/table/log/block/HoodieLogBlock.java | 3 +-
.../table/log/block/HoodieParquetDataBlock.java | 4 +-
.../table/timeline/HoodieActiveTimeline.java | 4 +-
.../table/timeline/HoodieArchivedTimeline.java | 4 +-
.../table/timeline/HoodieDefaultTimeline.java | 18 +-
.../hudi/common/table/timeline/HoodieTimeline.java | 12 +
.../common/table/timeline/dto/FilePathDTO.java | 2 +-
.../view/HoodieTablePreCommitFileSystemView.java | 2 +-
.../org/apache/hudi/common/util/AvroOrcUtils.java | 8 +-
.../apache/hudi/common/util/Base64CodecUtil.java | 15 +-
.../org/apache/hudi/common/util/BaseFileUtils.java | 15 +
.../org/apache/hudi/common/util/BinaryUtil.java | 5 +-
.../apache/hudi/common/util/ClusteringUtils.java | 30 +-
.../org/apache/hudi/common/util/ConfigUtils.java | 65 +-
.../hudi/common/util/InternalSchemaCache.java | 4 +-
.../org/apache/hudi/common/util/NumericUtils.java | 5 +-
.../java/org/apache/hudi/common/util/OrcUtils.java | 22 +-
.../org/apache/hudi/common/util/ParquetUtils.java | 31 +-
.../common/util/collection/LazyFileIterable.java | 9 +-
.../hudi/common/util/collection/RocksDBDAO.java | 23 +-
.../org/apache/hudi/common/util/hash/Hash.java | 112 +
.../org/apache/hudi/common/util/hash/HashID.java | 7 +-
.../apache/hudi/common/util/hash/JenkinsHash.java | 287 ++
.../apache/hudi/common/util/hash/MurmurHash.java | 92 +
.../util/queue/BaseHoodieQueueBasedExecutor.java | 2 +-
.../common/util/queue/BoundedInMemoryExecutor.java | 14 +
.../common/util/queue/BoundedInMemoryQueue.java | 28 +-
.../apache/hudi/expression/PartialBindVisitor.java | 4 +-
.../io/FileBasedInternalSchemaStorageManager.java | 7 +-
.../schema/utils/AvroSchemaEvolutionUtils.java | 2 +-
.../hudi/internal/schema/utils/SerDeHelper.java | 6 +-
.../io/storage/HoodieAvroFileReaderFactory.java | 30 +-
.../io/storage/HoodieAvroFileWriterFactory.java | 9 +-
.../io/storage/HoodieAvroHFileReaderImplBase.java | 154 +
.../hudi/io/storage/HoodieAvroHFileWriter.java | 56 +-
.../hudi/io/storage/HoodieAvroOrcWriter.java | 28 +-
.../hudi/io/storage/HoodieBaseParquetWriter.java | 10 +-
.../io/storage/HoodieBootstrapRecordIterator.java | 3 +-
.../hudi/io/storage/HoodieFileReaderFactory.java | 68 +-
.../hudi/io/storage/HoodieFileWriterFactory.java | 10 +
...Reader.java => HoodieHBaseAvroHFileReader.java} | 169 +-
.../apache/hudi/io/storage/HoodieHFileUtils.java | 3 +-
.../io/storage/HoodieNativeAvroHFileReader.java | 559 ++++
.../hudi/metadata/AbstractHoodieTableMetadata.java | 7 +-
.../apache/hudi/metadata/BaseTableMetadata.java | 2 +-
.../metadata/FileSystemBackedTableMetadata.java | 5 +-
.../hudi/metadata/HoodieBackedTableMetadata.java | 6 +-
.../metadata/HoodieMetadataLogRecordReader.java | 6 +
.../hudi/metadata/HoodieMetadataPayload.java | 8 +-
.../hudi/metadata/HoodieTableMetadataUtil.java | 44 +-
.../secondary/index/SecondaryIndexManager.java | 2 +-
.../org/apache/hudi/avro/TestHoodieAvroUtils.java | 7 +-
.../apache/hudi/common/bloom/TestBloomFilter.java | 70 +
.../bloom/TestInternalDynamicBloomFilter.java | 3 +-
.../org/apache/hudi/common/fs/TestFSUtils.java | 7 +-
.../fs/TestFSUtilsWithRetryWrapperEnable.java | 13 +-
.../common/fs/TestHoodieWrapperFileSystem.java | 11 +-
.../apache/hudi/common/fs/TestStorageSchemes.java | 4 +
... => TestInLineFileSystemHFileInLiningBase.java} | 94 +-
.../TestInLineFileSystemWithHBaseHFileReader.java | 124 +
.../TestInLineFileSystemWithHFileReader.java | 104 +
.../common/functional/TestHoodieLogFormat.java | 43 +-
.../debezium/TestPostgresDebeziumAvroPayload.java | 9 +-
.../common/table/TestHoodieTableMetaClient.java | 9 +-
.../hudi/common/table/TestTimelineUtils.java | 6 +-
.../hudi/common/table/log/TestLogReaderUtils.java | 11 +-
.../table/timeline/TestHoodieActiveTimeline.java | 8 +-
.../view/TestHoodieTableFSViewWithClustering.java | 4 +-
.../table/view/TestHoodieTableFileSystemView.java | 20 +-
.../table/view/TestIncrementalFSViewSync.java | 10 +-
.../hudi/common/testutils/FileCreateUtils.java | 12 +-
.../hudi/common/testutils/FileSystemTestUtils.java | 10 +
.../common/testutils/HoodieTestDataGenerator.java | 18 +-
.../hudi/common/testutils/HoodieTestUtils.java | 6 +-
.../hudi/common/testutils/RawTripTestPayload.java | 3 +-
.../hudi/common/testutils/SchemaTestUtil.java | 5 +-
.../minicluster/ZookeeperTestService.java | 6 +-
.../hudi/common/util/TestBase64CodecUtil.java | 9 +-
.../hudi/common/util/TestClusteringUtils.java | 17 +
.../util/TestDFSPropertiesConfiguration.java | 4 +-
.../apache/hudi/common/util/TestFileIOUtils.java | 10 +-
.../apache/hudi/common/util/TestMarkerUtils.java | 6 +-
.../hudi/common/util/TestOrcReaderIterator.java | 6 +-
.../apache/hudi/common/util/TestStringUtils.java | 7 +-
.../apache/hudi/common/util/hash/TestHashID.java | 4 +-
.../storage/TestHoodieAvroFileReaderFactory.java | 10 +-
.../storage/TestHoodieHBaseHFileReaderWriter.java | 142 +
.../io/storage/TestHoodieHFileReaderWriter.java | 87 +
.../storage/TestHoodieHFileReaderWriterBase.java | 486 +++
.../hudi/io/storage/TestHoodieOrcReaderWriter.java | 6 +-
.../io/storage/TestHoodieReaderWriterBase.java | 0
.../io/storage/TestHoodieReaderWriterUtils.java | 89 +
.../src/test/resources/exampleEvolvedSchema.avsc | 0
.../resources/exampleEvolvedSchemaChangeOrder.avsc | 0
.../exampleEvolvedSchemaColumnRequire.avsc | 0
.../resources/exampleEvolvedSchemaColumnType.avsc | 0
.../exampleEvolvedSchemaDeleteColumn.avsc | 0
.../src/test/resources/exampleSchema.avsc | 0
.../resources/exampleSchemaWithMetaFields.avsc | 0
.../src/test/resources/exampleSchemaWithUDT.avsc | 0
.../format/bloom-filter/hadoop/all_10000.keys.data | 19 +
.../dynamic_1000_000001_jenkins_5000.bf.data | 19 +
.../hadoop/dynamic_1000_000001_murmur_5000.bf.data | 19 +
.../hadoop/dynamic_200_000001_murmur_1000.bf.data | 19 +
.../hadoop/simple_10000_000001_murmur.bf.data | 19 +
.../hadoop/simple_1000_000001_murmur.bf.data | 19 +
.../hadoop/simple_200_000001_murmur.bf.data | 19 +
.../hadoop/simple_5000_000001_jenkins.bf.data | 19 +
.../hadoop/simple_5000_000001_murmur.bf.data | 19 +
hudi-examples/hudi-examples-common/pom.xml | 2 +-
hudi-examples/hudi-examples-flink/pom.xml | 2 +-
hudi-examples/hudi-examples-java/pom.xml | 2 +-
.../java/HoodieJavaWriteClientExample.java | 4 +-
hudi-examples/hudi-examples-spark/pom.xml | 2 +-
.../examples/spark/HoodieWriteClientExample.java | 4 +-
hudi-examples/pom.xml | 2 +-
hudi-flink-datasource/hudi-flink/pom.xml | 5 +-
.../apache/hudi/configuration/OptionsResolver.java | 2 +-
.../hudi/metrics/FlinkClusteringMetrics.java | 105 +
.../hudi/schema/FilebasedSchemaProvider.java | 6 +-
.../apache/hudi/schema/SchemaRegistryProvider.java | 4 +-
.../hudi/sink/clustering/ClusteringCommitSink.java | 12 +
.../hudi/sink/clustering/ClusteringOperator.java | 23 +-
.../sink/clustering/ClusteringPlanOperator.java | 25 +-
.../hudi/sink/compact/CompactionPlanOperator.java | 3 +
.../org/apache/hudi/sink/meta/CkpMetadata.java | 4 +-
.../sink/partitioner/profile/WriteProfiles.java | 4 +-
.../RowDataToHoodieFunctionWithRateLimit.java | 10 +-
.../apache/hudi/source/ExpressionPredicates.java | 26 +-
.../org/apache/hudi/source/StreamReadOperator.java | 41 +-
.../apache/hudi/streamer/HoodieFlinkStreamer.java | 2 +
.../org/apache/hudi/table/HoodieTableFactory.java | 12 +
.../org/apache/hudi/table/HoodieTableSink.java | 2 +
.../org/apache/hudi/table/HoodieTableSource.java | 22 +-
.../apache/hudi/table/catalog/HoodieCatalog.java | 4 +-
.../hudi/table/catalog/HoodieHiveCatalog.java | 40 +-
.../hudi/table/catalog/TableOptionProperties.java | 18 +-
.../apache/hudi/table/format/FilePathUtils.java | 6 +-
.../org/apache/hudi/table/format/FormatUtils.java | 8 +-
.../hudi/table/format/cdc/CdcInputFormat.java | 4 +-
.../table/format/cow/CopyOnWriteInputFormat.java | 12 +-
.../main/java/org/apache/hudi/util/ClientIds.java | 6 +-
.../java/org/apache/hudi/util/ClusteringUtil.java | 2 +-
.../java/org/apache/hudi/util/ExpressionUtils.java | 4 +-
.../apache/hudi/util/ImplicitTypeConverter.java | 134 +
.../hudi/util/JsonDeserializationFunction.java | 4 +-
.../java/org/apache/hudi/util/StreamerUtil.java | 7 +-
.../apache/hudi/util/StringToRowDataConverter.java | 5 +-
.../apache/hudi/util/ViewStorageProperties.java | 6 +-
.../hudi/configuration/TestOptionsResolver.java | 56 +
.../sink/TestStreamWriteOperatorCoordinator.java | 6 +-
.../hudi/sink/bucket/ITTestBucketStreamWrite.java | 3 +-
.../bucket/ITTestConsistentBucketStreamWrite.java | 4 +-
.../sink/compact/ITTestHoodieFlinkCompactor.java | 2 +-
.../org/apache/hudi/sink/meta/TestCkpMetadata.java | 4 +-
.../hudi/sink/utils/ClusteringFunctionWrapper.java | 6 +
.../apache/hudi/sink/utils/CollectorOutput.java | 10 +-
.../sink/utils/MockStateInitializationContext.java | 12 +-
.../sink/utils/MockStreamingRuntimeContext.java | 11 +-
.../org/apache/hudi/sink/utils/TestWriteBase.java | 4 +-
.../hudi/source/TestExpressionPredicates.java | 81 +-
.../hudi/source/TestIncrementalInputSplits.java | 6 +-
.../apache/hudi/table/ITTestHoodieDataSource.java | 2 +-
.../apache/hudi/table/TestHoodieTableFactory.java | 25 +
.../table/catalog/TestHoodieCatalogFactory.java | 2 +-
.../hudi/table/catalog/TestHoodieHiveCatalog.java | 16 +-
.../org/apache/hudi/util/TestExpressionUtils.java | 6 +-
.../org/apache/hudi/utils/TestClusteringUtil.java | 11 +
.../org/apache/hudi/utils/TestStreamerUtil.java | 4 +-
.../java/org/apache/hudi/utils}/TestTableEnvs.java | 2 +-
.../test/java/org/apache/hudi/utils/TestUtils.java | 5 +-
hudi-flink-datasource/hudi-flink1.13.x/pom.xml | 144 -
.../adapter/AbstractStreamOperatorAdapter.java | 35 -
.../AbstractStreamOperatorFactoryAdapter.java | 50 -
.../adapter/DataStreamScanProviderAdapter.java | 27 -
.../adapter/DataStreamSinkProviderAdapter.java | 27 -
.../hudi/adapter/MailboxExecutorAdapter.java | 37 -
.../apache/hudi/adapter/MaskingOutputAdapter.java | 61 -
.../hudi/adapter/OperatorCoordinatorAdapter.java | 27 -
.../hudi/adapter/SortCodeGeneratorAdapter.java | 33 -
.../adapter/SupportsRowLevelDeleteAdapter.java | 33 -
.../adapter/SupportsRowLevelUpdateAdapter.java | 37 -
.../main/java/org/apache/hudi/adapter/Utils.java | 83 -
.../apache/hudi/table/data/ColumnarArrayData.java | 270 --
.../apache/hudi/table/data/ColumnarMapData.java | 73 -
.../apache/hudi/table/data/ColumnarRowData.java | 231 --
.../hudi/table/data/vector/MapColumnVector.java | 29 -
.../hudi/table/data/vector/RowColumnVector.java | 30 -
.../table/data/vector/VectorizedColumnBatch.java | 148 -
.../table/format/cow/ParquetSplitReaderUtil.java | 579 ----
.../table/format/cow/vector/HeapArrayVector.java | 71 -
.../format/cow/vector/HeapMapColumnVector.java | 80 -
.../format/cow/vector/HeapRowColumnVector.java | 55 -
.../format/cow/vector/ParquetDecimalVector.java | 54 -
.../cow/vector/reader/AbstractColumnReader.java | 325 --
.../cow/vector/reader/ArrayColumnReader.java | 473 ---
.../vector/reader/BaseVectorizedColumnReader.java | 313 --
.../cow/vector/reader/EmptyColumnReader.java | 41 -
.../vector/reader/FixedLenBytesColumnReader.java | 84 -
.../vector/reader/Int64TimestampColumnReader.java | 119 -
.../format/cow/vector/reader/MapColumnReader.java | 76 -
.../reader/ParquetColumnarRowSplitReader.java | 390 ---
.../format/cow/vector/reader/RowColumnReader.java | 63 -
.../format/cow/vector/reader/RunLengthDecoder.java | 304 --
.../org/apache/hudi/adapter/OutputAdapter.java | 27 -
.../adapter/StateInitializationContextAdapter.java | 26 -
.../adapter/StreamingRuntimeContextAdapter.java | 43 -
.../org/apache/hudi/adapter/TestTableEnvs.java | 34 -
hudi-flink-datasource/hudi-flink1.14.x/pom.xml | 4 +-
.../adapter/AbstractStreamOperatorAdapter.java | 27 -
.../AbstractStreamOperatorFactoryAdapter.java | 33 -
.../apache/hudi/adapter/HiveCatalogConstants.java | 52 +
.../hudi/adapter/MailboxExecutorAdapter.java | 37 -
.../apache/hudi/adapter/RateLimiterAdapter.java | 40 -
.../main/java/org/apache/hudi/adapter/Utils.java | 23 -
.../table/format/cow/ParquetSplitReaderUtil.java | 56 +-
.../reader/ParquetColumnarRowSplitReader.java | 16 +-
.../org/apache/hudi/adapter/OutputAdapter.java | 32 -
.../adapter/StreamingRuntimeContextAdapter.java | 43 -
.../org/apache/hudi/adapter/TestTableEnvs.java | 52 -
hudi-flink-datasource/hudi-flink1.15.x/pom.xml | 4 +-
.../adapter/AbstractStreamOperatorAdapter.java | 27 -
.../AbstractStreamOperatorFactoryAdapter.java | 33 -
.../apache/hudi/adapter/HiveCatalogConstants.java | 52 +
.../hudi/adapter/MailboxExecutorAdapter.java | 37 -
.../apache/hudi/adapter/RateLimiterAdapter.java | 40 -
.../main/java/org/apache/hudi/adapter/Utils.java | 23 -
.../table/format/cow/ParquetSplitReaderUtil.java | 56 +-
.../reader/ParquetColumnarRowSplitReader.java | 16 +-
.../org/apache/hudi/adapter/OutputAdapter.java | 32 -
.../adapter/StreamingRuntimeContextAdapter.java | 43 -
.../org/apache/hudi/adapter/TestTableEnvs.java | 52 -
hudi-flink-datasource/hudi-flink1.16.x/pom.xml | 4 +-
.../adapter/AbstractStreamOperatorAdapter.java | 27 -
.../AbstractStreamOperatorFactoryAdapter.java | 33 -
.../apache/hudi/adapter/HiveCatalogConstants.java | 52 +
.../hudi/adapter/MailboxExecutorAdapter.java | 37 -
.../apache/hudi/adapter/RateLimiterAdapter.java | 40 -
.../main/java/org/apache/hudi/adapter/Utils.java | 23 -
.../table/format/cow/ParquetSplitReaderUtil.java | 56 +-
.../reader/ParquetColumnarRowSplitReader.java | 16 +-
.../org/apache/hudi/adapter/OutputAdapter.java | 32 -
.../adapter/StateInitializationContextAdapter.java | 31 -
.../adapter/StreamingRuntimeContextAdapter.java | 43 -
.../org/apache/hudi/adapter/TestTableEnvs.java | 52 -
hudi-flink-datasource/hudi-flink1.17.x/pom.xml | 4 +-
.../adapter/AbstractStreamOperatorAdapter.java | 27 -
.../AbstractStreamOperatorFactoryAdapter.java | 33 -
.../apache/hudi/adapter/HiveCatalogConstants.java | 52 +
.../hudi/adapter/MailboxExecutorAdapter.java | 37 -
.../apache/hudi/adapter/RateLimiterAdapter.java | 40 -
.../main/java/org/apache/hudi/adapter/Utils.java | 23 -
.../table/format/cow/ParquetSplitReaderUtil.java | 56 +-
.../reader/ParquetColumnarRowSplitReader.java | 16 +-
.../org/apache/hudi/adapter/OutputAdapter.java | 32 -
.../adapter/StreamingRuntimeContextAdapter.java | 43 -
.../{hudi-flink1.17.x => hudi-flink1.18.x}/pom.xml | 30 +-
.../adapter/DataStreamScanProviderAdapter.java | 0
.../adapter/DataStreamSinkProviderAdapter.java | 0
.../apache/hudi/adapter/HiveCatalogConstants.java | 49 +
.../apache/hudi/adapter/MaskingOutputAdapter.java | 0
.../hudi/adapter/OperatorCoordinatorAdapter.java | 0
.../hudi/adapter/SortCodeGeneratorAdapter.java | 0
.../adapter/SupportsRowLevelDeleteAdapter.java | 0
.../adapter/SupportsRowLevelUpdateAdapter.java | 0
.../main/java/org/apache/hudi/adapter/Utils.java | 25 +-
.../table/format/cow/ParquetSplitReaderUtil.java | 56 +-
.../table/format/cow/vector/HeapArrayVector.java | 0
.../table/format/cow/vector/HeapDecimalVector.java | 23 +-
.../format/cow/vector/HeapMapColumnVector.java | 0
.../format/cow/vector/HeapRowColumnVector.java | 0
.../format/cow/vector/ParquetDecimalVector.java | 0
.../cow/vector/reader/AbstractColumnReader.java | 0
.../cow/vector/reader/ArrayColumnReader.java | 0
.../vector/reader/BaseVectorizedColumnReader.java | 0
.../cow/vector/reader/EmptyColumnReader.java | 0
.../vector/reader/FixedLenBytesColumnReader.java | 0
.../vector/reader/Int64TimestampColumnReader.java | 0
.../format/cow/vector/reader/MapColumnReader.java | 0
.../reader/ParquetColumnarRowSplitReader.java | 16 +-
.../cow/vector/reader/ParquetDataColumnReader.java | 0
.../reader/ParquetDataColumnReaderFactory.java | 0
.../format/cow/vector/reader/RowColumnReader.java | 0
.../format/cow/vector/reader/RunLengthDecoder.java | 0
.../apache/hudi/adapter/TestStreamConfigs.java} | 23 +-
hudi-flink-datasource/pom.xml | 6 +-
hudi-gcp/pom.xml | 2 +-
.../apache/hudi/gcp/bigquery/BigQuerySyncTool.java | 2 +-
.../gcp/bigquery/HoodieBigQuerySyncClient.java | 12 +-
.../gcp/bigquery/TestHoodieBigQuerySyncClient.java | 35 +
.../pom.xml | 123 +-
.../hudi/hadoop}/fs/BoundedFsDataInputStream.java | 16 +-
.../org/apache/hudi/hadoop/fs}/CachingPath.java | 15 +-
.../apache/hudi/hadoop}/fs/ConsistencyGuard.java | 15 +-
.../org/apache/hudi/hadoop/fs/HadoopFSUtils.java | 85 +
.../hadoop}/fs/HoodieRetryWrapperFileSystem.java | 15 +-
.../hadoop}/fs/HoodieSerializableFileStatus.java | 15 +-
.../hudi/hadoop}/fs/HoodieWrapperFileSystem.java | 28 +-
.../hudi/hadoop}/fs/NoOpConsistencyGuard.java | 15 +-
.../hadoop}/fs/SchemeAwareFSDataInputStream.java | 15 +-
.../apache/hudi/hadoop/fs}/SerializablePath.java | 15 +-
.../hadoop}/fs/SizeAwareFSDataOutputStream.java | 15 +-
.../hudi/hadoop}/fs/TimedFSDataInputStream.java | 15 +-
.../hudi/storage/hadoop/HoodieHadoopStorage.java | 201 ++
.../hadoop/storage/TestHoodieHadoopStorage.java | 53 +
hudi-hadoop-mr/pom.xml | 2 +-
.../hadoop/HoodieCopyOnWriteTableInputFormat.java | 5 +-
.../hudi/hadoop/HoodieHFileRecordReader.java | 22 +-
.../org/apache/hudi/hadoop/InputSplitUtils.java | 4 +-
.../realtime/HoodieMergeOnReadSnapshotReader.java | 4 +-
.../realtime/RealtimeCompactedRecordReader.java | 4 +-
.../apache/hudi/hadoop/realtime/RealtimeSplit.java | 2 +-
.../realtime/RealtimeUnmergedRecordReader.java | 4 +-
.../utils/HoodieRealtimeRecordReaderUtils.java | 5 +-
.../hudi/hadoop/TestHoodieHFileInputFormat.java | 5 +-
.../hudi/hadoop/TestHoodieParquetInputFormat.java | 33 +-
.../TestHoodieMergeOnReadSnapshotReader.java | 2 +-
.../realtime/TestHoodieRealtimeFileSplit.java | 18 +-
.../realtime/TestHoodieRealtimeRecordReader.java | 14 +-
.../hudi/hadoop/testutils/InputFormatTestUtil.java | 3 +-
hudi-integ-test/pom.xml | 2 +-
.../hudi/integ/testsuite/HoodieTestSuiteJob.java | 8 +-
.../SparkDataSourceContinuousIngestTool.java | 4 +-
.../dag/nodes/BaseValidateDatasetNode.java | 13 +-
.../dag/nodes/ValidateAsyncOperations.java | 4 +-
.../integ/testsuite/generator/DeltaGenerator.java | 4 +-
.../testsuite/reader/DFSDeltaInputReader.java | 5 +-
.../reader/DFSHoodieDatasetInputReader.java | 5 +-
.../testsuite/writer/AvroFileDeltaInputWriter.java | 12 +-
.../nodes/spark/sql/SparkSqlCreateTableNode.scala | 4 +-
.../TestDFSHoodieTestSuiteWriterAdapter.java | 4 +-
.../integ/testsuite/TestFileDeltaInputWriter.java | 5 +-
.../reader/TestDFSAvroDeltaInputReader.java | 5 +-
.../README.md | 26 +-
hudi-io/hfile_format.md | 394 +++
hudi-io/pom.xml | 126 +
.../java/org/apache/hudi/ApiMaturityLevel.java | 0
.../main/java/org/apache/hudi/PublicAPIClass.java | 0
.../main/java/org/apache/hudi/PublicAPIMethod.java | 0
.../org/apache/hudi/common/metrics/Counter.java | 0
.../apache/hudi/common/metrics/LocalRegistry.java | 0
.../org/apache/hudi/common/metrics/Metric.java | 0
.../org/apache/hudi/common/metrics/Registry.java | 0
.../org/apache/hudi/common/util/FileIOUtils.java | 13 +-
.../org/apache/hudi/common/util/HoodieTimer.java | 0
.../java/org/apache/hudi/common/util/Option.java | 0
.../apache/hudi/common/util/ReflectionUtils.java | 0
.../org/apache/hudi/common/util/RetryHelper.java | 13 +-
.../org/apache/hudi/common/util/StringUtils.java | 23 +-
.../apache/hudi/common/util/ValidationUtils.java | 0
.../util/io/ByteBufferBackedInputStream.java | 0
.../org/apache/hudi/exception/HoodieException.java | 0
.../apache/hudi/exception/HoodieIOException.java | 0
.../apache/hudi/io/compress/CompressionCodec.java | 29 +-
.../hudi/io/compress/HoodieDecompressor.java | 29 +-
.../io/compress/HoodieDecompressorFactory.java | 23 +-
.../airlift/HoodieAirliftGzipDecompressor.java | 53 +
.../compress/builtin/HoodieNoneDecompressor.java | 25 +-
.../org/apache/hudi/io/hfile/BlockIndexEntry.java | 79 +
.../java/org/apache/hudi/io/hfile/DataSize.java | 29 +-
.../java/org/apache/hudi/io/hfile/HFileBlock.java | 216 ++
.../org/apache/hudi/io/hfile/HFileBlockReader.java | 94 +
.../org/apache/hudi/io/hfile/HFileBlockType.java | 171 +
.../org/apache/hudi/io/hfile/HFileContext.java | 65 +
.../java/org/apache/hudi/io/hfile/HFileCursor.java | 94 +
.../org/apache/hudi/io/hfile/HFileDataBlock.java | 134 +
.../apache/hudi/io/hfile/HFileFileInfoBlock.java | 64 +
.../java/org/apache/hudi/io/hfile/HFileInfo.java | 90 +
.../org/apache/hudi/io/hfile/HFileMetaBlock.java | 22 +-
.../java/org/apache/hudi/io/hfile/HFileReader.java | 127 +
.../org/apache/hudi/io/hfile/HFileReaderImpl.java | 302 ++
.../apache/hudi/io/hfile/HFileRootIndexBlock.java | 77 +
.../org/apache/hudi/io/hfile/HFileTrailer.java | 191 ++
.../java/org/apache/hudi/io/hfile/HFileUtils.java | 128 +
.../main/java/org/apache/hudi/io/hfile/Key.java | 98 +
.../java/org/apache/hudi/io/hfile/KeyValue.java | 100 +
.../org/apache/hudi/io/hfile/UTF8StringKey.java | 36 +-
.../main/java/org/apache/hudi/io/util/IOUtils.java | 280 ++
.../org/apache/hudi/storage/HoodieFileStatus.java | 120 +
.../org/apache/hudi/storage/HoodieLocation.java | 262 ++
.../apache/hudi/storage/HoodieLocationFilter.java | 27 +-
.../org/apache/hudi/storage/HoodieStorage.java | 355 ++
.../org/apache/hudi/storage}/StorageSchemes.java | 19 +-
hudi-io/src/main/protobuf/HFile.proto | 53 +
.../hudi/io/compress/TestHoodieDecompressor.java | 106 +
.../org/apache/hudi/io/hfile/TestHFileReader.java | 639 ++++
.../hudi/io/storage/TestHoodieFileStatus.java | 102 +
.../apache/hudi/io/storage/TestHoodieLocation.java | 192 ++
.../hudi/io/storage/TestHoodieLocationFilter.java | 73 +
.../hudi/io/storage/TestHoodieStorageBase.java | 353 ++
.../org/apache/hudi/io/util/TestHFileUtils.java | 44 +
.../java/org/apache/hudi/io/util/TestIOUtils.java | 110 +
...10_hbase_1_2_3_bootstrap_index_partitions.hfile | Bin
.../hfile}/hudi_0_10_hbase_1_2_3_complex.hfile | Bin
.../hfile}/hudi_0_10_hbase_1_2_3_simple.hfile | Bin
...11_hbase_2_4_9_bootstrap_index_partitions.hfile | Bin
.../hfile}/hudi_0_11_hbase_2_4_9_complex.hfile | Bin
.../hfile}/hudi_0_11_hbase_2_4_9_simple.hfile | Bin
..._9_hbase_1_2_3_bootstrap_index_partitions.hfile | Bin
.../hfile}/hudi_0_9_hbase_1_2_3_complex.hfile | Bin
.../hfile}/hudi_0_9_hbase_1_2_3_simple.hfile | Bin
.../hfile/hudi_1_0_hbase_2_4_9_16KB_GZ_20000.hfile | Bin 0 -> 105235 bytes
...1_0_hbase_2_4_9_16KB_GZ_200_20_non_unique.hfile | Bin 0 -> 19476 bytes
.../hudi_1_0_hbase_2_4_9_16KB_NONE_5000.hfile | Bin 0 -> 301098 bytes
.../hudi_1_0_hbase_2_4_9_512KB_GZ_20000.hfile | Bin 0 -> 101870 bytes
.../hudi_1_0_hbase_2_4_9_64KB_NONE_5000.hfile | Bin 0 -> 300065 bytes
.../hfile/hudi_1_0_hbase_2_4_9_no_entry.hfile | Bin 6115 -> 5087 bytes
hudi-kafka-connect/pom.xml | 4 +-
.../hudi/connect/utils/KafkaConnectUtils.java | 7 +-
.../writers/KafkaConnectTransactionServices.java | 4 +-
.../org/apache/hudi/helper/MockKafkaConnect.java | 6 +-
.../hudi/writers/TestAbstractConnectWriter.java | 3 +-
.../hudi-metaserver/hudi-metaserver-client/pom.xml | 2 +-
.../hudi-metaserver/hudi-metaserver-server/pom.xml | 2 +-
hudi-platform-service/hudi-metaserver/pom.xml | 4 +-
hudi-platform-service/pom.xml | 2 +-
hudi-spark-datasource/hudi-spark-common/pom.xml | 4 +-
.../BaseDatasetBulkInsertCommitActionExecutor.java | 2 +-
.../org/apache/hudi/ColumnStatsIndexSupport.scala | 16 +-
.../main/scala/org/apache/hudi/DefaultSource.scala | 25 +-
.../scala/org/apache/hudi/HoodieBaseRelation.scala | 40 +-
.../scala/org/apache/hudi/HoodieFileIndex.scala | 26 +-
.../org/apache/hudi/HoodieSparkSqlWriter.scala | 10 +-
.../org/apache/hudi/IncrementalRelation.scala | 11 +-
.../src/main/scala/org/apache/hudi/Iterators.scala | 7 +-
.../datasources/HoodieInMemoryFileIndex.scala | 5 +-
.../parquet/LegacyHoodieParquetFileFormat.scala | 8 +-
.../apache/spark/sql/hudi/DataSkippingUtils.scala | 12 +-
.../apache/spark/sql/hudi/HoodieOptionConfig.scala | 7 +
.../spark/sql/hudi/HoodieSqlCommonUtils.scala | 2 +-
.../AlterHoodieTableAddColumnsCommand.scala | 4 +-
.../sql/hudi/command/DropHoodieTableCommand.scala | 4 +-
.../hudi/command/TruncateHoodieTableCommand.scala | 4 +-
.../hudi/testutils/SparkDatasetTestUtils.java | 19 +-
hudi-spark-datasource/hudi-spark/pom.xml | 34 +-
.../apache/hudi/cli/HDFSParquetImporterUtils.java | 6 +-
.../org/apache/spark/sql/hudi/DedupeSparkJob.scala | 2 +-
.../spark/sql/hudi/analysis/HoodieAnalysis.scala | 19 +-
.../hudi/command/CallProcedureHoodieCommand.scala | 6 +-
.../hudi/command/CompactionHoodiePathCommand.scala | 5 +-
.../command/CompactionHoodieTableCommand.scala | 5 +-
.../command/CompactionShowHoodiePathCommand.scala | 5 +-
.../command/CompactionShowHoodieTableCommand.scala | 5 +-
.../command/InsertIntoHoodieTableCommand.scala | 10 +-
.../procedures/ExportInstantsProcedure.scala | 9 +-
.../RepairAddpartitionmetaProcedure.scala | 2 +-
.../procedures/RepairDeduplicateProcedure.scala | 8 +-
.../RepairOverwriteHoodiePropsProcedure.scala | 16 +-
.../command/procedures/RunBootstrapProcedure.scala | 7 +-
.../procedures/ShowFileSystemViewProcedure.scala | 9 +-
.../procedures/ShowFsPathDetailProcedure.scala | 6 +-
.../ShowHoodieLogFileMetadataProcedure.scala | 2 +-
.../procedures/ShowInvalidParquetProcedure.scala | 4 +-
.../procedures/StatsFileSizeProcedure.scala | 2 +-
.../TestBulkInsertInternalPartitionerForRows.java | 0
.../org/apache/hudi/functional/TestBootstrap.java | 3 +-
.../apache/hudi/functional/TestBootstrapRead.java | 30 +-
.../TestHoodieDatasetBulkInsertHelper.java | 72 +-
.../row/TestHoodieInternalRowParquetWriter.java | 0
.../io/storage/row/TestHoodieRowCreateHandle.java | 14 +-
.../hudi/testutils/KeyGeneratorTestUtilities.java | 20 +-
.../org/apache/hudi/TestAvroConversionUtils.scala | 2 +-
.../hudi/TestAvroSchemaResolutionSupport.scala | 120 +-
.../org/apache/hudi/TestDataSkippingUtils.scala | 41 +-
.../org/apache/hudi/TestHoodieSparkSqlWriter.scala | 56 +-
.../org/apache/hudi/TestHoodieSparkUtils.scala | 4 +
.../apache/hudi/functional/TestCOWDataSource.scala | 25 +-
.../hudi/functional/TestCOWDataSourceStorage.scala | 8 +-
.../functional/TestColumnStatsIndexWithSQL.scala | 2 +-
.../functional/TestDataSourceForBootstrap.scala | 41 +-
.../hudi/functional/TestMORDataSourceStorage.scala | 8 +-
.../hudi/functional/TestSparkDataSource.scala | 89 +-
.../hudi/functional/TestSparkSqlCoreFlow.scala | 9 +-
.../spark/hudi/benchmark/HoodieBenchmarkBase.scala | 7 +-
.../spark/sql/hudi/HoodieSparkSqlTestBase.scala | 6 +-
.../spark/sql/hudi/TestDataSkippingQuery.scala | 114 +
.../org/apache/spark/sql/hudi/TestDropTable.scala | 10 +-
.../apache/spark/sql/hudi/TestInsertTable.scala | 128 +-
.../apache/spark/sql/hudi/TestMergeIntoTable.scala | 6 +-
.../org/apache/spark/sql/hudi/TestSpark3DDL.scala | 9 +-
.../procedure/TestHdfsParquetImportProcedure.scala | 14 +-
.../sql/hudi/procedure/TestRepairsProcedure.scala | 32 +-
.../TestShowInvalidParquetProcedure.scala | 6 +-
hudi-spark-datasource/hudi-spark2-common/pom.xml | 2 +-
hudi-spark-datasource/hudi-spark2/pom.xml | 12 +-
.../sql/HoodieSpark2CatalystExpressionUtils.scala | 7 +-
.../apache/spark/sql/HoodieSpark2SchemaUtils.scala | 6 +
.../apache/spark/sql/adapter/Spark2Adapter.scala | 7 +-
.../HoodieSpark2PartitionedFileUtils.scala | 12 +-
.../HoodieBulkInsertInternalWriterTestBase.java | 0
hudi-spark-datasource/hudi-spark3-common/pom.xml | 2 +-
.../apache/hudi/spark3/internal/ReflectUtil.java | 8 +-
.../spark/sql/adapter/BaseSpark3Adapter.scala | 6 +-
.../datasources/Spark3ParsePartitionUtil.scala | 10 +-
hudi-spark-datasource/hudi-spark3.0.x/pom.xml | 19 +-
.../sql/HoodieSpark30CatalystExpressionUtils.scala | 7 +-
.../spark/sql/HoodieSpark30SchemaUtils.scala | 6 +
.../HoodieSpark30PartitionedFileUtils.scala | 12 +-
.../HoodieBulkInsertInternalWriterTestBase.java | 0
.../TestHoodieBulkInsertDataInternalWriter.java | 0
.../TestHoodieDataSourceInternalBatchWrite.java | 0
hudi-spark-datasource/hudi-spark3.1.x/pom.xml | 19 +-
.../sql/HoodieSpark31CatalystExpressionUtils.scala | 8 +-
.../spark/sql/HoodieSpark31SchemaUtils.scala | 6 +
.../HoodieSpark31PartitionedFileUtils.scala | 12 +-
.../HoodieBulkInsertInternalWriterTestBase.java | 0
.../TestHoodieBulkInsertDataInternalWriter.java | 0
.../TestHoodieDataSourceInternalBatchWrite.java | 0
hudi-spark-datasource/hudi-spark3.2.x/pom.xml | 12 +-
.../sql/HoodieSpark32CatalystExpressionUtils.scala | 7 +-
.../spark/sql/HoodieSpark32SchemaUtils.scala | 6 +
.../HoodieSpark32PartitionedFileUtils.scala | 12 +-
.../parquet/Spark32DataSourceUtils.scala} | 2 +-
.../Spark32LegacyHoodieParquetFileFormat.scala | 10 +-
.../sql/hudi/analysis/HoodieSpark32Analysis.scala | 66 +
.../HoodieBulkInsertInternalWriterTestBase.java | 0
.../TestHoodieBulkInsertDataInternalWriter.java | 0
.../TestHoodieDataSourceInternalBatchWrite.java | 0
.../hudi-spark3.2plus-common/pom.xml | 2 +-
.../hudi/analysis/HoodieSpark32PlusAnalysis.scala | 28 -
.../spark/sql/hudi/command/AlterTableCommand.scala | 4 +-
hudi-spark-datasource/hudi-spark3.3.x/pom.xml | 4 +-
.../sql/HoodieSpark33CatalystExpressionUtils.scala | 9 +-
.../spark/sql/HoodieSpark33SchemaUtils.scala | 6 +
.../HoodieSpark33PartitionedFileUtils.scala | 12 +-
.../parquet/Spark33DataSourceUtils.scala} | 2 +-
.../Spark33LegacyHoodieParquetFileFormat.scala | 22 +-
.../sql/hudi/analysis/HoodieSpark33Analysis.scala | 66 +
.../HoodieBulkInsertInternalWriterTestBase.java | 0
.../hudi/spark3/internal/TestReflectUtil.java | 3 +-
hudi-spark-datasource/hudi-spark3.4.x/pom.xml | 4 +-
.../sql/HoodieSpark34CatalystExpressionUtils.scala | 7 +-
.../spark/sql/HoodieSpark34SchemaUtils.scala | 6 +
.../HoodieSpark34PartitionedFileUtils.scala | 12 +-
.../parquet/Spark34DataSourceUtils.scala} | 2 +-
.../Spark34LegacyHoodieParquetFileFormat.scala | 29 +-
.../sql/hudi/analysis/HoodieSpark34Analysis.scala | 66 +
.../HoodieBulkInsertInternalWriterTestBase.java | 0
.../hudi/spark3/internal/TestReflectUtil.java | 3 +-
.../{hudi-spark3.4.x => hudi-spark3.5.x}/pom.xml | 18 +-
.../src/main/antlr4/imports/SqlBase.g4 | 1940 +++++++++++
.../apache/hudi/spark/sql/parser/HoodieSqlBase.g4 | 40 +
...org.apache.spark.sql.sources.DataSourceRegister | 0
.../org/apache/hudi/Spark35HoodieFileScanRDD.scala | 27 +-
.../spark/sql/HoodieSpark35CatalogUtils.scala | 19 +-
.../HoodieSpark35CatalystExpressionUtils.scala} | 11 +-
.../spark/sql/HoodieSpark35CatalystPlanUtils.scala | 83 +
.../spark/sql/HoodieSpark35SchemaUtils.scala} | 11 +-
.../apache/spark/sql/adapter/Spark3_5Adapter.scala | 130 +
.../apache/spark/sql/avro/AvroDeserializer.scala | 495 +++
.../org/apache/spark/sql/avro/AvroSerializer.scala | 450 +++
.../org/apache/spark/sql/avro/AvroUtils.scala | 0
.../sql/avro/HoodieSpark3_5AvroDeserializer.scala | 21 +-
.../sql/avro/HoodieSpark3_5AvroSerializer.scala | 18 +-
.../HoodieSpark35PartitionedFileUtils.scala} | 14 +-
.../datasources/Spark35NestedSchemaPruning.scala | 198 ++
.../parquet/Spark35DataSourceUtils.scala} | 9 +-
.../Spark35LegacyHoodieParquetFileFormat.scala} | 43 +-
.../hudi/Spark35ResolveHudiAlterTableCommand.scala | 71 +
.../sql/hudi/analysis/HoodieSpark35Analysis.scala | 66 +
.../HoodieSpark3_5ExtendedSqlAstBuilder.scala | 3426 ++++++++++++++++++++
.../parser/HoodieSpark3_5ExtendedSqlParser.scala | 201 ++
.../HoodieBulkInsertInternalWriterTestBase.java | 0
.../TestHoodieBulkInsertDataInternalWriter.java | 0
.../TestHoodieDataSourceInternalBatchWrite.java | 0
.../hudi/spark3/internal/TestReflectUtil.java | 11 +-
hudi-spark-datasource/pom.xml | 2 +-
hudi-sync/hudi-adb-sync/pom.xml | 2 +-
hudi-sync/hudi-datahub-sync/pom.xml | 2 +-
hudi-sync/hudi-hive-sync/pom.xml | 2 +-
.../org/apache/hudi/hive/ddl/HMSDDLExecutor.java | 2 +-
.../hudi/hive/ddl/QueryBasedDDLExecutor.java | 2 +-
.../hudi/hive/testutils/HiveTestCluster.java | 4 +-
.../apache/hudi/hive/testutils/HiveTestUtil.java | 14 +-
hudi-sync/hudi-sync-common/pom.xml | 2 +-
.../apache/hudi/sync/common/HoodieSyncClient.java | 2 +-
.../apache/hudi/sync/common/HoodieSyncConfig.java | 4 +-
.../hudi/sync/common/util/TestSyncUtilHelpers.java | 4 +-
hudi-sync/pom.xml | 2 +-
hudi-tests-common/pom.xml | 2 +-
hudi-timeline-service/pom.xml | 2 +-
.../hudi/timeline/service/TimelineService.java | 8 +-
.../service/handlers/FileSliceHandler.java | 4 +-
.../timeline/service/handlers/TimelineHandler.java | 4 +-
hudi-utilities/pom.xml | 2 +-
.../apache/hudi/utilities/HDFSParquetImporter.java | 4 +-
.../hudi/utilities/HoodieCompactionAdminTool.java | 4 +-
.../org/apache/hudi/utilities/HoodieCompactor.java | 4 +-
.../hudi/utilities/HoodieDropPartitionsTool.java | 4 +-
.../utilities/HoodieMetadataTableValidator.java | 29 +-
.../apache/hudi/utilities/HoodieRepairTool.java | 9 +-
.../hudi/utilities/HoodieSnapshotCopier.java | 7 +-
.../hudi/utilities/HoodieSnapshotExporter.java | 15 +-
.../org/apache/hudi/utilities/TableSizeStats.java | 4 +-
.../org/apache/hudi/utilities/UtilHelpers.java | 7 +-
.../utilities/config/ParquetDFSSourceConfig.java | 49 +
.../hudi/utilities/perf/TimelineServerPerf.java | 11 +-
.../utilities/schema/FilebasedSchemaProvider.java | 33 +-
.../hudi/utilities/schema/SchemaProvider.java | 5 +
.../schema/SchemaProviderWithPostProcessor.java | 13 +-
.../utilities/schema/SchemaRegistryProvider.java | 40 +-
.../converter/JsonToAvroSchemaConverter.java | 8 +-
.../hudi/utilities/sources/HiveIncrPullSource.java | 4 +-
.../apache/hudi/utilities/sources/InputBatch.java | 8 +-
.../hudi/utilities/sources/JsonDFSSource.java | 2 +-
.../hudi/utilities/sources/ParquetDFSSource.java | 6 +-
.../hudi/utilities/sources/SqlFileBasedSource.java | 4 +-
.../helpers/CloudObjectsSelectorCommon.java | 4 +-
.../utilities/sources/helpers/DFSPathSelector.java | 4 +-
.../sources/helpers/ProtoConversionUtil.java | 4 +-
.../apache/hudi/utilities/streamer/ErrorEvent.java | 6 +-
.../streamer/HoodieMultiTableStreamer.java | 4 +-
.../hudi/utilities/streamer/HoodieStreamer.java | 6 +-
.../utilities/streamer/HoodieStreamerUtils.java | 68 +-
.../utilities/streamer/SparkSampleWritesUtils.java | 8 +-
.../apache/hudi/utilities/streamer/StreamSync.java | 142 +-
.../utilities/transform/ChainedTransformer.java | 20 +-
.../ErrorTableAwareChainedTransformer.java | 5 +-
.../transform/SqlFileBasedTransformer.java | 4 +-
.../deltastreamer/HoodieDeltaStreamerTestBase.java | 11 +-
.../deltastreamer/TestHoodieDeltaStreamer.java | 115 +-
...TestHoodieDeltaStreamerSchemaEvolutionBase.java | 65 +
...oodieDeltaStreamerSchemaEvolutionExtensive.java | 100 +-
...estHoodieDeltaStreamerSchemaEvolutionQuick.java | 68 +-
.../deltastreamer/TestSourceFormatAdapter.java | 2 +-
.../functional/TestChainedTransformer.java | 29 +-
.../TestErrorTableAwareChainedTransformer.java | 4 +-
.../functional/TestHDFSParquetImporter.java | 5 +-
.../functional/TestHoodieSnapshotCopier.java | 3 +-
.../functional/TestHoodieSnapshotExporter.java | 4 +-
.../schema/TestSchemaRegistryProvider.java | 24 +-
.../utilities/sources/TestGcsEventsSource.java | 34 +-
.../utilities/sources/TestGenericRddTransform.java | 29 +
.../utilities/sources/TestProtoKafkaSource.java | 5 +-
.../sources/helpers/TestProtoConversionUtil.java | 9 +-
.../sources/helpers/TestSanitizationUtils.java | 4 +-
.../schema-evolution/testMissingRecordKey.json | 2 +
packaging/bundle-validation/README.md | 8 +-
...ark313.sh => build_flink1146hive313spark313.sh} | 6 +-
...ark332.sh => build_flink1180hive313spark332.sh} | 6 +-
...ark340.sh => build_flink1180hive313spark340.sh} | 6 +-
...ark332.sh => build_flink1180hive313spark350.sh} | 8 +-
packaging/bundle-validation/ci_run.sh | 24 +-
packaging/bundle-validation/run_docker_java17.sh | 18 +-
packaging/hudi-aws-bundle/pom.xml | 3 +-
packaging/hudi-cli-bundle/pom.xml | 2 +-
packaging/hudi-datahub-sync-bundle/pom.xml | 3 +-
packaging/hudi-flink-bundle/pom.xml | 3 +-
packaging/hudi-gcp-bundle/pom.xml | 3 +-
packaging/hudi-hadoop-mr-bundle/pom.xml | 3 +-
packaging/hudi-hive-sync-bundle/pom.xml | 3 +-
packaging/hudi-integ-test-bundle/pom.xml | 3 +-
packaging/hudi-kafka-connect-bundle/pom.xml | 3 +-
packaging/hudi-metaserver-server-bundle/pom.xml | 4 +-
packaging/hudi-presto-bundle/pom.xml | 3 +-
packaging/hudi-spark-bundle/pom.xml | 8 +-
packaging/hudi-timeline-server-bundle/pom.xml | 3 +-
packaging/hudi-trino-bundle/pom.xml | 3 +-
packaging/hudi-utilities-bundle/pom.xml | 14 +-
packaging/hudi-utilities-slim-bundle/pom.xml | 9 +-
pom.xml | 179 +-
rfc/README.md | 10 +-
rfc/rfc-76/rfc-76.md | 156 +
rfc/rfc-8/metadata_record_index.jpg | Bin 0 -> 42413 bytes
rfc/rfc-8/rfc-8.md | 219 ++
scripts/release/deploy_staging_jars.sh | 10 +-
scripts/release/validate_staged_bundles.sh | 8 +-
852 files changed, 22709 insertions(+), 9078 deletions(-)
copy
hudi-common/src/main/java/org/apache/hudi/common/fs/NoOpConsistencyGuard.java
=>
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/InProcessLockProviderWithRuntimeError.java
(57%)
delete mode 100644
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/io/storage/TestHoodieHFileReaderWriter.java
create mode 100644
hudi-common/src/main/java/org/apache/hudi/common/bloom/HashFunction.java
create mode 100644
hudi-common/src/main/java/org/apache/hudi/common/bloom/InternalBloomFilter.java
create mode 100644
hudi-common/src/main/java/org/apache/hudi/common/bloom/Key.java
copy
hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/HoodieSpark30SchemaUtils.scala
=>
hudi-common/src/main/java/org/apache/hudi/common/config/HoodieReaderConfig.java
(55%)
create mode 100644
hudi-common/src/main/java/org/apache/hudi/common/util/hash/Hash.java
create mode 100644
hudi-common/src/main/java/org/apache/hudi/common/util/hash/JenkinsHash.java
create mode 100644
hudi-common/src/main/java/org/apache/hudi/common/util/hash/MurmurHash.java
create mode 100644
hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieAvroHFileReaderImplBase.java
rename
hudi-common/src/main/java/org/apache/hudi/io/storage/{HoodieAvroHFileReader.java
=> HoodieHBaseAvroHFileReader.java} (78%)
create mode 100644
hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieNativeAvroHFileReader.java
rename
hudi-common/src/test/java/org/apache/hudi/common/fs/inline/{TestInLineFileSystemHFileInLining.java
=> TestInLineFileSystemHFileInLiningBase.java} (60%)
create mode 100644
hudi-common/src/test/java/org/apache/hudi/common/fs/inline/TestInLineFileSystemWithHBaseHFileReader.java
create mode 100644
hudi-common/src/test/java/org/apache/hudi/common/fs/inline/TestInLineFileSystemWithHFileReader.java
create mode 100644
hudi-common/src/test/java/org/apache/hudi/io/storage/TestHoodieHBaseHFileReaderWriter.java
create mode 100644
hudi-common/src/test/java/org/apache/hudi/io/storage/TestHoodieHFileReaderWriter.java
create mode 100644
hudi-common/src/test/java/org/apache/hudi/io/storage/TestHoodieHFileReaderWriterBase.java
rename {hudi-client/hudi-client-common =>
hudi-common}/src/test/java/org/apache/hudi/io/storage/TestHoodieOrcReaderWriter.java
(96%)
rename {hudi-client/hudi-client-common =>
hudi-common}/src/test/java/org/apache/hudi/io/storage/TestHoodieReaderWriterBase.java
(100%)
create mode 100644
hudi-common/src/test/java/org/apache/hudi/io/storage/TestHoodieReaderWriterUtils.java
rename {hudi-client/hudi-client-common =>
hudi-common}/src/test/resources/exampleEvolvedSchema.avsc (100%)
rename {hudi-client/hudi-client-common =>
hudi-common}/src/test/resources/exampleEvolvedSchemaChangeOrder.avsc (100%)
rename {hudi-client/hudi-client-common =>
hudi-common}/src/test/resources/exampleEvolvedSchemaColumnRequire.avsc (100%)
rename {hudi-client/hudi-client-common =>
hudi-common}/src/test/resources/exampleEvolvedSchemaColumnType.avsc (100%)
rename {hudi-client/hudi-client-common =>
hudi-common}/src/test/resources/exampleEvolvedSchemaDeleteColumn.avsc (100%)
rename {hudi-client/hudi-client-common =>
hudi-common}/src/test/resources/exampleSchema.avsc (100%)
rename {hudi-client/hudi-client-common =>
hudi-common}/src/test/resources/exampleSchemaWithMetaFields.avsc (100%)
rename {hudi-client/hudi-client-common =>
hudi-common}/src/test/resources/exampleSchemaWithUDT.avsc (100%)
create mode 100644
hudi-common/src/test/resources/format/bloom-filter/hadoop/all_10000.keys.data
create mode 100644
hudi-common/src/test/resources/format/bloom-filter/hadoop/dynamic_1000_000001_jenkins_5000.bf.data
create mode 100644
hudi-common/src/test/resources/format/bloom-filter/hadoop/dynamic_1000_000001_murmur_5000.bf.data
create mode 100644
hudi-common/src/test/resources/format/bloom-filter/hadoop/dynamic_200_000001_murmur_1000.bf.data
create mode 100644
hudi-common/src/test/resources/format/bloom-filter/hadoop/simple_10000_000001_murmur.bf.data
create mode 100644
hudi-common/src/test/resources/format/bloom-filter/hadoop/simple_1000_000001_murmur.bf.data
create mode 100644
hudi-common/src/test/resources/format/bloom-filter/hadoop/simple_200_000001_murmur.bf.data
create mode 100644
hudi-common/src/test/resources/format/bloom-filter/hadoop/simple_5000_000001_jenkins.bf.data
create mode 100644
hudi-common/src/test/resources/format/bloom-filter/hadoop/simple_5000_000001_murmur.bf.data
create mode 100644
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/metrics/FlinkClusteringMetrics.java
create mode 100644
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/ImplicitTypeConverter.java
create mode 100644
hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/configuration/TestOptionsResolver.java
rename
hudi-flink-datasource/{hudi-flink1.17.x/src/test/java/org/apache/hudi/adapter
=> hudi-flink/src/test/java/org/apache/hudi/utils}/TestTableEnvs.java (98%)
delete mode 100644 hudi-flink-datasource/hudi-flink1.13.x/pom.xml
delete mode 100644
hudi-flink-datasource/hudi-flink1.13.x/src/main/java/org/apache/hudi/adapter/AbstractStreamOperatorAdapter.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.13.x/src/main/java/org/apache/hudi/adapter/AbstractStreamOperatorFactoryAdapter.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.13.x/src/main/java/org/apache/hudi/adapter/DataStreamScanProviderAdapter.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.13.x/src/main/java/org/apache/hudi/adapter/DataStreamSinkProviderAdapter.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.13.x/src/main/java/org/apache/hudi/adapter/MailboxExecutorAdapter.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.13.x/src/main/java/org/apache/hudi/adapter/MaskingOutputAdapter.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.13.x/src/main/java/org/apache/hudi/adapter/OperatorCoordinatorAdapter.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.13.x/src/main/java/org/apache/hudi/adapter/SortCodeGeneratorAdapter.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.13.x/src/main/java/org/apache/hudi/adapter/SupportsRowLevelDeleteAdapter.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.13.x/src/main/java/org/apache/hudi/adapter/SupportsRowLevelUpdateAdapter.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.13.x/src/main/java/org/apache/hudi/adapter/Utils.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.13.x/src/main/java/org/apache/hudi/table/data/ColumnarArrayData.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.13.x/src/main/java/org/apache/hudi/table/data/ColumnarMapData.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.13.x/src/main/java/org/apache/hudi/table/data/ColumnarRowData.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.13.x/src/main/java/org/apache/hudi/table/data/vector/MapColumnVector.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.13.x/src/main/java/org/apache/hudi/table/data/vector/RowColumnVector.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.13.x/src/main/java/org/apache/hudi/table/data/vector/VectorizedColumnBatch.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.13.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.13.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayVector.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.13.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapMapColumnVector.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.13.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapRowColumnVector.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.13.x/src/main/java/org/apache/hudi/table/format/cow/vector/ParquetDecimalVector.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.13.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/AbstractColumnReader.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.13.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayColumnReader.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.13.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/BaseVectorizedColumnReader.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.13.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/EmptyColumnReader.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.13.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/FixedLenBytesColumnReader.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.13.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/Int64TimestampColumnReader.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.13.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/MapColumnReader.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.13.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.13.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/RowColumnReader.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.13.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/RunLengthDecoder.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.13.x/src/test/java/org/apache/hudi/adapter/OutputAdapter.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.13.x/src/test/java/org/apache/hudi/adapter/StateInitializationContextAdapter.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.13.x/src/test/java/org/apache/hudi/adapter/StreamingRuntimeContextAdapter.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.13.x/src/test/java/org/apache/hudi/adapter/TestTableEnvs.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.14.x/src/main/java/org/apache/hudi/adapter/AbstractStreamOperatorAdapter.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.14.x/src/main/java/org/apache/hudi/adapter/AbstractStreamOperatorFactoryAdapter.java
create mode 100644
hudi-flink-datasource/hudi-flink1.14.x/src/main/java/org/apache/hudi/adapter/HiveCatalogConstants.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.14.x/src/main/java/org/apache/hudi/adapter/MailboxExecutorAdapter.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.14.x/src/main/java/org/apache/hudi/adapter/RateLimiterAdapter.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.14.x/src/test/java/org/apache/hudi/adapter/OutputAdapter.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.14.x/src/test/java/org/apache/hudi/adapter/StreamingRuntimeContextAdapter.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.14.x/src/test/java/org/apache/hudi/adapter/TestTableEnvs.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.15.x/src/main/java/org/apache/hudi/adapter/AbstractStreamOperatorAdapter.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.15.x/src/main/java/org/apache/hudi/adapter/AbstractStreamOperatorFactoryAdapter.java
create mode 100644
hudi-flink-datasource/hudi-flink1.15.x/src/main/java/org/apache/hudi/adapter/HiveCatalogConstants.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.15.x/src/main/java/org/apache/hudi/adapter/MailboxExecutorAdapter.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.15.x/src/main/java/org/apache/hudi/adapter/RateLimiterAdapter.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.15.x/src/test/java/org/apache/hudi/adapter/OutputAdapter.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.15.x/src/test/java/org/apache/hudi/adapter/StreamingRuntimeContextAdapter.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.15.x/src/test/java/org/apache/hudi/adapter/TestTableEnvs.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.16.x/src/main/java/org/apache/hudi/adapter/AbstractStreamOperatorAdapter.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.16.x/src/main/java/org/apache/hudi/adapter/AbstractStreamOperatorFactoryAdapter.java
create mode 100644
hudi-flink-datasource/hudi-flink1.16.x/src/main/java/org/apache/hudi/adapter/HiveCatalogConstants.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.16.x/src/main/java/org/apache/hudi/adapter/MailboxExecutorAdapter.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.16.x/src/main/java/org/apache/hudi/adapter/RateLimiterAdapter.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.16.x/src/test/java/org/apache/hudi/adapter/OutputAdapter.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.16.x/src/test/java/org/apache/hudi/adapter/StateInitializationContextAdapter.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.16.x/src/test/java/org/apache/hudi/adapter/StreamingRuntimeContextAdapter.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.16.x/src/test/java/org/apache/hudi/adapter/TestTableEnvs.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/adapter/AbstractStreamOperatorAdapter.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/adapter/AbstractStreamOperatorFactoryAdapter.java
create mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/adapter/HiveCatalogConstants.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/adapter/MailboxExecutorAdapter.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/adapter/RateLimiterAdapter.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/test/java/org/apache/hudi/adapter/OutputAdapter.java
delete mode 100644
hudi-flink-datasource/hudi-flink1.17.x/src/test/java/org/apache/hudi/adapter/StreamingRuntimeContextAdapter.java
copy hudi-flink-datasource/{hudi-flink1.17.x => hudi-flink1.18.x}/pom.xml (86%)
copy hudi-flink-datasource/{hudi-flink1.15.x =>
hudi-flink1.18.x}/src/main/java/org/apache/hudi/adapter/DataStreamScanProviderAdapter.java
(100%)
copy hudi-flink-datasource/{hudi-flink1.15.x =>
hudi-flink1.18.x}/src/main/java/org/apache/hudi/adapter/DataStreamSinkProviderAdapter.java
(100%)
create mode 100644
hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/adapter/HiveCatalogConstants.java
copy hudi-flink-datasource/{hudi-flink1.14.x =>
hudi-flink1.18.x}/src/main/java/org/apache/hudi/adapter/MaskingOutputAdapter.java
(100%)
copy hudi-flink-datasource/{hudi-flink1.16.x =>
hudi-flink1.18.x}/src/main/java/org/apache/hudi/adapter/OperatorCoordinatorAdapter.java
(100%)
copy hudi-flink-datasource/{hudi-flink1.16.x =>
hudi-flink1.18.x}/src/main/java/org/apache/hudi/adapter/SortCodeGeneratorAdapter.java
(100%)
copy hudi-flink-datasource/{hudi-flink1.17.x =>
hudi-flink1.18.x}/src/main/java/org/apache/hudi/adapter/SupportsRowLevelDeleteAdapter.java
(100%)
copy hudi-flink-datasource/{hudi-flink1.17.x =>
hudi-flink1.18.x}/src/main/java/org/apache/hudi/adapter/SupportsRowLevelUpdateAdapter.java
(100%)
copy hudi-flink-datasource/{hudi-flink1.17.x =>
hudi-flink1.18.x}/src/main/java/org/apache/hudi/adapter/Utils.java (74%)
copy hudi-flink-datasource/{hudi-flink1.15.x =>
hudi-flink1.18.x}/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java
(90%)
copy hudi-flink-datasource/{hudi-flink1.15.x =>
hudi-flink1.18.x}/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayVector.java
(100%)
copy
hudi-spark-datasource/hudi-spark2/src/main/scala/org/apache/spark/sql/HoodieSpark2SchemaUtils.scala
=>
hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapDecimalVector.java
(56%)
copy hudi-flink-datasource/{hudi-flink1.15.x =>
hudi-flink1.18.x}/src/main/java/org/apache/hudi/table/format/cow/vector/HeapMapColumnVector.java
(100%)
copy hudi-flink-datasource/{hudi-flink1.15.x =>
hudi-flink1.18.x}/src/main/java/org/apache/hudi/table/format/cow/vector/HeapRowColumnVector.java
(100%)
copy hudi-flink-datasource/{hudi-flink1.15.x =>
hudi-flink1.18.x}/src/main/java/org/apache/hudi/table/format/cow/vector/ParquetDecimalVector.java
(100%)
copy hudi-flink-datasource/{hudi-flink1.15.x =>
hudi-flink1.18.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/AbstractColumnReader.java
(100%)
copy hudi-flink-datasource/{hudi-flink1.15.x =>
hudi-flink1.18.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayColumnReader.java
(100%)
copy hudi-flink-datasource/{hudi-flink1.15.x =>
hudi-flink1.18.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/BaseVectorizedColumnReader.java
(100%)
copy hudi-flink-datasource/{hudi-flink1.15.x =>
hudi-flink1.18.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/EmptyColumnReader.java
(100%)
copy hudi-flink-datasource/{hudi-flink1.15.x =>
hudi-flink1.18.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/FixedLenBytesColumnReader.java
(100%)
copy hudi-flink-datasource/{hudi-flink1.15.x =>
hudi-flink1.18.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/Int64TimestampColumnReader.java
(100%)
copy hudi-flink-datasource/{hudi-flink1.15.x =>
hudi-flink1.18.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/MapColumnReader.java
(100%)
copy hudi-flink-datasource/{hudi-flink1.16.x =>
hudi-flink1.18.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java
(97%)
rename hudi-flink-datasource/{hudi-flink1.13.x =>
hudi-flink1.18.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetDataColumnReader.java
(100%)
rename hudi-flink-datasource/{hudi-flink1.13.x =>
hudi-flink1.18.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetDataColumnReaderFactory.java
(100%)
copy hudi-flink-datasource/{hudi-flink1.15.x =>
hudi-flink1.18.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/RowColumnReader.java
(100%)
copy hudi-flink-datasource/{hudi-flink1.15.x =>
hudi-flink1.18.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/RunLengthDecoder.java
(100%)
rename
hudi-flink-datasource/{hudi-flink1.13.x/src/main/java/org/apache/hudi/adapter/RateLimiterAdapter.java
=>
hudi-flink1.18.x/src/test/java/org/apache/hudi/adapter/TestStreamConfigs.java}
(60%)
copy {hudi-sync/hudi-datahub-sync => hudi-hadoop-common}/pom.xml (63%)
rename {hudi-common/src/main/java/org/apache/hudi/common =>
hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop}/fs/BoundedFsDataInputStream.java
(81%)
rename {hudi-common/src/main/java/org/apache/hudi/hadoop =>
hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop/fs}/CachingPath.java
(93%)
rename {hudi-common/src/main/java/org/apache/hudi/common =>
hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop}/fs/ConsistencyGuard.java
(85%)
create mode 100644
hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop/fs/HadoopFSUtils.java
rename {hudi-common/src/main/java/org/apache/hudi/common =>
hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop}/fs/HoodieRetryWrapperFileSystem.java
(97%)
rename {hudi-common/src/main/java/org/apache/hudi/common =>
hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop}/fs/HoodieSerializableFileStatus.java
(90%)
rename {hudi-common/src/main/java/org/apache/hudi/common =>
hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop}/fs/HoodieWrapperFileSystem.java
(97%)
rename {hudi-common/src/main/java/org/apache/hudi/common =>
hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop}/fs/NoOpConsistencyGuard.java
(71%)
rename {hudi-common/src/main/java/org/apache/hudi/common =>
hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop}/fs/SchemeAwareFSDataInputStream.java
(75%)
rename {hudi-common/src/main/java/org/apache/hudi/hadoop =>
hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop/fs}/SerializablePath.java
(78%)
rename {hudi-common/src/main/java/org/apache/hudi/common =>
hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop}/fs/SizeAwareFSDataOutputStream.java
(86%)
rename {hudi-common/src/main/java/org/apache/hudi/common =>
hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop}/fs/TimedFSDataInputStream.java
(86%)
create mode 100644
hudi-hadoop-common/src/main/java/org/apache/hudi/storage/hadoop/HoodieHadoopStorage.java
create mode 100644
hudi-hadoop-common/src/test/java/org/apache/hudi/hadoop/storage/TestHoodieHadoopStorage.java
copy
hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/HoodieSpark34SchemaUtils.scala
=> hudi-io/README.md (62%)
create mode 100644 hudi-io/hfile_format.md
create mode 100644 hudi-io/pom.xml
rename {hudi-common =>
hudi-io}/src/main/java/org/apache/hudi/ApiMaturityLevel.java (100%)
rename {hudi-common =>
hudi-io}/src/main/java/org/apache/hudi/PublicAPIClass.java (100%)
rename {hudi-common =>
hudi-io}/src/main/java/org/apache/hudi/PublicAPIMethod.java (100%)
rename {hudi-common =>
hudi-io}/src/main/java/org/apache/hudi/common/metrics/Counter.java (100%)
rename {hudi-common =>
hudi-io}/src/main/java/org/apache/hudi/common/metrics/LocalRegistry.java (100%)
rename {hudi-common =>
hudi-io}/src/main/java/org/apache/hudi/common/metrics/Metric.java (100%)
rename {hudi-common =>
hudi-io}/src/main/java/org/apache/hudi/common/metrics/Registry.java (100%)
rename {hudi-common =>
hudi-io}/src/main/java/org/apache/hudi/common/util/FileIOUtils.java (95%)
rename {hudi-common =>
hudi-io}/src/main/java/org/apache/hudi/common/util/HoodieTimer.java (100%)
rename {hudi-common =>
hudi-io}/src/main/java/org/apache/hudi/common/util/Option.java (100%)
rename {hudi-common =>
hudi-io}/src/main/java/org/apache/hudi/common/util/ReflectionUtils.java (100%)
rename {hudi-common =>
hudi-io}/src/main/java/org/apache/hudi/common/util/RetryHelper.java (92%)
rename {hudi-common =>
hudi-io}/src/main/java/org/apache/hudi/common/util/StringUtils.java (87%)
rename {hudi-common =>
hudi-io}/src/main/java/org/apache/hudi/common/util/ValidationUtils.java (100%)
rename {hudi-common =>
hudi-io}/src/main/java/org/apache/hudi/common/util/io/ByteBufferBackedInputStream.java
(100%)
rename {hudi-common =>
hudi-io}/src/main/java/org/apache/hudi/exception/HoodieException.java (100%)
rename {hudi-common =>
hudi-io}/src/main/java/org/apache/hudi/exception/HoodieIOException.java (100%)
copy
hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/HoodieSpark34SchemaUtils.scala
=> hudi-io/src/main/java/org/apache/hudi/io/compress/CompressionCodec.java
(64%)
copy
hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/HoodieSchemaUtils.scala
=> hudi-io/src/main/java/org/apache/hudi/io/compress/HoodieDecompressor.java
(50%)
copy
hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/HoodieSpark31SchemaUtils.scala
=>
hudi-io/src/main/java/org/apache/hudi/io/compress/HoodieDecompressorFactory.java
(55%)
create mode 100644
hudi-io/src/main/java/org/apache/hudi/io/compress/airlift/HoodieAirliftGzipDecompressor.java
copy
hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/HoodieSpark31SchemaUtils.scala
=>
hudi-io/src/main/java/org/apache/hudi/io/compress/builtin/HoodieNoneDecompressor.java
(51%)
create mode 100644
hudi-io/src/main/java/org/apache/hudi/io/hfile/BlockIndexEntry.java
copy
hudi-spark-datasource/hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/HoodieSpark32SchemaUtils.scala
=> hudi-io/src/main/java/org/apache/hudi/io/hfile/DataSize.java (59%)
create mode 100644
hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileBlock.java
create mode 100644
hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileBlockReader.java
create mode 100644
hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileBlockType.java
create mode 100644
hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileContext.java
create mode 100644
hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileCursor.java
create mode 100644
hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileDataBlock.java
create mode 100644
hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileFileInfoBlock.java
create mode 100644
hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileInfo.java
copy
hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/HoodieSpark30SchemaUtils.scala
=> hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileMetaBlock.java (59%)
create mode 100644
hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileReader.java
create mode 100644
hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileReaderImpl.java
create mode 100644
hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileRootIndexBlock.java
create mode 100644
hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileTrailer.java
create mode 100644
hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileUtils.java
create mode 100644 hudi-io/src/main/java/org/apache/hudi/io/hfile/Key.java
create mode 100644 hudi-io/src/main/java/org/apache/hudi/io/hfile/KeyValue.java
copy
hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/HoodieSpark31SchemaUtils.scala
=> hudi-io/src/main/java/org/apache/hudi/io/hfile/UTF8StringKey.java (57%)
create mode 100644 hudi-io/src/main/java/org/apache/hudi/io/util/IOUtils.java
create mode 100644
hudi-io/src/main/java/org/apache/hudi/storage/HoodieFileStatus.java
create mode 100644
hudi-io/src/main/java/org/apache/hudi/storage/HoodieLocation.java
copy
hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/HoodieSchemaUtils.scala
=> hudi-io/src/main/java/org/apache/hudi/storage/HoodieLocationFilter.java
(53%)
create mode 100644
hudi-io/src/main/java/org/apache/hudi/storage/HoodieStorage.java
rename {hudi-common/src/main/java/org/apache/hudi/common/fs =>
hudi-io/src/main/java/org/apache/hudi/storage}/StorageSchemes.java (89%)
create mode 100644 hudi-io/src/main/protobuf/HFile.proto
create mode 100644
hudi-io/src/test/java/org/apache/hudi/io/compress/TestHoodieDecompressor.java
create mode 100644
hudi-io/src/test/java/org/apache/hudi/io/hfile/TestHFileReader.java
create mode 100644
hudi-io/src/test/java/org/apache/hudi/io/storage/TestHoodieFileStatus.java
create mode 100644
hudi-io/src/test/java/org/apache/hudi/io/storage/TestHoodieLocation.java
create mode 100644
hudi-io/src/test/java/org/apache/hudi/io/storage/TestHoodieLocationFilter.java
create mode 100644
hudi-io/src/test/java/org/apache/hudi/io/storage/TestHoodieStorageBase.java
create mode 100644
hudi-io/src/test/java/org/apache/hudi/io/util/TestHFileUtils.java
create mode 100644
hudi-io/src/test/java/org/apache/hudi/io/util/TestIOUtils.java
rename {hudi-client/hudi-client-common/src/test/resources =>
hudi-io/src/test/resources/hfile}/hudi_0_10_hbase_1_2_3_bootstrap_index_partitions.hfile
(100%)
rename {hudi-client/hudi-client-common/src/test/resources =>
hudi-io/src/test/resources/hfile}/hudi_0_10_hbase_1_2_3_complex.hfile (100%)
rename {hudi-client/hudi-client-common/src/test/resources =>
hudi-io/src/test/resources/hfile}/hudi_0_10_hbase_1_2_3_simple.hfile (100%)
rename {hudi-client/hudi-client-common/src/test/resources =>
hudi-io/src/test/resources/hfile}/hudi_0_11_hbase_2_4_9_bootstrap_index_partitions.hfile
(100%)
rename {hudi-client/hudi-client-common/src/test/resources =>
hudi-io/src/test/resources/hfile}/hudi_0_11_hbase_2_4_9_complex.hfile (100%)
rename {hudi-client/hudi-client-common/src/test/resources =>
hudi-io/src/test/resources/hfile}/hudi_0_11_hbase_2_4_9_simple.hfile (100%)
rename {hudi-client/hudi-client-common/src/test/resources =>
hudi-io/src/test/resources/hfile}/hudi_0_9_hbase_1_2_3_bootstrap_index_partitions.hfile
(100%)
rename {hudi-client/hudi-client-common/src/test/resources =>
hudi-io/src/test/resources/hfile}/hudi_0_9_hbase_1_2_3_complex.hfile (100%)
copy {hudi-client/hudi-client-common/src/test/resources =>
hudi-io/src/test/resources/hfile}/hudi_0_9_hbase_1_2_3_simple.hfile (100%)
create mode 100644
hudi-io/src/test/resources/hfile/hudi_1_0_hbase_2_4_9_16KB_GZ_20000.hfile
create mode 100644
hudi-io/src/test/resources/hfile/hudi_1_0_hbase_2_4_9_16KB_GZ_200_20_non_unique.hfile
create mode 100644
hudi-io/src/test/resources/hfile/hudi_1_0_hbase_2_4_9_16KB_NONE_5000.hfile
create mode 100644
hudi-io/src/test/resources/hfile/hudi_1_0_hbase_2_4_9_512KB_GZ_20000.hfile
create mode 100644
hudi-io/src/test/resources/hfile/hudi_1_0_hbase_2_4_9_64KB_NONE_5000.hfile
rename
hudi-client/hudi-client-common/src/test/resources/hudi_0_9_hbase_1_2_3_simple.hfile
=> hudi-io/src/test/resources/hfile/hudi_1_0_hbase_2_4_9_no_entry.hfile (64%)
rename {hudi-client/hudi-spark-client =>
hudi-spark-datasource/hudi-spark-common}/src/test/java/org/apache/hudi/testutils/SparkDatasetTestUtils.java
(93%)
rename {hudi-client/hudi-spark-client =>
hudi-spark-datasource/hudi-spark}/src/test/java/org/apache/hudi/execution/bulkinsert/TestBulkInsertInternalPartitionerForRows.java
(100%)
rename {hudi-client/hudi-spark-client =>
hudi-spark-datasource/hudi-spark}/src/test/java/org/apache/hudi/io/storage/row/TestHoodieInternalRowParquetWriter.java
(100%)
rename {hudi-client/hudi-spark-client =>
hudi-spark-datasource/hudi-spark}/src/test/java/org/apache/hudi/io/storage/row/TestHoodieRowCreateHandle.java
(94%)
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestDataSkippingQuery.scala
copy hudi-spark-datasource/{hudi-spark-common =>
hudi-spark2}/src/test/java/org/apache/hudi/internal/HoodieBulkInsertInternalWriterTestBase.java
(100%)
copy hudi-spark-datasource/{hudi-spark-common =>
hudi-spark3.0.x}/src/test/java/org/apache/hudi/internal/HoodieBulkInsertInternalWriterTestBase.java
(100%)
copy hudi-spark-datasource/{hudi-spark3-common =>
hudi-spark3.0.x}/src/test/java/org/apache/hudi/spark3/internal/TestHoodieBulkInsertDataInternalWriter.java
(100%)
copy hudi-spark-datasource/{hudi-spark3-common =>
hudi-spark3.0.x}/src/test/java/org/apache/hudi/spark3/internal/TestHoodieDataSourceInternalBatchWrite.java
(100%)
copy hudi-spark-datasource/{hudi-spark-common/src/test/java =>
hudi-spark3.1.x/src/test}/org/apache/hudi/internal/HoodieBulkInsertInternalWriterTestBase.java
(100%)
copy hudi-spark-datasource/{hudi-spark3-common/src/test/java =>
hudi-spark3.1.x/src/test}/org/apache/hudi/spark3/internal/TestHoodieBulkInsertDataInternalWriter.java
(100%)
copy hudi-spark-datasource/{hudi-spark3-common/src/test/java =>
hudi-spark3.1.x/src/test}/org/apache/hudi/spark3/internal/TestHoodieDataSourceInternalBatchWrite.java
(100%)
copy
hudi-spark-datasource/{hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark32PlusDataSourceUtils.scala
=>
hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark32DataSourceUtils.scala}
(98%)
create mode 100644
hudi-spark-datasource/hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSpark32Analysis.scala
copy hudi-spark-datasource/{hudi-spark-common =>
hudi-spark3.2.x}/src/test/java/org/apache/hudi/internal/HoodieBulkInsertInternalWriterTestBase.java
(100%)
rename hudi-spark-datasource/{hudi-spark3-common =>
hudi-spark3.2.x}/src/test/java/org/apache/hudi/spark3/internal/TestHoodieBulkInsertDataInternalWriter.java
(100%)
rename hudi-spark-datasource/{hudi-spark3-common =>
hudi-spark3.2.x}/src/test/java/org/apache/hudi/spark3/internal/TestHoodieDataSourceInternalBatchWrite.java
(100%)
copy
hudi-spark-datasource/{hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark32PlusDataSourceUtils.scala
=>
hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark33DataSourceUtils.scala}
(98%)
create mode 100644
hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSpark33Analysis.scala
copy hudi-spark-datasource/{hudi-spark-common =>
hudi-spark3.3.x}/src/test/java/org/apache/hudi/internal/HoodieBulkInsertInternalWriterTestBase.java
(100%)
copy
hudi-spark-datasource/{hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark32PlusDataSourceUtils.scala
=>
hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark34DataSourceUtils.scala}
(98%)
create mode 100644
hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSpark34Analysis.scala
copy hudi-spark-datasource/{hudi-spark-common =>
hudi-spark3.4.x}/src/test/java/org/apache/hudi/internal/HoodieBulkInsertInternalWriterTestBase.java
(100%)
copy hudi-spark-datasource/{hudi-spark3.4.x => hudi-spark3.5.x}/pom.xml (96%)
create mode 100644
hudi-spark-datasource/hudi-spark3.5.x/src/main/antlr4/imports/SqlBase.g4
create mode 100644
hudi-spark-datasource/hudi-spark3.5.x/src/main/antlr4/org/apache/hudi/spark/sql/parser/HoodieSqlBase.g4
copy hudi-spark-datasource/{hudi-spark3.2.x =>
hudi-spark3.5.x}/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
(100%)
copy
hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/DataFrameUtil.scala
=>
hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/hudi/Spark35HoodieFileScanRDD.scala
(52%)
rename
hudi-flink-datasource/hudi-flink1.17.x/src/test/java/org/apache/hudi/adapter/StateInitializationContextAdapter.java
=>
hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/HoodieSpark35CatalogUtils.scala
(64%)
copy
hudi-spark-datasource/{hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/HoodieSpark34CatalystExpressionUtils.scala
=>
hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/HoodieSpark35CatalystExpressionUtils.scala}
(94%)
create mode 100644
hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/HoodieSpark35CatalystPlanUtils.scala
copy
hudi-spark-datasource/{hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/HoodieSpark34SchemaUtils.scala
=>
hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/HoodieSpark35SchemaUtils.scala}
(76%)
create mode 100644
hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/adapter/Spark3_5Adapter.scala
create mode 100644
hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
create mode 100644
hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala
copy hudi-spark-datasource/{hudi-spark3.3.x =>
hudi-spark3.5.x}/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala (100%)
rename
hudi-flink-datasource/hudi-flink1.14.x/src/test/java/org/apache/hudi/adapter/StateInitializationContextAdapter.java
=>
hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_5AvroDeserializer.scala
(58%)
rename
hudi-flink-datasource/hudi-flink1.15.x/src/test/java/org/apache/hudi/adapter/StateInitializationContextAdapter.java
=>
hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_5AvroSerializer.scala
(65%)
copy
hudi-spark-datasource/{hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/execution/datasources/HoodieSpark34PartitionedFileUtils.scala
=>
hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/execution/datasources/HoodieSpark35PartitionedFileUtils.scala}
(75%)
create mode 100644
hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/execution/datasources/Spark35NestedSchemaPruning.scala
rename
hudi-spark-datasource/{hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark32PlusDataSourceUtils.scala
=>
hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark35DataSourceUtils.scala}
(91%)
copy
hudi-spark-datasource/{hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark34LegacyHoodieParquetFileFormat.scala
=>
hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark35LegacyHoodieParquetFileFormat.scala}
(93%)
create mode 100644
hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/hudi/Spark35ResolveHudiAlterTableCommand.scala
create mode 100644
hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSpark35Analysis.scala
create mode 100644
hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_5ExtendedSqlAstBuilder.scala
create mode 100644
hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_5ExtendedSqlParser.scala
rename hudi-spark-datasource/{hudi-spark-common =>
hudi-spark3.5.x}/src/test/java/org/apache/hudi/internal/HoodieBulkInsertInternalWriterTestBase.java
(100%)
copy hudi-spark-datasource/{hudi-spark3.3.x =>
hudi-spark3.5.x}/src/test/java/org/apache/hudi/spark3/internal/TestHoodieBulkInsertDataInternalWriter.java
(100%)
copy hudi-spark-datasource/{hudi-spark3.3.x =>
hudi-spark3.5.x}/src/test/java/org/apache/hudi/spark3/internal/TestHoodieDataSourceInternalBatchWrite.java
(100%)
rename hudi-spark-datasource/{hudi-spark3-common =>
hudi-spark3.5.x}/src/test/java/org/apache/hudi/spark3/internal/TestReflectUtil.java
(90%)
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/config/ParquetDFSSourceConfig.java
create mode 100644
hudi-utilities/src/test/resources/data/schema-evolution/testMissingRecordKey.json
rename packaging/bundle-validation/base/{build_flink1136hive313spark313.sh =>
build_flink1146hive313spark313.sh} (80%)
copy packaging/bundle-validation/base/{build_flink1170hive313spark332.sh =>
build_flink1180hive313spark332.sh} (81%)
rename packaging/bundle-validation/base/{build_flink1170hive313spark340.sh =>
build_flink1180hive313spark340.sh} (81%)
rename packaging/bundle-validation/base/{build_flink1170hive313spark332.sh =>
build_flink1180hive313spark350.sh} (77%)
create mode 100644 rfc/rfc-76/rfc-76.md
create mode 100644 rfc/rfc-8/metadata_record_index.jpg
create mode 100644 rfc/rfc-8/rfc-8.md
