This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch
dependabot/maven/com.google.protobuf-protobuf-java-3.21.7
in repository https://gitbox.apache.org/repos/asf/hudi.git
discard f7b3ee64963 Bump protobuf-java from 3.21.5 to 3.21.7
add 0bc3eb8aaba [HUDI-4971] Remove direct use of kryo from `SerDeUtils`
(#7014)
add fa04e814cd5 [HUDI-5081] Tests clean up in hudi-utilities (#7033)
add 5cf8699297d [HUDI-5027] Replace hardcoded hbase config keys with
constant variables (#6946)
add 86218c32435 [MINOR] add commit_action output in show_commits (#7012)
add fdd8a4b330c [HUDI-5061] bulk insert operation don't throw other
exception except IOE Exception (#7001)
add ceb402564de [MINOR] Skip loading last completed txn for single writer
(#6660)
add 4965526289e [HUDI-4281] Using hudi to build a large number of tables
in spark on hive causes OOM (#5903)
add ce3bac45a28 [HUDI-5042] Fix clustering schedule problem in flink when
enable schedule clustering and disable async clustering (#6976)
add 927e5bf4ea8 [HUDI-4753] more accurate record size estimation for log
writing and spillable map (#6632)
add 49b9ba02c78 [HUDI-4201] Cli tool to get warned about empty
non-completed instants from timeline (#6867)
add 47499225e89 [HUDI-5038] Increase default num_instants to fetch for
incremental source (#6955)
add 4f6f15c3c76 [HUDI-5049] Supports dropPartition for Flink catalog
(#6991)
add 5ff21ef88ed [HUDI-4809] glue support drop partitions (#7007)
add ca4003b7844 [HUDI-5057] Fix msck repair hudi table (#6999)
add b6c35394c60 [HUDI-4959] Fixing Avro's `Utf8` serialization in Kryo
(#7024)
add e13b2129dc1 temp_view_support (#6990)
add aa1dff4d84a [HUDI-4982] Add Utilities and Utilities Slim + Spark
Bundle testing to GH Actions (#7005)
add 22ccdc15400 [HUDI-5085]When a flink job has multiple sink tables, the
index loading status is abnormal (#7051)
add fb5fab5ef39 [HUDI-5089] Refactor HoodieCommitMetadata deserialization
(#7055)
add 20e8ec79c82 [HUDI-5058] Fix flink catalog read spark table error :
primary key col can not be nullable (#7009)
add 7accc476d3e [HUDI-5087] Fix incorrect merging sequence for Column
Stats Record in `HoodieMetadataPayload` (#7053)
add 5b9fcc4540b [HUDI-4946] fix merge into with no preCombineField having
dup row by only insert (#6824)
add ae541ba0a57 [HUDI-5072] Extract `ExecutionStrategy#transform`
duplicate code (#7030)
add c6e5fd02719 [HUDI-3287] Remove hudi-spark dependencies from
hudi-kafka-connect-bundle (#6079)
add 48d23ff233d [HUDI-5000] Support schema evolution for Hive/presto
(#6989)
add adf53806e88 [HUDI-4716] Avoid parquet-hadoop-bundle in hudi-hadoop-mr
(#6930)
add 6be2057376f [HUDI-5035] Remove usage of deprecated HoodieTimer
constructor (#6952)
add b4563ddca56 [HUDI-5083]Fixed a bug when schema evolution (#7045)
add 6e40a6347c9 [HUDI-5102] source operator(monitor and reader) support
user uid (#7085)
add db1694f5fb2 [HUDI-5057] Fix msck repair external hudi table (#7084)
add a7c56978df3 [MINOR] Fix typos in Spark client related classes (#7083)
add e222693d87d [HUDI-4741] hotfix to avoid partial failover cause
restored subtask timeout (#6796)
add af317599314 [MINOR] use default maven version since it already fix the
warnings recently (#6863)
add 95c2fb9c375 Revert "[HUDI-4741] hotfix to avoid partial failover cause
restored subtask timeout (#6796)" (#7090)
add a922318a3ba [MINOR] Fix doc of
org.apache.hudi.sink.meta.CkpMetadata#bootstrap (#7048)
add 84ade4c9049 [HUDI-4799] improve analyzer exception tip when cannot
resolve expression (#6625)
add 455a3bd615b [HUDI-5096] Upgrade jcommander to 1.78 (#7068)
add 79ad3571db6 [HUDI-5105] Add Call show_commit_extra_metadata for spark
sql (#7091)
add 2f53be36bfb [HUDI-5107] Fix hadoop config in DirectWriteMarkers,
HoodieFlinkEngineContext and StreamerUtil are not consistent issue (#7094)
add 13fd5b93ce2 [MINOR] Fix OverwriteWithLatestAvroPayload full class name
(#7096)
add d5ef8a39b8b [HUDI-5074] Warn if table for metastore sync has capitals
in it (#7077)
add 8ce9f9c5742 [HUDI-5124] Fix HoodieInternalRowFileWriter#canWrite error
return tag. (#7107)
add 6fea6fc36b4 [MINOR] update commons-codec:commons-codec 1.4 to 1.13
(#6959)
add fd12a6f6508 [HUDI-5148] Claim RFC-63 for Index on Function and Logical
Partitioning (#7114)
add cbf7d0645ee [HUDI-5065] Call close on SparkRDDWriteClient in
HoodieCleaner (#7101)
add 6817977686a [HUDI-4624] Implement Closable for S3EventsSource (#7086)
add 2d2a7a52864 [HUDI-5045] Adding support to configure index type with
integ tests (#6982)
add a5434b6b4d9 [HUDI-3963] Use Lock-Free Message Queue Disruptor
Improving Hoodie Writing Efficiency (#5416)
add 1ddf9d4f9de [HUDI-5076] Fixing non serializable path used in
engineContext with metadata table intialization (#7036)
add 14ec9630c13 [HUDI-5032] Add archive to cli (#7076)
add 0d7d3aa9ddd [HUDI-4880] Fix corrupted parquet file issue left over by
cancelled compaction task (#6733)
add 22811752a80 [HUDI-5147] Flink data skipping doesn't work when
HepPlanner calls copy()… (#7113)
add 7a1be3b9b59 [MINOR] Fixing broken test (#7123)
add cd314b8cfa5 [HUDI-4898] presto/hive respect payload during merge
parquet file and logfile when reading mor table (#6741)
add d36cc05ed5b [HUDI-5126] Delete duplicate configuration items
PAYLOAD_CLASS_NAME (#7103)
add b2a84eed02e [HUDI-4989] Fixing deltastreamer init failures (#6862)
add c84400b9e4a [MINOR] Fix flaky test in ITTestHoodieDataSource (#7134)
add e088faac9ec [HUDI-4071] Remove default value for mandatory record key
field (#6681)
add 7a1a6837e0c [HUDI-5088]Fix bug:Failed to synchronize the hive metadata
of the Flink table (#7056)
add 547a2b014ed [MINOR] Removing spark2 scala12 combinations from readme
(#7112)
add c1e1825eb12 [HUDI-5153] Fix the write token name resolution of cdc log
file (#7128)
add df7b767a332 [HUDI-5066] Support flink hoodie source metaclient cache
(#7017)
add 0ef9a863748 [HUDI-5132] Add hadoop-mr bundle validation (#7157)
add bc4c0fcb3d6 [HUDI-2673] Add kafka connect bundle to validation test
(#7131)
add 655afdb49f2 [HUDI-5082] Improve the cdc log file name format (#7042)
add b3292ebf4a8 [HUDI-5154] Improve hudi-spark-client Lambada writing
(#7127)
add 1d1181a4410 [HUDI-5178] Add Call show_table_properties for spark sql
(#7161)
add 6183f684dcd [HUDI-5067] Merge the columns stats of multiple log blocks
from the same log file (#7018)
add d047fbaed00 [HUDI-5025] Rollback failed with log file not found when
rollOver in rollback process (#6939)
add 371296173a7 [HUDI-4526] Improve spillableMapBasePath when disk
directory is full (#6284)
add f19f6e7f544 [minor] Refactor the code for CkpMetadata (#7166)
add 1469469d6ec [HUDI-5111] Improve integration test coverage (#7092)
add df9100b0cb4 [HUDI-5187] Remove the preCondition check of
BucketAssigner assign state (#7170)
add 7fe35fb895f [HUDI-5145] Avoid starting HDFS in hudi-utilities tests
(#7171)
add 6508b11d7c1 [MINOR] Performance improvement of flink ITs with reused
miniCluster (#7151)
add 0b7a6656ca8 [HUDI-5171] Ensure validateTableConfig also checks for
partition path field value switch (#7163)
add 702d2b0964f [HUDI-5056] Allow wildcards in partition paths for
DELETE_PARTITIONS (#7142)
add c8c25b111db [HUDI-4888] Throw exception if COW table and consistent
hashing bucket index (#7172)
add 49c5fcbf49f [HUDI-5176] Fix incremental source to consider inflight
commits before completed commits (#7160)
add a06b1c0c4e5 [MINOR] Update RFC-46 doc (#7050)
add 1f0a5eb7a78 [HUDI-4142] Claim RFC-64 for table spec APIs (#7184)
add 6cd87b13600 [HUDI-5184] Remove unnecessary line from pyspark example
(#7178)
add fd345453eda [MINOR] Balance CI jobs (#6838)
add 47df84db178 [HUDI-5185] Fix CLI run compaction failing with
--hoodieConfigs (#7168)
add 35d90939211 [HUDI-5198] Reduce test run time in hudi-utilities and
locking related tests (#7180)
add 4da18ab5db8 [HUDI-5191] Fix compatibility with avro 1.10 (#7175)
add 49a2110fc81 [HUDI-4496] Fix Orc support broken for Spark 3.x and more
(#6227)
add b10a7c32def [HUDI-5200] Clean up resources in hudi common UT (#7190)
add 5ac605c9e76 [HUDI-5201] Add totalRecordsDeleted metric (#7181)
add 3b8df4bb204 [HUDI-5206] RowColumnReader should not return null value
for certain null child columns (#7194)
add 6b0b03b12b5 [MINOR] Make sure Dictionary Encoding in Parquet enabled
by default (#7052)
add 7e7b3a866b5 [HUDI-5221] Make the decision for flink sql bucket index
case-insensitive (#7207)
add d3f957755ab [HUDI-5223] Partial failover for flink (#7208)
add dc5cc08cccd [HUDI-5228] Flink table service job fs view conf
overwrites the one of writing job (#7214)
add 866ebf91acb [HUDI-5227] Bump Javalin to 4.6.7 and Jetty to 9.4.48
(#7211)
add 57961c0e371 Use as.of.instant for IncrementalRelation (#6921)
add 6b73c814f93 [HUDI-5209] Fixing `QueueBasedExecutor` in Spark bundles
(missing Disruptor as dep) (#7188)
add 65ce431d0c7 [HUDI-5203] Handle null fields in debezium avro payloads
(#7193)
add c5590cdd0fe [HUDI-4812] Lazy fetching partition path & file slice for
HoodieFileIndex (#6680)
add e4e28836c23 [HUDI-5233] Fix bug when
InternalSchemaUtils.collectTypeChangedCols returns all columns (#7228)
add 7f331af97cf [HUDI-3625][RFC-60] Optimized storage layout for Cloud
Object Stores (#5113)
add 34b226c0cba [HUDI-4812] Fixing `FileIndex` impls to properly batch
partitions listing (#7233)
add d976671c7bb [HUDI-5162] Allow user specified start offset for
streaming query (#7138)
add 0e1f9653c0e [HUDI-5070] Move flaky cleaner tests to separate class
(#7251)
add 926794aa74b [HUDI-5247] Clean up java client tests (#7250)
add 21bcbfd7923 [HUDI-5237] Support for HoodieUnMergedLogRecordScanner
with InternalSchema (#7237)
add 4c218231bdb [HUDI-5244] Fix bugs in schema evolution client with lost
operation field and not found schema (#7248)
add e90ab52a3de [MINOR] Fix `TestSchemaEvolutionClient` compilation (#7256)
add 8d2ad715a54 [HUDI-712] Improve exporter file listing and copy perf
(#7267)
add ceb94b47d03 [HUDI-5157] Support dropping all meta fields from source
hudi table with hudi incr source (#7132)
add dd1a2f544eb [MINOR] Fix typos in HoodieTimelineArchiver (#7268)
add 91e0db57b94 [MINOR] Use direct marker for spark engine when timeline
server is disabled (#7272)
add 3109d890f13 [HUDI-5252] ClusteringCommitSink supports to rollback
clustering (#7263)
add 3f37d4fb081 [HUDI-5258] Fix checkstyle issues in hudi-common (#7270)
add 7846b509592 [HUDI-5260] Fix insert into sql command with strict sql
insert mode (#7269)
add b6124ff85a1 [HUDI-4588][HUDI-4472] Addressing schema handling issues
in the write path (#6358)
add 431ade06fc2 [HUDI-5234] Streaming read skip clustering (#7296)
add d10d987288b [HUDI-5277] Close HoodieWriteClient before exiting
RunClusteringProcedure (#7300)
add ddcd0458510 [HUDI-5283] Replace deprecated method Schema.parse with
Schema.Parser (#7308)
add 78a0047ed99 [HUDI-5241] Optimize HoodieDefaultTimeline API (#7241)
add 62b5989c079 [HUDI-4209] Avoid using HDFS in HoodieClientTestHarness
(#7305)
add e88b4748127 [HUDI-5286] UnsupportedOperationException throws when
enabling filesystem retry (#7313)
add b31ef12f4d3 [MINOR] Re-enable Cleaner tests (#7317)
add 1cdbf68d4ee [HUDI-5242] Do not fail Meta sync in Deltastreamer when
inline table service fails (#7243)
add 6baf733292b [HUDI-5007] Prevent Hudi from reading the entire
timeline's when performing a LATEST streaming read (#6920)
add 0d70df89fe6 [MINOR] Fix IncrementalInputSplits compilation failure
(#7319)
add 5b39ac1a02f [HUDI-5285] Exclude *-site.xml files from jar packaging
(#7310)
add 98c05045ab2 [MINOR] Remove the log refrence and imports in
HoodieHFileDataBlock (#7311)
add fe43e6f85d6 [HUDI-5253] HoodieMergeOnReadTableInputFormat could have
duplicate records issue if it contains delta files while still splittable
(#7264)
add 88db1ca9741 [HUDI-5151] Fix bug with broken flink data skipping caused
by ClassNotFoundException of InLineFileSystem (#7124)
add bf2ca54fe0c [HUDI-5269] Enhancing spark-sql write tests for some of
the core user flows (#7230)
add 418091bb4e7 [HUDI-5278] Support more conf to cluster procedure (#7304)
add 2cdaa60046d [MINOR] Reuse empty InternalSchema instance (#7287)
add da89e124eba [HUDI-3981] Flink engine support for comprehensive schema
evolution (#5830)
add fdce2b80c8a [HUDI-5279] move logic for deleting active instant to
HoodieActiveTimeline (#7196)
add d7a7285b299 Rebased MOR iterators onto a `CachingIterator` (to be
idempotent) (#7334)
add eb6da963f98 [MINOR] Bumping Azure Ubuntu image to 22.04, as 18.04 will
be deprecated soon (#7347)
add f23bed8534f [HUDI-5304] Disabling spark-sql core flow tests to unblock
CI (#7346)
add 07cc3e89a73 [HUDI-5306] Unify RecordIterator and HoodieParquetReader
with ClosableIterator (#7340)
add a3f7eef765e Revert "[MINOR] Bumping Azure Ubuntu image to 22.04, as
18.04 will be deprecated soon (#7347)" (#7350)
add 3dc76af95d8 [HUDI-5302] Fix: compute hash key from recordKey failed
when recordKeyValue contains ',' (#7342)
add 4a825dcdc4e [MINOR] Disable the `SparkSqlCoreFlow` tests (#7368)
add ea48a85efcf [HUDI-5179] Updated Hudi Release guide (#7212)
add 74f8d94b73b [HUDI-5331] Add schema settings with stream api (#7384)
add 06c8fa5a62f [HUDI-5294] Support type change for schema on read +
reconcile schema (#7326)
add 20eea46ca95 [HUDI-5023] Cleaning up `QueueBasedExecutor` impls (#7238)
add 6629bf7287f [MINOR] Fix locale specific NumberFormatException in
testutils HoodieTestDataGenerator (#7215)
add 4d9035e96ba [HUDI-5334] Fix checkpoint reading for structured
streaming (#7389)
add 86d1e39fb4e [HUDI-3661] Flink async compaction is not thread safe when
use watermark (#7399)
add 6180c043464 [HUDI-5290] Remove the lock in
HoodieFlinkWriteClient#writeTableMetadata (#7320)
add 7dfe960415f [HUDI-5314] add call help procedure (#7361)
add d06661a2b4d [HUDI-5344] Fix CVE - upgrade protobuf-java to 3.18.2
(#6957)
add e96abcb2ea8 Bump protobuf-java from 3.21.5 to 3.21.7
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (f7b3ee64963)
\
N -- N -- N
refs/heads/dependabot/maven/com.google.protobuf-protobuf-java-3.21.7
(e96abcb2ea8)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.github/workflows/bot.yml | 10 +-
README.md | 5 +-
azure-pipelines.yml | 65 +-
.../deltastreamer-immutable-dataset.yaml | 14 +-
...treamer-long-running-multi-partitions-hive.yaml | 6 +-
...mer-long-running-multi-partitions-metadata.yaml | 18 +-
...eltastreamer-long-running-multi-partitions.yaml | 18 +-
.../deltastreamer-medium-clustering.yaml | 19 +-
...ltastreamer-medium-full-dataset-validation.yaml | 18 +-
.../test-suite/deltastreamer-non-partitioned.yaml | 14 +-
.../detlastreamer-long-running-example.yaml | 18 +-
.../demo/config/test-suite/simple-clustering.yaml | 16 +-
.../config/test-suite/simple-deltastreamer.yaml | 12 +
.../config/test-suite/spark-immutable-dataset.yaml | 14 +-
...ataset.yaml => spark-immutable-to-mutable.yaml} | 23 +-
.../spark-long-running-non-partitioned.yaml | 12 +
.../demo/config/test-suite/spark-long-running.yaml | 16 +-
.../config/test-suite/spark-medium-clustering.yaml | 16 +-
docker/demo/config/test-suite/spark-simple.yaml | 14 +-
.../hudi/aws/sync/AWSGlueCatalogSyncClient.java | 33 +-
.../hudi/cli/commands/ArchivedCommitsCommand.java | 50 +-
.../apache/hudi/cli/commands/CommitsCommand.java | 22 +-
.../hudi/cli/commands/CompactionCommand.java | 6 +-
.../org/apache/hudi/cli/commands/DiffCommand.java | 2 +-
.../apache/hudi/cli/commands/ExportCommand.java | 2 +-
.../hudi/cli/commands/FileSystemViewCommand.java | 2 +-
.../cli/commands/HoodieSyncValidateCommand.java | 2 +-
.../apache/hudi/cli/commands/MetadataCommand.java | 15 +-
.../apache/hudi/cli/commands/RepairsCommand.java | 15 +-
.../hudi/cli/commands/SavepointsCommand.java | 2 +-
.../org/apache/hudi/cli/commands/SparkMain.java | 31 +-
.../org/apache/hudi/cli/commands/StatsCommand.java | 3 +-
.../apache/hudi/cli/commands/TimelineCommand.java | 5 +-
.../scala/org/apache/hudi/cli/SparkHelpers.scala | 11 +-
.../hudi/cli/commands/TestArchiveCommand.java | 79 ++
.../hudi/cli/commands/TestCleansCommand.java | 8 +-
.../hudi/cli/commands/TestCommitsCommand.java | 2 +-
.../hudi/cli/commands/TestRepairsCommand.java | 73 +-
.../hudi/cli/commands/TestRollbacksCommand.java | 4 +-
.../hudi/cli/integ/ITTestClusteringCommand.java | 6 +-
.../hudi/cli/integ/ITTestCompactionCommand.java | 6 +-
hudi-client/hudi-client-common/pom.xml | 6 +
.../apache/hudi/client/BaseHoodieWriteClient.java | 66 +-
.../apache/hudi/client/HoodieTimelineArchiver.java | 55 +-
.../bootstrap/HoodieBootstrapSchemaProvider.java | 2 +-
...urrentFileWritesConflictResolutionStrategy.java | 4 +-
.../client/transaction/TransactionManager.java | 3 +
.../hudi/client/utils/MetadataConversionUtils.java | 14 +-
.../apache/hudi/client/utils/TransactionUtils.java | 21 +-
.../apache/hudi/config/HoodieClusteringConfig.java | 52 ++
.../org/apache/hudi/config/HoodieIndexConfig.java | 6 +-
.../org/apache/hudi/config/HoodieMemoryConfig.java | 9 +-
.../org/apache/hudi/config/HoodieWriteConfig.java | 120 ++-
.../hudi/execution/CopyOnWriteInsertHandler.java | 32 +-
.../org/apache/hudi/index/HoodieIndexUtils.java | 7 +-
.../apache/hudi/index/bucket/BucketIdentifier.java | 9 +-
.../org/apache/hudi/io/HoodieAppendHandle.java | 23 +-
.../org/apache/hudi/io/HoodieCreateHandle.java | 4 +-
.../org/apache/hudi/io/HoodieKeyLookupHandle.java | 2 +-
.../java/org/apache/hudi/io/HoodieMergeHandle.java | 13 +-
.../hudi/io/HoodieMergeHandleWithChangeLog.java | 4 +-
.../apache/hudi/io/HoodieSortedMergeHandle.java | 8 +-
.../java/org/apache/hudi/io/HoodieWriteHandle.java | 77 +-
.../apache/hudi/io/storage/HoodieOrcWriter.java | 4 +-
.../java/org/apache/hudi/keygen/KeyGenUtils.java | 39 +-
.../factory/HoodieAvroKeyGeneratorFactory.java | 2 +-
.../metadata/HoodieBackedTableMetadataWriter.java | 25 +-
.../org/apache/hudi/metrics/HoodieMetrics.java | 3 +
.../java/org/apache/hudi/table/HoodieTable.java | 13 +-
.../action/bootstrap/BootstrapRecordConsumer.java | 11 +-
.../table/action/clean/CleanActionExecutor.java | 10 +-
.../hudi/table/action/clean/CleanPlanner.java | 4 +-
.../action/commit/BaseCommitActionExecutor.java | 3 +-
.../hudi/table/action/commit/BaseMergeHelper.java | 79 +-
.../table/action/commit/HoodieMergeHelper.java | 194 +++--
.../compact/ScheduleCompactionActionExecutor.java | 2 +-
.../table/action/index/RunIndexActionExecutor.java | 11 +-
.../action/restore/BaseRestoreActionExecutor.java | 3 +-
.../rollback/BaseRollbackActionExecutor.java | 6 +-
.../table/action/rollback/BaseRollbackHelper.java | 8 +-
.../CopyOnWriteRollbackActionExecutor.java | 3 +-
.../MergeOnReadRollbackActionExecutor.java | 3 +-
.../hudi/table/marker/DirectWriteMarkers.java | 4 +-
.../marker/TimelineServerBasedWriteMarkers.java | 2 +-
.../org/apache/hudi/table/repair/RepairUtils.java | 4 +-
.../table/upgrade/ZeroToOneUpgradeHandler.java | 2 +-
.../hudi/util/QueueBasedExecutorFactory.java | 60 ++
.../hudi/avro/TestHoodieAvroParquetWriter.java | 2 +-
.../client/transaction/TestTransactionManager.java | 5 +
.../apache/hudi/config/TestHoodieWriteConfig.java | 15 +-
.../hudi/index/bucket/TestBucketIdentifier.java | 6 +
.../org/apache/hudi/keygen/TestKeyGenUtils.java | 18 +
.../org/apache/hudi/metrics/TestHoodieMetrics.java | 7 +-
.../hudi/testutils/HoodieWriteableTestTable.java | 2 +-
.../apache/hudi/client/HoodieFlinkWriteClient.java | 40 +-
.../client/common/HoodieFlinkEngineContext.java | 4 +
.../hudi/execution/ExplicitWriteHandler.java | 16 +-
.../hudi/execution/FlinkLazyInsertIterable.java | 19 +-
.../FlinkMergeAndReplaceHandleWithChangeLog.java | 2 +-
.../java/org/apache/hudi/io/FlinkMergeHandle.java | 8 +-
.../hudi/io/FlinkMergeHandleWithChangeLog.java | 2 +-
.../io/storage/row/HoodieRowDataCreateHandle.java | 3 +-
.../row/HoodieRowDataFileWriterFactory.java | 3 +-
.../row/parquet/ParquetSchemaConverter.java | 8 +-
.../hudi/table/HoodieFlinkCopyOnWriteTable.java | 10 +-
.../org/apache/hudi/table/HoodieFlinkTable.java | 13 +
.../commit/BaseFlinkCommitActionExecutor.java | 2 +-
.../FlinkDeletePartitionCommitActionExecutor.java} | 47 +-
.../hudi/table/action/commit/FlinkMergeHelper.java | 119 ---
.../java/org/apache/hudi/util/WriteStatMerger.java | 159 ++++
.../index/bloom/TestFlinkHoodieBloomIndex.java | 2 +
.../testutils/HoodieFlinkClientTestHarness.java | 84 +-
.../testutils/HoodieFlinkWriteableTestTable.java | 2 +-
.../hudi/execution/JavaLazyInsertIterable.java | 16 +-
.../hudi/table/HoodieJavaCopyOnWriteTable.java | 4 +-
.../commit/BaseJavaCommitActionExecutor.java | 2 +-
.../hudi/table/action/commit/JavaMergeHelper.java | 118 ---
.../TestHoodieJavaWriteClientInsert.java} | 37 +-
.../TestJavaBulkInsertInternalPartitioner.java | 4 +-
.../commit/TestJavaCopyOnWriteActionExecutor.java | 4 +-
.../action/commit/TestSchemaEvolutionClient.java | 96 +++
.../testutils/HoodieJavaClientTestHarness.java | 125 +--
.../apache/hudi/client/SparkRDDWriteClient.java | 4 +-
...parkConsistentBucketClusteringPlanStrategy.java | 2 +-
.../run/strategy/ExecutionStrategyUtil.java | 73 ++
.../MultipleSparkJobExecutionStrategy.java | 24 +-
.../strategy/SingleSparkJobExecutionStrategy.java | 33 +-
.../update/strategy/SparkAllowUpdateStrategy.java | 2 +-
...arkConsistentBucketDuplicateUpdateStrategy.java | 2 +-
.../client/utils/SparkInternalSchemaConverter.java | 11 -
.../hudi/client/utils/SparkValidatorUtils.java | 12 +-
.../client/validator/SparkPreCommitValidator.java | 2 +-
.../hudi/execution/SparkLazyInsertIterable.java | 17 +-
.../bulkinsert/RDDConsistentBucketPartitioner.java | 4 +-
.../hudi/index/hbase/SparkHoodieHBaseIndex.java | 29 +-
.../storage/row/HoodieInternalRowFileWriter.java | 2 +-
.../apache/hudi/keygen/BuiltinKeyGenerator.java | 178 +---
.../apache/hudi/keygen/ComplexKeyGenerator.java | 4 +-
.../hudi/keygen/PartitionPathFormatterBase.java | 112 +++
.../org/apache/hudi/keygen/SimpleKeyGenerator.java | 23 +-
.../hudi/keygen/StringPartitionPathFormatter.java | 72 ++
.../keygen/UTF8StringPartitionPathFormatter.java | 80 ++
.../apache/hudi/metrics/DistributedRegistry.java | 10 +-
.../SparkDeletePartitionCommitActionExecutor.java | 16 +-
.../java/org/apache/hudi/util/DataTypeUtils.java | 53 --
.../org/apache/hudi/AvroConversionUtils.scala | 35 +-
.../org/apache/hudi/HoodieConversionUtils.scala | 7 +
.../scala/org/apache/hudi/HoodieSparkUtils.scala | 18 +-
.../scala/org/apache/hudi/util/JFunction.scala | 7 +-
.../spark/sql/HoodieCatalystPlansUtils.scala | 10 +
.../org/apache/spark/sql/HoodieDataTypeUtils.scala | 55 ++
.../org/apache/hudi/client/TestClientRollback.java | 12 +-
.../hudi/client/TestCompactionAdminClient.java | 38 +-
.../hudi/client/TestFileBasedLockProvider.java | 91 +-
.../hudi/client/TestHoodieClientMultiWriter.java | 4 +-
.../apache/hudi/client/TestHoodieReadClient.java | 37 +-
.../java/org/apache/hudi/client/TestMultiFS.java | 63 +-
.../hudi/client/TestTableSchemaEvolution.java | 305 ++-----
.../hudi/client/TestUpdateSchemaEvolution.java | 3 +-
...parkConsistentBucketClusteringPlanStrategy.java | 10 +-
...DataValidationCheckForLogCompactionActions.java | 35 +-
.../functional/TestHoodieBackedMetadata.java | 2 +-
.../TestHoodieClientOnCopyOnWriteStorage.java | 70 +-
.../TestHoodieClientOnMergeOnReadStorage.java | 3 +-
.../TestBoundedInMemoryExecutorInSpark.java | 72 +-
.../hudi/execution/TestBoundedInMemoryQueue.java | 8 +-
.../execution/TestDisruptorExecutionInSpark.java | 159 ++++
.../hudi/execution/TestDisruptorMessageQueue.java | 336 ++++++++
.../TestBulkInsertInternalPartitioner.java | 16 +-
.../apache/hudi/index/TestHoodieIndexConfigs.java | 8 +-
.../index/bucket/TestHoodieSimpleBucketIndex.java | 1 +
.../index/hbase/TestSparkHoodieHBaseIndex.java | 11 +-
.../apache/hudi/io/TestHoodieTimelineArchiver.java | 15 +-
.../java/org/apache/hudi/table/TestCleaner.java | 279 ++----
.../hudi/table/TestHoodieMergeOnReadTable.java | 4 +-
.../clean/TestCleanerInsertAndCleanByCommits.java | 45 +-
.../clean/TestCleanerInsertAndCleanByVersions.java | 238 ++++++
.../commit/TestCopyOnWriteActionExecutor.java | 8 +-
.../table/action/compact/CompactionTestBase.java | 4 +-
.../table/action/compact/TestAsyncCompaction.java | 4 +-
.../table/action/compact/TestInlineCompaction.java | 10 +-
.../TestMergeOnReadRollbackActionExecutor.java | 25 +-
.../TestHoodieSparkMergeOnReadTableCompaction.java | 10 +-
.../hudi/table/marker/TestDirectWriteMarkers.java | 13 +-
.../hudi/testutils/HoodieClientTestHarness.java | 112 +--
.../hudi/testutils/HoodieClientTestUtils.java | 8 +-
.../SparkClientFunctionalTestHarness.java | 13 +-
hudi-common/pom.xml | 10 +-
.../org/apache/hudi/BaseHoodieTableFileIndex.java | 379 ++++----
.../apache/hudi/avro/AvroSchemaCompatibility.java | 949 +++++++++++++++++++++
.../java/org/apache/hudi/avro/AvroSchemaUtils.java | 129 +++
.../java/org/apache/hudi/avro/HoodieAvroUtils.java | 165 ++--
.../apache/hudi/avro/HoodieAvroWriteSupport.java | 3 +-
.../apache/hudi/avro/MercifulJsonConverter.java | 7 +-
.../hudi/common/config/ConfigClassProperty.java | 9 +-
.../apache/hudi/common/config/ConfigGroups.java | 3 +
.../apache/hudi/common/config/ConfigProperty.java | 4 +-
.../hudi/common/config/HoodieCommonConfig.java | 16 +-
.../hudi/common/config/HoodieMetadataConfig.java | 3 +
.../hudi/common/config/HoodieMetastoreConfig.java | 4 +
.../hudi/common/data/HoodieBaseListData.java | 5 +
.../common/engine/HoodieLocalEngineContext.java | 5 +-
.../common/engine/LocalTaskContextSupplier.java | 3 +
.../hudi/common/fs/BoundedFsDataInputStream.java | 3 +
.../java/org/apache/hudi/common/fs/FSUtils.java | 16 +-
.../common/fs/HoodieRetryWrapperFileSystem.java | 13 +-
.../hudi/common/fs/HoodieWrapperFileSystem.java | 12 +-
.../hudi/common/function/SerializableSupplier.java | 5 +
.../org/apache/hudi/common/metrics/Registry.java | 5 +-
.../hudi/common/model/AWSDmsAvroPayload.java | 3 +-
.../org/apache/hudi/common/model/BaseFile.java | 3 +-
.../common/model/BootstrapBaseFileMapping.java | 3 +-
.../hudi/common/model/BootstrapFileMapping.java | 3 +-
.../apache/hudi/common/model/CleanFileInfo.java | 3 +
.../hudi/common/model/ConsistentHashingNode.java | 3 +
.../apache/hudi/common/model/HoodieAvroRecord.java | 5 +
.../common/model/HoodieColumnRangeMetadata.java | 38 +
.../hudi/common/model/HoodieCommitMetadata.java | 3 +-
.../hudi/common/model/HoodieDeltaWriteStat.java | 28 +-
.../apache/hudi/common/model/HoodieLogFile.java | 3 +-
.../hudi/common/model/HoodieMetadataWrapper.java | 3 +
.../apache/hudi/common/model/HoodieWriteStat.java | 3 +-
.../OverwriteNonDefaultsWithLatestAvroPayload.java | 4 +-
.../model/OverwriteWithLatestAvroPayload.java | 2 +-
.../common/model/PartialUpdateAvroPayload.java | 10 +-
.../hudi/common/model/RewriteAvroPayload.java | 3 +-
.../debezium/AbstractDebeziumAvroPayload.java | 3 +-
.../model/debezium/MySqlDebeziumAvroPayload.java | 16 +-
.../debezium/PostgresDebeziumAvroPayload.java | 17 +-
.../hudi/common/table/HoodieTableConfig.java | 2 +-
.../hudi/common/table/HoodieTableMetaClient.java | 28 +-
.../hudi/common/table/TableSchemaResolver.java | 96 +--
.../hudi/common/table/cdc/HoodieCDCExtractor.java | 22 +-
.../hudi/common/table/cdc/HoodieCDCFileSplit.java | 4 +-
.../hudi/common/table/cdc/HoodieCDCUtils.java | 2 +-
.../table/log/AbstractHoodieLogRecordReader.java | 99 ++-
.../table/log/HoodieCDCLogRecordIterator.java | 4 +-
.../hudi/common/table/log/HoodieLogFileReader.java | 14 +-
.../hudi/common/table/log/HoodieLogFormat.java | 15 +
.../common/table/log/HoodieLogFormatReader.java | 2 +-
.../table/log/HoodieMergedLogRecordScanner.java | 7 +-
.../table/log/HoodieUnMergedLogRecordScanner.java | 17 +-
.../table/log/block/HoodieAvroDataBlock.java | 50 +-
.../common/table/log/block/HoodieCDCDataBlock.java | 7 +-
.../common/table/log/block/HoodieDataBlock.java | 22 -
.../table/log/block/HoodieHFileDataBlock.java | 26 +-
.../common/table/log/block/HoodieLogBlock.java | 2 +-
.../table/log/block/HoodieParquetDataBlock.java | 34 +-
.../table/timeline/HoodieActiveTimeline.java | 16 +-
.../table/timeline/HoodieArchivedTimeline.java | 2 +-
.../table/timeline/HoodieDefaultTimeline.java | 93 +-
.../table/timeline/HoodieInstantTimeGenerator.java | 1 +
.../hudi/common/table/timeline/HoodieTimeline.java | 10 +-
.../common/table/timeline/TimelineDiffHelper.java | 12 +-
.../table/timeline/TimelineMetadataUtils.java | 15 +-
.../hudi/common/table/timeline/TimelineUtils.java | 26 +-
.../common/table/timeline/dto/ClusteringOpDTO.java | 5 +-
.../common/table/timeline/dto/TimelineDTO.java | 2 +-
.../versioning/clean/CleanMetadataMigrator.java | 3 +
.../clean/CleanMetadataV1MigrationHandler.java | 3 +
.../clean/CleanMetadataV2MigrationHandler.java | 3 +
.../clean/CleanPlanV1MigrationHandler.java | 13 +-
.../clean/CleanPlanV2MigrationHandler.java | 11 +-
.../table/view/AbstractTableFileSystemView.java | 9 +-
.../view/HoodieTablePreCommitFileSystemView.java | 3 +-
.../IncrementalTimelineSyncFileSystemView.java | 5 +-
.../table/view/RocksDbBasedFileSystemView.java | 5 +-
.../view/SpillableMapBasedFileSystemView.java | 4 +-
.../org/apache/hudi/common/util/AvroOrcUtils.java | 54 +-
.../apache/hudi/common/util/Base64CodecUtil.java | 3 +
.../org/apache/hudi/common/util/BaseFileUtils.java | 3 +
.../org/apache/hudi/common/util/BinaryUtil.java | 13 +
.../org/apache/hudi/common/util/CleanerUtils.java | 7 +-
.../apache/hudi/common/util/ClusteringUtils.java | 5 +-
.../apache/hudi/common/util/CollectionUtils.java | 7 +-
.../org/apache/hudi/common/util/CommitUtils.java | 2 +-
.../apache/hudi/common/util/CompactionUtils.java | 6 +-
.../hudi/common/util/CustomizedThreadFactory.java | 26 +-
.../org/apache/hudi/common/util/DateTimeUtils.java | 3 +
.../java/org/apache/hudi/common/util/Either.java | 14 +
.../org/apache/hudi/common/util/FileIOUtils.java | 36 +
.../org/apache/hudi/common/util/FutureUtils.java | 1 +
.../hudi/common/util/InternalSchemaCache.java | 38 +-
.../org/apache/hudi/common/util/JsonUtils.java | 7 +-
.../java/org/apache/hudi/common/util/MapUtils.java | 3 +
.../java/org/apache/hudi/common/util/Option.java | 11 +-
.../apache/hudi/common/util/OrcReaderIterator.java | 8 +-
.../java/org/apache/hudi/common/util/OrcUtils.java | 9 +-
.../org/apache/hudi/common/util/ParquetUtils.java | 6 +-
.../org/apache/hudi/common/util/RateLimiter.java | 6 +-
.../org/apache/hudi/common/util/RetryHelper.java | 17 +-
.../hudi/common/util/SerializationUtils.java | 39 +-
.../apache/hudi/common/util/TablePathUtils.java | 10 +-
.../org/apache/hudi/common/util/ThreadUtils.java | 3 +
.../org/apache/hudi/common/util/TypeUtils.java | 4 +
.../apache/hudi/common/util/ValidationUtils.java | 2 +-
.../CloseableMappingIterator.java} | 28 +-
.../util/collection/ExternalSpillableMap.java | 22 +-
.../MappingIterator.java} | 32 +-
.../hudi/common/util/collection/RocksDBDAO.java | 3 +-
.../org/apache/hudi/common/util/hash/HashID.java | 3 +-
.../util/io/ByteBufferBackedInputStream.java | 1 +
.../jvm/HotSpotMemoryLayoutSpecification32bit.java | 4 +
.../jvm/HotSpotMemoryLayoutSpecification64bit.java | 4 +
...otMemoryLayoutSpecification64bitCompressed.java | 4 +
.../jvm/OpenJ9MemoryLayoutSpecification32bit.java | 4 +
.../jvm/OpenJ9MemoryLayoutSpecification64bit.java | 4 +
...J9MemoryLayoutSpecification64bitCompressed.java | 4 +
.../util/queue/BaseHoodieQueueBasedExecutor.java | 193 +++++
.../common/util/queue/BoundedInMemoryExecutor.java | 165 +---
.../common/util/queue/BoundedInMemoryQueue.java | 26 +-
.../hudi/common/util/queue/DisruptorExecutor.java | 54 ++
.../common/util/queue/DisruptorMessageQueue.java | 125 +++
.../util/queue/DisruptorWaitStrategyType.java | 67 ++
...nMemoryQueueConsumer.java => ExecutorType.java} | 48 +-
.../util/queue/FunctionBasedQueueProducer.java | 15 +-
...emoryQueueProducer.java => HoodieConsumer.java} | 17 +-
...emoryQueueProducer.java => HoodieExecutor.java} | 25 +-
...yQueueConsumer.java => HoodieMessageQueue.java} | 50 +-
...emoryQueueProducer.java => HoodieProducer.java} | 13 +-
.../util/queue/IteratorBasedQueueProducer.java | 13 +-
.../common/util/queue/WaitStrategyFactory.java | 58 ++
.../hudi/exception/HoodieCompactException.java | 3 +
...ava => HoodieDebeziumAvroPayloadException.java} | 14 +-
.../exception/HoodieSecondaryIndexException.java | 3 +
.../java/org/apache/hudi/hadoop/CachingPath.java | 54 +-
.../hudi/internal/schema/InternalSchema.java | 39 +-
.../java/org/apache/hudi/internal/schema/Type.java | 32 +
.../org/apache/hudi/internal/schema/Types.java | 96 ++-
.../schema/action/InternalSchemaMerger.java | 2 +-
.../hudi/internal/schema/action/TableChange.java | 7 +-
.../hudi/internal/schema/action/TableChanges.java | 3 +
.../convert/AvroInternalSchemaConverter.java | 162 ++--
.../io/FileBasedInternalSchemaStorageManager.java | 13 +-
.../schema/utils/AvroSchemaEvolutionUtils.java | 42 +-
.../internal/schema/utils/InternalSchemaUtils.java | 6 +-
.../internal/schema/utils/SchemaChangeUtils.java | 13 +-
.../hudi/internal/schema/utils/SerDeHelper.java | 31 +-
.../internal/schema/visitor/NameToIDVisitor.java | 4 +-
.../apache/hudi/io/storage/HoodieFileReader.java | 10 +-
.../hudi/io/storage/HoodieFileReaderFactory.java | 5 +-
.../apache/hudi/io/storage/HoodieHFileReader.java | 21 +-
.../apache/hudi/io/storage/HoodieOrcReader.java | 14 +-
.../hudi/io/storage/HoodieParquetConfig.java | 5 -
.../hudi/io/storage/HoodieParquetReader.java | 69 +-
.../hudi/io/storage/HoodieParquetStreamWriter.java | 13 +-
.../org/apache/hudi/keygen/BaseKeyGenerator.java | 7 +-
.../apache/hudi/keygen/KeyGeneratorInterface.java | 3 +-
.../hudi/keygen/constant/KeyGeneratorOptions.java | 5 +-
.../apache/hudi/metadata/BaseTableMetadata.java | 22 +-
.../metadata/FileSystemBackedTableMetadata.java | 28 +-
.../hudi/metadata/HoodieBackedTableMetadata.java | 28 +-
.../HoodieMetadataMergedLogRecordReader.java | 14 +-
.../hudi/metadata/HoodieMetadataMetrics.java | 3 +
.../hudi/metadata/HoodieMetadataPayload.java | 57 +-
.../apache/hudi/metadata/HoodieTableMetadata.java | 18 +-
.../hudi/metadata/MetadataPartitionType.java | 3 +
.../hudi/parquet/io/ByteBufferBackedInputFile.java | 1 +
.../parquet/io/OutputStreamBackedOutputFile.java | 1 +
.../hudi/secondary/index/HoodieSecondaryIndex.java | 9 +
.../secondary/index/SecondaryIndexManager.java | 3 +
.../hudi/secondary/index/SecondaryIndexType.java | 3 +
.../hudi/secondary/index/SecondaryIndexUtils.java | 3 +
.../src/main/java/org/apache/hudi/util/Lazy.java | 1 +
.../org/apache/hudi/avro/TestAvroSchemaUtils.java | 175 ++++
.../org/apache/hudi/avro/TestHoodieAvroUtils.java | 74 +-
.../java/org/apache/hudi/common/TestRegistry.java | 4 +
.../hudi/common/bootstrap/TestBootstrapIndex.java | 6 +
.../hudi/common/config/TestConfigProperty.java | 4 +
.../common/data/TestHoodieListDataPairData.java | 8 +-
.../org/apache/hudi/common/fs/TestFSUtils.java | 132 ++-
.../fs/TestFSUtilsWithRetryWrapperEnable.java | 16 +-
.../hudi/common/fs/inline/InLineFSUtilsTest.java | 3 +-
.../common/functional/TestHoodieLogFormat.java | 66 +-
.../TestHoodieLogFormatAppendFailure.java | 9 +-
.../hudi/common/model/TestAWSDmsAvroPayload.java | 6 +-
.../model/TestHoodieConsistentHashingMetadata.java | 3 +
.../hudi/common/model/TestHoodieFileGroup.java | 3 +
.../common/model/TestHoodiePartitionMetadata.java | 10 +
.../model/TestHoodieReplaceCommitMetadata.java | 3 +
.../hudi/common/model/TestHoodieWriteStat.java | 1 +
...tOverwriteNonDefaultsWithLatestAvroPayload.java | 11 +-
.../common/model/TestPartialUpdateAvroPayload.java | 8 +-
.../debezium/TestMySqlDebeziumAvroPayload.java | 44 +-
.../debezium/TestPostgresDebeziumAvroPayload.java | 44 +-
.../common/properties/TestOrderedProperties.java | 3 +
.../common/properties/TestTypedProperties.java | 3 +
.../hudi/common/table/TestHoodieTableConfig.java | 9 +
.../common/table/TestHoodieTableMetaClient.java | 10 +-
.../hudi/common/table/TestTableSchemaResolver.java | 3 +
.../hudi/common/table/TestTimelineUtils.java | 34 +-
.../table/timeline/TestHoodieActiveTimeline.java | 22 +-
.../common/table/timeline/TestTimelineLayout.java | 3 +
.../view/TestHoodieTableFSViewWithClustering.java | 9 +
.../table/view/TestHoodieTableFileSystemView.java | 10 +
.../table/view/TestIncrementalFSViewSync.java | 170 ++--
.../view/TestPriorityBasedFileSystemView.java | 15 +-
.../hudi/common/testutils/CheckedFunction.java | 6 +
.../hudi/common/testutils/ClusteringTestUtils.java | 7 +-
.../hudi/common/testutils/FileCreateUtils.java | 3 +
.../hudi/common/testutils/FileSystemTestUtils.java | 8 +-
.../hudi/common/testutils/HadoopMapRedUtils.java | 6 +-
.../common/testutils/HoodieCommonTestHarness.java | 31 +-
.../common/testutils/HoodieTestDataGenerator.java | 185 +++-
.../hudi/common/testutils/HoodieTestTable.java | 8 +-
.../hudi/common/testutils/NetworkTestUtils.java | 3 +
.../testutils/minicluster/HdfsTestService.java | 39 +-
.../testutils/minicluster/MiniClusterUtil.java | 59 --
.../apache/hudi/common/util/TestAvroOrcUtils.java | 13 +-
.../hudi/common/util/TestBase64CodecUtil.java | 3 +
.../apache/hudi/common/util/TestBinaryUtil.java | 5 +
.../hudi/common/util/TestClusteringUtils.java | 1 +
.../apache/hudi/common/util/TestCommitUtils.java | 4 +
.../hudi/common/util/TestCompactionUtils.java | 6 +-
.../common/util/TestCustomizedThreadFactory.java | 4 +
.../util/TestDFSPropertiesConfiguration.java | 4 +-
.../apache/hudi/common/util/TestDateTimeUtils.java | 3 +
.../hudi/common/util/TestObjectSizeCalculator.java | 15 +-
.../hudi/common/util/TestOrcReaderIterator.java | 9 +-
.../common/util/TestParquetReaderIterator.java | 3 +
.../apache/hudi/common/util/TestRatelimiter.java | 6 +-
.../hudi/common/util/TestSerializableSchema.java | 3 +-
.../hudi/common/util/TestSerializationUtils.java | 33 +-
.../apache/hudi/common/util/TestStringUtils.java | 3 +
.../hudi/common/util/TestTablePathUtils.java | 7 +-
.../common/util/collection/TestRocksDBDAO.java | 3 +
.../apache/hudi/common/util/hash/TestHashID.java | 4 +
.../util/io/TestByteBufferBackedInputStream.java | 3 +
.../hudi/internal/schema/TestSerDeHelper.java | 22 +-
.../internal/schema/action/TestMergeSchema.java | 37 +-
.../internal/schema/action/TestTableChanges.java | 73 +-
.../TestFileBasedInternalSchemaStorageManager.java | 9 +-
.../schema/utils/TestAvroSchemaEvolutionUtils.java | 59 +-
.../schema/utils/TestInternalSchemaUtils.java | 36 +-
.../TestFileSystemBackedTableMetadata.java | 9 +-
.../hudi/metadata/TestHoodieMetadataPayload.java | 157 ++++
.../examples/quickstart/HoodieSparkQuickstart.java | 2 +-
.../quickstart/TestHoodieSparkQuickstart.java | 10 -
.../hudi-examples-spark/src/test/python/README.md | 1 -
.../apache/hudi/configuration/FlinkOptions.java | 8 +
.../apache/hudi/configuration/OptionsResolver.java | 17 +-
.../java/org/apache/hudi/sink/CleanFunction.java | 4 +-
.../hudi/sink/StreamWriteOperatorCoordinator.java | 3 +-
.../hudi/sink/bootstrap/BootstrapOperator.java | 6 +-
.../sink/bucket/BucketBulkInsertWriterHelper.java | 5 +-
.../hudi/sink/bulk/BulkInsertWriteFunction.java | 4 +-
.../hudi/sink/bulk/BulkInsertWriterHelper.java | 5 +-
.../hudi/sink/clustering/ClusteringCommitSink.java | 7 +-
.../hudi/sink/clustering/ClusteringOperator.java | 19 +-
.../sink/clustering/ClusteringPlanOperator.java | 4 +-
.../sink/clustering/HoodieFlinkClusteringJob.java | 3 +-
.../sink/common/AbstractStreamWriteFunction.java | 15 +-
.../{CompactFunction.java => CompactOperator.java} | 47 +-
.../hudi/sink/compact/CompactionCommitEvent.java | 2 +-
.../hudi/sink/compact/CompactionCommitSink.java | 4 +-
.../hudi/sink/compact/HoodieFlinkCompactor.java | 6 +-
.../compact/strategy/CompactionPlanStrategies.java | 7 +-
.../org/apache/hudi/sink/meta/CkpMetadata.java | 15 +-
.../sink/partitioner/BucketAssignFunction.java | 4 +-
.../hudi/sink/partitioner/BucketAssigner.java | 7 -
.../sink/partitioner/profile/WriteProfile.java | 2 +-
.../sink/partitioner/profile/WriteProfiles.java | 4 +-
.../java/org/apache/hudi/sink/utils/Pipelines.java | 35 +-
.../java/org/apache/hudi/source/FileIndex.java | 7 +-
.../apache/hudi/source/IncrementalInputSplits.java | 80 +-
.../hudi/source/StreamReadMonitoringFunction.java | 1 +
.../org/apache/hudi/table/HoodieTableSource.java | 39 +-
.../apache/hudi/table/catalog/HiveSchemaUtils.java | 8 +-
.../apache/hudi/table/catalog/HoodieCatalog.java | 67 +-
.../hudi/table/catalog/HoodieCatalogUtil.java | 59 ++
.../hudi/table/catalog/HoodieHiveCatalog.java | 88 +-
.../java/org/apache/hudi/table/format/CastMap.java | 223 +++++
.../org/apache/hudi/table/format/FormatUtils.java | 77 +-
.../hudi/table/format/InternalSchemaManager.java | 170 ++++
.../table/format/ParquetSplitRecordIterator.java | 61 ++
.../apache/hudi/table/format/RecordIterators.java | 91 ++
.../table/format/SchemaEvolvedRecordIterator.java | 52 ++
.../hudi/table/format/cdc/CdcInputFormat.java | 142 +--
.../table/format/cow/CopyOnWriteInputFormat.java | 27 +-
.../table/format/mor/MergeOnReadInputFormat.java | 219 +++--
.../java/org/apache/hudi/util/ClusteringUtil.java | 35 +
.../java/org/apache/hudi/util/DataTypeUtils.java | 44 +
.../java/org/apache/hudi/util/FlinkTables.java | 5 +-
.../org/apache/hudi/util/FlinkWriteClients.java | 245 ++++++
.../java/org/apache/hudi/util/HoodiePipeline.java | 15 +
.../apache/hudi/util/RowDataCastProjection.java | 40 +-
.../org/apache/hudi/util/RowDataProjection.java | 9 +-
.../java/org/apache/hudi/util/StreamerUtil.java | 195 +----
.../apache/hudi/sink/ITTestDataStreamWrite.java | 130 +++
.../sink/TestStreamWriteOperatorCoordinator.java | 53 +-
.../org/apache/hudi/sink/TestWriteCopyOnWrite.java | 6 +-
.../sink/cluster/ITTestHoodieFlinkClustering.java | 73 +-
.../sink/compact/ITTestHoodieFlinkCompactor.java | 18 +-
.../hudi/sink/partitioner/TestBucketAssigner.java | 5 +-
.../hudi/sink/utils/ClusteringFunctionWrapper.java | 2 +-
.../hudi/sink/utils/CompactFunctionWrapper.java | 53 +-
.../sink/utils/StreamWriteFunctionWrapper.java | 2 +-
.../hudi/source/TestIncrementalInputSplits.java | 88 ++
.../apache/hudi/table/ITTestHoodieDataSource.java | 113 ++-
.../apache/hudi/table/ITTestSchemaEvolution.java | 461 ++++++++++
.../apache/hudi/table/TestHoodieTableSource.java | 52 +-
.../hudi/table/catalog/TestHoodieCatalog.java | 55 +-
.../hudi/table/catalog/TestHoodieHiveCatalog.java | 62 ++
.../org/apache/hudi/table/format/TestCastMap.java | 120 +++
.../apache/hudi/table/format/TestInputFormat.java | 81 +-
.../org/apache/hudi/utils/FlinkMiniCluster.java | 82 ++
.../org/apache/hudi/utils/TestClusteringUtil.java | 127 +++
.../org/apache/hudi/utils/TestCompactionUtil.java | 15 +-
.../org/apache/hudi/utils/TestConfigurations.java | 23 +
.../test/java/org/apache/hudi/utils/TestData.java | 10 +-
.../test/java/org/apache/hudi/utils/TestSQL.java | 5 +
.../org/apache/hudi/utils/TestStreamerUtil.java | 11 -
.../test/java/org/apache/hudi/utils/TestUtils.java | 4 +-
.../hudi/utils/TestViewStorageProperties.java | 10 +
.../format/cow/vector/HeapRowColumnVector.java | 8 +
.../format/cow/vector/reader/RowColumnReader.java | 14 +-
.../format/cow/vector/HeapRowColumnVector.java | 8 +
.../format/cow/vector/reader/RowColumnReader.java | 14 +-
.../format/cow/vector/HeapRowColumnVector.java | 8 +
.../format/cow/vector/reader/RowColumnReader.java | 14 +-
.../hudi/hadoop/HiveHoodieTableFileIndex.java | 18 +-
.../hudi/hadoop/HoodieParquetInputFormat.java | 3 +
.../apache/hudi/hadoop/SchemaEvolutionContext.java | 353 ++++++++
.../realtime/AbstractRealtimeRecordReader.java | 81 +-
.../realtime/HoodieHFileRealtimeInputFormat.java | 2 +-
.../HoodieMergeOnReadTableInputFormat.java | 7 +-
.../realtime/HoodieParquetRealtimeInputFormat.java | 14 +-
.../hudi/hadoop/realtime/HoodieRealtimePath.java | 2 +-
.../realtime/RealtimeCompactedRecordReader.java | 29 +-
.../realtime/RealtimeUnmergedRecordReader.java | 83 +-
.../hudi/hadoop/utils/HiveAvroSerializer.java | 413 +++++++++
.../hudi/hadoop/utils/HoodieInputFormatUtils.java | 17 +-
.../utils/HoodieRealtimeInputFormatUtils.java | 19 +-
.../utils/HoodieRealtimeRecordReaderUtils.java | 5 +
.../hive/TestHoodieCombineHiveInputFormat.java | 32 +-
.../TestHoodieMergeOnReadTableInputFormat.java | 13 +
.../hudi/hadoop/testutils/InputFormatTestUtil.java | 2 +-
.../hudi/hadoop/utils/TestHiveAvroSerializer.java | 148 ++++
.../hudi/integ/testsuite/HoodieTestSuiteJob.java | 10 +
.../integ/testsuite/HoodieTestSuiteWriter.java | 3 +-
.../integ/testsuite/dag/nodes/PrestoQueryNode.java | 44 +-
.../dag/nodes/ValidateAsyncOperations.java | 2 +-
.../testsuite/dag/nodes/ValidateDatasetNode.java | 13 +-
.../testsuite/dag/nodes/SparkInsertNode.scala | 4 +-
.../TestDFSHoodieTestSuiteWriterAdapter.java | 8 +-
.../integ/testsuite/TestFileDeltaInputWriter.java | 10 +-
.../testsuite/job/TestHoodieTestSuiteJob.java | 84 +-
.../reader/TestDFSAvroDeltaInputReader.java | 10 +-
.../reader/TestDFSHoodieDatasetInputReader.java | 6 +-
hudi-kafka-connect/demo/setupKafka.sh | 7 +-
.../hudi/connect/utils/KafkaConnectUtils.java | 10 +-
.../writers/KafkaConnectTransactionServices.java | 2 +-
.../main/java/org/apache/hudi/DataSourceUtils.java | 4 +-
.../org/apache/hudi/ColumnStatsIndexSupport.scala | 6 +-
.../scala/org/apache/hudi/DataSourceOptions.scala | 50 +-
.../main/scala/org/apache/hudi/DefaultSource.scala | 15 +-
.../scala/org/apache/hudi/HoodieFileIndex.scala | 71 +-
.../org/apache/hudi/HoodieSparkSqlWriter.scala | 544 ++++++++----
.../scala/org/apache/hudi/HoodieWriterUtils.scala | 8 +-
.../org/apache/hudi/IncrementalRelation.scala | 22 +-
.../{LogFileIterator.scala => Iterators.scala} | 29 +-
.../hudi/MergeOnReadIncrementalRelation.scala | 7 +-
.../apache/hudi/SparkHoodieTableFileIndex.scala | 266 ++++--
.../org/apache/hudi/util/CachingIterator.scala | 38 +-
.../sql/catalyst/catalog/HoodieCatalogTable.scala | 13 +-
.../apache/spark/sql/hive/HiveClientUtils.scala | 14 +
.../apache/spark/sql/hudi/HoodieOptionConfig.scala | 1 -
.../spark/sql/hudi/HoodieSqlCommonUtils.scala | 16 +-
.../spark/sql/hudi/ProvidesHoodieConfig.scala | 7 +-
.../org/apache/spark/sql/hudi/SerDeUtils.scala | 31 +-
.../AlterHoodieTableChangeColumnCommand.scala | 5 +-
.../hudi/command/CreateHoodieTableCommand.scala | 3 +-
.../hudi/command/RepairHoodieTableCommand.scala | 169 ++++
.../sql/hudi/streaming/HoodieMetadataLog.scala | 79 ++
.../streaming/HoodieOffsetRangeLimit.scala} | 30 +-
.../sql/hudi/streaming/HoodieStreamSource.scala | 100 +--
hudi-spark-datasource/hudi-spark/pom.xml | 3 +
.../hudi/spark/sql/parser/HoodieSqlCommon.g4 | 6 +-
.../org/apache/hudi/HoodieDataSourceHelpers.java | 2 +-
.../org/apache/spark/sql/hudi/SparkHelpers.scala | 11 +-
.../spark/sql/hudi/analysis/HoodieAnalysis.scala | 16 +
.../hudi/command/MergeIntoHoodieTableCommand.scala | 92 +-
.../hudi/command/payload/ExpressionPayload.scala | 60 +-
.../hudi/command/procedures/BaseProcedure.scala | 4 +-
.../procedures/CommitsCompareProcedure.scala | 4 +-
.../hudi/command/procedures/CopyToTempView.scala | 114 +++
.../procedures/CreateMetadataTableProcedure.scala | 2 +-
.../hudi/command/procedures/HelpProcedure.scala | 125 +++
.../hudi/command/procedures/HoodieProcedures.scala | 8 +
.../procedures/InitMetadataTableProcedure.scala | 2 +-
.../procedures/RunClusteringProcedure.scala | 143 +++-
.../procedures/RunCompactionProcedure.scala | 6 +-
.../ShowCommitExtraMetadataProcedure.scala | 138 +++
.../command/procedures/ShowCommitsProcedure.scala | 5 +-
.../ShowMetadataTableFilesProcedure.scala | 2 +-
.../ShowMetadataTablePartitionsProcedure.scala | 2 +-
.../procedures/ShowTablePropertiesProcedure.scala | 71 ++
.../procedures/ValidateHoodieSyncProcedure.scala | 2 +-
.../ValidateMetadataTableFilesProcedure.scala | 2 +-
.../sql/parser/HoodieSqlCommonAstBuilder.scala | 21 +-
.../src/test/java/HoodieJavaStreamingApp.java | 9 +-
.../apache/hudi/functional/SparkSQLCoreFlow.java | 22 +-
.../org/apache/hudi/functional/TestBootstrap.java | 6 +-
.../functional/TestHiveTableSchemaEvolution.java | 155 ++++
.../apache/hudi/functional/TestOrcBootstrap.java | 6 +
.../apache/hudi/keygen/TestSimpleKeyGenerator.java | 5 +-
.../org/apache/hudi/ScalaAssertionSupport.scala | 4 +-
.../org/apache/hudi/TestDataSourceOptions.scala | 2 +
.../org/apache/hudi/TestHoodieFileIndex.scala | 265 ++++--
.../org/apache/hudi/TestHoodieSparkSqlWriter.scala | 270 ++++--
.../hudi/functional/TestBasicSchemaEvolution.scala | 356 ++++++++
.../apache/hudi/functional/TestCOWDataSource.scala | 122 +--
.../hudi/functional/TestCOWDataSourceStorage.scala | 3 +
.../TestIncrementalReadWithFullTableScan.scala | 2 +-
.../apache/hudi/functional/TestMORDataSource.scala | 8 +-
...cala => TestMORDataSourceWithBucketIndex.scala} | 16 +-
.../hudi/functional/TestSparkSqlCoreFlow.scala | 441 ++++++++++
.../hudi/functional/TestStreamingSource.scala | 60 ++
.../hudi/functional/cdc/HoodieCDCTestBase.scala | 5 +-
.../benchmark/BoundInMemoryExecutorBenchmark.scala | 135 +++
.../org/apache/spark/sql/hudi/TestAlterTable.scala | 1 -
.../apache/spark/sql/hudi/TestCreateTable.scala | 18 +-
.../apache/spark/sql/hudi/TestInsertTable.scala | 99 ++-
.../spark/sql/hudi/TestMergeIntoTable2.scala | 134 ++-
.../apache/spark/sql/hudi/TestRepairTable.scala | 197 +++++
.../org/apache/spark/sql/hudi/TestSpark3DDL.scala | 144 +++-
.../hudi/procedure/TestClusteringProcedure.scala | 269 +++++-
.../sql/hudi/procedure/TestCommitsProcedure.scala | 54 +-
.../procedure/TestCopyToTempViewProcedure.scala | 168 ++++
.../sql/hudi/procedure/TestHelpProcedure.scala | 84 ++
.../sql/hudi/procedure/TestRepairsProcedure.scala | 4 +-
.../TestShowTablePropertiesProcedure.scala | 45 +
.../spark/sql/HoodieSpark2CatalystPlanUtils.scala | 16 +-
.../apache/hudi/spark3/internal/DefaultSource.java | 5 +-
.../spark/sql/HoodieSpark31CatalystPlanUtils.scala | 16 +
.../parquet/Spark31HoodieParquetFileFormat.scala | 2 +-
.../hudi/command/Spark31AlterTableCommand.scala | 3 +-
.../spark/sql/HoodieSpark32CatalystPlanUtils.scala | 16 +
.../spark/sql/hudi/command/AlterTableCommand.scala | 3 +-
.../spark/sql/HoodieSpark33CatalystPlanUtils.scala | 15 +
.../org/apache/hudi/hive/HoodieHiveSyncClient.java | 2 +-
.../apache/hudi/hive/ddl/HiveQueryDDLExecutor.java | 2 +-
.../org/apache/hudi/hive/TestHiveSyncTool.java | 4 +-
.../org/apache/hudi/hive/TestHiveSyncToolArgs.java | 54 ++
.../hudi/sync/common/util/SyncUtilHelpers.java | 11 +-
hudi-timeline-service/pom.xml | 2 +-
.../hudi/timeline/service/RequestHandler.java | 9 +-
.../hudi/timeline/service/TimelineService.java | 10 +-
.../marker/BatchedMarkerCreationRunnable.java | 2 +-
.../handlers/marker/MarkerCreationFuture.java | 2 +-
.../service/handlers/marker/MarkerDirState.java | 2 +-
hudi-utilities/pom.xml | 4 +
.../hudi/utilities/HiveIncrementalPuller.java | 4 +-
.../org/apache/hudi/utilities/HoodieCleaner.java | 5 +-
.../hudi/utilities/HoodieDataTableValidator.java | 2 +-
.../utilities/HoodieMetadataTableValidator.java | 2 +-
.../hudi/utilities/HoodieSnapshotExporter.java | 74 +-
.../org/apache/hudi/utilities/UtilHelpers.java | 60 +-
.../hudi/utilities/deltastreamer/DeltaSync.java | 174 ++--
.../deltastreamer/HoodieDeltaStreamer.java | 64 +-
.../utilities/schema/SimpleSchemaProvider.java | 30 +-
.../utilities/schema/SparkAvroPostProcessor.java | 12 +-
.../hudi/utilities/sources/HoodieIncrSource.java | 19 +-
.../sources/S3EventsHoodieIncrSource.java | 4 +-
.../hudi/utilities/sources/S3EventsSource.java | 10 +-
.../sources/helpers/IncrSourceHelper.java | 21 +-
.../utilities/sources/helpers/gcs/QueryInfo.java | 5 +-
.../apache/hudi/utilities/TestHoodieIndexer.java | 205 ++---
.../hudi/utilities/TestSchemaPostProcessor.java | 23 +-
.../deser/TestKafkaAvroSchemaDeserializer.java | 3 +-
.../functional/HoodieDeltaStreamerTestBase.java | 11 +-
.../functional/TestHoodieDeltaStreamer.java | 457 +++++-----
.../TestHoodieDeltaStreamerWithMultiWriter.java | 3 +-
.../TestHoodieMultiTableDeltaStreamer.java | 40 +-
.../functional/TestHoodieSnapshotExporter.java | 7 +
.../hudi/utilities/sources/TestAvroDFSSource.java | 4 +-
.../hudi/utilities/sources/TestCsvDFSSource.java | 4 +-
.../utilities/sources/TestGcsEventsSource.java | 15 +-
.../utilities/sources/TestHoodieIncrSource.java | 233 ++++-
.../hudi/utilities/sources/TestJdbcSource.java | 6 +
.../hudi/utilities/sources/TestJsonDFSSource.java | 4 +-
.../utilities/sources/TestParquetDFSSource.java | 2 +-
.../hudi/utilities/sources/TestS3EventsSource.java | 4 +-
.../hudi/utilities/sources/TestSqlSource.java | 11 +-
.../debezium/TestAbstractDebeziumSource.java | 28 +-
.../sources/helpers/TestKafkaOffsetGen.java | 48 +-
.../utilities/testutils/UtilitiesTestBase.java | 94 +-
.../AbstractCloudObjectsSourceTestBase.java | 2 +-
.../sources/AbstractDFSSourceTestBase.java | 18 +-
.../transform/TestSqlFileBasedTransformer.java | 23 +-
.../delta-streamer-config/source_evolved.avsc | 225 ++---
.../source_evolved_post_processed.avsc | 156 ++++
.../spark-defaults.conf => Dockerfile} | 15 +-
.../Dockerfile => Dockerfile-base} | 25 +-
.../{spark-write-hive-sync => }/ci_run.sh | 51 +-
.../{spark-write-hive-sync => conf}/hive-site.xml | 0
.../hudi-defaults.conf} | 8 +-
.../spark-defaults.conf | 8 +-
packaging/bundle-validation/kafka/config-sink.json | 20 +
.../validate.sh => kafka/consume.sh} | 26 +-
.../validate.sh => kafka/produce.sh} | 20 +-
.../validate.scala => spark_hadoop_mr/write.scala} | 10 +-
.../utilities/hoodieapp.properties | 23 +
.../bundle-validation/utilities/validate.scala | 21 +-
packaging/bundle-validation/validate.sh | 195 +++++
packaging/hudi-flink-bundle/pom.xml | 4 +-
packaging/hudi-hadoop-mr-bundle/pom.xml | 22 +-
packaging/hudi-integ-test-bundle/pom.xml | 2 +-
packaging/hudi-kafka-connect-bundle/pom.xml | 8 +-
packaging/hudi-presto-bundle/pom.xml | 1 +
packaging/hudi-spark-bundle/pom.xml | 5 +-
packaging/hudi-timeline-server-bundle/pom.xml | 2 +-
packaging/hudi-utilities-bundle/pom.xml | 1 +
pom.xml | 49 +-
release/release_guide.md | 680 +++++++++++++++
release/release_guide_overview.jpg | Bin 0 -> 363329 bytes
rfc/README.md | 2 +
rfc/rfc-46/rfc-46.md | 168 +++-
rfc/rfc-60/rfc-60.md | 227 +++++
719 files changed, 20380 insertions(+), 6724 deletions(-)
copy docker/demo/config/test-suite/{spark-immutable-dataset.yaml =>
spark-immutable-to-mutable.yaml} (82%)
create mode 100644
hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestArchiveCommand.java
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/util/QueueBasedExecutorFactory.java
copy
hudi-client/{hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkDeletePartitionCommitActionExecutor.java
=>
hudi-flink-client/src/main/java/org/apache/hudi/table/action/commit/FlinkDeletePartitionCommitActionExecutor.java}
(74%)
delete mode 100644
hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/action/commit/FlinkMergeHelper.java
create mode 100644
hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/WriteStatMerger.java
delete mode 100644
hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/JavaMergeHelper.java
rename
hudi-client/hudi-java-client/src/test/java/org/apache/hudi/{table/action/commit/TestHoodieConcatHandle.java
=> client/TestHoodieJavaWriteClientInsert.java} (89%)
create mode 100644
hudi-client/hudi-java-client/src/test/java/org/apache/hudi/table/action/commit/TestSchemaEvolutionClient.java
create mode 100644
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/ExecutionStrategyUtil.java
create mode 100644
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/PartitionPathFormatterBase.java
create mode 100644
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/StringPartitionPathFormatter.java
create mode 100644
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/UTF8StringPartitionPathFormatter.java
delete mode 100644
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/util/DataTypeUtils.java
create mode 100644
hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/HoodieDataTypeUtils.scala
create mode 100644
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/execution/TestDisruptorExecutionInSpark.java
create mode 100644
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/execution/TestDisruptorMessageQueue.java
create mode 100644
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/clean/TestCleanerInsertAndCleanByVersions.java
create mode 100644
hudi-common/src/main/java/org/apache/hudi/avro/AvroSchemaCompatibility.java
copy
hudi-common/src/main/java/org/apache/hudi/common/util/{jvm/OpenJ9MemoryLayoutSpecification32bit.java
=> collection/CloseableMappingIterator.java} (65%)
copy
hudi-common/src/main/java/org/apache/hudi/common/util/{jvm/OpenJ9MemoryLayoutSpecification32bit.java
=> collection/MappingIterator.java} (62%)
create mode 100644
hudi-common/src/main/java/org/apache/hudi/common/util/queue/BaseHoodieQueueBasedExecutor.java
create mode 100644
hudi-common/src/main/java/org/apache/hudi/common/util/queue/DisruptorExecutor.java
create mode 100644
hudi-common/src/main/java/org/apache/hudi/common/util/queue/DisruptorMessageQueue.java
create mode 100644
hudi-common/src/main/java/org/apache/hudi/common/util/queue/DisruptorWaitStrategyType.java
copy
hudi-common/src/main/java/org/apache/hudi/common/util/queue/{BoundedInMemoryQueueConsumer.java
=> ExecutorType.java} (50%)
copy
hudi-common/src/main/java/org/apache/hudi/common/util/queue/{BoundedInMemoryQueueProducer.java
=> HoodieConsumer.java} (69%)
copy
hudi-common/src/main/java/org/apache/hudi/common/util/queue/{BoundedInMemoryQueueProducer.java
=> HoodieExecutor.java} (60%)
rename
hudi-common/src/main/java/org/apache/hudi/common/util/queue/{BoundedInMemoryQueueConsumer.java
=> HoodieMessageQueue.java} (50%)
rename
hudi-common/src/main/java/org/apache/hudi/common/util/queue/{BoundedInMemoryQueueProducer.java
=> HoodieProducer.java} (74%)
create mode 100644
hudi-common/src/main/java/org/apache/hudi/common/util/queue/WaitStrategyFactory.java
copy
hudi-common/src/main/java/org/apache/hudi/exception/{HoodieSecondaryIndexException.java
=> HoodieDebeziumAvroPayloadException.java} (77%)
create mode 100644
hudi-common/src/test/java/org/apache/hudi/avro/TestAvroSchemaUtils.java
delete mode 100644
hudi-common/src/test/java/org/apache/hudi/common/testutils/minicluster/MiniClusterUtil.java
create mode 100644
hudi-common/src/test/java/org/apache/hudi/metadata/TestHoodieMetadataPayload.java
rename
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/compact/{CompactFunction.java
=> CompactOperator.java} (76%)
create mode 100644
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/CastMap.java
create mode 100644
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/InternalSchemaManager.java
create mode 100644
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/ParquetSplitRecordIterator.java
create mode 100644
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/RecordIterators.java
create mode 100644
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/SchemaEvolvedRecordIterator.java
create mode 100644
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/FlinkWriteClients.java
copy
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/keygen/TestKeyGenUtils.java
=>
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/RowDataCastProjection.java
(50%)
create mode 100644
hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/TestIncrementalInputSplits.java
create mode 100644
hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestSchemaEvolution.java
create mode 100644
hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/format/TestCastMap.java
create mode 100644
hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/FlinkMiniCluster.java
create mode 100644
hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestClusteringUtil.java
create mode 100644
hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/SchemaEvolutionContext.java
create mode 100644
hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HiveAvroSerializer.java
create mode 100644
hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/utils/TestHiveAvroSerializer.java
rename
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/{LogFileIterator.scala
=> Iterators.scala} (95%)
rename
hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/HoodieJavaClientTestBase.java
=>
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/util/CachingIterator.scala
(50%)
create mode 100644
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/RepairHoodieTableCommand.scala
create mode 100644
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/streaming/HoodieMetadataLog.scala
copy
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/{hive/HiveClientUtils.scala
=> hudi/streaming/HoodieOffsetRangeLimit.scala} (53%)
create mode 100644
hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/CopyToTempView.scala
create mode 100644
hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/HelpProcedure.scala
create mode 100644
hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowCommitExtraMetadataProcedure.scala
create mode 100644
hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowTablePropertiesProcedure.scala
copy hudi-common/src/main/java/org/apache/hudi/common/util/MapUtils.java =>
hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/SparkSQLCoreFlow.java
(68%)
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestHiveTableSchemaEvolution.java
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestBasicSchemaEvolution.scala
rename
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/{TestMORDataSourceWithBucket.scala
=> TestMORDataSourceWithBucketIndex.scala} (99%)
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSparkSqlCoreFlow.scala
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/execution/benchmark/BoundInMemoryExecutorBenchmark.scala
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestRepairTable.scala
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestCopyToTempViewProcedure.scala
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestHelpProcedure.scala
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestShowTablePropertiesProcedure.scala
create mode 100644
hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncToolArgs.java
copy
hudi-common/src/main/java/org/apache/hudi/common/util/jvm/HotSpotMemoryLayoutSpecification64bit.java
=>
hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/SimpleSchemaProvider.java
(62%)
create mode 100644
hudi-utilities/src/test/resources/delta-streamer-config/source_evolved_post_processed.avsc
copy packaging/bundle-validation/{spark-write-hive-sync/spark-defaults.conf =>
Dockerfile} (57%)
rename packaging/bundle-validation/{spark-write-hive-sync/Dockerfile =>
Dockerfile-base} (64%)
rename packaging/bundle-validation/{spark-write-hive-sync => }/ci_run.sh (50%)
rename packaging/bundle-validation/{spark-write-hive-sync =>
conf}/hive-site.xml (100%)
copy packaging/bundle-validation/{spark-write-hive-sync/spark-defaults.conf =>
conf/hudi-defaults.conf} (78%)
rename packaging/bundle-validation/{spark-write-hive-sync =>
conf}/spark-defaults.conf (71%)
create mode 100644 packaging/bundle-validation/kafka/config-sink.json
copy packaging/bundle-validation/{spark-write-hive-sync/validate.sh =>
kafka/consume.sh} (60%)
rename packaging/bundle-validation/{spark-write-hive-sync/validate.sh =>
kafka/produce.sh} (57%)
rename packaging/bundle-validation/{spark-write-hive-sync/validate.scala =>
spark_hadoop_mr/write.scala} (87%)
create mode 100644 packaging/bundle-validation/utilities/hoodieapp.properties
copy
hudi-common/src/main/java/org/apache/hudi/exception/HoodieCompactException.java
=> packaging/bundle-validation/utilities/validate.scala (62%)
create mode 100755 packaging/bundle-validation/validate.sh
create mode 100644 release/release_guide.md
create mode 100644 release/release_guide_overview.jpg
create mode 100644 rfc/rfc-60/rfc-60.md
