This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a change to branch HUDI-8990-V2
in repository https://gitbox.apache.org/repos/asf/hudi.git
from f03fc80498c code review
add 8c58ef5d78a [HUDI-7037] Fix colstats reading for Decimal field (#12993)
add 07cec68850a [HUDI-9212] Fix metadata compaction failure post downgrade
(#13022)
add 5e6795d0534 [MINOR] Improve CI to parallelize tests more and reduce
completion time (#13031)
add 4f63cc06e74 [MINOR] Fix flaky test testCreateNewInstantTimes (#13032)
add fb8d27bc4e1 [HUDI-9216] Ignore validation of empty databaseName in
HoodieTableMetaClient (#13033)
add 0185ede3beb [MINOR] Follow up HUDI-8803, optimize vectorized reader by
cache 'batchIdxField' (#13023)
add 550c39b9383 [HUDI-9088] Fix unnecessary scanning of target table in
MERGE INTO on Spark (#12934)
add aef7a33fb54 [MINOR] Use configutils to handle defaults for all zk lock
provider configs (#13039)
add b6957f020d0 [HUDI-9205] Refactor Flink tests to avoid sleeping for
data results (#13027)
add e794f091d07 [HUDI-9227] Fix bulk insert overwrite after a failed
insert overwrite (#13041)
add 07328c76019 [HUDI-9221] Fix tests in TestHoodieBackedMetadata after
new default indexes enabled (#13035)
add eae549d24bd [HUDI-9233] Handle empty table edge case for col stats
initialization (#13045)
add badee10d290 [HUDI-9167] Remove redundant classes in
hudi-utilities-slim-bundle (#12962)
add b0ef89aaa02 [HUDI-9218] Implement HoodieRecordMerger for Flink
HoodieRecord (#13040)
add 906747a5bfb [HUDI-9042] Support `TimestampType.DATE_STRING` by
`RowDataKeyGen` (#13037)
add 12fa7d27084 [HUDI-9238] Fix bug with HudiIncrSource when source is
table version 6 (#13048)
add baea182cfc7 [HUDI-9239] Fix the bug of Spark Cache not releasing
cleanly (#13050)
add 5a5117c98d8 [HUDI-9133] Fallback to legacy schema fetching in case of
IllegalAccessError from schema registry client (#13030)
add bec60f2f6ae [HUDI-9240] Bump spark 35 version to 3.5.5 (#13053)
add 9cd1b610ffe [HUDI-9228] Support rowdata writing for MOR table with
consistent hasing (#13043)
add ee50661db20 [HUDI-9231] Show complete DAG for one single query in
spark web ui while inserting into hudi table (#13044)
add 666eb312acd [HUDI-9236] Handle markers for log files in table version
6 (#13007)
add 0d2877631a2 [HUDI-8409] Fixing merge mode config during upgrade and
downgrade from version 7 to 8 and back (#13046)
add 5e713cf95fd [HUDI-8635] Support numWrites metric for compaction
(#13047)
add d54f3432beb Merge branch 'master' into HUDI-8990-V2
No new revisions were added by this update.
Summary of changes:
.github/workflows/bot.yml | 380 ++++++++++++++++++--
azure-pipelines-20230430.yml | 137 ++++++--
.../apache/hudi/client/BaseHoodieWriteClient.java | 8 +-
.../apache/hudi/client/CommitMetadataResolver.java | 48 +++
.../hudi/client/CommitMetadataResolverFactory.java | 43 +++
.../client/MarkerBasedCommitMetadataResolver.java | 257 ++++++++++++++
.../NoOpCommitMetadataResolver.java} | 33 +-
.../lock/BaseZookeeperBasedLockProvider.java | 27 +-
.../java/org/apache/hudi/io/HoodieWriteHandle.java | 5 +-
.../keygen/TimestampBasedAvroKeyGenerator.java | 2 +-
.../metadata/HoodieBackedTableMetadataWriter.java | 5 +-
.../action/commit/BaseCommitActionExecutor.java | 8 +-
.../rollback/BaseRollbackActionExecutor.java | 11 +-
.../rollback/ListingBasedRollbackStrategy.java | 39 ++-
.../rollback/MarkerBasedRollbackStrategy.java | 49 ++-
...BaseRollbackHelper.java => RollbackHelper.java} | 13 +-
.../action/rollback/RollbackHelperFactory.java} | 27 +-
.../table/action/rollback/RollbackHelperV1.java | 389 +++++++++++++++++++++
.../marker/AppendMarkerHandler.java} | 23 +-
.../hudi/table/marker/DirectWriteMarkers.java | 58 +--
.../hudi/table/marker/DirectWriteMarkersV1.java | 103 ++++++
.../marker/TimelineServerBasedWriteMarkers.java | 9 +-
.../marker/TimelineServerBasedWriteMarkersV1.java | 74 ++++
.../org/apache/hudi/table/marker/WriteMarkers.java | 19 +
.../hudi/table/marker/WriteMarkersFactory.java | 31 +-
.../hudi/table/upgrade/DowngradeHandler.java | 7 +-
.../upgrade/EightToSevenDowngradeHandler.java | 43 ++-
.../table/upgrade/FiveToFourDowngradeHandler.java | 6 +-
.../table/upgrade/FourToThreeDowngradeHandler.java | 6 +-
.../table/upgrade/OneToZeroDowngradeHandler.java | 5 +-
.../table/upgrade/SevenToEightUpgradeHandler.java | 68 +++-
.../table/upgrade/SevenToSixDowngradeHandler.java | 6 +-
.../table/upgrade/SixToFiveDowngradeHandler.java | 7 +-
.../table/upgrade/ThreeToTwoDowngradeHandler.java | 6 +-
.../table/upgrade/TwoToOneDowngradeHandler.java | 5 +-
.../hudi/table/upgrade/UpgradeDowngrade.java | 57 ++-
.../table/upgrade/ZeroToOneUpgradeHandler.java | 4 +-
.../client/utils/TestCommitMetadataResolver.java | 242 +++++++++++++
.../rollback/TestMarkerBasedRollbackStrategy.java | 17 +-
...RollbackHelper.java => TestRollbackHelper.java} | 6 +-
.../hudi/table/marker/TestWriteMarkersFactory.java | 71 ++--
.../upgrade/TestEightToSevenDowngradeHandler.java | 45 ++-
.../upgrade/TestSevenToEightUpgradeHandler.java | 60 +++-
.../client/model/CommitTimeFlinkRecordMerger.java} | 13 +-
.../client/model/EventTimeFlinkRecordMerger.java | 57 ++-
.../hudi/client/model/HoodieFlinkRecord.java | 2 +-
.../client/model/HoodieFlinkRecordMerger.java} | 9 +-
.../hudi/merge/TestHoodieFlinkRecordMerger.java | 180 ++++++++++
.../org/apache/hudi/data/HoodieJavaPairRDD.java | 8 +
.../apache/hudi/data/TestHoodieJavaPairRDD.java | 110 ++++++
.../TestMarkerBasedRollbackStrategy.java | 27 +-
.../hudi/table/marker/TestDirectWriteMarkers.java | 15 +-
.../table/marker/TestDirectWriteMarkersV1.java | 35 +-
.../TestTimelineServerBasedWriteMarkers.java | 14 +-
.../TestTimelineServerBasedWriteMarkersV1.java | 52 +++
.../hudi/table/marker/TestWriteMarkersBase.java | 61 +++-
.../hudi/common/data/HoodieListPairData.java | 27 ++
.../apache/hudi/common/data/HoodiePairData.java | 12 +
.../java/org/apache/hudi/common/model/IOType.java | 1 +
.../hudi/common/table/HoodieTableMetaClient.java | 2 +-
.../table/log/BaseHoodieLogRecordReader.java | 1 +
.../hudi/common/table/marker/MarkerOperation.java | 2 +
.../common/table/read/FileGroupRecordBuffer.java | 10 +
.../table/read/HoodieFileGroupRecordBuffer.java | 5 +
.../read/PositionBasedFileGroupRecordBuffer.java | 18 +-
.../hudi/metadata/HoodieTableMetadataUtil.java | 39 +--
.../hudi/metadata/MetadataPartitionType.java | 5 +-
.../common/data/TestHoodieListDataPairData.java | 34 ++
.../hudi/common/testutils/FileCreateUtils.java | 56 +++
.../apache/hudi/configuration/OptionsResolver.java | 3 +-
.../hudi/sink/RowDataStreamWriteFunction.java | 2 +-
.../org/apache/hudi/sink/StreamWriteOperator.java | 2 +-
.../sink/bucket/BucketStreamWriteOperator.java | 18 +-
...RowDataConsistentBucketStreamWriteFunction.java | 99 ++++++
.../org/apache/hudi/sink/bulk/RowDataKeyGen.java | 5 +
.../FlinkConsistentBucketUpdateStrategy.java | 8 +-
... => RowDataConsistentBucketUpdateStrategy.java} | 89 +++--
.../apache/hudi/sink/ITTestDataStreamWrite.java | 56 +--
.../apache/hudi/sink/bulk/TestRowDataKeyGen.java | 17 +
.../utils/BucketStreamWriteFunctionWrapper.java | 4 +-
...ConsistentBucketStreamWriteFunctionWrapper.java | 12 +-
.../sink/utils/StreamWriteFunctionWrapper.java | 4 +-
.../org/apache/hudi/sink/utils/TestWriteBase.java | 2 +-
.../apache/hudi/table/ITTestHoodieDataSource.java | 172 ++++-----
.../apache/hudi/table/ITTestSchemaEvolution.java | 4 -
.../org/apache/hudi/utils/FlinkMiniCluster.java | 11 +
.../org/apache/hudi/utils/TestConfigurations.java | 15 +-
.../utils/factory/CollectSinkTableFactory.java | 62 +++-
.../common/table/TestHoodieTableMetaClient.java | 4 +-
.../hudi/common/testutils/HoodieTestTable.java | 41 ++-
.../hudi/hadoop/TestHoodieHFileInputFormat.java | 4 +-
.../hudi/hadoop/TestHoodieParquetInputFormat.java | 4 +-
.../org/apache/hudi/storage/HoodieStorage.java | 5 +-
.../org/apache/hudi/storage/StorageSchemes.java | 15 +
.../BaseDatasetBulkInsertCommitActionExecutor.java | 1 +
.../org/apache/hudi/ColumnStatsIndexSupport.scala | 15 +-
.../scala/org/apache/hudi/DataSourceOptions.scala | 2 +-
.../org/apache/hudi/HoodieSparkSqlWriter.scala | 10 +-
.../scala/org/apache/hudi/HoodieWriterUtils.scala | 46 ++-
.../org/apache/hudi/TestHoodieWriterUtils.java | 46 +++
.../spark/sql/hudi/analysis/HoodieAnalysis.scala | 15 +-
.../command/CreateHoodieTableAsSelectCommand.scala | 12 +-
.../command/InsertIntoHoodieTableCommand.scala | 25 +-
.../hudi/command/MergeIntoHoodieTableCommand.scala | 50 +--
.../functional/TestHoodieBackedMetadata.java | 139 ++++++--
...e.java => HoodieSparkFunctionalTestSuiteA.java} | 3 +-
...e.java => HoodieSparkFunctionalTestSuiteB.java} | 5 +-
.../functional/TestNewHoodieParquetFileFormat.java | 2 +-
.../hudi/table/TestHoodieMergeOnReadTable.java | 4 +
.../table/action/compact/TestHoodieCompactor.java | 22 +-
...arkCopyOnWriteTableRollbackTableVersionSix.java | 63 ++++
.../TestHoodieSparkMergeOnReadTableCompaction.java | 227 +++++++++++-
.../TestHoodieSparkMergeOnReadTableRollback.java | 233 +++++++++---
.../table/functional/TestHoodieSparkRollback.java | 307 ++++++++++++++++
.../hudi/functional/ColumnStatIndexTestBase.scala | 66 +++-
.../hudi/functional/TestColumnStatsIndex.scala | 43 ++-
.../hudi/functional/TestPartitionStatsIndex.scala | 2 +-
.../hudi/functional/TestSevenToEightUpgrade.scala | 220 +++++++++++-
.../sql/hudi/common/HoodieSparkSqlTestBase.scala | 16 +-
.../spark/sql/hudi/dml/TestInsertTable.scala | 45 ++-
.../spark/sql/hudi/dml/TestMergeIntoTable.scala | 208 ++++++-----
.../hudi/feature/index/TestSecondaryIndex.scala | 4 +-
.../Spark3HoodieVectorizedParquetRecordReader.java | 8 +-
.../hudi/timeline/service/RequestHandler.java | 8 +
.../timeline/service/handlers/MarkerHandler.java | 10 +
.../utilities/schema/SchemaRegistryProvider.java | 62 ++++
.../hudi/utilities/sources/HoodieIncrSource.java | 6 +-
.../schema/TestSchemaRegistryProvider.java | 24 ++
.../utilities/sources/TestHoodieIncrSource.java | 174 ++++-----
packaging/bundle-validation/flink/insert.sql | 2 +
packaging/bundle-validation/validate.sh | 2 +-
packaging/hudi-spark-bundle/pom.xml | 9 -
packaging/hudi-utilities-slim-bundle/pom.xml | 37 --
pom.xml | 63 +++-
rfc/README.md | 2 +-
135 files changed, 5152 insertions(+), 998 deletions(-)
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/CommitMetadataResolver.java
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/CommitMetadataResolverFactory.java
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/MarkerBasedCommitMetadataResolver.java
copy
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/{table/upgrade/SupportsUpgradeDowngrade.java
=> client/NoOpCommitMetadataResolver.java} (55%)
rename
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/{BaseRollbackHelper.java
=> RollbackHelper.java} (96%)
copy
hudi-client/{hudi-spark-client/src/test/java/org/apache/hudi/index/hbase/TestHBaseIndexUsage.java
=>
hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/RollbackHelperFactory.java}
(60%)
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/RollbackHelperV1.java
copy
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/{client/timeline/HoodieTimelineArchiver.java
=> table/marker/AppendMarkerHandler.java} (58%)
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/DirectWriteMarkersV1.java
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/TimelineServerBasedWriteMarkersV1.java
create mode 100644
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/utils/TestCommitMetadataResolver.java
rename
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/rollback/{TestBaseRollbackHelper.java
=> TestRollbackHelper.java} (98%)
copy
hudi-client/{hudi-spark-client/src/main/java/org/apache/hudi/OverwriteWithLatestSparkRecordMerger.java
=>
hudi-flink-client/src/main/java/org/apache/hudi/client/model/CommitTimeFlinkRecordMerger.java}
(75%)
copy
hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/DefaultHiveRecordMerger.java
=>
hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/model/EventTimeFlinkRecordMerger.java
(59%)
copy
hudi-client/{hudi-spark-client/src/main/java/org/apache/hudi/HoodieSparkRecordMerger.java
=>
hudi-flink-client/src/main/java/org/apache/hudi/client/model/HoodieFlinkRecordMerger.java}
(81%)
create mode 100644
hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/merge/TestHoodieFlinkRecordMerger.java
create mode 100644
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/data/TestHoodieJavaPairRDD.java
copy
hudi-io/src/main/java/org/apache/hudi/io/ByteArraySeekableDataInputStream.java
=>
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/marker/TestDirectWriteMarkersV1.java
(58%)
create mode 100644
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/marker/TestTimelineServerBasedWriteMarkersV1.java
create mode 100644
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/bucket/RowDataConsistentBucketStreamWriteFunction.java
copy
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/clustering/update/strategy/{FlinkConsistentBucketUpdateStrategy.java
=> RowDataConsistentBucketUpdateStrategy.java} (62%)
create mode 100644
hudi-spark-datasource/hudi-spark-common/src/test/java/org/apache/hudi/TestHoodieWriterUtils.java
copy
hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/{HoodieSparkFunctionalTestSuite.java
=> HoodieSparkFunctionalTestSuiteA.java} (95%)
rename
hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/{HoodieSparkFunctionalTestSuite.java
=> HoodieSparkFunctionalTestSuiteB.java} (93%)
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/functional/TestHoodieSparkCopyOnWriteTableRollbackTableVersionSix.java
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/functional/TestHoodieSparkRollback.java
