This is an automated email from the ASF dual-hosted git repository.
yihua pushed a change to branch release-0.13.0
in repository https://gitbox.apache.org/repos/asf/hudi.git
from ca72c96bd2e [MINOR] Fix validate_staged_release.sh (#7780)
new 7f67e6abd82 [HUDI-5592] Fixing some of the flaky tests in CI (#7720)
new 9ac8765a3a7 [HUDI-5534] Optimizing Bloom Index lookup when using Bloom
Filters from Metadata Table (#7642)
new c570e6bd271 [HUDI-5627] Improve the usability of Hudi CLI bundle
(#7762)
new df05628334c [HUDI-5623] Increase default time to wait between retries
by lock provider client (#7758)
new ee7764b9749 [HUDI-5023] Switching default Write Executor type to
`SIMPLE` (#7476)
new 01acf99e279 [HUDI-5630] Fixing flaky parquet projection tests (#7768)
new 281b29eb7b3 [HUDI-5629] Clean CDC log files for enable/disable
scenario (#7767)
new cf52dc23db8 [HUDI-5626] Rename CDC logging mode options (#7760)
new 44cd68bdbd6 [MINOR] Skip docs generation for table service manager
(#7773)
new 1b592c2be81 Revert "[MINOR] Add `hudi-platform-service` and
`hudi-metaserver-server-bundle` to root pom (#7774)" (#7782)
new 9495918c633 [HUDI-5631] Improve defaults of early conflict detection
configs (#7770)
new 01d56a6f3f9 [HUDI-5624] Fix HoodieAvroRecordMerger to use new
precombine API (#7759)
new 0d7d4c4624b [HUDI-5628] Fixing log record reader scan V2 config name
(#7764)
new be1aa615a3d [HUDI-5638] Fix metaserver test setup (#7785)
new 53c1d5dd6a1 [HUDI-5640] Add missing profiles in
`deploy_staging_jars.sh` (#7784)
new 8ccb80cb052 [HUDI-5639] Fixing stream identifier for single writer
with spark streaming ingest (#7783)
new 2ce0869ef34 [MINOR] Fix `HoodieCDCRDD` setting flag `usesVirtualKeys`
(#7777)
new 48977a61e26 [HUDI-5503] Optimize flink table factory option check
(#7608)
new 5765e1b9b96 [MINOR] Cleaning up recently introduced configs (#7772)
The 19 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
azure-pipelines.yml | 2 +-
.../hudi/cli/commands/HoodieLogFileCommand.java | 2 +-
.../cli/commands/TestHoodieLogFileCommand.java | 2 +-
.../client/embedded/EmbeddedTimelineService.java | 4 +-
.../apache/hudi/config/HoodieCompactionConfig.java | 11 +-
.../org/apache/hudi/config/HoodieLockConfig.java | 2 +-
.../org/apache/hudi/config/HoodieWriteConfig.java | 58 +++---
.../hudi/execution/HoodieLazyInsertIterable.java | 30 +--
.../org/apache/hudi/index/HoodieIndexUtils.java | 5 +-
.../index/bloom/BaseHoodieBloomIndexHelper.java | 5 +-
.../apache/hudi/index/bloom/HoodieBloomIndex.java | 77 +++----
...ion.java => HoodieBloomIndexCheckFunction.java} | 59 +++---
.../hudi/index/bloom/HoodieGlobalBloomIndex.java | 14 +-
.../bloom/ListBasedHoodieBloomIndexHelper.java | 26 +--
.../java/org/apache/hudi/io/HoodieCDCLogger.java | 7 +-
.../metadata/HoodieBackedTableMetadataWriter.java | 2 +-
.../hudi/table/action/clean/CleanPlanner.java | 26 +--
.../action/compact/CompactionExecutionHelper.java | 4 +-
.../hudi/table/action/compact/HoodieCompactor.java | 2 +-
.../compact/LogCompactionExecutionHelper.java | 2 +-
.../HoodieLogCompactionPlanGenerator.java | 2 +-
.../java/org/apache/hudi/util/ExecutorFactory.java | 24 +--
.../hudi/execution/FlinkLazyInsertIterable.java | 2 +-
.../index/bloom/TestFlinkHoodieBloomIndex.java | 8 +-
.../hudi/execution/JavaLazyInsertIterable.java | 2 +-
.../MultipleSparkJobExecutionStrategy.java | 2 +-
.../org/apache/hudi/data/HoodieJavaPairRDD.java | 5 +
.../java/org/apache/hudi/data/HoodieJavaRDD.java | 14 +-
.../hudi/execution/SparkLazyInsertIterable.java | 2 +-
.../bloom/BucketizedBloomCheckPartitioner.java | 18 +-
.../bloom/HoodieBloomFilterProbingResult.java | 16 +-
.../index/bloom/HoodieBloomIndexCheckFunction.java | 120 -----------
.../index/bloom/HoodieFileProbingFunction.java | 143 +++++++++++++
.../HoodieMetadataBloomFilterProbingFunction.java | 157 ++++++++++++++
.../HoodieMetadataBloomIndexCheckFunction.java | 154 --------------
.../index/bloom/SparkHoodieBloomIndexHelper.java | 229 ++++++++++++++++++---
.../org/apache/spark/sql/hudi/SparkAdapter.scala | 4 +-
.../TestHoodieClientOnMergeOnReadStorage.java | 75 +++----
.../TestBoundedInMemoryExecutorInSpark.java | 28 ++-
.../hudi/execution/TestBoundedInMemoryQueue.java | 23 ++-
.../execution/TestDisruptorExecutionInSpark.java | 52 +++--
.../hudi/execution/TestDisruptorMessageQueue.java | 30 +--
.../hudi/execution/TestSimpleExecutionInSpark.java | 51 ++---
.../bloom/TestBucketizedBloomCheckPartitioner.java | 43 ++--
.../hudi/index/bloom/TestHoodieBloomIndex.java | 10 +-
.../index/bloom/TestHoodieGlobalBloomIndex.java | 10 +-
.../hudi/common/config/HoodieMetadataConfig.java | 17 +-
.../config/HoodieTableServiceManagerConfig.java | 5 +-
.../org/apache/hudi/common/data/HoodieData.java | 17 +-
.../apache/hudi/common/data/HoodieListData.java | 10 +
.../hudi/common/data/HoodieListPairData.java | 23 +++
.../apache/hudi/common/data/HoodiePairData.java | 11 +-
...alizableSupplier.java => ThrowingConsumer.java} | 14 +-
.../hudi/common/model/HoodieAvroIndexedRecord.java | 2 +-
.../hudi/common/model/HoodieAvroRecordMerger.java | 13 +-
.../hudi/common/table/HoodieTableConfig.java | 19 +-
.../hudi/common/table/HoodieTableMetaClient.java | 4 +
.../hudi/common/table/cdc/HoodieCDCExtractor.java | 6 +-
.../hudi/common/table/cdc/HoodieCDCInferCase.java | 4 +-
.../cdc/HoodieCDCSupplementalLoggingMode.java | 37 +---
.../hudi/common/table/cdc/HoodieCDCUtils.java | 14 +-
.../table/log/AbstractHoodieLogRecordReader.java | 10 +-
.../table/log/HoodieMergedLogRecordScanner.java | 12 +-
.../table/log/HoodieUnMergedLogRecordScanner.java | 12 +-
.../common/util/collection/FlatteningIterator.java | 55 +++++
.../common/util/collection/MappingIterator.java | 4 +-
.../common/util/queue/BoundedInMemoryExecutor.java | 2 +-
.../hudi/common/util/queue/DisruptorExecutor.java | 19 +-
.../common/util/queue/DisruptorMessageQueue.java | 18 +-
.../hudi/common/util/queue/ExecutorType.java | 15 +-
...mpleHoodieExecutor.java => SimpleExecutor.java} | 51 +++--
.../common/util/queue/WaitStrategyFactory.java | 8 +-
.../apache/hudi/metadata/BaseTableMetadata.java | 53 +++--
.../hudi/metadata/HoodieBackedTableMetadata.java | 36 ++--
.../metadata/HoodieMetadataFileSystemView.java | 5 +-
.../metadata/HoodieMetadataLogRecordReader.java | 4 +-
.../main/java/org/apache/hudi/util/Transient.java | 108 ++++++++++
.../common/functional/TestHoodieLogFormat.java | 76 +++----
.../common/testutils/HoodieTestDataGenerator.java | 9 +-
.../hudi/common/util/collection/TestIterators.java | 48 +++++
.../apache/hudi/configuration/FlinkOptions.java | 12 +-
.../apache/hudi/configuration/OptionsResolver.java | 2 +-
.../org/apache/hudi/sink/bulk/RowDataKeyGen.java | 22 +-
.../org/apache/hudi/table/HoodieTableFactory.java | 51 ++++-
.../hudi/table/catalog/HoodieHiveCatalog.java | 7 +-
.../hudi/table/format/cdc/CdcInputFormat.java | 6 +-
.../java/org/apache/hudi/util/StreamerUtil.java | 11 +-
.../apache/hudi/sink/bulk/TestRowDataKeyGen.java | 20 ++
.../apache/hudi/table/ITTestHoodieDataSource.java | 2 +-
.../apache/hudi/table/TestHoodieTableFactory.java | 92 ++++++++-
.../apache/hudi/table/format/TestInputFormat.java | 4 +-
.../realtime/RealtimeCompactedRecordReader.java | 2 +-
.../reader/DFSHoodieDatasetInputReader.java | 2 +-
.../hudi-metaserver/hudi-metaserver-client/pom.xml | 15 +-
.../hudi-metaserver/hudi-metaserver-server/pom.xml | 2 +-
hudi-platform-service/hudi-metaserver/pom.xml | 51 ++++-
.../scala/org/apache/hudi/DataSourceOptions.scala | 15 +-
.../main/scala/org/apache/hudi/DefaultSource.scala | 20 +-
.../org/apache/hudi/HoodieSparkSqlWriter.scala | 21 +-
.../org/apache/hudi/HoodieStreamingSink.scala | 20 +-
.../scala/org/apache/hudi/cdc/CDCRelation.scala | 21 +-
.../scala/org/apache/hudi/cdc/HoodieCDCRDD.scala | 38 ++--
.../hudi/command/MergeIntoHoodieTableCommand.scala | 3 +-
.../functional/TestParquetColumnProjection.scala | 17 +-
.../functional/TestPartialUpdateAvroPayload.scala | 125 +++++++++++
.../hudi/functional/TestStructuredStreaming.scala | 30 ++-
.../hudi/functional/cdc/HoodieCDCTestBase.scala | 32 +--
.../functional/cdc/TestCDCDataFrameSuite.scala | 51 +++--
.../functional/cdc/TestCDCStreamingSuite.scala | 16 +-
.../apache/spark/sql/hudi/TestCDCForSparkSQL.scala | 11 +-
.../TestHoodiePruneFileSourcePartitions.scala | 4 +-
.../apache/spark/sql/adapter/Spark2Adapter.scala | 4 +-
.../hudi/timeline/service/TimelineService.java | 8 +-
packaging/hudi-cli-bundle/hudi-cli-with-bundle.sh | 23 ++-
pom.xml | 22 +-
scripts/release/deploy_staging_jars.sh | 17 +-
116 files changed, 1990 insertions(+), 1114 deletions(-)
rename
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bloom/{HoodieBaseBloomIndexCheckFunction.java
=> HoodieBloomIndexCheckFunction.java} (61%)
copy
hudi-aws/src/main/java/org/apache/hudi/aws/sync/HoodieGlueSyncException.java =>
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/index/bloom/HoodieBloomFilterProbingResult.java
(72%)
delete mode 100644
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/index/bloom/HoodieBloomIndexCheckFunction.java
create mode 100644
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/index/bloom/HoodieFileProbingFunction.java
create mode 100644
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/index/bloom/HoodieMetadataBloomFilterProbingFunction.java
delete mode 100644
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/index/bloom/HoodieMetadataBloomIndexCheckFunction.java
copy
hudi-common/src/main/java/org/apache/hudi/common/function/{SerializableSupplier.java
=> ThrowingConsumer.java} (82%)
create mode 100644
hudi-common/src/main/java/org/apache/hudi/common/util/collection/FlatteningIterator.java
rename
hudi-common/src/main/java/org/apache/hudi/common/util/queue/{SimpleHoodieExecutor.java
=> SimpleExecutor.java} (53%)
create mode 100644
hudi-common/src/main/java/org/apache/hudi/util/Transient.java
create mode 100644
hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestIterators.java
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPartialUpdateAvroPayload.scala
