This is an automated email from the ASF dual-hosted git repository.
pwason pushed a change to branch release-0.14.0
in repository https://gitbox.apache.org/repos/asf/hudi.git
from d995bb8262c [HUDI-6763] Optimize collect calls (#9561)
new 655904a6f29 [HUDI-6562] Fixed issue for delete events for
AWSDmsAvroPayload when CDC enabled (#9519)
new 2e7e1b3a7b7 [MINOR] Fix failing schema evolution tests in Flink
versions < 1.17 (#9586)
new d4de4597849 [HUDI-6066] HoodieTableSource supports parquet predicate
push down (#8437)
new 15ecee9674e [MINOR] Update operator name for compact&clustering test
class (#9583)
new 26cc766ded7 [HUDI-6579] Fix streaming write when meta cols dropped
(#9589)
new 4bc41844957 [HUDI-6732] Allow wildcards from Spark-SQL entrypoints for
drop partition DDL (#9491)
new 033a9f80ff9 [HUDI-6813] Support table name for meta sync in bootstrap
(#9600)
new b7a1f80062b [MINOR] Fix ut due to the scala compile ambiguity of
Properties#putAll (#9601)
new 8b273631cfd [MINOR] Catch EntityNotFoundException correctly (#9595)
new 605eb24b226 [HUDI-6808] SkipCompaction Config should not affect the
stream read of the cow table (#9584)
new 629ee75fe5f [HUDI-6812]Fix bootstrap operator null point exception
while lastInstantTime is null (#9599)
new 620ee24b02b [HUDI-6805] Print detailed error message in clustering
(#9577)
new a136369344f [HUDI-6804] Fix hive read schema evolution MOR table
(#9573)
new ed1d7c97d16 [HUDI-6818] Create a database automatically when using the
flink catalog dfs mode (#9592)
new 83cdca8bc5d [HUDI-6766] Fixing mysql debezium data loss (#9475)
new 46c170425a7 [HUDI-6819] Fix logic for throwing exception in
getRecordIndexUpdates. (#9616)
new 135387c3177 [HUDI-6397][HUDI-6759] Fixing misc bugs w/ metadata table
(#9546)
new bca4828bc08 [HUDI-2141] Support flink compaction metrics (#9515)
new ae3d886e991 [HUDI-6736] Fixing rollback completion and commit timeline
files removal (#9521)
new a948fa09158 [HUDI-6833] Add field for tracking log files from failed
commit in rollback metadata (#9653)
new fadde0317fc [HUDI-6820] Close write clients in tests (#9642)
new 688d6c07a21 [HUDI-6820] Fixing CI stability issues (#9661)
new bba95305a07 [HUDI-6758] Fixing deducing spurious log blocks due to
spark retries (#9611)
new 4af3b7eefa6 [HUDI-6831] Add back missing project_id to query statement
in BigQuerySyncTool (#9650)
new f1114af22b5 [HUDI-6835] Adjust spark sql core flow test scenarios
(#9664)
new a808f74ce03 [HUDI-6728] Update BigQuery manifest sync to support
schema evolution (#9482)
new 5b99ed406ca [HUDI-6738] - Apply object filter before checkpoint
batching in GcsEventsHoodieIncrSource (#9538)
new 225c2ab5bd0 [HUDI-6838] Fix file writers to honor bloom filter configs
(#9669)
new 456f6731cc4 [HUDI-6753] Fix parquet inline reading flaky test (#9618)
new 0081f0ab46f [MINOR] Fixing failing tests with BQ sync tests (#9684)
new f33265d3bd8 [MINOR] Add timeout for github check
test-hudi-hadoop-mr-and-hudi-java-client (#9682)
new a03483f09c0 [MINOR] Avoiding to ingest update records to RLI (#9675)
new c1a497059c4 [HUDI-6834] Fixing time travel queries when overlaps with
cleaner and archival time window (#9666)
new 88f744da58c [MINOR] Avoiding warn log for succeeding in first attempt
(#9686)
new da81614a0de [HUDI-6842] Fixing flaky tests for async clustering test
(#9671)
new 5af6d703994 [HUDI-6478] Deduce op as upsert for INSERT INTO (#9665)
new 63a37211384 [HUDI-6724] - Defaulting previous Instant time to init
time to enable full read of initial commit (#9473)
The 37 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/bot.yml | 34 +-
azure-pipelines-20230430.yml | 2 +
.../hudi/aws/sync/AWSGlueCatalogSyncClient.java | 21 +-
.../hudi/cli/commands/TestRestoresCommand.java | 24 +-
.../hudi/cli/integ/ITTestClusteringCommand.java | 8 +-
.../hudi/cli/integ/ITTestCompactionCommand.java | 9 +-
.../hudi/client/BaseHoodieTableServiceClient.java | 57 --
.../org/apache/hudi/config/HoodieIndexConfig.java | 63 +-
.../org/apache/hudi/config/HoodieWriteConfig.java | 8 +-
.../org/apache/hudi/io/HoodieAppendHandle.java | 22 +-
.../hudi/io/HoodieMergeHandleWithChangeLog.java | 2 +-
.../metadata/HoodieBackedTableMetadataWriter.java | 17 +-
.../java/org/apache/hudi/table/HoodieTable.java | 6 +-
.../rollback/BaseRollbackActionExecutor.java | 25 +-
.../hudi/table/action/rollback/RollbackUtils.java | 6 +-
.../table/upgrade/SixToFiveDowngradeHandler.java | 9 +-
.../org/apache/hudi/DummyTaskContextSupplier.java | 5 +
.../hudi/client/FlinkTaskContextSupplier.java | 5 +
.../java/org/apache/hudi/io/FlinkAppendHandle.java | 4 +
.../client/common/JavaTaskContextSupplier.java | 6 +
.../hudi/client/TestJavaHoodieBackedMetadata.java | 61 +-
.../TestHoodieJavaClientOnCopyOnWriteStorage.java | 2 +
.../testutils/HoodieJavaClientTestHarness.java | 5 +
.../hudi/client/SparkTaskContextSupplier.java | 6 +
.../hudi/io/storage/row/HoodieRowCreateHandle.java | 4 +
.../org/apache/hudi/table/HoodieSparkTable.java | 3 +-
.../callback/TestHoodieClientInitCallback.java | 61 +-
...DataValidationCheckForLogCompactionActions.java | 3 +-
.../functional/TestHoodieBackedMetadata.java | 91 +--
.../TestHoodieClientOnCopyOnWriteStorage.java | 3 +
.../TestHoodieClientOnMergeOnReadStorage.java | 171 +++---
.../index/hbase/TestSparkHoodieHBaseIndex.java | 300 +++++-----
.../apache/hudi/io/TestHoodieTimelineArchiver.java | 20 +-
.../java/org/apache/hudi/table/TestCleaner.java | 38 ++
.../TestCopyOnWriteRollbackActionExecutor.java | 47 --
.../TestHoodieSparkMergeOnReadTableCompaction.java | 8 +
...dieSparkMergeOnReadTableInsertUpdateDelete.java | 15 +-
.../TestHoodieSparkMergeOnReadTableRollback.java | 57 +-
.../hudi/testutils/HoodieClientTestBase.java | 44 --
.../src/main/avro/HoodieRollbackMetadata.avsc | 13 +-
.../org/apache/hudi/common/HoodieRollbackStat.java | 20 +-
.../hudi/common/config/HoodieStorageConfig.java | 41 ++
.../common/engine/LocalTaskContextSupplier.java | 6 +
.../hudi/common/engine/TaskContextSupplier.java | 5 +
.../model/debezium/MySqlDebeziumAvroPayload.java | 29 +-
.../table/log/AbstractHoodieLogRecordReader.java | 95 +--
.../common/table/log/block/HoodieLogBlock.java | 2 +-
.../table/timeline/TimelineMetadataUtils.java | 2 +-
.../hudi/common/table/timeline/TimelineUtils.java | 30 +
.../hudi/io/storage/HoodieFileWriterFactory.java | 9 +-
.../hudi/metadata/HoodieBackedTableMetadata.java | 1 +
.../hudi/metadata/HoodieTableMetadataUtil.java | 11 +-
.../common/functional/TestHoodieLogFormat.java | 2 +-
.../debezium/TestMySqlDebeziumAvroPayload.java | 6 +
.../hudi/common/table/TestTimelineUtils.java | 3 +-
.../table/view/TestIncrementalFSViewSync.java | 2 +-
.../hudi/common/testutils/HoodieTestTable.java | 8 -
.../hudi/metrics/FlinkCompactionMetrics.java | 106 ++++
.../org/apache/hudi/metrics/FlinkWriteMetrics.java | 111 ++++
.../apache/hudi/metrics/HoodieFlinkMetrics.java | 23 +
.../hudi/sink/bootstrap/BootstrapOperator.java | 4 +-
.../apache/hudi/sink/compact/CompactOperator.java | 16 +
.../hudi/sink/compact/CompactionCommitSink.java | 16 +
.../hudi/sink/compact/CompactionPlanOperator.java | 19 +-
.../apache/hudi/source/ExpressionPredicates.java | 654 +++++++++++++++++++++
.../apache/hudi/source/IncrementalInputSplits.java | 2 +-
.../org/apache/hudi/table/HoodieTableSource.java | 18 +-
.../apache/hudi/table/catalog/HoodieCatalog.java | 10 +
.../apache/hudi/table/format/RecordIterators.java | 60 +-
.../hudi/table/format/cdc/CdcInputFormat.java | 11 +-
.../table/format/cow/CopyOnWriteInputFormat.java | 9 +-
.../table/format/mor/MergeOnReadInputFormat.java | 17 +-
.../sink/TestStreamWriteOperatorCoordinator.java | 9 +-
.../ITTestFlinkConsistentHashingClustering.java | 30 +-
.../sink/cluster/ITTestHoodieFlinkClustering.java | 431 +++++++-------
.../sink/compact/ITTestHoodieFlinkCompactor.java | 193 +++---
.../hudi/sink/utils/CompactFunctionWrapper.java | 11 +-
.../hudi/source/TestExpressionPredicates.java | 167 ++++++
.../apache/hudi/table/ITTestHoodieDataSource.java | 14 +
.../apache/hudi/table/ITTestSchemaEvolution.java | 23 +-
.../apache/hudi/table/TestHoodieTableSource.java | 23 +
.../hudi/table/catalog/TestHoodieCatalog.java | 5 +-
.../apache/hudi/table/format/TestInputFormat.java | 1 +
.../org/apache/hudi/utils/TestClusteringUtil.java | 9 +
.../org/apache/hudi/utils/TestCompactionUtil.java | 23 +-
.../hudi/utils/TestViewStorageProperties.java | 8 +-
.../table/format/cow/ParquetSplitReaderUtil.java | 10 +-
.../reader/ParquetColumnarRowSplitReader.java | 10 +-
.../table/format/cow/ParquetSplitReaderUtil.java | 10 +-
.../reader/ParquetColumnarRowSplitReader.java | 10 +-
.../table/format/cow/ParquetSplitReaderUtil.java | 10 +-
.../reader/ParquetColumnarRowSplitReader.java | 10 +-
.../table/format/cow/ParquetSplitReaderUtil.java | 10 +-
.../reader/ParquetColumnarRowSplitReader.java | 10 +-
.../table/format/cow/ParquetSplitReaderUtil.java | 10 +-
.../reader/ParquetColumnarRowSplitReader.java | 10 +-
hudi-gcp/pom.xml | 13 +
.../hudi/gcp/bigquery/BigQuerySchemaResolver.java | 197 +++++++
.../hudi/gcp/bigquery/BigQuerySyncConfig.java | 3 +-
.../apache/hudi/gcp/bigquery/BigQuerySyncTool.java | 95 ++-
.../gcp/bigquery/HoodieBigQuerySyncClient.java | 52 +-
.../gcp/bigquery/TestBigQuerySchemaResolver.java | 299 ++++++++++
.../hudi/gcp/bigquery/TestBigQuerySyncTool.java | 137 +++++
.../gcp/bigquery/TestHoodieBigQuerySyncClient.java | 120 ++++
.../apache/hudi/hadoop/SchemaEvolutionContext.java | 11 +-
.../apache/hudi/AutoRecordKeyGenerationUtils.scala | 11 +-
.../main/scala/org/apache/hudi/DefaultSource.scala | 36 +-
.../org/apache/hudi/HoodieCreateRecordUtils.scala | 11 +-
.../org/apache/hudi/HoodieSparkSqlWriter.scala | 24 +-
.../scala/org/apache/hudi/HoodieWriterUtils.scala | 31 +-
.../spark/sql/hudi/ProvidesHoodieConfig.scala | 48 +-
.../apache/hudi/cli/BootstrapExecutorUtils.java | 2 +
.../org/apache/spark/sql/hudi/SparkHelpers.scala | 7 +-
.../functional/TestHiveTableSchemaEvolution.java | 159 ++---
.../org/apache/hudi/TestHoodieFileIndex.scala | 1 +
.../hudi/functional/RecordLevelIndexTestBase.scala | 7 +-
.../functional/TestColumnStatsIndexWithSQL.scala | 7 +-
.../hudi/functional/TestMetadataRecordIndex.scala | 6 +-
.../hudi/functional/TestSparkSqlCoreFlow.scala | 160 +++--
...TestStreamSourceReadByStateTransitionTime.scala | 1 +
.../hudi/functional/TestStructuredStreaming.scala | 6 +-
.../hudi/functional/TestTimeTravelQuery.scala | 104 +++-
.../functional/cdc/TestCDCDataFrameSuite.scala | 56 +-
.../sql/hudi/TestAlterTableDropPartition.scala | 37 +-
.../apache/spark/sql/hudi/TestInsertTable.scala | 161 +++--
.../spark/sql/hudi/TestTimeTravelTable.scala | 22 +-
.../org/apache/hudi/sync/adb/AdbSyncConfig.java | 2 +-
.../apache/hudi/sync/common/HoodieSyncClient.java | 4 +
.../hudi/sync/common/util/ManifestFileWriter.java | 28 +-
.../sync/common/util/TestManifestFileWriter.java | 8 +-
.../sources/GcsEventsHoodieIncrSource.java | 3 +-
.../sources/helpers/IncrSourceHelper.java | 11 +-
.../utilities/sources/helpers/QueryRunner.java | 6 +
.../helpers/gcs/GcsObjectMetadataFetcher.java | 17 +-
.../apache/hudi/utilities/TestHoodieIndexer.java | 45 +-
.../deltastreamer/HoodieDeltaStreamerTestBase.java | 283 ++++++++-
.../deltastreamer/TestHoodieDeltaStreamer.java | 513 +++++-----------
.../TestHoodieDeltaStreamerDAGExecution.java | 4 +-
.../TestHoodieDeltaStreamerWithMultiWriter.java | 127 ++--
.../TestHoodieMultiTableDeltaStreamer.java | 12 +-
.../utilities/deltastreamer/TestTransformer.java | 4 +-
.../functional/TestHoodieSnapshotExporter.java | 14 +-
.../offlinejob/HoodieOfflineJobTestBase.java | 8 +
.../sources/TestGcsEventsHoodieIncrSource.java | 194 ++----
.../utilities/sources/TestHoodieIncrSource.java | 347 ++++++-----
.../sources/TestS3EventsHoodieIncrSource.java | 21 +-
.../sources/helpers/TestIncrSourceHelper.java | 120 ++++
.../utilities/testutils/UtilitiesTestBase.java | 3 +-
148 files changed, 5023 insertions(+), 2255 deletions(-)
create mode 100644
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/metrics/FlinkCompactionMetrics.java
create mode 100644
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/metrics/FlinkWriteMetrics.java
create mode 100644
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/ExpressionPredicates.java
create mode 100644
hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/TestExpressionPredicates.java
create mode 100644
hudi-gcp/src/main/java/org/apache/hudi/gcp/bigquery/BigQuerySchemaResolver.java
create mode 100644
hudi-gcp/src/test/java/org/apache/hudi/gcp/bigquery/TestBigQuerySchemaResolver.java
create mode 100644
hudi-gcp/src/test/java/org/apache/hudi/gcp/bigquery/TestBigQuerySyncTool.java
create mode 100644
hudi-gcp/src/test/java/org/apache/hudi/gcp/bigquery/TestHoodieBigQuerySyncClient.java
