This is an automated email from the ASF dual-hosted git repository.
pwason pushed a change to branch release-0.14.0
in repository https://gitbox.apache.org/repos/asf/hudi.git
from d32bdbd8240 [MINOR] Fix consistent hashing bucket index FT failure
(#9401)
new 8f07023948e [HUDI-6675] Fix Clean action will delete the whole table
(#9413)
new 3db4745a23d [MINOR] Infer the preCombine field only if the value is
not null (#9447)
new 510ff1753a4 [HUDI-5361] Propagate all hoodie configs from spark
sqlconf (#8327)
new 89b8ae02bf4 [HUDI-6679] Fix initialization of metadata table
partitions upon failure (#9419)
new b8d0424c2c8 [MINOR] asyncService log prompt incomplete (#9407)
new 81a458aa33c [MINOR] Increase CI timeout for UT FT other modules to 4
hours (#9423)
new 0dca5aaceb3 [HUDI-6680] Fixing the info log to fetch column value by
name instead of index (#9421)
new e2a78d3fb43 [MINOR] Unify class name of Spark Procedure (#9414)
new d70c15f4041 [HUDI-6670] Fix timeline check in metadata table validator
(#9405)
new b27b1f688aa [HUDI-6663] New Parquet File Format remove broadcast to
fix performance issue for complex file slices (#9409)
new 612d02b35a0 [HUDI-6553] Speedup column stats and bloom index creation
on large datasets. (#9223)
new b335d00a22b [HUDI-6674] Add rollback info from metadata table in
timeline commands (#9411)
new c7f0e6902fa [HUDI-6690] Generate test jars for hudi-utilities and
hudi-hive-sync modules (#9297)
new 529fc04488b Duplicate switch branch in HoodieInputFormatUtils (#9438)
new 1726b828578 [HUDI-6214] Enabling compaction by default for batch
writes with MOR table (#8718)
new 6b848f028ec [HUDI-6676] Add command for CreateHoodieTableLike (#9412)
new 97f21f85e95 [HUDI-6683] Added kafka key as part of hudi metadata
columns for Json & Avro KafkaSource (#9403)
new d6358a9d602 [HUDI-6694] Fix log file CLI around command blocks (#9445)
new b10f52d85d3 [HUDI-6689] Add record index validation in MDT validator
(#9437)
new b8dc3a58220 Handling empty commits after s3 applyFilter api (#9433)
new a58ff06f20e [HUDI-6688] Fix partition validation to only consider
commits in metadata table validator (#9436)
new da699fea98d [HUDI-6553][FOLLOW-UP] Introduces Tuple3 for
HoodieTableMetadataUtil (#9449)
new 2c9024e4fad [HUDI-6673] Fix Incremental Query Syntax - Spark SQL Core
Flow Test (#9410)
new 77bf4357ed7 [HUDI-6683][FOLLOW-UP] Rename kafka record value variable
in JsonKafkaSource and replace casting to String by calling toString (#9451)
new 2538f544507 [HUDI-6359] Spark offline compaction/clustering will never
rollback when both requested and inflight states exist (#8944)
new 90e3378207d [HUDI-6704] Fix Flink metadata table update (#9456)
new 20b4438377b [MINOR] Fix sql core flow test (#9461)
new 6ffd4d5705a [MINOR] Fix meta client instantiation and some incorrect
configs (#9463)
new 9bc6a28010c [MINOR] Fix build on master (#9452)
The 29 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
azure-pipelines-20230430.yml | 2 +-
.../test-suite/multi-writer-local-3.properties | 4 +-
.../config/test-suite/test-clustering.properties | 4 +-
...essive-clean-archival-inline-compact.properties | 4 +-
.../apache/hudi/cli/HoodieTableHeaderFields.java | 1 -
.../hudi/cli/commands/HoodieLogFileCommand.java | 70 ++++--
.../apache/hudi/cli/commands/TimelineCommand.java | 99 ++++----
.../cli/commands/TestHoodieLogFileCommand.java | 33 ++-
.../org/apache/hudi/async/HoodieAsyncService.java | 4 +-
.../hudi/client/BaseHoodieTableServiceClient.java | 19 +-
.../apache/hudi/client/BaseHoodieWriteClient.java | 29 +--
.../metadata/HoodieBackedTableMetadataWriter.java | 13 +-
.../java/org/apache/hudi/table/HoodieTable.java | 22 --
.../table/action/clean/CleanActionExecutor.java | 10 +-
.../hudi/client/HoodieFlinkTableServiceClient.java | 13 +-
.../apache/hudi/client/HoodieFlinkWriteClient.java | 5 -
.../spark/sql/HoodieCatalystPlansUtils.scala | 7 +
.../org/apache/spark/sql/hudi/SparkAdapter.scala | 8 +-
.../functional/TestHoodieBackedMetadata.java | 123 ++++++++-
.../java/org/apache/hudi/table/TestCleaner.java | 51 ++++
.../apache/hudi/common/util/collection/Tuple3.java | 71 ++++++
.../hudi/metadata/HoodieMetadataPayload.java | 19 +-
.../hudi/metadata/HoodieTableMetadataUtil.java | 228 ++++++++++-------
.../hudi/source/stats/ColumnStatsIndices.java | 17 +-
.../java/org/apache/hudi/util/CompactionUtil.java | 4 +-
.../hudi/hadoop/utils/HoodieInputFormatUtils.java | 2 -
.../hudi/integ/testsuite/HoodieTestSuiteJob.java | 2 +
.../main/java/org/apache/hudi/DataSourceUtils.java | 12 +-
.../scala/org/apache/hudi/DataSourceOptions.scala | 3 +
.../main/scala/org/apache/hudi/DefaultSource.scala | 3 +-
.../scala/org/apache/hudi/HoodieFileIndex.scala | 10 +-
.../org/apache/hudi/HoodieSparkSqlWriter.scala | 53 ++--
.../org/apache/hudi/HoodieStreamingSink.scala | 19 +-
.../hudi/NewHoodieParquetFileFormatUtils.scala | 2 +-
.../apache/hudi/PartitionFileSliceMapping.scala | 7 +-
.../parquet/NewHoodieParquetFileFormat.scala | 8 +-
.../apache/spark/sql/hudi/HoodieOptionConfig.scala | 8 +
.../command/CreateHoodieTableLikeCommand.scala | 110 ++++++++
.../spark/sql/hudi/analysis/HoodieAnalysis.scala | 13 +-
...empView.scala => CopyToTempViewProcedure.scala} | 8 +-
.../hudi/command/procedures/HoodieProcedures.scala | 2 +-
.../org/apache/hudi/TestHoodieSparkSqlWriter.scala | 4 +-
.../apache/hudi/functional/TestMORDataSource.scala | 2 +
.../hudi/functional/TestMORDataSourceStorage.scala | 54 +++-
.../hudi/functional/TestSparkSqlCoreFlow.scala | 10 +-
.../sql/hudi/TestAlterTableDropPartition.scala | 10 +-
.../spark/sql/hudi/TestCompactionTable.scala | 8 +
.../apache/spark/sql/hudi/TestCreateTable.scala | 139 ++++++++++
.../org/apache/spark/sql/hudi/TestSpark3DDL.scala | 4 +
.../apache/spark/sql/hudi/TestUpdateTable.scala | 18 ++
.../hudi/procedure/TestClusteringProcedure.scala | 5 +
.../hudi/procedure/TestCompactionProcedure.scala | 16 +-
.../spark/sql/HoodieSpark2CatalystPlanUtils.scala | 9 +
.../spark/sql/HoodieSpark3CatalystPlanUtils.scala | 13 +-
hudi-sync/hudi-hive-sync/pom.xml | 3 +
hudi-utilities/pom.xml | 3 +
.../apache/hudi/utilities/HoodieClusteringJob.java | 3 +-
.../org/apache/hudi/utilities/HoodieCompactor.java | 4 +-
.../utilities/HoodieMetadataTableValidator.java | 280 +++++++++++++++++++--
.../utilities/schema/KafkaOffsetPostProcessor.java | 6 +-
.../sources/GcsEventsHoodieIncrSource.java | 94 ++++---
.../hudi/utilities/sources/JsonKafkaSource.java | 9 +-
.../sources/S3EventsHoodieIncrSource.java | 14 +-
.../utilities/sources/helpers/AvroConvertor.java | 3 +
.../sources/helpers/IncrSourceHelper.java | 15 +-
.../utilities/sources/TestAvroKafkaSource.java | 11 +-
.../utilities/sources/TestJsonKafkaSource.java | 9 +-
.../sources/TestS3EventsHoodieIncrSource.java | 104 +++++++-
.../sources/helpers/TestIncrSourceHelper.java | 47 ++--
69 files changed, 1522 insertions(+), 497 deletions(-)
create mode 100644
hudi-common/src/main/java/org/apache/hudi/common/util/collection/Tuple3.java
create mode 100644
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/CreateHoodieTableLikeCommand.scala
rename
hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/{CopyToTempView.scala
=> CopyToTempViewProcedure.scala} (95%)
