This is an automated email from the ASF dual-hosted git repository.
yihua pushed a change to branch release-1.1.0
in repository https://gitbox.apache.org/repos/asf/hudi.git
from eeb07995b3ac fix: Skip payload class validation when merge mode is not
custom with v9 tables (#14116)
new 68707164cb3b fix: Fixing error msg when table is not found with
MergeIntoCommand (#14118)
new b634cfc47ee0 fix: Handle deletes and updates properly in secondary
index (#14090)
new d13d8ae9c82e fix(core): add table level validation for decimal
evolution (#14089)
new d424808c6802 fix: Handle missing valueType column after upgrade
(#14105)
new ae90b950f20d perf: Reduce memory usage of writing HFile log block
(#14078)
new 3468a173c263 fix: Fix cleaning of historical internal schema files
(#14126)
new 0568b0a90df8 fix: Upgrade parquet-avro version to 1.15.1 in trino
bundle and plugin (#14140)
new 3b1dc608ef61 fix: Upgrade Java xmlbuilder version to fix
CVE-2014-125087 (#14144)
new 83eac32573a8 fix: fix partition stats delete properly for downgrade
from V9 to V8 (#14138)
new 90a886a153ad fix: updating error messages thrown to end users (#14115)
new dec9c8ed2adc fix: Fix the instant time issue for row writer bulk
insert hoodie streamer (#14153)
new 256523affc3e fix: Fixed the recovering method for the older versions
where checksum is not present (#14148)
new c2fd476a8bb2 refactor: Add required setter methods for Flink-CDC dev
(#14150)
new 63ec30032ff9 fi: Fixing secondary index read for V1 (#14149)
new 7dcccd607393 fix: Give proper error message for multi-writer scenarios
without lock provider set (#14119)
new 9510647bfa9b feat: Partition predicate fix for Databricks runtime
support (#14059)
new dfd26b46ffb6 fix: Partition stats should be controlled using column
stats config (#14165)
new 72f4452981eb fix: Fix upgrade handling for MySqlDebeziumAvroPayload
with deltastreamer (#14159)
new bfc236b0398f fix: fix downgrade to not delete unintended partitions in
MDT (#14162)
The 19 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../apache/hudi/client/BaseHoodieWriteClient.java | 7 +-
.../org/apache/hudi/config/HoodieWriteConfig.java | 5 +-
.../org/apache/hudi/io/HoodieAppendHandle.java | 2 +
.../metadata/HoodieBackedTableMetadataWriter.java | 37 +++--
.../hudi/metadata/MetadataIndexGenerator.java | 132 ------------------
.../apache/hudi/metadata/MetadataIndexMapper.java | 66 +++++++++
.../apache/hudi/metadata/RecordIndexMapper.java | 84 ++++++++++++
.../apache/hudi/metadata/SecondaryIndexMapper.java | 68 ++++++++++
.../SecondaryIndexRecordGenerationUtils.java | 13 +-
.../java/org/apache/hudi/table/HoodieTable.java | 12 +-
.../table/upgrade/EightToNineUpgradeHandler.java | 44 +++---
.../hudi/table/upgrade/UpgradeDowngrade.java | 2 +-
.../hudi/table/upgrade/UpgradeDowngradeUtils.java | 14 +-
...Generator.java => TestMetadataIndexMapper.java} | 13 +-
.../upgrade/TestEightToNineUpgradeHandler.java | 107 ++++++---------
.../upgrade/TestNineToEightDowngradeHandler.java | 120 +++++++++++++++++
.../FlinkHoodieBackedTableMetadataWriter.java | 5 -
.../org/apache/hudi/table/HoodieFlinkTable.java | 3 +-
.../JavaHoodieBackedTableMetadataWriter.java | 5 -
.../org/apache/hudi/table/HoodieJavaTable.java | 9 +-
.../hudi/client/StreamingMetadataWriteHandler.java | 2 +-
.../apache/hudi/index/HoodieSparkIndexClient.java | 25 +---
.../SparkHoodieBackedTableMetadataWriter.java | 4 -
...ieBackedTableMetadataWriterTableVersionSix.java | 5 -
.../org/apache/hudi/table/HoodieSparkTable.java | 7 +-
.../spark/sql/HoodieCatalystPlansUtils.scala | 5 +
.../table/functional/TestCleanPlanExecutor.java | 3 +-
.../apache/hudi/avro/AvroSchemaCompatibility.java | 40 +++++-
.../hudi/common/config/HoodieMetadataConfig.java | 30 ++---
.../hudi/common/model/HoodieAvroIndexedRecord.java | 2 +-
.../hudi/common/model/HoodieRecordPayload.java | 21 ++-
.../model/debezium/MySqlDebeziumAvroPayload.java | 5 +
.../hudi/common/table/HoodieTableConfig.java | 7 +
.../hudi/common/table/HoodieTableVersion.java | 9 +-
.../common/table/log/block/HoodieDataBlock.java | 2 +-
.../org/apache/hudi/common/util/CommitUtils.java | 2 +-
.../org/apache/hudi/common/util/ConfigUtils.java | 15 ++-
.../org/apache/hudi/common/util/HFileUtils.java | 92 ++++++-------
.../io/FileBasedInternalSchemaStorageManager.java | 4 +-
.../hudi/metadata/HoodieBackedTableMetadata.java | 8 +-
.../hudi/metadata/HoodieTableMetadataUtil.java | 40 ++++--
.../java/org/apache/hudi/stats/ValueMetadata.java | 5 +
.../org/apache/hudi/avro/TestAvroSchemaUtils.java | 70 ++++++++++
.../apache/hudi/common/util/TestConfigUtils.java | 27 ++++
.../hudi/sink/StreamWriteOperatorCoordinator.java | 6 +-
.../sink/common/AbstractStreamWriteFunction.java | 10 +-
.../org/apache/hudi/sink/event/Correspondent.java | 8 ++
.../org/apache/hudi/sink/TestWriteCopyOnWrite.java | 2 -
.../hudi/sink/TestWriteMergeOnReadWithCompact.java | 3 -
.../java/org/apache/hudi/source/TestFileIndex.java | 1 -
.../hudi/source/TestIncrementalInputSplits.java | 1 -
.../apache/hudi/table/ITTestHoodieDataSource.java | 2 -
.../apache/hudi/table/ITTestSchemaEvolution.java | 3 +-
.../apache/hudi/table/TestHoodieTableSource.java | 1 -
.../hudi/io/hadoop/HoodieBaseParquetWriter.java | 4 +
.../hudi/common/table/TestHoodieTableConfig.java | 13 +-
.../apache/hudi/common/util/TestCommitUtils.java | 35 +++++
.../TestFileBasedInternalSchemaStorageManager.java | 2 +
.../hudi/metadata/TestHoodieTableMetadataUtil.java | 2 -
.../main/java/org/apache/hudi/DataSourceUtils.java | 15 ++-
.../BaseDatasetBulkInsertCommitActionExecutor.java | 4 +-
.../DatasetBucketRescaleCommitActionExecutor.java | 4 +-
.../DatasetBulkInsertCommitActionExecutor.java | 4 +-
...setBulkInsertOverwriteCommitActionExecutor.java | 4 +-
...lkInsertOverwriteTableCommitActionExecutor.java | 4 +-
...eamerDatasetBulkInsertCommitActionExecutor.java | 4 +-
.../org/apache/hudi/HoodieSparkSqlWriter.scala | 12 +-
.../apache/hudi/SparkHoodieTableFileIndex.scala | 20 ++-
.../hudi/analysis/HoodieSparkBaseAnalysis.scala | 10 +-
.../TestDataSkippingWithMORColstats.java | 3 -
.../hudi/functional/TestHoodieBackedMetadata.java | 2 -
.../apache/hudi/io/TestMetadataWriterCommit.java | 6 +-
.../hudi/table/upgrade/TestUpgradeDowngrade.java | 108 ++++++++++++---
.../cow-tables/hudi-v8-table-cow.zip | Bin 0 -> 380008 bytes
.../generate-fixtures.sh | 3 +
...ixture-mor.scala => generate-fixture-cow.scala} | 28 ++--
.../org/apache/hudi/TestHoodieSparkSqlWriter.scala | 2 +-
.../functional/PartitionStatsIndexTestBase.scala | 3 +-
.../hudi/functional/TestBasicSchemaEvolution.scala | 3 +-
.../hudi/functional/TestPartitionStatsIndex.scala | 22 ---
.../TestPartitionStatsIndexWithSql.scala | 6 +-
.../functional/TestPartitionStatsPruning.scala | 19 +--
.../functional/TestRecordLevelIndexWithSQL.scala | 2 +
.../functional/TestSecondaryIndexDataTypes.scala | 5 +
.../functional/TestSecondaryIndexPruning.scala | 150 ++++++++++++++++++++-
.../sql/hudi/dml/others/TestMergeIntoTable.scala | 33 +++++
.../hudi/feature/index/TestExpressionIndex.scala | 5 +-
.../TestHoodieBackedTableMetadataIndexLookup.scala | 1 +
.../sql/hudi/feature/index/TestIndexSyntax.scala | 4 +
.../hudi/feature/index/TestSecondaryIndex.scala | 4 +
.../spark/sql/HoodieSpark33CatalystPlanUtils.scala | 4 +
.../spark/sql/HoodieSpark34CatalystPlanUtils.scala | 6 +
.../spark/sql/HoodieSpark35CatalystPlanUtils.scala | 6 +
.../spark/sql/HoodieSpark40CatalystPlanUtils.scala | 6 +
.../hudi/hive/util/IMetaStoreClientUtil.java | 23 +++-
.../apache/hudi/sync/common/HoodieSyncClient.java | 19 ++-
hudi-trino-plugin/pom.xml | 2 +
.../apache/hudi/utilities/sources/InputBatch.java | 7 +-
.../hudi/utilities/streamer/HoodieStreamer.java | 1 -
.../utilities/streamer/HoodieStreamerUtils.java | 8 +-
.../apache/hudi/utilities/streamer/StreamSync.java | 30 ++++-
.../TestHoodieMetadataTableValidator.java | 5 +-
.../deltastreamer/HoodieDeltaStreamerTestBase.java | 26 +++-
.../deltastreamer/TestHoodieDeltaStreamer.java | 54 ++++++--
...TestHoodieDeltaStreamerSchemaEvolutionBase.java | 2 +-
.../hudi/utilities/sources/TestInputBatch.java | 6 +-
.../streamer/TestHoodieStreamerUtils.java | 5 +-
packaging/hudi-trino-bundle/pom.xml | 2 +-
pom.xml | 6 +
109 files changed, 1397 insertions(+), 582 deletions(-)
delete mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/MetadataIndexGenerator.java
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/MetadataIndexMapper.java
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/RecordIndexMapper.java
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/SecondaryIndexMapper.java
rename
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metadata/{TestMetadataIndexGenerator.java
=> TestMetadataIndexMapper.java} (85%)
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/resources/upgrade-downgrade-fixtures/cow-tables/hudi-v8-table-cow.zip
copy
hudi-spark-datasource/hudi-spark/src/test/resources/upgrade-downgrade-fixtures/scala-templates/{generate-fixture-mor.scala
=> generate-fixture-cow.scala} (89%)
