This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 1217eaaf11d9 fix(flink): avoid partition stats pruning for incremental
reads (#20481)
1217eaaf11d9 is described below
commit 1217eaaf11d9ac177f718e3619e1df86c8403148
Author: Shuo Cheng <[email protected]>
AuthorDate: Wed Nov 4 10:19:03 2026 +0800
fix(flink): avoid partition stats pruning for incremental reads (#20481)
---
.../org/apache/hudi/table/HoodieTableSource.java | 5 ++-
.../hudi/source/TestIncrementalInputSplits.java | 10 ++++-
.../apache/hudi/table/ITTestHoodieDataSource.java | 47 +++++++++++++++++++++-
.../apache/hudi/table/TestHoodieTableSource.java | 27 +++++++++++++
4 files changed, 86 insertions(+), 3 deletions(-)
diff --git
a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableSource.java
b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableSource.java
index a32014a3c67f..0e7d54ec35a6 100644
---
a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableSource.java
+++
b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableSource.java
@@ -399,7 +399,10 @@ public class HoodieTableSource extends FileIndexReader
implements
Tuple2<List<ResolvedExpression>, List<ResolvedExpression>> splitFilters =
splitExprByPartitionCall(simpleFilters, this.partitionKeys, this.tableRowType);
this.predicates = ExpressionPredicates.fromExpression(splitFilters.f0);
this.columnStatsProbe = ColumnStatsProbe.newInstance(splitFilters.f0);
- this.partitionPruner = createPartitionPruner(splitFilters.f1,
columnStatsProbe);
+ // Partition stats only reflect the latest table state and can exclude
values in historical incremental reads.
+ boolean incrementalRead = conf.get(FlinkOptions.READ_AS_STREAMING)
+ ||
FlinkOptions.QUERY_TYPE_INCREMENTAL.equals(conf.get(FlinkOptions.QUERY_TYPE));
+ this.partitionPruner = createPartitionPruner(splitFilters.f1,
incrementalRead ? null : columnStatsProbe);
this.dataBucketFunc = getDataBucketFunc(splitFilters.f0);
// refuse all the filters now
return SupportsFilterPushDown.Result.of(new ArrayList<>(splitFilters.f1),
new ArrayList<>(filters));
diff --git
a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/TestIncrementalInputSplits.java
b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/TestIncrementalInputSplits.java
index 9c8e0a7f7db1..1435f3029cd4 100644
---
a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/TestIncrementalInputSplits.java
+++
b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/TestIncrementalInputSplits.java
@@ -402,6 +402,7 @@ public class TestIncrementalInputSplits extends
HoodieCommonTestHarness {
PartitionPruners.PartitionPruner partitionPruner =
PartitionPruners.builder().rowType(TestConfigurations.ROW_TYPE).basePath(basePath).metaClient(metaClient).conf(conf).columnStatsProbe(columnStatsProbe).build();
+ assertEquals(Collections.singleton("par3"),
partitionPruner.filter(Arrays.asList("par1", "par2", "par3", "par4")));
IncrementalInputSplits iis = IncrementalInputSplits.builder()
.conf(conf)
.path(new Path(basePath))
@@ -410,7 +411,14 @@ public class TestIncrementalInputSplits extends
HoodieCommonTestHarness {
.build();
IncrementalInputSplits.Result result = iis.inputSplits(metaClient, null,
false);
List<String> partitions = getFilteredPartitions(result);
- assertEquals(Arrays.asList("par3"), partitions);
+ assertEquals(Collections.singletonList("par3"), partitions);
+
+ PartitionPruners.PartitionPruner chainedPruner = PartitionPruners.builder()
+
.rowType(TestConfigurations.ROW_TYPE).basePath(basePath).metaClient(metaClient).conf(conf)
+
.columnStatsProbe(columnStatsProbe).candidatePartitions(Arrays.asList("par1",
"par3")).build();
+ IncrementalInputSplits chainedSplits = IncrementalInputSplits.builder()
+ .conf(conf).path(new
Path(basePath)).rowType(TestConfigurations.ROW_TYPE).partitionPruner(chainedPruner).build();
+ assertEquals(Collections.singletonList("par3"),
getFilteredPartitions(chainedSplits.inputSplits(metaClient, null, false)));
}
@Test
diff --git
a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java
b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java
index c877fda22097..09ba50f7edbc 100644
---
a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java
+++
b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java
@@ -2944,6 +2944,51 @@ public class ITTestHoodieDataSource {
+ "+I[id3, id3, Julian, 43, 1970-01-01T00:00:03, par1]]");
}
+ @ParameterizedTest
+ @MethodSource("tableTypeAndSourceV2AndBooleanTrueFalseParams")
+ void testHistoricalIncrementalReadWithDataSkipping(HoodieTableType
tableType, boolean useSourceV2, boolean streaming) throws Exception {
+ batchTableEnv.executeSql(sql("writer")
+ .option(FlinkOptions.PATH, tempFile.getAbsolutePath())
+ .options(getDefaultKeys())
+ .option(FlinkOptions.TABLE_TYPE, tableType)
+ .option(FlinkOptions.INDEX_BOOTSTRAP_ENABLED, true)
+ .option(FlinkOptions.METADATA_ENABLED, true)
+ .option(HoodieMetadataConfig.ENABLE_METADATA_INDEX_COLUMN_STATS.key(),
true)
+
.option(HoodieMetadataConfig.ENABLE_METADATA_INDEX_PARTITION_STATS.key(), true)
+ .end());
+ execInsertSql(batchTableEnv, "insert into writer values "
+ + "('id1', 'alice', 23, TIMESTAMP '1970-01-01 00:00:01', 'par1'),"
+ + "('id2', 'bob', 23, TIMESTAMP '1970-01-01 00:00:01', 'par2')");
+ String firstCommit =
HoodieTestUtils.createMetaClient(tempFile.getAbsolutePath())
+
.getCommitsTimeline().filterCompletedInstants().lastInstant().get().getCompletionTime();
+ // The update can remove age 23 from current stats of par1; par2 remains a
matching control.
+ execInsertSql(batchTableEnv, "insert into writer values ('id1', 'alice',
50, TIMESTAMP '1970-01-01 00:00:02', 'par1')");
+ assertRowsEquals(execSelectSql(batchTableEnv, "select * from writer where
age = 50"),
+ "[+I[id1, alice, 50, 1970-01-01T00:00:02, par1]]");
+
+ TableEnvironment readerEnv = streaming ? streamTableEnv : batchTableEnv;
+ readerEnv.executeSql(sql("t1")
+ .option(FlinkOptions.PATH, tempFile.getAbsolutePath())
+ .options(getDefaultKeys())
+ .option(FlinkOptions.TABLE_TYPE, tableType)
+ .option(FlinkOptions.READ_AS_STREAMING, streaming)
+ .option(FlinkOptions.READ_STREAMING_CHECK_INTERVAL, 2)
+ .option(FlinkOptions.READ_START_COMMIT, firstCommit)
+ .option(FlinkOptions.READ_END_COMMIT, firstCommit)
+ .option(FlinkOptions.READ_SOURCE_V2_ENABLED, useSourceV2)
+ .option(FlinkOptions.METADATA_ENABLED, true)
+ .option(FlinkOptions.READ_DATA_SKIPPING_ENABLED, true)
+ .end());
+ String query = "select * from t1 where age = 23";
+ String expected = "[+I[id1, alice, 23, 1970-01-01T00:00:01, par1], "
+ + "+I[id2, bob, 23, 1970-01-01T00:00:01, par2]]";
+ if (streaming) {
+ assertRowsEquals(execSelectSqlWithExpectedNum(readerEnv, query, 2),
expected);
+ } else {
+ assertRowsEquals(execSelectSql(readerEnv, query), expected);
+ }
+ }
+
@Test
void testDataSkippingWithMissingColumnStatsAfterIndexExpansion() throws
Exception {
TableEnvironment tableEnv = batchTableEnv;
@@ -4322,7 +4367,7 @@ public class ITTestHoodieDataSource {
}
/**
- * Return test params => (HoodieTableType, true/false, true/fase).
+ * Return test params => (HoodieTableType, true/false, true/false).
*/
private static Stream<Arguments>
tableTypeAndSourceV2AndBooleanTrueFalseParams() {
Object[][] data =
diff --git
a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/TestHoodieTableSource.java
b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/TestHoodieTableSource.java
index f82c1267fd38..a91c436547ad 100644
---
a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/TestHoodieTableSource.java
+++
b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/TestHoodieTableSource.java
@@ -236,6 +236,33 @@ public class TestHoodieTableSource {
assertEquals(expectedPartitions, hoodieTableSource.getReadPartitions());
}
+ @ParameterizedTest
+ @ValueSource(strings = {"snapshot", "read_optimized", "incremental",
"streaming"})
+ void testPartitionStatsPrunerForReadMode(String readMode) throws Exception {
+ Configuration conf =
TestConfigurations.getDefaultConf(tempFile.getAbsolutePath());
+ conf.set(FlinkOptions.METADATA_ENABLED, true);
+ conf.set(FlinkOptions.READ_DATA_SKIPPING_ENABLED, true);
+
conf.setString(HoodieMetadataConfig.ENABLE_METADATA_INDEX_COLUMN_STATS.key(),
"true");
+
conf.setString(HoodieMetadataConfig.ENABLE_METADATA_INDEX_PARTITION_STATS.key(),
"true");
+ TestData.writeData(TestData.DATA_SET_INSERT, conf);
+ if ("streaming".equals(readMode)) {
+ conf.set(FlinkOptions.READ_AS_STREAMING, true);
+ } else {
+ conf.set(FlinkOptions.QUERY_TYPE, readMode);
+ }
+
+ HoodieTableSource tableSource = createHoodieTableSource(conf);
+ tableSource.applyFilters(Arrays.asList(
+ createLitEquivalenceExpr("age", 2, DataTypes.INT().notNull(), 18),
+ createLitEquivalenceExpr("partition", 4, DataTypes.STRING().notNull(),
"par1")));
+ assertNotNull(tableSource.getColumnStatsProbe());
+ if ("incremental".equals(readMode) || "streaming".equals(readMode)) {
+ assertEquals(Collections.singletonList("par1"),
tableSource.getReadPartitions());
+ } else {
+ assertTrue(tableSource.getReadPartitions().isEmpty());
+ }
+ }
+
@Test
void testDataSkippingWithMetaColumns() throws Exception {
final String path = tempFile.getAbsolutePath();