This is an automated email from the ASF dual-hosted git repository.
amogh-jahagirdar pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new ebf2ff8a93 Core: Support projecting content stats in the V4 file
planner (#18740)
ebf2ff8a93 is described below
commit ebf2ff8a938177a6c37a165b4cc2543f15d9c4e1
Author: Anoop Johnson <[email protected]>
AuthorDate: Mon Nov 2 08:33:48 2026 -0800
Core: Support projecting content stats in the V4 file planner (#18740)
Add FilePlanner.projectStats(int...) so callers can request content stats
for specific table field IDs. The field IDs are forwarded to the root and
leaf manifest readers; when unset, no stats are returned.
Follow-up to
https://github.com/apache/iceberg/pull/17541#discussion_r4169414840
---
.../main/java/org/apache/iceberg/FilePlanner.java | 12 +++
.../java/org/apache/iceberg/TestFilePlanner.java | 92 +++++++++++++++++++---
2 files changed, 93 insertions(+), 11 deletions(-)
diff --git a/core/src/main/java/org/apache/iceberg/FilePlanner.java
b/core/src/main/java/org/apache/iceberg/FilePlanner.java
index 78c2206b47..ca63c2b64f 100644
--- a/core/src/main/java/org/apache/iceberg/FilePlanner.java
+++ b/core/src/main/java/org/apache/iceberg/FilePlanner.java
@@ -20,6 +20,7 @@ package org.apache.iceberg;
import java.io.IOException;
import java.io.UncheckedIOException;
+import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
@@ -56,6 +57,7 @@ class FilePlanner {
private String tableLocation = null;
private Expression dataFilter = Expressions.alwaysTrue();
+ private int[] statsFieldIds = null;
private boolean ignoreResiduals = false;
private boolean caseSensitive = true;
private ScanMetrics scanMetrics = ScanMetrics.noop();
@@ -85,6 +87,12 @@ class FilePlanner {
return this;
}
+ FilePlanner projectStats(int... fieldIds) {
+ Preconditions.checkArgument(fieldIds != null, "Invalid field IDs: null");
+ this.statsFieldIds = Arrays.copyOf(fieldIds, fieldIds.length);
+ return this;
+ }
+
FilePlanner ignoreResiduals() {
this.ignoreResiduals = true;
return this;
@@ -162,6 +170,10 @@ class FilePlanner {
builder.tableLocation(tableLocation);
}
+ if (statsFieldIds != null) {
+ builder.projectStats(statsFieldIds);
+ }
+
return builder.build();
}
diff --git a/core/src/test/java/org/apache/iceberg/TestFilePlanner.java
b/core/src/test/java/org/apache/iceberg/TestFilePlanner.java
index dd2e7d3ed1..318fe7df01 100644
--- a/core/src/test/java/org/apache/iceberg/TestFilePlanner.java
+++ b/core/src/test/java/org/apache/iceberg/TestFilePlanner.java
@@ -52,6 +52,7 @@ import
org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.apache.iceberg.relocated.com.google.common.collect.Iterables;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.types.Conversions;
import org.apache.iceberg.types.Types;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.FieldSource;
@@ -273,6 +274,58 @@ class TestFilePlanner {
.hasMessageContaining("Cannot find field 'ID'");
}
+ @ParameterizedTest
+ @FieldSource("MANIFEST_FORMATS")
+ void taskOmitsStatsUnlessProjected(FileFormat format) throws IOException {
+ ManifestFile root =
+ writeManifest(
+ format, EMPTY_PARTITION,
ImmutableList.of(dataFileWithStats(FILE_A, idStats(0, 99))));
+
+ DataFile file = Iterables.getOnlyElement(plan(root,
UNPARTITIONED_SPECS)).file();
+
+ assertThat(file.lowerBounds())
+ .as("no column stats are returned unless projectStats requests them")
+ .isNull();
+ assertThat(file.valueCounts()).as("value counts are also omitted by
default").isNull();
+ }
+
+ @ParameterizedTest
+ @FieldSource("MANIFEST_FORMATS")
+ void tasksContainProjectedStats(FileFormat format) throws IOException {
+ ManifestFile leaf =
+ writeManifest(
+ format,
+ EMPTY_PARTITION,
+ ImmutableList.of(dataFileWithStats(FILE_C, idAndDataStats(100,
199))));
+ ManifestFile root =
+ writeManifest(
+ format,
+ EMPTY_PARTITION,
+ ImmutableList.of(
+ dataFileWithStats(FILE_A, idAndDataStats(0, 99)),
dataManifest(leaf.path())));
+
+ int idFieldId = TABLE_SCHEMA.findField("id").fieldId();
+ int dataFieldId = TABLE_SCHEMA.findField("data").fieldId();
+ List<FileScanTask> tasks =
+ plan(root, UNPARTITIONED_SPECS, planner ->
planner.projectStats(idFieldId));
+
+ assertThat(tasks)
+ .extracting(
+ task -> task.file().location(),
+ task -> lowerBound(task.file(), idFieldId),
+ task -> upperBound(task.file(), idFieldId))
+ .as("projected id stats are returned for files in both the root and a
leaf")
+ .containsExactlyInAnyOrder(tuple(FILE_A, 0, 99), tuple(FILE_C, 100,
199));
+
+ assertThat(tasks)
+ .as("stats for unprojected fields are omitted")
+ .allSatisfy(
+ task -> {
+
assertThat(task.file().lowerBounds()).doesNotContainKey(dataFieldId);
+
assertThat(task.file().upperBounds()).doesNotContainKey(dataFieldId);
+ });
+ }
+
@ParameterizedTest
@FieldSource("MANIFEST_FORMATS")
void taskAttachesColocatedDV(FileFormat format) throws IOException {
@@ -590,22 +643,39 @@ class TestFilePlanner {
return manifestRefWithStats(FileContent.DATA_MANIFEST, location, stats,
dataManifestInfo(1));
}
+ private static Integer lowerBound(DataFile file, int fieldId) {
+ return Conversions.fromByteBuffer(Types.IntegerType.get(),
file.lowerBounds().get(fieldId));
+ }
+
+ private static Integer upperBound(DataFile file, int fieldId) {
+ return Conversions.fromByteBuffer(Types.IntegerType.get(),
file.upperBounds().get(fieldId));
+ }
+
private static ContentStats idStats(int lower, int upper) {
ContentStatsStruct stats = new ContentStatsStruct(STATS_TYPE);
- stats.setStats(
- TABLE_SCHEMA.findField("id").fieldId(),
- new FieldStatsStruct<>(
- STATS_TYPE.fieldType("id").asStructType(),
- lower,
- upper,
- true, // tightBounds
- RECORD_COUNT,
- 0, // nullValueCount
- 0, // nanValueCount
- 0L)); // totalBytes
+ stats.setStats(TABLE_SCHEMA.findField("id").fieldId(), fieldStats("id",
lower, upper));
return stats;
}
+ private static ContentStats idAndDataStats(int lower, int upper) {
+ ContentStatsStruct stats = new ContentStatsStruct(STATS_TYPE);
+ stats.setStats(TABLE_SCHEMA.findField("id").fieldId(), fieldStats("id",
lower, upper));
+ stats.setStats(TABLE_SCHEMA.findField("data").fieldId(),
fieldStats("data", "a", "z"));
+ return stats;
+ }
+
+ private static <T> FieldStatsStruct<T> fieldStats(String fieldName, T lower,
T upper) {
+ return new FieldStatsStruct<>(
+ STATS_TYPE.fieldType(fieldName).asStructType(),
+ lower,
+ upper,
+ true, // tightBounds
+ RECORD_COUNT,
+ 0, // nullValueCount
+ 0, // nanValueCount
+ 0L); // totalBytes
+ }
+
private static ManifestInfo dataManifestInfo(int addedFiles) {
return dataManifestInfo(addedFiles, FORMAT_VERSION_V4);
}
