This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 87de4a722e [iceberg] Omit unknown bounds in Iceberg manifest partition
summary (#9152)
87de4a722e is described below
commit 87de4a722ec0053d1c40051af58e7daac15f3817
Author: Eunbin Son <[email protected]>
AuthorDate: Wed Aug 12 12:29:42 2026 +0900
[iceberg] Omit unknown bounds in Iceberg manifest partition summary (#9152)
---
.../paimon/iceberg/IcebergCommitCallback.java | 13 +++--
.../iceberg/manifest/IcebergManifestFile.java | 7 ++-
.../paimon/iceberg/IcebergCompatibilityTest.java | 67 ++++++++++++++++++++++
3 files changed, 81 insertions(+), 6 deletions(-)
diff --git
a/paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergCommitCallback.java
b/paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergCommitCallback.java
index 54180fe3e0..cfeb6acb1c 100644
---
a/paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergCommitCallback.java
+++
b/paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergCommitCallback.java
@@ -1338,10 +1338,15 @@ public class IcebergCommitCallback implements
CommitCallback, TagCallback {
for (int i = 0; i < numFields; i++) {
IcebergPartitionSummary summary = fileMeta.partitions().get(i);
DataType fieldType = partitionType.getTypeAt(i);
- minValues.setField(
- i, IcebergConversions.toPaimonObject(fieldType,
summary.lowerBound()));
- maxValues.setField(
- i, IcebergConversions.toPaimonObject(fieldType,
summary.upperBound()));
+ // an omitted bound means the value is unknown; keep the slot
null
+ byte[] lowerBound = summary.lowerBound();
+ byte[] upperBound = summary.upperBound();
+ if (lowerBound != null) {
+ minValues.setField(i,
IcebergConversions.toPaimonObject(fieldType, lowerBound));
+ }
+ if (upperBound != null) {
+ maxValues.setField(i,
IcebergConversions.toPaimonObject(fieldType, upperBound));
+ }
// IcebergPartitionSummary only has `containsNull` field and
does not have the
// exact number of nulls.
nullCounts[i] = summary.containsNull() ? 1 : 0;
diff --git
a/paimon-core/src/main/java/org/apache/paimon/iceberg/manifest/IcebergManifestFile.java
b/paimon-core/src/main/java/org/apache/paimon/iceberg/manifest/IcebergManifestFile.java
index d5abb480a5..201b389570 100644
---
a/paimon-core/src/main/java/org/apache/paimon/iceberg/manifest/IcebergManifestFile.java
+++
b/paimon-core/src/main/java/org/apache/paimon/iceberg/manifest/IcebergManifestFile.java
@@ -266,12 +266,15 @@ public class IcebergManifestFile extends
ObjectsFile<IcebergManifestEntry> {
default:
// contains_nan is only meaningful for FLOAT/DOUBLE
per the Iceberg spec
}
+ // an unknown bound must be omitted, not published as a value
+ Object min = fieldStats.min();
+ Object max = fieldStats.max();
partitionSummaries.add(
new IcebergPartitionSummary(
Objects.requireNonNull(fieldStats.nullCount())
> 0,
containsNan,
- toByteBuffer(type, fieldStats.min()).array(),
- toByteBuffer(type, fieldStats.max()).array()));
+ min == null ? null : toByteBuffer(type,
min).array(),
+ max == null ? null : toByteBuffer(type,
max).array()));
}
return new IcebergManifestFileMeta(
path.toString(),
diff --git
a/paimon-core/src/test/java/org/apache/paimon/iceberg/IcebergCompatibilityTest.java
b/paimon-core/src/test/java/org/apache/paimon/iceberg/IcebergCompatibilityTest.java
index d8a22aaee4..6c6ceed190 100644
---
a/paimon-core/src/test/java/org/apache/paimon/iceberg/IcebergCompatibilityTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/iceberg/IcebergCompatibilityTest.java
@@ -1371,6 +1371,73 @@ public class IcebergCompatibilityTest {
assertThat(sawNanPartitionSummary).isTrue();
}
+ @Test
+ public void testNullPartitionValue() throws Exception {
+ RowType rowType =
+ RowType.of(
+ new DataType[] {DataTypes.VARCHAR(10),
DataTypes.INT()},
+ new String[] {"pt", "v"});
+ FileStoreTable table =
+ createPaimonTable(
+ rowType, Collections.singletonList("pt"),
Collections.emptyList(), -1);
+
+ String commitUser = UUID.randomUUID().toString();
+ TableWriteImpl<?> write = table.newWrite(commitUser);
+ TableCommitImpl commit = table.newCommit(commitUser);
+
+ write.write(GenericRow.of(BinaryString.fromString("a"), 1), 1);
+ commit.commit(1, write.prepareCommit(false, 1));
+
+ // every entry of this manifest has a null partition value, so its
partition
+ // statistics are unknown and the summary must omit both bounds
+ write.write(GenericRow.of(null, 2), 1);
+ commit.commit(2, write.prepareCommit(false, 2));
+
+ FileIO fileIO = table.fileIO();
+ IcebergMetadata metadata =
+ IcebergMetadata.fromPath(
+ fileIO, new Path(table.location(),
"metadata/v2.metadata.json"));
+ List<String> partitionSummaries = new ArrayList<>();
+ try (DataFileReader<GenericRecord> dataFileReader =
+ new DataFileReader<>(
+ new SeekableFileInput(new
File(metadata.currentSnapshot().manifestList())),
+ new GenericDatumReader<>())) {
+ while (dataFileReader.hasNext()) {
+
partitionSummaries.add(dataFileReader.next().get("partitions").toString());
+ }
+ }
+ assertThat(partitionSummaries)
+ .anySatisfy(
+ summary ->
+ assertThat(summary)
+ .contains("\"contains_null\": true")
+ .contains("\"lower_bound\": null")
+ .contains("\"upper_bound\": null"));
+ // known bounds are still recorded for non-null partition values
+ assertThat(partitionSummaries)
+ .anySatisfy(
+ summary ->
+ assertThat(summary)
+ .contains("\"lower_bound\": \"a\"")
+ .contains("\"upper_bound\": \"a\""));
+
+ write.write(GenericRow.of(BinaryString.fromString("b"), 3), 1);
+ commit.commit(3, write.prepareCommit(false, 3));
+
+ assertThat(getIcebergResult())
+ .containsExactlyInAnyOrder("Record(a, 1)", "Record(null, 2)",
"Record(b, 3)");
+
+ // a non add-only commit reads the omitted bounds back from the base
manifests
+ Map<String, String> partition = new HashMap<>();
+ partition.put("pt", "a");
+ commit.truncatePartitions(Collections.singletonList(partition));
+
+ assertThat(getIcebergResult()).containsExactlyInAnyOrder("Record(null,
2)", "Record(b, 3)");
+
+ write.close();
+ commit.close();
+ }
+
@Test
public void testStringPartitionNullPadding() throws Exception {
RowType rowType =
