This is an automated email from the ASF dual-hosted git repository.
amogh-jahagirdar pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new e0bf18cc0d Core: Extend V4 DeletionVector with key_metadata field
(#17438)
e0bf18cc0d is described below
commit e0bf18cc0d109a6a4fe7605a2bec95c5f1290e70
Author: gaborkaszab <[email protected]>
AuthorDate: Tue Aug 4 16:19:08 2026 +0200
Core: Extend V4 DeletionVector with key_metadata field (#17438)
---
.../java/org/apache/iceberg/DeletionVector.java | 12 +++-
.../org/apache/iceberg/DeletionVectorStruct.java | 69 +++++++++++++---------
.../org/apache/iceberg/TrackedFileAdapters.java | 2 +-
.../apache/iceberg/TestDeletionVectorStruct.java | 15 ++++-
.../apache/iceberg/TestTrackedFileAdapters.java | 5 +-
5 files changed, 71 insertions(+), 32 deletions(-)
diff --git a/core/src/main/java/org/apache/iceberg/DeletionVector.java
b/core/src/main/java/org/apache/iceberg/DeletionVector.java
index 0fc8f259f0..d29c150058 100644
--- a/core/src/main/java/org/apache/iceberg/DeletionVector.java
+++ b/core/src/main/java/org/apache/iceberg/DeletionVector.java
@@ -18,6 +18,7 @@
*/
package org.apache.iceberg;
+import java.nio.ByteBuffer;
import org.apache.iceberg.types.Types;
/**
@@ -45,9 +46,15 @@ interface DeletionVector {
"cardinality",
Types.LongType.get(),
"Number of set bits (deleted rows) in the vector");
+ Types.NestedField KEY_METADATA =
+ Types.NestedField.optional(
+ 149,
+ "key_metadata",
+ Types.BinaryType.get(),
+ "Implementation-specific key metadata for encryption");
static Types.StructType schema() {
- return Types.StructType.of(LOCATION, OFFSET, SIZE_IN_BYTES, CARDINALITY);
+ return Types.StructType.of(LOCATION, OFFSET, SIZE_IN_BYTES, CARDINALITY,
KEY_METADATA);
}
/** Returns the location of the file containing the deletion vector. */
@@ -62,6 +69,9 @@ interface DeletionVector {
/** Returns the number of set bits (deleted rows) in the vector. */
long cardinality();
+ /** Returns encryption key metadata, or null if the file is not encrypted. */
+ ByteBuffer keyMetadata();
+
/** Copies this deletion vector. */
DeletionVector copy();
}
diff --git a/core/src/main/java/org/apache/iceberg/DeletionVectorStruct.java
b/core/src/main/java/org/apache/iceberg/DeletionVectorStruct.java
index 04d23fa33a..461e472e59 100644
--- a/core/src/main/java/org/apache/iceberg/DeletionVectorStruct.java
+++ b/core/src/main/java/org/apache/iceberg/DeletionVectorStruct.java
@@ -19,10 +19,13 @@
package org.apache.iceberg;
import java.io.Serializable;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
import org.apache.iceberg.avro.SupportsIndexProjection;
import org.apache.iceberg.relocated.com.google.common.base.MoreObjects;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
import org.apache.iceberg.types.Types;
+import org.apache.iceberg.util.ByteBuffers;
/** Mutable {@link StructLike} implementation of {@link DeletionVector}. */
class DeletionVectorStruct extends SupportsIndexProjection implements
DeletionVector, Serializable {
@@ -31,12 +34,14 @@ class DeletionVectorStruct extends SupportsIndexProjection
implements DeletionVe
DeletionVector.LOCATION,
DeletionVector.OFFSET,
DeletionVector.SIZE_IN_BYTES,
- DeletionVector.CARDINALITY);
+ DeletionVector.CARDINALITY,
+ DeletionVector.KEY_METADATA);
private String location = null;
private long offset = -1L;
private long sizeInBytes = -1L;
private long cardinality = -1L;
+ private byte[] keyMetadata = null;
DeletionVectorStruct(Types.StructType type) {
super(BASE_TYPE, type);
@@ -48,14 +53,20 @@ class DeletionVectorStruct extends SupportsIndexProjection
implements DeletionVe
this.offset = toCopy.offset;
this.sizeInBytes = toCopy.sizeInBytes;
this.cardinality = toCopy.cardinality;
+ this.keyMetadata =
+ toCopy.keyMetadata != null
+ ? Arrays.copyOf(toCopy.keyMetadata, toCopy.keyMetadata.length)
+ : null;
}
- private DeletionVectorStruct(String location, long offset, long sizeInBytes,
long cardinality) {
+ private DeletionVectorStruct(
+ String location, long offset, long sizeInBytes, long cardinality,
ByteBuffer keyMetadata) {
super(BASE_TYPE.fields().size());
this.location = location;
this.offset = offset;
this.sizeInBytes = sizeInBytes;
this.cardinality = cardinality;
+ this.keyMetadata = ByteBuffers.toByteArray(keyMetadata);
}
@Override
@@ -78,6 +89,11 @@ class DeletionVectorStruct extends SupportsIndexProjection
implements DeletionVe
return cardinality;
}
+ @Override
+ public ByteBuffer keyMetadata() {
+ return keyMetadata != null ? ByteBuffer.wrap(keyMetadata) : null;
+ }
+
@Override
public DeletionVectorStruct copy() {
return new DeletionVectorStruct(this);
@@ -89,38 +105,28 @@ class DeletionVectorStruct extends SupportsIndexProjection
implements DeletionVe
}
private Object getByPos(int pos) {
- switch (pos) {
- case 0:
- return location;
- case 1:
- return offset;
- case 2:
- return sizeInBytes;
- case 3:
- return cardinality;
- default:
- throw new UnsupportedOperationException("Unknown field ordinal: " +
pos);
- }
+ return switch (pos) {
+ case 0 -> location;
+ case 1 -> offset;
+ case 2 -> sizeInBytes;
+ case 3 -> cardinality;
+ case 4 -> keyMetadata();
+ default -> throw new UnsupportedOperationException("Unknown field
ordinal: " + pos);
+ };
}
@Override
protected <T> void internalSet(int pos, T value) {
switch (pos) {
- case 0:
// always coerce to String for Serializable
- this.location = value.toString();
- break;
- case 1:
- this.offset = (Long) value;
- break;
- case 2:
- this.sizeInBytes = (Long) value;
- break;
- case 3:
- this.cardinality = (Long) value;
- break;
- default:
+ case 0 -> this.location = value.toString();
+ case 1 -> this.offset = (Long) value;
+ case 2 -> this.sizeInBytes = (Long) value;
+ case 3 -> this.cardinality = (Long) value;
+ case 4 -> this.keyMetadata = ByteBuffers.toByteArray((ByteBuffer) value);
+ default -> {
// ignore the object, it must be from a newer version of the format
+ }
}
}
@@ -135,6 +141,7 @@ class DeletionVectorStruct extends SupportsIndexProjection
implements DeletionVe
.add("offset", offset)
.add("size_in_bytes", sizeInBytes)
.add("cardinality", cardinality)
+ .add("key_metadata", keyMetadata == null ? "null" : "(redacted)")
.toString();
}
@@ -143,6 +150,7 @@ class DeletionVectorStruct extends SupportsIndexProjection
implements DeletionVe
private Long offset = null;
private Long sizeInBytes = null;
private Long cardinality = null;
+ private ByteBuffer keyMetadata = null;
Builder location(String dvLocation) {
Preconditions.checkArgument(dvLocation != null, "Invalid location:
null");
@@ -170,12 +178,17 @@ class DeletionVectorStruct extends
SupportsIndexProjection implements DeletionVe
return this;
}
+ Builder keyMetadata(ByteBuffer dvKeyMetadata) {
+ this.keyMetadata = dvKeyMetadata;
+ return this;
+ }
+
DeletionVectorStruct build() {
Preconditions.checkArgument(location != null, "Missing required value:
location");
Preconditions.checkArgument(offset != null, "Missing required value:
offset");
Preconditions.checkArgument(sizeInBytes != null, "Missing required
value: size in bytes");
Preconditions.checkArgument(cardinality != null, "Missing required
value: cardinality");
- return new DeletionVectorStruct(location, offset, sizeInBytes,
cardinality);
+ return new DeletionVectorStruct(location, offset, sizeInBytes,
cardinality, keyMetadata);
}
}
}
diff --git a/core/src/main/java/org/apache/iceberg/TrackedFileAdapters.java
b/core/src/main/java/org/apache/iceberg/TrackedFileAdapters.java
index 355e9602af..154abb9f74 100644
--- a/core/src/main/java/org/apache/iceberg/TrackedFileAdapters.java
+++ b/core/src/main/java/org/apache/iceberg/TrackedFileAdapters.java
@@ -331,7 +331,7 @@ class TrackedFileAdapters {
@Override
public ByteBuffer keyMetadata() {
- return null;
+ return dv.keyMetadata();
}
@Override
diff --git
a/core/src/test/java/org/apache/iceberg/TestDeletionVectorStruct.java
b/core/src/test/java/org/apache/iceberg/TestDeletionVectorStruct.java
index b20a096077..0589ebbb69 100644
--- a/core/src/test/java/org/apache/iceberg/TestDeletionVectorStruct.java
+++ b/core/src/test/java/org/apache/iceberg/TestDeletionVectorStruct.java
@@ -22,11 +22,14 @@ import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import java.io.IOException;
+import java.nio.ByteBuffer;
import org.apache.iceberg.types.Types;
import org.junit.jupiter.api.Test;
class TestDeletionVectorStruct {
+ private static final ByteBuffer KEY_METADATA = ByteBuffer.wrap(new byte[]
{1, 2, 3});
+
@Test
void fieldAccess() {
DeletionVectorStruct dv =
@@ -35,12 +38,14 @@ class TestDeletionVectorStruct {
.offset(256L)
.sizeInBytes(128L)
.cardinality(42L)
+ .keyMetadata(KEY_METADATA)
.build();
assertThat(dv.location()).isEqualTo("s3://bucket/data/dv.puffin");
assertThat(dv.offset()).isEqualTo(256L);
assertThat(dv.sizeInBytes()).isEqualTo(128L);
assertThat(dv.cardinality()).isEqualTo(42L);
+ assertThat(dv.keyMetadata()).isEqualTo(KEY_METADATA);
}
@Test
@@ -51,6 +56,7 @@ class TestDeletionVectorStruct {
.offset(256L)
.sizeInBytes(128L)
.cardinality(42L)
+ .keyMetadata(KEY_METADATA)
.build();
DeletionVectorStruct copy = dv.copy();
@@ -59,12 +65,13 @@ class TestDeletionVectorStruct {
assertThat(copy.offset()).isEqualTo(256L);
assertThat(copy.sizeInBytes()).isEqualTo(128L);
assertThat(copy.cardinality()).isEqualTo(42L);
+ assertThat(copy.keyMetadata()).isEqualTo(KEY_METADATA);
}
@Test
void size() {
DeletionVectorStruct dv = new
DeletionVectorStruct(DeletionVector.schema());
- assertThat(dv.size()).isEqualTo(4);
+ assertThat(dv.size()).isEqualTo(5);
}
@Test
@@ -95,6 +102,7 @@ class TestDeletionVectorStruct {
.offset(100L)
.sizeInBytes(512L)
.cardinality(42L)
+ .keyMetadata(KEY_METADATA)
.build();
// unknown ordinals from a newer format version are silently ignored
@@ -105,6 +113,7 @@ class TestDeletionVectorStruct {
assertThat(dv.offset()).isEqualTo(100L);
assertThat(dv.sizeInBytes()).isEqualTo(512L);
assertThat(dv.cardinality()).isEqualTo(42L);
+ assertThat(dv.keyMetadata()).isEqualTo(KEY_METADATA);
}
@Test
@@ -115,6 +124,7 @@ class TestDeletionVectorStruct {
.offset(256L)
.sizeInBytes(128L)
.cardinality(42L)
+ .keyMetadata(KEY_METADATA)
.build();
DeletionVectorStruct deserialized = TestHelpers.roundTripSerialize(dv);
@@ -123,6 +133,7 @@ class TestDeletionVectorStruct {
assertThat(deserialized.offset()).isEqualTo(256L);
assertThat(deserialized.sizeInBytes()).isEqualTo(128L);
assertThat(deserialized.cardinality()).isEqualTo(42L);
+ assertThat(deserialized.keyMetadata()).isEqualTo(KEY_METADATA);
}
@Test
@@ -190,6 +201,7 @@ class TestDeletionVectorStruct {
.offset(256L)
.sizeInBytes(128L)
.cardinality(42L)
+ .keyMetadata(KEY_METADATA)
.build();
DeletionVectorStruct deserialized =
TestHelpers.KryoHelpers.roundTripSerialize(dv);
@@ -198,5 +210,6 @@ class TestDeletionVectorStruct {
assertThat(deserialized.offset()).isEqualTo(256L);
assertThat(deserialized.sizeInBytes()).isEqualTo(128L);
assertThat(deserialized.cardinality()).isEqualTo(42L);
+ assertThat(deserialized.keyMetadata()).isEqualTo(KEY_METADATA);
}
}
diff --git a/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java
b/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java
index 1992390506..c9a45928ad 100644
--- a/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java
+++ b/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java
@@ -49,6 +49,8 @@ class TestTrackedFileAdapters {
private static final Map<Integer, PartitionSpec> UNPARTITIONED =
ImmutableMap.of(UNPARTITIONED_SPEC_ID, PartitionSpec.unpartitioned());
+ private static final ByteBuffer KEY_METADATA = ByteBuffer.wrap(new byte[]
{1, 2, 3});
+
private static final Schema PARTITION_SCHEMA =
new Schema(Types.NestedField.required(1, "category",
Types.StringType.get()));
private static final int PARTITIONED_SPEC_ID = 1;
@@ -242,6 +244,7 @@ class TestTrackedFileAdapters {
.offset(128L)
.sizeInBytes(256L)
.cardinality(10L)
+ .keyMetadata(KEY_METADATA)
.build();
TrackingStruct tracking =
@@ -285,6 +288,7 @@ class TestTrackedFileAdapters {
assertThat(dvFile.recordCount()).isEqualTo(dv.cardinality());
assertThat(dvFile.contentOffset()).isEqualTo(dv.offset());
assertThat(dvFile.contentSizeInBytes()).isEqualTo(dv.sizeInBytes());
+ assertThat(dvFile.keyMetadata()).isEqualTo(KEY_METADATA);
// fileSizeInBytes reports the DV blob size, not the full Puffin file size.
assertThat(dvFile.fileSizeInBytes()).isEqualTo(dv.sizeInBytes());
// referencedDataFile is delegated to the tracked data file's location.
@@ -301,7 +305,6 @@ class TestTrackedFileAdapters {
// fields that are null for DVs
assertThat(dvFile.sortOrderId()).isNull();
assertThat(dvFile.firstRowId()).isNull();
- assertThat(dvFile.keyMetadata()).isNull();
assertThat(dvFile.splitOffsets()).isNull();
assertThat(dvFile.equalityFieldIds()).isNull();
assertThat(dvFile.columnSizes()).isNull();
