This is an automated email from the ASF dual-hosted git repository.
danielcweeks pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new 063a80a9a7 Core: Move v4 format_version from tracked file to manifest
info (#18418)
063a80a9a7 is described below
commit 063a80a9a7427794890dae740c08521627100ee4
Author: Anoop Johnson <[email protected]>
AuthorDate: Fri Oct 9 11:10:57 2026 -0700
Core: Move v4 format_version from tracked file to manifest info (#18418)
Move the v4 format_version field (0: PRE-V4, 4: V4) off every tracked file
and onto manifest_info as a required field (ID 521), so it is tracked only for
manifest entries. The tracked_file field ID 157 is removed, and the
TrackedManifestFile adapter sources formatVersion() from manifest_info.
---
.../main/java/org/apache/iceberg/ManifestInfo.java | 12 +++-
.../org/apache/iceberg/ManifestInfoStruct.java | 32 +++++++++-
.../main/java/org/apache/iceberg/TrackedFile.java | 7 ---
.../org/apache/iceberg/TrackedFileAdapters.java | 2 +-
.../java/org/apache/iceberg/TrackedFileStruct.java | 65 ++++++++-----------
.../java/org/apache/iceberg/StatsTestUtil.java | 1 -
.../org/apache/iceberg/TestManifestInfoStruct.java | 72 +++++++++++++++++++++-
.../java/org/apache/iceberg/TestTrackedFile.java | 4 +-
.../apache/iceberg/TestTrackedFileAdapters.java | 9 +--
.../org/apache/iceberg/TestTrackedFileStruct.java | 17 -----
.../org/apache/iceberg/TestV4ManifestReader.java | 10 +--
.../java/org/apache/iceberg/V4TestComparators.java | 4 +-
.../java/org/apache/iceberg/V4TestHelpers.java | 1 -
13 files changed, 143 insertions(+), 93 deletions(-)
diff --git a/core/src/main/java/org/apache/iceberg/ManifestInfo.java
b/core/src/main/java/org/apache/iceberg/ManifestInfo.java
index c41a008526..45d5062aab 100644
--- a/core/src/main/java/org/apache/iceberg/ManifestInfo.java
+++ b/core/src/main/java/org/apache/iceberg/ManifestInfo.java
@@ -61,6 +61,12 @@ interface ManifestInfo {
Types.NestedField DV =
Types.NestedField.optional(
522, "dv", Types.BinaryType.get(), "Deletion vector for manifest
entries");
+ Types.NestedField FORMAT_VERSION =
+ Types.NestedField.required(
+ 521,
+ "format_version",
+ Types.IntegerType.get(),
+ "Format version used to write the manifest");
static Types.StructType schema() {
return Types.StructType.of(
@@ -75,7 +81,8 @@ interface ManifestInfo {
REPLACED_ROWS_COUNT,
MODIFIED_ROWS_COUNT,
MIN_SEQUENCE_NUMBER,
- DV);
+ DV,
+ FORMAT_VERSION);
}
/** Returns the number of files added by this manifest. */
@@ -114,6 +121,9 @@ interface ManifestInfo {
/** Returns the deletion vector for manifest entries, or null if not
present. */
ManifestBitmap manifestDeletionVector();
+ /** Returns the format version used to write the manifest. */
+ int formatVersion();
+
/** Copies this manifest info. */
ManifestInfo copy();
}
diff --git a/core/src/main/java/org/apache/iceberg/ManifestInfoStruct.java
b/core/src/main/java/org/apache/iceberg/ManifestInfoStruct.java
index 26a2fcef76..bc5765a855 100644
--- a/core/src/main/java/org/apache/iceberg/ManifestInfoStruct.java
+++ b/core/src/main/java/org/apache/iceberg/ManifestInfoStruct.java
@@ -43,7 +43,8 @@ class ManifestInfoStruct extends SupportsIndexProjection
implements ManifestInfo
ManifestInfo.REPLACED_ROWS_COUNT,
ManifestInfo.MODIFIED_ROWS_COUNT,
ManifestInfo.MIN_SEQUENCE_NUMBER,
- ManifestInfo.DV);
+ ManifestInfo.DV,
+ ManifestInfo.FORMAT_VERSION);
private int addedFilesCount = -1;
private int existingFilesCount = -1;
@@ -57,6 +58,7 @@ class ManifestInfoStruct extends SupportsIndexProjection
implements ManifestInfo
private long modifiedRowsCount = -1L;
private long minSequenceNumber = -1L;
private byte[] dv = null;
+ private int formatVersion = -1;
private transient ManifestBitmap mdv = null;
@@ -78,6 +80,7 @@ class ManifestInfoStruct extends SupportsIndexProjection
implements ManifestInfo
this.modifiedRowsCount = toCopy.modifiedRowsCount;
this.minSequenceNumber = toCopy.minSequenceNumber;
this.dv = toCopy.dv != null ? Arrays.copyOf(toCopy.dv, toCopy.dv.length) :
null;
+ this.formatVersion = toCopy.formatVersion;
}
ManifestInfoStruct(
@@ -92,7 +95,8 @@ class ManifestInfoStruct extends SupportsIndexProjection
implements ManifestInfo
long replacedRowsCount,
long modifiedRowsCount,
long minSequenceNumber,
- byte[] dv) {
+ byte[] dv,
+ int formatVersion) {
super(BASE_TYPE.fields().size());
this.addedFilesCount = addedFilesCount;
this.existingFilesCount = existingFilesCount;
@@ -106,6 +110,7 @@ class ManifestInfoStruct extends SupportsIndexProjection
implements ManifestInfo
this.modifiedRowsCount = modifiedRowsCount;
this.minSequenceNumber = minSequenceNumber;
this.dv = dv;
+ this.formatVersion = formatVersion;
}
@Override
@@ -172,6 +177,11 @@ class ManifestInfoStruct extends SupportsIndexProjection
implements ManifestInfo
return mdv;
}
+ @Override
+ public int formatVersion() {
+ return formatVersion;
+ }
+
@Override
public ManifestInfoStruct copy() {
return new ManifestInfoStruct(this);
@@ -208,6 +218,8 @@ class ManifestInfoStruct extends SupportsIndexProjection
implements ManifestInfo
return minSequenceNumber;
case 11:
return dv != null ? ByteBuffer.wrap(dv) : null;
+ case 12:
+ return formatVersion;
default:
throw new UnsupportedOperationException("Unknown field ordinal: " +
pos);
}
@@ -253,6 +265,9 @@ class ManifestInfoStruct extends SupportsIndexProjection
implements ManifestInfo
this.dv = ByteBuffers.toByteArray((ByteBuffer) value);
this.mdv = null;
break;
+ case 12:
+ this.formatVersion = (Integer) value;
+ break;
default:
// ignore the object, it must be from a newer version of the format
}
@@ -277,6 +292,7 @@ class ManifestInfoStruct extends SupportsIndexProjection
implements ManifestInfo
.add("modified_rows_count", modifiedRowsCount)
.add("min_sequence_number", minSequenceNumber)
.add("dv", dv == null ? "null" : "(binary)")
+ .add("format_version", formatVersion)
.toString();
}
@@ -293,6 +309,7 @@ class ManifestInfoStruct extends SupportsIndexProjection
implements ManifestInfo
private Long modifiedRowsCount = null;
private Long minSequenceNumber = null;
private byte[] dv = null;
+ private Integer formatVersion = null;
Builder addedFilesCount(int count) {
Preconditions.checkArgument(
@@ -376,6 +393,13 @@ class ManifestInfoStruct extends SupportsIndexProjection
implements ManifestInfo
return this;
}
+ Builder formatVersion(int version) {
+ Preconditions.checkArgument(
+ version == 0 || version == 4, "Invalid format version: %s (must be 0
or 4)", version);
+ this.formatVersion = version;
+ return this;
+ }
+
ManifestInfoStruct build() {
Preconditions.checkArgument(
addedFilesCount != null, "Missing required value: added files
count");
@@ -399,6 +423,7 @@ class ManifestInfoStruct extends SupportsIndexProjection
implements ManifestInfo
modifiedRowsCount != null, "Missing required value: modified rows
count");
Preconditions.checkArgument(
minSequenceNumber != null, "Missing required value: min sequence
number");
+ Preconditions.checkArgument(formatVersion != null, "Missing required
value: format version");
Preconditions.checkArgument(
addedRowsCount == 0 || addedFilesCount > 0,
"Invalid added counts: %s rows in %s files",
@@ -436,7 +461,8 @@ class ManifestInfoStruct extends SupportsIndexProjection
implements ManifestInfo
replacedRowsCount,
modifiedRowsCount,
minSequenceNumber,
- dv);
+ dv,
+ formatVersion);
}
}
}
diff --git a/core/src/main/java/org/apache/iceberg/TrackedFile.java
b/core/src/main/java/org/apache/iceberg/TrackedFile.java
index e2db837c48..94c55e471a 100644
--- a/core/src/main/java/org/apache/iceberg/TrackedFile.java
+++ b/core/src/main/java/org/apache/iceberg/TrackedFile.java
@@ -37,9 +37,6 @@ interface TrackedFile {
"content_type",
Types.IntegerType.get(),
"Type of content: 0=DATA, 2=EQUALITY_DELETES, 3=DATA_MANIFEST,
4=DELETE_MANIFEST");
- Types.NestedField FORMAT_VERSION =
- Types.NestedField.required(
- 157, "format_version", Types.IntegerType.get(), "Format version of
this file");
Types.NestedField LOCATION =
Types.NestedField.required(100, "location", Types.StringType.get(),
"Location of the file");
Types.NestedField FILE_FORMAT =
@@ -102,7 +99,6 @@ interface TrackedFile {
return ImmutableList.of(
TRACKING,
CONTENT_TYPE,
- FORMAT_VERSION,
LOCATION,
FILE_FORMAT,
RECORD_COUNT,
@@ -151,9 +147,6 @@ interface TrackedFile {
/** Returns the type of content stored by this entry. */
FileContent contentType();
- /** Returns the format version of this file. */
- int formatVersion();
-
/** Returns the location of the file. */
String location();
diff --git a/core/src/main/java/org/apache/iceberg/TrackedFileAdapters.java
b/core/src/main/java/org/apache/iceberg/TrackedFileAdapters.java
index 5acb03b908..2b351fc4bd 100644
--- a/core/src/main/java/org/apache/iceberg/TrackedFileAdapters.java
+++ b/core/src/main/java/org/apache/iceberg/TrackedFileAdapters.java
@@ -520,7 +520,7 @@ class TrackedFileAdapters {
@Override
public int formatVersion() {
- return file.formatVersion();
+ return file.manifestInfo().formatVersion();
}
@Override
diff --git a/core/src/main/java/org/apache/iceberg/TrackedFileStruct.java
b/core/src/main/java/org/apache/iceberg/TrackedFileStruct.java
index fffd822a56..fc400a4bfb 100644
--- a/core/src/main/java/org/apache/iceberg/TrackedFileStruct.java
+++ b/core/src/main/java/org/apache/iceberg/TrackedFileStruct.java
@@ -40,7 +40,6 @@ class TrackedFileStruct extends SupportsIndexProjection
implements TrackedFile,
Types.StructType.of(
TrackedFile.TRACKING,
TrackedFile.CONTENT_TYPE,
- TrackedFile.FORMAT_VERSION,
TrackedFile.LOCATION,
TrackedFile.FILE_FORMAT,
TrackedFile.RECORD_COUNT,
@@ -64,7 +63,6 @@ class TrackedFileStruct extends SupportsIndexProjection
implements TrackedFile,
TrackedFile.EQUALITY_IDS);
private FileContent contentType = null;
- private int formatVersion = -1;
private String location = null;
private FileFormat fileFormat = null;
private Tracking tracking = null;
@@ -97,7 +95,6 @@ class TrackedFileStruct extends SupportsIndexProjection
implements TrackedFile,
TrackedFileStruct(
Tracking tracking,
FileContent contentType,
- int formatVersion,
String location,
FileFormat fileFormat,
long recordCount,
@@ -114,7 +111,6 @@ class TrackedFileStruct extends SupportsIndexProjection
implements TrackedFile,
super(BASE_TYPE.fields().size());
this.tracking = tracking;
this.contentType = contentType;
- this.formatVersion = formatVersion;
this.location = location;
this.fileFormat = fileFormat;
this.recordCount = recordCount;
@@ -134,7 +130,6 @@ class TrackedFileStruct extends SupportsIndexProjection
implements TrackedFile,
private TrackedFileStruct(TrackedFileStruct toCopy, Set<Integer> statsIds) {
super(toCopy);
this.contentType = toCopy.contentType;
- this.formatVersion = toCopy.formatVersion;
this.location = toCopy.location;
this.fileFormat = toCopy.fileFormat;
this.recordCount = toCopy.recordCount;
@@ -177,11 +172,6 @@ class TrackedFileStruct extends SupportsIndexProjection
implements TrackedFile,
return contentType;
}
- @Override
- public int formatVersion() {
- return formatVersion;
- }
-
@Override
public String location() {
return location;
@@ -286,20 +276,19 @@ class TrackedFileStruct extends SupportsIndexProjection
implements TrackedFile,
return switch (pos) {
case 0 -> tracking;
case 1 -> contentType != null ? contentType.id() : null;
- case 2 -> formatVersion;
- case 3 -> location;
- case 4 -> fileFormat != null ? fileFormat.toString() : null;
- case 5 -> recordCount;
- case 6 -> fileSizeInBytes;
- case 7 -> specId;
- case 8 -> partition;
- case 9 -> contentStats;
- case 10 -> sortOrderId;
- case 11 -> deletionVector;
- case 12 -> manifestInfo;
- case 13 -> keyMetadata();
- case 14 -> splitOffsets();
- case 15 -> equalityIds();
+ case 2 -> location;
+ case 3 -> fileFormat != null ? fileFormat.toString() : null;
+ case 4 -> recordCount;
+ case 5 -> fileSizeInBytes;
+ case 6 -> specId;
+ case 7 -> partition;
+ case 8 -> contentStats;
+ case 9 -> sortOrderId;
+ case 10 -> deletionVector;
+ case 11 -> manifestInfo;
+ case 12 -> keyMetadata();
+ case 13 -> splitOffsets();
+ case 14 -> equalityIds();
default -> throw new UnsupportedOperationException("Unknown field
ordinal: " + pos);
};
}
@@ -309,22 +298,21 @@ class TrackedFileStruct extends SupportsIndexProjection
implements TrackedFile,
switch (pos) {
case 0 -> this.tracking = (Tracking) value;
case 1 -> this.contentType = FileContent.fromId((Integer) value);
- case 2 -> this.formatVersion = (int) value;
- case 3 ->
+ case 2 ->
// always coerce to String for Serializable
this.location = value.toString();
- case 4 -> this.fileFormat = FileFormat.fromString(value.toString());
- case 5 -> this.recordCount = (long) value;
- case 6 -> this.fileSizeInBytes = (long) value;
- case 7 -> this.specId = (Integer) value;
- case 8 -> this.partition = (StructLike) value;
- case 9 -> this.contentStats = (ContentStats) value;
- case 10 -> this.sortOrderId = (Integer) value;
- case 11 -> this.deletionVector = (DeletionVector) value;
- case 12 -> this.manifestInfo = (ManifestInfo) value;
- case 13 -> this.keyMetadata = ByteBuffers.toByteArray((ByteBuffer)
value);
- case 14 -> this.splitOffsets = ArrayUtil.toLongArray((List<Long>) value);
- case 15 -> this.equalityIds = ArrayUtil.toIntArray((List<Integer>)
value);
+ case 3 -> this.fileFormat = FileFormat.fromString(value.toString());
+ case 4 -> this.recordCount = (long) value;
+ case 5 -> this.fileSizeInBytes = (long) value;
+ case 6 -> this.specId = (Integer) value;
+ case 7 -> this.partition = (StructLike) value;
+ case 8 -> this.contentStats = (ContentStats) value;
+ case 9 -> this.sortOrderId = (Integer) value;
+ case 10 -> this.deletionVector = (DeletionVector) value;
+ case 11 -> this.manifestInfo = (ManifestInfo) value;
+ case 12 -> this.keyMetadata = ByteBuffers.toByteArray((ByteBuffer)
value);
+ case 13 -> this.splitOffsets = ArrayUtil.toLongArray((List<Long>) value);
+ case 14 -> this.equalityIds = ArrayUtil.toIntArray((List<Integer>)
value);
default -> {
// ignore the object, it must be from a newer version of the format
}
@@ -335,7 +323,6 @@ class TrackedFileStruct extends SupportsIndexProjection
implements TrackedFile,
public String toString() {
return MoreObjects.toStringHelper(this)
.add("content", contentType != null ? contentType.lowerCaseName() :
null)
- .add("format_version", formatVersion)
.add("location", location)
.add("file_format", fileFormat)
.add("record_count", recordCount)
diff --git a/core/src/test/java/org/apache/iceberg/StatsTestUtil.java
b/core/src/test/java/org/apache/iceberg/StatsTestUtil.java
index 93309ceb9b..a2bbf7bf53 100644
--- a/core/src/test/java/org/apache/iceberg/StatsTestUtil.java
+++ b/core/src/test/java/org/apache/iceberg/StatsTestUtil.java
@@ -29,7 +29,6 @@ class StatsTestUtil {
return new TrackedFileStruct(
null,
FileContent.DATA,
- 4,
location,
FileFormat.fromFileName(location),
recordCount,
diff --git a/core/src/test/java/org/apache/iceberg/TestManifestInfoStruct.java
b/core/src/test/java/org/apache/iceberg/TestManifestInfoStruct.java
index 1892f2d890..b1363249bb 100644
--- a/core/src/test/java/org/apache/iceberg/TestManifestInfoStruct.java
+++ b/core/src/test/java/org/apache/iceberg/TestManifestInfoStruct.java
@@ -26,6 +26,8 @@ import java.nio.ByteBuffer;
import org.apache.iceberg.mumbling.MumblingTestUtil;
import org.apache.iceberg.types.Types;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
class TestManifestInfoStruct {
@@ -44,7 +46,8 @@ class TestManifestInfoStruct {
200L,
100L,
5L,
- MumblingTestUtil.onlyFirstBitSetBytes());
+ MumblingTestUtil.onlyFirstBitSetBytes(),
+ 4);
assertThat(info.addedFilesCount()).isEqualTo(10);
assertThat(info.existingFilesCount()).isEqualTo(20);
@@ -60,6 +63,7 @@ class TestManifestInfoStruct {
assertThat(info.manifestDeletionVector().buffer())
.isEqualTo(ByteBuffer.wrap(MumblingTestUtil.onlyFirstBitSetBytes()));
assertThat(info.manifestDeletionVector().cardinality()).isEqualTo(1);
+ assertThat(info.formatVersion()).isEqualTo(4);
}
@Test
@@ -78,6 +82,7 @@ class TestManifestInfoStruct {
.modifiedRowsCount(100L)
.minSequenceNumber(5L)
.dv(ByteBuffer.wrap(MumblingTestUtil.onlyFirstBitSetBytes()))
+ .formatVersion(4)
.build();
ManifestInfoStruct copy = info.copy();
@@ -93,6 +98,7 @@ class TestManifestInfoStruct {
assertThat(copy.replacedRowsCount()).isEqualTo(200L);
assertThat(copy.modifiedRowsCount()).isEqualTo(100L);
assertThat(copy.minSequenceNumber()).isEqualTo(5L);
+ assertThat(copy.formatVersion()).isEqualTo(4);
// verify deep copy of dv byte array
assertThat(copy.manifestDeletionVector().buffer().array())
@@ -114,6 +120,7 @@ class TestManifestInfoStruct {
.replacedRowsCount(0L)
.modifiedRowsCount(0L)
.minSequenceNumber(0L)
+ .formatVersion(4)
.build();
assertThat(info.manifestDeletionVector()).isNull();
@@ -155,6 +162,7 @@ class TestManifestInfoStruct {
.modifiedRowsCount(100L)
.minSequenceNumber(5L)
.dv(ByteBuffer.wrap(MumblingTestUtil.onlyFirstBitSetBytes()))
+ .formatVersion(4)
.build();
ManifestBitmap mdv = info.manifestDeletionVector();
@@ -173,6 +181,7 @@ class TestManifestInfoStruct {
assertThat(info.replacedRowsCount()).isEqualTo(200L);
assertThat(info.modifiedRowsCount()).isEqualTo(100L);
assertThat(info.minSequenceNumber()).isEqualTo(5L);
+ assertThat(info.formatVersion()).isEqualTo(4);
assertThat(info.manifestDeletionVector()).isSameAs(mdv);
}
@@ -192,6 +201,7 @@ class TestManifestInfoStruct {
.modifiedRowsCount(100L)
.minSequenceNumber(5L)
.dv(ByteBuffer.wrap(MumblingTestUtil.onlyFirstBitSetBytes()))
+ .formatVersion(4)
.build();
ManifestInfoStruct deserialized = TestHelpers.roundTripSerialize(info);
@@ -207,6 +217,7 @@ class TestManifestInfoStruct {
assertThat(deserialized.replacedRowsCount()).isEqualTo(200L);
assertThat(deserialized.modifiedRowsCount()).isEqualTo(100L);
assertThat(deserialized.minSequenceNumber()).isEqualTo(5L);
+ assertThat(deserialized.formatVersion()).isEqualTo(4);
assertThat(deserialized.manifestDeletionVector().buffer())
.isEqualTo(ByteBuffer.wrap(MumblingTestUtil.onlyFirstBitSetBytes()));
}
@@ -524,6 +535,7 @@ class TestManifestInfoStruct {
.replacedRowsCount(0L)
.modifiedRowsCount(0L)
.minSequenceNumber(0L)
+ .formatVersion(4)
.build())
.isInstanceOf(IllegalArgumentException.class)
.hasMessage("Invalid added counts: 10 rows in 0 files");
@@ -542,6 +554,7 @@ class TestManifestInfoStruct {
.replacedRowsCount(0L)
.modifiedRowsCount(0L)
.minSequenceNumber(0L)
+ .formatVersion(4)
.build())
.isInstanceOf(IllegalArgumentException.class)
.hasMessage("Invalid existing counts: 5 rows in 0 files");
@@ -560,6 +573,7 @@ class TestManifestInfoStruct {
.replacedRowsCount(0L)
.modifiedRowsCount(0L)
.minSequenceNumber(0L)
+ .formatVersion(4)
.build())
.isInstanceOf(IllegalArgumentException.class)
.hasMessage("Invalid deleted counts: 3 rows in 0 files");
@@ -578,6 +592,7 @@ class TestManifestInfoStruct {
.replacedRowsCount(7L)
.modifiedRowsCount(0L)
.minSequenceNumber(0L)
+ .formatVersion(4)
.build())
.isInstanceOf(IllegalArgumentException.class)
.hasMessage("Invalid replaced counts: 7 rows in 0 files");
@@ -596,6 +611,7 @@ class TestManifestInfoStruct {
.replacedRowsCount(0L)
.modifiedRowsCount(4L)
.minSequenceNumber(0L)
+ .formatVersion(4)
.build())
.isInstanceOf(IllegalArgumentException.class)
.hasMessage("Invalid modified counts: 4 rows in 0 files");
@@ -616,6 +632,7 @@ class TestManifestInfoStruct {
.replacedRowsCount(0L)
.modifiedRowsCount(0L)
.minSequenceNumber(0L)
+ .formatVersion(4)
.build();
assertThat(info.addedFilesCount()).isEqualTo(5);
@@ -646,6 +663,7 @@ class TestManifestInfoStruct {
.modifiedRowsCount(100L)
.minSequenceNumber(5L)
.dv(ByteBuffer.wrap(MumblingTestUtil.onlyFirstBitSetBytes()))
+ .formatVersion(4)
.build();
ManifestInfoStruct deserialized =
TestHelpers.KryoHelpers.roundTripSerialize(info);
@@ -661,7 +679,59 @@ class TestManifestInfoStruct {
assertThat(deserialized.replacedRowsCount()).isEqualTo(200L);
assertThat(deserialized.modifiedRowsCount()).isEqualTo(100L);
assertThat(deserialized.minSequenceNumber()).isEqualTo(5L);
+ assertThat(deserialized.formatVersion()).isEqualTo(4);
assertThat(deserialized.manifestDeletionVector().buffer())
.isEqualTo(ByteBuffer.wrap(MumblingTestUtil.onlyFirstBitSetBytes()));
}
+
+ @Test
+ void builderMissingFormatVersion() {
+ assertThatThrownBy(
+ () ->
+ ManifestInfoStruct.builder()
+ .addedFilesCount(0)
+ .existingFilesCount(0)
+ .deletedFilesCount(0)
+ .replacedFilesCount(0)
+ .modifiedFilesCount(0)
+ .addedRowsCount(0L)
+ .existingRowsCount(0L)
+ .deletedRowsCount(0L)
+ .replacedRowsCount(0L)
+ .modifiedRowsCount(0L)
+ .minSequenceNumber(0L)
+ .build())
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Missing required value: format version");
+ }
+
+ @ParameterizedTest
+ @ValueSource(ints = {-1, 1, 2, 3, 5})
+ void builderRejectsUndefinedFormatVersion(int version) {
+ assertThatThrownBy(() ->
ManifestInfoStruct.builder().formatVersion(version))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Invalid format version: " + version + " (must be 0 or
4)");
+ }
+
+ @ParameterizedTest
+ @ValueSource(ints = {0, 4})
+ void builderAcceptsValidFormatVersion(int version) {
+ ManifestInfoStruct info =
+ ManifestInfoStruct.builder()
+ .addedFilesCount(0)
+ .existingFilesCount(0)
+ .deletedFilesCount(0)
+ .replacedFilesCount(0)
+ .modifiedFilesCount(0)
+ .addedRowsCount(0L)
+ .existingRowsCount(0L)
+ .deletedRowsCount(0L)
+ .replacedRowsCount(0L)
+ .modifiedRowsCount(0L)
+ .minSequenceNumber(0L)
+ .formatVersion(version)
+ .build();
+
+ assertThat(info.formatVersion()).isEqualTo(version);
+ }
}
diff --git a/core/src/test/java/org/apache/iceberg/TestTrackedFile.java
b/core/src/test/java/org/apache/iceberg/TestTrackedFile.java
index 9a32243cc1..e0a1ef19cc 100644
--- a/core/src/test/java/org/apache/iceberg/TestTrackedFile.java
+++ b/core/src/test/java/org/apache/iceberg/TestTrackedFile.java
@@ -48,7 +48,6 @@ public class TestTrackedFile {
.containsExactly(
"tracking",
"content_type",
- "format_version",
"location",
"file_format",
"record_count",
@@ -71,8 +70,7 @@ public class TestTrackedFile {
assertThat(fields)
.extracting(Types.NestedField::fieldId)
- .containsExactly(
- 147, 134, 157, 100, 101, 103, 104, 141, 102, 146, 140, 148, 150,
131, 132, 135);
+ .containsExactly(147, 134, 100, 101, 103, 104, 141, 102, 146, 140,
148, 150, 131, 132, 135);
}
@Test
diff --git a/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java
b/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java
index d3af965ec1..31d6758b80 100644
--- a/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java
+++ b/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java
@@ -123,6 +123,7 @@ class TestTrackedFileAdapters {
.modifiedRowsCount(0L)
.minSequenceNumber(7L)
.dv(ByteBuffer.wrap(MumblingTestUtil.onlyFirstBitSetBytes()))
+ .formatVersion(FORMAT_VERSION_V4)
.build();
@Test
@@ -145,7 +146,6 @@ class TestTrackedFileAdapters {
new TrackedFileStruct(
tracking,
FileContent.DATA,
- FORMAT_VERSION_V4,
DATA_FILE_LOCATION,
FileFormat.PARQUET,
100L,
@@ -225,7 +225,6 @@ class TestTrackedFileAdapters {
new TrackedFileStruct(
tracking,
FileContent.EQUALITY_DELETES,
- FORMAT_VERSION_V4,
"s3://bucket/eq-delete.avro",
FileFormat.AVRO,
50L,
@@ -314,7 +313,6 @@ class TestTrackedFileAdapters {
new TrackedFileStruct(
tracking,
FileContent.DATA,
- FORMAT_VERSION_V4,
DATA_FILE_LOCATION,
FileFormat.PARQUET,
100L,
@@ -446,7 +444,6 @@ class TestTrackedFileAdapters {
return new TrackedFileStruct(
tracking,
FileContent.DATA,
- FORMAT_VERSION_V4,
DATA_FILE_LOCATION,
FileFormat.PARQUET,
100L, // recordCount
@@ -490,7 +487,6 @@ class TestTrackedFileAdapters {
new TrackedFileStruct(
MANIFEST_TRACKING,
contentType,
- FORMAT_VERSION_V4,
MANIFEST_LOCATION,
FileFormat.PARQUET,
10L, // recordCount
@@ -597,7 +593,6 @@ class TestTrackedFileAdapters {
new TrackedFileStruct(
null,
FileContent.DATA,
- 0,
null,
null,
0L,
@@ -651,7 +646,6 @@ class TestTrackedFileAdapters {
new TrackedFileStruct(
null,
FileContent.DATA,
- 0,
null,
null,
0L,
@@ -695,7 +689,6 @@ class TestTrackedFileAdapters {
return new TrackedFileStruct(
null,
contentType,
- FORMAT_VERSION_V4,
DATA_FILE_LOCATION,
FileFormat.PARQUET,
1L,
diff --git a/core/src/test/java/org/apache/iceberg/TestTrackedFileStruct.java
b/core/src/test/java/org/apache/iceberg/TestTrackedFileStruct.java
index f5edda5567..d5e31215b8 100644
--- a/core/src/test/java/org/apache/iceberg/TestTrackedFileStruct.java
+++ b/core/src/test/java/org/apache/iceberg/TestTrackedFileStruct.java
@@ -40,8 +40,6 @@ import org.junit.jupiter.params.provider.MethodSource;
import org.mockito.Mockito;
class TestTrackedFileStruct {
- private static final int FORMAT_VERSION_V4 = 4;
-
private static final List<Types.NestedField> DEFAULT_FIELDS =
TrackedFile.schema(Types.StructType.of(),
Types.StructType.of()).asStruct().fields();
@@ -72,7 +70,6 @@ class TestTrackedFileStruct {
new TrackedFileStruct(
TRACKING,
FileContent.DATA,
- FORMAT_VERSION_V4,
"s3://bucket/data/00000-0-file.parquet",
FileFormat.PARQUET,
50L,
@@ -89,7 +86,6 @@ class TestTrackedFileStruct {
assertThat(file.tracking()).isSameAs(TRACKING);
assertThat(file.contentType()).isEqualTo(FileContent.DATA);
- assertThat(file.formatVersion()).isEqualTo(FORMAT_VERSION_V4);
assertThat(file.location()).isEqualTo("s3://bucket/data/00000-0-file.parquet");
assertThat(file.fileFormat()).isEqualTo(FileFormat.PARQUET);
assertThat(file.partition()).isSameAs(PARTITION);
@@ -110,7 +106,6 @@ class TestTrackedFileStruct {
TrackedFileStruct file = new TrackedFileStruct();
file.set(pos("tracking"), TRACKING);
file.set(pos("content_type"), FileContent.DATA.id());
- file.set(pos("format_version"), FORMAT_VERSION_V4);
file.set(pos("location"), "s3://bucket/data/00000-0-file.parquet");
file.set(pos("file_format"), "parquet");
file.set(pos("record_count"), 50L);
@@ -127,7 +122,6 @@ class TestTrackedFileStruct {
assertThat(file.tracking()).isSameAs(TRACKING);
assertThat(file.contentType()).isEqualTo(FileContent.DATA);
- assertThat(file.formatVersion()).isEqualTo(FORMAT_VERSION_V4);
assertThat(file.location()).isEqualTo("s3://bucket/data/00000-0-file.parquet");
assertThat(file.fileFormat()).isEqualTo(FileFormat.PARQUET);
assertThat(file.recordCount()).isEqualTo(50L);
@@ -149,7 +143,6 @@ class TestTrackedFileStruct {
new TrackedFileStruct(
TRACKING,
FileContent.DATA,
- FORMAT_VERSION_V4,
"s3://bucket/data/00000-0-file.parquet",
FileFormat.PARQUET,
50L,
@@ -166,7 +159,6 @@ class TestTrackedFileStruct {
assertThat(file.get(pos("tracking"), Tracking.class)).isSameAs(TRACKING);
assertThat(file.get(pos("content_type"),
Integer.class)).isEqualTo(FileContent.DATA.id());
- assertThat(file.get(pos("format_version"),
Integer.class)).isEqualTo(FORMAT_VERSION_V4);
assertThat(file.get(pos("location"), String.class))
.isEqualTo("s3://bucket/data/00000-0-file.parquet");
assertThat(file.get(pos("file_format"),
String.class)).isEqualTo(FileFormat.PARQUET.toString());
@@ -190,7 +182,6 @@ class TestTrackedFileStruct {
new TrackedFileStruct(
TRACKING,
FileContent.DATA,
- FORMAT_VERSION_V4,
"s3://bucket/data/00000-0-file.parquet",
FileFormat.PARQUET,
50L,
@@ -210,7 +201,6 @@ class TestTrackedFileStruct {
assertThat(copy).isInstanceOf(TrackedFileStruct.class);
assertThat(copy.tracking()).isSameAs(TRACKING_COPY);
assertThat(copy.contentType()).isEqualTo(FileContent.DATA);
- assertThat(copy.formatVersion()).isEqualTo(FORMAT_VERSION_V4);
assertThat(copy.location()).isEqualTo("s3://bucket/data/00000-0-file.parquet");
assertThat(copy.fileFormat()).isEqualTo(FileFormat.PARQUET);
assertThat(copy.recordCount()).isEqualTo(50L);
@@ -239,7 +229,6 @@ class TestTrackedFileStruct {
new TrackedFileStruct(
TRACKING,
FileContent.DATA,
- FORMAT_VERSION_V4,
"s3://bucket/data/00000-0-file.parquet",
FileFormat.PARQUET,
50L,
@@ -259,7 +248,6 @@ class TestTrackedFileStruct {
assertThat(copy).isInstanceOf(TrackedFileStruct.class);
assertThat(copy.tracking()).isSameAs(TRACKING_COPY);
assertThat(copy.contentType()).isEqualTo(FileContent.DATA);
- assertThat(copy.formatVersion()).isEqualTo(FORMAT_VERSION_V4);
assertThat(copy.location()).isEqualTo("s3://bucket/data/00000-0-file.parquet");
assertThat(copy.fileFormat()).isEqualTo(FileFormat.PARQUET);
assertThat(copy.recordCount()).isEqualTo(50L);
@@ -286,7 +274,6 @@ class TestTrackedFileStruct {
new TrackedFileStruct(
TRACKING,
FileContent.DATA,
- FORMAT_VERSION_V4,
"s3://bucket/data/00000-0-file.parquet",
FileFormat.PARQUET,
50L,
@@ -309,7 +296,6 @@ class TestTrackedFileStruct {
assertThat(copy).isInstanceOf(TrackedFileStruct.class);
assertThat(copy.tracking()).isSameAs(TRACKING_COPY);
assertThat(copy.contentType()).isEqualTo(FileContent.DATA);
- assertThat(copy.formatVersion()).isEqualTo(FORMAT_VERSION_V4);
assertThat(copy.location()).isEqualTo("s3://bucket/data/00000-0-file.parquet");
assertThat(copy.fileFormat()).isEqualTo(FileFormat.PARQUET);
assertThat(copy.recordCount()).isEqualTo(50L);
@@ -436,7 +422,6 @@ class TestTrackedFileStruct {
new TrackedFileStruct(
null, // TrackingStruct has its own serialization tests
FileContent.DATA,
- FORMAT_VERSION_V4,
"s3://bucket/data/file.parquet",
FileFormat.PARQUET,
100L,
@@ -455,7 +440,6 @@ class TestTrackedFileStruct {
assertThat(deserialized.tracking()).isNull();
assertThat(deserialized.contentType()).isEqualTo(FileContent.DATA);
- assertThat(deserialized.formatVersion()).isEqualTo(FORMAT_VERSION_V4);
assertThat(deserialized.location()).isEqualTo("s3://bucket/data/file.parquet");
assertThat(deserialized.fileFormat()).isEqualTo(FileFormat.PARQUET);
assertThat(deserialized.partition())
@@ -513,7 +497,6 @@ class TestTrackedFileStruct {
return new TrackedFileStruct(
null, // tracking
FileContent.DATA,
- FORMAT_VERSION_V4,
"s3://bucket/file.parquet",
FileFormat.PARQUET,
100L, // recordCount
diff --git a/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java
b/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java
index f2c17bb552..6db0059742 100644
--- a/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java
+++ b/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java
@@ -82,7 +82,7 @@ class TestV4ManifestReader {
private static final DeletionVector DV =
deletionVector("s3://bucket/dv.puffin");
private static final ManifestInfo MANIFEST_INFO =
- new ManifestInfoStruct(49, 51, 0, 0, 1, 4_900L, 5_100L, 0L, 0L, 100L,
1L, null);
+ new ManifestInfoStruct(49, 51, 0, 0, 1, 4_900L, 5_100L, 0L, 0L, 100L,
1L, null, 4);
private static final Schema TABLE_SCHEMA =
new Schema(
@@ -168,7 +168,6 @@ class TestV4ManifestReader {
new TrackedFileStruct(
ADDED_TRACKING,
FileContent.DATA,
- FORMAT_VERSION_V4,
"s3://bucket/data/file.parquet",
FileFormat.PARQUET,
RECORD_COUNT,
@@ -200,7 +199,6 @@ class TestV4ManifestReader {
new TrackedFileStruct(
ADDED_TRACKING,
FileContent.DATA,
- FORMAT_VERSION_V4,
"s3://bucket/data/file.parquet",
FileFormat.PARQUET,
RECORD_COUNT,
@@ -234,7 +232,6 @@ class TestV4ManifestReader {
new TrackedFileStruct(
ADDED_TRACKING,
FileContent.DATA,
- FORMAT_VERSION_V4,
"s3://bucket/data/file.parquet",
FileFormat.PARQUET,
RECORD_COUNT,
@@ -270,7 +267,6 @@ class TestV4ManifestReader {
new TrackedFileStruct(
ADDED_TRACKING,
FileContent.EQUALITY_DELETES,
- FORMAT_VERSION_V4,
"s3://bucket/eq-delete.parquet",
FileFormat.PARQUET,
RECORD_COUNT,
@@ -302,7 +298,6 @@ class TestV4ManifestReader {
new TrackedFileStruct(
ADDED_TRACKING,
FileContent.DATA_MANIFEST,
- FORMAT_VERSION_V4,
"s3://bucket/leaf-manifest.parquet",
FileFormat.PARQUET,
RECORD_COUNT,
@@ -1163,7 +1158,6 @@ class TestV4ManifestReader {
new TrackedFileStruct(
ADDED_TRACKING,
FileContent.DATA,
- FORMAT_VERSION_V4,
"s3://bucket/table/empty-file.parquet",
FileFormat.PARQUET,
0, // file contains no records
@@ -1200,7 +1194,6 @@ class TestV4ManifestReader {
new TrackedFileStruct(
ADDED_TRACKING,
FileContent.DATA,
- FORMAT_VERSION_V4,
"s3://bucket/table/very-old.avro",
FileFormat.AVRO,
-1, // mimic invalid record count in old Avro metadata
@@ -1385,7 +1378,6 @@ class TestV4ManifestReader {
new TrackedFileStruct(
ADDED_TRACKING,
FileContent.DATA,
- FORMAT_VERSION_V4,
"s3://bucket/table/bucket=7/data-file.parquet",
FileFormat.AVRO,
10,
diff --git a/core/src/test/java/org/apache/iceberg/V4TestComparators.java
b/core/src/test/java/org/apache/iceberg/V4TestComparators.java
index 08ea44ed35..088604b5a5 100644
--- a/core/src/test/java/org/apache/iceberg/V4TestComparators.java
+++ b/core/src/test/java/org/apache/iceberg/V4TestComparators.java
@@ -55,7 +55,6 @@ class V4TestComparators {
return Comparator.nullsFirst(
Comparator.comparing(TrackedFile::tracking, trackingComparator)
.thenComparing(TrackedFile::contentType, natural())
- .thenComparingInt(TrackedFile::formatVersion)
.thenComparing(TrackedFile::location, natural())
.thenComparing(TrackedFile::fileFormat, natural())
.thenComparingLong(TrackedFile::recordCount)
@@ -121,7 +120,8 @@ class V4TestComparators {
.thenComparingLong(ManifestInfo::replacedRowsCount)
.thenComparingLong(ManifestInfo::modifiedRowsCount)
.thenComparingLong(ManifestInfo::minSequenceNumber)
- .thenComparing(ManifestInfo::manifestDeletionVector, BITMAPS));
+ .thenComparing(ManifestInfo::manifestDeletionVector, BITMAPS)
+ .thenComparingInt(ManifestInfo::formatVersion));
private static final Comparator<ContentStats> CONTENT_STATS =
Comparator.nullsFirst(new ContentStatsComparator());
diff --git a/core/src/test/java/org/apache/iceberg/V4TestHelpers.java
b/core/src/test/java/org/apache/iceberg/V4TestHelpers.java
index 1b922b576c..e9db4950bd 100644
--- a/core/src/test/java/org/apache/iceberg/V4TestHelpers.java
+++ b/core/src/test/java/org/apache/iceberg/V4TestHelpers.java
@@ -119,7 +119,6 @@ class V4TestHelpers {
return new TrackedFileStruct(
tracking,
content,
- FORMAT_VERSION_V4,
location,
FileFormat.fromFileName(location),
RECORD_COUNT,
