amogh-jahagirdar commented on code in PR #19007:
URL: https://github.com/apache/iceberg/pull/19007#discussion_r4600917601
##########
core/src/main/java/org/apache/iceberg/mumbling/MumblingBitmaps.java:
##########
@@ -33,4 +34,35 @@ private MumblingBitmaps() {}
public static ManifestBitmap read(ByteBuffer buffer) {
return new MumblingBitmap(buffer);
}
+
+ /**
+ * Create a new empty mutable bitmap.
+ *
+ * @return a new mutable bitmap with no bits set
+ */
+ public static MutableBitmap newMutableBitmap() {
+ return MumblingBitmaps.newMutableBitmap();
Review Comment:
`MumblingBitmap.newMutableBitmap()`? (map vs maps)
##########
core/src/test/java/org/apache/iceberg/TestV4ManifestWriter.java:
##########
@@ -546,6 +620,379 @@ public void rootDataFileFirstRowIdAssignment() throws
IOException {
}
}
+ @Test
+ public void rootManifestFileAppend() throws IOException {
+ OutputFile output = new InMemoryOutputFile("manifest.parquet");
+ V4ManifestWriter writer =
+ V4ManifestWriter.createRoot(
+ EncryptionUtil.plainAsEncryptedOutput(output),
+ Map.of(),
+ STATS_TYPE,
+ Map.of(),
+ SCHEMA.schemaId(),
+ COMMIT_SNAPSHOT_ID,
+ COMMIT_SEQUENCE_NUMBER,
+ COMMIT_FIRST_ROW_ID);
+ try (writer) {
+ writer.append(LEAF_MANIFEST);
+ }
+
+ ManifestInfo summary = LEAF_MANIFEST.manifestInfo();
+ long leafRowsAssigned =
+ summary.addedRowsCount() + summary.existingRowsCount() +
summary.modifiedRowsCount();
+ assertThat(writer.assignedRowIds()).isEqualTo(leafRowsAssigned);
+
+ InputFile in = output.toInputFile();
+ for (TrackedFile file : List.of(readOne(writer, in), rawReadOne(in))) {
+ assertThat(file)
+ .usingComparator(V4TestComparators.trackedFileWithoutStatus())
+ .isEqualTo(LEAF_MANIFEST);
+ assertThat(file.tracking())
+ .usingComparator(V4TestComparators.TRACKING)
+ .as("Tracking uses the commit metadata")
+ .isEqualTo(
+ new TrackingStruct(
+ EntryStatus.ADDED,
+ COMMIT_SNAPSHOT_ID,
+ COMMIT_SEQUENCE_NUMBER,
+ COMMIT_SEQUENCE_NUMBER,
+ null, // not modified
+ COMMIT_FIRST_ROW_ID,
+ null, // no change bitmaps
+ null));
+ }
+ }
+
+ @Test
+ public void rootManifestFileAppendAtSeq() throws IOException {
+ V4ManifestWriter root =
+ V4ManifestWriter.createRoot(
+ EncryptionUtil.plainAsEncryptedOutput(new
InMemoryOutputFile("root.parquet")),
+ Map.of(),
+ STATS_TYPE,
+ Map.of(),
+ SCHEMA.schemaId(),
+ COMMIT_SNAPSHOT_ID,
+ COMMIT_SEQUENCE_NUMBER,
+ COMMIT_FIRST_ROW_ID);
+
+ try (root) {
+ assertThatThrownBy(() -> root.appendAtSeq(LEAF_MANIFEST, 33L))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Cannot append DATA_MANIFEST at a data sequence number");
+ }
+ }
+
+ @Test
+ public void rootManifestFileWithMDVAppend() throws IOException {
+ OutputFile output = new InMemoryOutputFile("manifest.parquet");
+ V4ManifestWriter writer =
+ V4ManifestWriter.createRoot(
+ EncryptionUtil.plainAsEncryptedOutput(output),
+ SPECS,
+ STATS_TYPE,
+ Map.of(),
+ SCHEMA.schemaId(),
+ COMMIT_SNAPSHOT_ID,
+ COMMIT_SEQUENCE_NUMBER,
+ COMMIT_FIRST_ROW_ID);
+ try (writer) {
+ writer.append(LEAF_MANIFEST_WITH_MDV);
+ }
+
+ ManifestInfo summary = LEAF_MANIFEST_WITH_MDV.manifestInfo();
+ long leafRowsAssigned =
+ summary.addedRowsCount() + summary.existingRowsCount() +
summary.modifiedRowsCount();
+ assertThat(writer.assignedRowIds()).isEqualTo(leafRowsAssigned);
+
+ InputFile in = output.toInputFile();
+ for (TrackedFile file : List.of(readOne(writer, in), rawReadOne(in))) {
+ assertThat(file)
+ .usingComparator(V4TestComparators.trackedFileWithoutStatus())
+ .isEqualTo(LEAF_MANIFEST_WITH_MDV);
+ assertThat(file.tracking())
+ .usingComparator(V4TestComparators.TRACKING)
+ .as("Tracking uses the commit metadata")
+ .isEqualTo(
+ new TrackingStruct(
+ EntryStatus.ADDED,
+ COMMIT_SNAPSHOT_ID,
+ COMMIT_SEQUENCE_NUMBER,
+ COMMIT_SEQUENCE_NUMBER,
+ null, // not modified
+ COMMIT_FIRST_ROW_ID,
+ null, // no change bitmaps
+ null));
+ }
+ }
+
+ @Test
+ public void rootManifestFileExisting() throws IOException {
+ OutputFile output = new InMemoryOutputFile("manifest.parquet");
+ V4ManifestWriter writer =
+ V4ManifestWriter.createRoot(
+ EncryptionUtil.plainAsEncryptedOutput(output),
+ SPECS,
+ STATS_TYPE,
+ Map.of(),
+ SCHEMA.schemaId(),
+ COMMIT_SNAPSHOT_ID,
+ COMMIT_SEQUENCE_NUMBER,
+ COMMIT_FIRST_ROW_ID);
+ try (writer) {
+ writer.existing(LEAF_MANIFEST_WITH_MDV);
+ }
+
+ assertThat(writer.assignedRowIds()).isEqualTo(0L);
+
+ InputFile in = output.toInputFile();
+ for (TrackedFile file : List.of(readOne(writer, in), rawReadOne(in))) {
+ assertThat(file)
+ .usingComparator(V4TestComparators.trackedFileWithoutStatus())
+ .isEqualTo(LEAF_MANIFEST_WITH_MDV);
+ Tracking originalTracking = LEAF_MANIFEST_WITH_MDV.tracking();
+ assertThat(file.tracking())
+ .usingComparator(V4TestComparators.TRACKING)
+ .as("Tracking uses the original metadata")
+ .isEqualTo(
+ new TrackingStruct(
+ EntryStatus.EXISTING,
+ originalTracking.snapshotId(),
+ originalTracking.dataSequenceNumber(),
+ originalTracking.fileSequenceNumber(),
+ originalTracking.modifiedSnapshotId(),
+ originalTracking.firstRowId(),
+ null, // no change bitmaps
+ null));
+ }
+ }
+
+ @Test
+ public void rootManifestFileDelete() throws IOException {
+ OutputFile output = new InMemoryOutputFile("manifest.parquet");
+ V4ManifestWriter writer =
+ V4ManifestWriter.createRoot(
+ EncryptionUtil.plainAsEncryptedOutput(output),
+ Map.of(),
+ STATS_TYPE,
+ Map.of(),
+ SCHEMA.schemaId(),
+ COMMIT_SNAPSHOT_ID,
+ COMMIT_SEQUENCE_NUMBER,
+ COMMIT_FIRST_ROW_ID);
+ try (writer) {
+ writer.delete(LEAF_MANIFEST);
+ }
+
+ assertThat(writer.assignedRowIds()).isEqualTo(0L);
+
+ assertThat(read(writer, output.toInputFile())).isEmpty();
+
+ TrackedFile file = rawReadOne(output.toInputFile());
+ assertThat(file)
+ .usingComparator(V4TestComparators.trackedFileWithoutStatus())
+ .isEqualTo(LEAF_MANIFEST);
+ Tracking originalTracking = LEAF_MANIFEST.tracking();
+ assertThat(file.tracking())
+ .usingComparator(V4TestComparators.TRACKING)
+ .as("Deleted entry preserves original metadata, sets snapshot ID")
+ .isEqualTo(
+ new TrackingStruct(
+ EntryStatus.DELETED,
+ COMMIT_SNAPSHOT_ID,
+ originalTracking.dataSequenceNumber(),
+ originalTracking.fileSequenceNumber(),
+ originalTracking.modifiedSnapshotId(),
+ originalTracking.firstRowId(),
+ null, // no change bitmaps
+ null));
+ }
+
+ @Test
+ public void rootManifestFileReplace() throws IOException {
+ OutputFile output = new InMemoryOutputFile("manifest.parquet");
+ V4ManifestWriter writer =
+ V4ManifestWriter.createRoot(
+ EncryptionUtil.plainAsEncryptedOutput(output),
+ Map.of(),
+ STATS_TYPE,
+ Map.of(),
+ SCHEMA.schemaId(),
+ COMMIT_SNAPSHOT_ID,
+ COMMIT_SEQUENCE_NUMBER,
+ COMMIT_FIRST_ROW_ID);
+ try (writer) {
+ writer.replace(LEAF_MANIFEST_WITH_MDV);
+ }
+
+ assertThat(writer.assignedRowIds()).isEqualTo(0L);
+
+ assertThat(read(writer, output.toInputFile())).isEmpty();
+
+ TrackedFile file = rawReadOne(output.toInputFile());
+ assertThat(file)
+ .usingComparator(V4TestComparators.trackedFileWithoutStatus())
+ .isEqualTo(LEAF_MANIFEST_WITH_MDV);
+ Tracking originalTracking = LEAF_MANIFEST_WITH_MDV.tracking();
+ assertThat(file.tracking())
+ .usingComparator(V4TestComparators.TRACKING)
+ .as("Replaced entry preserves original metadata, sets snapshot ID")
+ .isEqualTo(
+ new TrackingStruct(
+ EntryStatus.REPLACED,
+ COMMIT_SNAPSHOT_ID,
+ originalTracking.dataSequenceNumber(),
+ originalTracking.fileSequenceNumber(),
+ originalTracking.modifiedSnapshotId(),
+ originalTracking.firstRowId(),
+ null, // no change bitmaps
+ null));
+ }
+
+ @Test
+ public void rootManifestFileWithDeletionVector() throws IOException {
+ V4ManifestWriter writer =
+ V4ManifestWriter.createRoot(
+ EncryptionUtil.plainAsEncryptedOutput(new
InMemoryOutputFile("manifest.parquet")),
+ SPECS,
+ STATS_TYPE,
+ Map.of(),
+ SCHEMA.schemaId(),
+ COMMIT_SNAPSHOT_ID,
+ COMMIT_SEQUENCE_NUMBER,
+ COMMIT_FIRST_ROW_ID);
+
+ try (writer) {
+ assertThatThrownBy(() -> writer.modify(LEAF_MANIFEST, DV_A))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("Invalid manifest, data deletion vector must
be null");
+ }
+ }
+
+ @Test
+ public void rootMixedDataAndManifestFiles() throws IOException {
+ OutputFile output = new InMemoryOutputFile("manifest.parquet");
+ V4ManifestWriter writer =
+ V4ManifestWriter.createRoot(
+ EncryptionUtil.plainAsEncryptedOutput(output),
+ SPECS,
+ STATS_TYPE,
+ Map.of(),
+ SCHEMA.schemaId(),
+ COMMIT_SNAPSHOT_ID,
+ COMMIT_SEQUENCE_NUMBER,
+ COMMIT_FIRST_ROW_ID);
+ try (writer) {
+ writer.existing(FILE_A);
+ writer.append(FILE_B);
+ writer.existing(LEAF_MANIFEST_WITH_MDV);
+ }
+
+ assertThat(writer.assignedRowIds()).isEqualTo(FILE_B.recordCount());
+
+ List<TrackedFile> files = read(writer, output.toInputFile());
+ assertThat(files)
+ .usingComparatorForType(
+
V4TestComparators.trackedFileWithoutStatus(TRUNCATE_SPEC.partitionType()),
+ TrackedFile.class)
+ .containsExactly(FILE_A, FILE_B, LEAF_MANIFEST_WITH_MDV);
+ assertThat(files.get(0).tracking())
+ .usingComparator(V4TestComparators.TRACKING)
+ .as("Tracking should preserve FILE_A metadata")
+ .isEqualTo(FILE_A.tracking());
+ assertThat(files.get(1).tracking())
+ .usingComparator(V4TestComparators.TRACKING)
+ .as("Tracking uses the commit metadata for FILE_B")
+ .isEqualTo(
+ new TrackingStruct(
+ EntryStatus.ADDED,
+ COMMIT_SNAPSHOT_ID,
+ COMMIT_SEQUENCE_NUMBER,
+ COMMIT_SEQUENCE_NUMBER,
+ null, // not modified
+ COMMIT_FIRST_ROW_ID,
+ null, // no change bitmaps
+ null));
+ assertThat(files.get(2).tracking())
Review Comment:
Can we also check `rawRead` here like the other root tests? For the root, we
want to verify that `first_row_id` is actually physically materialized in the
file, not just that the reader returns it.
##########
core/src/test/java/org/apache/iceberg/TestV4ManifestWriter.java:
##########
@@ -546,6 +620,379 @@ public void rootDataFileFirstRowIdAssignment() throws
IOException {
}
}
+ @Test
+ public void rootManifestFileAppend() throws IOException {
Review Comment:
BTW I think the new style is that tests don't have `public` modifiers
##########
core/src/test/java/org/apache/iceberg/TestV4ManifestWriter.java:
##########
@@ -546,6 +620,379 @@ public void rootDataFileFirstRowIdAssignment() throws
IOException {
}
}
+ @Test
Review Comment:
Some other negative path tests (not sure if you intended to add these later)
related to rejecting invalid content types:
rejecting append on a delete manifest? same for all the operations but for
delete files (I don't think we can build a TrackedFile from DeleteFile so maybe
this is moot by design)?
##########
core/src/main/java/org/apache/iceberg/mumbling/MutableBitmap.java:
##########
@@ -19,12 +19,38 @@
package org.apache.iceberg.mumbling;
import org.apache.iceberg.ManifestBitmap;
+import org.apache.iceberg.util.ArrayUtil;
/** Mutation methods for ManifestBitmap. */
public interface MutableBitmap extends ManifestBitmap {
/** Set the bit at {@code position}. */
boolean set(int position);
+ /**
+ * Set multiple {@code positions} in the bitmap.
+ *
+ * @param positions positions to set in the bitmap
+ * @return true if any position was set, false otherwise
+ */
+ default boolean setAll(int... positions) {
Review Comment:
Minor: Do we need this? I think in practice we'd figure out which positions
are being removed/replaced in a manifest, put them in a collection and then
call the one below this. This one seems convenient only for testing but also
boxes every element into an integer just to loop over it again.
##########
core/src/main/java/org/apache/iceberg/V4ManifestWriter.java:
##########
Review Comment:
Do these need to be using `rowsToAssign`? For leaf manifests these counts
would just be the number of entries.
##########
core/src/test/java/org/apache/iceberg/TestV4ManifestWriter.java:
##########
@@ -546,6 +620,379 @@ public void rootDataFileFirstRowIdAssignment() throws
IOException {
}
}
+ @Test
+ public void rootManifestFileAppend() throws IOException {
+ OutputFile output = new InMemoryOutputFile("manifest.parquet");
+ V4ManifestWriter writer =
+ V4ManifestWriter.createRoot(
+ EncryptionUtil.plainAsEncryptedOutput(output),
+ Map.of(),
+ STATS_TYPE,
+ Map.of(),
+ SCHEMA.schemaId(),
+ COMMIT_SNAPSHOT_ID,
+ COMMIT_SEQUENCE_NUMBER,
+ COMMIT_FIRST_ROW_ID);
+ try (writer) {
+ writer.append(LEAF_MANIFEST);
+ }
+
+ ManifestInfo summary = LEAF_MANIFEST.manifestInfo();
+ long leafRowsAssigned =
+ summary.addedRowsCount() + summary.existingRowsCount() +
summary.modifiedRowsCount();
+ assertThat(writer.assignedRowIds()).isEqualTo(leafRowsAssigned);
+
+ InputFile in = output.toInputFile();
+ for (TrackedFile file : List.of(readOne(writer, in), rawReadOne(in))) {
+ assertThat(file)
+ .usingComparator(V4TestComparators.trackedFileWithoutStatus())
+ .isEqualTo(LEAF_MANIFEST);
+ assertThat(file.tracking())
+ .usingComparator(V4TestComparators.TRACKING)
+ .as("Tracking uses the commit metadata")
+ .isEqualTo(
+ new TrackingStruct(
+ EntryStatus.ADDED,
+ COMMIT_SNAPSHOT_ID,
+ COMMIT_SEQUENCE_NUMBER,
+ COMMIT_SEQUENCE_NUMBER,
+ null, // not modified
+ COMMIT_FIRST_ROW_ID,
+ null, // no change bitmaps
+ null));
+ }
+ }
+
+ @Test
+ public void rootManifestFileAppendAtSeq() throws IOException {
+ V4ManifestWriter root =
+ V4ManifestWriter.createRoot(
+ EncryptionUtil.plainAsEncryptedOutput(new
InMemoryOutputFile("root.parquet")),
+ Map.of(),
+ STATS_TYPE,
+ Map.of(),
+ SCHEMA.schemaId(),
+ COMMIT_SNAPSHOT_ID,
+ COMMIT_SEQUENCE_NUMBER,
+ COMMIT_FIRST_ROW_ID);
+
+ try (root) {
+ assertThatThrownBy(() -> root.appendAtSeq(LEAF_MANIFEST, 33L))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Cannot append DATA_MANIFEST at a data sequence number");
+ }
+ }
+
+ @Test
+ public void rootManifestFileWithMDVAppend() throws IOException {
+ OutputFile output = new InMemoryOutputFile("manifest.parquet");
+ V4ManifestWriter writer =
+ V4ManifestWriter.createRoot(
+ EncryptionUtil.plainAsEncryptedOutput(output),
+ SPECS,
+ STATS_TYPE,
+ Map.of(),
+ SCHEMA.schemaId(),
+ COMMIT_SNAPSHOT_ID,
+ COMMIT_SEQUENCE_NUMBER,
+ COMMIT_FIRST_ROW_ID);
+ try (writer) {
+ writer.append(LEAF_MANIFEST_WITH_MDV);
+ }
+
+ ManifestInfo summary = LEAF_MANIFEST_WITH_MDV.manifestInfo();
+ long leafRowsAssigned =
+ summary.addedRowsCount() + summary.existingRowsCount() +
summary.modifiedRowsCount();
+ assertThat(writer.assignedRowIds()).isEqualTo(leafRowsAssigned);
+
+ InputFile in = output.toInputFile();
+ for (TrackedFile file : List.of(readOne(writer, in), rawReadOne(in))) {
+ assertThat(file)
+ .usingComparator(V4TestComparators.trackedFileWithoutStatus())
+ .isEqualTo(LEAF_MANIFEST_WITH_MDV);
+ assertThat(file.tracking())
+ .usingComparator(V4TestComparators.TRACKING)
+ .as("Tracking uses the commit metadata")
+ .isEqualTo(
+ new TrackingStruct(
+ EntryStatus.ADDED,
+ COMMIT_SNAPSHOT_ID,
+ COMMIT_SEQUENCE_NUMBER,
+ COMMIT_SEQUENCE_NUMBER,
+ null, // not modified
+ COMMIT_FIRST_ROW_ID,
+ null, // no change bitmaps
+ null));
+ }
+ }
+
+ @Test
+ public void rootManifestFileExisting() throws IOException {
+ OutputFile output = new InMemoryOutputFile("manifest.parquet");
+ V4ManifestWriter writer =
+ V4ManifestWriter.createRoot(
+ EncryptionUtil.plainAsEncryptedOutput(output),
+ SPECS,
+ STATS_TYPE,
+ Map.of(),
+ SCHEMA.schemaId(),
+ COMMIT_SNAPSHOT_ID,
+ COMMIT_SEQUENCE_NUMBER,
+ COMMIT_FIRST_ROW_ID);
+ try (writer) {
+ writer.existing(LEAF_MANIFEST_WITH_MDV);
+ }
+
+ assertThat(writer.assignedRowIds()).isEqualTo(0L);
+
+ InputFile in = output.toInputFile();
+ for (TrackedFile file : List.of(readOne(writer, in), rawReadOne(in))) {
+ assertThat(file)
+ .usingComparator(V4TestComparators.trackedFileWithoutStatus())
+ .isEqualTo(LEAF_MANIFEST_WITH_MDV);
+ Tracking originalTracking = LEAF_MANIFEST_WITH_MDV.tracking();
+ assertThat(file.tracking())
+ .usingComparator(V4TestComparators.TRACKING)
+ .as("Tracking uses the original metadata")
+ .isEqualTo(
+ new TrackingStruct(
+ EntryStatus.EXISTING,
+ originalTracking.snapshotId(),
+ originalTracking.dataSequenceNumber(),
+ originalTracking.fileSequenceNumber(),
+ originalTracking.modifiedSnapshotId(),
+ originalTracking.firstRowId(),
+ null, // no change bitmaps
+ null));
+ }
+ }
+
+ @Test
+ public void rootManifestFileDelete() throws IOException {
+ OutputFile output = new InMemoryOutputFile("manifest.parquet");
+ V4ManifestWriter writer =
+ V4ManifestWriter.createRoot(
+ EncryptionUtil.plainAsEncryptedOutput(output),
+ Map.of(),
+ STATS_TYPE,
+ Map.of(),
+ SCHEMA.schemaId(),
+ COMMIT_SNAPSHOT_ID,
+ COMMIT_SEQUENCE_NUMBER,
+ COMMIT_FIRST_ROW_ID);
+ try (writer) {
+ writer.delete(LEAF_MANIFEST);
+ }
+
+ assertThat(writer.assignedRowIds()).isEqualTo(0L);
+
+ assertThat(read(writer, output.toInputFile())).isEmpty();
+
+ TrackedFile file = rawReadOne(output.toInputFile());
+ assertThat(file)
+ .usingComparator(V4TestComparators.trackedFileWithoutStatus())
+ .isEqualTo(LEAF_MANIFEST);
+ Tracking originalTracking = LEAF_MANIFEST.tracking();
+ assertThat(file.tracking())
+ .usingComparator(V4TestComparators.TRACKING)
+ .as("Deleted entry preserves original metadata, sets snapshot ID")
+ .isEqualTo(
+ new TrackingStruct(
+ EntryStatus.DELETED,
+ COMMIT_SNAPSHOT_ID,
+ originalTracking.dataSequenceNumber(),
+ originalTracking.fileSequenceNumber(),
+ originalTracking.modifiedSnapshotId(),
+ originalTracking.firstRowId(),
+ null, // no change bitmaps
+ null));
+ }
+
+ @Test
+ public void rootManifestFileReplace() throws IOException {
+ OutputFile output = new InMemoryOutputFile("manifest.parquet");
+ V4ManifestWriter writer =
+ V4ManifestWriter.createRoot(
+ EncryptionUtil.plainAsEncryptedOutput(output),
+ Map.of(),
+ STATS_TYPE,
+ Map.of(),
+ SCHEMA.schemaId(),
+ COMMIT_SNAPSHOT_ID,
+ COMMIT_SEQUENCE_NUMBER,
+ COMMIT_FIRST_ROW_ID);
+ try (writer) {
+ writer.replace(LEAF_MANIFEST_WITH_MDV);
+ }
+
+ assertThat(writer.assignedRowIds()).isEqualTo(0L);
+
+ assertThat(read(writer, output.toInputFile())).isEmpty();
+
+ TrackedFile file = rawReadOne(output.toInputFile());
+ assertThat(file)
+ .usingComparator(V4TestComparators.trackedFileWithoutStatus())
+ .isEqualTo(LEAF_MANIFEST_WITH_MDV);
+ Tracking originalTracking = LEAF_MANIFEST_WITH_MDV.tracking();
+ assertThat(file.tracking())
+ .usingComparator(V4TestComparators.TRACKING)
+ .as("Replaced entry preserves original metadata, sets snapshot ID")
+ .isEqualTo(
+ new TrackingStruct(
+ EntryStatus.REPLACED,
+ COMMIT_SNAPSHOT_ID,
+ originalTracking.dataSequenceNumber(),
+ originalTracking.fileSequenceNumber(),
+ originalTracking.modifiedSnapshotId(),
+ originalTracking.firstRowId(),
+ null, // no change bitmaps
+ null));
+ }
+
+ @Test
+ public void rootManifestFileWithDeletionVector() throws IOException {
+ V4ManifestWriter writer =
+ V4ManifestWriter.createRoot(
+ EncryptionUtil.plainAsEncryptedOutput(new
InMemoryOutputFile("manifest.parquet")),
+ SPECS,
+ STATS_TYPE,
+ Map.of(),
+ SCHEMA.schemaId(),
+ COMMIT_SNAPSHOT_ID,
+ COMMIT_SEQUENCE_NUMBER,
+ COMMIT_FIRST_ROW_ID);
+
+ try (writer) {
+ assertThatThrownBy(() -> writer.modify(LEAF_MANIFEST, DV_A))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("Invalid manifest, data deletion vector must
be null");
+ }
+ }
+
+ @Test
+ public void rootMixedDataAndManifestFiles() throws IOException {
+ OutputFile output = new InMemoryOutputFile("manifest.parquet");
+ V4ManifestWriter writer =
+ V4ManifestWriter.createRoot(
+ EncryptionUtil.plainAsEncryptedOutput(output),
+ SPECS,
+ STATS_TYPE,
+ Map.of(),
+ SCHEMA.schemaId(),
+ COMMIT_SNAPSHOT_ID,
+ COMMIT_SEQUENCE_NUMBER,
+ COMMIT_FIRST_ROW_ID);
+ try (writer) {
+ writer.existing(FILE_A);
+ writer.append(FILE_B);
+ writer.existing(LEAF_MANIFEST_WITH_MDV);
+ }
+
+ assertThat(writer.assignedRowIds()).isEqualTo(FILE_B.recordCount());
+
+ List<TrackedFile> files = read(writer, output.toInputFile());
+ assertThat(files)
+ .usingComparatorForType(
+
V4TestComparators.trackedFileWithoutStatus(TRUNCATE_SPEC.partitionType()),
+ TrackedFile.class)
+ .containsExactly(FILE_A, FILE_B, LEAF_MANIFEST_WITH_MDV);
+ assertThat(files.get(0).tracking())
+ .usingComparator(V4TestComparators.TRACKING)
+ .as("Tracking should preserve FILE_A metadata")
+ .isEqualTo(FILE_A.tracking());
+ assertThat(files.get(1).tracking())
+ .usingComparator(V4TestComparators.TRACKING)
+ .as("Tracking uses the commit metadata for FILE_B")
+ .isEqualTo(
+ new TrackingStruct(
+ EntryStatus.ADDED,
+ COMMIT_SNAPSHOT_ID,
+ COMMIT_SEQUENCE_NUMBER,
+ COMMIT_SEQUENCE_NUMBER,
+ null, // not modified
+ COMMIT_FIRST_ROW_ID,
+ null, // no change bitmaps
+ null));
+ assertThat(files.get(2).tracking())
+ .usingComparator(V4TestComparators.TRACKING)
+ .as("Tracking should preserve LEAF_MANIFEST_WITH_DV metadata")
+ .isEqualTo(LEAF_MANIFEST_WITH_MDV.tracking());
+ }
+
+ @Test
+ public void rootMixedDataAndManifestFilesFirstRowIdAssignment() throws
IOException {
Review Comment:
Are we able to have another test with mixed data files/data manifests and
delete manifests? I think that can be considered independent in that with prior
delete manifests we expect not to account for additional row IDs for that case
so a test would confirm that behavior.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]