rdblue commented on a change in pull request #1101:
URL: https://github.com/apache/iceberg/pull/1101#discussion_r467144625
##########
File path: core/src/test/java/org/apache/iceberg/TestMergeAppend.java
##########
@@ -193,46 +259,200 @@ public void testManifestMergeMinCount() throws
IOException {
.appendManifest(manifest3)
.commit();
+ Snapshot snap1 = table.currentSnapshot();
+ long commitId1 = snap1.snapshotId();
+ base = readMetadata();
+ V2Assert.assertEquals("Snapshot sequence number should be 1", 1,
snap1.sequenceNumber());
+ V2Assert.assertEquals("Last sequence number should be 1", 1,
base.lastSequenceNumber());
+ V1Assert.assertEquals("Table should end with last-sequence-number 0", 0,
base.lastSequenceNumber());
+
Assert.assertEquals("Should contain 2 merged manifest for first write",
2, readMetadata().currentSnapshot().allManifests().size());
+ validateManifest(snap1.allManifests().get(0),
+ seqs(1),
+ ids(commitId1),
+ files(FILE_A),
+ statuses(Status.ADDED));
+ validateManifest(snap1.allManifests().get(1),
+ seqs(1, 1),
+ ids(commitId1, commitId1),
+ files(FILE_C, FILE_D),
+ statuses(Status.ADDED, Status.ADDED));
table.newAppend()
.appendManifest(manifest)
.appendManifest(manifest2)
.appendManifest(manifest3)
.commit();
+ Snapshot snap2 = table.currentSnapshot();
+ long commitId2 = snap2.snapshotId();
+ base = readMetadata();
+ V2Assert.assertEquals("Snapshot sequence number should be 2", 2,
snap2.sequenceNumber());
+ V2Assert.assertEquals("Last sequence number should be 2", 2,
base.lastSequenceNumber());
+ V1Assert.assertEquals("Table should end with last-sequence-number 0", 0,
base.lastSequenceNumber());
Assert.assertEquals("Should contain 3 merged manifest for second write",
3, readMetadata().currentSnapshot().allManifests().size());
+ validateManifest(snap2.allManifests().get(0),
+ seqs(2),
+ ids(commitId2),
+ files(FILE_A),
+ statuses(Status.ADDED));
+ validateManifest(snap2.allManifests().get(1),
+ seqs(2, 2),
+ ids(commitId2, commitId2),
+ files(FILE_C, FILE_D),
+ statuses(Status.ADDED, Status.ADDED));
+ validateManifest(snap2.allManifests().get(2),
+ seqs(1, 1, 1),
+ ids(commitId1, commitId1, commitId1),
+ files(FILE_A, FILE_C, FILE_D),
+ statuses(Status.EXISTING, Status.EXISTING, Status.EXISTING));
// validate that the metadata summary is correct when using appendManifest
Assert.assertEquals("Summary metadata should include 3 added files",
"3",
readMetadata().currentSnapshot().summary().get("added-data-files"));
}
+ @Test
+ public void testManifestsMergeIntoOne() throws IOException {
+ Assert.assertEquals("Table should start empty", 0,
listManifestFiles().size());
+ table.newAppend().appendFile(FILE_A).commit();
+ Snapshot snap1 = table.currentSnapshot();
+ TableMetadata base = readMetadata();
+ V2Assert.assertEquals("Snapshot sequence number should be 1", 1,
snap1.sequenceNumber());
+ V2Assert.assertEquals("Last sequence number should be 1", 1,
base.lastSequenceNumber());
+ V1Assert.assertEquals("Table should end with last-sequence-number 0", 0,
base.lastSequenceNumber());
+ long commitId1 = snap1.snapshotId();
+
+ Assert.assertEquals("Should contain 1 manifest", 1,
snap1.allManifests().size());
+ validateManifest(snap1.allManifests().get(0), seqs(1), ids(commitId1),
files(FILE_A), statuses(Status.ADDED));
+
+ table.newAppend().appendFile(FILE_B).commit();
+ Snapshot snap2 = table.currentSnapshot();
+ long commitId2 = snap2.snapshotId();
+ base = readMetadata();
+ V2Assert.assertEquals("Snapshot sequence number should be 2", 2,
snap2.sequenceNumber());
+ V2Assert.assertEquals("Last sequence number should be 2", 2,
base.lastSequenceNumber());
+ V1Assert.assertEquals("Table should end with last-sequence-number 0", 0,
base.lastSequenceNumber());
+
+ Assert.assertEquals("Should contain 2 manifests", 2,
snap2.allManifests().size());
+ validateManifest(snap2.allManifests().get(0),
+ seqs(2),
+ ids(commitId2),
+ files(FILE_B),
+ statuses(Status.ADDED));
+ validateManifest(snap2.allManifests().get(1),
+ seqs(1),
+ ids(commitId1),
+ files(FILE_A),
+ statuses(Status.ADDED));
+
+ table.newAppend()
+ .appendManifest(writeManifest("input-m0.avro",
+ manifestEntry(ManifestEntry.Status.ADDED, null, FILE_C)))
+ .commit();
+ Snapshot snap3 = table.currentSnapshot();
+
+ base = readMetadata();
+ V2Assert.assertEquals("Snapshot sequence number should be 3", 3,
snap3.sequenceNumber());
+ V2Assert.assertEquals("Last sequence number should be 3", 3,
base.lastSequenceNumber());
+ V1Assert.assertEquals("Table should end with last-sequence-number 0", 0,
base.lastSequenceNumber());
+
+ Assert.assertEquals("Should contain 3 manifests", 3,
snap3.allManifests().size());
+ long commitId3 = snap3.snapshotId();
+ validateManifest(snap3.allManifests().get(0),
+ seqs(3),
+ ids(commitId3),
+ files(FILE_C),
+ statuses(Status.ADDED));
+ validateManifest(snap3.allManifests().get(1),
+ seqs(2),
+ ids(commitId2),
+ files(FILE_B),
+ statuses(Status.ADDED));
+ validateManifest(snap3.allManifests().get(2),
+ seqs(1),
+ ids(commitId1),
+ files(FILE_A),
+ statuses(Status.ADDED));
+
+ table.updateProperties()
+ .set(TableProperties.MANIFEST_MIN_MERGE_COUNT, "1")
+ .commit();
+
+ table.newAppend()
+ .appendManifest(writeManifest("input-m1.avro",
+ manifestEntry(ManifestEntry.Status.ADDED, null, FILE_D)))
+ .commit();
+ Snapshot snap4 = table.currentSnapshot();
+
+ base = readMetadata();
+ V2Assert.assertEquals("Snapshot sequence number should be 4", 4,
snap4.sequenceNumber());
+ V2Assert.assertEquals("Last sequence number should be 4", 4,
base.lastSequenceNumber());
+ V1Assert.assertEquals("Table should end with last-sequence-number 0", 0,
base.lastSequenceNumber());
+
+ long commitId4 = snap4.snapshotId();
+ Assert.assertEquals("Should only contains 1 merged manifest", 1,
snap4.allManifests().size());
+ validateManifest(snap4.allManifests().get(0),
+ seqs(4, 3, 2, 1),
+ ids(commitId4, commitId3, commitId2, commitId1),
+ files(FILE_D, FILE_C, FILE_B, FILE_A),
+ statuses(Status.ADDED, Status.EXISTING, Status.EXISTING,
Status.EXISTING));
+ }
+
@Test
public void testManifestDoNotMergeMinCount() throws IOException {
Assert.assertEquals("Table should start empty", 0,
listManifestFiles().size());
table.updateProperties().set("commit.manifest.min-count-to-merge",
"4").commit();
TableMetadata base = readMetadata();
Assert.assertNull("Should not have a current snapshot",
base.currentSnapshot());
+ V1Assert.assertEquals("Last sequence number should be 0", 0,
readMetadata().lastSequenceNumber());
+ V2Assert.assertEquals("Last sequence number should be 0", 0,
readMetadata().lastSequenceNumber());
Review comment:
Minor: we don't need duplicate assertions when the condition should be
true for both v1 and v2 tables.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
