RussellSpitzer commented on a change in pull request #1361:
URL: https://github.com/apache/iceberg/pull/1361#discussion_r474291294
##########
File path: core/src/test/java/org/apache/iceberg/TestRemoveSnapshots.java
##########
@@ -50,6 +51,257 @@ public TestRemoveSnapshots(int formatVersion) {
super(formatVersion);
}
+ private long waitUntilAfter(long timestampMillis) {
+ long t = System.currentTimeMillis();
+ while (t <= timestampMillis) {
+ t = System.currentTimeMillis();
+ }
+ return t;
+ }
+
+ @Test
+ public void testExpireOlderThan() {
+ table.newAppend()
+ .appendFile(FILE_A)
+ .commit();
+
+ Snapshot firstSnapshot = table.currentSnapshot();
+
+ waitUntilAfter(table.currentSnapshot().timestampMillis());
+
+ table.newAppend()
+ .appendFile(FILE_B)
+ .commit();
+
+ long snapshotId = table.currentSnapshot().snapshotId();
+
+ long tAfterCommits =
waitUntilAfter(table.currentSnapshot().timestampMillis());
+
+ Set<String> deletedFiles = Sets.newHashSet();
+
+ table.expireSnapshots()
+ .expireOlderThan(tAfterCommits)
+ .deleteWith(deletedFiles::add)
+ .commit();
+
+ Assert.assertEquals("Expire should not change current snapshot",
snapshotId, table.currentSnapshot().snapshotId());
+ Assert.assertNull("Expire should remove the oldest snapshot",
table.snapshot(firstSnapshot.snapshotId()));
+ Assert.assertEquals("Should not remove only the expired manifest list
location",
+ Sets.newHashSet(firstSnapshot.manifestListLocation()), deletedFiles);
+ }
+
+ @Test
+ public void testExpireOlderThanWithDelete() {
+ table.newAppend()
+ .appendFile(FILE_A)
+ .commit();
+
+ Snapshot firstSnapshot = table.currentSnapshot();
+ Assert.assertEquals("Should create one manifest",
+ 1, firstSnapshot.allManifests().size());
+
+ waitUntilAfter(table.currentSnapshot().timestampMillis());
+
+ table.newDelete()
+ .deleteFile(FILE_A)
+ .commit();
+
+ Snapshot secondSnapshot = table.currentSnapshot();
+ Assert.assertEquals("Should create replace manifest with a rewritten
manifest",
+ 1, secondSnapshot.allManifests().size());
+
+ table.newAppend()
+ .appendFile(FILE_B)
+ .commit();
+
+ waitUntilAfter(table.currentSnapshot().timestampMillis());
+
+ long snapshotId = table.currentSnapshot().snapshotId();
+
+ long tAfterCommits =
waitUntilAfter(table.currentSnapshot().timestampMillis());
+
+ Set<String> deletedFiles = Sets.newHashSet();
+
+ table.expireSnapshots()
+ .expireOlderThan(tAfterCommits)
+ .deleteWith(deletedFiles::add)
+ .commit();
+
+ Assert.assertEquals("Expire should not change current snapshot",
snapshotId, table.currentSnapshot().snapshotId());
+ Assert.assertNull("Expire should remove the oldest snapshot",
table.snapshot(firstSnapshot.snapshotId()));
+ Assert.assertNull("Expire should remove the second oldest snapshot",
table.snapshot(secondSnapshot.snapshotId()));
+
+ Assert.assertEquals("Should remove expired manifest lists and deleted data
file",
+ Sets.newHashSet(
+ firstSnapshot.manifestListLocation(), // snapshot expired
+ firstSnapshot.allManifests().get(0).path(), // manifest was
rewritten for delete
+ secondSnapshot.manifestListLocation(), // snapshot expired
+ secondSnapshot.allManifests().get(0).path(), // manifest contained
only deletes, was dropped
+ FILE_A.path()), // deleted
+ deletedFiles);
+ }
+
+ @Test
+ public void testExpireOlderThanWithDeleteInMergedManifests() {
+ // merge every commit
+ table.updateProperties()
+ .set(TableProperties.MANIFEST_MIN_MERGE_COUNT, "0")
+ .commit();
+
+ table.newAppend()
+ .appendFile(FILE_A)
+ .appendFile(FILE_B)
+ .commit();
+
+ Snapshot firstSnapshot = table.currentSnapshot();
+ Assert.assertEquals("Should create one manifest",
+ 1, firstSnapshot.allManifests().size());
+
+ waitUntilAfter(table.currentSnapshot().timestampMillis());
+
+ table.newDelete()
+ .deleteFile(FILE_A) // FILE_B is still in the dataset
+ .commit();
+
+ Snapshot secondSnapshot = table.currentSnapshot();
+ Assert.assertEquals("Should create replace manifest with a rewritten
manifest",
Review comment:
Should replace manifest with a rewritten manifest
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
