RussellSpitzer commented on code in PR #18477:
URL: https://github.com/apache/iceberg/pull/18477#discussion_r4518669551
##########
spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/actions/TestExpireSnapshotsAction.java:
##########
@@ -1416,4 +1420,125 @@ public void testCleanExpiredMetadata() {
.contains(FILE_A.location())
.doesNotContain(fileInNewSpec.location());
}
+
+ @TestTemplate
+ public void
testExpireWithMissingManifestListReachableOnlyFromExpiredSnapshots() {
+ assertExpireSkipsMissingMetadata(Snapshot::manifestListLocation);
+ }
+
+ @TestTemplate
+ public void testExpireWithMissingManifestReachableOnlyFromExpiredSnapshots()
{
+ assertExpireSkipsMissingMetadata(
+ snapshot ->
Iterables.getOnlyElement(snapshot.allManifests(table.io())).path());
+ }
+
+ private void assertExpireSkipsMissingMetadata(
+ Function<Snapshot, String> missingLocationResolver) {
+ table.newAppend().appendFile(FILE_A).commit();
+ String missingLocation =
missingLocationResolver.apply(table.currentSnapshot());
+
+ table.newOverwrite().deleteFile(FILE_A).addFile(FILE_B).commit();
+ table.newOverwrite().deleteFile(FILE_B).addFile(FILE_C).commit();
+ table.newAppend().appendFile(FILE_D).commit();
+
+ long end = rightAfterSnapshot();
+
+ table.io().deleteFile(missingLocation);
+
+ Set<String> deletedFiles = ConcurrentHashMap.newKeySet();
+
+ ExpireSnapshots.Result result =
+ SparkActions.get()
+ .expireSnapshots(table)
+ .expireOlderThan(end)
+ .deleteWith(deletedFiles::add)
+ .execute();
+
+ assertThat(table.snapshots()).as("Expiration should still
commit").hasSize(1);
+ assertThat(result.deletedDataFilesCount())
+ .as("Should report the files deleted despite the missing metadata")
+ .isEqualTo(1L);
+ assertThat(deletedFiles)
+ .as("Should delete files resolved from other expired snapshots")
+ .contains(FILE_B.location());
+ assertThat(deletedFiles)
+ .as("Should leave files reachable only through the missing metadata as
orphans")
+ .doesNotContain(FILE_A.location());
Review Comment:
While we are at it we can make sure C and D are also not deleted.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]