RussellSpitzer commented on code in PR #18477:
URL: https://github.com/apache/iceberg/pull/18477#discussion_r4518684472
##########
spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/actions/TestExpireSnapshotsAction.java:
##########
@@ -1416,4 +1420,125 @@ public void testCleanExpiredMetadata() {
.contains(FILE_A.location())
.doesNotContain(fileInNewSpec.location());
}
+
+ @TestTemplate
+ public void
testExpireWithMissingManifestListReachableOnlyFromExpiredSnapshots() {
+ assertExpireSkipsMissingMetadata(Snapshot::manifestListLocation);
+ }
+
+ @TestTemplate
+ public void testExpireWithMissingManifestReachableOnlyFromExpiredSnapshots()
{
+ assertExpireSkipsMissingMetadata(
+ snapshot ->
Iterables.getOnlyElement(snapshot.allManifests(table.io())).path());
+ }
+
+ private void assertExpireSkipsMissingMetadata(
+ Function<Snapshot, String> missingLocationResolver) {
+ table.newAppend().appendFile(FILE_A).commit();
+ String missingLocation =
missingLocationResolver.apply(table.currentSnapshot());
+
+ table.newOverwrite().deleteFile(FILE_A).addFile(FILE_B).commit();
+ table.newOverwrite().deleteFile(FILE_B).addFile(FILE_C).commit();
+ table.newAppend().appendFile(FILE_D).commit();
+
+ long end = rightAfterSnapshot();
+
+ table.io().deleteFile(missingLocation);
+
+ Set<String> deletedFiles = ConcurrentHashMap.newKeySet();
+
+ ExpireSnapshots.Result result =
+ SparkActions.get()
+ .expireSnapshots(table)
+ .expireOlderThan(end)
+ .deleteWith(deletedFiles::add)
+ .execute();
+
+ assertThat(table.snapshots()).as("Expiration should still
commit").hasSize(1);
+ assertThat(result.deletedDataFilesCount())
+ .as("Should report the files deleted despite the missing metadata")
+ .isEqualTo(1L);
+ assertThat(deletedFiles)
+ .as("Should delete files resolved from other expired snapshots")
+ .contains(FILE_B.location());
+ assertThat(deletedFiles)
+ .as("Should leave files reachable only through the missing metadata as
orphans")
+ .doesNotContain(FILE_A.location());
+ }
+
+ @TestTemplate
+ public void
testExpireDeletesStatisticsFileOfSnapshotWithMissingManifestList() {
Review Comment:
With the helpers I hinted at above,
```java
@TestTemplate
public void expireDeletesStatisticsFileOfSnapshotWithMissingManifestList() {
Snapshot firstSnapshot = commitSnapshotsForSkipTest();
String statisticsLocation = tableLocation + "/metadata/stats-" +
UUID.randomUUID();
table
.updateStatistics()
.setStatistics(
new GenericStatisticsFile(
firstSnapshot.snapshotId(), statisticsLocation, 1L, 1L,
List.of()))
.commit();
table.io().deleteFile(firstSnapshot.manifestListLocation());
Set<String> deletedFiles = expireAll();
assertThat(deletedFiles).contains(statisticsLocation);
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]