raunaqmorarka commented on code in PR #18074:
URL: https://github.com/apache/iceberg/pull/18074#discussion_r4442446463
##########
core/src/main/java/org/apache/iceberg/SnapshotProducer.java:
##########
@@ -109,6 +112,10 @@ public void accept(String file) {
private final AtomicInteger manifestCount = new AtomicInteger(0);
private final AtomicInteger attempt = new AtomicInteger(0);
private final List<String> manifestLists = Lists.newArrayList();
+
+ /** Manifest paths written to each manifest list, keyed by manifest list
location. */
+ private final Map<String, List<String>> manifestPathsByManifestList =
Maps.newHashMap();
Review Comment:
Merged into one `LinkedHashMap` of `Set`s keyed by manifest list location,
recorded before the write.
##########
core/src/test/java/org/apache/iceberg/TestTransaction.java:
##########
@@ -1119,6 +1119,91 @@ public void testExtendBaseTransaction() {
.hasSize(2);
}
+ @TestTemplate
+ public void testCommitTransactionDoesNotReadCommittedManifestLists() throws
IOException {
Review Comment:
Dropped it.
##########
core/src/test/java/org/apache/iceberg/TestSnapshotProducer.java:
##########
@@ -403,4 +409,82 @@ public void monotonicityIsScopedToTargetBranch() {
.isEqualTo(mainNewTs)
.isLessThan(branchTs);
}
+
+ @TestTemplate
+ public void testCommitDoesNotReadCommittedManifestList() {
+ String tableName = "no-manifest-list-reread";
+ List<String> openedInputFiles = Lists.newArrayList();
+ TestTables.LocalFileIO countingFileIO =
+ new TestTables.LocalFileIO() {
Review Comment:
Switched both new tests to a spied `LocalFileIO` and dropped the `test`
prefix.
##########
core/src/test/java/org/apache/iceberg/TestTransaction.java:
##########
@@ -683,7 +683,7 @@ public void
testTransactionRetryAndAppendManifestsWithSnapshotIdInheritance() th
}
@TestTemplate
- public void testTransactionCleanupReadsOnlyOwnManifestLists() throws
IOException {
+ public void testTransactionCleanupReadsNoManifestLists() throws IOException {
File location = java.nio.file.Files.createTempDirectory(temp,
"junit").toFile();
String tableName = "txnCleanupOwnManifestListsTest";
Review Comment:
Renamed it to `txnCleanupNoManifestListsTest`.
##########
core/src/main/java/org/apache/iceberg/BaseTransaction.java:
##########
@@ -455,17 +455,32 @@ private void applyUpdates(TableOperations underlyingOps) {
}
// returns the manifest lists and manifests referenced by the given
committed snapshots
- private static Set<String> committedFiles(FileIO io, Set<Snapshot>
snapshots) {
+ private Set<String> committedFiles(FileIO io, Set<Snapshot> snapshots) {
Set<String> committedFiles = Sets.newHashSet();
for (Snapshot snap : snapshots) {
committedFiles.add(snap.manifestListLocation());
- snap.allManifests(io).forEach(manifest ->
committedFiles.add(manifest.path()));
+ committedFiles.addAll(committedManifestPaths(io, snap));
}
return committedFiles;
}
+ // uses the manifest paths held by the update that wrote the snapshot's
manifest list, if any
+ private List<String> committedManifestPaths(FileIO io, Snapshot snapshot) {
Review Comment:
Done.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
