0dunay0 commented on code in PR #10995:
URL: https://github.com/apache/paimon/pull/10995#discussion_r4528868144


##########
paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergCommitCallback.java:
##########
@@ -789,60 +868,497 @@ private static void collectVariantFields(
     // Create metadata based on old ones
     // 
-------------------------------------------------------------------------------------
 
-    /**
-     * Whether the existing metadata for {@code snapshotId} was built from 
this very Paimon
-     * snapshot, judged by the commit identity in the snapshot summary. 
Unreadable counts as a
-     * mismatch; metadata without an identity (older releases) is trusted, so 
the protection only
-     * covers metadata written since.
-     *
-     * <p>A replacement reuses its metadata version (versions are keyed by 
Paimon snapshot id), so
-     * readers that already loaded the abandoned version converge only after 
reloading the table.
-     */
     /**
      * Deletes every metadata version above the current Paimon snapshots, 
which would otherwise
      * shadow the replaced timeline for readers probing past the hint. A 
failed deletion fails the
-     * commit so a retry finishes the job; referenced manifests are left to 
orphan cleanup (the
-     * shared prefix makes reference counting non-trivial).
+     * commit so a retry finishes the job; the manifests they reference stay 
on storage (the shared
+     * prefix makes reference counting non-trivial), and nothing removes them 
later.
+     *
+     * <p>Only the head retires them: the hint and the pointer of the catalog 
may still name one of
+     * the versions, until the head is published and moves them. The versions 
above the head are
+     * listed here, and deleted under the table lock, which checks the head 
again for every one of
+     * them: a commit that came after the listing makes its own version a 
valid one. Returns false,
+     * and deletes nothing more, when the snapshot is not the head or its 
publication was replaced.
      */
-    private void retireAbandonedSuffix() throws IOException {
+    private boolean retireAbandonedSuffix(Snapshot source) throws IOException {
+        List<Path> abandoned = new ArrayList<>();
+        Long latestNow = table.snapshotManager().latestSnapshotId();
         for (FileStatus status : 
table.fileIO().listStatus(pathFactory.metadataDirectory())) {
-            String name = status.getPath().getName();
-            if (!name.startsWith("v") || !name.endsWith(".metadata.json")) {
-                continue;
+            if (latestNow != null && metadataVersionOf(status.getPath()) > 
latestNow) {
+                abandoned.add(status.getPath());
             }
-            long version;
-            try {
-                version = Long.parseLong(name.substring(1, name.indexOf('.')));
-            } catch (NumberFormatException ignored) {
-                continue;
+        }
+        for (int from = 0; from < abandoned.size(); from += cleanupBatchSize) {
+            List<Path> batch =
+                    abandoned.subList(from, Math.min(abandoned.size(), from + 
cleanupBatchSize));
+            boolean done =
+                    cleanUnderLock(
+                            source,
+                            this::isPublishedHead,
+                            () -> {
+                                Long head = 
table.snapshotManager().latestSnapshotId();
+                                for (Path path : batch) {
+                                    if (head == null || 
metadataVersionOf(path) <= head) {
+                                        continue;
+                                    }
+                                    table.fileIO().deleteQuietly(path);
+                                    if (table.fileIO().exists(path)) {
+                                        throw new IllegalStateException(
+                                                "Failed to retire abandoned 
Iceberg metadata "
+                                                        + path);
+                                    }
+                                }
+                            });
+            if (!done) {
+                return false;
             }
-            Long latestNow = table.snapshotManager().latestSnapshotId();
-            if (latestNow == null || version <= latestNow) {
-                continue;
+        }
+        return true;
+    }
+
+    /** The version of a metadata file such as v5.metadata.json, or -1 for any 
other file. */
+    static long metadataVersionOf(Path path) {
+        String name = path.getName();
+        if (!name.startsWith("v") || !name.endsWith(".metadata.json")) {
+            return -1;
+        }
+        try {
+            return Long.parseLong(name.substring(1, name.indexOf('.')));
+        } catch (NumberFormatException ignored) {
+            return -1;
+        }
+    }
+
+    static final String RETIRE_PENDING_FILENAME = "retire-pending";
+
+    /** What the marker of a rollback holds, or null when there is none or it 
cannot be read. */
+    @Nullable
+    private String readRetirePending() {
+        try {
+            return table.fileIO()
+                    .readFileUtf8(
+                            new Path(pathFactory.metadataDirectory(), 
RETIRE_PENDING_FILENAME))
+                    .trim();
+        } catch (Exception e) {
+            return null;
+        }
+    }
+
+    /**
+     * Deletes the marker of a rollback once the leftovers of the rollback are 
retired, but only if
+     * it still holds what it held before they were listed. A rollback writes 
it again, and the
+     * leftovers of that one are not in the listing.
+     */
+    private void clearRetirePending(Snapshot source, @Nullable String 
seenBeforeListing)
+            throws IOException {
+        if (seenBeforeListing == null) {
+            return;
+        }
+        cleanUnderLock(
+                source,
+                published -> isPublished(published, null),
+                () -> {
+                    if (seenBeforeListing.equals(readRetirePending())) {
+                        table.fileIO()
+                                .deleteQuietly(
+                                        new Path(
+                                                
pathFactory.metadataDirectory(),
+                                                RETIRE_PENDING_FILENAME));
+                    }
+                });
+    }
+
+    @FunctionalInterface
+    interface LockedAction<T> {
+        T run(Lock lock) throws IOException;
+    }
+
+    /**
+     * Runs an operation under the lock that the commits of the table take, so 
that the operation
+     * and a rollback or a commit of the table cannot interleave. The lock 
comes from the catalog of
+     * the table, and a table without one is not protected.
+     */
+    private <T> T runUnderTableLock(LockedAction<T> action) throws IOException 
{
+        return runWithLock(table, tableLock(), action);
+    }
+
+    private static <T> T runWithLock(
+            FileStoreTable table, SnapshotCommit lock, LockedAction<T> action) 
throws IOException {
+        try {
+            return lock.runWithLock(() -> action.run(lock));
+        } catch (IOException | RuntimeException e) {
+            throw e;
+        } catch (Exception e) {
+            throw new RuntimeException(
+                    "Failed to run an operation with the lock of table " + 
table.fullName(), e);
+        }
+    }
+
+    /**
+     * The lock of the table, which the catalog of the table provides. One is 
made for the callback
+     * and closed with it: making one loads a catalog, and every publication 
and cleanup takes the
+     * lock.
+     */
+    private synchronized SnapshotCommit tableLock() {
+        if (tableLock == null) {
+            // the callbacks of a tag have no commit user, and a lock can ask 
for one
+            String lockUser = commitUser.isEmpty() ? 
UUID.randomUUID().toString() : commitUser;
+            tableLock =
+                    table.catalogEnvironment()
+                            .snapshotCommit(table.snapshotManager(), 
table.coreOptions(), lockUser);
+        }
+        return tableLock;
+    }
+
+    /**
+     * Runs the part of a rollback that deletes snapshots under the lock that 
the publication of
+     * Iceberg metadata takes, so a rollback never happens between the last 
check of a snapshot and
+     * the publication of its metadata. It takes the lock whatever the Iceberg 
setting of the handle
+     * that rolls back is, because one table can have handles with Iceberg 
metadata on and off, and
+     * a rollback through any of them replaces snapshots. The setting of the 
table decides, and a
+     * table that never has Iceberg metadata rolls back without a lock.
+     */
+    public static void runRollback(FileStoreTable table, Runnable rollback) {
+        if (!icebergMetadataEnabled(table)) {

Review Comment:
   `runRollback` no longer looks at the setting. A rollback takes the table 
lock through any handle, whatever the handle or the stored option says. A 
catalog without a lock gives one that does nothing, so the cost is a lock 
acquisition per rollback.
   
   This covers both of your cases:
   - an already-open writer that keeps publishing after 
metadata.iceberg.storage is set to disabled;
   - a job that enables Iceberg metadata only for itself.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to