0dunay0 commented on code in PR #10995:
URL: https://github.com/apache/paimon/pull/10995#discussion_r4630567538


##########
paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergCommitCallback.java:
##########
@@ -789,60 +885,568 @@ private static void collectVariantFields(
     // Create metadata based on old ones
     // 
-------------------------------------------------------------------------------------
 
-    /**
-     * Whether the existing metadata for {@code snapshotId} was built from 
this very Paimon
-     * snapshot, judged by the commit identity in the snapshot summary. 
Unreadable counts as a
-     * mismatch; metadata without an identity (older releases) is trusted, so 
the protection only
-     * covers metadata written since.
-     *
-     * <p>A replacement reuses its metadata version (versions are keyed by 
Paimon snapshot id), so
-     * readers that already loaded the abandoned version converge only after 
reloading the table.
-     */
     /**
      * Deletes every metadata version above the current Paimon snapshots, 
which would otherwise
      * shadow the replaced timeline for readers probing past the hint. A 
failed deletion fails the
-     * commit so a retry finishes the job; referenced manifests are left to 
orphan cleanup (the
-     * shared prefix makes reference counting non-trivial).
+     * commit so a retry finishes the job; the manifests they reference stay 
on storage (the shared
+     * prefix makes reference counting non-trivial), and nothing removes them 
later.
+     *
+     * <p>The head retires them, and so does a snapshot that a later commit 
overtook, once the hint
+     * names no version above the head: until a head is published, the hint 
and the pointer of the
+     * catalog may still name one of them. The versions above the head are 
listed here, and deleted
+     * under the table lock, which checks the head again for every one of 
them: a commit that came
+     * after the listing makes its own version a valid one. Returns false, and 
deletes nothing more,
+     * when the versions may not be retired or the publication of the snapshot 
was replaced.
      */
-    private void retireAbandonedSuffix() throws IOException {
+    private boolean retireAbandonedSuffix(Snapshot source) throws IOException {
+        List<Path> abandoned = new ArrayList<>();
+        Long latestNow = table.snapshotManager().latestSnapshotId();
         for (FileStatus status : 
table.fileIO().listStatus(pathFactory.metadataDirectory())) {
-            String name = status.getPath().getName();
-            if (!name.startsWith("v") || !name.endsWith(".metadata.json")) {
-                continue;
+            if (latestNow != null && metadataVersionOf(status.getPath()) > 
latestNow) {
+                abandoned.add(status.getPath());
+            }
+        }
+        for (int from = 0; from < abandoned.size(); from += cleanupBatchSize) {
+            List<Path> batch =
+                    abandoned.subList(from, Math.min(abandoned.size(), from + 
cleanupBatchSize));
+            boolean done =
+                    cleanUnderLock(
+                            source,
+                            this::mayRetireAboveHead,
+                            () -> {
+                                Long head = 
table.snapshotManager().latestSnapshotId();
+                                for (Path path : batch) {
+                                    if (head == null || 
metadataVersionOf(path) <= head) {
+                                        continue;
+                                    }
+                                    table.fileIO().deleteQuietly(path);
+                                    if (table.fileIO().exists(path)) {
+                                        throw new IllegalStateException(
+                                                "Failed to retire abandoned 
Iceberg metadata "
+                                                        + path);
+                                    }
+                                }
+                            });
+            if (!done) {
+                return false;
             }
-            long version;
+        }
+        return true;
+    }
+
+    /** The version of a metadata file such as v5.metadata.json, or -1 for any 
other file. */
+    static long metadataVersionOf(Path path) {
+        String name = path.getName();
+        if (!name.startsWith("v") || !name.endsWith(".metadata.json")) {
+            return -1;
+        }
+        try {
+            return Long.parseLong(name.substring(1, name.indexOf('.')));
+        } catch (NumberFormatException ignored) {
+            return -1;
+        }
+    }
+
+    static final String RETIRE_PENDING_FILENAME = "retire-pending";
+
+    /** What the marker of a rollback holds, or null when there is none or it 
cannot be read. */
+    @Nullable
+    private String readRetirePending() {
+        try {
+            return table.fileIO()
+                    .readFileUtf8(
+                            new Path(pathFactory.metadataDirectory(), 
RETIRE_PENDING_FILENAME))
+                    .trim();
+        } catch (Exception e) {
+            return null;
+        }
+    }
+
+    /**
+     * Deletes the marker of a rollback once the leftovers of the rollback are 
retired, but only if
+     * it still holds what it held before they were listed. A rollback writes 
it again, and the
+     * leftovers of that one are not in the listing.
+     */
+    private void clearRetirePending(Snapshot source, @Nullable String 
seenBeforeListing)
+            throws IOException {
+        if (seenBeforeListing == null) {
+            return;
+        }
+        cleanUnderLock(
+                source,
+                published -> isPublished(published, null),
+                () -> {
+                    if (seenBeforeListing.equals(readRetirePending())) {
+                        table.fileIO()
+                                .deleteQuietly(
+                                        new Path(
+                                                
pathFactory.metadataDirectory(),
+                                                RETIRE_PENDING_FILENAME));
+                    }
+                });
+    }
+
+    @FunctionalInterface
+    interface LockedAction<T> {
+        T run(Lock lock) throws IOException;
+    }
+
+    /**
+     * Runs an operation under the lock that the commits of the table take, so 
that the operation
+     * and a rollback or a commit of the table cannot interleave. The lock 
comes from the catalog of
+     * the table, and a table without one is not protected.
+     */
+    private <T> T runUnderTableLock(LockedAction<T> action) throws IOException 
{
+        return runWithLock(table, tableLock(), action);
+    }
+
+    private static <T> T runWithLock(
+            FileStoreTable table, SnapshotCommit lock, LockedAction<T> action) 
throws IOException {
+        try {
+            return lock.runWithLock(() -> action.run(lock));
+        } catch (IOException | RuntimeException e) {
+            throw e;
+        } catch (Exception e) {
+            throw new RuntimeException(
+                    "Failed to run an operation with the lock of table " + 
table.fullName(), e);
+        }
+    }
+
+    /**
+     * The lock of the table, which the catalog of the table provides. One is 
made for the callback
+     * and closed with it: making one loads a catalog, and every publication 
and cleanup takes the
+     * lock.
+     */
+    private synchronized SnapshotCommit tableLock() {
+        if (tableLock == null) {
+            // the callbacks of a tag have no commit user, and a lock can ask 
for one
+            String lockUser = commitUser.isEmpty() ? 
UUID.randomUUID().toString() : commitUser;
+            tableLock =
+                    table.catalogEnvironment()
+                            .snapshotCommit(table.snapshotManager(), 
table.coreOptions(), lockUser);
+        }
+        return tableLock;
+    }
+
+    /**
+     * Runs the part of a rollback that deletes snapshots under the lock that 
the publication of
+     * Iceberg metadata takes, so a rollback never happens between the last 
check of a snapshot and
+     * the publication of its metadata. It takes the lock whatever the Iceberg 
setting is, of the
+     * handle that rolls back or of the table as it is stored: a writer that 
opened the table
+     * earlier, or that turned Iceberg metadata on for its own job, can still 
publish after the
+     * stored setting says off. A catalog without a lock gives a lock that 
does nothing.
+     */
+    public static void runRollback(FileStoreTable table, Runnable rollback) {
+        SnapshotCommit lock =
+                table.catalogEnvironment()
+                        .snapshotCommit(
+                                table.snapshotManager(),
+                                table.coreOptions(),
+                                UUID.randomUUID().toString());
+        try {
+            runWithLock(
+                    table,
+                    lock,
+                    held -> {
+                        // under the lock, a cleanup of the leftovers of an 
earlier rollback cannot
+                        // clear this marker between its check and its delete
+                        markRetirePendingForRollback(table);
+                        rollback.run();
+                        return null;
+                    });
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        } finally {
+            IOUtils.closeQuietly(lock);
+        }
+    }
+
+    /** Whether the table, as it is stored, has Iceberg metadata on, whatever 
a handle sets. */
+    private static boolean icebergMetadataEnabled(FileStoreTable table) {
+        return table.schemaManager()
+                .latest()
+                .map(
+                        schema ->
+                                Options.fromMap(schema.options())
+                                                
.get(IcebergOptions.METADATA_ICEBERG_STORAGE)
+                                        != IcebergOptions.StorageType.DISABLED)
+                .orElse(false);
+    }
+
+    /** Whether the commits of the table take a lock that keeps other writers 
and rollbacks out. */
+    private boolean tableLockCoordinates() {
+        return tableLock().isCoordinating();
+    }
+
+    /**
+     * Publishes metadata under the lock that the commits and the rollbacks of 
the table take.
+     * Everything that decides what readers see goes through here: the 
metadata file, the version
+     * hint and the pointer of the external catalog. The source is checked 
again under the lock, so
+     * a rollback cannot replace the snapshot between that check and the 
publication. Dropping and
+     * recreating the table does not take the lock: a publication that passed 
the check can still
+     * write into the metadata directory of the new table. Building the 
metadata stays outside the
+     * lock, and so does the cleanup that follows.
+     *
+     * <p>Returns null, and does not run the publication, when the snapshot is 
not the one on disk
+     * any more. The files that were built for it stay on storage.
+     */
+    @Nullable
+    <T> T publishUnderLock(Snapshot source, LockedAction<T> publication) 
throws IOException {
+        replayPoint(ReplayPoint.BEFORE_WRITE, source.id());
+        return runUnderTableLock(
+                lock -> {
+                    if (!isCurrentSource(source)) {
+                        return null;
+                    }
+                    lock.ensureValid();
+                    replayPoint(ReplayPoint.LOCKED, source.id());
+                    return publication.run(lock);
+                });
+    }
+
+    /** Whether the Paimon snapshot on disk is still the one that the metadata 
is built from. */
+    private boolean isCurrentSource(Snapshot source) {
+        // the snapshot cache may still hold a rolled-back timeline under 
reused ids
+        table.snapshotManager().invalidateCache();
+        Path path = table.snapshotManager().snapshotPath(source.id());
+        try {
+            Snapshot onDisk = SnapshotManager.tryFromPath(table.fileIO(), 
path);
+            return commitIdentity(onDisk).equals(commitIdentity(source));
+        } catch (FileNotFoundException e) {
             try {
-                version = Long.parseLong(name.substring(1, name.indexOf('.')));
-            } catch (NumberFormatException ignored) {
-                continue;
+                confirmMissing(path, e);
+            } catch (IOException failure) {
+                throw new UncheckedIOException(failure);
             }
-            Long latestNow = table.snapshotManager().latestSnapshotId();
-            if (latestNow == null || version <= latestNow) {
-                continue;
+            return false;
+        }
+    }
+
+    /** Whether the Paimon snapshot on disk is still the head, and the one the 
metadata is from. */
+    private boolean isCurrentHead(Snapshot source) {
+        Long latest = table.snapshotManager().latestSnapshotId();
+        return isCurrentSource(source) && latest != null && latest == 
source.id();
+    }
+
+    /**
+     * Whether the metadata that was published for the snapshot is still in 
place: the snapshot is
+     * the one on disk, and the metadata file of its version is built from it. 
The commit identity
+     * is not enough for what depends on the files a metadata file lists, 
because two callbacks can
+     * build metadata files from the same snapshot, with different files, and 
only one of them is
+     * durable. Such a cleanup passes the generation it was chosen for, and 
the metadata file must
+     * be of that generation. A cleanup that does not depend on the listed 
files passes null.
+     */
+    private boolean isPublished(Snapshot source, @Nullable String generation) {
+        if (!isCurrentSource(source)) {
+            return false;
+        }
+        try {
+            IcebergMetadata durable = 
readUsableMetadata(pathFactory.toMetadataPath(source.id()));
+            return durable != null
+                    && metadataMatchesSnapshot(durable, source)
+                    && (generation == null || 
generation.equals(generationOf(durable)));
+        } catch (IOException e) {
+            // a failed read says nothing about the file: fail, and the retry 
cleans up
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    /**
+     * Whether the files chosen for the publication of the snapshot can still 
be deleted: the
+     * publication stands, see {@link #isPublished}, or a later head 
superseded it. A later head
+     * deletes the metadata file of the snapshot with the older versions once 
it published its own,
+     * and what the metadata of the snapshot did not need, nothing after it 
needs.
+     */
+    private boolean isPublishedOrSuperseded(Snapshot source, String 
generation) {
+        if (isPublished(source, generation)) {
+            return true;
+        }
+        try {
+            return isCurrentSource(source)
+                    && readVersionHint() > source.id()
+                    && 
!table.fileIO().exists(pathFactory.toMetadataPath(source.id()));
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    /**
+     * Whether the metadata of the snapshot is in place, and the snapshot is 
the head, or the hint,
+     * and the pointer of the catalog that moves with it, names no version 
above the head.
+     */
+    private boolean mayRetireAboveHead(Snapshot source) {
+        if (!isPublished(source, null)) {
+            return false;
+        }
+        Long latest = table.snapshotManager().latestSnapshotId();
+        return latest != null && (latest == source.id() || readVersionHint() 
<= latest);

Review Comment:
   Good catch, thanks for the repro. The hint can't vouch for the pointer, 
since it's written first. Fixed in latest commit:
   
   - Only the callback whose own publication made its snapshot the head retires 
versions above it. An older callback, like your delayed v3, now skips the 
retire and leaves the files and the `retire-pending` marker for the next head.
   - Under the lock it also requires the snapshot to still be the head and the 
hint to name it. So if a later commit moved the hint and a rollback came after, 
the retire is blocked too. I added a rollback test for this.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to