0dunay0 commented on code in PR #10995:
URL: https://github.com/apache/paimon/pull/10995#discussion_r4528868144
##########
paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergCommitCallback.java:
##########
@@ -789,60 +868,497 @@ private static void collectVariantFields(
// Create metadata based on old ones
//
-------------------------------------------------------------------------------------
- /**
- * Whether the existing metadata for {@code snapshotId} was built from
this very Paimon
- * snapshot, judged by the commit identity in the snapshot summary.
Unreadable counts as a
- * mismatch; metadata without an identity (older releases) is trusted, so
the protection only
- * covers metadata written since.
- *
- * <p>A replacement reuses its metadata version (versions are keyed by
Paimon snapshot id), so
- * readers that already loaded the abandoned version converge only after
reloading the table.
- */
/**
* Deletes every metadata version above the current Paimon snapshots,
which would otherwise
* shadow the replaced timeline for readers probing past the hint. A
failed deletion fails the
- * commit so a retry finishes the job; referenced manifests are left to
orphan cleanup (the
- * shared prefix makes reference counting non-trivial).
+ * commit so a retry finishes the job; the manifests they reference stay
on storage (the shared
+ * prefix makes reference counting non-trivial), and nothing removes them
later.
+ *
+ * <p>Only the head retires them: the hint and the pointer of the catalog
may still name one of
+ * the versions, until the head is published and moves them. The versions
above the head are
+ * listed here, and deleted under the table lock, which checks the head
again for every one of
+ * them: a commit that came after the listing makes its own version a
valid one. Returns false,
+ * and deletes nothing more, when the snapshot is not the head or its
publication was replaced.
*/
- private void retireAbandonedSuffix() throws IOException {
+ private boolean retireAbandonedSuffix(Snapshot source) throws IOException {
+ List<Path> abandoned = new ArrayList<>();
+ Long latestNow = table.snapshotManager().latestSnapshotId();
for (FileStatus status :
table.fileIO().listStatus(pathFactory.metadataDirectory())) {
- String name = status.getPath().getName();
- if (!name.startsWith("v") || !name.endsWith(".metadata.json")) {
- continue;
+ if (latestNow != null && metadataVersionOf(status.getPath()) >
latestNow) {
+ abandoned.add(status.getPath());
}
- long version;
- try {
- version = Long.parseLong(name.substring(1, name.indexOf('.')));
- } catch (NumberFormatException ignored) {
- continue;
+ }
+ for (int from = 0; from < abandoned.size(); from += cleanupBatchSize) {
+ List<Path> batch =
+ abandoned.subList(from, Math.min(abandoned.size(), from +
cleanupBatchSize));
+ boolean done =
+ cleanUnderLock(
+ source,
+ this::isPublishedHead,
+ () -> {
+ Long head =
table.snapshotManager().latestSnapshotId();
+ for (Path path : batch) {
+ if (head == null ||
metadataVersionOf(path) <= head) {
+ continue;
+ }
+ table.fileIO().deleteQuietly(path);
+ if (table.fileIO().exists(path)) {
+ throw new IllegalStateException(
+ "Failed to retire abandoned
Iceberg metadata "
+ + path);
+ }
+ }
+ });
+ if (!done) {
+ return false;
}
- Long latestNow = table.snapshotManager().latestSnapshotId();
- if (latestNow == null || version <= latestNow) {
- continue;
+ }
+ return true;
+ }
+
+ /** The version of a metadata file such as v5.metadata.json, or -1 for any
other file. */
+ static long metadataVersionOf(Path path) {
+ String name = path.getName();
+ if (!name.startsWith("v") || !name.endsWith(".metadata.json")) {
+ return -1;
+ }
+ try {
+ return Long.parseLong(name.substring(1, name.indexOf('.')));
+ } catch (NumberFormatException ignored) {
+ return -1;
+ }
+ }
+
+ static final String RETIRE_PENDING_FILENAME = "retire-pending";
+
+ /** What the marker of a rollback holds, or null when there is none or it
cannot be read. */
+ @Nullable
+ private String readRetirePending() {
+ try {
+ return table.fileIO()
+ .readFileUtf8(
+ new Path(pathFactory.metadataDirectory(),
RETIRE_PENDING_FILENAME))
+ .trim();
+ } catch (Exception e) {
+ return null;
+ }
+ }
+
+ /**
+ * Deletes the marker of a rollback once the leftovers of the rollback are
retired, but only if
+ * it still holds what it held before they were listed. A rollback writes
it again, and the
+ * leftovers of that one are not in the listing.
+ */
+ private void clearRetirePending(Snapshot source, @Nullable String
seenBeforeListing)
+ throws IOException {
+ if (seenBeforeListing == null) {
+ return;
+ }
+ cleanUnderLock(
+ source,
+ published -> isPublished(published, null),
+ () -> {
+ if (seenBeforeListing.equals(readRetirePending())) {
+ table.fileIO()
+ .deleteQuietly(
+ new Path(
+
pathFactory.metadataDirectory(),
+ RETIRE_PENDING_FILENAME));
+ }
+ });
+ }
+
+ @FunctionalInterface
+ interface LockedAction<T> {
+ T run(Lock lock) throws IOException;
+ }
+
+ /**
+ * Runs an operation under the lock that the commits of the table take, so
that the operation
+ * and a rollback or a commit of the table cannot interleave. The lock
comes from the catalog of
+ * the table, and a table without one is not protected.
+ */
+ private <T> T runUnderTableLock(LockedAction<T> action) throws IOException
{
+ return runWithLock(table, tableLock(), action);
+ }
+
+ private static <T> T runWithLock(
+ FileStoreTable table, SnapshotCommit lock, LockedAction<T> action)
throws IOException {
+ try {
+ return lock.runWithLock(() -> action.run(lock));
+ } catch (IOException | RuntimeException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new RuntimeException(
+ "Failed to run an operation with the lock of table " +
table.fullName(), e);
+ }
+ }
+
+ /**
+ * The lock of the table, which the catalog of the table provides. One is
made for the callback
+ * and closed with it: making one loads a catalog, and every publication
and cleanup takes the
+ * lock.
+ */
+ private synchronized SnapshotCommit tableLock() {
+ if (tableLock == null) {
+ // the callbacks of a tag have no commit user, and a lock can ask
for one
+ String lockUser = commitUser.isEmpty() ?
UUID.randomUUID().toString() : commitUser;
+ tableLock =
+ table.catalogEnvironment()
+ .snapshotCommit(table.snapshotManager(),
table.coreOptions(), lockUser);
+ }
+ return tableLock;
+ }
+
+ /**
+ * Runs the part of a rollback that deletes snapshots under the lock that
the publication of
+ * Iceberg metadata takes, so a rollback never happens between the last
check of a snapshot and
+ * the publication of its metadata. It takes the lock whatever the Iceberg
setting of the handle
+ * that rolls back is, because one table can have handles with Iceberg
metadata on and off, and
+ * a rollback through any of them replaces snapshots. The setting of the
table decides, and a
+ * table that never has Iceberg metadata rolls back without a lock.
+ */
+ public static void runRollback(FileStoreTable table, Runnable rollback) {
+ if (!icebergMetadataEnabled(table)) {
Review Comment:
`runRollback` no longer looks at the setting. A rollback takes the table
lock through any handle, whatever the handle or the stored option says. A
catalog without a lock gives one that does nothing, so the cost is a lock
acquisition per rollback.
This covers both of your cases:
- an already-open writer that keeps publishing after
metadata.iceberg.storage is set to disabled;
- a job that enables Iceberg metadata only for itself.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]