0dunay0 commented on code in PR #10995:
URL: https://github.com/apache/paimon/pull/10995#discussion_r4630567538
##########
paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergCommitCallback.java:
##########
@@ -789,60 +885,568 @@ private static void collectVariantFields(
// Create metadata based on old ones
//
-------------------------------------------------------------------------------------
- /**
- * Whether the existing metadata for {@code snapshotId} was built from
this very Paimon
- * snapshot, judged by the commit identity in the snapshot summary.
Unreadable counts as a
- * mismatch; metadata without an identity (older releases) is trusted, so
the protection only
- * covers metadata written since.
- *
- * <p>A replacement reuses its metadata version (versions are keyed by
Paimon snapshot id), so
- * readers that already loaded the abandoned version converge only after
reloading the table.
- */
/**
* Deletes every metadata version above the current Paimon snapshots,
which would otherwise
* shadow the replaced timeline for readers probing past the hint. A
failed deletion fails the
- * commit so a retry finishes the job; referenced manifests are left to
orphan cleanup (the
- * shared prefix makes reference counting non-trivial).
+ * commit so a retry finishes the job; the manifests they reference stay
on storage (the shared
+ * prefix makes reference counting non-trivial), and nothing removes them
later.
+ *
+ * <p>The head retires them, and so does a snapshot that a later commit
overtook, once the hint
+ * names no version above the head: until a head is published, the hint
and the pointer of the
+ * catalog may still name one of them. The versions above the head are
listed here, and deleted
+ * under the table lock, which checks the head again for every one of
them: a commit that came
+ * after the listing makes its own version a valid one. Returns false, and
deletes nothing more,
+ * when the versions may not be retired or the publication of the snapshot
was replaced.
*/
- private void retireAbandonedSuffix() throws IOException {
+ private boolean retireAbandonedSuffix(Snapshot source) throws IOException {
+ List<Path> abandoned = new ArrayList<>();
+ Long latestNow = table.snapshotManager().latestSnapshotId();
for (FileStatus status :
table.fileIO().listStatus(pathFactory.metadataDirectory())) {
- String name = status.getPath().getName();
- if (!name.startsWith("v") || !name.endsWith(".metadata.json")) {
- continue;
+ if (latestNow != null && metadataVersionOf(status.getPath()) >
latestNow) {
+ abandoned.add(status.getPath());
+ }
+ }
+ for (int from = 0; from < abandoned.size(); from += cleanupBatchSize) {
+ List<Path> batch =
+ abandoned.subList(from, Math.min(abandoned.size(), from +
cleanupBatchSize));
+ boolean done =
+ cleanUnderLock(
+ source,
+ this::mayRetireAboveHead,
+ () -> {
+ Long head =
table.snapshotManager().latestSnapshotId();
+ for (Path path : batch) {
+ if (head == null ||
metadataVersionOf(path) <= head) {
+ continue;
+ }
+ table.fileIO().deleteQuietly(path);
+ if (table.fileIO().exists(path)) {
+ throw new IllegalStateException(
+ "Failed to retire abandoned
Iceberg metadata "
+ + path);
+ }
+ }
+ });
+ if (!done) {
+ return false;
}
- long version;
+ }
+ return true;
+ }
+
+ /** The version of a metadata file such as v5.metadata.json, or -1 for any
other file. */
+ static long metadataVersionOf(Path path) {
+ String name = path.getName();
+ if (!name.startsWith("v") || !name.endsWith(".metadata.json")) {
+ return -1;
+ }
+ try {
+ return Long.parseLong(name.substring(1, name.indexOf('.')));
+ } catch (NumberFormatException ignored) {
+ return -1;
+ }
+ }
+
+ static final String RETIRE_PENDING_FILENAME = "retire-pending";
+
+ /** What the marker of a rollback holds, or null when there is none or it
cannot be read. */
+ @Nullable
+ private String readRetirePending() {
+ try {
+ return table.fileIO()
+ .readFileUtf8(
+ new Path(pathFactory.metadataDirectory(),
RETIRE_PENDING_FILENAME))
+ .trim();
+ } catch (Exception e) {
+ return null;
+ }
+ }
+
+ /**
+ * Deletes the marker of a rollback once the leftovers of the rollback are
retired, but only if
+ * it still holds what it held before they were listed. A rollback writes
it again, and the
+ * leftovers of that one are not in the listing.
+ */
+ private void clearRetirePending(Snapshot source, @Nullable String
seenBeforeListing)
+ throws IOException {
+ if (seenBeforeListing == null) {
+ return;
+ }
+ cleanUnderLock(
+ source,
+ published -> isPublished(published, null),
+ () -> {
+ if (seenBeforeListing.equals(readRetirePending())) {
+ table.fileIO()
+ .deleteQuietly(
+ new Path(
+
pathFactory.metadataDirectory(),
+ RETIRE_PENDING_FILENAME));
+ }
+ });
+ }
+
+ @FunctionalInterface
+ interface LockedAction<T> {
+ T run(Lock lock) throws IOException;
+ }
+
+ /**
+ * Runs an operation under the lock that the commits of the table take, so
that the operation
+ * and a rollback or a commit of the table cannot interleave. The lock
comes from the catalog of
+ * the table, and a table without one is not protected.
+ */
+ private <T> T runUnderTableLock(LockedAction<T> action) throws IOException
{
+ return runWithLock(table, tableLock(), action);
+ }
+
+ private static <T> T runWithLock(
+ FileStoreTable table, SnapshotCommit lock, LockedAction<T> action)
throws IOException {
+ try {
+ return lock.runWithLock(() -> action.run(lock));
+ } catch (IOException | RuntimeException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new RuntimeException(
+ "Failed to run an operation with the lock of table " +
table.fullName(), e);
+ }
+ }
+
+ /**
+ * The lock of the table, which the catalog of the table provides. One is
made for the callback
+ * and closed with it: making one loads a catalog, and every publication
and cleanup takes the
+ * lock.
+ */
+ private synchronized SnapshotCommit tableLock() {
+ if (tableLock == null) {
+ // the callbacks of a tag have no commit user, and a lock can ask
for one
+ String lockUser = commitUser.isEmpty() ?
UUID.randomUUID().toString() : commitUser;
+ tableLock =
+ table.catalogEnvironment()
+ .snapshotCommit(table.snapshotManager(),
table.coreOptions(), lockUser);
+ }
+ return tableLock;
+ }
+
+ /**
+ * Runs the part of a rollback that deletes snapshots under the lock that
the publication of
+ * Iceberg metadata takes, so a rollback never happens between the last
check of a snapshot and
+ * the publication of its metadata. It takes the lock whatever the Iceberg
setting is, of the
+ * handle that rolls back or of the table as it is stored: a writer that
opened the table
+ * earlier, or that turned Iceberg metadata on for its own job, can still
publish after the
+ * stored setting says off. A catalog without a lock gives a lock that
does nothing.
+ */
+ public static void runRollback(FileStoreTable table, Runnable rollback) {
+ SnapshotCommit lock =
+ table.catalogEnvironment()
+ .snapshotCommit(
+ table.snapshotManager(),
+ table.coreOptions(),
+ UUID.randomUUID().toString());
+ try {
+ runWithLock(
+ table,
+ lock,
+ held -> {
+ // under the lock, a cleanup of the leftovers of an
earlier rollback cannot
+ // clear this marker between its check and its delete
+ markRetirePendingForRollback(table);
+ rollback.run();
+ return null;
+ });
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ } finally {
+ IOUtils.closeQuietly(lock);
+ }
+ }
+
+ /** Whether the table, as it is stored, has Iceberg metadata on, whatever
a handle sets. */
+ private static boolean icebergMetadataEnabled(FileStoreTable table) {
+ return table.schemaManager()
+ .latest()
+ .map(
+ schema ->
+ Options.fromMap(schema.options())
+
.get(IcebergOptions.METADATA_ICEBERG_STORAGE)
+ != IcebergOptions.StorageType.DISABLED)
+ .orElse(false);
+ }
+
+ /** Whether the commits of the table take a lock that keeps other writers
and rollbacks out. */
+ private boolean tableLockCoordinates() {
+ return tableLock().isCoordinating();
+ }
+
+ /**
+ * Publishes metadata under the lock that the commits and the rollbacks of
the table take.
+ * Everything that decides what readers see goes through here: the
metadata file, the version
+ * hint and the pointer of the external catalog. The source is checked
again under the lock, so
+ * a rollback cannot replace the snapshot between that check and the
publication. Dropping and
+ * recreating the table does not take the lock: a publication that passed
the check can still
+ * write into the metadata directory of the new table. Building the
metadata stays outside the
+ * lock, and so does the cleanup that follows.
+ *
+ * <p>Returns null, and does not run the publication, when the snapshot is
not the one on disk
+ * any more. The files that were built for it stay on storage.
+ */
+ @Nullable
+ <T> T publishUnderLock(Snapshot source, LockedAction<T> publication)
throws IOException {
+ replayPoint(ReplayPoint.BEFORE_WRITE, source.id());
+ return runUnderTableLock(
+ lock -> {
+ if (!isCurrentSource(source)) {
+ return null;
+ }
+ lock.ensureValid();
+ replayPoint(ReplayPoint.LOCKED, source.id());
+ return publication.run(lock);
+ });
+ }
+
+ /** Whether the Paimon snapshot on disk is still the one that the metadata
is built from. */
+ private boolean isCurrentSource(Snapshot source) {
+ // the snapshot cache may still hold a rolled-back timeline under
reused ids
+ table.snapshotManager().invalidateCache();
+ Path path = table.snapshotManager().snapshotPath(source.id());
+ try {
+ Snapshot onDisk = SnapshotManager.tryFromPath(table.fileIO(),
path);
+ return commitIdentity(onDisk).equals(commitIdentity(source));
+ } catch (FileNotFoundException e) {
try {
- version = Long.parseLong(name.substring(1, name.indexOf('.')));
- } catch (NumberFormatException ignored) {
- continue;
+ confirmMissing(path, e);
+ } catch (IOException failure) {
+ throw new UncheckedIOException(failure);
}
- Long latestNow = table.snapshotManager().latestSnapshotId();
- if (latestNow == null || version <= latestNow) {
- continue;
+ return false;
+ }
+ }
+
+ /** Whether the Paimon snapshot on disk is still the head, and the one the
metadata is from. */
+ private boolean isCurrentHead(Snapshot source) {
+ Long latest = table.snapshotManager().latestSnapshotId();
+ return isCurrentSource(source) && latest != null && latest ==
source.id();
+ }
+
+ /**
+ * Whether the metadata that was published for the snapshot is still in
place: the snapshot is
+ * the one on disk, and the metadata file of its version is built from it.
The commit identity
+ * is not enough for what depends on the files a metadata file lists,
because two callbacks can
+ * build metadata files from the same snapshot, with different files, and
only one of them is
+ * durable. Such a cleanup passes the generation it was chosen for, and
the metadata file must
+ * be of that generation. A cleanup that does not depend on the listed
files passes null.
+ */
+ private boolean isPublished(Snapshot source, @Nullable String generation) {
+ if (!isCurrentSource(source)) {
+ return false;
+ }
+ try {
+ IcebergMetadata durable =
readUsableMetadata(pathFactory.toMetadataPath(source.id()));
+ return durable != null
+ && metadataMatchesSnapshot(durable, source)
+ && (generation == null ||
generation.equals(generationOf(durable)));
+ } catch (IOException e) {
+ // a failed read says nothing about the file: fail, and the retry
cleans up
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ /**
+ * Whether the files chosen for the publication of the snapshot can still
be deleted: the
+ * publication stands, see {@link #isPublished}, or a later head
superseded it. A later head
+ * deletes the metadata file of the snapshot with the older versions once
it published its own,
+ * and what the metadata of the snapshot did not need, nothing after it
needs.
+ */
+ private boolean isPublishedOrSuperseded(Snapshot source, String
generation) {
+ if (isPublished(source, generation)) {
+ return true;
+ }
+ try {
+ return isCurrentSource(source)
+ && readVersionHint() > source.id()
+ &&
!table.fileIO().exists(pathFactory.toMetadataPath(source.id()));
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ /**
+ * Whether the metadata of the snapshot is in place, and the snapshot is
the head, or the hint,
+ * and the pointer of the catalog that moves with it, names no version
above the head.
+ */
+ private boolean mayRetireAboveHead(Snapshot source) {
+ if (!isPublished(source, null)) {
+ return false;
+ }
+ Long latest = table.snapshotManager().latestSnapshotId();
+ return latest != null && (latest == source.id() || readVersionHint()
<= latest);
Review Comment:
Good catch, thanks for the repro. The hint can't vouch for the pointer,
since it's written first. Fixed in latest commit:
- Only the callback whose own publication made its snapshot the head retires
versions above it. An older callback, like your delayed v3, now skips the
retire and leaves the files and the `retire-pending` marker for the next head.
- Under the lock it also requires the snapshot to still be the head and the
hint to name it. So if a later commit moved the hint and a rollback came after,
the retire is blocked too. I added a rollback test for this.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]