JingsongLi commented on code in PR #10995:
URL: https://github.com/apache/paimon/pull/10995#discussion_r4523028190
##########
paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergCommitCallback.java:
##########
@@ -789,60 +868,497 @@ private static void collectVariantFields(
// Create metadata based on old ones
//
-------------------------------------------------------------------------------------
- /**
- * Whether the existing metadata for {@code snapshotId} was built from
this very Paimon
- * snapshot, judged by the commit identity in the snapshot summary.
Unreadable counts as a
- * mismatch; metadata without an identity (older releases) is trusted, so
the protection only
- * covers metadata written since.
- *
- * <p>A replacement reuses its metadata version (versions are keyed by
Paimon snapshot id), so
- * readers that already loaded the abandoned version converge only after
reloading the table.
- */
/**
* Deletes every metadata version above the current Paimon snapshots,
which would otherwise
* shadow the replaced timeline for readers probing past the hint. A
failed deletion fails the
- * commit so a retry finishes the job; referenced manifests are left to
orphan cleanup (the
- * shared prefix makes reference counting non-trivial).
+ * commit so a retry finishes the job; the manifests they reference stay
on storage (the shared
+ * prefix makes reference counting non-trivial), and nothing removes them
later.
+ *
+ * <p>Only the head retires them: the hint and the pointer of the catalog
may still name one of
+ * the versions, until the head is published and moves them. The versions
above the head are
+ * listed here, and deleted under the table lock, which checks the head
again for every one of
+ * them: a commit that came after the listing makes its own version a
valid one. Returns false,
+ * and deletes nothing more, when the snapshot is not the head or its
publication was replaced.
*/
- private void retireAbandonedSuffix() throws IOException {
+ private boolean retireAbandonedSuffix(Snapshot source) throws IOException {
+ List<Path> abandoned = new ArrayList<>();
+ Long latestNow = table.snapshotManager().latestSnapshotId();
for (FileStatus status :
table.fileIO().listStatus(pathFactory.metadataDirectory())) {
- String name = status.getPath().getName();
- if (!name.startsWith("v") || !name.endsWith(".metadata.json")) {
- continue;
+ if (latestNow != null && metadataVersionOf(status.getPath()) >
latestNow) {
+ abandoned.add(status.getPath());
}
- long version;
- try {
- version = Long.parseLong(name.substring(1, name.indexOf('.')));
- } catch (NumberFormatException ignored) {
- continue;
+ }
+ for (int from = 0; from < abandoned.size(); from += cleanupBatchSize) {
+ List<Path> batch =
+ abandoned.subList(from, Math.min(abandoned.size(), from +
cleanupBatchSize));
+ boolean done =
+ cleanUnderLock(
+ source,
+ this::isPublishedHead,
+ () -> {
+ Long head =
table.snapshotManager().latestSnapshotId();
+ for (Path path : batch) {
+ if (head == null ||
metadataVersionOf(path) <= head) {
+ continue;
+ }
+ table.fileIO().deleteQuietly(path);
+ if (table.fileIO().exists(path)) {
+ throw new IllegalStateException(
+ "Failed to retire abandoned
Iceberg metadata "
+ + path);
+ }
+ }
+ });
+ if (!done) {
+ return false;
}
- Long latestNow = table.snapshotManager().latestSnapshotId();
- if (latestNow == null || version <= latestNow) {
- continue;
+ }
+ return true;
+ }
+
+ /** The version of a metadata file such as v5.metadata.json, or -1 for any
other file. */
+ static long metadataVersionOf(Path path) {
+ String name = path.getName();
+ if (!name.startsWith("v") || !name.endsWith(".metadata.json")) {
+ return -1;
+ }
+ try {
+ return Long.parseLong(name.substring(1, name.indexOf('.')));
+ } catch (NumberFormatException ignored) {
+ return -1;
+ }
+ }
+
+ static final String RETIRE_PENDING_FILENAME = "retire-pending";
+
+ /** What the marker of a rollback holds, or null when there is none or it
cannot be read. */
+ @Nullable
+ private String readRetirePending() {
+ try {
+ return table.fileIO()
+ .readFileUtf8(
+ new Path(pathFactory.metadataDirectory(),
RETIRE_PENDING_FILENAME))
+ .trim();
+ } catch (Exception e) {
+ return null;
+ }
+ }
+
+ /**
+ * Deletes the marker of a rollback once the leftovers of the rollback are
retired, but only if
+ * it still holds what it held before they were listed. A rollback writes
it again, and the
+ * leftovers of that one are not in the listing.
+ */
+ private void clearRetirePending(Snapshot source, @Nullable String
seenBeforeListing)
+ throws IOException {
+ if (seenBeforeListing == null) {
+ return;
+ }
+ cleanUnderLock(
+ source,
+ published -> isPublished(published, null),
+ () -> {
+ if (seenBeforeListing.equals(readRetirePending())) {
+ table.fileIO()
+ .deleteQuietly(
+ new Path(
+
pathFactory.metadataDirectory(),
+ RETIRE_PENDING_FILENAME));
+ }
+ });
+ }
+
+ @FunctionalInterface
+ interface LockedAction<T> {
+ T run(Lock lock) throws IOException;
+ }
+
+ /**
+ * Runs an operation under the lock that the commits of the table take, so
that the operation
+ * and a rollback or a commit of the table cannot interleave. The lock
comes from the catalog of
+ * the table, and a table without one is not protected.
+ */
+ private <T> T runUnderTableLock(LockedAction<T> action) throws IOException
{
+ return runWithLock(table, tableLock(), action);
+ }
+
+ private static <T> T runWithLock(
+ FileStoreTable table, SnapshotCommit lock, LockedAction<T> action)
throws IOException {
+ try {
+ return lock.runWithLock(() -> action.run(lock));
+ } catch (IOException | RuntimeException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new RuntimeException(
+ "Failed to run an operation with the lock of table " +
table.fullName(), e);
+ }
+ }
+
+ /**
+ * The lock of the table, which the catalog of the table provides. One is
made for the callback
+ * and closed with it: making one loads a catalog, and every publication
and cleanup takes the
+ * lock.
+ */
+ private synchronized SnapshotCommit tableLock() {
+ if (tableLock == null) {
+ // the callbacks of a tag have no commit user, and a lock can ask
for one
+ String lockUser = commitUser.isEmpty() ?
UUID.randomUUID().toString() : commitUser;
+ tableLock =
+ table.catalogEnvironment()
+ .snapshotCommit(table.snapshotManager(),
table.coreOptions(), lockUser);
+ }
+ return tableLock;
+ }
+
+ /**
+ * Runs the part of a rollback that deletes snapshots under the lock that
the publication of
+ * Iceberg metadata takes, so a rollback never happens between the last
check of a snapshot and
+ * the publication of its metadata. It takes the lock whatever the Iceberg
setting of the handle
+ * that rolls back is, because one table can have handles with Iceberg
metadata on and off, and
+ * a rollback through any of them replaces snapshots. The setting of the
table decides, and a
+ * table that never has Iceberg metadata rolls back without a lock.
+ */
+ public static void runRollback(FileStoreTable table, Runnable rollback) {
+ if (!icebergMetadataEnabled(table)) {
Review Comment:
**[P2] Coordinate rollback while an older publisher may still be active**
This bypass depends on the latest persisted storage setting. An already-open
writer can still hold an enabled callback after `metadata.iceberg.storage` is
disabled, and a job-local enablement can also differ from the stored setting.
Such a callback still relies on the publication lock after its identity check,
while the rollback no longer acquires that lock. Could the decision to
coordinate be independent of the current flag, or could a persisted fence keep
older callbacks from publishing after a setting change? Please cover an
existing enabled writer across a storage-option change as well as the existing
disabled-handle case.
##########
paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergCommitCallback.java:
##########
@@ -484,6 +567,7 @@ private void createMetadataWithoutBase(
int lastColumnIdFloor,
long nextRowIdFloor)
throws IOException {
+ replayPoint(ReplayPoint.REBUILD, snapshotId);
SnapshotReader snapshotReader =
table.newSnapshotReader().withSnapshot(snapshotId);
Review Comment:
**[P1] Keep the original snapshot identity through replay fallback**
I reproduced a rollback boundary on `edb89d273`. A retry chose rebuild
because the gap exceeded the replay limit. While that fallback was pending, the
source snapshot was rolled back and its numeric ID was reused by a different
commit with Iceberg publication disabled. The old retry then wrote Iceberg
metadata stamped with the replacement commit's identity and advanced the
version hint. The fallback discards the Snapshot that was checked and reloads
by number here, so publication revalidates the replacement instead of the
original source. Could the fallback carry the checked Snapshot through both
manifest planning and final publication and abandon the work when that identity
changes?
##########
paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergGapReplay.java:
##########
@@ -0,0 +1,1064 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.iceberg;
+
+import org.apache.paimon.Snapshot;
+import org.apache.paimon.fs.Path;
+import org.apache.paimon.iceberg.IcebergCommitCallback.BuildResult;
+import org.apache.paimon.iceberg.IcebergCommitCallback.FileChangesCollector;
+import org.apache.paimon.iceberg.IcebergCommitCallback.MetadataNotWritten;
+import org.apache.paimon.iceberg.IcebergCommitCallback.Publication;
+import org.apache.paimon.iceberg.IcebergCommitCallback.RebuildRequest;
+import org.apache.paimon.iceberg.IcebergCommitCallback.ReplayPoint;
+import org.apache.paimon.iceberg.IcebergCommitCallback.SchemaCache;
+import org.apache.paimon.iceberg.manifest.IcebergManifestFile;
+import org.apache.paimon.iceberg.manifest.IcebergManifestFileMeta;
+import org.apache.paimon.iceberg.manifest.IcebergManifestList;
+import org.apache.paimon.iceberg.metadata.IcebergMetadata;
+import org.apache.paimon.iceberg.metadata.IcebergPartitionField;
+import org.apache.paimon.iceberg.metadata.IcebergPartitionSpec;
+import org.apache.paimon.iceberg.metadata.IcebergSchema;
+import org.apache.paimon.iceberg.metadata.IcebergSnapshot;
+import org.apache.paimon.iceberg.metadata.IcebergSnapshotSummary;
+import org.apache.paimon.manifest.ManifestFileMeta;
+import org.apache.paimon.manifest.ManifestList;
+import org.apache.paimon.operation.Lock;
+import org.apache.paimon.options.Options;
+import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.utils.JsonSerdeUtil;
+import org.apache.paimon.utils.SnapshotManager;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nullable;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+
+import static
org.apache.paimon.iceberg.IcebergCommitCallback.SNAPSHOT_SUMMARY_PAIMON_COMMIT_IDENTITY;
+import static
org.apache.paimon.iceberg.IcebergCommitCallback.SNAPSHOT_SUMMARY_REPLAY_ORIGINALS;
+import static
org.apache.paimon.iceberg.IcebergCommitCallback.SNAPSHOT_SUMMARY_TOTAL_RECORDS;
+import static org.apache.paimon.iceberg.IcebergCommitCallback.commitIdentity;
+import static org.apache.paimon.iceberg.IcebergCommitCallback.generationOf;
+import static org.apache.paimon.iceberg.IcebergCommitCallback.isMissingFile;
+import static
org.apache.paimon.iceberg.IcebergCommitCallback.lastPartitionIdOf;
+import static
org.apache.paimon.iceberg.IcebergCommitCallback.metadataVersionOf;
+import static
org.apache.paimon.iceberg.IcebergCommitCallback.replayedOriginals;
+
+/**
+ * The replay of the Iceberg snapshots that a missing base left out, for one
snapshot of Paimon: it
+ * extends the newest older metadata that is valid, instead of rebuilding from
every data file. See
+ * {@link IcebergCommitCallback}.
+ */
+final class IcebergGapReplay {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(IcebergGapReplay.class);
+
+ private static final int MAX_CANDIDATES = 5;
+
+ private final IcebergCommitCallback callback;
+ private final FileStoreTable table;
+ private final IcebergPathFactory pathFactory;
+ private final IcebergManifestList manifestList;
+ private final IcebergManifestFile manifestFile;
+
+ private final Snapshot snapshot;
+ private final long snapshotId;
+ private final FileChangesCollector collector;
+ private final RebuildRequest defaultRebuild;
+
+ private final int maxSnapshots;
+ private final long maxDeltaEntries;
+ private final long maxBaseManifestBytes;
+ private final long maxDurationNanos;
+ private final long startNanos = System.nanoTime();
+
+ private final List<Long> candidates = new ArrayList<>();
+ private int nextCandidate = 0;
+ private long chargedBytes = 0;
+
+ // what a rebuild after a failed replay has to inherit
+ @Nullable private String baseUuid;
+ private int maxLastColumnId;
+ private long maxNextRowId;
+
+ // true once the metadata file may exist: from then on no file of the
replay is deleted
+ private boolean exposing = false;
+
+ IcebergGapReplay(
+ IcebergCommitCallback callback,
+ Snapshot snapshot,
+ FileChangesCollector collector,
+ RebuildRequest rebuild) {
+ this.callback = callback;
+ this.table = callback.table;
+ this.pathFactory = callback.pathFactory;
+ this.manifestList = callback.manifestList;
+ this.manifestFile = callback.manifestFile;
+ this.snapshot = snapshot;
+ this.snapshotId = snapshot.id();
+ this.collector = collector;
+ this.defaultRebuild = rebuild;
+ this.maxLastColumnId = rebuild.lastColumnIdFloor;
+ this.maxNextRowId = rebuild.nextRowIdFloor;
+
+ Options options = table.coreOptions().toConfiguration();
+ this.maxSnapshots = options.get(IcebergOptions.REPLAY_MAX_SNAPSHOTS);
+ this.maxDeltaEntries =
options.get(IcebergOptions.REPLAY_MAX_DELTA_ENTRIES);
+ this.maxBaseManifestBytes =
+
options.get(IcebergOptions.REPLAY_MAX_BASE_MANIFEST_BYTES).getBytes();
+ this.maxDurationNanos =
options.get(IcebergOptions.REPLAY_MAX_DURATION).toNanos();
+ }
+
+ /**
+ * Replays the snapshot, and returns what a rebuild has to inherit when no
replay was possible,
+ * or it failed before it exposed anything, and the snapshot is still the
head. Returns null
+ * when nothing is left to do.
+ */
+ @Nullable
+ RebuildRequest run() throws IOException {
+ listCandidates();
+ ReplayBase base = nextBase();
+ if (base == null) {
+ return rebuildIfHead("there is no usable older Iceberg metadata");
+ }
+ pathFactory.startRecording();
+ try {
+ return replay(base);
+ } finally {
+ pathFactory.stopRecording();
+ }
+ }
+
+ @Nullable
+ private RebuildRequest replay(ReplayBase first) throws IOException {
+ try {
+ ReplayBase base = first;
+ List<Snapshot> steps;
+ Acquired acquired = null;
+ for (int attempt = 0; ; attempt++) {
+ try {
+ steps = readSteps(base);
+ acquired = acquire(base, steps);
+ break;
+ } catch (BrokenHistory e) {
+ LOG.info(
+ "The history of Iceberg metadata v{} of table {}
is broken: {}",
+ base.version,
+ table.fullName(),
+ e.getMessage());
+ deleteCreatedFiles();
+ callback.replayPoint(ReplayPoint.CANDIDATE_REJECTED,
base.version);
+ base = attempt == 0 ? nextBase() : null;
+ if (base == null) {
+ throw e;
+ }
+ }
+ }
+
+ IcebergMetadata result = runSteps(acquired, steps);
+ finish(result, acquired);
+ LOG.info(
+ "Replayed the Iceberg metadata of snapshots {} to {} of
table {} on top "
+ + "of v{}, and copied {} manifests.",
+ base.version + 1,
+ snapshotId,
+ table.fullName(),
+ base.version,
+ acquired.copied.size());
+ return null;
+ } catch (ReplaySourceReplaced e) {
+ LOG.info(
+ "Snapshot {} of table {} was replaced while its Iceberg
metadata was "
+ + "replayed, so nothing is written: {}",
+ snapshotId,
+ table.fullName(),
+ e.getMessage());
+ deleteCreatedFiles();
+ callback.replayPoint(ReplayPoint.ABANDONED, snapshotId);
+ return null;
+ } catch (ReplayAbort e) {
+ deleteCreatedFiles();
+ return rebuildIfHead(e.getMessage());
+ } catch (IOException | RuntimeException | Error e) {
+ if (!exposing) {
+ deleteCreatedFiles();
+ }
+ throw e;
+ }
+ }
+
+ //
---------------------------------------------------------------------------------
+ // Base selection
+ //
---------------------------------------------------------------------------------
+
+ private void listCandidates() throws IOException {
+ // the newest first, each version once, whatever else the directory
holds
+ Set<Long> versions = new TreeSet<>(Comparator.reverseOrder());
+ Iterator<Path> it =
+ pathFactory.getAllMetadataPathBefore(table.fileIO(),
snapshotId).iterator();
+ while (it.hasNext()) {
+ long version = metadataVersionOf(it.next());
+ if (version >= 0) {
+ versions.add(version);
+ }
+ }
+ for (long version : versions) {
+ if (candidates.size() == MAX_CANDIDATES) {
+ break;
+ }
+ candidates.add(version);
+ }
+ }
+
+ @Nullable
+ private ReplayBase nextBase() throws IOException {
+ while (nextCandidate < candidates.size()) {
+ long version = candidates.get(nextCandidate++);
+ String problem = null;
+ ReplayBase base = null;
+ try {
+ base = checkBase(version);
+ } catch (ReplayAbort e) {
+ problem = e.getMessage();
+ }
+ if (base != null) {
+ baseUuid = base.metadata.tableUuid();
+ maxLastColumnId = Math.max(maxLastColumnId,
base.metadata.lastColumnId());
+ if (base.metadata.nextRowId() != null) {
+ maxNextRowId = Math.max(maxNextRowId,
base.metadata.nextRowId());
+ }
+ callback.replayPoint(ReplayPoint.BASE_SELECTED, version);
+ return base;
+ }
+ LOG.info(
+ "Iceberg metadata v{} of table {} is not a base for
snapshot {}: {}",
+ version,
+ table.fullName(),
+ snapshotId,
+ problem);
+ callback.replayPoint(ReplayPoint.CANDIDATE_REJECTED, version);
+ }
+ return null;
+ }
+
+ private ReplayBase checkBase(long version) throws IOException {
+ if (snapshotId - version > maxSnapshots) {
Review Comment:
**[P2] Preserve the table UUID when a valid older base exceeds the replay
budget**
I created v1, left the metadata missing for snapshots 2 and 3, and retried
v3 with `replay.max-snapshots=1`. The fallback rebuilt all rows but published a
new table UUID. An already-open Iceberg reader refreshing from v1 to v3 then
failed with `Table UUID does not match`. With the same table and limit 2,
replay preserved the UUID and refresh succeeded. Could we separate validation
for identity inheritance from the gap-size check for actually replaying, so a
valid older version can still supply the UUID on rebuild?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]