JingsongLi commented on code in PR #9348:
URL: https://github.com/apache/paimon/pull/9348#discussion_r4492936811
##########
paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergCommitCallback.java:
##########
@@ -465,6 +467,463 @@ private void createMetadata(
}
}
+ /**
+ * Create Iceberg metadata when no usable base metadata exists: either the
very first Iceberg
+ * commit for this table, or a recovery after the previous metadata became
unusable (format
+ * version change, missing row lineage, Iceberg-layer commit failure).
+ *
+ * <p>By default only the current snapshot is exposed to Iceberg. With
{@link
+ * IcebergOptions#SYNC_FULL_HISTORY} the whole retained Paimon history is
replayed instead, so
+ * Iceberg readers keep time travel and tags (see <a
+ *
href="https://github.com/apache/paimon/issues/6107">apache/paimon#6107</a>).
+ */
+ private void recreateMetadata(
+ long snapshotId,
+ @Nullable String inheritUuid,
+ int lastColumnIdFloor,
+ long nextRowIdFloor)
+ throws IOException {
+ if (syncFullHistory) {
+ rebuildFullHistory(snapshotId, inheritUuid, lastColumnIdFloor,
nextRowIdFloor);
+ } else {
+ createMetadataWithoutBase(snapshotId, inheritUuid,
lastColumnIdFloor, nextRowIdFloor);
+ }
+ }
+
+ /**
+ * Rebuild Iceberg metadata from every Paimon snapshot that is still
retained, ending at {@code
+ * currentSnapshotId}: create metadata afresh for the earliest retained
snapshot, then replay
+ * each following snapshot on top of its predecessor, exactly like live
commits would have.
+ * Schemas, tags and (for format version 3) the row-id space therefore
accumulate consistently
+ * across the whole replayed history.
+ *
+ * <p>Every replay step writes to a uniquely named staging path; the
published chain, the
+ * version hint and the external catalog stay untouched until the final
staged metadata is
+ * durable, and only then is the staged chain promoted into place. A
rebuild that fails at any
+ * point therefore leaves the previously published metadata fully
readable, and its staged
+ * leftovers are removed by the next rebuild. Replayed snapshots keep
their original Paimon
+ * commit timestamps and are subject to the same retention policy ({@link
+ * CoreOptions#SNAPSHOT_NUM_RETAINED_MIN}, {@link
CoreOptions#SNAPSHOT_TIME_RETAINED}, ...) that
+ * live commits apply.
+ */
+ private void rebuildFullHistory(
+ long currentSnapshotId,
+ @Nullable String inheritUuid,
+ int lastColumnIdFloor,
+ long nextRowIdFloor)
+ throws IOException {
+ SnapshotManager snapshotManager = table.snapshotManager();
+ Long earliest = snapshotManager.earliestSnapshotId();
+ long startId = earliest == null ? currentSnapshotId :
Math.min(earliest, currentSnapshotId);
+
+ deleteStagedLeftovers(currentSnapshotId);
+
+ // Resume from the newest existing metadata below the current
snapshot, if it is usable.
+ // Anything older than the newest existing file is stale by
definition: live commits only
+ // ever read the immediately preceding metadata.
+ long baseId = -1;
+ for (long id = currentSnapshotId - 1; id >= startId; id--) {
+ Path metadataPath = pathFactory.toMetadataPath(id);
+ if (table.fileIO().exists(metadataPath)) {
+ try {
+ IcebergMetadata metadata =
+ IcebergMetadata.fromPath(table.fileIO(),
metadataPath);
+ if (isUsableReplayBase(metadata, id, startId)) {
+ baseId = id;
+ }
+ } catch (Exception e) {
+ LOG.warn(
+ "Failed to read existing Iceberg metadata {},
rebuilding history from scratch",
+ metadataPath,
+ e);
+ }
+ break;
+ }
+ }
+
+ // the head snapshot leads the name, so a later rebuild can tell what
it supersedes
+ String rebuildUuid = currentSnapshotId + "-" + UUID.randomUUID();
+ try {
+ replayAndPromote(
+ currentSnapshotId,
+ startId,
+ baseId,
+ rebuildUuid,
+ inheritUuid,
+ lastColumnIdFloor,
+ nextRowIdFloor);
+ } catch (IOException | RuntimeException e) {
+ Long latest = snapshotManager.latestSnapshotId();
+ if (latest != null && latest > currentSnapshotId) {
+ // a newer commit owns publication now and its own rebuild may
have removed
+ // this one's staged files; the result would be discarded
anyway
+ LOG.info(
+ "Full-history rebuild for snapshot {} was superseded
by snapshot {} and "
+ + "stopped.",
+ currentSnapshotId,
+ latest,
+ e);
+ return;
+ }
+ throw e;
+ }
+ }
+
+ private void replayAndPromote(
+ long currentSnapshotId,
+ long startId,
+ long baseId,
+ String rebuildUuid,
+ @Nullable String inheritUuid,
+ int lastColumnIdFloor,
+ long nextRowIdFloor)
+ throws IOException {
+ SnapshotManager snapshotManager = table.snapshotManager();
+ long firstStagedId;
+ boolean freshRebuild = baseId == -1;
+ StaleBuild staleBuild = null;
+ if (freshRebuild) {
+ // No usable base: the whole old build is stale. Nothing of it is
touched during the
+ // replay, so an external catalog that still points at the old
metadata keeps a fully
+ // readable table; the old files are replaced and cleaned only by
the promotion. Their
+ // references are collected up front, tolerating unreadable files
(that is what
+ // triggered some rebuilds in the first place).
+ staleBuild = collectStaleBuild(currentSnapshotId);
+ firstStagedId = startId;
+ createMetadataWithoutBase(
+ startId,
+ inheritUuid,
+ lastColumnIdFloor,
+ nextRowIdFloor,
+ stagedMetadataPath(rebuildUuid, startId));
+ } else {
+ firstStagedId = baseId + 1;
+ }
+
+ for (long id = firstStagedId == startId ? startId + 1 : firstStagedId;
+ id <= currentSnapshotId;
+ id++) {
+ long snapshotId = id;
+ Snapshot snapshot = snapshotManager.snapshot(snapshotId);
+ Path basePath =
+ snapshotId - 1 < firstStagedId
+ ? pathFactory.toMetadataPath(snapshotId - 1)
+ : stagedMetadataPath(rebuildUuid, snapshotId - 1);
+ createMetadataWithBase(
+ (removedFiles, addedFiles) ->
+ collectFileChanges(snapshotId, removedFiles,
addedFiles),
+ indexFileHandler.scan(snapshot, DELETION_VECTORS_INDEX),
+ snapshot,
+ basePath,
+ lastColumnIdFloor,
+ nextRowIdFloor,
+ stagedMetadataPath(rebuildUuid, snapshotId));
+ }
+
+ boolean promoted =
+ promoteStagedReplay(rebuildUuid, firstStagedId,
currentSnapshotId, startId);
+ if (promoted && staleBuild != null) {
+ deleteStaleBuild(staleBuild, currentSnapshotId, startId);
+ }
+ }
+
+ /**
+ * Whether an existing metadata file can serve as the resume base of a
full-history replay. On
+ * top of the structural checks, the base must describe the live timeline:
an abandoned base (a
+ * rolled-back Paimon snapshot reused the id) or one carrying re-evolved
schema definitions
+ * would be rejected again by the first replay step, and a rebuild must
never select a base its
+ * own replay refuses to extend.
+ */
+ private boolean isUsableReplayBase(IcebergMetadata metadata, long baseId,
long startId) {
+ if (!isSameFormatVersion(metadata.formatVersion())) {
+ return false;
+ }
+ if (formatVersion >= IcebergMetadata.FORMAT_VERSION_V3 &&
metadata.nextRowId() == null) {
+ return false;
+ }
+ if (!coversRetainedPrefix(metadata, baseId, startId)) {
+ return false;
+ }
+ SnapshotManager snapshotManager = table.snapshotManager();
+ if (!snapshotManager.snapshotExists(baseId)
+ || !metadataMatchesSnapshot(metadata,
snapshotManager.snapshot(baseId))) {
+ return false;
+ }
+ SchemaCache schemaCache = new SchemaCache();
+ long latestSchemaId = schemaCache.getLatestSchemaId();
+ for (IcebergSchema known : metadata.schemas()) {
+ if (known.schemaId() > latestSchemaId
+ || !known.equals(schemaCache.get(known.schemaId()))) {
+ return false;
+ }
+ }
+ return true;
Review Comment:
[P2] Reject resume metadata whose manifest lists were already retired
A retained JSON is not necessarily extendable. With
`metadata.iceberg.delete-after-commit.enabled=false`, a normal single-snapshot
recovery runs `expireAllBefore`, deleting old manifest lists while keeping
their metadata JSON. On a later missing-head recovery with full-history
enabled, this check accepts such an older JSON based on
format/history/identity/schema alone; the first staged step then fails reading
its deleted current manifest list. Subsequent attempts keep selecting the same
unusable candidate rather than rebuilding.
I reproduced this without manually deleting any manifests: publish snapshots
1–2; fail snapshot-3 JSON publication; recover snapshot 4 with full-history off
(which really deletes list 2 but keeps JSON 2); lose the latest JSON and enable
full-history; snapshot 5 selects JSON 2 and fails with FileNotFound. All Paimon
snapshots/data remain valid: discarding the dangling JSONs then cold-replaying
snapshot 6 reads all six rows. A diagnostic control that rejects a candidate
when one of its referenced manifest lists is absent makes the original recovery
pass. Please validate the dependencies needed to extend/read the candidate, or
fall back to a fresh replay when extension fails.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]