0dunay0 commented on code in PR #10995:
URL: https://github.com/apache/paimon/pull/10995#discussion_r4488322215


##########
paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergCommitCallback.java:
##########
@@ -2079,6 +2303,913 @@ private long 
computeTotalFilesSizeFromManifests(List<IcebergManifestFileMeta> ma
         return total;
     }
 
+    // 
-------------------------------------------------------------------------------------
+    // Gap replay
+    // 
-------------------------------------------------------------------------------------
+
+    /** The points of a replay that tests can observe and interrupt. */
+    enum ReplayPoint {
+        /** A base candidate was accepted, the value is its version. */
+        BASE_SELECTED,
+        /** A base candidate was rejected, the value is its version. */
+        CANDIDATE_REJECTED,
+        /** The lists of the retained history were read and charged, nothing 
is copied yet. */
+        LISTS_READ,
+        /** The retained history is copied, the value is the number of copied 
manifests. */
+        COPIED,
+        /** A snapshot was replayed, the value is its snapshot id. */
+        STEP_DONE,
+        /** The replayed metadata is complete, its file is not written yet. */
+        BEFORE_WRITE,
+        /** The metadata file of the replay is durable. */
+        EXPOSED,
+        /** The replay was dropped and nothing was written for the snapshot. */
+        ABANDONED,
+        /** A rebuild from every data file starts. */
+        REBUILD
+    }
+
+    /** Observes every {@link ReplayPoint}, and may throw to interrupt the 
replay. */
+    @FunctionalInterface
+    interface ReplayHook {
+        void at(ReplayPoint point, long value);
+    }
+
+    /** What a rebuild from the table inherits from the metadata it replaces. 
*/
+    private static final class RebuildRequest {
+
+        @Nullable private final String inheritUuid;
+        private final int lastColumnIdFloor;
+        private final long nextRowIdFloor;
+
+        private RebuildRequest(
+                @Nullable String inheritUuid, int lastColumnIdFloor, long 
nextRowIdFloor) {
+            this.inheritUuid = inheritUuid;
+            this.lastColumnIdFloor = lastColumnIdFloor;
+            this.nextRowIdFloor = nextRowIdFloor;
+        }
+    }
+
+    /** An expected failure of a replay: the replay is dropped, the caller 
decides what is next. */
+    private static class ReplayAbort extends RuntimeException {
+
+        private ReplayAbort(String message) {
+            super(message);
+        }
+
+        private ReplayAbort(String message, Throwable cause) {
+            super(message, cause);
+        }
+    }
+
+    /** A snapshot of the retained history is missing or damaged, so the base 
cannot be used. */
+    private static final class BrokenHistory extends ReplayAbort {
+
+        private BrokenHistory(String message, Throwable cause) {
+            super(message, cause);
+        }
+
+        private BrokenHistory(String message) {
+            super(message);
+        }
+    }
+
+    /** The Paimon snapshot was replaced by a rollback or a recreation while 
it was replayed. */
+    private static final class ReplaySourceReplaced extends RuntimeException {
+
+        private ReplaySourceReplaced(String message) {
+            super(message);
+        }
+    }
+
+    /** The reason a replay is not attempted, or null when the table is 
eligible. */
+    @Nullable
+    private String replayIneligibleReason() {
+        CoreOptions options = table.coreOptions();
+        if (!table.primaryKeys().isEmpty()) {
+            return "the table has primary keys";
+        }
+        if (options.deletionVectorsEnabled()) {
+            return "deletion vectors are enabled";
+        }
+        if (options.dataEvolutionEnabled()) {
+            return "data evolution is enabled";
+        }
+        if (formatVersion != IcebergMetadata.FORMAT_VERSION_V2) {
+            return "the Iceberg format version is " + formatVersion;
+        }
+        switch (storageType) {
+            case TABLE_LOCATION:
+            case HIVE_CATALOG:
+            case HADOOP_CATALOG:
+                return null;
+            default:
+                return "the storage type is " + storageType;
+        }
+    }
+
+    /**
+     * Replays the Iceberg snapshots after the newest valid older metadata 
version instead of
+     * rebuilding from every data file. Returns null when nothing is left to 
do, and what a rebuild
+     * has to inherit when the replay failed before it exposed anything and 
the snapshot is still
+     * the head.
+     */
+    @Nullable
+    private RebuildRequest replayOrRebuild(
+            Snapshot snapshot, FileChangesCollector collector, RebuildRequest 
rebuild)
+            throws IOException {
+        if (snapshot.id() == Snapshot.FIRST_SNAPSHOT_ID) {
+            return rebuild;
+        }
+        String reason = replayIneligibleReason();
+        if (reason != null) {
+            LOG.warn(
+                    "Table {} does not replay missing Iceberg metadata ({}), 
so it is rebuilt.",
+                    table.fullName(),
+                    reason);
+            return rebuild;
+        }
+        return new GapReplay(snapshot, collector, rebuild).run();
+    }
+
+    /** One run of the replay for one snapshot. */
+    private final class GapReplay {
+
+        private static final int MAX_CANDIDATES = 5;
+
+        private final Snapshot snapshot;
+        private final long snapshotId;
+        private final FileChangesCollector collector;
+        private final RebuildRequest defaultRebuild;
+
+        private final int maxSnapshots;
+        private final long maxDeltaEntries;
+        private final long maxBaseManifestBytes;
+        private final long maxDurationNanos;
+        private final long startNanos = System.nanoTime();
+
+        private final List<Long> candidates = new ArrayList<>();
+        private int nextCandidate = 0;
+        private long chargedBytes = 0;
+
+        // what a rebuild after a failed replay has to inherit
+        @Nullable private String baseUuid;
+        private int maxLastColumnId;
+        private long maxNextRowId;
+
+        // true once the metadata file may exist: from then on no file of the 
replay is deleted
+        private boolean exposing = false;
+
+        private GapReplay(
+                Snapshot snapshot, FileChangesCollector collector, 
RebuildRequest rebuild) {
+            this.snapshot = snapshot;
+            this.snapshotId = snapshot.id();
+            this.collector = collector;
+            this.defaultRebuild = rebuild;
+            this.maxLastColumnId = rebuild.lastColumnIdFloor;
+            this.maxNextRowId = rebuild.nextRowIdFloor;
+
+            Options options = table.coreOptions().toConfiguration();
+            this.maxSnapshots = 
options.get(IcebergOptions.REPLAY_MAX_SNAPSHOTS);
+            this.maxDeltaEntries = 
options.get(IcebergOptions.REPLAY_MAX_DELTA_ENTRIES);
+            this.maxBaseManifestBytes =
+                    
options.get(IcebergOptions.REPLAY_MAX_BASE_MANIFEST_BYTES).getBytes();
+            this.maxDurationNanos = 
options.get(IcebergOptions.REPLAY_MAX_DURATION).toNanos();
+        }
+
+        /** See {@link #replayOrRebuild}. */
+        @Nullable
+        private RebuildRequest run() throws IOException {
+            listCandidates();
+            ReplayBase base = nextBase();
+            if (base == null) {
+                LOG.info(
+                        "No usable older Iceberg metadata for snapshot {} of 
table {}, "
+                                + "so it is rebuilt.",
+                        snapshotId,
+                        table.fullName());
+                return defaultRebuild;
+            }
+            pathFactory.startRecording();
+            try {
+                return replay(base);
+            } finally {
+                pathFactory.stopRecording();
+            }
+        }
+
+        @Nullable
+        private RebuildRequest replay(ReplayBase first) throws IOException {
+            try {
+                ReplayBase base = first;
+                List<Snapshot> steps;
+                Acquired acquired = null;
+                for (int attempt = 0; ; attempt++) {
+                    try {
+                        steps = readSteps(base);
+                        acquired = acquire(base, steps);
+                        break;
+                    } catch (BrokenHistory e) {
+                        LOG.info(
+                                "The history of Iceberg metadata v{} of table 
{} is broken: {}",
+                                base.version,
+                                table.fullName(),
+                                e.getMessage());
+                        deleteCreatedFiles();
+                        replayPoint(ReplayPoint.CANDIDATE_REJECTED, 
base.version);
+                        base = attempt == 0 ? nextBase() : null;
+                        if (base == null) {
+                            throw e;
+                        }
+                    }
+                }
+
+                IcebergMetadata result = runSteps(acquired, steps);
+                finish(result, acquired);
+                LOG.info(
+                        "Replayed the Iceberg metadata of snapshots {} to {} 
of table {} on top "
+                                + "of v{}, and copied {} manifests.",
+                        base.version + 1,
+                        snapshotId,
+                        table.fullName(),
+                        base.version,
+                        acquired.copied.size());
+                return null;
+            } catch (ReplaySourceReplaced e) {
+                LOG.info(
+                        "Snapshot {} of table {} was replaced while its 
Iceberg metadata was "
+                                + "replayed, so nothing is written: {}",
+                        snapshotId,
+                        table.fullName(),
+                        e.getMessage());
+                deleteCreatedFiles();
+                replayPoint(ReplayPoint.ABANDONED, snapshotId);
+                return null;
+            } catch (ReplayAbort e) {
+                deleteCreatedFiles();
+                Long latest = table.snapshotManager().latestSnapshotId();
+                if (latest != null && latest == snapshotId) {
+                    try {
+                        // the failure may have come before any step checked 
the source
+                        verifySource(snapshot);
+                    } catch (ReplaySourceReplaced replaced) {
+                        LOG.info(
+                                "Replaying the Iceberg metadata of snapshot {} 
of table {} failed "
+                                        + "and the snapshot was replaced, so 
nothing is "
+                                        + "written: {}",
+                                snapshotId,
+                                table.fullName(),
+                                replaced.getMessage());
+                        replayPoint(ReplayPoint.ABANDONED, snapshotId);
+                        return null;
+                    }
+                    LOG.info(
+                            "Replaying the Iceberg metadata of snapshot {} of 
table {} failed, "
+                                    + "so it is rebuilt: {}",
+                            snapshotId,
+                            table.fullName(),
+                            e.getMessage());
+                    return rebuildRequest();
+                }
+                LOG.info(
+                        "Replaying the Iceberg metadata of snapshot {} of 
table {} failed and "
+                                + "the head is {}, so it is not rebuilt: {}",
+                        snapshotId,
+                        table.fullName(),
+                        latest,
+                        e.getMessage());
+                replayPoint(ReplayPoint.ABANDONED, snapshotId);
+                return null;
+            } catch (IOException | RuntimeException | Error e) {
+                if (!exposing) {
+                    deleteCreatedFiles();
+                }
+                throw e;
+            }
+        }
+
+        // 
---------------------------------------------------------------------------------
+        // Base selection
+        // 
---------------------------------------------------------------------------------
+
+        private void listCandidates() throws IOException {
+            List<Long> versions = new ArrayList<>();
+            Iterator<Path> it =
+                    pathFactory.getAllMetadataPathBefore(table.fileIO(), 
snapshotId).iterator();
+            while (it.hasNext()) {
+                String name = it.next().getName();
+                try {
+                    versions.add(Long.parseLong(name.substring(1, 
name.indexOf('.'))));
+                } catch (NumberFormatException ignored) {
+                    // not a metadata version
+                }
+            }
+            versions.sort(Comparator.reverseOrder());
+            candidates.addAll(versions.subList(0, Math.min(versions.size(), 
MAX_CANDIDATES)));
+        }
+
+        @Nullable
+        private ReplayBase nextBase() throws IOException {
+            while (nextCandidate < candidates.size()) {
+                long version = candidates.get(nextCandidate++);
+                String problem = null;
+                ReplayBase base = null;
+                try {
+                    base = checkBase(version);
+                } catch (ReplayAbort e) {
+                    problem = e.getMessage();
+                }
+                if (base != null) {
+                    baseUuid = base.metadata.tableUuid();
+                    maxLastColumnId = Math.max(maxLastColumnId, 
base.metadata.lastColumnId());
+                    if (base.metadata.nextRowId() != null) {
+                        maxNextRowId = Math.max(maxNextRowId, 
base.metadata.nextRowId());
+                    }
+                    replayPoint(ReplayPoint.BASE_SELECTED, version);
+                    return base;
+                }
+                LOG.info(
+                        "Iceberg metadata v{} of table {} is not a base for 
snapshot {}: {}",
+                        version,
+                        table.fullName(),
+                        snapshotId,
+                        problem);
+                replayPoint(ReplayPoint.CANDIDATE_REJECTED, version);
+            }
+            return null;
+        }
+
+        private ReplayBase checkBase(long version) throws IOException {
+            if (snapshotId - version > maxSnapshots) {
+                throw new ReplayAbort(
+                        "it is "
+                                + (snapshotId - version)
+                                + " snapshots behind, more than "
+                                + IcebergOptions.REPLAY_MAX_SNAPSHOTS.key());
+            }
+            Snapshot paimonSnapshot = readSnapshot(version);
+            if (paimonSnapshot == null) {
+                throw new ReplayAbort("there is no Paimon snapshot " + 
version);
+            }
+            IcebergMetadata metadata = 
readUsableMetadata(pathFactory.toMetadataPath(version));
+            if (metadata == null) {
+                throw new ReplayAbort("it is missing or malformed");
+            }
+            if (metadata.formatVersion() != formatVersion) {
+                throw new ReplayAbort("its format version is " + 
metadata.formatVersion());
+            }
+            IcebergSnapshot current;
+            try {
+                current = metadata.currentSnapshot();
+            } catch (RuntimeException e) {
+                throw new ReplayAbort("its current snapshot is missing", e);
+            }
+            if (current.snapshotId() != version) {
+                throw new ReplayAbort("its current snapshot is " + 
current.snapshotId());
+            }
+            String identity =
+                    current.summary() == null
+                            ? null
+                            : 
current.summary().get(SNAPSHOT_SUMMARY_PAIMON_COMMIT_IDENTITY);
+            if (identity == null || 
!identity.equals(commitIdentity(paimonSnapshot))) {
+                throw new ReplayAbort("it was not built from Paimon snapshot " 
+ version);
+            }
+            return new ReplayBase(version, metadata);
+        }
+
+        // 
---------------------------------------------------------------------------------
+        // Pre-flight and acquisition of the history
+        // 
---------------------------------------------------------------------------------
+
+        /** The Paimon snapshots after the base up to N, and the check that 
they are replayable. */
+        private List<Snapshot> readSteps(ReplayBase base) {
+            checkDuration();
+            ManifestList deltaLists = 
table.store().manifestListFactory().create();
+            List<Snapshot> steps = new ArrayList<>();
+            long entries = 0;
+            for (long id = base.version + 1; id <= snapshotId; id++) {
+                Snapshot step = id == snapshotId ? snapshot : readSnapshot(id);
+                if (step == null) {
+                    throw new ReplayAbort("Paimon snapshot " + id + " does not 
exist");
+                }
+                try {
+                    for (ManifestFileMeta meta : 
deltaLists.readDeltaManifests(step)) {
+                        entries += meta.numAddedFiles() + 
meta.numDeletedFiles();
+                    }
+                } catch (RuntimeException e) {
+                    if (isMissingFile(e)) {
+                        throw new ReplayAbort(
+                                "the delta of Paimon snapshot " + id + " 
cannot be read", e);
+                    }
+                    throw e;
+                }
+                if (entries > maxDeltaEntries) {
+                    throw new ReplayAbort(
+                            "the snapshots after v"
+                                    + base.version
+                                    + " change more than "
+                                    + maxDeltaEntries
+                                    + " files");
+                }
+                steps.add(step);
+            }
+            return steps;
+        }
+
+        /** Index of the first snapshot of the base that retention keeps for 
snapshot N. */
+        private int firstRetained(IcebergMetadata base, List<Snapshot> steps) {
+            List<IcebergSnapshot> baseSnapshots = base.snapshots();
+            int total = baseSnapshots.size() + steps.size();
+            for (int i = 0; i + 1 < total; i++) {
+                boolean expire;
+                if (i < baseSnapshots.size()) {
+                    expire = shouldExpire(baseSnapshots.get(i), snapshotId);
+                } else {
+                    Snapshot step = steps.get(i - baseSnapshots.size());
+                    expire = shouldExpire(step.id(), step.timeMillis(), 
snapshotId);
+                }
+                if (!expire) {
+                    return Math.min(i, baseSnapshots.size());
+                }
+            }
+            // like the build helper, nothing is dropped when every older 
snapshot expires
+            return 0;
+        }
+
+        /**
+         * Copies the manifests and lists the output keeps, and those of the 
working base, so the
+         * replay owns everything it reads and publishes.
+         */
+        private Acquired acquire(ReplayBase base, List<Snapshot> steps) throws 
IOException {
+            List<IcebergSnapshot> baseSnapshots = base.metadata.snapshots();
+            int first = firstRetained(base.metadata, steps);
+            List<IcebergSnapshot> retained =
+                    new ArrayList<>(baseSnapshots.subList(first, 
baseSnapshots.size()));
+            IcebergSnapshot working = base.metadata.currentSnapshot();
+            List<IcebergSnapshot> toCopy = new ArrayList<>(retained);
+            boolean workingRetained = first < baseSnapshots.size();
+            if (!workingRetained) {
+                // the working base is always acquired, even when the output 
drops its own entry
+                toCopy.add(working);
+            }
+
+            Map<String, Long> unique = new LinkedHashMap<>();
+            for (IcebergSnapshot s : toCopy) {
+                for (IcebergManifestFileMeta meta : readBaseList(s)) {
+                    unique.putIfAbsent(meta.manifestPath(), 
meta.manifestLength());
+                }
+            }
+            long bytes = 0;
+            for (long length : unique.values()) {
+                bytes += length;
+            }
+            charge(bytes);
+            replayPoint(ReplayPoint.LISTS_READ, base.version);
+
+            Map<String, String> copied = new HashMap<>();
+            Map<Long, String> originalLists = new HashMap<>();
+            List<IcebergSnapshot> retainedCopies = new ArrayList<>();
+            IcebergSnapshot workingCopy = null;
+            for (IcebergSnapshot s : toCopy) {
+                checkDuration();
+                List<IcebergManifestFileMeta> owned = new ArrayList<>();
+                for (IcebergManifestFileMeta meta : readBaseList(s)) {
+                    String target = copied.get(meta.manifestPath());
+                    if (target == null) {
+                        checkDuration();
+                        target = copyManifest(meta);
+                        copied.put(meta.manifestPath(), target);
+                    }
+                    owned.add(meta.withManifestPath(target));
+                }
+                String listName = manifestList.writeWithoutRolling(owned);
+                IcebergSnapshot copy =
+                        
s.withManifestList(pathFactory.toManifestListPath(listName).toString());
+                originalLists.put(s.snapshotId(), new 
Path(s.manifestList()).getName());
+                if (s.snapshotId() == working.snapshotId()) {
+                    workingCopy = copy;
+                }
+                if (retained.contains(s)) {
+                    retainedCopies.add(copy);
+                }
+            }
+            replayPoint(ReplayPoint.COPIED, copied.size());
+
+            List<IcebergSnapshot> ownedSnapshots = new 
ArrayList<>(retainedCopies);
+            if (!workingRetained) {
+                ownedSnapshots.add(workingCopy);
+            }
+            return new Acquired(
+                    base, base.metadata.withSnapshots(ownedSnapshots), 
originalLists, copied);
+        }
+
+        private List<IcebergManifestFileMeta> readBaseList(IcebergSnapshot 
icebergSnapshot) {
+            try {
+                return manifestList.read(new 
Path(icebergSnapshot.manifestList()).getName());
+            } catch (RuntimeException e) {
+                if (isMissingFile(e)) {
+                    throw new BrokenHistory(
+                            "the manifest list of snapshot "
+                                    + icebergSnapshot.snapshotId()
+                                    + " cannot be read",
+                            e);
+                }
+                throw e;
+            }
+        }
+
+        private String copyManifest(IcebergManifestFileMeta meta) throws 
IOException {
+            Path source = new Path(meta.manifestPath());
+            Path target = pathFactory.newManifestFile();
+            try {
+                table.fileIO().copyFile(source, target, false);
+            } catch (IOException e) {
+                if (isMissingFile(e)) {
+                    throw new BrokenHistory("manifest " + source + " cannot be 
copied", e);
+                }
+                throw e;
+            }
+            long length = table.fileIO().getFileSize(target);
+            if (length != meta.manifestLength()) {

Review Comment:
   Every copied manifest is now decoded entry by entry, and the number of files 
and rows it holds is compared with the counters of its manifest list entry, 
before the copy is relied on. Entries are dropped as they are read, so memory 
does not depend on the size of the file. A copy that does not decode, or whose 
counts differ, rejects the base like a missing file does: one other base is 
tried, and then the snapshot is rebuilt if it is still the head.
   
   A storage error or a read that ends early is not damage. It fails the commit 
and does not reject the base. The second commit fixes a case where a read that 
ended at an Avro block boundary looked like a complete file with the wrong 
counts. The check is charged against 
metadata.iceberg.replay.max-base-manifest-bytes, so the retained history counts 
twice.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to