0dunay0 commented on code in PR #10995: URL: https://github.com/apache/paimon/pull/10995#discussion_r4528862336
########## paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergGapReplay.java: ########## @@ -0,0 +1,1064 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.iceberg; + +import org.apache.paimon.Snapshot; +import org.apache.paimon.fs.Path; +import org.apache.paimon.iceberg.IcebergCommitCallback.BuildResult; +import org.apache.paimon.iceberg.IcebergCommitCallback.FileChangesCollector; +import org.apache.paimon.iceberg.IcebergCommitCallback.MetadataNotWritten; +import org.apache.paimon.iceberg.IcebergCommitCallback.Publication; +import org.apache.paimon.iceberg.IcebergCommitCallback.RebuildRequest; +import org.apache.paimon.iceberg.IcebergCommitCallback.ReplayPoint; +import org.apache.paimon.iceberg.IcebergCommitCallback.SchemaCache; +import org.apache.paimon.iceberg.manifest.IcebergManifestFile; +import org.apache.paimon.iceberg.manifest.IcebergManifestFileMeta; +import org.apache.paimon.iceberg.manifest.IcebergManifestList; +import org.apache.paimon.iceberg.metadata.IcebergMetadata; +import org.apache.paimon.iceberg.metadata.IcebergPartitionField; +import org.apache.paimon.iceberg.metadata.IcebergPartitionSpec; +import org.apache.paimon.iceberg.metadata.IcebergSchema; +import org.apache.paimon.iceberg.metadata.IcebergSnapshot; +import org.apache.paimon.iceberg.metadata.IcebergSnapshotSummary; +import org.apache.paimon.manifest.ManifestFileMeta; +import org.apache.paimon.manifest.ManifestList; +import org.apache.paimon.operation.Lock; +import org.apache.paimon.options.Options; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.utils.JsonSerdeUtil; +import org.apache.paimon.utils.SnapshotManager; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nullable; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.TreeSet; +import java.util.stream.Collectors; + +import static org.apache.paimon.iceberg.IcebergCommitCallback.SNAPSHOT_SUMMARY_PAIMON_COMMIT_IDENTITY; +import static org.apache.paimon.iceberg.IcebergCommitCallback.SNAPSHOT_SUMMARY_REPLAY_ORIGINALS; +import static org.apache.paimon.iceberg.IcebergCommitCallback.SNAPSHOT_SUMMARY_TOTAL_RECORDS; +import static org.apache.paimon.iceberg.IcebergCommitCallback.commitIdentity; +import static org.apache.paimon.iceberg.IcebergCommitCallback.generationOf; +import static org.apache.paimon.iceberg.IcebergCommitCallback.isMissingFile; +import static org.apache.paimon.iceberg.IcebergCommitCallback.lastPartitionIdOf; +import static org.apache.paimon.iceberg.IcebergCommitCallback.metadataVersionOf; +import static org.apache.paimon.iceberg.IcebergCommitCallback.replayedOriginals; + +/** + * The replay of the Iceberg snapshots that a missing base left out, for one snapshot of Paimon: it + * extends the newest older metadata that is valid, instead of rebuilding from every data file. See + * {@link IcebergCommitCallback}. + */ +final class IcebergGapReplay { + + private static final Logger LOG = LoggerFactory.getLogger(IcebergGapReplay.class); + + private static final int MAX_CANDIDATES = 5; + + private final IcebergCommitCallback callback; + private final FileStoreTable table; + private final IcebergPathFactory pathFactory; + private final IcebergManifestList manifestList; + private final IcebergManifestFile manifestFile; + + private final Snapshot snapshot; + private final long snapshotId; + private final FileChangesCollector collector; + private final RebuildRequest defaultRebuild; + + private final int maxSnapshots; + private final long maxDeltaEntries; + private final long maxBaseManifestBytes; + private final long maxDurationNanos; + private final long startNanos = System.nanoTime(); + + private final List<Long> candidates = new ArrayList<>(); + private int nextCandidate = 0; + private long chargedBytes = 0; + + // what a rebuild after a failed replay has to inherit + @Nullable private String baseUuid; + private int maxLastColumnId; + private long maxNextRowId; + + // true once the metadata file may exist: from then on no file of the replay is deleted + private boolean exposing = false; + + IcebergGapReplay( + IcebergCommitCallback callback, + Snapshot snapshot, + FileChangesCollector collector, + RebuildRequest rebuild) { + this.callback = callback; + this.table = callback.table; + this.pathFactory = callback.pathFactory; + this.manifestList = callback.manifestList; + this.manifestFile = callback.manifestFile; + this.snapshot = snapshot; + this.snapshotId = snapshot.id(); + this.collector = collector; + this.defaultRebuild = rebuild; + this.maxLastColumnId = rebuild.lastColumnIdFloor; + this.maxNextRowId = rebuild.nextRowIdFloor; + + Options options = table.coreOptions().toConfiguration(); + this.maxSnapshots = options.get(IcebergOptions.REPLAY_MAX_SNAPSHOTS); + this.maxDeltaEntries = options.get(IcebergOptions.REPLAY_MAX_DELTA_ENTRIES); + this.maxBaseManifestBytes = + options.get(IcebergOptions.REPLAY_MAX_BASE_MANIFEST_BYTES).getBytes(); + this.maxDurationNanos = options.get(IcebergOptions.REPLAY_MAX_DURATION).toNanos(); + } + + /** + * Replays the snapshot, and returns what a rebuild has to inherit when no replay was possible, + * or it failed before it exposed anything, and the snapshot is still the head. Returns null + * when nothing is left to do. + */ + @Nullable + RebuildRequest run() throws IOException { + listCandidates(); + ReplayBase base = nextBase(); + if (base == null) { + return rebuildIfHead("there is no usable older Iceberg metadata"); + } + pathFactory.startRecording(); + try { + return replay(base); + } finally { + pathFactory.stopRecording(); + } + } + + @Nullable + private RebuildRequest replay(ReplayBase first) throws IOException { + try { + ReplayBase base = first; + List<Snapshot> steps; + Acquired acquired = null; + for (int attempt = 0; ; attempt++) { + try { + steps = readSteps(base); + acquired = acquire(base, steps); + break; + } catch (BrokenHistory e) { + LOG.info( + "The history of Iceberg metadata v{} of table {} is broken: {}", + base.version, + table.fullName(), + e.getMessage()); + deleteCreatedFiles(); + callback.replayPoint(ReplayPoint.CANDIDATE_REJECTED, base.version); + base = attempt == 0 ? nextBase() : null; + if (base == null) { + throw e; + } + } + } + + IcebergMetadata result = runSteps(acquired, steps); + finish(result, acquired); + LOG.info( + "Replayed the Iceberg metadata of snapshots {} to {} of table {} on top " + + "of v{}, and copied {} manifests.", + base.version + 1, + snapshotId, + table.fullName(), + base.version, + acquired.copied.size()); + return null; + } catch (ReplaySourceReplaced e) { + LOG.info( + "Snapshot {} of table {} was replaced while its Iceberg metadata was " + + "replayed, so nothing is written: {}", + snapshotId, + table.fullName(), + e.getMessage()); + deleteCreatedFiles(); + callback.replayPoint(ReplayPoint.ABANDONED, snapshotId); + return null; + } catch (ReplayAbort e) { + deleteCreatedFiles(); + return rebuildIfHead(e.getMessage()); + } catch (IOException | RuntimeException | Error e) { + if (!exposing) { + deleteCreatedFiles(); + } + throw e; + } + } + + // --------------------------------------------------------------------------------- + // Base selection + // --------------------------------------------------------------------------------- + + private void listCandidates() throws IOException { + // the newest first, each version once, whatever else the directory holds + Set<Long> versions = new TreeSet<>(Comparator.reverseOrder()); + Iterator<Path> it = + pathFactory.getAllMetadataPathBefore(table.fileIO(), snapshotId).iterator(); + while (it.hasNext()) { + long version = metadataVersionOf(it.next()); + if (version >= 0) { + versions.add(version); + } + } + for (long version : versions) { + if (candidates.size() == MAX_CANDIDATES) { + break; + } + candidates.add(version); + } + } + + @Nullable + private ReplayBase nextBase() throws IOException { + while (nextCandidate < candidates.size()) { + long version = candidates.get(nextCandidate++); + String problem = null; + ReplayBase base = null; + try { + base = checkBase(version); + } catch (ReplayAbort e) { + problem = e.getMessage(); + } + if (base != null) { + baseUuid = base.metadata.tableUuid(); + maxLastColumnId = Math.max(maxLastColumnId, base.metadata.lastColumnId()); + if (base.metadata.nextRowId() != null) { + maxNextRowId = Math.max(maxNextRowId, base.metadata.nextRowId()); + } + callback.replayPoint(ReplayPoint.BASE_SELECTED, version); + return base; + } + LOG.info( + "Iceberg metadata v{} of table {} is not a base for snapshot {}: {}", + version, + table.fullName(), + snapshotId, + problem); + callback.replayPoint(ReplayPoint.CANDIDATE_REJECTED, version); + } + return null; + } + + private ReplayBase checkBase(long version) throws IOException { + if (snapshotId - version > maxSnapshots) { Review Comment: The gap-size check is now separate from validation. A valid base beyond replay.max-snapshots is rejected for replay, but it still gives the rebuild its table UUID, and the last column id and next row id as floors. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
