0dunay0 commented on code in PR #10995:
URL: https://github.com/apache/paimon/pull/10995#discussion_r4495262292
##########
paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergCommitCallback.java:
##########
@@ -2079,6 +2822,926 @@ private long
computeTotalFilesSizeFromManifests(List<IcebergManifestFileMeta> ma
return total;
}
+ //
-------------------------------------------------------------------------------------
+ // Gap replay
+ //
-------------------------------------------------------------------------------------
+
+ /** The points of a replay that tests can observe and interrupt. */
+ enum ReplayPoint {
+ /** A base candidate was accepted, the value is its version. */
+ BASE_SELECTED,
+ /** A base candidate was rejected, the value is its version. */
+ CANDIDATE_REJECTED,
+ /** The lists of the retained history were read and charged, nothing
is copied yet. */
+ LISTS_READ,
+ /** The retained history is copied, the value is the number of copied
manifests. */
+ COPIED,
+ /** A snapshot was replayed, the value is its snapshot id. */
+ STEP_DONE,
+ /** The metadata is complete, its file is not written yet. */
+ BEFORE_WRITE,
+ /** A publication holds the table lock and checked its source, nothing
is written yet. */
+ LOCKED,
+ /** A cleanup of files that a publication made unneeded is about to
take the lock. */
+ BEFORE_CLEANUP,
+ /** A cleanup holds the table lock and checked its publication,
nothing is deleted yet. */
+ CLEANUP_LOCKED,
+ /** The metadata file of the replay is durable. */
+ EXPOSED,
+ /** The replay was dropped and nothing was written for the snapshot. */
+ ABANDONED,
+ /** A rebuild from every data file starts. */
+ REBUILD
+ }
+
+ /** Observes every {@link ReplayPoint}, and may throw to interrupt the
replay. */
+ @FunctionalInterface
+ interface ReplayHook {
+ void at(ReplayPoint point, long value);
+ }
+
+ /** What a rebuild from the table inherits from the metadata it replaces.
*/
+ private static final class RebuildRequest {
+
+ @Nullable private final String inheritUuid;
+ private final int lastColumnIdFloor;
+ private final long nextRowIdFloor;
+
+ private RebuildRequest(
+ @Nullable String inheritUuid, int lastColumnIdFloor, long
nextRowIdFloor) {
+ this.inheritUuid = inheritUuid;
+ this.lastColumnIdFloor = lastColumnIdFloor;
+ this.nextRowIdFloor = nextRowIdFloor;
+ }
+ }
+
+ /** An expected failure of a replay: the replay is dropped, the caller
decides what is next. */
+ private static class ReplayAbort extends RuntimeException {
+
+ private ReplayAbort(String message) {
+ super(message);
+ }
+
+ private ReplayAbort(String message, Throwable cause) {
+ super(message, cause);
+ }
+ }
+
+ /** A snapshot of the retained history is missing or damaged, so the base
cannot be used. */
+ private static final class BrokenHistory extends ReplayAbort {
+
+ private BrokenHistory(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ private BrokenHistory(String message) {
+ super(message);
+ }
+ }
+
+ /** The Paimon snapshot was replaced by a rollback or a recreation while
it was replayed. */
+ private static final class ReplaySourceReplaced extends RuntimeException {
+
+ private ReplaySourceReplaced(String message) {
+ super(message);
+ }
+ }
+
+ /** The reason a replay is not attempted, or null when the table is
eligible. */
+ @Nullable
+ private String replayIneligibleReason() {
+ CoreOptions options = table.coreOptions();
+ if (!table.primaryKeys().isEmpty()) {
+ return "the table has primary keys";
+ }
+ if (options.deletionVectorsEnabled()) {
+ return "deletion vectors are enabled";
+ }
+ if (options.dataEvolutionEnabled()) {
+ return "data evolution is enabled";
+ }
+ if (formatVersion != IcebergMetadata.FORMAT_VERSION_V2) {
+ return "the Iceberg format version is " + formatVersion;
+ }
+ switch (storageType) {
+ case TABLE_LOCATION:
+ case HIVE_CATALOG:
+ case HADOOP_CATALOG:
+ return null;
+ default:
+ return "the storage type is " + storageType;
+ }
+ }
+
+ /**
+ * Replays the Iceberg snapshots after the newest valid older metadata
version instead of
+ * rebuilding from every data file. Returns null when nothing is left to
do, and what a rebuild
+ * has to inherit when no replay was possible, or it failed before it
exposed anything, and the
+ * snapshot is still the head.
+ */
+ @Nullable
+ private RebuildRequest replayOrRebuild(
+ Snapshot snapshot, FileChangesCollector collector, RebuildRequest
rebuild)
+ throws IOException {
+ if (snapshot.id() == Snapshot.FIRST_SNAPSHOT_ID) {
+ return rebuild;
+ }
+ String reason = replayIneligibleReason();
+ if (reason != null) {
+ LOG.warn(
+ "Table {} does not replay missing Iceberg metadata ({}),
so it is rebuilt.",
+ table.fullName(),
+ reason);
+ return rebuild;
+ }
+ // A rollback can replace the snapshot after the last check of it and
before the metadata
+ // is written, and only a lock that the rollbacks and the commits of
the table take closes
+ // that. Without one the replay would publish metadata of the wrong
timeline, and a
+ // rebuild would too, so neither is attempted.
+ if (!tableLockCoordinates()) {
+ throw new IllegalStateException(
+ String.format(
+ "%s = %s needs a lock that keeps the other commits
and rollbacks of "
+ + "table %s out while its Iceberg metadata
is published, and "
+ + "no lock is configured for it. Configure
a lock for the "
+ + "catalog, or set %s = %s.",
+
IcebergOptions.METADATA_MISSING_BASE_STRATEGY.key(),
+ IcebergOptions.MissingBaseStrategy.REPLAY,
+ table.fullName(),
+
IcebergOptions.METADATA_MISSING_BASE_STRATEGY.key(),
+ IcebergOptions.MissingBaseStrategy.REBUILD));
+ }
+ return new GapReplay(snapshot, collector, rebuild).run();
+ }
+
+ /** One run of the replay for one snapshot. */
+ private final class GapReplay {
+
+ private static final int MAX_CANDIDATES = 5;
+
+ private final Snapshot snapshot;
+ private final long snapshotId;
+ private final FileChangesCollector collector;
+ private final RebuildRequest defaultRebuild;
+
+ private final int maxSnapshots;
+ private final long maxDeltaEntries;
+ private final long maxBaseManifestBytes;
+ private final long maxDurationNanos;
+ private final long startNanos = System.nanoTime();
+
+ private final List<Long> candidates = new ArrayList<>();
+ private int nextCandidate = 0;
+ private long chargedBytes = 0;
+
+ // what a rebuild after a failed replay has to inherit
+ @Nullable private String baseUuid;
+ private int maxLastColumnId;
+ private long maxNextRowId;
+
+ // true once the metadata file may exist: from then on no file of the
replay is deleted
+ private boolean exposing = false;
+
+ private GapReplay(
+ Snapshot snapshot, FileChangesCollector collector,
RebuildRequest rebuild) {
+ this.snapshot = snapshot;
+ this.snapshotId = snapshot.id();
+ this.collector = collector;
+ this.defaultRebuild = rebuild;
+ this.maxLastColumnId = rebuild.lastColumnIdFloor;
+ this.maxNextRowId = rebuild.nextRowIdFloor;
+
+ Options options = table.coreOptions().toConfiguration();
+ this.maxSnapshots =
options.get(IcebergOptions.REPLAY_MAX_SNAPSHOTS);
+ this.maxDeltaEntries =
options.get(IcebergOptions.REPLAY_MAX_DELTA_ENTRIES);
+ this.maxBaseManifestBytes =
+
options.get(IcebergOptions.REPLAY_MAX_BASE_MANIFEST_BYTES).getBytes();
+ this.maxDurationNanos =
options.get(IcebergOptions.REPLAY_MAX_DURATION).toNanos();
+ }
+
+ /** See {@link #replayOrRebuild}. */
+ @Nullable
+ private RebuildRequest run() throws IOException {
+ listCandidates();
+ ReplayBase base = nextBase();
+ if (base == null) {
+ return rebuildIfHead("there is no usable older Iceberg
metadata");
+ }
+ pathFactory.startRecording();
+ try {
+ return replay(base);
+ } finally {
+ pathFactory.stopRecording();
+ }
+ }
+
+ @Nullable
+ private RebuildRequest replay(ReplayBase first) throws IOException {
+ try {
+ ReplayBase base = first;
+ List<Snapshot> steps;
+ Acquired acquired = null;
+ for (int attempt = 0; ; attempt++) {
+ try {
+ steps = readSteps(base);
+ acquired = acquire(base, steps);
+ break;
+ } catch (BrokenHistory e) {
+ LOG.info(
+ "The history of Iceberg metadata v{} of table
{} is broken: {}",
+ base.version,
+ table.fullName(),
+ e.getMessage());
+ deleteCreatedFiles();
+ replayPoint(ReplayPoint.CANDIDATE_REJECTED,
base.version);
+ base = attempt == 0 ? nextBase() : null;
+ if (base == null) {
+ throw e;
+ }
+ }
+ }
+
+ IcebergMetadata result = runSteps(acquired, steps);
+ finish(result, acquired);
+ LOG.info(
+ "Replayed the Iceberg metadata of snapshots {} to {}
of table {} on top "
+ + "of v{}, and copied {} manifests.",
+ base.version + 1,
+ snapshotId,
+ table.fullName(),
+ base.version,
+ acquired.copied.size());
+ return null;
+ } catch (ReplaySourceReplaced e) {
+ LOG.info(
+ "Snapshot {} of table {} was replaced while its
Iceberg metadata was "
+ + "replayed, so nothing is written: {}",
+ snapshotId,
+ table.fullName(),
+ e.getMessage());
+ deleteCreatedFiles();
+ replayPoint(ReplayPoint.ABANDONED, snapshotId);
+ return null;
+ } catch (ReplayAbort e) {
+ deleteCreatedFiles();
+ return rebuildIfHead(e.getMessage());
+ } catch (IOException | RuntimeException | Error e) {
+ if (!exposing) {
+ deleteCreatedFiles();
+ }
+ throw e;
+ }
+ }
+
+ //
---------------------------------------------------------------------------------
+ // Base selection
+ //
---------------------------------------------------------------------------------
+
+ private void listCandidates() throws IOException {
+ List<Long> versions = new ArrayList<>();
+ Iterator<Path> it =
+ pathFactory.getAllMetadataPathBefore(table.fileIO(),
snapshotId).iterator();
+ while (it.hasNext()) {
+ String name = it.next().getName();
+ try {
+ versions.add(Long.parseLong(name.substring(1,
name.indexOf('.'))));
+ } catch (NumberFormatException ignored) {
+ // not a metadata version
+ }
+ }
+ versions.sort(Comparator.reverseOrder());
+ candidates.addAll(versions.subList(0, Math.min(versions.size(),
MAX_CANDIDATES)));
+ }
+
+ @Nullable
+ private ReplayBase nextBase() throws IOException {
+ while (nextCandidate < candidates.size()) {
+ long version = candidates.get(nextCandidate++);
+ String problem = null;
+ ReplayBase base = null;
+ try {
+ base = checkBase(version);
+ } catch (ReplayAbort e) {
+ problem = e.getMessage();
+ }
+ if (base != null) {
+ baseUuid = base.metadata.tableUuid();
+ maxLastColumnId = Math.max(maxLastColumnId,
base.metadata.lastColumnId());
+ if (base.metadata.nextRowId() != null) {
+ maxNextRowId = Math.max(maxNextRowId,
base.metadata.nextRowId());
+ }
+ replayPoint(ReplayPoint.BASE_SELECTED, version);
+ return base;
+ }
+ LOG.info(
+ "Iceberg metadata v{} of table {} is not a base for
snapshot {}: {}",
+ version,
+ table.fullName(),
+ snapshotId,
+ problem);
+ replayPoint(ReplayPoint.CANDIDATE_REJECTED, version);
+ }
+ return null;
+ }
+
+ private ReplayBase checkBase(long version) throws IOException {
+ if (snapshotId - version > maxSnapshots) {
+ throw new ReplayAbort(
+ "it is "
+ + (snapshotId - version)
+ + " snapshots behind, more than "
+ + IcebergOptions.REPLAY_MAX_SNAPSHOTS.key());
+ }
+ Snapshot paimonSnapshot = readSnapshot(version);
+ if (paimonSnapshot == null) {
+ throw new ReplayAbort("there is no Paimon snapshot " +
version);
+ }
+ IcebergMetadata metadata =
readUsableMetadata(pathFactory.toMetadataPath(version));
+ if (metadata == null) {
+ throw new ReplayAbort("it is missing or malformed");
+ }
+ if (metadata.formatVersion() != formatVersion) {
+ throw new ReplayAbort("its format version is " +
metadata.formatVersion());
+ }
+ IcebergSnapshot current;
+ try {
+ current = metadata.currentSnapshot();
+ } catch (RuntimeException e) {
+ throw new ReplayAbort("its current snapshot is missing", e);
+ }
+ if (current.snapshotId() != version) {
+ throw new ReplayAbort("its current snapshot is " +
current.snapshotId());
+ }
+ String identity =
+ current.summary() == null
+ ? null
+ :
current.summary().get(SNAPSHOT_SUMMARY_PAIMON_COMMIT_IDENTITY);
+ if (identity == null ||
!identity.equals(commitIdentity(paimonSnapshot))) {
+ throw new ReplayAbort("it was not built from Paimon snapshot "
+ version);
+ }
+ return new ReplayBase(version, metadata);
+ }
+
+ //
---------------------------------------------------------------------------------
+ // Pre-flight and acquisition of the history
+ //
---------------------------------------------------------------------------------
+
+ /** The Paimon snapshots after the base up to N, and the check that
they are replayable. */
+ private List<Snapshot> readSteps(ReplayBase base) {
+ checkDuration();
+ ManifestList deltaLists =
table.store().manifestListFactory().create();
+ List<Snapshot> steps = new ArrayList<>();
+ long entries = 0;
+ for (long id = base.version + 1; id <= snapshotId; id++) {
+ Snapshot step = id == snapshotId ? snapshot : readSnapshot(id);
+ if (step == null) {
+ throw new ReplayAbort("Paimon snapshot " + id + " does not
exist");
+ }
+ try {
+ for (ManifestFileMeta meta :
deltaLists.readDeltaManifests(step)) {
+ entries += meta.numAddedFiles() +
meta.numDeletedFiles();
+ }
+ } catch (RuntimeException e) {
+ if (isMissingFile(e)) {
+ throw new ReplayAbort(
+ "the delta of Paimon snapshot " + id + "
cannot be read", e);
+ }
+ throw e;
+ }
+ if (entries > maxDeltaEntries) {
+ throw new ReplayAbort(
+ "the snapshots after v"
+ + base.version
+ + " change more than "
+ + maxDeltaEntries
+ + " files");
+ }
+ steps.add(step);
+ }
+ return steps;
+ }
+
+ /** Index of the first snapshot of the base that retention keeps for
snapshot N. */
+ private int firstRetained(IcebergMetadata base, List<Snapshot> steps) {
+ List<IcebergSnapshot> baseSnapshots = base.snapshots();
+ int total = baseSnapshots.size() + steps.size();
+ for (int i = 0; i + 1 < total; i++) {
+ boolean expire;
+ if (i < baseSnapshots.size()) {
+ expire = shouldExpire(baseSnapshots.get(i), snapshotId);
+ } else {
+ Snapshot step = steps.get(i - baseSnapshots.size());
+ expire = shouldExpire(step.id(), step.timeMillis(),
snapshotId);
+ }
+ if (!expire) {
+ return Math.min(i, baseSnapshots.size());
+ }
+ }
+ // like the build helper, nothing is dropped when every older
snapshot expires
+ return 0;
+ }
+
+ /**
+ * Copies the manifests and lists the output keeps, and those of the
working base, so the
+ * replay owns everything it reads and publishes.
+ */
+ private Acquired acquire(ReplayBase base, List<Snapshot> steps) throws
IOException {
+ List<IcebergSnapshot> baseSnapshots = base.metadata.snapshots();
+ int first = firstRetained(base.metadata, steps);
+ List<IcebergSnapshot> retained =
+ new ArrayList<>(baseSnapshots.subList(first,
baseSnapshots.size()));
+ IcebergSnapshot working = base.metadata.currentSnapshot();
+ List<IcebergSnapshot> toCopy = new ArrayList<>(retained);
+ boolean workingRetained = first < baseSnapshots.size();
+ if (!workingRetained) {
+ // the working base is always acquired, even when the output
drops its own entry
+ toCopy.add(working);
+ }
+
+ Map<String, Long> unique = new LinkedHashMap<>();
+ for (IcebergSnapshot s : toCopy) {
+ for (IcebergManifestFileMeta meta : readBaseList(s)) {
+ unique.putIfAbsent(meta.manifestPath(),
meta.manifestLength());
+ }
+ }
+ long bytes = 0;
+ for (long length : unique.values()) {
+ bytes += length;
+ }
+ // every manifest is read twice, to copy it and to validate the
copy
+ charge(2 * bytes);
+ replayPoint(ReplayPoint.LISTS_READ, base.version);
+
+ Map<String, String> copied = new HashMap<>();
+ Map<Long, String> originalLists = new HashMap<>();
+ List<IcebergSnapshot> retainedCopies = new ArrayList<>();
+ IcebergSnapshot workingCopy = null;
+ for (IcebergSnapshot s : toCopy) {
+ checkDuration();
+ List<IcebergManifestFileMeta> owned = new ArrayList<>();
+ for (IcebergManifestFileMeta meta : readBaseList(s)) {
+ String target = copied.get(meta.manifestPath());
+ if (target == null) {
+ checkDuration();
+ target = copyManifest(meta);
+ copied.put(meta.manifestPath(), target);
+ }
+ owned.add(meta.withManifestPath(target));
Review Comment:
Fixed. A manifest that several lists name is decoded once, and every list
entry that names it is now compared with what the manifest holds: manifest
length, partition spec, sequence numbers, and the added, existing and deleted
file and row counts. A base whose entry disagrees is rejected before anything
is published, and the replay tries the next older base or rebuilds. Your v1/v2
case (m1 listed with `addedFilesCount=0` in v2 only) is now a test, and it
rejects that base. The entry fields are also held to invariants of their own,
such as spec id 0, sequence number equal to the added snapshot, and no sequence
number newer than the list's snapshot.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]