mvanhorn commented on code in PR #11102:
URL: https://github.com/apache/paimon/pull/11102#discussion_r4590186302
##########
paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java:
##########
@@ -333,6 +350,223 @@ public List<ManifestCommittable>
filterCommitted(List<ManifestCommittable> commi
}
}
+ @Nullable
+ private ManifestCommittable recoverCompact(ManifestCommittable
committable) {
+ List<CommitMessage> messages = new ArrayList<>();
+ Set<Path> outputs = new HashSet<>();
+ DataFilePathFactories dataFactories = new
DataFilePathFactories(pathFactory);
+ IndexFilePathFactories indexFactories = new
IndexFilePathFactories(pathFactory);
+ for (CommitMessage message : committable.fileCommittables()) {
+ CompactIncrement compact = ((CommitMessageImpl)
message).compactIncrement();
+ if (compact.isEmpty()) {
+ continue;
+ }
+ messages.add(
+ new CommitMessageImpl(
+ message.partition(),
+ message.bucket(),
+ message.totalBuckets(),
+ DataIncrement.emptyIncrement(),
+ compact,
+ message.checkFromSnapshot()));
+ DataFilePathFactory dataFactory =
+ dataFactories.get(message.partition(), message.bucket());
+ compact.compactAfter().forEach(file ->
outputs.addAll(file.collectFiles(dataFactory)));
+ compact.changelogFiles()
+ .forEach(file ->
outputs.addAll(file.collectFiles(dataFactory)));
+ IndexPathFactory indexFactory =
+ indexFactories.get(message.partition(), message.bucket());
+ compact.newIndexFiles().forEach(file ->
outputs.add(indexFactory.toPath(file)));
+ }
+ if (messages.isEmpty()) {
+ return null;
+ }
+ // An expired or orphan-cleaned output cannot be replayed. Skip the
whole identifier,
+ // rather than publishing only some of its buckets. I/O failures must
still fail recovery.
+ for (Path output : outputs) {
+ Path physicalPath =
+
CompactedChangelogPathResolver.resolveCompactedChangelogPath(output);
+ try {
+ if (!fileIO.exists(physicalPath)) {
+ LOG.warn(
+ "Skipping compaction recovery for table {}, user
{}, identifier {}: output {} is missing",
+ tableName,
+ commitUser,
+ committable.identifier(),
+ physicalPath);
+ return null;
+ }
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+ if (!compactInputsStillLive(committable) ||
!compactIndexDeletionsStillLive(committable)) {
+ return null;
+ }
+ return new CompactRecoveryCommittable(committable, messages);
+ }
+
+ /**
+ * A concurrent compaction may already have removed or upgraded some
compact-before files.
+ * Replaying such an increment would fail conflict detection on every
restart, so only recover
+ * it when all of its inputs are still live in the latest snapshot with
the same {@link
+ * FileEntry.Identifier}. Conflict detection compares the full identifier,
so an in-place level
+ * upgrade that keeps the file name still makes the recovery stale.
+ */
+ private boolean compactInputsStillLive(ManifestCommittable committable) {
+ Set<FileEntry.Identifier> inputs = new HashSet<>();
+ Set<String> inputNames = new HashSet<>();
+ Set<BinaryRow> partitions = new HashSet<>();
+ for (CommitMessage message : committable.fileCommittables()) {
+ Integer totalBuckets = message.totalBuckets();
+ for (DataFileMeta file :
+ ((CommitMessageImpl)
message).compactIncrement().compactBefore()) {
+ inputs.add(
+ ManifestEntry.create(
+ FileKind.DELETE,
+ message.partition(),
+ message.bucket(),
+ totalBuckets == null ? 0 :
totalBuckets,
+ file)
+ .identifier());
+ inputNames.add(file.fileName());
+ partitions.add(message.partition());
+ }
+ }
+ if (inputs.isEmpty()) {
+ return true;
+ }
+ Snapshot latest = snapshotManager.latestSnapshot();
+ Set<FileEntry.Identifier> live = new HashSet<>();
+ if (latest != null) {
+ for (SimpleFileEntry entry :
+ scanner.readAllEntriesFromDataFiles(
+ latest, new ArrayList<>(partitions), inputNames)) {
+ if (entry.kind() == FileKind.ADD) {
+ live.add(entry.identifier());
+ }
+ }
+ }
+ for (FileEntry.Identifier input : inputs) {
+ if (!live.contains(input)) {
+ LOG.warn(
+ "Skipping compaction recovery for table {}, user {},
identifier {}: input {} was removed or changed by a concurrent commit",
+ tableName,
+ commitUser,
+ committable.identifier(),
+ input);
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * A compact increment can also delete index files, even without any
compact-before file. A
+ * concurrent commit such as DROP GLOBAL INDEX may already have removed
such an index, and the
+ * index manifest combiners reject deleting a missing or changed entry on
every restart. Only
+ * recover the increment when each deleted index file is still live in the
latest snapshot.
+ */
+ private boolean compactIndexDeletionsStillLive(ManifestCommittable
committable) {
+ List<IndexManifestEntry> deletions = new ArrayList<>();
+ for (CommitMessage message : committable.fileCommittables()) {
+ for (IndexFileMeta file :
+ ((CommitMessageImpl)
message).compactIncrement().deletedIndexFiles()) {
+ deletions.add(
+ new IndexManifestEntry(
+ FileKind.DELETE, message.partition(),
message.bucket(), file));
+ }
+ }
+ if (deletions.isEmpty()) {
+ return true;
+ }
+ Snapshot latest = snapshotManager.latestSnapshot();
+ Map<String, IndexManifestEntry> live = new HashMap<>();
+ if (latest != null && latest.indexManifest() != null) {
+ for (IndexManifestEntry entry :
indexManifestFile.read(latest.indexManifest())) {
+ live.put(entry.indexFile().fileName(), entry);
+ }
+ }
+ for (IndexManifestEntry deletion : deletions) {
+ IndexManifestEntry current =
live.get(deletion.indexFile().fileName());
+ if (current == null
+ || !current.partition().equals(deletion.partition())
+ || current.bucket() != deletion.bucket()
+ || !current.indexFile().equals(deletion.indexFile())) {
+ LOG.warn(
+ "Skipping compaction recovery for table {}, user {},
identifier {}: deleted index file {} was removed or changed by a concurrent
commit",
+ tableName,
+ commitUser,
+ committable.identifier(),
+ deletion.indexFile().fileName());
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Conflict detection rejected a recovered compaction. A concurrent commit
changed the files it
+ * would replace, so replaying it would fail the same way after every
restart.
+ */
+ private static class StaleCompactRecoveryException extends
RuntimeException {
+
+ private StaleCompactRecoveryException(RuntimeException conflict) {
+ super(conflict.getMessage(), conflict);
+ }
+ }
+
+ /**
+ * Commits the compact phase of a committable. A conflict on a recovered
compaction drops that
+ * recovery with a warning: a concurrent commit changed the files it would
replace, so a restart
+ * would replay it and fail the same way. Other failures still propagate.
+ *
+ * @return the number of attempts, or -1 if the recovered compaction was
dropped
+ */
+ private int tryCommitCompact(
+ ManifestCommittable committable,
+ CommitChangesProvider compactProvider,
+ Map<Pair<BinaryRow, Integer>, Long> compactionBaselines) {
+ committingCompactRecovery = committable instanceof
CompactRecoveryCommittable;
+ try {
+ return tryCommit(
+ compactProvider,
+ compactionBaselines,
+ committable.identifier(),
+ committable.watermark(),
+ committable.properties(),
+ CommitKind.COMPACT,
+ false,
+ true,
+ null);
+ } catch (StaleCompactRecoveryException e) {
Review Comment:
Status: this change is not pushed yet, so head 96ad422b still has the gap
you found. The fix is the StaleCompactRecoveryException wrapping described
above plus your DV-enabled two-writer regression test, and this thread stays
open until a commit carrying both lands on this PR.
<!-- r:a1880a580f0f.e08f0c7d0c73 -->
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]