mvanhorn commented on code in PR #11102:
URL: https://github.com/apache/paimon/pull/11102#discussion_r4565881833
##########
paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java:
##########
@@ -333,6 +348,120 @@ public List<ManifestCommittable>
filterCommitted(List<ManifestCommittable> commi
}
}
+ @Nullable
+ private ManifestCommittable recoverCompact(ManifestCommittable
committable) {
+ List<CommitMessage> messages = new ArrayList<>();
+ Set<Path> outputs = new HashSet<>();
+ DataFilePathFactories dataFactories = new
DataFilePathFactories(pathFactory);
+ IndexFilePathFactories indexFactories = new
IndexFilePathFactories(pathFactory);
+ for (CommitMessage message : committable.fileCommittables()) {
+ CompactIncrement compact = ((CommitMessageImpl)
message).compactIncrement();
+ if (compact.isEmpty()) {
+ continue;
+ }
+ messages.add(
+ new CommitMessageImpl(
+ message.partition(),
+ message.bucket(),
+ message.totalBuckets(),
+ DataIncrement.emptyIncrement(),
+ compact,
+ message.checkFromSnapshot()));
+ DataFilePathFactory dataFactory =
+ dataFactories.get(message.partition(), message.bucket());
+ compact.compactAfter().forEach(file ->
outputs.addAll(file.collectFiles(dataFactory)));
+ compact.changelogFiles()
+ .forEach(file ->
outputs.addAll(file.collectFiles(dataFactory)));
+ IndexPathFactory indexFactory =
+ indexFactories.get(message.partition(), message.bucket());
+ compact.newIndexFiles().forEach(file ->
outputs.add(indexFactory.toPath(file)));
+ }
+ if (messages.isEmpty()) {
+ return null;
+ }
+ // An expired or orphan-cleaned output cannot be replayed. Skip the
whole identifier,
+ // rather than publishing only some of its buckets. I/O failures must
still fail recovery.
+ for (Path output : outputs) {
+ Path physicalPath =
+
CompactedChangelogPathResolver.resolveCompactedChangelogPath(output);
+ try {
+ if (!fileIO.exists(physicalPath)) {
+ LOG.warn(
+ "Skipping compaction recovery for table {}, user
{}, identifier {}: output {} is missing",
+ tableName,
+ commitUser,
+ committable.identifier(),
+ physicalPath);
+ return null;
+ }
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+ if (!compactInputsStillLive(committable)) {
+ return null;
+ }
+ return new CompactRecoveryCommittable(committable, messages);
+ }
+
+ /**
+ * A concurrent compaction may already have removed some compact-before
files. Replaying such an
+ * increment would fail conflict detection on every restart, so only
recover it when all of its
+ * inputs are still live in the latest snapshot.
+ */
+ private boolean compactInputsStillLive(ManifestCommittable committable) {
+ Set<List<Object>> inputs = new HashSet<>();
+ Set<String> inputNames = new HashSet<>();
+ Set<BinaryRow> partitions = new HashSet<>();
+ for (CommitMessage message : committable.fileCommittables()) {
+ for (DataFileMeta file :
+ ((CommitMessageImpl)
message).compactIncrement().compactBefore()) {
+ inputs.add(Arrays.asList(message.partition(),
message.bucket(), file.fileName()));
Review Comment:
Done in 4820c4ade789c0ae435a4880cc6afc73828db9f8. compactInputsStillLive now
builds the full FileEntry.Identifier for each compact-before file and checks it
against the identifiers of the live ADD entries, so a file that a concurrent
compaction upgraded in place no longer counts as live and the stale recovery is
skipped. Your testFilterHalfCommittedIdentifierWithConcurrentlyUpgradedInput
regression is in FileStoreCommitTest and passes along with the other
testFilterHalfCommitted cases.
<!-- r:97d309942405.f3445640000e -->
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]