JingsongLi commented on code in PR #11102:
URL: https://github.com/apache/paimon/pull/11102#discussion_r4583829635


##########
paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java:
##########
@@ -384,6 +390,550 @@ public void testFilterAllCommits() throws Exception {
         }
     }
 
+    @ParameterizedTest
+    @ValueSource(booleans = {false, true})
+    public void testFilterHalfCommittedIdentifier(boolean overwrite) throws 
Exception {
+        TestFileStore store =
+                createStore(
+                        false,
+                        
Collections.singletonMap(CoreOptions.OVERWRITE_UPGRADE.key(), "false"));
+        KeyValue row = gen.nextInsert("20211110", 8, 1L, null, "replacement");
+        List<ManifestCommittable> prepared = new ArrayList<>();
+        for (int i = 0; i < 2; i++) {
+            store.commitDataImpl(
+                    Collections.singletonList(row),
+                    gen::getPartition,
+                    value -> 0,
+                    false,
+                    2L,
+                    123L,
+                    Collections.emptyList(),
+                    (commit, committable) -> prepared.add(committable));
+        }
+        CommitMessageImpl append = (CommitMessageImpl) 
prepared.get(0).fileCommittables().get(0);
+        CommitMessageImpl replacement =
+                (CommitMessageImpl) prepared.get(1).fileCommittables().get(0);
+        IndexPathFactory indexFactory = 
store.pathFactory().indexFileFactory(append.partition(), 0);
+        IndexFileMeta oldIndex = new IndexFileMeta(HASH_INDEX, "old-index", 0, 
0, null, null, null);
+        IndexFileMeta newIndex = new IndexFileMeta(HASH_INDEX, "new-index", 0, 
0, null, null, null);
+        store.fileIO().newOutputStream(indexFactory.toPath(newIndex), 
false).close();
+        append.newFilesIncrement().newIndexFiles().add(oldIndex);
+        CompactIncrement compact =
+                new CompactIncrement(
+                        append.newFilesIncrement().newFiles(),
+                        replacement.newFilesIncrement().newFiles(),
+                        Collections.emptyList(),
+                        Collections.singletonList(newIndex),
+                        Collections.singletonList(oldIndex));
+        CommitMessageImpl message =
+                new CommitMessageImpl(
+                        append.partition(),
+                        append.bucket(),
+                        append.totalBuckets(),
+                        append.newFilesIncrement(),
+                        compact,
+                        1L);
+        ManifestCommittable pending =
+                new ManifestCommittable(2, 123L, 
Collections.singletonList(message));
+        pending.addProperty("recovery-test", "preserved");
+        ManifestCommittable next = new ManifestCommittable(3);
+        CommitCallback callback = mock(CommitCallback.class);
+        try (FileStoreCommitImpl commit =
+                newCommitWithSnapshotCommit(
+                        store,
+                        "writer",
+                        new RenamingSnapshotCommit(store.snapshotManager(), 
Lock.empty()),
+                        store.options(),
+                        false,
+                        Collections.singletonList(callback))) {
+            commit.ignoreEmptyCommit(false);
+            
assertThat(commit.filterCommitted(Collections.singletonList(pending)))
+                    .containsExactly(pending);
+            if (overwrite) {
+                commit.overwritePartition(Collections.emptyMap(), 
prepared.get(0));
+            } else {
+                commit.commit(prepared.get(0), false);
+            }
+            assertThat(store.snapshotManager().latestSnapshot().commitKind())
+                    .isEqualTo(
+                            overwrite ? Snapshot.CommitKind.OVERWRITE : 
Snapshot.CommitKind.APPEND);
+            
assertThat(commit.filterCommitted(Collections.singletonList(prepared.get(0))))
+                    .isEmpty();
+            // A newer snapshot from another user must not hide the 
interrupted first phase.
+            try (FileStoreCommit other = store.newCommit("other", null)) {
+                other.ignoreEmptyCommit(false);
+                other.commit(new ManifestCommittable(100), false);
+            }
+            List<ManifestCommittable> filtered =
+                    commit.filterCommitted(
+                            Arrays.asList(
+                                    new ManifestCommittable(1, null, 
pending.fileCommittables()),
+                                    pending,
+                                    next));
+            assertThat(filtered).hasSize(2);
+            assertThat(filtered.get(1)).isSameAs(next);
+            ManifestCommittable recovered = filtered.get(0);
+            assertThat(recovered).isNotSameAs(pending);
+            assertThat(recovered.identifier()).isEqualTo(2);
+            assertThat(recovered.watermark()).isEqualTo(123L);
+            assertThat(recovered.properties())
+                    .isEqualTo(pending.properties())
+                    .isNotSameAs(pending.properties());
+            assertThat(recovered.fileCommittables()).hasSize(1);
+            CommitMessageImpl recoveredMessage =
+                    (CommitMessageImpl) recovered.fileCommittables().get(0);
+            
assertThat(recoveredMessage.partition()).isEqualTo(message.partition());
+            assertThat(recoveredMessage.bucket()).isEqualTo(message.bucket());
+            
assertThat(recoveredMessage.totalBuckets()).isEqualTo(message.totalBuckets());
+            assertThat(recoveredMessage.checkFromSnapshot()).isEqualTo(1L);
+            
assertThat(recoveredMessage.newFilesIncrement().isEmpty()).isTrue();
+            assertThat(recoveredMessage.compactIncrement()).isSameAs(compact);
+            
assertThat(message.newFilesIncrement()).isSameAs(append.newFilesIncrement());
+            verify(callback).retry(same(pending));
+            assertThat(
+                            overwrite
+                                    ? 
commit.overwritePartition(Collections.emptyMap(), recovered)
+                                    : commit.commit(recovered, false))
+                    .isEqualTo(1);
+            Snapshot snapshot = store.snapshotManager().latestSnapshot();
+            assertThat(snapshot.id()).isEqualTo(3L);
+            
assertThat(snapshot.commitKind()).isEqualTo(Snapshot.CommitKind.COMPACT);
+            verify(callback)
+                    .call(
+                            argThat(
+                                    context ->
+                                            context.snapshot.commitKind()
+                                                    == 
Snapshot.CommitKind.COMPACT));
+            assertThat(store.toKvMap(store.readKvsFromSnapshot(snapshot.id())))
+                    .isEqualTo(store.toKvMap(Collections.singletonList(row)));
+            assertThat(
+                            store.newIndexFileHandler()
+                                    .scanEntries(snapshot, HASH_INDEX, 
append.partition()))
+                    .extracting(IndexManifestEntry::indexFile)
+                    .containsExactly(newIndex);
+            assertThat(commit.filterCommitted(Arrays.asList(pending, 
next))).containsExactly(next);
+
+            // The recovery marker must not change ordinary empty-commit 
behavior.
+            assertThat(commit.commit(next, false)).isEqualTo(1);
+            assertThat(store.snapshotManager().latestSnapshot().commitKind())
+                    .isEqualTo(Snapshot.CommitKind.APPEND);
+            commit.commitStatistics(new Statistics(4, 0, 1L, 1L), 4);
+            assertThat(store.snapshotManager().latestSnapshot().commitKind())
+                    .isEqualTo(Snapshot.CommitKind.ANALYZE);
+            assertThat(
+                            commit.filterCommitted(
+                                    Collections.singletonList(
+                                            new ManifestCommittable(
+                                                    4, null, 
pending.fileCommittables()))))
+                    .isEmpty();
+        }
+    }
+
+    @Test
+    public void 
testFilterHalfCommittedIdentifierWithConcurrentlyCompactedInput() throws 
Exception {
+        TestFileStore store = createStore(false);
+        KeyValue row = gen.nextInsert("20211110", 8, 1L, null, "replacement");
+        List<ManifestCommittable> prepared = new ArrayList<>();
+        for (int i = 0; i < 3; i++) {
+            store.commitDataImpl(
+                    Collections.singletonList(row),
+                    gen::getPartition,
+                    value -> 0,
+                    false,
+                    2L,
+                    123L,
+                    Collections.emptyList(),
+                    (commit, committable) -> prepared.add(committable));
+        }
+        CommitMessageImpl append = (CommitMessageImpl) 
prepared.get(0).fileCommittables().get(0);
+        CommitMessageImpl ours = (CommitMessageImpl) 
prepared.get(1).fileCommittables().get(0);
+        CommitMessageImpl theirs = (CommitMessageImpl) 
prepared.get(2).fileCommittables().get(0);
+        ManifestCommittable pending =
+                new ManifestCommittable(
+                        2,
+                        123L,
+                        Collections.singletonList(
+                                new CommitMessageImpl(
+                                        append.partition(),
+                                        append.bucket(),
+                                        append.totalBuckets(),
+                                        append.newFilesIncrement(),
+                                        new CompactIncrement(
+                                                
append.newFilesIncrement().newFiles(),
+                                                
ours.newFilesIncrement().newFiles(),
+                                                Collections.emptyList()))));
+        try (FileStoreCommit commit = store.newCommit("writer", null)) {
+            // The first phase of identifier 2 publishes the append files.
+            commit.commit(prepared.get(0), false);
+            
assertThat(commit.filterCommitted(Collections.singletonList(pending))).hasSize(1);
+
+            // Another job compacts the same input away before the writer 
restarts.
+            try (FileStoreCommit other = store.newCommit("other", null)) {
+                other.commit(
+                        new ManifestCommittable(
+                                1,
+                                null,
+                                Collections.singletonList(
+                                        new CommitMessageImpl(
+                                                append.partition(),
+                                                append.bucket(),
+                                                append.totalBuckets(),
+                                                DataIncrement.emptyIncrement(),
+                                                new CompactIncrement(
+                                                        
append.newFilesIncrement().newFiles(),
+                                                        
theirs.newFilesIncrement().newFiles(),
+                                                        
Collections.emptyList())))),
+                        false);
+            }
+            Snapshot compacted = store.snapshotManager().latestSnapshot();
+            
assertThat(compacted.commitKind()).isEqualTo(Snapshot.CommitKind.COMPACT);
+
+            // The stale compaction cannot be replayed, so recovery must 
discard it instead of
+            // rebuilding a committable that fails conflict detection on every 
restart.
+            ManifestCommittable next = new ManifestCommittable(3);
+            assertThat(commit.filterCommitted(Arrays.asList(pending, 
next))).containsExactly(next);
+            
assertThat(store.snapshotManager().latestSnapshotId()).isEqualTo(compacted.id());
+            
assertThat(store.toKvMap(store.readKvsFromSnapshot(compacted.id())))
+                    .isEqualTo(store.toKvMap(Collections.singletonList(row)));
+        }
+    }
+
+    @Test
+    public void 
testFilterHalfCommittedIdentifierWithConcurrentlyUpgradedInput() throws 
Exception {
+        TestFileStore store = createStore(false);
+        KeyValue row = gen.nextInsert("20211110", 8, 1L, null, "replacement");
+        List<ManifestCommittable> prepared = new ArrayList<>();
+        for (int i = 0; i < 2; i++) {
+            store.commitDataImpl(
+                    Collections.singletonList(row),
+                    gen::getPartition,
+                    value -> 0,
+                    false,
+                    2L,
+                    123L,
+                    Collections.emptyList(),
+                    (commit, committable) -> prepared.add(committable));
+        }
+        CommitMessageImpl append = (CommitMessageImpl) 
prepared.get(0).fileCommittables().get(0);
+        CommitMessageImpl ours = (CommitMessageImpl) 
prepared.get(1).fileCommittables().get(0);
+        List<DataFileMeta> inputs = append.newFilesIncrement().newFiles();
+        assertThat(inputs).allMatch(file -> file.level() == 0);
+        ManifestCommittable pending =
+                new ManifestCommittable(
+                        2,
+                        123L,
+                        Collections.singletonList(
+                                new CommitMessageImpl(
+                                        append.partition(),
+                                        append.bucket(),
+                                        append.totalBuckets(),
+                                        append.newFilesIncrement(),
+                                        new CompactIncrement(
+                                                inputs,
+                                                
ours.newFilesIncrement().newFiles(),
+                                                Collections.emptyList()))));
+        try (FileStoreCommit commit = store.newCommit("writer", null)) {
+            // The first phase of identifier 2 publishes the append files.
+            commit.commit(prepared.get(0), false);
+            
assertThat(commit.filterCommitted(Collections.singletonList(pending))).hasSize(1);
+
+            // Another job upgrades the same input in place: same file name, 
new level.
+            List<DataFileMeta> upgraded = new ArrayList<>();
+            for (DataFileMeta file : inputs) {
+                upgraded.add(file.upgrade(1));
+            }
+            try (FileStoreCommit other = store.newCommit("other", null)) {
+                other.commit(
+                        new ManifestCommittable(
+                                1,
+                                null,
+                                Collections.singletonList(
+                                        new CommitMessageImpl(
+                                                append.partition(),
+                                                append.bucket(),
+                                                append.totalBuckets(),
+                                                DataIncrement.emptyIncrement(),
+                                                new CompactIncrement(
+                                                        inputs,
+                                                        upgraded,
+                                                        
Collections.emptyList())))),
+                        false);
+            }
+            Snapshot compacted = store.snapshotManager().latestSnapshot();
+            
assertThat(compacted.commitKind()).isEqualTo(Snapshot.CommitKind.COMPACT);
+
+            // The level 0 deletion no longer matches the live level 1 entry, 
so replaying it
+            // would fail conflict detection. Recovery must discard it.
+            ManifestCommittable next = new ManifestCommittable(3);
+            assertThat(commit.filterCommitted(Arrays.asList(pending, 
next))).containsExactly(next);
+            
assertThat(store.snapshotManager().latestSnapshotId()).isEqualTo(compacted.id());
+            
assertThat(store.toKvMap(store.readKvsFromSnapshot(compacted.id())))
+                    .isEqualTo(store.toKvMap(Collections.singletonList(row)));
+        }
+    }
+
+    @ParameterizedTest
+    @ValueSource(booleans = {false, true})
+    public void 
testRecoverHalfCommittedIdentifierWithConcurrentOverlap(boolean overwrite)

Review Comment:
   [P2] Move recovery regressions out of the over-limit test file
   
   The added regression pushes this file to 4,019 lines; 
tools/maven/checkstyle.xml enforces max=4000 through defaultFileLength. Normal 
mvn -pl paimon-core -am ... test now stops before compilation/tests, and the 
current Core and downstream CI jobs fail for the same reason. 
checkstyle-result.xml contains the explicit file-length error; the Maven 
message 'begin 0, end -1, length 17' is a secondary error from rendering that 
rule id. The previous head had 3,924 lines. Please extract a cohesive recovery 
test class while retaining these cases, so the standard build passes without 
disabling Checkstyle.



##########
paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java:
##########
@@ -333,6 +350,223 @@ public List<ManifestCommittable> 
filterCommitted(List<ManifestCommittable> commi
         }
     }
 
+    @Nullable
+    private ManifestCommittable recoverCompact(ManifestCommittable 
committable) {
+        List<CommitMessage> messages = new ArrayList<>();
+        Set<Path> outputs = new HashSet<>();
+        DataFilePathFactories dataFactories = new 
DataFilePathFactories(pathFactory);
+        IndexFilePathFactories indexFactories = new 
IndexFilePathFactories(pathFactory);
+        for (CommitMessage message : committable.fileCommittables()) {
+            CompactIncrement compact = ((CommitMessageImpl) 
message).compactIncrement();
+            if (compact.isEmpty()) {
+                continue;
+            }
+            messages.add(
+                    new CommitMessageImpl(
+                            message.partition(),
+                            message.bucket(),
+                            message.totalBuckets(),
+                            DataIncrement.emptyIncrement(),
+                            compact,
+                            message.checkFromSnapshot()));
+            DataFilePathFactory dataFactory =
+                    dataFactories.get(message.partition(), message.bucket());
+            compact.compactAfter().forEach(file -> 
outputs.addAll(file.collectFiles(dataFactory)));
+            compact.changelogFiles()
+                    .forEach(file -> 
outputs.addAll(file.collectFiles(dataFactory)));
+            IndexPathFactory indexFactory =
+                    indexFactories.get(message.partition(), message.bucket());
+            compact.newIndexFiles().forEach(file -> 
outputs.add(indexFactory.toPath(file)));
+        }
+        if (messages.isEmpty()) {
+            return null;
+        }
+        // An expired or orphan-cleaned output cannot be replayed. Skip the 
whole identifier,
+        // rather than publishing only some of its buckets. I/O failures must 
still fail recovery.
+        for (Path output : outputs) {
+            Path physicalPath =
+                    
CompactedChangelogPathResolver.resolveCompactedChangelogPath(output);
+            try {
+                if (!fileIO.exists(physicalPath)) {
+                    LOG.warn(
+                            "Skipping compaction recovery for table {}, user 
{}, identifier {}: output {} is missing",
+                            tableName,
+                            commitUser,
+                            committable.identifier(),
+                            physicalPath);
+                    return null;
+                }
+            } catch (IOException e) {
+                throw new UncheckedIOException(e);
+            }
+        }
+        if (!compactInputsStillLive(committable) || 
!compactIndexDeletionsStillLive(committable)) {
+            return null;
+        }
+        return new CompactRecoveryCommittable(committable, messages);
+    }
+
+    /**
+     * A concurrent compaction may already have removed or upgraded some 
compact-before files.
+     * Replaying such an increment would fail conflict detection on every 
restart, so only recover
+     * it when all of its inputs are still live in the latest snapshot with 
the same {@link
+     * FileEntry.Identifier}. Conflict detection compares the full identifier, 
so an in-place level
+     * upgrade that keeps the file name still makes the recovery stale.
+     */
+    private boolean compactInputsStillLive(ManifestCommittable committable) {
+        Set<FileEntry.Identifier> inputs = new HashSet<>();
+        Set<String> inputNames = new HashSet<>();
+        Set<BinaryRow> partitions = new HashSet<>();
+        for (CommitMessage message : committable.fileCommittables()) {
+            Integer totalBuckets = message.totalBuckets();
+            for (DataFileMeta file :
+                    ((CommitMessageImpl) 
message).compactIncrement().compactBefore()) {
+                inputs.add(
+                        ManifestEntry.create(
+                                        FileKind.DELETE,
+                                        message.partition(),
+                                        message.bucket(),
+                                        totalBuckets == null ? 0 : 
totalBuckets,
+                                        file)
+                                .identifier());
+                inputNames.add(file.fileName());
+                partitions.add(message.partition());
+            }
+        }
+        if (inputs.isEmpty()) {
+            return true;
+        }
+        Snapshot latest = snapshotManager.latestSnapshot();
+        Set<FileEntry.Identifier> live = new HashSet<>();
+        if (latest != null) {
+            for (SimpleFileEntry entry :
+                    scanner.readAllEntriesFromDataFiles(
+                            latest, new ArrayList<>(partitions), inputNames)) {
+                if (entry.kind() == FileKind.ADD) {
+                    live.add(entry.identifier());
+                }
+            }
+        }
+        for (FileEntry.Identifier input : inputs) {
+            if (!live.contains(input)) {
+                LOG.warn(
+                        "Skipping compaction recovery for table {}, user {}, 
identifier {}: input {} was removed or changed by a concurrent commit",
+                        tableName,
+                        commitUser,
+                        committable.identifier(),
+                        input);
+                return false;
+            }
+        }
+        return true;
+    }
+
+    /**
+     * A compact increment can also delete index files, even without any 
compact-before file. A
+     * concurrent commit such as DROP GLOBAL INDEX may already have removed 
such an index, and the
+     * index manifest combiners reject deleting a missing or changed entry on 
every restart. Only
+     * recover the increment when each deleted index file is still live in the 
latest snapshot.
+     */
+    private boolean compactIndexDeletionsStillLive(ManifestCommittable 
committable) {
+        List<IndexManifestEntry> deletions = new ArrayList<>();
+        for (CommitMessage message : committable.fileCommittables()) {
+            for (IndexFileMeta file :
+                    ((CommitMessageImpl) 
message).compactIncrement().deletedIndexFiles()) {
+                deletions.add(
+                        new IndexManifestEntry(
+                                FileKind.DELETE, message.partition(), 
message.bucket(), file));
+            }
+        }
+        if (deletions.isEmpty()) {
+            return true;
+        }
+        Snapshot latest = snapshotManager.latestSnapshot();
+        Map<String, IndexManifestEntry> live = new HashMap<>();
+        if (latest != null && latest.indexManifest() != null) {
+            for (IndexManifestEntry entry : 
indexManifestFile.read(latest.indexManifest())) {
+                live.put(entry.indexFile().fileName(), entry);
+            }
+        }
+        for (IndexManifestEntry deletion : deletions) {
+            IndexManifestEntry current = 
live.get(deletion.indexFile().fileName());
+            if (current == null
+                    || !current.partition().equals(deletion.partition())
+                    || current.bucket() != deletion.bucket()
+                    || !current.indexFile().equals(deletion.indexFile())) {
+                LOG.warn(
+                        "Skipping compaction recovery for table {}, user {}, 
identifier {}: deleted index file {} was removed or changed by a concurrent 
commit",
+                        tableName,
+                        commitUser,
+                        committable.identifier(),
+                        deletion.indexFile().fileName());
+                return false;
+            }
+        }
+        return true;
+    }
+
+    /**
+     * Conflict detection rejected a recovered compaction. A concurrent commit 
changed the files it
+     * would replace, so replaying it would fail the same way after every 
restart.
+     */
+    private static class StaleCompactRecoveryException extends 
RuntimeException {
+
+        private StaleCompactRecoveryException(RuntimeException conflict) {
+            super(conflict.getMessage(), conflict);
+        }
+    }
+
+    /**
+     * Commits the compact phase of a committable. A conflict on a recovered 
compaction drops that
+     * recovery with a warning: a concurrent commit changed the files it would 
replace, so a restart
+     * would replay it and fail the same way. Other failures still propagate.
+     *
+     * @return the number of attempts, or -1 if the recovered compaction was 
dropped
+     */
+    private int tryCommitCompact(
+            ManifestCommittable committable,
+            CommitChangesProvider compactProvider,
+            Map<Pair<BinaryRow, Integer>, Long> compactionBaselines) {
+        committingCompactRecovery = committable instanceof 
CompactRecoveryCommittable;
+        try {
+            return tryCommit(
+                    compactProvider,
+                    compactionBaselines,
+                    committable.identifier(),
+                    committable.watermark(),
+                    committable.properties(),
+                    CommitKind.COMPACT,
+                    false,
+                    true,
+                    null);
+        } catch (StaleCompactRecoveryException e) {

Review Comment:
   [P1] Handle stale recovery rejected by the DV bucket-history check
   
   This catch only handles conflicts wrapped later by 
conflictDetection.checkConflicts, but 
SnapshotConflictChecker.checkBucketCompaction runs earlier in 
tryCommitOnceWithoutLock and throws a plain RuntimeException. I reproduced this 
with a real Parquet PK table and deletion-vectors.enabled=true: writers A/B 
prepare distinct keys in one bucket, publish only A's APPEND phase for 
identifier 2, then B publishes its own APPEND and an in-place level upgrade. 
A's original level-0 input and pending upgraded output still exist, so 
filterCommitted returns its recovery. Two fresh A committers both fail with 
'Concurrent compaction conflict ... COMPACT snapshot 3 by user B ... after this 
writer restored its state'. The same public-table probe with DV disabled 
recovers successfully. Thus every restart replays the rejected historical 
baseline rather than progressing; the base code filtered this already-appended 
identifier out. Classify the known bucket-history stale-compaction rejection 
for recove
 ry as well, and discard/recompute that recovery while continuing to propagate 
unrelated I/O or publication failures.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to