mvanhorn commented on code in PR #11102:
URL: https://github.com/apache/paimon/pull/11102#discussion_r4591066516


##########
paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java:
##########
@@ -384,6 +390,550 @@ public void testFilterAllCommits() throws Exception {
         }
     }
 
+    @ParameterizedTest
+    @ValueSource(booleans = {false, true})
+    public void testFilterHalfCommittedIdentifier(boolean overwrite) throws 
Exception {
+        TestFileStore store =
+                createStore(
+                        false,
+                        
Collections.singletonMap(CoreOptions.OVERWRITE_UPGRADE.key(), "false"));
+        KeyValue row = gen.nextInsert("20211110", 8, 1L, null, "replacement");
+        List<ManifestCommittable> prepared = new ArrayList<>();
+        for (int i = 0; i < 2; i++) {
+            store.commitDataImpl(
+                    Collections.singletonList(row),
+                    gen::getPartition,
+                    value -> 0,
+                    false,
+                    2L,
+                    123L,
+                    Collections.emptyList(),
+                    (commit, committable) -> prepared.add(committable));
+        }
+        CommitMessageImpl append = (CommitMessageImpl) 
prepared.get(0).fileCommittables().get(0);
+        CommitMessageImpl replacement =
+                (CommitMessageImpl) prepared.get(1).fileCommittables().get(0);
+        IndexPathFactory indexFactory = 
store.pathFactory().indexFileFactory(append.partition(), 0);
+        IndexFileMeta oldIndex = new IndexFileMeta(HASH_INDEX, "old-index", 0, 
0, null, null, null);
+        IndexFileMeta newIndex = new IndexFileMeta(HASH_INDEX, "new-index", 0, 
0, null, null, null);
+        store.fileIO().newOutputStream(indexFactory.toPath(newIndex), 
false).close();
+        append.newFilesIncrement().newIndexFiles().add(oldIndex);
+        CompactIncrement compact =
+                new CompactIncrement(
+                        append.newFilesIncrement().newFiles(),
+                        replacement.newFilesIncrement().newFiles(),
+                        Collections.emptyList(),
+                        Collections.singletonList(newIndex),
+                        Collections.singletonList(oldIndex));
+        CommitMessageImpl message =
+                new CommitMessageImpl(
+                        append.partition(),
+                        append.bucket(),
+                        append.totalBuckets(),
+                        append.newFilesIncrement(),
+                        compact,
+                        1L);
+        ManifestCommittable pending =
+                new ManifestCommittable(2, 123L, 
Collections.singletonList(message));
+        pending.addProperty("recovery-test", "preserved");
+        ManifestCommittable next = new ManifestCommittable(3);
+        CommitCallback callback = mock(CommitCallback.class);
+        try (FileStoreCommitImpl commit =
+                newCommitWithSnapshotCommit(
+                        store,
+                        "writer",
+                        new RenamingSnapshotCommit(store.snapshotManager(), 
Lock.empty()),
+                        store.options(),
+                        false,
+                        Collections.singletonList(callback))) {
+            commit.ignoreEmptyCommit(false);
+            
assertThat(commit.filterCommitted(Collections.singletonList(pending)))
+                    .containsExactly(pending);
+            if (overwrite) {
+                commit.overwritePartition(Collections.emptyMap(), 
prepared.get(0));
+            } else {
+                commit.commit(prepared.get(0), false);
+            }
+            assertThat(store.snapshotManager().latestSnapshot().commitKind())
+                    .isEqualTo(
+                            overwrite ? Snapshot.CommitKind.OVERWRITE : 
Snapshot.CommitKind.APPEND);
+            
assertThat(commit.filterCommitted(Collections.singletonList(prepared.get(0))))
+                    .isEmpty();
+            // A newer snapshot from another user must not hide the 
interrupted first phase.
+            try (FileStoreCommit other = store.newCommit("other", null)) {
+                other.ignoreEmptyCommit(false);
+                other.commit(new ManifestCommittable(100), false);
+            }
+            List<ManifestCommittable> filtered =
+                    commit.filterCommitted(
+                            Arrays.asList(
+                                    new ManifestCommittable(1, null, 
pending.fileCommittables()),
+                                    pending,
+                                    next));
+            assertThat(filtered).hasSize(2);
+            assertThat(filtered.get(1)).isSameAs(next);
+            ManifestCommittable recovered = filtered.get(0);
+            assertThat(recovered).isNotSameAs(pending);
+            assertThat(recovered.identifier()).isEqualTo(2);
+            assertThat(recovered.watermark()).isEqualTo(123L);
+            assertThat(recovered.properties())
+                    .isEqualTo(pending.properties())
+                    .isNotSameAs(pending.properties());
+            assertThat(recovered.fileCommittables()).hasSize(1);
+            CommitMessageImpl recoveredMessage =
+                    (CommitMessageImpl) recovered.fileCommittables().get(0);
+            
assertThat(recoveredMessage.partition()).isEqualTo(message.partition());
+            assertThat(recoveredMessage.bucket()).isEqualTo(message.bucket());
+            
assertThat(recoveredMessage.totalBuckets()).isEqualTo(message.totalBuckets());
+            assertThat(recoveredMessage.checkFromSnapshot()).isEqualTo(1L);
+            
assertThat(recoveredMessage.newFilesIncrement().isEmpty()).isTrue();
+            assertThat(recoveredMessage.compactIncrement()).isSameAs(compact);
+            
assertThat(message.newFilesIncrement()).isSameAs(append.newFilesIncrement());
+            verify(callback).retry(same(pending));
+            assertThat(
+                            overwrite
+                                    ? 
commit.overwritePartition(Collections.emptyMap(), recovered)
+                                    : commit.commit(recovered, false))
+                    .isEqualTo(1);
+            Snapshot snapshot = store.snapshotManager().latestSnapshot();
+            assertThat(snapshot.id()).isEqualTo(3L);
+            
assertThat(snapshot.commitKind()).isEqualTo(Snapshot.CommitKind.COMPACT);
+            verify(callback)
+                    .call(
+                            argThat(
+                                    context ->
+                                            context.snapshot.commitKind()
+                                                    == 
Snapshot.CommitKind.COMPACT));
+            assertThat(store.toKvMap(store.readKvsFromSnapshot(snapshot.id())))
+                    .isEqualTo(store.toKvMap(Collections.singletonList(row)));
+            assertThat(
+                            store.newIndexFileHandler()
+                                    .scanEntries(snapshot, HASH_INDEX, 
append.partition()))
+                    .extracting(IndexManifestEntry::indexFile)
+                    .containsExactly(newIndex);
+            assertThat(commit.filterCommitted(Arrays.asList(pending, 
next))).containsExactly(next);
+
+            // The recovery marker must not change ordinary empty-commit 
behavior.
+            assertThat(commit.commit(next, false)).isEqualTo(1);
+            assertThat(store.snapshotManager().latestSnapshot().commitKind())
+                    .isEqualTo(Snapshot.CommitKind.APPEND);
+            commit.commitStatistics(new Statistics(4, 0, 1L, 1L), 4);
+            assertThat(store.snapshotManager().latestSnapshot().commitKind())
+                    .isEqualTo(Snapshot.CommitKind.ANALYZE);
+            assertThat(
+                            commit.filterCommitted(
+                                    Collections.singletonList(
+                                            new ManifestCommittable(
+                                                    4, null, 
pending.fileCommittables()))))
+                    .isEmpty();
+        }
+    }
+
+    @Test
+    public void 
testFilterHalfCommittedIdentifierWithConcurrentlyCompactedInput() throws 
Exception {
+        TestFileStore store = createStore(false);
+        KeyValue row = gen.nextInsert("20211110", 8, 1L, null, "replacement");
+        List<ManifestCommittable> prepared = new ArrayList<>();
+        for (int i = 0; i < 3; i++) {
+            store.commitDataImpl(
+                    Collections.singletonList(row),
+                    gen::getPartition,
+                    value -> 0,
+                    false,
+                    2L,
+                    123L,
+                    Collections.emptyList(),
+                    (commit, committable) -> prepared.add(committable));
+        }
+        CommitMessageImpl append = (CommitMessageImpl) 
prepared.get(0).fileCommittables().get(0);
+        CommitMessageImpl ours = (CommitMessageImpl) 
prepared.get(1).fileCommittables().get(0);
+        CommitMessageImpl theirs = (CommitMessageImpl) 
prepared.get(2).fileCommittables().get(0);
+        ManifestCommittable pending =
+                new ManifestCommittable(
+                        2,
+                        123L,
+                        Collections.singletonList(
+                                new CommitMessageImpl(
+                                        append.partition(),
+                                        append.bucket(),
+                                        append.totalBuckets(),
+                                        append.newFilesIncrement(),
+                                        new CompactIncrement(
+                                                
append.newFilesIncrement().newFiles(),
+                                                
ours.newFilesIncrement().newFiles(),
+                                                Collections.emptyList()))));
+        try (FileStoreCommit commit = store.newCommit("writer", null)) {
+            // The first phase of identifier 2 publishes the append files.
+            commit.commit(prepared.get(0), false);
+            
assertThat(commit.filterCommitted(Collections.singletonList(pending))).hasSize(1);
+
+            // Another job compacts the same input away before the writer 
restarts.
+            try (FileStoreCommit other = store.newCommit("other", null)) {
+                other.commit(
+                        new ManifestCommittable(
+                                1,
+                                null,
+                                Collections.singletonList(
+                                        new CommitMessageImpl(
+                                                append.partition(),
+                                                append.bucket(),
+                                                append.totalBuckets(),
+                                                DataIncrement.emptyIncrement(),
+                                                new CompactIncrement(
+                                                        
append.newFilesIncrement().newFiles(),
+                                                        
theirs.newFilesIncrement().newFiles(),
+                                                        
Collections.emptyList())))),
+                        false);
+            }
+            Snapshot compacted = store.snapshotManager().latestSnapshot();
+            
assertThat(compacted.commitKind()).isEqualTo(Snapshot.CommitKind.COMPACT);
+
+            // The stale compaction cannot be replayed, so recovery must 
discard it instead of
+            // rebuilding a committable that fails conflict detection on every 
restart.
+            ManifestCommittable next = new ManifestCommittable(3);
+            assertThat(commit.filterCommitted(Arrays.asList(pending, 
next))).containsExactly(next);
+            
assertThat(store.snapshotManager().latestSnapshotId()).isEqualTo(compacted.id());
+            
assertThat(store.toKvMap(store.readKvsFromSnapshot(compacted.id())))
+                    .isEqualTo(store.toKvMap(Collections.singletonList(row)));
+        }
+    }
+
+    @Test
+    public void 
testFilterHalfCommittedIdentifierWithConcurrentlyUpgradedInput() throws 
Exception {
+        TestFileStore store = createStore(false);
+        KeyValue row = gen.nextInsert("20211110", 8, 1L, null, "replacement");
+        List<ManifestCommittable> prepared = new ArrayList<>();
+        for (int i = 0; i < 2; i++) {
+            store.commitDataImpl(
+                    Collections.singletonList(row),
+                    gen::getPartition,
+                    value -> 0,
+                    false,
+                    2L,
+                    123L,
+                    Collections.emptyList(),
+                    (commit, committable) -> prepared.add(committable));
+        }
+        CommitMessageImpl append = (CommitMessageImpl) 
prepared.get(0).fileCommittables().get(0);
+        CommitMessageImpl ours = (CommitMessageImpl) 
prepared.get(1).fileCommittables().get(0);
+        List<DataFileMeta> inputs = append.newFilesIncrement().newFiles();
+        assertThat(inputs).allMatch(file -> file.level() == 0);
+        ManifestCommittable pending =
+                new ManifestCommittable(
+                        2,
+                        123L,
+                        Collections.singletonList(
+                                new CommitMessageImpl(
+                                        append.partition(),
+                                        append.bucket(),
+                                        append.totalBuckets(),
+                                        append.newFilesIncrement(),
+                                        new CompactIncrement(
+                                                inputs,
+                                                
ours.newFilesIncrement().newFiles(),
+                                                Collections.emptyList()))));
+        try (FileStoreCommit commit = store.newCommit("writer", null)) {
+            // The first phase of identifier 2 publishes the append files.
+            commit.commit(prepared.get(0), false);
+            
assertThat(commit.filterCommitted(Collections.singletonList(pending))).hasSize(1);
+
+            // Another job upgrades the same input in place: same file name, 
new level.
+            List<DataFileMeta> upgraded = new ArrayList<>();
+            for (DataFileMeta file : inputs) {
+                upgraded.add(file.upgrade(1));
+            }
+            try (FileStoreCommit other = store.newCommit("other", null)) {
+                other.commit(
+                        new ManifestCommittable(
+                                1,
+                                null,
+                                Collections.singletonList(
+                                        new CommitMessageImpl(
+                                                append.partition(),
+                                                append.bucket(),
+                                                append.totalBuckets(),
+                                                DataIncrement.emptyIncrement(),
+                                                new CompactIncrement(
+                                                        inputs,
+                                                        upgraded,
+                                                        
Collections.emptyList())))),
+                        false);
+            }
+            Snapshot compacted = store.snapshotManager().latestSnapshot();
+            
assertThat(compacted.commitKind()).isEqualTo(Snapshot.CommitKind.COMPACT);
+
+            // The level 0 deletion no longer matches the live level 1 entry, 
so replaying it
+            // would fail conflict detection. Recovery must discard it.
+            ManifestCommittable next = new ManifestCommittable(3);
+            assertThat(commit.filterCommitted(Arrays.asList(pending, 
next))).containsExactly(next);
+            
assertThat(store.snapshotManager().latestSnapshotId()).isEqualTo(compacted.id());
+            
assertThat(store.toKvMap(store.readKvsFromSnapshot(compacted.id())))
+                    .isEqualTo(store.toKvMap(Collections.singletonList(row)));
+        }
+    }
+
+    @ParameterizedTest
+    @ValueSource(booleans = {false, true})
+    public void 
testRecoverHalfCommittedIdentifierWithConcurrentOverlap(boolean overwrite)

Review Comment:
   Agreed, FileStoreCommitTest is 4,019 lines on 96ad422b and the 
defaultFileLength rule in tools/maven/checkstyle.xml caps it at 4000. The right 
fix is a separate recovery test class that keeps all of these cases, rather 
than trimming them or relaxing the rule. That extraction is not on this head 
yet.
   
   <!-- r:a1880a580f0f.7cdefd0f4d41 -->



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to