mvanhorn commented on code in PR #11102:
URL: https://github.com/apache/paimon/pull/11102#discussion_r4591066516
##########
paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java:
##########
@@ -384,6 +390,550 @@ public void testFilterAllCommits() throws Exception {
}
}
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ public void testFilterHalfCommittedIdentifier(boolean overwrite) throws
Exception {
+ TestFileStore store =
+ createStore(
+ false,
+
Collections.singletonMap(CoreOptions.OVERWRITE_UPGRADE.key(), "false"));
+ KeyValue row = gen.nextInsert("20211110", 8, 1L, null, "replacement");
+ List<ManifestCommittable> prepared = new ArrayList<>();
+ for (int i = 0; i < 2; i++) {
+ store.commitDataImpl(
+ Collections.singletonList(row),
+ gen::getPartition,
+ value -> 0,
+ false,
+ 2L,
+ 123L,
+ Collections.emptyList(),
+ (commit, committable) -> prepared.add(committable));
+ }
+ CommitMessageImpl append = (CommitMessageImpl)
prepared.get(0).fileCommittables().get(0);
+ CommitMessageImpl replacement =
+ (CommitMessageImpl) prepared.get(1).fileCommittables().get(0);
+ IndexPathFactory indexFactory =
store.pathFactory().indexFileFactory(append.partition(), 0);
+ IndexFileMeta oldIndex = new IndexFileMeta(HASH_INDEX, "old-index", 0,
0, null, null, null);
+ IndexFileMeta newIndex = new IndexFileMeta(HASH_INDEX, "new-index", 0,
0, null, null, null);
+ store.fileIO().newOutputStream(indexFactory.toPath(newIndex),
false).close();
+ append.newFilesIncrement().newIndexFiles().add(oldIndex);
+ CompactIncrement compact =
+ new CompactIncrement(
+ append.newFilesIncrement().newFiles(),
+ replacement.newFilesIncrement().newFiles(),
+ Collections.emptyList(),
+ Collections.singletonList(newIndex),
+ Collections.singletonList(oldIndex));
+ CommitMessageImpl message =
+ new CommitMessageImpl(
+ append.partition(),
+ append.bucket(),
+ append.totalBuckets(),
+ append.newFilesIncrement(),
+ compact,
+ 1L);
+ ManifestCommittable pending =
+ new ManifestCommittable(2, 123L,
Collections.singletonList(message));
+ pending.addProperty("recovery-test", "preserved");
+ ManifestCommittable next = new ManifestCommittable(3);
+ CommitCallback callback = mock(CommitCallback.class);
+ try (FileStoreCommitImpl commit =
+ newCommitWithSnapshotCommit(
+ store,
+ "writer",
+ new RenamingSnapshotCommit(store.snapshotManager(),
Lock.empty()),
+ store.options(),
+ false,
+ Collections.singletonList(callback))) {
+ commit.ignoreEmptyCommit(false);
+
assertThat(commit.filterCommitted(Collections.singletonList(pending)))
+ .containsExactly(pending);
+ if (overwrite) {
+ commit.overwritePartition(Collections.emptyMap(),
prepared.get(0));
+ } else {
+ commit.commit(prepared.get(0), false);
+ }
+ assertThat(store.snapshotManager().latestSnapshot().commitKind())
+ .isEqualTo(
+ overwrite ? Snapshot.CommitKind.OVERWRITE :
Snapshot.CommitKind.APPEND);
+
assertThat(commit.filterCommitted(Collections.singletonList(prepared.get(0))))
+ .isEmpty();
+ // A newer snapshot from another user must not hide the
interrupted first phase.
+ try (FileStoreCommit other = store.newCommit("other", null)) {
+ other.ignoreEmptyCommit(false);
+ other.commit(new ManifestCommittable(100), false);
+ }
+ List<ManifestCommittable> filtered =
+ commit.filterCommitted(
+ Arrays.asList(
+ new ManifestCommittable(1, null,
pending.fileCommittables()),
+ pending,
+ next));
+ assertThat(filtered).hasSize(2);
+ assertThat(filtered.get(1)).isSameAs(next);
+ ManifestCommittable recovered = filtered.get(0);
+ assertThat(recovered).isNotSameAs(pending);
+ assertThat(recovered.identifier()).isEqualTo(2);
+ assertThat(recovered.watermark()).isEqualTo(123L);
+ assertThat(recovered.properties())
+ .isEqualTo(pending.properties())
+ .isNotSameAs(pending.properties());
+ assertThat(recovered.fileCommittables()).hasSize(1);
+ CommitMessageImpl recoveredMessage =
+ (CommitMessageImpl) recovered.fileCommittables().get(0);
+
assertThat(recoveredMessage.partition()).isEqualTo(message.partition());
+ assertThat(recoveredMessage.bucket()).isEqualTo(message.bucket());
+
assertThat(recoveredMessage.totalBuckets()).isEqualTo(message.totalBuckets());
+ assertThat(recoveredMessage.checkFromSnapshot()).isEqualTo(1L);
+
assertThat(recoveredMessage.newFilesIncrement().isEmpty()).isTrue();
+ assertThat(recoveredMessage.compactIncrement()).isSameAs(compact);
+
assertThat(message.newFilesIncrement()).isSameAs(append.newFilesIncrement());
+ verify(callback).retry(same(pending));
+ assertThat(
+ overwrite
+ ?
commit.overwritePartition(Collections.emptyMap(), recovered)
+ : commit.commit(recovered, false))
+ .isEqualTo(1);
+ Snapshot snapshot = store.snapshotManager().latestSnapshot();
+ assertThat(snapshot.id()).isEqualTo(3L);
+
assertThat(snapshot.commitKind()).isEqualTo(Snapshot.CommitKind.COMPACT);
+ verify(callback)
+ .call(
+ argThat(
+ context ->
+ context.snapshot.commitKind()
+ ==
Snapshot.CommitKind.COMPACT));
+ assertThat(store.toKvMap(store.readKvsFromSnapshot(snapshot.id())))
+ .isEqualTo(store.toKvMap(Collections.singletonList(row)));
+ assertThat(
+ store.newIndexFileHandler()
+ .scanEntries(snapshot, HASH_INDEX,
append.partition()))
+ .extracting(IndexManifestEntry::indexFile)
+ .containsExactly(newIndex);
+ assertThat(commit.filterCommitted(Arrays.asList(pending,
next))).containsExactly(next);
+
+ // The recovery marker must not change ordinary empty-commit
behavior.
+ assertThat(commit.commit(next, false)).isEqualTo(1);
+ assertThat(store.snapshotManager().latestSnapshot().commitKind())
+ .isEqualTo(Snapshot.CommitKind.APPEND);
+ commit.commitStatistics(new Statistics(4, 0, 1L, 1L), 4);
+ assertThat(store.snapshotManager().latestSnapshot().commitKind())
+ .isEqualTo(Snapshot.CommitKind.ANALYZE);
+ assertThat(
+ commit.filterCommitted(
+ Collections.singletonList(
+ new ManifestCommittable(
+ 4, null,
pending.fileCommittables()))))
+ .isEmpty();
+ }
+ }
+
+ @Test
+ public void
testFilterHalfCommittedIdentifierWithConcurrentlyCompactedInput() throws
Exception {
+ TestFileStore store = createStore(false);
+ KeyValue row = gen.nextInsert("20211110", 8, 1L, null, "replacement");
+ List<ManifestCommittable> prepared = new ArrayList<>();
+ for (int i = 0; i < 3; i++) {
+ store.commitDataImpl(
+ Collections.singletonList(row),
+ gen::getPartition,
+ value -> 0,
+ false,
+ 2L,
+ 123L,
+ Collections.emptyList(),
+ (commit, committable) -> prepared.add(committable));
+ }
+ CommitMessageImpl append = (CommitMessageImpl)
prepared.get(0).fileCommittables().get(0);
+ CommitMessageImpl ours = (CommitMessageImpl)
prepared.get(1).fileCommittables().get(0);
+ CommitMessageImpl theirs = (CommitMessageImpl)
prepared.get(2).fileCommittables().get(0);
+ ManifestCommittable pending =
+ new ManifestCommittable(
+ 2,
+ 123L,
+ Collections.singletonList(
+ new CommitMessageImpl(
+ append.partition(),
+ append.bucket(),
+ append.totalBuckets(),
+ append.newFilesIncrement(),
+ new CompactIncrement(
+
append.newFilesIncrement().newFiles(),
+
ours.newFilesIncrement().newFiles(),
+ Collections.emptyList()))));
+ try (FileStoreCommit commit = store.newCommit("writer", null)) {
+ // The first phase of identifier 2 publishes the append files.
+ commit.commit(prepared.get(0), false);
+
assertThat(commit.filterCommitted(Collections.singletonList(pending))).hasSize(1);
+
+ // Another job compacts the same input away before the writer
restarts.
+ try (FileStoreCommit other = store.newCommit("other", null)) {
+ other.commit(
+ new ManifestCommittable(
+ 1,
+ null,
+ Collections.singletonList(
+ new CommitMessageImpl(
+ append.partition(),
+ append.bucket(),
+ append.totalBuckets(),
+ DataIncrement.emptyIncrement(),
+ new CompactIncrement(
+
append.newFilesIncrement().newFiles(),
+
theirs.newFilesIncrement().newFiles(),
+
Collections.emptyList())))),
+ false);
+ }
+ Snapshot compacted = store.snapshotManager().latestSnapshot();
+
assertThat(compacted.commitKind()).isEqualTo(Snapshot.CommitKind.COMPACT);
+
+ // The stale compaction cannot be replayed, so recovery must
discard it instead of
+ // rebuilding a committable that fails conflict detection on every
restart.
+ ManifestCommittable next = new ManifestCommittable(3);
+ assertThat(commit.filterCommitted(Arrays.asList(pending,
next))).containsExactly(next);
+
assertThat(store.snapshotManager().latestSnapshotId()).isEqualTo(compacted.id());
+
assertThat(store.toKvMap(store.readKvsFromSnapshot(compacted.id())))
+ .isEqualTo(store.toKvMap(Collections.singletonList(row)));
+ }
+ }
+
+ @Test
+ public void
testFilterHalfCommittedIdentifierWithConcurrentlyUpgradedInput() throws
Exception {
+ TestFileStore store = createStore(false);
+ KeyValue row = gen.nextInsert("20211110", 8, 1L, null, "replacement");
+ List<ManifestCommittable> prepared = new ArrayList<>();
+ for (int i = 0; i < 2; i++) {
+ store.commitDataImpl(
+ Collections.singletonList(row),
+ gen::getPartition,
+ value -> 0,
+ false,
+ 2L,
+ 123L,
+ Collections.emptyList(),
+ (commit, committable) -> prepared.add(committable));
+ }
+ CommitMessageImpl append = (CommitMessageImpl)
prepared.get(0).fileCommittables().get(0);
+ CommitMessageImpl ours = (CommitMessageImpl)
prepared.get(1).fileCommittables().get(0);
+ List<DataFileMeta> inputs = append.newFilesIncrement().newFiles();
+ assertThat(inputs).allMatch(file -> file.level() == 0);
+ ManifestCommittable pending =
+ new ManifestCommittable(
+ 2,
+ 123L,
+ Collections.singletonList(
+ new CommitMessageImpl(
+ append.partition(),
+ append.bucket(),
+ append.totalBuckets(),
+ append.newFilesIncrement(),
+ new CompactIncrement(
+ inputs,
+
ours.newFilesIncrement().newFiles(),
+ Collections.emptyList()))));
+ try (FileStoreCommit commit = store.newCommit("writer", null)) {
+ // The first phase of identifier 2 publishes the append files.
+ commit.commit(prepared.get(0), false);
+
assertThat(commit.filterCommitted(Collections.singletonList(pending))).hasSize(1);
+
+ // Another job upgrades the same input in place: same file name,
new level.
+ List<DataFileMeta> upgraded = new ArrayList<>();
+ for (DataFileMeta file : inputs) {
+ upgraded.add(file.upgrade(1));
+ }
+ try (FileStoreCommit other = store.newCommit("other", null)) {
+ other.commit(
+ new ManifestCommittable(
+ 1,
+ null,
+ Collections.singletonList(
+ new CommitMessageImpl(
+ append.partition(),
+ append.bucket(),
+ append.totalBuckets(),
+ DataIncrement.emptyIncrement(),
+ new CompactIncrement(
+ inputs,
+ upgraded,
+
Collections.emptyList())))),
+ false);
+ }
+ Snapshot compacted = store.snapshotManager().latestSnapshot();
+
assertThat(compacted.commitKind()).isEqualTo(Snapshot.CommitKind.COMPACT);
+
+ // The level 0 deletion no longer matches the live level 1 entry,
so replaying it
+ // would fail conflict detection. Recovery must discard it.
+ ManifestCommittable next = new ManifestCommittable(3);
+ assertThat(commit.filterCommitted(Arrays.asList(pending,
next))).containsExactly(next);
+
assertThat(store.snapshotManager().latestSnapshotId()).isEqualTo(compacted.id());
+
assertThat(store.toKvMap(store.readKvsFromSnapshot(compacted.id())))
+ .isEqualTo(store.toKvMap(Collections.singletonList(row)));
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ public void
testRecoverHalfCommittedIdentifierWithConcurrentOverlap(boolean overwrite)
Review Comment:
Agreed, FileStoreCommitTest is 4,019 lines on 96ad422b and the
defaultFileLength rule in tools/maven/checkstyle.xml caps it at 4000. The right
fix is a separate recovery test class that keeps all of these cases, rather
than trimming them or relaxing the rule. That extraction is not on this head
yet.
<!-- r:a1880a580f0f.7cdefd0f4d41 -->
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]