This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 5d444c508 [test] add random test case for manifest file full
compaction (#4326)
5d444c508 is described below
commit 5d444c508c5fe35b29c89c6fdebe28f3131c1dcf
Author: LsomeYeah <[email protected]>
AuthorDate: Tue Oct 15 17:22:59 2024 +0800
[test] add random test case for manifest file full compaction (#4326)
---
.../paimon/manifest/ManifestFileMetaTest.java | 105 +++++++++++++++++++++
1 file changed, 105 insertions(+)
diff --git
a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java
b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java
index c3d6cc3be..32b64aa7e 100644
---
a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java
@@ -39,8 +39,10 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Optional;
+import java.util.Set;
import java.util.TreeSet;
import java.util.UUID;
import java.util.concurrent.ThreadLocalRandom;
@@ -446,6 +448,109 @@ public class ManifestFileMetaTest extends
ManifestFileMetaTestBase {
containSameIdentifyEntryFile(fullCompacted, entryIdentifierExpected);
}
+ @RepeatedTest(100)
+ public void testRandomFullCompaction() throws Exception {
+ List<ManifestFileMeta> input = new ArrayList<>();
+ Set<FileEntry.Identifier> manifestEntrySet = new HashSet<>();
+ Set<FileEntry.Identifier> deleteManifestEntrySet = new HashSet<>();
+ int inputSize = ThreadLocalRandom.current().nextInt(100) + 1;
+ int totalEntryNums = 0;
+ for (int i = 0; i < inputSize; i++) {
+ int entryNums = ThreadLocalRandom.current().nextInt(100) + 1;
+ input.add(
+ generateRandomData(
+ entryNums, totalEntryNums, manifestEntrySet,
deleteManifestEntrySet));
+ totalEntryNums += entryNums;
+ }
+ int suggerstSize = ThreadLocalRandom.current().nextInt(3000) + 1;
+ int sizeTrigger = ThreadLocalRandom.current().nextInt(40000) + 1;
+ List<ManifestFileMeta> newMetas = new ArrayList<>();
+ Optional<List<ManifestFileMeta>> fullCompacted =
+ ManifestFileMerger.tryFullCompaction(
+ input,
+ newMetas,
+ manifestFile,
+ suggerstSize,
+ sizeTrigger,
+ getPartitionType(),
+ null);
+
+ // *****verify result*****
+ List<ManifestFileMeta> mustMergedFiles =
+ input.stream()
+ .filter(
+ manifest ->
+ manifest.fileSize() < suggerstSize
+ || manifest.numDeletedFiles()
> 0)
+ .collect(Collectors.toList());
+ long mustMergeSize =
+
mustMergedFiles.stream().map(ManifestFileMeta::fileSize).reduce(0L, Long::sum);
+ // manifest files which were not written after full compaction
+ List<ManifestFileMeta> notMergedFiles =
+ input.stream()
+ .filter(
+ manifest ->
+ manifest.fileSize() >= suggerstSize
+ && manifest.numDeletedFiles()
== 0)
+ .filter(
+ manifest ->
+ manifestFile.read(manifest.fileName(),
manifest.fileSize())
+ .stream()
+ .map(ManifestEntry::identifier)
+
.noneMatch(deleteManifestEntrySet::contains))
+ .collect(Collectors.toList());
+
+ if (mustMergeSize < sizeTrigger) {
+ assertThat(fullCompacted).isEmpty();
+ assertThat(newMetas).isEmpty();
+ } else if (mustMergedFiles.size() <= 1) {
+ assertThat(fullCompacted).isEmpty();
+ assertThat(newMetas).isEmpty();
+ } else {
+
assertThat(fullCompacted.get().size()).isEqualTo(notMergedFiles.size() + 1);
+ assertThat(newMetas).size().isEqualTo(1);
+ }
+ }
+
+ private ManifestFileMeta generateRandomData(
+ int entryNums,
+ int totalEntryNums,
+ Set<FileEntry.Identifier> manifestEntrySet,
+ Set<FileEntry.Identifier> deleteManifestEntrySet) {
+ List<ManifestEntry> entries = new ArrayList<>();
+ for (int i = 0; i < entryNums; i++) {
+ // 70% add, 30% delete
+ boolean isAdd = ThreadLocalRandom.current().nextInt(10) < 7;
+ if (manifestEntrySet.isEmpty() || isAdd) {
+ String fileName = String.format("file-%d", totalEntryNums + i);
+ Integer partition = ThreadLocalRandom.current().nextInt(10);
+ int level = ThreadLocalRandom.current().nextInt(6);
+ List<String> extraFiles =
Lists.newArrayList(String.format("index-%s", fileName));
+ byte[] embeddedIndex = new byte[] {1, 2, 3};
+ entries.add(makeEntry(true, fileName, partition, level,
extraFiles, embeddedIndex));
+ } else {
+ FileEntry.Identifier identifier =
+ manifestEntrySet.stream()
+
.skip(ThreadLocalRandom.current().nextInt(manifestEntrySet.size()))
+ .findFirst()
+ .get();
+ entries.add(
+ makeEntry(
+ false,
+ identifier.fileName,
+ identifier.partition.getInt(0),
+ identifier.level,
+ identifier.extraFiles,
+ new byte[] {1, 2, 3}));
+ manifestEntrySet.remove(identifier);
+ deleteManifestEntrySet.add(identifier);
+ }
+ }
+ manifestEntrySet.addAll(
+
entries.stream().map(ManifestEntry::identifier).collect(Collectors.toSet()));
+ return makeManifest(entries.toArray(new ManifestEntry[0]));
+ }
+
private void createData(
int numLastBits, List<ManifestFileMeta> input,
List<ManifestFileMeta> expected) {
// suggested size 500 and suggested count 3
