This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 414cd52fe [core] Add benchmark for reader and writer with bloom filter
in sort file store (#4173)
414cd52fe is described below
commit 414cd52feb14653ed232cf24619aef317775e3e2
Author: Fang Yong <[email protected]>
AuthorDate: Mon Oct 14 20:52:05 2024 +0800
[core] Add benchmark for reader and writer with bloom filter in sort file
store (#4173)
---
.../benchmark/lookup/AbstractLookupBenchmark.java | 27 ++++++++++++++++++++--
.../benchmark/lookup/LookupReaderBenchmark.java | 14 ++++++-----
.../benchmark/lookup/LookupWriterBenchmark.java | 20 +++++++++++-----
3 files changed, 47 insertions(+), 14 deletions(-)
diff --git
a/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/lookup/AbstractLookupBenchmark.java
b/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/lookup/AbstractLookupBenchmark.java
index 430a207f5..0c8901bb4 100644
---
a/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/lookup/AbstractLookupBenchmark.java
+++
b/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/lookup/AbstractLookupBenchmark.java
@@ -27,11 +27,13 @@ import org.apache.paimon.lookup.LookupStoreWriter;
import org.apache.paimon.options.MemorySize;
import org.apache.paimon.types.IntType;
import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.BloomFilter;
import org.apache.paimon.utils.Pair;
import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.UUID;
@@ -47,6 +49,15 @@ abstract class AbstractLookupBenchmark {
new RowCompactedSerializer(RowType.of(new IntType()));
private final GenericRow reusedKey = new GenericRow(1);
+ protected static List<List<Object>> getCountBloomList() {
+ List<List<Object>> countBloomList = new ArrayList<>();
+ for (Integer recordCount : RECORD_COUNT_LIST) {
+ countBloomList.add(Arrays.asList(recordCount, false));
+ countBloomList.add(Arrays.asList(recordCount, true));
+ }
+ return countBloomList;
+ }
+
protected byte[][] generateSequenceInputs(int start, int end) {
int count = end - start;
byte[][] result = new byte[count][6];
@@ -74,7 +85,12 @@ abstract class AbstractLookupBenchmark {
}
protected Pair<String, LookupStoreFactory.Context> writeData(
- Path tempDir, CoreOptions options, byte[][] inputs, int
valueLength, boolean sameValue)
+ Path tempDir,
+ CoreOptions options,
+ byte[][] inputs,
+ int valueLength,
+ boolean sameValue,
+ boolean bloomFilterEnabled)
throws IOException {
byte[] value1 = new byte[valueLength];
byte[] value2 = new byte[valueLength];
@@ -87,7 +103,7 @@ abstract class AbstractLookupBenchmark {
keySerializer.createSliceComparator());
File file = new File(tempDir.toFile(), UUID.randomUUID().toString());
- LookupStoreWriter writer = factory.createWriter(file, null);
+ LookupStoreWriter writer = factory.createWriter(file,
createBloomFiler(bloomFilterEnabled));
int i = 0;
for (byte[] input : inputs) {
if (sameValue) {
@@ -104,4 +120,11 @@ abstract class AbstractLookupBenchmark {
LookupStoreFactory.Context context = writer.close();
return Pair.of(file.getAbsolutePath(), context);
}
+
+ private BloomFilter.Builder createBloomFiler(boolean enabled) {
+ if (!enabled) {
+ return null;
+ }
+ return BloomFilter.builder(100, 0.01);
+ }
}
diff --git
a/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/lookup/LookupReaderBenchmark.java
b/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/lookup/LookupReaderBenchmark.java
index 6327d703a..b3f50732d 100644
---
a/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/lookup/LookupReaderBenchmark.java
+++
b/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/lookup/LookupReaderBenchmark.java
@@ -49,15 +49,17 @@ import static org.assertj.core.api.Assertions.assertThat;
public class LookupReaderBenchmark extends AbstractLookupBenchmark {
private static final int QUERY_KEY_COUNT = 10000;
private final int recordCount;
+ private final boolean bloomFilterEnabled;
@TempDir Path tempDir;
- public LookupReaderBenchmark(int recordCount) {
- this.recordCount = recordCount;
+ public LookupReaderBenchmark(List<Object> countBloomList) {
+ this.recordCount = (Integer) countBloomList.get(0);
+ this.bloomFilterEnabled = (Boolean) countBloomList.get(1);
}
- @Parameters(name = "record-count-{0}")
- public static List<Integer> getVarSeg() {
- return RECORD_COUNT_LIST;
+ @Parameters(name = "countBloom-{0}")
+ public static List<List<Object>> getVarSeg() {
+ return getCountBloomList();
}
@TestTemplate
@@ -81,7 +83,7 @@ public class LookupReaderBenchmark extends
AbstractLookupBenchmark {
Collections.singletonMap(
LOOKUP_LOCAL_FILE_TYPE.key(),
fileType.name()));
Pair<String, LookupStoreFactory.Context> pair =
- writeData(tempDir, options, inputs, valueLength,
false);
+ writeData(tempDir, options, inputs, valueLength,
false, bloomFilterEnabled);
benchmark.addCase(
String.format(
"%s-read-%dB-value-%d-num",
diff --git
a/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/lookup/LookupWriterBenchmark.java
b/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/lookup/LookupWriterBenchmark.java
index fe4f1cd24..a590866ef 100644
---
a/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/lookup/LookupWriterBenchmark.java
+++
b/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/lookup/LookupWriterBenchmark.java
@@ -38,15 +38,17 @@ import static
org.apache.paimon.CoreOptions.LOOKUP_LOCAL_FILE_TYPE;
@ExtendWith(ParameterizedTestExtension.class)
public class LookupWriterBenchmark extends AbstractLookupBenchmark {
private final int recordCount;
+ private final boolean bloomFilterEnabled;
@TempDir Path tempDir;
- public LookupWriterBenchmark(int recordCount) {
- this.recordCount = recordCount;
+ public LookupWriterBenchmark(List<Object> countBloomList) {
+ this.recordCount = (Integer) countBloomList.get(0);
+ this.bloomFilterEnabled = (Boolean) countBloomList.get(1);
}
- @Parameters(name = "record-count-{0}")
- public static List<Integer> getVarSeg() {
- return RECORD_COUNT_LIST;
+ @Parameters(name = "countBloom-{0}")
+ public static List<List<Object>> getVarSeg() {
+ return getCountBloomList();
}
@TestTemplate
@@ -78,7 +80,13 @@ public class LookupWriterBenchmark extends
AbstractLookupBenchmark {
5,
() -> {
try {
- writeData(tempDir, options, inputs,
valueLength, sameValue);
+ writeData(
+ tempDir,
+ options,
+ inputs,
+ valueLength,
+ sameValue,
+ bloomFilterEnabled);
} catch (IOException e) {
throw new RuntimeException(e);
}
