(paimon) branch master updated: [core] Update writer/reader benchmark for bloomfilter (#4350)

[lzljs3620320]

This is an automated email from the ASF dual-hosted git repository.

lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git


The following commit(s) were added to refs/heads/master by this push:
     new 414d61745 [core] Update writer/reader benchmark for bloomfilter (#4350)
414d61745 is described below

commit 414d617451016f56045b5c3c88a49adf6d61a0ea
Author: Fang Yong <zjur...@gmail.com>
AuthorDate: Mon Oct 21 23:22:46 2024 +0800

    [core] Update writer/reader benchmark for bloomfilter (#4350)
---
 .../benchmark/lookup/AbstractLookupBenchmark.java  |  2 +-
 .../benchmark/lookup/LookupReaderBenchmark.java    | 31 ++++++++++++++++++----
 2 files changed, 27 insertions(+), 6 deletions(-)

diff --git 
a/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/lookup/AbstractLookupBenchmark.java
 
b/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/lookup/AbstractLookupBenchmark.java
index 0c8901bb4..635d876f7 100644
--- 
a/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/lookup/AbstractLookupBenchmark.java
+++ 
b/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/lookup/AbstractLookupBenchmark.java
@@ -125,6 +125,6 @@ abstract class AbstractLookupBenchmark {
         if (!enabled) {
             return null;
         }
-        return BloomFilter.builder(100, 0.01);
+        return BloomFilter.builder(5000000, 0.01);
     }
 }
diff --git 
a/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/lookup/LookupReaderBenchmark.java
 
b/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/lookup/LookupReaderBenchmark.java
index b3f50732d..9947b54a7 100644
--- 
a/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/lookup/LookupReaderBenchmark.java
+++ 
b/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/lookup/LookupReaderBenchmark.java
@@ -62,14 +62,25 @@ public class LookupReaderBenchmark extends 
AbstractLookupBenchmark {
         return getCountBloomList();
     }
 
+    /** Query data based on some keys that are definitely stored. */
     @TestTemplate
     void testLookupReader() throws IOException {
         readLookupDataBenchmark(
                 generateSequenceInputs(0, recordCount),
-                generateRandomInputs(0, recordCount, QUERY_KEY_COUNT));
+                generateRandomInputs(0, recordCount, QUERY_KEY_COUNT),
+                false);
     }
 
-    private void readLookupDataBenchmark(byte[][] inputs, byte[][] 
randomInputs)
+    /** Query data based on some keys that are definitely not stored. */
+    @TestTemplate
+    void testLookupReaderMiss() throws IOException {
+        readLookupDataBenchmark(
+                generateSequenceInputs(0, recordCount),
+                generateRandomInputs(recordCount + 1, recordCount * 2, 
QUERY_KEY_COUNT),
+                true);
+    }
+
+    private void readLookupDataBenchmark(byte[][] inputs, byte[][] 
randomInputs, boolean nullResult)
             throws IOException {
         Benchmark benchmark =
                 new Benchmark("reader-" + randomInputs.length, 
randomInputs.length)
@@ -91,7 +102,12 @@ public class LookupReaderBenchmark extends 
AbstractLookupBenchmark {
                         5,
                         () -> {
                             try {
-                                readData(options, randomInputs, 
pair.getLeft(), pair.getRight());
+                                readData(
+                                        options,
+                                        randomInputs,
+                                        pair.getLeft(),
+                                        pair.getRight(),
+                                        nullResult);
                             } catch (IOException e) {
                                 throw new RuntimeException(e);
                             }
@@ -106,7 +122,8 @@ public class LookupReaderBenchmark extends 
AbstractLookupBenchmark {
             CoreOptions options,
             byte[][] randomInputs,
             String filePath,
-            LookupStoreFactory.Context context)
+            LookupStoreFactory.Context context,
+            boolean nullResult)
             throws IOException {
         LookupStoreFactory factory =
                 LookupStoreFactory.create(
@@ -118,7 +135,11 @@ public class LookupReaderBenchmark extends 
AbstractLookupBenchmark {
         File file = new File(filePath);
         LookupStoreReader reader = factory.createReader(file, context);
         for (byte[] input : randomInputs) {
-            assertThat(reader.lookup(input)).isNotNull();
+            if (nullResult) {
+                assertThat(reader.lookup(input)).isNull();
+            } else {
+                assertThat(reader.lookup(input)).isNotNull();
+            }
         }
         reader.close();
     }