This is an automated email from the ASF dual-hosted git repository. jackie pushed a commit to branch bloom_filter_test in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 73972a602c4680167963cb5373cc792d992334ec Author: Xiaotian (Jackie) Jiang <[email protected]> AuthorDate: Tue Oct 6 15:40:54 2020 -0700 Reduce fpp to 0.001 and rmove the size limit for bloom filter --- .../creator/impl/bloom/BloomFilterCreator.java | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/pinot-core/src/main/java/org/apache/pinot/core/segment/creator/impl/bloom/BloomFilterCreator.java b/pinot-core/src/main/java/org/apache/pinot/core/segment/creator/impl/bloom/BloomFilterCreator.java index 7c1a6c4..547e478 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/segment/creator/impl/bloom/BloomFilterCreator.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/segment/creator/impl/bloom/BloomFilterCreator.java @@ -23,7 +23,6 @@ import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import org.apache.pinot.core.bloom.BloomFilter; -import org.apache.pinot.core.bloom.BloomFilterUtil; import org.apache.pinot.core.bloom.SegmentBloomFilterFactory; import org.apache.pinot.core.segment.creator.impl.V1Constants; @@ -39,17 +38,18 @@ import org.apache.pinot.core.segment.creator.impl.V1Constants; * 0.45 when the filter size is 1MB. */ public class BloomFilterCreator implements AutoCloseable { - private static double DEFAULT_MAX_FALSE_POS_PROBABILITY = 0.05; - private static int MB_IN_BITS = 8388608; + private static final double DEFAULT_MAX_FALSE_POS_PROBABILITY = 0.001; - private BloomFilter _bloomFilter; - private File _bloomFilterFile; + private final BloomFilter _bloomFilter; + private final File _bloomFilterFile; public BloomFilterCreator(File indexDir, String columnName, int cardinality) { + _bloomFilter = SegmentBloomFilterFactory.createSegmentBloomFilter(cardinality, DEFAULT_MAX_FALSE_POS_PROBABILITY); _bloomFilterFile = new File(indexDir, columnName + V1Constants.Indexes.BLOOM_FILTER_FILE_EXTENSION); - double maxFalsePosProbability = BloomFilterUtil - .computeMaxFalsePositiveProbabilityForNumBits(cardinality, MB_IN_BITS, DEFAULT_MAX_FALSE_POS_PROBABILITY); - _bloomFilter = SegmentBloomFilterFactory.createSegmentBloomFilter(cardinality, maxFalsePosProbability); + } + + public void add(Object input) { + _bloomFilter.add(input.toString()); } @Override @@ -61,8 +61,4 @@ public class BloomFilterCreator implements AutoCloseable { _bloomFilter.writeTo(outputStream); } } - - public void add(Object input) { - _bloomFilter.add(input.toString()); - } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
