This is an automated email from the ASF dual-hosted git repository. lzljs3620320 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git
commit 3d8f14bd075a1d93fb0fca1528e2b4f5582c8d90 Author: Jingsong <[email protected]> AuthorDate: Wed Jan 24 17:55:38 2024 +0800 [core] Introduce refreshCount to refresh cache in LRU --- .../java/org/apache/paimon/utils/FileBasedBloomFilter.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/paimon-common/src/main/java/org/apache/paimon/utils/FileBasedBloomFilter.java b/paimon-common/src/main/java/org/apache/paimon/utils/FileBasedBloomFilter.java index 541185e7b..06edd34e9 100644 --- a/paimon-common/src/main/java/org/apache/paimon/utils/FileBasedBloomFilter.java +++ b/paimon-common/src/main/java/org/apache/paimon/utils/FileBasedBloomFilter.java @@ -31,12 +31,16 @@ import static org.apache.paimon.utils.Preconditions.checkArgument; /** Util to apply a built bloom filter . */ public class FileBasedBloomFilter { + private static final int FORCE_REFRESH_CACHE = 30; + private final RandomAccessFile file; private final CacheManager cacheManager; private final BloomFilter filter; private final long readOffset; private final int readLength; + private int refreshCount; + public FileBasedBloomFilter( RandomAccessFile file, CacheManager cacheManager, @@ -49,10 +53,14 @@ public class FileBasedBloomFilter { this.filter = new BloomFilter(numRecords, readLength); this.readOffset = readOffset; this.readLength = readLength; + this.refreshCount = 0; } public boolean testHash(int hash) { - if (filter.getMemorySegment() == null) { + refreshCount++; + // we should refresh cache in LRU, but we cannot refresh everytime, it is costly. + // so we introduce a refresh count to reduce refresh + if (refreshCount == FORCE_REFRESH_CACHE || filter.getMemorySegment() == null) { MemorySegment segment = cacheManager.getPage( file, @@ -60,6 +68,7 @@ public class FileBasedBloomFilter { readLength, (position, length) -> filter.unsetMemorySegment()); filter.setMemorySegment(segment, 0); + refreshCount = 0; } return filter.testHash(hash); }
