Repository: hbase Updated Branches: refs/heads/branch-1.2 93bffa40d -> 2c8060e1e (forced update) refs/heads/branch-1.3 82f72b5be -> bfe373d18 (forced update)
HBASE-21168 BloomFilterUtil uses hardcoded randomness (Mike Drob) Signed-off-by: Andrew Purtell <apurt...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2c8060e1 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2c8060e1 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2c8060e1 Branch: refs/heads/branch-1.2 Commit: 2c8060e1e8be34b92f17645c80b8985354485281 Parents: d8790be Author: Mingliang Liu <lium...@apache.org> Authored: Wed Sep 12 12:31:09 2018 -0700 Committer: Andrew Purtell <apurt...@apache.org> Committed: Wed Sep 12 13:29:38 2018 -0700 ---------------------------------------------------------------------- .../hadoop/hbase/util/ByteBloomFilter.java | 54 +++++++++++--------- .../regionserver/TestCompoundBloomFilter.java | 6 ++- 2 files changed, 34 insertions(+), 26 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/2c8060e1/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ByteBloomFilter.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ByteBloomFilter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ByteBloomFilter.java index b8ec4c3..723571d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ByteBloomFilter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ByteBloomFilter.java @@ -26,6 +26,7 @@ import java.nio.ByteBuffer; import java.text.NumberFormat; import java.util.Random; +import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValue.KVComparator; @@ -423,26 +424,26 @@ public class ByteBloomFilter implements BloomFilter, BloomFilterWriter { int hashCount) { int hash1 = hash.hash(buf, offset, length, 0); - int hash2 = hash.hash(buf, offset, length, hash1); int bloomBitSize = bloomSize << 3; - + + int hash2 = 0; + int compositeHash = 0; + if (randomGeneratorForTest == null) { - // Production mode. - int compositeHash = hash1; - for (int i = 0; i < hashCount; i++) { - int hashLoc = Math.abs(compositeHash % bloomBitSize); - compositeHash += hash2; - if (!get(hashLoc, bloomBuf, bloomOffset)) { - return false; - } - } - } else { - // Test mode with "fake lookups" to estimate "ideal false positive rate". - for (int i = 0; i < hashCount; i++) { - int hashLoc = randomGeneratorForTest.nextInt(bloomBitSize); - if (!get(hashLoc, bloomBuf, bloomOffset)){ - return false; - } + // Production mode + compositeHash = hash1; + hash2 = hash.hash(buf, offset, length, hash1); + } + + for (int i = 0; i < hashCount; i++) { + int hashLoc = (randomGeneratorForTest == null + // Production mode + ? Math.abs(compositeHash % bloomBitSize) + // Test mode with "fake look-ups" to estimate "ideal false positive rate" + : randomGeneratorForTest.nextInt(bloomBitSize)); + compositeHash += hash2; + if (!get(hashLoc, bloomBuf, bloomOffset)) { + return false; } } return true; @@ -598,12 +599,17 @@ public class ByteBloomFilter implements BloomFilter, BloomFilterWriter { return bloom != null; } - public static void setFakeLookupMode(boolean enabled) { - if (enabled) { - randomGeneratorForTest = new Random(283742987L); - } else { - randomGeneratorForTest = null; - } + /** + * Sets a random generator to be used for look-ups instead of computing hashes. Can be used to + * simulate uniformity of accesses better in a test environment. Should not be set in a real + * environment where correctness matters! + * <p> + * This gets used in {@link #contains(byte[], int, int, ByteBuffer, int, int, Hash, int)} + * @param random The random number source to use, or null to compute actual hashes + */ + @VisibleForTesting + public static void setRandomGeneratorForTest(Random random) { + randomGeneratorForTest = random; } /** http://git-wip-us.apache.org/repos/asf/hbase/blob/2c8060e1/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java index 0129fad..a0bf6c5 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java @@ -218,7 +218,9 @@ public class TestCompoundBloomFilter { // Test for false positives (some percentage allowed). We test in two modes: // "fake lookup" which ignores the key distribution, and production mode. for (boolean fakeLookupEnabled : new boolean[] { true, false }) { - ByteBloomFilter.setFakeLookupMode(fakeLookupEnabled); + if (fakeLookupEnabled) { + ByteBloomFilter.setRandomGeneratorForTest(new Random(283742987L)); + } try { String fakeLookupModeStr = ", fake lookup is " + (fakeLookupEnabled ? "enabled" : "disabled"); @@ -268,7 +270,7 @@ public class TestCompoundBloomFilter { validateFalsePosRate(falsePosRate, nTrials, -2.58, cbf, fakeLookupModeStr); } finally { - ByteBloomFilter.setFakeLookupMode(false); + ByteBloomFilter.setRandomGeneratorForTest(null); } }
