Repository: hive Updated Branches: refs/heads/master cdec1b8de -> b63f7d7dd
HIVE-15892: Vectorization: Fast Hash tables need to do bounds checking during expand (Matt McCline, reviewed by Jason Dere Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b63f7d7d Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b63f7d7d Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b63f7d7d Branch: refs/heads/master Commit: b63f7d7dd82c493e95bb7c7d303c48e340d6020b Parents: cdec1b8 Author: Matt McCline <[email protected]> Authored: Wed Feb 15 18:22:27 2017 -0600 Committer: Matt McCline <[email protected]> Committed: Wed Feb 15 18:22:27 2017 -0600 ---------------------------------------------------------------------- .../fast/VectorMapJoinFastBytesHashMap.java | 4 +- .../VectorMapJoinFastBytesHashMultiSet.java | 4 +- .../fast/VectorMapJoinFastBytesHashSet.java | 4 +- .../fast/VectorMapJoinFastBytesHashTable.java | 8 ++- .../mapjoin/fast/VectorMapJoinFastHashMap.java | 4 +- .../fast/VectorMapJoinFastHashMultiSet.java | 4 +- .../mapjoin/fast/VectorMapJoinFastHashSet.java | 4 +- .../fast/VectorMapJoinFastHashTable.java | 20 ++++++- .../fast/VectorMapJoinFastLongHashMap.java | 4 +- .../fast/VectorMapJoinFastLongHashMultiSet.java | 4 +- .../fast/VectorMapJoinFastLongHashSet.java | 4 +- .../fast/VectorMapJoinFastLongHashTable.java | 8 ++- .../fast/VectorMapJoinFastMultiKeyHashMap.java | 4 +- .../VectorMapJoinFastMultiKeyHashMultiSet.java | 4 +- .../fast/VectorMapJoinFastMultiKeyHashSet.java | 4 +- .../fast/VectorMapJoinFastStringHashMap.java | 4 +- .../VectorMapJoinFastStringHashMultiSet.java | 4 +- .../fast/VectorMapJoinFastStringHashSet.java | 4 +- .../fast/VectorMapJoinFastTableContainer.java | 27 +++++----- .../fast/TestVectorMapJoinFastBytesHashMap.java | 56 ++++++++++++++++---- .../TestVectorMapJoinFastBytesHashMultiSet.java | 14 ++--- .../fast/TestVectorMapJoinFastBytesHashSet.java | 14 ++--- .../fast/TestVectorMapJoinFastLongHashMap.java | 50 ++++++++++++++--- .../TestVectorMapJoinFastLongHashMultiSet.java | 14 ++--- .../fast/TestVectorMapJoinFastLongHashSet.java | 14 ++--- .../fast/TestVectorMapJoinFastRowHashMap.java | 48 ++++++++--------- 26 files changed, 215 insertions(+), 118 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/b63f7d7d/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMap.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMap.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMap.java index d878f65..6242daf 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMap.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMap.java @@ -99,8 +99,8 @@ public abstract class VectorMapJoinFastBytesHashMap } public VectorMapJoinFastBytesHashMap( - int initialCapacity, float loadFactor, int writeBuffersSize) { - super(initialCapacity, loadFactor, writeBuffersSize); + int initialCapacity, float loadFactor, int writeBuffersSize, long estimatedKeyCount) { + super(initialCapacity, loadFactor, writeBuffersSize, estimatedKeyCount); valueStore = new VectorMapJoinFastValueStore(writeBuffersSize); http://git-wip-us.apache.org/repos/asf/hive/blob/b63f7d7d/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMultiSet.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMultiSet.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMultiSet.java index b328efd..1a41961 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMultiSet.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMultiSet.java @@ -92,8 +92,8 @@ public abstract class VectorMapJoinFastBytesHashMultiSet } public VectorMapJoinFastBytesHashMultiSet( - int initialCapacity, float loadFactor, int writeBuffersSize) { - super(initialCapacity, loadFactor, writeBuffersSize); + int initialCapacity, float loadFactor, int writeBuffersSize, long estimatedKeyCount) { + super(initialCapacity, loadFactor, writeBuffersSize, estimatedKeyCount); keyStore = new VectorMapJoinFastKeyStore(writeBuffersSize); } http://git-wip-us.apache.org/repos/asf/hive/blob/b63f7d7d/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashSet.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashSet.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashSet.java index c9b23bf..331867c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashSet.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashSet.java @@ -79,8 +79,8 @@ public abstract class VectorMapJoinFastBytesHashSet } public VectorMapJoinFastBytesHashSet( - int initialCapacity, float loadFactor, int writeBuffersSize) { - super(initialCapacity, loadFactor, writeBuffersSize); + int initialCapacity, float loadFactor, int writeBuffersSize, long estimatedKeyCount) { + super(initialCapacity, loadFactor, writeBuffersSize, estimatedKeyCount); keyStore = new VectorMapJoinFastKeyStore(writeBuffersSize); } http://git-wip-us.apache.org/repos/asf/hive/blob/b63f7d7d/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashTable.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashTable.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashTable.java index dc0476b..b93e977 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashTable.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashTable.java @@ -105,6 +105,10 @@ public abstract class VectorMapJoinFastBytesHashTable private void expandAndRehash() { + // We allocate triples, so we cannot go above highest Integer power of 2 / 6. + if (logicalHashBucketCount > ONE_SIXTH_LIMIT) { + throwExpandError(ONE_SIXTH_LIMIT, "Bytes"); + } int newLogicalHashBucketCount = logicalHashBucketCount * 2; int newLogicalHashBucketMask = newLogicalHashBucketCount - 1; int newMetricPutConflict = 0; @@ -210,8 +214,8 @@ public abstract class VectorMapJoinFastBytesHashTable } public VectorMapJoinFastBytesHashTable( - int initialCapacity, float loadFactor, int writeBuffersSize) { - super(initialCapacity, loadFactor, writeBuffersSize); + int initialCapacity, float loadFactor, int writeBuffersSize, long estimatedKeyCount) { + super(initialCapacity, loadFactor, writeBuffersSize, estimatedKeyCount); allocateBucketArray(); } } http://git-wip-us.apache.org/repos/asf/hive/blob/b63f7d7d/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashMap.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashMap.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashMap.java index 262b619..c32964b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashMap.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashMap.java @@ -32,7 +32,7 @@ public abstract class VectorMapJoinFastHashMap public VectorMapJoinFastHashMap( boolean isOuterJoin, - int initialCapacity, float loadFactor, int writeBuffersSize) { - super(initialCapacity, loadFactor, writeBuffersSize); + int initialCapacity, float loadFactor, int writeBuffersSize, long estimatedKeyCount) { + super(initialCapacity, loadFactor, writeBuffersSize, estimatedKeyCount); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/b63f7d7d/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashMultiSet.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashMultiSet.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashMultiSet.java index 5f7c6a7..2b4a2b8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashMultiSet.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashMultiSet.java @@ -42,7 +42,7 @@ public abstract class VectorMapJoinFastHashMultiSet public VectorMapJoinFastHashMultiSet( boolean isOuterJoin, - int initialCapacity, float loadFactor, int writeBuffersSize) { - super(initialCapacity, loadFactor, writeBuffersSize); + int initialCapacity, float loadFactor, int writeBuffersSize, long estimatedKeyCount) { + super(initialCapacity, loadFactor, writeBuffersSize, estimatedKeyCount); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/b63f7d7d/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashSet.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashSet.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashSet.java index 8509971..9339250 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashSet.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashSet.java @@ -38,7 +38,7 @@ public abstract class VectorMapJoinFastHashSet public VectorMapJoinFastHashSet( boolean isOuterJoin, - int initialCapacity, float loadFactor, int writeBuffersSize) { - super(initialCapacity, loadFactor, writeBuffersSize); + int initialCapacity, float loadFactor, int writeBuffersSize, long estimatedKeyCount) { + super(initialCapacity, loadFactor, writeBuffersSize, estimatedKeyCount); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/b63f7d7d/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTable.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTable.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTable.java index 7df9eed..9030e5f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTable.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTable.java @@ -31,12 +31,28 @@ public abstract class VectorMapJoinFastHashTable implements VectorMapJoinHashTab protected float loadFactor; protected final int writeBuffersSize; + protected long estimatedKeyCount; + protected int metricPutConflict; protected int largestNumberOfSteps; protected int keysAssigned; protected int resizeThreshold; protected int metricExpands; + // 2^30 (we cannot use Integer.MAX_VALUE which is 2^31-1). + public static int HIGHEST_INT_POWER_OF_2 = 1073741824; + + public static int ONE_QUARTER_LIMIT = HIGHEST_INT_POWER_OF_2 / 4; + public static int ONE_SIXTH_LIMIT = HIGHEST_INT_POWER_OF_2 / 6; + + public void throwExpandError(int limit, String dataTypeName) { + throw new RuntimeException( + "Vector MapJoin " + dataTypeName + " Hash Table cannot grow any more -- use a smaller container size. " + + "Current logical size is " + logicalHashBucketCount + " and " + + "the limit is " + limit + ". " + + "Estimated key count was " + (estimatedKeyCount == -1 ? "not available" : estimatedKeyCount) + "."); + } + private static void validateCapacity(long capacity) { if (Long.bitCount(capacity) != 1) { throw new AssertionError("Capacity must be a power of two"); @@ -51,13 +67,15 @@ public abstract class VectorMapJoinFastHashTable implements VectorMapJoinHashTab } public VectorMapJoinFastHashTable( - int initialCapacity, float loadFactor, int writeBuffersSize) { + int initialCapacity, float loadFactor, int writeBuffersSize, long estimatedKeyCount) { initialCapacity = (Long.bitCount(initialCapacity) == 1) ? initialCapacity : nextHighestPowerOfTwo(initialCapacity); validateCapacity(initialCapacity); + this.estimatedKeyCount = estimatedKeyCount; + logicalHashBucketCount = initialCapacity; logicalHashBucketMask = logicalHashBucketCount - 1; resizeThreshold = (int)(logicalHashBucketCount * loadFactor); http://git-wip-us.apache.org/repos/asf/hive/blob/b63f7d7d/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMap.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMap.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMap.java index cd51d0d..6fe98f9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMap.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMap.java @@ -107,9 +107,9 @@ public class VectorMapJoinFastLongHashMap public VectorMapJoinFastLongHashMap( boolean minMaxEnabled, boolean isOuterJoin, HashTableKeyType hashTableKeyType, - int initialCapacity, float loadFactor, int writeBuffersSize) { + int initialCapacity, float loadFactor, int writeBuffersSize, long estimatedKeyCount) { super(minMaxEnabled, isOuterJoin, hashTableKeyType, - initialCapacity, loadFactor, writeBuffersSize); + initialCapacity, loadFactor, writeBuffersSize, estimatedKeyCount); valueStore = new VectorMapJoinFastValueStore(writeBuffersSize); } } http://git-wip-us.apache.org/repos/asf/hive/blob/b63f7d7d/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMultiSet.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMultiSet.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMultiSet.java index 032233a..9140aee 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMultiSet.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMultiSet.java @@ -96,8 +96,8 @@ public class VectorMapJoinFastLongHashMultiSet public VectorMapJoinFastLongHashMultiSet( boolean minMaxEnabled, boolean isOuterJoin, HashTableKeyType hashTableKeyType, - int initialCapacity, float loadFactor, int writeBuffersSize) { + int initialCapacity, float loadFactor, int writeBuffersSize, long estimatedKeyCount) { super(minMaxEnabled, isOuterJoin, hashTableKeyType, - initialCapacity, loadFactor, writeBuffersSize); + initialCapacity, loadFactor, writeBuffersSize, estimatedKeyCount); } } http://git-wip-us.apache.org/repos/asf/hive/blob/b63f7d7d/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashSet.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashSet.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashSet.java index 21701d4..d3efb11 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashSet.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashSet.java @@ -92,8 +92,8 @@ public class VectorMapJoinFastLongHashSet public VectorMapJoinFastLongHashSet( boolean minMaxEnabled, boolean isOuterJoin, HashTableKeyType hashTableKeyType, - int initialCapacity, float loadFactor, int writeBuffersSize) { + int initialCapacity, float loadFactor, int writeBuffersSize, long estimatedKeyCount) { super(minMaxEnabled, isOuterJoin, hashTableKeyType, - initialCapacity, loadFactor, writeBuffersSize); + initialCapacity, loadFactor, writeBuffersSize, estimatedKeyCount); } } http://git-wip-us.apache.org/repos/asf/hive/blob/b63f7d7d/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashTable.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashTable.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashTable.java index bc892ba..8bfa07c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashTable.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashTable.java @@ -156,6 +156,10 @@ public abstract class VectorMapJoinFastLongHashTable private void expandAndRehash() { + // We allocate pairs, so we cannot go above highest Integer power of 2 / 4. + if (logicalHashBucketCount > ONE_QUARTER_LIMIT) { + throwExpandError(ONE_QUARTER_LIMIT, "Long"); + } int newLogicalHashBucketCount = logicalHashBucketCount * 2; int newLogicalHashBucketMask = newLogicalHashBucketCount - 1; int newMetricPutConflict = 0; @@ -262,8 +266,8 @@ public abstract class VectorMapJoinFastLongHashTable public VectorMapJoinFastLongHashTable( boolean minMaxEnabled, boolean isOuterJoin, HashTableKeyType hashTableKeyType, - int initialCapacity, float loadFactor, int writeBuffersSize) { - super(initialCapacity, loadFactor, writeBuffersSize); + int initialCapacity, float loadFactor, int writeBuffersSize, long estimatedKeyCount) { + super(initialCapacity, loadFactor, writeBuffersSize, estimatedKeyCount); this.isOuterJoin = isOuterJoin; this.hashTableKeyType = hashTableKeyType; PrimitiveTypeInfo[] primitiveTypeInfos = { hashTableKeyType.getPrimitiveTypeInfo() }; http://git-wip-us.apache.org/repos/asf/hive/blob/b63f7d7d/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashMap.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashMap.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashMap.java index cee3b3b..add4788 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashMap.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashMap.java @@ -50,7 +50,7 @@ public class VectorMapJoinFastMultiKeyHashMap public VectorMapJoinFastMultiKeyHashMap( boolean isOuterJoin, - int initialCapacity, float loadFactor, int writeBuffersSize) { - super(initialCapacity, loadFactor, writeBuffersSize); + int initialCapacity, float loadFactor, int writeBuffersSize, long estimatedKeyCount) { + super(initialCapacity, loadFactor, writeBuffersSize, estimatedKeyCount); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/b63f7d7d/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashMultiSet.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashMultiSet.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashMultiSet.java index ff82ac4..faefdbb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashMultiSet.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashMultiSet.java @@ -48,8 +48,8 @@ public class VectorMapJoinFastMultiKeyHashMultiSet public VectorMapJoinFastMultiKeyHashMultiSet( boolean isOuterJoin, - int initialCapacity, float loadFactor, int writeBuffersSize) { - super(initialCapacity, loadFactor, writeBuffersSize); + int initialCapacity, float loadFactor, int writeBuffersSize, long estimatedKeyCount) { + super(initialCapacity, loadFactor, writeBuffersSize, estimatedKeyCount); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/b63f7d7d/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashSet.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashSet.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashSet.java index de0666d..5328910 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashSet.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashSet.java @@ -48,8 +48,8 @@ public class VectorMapJoinFastMultiKeyHashSet public VectorMapJoinFastMultiKeyHashSet( boolean isOuterJoin, - int initialCapacity, float loadFactor, int writeBuffersSize) { - super(initialCapacity, loadFactor, writeBuffersSize); + int initialCapacity, float loadFactor, int writeBuffersSize, long estimatedKeyCount) { + super(initialCapacity, loadFactor, writeBuffersSize, estimatedKeyCount); } http://git-wip-us.apache.org/repos/asf/hive/blob/b63f7d7d/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashMap.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashMap.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashMap.java index 35af1d1..f13034f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashMap.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashMap.java @@ -39,8 +39,8 @@ public class VectorMapJoinFastStringHashMap extends VectorMapJoinFastBytesHashMa public VectorMapJoinFastStringHashMap( boolean isOuterJoin, - int initialCapacity, float loadFactor, int writeBuffersSize) { - super(initialCapacity, loadFactor, writeBuffersSize); + int initialCapacity, float loadFactor, int writeBuffersSize, long estimatedKeyCount) { + super(initialCapacity, loadFactor, writeBuffersSize, estimatedKeyCount); stringCommon = new VectorMapJoinFastStringCommon(isOuterJoin); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/b63f7d7d/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashMultiSet.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashMultiSet.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashMultiSet.java index 36120b7..53ad7b4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashMultiSet.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashMultiSet.java @@ -39,8 +39,8 @@ public class VectorMapJoinFastStringHashMultiSet extends VectorMapJoinFastBytesH public VectorMapJoinFastStringHashMultiSet( boolean isOuterJoin, - int initialCapacity, float loadFactor, int writeBuffersSize) { - super(initialCapacity, loadFactor, writeBuffersSize); + int initialCapacity, float loadFactor, int writeBuffersSize, long estimatedKeyCount) { + super(initialCapacity, loadFactor, writeBuffersSize, estimatedKeyCount); stringCommon = new VectorMapJoinFastStringCommon(isOuterJoin); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/b63f7d7d/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashSet.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashSet.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashSet.java index 2ed6ab3..723c729 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashSet.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashSet.java @@ -39,8 +39,8 @@ public class VectorMapJoinFastStringHashSet extends VectorMapJoinFastBytesHashSe public VectorMapJoinFastStringHashSet( boolean isOuterJoin, - int initialCapacity, float loadFactor, int writeBuffersSize) { - super(initialCapacity, loadFactor, writeBuffersSize); + int initialCapacity, float loadFactor, int writeBuffersSize, long estimatedKeyCount) { + super(initialCapacity, loadFactor, writeBuffersSize, estimatedKeyCount); stringCommon = new VectorMapJoinFastStringCommon(isOuterJoin); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/b63f7d7d/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastTableContainer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastTableContainer.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastTableContainer.java index 069cc9a..05f1cf1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastTableContainer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastTableContainer.java @@ -55,13 +55,14 @@ public class VectorMapJoinFastTableContainer implements VectorMapJoinTableContai private final int threshold; private final float loadFactor; private final int wbSize; - private final long keyCount; + + private final long estimatedKeyCount; private final VectorMapJoinFastHashTable vectorMapJoinFastHashTable; public VectorMapJoinFastTableContainer(MapJoinDesc desc, Configuration hconf, - long keyCount) throws SerDeException { + long estimatedKeyCount) throws SerDeException { this.desc = desc; this.hconf = hconf; @@ -71,7 +72,7 @@ public class VectorMapJoinFastTableContainer implements VectorMapJoinTableContai loadFactor = HiveConf.getFloatVar(hconf, HiveConf.ConfVars.HIVEHASHTABLELOADFACTOR); wbSize = HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVEHASHTABLEWBSIZE); - this.keyCount = keyCount; + this.estimatedKeyCount = estimatedKeyCount; // LOG.info("VectorMapJoinFastTableContainer load keyCountAdj " + keyCountAdj); // LOG.info("VectorMapJoinFastTableContainer load threshold " + threshold); @@ -79,7 +80,7 @@ public class VectorMapJoinFastTableContainer implements VectorMapJoinTableContai // LOG.info("VectorMapJoinFastTableContainer load wbSize " + wbSize); int newThreshold = HashMapWrapper.calculateTableSize( - keyCountAdj, threshold, loadFactor, keyCount); + keyCountAdj, threshold, loadFactor, estimatedKeyCount); // LOG.debug("VectorMapJoinFastTableContainer load newThreshold " + newThreshold); @@ -114,17 +115,17 @@ public class VectorMapJoinFastTableContainer implements VectorMapJoinTableContai case HASH_MAP: hashTable = new VectorMapJoinFastLongHashMap( minMaxEnabled, isOuterJoin, hashTableKeyType, - newThreshold, loadFactor, writeBufferSize); + newThreshold, loadFactor, writeBufferSize, estimatedKeyCount); break; case HASH_MULTISET: hashTable = new VectorMapJoinFastLongHashMultiSet( minMaxEnabled, isOuterJoin, hashTableKeyType, - newThreshold, loadFactor, writeBufferSize); + newThreshold, loadFactor, writeBufferSize, estimatedKeyCount); break; case HASH_SET: hashTable = new VectorMapJoinFastLongHashSet( minMaxEnabled, isOuterJoin, hashTableKeyType, - newThreshold, loadFactor, writeBufferSize); + newThreshold, loadFactor, writeBufferSize, estimatedKeyCount); break; } break; @@ -134,17 +135,17 @@ public class VectorMapJoinFastTableContainer implements VectorMapJoinTableContai case HASH_MAP: hashTable = new VectorMapJoinFastStringHashMap( isOuterJoin, - newThreshold, loadFactor, writeBufferSize); + newThreshold, loadFactor, writeBufferSize, estimatedKeyCount); break; case HASH_MULTISET: hashTable = new VectorMapJoinFastStringHashMultiSet( isOuterJoin, - newThreshold, loadFactor, writeBufferSize); + newThreshold, loadFactor, writeBufferSize, estimatedKeyCount); break; case HASH_SET: hashTable = new VectorMapJoinFastStringHashSet( isOuterJoin, - newThreshold, loadFactor, writeBufferSize); + newThreshold, loadFactor, writeBufferSize, estimatedKeyCount); break; } break; @@ -154,17 +155,17 @@ public class VectorMapJoinFastTableContainer implements VectorMapJoinTableContai case HASH_MAP: hashTable = new VectorMapJoinFastMultiKeyHashMap( isOuterJoin, - newThreshold, loadFactor, writeBufferSize); + newThreshold, loadFactor, writeBufferSize, estimatedKeyCount); break; case HASH_MULTISET: hashTable = new VectorMapJoinFastMultiKeyHashMultiSet( isOuterJoin, - newThreshold, loadFactor, writeBufferSize); + newThreshold, loadFactor, writeBufferSize, estimatedKeyCount); break; case HASH_SET: hashTable = new VectorMapJoinFastMultiKeyHashSet( isOuterJoin, - newThreshold, loadFactor, writeBufferSize); + newThreshold, loadFactor, writeBufferSize, estimatedKeyCount); break; } break; http://git-wip-us.apache.org/repos/asf/hive/blob/b63f7d7d/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastBytesHashMap.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastBytesHashMap.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastBytesHashMap.java index 8525e99..9051804 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastBytesHashMap.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastBytesHashMap.java @@ -42,7 +42,7 @@ public class TestVectorMapJoinFastBytesHashMap extends CommonFastHashTable { VectorMapJoinFastMultiKeyHashMap map = new VectorMapJoinFastMultiKeyHashMap( - false,CAPACITY, LOAD_FACTOR, WB_SIZE); + false,CAPACITY, LOAD_FACTOR, WB_SIZE, -1); VerifyFastBytesHashMap verifyTable = new VerifyFastBytesHashMap(); @@ -76,7 +76,7 @@ public class TestVectorMapJoinFastBytesHashMap extends CommonFastHashTable { VectorMapJoinFastMultiKeyHashMap map = new VectorMapJoinFastMultiKeyHashMap( - false,CAPACITY, LOAD_FACTOR, WB_SIZE); + false,CAPACITY, LOAD_FACTOR, WB_SIZE, -1); VerifyFastBytesHashMap verifyTable = new VerifyFastBytesHashMap(); @@ -103,7 +103,7 @@ public class TestVectorMapJoinFastBytesHashMap extends CommonFastHashTable { VectorMapJoinFastMultiKeyHashMap map = new VectorMapJoinFastMultiKeyHashMap( - false,CAPACITY, LOAD_FACTOR, WB_SIZE); + false,CAPACITY, LOAD_FACTOR, WB_SIZE, -1); VerifyFastBytesHashMap verifyTable = new VerifyFastBytesHashMap(); @@ -141,7 +141,7 @@ public class TestVectorMapJoinFastBytesHashMap extends CommonFastHashTable { // Make sure the map does not expand; should be able to find space. VectorMapJoinFastMultiKeyHashMap map = - new VectorMapJoinFastMultiKeyHashMap(false,CAPACITY, 1f, WB_SIZE); + new VectorMapJoinFastMultiKeyHashMap(false,CAPACITY, 1f, WB_SIZE, -1); VerifyFastBytesHashMap verifyTable = new VerifyFastBytesHashMap(); @@ -185,7 +185,7 @@ public class TestVectorMapJoinFastBytesHashMap extends CommonFastHashTable { // Start with capacity 1; make sure we expand on every put. VectorMapJoinFastMultiKeyHashMap map = - new VectorMapJoinFastMultiKeyHashMap(false,1, 0.0000001f, WB_SIZE); + new VectorMapJoinFastMultiKeyHashMap(false,1, 0.0000001f, WB_SIZE, -1); VerifyFastBytesHashMap verifyTable = new VerifyFastBytesHashMap(); @@ -213,15 +213,27 @@ public class TestVectorMapJoinFastBytesHashMap extends CommonFastHashTable { public void addAndVerifyMultipleKeyMultipleValue(int keyCount, VectorMapJoinFastMultiKeyHashMap map, VerifyFastBytesHashMap verifyTable) throws HiveException, IOException { + addAndVerifyMultipleKeyMultipleValue(keyCount, map, verifyTable, MAX_KEY_LENGTH, -1); + } + + public void addAndVerifyMultipleKeyMultipleValue(int keyCount, + VectorMapJoinFastMultiKeyHashMap map, VerifyFastBytesHashMap verifyTable, + int maxKeyLength, int fixedValueLength) + throws HiveException, IOException { for (int i = 0; i < keyCount; i++) { - byte[] value = new byte[generateLargeCount() - 1]; + byte[] value; + if (fixedValueLength == -1) { + value = new byte[generateLargeCount() - 1]; + } else { + value = new byte[fixedValueLength]; + } random.nextBytes(value); // Add a new key or add a value to an existing key? if (random.nextBoolean() || verifyTable.getCount() == 0) { byte[] key; while (true) { - key = new byte[random.nextInt(MAX_KEY_LENGTH)]; + key = new byte[random.nextInt(maxKeyLength)]; random.nextBytes(key); if (!verifyTable.contains(key)) { // Unique keys for this test. @@ -240,6 +252,7 @@ public class TestVectorMapJoinFastBytesHashMap extends CommonFastHashTable { } verifyTable.verify(map); } + @Test public void testMultipleKeysMultipleValue() throws Exception { random = new Random(9332); @@ -247,7 +260,7 @@ public class TestVectorMapJoinFastBytesHashMap extends CommonFastHashTable { // Use a large capacity that doesn't require expansion, yet. VectorMapJoinFastMultiKeyHashMap map = new VectorMapJoinFastMultiKeyHashMap( - false,LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + false,LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastBytesHashMap verifyTable = new VerifyFastBytesHashMap(); @@ -262,7 +275,7 @@ public class TestVectorMapJoinFastBytesHashMap extends CommonFastHashTable { // Use a large capacity that doesn't require expansion, yet. VectorMapJoinFastMultiKeyHashMap map = new VectorMapJoinFastMultiKeyHashMap( - false,MODERATE_CAPACITY, LOAD_FACTOR, MODERATE_WB_SIZE); + false,MODERATE_CAPACITY, LOAD_FACTOR, MODERATE_WB_SIZE, -1); VerifyFastBytesHashMap verifyTable = new VerifyFastBytesHashMap(); @@ -277,11 +290,34 @@ public class TestVectorMapJoinFastBytesHashMap extends CommonFastHashTable { // Use a large capacity that doesn't require expansion, yet. VectorMapJoinFastMultiKeyHashMap map = new VectorMapJoinFastMultiKeyHashMap( - false,LARGE_CAPACITY, LOAD_FACTOR, MODERATE_WB_SIZE); + false,LARGE_CAPACITY, LOAD_FACTOR, MODERATE_WB_SIZE, -1); VerifyFastBytesHashMap verifyTable = new VerifyFastBytesHashMap(); int keyCount = 1000000; addAndVerifyMultipleKeyMultipleValue(keyCount, map, verifyTable); } + + /* + // Can't seem to get mvn to give enough memory to run this successfully. + @Test + public void testKeyCountLimit() throws Exception { + random = new Random(28400); + + // Use a large capacity that doesn't require expansion, yet. + VectorMapJoinFastMultiKeyHashMap map = + new VectorMapJoinFastMultiKeyHashMap( + false, LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, 10000000); + + VerifyFastBytesHashMap verifyTable = new VerifyFastBytesHashMap(); + + int keyCount = Integer.MAX_VALUE; + try { + addAndVerifyMultipleKeyMultipleValue(keyCount, map, verifyTable, 10, 1); + } catch (RuntimeException re) { + System.out.println(re.toString()); + assertTrue(re.toString().startsWith("Vector MapJoin Bytes Hash Table cannot grow any more")); + } + } + */ } http://git-wip-us.apache.org/repos/asf/hive/blob/b63f7d7d/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastBytesHashMultiSet.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastBytesHashMultiSet.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastBytesHashMultiSet.java index 449a8b2..de30b31 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastBytesHashMultiSet.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastBytesHashMultiSet.java @@ -37,7 +37,7 @@ public class TestVectorMapJoinFastBytesHashMultiSet extends CommonFastHashTable VectorMapJoinFastMultiKeyHashMultiSet map = new VectorMapJoinFastMultiKeyHashMultiSet( - false,CAPACITY, LOAD_FACTOR, WB_SIZE); + false,CAPACITY, LOAD_FACTOR, WB_SIZE, -1); VerifyFastBytesHashMultiSet verifyTable = new VerifyFastBytesHashMultiSet(); @@ -65,7 +65,7 @@ public class TestVectorMapJoinFastBytesHashMultiSet extends CommonFastHashTable VectorMapJoinFastMultiKeyHashMultiSet map = new VectorMapJoinFastMultiKeyHashMultiSet( - false,CAPACITY, LOAD_FACTOR, WB_SIZE); + false,CAPACITY, LOAD_FACTOR, WB_SIZE, -1); VerifyFastBytesHashMultiSet verifyTable = new VerifyFastBytesHashMultiSet(); @@ -91,7 +91,7 @@ public class TestVectorMapJoinFastBytesHashMultiSet extends CommonFastHashTable VectorMapJoinFastMultiKeyHashMultiSet map = new VectorMapJoinFastMultiKeyHashMultiSet( - false,CAPACITY, LOAD_FACTOR, WB_SIZE); + false,CAPACITY, LOAD_FACTOR, WB_SIZE, -1); VerifyFastBytesHashMultiSet verifyTable = new VerifyFastBytesHashMultiSet(); @@ -126,7 +126,7 @@ public class TestVectorMapJoinFastBytesHashMultiSet extends CommonFastHashTable // Make sure the map does not expand; should be able to find space. VectorMapJoinFastMultiKeyHashMultiSet map = - new VectorMapJoinFastMultiKeyHashMultiSet(false,CAPACITY, 1f, WB_SIZE); + new VectorMapJoinFastMultiKeyHashMultiSet(false,CAPACITY, 1f, WB_SIZE, -1); VerifyFastBytesHashMultiSet verifyTable = new VerifyFastBytesHashMultiSet(); @@ -168,7 +168,7 @@ public class TestVectorMapJoinFastBytesHashMultiSet extends CommonFastHashTable // Start with capacity 1; make sure we expand on every put. VectorMapJoinFastMultiKeyHashMultiSet map = - new VectorMapJoinFastMultiKeyHashMultiSet(false,1, 0.0000001f, WB_SIZE); + new VectorMapJoinFastMultiKeyHashMultiSet(false,1, 0.0000001f, WB_SIZE, -1); VerifyFastBytesHashMultiSet verifyTable = new VerifyFastBytesHashMultiSet(); @@ -228,7 +228,7 @@ public class TestVectorMapJoinFastBytesHashMultiSet extends CommonFastHashTable // Use a large capacity that doesn't require expansion, yet. VectorMapJoinFastMultiKeyHashMultiSet map = new VectorMapJoinFastMultiKeyHashMultiSet( - false,LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + false,LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastBytesHashMultiSet verifyTable = new VerifyFastBytesHashMultiSet(); @@ -243,7 +243,7 @@ public class TestVectorMapJoinFastBytesHashMultiSet extends CommonFastHashTable // Use a large capacity that doesn't require expansion, yet. VectorMapJoinFastMultiKeyHashMultiSet map = new VectorMapJoinFastMultiKeyHashMultiSet( - false,MODERATE_CAPACITY, LOAD_FACTOR, MODERATE_WB_SIZE); + false,MODERATE_CAPACITY, LOAD_FACTOR, MODERATE_WB_SIZE, -1); VerifyFastBytesHashMultiSet verifyTable = new VerifyFastBytesHashMultiSet(); http://git-wip-us.apache.org/repos/asf/hive/blob/b63f7d7d/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastBytesHashSet.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastBytesHashSet.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastBytesHashSet.java index ef7c91c..969c7d1 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastBytesHashSet.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastBytesHashSet.java @@ -37,7 +37,7 @@ public class TestVectorMapJoinFastBytesHashSet extends CommonFastHashTable { VectorMapJoinFastMultiKeyHashSet map = new VectorMapJoinFastMultiKeyHashSet( - false,CAPACITY, LOAD_FACTOR, WB_SIZE); + false,CAPACITY, LOAD_FACTOR, WB_SIZE, -1); VerifyFastBytesHashSet verifyTable = new VerifyFastBytesHashSet(); @@ -65,7 +65,7 @@ public class TestVectorMapJoinFastBytesHashSet extends CommonFastHashTable { VectorMapJoinFastMultiKeyHashSet map = new VectorMapJoinFastMultiKeyHashSet( - false,CAPACITY, LOAD_FACTOR, WB_SIZE); + false,CAPACITY, LOAD_FACTOR, WB_SIZE, -1); VerifyFastBytesHashSet verifyTable = new VerifyFastBytesHashSet(); @@ -91,7 +91,7 @@ public class TestVectorMapJoinFastBytesHashSet extends CommonFastHashTable { VectorMapJoinFastMultiKeyHashSet map = new VectorMapJoinFastMultiKeyHashSet( - false,CAPACITY, LOAD_FACTOR, WB_SIZE); + false,CAPACITY, LOAD_FACTOR, WB_SIZE, -1); VerifyFastBytesHashSet verifyTable = new VerifyFastBytesHashSet(); @@ -125,7 +125,7 @@ public class TestVectorMapJoinFastBytesHashSet extends CommonFastHashTable { // Make sure the map does not expand; should be able to find space. VectorMapJoinFastMultiKeyHashSet map = - new VectorMapJoinFastMultiKeyHashSet(false,CAPACITY, 1f, WB_SIZE); + new VectorMapJoinFastMultiKeyHashSet(false,CAPACITY, 1f, WB_SIZE, -1); VerifyFastBytesHashSet verifyTable = new VerifyFastBytesHashSet(); @@ -167,7 +167,7 @@ public class TestVectorMapJoinFastBytesHashSet extends CommonFastHashTable { // Start with capacity 1; make sure we expand on every put. VectorMapJoinFastMultiKeyHashSet map = - new VectorMapJoinFastMultiKeyHashSet(false,1, 0.0000001f, WB_SIZE); + new VectorMapJoinFastMultiKeyHashSet(false,1, 0.0000001f, WB_SIZE, -1); VerifyFastBytesHashSet verifyTable = new VerifyFastBytesHashSet(); @@ -227,7 +227,7 @@ public class TestVectorMapJoinFastBytesHashSet extends CommonFastHashTable { // Use a large capacity that doesn't require expansion, yet. VectorMapJoinFastMultiKeyHashSet map = new VectorMapJoinFastMultiKeyHashSet( - false,LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + false,LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastBytesHashSet verifyTable = new VerifyFastBytesHashSet(); @@ -242,7 +242,7 @@ public class TestVectorMapJoinFastBytesHashSet extends CommonFastHashTable { // Use a large capacity that doesn't require expansion, yet. VectorMapJoinFastMultiKeyHashSet map = new VectorMapJoinFastMultiKeyHashSet( - false,MODERATE_CAPACITY, LOAD_FACTOR, MODERATE_WB_SIZE); + false,MODERATE_CAPACITY, LOAD_FACTOR, MODERATE_WB_SIZE, -1); VerifyFastBytesHashSet verifyTable = new VerifyFastBytesHashSet(); http://git-wip-us.apache.org/repos/asf/hive/blob/b63f7d7d/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastLongHashMap.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastLongHashMap.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastLongHashMap.java index e8bbee3..315a54d 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastLongHashMap.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastLongHashMap.java @@ -39,7 +39,7 @@ public class TestVectorMapJoinFastLongHashMap extends CommonFastHashTable { VectorMapJoinFastLongHashMap map = new VectorMapJoinFastLongHashMap( - false, false, HashTableKeyType.LONG, CAPACITY, LOAD_FACTOR, WB_SIZE); + false, false, HashTableKeyType.LONG, CAPACITY, LOAD_FACTOR, WB_SIZE, -1); VerifyFastLongHashMap verifyTable = new VerifyFastLongHashMap(); @@ -72,7 +72,7 @@ public class TestVectorMapJoinFastLongHashMap extends CommonFastHashTable { VectorMapJoinFastLongHashMap map = new VectorMapJoinFastLongHashMap( - false, false, HashTableKeyType.LONG, CAPACITY, LOAD_FACTOR, WB_SIZE); + false, false, HashTableKeyType.LONG, CAPACITY, LOAD_FACTOR, WB_SIZE, -1); VerifyFastLongHashMap verifyTable = new VerifyFastLongHashMap(); @@ -102,7 +102,7 @@ public class TestVectorMapJoinFastLongHashMap extends CommonFastHashTable { VectorMapJoinFastLongHashMap map = new VectorMapJoinFastLongHashMap( - false, false, HashTableKeyType.LONG, CAPACITY, LOAD_FACTOR, WB_SIZE); + false, false, HashTableKeyType.LONG, CAPACITY, LOAD_FACTOR, WB_SIZE, -1); VerifyFastLongHashMap verifyTable = new VerifyFastLongHashMap(); @@ -138,7 +138,7 @@ public class TestVectorMapJoinFastLongHashMap extends CommonFastHashTable { // Make sure the map does not expand; should be able to find space. VectorMapJoinFastLongHashMap map = new VectorMapJoinFastLongHashMap( - false, false, HashTableKeyType.LONG, CAPACITY, 1f, WB_SIZE); + false, false, HashTableKeyType.LONG, CAPACITY, 1f, WB_SIZE, -1); VerifyFastLongHashMap verifyTable = new VerifyFastLongHashMap(); @@ -181,7 +181,7 @@ public class TestVectorMapJoinFastLongHashMap extends CommonFastHashTable { // Start with capacity 1; make sure we expand on every put. VectorMapJoinFastLongHashMap map = new VectorMapJoinFastLongHashMap( - false, false, HashTableKeyType.LONG, 1, 0.0000001f, WB_SIZE); + false, false, HashTableKeyType.LONG, 1, 0.0000001f, WB_SIZE, -1); VerifyFastLongHashMap verifyTable = new VerifyFastLongHashMap(); @@ -208,8 +208,19 @@ public class TestVectorMapJoinFastLongHashMap extends CommonFastHashTable { public void addAndVerifyMultipleKeyMultipleValue(int keyCount, VectorMapJoinFastLongHashMap map, VerifyFastLongHashMap verifyTable) throws HiveException, IOException { + addAndVerifyMultipleKeyMultipleValue(keyCount, map, verifyTable, -1); + } + + public void addAndVerifyMultipleKeyMultipleValue(int keyCount, + VectorMapJoinFastLongHashMap map, VerifyFastLongHashMap verifyTable, int fixedValueLength) + throws HiveException, IOException { for (int i = 0; i < keyCount; i++) { - byte[] value = new byte[generateLargeCount() - 1]; + byte[] value; + if (fixedValueLength == -1) { + value = new byte[generateLargeCount() - 1]; + } else { + value = new byte[fixedValueLength]; + } random.nextBytes(value); // Add a new key or add a value to an existing key? @@ -241,7 +252,7 @@ public class TestVectorMapJoinFastLongHashMap extends CommonFastHashTable { // Use a large capacity that doesn't require expansion, yet. VectorMapJoinFastLongHashMap map = new VectorMapJoinFastLongHashMap( - false, false, HashTableKeyType.LONG, LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + false, false, HashTableKeyType.LONG, LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastLongHashMap verifyTable = new VerifyFastLongHashMap(); @@ -256,11 +267,34 @@ public class TestVectorMapJoinFastLongHashMap extends CommonFastHashTable { // Use a large capacity that doesn't require expansion, yet. VectorMapJoinFastLongHashMap map = new VectorMapJoinFastLongHashMap( - false, false, HashTableKeyType.LONG, MODERATE_CAPACITY, LOAD_FACTOR, MODERATE_WB_SIZE); + false, false, HashTableKeyType.LONG, MODERATE_CAPACITY, LOAD_FACTOR, MODERATE_WB_SIZE, -1); VerifyFastLongHashMap verifyTable = new VerifyFastLongHashMap(); int keyCount = 1000; addAndVerifyMultipleKeyMultipleValue(keyCount, map, verifyTable); } + + /* + // Doesn't finish in a reasonable amount of time.... + @Test + public void testKeyCountLimit() throws Exception { + random = new Random(28400); + + // Use a large capacity that doesn't require expansion, yet. + VectorMapJoinFastLongHashMap map = + new VectorMapJoinFastLongHashMap( + false, false, HashTableKeyType.LONG, MODERATE_CAPACITY, LOAD_FACTOR, MODERATE_WB_SIZE, 10000000); + + VerifyFastLongHashMap verifyTable = new VerifyFastLongHashMap(); + + int keyCount = Integer.MAX_VALUE; + try { + addAndVerifyMultipleKeyMultipleValue(keyCount, map, verifyTable, 1); + } catch (RuntimeException re) { + System.out.println(re.toString()); + assertTrue(re.toString().startsWith("Vector MapJoin Long Hash Table cannot grow any more")); + } + } + */ } http://git-wip-us.apache.org/repos/asf/hive/blob/b63f7d7d/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastLongHashMultiSet.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastLongHashMultiSet.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastLongHashMultiSet.java index 9e94611..beae727 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastLongHashMultiSet.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastLongHashMultiSet.java @@ -39,7 +39,7 @@ public class TestVectorMapJoinFastLongHashMultiSet extends CommonFastHashTable { VectorMapJoinFastLongHashMultiSet map = new VectorMapJoinFastLongHashMultiSet( - false, false, HashTableKeyType.LONG, CAPACITY, LOAD_FACTOR, WB_SIZE); + false, false, HashTableKeyType.LONG, CAPACITY, LOAD_FACTOR, WB_SIZE, -1); VerifyFastLongHashMultiSet verifyTable = new VerifyFastLongHashMultiSet(); @@ -66,7 +66,7 @@ public class TestVectorMapJoinFastLongHashMultiSet extends CommonFastHashTable { VectorMapJoinFastLongHashMultiSet map = new VectorMapJoinFastLongHashMultiSet( - false, false, HashTableKeyType.LONG, CAPACITY, LOAD_FACTOR, WB_SIZE); + false, false, HashTableKeyType.LONG, CAPACITY, LOAD_FACTOR, WB_SIZE, -1); VerifyFastLongHashMultiSet verifyTable = new VerifyFastLongHashMultiSet(); @@ -94,7 +94,7 @@ public class TestVectorMapJoinFastLongHashMultiSet extends CommonFastHashTable { VectorMapJoinFastLongHashMultiSet map = new VectorMapJoinFastLongHashMultiSet( - false, false, HashTableKeyType.LONG, CAPACITY, LOAD_FACTOR, WB_SIZE); + false, false, HashTableKeyType.LONG, CAPACITY, LOAD_FACTOR, WB_SIZE, -1); VerifyFastLongHashMultiSet verifyTable = new VerifyFastLongHashMultiSet(); @@ -128,7 +128,7 @@ public class TestVectorMapJoinFastLongHashMultiSet extends CommonFastHashTable { // Make sure the map does not expand; should be able to find space. VectorMapJoinFastLongHashMultiSet map = new VectorMapJoinFastLongHashMultiSet( - false, false, HashTableKeyType.LONG, CAPACITY, 1f, WB_SIZE); + false, false, HashTableKeyType.LONG, CAPACITY, 1f, WB_SIZE, -1); VerifyFastLongHashMultiSet verifyTable = new VerifyFastLongHashMultiSet(); @@ -169,7 +169,7 @@ public class TestVectorMapJoinFastLongHashMultiSet extends CommonFastHashTable { // Start with capacity 1; make sure we expand on every put. VectorMapJoinFastLongHashMultiSet map = new VectorMapJoinFastLongHashMultiSet( - false, false, HashTableKeyType.LONG, 1, 0.0000001f, WB_SIZE); + false, false, HashTableKeyType.LONG, 1, 0.0000001f, WB_SIZE, -1); VerifyFastLongHashMultiSet verifyTable = new VerifyFastLongHashMultiSet(); @@ -227,7 +227,7 @@ public class TestVectorMapJoinFastLongHashMultiSet extends CommonFastHashTable { // Use a large capacity that doesn't require expansion, yet. VectorMapJoinFastLongHashMultiSet map = new VectorMapJoinFastLongHashMultiSet( - false, false, HashTableKeyType.LONG, LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + false, false, HashTableKeyType.LONG, LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastLongHashMultiSet verifyTable = new VerifyFastLongHashMultiSet(); @@ -242,7 +242,7 @@ public class TestVectorMapJoinFastLongHashMultiSet extends CommonFastHashTable { // Use a large capacity that doesn't require expansion, yet. VectorMapJoinFastLongHashMultiSet map = new VectorMapJoinFastLongHashMultiSet( - false, false, HashTableKeyType.LONG, MODERATE_CAPACITY, LOAD_FACTOR, MODERATE_WB_SIZE); + false, false, HashTableKeyType.LONG, MODERATE_CAPACITY, LOAD_FACTOR, MODERATE_WB_SIZE, -1); VerifyFastLongHashMultiSet verifyTable = new VerifyFastLongHashMultiSet(); http://git-wip-us.apache.org/repos/asf/hive/blob/b63f7d7d/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastLongHashSet.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastLongHashSet.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastLongHashSet.java index 698bcdc..2e77abf 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastLongHashSet.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastLongHashSet.java @@ -39,7 +39,7 @@ public class TestVectorMapJoinFastLongHashSet extends CommonFastHashTable { VectorMapJoinFastLongHashSet map = new VectorMapJoinFastLongHashSet( - false, false, HashTableKeyType.LONG, CAPACITY, LOAD_FACTOR, WB_SIZE); + false, false, HashTableKeyType.LONG, CAPACITY, LOAD_FACTOR, WB_SIZE, -1); VerifyFastLongHashSet verifyTable = new VerifyFastLongHashSet(); @@ -66,7 +66,7 @@ public class TestVectorMapJoinFastLongHashSet extends CommonFastHashTable { VectorMapJoinFastLongHashSet map = new VectorMapJoinFastLongHashSet( - false, false, HashTableKeyType.LONG, CAPACITY, LOAD_FACTOR, WB_SIZE); + false, false, HashTableKeyType.LONG, CAPACITY, LOAD_FACTOR, WB_SIZE, -1); VerifyFastLongHashSet verifyTable = new VerifyFastLongHashSet(); @@ -94,7 +94,7 @@ public class TestVectorMapJoinFastLongHashSet extends CommonFastHashTable { VectorMapJoinFastLongHashSet map = new VectorMapJoinFastLongHashSet( - false, false, HashTableKeyType.LONG, CAPACITY, LOAD_FACTOR, WB_SIZE); + false, false, HashTableKeyType.LONG, CAPACITY, LOAD_FACTOR, WB_SIZE, -1); VerifyFastLongHashSet verifyTable = new VerifyFastLongHashSet(); @@ -126,7 +126,7 @@ public class TestVectorMapJoinFastLongHashSet extends CommonFastHashTable { // Make sure the map does not expand; should be able to find space. VectorMapJoinFastLongHashSet map = new VectorMapJoinFastLongHashSet( - false, false, HashTableKeyType.LONG, CAPACITY, 1f, WB_SIZE); + false, false, HashTableKeyType.LONG, CAPACITY, 1f, WB_SIZE, -1); VerifyFastLongHashSet verifyTable = new VerifyFastLongHashSet(); @@ -167,7 +167,7 @@ public class TestVectorMapJoinFastLongHashSet extends CommonFastHashTable { // Start with capacity 1; make sure we expand on every put. VectorMapJoinFastLongHashSet map = new VectorMapJoinFastLongHashSet( - false, false, HashTableKeyType.LONG, 1, 0.0000001f, WB_SIZE); + false, false, HashTableKeyType.LONG, 1, 0.0000001f, WB_SIZE, -1); VerifyFastLongHashSet verifyTable = new VerifyFastLongHashSet(); @@ -225,7 +225,7 @@ public class TestVectorMapJoinFastLongHashSet extends CommonFastHashTable { // Use a large capacity that doesn't require expansion, yet. VectorMapJoinFastLongHashSet map = new VectorMapJoinFastLongHashSet( - false, false, HashTableKeyType.LONG, LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + false, false, HashTableKeyType.LONG, LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastLongHashSet verifyTable = new VerifyFastLongHashSet(); @@ -240,7 +240,7 @@ public class TestVectorMapJoinFastLongHashSet extends CommonFastHashTable { // Use a large capacity that doesn't require expansion, yet. VectorMapJoinFastLongHashSet map = new VectorMapJoinFastLongHashSet( - false, false, HashTableKeyType.LONG, MODERATE_CAPACITY, LOAD_FACTOR, MODERATE_WB_SIZE); + false, false, HashTableKeyType.LONG, MODERATE_CAPACITY, LOAD_FACTOR, MODERATE_WB_SIZE, -1); VerifyFastLongHashSet verifyTable = new VerifyFastLongHashSet(); http://git-wip-us.apache.org/repos/asf/hive/blob/b63f7d7d/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastRowHashMap.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastRowHashMap.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastRowHashMap.java index 3f02eb3..ebb243e 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastRowHashMap.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastRowHashMap.java @@ -147,7 +147,7 @@ public class TestVectorMapJoinFastRowHashMap extends CommonFastHashTable { VectorMapJoinFastLongHashMap map = new VectorMapJoinFastLongHashMap( false, false, HashTableKeyType.LONG, - LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastRowHashMap verifyTable = new VerifyFastRowHashMap(); @@ -171,7 +171,7 @@ public class TestVectorMapJoinFastRowHashMap extends CommonFastHashTable { VectorMapJoinFastLongHashMap map = new VectorMapJoinFastLongHashMap( false, false, HashTableKeyType.INT, - LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastRowHashMap verifyTable = new VerifyFastRowHashMap(); @@ -195,7 +195,7 @@ public class TestVectorMapJoinFastRowHashMap extends CommonFastHashTable { VectorMapJoinFastStringHashMap map = new VectorMapJoinFastStringHashMap( false, - LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastRowHashMap verifyTable = new VerifyFastRowHashMap(); @@ -219,7 +219,7 @@ public class TestVectorMapJoinFastRowHashMap extends CommonFastHashTable { VectorMapJoinFastMultiKeyHashMap map = new VectorMapJoinFastMultiKeyHashMap( false, - LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastRowHashMap verifyTable = new VerifyFastRowHashMap(); @@ -243,7 +243,7 @@ public class TestVectorMapJoinFastRowHashMap extends CommonFastHashTable { VectorMapJoinFastMultiKeyHashMap map = new VectorMapJoinFastMultiKeyHashMap( false, - LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastRowHashMap verifyTable = new VerifyFastRowHashMap(); @@ -267,7 +267,7 @@ public class TestVectorMapJoinFastRowHashMap extends CommonFastHashTable { VectorMapJoinFastMultiKeyHashMap map = new VectorMapJoinFastMultiKeyHashMap( false, - LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastRowHashMap verifyTable = new VerifyFastRowHashMap(); @@ -291,7 +291,7 @@ public class TestVectorMapJoinFastRowHashMap extends CommonFastHashTable { VectorMapJoinFastLongHashMap map = new VectorMapJoinFastLongHashMap( false, false, HashTableKeyType.LONG, - LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastRowHashMap verifyTable = new VerifyFastRowHashMap(); @@ -315,7 +315,7 @@ public class TestVectorMapJoinFastRowHashMap extends CommonFastHashTable { VectorMapJoinFastLongHashMap map = new VectorMapJoinFastLongHashMap( false, false, HashTableKeyType.INT, - LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastRowHashMap verifyTable = new VerifyFastRowHashMap(); @@ -339,7 +339,7 @@ public class TestVectorMapJoinFastRowHashMap extends CommonFastHashTable { VectorMapJoinFastStringHashMap map = new VectorMapJoinFastStringHashMap( false, - LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastRowHashMap verifyTable = new VerifyFastRowHashMap(); @@ -363,7 +363,7 @@ public class TestVectorMapJoinFastRowHashMap extends CommonFastHashTable { VectorMapJoinFastMultiKeyHashMap map = new VectorMapJoinFastMultiKeyHashMap( false, - LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastRowHashMap verifyTable = new VerifyFastRowHashMap(); @@ -387,7 +387,7 @@ public class TestVectorMapJoinFastRowHashMap extends CommonFastHashTable { VectorMapJoinFastMultiKeyHashMap map = new VectorMapJoinFastMultiKeyHashMap( false, - LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastRowHashMap verifyTable = new VerifyFastRowHashMap(); @@ -411,7 +411,7 @@ public class TestVectorMapJoinFastRowHashMap extends CommonFastHashTable { VectorMapJoinFastMultiKeyHashMap map = new VectorMapJoinFastMultiKeyHashMap( false, - LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastRowHashMap verifyTable = new VerifyFastRowHashMap(); @@ -436,7 +436,7 @@ public class TestVectorMapJoinFastRowHashMap extends CommonFastHashTable { VectorMapJoinFastLongHashMap map = new VectorMapJoinFastLongHashMap( false, false, HashTableKeyType.LONG, - LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastRowHashMap verifyTable = new VerifyFastRowHashMap(); @@ -460,7 +460,7 @@ public class TestVectorMapJoinFastRowHashMap extends CommonFastHashTable { VectorMapJoinFastLongHashMap map = new VectorMapJoinFastLongHashMap( false, false, HashTableKeyType.INT, - LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastRowHashMap verifyTable = new VerifyFastRowHashMap(); @@ -484,7 +484,7 @@ public class TestVectorMapJoinFastRowHashMap extends CommonFastHashTable { VectorMapJoinFastStringHashMap map = new VectorMapJoinFastStringHashMap( false, - LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastRowHashMap verifyTable = new VerifyFastRowHashMap(); @@ -508,7 +508,7 @@ public class TestVectorMapJoinFastRowHashMap extends CommonFastHashTable { VectorMapJoinFastMultiKeyHashMap map = new VectorMapJoinFastMultiKeyHashMap( false, - LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastRowHashMap verifyTable = new VerifyFastRowHashMap(); @@ -532,7 +532,7 @@ public class TestVectorMapJoinFastRowHashMap extends CommonFastHashTable { VectorMapJoinFastMultiKeyHashMap map = new VectorMapJoinFastMultiKeyHashMap( false, - LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastRowHashMap verifyTable = new VerifyFastRowHashMap(); @@ -556,7 +556,7 @@ public class TestVectorMapJoinFastRowHashMap extends CommonFastHashTable { VectorMapJoinFastMultiKeyHashMap map = new VectorMapJoinFastMultiKeyHashMap( false, - LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastRowHashMap verifyTable = new VerifyFastRowHashMap(); @@ -580,7 +580,7 @@ public class TestVectorMapJoinFastRowHashMap extends CommonFastHashTable { VectorMapJoinFastLongHashMap map = new VectorMapJoinFastLongHashMap( false, false, HashTableKeyType.LONG, - LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastRowHashMap verifyTable = new VerifyFastRowHashMap(); @@ -604,7 +604,7 @@ public class TestVectorMapJoinFastRowHashMap extends CommonFastHashTable { VectorMapJoinFastLongHashMap map = new VectorMapJoinFastLongHashMap( false, false, HashTableKeyType.INT, - LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastRowHashMap verifyTable = new VerifyFastRowHashMap(); @@ -628,7 +628,7 @@ public class TestVectorMapJoinFastRowHashMap extends CommonFastHashTable { VectorMapJoinFastStringHashMap map = new VectorMapJoinFastStringHashMap( false, - LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastRowHashMap verifyTable = new VerifyFastRowHashMap(); @@ -652,7 +652,7 @@ public class TestVectorMapJoinFastRowHashMap extends CommonFastHashTable { VectorMapJoinFastMultiKeyHashMap map = new VectorMapJoinFastMultiKeyHashMap( false, - LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastRowHashMap verifyTable = new VerifyFastRowHashMap(); @@ -676,7 +676,7 @@ public class TestVectorMapJoinFastRowHashMap extends CommonFastHashTable { VectorMapJoinFastMultiKeyHashMap map = new VectorMapJoinFastMultiKeyHashMap( false, - LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastRowHashMap verifyTable = new VerifyFastRowHashMap(); @@ -700,7 +700,7 @@ public class TestVectorMapJoinFastRowHashMap extends CommonFastHashTable { VectorMapJoinFastMultiKeyHashMap map = new VectorMapJoinFastMultiKeyHashMap( false, - LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastRowHashMap verifyTable = new VerifyFastRowHashMap();
