This is an automated email from the ASF dual-hosted git repository.
dschneider pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/geode.git
The following commit(s) were added to refs/heads/develop by this push:
new 1210cbc GEODE-9493: rework sizing of RedisString, RedisHash,
RedisSet, and RedisSortedSet (#6727)
1210cbc is described below
commit 1210cbc48c80df9d4f87a94cd417e424b85cbccf
Author: Darrel Schneider <dar...@vmware.com>
AuthorDate: Mon Aug 16 09:09:42 2021 -0700
GEODE-9493: rework sizing of RedisString, RedisHash, RedisSet, and
RedisSortedSet (#6727)
Removed SizeableObjectSizer and made the Sizeable redis classes abstract.
They now require subclasses that implement sizeKey, sizeValue, or
sizeElement.
Instead of having hard coded constants stating what the size of a redis
base class is,
the code now uses JvmSizeUtils.memoryOverhead(Class) to statically compute
the class size overhead.
This allows the size to be correct for different jvm configs.
Also simplified the sizing logic to use int and if needed use casts instead
of narrow.
Narrow was more expensive and would still cause problems as the size goes
back down toward zero.
The sizing of OrderStatisticsTree was simplified some more since it did not
need an extra int field
since we do not keep track of its element sizes.
Also the size of the hashing strategy object is no longer accounted for
since it is a singleton.
OrderStatisticsTree has been simplified to not size the elements it
contains.
It is currently only used by RedisSortedSet and it does not need the element
size to be computed (it already is done in the hash map it also uses).
If in the future we need to also size
the elements of an OrderStatisticsTree we can figure out the best
way to do that then in that new context.
---
.../hash/MemoryOverheadIntegrationTest.java | 2 +-
.../apache/geode/codeAnalysis/excludedClasses.txt | 3 +
.../codeAnalysis/sanctionedDataSerializables.txt | 6 +-
...orBenchmark.java => RedisHashMapBenchmark.java} | 10 +-
.../internal/collections/OrderStatisticsTree.java | 39 ++---
...leObject2ObjectOpenCustomHashMapWithCursor.java | 45 +++---
.../SizeableObjectOpenCustomHashSet.java | 36 ++---
.../geode/redis/internal/data/RedisHash.java | 47 ++++--
.../apache/geode/redis/internal/data/RedisSet.java | 39 +++--
.../geode/redis/internal/data/RedisSortedSet.java | 94 ++++++-----
.../geode/redis/internal/data/RedisString.java | 15 +-
.../redis/internal/data/SizeableObjectSizer.java | 38 -----
.../collections/OrderStatisticsTreeTest.java | 45 ++++--
.../OrderedStatisticTreeQuickCheckTest.java | 1 +
...tOpenCustomHashMapWithCursorQuickCheckTest.java | 24 ++-
...ject2ObjectOpenCustomHashMapWithCursorTest.java | 175 ++++++++-------------
.../SizeableObjectOpenCustomHashSetTest.java | 123 +--------------
.../geode/redis/internal/data/RedisHashTest.java | 24 +--
.../geode/redis/internal/data/RedisSetTest.java | 41 ++---
.../redis/internal/data/RedisSortedSetTest.java | 48 +++---
.../geode/redis/internal/data/RedisStringTest.java | 37 ++---
.../cache30/MemLRUEvictionControllerDUnitTest.java | 18 ++-
.../eviction/EvictionObjectSizerDUnitTest.java | 12 +-
.../org/apache/geode/internal/JvmSizeUtils.java | 57 +++++++
.../internal/size/ReflectionSingleObjectSizer.java | 13 +-
.../geode/internal/size/WellKnownClassSizer.java | 8 +-
26 files changed, 447 insertions(+), 553 deletions(-)
diff --git
a/geode-apis-compatible-with-redis/src/integrationTest/java/org/apache/geode/redis/internal/executor/hash/MemoryOverheadIntegrationTest.java
b/geode-apis-compatible-with-redis/src/integrationTest/java/org/apache/geode/redis/internal/executor/hash/MemoryOverheadIntegrationTest.java
index b096671..97750f3 100755
---
a/geode-apis-compatible-with-redis/src/integrationTest/java/org/apache/geode/redis/internal/executor/hash/MemoryOverheadIntegrationTest.java
+++
b/geode-apis-compatible-with-redis/src/integrationTest/java/org/apache/geode/redis/internal/executor/hash/MemoryOverheadIntegrationTest.java
@@ -58,7 +58,7 @@ public class MemoryOverheadIntegrationTest extends
AbstractMemoryOverheadIntegra
result.put(Measurement.SET_ENTRY, 25);
result.put(Measurement.HASH, 339);
result.put(Measurement.HASH_ENTRY, 50);
- result.put(Measurement.SORTED_SET, 455);
+ result.put(Measurement.SORTED_SET, 435);
result.put(Measurement.SORTED_SET_ENTRY, 126);
return result;
diff --git
a/geode-apis-compatible-with-redis/src/integrationTest/resources/org/apache/geode/codeAnalysis/excludedClasses.txt
b/geode-apis-compatible-with-redis/src/integrationTest/resources/org/apache/geode/codeAnalysis/excludedClasses.txt
index 850671e..272f2dc 100644
---
a/geode-apis-compatible-with-redis/src/integrationTest/resources/org/apache/geode/codeAnalysis/excludedClasses.txt
+++
b/geode-apis-compatible-with-redis/src/integrationTest/resources/org/apache/geode/codeAnalysis/excludedClasses.txt
@@ -5,7 +5,10 @@ org/apache/geode/redis/internal/collections/IndexibleTreeSet
org/apache/geode/redis/internal/data/RedisDataMovedException
org/apache/geode/redis/internal/data/RedisDataTypeMismatchException
org/apache/geode/redis/internal/RedisException
+org/apache/geode/redis/internal/data/RedisHash$Hash
org/apache/geode/redis/internal/data/RedisRestoreKeyExistsException
+org/apache/geode/redis/internal/data/RedisSet$MemberSet
org/apache/geode/redis/internal/data/RedisSetCommandsFunctionExecutor$SetOp
+org/apache/geode/redis/internal/data/RedisSortedSet$MemberMap
org/apache/geode/redis/internal/data/RedisStringCommandsFunctionExecutor$BitOp
org/apache/geode/redis/internal/services/StripedExecutorService$State
diff --git
a/geode-apis-compatible-with-redis/src/integrationTest/resources/org/apache/geode/codeAnalysis/sanctionedDataSerializables.txt
b/geode-apis-compatible-with-redis/src/integrationTest/resources/org/apache/geode/codeAnalysis/sanctionedDataSerializables.txt
index 8735ef5..cad8be5 100644
---
a/geode-apis-compatible-with-redis/src/integrationTest/resources/org/apache/geode/codeAnalysis/sanctionedDataSerializables.txt
+++
b/geode-apis-compatible-with-redis/src/integrationTest/resources/org/apache/geode/codeAnalysis/sanctionedDataSerializables.txt
@@ -12,7 +12,7 @@ toData,8
org/apache/geode/redis/internal/data/RedisHash,2
toData,90
-fromData,61
+fromData,58
org/apache/geode/redis/internal/data/RedisKey,2
fromData,22
@@ -20,11 +20,11 @@ toData,19
org/apache/geode/redis/internal/data/RedisSet,2
toData,55
-fromData,54
+fromData,51
org/apache/geode/redis/internal/data/RedisSortedSet,2
toData,93
-fromData,102
+fromData,86
org/apache/geode/redis/internal/data/RedisString,2
toData,23
diff --git
a/geode-apis-compatible-with-redis/src/jmh/java/org/apache/geode/redis/internal/collections/SizeableObject2ObjectOpenCustomHashmapWithCursorBenchmark.java
b/geode-apis-compatible-with-redis/src/jmh/java/org/apache/geode/redis/internal/collections/RedisHashMapBenchmark.java
similarity index 86%
rename from
geode-apis-compatible-with-redis/src/jmh/java/org/apache/geode/redis/internal/collections/SizeableObject2ObjectOpenCustomHashmapWithCursorBenchmark.java
rename to
geode-apis-compatible-with-redis/src/jmh/java/org/apache/geode/redis/internal/collections/RedisHashMapBenchmark.java
index cb3be45..8064c32 100644
---
a/geode-apis-compatible-with-redis/src/jmh/java/org/apache/geode/redis/internal/collections/SizeableObject2ObjectOpenCustomHashmapWithCursorBenchmark.java
+++
b/geode-apis-compatible-with-redis/src/jmh/java/org/apache/geode/redis/internal/collections/RedisHashMapBenchmark.java
@@ -17,7 +17,6 @@ package org.apache.geode.redis.internal.collections;
import java.util.Iterator;
import java.util.Random;
-import it.unimi.dsi.fastutil.bytes.ByteArrays;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.Param;
import org.openjdk.jmh.annotations.Scope;
@@ -25,7 +24,9 @@ import org.openjdk.jmh.annotations.Setup;
import org.openjdk.jmh.annotations.State;
import org.openjdk.jmh.infra.Blackhole;
-public class SizeableObject2ObjectOpenCustomHashmapWithCursorBenchmark {
+import org.apache.geode.redis.internal.data.RedisHash;
+
+public class RedisHashMapBenchmark {
@State(Scope.Benchmark)
public static class BenchmarkState {
@@ -33,15 +34,14 @@ public class
SizeableObject2ObjectOpenCustomHashmapWithCursorBenchmark {
private int numEntries;
@Param({"32"})
private int keySize;
- private SizeableObject2ObjectOpenCustomHashMapWithCursor<byte[], byte[]>
map;
+ private RedisHash.Hash map;
private int cursor;
private Iterator<byte[]> iterator;
@Setup
public void createMap() {
Random random = new Random(0);
- map = new SizeableObject2ObjectOpenCustomHashMapWithCursor<>(numEntries,
- ByteArrays.HASH_STRATEGY);
+ map = new RedisHash.Hash(numEntries);
for (int i = 0; i < numEntries; i++) {
byte[] key = new byte[keySize];
random.nextBytes(key);
diff --git
a/geode-apis-compatible-with-redis/src/main/java/org/apache/geode/redis/internal/collections/OrderStatisticsTree.java
b/geode-apis-compatible-with-redis/src/main/java/org/apache/geode/redis/internal/collections/OrderStatisticsTree.java
index 96c3f5e..8cad3a1 100644
---
a/geode-apis-compatible-with-redis/src/main/java/org/apache/geode/redis/internal/collections/OrderStatisticsTree.java
+++
b/geode-apis-compatible-with-redis/src/main/java/org/apache/geode/redis/internal/collections/OrderStatisticsTree.java
@@ -26,6 +26,7 @@
package org.apache.geode.redis.internal.collections;
+import static org.apache.geode.internal.JvmSizeUtils.memoryOverhead;
import static org.apache.geode.util.internal.UncheckedUtils.uncheckedCast;
import java.util.Arrays;
@@ -38,7 +39,6 @@ import java.util.Objects;
import java.util.Set;
import org.apache.geode.annotations.VisibleForTesting;
-import org.apache.geode.redis.internal.data.SizeableObjectSizer;
/**
* This class implements an order statistic tree which is based on AVL-trees.
@@ -47,19 +47,17 @@ import
org.apache.geode.redis.internal.data.SizeableObjectSizer;
* @author Rodion "rodde" Efremov
* @version 1.6 (Feb 11, 2016)
*/
-public class OrderStatisticsTree<E extends Comparable<? super E>> implements
OrderStatisticsSet<E> {
-
- // The following constants were calculated using reflection. You can find
the tests for these
- // values in OrderStatisticsTreeTest, which shows the way these numbers were
calculated. If our
- // internal implementation changes, these values may be incorrect. An
increase in overhead should
- // be carefully considered.
- public static final int ORDER_STATISTICS_TREE_BASE_SIZE = 32;
- public static final int PER_ENTRY_OVERHEAD = 40;
+public class OrderStatisticsTree<E extends Comparable<? super E>>
+ implements OrderStatisticsSet<E> {
+
+ private static final int ORDER_STATISTICS_TREE_OVERHEAD =
+ memoryOverhead(OrderStatisticsTree.class);
+ @VisibleForTesting
+ static final int NODE_OVERHEAD = memoryOverhead(Node.class);
+
private Node<E> root;
private int size;
private int modCount;
- private int sizeInBytes = ORDER_STATISTICS_TREE_BASE_SIZE;
- private static final SizeableObjectSizer elementSizer = new
SizeableObjectSizer();
@Override
public Iterator<E> iterator() {
@@ -162,7 +160,6 @@ public class OrderStatisticsTree<E extends Comparable<?
super E>> implements Ord
root = new Node<>(element);
size = 1;
modCount++;
- incrementSize(element);
return true;
}
@@ -198,7 +195,6 @@ public class OrderStatisticsTree<E extends Comparable<?
super E>> implements Ord
newNode.parent = parent;
size++;
modCount++;
- incrementSize(element);
Node<E> hi = parent;
Node<E> lo = newNode;
@@ -269,7 +265,6 @@ public class OrderStatisticsTree<E extends Comparable<?
super E>> implements Ord
}
x = deleteNode(x);
- decrementSize(x.key);
fixAfterModification(x, false);
size--;
modCount++;
@@ -743,17 +738,14 @@ public class OrderStatisticsTree<E extends Comparable<?
super E>> implements Ord
return leftTreeSize + 1 + rightTreeSize;
}
- void incrementSize(E element) {
- sizeInBytes += elementSizer.sizeof(element) + PER_ENTRY_OVERHEAD;
- }
-
- void decrementSize(E element) {
- sizeInBytes -= elementSizer.sizeof(element) + PER_ENTRY_OVERHEAD;
- }
-
+ /**
+ * Returns the amount of memory used by this instance and its Node instances.
+ * Note that this does not include the memory used by the elements referenced
+ * by the nodes.
+ */
@Override
public int getSizeInBytes() {
- return sizeInBytes;
+ return ORDER_STATISTICS_TREE_OVERHEAD + (size * NODE_OVERHEAD);
}
private static final class Node<T> {
@@ -825,7 +817,6 @@ public class OrderStatisticsTree<E extends Comparable<?
super E>> implements Ord
checkConcurrentModification();
Node<E> x = deleteNode(previousNode);
- decrementSize(x.key);
fixAfterModification(x, false);
if (x == nextNode) {
diff --git
a/geode-apis-compatible-with-redis/src/main/java/org/apache/geode/redis/internal/collections/SizeableObject2ObjectOpenCustomHashMapWithCursor.java
b/geode-apis-compatible-with-redis/src/main/java/org/apache/geode/redis/internal/collections/SizeableObject2ObjectOpenCustomHashMapWithCursor.java
index b2239dd..dc552c5 100644
---
a/geode-apis-compatible-with-redis/src/main/java/org/apache/geode/redis/internal/collections/SizeableObject2ObjectOpenCustomHashMapWithCursor.java
+++
b/geode-apis-compatible-with-redis/src/main/java/org/apache/geode/redis/internal/collections/SizeableObject2ObjectOpenCustomHashMapWithCursor.java
@@ -15,31 +15,29 @@
package org.apache.geode.redis.internal.collections;
import static it.unimi.dsi.fastutil.HashCommon.mix;
+import static org.apache.geode.internal.JvmSizeUtils.memoryOverhead;
import java.util.Map;
import it.unimi.dsi.fastutil.objects.Object2ObjectMap;
import it.unimi.dsi.fastutil.objects.Object2ObjectOpenCustomHashMap;
-import org.apache.geode.annotations.VisibleForTesting;
import org.apache.geode.internal.size.Sizeable;
-import org.apache.geode.redis.internal.data.SizeableObjectSizer;
/**
- * An extention of {@link Object2ObjectOpenCustomHashMap} that supports
+ * An extension of {@link Object2ObjectOpenCustomHashMap} that supports
* a method of iteration where each scan operation returns an integer cursor
* that allows future scan operations to start from that same point.
*
* The scan method provides the same guarantees as Redis's HSCAN, and in fact
* uses the same algorithm.
*/
-public class SizeableObject2ObjectOpenCustomHashMapWithCursor<K, V>
+public abstract class SizeableObject2ObjectOpenCustomHashMapWithCursor<K, V>
extends Object2ObjectOpenCustomHashMap<K, V> implements Sizeable {
private static final long serialVersionUID = 9079713776660851891L;
- public static final int BACKING_ARRAY_OVERHEAD_CONSTANT = 128;
- public static final int BACKING_ARRAY_LENGTH_COEFFICIENT = 4;
- private static final SizeableObjectSizer elementSizer = new
SizeableObjectSizer();
+ public static final int OPEN_HASH_MAP_OVERHEAD =
+ memoryOverhead(SizeableObject2ObjectOpenCustomHashMapWithCursor.class);
private int arrayContentsOverhead;
@@ -196,46 +194,39 @@ public class
SizeableObject2ObjectOpenCustomHashMapWithCursor<K, V>
V oldValue = super.put(k, v);
if (oldValue == null) {
// A create
- arrayContentsOverhead += elementSizer.sizeof(k) + elementSizer.sizeof(v);
+ arrayContentsOverhead += sizeKey(k) + sizeValue(v);
} else {
// An update
- arrayContentsOverhead += elementSizer.sizeof(v) -
elementSizer.sizeof(oldValue);
+ arrayContentsOverhead += sizeValue(v) - sizeValue(oldValue);
}
return oldValue;
}
@Override
+ @SuppressWarnings("unchecked")
public V remove(Object k) {
V oldValue = super.remove(k);
if (oldValue != null) {
- arrayContentsOverhead -= elementSizer.sizeof(k) +
elementSizer.sizeof(oldValue);
+ arrayContentsOverhead -= sizeKey((K) k) + sizeValue(oldValue);
}
return oldValue;
}
@Override
public int getSizeInBytes() {
- return arrayContentsOverhead + calculateBackingArraysOverhead();
- }
-
- @VisibleForTesting
- int calculateBackingArraysOverhead() {
- // This formula determined experimentally using tests.
- return BACKING_ARRAY_OVERHEAD_CONSTANT
- + BACKING_ARRAY_LENGTH_COEFFICIENT * getTotalBackingArrayLength();
- }
-
- @VisibleForTesting
- int getArrayContentsOverhead() {
- return arrayContentsOverhead;
- }
+ // The size of the object referenced by the "strategy" field is not
included
+ // here because in most cases it is a static singleton.
- @VisibleForTesting
- int getTotalBackingArrayLength() {
- return key.length + value.length;
+ return OPEN_HASH_MAP_OVERHEAD + memoryOverhead(key) + memoryOverhead(value)
+ + arrayContentsOverhead;
}
public interface EntryConsumer<K, V, D> {
void consume(D privateData, K key, V value);
}
+
+ protected abstract int sizeKey(K key);
+
+ protected abstract int sizeValue(V value);
+
}
diff --git
a/geode-apis-compatible-with-redis/src/main/java/org/apache/geode/redis/internal/collections/SizeableObjectOpenCustomHashSet.java
b/geode-apis-compatible-with-redis/src/main/java/org/apache/geode/redis/internal/collections/SizeableObjectOpenCustomHashSet.java
index 4b64caf..aa526ba 100644
---
a/geode-apis-compatible-with-redis/src/main/java/org/apache/geode/redis/internal/collections/SizeableObjectOpenCustomHashSet.java
+++
b/geode-apis-compatible-with-redis/src/main/java/org/apache/geode/redis/internal/collections/SizeableObjectOpenCustomHashSet.java
@@ -15,22 +15,21 @@
package org.apache.geode.redis.internal.collections;
+import static org.apache.geode.internal.JvmSizeUtils.memoryOverhead;
+
import java.util.Collection;
import java.util.Iterator;
import it.unimi.dsi.fastutil.objects.ObjectCollection;
import it.unimi.dsi.fastutil.objects.ObjectOpenCustomHashSet;
-import org.apache.geode.annotations.VisibleForTesting;
import org.apache.geode.internal.size.Sizeable;
-import org.apache.geode.redis.internal.data.SizeableObjectSizer;
-public class SizeableObjectOpenCustomHashSet<K> extends
ObjectOpenCustomHashSet<K>
+public abstract class SizeableObjectOpenCustomHashSet<K> extends
ObjectOpenCustomHashSet<K>
implements Sizeable {
private static final long serialVersionUID = 9174920505089089517L;
- public static final int BACKING_ARRAY_OVERHEAD_CONSTANT = 92;
- public static final int BACKING_ARRAY_LENGTH_COEFFICIENT = 4;
- private static final SizeableObjectSizer elementSizer = new
SizeableObjectSizer();
+ private static final int OPEN_HASH_SET_OVERHEAD =
+ memoryOverhead(SizeableObjectOpenCustomHashSet.class);
private int memberOverhead;
@@ -96,38 +95,27 @@ public class SizeableObjectOpenCustomHashSet<K> extends
ObjectOpenCustomHashSet<
public boolean add(K k) {
boolean added = super.add(k);
if (added) {
- memberOverhead += elementSizer.sizeof(k);
+ memberOverhead += sizeElement(k);
}
return added;
}
+ @SuppressWarnings("unchecked")
@Override
public boolean remove(Object k) {
boolean removed = super.remove(k);
if (removed) {
- memberOverhead -= elementSizer.sizeof(k);
+ memberOverhead -= sizeElement((K) k);
}
return removed;
}
@Override
public int getSizeInBytes() {
- return memberOverhead + calculateBackingArrayOverhead();
- }
-
- @VisibleForTesting
- int getMemberOverhead() {
- return memberOverhead;
+ // The object referenced by the "strategy" field is not sized
+ // since it is usually a singleton instance.
+ return OPEN_HASH_SET_OVERHEAD + memoryOverhead(key) + memberOverhead;
}
- @VisibleForTesting
- int getBackingArrayLength() {
- return key.length;
- }
-
- @VisibleForTesting
- int calculateBackingArrayOverhead() {
- // This formula determined experimentally using tests
- return BACKING_ARRAY_OVERHEAD_CONSTANT + (BACKING_ARRAY_LENGTH_COEFFICIENT
* key.length);
- }
+ protected abstract int sizeElement(K element);
}
diff --git
a/geode-apis-compatible-with-redis/src/main/java/org/apache/geode/redis/internal/data/RedisHash.java
b/geode-apis-compatible-with-redis/src/main/java/org/apache/geode/redis/internal/data/RedisHash.java
index d685205..5969d3a 100644
---
a/geode-apis-compatible-with-redis/src/main/java/org/apache/geode/redis/internal/data/RedisHash.java
+++
b/geode-apis-compatible-with-redis/src/main/java/org/apache/geode/redis/internal/data/RedisHash.java
@@ -16,6 +16,8 @@
package org.apache.geode.redis.internal.data;
+import static it.unimi.dsi.fastutil.bytes.ByteArrays.HASH_STRATEGY;
+import static org.apache.geode.internal.JvmSizeUtils.memoryOverhead;
import static org.apache.geode.redis.internal.RedisConstants.ERROR_NOT_INTEGER;
import static org.apache.geode.redis.internal.RedisConstants.ERROR_OVERFLOW;
import static org.apache.geode.redis.internal.netty.Coder.bytesToLong;
@@ -34,7 +36,6 @@ import java.util.Map;
import java.util.Objects;
import java.util.regex.Pattern;
-import it.unimi.dsi.fastutil.bytes.ByteArrays;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.geode.DataSerializer;
@@ -51,12 +52,9 @@ import org.apache.geode.redis.internal.delta.RemsDeltaInfo;
import org.apache.geode.redis.internal.netty.Coder;
public class RedisHash extends AbstractRedisData {
- // The following constant was calculated using reflection. you can find the
test for this value in
- // RedisHashTest, which shows the way this number was calculated. If our
internal implementation
- // changes, these values may be incorrect. An increase in overhead should be
carefully considered.
- protected static final int BASE_REDIS_HASH_OVERHEAD = 32;
+ protected static final int REDIS_HASH_OVERHEAD =
memoryOverhead(RedisHash.class);
- private SizeableObject2ObjectOpenCustomHashMapWithCursor<byte[], byte[]>
hash;
+ private Hash hash;
@VisibleForTesting
public RedisHash(List<byte[]> fieldsToSet) {
@@ -66,8 +64,7 @@ public class RedisHash extends AbstractRedisData {
"fieldsToSet should have an even number of elements but was size " +
numKeysAndValues);
}
- hash = new
SizeableObject2ObjectOpenCustomHashMapWithCursor<>(numKeysAndValues / 2,
- ByteArrays.HASH_STRATEGY);
+ hash = new Hash(numKeysAndValues / 2);
Iterator<byte[]> iterator = fieldsToSet.iterator();
while (iterator.hasNext()) {
hashPut(iterator.next(), iterator.next());
@@ -101,7 +98,7 @@ public class RedisHash extends AbstractRedisData {
throws IOException, ClassNotFoundException {
super.fromData(in, context);
int size = DataSerializer.readInteger(in);
- hash = new SizeableObject2ObjectOpenCustomHashMapWithCursor<>(size,
ByteArrays.HASH_STRATEGY);
+ hash = new Hash(size);
for (int i = 0; i < size; i++) {
hash.put(DataSerializer.readByteArray(in),
DataSerializer.readByteArray(in));
}
@@ -392,7 +389,37 @@ public class RedisHash extends AbstractRedisData {
@Override
public int getSizeInBytes() {
- return BASE_REDIS_HASH_OVERHEAD + hash.getSizeInBytes();
+ return REDIS_HASH_OVERHEAD + hash.getSizeInBytes();
+ }
+
+ public static class Hash
+ extends SizeableObject2ObjectOpenCustomHashMapWithCursor<byte[], byte[]>
{
+
+ public Hash() {
+ super(HASH_STRATEGY);
+ }
+
+ public Hash(int expected) {
+ super(expected, HASH_STRATEGY);
+ }
+
+ public Hash(int expected, float f) {
+ super(expected, f, HASH_STRATEGY);
+ }
+
+ public Hash(Map<byte[], byte[]> m) {
+ super(m, HASH_STRATEGY);
+ }
+
+ @Override
+ protected int sizeKey(byte[] key) {
+ return memoryOverhead(key);
+ }
+
+ @Override
+ protected int sizeValue(byte[] value) {
+ return sizeKey(value);
+ }
}
}
diff --git
a/geode-apis-compatible-with-redis/src/main/java/org/apache/geode/redis/internal/data/RedisSet.java
b/geode-apis-compatible-with-redis/src/main/java/org/apache/geode/redis/internal/data/RedisSet.java
index 5c2e18c..08bf977 100644
---
a/geode-apis-compatible-with-redis/src/main/java/org/apache/geode/redis/internal/data/RedisSet.java
+++
b/geode-apis-compatible-with-redis/src/main/java/org/apache/geode/redis/internal/data/RedisSet.java
@@ -17,6 +17,7 @@
package org.apache.geode.redis.internal.data;
import static java.util.Collections.emptyList;
+import static org.apache.geode.internal.JvmSizeUtils.memoryOverhead;
import static org.apache.geode.redis.internal.data.RedisDataType.REDIS_SET;
import static org.apache.geode.redis.internal.netty.Coder.bytesToString;
@@ -50,16 +51,12 @@ import org.apache.geode.redis.internal.delta.DeltaInfo;
import org.apache.geode.redis.internal.delta.RemsDeltaInfo;
public class RedisSet extends AbstractRedisData {
- private SizeableObjectOpenCustomHashSet<byte[]> members;
+ protected static final int REDIS_SET_OVERHEAD =
memoryOverhead(RedisSet.class);
- // The following constant was calculated using reflection. you can find the
test for this value in
- // RedisSetTest, which shows the way this number was calculated. If our
internal implementation
- // changes, these values may be incorrect. An increase in overhead should be
carefully considered.
- protected static final int BASE_REDIS_SET_OVERHEAD = 32;
+ private MemberSet members;
RedisSet(Collection<byte[]> members) {
- this.members =
- new SizeableObjectOpenCustomHashSet<>(members.size(),
ByteArrays.HASH_STRATEGY);
+ this.members = new MemberSet(members.size());
for (byte[] member : members) {
membersAdd(member);
}
@@ -218,7 +215,7 @@ public class RedisSet extends AbstractRedisData {
throws IOException, ClassNotFoundException {
super.fromData(in, context);
int size = DataSerializer.readPrimitiveInt(in);
- members = new SizeableObjectOpenCustomHashSet<>(size,
ByteArrays.HASH_STRATEGY);
+ members = new MemberSet(size);
for (int i = 0; i < size; ++i) {
members.add(DataSerializer.readByteArray(in));
}
@@ -332,6 +329,30 @@ public class RedisSet extends AbstractRedisData {
@Override
public int getSizeInBytes() {
- return BASE_REDIS_SET_OVERHEAD + members.getSizeInBytes();
+ return REDIS_SET_OVERHEAD + members.getSizeInBytes();
}
+
+ public static class MemberSet extends
SizeableObjectOpenCustomHashSet<byte[]> {
+ public MemberSet() {
+ super(ByteArrays.HASH_STRATEGY);
+ }
+
+ public MemberSet(int size) {
+ super(size, ByteArrays.HASH_STRATEGY);
+ }
+
+ public MemberSet(int expected, float f) {
+ super(expected, f, ByteArrays.HASH_STRATEGY);
+ }
+
+ public MemberSet(Collection<byte[]> initialElements) {
+ super(initialElements, ByteArrays.HASH_STRATEGY);
+ }
+
+ @Override
+ protected int sizeElement(byte[] element) {
+ return memoryOverhead(element);
+ }
+ }
+
}
diff --git
a/geode-apis-compatible-with-redis/src/main/java/org/apache/geode/redis/internal/data/RedisSortedSet.java
b/geode-apis-compatible-with-redis/src/main/java/org/apache/geode/redis/internal/data/RedisSortedSet.java
index 0a6f2dd..1299be4 100644
---
a/geode-apis-compatible-with-redis/src/main/java/org/apache/geode/redis/internal/data/RedisSortedSet.java
+++
b/geode-apis-compatible-with-redis/src/main/java/org/apache/geode/redis/internal/data/RedisSortedSet.java
@@ -16,12 +16,12 @@
package org.apache.geode.redis.internal.data;
-import static
org.apache.geode.internal.size.ReflectionSingleObjectSizer.roundUpSize;
+import static it.unimi.dsi.fastutil.bytes.ByteArrays.HASH_STRATEGY;
+import static org.apache.geode.internal.JvmSizeUtils.memoryOverhead;
import static
org.apache.geode.redis.internal.RedisConstants.ERROR_NOT_A_VALID_FLOAT;
import static
org.apache.geode.redis.internal.data.RedisDataType.REDIS_SORTED_SET;
import static org.apache.geode.redis.internal.netty.Coder.bytesToDouble;
import static org.apache.geode.redis.internal.netty.Coder.doubleToBytes;
-import static org.apache.geode.redis.internal.netty.Coder.narrowLongToInt;
import static
org.apache.geode.redis.internal.netty.Coder.stripTrailingZeroFromDouble;
import static
org.apache.geode.redis.internal.netty.StringBytesGlossary.bGREATEST_MEMBER_NAME;
import static
org.apache.geode.redis.internal.netty.StringBytesGlossary.bLEAST_MEMBER_NAME;
@@ -37,8 +37,6 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
-import it.unimi.dsi.fastutil.bytes.ByteArrays;
-
import org.apache.geode.annotations.VisibleForTesting;
import org.apache.geode.cache.Region;
import org.apache.geode.internal.InternalDataSerializer;
@@ -47,7 +45,6 @@ import org.apache.geode.internal.serialization.KnownVersion;
import org.apache.geode.internal.serialization.SerializationContext;
import org.apache.geode.internal.size.Sizeable;
import org.apache.geode.redis.internal.RedisConstants;
-import org.apache.geode.redis.internal.collections.OrderStatisticsSet;
import org.apache.geode.redis.internal.collections.OrderStatisticsTree;
import
org.apache.geode.redis.internal.collections.SizeableObject2ObjectOpenCustomHashMapWithCursor;
import org.apache.geode.redis.internal.delta.AddsDeltaInfo;
@@ -59,29 +56,18 @@ import
org.apache.geode.redis.internal.executor.sortedset.SortedSetScoreRangeOpt
import org.apache.geode.redis.internal.executor.sortedset.ZAddOptions;
public class RedisSortedSet extends AbstractRedisData {
- private SizeableObject2ObjectOpenCustomHashMapWithCursor<byte[],
OrderedSetEntry> members;
- private OrderStatisticsSet<AbstractOrderedSetEntry> scoreSet;
- // This field is used to keep track of the size associated with objects that
are referenced by
- // both backing collections, since they will be counted twice otherwise
- private int sizeInBytesAdjustment = 0;
-
- // The following constant was calculated using reflection. You can find the
test for this value in
- // RedisSortedSetTest, which shows the way this number was calculated. If
our internal
- // implementation changes, this value may be incorrect. An increase in
overhead should be
- // carefully considered.
- protected static final int BASE_REDIS_SORTED_SET_OVERHEAD = 40;
+ protected static final int REDIS_SORTED_SET_OVERHEAD =
memoryOverhead(RedisSortedSet.class);
+
+ private MemberMap members;
+ private final ScoreSet scoreSet = new ScoreSet();
@Override
public int getSizeInBytes() {
- return BASE_REDIS_SORTED_SET_OVERHEAD + members.getSizeInBytes() +
scoreSet.getSizeInBytes()
- - sizeInBytesAdjustment;
+ return REDIS_SORTED_SET_OVERHEAD + members.getSizeInBytes() +
scoreSet.getSizeInBytes();
}
RedisSortedSet(List<byte[]> members) {
- this.members =
- new SizeableObject2ObjectOpenCustomHashMapWithCursor<>(members.size()
/ 2,
- ByteArrays.HASH_STRATEGY);
- scoreSet = new OrderStatisticsTree<>();
+ this.members = new MemberMap(members.size() / 2);
Iterator<byte[]> iterator = members.iterator();
@@ -133,9 +119,7 @@ public class RedisSortedSet extends AbstractRedisData {
throws IOException, ClassNotFoundException {
super.fromData(in, context);
int size = InternalDataSerializer.readPrimitiveInt(in);
- members =
- new SizeableObject2ObjectOpenCustomHashMapWithCursor<>(size,
ByteArrays.HASH_STRATEGY);
- scoreSet = new OrderStatisticsTree<>();
+ members = new MemberMap(size);
for (int i = 0; i < size; i++) {
byte[] member = InternalDataSerializer.readByteArray(in);
byte[] score = InternalDataSerializer.readByteArray(in);
@@ -184,9 +168,6 @@ public class RedisSortedSet extends AbstractRedisData {
OrderedSetEntry newEntry = new OrderedSetEntry(memberToAdd, scoreToAdd);
members.put(memberToAdd, newEntry);
scoreSet.add(newEntry);
- // Without this adjustment, we count the entry and member name array
twice, since references
- // to them appear in both backing collections.
- sizeInBytesAdjustment += newEntry.getSizeInBytes() +
calculateByteArraySize(memberToAdd);
return null;
} else {
scoreSet.remove(existingEntry);
@@ -204,17 +185,11 @@ public class RedisSortedSet extends AbstractRedisData {
if (orderedSetEntry != null) {
scoreSet.remove(orderedSetEntry);
oldValue = orderedSetEntry.getScoreBytes();
- // Adjust for removal.
- adjustSizeForRemoval(orderedSetEntry);
}
return oldValue;
}
- private void adjustSizeForRemoval(AbstractOrderedSetEntry entry) {
- sizeInBytesAdjustment -= entry.getSizeInBytes() +
calculateByteArraySize(entry.getMember());
- }
-
private synchronized void membersAddAll(AddsDeltaInfo addsDeltaInfo) {
Iterator<byte[]> iterator = addsDeltaInfo.getAdds().iterator();
while (iterator.hasNext()) {
@@ -427,7 +402,6 @@ public class RedisSortedSet extends AbstractRedisData {
AbstractOrderedSetEntry entry = scoresIterator.next();
scoresIterator.remove();
members.remove(entry.member);
- adjustSizeForRemoval(entry);
result.add(entry.member);
result.add(entry.scoreBytes);
@@ -635,11 +609,8 @@ public class RedisSortedSet extends AbstractRedisData {
// with same name...
}
- private static int calculateByteArraySize(byte[] bytes) {
- return narrowLongToInt(roundUpSize(bytes.length) + 16);
- }
-
- abstract static class AbstractOrderedSetEntry implements
Comparable<AbstractOrderedSetEntry>,
+ public abstract static class AbstractOrderedSetEntry
+ implements Comparable<AbstractOrderedSetEntry>,
Sizeable {
byte[] member;
byte[] scoreBytes;
@@ -676,11 +647,7 @@ public class RedisSortedSet extends AbstractRedisData {
// Entry used to store data in the scoreSet
public static class OrderedSetEntry extends AbstractOrderedSetEntry {
- // The following constant was calculated using reflection. You can find
the test for this value
- // in RedisSortedSetTest, which shows the way this number was calculated.
If our internal
- // implementation changes, this value may be incorrect. An increase in
overhead should be
- // carefully considered.
- public static final int BASE_ORDERED_SET_ENTRY_SIZE = 32;
+ public static final int ORDERED_SET_ENTRY_OVERHEAD =
memoryOverhead(OrderedSetEntry.class);
public OrderedSetEntry(byte[] member, byte[] score) {
this.member = member;
@@ -695,11 +662,12 @@ public class RedisSortedSet extends AbstractRedisData {
@Override
public int getSizeInBytes() {
- return BASE_ORDERED_SET_ENTRY_SIZE + calculateByteArraySize(member)
- + calculateByteArraySize(scoreBytes);
+ // don't include the member size since it is accounted
+ // for as the key on the Hash.
+ return ORDERED_SET_ENTRY_OVERHEAD + memoryOverhead(scoreBytes);
}
- void updateScore(byte[] newScore) {
+ public void updateScore(byte[] newScore) {
if (!Arrays.equals(newScore, scoreBytes)) {
scoreBytes = newScore;
score = processByteArrayAsDouble(newScore);
@@ -770,4 +738,34 @@ public class RedisSortedSet extends AbstractRedisData {
return 0;
}
}
+
+ public static class MemberMap
+ extends SizeableObject2ObjectOpenCustomHashMapWithCursor<byte[],
OrderedSetEntry> {
+
+ public MemberMap(int size) {
+ super(size, HASH_STRATEGY);
+ }
+
+ public MemberMap(Map<byte[], RedisSortedSet.OrderedSetEntry>
initialElements) {
+ super(initialElements, HASH_STRATEGY);
+ }
+
+ @Override
+ protected int sizeKey(byte[] key) {
+ return memoryOverhead(key);
+ }
+
+ @Override
+ protected int sizeValue(OrderedSetEntry value) {
+ return value.getSizeInBytes();
+ }
+ }
+
+ /**
+ * ScoreSet does not keep track of the size of the element instances since
they
+ * are already accounted for by the MemberMap.
+ */
+ public static class ScoreSet extends
OrderStatisticsTree<AbstractOrderedSetEntry> {
+ }
+
}
diff --git
a/geode-apis-compatible-with-redis/src/main/java/org/apache/geode/redis/internal/data/RedisString.java
b/geode-apis-compatible-with-redis/src/main/java/org/apache/geode/redis/internal/data/RedisString.java
index 6de5e45..5012426 100644
---
a/geode-apis-compatible-with-redis/src/main/java/org/apache/geode/redis/internal/data/RedisString.java
+++
b/geode-apis-compatible-with-redis/src/main/java/org/apache/geode/redis/internal/data/RedisString.java
@@ -16,6 +16,7 @@
package org.apache.geode.redis.internal.data;
+import static org.apache.geode.internal.JvmSizeUtils.memoryOverhead;
import static org.apache.geode.redis.internal.netty.Coder.bytesToString;
import java.io.DataInput;
@@ -37,18 +38,14 @@ import
org.apache.geode.redis.internal.executor.string.SetOptions;
import org.apache.geode.redis.internal.netty.Coder;
public class RedisString extends AbstractRedisData {
- private int appendSequence;
-
- private byte[] value;
-
- // this value is empirically derived using ReflectionObjectSizer, which
provides an exact size
- // of the object. It can't be used directly because of its performance
impact. This value causes
- // the size we keep track of to converge to the actual size as it increases.
- protected static final int BASE_REDIS_STRING_OVERHEAD = 48;
+ private static final int REDIS_STRING_OVERHEAD =
memoryOverhead(RedisString.class);
// An array containing the number of set bits for each value from 0x00 to
0xff
private static final byte[] bitCountTable = getBitCountTable();
+ private int appendSequence;
+ private byte[] value;
+
public RedisString(byte[] value) {
this.value = value;
}
@@ -475,7 +472,7 @@ public class RedisString extends AbstractRedisData {
@Override
public int getSizeInBytes() {
- return BASE_REDIS_STRING_OVERHEAD + value.length;
+ return REDIS_STRING_OVERHEAD + memoryOverhead(value);
}
private static byte[] getBitCountTable() {
diff --git
a/geode-apis-compatible-with-redis/src/main/java/org/apache/geode/redis/internal/data/SizeableObjectSizer.java
b/geode-apis-compatible-with-redis/src/main/java/org/apache/geode/redis/internal/data/SizeableObjectSizer.java
deleted file mode 100644
index 9243936..0000000
---
a/geode-apis-compatible-with-redis/src/main/java/org/apache/geode/redis/internal/data/SizeableObjectSizer.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
contributor license
- * agreements. See the NOTICE file distributed with this work for additional
information regarding
- * copyright ownership. The ASF licenses this file to You under the Apache
License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the
License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express
- * or implied. See the License for the specific language governing permissions
and limitations under
- * the License.
- */
-package org.apache.geode.redis.internal.data;
-
-import static org.apache.geode.redis.internal.netty.Coder.narrowLongToInt;
-
-import org.apache.geode.internal.size.ReflectionSingleObjectSizer;
-import org.apache.geode.internal.size.SingleObjectSizer;
-import org.apache.geode.internal.size.Sizeable;
-
-/**
- * A sizer that allows for efficient sizing of objects that implement the
{@link Sizeable}
- * interface, and delegates to {@link ReflectionSingleObjectSizer} otherwise.
- */
-public class SizeableObjectSizer {
-
- private final SingleObjectSizer sizer = new ReflectionSingleObjectSizer();
-
- public int sizeof(Object object) {
- if (object instanceof Sizeable) {
- return ((Sizeable) object).getSizeInBytes();
- } else {
- return narrowLongToInt(sizer.sizeof(object));
- }
- }
-}
diff --git
a/geode-apis-compatible-with-redis/src/test/java/org/apache/geode/redis/internal/collections/OrderStatisticsTreeTest.java
b/geode-apis-compatible-with-redis/src/test/java/org/apache/geode/redis/internal/collections/OrderStatisticsTreeTest.java
index 04616ec..9079314 100644
---
a/geode-apis-compatible-with-redis/src/test/java/org/apache/geode/redis/internal/collections/OrderStatisticsTreeTest.java
+++
b/geode-apis-compatible-with-redis/src/test/java/org/apache/geode/redis/internal/collections/OrderStatisticsTreeTest.java
@@ -25,8 +25,7 @@
*/
package org.apache.geode.redis.internal.collections;
-import static
org.apache.geode.redis.internal.collections.OrderStatisticsTree.ORDER_STATISTICS_TREE_BASE_SIZE;
-import static
org.apache.geode.redis.internal.collections.OrderStatisticsTree.PER_ENTRY_OVERHEAD;
+import static
org.apache.geode.redis.internal.collections.OrderStatisticsTree.NODE_OVERHEAD;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
@@ -674,34 +673,41 @@ public class OrderStatisticsTreeTest {
@Test
public void getSizeInBytes_isAccurate_forAdds() {
+ int elementSize = 0;
for (int i = 0; i < 1000; ++i) {
tree.add(i);
- assertThat(tree.getSizeInBytes()).isEqualTo(sizer.sizeof(tree));
+ elementSize += sizer.sizeof(i);
+ assertThat(tree.getSizeInBytes()).isEqualTo(sizer.sizeof(tree) -
elementSize);
}
}
@Test
public void getSizeInBytes_isAccurate_forRemoves() {
int entries = 1000;
+ int elementSize = 0;
for (int i = 0; i < entries; ++i) {
tree.add(i);
+ elementSize += sizer.sizeof(i);
}
for (int i = 0; i < entries; ++i) {
tree.remove(i);
- assertThat(tree.getSizeInBytes()).isEqualTo(sizer.sizeof(tree));
+ elementSize -= sizer.sizeof(i);
+ assertThat(tree.getSizeInBytes()).isEqualTo(sizer.sizeof(tree) -
elementSize);
}
}
@Test
public void getSizeInBytes_isAccurate_forIteratorRemoves() {
+ int elementSize = 0;
for (int i = 0; i < 1000; ++i) {
tree.add(i);
+ elementSize += sizer.sizeof(i);
}
Iterator<Integer> iterator = tree.iterator();
while (iterator.hasNext()) {
- iterator.next();
+ elementSize -= sizer.sizeof(iterator.next());
iterator.remove();
- assertThat(tree.getSizeInBytes()).isEqualTo(sizer.sizeof(tree));
+ assertThat(tree.getSizeInBytes()).isEqualTo(sizer.sizeof(tree) -
elementSize);
}
}
@@ -711,29 +717,32 @@ public class OrderStatisticsTreeTest {
byte[] member;
byte[] scoreBytes;
List<RedisSortedSet.OrderedSetEntry> entries = new ArrayList<>();
- OrderStatisticsSet<RedisSortedSet.OrderedSetEntry> tree = new
OrderStatisticsTree<>();
+ RedisSortedSet.ScoreSet tree = new RedisSortedSet.ScoreSet();
assertThat(tree.getSizeInBytes()).isEqualTo(sizer.sizeof(tree));
// Populate the tree set with randomly-sized OrderedSetEntry instances and
verify that the
// reported size is accurate after each addition
+ int sizeOfEntries = 0;
for (int i = 0; i < 100; ++i) {
member = new byte[random.nextInt(50_000)];
scoreBytes = String.valueOf(random.nextDouble()).getBytes();
RedisSortedSet.OrderedSetEntry entry = new
RedisSortedSet.OrderedSetEntry(member, scoreBytes);
+ sizeOfEntries += sizer.sizeof(entry);
entries.add(entry);
tree.add(entry);
- assertThat(tree.getSizeInBytes()).isEqualTo(sizer.sizeof(tree));
+ assertThat(tree.getSizeInBytes()).isEqualTo(sizer.sizeof(tree) -
sizeOfEntries);
}
// Remove half the entries using an iterator and verify that the reported
size is accurate after
// each removal
- Iterator<RedisSortedSet.OrderedSetEntry> iterator =
+ Iterator<RedisSortedSet.AbstractOrderedSetEntry> iterator =
tree.getIndexRange(tree.size() / 2, tree.size(), false);
while (iterator.hasNext()) {
- iterator.next();
+ RedisSortedSet.AbstractOrderedSetEntry entry = iterator.next();
+ sizeOfEntries -= sizer.sizeof(entry);
iterator.remove();
- assertThat(tree.getSizeInBytes()).isEqualTo(sizer.sizeof(tree));
+ assertThat(tree.getSizeInBytes()).isEqualTo(sizer.sizeof(tree) -
sizeOfEntries);
}
// Remove the rest of the entries using tree.remove() and verify that the
reported size is
@@ -741,8 +750,10 @@ public class OrderStatisticsTreeTest {
int entriesSize = entries.size();
for (int i = 0; i < entriesSize; i++) {
RedisSortedSet.OrderedSetEntry entry = entries.get(i);
- tree.remove(entry);
- assertThat(tree.getSizeInBytes()).isEqualTo(sizer.sizeof(tree));
+ if (tree.remove(entry)) {
+ sizeOfEntries -= sizer.sizeof(entry);
+ }
+ assertThat(tree.getSizeInBytes()).isEqualTo(sizer.sizeof(tree) -
sizeOfEntries);
}
}
@@ -752,13 +763,13 @@ public class OrderStatisticsTreeTest {
// that. If they have increased, carefully consider that increase before
changing the constants.
@Test
public void orderStatisticsTreeBaseSize_shouldMatchReflectedSize() {
- OrderStatisticsSet<RedisSortedSet.OrderedSetEntry> tree = new
OrderStatisticsTree<>();
- assertThat(ORDER_STATISTICS_TREE_BASE_SIZE).isEqualTo(sizer.sizeof(tree));
+ RedisSortedSet.ScoreSet tree = new RedisSortedSet.ScoreSet();
+ assertThat(tree.getSizeInBytes()).isEqualTo(sizer.sizeof(tree));
}
@Test
public void orderStatisticsTreePerEntryOverhead_shouldMatchReflectedSize() {
- OrderStatisticsSet<RedisSortedSet.OrderedSetEntry> tree = new
OrderStatisticsTree<>();
+ RedisSortedSet.ScoreSet tree = new RedisSortedSet.ScoreSet();
int beforeSize = sizer.sizeof(tree);
for (int i = 0; i < 100; ++i) {
byte[] member = new byte[i];
@@ -767,7 +778,7 @@ public class OrderStatisticsTreeTest {
tree.add(entry);
int afterSize = sizer.sizeof(tree);
int perMemberOverhead = afterSize - beforeSize - sizer.sizeof(entry);
- assertThat(PER_ENTRY_OVERHEAD).isEqualTo(perMemberOverhead);
+ assertThat(NODE_OVERHEAD).isEqualTo(perMemberOverhead);
beforeSize = afterSize;
}
}
diff --git
a/geode-apis-compatible-with-redis/src/test/java/org/apache/geode/redis/internal/collections/OrderedStatisticTreeQuickCheckTest.java
b/geode-apis-compatible-with-redis/src/test/java/org/apache/geode/redis/internal/collections/OrderedStatisticTreeQuickCheckTest.java
index b853626..f6e24f1 100644
---
a/geode-apis-compatible-with-redis/src/test/java/org/apache/geode/redis/internal/collections/OrderedStatisticTreeQuickCheckTest.java
+++
b/geode-apis-compatible-with-redis/src/test/java/org/apache/geode/redis/internal/collections/OrderedStatisticTreeQuickCheckTest.java
@@ -34,6 +34,7 @@ import org.junit.runner.RunWith;
*/
@RunWith(JUnitQuickcheck.class)
public class OrderedStatisticTreeQuickCheckTest {
+
OrderStatisticsSet<Long> list = new OrderStatisticsTree<>();
OrderStatisticsSet<Long> expected = new IndexibleTreeSet<>();
diff --git
a/geode-apis-compatible-with-redis/src/test/java/org/apache/geode/redis/internal/collections/SizeableObject2ObjectOpenCustomHashMapWithCursorQuickCheckTest.java
b/geode-apis-compatible-with-redis/src/test/java/org/apache/geode/redis/internal/collections/SizeableObject2ObjectOpenCustomHashMapWithCursorQuickCheckTest.java
index cef48ae..1f2c0e8 100644
---
a/geode-apis-compatible-with-redis/src/test/java/org/apache/geode/redis/internal/collections/SizeableObject2ObjectOpenCustomHashMapWithCursorQuickCheckTest.java
+++
b/geode-apis-compatible-with-redis/src/test/java/org/apache/geode/redis/internal/collections/SizeableObject2ObjectOpenCustomHashMapWithCursorQuickCheckTest.java
@@ -44,14 +44,31 @@ public class
SizeableObject2ObjectOpenCustomHashMapWithCursorQuickCheckTest {
}
};
+ private static class Integer2IntegerMap
+ extends SizeableObject2ObjectOpenCustomHashMapWithCursor<Integer,
Integer> {
+
+ public Integer2IntegerMap() {
+ super(NATURAL_HASH);
+ }
+
+ @Override
+ protected int sizeKey(Integer key) {
+ return 0;
+ }
+
+ @Override
+ protected int sizeValue(Integer value) {
+ return 0;
+ }
+ }
+
@Property
public void scanWithConcurrentModifications_ReturnsExpectedElements(
@Size(min = 2, max = 500) Set<@InRange(minInt = 0, maxInt = 500)
Integer> initialData,
@Size(max = 500) Set<@InRange(minInt = 0, maxInt = 1000) Integer>
dataToAdd,
@Size(max = 500) Set<@InRange(minInt = 0, maxInt = 500) Integer>
keysToRemove) {
- SizeableObject2ObjectOpenCustomHashMapWithCursor<Integer, Integer> map =
- new SizeableObject2ObjectOpenCustomHashMapWithCursor<>(NATURAL_HASH);
+ Integer2IntegerMap map = new Integer2IntegerMap();
initialData.forEach(i -> map.put(i, i));
HashMap<Integer, Integer> scanned = new HashMap<>();
@@ -72,8 +89,7 @@ public class
SizeableObject2ObjectOpenCustomHashMapWithCursorQuickCheckTest {
@Property
public void scanWithNoModificationsDoesNotReturnDuplicates(
@Size(min = 2, max = 500) Set<@InRange(minInt = 0, maxInt = 500)
Integer> initialData) {
- SizeableObject2ObjectOpenCustomHashMapWithCursor<Integer, Integer> map =
- new SizeableObject2ObjectOpenCustomHashMapWithCursor<>(NATURAL_HASH);
+ Integer2IntegerMap map = new Integer2IntegerMap();
initialData.forEach(i -> map.put(i, i));
List<Integer> scanned = new ArrayList<>();
diff --git
a/geode-apis-compatible-with-redis/src/test/java/org/apache/geode/redis/internal/collections/SizeableObject2ObjectOpenCustomHashMapWithCursorTest.java
b/geode-apis-compatible-with-redis/src/test/java/org/apache/geode/redis/internal/collections/SizeableObject2ObjectOpenCustomHashMapWithCursorTest.java
index 8e11f9a..6b2097f 100644
---
a/geode-apis-compatible-with-redis/src/test/java/org/apache/geode/redis/internal/collections/SizeableObject2ObjectOpenCustomHashMapWithCursorTest.java
+++
b/geode-apis-compatible-with-redis/src/test/java/org/apache/geode/redis/internal/collections/SizeableObject2ObjectOpenCustomHashMapWithCursorTest.java
@@ -14,22 +14,20 @@
*/
package org.apache.geode.redis.internal.collections;
-import static
org.apache.geode.redis.internal.collections.SizeableObject2ObjectOpenCustomHashMapWithCursor.BACKING_ARRAY_LENGTH_COEFFICIENT;
-import static
org.apache.geode.redis.internal.collections.SizeableObject2ObjectOpenCustomHashMapWithCursor.BACKING_ARRAY_OVERHEAD_CONSTANT;
+import static org.apache.geode.internal.JvmSizeUtils.memoryOverhead;
import static org.assertj.core.api.Assertions.assertThat;
import java.util.HashMap;
import java.util.Map;
-import java.util.Random;
import java.util.stream.IntStream;
import it.unimi.dsi.fastutil.Hash;
import it.unimi.dsi.fastutil.bytes.ByteArrays;
-import org.assertj.core.api.SoftAssertions;
import org.junit.Test;
import org.apache.geode.cache.util.ObjectSizer;
import org.apache.geode.internal.size.ReflectionObjectSizer;
+import org.apache.geode.redis.internal.data.RedisHash;
import org.apache.geode.redis.internal.data.RedisSortedSet;
public class SizeableObject2ObjectOpenCustomHashMapWithCursorTest {
@@ -47,7 +45,7 @@ public class
SizeableObject2ObjectOpenCustomHashMapWithCursorTest {
}
};
- private static Hash.Strategy<Integer> COLLIDING_HASH = new
Hash.Strategy<Integer>() {
+ private static final Hash.Strategy<Integer> COLLIDING_HASH = new
Hash.Strategy<Integer>() {
@Override
public int hashCode(Integer o) {
return o % 5;
@@ -59,10 +57,30 @@ public class
SizeableObject2ObjectOpenCustomHashMapWithCursorTest {
}
};
+ private static class Integer2StringMap
+ extends SizeableObject2ObjectOpenCustomHashMapWithCursor<Integer,
String> {
+ public Integer2StringMap() {
+ super(NATURAL_HASH);
+ }
+
+ public Integer2StringMap(Hash.Strategy<Integer> hashStrategy) {
+ super(hashStrategy);
+ }
+
+ @Override
+ protected int sizeKey(Integer key) {
+ return 0;
+ }
+
+ @Override
+ protected int sizeValue(String value) {
+ return 0;
+ }
+ }
+
@Test
public void scanEntireMap_ReturnsExpectedElements() {
- SizeableObject2ObjectOpenCustomHashMapWithCursor<Integer, String> map =
- new SizeableObject2ObjectOpenCustomHashMapWithCursor<>(NATURAL_HASH);
+ Integer2StringMap map = new Integer2StringMap();
IntStream.range(0, 10).forEach(i -> map.put(i, "value-" + i));
Map<Integer, String> scanned = new HashMap<>();
@@ -73,8 +91,7 @@ public class
SizeableObject2ObjectOpenCustomHashMapWithCursorTest {
@Test
public void twoScansWithNoModifications_ReturnsExpectedElements() {
- SizeableObject2ObjectOpenCustomHashMapWithCursor<Integer, String> map =
- new SizeableObject2ObjectOpenCustomHashMapWithCursor<>(NATURAL_HASH);
+ Integer2StringMap map = new Integer2StringMap();
IntStream.range(0, 10).forEach(i -> map.put(i, "value-" + i));
Map<Integer, String> scanned = new HashMap<>();
@@ -94,8 +111,7 @@ public class
SizeableObject2ObjectOpenCustomHashMapWithCursorTest {
@Test
public void scanWithConcurrentRemoves_ReturnsExpectedElements() {
- SizeableObject2ObjectOpenCustomHashMapWithCursor<Integer, String> map =
- new SizeableObject2ObjectOpenCustomHashMapWithCursor<>(NATURAL_HASH);
+ Integer2StringMap map = new Integer2StringMap();
IntStream.range(0, 10).forEach(i -> map.put(i, "value-" + i));
Map<Integer, String> scanned = new HashMap<>();
@@ -116,8 +132,7 @@ public class
SizeableObject2ObjectOpenCustomHashMapWithCursorTest {
@Test
public void scanWithHashcodeCollisions_ReturnsExpectedElements() {
- SizeableObject2ObjectOpenCustomHashMapWithCursor<Integer, String> map =
- new SizeableObject2ObjectOpenCustomHashMapWithCursor<>(COLLIDING_HASH);
+ Integer2StringMap map = new Integer2StringMap(COLLIDING_HASH);
IntStream.range(0, 10).forEach(i -> map.put(i, "value-" + i));
// The colliding hash is just key % 5. So 0 and 5 will have the same
hashcode, etc.
@@ -144,8 +159,7 @@ public class
SizeableObject2ObjectOpenCustomHashMapWithCursorTest {
@Test
public void
scanWithHashcodeCollisionsAndConcurrentRemoves_ReturnsExpectedElements() {
- SizeableObject2ObjectOpenCustomHashMapWithCursor<Integer, String> map =
- new SizeableObject2ObjectOpenCustomHashMapWithCursor<>(COLLIDING_HASH);
+ Integer2StringMap map = new Integer2StringMap(COLLIDING_HASH);
IntStream.range(0, 10).forEach(i -> map.put(i, "value-" + i));
Map<Integer, String> scanned = new HashMap<>();
@@ -166,8 +180,7 @@ public class
SizeableObject2ObjectOpenCustomHashMapWithCursorTest {
@Test
public void scanWithGrowingTable_DoesNotMissElements() {
- SizeableObject2ObjectOpenCustomHashMapWithCursor<Integer, String> map =
- new SizeableObject2ObjectOpenCustomHashMapWithCursor<>(NATURAL_HASH);
+ Integer2StringMap map = new Integer2StringMap();
IntStream.range(0, 10).forEach(i -> map.put(i, "value-" + i));
Map<Integer, String> scanned = new HashMap<>();
@@ -188,8 +201,7 @@ public class
SizeableObject2ObjectOpenCustomHashMapWithCursorTest {
@Test
public void scanWithShrinkingTable_DoesNotMissElements() {
- SizeableObject2ObjectOpenCustomHashMapWithCursor<Integer, String> map =
- new SizeableObject2ObjectOpenCustomHashMapWithCursor<>(NATURAL_HASH);
+ Integer2StringMap map = new Integer2StringMap();
IntStream.range(0, 500).forEach(i -> map.put(i, "value-" + i));
Map<Integer, String> scanned = new HashMap<>();
@@ -213,104 +225,48 @@ public class
SizeableObject2ObjectOpenCustomHashMapWithCursorTest {
assertThat(SizeableObject2ObjectOpenCustomHashMapWithCursor.rev(0xFF)).isEqualTo(0xFF000000);
}
- // This test can be used to derive the formula for calculating overhead
associated with resizing
- // the backing arrays of the map. If it fails, examine the output of this
test and determine if
- // the constant or the formula needs to be adjusted. If all the assertions
fail with a constant
- // difference between the expected and actual, adjust the constant. If they
fail with inconsistent
- // differences, adjust the formula
- @Test
- public void backingArrayOverheadCalculationTest() {
- SizeableObject2ObjectOpenCustomHashMapWithCursor<byte[], byte[]> map =
- new SizeableObject2ObjectOpenCustomHashMapWithCursor<>(0,
ByteArrays.HASH_STRATEGY);
- int backingArrayOverhead;
- SoftAssertions softly = new SoftAssertions();
- for (int i = 0; i < 250; ++i) {
- map.put(new byte[i], new byte[250 - i]);
-
- // Calculate the overhead associated only with the backing array
- backingArrayOverhead = sizer.sizeof(map) -
map.getArrayContentsOverhead();
- int expected = BACKING_ARRAY_OVERHEAD_CONSTANT
- + BACKING_ARRAY_LENGTH_COEFFICIENT *
map.getTotalBackingArrayLength();
- softly.assertThat(backingArrayOverhead)
- .as("Expecting backing array overhead to = "
- + "backing array constant + (backing array length coefficient *
total backing array length)"
- + " = " + BACKING_ARRAY_OVERHEAD_CONSTANT + " + (" +
BACKING_ARRAY_LENGTH_COEFFICIENT
- + " * " + map.getTotalBackingArrayLength() + ") but was off by "
- + (expected - backingArrayOverhead))
- .isEqualTo(expected);
- }
- softly.assertAll();
- }
-
@Test
public void putUpdatesSizeWhenCreatingNewEntry() {
- SizeableObject2ObjectOpenCustomHashMapWithCursor<byte[], byte[]> hash =
- new
SizeableObject2ObjectOpenCustomHashMapWithCursor<>(ByteArrays.HASH_STRATEGY);
- assertThat(hash.getSizeInBytes()).isEqualTo(sizer.sizeof(hash));
+ RedisHash.Hash hash = new RedisHash.Hash();
+ assertThat(hash.getSizeInBytes()).isEqualTo(expectedSize(hash));
hash.put(new byte[] {(byte) 1}, new byte[] {(byte) 1});
- assertThat(hash.getSizeInBytes()).isEqualTo(sizer.sizeof(hash));
+ assertThat(hash.getSizeInBytes()).isEqualTo(expectedSize(hash));
}
@Test
public void putUpdatesSizeWhenUpdatingExistingEntry() {
- SizeableObject2ObjectOpenCustomHashMapWithCursor<byte[], byte[]> hash =
- new
SizeableObject2ObjectOpenCustomHashMapWithCursor<>(ByteArrays.HASH_STRATEGY);
- assertThat(hash.getSizeInBytes()).isEqualTo(sizer.sizeof(hash));
+ RedisHash.Hash hash = new RedisHash.Hash();
+ assertThat(hash.getSizeInBytes()).isEqualTo(expectedSize(hash));
byte[] key = new byte[1];
byte[] initialValue = new byte[1];
hash.put(key, initialValue);
- assertThat(hash.getSizeInBytes()).isEqualTo(sizer.sizeof(hash));
+ assertThat(hash.getSizeInBytes()).isEqualTo(expectedSize(hash));
byte[] largerValue = new byte[100];
hash.put(key, largerValue);
- assertThat(hash.getSizeInBytes()).isEqualTo(sizer.sizeof(hash));
+ assertThat(hash.getSizeInBytes()).isEqualTo(expectedSize(hash));
byte[] smallerValue = new byte[2];
hash.put(key, smallerValue);
- assertThat(hash.getSizeInBytes()).isEqualTo(sizer.sizeof(hash));
+ assertThat(hash.getSizeInBytes()).isEqualTo(expectedSize(hash));
}
@Test
public void removeUpdatesSize() {
- SizeableObject2ObjectOpenCustomHashMapWithCursor<byte[], byte[]> hash =
- new
SizeableObject2ObjectOpenCustomHashMapWithCursor<>(ByteArrays.HASH_STRATEGY);
+ RedisHash.Hash hash = new RedisHash.Hash();
byte[] key = new byte[1];
byte[] initialValue = new byte[100];
hash.put(key, initialValue);
- assertThat(hash.getSizeInBytes()).isEqualTo(sizer.sizeof(hash));
+ assertThat(hash.getSizeInBytes()).isEqualTo(expectedSize(hash));
hash.remove(key);
- assertThat(hash.getSizeInBytes()).isEqualTo(sizer.sizeof(hash));
+ assertThat(hash.getSizeInBytes()).isEqualTo(expectedSize(hash));
}
- @Test
- public void calculateBackingArraysOverheadForDifferentInitialSizes() {
- for (int i = 0; i < 1000; ++i) {
- SizeableObject2ObjectOpenCustomHashMapWithCursor<byte[], byte[]> hash =
- new SizeableObject2ObjectOpenCustomHashMapWithCursor<>(i,
ByteArrays.HASH_STRATEGY);
-
assertThat(hash.calculateBackingArraysOverhead()).isEqualTo(sizer.sizeof(hash));
- }
- }
-
- @Test
- public void
calculateBackingArraysOverheadForDifferentLoadFactorsAndInitialSizes() {
- Random random = new Random(42);
- for (int i = 0; i < 1000; ++i) {
- float loadFactor = random.nextFloat();
- int initialSize = random.nextInt(1000);
-
- // Create a map with a random initial size and load factor
- SizeableObject2ObjectOpenCustomHashMapWithCursor<byte[], byte[]> hash =
- new SizeableObject2ObjectOpenCustomHashMapWithCursor<>(initialSize,
loadFactor,
- ByteArrays.HASH_STRATEGY);
-
- // Confirm that the calculated value matches the actual value
- assertThat(hash.calculateBackingArraysOverhead())
- .as("load factor = " + loadFactor + ", initial size = " +
initialSize)
- .isEqualTo(sizer.sizeof(hash));
- }
+ private int
expectedSize(SizeableObject2ObjectOpenCustomHashMapWithCursor<byte[], ?> map) {
+ return sizer.sizeof(map) - sizer.sizeof(ByteArrays.HASH_STRATEGY);
}
@Test
@@ -325,11 +281,9 @@ public class
SizeableObject2ObjectOpenCustomHashMapWithCursorTest {
byte[] value = {(byte) (initialNumberOfElements - i)};
initialElements.put(key, value);
}
- SizeableObject2ObjectOpenCustomHashMapWithCursor<byte[], byte[]> hash =
- new SizeableObject2ObjectOpenCustomHashMapWithCursor<>(initialElements,
- ByteArrays.HASH_STRATEGY);
+ RedisHash.Hash hash = new RedisHash.Hash(initialElements);
- assertThat(hash.getSizeInBytes()).isEqualTo(sizer.sizeof(hash));
+ assertThat(hash.getSizeInBytes()).isEqualTo(expectedSize(hash));
// Add more elements to force a resizing of the backing arrays and confirm
that size changes as
// expected
@@ -338,7 +292,7 @@ public class
SizeableObject2ObjectOpenCustomHashMapWithCursorTest {
byte[] key = {(byte) i};
byte[] value = {(byte) (totalNumberOfElements - i)};
hash.put(key, value);
- assertThat(hash.getSizeInBytes()).isEqualTo(sizer.sizeof(hash));
+ assertThat(hash.getSizeInBytes()).isEqualTo(expectedSize(hash));
}
// Update values and confirm that size changes as expected
@@ -346,7 +300,7 @@ public class
SizeableObject2ObjectOpenCustomHashMapWithCursorTest {
byte[] key = {(byte) i};
byte[] value = {(byte) i};
hash.put(key, value);
- assertThat(hash.getSizeInBytes()).isEqualTo(sizer.sizeof(hash));
+ assertThat(hash.getSizeInBytes()).isEqualTo(expectedSize(hash));
}
assertThat(hash.size()).isEqualTo(totalNumberOfElements);
@@ -354,7 +308,7 @@ public class
SizeableObject2ObjectOpenCustomHashMapWithCursorTest {
// Remove all elements and confirm that size changes as expected
for (int i = 0; i < totalNumberOfElements; ++i) {
hash.remove(new byte[] {(byte) i});
- assertThat(hash.getSizeInBytes()).isEqualTo(sizer.sizeof(hash));
+ assertThat(hash.getSizeInBytes()).isEqualTo(expectedSize(hash));
}
assertThat(hash.size()).isEqualTo(0);
@@ -369,37 +323,44 @@ public class
SizeableObject2ObjectOpenCustomHashMapWithCursorTest {
// Create a map with an initial size
for (int i = 0; i < initialNumberOfElements; ++i) {
byte[] key = {(byte) i};
- byte[] member = new byte[i];
+ byte[] member = key;
byte[] scoreBytes = String.valueOf(i).getBytes();
RedisSortedSet.OrderedSetEntry value = new
RedisSortedSet.OrderedSetEntry(member, scoreBytes);
initialElements.put(key, value);
}
- SizeableObject2ObjectOpenCustomHashMapWithCursor<byte[],
RedisSortedSet.OrderedSetEntry> hash =
- new SizeableObject2ObjectOpenCustomHashMapWithCursor<>(initialElements,
- ByteArrays.HASH_STRATEGY);
+ RedisSortedSet.MemberMap hash = new
RedisSortedSet.MemberMap(initialElements);
- assertThat(hash.getSizeInBytes()).isEqualTo(sizer.sizeof(hash));
+ assertThat(hash.getSizeInBytes()).isEqualTo(expectedSize(hash));
// Add more elements to force a resizing of the backing arrays and confirm
that size changes as
// expected
int totalNumberOfElements = initialNumberOfElements + elementsToAdd;
for (int i = initialNumberOfElements; i < totalNumberOfElements; ++i) {
byte[] key = {(byte) i};
- byte[] member = new byte[i];
+ byte[] member = key;
byte[] scoreBytes = String.valueOf(totalNumberOfElements - i).getBytes();
RedisSortedSet.OrderedSetEntry value = new
RedisSortedSet.OrderedSetEntry(member, scoreBytes);
hash.put(key, value);
- assertThat(hash.getSizeInBytes()).isEqualTo(sizer.sizeof(hash));
+ assertThat(hash.getSizeInBytes()).isEqualTo(expectedSize(hash));
}
// Update values and confirm that size changes as expected
for (int i = initialNumberOfElements; i < totalNumberOfElements; ++i) {
byte[] key = {(byte) i};
- byte[] member = new byte[i];
+ byte[] member = key;
byte[] scoreBytes = String.valueOf(i).getBytes();
- RedisSortedSet.OrderedSetEntry value = new
RedisSortedSet.OrderedSetEntry(member, scoreBytes);
- hash.put(key, value);
- assertThat(hash.getSizeInBytes()).isEqualTo(sizer.sizeof(hash));
+ RedisSortedSet.OrderedSetEntry value = hash.get(key);
+ byte[] oldScoreBytes = value.getScoreBytes();
+ int scoreDelta = memoryOverhead(scoreBytes)
+ - memoryOverhead(oldScoreBytes);
+
+ int oldSize = hash.getSizeInBytes();
+ value.updateScore(scoreBytes);
+ int sizeDelta = hash.getSizeInBytes() - oldSize;
+
+ assertThat(sizeDelta).isEqualTo(scoreDelta);
+
+ assertThat(hash.getSizeInBytes()).isEqualTo(expectedSize(hash));
}
assertThat(hash.size()).isEqualTo(totalNumberOfElements);
@@ -407,7 +368,7 @@ public class
SizeableObject2ObjectOpenCustomHashMapWithCursorTest {
// Remove all elements and confirm that size changes as expected
for (int i = 0; i < totalNumberOfElements; ++i) {
hash.remove(new byte[] {(byte) i});
- assertThat(hash.getSizeInBytes()).isEqualTo(sizer.sizeof(hash));
+ assertThat(hash.getSizeInBytes()).isEqualTo(expectedSize(hash));
}
assertThat(hash.size()).isEqualTo(0);
diff --git
a/geode-apis-compatible-with-redis/src/test/java/org/apache/geode/redis/internal/collections/SizeableObjectOpenCustomHashSetTest.java
b/geode-apis-compatible-with-redis/src/test/java/org/apache/geode/redis/internal/collections/SizeableObjectOpenCustomHashSetTest.java
index 2d90e93..d2667c3 100644
---
a/geode-apis-compatible-with-redis/src/test/java/org/apache/geode/redis/internal/collections/SizeableObjectOpenCustomHashSetTest.java
+++
b/geode-apis-compatible-with-redis/src/test/java/org/apache/geode/redis/internal/collections/SizeableObjectOpenCustomHashSetTest.java
@@ -14,131 +14,23 @@
*/
package org.apache.geode.redis.internal.collections;
-import static
org.apache.geode.redis.internal.collections.SizeableObjectOpenCustomHashSet.BACKING_ARRAY_LENGTH_COEFFICIENT;
-import static
org.apache.geode.redis.internal.collections.SizeableObjectOpenCustomHashSet.BACKING_ARRAY_OVERHEAD_CONSTANT;
import static org.assertj.core.api.Assertions.assertThat;
import java.util.ArrayList;
import java.util.List;
-import java.util.Random;
import it.unimi.dsi.fastutil.bytes.ByteArrays;
-import org.assertj.core.api.SoftAssertions;
import org.junit.Test;
import org.apache.geode.cache.util.ObjectSizer;
import org.apache.geode.internal.size.ReflectionObjectSizer;
-import org.apache.geode.internal.size.ReflectionSingleObjectSizer;
-import org.apache.geode.internal.size.SingleObjectSizer;
+import org.apache.geode.redis.internal.data.RedisSet;
public class SizeableObjectOpenCustomHashSetTest {
private final ObjectSizer sizer = ReflectionObjectSizer.getInstance();
- private final SingleObjectSizer elementSizer = new
ReflectionSingleObjectSizer();
- // This test can be used to derive the formula for calculating overhead
associated with resizing
- // the backing array of the set. If it fails examine the output of this test
and determine if the
- // constant or the formula needs to be adjusted. If all the assertions fail
with a constant
- // difference between the expected and actual, adjust the constant. If they
fail with inconsistent
- // differences, adjust the formula
- @Test
- public void backingArrayOverheadCalculationTest() {
- SizeableObjectOpenCustomHashSet<byte[]> set =
- new SizeableObjectOpenCustomHashSet<>(0, ByteArrays.HASH_STRATEGY);
- int backingArrayOverhead;
- SoftAssertions softly = new SoftAssertions();
- for (int i = 0; i < 250; ++i) {
- set.add(new byte[i]);
-
- // Calculate the overhead associated only with the backing array
- backingArrayOverhead = sizer.sizeof(set) - set.getMemberOverhead();
- int expected = BACKING_ARRAY_OVERHEAD_CONSTANT
- + BACKING_ARRAY_LENGTH_COEFFICIENT * set.getBackingArrayLength();
- softly.assertThat(backingArrayOverhead)
- .as("Expecting backing array overhead to = "
- + "backing array constant + (backing array length coefficient *
backing array length)"
- + " = " + BACKING_ARRAY_OVERHEAD_CONSTANT + " + (" +
BACKING_ARRAY_LENGTH_COEFFICIENT
- + " * " + set.getBackingArrayLength() + ") but was off by "
- + (expected - backingArrayOverhead))
- .isEqualTo(expected);
- }
- softly.assertAll();
- }
-
- @Test
- public void addIncreasesMemberOverheadByCorrectAmount() {
- SizeableObjectOpenCustomHashSet<byte[]> set =
- new SizeableObjectOpenCustomHashSet<>(ByteArrays.HASH_STRATEGY);
- int initialSize = set.getMemberOverhead();
- List<byte[]> members = new ArrayList<>();
- for (int i = 0; i < 100; ++i) {
- members.add(new byte[i]);
- }
-
- // Add a duplicate member to check that member overhead isn't increased if
a member isn't added
- // to the set
- members.add(new byte[100]);
-
- for (byte[] bytes : members) {
- boolean added = set.add(bytes);
- if (added) {
- long expectedOverhead = elementSizer.sizeof(bytes);
- assertThat(expectedOverhead).isEqualTo(set.getMemberOverhead() -
initialSize);
- initialSize = set.getMemberOverhead();
- } else {
- assertThat(set.getMemberOverhead() - initialSize).isZero();
- }
- }
- }
-
- @Test
- public void removeDecreasesMemberOverheadByCorrectAmount() {
- SizeableObjectOpenCustomHashSet<byte[]> set =
- new SizeableObjectOpenCustomHashSet<>(ByteArrays.HASH_STRATEGY);
- List<byte[]> members = new ArrayList<>();
- for (int i = 0; i < 100; ++i) {
- members.add(new byte[i]);
- }
- set.addAll(members);
-
- // Add a byte to the list that isn't present in the set to ensure that
member overhead isn't
- // decreased when a member isn't actually removed
- members.add(new byte[101]);
-
- int initialSize = set.getMemberOverhead();
-
- for (byte[] bytes : members) {
- boolean removed = set.remove(bytes);
- if (removed) {
- long expectedOverhead = elementSizer.sizeof(bytes);
- assertThat(expectedOverhead).isEqualTo(initialSize -
set.getMemberOverhead());
- initialSize = set.getMemberOverhead();
- } else {
- assertThat(set.getMemberOverhead() - initialSize).isZero();
- }
- }
- }
-
- @Test
- public void calculateBackingArrayOverheadForDifferentInitialSizes() {
- for (int i = 0; i < 1000; ++i) {
- SizeableObjectOpenCustomHashSet<byte[]> set =
- new SizeableObjectOpenCustomHashSet<>(i, ByteArrays.HASH_STRATEGY);
-
assertThat(set.calculateBackingArrayOverhead()).isEqualTo(sizer.sizeof(set));
- }
- }
-
- @Test
- public void
calculateBackingArrayOverheadForDifferentLoadFactorsAndInitialSizes() {
- Random random = new Random(42);
- for (int i = 0; i < 1000; ++i) {
- float loadFactor = random.nextFloat();
- int initialSize = random.nextInt(1000);
- SizeableObjectOpenCustomHashSet<byte[]> set =
- new SizeableObjectOpenCustomHashSet<>(initialSize, loadFactor,
ByteArrays.HASH_STRATEGY);
- assertThat(set.calculateBackingArrayOverhead())
- .as("load factor = " + loadFactor + ", initial size = " +
initialSize)
- .isEqualTo(sizer.sizeof(set));
- }
+ private int expectedSize(RedisSet.MemberSet set) {
+ return sizer.sizeof(set) - sizer.sizeof(ByteArrays.HASH_STRATEGY);
}
@Test
@@ -150,21 +42,20 @@ public class SizeableObjectOpenCustomHashSetTest {
initialElements.add(new byte[] {(byte) i});
}
// Create a set with an initial size and confirm that it correctly reports
its size
- SizeableObjectOpenCustomHashSet<byte[]> set =
- new SizeableObjectOpenCustomHashSet<>(initialElements,
ByteArrays.HASH_STRATEGY);
- assertThat(set.getSizeInBytes()).isEqualTo(sizer.sizeof(set));
+ RedisSet.MemberSet set = new RedisSet.MemberSet(initialElements);
+ assertThat(set.getSizeInBytes()).isEqualTo(expectedSize(set));
// Add enough members to force a resize and assert that the size is
correct after each add
for (int i = initialNumberOfElements; i < initialNumberOfElements +
elementsToAdd; ++i) {
set.add(new byte[] {(byte) i});
- assertThat(set.getSizeInBytes()).isEqualTo(sizer.sizeof(set));
+ assertThat(set.getSizeInBytes()).isEqualTo(expectedSize(set));
}
assertThat(set.size()).isEqualTo(initialNumberOfElements + elementsToAdd);
// Remove all the members and assert that the size is correct after each
remove
for (int i = 0; i < initialNumberOfElements + elementsToAdd; ++i) {
set.remove(new byte[] {(byte) i});
- assertThat(set.getSizeInBytes()).isEqualTo(sizer.sizeof(set));
+ assertThat(set.getSizeInBytes()).isEqualTo(expectedSize(set));
}
assertThat(set.size()).isEqualTo(0);
}
diff --git
a/geode-apis-compatible-with-redis/src/test/java/org/apache/geode/redis/internal/data/RedisHashTest.java
b/geode-apis-compatible-with-redis/src/test/java/org/apache/geode/redis/internal/data/RedisHashTest.java
index 1862cb1..db8531c 100644
---
a/geode-apis-compatible-with-redis/src/test/java/org/apache/geode/redis/internal/data/RedisHashTest.java
+++
b/geode-apis-compatible-with-redis/src/test/java/org/apache/geode/redis/internal/data/RedisHashTest.java
@@ -16,7 +16,7 @@
package org.apache.geode.redis.internal.data;
-import static
org.apache.geode.redis.internal.data.RedisHash.BASE_REDIS_HASH_OVERHEAD;
+import static
org.apache.geode.redis.internal.data.RedisHash.REDIS_HASH_OVERHEAD;
import static org.apache.geode.redis.internal.netty.Coder.stringToBytes;
import static org.apache.geode.util.internal.UncheckedUtils.uncheckedCast;
import static org.assertj.core.api.Assertions.assertThat;
@@ -52,7 +52,6 @@ import
org.apache.geode.internal.serialization.DataSerializableFixedID;
import org.apache.geode.internal.serialization.SerializationContext;
import org.apache.geode.internal.size.ReflectionObjectSizer;
import org.apache.geode.internal.size.ReflectionSingleObjectSizer;
-import
org.apache.geode.redis.internal.collections.SizeableObject2ObjectOpenCustomHashMapWithCursor;
import org.apache.geode.redis.internal.netty.Coder;
public class RedisHashTest {
@@ -224,10 +223,9 @@ public class RedisHashTest {
@Test
public void constantBaseRedisHashOverhead_shouldEqualCalculatedOverhead() {
RedisHash hash = new RedisHash(Collections.emptyList());
- SizeableObject2ObjectOpenCustomHashMapWithCursor<byte[], byte[]>
backingHash =
- new SizeableObject2ObjectOpenCustomHashMapWithCursor<>(0,
ByteArrays.HASH_STRATEGY);
+ RedisHash.Hash backingHash = new RedisHash.Hash(0);
- assertThat(sizer.sizeof(hash) -
sizer.sizeof(backingHash)).isEqualTo(BASE_REDIS_HASH_OVERHEAD);
+ assertThat(sizer.sizeof(hash) -
sizer.sizeof(backingHash)).isEqualTo(REDIS_HASH_OVERHEAD);
}
/******* constructor *******/
@@ -240,7 +238,7 @@ public class RedisHashTest {
RedisHash redisHash = new RedisHash(data);
- final int expected = sizer.sizeof(redisHash);
+ final int expected = sizer.sizeof(redisHash) -
sizer.sizeof(ByteArrays.HASH_STRATEGY);
final int actual = redisHash.getSizeInBytes();
assertThat(actual).isEqualTo(expected);
@@ -251,12 +249,16 @@ public class RedisHashTest {
RedisHash redisHash =
createRedisHash("aSuperLongField", "value", "field",
"aSuperLongValue");
- final int expected = sizer.sizeof(redisHash);
+ final int expected = expectedSize(redisHash);
final int actual = redisHash.getSizeInBytes();
assertThat(actual).isEqualTo(expected);
}
+ private int expectedSize(RedisHash hash) {
+ return sizer.sizeof(hash) - sizer.sizeof(ByteArrays.HASH_STRATEGY);
+ }
+
@Test
public void should_calculateSize_equalToROSSize_withManyEntries() {
final String baseField = "longerbase";
@@ -270,7 +272,7 @@ public class RedisHashTest {
RedisHash hash = new RedisHash(elements);
Integer actual = hash.getSizeInBytes();
- int expected = sizer.sizeof(hash);
+ int expected = expectedSize(hash);
assertThat(actual).isEqualTo(expected);
}
@@ -343,7 +345,7 @@ public class RedisHashTest {
hash.hset(region, key, initialData, false);
hash.clearDelta();
- long initialSize = sizer.sizeof(hash);
+ long initialSize = expectedSize(hash);
List<byte[]> finalData = new ArrayList<>();
finalData.add(stringToBytes(field));
@@ -352,7 +354,7 @@ public class RedisHashTest {
hash.hset(region, key, finalData, false);
hash.clearDelta();
- assertThat(hash.getSizeInBytes()).isEqualTo(sizer.sizeof(hash));
+ assertThat(hash.getSizeInBytes()).isEqualTo(expectedSize(hash));
long expectedSizeChange =
elementSizer.sizeof(Coder.stringToBytes(finalValue))
- elementSizer.sizeof(Coder.stringToBytes(initialValue));
@@ -477,7 +479,7 @@ public class RedisHashTest {
int finalSize = redisHash.getSizeInBytes();
assertThat(finalSize).isLessThan(initialSize);
- assertThat(finalSize).isEqualTo(sizer.sizeof(redisHash));
+ assertThat(finalSize).isEqualTo(expectedSize(redisHash));
}
private RedisHash createRedisHash(String... keysAndValues) {
diff --git
a/geode-apis-compatible-with-redis/src/test/java/org/apache/geode/redis/internal/data/RedisSetTest.java
b/geode-apis-compatible-with-redis/src/test/java/org/apache/geode/redis/internal/data/RedisSetTest.java
index 413c7e0..f01d5f1 100644
---
a/geode-apis-compatible-with-redis/src/test/java/org/apache/geode/redis/internal/data/RedisSetTest.java
+++
b/geode-apis-compatible-with-redis/src/test/java/org/apache/geode/redis/internal/data/RedisSetTest.java
@@ -17,7 +17,6 @@
package org.apache.geode.redis.internal.data;
import static
org.apache.geode.redis.internal.data.NullRedisDataStructures.NULL_REDIS_SET;
-import static
org.apache.geode.redis.internal.data.RedisSet.BASE_REDIS_SET_OVERHEAD;
import static org.apache.geode.redis.internal.netty.Coder.stringToBytes;
import static org.apache.geode.util.internal.UncheckedUtils.uncheckedCast;
import static org.assertj.core.api.Assertions.assertThat;
@@ -48,7 +47,6 @@ import
org.apache.geode.internal.serialization.ByteArrayDataInput;
import org.apache.geode.internal.serialization.DataSerializableFixedID;
import org.apache.geode.internal.serialization.SerializationContext;
import org.apache.geode.internal.size.ReflectionObjectSizer;
-import
org.apache.geode.redis.internal.collections.SizeableObjectOpenCustomHashSet;
import org.apache.geode.redis.internal.netty.Coder;
public class RedisSetTest {
@@ -189,7 +187,7 @@ public class RedisSetTest {
Set<byte[]> members = new
ObjectOpenCustomHashSet<>(ByteArrays.HASH_STRATEGY);
RedisSet set = new RedisSet(members);
- int expected = sizer.sizeof(set);
+ int expected = expectedSize(set);
int actual = set.getSizeInBytes();
assertThat(actual).isEqualTo(expected);
@@ -202,7 +200,7 @@ public class RedisSetTest {
RedisSet set = new RedisSet(members);
int actual = set.getSizeInBytes();
- int expected = sizer.sizeof(set);
+ int expected = expectedSize(set);
assertThat(actual).isEqualTo(expected);
}
@@ -212,18 +210,22 @@ public class RedisSetTest {
for (int i = 0; i < 1024; i += 16) {
RedisSet set = createRedisSetOfSpecifiedSize(i);
- int expected = sizer.sizeof(set);
+ int expected = expectedSize(set);
int actual = set.getSizeInBytes();
assertThat(actual).isEqualTo(expected);
}
}
+ private int expectedSize(RedisSet set) {
+ return sizer.sizeof(set) - sizer.sizeof(ByteArrays.HASH_STRATEGY);
+ }
+
@Test
public void should_calculateSize_equalToROS_withVaryingMemberSize() {
for (int i = 0; i < 1024; i += 16) {
RedisSet set = createRedisSetWithMemberOfSpecifiedSize(i * 64);
- int expected = sizer.sizeof(set);
+ int expected = expectedSize(set);
int actual = set.getSizeInBytes();
assertThat(actual).isEqualTo(expected);
@@ -248,7 +250,7 @@ public class RedisSetTest {
set.clearDelta(); // because the region is mocked, the delta has to be
explicitly cleared
- assertThat(set.getSizeInBytes()).isEqualTo(sizer.sizeof(set));
+ assertThat(set.getSizeInBytes()).isEqualTo(expectedSize(set));
}
@Test
@@ -270,7 +272,7 @@ public class RedisSetTest {
set.clearDelta(); // because the region is mocked, the delta has to be
explicitly cleared
long actual = set.getSizeInBytes();
- long expected = sizer.sizeof(set);
+ long expected = expectedSize(set);
assertThat(actual).isEqualTo(expected);
}
@@ -298,7 +300,7 @@ public class RedisSetTest {
set.clearDelta(); // because the region is mocked, the delta has to be
explicitly cleared
long finalSize = set.getSizeInBytes();
- long expectedSize = sizer.sizeof(set);
+ long expectedSize = expectedSize(set);
assertThat(finalSize).isEqualTo(expectedSize);
}
@@ -318,7 +320,7 @@ public class RedisSetTest {
RedisSet set = new RedisSet(initialMembers);
- assertThat(set.getSizeInBytes()).isEqualTo(sizer.sizeof(set));
+ assertThat(set.getSizeInBytes()).isEqualTo(expectedSize(set));
int membersToAdd = numOfInitialMembers * 3;
doAddsAndAssertSize(set, membersToAdd);
@@ -328,21 +330,6 @@ public class RedisSetTest {
doAddsAndAssertSize(set, membersToAdd);
}
- /******** constants *******/
- // This test contains the math that is used to derive the constant in
RedisSet. If this test
- // starts failing, it is because the overhead of RedisSet has changed. If it
has decreased, good
- // job! You can change the constant in RedisSet to reflect that. If it has
increased, carefully
- // consider that increase before changing the constants.
- @Test
- public void baseOverheadConstant_shouldMatchReflectedSize() {
- RedisSet set = new RedisSet(Collections.emptyList());
- SizeableObjectOpenCustomHashSet<byte[]> backingSet =
- new SizeableObjectOpenCustomHashSet<>(0, ByteArrays.HASH_STRATEGY);
- int baseRedisSetOverhead = sizer.sizeof(set) - sizer.sizeof(backingSet);
-
- assertThat(baseRedisSetOverhead).isEqualTo(BASE_REDIS_SET_OVERHEAD);
- }
-
/******* helper methods *******/
private RedisSet createRedisSetOfSpecifiedSize(int setSize) {
List<byte[]> arrayList = new ArrayList<>();
@@ -388,7 +375,7 @@ public class RedisSetTest {
assertThat(calculatedOH).isEqualTo(actualOverhead);
}
- assertThat(set.getSizeInBytes()).isEqualTo(sizer.sizeof(set));
+ assertThat(set.getSizeInBytes()).isEqualTo(expectedSize(set));
}
void doRemovesAndAssertSize(RedisSet set, int membersToRemove) {
@@ -405,6 +392,6 @@ public class RedisSetTest {
assertThat(calculatedOH).isEqualTo(actualOverhead);
}
- assertThat(set.getSizeInBytes()).isEqualTo(sizer.sizeof(set));
+ assertThat(set.getSizeInBytes()).isEqualTo(expectedSize(set));
}
}
diff --git
a/geode-apis-compatible-with-redis/src/test/java/org/apache/geode/redis/internal/data/RedisSortedSetTest.java
b/geode-apis-compatible-with-redis/src/test/java/org/apache/geode/redis/internal/data/RedisSortedSetTest.java
index 0e3bc3e..499b01a 100644
---
a/geode-apis-compatible-with-redis/src/test/java/org/apache/geode/redis/internal/data/RedisSortedSetTest.java
+++
b/geode-apis-compatible-with-redis/src/test/java/org/apache/geode/redis/internal/data/RedisSortedSetTest.java
@@ -16,8 +16,8 @@
package org.apache.geode.redis.internal.data;
-import static
org.apache.geode.redis.internal.data.RedisSortedSet.BASE_REDIS_SORTED_SET_OVERHEAD;
-import static
org.apache.geode.redis.internal.data.RedisSortedSet.OrderedSetEntry.BASE_ORDERED_SET_ENTRY_SIZE;
+import static
org.apache.geode.redis.internal.data.RedisSortedSet.OrderedSetEntry.ORDERED_SET_ENTRY_OVERHEAD;
+import static
org.apache.geode.redis.internal.data.RedisSortedSet.REDIS_SORTED_SET_OVERHEAD;
import static org.apache.geode.redis.internal.netty.Coder.doubleToBytes;
import static org.apache.geode.redis.internal.netty.Coder.stringToBytes;
import static
org.apache.geode.redis.internal.netty.StringBytesGlossary.bGREATEST_MEMBER_NAME;
@@ -55,9 +55,6 @@ import
org.apache.geode.internal.serialization.ByteArrayDataInput;
import org.apache.geode.internal.serialization.DataSerializableFixedID;
import org.apache.geode.internal.serialization.SerializationContext;
import org.apache.geode.internal.size.ReflectionObjectSizer;
-import org.apache.geode.redis.internal.collections.OrderStatisticsSet;
-import org.apache.geode.redis.internal.collections.OrderStatisticsTree;
-import
org.apache.geode.redis.internal.collections.SizeableObject2ObjectOpenCustomHashMapWithCursor;
import org.apache.geode.redis.internal.delta.RemsDeltaInfo;
import org.apache.geode.redis.internal.executor.sortedset.ZAddOptions;
import org.apache.geode.redis.internal.netty.Coder;
@@ -549,13 +546,12 @@ public class RedisSortedSetTest {
@Test
public void baseRedisSortedSetOverheadConstant_shouldMatchReflectedSize() {
RedisSortedSet set = new RedisSortedSet(Collections.emptyList());
- SizeableObject2ObjectOpenCustomHashMapWithCursor<byte[],
RedisSortedSet.OrderedSetEntry> backingMap =
- new SizeableObject2ObjectOpenCustomHashMapWithCursor<>(0,
ByteArrays.HASH_STRATEGY);
- OrderStatisticsSet<RedisSortedSet.OrderedSetEntry> backingTree = new
OrderStatisticsTree<>();
+ RedisSortedSet.MemberMap backingMap = new RedisSortedSet.MemberMap(0);
+ RedisSortedSet.ScoreSet backingTree = new RedisSortedSet.ScoreSet();
int baseRedisSetOverhead =
sizer.sizeof(set) - sizer.sizeof(backingMap) -
sizer.sizeof(backingTree);
- assertThat(BASE_REDIS_SORTED_SET_OVERHEAD).isEqualTo(baseRedisSetOverhead);
+ assertThat(REDIS_SORTED_SET_OVERHEAD).isEqualTo(baseRedisSetOverhead);
}
@Test
@@ -566,7 +562,7 @@ public class RedisSortedSetTest {
new RedisSortedSet.OrderedSetEntry(memberBytes, scoreBytes);
int expectedSize = sizer.sizeof(entry) - sizer.sizeof(scoreBytes) -
sizer.sizeof(memberBytes);
- assertThat(BASE_ORDERED_SET_ENTRY_SIZE).isEqualTo(expectedSize);
+ assertThat(ORDERED_SET_ENTRY_OVERHEAD).isEqualTo(expectedSize);
}
/****************** Size ******************/
@@ -578,19 +574,19 @@ public class RedisSortedSetTest {
ZAddOptions options = new ZAddOptions(ZAddOptions.Exists.NONE, false,
false);
RedisSortedSet sortedSet = new RedisSortedSet(Collections.emptyList());
- int actualSize = sizer.sizeof(sortedSet);
- int calculatedSize = sortedSet.getSizeInBytes();
- assertThat(calculatedSize).isEqualTo(actualSize);
+ int expectedSize = sizer.sizeof(sortedSet) -
sizer.sizeof(ByteArrays.HASH_STRATEGY);
+ int actualSize = sortedSet.getSizeInBytes();
+ assertThat(actualSize).isEqualTo(expectedSize);
- // Add members and scores and confirm that the calculated size is accurate
after each operation
+ // Add members and scores and confirm that the actual size is accurate
after each operation
int numberOfEntries = 100;
for (int i = 0; i < numberOfEntries; ++i) {
List<byte[]> scoreAndMember = Arrays.asList(doubleToBytes(i), new
byte[i]);
sortedSet.zadd(mockRegion, mockKey, scoreAndMember, options);
sortedSet.clearDelta();
- actualSize = sizer.sizeof(sortedSet);
- calculatedSize = sortedSet.getSizeInBytes();
- assertThat(calculatedSize).isEqualTo(actualSize);
+ expectedSize = sizer.sizeof(sortedSet) -
sizer.sizeof(ByteArrays.HASH_STRATEGY);
+ actualSize = sortedSet.getSizeInBytes();
+ assertThat(actualSize).isEqualTo(expectedSize);
}
}
@@ -607,12 +603,14 @@ public class RedisSortedSetTest {
sortedSet.zadd(mockRegion, mockKey, scoreAndMember, options);
}
- // Update half the scores and confirm that the calculated size is accurate
after each operation
+ // Update half the scores and confirm that the actual size is accurate
after each operation
for (int i = 0; i < numberOfEntries / 2; ++i) {
List<byte[]> scoreAndMember = Arrays.asList(doubleToBytes(i * 2), new
byte[i]);
sortedSet.zadd(mockRegion, mockKey, scoreAndMember, options);
sortedSet.clearDelta();
-
assertThat(sizer.sizeof(sortedSet)).isEqualTo(sortedSet.getSizeInBytes());
+ int expectedSize = sizer.sizeof(sortedSet) -
sizer.sizeof(ByteArrays.HASH_STRATEGY);
+ int actualSize = sortedSet.getSizeInBytes();
+ assertThat(actualSize).isEqualTo(expectedSize);
}
}
@@ -629,12 +627,14 @@ public class RedisSortedSetTest {
sortedSet.zadd(mockRegion, mockKey, scoreAndMember, options);
}
- // Remove all members and confirm that the calculated size is accurate
after each operation
+ // Remove all members and confirm that the actual size is accurate after
each operation
for (int i = 0; i < numberOfEntries; ++i) {
List<byte[]> memberToRemove = Collections.singletonList(new byte[i]);
sortedSet.zrem(mockRegion, mockKey, memberToRemove);
sortedSet.clearDelta();
-
assertThat(sizer.sizeof(sortedSet)).isEqualTo(sortedSet.getSizeInBytes());
+ int expectedSize = sizer.sizeof(sortedSet) -
sizer.sizeof(ByteArrays.HASH_STRATEGY);
+ int actualSize = sortedSet.getSizeInBytes();
+ assertThat(actualSize).isEqualTo(expectedSize);
}
}
@@ -656,7 +656,9 @@ public class RedisSortedSetTest {
for (int i = 0; i < numberOfEntries; ++i) {
sortedSet.zpopmax(mockRegion, mockKey, 1);
sortedSet.clearDelta();
-
assertThat(sizer.sizeof(sortedSet)).isEqualTo(sortedSet.getSizeInBytes());
+ int expectedSize = sizer.sizeof(sortedSet) -
sizer.sizeof(ByteArrays.HASH_STRATEGY);
+ int actualSize = sortedSet.getSizeInBytes();
+ assertThat(actualSize).isEqualTo(expectedSize);
}
}
@@ -669,7 +671,7 @@ public class RedisSortedSetTest {
member = new byte[random.nextInt(50_000)];
scoreBytes = String.valueOf(random.nextDouble()).getBytes();
RedisSortedSet.OrderedSetEntry entry = new
RedisSortedSet.OrderedSetEntry(member, scoreBytes);
- assertThat(entry.getSizeInBytes()).isEqualTo(sizer.sizeof(entry));
+ assertThat(entry.getSizeInBytes()).isEqualTo(sizer.sizeof(entry) -
sizer.sizeof(member));
}
}
diff --git
a/geode-apis-compatible-with-redis/src/test/java/org/apache/geode/redis/internal/data/RedisStringTest.java
b/geode-apis-compatible-with-redis/src/test/java/org/apache/geode/redis/internal/data/RedisStringTest.java
index 7585341..572cd3b 100644
---
a/geode-apis-compatible-with-redis/src/test/java/org/apache/geode/redis/internal/data/RedisStringTest.java
+++
b/geode-apis-compatible-with-redis/src/test/java/org/apache/geode/redis/internal/data/RedisStringTest.java
@@ -16,7 +16,6 @@
package org.apache.geode.redis.internal.data;
-import static
org.apache.geode.redis.internal.data.RedisString.BASE_REDIS_STRING_OVERHEAD;
import static org.apache.geode.redis.internal.netty.Coder.longToBytes;
import static org.apache.geode.redis.internal.netty.Coder.stringToBytes;
import static org.apache.geode.util.internal.UncheckedUtils.uncheckedCast;
@@ -356,16 +355,18 @@ public class RedisStringTest {
@Test
public void changingStringValue_toShorterString_shouldDecreaseSizeInBytes() {
String baseString = "baseString";
- String stringToRemove = "asdf";
+ String stringToRemove = "asdf1234567890";
RedisString string = new RedisString(stringToBytes((baseString +
stringToRemove)));
int initialSize = string.getSizeInBytes();
+ assertThat(initialSize).isEqualTo(reflectionObjectSizer.sizeof(string));
string.set(stringToBytes(baseString));
int finalSize = string.getSizeInBytes();
+ assertThat(finalSize).isEqualTo(reflectionObjectSizer.sizeof(string));
- assertThat(finalSize).isEqualTo(initialSize - stringToRemove.length());
+ assertThat(finalSize).isLessThan(initialSize);
}
@Test
@@ -374,38 +375,30 @@ public class RedisStringTest {
RedisString string = new RedisString(stringToBytes(baseString));
int initialSize = string.getSizeInBytes();
+ assertThat(initialSize).isEqualTo(reflectionObjectSizer.sizeof(string));
- String addedString = "asdf";
+ String addedString = "asdf1234567890";
string.set(stringToBytes((baseString + addedString)));
int finalSize = string.getSizeInBytes();
+ assertThat(finalSize).isEqualTo(reflectionObjectSizer.sizeof(string));
- assertThat(finalSize).isEqualTo(initialSize + addedString.length());
+ assertThat(finalSize).isGreaterThan(initialSize);
}
@Test
- public void
changingStringValue_toEmptyString_shouldDecreaseSizeInBytes_toBaseSize() {
- String baseString = "baseString";
- RedisString string = new RedisString(stringToBytes((baseString + "asdf")));
+ public void changingStringValue_toEmptyString_shouldDecreaseSizeInBytes() {
+ String baseString = "baseString1234567890";
+ final int emptySize = reflectionObjectSizer.sizeof(new
RedisString(stringToBytes("")));
+ RedisString string = new RedisString(stringToBytes((baseString)));
+ int baseSize = string.getSizeInBytes();
string.set(stringToBytes(""));
int finalSize = string.getSizeInBytes();
- assertThat(finalSize).isEqualTo(BASE_REDIS_STRING_OVERHEAD);
- }
-
- /******* constants *******/
- // this test contains the math that was used to derive the constants in
RedisString. If this test
- // starts failing, it is because the overhead of RedisString has changed. If
it has decreased,
- // good job! You can change the constant in RedisString to reflect that. If
it has increased,
- // carefully consider that increase before changing the constant.
- @Test
- public void overheadConstants_shouldMatchCalculatedValue() {
- RedisString redisString = new RedisString(stringToBytes(""));
- int calculatedSize = reflectionObjectSizer.sizeof(redisString);
-
- assertThat(BASE_REDIS_STRING_OVERHEAD).isEqualTo(calculatedSize);
+ assertThat(finalSize).isEqualTo(emptySize);
+ assertThat(finalSize).isLessThan(baseSize);
}
/******* helper methods *******/
diff --git
a/geode-core/src/distributedTest/java/org/apache/geode/cache30/MemLRUEvictionControllerDUnitTest.java
b/geode-core/src/distributedTest/java/org/apache/geode/cache30/MemLRUEvictionControllerDUnitTest.java
index 92221f2..ab00d20 100644
---
a/geode-core/src/distributedTest/java/org/apache/geode/cache30/MemLRUEvictionControllerDUnitTest.java
+++
b/geode-core/src/distributedTest/java/org/apache/geode/cache30/MemLRUEvictionControllerDUnitTest.java
@@ -14,6 +14,8 @@
*/
package org.apache.geode.cache30;
+import static org.apache.geode.internal.JvmSizeUtils.getObjectHeaderSize;
+import static org.apache.geode.internal.JvmSizeUtils.roundUpSize;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
@@ -100,16 +102,16 @@ public class MemLRUEvictionControllerDUnitTest extends
JUnit4CacheTestCase {
assertNotNull(lruStats);
String sampleKey = new String("10000");
- int stringSize = JvmSizeUtils.getObjectHeaderSize() // String object
+ int stringSize = getObjectHeaderSize() // String object
+ (2 * 4) + JvmSizeUtils.getReferenceSize(); // 2 ints and a reference
on a string
- stringSize = (int) ReflectionSingleObjectSizer.roundUpSize(stringSize);
+ stringSize = roundUpSize(stringSize);
- int charArraySize = sampleKey.length() * 2 +
JvmSizeUtils.getObjectHeaderSize() // char array
-
// object
+ int charArraySize = sampleKey.length() * 2 + getObjectHeaderSize() // char
array
+ //
object
+ 4; // length of char array
- charArraySize = (int)
ReflectionSingleObjectSizer.roundUpSize(charArraySize);
+ charArraySize = roundUpSize(charArraySize);
assertEquals(stringSize, ReflectionSingleObjectSizer.sizeof(String.class));
-
assertEquals(ReflectionSingleObjectSizer.roundUpSize(JvmSizeUtils.getObjectHeaderSize()
+ 4),
+ assertEquals(roundUpSize(getObjectHeaderSize() + 4),
(new ReflectionSingleObjectSizer()).sizeof(new char[0]));
assertEquals(charArraySize, (new ReflectionSingleObjectSizer()).sizeof(new
char[5]));
@@ -119,9 +121,9 @@ public class MemLRUEvictionControllerDUnitTest extends
JUnit4CacheTestCase {
// now that keys are inlined the keySize is 0
keySize = 0;
byte[] sampleValue = new byte[1000];
- int valueSize = sampleValue.length + JvmSizeUtils.getObjectHeaderSize() //
byte array object;
+ int valueSize = sampleValue.length + getObjectHeaderSize() // byte array
object;
+ 4; // length of byte array
- valueSize = (int) ReflectionSingleObjectSizer.roundUpSize(valueSize);
+ valueSize = roundUpSize(valueSize);
int entrySize = keySize + valueSize + getEntryOverhead(region);
assertEquals(valueSize, ObjectSizer.DEFAULT.sizeof(sampleValue));
diff --git
a/geode-core/src/distributedTest/java/org/apache/geode/internal/cache/eviction/EvictionObjectSizerDUnitTest.java
b/geode-core/src/distributedTest/java/org/apache/geode/internal/cache/eviction/EvictionObjectSizerDUnitTest.java
index 0792797..49eea79 100644
---
a/geode-core/src/distributedTest/java/org/apache/geode/internal/cache/eviction/EvictionObjectSizerDUnitTest.java
+++
b/geode-core/src/distributedTest/java/org/apache/geode/internal/cache/eviction/EvictionObjectSizerDUnitTest.java
@@ -14,6 +14,7 @@
*/
package org.apache.geode.internal.cache.eviction;
+import static org.apache.geode.internal.JvmSizeUtils.roundUpSize;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
@@ -44,7 +45,6 @@ import org.apache.geode.internal.cache.RegionMap;
import org.apache.geode.internal.cache.TestNonSizerObject;
import org.apache.geode.internal.cache.TestObjectSizerImpl;
import org.apache.geode.internal.cache.entries.AbstractLRURegionEntry;
-import org.apache.geode.internal.size.ReflectionSingleObjectSizer;
import org.apache.geode.internal.size.Sizeable;
import org.apache.geode.logging.internal.OSProcess;
import org.apache.geode.test.dunit.LogWriterUtils;
@@ -89,7 +89,7 @@ public class EvictionObjectSizerDUnitTest extends
CacheTestCase {
int keySize = 0;
int valueSize =
JvmSizeUtils.getObjectHeaderSize() + 4 /* array length */ + (1024 *
1024) /* bytes */;
- valueSize = (int) ReflectionSingleObjectSizer.roundUpSize(valueSize);
+ valueSize = roundUpSize(valueSize);
int entrySize = keySize + valueSize
+ ((HeapLRUController) ((PartitionedRegion)
region).getEvictionController())
.getPerEntryOverhead();
@@ -117,7 +117,7 @@ public class EvictionObjectSizerDUnitTest extends
CacheTestCase {
int keySize = 0;
int valueSize =
JvmSizeUtils.getObjectHeaderSize() + 4 /* array length */ + (1024 *
1024) /* bytes */;
- valueSize = (int) ReflectionSingleObjectSizer.roundUpSize(valueSize);
+ valueSize = roundUpSize(valueSize);
int entrySize = keySize + valueSize
+ ((HeapLRUController) ((PartitionedRegion)
region).getEvictionController())
.getPerEntryOverhead();
@@ -134,7 +134,7 @@ public class EvictionObjectSizerDUnitTest extends
CacheTestCase {
int keySize = 0;
int valueSize = JvmSizeUtils.getObjectHeaderSize() + 4 /* array length */
+ (1024 * 1024 * 2) /* bytes */;
- valueSize = (int) ReflectionSingleObjectSizer.roundUpSize(valueSize);
+ valueSize = roundUpSize(valueSize);
int entrySize = keySize + valueSize
+ ((HeapLRUController) ((PartitionedRegion)
region).getEvictionController())
.getPerEntryOverhead();
@@ -158,7 +158,7 @@ public class EvictionObjectSizerDUnitTest extends
CacheTestCase {
{
int keySize = 0;
int valueSize = JvmSizeUtils.getObjectHeaderSize() + 4 /* array length
*/ + 0 /* bytes */;
- valueSize = (int) ReflectionSingleObjectSizer.roundUpSize(valueSize);
+ valueSize = roundUpSize(valueSize);
int entrySize = keySize + valueSize
+ ((HeapLRUController) ((PartitionedRegion)
region).getEvictionController())
.getPerEntryOverhead();
@@ -173,7 +173,7 @@ public class EvictionObjectSizerDUnitTest extends
CacheTestCase {
{
int keySize = 0;
int valueSize = JvmSizeUtils.getObjectHeaderSize() + 4 /* array length
*/ + 4 /* bytes */;
- valueSize = (int) ReflectionSingleObjectSizer.roundUpSize(valueSize);
+ valueSize = roundUpSize(valueSize);
int entrySize = keySize + valueSize
+ ((HeapLRUController) ((PartitionedRegion)
region).getEvictionController())
.getPerEntryOverhead();
diff --git
a/geode-core/src/main/java/org/apache/geode/internal/JvmSizeUtils.java
b/geode-core/src/main/java/org/apache/geode/internal/JvmSizeUtils.java
index 1d90d7a..260a8d6 100644
--- a/geode-core/src/main/java/org/apache/geode/internal/JvmSizeUtils.java
+++ b/geode-core/src/main/java/org/apache/geode/internal/JvmSizeUtils.java
@@ -15,6 +15,8 @@
package org.apache.geode.internal;
+import static
org.apache.geode.internal.size.ReflectionSingleObjectSizer.sizeof;
+
import org.apache.commons.lang3.JavaVersion;
import org.apache.geode.annotations.Immutable;
@@ -132,4 +134,59 @@ public class JvmSizeUtils {
public static int getObjectHeaderSize() {
return objectHeaderSize;
}
+
+ public static long roundUpSize(long size) {
+ // Round up to the nearest 8 bytes. Experimentally, this
+ // is what we've seen the sun 32 bit VM do with object size.
+ // See https://wiki.gemstone.com/display/rusage/Per+Entry+Overhead
+ long remainder = size % 8;
+ if (remainder != 0) {
+ size += 8 - remainder;
+ }
+ return size;
+ }
+
+ public static int roundUpSize(int size) {
+ return (int) roundUpSize((long) size);
+ }
+
+ /**
+ * Returns the amount of memory used to store the given
+ * byte array. Note that this does include the
+ * memory used to store the bytes in the array.
+ */
+ public static int memoryOverhead(byte[] byteArray) {
+ if (byteArray == null) {
+ return 0;
+ }
+ int result = getObjectHeaderSize();
+ result += 4; // array length field
+ result += byteArray.length;
+ return roundUpSize(result);
+ }
+
+ /**
+ * Returns the amount of memory used to store the given
+ * object array. Note that this does not include the
+ * memory used by objects referenced by the array.
+ */
+ public static int memoryOverhead(Object[] objectArray) {
+ if (objectArray == null) {
+ return 0;
+ }
+ int result = getObjectHeaderSize();
+ result += 4; // array length field
+ result += objectArray.length * getReferenceSize();
+ return roundUpSize(result);
+ }
+
+ /**
+ * Returns the amount of memory used to store an instance
+ * of the given class. Note that this does not include
+ * memory used by objects referenced from fields of the
+ * instance.
+ */
+ public static int memoryOverhead(Class<?> clazz) {
+ return (int) sizeof(clazz);
+ }
}
diff --git
a/geode-core/src/main/java/org/apache/geode/internal/size/ReflectionSingleObjectSizer.java
b/geode-core/src/main/java/org/apache/geode/internal/size/ReflectionSingleObjectSizer.java
index f1602e0..c88d71c 100644
---
a/geode-core/src/main/java/org/apache/geode/internal/size/ReflectionSingleObjectSizer.java
+++
b/geode-core/src/main/java/org/apache/geode/internal/size/ReflectionSingleObjectSizer.java
@@ -14,6 +14,8 @@
*/
package org.apache.geode.internal.size;
+import static org.apache.geode.internal.JvmSizeUtils.roundUpSize;
+
import java.lang.reflect.Array;
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
@@ -143,17 +145,6 @@ public class ReflectionSingleObjectSizer implements
SingleObjectSizer {
return size;
}
- public static long roundUpSize(long size) {
- // Round up to the nearest 8 bytes. Experimentally, this
- // is what we've seen the sun 32 bit VM do with object size.
- // See https://wiki.gemstone.com/display/rusage/Per+Entry+Overhead
- long remainder = size % 8;
- if (remainder != 0) {
- size = size - remainder + 8;
- }
- return size;
- }
-
private static int sizeType(Class<?> t) {
if (t == Boolean.TYPE) {
diff --git
a/geode-core/src/main/java/org/apache/geode/internal/size/WellKnownClassSizer.java
b/geode-core/src/main/java/org/apache/geode/internal/size/WellKnownClassSizer.java
index 96b3a97..513ab3c 100644
---
a/geode-core/src/main/java/org/apache/geode/internal/size/WellKnownClassSizer.java
+++
b/geode-core/src/main/java/org/apache/geode/internal/size/WellKnownClassSizer.java
@@ -15,6 +15,9 @@
package org.apache.geode.internal.size;
+
+import static org.apache.geode.internal.JvmSizeUtils.roundUpSize;
+
/**
* An efficient sizer for some commonly used classes.
*
@@ -38,7 +41,7 @@ public class WellKnownClassSizer {
}
public static int sizeof(Object o) {
- int size = 0;
+ int size;
if (o instanceof byte[]) {
size = BYTE_ARRAY_OVERHEAD + ((byte[]) o).length;
@@ -48,8 +51,7 @@ public class WellKnownClassSizer {
return 0;
}
- size = (int) ReflectionSingleObjectSizer.roundUpSize(size);
- return size;
+ return roundUpSize(size);
}
}
