Fix issue with map manager key sorting. Enhance buffer sizing by using byte length of string components.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/fcbe8036 Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/fcbe8036 Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/fcbe8036 Branch: refs/heads/asf-site Commit: fcbe803679ab4832a9ec2b3d26fd35eaec61202d Parents: 92d5f40 Author: Michael Russo <[email protected]> Authored: Thu Aug 18 23:55:30 2016 -0700 Committer: Michael Russo <[email protected]> Committed: Thu Aug 18 23:55:30 2016 -0700 ---------------------------------------------------------------------- .../impl/ScopedCacheSerializationImpl.java | 2 +- .../UniqueValueSerializationStrategyV1Impl.java | 23 ++++++---- .../UniqueValueSerializationStrategyV2Impl.java | 20 ++++---- .../usergrid/persistence/map/MapManager.java | 2 +- .../map/impl/MapSerializationImpl.java | 4 +- .../persistence/map/MapManagerTest.java | 48 +++++++++++++++++--- 6 files changed, 70 insertions(+), 29 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/fcbe8036/stack/corepersistence/cache/src/main/java/org/apache/usergrid/persistence/cache/impl/ScopedCacheSerializationImpl.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/cache/src/main/java/org/apache/usergrid/persistence/cache/impl/ScopedCacheSerializationImpl.java b/stack/corepersistence/cache/src/main/java/org/apache/usergrid/persistence/cache/impl/ScopedCacheSerializationImpl.java index 1334650..2a44f2b 100644 --- a/stack/corepersistence/cache/src/main/java/org/apache/usergrid/persistence/cache/impl/ScopedCacheSerializationImpl.java +++ b/stack/corepersistence/cache/src/main/java/org/apache/usergrid/persistence/cache/impl/ScopedCacheSerializationImpl.java @@ -303,7 +303,7 @@ public class ScopedCacheSerializationImpl<K,V> implements ScopedCacheSerializati keys.add(3, rowKeyString); // UUIDs are 16 bytes, allocate the buffer accordingly - int size = 16+ownerType.length()+rowKeyString.length(); + int size = 16+ownerType.getBytes().length+rowKeyString.getBytes().length; // ints are 4 bytes, add for the bucket size += 4; http://git-wip-us.apache.org/repos/asf/usergrid/blob/fcbe8036/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV1Impl.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV1Impl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV1Impl.java index 55ba011..1435c1e 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV1Impl.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV1Impl.java @@ -198,14 +198,15 @@ public class UniqueValueSerializationStrategyV1Impl extends UniqueValueSerializ String comparator = UUID_TYPE_REVERSED; - int size = 16+fieldEntry.getField().getName().length()+fieldEntry.getField().getValue().toString().length()+ - fieldEntry.getField().getTypeName().name().length(); + int size = 16+fieldEntry.getField().getName().getBytes().length + +fieldEntry.getField().getValue().toString().getBytes().length+ + fieldEntry.getField().getTypeName().name().getBytes().length; // we always need to add length for the 2 byte comparator short, 2 byte length short and 1 byte equality size += keys.size()*5; // uuid type comparator is longest, ensure we allocate buffer using the max size to avoid overflow - size += keys.size()*comparator.length(); + size += keys.size()*comparator.getBytes().length; ByteBuffer stuff = ByteBuffer.allocate(size); @@ -228,7 +229,7 @@ public class UniqueValueSerializationStrategyV1Impl extends UniqueValueSerializ stuff.putShort((short) ('è' | a)); }else{ comparator = "UTF8Type"; // only strings are being serialized other than UUIDs here - stuff.putShort((short)comparator.length()); + stuff.putShort((short)comparator.getBytes().length); stuff.put(DataType.serializeValue(comparator, ProtocolVersion.NEWEST_SUPPORTED)); } @@ -293,13 +294,13 @@ public class UniqueValueSerializationStrategyV1Impl extends UniqueValueSerializ keys.add(entityVersion.getEntityId().getType()); // UUIDs are 16 bytes - int size = 16+16+entityVersion.getEntityId().getType().length(); + int size = 16+16+entityVersion.getEntityId().getType().getBytes().length; // we always need to add length for the 2 byte comparator short, 2 byte length short and 1 byte equality size += keys.size()*5; // we always add comparator to the buffer as well - size += keys.size()*comparator.length(); + size += keys.size()*comparator.getBytes().length; ByteBuffer stuff = ByteBuffer.allocate(size); @@ -311,7 +312,7 @@ public class UniqueValueSerializationStrategyV1Impl extends UniqueValueSerializ comparator = "UTF8Type"; // if it's not a UUID, the only other thing we're serializing is text } - stuff.putShort((short)comparator.length()); + stuff.putShort((short)comparator.getBytes().length); stuff.put(DataType.serializeValue(comparator, ProtocolVersion.NEWEST_SUPPORTED)); ByteBuffer kb = DataType.serializeValue(key, ProtocolVersion.NEWEST_SUPPORTED); @@ -457,8 +458,9 @@ public class UniqueValueSerializationStrategyV1Impl extends UniqueValueSerializ // UUIDs are 16 bytes, allocate the buffer accordingly - int size = 16 + applicationType.length() + 16 + applicationType.length() + collectionName.length() + - fieldType.length() + fieldName.length()+fieldValueString.length(); + int size = 16 + applicationType.getBytes().length + 16 + applicationType.getBytes().length + + collectionName.getBytes().length + fieldType.getBytes().length + fieldName.getBytes().length + + fieldValueString.getBytes().length; // we always need to add length for the 2 byte short and 1 byte equality @@ -503,7 +505,8 @@ public class UniqueValueSerializationStrategyV1Impl extends UniqueValueSerializ keys.add(entityId); keys.add(entityType); - int size = 16+applicationType.length()+16+applicationType.length()+collectionName.length()+16+entityType.length(); + int size = 16+applicationType.getBytes().length+16+applicationType.getBytes().length + +collectionName.getBytes().length+16+entityType.getBytes().length; // we always need to add length for the 2 byte short and 1 byte equality size += keys.size()*3; http://git-wip-us.apache.org/repos/asf/usergrid/blob/fcbe8036/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV2Impl.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV2Impl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV2Impl.java index a5fceeb..518937d 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV2Impl.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV2Impl.java @@ -187,14 +187,15 @@ public class UniqueValueSerializationStrategyV2Impl extends UniqueValueSerializ String comparator = UUID_TYPE_REVERSED; - int size = 16+fieldEntry.getField().getName().length()+fieldEntry.getField().getValue().toString().length()+ - fieldEntry.getField().getTypeName().name().length(); + int size = 16+fieldEntry.getField().getName().getBytes().length + +fieldEntry.getField().getValue().toString().getBytes().length+ + fieldEntry.getField().getTypeName().name().getBytes().length; // we always need to add length for the 2 byte comparator short, 2 byte length short and 1 byte equality size += keys.size()*5; // uuid type comparator is longest, ensure we allocate buffer using the max size to avoid overflow - size += keys.size()*comparator.length(); + size += keys.size()*comparator.getBytes().length; ByteBuffer stuff = ByteBuffer.allocate(size); @@ -217,7 +218,7 @@ public class UniqueValueSerializationStrategyV2Impl extends UniqueValueSerializ stuff.putShort((short) ('è' | a)); }else{ comparator = "UTF8Type"; // only strings are being serialized other than UUIDs here - stuff.putShort((short)comparator.length()); + stuff.putShort((short)comparator.getBytes().length); stuff.put(DataType.serializeValue(comparator, ProtocolVersion.NEWEST_SUPPORTED)); } @@ -282,13 +283,13 @@ public class UniqueValueSerializationStrategyV2Impl extends UniqueValueSerializ keys.add(entityVersion.getEntityId().getType()); // UUIDs are 16 bytes - int size = 16+16+entityVersion.getEntityId().getType().length(); + int size = 16+16+entityVersion.getEntityId().getType().getBytes().length; // we always need to add length for the 2 byte comparator short, 2 byte length short and 1 byte equality size += keys.size()*5; // we always add comparator to the buffer as well - size += keys.size()*comparator.length(); + size += keys.size()*comparator.getBytes().length; ByteBuffer stuff = ByteBuffer.allocate(size); @@ -303,7 +304,7 @@ public class UniqueValueSerializationStrategyV2Impl extends UniqueValueSerializ comparator = "UTF8Type"; // if it's not a UUID, the only other thing we're serializing is text } - stuff.putShort((short)comparator.length()); + stuff.putShort((short)comparator.getBytes().length); stuff.put(DataType.serializeValue(comparator, ProtocolVersion.NEWEST_SUPPORTED)); ByteBuffer kb = DataType.serializeValue(key, ProtocolVersion.NEWEST_SUPPORTED); @@ -436,7 +437,8 @@ public class UniqueValueSerializationStrategyV2Impl extends UniqueValueSerializ // UUIDs are 16 bytes, allocate the buffer accordingly - int size = 16 + applicationType.length() + entityType.length() + fieldType.length() + fieldName.length()+fieldValueString.length(); + int size = 16 + applicationType.getBytes().length + entityType.getBytes().length + + fieldType.getBytes().length + fieldName.getBytes().length+fieldValueString.getBytes().length; // we always need to add length for the 2 byte short and 1 byte equality @@ -470,7 +472,7 @@ public class UniqueValueSerializationStrategyV2Impl extends UniqueValueSerializ keys.add(entityId); keys.add(entityType); - int size = 16+applicationType.length()+16+entityType.length(); + int size = 16+applicationType.getBytes().length+16+entityType.getBytes().length; // we always need to add length for the 2 byte short and 1 byte equality size += keys.size()*3; http://git-wip-us.apache.org/repos/asf/usergrid/blob/fcbe8036/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/MapManager.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/MapManager.java b/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/MapManager.java index e1d121e..bd1f444 100644 --- a/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/MapManager.java +++ b/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/MapManager.java @@ -92,7 +92,7 @@ public interface MapManager { void delete( final String key ); /** - * Return a page of keys that exist within the map. Keys are sorted descending. + * Return a page of keys that exist within the map. Keys are sorted ascending in lexicographical ordering. * @param cursor * @param limit */ http://git-wip-us.apache.org/repos/asf/usergrid/blob/fcbe8036/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerializationImpl.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerializationImpl.java b/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerializationImpl.java index 282974c..2c77ebf64 100644 --- a/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerializationImpl.java +++ b/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerializationImpl.java @@ -70,7 +70,7 @@ public class MapSerializationImpl implements MapSerialization { put( "column1", DataType.Name.BLOB ); put( "value", DataType.Name.BLOB ); }}; private static final Map<String, String> MAP_KEYS_CLUSTERING_ORDER = - new HashMap<String, String>(){{ put( "column1", "DESC" ); }}; + new HashMap<String, String>(){{ put( "column1", "ASC" ); }}; @@ -465,7 +465,7 @@ public class MapSerializationImpl implements MapSerialization { } // UUIDs are 16 bytes, allocate the buffer accordingly - int size = 16+ownerType.length()+mapName.length()+mapKey.length(); + int size = 16+ownerType.getBytes().length+mapName.getBytes().length+mapKey.getBytes().length; if(bucketNumber > 0 ){ // ints are 4 bytes size += 4; http://git-wip-us.apache.org/repos/asf/usergrid/blob/fcbe8036/stack/corepersistence/map/src/test/java/org/apache/usergrid/persistence/map/MapManagerTest.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/map/src/test/java/org/apache/usergrid/persistence/map/MapManagerTest.java b/stack/corepersistence/map/src/test/java/org/apache/usergrid/persistence/map/MapManagerTest.java index 89f6799..07bec81 100644 --- a/stack/corepersistence/map/src/test/java/org/apache/usergrid/persistence/map/MapManagerTest.java +++ b/stack/corepersistence/map/src/test/java/org/apache/usergrid/persistence/map/MapManagerTest.java @@ -41,6 +41,7 @@ import com.google.inject.Inject; import static junit.framework.TestCase.assertNotNull; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; @RunWith( ITRunner.class ) @@ -121,20 +122,20 @@ public class MapManagerTest { MapKeyResults keyResults = mm.getKeys(null, 3); assertEquals(3, keyResults.getKeys().size()); - assertEquals(key6, keyResults.getKeys().get(0)); + assertTrue("should contain key1", keyResults.getKeys().contains(key1)); assertNotNull(keyResults.getCursor()); MapKeyResults keyResults2 = mm.getKeys(keyResults.getCursor(), 3); assertEquals(3, keyResults2.getKeys().size()); - assertEquals(key3, keyResults2.getKeys().get(0)); + assertTrue("should contain key4", keyResults2.getKeys().contains(key4)); } @Test - public void getAllKeysAfterDelete(){ + public void testKeysOrdering(){ MapManager mm = mmf.createMapManager(this.scope); @@ -157,8 +158,45 @@ public class MapManagerTest { MapKeyResults keyResults = mm.getKeys(null, 6); assertEquals(6, keyResults.getKeys().size()); - assertEquals(key6, keyResults.getKeys().get(0)); + assertEquals(key1, keyResults.getKeys().get(0)); + + + mm.delete(key1); + mm.delete(key2); + mm.delete(key3); + + MapKeyResults keyResults2 = mm.getKeys(null, 6); + + assertEquals(3, keyResults2.getKeys().size()); + assertEquals(key4, keyResults2.getKeys().get(0)); + + + } + @Test + public void getAllKeysAfterDelete(){ + + MapManager mm = mmf.createMapManager(this.scope); + + final String value = "value"; + + final String key1 = "key1"; + final String key2 = "key2"; + final String key3 = "key3"; + final String key4 = "key4"; + final String key5 = "key5"; + final String key6 = "key6"; + + mm.putString( key1, value ); + mm.putString( key2, value ); + mm.putString( key3, value ); + mm.putString( key4, value ); + mm.putString( key5, value ); + mm.putString( key6, value ); + + MapKeyResults keyResults = mm.getKeys(null, 6); + + assertEquals(6, keyResults.getKeys().size()); mm.delete(key4); mm.delete(key5); @@ -167,8 +205,6 @@ public class MapManagerTest { MapKeyResults keyResults2 = mm.getKeys(null, 6); assertEquals(3, keyResults2.getKeys().size()); - assertEquals(key3, keyResults2.getKeys().get(0)); - }
