Repository: incubator-usergrid Updated Branches: refs/heads/two-dot-o-dev 45ce5d288 -> 42f45efe8
Fixes issue with invalid UUID test Fixes GeoIT tests Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/29e885ff Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/29e885ff Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/29e885ff Branch: refs/heads/two-dot-o-dev Commit: 29e885ff2f002ecc8809ba51573bc33022ae6ff6 Parents: bfe1337 Author: Todd Nine <tn...@apigee.com> Authored: Mon Jun 1 13:37:15 2015 -0600 Committer: Todd Nine <tn...@apigee.com> Committed: Mon Jun 1 13:37:15 2015 -0600 ---------------------------------------------------------------------- .../corepersistence/CpEntityManager.java | 2 +- .../corepersistence/util/CpNamingUtils.java | 5 +- .../org/apache/usergrid/utils/UUIDUtils.java | 32 ++--------- .../org/apache/usergrid/persistence/GeoIT.java | 5 +- .../apache/usergrid/utils/UUIDUtilsTest.java | 26 ++++----- stack/core/src/test/resources/log4j.properties | 1 + .../persistence/model/util/UUIDGenerator.java | 58 ++------------------ .../impl/SearchRequestBuilderStrategy.java | 50 +++++++---------- 8 files changed, 48 insertions(+), 131 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/29e885ff/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java index 14d1bed..50f04ed 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java @@ -2500,7 +2500,7 @@ public class CpEntityManager implements EntityManager { long timestamp = UUIDUtils.getUUIDLong( timestampUuid ); - UUID itemId = UUIDUtils.newTimeUUID(); + UUID itemId = UUIDGenerator.newTimeUUID(); if ( is_application ) { itemId = applicationId; http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/29e885ff/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java index 31dabea..471eda6 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java @@ -39,6 +39,7 @@ import org.apache.usergrid.persistence.map.MapScope; import org.apache.usergrid.persistence.map.impl.MapScopeImpl; import org.apache.usergrid.persistence.model.entity.Id; import org.apache.usergrid.persistence.model.entity.SimpleId; +import org.apache.usergrid.persistence.model.util.UUIDGenerator; import org.apache.usergrid.utils.UUIDUtils; @@ -141,7 +142,7 @@ public class CpNamingUtils { //if they don't use a time based uuid (such as in devices) we need to create a timestamp from "now" since // this is when the entity //will be added to the collection - final UUID timeStampUuid = UUIDUtils.isTimeBased( entityIdUUID ) ? entityIdUUID : UUIDUtils.newTimeUUID(); + final UUID timeStampUuid = UUIDUtils.isTimeBased( entityIdUUID ) ? entityIdUUID : UUIDGenerator.newTimeUUID(); long uuidTimestamp = UUIDUtils.getUUIDLong( timeStampUuid ); @@ -167,7 +168,7 @@ public class CpNamingUtils { final String edgeType = getEdgeTypeFromConnectionType( connectionType ); // create graph edge connection from head entity to member entity - return new SimpleEdge( sourceEntityId, edgeType, targetEntityId, System.currentTimeMillis() ); + return new SimpleEdge( sourceEntityId, edgeType, targetEntityId, UUIDGenerator.newTimeUUID().timestamp() ); } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/29e885ff/stack/core/src/main/java/org/apache/usergrid/utils/UUIDUtils.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/utils/UUIDUtils.java b/stack/core/src/main/java/org/apache/usergrid/utils/UUIDUtils.java index ecb1f61..adde4b6 100644 --- a/stack/core/src/main/java/org/apache/usergrid/utils/UUIDUtils.java +++ b/stack/core/src/main/java/org/apache/usergrid/utils/UUIDUtils.java @@ -37,6 +37,8 @@ import static com.fasterxml.uuid.impl.UUIDUtil.BYTE_OFFSET_CLOCK_SEQUENCE; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.usergrid.persistence.model.util.UUIDGenerator; + import static org.apache.commons.codec.binary.Base64.decodeBase64; import static org.apache.commons.codec.binary.Base64.encodeBase64URLSafeString; @@ -82,27 +84,8 @@ public class UUIDUtils { * 'unique.' */ public static java.util.UUID newTimeUUID() { - // get & inc counter, but roll on 1k (because we divide by 10 on retrieval) - // if count + currentMicro > 1k, block and roll - tsLock.lock(); - long ts = System.currentTimeMillis(); - if ( ts > timestampMillisNow ) { - timestampMillisNow = ts; - currentMicrosPoint.set( 0 ); - } - int pointer = currentMicrosPoint.getAndIncrement(); - try { - if ( pointer > 990 ) { - TimeUnit.MILLISECONDS.sleep( 1L ); - } - } - catch ( Exception ex ) { - ex.printStackTrace(); - } - finally { - tsLock.unlock(); - } - return newTimeUUID( ts, MICROS[pointer] ); + //replaced with our new impl. The old impl does not properly generate sequential times when under heavy load in the same millisecond. Causes edge ordering issues. See USERGRID-628 + return UUIDGenerator.newTimeUUID(); } @@ -210,13 +193,6 @@ public class UUIDUtils { } - public static UUID minTimeUUID( long ts ) { - byte[] uuidBytes = new byte[16]; - setTimestamp( ts, uuidBytes, 0, 0 ); - - return uuid( uuidBytes ); - } - public static UUID maxTimeUUID( long ts ) { byte[] uuidBytes = new byte[16]; http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/29e885ff/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java ---------------------------------------------------------------------- diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java index c837ab7..542a1c5 100644 --- a/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java +++ b/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java @@ -441,6 +441,7 @@ public class GeoIT extends AbstractCoreIT { setPos(data, 0, 0); em.create("store", data); +// Thread.sleep(1); } app.refreshIndex(); @@ -459,7 +460,9 @@ public class GeoIT extends AbstractCoreIT { for (Entity entity : results.getEntities()) { //TODO:can we assert order - assertEquals(String.valueOf(count), entity.getName()); + final int expected = numEntities - count - 1; + + assertEquals(String.valueOf(expected), entity.getName()); count++; } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/29e885ff/stack/core/src/test/java/org/apache/usergrid/utils/UUIDUtilsTest.java ---------------------------------------------------------------------- diff --git a/stack/core/src/test/java/org/apache/usergrid/utils/UUIDUtilsTest.java b/stack/core/src/test/java/org/apache/usergrid/utils/UUIDUtilsTest.java index e59dac8..0493d31 100644 --- a/stack/core/src/test/java/org/apache/usergrid/utils/UUIDUtilsTest.java +++ b/stack/core/src/test/java/org/apache/usergrid/utils/UUIDUtilsTest.java @@ -22,7 +22,6 @@ import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.UUID; -import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; @@ -166,10 +165,10 @@ public class UUIDUtilsTest { /** Populate timestamp set for the methods testing uuid contention */ @SuppressWarnings("unchecked") - private static Set buildTsMicros( int count ) { - HashSet created = new HashSet( count ); + private static Set<UUID> buildTsMicros( int count ) { + HashSet<UUID> created = new HashSet<>( count ); for ( int x = 0; x < count; x++ ) { - created.add( UUIDUtils.getTimestampInMicros( UUIDUtils.newTimeUUID() ) ); + created.add( UUIDUtils.newTimeUUID() ); } return created; } @@ -207,20 +206,17 @@ public class UUIDUtilsTest { List<Future> jobs = new ArrayList<Future>( 10 ); for ( int x = 0; x < 10; x++ ) { - jobs.add( exec.submit( new Callable<Object>() { - @Override - public Object call() throws Exception { - logger.info("call invoked"); + jobs.add( exec.submit( () -> { + logger.info("call invoked"); - int count = 1000 * 100; - Set created = buildTsMicros( count ); + int count = 1000 * 100; + Set created = buildTsMicros( count ); - assertEquals( count, created.size() ); - assertTrue( created.size() > 0 ); + assertEquals( count, created.size() ); + assertTrue( created.size() > 0 ); - logger.info("run complete"); - return null; - } + logger.info("run complete"); + return null; } ) ); } return jobs; http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/29e885ff/stack/core/src/test/resources/log4j.properties ---------------------------------------------------------------------- diff --git a/stack/core/src/test/resources/log4j.properties b/stack/core/src/test/resources/log4j.properties index 34bd708..dfcb260 100644 --- a/stack/core/src/test/resources/log4j.properties +++ b/stack/core/src/test/resources/log4j.properties @@ -66,3 +66,4 @@ log4j.logger.org.apache.usergrid.persistence.index.impl.EsApplicationEntityIndex #log4j.logger.org.apache.cassandra.service.StorageProxy=DEBUG, stdout log4j.logger.org.apache.usergrid.corepersistence.index.IndexServiceImpl=DEBUG +log4j.logger.org.apache.usergrid.utils.UUIDUtilsTest=DEBUG http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/29e885ff/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/util/UUIDGenerator.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/util/UUIDGenerator.java b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/util/UUIDGenerator.java index f2c4fa8..176c97b 100644 --- a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/util/UUIDGenerator.java +++ b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/util/UUIDGenerator.java @@ -4,86 +4,36 @@ package org.apache.usergrid.persistence.model.util; import java.io.IOException; import java.util.Random; import java.util.UUID; -import java.util.concurrent.atomic.AtomicInteger; import com.fasterxml.uuid.EthernetAddress; -import com.fasterxml.uuid.TimestampSynchronizer; import com.fasterxml.uuid.UUIDTimer; import com.fasterxml.uuid.impl.TimeBasedGenerator; /** * TODO replace this with the Astyanax generator libs - * @author: tnine - * */ public class UUIDGenerator { - private static final TimestampSynchronizer synchronize = new TimestampSynchronizer() { - - /** - * Pointer to the last value we returned - */ - private long last = 0; - - /** - * The number of ticks that can be used in the millisecond. In a time UUID a tick is divided into 1/10000 of - * a millisecond - * - */ - private AtomicInteger ticks = new AtomicInteger(); - - - @Override - protected long initialize() throws IOException { - - last = System.currentTimeMillis(); - return last; - } - - - @Override - protected void deactivate() throws IOException { - //no op - } - - - @Override - protected long update( long now ) throws IOException { - /** - * Our timestamp is greater just use that and reset last - */ - if ( now > last ) { - last = now; - ticks.set( 0 ); - return last; - } - - //we have the same value (since now should always be increasing) increment a tick - return last + ticks.incrementAndGet(); - } - }; - - private static final Random random = new Random(); private static final UUIDTimer timer; - /** * Lame, but required */ static { try { - timer = new UUIDTimer( random, synchronize ); + timer = new UUIDTimer( random, null ); } catch ( IOException e ) { - throw new RuntimeException( "Couldn't intialize timer", e ); + throw new RuntimeException( "Couldn't initialize timer", e ); } } - private static final TimeBasedGenerator generator = new TimeBasedGenerator( EthernetAddress.fromInterface(), timer ); + private static final TimeBasedGenerator generator = + new TimeBasedGenerator( EthernetAddress.fromInterface(), timer ); /** Create a new time uuid */ http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/29e885ff/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java index 33a9adf..2ac6bcb 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java @@ -20,12 +20,8 @@ package org.apache.usergrid.persistence.index.impl; -import java.util.Set; - import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchType; -import org.elasticsearch.common.geo.GeoDistance; -import org.elasticsearch.common.unit.DistanceUnit; import org.elasticsearch.index.query.BoolFilterBuilder; import org.elasticsearch.index.query.FilterBuilder; import org.elasticsearch.index.query.FilterBuilders; @@ -82,13 +78,14 @@ public class SearchRequestBuilderStrategy { * Get the search request builder */ public SearchRequestBuilder getBuilder( final SearchEdge searchEdge, final SearchTypes searchTypes, - final ParsedQuery query, final int limit, final int from) { + final ParsedQuery query, final int limit, final int from ) { Preconditions - .checkArgument( limit <= EntityIndex.MAX_LIMIT, "limit is greater than max " + EntityIndex.MAX_LIMIT ); + .checkArgument( limit <= EntityIndex.MAX_LIMIT, "limit is greater than max " + EntityIndex.MAX_LIMIT ); SearchRequestBuilder srb = - esProvider.getClient().prepareSearch( alias.getReadAlias() ).setTypes( IndexingUtils.ES_ENTITY_TYPE ).setSearchType(SearchType.QUERY_THEN_FETCH); + esProvider.getClient().prepareSearch( alias.getReadAlias() ).setTypes( IndexingUtils.ES_ENTITY_TYPE ) + .setSearchType( SearchType.QUERY_THEN_FETCH ); final QueryVisitor visitor = visitParsedQuery( query ); @@ -126,28 +123,22 @@ public class SearchRequestBuilderStrategy { * Apply our default sort predicate logic */ private void applyDefaultSortPredicates( final SearchRequestBuilder srb, final GeoSortFields geoFields ) { - - - if ( geoFields.isEmpty() ) { - - //sort by the edge timestamp - srb.addSort( SortBuilders.fieldSort( IndexingUtils.EDGE_TIMESTAMP_FIELDNAME ).order( SortOrder.DESC ) ); - - //sort by the entity id if our times are equal - srb.addSort( SortBuilders.fieldSort( IndexingUtils.ENTITY_ID_FIELDNAME ).order( SortOrder.ASC ) ); - - return; - } - //we have geo fields, sort through them in visit order - - for ( String geoField : geoFields.fields() ) { final GeoDistanceSortBuilder geoSort = geoFields.applyOrder( geoField, SortOrder.ASC ); srb.addSort( geoSort ); } + + //now sort by edge timestamp, then entity id + //sort by the edge timestamp + srb.addSort( SortBuilders.fieldSort( IndexingUtils.EDGE_TIMESTAMP_FIELDNAME ).order( SortOrder.DESC ) ); + + //sort by the entity id if our times are equal + srb.addSort( SortBuilders.fieldSort( IndexingUtils.ENTITY_ID_FIELDNAME ).order( SortOrder.ASC ) ); + + return; } @@ -170,9 +161,10 @@ public class SearchRequestBuilderStrategy { final String propertyName = sp.getPropertyName(); - //if the user specified a geo field in their sort, then honor their sort order and use the point they specified + //if the user specified a geo field in their sort, then honor their sort order and use the point they + // specified if ( geoFields.contains( propertyName ) ) { - final GeoDistanceSortBuilder geoSort = geoFields.applyOrder(propertyName, SortOrder.ASC); + final GeoDistanceSortBuilder geoSort = geoFields.applyOrder( propertyName, SortOrder.ASC ); srb.addSort( geoSort ); } @@ -202,8 +194,8 @@ public class SearchRequestBuilderStrategy { * Create our filter builder. We need to restrict our results on edge search, as well as on types, and any filters * that came from the grammar. */ - private FilterBuilder createFilterBuilder( final SearchEdge searchEdge, final QueryVisitor visitor, final - SearchTypes searchTypes ) { + private FilterBuilder createFilterBuilder( final SearchEdge searchEdge, final QueryVisitor visitor, + final SearchTypes searchTypes ) { String context = createContextName( applicationScope, searchEdge ); @@ -227,7 +219,7 @@ public class SearchRequestBuilderStrategy { final String[] sourceTypes = searchTypes.getTypeNames( applicationScope ); - if(sourceTypes.length > 0 ) { + if ( sourceTypes.length > 0 ) { final FilterBuilder[] typeTerms = new FilterBuilder[sourceTypes.length]; for ( int i = 0; i < sourceTypes.length; i++ ) { @@ -270,7 +262,6 @@ public class SearchRequestBuilderStrategy { } - /** * Create a sort for the property name and field name specified * @@ -284,7 +275,6 @@ public class SearchRequestBuilderStrategy { final TermFilterBuilder propertyFilter = sortPropertyTermFilter( propertyName ); - return SortBuilders.fieldSort( fieldName ).order( sortOrder ) - .setNestedFilter( propertyFilter ); + return SortBuilders.fieldSort( fieldName ).order( sortOrder ).setNestedFilter( propertyFilter ); } }
