change entityindex interface
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/8a62d117 Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/8a62d117 Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/8a62d117 Branch: refs/heads/two-dot-o-dev Commit: 8a62d1177783612189be49fb924dfead3f348e2f Parents: e5eb57f Author: Shawn Feldman <[email protected]> Authored: Wed Mar 25 14:26:10 2015 -0600 Committer: Shawn Feldman <[email protected]> Committed: Wed Mar 25 14:26:10 2015 -0600 ---------------------------------------------------------------------- .../corepersistence/CpEntityManager.java | 122 ++++++------ .../index/ApplicationEntityIndex.java | 13 +- .../impl/EsApplicationEntityIndexImpl.java | 199 +++++++------------ .../impl/SearchRequestBuilderStrategy.java | 197 ++++++++++++++++++ .../index/query/CandidateResults.java | 24 ++- .../usergrid/persistence/index/query/Query.java | 160 +-------------- .../persistence/index/impl/EntityIndexTest.java | 28 +-- .../persistence/query/tree/GrammarTreeTest.java | 8 +- 8 files changed, 382 insertions(+), 369 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8a62d117/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java index f420d7b..481fff5 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java @@ -753,14 +753,14 @@ public class CpEntityManager implements EntityManager { @Override public void updateApplication( Application app ) throws Exception { - update( app ); + update(app); this.application = app; } @Override public void updateApplication( Map<String, Object> properties ) throws Exception { - this.updateProperties( new SimpleEntityRef( Application.ENTITY_TYPE, applicationId ), properties ); + this.updateProperties(new SimpleEntityRef(Application.ENTITY_TYPE, applicationId), properties); this.application = get( applicationId, Application.class ); } @@ -857,7 +857,7 @@ public class CpEntityManager implements EntityManager { @Override public EntityRef getAlias( String aliasType, String alias ) throws Exception { - return getAlias( new SimpleEntityRef( Application.ENTITY_TYPE, applicationId ), aliasType, alias ); + return getAlias(new SimpleEntityRef(Application.ENTITY_TYPE, applicationId), aliasType, alias); } @@ -886,7 +886,7 @@ public class CpEntityManager implements EntityManager { new Object[] { ownerRef, collectionType, aliasValue } ); } - return results.get( aliasValue ); + return results.get(aliasValue); } @@ -936,7 +936,7 @@ public class CpEntityManager implements EntityManager { } - return Collections.<EntityRef>singleton( new SimpleEntityRef( id.getType(), id.getUuid() ) ); + return Collections.<EntityRef>singleton(new SimpleEntityRef(id.getType(), id.getUuid())); } @@ -1003,7 +1003,7 @@ public class CpEntityManager implements EntityManager { public void setProperty( EntityRef entityRef, String propertyName, Object propertyValue ) throws Exception { - setProperty( entityRef, propertyName, propertyValue, false ); + setProperty(entityRef, propertyName, propertyValue, false); } @@ -1021,7 +1021,7 @@ public class CpEntityManager implements EntityManager { entity.getType(), propertyName, propertyValue ); entity.setProperty( propertyName, propertyValue ); - entity.setProperty( PROPERTY_MODIFIED, UUIDUtils.getTimestampInMillis( UUIDUtils.newTimeUUID() ) ); + entity.setProperty(PROPERTY_MODIFIED, UUIDUtils.getTimestampInMillis(UUIDUtils.newTimeUUID())); update( entity ); } @@ -1127,7 +1127,7 @@ public class CpEntityManager implements EntityManager { public void addToDictionary( EntityRef entityRef, String dictionaryName, Object elementValue ) throws Exception { - addToDictionary( entityRef, dictionaryName, elementValue, null ); + addToDictionary(entityRef, dictionaryName, elementValue, null); } @@ -1188,7 +1188,7 @@ public class CpEntityManager implements EntityManager { EntityRef entity = get( entityRef ); UUID timestampUuid = UUIDUtils.newTimeUUID(); - Mutator<ByteBuffer> batch = createMutator( cass.getApplicationKeyspace( applicationId ), be ); + Mutator<ByteBuffer> batch = createMutator(cass.getApplicationKeyspace(applicationId), be); for ( Map.Entry<?, ?> elementValue : elementValues.entrySet() ) { batch = batchUpdateDictionary( batch, entity, dictionaryName, elementValue.getKey(), @@ -1225,8 +1225,8 @@ public class CpEntityManager implements EntityManager { boolean coTypeIsBasic = ClassUtils.isBasicType( setCoType ); List<HColumn<ByteBuffer, ByteBuffer>> results = - cass.getAllColumns( cass.getApplicationKeyspace( applicationId ), dictionaryCf, - CassandraPersistenceUtils.key( entity.getUuid(), dictionaryName ), be, be ); + cass.getAllColumns(cass.getApplicationKeyspace(applicationId), dictionaryCf, + CassandraPersistenceUtils.key(entity.getUuid(), dictionaryName), be, be); for ( HColumn<ByteBuffer, ByteBuffer> result : results ) { Object name = null; if ( entityHasDictionary ) { @@ -1275,7 +1275,7 @@ public class CpEntityManager implements EntityManager { ApplicationCF dictionaryCf = null; - boolean entityHasDictionary = Schema.getDefaultSchema().hasDictionary( entity.getType(), dictionaryName ); + boolean entityHasDictionary = Schema.getDefaultSchema().hasDictionary(entity.getType(), dictionaryName); if ( entityHasDictionary ) { dictionaryCf = ENTITY_DICTIONARIES; @@ -1285,14 +1285,14 @@ public class CpEntityManager implements EntityManager { } Class<?> dictionaryCoType = - Schema.getDefaultSchema().getDictionaryValueType( entity.getType(), dictionaryName ); + Schema.getDefaultSchema().getDictionaryValueType(entity.getType(), dictionaryName); boolean coTypeIsBasic = ClassUtils.isBasicType( dictionaryCoType ); HColumn<ByteBuffer, ByteBuffer> result = - cass.getColumn( cass.getApplicationKeyspace( applicationId ), dictionaryCf, - CassandraPersistenceUtils.key( entity.getUuid(), dictionaryName ), - entityHasDictionary ? bytebuffer( elementName ) : DynamicComposite.toByteBuffer( elementName ), - be, be ); + cass.getColumn(cass.getApplicationKeyspace(applicationId), dictionaryCf, + CassandraPersistenceUtils.key(entity.getUuid(), dictionaryName), + entityHasDictionary ? bytebuffer(elementName) : DynamicComposite.toByteBuffer(elementName), + be, be); if ( result != null ) { if ( entityHasDictionary && coTypeIsBasic ) { @@ -1303,7 +1303,7 @@ public class CpEntityManager implements EntityManager { } } else { - logger.info( "Results of CpEntityManagerImpl.getDictionaryElementValue is null" ); + logger.info("Results of CpEntityManagerImpl.getDictionaryElementValue is null"); } return value; @@ -1327,8 +1327,8 @@ public class CpEntityManager implements EntityManager { } Class<?> dictionaryCoType = - Schema.getDefaultSchema().getDictionaryValueType( entity.getType(), dictionaryName ); - boolean coTypeIsBasic = ClassUtils.isBasicType( dictionaryCoType ); + Schema.getDefaultSchema().getDictionaryValueType(entity.getType(), dictionaryName); + boolean coTypeIsBasic = ClassUtils.isBasicType(dictionaryCoType); ByteBuffer[] columnNames = new ByteBuffer[elementNames.length]; for ( int i = 0; i < elementNames.length; i++ ) { @@ -1337,8 +1337,8 @@ public class CpEntityManager implements EntityManager { } ColumnSlice<ByteBuffer, ByteBuffer> results = - cass.getColumns( cass.getApplicationKeyspace( applicationId ), dictionaryCf, - CassandraPersistenceUtils.key( entity.getUuid(), dictionaryName ), columnNames, be, be ); + cass.getColumns(cass.getApplicationKeyspace(applicationId), dictionaryCf, + CassandraPersistenceUtils.key(entity.getUuid(), dictionaryName), columnNames, be, be); if ( results != null ) { values = new HashMap<String, Object>(); for ( HColumn<ByteBuffer, ByteBuffer> result : results.getColumns() ) { @@ -1384,7 +1384,7 @@ public class CpEntityManager implements EntityManager { @Override public Set<String> getDictionaries( EntityRef entity ) throws Exception { - return getDictionaryNames( entity ); + return getDictionaryNames(entity); } @@ -1398,14 +1398,14 @@ public class CpEntityManager implements EntityManager { @Override public boolean isCollectionMember( EntityRef owner, String collectionName, EntityRef entity ) throws Exception { - return getRelationManager( owner ).isCollectionMember( collectionName, entity ); + return getRelationManager( owner ).isCollectionMember(collectionName, entity); } @Override public boolean isConnectionMember( EntityRef owner, String connectionName, EntityRef entity ) throws Exception { - return getRelationManager( owner ).isConnectionMember( connectionName, entity ); + return getRelationManager( owner ).isConnectionMember(connectionName, entity); } @@ -1430,7 +1430,7 @@ public class CpEntityManager implements EntityManager { throws Exception { return getRelationManager( get( entityId )) - .getCollection ( collectionName, query, resultsLevel ); + .getCollection(collectionName, query, resultsLevel); } @@ -1495,8 +1495,8 @@ public class CpEntityManager implements EntityManager { @Override public ConnectionRef createConnection( ConnectionRef connection ) throws Exception { - return createConnection( connection.getConnectingEntity(), connection.getConnectionType(), - connection.getConnectedEntity() ); + return createConnection(connection.getConnectingEntity(), connection.getConnectionType(), + connection.getConnectedEntity()); } @@ -1504,7 +1504,7 @@ public class CpEntityManager implements EntityManager { public ConnectionRef createConnection( EntityRef connectingEntity, String connectionType, EntityRef connectedEntityRef ) throws Exception { - return getRelationManager( connectingEntity ).createConnection( connectionType, connectedEntityRef ); + return getRelationManager( connectingEntity ).createConnection(connectionType, connectedEntityRef); } @@ -1514,7 +1514,7 @@ public class CpEntityManager implements EntityManager { throws Exception { return getRelationManager( connectingEntity ) - .createConnection( pairedConnectionType, pairedEntity, connectionType, connectedEntityRef ); + .createConnection(pairedConnectionType, pairedEntity, connectionType, connectedEntityRef); } @@ -1540,7 +1540,7 @@ public class CpEntityManager implements EntityManager { EntityRef pairedEntity, String connectionType, EntityRef connectedEntityRef ) throws Exception { return getRelationManager( connectingEntity ) - .connectionRef( pairedConnectionType, pairedEntity, connectionType, connectedEntityRef ); + .connectionRef(pairedConnectionType, pairedEntity, connectionType, connectedEntityRef); } @@ -1572,7 +1572,7 @@ public class CpEntityManager implements EntityManager { String connectedEntityType, Level resultsLevel ) throws Exception { return getRelationManager( entityRef ) - .getConnectedEntities( connectionType, connectedEntityType, resultsLevel ); + .getConnectedEntities(connectionType, connectedEntityType, resultsLevel); } @@ -1581,7 +1581,7 @@ public class CpEntityManager implements EntityManager { String connectedEntityType, Level resultsLevel ) throws Exception { return getRelationManager( entityRef ) - .getConnectingEntities( connectionType, connectedEntityType, resultsLevel ); + .getConnectingEntities(connectionType, connectedEntityType, resultsLevel); } @@ -1589,21 +1589,21 @@ public class CpEntityManager implements EntityManager { public Results getConnectingEntities( EntityRef entityRef, String connectionType, String entityType, Level level, int count ) throws Exception { - return getRelationManager( entityRef ).getConnectingEntities( connectionType, entityType, level, count ); + return getRelationManager( entityRef ).getConnectingEntities(connectionType, entityType, level, count); } @Override public Results searchConnectedEntities( EntityRef connectingEntity, Query query ) throws Exception { - return getRelationManager( connectingEntity ).searchConnectedEntities( query ); + return getRelationManager( connectingEntity ).searchConnectedEntities(query); } @Override public Set<String> getConnectionIndexes( EntityRef entity, String connectionType ) throws Exception { - return getRelationManager( entity ).getConnectionIndexes( connectionType ); + return getRelationManager( entity ).getConnectionIndexes(connectionType); } @@ -1829,14 +1829,14 @@ public class CpEntityManager implements EntityManager { String roleTitle = batchRoleName; String propertyName = groupId + ":" + batchRoleName; Map<String, Object> properties = new TreeMap<String, Object>( CASE_INSENSITIVE_ORDER ); - properties.put( "group", groupId ); + properties.put("group", groupId); - Entity entity = batchCreateRole( roleName, roleTitle, inactivity, propertyName, groupId, properties ); + Entity entity = batchCreateRole(roleName, roleTitle, inactivity, propertyName, groupId, properties); getRelationManager( new SimpleEntityRef( Group.ENTITY_TYPE, groupId ) ) - .addToCollection( COLLECTION_ROLES, entity ); + .addToCollection(COLLECTION_ROLES, entity); - logger.info( "Created role {} with id {} in group {}", - new String[] { roleName, entity.getUuid().toString(), groupId.toString() } ); + logger.info("Created role {} with id {} in group {}", + new String[]{roleName, entity.getUuid().toString(), groupId.toString()}); return entity; } @@ -1848,8 +1848,8 @@ public class CpEntityManager implements EntityManager { permission = permission.toLowerCase(); long timestamp = cass.createTimestamp(); Mutator<ByteBuffer> batch = createMutator( cass.getApplicationKeyspace( applicationId ), be ); - CassandraPersistenceUtils.addInsertToMutator( batch, ApplicationCF.ENTITY_DICTIONARIES, - getRolePermissionsKey( groupId, roleName ), permission, ByteBuffer.allocate( 0 ), timestamp ); + CassandraPersistenceUtils.addInsertToMutator(batch, ApplicationCF.ENTITY_DICTIONARIES, + getRolePermissionsKey(groupId, roleName), permission, ByteBuffer.allocate(0), timestamp); //Adding graphite metrics Timer.Context timeGroupRolePermission = entGrantGroupPermissionTimer.time(); @@ -1927,7 +1927,7 @@ public class CpEntityManager implements EntityManager { @Override public void grantUserPermission( UUID userId, String permission ) throws Exception { permission = permission.toLowerCase(); - addToDictionary( userRef( userId ), DICTIONARY_PERMISSIONS, permission ); + addToDictionary(userRef(userId), DICTIONARY_PERMISSIONS, permission); } @@ -1951,9 +1951,9 @@ public class CpEntityManager implements EntityManager { roleName = roleName.toLowerCase(); EntityRef userRef = userRef( userId ); EntityRef roleRef = getRoleRef( roleName ); - addToDictionary( userRef, DICTIONARY_ROLENAMES, roleName, roleName ); - addToCollection( userRef, COLLECTION_ROLES, roleRef ); - addToCollection( roleRef, COLLECTION_USERS, userRef ); + addToDictionary(userRef, DICTIONARY_ROLENAMES, roleName, roleName); + addToCollection(userRef, COLLECTION_ROLES, roleRef); + addToCollection(roleRef, COLLECTION_USERS, userRef); } @@ -1988,8 +1988,8 @@ public class CpEntityManager implements EntityManager { private EntityRef getRoleRef( String roleName ) throws Exception { - Results results = this.searchCollection( new SimpleEntityRef( Application.ENTITY_TYPE, applicationId ), - Schema.defaultCollectionName( Role.ENTITY_TYPE ), Query.findForProperty( "roleName", roleName ) ); + Results results = this.searchCollection(new SimpleEntityRef(Application.ENTITY_TYPE, applicationId), + Schema.defaultCollectionName(Role.ENTITY_TYPE), Query.findForProperty("roleName", roleName)); Iterator<Entity> iterator = results.iterator(); EntityRef roleRef = null; while ( iterator.hasNext() ) { @@ -2235,8 +2235,8 @@ public class CpEntityManager implements EntityManager { @Override public Set<String> getCounterNames() throws Exception { Set<String> names = new TreeSet<String>( CASE_INSENSITIVE_ORDER ); - Set<String> nameSet = cast( getDictionaryAsSet( getApplicationRef(), Schema.DICTIONARY_COUNTERS ) ); - names.addAll( nameSet ); + Set<String> nameSet = cast(getDictionaryAsSet(getApplicationRef(), Schema.DICTIONARY_COUNTERS)); + names.addAll(nameSet); return names; } @@ -2300,7 +2300,7 @@ public class CpEntityManager implements EntityManager { CollectionScope collectionScope = getCollectionScopeNameFromEntityType( getApplicationScope().getApplication(), collectionName); - final EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope ); + final EntityCollectionManager ecm = managerCache.getEntityCollectionManager(collectionScope); //convert to a string, that's what we store final Id results = ecm.getIdField( new StringField( @@ -2313,7 +2313,7 @@ public class CpEntityManager implements EntityManager { public Entity get( UUID uuid ) throws Exception { MapManager mm = getMapManagerForTypes(); - String entityType = mm.getString( uuid.toString() ); + String entityType = mm.getString(uuid.toString()); final Entity entity; @@ -2335,9 +2335,9 @@ public class CpEntityManager implements EntityManager { private MapManager getMapManagerForTypes() { Id mapOwner = new SimpleId( applicationId, TYPE_APPLICATION ); - final MapScope ms = CpNamingUtils.getEntityTypeMapScope( mapOwner ); + final MapScope ms = CpNamingUtils.getEntityTypeMapScope(mapOwner); - MapManager mm = managerCache.getMapManager( ms ); + MapManager mm = managerCache.getMapManager(ms); return mm; } @@ -2440,7 +2440,7 @@ public class CpEntityManager implements EntityManager { @Override public Map<String, Role> getUserRolesWithTitles( UUID userId ) throws Exception { return getRolesWithTitles( - ( Set<String> ) cast( getDictionaryAsSet( userRef( userId ), DICTIONARY_ROLENAMES ) ) ); + ( Set<String> ) cast( getDictionaryAsSet( userRef(userId), DICTIONARY_ROLENAMES ) ) ); } @@ -2455,8 +2455,8 @@ public class CpEntityManager implements EntityManager { @Override public void addGroupToRole( UUID groupId, String roleName ) throws Exception { roleName = roleName.toLowerCase(); - addToDictionary( groupRef( groupId ), DICTIONARY_ROLENAMES, roleName, roleName ); - addToCollection( groupRef( groupId ), COLLECTION_ROLES, getRoleRef( roleName ) ); + addToDictionary(groupRef(groupId), DICTIONARY_ROLENAMES, roleName, roleName); + addToCollection(groupRef(groupId), COLLECTION_ROLES, getRoleRef(roleName)); } @@ -2477,7 +2477,7 @@ public class CpEntityManager implements EntityManager { @Override public void grantGroupPermission( UUID groupId, String permission ) throws Exception { permission = permission.toLowerCase(); - addToDictionary( groupRef( groupId ), DICTIONARY_PERMISSIONS, permission ); + addToDictionary(groupRef(groupId), DICTIONARY_PERMISSIONS, permission); } @@ -2663,8 +2663,8 @@ public class CpEntityManager implements EntityManager { catch ( HystrixRuntimeException hre ) { if ( hre.getCause() instanceof WriteUniqueVerifyException ) { - WriteUniqueVerifyException wuve = ( WriteUniqueVerifyException ) hre.getCause(); - handleWriteUniqueVerifyException( entity, wuve ); + WriteUniqueVerifyException wuve = (WriteUniqueVerifyException) hre.getCause(); + handleWriteUniqueVerifyException(entity, wuve); } } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8a62d117/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java index dd1c5fa..2167efa 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java @@ -39,11 +39,22 @@ public interface ApplicationEntityIndex { */ public EntityIndexBatch createBatch(); + /** + * Execute query in Usergrid syntax. + */ + public CandidateResults search(final IndexScope indexScope, final SearchTypes searchTypes, final Query query); /** * Execute query in Usergrid syntax. */ - public CandidateResults search(final IndexScope indexScope, final SearchTypes searchType, Query query ); + public CandidateResults search(final IndexScope indexScope, final SearchTypes searchType, final Query query, final int limit ); + + /** + * get next page of results + * @param cursor + * @return + */ + public CandidateResults getNextPage(final String cursor); /** * delete all application records http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8a62d117/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java index ade2286..ca071e2 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java @@ -46,15 +46,9 @@ import org.elasticsearch.action.deletebyquery.IndexDeleteByQueryResponse; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchScrollRequestBuilder; -import org.elasticsearch.index.query.FilterBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.index.query.TermQueryBuilder; +import org.elasticsearch.index.query.*; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; -import org.elasticsearch.search.sort.FieldSortBuilder; -import org.elasticsearch.search.sort.SortBuilders; -import org.elasticsearch.search.sort.SortOrder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import rx.Observable; @@ -85,6 +79,7 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex{ private final IndexIdentifier.IndexAlias alias; private final Timer deleteApplicationTimer; private final Meter deleteApplicationMeter; + private final SearchRequestBuilderStrategy searchRequest; private FailureMonitor failureMonitor; private final int cursorTimeout; @Inject @@ -117,6 +112,8 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex{ this.alias = indexIdentifier.getAlias(); + this.searchRequest = new SearchRequestBuilderStrategy(esProvider,appScope,alias,cursorTimeout); + } @Override @@ -139,132 +136,78 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex{ } @Override - public CandidateResults search(final IndexScope indexScope, final SearchTypes searchTypes, - final Query query ) { - - SearchResponse searchResponse; - - if ( query.getCursor() == null ) { - String[] contexts = new String[]{createContextName(applicationScope, indexScope),createLegacyContextName(applicationScope,indexScope)}; - SearchRequestBuilder srb = esProvider.getClient().prepareSearch( alias.getReadAlias() ) - .setTypes(searchTypes.getTypeNames(applicationScope)) - .setScroll(cursorTimeout + "m") - .setQuery(query.createQueryBuilder(contexts)); - - final FilterBuilder fb = query.createFilterBuilder(); - - //we have post filters, apply them - if ( fb != null ) { - logger.debug( " Filter: {} ", fb.toString() ); - srb = srb.setPostFilter( fb ); - } - - - srb = srb.setFrom( 0 ).setSize( query.getLimit() ); - - for ( Query.SortPredicate sp : query.getSortPredicates() ) { - - final SortOrder order; - if ( sp.getDirection().equals( Query.SortDirection.ASCENDING ) ) { - order = SortOrder.ASC; - } - else { - order = SortOrder.DESC; - } - - // we do not know the type of the "order by" property and so we do not know what - // type prefix to use. So, here we add an order by clause for every possible type - // that you can order by: string, number and boolean and we ask ElasticSearch - // to ignore any fields that are not present. - - final String stringFieldName = STRING_PREFIX + sp.getPropertyName(); - final FieldSortBuilder stringSort = SortBuilders.fieldSort(stringFieldName) - .order( order ).ignoreUnmapped( true ); - srb.addSort( stringSort ); - - logger.debug( " Sort: {} order by {}", stringFieldName, order.toString() ); + public CandidateResults search(final IndexScope indexScope, final SearchTypes searchTypes, final Query query){ + return search(indexScope,searchTypes,query,10); + } - final String longFieldName = LONG_PREFIX + sp.getPropertyName(); - final FieldSortBuilder longSort = SortBuilders.fieldSort( longFieldName ) - .order( order ).ignoreUnmapped( true ); - srb.addSort( longSort ); - logger.debug( " Sort: {} order by {}", longFieldName, order.toString() ); + @Override + public CandidateResults search(final IndexScope indexScope, final SearchTypes searchTypes, final Query query, final int limit) { + SearchResponse searchResponse; - final String doubleFieldName = DOUBLE_PREFIX + sp.getPropertyName(); - final FieldSortBuilder doubleSort = SortBuilders.fieldSort( doubleFieldName ) - .order( order ).ignoreUnmapped( true ); - srb.addSort( doubleSort ); - logger.debug( " Sort: {} order by {}", doubleFieldName, order.toString() ); + SearchRequestBuilder srb = searchRequest.getBuilder(indexScope, searchTypes, query, limit); + if (logger.isDebugEnabled()) { + logger.debug("Searching index (read alias): {}\n scope: {} \n type: {}\n query: {} ", + this.alias.getReadAlias(), indexScope.getOwner(), searchTypes.getTypeNames(applicationScope), srb); + } - final String booleanFieldName = BOOLEAN_PREFIX + sp.getPropertyName(); - final FieldSortBuilder booleanSort = SortBuilders.fieldSort( booleanFieldName ) - .order( order ).ignoreUnmapped( true ); - srb.addSort( booleanSort ); - logger.debug( " Sort: {} order by {}", booleanFieldName, order.toString() ); - } + try { + //Added For Graphite Metrics + Timer.Context timeSearch = searchTimer.time(); + searchResponse = srb.execute().actionGet(); + timeSearch.stop(); + } catch (Throwable t) { + logger.error("Unable to communicate with Elasticsearch", t); + failureMonitor.fail("Unable to execute batch", t); + throw t; + } + failureMonitor.success(); - if ( logger.isDebugEnabled() ) { - logger.debug( "Searching index (read alias): {}\n scope: {} \n type: {}\n query: {} ", - this.alias.getReadAlias(), indexScope.getOwner(), searchTypes.getTypeNames(applicationScope), srb ); - } + return parseResults(searchResponse); + } - try { - //Added For Graphite Metrics - Timer.Context timeSearch = searchTimer.time(); - searchResponse = srb.execute().actionGet(); - timeSearch.stop(); - } - catch ( Throwable t ) { - logger.error( "Unable to communicate with Elasticsearch", t ); - failureMonitor.fail( "Unable to execute batch", t ); - throw t; - } + public CandidateResults getNextPage(final String cursor){ + SearchResponse searchResponse; - failureMonitor.success(); + String userCursorString = cursor; + if ( userCursorString.startsWith( "\"" ) ) { + userCursorString = userCursorString.substring( 1 ); + } + if ( userCursorString.endsWith( "\"" ) ) { + userCursorString = userCursorString.substring( 0, userCursorString.length() - 1 ); } - else { - String userCursorString = query.getCursor(); - if ( userCursorString.startsWith( "\"" ) ) { - userCursorString = userCursorString.substring( 1 ); - } - if ( userCursorString.endsWith( "\"" ) ) { - userCursorString = userCursorString.substring( 0, userCursorString.length() - 1 ); - } - - //now get the cursor from the map and validate - final String esScrollCursor = mapManager.getString( userCursorString ); - - Preconditions.checkArgument(esScrollCursor != null, "Could not find a cursor for the value '{}' ", esScrollCursor); + //now get the cursor from the map and validate + final String esScrollCursor = mapManager.getString(userCursorString); + Preconditions.checkArgument(esScrollCursor != null, "Could not find a cursor for the value '{}' ", esScrollCursor); - logger.debug( "Executing query with cursor: {} ", esScrollCursor ); - SearchScrollRequestBuilder ssrb = esProvider.getClient() - .prepareSearchScroll(esScrollCursor).setScroll( cursorTimeout + "m" ); + logger.debug( "Executing query with cursor: {} ", esScrollCursor ); - try { - //Added For Graphite Metrics - Timer.Context timeSearchCursor = cursorTimer.time(); - searchResponse = ssrb.execute().actionGet(); - timeSearchCursor.stop(); - } - catch ( Throwable t ) { - logger.error( "Unable to communicate with elasticsearch", t ); - failureMonitor.fail( "Unable to execute batch", t ); - throw t; - } + SearchScrollRequestBuilder ssrb = esProvider.getClient() + .prepareSearchScroll(esScrollCursor).setScroll(cursorTimeout + "m"); - failureMonitor.success(); + try { + //Added For Graphite Metrics + Timer.Context timeSearchCursor = cursorTimer.time(); + searchResponse = ssrb.execute().actionGet(); + timeSearchCursor.stop(); + } + catch ( Throwable t ) { + logger.error( "Unable to communicate with elasticsearch", t ); + failureMonitor.fail( "Unable to execute batch", t ); + throw t; } - return parseResults(searchResponse, query); + + failureMonitor.success(); + return parseResults(searchResponse); } /** @@ -327,7 +270,7 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex{ - private CandidateResults parseResults( final SearchResponse searchResponse, final Query query ) { + private CandidateResults parseResults( final SearchResponse searchResponse) { final SearchHits searchHits = searchResponse.getHits(); final SearchHit[] hits = searchHits.getHits(); @@ -335,41 +278,39 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex{ logger.debug(" Hit count: {} Total hits: {}", length, searchHits.getTotalHits()); - List<CandidateResult> candidates = new ArrayList<>( length ); + List<CandidateResult> candidates = new ArrayList<>(length); - for ( SearchHit hit : hits ) { + for (SearchHit hit : hits) { - String[] idparts = hit.getId().split( SPLITTER ); + String[] idparts = hit.getId().split(SPLITTER); String id = idparts[0]; String type = idparts[1]; String version = idparts[2]; - Id entityId = new SimpleId( UUID.fromString(id), type ); + Id entityId = new SimpleId(UUID.fromString(id), type); - candidates.add( new CandidateResult( entityId, UUID.fromString( version ) ) ); + candidates.add(new CandidateResult(entityId, UUID.fromString(version))); } - CandidateResults candidateResults = new CandidateResults( query, candidates ); - - if ( candidates.size() >= query.getLimit() ) { - //USERGRID-461 our cursor is getting too large, map it to a new time UUID - //TODO T.N., this shouldn't live here. This should live at the UG core tier. However the RM/EM are an absolute mess, so until they're refactored, this is it's home - - final String userCursorString = org.apache.usergrid.persistence.index.utils.StringUtils.sanitizeUUID( UUIDGenerator.newTimeUUID() ); + CandidateResults candidateResults = new CandidateResults(candidates); + if(candidateResults.hasCursor()) { + candidateResults.initializeCursor(); final String esScrollCursor = searchResponse.getScrollId(); - //now set this into our map module final int minutes = indexFig.getQueryCursorTimeout(); //just truncate it, we'll never hit a long value anyway - mapManager.putString( userCursorString, esScrollCursor, ( int ) TimeUnit.MINUTES.toSeconds( minutes ) ); + mapManager.putString(candidateResults.getCursor(), esScrollCursor, (int) TimeUnit.MINUTES.toSeconds(minutes)); - candidateResults.setCursor( userCursorString ); - logger.debug(" User cursor = {}, Cursor = {} ", userCursorString, esScrollCursor); + logger.debug(" User cursor = {}, Cursor = {} ", candidateResults.getCursor(), esScrollCursor); } return candidateResults; } + + + + } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8a62d117/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java new file mode 100644 index 0000000..d146e4c --- /dev/null +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java @@ -0,0 +1,197 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one or more + * * contributor license agreements. The ASF licenses this file to You + * * under the Apache License, Version 2.0 (the "License"); you may not + * * use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. For additional information regarding + * * copyright in this work, please see the NOTICE file in the top level + * * directory of this distribution. + * + */ +package org.apache.usergrid.persistence.index.impl; + +import com.google.common.base.Preconditions; +import org.apache.usergrid.persistence.core.scope.ApplicationScope; +import org.apache.usergrid.persistence.core.util.ValidationUtils; +import org.apache.usergrid.persistence.index.IndexIdentifier; +import org.apache.usergrid.persistence.index.IndexScope; +import org.apache.usergrid.persistence.index.SearchTypes; +import org.apache.usergrid.persistence.index.exceptions.IndexException; +import org.apache.usergrid.persistence.index.query.Query; +import org.apache.usergrid.persistence.index.query.tree.QueryVisitor; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.FilterBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.sort.FieldSortBuilder; +import org.elasticsearch.search.sort.SortBuilders; +import org.elasticsearch.search.sort.SortOrder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.usergrid.persistence.index.impl.IndexingUtils.*; +import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createContextName; +import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createLegacyContextName; + +/** + * Classy class class. + */ + +public class SearchRequestBuilderStrategy { + + private static final Logger logger = LoggerFactory.getLogger(SearchRequestBuilderStrategy.class); + + private final EsProvider esProvider; + private final ApplicationScope applicationScope; + private final IndexIdentifier.IndexAlias alias; + private final int cursorTimeout; + public static final int MAX_LIMIT = 1000; + + public SearchRequestBuilderStrategy(final EsProvider esProvider, final ApplicationScope applicationScope, final IndexIdentifier.IndexAlias alias, int cursorTimeout){ + + this.esProvider = esProvider; + this.applicationScope = applicationScope; + this.alias = alias; + this.cursorTimeout = cursorTimeout; + } + + public SearchRequestBuilder getBuilder(final IndexScope indexScope, final SearchTypes searchTypes, final Query query, final int limit) { + Preconditions.checkArgument(limit <= MAX_LIMIT, "limit is greater than max "+ MAX_LIMIT); + + SearchRequestBuilder srb = esProvider.getClient().prepareSearch(alias.getReadAlias()) + .setTypes(searchTypes.getTypeNames(applicationScope)) + .setScroll(cursorTimeout + "m") + .setQuery(createQueryBuilder( indexScope,query)); + + final FilterBuilder fb = createFilterBuilder(query); + + //we have post filters, apply them + if (fb != null) { + logger.debug(" Filter: {} ", fb.toString()); + srb = srb.setPostFilter(fb); + } + + + srb = srb.setFrom(0).setSize(limit); + + for (Query.SortPredicate sp : query.getSortPredicates()) { + + final SortOrder order; + if (sp.getDirection().equals(Query.SortDirection.ASCENDING)) { + order = SortOrder.ASC; + } else { + order = SortOrder.DESC; + } + + // we do not know the type of the "order by" property and so we do not know what + // type prefix to use. So, here we add an order by clause for every possible type + // that you can order by: string, number and boolean and we ask ElasticSearch + // to ignore any fields that are not present. + + final String stringFieldName = STRING_PREFIX + sp.getPropertyName(); + final FieldSortBuilder stringSort = SortBuilders.fieldSort(stringFieldName) + .order(order).ignoreUnmapped(true); + srb.addSort(stringSort); + + logger.debug(" Sort: {} order by {}", stringFieldName, order.toString()); + + final String longFieldName = LONG_PREFIX + sp.getPropertyName(); + final FieldSortBuilder longSort = SortBuilders.fieldSort(longFieldName) + .order(order).ignoreUnmapped(true); + srb.addSort(longSort); + logger.debug(" Sort: {} order by {}", longFieldName, order.toString()); + + + final String doubleFieldName = DOUBLE_PREFIX + sp.getPropertyName(); + final FieldSortBuilder doubleSort = SortBuilders.fieldSort(doubleFieldName) + .order(order).ignoreUnmapped(true); + srb.addSort(doubleSort); + logger.debug(" Sort: {} order by {}", doubleFieldName, order.toString()); + + + final String booleanFieldName = BOOLEAN_PREFIX + sp.getPropertyName(); + final FieldSortBuilder booleanSort = SortBuilders.fieldSort(booleanFieldName) + .order(order).ignoreUnmapped(true); + srb.addSort(booleanSort); + logger.debug(" Sort: {} order by {}", booleanFieldName, order.toString()); + } + return srb; + } + + + public QueryBuilder createQueryBuilder( IndexScope indexScope, Query query) { + String[] contexts = new String[]{createContextName(applicationScope, indexScope),createLegacyContextName(applicationScope,indexScope)}; + + QueryBuilder queryBuilder = null; + + //we have a root operand. Translate our AST into an ES search + if ( query.getRootOperand() != null ) { + // In the case of geo only queries, this will return null into the query builder. + // Once we start using tiles, we won't need this check any longer, since a geo query + // will return a tile query + post filter + QueryVisitor v = new EsQueryVistor(); + + try { + query.getRootOperand().visit( v ); + } + catch ( IndexException ex ) { + throw new RuntimeException( "Error building ElasticSearch query", ex ); + } + + + queryBuilder = v.getQueryBuilder(); + } + + + // Add our filter for context to our query for fast execution. + // Fast because it utilizes bitsets internally. See this post for more detail. + // http://www.elasticsearch.org/blog/all-about-elasticsearch-filter-bitsets/ + + // TODO evaluate performance when it's an all query. + // Do we need to put the context term first for performance? + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + for(String context : contexts){ + boolQueryBuilder = boolQueryBuilder.should(QueryBuilders.termQuery( IndexingUtils.ENTITY_CONTEXT_FIELDNAME, context )); + } + boolQueryBuilder = boolQueryBuilder.minimumNumberShouldMatch(1); + if ( queryBuilder != null ) { + queryBuilder = boolQueryBuilder.must( queryBuilder ); + } + + //nothing was specified ensure we specify the context in the search + else { + queryBuilder = boolQueryBuilder; + } + + return queryBuilder; + } + + + public FilterBuilder createFilterBuilder(Query query) { + FilterBuilder filterBuilder = null; + + if ( query.getRootOperand() != null ) { + QueryVisitor v = new EsQueryVistor(); + try { + query.getRootOperand().visit( v ); + + } catch ( IndexException ex ) { + throw new RuntimeException( "Error building ElasticSearch query", ex ); + } + filterBuilder = v.getFilterBuilder(); + } + + return filterBuilder; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8a62d117/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/CandidateResults.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/CandidateResults.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/CandidateResults.java index be90446..5cb6680 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/CandidateResults.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/CandidateResults.java @@ -19,12 +19,14 @@ package org.apache.usergrid.persistence.index.query; import java.util.Iterator; import java.util.List; + +import org.apache.usergrid.persistence.model.util.UUIDGenerator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * Internal results class, should not be returned as results to a user. + * Internal results class, should not be returned as results to a user. * Only returns candidate entity results */ public class CandidateResults implements Iterable<CandidateResult> { @@ -33,16 +35,25 @@ public class CandidateResults implements Iterable<CandidateResult> { private String cursor = null; - private final Query query; private final List<CandidateResult> candidates; - public CandidateResults( Query query, List<CandidateResult> candidates ) { - this.query = query; + public CandidateResults( List<CandidateResult> candidates ) { this.candidates = candidates; } + public void add( List<CandidateResult> candidates ) { + this.candidates.addAll( candidates); + } + + public void initializeCursor(){ + //USERGRID-461 our cursor is getting too large, map it to a new time UUID + //TODO T.N., this shouldn't live here. This should live at the UG core tier. However the RM/EM are an absolute mess, so until they're refactored, this is it's home + + cursor = org.apache.usergrid.persistence.index.utils.StringUtils.sanitizeUUID(UUIDGenerator.newTimeUUID()); + } + public boolean hasCursor() { return cursor != null; @@ -59,11 +70,8 @@ public class CandidateResults implements Iterable<CandidateResult> { } - public Query getQuery() { - return query; - } - + public int size() { return candidates.size(); } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8a62d117/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java index 438d82e..6454b48 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java @@ -40,10 +40,7 @@ import org.antlr.runtime.Token; import org.antlr.runtime.TokenRewriteStream; import org.apache.commons.codec.binary.Base64; import org.apache.commons.lang.StringUtils; -import org.apache.usergrid.persistence.index.exceptions.IndexException; import org.apache.usergrid.persistence.index.exceptions.QueryParseException; -import org.apache.usergrid.persistence.index.impl.EsQueryVistor; -import org.apache.usergrid.persistence.index.impl.IndexingUtils; import org.apache.usergrid.persistence.index.query.tree.AndOperand; import org.apache.usergrid.persistence.index.query.tree.ContainsOperand; import org.apache.usergrid.persistence.index.query.tree.CpQueryFilterLexer; @@ -55,15 +52,10 @@ import org.apache.usergrid.persistence.index.query.tree.GreaterThanEqual; import org.apache.usergrid.persistence.index.query.tree.LessThan; import org.apache.usergrid.persistence.index.query.tree.LessThanEqual; import org.apache.usergrid.persistence.index.query.tree.Operand; -import org.apache.usergrid.persistence.index.query.tree.QueryVisitor; import org.apache.usergrid.persistence.index.utils.ClassUtils; import org.apache.usergrid.persistence.index.utils.ConversionUtils; import org.apache.usergrid.persistence.index.utils.ListUtils; import org.apache.usergrid.persistence.index.utils.MapUtils; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.FilterBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -80,9 +72,7 @@ public class Query { IDS, REFS, CORE_PROPERTIES, ALL_PROPERTIES, LINKED_PROPERTIES } - public static final int DEFAULT_LIMIT = 10; - public static final int MAX_LIMIT = 1000; public static final String PROPERTY_UUID = "uuid"; @@ -90,8 +80,6 @@ public class Query { private List<SortPredicate> sortPredicates = new ArrayList<SortPredicate>(); private Operand rootOperand; private UUID startResult; - private String cursor; - private int limit = 0; private Map<String, String> selectAssignments = new LinkedHashMap<String, String>(); private boolean mergeSelectResults = false; @@ -124,8 +112,6 @@ public class Query { sortPredicates = q.sortPredicates != null ? new ArrayList<SortPredicate>( q.sortPredicates ) : null; startResult = q.startResult; - cursor = q.cursor; - limit = q.limit; selectAssignments = q.selectAssignments != null ? new LinkedHashMap<String, String>( q.selectAssignments ) : null; mergeSelectResults = q.mergeSelectResults; @@ -148,71 +134,6 @@ public class Query { } - public QueryBuilder createQueryBuilder( final String[] contexts ) { - - - QueryBuilder queryBuilder = null; - - - //we have a root operand. Translate our AST into an ES search - if ( getRootOperand() != null ) { - // In the case of geo only queries, this will return null into the query builder. - // Once we start using tiles, we won't need this check any longer, since a geo query - // will return a tile query + post filter - QueryVisitor v = new EsQueryVistor(); - - try { - getRootOperand().visit( v ); - } - catch ( IndexException ex ) { - throw new RuntimeException( "Error building ElasticSearch query", ex ); - } - - - queryBuilder = v.getQueryBuilder(); - } - - - // Add our filter for context to our query for fast execution. - // Fast because it utilizes bitsets internally. See this post for more detail. - // http://www.elasticsearch.org/blog/all-about-elasticsearch-filter-bitsets/ - - // TODO evaluate performance when it's an all query. - // Do we need to put the context term first for performance? - BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); - for(String context : contexts){ - boolQueryBuilder = boolQueryBuilder.should(QueryBuilders.termQuery( IndexingUtils.ENTITY_CONTEXT_FIELDNAME, context )); - } - boolQueryBuilder = boolQueryBuilder.minimumNumberShouldMatch(1); - if ( queryBuilder != null ) { - queryBuilder = boolQueryBuilder.must( queryBuilder ); - } - - //nothing was specified ensure we specify the context in the search - else { - queryBuilder = boolQueryBuilder; - } - - return queryBuilder; - } - - - public FilterBuilder createFilterBuilder() { - FilterBuilder filterBuilder = null; - - if ( getRootOperand() != null ) { - QueryVisitor v = new EsQueryVistor(); - try { - getRootOperand().visit( v ); - - } catch ( IndexException ex ) { - throw new RuntimeException( "Error building ElasticSearch query", ex ); - } - filterBuilder = v.getFilterBuilder(); - } - - return filterBuilder; - } /** @@ -315,7 +236,6 @@ public class Query { String connection = ListUtils.first( params.get( "connectionType" ) ); UUID start = ListUtils.firstUuid( params.get( "start" ) ); String cursor = ListUtils.first( params.get( "cursor" ) ); - Integer limit = ListUtils.firstInteger( params.get( "limit" ) ); List<String> permissions = params.get( "permission" ); Long startTime = ListUtils.firstLong( params.get( "start_time" ) ); Long finishTime = ListUtils.firstLong( params.get( "end_time" ) ); @@ -386,15 +306,7 @@ public class Query { q.setStartResult( start ); } - if ( cursor != null ) { - q = newQueryIfNull( q ); - q.setCursor( cursor ); - } - if ( limit != null ) { - q = newQueryIfNull( q ); - q.setLimit( limit ); - } if ( startTime != null ) { q = newQueryIfNull( q ); @@ -437,22 +349,21 @@ public class Query { public static Query searchForProperty( String propertyName, Object propertyValue ) { Query q = new Query(); - q.addEqualityFilter( propertyName, propertyValue ); + q.addEqualityFilter(propertyName, propertyValue); return q; } public static Query findForProperty( String propertyName, Object propertyValue ) { Query q = new Query(); - q.addEqualityFilter( propertyName, propertyValue ); - q.setLimit( 1 ); + q.addEqualityFilter(propertyName, propertyValue); return q; } public static Query fromUUID( UUID uuid ) { Query q = new Query(); - q.addIdentifier( Identifier.fromUUID( uuid ) ); + q.addIdentifier( Identifier.fromUUID(uuid) ); return q; } @@ -809,8 +720,8 @@ public class Query { public Query addContainsFilter( String propName, String keyword ) { ContainsOperand equality = new ContainsOperand( new ClassicToken( 0, "contains" ) ); - equality.setProperty( propName ); - equality.setLiteral( keyword ); + equality.setProperty(propName); + equality.setLiteral(keyword); addClause( equality ); @@ -819,8 +730,8 @@ public class Query { private void addClause( EqualityOperand equals, String propertyName, Object value ) { - equals.setProperty( propertyName ); - equals.setLiteral( value ); + equals.setProperty(propertyName); + equals.setLiteral(value); addClause( equals ); } @@ -882,7 +793,7 @@ public class Query { } - public UUID getStartResult() { + public UUID getStartResult(String cursor) { if ( ( startResult == null ) && ( cursor != null ) ) { byte[] cursorBytes = Base64.decodeBase64( cursor ); if ( ( cursorBytes != null ) && ( cursorBytes.length == 16 ) ) { @@ -893,61 +804,6 @@ public class Query { } - public String getCursor() { - return cursor; - } - - - public void setCursor( String cursor ) { - this.cursor = cursor; - } - - - public Query withCursor( String cursor ) { - setCursor( cursor ); - return this; - } - - - public int getLimit() { - return getLimit( DEFAULT_LIMIT ); - } - - - public int getLimit( int defaultLimit ) { - if ( limit <= 0 ) { - if ( defaultLimit > 0 ) { - return defaultLimit; - } - else { - return DEFAULT_LIMIT; - } - } - return limit; - } - - - public void setLimit( int limit ) { - - // TODO tnine. After users have had time to change their query limits, - // this needs to be uncommented and enforced. - // if(limit > MAX_LIMIT){ - // throw new IllegalArgumentException( - // String.format("Query limit must be <= to %d", MAX_LIMIT)); - // } - - if ( limit > MAX_LIMIT ) { - limit = MAX_LIMIT; - } - - this.limit = limit; - } - - - public Query withLimit( int limit ) { - setLimit( limit ); - return this; - } public boolean isReversed() { http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8a62d117/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java index 6df5a6d..2c03635 100644 --- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java +++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java @@ -118,7 +118,7 @@ public class EntityIndexTest extends BaseIT { final String entityType = "thing"; IndexScope indexScope = new IndexScopeImpl( appId, "things" ); - final SearchTypes searchTypes = SearchTypes.fromTypes( entityType ); + final SearchTypes searchTypes = SearchTypes.fromTypes(entityType); EntityIndexBatch batch = entityIndex.createBatch(); Entity entity = new Entity( entityType ); EntityUtils.setVersion(entity, UUIDGenerator.newTimeUUID()); @@ -143,7 +143,7 @@ public class EntityIndexTest extends BaseIT { ei.refresh(); - testQueries( indexScope, searchTypes, entityIndex ); + testQueries(indexScope, searchTypes, entityIndex); } @Test @@ -219,17 +219,17 @@ public class EntityIndexTest extends BaseIT { final String entityType = "thing"; IndexScope indexScope = new IndexScopeImpl( appId, "things" ); - final SearchTypes searchTypes = SearchTypes.fromTypes( entityType ); + final SearchTypes searchTypes = SearchTypes.fromTypes(entityType); insertJsonBlob(entityIndex, entityType, indexScope, "/sample-large.json",101,0); ei.refresh(); - testQueries( indexScope, searchTypes, entityIndex ); + testQueries(indexScope, searchTypes, entityIndex); - ei.addIndex("v2", 1,0,"one"); + ei.addIndex("v2", 1, 0, "one"); - insertJsonBlob(entityIndex, entityType, indexScope, "/sample-large.json",101,100); + insertJsonBlob(entityIndex, entityType, indexScope, "/sample-large.json", 101, 100); ei.refresh(); @@ -364,11 +364,12 @@ public class EntityIndexTest extends BaseIT { StopWatch timer = new StopWatch(); timer.start(); Query query = Query.fromQL( queryString ); - query.setLimit( 1000 ); - CandidateResults candidateResults = entityIndex.search( scope, searchTypes, query ); + CandidateResults candidateResults = null; + candidateResults = entityIndex.search( scope, searchTypes, query,num+1 ); + timer.stop(); - assertEquals( num, candidateResults.size() ); + assertEquals( num,candidateResults.size() ); log.debug( "Query time {}ms", timer.getTime() ); return candidateResults; } @@ -652,13 +653,8 @@ public class EntityIndexTest extends BaseIT { for ( int i = 0; i < expectedPages; i++ ) { //** Query query = Query.fromQL( "select * order by created" ); - query.setLimit( limit ); - - if ( cursor != null ) { - query.setCursor( cursor ); - } - final CandidateResults results = entityIndex.search( indexScope, SearchTypes.allTypes(), query ); + final CandidateResults results = cursor == null ? entityIndex.search( indexScope, SearchTypes.allTypes(), query , limit) : entityIndex.getNextPage(cursor); assertTrue( results.hasCursor() ); @@ -667,8 +663,6 @@ public class EntityIndexTest extends BaseIT { assertEquals("Should be 16 bytes as hex", 32, cursor.length()); - - assertEquals( 1, results.size() ); http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8a62d117/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java index 102e222..792fe3a 100644 --- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java +++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java @@ -19,6 +19,9 @@ package org.apache.usergrid.persistence.query.tree; +import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl; +import org.apache.usergrid.persistence.index.impl.IndexScopeImpl; +import org.apache.usergrid.persistence.index.impl.SearchRequestBuilderStrategy; import org.apache.usergrid.persistence.index.query.tree.LongLiteral; import org.apache.usergrid.persistence.index.query.tree.CpQueryFilterLexer; import org.apache.usergrid.persistence.index.query.tree.CpQueryFilterParser; @@ -43,6 +46,7 @@ import org.antlr.runtime.ANTLRStringStream; import org.antlr.runtime.RecognitionException; import org.antlr.runtime.TokenRewriteStream; import org.apache.usergrid.persistence.index.exceptions.QueryParseException; +import org.apache.usergrid.persistence.model.entity.SimpleId; import org.junit.Test; import org.apache.usergrid.persistence.index.query.Query; import org.elasticsearch.index.query.QueryBuilder; @@ -429,7 +433,9 @@ public class GrammarTreeTest { assertEquals( 37f, withinOperand.getLatitude().getFloatValue(), 0 ); assertEquals( -75f, withinOperand.getLongitude().getFloatValue(), 0 ); - QueryBuilder qb = query.createQueryBuilder(new String[]{"testcontext"}); + SearchRequestBuilderStrategy builderStrategy = new SearchRequestBuilderStrategy(null,new ApplicationScopeImpl(new SimpleId("test")),null,100); + QueryBuilder qb =builderStrategy.createQueryBuilder(new IndexScopeImpl(new SimpleId("owner"),"app"),query); + }
