Updates constants. Fixes scope in query
Need to finish version query Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/707747d1 Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/707747d1 Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/707747d1 Branch: refs/heads/es-type-updates Commit: 707747d1e4a7fe4558872717876493bbe7d8cfa2 Parents: 3cd6040 Author: Todd Nine <[email protected]> Authored: Thu Nov 6 17:42:16 2014 -0700 Committer: Todd Nine <[email protected]> Committed: Thu Nov 6 17:42:16 2014 -0700 ---------------------------------------------------------------------- .../index/impl/EsEntityIndexBatchImpl.java | 9 +- .../index/impl/EsEntityIndexImpl.java | 36 +- .../persistence/index/impl/EsQueryVistor.java | 3 +- .../persistence/index/impl/IndexingUtils.java | 74 +- .../usergrid/persistence/index/query/Query.java | 28 +- .../persistence/index/impl/EntityIndexTest.java | 785 ++++++++++--------- .../persistence/query/tree/GrammarTreeTest.java | 2 +- 7 files changed, 486 insertions(+), 451 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/707747d1/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java index a70a8d8..01f22c0 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java @@ -61,8 +61,8 @@ import com.google.common.base.Joiner; import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ANALYZED_STRING_PREFIX; import static org.apache.usergrid.persistence.index.impl.IndexingUtils.BOOLEAN_PREFIX; -import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITYID_FIELDNAME; -import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITY_CONTEXT; +import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITYID_ID_FIELDNAME; +import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITY_CONTEXT_FIELDNAME; import static org.apache.usergrid.persistence.index.impl.IndexingUtils.GEO_PREFIX; import static org.apache.usergrid.persistence.index.impl.IndexingUtils.NUMBER_PREFIX; import static org.apache.usergrid.persistence.index.impl.IndexingUtils.STRING_PREFIX; @@ -255,10 +255,11 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch { final Map entityMap = entityToMap( entity ); //add the context for filtering later - entityMap.put( ENTITY_CONTEXT, context ); + entityMap.put( ENTITY_CONTEXT_FIELDNAME, context ); //but the fieldname - entityMap.put( ENTITYID_FIELDNAME, entity.getId().getUuid().toString().toLowerCase() ); + //we have to prefix because we use query equality to seek this later. TODO see if we can make this more declarative + entityMap.put( STRING_PREFIX+ ENTITYID_ID_FIELDNAME, IndexingUtils.idString(entity.getId()).toLowerCase() ); return entityMap; } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/707747d1/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java index 6d8810c..d7955c4 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java @@ -71,9 +71,9 @@ import com.google.inject.Inject; import com.google.inject.assistedinject.Assisted; import static org.apache.usergrid.persistence.index.impl.IndexingUtils.BOOLEAN_PREFIX; -import static org.apache.usergrid.persistence.index.impl.IndexingUtils.DOC_ID_SEPARATOR_SPLITTER; -import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITYID_FIELDNAME; +import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITYID_ID_FIELDNAME; import static org.apache.usergrid.persistence.index.impl.IndexingUtils.NUMBER_PREFIX; +import static org.apache.usergrid.persistence.index.impl.IndexingUtils.SPLITTER; import static org.apache.usergrid.persistence.index.impl.IndexingUtils.STRING_PREFIX; @@ -232,45 +232,25 @@ public class EsEntityIndexImpl implements EntityIndex { final String context = IndexingUtils.createContextName( indexScope ); final String[] entityTypes = searchTypes.getTypeNames(); - QueryBuilder qb = query.createQueryBuilder(); + QueryBuilder qb = query.createQueryBuilder(context); SearchResponse searchResponse; if ( query.getCursor() == null ) { SearchRequestBuilder srb = esProvider.getClient().prepareSearch( indexName ).setTypes( entityTypes ) - .setScroll( cursorTimeout + "m" ); + .setScroll( cursorTimeout + "m" ).setQuery( qb );; //we must add a "must" operation to our entity context. - /** - * Add our filter for context to our query for fast execution. Fast because it utilizes bitsets - * internally. See this post for more detail. - * http://www.elasticsearch.org/blog/all-about-elasticsearch-filter-bitsets/ - * - * - * TODO evaluate performance when it's an all query. Do we need to put the context term first for - * performance? - */ - - - final QueryBuilder finalQuery; - - if ( qb instanceof MatchAllQueryBuilder ) { - finalQuery = QueryBuilders.termQuery( IndexingUtils.ENTITY_CONTEXT, context ); - } - else { - finalQuery = QueryBuilders.boolQuery().must( qb ) - .must( QueryBuilders.termQuery( IndexingUtils.ENTITY_CONTEXT, context ) ); - } //set our final query - srb.setQuery( finalQuery ); + if ( logger.isDebugEnabled() ) { logger.debug( "Searching index {}\n scope{} \n type {}\n query {} limit {}", new Object[] { - this.indexName, context, entityTypes, finalQuery.toString().replace( "\n", " " ), query.getLimit() + this.indexName, context, entityTypes, qb.toString().replace( "\n", " " ), query.getLimit() } ); } @@ -367,7 +347,7 @@ public class EsEntityIndexImpl implements EntityIndex { for ( SearchHit hit : hits.getHits() ) { - String[] idparts = hit.getId().split( DOC_ID_SEPARATOR_SPLITTER ); + String[] idparts = hit.getId().split( SPLITTER ); String id = idparts[0]; String type = idparts[1]; String version = idparts[2]; @@ -417,7 +397,7 @@ public class EsEntityIndexImpl implements EntityIndex { @Override public CandidateResults getEntityVersions( final IndexScope scope, final Id id ) { Query query = new Query(); - query.addEqualityFilter( ENTITYID_FIELDNAME, id.getUuid().toString() ); + query.addEqualityFilter( ENTITYID_ID_FIELDNAME, IndexingUtils.idString( id ).toLowerCase() ); CandidateResults results = search( scope, SearchTypes.fromTypes( id.getType() ), query ); return results; } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/707747d1/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java index 0afe992..0e4b0ad 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java @@ -219,9 +219,10 @@ public class EsQueryVistor implements QueryVisitor { String svalue = (String)value; BoolQueryBuilder qb = QueryBuilders.boolQuery(); // let's do a boolean OR - qb.minimumNumberShouldMatch(1); + qb.minimumNumberShouldMatch(1); // field is an entity/array that does not need a prefix on its name + //TODO is this right now that we've updated our doc structure? Should this be "must" instead of should? qb = qb.should( QueryBuilders.wildcardQuery( name, svalue ) ); // or field is just a string that does need a prefix http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/707747d1/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java index cb82f6f..c7592bc 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java @@ -39,26 +39,19 @@ public class IndexingUtils { - public static final String DOC_ID_SEPARATOR = "__"; - public static final String DOC_ID_SEPARATOR_SPLITTER = "\\__"; + public static final String SPLITTER = "\\__"; // These are not allowed in document type names: _ . , | # - public static final String DOC_TYPE_SEPARATOR = "__"; - - public static final String INDEX_NAME_SEPARATOR = "__"; + public static final String SEPARATOR = "__"; /** * Reserved UG fields. */ - public static final String ENTITY_CONTEXT = "ug_context"; + public static final String ENTITY_CONTEXT_FIELDNAME = "ug_context"; - public static final String ENTITYID_FIELDNAME = "ug_entityId"; + public static final String ENTITYID_ID_FIELDNAME = "ug_entityId"; - /** - * To be used when we want to search all types within a scope - */ - public static final String ALL_TYPES = "ALL"; /** @@ -68,13 +61,35 @@ public class IndexingUtils { */ public static String createContextName( IndexScope scope ) { StringBuilder sb = new StringBuilder(); - sb.append( scope.getOwner().getUuid() ).append(DOC_TYPE_SEPARATOR); - sb.append( scope.getOwner().getType() ).append(DOC_TYPE_SEPARATOR); + idString(sb, scope.getOwner()); + sb.append( SEPARATOR ); sb.append( scope.getName() ); return sb.toString(); } + /** + * Append the id to the string + * @param builder + * @param id + */ + public static final void idString(final StringBuilder builder, final Id id){ + builder.append( id.getUuid() ).append( SEPARATOR ) + .append(id.getType()); + } + + + /** + * Turn the id into a string + * @param id + * @return + */ + public static final String idString(final Id id){ + final StringBuilder sb = new StringBuilder( ); + idString(sb, id); + return sb.toString(); + } + /** * Create the index name based on our prefix+appUUID+AppType @@ -85,9 +100,8 @@ public class IndexingUtils { public static String createIndexName( String prefix, ApplicationScope applicationScope) { StringBuilder sb = new StringBuilder(); - sb.append( prefix ).append(INDEX_NAME_SEPARATOR); - sb.append( applicationScope.getApplication().getUuid() ).append(INDEX_NAME_SEPARATOR); - sb.append( applicationScope.getApplication().getType() ); + sb.append( prefix ).append( SEPARATOR ); + idString( sb, applicationScope.getApplication() ); return sb.toString(); } @@ -112,9 +126,9 @@ public class IndexingUtils { */ public static String createIndexDocId(final Id entityId, final UUID version, final String context) { StringBuilder sb = new StringBuilder(); - sb.append( entityId.getUuid() ).append(DOC_ID_SEPARATOR); - sb.append( entityId.getType() ).append(DOC_ID_SEPARATOR); - sb.append( version.toString() ).append( DOC_ID_SEPARATOR ); + idString(sb, entityId); + sb.append( SEPARATOR ); + sb.append( version.toString() ).append( SEPARATOR ); sb.append( context); return sb.toString(); } @@ -143,8 +157,21 @@ public class IndexingUtils { .startArray( "dynamic_templates" ) //we need most specific mappings first since it's a stop on match algorithm + + .startObject() + .startObject( "context_template" ) + .field( "match", IndexingUtils.ENTITYID_ID_FIELDNAME ) + .field( "match_mapping_type", "string" ) + .startObject( "mapping" ).field( "type", "string" ) + .field( "index", "not_analyzed" ) + .endObject() + .endObject() + .endObject() + + + .startObject() .startObject( "context_template" ) - .field( "match", IndexingUtils.ENTITY_CONTEXT ) + .field( "match", IndexingUtils.ENTITY_CONTEXT_FIELDNAME ) .field( "match_mapping_type", "string" ) .startObject( "mapping" ).field( "type", "string" ) .field( "index", "not_analyzed" ).endObject() @@ -154,11 +181,8 @@ public class IndexingUtils { // any string with field name that starts with sa_ gets analyzed .startObject() .startObject( "template_1" ).field( "match", ANALYZED_STRING_PREFIX + "*" ) - .field( "match_mapping_type", "string" ) - .startObject( "mapping" ).field( "type", "string" ) - .field( "index", "analyzed" ) - .endObject() - .endObject() + .field( "match_mapping_type", "string" ).startObject( "mapping" ).field( "type", "string" ) + .field( "index", "analyzed" ).endObject().endObject() .endObject() // all other strings are not analyzed http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/707747d1/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java index 778134e..07bee44 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java @@ -43,6 +43,7 @@ import org.apache.commons.lang.StringUtils; import org.apache.usergrid.persistence.index.exceptions.IndexException; import org.apache.usergrid.persistence.index.exceptions.QueryParseException; import org.apache.usergrid.persistence.index.impl.EsQueryVistor; +import org.apache.usergrid.persistence.index.impl.IndexingUtils; import org.apache.usergrid.persistence.index.query.tree.AndOperand; import org.apache.usergrid.persistence.index.query.tree.ContainsOperand; import org.apache.usergrid.persistence.index.query.tree.CpQueryFilterLexer; @@ -142,10 +143,23 @@ public class Query { } - public QueryBuilder createQueryBuilder() { + public QueryBuilder createQueryBuilder(final String context) { - QueryBuilder queryBuilder = null; + QueryBuilder queryBuilder; + + + /** + * Add our filter for context to our query for fast execution. Fast because it utilizes bitsets + * internally. See this post for more detail. + * http://www.elasticsearch.org/blog/all-about-elasticsearch-filter-bitsets/ + */ + + + + + + //we have a root operand. Translate our AST into an ES search if ( getRootOperand() != null ) { QueryVisitor v = new EsQueryVistor(); try { @@ -154,11 +168,15 @@ public class Query { } catch ( IndexException ex ) { throw new RuntimeException( "Error building ElasticSearch query", ex ); } - queryBuilder = v.getQueryBuilder(); + + // TODO evaluate performance when it's an all query. Do we need to put the context term first for performance? + queryBuilder = QueryBuilders.boolQuery().must( v.getQueryBuilder() ) + .must( QueryBuilders.termQuery( IndexingUtils.ENTITY_CONTEXT_FIELDNAME, context ) ); } - if ( queryBuilder == null ) { - queryBuilder = QueryBuilders.matchAllQuery(); + //nothing was specified ensure we specify the context in the search + else { + queryBuilder = QueryBuilders.termQuery( IndexingUtils.ENTITY_CONTEXT_FIELDNAME, context ); } return queryBuilder; http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/707747d1/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java index 1bbb301..a1240f1 100644 --- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java +++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java @@ -45,6 +45,7 @@ import org.apache.usergrid.persistence.index.EntityIndex; import org.apache.usergrid.persistence.index.EntityIndexBatch; import org.apache.usergrid.persistence.index.EntityIndexFactory; import org.apache.usergrid.persistence.index.IndexScope; +import org.apache.usergrid.persistence.index.SearchTypes; import org.apache.usergrid.persistence.index.guice.TestIndexModule; import org.apache.usergrid.persistence.index.query.CandidateResults; import org.apache.usergrid.persistence.index.query.Query; @@ -69,393 +70,403 @@ import static org.junit.Assert.assertNotEquals; @UseModules({ TestIndexModule.class }) public class EntityIndexTest extends BaseIT { -// private static final Logger log = LoggerFactory.getLogger( EntityIndexTest.class ); -// -// @ClassRule -// public static CassandraRule cass = new CassandraRule(); -// -// @Rule -// public ElasticSearchResource elasticSearchResource = new ElasticSearchResource(); -// -// @Inject -// @Rule -// public MigrationManagerRule migrationManagerRule; -// -// @Inject -// public EntityIndexFactory eif; -// -// -// -// @Test -// public void testIndex() throws IOException { -// -// final int MAX_ENTITIES = 100; -// -// Id appId = new SimpleId( "application" ); -// -// ApplicationScope applicationScope = new ApplicationScopeImpl( appId ); -// -// IndexScope indexScope = new IndexScopeImpl( appId, "things" ); -// -// -// EntityIndex entityIndex = eif.createEntityIndex( applicationScope ); -// entityIndex.initializeIndex(); -// -// InputStream is = this.getClass().getResourceAsStream( "/sample-large.json" ); -// ObjectMapper mapper = new ObjectMapper(); -// List<Object> sampleJson = mapper.readValue( is, new TypeReference<List<Object>>() {} ); -// -// int count = 0; -// StopWatch timer = new StopWatch(); -// timer.start(); -// -// final EntityIndexBatch batch = entityIndex.createBatch(); -// -// for ( Object o : sampleJson ) { -// -// Map<String, Object> item = ( Map<String, Object> ) o; -// -// Entity entity = new Entity( indexScope.getName() ); -// entity = EntityIndexMapUtils.fromMap( entity, item ); -// EntityUtils.setVersion( entity, UUIDGenerator.newTimeUUID() ); -// -// batch.index( indexScope, entity ); -// -// if(count %1000 == 0){ -// batch.execute(); -// } -// -// -// -// if ( count++ > MAX_ENTITIES ) { -// break; -// } -// } -// -// batch.execute(); -// timer.stop(); -// log.info( "Total time to index {} entries {}ms, average {}ms/entry", -// new Object[] { count, timer.getTime(), timer.getTime() / count } ); -// -// entityIndex.refresh(); -// -// -// testQueries( indexScope, entityIndex ); -// } -// -// -// @Test -// public void testDeindex() { -// -// Id appId = new SimpleId( "application" ); -// -// ApplicationScope applicationScope = new ApplicationScopeImpl( appId ); -// -// IndexScope indexScope = new IndexScopeImpl( appId, "fastcars", entityType ); -// -// EntityIndex entityIndex = eif.createEntityIndex( applicationScope ); -// entityIndex.initializeIndex(); -// -// Map entityMap = new HashMap() {{ -// put( "name", "Ferrari 212 Inter" ); -// put( "introduced", 1952 ); -// put( "topspeed", 215 ); -// }}; -// -// -// Entity entity = EntityIndexMapUtils.fromMap( entityMap ); -// EntityUtils.setId( entity, new SimpleId( "fastcar" ) ); -// EntityUtils.setVersion( entity, UUIDGenerator.newTimeUUID() ); -// entityIndex.createBatch().index(indexScope , entity ).executeAndRefresh(); -// -// CandidateResults candidateResults = entityIndex.search(indexScope, Query.fromQL( "name contains 'Ferrari*'" ) ); -// assertEquals( 1, candidateResults.size() ); -// -// entityIndex.createBatch().deindex( indexScope, entity ).execute(); -// -// entityIndex.refresh(); -// -// candidateResults = entityIndex.search( indexScope, Query.fromQL( "name contains 'Ferrari*'" ) ); -// assertEquals( 0, candidateResults.size() ); -// } -// -// -// private void testQuery(final IndexScope scope, final EntityIndex entityIndex, final String queryString, final int num ) { -// -// StopWatch timer = new StopWatch(); -// timer.start(); -// Query query = Query.fromQL( queryString ); -// query.setLimit( 1000 ); -// CandidateResults candidateResults = entityIndex.search( scope, query ); -// timer.stop(); -// -// assertEquals( num, candidateResults.size() ); -// log.debug( "Query time {}ms", timer.getTime() ); -// } -// -// -// private void testQueries(final IndexScope scope, final EntityIndex entityIndex ) { -// -// -// testQuery(scope, entityIndex, "name = 'Morgan Pierce'", 1 ); -// -// testQuery(scope, entityIndex, "name = 'morgan pierce'", 1 ); -// -// testQuery(scope, entityIndex, "name = 'Morgan'", 0 ); -// -// testQuery(scope, entityIndex, "name contains 'Morgan'", 1 ); -// -// testQuery(scope, entityIndex, "company > 'GeoLogix'", 64 ); -// -// testQuery(scope, entityIndex, "gender = 'female'", 45 ); -// -// testQuery(scope, entityIndex, "name = 'Minerva Harrell' and age > 39", 1 ); -// -// testQuery(scope, entityIndex, "name = 'Minerva Harrell' and age > 39 and age < 41", 1 ); -// -// testQuery(scope, entityIndex, "name = 'Minerva Harrell' and age > 40", 0 ); -// -// testQuery(scope, entityIndex, "name = 'Minerva Harrell' and age >= 40", 1 ); -// -// testQuery(scope, entityIndex, "name = 'Minerva Harrell' and age <= 40", 1 ); -// -// testQuery(scope, entityIndex, "name = 'Morgan* '", 1 ); -// -// testQuery(scope, entityIndex, "name = 'Morgan*'", 1 ); -// -// -// // test a couple of array sub-property queries -// -// int totalUsers = 102; -// -// // nobody has a friend named Jack the Ripper -// testQuery(scope, entityIndex, "friends.name = 'Jack the Ripper'", 0 ); -// -// // everybody doesn't have a friend named Jack the Ripper -// testQuery(scope, entityIndex, "not (friends.name = 'Jack the Ripper')", totalUsers ); -// -// // one person has a friend named Shari Hahn -// testQuery(scope, entityIndex, "friends.name = 'Wendy Moody'", 1 ); -// -// // everybody but 1 doesn't have a friend named Shari Hahh -// testQuery(scope, entityIndex, "not (friends.name = 'Shari Hahn')", totalUsers - 1); -// -// } -// -// -// /** -// * Tests that Entity-to-map and Map-to-entity round trip works. -// */ -// @Test -// public void testEntityIndexMapUtils() throws IOException { -// -// InputStream is = this.getClass().getResourceAsStream( "/sample-small.json" ); -// ObjectMapper mapper = new ObjectMapper(); -// List<Object> contacts = mapper.readValue( is, new TypeReference<List<Object>>() {} ); -// -// for ( Object o : contacts ) { -// -// Map<String, Object> map1 = ( Map<String, Object> ) o; -// -// // convert map to entity -// Entity entity1 = EntityIndexMapUtils.fromMap( map1 ); -// -// // convert entity back to map -// Map map2 = EntityIndexMapUtils.toMap( entity1 ); -// -// // the two maps should be the same -// Map diff = Maps.difference( map1, map2 ).entriesDiffering(); -// assertEquals( 0, diff.size() ); -// } -// } -// -// -// @Test -// public void getEntityVersions() throws Exception { -// -// Id appId = new SimpleId( "application" ); -// Id ownerId = new SimpleId( "owner" ); -// -// ApplicationScope applicationScope = new ApplicationScopeImpl( appId ); -// -// IndexScope indexScope = new IndexScopeImpl( ownerId, "user", entityType ); -// -// -// -// EntityIndex entityIndex = eif.createEntityIndex( applicationScope ); -// entityIndex.initializeIndex(); -// -// final String middleName = "middleName" + UUIDUtils.newTimeUUID(); -// Map<String, Object> properties = new LinkedHashMap<String, Object>(); -// properties.put( "username", "edanuff" ); -// properties.put( "email", "[email protected]" ); -// properties.put( "middlename", middleName ); -// -// Map entityMap = new HashMap() {{ -// put( "username", "edanuff" ); -// put( "email", "[email protected]" ); -// put( "middlename", middleName ); -// }}; -// -// Entity user = EntityIndexMapUtils.fromMap( entityMap ); -// EntityUtils.setId( user, new SimpleId( "edanuff" ) ); -// EntityUtils.setVersion( user, UUIDGenerator.newTimeUUID() ); -// -// -// final EntityIndexBatch batch = entityIndex.createBatch(); -// -// batch.index( indexScope, user ); -// -// user.setField( new StringField( "address1", "1782 address st" ) ); -// batch.index( indexScope, user ); -// user.setField( new StringField( "address2", "apt 508" ) ); -// batch.index( indexScope, user ); -// user.setField( new StringField( "address3", "apt 508" ) ); -// batch.index( indexScope, user ); -// batch.executeAndRefresh(); -// -// CandidateResults results = entityIndex.getEntityVersions(indexScope, user.getId() ); -// -// assertEquals(1, results.size()); -// assertEquals( results.get( 0 ).getId(), user.getId() ); -// assertEquals( results.get(0).getVersion(), user.getVersion()); -// } -// -// -// @Test -// public void deleteVerification() throws Throwable { -// -// Id appId = new SimpleId( "application" ); -// Id ownerId = new SimpleId( "owner" ); -// -// ApplicationScope applicationScope = new ApplicationScopeImpl( appId ); -// -// IndexScope appScope = new IndexScopeImpl( ownerId, "user", entityType ); -// -// EntityIndex ei = eif.createEntityIndex( applicationScope ); -// ei.initializeIndex(); -// -// final String middleName = "middleName" + UUIDUtils.newTimeUUID(); -// -// Map entityMap = new HashMap() {{ -// put( "username", "edanuff" ); -// put( "email", "[email protected]" ); -// put( "middlename", middleName ); -// }}; -// -// Entity user = EntityIndexMapUtils.fromMap( entityMap ); -// EntityUtils.setId( user, new SimpleId( "edanuff" ) ); -// EntityUtils.setVersion( user, UUIDGenerator.newTimeUUID() ); -// -// -// EntityIndexBatch batch = ei.createBatch(); -// -// batch.index( appScope, user ).executeAndRefresh(); -// Query query = new Query(); -// query.addEqualityFilter( "username", "edanuff" ); -// CandidateResults r = ei.search( appScope, query ); -// assertEquals( user.getId(), r.get( 0 ).getId() ); -// -// batch.deindex(appScope, user.getId(), user.getVersion() ).executeAndRefresh(); -// -// -// // EntityRef -// query = new Query(); -// query.addEqualityFilter( "username", "edanuff" ); -// r = ei.search(appScope, query ); -// -// assertFalse( r.iterator().hasNext() ); -// } -// -// @Test -// public void multiValuedTypes() { -// -// Id appId = new SimpleId( "entityindextest" ); -// Id ownerId = new SimpleId( "multivaluedtype" ); -// ApplicationScope applicationScope = new ApplicationScopeImpl( appId ); -// -// IndexScope appScope = new IndexScopeImpl( ownerId, "user", entityType ); -// -// EntityIndex ei = eif.createEntityIndex( applicationScope ); -// ei.createBatch(); -// -// // Bill has favorites as string, age as string and retirement goal as number -// Map billMap = new HashMap() {{ -// put( "username", "bill" ); -// put( "email", "[email protected]" ); -// put( "age", "thirtysomething"); -// put( "favorites", "scallops, croquet, wine"); -// put( "retirementGoal", 100000); -// }}; -// Entity bill = EntityIndexMapUtils.fromMap( billMap ); -// EntityUtils.setId( bill, new SimpleId( UUIDGenerator.newTimeUUID(), "user" ) ); -// EntityUtils.setVersion( bill, UUIDGenerator.newTimeUUID() ); -// -// EntityIndexBatch batch = ei.createBatch(); -// -// batch.index( appScope, bill ); -// -// // Fred has age as int, favorites as object and retirement goal as object -// Map fredMap = new HashMap() {{ -// put( "username", "fred" ); -// put( "email", "[email protected]" ); -// put( "age", 41 ); -// put( "favorites", new HashMap<String, Object>() {{ -// put("food", "cheezewiz"); -// put("sport", "nascar"); -// put("beer", "budwizer"); -// }}); -// put( "retirementGoal", new HashMap<String, Object>() {{ -// put("car", "Firebird"); -// put("home", "Mobile"); -// }}); -// }}; -// Entity fred = EntityIndexMapUtils.fromMap( fredMap ); -// EntityUtils.setId( fred, new SimpleId( UUIDGenerator.newTimeUUID(), "user" ) ); -// EntityUtils.setVersion( fred, UUIDGenerator.newTimeUUID() ); -// batch.index( appScope, fred ); -// -// batch.executeAndRefresh(); -// -// Query query = new Query(); -// query.addEqualityFilter( "username", "bill" ); -// CandidateResults r = ei.search( appScope, query ); -// assertEquals( bill.getId(), r.get( 0 ).getId() ); -// -// query = new Query(); -// query.addEqualityFilter( "username", "fred" ); -// r = ei.search( appScope, query ); -// assertEquals( fred.getId(), r.get( 0 ).getId() ); -// -// query = new Query(); -// query.addEqualityFilter( "age", 41 ); -// r = ei.search( appScope, query ); -// assertEquals( fred.getId(), r.get( 0 ).getId() ); -// -// query = new Query(); -// query.addEqualityFilter( "age", "thirtysomething" ); -// r = ei.search( appScope, query ); -// assertEquals( bill.getId(), r.get( 0 ).getId() ); -// } -// -// -// @Test -// public void healthTest() { -// -// Id appId = new SimpleId( "application" ); -// ApplicationScope applicationScope = new ApplicationScopeImpl( appId ); -// -// EntityIndex ei = eif.createEntityIndex( applicationScope ); -// -// assertNotEquals( "cluster should be ok", Health.RED, ei.getClusterHealth() ); -// assertEquals( "index not be ready yet", Health.RED, ei.getIndexHealth() ); -// -// ei.initializeIndex(); -// ei.refresh(); -// -// assertNotEquals( "cluster should be fine", Health.RED, ei.getIndexHealth() ); -// assertNotEquals( "cluster should be ready now", Health.RED, ei.getClusterHealth() ); -// } + private static final Logger log = LoggerFactory.getLogger( EntityIndexTest.class ); + + @ClassRule + public static CassandraRule cass = new CassandraRule(); + + @Rule + public ElasticSearchResource elasticSearchResource = new ElasticSearchResource(); + + @Inject + @Rule + public MigrationManagerRule migrationManagerRule; + + @Inject + public EntityIndexFactory eif; + + + + @Test + public void testIndex() throws IOException { + + final int MAX_ENTITIES = 100; + + Id appId = new SimpleId( "application" ); + + ApplicationScope applicationScope = new ApplicationScopeImpl( appId ); + + IndexScope indexScope = new IndexScopeImpl( appId, "things" ); + + final String entityType = "thing"; + + + final SearchTypes searchTypes = SearchTypes.fromTypes( entityType ); + + + EntityIndex entityIndex = eif.createEntityIndex( applicationScope ); + entityIndex.initializeIndex(); + + InputStream is = this.getClass().getResourceAsStream( "/sample-large.json" ); + ObjectMapper mapper = new ObjectMapper(); + List<Object> sampleJson = mapper.readValue( is, new TypeReference<List<Object>>() {} ); + + int count = 0; + StopWatch timer = new StopWatch(); + timer.start(); + + final EntityIndexBatch batch = entityIndex.createBatch(); + + for ( Object o : sampleJson ) { + + Map<String, Object> item = ( Map<String, Object> ) o; + + Entity entity = new Entity( entityType ); + entity = EntityIndexMapUtils.fromMap( entity, item ); + EntityUtils.setVersion( entity, UUIDGenerator.newTimeUUID() ); + + batch.index( indexScope, entity ); + + if(count %1000 == 0){ + batch.execute(); + } + + + + if ( count++ > MAX_ENTITIES ) { + break; + } + } + + batch.execute(); + timer.stop(); + log.info( "Total time to index {} entries {}ms, average {}ms/entry", + new Object[] { count, timer.getTime(), timer.getTime() / count } ); + + entityIndex.refresh(); + + + testQueries( indexScope, searchTypes, entityIndex ); + } + + + @Test + public void testDeindex() { + + Id appId = new SimpleId( "application" ); + + ApplicationScope applicationScope = new ApplicationScopeImpl( appId ); + + IndexScope indexScope = new IndexScopeImpl( appId, "fastcars" ); + + EntityIndex entityIndex = eif.createEntityIndex( applicationScope ); + entityIndex.initializeIndex(); + + Map entityMap = new HashMap() {{ + put( "name", "Ferrari 212 Inter" ); + put( "introduced", 1952 ); + put( "topspeed", 215 ); + }}; + + + Entity entity = EntityIndexMapUtils.fromMap( entityMap ); + EntityUtils.setId( entity, new SimpleId( "fastcar" ) ); + EntityUtils.setVersion( entity, UUIDGenerator.newTimeUUID() ); + entityIndex.createBatch().index(indexScope , entity ).executeAndRefresh(); + + CandidateResults candidateResults = entityIndex.search( indexScope, SearchTypes.fromTypes(entity.getId().getType()), + Query.fromQL( "name contains 'Ferrari*'" ) ); + assertEquals( 1, candidateResults.size() ); + + entityIndex.createBatch().deindex( indexScope, entity ).execute(); + + entityIndex.refresh(); + + candidateResults = entityIndex.search( indexScope, SearchTypes.fromTypes(entity.getId().getType()), Query.fromQL( "name contains 'Ferrari*'" ) ); + assertEquals( 0, candidateResults.size() ); + } + + + private void testQuery(final IndexScope scope, final SearchTypes searchTypes, final EntityIndex entityIndex, final String queryString, final int num ) { + + StopWatch timer = new StopWatch(); + timer.start(); + Query query = Query.fromQL( queryString ); + query.setLimit( 1000 ); + CandidateResults candidateResults = entityIndex.search( scope, searchTypes, query ); + timer.stop(); + + assertEquals( num, candidateResults.size() ); + log.debug( "Query time {}ms", timer.getTime() ); + } + + + private void testQueries(final IndexScope scope, SearchTypes searchTypes, final EntityIndex entityIndex ) { + + + testQuery(scope, searchTypes, entityIndex, "name = 'Morgan Pierce'", 1 ); + + testQuery(scope, searchTypes, entityIndex, "name = 'morgan pierce'", 1 ); + + testQuery(scope, searchTypes, entityIndex, "name = 'Morgan'", 0 ); + + testQuery(scope, searchTypes, entityIndex, "name contains 'Morgan'", 1 ); + + testQuery(scope, searchTypes, entityIndex, "company > 'GeoLogix'", 64 ); + + testQuery(scope, searchTypes, entityIndex, "gender = 'female'", 45 ); + + testQuery(scope, searchTypes, entityIndex, "name = 'Minerva Harrell' and age > 39", 1 ); + + testQuery(scope, searchTypes, entityIndex, "name = 'Minerva Harrell' and age > 39 and age < 41", 1 ); + + testQuery(scope, searchTypes, entityIndex, "name = 'Minerva Harrell' and age > 40", 0 ); + + testQuery(scope, searchTypes, entityIndex, "name = 'Minerva Harrell' and age >= 40", 1 ); + + testQuery(scope, searchTypes, entityIndex, "name = 'Minerva Harrell' and age <= 40", 1 ); + + testQuery(scope, searchTypes, entityIndex, "name = 'Morgan* '", 1 ); + + testQuery(scope, searchTypes, entityIndex, "name = 'Morgan*'", 1 ); + + + // test a couple of array sub-property queries + + int totalUsers = 102; + + // nobody has a friend named Jack the Ripper + testQuery(scope, searchTypes, entityIndex, "friends.name = 'Jack the Ripper'", 0 ); + + // everybody doesn't have a friend named Jack the Ripper + testQuery(scope, searchTypes,entityIndex, "not (friends.name = 'Jack the Ripper')", totalUsers ); + + // one person has a friend named Shari Hahn + testQuery(scope, searchTypes, entityIndex, "friends.name = 'Wendy Moody'", 1 ); + + // everybody but 1 doesn't have a friend named Shari Hahh + testQuery(scope, searchTypes, entityIndex, "not (friends.name = 'Shari Hahn')", totalUsers - 1); + + } + + + /** + * Tests that Entity-to-map and Map-to-entity round trip works. + */ + @Test + public void testEntityIndexMapUtils() throws IOException { + + InputStream is = this.getClass().getResourceAsStream( "/sample-small.json" ); + ObjectMapper mapper = new ObjectMapper(); + List<Object> contacts = mapper.readValue( is, new TypeReference<List<Object>>() {} ); + + for ( Object o : contacts ) { + + Map<String, Object> map1 = ( Map<String, Object> ) o; + + // convert map to entity + Entity entity1 = EntityIndexMapUtils.fromMap( map1 ); + + // convert entity back to map + Map map2 = EntityIndexMapUtils.toMap( entity1 ); + + // the two maps should be the same + Map diff = Maps.difference( map1, map2 ).entriesDiffering(); + assertEquals( 0, diff.size() ); + } + } + + + @Test + public void getEntityVersions() throws Exception { + + Id appId = new SimpleId( "application" ); + Id ownerId = new SimpleId( "owner" ); + + ApplicationScope applicationScope = new ApplicationScopeImpl( appId ); + + IndexScope indexScope = new IndexScopeImpl( ownerId, "users" ); + + + + EntityIndex entityIndex = eif.createEntityIndex( applicationScope ); + entityIndex.initializeIndex(); + + final String middleName = "middleName" + UUIDUtils.newTimeUUID(); + Map<String, Object> properties = new LinkedHashMap<String, Object>(); + properties.put( "username", "edanuff" ); + properties.put( "email", "[email protected]" ); + properties.put( "middlename", middleName ); + + Map entityMap = new HashMap() {{ + put( "username", "edanuff" ); + put( "email", "[email protected]" ); + put( "middlename", middleName ); + }}; + + final Id userId = new SimpleId("user"); + + Entity user = EntityIndexMapUtils.fromMap( entityMap ); + EntityUtils.setId( user, userId); + EntityUtils.setVersion( user, UUIDGenerator.newTimeUUID() ); + + + final EntityIndexBatch batch = entityIndex.createBatch(); + + batch.index( indexScope, user ); + + user.setField( new StringField( "address1", "1782 address st" ) ); + batch.index( indexScope, user ); + user.setField( new StringField( "address2", "apt 508" ) ); + batch.index( indexScope, user ); + user.setField( new StringField( "address3", "apt 508" ) ); + batch.index( indexScope, user ); + batch.executeAndRefresh(); + + CandidateResults results = entityIndex.getEntityVersions(indexScope, user.getId() ); + + assertEquals(1, results.size()); + assertEquals( results.get( 0 ).getId(), user.getId() ); + assertEquals( results.get(0).getVersion(), user.getVersion()); + } + + + @Test + public void deleteVerification() throws Throwable { + + Id appId = new SimpleId( "application" ); + Id ownerId = new SimpleId( "owner" ); + + ApplicationScope applicationScope = new ApplicationScopeImpl( appId ); + + IndexScope appScope = new IndexScopeImpl( ownerId, "user" ); + + EntityIndex ei = eif.createEntityIndex( applicationScope ); + ei.initializeIndex(); + + final String middleName = "middleName" + UUIDUtils.newTimeUUID(); + + Map entityMap = new HashMap() {{ + put( "username", "edanuff" ); + put( "email", "[email protected]" ); + put( "middlename", middleName ); + }}; + + Entity user = EntityIndexMapUtils.fromMap( entityMap ); + EntityUtils.setId( user, new SimpleId( "edanuff" ) ); + EntityUtils.setVersion( user, UUIDGenerator.newTimeUUID() ); + + + EntityIndexBatch batch = ei.createBatch(); + + batch.index( appScope, user ).executeAndRefresh(); + Query query = new Query(); + query.addEqualityFilter( "username", "edanuff" ); + CandidateResults r = ei.search( appScope, SearchTypes.fromTypes( "edanuff" ), query ); + assertEquals( user.getId(), r.get( 0 ).getId() ); + + batch.deindex(appScope, user.getId(), user.getVersion() ).executeAndRefresh(); + + + // EntityRef + query = new Query(); + query.addEqualityFilter( "username", "edanuff" ); + r = ei.search(appScope,SearchTypes.fromTypes( "edanuff" ), query ); + + assertFalse( r.iterator().hasNext() ); + } + + @Test + public void multiValuedTypes() { + + Id appId = new SimpleId( "entityindextest" ); + Id ownerId = new SimpleId( "multivaluedtype" ); + ApplicationScope applicationScope = new ApplicationScopeImpl( appId ); + + IndexScope appScope = new IndexScopeImpl( ownerId, "user" ); + + EntityIndex ei = eif.createEntityIndex( applicationScope ); + ei.createBatch(); + + // Bill has favorites as string, age as string and retirement goal as number + Map billMap = new HashMap() {{ + put( "username", "bill" ); + put( "email", "[email protected]" ); + put( "age", "thirtysomething"); + put( "favorites", "scallops, croquet, wine"); + put( "retirementGoal", 100000); + }}; + Entity bill = EntityIndexMapUtils.fromMap( billMap ); + EntityUtils.setId( bill, new SimpleId( UUIDGenerator.newTimeUUID(), "user" ) ); + EntityUtils.setVersion( bill, UUIDGenerator.newTimeUUID() ); + + EntityIndexBatch batch = ei.createBatch(); + + batch.index( appScope, bill ); + + // Fred has age as int, favorites as object and retirement goal as object + Map fredMap = new HashMap() {{ + put( "username", "fred" ); + put( "email", "[email protected]" ); + put( "age", 41 ); + put( "favorites", new HashMap<String, Object>() {{ + put("food", "cheezewiz"); + put("sport", "nascar"); + put("beer", "budwizer"); + }}); + put( "retirementGoal", new HashMap<String, Object>() {{ + put("car", "Firebird"); + put("home", "Mobile"); + }}); + }}; + Entity fred = EntityIndexMapUtils.fromMap( fredMap ); + EntityUtils.setId( fred, new SimpleId( UUIDGenerator.newTimeUUID(), "user" ) ); + EntityUtils.setVersion( fred, UUIDGenerator.newTimeUUID() ); + batch.index( appScope, fred ); + + batch.executeAndRefresh(); + + final SearchTypes searchTypes = SearchTypes.fromTypes( "user" ); + + Query query = new Query(); + query.addEqualityFilter( "username", "bill" ); + CandidateResults r = ei.search( appScope, searchTypes, query ); + assertEquals( bill.getId(), r.get( 0 ).getId() ); + + query = new Query(); + query.addEqualityFilter( "username", "fred" ); + r = ei.search( appScope, searchTypes, query ); + assertEquals( fred.getId(), r.get( 0 ).getId() ); + + query = new Query(); + query.addEqualityFilter( "age", 41 ); + r = ei.search( appScope, searchTypes, query ); + assertEquals( fred.getId(), r.get( 0 ).getId() ); + + query = new Query(); + query.addEqualityFilter( "age", "thirtysomething" ); + r = ei.search( appScope, searchTypes, query ); + assertEquals( bill.getId(), r.get( 0 ).getId() ); + } + + + @Test + public void healthTest() { + + Id appId = new SimpleId( "application" ); + ApplicationScope applicationScope = new ApplicationScopeImpl( appId ); + + EntityIndex ei = eif.createEntityIndex( applicationScope ); + + assertNotEquals( "cluster should be ok", Health.RED, ei.getClusterHealth() ); + assertEquals( "index not be ready yet", Health.RED, ei.getIndexHealth() ); + + ei.initializeIndex(); + ei.refresh(); + + assertNotEquals( "cluster should be fine", Health.RED, ei.getIndexHealth() ); + assertNotEquals( "cluster should be ready now", Health.RED, ei.getClusterHealth() ); + } } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/707747d1/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java index d0e2d11..34185b3 100644 --- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java +++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java @@ -429,7 +429,7 @@ public class GrammarTreeTest { assertEquals( 37f, withinOperand.getLatitude().getFloatValue(), 0 ); assertEquals( -75f, withinOperand.getLongitude().getFloatValue(), 0 ); - QueryBuilder qb = query.createQueryBuilder(); + QueryBuilder qb = query.createQueryBuilder("testcontext"); }
