Updates constants. Fixes scope in query
Need to finish version query Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/ef77a5a4 Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/ef77a5a4 Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/ef77a5a4 Branch: refs/heads/two-dot-o Commit: ef77a5a4eeca44496bf0ced75cf662f49040cba8 Parents: 2db132c Author: Todd Nine <[email protected]> Authored: Thu Nov 6 16:55:13 2014 -0700 Committer: Todd Nine <[email protected]> Committed: Thu Nov 6 17:43:18 2014 -0700 ---------------------------------------------------------------------- .../index/impl/EsEntityIndexBatchImpl.java | 11 +- .../index/impl/EsEntityIndexImpl.java | 126 ++- .../persistence/index/impl/EsQueryVistor.java | 3 +- .../persistence/index/impl/IndexingUtils.java | 116 +-- .../usergrid/persistence/index/query/Query.java | 28 +- .../impl/EntityConnectionIndexImplTest.java | 45 +- .../persistence/index/impl/EntityIndexTest.java | 785 ++++++++++--------- .../persistence/index/impl/EsRunner.java | 2 +- .../persistence/query/tree/GrammarTreeTest.java | 2 +- 9 files changed, 590 insertions(+), 528 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ef77a5a4/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java index 5a19d1e..01f22c0 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java @@ -61,8 +61,8 @@ import com.google.common.base.Joiner; import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ANALYZED_STRING_PREFIX; import static org.apache.usergrid.persistence.index.impl.IndexingUtils.BOOLEAN_PREFIX; -import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITYID_FIELDNAME; -import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITY_CONTEXT; +import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITYID_ID_FIELDNAME; +import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITY_CONTEXT_FIELDNAME; import static org.apache.usergrid.persistence.index.impl.IndexingUtils.GEO_PREFIX; import static org.apache.usergrid.persistence.index.impl.IndexingUtils.NUMBER_PREFIX; import static org.apache.usergrid.persistence.index.impl.IndexingUtils.STRING_PREFIX; @@ -116,7 +116,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch { if ( log.isDebugEnabled() ) { log.debug( "Indexing entity {}:{} in scope\n app {}\n " - + "owner {}\n name {}\n type {} \n scope type{}", new Object[] { + + "owner {}\n name {}\n type {} \n scope type {}", new Object[] { entity.getId().getType(), entity.getId().getUuid(), applicationScope.getApplication(), indexScope.getOwner(), indexScope.getName(), entityType, context } ); @@ -255,10 +255,11 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch { final Map entityMap = entityToMap( entity ); //add the context for filtering later - entityMap.put( ENTITY_CONTEXT, context ); + entityMap.put( ENTITY_CONTEXT_FIELDNAME, context ); //but the fieldname - entityMap.put( ENTITYID_FIELDNAME, entity.getId().getUuid().toString().toLowerCase() ); + //we have to prefix because we use query equality to seek this later. TODO see if we can make this more declarative + entityMap.put( STRING_PREFIX+ ENTITYID_ID_FIELDNAME, IndexingUtils.idString(entity.getId()).toLowerCase() ); return entityMap; } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ef77a5a4/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java index 7378ecd..d7955c4 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java @@ -33,17 +33,14 @@ import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchScrollRequestBuilder; import org.elasticsearch.client.AdminClient; -import org.elasticsearch.client.Client; import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.query.FilterBuilder; -import org.elasticsearch.index.query.FilterBuilders; import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.index.query.TermFilterBuilder; import org.elasticsearch.indices.IndexAlreadyExistsException; import org.elasticsearch.indices.IndexMissingException; import org.elasticsearch.search.SearchHit; @@ -54,8 +51,6 @@ import org.elasticsearch.search.sort.SortOrder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.lucene.queryparser.xml.FilterBuilderFactory; - import org.apache.usergrid.persistence.core.scope.ApplicationScope; import org.apache.usergrid.persistence.core.util.Health; import org.apache.usergrid.persistence.core.util.ValidationUtils; @@ -76,9 +71,9 @@ import com.google.inject.Inject; import com.google.inject.assistedinject.Assisted; import static org.apache.usergrid.persistence.index.impl.IndexingUtils.BOOLEAN_PREFIX; -import static org.apache.usergrid.persistence.index.impl.IndexingUtils.DOC_ID_SEPARATOR_SPLITTER; -import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITYID_FIELDNAME; +import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITYID_ID_FIELDNAME; import static org.apache.usergrid.persistence.index.impl.IndexingUtils.NUMBER_PREFIX; +import static org.apache.usergrid.persistence.index.impl.IndexingUtils.SPLITTER; import static org.apache.usergrid.persistence.index.impl.IndexingUtils.STRING_PREFIX; @@ -148,12 +143,12 @@ public class EsEntityIndexImpl implements EntityIndex { final int numberOfShards = config.getNumberOfShards(); final int numberOfReplicas = config.getNumberOfReplicas(); - Settings settings = ImmutableSettings.settingsBuilder() - .put("index.number_of_shards", numberOfShards ) - .put("index.number_of_replicas", numberOfReplicas).build(); + Settings settings = ImmutableSettings.settingsBuilder().put( "index.number_of_shards", numberOfShards ) + .put( "index.number_of_replicas", numberOfReplicas ).build(); - final CreateIndexResponse cir = admin.indices().prepareCreate( indexName ).setSettings( settings ).execute().actionGet(); + final CreateIndexResponse cir = + admin.indices().prepareCreate( indexName ).setSettings( settings ).execute().actionGet(); logger.info( "Created new Index Name [{}] ACK=[{}]", indexName, cir.isAcknowledged() ); @@ -175,8 +170,8 @@ public class EsEntityIndexImpl implements EntityIndex { /** - * Tests writing a document to a new index to ensure it's working correctly. - * See this post: http://s.apache.org/index-missing-exception + * Tests writing a document to a new index to ensure it's working correctly. See this post: + * http://s.apache.org/index-missing-exception */ private void testNewIndex() { @@ -188,18 +183,17 @@ public class EsEntityIndexImpl implements EntityIndex { public boolean doOp() { final String tempId = UUIDGenerator.newTimeUUID().toString(); - esProvider.getClient().prepareIndex( indexName, VERIFY_TYPE, tempId ) - .setSource( DEFAULT_PAYLOAD ).get(); + esProvider.getClient().prepareIndex( indexName, VERIFY_TYPE, tempId ).setSource( DEFAULT_PAYLOAD ) + .get(); - logger.info( "Successfully created new document with docId {} in index {} and type {}", - tempId, indexName, VERIFY_TYPE ); + logger.info( "Successfully created new document with docId {} in index {} and type {}", tempId, + indexName, VERIFY_TYPE ); // delete all types, this way if we miss one it will get cleaned up esProvider.getClient().prepareDeleteByQuery( indexName ).setTypes( VERIFY_TYPE ) - .setQuery( MATCH_ALL_QUERY_BUILDER ).get(); + .setQuery( MATCH_ALL_QUERY_BUILDER ).get(); - logger.info( "Successfully deleted all documents in index {} and type {}", - indexName, VERIFY_TYPE ); + logger.info( "Successfully deleted all documents in index {} and type {}", indexName, VERIFY_TYPE ); return true; } @@ -218,10 +212,11 @@ public class EsEntityIndexImpl implements EntityIndex { XContentBuilder xcb = IndexingUtils.createDoubleStringIndexMapping( XContentFactory.jsonBuilder(), "_default_" ); - PutIndexTemplateResponse pitr = esProvider.getClient().admin().indices().preparePutTemplate( "usergrid_template" ) - .setTemplate( config.getIndexPrefix() + "*" ).addMapping( "_default_", + PutIndexTemplateResponse pitr = + esProvider.getClient().admin().indices().preparePutTemplate( "usergrid_template" ) + .setTemplate( config.getIndexPrefix() + "*" ).addMapping( "_default_", xcb ) // set mapping as the default for all types - .execute().actionGet(); + .execute().actionGet(); } @@ -237,40 +232,40 @@ public class EsEntityIndexImpl implements EntityIndex { final String context = IndexingUtils.createContextName( indexScope ); final String[] entityTypes = searchTypes.getTypeNames(); - QueryBuilder qb = query.createQueryBuilder(); + QueryBuilder qb = query.createQueryBuilder(context); - if ( logger.isDebugEnabled() ) { - logger.debug( "Searching index {}\n scope{} \n type {}\n query {} limit {}", new Object[] { - this.indexName, context, entityTypes, qb.toString().replace( "\n", " " ), query.getLimit() - } ); - } SearchResponse searchResponse; if ( query.getCursor() == null ) { - SearchRequestBuilder srb = - esProvider.getClient().prepareSearch( indexName ).setTypes( entityTypes ).setScroll( - cursorTimeout + "m" ) - .setQuery( qb ); + SearchRequestBuilder srb = esProvider.getClient().prepareSearch( indexName ).setTypes( entityTypes ) + .setScroll( cursorTimeout + "m" ).setQuery( qb );; - final TermFilterBuilder contextFilter = FilterBuilders.termFilter( IndexingUtils.ENTITY_CONTEXT, context ); - final FilterBuilder fb = query.createFilterBuilder(); + //we must add a "must" operation to our entity context. + + //set our final query + + + if ( logger.isDebugEnabled() ) { + logger.debug( "Searching index {}\n scope{} \n type {}\n query {} limit {}", new Object[] { + this.indexName, context, entityTypes, qb.toString().replace( "\n", " " ), query.getLimit() + } ); + } + + + + final FilterBuilder fb = query.createFilterBuilder(); //we have post filters, apply them if ( fb != null ) { -// final FilterBuilder postFilters = FilterBuilders.andFilter(fb, contextFilter ); - logger.debug( " Filter: {} ", fb.toString() ); -// srb = srb.setPostFilter( postFilters ); - } - //no other post filters, just the types - else{ -// srb.setPostFilter( contextFilter ); + srb = srb.setPostFilter( fb ); } + srb = srb.setFrom( 0 ).setSize( query.getLimit() ); for ( Query.SortPredicate sp : query.getSortPredicates() ) { @@ -289,20 +284,20 @@ public class EsEntityIndexImpl implements EntityIndex { // to ignore any fields that are not present. final String stringFieldName = STRING_PREFIX + sp.getPropertyName(); - final FieldSortBuilder stringSort = SortBuilders.fieldSort( stringFieldName ) - .order( order ).ignoreUnmapped( true ); + final FieldSortBuilder stringSort = + SortBuilders.fieldSort( stringFieldName ).order( order ).ignoreUnmapped( true ); srb.addSort( stringSort ); logger.debug( " Sort: {} order by {}", stringFieldName, order.toString() ); final String numberFieldName = NUMBER_PREFIX + sp.getPropertyName(); - final FieldSortBuilder numberSort = SortBuilders.fieldSort( numberFieldName ) - .order( order ).ignoreUnmapped( true ); + final FieldSortBuilder numberSort = + SortBuilders.fieldSort( numberFieldName ).order( order ).ignoreUnmapped( true ); srb.addSort( numberSort ); logger.debug( " Sort: {} order by {}", numberFieldName, order.toString() ); final String booleanFieldName = BOOLEAN_PREFIX + sp.getPropertyName(); - final FieldSortBuilder booleanSort = SortBuilders.fieldSort( booleanFieldName ) - .order( order ).ignoreUnmapped( true ); + final FieldSortBuilder booleanSort = + SortBuilders.fieldSort( booleanFieldName ).order( order ).ignoreUnmapped( true ); srb.addSort( booleanSort ); logger.debug( " Sort: {} order by {}", booleanFieldName, order.toString() ); } @@ -329,7 +324,8 @@ public class EsEntityIndexImpl implements EntityIndex { } logger.debug( "Executing query with cursor: {} ", scrollId ); - SearchScrollRequestBuilder ssrb = esProvider.getClient().prepareSearchScroll( scrollId ).setScroll( cursorTimeout + "m" ); + SearchScrollRequestBuilder ssrb = + esProvider.getClient().prepareSearchScroll( scrollId ).setScroll( cursorTimeout + "m" ); try { searchResponse = ssrb.execute().actionGet(); @@ -351,7 +347,7 @@ public class EsEntityIndexImpl implements EntityIndex { for ( SearchHit hit : hits.getHits() ) { - String[] idparts = hit.getId().split( DOC_ID_SEPARATOR_SPLITTER ); + String[] idparts = hit.getId().split( SPLITTER ); String id = idparts[0]; String type = idparts[1]; String version = idparts[2]; @@ -386,7 +382,7 @@ public class EsEntityIndexImpl implements EntityIndex { return true; } catch ( IndexMissingException e ) { - logger.error( "Unable to refresh index after create. Waiting before sleeping.", e); + logger.error( "Unable to refresh index after create. Waiting before sleeping.", e ); throw e; } } @@ -401,7 +397,7 @@ public class EsEntityIndexImpl implements EntityIndex { @Override public CandidateResults getEntityVersions( final IndexScope scope, final Id id ) { Query query = new Query(); - query.addEqualityFilter( ENTITYID_FIELDNAME, id.getUuid().toString() ); + query.addEqualityFilter( ENTITYID_ID_FIELDNAME, IndexingUtils.idString( id ).toLowerCase() ); CandidateResults results = search( scope, SearchTypes.fromTypes( id.getType() ), query ); return results; } @@ -424,7 +420,6 @@ public class EsEntityIndexImpl implements EntityIndex { /** * Do the retry operation - * @param operation */ private void doInRetry( final RetryOperation operation ) { for ( int i = 0; i < MAX_WAITS; i++ ) { @@ -456,13 +451,13 @@ public class EsEntityIndexImpl implements EntityIndex { public Health getClusterHealth() { try { - ClusterHealthResponse chr = esProvider.getClient().admin().cluster() - .health( new ClusterHealthRequest() ).get(); + ClusterHealthResponse chr = + esProvider.getClient().admin().cluster().health( new ClusterHealthRequest() ).get(); return Health.valueOf( chr.getStatus().name() ); - } - catch (Exception ex) { - logger.error("Error connecting to ElasticSearch", ex); - } + } + catch ( Exception ex ) { + logger.error( "Error connecting to ElasticSearch", ex ); + } // this is bad, red alert! return Health.RED; @@ -474,15 +469,16 @@ public class EsEntityIndexImpl implements EntityIndex { */ @Override public Health getIndexHealth() { - + try { ClusterHealthResponse chr = esProvider.getClient().admin().cluster() - .health( new ClusterHealthRequest( new String[] { indexName } ) ).get(); + .health( new ClusterHealthRequest( new String[] { indexName } ) ) + .get(); return Health.valueOf( chr.getStatus().name() ); - } - catch (Exception ex) { - logger.error("Error connecting to ElasticSearch", ex); - } + } + catch ( Exception ex ) { + logger.error( "Error connecting to ElasticSearch", ex ); + } // this is bad, red alert! return Health.RED; http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ef77a5a4/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java index 0afe992..0e4b0ad 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java @@ -219,9 +219,10 @@ public class EsQueryVistor implements QueryVisitor { String svalue = (String)value; BoolQueryBuilder qb = QueryBuilders.boolQuery(); // let's do a boolean OR - qb.minimumNumberShouldMatch(1); + qb.minimumNumberShouldMatch(1); // field is an entity/array that does not need a prefix on its name + //TODO is this right now that we've updated our doc structure? Should this be "must" instead of should? qb = qb.should( QueryBuilders.wildcardQuery( name, svalue ) ); // or field is just a string that does need a prefix http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ef77a5a4/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java index c8e0b86..c7592bc 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java @@ -37,22 +37,21 @@ public class IndexingUtils { public static final String NUMBER_PREFIX = "nu_"; public static final String BOOLEAN_PREFIX = "bu_"; - public static final String ENTITYID_FIELDNAME = "entityId"; - public static final String DOC_ID_SEPARATOR = "|"; - public static final String DOC_ID_SEPARATOR_SPLITTER = "\\|"; - // These are not allowed in document type names: _ . , | # - public static final String DOC_TYPE_SEPARATOR = "^"; - - public static final String INDEX_NAME_SEPARATOR = "^"; + public static final String SPLITTER = "\\__"; - public static final String ENTITY_CONTEXT = "_context"; + // These are not allowed in document type names: _ . , | # + public static final String SEPARATOR = "__"; /** - * To be used when we want to search all types within a scope + * Reserved UG fields. */ - public static final String ALL_TYPES = "ALL"; + + public static final String ENTITY_CONTEXT_FIELDNAME = "ug_context"; + + public static final String ENTITYID_ID_FIELDNAME = "ug_entityId"; + /** @@ -62,13 +61,35 @@ public class IndexingUtils { */ public static String createContextName( IndexScope scope ) { StringBuilder sb = new StringBuilder(); - sb.append( scope.getOwner().getUuid() ).append(DOC_TYPE_SEPARATOR); - sb.append( scope.getOwner().getType() ).append(DOC_TYPE_SEPARATOR); + idString(sb, scope.getOwner()); + sb.append( SEPARATOR ); sb.append( scope.getName() ); return sb.toString(); } + /** + * Append the id to the string + * @param builder + * @param id + */ + public static final void idString(final StringBuilder builder, final Id id){ + builder.append( id.getUuid() ).append( SEPARATOR ) + .append(id.getType()); + } + + + /** + * Turn the id into a string + * @param id + * @return + */ + public static final String idString(final Id id){ + final StringBuilder sb = new StringBuilder( ); + idString(sb, id); + return sb.toString(); + } + /** * Create the index name based on our prefix+appUUID+AppType @@ -79,9 +100,8 @@ public class IndexingUtils { public static String createIndexName( String prefix, ApplicationScope applicationScope) { StringBuilder sb = new StringBuilder(); - sb.append( prefix ).append(INDEX_NAME_SEPARATOR); - sb.append( applicationScope.getApplication().getUuid() ).append(INDEX_NAME_SEPARATOR); - sb.append( applicationScope.getApplication().getType() ); + sb.append( prefix ).append( SEPARATOR ); + idString( sb, applicationScope.getApplication() ); return sb.toString(); } @@ -92,8 +112,8 @@ public class IndexingUtils { * @param entity * @return */ - public static String createIndexDocId(final Entity entity, final String scopeType) { - return createIndexDocId(entity.getId(), entity.getVersion(), scopeType); + public static String createIndexDocId(final Entity entity, final String context) { + return createIndexDocId(entity.getId(), entity.getVersion(), context); } @@ -101,14 +121,15 @@ public class IndexingUtils { * Create the doc Id. This is the entitie's type + uuid + version * @param entityId * @param version + * @para context The context it's indexed in * @return */ - public static String createIndexDocId(final Id entityId, final UUID version, final String scopeType) { + public static String createIndexDocId(final Id entityId, final UUID version, final String context) { StringBuilder sb = new StringBuilder(); - sb.append( entityId.getUuid() ).append(DOC_ID_SEPARATOR); - sb.append( entityId.getType() ).append(DOC_ID_SEPARATOR); - sb.append( version.toString() ).append( DOC_ID_SEPARATOR ); - sb.append( scopeType); + idString(sb, entityId); + sb.append( SEPARATOR ); + sb.append( version.toString() ).append( SEPARATOR ); + sb.append( context); return sb.toString(); } @@ -135,25 +156,41 @@ public class IndexingUtils { .startArray( "dynamic_templates" ) - // any string with field name that starts with sa_ gets analyzed + //we need most specific mappings first since it's a stop on match algorithm + .startObject() - .startObject( "template_1" ) - .field( "match", ANALYZED_STRING_PREFIX + "*" ) - .field( "match_mapping_type", "string" ) + .startObject( "context_template" ) + .field( "match", IndexingUtils.ENTITYID_ID_FIELDNAME ) + .field( "match_mapping_type", "string" ) + .startObject( "mapping" ).field( "type", "string" ) + .field( "index", "not_analyzed" ) + .endObject() + .endObject() + .endObject() + + + .startObject() + .startObject( "context_template" ) + .field( "match", IndexingUtils.ENTITY_CONTEXT_FIELDNAME ) + .field( "match_mapping_type", "string" ) .startObject( "mapping" ).field( "type", "string" ) - .field( "index", "analyzed" ) + .field( "index", "not_analyzed" ).endObject() .endObject() - .endObject() + .endObject() + + // any string with field name that starts with sa_ gets analyzed + .startObject() + .startObject( "template_1" ).field( "match", ANALYZED_STRING_PREFIX + "*" ) + .field( "match_mapping_type", "string" ).startObject( "mapping" ).field( "type", "string" ) + .field( "index", "analyzed" ).endObject().endObject() .endObject() - // all other strings are not analyzed + // all other strings are not analyzed .startObject() .startObject( "template_2" ) - //todo, should be string prefix, remove 2 field mapping - .field( "match", "*" ) - .field( "match_mapping_type", "string" ) - .startObject( "mapping" ) - .field( "type", "string" ) + //todo, should be string prefix, remove 2 field mapping + .field( "match", "*" ).field( "match_mapping_type", "string" ).startObject( "mapping" ) + .field( "type", "string" ) .field( "index", "not_analyzed" ) .endObject() .endObject() @@ -169,16 +206,7 @@ public class IndexingUtils { .endObject() .endObject() - //types for context direct string matching - .startObject( "context_template" ) - .field( "match", IndexingUtils.ENTITY_CONTEXT ) - .field( "match_mapping_type", "string" ) - .startObject( "mapping" ) - .field( "type", "string" ) - .field( "index", "not_analyzed" ) - .endObject() - .endObject() - .endObject() + .endArray() http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ef77a5a4/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java index 778134e..07bee44 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java @@ -43,6 +43,7 @@ import org.apache.commons.lang.StringUtils; import org.apache.usergrid.persistence.index.exceptions.IndexException; import org.apache.usergrid.persistence.index.exceptions.QueryParseException; import org.apache.usergrid.persistence.index.impl.EsQueryVistor; +import org.apache.usergrid.persistence.index.impl.IndexingUtils; import org.apache.usergrid.persistence.index.query.tree.AndOperand; import org.apache.usergrid.persistence.index.query.tree.ContainsOperand; import org.apache.usergrid.persistence.index.query.tree.CpQueryFilterLexer; @@ -142,10 +143,23 @@ public class Query { } - public QueryBuilder createQueryBuilder() { + public QueryBuilder createQueryBuilder(final String context) { - QueryBuilder queryBuilder = null; + QueryBuilder queryBuilder; + + + /** + * Add our filter for context to our query for fast execution. Fast because it utilizes bitsets + * internally. See this post for more detail. + * http://www.elasticsearch.org/blog/all-about-elasticsearch-filter-bitsets/ + */ + + + + + + //we have a root operand. Translate our AST into an ES search if ( getRootOperand() != null ) { QueryVisitor v = new EsQueryVistor(); try { @@ -154,11 +168,15 @@ public class Query { } catch ( IndexException ex ) { throw new RuntimeException( "Error building ElasticSearch query", ex ); } - queryBuilder = v.getQueryBuilder(); + + // TODO evaluate performance when it's an all query. Do we need to put the context term first for performance? + queryBuilder = QueryBuilders.boolQuery().must( v.getQueryBuilder() ) + .must( QueryBuilders.termQuery( IndexingUtils.ENTITY_CONTEXT_FIELDNAME, context ) ); } - if ( queryBuilder == null ) { - queryBuilder = QueryBuilders.matchAllQuery(); + //nothing was specified ensure we specify the context in the search + else { + queryBuilder = QueryBuilders.termQuery( IndexingUtils.ENTITY_CONTEXT_FIELDNAME, context ); } return queryBuilder; http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ef77a5a4/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java index c1c075e..d2423bb 100644 --- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java +++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java @@ -40,6 +40,7 @@ import org.apache.usergrid.persistence.index.EntityIndexFactory; import org.apache.usergrid.persistence.index.IndexScope; import org.apache.usergrid.persistence.index.SearchTypes; import org.apache.usergrid.persistence.index.guice.TestIndexModule; +import org.apache.usergrid.persistence.index.query.CandidateResult; import org.apache.usergrid.persistence.index.query.CandidateResults; import org.apache.usergrid.persistence.index.query.Query; import org.apache.usergrid.persistence.model.entity.Entity; @@ -51,6 +52,7 @@ import com.google.inject.Inject; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.fail; @RunWith( EsRunner.class ) @@ -59,15 +61,9 @@ public class EntityConnectionIndexImplTest extends BaseIT { private static final Logger log = LoggerFactory.getLogger( EntityConnectionIndexImplTest.class ); - @ClassRule - public static ElasticSearchResource es = new ElasticSearchResource(); +// @ClassRule +// public static ElasticSearchResource es = new ElasticSearchResource(); - @ClassRule - public static CassandraRule cass = new CassandraRule(); - - @Inject - @Rule - public MigrationManagerRule migrationManagerRule; @Inject public EntityIndexFactory ecif; @@ -121,7 +117,7 @@ public class EntityConnectionIndexImplTest extends BaseIT { IndexScope searchScope = new IndexScopeImpl( personId, "likes" ); //create another scope we index in, want to be sure these scopes are filtered - IndexScope otherIndexScope = new IndexScopeImpl( new SimpleId( UUIDGenerator.newTimeUUID(), "person" ), "likes" ); + IndexScope otherIndexScope = new IndexScopeImpl( new SimpleId( UUIDGenerator.newTimeUUID(), "animal" ), "likes" ); EntityIndex personLikesIndex = ecif.createEntityIndex( applicationScope ); @@ -142,6 +138,7 @@ public class EntityConnectionIndexImplTest extends BaseIT { batch.index( otherIndexScope, oj ); batch.executeAndRefresh(); + personLikesIndex.refresh(); // now, let's search for muffins CandidateResults likes = personLikesIndex @@ -167,18 +164,18 @@ public class EntityConnectionIndexImplTest extends BaseIT { SearchTypes.fromTypes( muffin.getId().getType(), egg.getId().getType(), oj.getId().getType() ), Query.fromQL( "select *" ) ); assertEquals( 3, likes.size() ); - assertEquals( muffin.getId(), likes.get( 0 ).getId() ); - assertEquals( egg.getId(), likes.get( 1 ).getId() ); - assertEquals( oj.getId(), likes.get( 2 ).getId() ); + assertContains( egg.getId(), likes ); + assertContains( muffin.getId(), likes ); + assertContains( oj.getId(), likes ); //now lets search for all explicitly likes = personLikesIndex.search( searchScope, SearchTypes.allTypes(), Query.fromQL( "select *" ) ); assertEquals( 3, likes.size() ); - assertEquals( muffin.getId(), likes.get( 0 ).getId() ); - assertEquals( egg.getId(), likes.get( 1 ).getId() ); - assertEquals( oj.getId(), likes.get( 2 ).getId() ); + assertContains( egg.getId(), likes ); + assertContains( muffin.getId(), likes); + assertContains( oj.getId(), likes ); @@ -187,16 +184,26 @@ public class EntityConnectionIndexImplTest extends BaseIT { SearchTypes.fromTypes( muffin.getId().getType(), egg.getId().getType(), oj.getId().getType() ), Query.fromQL( "select * where stars = 5" ) ); assertEquals( 2, likes.size() ); - assertEquals( muffin.getId(), likes.get( 0 ).getId() ); - assertEquals( egg.getId(), likes.get( 1 ).getId() ); + assertContains( egg.getId(), likes); + assertContains( muffin.getId(), likes ); //now search with no types, we should get only the results that match likes = personLikesIndex.search( searchScope, SearchTypes.allTypes(), Query.fromQL( "select * where stars = 5" ) ); assertEquals( 2, likes.size() ); - assertEquals( muffin.getId(), likes.get( 0 ).getId() ); - assertEquals( egg.getId(), likes.get( 1 ).getId() ); + assertContains( egg.getId(), likes); + assertContains( muffin.getId(), likes ); + } + + + private void assertContains(final Id id, final CandidateResults results ){ + for( CandidateResult result: results){ + if(result.getId().equals(id)){ + return; + } + } + fail(String.format("Could not find id %s in candidate results", id)); } } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ef77a5a4/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java index 1bbb301..a1240f1 100644 --- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java +++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java @@ -45,6 +45,7 @@ import org.apache.usergrid.persistence.index.EntityIndex; import org.apache.usergrid.persistence.index.EntityIndexBatch; import org.apache.usergrid.persistence.index.EntityIndexFactory; import org.apache.usergrid.persistence.index.IndexScope; +import org.apache.usergrid.persistence.index.SearchTypes; import org.apache.usergrid.persistence.index.guice.TestIndexModule; import org.apache.usergrid.persistence.index.query.CandidateResults; import org.apache.usergrid.persistence.index.query.Query; @@ -69,393 +70,403 @@ import static org.junit.Assert.assertNotEquals; @UseModules({ TestIndexModule.class }) public class EntityIndexTest extends BaseIT { -// private static final Logger log = LoggerFactory.getLogger( EntityIndexTest.class ); -// -// @ClassRule -// public static CassandraRule cass = new CassandraRule(); -// -// @Rule -// public ElasticSearchResource elasticSearchResource = new ElasticSearchResource(); -// -// @Inject -// @Rule -// public MigrationManagerRule migrationManagerRule; -// -// @Inject -// public EntityIndexFactory eif; -// -// -// -// @Test -// public void testIndex() throws IOException { -// -// final int MAX_ENTITIES = 100; -// -// Id appId = new SimpleId( "application" ); -// -// ApplicationScope applicationScope = new ApplicationScopeImpl( appId ); -// -// IndexScope indexScope = new IndexScopeImpl( appId, "things" ); -// -// -// EntityIndex entityIndex = eif.createEntityIndex( applicationScope ); -// entityIndex.initializeIndex(); -// -// InputStream is = this.getClass().getResourceAsStream( "/sample-large.json" ); -// ObjectMapper mapper = new ObjectMapper(); -// List<Object> sampleJson = mapper.readValue( is, new TypeReference<List<Object>>() {} ); -// -// int count = 0; -// StopWatch timer = new StopWatch(); -// timer.start(); -// -// final EntityIndexBatch batch = entityIndex.createBatch(); -// -// for ( Object o : sampleJson ) { -// -// Map<String, Object> item = ( Map<String, Object> ) o; -// -// Entity entity = new Entity( indexScope.getName() ); -// entity = EntityIndexMapUtils.fromMap( entity, item ); -// EntityUtils.setVersion( entity, UUIDGenerator.newTimeUUID() ); -// -// batch.index( indexScope, entity ); -// -// if(count %1000 == 0){ -// batch.execute(); -// } -// -// -// -// if ( count++ > MAX_ENTITIES ) { -// break; -// } -// } -// -// batch.execute(); -// timer.stop(); -// log.info( "Total time to index {} entries {}ms, average {}ms/entry", -// new Object[] { count, timer.getTime(), timer.getTime() / count } ); -// -// entityIndex.refresh(); -// -// -// testQueries( indexScope, entityIndex ); -// } -// -// -// @Test -// public void testDeindex() { -// -// Id appId = new SimpleId( "application" ); -// -// ApplicationScope applicationScope = new ApplicationScopeImpl( appId ); -// -// IndexScope indexScope = new IndexScopeImpl( appId, "fastcars", entityType ); -// -// EntityIndex entityIndex = eif.createEntityIndex( applicationScope ); -// entityIndex.initializeIndex(); -// -// Map entityMap = new HashMap() {{ -// put( "name", "Ferrari 212 Inter" ); -// put( "introduced", 1952 ); -// put( "topspeed", 215 ); -// }}; -// -// -// Entity entity = EntityIndexMapUtils.fromMap( entityMap ); -// EntityUtils.setId( entity, new SimpleId( "fastcar" ) ); -// EntityUtils.setVersion( entity, UUIDGenerator.newTimeUUID() ); -// entityIndex.createBatch().index(indexScope , entity ).executeAndRefresh(); -// -// CandidateResults candidateResults = entityIndex.search(indexScope, Query.fromQL( "name contains 'Ferrari*'" ) ); -// assertEquals( 1, candidateResults.size() ); -// -// entityIndex.createBatch().deindex( indexScope, entity ).execute(); -// -// entityIndex.refresh(); -// -// candidateResults = entityIndex.search( indexScope, Query.fromQL( "name contains 'Ferrari*'" ) ); -// assertEquals( 0, candidateResults.size() ); -// } -// -// -// private void testQuery(final IndexScope scope, final EntityIndex entityIndex, final String queryString, final int num ) { -// -// StopWatch timer = new StopWatch(); -// timer.start(); -// Query query = Query.fromQL( queryString ); -// query.setLimit( 1000 ); -// CandidateResults candidateResults = entityIndex.search( scope, query ); -// timer.stop(); -// -// assertEquals( num, candidateResults.size() ); -// log.debug( "Query time {}ms", timer.getTime() ); -// } -// -// -// private void testQueries(final IndexScope scope, final EntityIndex entityIndex ) { -// -// -// testQuery(scope, entityIndex, "name = 'Morgan Pierce'", 1 ); -// -// testQuery(scope, entityIndex, "name = 'morgan pierce'", 1 ); -// -// testQuery(scope, entityIndex, "name = 'Morgan'", 0 ); -// -// testQuery(scope, entityIndex, "name contains 'Morgan'", 1 ); -// -// testQuery(scope, entityIndex, "company > 'GeoLogix'", 64 ); -// -// testQuery(scope, entityIndex, "gender = 'female'", 45 ); -// -// testQuery(scope, entityIndex, "name = 'Minerva Harrell' and age > 39", 1 ); -// -// testQuery(scope, entityIndex, "name = 'Minerva Harrell' and age > 39 and age < 41", 1 ); -// -// testQuery(scope, entityIndex, "name = 'Minerva Harrell' and age > 40", 0 ); -// -// testQuery(scope, entityIndex, "name = 'Minerva Harrell' and age >= 40", 1 ); -// -// testQuery(scope, entityIndex, "name = 'Minerva Harrell' and age <= 40", 1 ); -// -// testQuery(scope, entityIndex, "name = 'Morgan* '", 1 ); -// -// testQuery(scope, entityIndex, "name = 'Morgan*'", 1 ); -// -// -// // test a couple of array sub-property queries -// -// int totalUsers = 102; -// -// // nobody has a friend named Jack the Ripper -// testQuery(scope, entityIndex, "friends.name = 'Jack the Ripper'", 0 ); -// -// // everybody doesn't have a friend named Jack the Ripper -// testQuery(scope, entityIndex, "not (friends.name = 'Jack the Ripper')", totalUsers ); -// -// // one person has a friend named Shari Hahn -// testQuery(scope, entityIndex, "friends.name = 'Wendy Moody'", 1 ); -// -// // everybody but 1 doesn't have a friend named Shari Hahh -// testQuery(scope, entityIndex, "not (friends.name = 'Shari Hahn')", totalUsers - 1); -// -// } -// -// -// /** -// * Tests that Entity-to-map and Map-to-entity round trip works. -// */ -// @Test -// public void testEntityIndexMapUtils() throws IOException { -// -// InputStream is = this.getClass().getResourceAsStream( "/sample-small.json" ); -// ObjectMapper mapper = new ObjectMapper(); -// List<Object> contacts = mapper.readValue( is, new TypeReference<List<Object>>() {} ); -// -// for ( Object o : contacts ) { -// -// Map<String, Object> map1 = ( Map<String, Object> ) o; -// -// // convert map to entity -// Entity entity1 = EntityIndexMapUtils.fromMap( map1 ); -// -// // convert entity back to map -// Map map2 = EntityIndexMapUtils.toMap( entity1 ); -// -// // the two maps should be the same -// Map diff = Maps.difference( map1, map2 ).entriesDiffering(); -// assertEquals( 0, diff.size() ); -// } -// } -// -// -// @Test -// public void getEntityVersions() throws Exception { -// -// Id appId = new SimpleId( "application" ); -// Id ownerId = new SimpleId( "owner" ); -// -// ApplicationScope applicationScope = new ApplicationScopeImpl( appId ); -// -// IndexScope indexScope = new IndexScopeImpl( ownerId, "user", entityType ); -// -// -// -// EntityIndex entityIndex = eif.createEntityIndex( applicationScope ); -// entityIndex.initializeIndex(); -// -// final String middleName = "middleName" + UUIDUtils.newTimeUUID(); -// Map<String, Object> properties = new LinkedHashMap<String, Object>(); -// properties.put( "username", "edanuff" ); -// properties.put( "email", "[email protected]" ); -// properties.put( "middlename", middleName ); -// -// Map entityMap = new HashMap() {{ -// put( "username", "edanuff" ); -// put( "email", "[email protected]" ); -// put( "middlename", middleName ); -// }}; -// -// Entity user = EntityIndexMapUtils.fromMap( entityMap ); -// EntityUtils.setId( user, new SimpleId( "edanuff" ) ); -// EntityUtils.setVersion( user, UUIDGenerator.newTimeUUID() ); -// -// -// final EntityIndexBatch batch = entityIndex.createBatch(); -// -// batch.index( indexScope, user ); -// -// user.setField( new StringField( "address1", "1782 address st" ) ); -// batch.index( indexScope, user ); -// user.setField( new StringField( "address2", "apt 508" ) ); -// batch.index( indexScope, user ); -// user.setField( new StringField( "address3", "apt 508" ) ); -// batch.index( indexScope, user ); -// batch.executeAndRefresh(); -// -// CandidateResults results = entityIndex.getEntityVersions(indexScope, user.getId() ); -// -// assertEquals(1, results.size()); -// assertEquals( results.get( 0 ).getId(), user.getId() ); -// assertEquals( results.get(0).getVersion(), user.getVersion()); -// } -// -// -// @Test -// public void deleteVerification() throws Throwable { -// -// Id appId = new SimpleId( "application" ); -// Id ownerId = new SimpleId( "owner" ); -// -// ApplicationScope applicationScope = new ApplicationScopeImpl( appId ); -// -// IndexScope appScope = new IndexScopeImpl( ownerId, "user", entityType ); -// -// EntityIndex ei = eif.createEntityIndex( applicationScope ); -// ei.initializeIndex(); -// -// final String middleName = "middleName" + UUIDUtils.newTimeUUID(); -// -// Map entityMap = new HashMap() {{ -// put( "username", "edanuff" ); -// put( "email", "[email protected]" ); -// put( "middlename", middleName ); -// }}; -// -// Entity user = EntityIndexMapUtils.fromMap( entityMap ); -// EntityUtils.setId( user, new SimpleId( "edanuff" ) ); -// EntityUtils.setVersion( user, UUIDGenerator.newTimeUUID() ); -// -// -// EntityIndexBatch batch = ei.createBatch(); -// -// batch.index( appScope, user ).executeAndRefresh(); -// Query query = new Query(); -// query.addEqualityFilter( "username", "edanuff" ); -// CandidateResults r = ei.search( appScope, query ); -// assertEquals( user.getId(), r.get( 0 ).getId() ); -// -// batch.deindex(appScope, user.getId(), user.getVersion() ).executeAndRefresh(); -// -// -// // EntityRef -// query = new Query(); -// query.addEqualityFilter( "username", "edanuff" ); -// r = ei.search(appScope, query ); -// -// assertFalse( r.iterator().hasNext() ); -// } -// -// @Test -// public void multiValuedTypes() { -// -// Id appId = new SimpleId( "entityindextest" ); -// Id ownerId = new SimpleId( "multivaluedtype" ); -// ApplicationScope applicationScope = new ApplicationScopeImpl( appId ); -// -// IndexScope appScope = new IndexScopeImpl( ownerId, "user", entityType ); -// -// EntityIndex ei = eif.createEntityIndex( applicationScope ); -// ei.createBatch(); -// -// // Bill has favorites as string, age as string and retirement goal as number -// Map billMap = new HashMap() {{ -// put( "username", "bill" ); -// put( "email", "[email protected]" ); -// put( "age", "thirtysomething"); -// put( "favorites", "scallops, croquet, wine"); -// put( "retirementGoal", 100000); -// }}; -// Entity bill = EntityIndexMapUtils.fromMap( billMap ); -// EntityUtils.setId( bill, new SimpleId( UUIDGenerator.newTimeUUID(), "user" ) ); -// EntityUtils.setVersion( bill, UUIDGenerator.newTimeUUID() ); -// -// EntityIndexBatch batch = ei.createBatch(); -// -// batch.index( appScope, bill ); -// -// // Fred has age as int, favorites as object and retirement goal as object -// Map fredMap = new HashMap() {{ -// put( "username", "fred" ); -// put( "email", "[email protected]" ); -// put( "age", 41 ); -// put( "favorites", new HashMap<String, Object>() {{ -// put("food", "cheezewiz"); -// put("sport", "nascar"); -// put("beer", "budwizer"); -// }}); -// put( "retirementGoal", new HashMap<String, Object>() {{ -// put("car", "Firebird"); -// put("home", "Mobile"); -// }}); -// }}; -// Entity fred = EntityIndexMapUtils.fromMap( fredMap ); -// EntityUtils.setId( fred, new SimpleId( UUIDGenerator.newTimeUUID(), "user" ) ); -// EntityUtils.setVersion( fred, UUIDGenerator.newTimeUUID() ); -// batch.index( appScope, fred ); -// -// batch.executeAndRefresh(); -// -// Query query = new Query(); -// query.addEqualityFilter( "username", "bill" ); -// CandidateResults r = ei.search( appScope, query ); -// assertEquals( bill.getId(), r.get( 0 ).getId() ); -// -// query = new Query(); -// query.addEqualityFilter( "username", "fred" ); -// r = ei.search( appScope, query ); -// assertEquals( fred.getId(), r.get( 0 ).getId() ); -// -// query = new Query(); -// query.addEqualityFilter( "age", 41 ); -// r = ei.search( appScope, query ); -// assertEquals( fred.getId(), r.get( 0 ).getId() ); -// -// query = new Query(); -// query.addEqualityFilter( "age", "thirtysomething" ); -// r = ei.search( appScope, query ); -// assertEquals( bill.getId(), r.get( 0 ).getId() ); -// } -// -// -// @Test -// public void healthTest() { -// -// Id appId = new SimpleId( "application" ); -// ApplicationScope applicationScope = new ApplicationScopeImpl( appId ); -// -// EntityIndex ei = eif.createEntityIndex( applicationScope ); -// -// assertNotEquals( "cluster should be ok", Health.RED, ei.getClusterHealth() ); -// assertEquals( "index not be ready yet", Health.RED, ei.getIndexHealth() ); -// -// ei.initializeIndex(); -// ei.refresh(); -// -// assertNotEquals( "cluster should be fine", Health.RED, ei.getIndexHealth() ); -// assertNotEquals( "cluster should be ready now", Health.RED, ei.getClusterHealth() ); -// } + private static final Logger log = LoggerFactory.getLogger( EntityIndexTest.class ); + + @ClassRule + public static CassandraRule cass = new CassandraRule(); + + @Rule + public ElasticSearchResource elasticSearchResource = new ElasticSearchResource(); + + @Inject + @Rule + public MigrationManagerRule migrationManagerRule; + + @Inject + public EntityIndexFactory eif; + + + + @Test + public void testIndex() throws IOException { + + final int MAX_ENTITIES = 100; + + Id appId = new SimpleId( "application" ); + + ApplicationScope applicationScope = new ApplicationScopeImpl( appId ); + + IndexScope indexScope = new IndexScopeImpl( appId, "things" ); + + final String entityType = "thing"; + + + final SearchTypes searchTypes = SearchTypes.fromTypes( entityType ); + + + EntityIndex entityIndex = eif.createEntityIndex( applicationScope ); + entityIndex.initializeIndex(); + + InputStream is = this.getClass().getResourceAsStream( "/sample-large.json" ); + ObjectMapper mapper = new ObjectMapper(); + List<Object> sampleJson = mapper.readValue( is, new TypeReference<List<Object>>() {} ); + + int count = 0; + StopWatch timer = new StopWatch(); + timer.start(); + + final EntityIndexBatch batch = entityIndex.createBatch(); + + for ( Object o : sampleJson ) { + + Map<String, Object> item = ( Map<String, Object> ) o; + + Entity entity = new Entity( entityType ); + entity = EntityIndexMapUtils.fromMap( entity, item ); + EntityUtils.setVersion( entity, UUIDGenerator.newTimeUUID() ); + + batch.index( indexScope, entity ); + + if(count %1000 == 0){ + batch.execute(); + } + + + + if ( count++ > MAX_ENTITIES ) { + break; + } + } + + batch.execute(); + timer.stop(); + log.info( "Total time to index {} entries {}ms, average {}ms/entry", + new Object[] { count, timer.getTime(), timer.getTime() / count } ); + + entityIndex.refresh(); + + + testQueries( indexScope, searchTypes, entityIndex ); + } + + + @Test + public void testDeindex() { + + Id appId = new SimpleId( "application" ); + + ApplicationScope applicationScope = new ApplicationScopeImpl( appId ); + + IndexScope indexScope = new IndexScopeImpl( appId, "fastcars" ); + + EntityIndex entityIndex = eif.createEntityIndex( applicationScope ); + entityIndex.initializeIndex(); + + Map entityMap = new HashMap() {{ + put( "name", "Ferrari 212 Inter" ); + put( "introduced", 1952 ); + put( "topspeed", 215 ); + }}; + + + Entity entity = EntityIndexMapUtils.fromMap( entityMap ); + EntityUtils.setId( entity, new SimpleId( "fastcar" ) ); + EntityUtils.setVersion( entity, UUIDGenerator.newTimeUUID() ); + entityIndex.createBatch().index(indexScope , entity ).executeAndRefresh(); + + CandidateResults candidateResults = entityIndex.search( indexScope, SearchTypes.fromTypes(entity.getId().getType()), + Query.fromQL( "name contains 'Ferrari*'" ) ); + assertEquals( 1, candidateResults.size() ); + + entityIndex.createBatch().deindex( indexScope, entity ).execute(); + + entityIndex.refresh(); + + candidateResults = entityIndex.search( indexScope, SearchTypes.fromTypes(entity.getId().getType()), Query.fromQL( "name contains 'Ferrari*'" ) ); + assertEquals( 0, candidateResults.size() ); + } + + + private void testQuery(final IndexScope scope, final SearchTypes searchTypes, final EntityIndex entityIndex, final String queryString, final int num ) { + + StopWatch timer = new StopWatch(); + timer.start(); + Query query = Query.fromQL( queryString ); + query.setLimit( 1000 ); + CandidateResults candidateResults = entityIndex.search( scope, searchTypes, query ); + timer.stop(); + + assertEquals( num, candidateResults.size() ); + log.debug( "Query time {}ms", timer.getTime() ); + } + + + private void testQueries(final IndexScope scope, SearchTypes searchTypes, final EntityIndex entityIndex ) { + + + testQuery(scope, searchTypes, entityIndex, "name = 'Morgan Pierce'", 1 ); + + testQuery(scope, searchTypes, entityIndex, "name = 'morgan pierce'", 1 ); + + testQuery(scope, searchTypes, entityIndex, "name = 'Morgan'", 0 ); + + testQuery(scope, searchTypes, entityIndex, "name contains 'Morgan'", 1 ); + + testQuery(scope, searchTypes, entityIndex, "company > 'GeoLogix'", 64 ); + + testQuery(scope, searchTypes, entityIndex, "gender = 'female'", 45 ); + + testQuery(scope, searchTypes, entityIndex, "name = 'Minerva Harrell' and age > 39", 1 ); + + testQuery(scope, searchTypes, entityIndex, "name = 'Minerva Harrell' and age > 39 and age < 41", 1 ); + + testQuery(scope, searchTypes, entityIndex, "name = 'Minerva Harrell' and age > 40", 0 ); + + testQuery(scope, searchTypes, entityIndex, "name = 'Minerva Harrell' and age >= 40", 1 ); + + testQuery(scope, searchTypes, entityIndex, "name = 'Minerva Harrell' and age <= 40", 1 ); + + testQuery(scope, searchTypes, entityIndex, "name = 'Morgan* '", 1 ); + + testQuery(scope, searchTypes, entityIndex, "name = 'Morgan*'", 1 ); + + + // test a couple of array sub-property queries + + int totalUsers = 102; + + // nobody has a friend named Jack the Ripper + testQuery(scope, searchTypes, entityIndex, "friends.name = 'Jack the Ripper'", 0 ); + + // everybody doesn't have a friend named Jack the Ripper + testQuery(scope, searchTypes,entityIndex, "not (friends.name = 'Jack the Ripper')", totalUsers ); + + // one person has a friend named Shari Hahn + testQuery(scope, searchTypes, entityIndex, "friends.name = 'Wendy Moody'", 1 ); + + // everybody but 1 doesn't have a friend named Shari Hahh + testQuery(scope, searchTypes, entityIndex, "not (friends.name = 'Shari Hahn')", totalUsers - 1); + + } + + + /** + * Tests that Entity-to-map and Map-to-entity round trip works. + */ + @Test + public void testEntityIndexMapUtils() throws IOException { + + InputStream is = this.getClass().getResourceAsStream( "/sample-small.json" ); + ObjectMapper mapper = new ObjectMapper(); + List<Object> contacts = mapper.readValue( is, new TypeReference<List<Object>>() {} ); + + for ( Object o : contacts ) { + + Map<String, Object> map1 = ( Map<String, Object> ) o; + + // convert map to entity + Entity entity1 = EntityIndexMapUtils.fromMap( map1 ); + + // convert entity back to map + Map map2 = EntityIndexMapUtils.toMap( entity1 ); + + // the two maps should be the same + Map diff = Maps.difference( map1, map2 ).entriesDiffering(); + assertEquals( 0, diff.size() ); + } + } + + + @Test + public void getEntityVersions() throws Exception { + + Id appId = new SimpleId( "application" ); + Id ownerId = new SimpleId( "owner" ); + + ApplicationScope applicationScope = new ApplicationScopeImpl( appId ); + + IndexScope indexScope = new IndexScopeImpl( ownerId, "users" ); + + + + EntityIndex entityIndex = eif.createEntityIndex( applicationScope ); + entityIndex.initializeIndex(); + + final String middleName = "middleName" + UUIDUtils.newTimeUUID(); + Map<String, Object> properties = new LinkedHashMap<String, Object>(); + properties.put( "username", "edanuff" ); + properties.put( "email", "[email protected]" ); + properties.put( "middlename", middleName ); + + Map entityMap = new HashMap() {{ + put( "username", "edanuff" ); + put( "email", "[email protected]" ); + put( "middlename", middleName ); + }}; + + final Id userId = new SimpleId("user"); + + Entity user = EntityIndexMapUtils.fromMap( entityMap ); + EntityUtils.setId( user, userId); + EntityUtils.setVersion( user, UUIDGenerator.newTimeUUID() ); + + + final EntityIndexBatch batch = entityIndex.createBatch(); + + batch.index( indexScope, user ); + + user.setField( new StringField( "address1", "1782 address st" ) ); + batch.index( indexScope, user ); + user.setField( new StringField( "address2", "apt 508" ) ); + batch.index( indexScope, user ); + user.setField( new StringField( "address3", "apt 508" ) ); + batch.index( indexScope, user ); + batch.executeAndRefresh(); + + CandidateResults results = entityIndex.getEntityVersions(indexScope, user.getId() ); + + assertEquals(1, results.size()); + assertEquals( results.get( 0 ).getId(), user.getId() ); + assertEquals( results.get(0).getVersion(), user.getVersion()); + } + + + @Test + public void deleteVerification() throws Throwable { + + Id appId = new SimpleId( "application" ); + Id ownerId = new SimpleId( "owner" ); + + ApplicationScope applicationScope = new ApplicationScopeImpl( appId ); + + IndexScope appScope = new IndexScopeImpl( ownerId, "user" ); + + EntityIndex ei = eif.createEntityIndex( applicationScope ); + ei.initializeIndex(); + + final String middleName = "middleName" + UUIDUtils.newTimeUUID(); + + Map entityMap = new HashMap() {{ + put( "username", "edanuff" ); + put( "email", "[email protected]" ); + put( "middlename", middleName ); + }}; + + Entity user = EntityIndexMapUtils.fromMap( entityMap ); + EntityUtils.setId( user, new SimpleId( "edanuff" ) ); + EntityUtils.setVersion( user, UUIDGenerator.newTimeUUID() ); + + + EntityIndexBatch batch = ei.createBatch(); + + batch.index( appScope, user ).executeAndRefresh(); + Query query = new Query(); + query.addEqualityFilter( "username", "edanuff" ); + CandidateResults r = ei.search( appScope, SearchTypes.fromTypes( "edanuff" ), query ); + assertEquals( user.getId(), r.get( 0 ).getId() ); + + batch.deindex(appScope, user.getId(), user.getVersion() ).executeAndRefresh(); + + + // EntityRef + query = new Query(); + query.addEqualityFilter( "username", "edanuff" ); + r = ei.search(appScope,SearchTypes.fromTypes( "edanuff" ), query ); + + assertFalse( r.iterator().hasNext() ); + } + + @Test + public void multiValuedTypes() { + + Id appId = new SimpleId( "entityindextest" ); + Id ownerId = new SimpleId( "multivaluedtype" ); + ApplicationScope applicationScope = new ApplicationScopeImpl( appId ); + + IndexScope appScope = new IndexScopeImpl( ownerId, "user" ); + + EntityIndex ei = eif.createEntityIndex( applicationScope ); + ei.createBatch(); + + // Bill has favorites as string, age as string and retirement goal as number + Map billMap = new HashMap() {{ + put( "username", "bill" ); + put( "email", "[email protected]" ); + put( "age", "thirtysomething"); + put( "favorites", "scallops, croquet, wine"); + put( "retirementGoal", 100000); + }}; + Entity bill = EntityIndexMapUtils.fromMap( billMap ); + EntityUtils.setId( bill, new SimpleId( UUIDGenerator.newTimeUUID(), "user" ) ); + EntityUtils.setVersion( bill, UUIDGenerator.newTimeUUID() ); + + EntityIndexBatch batch = ei.createBatch(); + + batch.index( appScope, bill ); + + // Fred has age as int, favorites as object and retirement goal as object + Map fredMap = new HashMap() {{ + put( "username", "fred" ); + put( "email", "[email protected]" ); + put( "age", 41 ); + put( "favorites", new HashMap<String, Object>() {{ + put("food", "cheezewiz"); + put("sport", "nascar"); + put("beer", "budwizer"); + }}); + put( "retirementGoal", new HashMap<String, Object>() {{ + put("car", "Firebird"); + put("home", "Mobile"); + }}); + }}; + Entity fred = EntityIndexMapUtils.fromMap( fredMap ); + EntityUtils.setId( fred, new SimpleId( UUIDGenerator.newTimeUUID(), "user" ) ); + EntityUtils.setVersion( fred, UUIDGenerator.newTimeUUID() ); + batch.index( appScope, fred ); + + batch.executeAndRefresh(); + + final SearchTypes searchTypes = SearchTypes.fromTypes( "user" ); + + Query query = new Query(); + query.addEqualityFilter( "username", "bill" ); + CandidateResults r = ei.search( appScope, searchTypes, query ); + assertEquals( bill.getId(), r.get( 0 ).getId() ); + + query = new Query(); + query.addEqualityFilter( "username", "fred" ); + r = ei.search( appScope, searchTypes, query ); + assertEquals( fred.getId(), r.get( 0 ).getId() ); + + query = new Query(); + query.addEqualityFilter( "age", 41 ); + r = ei.search( appScope, searchTypes, query ); + assertEquals( fred.getId(), r.get( 0 ).getId() ); + + query = new Query(); + query.addEqualityFilter( "age", "thirtysomething" ); + r = ei.search( appScope, searchTypes, query ); + assertEquals( bill.getId(), r.get( 0 ).getId() ); + } + + + @Test + public void healthTest() { + + Id appId = new SimpleId( "application" ); + ApplicationScope applicationScope = new ApplicationScopeImpl( appId ); + + EntityIndex ei = eif.createEntityIndex( applicationScope ); + + assertNotEquals( "cluster should be ok", Health.RED, ei.getClusterHealth() ); + assertEquals( "index not be ready yet", Health.RED, ei.getIndexHealth() ); + + ei.initializeIndex(); + ei.refresh(); + + assertNotEquals( "cluster should be fine", Health.RED, ei.getIndexHealth() ); + assertNotEquals( "cluster should be ready now", Health.RED, ei.getClusterHealth() ); + } } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ef77a5a4/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EsRunner.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EsRunner.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EsRunner.java index c972851..2521a38 100644 --- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EsRunner.java +++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EsRunner.java @@ -36,7 +36,7 @@ public class EsRunner extends ITRunner { ElasticSearchResource rule = new ElasticSearchResource(); try { - rule.before(); +// rule.before(); } catch ( Throwable throwable ) { throw new RuntimeException( "Unable to start ES", throwable ); http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ef77a5a4/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java index d0e2d11..34185b3 100644 --- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java +++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java @@ -429,7 +429,7 @@ public class GrammarTreeTest { assertEquals( 37f, withinOperand.getLatitude().getFloatValue(), 0 ); assertEquals( -75f, withinOperand.getLongitude().getFloatValue(), 0 ); - QueryBuilder qb = query.createQueryBuilder(); + QueryBuilder qb = query.createQueryBuilder("testcontext"); }
