move back to old Query object
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/96d27b95 Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/96d27b95 Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/96d27b95 Branch: refs/heads/two-dot-o-dev Commit: 96d27b95336b41aee16180fe80fbf9bc9fb21f4d Parents: c09e6b4 Author: Shawn Feldman <[email protected]> Authored: Thu Mar 26 13:34:01 2015 -0600 Committer: Shawn Feldman <[email protected]> Committed: Thu Mar 26 13:34:01 2015 -0600 ---------------------------------------------------------------------- .../index/ApplicationEntityIndex.java | 6 +- .../usergrid/persistence/index/EntityIndex.java | 2 + .../impl/EsApplicationEntityIndexImpl.java | 9 +- .../impl/SearchRequestBuilderStrategy.java | 11 +- .../usergrid/persistence/index/query/Query.java | 184 ++++++++++++++++--- 5 files changed, 174 insertions(+), 38 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/96d27b95/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java index 2167efa..59a19eb 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java @@ -43,11 +43,9 @@ public interface ApplicationEntityIndex { * Execute query in Usergrid syntax. */ public CandidateResults search(final IndexScope indexScope, final SearchTypes searchTypes, final Query query); + public CandidateResults search(final IndexScope indexScope, final SearchTypes searchTypes, final Query query, final int limit); + - /** - * Execute query in Usergrid syntax. - */ - public CandidateResults search(final IndexScope indexScope, final SearchTypes searchType, final Query query, final int limit ); /** * get next page of results http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/96d27b95/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java index af6b013..6783fb0 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java @@ -39,6 +39,8 @@ import java.util.concurrent.Future; public interface EntityIndex extends CPManager { + public static final int MAX_LIMIT = 1000; + /** * Create an index and add to alias, will create alias and remove any old index from write alias if alias already exists * @param indexSuffix index name http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/96d27b95/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java index 27791ae..603bee4 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java @@ -136,16 +136,15 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex{ } @Override - public CandidateResults search(final IndexScope indexScope, final SearchTypes searchTypes, final Query query){ - return search(indexScope,searchTypes,query,10); + public CandidateResults search(final IndexScope indexScope, final SearchTypes searchTypes, final Query query) { + return search(indexScope,searchTypes,query,query.getLimit()); } - @Override - public CandidateResults search(final IndexScope indexScope, final SearchTypes searchTypes, final Query query, final int limit) { + public CandidateResults search(final IndexScope indexScope, final SearchTypes searchTypes, final Query query, final int limit){ SearchResponse searchResponse; - SearchRequestBuilder srb = searchRequest.getBuilder(indexScope, searchTypes, query, limit); + SearchRequestBuilder srb = searchRequest.getBuilder(indexScope, searchTypes, query,limit); if (logger.isDebugEnabled()) { logger.debug("Searching index (read alias): {}\n scope: {} \n type: {}\n query: {} ", http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/96d27b95/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java index 15796f9..207a7a8 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java @@ -22,10 +22,7 @@ package org.apache.usergrid.persistence.index.impl; import com.google.common.base.Preconditions; import org.apache.usergrid.persistence.core.scope.ApplicationScope; import org.apache.usergrid.persistence.core.util.ValidationUtils; -import org.apache.usergrid.persistence.index.IndexAlias; -import org.apache.usergrid.persistence.index.IndexIdentifier; -import org.apache.usergrid.persistence.index.IndexScope; -import org.apache.usergrid.persistence.index.SearchTypes; +import org.apache.usergrid.persistence.index.*; import org.apache.usergrid.persistence.index.exceptions.IndexException; import org.apache.usergrid.persistence.index.query.Query; import org.apache.usergrid.persistence.index.query.tree.QueryVisitor; @@ -56,7 +53,6 @@ public class SearchRequestBuilderStrategy { private final ApplicationScope applicationScope; private final IndexAlias alias; private final int cursorTimeout; - public static final int MAX_LIMIT = 1000; public SearchRequestBuilderStrategy(final EsProvider esProvider, final ApplicationScope applicationScope, final IndexAlias alias, int cursorTimeout){ @@ -66,8 +62,9 @@ public class SearchRequestBuilderStrategy { this.cursorTimeout = cursorTimeout; } - public SearchRequestBuilder getBuilder(final IndexScope indexScope, final SearchTypes searchTypes, final Query query, final int limit) { - Preconditions.checkArgument(limit <= MAX_LIMIT, "limit is greater than max "+ MAX_LIMIT); + public SearchRequestBuilder getBuilder(final IndexScope indexScope, final SearchTypes searchTypes, final Query query, final int limit) { + + Preconditions.checkArgument(limit <= EntityIndex.MAX_LIMIT, "limit is greater than max "+ EntityIndex.MAX_LIMIT); SearchRequestBuilder srb = esProvider.getClient().prepareSearch(alias.getReadAlias()) .setTypes(searchTypes.getTypeNames(applicationScope)) http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/96d27b95/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java index fa7def4..5567382 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java @@ -40,7 +40,10 @@ import org.antlr.runtime.Token; import org.antlr.runtime.TokenRewriteStream; import org.apache.commons.codec.binary.Base64; import org.apache.commons.lang.StringUtils; +import org.apache.usergrid.persistence.index.exceptions.IndexException; import org.apache.usergrid.persistence.index.exceptions.QueryParseException; +import org.apache.usergrid.persistence.index.impl.EsQueryVistor; +import org.apache.usergrid.persistence.index.impl.IndexingUtils; import org.apache.usergrid.persistence.index.query.tree.AndOperand; import org.apache.usergrid.persistence.index.query.tree.ContainsOperand; import org.apache.usergrid.persistence.index.query.tree.CpQueryFilterLexer; @@ -52,10 +55,14 @@ import org.apache.usergrid.persistence.index.query.tree.GreaterThanEqual; import org.apache.usergrid.persistence.index.query.tree.LessThan; import org.apache.usergrid.persistence.index.query.tree.LessThanEqual; import org.apache.usergrid.persistence.index.query.tree.Operand; +import org.apache.usergrid.persistence.index.query.tree.QueryVisitor; import org.apache.usergrid.persistence.index.utils.ClassUtils; import org.apache.usergrid.persistence.index.utils.ConversionUtils; import org.apache.usergrid.persistence.index.utils.ListUtils; import org.apache.usergrid.persistence.index.utils.MapUtils; +import org.elasticsearch.index.query.FilterBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -72,7 +79,9 @@ public class Query { IDS, REFS, CORE_PROPERTIES, ALL_PROPERTIES, LINKED_PROPERTIES } + public static final int DEFAULT_LIMIT = 10; + public static final int MAX_LIMIT = 1000; public static final String PROPERTY_UUID = "uuid"; @@ -80,6 +89,8 @@ public class Query { private List<SortPredicate> sortPredicates = new ArrayList<SortPredicate>(); private Operand rootOperand; private UUID startResult; + private String cursor; + private int limit = 0; private Map<String, String> selectAssignments = new LinkedHashMap<String, String>(); private boolean mergeSelectResults = false; @@ -111,19 +122,22 @@ public class Query { * @param q */ public Query( Query q ) { - if (q == null) { + if ( q == null ) { return; } + type = q.type; sortPredicates = q.sortPredicates != null - ? new ArrayList<SortPredicate>(q.sortPredicates) : null; + ? new ArrayList<>( q.sortPredicates ) : null; startResult = q.startResult; + cursor = q.cursor; + limit = q.limit; selectAssignments = q.selectAssignments != null - ? new LinkedHashMap<String, String>(q.selectAssignments) : null; + ? new LinkedHashMap<>( q.selectAssignments ) : null; mergeSelectResults = q.mergeSelectResults; //level = q.level; connectionType = q.connectionType; - permissions = q.permissions != null ? new ArrayList<String>(q.permissions) : null; + permissions = q.permissions != null ? new ArrayList<>( q.permissions ) : null; reversed = q.reversed; reversedSet = q.reversedSet; startTime = q.startTime; @@ -132,16 +146,76 @@ public class Query { pad = q.pad; rootOperand = q.rootOperand; identifiers = q.identifiers != null - ? new ArrayList<Identifier>(q.identifiers) : null; + ? new ArrayList<>( q.identifiers ) : null; counterFilters = q.counterFilters != null - ? new ArrayList<CounterFilterPredicate>(q.counterFilters) : null; + ? new ArrayList<>( q.counterFilters ) : null; collection = q.collection; - level = q.level; } + public QueryBuilder createQueryBuilder( final String context ) { + + + QueryBuilder queryBuilder = null; + + + //we have a root operand. Translate our AST into an ES search + if ( getRootOperand() != null ) { + // In the case of geo only queries, this will return null into the query builder. + // Once we start using tiles, we won't need this check any longer, since a geo query + // will return a tile query + post filter + QueryVisitor v = new EsQueryVistor(); + + try { + getRootOperand().visit( v ); + } + catch ( IndexException ex ) { + throw new RuntimeException( "Error building ElasticSearch query", ex ); + } + + + queryBuilder = v.getQueryBuilder(); + } + + + // Add our filter for context to our query for fast execution. + // Fast because it utilizes bitsets internally. See this post for more detail. + // http://www.elasticsearch.org/blog/all-about-elasticsearch-filter-bitsets/ + + // TODO evaluate performance when it's an all query. + // Do we need to put the context term first for performance? + if ( queryBuilder != null ) { + queryBuilder = QueryBuilders.boolQuery().must( queryBuilder ).must( QueryBuilders + .termQuery( IndexingUtils.ENTITY_CONTEXT_FIELDNAME, context ) ); + } + + //nothing was specified ensure we specify the context in the search + else { + queryBuilder = QueryBuilders.termQuery( IndexingUtils.ENTITY_CONTEXT_FIELDNAME, context ); + } + + return queryBuilder; + } + + + public FilterBuilder createFilterBuilder() { + FilterBuilder filterBuilder = null; + + if ( getRootOperand() != null ) { + QueryVisitor v = new EsQueryVistor(); + try { + getRootOperand().visit( v ); + + } catch ( IndexException ex ) { + throw new RuntimeException( "Error building ElasticSearch query", ex ); + } + filterBuilder = v.getFilterBuilder(); + } + + return filterBuilder; + } /** @@ -169,8 +243,8 @@ public class Query { String qlt = ql.toLowerCase(); if ( !qlt.startsWith( "select" ) - && !qlt.startsWith( "insert" ) - && !qlt.startsWith( "update" ) && !qlt.startsWith( "delete" ) ) { + && !qlt.startsWith( "insert" ) + && !qlt.startsWith( "update" ) && !qlt.startsWith( "delete" ) ) { if ( qlt.startsWith( "order by" ) ) { ql = "select * " + ql; @@ -224,7 +298,7 @@ public class Query { if ( o instanceof Map ) { @SuppressWarnings({ "unchecked", "rawtypes" }) Map<String, List<String>> params = - ClassUtils.cast( MapUtils.toMapList( ( Map ) o ) ); + ClassUtils.cast( MapUtils.toMapList( ( Map ) o ) ); return fromQueryParams( params ); } return null; @@ -232,7 +306,7 @@ public class Query { public static Query fromQueryParams( Map<String, List<String>> params ) - throws QueryParseException { + throws QueryParseException { Query q = null; CounterResolution resolution = null; List<Identifier> identifiers = null; @@ -243,6 +317,8 @@ public class Query { Boolean reversed = ListUtils.firstBoolean( params.get( "reversed" ) ); String connection = ListUtils.first( params.get( "connectionType" ) ); UUID start = ListUtils.firstUuid( params.get( "start" ) ); + String cursor = ListUtils.first( params.get( "cursor" ) ); + Integer limit = ListUtils.firstInteger( params.get( "limit" ) ); List<String> permissions = params.get( "permission" ); Long startTime = ListUtils.firstLong( params.get( "start_time" ) ); Long finishTime = ListUtils.firstLong( params.get( "end_time" ) ); @@ -313,7 +389,15 @@ public class Query { q.setStartResult( start ); } + if ( cursor != null ) { + q = newQueryIfNull( q ); + q.setCursor( cursor ); + } + if ( limit != null ) { + q = newQueryIfNull( q ); + q.setLimit( limit ); + } if ( startTime != null ) { q = newQueryIfNull( q ); @@ -356,21 +440,22 @@ public class Query { public static Query searchForProperty( String propertyName, Object propertyValue ) { Query q = new Query(); - q.addEqualityFilter(propertyName, propertyValue); + q.addEqualityFilter( propertyName, propertyValue ); return q; } public static Query findForProperty( String propertyName, Object propertyValue ) { Query q = new Query(); - q.addEqualityFilter(propertyName, propertyValue); + q.addEqualityFilter( propertyName, propertyValue ); + q.setLimit( 1 ); return q; } public static Query fromUUID( UUID uuid ) { Query q = new Query(); - q.addIdentifier( Identifier.fromUUID(uuid) ); + q.addIdentifier( Identifier.fromUUID( uuid ) ); return q; } @@ -629,8 +714,8 @@ public class Query { for ( SortPredicate s : sortPredicates ) { if ( s.getPropertyName().equals( propertyName ) ) { logger.error( - "Attempted to set sort order for " + s.getPropertyName() - + " more than once, discarding..." ); + "Attempted to set sort order for " + s.getPropertyName() + + " more than once, discarding..." ); return this; } } @@ -727,8 +812,8 @@ public class Query { public Query addContainsFilter( String propName, String keyword ) { ContainsOperand equality = new ContainsOperand( new ClassicToken( 0, "contains" ) ); - equality.setProperty(propName); - equality.setLiteral(keyword); + equality.setProperty( propName ); + equality.setLiteral( keyword ); addClause( equality ); @@ -737,8 +822,8 @@ public class Query { private void addClause( EqualityOperand equals, String propertyName, Object value ) { - equals.setProperty(propertyName); - equals.setLiteral(value); + equals.setProperty( propertyName ); + equals.setLiteral( value ); addClause( equals ); } @@ -800,7 +885,7 @@ public class Query { } - public UUID getStartResult(String cursor) { + public UUID getStartResult() { if ( ( startResult == null ) && ( cursor != null ) ) { byte[] cursorBytes = Base64.decodeBase64( cursor ); if ( ( cursorBytes != null ) && ( cursorBytes.length == 16 ) ) { @@ -811,6 +896,61 @@ public class Query { } + public String getCursor() { + return cursor; + } + + + public void setCursor( String cursor ) { + this.cursor = cursor; + } + + + public Query withCursor( String cursor ) { + setCursor( cursor ); + return this; + } + + + public int getLimit() { + return getLimit( DEFAULT_LIMIT ); + } + + + public int getLimit( int defaultLimit ) { + if ( limit <= 0 ) { + if ( defaultLimit > 0 ) { + return defaultLimit; + } + else { + return DEFAULT_LIMIT; + } + } + return limit; + } + + + public void setLimit( int limit ) { + + // TODO tnine. After users have had time to change their query limits, + // this needs to be uncommented and enforced. + // if(limit > MAX_LIMIT){ + // throw new IllegalArgumentException( + // String.format("Query limit must be <= to %d", MAX_LIMIT)); + // } + + if ( limit > MAX_LIMIT ) { + limit = MAX_LIMIT; + } + + this.limit = limit; + } + + + public Query withLimit( int limit ) { + setLimit( limit ); + return this; + } public boolean isReversed() { @@ -998,7 +1138,7 @@ public class Query { public SortPredicate(@JsonProperty("propertyName") String propertyName, - @JsonProperty("direction") Query.SortDirection direction ) { + @JsonProperty("direction") Query.SortDirection direction ) { if ( propertyName == null ) { throw new NullPointerException( "Property name was null" );
