ATLAS-1880: search API with support for entity/tag attribute filters
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/7c262b40 Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/7c262b40 Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/7c262b40 Branch: refs/heads/master Commit: 7c262b40ba4bbf6946c2423f36ddb0f3ee12c8f8 Parents: 8101883 Author: apoorvnaik <[email protected]> Authored: Wed Jun 28 06:22:34 2017 -0700 Committer: Madhan Neethiraj <[email protected]> Committed: Fri Jun 30 01:49:08 2017 -0700 ---------------------------------------------------------------------- .../java/org/apache/atlas/AtlasClientV2.java | 22 + .../org/apache/atlas/repository/Constants.java | 4 +- distro/src/conf/atlas-log4j.xml | 17 + .../repository/graphdb/AtlasGraphQuery.java | 39 +- .../repository/graphdb/AtlasIndexQuery.java | 8 + .../repository/graphdb/AtlasVertexQuery.java | 16 +- .../titan/query/NativeTitanGraphQuery.java | 25 +- .../graphdb/titan/query/TitanGraphQuery.java | 65 +- .../graphdb/titan/query/expr/AndCondition.java | 8 +- .../graphdb/titan/query/expr/HasPredicate.java | 7 +- .../query/graph/GraphCentricQueryBuilder.java | 28 +- .../graphdb/titan0/Titan0IndexQuery.java | 21 + .../graphdb/titan0/Titan0VertexQuery.java | 15 + .../titan0/query/NativeTitan0GraphQuery.java | 92 ++- .../graphdb/titan1/Titan1IndexQuery.java | 22 + .../graphdb/titan1/Titan1VertexQuery.java | 14 + .../titan1/query/NativeTitan1GraphQuery.java | 90 ++- .../model/discovery/AtlasSearchResult.java | 28 +- .../atlas/model/discovery/SearchParameters.java | 366 +++++++++++ .../atlas/model/impexp/AtlasImportRequest.java | 15 +- .../atlas/discovery/AtlasDiscoveryService.java | 9 + .../atlas/discovery/EntityDiscoveryService.java | 34 +- .../atlas/discovery/EntityLineageService.java | 2 + .../org/apache/atlas/discovery/GremlinStep.java | 388 ++++++++++++ .../apache/atlas/discovery/SearchPipeline.java | 611 +++++++++++++++++++ .../org/apache/atlas/discovery/SolrStep.java | 288 +++++++++ .../graph/GraphBackedSearchIndexer.java | 41 +- .../atlas/repository/impexp/ImportService.java | 5 +- .../store/graph/v1/AtlasGraphUtilsV1.java | 5 +- .../store/graph/v1/EntityGraphRetriever.java | 26 +- .../org/apache/atlas/util/SearchTracker.java | 73 +++ .../test/java/org/apache/atlas/TestModules.java | 17 +- .../services/EntityDiscoveryServiceTest.java | 8 + .../filters/StaleTransactionCleanupFilter.java | 2 +- .../atlas/web/resources/AdminResource.java | 43 +- .../apache/atlas/web/rest/DiscoveryREST.java | 47 ++ .../atlas/web/resources/AdminResourceTest.java | 4 +- 37 files changed, 2381 insertions(+), 124 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/7c262b40/client/src/main/java/org/apache/atlas/AtlasClientV2.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/atlas/AtlasClientV2.java b/client/src/main/java/org/apache/atlas/AtlasClientV2.java index 6141342..7e287e7 100644 --- a/client/src/main/java/org/apache/atlas/AtlasClientV2.java +++ b/client/src/main/java/org/apache/atlas/AtlasClientV2.java @@ -22,6 +22,7 @@ import com.sun.jersey.api.client.WebResource; import com.sun.jersey.core.util.MultivaluedMapImpl; import org.apache.atlas.model.SearchFilter; import org.apache.atlas.model.discovery.AtlasSearchResult; +import org.apache.atlas.model.discovery.SearchParameters; import org.apache.atlas.model.instance.AtlasClassification; import org.apache.atlas.model.instance.AtlasClassification.AtlasClassifications; import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo; @@ -100,9 +101,13 @@ public class AtlasClientV2 extends AtlasBaseClient { private static final String DISCOVERY_URI = BASE_URI + "v2/search"; private static final String DSL_URI = DISCOVERY_URI + "/dsl"; private static final String FULL_TEXT_URI = DISCOVERY_URI + "/fulltext"; + private static final String BASIC_SEARCH_URI = DISCOVERY_URI + "/basic"; + private static final String FACETED_SEARCH_URI = BASIC_SEARCH_URI; private static final APIInfo DSL_SEARCH = new APIInfo(DSL_URI, HttpMethod.GET, Response.Status.OK); private static final APIInfo FULL_TEXT_SEARCH = new APIInfo(FULL_TEXT_URI, HttpMethod.GET, Response.Status.OK); + private static final APIInfo BASIC_SEARCH = new APIInfo(BASIC_SEARCH_URI, HttpMethod.GET, Response.Status.OK); + private static final APIInfo FACETED_SEARCH = new APIInfo(FACETED_SEARCH_URI, HttpMethod.POST, Response.Status.OK); public AtlasClientV2(String[] baseUrl, String[] basicAuthUserNamePassword) { @@ -398,6 +403,23 @@ public class AtlasClientV2 extends AtlasBaseClient { return callAPI(FULL_TEXT_SEARCH, AtlasSearchResult.class, queryParams); } + public AtlasSearchResult basicSearch(final String typeName, final String classification, final String query, + final boolean excludeDeletedEntities, final int limit, final int offset) throws AtlasServiceException { + MultivaluedMap<String, String> queryParams = new MultivaluedMapImpl(); + queryParams.add("typeName", typeName); + queryParams.add("classification", classification); + queryParams.add(QUERY, query); + queryParams.add("excludeDeletedEntities", String.valueOf(excludeDeletedEntities)); + queryParams.add(LIMIT, String.valueOf(limit)); + queryParams.add(OFFSET, String.valueOf(offset)); + + return callAPI(BASIC_SEARCH, AtlasSearchResult.class, queryParams); + } + + public AtlasSearchResult facetedSearch(SearchParameters searchParameters) throws AtlasServiceException { + return callAPI(FACETED_SEARCH, AtlasSearchResult.class, searchParameters); + } + private <T> T getTypeDefByName(final String name, Class<T> typeDefClass) throws AtlasServiceException { String atlasPath = getAtlasPath(typeDefClass); APIInfo apiInfo = new APIInfo(String.format(GET_BY_NAME_TEMPLATE, atlasPath, name), HttpMethod.GET, Response.Status.OK); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/7c262b40/common/src/main/java/org/apache/atlas/repository/Constants.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java index ac02252..e8621cf 100644 --- a/common/src/main/java/org/apache/atlas/repository/Constants.java +++ b/common/src/main/java/org/apache/atlas/repository/Constants.java @@ -96,7 +96,9 @@ public final class Constants { public static final String QUALIFIED_NAME = "Referenceable.qualifiedName"; public static final String TYPE_NAME_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "typeName"; - + public static final String INDEX_SEARCH_MAX_RESULT_SET_SIZE = "atlas.graph.index.search.max-result-set-size"; + public static final String INDEX_SEARCH_MAX_TYPES_COUNT = "atlas.graph.index.search.max-types-count"; + public static final String INDEX_SEARCH_MAX_TAGS_COUNT = "atlas.graph.index.search.max-tags-count"; private Constants() { } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/7c262b40/distro/src/conf/atlas-log4j.xml ---------------------------------------------------------------------- diff --git a/distro/src/conf/atlas-log4j.xml b/distro/src/conf/atlas-log4j.xml index e6c0d9f..a0ef6a9 100755 --- a/distro/src/conf/atlas-log4j.xml +++ b/distro/src/conf/atlas-log4j.xml @@ -59,6 +59,23 @@ </layout> </appender> + <!-- Uncomment the following for perf logs --> + <!-- + <appender name="perf_appender" class="org.apache.log4j.DailyRollingFileAppender"> + <param name="file" value="${atlas.log.dir}/atlas_perf.log" /> + <param name="datePattern" value="'.'yyyy-MM-dd" /> + <param name="append" value="true" /> + <layout class="org.apache.log4j.PatternLayout"> + <param name="ConversionPattern" value="%d|%t|%m%n" /> + </layout> + </appender> + + <logger name="org.apache.atlas.perf" additivity="false"> + <level value="debug" /> + <appender-ref ref="perf_appender" /> + </logger> + --> + <logger name="org.apache.atlas" additivity="false"> <level value="info"/> <appender-ref ref="FILE"/> http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/7c262b40/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphQuery.java ---------------------------------------------------------------------- diff --git a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphQuery.java b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphQuery.java index 841edf7..73db22e 100644 --- a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphQuery.java +++ b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphQuery.java @@ -47,7 +47,7 @@ public interface AtlasGraphQuery<V, E> { * the specified list of values. * * @param propertyKey - * @param value + * @param values * @return */ AtlasGraphQuery<V, E> in(String propertyKey, Collection<?> values); @@ -56,7 +56,6 @@ public interface AtlasGraphQuery<V, E> { /** * Executes the query and returns the matching vertices. * @return - * @throws AtlasException */ Iterable<AtlasVertex<V, E>> vertices(); @@ -66,16 +65,32 @@ public interface AtlasGraphQuery<V, E> { */ Iterable<AtlasEdge<V, E>> edges(); + /** + * Executes the query and returns the matching vertices from given offset till the max limit + * @param limit max number of vertices + * @return + */ + Iterable<AtlasVertex<V, E>> vertices(int limit); + + /** + * Executes the query and returns the matching vertices from given offset till the max limit + * @param offset starting offset + * @param limit max number of vertices + * @return + */ + Iterable<AtlasVertex<V, E>> vertices(int offset, int limit); + /** * Adds a predicate that the returned vertices must have the specified * property and that its value matches the criterion specified. * * @param propertyKey - * @param value + * @param op + * @param values * @return */ - AtlasGraphQuery<V, E> has(String propertyKey, ComparisionOperator compMethod, Object values); + AtlasGraphQuery<V, E> has(String propertyKey, QueryOperator op, Object values); /** * Adds a predicate that the vertices returned must satisfy the @@ -94,17 +109,31 @@ public interface AtlasGraphQuery<V, E> { AtlasGraphQuery<V, E> createChildQuery(); + interface QueryOperator {} + /** * Comparison operators that can be used in an AtlasGraphQuery. */ - enum ComparisionOperator { + enum ComparisionOperator implements QueryOperator { + GREATER_THAN, GREATER_THAN_EQUAL, EQUAL, + LESS_THAN, LESS_THAN_EQUAL, NOT_EQUAL } /** + * String/text matching that can be used in AtlasGraphQuery + */ + enum MatchingOperator implements QueryOperator { + CONTAINS, + PREFIX, + SUFFIX, + REGEX + } + + /** * Adds all of the predicates that have been added to this query to the * specified query. * @param otherQuery http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/7c262b40/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasIndexQuery.java ---------------------------------------------------------------------- diff --git a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasIndexQuery.java b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasIndexQuery.java index 1ff9d5e..6bad173 100644 --- a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasIndexQuery.java +++ b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasIndexQuery.java @@ -36,6 +36,14 @@ public interface AtlasIndexQuery<V, E> { Iterator<Result<V, E>> vertices(); /** + * Gets the query results + * @param offset starting offset + * @param limit max number of results + * @return + */ + Iterator<Result<V, E>> vertices(int offset, int limit); + + /** * Query result from an index query. * * @param <V> http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/7c262b40/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasVertexQuery.java ---------------------------------------------------------------------- diff --git a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasVertexQuery.java b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasVertexQuery.java index 53f490f..9beb6a3 100644 --- a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasVertexQuery.java +++ b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasVertexQuery.java @@ -42,15 +42,29 @@ public interface AtlasVertexQuery<V, E> { Iterable<AtlasVertex<V, E>> vertices(); /** + * Returns the vertices that satisfy the query condition. + * + * @param limit Max number of vertices + * @return + */ + Iterable<AtlasVertex<V, E>> vertices(int limit); + + /** * Returns the incident edges that satisfy the query condition. * @return */ Iterable<AtlasEdge<V, E>> edges(); /** + * Returns the incident edges that satisfy the query condition. + * @param limit Max number of edges + * @return + */ + Iterable<AtlasEdge<V, E>> edges(int limit); + + /** * Returns the number of elements that match the query. * @return */ long count(); - } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/7c262b40/graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/NativeTitanGraphQuery.java ---------------------------------------------------------------------- diff --git a/graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/NativeTitanGraphQuery.java b/graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/NativeTitanGraphQuery.java index 0211ff0..288b325 100644 --- a/graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/NativeTitanGraphQuery.java +++ b/graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/NativeTitanGraphQuery.java @@ -17,12 +17,12 @@ */ package org.apache.atlas.repository.graphdb.titan.query; -import java.util.Collection; - import org.apache.atlas.repository.graphdb.AtlasEdge; -import org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator; +import org.apache.atlas.repository.graphdb.AtlasGraphQuery.QueryOperator; import org.apache.atlas.repository.graphdb.AtlasVertex; +import java.util.Collection; + /** * Interfaces that provides a thin wrapper around GraphQuery (used by Titan0) and * TitanGraphQuery (used by Titan 1). @@ -47,6 +47,22 @@ public interface NativeTitanGraphQuery<V, E> { Iterable<AtlasEdge<V, E>> edges(); /** + * Executes graph query + * @param limit Max vertices to return + * @return + */ + Iterable<AtlasVertex<V, E>> vertices(int limit); + + /** + * Executes graph query + * @param offset Starting offset + * @param limit Max vertices to return + * @return + */ + Iterable<AtlasVertex<V, E>> vertices(int offset, int limit); + + + /** * Adds an in condition to the query. * * @param propertyName @@ -61,6 +77,5 @@ public interface NativeTitanGraphQuery<V, E> { * @param op * @param value */ - void has(String propertyName, ComparisionOperator op, Object value); - + void has(String propertyName, QueryOperator op, Object value); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/7c262b40/graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/TitanGraphQuery.java ---------------------------------------------------------------------- diff --git a/graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/TitanGraphQuery.java b/graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/TitanGraphQuery.java index 0077a21..c38f6cc 100644 --- a/graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/TitanGraphQuery.java +++ b/graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/TitanGraphQuery.java @@ -17,11 +17,8 @@ */ package org.apache.atlas.repository.graphdb.titan.query; -import java.util.Collection; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; import org.apache.atlas.repository.graphdb.AtlasEdge; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasGraphQuery; @@ -33,6 +30,13 @@ import org.apache.atlas.repository.graphdb.titan.query.expr.OrCondition; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; + /** * Abstract implementation of AtlasGraphQuery that is used by both Titan 0.5.4 * and Titan 1.0.0. @@ -123,11 +127,10 @@ public abstract class TitanGraphQuery<V, E> implements AtlasGraphQuery<V, E> { @Override public Iterable<AtlasVertex<V, E>> vertices() { if (LOG.isDebugEnabled()) { - LOG.debug("Executing: " + queryCondition.toString()); + LOG.debug("Executing: " + queryCondition); } - //compute the overall result by unioning the results from all of the - //AndConditions together. + // Compute the overall result by combining the results of all the AndConditions (nested within OR) together. Set<AtlasVertex<V, E>> result = new HashSet<>(); for(AndCondition andExpr : queryCondition.getAndTerms()) { NativeTitanGraphQuery<V, E> andQuery = andExpr.create(getQueryFactory()); @@ -141,11 +144,10 @@ public abstract class TitanGraphQuery<V, E> implements AtlasGraphQuery<V, E> { @Override public Iterable<AtlasEdge<V, E>> edges() { if (LOG.isDebugEnabled()) { - LOG.debug("Executing: " + queryCondition.toString()); + LOG.debug("Executing: " + queryCondition); } - //compute the overall result by unioning the results from all of the - //AndConditions together. + // Compute the overall result by combining the results of all the AndConditions (nested within OR) together. Set<AtlasEdge<V, E>> result = new HashSet<>(); for(AndCondition andExpr : queryCondition.getAndTerms()) { NativeTitanGraphQuery<V, E> andQuery = andExpr.create(getQueryFactory()); @@ -157,7 +159,46 @@ public abstract class TitanGraphQuery<V, E> implements AtlasGraphQuery<V, E> { } @Override - public AtlasGraphQuery<V, E> has(String propertyKey, ComparisionOperator operator, + public Iterable<AtlasVertex<V, E>> vertices(int limit) { + return vertices(0, limit); + } + + @Override + public Iterable<AtlasVertex<V, E>> vertices(int offset, int limit) { + if (LOG.isDebugEnabled()) { + LOG.debug("Executing: " + queryCondition); + } + + Preconditions.checkArgument(offset >= 0, "Offset must be non-negative"); + Preconditions.checkArgument(limit >= 0, "Limit must be non-negative"); + + // Compute the overall result by combining the results of all the AndConditions (nested within OR) together. + Set<AtlasVertex<V, E>> result = new HashSet<>(); + long resultIdx = 0; + for(AndCondition andExpr : queryCondition.getAndTerms()) { + if (result.size() == limit) { + break; + } + + NativeTitanGraphQuery<V, E> andQuery = andExpr.create(getQueryFactory()); + for(AtlasVertex<V, E> vertex : andQuery.vertices(offset + limit)) { + if (resultIdx >= offset) { + result.add(vertex); + + if (result.size() == limit) { + break; + } + } + + resultIdx++; + } + } + + return result; + } + + @Override + public AtlasGraphQuery<V, E> has(String propertyKey, QueryOperator operator, Object value) { queryCondition.andWith(new HasPredicate(propertyKey, operator, value)); return this; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/7c262b40/graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/expr/AndCondition.java ---------------------------------------------------------------------- diff --git a/graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/expr/AndCondition.java b/graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/expr/AndCondition.java index 68f0eb2..db5093f 100644 --- a/graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/expr/AndCondition.java +++ b/graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/expr/AndCondition.java @@ -17,12 +17,12 @@ */ package org.apache.atlas.repository.graphdb.titan.query.expr; -import java.util.ArrayList; -import java.util.List; - import org.apache.atlas.repository.graphdb.titan.query.NativeTitanGraphQuery; import org.apache.atlas.repository.graphdb.titan.query.NativeTitanQueryFactory; +import java.util.ArrayList; +import java.util.List; + /** * Represents an AndCondition in a graph query. Only vertices that * satisfy the conditions in all of the query predicates will be returned @@ -78,7 +78,7 @@ public class AndCondition { /** * Creates a NativeTitanGraphQuery that can be used to evaluate this condition. * - * @param graph + * @param factory * @return */ public <V, E> NativeTitanGraphQuery<V, E> create(NativeTitanQueryFactory<V, E> factory) { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/7c262b40/graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/expr/HasPredicate.java ---------------------------------------------------------------------- diff --git a/graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/expr/HasPredicate.java b/graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/expr/HasPredicate.java index 24e4f5b..0652c41 100644 --- a/graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/expr/HasPredicate.java +++ b/graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/expr/HasPredicate.java @@ -17,7 +17,7 @@ */ package org.apache.atlas.repository.graphdb.titan.query.expr; -import org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator; +import org.apache.atlas.repository.graphdb.AtlasGraphQuery.QueryOperator; import org.apache.atlas.repository.graphdb.titan.query.NativeTitanGraphQuery; /** @@ -27,11 +27,10 @@ import org.apache.atlas.repository.graphdb.titan.query.NativeTitanGraphQuery; public class HasPredicate implements QueryPredicate { private String propertyName; - private ComparisionOperator op; + private QueryOperator op; private Object value; - public HasPredicate(String propertyName, ComparisionOperator op, Object value) { - super(); + public HasPredicate(String propertyName, QueryOperator op, Object value) { this.propertyName = propertyName; this.op = op; this.value = value; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/7c262b40/graphdb/titan0/src/main/java/com/thinkaurelius/titan/graphdb/query/graph/GraphCentricQueryBuilder.java ---------------------------------------------------------------------- diff --git a/graphdb/titan0/src/main/java/com/thinkaurelius/titan/graphdb/query/graph/GraphCentricQueryBuilder.java b/graphdb/titan0/src/main/java/com/thinkaurelius/titan/graphdb/query/graph/GraphCentricQueryBuilder.java index 54ff7cb..8e0928c 100644 --- a/graphdb/titan0/src/main/java/com/thinkaurelius/titan/graphdb/query/graph/GraphCentricQueryBuilder.java +++ b/graphdb/titan0/src/main/java/com/thinkaurelius/titan/graphdb/query/graph/GraphCentricQueryBuilder.java @@ -285,7 +285,33 @@ public class GraphCentricQueryBuilder implements TitanGraphQuery<GraphCentricQue } if (index.isCompositeIndex()) { - subcondition = indexCover((CompositeIndexType) index, conditions, subcover); + CompositeIndexType compositeIndex = (CompositeIndexType)index; + + subcondition = indexCover(compositeIndex, conditions, subcover); + + // if this is unique index, use it!! + if (compositeIndex.getCardinality() == Cardinality.SINGLE && subcondition != null) { + bestCandidate = null; // will cause the outer while() to bail out + candidateSubcover = subcover; + candidateSubcondition = subcondition; + candidateSupportsSort = supportsSort; + + if (log.isDebugEnabled()) { + log.debug("selected unique index {}", compositeIndex.getName()); + } + + if (coveredClauses.isEmpty()) { + isSorted = candidateSupportsSort; + } + + coveredClauses.clear();; + coveredClauses.addAll(candidateSubcover); + + jointQuery = new JointIndexQuery(); + jointQuery.add(compositeIndex, serializer.getQuery(compositeIndex, (List<Object[]>)candidateSubcondition)); + + break; + } } else { subcondition = indexCover((MixedIndexType) index, conditions, serializer, subcover); if (coveredClauses.isEmpty() && !supportsSort http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/7c262b40/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0IndexQuery.java ---------------------------------------------------------------------- diff --git a/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0IndexQuery.java b/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0IndexQuery.java index 1ed1734..c4a312d 100644 --- a/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0IndexQuery.java +++ b/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0IndexQuery.java @@ -19,6 +19,7 @@ package org.apache.atlas.repository.graphdb.titan0; import java.util.Iterator; +import com.google.common.base.Preconditions; import org.apache.atlas.repository.graphdb.AtlasIndexQuery; import org.apache.atlas.repository.graphdb.AtlasVertex; @@ -56,6 +57,26 @@ public class Titan0IndexQuery implements AtlasIndexQuery<Titan0Vertex, Titan0Edg return Iterators.transform(results, function); } + @Override + public Iterator<Result<Titan0Vertex, Titan0Edge>> vertices(int offset, int limit) { + Preconditions.checkArgument(offset >=0, "Index offset should be greater than or equals to 0"); + Preconditions.checkArgument(limit >=0, "Index limit should be greater than or equals to 0"); + Iterator<TitanIndexQuery.Result<Vertex>> results = wrappedIndexQuery + .offset(offset) + .limit(limit) + .vertices().iterator(); + + Function<TitanIndexQuery.Result<Vertex>, AtlasIndexQuery.Result<Titan0Vertex, Titan0Edge>> function = + new Function<TitanIndexQuery.Result<Vertex>, AtlasIndexQuery.Result<Titan0Vertex, Titan0Edge>>() { + + @Override + public AtlasIndexQuery.Result<Titan0Vertex, Titan0Edge> apply(TitanIndexQuery.Result<Vertex> source) { + return new ResultImpl(source); + } + }; + return Iterators.transform(results, function); + } + private final class ResultImpl implements AtlasIndexQuery.Result<Titan0Vertex, Titan0Edge> { private TitanIndexQuery.Result<Vertex> wrappedResult; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/7c262b40/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0VertexQuery.java ---------------------------------------------------------------------- diff --git a/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0VertexQuery.java b/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0VertexQuery.java index bd8b897..091e7d4 100644 --- a/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0VertexQuery.java +++ b/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0VertexQuery.java @@ -17,6 +17,7 @@ */ package org.apache.atlas.repository.graphdb.titan0; +import com.google.common.base.Preconditions; import org.apache.atlas.repository.graphdb.AtlasEdge; import org.apache.atlas.repository.graphdb.AtlasEdgeDirection; import org.apache.atlas.repository.graphdb.AtlasVertex; @@ -53,12 +54,26 @@ public class Titan0VertexQuery implements AtlasVertexQuery<Titan0Vertex, Titan0E } @Override + public Iterable<AtlasVertex<Titan0Vertex, Titan0Edge>> vertices(int limit) { + Preconditions.checkArgument(limit >=0, "Limit should be greater than or equals to 0"); + Iterable<Vertex> vertices = vertexQuery.limit(limit).vertices(); + return graph.wrapVertices(vertices); + } + + @Override public Iterable<AtlasEdge<Titan0Vertex, Titan0Edge>> edges() { Iterable<Edge> edges = vertexQuery.edges(); return graph.wrapEdges(edges); } @Override + public Iterable<AtlasEdge<Titan0Vertex, Titan0Edge>> edges(int limit) { + Preconditions.checkArgument(limit >=0, "Limit should be greater than or equals to 0"); + Iterable<Edge> edges = vertexQuery.limit(limit).edges(); + return graph.wrapEdges(edges); + } + + @Override public long count() { return vertexQuery.count(); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/7c262b40/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/query/NativeTitan0GraphQuery.java ---------------------------------------------------------------------- diff --git a/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/query/NativeTitan0GraphQuery.java b/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/query/NativeTitan0GraphQuery.java index 7ec6ffe..f1f1adb 100644 --- a/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/query/NativeTitan0GraphQuery.java +++ b/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/query/NativeTitan0GraphQuery.java @@ -17,21 +17,25 @@ */ package org.apache.atlas.repository.graphdb.titan0.query; -import java.util.Collection; - +import com.google.common.collect.Lists; +import com.thinkaurelius.titan.core.TitanGraphQuery; +import com.thinkaurelius.titan.core.attribute.Contain; +import com.thinkaurelius.titan.core.attribute.Text; +import com.thinkaurelius.titan.graphdb.query.TitanPredicate; +import com.tinkerpop.blueprints.Compare; +import com.tinkerpop.blueprints.Vertex; import org.apache.atlas.repository.graphdb.AtlasEdge; import org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator; +import org.apache.atlas.repository.graphdb.AtlasGraphQuery.MatchingOperator; +import org.apache.atlas.repository.graphdb.AtlasGraphQuery.QueryOperator; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.graphdb.titan.query.NativeTitanGraphQuery; -import org.apache.atlas.repository.graphdb.titan0.Titan0GraphDatabase; import org.apache.atlas.repository.graphdb.titan0.Titan0Edge; import org.apache.atlas.repository.graphdb.titan0.Titan0Graph; +import org.apache.atlas.repository.graphdb.titan0.Titan0GraphDatabase; import org.apache.atlas.repository.graphdb.titan0.Titan0Vertex; -import com.thinkaurelius.titan.core.TitanGraphQuery; -import com.thinkaurelius.titan.core.attribute.Contain; -import com.thinkaurelius.titan.graphdb.query.TitanPredicate; -import com.tinkerpop.blueprints.Compare; +import java.util.*; /** * Titan 0.5.4 implementation of NativeTitanGraphQuery. @@ -60,6 +64,28 @@ public class NativeTitan0GraphQuery implements NativeTitanGraphQuery<Titan0Verte Iterable it = query.edges(); return graph.wrapEdges(it); } + @Override + public Iterable<AtlasVertex<Titan0Vertex, Titan0Edge>> vertices(int limit) { + Iterable it = query.limit(limit).vertices(); + return graph.wrapVertices(it); + } + + @Override + public Iterable<AtlasVertex<Titan0Vertex, Titan0Edge>> vertices(int offset, int limit) { + List<Vertex> result = new ArrayList<>(limit); + Iterator<Vertex> iter = query.limit(offset + limit).vertices().iterator(); + + for (long resultIdx = 0; iter.hasNext() && result.size() < limit; resultIdx++) { + if (resultIdx < offset) { + continue; + } + + result.add(iter.next()); + } + + return graph.wrapVertices(result); + } + @Override public void in(String propertyName, Collection<?> values) { @@ -68,26 +94,48 @@ public class NativeTitan0GraphQuery implements NativeTitanGraphQuery<Titan0Verte } @Override - public void has(String propertyName, ComparisionOperator op, Object value) { - - Compare c = getGremlinPredicate(op); - TitanPredicate pred = TitanPredicate.Converter.convert(c); + public void has(String propertyName, QueryOperator op, Object value) { + TitanPredicate pred; + if (op instanceof ComparisionOperator) { + Compare c = getGremlinPredicate((ComparisionOperator) op); + pred = TitanPredicate.Converter.convert(c); + } else { + pred = getGremlinPredicate((MatchingOperator) op); + } query.has(propertyName, pred, value); } + private Text getGremlinPredicate(MatchingOperator op) { + switch (op) { + case CONTAINS: + return Text.CONTAINS; + case PREFIX: + return Text.PREFIX; + case SUFFIX: + return Text.CONTAINS_REGEX; + case REGEX: + return Text.REGEX; + default: + throw new RuntimeException("Unsupported matching operator:" + op); + } + } + private Compare getGremlinPredicate(ComparisionOperator op) { switch (op) { - case EQUAL: - return Compare.EQUAL; - case GREATER_THAN_EQUAL: - return Compare.GREATER_THAN_EQUAL; - case LESS_THAN_EQUAL: - return Compare.LESS_THAN_EQUAL; - case NOT_EQUAL: - return Compare.NOT_EQUAL; - - default: - throw new RuntimeException("Unsupported comparison operator:" + op); + case EQUAL: + return Compare.EQUAL; + case GREATER_THAN: + return Compare.GREATER_THAN; + case GREATER_THAN_EQUAL: + return Compare.GREATER_THAN_EQUAL; + case LESS_THAN: + return Compare.LESS_THAN; + case LESS_THAN_EQUAL: + return Compare.LESS_THAN_EQUAL; + case NOT_EQUAL: + return Compare.NOT_EQUAL; + default: + throw new RuntimeException("Unsupported comparison operator:" + op); } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/7c262b40/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/Titan1IndexQuery.java ---------------------------------------------------------------------- diff --git a/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/Titan1IndexQuery.java b/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/Titan1IndexQuery.java index 4073dd2..c7e4150 100644 --- a/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/Titan1IndexQuery.java +++ b/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/Titan1IndexQuery.java @@ -19,6 +19,7 @@ package org.apache.atlas.repository.graphdb.titan1; import java.util.Iterator; +import com.google.common.base.Preconditions; import org.apache.atlas.repository.graphdb.AtlasIndexQuery; import org.apache.atlas.repository.graphdb.AtlasVertex; @@ -56,6 +57,27 @@ public class Titan1IndexQuery implements AtlasIndexQuery<Titan1Vertex, Titan1Edg return Iterators.transform(results, function); } + @Override + public Iterator<Result<Titan1Vertex, Titan1Edge>> vertices(int offset, int limit) { + Preconditions.checkArgument(offset >=0, "Index offset should be greater than or equals to 0"); + Preconditions.checkArgument(limit >=0, "Index limit should be greater than or equals to 0"); + Iterator<TitanIndexQuery.Result<TitanVertex>> results = query + .offset(offset) + .limit(limit) + .vertices().iterator(); + + Function<TitanIndexQuery.Result<TitanVertex>, Result<Titan1Vertex, Titan1Edge>> function = + new Function<TitanIndexQuery.Result<TitanVertex>, Result<Titan1Vertex, Titan1Edge>>() { + + @Override + public Result<Titan1Vertex, Titan1Edge> apply(TitanIndexQuery.Result<TitanVertex> source) { + return new ResultImpl(source); + } + }; + + return Iterators.transform(results, function); + } + /** * Titan 1.0.0 implementation of AtlasIndexQuery.Result. */ http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/7c262b40/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/Titan1VertexQuery.java ---------------------------------------------------------------------- diff --git a/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/Titan1VertexQuery.java b/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/Titan1VertexQuery.java index 4452bcd..a761a74 100644 --- a/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/Titan1VertexQuery.java +++ b/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/Titan1VertexQuery.java @@ -17,6 +17,7 @@ */ package org.apache.atlas.repository.graphdb.titan1; +import com.google.common.base.Preconditions; import org.apache.atlas.repository.graphdb.AtlasEdge; import org.apache.atlas.repository.graphdb.AtlasEdgeDirection; import org.apache.atlas.repository.graphdb.AtlasVertex; @@ -51,10 +52,23 @@ public class Titan1VertexQuery implements AtlasVertexQuery<Titan1Vertex, Titan1E } @Override + public Iterable<AtlasVertex<Titan1Vertex, Titan1Edge>> vertices(int limit) { + Preconditions.checkArgument(limit >=0, "Limit should be greater than or equals to 0"); + Iterable vertices = query.limit(limit).vertices(); + return graph.wrapVertices(vertices); + } + + @Override public Iterable<AtlasEdge<Titan1Vertex, Titan1Edge>> edges() { Iterable edges = query.edges(); return graph.wrapEdges(edges); + } + @Override + public Iterable<AtlasEdge<Titan1Vertex, Titan1Edge>> edges(int limit) { + Preconditions.checkArgument(limit >=0, "Limit should be greater than or equals to 0"); + Iterable edges = query.limit(limit).edges(); + return graph.wrapEdges(edges); } @Override http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/7c262b40/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/query/NativeTitan1GraphQuery.java ---------------------------------------------------------------------- diff --git a/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/query/NativeTitan1GraphQuery.java b/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/query/NativeTitan1GraphQuery.java index 1ca900d..9293dbd 100644 --- a/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/query/NativeTitan1GraphQuery.java +++ b/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/query/NativeTitan1GraphQuery.java @@ -17,11 +17,16 @@ */ package org.apache.atlas.repository.graphdb.titan1.query; -import java.util.Collection; - import com.thinkaurelius.titan.core.TitanEdge; +import com.thinkaurelius.titan.core.TitanGraphQuery; +import com.thinkaurelius.titan.core.TitanVertex; +import com.thinkaurelius.titan.core.attribute.Contain; +import com.thinkaurelius.titan.core.attribute.Text; +import com.thinkaurelius.titan.graphdb.query.TitanPredicate; import org.apache.atlas.repository.graphdb.AtlasEdge; import org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator; +import org.apache.atlas.repository.graphdb.AtlasGraphQuery.MatchingOperator; +import org.apache.atlas.repository.graphdb.AtlasGraphQuery.QueryOperator; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.graphdb.titan.query.NativeTitanGraphQuery; import org.apache.atlas.repository.graphdb.titan1.Titan1Edge; @@ -29,11 +34,9 @@ import org.apache.atlas.repository.graphdb.titan1.Titan1Graph; import org.apache.atlas.repository.graphdb.titan1.Titan1GraphDatabase; import org.apache.atlas.repository.graphdb.titan1.Titan1Vertex; import org.apache.tinkerpop.gremlin.process.traversal.Compare; +import org.apache.tinkerpop.gremlin.structure.Vertex; -import com.thinkaurelius.titan.core.TitanGraphQuery; -import com.thinkaurelius.titan.core.TitanVertex; -import com.thinkaurelius.titan.core.attribute.Contain; -import com.thinkaurelius.titan.graphdb.query.TitanPredicate; +import java.util.*; /** * Titan 1.0.0 implementation of NativeTitanGraphQuery. @@ -61,32 +64,77 @@ public class NativeTitan1GraphQuery implements NativeTitanGraphQuery<Titan1Verte } @Override + public Iterable<AtlasVertex<Titan1Vertex, Titan1Edge>> vertices(int limit) { + Iterable<TitanVertex> it = query.limit(limit).vertices(); + return graph.wrapVertices(it); + } + + @Override + public Iterable<AtlasVertex<Titan1Vertex, Titan1Edge>> vertices(int offset, int limit) { + List<Vertex> result = new ArrayList<>(limit); + Iterator<? extends Vertex> iter = query.limit(offset + limit).vertices().iterator(); + + for (long resultIdx = 0; iter.hasNext() && result.size() < limit; resultIdx++) { + if (resultIdx < offset) { + continue; + } + + result.add(iter.next()); + } + + return graph.wrapVertices(result); + } + + @Override public void in(String propertyName, Collection<? extends Object> values) { query.has(propertyName, Contain.IN, values); } @Override - public void has(String propertyName, ComparisionOperator op, Object value) { - - Compare c = getGremlinPredicate(op); - TitanPredicate pred = TitanPredicate.Converter.convert(c); + public void has(String propertyName, QueryOperator op, Object value) { + TitanPredicate pred; + if (op instanceof ComparisionOperator) { + Compare c = getGremlinPredicate((ComparisionOperator) op); + pred = TitanPredicate.Converter.convert(c); + } else { + pred = getGremlinPredicate((MatchingOperator)op); + } query.has(propertyName, pred, value); } + private Text getGremlinPredicate(MatchingOperator op) { + switch (op) { + case CONTAINS: + return Text.CONTAINS; + case PREFIX: + return Text.PREFIX; + case SUFFIX: + return Text.CONTAINS_REGEX; + case REGEX: + return Text.REGEX; + default: + throw new RuntimeException("Unsupported matching operator:" + op); + } + } + private Compare getGremlinPredicate(ComparisionOperator op) { switch (op) { - case EQUAL: - return Compare.eq; - case GREATER_THAN_EQUAL: - return Compare.gte; - case LESS_THAN_EQUAL: - return Compare.lte; - case NOT_EQUAL: - return Compare.neq; - - default: - throw new RuntimeException("Unsupported comparison operator:" + op); + case EQUAL: + return Compare.eq; + case GREATER_THAN: + return Compare.gt; + case GREATER_THAN_EQUAL: + return Compare.gte; + case LESS_THAN: + return Compare.lt; + case LESS_THAN_EQUAL: + return Compare.lte; + case NOT_EQUAL: + return Compare.neq; + + default: + throw new RuntimeException("Unsupported comparison operator:" + op); } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/7c262b40/intg/src/main/java/org/apache/atlas/model/discovery/AtlasSearchResult.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/model/discovery/AtlasSearchResult.java b/intg/src/main/java/org/apache/atlas/model/discovery/AtlasSearchResult.java index a402c62..9513dcb 100644 --- a/intg/src/main/java/org/apache/atlas/model/discovery/AtlasSearchResult.java +++ b/intg/src/main/java/org/apache/atlas/model/discovery/AtlasSearchResult.java @@ -39,10 +39,9 @@ import static org.codehaus.jackson.annotate.JsonAutoDetect.Visibility.PUBLIC_ONL @JsonAutoDetect(getterVisibility = PUBLIC_ONLY, setterVisibility = PUBLIC_ONLY, fieldVisibility = NONE) @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) @JsonIgnoreProperties(ignoreUnknown = true) -@XmlRootElement -@XmlAccessorType(XmlAccessType.PROPERTY) public class AtlasSearchResult implements Serializable { private AtlasQueryType queryType; + private SearchParameters searchParameters; private String queryText; private String type; private String classification; @@ -59,11 +58,24 @@ public class AtlasSearchResult implements Serializable { public AtlasSearchResult(String queryText, AtlasQueryType queryType) { setQueryText(queryText); setQueryType(queryType); + setSearchParameters(null); setEntities(null); setAttributes(null); setFullTextResult(null); } + public AtlasSearchResult(SearchParameters searchParameters) { + setQueryType(AtlasQueryType.BASIC); + + if (searchParameters != null) { + setQueryText(searchParameters.getQuery()); + setSearchParameters(searchParameters); + setEntities(null); + setAttributes(null); + setFullTextResult(null); + } + } + public AtlasQueryType getQueryType() { return queryType; } public void setQueryType(AtlasQueryType queryType) { this.queryType = queryType; } @@ -98,6 +110,7 @@ public class AtlasSearchResult implements Serializable { if (o == null || getClass() != o.getClass()) return false; AtlasSearchResult that = (AtlasSearchResult) o; return Objects.equals(queryType, that.queryType) && + Objects.equals(searchParameters, that.searchParameters) && Objects.equals(queryText, that.queryText) && Objects.equals(type, that.type) && Objects.equals(classification, that.classification) && @@ -107,12 +120,13 @@ public class AtlasSearchResult implements Serializable { } @Override - public int hashCode() { return Objects.hash(queryText, queryType, entities, attributes, fullTextResult, type, classification); } + public int hashCode() { return Objects.hash(queryType, searchParameters, queryText, type, classification, entities, attributes, fullTextResult); } @Override public String toString() { return "AtlasSearchResult{" + "queryType=" + queryType + + ", searchParameters='" + searchParameters + '\'' + ", queryText='" + queryText + '\'' + ", type=" + type + ", classification=" + classification + @@ -149,6 +163,14 @@ public class AtlasSearchResult implements Serializable { } } + public void setSearchParameters(SearchParameters searchParameters) { + this.searchParameters = searchParameters; + } + + public SearchParameters getSearchParameters() { + return searchParameters; + } + public enum AtlasQueryType { DSL, FULL_TEXT, GREMLIN, BASIC, ATTRIBUTE } @JsonAutoDetect(getterVisibility = PUBLIC_ONLY, setterVisibility = PUBLIC_ONLY, fieldVisibility = NONE) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/7c262b40/intg/src/main/java/org/apache/atlas/model/discovery/SearchParameters.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/model/discovery/SearchParameters.java b/intg/src/main/java/org/apache/atlas/model/discovery/SearchParameters.java new file mode 100644 index 0000000..30855dc --- /dev/null +++ b/intg/src/main/java/org/apache/atlas/model/discovery/SearchParameters.java @@ -0,0 +1,366 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.model.discovery; + + +import org.codehaus.jackson.annotate.JsonAutoDetect; +import org.codehaus.jackson.annotate.JsonCreator; +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.annotate.JsonValue; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +import static org.codehaus.jackson.annotate.JsonAutoDetect.Visibility.NONE; +import static org.codehaus.jackson.annotate.JsonAutoDetect.Visibility.PUBLIC_ONLY; + +@JsonAutoDetect(getterVisibility = PUBLIC_ONLY, setterVisibility = PUBLIC_ONLY, fieldVisibility = NONE) +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +public class SearchParameters { + private String query; + private String typeName; + private String classification; + private boolean excludeDeletedEntities; + private int limit; + private int offset; + + private FilterCriteria entityFilters; + private FilterCriteria tagFilters; + private Set<String> attributes; + + /** + * @return The type of query + */ + public String getQuery() { + return query; + } + + /** + * Set query type + * @param query type + */ + public void setQuery(String query) { + this.query = query; + } + + /** + * @return Type name to search on + */ + public String getTypeName() { + return typeName; + } + + /** + * Set the type name to search on + * @param typeName type name + */ + public void setTypeName(String typeName) { + this.typeName = typeName; + } + + /** + * + * @return Classification/tag to search on + */ + public String getClassification() { + return classification; + } + + /** + * Set the classification/tag to search on + * @param classification classification/tag name + */ + public void setClassification(String classification) { + this.classification = classification; + } + + /** + * @return True iff deleted entities are excluded + */ + public boolean getExcludeDeletedEntities() { + return excludeDeletedEntities; + } + + /** + * Exclude deleted entities from search + * @param excludeDeletedEntities boolean flag + */ + public void setExcludeDeletedEntities(boolean excludeDeletedEntities) { + this.excludeDeletedEntities = excludeDeletedEntities; + } + + /** + * @return Max number of results to be returned + */ + public int getLimit() { + return limit; + } + + /** + * Restrict the results to the specified limit + * @param limit max number of results + */ + public void setLimit(int limit) { + this.limit = limit; + } + + /** + * @return Offset(pagination) of the results + */ + public int getOffset() { + return offset; + } + + /** + * @param offset + */ + public void setOffset(int offset) { + this.offset = offset; + } + + /** + * Entity attribute filters for the type (if type name is specified) + * @return + */ + public FilterCriteria getEntityFilters() { + return entityFilters; + } + + /** + * Filter the entities on this criteria + * @param entityFilters + */ + public void setEntityFilters(FilterCriteria entityFilters) { + this.entityFilters = entityFilters; + } + + /** + * Tag attribute filters for the classification/tag (if tag name is specified) + * @return + */ + public FilterCriteria getTagFilters() { + return tagFilters; + } + + /** + * Filter the tag/classification on this criteria + * @param tagFilters + */ + public void setTagFilters(FilterCriteria tagFilters) { + this.tagFilters = tagFilters; + } + + /** + * Attribute values included in the results + * @return + */ + public Set<String> getAttributes() { + return attributes; + } + + /** + * Return these attributes in the result response + * @param attributes + */ + public void setAttributes(Set<String> attributes) { + this.attributes = attributes; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + SearchParameters that = (SearchParameters) o; + return excludeDeletedEntities == that.excludeDeletedEntities && + limit == that.limit && + offset == that.offset && + Objects.equals(query, that.query) && + Objects.equals(typeName, that.typeName) && + Objects.equals(classification, that.classification) && + Objects.equals(entityFilters, that.entityFilters) && + Objects.equals(tagFilters, that.tagFilters) && + Objects.equals(attributes, that.attributes); + } + + @Override + public int hashCode() { + return Objects.hash(query, typeName, classification, excludeDeletedEntities, limit, offset, entityFilters, tagFilters, attributes); + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("SearchParameters{"); + sb.append("query='").append(query).append('\''); + sb.append(", typeName='").append(typeName).append('\''); + sb.append(", classification='").append(classification).append('\''); + sb.append(", excludeDeletedEntities=").append(excludeDeletedEntities); + sb.append(", limit=").append(limit); + sb.append(", offset=").append(offset); + sb.append(", entityFilters=").append(entityFilters); + sb.append(", tagFilters=").append(tagFilters); + sb.append(", attributes=").append(attributes); + sb.append('}'); + return sb.toString(); + } + + + @JsonAutoDetect(getterVisibility = PUBLIC_ONLY, setterVisibility = PUBLIC_ONLY, fieldVisibility = NONE) + @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) + @JsonIgnoreProperties(ignoreUnknown = true) + public static class FilterCriteria { + // Can be presented as a group of conditions or a single condition + public enum Condition { AND, OR } + + // Single condition + private String attributeName; + private Operator operator; + private String attributeValue; + + // Complex conditions + private Condition condition; + private List<FilterCriteria> criterion; + + public String getAttributeName() { + return attributeName; + } + + public void setAttributeName(String attributeName) { + this.attributeName = attributeName; + } + + public Operator getOperator() { + return operator; + } + + public void setOperator(Operator operator) { + this.operator = operator; + } + + public String getAttributeValue() { + return attributeValue; + } + + public void setAttributeValue(String attributeValue) { + this.attributeValue = attributeValue; + } + + public Condition getCondition() { + return condition; + } + + public void setCondition(Condition condition) { + this.condition = condition; + } + + public List<FilterCriteria> getCriterion() { + return criterion; + } + + public void setCriterion(List<FilterCriteria> criterion) { + this.criterion = criterion; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + FilterCriteria that = (FilterCriteria) o; + return Objects.equals(attributeName, that.attributeName) && + Objects.equals(operator, that.operator) && + Objects.equals(attributeValue, that.attributeValue) && + condition == that.condition && + Objects.equals(criterion, that.criterion); + } + + @Override + public int hashCode() { + return Objects.hash(attributeName, operator, attributeValue, condition, criterion); + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("FilterCriteria{"); + sb.append("attributeName='").append(attributeName).append('\''); + sb.append(", operator=").append(operator); + sb.append(", attributeValue='").append(attributeValue).append('\''); + sb.append(", condition=").append(condition); + sb.append(", criterion=").append(criterion); + sb.append('}'); + return sb.toString(); + } + } + + /** + * Supported search operations + * Logical comparision operators can only be used with numbers or dates + * IN, LIKE, startsWith, endsWith, CONTAINS can only be used with strings or text + */ + public enum Operator { + LT(new String[]{"<", "lt"}), + GT(new String[]{">", "gt"}), + LTE(new String[]{"<=", "lte"}), + GTE(new String[]{">=", "gte"}), + EQ(new String[]{"eq", "="}), + NEQ(new String[]{"neq", "!="}), + IN(new String[]{"in", "IN"}), + LIKE(new String[]{"like", "LIKE"}), + STARTS_WITH(new String[]{"startsWith", "STARTSWITH", "begins_with", "BEGINS_WITH"}), + ENDS_WITH(new String[]{"endsWith", "ENDSWITH", "ends_with", "BEGINS_WITH"}), + CONTAINS(new String[]{"contains", "CONTAINS"}) + ; + static final Map<String, Operator> operatorsMap = new HashMap<>(); + + private String[] symbols; + + static { + for (Operator operator : Operator.values()) { + for (String s : operator.symbols) { + operatorsMap.put(s, operator); + } + } + } + + Operator(String[] symbols) { + this.symbols = symbols; + } + + @JsonCreator + public static Operator fromString(String symbol) { + return operatorsMap.get(symbol); + } + + @JsonValue + public String getSymbol() { + return symbols[0]; + } + + public String[] getSymbols() { + return symbols; + } + + @Override + public String toString() { + return getSymbol(); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/7c262b40/intg/src/main/java/org/apache/atlas/model/impexp/AtlasImportRequest.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/model/impexp/AtlasImportRequest.java b/intg/src/main/java/org/apache/atlas/model/impexp/AtlasImportRequest.java index 21b6427..4d2ac62 100644 --- a/intg/src/main/java/org/apache/atlas/model/impexp/AtlasImportRequest.java +++ b/intg/src/main/java/org/apache/atlas/model/impexp/AtlasImportRequest.java @@ -18,14 +18,12 @@ package org.apache.atlas.model.impexp; import org.apache.atlas.model.typedef.AtlasBaseTypeDef; +import org.codehaus.jackson.annotate.JsonAnySetter; import org.codehaus.jackson.annotate.JsonAutoDetect; import org.codehaus.jackson.annotate.JsonIgnore; import org.codehaus.jackson.annotate.JsonIgnoreProperties; import org.codehaus.jackson.map.annotate.JsonSerialize; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlRootElement; import java.io.Serializable; import java.util.HashMap; import java.util.Map; @@ -33,12 +31,9 @@ import java.util.Map; import static org.codehaus.jackson.annotate.JsonAutoDetect.Visibility.NONE; import static org.codehaus.jackson.annotate.JsonAutoDetect.Visibility.PUBLIC_ONLY; - @JsonAutoDetect(getterVisibility=PUBLIC_ONLY, setterVisibility=PUBLIC_ONLY, fieldVisibility=NONE) @JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) @JsonIgnoreProperties(ignoreUnknown=true) -@XmlRootElement -@XmlAccessorType(XmlAccessType.PROPERTY) public class AtlasImportRequest implements Serializable { private static final long serialVersionUID = 1L; public static final String TRANSFORMS_KEY = "transforms"; @@ -97,4 +92,10 @@ public class AtlasImportRequest implements Serializable { return (String) this.options.get(key); } -} + @JsonAnySetter + public void setOption(String key, String value) { + if (null == options) { + options = new HashMap<>(); + } + options.put(key, value); + }} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/7c262b40/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java index 923a198..030a957 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java @@ -21,6 +21,7 @@ package org.apache.atlas.discovery; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.discovery.AtlasSearchResult; +import org.apache.atlas.model.discovery.SearchParameters; public interface AtlasDiscoveryService { /** @@ -56,4 +57,12 @@ public interface AtlasDiscoveryService { */ AtlasSearchResult searchUsingBasicQuery(String query, String type, String classification, String attrName, String attrValuePrefix, boolean excludeDeletedEntities, int limit, int offset) throws AtlasBaseException; + + /** + * Search for entities matching the search criteria + * @param searchParameters Search criteria + * @return Matching entities + * @throws AtlasBaseException + */ + AtlasSearchResult searchUsingBasicQuery(SearchParameters searchParameters) throws AtlasBaseException; } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/7c262b40/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java index 128cdbf..5068fa5 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java @@ -20,12 +20,14 @@ package org.apache.atlas.discovery; import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasConfiguration; import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.annotation.GraphTransaction; import org.apache.atlas.discovery.graph.DefaultGraphPersistenceStrategy; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.discovery.AtlasSearchResult; import org.apache.atlas.model.discovery.AtlasSearchResult.AtlasFullTextResult; import org.apache.atlas.model.discovery.AtlasSearchResult.AtlasQueryType; import org.apache.atlas.model.discovery.AtlasSearchResult.AttributeSearchResult; +import org.apache.atlas.model.discovery.SearchParameters; import org.apache.atlas.model.instance.AtlasEntity.Status; import org.apache.atlas.AtlasException; import org.apache.atlas.model.instance.AtlasEntityHeader; @@ -86,23 +88,28 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { private final EntityGraphRetriever entityRetriever; private final AtlasGremlinQueryProvider gremlinQueryProvider; private final AtlasTypeRegistry typeRegistry; + private final SearchPipeline searchPipeline; private final int maxResultSetSize; private final int maxTypesCountInIdxQuery; private final int maxTagsCountInIdxQuery; @Inject - EntityDiscoveryService(MetadataRepository metadataRepository, AtlasTypeRegistry typeRegistry, AtlasGraph graph) throws AtlasException { + EntityDiscoveryService(MetadataRepository metadataRepository, AtlasTypeRegistry typeRegistry, + AtlasGraph graph, SearchPipeline searchPipeline) throws AtlasException { this.graph = graph; this.graphPersistenceStrategy = new DefaultGraphPersistenceStrategy(metadataRepository); this.entityRetriever = new EntityGraphRetriever(typeRegistry); this.gremlinQueryProvider = AtlasGremlinQueryProvider.INSTANCE; this.typeRegistry = typeRegistry; - this.maxResultSetSize = ApplicationProperties.get().getInt("atlas.graph.index.search.max-result-set-size", 150); - this.maxTypesCountInIdxQuery = ApplicationProperties.get().getInt("atlas.graph.index.search.max-types-count", 10); - this.maxTagsCountInIdxQuery = ApplicationProperties.get().getInt("atlas.graph.index.search.max-tags-count", 10); + this.searchPipeline = searchPipeline; + + this.maxResultSetSize = ApplicationProperties.get().getInt(Constants.INDEX_SEARCH_MAX_RESULT_SET_SIZE, 150); + this.maxTypesCountInIdxQuery = ApplicationProperties.get().getInt(Constants.INDEX_SEARCH_MAX_TYPES_COUNT, 10); + this.maxTagsCountInIdxQuery = ApplicationProperties.get().getInt(Constants.INDEX_SEARCH_MAX_TAGS_COUNT, 10); } @Override + @GraphTransaction public AtlasSearchResult searchUsingDslQuery(String dslQuery, int limit, int offset) throws AtlasBaseException { AtlasSearchResult ret = new AtlasSearchResult(dslQuery, AtlasQueryType.DSL); GremlinQuery gremlinQuery = toGremlinQuery(dslQuery, limit, offset); @@ -155,6 +162,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { } @Override + @GraphTransaction public AtlasSearchResult searchUsingFullTextQuery(String fullTextQuery, boolean excludeDeletedEntities, int limit, int offset) throws AtlasBaseException { AtlasSearchResult ret = new AtlasSearchResult(fullTextQuery, AtlasQueryType.FULL_TEXT); @@ -170,6 +178,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { } @Override + @GraphTransaction public AtlasSearchResult searchUsingBasicQuery(String query, String typeName, String classification, String attrName, String attrValuePrefix, boolean excludeDeletedEntities, int limit, int offset) throws AtlasBaseException { @@ -393,6 +402,22 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { return ret; } + @Override + @GraphTransaction + public AtlasSearchResult searchUsingBasicQuery(SearchParameters searchParameters) throws AtlasBaseException { + AtlasSearchResult ret = new AtlasSearchResult(searchParameters); + + List<AtlasVertex> resultList = searchPipeline.run(searchParameters); + + for (AtlasVertex atlasVertex : resultList) { + AtlasEntityHeader entity = entityRetriever.toAtlasEntityHeader(atlasVertex, searchParameters.getAttributes()); + + ret.addEntity(entity); + } + + return ret; + } + private String getQueryForFullTextSearch(String userKeyedString, String typeName, String classification) { String typeFilter = getTypeFilter(typeRegistry, typeName, maxTypesCountInIdxQuery); String classficationFilter = getClassificationFilter(typeRegistry, classification, maxTagsCountInIdxQuery); @@ -548,4 +573,5 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { public int getMaxResultSetSize() { return maxResultSetSize; } + } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/7c262b40/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java index b07091a..3ae41c8 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java @@ -21,6 +21,7 @@ package org.apache.atlas.discovery; import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.annotation.GraphTransaction; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.model.lineage.AtlasLineageInfo; @@ -62,6 +63,7 @@ public class EntityLineageService implements AtlasLineageService { } @Override + @GraphTransaction public AtlasLineageInfo getAtlasLineageInfo(String guid, LineageDirection direction, int depth) throws AtlasBaseException { AtlasLineageInfo lineageInfo;
