Repository: atlas Updated Branches: refs/heads/0.8-incubating b8255d418 -> 0183beca0
ATLAS-1988: Implement REST API to search for related entities - Add excludeDeletedEntities flag Project: http://git-wip-us.apache.org/repos/asf/atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/0183beca Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/0183beca Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/0183beca Branch: refs/heads/0.8-incubating Commit: 0183beca085b95de4e8a24da85bae843011ce732 Parents: b8255d4 Author: Sarath Subramanian <ssubraman...@hortonworks.com> Authored: Thu Aug 17 13:51:28 2017 -0700 Committer: Sarath Subramanian <ssubraman...@hortonworks.com> Committed: Thu Aug 17 13:51:28 2017 -0700 ---------------------------------------------------------------------- .../main/java/org/apache/atlas/AtlasClient.java | 1 + .../atlas/discovery/AtlasDiscoveryService.java | 3 +- .../atlas/discovery/EntityDiscoveryService.java | 61 ++++++++++++-------- .../store/graph/v1/EntityGraphRetriever.java | 2 + .../atlas/util/AtlasGremlin2QueryProvider.java | 8 +-- .../apache/atlas/web/rest/DiscoveryREST.java | 17 +++--- 6 files changed, 54 insertions(+), 38 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/atlas/blob/0183beca/client/src/main/java/org/apache/atlas/AtlasClient.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/atlas/AtlasClient.java b/client/src/main/java/org/apache/atlas/AtlasClient.java index c50f3f5..5fa5df7 100755 --- a/client/src/main/java/org/apache/atlas/AtlasClient.java +++ b/client/src/main/java/org/apache/atlas/AtlasClient.java @@ -101,6 +101,7 @@ public class AtlasClient extends AtlasBaseClient { public static final String NAME = "name"; public static final String DESCRIPTION = "description"; public static final String OWNER = "owner"; + public static final String CREATE_TIME = "createTime"; public static final String INFRASTRUCTURE_SUPER_TYPE = "Infrastructure"; public static final String DATA_SET_SUPER_TYPE = "DataSet"; http://git-wip-us.apache.org/repos/asf/atlas/blob/0183beca/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java index ead5d3c..8196a67 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java @@ -73,9 +73,10 @@ public interface AtlasDiscoveryService { * @param relation relation name. * @param sortByAttribute sort the result using this attribute name, default value is 'name' * @param sortOrder sorting order + * @param excludeDeletedEntities exclude deleted entities in search result. * @param limit number of resultant rows (for pagination). [ limit > 0 ] and [ limit < maxlimit ]. -1 maps to atlas.search.defaultlimit property. * @param offset offset to the results returned (for pagination). [ offset >= 0 ]. -1 maps to offset 0. * @return AtlasSearchResult */ - AtlasSearchResult searchRelatedEntities(String guid, String relation, String sortByAttribute, SortOrder sortOrder, int limit, int offset) throws AtlasBaseException; + AtlasSearchResult searchRelatedEntities(String guid, String relation, String sortByAttribute, SortOrder sortOrder, boolean excludeDeletedEntities, int limit, int offset) throws AtlasBaseException; } http://git-wip-us.apache.org/repos/asf/atlas/blob/0183beca/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java index d5062a7..1e68835 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java @@ -76,14 +76,19 @@ import java.util.*; import static org.apache.atlas.AtlasErrorCode.CLASSIFICATION_NOT_FOUND; import static org.apache.atlas.AtlasErrorCode.DISCOVERY_QUERY_FAILED; import static org.apache.atlas.AtlasErrorCode.UNKNOWN_TYPENAME; +import static org.apache.atlas.SortOrder.ASCENDING; import static org.apache.atlas.SortOrder.DESCENDING; import static org.apache.atlas.model.TypeCategory.ARRAY; import static org.apache.atlas.model.TypeCategory.MAP; import static org.apache.atlas.model.TypeCategory.OBJECT_ID_TYPE; +import static org.apache.atlas.model.instance.AtlasEntity.Status.ACTIVE; +import static org.apache.atlas.model.instance.AtlasEntity.Status.DELETED; import static org.apache.atlas.repository.graph.GraphHelper.EDGE_LABEL_PREFIX; +import static org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery.BASIC_SEARCH_STATE_FILTER; import static org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery.RELATIONSHIP_SEARCH; import static org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery.RELATIONSHIP_SEARCH_DESCENDING_SORT; import static org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery.RELATIONSHIP_SEARCH_ASCENDING_SORT; +import static org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery.TO_RANGE_LIST; @Component public class EntityDiscoveryService implements AtlasDiscoveryService { @@ -364,9 +369,9 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { } if (excludeDeletedEntities) { - bindings.put("state", Status.ACTIVE.toString()); + bindings.put("state", ACTIVE.toString()); - basicQuery += gremlinQueryProvider.getQuery(AtlasGremlinQuery.BASIC_SEARCH_STATE_FILTER); + basicQuery += gremlinQueryProvider.getQuery(BASIC_SEARCH_STATE_FILTER); } if (isGuidPrefixSearch) { @@ -378,7 +383,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { bindings.put("startIdx", params.offset()); bindings.put("endIdx", params.offset() + params.limit()); - basicQuery += gremlinQueryProvider.getQuery(AtlasGremlinQuery.TO_RANGE_LIST); + basicQuery += gremlinQueryProvider.getQuery(TO_RANGE_LIST); ScriptEngine scriptEngine = graph.getGremlinScriptEngine(); @@ -504,8 +509,8 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { @Override @GraphTransaction - public AtlasSearchResult searchRelatedEntities(String guid, String relation, String sortByAttributeName, - SortOrder sortOrder, int limit, int offset) throws AtlasBaseException { + public AtlasSearchResult searchRelatedEntities(String guid, String relation, String sortByAttributeName, SortOrder sortOrder, + boolean excludeDeletedEntities, int limit, int offset) throws AtlasBaseException { AtlasSearchResult ret = new AtlasSearchResult(AtlasQueryType.RELATIONSHIP); if (StringUtils.isEmpty(guid) || StringUtils.isEmpty(relation)) { @@ -543,21 +548,37 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { sortByAttributeName = sortByAttribute.getQualifiedName(); if (sortOrder == null) { - sortOrder = SortOrder.ASCENDING; + sortOrder = ASCENDING; } } - String relatedEntitiesQuery = getRelatedEntitiesQuery(sortOrder); + QueryParams params = validateSearchParams(limit, offset); ScriptEngine scriptEngine = graph.getGremlinScriptEngine(); Bindings bindings = scriptEngine.createBindings(); - QueryParams params = validateSearchParams(limit, offset); + Set<String> states = getEntityStates(); + String relatedEntitiesQuery = gremlinQueryProvider.getQuery(RELATIONSHIP_SEARCH); + + if (excludeDeletedEntities) { + states.remove(DELETED.toString()); + } + + if (sortOrder == ASCENDING) { + relatedEntitiesQuery += gremlinQueryProvider.getQuery(RELATIONSHIP_SEARCH_ASCENDING_SORT); + bindings.put("sortAttributeName", sortByAttributeName); + + } else if (sortOrder == DESCENDING) { + relatedEntitiesQuery += gremlinQueryProvider.getQuery(RELATIONSHIP_SEARCH_DESCENDING_SORT); + bindings.put("sortAttributeName", sortByAttributeName); + } + + relatedEntitiesQuery += gremlinQueryProvider.getQuery(TO_RANGE_LIST); bindings.put("g", graph); bindings.put("guid", guid); bindings.put("relation", relation); - bindings.put("sortAttributeName", sortByAttributeName); - bindings.put("offset", params.offset()); - bindings.put("limit", params.offset() + params.limit()); + bindings.put("states", Collections.unmodifiableSet(states)); + bindings.put("startIdx", params.offset()); + bindings.put("endIdx", params.offset() + params.limit()); try { Object result = graph.executeGremlinScript(scriptEngine, bindings, relatedEntitiesQuery, false); @@ -725,7 +746,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { } private boolean skipDeletedEntities(boolean excludeDeletedEntities, AtlasVertex<?, ?> vertex) { - return excludeDeletedEntities && GraphHelper.getStatus(vertex) == Status.DELETED; + return excludeDeletedEntities && GraphHelper.getStatus(vertex) == DELETED; } private static String getClassificationFilter(AtlasTypeRegistry typeRegistry, String classificationName, int maxTypesLengthInIdxQuery) { @@ -767,17 +788,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { return ret; } - private String getRelatedEntitiesQuery(SortOrder sortOrder) { - final String ret; - - if (sortOrder == null) { - ret = gremlinQueryProvider.getQuery(RELATIONSHIP_SEARCH); - } else if (sortOrder == DESCENDING) { - ret = gremlinQueryProvider.getQuery(RELATIONSHIP_SEARCH_DESCENDING_SORT); - } else { - ret = gremlinQueryProvider.getQuery(RELATIONSHIP_SEARCH_ASCENDING_SORT); - } - - return ret; + private Set<String> getEntityStates() { + return new HashSet<>(Arrays.asList(ACTIVE.toString(), DELETED.toString())); } -} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/atlas/blob/0183beca/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java index f6c8c72..83c9c2b 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java @@ -218,11 +218,13 @@ public final class EntityGraphRetriever { Object name = getVertexAttribute(entityVertex, entityType.getAttribute(AtlasClient.NAME)); Object description = getVertexAttribute(entityVertex, entityType.getAttribute(AtlasClient.DESCRIPTION)); Object owner = getVertexAttribute(entityVertex, entityType.getAttribute(AtlasClient.OWNER)); + Object createTime = entityVertex.getProperty(Constants.TIMESTAMP_PROPERTY_KEY, Long.class); Object displayText = name != null ? name : ret.getAttribute(AtlasClient.QUALIFIED_NAME); ret.setAttribute(AtlasClient.NAME, name); ret.setAttribute(AtlasClient.DESCRIPTION, description); ret.setAttribute(AtlasClient.OWNER, owner); + ret.setAttribute(AtlasClient.CREATE_TIME, createTime); if (displayText != null) { ret.setDisplayText(displayText.toString()); http://git-wip-us.apache.org/repos/asf/atlas/blob/0183beca/repository/src/main/java/org/apache/atlas/util/AtlasGremlin2QueryProvider.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/util/AtlasGremlin2QueryProvider.java b/repository/src/main/java/org/apache/atlas/util/AtlasGremlin2QueryProvider.java index 2869e88..ea4676a 100644 --- a/repository/src/main/java/org/apache/atlas/util/AtlasGremlin2QueryProvider.java +++ b/repository/src/main/java/org/apache/atlas/util/AtlasGremlin2QueryProvider.java @@ -76,11 +76,11 @@ public class AtlasGremlin2QueryProvider extends AtlasGremlinQueryProvider { case GUID_PREFIX_FILTER: return ".filter{it.'__guid'.matches(guid)}"; case RELATIONSHIP_SEARCH: - return "g.V('__guid', guid).both(relation)[offset..<limit].toList()"; - case RELATIONSHIP_SEARCH_DESCENDING_SORT: - return "g.V('__guid', guid).both(relation)[offset..<limit].order{it.b.getProperty(sortAttributeName) <=> it.a.getProperty(sortAttributeName)}.toList()"; + return "g.V('__guid', guid).both(relation).has('__state', T.in, states)"; case RELATIONSHIP_SEARCH_ASCENDING_SORT: - return "g.V('__guid', guid).both(relation)[offset..<limit].order{it.a.getProperty(sortAttributeName) <=> it.b.getProperty(sortAttributeName)}.toList()"; + return ".order{it.a.getProperty(sortAttributeName) <=> it.b.getProperty(sortAttributeName)}"; + case RELATIONSHIP_SEARCH_DESCENDING_SORT: + return ".order{it.b.getProperty(sortAttributeName) <=> it.a.getProperty(sortAttributeName)}"; case COMPARE_LT: return ".has('%s', T.lt, %s)"; case COMPARE_LTE: http://git-wip-us.apache.org/repos/asf/atlas/blob/0183beca/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java index 032bde8..52258e3 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java @@ -285,21 +285,22 @@ public class DiscoveryREST { @Path("relationship") @Consumes(Servlets.JSON_MEDIA_TYPE) @Produces(Servlets.JSON_MEDIA_TYPE) - public AtlasSearchResult searchRelatedEntities(@QueryParam("guid") String guid, - @QueryParam("relation") String relation, - @QueryParam("sortBy") String sortByAttribute, - @QueryParam("sortOrder") SortOrder sortOrder, - @QueryParam("limit") int limit, - @QueryParam("offset") int offset) throws AtlasBaseException { + public AtlasSearchResult searchRelatedEntities(@QueryParam("guid") String guid, + @QueryParam("relation") String relation, + @QueryParam("sortBy") String sortByAttribute, + @QueryParam("sortOrder") SortOrder sortOrder, + @QueryParam("excludeDeletedEntities") boolean excludeDeletedEntities, + @QueryParam("limit") int limit, + @QueryParam("offset") int offset) throws AtlasBaseException { AtlasPerfTracer perf = null; try { if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "DiscoveryREST.relatedEntitiesSearchUsingGremlin(" + guid + - ", " + relation + ", " + sortByAttribute + ", " + sortOrder + ", " + limit + ", " + offset + ")"); + ", " + relation + ", " + sortByAttribute + ", " + sortOrder + ", " + excludeDeletedEntities + ", " + ", " + limit + ", " + offset + ")"); } - return atlasDiscoveryService.searchRelatedEntities(guid, relation, sortByAttribute, sortOrder, limit, offset); + return atlasDiscoveryService.searchRelatedEntities(guid, relation, sortByAttribute, sortOrder, excludeDeletedEntities, limit, offset); } finally { AtlasPerfTracer.log(perf); }
