Repository: atlas Updated Branches: refs/heads/master 0dad95296 -> f622751db
ATLAS-2561: basic-search update to enable search by glossary-term Project: http://git-wip-us.apache.org/repos/asf/atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/f622751d Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/f622751d Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/f622751d Branch: refs/heads/master Commit: f622751dbbc01aab5f0398bfd65674e9bcb92e72 Parents: 0dad952 Author: Madhan Neethiraj <[email protected]> Authored: Fri Apr 13 01:18:28 2018 -0700 Committer: Madhan Neethiraj <[email protected]> Committed: Sun Apr 15 01:48:48 2018 -0700 ---------------------------------------------------------------------- .../java/org/apache/atlas/AtlasErrorCode.java | 3 +- .../atlas/model/discovery/SearchParameters.java | 21 +++- .../apache/atlas/discovery/SearchContext.java | 62 ++++++++++ .../atlas/discovery/TermSearchProcessor.java | 117 +++++++++++++++++++ .../apache/atlas/web/rest/DiscoveryREST.java | 2 +- 5 files changed, 202 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/atlas/blob/f622751d/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java index 539a225..bbb1b1a 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java +++ b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java @@ -91,7 +91,7 @@ public enum AtlasErrorCode { RELATIONSHIPDEF_INVALID(400, "ATLAS-400-00-044", "Invalid relationshipDef: {0}"), RELATIONSHIP_INVALID_ENDTYPE(400, "ATLAS-400-00-045", "Invalid entity-type for relationship attribute â{0}â: entity specified (guid={1}) is of type â{2}â, but expected type is â{3}â"), UNKNOWN_CLASSIFICATION(400, "ATLAS-400-00-046", "{0}: Unknown/invalid classification"), - INVALID_SEARCH_PARAMS(400, "ATLAS-400-00-047", "No search parameter was found. One of the following MUST be specified in the request; typeName, classification or queryText"), + INVALID_SEARCH_PARAMS(400, "ATLAS-400-00-047", "No search parameter was found. One of the following MUST be specified in the request; typeName, classification, termName or queryText"), INVALID_RELATIONSHIP_ATTRIBUTE(400, "ATLAS-400-00-048", "Expected attribute {0} to be a relationship but found type {1}"), INVALID_RELATIONSHIP_TYPE(400, "ATLAS-400-00-049", "Invalid entity type '{0}', guid '{1}' in relationship search"), INVALID_IMPORT_ATTRIBUTE_TYPE_CHANGED(400, "ATLAS-400-00-050", "Attribute {0}.{1} is of type {2}. Import has this attribute type as {3}"), @@ -138,6 +138,7 @@ public enum AtlasErrorCode { GLOSSARY_QUALIFIED_NAME_CANT_BE_DERIVED(400, "ATLAS-400-00-078", "Attributes qualifiedName and displayName are missing. Failed to derive a unique name for Glossary"), GLOSSARY_TERM_QUALIFIED_NAME_CANT_BE_DERIVED(400, "ATLAS-400-00-079", "Attributes qualifiedName, displayName & glossary name are missing. Failed to derive a unique name for Glossary term"), GLOSSARY_CATEGORY_QUALIFIED_NAME_CANT_BE_DERIVED(400, "ATLAS-400-00-07A", "Attributes qualifiedName, displayName & glossary name are missing. Failed to derive a unique name for Glossary category"), + UNKNOWN_GLOSSARY_TERM(400, "ATLAS-400-00-07B", "{0}: Unknown/invalid glossary term"), UNAUTHORIZED_ACCESS(403, "ATLAS-403-00-001", "{0} is not authorized to perform {1}"), http://git-wip-us.apache.org/repos/asf/atlas/blob/f622751d/intg/src/main/java/org/apache/atlas/model/discovery/SearchParameters.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/model/discovery/SearchParameters.java b/intg/src/main/java/org/apache/atlas/model/discovery/SearchParameters.java index 347a314..39ebf23 100644 --- a/intg/src/main/java/org/apache/atlas/model/discovery/SearchParameters.java +++ b/intg/src/main/java/org/apache/atlas/model/discovery/SearchParameters.java @@ -43,6 +43,7 @@ public class SearchParameters implements Serializable { private String query; private String typeName; private String classification; + private String termName; private boolean excludeDeletedEntities; private boolean includeClassificationAttributes; private boolean includeSubTypes = true; @@ -86,6 +87,22 @@ public class SearchParameters implements Serializable { /** * + * @return termName to search on + */ + public String getTermName() { + return termName; + } + + /** + * Set the classification/tag to search on + * @param termName classification/tag name + */ + public void setTermName(String termName) { + this.termName = termName; + } + + /** + * * @return Classification/tag to search on */ public String getClassification() { @@ -249,6 +266,7 @@ public class SearchParameters implements Serializable { Objects.equals(query, that.query) && Objects.equals(typeName, that.typeName) && Objects.equals(classification, that.classification) && + Objects.equals(termName, that.termName) && Objects.equals(entityFilters, that.entityFilters) && Objects.equals(tagFilters, that.tagFilters) && Objects.equals(attributes, that.attributes); @@ -256,7 +274,7 @@ public class SearchParameters implements Serializable { @Override public int hashCode() { - return Objects.hash(query, typeName, classification, excludeDeletedEntities, includeClassificationAttributes, + return Objects.hash(query, typeName, classification, termName, excludeDeletedEntities, includeClassificationAttributes, limit, offset, entityFilters, tagFilters, attributes); } @@ -269,6 +287,7 @@ public class SearchParameters implements Serializable { sb.append("query='").append(query).append('\''); sb.append(", typeName='").append(typeName).append('\''); sb.append(", classification='").append(classification).append('\''); + sb.append(", termName='").append(termName).append('\''); sb.append(", excludeDeletedEntities=").append(excludeDeletedEntities); sb.append(", includeClassificationAttributes=").append(includeClassificationAttributes); sb.append(", limit=").append(limit); http://git-wip-us.apache.org/repos/asf/atlas/blob/f622751d/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java index 341a047..1890b1d 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java +++ b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java @@ -22,16 +22,28 @@ import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.discovery.SearchParameters; import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria; +import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.typedef.AtlasClassificationDef; +import org.apache.atlas.repository.Constants; +import org.apache.atlas.repository.graph.GraphHelper; +import org.apache.atlas.repository.graphdb.AtlasEdge; +import org.apache.atlas.repository.graphdb.AtlasEdgeDirection; import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.apache.atlas.repository.graphdb.AtlasGraphQuery; +import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.type.AtlasClassificationType; import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasStructType; +import org.apache.atlas.type.AtlasStructType.AtlasAttribute; import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.atlas.v1.model.instance.Id; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; +import java.util.ArrayList; import java.util.HashSet; +import java.util.Iterator; +import java.util.List; import java.util.Set; /* @@ -73,12 +85,23 @@ public class SearchContext { throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_CLASSIFICATION, classificationName); } + AtlasVertex glossaryTermVertex = getGlossaryTermVertex(searchParameters.getTermName()); + + // Validate if the term exists + if (StringUtils.isNotEmpty(searchParameters.getTermName()) && glossaryTermVertex == null) { + throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_GLOSSARY_TERM, searchParameters.getTermName()); + } + // Invalid attributes will raise an exception with 400 error code validateAttributes(entityType, searchParameters.getEntityFilters()); // Invalid attributes will raise an exception with 400 error code validateAttributes(classificationType, searchParameters.getTagFilters()); + if (glossaryTermVertex != null) { + addProcessor(new TermSearchProcessor(this, getAssignedEntities(glossaryTermVertex))); + } + if (needFullTextProcessor()) { addProcessor(new FullTextSearchProcessor(this)); } @@ -185,4 +208,43 @@ public class SearchContext { return ret; } + + private AtlasVertex getGlossaryTermVertex(String termName) { + AtlasVertex ret = null; + + if (StringUtils.isNotEmpty(termName)) { + AtlasEntityType termType = getTermEntityType(); + AtlasAttribute attrName = termType.getAttribute(TermSearchProcessor.ATLAS_GLOSSARY_TERM_ATTR_QNAME); + AtlasGraphQuery query = graph.query().has(Constants.ENTITY_TYPE_PROPERTY_KEY, termType.getTypeName()) + .has(attrName.getVertexPropertyName(), termName) + .has(Constants.STATE_PROPERTY_KEY, AtlasEntity.Status.ACTIVE.name()); + + Iterator<AtlasVertex> results = query.vertices().iterator(); + + ret = results.hasNext() ? results.next() : null; + } + + return ret; + } + + private List<AtlasVertex> getAssignedEntities(AtlasVertex glossaryTerm) { + List<AtlasVertex> ret = new ArrayList<>(); + AtlasEntityType termType = getTermEntityType(); + AtlasAttribute attr = termType.getRelationshipAttribute(TermSearchProcessor.ATLAS_GLOSSARY_TERM_ATTR_ASSIGNED_ENTITIES); + Iterator<AtlasEdge> edges = GraphHelper.getEdgesForLabel(glossaryTerm, attr.getRelationshipEdgeLabel(), attr.getRelationshipEdgeDirection()); + + if (edges != null) { + while (edges.hasNext()) { + AtlasEdge edge = edges.next(); + + ret.add(edge.getInVertex()); + } + } + + return ret; + } + + private AtlasEntityType getTermEntityType() { + return typeRegistry.getEntityTypeByName(TermSearchProcessor.ATLAS_GLOSSARY_TERM_ENTITY_TYPE); + } } http://git-wip-us.apache.org/repos/asf/atlas/blob/f622751d/repository/src/main/java/org/apache/atlas/discovery/TermSearchProcessor.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/discovery/TermSearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/TermSearchProcessor.java new file mode 100644 index 0000000..1f51afb --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/discovery/TermSearchProcessor.java @@ -0,0 +1,117 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.discovery; + +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.utils.AtlasPerfTracer; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.Predicate; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; + + + +public class TermSearchProcessor extends SearchProcessor { + private static final Logger LOG = LoggerFactory.getLogger(TermSearchProcessor.class); + private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("TermSearchProcessor"); + + public static final String ATLAS_GLOSSARY_TERM_ENTITY_TYPE = "__AtlasGlossaryTerm"; + public static final String ATLAS_GLOSSARY_TERM_ATTR_QNAME = "qualifiedName"; + public static final String ATLAS_GLOSSARY_TERM_ATTR_ASSIGNED_ENTITIES = "assignedEntities"; + + final List<AtlasVertex> assignedEntities; + + public TermSearchProcessor(SearchContext context, List<AtlasVertex> assignedEntities) { + super(context); + + this.assignedEntities = assignedEntities; + } + + @Override + public List<AtlasVertex> execute() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> TermSearchProcessor.execute({})", context); + } + + List<AtlasVertex> ret = new ArrayList<>(); + AtlasPerfTracer perf = null; + + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "TermSearchProcessor.execute(" + context + ")"); + } + + try { + if (CollectionUtils.isNotEmpty(assignedEntities)) { + final int startIdx = context.getSearchParameters().getOffset(); + final int limit = context.getSearchParameters().getLimit(); + final List<AtlasVertex> tmpList = new ArrayList<>(assignedEntities); + + super.filter(tmpList); + + collectResultVertices(ret, startIdx, limit, 0, tmpList); + } + } finally { + AtlasPerfTracer.log(perf); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== TermSearchProcessor.execute({}): ret.size()={}", context, ret.size()); + } + + return ret; + } + + @Override + public void filter(List<AtlasVertex> entityVertices) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> TermSearchProcessor.filter({})", entityVertices.size()); + } + + if (CollectionUtils.isNotEmpty(entityVertices)) { + if (CollectionUtils.isEmpty(assignedEntities)) { + entityVertices.clear(); + } else { + CollectionUtils.filter(entityVertices, new Predicate() { + @Override + public boolean evaluate(Object o) { + if (o instanceof AtlasVertex) { + AtlasVertex entityVertex = (AtlasVertex) o; + + for (AtlasVertex assignedEntity : assignedEntities) { + if (assignedEntity.getId().equals(entityVertex.getId())) { + return true; + } + } + } + + return false; + } + }); + } + } + + super.filter(entityVertices); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== TermSearchProcessor.filter(): ret.size()={}", entityVertices.size()); + } + } +} http://git-wip-us.apache.org/repos/asf/atlas/blob/f622751d/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java index ee68d63..af309a1 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java @@ -290,7 +290,7 @@ public class DiscoveryREST { throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "TagFilters specified without tag name"); } - if (StringUtils.isEmpty(parameters.getTypeName()) && StringUtils.isEmpty(parameters.getClassification()) && StringUtils.isEmpty(parameters.getQuery())) { + if (StringUtils.isEmpty(parameters.getTypeName()) && StringUtils.isEmpty(parameters.getClassification()) && StringUtils.isEmpty(parameters.getQuery()) && StringUtils.isEmpty(parameters.getTermName())) { throw new AtlasBaseException(AtlasErrorCode.INVALID_SEARCH_PARAMS); }
