This is an automated email from the ASF dual-hosted git repository. nixon pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/atlas.git
commit 61c2fe3ebeb7a71620bc9d0a231710d2b42aa800 Author: Pinal Shah <[email protected]> AuthorDate: Mon Mar 30 11:15:35 2020 +0530 ATLAS-3703 : BasicSearch: Search by terms/Search by text in combination of Classification filter, filter doesn't work Signed-off-by: nixonrodrigues <[email protected]> --- .../discovery/ClassificationSearchProcessor.java | 93 ++++++++++++++-------- .../apache/atlas/discovery/SearchProcessor.java | 9 ++- 2 files changed, 67 insertions(+), 35 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java index 1343c9b..74c088c 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java +++ b/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java @@ -42,13 +42,7 @@ import org.slf4j.LoggerFactory; import javax.script.ScriptEngine; import javax.script.ScriptException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFIED; import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_NOT_CLASSIFIED; @@ -109,13 +103,14 @@ public class ClassificationSearchProcessor extends SearchProcessor { /* If classification's attributes can be applied index filter, we can use direct index * to query classification index as well. */ - final boolean useIndexSearchForClassification = (classificationType != MATCH_ALL_WILDCARD_CLASSIFICATION && - classificationType != MATCH_ALL_CLASSIFIED && - classificationType != MATCH_ALL_NOT_CLASSIFIED && !isWildcardSearch) && + final boolean useIndexSearchForClassification = (!isBuiltInType && !isWildcardSearch) && (typeAndSubTypesQryStr.length() <= MAX_QUERY_STR_LENGTH_TAGS) && - CollectionUtils.isEmpty(graphAttributes) && + CollectionUtils.isNotEmpty(indexAttributes) && canApplyIndexFilter(classificationType, filterCriteria, false); + traitPredicate = buildTraitPredict(classificationType); + isEntityPredicate = SearchPredicateUtil.generateIsEntityVertexPredicate(context.getTypeRegistry()); + AtlasGraph graph = context.getGraph(); // index query directly on entity @@ -148,15 +143,8 @@ public class ClassificationSearchProcessor extends SearchProcessor { indexQuery = graph.indexQuery(Constants.VERTEX_INDEX, indexQueryString); LOG.debug("Using query string '{}'.", indexQuery); - - isEntityPredicate = SearchPredicateUtil.generateIsEntityVertexPredicate(context.getTypeRegistry()); - traitPredicate = buildTraitPredict(classificationType); - inMemoryPredicate = inMemoryPredicate == null ? traitPredicate : PredicateUtils.andPredicate(inMemoryPredicate, traitPredicate); - } else { - indexQuery = null; - traitPredicate = null; - isEntityPredicate = null; + indexQuery = null; } // index query directly on classification @@ -286,25 +274,27 @@ public class ClassificationSearchProcessor extends SearchProcessor { getVerticesFromIndexQueryResult(queryResult, entityVertices); isLastResultPage = entityVertices.size() < limit; - if (isEntityPredicate != null) { - // Do in-memory filtering - CollectionUtils.filter(entityVertices, isEntityPredicate); - } - } else { - if (classificationIndexQuery != null) { - Iterator<AtlasIndexQuery.Result> queryResult = classificationIndexQuery.vertices(qryOffset, limit); - - getVerticesFromIndexQueryResult(queryResult, classificationVertices); + // Do in-memory filtering + CollectionUtils.filter(entityVertices, traitPredicate); + CollectionUtils.filter(entityVertices, isEntityPredicate); - // Do in-memory filtering before the graph query - CollectionUtils.filter(classificationVertices, inMemoryPredicate); + } else { + if (tagGraphQueryWithAttributes != null) { - } else if (context.getSearchParameters().getTagFilters() != null) { Iterator<AtlasVertex> queryResult = tagGraphQueryWithAttributes.vertices(qryOffset, limit).iterator(); getVertices(queryResult, classificationVertices); isLastResultPage = classificationVertices.size() < limit; + + } else if (classificationIndexQuery != null){ + + Iterator<AtlasIndexQuery.Result> queryResult = classificationIndexQuery.vertices(qryOffset, limit); + + getVerticesFromIndexQueryResult(queryResult, classificationVertices); + + // Do in-memory filtering before the graph query + CollectionUtils.filter(classificationVertices, inMemoryPredicate); } } @@ -333,6 +323,8 @@ public class ClassificationSearchProcessor extends SearchProcessor { if (whiteSpaceFilter) { filterWhiteSpaceClassification(entityVertices); } + // Do in-memory filtering + CollectionUtils.filter(entityVertices, isEntityPredicate); super.filter(entityVertices); @@ -358,7 +350,7 @@ public class ClassificationSearchProcessor extends SearchProcessor { if (LOG.isDebugEnabled()) { LOG.debug("==> ClassificationSearchProcessor.filter({})", entityVertices.size()); } - + //in case of classification type + graph attributes if (gremlinTagFilterQuery != null && gremlinQueryBindings != null) { // Now filter on the tag attributes Set<String> guids = getGuids(entityVertices); @@ -381,8 +373,43 @@ public class ClassificationSearchProcessor extends SearchProcessor { LOG.warn(e.getMessage(), e); } } - } else if (traitPredicate != null) { + } else if (inMemoryPredicate != null) { + //in case of classification type + index attributes + CollectionUtils.filter(entityVertices, traitPredicate); + + //filter attributes (filterCriteria). Find classification vertex(typeName = classification) from entity vertex (traitName = classification) + final Set<String> processedGuids = new HashSet<>(); + List<AtlasVertex> matchEntityVertices = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(entityVertices)) { + for (AtlasVertex entityVertex : entityVertices) { + Iterable<AtlasEdge> edges = entityVertex.getEdges(AtlasEdgeDirection.OUT, Constants.CLASSIFICATION_LABEL); + + for (AtlasEdge edge : edges) { + AtlasVertex classificationVertex = edge.getInVertex(); + + AtlasVertex matchVertex = (AtlasVertex) CollectionUtils.find(Collections.singleton(classificationVertex), inMemoryPredicate); + if (matchVertex != null) { + String guid = AtlasGraphUtilsV2.getIdFromVertex(entityVertex); + + if (processedGuids.contains(guid)) { + continue; + } + + matchEntityVertices.add(entityVertex); + processedGuids.add(guid); + break; + + } + } + } + } + entityVertices.clear(); + entityVertices.addAll(matchEntityVertices); + + } else { + //in case of only classsification type CollectionUtils.filter(entityVertices, traitPredicate); + CollectionUtils.filter(entityVertices, isEntityPredicate); } super.filter(entityVertices); diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java index ddd3e60..074ad8b 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java +++ b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java @@ -79,7 +79,7 @@ public abstract class SearchProcessor { public static final char CUSTOM_ATTR_SEPARATOR = '='; public static final String CUSTOM_ATTR_SEARCH_FORMAT = "\"\\\"%s\\\":\\\"%s\\\"\""; public static final String CUSTOM_ATTR_SEARCH_FORMAT_GRAPH = "\"%s\":\"%s\""; - + public static final String WILDCARD_CHAR = "*"; private static final Map<SearchParameters.Operator, String> OPERATOR_MAP = new HashMap<>(); private static final Map<SearchParameters.Operator, VertexAttributePredicateGenerator> OPERATOR_PREDICATE_MAP = new HashMap<>(); @@ -195,12 +195,17 @@ public abstract class SearchProcessor { protected Predicate buildTraitPredict(AtlasClassificationType classificationType) { Predicate traitPredicate; - if (classificationType == MATCH_ALL_WILDCARD_CLASSIFICATION || classificationType == MATCH_ALL_CLASSIFIED || context.isWildCardSearch() || classificationType == MATCH_ALL_CLASSIFICATION_TYPES) { + if (classificationType == MATCH_ALL_WILDCARD_CLASSIFICATION || classificationType == MATCH_ALL_CLASSIFIED || classificationType == MATCH_ALL_CLASSIFICATION_TYPES) { traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, null, List.class), SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class)); } else if (classificationType == MATCH_ALL_NOT_CLASSIFIED) { traitPredicate = PredicateUtils.andPredicate(SearchPredicateUtil.getIsNullOrEmptyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, null, List.class), SearchPredicateUtil.getIsNullOrEmptyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class)); + } else if (context.isWildCardSearch()) { + String wildcardName = context.getClassificationName(); + wildcardName = wildcardName.replace(WILDCARD_CHAR,""); + traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getContainsPredicateGenerator().generatePredicate(CLASSIFICATION_NAMES_KEY, wildcardName, String.class), + SearchPredicateUtil.getContainsPredicateGenerator().generatePredicate(PROPAGATED_CLASSIFICATION_NAMES_KEY, wildcardName, String.class)); } else { traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, context.getClassificationTypes(), List.class), SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, context.getClassificationTypes(), List.class));
