ATLAS-1961: Basic search improvement in use of Solr index for attribute filtering
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/377fe19f Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/377fe19f Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/377fe19f Branch: refs/heads/feature-odf Commit: 377fe19f023a4036f9fa1503dec6ea698fdb214b Parents: 71f9c06 Author: Madhan Neethiraj <mad...@apache.org> Authored: Tue Jul 18 00:25:36 2017 -0700 Committer: Madhan Neethiraj <mad...@apache.org> Committed: Tue Jul 18 08:05:53 2017 -0700 ---------------------------------------------------------------------- .../atlas/discovery/EntitySearchProcessor.java | 73 ++++++++++---------- .../apache/atlas/discovery/SearchContext.java | 2 +- .../apache/atlas/discovery/SearchProcessor.java | 14 +++- 3 files changed, 50 insertions(+), 39 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/atlas/blob/377fe19f/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java index 605cb15..3204ecf 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java @@ -20,7 +20,6 @@ package org.apache.atlas.discovery; import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria; import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.graphdb.*; -import org.apache.atlas.type.AtlasClassificationType; import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.utils.AtlasPerfTracer; import org.apache.commons.collections.CollectionUtils; @@ -34,60 +33,70 @@ public class EntitySearchProcessor extends SearchProcessor { private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("EntitySearchProcessor"); private final AtlasIndexQuery indexQuery; - private final AtlasGraphQuery partialGraphQuery; - private final AtlasGraphQuery allGraphQuery; + private final AtlasGraphQuery graphQuery; + private final AtlasGraphQuery filterGraphQuery; public EntitySearchProcessor(SearchContext context) { super(context); - AtlasEntityType entityType = context.getEntityType(); - AtlasClassificationType classificationType = context.getClassificationType(); - FilterCriteria filterCriteria = context.getSearchParameters().getEntityFilters(); - Set<String> typeAndSubTypes = entityType.getTypeAndAllSubTypes(); - Set<String> solrAttributes = new HashSet<>(); - Set<String> gremlinAttributes = new HashSet<>(); - Set<String> allAttributes = new HashSet<>(); + final AtlasEntityType entityType = context.getEntityType(); + final FilterCriteria filterCriteria = context.getSearchParameters().getEntityFilters(); + final Set<String> typeAndSubTypes = entityType.getTypeAndAllSubTypes(); + final Set<String> solrAttributes = new HashSet<>(); + final Set<String> gremlinAttributes = new HashSet<>(); + final Set<String> allAttributes = new HashSet<>(); processSearchAttributes(entityType, filterCriteria, solrAttributes, gremlinAttributes, allAttributes); - boolean useSolrSearch = typeAndSubTypes.size() <= MAX_ENTITY_TYPES_IN_INDEX_QUERY && canApplySolrFilter(entityType, filterCriteria, false); + final boolean typeSearchBySolr = typeAndSubTypes.size() <= MAX_ENTITY_TYPES_IN_INDEX_QUERY; + final boolean attrSearchBySolr = canApplySolrFilter(entityType, filterCriteria, false); - if (useSolrSearch) { - StringBuilder solrQuery = new StringBuilder(); + StringBuilder solrQuery = new StringBuilder(); + if (typeSearchBySolr) { constructTypeTestQuery(solrQuery, typeAndSubTypes); + } + + if (attrSearchBySolr) { constructFilterQuery(solrQuery, entityType, filterCriteria, solrAttributes); + } else { + gremlinAttributes.addAll(solrAttributes); + } + if (solrQuery.length() > 0) { String solrQueryString = STRAY_AND_PATTERN.matcher(solrQuery).replaceAll(")"); solrQueryString = STRAY_OR_PATTERN.matcher(solrQueryString).replaceAll(")"); solrQueryString = STRAY_ELIPSIS_PATTERN.matcher(solrQueryString).replaceAll(""); indexQuery = context.getGraph().indexQuery(Constants.VERTEX_INDEX, solrQueryString); + } else { + indexQuery = null; + } - if (CollectionUtils.isNotEmpty(gremlinAttributes) || classificationType != null) { - AtlasGraphQuery query = context.getGraph().query(); + if (CollectionUtils.isNotEmpty(gremlinAttributes) || !typeSearchBySolr) { + AtlasGraphQuery query = context.getGraph().query(); - addClassificationNameConditionIfNecessary(query); + if (!typeSearchBySolr) { + query.in(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes); + } + + graphQuery = toGremlinFilterQuery(entityType, filterCriteria, gremlinAttributes, query); - partialGraphQuery = toGremlinFilterQuery(entityType, filterCriteria, gremlinAttributes, query); - } else { - partialGraphQuery = null; + if (context.getSearchParameters().getExcludeDeletedEntities() && indexQuery == null) { + graphQuery.has(Constants.STATE_PROPERTY_KEY, "ACTIVE"); } } else { - indexQuery = null; - partialGraphQuery = null; + graphQuery = null; } AtlasGraphQuery query = context.getGraph().query().in(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes); - addClassificationNameConditionIfNecessary(query); - - allGraphQuery = toGremlinFilterQuery(entityType, filterCriteria, allAttributes, query); + filterGraphQuery = toGremlinFilterQuery(entityType, filterCriteria, allAttributes, query); if (context.getSearchParameters().getExcludeDeletedEntities()) { - allGraphQuery.has(Constants.STATE_PROPERTY_KEY, "ACTIVE"); + filterGraphQuery.has(Constants.STATE_PROPERTY_KEY, "ACTIVE"); } } @@ -128,15 +137,15 @@ public class EntitySearchProcessor extends SearchProcessor { vertices = getVerticesFromIndexQueryResult(queryResult); - if (partialGraphQuery != null) { + if (graphQuery != null) { AtlasGraphQuery guidQuery = context.getGraph().query().in(Constants.GUID_PROPERTY_KEY, getGuids(vertices)); - guidQuery.addConditionsFrom(partialGraphQuery); + guidQuery.addConditionsFrom(graphQuery); vertices = getVertices(guidQuery.vertices().iterator()); } } else { - Iterator<AtlasVertex> queryResult = allGraphQuery.vertices(qryOffset, limit).iterator(); + Iterator<AtlasVertex> queryResult = graphQuery.vertices(qryOffset, limit).iterator(); if (!queryResult.hasNext()) { // no more results from query - end of search break; @@ -182,7 +191,7 @@ public class EntitySearchProcessor extends SearchProcessor { AtlasGraphQuery query = context.getGraph().query().in(Constants.GUID_PROPERTY_KEY, getGuids(entityVertices)); - query.addConditionsFrom(allGraphQuery); + query.addConditionsFrom(filterGraphQuery); List<AtlasVertex> ret = getVertices(query.vertices().iterator()); @@ -194,10 +203,4 @@ public class EntitySearchProcessor extends SearchProcessor { return ret; } - - private void addClassificationNameConditionIfNecessary(AtlasGraphQuery query) { - if (context.getClassificationType() != null && !context.needClassificationProcessor()) { - query.in(Constants.TRAIT_NAMES_PROPERTY_KEY, context.getClassificationType().getTypeAndAllSubTypes()); - } - } } http://git-wip-us.apache.org/repos/asf/atlas/blob/377fe19f/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java index 55a07f3..8dd7667 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java +++ b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java @@ -109,7 +109,7 @@ public class SearchContext { } public boolean needClassificationProcessor() { - return classificationType != null && (hasAttributeFilter(searchParameters.getTagFilters()) || entityType == null); + return classificationType != null; } public boolean needEntityProcessor() { http://git-wip-us.apache.org/repos/asf/atlas/blob/377fe19f/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java index 1481809..ff0bd2e 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java +++ b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java @@ -88,7 +88,7 @@ public abstract class SearchProcessor { public abstract List<AtlasVertex> execute(); public List<AtlasVertex> filter(List<AtlasVertex> entityVertices) { - return nextProcessor == null ? entityVertices : nextProcessor.filter(entityVertices); + return nextProcessor == null || CollectionUtils.isEmpty(entityVertices) ? entityVertices : nextProcessor.filter(entityVertices); } @@ -193,12 +193,20 @@ public abstract class SearchProcessor { String filterQuery = toSolrQuery(type, filterCriteria, solrAttributes, 0); if (StringUtils.isNotEmpty(filterQuery)) { - solrQuery.append(AND_STR).append(filterQuery); + if (solrQuery.length() > 0) { + solrQuery.append(AND_STR); + } + + solrQuery.append(filterQuery); } } if (type instanceof AtlasEntityType && context.getSearchParameters().getExcludeDeletedEntities()) { - solrQuery.append(AND_STR).append("v.\"__state\":").append("ACTIVE"); + if (solrQuery.length() > 0) { + solrQuery.append(AND_STR); + } + + solrQuery.append("v.\"__state\":").append("ACTIVE"); } }
