Repository: incubator-atlas
Updated Branches:
  refs/heads/master 377fe19f0 -> cfb6b84f4


ATLAS-1961: Basic search improvement in use of Solr index for attribute 
filtering (# 2)


Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/cfb6b84f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/cfb6b84f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/cfb6b84f

Branch: refs/heads/master
Commit: cfb6b84f41b05275db826bbd43e4c39145b6d2d5
Parents: 377fe19
Author: Madhan Neethiraj <mad...@apache.org>
Authored: Tue Jul 18 13:27:46 2017 -0700
Committer: Madhan Neethiraj <mad...@apache.org>
Committed: Tue Jul 18 16:59:02 2017 -0700

----------------------------------------------------------------------
 .../atlas/discovery/EntitySearchProcessor.java  | 30 +++++++++++-----
 .../discovery/FullTextSearchProcessor.java      | 36 ++++++++++++++++++--
 .../apache/atlas/discovery/SearchContext.java   |  9 +++--
 .../apache/atlas/discovery/SearchProcessor.java |  4 +--
 4 files changed, 61 insertions(+), 18 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/cfb6b84f/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
----------------------------------------------------------------------
diff --git 
a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
 
b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
index 3204ecf..50376ef 100644
--- 
a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
+++ 
b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
@@ -20,6 +20,7 @@ package org.apache.atlas.discovery;
 import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
 import org.apache.atlas.repository.Constants;
 import org.apache.atlas.repository.graphdb.*;
+import org.apache.atlas.type.AtlasClassificationType;
 import org.apache.atlas.type.AtlasEntityType;
 import org.apache.atlas.utils.AtlasPerfTracer;
 import org.apache.commons.collections.CollectionUtils;
@@ -39,18 +40,21 @@ public class EntitySearchProcessor extends SearchProcessor {
     public EntitySearchProcessor(SearchContext context) {
         super(context);
 
-        final AtlasEntityType entityType         = context.getEntityType();
-        final FilterCriteria  filterCriteria     = 
context.getSearchParameters().getEntityFilters();
-        final Set<String>     typeAndSubTypes    = 
entityType.getTypeAndAllSubTypes();
-        final Set<String>     solrAttributes     = new HashSet<>();
-        final Set<String>     gremlinAttributes  = new HashSet<>();
-        final Set<String>     allAttributes      = new HashSet<>();
+        final AtlasEntityType entityType        = context.getEntityType();
+        final FilterCriteria  filterCriteria    = 
context.getSearchParameters().getEntityFilters();
+        final Set<String>     typeAndSubTypes   = 
entityType.getTypeAndAllSubTypes();
+        final Set<String>     solrAttributes    = new HashSet<>();
+        final Set<String>     gremlinAttributes = new HashSet<>();
+        final Set<String>     allAttributes     = new HashSet<>();
+
+        final AtlasClassificationType classificationType   = 
context.getClassificationType();
+        final boolean                 filterClassification = 
classificationType != null && !context.needClassificationProcessor();
 
 
         processSearchAttributes(entityType, filterCriteria, solrAttributes, 
gremlinAttributes, allAttributes);
 
-        final boolean typeSearchBySolr = typeAndSubTypes.size() <= 
MAX_ENTITY_TYPES_IN_INDEX_QUERY;
-        final boolean attrSearchBySolr = canApplySolrFilter(entityType, 
filterCriteria, false);
+        final boolean typeSearchBySolr = !filterClassification && 
typeAndSubTypes.size() <= MAX_ENTITY_TYPES_IN_INDEX_QUERY;
+        final boolean attrSearchBySolr = !filterClassification && 
CollectionUtils.isNotEmpty(solrAttributes) && canApplySolrFilter(entityType, 
filterCriteria, false);
 
         StringBuilder solrQuery = new StringBuilder();
 
@@ -82,6 +86,10 @@ public class EntitySearchProcessor extends SearchProcessor {
                 query.in(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes);
             }
 
+            if (filterClassification) {
+                query.in(Constants.TRAIT_NAMES_PROPERTY_KEY, 
classificationType.getTypeAndAllSubTypes());
+            }
+
             graphQuery = toGremlinFilterQuery(entityType, filterCriteria, 
gremlinAttributes, query);
 
             if (context.getSearchParameters().getExcludeDeletedEntities() && 
indexQuery == null) {
@@ -93,6 +101,10 @@ public class EntitySearchProcessor extends SearchProcessor {
 
         AtlasGraphQuery query = 
context.getGraph().query().in(Constants.TYPE_NAME_PROPERTY_KEY, 
typeAndSubTypes);
 
+        if (filterClassification) {
+            query.in(Constants.TRAIT_NAMES_PROPERTY_KEY, 
classificationType.getTypeAndAllSubTypes());
+        }
+
         filterGraphQuery = toGremlinFilterQuery(entityType, filterCriteria, 
allAttributes, query);
 
         if (context.getSearchParameters().getExcludeDeletedEntities()) {
@@ -115,7 +127,7 @@ public class EntitySearchProcessor extends SearchProcessor {
         }
 
         try {
-            int qryOffset = (nextProcessor == null) ? 
context.getSearchParameters().getOffset() : 0;
+            int qryOffset = (nextProcessor == null && (graphQuery == null || 
indexQuery == null)) ? context.getSearchParameters().getOffset() : 0;
             int limit     = context.getSearchParameters().getLimit();
             int resultIdx = qryOffset;
 

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/cfb6b84f/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
----------------------------------------------------------------------
diff --git 
a/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
 
b/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
index 4ddd642..83368c2 100644
--- 
a/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
+++ 
b/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
@@ -22,12 +22,14 @@ import org.apache.atlas.repository.Constants;
 import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
 import org.apache.atlas.repository.graphdb.AtlasVertex;
 import org.apache.atlas.utils.AtlasPerfTracer;
+import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Set;
 
 
 public class FullTextSearchProcessor extends SearchProcessor {
@@ -40,9 +42,39 @@ public class FullTextSearchProcessor extends SearchProcessor 
{
         super(context);
 
         SearchParameters searchParameters = context.getSearchParameters();
-        String           queryString      = String.format("v.\"%s\":(%s)", 
Constants.ENTITY_TEXT_PROPERTY_KEY, searchParameters.getQuery());
+        StringBuilder    queryString      = new StringBuilder();
 
-        indexQuery = context.getGraph().indexQuery(Constants.FULLTEXT_INDEX, 
queryString);
+        
queryString.append("v.\"").append(Constants.ENTITY_TEXT_PROPERTY_KEY).append("\":(").append(searchParameters.getQuery());
+
+        // if search includes entity-type criteria, adding a filter here can 
help avoid unnecessary
+        // processing (and rejection) by subsequent EntitySearchProcessor
+        if (context.getEntityType() != null) {
+            Set<String> typeAndSubTypeNames = 
context.getEntityType().getTypeAndAllSubTypes();
+
+            if (typeAndSubTypeNames.size() <= MAX_ENTITY_TYPES_IN_INDEX_QUERY) 
{
+                
queryString.append(AND_STR).append("(").append(StringUtils.join(typeAndSubTypeNames,
 SPACE_STRING)).append(")");
+            } else {
+                LOG.warn("'{}' has too many subtypes ({}) to include in 
index-query; might cause poor performance",
+                         context.getEntityType().getTypeName(), 
typeAndSubTypeNames.size());
+            }
+        }
+
+        // if search includes classification criteria, adding a filter here 
can help avoid unnecessary
+        // processing (and rejection) by subsequent 
ClassificationSearchProcessor or EntitySearchProcessor
+        if (context.getClassificationType() != null) {
+            Set<String> typeAndSubTypeNames = 
context.getClassificationType().getTypeAndAllSubTypes();
+
+            if (typeAndSubTypeNames.size() <= 
MAX_CLASSIFICATION_TYPES_IN_INDEX_QUERY) {
+                
queryString.append(AND_STR).append("(").append(StringUtils.join(typeAndSubTypeNames,
 SPACE_STRING)).append(")");
+            } else {
+                LOG.warn("'{}' has too many subtypes ({}) to include in 
index-query; might cause poor performance",
+                        context.getEntityType().getTypeName(), 
typeAndSubTypeNames.size());
+            }
+        }
+
+        queryString.append(")");
+
+        indexQuery = context.getGraph().indexQuery(Constants.FULLTEXT_INDEX, 
queryString.toString());
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/cfb6b84f/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
----------------------------------------------------------------------
diff --git 
a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java 
b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
index 8dd7667..929f8d0 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
@@ -61,7 +61,6 @@ public class SearchContext {
 
         if (needEntityProcessor()) {
             addProcessor(new EntitySearchProcessor(this));
-
         }
     }
 
@@ -104,15 +103,15 @@ public class SearchContext {
         return toString(new StringBuilder()).toString();
     }
 
-    public boolean needFullTextrocessor() {
+    boolean needFullTextrocessor() {
         return StringUtils.isNotEmpty(searchParameters.getQuery());
     }
 
-    public boolean needClassificationProcessor() {
-        return classificationType != null;
+    boolean needClassificationProcessor() {
+        return classificationType != null && (entityType == null || 
hasAttributeFilter(searchParameters.getTagFilters()));
     }
 
-    public boolean needEntityProcessor() {
+    boolean needEntityProcessor() {
         return entityType != null;
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/cfb6b84f/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
----------------------------------------------------------------------
diff --git 
a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java 
b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
index ff0bd2e..596b43b 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
@@ -181,7 +181,7 @@ public abstract class SearchProcessor {
     protected void constructTypeTestQuery(StringBuilder solrQuery, Set<String> 
typeAndAllSubTypes) {
         String typeAndSubtypesString = StringUtils.join(typeAndAllSubTypes, 
SPACE_STRING);
 
-        solrQuery.append("v.\"__typeName\": (")
+        
solrQuery.append("v.\"").append(Constants.TYPE_NAME_PROPERTY_KEY).append("\": 
(")
                 .append(typeAndSubtypesString)
                 .append(")");
     }
@@ -206,7 +206,7 @@ public abstract class SearchProcessor {
                 solrQuery.append(AND_STR);
             }
 
-            solrQuery.append("v.\"__state\":").append("ACTIVE");
+            
solrQuery.append("v.\"").append(Constants.STATE_PROPERTY_KEY).append("\":ACTIVE");
         }
     }
 

Reply via email to