This is an automated email from the ASF dual-hosted git repository. nixon pushed a commit to branch branch-2.0 in repository https://gitbox.apache.org/repos/asf/atlas.git
commit 01d8c3abffd053024ed530a4686812fe2de5777f Author: Pinal Shah <[email protected]> AuthorDate: Fri Jun 26 11:24:46 2020 +0530 ATLAS-3838: Support multiple tag/classification in basic/quick search API ATLAS-3652: Quick Search: API requirement for GET request on multiple entity types Signed-off-by: Sarath Subramanian <[email protected]> both JIRA's addressed in the same commit. (cherry picked from commit 8b5cb9d9aa9f977ba5dfc455dfe440dd55eace06) --- .../repository/graphdb/AggregationContext.java | 38 +-- .../graphdb/janus/AtlasJanusGraphIndexClient.java | 2 +- .../graphdb/janus/AtlasSolrQueryBuilder.java | 151 +++++---- .../graphdb/janus/AtlasSolrQueryBuilderTest.java | 54 +++- .../src/test/resources/searchparameters2Types.json | 19 ++ .../discovery/ClassificationSearchProcessor.java | 34 +- .../atlas/discovery/EntityDiscoveryService.java | 8 +- .../atlas/discovery/EntitySearchProcessor.java | 53 ++-- .../atlas/discovery/FreeTextSearchProcessor.java | 7 +- .../atlas/discovery/FullTextSearchProcessor.java | 19 +- .../atlas/discovery/GraphIndexQueryBuilder.java | 25 +- .../atlas/discovery/SearchAggregatorImpl.java | 7 +- .../org/apache/atlas/discovery/SearchContext.java | 253 +++++++++++---- .../apache/atlas/discovery/SearchProcessor.java | 343 +++++++++++++-------- .../atlas/discovery/EntitySearchProcessorTest.java | 114 ++++++- 15 files changed, 750 insertions(+), 377 deletions(-) diff --git a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AggregationContext.java b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AggregationContext.java index 6006fef..efde79b 100644 --- a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AggregationContext.java +++ b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AggregationContext.java @@ -25,14 +25,14 @@ import java.util.Map; import java.util.Set; public class AggregationContext { - private final String queryString; - private final FilterCriteria filterCriteria; - private final AtlasEntityType searchForEntityType; - private final Set<String> aggregationFieldNames; - private final Set<AtlasAttribute> aggregationAttributes; - private final Map<String, String> indexFieldNameCache; - private final boolean excludeDeletedEntities; - private final boolean includeSubTypes; + private final String queryString; + private final FilterCriteria filterCriteria; + private final Set<AtlasEntityType> searchForEntityTypes; + private final Set<String> aggregationFieldNames; + private final Set<AtlasAttribute> aggregationAttributes; + private final Map<String, String> indexFieldNameCache; + private final boolean excludeDeletedEntities; + private final boolean includeSubTypes; /** * @param queryString the query string whose aggregation metrics need to be retrieved. @@ -41,17 +41,17 @@ public class AggregationContext { * @param indexFieldNameCache * @param excludeDeletedEntities a boolean flag to indicate if the deleted entities need to be excluded in search */ - public AggregationContext(String queryString, - FilterCriteria filterCriteria, - AtlasEntityType searchForEntityType, - Set<String> aggregationFieldNames, - Set<AtlasAttribute> aggregationAttributes, - Map<String, String> indexFieldNameCache, - boolean excludeDeletedEntities, - boolean includeSubTypes) { + public AggregationContext(String queryString, + FilterCriteria filterCriteria, + Set<AtlasEntityType> searchForEntityType, + Set<String> aggregationFieldNames, + Set<AtlasAttribute> aggregationAttributes, + Map<String, String> indexFieldNameCache, + boolean excludeDeletedEntities, + boolean includeSubTypes) { this.queryString = queryString; this.filterCriteria = filterCriteria; - this.searchForEntityType = searchForEntityType; + this.searchForEntityTypes = searchForEntityType; this.aggregationFieldNames = aggregationFieldNames; this.aggregationAttributes = aggregationAttributes; this.indexFieldNameCache = indexFieldNameCache; @@ -67,8 +67,8 @@ public class AggregationContext { return filterCriteria; } - public AtlasEntityType getSearchForEntityType() { - return searchForEntityType; + public Set<AtlasEntityType> getSearchForEntityTypes() { + return searchForEntityTypes; } public Set<String> getAggregationFieldNames() { diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphIndexClient.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphIndexClient.java index 29bd2a4..8142514 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphIndexClient.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphIndexClient.java @@ -175,7 +175,7 @@ public class AtlasJanusGraphIndexClient implements AtlasGraphIndexClient { Map<String, String> indexFieldName2PropertyKeyNameMap = new HashMap<>(); AtlasSolrQueryBuilder solrQueryBuilder = new AtlasSolrQueryBuilder(); - solrQueryBuilder.withEntityType(aggregationContext.getSearchForEntityType()) + solrQueryBuilder.withEntityTypes(aggregationContext.getSearchForEntityTypes()) .withQueryString(aggregationContext.getQueryString()) .withCriteria(aggregationContext.getFilterCriteria()) .withExcludedDeletedEntities(aggregationContext.isExcludeDeletedEntities()) diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilder.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilder.java index b8fd4a0..6c06a3c 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilder.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilder.java @@ -29,31 +29,28 @@ import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import static org.apache.atlas.repository.Constants.CUSTOM_ATTRIBUTES_PROPERTY_KEY; public class AtlasSolrQueryBuilder { private static final Logger LOG = LoggerFactory.getLogger(AtlasSolrQueryBuilder.class); - private AtlasEntityType entityType; - private String queryString; - private FilterCriteria criteria; - private boolean excludeDeletedEntities; - private boolean includeSubtypes; - private Map<String, String> indexFieldNameCache; - public static final char CUSTOM_ATTR_SEPARATOR = '='; - public static final String CUSTOM_ATTR_SEARCH_FORMAT = "\"\\\"%s\\\":\\\"%s\\\"\""; + private Set<AtlasEntityType> entityTypes; + private String queryString; + private FilterCriteria criteria; + private boolean excludeDeletedEntities; + private boolean includeSubtypes; + private Map<String, String> indexFieldNameCache; + public static final char CUSTOM_ATTR_SEPARATOR = '='; + public static final String CUSTOM_ATTR_SEARCH_FORMAT = "\"\\\"%s\\\":\\\"%s\\\"\""; public AtlasSolrQueryBuilder() { } - public AtlasSolrQueryBuilder withEntityType(AtlasEntityType searchForEntityType) { - this.entityType = searchForEntityType; + public AtlasSolrQueryBuilder withEntityTypes(Set<AtlasEntityType> searchForEntityTypes) { + this.entityTypes = searchForEntityTypes; return this; } @@ -112,7 +109,7 @@ public class AtlasSolrQueryBuilder { isAndNeeded = true; } - if (entityType != null) { + if (CollectionUtils.isNotEmpty(entityTypes)) { if (isAndNeeded) { queryBuilder.append(" AND "); } @@ -140,29 +137,25 @@ public class AtlasSolrQueryBuilder { } private void buildForEntityType(StringBuilder queryBuilder) { - if (LOG.isDebugEnabled()) { - LOG.debug("Search is being done for entities of type {}", entityType.getTypeName()); - } String typeIndexFieldName = indexFieldNameCache.get(Constants.ENTITY_TYPE_PROPERTY_KEY); queryBuilder.append(" +") .append(typeIndexFieldName) - .append(":(") - .append(entityType.getTypeName()) - .append(" "); + .append(":("); - if (includeSubtypes) { - Set<String> allSubTypes = entityType.getAllSubTypes(); + Set<String> typesToSearch = new HashSet<>(); + for (AtlasEntityType type : entityTypes) { - if(allSubTypes.size() != 0 ) { - for(String subTypeName: allSubTypes) { - queryBuilder.append(subTypeName).append(" "); - } + if (includeSubtypes) { + typesToSearch.addAll(type.getTypeAndAllSubTypes()); + } else { + typesToSearch.add(type.getTypeName()); } } - queryBuilder.append(" ) "); + queryBuilder.append(StringUtils.join(typesToSearch, " ")).append(" ) "); + } private void dropDeletedEntities(StringBuilder queryBuilder) throws AtlasBaseException { @@ -173,9 +166,8 @@ public class AtlasSolrQueryBuilder { String indexFieldName = indexFieldNameCache.get(Constants.STATE_PROPERTY_KEY); if (indexFieldName == null) { - String msg = String.format("There is no index field name defined for attribute '%s' for entity '%s'", - Constants.STATE_PROPERTY_KEY, - entityType.getTypeName()); + String msg = String.format("There is no index field name defined for attribute '%s'", + Constants.STATE_PROPERTY_KEY); LOG.error(msg); @@ -187,10 +179,46 @@ public class AtlasSolrQueryBuilder { private AtlasSolrQueryBuilder withCriteria(StringBuilder queryBuilder, FilterCriteria criteria) throws AtlasBaseException { List<FilterCriteria> criterion = criteria.getCriterion(); + Set<String> indexAttributes = new HashSet<>(); + if (StringUtils.isNotEmpty(criteria.getAttributeName()) && CollectionUtils.isEmpty(criterion)) { // no child criterion + + String attributeName = criteria.getAttributeName(); + String attributeValue = criteria.getAttributeValue(); + Operator operator = criteria.getOperator(); + + ArrayList<StringBuilder> orExpQuery = new ArrayList<>(); + + for (AtlasEntityType type : entityTypes) { + String indexAttributeName = getIndexAttributeName(type, attributeName); + + //check to remove duplicate attribute query (for eg. name) + if (!indexAttributes.contains(indexAttributeName)) { + StringBuilder sb = new StringBuilder(); + + if (attributeName.equals(CUSTOM_ATTRIBUTES_PROPERTY_KEY) && operator.equals(Operator.CONTAINS)) { + // CustomAttributes stores key value pairs in String format, so ideally it should be 'contains' operator to search for one pair, + // for use-case, E1 having key1=value1 and E2 having key1=value2, searching key1=value1 results both E1,E2 + // surrounding inverted commas to attributeValue works + operator = Operator.EQ; + attributeValue = getIndexQueryAttributeValue(attributeValue); + } + + withPropertyCondition(sb, indexAttributeName, operator, attributeValue); + indexAttributes.add(indexAttributeName); + orExpQuery.add(sb); + } + } + + if (CollectionUtils.isNotEmpty(orExpQuery)) { + if (orExpQuery.size() > 1) { + String orExpStr = StringUtils.join(orExpQuery, FilterCriteria.Condition.OR.name()); + queryBuilder.append(" ( ").append(orExpStr).append(" ) "); + } else { + queryBuilder.append(orExpQuery.iterator().next()); + } + } - if(criterion == null || CollectionUtils.isEmpty(criteria.getCriterion())) { // no child criterion - withPropertyCondition(queryBuilder, criteria.getAttributeName(), criteria.getOperator(), criteria.getAttributeValue()); - } else { + } else if (CollectionUtils.isNotEmpty(criterion)) { beginCriteria(queryBuilder); for (Iterator<FilterCriteria> iterator = criterion.iterator(); iterator.hasNext(); ) { @@ -209,40 +237,12 @@ public class AtlasSolrQueryBuilder { return this; } - private void withPropertyCondition(StringBuilder queryBuilder, String attributeName, Operator operator, String attributeValue) throws AtlasBaseException { - if (StringUtils.isNotEmpty(attributeName) && operator != null) { + private void withPropertyCondition(StringBuilder queryBuilder, String indexFieldName, Operator operator, String attributeValue) throws AtlasBaseException { + if (StringUtils.isNotEmpty(indexFieldName) && operator != null) { if (attributeValue != null) { attributeValue = attributeValue.trim(); } - if (attributeName.equals(CUSTOM_ATTRIBUTES_PROPERTY_KEY) && operator.equals(Operator.CONTAINS)) { - // CustomAttributes stores key value pairs in String format, so ideally it should be 'contains' operator to search for one pair, - // for use-case, E1 having key1=value1 and E2 having key1=value2, searching key1=value1 results both E1,E2 - // surrounding inverted commas to attributeValue works - operator = Operator.EQ; - attributeValue = getIndexQueryAttributeValue(attributeValue); - } - - AtlasAttribute attribute = entityType.getAttribute(attributeName); - - if (attribute == null) { - String msg = String.format("Received unknown attribute '%s' for type '%s'.", attributeName, entityType.getTypeName()); - - LOG.error(msg); - - throw new AtlasBaseException(msg); - } - - String indexFieldName = attribute.getIndexFieldName(); - - if (indexFieldName == null) { - String msg = String.format("Received non-index attribute %s for type %s.", attributeName, entityType.getTypeName()); - - LOG.error(msg); - - throw new AtlasBaseException(msg); - } - beginCriteria(queryBuilder); switch (operator) { @@ -308,6 +308,29 @@ public class AtlasSolrQueryBuilder { return attributeValue; } + private String getIndexAttributeName(AtlasEntityType type, String attrName) throws AtlasBaseException { + AtlasAttribute ret = type.getAttribute(attrName); + + if (ret == null) { + String msg = String.format("Received unknown attribute '%s' for type '%s'.", attrName, type.getTypeName()); + + LOG.error(msg); + + throw new AtlasBaseException(msg); + } + + String indexFieldName = ret.getIndexFieldName(); + + if (indexFieldName == null) { + String msg = String.format("Received non-index attribute %s for type %s.", attrName, type.getTypeName()); + + LOG.error(msg); + + throw new AtlasBaseException(msg); + } + + return indexFieldName; + } private void beginCriteria(StringBuilder queryBuilder) { queryBuilder.append("( "); diff --git a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilderTest.java b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilderTest.java index 60d8610..06c7221 100644 --- a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilderTest.java +++ b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilderTest.java @@ -33,8 +33,10 @@ import org.testng.annotations.Test; import java.io.FileInputStream; import java.io.IOException; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import static org.mockito.Mockito.when; @@ -45,6 +47,9 @@ public class AtlasSolrQueryBuilderTest { private AtlasEntityType hiveTableEntityTypeMock; @Mock + private AtlasEntityType hiveTableEntityTypeMock2; + + @Mock private AtlasStructType.AtlasAttribute nameAttributeMock; @Mock @@ -95,6 +100,7 @@ public class AtlasSolrQueryBuilderTest { when(hiveTableEntityTypeMock.getTypeName()).thenReturn("hive_table"); + when(hiveTableEntityTypeMock2.getTypeName()).thenReturn("hive_db"); when(nameAttributeMock.getIndexFieldName()).thenReturn("name_index"); when(commentAttributeMock.getIndexFieldName()).thenReturn("comment_index"); @@ -114,7 +120,7 @@ public class AtlasSolrQueryBuilderTest { processSearchParameters(fileName, underTest); - Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +name_index:t10 ) OR ( +comment_index:*t10* ) )"); + Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +name_index:t10 ) OR ( +comment_index:*t10* ) )"); } @Test @@ -124,7 +130,7 @@ public class AtlasSolrQueryBuilderTest { processSearchParameters(fileName, underTest); - Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +name_index:t10 ) )"); + Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +name_index:t10 ) )"); } @Test @@ -134,7 +140,7 @@ public class AtlasSolrQueryBuilderTest { processSearchParameters(fileName, underTest); - Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +name_index:t10 ) AND ( +comment_index:*t10* ) )"); + Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +name_index:t10 ) AND ( +comment_index:*t10* ) )"); } @Test @@ -144,7 +150,7 @@ public class AtlasSolrQueryBuilderTest { processSearchParameters(fileName, underTest); - Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +name_index:t10 ) )"); + Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +name_index:t10 ) )"); } @Test @@ -154,7 +160,7 @@ public class AtlasSolrQueryBuilderTest { processSearchParameters(fileName, underTest); - Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( +name_index:t10 )"); + Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( +name_index:t10 )"); } @Test @@ -164,7 +170,7 @@ public class AtlasSolrQueryBuilderTest { processSearchParameters(fileName, underTest); - Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +comment_index:*United States* ) AND ( +descrption__index:*nothing* ) AND ( +name_index:*t100* ) )"); + Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +comment_index:*United States* ) AND ( +descrption__index:*nothing* ) AND ( +name_index:*t100* ) )"); } @Test @@ -174,7 +180,7 @@ public class AtlasSolrQueryBuilderTest { processSearchParameters(fileName, underTest); - Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +created__index:{ 100 TO * ] ) )"); + Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +created__index:{ 100 TO * ] ) )"); } @Test @@ -184,7 +190,7 @@ public class AtlasSolrQueryBuilderTest { processSearchParameters(fileName, underTest); - Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +created__index:[ 100 TO * ] ) AND ( +started__index:[ 100 TO * ] ) )"); + Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +created__index:[ 100 TO * ] ) AND ( +started__index:[ 100 TO * ] ) )"); } @Test @@ -194,7 +200,7 @@ public class AtlasSolrQueryBuilderTest { processSearchParameters(fileName, underTest); - Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +created__index:[ * TO100} ) )"); + Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +created__index:[ * TO100} ) )"); } @Test @@ -204,7 +210,7 @@ public class AtlasSolrQueryBuilderTest { processSearchParameters(fileName, underTest); - Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +created__index:[ * TO 100 ] ) AND ( +started__index:[ * TO 100 ] ) )"); + Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +created__index:[ * TO 100 ] ) AND ( +started__index:[ * TO 100 ] ) )"); } @Test @@ -214,9 +220,18 @@ public class AtlasSolrQueryBuilderTest { processSearchParameters(fileName, underTest); - Assert.assertEquals(underTest.build(), " -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +qualifiedName__index:testdb.t1* ) )"); + Assert.assertEquals(underTest.build(), " -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +qualifiedName__index:testdb.t1* ) )"); } + @Test + public void testGenerateSolrQueryString2TypeNames() throws IOException, AtlasBaseException { + final String fileName = "src/test/resources/searchparameters2Types.json"; + AtlasSolrQueryBuilder underTest = new AtlasSolrQueryBuilder(); + + processSearchParametersForMultipleTypeNames(fileName, underTest); + + Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table hive_db ) "); + } @@ -243,8 +258,23 @@ public class AtlasSolrQueryBuilderTest { ObjectMapper mapper = new ObjectMapper(); SearchParameters searchParameters = mapper.readValue(new FileInputStream(fileName), SearchParameters.class); + Set<AtlasEntityType> hiveTableEntityTypeMocks = new HashSet<>(); + hiveTableEntityTypeMocks.add(hiveTableEntityTypeMock); + underTest.withEntityTypes(hiveTableEntityTypeMocks) + .withQueryString(searchParameters.getQuery()) + .withCriteria(searchParameters.getEntityFilters()) + .withExcludedDeletedEntities(searchParameters.getExcludeDeletedEntities()) + .withCommonIndexFieldNames(indexFieldNamesMap); + } + + private void processSearchParametersForMultipleTypeNames(String fileName, AtlasSolrQueryBuilder underTest) throws IOException, AtlasBaseException { + ObjectMapper mapper = new ObjectMapper(); + SearchParameters searchParameters = mapper.readValue(new FileInputStream(fileName), SearchParameters.class); - underTest.withEntityType(hiveTableEntityTypeMock) + Set<AtlasEntityType> hiveTableEntityTypeMocks = new HashSet<>(); + hiveTableEntityTypeMocks.add(hiveTableEntityTypeMock); + hiveTableEntityTypeMocks.add(hiveTableEntityTypeMock2); + underTest.withEntityTypes(hiveTableEntityTypeMocks) .withQueryString(searchParameters.getQuery()) .withCriteria(searchParameters.getEntityFilters()) .withExcludedDeletedEntities(searchParameters.getExcludeDeletedEntities()) diff --git a/graphdb/janus/src/test/resources/searchparameters2Types.json b/graphdb/janus/src/test/resources/searchparameters2Types.json new file mode 100644 index 0000000..77fc70f --- /dev/null +++ b/graphdb/janus/src/test/resources/searchparameters2Types.json @@ -0,0 +1,19 @@ + +{ + "excludeDeletedEntities":true, + "includeSubClassifications":true, + "includeSubTypes":true, + "includeClassificationAttributes":true, + "entityFilters":{ + }, + "tagFilters":null, + "attributes":[ + "comment" + ], + "query":"t", + "limit":25, + "offset":0, + "typeName":"hive_table,hive_db", + "classification":null, + "termName":null +} diff --git a/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java index 5dd0d7f..9c72cd4 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java +++ b/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java @@ -67,17 +67,16 @@ public class ClassificationSearchProcessor extends SearchProcessor { public ClassificationSearchProcessor(SearchContext context) { super(context); - final AtlasClassificationType classificationType = context.getClassificationType(); final FilterCriteria filterCriteria = context.getSearchParameters().getTagFilters(); final Set<String> indexAttributes = new HashSet<>(); final Set<String> graphAttributes = new HashSet<>(); final Set<String> allAttributes = new HashSet<>(); - final Set<String> typeAndSubTypes = context.getClassificationTypes(); + final Set<String> typeAndSubTypes = context.getClassificationTypeNames(); final String typeAndSubTypesQryStr = context.getClassificationTypesQryStr(); - final boolean isBuiltInType = context.isBuiltInClassificationType(); final boolean isWildcardSearch = context.isWildCardSearch(); + final Set<AtlasClassificationType> classificationTypes = context.getClassificationTypes(); - processSearchAttributes(classificationType, filterCriteria, indexAttributes, graphAttributes, allAttributes); + processSearchAttributes(classificationTypes, filterCriteria, indexAttributes, graphAttributes, allAttributes); /* for classification search, if any attribute can't be handled by index query - switch to all filter by Graph query There are four cases in the classification type : @@ -87,19 +86,25 @@ public class ClassificationSearchProcessor extends SearchProcessor { 4. classification is not present in the search parameter each of above cases with either has empty/or not tagFilters */ - final boolean useIndexSearchForEntity = (classificationType != null || isWildcardSearch) && + final boolean useIndexSearchForEntity = (CollectionUtils.isNotEmpty(classificationTypes) || isWildcardSearch) && !context.hasAttributeFilter(filterCriteria) && (typeAndSubTypesQryStr.length() <= MAX_QUERY_STR_LENGTH_TAGS); /* If classification's attributes can be applied index filter, we can use direct index * to query classification index as well. */ - final boolean useIndexSearchForClassification = (!isBuiltInType && !isWildcardSearch) && + final boolean useIndexSearchForClassification = (CollectionUtils.isNotEmpty(classificationTypes) && + classificationTypes.iterator().next() != SearchContext.MATCH_ALL_NOT_CLASSIFIED && + !isWildcardSearch) && (typeAndSubTypesQryStr.length() <= MAX_QUERY_STR_LENGTH_TAGS) && CollectionUtils.isNotEmpty(indexAttributes) && - canApplyIndexFilter(classificationType, filterCriteria, false); + canApplyIndexFilter(classificationTypes, filterCriteria, false); - traitPredicate = buildTraitPredict(classificationType); + final boolean useGraphSearchForClassification = (CollectionUtils.isNotEmpty(classificationTypes) && + classificationTypes.iterator().next() != SearchContext.MATCH_ALL_NOT_CLASSIFIED && + !isWildcardSearch && CollectionUtils.isNotEmpty(graphAttributes)); + + traitPredicate = buildTraitPredict(classificationTypes); isEntityPredicate = SearchPredicateUtil.generateIsEntityVertexPredicate(context.getTypeRegistry()); AtlasGraph graph = context.getGraph(); @@ -115,8 +120,7 @@ public class ClassificationSearchProcessor extends SearchProcessor { // tagFilters is not allowed in wildcard search graphIndexQueryBuilder.addClassificationTypeFilter(queryString); } else { - if (isBuiltInType) { - + if (classificationTypes.iterator().next() == SearchContext.MATCH_ALL_NOT_CLASSIFIED) { // tagFilters is not allowed in unique classificationType search graphIndexQueryBuilder.addClassificationFilterForBuiltInTypes(queryString); @@ -146,7 +150,7 @@ public class ClassificationSearchProcessor extends SearchProcessor { graphIndexQueryBuilder.addActiveStateQueryFilter(queryString); graphIndexQueryBuilder.addTypeAndSubTypesQueryFilter(queryString, typeAndSubTypesQryStr); - constructFilterQuery(queryString, classificationType, filterCriteria, indexAttributes); + constructFilterQuery(queryString, classificationTypes, filterCriteria, indexAttributes); String indexQueryString = STRAY_AND_PATTERN.matcher(queryString).replaceAll(")"); indexQueryString = STRAY_OR_PATTERN.matcher(indexQueryString).replaceAll(")"); @@ -158,7 +162,7 @@ public class ClassificationSearchProcessor extends SearchProcessor { inMemoryPredicate = inMemoryPredicate == null ? typeNamePredicate : PredicateUtils.andPredicate(inMemoryPredicate, typeNamePredicate); } - Predicate attributePredicate = constructInMemoryPredicate(classificationType, filterCriteria, indexAttributes); + Predicate attributePredicate = constructInMemoryPredicate(classificationTypes, filterCriteria, indexAttributes); if (attributePredicate != null) { inMemoryPredicate = inMemoryPredicate == null ? attributePredicate : PredicateUtils.andPredicate(inMemoryPredicate, attributePredicate); @@ -170,7 +174,7 @@ public class ClassificationSearchProcessor extends SearchProcessor { } // only registered classification will search with tag filters - if (!isWildcardSearch && !isBuiltInType && !graphAttributes.isEmpty()) { + if (useGraphSearchForClassification) { AtlasGremlinQueryProvider queryProvider = AtlasGremlinQueryProvider.INSTANCE; AtlasGraphQuery query = graph.query(); @@ -179,7 +183,7 @@ public class ClassificationSearchProcessor extends SearchProcessor { query.in(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes); } - tagGraphQueryWithAttributes = toGraphFilterQuery(classificationType, filterCriteria, allAttributes, query); + tagGraphQueryWithAttributes = toGraphFilterQuery(classificationTypes, filterCriteria, allAttributes, query); gremlinQueryBindings = new HashMap<>(); StringBuilder gremlinQuery = new StringBuilder(); @@ -188,7 +192,7 @@ public class ClassificationSearchProcessor extends SearchProcessor { gremlinQuery.append(".as('e').filter(out()"); gremlinQuery.append(queryProvider.getQuery(AtlasGremlinQueryProvider.AtlasGremlinQuery.BASIC_SEARCH_TYPE_FILTER)); - constructGremlinFilterQuery(gremlinQuery, gremlinQueryBindings, context.getClassificationType(), context.getSearchParameters().getTagFilters()); + // constructGremlinFilterQuery(gremlinQuery, gremlinQueryBindings, context.getClassificationType(), context.getSearchParameters().getTagFilters()); // After filtering on tags go back to e and output the list of entity vertices gremlinQuery.append(").toList()"); diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java index 37eef90..f814345 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java @@ -491,9 +491,11 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { resultAttributes.addAll(searchContext.getEntityAttributes()); } - AtlasEntityType entityType = searchContext.getEntityType(); - if (entityType != null) { - for (String resultAttribute : resultAttributes) { + if (CollectionUtils.isNotEmpty(searchContext.getEntityTypes())) { + + AtlasEntityType entityType = searchContext.getEntityTypes().iterator().next(); + + for (String resultAttribute : resultAttributes) { AtlasAttribute attribute = entityType.getAttribute(resultAttribute); if (attribute == null) { diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java index 56956e6..5dcff3b 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java @@ -31,27 +31,15 @@ import org.apache.atlas.utils.AtlasPerfTracer; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.Predicate; import org.apache.commons.collections.PredicateUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.tinkerpop.gremlin.process.traversal.Order; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.Set; +import java.util.*; import java.util.stream.StreamSupport; import static org.apache.atlas.SortOrder.ASCENDING; -import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFICATION_TYPES; -import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFIED; -import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_NOT_CLASSIFIED; -import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_WILDCARD_CLASSIFICATION; -import static org.apache.atlas.repository.Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY; -import static org.apache.atlas.repository.Constants.TRAIT_NAMES_PROPERTY_KEY; -import static org.apache.atlas.repository.Constants.TYPE_NAME_PROPERTY_KEY; +import static org.apache.atlas.discovery.SearchContext.*; +import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator.EQUAL; import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator.NOT_EQUAL; import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.SortOrder.ASC; @@ -69,28 +57,28 @@ public class EntitySearchProcessor extends SearchProcessor { public EntitySearchProcessor(SearchContext context) { super(context); - final AtlasEntityType entityType = context.getEntityType(); + final Set<AtlasEntityType> entityTypes = context.getEntityTypes(); final FilterCriteria filterCriteria = context.getSearchParameters().getEntityFilters(); final Set<String> indexAttributes = new HashSet<>(); final Set<String> graphAttributes = new HashSet<>(); final Set<String> allAttributes = new HashSet<>(); - final Set<String> typeAndSubTypes = context.getEntityTypes(); + final Set<String> typeAndSubTypes = context.getEntityTypeNames(); final String typeAndSubTypesQryStr = context.getEntityTypesQryStr(); final String sortBy = context.getSearchParameters().getSortBy(); final SortOrder sortOrder = context.getSearchParameters().getSortOrder(); - final AtlasClassificationType classificationType = context.getClassificationType(); - final Set<String> classificationTypeAndSubTypes = context.getClassificationTypes(); - final boolean filterClassification; + final Set<AtlasClassificationType> classificationTypes = context.getClassificationTypes(); + final Set<String> classificationTypeAndSubTypes = context.getClassificationTypeNames(); + final boolean filterClassification; - if (classificationType != null) { + if (CollectionUtils.isNotEmpty(classificationTypes)) { filterClassification = !context.needClassificationProcessor(); } else { filterClassification = false; } final Predicate typeNamePredicate; - final Predicate traitPredicate = buildTraitPredict(classificationType); + final Predicate traitPredicate = buildTraitPredict(classificationTypes); final Predicate activePredicate = SearchPredicateUtil.getEQPredicateGenerator() .generatePredicate(Constants.STATE_PROPERTY_KEY, "ACTIVE", String.class); @@ -101,10 +89,10 @@ public class EntitySearchProcessor extends SearchProcessor { typeNamePredicate = SearchPredicateUtil.generateIsEntityVertexPredicate(context.getTypeRegistry()); } - processSearchAttributes(entityType, filterCriteria, indexAttributes, graphAttributes, allAttributes); + processSearchAttributes(entityTypes, filterCriteria, indexAttributes, graphAttributes, allAttributes); final boolean typeSearchByIndex = !filterClassification && typeAndSubTypesQryStr.length() <= MAX_QUERY_STR_LENGTH_TYPES; - final boolean attrSearchByIndex = !filterClassification && CollectionUtils.isNotEmpty(indexAttributes) && canApplyIndexFilter(entityType, filterCriteria, false); + final boolean attrSearchByIndex = !filterClassification && CollectionUtils.isNotEmpty(indexAttributes) && canApplyIndexFilter(entityTypes, filterCriteria, false); StringBuilder indexQuery = new StringBuilder(); @@ -116,9 +104,9 @@ public class EntitySearchProcessor extends SearchProcessor { } if (attrSearchByIndex) { - constructFilterQuery(indexQuery, entityType, filterCriteria, indexAttributes); + constructFilterQuery(indexQuery, entityTypes, filterCriteria, indexAttributes); - Predicate attributePredicate = constructInMemoryPredicate(entityType, filterCriteria, indexAttributes); + Predicate attributePredicate = constructInMemoryPredicate(entityTypes, filterCriteria, indexAttributes); if (attributePredicate != null) { inMemoryPredicate = PredicateUtils.andPredicate(inMemoryPredicate, attributePredicate); } @@ -149,6 +137,7 @@ public class EntitySearchProcessor extends SearchProcessor { // If we need to filter on the trait names then we need to build the query and equivalent in-memory predicate if (filterClassification) { + AtlasClassificationType classificationType = classificationTypes.iterator().next(); List<AtlasGraphQuery> orConditions = new LinkedList<>(); if (classificationType == MATCH_ALL_WILDCARD_CLASSIFICATION || classificationType == MATCH_ALL_CLASSIFIED || classificationType == MATCH_ALL_CLASSIFICATION_TYPES) { @@ -176,10 +165,10 @@ public class EntitySearchProcessor extends SearchProcessor { } } - graphQuery = toGraphFilterQuery(entityType, filterCriteria, graphAttributes, query); + graphQuery = toGraphFilterQuery(entityTypes, filterCriteria, graphAttributes, query); // Prepare in-memory predicate for attribute filtering - Predicate attributePredicate = constructInMemoryPredicate(entityType, filterCriteria, graphAttributes); + Predicate attributePredicate = constructInMemoryPredicate(entityTypes, filterCriteria, graphAttributes); if (attributePredicate != null) { if (graphQueryPredicate != null) { @@ -199,7 +188,8 @@ public class EntitySearchProcessor extends SearchProcessor { } } if (sortBy != null && !sortBy.isEmpty()) { - AtlasAttribute sortByAttribute = context.getEntityType().getAttribute(sortBy); + final AtlasEntityType entityType = context.getEntityTypes().iterator().next(); + AtlasAttribute sortByAttribute = entityType.getAttribute(sortBy); if (sortByAttribute != null) { AtlasGraphQuery.SortOrder qrySortOrder = sortOrder == SortOrder.ASCENDING ? ASC : DESC; @@ -215,8 +205,7 @@ public class EntitySearchProcessor extends SearchProcessor { // Prepare the graph query and in-memory filter for the filtering phase filterGraphQueryPredicate = typeNamePredicate; - - Predicate attributesPredicate = constructInMemoryPredicate(entityType, filterCriteria, allAttributes); + Predicate attributesPredicate = constructInMemoryPredicate(entityTypes, filterCriteria, allAttributes); if (attributesPredicate != null) { filterGraphQueryPredicate = filterGraphQueryPredicate == null ? attributesPredicate : @@ -265,7 +254,7 @@ public class EntitySearchProcessor extends SearchProcessor { SortOrder sortOrder = context.getSearchParameters().getSortOrder(); String sortBy = context.getSearchParameters().getSortBy(); - final AtlasEntityType entityType = context.getEntityType(); + final AtlasEntityType entityType = context.getEntityTypes().iterator().next(); AtlasAttribute sortByAttribute = entityType.getAttribute(sortBy); if (sortByAttribute == null) { sortBy = null; diff --git a/repository/src/main/java/org/apache/atlas/discovery/FreeTextSearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/FreeTextSearchProcessor.java index 6e3a760..d9981e6 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/FreeTextSearchProcessor.java +++ b/repository/src/main/java/org/apache/atlas/discovery/FreeTextSearchProcessor.java @@ -18,7 +18,6 @@ package org.apache.atlas.discovery; import org.apache.atlas.model.discovery.SearchParameters; -import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.graph.GraphHelper; import org.apache.atlas.repository.graphdb.*; @@ -53,13 +52,13 @@ public class FreeTextSearchProcessor extends SearchProcessor { queryString.append(searchParameters.getQuery()); - if (CollectionUtils.isNotEmpty(context.getEntityTypes()) && context.getEntityTypesQryStr().length() <= MAX_QUERY_STR_LENGTH_TYPES) { + if (CollectionUtils.isNotEmpty(context.getEntityTypeNames()) && context.getEntityTypesQryStr().length() <= MAX_QUERY_STR_LENGTH_TYPES) { queryString.append(AND_STR).append(context.getEntityTypesQryStr()); } graphIndexQueryBuilder.addActiveStateQueryFilter(queryString); - if (CollectionUtils.isNotEmpty(context.getClassificationTypes()) && context.getClassificationTypesQryStr().length() <= MAX_QUERY_STR_LENGTH_TYPES) { + if (CollectionUtils.isNotEmpty(context.getClassificationTypeNames()) && context.getClassificationTypesQryStr().length() <= MAX_QUERY_STR_LENGTH_TYPES) { queryString.append(AND_STR).append(context.getClassificationTypesQryStr()); } @@ -138,7 +137,7 @@ public class FreeTextSearchProcessor extends SearchProcessor { continue; } - if (context.getClassificationType() != null) { + if (CollectionUtils.isNotEmpty(context.getClassificationTypes())) { List<String> entityClassifications = GraphHelper.getAllTraitNames(vertex); if (!context.includeClassificationTypes(entityClassifications)) { diff --git a/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java index 99cb1d0..b37d93a 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java +++ b/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java @@ -24,6 +24,7 @@ import org.apache.atlas.repository.graphdb.AtlasIndexQuery; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; import org.apache.atlas.utils.AtlasPerfTracer; +import org.apache.commons.collections.CollectionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,10 +32,7 @@ import java.util.ArrayList; import java.util.Iterator; import java.util.List; -import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFIED; -import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_ENTITY_TYPES; import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_NOT_CLASSIFIED; -import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_WILDCARD_CLASSIFICATION; public class FullTextSearchProcessor extends SearchProcessor { @@ -53,29 +51,28 @@ public class FullTextSearchProcessor extends SearchProcessor { // if search includes entity-type criteria, adding a filter here can help avoid unnecessary // processing (and rejection) by subsequent EntitySearchProcessor - if (context.getEntityType() != null && context.getEntityType() != MATCH_ALL_ENTITY_TYPES) { - String typeAndSubTypeNamesQryStr = context.getEntityType().getTypeAndAllSubTypesQryStr(); + if (CollectionUtils.isNotEmpty(context.getEntityTypes())) { + String typeAndSubTypeNamesQryStr = context.getEntityTypesQryStr(); if (typeAndSubTypeNamesQryStr.length() <= MAX_QUERY_STR_LENGTH_TYPES) { queryString.append(AND_STR).append(typeAndSubTypeNamesQryStr); } else { LOG.warn("'{}' has too many subtypes (query-string-length={}) to include in index-query; might cause poor performance", - context.getEntityType().getTypeName(), typeAndSubTypeNamesQryStr.length()); + searchParameters.getTypeName(), typeAndSubTypeNamesQryStr.length()); } } // if search includes classification criteria, adding a filter here can help avoid unnecessary // processing (and rejection) by subsequent ClassificationSearchProcessor or EntitySearchProcessor - if (context.getClassificationType() != null && context.getClassificationType() != MATCH_ALL_WILDCARD_CLASSIFICATION && - context.getClassificationType() != MATCH_ALL_CLASSIFIED && - context.getClassificationType() != MATCH_ALL_NOT_CLASSIFIED) { - String typeAndSubTypeNamesStr = context.getClassificationType().getTypeAndAllSubTypesQryStr(); + if (CollectionUtils.isNotEmpty(context.getClassificationTypes()) && + context.getClassificationTypes().iterator().next() != MATCH_ALL_NOT_CLASSIFIED) { + String typeAndSubTypeNamesStr = context.getClassificationTypesQryStr(); if (typeAndSubTypeNamesStr.length() <= MAX_QUERY_STR_LENGTH_TAGS) { queryString.append(AND_STR).append(typeAndSubTypeNamesStr); } else { LOG.warn("'{}' has too many subtypes (query-string-length={}) to include in index-query; might cause poor performance", - context.getClassificationType().getTypeName(), typeAndSubTypeNamesStr.length()); + searchParameters.getClassification(), typeAndSubTypeNamesStr.length()); } } diff --git a/repository/src/main/java/org/apache/atlas/discovery/GraphIndexQueryBuilder.java b/repository/src/main/java/org/apache/atlas/discovery/GraphIndexQueryBuilder.java index 28d2086..35d64b7 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/GraphIndexQueryBuilder.java +++ b/repository/src/main/java/org/apache/atlas/discovery/GraphIndexQueryBuilder.java @@ -26,6 +26,8 @@ import static org.apache.atlas.repository.Constants.PROPAGATED_CLASSIFICATION_NA import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY; import org.apache.atlas.repository.Constants; +import org.apache.atlas.type.AtlasStructType; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; public class GraphIndexQueryBuilder { @@ -36,20 +38,20 @@ public class GraphIndexQueryBuilder { } void addClassificationTypeFilter(StringBuilder indexQuery) { - if (indexQuery != null && StringUtils.isNotEmpty(context.getSearchParameters().getClassification())) { - String classificationName = context.getSearchParameters().getClassification(); + if (indexQuery != null && CollectionUtils.isNotEmpty(context.getClassificationNames())) { + String classificationNames = AtlasStructType.AtlasAttribute.escapeIndexQueryValue(context.getClassificationNames()); if (indexQuery.length() != 0) { indexQuery.append(" AND "); } - indexQuery.append("(").append(INDEX_SEARCH_PREFIX).append('\"').append(CLASSIFICATION_NAMES_KEY).append('\"').append(':').append(classificationName) + indexQuery.append("(").append(INDEX_SEARCH_PREFIX).append('\"').append(CLASSIFICATION_NAMES_KEY).append('\"').append(':').append(classificationNames) .append(" OR ").append(INDEX_SEARCH_PREFIX).append('\"').append(PROPAGATED_CLASSIFICATION_NAMES_KEY) - .append('\"').append(':').append(classificationName).append(")"); + .append('\"').append(':').append(classificationNames).append(")"); } } void addClassificationAndSubTypesQueryFilter(StringBuilder indexQuery) { - if (indexQuery != null && StringUtils.isNotEmpty(context.getSearchParameters().getClassification())) { + if (indexQuery != null && CollectionUtils.isNotEmpty(context.getClassificationTypes())) { String classificationTypesQryStr = context.getClassificationTypesQryStr(); if (indexQuery.length() != 0) { @@ -63,17 +65,8 @@ public class GraphIndexQueryBuilder { } void addClassificationFilterForBuiltInTypes(StringBuilder indexQuery) { - if (indexQuery != null && context.getClassificationType() != null) { - if (context.getClassificationType() == MATCH_ALL_WILDCARD_CLASSIFICATION || context.getClassificationType() == MATCH_ALL_CLASSIFIED) { - if (indexQuery.length() != 0) { - indexQuery.append(" AND "); - } - indexQuery.append("(").append(INDEX_SEARCH_PREFIX).append("\"") - .append(CLASSIFICATION_NAMES_KEY).append("\"").append(":" + "[* TO *]") - .append(" OR ").append(INDEX_SEARCH_PREFIX).append("\"") - .append(PROPAGATED_CLASSIFICATION_NAMES_KEY).append("\"").append(":" + "[* TO *]").append(")"); - - } else if (context.getClassificationType() == MATCH_ALL_NOT_CLASSIFIED) { + if (indexQuery != null && CollectionUtils.isNotEmpty(context.getClassificationTypes())) { + if (context.getClassificationTypes().iterator().next() == MATCH_ALL_NOT_CLASSIFIED) { if (indexQuery.length() != 0) { indexQuery.append(" AND "); } diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchAggregatorImpl.java b/repository/src/main/java/org/apache/atlas/discovery/SearchAggregatorImpl.java index e8f7dbc..2d095be 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/SearchAggregatorImpl.java +++ b/repository/src/main/java/org/apache/atlas/discovery/SearchAggregatorImpl.java @@ -56,12 +56,7 @@ public class SearchAggregatorImpl implements SearchAggregator { try { AtlasGraphIndexClient graphIndexClient = graph.getGraphIndexClient(); - String searchedOnTypeName = searchParameters.getTypeName(); - AtlasEntityType searchForEntityType = null; - - if (searchedOnTypeName != null) { - searchForEntityType = typeRegistry.getEntityTypeByName(searchedOnTypeName); - } + Set<AtlasEntityType> searchForEntityType = searchContext.getEntityTypes(); Map<String, String> indexFieldNameCache = new HashMap<>(); diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java index 2c2888e..04e8218 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java +++ b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java @@ -21,7 +21,7 @@ package org.apache.atlas.discovery; import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.discovery.SearchParameters; -import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria; +import org.apache.atlas.model.discovery.SearchParameters.*; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.typedef.AtlasClassificationDef; import org.apache.atlas.repository.Constants; @@ -44,13 +44,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.*; +import java.util.stream.Collectors; import static org.apache.atlas.discovery.SearchProcessor.ALL_TYPE_QUERY; -import static org.apache.atlas.model.discovery.SearchParameters.ALL_CLASSIFICATIONS; -import static org.apache.atlas.model.discovery.SearchParameters.ALL_CLASSIFICATION_TYPES; -import static org.apache.atlas.model.discovery.SearchParameters.ALL_ENTITY_TYPES; -import static org.apache.atlas.model.discovery.SearchParameters.NO_CLASSIFICATIONS; -import static org.apache.atlas.model.discovery.SearchParameters.WILDCARD_CLASSIFICATIONS; +import static org.apache.atlas.model.discovery.SearchParameters.*; /* * Search context captures elements required for performing a basic search @@ -62,12 +59,12 @@ public class SearchContext { private final AtlasTypeRegistry typeRegistry; private final AtlasGraph graph; - private final AtlasEntityType entityType; + private final Set<AtlasEntityType> entityTypes; private final Set<String> indexedKeys; private final Set<String> entityAttributes; private final SearchParameters searchParameters; - private final AtlasClassificationType classificationType; - private final String classificationName; + private final Set<AtlasClassificationType> classificationTypes; + private final Set<String> classificationNames; private final Set<String> typeAndSubTypes; private final Set<String> classificationTypeAndSubTypes; private final String typeAndSubTypesQryStr; @@ -80,27 +77,18 @@ public class SearchContext { public final static AtlasClassificationType MATCH_ALL_NOT_CLASSIFIED = new AtlasClassificationType(new AtlasClassificationDef(NO_CLASSIFICATIONS)); public final static AtlasClassificationType MATCH_ALL_CLASSIFICATION_TYPES = AtlasClassificationType.getClassificationRoot(); public final static AtlasEntityType MATCH_ALL_ENTITY_TYPES = AtlasEntityType.getEntityRoot(); + public final static String TYPENAME_DELIMITER = ","; public SearchContext(SearchParameters searchParameters, AtlasTypeRegistry typeRegistry, AtlasGraph graph, Set<String> indexedKeys) throws AtlasBaseException { - this.classificationName = searchParameters.getClassification(); this.searchParameters = searchParameters; this.typeRegistry = typeRegistry; this.graph = graph; this.indexedKeys = indexedKeys; this.entityAttributes = new HashSet<>(); - this.entityType = getEntityType(searchParameters.getTypeName()); - this.classificationType = getClassificationType(classificationName); - - // Validate if the type name exists - if (StringUtils.isNotEmpty(searchParameters.getTypeName()) && entityType == null) { - throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_TYPENAME, searchParameters.getTypeName()); - } - - // Validate if the classification exists - if ((StringUtils.isNotEmpty(classificationName) && classificationType == null && !classificationName.contains(WILDCARD_CLASSIFICATIONS))) { - throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_CLASSIFICATION, classificationName); - } + this.entityTypes = getEntityTypes(searchParameters.getTypeName()); + this.classificationNames = getClassificationNames(searchParameters.getClassification()); + this.classificationTypes = getClassificationTypes(this.classificationNames); AtlasVertex glossaryTermVertex = getGlossaryTermVertex(searchParameters.getTermName()); @@ -109,40 +97,77 @@ public class SearchContext { throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_GLOSSARY_TERM, searchParameters.getTermName()); } - // Invalid attributes will raise an exception with 400 error code - validateAttributes(entityType, searchParameters.getEntityFilters()); + // Invalid attributes or unsupported attribute in a type, will raise an exception with 400 error code + if (CollectionUtils.isNotEmpty(entityTypes)) { + for (AtlasEntityType entityType : entityTypes) { + validateAttributes(entityType, searchParameters.getEntityFilters()); - // Invalid attribute will raise an exception with 400 error code - validateAttributes(entityType, searchParameters.getSortBy()); + validateAttributes(entityType, searchParameters.getSortBy()); + } + } // Invalid attributes will raise an exception with 400 error code - validateAttributes(classificationType, searchParameters.getTagFilters()); + if (CollectionUtils.isNotEmpty(classificationTypes)) { + for (AtlasClassificationType classificationType : classificationTypes) { + validateAttributes(classificationType, searchParameters.getTagFilters()); + } + } - if (classificationType != null && !isBuiltInClassificationType()) { - if (classificationType == MATCH_ALL_CLASSIFICATION_TYPES) { - classificationTypeAndSubTypes = Collections.emptySet(); - classificationTypeAndSubTypesQryStr = ALL_TYPE_QUERY; - } else { - classificationTypeAndSubTypes = searchParameters.getIncludeSubClassifications() ? classificationType.getTypeAndAllSubTypes() : Collections.singleton(classificationType.getTypeName()); - classificationTypeAndSubTypesQryStr = searchParameters.getIncludeSubClassifications() ? classificationType.getTypeAndAllSubTypesQryStr() : classificationType.getTypeQryStr(); + //remove other types if builtin type is present + filterStructTypes(); + + //gather all classifications and its corresponding subtypes + Set<String> classificationTypeAndSubTypes = new HashSet<>(); + String classificationTypeAndSubTypesQryStr = null; + + if (CollectionUtils.isNotEmpty(classificationTypes) && classificationTypes.iterator().next() != MATCH_ALL_NOT_CLASSIFIED ) { + for (AtlasClassificationType classificationType : classificationTypes) { + + if (classificationType == MATCH_ALL_CLASSIFICATION_TYPES) { + classificationTypeAndSubTypes = Collections.emptySet(); + classificationTypeAndSubTypesQryStr = ALL_TYPE_QUERY; + break; + } else { + Set<String> allTypes = searchParameters.getIncludeSubClassifications() ? classificationType.getTypeAndAllSubTypes() : Collections.singleton(classificationType.getTypeName()); + classificationTypeAndSubTypes.addAll(allTypes); } + } + + if (CollectionUtils.isNotEmpty(classificationTypeAndSubTypes)) { + classificationTypeAndSubTypesQryStr = AtlasAttribute.escapeIndexQueryValue(classificationTypeAndSubTypes); } } else { classificationTypeAndSubTypes = Collections.emptySet(); classificationTypeAndSubTypesQryStr = ""; } + this.classificationTypeAndSubTypes = classificationTypeAndSubTypes; + this.classificationTypeAndSubTypesQryStr = classificationTypeAndSubTypesQryStr; + + //gather all types and its corresponding subtypes + Set<String> typeAndSubTypes = new HashSet<>(); + String typeAndSubTypesQryStr = null; + + if (CollectionUtils.isNotEmpty(entityTypes)) { + for (AtlasEntityType entityType : entityTypes) { + + if (entityType.equals(MATCH_ALL_ENTITY_TYPES)) { + typeAndSubTypes = Collections.emptySet(); + typeAndSubTypesQryStr = ALL_TYPE_QUERY; + break; + } else { + Set<String> allTypes = searchParameters.getIncludeSubTypes() ? entityType.getTypeAndAllSubTypes() : Collections.singleton(entityType.getTypeName()); + typeAndSubTypes.addAll(allTypes); + } + } - if (entityType != null) { - if (entityType.equals(MATCH_ALL_ENTITY_TYPES)) { - typeAndSubTypes = Collections.emptySet(); - typeAndSubTypesQryStr = ALL_TYPE_QUERY; - } else { - typeAndSubTypes = searchParameters.getIncludeSubTypes() ? entityType.getTypeAndAllSubTypes() : Collections.singleton(entityType.getTypeName()); - typeAndSubTypesQryStr = searchParameters.getIncludeSubTypes() ? entityType.getTypeAndAllSubTypesQryStr() : entityType.getTypeQryStr(); + if (CollectionUtils.isNotEmpty(typeAndSubTypes)) { + typeAndSubTypesQryStr = AtlasAttribute.escapeIndexQueryValue(typeAndSubTypes); } } else { typeAndSubTypes = Collections.emptySet(); typeAndSubTypesQryStr = ""; } + this.typeAndSubTypes = typeAndSubTypes; + this.typeAndSubTypesQryStr = typeAndSubTypesQryStr; if (glossaryTermVertex != null) { addProcessor(new TermSearchProcessor(this, getAssignedEntities(glossaryTermVertex))); @@ -179,37 +204,37 @@ public class SearchContext { public Set<String> getEntityAttributes() { return entityAttributes; } - public AtlasEntityType getEntityType() { return entityType; } + public Set<AtlasClassificationType> getClassificationTypes() { return classificationTypes; } - public AtlasClassificationType getClassificationType() { return classificationType; } + public Set<String> getEntityTypeNames() { return typeAndSubTypes; } - public Set<String> getEntityTypes() { return typeAndSubTypes; } - - public Set<String> getClassificationTypes() { return classificationTypeAndSubTypes; } + public Set<String> getClassificationTypeNames() { return classificationTypeAndSubTypes; } public String getEntityTypesQryStr() { return typeAndSubTypesQryStr; } public String getClassificationTypesQryStr() { return classificationTypeAndSubTypesQryStr; } + public Set<AtlasEntityType> getEntityTypes() { return entityTypes; } + public SearchProcessor getSearchProcessor() { return searchProcessor; } - public String getClassificationName() {return classificationName;} + public Set<String> getClassificationNames() {return classificationNames;} public boolean includeEntityType(String entityType) { return typeAndSubTypes.isEmpty() || typeAndSubTypes.contains(entityType); } - public boolean includeClassificationTypes(Collection<String> classificationTypes) { + public boolean includeClassificationTypes(Collection<String> traitNames) { final boolean ret; - if (classificationType == null || classificationTypeAndSubTypes.isEmpty()) { + if (CollectionUtils.isEmpty(classificationTypes) || classificationTypeAndSubTypes.isEmpty()) { ret = true; - } else if (classificationType == MATCH_ALL_NOT_CLASSIFIED) { - ret = CollectionUtils.isEmpty(classificationTypes); - } else if (classificationType == MATCH_ALL_CLASSIFIED || classificationType == MATCH_ALL_WILDCARD_CLASSIFICATION) { - ret = CollectionUtils.isNotEmpty(classificationTypes); + } else if (classificationTypes.iterator().next() == MATCH_ALL_NOT_CLASSIFIED) { + ret = CollectionUtils.isEmpty(traitNames); + } else if (classificationTypes.iterator().next() == MATCH_ALL_CLASSIFICATION_TYPES) { + ret = CollectionUtils.isNotEmpty(traitNames); } else { - ret = CollectionUtils.containsAny(classificationTypeAndSubTypes, classificationTypes); + ret = CollectionUtils.containsAny(classificationTypeAndSubTypes, traitNames); } return ret; @@ -243,25 +268,18 @@ public class SearchContext { } boolean needClassificationProcessor() { - return (classificationType != null && (entityType == null || hasAttributeFilter(searchParameters.getTagFilters()))) || isWildCardSearch() ; - } - - boolean isBuiltInClassificationType() { - return getClassificationType() == MATCH_ALL_WILDCARD_CLASSIFICATION - || getClassificationType() == MATCH_ALL_CLASSIFIED - || getClassificationType() == MATCH_ALL_NOT_CLASSIFIED; + return (CollectionUtils.isNotEmpty(classificationTypes) && (CollectionUtils.isEmpty(entityTypes) || hasAttributeFilter(searchParameters.getTagFilters()))) || isWildCardSearch() ; } boolean isWildCardSearch () { - String classification = getSearchParameters().getClassification(); - if (StringUtils.isNotEmpty(classification) && getClassificationType() == null) { - return classification.contains("*"); + if (CollectionUtils.isNotEmpty(classificationNames)) { + return classificationNames.stream().anyMatch(classification -> classification.contains(WILDCARD_CLASSIFICATIONS)); } return false; } boolean needEntityProcessor() { - return entityType != null; + return CollectionUtils.isNotEmpty(entityTypes); } private void validateAttributes(final AtlasStructType structType, final FilterCriteria filterCriteria) throws AtlasBaseException { @@ -285,7 +303,14 @@ public class SearchContext { if (structType == null) { throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_TYPENAME, "NULL"); } - throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_ATTRIBUTE, attributeName, structType.getTypeName()); + + String name = structType.getTypeName(); + if (name.equals(MATCH_ALL_ENTITY_TYPES.getTypeName())) { + name = ALL_ENTITY_TYPES; + } else if (name.equals(MATCH_ALL_CLASSIFICATION_TYPES.getTypeName())) { + name = ALL_CLASSIFICATION_TYPES; + } + throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_ATTRIBUTE, attributeName, name); } } } @@ -321,11 +346,107 @@ public class SearchContext { return ret; } + private Set<AtlasClassificationType> getClassificationTypes(Set<String> classificationNames) { + if (CollectionUtils.isNotEmpty(classificationNames)) { + return classificationNames.stream().map(n -> + getClassificationType(n)).filter(Objects::nonNull).collect(Collectors.toSet()); + } + + return null; + } + + private Set<String> getClassificationNames(String classification) throws AtlasBaseException { + Set<String> classificationNames = new HashSet<>(); + + if (StringUtils.isNotEmpty(classification)) { + String[] types = classification.split(TYPENAME_DELIMITER); + Set<String> names = new HashSet<>(Arrays.asList(types)); + + names.forEach(name -> { + AtlasClassificationType type = getClassificationType(name); + if (type != null || name.contains(WILDCARD_CLASSIFICATIONS)) { + classificationNames.add(name); + } + }); + + // Validate if the classification exists + if (CollectionUtils.isEmpty(classificationNames) || classificationNames.size() != names.size()) { + if (CollectionUtils.isNotEmpty(classificationNames)) { + names.removeAll(classificationNames); + } + throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_CLASSIFICATION, String.join(TYPENAME_DELIMITER, names)); + } + } + + return classificationNames; + } + private AtlasEntityType getEntityType(String entityName) { return StringUtils.equals(entityName, ALL_ENTITY_TYPES) ? MATCH_ALL_ENTITY_TYPES : typeRegistry.getEntityTypeByName(entityName); } + private Set<AtlasEntityType> getEntityTypes(String typeName) throws AtlasBaseException { + + Set<AtlasEntityType> entityTypes = null; + //split multiple typeNames by comma + if (StringUtils.isNotEmpty(typeName)) { + + String[] types = typeName.split(TYPENAME_DELIMITER); + Set<String> typeNames = new HashSet<>(Arrays.asList(types)); + entityTypes = typeNames.stream().map(n -> + getEntityType(n)).filter(Objects::nonNull).collect(Collectors.toSet()); + + // Validate if the type name is incorrect + if (CollectionUtils.isEmpty(entityTypes) || entityTypes.size() != typeNames.size()) { + if (CollectionUtils.isNotEmpty(entityTypes)) { + Set<String> validEntityTypes = new HashSet<>(); + for (AtlasEntityType entityType : entityTypes) { + String name = entityType.getTypeName(); + if (name.equals(MATCH_ALL_ENTITY_TYPES.getTypeName())) { + validEntityTypes.add(ALL_ENTITY_TYPES); + continue; + } + validEntityTypes.add(entityType.getTypeName()); + } + + typeNames.removeAll(validEntityTypes); + } + + throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_TYPENAME, String.join(TYPENAME_DELIMITER, typeNames)); + } + + } + + return entityTypes; + } + + private void filterStructTypes(){ + //if typeName contains ALL_ENTITY_TYPES, remove others as OR condition will not effect any other + if (CollectionUtils.isNotEmpty(entityTypes) && entityTypes.contains(MATCH_ALL_ENTITY_TYPES)) { + entityTypes.clear(); + entityTypes.add(MATCH_ALL_ENTITY_TYPES); + } + + //No Builtin Classification can be together + if (CollectionUtils.isNotEmpty(classificationTypes)) { + if (classificationTypes.contains(MATCH_ALL_NOT_CLASSIFIED)) { + classificationTypes.clear(); + classificationTypes.add(MATCH_ALL_NOT_CLASSIFIED); + + classificationNames.clear(); + classificationNames.add(MATCH_ALL_NOT_CLASSIFIED.getTypeName()); + } else if (classificationTypes.contains(MATCH_ALL_WILDCARD_CLASSIFICATION) || classificationTypes.contains(MATCH_ALL_CLASSIFICATION_TYPES) || classificationTypes.contains(MATCH_ALL_CLASSIFIED)) { + classificationTypes.clear(); + classificationTypes.add(MATCH_ALL_CLASSIFICATION_TYPES); + + classificationNames.clear(); + classificationNames.add(ALL_CLASSIFICATION_TYPES); + } + } + } + + private AtlasVertex getGlossaryTermVertex(String termName) { AtlasVertex ret = null; diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java index 981ba6a..c62f805 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java +++ b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java @@ -51,16 +51,8 @@ import java.util.regex.Pattern; import static org.apache.atlas.SortOrder.ASCENDING; import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFICATION_TYPES; -import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFIED; import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_NOT_CLASSIFIED; -import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_WILDCARD_CLASSIFICATION; -import static org.apache.atlas.repository.Constants.CLASSIFICATION_NAMES_KEY; -import static org.apache.atlas.repository.Constants.CLASSIFICATION_NAME_DELIMITER; -import static org.apache.atlas.repository.Constants.CUSTOM_ATTRIBUTES_PROPERTY_KEY; -import static org.apache.atlas.repository.Constants.LABELS_PROPERTY_KEY; -import static org.apache.atlas.repository.Constants.PROPAGATED_CLASSIFICATION_NAMES_KEY; -import static org.apache.atlas.repository.Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY; -import static org.apache.atlas.repository.Constants.TRAIT_NAMES_PROPERTY_KEY; +import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.util.SearchPredicateUtil.*; public abstract class SearchProcessor { @@ -153,11 +145,19 @@ public abstract class SearchProcessor { public abstract long getResultCount(); protected boolean isEntityRootType() { - return context.getEntityType() == SearchContext.MATCH_ALL_ENTITY_TYPES; + //always size will be one if in case of _ALL_ENTITY_TYPES + if (CollectionUtils.isNotEmpty(context.getEntityTypes())) { + return context.getEntityTypes().iterator().next() == SearchContext.MATCH_ALL_ENTITY_TYPES; + } + return false; } protected boolean isClassificationRootType() { - return context.getClassificationType() == SearchContext.MATCH_ALL_CLASSIFICATION_TYPES; + //always size will be one if in case of _ALL_CLASSIFICATION_TYPES + if (CollectionUtils.isNotEmpty(context.getClassificationTypes())) { + return context.getClassificationTypes().iterator().next() == SearchContext.MATCH_ALL_CLASSIFICATION_TYPES; + } + return false; } protected boolean isSystemAttribute(String attrName) { @@ -195,9 +195,14 @@ public abstract class SearchProcessor { } } - protected Predicate buildTraitPredict(AtlasClassificationType classificationType) { + protected Predicate buildTraitPredict(Set<AtlasClassificationType> classificationTypes) { Predicate traitPredicate; - if (classificationType == MATCH_ALL_WILDCARD_CLASSIFICATION || classificationType == MATCH_ALL_CLASSIFIED || classificationType == MATCH_ALL_CLASSIFICATION_TYPES) { + AtlasClassificationType classificationType = null; + if (CollectionUtils.isNotEmpty(classificationTypes)) { + classificationType = classificationTypes.iterator().next(); + } + + if (classificationType == MATCH_ALL_CLASSIFICATION_TYPES) { traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, null, List.class), SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class)); } else if (classificationType == MATCH_ALL_NOT_CLASSIFIED) { @@ -205,20 +210,27 @@ public abstract class SearchProcessor { SearchPredicateUtil.getIsNullOrEmptyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class)); } else if (context.isWildCardSearch()) { //For wildcard search __classificationNames which of String type is taken instead of _traitNames which is of Array type - //No need to escape, as classification Names only support letters,numbers,space and underscore - String regexString = getRegexString("\\|" + context.getClassificationName() + "\\|"); - traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getRegexPredicateGenerator().generatePredicate(CLASSIFICATION_NAMES_KEY, regexString, String.class), - SearchPredicateUtil.getRegexPredicateGenerator().generatePredicate(PROPAGATED_CLASSIFICATION_NAMES_KEY, regexString, String.class)); + Set<String> classificationNames = context.getClassificationNames(); + List<Predicate> predicates = new ArrayList<>(); + + classificationNames.forEach(classificationName -> { + //No need to escape, as classification Names only support letters,numbers,space and underscore + String regexString = getRegexString("\\|" + classificationName + "\\|"); + predicates.add(SearchPredicateUtil.getRegexPredicateGenerator().generatePredicate(CLASSIFICATION_NAMES_KEY, regexString, String.class)); + predicates.add(SearchPredicateUtil.getRegexPredicateGenerator().generatePredicate(PROPAGATED_CLASSIFICATION_NAMES_KEY, regexString, String.class)); + }); + + traitPredicate = PredicateUtils.anyPredicate(predicates); } else { - traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, context.getClassificationTypes(), List.class), - SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, context.getClassificationTypes(), List.class)); + traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, context.getClassificationTypeNames(), List.class), + SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, context.getClassificationTypeNames(), List.class)); } return traitPredicate; } - protected void processSearchAttributes(AtlasStructType structType, FilterCriteria filterCriteria, Set<String> indexFiltered, Set<String> graphFiltered, Set<String> allAttributes) { - if (structType == null || filterCriteria == null) { + protected void processSearchAttributes(Set<? extends AtlasStructType> structTypes, FilterCriteria filterCriteria, Set<String> indexFiltered, Set<String> graphFiltered, Set<String> allAttributes) { + if (CollectionUtils.isEmpty(structTypes) || filterCriteria == null) { return; } @@ -227,7 +239,7 @@ public abstract class SearchProcessor { if (filterCondition != null && CollectionUtils.isNotEmpty(criterion)) { for (SearchParameters.FilterCriteria criteria : criterion) { - processSearchAttributes(structType, criteria, indexFiltered, graphFiltered, allAttributes); + processSearchAttributes(structTypes, criteria, indexFiltered, graphFiltered, allAttributes); } } else if (StringUtils.isNotEmpty(filterCriteria.getAttributeName())) { String attributeName = filterCriteria.getAttributeName(); @@ -267,20 +279,23 @@ public abstract class SearchProcessor { } try { - if (isIndexSearchable(filterCriteria, structType)) { - indexFiltered.add(attributeName); - } else { - LOG.warn("not using index-search for attribute '{}'; might cause poor performance", structType.getVertexPropertyName(attributeName)); + for (AtlasStructType structType : structTypes) { + String qualifiedName = structType.getVertexPropertyName(attributeName); + if (isIndexSearchable(filterCriteria, structType)) { + indexFiltered.add(qualifiedName); + } else { + LOG.warn("not using index-search for attribute '{}'; might cause poor performance", structType.getVertexPropertyName(attributeName)); - graphFiltered.add(attributeName); - } + graphFiltered.add(qualifiedName); + } - if (structType instanceof AtlasEntityType && !isSystemAttribute(attributeName)) { - // Capture the entity attributes - context.getEntityAttributes().add(attributeName); - } + if (structType instanceof AtlasEntityType && !isSystemAttribute(attributeName)) { + // Capture the entity attributes + context.getEntityAttributes().add(attributeName); + } - allAttributes.add(attributeName); + allAttributes.add(qualifiedName); + } } catch (AtlasBaseException e) { LOG.warn(e.getMessage()); } @@ -298,7 +313,7 @@ public abstract class SearchProcessor { // (AND (OR idx-att1=x idx-attr1=y) non-idx-attr=z) // (AND (OR idx-att1=x idx-attr1=y) non-idx-attr=z (AND idx-attr2=xyz idx-attr2=abc)) // - protected boolean canApplyIndexFilter(AtlasStructType structType, FilterCriteria filterCriteria, boolean insideOrCondition) { + protected boolean canApplyIndexFilter(Set<? extends AtlasStructType> structTypes, FilterCriteria filterCriteria, boolean insideOrCondition) { if (!context.hasAttributeFilter(filterCriteria)) { return true; } @@ -314,7 +329,7 @@ public abstract class SearchProcessor { // If we have nested criterion let's find any nested ORs with non-indexed attr for (FilterCriteria criteria : criterion) { - ret = canApplyIndexFilter(structType, criteria, insideOrCondition); + ret = canApplyIndexFilter(structTypes, criteria, insideOrCondition); if (!ret) { break; @@ -322,10 +337,11 @@ public abstract class SearchProcessor { } } else if (StringUtils.isNotEmpty(filterCriteria.getAttributeName())) { try { - - - if (insideOrCondition && !isIndexSearchable(filterCriteria, structType)) { - ret = false; + for (AtlasStructType structType : structTypes) { + if (insideOrCondition && !isIndexSearchable(filterCriteria, structType)) { + ret = false; + break; + } } } catch (AtlasBaseException e) { LOG.warn(e.getMessage()); @@ -338,7 +354,7 @@ public abstract class SearchProcessor { protected void filterWhiteSpaceClassification(List<AtlasVertex> entityVertices) { if (CollectionUtils.isNotEmpty(entityVertices)) { final Iterator<AtlasVertex> it = entityVertices.iterator(); - final Set<String> typeAndSubTypes = context.getClassificationTypes(); + final Set<String> typeAndSubTypes = context.getClassificationTypeNames(); while (it.hasNext()) { AtlasVertex entityVertex = it.next(); @@ -363,13 +379,13 @@ public abstract class SearchProcessor { } } - protected void constructFilterQuery(StringBuilder indexQuery, AtlasStructType type, FilterCriteria filterCriteria, Set<String> indexAttributes) { + protected void constructFilterQuery(StringBuilder indexQuery, Set<? extends AtlasStructType> structTypes, FilterCriteria filterCriteria, Set<String> indexAttributes) { if (filterCriteria != null) { if (LOG.isDebugEnabled()) { LOG.debug("Processing Filters"); } - String filterQuery = toIndexQuery(type, filterCriteria, indexAttributes, 0); + String filterQuery = toIndexQuery(structTypes, filterCriteria, indexAttributes, 0); if (StringUtils.isNotEmpty(filterQuery)) { if (indexQuery.length() > 0) { @@ -381,14 +397,14 @@ public abstract class SearchProcessor { } } - protected Predicate constructInMemoryPredicate(AtlasStructType type, FilterCriteria filterCriteria, Set<String> indexAttributes) { + protected Predicate constructInMemoryPredicate(Set<? extends AtlasStructType> structTypes, FilterCriteria filterCriteria, Set<String> indexAttributes) { Predicate ret = null; if (filterCriteria != null) { if (LOG.isDebugEnabled()) { LOG.debug("Processing Filters"); } - ret = toInMemoryPredicate(type, filterCriteria, indexAttributes); + ret = toInMemoryPredicate(structTypes, filterCriteria, indexAttributes); } return ret; } @@ -473,17 +489,20 @@ public abstract class SearchProcessor { return ret; } - private String toIndexQuery(AtlasStructType type, FilterCriteria criteria, Set<String> indexAttributes, int level) { - return toIndexQuery(type, criteria, indexAttributes, new StringBuilder(), level); + private String toIndexQuery(Set<? extends AtlasStructType> structTypes, FilterCriteria criteria, Set<String> indexAttributes, int level) { + return toIndexQuery(structTypes, criteria, indexAttributes, new StringBuilder(), level); } - private String toIndexQuery(AtlasStructType type, FilterCriteria criteria, Set<String> indexAttributes, StringBuilder sb, int level) { + private String toIndexQuery(Set<? extends AtlasStructType> structTypes, FilterCriteria criteria, Set<String> indexAttributes, StringBuilder sb, int level) { + Set<String> filterAttributes = new HashSet<>(); + filterAttributes.addAll(indexAttributes); + Condition condition = criteria.getCondition(); if (condition != null && CollectionUtils.isNotEmpty(criteria.getCriterion())) { StringBuilder nestedExpression = new StringBuilder(); for (FilterCriteria filterCriteria : criteria.getCriterion()) { - String nestedQuery = toIndexQuery(type, filterCriteria, indexAttributes, level + 1); + String nestedQuery = toIndexQuery(structTypes, filterCriteria, filterAttributes, level + 1); if (StringUtils.isNotEmpty(nestedQuery)) { if (nestedExpression.length() > 0) { @@ -502,19 +521,45 @@ public abstract class SearchProcessor { } else { return EMPTY_STRING; } - } else if (indexAttributes.contains(criteria.getAttributeName())){ - return toIndexExpression(type, criteria.getAttributeName(), criteria.getOperator(), criteria.getAttributeValue()); - } else { - return EMPTY_STRING; + } else if (StringUtils.isNotEmpty(criteria.getAttributeName())) { + try { + ArrayList<String> orExpQuery = new ArrayList<>(); + for (AtlasStructType structType : structTypes) { + String name = structType.getVertexPropertyName(criteria.getAttributeName()); + + if (filterAttributes.contains(name)) { + String nestedQuery = toIndexExpression(structType, criteria.getAttributeName(), criteria.getOperator(), criteria.getAttributeValue()); + orExpQuery.add(nestedQuery); + filterAttributes.remove(name); + } + } + + if (CollectionUtils.isNotEmpty(orExpQuery)) { + if (orExpQuery.size() > 1) { + String orExpStr = StringUtils.join(orExpQuery, " "+Condition.OR.name()+" "); + return BRACE_OPEN_STR + " " + orExpStr + " " + BRACE_CLOSE_STR; + } else { + return orExpQuery.iterator().next(); + } + } else { + return EMPTY_STRING; + } + } catch (AtlasBaseException e) { + LOG.warn(e.getMessage()); + } } + return EMPTY_STRING; } - private Predicate toInMemoryPredicate(AtlasStructType type, FilterCriteria criteria, Set<String> indexAttributes) { + private Predicate toInMemoryPredicate(Set<? extends AtlasStructType> structTypes, FilterCriteria criteria, Set<String> indexAttributes) { + Set<String> filterAttributes = new HashSet<>(); + filterAttributes.addAll(indexAttributes); + if (criteria.getCondition() != null && CollectionUtils.isNotEmpty(criteria.getCriterion())) { List<Predicate> predicates = new ArrayList<>(); for (FilterCriteria filterCriteria : criteria.getCriterion()) { - Predicate predicate = toInMemoryPredicate(type, filterCriteria, indexAttributes); + Predicate predicate = toInMemoryPredicate(structTypes, filterCriteria, filterAttributes); if (predicate != null) { predicates.add(predicate); @@ -528,21 +573,40 @@ public abstract class SearchProcessor { return PredicateUtils.anyPredicate(predicates); } } - } else if (indexAttributes.contains(criteria.getAttributeName())) { - String attrName = criteria.getAttributeName(); - String attrValue = criteria.getAttributeValue(); - SearchParameters.Operator operator = criteria.getOperator(); + } else if (StringUtils.isNotEmpty(criteria.getAttributeName())) { + try { + ArrayList<Predicate> predicates = new ArrayList<>(); + for (AtlasStructType structType : structTypes) { + String name = structType.getVertexPropertyName(criteria.getAttributeName()); + + if (filterAttributes.contains(name)) { + String attrName = criteria.getAttributeName(); + String attrValue = criteria.getAttributeValue(); + SearchParameters.Operator operator = criteria.getOperator(); + + //process attribute value and attribute operator for pipeSeperated fields + if (isPipeSeparatedSystemAttribute(attrName)) { + FilterCriteria processedCriteria = processPipeSeperatedSystemAttribute(attrName, operator, attrValue); + attrValue = processedCriteria.getAttributeValue(); + operator = processedCriteria.getOperator(); + } - //process attribute value and attribute operator for pipeSeperated fields - if (isPipeSeparatedSystemAttribute(attrName)) { - FilterCriteria processedCriteria = processPipeSeperatedSystemAttribute(attrName, operator, attrValue); - attrValue = processedCriteria.getAttributeValue(); - operator = processedCriteria.getOperator(); - } + predicates.add(toInMemoryPredicate(structType, attrName, operator, attrValue)); + filterAttributes.remove(name); + } + } - return toInMemoryPredicate(type, attrName, operator, attrValue); + if (CollectionUtils.isNotEmpty(predicates)) { + if (predicates.size() > 1) { + return PredicateUtils.anyPredicate(predicates); + } else { + return predicates.iterator().next(); + } + } + } catch (AtlasBaseException e) { + LOG.warn(e.getMessage()); + } } - return null; } @@ -704,12 +768,15 @@ public abstract class SearchProcessor { return ret; } - protected AtlasGraphQuery toGraphFilterQuery(AtlasStructType type, FilterCriteria criteria, Set<String> graphAttributes, AtlasGraphQuery query) { + protected AtlasGraphQuery toGraphFilterQuery(Set<? extends AtlasStructType> structTypes, FilterCriteria criteria, Set<String> graphAttributes, AtlasGraphQuery query) { + Set<String> filterAttributes = new HashSet<>(); + filterAttributes.addAll(graphAttributes); + if (criteria != null) { if (criteria.getCondition() != null) { if (criteria.getCondition() == Condition.AND) { for (FilterCriteria filterCriteria : criteria.getCriterion()) { - AtlasGraphQuery nestedQuery = toGraphFilterQuery(type, filterCriteria, graphAttributes, context.getGraph().query()); + AtlasGraphQuery nestedQuery = toGraphFilterQuery(structTypes, filterCriteria, filterAttributes, context.getGraph().query()); query.addConditionsFrom(nestedQuery); } @@ -717,7 +784,7 @@ public abstract class SearchProcessor { List<AtlasGraphQuery> orConditions = new LinkedList<>(); for (FilterCriteria filterCriteria : criteria.getCriterion()) { - AtlasGraphQuery nestedQuery = toGraphFilterQuery(type, filterCriteria, graphAttributes, context.getGraph().query()); + AtlasGraphQuery nestedQuery = toGraphFilterQuery(structTypes, filterCriteria, filterAttributes, context.getGraph().query()); orConditions.add(context.getGraph().query().createChildQuery().addConditionsFrom(nestedQuery)); } @@ -726,60 +793,80 @@ public abstract class SearchProcessor { query.or(orConditions); } } - } else if (graphAttributes.contains(criteria.getAttributeName())) { - String attrName = criteria.getAttributeName(); - String attrValue = criteria.getAttributeValue(); - SearchParameters.Operator operator = criteria.getOperator(); - - //process attribute value and attribute operator for pipeSeperated fields - if (isPipeSeparatedSystemAttribute(attrName)) { - FilterCriteria processedCriteria = processPipeSeperatedSystemAttribute(attrName, operator, attrValue); - attrValue = processedCriteria.getAttributeValue(); - operator = processedCriteria.getOperator(); - } - - final String qualifiedName = type.getAttribute(attrName).getVertexPropertyName(); + } else if (StringUtils.isNotEmpty(criteria.getAttributeName())) { + try { + ArrayList<AtlasGraphQuery> queries = new ArrayList<>(); + for (AtlasStructType structType : structTypes) { + String qualifiedName = structType.getVertexPropertyName(criteria.getAttributeName()); + if (filterAttributes.contains(qualifiedName)) { + + String attrName = criteria.getAttributeName(); + String attrValue = criteria.getAttributeValue(); + SearchParameters.Operator operator = criteria.getOperator(); + + //process attribute value and attribute operator for pipeSeperated fields + if (isPipeSeparatedSystemAttribute(attrName)) { + FilterCriteria processedCriteria = processPipeSeperatedSystemAttribute(attrName, operator, attrValue); + attrValue = processedCriteria.getAttributeValue(); + operator = processedCriteria.getOperator(); + } + + AtlasGraphQuery innerQry = context.getGraph().query().createChildQuery(); + switch (operator) { + case LT: + innerQry.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.LESS_THAN, attrValue); + break; + case LTE: + innerQry.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.LESS_THAN_EQUAL, attrValue); + break; + case GT: + innerQry.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.GREATER_THAN, attrValue); + break; + case GTE: + innerQry.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.GREATER_THAN_EQUAL, attrValue); + break; + case EQ: + innerQry.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.EQUAL, attrValue); + break; + case NEQ: + innerQry.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.NOT_EQUAL, attrValue); + break; + case LIKE: + innerQry.has(qualifiedName, AtlasGraphQuery.MatchingOperator.REGEX, attrValue); + break; + case CONTAINS: + innerQry.has(qualifiedName, AtlasGraphQuery.MatchingOperator.REGEX, getContainsRegex(attrValue)); + break; + case STARTS_WITH: + innerQry.has(qualifiedName, AtlasGraphQuery.MatchingOperator.PREFIX, attrValue); + break; + case ENDS_WITH: + innerQry.has(qualifiedName, AtlasGraphQuery.MatchingOperator.REGEX, getSuffixRegex(attrValue)); + break; + case IS_NULL: + innerQry.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.EQUAL, null); + break; + case NOT_NULL: + innerQry.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.NOT_EQUAL, null); + break; + default: + LOG.warn("{}: unsupported operator. Ignored", operator); + break; + } + queries.add(context.getGraph().query().createChildQuery().addConditionsFrom(innerQry)); + filterAttributes.remove(qualifiedName); + } + } - switch (operator) { - case LT: - query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.LESS_THAN, attrValue); - break; - case LTE: - query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.LESS_THAN_EQUAL, attrValue); - break; - case GT: - query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.GREATER_THAN, attrValue); - break; - case GTE: - query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.GREATER_THAN_EQUAL, attrValue); - break; - case EQ: - query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.EQUAL, attrValue); - break; - case NEQ: - query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.NOT_EQUAL, attrValue); - break; - case LIKE: - query.has(qualifiedName, AtlasGraphQuery.MatchingOperator.REGEX, attrValue); - break; - case CONTAINS: - query.has(qualifiedName, AtlasGraphQuery.MatchingOperator.REGEX, getContainsRegex(attrValue)); - break; - case STARTS_WITH: - query.has(qualifiedName, AtlasGraphQuery.MatchingOperator.PREFIX, attrValue); - break; - case ENDS_WITH: - query.has(qualifiedName, AtlasGraphQuery.MatchingOperator.REGEX, getSuffixRegex(attrValue)); - break; - case IS_NULL: - query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.EQUAL, null); - break; - case NOT_NULL: - query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.NOT_EQUAL, null); - break; - default: - LOG.warn("{}: unsupported operator. Ignored", operator); - break; + if (CollectionUtils.isNotEmpty(queries)) { + if (queries.size() > 1) { + return context.getGraph().query().createChildQuery().or(queries); + } else { + return queries.iterator().next(); + } + } + } catch (AtlasBaseException e) { + LOG.warn(e.getMessage()); } } } @@ -987,11 +1074,13 @@ public abstract class SearchProcessor { } private static String getSortByAttribute(SearchContext context) { - final AtlasEntityType entityType = context.getEntityType(); - String sortBy = context.getSearchParameters().getSortBy(); - AtlasStructType.AtlasAttribute sortByAttribute = entityType != null ? entityType.getAttribute(sortBy) : null; - if (sortByAttribute != null) { - return sortByAttribute.getVertexPropertyName(); + if (CollectionUtils.isNotEmpty(context.getEntityTypes())) { + final AtlasEntityType entityType = context.getEntityTypes().iterator().next(); + String sortBy = context.getSearchParameters().getSortBy(); + AtlasStructType.AtlasAttribute sortByAttribute = entityType.getAttribute(sortBy); + if (sortByAttribute != null) { + return sortByAttribute.getVertexPropertyName(); + } } return null; } diff --git a/repository/src/test/java/org/apache/atlas/discovery/EntitySearchProcessorTest.java b/repository/src/test/java/org/apache/atlas/discovery/EntitySearchProcessorTest.java index 43f11d1..b7ce978 100644 --- a/repository/src/test/java/org/apache/atlas/discovery/EntitySearchProcessorTest.java +++ b/repository/src/test/java/org/apache/atlas/discovery/EntitySearchProcessorTest.java @@ -130,7 +130,7 @@ public class EntitySearchProcessorTest extends BasicTestSetup { new EntitySearchProcessor(context); } - @Test + @Test(priority = -1) public void searchWithNEQ_stringAttr() throws AtlasBaseException { String expectedEntityName = "hive_Table_Null_tableType"; createDummyEntity(expectedEntityName,HIVE_TABLE_TYPE); @@ -197,4 +197,116 @@ public class EntitySearchProcessorTest extends BasicTestSetup { assertTrue(nameList.contains("hive_Table_Null_tableType")); } + + @Test + public void ALLEntityType() throws AtlasBaseException { + SearchParameters params = new SearchParameters(); + params.setTypeName(SearchParameters.ALL_ENTITY_TYPES); + params.setLimit(20); + + SearchContext context = new SearchContext(params, typeRegistry, graph, Collections.<String>emptySet()); + + EntitySearchProcessor processor = new EntitySearchProcessor(context); + assertEquals(processor.execute().size(), 20); + } + + @Test + public void ALLEntityTypeWithTag() throws AtlasBaseException { + SearchParameters params = new SearchParameters(); + params.setTypeName(SearchParameters.ALL_ENTITY_TYPES); + params.setClassification(FACT_CLASSIFICATION); + params.setLimit(20); + + SearchContext context = new SearchContext(params, typeRegistry, graph, Collections.<String>emptySet()); + + EntitySearchProcessor processor = new EntitySearchProcessor(context); + assertEquals(processor.execute().size(), 5); + } + + @Test + public void entityType() throws AtlasBaseException { + SearchParameters params = new SearchParameters(); + params.setTypeName(DATABASE_TYPE); + params.setLimit(20); + + SearchContext context = new SearchContext(params, typeRegistry, graph, Collections.<String>emptySet()); + + EntitySearchProcessor processor = new EntitySearchProcessor(context); + assertEquals(processor.execute().size(), 3); + } + + @Test + public void entityTypes() throws AtlasBaseException { + SearchParameters params = new SearchParameters(); + params.setTypeName(DATABASE_TYPE+","+HIVE_TABLE_TYPE); + params.setLimit(20); + + SearchContext context = new SearchContext(params, typeRegistry, graph, Collections.<String>emptySet()); + + EntitySearchProcessor processor = new EntitySearchProcessor(context); + assertEquals(processor.execute().size(), 14); + } + + @Test(expectedExceptions = AtlasBaseException.class, expectedExceptionsMessageRegExp = "Not_Exists: Unknown/invalid typename") + public void entityTypesNotAllowed() throws AtlasBaseException { + SearchParameters params = new SearchParameters(); + params.setTypeName(DATABASE_TYPE+",Not_Exists"); + params.setLimit(20); + + SearchContext context = new SearchContext(params, typeRegistry, graph, Collections.<String>emptySet()); + } + + @Test(expectedExceptions = AtlasBaseException.class, expectedExceptionsMessageRegExp = "Attribute tableType not found for type "+DATABASE_TYPE) + public void entityFiltersNotAllowed() throws AtlasBaseException { + SearchParameters params = new SearchParameters(); + params.setTypeName(DATABASE_TYPE+","+HIVE_TABLE_TYPE); + SearchParameters.FilterCriteria filterCriteria = getSingleFilterCondition("tableType", SearchParameters.Operator.CONTAINS, "ETL"); + params.setEntityFilters(filterCriteria); + params.setLimit(20); + + SearchContext context = new SearchContext(params, typeRegistry, graph, Collections.<String>emptySet()); + } + + @Test + public void entityTypesAndTag() throws AtlasBaseException { + SearchParameters params = new SearchParameters(); + params.setTypeName(DATABASE_TYPE+","+HIVE_TABLE_TYPE); + params.setClassification(FACT_CLASSIFICATION); + params.setLimit(20); + + SearchContext context = new SearchContext(params, typeRegistry, graph, Collections.<String>emptySet()); + + EntitySearchProcessor processor = new EntitySearchProcessor(context); + assertEquals(processor.execute().size(), 3); + } + + @Test + public void searchWithEntityTypesAndEntityFilters() throws AtlasBaseException { + SearchParameters params = new SearchParameters(); + params.setTypeName(DATABASE_TYPE+","+HIVE_TABLE_TYPE); + SearchParameters.FilterCriteria filterCriteria = getSingleFilterCondition("owner", SearchParameters.Operator.CONTAINS, "ETL"); + params.setEntityFilters(filterCriteria); + params.setLimit(20); + + SearchContext context = new SearchContext(params, typeRegistry, graph, Collections.<String>emptySet()); + + EntitySearchProcessor processor = new EntitySearchProcessor(context); + assertEquals(processor.execute().size(), 4); + } + + @Test + public void searchWithEntityTypesAndEntityFiltersAndTag() throws AtlasBaseException { + SearchParameters params = new SearchParameters(); + params.setTypeName(DATABASE_TYPE+","+HIVE_TABLE_TYPE); + SearchParameters.FilterCriteria filterCriteria = getSingleFilterCondition("owner", SearchParameters.Operator.CONTAINS, "ETL"); + params.setEntityFilters(filterCriteria); + params.setClassification(LOGDATA_CLASSIFICATION); + params.setLimit(20); + + SearchContext context = new SearchContext(params, typeRegistry, graph, Collections.<String>emptySet()); + + EntitySearchProcessor processor = new EntitySearchProcessor(context); + assertEquals(processor.execute().size(), 2); + } + }
