This is an automated email from the ASF dual-hosted git repository.
sarath pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/master by this push:
new 8b5cb9d ATLAS-3838: Support multiple tag/classification in
basic/quick search API ATLAS-3652: Quick Search: API requirement for GET
request on multiple entity types
8b5cb9d is described below
commit 8b5cb9d9aa9f977ba5dfc455dfe440dd55eace06
Author: Pinal Shah <[email protected]>
AuthorDate: Fri Jun 26 11:24:46 2020 +0530
ATLAS-3838: Support multiple tag/classification in basic/quick search API
ATLAS-3652: Quick Search: API requirement for GET request on multiple
entity types
Signed-off-by: Sarath Subramanian <[email protected]>
both JIRA's addressed in the same commit.
---
.../repository/graphdb/AggregationContext.java | 38 +--
.../graphdb/janus/AtlasJanusGraphIndexClient.java | 2 +-
.../graphdb/janus/AtlasSolrQueryBuilder.java | 151 +++++----
.../graphdb/janus/AtlasSolrQueryBuilderTest.java | 54 +++-
.../src/test/resources/searchparameters2Types.json | 19 ++
.../discovery/ClassificationSearchProcessor.java | 34 +-
.../atlas/discovery/EntityDiscoveryService.java | 8 +-
.../atlas/discovery/EntitySearchProcessor.java | 53 ++--
.../atlas/discovery/FreeTextSearchProcessor.java | 7 +-
.../atlas/discovery/FullTextSearchProcessor.java | 19 +-
.../atlas/discovery/GraphIndexQueryBuilder.java | 25 +-
.../atlas/discovery/SearchAggregatorImpl.java | 7 +-
.../org/apache/atlas/discovery/SearchContext.java | 253 +++++++++++----
.../apache/atlas/discovery/SearchProcessor.java | 343 +++++++++++++--------
.../atlas/discovery/EntitySearchProcessorTest.java | 114 ++++++-
15 files changed, 750 insertions(+), 377 deletions(-)
diff --git
a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AggregationContext.java
b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AggregationContext.java
index 6006fef..efde79b 100644
---
a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AggregationContext.java
+++
b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AggregationContext.java
@@ -25,14 +25,14 @@ import java.util.Map;
import java.util.Set;
public class AggregationContext {
- private final String queryString;
- private final FilterCriteria filterCriteria;
- private final AtlasEntityType searchForEntityType;
- private final Set<String> aggregationFieldNames;
- private final Set<AtlasAttribute> aggregationAttributes;
- private final Map<String, String> indexFieldNameCache;
- private final boolean excludeDeletedEntities;
- private final boolean includeSubTypes;
+ private final String queryString;
+ private final FilterCriteria filterCriteria;
+ private final Set<AtlasEntityType> searchForEntityTypes;
+ private final Set<String> aggregationFieldNames;
+ private final Set<AtlasAttribute> aggregationAttributes;
+ private final Map<String, String> indexFieldNameCache;
+ private final boolean excludeDeletedEntities;
+ private final boolean includeSubTypes;
/**
* @param queryString the query string whose aggregation metrics need to
be retrieved.
@@ -41,17 +41,17 @@ public class AggregationContext {
* @param indexFieldNameCache
* @param excludeDeletedEntities a boolean flag to indicate if the deleted
entities need to be excluded in search
*/
- public AggregationContext(String queryString,
- FilterCriteria filterCriteria,
- AtlasEntityType searchForEntityType,
- Set<String> aggregationFieldNames,
- Set<AtlasAttribute> aggregationAttributes,
- Map<String, String> indexFieldNameCache,
- boolean excludeDeletedEntities,
- boolean includeSubTypes) {
+ public AggregationContext(String queryString,
+ FilterCriteria filterCriteria,
+ Set<AtlasEntityType> searchForEntityType,
+ Set<String> aggregationFieldNames,
+ Set<AtlasAttribute> aggregationAttributes,
+ Map<String, String> indexFieldNameCache,
+ boolean excludeDeletedEntities,
+ boolean includeSubTypes) {
this.queryString = queryString;
this.filterCriteria = filterCriteria;
- this.searchForEntityType = searchForEntityType;
+ this.searchForEntityTypes = searchForEntityType;
this.aggregationFieldNames = aggregationFieldNames;
this.aggregationAttributes = aggregationAttributes;
this.indexFieldNameCache = indexFieldNameCache;
@@ -67,8 +67,8 @@ public class AggregationContext {
return filterCriteria;
}
- public AtlasEntityType getSearchForEntityType() {
- return searchForEntityType;
+ public Set<AtlasEntityType> getSearchForEntityTypes() {
+ return searchForEntityTypes;
}
public Set<String> getAggregationFieldNames() {
diff --git
a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphIndexClient.java
b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphIndexClient.java
index 29bd2a4..8142514 100644
---
a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphIndexClient.java
+++
b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphIndexClient.java
@@ -175,7 +175,7 @@ public class AtlasJanusGraphIndexClient implements
AtlasGraphIndexClient {
Map<String, String> indexFieldName2PropertyKeyNameMap = new
HashMap<>();
AtlasSolrQueryBuilder solrQueryBuilder = new
AtlasSolrQueryBuilder();
-
solrQueryBuilder.withEntityType(aggregationContext.getSearchForEntityType())
+
solrQueryBuilder.withEntityTypes(aggregationContext.getSearchForEntityTypes())
.withQueryString(aggregationContext.getQueryString())
.withCriteria(aggregationContext.getFilterCriteria())
.withExcludedDeletedEntities(aggregationContext.isExcludeDeletedEntities())
diff --git
a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilder.java
b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilder.java
index b8fd4a0..6c06a3c 100644
---
a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilder.java
+++
b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilder.java
@@ -29,31 +29,28 @@ import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
import static
org.apache.atlas.repository.Constants.CUSTOM_ATTRIBUTES_PROPERTY_KEY;
public class AtlasSolrQueryBuilder {
private static final Logger LOG =
LoggerFactory.getLogger(AtlasSolrQueryBuilder.class);
- private AtlasEntityType entityType;
- private String queryString;
- private FilterCriteria criteria;
- private boolean excludeDeletedEntities;
- private boolean includeSubtypes;
- private Map<String, String> indexFieldNameCache;
- public static final char CUSTOM_ATTR_SEPARATOR = '=';
- public static final String CUSTOM_ATTR_SEARCH_FORMAT =
"\"\\\"%s\\\":\\\"%s\\\"\"";
+ private Set<AtlasEntityType> entityTypes;
+ private String queryString;
+ private FilterCriteria criteria;
+ private boolean excludeDeletedEntities;
+ private boolean includeSubtypes;
+ private Map<String, String> indexFieldNameCache;
+ public static final char CUSTOM_ATTR_SEPARATOR = '=';
+ public static final String CUSTOM_ATTR_SEARCH_FORMAT =
"\"\\\"%s\\\":\\\"%s\\\"\"";
public AtlasSolrQueryBuilder() {
}
- public AtlasSolrQueryBuilder withEntityType(AtlasEntityType
searchForEntityType) {
- this.entityType = searchForEntityType;
+ public AtlasSolrQueryBuilder withEntityTypes(Set<AtlasEntityType>
searchForEntityTypes) {
+ this.entityTypes = searchForEntityTypes;
return this;
}
@@ -112,7 +109,7 @@ public class AtlasSolrQueryBuilder {
isAndNeeded = true;
}
- if (entityType != null) {
+ if (CollectionUtils.isNotEmpty(entityTypes)) {
if (isAndNeeded) {
queryBuilder.append(" AND ");
}
@@ -140,29 +137,25 @@ public class AtlasSolrQueryBuilder {
}
private void buildForEntityType(StringBuilder queryBuilder) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Search is being done for entities of type {}",
entityType.getTypeName());
- }
String typeIndexFieldName =
indexFieldNameCache.get(Constants.ENTITY_TYPE_PROPERTY_KEY);
queryBuilder.append(" +")
.append(typeIndexFieldName)
- .append(":(")
- .append(entityType.getTypeName())
- .append(" ");
+ .append(":(");
- if (includeSubtypes) {
- Set<String> allSubTypes = entityType.getAllSubTypes();
+ Set<String> typesToSearch = new HashSet<>();
+ for (AtlasEntityType type : entityTypes) {
- if(allSubTypes.size() != 0 ) {
- for(String subTypeName: allSubTypes) {
- queryBuilder.append(subTypeName).append(" ");
- }
+ if (includeSubtypes) {
+ typesToSearch.addAll(type.getTypeAndAllSubTypes());
+ } else {
+ typesToSearch.add(type.getTypeName());
}
}
- queryBuilder.append(" ) ");
+ queryBuilder.append(StringUtils.join(typesToSearch, " ")).append(" )
");
+
}
private void dropDeletedEntities(StringBuilder queryBuilder) throws
AtlasBaseException {
@@ -173,9 +166,8 @@ public class AtlasSolrQueryBuilder {
String indexFieldName =
indexFieldNameCache.get(Constants.STATE_PROPERTY_KEY);
if (indexFieldName == null) {
- String msg = String.format("There is no index field name defined
for attribute '%s' for entity '%s'",
- Constants.STATE_PROPERTY_KEY,
- entityType.getTypeName());
+ String msg = String.format("There is no index field name defined
for attribute '%s'",
+ Constants.STATE_PROPERTY_KEY);
LOG.error(msg);
@@ -187,10 +179,46 @@ public class AtlasSolrQueryBuilder {
private AtlasSolrQueryBuilder withCriteria(StringBuilder queryBuilder,
FilterCriteria criteria) throws AtlasBaseException {
List<FilterCriteria> criterion = criteria.getCriterion();
+ Set<String> indexAttributes = new HashSet<>();
+ if (StringUtils.isNotEmpty(criteria.getAttributeName()) &&
CollectionUtils.isEmpty(criterion)) { // no child criterion
+
+ String attributeName = criteria.getAttributeName();
+ String attributeValue = criteria.getAttributeValue();
+ Operator operator = criteria.getOperator();
+
+ ArrayList<StringBuilder> orExpQuery = new ArrayList<>();
+
+ for (AtlasEntityType type : entityTypes) {
+ String indexAttributeName = getIndexAttributeName(type,
attributeName);
+
+ //check to remove duplicate attribute query (for eg. name)
+ if (!indexAttributes.contains(indexAttributeName)) {
+ StringBuilder sb = new StringBuilder();
+
+ if (attributeName.equals(CUSTOM_ATTRIBUTES_PROPERTY_KEY)
&& operator.equals(Operator.CONTAINS)) {
+ // CustomAttributes stores key value pairs in String
format, so ideally it should be 'contains' operator to search for one pair,
+ // for use-case, E1 having key1=value1 and E2 having
key1=value2, searching key1=value1 results both E1,E2
+ // surrounding inverted commas to attributeValue works
+ operator = Operator.EQ;
+ attributeValue =
getIndexQueryAttributeValue(attributeValue);
+ }
+
+ withPropertyCondition(sb, indexAttributeName, operator,
attributeValue);
+ indexAttributes.add(indexAttributeName);
+ orExpQuery.add(sb);
+ }
+ }
+
+ if (CollectionUtils.isNotEmpty(orExpQuery)) {
+ if (orExpQuery.size() > 1) {
+ String orExpStr = StringUtils.join(orExpQuery,
FilterCriteria.Condition.OR.name());
+ queryBuilder.append(" ( ").append(orExpStr).append(" ) ");
+ } else {
+ queryBuilder.append(orExpQuery.iterator().next());
+ }
+ }
- if(criterion == null ||
CollectionUtils.isEmpty(criteria.getCriterion())) { // no child criterion
- withPropertyCondition(queryBuilder, criteria.getAttributeName(),
criteria.getOperator(), criteria.getAttributeValue());
- } else {
+ } else if (CollectionUtils.isNotEmpty(criterion)) {
beginCriteria(queryBuilder);
for (Iterator<FilterCriteria> iterator = criterion.iterator();
iterator.hasNext(); ) {
@@ -209,40 +237,12 @@ public class AtlasSolrQueryBuilder {
return this;
}
- private void withPropertyCondition(StringBuilder queryBuilder, String
attributeName, Operator operator, String attributeValue) throws
AtlasBaseException {
- if (StringUtils.isNotEmpty(attributeName) && operator != null) {
+ private void withPropertyCondition(StringBuilder queryBuilder, String
indexFieldName, Operator operator, String attributeValue) throws
AtlasBaseException {
+ if (StringUtils.isNotEmpty(indexFieldName) && operator != null) {
if (attributeValue != null) {
attributeValue = attributeValue.trim();
}
- if (attributeName.equals(CUSTOM_ATTRIBUTES_PROPERTY_KEY) &&
operator.equals(Operator.CONTAINS)) {
- // CustomAttributes stores key value pairs in String format,
so ideally it should be 'contains' operator to search for one pair,
- // for use-case, E1 having key1=value1 and E2 having
key1=value2, searching key1=value1 results both E1,E2
- // surrounding inverted commas to attributeValue works
- operator = Operator.EQ;
- attributeValue = getIndexQueryAttributeValue(attributeValue);
- }
-
- AtlasAttribute attribute = entityType.getAttribute(attributeName);
-
- if (attribute == null) {
- String msg = String.format("Received unknown attribute '%s'
for type '%s'.", attributeName, entityType.getTypeName());
-
- LOG.error(msg);
-
- throw new AtlasBaseException(msg);
- }
-
- String indexFieldName = attribute.getIndexFieldName();
-
- if (indexFieldName == null) {
- String msg = String.format("Received non-index attribute %s
for type %s.", attributeName, entityType.getTypeName());
-
- LOG.error(msg);
-
- throw new AtlasBaseException(msg);
- }
-
beginCriteria(queryBuilder);
switch (operator) {
@@ -308,6 +308,29 @@ public class AtlasSolrQueryBuilder {
return attributeValue;
}
+ private String getIndexAttributeName(AtlasEntityType type, String
attrName) throws AtlasBaseException {
+ AtlasAttribute ret = type.getAttribute(attrName);
+
+ if (ret == null) {
+ String msg = String.format("Received unknown attribute '%s' for
type '%s'.", attrName, type.getTypeName());
+
+ LOG.error(msg);
+
+ throw new AtlasBaseException(msg);
+ }
+
+ String indexFieldName = ret.getIndexFieldName();
+
+ if (indexFieldName == null) {
+ String msg = String.format("Received non-index attribute %s for
type %s.", attrName, type.getTypeName());
+
+ LOG.error(msg);
+
+ throw new AtlasBaseException(msg);
+ }
+
+ return indexFieldName;
+ }
private void beginCriteria(StringBuilder queryBuilder) {
queryBuilder.append("( ");
diff --git
a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilderTest.java
b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilderTest.java
index 60d8610..06c7221 100644
---
a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilderTest.java
+++
b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilderTest.java
@@ -33,8 +33,10 @@ import org.testng.annotations.Test;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import static org.mockito.Mockito.when;
@@ -45,6 +47,9 @@ public class AtlasSolrQueryBuilderTest {
private AtlasEntityType hiveTableEntityTypeMock;
@Mock
+ private AtlasEntityType hiveTableEntityTypeMock2;
+
+ @Mock
private AtlasStructType.AtlasAttribute nameAttributeMock;
@Mock
@@ -95,6 +100,7 @@ public class AtlasSolrQueryBuilderTest {
when(hiveTableEntityTypeMock.getTypeName()).thenReturn("hive_table");
+ when(hiveTableEntityTypeMock2.getTypeName()).thenReturn("hive_db");
when(nameAttributeMock.getIndexFieldName()).thenReturn("name_index");
when(commentAttributeMock.getIndexFieldName()).thenReturn("comment_index");
@@ -114,7 +120,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest);
- Assert.assertEquals(underTest.build(), "+t AND
-__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( (
+name_index:t10 ) OR ( +comment_index:*t10* ) )");
+ Assert.assertEquals(underTest.build(), "+t AND
-__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( (
+name_index:t10 ) OR ( +comment_index:*t10* ) )");
}
@Test
@@ -124,7 +130,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest);
- Assert.assertEquals(underTest.build(), "+t AND
-__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( (
+name_index:t10 ) )");
+ Assert.assertEquals(underTest.build(), "+t AND
-__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( (
+name_index:t10 ) )");
}
@Test
@@ -134,7 +140,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest);
- Assert.assertEquals(underTest.build(), "+t AND
-__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( (
+name_index:t10 ) AND ( +comment_index:*t10* ) )");
+ Assert.assertEquals(underTest.build(), "+t AND
-__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( (
+name_index:t10 ) AND ( +comment_index:*t10* ) )");
}
@Test
@@ -144,7 +150,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest);
- Assert.assertEquals(underTest.build(), "+t AND
-__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( (
+name_index:t10 ) )");
+ Assert.assertEquals(underTest.build(), "+t AND
-__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( (
+name_index:t10 ) )");
}
@Test
@@ -154,7 +160,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest);
- Assert.assertEquals(underTest.build(), "+t AND
-__state_index:DELETED AND +__typeName__index:(hive_table ) AND (
+name_index:t10 )");
+ Assert.assertEquals(underTest.build(), "+t AND
-__state_index:DELETED AND +__typeName__index:(hive_table ) AND (
+name_index:t10 )");
}
@Test
@@ -164,7 +170,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest);
- Assert.assertEquals(underTest.build(), "+t10 AND
-__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( (
+comment_index:*United States* ) AND ( +descrption__index:*nothing* ) AND (
+name_index:*t100* ) )");
+ Assert.assertEquals(underTest.build(), "+t10 AND
-__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( (
+comment_index:*United States* ) AND ( +descrption__index:*nothing* ) AND (
+name_index:*t100* ) )");
}
@Test
@@ -174,7 +180,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest);
- Assert.assertEquals(underTest.build(), "+t10 AND
-__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( (
+created__index:{ 100 TO * ] ) )");
+ Assert.assertEquals(underTest.build(), "+t10 AND
-__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( (
+created__index:{ 100 TO * ] ) )");
}
@Test
@@ -184,7 +190,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest);
- Assert.assertEquals(underTest.build(), "+t10 AND
-__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( (
+created__index:[ 100 TO * ] ) AND ( +started__index:[ 100 TO * ] ) )");
+ Assert.assertEquals(underTest.build(), "+t10 AND
-__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( (
+created__index:[ 100 TO * ] ) AND ( +started__index:[ 100 TO * ] ) )");
}
@Test
@@ -194,7 +200,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest);
- Assert.assertEquals(underTest.build(), "+t10 AND
-__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( (
+created__index:[ * TO100} ) )");
+ Assert.assertEquals(underTest.build(), "+t10 AND
-__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( (
+created__index:[ * TO100} ) )");
}
@Test
@@ -204,7 +210,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest);
- Assert.assertEquals(underTest.build(), "+t10 AND
-__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( (
+created__index:[ * TO 100 ] ) AND ( +started__index:[ * TO 100 ] ) )");
+ Assert.assertEquals(underTest.build(), "+t10 AND
-__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( (
+created__index:[ * TO 100 ] ) AND ( +started__index:[ * TO 100 ] ) )");
}
@Test
@@ -214,9 +220,18 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest);
- Assert.assertEquals(underTest.build(), " -__state_index:DELETED AND
+__typeName__index:(hive_table ) AND ( ( +qualifiedName__index:testdb.t1* )
)");
+ Assert.assertEquals(underTest.build(), " -__state_index:DELETED AND
+__typeName__index:(hive_table ) AND ( ( +qualifiedName__index:testdb.t1* )
)");
}
+ @Test
+ public void testGenerateSolrQueryString2TypeNames() throws IOException,
AtlasBaseException {
+ final String fileName =
"src/test/resources/searchparameters2Types.json";
+ AtlasSolrQueryBuilder underTest = new AtlasSolrQueryBuilder();
+
+ processSearchParametersForMultipleTypeNames(fileName, underTest);
+
+ Assert.assertEquals(underTest.build(), "+t AND
-__state_index:DELETED AND +__typeName__index:(hive_table hive_db ) ");
+ }
@@ -243,8 +258,23 @@ public class AtlasSolrQueryBuilderTest {
ObjectMapper mapper = new ObjectMapper();
SearchParameters searchParameters = mapper.readValue(new
FileInputStream(fileName), SearchParameters.class);
+ Set<AtlasEntityType> hiveTableEntityTypeMocks = new HashSet<>();
+ hiveTableEntityTypeMocks.add(hiveTableEntityTypeMock);
+ underTest.withEntityTypes(hiveTableEntityTypeMocks)
+ .withQueryString(searchParameters.getQuery())
+ .withCriteria(searchParameters.getEntityFilters())
+
.withExcludedDeletedEntities(searchParameters.getExcludeDeletedEntities())
+ .withCommonIndexFieldNames(indexFieldNamesMap);
+ }
+
+ private void processSearchParametersForMultipleTypeNames(String fileName,
AtlasSolrQueryBuilder underTest) throws IOException, AtlasBaseException {
+ ObjectMapper mapper = new ObjectMapper();
+ SearchParameters searchParameters = mapper.readValue(new
FileInputStream(fileName), SearchParameters.class);
- underTest.withEntityType(hiveTableEntityTypeMock)
+ Set<AtlasEntityType> hiveTableEntityTypeMocks = new HashSet<>();
+ hiveTableEntityTypeMocks.add(hiveTableEntityTypeMock);
+ hiveTableEntityTypeMocks.add(hiveTableEntityTypeMock2);
+ underTest.withEntityTypes(hiveTableEntityTypeMocks)
.withQueryString(searchParameters.getQuery())
.withCriteria(searchParameters.getEntityFilters())
.withExcludedDeletedEntities(searchParameters.getExcludeDeletedEntities())
diff --git a/graphdb/janus/src/test/resources/searchparameters2Types.json
b/graphdb/janus/src/test/resources/searchparameters2Types.json
new file mode 100644
index 0000000..77fc70f
--- /dev/null
+++ b/graphdb/janus/src/test/resources/searchparameters2Types.json
@@ -0,0 +1,19 @@
+
+{
+ "excludeDeletedEntities":true,
+ "includeSubClassifications":true,
+ "includeSubTypes":true,
+ "includeClassificationAttributes":true,
+ "entityFilters":{
+ },
+ "tagFilters":null,
+ "attributes":[
+ "comment"
+ ],
+ "query":"t",
+ "limit":25,
+ "offset":0,
+ "typeName":"hive_table,hive_db",
+ "classification":null,
+ "termName":null
+}
diff --git
a/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
b/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
index 5dd0d7f..9c72cd4 100644
---
a/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
+++
b/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
@@ -67,17 +67,16 @@ public class ClassificationSearchProcessor extends
SearchProcessor {
public ClassificationSearchProcessor(SearchContext context) {
super(context);
- final AtlasClassificationType classificationType =
context.getClassificationType();
final FilterCriteria filterCriteria =
context.getSearchParameters().getTagFilters();
final Set<String> indexAttributes = new HashSet<>();
final Set<String> graphAttributes = new HashSet<>();
final Set<String> allAttributes = new HashSet<>();
- final Set<String> typeAndSubTypes =
context.getClassificationTypes();
+ final Set<String> typeAndSubTypes =
context.getClassificationTypeNames();
final String typeAndSubTypesQryStr =
context.getClassificationTypesQryStr();
- final boolean isBuiltInType =
context.isBuiltInClassificationType();
final boolean isWildcardSearch =
context.isWildCardSearch();
+ final Set<AtlasClassificationType> classificationTypes =
context.getClassificationTypes();
- processSearchAttributes(classificationType, filterCriteria,
indexAttributes, graphAttributes, allAttributes);
+ processSearchAttributes(classificationTypes, filterCriteria,
indexAttributes, graphAttributes, allAttributes);
/* for classification search, if any attribute can't be handled by
index query - switch to all filter by Graph query
There are four cases in the classification type :
@@ -87,19 +86,25 @@ public class ClassificationSearchProcessor extends
SearchProcessor {
4. classification is not present in the search parameter
each of above cases with either has empty/or not tagFilters
*/
- final boolean useIndexSearchForEntity = (classificationType != null ||
isWildcardSearch) &&
+ final boolean useIndexSearchForEntity =
(CollectionUtils.isNotEmpty(classificationTypes) || isWildcardSearch) &&
!context.hasAttributeFilter(filterCriteria) &&
(typeAndSubTypesQryStr.length() <= MAX_QUERY_STR_LENGTH_TAGS);
/* If classification's attributes can be applied index filter, we can
use direct index
* to query classification index as well.
*/
- final boolean useIndexSearchForClassification = (!isBuiltInType &&
!isWildcardSearch) &&
+ final boolean useIndexSearchForClassification =
(CollectionUtils.isNotEmpty(classificationTypes) &&
+
classificationTypes.iterator().next() != SearchContext.MATCH_ALL_NOT_CLASSIFIED
&&
+ !isWildcardSearch) &&
(typeAndSubTypesQryStr.length() <= MAX_QUERY_STR_LENGTH_TAGS) &&
CollectionUtils.isNotEmpty(indexAttributes) &&
-
canApplyIndexFilter(classificationType, filterCriteria, false);
+
canApplyIndexFilter(classificationTypes, filterCriteria, false);
- traitPredicate = buildTraitPredict(classificationType);
+ final boolean useGraphSearchForClassification =
(CollectionUtils.isNotEmpty(classificationTypes) &&
+
classificationTypes.iterator().next() != SearchContext.MATCH_ALL_NOT_CLASSIFIED
&&
+ !isWildcardSearch &&
CollectionUtils.isNotEmpty(graphAttributes));
+
+ traitPredicate = buildTraitPredict(classificationTypes);
isEntityPredicate =
SearchPredicateUtil.generateIsEntityVertexPredicate(context.getTypeRegistry());
AtlasGraph graph = context.getGraph();
@@ -115,8 +120,7 @@ public class ClassificationSearchProcessor extends
SearchProcessor {
// tagFilters is not allowed in wildcard search
graphIndexQueryBuilder.addClassificationTypeFilter(queryString);
} else {
- if (isBuiltInType) {
-
+ if (classificationTypes.iterator().next() ==
SearchContext.MATCH_ALL_NOT_CLASSIFIED) {
// tagFilters is not allowed in unique classificationType
search
graphIndexQueryBuilder.addClassificationFilterForBuiltInTypes(queryString);
@@ -146,7 +150,7 @@ public class ClassificationSearchProcessor extends
SearchProcessor {
graphIndexQueryBuilder.addActiveStateQueryFilter(queryString);
graphIndexQueryBuilder.addTypeAndSubTypesQueryFilter(queryString,
typeAndSubTypesQryStr);
- constructFilterQuery(queryString, classificationType,
filterCriteria, indexAttributes);
+ constructFilterQuery(queryString, classificationTypes,
filterCriteria, indexAttributes);
String indexQueryString =
STRAY_AND_PATTERN.matcher(queryString).replaceAll(")");
indexQueryString =
STRAY_OR_PATTERN.matcher(indexQueryString).replaceAll(")");
@@ -158,7 +162,7 @@ public class ClassificationSearchProcessor extends
SearchProcessor {
inMemoryPredicate = inMemoryPredicate == null ?
typeNamePredicate : PredicateUtils.andPredicate(inMemoryPredicate,
typeNamePredicate);
}
- Predicate attributePredicate =
constructInMemoryPredicate(classificationType, filterCriteria, indexAttributes);
+ Predicate attributePredicate =
constructInMemoryPredicate(classificationTypes, filterCriteria,
indexAttributes);
if (attributePredicate != null) {
inMemoryPredicate = inMemoryPredicate == null ?
attributePredicate : PredicateUtils.andPredicate(inMemoryPredicate,
attributePredicate);
@@ -170,7 +174,7 @@ public class ClassificationSearchProcessor extends
SearchProcessor {
}
// only registered classification will search with tag filters
- if (!isWildcardSearch && !isBuiltInType && !graphAttributes.isEmpty())
{
+ if (useGraphSearchForClassification) {
AtlasGremlinQueryProvider queryProvider =
AtlasGremlinQueryProvider.INSTANCE;
AtlasGraphQuery query = graph.query();
@@ -179,7 +183,7 @@ public class ClassificationSearchProcessor extends
SearchProcessor {
query.in(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes);
}
- tagGraphQueryWithAttributes =
toGraphFilterQuery(classificationType, filterCriteria, allAttributes, query);
+ tagGraphQueryWithAttributes =
toGraphFilterQuery(classificationTypes, filterCriteria, allAttributes, query);
gremlinQueryBindings = new HashMap<>();
StringBuilder gremlinQuery = new StringBuilder();
@@ -188,7 +192,7 @@ public class ClassificationSearchProcessor extends
SearchProcessor {
gremlinQuery.append(".as('e').filter(out()");
gremlinQuery.append(queryProvider.getQuery(AtlasGremlinQueryProvider.AtlasGremlinQuery.BASIC_SEARCH_TYPE_FILTER));
- constructGremlinFilterQuery(gremlinQuery, gremlinQueryBindings,
context.getClassificationType(), context.getSearchParameters().getTagFilters());
+ // constructGremlinFilterQuery(gremlinQuery, gremlinQueryBindings,
context.getClassificationType(), context.getSearchParameters().getTagFilters());
// After filtering on tags go back to e and output the list of
entity vertices
gremlinQuery.append(").toList()");
diff --git
a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
index dd4d1b4..b2737af 100644
---
a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
+++
b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
@@ -486,9 +486,11 @@ public class EntityDiscoveryService implements
AtlasDiscoveryService {
resultAttributes.addAll(searchContext.getEntityAttributes());
}
- AtlasEntityType entityType = searchContext.getEntityType();
- if (entityType != null) {
- for (String resultAttribute : resultAttributes) {
+ if (CollectionUtils.isNotEmpty(searchContext.getEntityTypes())) {
+
+ AtlasEntityType entityType =
searchContext.getEntityTypes().iterator().next();
+
+ for (String resultAttribute : resultAttributes) {
AtlasAttribute attribute =
entityType.getAttribute(resultAttribute);
if (attribute == null) {
diff --git
a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
index 56956e6..5dcff3b 100644
---
a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
+++
b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
@@ -31,27 +31,15 @@ import org.apache.atlas.utils.AtlasPerfTracer;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.Predicate;
import org.apache.commons.collections.PredicateUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.tinkerpop.gremlin.process.traversal.Order;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Set;
+import java.util.*;
import java.util.stream.StreamSupport;
import static org.apache.atlas.SortOrder.ASCENDING;
-import static
org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFICATION_TYPES;
-import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFIED;
-import static
org.apache.atlas.discovery.SearchContext.MATCH_ALL_NOT_CLASSIFIED;
-import static
org.apache.atlas.discovery.SearchContext.MATCH_ALL_WILDCARD_CLASSIFICATION;
-import static
org.apache.atlas.repository.Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY;
-import static org.apache.atlas.repository.Constants.TRAIT_NAMES_PROPERTY_KEY;
-import static org.apache.atlas.repository.Constants.TYPE_NAME_PROPERTY_KEY;
+import static org.apache.atlas.discovery.SearchContext.*;
+import static org.apache.atlas.repository.Constants.*;
import static
org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator.EQUAL;
import static
org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator.NOT_EQUAL;
import static
org.apache.atlas.repository.graphdb.AtlasGraphQuery.SortOrder.ASC;
@@ -69,28 +57,28 @@ public class EntitySearchProcessor extends SearchProcessor {
public EntitySearchProcessor(SearchContext context) {
super(context);
- final AtlasEntityType entityType = context.getEntityType();
+ final Set<AtlasEntityType> entityTypes = context.getEntityTypes();
final FilterCriteria filterCriteria =
context.getSearchParameters().getEntityFilters();
final Set<String> indexAttributes = new HashSet<>();
final Set<String> graphAttributes = new HashSet<>();
final Set<String> allAttributes = new HashSet<>();
- final Set<String> typeAndSubTypes = context.getEntityTypes();
+ final Set<String> typeAndSubTypes =
context.getEntityTypeNames();
final String typeAndSubTypesQryStr =
context.getEntityTypesQryStr();
final String sortBy =
context.getSearchParameters().getSortBy();
final SortOrder sortOrder =
context.getSearchParameters().getSortOrder();
- final AtlasClassificationType classificationType =
context.getClassificationType();
- final Set<String> classificationTypeAndSubTypes =
context.getClassificationTypes();
- final boolean filterClassification;
+ final Set<AtlasClassificationType> classificationTypes =
context.getClassificationTypes();
+ final Set<String> classificationTypeAndSubTypes =
context.getClassificationTypeNames();
+ final boolean filterClassification;
- if (classificationType != null) {
+ if (CollectionUtils.isNotEmpty(classificationTypes)) {
filterClassification = !context.needClassificationProcessor();
} else {
filterClassification = false;
}
final Predicate typeNamePredicate;
- final Predicate traitPredicate =
buildTraitPredict(classificationType);
+ final Predicate traitPredicate =
buildTraitPredict(classificationTypes);
final Predicate activePredicate =
SearchPredicateUtil.getEQPredicateGenerator()
.generatePredicate(Constants.STATE_PROPERTY_KEY, "ACTIVE", String.class);
@@ -101,10 +89,10 @@ public class EntitySearchProcessor extends SearchProcessor
{
typeNamePredicate =
SearchPredicateUtil.generateIsEntityVertexPredicate(context.getTypeRegistry());
}
- processSearchAttributes(entityType, filterCriteria, indexAttributes,
graphAttributes, allAttributes);
+ processSearchAttributes(entityTypes, filterCriteria, indexAttributes,
graphAttributes, allAttributes);
final boolean typeSearchByIndex = !filterClassification &&
typeAndSubTypesQryStr.length() <= MAX_QUERY_STR_LENGTH_TYPES;
- final boolean attrSearchByIndex = !filterClassification &&
CollectionUtils.isNotEmpty(indexAttributes) && canApplyIndexFilter(entityType,
filterCriteria, false);
+ final boolean attrSearchByIndex = !filterClassification &&
CollectionUtils.isNotEmpty(indexAttributes) && canApplyIndexFilter(entityTypes,
filterCriteria, false);
StringBuilder indexQuery = new StringBuilder();
@@ -116,9 +104,9 @@ public class EntitySearchProcessor extends SearchProcessor {
}
if (attrSearchByIndex) {
- constructFilterQuery(indexQuery, entityType, filterCriteria,
indexAttributes);
+ constructFilterQuery(indexQuery, entityTypes, filterCriteria,
indexAttributes);
- Predicate attributePredicate =
constructInMemoryPredicate(entityType, filterCriteria, indexAttributes);
+ Predicate attributePredicate =
constructInMemoryPredicate(entityTypes, filterCriteria, indexAttributes);
if (attributePredicate != null) {
inMemoryPredicate =
PredicateUtils.andPredicate(inMemoryPredicate, attributePredicate);
}
@@ -149,6 +137,7 @@ public class EntitySearchProcessor extends SearchProcessor {
// If we need to filter on the trait names then we need to build
the query and equivalent in-memory predicate
if (filterClassification) {
+ AtlasClassificationType classificationType =
classificationTypes.iterator().next();
List<AtlasGraphQuery> orConditions = new LinkedList<>();
if (classificationType == MATCH_ALL_WILDCARD_CLASSIFICATION ||
classificationType == MATCH_ALL_CLASSIFIED || classificationType ==
MATCH_ALL_CLASSIFICATION_TYPES) {
@@ -176,10 +165,10 @@ public class EntitySearchProcessor extends
SearchProcessor {
}
}
- graphQuery = toGraphFilterQuery(entityType, filterCriteria,
graphAttributes, query);
+ graphQuery = toGraphFilterQuery(entityTypes, filterCriteria,
graphAttributes, query);
// Prepare in-memory predicate for attribute filtering
- Predicate attributePredicate =
constructInMemoryPredicate(entityType, filterCriteria, graphAttributes);
+ Predicate attributePredicate =
constructInMemoryPredicate(entityTypes, filterCriteria, graphAttributes);
if (attributePredicate != null) {
if (graphQueryPredicate != null) {
@@ -199,7 +188,8 @@ public class EntitySearchProcessor extends SearchProcessor {
}
}
if (sortBy != null && !sortBy.isEmpty()) {
- AtlasAttribute sortByAttribute =
context.getEntityType().getAttribute(sortBy);
+ final AtlasEntityType entityType =
context.getEntityTypes().iterator().next();
+ AtlasAttribute sortByAttribute =
entityType.getAttribute(sortBy);
if (sortByAttribute != null) {
AtlasGraphQuery.SortOrder qrySortOrder = sortOrder ==
SortOrder.ASCENDING ? ASC : DESC;
@@ -215,8 +205,7 @@ public class EntitySearchProcessor extends SearchProcessor {
// Prepare the graph query and in-memory filter for the filtering phase
filterGraphQueryPredicate = typeNamePredicate;
-
- Predicate attributesPredicate = constructInMemoryPredicate(entityType,
filterCriteria, allAttributes);
+ Predicate attributesPredicate =
constructInMemoryPredicate(entityTypes, filterCriteria, allAttributes);
if (attributesPredicate != null) {
filterGraphQueryPredicate = filterGraphQueryPredicate == null ?
attributesPredicate :
@@ -265,7 +254,7 @@ public class EntitySearchProcessor extends SearchProcessor {
SortOrder sortOrder = context.getSearchParameters().getSortOrder();
String sortBy = context.getSearchParameters().getSortBy();
- final AtlasEntityType entityType = context.getEntityType();
+ final AtlasEntityType entityType =
context.getEntityTypes().iterator().next();
AtlasAttribute sortByAttribute = entityType.getAttribute(sortBy);
if (sortByAttribute == null) {
sortBy = null;
diff --git
a/repository/src/main/java/org/apache/atlas/discovery/FreeTextSearchProcessor.java
b/repository/src/main/java/org/apache/atlas/discovery/FreeTextSearchProcessor.java
index 6e3a760..d9981e6 100644
---
a/repository/src/main/java/org/apache/atlas/discovery/FreeTextSearchProcessor.java
+++
b/repository/src/main/java/org/apache/atlas/discovery/FreeTextSearchProcessor.java
@@ -18,7 +18,6 @@
package org.apache.atlas.discovery;
import org.apache.atlas.model.discovery.SearchParameters;
-import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.*;
@@ -53,13 +52,13 @@ public class FreeTextSearchProcessor extends
SearchProcessor {
queryString.append(searchParameters.getQuery());
- if (CollectionUtils.isNotEmpty(context.getEntityTypes()) &&
context.getEntityTypesQryStr().length() <= MAX_QUERY_STR_LENGTH_TYPES) {
+ if (CollectionUtils.isNotEmpty(context.getEntityTypeNames()) &&
context.getEntityTypesQryStr().length() <= MAX_QUERY_STR_LENGTH_TYPES) {
queryString.append(AND_STR).append(context.getEntityTypesQryStr());
}
graphIndexQueryBuilder.addActiveStateQueryFilter(queryString);
- if (CollectionUtils.isNotEmpty(context.getClassificationTypes()) &&
context.getClassificationTypesQryStr().length() <= MAX_QUERY_STR_LENGTH_TYPES) {
+ if (CollectionUtils.isNotEmpty(context.getClassificationTypeNames())
&& context.getClassificationTypesQryStr().length() <=
MAX_QUERY_STR_LENGTH_TYPES) {
queryString.append(AND_STR).append(context.getClassificationTypesQryStr());
}
@@ -138,7 +137,7 @@ public class FreeTextSearchProcessor extends
SearchProcessor {
continue;
}
- if (context.getClassificationType() != null) {
+ if
(CollectionUtils.isNotEmpty(context.getClassificationTypes())) {
List<String> entityClassifications =
GraphHelper.getAllTraitNames(vertex);
if
(!context.includeClassificationTypes(entityClassifications)) {
diff --git
a/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
b/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
index 99cb1d0..b37d93a 100644
---
a/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
+++
b/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
@@ -24,6 +24,7 @@ import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2;
import org.apache.atlas.utils.AtlasPerfTracer;
+import org.apache.commons.collections.CollectionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -31,10 +32,7 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
-import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFIED;
-import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_ENTITY_TYPES;
import static
org.apache.atlas.discovery.SearchContext.MATCH_ALL_NOT_CLASSIFIED;
-import static
org.apache.atlas.discovery.SearchContext.MATCH_ALL_WILDCARD_CLASSIFICATION;
public class FullTextSearchProcessor extends SearchProcessor {
@@ -53,29 +51,28 @@ public class FullTextSearchProcessor extends
SearchProcessor {
// if search includes entity-type criteria, adding a filter here can
help avoid unnecessary
// processing (and rejection) by subsequent EntitySearchProcessor
- if (context.getEntityType() != null && context.getEntityType() !=
MATCH_ALL_ENTITY_TYPES) {
- String typeAndSubTypeNamesQryStr =
context.getEntityType().getTypeAndAllSubTypesQryStr();
+ if (CollectionUtils.isNotEmpty(context.getEntityTypes())) {
+ String typeAndSubTypeNamesQryStr = context.getEntityTypesQryStr();
if (typeAndSubTypeNamesQryStr.length() <=
MAX_QUERY_STR_LENGTH_TYPES) {
queryString.append(AND_STR).append(typeAndSubTypeNamesQryStr);
} else {
LOG.warn("'{}' has too many subtypes (query-string-length={})
to include in index-query; might cause poor performance",
- context.getEntityType().getTypeName(),
typeAndSubTypeNamesQryStr.length());
+ searchParameters.getTypeName(),
typeAndSubTypeNamesQryStr.length());
}
}
// if search includes classification criteria, adding a filter here
can help avoid unnecessary
// processing (and rejection) by subsequent
ClassificationSearchProcessor or EntitySearchProcessor
- if (context.getClassificationType() != null &&
context.getClassificationType() != MATCH_ALL_WILDCARD_CLASSIFICATION &&
-
context.getClassificationType() != MATCH_ALL_CLASSIFIED &&
-
context.getClassificationType() != MATCH_ALL_NOT_CLASSIFIED) {
- String typeAndSubTypeNamesStr =
context.getClassificationType().getTypeAndAllSubTypesQryStr();
+ if (CollectionUtils.isNotEmpty(context.getClassificationTypes()) &&
+
context.getClassificationTypes().iterator().next() != MATCH_ALL_NOT_CLASSIFIED)
{
+ String typeAndSubTypeNamesStr =
context.getClassificationTypesQryStr();
if (typeAndSubTypeNamesStr.length() <= MAX_QUERY_STR_LENGTH_TAGS) {
queryString.append(AND_STR).append(typeAndSubTypeNamesStr);
} else {
LOG.warn("'{}' has too many subtypes (query-string-length={})
to include in index-query; might cause poor performance",
- context.getClassificationType().getTypeName(),
typeAndSubTypeNamesStr.length());
+ searchParameters.getClassification(),
typeAndSubTypeNamesStr.length());
}
}
diff --git
a/repository/src/main/java/org/apache/atlas/discovery/GraphIndexQueryBuilder.java
b/repository/src/main/java/org/apache/atlas/discovery/GraphIndexQueryBuilder.java
index 28d2086..35d64b7 100644
---
a/repository/src/main/java/org/apache/atlas/discovery/GraphIndexQueryBuilder.java
+++
b/repository/src/main/java/org/apache/atlas/discovery/GraphIndexQueryBuilder.java
@@ -26,6 +26,8 @@ import static
org.apache.atlas.repository.Constants.PROPAGATED_CLASSIFICATION_NA
import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY;
import org.apache.atlas.repository.Constants;
+import org.apache.atlas.type.AtlasStructType;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
public class GraphIndexQueryBuilder {
@@ -36,20 +38,20 @@ public class GraphIndexQueryBuilder {
}
void addClassificationTypeFilter(StringBuilder indexQuery) {
- if (indexQuery != null &&
StringUtils.isNotEmpty(context.getSearchParameters().getClassification())) {
- String classificationName =
context.getSearchParameters().getClassification();
+ if (indexQuery != null &&
CollectionUtils.isNotEmpty(context.getClassificationNames())) {
+ String classificationNames =
AtlasStructType.AtlasAttribute.escapeIndexQueryValue(context.getClassificationNames());
if (indexQuery.length() != 0) {
indexQuery.append(" AND ");
}
-
indexQuery.append("(").append(INDEX_SEARCH_PREFIX).append('\"').append(CLASSIFICATION_NAMES_KEY).append('\"').append(':').append(classificationName)
+
indexQuery.append("(").append(INDEX_SEARCH_PREFIX).append('\"').append(CLASSIFICATION_NAMES_KEY).append('\"').append(':').append(classificationNames)
.append(" OR
").append(INDEX_SEARCH_PREFIX).append('\"').append(PROPAGATED_CLASSIFICATION_NAMES_KEY)
-
.append('\"').append(':').append(classificationName).append(")");
+
.append('\"').append(':').append(classificationNames).append(")");
}
}
void addClassificationAndSubTypesQueryFilter(StringBuilder indexQuery) {
- if (indexQuery != null &&
StringUtils.isNotEmpty(context.getSearchParameters().getClassification())) {
+ if (indexQuery != null &&
CollectionUtils.isNotEmpty(context.getClassificationTypes())) {
String classificationTypesQryStr =
context.getClassificationTypesQryStr();
if (indexQuery.length() != 0) {
@@ -63,17 +65,8 @@ public class GraphIndexQueryBuilder {
}
void addClassificationFilterForBuiltInTypes(StringBuilder indexQuery) {
- if (indexQuery != null && context.getClassificationType() != null) {
- if (context.getClassificationType() ==
MATCH_ALL_WILDCARD_CLASSIFICATION || context.getClassificationType() ==
MATCH_ALL_CLASSIFIED) {
- if (indexQuery.length() != 0) {
- indexQuery.append(" AND ");
- }
- indexQuery.append("(").append(INDEX_SEARCH_PREFIX).append("\"")
- .append(CLASSIFICATION_NAMES_KEY).append("\"").append(":"
+ "[* TO *]")
- .append(" OR ").append(INDEX_SEARCH_PREFIX).append("\"")
-
.append(PROPAGATED_CLASSIFICATION_NAMES_KEY).append("\"").append(":" + "[* TO
*]").append(")");
-
- } else if (context.getClassificationType() ==
MATCH_ALL_NOT_CLASSIFIED) {
+ if (indexQuery != null &&
CollectionUtils.isNotEmpty(context.getClassificationTypes())) {
+ if (context.getClassificationTypes().iterator().next() ==
MATCH_ALL_NOT_CLASSIFIED) {
if (indexQuery.length() != 0) {
indexQuery.append(" AND ");
}
diff --git
a/repository/src/main/java/org/apache/atlas/discovery/SearchAggregatorImpl.java
b/repository/src/main/java/org/apache/atlas/discovery/SearchAggregatorImpl.java
index e8f7dbc..2d095be 100644
---
a/repository/src/main/java/org/apache/atlas/discovery/SearchAggregatorImpl.java
+++
b/repository/src/main/java/org/apache/atlas/discovery/SearchAggregatorImpl.java
@@ -56,12 +56,7 @@ public class SearchAggregatorImpl implements
SearchAggregator {
try {
AtlasGraphIndexClient graphIndexClient =
graph.getGraphIndexClient();
- String searchedOnTypeName =
searchParameters.getTypeName();
- AtlasEntityType searchForEntityType = null;
-
- if (searchedOnTypeName != null) {
- searchForEntityType =
typeRegistry.getEntityTypeByName(searchedOnTypeName);
- }
+ Set<AtlasEntityType> searchForEntityType =
searchContext.getEntityTypes();
Map<String, String> indexFieldNameCache = new HashMap<>();
diff --git
a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
index 2c2888e..04e8218 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
@@ -21,7 +21,7 @@ package org.apache.atlas.discovery;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.discovery.SearchParameters;
-import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
+import org.apache.atlas.model.discovery.SearchParameters.*;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.typedef.AtlasClassificationDef;
import org.apache.atlas.repository.Constants;
@@ -44,13 +44,10 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.*;
+import java.util.stream.Collectors;
import static org.apache.atlas.discovery.SearchProcessor.ALL_TYPE_QUERY;
-import static
org.apache.atlas.model.discovery.SearchParameters.ALL_CLASSIFICATIONS;
-import static
org.apache.atlas.model.discovery.SearchParameters.ALL_CLASSIFICATION_TYPES;
-import static
org.apache.atlas.model.discovery.SearchParameters.ALL_ENTITY_TYPES;
-import static
org.apache.atlas.model.discovery.SearchParameters.NO_CLASSIFICATIONS;
-import static
org.apache.atlas.model.discovery.SearchParameters.WILDCARD_CLASSIFICATIONS;
+import static org.apache.atlas.model.discovery.SearchParameters.*;
/*
* Search context captures elements required for performing a basic search
@@ -62,12 +59,12 @@ public class SearchContext {
private final AtlasTypeRegistry typeRegistry;
private final AtlasGraph graph;
- private final AtlasEntityType entityType;
+ private final Set<AtlasEntityType> entityTypes;
private final Set<String> indexedKeys;
private final Set<String> entityAttributes;
private final SearchParameters searchParameters;
- private final AtlasClassificationType classificationType;
- private final String classificationName;
+ private final Set<AtlasClassificationType> classificationTypes;
+ private final Set<String> classificationNames;
private final Set<String> typeAndSubTypes;
private final Set<String> classificationTypeAndSubTypes;
private final String typeAndSubTypesQryStr;
@@ -80,27 +77,18 @@ public class SearchContext {
public final static AtlasClassificationType MATCH_ALL_NOT_CLASSIFIED
= new AtlasClassificationType(new
AtlasClassificationDef(NO_CLASSIFICATIONS));
public final static AtlasClassificationType MATCH_ALL_CLASSIFICATION_TYPES
= AtlasClassificationType.getClassificationRoot();
public final static AtlasEntityType MATCH_ALL_ENTITY_TYPES
= AtlasEntityType.getEntityRoot();
+ public final static String TYPENAME_DELIMITER
= ",";
public SearchContext(SearchParameters searchParameters, AtlasTypeRegistry
typeRegistry, AtlasGraph graph, Set<String> indexedKeys) throws
AtlasBaseException {
- this.classificationName = searchParameters.getClassification();
this.searchParameters = searchParameters;
this.typeRegistry = typeRegistry;
this.graph = graph;
this.indexedKeys = indexedKeys;
this.entityAttributes = new HashSet<>();
- this.entityType =
getEntityType(searchParameters.getTypeName());
- this.classificationType = getClassificationType(classificationName);
-
- // Validate if the type name exists
- if (StringUtils.isNotEmpty(searchParameters.getTypeName()) &&
entityType == null) {
- throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_TYPENAME,
searchParameters.getTypeName());
- }
-
- // Validate if the classification exists
- if ((StringUtils.isNotEmpty(classificationName) && classificationType
== null && !classificationName.contains(WILDCARD_CLASSIFICATIONS))) {
- throw new
AtlasBaseException(AtlasErrorCode.UNKNOWN_CLASSIFICATION, classificationName);
- }
+ this.entityTypes =
getEntityTypes(searchParameters.getTypeName());
+ this.classificationNames =
getClassificationNames(searchParameters.getClassification());
+ this.classificationTypes =
getClassificationTypes(this.classificationNames);
AtlasVertex glossaryTermVertex =
getGlossaryTermVertex(searchParameters.getTermName());
@@ -109,40 +97,77 @@ public class SearchContext {
throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_GLOSSARY_TERM,
searchParameters.getTermName());
}
- // Invalid attributes will raise an exception with 400 error code
- validateAttributes(entityType, searchParameters.getEntityFilters());
+ // Invalid attributes or unsupported attribute in a type, will raise
an exception with 400 error code
+ if (CollectionUtils.isNotEmpty(entityTypes)) {
+ for (AtlasEntityType entityType : entityTypes) {
+ validateAttributes(entityType,
searchParameters.getEntityFilters());
- // Invalid attribute will raise an exception with 400 error code
- validateAttributes(entityType, searchParameters.getSortBy());
+ validateAttributes(entityType, searchParameters.getSortBy());
+ }
+ }
// Invalid attributes will raise an exception with 400 error code
- validateAttributes(classificationType,
searchParameters.getTagFilters());
+ if (CollectionUtils.isNotEmpty(classificationTypes)) {
+ for (AtlasClassificationType classificationType :
classificationTypes) {
+ validateAttributes(classificationType,
searchParameters.getTagFilters());
+ }
+ }
- if (classificationType != null && !isBuiltInClassificationType()) {
- if (classificationType == MATCH_ALL_CLASSIFICATION_TYPES) {
- classificationTypeAndSubTypes = Collections.emptySet();
- classificationTypeAndSubTypesQryStr = ALL_TYPE_QUERY;
- } else {
- classificationTypeAndSubTypes =
searchParameters.getIncludeSubClassifications() ?
classificationType.getTypeAndAllSubTypes() :
Collections.singleton(classificationType.getTypeName());
- classificationTypeAndSubTypesQryStr =
searchParameters.getIncludeSubClassifications() ?
classificationType.getTypeAndAllSubTypesQryStr() :
classificationType.getTypeQryStr();
+ //remove other types if builtin type is present
+ filterStructTypes();
+
+ //gather all classifications and its corresponding subtypes
+ Set<String> classificationTypeAndSubTypes = new HashSet<>();
+ String classificationTypeAndSubTypesQryStr = null;
+
+ if (CollectionUtils.isNotEmpty(classificationTypes) &&
classificationTypes.iterator().next() != MATCH_ALL_NOT_CLASSIFIED ) {
+ for (AtlasClassificationType classificationType :
classificationTypes) {
+
+ if (classificationType == MATCH_ALL_CLASSIFICATION_TYPES) {
+ classificationTypeAndSubTypes =
Collections.emptySet();
+ classificationTypeAndSubTypesQryStr = ALL_TYPE_QUERY;
+ break;
+ } else {
+ Set<String> allTypes =
searchParameters.getIncludeSubClassifications() ?
classificationType.getTypeAndAllSubTypes() :
Collections.singleton(classificationType.getTypeName());
+ classificationTypeAndSubTypes.addAll(allTypes);
}
+ }
+
+ if (CollectionUtils.isNotEmpty(classificationTypeAndSubTypes)) {
+ classificationTypeAndSubTypesQryStr =
AtlasAttribute.escapeIndexQueryValue(classificationTypeAndSubTypes);
}
} else {
classificationTypeAndSubTypes = Collections.emptySet();
classificationTypeAndSubTypesQryStr = "";
}
+ this.classificationTypeAndSubTypes =
classificationTypeAndSubTypes;
+ this.classificationTypeAndSubTypesQryStr =
classificationTypeAndSubTypesQryStr;
+
+ //gather all types and its corresponding subtypes
+ Set<String> typeAndSubTypes = new HashSet<>();
+ String typeAndSubTypesQryStr = null;
+
+ if (CollectionUtils.isNotEmpty(entityTypes)) {
+ for (AtlasEntityType entityType : entityTypes) {
+
+ if (entityType.equals(MATCH_ALL_ENTITY_TYPES)) {
+ typeAndSubTypes = Collections.emptySet();
+ typeAndSubTypesQryStr = ALL_TYPE_QUERY;
+ break;
+ } else {
+ Set<String> allTypes =
searchParameters.getIncludeSubTypes() ? entityType.getTypeAndAllSubTypes() :
Collections.singleton(entityType.getTypeName());
+ typeAndSubTypes.addAll(allTypes);
+ }
+ }
- if (entityType != null) {
- if (entityType.equals(MATCH_ALL_ENTITY_TYPES)) {
- typeAndSubTypes = Collections.emptySet();
- typeAndSubTypesQryStr = ALL_TYPE_QUERY;
- } else {
- typeAndSubTypes = searchParameters.getIncludeSubTypes()
? entityType.getTypeAndAllSubTypes() :
Collections.singleton(entityType.getTypeName());
- typeAndSubTypesQryStr = searchParameters.getIncludeSubTypes()
? entityType.getTypeAndAllSubTypesQryStr() : entityType.getTypeQryStr();
+ if (CollectionUtils.isNotEmpty(typeAndSubTypes)) {
+ typeAndSubTypesQryStr =
AtlasAttribute.escapeIndexQueryValue(typeAndSubTypes);
}
} else {
typeAndSubTypes = Collections.emptySet();
typeAndSubTypesQryStr = "";
}
+ this.typeAndSubTypes = typeAndSubTypes;
+ this.typeAndSubTypesQryStr = typeAndSubTypesQryStr;
if (glossaryTermVertex != null) {
addProcessor(new TermSearchProcessor(this,
getAssignedEntities(glossaryTermVertex)));
@@ -179,37 +204,37 @@ public class SearchContext {
public Set<String> getEntityAttributes() { return entityAttributes; }
- public AtlasEntityType getEntityType() { return entityType; }
+ public Set<AtlasClassificationType> getClassificationTypes() { return
classificationTypes; }
- public AtlasClassificationType getClassificationType() { return
classificationType; }
+ public Set<String> getEntityTypeNames() { return typeAndSubTypes; }
- public Set<String> getEntityTypes() { return typeAndSubTypes; }
-
- public Set<String> getClassificationTypes() { return
classificationTypeAndSubTypes; }
+ public Set<String> getClassificationTypeNames() { return
classificationTypeAndSubTypes; }
public String getEntityTypesQryStr() { return typeAndSubTypesQryStr; }
public String getClassificationTypesQryStr() { return
classificationTypeAndSubTypesQryStr; }
+ public Set<AtlasEntityType> getEntityTypes() { return entityTypes; }
+
public SearchProcessor getSearchProcessor() { return searchProcessor; }
- public String getClassificationName() {return classificationName;}
+ public Set<String> getClassificationNames() {return classificationNames;}
public boolean includeEntityType(String entityType) {
return typeAndSubTypes.isEmpty() ||
typeAndSubTypes.contains(entityType);
}
- public boolean includeClassificationTypes(Collection<String>
classificationTypes) {
+ public boolean includeClassificationTypes(Collection<String> traitNames) {
final boolean ret;
- if (classificationType == null ||
classificationTypeAndSubTypes.isEmpty()) {
+ if (CollectionUtils.isEmpty(classificationTypes) ||
classificationTypeAndSubTypes.isEmpty()) {
ret = true;
- } else if (classificationType == MATCH_ALL_NOT_CLASSIFIED) {
- ret = CollectionUtils.isEmpty(classificationTypes);
- } else if (classificationType == MATCH_ALL_CLASSIFIED ||
classificationType == MATCH_ALL_WILDCARD_CLASSIFICATION) {
- ret = CollectionUtils.isNotEmpty(classificationTypes);
+ } else if (classificationTypes.iterator().next() ==
MATCH_ALL_NOT_CLASSIFIED) {
+ ret = CollectionUtils.isEmpty(traitNames);
+ } else if (classificationTypes.iterator().next() ==
MATCH_ALL_CLASSIFICATION_TYPES) {
+ ret = CollectionUtils.isNotEmpty(traitNames);
} else {
- ret = CollectionUtils.containsAny(classificationTypeAndSubTypes,
classificationTypes);
+ ret = CollectionUtils.containsAny(classificationTypeAndSubTypes,
traitNames);
}
return ret;
@@ -243,25 +268,18 @@ public class SearchContext {
}
boolean needClassificationProcessor() {
- return (classificationType != null && (entityType == null ||
hasAttributeFilter(searchParameters.getTagFilters()))) || isWildCardSearch() ;
- }
-
- boolean isBuiltInClassificationType() {
- return getClassificationType() == MATCH_ALL_WILDCARD_CLASSIFICATION
- || getClassificationType() == MATCH_ALL_CLASSIFIED
- || getClassificationType() == MATCH_ALL_NOT_CLASSIFIED;
+ return (CollectionUtils.isNotEmpty(classificationTypes) &&
(CollectionUtils.isEmpty(entityTypes) ||
hasAttributeFilter(searchParameters.getTagFilters()))) || isWildCardSearch() ;
}
boolean isWildCardSearch () {
- String classification = getSearchParameters().getClassification();
- if (StringUtils.isNotEmpty(classification) && getClassificationType()
== null) {
- return classification.contains("*");
+ if (CollectionUtils.isNotEmpty(classificationNames)) {
+ return classificationNames.stream().anyMatch(classification ->
classification.contains(WILDCARD_CLASSIFICATIONS));
}
return false;
}
boolean needEntityProcessor() {
- return entityType != null;
+ return CollectionUtils.isNotEmpty(entityTypes);
}
private void validateAttributes(final AtlasStructType structType, final
FilterCriteria filterCriteria) throws AtlasBaseException {
@@ -285,7 +303,14 @@ public class SearchContext {
if (structType == null) {
throw new
AtlasBaseException(AtlasErrorCode.UNKNOWN_TYPENAME, "NULL");
}
- throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_ATTRIBUTE,
attributeName, structType.getTypeName());
+
+ String name = structType.getTypeName();
+ if (name.equals(MATCH_ALL_ENTITY_TYPES.getTypeName())) {
+ name = ALL_ENTITY_TYPES;
+ } else if
(name.equals(MATCH_ALL_CLASSIFICATION_TYPES.getTypeName())) {
+ name = ALL_CLASSIFICATION_TYPES;
+ }
+ throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_ATTRIBUTE,
attributeName, name);
}
}
}
@@ -321,11 +346,107 @@ public class SearchContext {
return ret;
}
+ private Set<AtlasClassificationType> getClassificationTypes(Set<String>
classificationNames) {
+ if (CollectionUtils.isNotEmpty(classificationNames)) {
+ return classificationNames.stream().map(n ->
+
getClassificationType(n)).filter(Objects::nonNull).collect(Collectors.toSet());
+ }
+
+ return null;
+ }
+
+ private Set<String> getClassificationNames(String classification) throws
AtlasBaseException {
+ Set<String> classificationNames = new HashSet<>();
+
+ if (StringUtils.isNotEmpty(classification)) {
+ String[] types =
classification.split(TYPENAME_DELIMITER);
+ Set<String> names = new HashSet<>(Arrays.asList(types));
+
+ names.forEach(name -> {
+ AtlasClassificationType type = getClassificationType(name);
+ if (type != null || name.contains(WILDCARD_CLASSIFICATIONS)) {
+ classificationNames.add(name);
+ }
+ });
+
+ // Validate if the classification exists
+ if (CollectionUtils.isEmpty(classificationNames) ||
classificationNames.size() != names.size()) {
+ if (CollectionUtils.isNotEmpty(classificationNames)) {
+ names.removeAll(classificationNames);
+ }
+ throw new
AtlasBaseException(AtlasErrorCode.UNKNOWN_CLASSIFICATION,
String.join(TYPENAME_DELIMITER, names));
+ }
+ }
+
+ return classificationNames;
+ }
+
private AtlasEntityType getEntityType(String entityName) {
return StringUtils.equals(entityName, ALL_ENTITY_TYPES) ?
MATCH_ALL_ENTITY_TYPES :
typeRegistry.getEntityTypeByName(entityName);
}
+ private Set<AtlasEntityType> getEntityTypes(String typeName) throws
AtlasBaseException {
+
+ Set<AtlasEntityType> entityTypes = null;
+ //split multiple typeNames by comma
+ if (StringUtils.isNotEmpty(typeName)) {
+
+ String[] types = typeName.split(TYPENAME_DELIMITER);
+ Set<String> typeNames = new HashSet<>(Arrays.asList(types));
+ entityTypes = typeNames.stream().map(n ->
+
getEntityType(n)).filter(Objects::nonNull).collect(Collectors.toSet());
+
+ // Validate if the type name is incorrect
+ if (CollectionUtils.isEmpty(entityTypes) || entityTypes.size() !=
typeNames.size()) {
+ if (CollectionUtils.isNotEmpty(entityTypes)) {
+ Set<String> validEntityTypes = new HashSet<>();
+ for (AtlasEntityType entityType : entityTypes) {
+ String name = entityType.getTypeName();
+ if (name.equals(MATCH_ALL_ENTITY_TYPES.getTypeName()))
{
+ validEntityTypes.add(ALL_ENTITY_TYPES);
+ continue;
+ }
+ validEntityTypes.add(entityType.getTypeName());
+ }
+
+ typeNames.removeAll(validEntityTypes);
+ }
+
+ throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_TYPENAME,
String.join(TYPENAME_DELIMITER, typeNames));
+ }
+
+ }
+
+ return entityTypes;
+ }
+
+ private void filterStructTypes(){
+ //if typeName contains ALL_ENTITY_TYPES, remove others as OR condition
will not effect any other
+ if (CollectionUtils.isNotEmpty(entityTypes) &&
entityTypes.contains(MATCH_ALL_ENTITY_TYPES)) {
+ entityTypes.clear();
+ entityTypes.add(MATCH_ALL_ENTITY_TYPES);
+ }
+
+ //No Builtin Classification can be together
+ if (CollectionUtils.isNotEmpty(classificationTypes)) {
+ if (classificationTypes.contains(MATCH_ALL_NOT_CLASSIFIED)) {
+ classificationTypes.clear();
+ classificationTypes.add(MATCH_ALL_NOT_CLASSIFIED);
+
+ classificationNames.clear();
+
classificationNames.add(MATCH_ALL_NOT_CLASSIFIED.getTypeName());
+ } else if
(classificationTypes.contains(MATCH_ALL_WILDCARD_CLASSIFICATION) ||
classificationTypes.contains(MATCH_ALL_CLASSIFICATION_TYPES) ||
classificationTypes.contains(MATCH_ALL_CLASSIFIED)) {
+ classificationTypes.clear();
+ classificationTypes.add(MATCH_ALL_CLASSIFICATION_TYPES);
+
+ classificationNames.clear();
+ classificationNames.add(ALL_CLASSIFICATION_TYPES);
+ }
+ }
+ }
+
+
private AtlasVertex getGlossaryTermVertex(String termName) {
AtlasVertex ret = null;
diff --git
a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
index ad48be1..c9a6053 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
@@ -51,16 +51,8 @@ import java.util.regex.Pattern;
import static org.apache.atlas.SortOrder.ASCENDING;
import static
org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFICATION_TYPES;
-import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFIED;
import static
org.apache.atlas.discovery.SearchContext.MATCH_ALL_NOT_CLASSIFIED;
-import static
org.apache.atlas.discovery.SearchContext.MATCH_ALL_WILDCARD_CLASSIFICATION;
-import static org.apache.atlas.repository.Constants.CLASSIFICATION_NAMES_KEY;
-import static
org.apache.atlas.repository.Constants.CLASSIFICATION_NAME_DELIMITER;
-import static
org.apache.atlas.repository.Constants.CUSTOM_ATTRIBUTES_PROPERTY_KEY;
-import static org.apache.atlas.repository.Constants.LABELS_PROPERTY_KEY;
-import static
org.apache.atlas.repository.Constants.PROPAGATED_CLASSIFICATION_NAMES_KEY;
-import static
org.apache.atlas.repository.Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY;
-import static org.apache.atlas.repository.Constants.TRAIT_NAMES_PROPERTY_KEY;
+import static org.apache.atlas.repository.Constants.*;
import static org.apache.atlas.util.SearchPredicateUtil.*;
public abstract class SearchProcessor {
@@ -153,11 +145,19 @@ public abstract class SearchProcessor {
public abstract long getResultCount();
protected boolean isEntityRootType() {
- return context.getEntityType() == SearchContext.MATCH_ALL_ENTITY_TYPES;
+ //always size will be one if in case of _ALL_ENTITY_TYPES
+ if (CollectionUtils.isNotEmpty(context.getEntityTypes())) {
+ return context.getEntityTypes().iterator().next() ==
SearchContext.MATCH_ALL_ENTITY_TYPES;
+ }
+ return false;
}
protected boolean isClassificationRootType() {
- return context.getClassificationType() ==
SearchContext.MATCH_ALL_CLASSIFICATION_TYPES;
+ //always size will be one if in case of _ALL_CLASSIFICATION_TYPES
+ if (CollectionUtils.isNotEmpty(context.getClassificationTypes())) {
+ return context.getClassificationTypes().iterator().next() ==
SearchContext.MATCH_ALL_CLASSIFICATION_TYPES;
+ }
+ return false;
}
protected boolean isSystemAttribute(String attrName) {
@@ -195,9 +195,14 @@ public abstract class SearchProcessor {
}
}
- protected Predicate buildTraitPredict(AtlasClassificationType
classificationType) {
+ protected Predicate buildTraitPredict(Set<AtlasClassificationType>
classificationTypes) {
Predicate traitPredicate;
- if (classificationType == MATCH_ALL_WILDCARD_CLASSIFICATION ||
classificationType == MATCH_ALL_CLASSIFIED || classificationType ==
MATCH_ALL_CLASSIFICATION_TYPES) {
+ AtlasClassificationType classificationType = null;
+ if (CollectionUtils.isNotEmpty(classificationTypes)) {
+ classificationType = classificationTypes.iterator().next();
+ }
+
+ if (classificationType == MATCH_ALL_CLASSIFICATION_TYPES) {
traitPredicate =
PredicateUtils.orPredicate(SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY,
null, List.class),
SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY,
null, List.class));
} else if (classificationType == MATCH_ALL_NOT_CLASSIFIED) {
@@ -205,20 +210,27 @@ public abstract class SearchProcessor {
SearchPredicateUtil.getIsNullOrEmptyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY,
null, List.class));
} else if (context.isWildCardSearch()) {
//For wildcard search __classificationNames which of String type
is taken instead of _traitNames which is of Array type
- //No need to escape, as classification Names only support
letters,numbers,space and underscore
- String regexString = getRegexString("\\|" +
context.getClassificationName() + "\\|");
- traitPredicate =
PredicateUtils.orPredicate(SearchPredicateUtil.getRegexPredicateGenerator().generatePredicate(CLASSIFICATION_NAMES_KEY,
regexString, String.class),
-
SearchPredicateUtil.getRegexPredicateGenerator().generatePredicate(PROPAGATED_CLASSIFICATION_NAMES_KEY,
regexString, String.class));
+ Set<String> classificationNames = context.getClassificationNames();
+ List<Predicate> predicates = new ArrayList<>();
+
+ classificationNames.forEach(classificationName -> {
+ //No need to escape, as classification Names only support
letters,numbers,space and underscore
+ String regexString = getRegexString("\\|" + classificationName
+ "\\|");
+
predicates.add(SearchPredicateUtil.getRegexPredicateGenerator().generatePredicate(CLASSIFICATION_NAMES_KEY,
regexString, String.class));
+
predicates.add(SearchPredicateUtil.getRegexPredicateGenerator().generatePredicate(PROPAGATED_CLASSIFICATION_NAMES_KEY,
regexString, String.class));
+ });
+
+ traitPredicate = PredicateUtils.anyPredicate(predicates);
} else {
- traitPredicate =
PredicateUtils.orPredicate(SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY,
context.getClassificationTypes(), List.class),
-
SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY,
context.getClassificationTypes(), List.class));
+ traitPredicate =
PredicateUtils.orPredicate(SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY,
context.getClassificationTypeNames(), List.class),
+
SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY,
context.getClassificationTypeNames(), List.class));
}
return traitPredicate;
}
- protected void processSearchAttributes(AtlasStructType structType,
FilterCriteria filterCriteria, Set<String> indexFiltered, Set<String>
graphFiltered, Set<String> allAttributes) {
- if (structType == null || filterCriteria == null) {
+ protected void processSearchAttributes(Set<? extends AtlasStructType>
structTypes, FilterCriteria filterCriteria, Set<String> indexFiltered,
Set<String> graphFiltered, Set<String> allAttributes) {
+ if (CollectionUtils.isEmpty(structTypes) || filterCriteria == null) {
return;
}
@@ -227,7 +239,7 @@ public abstract class SearchProcessor {
if (filterCondition != null && CollectionUtils.isNotEmpty(criterion)) {
for (SearchParameters.FilterCriteria criteria : criterion) {
- processSearchAttributes(structType, criteria, indexFiltered,
graphFiltered, allAttributes);
+ processSearchAttributes(structTypes, criteria, indexFiltered,
graphFiltered, allAttributes);
}
} else if (StringUtils.isNotEmpty(filterCriteria.getAttributeName())) {
String attributeName = filterCriteria.getAttributeName();
@@ -267,20 +279,23 @@ public abstract class SearchProcessor {
}
try {
- if (isIndexSearchable(filterCriteria, structType)) {
- indexFiltered.add(attributeName);
- } else {
- LOG.warn("not using index-search for attribute '{}'; might
cause poor performance", structType.getVertexPropertyName(attributeName));
+ for (AtlasStructType structType : structTypes) {
+ String qualifiedName =
structType.getVertexPropertyName(attributeName);
+ if (isIndexSearchable(filterCriteria, structType)) {
+ indexFiltered.add(qualifiedName);
+ } else {
+ LOG.warn("not using index-search for attribute '{}';
might cause poor performance", structType.getVertexPropertyName(attributeName));
- graphFiltered.add(attributeName);
- }
+ graphFiltered.add(qualifiedName);
+ }
- if (structType instanceof AtlasEntityType &&
!isSystemAttribute(attributeName)) {
- // Capture the entity attributes
- context.getEntityAttributes().add(attributeName);
- }
+ if (structType instanceof AtlasEntityType &&
!isSystemAttribute(attributeName)) {
+ // Capture the entity attributes
+ context.getEntityAttributes().add(attributeName);
+ }
- allAttributes.add(attributeName);
+ allAttributes.add(qualifiedName);
+ }
} catch (AtlasBaseException e) {
LOG.warn(e.getMessage());
}
@@ -298,7 +313,7 @@ public abstract class SearchProcessor {
// (AND (OR idx-att1=x idx-attr1=y) non-idx-attr=z)
// (AND (OR idx-att1=x idx-attr1=y) non-idx-attr=z (AND idx-attr2=xyz
idx-attr2=abc))
//
- protected boolean canApplyIndexFilter(AtlasStructType structType,
FilterCriteria filterCriteria, boolean insideOrCondition) {
+ protected boolean canApplyIndexFilter(Set<? extends AtlasStructType>
structTypes, FilterCriteria filterCriteria, boolean insideOrCondition) {
if (!context.hasAttributeFilter(filterCriteria)) {
return true;
}
@@ -314,7 +329,7 @@ public abstract class SearchProcessor {
// If we have nested criterion let's find any nested ORs with
non-indexed attr
for (FilterCriteria criteria : criterion) {
- ret = canApplyIndexFilter(structType, criteria,
insideOrCondition);
+ ret = canApplyIndexFilter(structTypes, criteria,
insideOrCondition);
if (!ret) {
break;
@@ -322,10 +337,11 @@ public abstract class SearchProcessor {
}
} else if (StringUtils.isNotEmpty(filterCriteria.getAttributeName())) {
try {
-
-
- if (insideOrCondition && !isIndexSearchable(filterCriteria,
structType)) {
- ret = false;
+ for (AtlasStructType structType : structTypes) {
+ if (insideOrCondition &&
!isIndexSearchable(filterCriteria, structType)) {
+ ret = false;
+ break;
+ }
}
} catch (AtlasBaseException e) {
LOG.warn(e.getMessage());
@@ -338,7 +354,7 @@ public abstract class SearchProcessor {
protected void filterWhiteSpaceClassification(List<AtlasVertex>
entityVertices) {
if (CollectionUtils.isNotEmpty(entityVertices)) {
final Iterator<AtlasVertex> it =
entityVertices.iterator();
- final Set<String> typeAndSubTypes =
context.getClassificationTypes();
+ final Set<String> typeAndSubTypes =
context.getClassificationTypeNames();
while (it.hasNext()) {
AtlasVertex entityVertex = it.next();
@@ -363,13 +379,13 @@ public abstract class SearchProcessor {
}
}
- protected void constructFilterQuery(StringBuilder indexQuery,
AtlasStructType type, FilterCriteria filterCriteria, Set<String>
indexAttributes) {
+ protected void constructFilterQuery(StringBuilder indexQuery, Set<?
extends AtlasStructType> structTypes, FilterCriteria filterCriteria,
Set<String> indexAttributes) {
if (filterCriteria != null) {
if (LOG.isDebugEnabled()) {
LOG.debug("Processing Filters");
}
- String filterQuery = toIndexQuery(type, filterCriteria,
indexAttributes, 0);
+ String filterQuery = toIndexQuery(structTypes, filterCriteria,
indexAttributes, 0);
if (StringUtils.isNotEmpty(filterQuery)) {
if (indexQuery.length() > 0) {
@@ -381,14 +397,14 @@ public abstract class SearchProcessor {
}
}
- protected Predicate constructInMemoryPredicate(AtlasStructType type,
FilterCriteria filterCriteria, Set<String> indexAttributes) {
+ protected Predicate constructInMemoryPredicate(Set<? extends
AtlasStructType> structTypes, FilterCriteria filterCriteria, Set<String>
indexAttributes) {
Predicate ret = null;
if (filterCriteria != null) {
if (LOG.isDebugEnabled()) {
LOG.debug("Processing Filters");
}
- ret = toInMemoryPredicate(type, filterCriteria, indexAttributes);
+ ret = toInMemoryPredicate(structTypes, filterCriteria,
indexAttributes);
}
return ret;
}
@@ -473,17 +489,20 @@ public abstract class SearchProcessor {
return ret;
}
- private String toIndexQuery(AtlasStructType type, FilterCriteria criteria,
Set<String> indexAttributes, int level) {
- return toIndexQuery(type, criteria, indexAttributes, new
StringBuilder(), level);
+ private String toIndexQuery(Set<? extends AtlasStructType> structTypes,
FilterCriteria criteria, Set<String> indexAttributes, int level) {
+ return toIndexQuery(structTypes, criteria, indexAttributes, new
StringBuilder(), level);
}
- private String toIndexQuery(AtlasStructType type, FilterCriteria criteria,
Set<String> indexAttributes, StringBuilder sb, int level) {
+ private String toIndexQuery(Set<? extends AtlasStructType> structTypes,
FilterCriteria criteria, Set<String> indexAttributes, StringBuilder sb, int
level) {
+ Set<String> filterAttributes = new HashSet<>();
+ filterAttributes.addAll(indexAttributes);
+
Condition condition = criteria.getCondition();
if (condition != null &&
CollectionUtils.isNotEmpty(criteria.getCriterion())) {
StringBuilder nestedExpression = new StringBuilder();
for (FilterCriteria filterCriteria : criteria.getCriterion()) {
- String nestedQuery = toIndexQuery(type, filterCriteria,
indexAttributes, level + 1);
+ String nestedQuery = toIndexQuery(structTypes, filterCriteria,
filterAttributes, level + 1);
if (StringUtils.isNotEmpty(nestedQuery)) {
if (nestedExpression.length() > 0) {
@@ -502,19 +521,45 @@ public abstract class SearchProcessor {
} else {
return EMPTY_STRING;
}
- } else if (indexAttributes.contains(criteria.getAttributeName())){
- return toIndexExpression(type, criteria.getAttributeName(),
criteria.getOperator(), criteria.getAttributeValue());
- } else {
- return EMPTY_STRING;
+ } else if (StringUtils.isNotEmpty(criteria.getAttributeName())) {
+ try {
+ ArrayList<String> orExpQuery = new ArrayList<>();
+ for (AtlasStructType structType : structTypes) {
+ String name =
structType.getVertexPropertyName(criteria.getAttributeName());
+
+ if (filterAttributes.contains(name)) {
+ String nestedQuery = toIndexExpression(structType,
criteria.getAttributeName(), criteria.getOperator(),
criteria.getAttributeValue());
+ orExpQuery.add(nestedQuery);
+ filterAttributes.remove(name);
+ }
+ }
+
+ if (CollectionUtils.isNotEmpty(orExpQuery)) {
+ if (orExpQuery.size() > 1) {
+ String orExpStr = StringUtils.join(orExpQuery, "
"+Condition.OR.name()+" ");
+ return BRACE_OPEN_STR + " " + orExpStr + " " +
BRACE_CLOSE_STR;
+ } else {
+ return orExpQuery.iterator().next();
+ }
+ } else {
+ return EMPTY_STRING;
+ }
+ } catch (AtlasBaseException e) {
+ LOG.warn(e.getMessage());
+ }
}
+ return EMPTY_STRING;
}
- private Predicate toInMemoryPredicate(AtlasStructType type, FilterCriteria
criteria, Set<String> indexAttributes) {
+ private Predicate toInMemoryPredicate(Set<? extends AtlasStructType>
structTypes, FilterCriteria criteria, Set<String> indexAttributes) {
+ Set<String> filterAttributes = new HashSet<>();
+ filterAttributes.addAll(indexAttributes);
+
if (criteria.getCondition() != null &&
CollectionUtils.isNotEmpty(criteria.getCriterion())) {
List<Predicate> predicates = new ArrayList<>();
for (FilterCriteria filterCriteria : criteria.getCriterion()) {
- Predicate predicate = toInMemoryPredicate(type,
filterCriteria, indexAttributes);
+ Predicate predicate = toInMemoryPredicate(structTypes,
filterCriteria, filterAttributes);
if (predicate != null) {
predicates.add(predicate);
@@ -528,21 +573,40 @@ public abstract class SearchProcessor {
return PredicateUtils.anyPredicate(predicates);
}
}
- } else if (indexAttributes.contains(criteria.getAttributeName())) {
- String attrName = criteria.getAttributeName();
- String attrValue = criteria.getAttributeValue();
- SearchParameters.Operator operator = criteria.getOperator();
+ } else if (StringUtils.isNotEmpty(criteria.getAttributeName())) {
+ try {
+ ArrayList<Predicate> predicates = new ArrayList<>();
+ for (AtlasStructType structType : structTypes) {
+ String name =
structType.getVertexPropertyName(criteria.getAttributeName());
+
+ if (filterAttributes.contains(name)) {
+ String attrName =
criteria.getAttributeName();
+ String attrValue =
criteria.getAttributeValue();
+ SearchParameters.Operator operator =
criteria.getOperator();
+
+ //process attribute value and attribute operator for
pipeSeperated fields
+ if (isPipeSeparatedSystemAttribute(attrName)) {
+ FilterCriteria processedCriteria =
processPipeSeperatedSystemAttribute(attrName, operator, attrValue);
+ attrValue =
processedCriteria.getAttributeValue();
+ operator =
processedCriteria.getOperator();
+ }
- //process attribute value and attribute operator for pipeSeperated
fields
- if (isPipeSeparatedSystemAttribute(attrName)) {
- FilterCriteria processedCriteria =
processPipeSeperatedSystemAttribute(attrName, operator, attrValue);
- attrValue =
processedCriteria.getAttributeValue();
- operator =
processedCriteria.getOperator();
- }
+ predicates.add(toInMemoryPredicate(structType,
attrName, operator, attrValue));
+ filterAttributes.remove(name);
+ }
+ }
- return toInMemoryPredicate(type, attrName, operator, attrValue);
+ if (CollectionUtils.isNotEmpty(predicates)) {
+ if (predicates.size() > 1) {
+ return PredicateUtils.anyPredicate(predicates);
+ } else {
+ return predicates.iterator().next();
+ }
+ }
+ } catch (AtlasBaseException e) {
+ LOG.warn(e.getMessage());
+ }
}
-
return null;
}
@@ -704,12 +768,15 @@ public abstract class SearchProcessor {
return ret;
}
- protected AtlasGraphQuery toGraphFilterQuery(AtlasStructType type,
FilterCriteria criteria, Set<String> graphAttributes, AtlasGraphQuery query) {
+ protected AtlasGraphQuery toGraphFilterQuery(Set<? extends
AtlasStructType> structTypes, FilterCriteria criteria, Set<String>
graphAttributes, AtlasGraphQuery query) {
+ Set<String> filterAttributes = new HashSet<>();
+ filterAttributes.addAll(graphAttributes);
+
if (criteria != null) {
if (criteria.getCondition() != null) {
if (criteria.getCondition() == Condition.AND) {
for (FilterCriteria filterCriteria :
criteria.getCriterion()) {
- AtlasGraphQuery nestedQuery = toGraphFilterQuery(type,
filterCriteria, graphAttributes, context.getGraph().query());
+ AtlasGraphQuery nestedQuery =
toGraphFilterQuery(structTypes, filterCriteria, filterAttributes,
context.getGraph().query());
query.addConditionsFrom(nestedQuery);
}
@@ -717,7 +784,7 @@ public abstract class SearchProcessor {
List<AtlasGraphQuery> orConditions = new LinkedList<>();
for (FilterCriteria filterCriteria :
criteria.getCriterion()) {
- AtlasGraphQuery nestedQuery = toGraphFilterQuery(type,
filterCriteria, graphAttributes, context.getGraph().query());
+ AtlasGraphQuery nestedQuery =
toGraphFilterQuery(structTypes, filterCriteria, filterAttributes,
context.getGraph().query());
orConditions.add(context.getGraph().query().createChildQuery().addConditionsFrom(nestedQuery));
}
@@ -726,60 +793,80 @@ public abstract class SearchProcessor {
query.or(orConditions);
}
}
- } else if (graphAttributes.contains(criteria.getAttributeName())) {
- String attrName =
criteria.getAttributeName();
- String attrValue =
criteria.getAttributeValue();
- SearchParameters.Operator operator = criteria.getOperator();
-
- //process attribute value and attribute operator for
pipeSeperated fields
- if (isPipeSeparatedSystemAttribute(attrName)) {
- FilterCriteria processedCriteria =
processPipeSeperatedSystemAttribute(attrName, operator, attrValue);
- attrValue =
processedCriteria.getAttributeValue();
- operator =
processedCriteria.getOperator();
- }
-
- final String qualifiedName =
type.getAttribute(attrName).getVertexPropertyName();
+ } else if (StringUtils.isNotEmpty(criteria.getAttributeName())) {
+ try {
+ ArrayList<AtlasGraphQuery> queries = new ArrayList<>();
+ for (AtlasStructType structType : structTypes) {
+ String qualifiedName =
structType.getVertexPropertyName(criteria.getAttributeName());
+ if (filterAttributes.contains(qualifiedName)) {
+
+ String attrName =
criteria.getAttributeName();
+ String attrValue =
criteria.getAttributeValue();
+ SearchParameters.Operator operator =
criteria.getOperator();
+
+ //process attribute value and attribute operator
for pipeSeperated fields
+ if (isPipeSeparatedSystemAttribute(attrName)) {
+ FilterCriteria processedCriteria =
processPipeSeperatedSystemAttribute(attrName, operator, attrValue);
+ attrValue =
processedCriteria.getAttributeValue();
+ operator = processedCriteria.getOperator();
+ }
+
+ AtlasGraphQuery innerQry =
context.getGraph().query().createChildQuery();
+ switch (operator) {
+ case LT:
+ innerQry.has(qualifiedName,
AtlasGraphQuery.ComparisionOperator.LESS_THAN, attrValue);
+ break;
+ case LTE:
+ innerQry.has(qualifiedName,
AtlasGraphQuery.ComparisionOperator.LESS_THAN_EQUAL, attrValue);
+ break;
+ case GT:
+ innerQry.has(qualifiedName,
AtlasGraphQuery.ComparisionOperator.GREATER_THAN, attrValue);
+ break;
+ case GTE:
+ innerQry.has(qualifiedName,
AtlasGraphQuery.ComparisionOperator.GREATER_THAN_EQUAL, attrValue);
+ break;
+ case EQ:
+ innerQry.has(qualifiedName,
AtlasGraphQuery.ComparisionOperator.EQUAL, attrValue);
+ break;
+ case NEQ:
+ innerQry.has(qualifiedName,
AtlasGraphQuery.ComparisionOperator.NOT_EQUAL, attrValue);
+ break;
+ case LIKE:
+ innerQry.has(qualifiedName,
AtlasGraphQuery.MatchingOperator.REGEX, attrValue);
+ break;
+ case CONTAINS:
+ innerQry.has(qualifiedName,
AtlasGraphQuery.MatchingOperator.REGEX, getContainsRegex(attrValue));
+ break;
+ case STARTS_WITH:
+ innerQry.has(qualifiedName,
AtlasGraphQuery.MatchingOperator.PREFIX, attrValue);
+ break;
+ case ENDS_WITH:
+ innerQry.has(qualifiedName,
AtlasGraphQuery.MatchingOperator.REGEX, getSuffixRegex(attrValue));
+ break;
+ case IS_NULL:
+ innerQry.has(qualifiedName,
AtlasGraphQuery.ComparisionOperator.EQUAL, null);
+ break;
+ case NOT_NULL:
+ innerQry.has(qualifiedName,
AtlasGraphQuery.ComparisionOperator.NOT_EQUAL, null);
+ break;
+ default:
+ LOG.warn("{}: unsupported operator.
Ignored", operator);
+ break;
+ }
+
queries.add(context.getGraph().query().createChildQuery().addConditionsFrom(innerQry));
+ filterAttributes.remove(qualifiedName);
+ }
+ }
- switch (operator) {
- case LT:
- query.has(qualifiedName,
AtlasGraphQuery.ComparisionOperator.LESS_THAN, attrValue);
- break;
- case LTE:
- query.has(qualifiedName,
AtlasGraphQuery.ComparisionOperator.LESS_THAN_EQUAL, attrValue);
- break;
- case GT:
- query.has(qualifiedName,
AtlasGraphQuery.ComparisionOperator.GREATER_THAN, attrValue);
- break;
- case GTE:
- query.has(qualifiedName,
AtlasGraphQuery.ComparisionOperator.GREATER_THAN_EQUAL, attrValue);
- break;
- case EQ:
- query.has(qualifiedName,
AtlasGraphQuery.ComparisionOperator.EQUAL, attrValue);
- break;
- case NEQ:
- query.has(qualifiedName,
AtlasGraphQuery.ComparisionOperator.NOT_EQUAL, attrValue);
- break;
- case LIKE:
- query.has(qualifiedName,
AtlasGraphQuery.MatchingOperator.REGEX, attrValue);
- break;
- case CONTAINS:
- query.has(qualifiedName,
AtlasGraphQuery.MatchingOperator.REGEX, getContainsRegex(attrValue));
- break;
- case STARTS_WITH:
- query.has(qualifiedName,
AtlasGraphQuery.MatchingOperator.PREFIX, attrValue);
- break;
- case ENDS_WITH:
- query.has(qualifiedName,
AtlasGraphQuery.MatchingOperator.REGEX, getSuffixRegex(attrValue));
- break;
- case IS_NULL:
- query.has(qualifiedName,
AtlasGraphQuery.ComparisionOperator.EQUAL, null);
- break;
- case NOT_NULL:
- query.has(qualifiedName,
AtlasGraphQuery.ComparisionOperator.NOT_EQUAL, null);
- break;
- default:
- LOG.warn("{}: unsupported operator. Ignored",
operator);
- break;
+ if (CollectionUtils.isNotEmpty(queries)) {
+ if (queries.size() > 1) {
+ return
context.getGraph().query().createChildQuery().or(queries);
+ } else {
+ return queries.iterator().next();
+ }
+ }
+ } catch (AtlasBaseException e) {
+ LOG.warn(e.getMessage());
}
}
}
@@ -987,11 +1074,13 @@ public abstract class SearchProcessor {
}
private static String getSortByAttribute(SearchContext context) {
- final AtlasEntityType entityType = context.getEntityType();
- String sortBy = context.getSearchParameters().getSortBy();
- AtlasStructType.AtlasAttribute sortByAttribute = entityType != null ?
entityType.getAttribute(sortBy) : null;
- if (sortByAttribute != null) {
- return sortByAttribute.getVertexPropertyName();
+ if (CollectionUtils.isNotEmpty(context.getEntityTypes())) {
+ final AtlasEntityType entityType =
context.getEntityTypes().iterator().next();
+ String sortBy = context.getSearchParameters().getSortBy();
+ AtlasStructType.AtlasAttribute sortByAttribute =
entityType.getAttribute(sortBy);
+ if (sortByAttribute != null) {
+ return sortByAttribute.getVertexPropertyName();
+ }
}
return null;
}
diff --git
a/repository/src/test/java/org/apache/atlas/discovery/EntitySearchProcessorTest.java
b/repository/src/test/java/org/apache/atlas/discovery/EntitySearchProcessorTest.java
index 43f11d1..b7ce978 100644
---
a/repository/src/test/java/org/apache/atlas/discovery/EntitySearchProcessorTest.java
+++
b/repository/src/test/java/org/apache/atlas/discovery/EntitySearchProcessorTest.java
@@ -130,7 +130,7 @@ public class EntitySearchProcessorTest extends
BasicTestSetup {
new EntitySearchProcessor(context);
}
- @Test
+ @Test(priority = -1)
public void searchWithNEQ_stringAttr() throws AtlasBaseException {
String expectedEntityName = "hive_Table_Null_tableType";
createDummyEntity(expectedEntityName,HIVE_TABLE_TYPE);
@@ -197,4 +197,116 @@ public class EntitySearchProcessorTest extends
BasicTestSetup {
assertTrue(nameList.contains("hive_Table_Null_tableType"));
}
+
+ @Test
+ public void ALLEntityType() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setTypeName(SearchParameters.ALL_ENTITY_TYPES);
+ params.setLimit(20);
+
+ SearchContext context = new SearchContext(params, typeRegistry, graph,
Collections.<String>emptySet());
+
+ EntitySearchProcessor processor = new EntitySearchProcessor(context);
+ assertEquals(processor.execute().size(), 20);
+ }
+
+ @Test
+ public void ALLEntityTypeWithTag() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setTypeName(SearchParameters.ALL_ENTITY_TYPES);
+ params.setClassification(FACT_CLASSIFICATION);
+ params.setLimit(20);
+
+ SearchContext context = new SearchContext(params, typeRegistry, graph,
Collections.<String>emptySet());
+
+ EntitySearchProcessor processor = new EntitySearchProcessor(context);
+ assertEquals(processor.execute().size(), 5);
+ }
+
+ @Test
+ public void entityType() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setTypeName(DATABASE_TYPE);
+ params.setLimit(20);
+
+ SearchContext context = new SearchContext(params, typeRegistry, graph,
Collections.<String>emptySet());
+
+ EntitySearchProcessor processor = new EntitySearchProcessor(context);
+ assertEquals(processor.execute().size(), 3);
+ }
+
+ @Test
+ public void entityTypes() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setTypeName(DATABASE_TYPE+","+HIVE_TABLE_TYPE);
+ params.setLimit(20);
+
+ SearchContext context = new SearchContext(params, typeRegistry, graph,
Collections.<String>emptySet());
+
+ EntitySearchProcessor processor = new EntitySearchProcessor(context);
+ assertEquals(processor.execute().size(), 14);
+ }
+
+ @Test(expectedExceptions = AtlasBaseException.class,
expectedExceptionsMessageRegExp = "Not_Exists: Unknown/invalid typename")
+ public void entityTypesNotAllowed() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setTypeName(DATABASE_TYPE+",Not_Exists");
+ params.setLimit(20);
+
+ SearchContext context = new SearchContext(params, typeRegistry, graph,
Collections.<String>emptySet());
+ }
+
+ @Test(expectedExceptions = AtlasBaseException.class,
expectedExceptionsMessageRegExp = "Attribute tableType not found for type
"+DATABASE_TYPE)
+ public void entityFiltersNotAllowed() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setTypeName(DATABASE_TYPE+","+HIVE_TABLE_TYPE);
+ SearchParameters.FilterCriteria filterCriteria =
getSingleFilterCondition("tableType", SearchParameters.Operator.CONTAINS,
"ETL");
+ params.setEntityFilters(filterCriteria);
+ params.setLimit(20);
+
+ SearchContext context = new SearchContext(params, typeRegistry, graph,
Collections.<String>emptySet());
+ }
+
+ @Test
+ public void entityTypesAndTag() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setTypeName(DATABASE_TYPE+","+HIVE_TABLE_TYPE);
+ params.setClassification(FACT_CLASSIFICATION);
+ params.setLimit(20);
+
+ SearchContext context = new SearchContext(params, typeRegistry, graph,
Collections.<String>emptySet());
+
+ EntitySearchProcessor processor = new EntitySearchProcessor(context);
+ assertEquals(processor.execute().size(), 3);
+ }
+
+ @Test
+ public void searchWithEntityTypesAndEntityFilters() throws
AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setTypeName(DATABASE_TYPE+","+HIVE_TABLE_TYPE);
+ SearchParameters.FilterCriteria filterCriteria =
getSingleFilterCondition("owner", SearchParameters.Operator.CONTAINS, "ETL");
+ params.setEntityFilters(filterCriteria);
+ params.setLimit(20);
+
+ SearchContext context = new SearchContext(params, typeRegistry, graph,
Collections.<String>emptySet());
+
+ EntitySearchProcessor processor = new EntitySearchProcessor(context);
+ assertEquals(processor.execute().size(), 4);
+ }
+
+ @Test
+ public void searchWithEntityTypesAndEntityFiltersAndTag() throws
AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setTypeName(DATABASE_TYPE+","+HIVE_TABLE_TYPE);
+ SearchParameters.FilterCriteria filterCriteria =
getSingleFilterCondition("owner", SearchParameters.Operator.CONTAINS, "ETL");
+ params.setEntityFilters(filterCriteria);
+ params.setClassification(LOGDATA_CLASSIFICATION);
+ params.setLimit(20);
+
+ SearchContext context = new SearchContext(params, typeRegistry, graph,
Collections.<String>emptySet());
+
+ EntitySearchProcessor processor = new EntitySearchProcessor(context);
+ assertEquals(processor.execute().size(), 2);
+ }
+
}
