This is an automated email from the ASF dual-hosted git repository.
pinal pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new dff0fcb88 ATLAS-4671: Basic Search : Exclude Header attributes of
entities from the response
dff0fcb88 is described below
commit dff0fcb88a7b34365b1262f71e71f68749fbd35f
Author: Pinal Shah <[email protected]>
AuthorDate: Wed Sep 14 15:36:39 2022 +0530
ATLAS-4671: Basic Search : Exclude Header attributes of entities from the
response
Signed-off-by: Pinal Shah <[email protected]>
(cherry picked from commit 37c242755cdf5320c2eafce925d613061e584e6a)
---
.../model/discovery/QuickSearchParameters.java | 9 ++
.../atlas/model/discovery/SearchParameters.java | 14 ++-
.../atlas/discovery/EntityDiscoveryService.java | 27 +++++
.../org/apache/atlas/discovery/SearchContext.java | 38 +++++++
.../atlas/discovery/AtlasDiscoveryServiceTest.java | 117 +++++++++++++++++++++
.../search-parameters/combination-filters.json | 12 +--
.../json/search-parameters/entity-filters.json | 13 ---
7 files changed, 210 insertions(+), 20 deletions(-)
diff --git
a/intg/src/main/java/org/apache/atlas/model/discovery/QuickSearchParameters.java
b/intg/src/main/java/org/apache/atlas/model/discovery/QuickSearchParameters.java
index 79f5aae0d..dcad83c01 100644
---
a/intg/src/main/java/org/apache/atlas/model/discovery/QuickSearchParameters.java
+++
b/intg/src/main/java/org/apache/atlas/model/discovery/QuickSearchParameters.java
@@ -50,6 +50,7 @@ public class QuickSearchParameters implements Serializable {
private Set<String> attributes;
private String sortBy;
private SortOrder sortOrder;
+ private boolean excludeHeaderAttributes;
/**
* for framework use.
@@ -158,4 +159,12 @@ public class QuickSearchParameters implements Serializable
{
public void setSortOrder(SortOrder sortOrder) {
this.sortOrder = sortOrder;
}
+
+ public boolean getExcludeHeaderAttributes() {
+ return excludeHeaderAttributes;
+ }
+
+ public void setExcludeHeaderAttributes(boolean excludeHeaderAttributes) {
+ this.excludeHeaderAttributes = excludeHeaderAttributes;
+ }
}
diff --git
a/intg/src/main/java/org/apache/atlas/model/discovery/SearchParameters.java
b/intg/src/main/java/org/apache/atlas/model/discovery/SearchParameters.java
index 78fb4a48f..8e68d0e82 100644
--- a/intg/src/main/java/org/apache/atlas/model/discovery/SearchParameters.java
+++ b/intg/src/main/java/org/apache/atlas/model/discovery/SearchParameters.java
@@ -50,6 +50,8 @@ public class SearchParameters implements Serializable {
private boolean includeClassificationAttributes;
private boolean includeSubTypes = true;
private boolean includeSubClassifications = true;
+ private boolean excludeHeaderAttributes = false;
+
private int limit;
private int offset;
private String marker;
@@ -258,6 +260,14 @@ public class SearchParameters implements Serializable {
this.tagFilters = tagFilters;
}
+ public boolean getExcludeHeaderAttributes() {
+ return excludeHeaderAttributes;
+ }
+
+ public void setExcludeHeaderAttributes(boolean excludeHeaderAttributes) {
+ this.excludeHeaderAttributes = excludeHeaderAttributes;
+ }
+
/**
* Attribute values included in the results
* @return
@@ -307,6 +317,7 @@ public class SearchParameters implements Serializable {
includeClassificationAttributes ==
that.includeClassificationAttributes &&
includeSubTypes == that.includeSubTypes &&
includeSubClassifications == that.includeSubClassifications &&
+ excludeHeaderAttributes == that.excludeHeaderAttributes &&
limit == that.limit &&
offset == that.offset &&
Objects.equals(query, that.query) &&
@@ -323,7 +334,7 @@ public class SearchParameters implements Serializable {
@Override
public int hashCode() {
return Objects.hash(query, typeName, classification, termName,
includeSubTypes, includeSubClassifications,
- excludeDeletedEntities,
includeClassificationAttributes, limit, offset, entityFilters,
+ excludeDeletedEntities,
includeClassificationAttributes, excludeHeaderAttributes, limit, offset,
entityFilters,
tagFilters, attributes, sortBy, sortOrder);
}
@@ -341,6 +352,7 @@ public class SearchParameters implements Serializable {
sb.append(",
includeSubClassifications='").append(includeSubClassifications).append('\'');
sb.append(", excludeDeletedEntities=").append(excludeDeletedEntities);
sb.append(",
includeClassificationAttributes=").append(includeClassificationAttributes);
+ sb.append(",
excludeHeaderAttributes=").append(excludeHeaderAttributes);
sb.append(", limit=").append(limit);
sb.append(", offset=").append(offset);
sb.append(", entityFilters=").append(entityFilters);
diff --git
a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
index 8fbc22fa0..4b113dbef 100644
---
a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
+++
b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
@@ -468,6 +468,32 @@ public class EntityDiscoveryService implements
AtlasDiscoveryService {
ret.setNextMarker(nextMarker);
}
+ //If excludeHeaderAttributes is true, only primitive attributes
requested in 'attributes' field will be sent in the response
+ Set<String> attributes = searchParameters.getAttributes();
+ if (searchContext.excludeHeaderAttributes()) {
+
+ AtlasSearchResult.AttributeSearchResult attributeSearchResult
= new AtlasSearchResult.AttributeSearchResult();
+ attributeSearchResult.setName(new ArrayList<>(attributes));
+
+ Collection<List<Object>> values = new ArrayList<>();
+ for (AtlasVertex vertex : resultList) {
+ List<Object> row = new ArrayList<>();
+
+ for (String attrName : attributes) {
+ AtlasEntityType entityType =
searchContext.getEntityTypes().iterator().next();
+ AtlasAttribute attribute =
entityType.getAttribute(attrName);
+ Object value =
vertex.getProperty(attribute.getVertexPropertyName(), Object.class);
+
+ row.add(value != null ? value : StringUtils.EMPTY);
+ }
+ values.add(row);
+ }
+ attributeSearchResult.setValues(new ArrayList<>(values));
+
+ ret.setAttributes(attributeSearchResult);
+ return ret;
+ }
+
// By default any attribute that shows up in the search parameter
should be sent back in the response
// If additional values are requested then the entityAttributes
will be a superset of the all search attributes
// and the explicitly requested attribute(s)
@@ -922,6 +948,7 @@ public class EntityDiscoveryService implements
AtlasDiscoveryService {
searchParameters.setAttributes(quickSearchParameters.getAttributes());
searchParameters.setSortBy(quickSearchParameters.getSortBy());
searchParameters.setSortOrder(quickSearchParameters.getSortOrder());
+
searchParameters.setExcludeHeaderAttributes(quickSearchParameters.getExcludeHeaderAttributes());
return searchParameters;
}
diff --git
a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
index 01954d07e..b8976e079 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
@@ -32,10 +32,13 @@ import org.apache.atlas.repository.graphdb.AtlasGraphQuery;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2;
import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever;
+import org.apache.atlas.type.AtlasArrayType;
+import org.apache.atlas.type.AtlasBuiltInTypes;
import org.apache.atlas.type.AtlasClassificationType;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasStructType;
import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
+import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.util.AtlasRepositoryConfiguration;
import org.apache.commons.collections.CollectionUtils;
@@ -86,6 +89,7 @@ public class SearchContext {
private boolean terminateSearch = false;
private SearchProcessor searchProcessor;
private Integer marker;
+ private boolean hasRelationshipAttributes = false;
public final static AtlasClassificationType
MATCH_ALL_WILDCARD_CLASSIFICATION = new AtlasClassificationType(new
AtlasClassificationDef(WILDCARD_CLASSIFICATIONS));
public final static AtlasClassificationType MATCH_ALL_CLASSIFIED
= new AtlasClassificationType(new
AtlasClassificationDef(ALL_CLASSIFICATIONS));
@@ -146,6 +150,9 @@ public class SearchContext {
//remove other types if builtin type is present
filterStructTypes();
+ //validate 'attributes' field
+ validateAttributes();
+
//gather all classifications and its corresponding subtypes
Set<String> classificationTypeAndSubTypes = new HashSet<>();
String classificationTypeAndSubTypesQryStr = null;
@@ -345,6 +352,37 @@ public class SearchContext {
}
}
+ private void validateAttributes() throws AtlasBaseException {
+ Set<String> attributes = searchParameters.getAttributes();
+ if (CollectionUtils.isNotEmpty(attributes) &&
CollectionUtils.isNotEmpty(entityTypes)) {
+
+ AtlasEntityType entityType = entityTypes.iterator().next();
+ for (String attr : attributes) {
+ AtlasAttribute attribute = entityType.getAttribute(attr);
+
+ if (attribute == null) {
+ attribute = entityType.getRelationshipAttribute(attr,
null);
+ hasRelationshipAttributes = attribute != null;
+ }
+
+ if (attribute == null) {
+ throw new
AtlasBaseException(AtlasErrorCode.UNKNOWN_ATTRIBUTE, attr,
entityType.getTypeName());
+ }
+ }
+ }
+ }
+
+ public boolean excludeHeaderAttributes() {
+ if (CollectionUtils.isNotEmpty(entityTypes) &&
+ searchParameters.getExcludeHeaderAttributes() &&
+ CollectionUtils.isNotEmpty(searchParameters.getAttributes()) &&
+ !hasRelationshipAttributes){
+ return true;
+ }
+
+ return false;
+ }
+
public boolean hasAttributeFilter(FilterCriteria filterCriteria) {
return filterCriteria != null &&
(CollectionUtils.isNotEmpty(filterCriteria.getCriterion()) ||
StringUtils.isNotEmpty(filterCriteria.getAttributeName()));
diff --git
a/repository/src/test/java/org/apache/atlas/discovery/AtlasDiscoveryServiceTest.java
b/repository/src/test/java/org/apache/atlas/discovery/AtlasDiscoveryServiceTest.java
index fbc739652..ecb398a44 100644
---
a/repository/src/test/java/org/apache/atlas/discovery/AtlasDiscoveryServiceTest.java
+++
b/repository/src/test/java/org/apache/atlas/discovery/AtlasDiscoveryServiceTest.java
@@ -31,6 +31,7 @@ import org.apache.atlas.model.discovery.AtlasAggregationEntry;
import org.apache.atlas.model.instance.AtlasClassification;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntityHeader;
+import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.store.graph.v2.AtlasEntityStream;
@@ -42,6 +43,7 @@ import org.testng.annotations.*;
import javax.inject.Inject;
import java.util.Arrays;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -929,6 +931,109 @@ public class AtlasDiscoveryServiceTest extends
BasicTestSetup {
Assert.assertTrue(list.get(0).getDisplayText().equalsIgnoreCase("time_id"));
}
+ //test excludeHeaderAttributes
+ @Test
+ public void excludeHeaderAttributesStringAttr() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setTypeName(HIVE_TABLE_TYPE);
+ params.setExcludeHeaderAttributes(true);
+ SearchParameters.FilterCriteria filterCriteria =
getSingleFilterCondition("tableType", Operator.EQ, "Managed");
+ params.setEntityFilters(filterCriteria);
+ params.setSortBy("name");
+ params.setAttributes(new HashSet<String>() {{ add("name");}});
+ params.setLimit(1);
+
+ AtlasSearchResult searchResult =
discoveryService.searchWithParameters(params);
+ AtlasSearchResult.AttributeSearchResult expected = new
AtlasSearchResult.AttributeSearchResult();
+ expected.setName(Arrays.asList("name"));
+ expected.setValues(Arrays.asList(Arrays.asList("log_fact_daily_mv")));
+ assertSearchResult(searchResult,expected);
+ }
+
+ @Test
+ public void excludeHeaderAttributesRelationAttr() throws
AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setTypeName(HIVE_TABLE_TYPE);
+ params.setExcludeHeaderAttributes(true);
+ SearchParameters.FilterCriteria filterCriteria =
getSingleFilterCondition("name", Operator.EQ, "time_dim");
+ params.setEntityFilters(filterCriteria);
+ params.setAttributes(new HashSet<String>() {{ add("name");
add("db");}});
+ params.setLimit(1);
+
+ AtlasSearchResult searchResult =
discoveryService.searchWithParameters(params);
+
+ assertNotNull(searchResult);
+ assertNotNull(searchResult.getEntities());
+ assertNotNull(searchResult.getReferredEntities());
+ }
+
+ @Test
+ public void excludeHeaderAttributesSystemAttr() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setTypeName(HIVE_TABLE_TYPE);
+ params.setExcludeHeaderAttributes(true);
+ params.setAttributes(new HashSet<String>() {{ add("name");
add("__state");}});
+ params.setLimit(1);
+ SearchParameters.FilterCriteria filterCriteria =
getSingleFilterCondition("tableType", Operator.EQ, "Managed");
+ params.setEntityFilters(filterCriteria);
+ params.setSortBy("name");
+
+ AtlasSearchResult searchResult =
discoveryService.searchWithParameters(params);
+ AtlasSearchResult.AttributeSearchResult expected = new
AtlasSearchResult.AttributeSearchResult();
+ expected.setName(Arrays.asList("name","__state"));
+
expected.setValues(Arrays.asList(Arrays.asList("log_fact_daily_mv","ACTIVE")));
+ assertSearchResult(searchResult,expected);
+ }
+
+ @Test
+ public void excludeHeaderAttributesAllEntityTypeSysAttr() throws
AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setTypeName(HIVE_TABLE_TYPE+","+ALL_ENTITY_TYPES);
+ params.setExcludeHeaderAttributes(true);
+ params.setAttributes(new HashSet<String>() {{ add("__state");}});
+ params.setLimit(2);
+
+ AtlasSearchResult searchResult =
discoveryService.searchWithParameters(params);
+ AtlasSearchResult.AttributeSearchResult expected = new
AtlasSearchResult.AttributeSearchResult();
+ expected.setName(Arrays.asList("__state"));
+ expected.setValues(Arrays.asList(Arrays.asList("ACTIVE"),
Arrays.asList("ACTIVE")));
+ assertSearchResult(searchResult,expected);
+ }
+ @Test
+ public void excludeHeaderAttributesAllEntityTypeSysAttrs() throws
AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setTypeName(HIVE_TABLE_TYPE+","+ALL_ENTITY_TYPES);
+ params.setExcludeHeaderAttributes(true);
+ params.setAttributes(new HashSet<String>() {{ add("__state");
add("__guid");}});
+ params.setLimit(2);
+
+ AtlasSearchResult searchResult =
discoveryService.searchWithParameters(params);
+ assertEquals(searchResult.getAttributes().getValues().size(), 2);
+ }
+
+ @Test(expectedExceptions = AtlasBaseException.class,
expectedExceptionsMessageRegExp = "Attribute name not found for type
__ENTITY_ROOT")
+ public void excludeHeaderAttributesAllEntityType() throws
AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setTypeName(HIVE_TABLE_TYPE+","+ALL_ENTITY_TYPES);
+ params.setExcludeHeaderAttributes(true);
+ params.setAttributes(new HashSet<String>() {{ add("name");}});
+ params.setLimit(1);
+
+ discoveryService.searchWithParameters(params);
+ }
+
+ @Test(expectedExceptions = AtlasBaseException.class,
expectedExceptionsMessageRegExp = "Attribute name1 not found for type
hive_table")
+ public void excludeHeaderAttributesInvalidAttr() throws AtlasBaseException
{
+ SearchParameters params = new SearchParameters();
+ params.setTypeName(HIVE_TABLE_TYPE);
+ params.setExcludeHeaderAttributes(true);
+ params.setAttributes(new HashSet<String>() {{ add("name1");}});
+ params.setLimit(1);
+
+ discoveryService.searchWithParameters(params);
+ }
+
+
private String gethiveTableSalesFactGuid() throws AtlasBaseException {
if (salesFactGuid == null) {
SearchParameters params = new SearchParameters();
@@ -958,6 +1063,18 @@ public class AtlasDiscoveryServiceTest extends
BasicTestSetup {
}
}
+ private void assertSearchResult(AtlasSearchResult searchResult,
AtlasSearchResult.AttributeSearchResult expected) {
+ assertNotNull(searchResult);
+ AtlasSearchResult.AttributeSearchResult result =
searchResult.getAttributes();
+ assertNotNull(result);
+ assertTrue(result.getName().containsAll(expected.getName()));
+ int i = 0;
+ for (List<Object> value : result.getValues()) {
+ assertTrue(value.containsAll(expected.getValues().get(i)));
+ i++;
+ }
+ }
+
private void assertAggregationMetrics(AtlasQuickSearchResult searchResult)
{
Map<String, List<AtlasAggregationEntry>> agg =
searchResult.getAggregationMetrics();
Assert.assertTrue(CollectionUtils.isNotEmpty(agg.get("__typeName")));
diff --git
a/webapp/src/test/resources/json/search-parameters/combination-filters.json
b/webapp/src/test/resources/json/search-parameters/combination-filters.json
index dc52d33d1..f65e2bae3 100644
--- a/webapp/src/test/resources/json/search-parameters/combination-filters.json
+++ b/webapp/src/test/resources/json/search-parameters/combination-filters.json
@@ -10,7 +10,7 @@
"offset": 0,
"entityFilters": null,
"tagFilters": null,
- "attributes": [""]
+ "attributes": []
},
"expectedCount": 1
},
@@ -25,7 +25,7 @@
"offset": 0,
"entityFilters": null,
"tagFilters": null,
- "attributes": [""]
+ "attributes": []
},
"expectedCount": 0
},
@@ -129,7 +129,7 @@
]
},
"tagFilters": null,
- "attributes": [""]
+ "attributes": []
},
"expectedCount": 1
},
@@ -144,7 +144,7 @@
"offset": 0,
"entityFilters": null,
"tagFilters": null,
- "attributes": [""]
+ "attributes": []
},
"expectedCount": 1
},
@@ -159,7 +159,7 @@
"offset": 0,
"entityFilters": null,
"tagFilters": null,
- "attributes": [""]
+ "attributes": []
},
"expectedCount": 3
},
@@ -178,7 +178,7 @@
"attributeValue" : "+"
},
"tagFilters": null,
- "attributes": [""]
+ "attributes": []
},
"expectedCount": 1
}
diff --git
a/webapp/src/test/resources/json/search-parameters/entity-filters.json
b/webapp/src/test/resources/json/search-parameters/entity-filters.json
index 93d2d7d36..255ed8adb 100644
--- a/webapp/src/test/resources/json/search-parameters/entity-filters.json
+++ b/webapp/src/test/resources/json/search-parameters/entity-filters.json
@@ -15,7 +15,6 @@
},
"tagFilters": null,
"attributes": [
- ""
]
},
"expectedCount": 3
@@ -38,7 +37,6 @@
},
"tagFilters": null,
"attributes": [
- ""
]
},
"expectedCount": 3
@@ -59,7 +57,6 @@
},
"tagFilters": null,
"attributes": [
- ""
]
},
"expectedCount": 2
@@ -81,7 +78,6 @@
},
"tagFilters": null,
"attributes": [
- ""
]
},
"expectedCount": 1
@@ -113,7 +109,6 @@
},
"tagFilters": null,
"attributes": [
- ""
]
},
"expectedCount": 3
@@ -145,7 +140,6 @@
},
"tagFilters": null,
"attributes": [
- ""
]
},
"expectedCount": 3
@@ -177,7 +171,6 @@
},
"tagFilters": null,
"attributes": [
- ""
]
},
"expectedCount": 3
@@ -199,7 +192,6 @@
},
"tagFilters": null,
"attributes": [
- ""
]
},
"expectedCount": 3
@@ -231,7 +223,6 @@
},
"tagFilters": null,
"attributes": [
- ""
]
},
"expectedCount": 1
@@ -263,7 +254,6 @@
},
"tagFilters": null,
"attributes": [
- ""
]
},
"expectedCount": 18
@@ -295,7 +285,6 @@
},
"tagFilters": null,
"attributes": [
- ""
]
},
"expectedCount": 3
@@ -327,7 +316,6 @@
},
"tagFilters": null,
"attributes": [
- ""
]
},
"expectedCount": 3
@@ -359,7 +347,6 @@
},
"tagFilters": null,
"attributes": [
- ""
]
},
"expectedCount": 3
