This is an automated email from the ASF dual-hosted git repository.
sarath pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/master by this push:
new 7423add ATLAS-3563: Improve tag propagation performance using
in-memory traversal
7423add is described below
commit 7423addb220330cceadbf690d3bf4e4e5fcde99f
Author: Sarath Subramanian <[email protected]>
AuthorDate: Tue Dec 17 17:30:07 2019 -0800
ATLAS-3563: Improve tag propagation performance using in-memory traversal
---
.../atlas/repository/graphdb/AtlasVertex.java | 10 ++
.../repository/graphdb/janus/AtlasJanusVertex.java | 8 ++
.../main/java/org/apache/atlas/AtlasErrorCode.java | 1 +
.../org/apache/atlas/type/AtlasEntityType.java | 49 ++++++-
.../apache/atlas/repository/graph/GraphHelper.java | 153 +--------------------
.../repository/store/graph/v1/DeleteHandlerV1.java | 12 +-
.../store/graph/v2/AtlasRelationshipStoreV2.java | 112 ++++++++-------
.../store/graph/v2/EntityGraphMapper.java | 4 +-
.../store/graph/v2/EntityGraphRetriever.java | 138 ++++++++++++++++++-
.../atlas/util/AtlasGremlin3QueryProvider.java | 26 ----
.../atlas/util/AtlasGremlinQueryProvider.java | 5 -
.../ClassificationPropagationTest.java | 19 +--
12 files changed, 286 insertions(+), 251 deletions(-)
diff --git
a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasVertex.java
b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasVertex.java
index 6de4dcf..9406e26 100644
---
a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasVertex.java
+++
b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasVertex.java
@@ -37,6 +37,16 @@ public interface AtlasVertex<V, E> extends AtlasElement {
/**
* Gets the edges associated with this vertex going the
+ * specified direction that have the specified edgeLabels.
+ *
+ * @param direction
+ * @param edgeLabels
+ * @return
+ */
+ Iterable<AtlasEdge<V, E>> getEdges(AtlasEdgeDirection direction, String[]
edgeLabels);
+
+ /**
+ * Gets the edges associated with this vertex going the
* specified direction.
*
* @param in
diff --git
a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusVertex.java
b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusVertex.java
index 71b2857..fdc9fd0 100644
---
a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusVertex.java
+++
b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusVertex.java
@@ -70,6 +70,14 @@ public class AtlasJanusVertex extends
AtlasJanusElement<Vertex> implements Atlas
return graph.wrapEdges(edges);
}
+ @Override
+ public Iterable<AtlasEdge<AtlasJanusVertex, AtlasJanusEdge>>
getEdges(AtlasEdgeDirection dir, String[] edgeLabels) {
+ Direction direction =
AtlasJanusObjectFactory.createDirection(dir);
+ Iterator<Edge> edges = getWrappedElement().edges(direction,
edgeLabels);
+
+ return graph.wrapEdges(edges);
+ }
+
private JanusGraphVertex getAsJanusVertex() {
return (JanusGraphVertex)getWrappedElement();
}
diff --git a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
index 7a2aae2..3ebd70d 100644
--- a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
+++ b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
@@ -160,6 +160,7 @@ public enum AtlasErrorCode {
INVALID_CUSTOM_ATTRIBUTE_VALUE(400, "ATLAS-400-00-9A", "Invalid value: {0}
in custom attribute, value length is greater than {1}"),
INVALID_LABEL_LENGTH(400, "ATLAS-400-00-9B", "Invalid label: {0}, label
size should not be greater than {1}"),
INVALID_LABEL_CHARACTERS(400, "ATLAS-400-00-9C", "Invalid label: {0},
label should contain alphanumeric characters, '_' or '-'"),
+ INVALID_PROPAGATION_TYPE(400, "ATLAS-400-00-9D", "Invalid propagation {0}
for relationship-type={1}. Default value is {2}"),
UNAUTHORIZED_ACCESS(403, "ATLAS-403-00-001", "{0} is not authorized to
perform {1}"),
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
index 928ac0d..4742d1c 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
@@ -19,7 +19,6 @@ package org.apache.atlas.type;
import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_DATE;
import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_INT;
-import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_LONG;
import static
org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_STRING;
import static org.apache.atlas.type.Constants.*;
@@ -29,6 +28,7 @@ import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.model.typedef.AtlasEntityDef;
import
org.apache.atlas.model.typedef.AtlasEntityDef.AtlasRelationshipAttributeDef;
+import org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.type.AtlasBuiltInTypes.AtlasObjectIdType;
import org.apache.atlas.utils.AtlasEntityUtil;
@@ -87,6 +87,7 @@ public class AtlasEntityType extends AtlasStructType {
private List<AtlasAttribute> dynAttributes
= Collections.emptyList();
private List<AtlasAttribute> dynEvalTriggerAttributes
= Collections.emptyList();
private Map<String,List<TemplateToken>> parsedTemplates
= Collections.emptyMap();
+ private Set<String> tagPropagationEdges
= Collections.emptySet();
public AtlasEntityType(AtlasEntityDef entityDef) {
super(entityDef);
@@ -138,6 +139,7 @@ public class AtlasEntityType extends AtlasStructType {
this.allSubTypes = new HashSet<>(); // this will be
populated in resolveReferencesPhase2()
this.typeAndAllSubTypes = new HashSet<>(); // this will be
populated in resolveReferencesPhase2()
this.relationshipAttributes = new HashMap<>(); // this will be
populated in resolveReferencesPhase3()
+ this.tagPropagationEdges = new HashSet<>(); // this will be
populated in resolveReferencesPhase2()
this.typeAndAllSubTypes.add(this.getTypeName());
@@ -230,6 +232,8 @@ public class AtlasEntityType extends AtlasStructType {
}
}
}
+
+ tagPropagationEdges.addAll(superType.tagPropagationEdges);
}
ownedRefAttributes = new ArrayList<>();
@@ -254,6 +258,7 @@ public class AtlasEntityType extends AtlasStructType {
typeAndAllSubTypesQryStr = ""; // will be computed on next access
relationshipAttributes =
Collections.unmodifiableMap(relationshipAttributes);
ownedRefAttributes =
Collections.unmodifiableList(ownedRefAttributes);
+ tagPropagationEdges =
Collections.unmodifiableSet(tagPropagationEdges);
entityDef.setSubTypes(subTypes);
@@ -275,6 +280,8 @@ public class AtlasEntityType extends AtlasStructType {
this.parsedTemplates = parseDynAttributeTemplates();
populateDynFlagsInfo();
+
+ LOG.info("resolveReferencesPhase3({}): tagPropagationEdges={}",
getTypeName(), tagPropagationEdges);
}
public Set<String> getSuperTypes() {
@@ -347,6 +354,14 @@ public class AtlasEntityType extends AtlasStructType {
@VisibleForTesting
public void setDynEvalTriggerAttributes(List<AtlasAttribute>
dynEvalTriggerAttributes) { this.dynEvalTriggerAttributes =
dynEvalTriggerAttributes; }
+ public Set<String> getTagPropagationEdges() {
+ return this.tagPropagationEdges;
+ }
+
+ public String[] getTagPropagationEdgesArray() {
+ return CollectionUtils.isNotEmpty(tagPropagationEdges) ?
tagPropagationEdges.toArray(new String[tagPropagationEdges.size()]) : null;
+ }
+
public Map<String,List<TemplateToken>> getParsedTemplates() { return
parsedTemplates; }
public AtlasAttribute getRelationshipAttribute(String attributeName,
String relationshipType) {
@@ -377,6 +392,38 @@ public class AtlasEntityType extends AtlasStructType {
}
attributes.put(relationshipType.getTypeName(), attribute);
+
+ // determine if tags from this entity-type propagate via this
relationship
+ PropagateTags propagation =
relationshipType.getRelationshipDef().getPropagateTags();
+
+ if (propagation == null) {
+ propagation = PropagateTags.NONE;
+ }
+
+ final boolean propagatesTags;
+
+ switch (propagation) {
+ case BOTH:
+ propagatesTags = true;
+ break;
+
+ case ONE_TO_TWO:
+ propagatesTags =
StringUtils.equals(relationshipType.getEnd1Type().getTypeName(), getTypeName());
+ break;
+
+ case TWO_TO_ONE:
+ propagatesTags =
StringUtils.equals(relationshipType.getEnd2Type().getTypeName(), getTypeName());
+ break;
+
+ case NONE:
+ default:
+ propagatesTags = false;
+ break;
+ }
+
+ if (propagatesTags) {
+ tagPropagationEdges.add(relationshipType.getRelationshipLabel());
+ }
}
public Set<String> getAttributeRelationshipTypes(String attributeName) {
diff --git
a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
index 1e7acf1..5ab9f4d 100755
---
a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
+++
b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
@@ -60,9 +60,6 @@ import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.script.Bindings;
-import javax.script.ScriptEngine;
-import javax.script.ScriptException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -84,10 +81,6 @@ import static
org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.isRef
import static
org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.BOTH;
import static
org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.IN;
import static
org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.OUT;
-import static
org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery.TAG_PROPAGATION_IMPACTED_INSTANCES;
-import static
org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery.TAG_PROPAGATION_IMPACTED_INSTANCES_EXCLUDE_RELATIONSHIP;
-import static
org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery.TAG_PROPAGATION_IMPACTED_INSTANCES_FOR_REMOVAL;
-import static
org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery.TAG_PROPAGATION_IMPACTED_INSTANCES_WITH_RESTRICTIONS;
/**
* Utility class for graph operations.
@@ -728,130 +721,6 @@ public final class GraphHelper {
return result;
}
- public List<AtlasVertex>
getIncludedImpactedVerticesWithReferences(AtlasVertex entityVertex, String
relationshipGuid) throws AtlasBaseException {
- List<AtlasVertex> ret = new ArrayList<>();
- List<AtlasVertex> impactedVertices =
getImpactedVerticesWithReferences(getGuid(entityVertex), relationshipGuid);
-
- ret.add(entityVertex);
-
- if (CollectionUtils.isNotEmpty(impactedVertices)) {
- ret.addAll(impactedVertices);
- }
-
- return ret;
- }
-
- public List<AtlasVertex> getImpactedVertices(String guid) throws
AtlasBaseException {
- ScriptEngine scriptEngine = graph.getGremlinScriptEngine();
- Bindings bindings = scriptEngine.createBindings();
- String query =
queryProvider.getQuery(TAG_PROPAGATION_IMPACTED_INSTANCES);
- List<AtlasVertex> ret = new ArrayList<>();
-
- bindings.put("g", graph);
- bindings.put("guid", guid);
-
- try {
- Object resultObj = graph.executeGremlinScript(scriptEngine,
bindings, query, false);
-
- if (resultObj instanceof List && CollectionUtils.isNotEmpty((List)
resultObj)) {
- List<?> results = (List) resultObj;
- Object firstElement = results.get(0);
-
- if (firstElement instanceof AtlasVertex) {
- ret = (List<AtlasVertex>) results;
- }
- }
- } catch (ScriptException e) {
- throw new
AtlasBaseException(AtlasErrorCode.GREMLIN_SCRIPT_EXECUTION_FAILED, e);
- }
-
- return ret;
- }
-
- public List<AtlasVertex> getPropagatedEntityVertices(AtlasVertex
classificationVertex) throws AtlasBaseException {
- List<AtlasVertex> ret = new ArrayList<>();
-
- if (classificationVertex != null) {
- String entityGuid =
getClassificationEntityGuid(classificationVertex);
- String classificationId =
classificationVertex.getIdForDisplay();
- List<AtlasVertex> impactedEntityVertices =
getAllPropagatedEntityVertices(classificationVertex);
- List<AtlasVertex> impactedEntityVerticesWithRestrictions =
getImpactedVerticesWithRestrictions(entityGuid, classificationId);
-
- if (impactedEntityVertices.size() >
impactedEntityVerticesWithRestrictions.size()) {
- ret = (List<AtlasVertex>)
CollectionUtils.subtract(impactedEntityVertices,
impactedEntityVerticesWithRestrictions);
- } else {
- ret = (List<AtlasVertex>)
CollectionUtils.subtract(impactedEntityVerticesWithRestrictions,
impactedEntityVertices);
- }
- }
-
- return ret;
- }
-
- public List<AtlasVertex> getImpactedVerticesWithRestrictions(String guid,
String classificationId) throws AtlasBaseException {
- return getImpactedVerticesWithRestrictions(guid, classificationId,
null);
- }
-
- public List<AtlasVertex> getImpactedVerticesWithRestrictions(String guid,
String classificationId, String guidRelationshipToExclude) throws
AtlasBaseException {
- ScriptEngine scriptEngine = graph.getGremlinScriptEngine();
- Bindings bindings = scriptEngine.createBindings();
- List<AtlasVertex> ret = new ArrayList<>();
- String query =
queryProvider.getQuery(TAG_PROPAGATION_IMPACTED_INSTANCES_WITH_RESTRICTIONS);
-
- bindings.put("g", graph);
- bindings.put("guid", guid);
- bindings.put("classificationId", classificationId);
-
- if (guidRelationshipToExclude != null) {
- query =
queryProvider.getQuery(TAG_PROPAGATION_IMPACTED_INSTANCES_EXCLUDE_RELATIONSHIP);
- bindings.put("guidRelationshipToExclude",
guidRelationshipToExclude);
- }
-
- try {
- Object resultObj = graph.executeGremlinScript(scriptEngine,
bindings, query, false);
-
- if (resultObj instanceof List && CollectionUtils.isNotEmpty((List)
resultObj)) {
- List<?> results = (List) resultObj;
- Object firstElement = results.get(0);
-
- if (firstElement instanceof AtlasVertex) {
- ret = (List<AtlasVertex>) results;
- }
- }
- } catch (ScriptException e) {
- throw new
AtlasBaseException(AtlasErrorCode.GREMLIN_SCRIPT_EXECUTION_FAILED, e);
- }
-
- return ret;
- }
-
- public List<AtlasVertex> getImpactedVerticesWithReferences(String guid,
String relationshipGuid) throws AtlasBaseException {
- ScriptEngine scriptEngine = graph.getGremlinScriptEngine();
- Bindings bindings = scriptEngine.createBindings();
- String query =
queryProvider.getQuery(TAG_PROPAGATION_IMPACTED_INSTANCES_FOR_REMOVAL);
- List<AtlasVertex> ret = new ArrayList<>();
-
- bindings.put("g", graph);
- bindings.put("guid", guid);
- bindings.put("relationshipGuid", relationshipGuid);
-
- try {
- Object resultObj = graph.executeGremlinScript(scriptEngine,
bindings, query, false);
-
- if (resultObj instanceof List && CollectionUtils.isNotEmpty((List)
resultObj)) {
- List<?> results = (List) resultObj;
- Object firstElement = results.get(0);
-
- if (firstElement instanceof AtlasVertex) {
- ret = (List<AtlasVertex>) results;
- }
- }
- } catch (ScriptException e) {
- throw new
AtlasBaseException(AtlasErrorCode.GREMLIN_SCRIPT_EXECUTION_FAILED, e);
- }
-
- return ret;
- }
-
/**
* Finds the Vertices that correspond to the given GUIDs. GUIDs
* that are not found in the graph will not be in the map.
@@ -915,7 +784,7 @@ public final class GraphHelper {
return ret;
}
- public static List<AtlasVertex> getClassificationVertices(AtlasEdge edge) {
+ public static List<AtlasVertex> getPropagatableClassifications(AtlasEdge
edge) {
List<AtlasVertex> ret = new ArrayList<>();
if (edge != null && getStatus(edge) != DELETED) {
@@ -935,26 +804,6 @@ public final class GraphHelper {
return ret;
}
- public Map<AtlasVertex, List<AtlasVertex>>
getClassificationPropagatedEntitiesMapping(List<AtlasVertex>
classificationVertices) throws AtlasBaseException {
- return
getClassificationPropagatedEntitiesMapping(classificationVertices, null);
- }
-
- public Map<AtlasVertex, List<AtlasVertex>>
getClassificationPropagatedEntitiesMapping(List<AtlasVertex>
classificationVertices, String guidRelationshipToExclude) throws
AtlasBaseException {
- Map<AtlasVertex, List<AtlasVertex>> ret = new HashMap<>();
-
- if (CollectionUtils.isNotEmpty(classificationVertices)) {
- for (AtlasVertex classificationVertex : classificationVertices) {
- String classificationId =
classificationVertex.getIdForDisplay();
- String sourceEntityId =
getClassificationEntityGuid(classificationVertex);
- List<AtlasVertex> entitiesPropagatingTo =
getImpactedVerticesWithRestrictions(sourceEntityId, classificationId,
guidRelationshipToExclude);
-
- ret.put(classificationVertex, entitiesPropagatingTo);
- }
- }
-
- return ret;
- }
-
public static List<AtlasVertex>
getPropagationEnabledClassificationVertices(AtlasVertex entityVertex) {
List<AtlasVertex> ret = new ArrayList<>();
Iterable edges =
entityVertex.query().direction(AtlasEdgeDirection.OUT).label(CLASSIFICATION_LABEL).edges();
diff --git
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java
index c9ed797..a87e2b5 100644
---
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java
+++
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java
@@ -402,7 +402,7 @@ public abstract class DeleteHandlerV1 {
private void addTagPropagation(AtlasVertex fromVertex, AtlasVertex
toVertex, AtlasEdge edge) throws AtlasBaseException {
final List<AtlasVertex> classificationVertices =
getPropagationEnabledClassificationVertices(fromVertex);
- final List<AtlasVertex> propagatedEntityVertices =
CollectionUtils.isNotEmpty(classificationVertices) ?
graphHelper.getIncludedImpactedVerticesWithReferences(toVertex,
getRelationshipGuid(edge)) : null;
+ final List<AtlasVertex> propagatedEntityVertices =
CollectionUtils.isNotEmpty(classificationVertices) ?
entityRetriever.getIncludedImpactedVerticesV2(toVertex,
getRelationshipGuid(edge)) : null;
if (CollectionUtils.isNotEmpty(propagatedEntityVertices)) {
if (LOG.isDebugEnabled()) {
@@ -490,9 +490,9 @@ public abstract class DeleteHandlerV1 {
return;
}
- List<AtlasVertex> currentClassificationVertices =
getClassificationVertices(edge);
- Map<AtlasVertex, List<AtlasVertex>> currentClassificationsMap =
graphHelper.getClassificationPropagatedEntitiesMapping(currentClassificationVertices);
- Map<AtlasVertex, List<AtlasVertex>> updatedClassificationsMap =
graphHelper.getClassificationPropagatedEntitiesMapping(currentClassificationVertices,
getRelationshipGuid(edge));
+ List<AtlasVertex> currentClassificationVertices =
getPropagatableClassifications(edge);
+ Map<AtlasVertex, List<AtlasVertex>> currentClassificationsMap =
entityRetriever.getClassificationPropagatedEntitiesMapping(currentClassificationVertices);
+ Map<AtlasVertex, List<AtlasVertex>> updatedClassificationsMap =
entityRetriever.getClassificationPropagatedEntitiesMapping(currentClassificationVertices,
getRelationshipGuid(edge));
Map<AtlasVertex, List<AtlasVertex>> removePropagationsMap =
new HashMap<>();
if (MapUtils.isNotEmpty(currentClassificationsMap) &&
MapUtils.isEmpty(updatedClassificationsMap)) {
@@ -598,7 +598,7 @@ public abstract class DeleteHandlerV1 {
private void removeTagPropagation(AtlasVertex fromVertex, AtlasVertex
toVertex, AtlasEdge edge) throws AtlasBaseException {
final List<AtlasVertex> classificationVertices =
getPropagationEnabledClassificationVertices(fromVertex);
- final List<AtlasVertex> impactedEntityVertices =
CollectionUtils.isNotEmpty(classificationVertices) ?
graphHelper.getIncludedImpactedVerticesWithReferences(toVertex,
getRelationshipGuid(edge)) : null;
+ final List<AtlasVertex> impactedEntityVertices =
CollectionUtils.isNotEmpty(classificationVertices) ?
entityRetriever.getIncludedImpactedVerticesV2(toVertex,
getRelationshipGuid(edge)) : null;
if (CollectionUtils.isNotEmpty(impactedEntityVertices)) {
if (LOG.isDebugEnabled()) {
@@ -608,7 +608,7 @@ public abstract class DeleteHandlerV1 {
for (AtlasVertex classificationVertex : classificationVertices) {
String classificationName =
getTypeName(classificationVertex);
AtlasVertex associatedEntityVertex =
getAssociatedEntityVertex(classificationVertex);
- List<AtlasVertex> referrals =
graphHelper.getIncludedImpactedVerticesWithReferences(associatedEntityVertex,
getRelationshipGuid(edge));
+ List<AtlasVertex> referrals =
entityRetriever.getIncludedImpactedVerticesV2(associatedEntityVertex,
getRelationshipGuid(edge));
for (AtlasVertex impactedEntityVertex :
impactedEntityVertices) {
if (referrals.contains(impactedEntityVertex)) {
diff --git
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java
index 1c8b057..fdf117a 100644
---
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java
+++
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java
@@ -69,8 +69,7 @@ import java.util.UUID;
import static org.apache.atlas.model.instance.AtlasEntity.Status.ACTIVE;
import static org.apache.atlas.model.instance.AtlasEntity.Status.DELETED;
-import static
org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags.ONE_TO_TWO;
-import static
org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags.TWO_TO_ONE;
+import static
org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags.*;
import static org.apache.atlas.repository.Constants.ENTITY_TYPE_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.HOME_ID_KEY;
import static org.apache.atlas.repository.Constants.PROVENANCE_TYPE_KEY;
@@ -83,7 +82,7 @@ import static
org.apache.atlas.AtlasConfiguration.NOTIFICATION_RELATIONSHIPS_ENA
import static
org.apache.atlas.repository.graph.GraphHelper.getBlockedClassificationIds;
import static
org.apache.atlas.repository.graph.GraphHelper.getClassificationEntityGuid;
import static
org.apache.atlas.repository.graph.GraphHelper.getClassificationName;
-import static
org.apache.atlas.repository.graph.GraphHelper.getClassificationVertices;
+import static
org.apache.atlas.repository.graph.GraphHelper.getPropagatableClassifications;
import static
org.apache.atlas.repository.graph.GraphHelper.getIncomingEdgesByLabel;
import static org.apache.atlas.repository.graph.GraphHelper.getPropagateTags;
import static
org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.getState;
@@ -453,60 +452,53 @@ public class AtlasRelationshipStoreV2 implements
AtlasRelationshipStore {
return entityRetriever.mapEdgeToAtlasRelationship(relationshipEdge);
}
- private void handleBlockedClassifications(AtlasEdge edge,
Set<AtlasClassification> blockedPropagatedClassifications) throws
AtlasBaseException {
- if (blockedPropagatedClassifications != null) {
- List<AtlasVertex> propagatedClassificationVertices =
getClassificationVertices(edge);
- List<String> currentClassificationIds =
getBlockedClassificationIds(edge);
- List<AtlasVertex> currentBlockedPropagatedClassificationVertices =
getBlockedClassificationVertices(propagatedClassificationVertices,
currentClassificationIds);
- List<AtlasVertex> updatedBlockedPropagatedClassificationVertices =
new ArrayList<>();
- List<String> updatedClassificationIds =
new ArrayList<>();
+ private void handleBlockedClassifications(AtlasEdge edge,
Set<AtlasClassification> blockedClassifications) throws AtlasBaseException {
+ if (blockedClassifications != null) {
+ List<AtlasVertex> propagatableClassifications =
getPropagatableClassifications(edge);
+ List<String> currBlockedClassificationIds =
getBlockedClassificationIds(edge);
+ List<AtlasVertex> currBlockedClassifications =
getVerticesForIds(propagatableClassifications, currBlockedClassificationIds);
+ List<AtlasVertex> classificationsToBlock = new ArrayList<>();
+ List<String> classificationIdsToBlock = new ArrayList<>();
- for (AtlasClassification classification :
blockedPropagatedClassifications) {
- AtlasVertex classificationVertex =
validateBlockedPropagatedClassification(propagatedClassificationVertices,
classification);
+ for (AtlasClassification blockedClassification :
blockedClassifications) {
+ AtlasVertex classificationVertex =
validateBlockedPropagatedClassification(propagatableClassifications,
blockedClassification);
- // ignore invalid blocked propagated classification
- if (classificationVertex == null) {
- continue;
+ if (classificationVertex != null) {
+ classificationsToBlock.add(classificationVertex);
+
classificationIdsToBlock.add(classificationVertex.getIdForDisplay());
}
-
-
updatedBlockedPropagatedClassificationVertices.add(classificationVertex);
-
- String classificationId =
classificationVertex.getIdForDisplay();
-
- updatedClassificationIds.add(classificationId);
}
- addToBlockedClassificationIds(edge, updatedClassificationIds);
+ setBlockedClassificationIds(edge, classificationIdsToBlock);
- // remove propagated tag for added entry
- List<AtlasVertex> addedBlockedClassifications =
(List<AtlasVertex>)
CollectionUtils.subtract(updatedBlockedPropagatedClassificationVertices,
currentBlockedPropagatedClassificationVertices);
+ List<AtlasVertex> propagationChangedClassifications =
(List<AtlasVertex>) CollectionUtils.disjunction(classificationsToBlock,
currBlockedClassifications);
- for (AtlasVertex classificationVertex :
addedBlockedClassifications) {
- List<AtlasVertex> removePropagationFromVertices =
graphHelper.getPropagatedEntityVertices(classificationVertex);
-
-
deleteDelegate.getHandler().removeTagPropagation(classificationVertex,
removePropagationFromVertices);
- }
+ for (AtlasVertex classificationVertex :
propagationChangedClassifications) {
+ List<AtlasVertex> propagationsToRemove = new ArrayList<>();
+ List<AtlasVertex> propagationsToAdd = new ArrayList<>();
- // add propagated tag for removed entry
- List<AtlasVertex> removedBlockedClassifications =
(List<AtlasVertex>)
CollectionUtils.subtract(currentBlockedPropagatedClassificationVertices,
updatedBlockedPropagatedClassificationVertices);
+
entityRetriever.evaluateClassificationPropagation(classificationVertex,
propagationsToAdd, propagationsToRemove);
- for (AtlasVertex classificationVertex :
removedBlockedClassifications) {
- List<AtlasVertex> addPropagationToVertices =
graphHelper.getPropagatedEntityVertices(classificationVertex);
+ if (CollectionUtils.isNotEmpty(propagationsToAdd)) {
+
deleteDelegate.getHandler().addTagPropagation(classificationVertex,
propagationsToAdd);
+ }
-
deleteDelegate.getHandler().addTagPropagation(classificationVertex,
addPropagationToVertices);
+ if (CollectionUtils.isNotEmpty(propagationsToRemove)) {
+
deleteDelegate.getHandler().removeTagPropagation(classificationVertex,
propagationsToRemove);
+ }
}
}
}
- private List<AtlasVertex>
getBlockedClassificationVertices(List<AtlasVertex> classificationVertices,
List<String> blockedClassificationIds) {
+ private List<AtlasVertex> getVerticesForIds(List<AtlasVertex> vertices,
List<String> vertexIds) {
List<AtlasVertex> ret = new ArrayList<>();
- if (CollectionUtils.isNotEmpty(blockedClassificationIds)) {
- for (AtlasVertex classificationVertex : classificationVertices) {
- String classificationId =
classificationVertex.getIdForDisplay();
+ if (CollectionUtils.isNotEmpty(vertexIds)) {
+ for (AtlasVertex vertex : vertices) {
+ String vertexId = vertex.getIdForDisplay();
- if (blockedClassificationIds.contains(classificationId)) {
- ret.add(classificationVertex);
+ if (vertexIds.contains(vertexId)) {
+ ret.add(vertex);
}
}
}
@@ -515,7 +507,6 @@ public class AtlasRelationshipStoreV2 implements
AtlasRelationshipStore {
}
// propagated classifications should contain blocked propagated
classification
-
private AtlasVertex
validateBlockedPropagatedClassification(List<AtlasVertex>
classificationVertices, AtlasClassification classification) {
AtlasVertex ret = null;
@@ -531,7 +522,8 @@ public class AtlasRelationshipStoreV2 implements
AtlasRelationshipStore {
return ret;
}
- private void addToBlockedClassificationIds(AtlasEdge edge, List<String>
classificationIds) {
+
+ private void setBlockedClassificationIds(AtlasEdge edge, List<String>
classificationIds) {
if (edge != null) {
if (classificationIds.isEmpty()) {
edge.removeProperty(Constants.RELATIONSHIPTYPE_BLOCKED_PROPAGATED_CLASSIFICATIONS_KEY);
@@ -546,15 +538,15 @@ public class AtlasRelationshipStoreV2 implements
AtlasRelationshipStore {
PropagateTags newTagPropagation = relationship.getPropagateTags();
if (newTagPropagation != oldTagPropagation) {
- List<AtlasVertex> currentClassificationVertices
= getClassificationVertices(edge);
- Map<AtlasVertex, List<AtlasVertex>> currentClassificationsMap
=
graphHelper.getClassificationPropagatedEntitiesMapping(currentClassificationVertices);
+ List<AtlasVertex> currentClassificationVertices
= getPropagatableClassifications(edge);
+ Map<AtlasVertex, List<AtlasVertex>> currentClassificationsMap
=
entityRetriever.getClassificationPropagatedEntitiesMapping(currentClassificationVertices);
// Update propagation edge
AtlasGraphUtilsV2.setEncodedProperty(edge,
RELATIONSHIPTYPE_TAG_PROPAGATION_KEY, newTagPropagation.name());
- List<AtlasVertex> updatedClassificationVertices
= getClassificationVertices(edge);
+ List<AtlasVertex> updatedClassificationVertices
= getPropagatableClassifications(edge);
List<AtlasVertex> classificationVerticesUnion
= (List<AtlasVertex>) CollectionUtils.union(currentClassificationVertices,
updatedClassificationVertices);
- Map<AtlasVertex, List<AtlasVertex>> updatedClassificationsMap
=
graphHelper.getClassificationPropagatedEntitiesMapping(classificationVerticesUnion);
+ Map<AtlasVertex, List<AtlasVertex>> updatedClassificationsMap
=
entityRetriever.getClassificationPropagatedEntitiesMapping(classificationVerticesUnion);
// compute add/remove propagations list
Map<AtlasVertex, List<AtlasVertex>> addPropagationsMap = new
HashMap<>();
@@ -665,6 +657,34 @@ public class AtlasRelationshipStoreV2 implements
AtlasRelationshipStore {
throw new
AtlasBaseException(AtlasErrorCode.INVALID_RELATIONSHIP_END_TYPE,
relationshipName, relationshipType.getEnd2Type().getTypeName(), end1TypeName);
}
+ PropagateTags typePropagation =
relationshipType.getRelationshipDef().getPropagateTags();
+ PropagateTags edgePropagation = relationship.getPropagateTags();
+
+ if (typePropagation == null) {
+ typePropagation = NONE;
+ }
+
+ if (edgePropagation == null) {
+ edgePropagation = NONE;
+ }
+
+ /*
+ +-------------+----------------------------------------+
+ | type | edge |
+ +-------------+-------+------------+------------+------+
+ | | NONE | ONE_TO_TWO | TWO_TO_ONE | BOTH |
+ |-------------+-------+------------+------------+------|
+ | NONE | Y | N | N | N |
+ | ONE_TO_TWO | Y | Y | N | N |
+ | TWO_TO_ONE | Y | N | Y | N |
+ | BOTH | Y | Y | Y | Y |
+ +-------------+-------+------------+------------+------+
+ */
+
+ if (edgePropagation != NONE && typePropagation != BOTH &&
edgePropagation != typePropagation) {
+ throw new
AtlasBaseException(AtlasErrorCode.INVALID_PROPAGATION_TYPE,
edgePropagation.toString(), relationshipName, typePropagation.toString());
+ }
+
List<String> messages = new ArrayList<>();
relationshipType.validateValue(relationship, relationshipName,
messages);
diff --git
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
index a415d30..7a2ccb9 100644
---
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
+++
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
@@ -1757,7 +1757,7 @@ public class EntityGraphMapper {
if (propagateTags) {
// compute propagatedEntityVertices only once
if (entitiesToPropagateTo == null) {
- entitiesToPropagateTo =
graphHelper.getImpactedVertices(guid);
+ entitiesToPropagateTo =
entityRetriever.getImpactedVerticesV2(entityVertex);
}
if (CollectionUtils.isNotEmpty(entitiesToPropagateTo)) {
@@ -2092,7 +2092,7 @@ public class EntityGraphMapper {
if (updatedTagPropagation != null && currentTagPropagation !=
updatedTagPropagation) {
if (updatedTagPropagation) {
if (CollectionUtils.isEmpty(entitiesToPropagateTo)) {
- entitiesToPropagateTo =
graphHelper.getImpactedVerticesWithRestrictions(guid,
classificationVertex.getIdForDisplay());
+ entitiesToPropagateTo =
entityRetriever.getImpactedVerticesV2(entityVertex, null,
classificationVertex.getIdForDisplay());
}
if (CollectionUtils.isNotEmpty(entitiesToPropagateTo)) {
diff --git
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java
index 8a24fa1..4becb33 100644
---
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java
+++
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java
@@ -35,6 +35,7 @@ import org.apache.atlas.model.instance.AtlasRelationship;
import
org.apache.atlas.model.instance.AtlasRelationship.AtlasRelationshipWithExtInfo;
import org.apache.atlas.model.instance.AtlasStruct;
import org.apache.atlas.model.typedef.AtlasRelationshipDef;
+import org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags;
import org.apache.atlas.model.typedef.AtlasRelationshipEndDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.repository.Constants;
@@ -66,6 +67,7 @@ import javax.inject.Inject;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
@@ -97,6 +99,9 @@ import static
org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_INT;
import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_LONG;
import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_SHORT;
import static
org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_STRING;
+import static
org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags.NONE;
+import static
org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags.ONE_TO_TWO;
+import static
org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags.TWO_TO_ONE;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_ENTITY_GUID;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_LABEL;
import static
org.apache.atlas.repository.Constants.CLASSIFICATION_VALIDITY_PERIODS_KEY;
@@ -397,6 +402,137 @@ public class EntityGraphRetriever {
return ret;
}
+ public void evaluateClassificationPropagation(AtlasVertex
classificationVertex, List<AtlasVertex> entitiesToAddPropagation,
List<AtlasVertex> entitiesToRemovePropagation) {
+ if (classificationVertex != null) {
+ String entityGuid =
getClassificationEntityGuid(classificationVertex);
+ AtlasVertex entityVertex =
AtlasGraphUtilsV2.findByGuid(entityGuid);
+ String classificationId =
classificationVertex.getIdForDisplay();
+ List<AtlasVertex> propagatedEntities =
getAllPropagatedEntityVertices(classificationVertex);
+ List<AtlasVertex> impactedEntities =
getImpactedVerticesV2(entityVertex, null, classificationId);
+
+ List<AtlasVertex> entityVertices = (List<AtlasVertex>)
CollectionUtils.subtract(propagatedEntities, impactedEntities);
+
+ if (CollectionUtils.isNotEmpty(entityVertices)) {
+ entitiesToRemovePropagation.addAll(entityVertices);
+ }
+
+ entityVertices = (List<AtlasVertex>)
CollectionUtils.subtract(impactedEntities, propagatedEntities);
+
+ if (CollectionUtils.isNotEmpty(entityVertices)) {
+ entitiesToAddPropagation.addAll(entityVertices);
+ }
+ }
+ }
+
+ public Map<AtlasVertex, List<AtlasVertex>>
getClassificationPropagatedEntitiesMapping(List<AtlasVertex>
classificationVertices) {
+ return
getClassificationPropagatedEntitiesMapping(classificationVertices, null);
+ }
+
+ public Map<AtlasVertex, List<AtlasVertex>>
getClassificationPropagatedEntitiesMapping(List<AtlasVertex>
classificationVertices, String relationshipGuidToExclude) {
+ Map<AtlasVertex, List<AtlasVertex>> ret = new HashMap<>();
+
+ if (CollectionUtils.isNotEmpty(classificationVertices)) {
+ for (AtlasVertex classificationVertex : classificationVertices) {
+ String classificationId =
classificationVertex.getIdForDisplay();
+ String sourceEntityId =
getClassificationEntityGuid(classificationVertex);
+ AtlasVertex sourceEntityVertex =
AtlasGraphUtilsV2.findByGuid(sourceEntityId);
+ List<AtlasVertex> entitiesPropagatingTo =
getImpactedVerticesV2(sourceEntityVertex, relationshipGuidToExclude,
classificationId);
+
+ ret.put(classificationVertex, entitiesPropagatingTo);
+ }
+ }
+
+ return ret;
+ }
+
+ public List<AtlasVertex> getImpactedVerticesV2(AtlasVertex entityVertex) {
+ return getImpactedVerticesV2(entityVertex, null);
+ }
+
+ public List<AtlasVertex> getImpactedVerticesV2(AtlasVertex entityVertex,
String relationshipGuidToExclude) {
+ List<AtlasVertex> ret = new ArrayList<>();
+
+ traverseImpactedVertices(entityVertex, relationshipGuidToExclude,
null, new HashSet<>(), ret);
+
+ return ret;
+ }
+
+ public List<AtlasVertex> getIncludedImpactedVerticesV2(AtlasVertex
entityVertex, String relationshipGuidToExclude) {
+ List<AtlasVertex> ret = new ArrayList<>(Arrays.asList(entityVertex));
+
+ traverseImpactedVertices(entityVertex, relationshipGuidToExclude,
null, new HashSet<>(), ret);
+
+ return ret;
+ }
+
+ public List<AtlasVertex> getImpactedVerticesV2(AtlasVertex entityVertex,
String relationshipGuidToExclude, String classificationId) {
+ List<AtlasVertex> ret = new ArrayList<>();
+
+ traverseImpactedVertices(entityVertex, relationshipGuidToExclude,
classificationId, new HashSet<>(), ret);
+
+ return ret;
+ }
+
+ private void traverseImpactedVertices(AtlasVertex entityVertex, String
relationshipGuidToExclude, String classificationId, Set<String>
visitedVertices, List<AtlasVertex> result) {
+ visitedVertices.add(entityVertex.getIdForDisplay());
+
+ AtlasEntityType entityType =
typeRegistry.getEntityTypeByName(getTypeName(entityVertex));
+ String[] tagPropagationEdges = entityType != null ?
entityType.getTagPropagationEdgesArray() : null;
+
+ if (tagPropagationEdges != null) {
+ Iterable<AtlasEdge> propagationEdges =
entityVertex.getEdges(AtlasEdgeDirection.BOTH, tagPropagationEdges);
+
+ for (AtlasEdge propagationEdge : propagationEdges) {
+ PropagateTags tagPropagation =
getPropagateTags(propagationEdge);
+
+ if (tagPropagation == null || tagPropagation == NONE) {
+ continue;
+ } else if (tagPropagation == TWO_TO_ONE) {
+ if (isOutVertex(entityVertex, propagationEdge)) {
+ continue;
+ }
+ } else if (tagPropagation == ONE_TO_TWO) {
+ if (!isOutVertex(entityVertex, propagationEdge)) {
+ continue;
+ }
+ }
+
+ if (relationshipGuidToExclude != null) {
+ if
(StringUtils.equals(getRelationshipGuid(propagationEdge),
relationshipGuidToExclude)) {
+ continue;
+ }
+ }
+
+ if (classificationId != null) {
+ List<String> blockedClassificationIds =
getBlockedClassificationIds(propagationEdge);
+
+ if (CollectionUtils.isNotEmpty(blockedClassificationIds)
&& blockedClassificationIds.contains(classificationId)) {
+ continue;
+ }
+ }
+
+ AtlasVertex adjacentVertex = getOtherVertex(propagationEdge,
entityVertex);
+
+ if
(!visitedVertices.contains(adjacentVertex.getIdForDisplay())) {
+ result.add(adjacentVertex);
+
+ traverseImpactedVertices(adjacentVertex,
relationshipGuidToExclude, classificationId, visitedVertices, result);
+ }
+ }
+ }
+ }
+
+ private boolean isOutVertex(AtlasVertex vertex, AtlasEdge edge) {
+ return StringUtils.equals(vertex.getIdForDisplay(),
edge.getOutVertex().getIdForDisplay());
+ }
+
+ private AtlasVertex getOtherVertex(AtlasEdge edge, AtlasVertex vertex) {
+ AtlasVertex outVertex = edge.getOutVertex();
+ AtlasVertex inVertex = edge.getInVertex();
+
+ return StringUtils.equals(outVertex.getIdForDisplay(),
vertex.getIdForDisplay()) ? inVertex : outVertex;
+ }
+
private AtlasVertex getEntityVertex(AtlasObjectId objId) throws
AtlasBaseException {
AtlasVertex ret = null;
@@ -1390,7 +1526,7 @@ public class EntityGraphRetriever {
}
private void readClassificationsFromEdge(AtlasEdge edge,
AtlasRelationshipWithExtInfo relationshipWithExtInfo, boolean extendedInfo)
throws AtlasBaseException {
- List<AtlasVertex> classificationVertices =
getClassificationVertices(edge);
+ List<AtlasVertex> classificationVertices =
getPropagatableClassifications(edge);
List<String> blockedClassificationIds =
getBlockedClassificationIds(edge);
AtlasRelationship relationship =
relationshipWithExtInfo.getRelationship();
Set<AtlasClassification> propagatedClassifications = new HashSet<>();
diff --git
a/repository/src/main/java/org/apache/atlas/util/AtlasGremlin3QueryProvider.java
b/repository/src/main/java/org/apache/atlas/util/AtlasGremlin3QueryProvider.java
index 20c570f..547a43d 100644
---
a/repository/src/main/java/org/apache/atlas/util/AtlasGremlin3QueryProvider.java
+++
b/repository/src/main/java/org/apache/atlas/util/AtlasGremlin3QueryProvider.java
@@ -65,32 +65,6 @@ public class AtlasGremlin3QueryProvider extends
AtlasGremlin2QueryProvider {
return "g.V().range(0,1).toList()";
case GREMLIN_SEARCH_RETURNS_EDGE_ID:
return "g.E().range(0,1).toList()";
-
- case TAG_PROPAGATION_IMPACTED_INSTANCES:
- return "g.V().has('__guid', guid).aggregate('src')" +
- ".repeat(union(outE().has('__state',
'ACTIVE').has('tagPropagation', within('ONE_TO_TWO', 'BOTH')).inV(), " +
- "inE().has('__state',
'ACTIVE').has('tagPropagation', within('TWO_TO_ONE', 'BOTH')).outV())" +
-
".dedup().where(without('src')).simplePath()).emit().toList();";
-
- case TAG_PROPAGATION_IMPACTED_INSTANCES_WITH_RESTRICTIONS:
- return "g.V().has('__guid', guid).aggregate('src')" +
- ".repeat(union(outE().has('__state',
'ACTIVE').has('tagPropagation', within('ONE_TO_TWO',
'BOTH')).not(has('blockedPropagatedClassifications',
org.janusgraph.core.attribute.Text.textContains(classificationId))).inV(), " +
- "inE().has('__state',
'ACTIVE').has('tagPropagation', within('TWO_TO_ONE',
'BOTH')).not(has('blockedPropagatedClassifications',
org.janusgraph.core.attribute.Text.textContains(classificationId))).outV())" +
-
".dedup().where(without('src')).simplePath()).emit().toList();";
-
- case TAG_PROPAGATION_IMPACTED_INSTANCES_FOR_REMOVAL:
- return "g.V().has('__guid', guid).aggregate('src')" +
- ".repeat(union(outE().has('__state',
'ACTIVE').has('tagPropagation', within('ONE_TO_TWO', 'BOTH')).has('_r__guid',
neq(relationshipGuid)).inV(), " +
- "inE().has('__state',
'ACTIVE').has('tagPropagation', within('TWO_TO_ONE', 'BOTH')).has('_r__guid',
neq(relationshipGuid)).outV())" +
-
".dedup().where(without('src')).simplePath()).emit().toList();";
-
- case TAG_PROPAGATION_IMPACTED_INSTANCES_EXCLUDE_RELATIONSHIP:
- return "g.V().has('__guid', guid).aggregate('src')" +
- ".repeat(union(outE().has('__state',
'ACTIVE').has('tagPropagation', within('ONE_TO_TWO', 'BOTH')).has('_r__guid',
neq(guidRelationshipToExclude))" +
-
".not(has('blockedPropagatedClassifications',
org.janusgraph.core.attribute.Text.textContains(classificationId))).inV(), " +
- "inE().has('__state',
'ACTIVE').has('tagPropagation', within('TWO_TO_ONE', 'BOTH')).has('_r__guid',
neq(guidRelationshipToExclude))" +
-
".not(has('blockedPropagatedClassifications',
org.janusgraph.core.attribute.Text.textContains(classificationId))).outV())" +
-
".dedup().where(without('src')).simplePath()).emit().toList();";
}
return super.getQuery(gremlinQuery);
}
diff --git
a/repository/src/main/java/org/apache/atlas/util/AtlasGremlinQueryProvider.java
b/repository/src/main/java/org/apache/atlas/util/AtlasGremlinQueryProvider.java
index d201db3..1bb725a 100644
---
a/repository/src/main/java/org/apache/atlas/util/AtlasGremlinQueryProvider.java
+++
b/repository/src/main/java/org/apache/atlas/util/AtlasGremlinQueryProvider.java
@@ -85,10 +85,5 @@ public abstract class AtlasGremlinQueryProvider {
COMPARE_CONTAINS,
COMPARE_IS_NULL,
COMPARE_NOT_NULL,
-
- TAG_PROPAGATION_IMPACTED_INSTANCES,
- TAG_PROPAGATION_IMPACTED_INSTANCES_FOR_REMOVAL,
- TAG_PROPAGATION_IMPACTED_INSTANCES_WITH_RESTRICTIONS,
- TAG_PROPAGATION_IMPACTED_INSTANCES_EXCLUDE_RELATIONSHIP
}
}
diff --git
a/repository/src/test/java/org/apache/atlas/repository/tagpropagation/ClassificationPropagationTest.java
b/repository/src/test/java/org/apache/atlas/repository/tagpropagation/ClassificationPropagationTest.java
index 6f9c05e..6269d3f 100644
---
a/repository/src/test/java/org/apache/atlas/repository/tagpropagation/ClassificationPropagationTest.java
+++
b/repository/src/test/java/org/apache/atlas/repository/tagpropagation/ClassificationPropagationTest.java
@@ -18,6 +18,7 @@
package org.apache.atlas.repository.tagpropagation;
import com.vividsolutions.jts.util.Assert;
+import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.RequestContext;
import org.apache.atlas.TestModules;
import org.apache.atlas.discovery.AtlasLineageService;
@@ -35,7 +36,6 @@ import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.impexp.ImportService;
import org.apache.atlas.repository.impexp.ZipFileResourceTestUtils;
-import org.apache.atlas.repository.impexp.ZipSource;
import org.apache.atlas.repository.store.graph.AtlasEntityStore;
import org.apache.atlas.repository.store.graph.AtlasRelationshipStore;
import org.apache.atlas.runner.LocalSolrRunner;
@@ -49,7 +49,6 @@ import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import javax.inject.Inject;
-import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Arrays;
@@ -320,20 +319,16 @@ public class ClassificationPropagationTest {
// validate tag2 is propagated to employees_union
assertClassificationExistInEntity(EMPLOYEES_UNION_TABLE, tag2);
- //update propagation to BOTH for edge process3 --> employee_union
+ //update propagation to BOTH for edge process3 --> employee_union.
This should fail
AtlasRelationship process3_employee_union_relationship =
getRelationship(EMPLOYEES_UNION_PROCESS, EMPLOYEES_UNION_TABLE);
assertEquals(process3_employee_union_relationship.getPropagateTags(),
ONE_TO_TWO);
process3_employee_union_relationship.setPropagateTags(BOTH);
- relationshipStore.update(process3_employee_union_relationship);
-
- // process3 should get 'tag4' from employee_union and employee_union
should get tag3 from process3 (BOTH)
- assertClassificationExistInEntity(EMPLOYEES_UNION_PROCESS, tag4);
- assertClassificationExistInEntity(EMPLOYEES_UNION_TABLE, tag3);
- //update propagation to ONE_TO_TWO for edge process3 --> employee_union
- process3_employee_union_relationship.setPropagateTags(ONE_TO_TWO);
- relationshipStore.update(process3_employee_union_relationship);
- assertClassificationNotExistInEntity(EMPLOYEES_UNION_PROCESS, tag4);
+ try {
+ relationshipStore.update(process3_employee_union_relationship);
+ } catch (AtlasBaseException ex) {
+ assertEquals(ex.getAtlasErrorCode(),
AtlasErrorCode.INVALID_PROPAGATION_TYPE);
+ }
//cleanup
deleteClassification(hdfs_employees, tag1);
