Repository: atlas Updated Branches: refs/heads/master ad6b07a98 -> 154dda0e5
ATLAS-2169: delete request fails when hard-delete is configured Project: http://git-wip-us.apache.org/repos/asf/atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/154dda0e Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/154dda0e Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/154dda0e Branch: refs/heads/master Commit: 154dda0e5b03020b405faed8fd37f15654f492f4 Parents: ad6b07a Author: Madhan Neethiraj <[email protected]> Authored: Sun Feb 11 02:26:28 2018 -0800 Committer: Madhan Neethiraj <[email protected]> Committed: Sun Feb 11 22:31:36 2018 -0800 ---------------------------------------------------------------------- .../model/instance/EntityMutationResponse.java | 35 +- .../atlas/repository/graph/GraphHelper.java | 64 +-- .../graph/v1/AtlasEntityChangeNotifier.java | 15 +- .../store/graph/v1/AtlasEntityStoreV1.java | 14 +- .../store/graph/v1/DeleteHandlerV1.java | 522 ++++++++++--------- .../store/graph/v1/EntityGraphMapper.java | 50 +- .../store/graph/v1/EntityGraphRetriever.java | 64 +++ .../java/org/apache/atlas/RequestContextV1.java | 63 +-- 8 files changed, 452 insertions(+), 375 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/atlas/blob/154dda0e/intg/src/main/java/org/apache/atlas/model/instance/EntityMutationResponse.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/model/instance/EntityMutationResponse.java b/intg/src/main/java/org/apache/atlas/model/instance/EntityMutationResponse.java index 05411d6..4589262 100644 --- a/intg/src/main/java/org/apache/atlas/model/instance/EntityMutationResponse.java +++ b/intg/src/main/java/org/apache/atlas/model/instance/EntityMutationResponse.java @@ -195,7 +195,7 @@ public class EntityMutationResponse { public void addEntity(EntityOperation op, AtlasEntityHeader header) { // if an entity is already included in CREATE, ignore subsequent UPDATE, PARTIAL_UPDATE if (op == EntityOperation.UPDATE || op == EntityOperation.PARTIAL_UPDATE) { - if (entityHeaderExists(getCreatedEntities(), header)) { + if (entityHeaderExists(getCreatedEntities(), header.getGuid())) { return; } } @@ -211,17 +211,42 @@ public class EntityMutationResponse { mutatedEntities.put(op, opEntities); } - if (!entityHeaderExists(opEntities, header)) { + if (!entityHeaderExists(opEntities, header.getGuid())) { opEntities.add(header); } } - private boolean entityHeaderExists(List<AtlasEntityHeader> entityHeaders, AtlasEntityHeader newEntityHeader) { + @JsonIgnore + public void addEntity(EntityOperation op, AtlasObjectId entity) { + if (mutatedEntities == null) { + mutatedEntities = new HashMap<>(); + } else { + // if an entity is already included in CREATE, ignore subsequent UPDATE, PARTIAL_UPDATE + if (op == EntityOperation.UPDATE || op == EntityOperation.PARTIAL_UPDATE) { + if (entityHeaderExists(getCreatedEntities(), entity.getGuid())) { + return; + } + } + } + + List<AtlasEntityHeader> opEntities = mutatedEntities.get(op); + + if (opEntities == null) { + opEntities = new ArrayList<>(); + mutatedEntities.put(op, opEntities); + } + + if (!entityHeaderExists(opEntities, entity.getGuid())) { + opEntities.add(new AtlasEntityHeader(entity.getTypeName(), entity.getGuid(), entity.getUniqueAttributes())); + } + } + + private boolean entityHeaderExists(List<AtlasEntityHeader> entityHeaders, String guid) { boolean ret = false; - if (CollectionUtils.isNotEmpty(entityHeaders) && newEntityHeader != null) { + if (CollectionUtils.isNotEmpty(entityHeaders) && guid != null) { for (AtlasEntityHeader entityHeader : entityHeaders) { - if (StringUtils.equals(entityHeader.getGuid(), newEntityHeader.getGuid())) { + if (StringUtils.equals(entityHeader.getGuid(), guid)) { ret = true; break; } http://git-wip-us.apache.org/repos/asf/atlas/blob/154dda0e/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java index 4d2b284..d61bff2 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java @@ -68,9 +68,6 @@ import java.util.Objects; import java.util.Set; import java.util.UUID; -import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.BOTH; -import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.IN; -import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.OUT; /** * Utility class for graph operations. @@ -888,24 +885,23 @@ public final class GraphHelper { * Guid and AtlasVertex combo */ public static class VertexInfo { - private String guid; - private AtlasVertex vertex; - private String typeName; + private final AtlasObjectId entity; + private final AtlasVertex vertex; - public VertexInfo(String guid, AtlasVertex vertex, String typeName) { - this.guid = guid; + public VertexInfo(AtlasObjectId entity, AtlasVertex vertex) { + this.entity = entity; this.vertex = vertex; - this.typeName = typeName; } - public String getGuid() { - return guid; - } + public AtlasObjectId getEntity() { return entity; } public AtlasVertex getVertex() { return vertex; } + public String getGuid() { + return entity.getGuid(); + } public String getTypeName() { - return typeName; + return entity.getTypeName(); } @Override @@ -913,14 +909,13 @@ public final class GraphHelper { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; VertexInfo that = (VertexInfo) o; - return Objects.equals(guid, that.guid) && - Objects.equals(vertex, that.vertex) && - Objects.equals(typeName, that.typeName); + return Objects.equals(entity, that.entity) && + Objects.equals(vertex, that.vertex); } @Override public int hashCode() { - return Objects.hash(guid, vertex, typeName); + return Objects.hash(entity, vertex); } } @@ -1304,39 +1299,4 @@ public final class GraphHelper { return StringUtils.isNotEmpty(edge.getLabel()) ? edgeLabel.startsWith("r:") : false; } - - public static AtlasObjectId getReferenceObjectId(AtlasEdge edge, AtlasRelationshipEdgeDirection relationshipDirection, - AtlasVertex parentVertex) { - AtlasObjectId ret = null; - - if (relationshipDirection == OUT) { - ret = getAtlasObjectIdForInVertex(edge); - - } else if (relationshipDirection == IN) { - ret = getAtlasObjectIdForOutVertex(edge); - - } else if (relationshipDirection == BOTH){ - // since relationship direction is BOTH, edge direction can be inward or outward - // compare with parent entity vertex and pick the right reference vertex - if (verticesEquals(parentVertex, edge.getOutVertex())) { - ret = getAtlasObjectIdForInVertex(edge); - } else { - ret = getAtlasObjectIdForOutVertex(edge); - } - } - - return ret; - } - - public static AtlasObjectId getAtlasObjectIdForOutVertex(AtlasEdge edge) { - return new AtlasObjectId(getGuid(edge.getOutVertex()), getTypeName(edge.getOutVertex())); - } - - public static AtlasObjectId getAtlasObjectIdForInVertex(AtlasEdge edge) { - return new AtlasObjectId(getGuid(edge.getInVertex()), getTypeName(edge.getInVertex())); - } - - private static boolean verticesEquals(AtlasVertex vertexA, AtlasVertex vertexB) { - return StringUtils.equals(getGuid(vertexB), getGuid(vertexA)); - } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/atlas/blob/154dda0e/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityChangeNotifier.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityChangeNotifier.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityChangeNotifier.java index 1bda8ff..2b6bead 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityChangeNotifier.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityChangeNotifier.java @@ -158,7 +158,7 @@ public class AtlasEntityChangeNotifier { return; } - List<Referenceable> typedRefInsts = toReferenceables(entityHeaders); + List<Referenceable> typedRefInsts = toReferenceables(entityHeaders, operation); for (EntityChangeListener listener : entityChangeListeners) { try { @@ -180,11 +180,18 @@ public class AtlasEntityChangeNotifier { } } - private List<Referenceable> toReferenceables(List<AtlasEntityHeader> entityHeaders) throws AtlasBaseException { + private List<Referenceable> toReferenceables(List<AtlasEntityHeader> entityHeaders, EntityOperation operation) throws AtlasBaseException { List<Referenceable> ret = new ArrayList<>(entityHeaders.size()); - for (AtlasEntityHeader entityHeader : entityHeaders) { - ret.add(toReferenceable(entityHeader.getGuid())); + // delete notifications don't need all attributes. Hence the special handling for delete operation + if (operation == EntityOperation.DELETE) { + for (AtlasEntityHeader entityHeader : entityHeaders) { + ret.add(new Referenceable(entityHeader.getGuid(), entityHeader.getTypeName(), entityHeader.getAttributes())); + } + } else { + for (AtlasEntityHeader entityHeader : entityHeaders) { + ret.add(toReferenceable(entityHeader.getGuid())); + } } return ret; http://git-wip-us.apache.org/repos/asf/atlas/blob/154dda0e/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1.java index 3a6f733..ca0eeeb 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1.java @@ -647,14 +647,16 @@ public class AtlasEntityStoreV1 implements AtlasEntityStore { private EntityMutationResponse deleteVertices(Collection<AtlasVertex> deletionCandidates) throws AtlasBaseException { EntityMutationResponse response = new EntityMutationResponse(); - deleteHandler.deleteEntities(deletionCandidates); - RequestContextV1 req = RequestContextV1.get(); - for (AtlasObjectId id : req.getDeletedEntityIds()) { - response.addEntity(DELETE, EntityGraphMapper.constructHeader(id)); + RequestContextV1 req = RequestContextV1.get(); + + deleteHandler.deleteEntities(deletionCandidates); // this will update req with list of deleted/updated entities + + for (AtlasObjectId entity : req.getDeletedEntities()) { + response.addEntity(DELETE, entity); } - for (AtlasObjectId id : req.getUpdatedEntityIds()) { - response.addEntity(UPDATE, EntityGraphMapper.constructHeader(id)); + for (AtlasObjectId entity : req.getUpdatedEntities()) { + response.addEntity(UPDATE, entity); } return response; http://git-wip-us.apache.org/repos/asf/atlas/blob/154dda0e/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java index e011c78..2b62a68 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java @@ -43,38 +43,30 @@ import org.apache.atlas.type.AtlasTypeRegistry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Set; -import java.util.Stack; +import java.util.*; import static org.apache.atlas.model.instance.AtlasEntity.Status.DELETED; import static org.apache.atlas.repository.graph.GraphHelper.EDGE_LABEL_PREFIX; -import static org.apache.atlas.repository.graph.GraphHelper.getReferenceObjectId; import static org.apache.atlas.repository.graph.GraphHelper.isRelationshipEdge; import static org.apache.atlas.repository.graph.GraphHelper.string; import static org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1.getIdFromEdge; import static org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1.getState; public abstract class DeleteHandlerV1 { - public static final Logger LOG = LoggerFactory.getLogger(DeleteHandlerV1.class); - private AtlasTypeRegistry typeRegistry; - private boolean shouldUpdateInverseReferences; - private boolean softDelete; + private final AtlasTypeRegistry typeRegistry; + private final EntityGraphRetriever entityRetriever; + private final boolean shouldUpdateInverseReferences; + private final boolean softDelete; protected static final GraphHelper graphHelper = GraphHelper.getInstance(); public DeleteHandlerV1(AtlasTypeRegistry typeRegistry, boolean shouldUpdateInverseReference, boolean softDelete) { - this.typeRegistry = typeRegistry; + this.typeRegistry = typeRegistry; + this.entityRetriever = new EntityGraphRetriever(typeRegistry); this.shouldUpdateInverseReferences = shouldUpdateInverseReference; - this.softDelete = softDelete; + this.softDelete = softDelete; } /** @@ -86,34 +78,25 @@ public abstract class DeleteHandlerV1 { * @throws AtlasException */ public void deleteEntities(Collection<AtlasVertex> instanceVertices) throws AtlasBaseException { - RequestContextV1 requestContext = RequestContextV1.get(); - + RequestContextV1 requestContext = RequestContextV1.get(); Set<AtlasVertex> deletionCandidateVertices = new HashSet<>(); for (AtlasVertex instanceVertex : instanceVertices) { String guid = AtlasGraphUtilsV1.getIdFromVertex(instanceVertex); AtlasEntity.Status state = getState(instanceVertex); - if (state == DELETED) { - LOG.debug("Skipping deletion of {} as it is already deleted", guid); - continue; - } - - String typeName = AtlasGraphUtilsV1.getTypeName(instanceVertex); - AtlasObjectId objId = new AtlasObjectId(guid, typeName); + if (state == DELETED || requestContext.isDeletedEntity(guid)) { + if (LOG.isDebugEnabled()) { + LOG.debug("Skipping deletion of {} as it is already deleted", guid); + } - if (requestContext.getDeletedEntityIds().contains(objId)) { - LOG.debug("Skipping deletion of {} as it is already deleted", guid); continue; } - // Get GUIDs and vertices for all deletion candidates. - Set<GraphHelper.VertexInfo> compositeVertices = getOwnedVertices(instanceVertex); - - // Record all deletion candidate GUIDs in RequestContext + // Record all deletion candidate entities in RequestContext // and gather deletion candidate vertices. - for (GraphHelper.VertexInfo vertexInfo : compositeVertices) { - requestContext.recordEntityDelete(new AtlasObjectId(vertexInfo.getGuid(), vertexInfo.getTypeName())); + for (GraphHelper.VertexInfo vertexInfo : getOwnedVertices(instanceVertex)) { + requestContext.recordEntityDelete(vertexInfo.getEntity()); deletionCandidateVertices.add(vertexInfo.getVertex()); } } @@ -134,7 +117,10 @@ public abstract class DeleteHandlerV1 { public void deleteRelationships(Collection<AtlasEdge> edges) throws AtlasBaseException { for (AtlasEdge edge : edges) { if (getState(edge) == DELETED) { - LOG.debug("Skipping deletion of {} as it is already deleted", getIdFromEdge(edge)); + if (LOG.isDebugEnabled()) { + LOG.debug("Skipping deletion of {} as it is already deleted", getIdFromEdge(edge)); + } + continue; } @@ -150,83 +136,104 @@ public abstract class DeleteHandlerV1 { * @return set of VertexInfo for all composite entities * @throws AtlasException */ - public Set<GraphHelper.VertexInfo> getOwnedVertices(AtlasVertex entityVertex) throws AtlasBaseException { - Set<GraphHelper.VertexInfo> result = new LinkedHashSet<>(); - Stack<AtlasVertex> vertices = new Stack<>(); + public Collection<GraphHelper.VertexInfo> getOwnedVertices(AtlasVertex entityVertex) throws AtlasBaseException { + Map<String, GraphHelper.VertexInfo> vertexInfoMap = new HashMap<>(); + Stack<AtlasVertex> vertices = new Stack<>(); + vertices.push(entityVertex); + while (vertices.size() > 0) { - AtlasVertex vertex = vertices.pop(); + AtlasVertex vertex = vertices.pop(); + AtlasEntity.Status state = getState(vertex); - AtlasEntity.Status state = getState(vertex); if (state == DELETED) { //If the reference vertex is marked for deletion, skip it continue; } - String typeName = GraphHelper.getTypeName(vertex); String guid = GraphHelper.getGuid(vertex); - result.add(new GraphHelper.VertexInfo(guid, vertex, typeName)); + if (vertexInfoMap.containsKey(guid)) { + continue; + } + + AtlasObjectId entity = entityRetriever.toAtlasObjectId(vertex); + String typeName = entity.getTypeName(); AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName); if (entityType == null) { throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, TypeCategory.ENTITY.name(), typeName); } + vertexInfoMap.put(guid, new GraphHelper.VertexInfo(entity, vertex)); + for (AtlasStructType.AtlasAttribute attributeInfo : entityType.getAllAttributes().values()) { if (! attributeInfo.isOwnedRef()) { continue; } - String edgeLabel = AtlasGraphUtilsV1.getAttributeEdgeLabel(entityType, attributeInfo.getName()); - AtlasType attrType = attributeInfo.getAttributeType(); + + String edgeLabel = AtlasGraphUtilsV1.getAttributeEdgeLabel(entityType, attributeInfo.getName()); + AtlasType attrType = attributeInfo.getAttributeType(); + switch (attrType.getTypeCategory()) { - case OBJECT_ID_TYPE: - AtlasEdge edge = graphHelper.getEdgeForLabel(vertex, edgeLabel); - if (edge != null && getState(edge) == AtlasEntity.Status.ACTIVE) { - AtlasVertex compositeVertex = edge.getInVertex(); - vertices.push(compositeVertex); + case OBJECT_ID_TYPE: { + AtlasEdge edge = graphHelper.getEdgeForLabel(vertex, edgeLabel); + + if (edge != null && getState(edge) == AtlasEntity.Status.ACTIVE) { + vertices.push(edge.getInVertex()); + } } break; - case ARRAY: - AtlasArrayType arrType = (AtlasArrayType) attrType; - if (arrType.getElementType().getTypeCategory() != TypeCategory.OBJECT_ID_TYPE) { - continue; - } - Iterator<AtlasEdge> edges = graphHelper.getOutGoingEdgesByLabel(vertex, edgeLabel); - if (edges != null) { - while (edges.hasNext()) { - edge = edges.next(); - if (edge != null && getState(edge) == AtlasEntity.Status.ACTIVE) { - AtlasVertex compositeVertex = edge.getInVertex(); - vertices.push(compositeVertex); + + case ARRAY: { + AtlasArrayType arrType = (AtlasArrayType) attrType; + + if (arrType.getElementType().getTypeCategory() != TypeCategory.OBJECT_ID_TYPE) { + continue; + } + + Iterator<AtlasEdge> edges = graphHelper.getOutGoingEdgesByLabel(vertex, edgeLabel); + + if (edges != null) { + while (edges.hasNext()) { + AtlasEdge edge = edges.next(); + + if (edge != null && getState(edge) == AtlasEntity.Status.ACTIVE) { + vertices.push(edge.getInVertex()); + } } } } break; - case MAP: - AtlasMapType mapType = (AtlasMapType) attrType; - TypeCategory valueTypeCategory = mapType.getValueType().getTypeCategory(); - if (valueTypeCategory != TypeCategory.OBJECT_ID_TYPE) { - continue; - } - String propertyName = AtlasGraphUtilsV1.getQualifiedAttributePropertyKey(entityType, attributeInfo.getName()); - List<String> keys = vertex.getProperty(propertyName, List.class); - if (keys != null) { - for (String key : keys) { - String mapEdgeLabel = GraphHelper.getQualifiedNameForMapKey(edgeLabel, key); - edge = graphHelper.getEdgeForLabel(vertex, mapEdgeLabel); - if (edge != null && getState(edge) == AtlasEntity.Status.ACTIVE) { - AtlasVertex compositeVertex = edge.getInVertex(); - vertices.push(compositeVertex); + + case MAP: { + AtlasMapType mapType = (AtlasMapType) attrType; + TypeCategory valueTypeCategory = mapType.getValueType().getTypeCategory(); + + if (valueTypeCategory != TypeCategory.OBJECT_ID_TYPE) { + continue; + } + + String propertyName = AtlasGraphUtilsV1.getQualifiedAttributePropertyKey(entityType, attributeInfo.getName()); + List<String> keys = vertex.getProperty(propertyName, List.class); + + if (keys != null) { + for (String key : keys) { + String mapEdgeLabel = GraphHelper.getQualifiedNameForMapKey(edgeLabel, key); + AtlasEdge edge = graphHelper.getEdgeForLabel(vertex, mapEdgeLabel); + + if (edge != null && getState(edge) == AtlasEntity.Status.ACTIVE) { + vertices.push(edge.getInVertex()); + } } } } break; - default: } } } - return result; + + return vertexInfoMap.values(); } /** @@ -240,19 +247,19 @@ public abstract class DeleteHandlerV1 { */ public boolean deleteEdgeReference(AtlasEdge edge, TypeCategory typeCategory, boolean isOwned, boolean forceDeleteStructTrait, AtlasVertex vertex) throws AtlasBaseException { - // default edge direction is outward return deleteEdgeReference(edge, typeCategory, isOwned, forceDeleteStructTrait, AtlasRelationshipEdgeDirection.OUT, vertex); } public boolean deleteEdgeReference(AtlasEdge edge, TypeCategory typeCategory, boolean isOwned, boolean forceDeleteStructTrait, AtlasRelationshipEdgeDirection relationshipDirection, AtlasVertex entityVertex) throws AtlasBaseException { - LOG.debug("Deleting {}", string(edge)); - boolean forceDelete = - (typeCategory == TypeCategory.STRUCT || typeCategory == TypeCategory.CLASSIFICATION) && forceDeleteStructTrait; + if (LOG.isDebugEnabled()) { + LOG.debug("Deleting {}", string(edge)); + } - if (typeCategory == TypeCategory.STRUCT || typeCategory == TypeCategory.CLASSIFICATION - || (typeCategory == TypeCategory.OBJECT_ID_TYPE && isOwned)) { + boolean forceDelete = (typeCategory == TypeCategory.STRUCT || typeCategory == TypeCategory.CLASSIFICATION) && forceDeleteStructTrait; + + if (typeCategory == TypeCategory.STRUCT || typeCategory == TypeCategory.CLASSIFICATION || (typeCategory == TypeCategory.OBJECT_ID_TYPE && isOwned)) { //If the vertex is of type struct/trait, delete the edge and then the reference vertex as the vertex is not shared by any other entities. //If the vertex is of type class, and its composite attribute, this reference vertex' lifecycle is controlled //through this delete, hence delete the edge and the reference vertex. @@ -270,8 +277,18 @@ public abstract class DeleteHandlerV1 { if (isRelationshipEdge(edge)) { deleteEdge(edge, false); - AtlasObjectId deletedReferenceObjectId = getReferenceObjectId(edge, relationshipDirection, entityVertex); - RequestContextV1.get().recordEntityUpdate(deletedReferenceObjectId); + AtlasVertex referencedVertex = entityRetriever.getReferencedEntityVertex(edge, relationshipDirection, entityVertex); + + if (referencedVertex != null) { + RequestContextV1 requestContext = RequestContextV1.get(); + + if (!requestContext.isUpdatedEntity(GraphHelper.getGuid(referencedVertex))) { + GraphHelper.setProperty(referencedVertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, requestContext.getRequestTime()); + GraphHelper.setProperty(referencedVertex, Constants.MODIFIED_BY_KEY, requestContext.getUser()); + + requestContext.recordEntityUpdate(entityRetriever.toAtlasObjectId(referencedVertex)); + } + } } else { //legacy case - not a relationship edge //If deleting just the edge, reverse attribute should be updated for any references @@ -279,6 +296,7 @@ public abstract class DeleteHandlerV1 { deleteEdge(edge, true, false); } } + return !softDelete || forceDelete; } @@ -286,13 +304,12 @@ public abstract class DeleteHandlerV1 { //update inverse attribute if (updateInverseAttribute) { AtlasEdgeLabel atlasEdgeLabel = new AtlasEdgeLabel(edge.getLabel()); - - AtlasType parentType = typeRegistry.getType(atlasEdgeLabel.getTypeName()); + AtlasType parentType = typeRegistry.getType(atlasEdgeLabel.getTypeName()); if (parentType instanceof AtlasEntityType) { - AtlasEntityType parentEntityType = (AtlasEntityType) parentType; + AtlasEntityType parentEntityType = (AtlasEntityType) parentType; + AtlasStructType.AtlasAttribute attribute = parentEntityType.getAttribute(atlasEdgeLabel.getAttributeName()); - AtlasStructType.AtlasAttribute attribute = parentEntityType.getAttribute(atlasEdgeLabel.getAttributeName()); if (attribute.getInverseRefAttribute() != null) { deleteEdgeBetweenVertices(edge.getInVertex(), edge.getOutVertex(), attribute.getInverseRefAttribute()); } @@ -305,18 +322,18 @@ public abstract class DeleteHandlerV1 { protected void deleteTypeVertex(AtlasVertex instanceVertex, TypeCategory typeCategory, boolean force) throws AtlasBaseException { switch (typeCategory) { - case STRUCT: - case CLASSIFICATION: - deleteTypeVertex(instanceVertex, force); + case STRUCT: + case CLASSIFICATION: + deleteTypeVertex(instanceVertex, force); break; - case ENTITY: - case OBJECT_ID_TYPE: - deleteEntities(Collections.singletonList(instanceVertex)); + case ENTITY: + case OBJECT_ID_TYPE: + deleteEntities(Collections.singletonList(instanceVertex)); break; - default: - throw new IllegalStateException("Type category " + typeCategory + " not handled"); + default: + throw new IllegalStateException("Type category " + typeCategory + " not handled"); } } @@ -326,10 +343,11 @@ public abstract class DeleteHandlerV1 { * @throws AtlasException */ protected void deleteTypeVertex(AtlasVertex instanceVertex, boolean force) throws AtlasBaseException { - LOG.debug("Deleting {}", string(instanceVertex)); - String typeName = GraphHelper.getTypeName(instanceVertex); - + if (LOG.isDebugEnabled()) { + LOG.debug("Deleting {}", string(instanceVertex)); + } + String typeName = GraphHelper.getTypeName(instanceVertex); AtlasType parentType = typeRegistry.getType(typeName); if (parentType instanceof AtlasStructType) { @@ -337,55 +355,62 @@ public abstract class DeleteHandlerV1 { boolean isEntityType = (parentType instanceof AtlasEntityType); for (AtlasStructType.AtlasAttribute attributeInfo : structType.getAllAttributes().values()) { - LOG.debug("Deleting attribute {} for {}", attributeInfo.getName(), string(instanceVertex)); - boolean isOwned = isEntityType && attributeInfo.isOwnedRef(); - - AtlasType attrType = attributeInfo.getAttributeType(); + if (LOG.isDebugEnabled()) { + LOG.debug("Deleting attribute {} for {}", attributeInfo.getName(), string(instanceVertex)); + } - String edgeLabel = AtlasGraphUtilsV1.getAttributeEdgeLabel(structType, attributeInfo.getName()); + boolean isOwned = isEntityType && attributeInfo.isOwnedRef(); + AtlasType attrType = attributeInfo.getAttributeType(); + String edgeLabel = AtlasGraphUtilsV1.getAttributeEdgeLabel(structType, attributeInfo.getName()); switch (attrType.getTypeCategory()) { - case OBJECT_ID_TYPE: - //If its class attribute, delete the reference - deleteEdgeReference(instanceVertex, edgeLabel, attrType.getTypeCategory(), isOwned); + case OBJECT_ID_TYPE: + //If its class attribute, delete the reference + deleteEdgeReference(instanceVertex, edgeLabel, attrType.getTypeCategory(), isOwned); break; - case STRUCT: - //If its struct attribute, delete the reference - deleteEdgeReference(instanceVertex, edgeLabel, attrType.getTypeCategory(), false); + case STRUCT: + //If its struct attribute, delete the reference + deleteEdgeReference(instanceVertex, edgeLabel, attrType.getTypeCategory(), false); break; - case ARRAY: - //For array attribute, if the element is struct/class, delete all the references - AtlasArrayType arrType = (AtlasArrayType) attrType; - AtlasType elemType = arrType.getElementType(); - if (AtlasGraphUtilsV1.isReference(elemType.getTypeCategory())) { - Iterator<AtlasEdge> edges = graphHelper.getOutGoingEdgesByLabel(instanceVertex, edgeLabel); - if (edges != null) { - while (edges.hasNext()) { - AtlasEdge edge = edges.next(); - deleteEdgeReference(edge, elemType.getTypeCategory(), isOwned, false, instanceVertex); + case ARRAY: + //For array attribute, if the element is struct/class, delete all the references + AtlasArrayType arrType = (AtlasArrayType) attrType; + AtlasType elemType = arrType.getElementType(); + + if (AtlasGraphUtilsV1.isReference(elemType.getTypeCategory())) { + Iterator<AtlasEdge> edges = graphHelper.getOutGoingEdgesByLabel(instanceVertex, edgeLabel); + + if (edges != null) { + while (edges.hasNext()) { + AtlasEdge edge = edges.next(); + + deleteEdgeReference(edge, elemType.getTypeCategory(), isOwned, false, instanceVertex); + } } } - } break; - case MAP: - //For map attribute, if the value type is struct/class, delete all the references - AtlasMapType mapType = (AtlasMapType) attrType; - AtlasType keyType = mapType.getKeyType(); - TypeCategory valueTypeCategory = mapType.getValueType().getTypeCategory(); - String propertyName = AtlasGraphUtilsV1.getQualifiedAttributePropertyKey(structType, attributeInfo.getName()); + case MAP: + //For map attribute, if the value type is struct/class, delete all the references + AtlasMapType mapType = (AtlasMapType) attrType; + AtlasType keyType = mapType.getKeyType(); + TypeCategory valueTypeCategory = mapType.getValueType().getTypeCategory(); + String propertyName = AtlasGraphUtilsV1.getQualifiedAttributePropertyKey(structType, attributeInfo.getName()); - if (AtlasGraphUtilsV1.isReference(valueTypeCategory)) { - List<Object> keys = EntityGraphMapper.getArrayElementsProperty(keyType, instanceVertex, propertyName); - if (keys != null) { - for (Object key : keys) { - String mapEdgeLabel = GraphHelper.getQualifiedNameForMapKey(edgeLabel, (String) key); - deleteEdgeReference(instanceVertex, mapEdgeLabel, valueTypeCategory, isOwned); + if (AtlasGraphUtilsV1.isReference(valueTypeCategory)) { + List<Object> keys = EntityGraphMapper.getArrayElementsProperty(keyType, instanceVertex, propertyName); + + if (keys != null) { + for (Object key : keys) { + String mapEdgeLabel = GraphHelper.getQualifiedNameForMapKey(edgeLabel, (String) key); + + deleteEdgeReference(instanceVertex, mapEdgeLabel, valueTypeCategory, isOwned); + } } } - } + break; } } } @@ -393,9 +418,9 @@ public abstract class DeleteHandlerV1 { deleteVertex(instanceVertex, force); } - public void deleteEdgeReference(AtlasVertex outVertex, String edgeLabel, TypeCategory typeCategory, - boolean isOwned) throws AtlasBaseException { + public void deleteEdgeReference(AtlasVertex outVertex, String edgeLabel, TypeCategory typeCategory, boolean isOwned) throws AtlasBaseException { AtlasEdge edge = graphHelper.getEdgeForLabel(outVertex, edgeLabel); + if (edge != null) { deleteEdgeReference(edge, typeCategory, isOwned, false, outVertex); } @@ -407,20 +432,23 @@ public abstract class DeleteHandlerV1 { * @throws AtlasException */ private void deleteAllTraits(AtlasVertex instanceVertex) throws AtlasBaseException { + String typeName = GraphHelper.getTypeName(instanceVertex); List<String> traitNames = GraphHelper.getTraitNames(instanceVertex); - LOG.debug("Deleting traits {} for {}", traitNames, string(instanceVertex)); - String typeName = GraphHelper.getTypeName(instanceVertex); + + if (LOG.isDebugEnabled()) { + LOG.debug("Deleting traits {} for {}", traitNames, string(instanceVertex)); + } for (String traitNameToBeDeleted : traitNames) { String relationshipLabel = GraphHelper.getTraitLabel(typeName, traitNameToBeDeleted); + deleteEdgeReference(instanceVertex, relationshipLabel, TypeCategory.CLASSIFICATION, false); } } protected AtlasAttribute getAttributeForEdge(String edgeLabel) throws AtlasBaseException { - AtlasEdgeLabel atlasEdgeLabel = new AtlasEdgeLabel(edgeLabel); - - AtlasType parentType = typeRegistry.getType(atlasEdgeLabel.getTypeName()); + AtlasEdgeLabel atlasEdgeLabel = new AtlasEdgeLabel(edgeLabel); + AtlasType parentType = typeRegistry.getType(atlasEdgeLabel.getTypeName()); AtlasStructType parentStructType = (AtlasStructType) parentType; return parentStructType.getAttribute(atlasEdgeLabel.getAttributeName()); @@ -438,148 +466,161 @@ public abstract class DeleteHandlerV1 { * @throws AtlasException */ protected void deleteEdgeBetweenVertices(AtlasVertex outVertex, AtlasVertex inVertex, AtlasAttribute attribute) throws AtlasBaseException { - LOG.debug("Removing edge from {} to {} with attribute name {}", string(outVertex), string(inVertex), - attribute.getName()); - String typeName = GraphHelper.getTypeName(outVertex); - String outId = GraphHelper.getGuid(outVertex); + if (LOG.isDebugEnabled()) { + LOG.debug("Removing edge from {} to {} with attribute name {}", string(outVertex), string(inVertex), attribute.getName()); + } - AtlasObjectId objId = new AtlasObjectId(outId, typeName); - AtlasEntity.Status state = getState(outVertex); + final String typeName = GraphHelper.getTypeName(outVertex); + final String outId = GraphHelper.getGuid(outVertex); + final AtlasEntity.Status state = getState(outVertex); - if (state == DELETED || (outId != null && RequestContextV1.get().isDeletedEntity(objId))) { + if (state == DELETED || (outId != null && RequestContextV1.get().isDeletedEntity(outId))) { //If the reference vertex is marked for deletion, skip updating the reference return; } - AtlasStructType parentType = (AtlasStructType) typeRegistry.getType(typeName); - String propertyName = AtlasGraphUtilsV1.getQualifiedAttributePropertyKey(parentType, attribute.getName()); - String edgeLabel = EDGE_LABEL_PREFIX + propertyName; - AtlasEdge edge = null; - - AtlasAttributeDef attrDef = attribute.getAttributeDef(); - AtlasType attrType = attribute.getAttributeType(); + AtlasStructType parentType = (AtlasStructType) typeRegistry.getType(typeName); + String propertyName = AtlasGraphUtilsV1.getQualifiedAttributePropertyKey(parentType, attribute.getName()); + String edgeLabel = EDGE_LABEL_PREFIX + propertyName; + AtlasEdge edge = null; + AtlasAttributeDef attrDef = attribute.getAttributeDef(); + AtlasType attrType = attribute.getAttributeType(); switch (attrType.getTypeCategory()) { - case OBJECT_ID_TYPE: - //If its class attribute, its the only edge between two vertices - if (attrDef.getIsOptional()) { - edge = graphHelper.getEdgeForLabel(outVertex, edgeLabel); - if (shouldUpdateInverseReferences) { - GraphHelper.setProperty(outVertex, propertyName, null); + case OBJECT_ID_TYPE: { + //If its class attribute, its the only edge between two vertices + if (attrDef.getIsOptional()) { + edge = graphHelper.getEdgeForLabel(outVertex, edgeLabel); + + if (shouldUpdateInverseReferences) { + GraphHelper.setProperty(outVertex, propertyName, null); + } + } else { + // Cannot unset a required attribute. + throw new AtlasBaseException("Cannot unset required attribute " + propertyName + " on " + GraphHelper.vertexString(outVertex) + " edge = " + edgeLabel); } - } else { - // Cannot unset a required attribute. - throw new AtlasBaseException("Cannot unset required attribute " + propertyName + - " on " + GraphHelper.vertexString(outVertex) + " edge = " + edgeLabel); } break; - case ARRAY: - //If its array attribute, find the right edge between the two vertices and update array property - List<String> elements = GraphHelper.getListProperty(outVertex, propertyName); - if (elements != null) { - elements = new ArrayList<>(elements); //Make a copy, else list.remove reflects on titan.getProperty() - for (String elementEdgeId : elements) { - AtlasEdge elementEdge = graphHelper.getEdgeByEdgeId(outVertex, edgeLabel, elementEdgeId); - if (elementEdge == null) { - continue; - } + case ARRAY: { + //If its array attribute, find the right edge between the two vertices and update array property + List<String> elements = GraphHelper.getListProperty(outVertex, propertyName); - AtlasVertex elementVertex = elementEdge.getInVertex(); - if (elementVertex.equals(inVertex)) { - edge = elementEdge; - - //TODO element.size includes deleted items as well. should exclude - if (!attrDef.getIsOptional() - && elements.size() <= attrDef.getValuesMinCount()) { - // Deleting this edge would violate the attribute's lower bound. - throw new AtlasBaseException( - "Cannot remove array element from required attribute " + - propertyName + " on " - + GraphHelper.getVertexDetails(outVertex) + " " + GraphHelper.getEdgeDetails(elementEdge)); + if (elements != null) { + elements = new ArrayList<>(elements); //Make a copy, else list.remove reflects on titan.getProperty() + + for (String elementEdgeId : elements) { + AtlasEdge elementEdge = graphHelper.getEdgeByEdgeId(outVertex, edgeLabel, elementEdgeId); + + if (elementEdge == null) { + continue; } - if (shouldUpdateInverseReferences) { - //if composite attribute, remove the reference as well. else, just remove the edge - //for example, when table is deleted, process still references the table - //but when column is deleted, table will not reference the deleted column - LOG.debug("Removing edge {} from the array attribute {}", string(elementEdge), - attribute.getName()); - // Remove all occurrences of the edge ID from the list. - // This prevents dangling edge IDs (i.e. edge IDs for deleted edges) - // from the remaining in the list if there are duplicates. - elements.removeAll(Collections.singletonList(elementEdge.getId().toString())); - GraphHelper.setProperty(outVertex, propertyName, elements); - break; + AtlasVertex elementVertex = elementEdge.getInVertex(); + + if (elementVertex.equals(inVertex)) { + edge = elementEdge; + //TODO element.size includes deleted items as well. should exclude + if (!attrDef.getIsOptional() && elements.size() <= attrDef.getValuesMinCount()) { + // Deleting this edge would violate the attribute's lower bound. + throw new AtlasBaseException("Cannot remove array element from required attribute " + propertyName + " on " + GraphHelper.getVertexDetails(outVertex) + " " + GraphHelper.getEdgeDetails(elementEdge)); + } + + if (shouldUpdateInverseReferences) { + //if composite attribute, remove the reference as well. else, just remove the edge + //for example, when table is deleted, process still references the table + //but when column is deleted, table will not reference the deleted column + if (LOG.isDebugEnabled()) { + LOG.debug("Removing edge {} from the array attribute {}", string(elementEdge), attribute.getName()); + } + + // Remove all occurrences of the edge ID from the list. + // This prevents dangling edge IDs (i.e. edge IDs for deleted edges) + // from the remaining in the list if there are duplicates. + elements.removeAll(Collections.singletonList(elementEdge.getId().toString())); + GraphHelper.setProperty(outVertex, propertyName, elements); + break; + } } } } } break; - case MAP: - //If its map attribute, find the right edge between two vertices and update map property - List<String> keys = GraphHelper.getListProperty(outVertex, propertyName); - if (keys != null) { - keys = new ArrayList<>(keys); //Make a copy, else list.remove reflects on titan.getProperty() - for (String key : keys) { - String keyPropertyName = GraphHelper.getQualifiedNameForMapKey(propertyName, key); - String mapEdgeId = GraphHelper.getSingleValuedProperty(outVertex, keyPropertyName, String.class); - AtlasEdge mapEdge = graphHelper.getEdgeByEdgeId(outVertex, keyPropertyName, mapEdgeId); - if(mapEdge != null) { - AtlasVertex mapVertex = mapEdge.getInVertex(); - if (mapVertex.getId().toString().equals(inVertex.getId().toString())) { - //TODO keys.size includes deleted items as well. should exclude - if (attrDef.getIsOptional() || keys.size() > attrDef.getValuesMinCount()) { - edge = mapEdge; - } else { - // Deleting this entry would violate the attribute's lower bound. - throw new AtlasBaseException( - "Cannot remove map entry " + keyPropertyName + " from required attribute " + - propertyName + " on " + GraphHelper.getVertexDetails(outVertex) + " " + GraphHelper.getEdgeDetails(mapEdge)); - } - - if (shouldUpdateInverseReferences) { - //remove this key - LOG.debug("Removing edge {}, key {} from the map attribute {}", string(mapEdge), key, - attribute.getName()); - keys.remove(key); - GraphHelper.setProperty(outVertex, propertyName, keys); - GraphHelper.setProperty(outVertex, keyPropertyName, null); + case MAP: { + //If its map attribute, find the right edge between two vertices and update map property + List<String> keys = GraphHelper.getListProperty(outVertex, propertyName); + + if (keys != null) { + keys = new ArrayList<>(keys); //Make a copy, else list.remove reflects on titan.getProperty() + + for (String key : keys) { + String keyPropertyName = GraphHelper.getQualifiedNameForMapKey(propertyName, key); + String mapEdgeId = GraphHelper.getSingleValuedProperty(outVertex, keyPropertyName, String.class); + AtlasEdge mapEdge = graphHelper.getEdgeByEdgeId(outVertex, keyPropertyName, mapEdgeId); + + if (mapEdge != null) { + AtlasVertex mapVertex = mapEdge.getInVertex(); + + if (mapVertex.getId().toString().equals(inVertex.getId().toString())) { + //TODO keys.size includes deleted items as well. should exclude + if (attrDef.getIsOptional() || keys.size() > attrDef.getValuesMinCount()) { + edge = mapEdge; + } else { + // Deleting this entry would violate the attribute's lower bound. + throw new AtlasBaseException("Cannot remove map entry " + keyPropertyName + " from required attribute " + propertyName + " on " + GraphHelper.getVertexDetails(outVertex) + " " + GraphHelper.getEdgeDetails(mapEdge)); + } + + if (shouldUpdateInverseReferences) { + //remove this key + if (LOG.isDebugEnabled()) { + LOG.debug("Removing edge {}, key {} from the map attribute {}", string(mapEdge), key, attribute.getName()); + } + + keys.remove(key); + GraphHelper.setProperty(outVertex, propertyName, keys); + GraphHelper.setProperty(outVertex, keyPropertyName, null); + } + break; } - break; } } } } break; - case STRUCT: - case CLASSIFICATION: + case STRUCT: + case CLASSIFICATION: break; - default: - throw new IllegalStateException("There can't be an edge from " + GraphHelper.getVertexDetails(outVertex) + " to " - + GraphHelper.getVertexDetails(inVertex) + " with attribute name " + attribute.getName() + " which is not class/array/map attribute. found " + attrType.getTypeCategory().name()); + default: + throw new IllegalStateException("There can't be an edge from " + GraphHelper.getVertexDetails(outVertex) + " to " + GraphHelper.getVertexDetails(inVertex) + " with attribute name " + attribute.getName() + " which is not class/array/map attribute. found " + attrType.getTypeCategory().name()); } if (edge != null) { deleteEdge(edge, false); + RequestContextV1 requestContext = RequestContextV1.get(); - GraphHelper.setProperty(outVertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, - requestContext.getRequestTime()); - GraphHelper.setProperty(outVertex, Constants.MODIFIED_BY_KEY, requestContext.getUser()); - requestContext.recordEntityUpdate(new AtlasObjectId(outId, typeName)); + + if (! requestContext.isUpdatedEntity(outId)) { + GraphHelper.setProperty(outVertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, requestContext.getRequestTime()); + GraphHelper.setProperty(outVertex, Constants.MODIFIED_BY_KEY, requestContext.getUser()); + + requestContext.recordEntityUpdate(entityRetriever.toAtlasObjectId(outVertex)); + } } } protected void deleteVertex(AtlasVertex instanceVertex, boolean force) throws AtlasBaseException { //Update external references(incoming edges) to this vertex - LOG.debug("Setting the external references to {} to null(removing edges)", string(instanceVertex)); + if (LOG.isDebugEnabled()) { + LOG.debug("Setting the external references to {} to null(removing edges)", string(instanceVertex)); + } for (AtlasEdge edge : (Iterable<AtlasEdge>) instanceVertex.getEdges(AtlasEdgeDirection.IN)) { AtlasEntity.Status edgeState = getState(edge); + if (edgeState == AtlasEntity.Status.ACTIVE) { //Delete only the active edge references AtlasAttribute attribute = getAttributeForEdge(edge.getLabel()); @@ -587,6 +628,7 @@ public abstract class DeleteHandlerV1 { deleteEdgeBetweenVertices(edge.getOutVertex(), edge.getInVertex(), attribute); } } + _deleteVertex(instanceVertex, force); } } http://git-wip-us.apache.org/repos/asf/atlas/blob/154dda0e/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java index 0fd4355..779bc38 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java @@ -85,6 +85,7 @@ public class EntityGraphMapper { private final AtlasGraph graph; private final DeleteHandlerV1 deleteHandler; private final AtlasTypeRegistry typeRegistry; + private final EntityGraphRetriever entityRetriever; private final AtlasRelationshipStore relationshipStore; @Inject @@ -92,6 +93,7 @@ public class EntityGraphMapper { AtlasRelationshipStore relationshipStore) { this.deleteHandler = deleteHandler; this.typeRegistry = typeRegistry; + this.entityRetriever = new EntityGraphRetriever(typeRegistry); this.graph = atlasGraph; this.relationshipStore = relationshipStore; } @@ -192,16 +194,16 @@ public class EntityGraphMapper { RequestContextV1 req = RequestContextV1.get(); - for (AtlasObjectId id : req.getDeletedEntityIds()) { - resp.addEntity(DELETE, constructHeader(id)); + for (AtlasObjectId entity : req.getDeletedEntities()) { + resp.addEntity(DELETE, entity); } - for (AtlasObjectId id : req.getUpdatedEntityIds()) { + for (AtlasObjectId entity : req.getUpdatedEntities()) { if (isPartialUpdate) { - resp.addEntity(PARTIAL_UPDATE, constructHeader(id)); + resp.addEntity(PARTIAL_UPDATE, entity); } else { - resp.addEntity(UPDATE, constructHeader(id)); + resp.addEntity(UPDATE, entity); } } @@ -423,7 +425,6 @@ public class EntityGraphMapper { AtlasVertex attrVertex = context.getDiscoveryContext().getResolvedEntityVertex(getGuid(ctx.getValue())); recordEntityUpdate(attrVertex); - updateModificationMetadata(attrVertex); } //delete old reference @@ -494,9 +495,13 @@ public class EntityGraphMapper { } if (inverseUpdated) { - updateModificationMetadata(inverseVertex); - AtlasObjectId inverseEntityId = new AtlasObjectId(getIdFromVertex(inverseVertex), inverseType.getTypeName()); - RequestContextV1.get().recordEntityUpdate(inverseEntityId); + RequestContextV1 requestContext = RequestContextV1.get(); + + if (!requestContext.isDeletedEntity(GraphHelper.getGuid(inverseVertex))) { + updateModificationMetadata(inverseVertex); + + requestContext.recordEntityUpdate(entityRetriever.toAtlasObjectId(inverseVertex)); + } } } @@ -1441,31 +1446,14 @@ public class EntityGraphMapper { return ret; } - private void recordEntityUpdate(AtlasVertex vertex) { - AtlasObjectId objectId = new AtlasObjectId(GraphHelper.getGuid(vertex), GraphHelper.getTypeName(vertex)); - RequestContextV1 req = RequestContextV1.get(); - - if (!objectIdsContain(req.getUpdatedEntityIds(), objectId) && !objectIdsContain(req.getCreatedEntityIds(), objectId)) { - req.recordEntityUpdate(objectId); - } - } - - private boolean objectIdsContain(Collection<AtlasObjectId> objectIds, AtlasObjectId objectId) { - boolean ret = false; + private void recordEntityUpdate(AtlasVertex vertex) throws AtlasBaseException { + RequestContextV1 req = RequestContextV1.get(); - if (CollectionUtils.isEmpty(objectIds)) { - ret = false; + if (!req.isUpdatedEntity(GraphHelper.getGuid(vertex))) { + updateModificationMetadata(vertex); - } else { - for (AtlasObjectId id : objectIds) { - if (StringUtils.equals(id.getGuid(), objectId.getGuid())) { - ret = true; - break; - } - } + req.recordEntityUpdate(entityRetriever.toAtlasObjectId(vertex)); } - - return ret; } private static void compactAttributes(AtlasEntity entity) { http://git-wip-us.apache.org/repos/asf/atlas/blob/154dda0e/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java index e9051a6..b05a9a3 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java @@ -147,6 +147,70 @@ public final class EntityGraphRetriever { return atlasVertex != null ? mapVertexToAtlasEntityHeader(atlasVertex, attributes) : null; } + public AtlasEntityHeader toAtlasEntityHeader(AtlasEntity entity) { + AtlasEntityHeader ret = null; + String typeName = entity.getTypeName(); + AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName); + + if (entityType != null) { + Map<String, Object> uniqueAttributes = new HashMap<>(); + + for (AtlasAttribute attribute : entityType.getUniqAttributes().values()) { + Object attrValue = entity.getAttribute(attribute.getName()); + + if (attrValue != null) { + uniqueAttributes.put(attribute.getName(), attrValue); + } + } + + ret = new AtlasEntityHeader(entity.getTypeName(), entity.getGuid(), uniqueAttributes); + } + + return ret; + } + + public AtlasObjectId toAtlasObjectId(AtlasVertex entityVertex) throws AtlasBaseException { + AtlasObjectId ret = null; + String typeName = entityVertex.getProperty(Constants.TYPE_NAME_PROPERTY_KEY, String.class); + AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName); + + if (entityType != null) { + Map<String, Object> uniqueAttributes = new HashMap<>(); + + for (AtlasAttribute attribute : entityType.getUniqAttributes().values()) { + Object attrValue = getVertexAttribute(entityVertex, attribute); + + if (attrValue != null) { + uniqueAttributes.put(attribute.getName(), attrValue); + } + } + + ret = new AtlasObjectId(entityVertex.getProperty(Constants.GUID_PROPERTY_KEY, String.class), typeName, uniqueAttributes); + } + + return ret; + } + + public AtlasVertex getReferencedEntityVertex(AtlasEdge edge, AtlasRelationshipEdgeDirection relationshipDirection, AtlasVertex parentVertex) throws AtlasBaseException { + AtlasVertex entityVertex = null; + + if (relationshipDirection == OUT) { + entityVertex = edge.getInVertex(); + } else if (relationshipDirection == IN) { + entityVertex = edge.getOutVertex(); + } else if (relationshipDirection == BOTH){ + // since relationship direction is BOTH, edge direction can be inward or outward + // compare with parent entity vertex and pick the right reference vertex + if (StringUtils.equals(GraphHelper.getGuid(parentVertex), GraphHelper.getGuid(edge.getOutVertex()))) { + entityVertex = edge.getInVertex(); + } else { + entityVertex = edge.getOutVertex(); + } + } + + return entityVertex; + } + public AtlasVertex getEntityVertex(String guid) throws AtlasBaseException { AtlasVertex ret = AtlasGraphUtilsV1.findByGuid(guid); http://git-wip-us.apache.org/repos/asf/atlas/blob/154dda0e/server-api/src/main/java/org/apache/atlas/RequestContextV1.java ---------------------------------------------------------------------- diff --git a/server-api/src/main/java/org/apache/atlas/RequestContextV1.java b/server-api/src/main/java/org/apache/atlas/RequestContextV1.java index 27fe3d2..9177cb8 100644 --- a/server-api/src/main/java/org/apache/atlas/RequestContextV1.java +++ b/server-api/src/main/java/org/apache/atlas/RequestContextV1.java @@ -31,18 +31,15 @@ public class RequestContextV1 { private static final ThreadLocal<RequestContextV1> CURRENT_CONTEXT = new ThreadLocal<>(); - private Set<AtlasObjectId> createdEntityIds = new LinkedHashSet<>(); - private Set<AtlasObjectId> updatedEntityIds = new LinkedHashSet<>(); - private Set<AtlasObjectId> deletedEntityIds = new LinkedHashSet<>(); - private Map<String, AtlasEntityWithExtInfo> entityCacheV2 = new HashMap<>(); + private final Map<String, AtlasObjectId> updatedEntities = new HashMap<>(); + private final Map<String, AtlasObjectId> deletedEntities = new HashMap<>(); + private final Map<String, AtlasEntityWithExtInfo> entityCacheV2 = new HashMap<>(); + private final Metrics metrics = new Metrics(); + private final long requestTime = System.currentTimeMillis(); private String user; - private final long requestTime; - - private Metrics metrics = new Metrics(); private RequestContextV1() { - requestTime = System.currentTimeMillis(); } //To handle gets from background threads where createContext() is not called @@ -62,9 +59,9 @@ public class RequestContextV1 { RequestContextV1 instance = CURRENT_CONTEXT.get(); if (instance != null) { - if (instance.entityCacheV2 != null) { - instance.entityCacheV2.clear(); - } + instance.updatedEntities.clear(); + instance.deletedEntities.clear(); + instance.entityCacheV2.clear(); } CURRENT_CONTEXT.remove(); @@ -78,24 +75,16 @@ public class RequestContextV1 { this.user = user; } - public void recordEntityCreate(Collection<AtlasObjectId> createdEntityIds) { - this.createdEntityIds.addAll(createdEntityIds); - } - - public void recordEntityCreate(AtlasObjectId createdEntityId) { - this.createdEntityIds.add(createdEntityId); - } - - public void recordEntityUpdate(Collection<AtlasObjectId> updatedEntityIds) { - this.updatedEntityIds.addAll(updatedEntityIds); - } - - public void recordEntityUpdate(AtlasObjectId entityId) { - this.updatedEntityIds.add(entityId); + public void recordEntityUpdate(AtlasObjectId entity) { + if (entity != null && entity.getGuid() != null) { + updatedEntities.put(entity.getGuid(), entity); + } } - public void recordEntityDelete(AtlasObjectId entityId) { - deletedEntityIds.add(entityId); + public void recordEntityDelete(AtlasObjectId entity) { + if (entity != null && entity.getGuid() != null) { + deletedEntities.put(entity.getGuid(), entity); + } } /** @@ -108,16 +97,12 @@ public class RequestContextV1 { } } - public Collection<AtlasObjectId> getCreatedEntityIds() { - return createdEntityIds; - } - - public Collection<AtlasObjectId> getUpdatedEntityIds() { - return updatedEntityIds; + public Collection<AtlasObjectId> getUpdatedEntities() { + return updatedEntities.values(); } - public Collection<AtlasObjectId> getDeletedEntityIds() { - return deletedEntityIds; + public Collection<AtlasObjectId> getDeletedEntities() { + return deletedEntities.values(); } /** @@ -135,8 +120,12 @@ public class RequestContextV1 { return requestTime; } - public boolean isDeletedEntity(AtlasObjectId entityId) { - return deletedEntityIds.contains(entityId); + public boolean isUpdatedEntity(String guid) { + return updatedEntities.containsKey(guid); + } + + public boolean isDeletedEntity(String guid) { + return deletedEntities.containsKey(guid); } public static Metrics getMetrics() {
