Repository: incubator-atlas Updated Branches: refs/heads/master a77d1ab53 -> 7ebb20136
ATLAS-463 Disconnect inverse references ( dkantor via sumasai) Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/7ebb2013 Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/7ebb2013 Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/7ebb2013 Branch: refs/heads/master Commit: 7ebb20136cefb6efe325ee1f30455abd9a3d57da Parents: a77d1ab Author: Suma Shivaprasad <[email protected]> Authored: Mon Mar 7 15:21:20 2016 -0800 Committer: Suma Shivaprasad <[email protected]> Committed: Mon Mar 7 15:21:20 2016 -0800 ---------------------------------------------------------------------- release-log.txt | 1 + .../atlas/repository/graph/AtlasEdgeLabel.java | 97 ++++ .../graph/TypedInstanceToGraphMapper.java | 285 ++++++++++-- .../test/java/org/apache/atlas/TestUtils.java | 2 +- ...kedMetadataRepositoryDeleteEntitiesTest.java | 443 ++++++++++++++++++- .../GraphBackedMetadataRepositoryTest.java | 25 +- .../service/DefaultMetadataServiceTest.java | 5 +- 7 files changed, 811 insertions(+), 47 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/7ebb2013/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index d0a5fd5..94e0780 100644 --- a/release-log.txt +++ b/release-log.txt @@ -10,6 +10,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags) ALL CHANGES: +ATLAS-463 Disconnect inverse references ( dkantor via sumasai) ATLAS-479 Add description for different types during create time (guptaneeru via shwethags) ATLAS-508 Apache nightly build failure - UnsupportedOperationException: Not a single key: __traitNames (shwethags) ATLAS-422 JavaDoc NotificationConsumer and NotificationInterface.(tbeerbower via sumasai) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/7ebb2013/repository/src/main/java/org/apache/atlas/repository/graph/AtlasEdgeLabel.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/AtlasEdgeLabel.java b/repository/src/main/java/org/apache/atlas/repository/graph/AtlasEdgeLabel.java new file mode 100644 index 0000000..da2ad9a --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/graph/AtlasEdgeLabel.java @@ -0,0 +1,97 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.graph; + +/** + * Represents an edge label used in Atlas. + * The format of an Atlas edge label is EDGE_LABEL_PREFIX<<typeName>>.<<attributeName>>[.mapKey] + * + */ +public class AtlasEdgeLabel { + private final String typeName_; + private final String attributeName_; + private final String mapKey_; + private final String edgeLabel_; + private final String qualifiedMapKey_; + private final String qualifiedAttributeName_; + + public AtlasEdgeLabel(String edgeLabel) { + if (!edgeLabel.startsWith(GraphHelper.EDGE_LABEL_PREFIX)) { + throw new IllegalArgumentException("Invalid edge label " + edgeLabel + ": missing required prefix " + GraphHelper.EDGE_LABEL_PREFIX); + } + String labelWithoutPrefix = edgeLabel.substring(GraphHelper.EDGE_LABEL_PREFIX.length()); + String[] fields = labelWithoutPrefix.split("\\.", 3); + if (fields.length < 2 || fields.length > 3) { + throw new IllegalArgumentException("Invalid edge label " + edgeLabel + + ": expected 2 or 3 label components but found " + fields.length); + } + typeName_ = fields[0]; + attributeName_ = fields[1]; + if (fields.length == 3) { + mapKey_ = fields[2]; + qualifiedMapKey_ = labelWithoutPrefix; + qualifiedAttributeName_ = typeName_ + '.' + attributeName_; + } + else { + mapKey_ = null; + qualifiedMapKey_ = null; + qualifiedAttributeName_ = labelWithoutPrefix; + } + edgeLabel_ = edgeLabel; + } + + public String getTypeName() { + return typeName_; + } + + public String getAttributeName() { + return attributeName_; + } + + public String getMapKey() { + return mapKey_; + } + + public String getEdgeLabel() { + return edgeLabel_; + } + + public String getQualifiedMapKey() { + return qualifiedMapKey_; + } + + + public String getQualifiedAttributeName() { + + return qualifiedAttributeName_; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append('(').append("typeName: ").append(typeName_); + sb.append(", attributeName: ").append(attributeName_); + if (mapKey_ != null) { + sb.append(", mapKey: ").append(mapKey_); + sb.append(", qualifiedMapKey: ").append(qualifiedMapKey_); + } + sb.append(", edgeLabel: ").append(edgeLabel_).append(')'); + return sb.toString(); + } + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/7ebb2013/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java index 7b2890c..ed403e7 100644 --- a/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java @@ -21,6 +21,7 @@ import com.thinkaurelius.titan.core.SchemaViolationException; import com.tinkerpop.blueprints.Direction; import com.tinkerpop.blueprints.Edge; import com.tinkerpop.blueprints.Vertex; + import org.apache.atlas.AtlasException; import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.RepositoryException; @@ -35,13 +36,17 @@ import org.apache.atlas.typesystem.persistence.ReferenceableInstance; import org.apache.atlas.typesystem.types.AttributeInfo; import org.apache.atlas.typesystem.types.ClassType; import org.apache.atlas.typesystem.types.DataTypes; +import org.apache.atlas.typesystem.types.DataTypes.TypeCategory; import org.apache.atlas.typesystem.types.EnumValue; +import org.apache.atlas.typesystem.types.IConstructableType; import org.apache.atlas.typesystem.types.IDataType; import org.apache.atlas.typesystem.types.Multiplicity; import org.apache.atlas.typesystem.types.ObjectGraphWalker; +import org.apache.atlas.typesystem.types.StructType; import org.apache.atlas.typesystem.types.TraitType; import org.apache.atlas.typesystem.types.TypeSystem; import org.apache.atlas.typesystem.types.TypeUtils; +import org.apache.atlas.typesystem.types.TypeUtils.Pair; import org.apache.atlas.utils.MD5Utils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -64,7 +69,6 @@ public final class TypedInstanceToGraphMapper { private final List<String> deletedEntityGuids = new ArrayList<>(); private final List<ITypedReferenceableInstance> deletedEntities = new ArrayList<>(); private final GraphToTypedInstanceMapper graphToTypedInstanceMapper; - private static final GraphHelper graphHelper = GraphHelper.getInstance(); private final String SIGNATURE_HASH_PROPERTY_KEY = Constants.INTERNAL_PROPERTY_KEY_PREFIX + "signature"; @@ -174,6 +178,7 @@ public final class TypedInstanceToGraphMapper { void mapInstanceToVertex(ITypedInstance typedInstance, Vertex instanceVertex, Map<String, AttributeInfo> fields, boolean mapOnlyUniqueAttributes, Operation operation) throws AtlasException { + LOG.debug("Mapping instance {} of {} to vertex {}", typedInstance, typedInstance.getTypeName(), instanceVertex); for (AttributeInfo attributeInfo : fields.values()) { @@ -184,12 +189,27 @@ public final class TypedInstanceToGraphMapper { } if (operation == Operation.DELETE) { + // Remove uni-directional references to the deletion candidate. + removeUnidirectionalReferences(instanceVertex); + // Remove vertex for deletion candidate. graphHelper.removeVertex(instanceVertex); } } + private String getInstanceName(Vertex referencingVertex, IConstructableType referencingType) { + + if (referencingType.getTypeCategory() == TypeCategory.CLASS) { + Id idFromVertex = GraphHelper.getIdFromVertex(referencingType.getName(), referencingVertex); + String instanceId = referencingType.getName() + ":" + idFromVertex._getId(); + return instanceId; + } + else { + return referencingType.getName(); + } + } + void mapAttributesToVertex(ITypedInstance typedInstance, Vertex instanceVertex, AttributeInfo attributeInfo, Operation operation) throws AtlasException { Object attrValue = typedInstance.get(attributeInfo.name); @@ -361,7 +381,6 @@ public final class TypedInstanceToGraphMapper { IDataType elementType = ((DataTypes.ArrayType) attributeInfo.dataType()).getElemType(); List<String> newEntries = new ArrayList<>(); - if (newElements != null && !newElements.isEmpty()) { int index = 0; for (; index < newElements.size(); index++) { @@ -404,7 +423,7 @@ public final class TypedInstanceToGraphMapper { @SuppressWarnings("unchecked") Map<Object, Object> collection = (Map<Object, Object>) typedInstance.get(attributeInfo.name); boolean empty = (collection == null || collection.isEmpty()); - if (!empty || operation == Operation.UPDATE_FULL) { + if (!empty || operation == Operation.UPDATE_FULL || operation == Operation.DELETE) { String propertyName = GraphHelper.getQualifiedFieldName(typedInstance, attributeInfo); IDataType elementType = ((DataTypes.MapType) attributeInfo.dataType()).getValueType(); @@ -420,23 +439,22 @@ public final class TypedInstanceToGraphMapper { //Add/Update/Remove property value GraphHelper.setProperty(instanceVertex, myPropertyName, newEntry); } - - //Remove unused key references - List<Object> origKeys = instanceVertex.getProperty(propertyName); - if (origKeys != null) { - if (collection != null) { - origKeys.removeAll(collection.keySet()); - } - for (Object unusedKey : origKeys) { - String edgeLabel = GraphHelper.getEdgeLabel(typedInstance, attributeInfo) + "." + unusedKey; - if (instanceVertex.getEdges(Direction.OUT, edgeLabel).iterator().hasNext()) { - Edge edge = instanceVertex.getEdges(Direction.OUT, edgeLabel).iterator().next(); - removeUnusedReference(edge.getId().toString(), attributeInfo, - ((DataTypes.MapType) attributeInfo.dataType()).getValueType()); - } + } + + //Remove unused key references + List<Object> origKeys = instanceVertex.getProperty(propertyName); + if (origKeys != null) { + if (collection != null) { + origKeys.removeAll(collection.keySet()); + } + for (Object unusedKey : origKeys) { + String edgeLabel = GraphHelper.getEdgeLabel(typedInstance, attributeInfo) + "." + unusedKey; + if (instanceVertex.getEdges(Direction.OUT, edgeLabel).iterator().hasNext()) { + Edge edge = instanceVertex.getEdges(Direction.OUT, edgeLabel).iterator().next(); + removeUnusedReference(edge.getId().toString(), attributeInfo, + ((DataTypes.MapType) attributeInfo.dataType()).getValueType()); } } - } // for dereference on way out @@ -653,42 +671,241 @@ public final class TypedInstanceToGraphMapper { } private Edge removeUnusedReference(String edgeId, AttributeInfo attributeInfo, IDataType<?> elementType) throws AtlasException { - //Remove edges for property values which do not exist any more + TypeCategory typeCategory = elementType.getTypeCategory(); + if (typeCategory != TypeCategory.STRUCT && elementType.getTypeCategory() != TypeCategory.CLASS) { + // Only class and struct references have edges. + return null; + } + + // Remove edge to disconnect struct or class reference. + // For struct or composite class reference, also delete the target instance. Edge removedRelation = null; - switch (elementType.getTypeCategory()) { - case STRUCT: - removedRelation = graphHelper.removeRelation(edgeId, true); - //Remove the vertex from state so that further processing no longer uses this - break; - case CLASS: - // TODO: disconnect inverse reference if attributeInfo.reverseAttributeName is non-null. + TypeUtils.Pair<Edge, Vertex> edgeAndVertex = graphHelper.getEdgeAndTargetVertex(edgeId); + if (typeCategory == TypeCategory.STRUCT) { + graphHelper.removeEdge(edgeAndVertex.left); + removedRelation = edgeAndVertex.left; + + // Create an empty instance to use for clearing all struct attributes. + StructType structType = (StructType) elementType; + ITypedStruct typedInstance = structType.createInstance(); + + // Delete target vertex and any underlying structs and composite entities owned by this struct. + mapInstanceToVertex(typedInstance, edgeAndVertex.right, structType.fieldMapping().fields, false, Operation.DELETE); + } + else { + // Class reference if (attributeInfo.isComposite) { - // Delete contained entity. - TypeUtils.Pair<Edge, Vertex> edgeAndVertex = graphHelper.getEdgeAndTargetVertex(edgeId); + // For uni-directional reference, remove the edge. + // For bi-directional reference, the edges are removed + // when the composite entity is deleted. + if (attributeInfo.reverseAttributeName == null) { + graphHelper.removeEdge(edgeAndVertex.left); + removedRelation = edgeAndVertex.left; + } + // Delete the contained entity. + if (LOG.isDebugEnabled()) { + Vertex sourceVertex = edgeAndVertex.left.getVertex(Direction.OUT); + String sourceTypeName = GraphHelper.getTypeName(sourceVertex); + LOG.debug("Deleting composite entity {}:{} contained by {}:{} through reference {}", + elementType.getName(), GraphHelper.getIdFromVertex(elementType.getName(), edgeAndVertex.right)._getId(), + sourceTypeName, GraphHelper.getIdFromVertex(sourceTypeName, sourceVertex)._getId(), + attributeInfo.name); + } deleteEntity(elementType.getName(), edgeAndVertex.right); - graphHelper.removeEdge(edgeAndVertex.left); - removedRelation = edgeAndVertex.left; } else { - removedRelation = graphHelper.removeRelation(edgeId, false); + if (attributeInfo.reverseAttributeName != null) { + // Disconnect both ends of the bi-directional reference + removeReverseReference(edgeAndVertex, attributeInfo); + } + graphHelper.removeEdge(edgeAndVertex.left); + removedRelation = edgeAndVertex.left; } - break; } return removedRelation; } + + /** + * Remove the reverse reference value for the specified edge and vertex. + * + * @param edgeAndVertex + * @param attributeInfo + * @throws AtlasException + */ + private void removeReverseReference(TypeUtils.Pair<Edge, Vertex> edgeAndVertex, + AttributeInfo attributeInfo) throws AtlasException { + + Vertex sourceVertex = edgeAndVertex.left.getVertex(Direction.OUT); + String inverseTypeName = GraphHelper.getTypeName(edgeAndVertex.right); + IConstructableType inverseType = typeSystem.getDataType(IConstructableType.class, inverseTypeName); + AttributeInfo inverseAttributeInfo = inverseType.fieldMapping().fields.get(attributeInfo.reverseAttributeName); + String inverseEdgeLabel = GraphHelper.getEdgeLabel(inverseType, inverseAttributeInfo); + TypeCategory inverseTypeCategory = inverseAttributeInfo.dataType().getTypeCategory(); + // Find and remove the edge which represents the inverse reference value. + Iterable<Edge> inverseEdges = GraphHelper.getOutGoingEdgesByLabel(edgeAndVertex.right, inverseEdgeLabel); + Edge removedEdge = null; + // Search for the edge which references the source vertex. + for (Edge edge : inverseEdges) { + Vertex vertex = edge.getVertex(Direction.IN); + if (vertex.equals(sourceVertex)) { + // Found the edge which points back at source vertex. + // Disconnect the reference by removing the edge and + // removing the edge ID from the vertex property. + removeReferenceValue(edge, new AtlasEdgeLabel(edge.getLabel()), edgeAndVertex.right, inverseType, inverseTypeCategory); + removedEdge = edge; + break; + } + } + if (removedEdge != null) { + if (LOG.isDebugEnabled()) { + String sourceTypeName = GraphHelper.getTypeName(sourceVertex); + LOG.debug("Removed edge {} for reverse reference {} from {}:{} to {}:{} ", removedEdge, + GraphHelper.getQualifiedFieldName(inverseType, inverseAttributeInfo.name), + inverseTypeName, GraphHelper.getIdFromVertex(inverseTypeName, edgeAndVertex.right)._getId(), + sourceTypeName, GraphHelper.getIdFromVertex(sourceTypeName, sourceVertex)._getId()); + } + } + else { + // We didn't find the edge for the inverse reference. + // Since Atlas currently does not automatically set + // the inverse reference when a reference value is updated, + // unbalanced references are not unexpected. + // The presence of inverse reference values depends on + // well behaved client applications which explicitly set + // both ends of the reference. + // TODO: throw an exception as it indicates a unbalanced reference? + String sourceTypeName = GraphHelper.getTypeName(sourceVertex); + LOG.warn("No edge found for inverse reference {} on vertex {} for entity instance {}:{} which points back to vertex {} for {}:{}", + inverseAttributeInfo.name, edgeAndVertex.right, + inverseTypeName, GraphHelper.getIdFromVertex(inverseTypeName, edgeAndVertex.right)._getId(), + sourceVertex, sourceTypeName, GraphHelper.getIdFromVertex(sourceTypeName, sourceVertex)._getId()); + } + } + + /** + * Remove any unidirectional map or array reference to a class, struct, or trait vertex. + * This involves removing appropriate value from the vertex property which holds the + * reference values. + * + * @param targetVertex a vertex which represents a class, struct, or trait instance + * @throws AtlasException + */ + private void removeUnidirectionalReferences(Vertex targetVertex) throws AtlasException { + + // Search for any remaining incoming edges that represent unidirectional references + // to the target vertex. + Iterable<Edge> incomingEdges = targetVertex.getEdges(Direction.IN); + for (Edge edge : incomingEdges) { + String label = edge.getLabel(); + AtlasEdgeLabel atlasEdgeLabel = new AtlasEdgeLabel(label); + Vertex referencingVertex = edge.getVertex(Direction.OUT); + String typeName = atlasEdgeLabel.getTypeName(); + IConstructableType referencingType = typeSystem.getDataType(IConstructableType.class, typeName); + + AttributeInfo attributeInfo = referencingType.fieldMapping().fields.get(atlasEdgeLabel.getAttributeName()); + if (attributeInfo == null) { + String instanceId = getInstanceName(referencingVertex, referencingType); + throw new AtlasException("Outgoing edge " + edge.getId().toString() + + " for " + instanceId + "(vertex " + referencingVertex + "): label " + label + + " has an attribute name " + atlasEdgeLabel.getAttributeName() + " that is undefined on " + + referencingType.getTypeCategory() + " " + typeName); + } + // Remove the appropriate value from the vertex property for this reference. + removeReferenceValue(edge, atlasEdgeLabel, referencingVertex, referencingType, attributeInfo.dataType().getTypeCategory()); + } + } + + private Pair<String, Boolean> removeReferenceValue(Edge edge, AtlasEdgeLabel atlasEdgeLabel, + Vertex referencingVertex, IConstructableType referencingType, TypeCategory attrTypeCategory) + throws AtlasException { + + graphHelper.removeEdge(edge); + if (attrTypeCategory != TypeCategory.ARRAY && attrTypeCategory != TypeCategory.MAP) { + // Multiplicity-one reference is represented by the edge, + // there is no vertex property to update. So just remove the edge. + return new Pair<String, Boolean>(edge.getId().toString(), Boolean.TRUE); + } + List<String> currentRefValues = referencingVertex.getProperty(atlasEdgeLabel.getQualifiedAttributeName()); + List<String> newRefValues = new ArrayList<>(currentRefValues); + Pair<String, Boolean> refValueRemoved = null; + if (attrTypeCategory == TypeCategory.ARRAY) { + refValueRemoved = removeArrayReferenceValue(atlasEdgeLabel, referencingVertex, edge, newRefValues); + } + else { + refValueRemoved = removeMapReferenceValue(atlasEdgeLabel, referencingVertex, edge, newRefValues); + } + if (refValueRemoved.right) { + if (LOG.isDebugEnabled()) { + String instanceId = getInstanceName(referencingVertex, referencingType); + LOG.debug("Reference value {} removed from reference {} on vertex {} for instance of {} {}", + refValueRemoved.left, atlasEdgeLabel.getAttributeName(), referencingVertex, + referencingType.getTypeCategory(), instanceId); + } + // If the referencing instance is an entity, update the modification timestamp. + if (referencingType instanceof ClassType) { + GraphHelper.setProperty(referencingVertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, System.currentTimeMillis()); + } + } + else { + // The expected value is missing from the reference property values - log a warning. + String instanceId = getInstanceName(referencingVertex, referencingType); + LOG.warn("Reference value {} expected but not found in array reference {} on vertex {} for instance of {} {}", + refValueRemoved.left, atlasEdgeLabel.getAttributeName(), referencingVertex, + referencingType.getTypeCategory(), instanceId); + } + return refValueRemoved; + } + + private TypeUtils.Pair<String, Boolean> removeArrayReferenceValue(AtlasEdgeLabel atlasEdgeLabel, Vertex referencingVertex, + Edge edge, List<String> newRefValues) { + + String refValueToRemove = edge.getId().toString(); + boolean valueRemoved = newRefValues.remove(refValueToRemove); + if (valueRemoved) { + GraphHelper.setProperty(referencingVertex, atlasEdgeLabel.getQualifiedAttributeName(), newRefValues); + } + return new TypeUtils.Pair<String, Boolean>(refValueToRemove, Boolean.valueOf(valueRemoved)); + } + + private TypeUtils.Pair<String, Boolean> removeMapReferenceValue(AtlasEdgeLabel atlasEdgeLabel, Vertex referencingVertex, + Edge edge, List<String> newRefValues) throws AtlasException { + + String refValueToRemove = atlasEdgeLabel.getMapKey(); + if (refValueToRemove == null) { + // Edge label is missing the map key - throw an exception. + String typeName = atlasEdgeLabel.getTypeName(); + throw new AtlasException("Outgoing edge " + edge.getId().toString() + + " for vertex " + referencingVertex + "): label " + atlasEdgeLabel.getEdgeLabel() + + " for map attribute " + atlasEdgeLabel.getAttributeName() + " on type " + + typeName + " is missing the map key"); + } + boolean valueRemoved = newRefValues.remove(refValueToRemove); + if (valueRemoved) { + GraphHelper.setProperty(referencingVertex, atlasEdgeLabel.getQualifiedAttributeName(), newRefValues); + // For maps, also remove the key-value pair property value. + GraphHelper.setProperty(referencingVertex, atlasEdgeLabel.getQualifiedMapKey(), null); + } + return new TypeUtils.Pair<String, Boolean>(refValueToRemove, Boolean.valueOf(valueRemoved)); + } + void deleteEntity(String typeName, Vertex instanceVertex) throws AtlasException { + // Check if this entity has already been processed. + Id id = GraphHelper.getIdFromVertex(typeName, instanceVertex); + if (deletedEntityGuids.contains(id._getId())) { + return; + } + deletedEntityGuids.add(id._getId()); + // Remove traits owned by this entity. deleteAllTraits(instanceVertex); // Create an empty instance to use for clearing all attributes. - Id id = GraphHelper.getIdFromVertex(typeName, instanceVertex); ClassType classType = typeSystem.getDataType(ClassType.class, typeName); ITypedReferenceableInstance typedInstance = classType.createInstance(id); // Remove any underlying structs and composite entities owned by this entity. mapInstanceToVertex(typedInstance, instanceVertex, classType.fieldMapping().fields, false, Operation.DELETE); - deletedEntityGuids.add(id._getId()); deletedEntities.add(typedInstance); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/7ebb2013/repository/src/test/java/org/apache/atlas/TestUtils.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/TestUtils.java b/repository/src/test/java/org/apache/atlas/TestUtils.java index d54dfab..ce59c43 100755 --- a/repository/src/test/java/org/apache/atlas/TestUtils.java +++ b/repository/src/test/java/org/apache/atlas/TestUtils.java @@ -170,7 +170,7 @@ public final class TestUtils { max.set("mentor", julius); john.set("manager", jane); - + john.set("mentor", max); hrDept.set("employees", ImmutableList.of(john, jane, julius, max)); jane.set("subordinates", ImmutableList.of(john, max)); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/7ebb2013/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteEntitiesTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteEntitiesTest.java b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteEntitiesTest.java index b024152..3681de7 100644 --- a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteEntitiesTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteEntitiesTest.java @@ -18,6 +18,10 @@ package org.apache.atlas.repository.graph; +import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAttrDef; +import static org.apache.atlas.typesystem.types.utils.TypesUtil.createRequiredAttrDef; + +import com.google.common.collect.ImmutableList; import com.thinkaurelius.titan.core.TitanGraph; import com.thinkaurelius.titan.core.util.TitanCleanup; import com.tinkerpop.blueprints.Vertex; @@ -27,13 +31,25 @@ import org.apache.atlas.TestUtils; import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService; import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.RepositoryException; +import org.apache.atlas.typesystem.IStruct; import org.apache.atlas.typesystem.ITypedReferenceableInstance; +import org.apache.atlas.typesystem.ITypedStruct; import org.apache.atlas.typesystem.Referenceable; +import org.apache.atlas.typesystem.Struct; +import org.apache.atlas.typesystem.TypesDef; import org.apache.atlas.typesystem.exception.EntityNotFoundException; +import org.apache.atlas.typesystem.persistence.Id; +import org.apache.atlas.typesystem.types.AttributeDefinition; import org.apache.atlas.typesystem.types.ClassType; +import org.apache.atlas.typesystem.types.DataTypes; +import org.apache.atlas.typesystem.types.EnumTypeDefinition; +import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition; import org.apache.atlas.typesystem.types.Multiplicity; +import org.apache.atlas.typesystem.types.StructTypeDefinition; +import org.apache.atlas.typesystem.types.TraitType; import org.apache.atlas.typesystem.types.TypeSystem; import org.apache.atlas.typesystem.types.TypeUtils.Pair; +import org.apache.atlas.typesystem.types.utils.TypesUtil; import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; @@ -44,7 +60,10 @@ import javax.inject.Inject; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.Date; import java.util.List; +import java.util.Map; /** * Test for GraphBackedMetadataRepository.deleteEntities @@ -95,8 +114,12 @@ public class GraphBackedMetadataRepositoryDeleteEntitiesTest { + /** + * Verify deleting entities with composite references to other entities. + * The composite entities should also be deleted. + */ @Test - public void testDeleteEntities() throws Exception { + public void testDeleteEntitiesWithCompositeArrayReference() throws Exception { String hrDeptGuid = createHrDeptGraph(); ITypedReferenceableInstance hrDept = repositoryService.getEntityDefinition(hrDeptGuid); @@ -136,25 +159,397 @@ public class GraphBackedMetadataRepositoryDeleteEntitiesTest { Assert.assertEquals(vertexCount, 0); } - @Test(dependsOnMethods = "testDeleteEntities") - public void testDeleteContainedEntity() throws Exception { + @Test + public void testDeleteEntitiesWithCompositeMapReference() throws Exception { + // Define type for map value. + HierarchicalTypeDefinition<ClassType> mapValueDef = TypesUtil.createClassTypeDef("CompositeMapValue", + ImmutableList.<String>of(), + TypesUtil.createOptionalAttrDef("attr1", DataTypes.STRING_TYPE)); + + // Define type with map where the value is a composite class reference to MapValue. + HierarchicalTypeDefinition<ClassType> mapOwnerDef = TypesUtil.createClassTypeDef("CompositeMapOwner", + ImmutableList.<String>of(), + new AttributeDefinition("map", DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), + "CompositeMapValue"), Multiplicity.OPTIONAL, true, null)); + TypesDef typesDef = TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), + ImmutableList.<StructTypeDefinition>of(), ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(), + ImmutableList.of(mapOwnerDef, mapValueDef)); + typeSystem.defineTypes(typesDef); + ClassType mapOwnerType = typeSystem.getDataType(ClassType.class, "CompositeMapOwner"); + ClassType mapValueType = typeSystem.getDataType(ClassType.class, "CompositeMapValue"); + + // Create instances of MapOwner and MapValue. + // Set MapOwner.map with one entry that references MapValue instance. + ITypedReferenceableInstance mapOwnerInstance = mapOwnerType.createInstance(); + ITypedReferenceableInstance mapValueInstance = mapValueType.createInstance(); + mapOwnerInstance.set("map", Collections.singletonMap("value1", mapValueInstance)); + List<String> createEntitiesResult = repositoryService.createEntities(mapOwnerInstance, mapValueInstance); + Assert.assertEquals(createEntitiesResult.size(), 2); + List<String> guids = repositoryService.getEntityList("CompositeMapOwner"); + Assert.assertEquals(guids.size(), 1); + String mapOwnerGuid = guids.get(0); + + // Verify MapOwner.map attribute has expected value. + mapOwnerInstance = repositoryService.getEntityDefinition(mapOwnerGuid); + Object object = mapOwnerInstance.get("map"); + Assert.assertNotNull(object); + Assert.assertTrue(object instanceof Map); + Map<String, ITypedReferenceableInstance> map = (Map<String, ITypedReferenceableInstance>)object; + Assert.assertEquals(map.size(), 1); + mapValueInstance = map.get("value1"); + Assert.assertNotNull(mapValueInstance); + String mapValueGuid = mapValueInstance.getId()._getId(); + String edgeLabel = GraphHelper.getEdgeLabel(mapOwnerType, mapOwnerType.fieldMapping.fields.get("map")); + String mapEntryLabel = edgeLabel + "." + "value1"; + AtlasEdgeLabel atlasEdgeLabel = new AtlasEdgeLabel(mapEntryLabel); + Vertex mapOwnerVertex = GraphHelper.getInstance().getVertexForGUID(mapOwnerGuid); + object = mapOwnerVertex.getProperty(atlasEdgeLabel.getQualifiedMapKey()); + Assert.assertNotNull(object); + + Pair<List<String>, List<ITypedReferenceableInstance>> deleteEntitiesResult = + repositoryService.deleteEntities(Arrays.asList(mapOwnerGuid)); + Assert.assertEquals(deleteEntitiesResult.left.size(), 2); + Assert.assertTrue(deleteEntitiesResult.left.containsAll(guids)); + verifyEntityDoesNotExist(mapOwnerGuid); + verifyEntityDoesNotExist(mapValueGuid); + } + + /** + * Verify deleting an entity which is contained by another + * entity through a bi-directional composite reference. + * + * @throws Exception + */ + @Test(dependsOnMethods = "testDeleteEntitiesWithCompositeArrayReference") + public void testDisconnectBidirectionalReferences() throws Exception { String hrDeptGuid = createHrDeptGraph(); ITypedReferenceableInstance hrDept = repositoryService.getEntityDefinition(hrDeptGuid); Object refValue = hrDept.get("employees"); Assert.assertTrue(refValue instanceof List); List<Object> employees = (List<Object>)refValue; Assert.assertEquals(employees.size(), 4); - Object listValue = employees.get(2); - Assert.assertTrue(listValue instanceof ITypedReferenceableInstance); - ITypedReferenceableInstance employee = (ITypedReferenceableInstance) listValue; - String employeeGuid = employee.getId()._getId(); + String employeeGuid = null; + for (Object listValue : employees) { + Assert.assertTrue(listValue instanceof ITypedReferenceableInstance); + ITypedReferenceableInstance employee = (ITypedReferenceableInstance) listValue; + if (employee.get("name").equals("Max")) { + employeeGuid = employee.getId()._getId(); + } + } + Assert.assertNotNull(employeeGuid); + + // Verify that Max is one of Jane's subordinates. + ITypedReferenceableInstance jane = repositoryService.getEntityDefinition("Manager", "name", "Jane"); + refValue = jane.get("subordinates"); + Assert.assertTrue(refValue instanceof List); + List<Object> subordinates = (List<Object>)refValue; + Assert.assertEquals(subordinates.size(), 2); + List<String> subordinateIds = new ArrayList<>(2); + for (Object listValue : employees) { + Assert.assertTrue(listValue instanceof ITypedReferenceableInstance); + ITypedReferenceableInstance employee = (ITypedReferenceableInstance) listValue; + subordinateIds.add(employee.getId()._getId()); + } + Assert.assertTrue(subordinateIds.contains(employeeGuid)); Pair<List<String>, List<ITypedReferenceableInstance>> deletedEntities = repositoryService.deleteEntities(Arrays.asList(employeeGuid)); Assert.assertTrue(deletedEntities.left.contains(employeeGuid)); verifyEntityDoesNotExist(employeeGuid); + // Verify that the Department.employees reference to the deleted employee + // was disconnected. + hrDept = repositoryService.getEntityDefinition(hrDeptGuid); + refValue = hrDept.get("employees"); + Assert.assertTrue(refValue instanceof List); + employees = (List<Object>)refValue; + Assert.assertEquals(employees.size(), 3); + for (Object listValue : employees) { + Assert.assertTrue(listValue instanceof ITypedReferenceableInstance); + ITypedReferenceableInstance employee = (ITypedReferenceableInstance) listValue; + Assert.assertNotEquals(employee.getId()._getId(), employeeGuid); + } + + // Verify that the Manager.subordinates reference to the deleted employee + // Max was disconnected. + jane = repositoryService.getEntityDefinition("Manager", "name", "Jane"); + refValue = jane.get("subordinates"); + Assert.assertTrue(refValue instanceof List); + subordinates = (List<Object>)refValue; + Assert.assertEquals(subordinates.size(), 1); + Object listValue = subordinates.get(0); + Assert.assertTrue(listValue instanceof ITypedReferenceableInstance); + ITypedReferenceableInstance subordinate = (ITypedReferenceableInstance) listValue; + String subordinateGuid = subordinate.getId()._getId(); + Assert.assertNotEquals(subordinateGuid, employeeGuid); + + // Verify that max's Person.mentor unidirectional reference to john was disconnected. + ITypedReferenceableInstance john = repositoryService.getEntityDefinition("Manager", "name", "John"); + refValue = john.get("mentor"); + Assert.assertNull(refValue); + + // Now delete jane - this should disconnect the manager reference from her + // subordinate. + String janeGuid = jane.getId()._getId(); + deletedEntities = repositoryService.deleteEntities(Arrays.asList(janeGuid)); + Assert.assertTrue(deletedEntities.left.contains(janeGuid)); + verifyEntityDoesNotExist(janeGuid); + subordinate = repositoryService.getEntityDefinition(subordinateGuid); + Assert.assertNull(subordinate.get("manager")); } + /** + * Verify deleting entity that is the target of a unidirectional class array reference + * from a class instance. + */ + @Test + public void testDisconnectUnidirectionalArrayReferenceFromClassType() throws Exception { + createDbTableGraph(); + + // Get the guid for one of the table's columns. + ITypedReferenceableInstance table = repositoryService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", TestUtils.TABLE_NAME); + String tableGuid = table.getId()._getId(); + Object refValues = table.get("columns"); + Assert.assertTrue(refValues instanceof List); + List<Object> refList = (List<Object>) refValues; + Assert.assertEquals(refList.size(), 5); + Assert.assertTrue(refList.get(0) instanceof ITypedReferenceableInstance); + ITypedReferenceableInstance column = (ITypedReferenceableInstance) refList.get(0); + String columnGuid = column.getId()._getId(); + + // Delete the column. + Pair<List<String>, List<ITypedReferenceableInstance>> deletedEntities = + repositoryService.deleteEntities(Arrays.asList(columnGuid)); + Assert.assertEquals(deletedEntities.left.size(), 1); + Assert.assertEquals(deletedEntities.right.size(), 1); + verifyEntityDoesNotExist(columnGuid); + + // Verify table.columns reference to the deleted column has been disconnected. + table = repositoryService.getEntityDefinition(tableGuid); + refList = (List<Object>) table.get("columns"); + Assert.assertEquals(refList.size(), 4); + for (Object refValue : refList) { + Assert.assertTrue(refValue instanceof ITypedReferenceableInstance); + column = (ITypedReferenceableInstance)refValue; + Assert.assertFalse(column.getId()._getId().equals(columnGuid)); + } + } + + /** + * Verify deleting entities that are the target of a unidirectional class array reference + * from a struct or trait instance. + */ + @Test + public void testDisconnectUnidirectionalArrayReferenceFromStructAndTraitTypes() throws Exception { + // Define class types. + HierarchicalTypeDefinition<ClassType> structTargetDef = TypesUtil.createClassTypeDef("StructTarget", + ImmutableList.<String>of(), TypesUtil.createOptionalAttrDef("attr1", DataTypes.STRING_TYPE)); + HierarchicalTypeDefinition<ClassType> traitTargetDef = TypesUtil.createClassTypeDef("TraitTarget", + ImmutableList.<String>of(), TypesUtil.createOptionalAttrDef("attr1", DataTypes.STRING_TYPE)); + HierarchicalTypeDefinition<ClassType> structContainerDef = TypesUtil.createClassTypeDef("StructContainer", + ImmutableList.<String>of(), TypesUtil.createOptionalAttrDef("struct", "TestStruct")); + + // Define struct and trait types which have a unidirectional array reference + // to a class type. + StructTypeDefinition structDef = TypesUtil.createStructTypeDef("TestStruct", + new AttributeDefinition("target", DataTypes.arrayTypeName("StructTarget"), Multiplicity.OPTIONAL, false, null), + new AttributeDefinition("nestedStructs", DataTypes.arrayTypeName("NestedStruct"), Multiplicity.OPTIONAL, false, null)); + StructTypeDefinition nestedStructDef = TypesUtil.createStructTypeDef("NestedStruct", + TypesUtil.createOptionalAttrDef("attr1", DataTypes.STRING_TYPE)); + HierarchicalTypeDefinition<TraitType> traitDef = TypesUtil.createTraitTypeDef("TestTrait", ImmutableList.<String>of(), + new AttributeDefinition("target", DataTypes.arrayTypeName("TraitTarget"), Multiplicity.OPTIONAL, false, null)); + + TypesDef typesDef = TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.of(structDef, nestedStructDef), + ImmutableList.of(traitDef), ImmutableList.of(structTargetDef, traitTargetDef, structContainerDef)); + typeSystem.defineTypes(typesDef); + + // Create instances of class, struct, and trait types. + Referenceable structTargetEntity = new Referenceable("StructTarget"); + Referenceable traitTargetEntity = new Referenceable("TraitTarget"); + Referenceable structContainerEntity = new Referenceable("StructContainer"); + Referenceable structInstance = new Referenceable("TestStruct"); + Referenceable nestedStructInstance = new Referenceable("NestedStruct"); + Referenceable traitInstance = new Referenceable("TestTrait"); + structContainerEntity.set("struct", structInstance); + structInstance.set("target", ImmutableList.of(structTargetEntity)); + structInstance.set("nestedStructs", ImmutableList.of(nestedStructInstance)); + + ClassType structTargetType = typeSystem.getDataType(ClassType.class, "StructTarget"); + ClassType traitTargetType = typeSystem.getDataType(ClassType.class, "TraitTarget"); + ClassType structContainerType = typeSystem.getDataType(ClassType.class, "StructContainer"); + + ITypedReferenceableInstance structTargetConvertedEntity = + structTargetType.convert(structTargetEntity, Multiplicity.REQUIRED); + ITypedReferenceableInstance traitTargetConvertedEntity = + traitTargetType.convert(traitTargetEntity, Multiplicity.REQUIRED); + ITypedReferenceableInstance structContainerConvertedEntity = + structContainerType.convert(structContainerEntity, Multiplicity.REQUIRED); + + List<String> guids = repositoryService.createEntities( + structTargetConvertedEntity, traitTargetConvertedEntity, structContainerConvertedEntity); + Assert.assertEquals(guids.size(), 3); + + guids = repositoryService.getEntityList("StructTarget"); + Assert.assertEquals(guids.size(), 1); + String structTargetGuid = guids.get(0); + + guids = repositoryService.getEntityList("TraitTarget"); + Assert.assertEquals(guids.size(), 1); + String traitTargetGuid = guids.get(0); + + guids = repositoryService.getEntityList("StructContainer"); + Assert.assertEquals(guids.size(), 1); + String structContainerGuid = guids.get(0); + + // Add TestTrait to StructContainer instance + traitInstance.set("target", ImmutableList.of(new Id(traitTargetGuid, 0, "TraitTarget"))); + TraitType traitType = typeSystem.getDataType(TraitType.class, "TestTrait"); + ITypedStruct convertedTrait = traitType.convert(traitInstance, Multiplicity.REQUIRED); + repositoryService.addTrait(structContainerGuid, convertedTrait); + + // Verify that the unidirectional references from the struct and trait instances + // are pointing at the target entities. + structContainerConvertedEntity = repositoryService.getEntityDefinition(structContainerGuid); + Object object = structContainerConvertedEntity.get("struct"); + Assert.assertNotNull(object); + Assert.assertTrue(object instanceof ITypedStruct); + ITypedStruct struct = (ITypedStruct) object; + object = struct.get("target"); + Assert.assertNotNull(object); + Assert.assertTrue(object instanceof List); + List<ITypedReferenceableInstance> refList = (List<ITypedReferenceableInstance>)object; + Assert.assertEquals(refList.size(), 1); + Assert.assertEquals(refList.get(0).getId()._getId(), structTargetGuid); + + IStruct trait = structContainerConvertedEntity.getTrait("TestTrait"); + Assert.assertNotNull(trait); + object = trait.get("target"); + Assert.assertNotNull(object); + Assert.assertTrue(object instanceof List); + refList = (List<ITypedReferenceableInstance>)object; + Assert.assertEquals(refList.size(), 1); + Assert.assertEquals(refList.get(0).getId()._getId(), traitTargetGuid); + + // Delete the entities that are targets of the struct and trait instances. + Pair<List<String>, List<ITypedReferenceableInstance>> deleteEntitiesResult = + repositoryService.deleteEntities(Arrays.asList(structTargetGuid, traitTargetGuid)); + verifyEntityDoesNotExist(structTargetGuid); + verifyEntityDoesNotExist(traitTargetGuid); + Assert.assertEquals(deleteEntitiesResult.left.size(), 2); + Assert.assertTrue(deleteEntitiesResult.left.containsAll(Arrays.asList(structTargetGuid, traitTargetGuid))); + + // Verify that the unidirectional references from the struct and trait instances + // to the deleted entities were disconnected. + structContainerConvertedEntity = repositoryService.getEntityDefinition(structContainerGuid); + object = structContainerConvertedEntity.get("struct"); + Assert.assertNotNull(object); + Assert.assertTrue(object instanceof ITypedStruct); + struct = (ITypedStruct) object; + Assert.assertNull(struct.get("target")); + trait = structContainerConvertedEntity.getTrait("TestTrait"); + Assert.assertNotNull(trait); + Assert.assertNull(trait.get("target")); + + // Delete the entity which contains nested structs and has the TestTrait trait. + deleteEntitiesResult = + repositoryService.deleteEntities(Arrays.asList(structContainerGuid)); + verifyEntityDoesNotExist(structContainerGuid); + Assert.assertEquals(deleteEntitiesResult.left.size(), 1); + Assert.assertTrue(deleteEntitiesResult.left.contains(structContainerGuid)); + + // Verify all TestStruct struct vertices were removed. + int vertexCount = countVertices(Constants.ENTITY_TYPE_PROPERTY_KEY, "TestStruct"); + Assert.assertEquals(vertexCount, 0); + + // Verify all NestedStruct struct vertices were removed. + vertexCount = countVertices(Constants.ENTITY_TYPE_PROPERTY_KEY, "NestedStruct"); + Assert.assertEquals(vertexCount, 0); + + // Verify all TestTrait trait vertices were removed. + vertexCount = countVertices(Constants.ENTITY_TYPE_PROPERTY_KEY, "TestTrait"); + Assert.assertEquals(vertexCount, 0); + } + + /** + * Verify deleting entities that are the target of class map references. + */ + @Test + public void testDisconnectMapReferenceFromClassType() throws Exception { + // Define type for map value. + HierarchicalTypeDefinition<ClassType> mapValueDef = TypesUtil.createClassTypeDef("MapValue", + ImmutableList.<String>of(), + new AttributeDefinition("biMapOwner", "MapOwner", Multiplicity.OPTIONAL, false, "biMap")); + + // Define type with unidirectional and bidirectional map references, + // where the map value is a class reference to MapValue. + HierarchicalTypeDefinition<ClassType> mapOwnerDef = TypesUtil.createClassTypeDef("MapOwner", + ImmutableList.<String>of(), + new AttributeDefinition("map", DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), + "MapValue"), Multiplicity.OPTIONAL, false, null), + new AttributeDefinition("biMap", DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), + "MapValue"), Multiplicity.OPTIONAL, false, "biMapOwner")); + TypesDef typesDef = TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), + ImmutableList.<StructTypeDefinition>of(), ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(), + ImmutableList.of(mapOwnerDef, mapValueDef)); + typeSystem.defineTypes(typesDef); + ClassType mapOwnerType = typeSystem.getDataType(ClassType.class, "MapOwner"); + ClassType mapValueType = typeSystem.getDataType(ClassType.class, "MapValue"); + + // Create instances of MapOwner and MapValue. + // Set MapOwner.map and MapOwner.biMap with one entry that references MapValue instance. + ITypedReferenceableInstance mapOwnerInstance = mapOwnerType.createInstance(); + ITypedReferenceableInstance mapValueInstance = mapValueType.createInstance(); + mapOwnerInstance.set("map", Collections.singletonMap("value1", mapValueInstance)); + mapOwnerInstance.set("biMap", Collections.singletonMap("value1", mapValueInstance)); + // Set biMapOwner reverse reference on MapValue. + mapValueInstance.set("biMapOwner", mapOwnerInstance); + List<String> createEntitiesResult = repositoryService.createEntities(mapOwnerInstance, mapValueInstance); + Assert.assertEquals(createEntitiesResult.size(), 2); + List<String> guids = repositoryService.getEntityList("MapOwner"); + Assert.assertEquals(guids.size(), 1); + String mapOwnerGuid = guids.get(0); + + String edgeLabel = GraphHelper.getEdgeLabel(mapOwnerType, mapOwnerType.fieldMapping.fields.get("map")); + String mapEntryLabel = edgeLabel + "." + "value1"; + AtlasEdgeLabel atlasEdgeLabel = new AtlasEdgeLabel(mapEntryLabel); + edgeLabel = GraphHelper.getEdgeLabel(mapOwnerType, mapOwnerType.fieldMapping.fields.get("biMap")); + mapEntryLabel = edgeLabel + "." + "value1"; + AtlasEdgeLabel biMapAtlasEdgeLabel = new AtlasEdgeLabel(mapEntryLabel); + + // Verify MapOwner.map attribute has expected value. + String mapValueGuid = null; + Vertex mapOwnerVertex = null; + mapOwnerInstance = repositoryService.getEntityDefinition(mapOwnerGuid); + for (String mapAttrName : Arrays.asList("map", "biMap")) { + Object object = mapOwnerInstance.get(mapAttrName); + Assert.assertNotNull(object); + Assert.assertTrue(object instanceof Map); + Map<String, ITypedReferenceableInstance> map = (Map<String, ITypedReferenceableInstance>)object; + Assert.assertEquals(map.size(), 1); + mapValueInstance = map.get("value1"); + Assert.assertNotNull(mapValueInstance); + mapValueGuid = mapValueInstance.getId()._getId(); + mapOwnerVertex = GraphHelper.getInstance().getVertexForGUID(mapOwnerGuid); + object = mapOwnerVertex.getProperty(atlasEdgeLabel.getQualifiedMapKey()); + Assert.assertNotNull(object); + } + + // Delete the map value instance. + // This should disconnect the references from the map owner instance. + Pair<List<String>, List<ITypedReferenceableInstance>> deleteEntitiesResult = + repositoryService.deleteEntities(Arrays.asList(mapValueGuid)); + verifyEntityDoesNotExist(mapValueGuid); + + // Verify map references from mapOwner were disconnected. + mapOwnerInstance = repositoryService.getEntityDefinition(mapOwnerGuid); + Assert.assertNull(mapOwnerInstance.get("map")); + Assert.assertNull(mapOwnerInstance.get("biMap")); + mapOwnerVertex = GraphHelper.getInstance().getVertexForGUID(mapOwnerGuid); + Object object = mapOwnerVertex.getProperty(atlasEdgeLabel.getQualifiedMapKey()); + Assert.assertNull(object); + object = mapOwnerVertex.getProperty(biMapAtlasEdgeLabel.getQualifiedMapKey()); + Assert.assertNull(object); + } + private String createHrDeptGraph() throws Exception { Referenceable deptEg1 = TestUtils.createDeptEg1(typeSystem); ClassType deptType = typeSystem.getDataType(ClassType.class, "Department"); @@ -170,6 +565,36 @@ public class GraphBackedMetadataRepositoryDeleteEntitiesTest { return entityList.get(0); } + private void createDbTableGraph() throws Exception { + Referenceable databaseInstance = new Referenceable(TestUtils.DATABASE_TYPE); + databaseInstance.set("name", TestUtils.DATABASE_NAME); + databaseInstance.set("description", "foo database"); + + ClassType dbType = typeSystem.getDataType(ClassType.class, TestUtils.DATABASE_TYPE); + ITypedReferenceableInstance db = dbType.convert(databaseInstance, Multiplicity.REQUIRED); + Referenceable tableInstance = new Referenceable(TestUtils.TABLE_TYPE, TestUtils.CLASSIFICATION); + tableInstance.set("name", TestUtils.TABLE_NAME); + tableInstance.set("description", "bar table"); + tableInstance.set("type", "managed"); + Struct traitInstance = (Struct) tableInstance.getTrait(TestUtils.CLASSIFICATION); + traitInstance.set("tag", "foundation_etl"); + tableInstance.set("tableType", 1); // enum + + tableInstance.set("database", databaseInstance); + ArrayList<Referenceable> columns = new ArrayList<>(); + for (int index = 0; index < 5; index++) { + Referenceable columnInstance = new Referenceable("column_type"); + final String name = "column_" + index; + columnInstance.set("name", name); + columnInstance.set("type", "string"); + columns.add(columnInstance); + } + tableInstance.set("columns", columns); + ClassType tableType = typeSystem.getDataType(ClassType.class, TestUtils.TABLE_TYPE); + ITypedReferenceableInstance table = tableType.convert(tableInstance, Multiplicity.REQUIRED); + repositoryService.createEntities(db, table); + } + private int countVertices(String propertyName, Object value) { Iterable<Vertex> vertices = graphProvider.get().getVertices(propertyName, value); int vertexCount = 0; @@ -179,9 +604,9 @@ public class GraphBackedMetadataRepositoryDeleteEntitiesTest { return vertexCount; } - private void verifyEntityDoesNotExist(String hrDeptGuid) throws RepositoryException { + private void verifyEntityDoesNotExist(String guid) throws RepositoryException { try { - repositoryService.getEntityDefinition(hrDeptGuid); + repositoryService.getEntityDefinition(guid); Assert.fail("EntityNotFoundException was expected but none thrown"); } catch(EntityNotFoundException e) { // good http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/7ebb2013/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java index 21789fd..83e4d85 100755 --- a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java @@ -534,7 +534,6 @@ public class GraphBackedMetadataRepositoryTest { Assert.assertTrue(creationTimestamp < modificationTimestampPostUpdate); // Update max's mentor reference to jane. - instance = personType.createInstance(max.getId()); instance.set("mentor", janeGuid); repositoryService.updatePartial(instance); @@ -550,6 +549,30 @@ public class GraphBackedMetadataRepositoryTest { Long modificationTimestampPost2ndUpdate = vertex.getProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY); Assert.assertNotNull(modificationTimestampPost2ndUpdate); Assert.assertTrue(modificationTimestampPostUpdate < modificationTimestampPost2ndUpdate); + + ITypedReferenceableInstance julius = repositoryService.getEntityDefinition("Person", "name", "Julius"); + Id juliusGuid = julius.getId(); + instance = personType.createInstance(max.getId()); + instance.set("manager", juliusGuid); + repositoryService.updatePartial(instance); + + // Verify the update was applied correctly - julius should now be max's manager. + max = repositoryService.getEntityDefinition(maxGuid); + object = max.get("manager"); + Assert.assertTrue(object instanceof ITypedReferenceableInstance); + refTarget = (ITypedReferenceableInstance) object; + Assert.assertEquals(refTarget.getId()._getId(), juliusGuid._getId()); + + // Verify that max is no longer a subordinate of jane. + jane = repositoryService.getEntityDefinition(janeGuid._getId()); + Object refValue = jane.get("subordinates"); + Assert.assertTrue(refValue instanceof List); + List<Object> subordinates = (List<Object>)refValue; + Assert.assertEquals(subordinates.size(), 1); + Object listValue = subordinates.get(0); + Assert.assertTrue(listValue instanceof ITypedReferenceableInstance); + ITypedReferenceableInstance subordinate = (ITypedReferenceableInstance) listValue; + Assert.assertNotEquals(subordinate.getId()._getId(), maxGuid); } private ITypedReferenceableInstance createHiveTableInstance(Referenceable databaseInstance) throws Exception { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/7ebb2013/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java b/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java index 019e21b..4eacfb2 100644 --- a/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java +++ b/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java @@ -389,7 +389,7 @@ public class DefaultMetadataServiceTest { Assert.assertEquals(arrColumnsList.size(), columns.size()); assertReferenceables(arrColumnsList.get(1), columns.get(1)); assertReferenceables(arrColumnsList.get(2), columns.get(2)); - + //Remove a class reference/Id and insert another reference //Also covers isComposite case since columns is a composite values.clear(); @@ -788,7 +788,8 @@ public class DefaultMetadataServiceTest { for (ITypedReferenceableInstance deletedEntity : deletedEntitiesFromListener) { deletedGuidsFromListener.add(deletedEntity.getId()._getId()); } - Assert.assertEquals(deletedGuidsFromListener, deletedGuids); + Assert.assertEquals(deletedGuidsFromListener.size(), deletedGuids.size()); + Assert.assertTrue(deletedGuidsFromListener.containsAll(deletedGuids)); } private static class DeleteEntitiesChangeListener implements EntityChangeListener {
