Repository: incubator-atlas Updated Branches: refs/heads/master 101abe6ee -> 8bb31fdf9
ATLAS-1584 Fix issues with owned map reference and add tests (sumasai) Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/8bb31fdf Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/8bb31fdf Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/8bb31fdf Branch: refs/heads/master Commit: 8bb31fdf98d214063b63ee8d730828e04c534fdc Parents: 101abe6 Author: Suma Shivaprasad <[email protected]> Authored: Tue Feb 28 16:36:45 2017 -0800 Committer: Suma Shivaprasad <[email protected]> Committed: Tue Feb 28 16:36:45 2017 -0800 ---------------------------------------------------------------------- .../org/apache/atlas/type/AtlasStructType.java | 4 + .../test/java/org/apache/atlas/TestUtilsV2.java | 5 +- release-log.txt | 3 +- .../store/graph/v1/AtlasGraphUtilsV1.java | 3 +- .../store/graph/v1/EntityGraphMapper.java | 45 ++- .../graph/v1/AtlasDeleteHandlerV1Test.java | 362 ++++++++++++++++++- .../store/graph/v1/AtlasEntityStoreV1Test.java | 24 +- .../store/graph/v1/HardDeleteHandlerV1Test.java | 18 + .../store/graph/v1/SoftDeleteHandlerV1Test.java | 13 + 9 files changed, 439 insertions(+), 38 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/8bb31fdf/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java index 57ad106..bb7eef8 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java @@ -559,6 +559,10 @@ public class AtlasStructType extends AtlasType { type = ((AtlasArrayType)type).getElementType(); } + if (type instanceof AtlasMapType) { + type = ((AtlasMapType)type).getValueType(); + } + return type instanceof AtlasEntityType ? (AtlasEntityType)type : null; } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/8bb31fdf/intg/src/test/java/org/apache/atlas/TestUtilsV2.java ---------------------------------------------------------------------- diff --git a/intg/src/test/java/org/apache/atlas/TestUtilsV2.java b/intg/src/test/java/org/apache/atlas/TestUtilsV2.java index 84e17cb..1fbf10e 100755 --- a/intg/src/test/java/org/apache/atlas/TestUtilsV2.java +++ b/intg/src/test/java/org/apache/atlas/TestUtilsV2.java @@ -713,7 +713,10 @@ public final class TestUtilsV2 { true, AtlasAttributeDef.Cardinality.SINGLE, 0, 1, false, false, - Collections.<AtlasStructDef.AtlasConstraintDef>emptyList() + new ArrayList<AtlasStructDef.AtlasConstraintDef>() {{ + add(new AtlasStructDef.AtlasConstraintDef( + AtlasConstraintDef.CONSTRAINT_TYPE_OWNED_REF)); + }} ), //map of structs new AtlasAttributeDef("partitionsMap", http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/8bb31fdf/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index f87a8de..4f59f57 100644 --- a/release-log.txt +++ b/release-log.txt @@ -9,7 +9,8 @@ ATLAS-1060 Add composite indexes for exact match performance improvements for al ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai) ALL CHANGES: -ATLAS_1589 DSL queries return wrong object when filter traverses an edge (jnhagelberg) +ATLAS-1584 Fix issues with owned map reference and add tests (sumasai) +ATLAS-1589 DSL queries return wrong object when filter traverses an edge (jnhagelberg) ATLAS-1590 UI : Edit Button is enabled for Deleted entities. (kevalbhatt) ATLAS-1487 Create Entity in UI : types list doesn't list fs_permissions (struct type) hence no entity could be created for it. (kevalbhatt) ATLAS-1573 Full text mapping for Entity store V2 http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/8bb31fdf/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasGraphUtilsV1.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasGraphUtilsV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasGraphUtilsV1.java index 49d5a08..13b4e9b 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasGraphUtilsV1.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasGraphUtilsV1.java @@ -225,8 +225,7 @@ public class AtlasGraphUtilsV1 { public static AtlasVertex findByGuid(String guid) { AtlasGraphQuery query = AtlasGraphProvider.getGraphInstance().query() - .has(Constants.GUID_PROPERTY_KEY, guid) - .has(Constants.STATE_PROPERTY_KEY, AtlasEntity.Status.ACTIVE.name()); + .has(Constants.GUID_PROPERTY_KEY, guid); Iterator<AtlasVertex> results = query.vertices().iterator(); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/8bb31fdf/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java index a5abac2..e6a7f41 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java @@ -59,6 +59,7 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -68,6 +69,7 @@ import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.CR import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.PARTIAL_UPDATE; import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.UPDATE; import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.DELETE; +import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY; import static org.apache.atlas.repository.graph.GraphHelper.string; @@ -437,7 +439,11 @@ public class EntityGraphMapper { Map<String, Object> finalMap = removeUnusedMapEntries(attribute, ctx.getReferringVertex(), ctx.getVertexProperty(), currentMap, newMap); - Set<String> newKeys = new HashSet<>(newMap.keySet()); + for (Object newEntry : newMap.values()) { + updateInConsistentOwnedMapVertices(ctx, mapType, newEntry); + } + + Set<String> newKeys = new LinkedHashSet<>(newMap.keySet()); newKeys.addAll(finalMap.keySet()); // for dereference on way out @@ -534,21 +540,21 @@ public class EntityGraphMapper { private Object mapCollectionElementsToVertex(AttributeMutationContext ctx, EntityMutationContext context) throws AtlasBaseException { switch(ctx.getAttrType().getTypeCategory()) { - case PRIMITIVE: - case ENUM: - return ctx.getValue(); + case PRIMITIVE: + case ENUM: + return ctx.getValue(); - case STRUCT: - return mapStructValue(ctx, context); + case STRUCT: + return mapStructValue(ctx, context); - case OBJECT_ID_TYPE: - AtlasEntityType instanceType = getInstanceType(ctx.getValue()); - ctx.setElementType(instanceType); - return mapObjectIdValue(ctx, context); + case OBJECT_ID_TYPE: + AtlasEntityType instanceType = getInstanceType(ctx.getValue()); + ctx.setElementType(instanceType); + return mapObjectIdValue(ctx, context); - case MAP: - case ARRAY: - default: + case MAP: + case ARRAY: + default: throw new AtlasBaseException(AtlasErrorCode.TYPE_CATEGORY_INVALID, ctx.getAttrType().getTypeCategory().name()); } } @@ -776,6 +782,19 @@ public class EntityGraphMapper { return new AtlasEntityHeader(id.getTypeName(), id.getGuid(), id.getUniqueAttributes()); } + private void updateInConsistentOwnedMapVertices(AttributeMutationContext ctx, AtlasMapType mapType, Object val) { + if (mapType.getValueType().getTypeCategory() == TypeCategory.OBJECT_ID_TYPE) { + AtlasEdge edge = (AtlasEdge) val; + if (ctx.getAttribute().isOwnedRef() && + GraphHelper.getStatus(edge) == AtlasEntity.Status.DELETED && + GraphHelper.getStatus(edge.getInVertex()) == AtlasEntity.Status.DELETED) { + //Resurrect the vertex and edge to ACTIVE state + GraphHelper.setProperty(edge, STATE_PROPERTY_KEY, AtlasEntity.Status.ACTIVE.name()); + GraphHelper.setProperty(edge.getInVertex(), STATE_PROPERTY_KEY, AtlasEntity.Status.ACTIVE.name()); + } + } + } + public void addClassifications(final EntityMutationContext context, String guid, List<AtlasClassification> classifications) throws AtlasBaseException { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/8bb31fdf/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasDeleteHandlerV1Test.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasDeleteHandlerV1Test.java b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasDeleteHandlerV1Test.java index 555f0ac..aab0d3e 100644 --- a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasDeleteHandlerV1Test.java +++ b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasDeleteHandlerV1Test.java @@ -21,6 +21,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import org.apache.atlas.AtlasException; import org.apache.atlas.RepositoryMetadataModule; +import org.apache.atlas.RequestContext; import org.apache.atlas.RequestContextV1; import org.apache.atlas.TestUtils; import org.apache.atlas.TestUtilsV2; @@ -37,6 +38,8 @@ import org.apache.atlas.model.typedef.AtlasEnumDef; import org.apache.atlas.model.typedef.AtlasStructDef; import org.apache.atlas.model.typedef.AtlasTypesDef; import org.apache.atlas.repository.Constants; +import org.apache.atlas.repository.RepositoryException; +import org.apache.atlas.repository.graph.AtlasEdgeLabel; import org.apache.atlas.repository.graph.AtlasGraphProvider; import org.apache.atlas.repository.graph.GraphBackedSearchIndexer; import org.apache.atlas.repository.graph.GraphHelper; @@ -128,7 +131,11 @@ public abstract class AtlasDeleteHandlerV1Test { AtlasEntityDef mapOwnerDef = AtlasTypeUtil.createClassTypeDef("CompositeMapOwner", "CompositeMapOwner_description", ImmutableSet.<String>of(), AtlasTypeUtil.createUniqueRequiredAttrDef("name", "string"), - AtlasTypeUtil.createOptionalAttrDef("map", "map<string,string>") + new AtlasStructDef.AtlasAttributeDef("map", "map<string,CompositeMapValue>", true, + AtlasStructDef.AtlasAttributeDef.Cardinality.SINGLE, 0, 1, false, false, + new ArrayList<AtlasStructDef.AtlasConstraintDef>() {{ + add(new AtlasStructDef.AtlasConstraintDef(AtlasStructDef.AtlasConstraintDef.CONSTRAINT_TYPE_OWNED_REF)); + }}) ); final AtlasTypesDef typesDef = AtlasTypeUtil.getTypesDef(ImmutableList.<AtlasEnumDef>of(), @@ -713,6 +720,109 @@ public abstract class AtlasDeleteHandlerV1Test { assertVerticesDeleted(getVertices(Constants.ENTITY_TYPE_PROPERTY_KEY, "TestTrait")); } + + /** + * Verify deleting entities that are the target of class map references. + */ + @Test + public void testDisconnectMapReferenceFromClassType() throws Exception { + // Define type for map value. + AtlasStructDef.AtlasAttributeDef[] mapValueAttributes = new AtlasStructDef.AtlasAttributeDef[]{ + new AtlasStructDef.AtlasAttributeDef("biMapOwner", "MapOwner", + true, + AtlasStructDef.AtlasAttributeDef.Cardinality.SINGLE, 0, 1, + false, false, + new ArrayList<AtlasStructDef.AtlasConstraintDef>() {{ + add(new AtlasStructDef.AtlasConstraintDef( + AtlasStructDef.AtlasConstraintDef.CONSTRAINT_TYPE_INVERSE_REF, new HashMap<String, Object>() {{ + put(AtlasStructDef.AtlasConstraintDef.CONSTRAINT_PARAM_ATTRIBUTE, "biMap"); + }})); + }})}; + + AtlasEntityDef mapValueContainerDef = + new AtlasEntityDef("MapValue", "MapValue_desc", "1.0", + Arrays.asList(mapValueAttributes), Collections.<String>emptySet()); + + // Define type with unidirectional and bidirectional map references, + // where the map value is a class reference to MapValue. + + AtlasStructDef.AtlasAttributeDef[] mapOwnerAttributes = new AtlasStructDef.AtlasAttributeDef[]{ + new AtlasStructDef.AtlasAttributeDef("map", "map<string,MapValue>", + true, + AtlasStructDef.AtlasAttributeDef.Cardinality.SINGLE, 0, 1, + false, false, + Collections.<AtlasStructDef.AtlasConstraintDef>emptyList()), + new AtlasStructDef.AtlasAttributeDef("biMap", "map<string,MapValue>", + true, + AtlasStructDef.AtlasAttributeDef.Cardinality.SINGLE, 0, 1, + false, false, + new ArrayList<AtlasStructDef.AtlasConstraintDef>() {{ + add(new AtlasStructDef.AtlasConstraintDef( + AtlasStructDef.AtlasConstraintDef.CONSTRAINT_TYPE_INVERSE_REF, new HashMap<String, Object>() {{ + put(AtlasStructDef.AtlasConstraintDef.CONSTRAINT_PARAM_ATTRIBUTE, "biMapOwner"); + }})); + }})}; + + AtlasEntityDef mapOwnerContainerDef = + new AtlasEntityDef("MapOwner", "MapOwner_desc", "1.0", + Arrays.asList(mapOwnerAttributes), Collections.<String>emptySet()); + + AtlasTypesDef typesDef = AtlasTypeUtil.getTypesDef(ImmutableList.<AtlasEnumDef>of(), + ImmutableList.<AtlasStructDef>of(), + ImmutableList.<AtlasClassificationDef>of(), + ImmutableList.<AtlasEntityDef>of(mapValueContainerDef, mapOwnerContainerDef)); + + typeDefStore.createTypesDef(typesDef); + + // Create instances of MapOwner and MapValue. + // Set MapOwner.map and MapOwner.biMap with one entry that references MapValue instance. + AtlasEntity mapOwnerInstance = new AtlasEntity("MapOwner"); + AtlasEntity mapValueInstance = new AtlasEntity("MapValue"); + + mapOwnerInstance.setAttribute("map", Collections.singletonMap("value1", AtlasTypeUtil.getAtlasObjectId(mapValueInstance))); + mapOwnerInstance.setAttribute("biMap", Collections.singletonMap("value1", AtlasTypeUtil.getAtlasObjectId(mapValueInstance))); + // Set biMapOwner reverse reference on MapValue. + mapValueInstance.setAttribute("biMapOwner", AtlasTypeUtil.getAtlasObjectId(mapOwnerInstance)); + + AtlasEntity.AtlasEntitiesWithExtInfo entities = new AtlasEntity.AtlasEntitiesWithExtInfo(); + entities.addReferredEntity(mapValueInstance); + entities.addEntity(mapOwnerInstance); + + final EntityMutationResponse response = entityStore.createOrUpdate(new AtlasEntityStream(entities), false); + Assert.assertEquals(response.getCreatedEntities().size(), 2); + final List<AtlasEntityHeader> mapOwnerCreated = response.getCreatedEntitiesByTypeName("MapOwner"); + AtlasEntity.AtlasEntityWithExtInfo mapOwnerEntity = entityStore.getById(mapOwnerCreated.get(0).getGuid()); + + String edgeLabel = AtlasGraphUtilsV1.getAttributeEdgeLabel(typeRegistry.getEntityTypeByName("MapOwner"), "map"); + String mapEntryLabel = edgeLabel + "." + "value1"; + AtlasEdgeLabel atlasEdgeLabel = new AtlasEdgeLabel(mapEntryLabel); + + // Verify MapOwner.map attribute has expected value. + String mapValueGuid = null; + AtlasVertex mapOwnerVertex = null; + for (String mapAttrName : Arrays.asList("map", "biMap")) { + Object object = mapOwnerEntity.getEntity().getAttribute(mapAttrName); + Assert.assertNotNull(object); + Assert.assertTrue(object instanceof Map); + Map<String, AtlasObjectId> map = (Map<String, AtlasObjectId>)object; + Assert.assertEquals(map.size(), 1); + AtlasObjectId value1Id = map.get("value1"); + Assert.assertNotNull(value1Id); + mapValueGuid = value1Id.getGuid(); + mapOwnerVertex = GraphHelper.getInstance().getVertexForGUID(mapOwnerEntity.getEntity().getGuid()); + object = mapOwnerVertex.getProperty(atlasEdgeLabel.getQualifiedMapKey(), Object.class); + Assert.assertNotNull(object); + } + + // Delete the map value instance. + // This should disconnect the references from the map owner instance. + entityStore.deleteById(mapValueGuid); + assertEntityDeleted(mapValueGuid); + assertTestDisconnectMapReferenceFromClassType(mapOwnerEntity.getEntity().getGuid()); + } + + protected abstract void assertTestDisconnectMapReferenceFromClassType(String mapOwnerGuid) throws Exception; + @Test public void testDeleteByUniqueAttribute() throws Exception { // Create a table entity, with 3 composite column entities @@ -764,6 +874,231 @@ public abstract class AtlasDeleteHandlerV1Test { assertEntityDeleted(colId); } + @Test + public void testDeleteEntitiesWithCompositeMapReference() throws Exception { + // Create instances of MapOwner and MapValue. + // Set MapOwner.map with one entry that references MapValue instance. + AtlasEntity.AtlasEntityWithExtInfo entityDefinition = createMapOwnerAndValueEntities(); + String mapOwnerGuid = entityDefinition.getEntity().getGuid(); + + // Verify MapOwner.map attribute has expected value. + AtlasEntity.AtlasEntityWithExtInfo mapOwnerInstance = entityStore.getById(mapOwnerGuid); + Object object = mapOwnerInstance.getEntity().getAttribute("map"); + Assert.assertNotNull(object); + Assert.assertTrue(object instanceof Map); + Map<String, AtlasObjectId> map = (Map<String, AtlasObjectId>)object; + Assert.assertEquals(map.size(), 1); + AtlasObjectId mapValueInstance = map.get("value1"); + Assert.assertNotNull(mapValueInstance); + String mapValueGuid = mapValueInstance.getGuid(); + String edgeLabel = AtlasGraphUtilsV1.getAttributeEdgeLabel(compositeMapOwnerType, "map"); + String mapEntryLabel = edgeLabel + "." + "value1"; + AtlasEdgeLabel atlasEdgeLabel = new AtlasEdgeLabel(mapEntryLabel); + AtlasVertex mapOwnerVertex = GraphHelper.getInstance().getVertexForGUID(mapOwnerGuid); + object = mapOwnerVertex.getProperty(atlasEdgeLabel.getQualifiedMapKey(), Object.class); + Assert.assertNotNull(object); + + RequestContextV1.clear(); + List<AtlasEntityHeader> deletedEntities = entityStore.deleteById(mapOwnerGuid).getDeletedEntities(); + Assert.assertEquals(deletedEntities.size(), 2); + Assert.assertTrue(extractGuids(deletedEntities).contains(mapOwnerGuid)); + Assert.assertTrue(extractGuids(deletedEntities).contains(mapValueGuid)); + + assertEntityDeleted(mapOwnerGuid); + assertEntityDeleted(mapValueGuid); + } + + @Test + public void testDeleteTargetOfRequiredMapReference() throws Exception { + // Define type for map value. + AtlasEntityDef mapValueDef = + new AtlasEntityDef("RequiredMapValue", "RequiredMapValue_description", "1.0", + Collections.<AtlasStructDef.AtlasAttributeDef>emptyList(), Collections.<String>emptySet()); + + AtlasStructDef.AtlasAttributeDef[] mapOwnerAttributes = new AtlasStructDef.AtlasAttributeDef[]{ + new AtlasStructDef.AtlasAttributeDef("map", "map<string,RequiredMapValue>", + false, + AtlasStructDef.AtlasAttributeDef.Cardinality.SINGLE, 1, 1, + false, false, + Collections.<AtlasStructDef.AtlasConstraintDef>emptyList()) + }; + + AtlasEntityDef mapOwnerDef = + new AtlasEntityDef("RequiredMapOwner", "RequiredMapOwner_description", "1.0", + Arrays.asList(mapOwnerAttributes), Collections.<String>emptySet()); + + AtlasTypesDef typesDef = AtlasTypeUtil.getTypesDef(ImmutableList.<AtlasEnumDef>of(), + ImmutableList.<AtlasStructDef>of(), + ImmutableList.<AtlasClassificationDef>of(), + ImmutableList.<AtlasEntityDef>of(mapValueDef, mapOwnerDef)); + + typeDefStore.createTypesDef(typesDef); + + AtlasEntityType mapOwnerType = typeRegistry.getEntityTypeByName("RequiredMapOwner"); + AtlasEntityType mapValueType = typeRegistry.getEntityTypeByName("RequiredMapValue"); + + // Create instances of RequiredMapOwner and RequiredMapValue. + // Set RequiredMapOwner.map with one entry that references RequiredMapValue instance. + AtlasEntity mapOwnerInstance = new AtlasEntity(mapOwnerType.getTypeName()); + AtlasEntity mapValueInstance = new AtlasEntity(mapValueType.getTypeName()); + mapOwnerInstance.setAttribute("map", Collections.singletonMap("value1", AtlasTypeUtil.getAtlasObjectId(mapValueInstance))); + + AtlasEntity.AtlasEntitiesWithExtInfo entities = new AtlasEntity.AtlasEntitiesWithExtInfo(); + entities.addReferredEntity(mapValueInstance); + entities.addEntity(mapOwnerInstance); + + List<AtlasEntityHeader> createEntitiesResult = entityStore.createOrUpdate(new AtlasEntityStream(entities), false).getCreatedEntities(); + Assert.assertEquals(createEntitiesResult.size(), 2); + List<String> guids = metadataService.getEntityList("RequiredMapOwner"); + Assert.assertEquals(guids.size(), 1); + String mapOwnerGuid = guids.get(0); + guids = metadataService.getEntityList("RequiredMapValue"); + Assert.assertEquals(guids.size(), 1); + String mapValueGuid = guids.get(0); + + // Verify MapOwner.map attribute has expected value. + final AtlasEntity.AtlasEntityWithExtInfo mapOwnerInstance1 = entityStore.getById(mapOwnerGuid); + Object object = mapOwnerInstance1.getEntity().getAttribute("map"); + Assert.assertNotNull(object); + Assert.assertTrue(object instanceof Map); + Map<String, AtlasObjectId> map = (Map<String, AtlasObjectId>)object; + Assert.assertEquals(map.size(), 1); + AtlasObjectId mapValueInstance1 = map.get("value1"); + Assert.assertNotNull(mapValueInstance1); + Assert.assertEquals(mapValueInstance1.getGuid(), mapValueGuid); + String edgeLabel = AtlasGraphUtilsV1.getAttributeEdgeLabel(mapOwnerType, "map"); + String mapEntryLabel = edgeLabel + "." + "value1"; + AtlasEdgeLabel atlasEdgeLabel = new AtlasEdgeLabel(mapEntryLabel); + AtlasVertex mapOwnerVertex = GraphHelper.getInstance().getVertexForGUID(mapOwnerGuid); + object = mapOwnerVertex.getProperty(atlasEdgeLabel.getQualifiedMapKey(), Object.class); + Assert.assertNotNull(object); + + // Verify deleting the target of required map attribute throws a AtlasBaseException. + try { + entityStore.deleteById(mapValueGuid); + Assert.fail(AtlasBaseException.class.getSimpleName() + " was expected but none thrown."); + } + catch (Exception e) { + verifyExceptionThrown(e, AtlasBaseException.class); + } + } + + @Test + public void testLowerBoundsIgnoredWhenDeletingCompositeEntitesOwnedByMap() throws Exception { + // Define MapValueReferencer type with required reference to CompositeMapValue. + AtlasStructDef.AtlasAttributeDef[] mapValueAttributes = new AtlasStructDef.AtlasAttributeDef[]{ + new AtlasStructDef.AtlasAttributeDef("refToMapValue", "CompositeMapValue", + false, + AtlasStructDef.AtlasAttributeDef.Cardinality.SINGLE, 1, 1, + false, false, + Collections.<AtlasStructDef.AtlasConstraintDef>emptyList()) + }; + + AtlasEntityDef mapValueDef = + new AtlasEntityDef("MapValueReferencer", "RequiredMapValue_description", "1.0", + Arrays.asList(mapValueAttributes), Collections.<String>emptySet()); + + + AtlasStructDef.AtlasAttributeDef[] mapContainerAttributes = new AtlasStructDef.AtlasAttributeDef[]{ + new AtlasStructDef.AtlasAttributeDef("requiredMap", "map<string,MapValueReferencer>", + false, + AtlasStructDef.AtlasAttributeDef.Cardinality.SINGLE, 1, 1, + false, false, + new ArrayList<AtlasStructDef.AtlasConstraintDef>() {{ + add(new AtlasStructDef.AtlasConstraintDef(AtlasStructDef.AtlasConstraintDef.CONSTRAINT_TYPE_OWNED_REF)); + }}) + }; + + AtlasEntityDef mapContainerDef = + new AtlasEntityDef("MapValueReferencerContainer", "MapValueReferencerContainer_description", "1.0", + Arrays.asList(mapContainerAttributes), Collections.<String>emptySet()); + + + AtlasTypesDef typesDef = AtlasTypeUtil.getTypesDef(ImmutableList.<AtlasEnumDef>of(), + ImmutableList.<AtlasStructDef>of(), + ImmutableList.<AtlasClassificationDef>of(), + ImmutableList.<AtlasEntityDef>of(mapValueDef, mapContainerDef)); + + typeDefStore.createTypesDef(typesDef); + + // Create instances of CompositeMapOwner and CompositeMapValue. + // Set MapOwner.map with one entry that references MapValue instance. + AtlasEntity.AtlasEntityWithExtInfo entityDefinition = createMapOwnerAndValueEntities(); + String mapOwnerGuid = entityDefinition.getEntity().getGuid(); + + // Verify MapOwner.map attribute has expected value. + ITypedReferenceableInstance mapOwnerInstance = metadataService.getEntityDefinition(mapOwnerGuid); + Object object = mapOwnerInstance.get("map"); + Assert.assertNotNull(object); + Assert.assertTrue(object instanceof Map); + Map<String, ITypedReferenceableInstance> map = (Map<String, ITypedReferenceableInstance>)object; + Assert.assertEquals(map.size(), 1); + ITypedReferenceableInstance mapValueInstance = map.get("value1"); + Assert.assertNotNull(mapValueInstance); + String mapValueGuid = mapValueInstance.getId()._getId(); + + // Create instance of MapValueReferencerContainer + RequestContextV1.clear(); + AtlasEntity mapValueReferencer = new AtlasEntity(mapValueDef.getName()); + mapValueReferencer.setAttribute("refToMapValue", new AtlasObjectId(mapValueInstance.getId()._getId(), mapValueInstance.getTypeName())); + AtlasEntity.AtlasEntitiesWithExtInfo entities = new AtlasEntity.AtlasEntitiesWithExtInfo(); + entities.addEntity(mapValueReferencer); + + List<AtlasEntityHeader> createEntitiesResult = entityStore.createOrUpdate(new AtlasEntityStream(entities), false).getCreatedEntities(); + Assert.assertEquals(createEntitiesResult.size(), 1); + + // Create instance of MapValueReferencer, and update mapValueReferencerContainer + // to reference it. + AtlasEntity mapValueReferenceContainer = new AtlasEntity(mapContainerDef.getName()); + entities = new AtlasEntity.AtlasEntitiesWithExtInfo(); + entities.addEntity(mapValueReferenceContainer); + entities.addReferredEntity(mapValueReferencer); + mapValueReferenceContainer.setAttribute("requiredMap", Collections.singletonMap("value1", AtlasTypeUtil.getAtlasObjectId(mapValueReferencer))); + + + RequestContextV1.clear(); + EntityMutationResponse updateEntitiesResult = entityStore.createOrUpdate(new AtlasEntityStream(entities), false); + + String mapValueReferencerContainerGuid = updateEntitiesResult.getCreatedEntitiesByTypeName("MapValueReferencerContainer").get(0).getGuid(); + String mapValueReferencerGuid = updateEntitiesResult.getUpdatedEntitiesByTypeName("MapValueReferencer").get(0).getGuid(); + + Assert.assertEquals(updateEntitiesResult.getCreatedEntities().size(), 1); + Assert.assertEquals(updateEntitiesResult.getUpdatedEntities().size(), 1); + Assert.assertEquals(updateEntitiesResult.getUpdatedEntities().get(0).getGuid(), mapValueReferencerGuid); + + + // Delete map owner and map referencer container. A total of 4 entities should be deleted, + // including the composite entities. The lower bound constraint on MapValueReferencer.refToMapValue + // should not be enforced on the composite MapValueReferencer since it is being deleted. + EntityMutationResponse deleteEntitiesResult = entityStore.deleteByIds(Arrays.asList(mapOwnerGuid, mapValueReferencerContainerGuid)); + Assert.assertEquals(deleteEntitiesResult.getDeletedEntities().size(), 4); + Assert.assertTrue(extractGuids(deleteEntitiesResult.getDeletedEntities()).containsAll( + Arrays.asList(mapOwnerGuid, mapValueGuid, mapValueReferencerContainerGuid, mapValueReferencerGuid))); + } + + private AtlasEntity.AtlasEntityWithExtInfo createMapOwnerAndValueEntities() + throws AtlasException, AtlasBaseException { + + final AtlasEntity mapOwnerInstance = new AtlasEntity(compositeMapOwnerType.getTypeName()); + mapOwnerInstance.setAttribute(NAME, TestUtils.randomString()); + AtlasEntity mapValueInstance = new AtlasEntity(compositeMapValueType.getTypeName()); + mapValueInstance.setAttribute(NAME, TestUtils.randomString()); + mapOwnerInstance.setAttribute("map", Collections.singletonMap("value1", AtlasTypeUtil.getAtlasObjectId(mapValueInstance))); + + AtlasEntity.AtlasEntitiesWithExtInfo entities = new AtlasEntity.AtlasEntitiesWithExtInfo(); + entities.addReferredEntity(mapValueInstance); + entities.addEntity(mapOwnerInstance); + + List<AtlasEntityHeader> createEntitiesResult = entityStore.createOrUpdate(new AtlasEntityStream(entities), false).getCreatedEntities(); + Assert.assertEquals(createEntitiesResult.size(), 2); + AtlasEntity.AtlasEntityWithExtInfo entityDefinition = entityStore.getByUniqueAttributes(compositeMapOwnerType, + new HashMap<String, Object>() {{ + put(NAME, mapOwnerInstance.getAttribute(NAME)); + }}); + return entityDefinition; + } + + protected abstract void assertTestDisconnectUnidirectionalArrayReferenceFromStructAndTraitTypes( String structContainerGuid) throws Exception; @@ -779,4 +1114,29 @@ public abstract class AtlasDeleteHandlerV1Test { return list; } + /** + * Search exception cause chain for specified exception. + * + * @param thrown root of thrown exception chain + * @param expected class of expected exception + */ + private void verifyExceptionThrown(Exception thrown, Class expected) { + + boolean exceptionFound = false; + Throwable cause = thrown; + while (cause != null) { + if (expected.isInstance(cause)) { + // good + exceptionFound = true; + break; + } + else { + cause = cause.getCause(); + } + } + if (!exceptionFound) { + Assert.fail(expected.getSimpleName() + " was expected but not thrown", thrown); + } + } + } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/8bb31fdf/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1Test.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1Test.java b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1Test.java index 845bbb4..94d313f 100644 --- a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1Test.java +++ b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1Test.java @@ -261,7 +261,6 @@ public class AtlasEntityStoreV1Test { partsMap.put("part0", new AtlasStruct(TestUtils.PARTITION_STRUCT_TYPE, TestUtilsV2.NAME, "test")); tableEntity.setAttribute("partitionsMap", partsMap); - entitiesInfo.addReferredEntity(tableEntity); init(); EntityMutationResponse response = entityStore.createOrUpdate(new AtlasEntityStream(entitiesInfo), false); @@ -365,10 +364,14 @@ public class AtlasEntityStoreV1Test { AtlasEntityHeader tableDefinition6 = response.getFirstUpdatedEntityByTypeName(TABLE_TYPE); validateEntity(entitiesInfo, getEntityFromStore(tableDefinition6)); + Assert.assertEquals(entityStore.getById(col0.getGuid()).getEntity().getStatus(), AtlasEntity.Status.ACTIVE); + Assert.assertEquals(entityStore.getById(col1.getGuid()).getEntity().getStatus(), AtlasEntity.Status.ACTIVE); + //Drop the first key and change the class type as well to col0 columnsMap.clear(); columnsMap.put("col0", AtlasTypeUtil.getAtlasObjectId(col0)); init(); + response = entityStore.createOrUpdate(new AtlasEntityStream(entitiesInfo), false); AtlasEntityHeader tableDefinition7 = response.getFirstUpdatedEntityByTypeName(TABLE_TYPE); validateEntity(entitiesInfo, getEntityFromStore(tableDefinition7)); @@ -494,25 +497,6 @@ public class AtlasEntityStoreV1Test { validateEntity(entitiesInfo, getEntityFromStore(updatedTable)); } -// private AtlasEntity clearSubOrdinates(List<AtlasObjectId> employees, int index) { -// -// AtlasEntity ret = null; -// AtlasObjectId employee = employees.get(index); -// AtlasEntity subOrdClone = new ArrayList<>(subOrdinates); -// ret = subOrdClone.remove(index); -// -// employees.get(index).setAttribute("subordinates", subOrdClone); -// return ret; -// } -// -// private int addSubordinate(AtlasEntity manager, AtlasEntity employee) { -// List<AtlasEntity> subOrdinates = (List<AtlasEntity>) manager.getAttribute("subordinates"); -// subOrdinates.add(employee); -// -// manager.setAttribute("subordinates", subOrdinates); -// return subOrdinates.size() - 1; -// } - @Test(dependsOnMethods = "testCreate") public void testClassUpdate() throws Exception { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/8bb31fdf/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/HardDeleteHandlerV1Test.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/HardDeleteHandlerV1Test.java b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/HardDeleteHandlerV1Test.java index fd685c6..4403b12 100644 --- a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/HardDeleteHandlerV1Test.java +++ b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/HardDeleteHandlerV1Test.java @@ -26,6 +26,7 @@ import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.model.instance.AtlasObjectId; import org.apache.atlas.repository.Constants; +import org.apache.atlas.repository.graph.GraphHelper; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.typesystem.IStruct; @@ -146,6 +147,23 @@ public class HardDeleteHandlerV1Test extends AtlasDeleteHandlerV1Test { } } + protected void assertTestDisconnectMapReferenceFromClassType(final String mapOwnerGuid) throws Exception { + // Verify map references from mapOwner were disconnected. + AtlasEntity.AtlasEntityWithExtInfo mapOwnerInstance = entityStore.getById(mapOwnerGuid); + Map<String, AtlasObjectId> map = + (Map<String, AtlasObjectId>) mapOwnerInstance.getEntity().getAttribute("map"); + Assert.assertNull(map); + Map<String, AtlasObjectId> biMap = + (Map<String, AtlasObjectId>) mapOwnerInstance.getEntity().getAttribute("biMap"); + Assert.assertNull(biMap); + + AtlasVertex mapOwnerVertex = GraphHelper.getInstance().getVertexForGUID(mapOwnerGuid); + Object object = mapOwnerVertex.getProperty("MapOwner.map.value1", String.class); + assertNull(object); + object = mapOwnerVertex.getProperty("MapOwner.biMap.value1", String.class); + assertNull(object); + } + @Override protected void assertTestDisconnectUnidirectionalArrayReferenceFromStructAndTraitTypes(String structContainerGuid) throws Exception { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/8bb31fdf/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/SoftDeleteHandlerV1Test.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/SoftDeleteHandlerV1Test.java b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/SoftDeleteHandlerV1Test.java index eba9c77..8935faf 100644 --- a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/SoftDeleteHandlerV1Test.java +++ b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/SoftDeleteHandlerV1Test.java @@ -183,6 +183,19 @@ public class SoftDeleteHandlerV1Test extends AtlasDeleteHandlerV1Test { } @Override + protected void assertTestDisconnectMapReferenceFromClassType(final String mapOwnerGuid) throws Exception { + AtlasEntity.AtlasEntityWithExtInfo mapOwnerInstance = entityStore.getById(mapOwnerGuid); + Map<String, AtlasObjectId> map = + (Map<String, AtlasObjectId>) mapOwnerInstance.getEntity().getAttribute("map"); + assertNotNull(map); + assertEquals(map.size(), 1); + Map<String, AtlasObjectId> biMap = + (Map<String, AtlasObjectId>) mapOwnerInstance.getEntity().getAttribute("biMap"); + assertNotNull(biMap); + assertEquals(biMap.size(), 1); + } + + @Override protected void assertTestDisconnectUnidirectionalArrayReferenceFromStructAndTraitTypes(final String structContainerGuid) throws Exception { // Verify that the unidirectional references from the struct and trait instances // to the deleted entities were not disconnected.
