Repository: atlas Updated Branches: refs/heads/branch-0.8 6641a17b1 -> 3021027c9
ATLAS-2813: SoftRef implementation. Part - 2 Project: http://git-wip-us.apache.org/repos/asf/atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/3021027c Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/3021027c Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/3021027c Branch: refs/heads/branch-0.8 Commit: 3021027c999c56767e65f410492a378f0e5b3a1f Parents: 6641a17 Author: Ashutosh Mestry <[email protected]> Authored: Thu Aug 16 11:17:00 2018 -0700 Committer: Ashutosh Mestry <[email protected]> Committed: Thu Aug 16 11:17:00 2018 -0700 ---------------------------------------------------------------------- .../graph/v1/AttributeMutationContext.java | 4 - .../store/graph/v1/EntityGraphMapper.java | 30 ++++--- .../store/graph/v1/EntityGraphRetriever.java | 35 +++++--- .../store/graph/v1/SoftReferenceTest.java | 84 ++++++++++++++++++-- 4 files changed, 125 insertions(+), 28 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/atlas/blob/3021027c/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AttributeMutationContext.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AttributeMutationContext.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AttributeMutationContext.java index b32c092..b6d82dd 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AttributeMutationContext.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AttributeMutationContext.java @@ -126,10 +126,6 @@ public class AttributeMutationContext { return value; } - public void setValue(Object value) { - this.value = value; - } - public String getVertexProperty() { return vertexProperty; } public AtlasVertex getReferringVertex() { return referringVertex; } http://git-wip-us.apache.org/repos/asf/atlas/blob/3021027c/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java index 3811919..fcdd379 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java @@ -67,6 +67,7 @@ import static org.apache.atlas.repository.graph.GraphHelper.string; @Component public class EntityGraphMapper { private static final Logger LOG = LoggerFactory.getLogger(EntityGraphMapper.class); + private static final String SOFT_REF_FORMAT = "%s:%s"; private final GraphHelper graphHelper = GraphHelper.getInstance(); private final AtlasGraph graph; @@ -342,16 +343,24 @@ public class EntityGraphMapper { } private Object mapSoftRefValue(AttributeMutationContext ctx, EntityMutationContext context) { - if(ctx.getValue() == null || !(ctx.getValue() instanceof AtlasObjectId)) { + if(ctx.getValue() != null && !(ctx.getValue() instanceof AtlasObjectId)) { + LOG.warn("mapSoftRefValue: Was expecting AtlasObjectId, but found: {}", ctx.getValue().getClass()); return null; } - AtlasObjectId objectId = (AtlasObjectId) ctx.getValue(); - String resolvedGuid = context.getGuidAssignments().get(objectId.getGuid()); - String softRefValue = String.format("%s:%s", objectId.getTypeName(), resolvedGuid); + String softRefValue = null; + if(ctx.getValue() != null) { + AtlasObjectId objectId = (AtlasObjectId) ctx.getValue(); + String resolvedGuid = AtlasTypeUtil.isUnAssignedGuid(objectId.getGuid()) + ? context.getGuidAssignments().get(objectId.getGuid()) + : objectId.getGuid(); - ctx.setValue(softRefValue); - return mapPrimitiveValue(ctx); + softRefValue = String.format(SOFT_REF_FORMAT, objectId.getTypeName(), resolvedGuid); + } + + AtlasGraphUtilsV1.setProperty(ctx.getReferringVertex(), ctx.getVertexProperty(), softRefValue); + + return softRefValue; } private void addInverseReference(AttributeMutationContext ctx, AtlasAttribute inverseAttribute, AtlasEdge edge) throws AtlasBaseException { @@ -588,8 +597,8 @@ public class EntityGraphMapper { AtlasArrayType arrType = (AtlasArrayType) attribute.getAttributeType(); AtlasType elementType = arrType.getElementType(); List<Object> currentElements = getArrayElementsProperty(elementType, ctx.getReferringVertex(), ctx.getVertexProperty()); - boolean isReference = AtlasGraphUtilsV1.isReference(elementType); - boolean isSoftReference = ctx.getAttribute().getAttributeDef().isSoftReferenced(); + boolean isReference = AtlasGraphUtilsV1.isReference(elementType); + boolean isSoftReference = ctx.getAttribute().getAttributeDef().isSoftReferenced(); AtlasAttribute inverseRefAttribute = attribute.getInverseRefAttribute(); Cardinality cardinality = attribute.getAttributeDef().getCardinality(); List<Object> newElementsCreated = new ArrayList<>(); @@ -614,7 +623,7 @@ public class EntityGraphMapper { } } - if (isReference) { + if (isReference && !isSoftReference) { List<AtlasEdge> additionalEdges = removeUnusedArrayEntries(attribute, (List) currentElements, (List) newElementsCreated); newElementsCreated.addAll(additionalEdges); } @@ -777,13 +786,14 @@ public class EntityGraphMapper { private Map<String, Object> removeUnusedMapEntries(AtlasAttribute attribute, AtlasVertex vertex, String propertyName, Map<String, Object> currentMap, Map<String, Object> newMap) throws AtlasException, AtlasBaseException { + boolean isSoftRef = attribute.getAttributeDef().isSoftReferenced(); AtlasMapType mapType = (AtlasMapType) attribute.getAttributeType(); Map<String, Object> additionalMap = new HashMap<>(); for (String currentKey : currentMap.keySet()) { boolean shouldDeleteKey = !newMap.containsKey(currentKey); - if (AtlasGraphUtilsV1.isReference(mapType.getValueType())) { + if (AtlasGraphUtilsV1.isReference(mapType.getValueType()) && !isSoftRef) { //Delete the edge reference if its not part of new edges created/updated AtlasEdge currentEdge = (AtlasEdge)currentMap.get(currentKey); http://git-wip-us.apache.org/repos/asf/atlas/blob/3021027c/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java index 81b553e..f99b4a9 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java @@ -478,7 +478,7 @@ public final class EntityGraphRetriever { return ret; } - private Object mapVertexToMapForSoftRef(AtlasVertex entityVertex, String propertyName) { + private Map<String, AtlasObjectId> mapVertexToMapForSoftRef(AtlasVertex entityVertex, String propertyName) { List mapKeys = entityVertex.getListProperty(propertyName); if (CollectionUtils.isEmpty(mapKeys)) { return null; @@ -488,12 +488,12 @@ public final class EntityGraphRetriever { LOG.debug("Mapping map attribute {} for vertex {}", propertyName, entityVertex); } - Map<String, Object> ret = new HashMap<>(mapKeys.size()); + Map<String, AtlasObjectId> ret = new HashMap<>(mapKeys.size()); for (Object mapKey : mapKeys) { final String keyPropertyName = String.format(MAP_VALUE_FORMAT, propertyName, mapKey); - Object mapValue = mapVertexToObjectIdForSoftRef(entityVertex, keyPropertyName); + AtlasObjectId mapValue = mapVertexToObjectIdForSoftRef(entityVertex, keyPropertyName); if (mapValue != null) { ret.put((String) mapKey, mapValue); } @@ -502,29 +502,46 @@ public final class EntityGraphRetriever { return ret; } - private Object mapVertexToArrayForSoftRef(AtlasVertex entityVertex, String propertyName) { - List<AtlasObjectId> objectIds = new ArrayList<>(); + private List<AtlasObjectId> mapVertexToArrayForSoftRef(AtlasVertex entityVertex, String propertyName) { List list = entityVertex.getListProperty(propertyName); + if (CollectionUtils.isEmpty(list)) { + return null; + } + + List<AtlasObjectId> objectIds = new ArrayList<>(); for (Object o : list) { - if(!(o instanceof String)) { + if (!(o instanceof String)) { continue; } AtlasObjectId objectId = getAtlasObjectIdFromSoftRefFormat((String) o); + if(objectId == null) { + continue; + } + objectIds.add(objectId); } return objectIds; } - private Object mapVertexToObjectIdForSoftRef(AtlasVertex entityVertex, String vertexPropertyName) { - Object rawValue = GraphHelper.getSingleValuedProperty(entityVertex, vertexPropertyName, String.class); - return getAtlasObjectIdFromSoftRefFormat((String) rawValue); + private AtlasObjectId mapVertexToObjectIdForSoftRef(AtlasVertex entityVertex, String vertexPropertyName) { + String rawValue = GraphHelper.getSingleValuedProperty(entityVertex, vertexPropertyName, String.class); + if(StringUtils.isEmpty(rawValue)) { + return null; + } + + return getAtlasObjectIdFromSoftRefFormat(rawValue); } private AtlasObjectId getAtlasObjectIdFromSoftRefFormat(String rawValue) { + if(StringUtils.isEmpty(rawValue)) { + return null; + } + String[] objectIdParts = StringUtils.split(rawValue, SOFT_REFERENCE_FORMAT_SEPERATOR); if(objectIdParts.length < 2) { + LOG.warn("Expecting value to be formatted for softRef. Instead found: {}", rawValue); return null; } http://git-wip-us.apache.org/repos/asf/atlas/blob/3021027c/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/SoftReferenceTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/SoftReferenceTest.java b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/SoftReferenceTest.java index 665f17a..95b99f9 100644 --- a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/SoftReferenceTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/SoftReferenceTest.java @@ -30,6 +30,7 @@ import org.apache.atlas.repository.graphdb.AtlasEdgeDirection; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.store.graph.AtlasEntityStore; import org.apache.atlas.store.AtlasTypeDefStore; +import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasType; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.utils.TestResourceFileUtils; @@ -69,6 +70,8 @@ public class SoftReferenceTest { private AtlasEntityStore entityStore; private AtlasType dbType; + private String dbGuid; + private String storageGuid; @Test public void typeCreationFromFile() throws IOException, AtlasBaseException { @@ -92,16 +95,87 @@ public class SoftReferenceTest { @Test(dependsOnMethods = "typeCreationFromFile") public void entityCreationUsingSoftRef() throws IOException, AtlasBaseException { + final int EXPECTED_ENTITY_COUNT = 6; AtlasEntity.AtlasEntityWithExtInfo dbEntity = AtlasType.fromJson( TestResourceFileUtils.getJson(RDBMS_DB_FILE), AtlasEntity.AtlasEntityWithExtInfo.class); EntityMutationResponse response = entityStore.createOrUpdate(new AtlasEntityStream(dbEntity), false); + assertNotNull(response); - assertTrue(response.getCreatedEntities().size() == 6); + assertTrue(response.getCreatedEntities().size() == EXPECTED_ENTITY_COUNT); assertGraphStructure(response.getCreatedEntities().get(0).getGuid(), response.getCreatedEntities().get(1).getGuid(), RDBMS_SD_PROPERTY); + + dbGuid = response.getCreatedEntities().get(0).getGuid(); + storageGuid = response.getCreatedEntities().get(1).getGuid(); + } + + @Test(dependsOnMethods = "entityCreationUsingSoftRef") + public void deletetingCollections() throws AtlasBaseException { + AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo = entityStore.getById(dbGuid); + + assertNotNull(entityWithExtInfo); + List list = (List)entityWithExtInfo.getEntity().getAttribute(RDBMS_DB_TABLES_PROPERTY); + list.remove(1); + + Map map = (Map) entityWithExtInfo.getEntity().getAttribute(RDBMS_DB_REGIONS_PROPERTY); + map.remove("east"); + + EntityMutationResponse response = entityStore.createOrUpdate(new AtlasEntityStream(entityWithExtInfo), true); + assertNotNull(response); + assertTrue(response.getPartialUpdatedEntities().size() > 0); + assertAttribute(dbGuid, storageGuid, 1, 1); + } + + @Test(dependsOnMethods = "deletetingCollections") + public void addingCollections() throws AtlasBaseException { + AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo = entityStore.getById(dbGuid); + + assertNotNull(entityWithExtInfo); + addNewTables(entityWithExtInfo); + addNewRegions(entityWithExtInfo); + + EntityMutationResponse response = entityStore.createOrUpdate(new AtlasEntityStream(entityWithExtInfo), true); + assertNotNull(response); + assertTrue(response.getPartialUpdatedEntities().size() > 0); + assertAttribute(dbGuid, storageGuid, 3, 3); } + private void addNewRegions(AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo) throws AtlasBaseException { + Map map = (Map) entityWithExtInfo.getEntity().getAttribute(RDBMS_DB_REGIONS_PROPERTY); + + AtlasEntity region1 = getDefaultTableEntity("r1"); + AtlasEntity region2 = getDefaultTableEntity("r2"); + + map.put("north", new AtlasObjectId(region1.getGuid(), region1.getTypeName())); + map.put("south", new AtlasObjectId(region2.getGuid(), region2.getTypeName())); + + entityWithExtInfo.addReferredEntity(region1); + entityWithExtInfo.addReferredEntity(region2); + } + + private void addNewTables(AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo) throws AtlasBaseException { + List list = (List)entityWithExtInfo.getEntity().getAttribute(RDBMS_DB_TABLES_PROPERTY); + AtlasEntity table1 = getDefaultTableEntity("newTable-1"); + AtlasEntity table2 = getDefaultTableEntity("newTable-2"); + + entityWithExtInfo.addReferredEntity(table1); + entityWithExtInfo.addReferredEntity(table2); + + list.add(new AtlasObjectId(table1.getGuid(), table1.getTypeName())); + list.add(new AtlasObjectId(table2.getGuid(), table2.getTypeName())); + } + + private AtlasEntity getDefaultTableEntity(String name) throws AtlasBaseException { + AtlasEntityType type = (AtlasEntityType) typeRegistry.getType(TYPE_RDBMS_TABLES); + + AtlasEntity ret = type.createDefaultValue(); + ret.setAttribute("name", name); + + return ret; + } + + private void assertGraphStructure(String dbGuid, String storageGuid, String propertyName) throws AtlasBaseException { AtlasVertex vertex = AtlasGraphUtilsV1.findByGuid(dbGuid); Iterator<AtlasEdge> edgesOut = vertex.getEdges(AtlasEdgeDirection.OUT).iterator(); @@ -110,13 +184,13 @@ public class SoftReferenceTest { String sd = AtlasGraphUtilsV1.getProperty(vertex, propertyName, String.class); assertNotNull(sd); - assertAttribute(dbGuid, storageGuid); + assertAttribute(dbGuid, storageGuid, 2, 2); assertFalse(edgesOut.hasNext()); assertFalse(edgesIn.hasNext()); assertNotNull(vertex); } - private void assertAttribute(String dbGuid, String storageGuid) throws AtlasBaseException { + private void assertAttribute(String dbGuid, String storageGuid, int expectedTableCount, int expectedRegionCount) throws AtlasBaseException { AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo = entityStore.getById(dbGuid); AtlasEntity entity = entityWithExtInfo.getEntity(); @@ -125,8 +199,8 @@ public class SoftReferenceTest { assertEquals(((AtlasObjectId) val).getTypeName(), TYPE_RDBMS_STORAGE); assertEquals(((AtlasObjectId) val).getGuid(), storageGuid); assertNotNull(entity.getAttribute(RDBMS_DB_TABLES_PROPERTY)); - assertEquals(((List) entity.getAttribute(RDBMS_DB_TABLES_PROPERTY)).size(), 2); + assertEquals(((List) entity.getAttribute(RDBMS_DB_TABLES_PROPERTY)).size(), expectedTableCount); assertNotNull(entity.getAttribute(RDBMS_DB_REGIONS_PROPERTY)); - assertEquals(((Map) entity.getAttribute(RDBMS_DB_REGIONS_PROPERTY)).size(), 2); + assertEquals(((Map) entity.getAttribute(RDBMS_DB_REGIONS_PROPERTY)).size(), expectedRegionCount); } }
