Repository: atlas
Updated Branches:
  refs/heads/branch-0.8 daabed131 -> a36b27ae7


ATLAS-2965: Duplicate entities are created when when same qualifiedName is 
given but different guids


Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/a36b27ae
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/a36b27ae
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/a36b27ae

Branch: refs/heads/branch-0.8
Commit: a36b27ae798d97b5b30238d13b8d63a3652a0260
Parents: daabed1
Author: Sarath Subramanian <ssubraman...@hortonworks.com>
Authored: Thu Nov 15 15:43:29 2018 -0800
Committer: Sarath Subramanian <ssubraman...@hortonworks.com>
Committed: Thu Nov 15 15:43:29 2018 -0800

----------------------------------------------------------------------
 .../test/java/org/apache/atlas/TestUtilsV2.java | 33 ++++++++++++++++++++
 .../store/graph/v1/AtlasEntityStoreV1.java      | 29 ++++++++++++++++-
 .../store/graph/v1/EntityGraphMapper.java       | 24 +++++++++++++-
 .../store/graph/v1/AtlasEntityStoreV1Test.java  | 23 ++++++++++++++
 4 files changed, 107 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/atlas/blob/a36b27ae/intg/src/test/java/org/apache/atlas/TestUtilsV2.java
----------------------------------------------------------------------
diff --git a/intg/src/test/java/org/apache/atlas/TestUtilsV2.java 
b/intg/src/test/java/org/apache/atlas/TestUtilsV2.java
index 8470054..18f8b25 100755
--- a/intg/src/test/java/org/apache/atlas/TestUtilsV2.java
+++ b/intg/src/test/java/org/apache/atlas/TestUtilsV2.java
@@ -1057,6 +1057,39 @@ public final class TestUtilsV2 {
         return ret;
     }
 
+    public static AtlasEntityWithExtInfo 
createTableEntityDuplicatesV2(AtlasEntity dbEntity) {
+        AtlasEntity tblEntity = new AtlasEntity(TABLE_TYPE);
+
+        tblEntity.setAttribute(NAME, RandomStringUtils.randomAlphanumeric(10));
+        tblEntity.setAttribute("description", "random table");
+        tblEntity.setAttribute("type", "type");
+        tblEntity.setAttribute("tableType", "MANAGED");
+        tblEntity.setAttribute("database", 
AtlasTypeUtil.getAtlasObjectId(dbEntity));
+
+        AtlasEntity col1 = createColumnEntity(tblEntity);
+        col1.setAttribute(NAME, "col1");
+
+        AtlasEntity col2 = createColumnEntity(tblEntity);
+        col2.setAttribute(NAME, "col1");
+
+        AtlasEntity col3 = createColumnEntity(tblEntity);
+        col3.setAttribute(NAME, "col1");
+
+        // all 3 columns have different guid but same typeName and unique 
attributes
+        tblEntity.setAttribute(COLUMNS_ATTR_NAME, 
Arrays.asList(AtlasTypeUtil.getAtlasObjectId(col1),
+                                                                
AtlasTypeUtil.getAtlasObjectId(col2),
+                                                                
AtlasTypeUtil.getAtlasObjectId(col3)));
+
+        AtlasEntityWithExtInfo ret = new AtlasEntityWithExtInfo(tblEntity);
+
+        ret.addReferredEntity(dbEntity);
+        ret.addReferredEntity(col1);
+        ret.addReferredEntity(col2);
+        ret.addReferredEntity(col3);
+
+        return ret;
+    }
+
     public static AtlasEntity createColumnEntity(AtlasEntity tableEntity) {
         return createColumnEntity(tableEntity, "col" + seq.addAndGet(1));
     }

http://git-wip-us.apache.org/repos/asf/atlas/blob/a36b27ae/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1.java
----------------------------------------------------------------------
diff --git 
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1.java
 
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1.java
index c98c8c4..f578ded 100644
--- 
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1.java
+++ 
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1.java
@@ -612,10 +612,10 @@ public class AtlasEntityStoreV1 implements 
AtlasEntityStore {
         RequestContextV1            requestContext   = RequestContextV1.get();
 
         for (String guid : discoveryContext.getReferencedGuids()) {
-            AtlasVertex vertex = 
discoveryContext.getResolvedEntityVertex(guid);
             AtlasEntity entity = entityStream.getByGuid(guid);
 
             if (entity != null) {
+                AtlasVertex vertex = getResolvedEntityVertex(discoveryContext, 
entity);
 
                 if (vertex != null) {
                     // entity would be null if guid is not in the stream but 
referenced by an entity in the stream
@@ -650,6 +650,8 @@ public class AtlasEntityStoreV1 implements AtlasEntityStore 
{
 
                     discoveryContext.addResolvedGuid(guid, vertex);
 
+                    
discoveryContext.addResolvedIdByUniqAttribs(getAtlasObjectId(entity), vertex);
+
                     String generatedGuid = 
AtlasGraphUtilsV1.getIdFromVertex(vertex);
 
                     entity.setGuid(generatedGuid);
@@ -669,6 +671,31 @@ public class AtlasEntityStoreV1 implements 
AtlasEntityStore {
         return context;
     }
 
+    private AtlasVertex getResolvedEntityVertex(EntityGraphDiscoveryContext 
context, AtlasEntity entity) throws AtlasBaseException {
+        AtlasVertex ret = context.getResolvedEntityVertex(entity.getGuid());
+
+        if (ret == null) {
+            ret = context.getResolvedEntityVertex(getAtlasObjectId(entity));
+
+            if (ret != null) {
+                context.addResolvedGuid(entity.getGuid(), ret);
+            }
+        }
+
+        return ret;
+    }
+
+    private AtlasObjectId getAtlasObjectId(AtlasEntity entity) {
+        AtlasObjectId ret = entityGraphMapper.toAtlasObjectId(entity);
+
+        if (ret != null && MapUtils.isNotEmpty(ret.getUniqueAttributes())) {
+            // if uniqueAttributes is not empty, reset guid to null.
+            ret.setGuid(null);
+        }
+
+        return ret;
+    }
+
     private EntityMutationResponse deleteVertices(Collection<AtlasVertex> 
deletionCandidates) throws AtlasBaseException {
         EntityMutationResponse response = new EntityMutationResponse();
         deleteDelegate.getHandlerV1().deleteEntities(deletionCandidates);

http://git-wip-us.apache.org/repos/asf/atlas/blob/a36b27ae/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java
----------------------------------------------------------------------
diff --git 
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java
 
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java
index df987be..e818ed5 100644
--- 
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java
+++ 
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java
@@ -949,7 +949,8 @@ public class EntityGraphMapper {
 
     private void setArrayElementsProperty(AtlasType elementType, boolean 
isSoftReference, AtlasVertex vertex, String vertexPropertyName, List<Object> 
values) {
         if (AtlasGraphUtilsV1.isReference(elementType) && !isSoftReference) {
-            vertex.setPropertyFromElementsIds(vertexPropertyName, (List) 
values);
+            // avoid storing duplicate edge ids in vertex property
+            vertex.setPropertyFromElementsIds(vertexPropertyName, 
getUniqueElementsList(values));
         }
         else {
             AtlasGraphUtilsV1.setEncodedProperty(vertex, vertexPropertyName, 
values);
@@ -1146,6 +1147,27 @@ public class EntityGraphMapper {
         return ret;
     }
 
+    public AtlasObjectId toAtlasObjectId(AtlasEntity entity) {
+        AtlasObjectId   ret        = null;
+        AtlasEntityType entityType = 
typeRegistry.getEntityTypeByName(entity.getTypeName());
+
+        if (entityType != null) {
+            Map<String, Object> uniqueAttributes = new HashMap<>();
+
+            for (String attributeName : 
entityType.getUniqAttributes().keySet()) {
+                Object attrValue = entity.getAttribute(attributeName);
+
+                if (attrValue != null) {
+                    uniqueAttributes.put(attributeName, attrValue);
+                }
+            }
+
+            ret = new AtlasObjectId(entity.getGuid(), entity.getTypeName(), 
uniqueAttributes);
+        }
+
+        return ret;
+    }
+
     public static String getSoftRefFormattedValue(AtlasObjectId objectId) {
         return getSoftRefFormattedString(objectId.getTypeName(), 
objectId.getGuid());
     }

http://git-wip-us.apache.org/repos/asf/atlas/blob/a36b27ae/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1Test.java
----------------------------------------------------------------------
diff --git 
a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1Test.java
 
b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1Test.java
index b991fd3..89b0cea 100644
--- 
a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1Test.java
+++ 
b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1Test.java
@@ -70,8 +70,10 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import static org.apache.atlas.TestUtils.COLUMNS_ATTR_NAME;
 import static org.apache.atlas.TestUtils.COLUMN_TYPE;
@@ -917,6 +919,27 @@ public class AtlasEntityStoreV1Test {
 
     }
 
+    @Test
+    public void testCreateWithDuplicateGuids() throws Exception {
+        init();
+        AtlasEntityWithExtInfo tblEntity2 = 
TestUtilsV2.createTableEntityDuplicatesV2(dbEntity.getEntity());
+        EntityMutationResponse response   = entityStore.createOrUpdate(new 
AtlasEntityStream(tblEntity2), false);
+
+        List<AtlasEntityHeader> createdEntities = 
response.getCreatedEntities();
+        assertTrue(CollectionUtils.isNotEmpty(createdEntities));
+        assertEquals(createdEntities.size(), 2);
+
+        String tableGuid = createdEntities.get(0).getGuid();
+        AtlasEntityWithExtInfo tableEntity  = entityStore.getById(tableGuid);
+        assertEquals(tableEntity.getReferredEntities().size(), 1);
+
+        List<AtlasObjectId> columns = (List<AtlasObjectId>) 
tableEntity.getEntity().getAttribute("columns");
+        assertEquals(columns.size(), 1);
+
+        Set<AtlasObjectId> uniqueColumns = new HashSet<>(columns);
+        assertTrue(uniqueColumns.size() == 1);
+    }
+
     private String randomStrWithReservedChars() {
         return randomString() + "\"${}%";
     }

Reply via email to