Repository: incubator-atlas Updated Branches: refs/heads/master 4367c4915 -> a810f1564
ATLAS-1514 Remove duplicates from class array attribute when target is deleted (dkantor) Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/a810f156 Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/a810f156 Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/a810f156 Branch: refs/heads/master Commit: a810f15648e764d1e15f1e810718da8eba7758f6 Parents: 4367c49 Author: Dave Kantor <[email protected]> Authored: Wed Feb 1 17:45:57 2017 -0500 Committer: Dave Kantor <[email protected]> Committed: Wed Feb 1 17:45:57 2017 -0500 ---------------------------------------------------------------------- release-log.txt | 1 + .../atlas/repository/graph/DeleteHandler.java | 5 +- .../store/graph/v1/DeleteHandlerV1.java | 5 +- ...hBackedMetadataRepositoryDeleteTestBase.java | 91 ++++++++++++++++++++ .../GraphBackedRepositoryHardDeleteTest.java | 8 ++ .../GraphBackedRepositorySoftDeleteTest.java | 8 ++ 6 files changed, 116 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/a810f156/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index b0fc288..4459f9d 100644 --- a/release-log.txt +++ b/release-log.txt @@ -9,6 +9,7 @@ ATLAS-1060 Add composite indexes for exact match performance improvements for al ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai) ALL CHANGES: +ATLAS-1514 Remove duplicates from class array attribute when target is deleted (dkantor) ATLAS-1369 Optimize Gremlin queries generated by DSL translator (jnhagelb) ATLAS-1513 updated AtlasEntityType with methods to get foreign-key references; added helper methods in AtlasAttribute (mneethiraj via kevalbhatt) ATLAS-1502 added configuration to restrict entity-types editable via UI (Kalyanikashikar via mneethiraj) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/a810f156/repository/src/main/java/org/apache/atlas/repository/graph/DeleteHandler.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/DeleteHandler.java b/repository/src/main/java/org/apache/atlas/repository/graph/DeleteHandler.java index 4973a33..f0fef1f 100644 --- a/repository/src/main/java/org/apache/atlas/repository/graph/DeleteHandler.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/DeleteHandler.java @@ -354,7 +354,10 @@ public abstract class DeleteHandler { attributeName); } - elements.remove(elementEdge.getId().toString()); + // Remove all occurrences of the edge ID from the list. + // This prevents dangling edge IDs (i.e. edge IDs for deleted edges) + // from the remaining in the list if there are duplicates. + elements.removeAll(Collections.singletonList(elementEdge.getId().toString())); GraphHelper.setProperty(outVertex, propertyName, elements); break; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/a810f156/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java index 08361ea..a989f76 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java @@ -447,7 +447,10 @@ public abstract class DeleteHandlerV1 { //but when column is deleted, table will not reference the deleted column LOG.debug("Removing edge {} from the array attribute {}", string(elementEdge), attributeName); - elements.remove(elementEdge.getId().toString()); + // Remove all occurrences of the edge ID from the list. + // This prevents dangling edge IDs (i.e. edge IDs for deleted edges) + // from the remaining in the list if there are duplicates. + elements.removeAll(Collections.singletonList(elementEdge.getId().toString())); GraphHelper.setProperty(outVertex, propertyName, elements); break; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/a810f156/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteTestBase.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteTestBase.java b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteTestBase.java index 9e850a9..4919b08 100644 --- a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteTestBase.java +++ b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteTestBase.java @@ -1048,6 +1048,97 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase { Arrays.asList(mapOwnerGuid, mapValueGuid, mapValueReferencerContainerGuid, mapValueReferencerGuid))); } + @Test + public void testDeleteMixOfExistentAndNonExistentEntities() throws Exception { + ITypedReferenceableInstance entity1 = compositeMapValueType.createInstance(); + ITypedReferenceableInstance entity2 = compositeMapValueType.createInstance(); + List<String> createEntitiesResult = repositoryService.createEntities(entity1, entity2); + Assert.assertEquals(createEntitiesResult.size(), 2); + List<String> guids = Arrays.asList(createEntitiesResult.get(0), "non-existent-guid1", "non-existent-guid2", createEntitiesResult.get(1)); + EntityResult deleteEntitiesResult = repositoryService.deleteEntities(guids); + Assert.assertEquals(deleteEntitiesResult.getDeletedEntities().size(), 2); + Assert.assertTrue(deleteEntitiesResult.getDeletedEntities().containsAll(createEntitiesResult)); + } + + @Test + public void testDeleteMixOfNullAndNonNullGuids() throws Exception { + ITypedReferenceableInstance entity1 = compositeMapValueType.createInstance(); + ITypedReferenceableInstance entity2 = compositeMapValueType.createInstance(); + List<String> createEntitiesResult = repositoryService.createEntities(entity1, entity2); + Assert.assertEquals(createEntitiesResult.size(), 2); + List<String> guids = Arrays.asList(createEntitiesResult.get(0), null, null, createEntitiesResult.get(1)); + EntityResult deleteEntitiesResult = repositoryService.deleteEntities(guids); + Assert.assertEquals(deleteEntitiesResult.getDeletedEntities().size(), 2); + Assert.assertTrue(deleteEntitiesResult.getDeletedEntities().containsAll(createEntitiesResult)); + } + + @Test + public void testDeleteCompositeEntityAndContainer() throws Exception { + Referenceable db = createDBEntity(); + String dbId = createInstance(db); + + Referenceable column = createColumnEntity(); + String colId = createInstance(column); + + Referenceable table1 = createTableEntity(dbId); + table1.set(COLUMNS_ATTR_NAME, Arrays.asList(new Id(colId, 0, COLUMN_TYPE))); + String table1Id = createInstance(table1); + Referenceable table2 = createTableEntity(dbId); + String table2Id = createInstance(table2); + + // Delete the tables and column + AtlasClient.EntityResult entityResult = deleteEntities(table1Id, colId, table2Id); + Assert.assertEquals(entityResult.getDeletedEntities().size(), 3); + Assert.assertTrue(entityResult.getDeletedEntities().containsAll(Arrays.asList(colId, table1Id, table2Id))); + assertEntityDeleted(table1Id); + assertEntityDeleted(colId); + assertEntityDeleted(table2Id); + } + + @Test + public void testDeleteEntityWithDuplicateReferenceListElements() throws Exception { + // Create a table entity, with 2 composite column entities + Referenceable dbEntity = createDBEntity(); + String dbGuid = createInstance(dbEntity); + Referenceable table1Entity = createTableEntity(dbGuid); + String tableName = TestUtils.randomString(); + table1Entity.set(NAME, tableName); + Referenceable col1 = createColumnEntity(); + col1.set(NAME, TestUtils.randomString()); + Referenceable col2 = createColumnEntity(); + col2.set(NAME, TestUtils.randomString()); + // Populate columns reference list with duplicates. + table1Entity.set(COLUMNS_ATTR_NAME, ImmutableList.of(col1, col2, col1, col2)); + ClassType dataType = typeSystem.getDataType(ClassType.class, table1Entity.getTypeName()); + ITypedReferenceableInstance instance = dataType.convert(table1Entity, Multiplicity.REQUIRED); + TestUtils.resetRequestContext(); + List<String> result = repositoryService.createEntities(instance); + Assert.assertEquals(result.size(), 3); + ITypedReferenceableInstance entityDefinition = repositoryService.getEntityDefinition(TABLE_TYPE, NAME, tableName); + String tableGuid = entityDefinition.getId()._getId(); + Object attrValue = entityDefinition.get(COLUMNS_ATTR_NAME); + assertTrue(attrValue instanceof List); + List<ITypedReferenceableInstance> columns = (List<ITypedReferenceableInstance>) attrValue; + Assert.assertEquals(columns.size(), 4); + TestUtils.resetRequestContext(); + String columnGuid = columns.get(0).getId()._getId(); + + // Delete one of the columns. + EntityResult deleteResult = repositoryService.deleteEntities(Collections.singletonList(columnGuid)); + Assert.assertEquals(deleteResult.getDeletedEntities().size(), 1); + Assert.assertTrue(deleteResult.getDeletedEntities().contains(columnGuid)); + Assert.assertEquals(deleteResult.getUpdateEntities().size(), 1); + Assert.assertTrue(deleteResult.getUpdateEntities().contains(tableGuid)); + + // Verify the duplicate edge IDs were all removed from reference property list. + AtlasVertex tableVertex = GraphHelper.getInstance().getVertexForGUID(tableGuid); + String columnsPropertyName = GraphHelper.getQualifiedFieldName(dataType, COLUMNS_ATTR_NAME); + List columnsPropertyValue = tableVertex.getProperty(columnsPropertyName, List.class); + verifyTestDeleteEntityWithDuplicateReferenceListElements(columnsPropertyValue); + } + + protected abstract void verifyTestDeleteEntityWithDuplicateReferenceListElements(List columnsPropertyValue); + private String createHrDeptGraph() throws Exception { ITypedReferenceableInstance hrDept = TestUtils.createDeptEg1(typeSystem); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/a810f156/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedRepositoryHardDeleteTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedRepositoryHardDeleteTest.java b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedRepositoryHardDeleteTest.java index 7929505..43b08c4 100644 --- a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedRepositoryHardDeleteTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedRepositoryHardDeleteTest.java @@ -202,4 +202,12 @@ public class GraphBackedRepositoryHardDeleteTest extends GraphBackedMetadataRepo // good } } + + @Override + protected void verifyTestDeleteEntityWithDuplicateReferenceListElements(List columnsPropertyValue) { + + // With hard deletes enabled, verify that duplicate edge IDs for deleted edges + // were removed from the array property list. + Assert.assertEquals(columnsPropertyValue.size(), 2); + } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/a810f156/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedRepositorySoftDeleteTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedRepositorySoftDeleteTest.java b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedRepositorySoftDeleteTest.java index 93926ef..5ac7c8f 100644 --- a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedRepositorySoftDeleteTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedRepositorySoftDeleteTest.java @@ -229,4 +229,12 @@ public class GraphBackedRepositorySoftDeleteTest extends GraphBackedMetadataRepo ITypedReferenceableInstance hrDept = repositoryService.getEntityDefinition(hrDeptGuid); Assert.assertEquals(hrDept.getId().getState(), EntityState.DELETED); } + + @Override + protected void verifyTestDeleteEntityWithDuplicateReferenceListElements(List columnsPropertyValue) { + + // With soft deletes enabled, verify that edge IDs for deleted edges + // were not removed from the array property list. + Assert.assertEquals(columnsPropertyValue.size(), 4); + } }
