Repository: incubator-atlas Updated Branches: refs/heads/master 153fc3623 -> 705014eb3
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/705014eb/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java b/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java index 1f906ed..e6dd230 100644 --- a/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java +++ b/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java @@ -73,6 +73,7 @@ import java.util.Map; import static org.apache.atlas.TestUtils.COLUMNS_ATTR_NAME; import static org.apache.atlas.TestUtils.COLUMN_TYPE; +import static org.apache.atlas.TestUtils.PII; import static org.apache.atlas.TestUtils.TABLE_TYPE; import static org.apache.atlas.TestUtils.createColumnEntity; import static org.apache.atlas.TestUtils.createDBEntity; @@ -80,6 +81,7 @@ import static org.apache.atlas.TestUtils.createTableEntity; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; @Guice(modules = RepositoryMetadataModule.class) @@ -154,23 +156,21 @@ public class DefaultMetadataServiceTest { String entityjson = InstanceSerialization.toJson(entity, true); JSONArray entitiesJson = new JSONArray(); entitiesJson.put(entityjson); - String response = metadataService.createEntities(entitiesJson.toString()); - JSONArray guids = new JSONArray(response); - if (guids != null && guids.length() > 0) { - return guids.getString(guids.length() - 1); + List<String> guids = metadataService.createEntities(entitiesJson.toString()); + if (guids != null && guids.size() > 0) { + return guids.get(guids.size() - 1); } return null; } - private String updateInstance(Referenceable entity) throws Exception { + private AtlasClient.EntityResult updateInstance(Referenceable entity) throws Exception { RequestContext.createContext(); ParamChecker.notNull(entity, "Entity"); ParamChecker.notNull(entity.getId(), "Entity"); String entityjson = InstanceSerialization.toJson(entity, true); JSONArray entitiesJson = new JSONArray(); entitiesJson.put(entityjson); - String response = metadataService.updateEntities(entitiesJson.toString()); - return new JSONArray(response).getString(0); + return metadataService.updateEntities(entitiesJson.toString()); } @Test(expectedExceptions = TypeNotFoundException.class) @@ -201,6 +201,32 @@ public class DefaultMetadataServiceTest { } @Test + public void testAddDeleteTrait() throws Exception { + Referenceable entity = createDBEntity(); + String id = createInstance(entity); + + //add trait + Struct tag = new Struct(TestUtils.PII); + metadataService.addTrait(id, InstanceSerialization.toJson(tag, true)); + + List<String> traits = metadataService.getTraitNames(id); + assertEquals(traits.size(), 1); + assertEquals(traits.get(0), PII); + + //delete trait + metadataService.deleteTrait(id, PII); + traits = metadataService.getTraitNames(id); + assertEquals(traits.size(), 0); + + //add trait again + metadataService.addTrait(id, InstanceSerialization.toJson(tag, true)); + + traits = metadataService.getTraitNames(id); + assertEquals(traits.size(), 1); + assertEquals(traits.get(0), PII); + } + + @Test public void testEntityAudit() throws Exception { //create entity Referenceable entity = createDBEntity(); @@ -221,7 +247,7 @@ public class DefaultMetadataServiceTest { assertAuditEvents(id, EntityAuditEvent.EntityAuditAction.ENTITY_DELETE); } - private List<String> deleteEntities(String... guids) throws AtlasException { + private AtlasClient.EntityResult deleteEntities(String... guids) throws AtlasException { RequestContext.createContext(); return metadataService.deleteEntities(Arrays.asList(guids)); } @@ -350,7 +376,7 @@ public class DefaultMetadataServiceTest { Assert.assertTrue(partsMap.get("part2").equalsContents(((Map<String, Struct>)tableDefinition.get("partitionsMap")).get("part2"))); //Test map pointing to a class - final Map<String, Struct> columnsMap = new HashMap<>(); + final Map<String, Referenceable> columnsMap = new HashMap<>(); Referenceable col0Type = new Referenceable(TestUtils.COLUMN_TYPE, new HashMap<String, Object>() {{ put(NAME, "test1"); @@ -393,17 +419,33 @@ public class DefaultMetadataServiceTest { verifyMapUpdates(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME), null, TestUtils.COLUMNS_MAP); } - private void verifyMapUpdates(String typeName, String uniqAttrName, String uniqAttrValue, Map<String, Struct> expectedMap, String mapAttrName) throws AtlasException { + private void verifyMapUpdates(String typeName, String uniqAttrName, String uniqAttrValue, + Map<String, Referenceable> expectedMap, String mapAttrName) throws AtlasException { String json = metadataService.getEntityDefinition(typeName, uniqAttrName, uniqAttrValue); Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(json, true); + Map<String, Referenceable> actualMap = (Map<String, Referenceable>) tableDefinition.get(mapAttrName); - if(expectedMap == null) { - Assert.assertNull(tableDefinition.get(TestUtils.COLUMNS_MAP)); + if (expectedMap == null && actualMap != null) { + //all are marked as deleted in case of soft delete + for (String key : actualMap.keySet()) { + assertEquals(actualMap.get(key).getId().state, Id.EntityState.DELETED); + } + } else if(expectedMap == null) { + //hard delete case + assertNull(actualMap); } else { - Assert.assertEquals(((Map<String, Referenceable>)tableDefinition.get(mapAttrName)).size(), expectedMap.size()); + assertTrue(actualMap.size() >= expectedMap.size()); + for (String key : expectedMap.keySet()) { - Assert.assertTrue(((Map<String, Referenceable>) tableDefinition.get(mapAttrName)).get(key).equalsContents(expectedMap.get(key))); + assertTrue(actualMap.get(key).equalsContents(expectedMap.get(key))); + } + + //rest of the keys are marked as deleted + List<String> extraKeys = new ArrayList<>(actualMap.keySet()); + extraKeys.removeAll(expectedMap.keySet()); + for (String key : extraKeys) { + assertEquals(actualMap.get(key).getId().getState(), Id.EntityState.DELETED); } } } @@ -438,14 +480,13 @@ public class DefaultMetadataServiceTest { Assert.assertEquals(actualColumns, updatedColNameList); } - private void updateEntityPartial(String guid, Referenceable entity) throws AtlasException { + private AtlasClient.EntityResult updateEntityPartial(String guid, Referenceable entity) throws AtlasException { RequestContext.createContext(); - metadataService.updateEntityPartialByGuid(guid, entity); + return metadataService.updateEntityPartialByGuid(guid, entity); } @Test public void testUpdateEntityArrayOfClass() throws Exception { - //test array of class with id final List<Referenceable> columns = new ArrayList<>(); Map<String, Object> values = new HashMap<>(); @@ -456,63 +497,67 @@ public class DefaultMetadataServiceTest { Referenceable tableUpdated = new Referenceable(TestUtils.TABLE_TYPE, new HashMap<String, Object>() {{ put(COLUMNS_ATTR_NAME, columns); }}); - updateEntityPartial(tableId._getId(), tableUpdated); + AtlasClient.EntityResult entityResult = updateEntityPartial(tableId._getId(), tableUpdated); + assertEquals(entityResult.getCreatedEntities().size(), 1); //col1 created + assertEquals(entityResult.getUpdateEntities().size(), 1); //table updated + assertEquals(entityResult.getUpdateEntities().get(0), tableId._getId()); verifyArrayUpdates(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME), columns, COLUMNS_ATTR_NAME); - //Partial update. Add col5 But also update col1 + //Partial update. Add col2 But also update col1 Map<String, Object> valuesCol5 = new HashMap<>(); - valuesCol5.put(NAME, "col5"); + valuesCol5.put(NAME, "col2"); valuesCol5.put("type", "type"); Referenceable col2 = new Referenceable(TestUtils.COLUMN_TYPE, valuesCol5); //update col1 col1.set("type", "type1"); - - //add col5 - final List<Referenceable> updateColumns = Arrays.asList(col1, col2); + columns.add(col2); tableUpdated = new Referenceable(TestUtils.TABLE_TYPE, new HashMap<String, Object>() {{ - put(COLUMNS_ATTR_NAME, updateColumns); + put(COLUMNS_ATTR_NAME, columns); }}); - updateEntityPartial(tableId._getId(), tableUpdated); + entityResult = updateEntityPartial(tableId._getId(), tableUpdated); + assertEquals(entityResult.getCreatedEntities().size(), 1); //col2 created + assertEquals(entityResult.getUpdateEntities().size(), 2); //table, col1 updated - verifyArrayUpdates(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME), updateColumns, COLUMNS_ATTR_NAME); + verifyArrayUpdates(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME), columns, COLUMNS_ATTR_NAME); - //Complete update. Add array elements - col3,4 + //Complete update. Add array elements - col3,col4 Map<String, Object> values1 = new HashMap<>(); values1.put(NAME, "col3"); values1.put("type", "type"); - Referenceable ref1 = new Referenceable(TestUtils.COLUMN_TYPE, values1); - columns.add(ref1); + Referenceable col3 = new Referenceable(TestUtils.COLUMN_TYPE, values1); + columns.add(col3); Map<String, Object> values2 = new HashMap<>(); values2.put(NAME, "col4"); values2.put("type", "type"); - Referenceable ref2 = new Referenceable(TestUtils.COLUMN_TYPE, values2); - columns.add(ref2); + Referenceable col4 = new Referenceable(TestUtils.COLUMN_TYPE, values2); + columns.add(col4); table.set(COLUMNS_ATTR_NAME, columns); - updateInstance(table); + entityResult = updateInstance(table); + assertEquals(entityResult.getCreatedEntities().size(), 2); //col3, col4 created verifyArrayUpdates(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME), columns, COLUMNS_ATTR_NAME); //Swap elements columns.clear(); - columns.add(ref2); - columns.add(ref1); + columns.add(col4); + columns.add(col3); table.set(COLUMNS_ATTR_NAME, columns); - updateInstance(table); - + entityResult = updateInstance(table); + assertEquals(entityResult.getDeletedEntities().size(), 2); //col1, col2 are deleted verifyArrayUpdates(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME), columns, COLUMNS_ATTR_NAME); //drop a single column columns.clear(); - columns.add(ref1); + columns.add(col3); table.set(COLUMNS_ATTR_NAME, columns); - updateInstance(table); - + entityResult = updateInstance(table); + assertEquals(entityResult.getDeletedEntities().size(), 1); //col4 deleted verifyArrayUpdates(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME), columns, COLUMNS_ATTR_NAME); //Remove a class reference/Id and insert another reference @@ -520,34 +565,46 @@ public class DefaultMetadataServiceTest { values.clear(); columns.clear(); - values.put(NAME, "col2"); + values.put(NAME, "col5"); values.put("type", "type"); - col1 = new Referenceable(TestUtils.COLUMN_TYPE, values); - columns.add(col1); + Referenceable col5 = new Referenceable(TestUtils.COLUMN_TYPE, values); + columns.add(col5); table.set(COLUMNS_ATTR_NAME, columns); - updateInstance(table); + entityResult = updateInstance(table); + assertEquals(entityResult.getCreatedEntities().size(), 1); //col5 created + assertEquals(entityResult.getDeletedEntities().size(), 1); //col3 deleted verifyArrayUpdates(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME), columns, COLUMNS_ATTR_NAME); //Update array column to null table.setNull(COLUMNS_ATTR_NAME); - String newtableId = updateInstance(table); - Assert.assertEquals(newtableId, tableId._getId()); - + entityResult = updateInstance(table); + assertEquals(entityResult.getDeletedEntities().size(), 1); verifyArrayUpdates(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME), null, COLUMNS_ATTR_NAME); } - private void verifyArrayUpdates(String typeName, String uniqAttrName, String uniqAttrValue, List<? extends Struct> expectedArray, String arrAttrName) throws AtlasException { - String json = - metadataService.getEntityDefinition(typeName, uniqAttrName, uniqAttrValue); + private void verifyArrayUpdates(String typeName, String uniqAttrName, String uniqAttrValue, + List<Referenceable> expectedArray, String arrAttrName) throws AtlasException { + String json = metadataService.getEntityDefinition(typeName, uniqAttrName, uniqAttrValue); Referenceable entityDefinition = InstanceSerialization.fromJsonReferenceable(json, true); - - if (expectedArray == null) { - Assert.assertNull(entityDefinition.get(arrAttrName)); + List<Referenceable> actualArray = (List<Referenceable>) entityDefinition.get(arrAttrName); + if (expectedArray == null && actualArray != null) { + //all are marked as deleted in case of soft delete + for (int index = 0; index < actualArray.size(); index++) { + assertEquals(actualArray.get(index).getId().state, Id.EntityState.DELETED); + } + } else if(expectedArray == null) { + //hard delete case + assertNull(actualArray); } else { - Assert.assertEquals(((List<Referenceable>)entityDefinition.get(arrAttrName)).size(), expectedArray.size()); - for (int index = 0; index < expectedArray.size(); index++) { - Assert.assertTrue(((List<Referenceable>) entityDefinition.get(arrAttrName)).get(index).equalsContents(expectedArray.get(index))); + int index; + for (index = 0; index < expectedArray.size(); index++) { + Assert.assertTrue(actualArray.get(index).equalsContents(expectedArray.get(index))); + } + + //Rest of the entities in the list are marked as deleted + for (; index < actualArray.size(); index++) { + assertEquals(actualArray.get(index).getId().state, Id.EntityState.DELETED); } } } @@ -560,7 +617,7 @@ public class DefaultMetadataServiceTest { serdeInstance.set("description", "testDesc"); table.set("serde1", serdeInstance); - String newtableId = updateInstance(table); + String newtableId = updateInstance(table).getUpdateEntities().get(0); Assert.assertEquals(newtableId, tableId._getId()); String tableDefinitionJson = @@ -664,7 +721,7 @@ public class DefaultMetadataServiceTest { List<Struct> partitions = new ArrayList<Struct>(){{ add(partition1); add(partition2); }}; table.set("partitions", partitions); - String newtableId = updateInstance(table); + String newtableId = updateInstance(table).getUpdateEntities().get(0); Assert.assertEquals(newtableId, tableId._getId()); String tableDefinitionJson = @@ -673,15 +730,14 @@ public class DefaultMetadataServiceTest { Assert.assertNotNull(tableDefinition.get("partitions")); List<Struct> partitionsActual = (List<Struct>) tableDefinition.get("partitions"); - Assert.assertEquals(partitionsActual.size(), 2); - Assert.assertTrue(partitions.get(0).equalsContents(partitionsActual.get(0))); + assertPartitions(partitionsActual, partitions); //add a new element to array of struct final Struct partition3 = new Struct(TestUtils.PARTITION_STRUCT_TYPE); partition3.set(NAME, "part3"); partitions.add(partition3); table.set("partitions", partitions); - newtableId = updateInstance(table); + newtableId = updateInstance(table).getUpdateEntities().get(0); Assert.assertEquals(newtableId, tableId._getId()); tableDefinitionJson = @@ -690,13 +746,12 @@ public class DefaultMetadataServiceTest { Assert.assertNotNull(tableDefinition.get("partitions")); partitionsActual = (List<Struct>) tableDefinition.get("partitions"); - Assert.assertEquals(partitionsActual.size(), 3); - Assert.assertTrue(partitions.get(2).equalsContents(partitionsActual.get(2))); + assertPartitions(partitionsActual, partitions); //remove one of the struct values partitions.remove(1); table.set("partitions", partitions); - newtableId = updateInstance(table); + newtableId = updateInstance(table).getUpdateEntities().get(0); Assert.assertEquals(newtableId, tableId._getId()); tableDefinitionJson = @@ -705,13 +760,11 @@ public class DefaultMetadataServiceTest { Assert.assertNotNull(tableDefinition.get("partitions")); partitionsActual = (List<Struct>) tableDefinition.get("partitions"); - Assert.assertEquals(partitionsActual.size(), 2); - Assert.assertTrue(partitions.get(0).equalsContents(partitionsActual.get(0))); - Assert.assertTrue(partitions.get(1).equalsContents(partitionsActual.get(1))); + assertPartitions(partitionsActual, partitions); //Update struct value within array of struct - partition1.set(NAME, "part4"); - newtableId = updateInstance(table); + partitions.get(0).set(NAME, "part4"); + newtableId = updateInstance(table).getUpdateEntities().get(0); Assert.assertEquals(newtableId, tableId._getId()); tableDefinitionJson = @@ -720,15 +773,14 @@ public class DefaultMetadataServiceTest { Assert.assertNotNull(tableDefinition.get("partitions")); partitionsActual = (List<Struct>) tableDefinition.get("partitions"); - Assert.assertEquals(partitionsActual.size(), 2); - Assert.assertTrue(partitions.get(0).equalsContents(partitionsActual.get(0))); + assertPartitions(partitionsActual, partitions); //add a repeated element to array of struct final Struct partition4 = new Struct(TestUtils.PARTITION_STRUCT_TYPE); partition4.set(NAME, "part4"); partitions.add(partition4); table.set("partitions", partitions); - newtableId = updateInstance(table); + newtableId = updateInstance(table).getUpdateEntities().get(0); Assert.assertEquals(newtableId, tableId._getId()); tableDefinitionJson = @@ -737,15 +789,12 @@ public class DefaultMetadataServiceTest { Assert.assertNotNull(tableDefinition.get("partitions")); partitionsActual = (List<Struct>) tableDefinition.get("partitions"); - Assert.assertEquals(partitionsActual.size(), 3); - Assert.assertEquals(partitionsActual.get(2).get(NAME), "part4"); - Assert.assertEquals(partitionsActual.get(0).get(NAME), "part4"); - Assert.assertTrue(partitions.get(2).equalsContents(partitionsActual.get(2))); + assertPartitions(partitionsActual, partitions); // Remove all elements. Should set array attribute to null partitions.clear(); - newtableId = updateInstance(table); + newtableId = updateInstance(table).getUpdateEntities().get(0); Assert.assertEquals(newtableId, tableId._getId()); tableDefinitionJson = @@ -755,6 +804,13 @@ public class DefaultMetadataServiceTest { Assert.assertNull(tableDefinition.get("partitions")); } + private void assertPartitions(List<Struct> partitionsActual, List<Struct> partitions) { + assertEquals(partitionsActual.size(), partitions.size()); + for (int index = 0; index < partitions.size(); index++) { + assertTrue(partitionsActual.get(index).equalsContents(partitions.get(index))); + } + } + @Test(expectedExceptions = ValueConversionException.class) public void testUpdateRequiredAttrToNull() throws Exception { @@ -772,7 +828,6 @@ public class DefaultMetadataServiceTest { @Test public void testUpdateOptionalAttrToNull() throws Exception { - String tableDefinitionJson = metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME)); Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true); @@ -782,7 +837,7 @@ public class DefaultMetadataServiceTest { //Update optional attribute table.setNull("created"); - String newtableId = updateInstance(table); + String newtableId = updateInstance(table).getUpdateEntities().get(0); Assert.assertEquals(newtableId, tableId._getId()); tableDefinitionJson = @@ -853,35 +908,39 @@ public class DefaultMetadataServiceTest { // Register an EntityChangeListener to verify the notification mechanism // is working for deleteEntities(). - DeleteEntitiesChangeListener listener = new DeleteEntitiesChangeListener(); + EntitiesChangeListener listener = new EntitiesChangeListener(); metadataService.registerListener(listener); + //Delete one column + String columnId = table1Columns.get(0).getId()._getId(); + AtlasClient.EntityResult entityResult = deleteEntities(columnId); + //column is deleted and table is updated + assertEquals(entityResult.getDeletedEntities().get(0), columnId); + assertEquals(entityResult.getUpdateEntities().get(0), table1Entity.getId()._getId()); + + //verify listener was called for updates and deletes + assertEquals(entityResult.getDeletedEntities(), listener.getDeletedEntities()); + assertEquals(entityResult.getUpdateEntities(), listener.getUpdatedEntities()); + // Delete the table entities. The deletion should cascade // to their composite columns. - List<String> deletedGuids = deleteEntities(table1Entity.getId()._getId()); + entityResult = deleteEntities(table1Entity.getId()._getId()); // Verify that deleteEntities() response has guids for tables and their composite columns. - Assert.assertTrue(deletedGuids.contains(table1Entity.getId()._getId())); - for (IReferenceableInstance column : table1Columns) { - Assert.assertTrue(deletedGuids.contains(column.getId()._getId())); - } + Assert.assertTrue(entityResult.getDeletedEntities().contains(table1Entity.getId()._getId())); + Assert.assertTrue(entityResult.getDeletedEntities().contains(table1Columns.get(1).getId()._getId())); + Assert.assertTrue(entityResult.getDeletedEntities().contains(table1Columns.get(2).getId()._getId())); // Verify that tables and their composite columns have been deleted from the repository. assertEntityDeleted(TABLE_TYPE, NAME, table1Entity.get(NAME)); - assertEntityDeleted(COLUMN_TYPE, NAME, col1.get(NAME)); assertEntityDeleted(COLUMN_TYPE, NAME, col2.get(NAME)); assertEntityDeleted(COLUMN_TYPE, NAME, col3.get(NAME)); // Verify that the listener was notified about the deleted entities. - Collection<ITypedReferenceableInstance> deletedEntitiesFromListener = listener.getDeletedEntities(); + List<String> deletedEntitiesFromListener = listener.getDeletedEntities(); Assert.assertNotNull(deletedEntitiesFromListener); - Assert.assertEquals(deletedEntitiesFromListener.size(), deletedGuids.size()); - List<String> deletedGuidsFromListener = new ArrayList<>(deletedGuids.size()); - for (ITypedReferenceableInstance deletedEntity : deletedEntitiesFromListener) { - deletedGuidsFromListener.add(deletedEntity.getId()._getId()); - } - Assert.assertEquals(deletedGuidsFromListener.size(), deletedGuids.size()); - Assert.assertTrue(deletedGuidsFromListener.containsAll(deletedGuids)); + Assert.assertEquals(deletedEntitiesFromListener.size(), entityResult.getDeletedEntities().size()); + Assert.assertTrue(deletedEntitiesFromListener.containsAll(entityResult.getDeletedEntities())); } private void assertEntityDeleted(String typeName, String attributeName, Object attributeValue) @@ -915,12 +974,13 @@ public class DefaultMetadataServiceTest { // Register an EntityChangeListener to verify the notification mechanism // is working for deleteEntityByUniqueAttribute(). - DeleteEntitiesChangeListener listener = new DeleteEntitiesChangeListener(); + EntitiesChangeListener listener = new EntitiesChangeListener(); metadataService.registerListener(listener); // Delete the table entities. The deletion should cascade // to their composite columns. - List<String> deletedGuids = metadataService.deleteEntityByUniqueAttribute(TestUtils.TABLE_TYPE, NAME, (String) table1Entity.get(NAME)); + List<String> deletedGuids = metadataService.deleteEntityByUniqueAttribute(TestUtils.TABLE_TYPE, NAME, + (String) table1Entity.get(NAME)).getDeletedEntities(); // Verify that deleteEntities() response has guids for tables and their composite columns. Assert.assertTrue(deletedGuids.contains(table1Entity.getId()._getId())); @@ -936,15 +996,10 @@ public class DefaultMetadataServiceTest { assertEntityDeleted(COLUMN_TYPE, NAME, col3.get(NAME)); // Verify that the listener was notified about the deleted entities. - Collection<ITypedReferenceableInstance> deletedEntitiesFromListener = listener.getDeletedEntities(); + List<String> deletedEntitiesFromListener = listener.getDeletedEntities(); Assert.assertNotNull(deletedEntitiesFromListener); Assert.assertEquals(deletedEntitiesFromListener.size(), deletedGuids.size()); - List<String> deletedGuidsFromListener = new ArrayList<>(deletedGuids.size()); - for (ITypedReferenceableInstance deletedEntity : deletedEntitiesFromListener) { - deletedGuidsFromListener.add(deletedEntity.getId()._getId()); - } - Assert.assertEquals(deletedGuidsFromListener.size(), deletedGuids.size()); - Assert.assertTrue(deletedGuidsFromListener.containsAll(deletedGuids)); + Assert.assertTrue(deletedEntitiesFromListener.containsAll(deletedGuids)); } @Test @@ -1024,10 +1079,10 @@ public class DefaultMetadataServiceTest { } } - private static class DeleteEntitiesChangeListener implements EntityChangeListener { - - private Collection<ITypedReferenceableInstance> deletedEntities_; - + private static class EntitiesChangeListener implements EntityChangeListener { + private List<String> deletedEntities = new ArrayList<>(); + private List<String> updatedEntities = new ArrayList<>(); + @Override public void onEntitiesAdded(Collection<ITypedReferenceableInstance> entities) throws AtlasException { @@ -1036,6 +1091,10 @@ public class DefaultMetadataServiceTest { @Override public void onEntitiesUpdated(Collection<ITypedReferenceableInstance> entities) throws AtlasException { + updatedEntities.clear(); + for (ITypedReferenceableInstance entity : entities) { + updatedEntities.add(entity.getId()._getId()); + } } @Override @@ -1051,11 +1110,18 @@ public class DefaultMetadataServiceTest { @Override public void onEntitiesDeleted(Collection<ITypedReferenceableInstance> entities) throws AtlasException { - deletedEntities_ = entities; + deletedEntities.clear(); + for (ITypedReferenceableInstance entity : entities) { + deletedEntities.add(entity.getId()._getId()); + } } - public Collection<ITypedReferenceableInstance> getDeletedEntities() { - return deletedEntities_; + public List<String> getDeletedEntities() { + return deletedEntities; + } + + public List<String> getUpdatedEntities() { + return updatedEntities; } } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/705014eb/server-api/src/main/java/org/apache/atlas/RequestContext.java ---------------------------------------------------------------------- diff --git a/server-api/src/main/java/org/apache/atlas/RequestContext.java b/server-api/src/main/java/org/apache/atlas/RequestContext.java index b1d87ea..ec38c11 100644 --- a/server-api/src/main/java/org/apache/atlas/RequestContext.java +++ b/server-api/src/main/java/org/apache/atlas/RequestContext.java @@ -49,7 +49,16 @@ public class RequestContext { private RequestContext() { } + //To handle gets from background threads where createContext() is not called + //createContext called for every request in the filter public static RequestContext get() { + if (CURRENT_CONTEXT.get() == null) { + synchronized (RequestContext.class) { + if (CURRENT_CONTEXT.get() == null) { + createContext(); + } + } + } return CURRENT_CONTEXT.get(); } @@ -72,15 +81,19 @@ public class RequestContext { this.user = user; } - public void recordCreatedEntities(Collection<String> createdEntityIds) { + public void recordEntityCreate(Collection<String> createdEntityIds) { this.createdEntityIds.addAll(createdEntityIds); } - public void recordUpdatedEntities(Collection<String> updatedEntityIds) { + public void recordEntityUpdate(Collection<String> updatedEntityIds) { this.updatedEntityIds.addAll(updatedEntityIds); } - public void recordDeletedEntity(String entityId, String typeName) throws AtlasException { + public void recordEntityUpdate(String entityId) { + this.updatedEntityIds.add(entityId); + } + + public void recordEntityDelete(String entityId, String typeName) throws AtlasException { ClassType type = typeSystem.getDataType(ClassType.class, typeName); ITypedReferenceableInstance entity = type.createInstance(new Id(entityId, 0, typeName)); if (deletedEntityIds.add(entityId)) { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/705014eb/server-api/src/main/java/org/apache/atlas/services/MetadataService.java ---------------------------------------------------------------------- diff --git a/server-api/src/main/java/org/apache/atlas/services/MetadataService.java b/server-api/src/main/java/org/apache/atlas/services/MetadataService.java index c8c1067..6bca3df 100644 --- a/server-api/src/main/java/org/apache/atlas/services/MetadataService.java +++ b/server-api/src/main/java/org/apache/atlas/services/MetadataService.java @@ -18,6 +18,7 @@ package org.apache.atlas.services; +import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasException; import org.apache.atlas.EntityAuditEvent; import org.apache.atlas.listener.EntityChangeListener; @@ -80,7 +81,7 @@ public interface MetadataService { * @param entityDefinition definition * @return json array of guids of entities created */ - String createEntities(String entityDefinition) throws AtlasException; + List<String> createEntities(String entityDefinition) throws AtlasException; /** * Get a typed entity instance. @@ -136,7 +137,7 @@ public interface MetadataService { * @param value property value * @return json array of guids of entities created/updated */ - String updateEntityAttributeByGuid(String guid, String attribute, String value) throws AtlasException; + AtlasClient.EntityResult updateEntityAttributeByGuid(String guid, String attribute, String value) throws AtlasException; /** * Supports Partial updates of an entity. Users can update a subset of attributes for an entity identified by its guid @@ -146,7 +147,7 @@ public interface MetadataService { * @return json array of guids of entities created/updated * @throws AtlasException */ - String updateEntityPartialByGuid(String guid, Referenceable entity) throws AtlasException; + AtlasClient.EntityResult updateEntityPartialByGuid(String guid, Referenceable entity) throws AtlasException; /** * Batch API - Adds/Updates the given entity id(guid). @@ -154,7 +155,7 @@ public interface MetadataService { * @param entityJson entity json * @return json array of guids of entities created/updated */ - String updateEntities(String entityJson) throws AtlasException; + AtlasClient.EntityResult updateEntities(String entityJson) throws AtlasException; // Trait management functions @@ -168,8 +169,9 @@ public interface MetadataService { * @return Guid of updated entity * @throws AtlasException */ - String updateEntityByUniqueAttribute(String typeName, String uniqueAttributeName, String attrValue, - Referenceable updatedEntity) throws AtlasException; + AtlasClient.EntityResult updateEntityByUniqueAttribute(String typeName, String uniqueAttributeName, + String attrValue, + Referenceable updatedEntity) throws AtlasException; /** * Gets the list of trait names for a given entity represented by a guid. @@ -207,10 +209,10 @@ public interface MetadataService { * Delete the specified entities from the repository * * @param guids entity guids to be deleted - * @return List of guids for deleted entities + * @return List of guids for deleted entities * @throws AtlasException */ - List<String> deleteEntities(List<String> guids) throws AtlasException; + AtlasClient.EntityResult deleteEntities(List<String> guids) throws AtlasException; /** * Register a listener for entity change. @@ -235,7 +237,8 @@ public interface MetadataService { * @return List of guids for deleted entities (including their composite references) * @throws AtlasException */ - List<String> deleteEntityByUniqueAttribute(String typeName, String uniqueAttributeName, String attrValue) throws AtlasException; + AtlasClient.EntityResult deleteEntityByUniqueAttribute(String typeName, String uniqueAttributeName, + String attrValue) throws AtlasException; /** * Returns entity audit events for entity id in the decreasing order of timestamp http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/705014eb/typesystem/src/main/java/org/apache/atlas/typesystem/Referenceable.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/Referenceable.java b/typesystem/src/main/java/org/apache/atlas/typesystem/Referenceable.java index 31f157e..5b8e157 100755 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/Referenceable.java +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/Referenceable.java @@ -100,7 +100,7 @@ public class Referenceable extends Struct implements IReferenceableInstance { * @throws AtlasException if the referenceable can not be created */ public Referenceable(IReferenceableInstance instance) throws AtlasException { - this(instance.getId()._getId(), instance.getTypeName(), instance.getValuesMap(), instance.getTraits(), + this(instance.getId(), instance.getTypeName(), instance.getValuesMap(), instance.getTraits(), getTraits(instance)); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/705014eb/webapp/src/main/java/org/apache/atlas/LocalAtlasClient.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/LocalAtlasClient.java b/webapp/src/main/java/org/apache/atlas/LocalAtlasClient.java index c6ed85d..7173d4d 100644 --- a/webapp/src/main/java/org/apache/atlas/LocalAtlasClient.java +++ b/webapp/src/main/java/org/apache/atlas/LocalAtlasClient.java @@ -83,13 +83,13 @@ public class LocalAtlasClient extends AtlasClient { } }; JSONObject response = entityOperation.run(); - List<String> results = extractResults(response, GUID, new ExtractOperation<String, String>()); + EntityResult results = extractEntityResult(response); LOG.debug("Create entities returned results: {}", results); - return results; + return results.getCreatedEntities(); } @Override - protected List<String> updateEntities(final JSONArray entities) throws AtlasServiceException { + protected EntityResult updateEntities(final JSONArray entities) throws AtlasServiceException { LOG.debug("Updating entities: {}", entities); EntityOperation entityOperation = new EntityOperation(API.UPDATE_ENTITY) { @Override @@ -98,7 +98,7 @@ public class LocalAtlasClient extends AtlasClient { } }; JSONObject response = entityOperation.run(); - List<String> results = extractResults(response, GUID, new ExtractOperation<String, String>()); + EntityResult results = extractEntityResult(response); LOG.debug("Update entities returned results: {}", results); return results; } @@ -130,7 +130,7 @@ public class LocalAtlasClient extends AtlasClient { } @Override - public String updateEntity(final String entityType, final String uniqueAttributeName, + public EntityResult updateEntity(final String entityType, final String uniqueAttributeName, final String uniqueAttributeValue, Referenceable entity) throws AtlasServiceException { final String entityJson = InstanceSerialization.toJson(entity, true); LOG.debug("Updating entity type: {}, attributeName: {}, attributeValue: {}, entity: {}", entityType, @@ -143,13 +143,13 @@ public class LocalAtlasClient extends AtlasClient { } }; JSONObject response = entityOperation.run(); - String result = getString(response, GUID); + EntityResult result = extractEntityResult(response); LOG.debug("Update entity returned result: {}", result); return result; } @Override - public List<String> deleteEntity(final String entityType, final String uniqueAttributeName, + public EntityResult deleteEntity(final String entityType, final String uniqueAttributeName, final String uniqueAttributeValue) throws AtlasServiceException { LOG.debug("Deleting entity type: {}, attributeName: {}, attributeValue: {}", entityType, uniqueAttributeName, uniqueAttributeValue); @@ -160,7 +160,7 @@ public class LocalAtlasClient extends AtlasClient { } }; JSONObject response = entityOperation.run(); - List<String> results = extractResults(response, GUID, new ExtractOperation<String, String>()); + EntityResult results = extractEntityResult(response); LOG.debug("Delete entities returned results: {}", results); return results; } @@ -191,18 +191,18 @@ public class LocalAtlasClient extends AtlasClient { } @Override - public void updateEntityAttribute(final String guid, final String attribute, String value) throws AtlasServiceException { + public EntityResult updateEntityAttribute(final String guid, final String attribute, String value) throws AtlasServiceException { throw new IllegalStateException("Not supported in LocalAtlasClient"); } @Override - public void updateEntity(String guid, Referenceable entity) throws AtlasServiceException { + public EntityResult updateEntity(String guid, Referenceable entity) throws AtlasServiceException { throw new IllegalStateException("Not supported in LocalAtlasClient"); } @Override - public List<String> deleteEntities(final String ... guids) throws AtlasServiceException { + public EntityResult deleteEntities(final String ... guids) throws AtlasServiceException { throw new IllegalStateException("Not supported in LocalAtlasClient"); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/705014eb/webapp/src/main/java/org/apache/atlas/notification/NotificationEntityChangeListener.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/notification/NotificationEntityChangeListener.java b/webapp/src/main/java/org/apache/atlas/notification/NotificationEntityChangeListener.java new file mode 100644 index 0000000..d10194d --- /dev/null +++ b/webapp/src/main/java/org/apache/atlas/notification/NotificationEntityChangeListener.java @@ -0,0 +1,161 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.notification; + +import com.google.common.annotations.VisibleForTesting; +import com.google.inject.Inject; +import org.apache.atlas.AtlasException; +import org.apache.atlas.listener.EntityChangeListener; +import org.apache.atlas.notification.entity.EntityNotification; +import org.apache.atlas.notification.entity.EntityNotificationImpl; +import org.apache.atlas.typesystem.IReferenceableInstance; +import org.apache.atlas.typesystem.IStruct; +import org.apache.atlas.typesystem.ITypedReferenceableInstance; +import org.apache.atlas.typesystem.Referenceable; +import org.apache.atlas.typesystem.Struct; +import org.apache.atlas.typesystem.types.FieldMapping; +import org.apache.atlas.typesystem.types.TraitType; +import org.apache.atlas.typesystem.types.TypeSystem; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Listen to the repository for entity changes and produce entity change notifications. + */ +public class NotificationEntityChangeListener implements EntityChangeListener { + + private final NotificationInterface notificationInterface; + private final TypeSystem typeSystem; + + + // ----- Constructors ------------------------------------------------------ + + /** + * Construct a NotificationEntityChangeListener. + * + * @param notificationInterface the notification framework interface + * @param typeSystem the Atlas type system + */ + @Inject + public NotificationEntityChangeListener(NotificationInterface notificationInterface, TypeSystem typeSystem) { + this.notificationInterface = notificationInterface; + this.typeSystem = typeSystem; + } + + + // ----- EntityChangeListener ---------------------------------------------- + + @Override + public void onEntitiesAdded(Collection<ITypedReferenceableInstance> entities) throws AtlasException { + notifyOfEntityEvent(entities, EntityNotification.OperationType.ENTITY_CREATE); + } + + @Override + public void onEntitiesUpdated(Collection<ITypedReferenceableInstance> entities) throws AtlasException { + notifyOfEntityEvent(entities, EntityNotification.OperationType.ENTITY_UPDATE); + } + + @Override + public void onTraitAdded(ITypedReferenceableInstance entity, IStruct trait) throws AtlasException { + notifyOfEntityEvent(Collections.singleton(entity), EntityNotification.OperationType.TRAIT_ADD); + } + + @Override + public void onTraitDeleted(ITypedReferenceableInstance entity, String traitName) throws AtlasException { + notifyOfEntityEvent(Collections.singleton(entity), EntityNotification.OperationType.TRAIT_DELETE); + } + + @Override + public void onEntitiesDeleted(Collection<ITypedReferenceableInstance> entities) throws AtlasException { + notifyOfEntityEvent(entities, EntityNotification.OperationType.ENTITY_DELETE); + } + + + // ----- helper methods ------------------------------------------------- + + + // ----- helper methods ---------------------------------------------------- + @VisibleForTesting + public static List<IStruct> getAllTraits(IReferenceableInstance entityDefinition, + TypeSystem typeSystem) throws AtlasException { + List<IStruct> traitInfo = new LinkedList<>(); + for (String traitName : entityDefinition.getTraits()) { + IStruct trait = entityDefinition.getTrait(traitName); + String typeName = trait.getTypeName(); + Map<String, Object> valuesMap = trait.getValuesMap(); + traitInfo.add(new Struct(typeName, valuesMap)); + traitInfo.addAll(getSuperTraits(typeName, valuesMap, typeSystem)); + } + return traitInfo; + } + + private static List<IStruct> getSuperTraits( + String typeName, Map<String, Object> values, TypeSystem typeSystem) throws AtlasException { + + List<IStruct> superTypes = new LinkedList<>(); + + TraitType traitDef = typeSystem.getDataType(TraitType.class, typeName); + Set<String> superTypeNames = traitDef.getAllSuperTypeNames(); + + for (String superTypeName : superTypeNames) { + TraitType superTraitDef = typeSystem.getDataType(TraitType.class, superTypeName); + + Map<String, Object> superTypeValues = new HashMap<>(); + + FieldMapping fieldMapping = superTraitDef.fieldMapping(); + + if (fieldMapping != null) { + Set<String> superTypeAttributeNames = fieldMapping.fields.keySet(); + + for (String superTypeAttributeName : superTypeAttributeNames) { + if (values.containsKey(superTypeAttributeName)) { + superTypeValues.put(superTypeAttributeName, values.get(superTypeAttributeName)); + } + } + } + IStruct superTrait = new Struct(superTypeName, superTypeValues); + superTypes.add(superTrait); + superTypes.addAll(getSuperTraits(superTypeName, values, typeSystem)); + } + + return superTypes; + } + + // send notification of entity change + private void notifyOfEntityEvent(Collection<ITypedReferenceableInstance> entityDefinitions, + EntityNotification.OperationType operationType) throws AtlasException { + List<EntityNotification> messages = new LinkedList<>(); + + for (IReferenceableInstance entityDefinition : entityDefinitions) { + Referenceable entity = new Referenceable(entityDefinition); + + EntityNotificationImpl notification = + new EntityNotificationImpl(entity, operationType, getAllTraits(entity, typeSystem)); + + messages.add(notification); + } + + notificationInterface.send(NotificationInterface.NotificationType.ENTITIES, messages); + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/705014eb/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java index 709fec5..487270d 100755 --- a/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java +++ b/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java @@ -19,7 +19,6 @@ package org.apache.atlas.web.resources; import com.google.common.base.Preconditions; - import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasException; import org.apache.atlas.EntityAuditEvent; @@ -59,9 +58,7 @@ import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import javax.ws.rs.core.UriBuilder; import javax.ws.rs.core.UriInfo; - import java.net.URI; -import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -119,19 +116,11 @@ public class EntityResource { LOG.debug("submitting entities {} ", AtlasClient.toString(new JSONArray(entities))); - final String guids = metadataService.createEntities(entities); + final List<String> guids = metadataService.createEntities(entities); + JSONObject response = getResponse(new AtlasClient.EntityResult(guids, null, null)); UriBuilder ub = uriInfo.getAbsolutePathBuilder(); - URI locationURI = ub.path(guids).build(); - - JSONObject response = new JSONObject(); - response.put(AtlasClient.REQUEST_ID, Servlets.getRequestId()); - JSONArray guidArray = new JSONArray(guids); - response.put(AtlasClient.GUID, guidArray); - if (guidArray.length() > 0) { - response.put(AtlasClient.DEFINITION, - new JSONObject(metadataService.getEntityDefinition(new JSONArray(guids).getString(0)))); - } + URI locationURI = guids.isEmpty() ? null : ub.path(guids.get(0)).build(); return Response.created(locationURI).entity(response).build(); @@ -150,6 +139,18 @@ public class EntityResource { } } + private JSONObject getResponse(AtlasClient.EntityResult entityResult) throws AtlasException, JSONException { + JSONObject response = new JSONObject(); + response.put(AtlasClient.REQUEST_ID, Servlets.getRequestId()); + response.put(AtlasClient.ENTITIES, new JSONObject(entityResult.toString()).get(AtlasClient.ENTITIES)); + String sampleEntityId = getSample(entityResult); + if (sampleEntityId != null) { + String entityDefinition = metadataService.getEntityDefinition(sampleEntityId); + response.put(AtlasClient.DEFINITION, new JSONObject(entityDefinition)); + } + return response; + } + /** * Complete update of a set of entities - the values not specified will be replaced with null/removed * Adds/Updates given entities identified by its GUID or unique attribute @@ -163,14 +164,8 @@ public class EntityResource { final String entities = Servlets.getRequestPayload(request); LOG.debug("updating entities {} ", AtlasClient.toString(new JSONArray(entities))); - final String guids = metadataService.updateEntities(entities); - - JSONObject response = new JSONObject(); - response.put(AtlasClient.REQUEST_ID, Servlets.getRequestId()); - JSONArray guidsArray = new JSONArray(guids); - response.put(AtlasClient.GUID, guidsArray); - String entityDefinition = metadataService.getEntityDefinition(guidsArray.getString(0)); - response.put(AtlasClient.DEFINITION, new JSONObject(entityDefinition)); + AtlasClient.EntityResult entityResult = metadataService.updateEntities(entities); + JSONObject response = getResponse(entityResult); return Response.ok(response).build(); } catch(EntityExistsException e) { LOG.error("Unique constraint violation", e); @@ -187,6 +182,25 @@ public class EntityResource { } } + private String getSample(AtlasClient.EntityResult entityResult) { + String sample = getSample(entityResult.getCreatedEntities()); + if (sample == null) { + sample = getSample(entityResult.getUpdateEntities()); + } + if (sample == null) { + sample = getSample(entityResult.getDeletedEntities()); + } + return sample; + } + + + private String getSample(List<String> list) { + if (list != null && list.size() > 0) { + return list.get(0); + } + return null; + } + /** * Adds/Updates given entity identified by its unique attribute( entityType, attributeName and value) * Updates support only partial update of an entity - Adds/updates any new values specified @@ -214,11 +228,10 @@ public class EntityResource { Referenceable updatedEntity = InstanceSerialization.fromJsonReferenceable(entities, true); - final String guid = metadataService.updateEntityByUniqueAttribute(entityType, attribute, value, updatedEntity); + AtlasClient.EntityResult entityResult = + metadataService.updateEntityByUniqueAttribute(entityType, attribute, value, updatedEntity); - JSONObject response = new JSONObject(); - response.put(AtlasClient.REQUEST_ID, Thread.currentThread().getName()); - response.put(AtlasClient.GUID, guid); + JSONObject response = getResponse(entityResult); return Response.ok(response).build(); } catch (ValueConversionException ve) { LOG.error("Unable to persist entity instance due to a desrialization error ", ve); @@ -268,10 +281,8 @@ public class EntityResource { Referenceable updatedEntity = InstanceSerialization.fromJsonReferenceable(entityJson, true); - metadataService.updateEntityPartialByGuid(guid, updatedEntity); - - JSONObject response = new JSONObject(); - response.put(AtlasClient.REQUEST_ID, Thread.currentThread().getName()); + AtlasClient.EntityResult entityResult = metadataService.updateEntityPartialByGuid(guid, updatedEntity); + JSONObject response = getResponse(entityResult); return Response.ok(response).build(); } catch (EntityNotFoundException e) { LOG.error("An entity with GUID={} does not exist", guid, e); @@ -301,12 +312,8 @@ public class EntityResource { String value = Servlets.getRequestPayload(request); Preconditions.checkNotNull(value, "Entity value cannot be null"); - metadataService.updateEntityAttributeByGuid(guid, property, value); - - JSONObject response = new JSONObject(); - response.put(AtlasClient.REQUEST_ID, Thread.currentThread().getName()); - response.put(AtlasClient.GUID, guid); - + AtlasClient.EntityResult entityResult = metadataService.updateEntityAttributeByGuid(guid, property, value); + JSONObject response = getResponse(entityResult); return Response.ok(response).build(); } catch (EntityNotFoundException e) { LOG.error("An entity with GUID={} does not exist", guid, e); @@ -340,19 +347,13 @@ public class EntityResource { @QueryParam("value") String value) { try { - List<String> deletedGuids = new ArrayList<>(); + AtlasClient.EntityResult entityResult; if (guids != null && !guids.isEmpty()) { - deletedGuids = metadataService.deleteEntities(guids); + entityResult = metadataService.deleteEntities(guids); } else { - deletedGuids = metadataService.deleteEntityByUniqueAttribute(entityType, attribute, value); - } - JSONObject response = new JSONObject(); - response.put(AtlasClient.REQUEST_ID, Servlets.getRequestId()); - JSONArray guidArray = new JSONArray(deletedGuids.size()); - for (String guid : deletedGuids) { - guidArray.put(guid); + entityResult = metadataService.deleteEntityByUniqueAttribute(entityType, attribute, value); } - response.put(AtlasClient.GUID, guidArray); + JSONObject response = getResponse(entityResult); return Response.ok(response).build(); } catch (EntityNotFoundException e) { if(guids != null || !guids.isEmpty()) { @@ -386,7 +387,6 @@ public class EntityResource { JSONObject response = new JSONObject(); response.put(AtlasClient.REQUEST_ID, Servlets.getRequestId()); - response.put(AtlasClient.GUID, guid); Response.Status status = Response.Status.NOT_FOUND; if (entityDefinition != null) { @@ -518,7 +518,6 @@ public class EntityResource { JSONObject response = new JSONObject(); response.put(AtlasClient.REQUEST_ID, Servlets.getRequestId()); - response.put(AtlasClient.GUID, guid); response.put(AtlasClient.RESULTS, new JSONArray(traitNames)); response.put(AtlasClient.COUNT, traitNames.size()); @@ -555,7 +554,6 @@ public class EntityResource { JSONObject response = new JSONObject(); response.put(AtlasClient.REQUEST_ID, Servlets.getRequestId()); - response.put(AtlasClient.GUID, guid); return Response.created(locationURI).entity(response).build(); } catch (EntityNotFoundException | TypeNotFoundException e) { @@ -588,7 +586,6 @@ public class EntityResource { JSONObject response = new JSONObject(); response.put(AtlasClient.REQUEST_ID, Servlets.getRequestId()); - response.put(AtlasClient.GUID, guid); response.put(TRAIT_NAME, traitName); return Response.ok(response).build(); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/705014eb/webapp/src/main/java/org/apache/atlas/web/service/ServiceModule.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/service/ServiceModule.java b/webapp/src/main/java/org/apache/atlas/web/service/ServiceModule.java index 0f8bcb1..2128b7c 100644 --- a/webapp/src/main/java/org/apache/atlas/web/service/ServiceModule.java +++ b/webapp/src/main/java/org/apache/atlas/web/service/ServiceModule.java @@ -23,7 +23,7 @@ import com.google.inject.multibindings.Multibinder; import org.apache.atlas.kafka.KafkaNotification; import org.apache.atlas.listener.EntityChangeListener; import org.apache.atlas.notification.NotificationHookConsumer; -import org.apache.atlas.notification.entity.NotificationEntityChangeListener; +import org.apache.atlas.notification.NotificationEntityChangeListener; import org.apache.atlas.service.Service; public class ServiceModule extends AbstractModule { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/705014eb/webapp/src/test/java/org/apache/atlas/LocalAtlasClientTest.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/LocalAtlasClientTest.java b/webapp/src/test/java/org/apache/atlas/LocalAtlasClientTest.java index 7f20652..e774399 100644 --- a/webapp/src/test/java/org/apache/atlas/LocalAtlasClientTest.java +++ b/webapp/src/test/java/org/apache/atlas/LocalAtlasClientTest.java @@ -23,7 +23,6 @@ import org.apache.atlas.typesystem.Referenceable; import org.apache.atlas.web.resources.EntityResource; import org.apache.atlas.web.service.ServiceState; import org.apache.commons.lang.RandomStringUtils; -import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONObject; import org.mockito.Mock; import org.mockito.MockitoAnnotations; @@ -36,6 +35,7 @@ import javax.ws.rs.core.Response; import java.util.Arrays; import java.util.List; +import static org.apache.atlas.AtlasClient.ENTITIES; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyListOf; import static org.mockito.Matchers.anyString; @@ -64,7 +64,8 @@ public class LocalAtlasClientTest { when(entityResource.submit(any(HttpServletRequest.class))).thenReturn(response); final String guid = random(); when(response.getEntity()).thenReturn(new JSONObject() {{ - put(AtlasClient.GUID, new JSONArray(Arrays.asList(guid))); + put(ENTITIES, new JSONObject( + new AtlasClient.EntityResult(Arrays.asList(guid), null, null).toString()).get(ENTITIES)); }}); LocalAtlasClient atlasClient = new LocalAtlasClient(serviceState, entityResource); @@ -119,12 +120,14 @@ public class LocalAtlasClientTest { when(entityResource.updateByUniqueAttribute(anyString(), anyString(), anyString(), any(HttpServletRequest.class))).thenReturn(response); when(response.getEntity()).thenReturn(new JSONObject() {{ - put(AtlasClient.GUID, guid); + put(ENTITIES, new JSONObject( + new AtlasClient.EntityResult(null, Arrays.asList(guid), null).toString()).get(ENTITIES)); }}); LocalAtlasClient atlasClient = new LocalAtlasClient(serviceState, entityResource); - String actualId = atlasClient.updateEntity(random(), random(), random(), new Referenceable(random())); - assertEquals(actualId, guid); + AtlasClient.EntityResult + entityResult = atlasClient.updateEntity(random(), random(), random(), new Referenceable(random())); + assertEquals(entityResult.getUpdateEntities(), Arrays.asList(guid)); } @Test @@ -132,14 +135,14 @@ public class LocalAtlasClientTest { final String guid = random(); Response response = mock(Response.class); when(response.getEntity()).thenReturn(new JSONObject() {{ - put(AtlasClient.GUID, new JSONArray(Arrays.asList(guid))); + put(ENTITIES, new JSONObject( + new AtlasClient.EntityResult(null, null, Arrays.asList(guid)).toString()).get(ENTITIES)); }}); when(entityResource.deleteEntities(anyListOf(String.class), anyString(), anyString(), anyString())).thenReturn(response); LocalAtlasClient atlasClient = new LocalAtlasClient(serviceState, entityResource); - List<String> results = atlasClient.deleteEntity(random(), random(), random()); - assertEquals(results.size(), 1); - assertEquals(results.get(0), guid); + AtlasClient.EntityResult entityResult = atlasClient.deleteEntity(random(), random(), random()); + assertEquals(entityResult.getDeletedEntities(), Arrays.asList(guid)); } private String random() { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/705014eb/webapp/src/test/java/org/apache/atlas/notification/EntityNotificationIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/notification/EntityNotificationIT.java b/webapp/src/test/java/org/apache/atlas/notification/EntityNotificationIT.java index 6985152..52f5b83 100644 --- a/webapp/src/test/java/org/apache/atlas/notification/EntityNotificationIT.java +++ b/webapp/src/test/java/org/apache/atlas/notification/EntityNotificationIT.java @@ -22,7 +22,6 @@ import com.google.common.collect.ImmutableSet; import com.google.inject.Inject; import com.sun.jersey.api.client.ClientResponse; import com.sun.jersey.api.client.WebResource; - import org.apache.atlas.notification.entity.EntityNotification; import org.apache.atlas.typesystem.IReferenceableInstance; import org.apache.atlas.typesystem.IStruct; @@ -43,7 +42,6 @@ import org.testng.annotations.Test; import javax.ws.rs.HttpMethod; import javax.ws.rs.core.Response; - import java.util.Collections; import java.util.LinkedList; import java.util.List; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/705014eb/webapp/src/test/java/org/apache/atlas/notification/NotificationEntityChangeListenerTest.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/notification/NotificationEntityChangeListenerTest.java b/webapp/src/test/java/org/apache/atlas/notification/NotificationEntityChangeListenerTest.java new file mode 100644 index 0000000..a988915 --- /dev/null +++ b/webapp/src/test/java/org/apache/atlas/notification/NotificationEntityChangeListenerTest.java @@ -0,0 +1,90 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.notification; + +import org.apache.atlas.typesystem.IStruct; +import org.apache.atlas.typesystem.Referenceable; +import org.apache.atlas.typesystem.Struct; +import org.apache.atlas.typesystem.types.TraitType; +import org.apache.atlas.typesystem.types.TypeSystem; +import org.testng.annotations.Test; + +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; + +public class NotificationEntityChangeListenerTest { + @Test + public void testGetAllTraitsSuperTraits() throws Exception { + + TypeSystem typeSystem = mock(TypeSystem.class); + + String traitName = "MyTrait"; + IStruct myTrait = new Struct(traitName); + + String superTraitName = "MySuperTrait"; + + TraitType traitDef = mock(TraitType.class); + Set<String> superTypeNames = Collections.singleton(superTraitName); + + TraitType superTraitDef = mock(TraitType.class); + Set<String> superSuperTypeNames = Collections.emptySet(); + + Referenceable entity = getEntity("id", myTrait); + + when(typeSystem.getDataType(TraitType.class, traitName)).thenReturn(traitDef); + when(typeSystem.getDataType(TraitType.class, superTraitName)).thenReturn(superTraitDef); + + when(traitDef.getAllSuperTypeNames()).thenReturn(superTypeNames); + when(superTraitDef.getAllSuperTypeNames()).thenReturn(superSuperTypeNames); + + List<IStruct> allTraits = NotificationEntityChangeListener.getAllTraits(entity, typeSystem); + + assertEquals(2, allTraits.size()); + + for (IStruct trait : allTraits) { + String typeName = trait.getTypeName(); + assertTrue(typeName.equals(traitName) || typeName.equals(superTraitName)); + } + } + + private Referenceable getEntity(String id, IStruct... traits) { + String typeName = "typeName"; + Map<String, Object> values = new HashMap<>(); + + List<String> traitNames = new LinkedList<>(); + Map<String, IStruct> traitMap = new HashMap<>(); + + for (IStruct trait : traits) { + String traitName = trait.getTypeName(); + + traitNames.add(traitName); + traitMap.put(traitName, trait); + } + return new Referenceable(id, typeName, values, traitNames, traitMap); + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/705014eb/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java b/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java index aa92bc0..36db646 100755 --- a/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java +++ b/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java @@ -71,6 +71,7 @@ import java.util.Map; import java.util.UUID; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; import static org.testng.Assert.fail; @@ -154,7 +155,10 @@ public class EntityJerseyResourceIT extends BaseResourceIT { JSONObject response = new JSONObject(responseAsString); Assert.assertNotNull(response.get(AtlasClient.REQUEST_ID)); - Assert.assertNotNull(response.get(AtlasClient.GUID)); + + AtlasClient.EntityResult entityResult = AtlasClient.EntityResult.fromString(response.toString()); + assertEquals(entityResult.getCreatedEntities().size(), 1); + assertNotNull(entityResult.getCreatedEntities().get(0)); } @Test @@ -376,7 +380,9 @@ public class EntityJerseyResourceIT extends BaseResourceIT { } private void addProperty(String guid, String property, String value) throws AtlasServiceException { - serviceClient.updateEntityAttribute(guid, property, value); + AtlasClient.EntityResult entityResult = serviceClient.updateEntityAttribute(guid, property, value); + assertEquals(entityResult.getUpdateEntities().size(), 1); + assertEquals(entityResult.getUpdateEntities().get(0), guid); } private ClientResponse getEntityDefinition(String guid) { @@ -482,7 +488,6 @@ public class EntityJerseyResourceIT extends BaseResourceIT { JSONObject response = new JSONObject(responseAsString); Assert.assertNotNull(response.get(AtlasClient.REQUEST_ID)); - Assert.assertNotNull(response.get("GUID")); final JSONArray list = response.getJSONArray(AtlasClient.RESULTS); Assert.assertEquals(list.length(), 7); @@ -513,7 +518,6 @@ public class EntityJerseyResourceIT extends BaseResourceIT { JSONObject response = new JSONObject(responseAsString); Assert.assertNotNull(response.get(AtlasClient.REQUEST_ID)); - Assert.assertNotNull(response.get(AtlasClient.GUID)); assertEntityAudit(guid, EntityAuditEvent.EntityAuditAction.TAG_ADD); } @@ -561,7 +565,6 @@ public class EntityJerseyResourceIT extends BaseResourceIT { JSONObject response = new JSONObject(responseAsString); Assert.assertNotNull(response.get(AtlasClient.REQUEST_ID)); - Assert.assertNotNull(response.get(AtlasClient.GUID)); // verify the response clientResponse = getEntityDefinition(guid); @@ -612,7 +615,6 @@ public class EntityJerseyResourceIT extends BaseResourceIT { JSONObject response = new JSONObject(responseAsString); Assert.assertNotNull(response.get(AtlasClient.REQUEST_ID)); - Assert.assertNotNull(response.get("GUID")); Assert.assertNotNull(response.get("traitName")); assertEntityAudit(guid, EntityAuditEvent.EntityAuditAction.TAG_DELETE); } @@ -635,7 +637,8 @@ public class EntityJerseyResourceIT extends BaseResourceIT { "trait=" + traitName + " should be defined in type system before it can be deleted"); Assert.assertNotNull(response.get(AtlasClient.STACKTRACE)); } -@Test(dependsOnMethods = "testSubmitEntity()") + + @Test(dependsOnMethods = "testSubmitEntity()") public void testDeleteExistentTraitNonExistentForEntity() throws Exception { final String guid = tableId._getId(); @@ -704,7 +707,9 @@ public class EntityJerseyResourceIT extends BaseResourceIT { }}); LOG.debug("Updating entity= " + tableUpdated); - serviceClient.updateEntity(tableId._getId(), tableUpdated); + AtlasClient.EntityResult entityResult = serviceClient.updateEntity(tableId._getId(), tableUpdated); + assertEquals(entityResult.getUpdateEntities().size(), 1); + assertEquals(entityResult.getUpdateEntities().get(0), tableId._getId()); ClientResponse response = getEntityDefinition(tableId._getId()); String definition = getEntityDefinition(response); @@ -722,8 +727,10 @@ public class EntityJerseyResourceIT extends BaseResourceIT { }}); LOG.debug("Updating entity= " + tableUpdated); - serviceClient.updateEntity(BaseResourceIT.HIVE_TABLE_TYPE, "name", (String) tableInstance.get("name"), - tableUpdated); + entityResult = serviceClient.updateEntity(BaseResourceIT.HIVE_TABLE_TYPE, "name", + (String) tableInstance.get("name"), tableUpdated); + assertEquals(entityResult.getUpdateEntities().size(), 1); + assertEquals(entityResult.getUpdateEntities().get(0), tableId._getId()); response = getEntityDefinition(tableId._getId()); definition = getEntityDefinition(response); @@ -732,7 +739,6 @@ public class EntityJerseyResourceIT extends BaseResourceIT { Assert.assertTrue(refs.get(0).equalsContents(columns.get(0))); Assert.assertEquals(refs.get(0).get("dataType"), "int"); - } @Test(dependsOnMethods = "testSubmitEntity") @@ -765,9 +771,8 @@ public class EntityJerseyResourceIT extends BaseResourceIT { // ATLAS-586: verify response entity can be parsed by GSON. String entity = clientResponse.getEntity(String.class); Gson gson = new Gson(); - UpdateEntitiesResponse updateEntitiesResponse = null; try { - updateEntitiesResponse = gson.fromJson(entity, UpdateEntitiesResponse.class); + UpdateEntitiesResponse updateEntitiesResponse = gson.fromJson(entity, UpdateEntitiesResponse.class); } catch (JsonSyntaxException e) { Assert.fail("Response entity from " + service.path(ENTITIES).getURI() + " not parseable by GSON", e); @@ -785,7 +790,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT { private static class UpdateEntitiesResponse { String requestId; - String[] GUID; + AtlasClient.EntityResult entities; AtlasEntity definition; } @@ -811,15 +816,9 @@ public class EntityJerseyResourceIT extends BaseResourceIT { queryParam(AtlasClient.GUID.toLowerCase(), db1Id._getId()). queryParam(AtlasClient.GUID.toLowerCase(), db2Id._getId()). accept(Servlets.JSON_MEDIA_TYPE).type(Servlets.JSON_MEDIA_TYPE).method(HttpMethod.DELETE, ClientResponse.class); + JSONObject response = getEntity(clientResponse); - final String deletedGuidsJson = response.getString(AtlasClient.GUID); - Assert.assertNotNull(deletedGuidsJson); - JSONArray guidsArray = new JSONArray(deletedGuidsJson); - Assert.assertEquals(guidsArray.length(), 2); - List<String> deletedGuidsList = new ArrayList<>(2); - for (int index = 0; index < guidsArray.length(); index++) { - deletedGuidsList.add(guidsArray.getString(index)); - } + List<String> deletedGuidsList = AtlasClient.EntityResult.fromString(response.toString()).getDeletedEntities(); Assert.assertTrue(deletedGuidsList.contains(db1Id._getId())); Assert.assertTrue(deletedGuidsList.contains(db2Id._getId())); @@ -843,7 +842,8 @@ public class EntityJerseyResourceIT extends BaseResourceIT { Id db2Id = createInstance(db2); // Delete the database entities - List<String> deletedGuidsList = serviceClient.deleteEntities(db1Id._getId(), db2Id._getId()); + List<String> deletedGuidsList = + serviceClient.deleteEntities(db1Id._getId(), db2Id._getId()).getDeletedEntities(); // Verify that deleteEntities() response has database entity guids Assert.assertEquals(deletedGuidsList.size(), 2); @@ -867,7 +867,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT { Id db1Id = createInstance(db1); // Delete the database entity - List<String> deletedGuidsList = serviceClient.deleteEntity(DATABASE_TYPE, "name", dbName); + List<String> deletedGuidsList = serviceClient.deleteEntity(DATABASE_TYPE, "name", dbName).getDeletedEntities(); // Verify that deleteEntities() response has database entity guids Assert.assertEquals(deletedGuidsList.size(), 1); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/705014eb/webapp/src/test/java/org/apache/atlas/web/service/CuratorFactoryTest.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/web/service/CuratorFactoryTest.java b/webapp/src/test/java/org/apache/atlas/web/service/CuratorFactoryTest.java index 2d510a0..0e48509 100644 --- a/webapp/src/test/java/org/apache/atlas/web/service/CuratorFactoryTest.java +++ b/webapp/src/test/java/org/apache/atlas/web/service/CuratorFactoryTest.java @@ -23,21 +23,16 @@ import org.apache.atlas.ha.HAConfiguration; import org.apache.commons.configuration.Configuration; import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.framework.api.ACLProvider; -import org.apache.zookeeper.ZooDefs; import org.apache.zookeeper.data.ACL; import org.mockito.ArgumentMatcher; import org.mockito.Mock; -import org.mockito.Mockito; import org.mockito.MockitoAnnotations; -import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; -import static junit.framework.TestCase.assertEquals; import static org.mockito.Matchers.any; import static org.mockito.Matchers.argThat; import static org.mockito.Matchers.eq; -import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyZeroInteractions; import static org.mockito.Mockito.when;
