Repository: incubator-atlas Updated Branches: refs/heads/master 98f4d40a1 -> ef9ef3c10
ATLAS-607 Add Support for delete entity through a qualifiedName (sumasai via yhemanth) Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/ef9ef3c1 Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/ef9ef3c1 Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/ef9ef3c1 Branch: refs/heads/master Commit: ef9ef3c10e8caf0ecf0cf729fe19809d5c4fff09 Parents: 98f4d40 Author: Suma Shivaprasad <[email protected]> Authored: Mon Apr 4 17:51:32 2016 -0700 Committer: Suma Shivaprasad <[email protected]> Committed: Mon Apr 4 17:51:32 2016 -0700 ---------------------------------------------------------------------- .../main/java/org/apache/atlas/AtlasClient.java | 19 ++++- release-log.txt | 1 + .../graph/GraphBackedMetadataRepository.java | 2 - .../atlas/services/DefaultMetadataService.java | 21 ++++- .../service/DefaultMetadataServiceTest.java | 82 +++++++++++++++++--- .../apache/atlas/services/MetadataService.java | 11 +++ .../atlas/web/resources/EntityResource.java | 35 +++++++-- .../web/resources/EntityJerseyResourceIT.java | 30 +++++++ 8 files changed, 180 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ef9ef3c1/client/src/main/java/org/apache/atlas/AtlasClient.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/atlas/AtlasClient.java b/client/src/main/java/org/apache/atlas/AtlasClient.java index 21f21eb..ccd6fbf 100755 --- a/client/src/main/java/org/apache/atlas/AtlasClient.java +++ b/client/src/main/java/org/apache/atlas/AtlasClient.java @@ -346,7 +346,7 @@ public class AtlasClient { UPDATE_ENTITY_PARTIAL(BASE_URI + URI_ENTITY, HttpMethod.POST, Response.Status.OK), LIST_ENTITIES(BASE_URI + URI_ENTITY, HttpMethod.GET, Response.Status.OK), DELETE_ENTITIES(BASE_URI + URI_ENTITY, HttpMethod.DELETE, Response.Status.OK), - + DELETE_ENTITY(BASE_URI + URI_ENTITY, HttpMethod.DELETE, Response.Status.OK), //Trait operations ADD_TRAITS(BASE_URI + URI_ENTITY, HttpMethod.POST, Response.Status.CREATED), @@ -610,6 +610,23 @@ public class AtlasClient { }); return extractResults(jsonResponse, GUID); } + + /** + * Supports Deletion of an entity identified by its unique attribute value + * @param entityType Type of the entity being deleted + * @param uniqueAttributeName Attribute Name that uniquely identifies the entity + * @param uniqueAttributeValue Attribute Value that uniquely identifies the entity + * @return List of deleted entity guids(including composite references from that entity) + */ + public List<String> deleteEntity(String entityType, String uniqueAttributeName, String uniqueAttributeValue) throws AtlasServiceException { + API api = API.DELETE_ENTITY; + WebResource resource = getResource(api); + resource = resource.queryParam(TYPE, entityType); + resource = resource.queryParam(ATTRIBUTE_NAME, uniqueAttributeName); + resource = resource.queryParam(ATTRIBUTE_VALUE, uniqueAttributeValue); + JSONObject jsonResponse = callAPIWithResource(API.DELETE_ENTITIES, resource, null); + return extractResults(jsonResponse, GUID); + } /** * Get an entity given the entity id http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ef9ef3c1/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 10fcdbb..1ae8a72 100644 --- a/release-log.txt +++ b/release-log.txt @@ -13,6 +13,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags) ALL CHANGES: +ATLAS-607 Add Support for delete entity through a qualifiedName (sumasai via yhemanth) ATLAS-571 Modify Atlas client for necessary changes in context of HA (yhemanth via sumasai) ATLAS-620 Disable hbase based entity audit (shwethags) ATLAS-618 Fix assembly for hdfs-module (sumasai via yhemanth) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ef9ef3c1/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java index 8a7c76e..b94ff5a 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java @@ -252,8 +252,6 @@ public class GraphBackedMetadataRepository implements MetadataRepository { } try { - - final String entityTypeName = GraphHelper.getTypeName(instanceVertex); String relationshipLabel = GraphHelper.getTraitLabel(entityTypeName, traitNameToBeDeleted); Iterator<Edge> results = instanceVertex.getEdges(Direction.OUT, relationshipLabel).iterator(); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ef9ef3c1/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java b/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java index cd1161a..0a04c5f 100755 --- a/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java +++ b/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java @@ -76,7 +76,6 @@ import java.util.Collection; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; -import java.util.concurrent.atomic.AtomicBoolean; /** * Simple wrapper over TypeSystem and MetadataRepository services with hooks @@ -691,7 +690,25 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang */ @Override public List<String> deleteEntities(List<String> deleteCandidateGuids) throws AtlasException { - ParamChecker.notEmpty(deleteCandidateGuids, "delete candidate guids cannot be empty"); + ParamChecker.notEmpty(deleteCandidateGuids, "delete candidate guids"); + return deleteGuids(deleteCandidateGuids); + } + + @Override + public List<String> deleteEntityByUniqueAttribute(String typeName, String uniqueAttributeName, String attrValue) throws AtlasException { + ParamChecker.notEmpty(typeName, "delete candidate typeName"); + ParamChecker.notEmpty(uniqueAttributeName, "delete candidate unique attribute name"); + ParamChecker.notEmpty(attrValue, "delete candidate unique attribute value"); + + //Throws EntityNotFoundException if the entity could not be found by its unique attribute + ITypedReferenceableInstance instance = getEntityDefinitionReference(typeName, uniqueAttributeName, attrValue); + final Id instanceId = instance.getId(); + List<String> deleteCandidateGuids = new ArrayList<String>() {{ add(instanceId._getId());}}; + + return deleteGuids(deleteCandidateGuids); + } + + private List<String> deleteGuids(List<String> deleteCandidateGuids) throws AtlasException { Pair<List<String>, List<ITypedReferenceableInstance>> deleteEntitiesResult = repository.deleteEntities(deleteCandidateGuids); if (deleteEntitiesResult.right.size() > 0) { onEntitiesDeleted(deleteEntitiesResult.right); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ef9ef3c1/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java b/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java index aa00417..156eb3d 100644 --- a/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java +++ b/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java @@ -24,14 +24,21 @@ import com.google.inject.Inject; import com.thinkaurelius.titan.core.TitanGraph; import com.thinkaurelius.titan.core.util.TitanCleanup; import org.apache.atlas.AtlasClient; +import org.apache.atlas.repository.audit.EntityAuditRepository; +import org.apache.atlas.repository.audit.HBaseBasedAuditRepository; +import org.apache.atlas.repository.audit.HBaseTestUtils; +import org.apache.atlas.typesystem.exception.TypeNotFoundException; +import org.apache.atlas.typesystem.exception.EntityNotFoundException; +import org.apache.atlas.typesystem.types.ClassType; +import org.apache.atlas.typesystem.types.DataTypes; +import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition; +import org.apache.atlas.typesystem.types.utils.TypesUtil; +import org.apache.atlas.utils.ParamChecker; import org.apache.atlas.AtlasException; import org.apache.atlas.RepositoryMetadataModule; import org.apache.atlas.RequestContext; import org.apache.atlas.TestUtils; import org.apache.atlas.listener.EntityChangeListener; -import org.apache.atlas.repository.audit.EntityAuditRepository; -import org.apache.atlas.repository.audit.HBaseBasedAuditRepository; -import org.apache.atlas.repository.audit.HBaseTestUtils; import org.apache.atlas.repository.graph.GraphProvider; import org.apache.atlas.services.MetadataService; import org.apache.atlas.typesystem.IReferenceableInstance; @@ -40,19 +47,12 @@ import org.apache.atlas.typesystem.ITypedReferenceableInstance; import org.apache.atlas.typesystem.Referenceable; import org.apache.atlas.typesystem.Struct; import org.apache.atlas.typesystem.TypesDef; -import org.apache.atlas.typesystem.exception.EntityNotFoundException; -import org.apache.atlas.typesystem.exception.TypeNotFoundException; import org.apache.atlas.typesystem.json.InstanceSerialization; import org.apache.atlas.typesystem.json.TypesSerialization; import org.apache.atlas.typesystem.persistence.Id; -import org.apache.atlas.typesystem.types.ClassType; -import org.apache.atlas.typesystem.types.DataTypes; import org.apache.atlas.typesystem.types.EnumValue; -import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition; import org.apache.atlas.typesystem.types.TypeSystem; import org.apache.atlas.typesystem.types.ValueConversionException; -import org.apache.atlas.typesystem.types.utils.TypesUtil; -import org.apache.atlas.utils.ParamChecker; import org.apache.commons.lang.RandomStringUtils; import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONException; @@ -919,6 +919,68 @@ public class DefaultMetadataServiceTest { } @Test + public void testDeleteEntityByUniqueAttribute() throws Exception { + // Create 2 table entities, each with 3 composite column entities + Referenceable dbEntity = createDBEntity(); + String dbGuid = createInstance(dbEntity); + Id dbId = new Id(dbGuid, 0, TestUtils.DATABASE_TYPE); + Referenceable table1Entity = createTableEntity(dbId); + Referenceable col1 = createColumnEntity(); + Referenceable col2 = createColumnEntity(); + Referenceable col3 = createColumnEntity(); + table1Entity.set(COLUMNS_ATTR_NAME, ImmutableList.of(col1, col2, col3)); + createInstance(table1Entity); + + // to get their guids and the composite column guids. + String entityJson = metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, + NAME, (String)table1Entity.get(NAME)); + Assert.assertNotNull(entityJson); + table1Entity = InstanceSerialization.fromJsonReferenceable(entityJson, true); + Object val = table1Entity.get(COLUMNS_ATTR_NAME); + Assert.assertTrue(val instanceof List); + List<IReferenceableInstance> table1Columns = (List<IReferenceableInstance>) val; + + // Register an EntityChangeListener to verify the notification mechanism + // is working for deleteEntityByUniqueAttribute(). + DeleteEntitiesChangeListener listener = new DeleteEntitiesChangeListener(); + metadataService.registerListener(listener); + + // Delete the table entities. The deletion should cascade + // to their composite columns. + List<String> deletedGuids = metadataService.deleteEntityByUniqueAttribute(TestUtils.TABLE_TYPE, NAME, (String) table1Entity.get(NAME)); + + // Verify that deleteEntities() response has guids for tables and their composite columns. + Assert.assertTrue(deletedGuids.contains(table1Entity.getId()._getId())); + for (IReferenceableInstance column : table1Columns) { + Assert.assertTrue(deletedGuids.contains(column.getId()._getId())); + } + + // Verify that tables and their composite columns have been deleted from the repository. + for (String guid : deletedGuids) { + try { + metadataService.getEntityDefinition(guid); + Assert.fail(EntityNotFoundException.class.getSimpleName() + + " expected but not thrown. The entity with guid " + guid + + " still exists in the repository after being deleted." ); + } + catch(EntityNotFoundException e) { + // The entity does not exist in the repository, so deletion was successful. + } + } + + // Verify that the listener was notified about the deleted entities. + Collection<ITypedReferenceableInstance> deletedEntitiesFromListener = listener.getDeletedEntities(); + Assert.assertNotNull(deletedEntitiesFromListener); + Assert.assertEquals(deletedEntitiesFromListener.size(), deletedGuids.size()); + List<String> deletedGuidsFromListener = new ArrayList<>(deletedGuids.size()); + for (ITypedReferenceableInstance deletedEntity : deletedEntitiesFromListener) { + deletedGuidsFromListener.add(deletedEntity.getId()._getId()); + } + Assert.assertEquals(deletedGuidsFromListener.size(), deletedGuids.size()); + Assert.assertTrue(deletedGuidsFromListener.containsAll(deletedGuids)); + } + + @Test public void testTypeUpdateWithReservedAttributes() throws AtlasException, JSONException { String typeName = "test_type_"+ RandomStringUtils.randomAlphanumeric(10); HierarchicalTypeDefinition<ClassType> typeDef = TypesUtil.createClassTypeDef( http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ef9ef3c1/server-api/src/main/java/org/apache/atlas/services/MetadataService.java ---------------------------------------------------------------------- diff --git a/server-api/src/main/java/org/apache/atlas/services/MetadataService.java b/server-api/src/main/java/org/apache/atlas/services/MetadataService.java index ab402d7..a2c347d 100644 --- a/server-api/src/main/java/org/apache/atlas/services/MetadataService.java +++ b/server-api/src/main/java/org/apache/atlas/services/MetadataService.java @@ -196,4 +196,15 @@ public interface MetadataService { * @param listener the listener to unregister */ void unregisterListener(EntityChangeListener listener); + + /** + * Delete the specified entity from the repository identified by its unique attribute (including its composite references) + * + * @param typeName The entity's type + * @param uniqueAttributeName attribute name by which the entity could be identified uniquely + * @param attrValue attribute value by which the entity could be identified uniquely + * @return List of guids for deleted entities (including their composite references) + * @throws AtlasException + */ + List<String> deleteEntityByUniqueAttribute(String typeName, String uniqueAttributeName, String attrValue) throws AtlasException; } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ef9ef3c1/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java index 53c503b..f39a80e 100755 --- a/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java +++ b/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java @@ -56,6 +56,7 @@ import javax.ws.rs.core.Response; import javax.ws.rs.core.UriBuilder; import javax.ws.rs.core.UriInfo; import java.net.URI; +import java.util.ArrayList; import java.util.List; @@ -313,19 +314,34 @@ public class EntityResource { } /** - * Delete entities from the repository + * Delete entities from the repository identified by their guids (including their composite references) + * or + * Deletes a single entity identified by its type and unique attribute value from the repository (including their composite references) * - * @param guids deletion candidate guids - * @param request - * @return response payload as json + * @param guids list of deletion candidate guids + * or + * @param entityType the entity type + * @param attribute the unique attribute used to identify the entity + * @param value the unique attribute value used to identify the entity + * @param request - Ignored + * @return response payload as json - including guids of entities(including composite references from that entity) that were deleted */ @DELETE @Consumes(Servlets.JSON_MEDIA_TYPE) @Produces(Servlets.JSON_MEDIA_TYPE) - public Response deleteEntities(@QueryParam("guid") List<String> guids, @Context HttpServletRequest request) { + public Response deleteEntities(@QueryParam("guid") List<String> guids, + @QueryParam("type") String entityType, + @QueryParam("property") String attribute, + @QueryParam("value") String value, + @Context HttpServletRequest request) { try { - List<String> deletedGuids = metadataService.deleteEntities(guids); + List<String> deletedGuids = new ArrayList<>(); + if (guids != null && !guids.isEmpty()) { + deletedGuids = metadataService.deleteEntities(guids); + } else { + deletedGuids = metadataService.deleteEntityByUniqueAttribute(entityType, attribute, value); + } JSONObject response = new JSONObject(); response.put(AtlasClient.REQUEST_ID, Servlets.getRequestId()); JSONArray guidArray = new JSONArray(deletedGuids.size()); @@ -334,6 +350,13 @@ public class EntityResource { } response.put(AtlasClient.GUID, guidArray); return Response.ok(response).build(); + } catch (EntityNotFoundException e) { + if(guids != null || !guids.isEmpty()) { + LOG.error("An entity with GUID={} does not exist", guids, e); + } else { + LOG.error("An entity with qualifiedName {}-{}-{} does not exist", entityType, attribute, value, e); + } + throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.NOT_FOUND)); } catch (AtlasException | IllegalArgumentException e) { LOG.error("Unable to delete entities {}", guids, e); throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ef9ef3c1/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java b/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java index a8cd1ef..6a939d3 100755 --- a/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java +++ b/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java @@ -808,4 +808,34 @@ public class EntityJerseyResourceIT extends BaseResourceIT { } } + @Test + public void testDeleteEntityByUniqAttribute() throws Exception { + // Create database entity + Referenceable db1 = new Referenceable(DATABASE_TYPE); + String dbName = randomString(); + db1.set("name", dbName); + db1.set("description", randomString()); + Id db1Id = createInstance(db1); + + // Delete the database entity + List<String> deletedGuidsList = serviceClient.deleteEntity(DATABASE_TYPE, "name", dbName); + + // Verify that deleteEntities() response has database entity guids + Assert.assertEquals(deletedGuidsList.size(), 1); + Assert.assertTrue(deletedGuidsList.contains(db1Id._getId())); + + // Verify entities were deleted from the repository. + for (String guid : deletedGuidsList) { + try { + serviceClient.getEntity(guid); + Assert.fail(AtlasServiceException.class.getSimpleName() + + " was expected but not thrown. The entity with guid " + guid + + " still exists in the repository after being deleted."); + } + catch (AtlasServiceException e) { + Assert.assertTrue(e.getMessage().contains(Integer.toString(Response.Status.NOT_FOUND.getStatusCode()))); + } + } + } + }
