ATLAS-712 Support getTrait() API (svimal2106 via shwethags) (cherry picked from commit dd0b051ec58ca292fca1862701e1089ad408e18d)
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/0de7ce34 Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/0de7ce34 Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/0de7ce34 Branch: refs/heads/0.7-incubating Commit: 0de7ce34b10ad6d282c7ce01343e852aef93b38e Parents: dc88832 Author: Shwetha GS <[email protected]> Authored: Mon Sep 19 10:46:34 2016 +0530 Committer: Madhan Neethiraj <[email protected]> Committed: Thu Dec 22 15:32:41 2016 -0800 ---------------------------------------------------------------------- .../main/java/org/apache/atlas/AtlasClient.java | 39 +++++++++ release-log.txt | 1 + .../atlas/services/DefaultMetadataService.java | 7 +- .../service/DefaultMetadataServiceTest.java | 7 +- .../apache/atlas/services/MetadataService.java | 3 +- .../atlas/web/resources/EntityResource.java | 83 ++++++++++++++++++++ .../web/resources/EntityJerseyResourceIT.java | 32 ++++++++ 7 files changed, 162 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/0de7ce34/client/src/main/java/org/apache/atlas/AtlasClient.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/atlas/AtlasClient.java b/client/src/main/java/org/apache/atlas/AtlasClient.java index 5ed79bc..6c13ec8 100755 --- a/client/src/main/java/org/apache/atlas/AtlasClient.java +++ b/client/src/main/java/org/apache/atlas/AtlasClient.java @@ -100,6 +100,9 @@ public class AtlasClient { public static final String URI_NAME_LINEAGE = "lineage/hive/table"; public static final String URI_LINEAGE = "lineage/"; public static final String URI_TRAITS = "traits"; + public static final String TRAITS = "traits"; + public static final String TRAIT_DEFINITIONS = "traitDefinitions"; + public static final String QUERY = "query"; public static final String LIMIT = "limit"; @@ -492,6 +495,8 @@ public class AtlasClient { ADD_TRAITS(BASE_URI + URI_ENTITY, HttpMethod.POST, Response.Status.CREATED), DELETE_TRAITS(BASE_URI + URI_ENTITY, HttpMethod.DELETE, Response.Status.OK), LIST_TRAITS(BASE_URI + URI_ENTITY, HttpMethod.GET, Response.Status.OK), + GET_ALL_TRAIT_DEFINITIONS(BASE_URI + URI_ENTITY, HttpMethod.GET, Response.Status.OK), + GET_TRAIT_DEFINITION(BASE_URI + URI_ENTITY, HttpMethod.GET, Response.Status.OK), //Search operations SEARCH(BASE_URI + URI_SEARCH, HttpMethod.GET, Response.Status.OK), @@ -987,6 +992,40 @@ public class AtlasClient { return extractResults(jsonResponse, AtlasClient.RESULTS, new ExtractOperation<String, String>()); } + /** + * Get all trait definitions for an entity + * @param guid GUID of the entity + * @return List<String> trait definitions of the traits associated to the entity + * @throws AtlasServiceException + */ + public List<Struct> listTraitDefinitions(final String guid) throws AtlasServiceException{ + JSONObject jsonResponse = callAPI(API.GET_ALL_TRAIT_DEFINITIONS, null, guid, TRAIT_DEFINITIONS); + List<JSONObject> traitDefList = extractResults(jsonResponse, AtlasClient.RESULTS, new ExtractOperation<JSONObject, JSONObject>()); + ArrayList<Struct> traitStructList = new ArrayList<>(); + for(JSONObject traitDef:traitDefList){ + Struct traitStruct = InstanceSerialization.fromJsonStruct(traitDef.toString(), true); + traitStructList.add(traitStruct); + } + return traitStructList; + } + + /** + * Get trait definition for a given entity and traitname + * @param guid GUID of the entity + * @param traitName + * @return trait definition + * @throws AtlasServiceException + */ + public Struct getTraitDefinition(final String guid, final String traitName) throws AtlasServiceException{ + JSONObject jsonResponse = callAPI(API.GET_TRAIT_DEFINITION, null, guid, TRAIT_DEFINITIONS, traitName); + + try { + return InstanceSerialization.fromJsonStruct(jsonResponse.getString(AtlasClient.RESULTS), false); + }catch (JSONException e){ + throw new AtlasServiceException(API.GET_TRAIT_DEFINITION, e); + } + } + protected class ExtractOperation<T, U> { T extractElement(U element) throws JSONException { return (T) element; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/0de7ce34/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 3d3b829..344d959 100644 --- a/release-log.txt +++ b/release-log.txt @@ -30,6 +30,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags) ALL CHANGES: +ATLAS-712 Support getTrait() API (svimal2106 via shwethags) ATLAS-1173 Doc: Minor editorial bug in the example given for property atlas.server.ha.zookeeper.auth (yhemanth via shwethags) ATLAS-1133 Jetty Server start doesn't throw exception when user-credential.properties file is not found (nixonrodrigues,svimal2106 via kevalbhatt) ATLAS-1149 Changes to UI to sort the hive table schema based on "position" attribute of hive_column (Kalyanikashikar via kevalbhatt) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/0de7ce34/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java b/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java index 6a937f4..fdb749a 100755 --- a/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java +++ b/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java @@ -22,7 +22,6 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.inject.Provider; - import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasException; @@ -33,7 +32,6 @@ import org.apache.atlas.ha.HAConfiguration; import org.apache.atlas.listener.ActiveStateChangeHandler; import org.apache.atlas.listener.EntityChangeListener; import org.apache.atlas.listener.TypesChangeListener; -import org.apache.atlas.query.QueryKeywords; import org.apache.atlas.query.QueryParser; import org.apache.atlas.repository.MetadataRepository; import org.apache.atlas.repository.RepositoryException; @@ -77,7 +75,6 @@ import scala.collection.Set; import javax.inject.Inject; import javax.inject.Singleton; - import java.util.ArrayList; import java.util.Collection; import java.util.LinkedHashSet; @@ -701,12 +698,12 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang } @Override - public String getTraitDefinition(String guid, final String traitName) throws AtlasException { + public IStruct getTraitDefinition(String guid, final String traitName) throws AtlasException { guid = ParamChecker.notEmpty(guid, "entity id"); final ITypedReferenceableInstance instance = repository.getEntityDefinition(guid); IStruct struct = instance.getTrait(traitName); - return InstanceSerialization.toJson(struct, true); + return struct; } /** http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/0de7ce34/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java b/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java index 52dcfde..6782970 100644 --- a/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java +++ b/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java @@ -269,10 +269,9 @@ public class DefaultMetadataServiceTest { assertEquals(traits.get(0), PII); //getTrait - String traitDefinition = metadataService.getTraitDefinition(id, PII); - Struct traitResult = InstanceSerialization.fromJsonStruct(traitDefinition, true); - Assert.assertNotNull(traitResult); - assertEquals(traitResult.getValuesMap().size(), 0); + IStruct traitDefinition = metadataService.getTraitDefinition(id, PII); + Assert.assertNotNull(traitDefinition); + assertEquals(traitDefinition.getValuesMap().size(), 0); //delete trait metadataService.deleteTrait(id, PII); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/0de7ce34/server-api/src/main/java/org/apache/atlas/services/MetadataService.java ---------------------------------------------------------------------- diff --git a/server-api/src/main/java/org/apache/atlas/services/MetadataService.java b/server-api/src/main/java/org/apache/atlas/services/MetadataService.java index 60d8790..2351891 100644 --- a/server-api/src/main/java/org/apache/atlas/services/MetadataService.java +++ b/server-api/src/main/java/org/apache/atlas/services/MetadataService.java @@ -26,6 +26,7 @@ import org.apache.atlas.typesystem.ITypedReferenceableInstance; import org.apache.atlas.typesystem.ITypedStruct; import org.apache.atlas.typesystem.Referenceable; import org.apache.atlas.typesystem.Struct; +import org.apache.atlas.typesystem.IStruct; import org.apache.atlas.typesystem.types.cache.TypeCache; import org.codehaus.jettison.json.JSONObject; @@ -213,7 +214,7 @@ public interface MetadataService { * @return * @throws AtlasException */ - String getTraitDefinition(String guid, String traitName) throws AtlasException; + IStruct getTraitDefinition(String guid, String traitName) throws AtlasException; /** * Deletes a given trait from an existing entity represented by a guid. http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/0de7ce34/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java index 82016d0..bbf01a6 100755 --- a/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java +++ b/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java @@ -25,6 +25,7 @@ import org.apache.atlas.AtlasConstants; import org.apache.atlas.AtlasException; import org.apache.atlas.EntityAuditEvent; import org.apache.atlas.services.MetadataService; +import org.apache.atlas.typesystem.IStruct; import org.apache.atlas.typesystem.Referenceable; import org.apache.atlas.typesystem.exception.EntityExistsException; import org.apache.atlas.typesystem.exception.EntityNotFoundException; @@ -639,6 +640,88 @@ public class EntityResource { } /** + * Fetches the trait definitions of all the traits associated to the given entity + * @param guid globally unique identifier for the entity + */ + @GET + @Path("{guid}/traitDefinitions") + @Produces(Servlets.JSON_MEDIA_TYPE) + public Response getTraitDefinitionsForEntity(@PathParam("guid") String guid){ + AtlasPerfTracer perf = null; + try { + if(AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityResource.getTraitDefinitionsForEntity(" + guid + ")"); + } + LOG.debug("Fetching all trait definitions for entity={}", guid); + final String entityDefinition = metadataService.getEntityDefinition(guid); + + Referenceable entity = InstanceSerialization.fromJsonReferenceable(entityDefinition, true); + JSONArray traits = new JSONArray(); + for (String traitName : entity.getTraits()) { + IStruct trait = entity.getTrait(traitName); + traits.put(new JSONObject(InstanceSerialization.toJson(trait, true))); + } + + JSONObject response = new JSONObject(); + response.put(AtlasClient.REQUEST_ID, Servlets.getRequestId()); + response.put(AtlasClient.RESULTS, traits); + response.put(AtlasClient.COUNT, traits.length()); + + return Response.ok(response).build(); + } catch (EntityNotFoundException e){ + LOG.error("An entity with GUID={} does not exist", guid, e); + throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.NOT_FOUND)); + } catch (AtlasException | IllegalArgumentException e) { + LOG.error("Unable to get trait definitions for entity {}", guid, e); + throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); + } catch (Throwable e) { + LOG.error("Unable to get trait definitions for entity {}", guid, e); + throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); + } finally { + AtlasPerfTracer.log(perf); + } + + } + + /** + * Fetches the trait definition for an entity given its guid and trait name + * + * @param guid globally unique identifier for the entity + * @param traitName name of the trait + */ + @GET + @Path("{guid}/traitDefinitions/{traitName}") + @Produces(Servlets.JSON_MEDIA_TYPE) + public Response getTraitDefinitionForEntity(@PathParam("guid") String guid, @PathParam("traitName") String traitName){ + AtlasPerfTracer perf = null; + try { + if(AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityResource.getTraitDefinitionForEntity(" + guid + ", " + traitName + ")"); + } + LOG.debug("Fetching trait definition for entity {} and trait name {}", guid, traitName); + final IStruct traitDefinition = metadataService.getTraitDefinition(guid, traitName); + + JSONObject response = new JSONObject(); + response.put(AtlasClient.REQUEST_ID, Servlets.getRequestId()); + response.put(AtlasClient.RESULTS, new JSONObject(InstanceSerialization.toJson(traitDefinition, true))); + + return Response.ok(response).build(); + + } catch (EntityNotFoundException e){ + LOG.error("An entity with GUID={} does not exist", guid, e); + throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.NOT_FOUND)); + } catch (AtlasException | IllegalArgumentException e) { + LOG.error("Unable to get trait definition for entity {} and trait {}", guid, traitName, e); + throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); + } catch (Throwable e) { + LOG.error("Unable to get trait definition for entity {} and trait {}", guid, traitName, e); + throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); + } finally { + AtlasPerfTracer.log(perf); + } + } + + /** * Adds a new trait to an existing entity represented by a guid. * * @param guid globally unique identifier for the entity http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/0de7ce34/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java b/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java index a1988ef..ea86cc6 100755 --- a/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java +++ b/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java @@ -87,6 +87,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT { private final String TABLE_NAME = "table" + randomString(); private static final String ENTITIES = "api/atlas/entities"; private static final String TRAITS = "traits"; + private static final String TRAIT_DEFINITION = "traitDefinitions"; private Referenceable tableInstance; private Id tableId; @@ -526,6 +527,37 @@ public class EntityJerseyResourceIT extends BaseResourceIT { assertEntityAudit(guid, EntityAuditEvent.EntityAuditAction.TAG_ADD); } + @Test(dependsOnMethods = "testSubmitEntity") + public void testgetTraitDefinitionForEntity() throws Exception{ + traitName = "PII_Trait" + randomString(); + HierarchicalTypeDefinition<TraitType> piiTrait = + TypesUtil.createTraitTypeDef(traitName, ImmutableSet.<String>of()); + String traitDefinitionAsJSON = TypesSerialization$.MODULE$.toJson(piiTrait, true); + LOG.debug("traitDefinitionAsJSON = " + traitDefinitionAsJSON); + createType(traitDefinitionAsJSON); + + Struct traitInstance = new Struct(traitName); + String traitInstanceAsJSON = InstanceSerialization.toJson(traitInstance, true); + LOG.debug("traitInstanceAsJSON = " + traitInstanceAsJSON); + + final String guid = tableId._getId(); + ClientResponse clientResponse = + service.path(ENTITIES).path(guid).path(TRAITS).accept(Servlets.JSON_MEDIA_TYPE) + .type(Servlets.JSON_MEDIA_TYPE) + .method(HttpMethod.POST, ClientResponse.class, traitInstanceAsJSON); + Assert.assertEquals(clientResponse.getStatus(), Response.Status.CREATED.getStatusCode()); + + Struct traitDef = serviceClient.getTraitDefinition(guid, traitName); + System.out.println(traitDef.toString()); + JSONObject responseAsJSON = new JSONObject(InstanceSerialization.toJson(traitDef, true)); + Assert.assertEquals(responseAsJSON.get("typeName"), traitName); + + + List<Struct> allTraitDefs = serviceClient.listTraitDefinitions(guid); + System.out.println(allTraitDefs.toString()); + Assert.assertEquals(allTraitDefs.size(), 9); + } + @Test(dependsOnMethods = "testAddTrait") public void testAddExistingTrait() throws Exception { final String traitName = "PII_Trait" + randomString();
