ATLAS-2179: Split Atlas client library to avoid unnecessary dependencies
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/187730dd Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/187730dd Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/187730dd Branch: refs/heads/master Commit: 187730dd745f5f558e00393e28b919963c24bf08 Parents: 049c512 Author: apoorvnaik <[email protected]> Authored: Tue Sep 26 12:13:02 2017 -0700 Committer: Madhan Neethiraj <[email protected]> Committed: Tue Sep 26 18:25:59 2017 -0700 ---------------------------------------------------------------------- addons/falcon-bridge/pom.xml | 4 +- addons/hdfs-model/pom.xml | 2 +- addons/hive-bridge/pom.xml | 4 +- addons/sqoop-bridge/pom.xml | 4 +- addons/storm-bridge/pom.xml | 4 +- authorization/pom.xml | 5 - .../simple/AtlasAuthorizationUtils.java | 10 +- .../atlas/authorize/simple/PolicyParser.java | 3 +- client/client-v1/pom.xml | 48 + .../java/org/apache/atlas/AtlasAdminClient.java | 140 +++ .../main/java/org/apache/atlas/AtlasClient.java | 974 +++++++++++++++++++ .../atlas/CreateUpdateEntitiesResult.java | 124 +++ .../java/org/apache/atlas/EntityAuditEvent.java | 146 +++ .../src/main/java/org/apache/atlas/SerDe.java | 80 ++ .../java/org/apache/atlas/AtlasClientTest.java | 474 +++++++++ client/client-v2/pom.xml | 42 + .../java/org/apache/atlas/AtlasClientV2.java | 460 +++++++++ client/common/pom.xml | 48 + .../java/org/apache/atlas/AtlasBaseClient.java | 621 ++++++++++++ .../org/apache/atlas/AtlasServerEnsemble.java | 52 + .../org/apache/atlas/AtlasServiceException.java | 56 ++ .../java/org/apache/atlas/ResourceCreator.java | 29 + .../atlas/security/SecureClientUtils.java | 248 +++++ client/pom.xml | 54 +- client/src/main/assembly/all-jar.xml | 35 + .../java/org/apache/atlas/AtlasAdminClient.java | 140 --- .../java/org/apache/atlas/AtlasBaseClient.java | 620 ------------ .../main/java/org/apache/atlas/AtlasClient.java | 962 ------------------ .../java/org/apache/atlas/AtlasClientV2.java | 466 --------- .../org/apache/atlas/AtlasServerEnsemble.java | 52 - .../org/apache/atlas/AtlasServiceException.java | 75 -- .../atlas/CreateUpdateEntitiesResult.java | 124 --- .../java/org/apache/atlas/EntityAuditEvent.java | 146 --- .../java/org/apache/atlas/ResourceCreator.java | 29 - .../src/main/java/org/apache/atlas/SerDe.java | 80 -- .../atlas/security/SecureClientUtils.java | 248 ----- .../java/org/apache/atlas/AtlasClientTest.java | 474 --------- common/pom.xml | 5 + .../org/apache/atlas/ApplicationProperties.java | 203 ---- .../java/org/apache/atlas/AtlasException.java | 43 - .../atlas/groovy/ArithmeticExpression.java | 4 +- .../security/InMemoryJAASConfiguration.java | 401 -------- .../atlas/security/SecurityProperties.java | 49 - .../apache/atlas/utils/AuthenticationUtil.java | 70 -- .../apache/atlas/ApplicationPropertiesTest.java | 129 --- .../security/InMemoryJAASConfigurationTest.java | 89 -- ...ConfigurationTicketBasedKafkaClientTest.java | 60 -- .../org/apache/atlas/ApplicationProperties.java | 203 ++++ .../java/org/apache/atlas/AtlasException.java | 43 + .../security/InMemoryJAASConfiguration.java | 401 ++++++++ .../atlas/security/SecurityProperties.java | 49 + .../apache/atlas/utils/AuthenticationUtil.java | 70 ++ .../apache/atlas/ApplicationPropertiesTest.java | 129 +++ .../security/InMemoryJAASConfigurationTest.java | 89 ++ ...ConfigurationTicketBasedKafkaClientTest.java | 60 ++ intg/src/test/resources/atlas-jaas.properties | 62 ++ intg/src/test/resources/test.properties | 19 + notification/pom.xml | 2 +- pom.xml | 12 + repository/pom.xml | 2 +- .../converters/AtlasInstanceConverter.java | 4 +- .../graph/GraphBackedMetadataRepository.java | 13 +- .../store/graph/v1/EntityGraphRetriever.java | 53 +- .../test/java/org/apache/atlas/TestUtils.java | 6 +- ...hBackedMetadataRepositoryDeleteTestBase.java | 6 +- .../GraphBackedMetadataRepositoryTest.java | 2 +- .../graph/ReverseReferenceUpdateTestBase.java | 8 +- .../service/DefaultMetadataServiceTest.java | 4 +- server-api/pom.xml | 2 +- .../apache/atlas/services/MetadataService.java | 2 +- .../test/resources/atlas-application.properties | 6 +- webapp/pom.xml | 7 +- .../notification/NotificationHookConsumer.java | 22 +- .../atlas/web/resources/EntityResource.java | 4 +- .../web/integration/AdminJerseyResourceIT.java | 2 +- .../DataSetLineageJerseyResourceIT.java | 10 +- .../web/integration/EntityJerseyResourceIT.java | 20 +- .../EntityLineageJerseyResourceIT.java | 17 +- .../MetadataDiscoveryJerseyResourceIT.java | 14 +- .../web/integration/TypesJerseyResourceIT.java | 10 +- 80 files changed, 4857 insertions(+), 4632 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/atlas/blob/187730dd/addons/falcon-bridge/pom.xml ---------------------------------------------------------------------- diff --git a/addons/falcon-bridge/pom.xml b/addons/falcon-bridge/pom.xml index f0e668f..442be48 100644 --- a/addons/falcon-bridge/pom.xml +++ b/addons/falcon-bridge/pom.xml @@ -53,7 +53,7 @@ <dependency> <groupId>org.apache.atlas</groupId> - <artifactId>atlas-client</artifactId> + <artifactId>atlas-client-v1</artifactId> </dependency> <dependency> @@ -157,7 +157,7 @@ </artifactItem> <artifactItem> <groupId>${project.groupId}</groupId> - <artifactId>atlas-client</artifactId> + <artifactId>atlas-client-v1</artifactId> <version>${project.version}</version> </artifactItem> <artifactItem> http://git-wip-us.apache.org/repos/asf/atlas/blob/187730dd/addons/hdfs-model/pom.xml ---------------------------------------------------------------------- diff --git a/addons/hdfs-model/pom.xml b/addons/hdfs-model/pom.xml index fe78442..6ed444a 100644 --- a/addons/hdfs-model/pom.xml +++ b/addons/hdfs-model/pom.xml @@ -50,7 +50,7 @@ <dependency> <groupId>org.apache.atlas</groupId> - <artifactId>atlas-client</artifactId> + <artifactId>atlas-client-v1</artifactId> </dependency> <dependency> http://git-wip-us.apache.org/repos/asf/atlas/blob/187730dd/addons/hive-bridge/pom.xml ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/pom.xml b/addons/hive-bridge/pom.xml index d16fde7..0909dc2 100755 --- a/addons/hive-bridge/pom.xml +++ b/addons/hive-bridge/pom.xml @@ -100,7 +100,7 @@ <dependency> <groupId>org.apache.atlas</groupId> - <artifactId>atlas-client</artifactId> + <artifactId>atlas-client-v1</artifactId> </dependency> <dependency> @@ -205,7 +205,7 @@ </artifactItem> <artifactItem> <groupId>${project.groupId}</groupId> - <artifactId>atlas-client</artifactId> + <artifactId>atlas-client-v1</artifactId> <version>${project.version}</version> </artifactItem> <artifactItem> http://git-wip-us.apache.org/repos/asf/atlas/blob/187730dd/addons/sqoop-bridge/pom.xml ---------------------------------------------------------------------- diff --git a/addons/sqoop-bridge/pom.xml b/addons/sqoop-bridge/pom.xml index c866f16..7afc806 100644 --- a/addons/sqoop-bridge/pom.xml +++ b/addons/sqoop-bridge/pom.xml @@ -101,7 +101,7 @@ <dependency> <groupId>org.apache.atlas</groupId> - <artifactId>atlas-client</artifactId> + <artifactId>atlas-client-v1</artifactId> </dependency> <dependency> @@ -222,7 +222,7 @@ </artifactItem> <artifactItem> <groupId>${project.groupId}</groupId> - <artifactId>atlas-client</artifactId> + <artifactId>atlas-client-v1</artifactId> <version>${project.version}</version> </artifactItem> <artifactItem> http://git-wip-us.apache.org/repos/asf/atlas/blob/187730dd/addons/storm-bridge/pom.xml ---------------------------------------------------------------------- diff --git a/addons/storm-bridge/pom.xml b/addons/storm-bridge/pom.xml index 8726681..279dd6e 100644 --- a/addons/storm-bridge/pom.xml +++ b/addons/storm-bridge/pom.xml @@ -43,7 +43,7 @@ <dependency> <groupId>org.apache.atlas</groupId> - <artifactId>atlas-client</artifactId> + <artifactId>atlas-client-v1</artifactId> </dependency> <dependency> @@ -171,7 +171,7 @@ </artifactItem> <artifactItem> <groupId>${project.groupId}</groupId> - <artifactId>atlas-client</artifactId> + <artifactId>atlas-client-v1</artifactId> <version>${project.version}</version> </artifactItem> <artifactItem> http://git-wip-us.apache.org/repos/asf/atlas/blob/187730dd/authorization/pom.xml ---------------------------------------------------------------------- diff --git a/authorization/pom.xml b/authorization/pom.xml index bc0882b..143c4e4 100644 --- a/authorization/pom.xml +++ b/authorization/pom.xml @@ -36,11 +36,6 @@ </dependency> <dependency> - <groupId>org.apache.atlas</groupId> - <artifactId>atlas-client</artifactId> - </dependency> - - <dependency> <groupId>javax.servlet</groupId> <artifactId>javax.servlet-api</artifactId> <version>${javax.servlet.version}</version> http://git-wip-us.apache.org/repos/asf/atlas/blob/187730dd/authorization/src/main/java/org/apache/atlas/authorize/simple/AtlasAuthorizationUtils.java ---------------------------------------------------------------------- diff --git a/authorization/src/main/java/org/apache/atlas/authorize/simple/AtlasAuthorizationUtils.java b/authorization/src/main/java/org/apache/atlas/authorize/simple/AtlasAuthorizationUtils.java index d67376f..cc42a73 100644 --- a/authorization/src/main/java/org/apache/atlas/authorize/simple/AtlasAuthorizationUtils.java +++ b/authorization/src/main/java/org/apache/atlas/authorize/simple/AtlasAuthorizationUtils.java @@ -19,16 +19,16 @@ package org.apache.atlas.authorize.simple; -import javax.servlet.http.HttpServletRequest; -import org.apache.atlas.AtlasClient; +import org.apache.atlas.authorize.AtlasAccessRequest; import org.apache.atlas.authorize.AtlasActionTypes; -import org.apache.atlas.authorize.AtlasResourceTypes; import org.apache.atlas.authorize.AtlasAuthorizationException; import org.apache.atlas.authorize.AtlasAuthorizer; -import org.apache.atlas.authorize.AtlasAccessRequest; import org.apache.atlas.authorize.AtlasAuthorizerFactory; +import org.apache.atlas.authorize.AtlasResourceTypes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + +import javax.servlet.http.HttpServletRequest; import java.net.InetAddress; import java.net.UnknownHostException; import java.util.HashSet; @@ -38,7 +38,7 @@ import java.util.regex.Pattern; public class AtlasAuthorizationUtils { private static final Logger LOG = LoggerFactory.getLogger(AtlasAuthorizationUtils.class); private static boolean isDebugEnabled = LOG.isDebugEnabled(); - private static final String BASE_URL = "/" + AtlasClient.BASE_URI; + private static final String BASE_URL = "/api/atlas/"; public static String getApi(String contextPath) { if (isDebugEnabled) { http://git-wip-us.apache.org/repos/asf/atlas/blob/187730dd/authorization/src/main/java/org/apache/atlas/authorize/simple/PolicyParser.java ---------------------------------------------------------------------- diff --git a/authorization/src/main/java/org/apache/atlas/authorize/simple/PolicyParser.java b/authorization/src/main/java/org/apache/atlas/authorize/simple/PolicyParser.java index fc611c8..aabac90 100644 --- a/authorization/src/main/java/org/apache/atlas/authorize/simple/PolicyParser.java +++ b/authorization/src/main/java/org/apache/atlas/authorize/simple/PolicyParser.java @@ -21,7 +21,6 @@ import org.apache.atlas.authorize.AtlasActionTypes; import org.apache.atlas.authorize.AtlasResourceTypes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import scala.tools.jline_embedded.internal.Log; import java.util.ArrayList; import java.util.HashMap; @@ -234,7 +233,7 @@ public class PolicyParser { } else if (type.equalsIgnoreCase("RELATIONSHIP")) { resourceType = AtlasResourceTypes.RELATIONSHIP; } else { - Log.warn(type + " is invalid resource please check PolicyStore file"); + LOG.warn(type + " is invalid resource please check PolicyStore file"); continue; } http://git-wip-us.apache.org/repos/asf/atlas/blob/187730dd/client/client-v1/pom.xml ---------------------------------------------------------------------- diff --git a/client/client-v1/pom.xml b/client/client-v1/pom.xml new file mode 100644 index 0000000..2b492bb --- /dev/null +++ b/client/client-v1/pom.xml @@ -0,0 +1,48 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <parent> + <artifactId>atlas-client</artifactId> + <groupId>org.apache.atlas</groupId> + <version>1.0.0-SNAPSHOT</version> + </parent> + <modelVersion>4.0.0</modelVersion> + + <artifactId>atlas-client-v1</artifactId> + + <dependencies> + <dependency> + <groupId>org.apache.atlas</groupId> + <artifactId>atlas-typesystem</artifactId> + </dependency> + <dependency> + <groupId>org.apache.atlas</groupId> + <artifactId>atlas-client-common</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.atlas</groupId> + <artifactId>atlas-typesystem</artifactId> + <classifier>tests</classifier> + <scope>test</scope> + </dependency> + </dependencies> +</project> http://git-wip-us.apache.org/repos/asf/atlas/blob/187730dd/client/client-v1/src/main/java/org/apache/atlas/AtlasAdminClient.java ---------------------------------------------------------------------- diff --git a/client/client-v1/src/main/java/org/apache/atlas/AtlasAdminClient.java b/client/client-v1/src/main/java/org/apache/atlas/AtlasAdminClient.java new file mode 100644 index 0000000..f334f6c --- /dev/null +++ b/client/client-v1/src/main/java/org/apache/atlas/AtlasAdminClient.java @@ -0,0 +1,140 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas; + +import org.apache.atlas.model.metrics.AtlasMetrics; +import org.apache.atlas.type.AtlasType; +import org.apache.atlas.utils.AuthenticationUtil; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.GnuParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.commons.configuration.Configuration; + +import java.util.Arrays; + + +/** + * An application that allows users to run admin commands against an Atlas server. + * + * The application uses {@link AtlasClient} to send REST requests to the Atlas server. The details of connections + * and other configuration is specified in the Atlas properties file. + * Exit status of the application will be as follows: + * <li>0: successful execution</li> + * <li>1: error in options used for the application</li> + * <li>-1/255: application error</li> + */ +public class AtlasAdminClient { + + private static final Option STATUS = new Option("status", false, "Get the status of an atlas instance"); + private static final Option STATS = new Option("stats", false, "Get the metrics of an atlas instance"); + private static final Options OPTIONS = new Options(); + + private static final int INVALID_OPTIONS_STATUS = 1; + private static final int PROGRAM_ERROR_STATUS = -1; + + static { + OPTIONS.addOption(STATUS); + OPTIONS.addOption(STATS); + } + + public static void main(String[] args) throws AtlasException, ParseException { + AtlasAdminClient atlasAdminClient = new AtlasAdminClient(); + int result = atlasAdminClient.run(args); + System.exit(result); + } + + private int run(String[] args) throws AtlasException { + CommandLine commandLine = parseCommandLineOptions(args); + Configuration configuration = ApplicationProperties.get(); + String[] atlasServerUri = configuration.getStringArray(AtlasConstants.ATLAS_REST_ADDRESS_KEY); + + if (atlasServerUri == null || atlasServerUri.length == 0) { + atlasServerUri = new String[] { AtlasConstants.DEFAULT_ATLAS_REST_ADDRESS }; + } + + AtlasClient atlasClient = null; + if (!AuthenticationUtil.isKerberosAuthenticationEnabled()) { + String[] basicAuthUsernamePassword = AuthenticationUtil.getBasicAuthenticationInput(); + atlasClient = new AtlasClient(atlasServerUri, basicAuthUsernamePassword); + } else { + atlasClient = new AtlasClient(atlasServerUri); + } + return handleCommand(commandLine, atlasServerUri, atlasClient); + } + + private int handleCommand(CommandLine commandLine, String[] atlasServerUri, AtlasClient atlasClient) { + int cmdStatus = PROGRAM_ERROR_STATUS; + if (commandLine.hasOption(STATUS.getOpt())) { + try { + System.out.println(atlasClient.getAdminStatus()); + cmdStatus = 0; + } catch (AtlasServiceException e) { + System.err.println("Could not retrieve status of the server at " + Arrays.toString(atlasServerUri)); + printStandardHttpErrorDetails(e); + } + } else if (commandLine.hasOption(STATS.getOpt())) { + try { + AtlasMetrics atlasMetrics = atlasClient.getAtlasMetrics(); + String json = AtlasType.toJson(atlasMetrics); + System.out.println(json); + cmdStatus = 0; + } catch (AtlasServiceException e) { + System.err.println("Could not retrieve metrics of the server at " + Arrays.toString(atlasServerUri)); + printStandardHttpErrorDetails(e); + } + } else { + System.err.println("Unsupported option. Refer to usage for valid options."); + printUsage(INVALID_OPTIONS_STATUS); + } + return cmdStatus; + } + + private void printStandardHttpErrorDetails(AtlasServiceException e) { + System.err.println("Error details: "); + System.err.println("HTTP Status: " + e.getStatus().getStatusCode() + "," + + e.getStatus().getReasonPhrase()); + System.err.println("Exception message: " + e.getMessage()); + } + + private CommandLine parseCommandLineOptions(String[] args) { + if (args.length == 0) { + printUsage(INVALID_OPTIONS_STATUS); + } + CommandLineParser parser = new GnuParser(); + CommandLine commandLine = null; + try { + commandLine = parser.parse(OPTIONS, args); + } catch (ParseException e) { + System.err.println("Could not parse command line options."); + printUsage(INVALID_OPTIONS_STATUS); + } + return commandLine; + } + + private void printUsage(int statusCode) { + HelpFormatter helpFormatter = new HelpFormatter(); + helpFormatter.printHelp("atlas_admin.py", OPTIONS); + System.exit(statusCode); + } + +} http://git-wip-us.apache.org/repos/asf/atlas/blob/187730dd/client/client-v1/src/main/java/org/apache/atlas/AtlasClient.java ---------------------------------------------------------------------- diff --git a/client/client-v1/src/main/java/org/apache/atlas/AtlasClient.java b/client/client-v1/src/main/java/org/apache/atlas/AtlasClient.java new file mode 100644 index 0000000..6d0b83d --- /dev/null +++ b/client/client-v1/src/main/java/org/apache/atlas/AtlasClient.java @@ -0,0 +1,974 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableSet; +import com.sun.jersey.api.client.WebResource; +import org.apache.atlas.model.legacy.EntityResult; +import org.apache.atlas.typesystem.Referenceable; +import org.apache.atlas.typesystem.Struct; +import org.apache.atlas.typesystem.TypesDef; +import org.apache.atlas.typesystem.json.InstanceSerialization; +import org.apache.atlas.typesystem.json.TypesSerialization; +import org.apache.atlas.typesystem.types.AttributeDefinition; +import org.apache.atlas.typesystem.types.DataTypes; +import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition; +import org.apache.atlas.typesystem.types.TraitType; +import org.apache.atlas.typesystem.types.utils.TypesUtil; +import org.apache.commons.configuration.Configuration; +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.security.UserGroupInformation; +import org.codehaus.jettison.json.JSONArray; +import org.codehaus.jettison.json.JSONException; +import org.codehaus.jettison.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.ws.rs.HttpMethod; +import javax.ws.rs.core.Cookie; +import javax.ws.rs.core.MultivaluedMap; +import javax.ws.rs.core.Response; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; + +/** + * Client for metadata. + */ +@Deprecated +public class AtlasClient extends AtlasBaseClient { + private static final Logger LOG = LoggerFactory.getLogger(AtlasClient.class); + + public static final String TYPE = "type"; + public static final String TYPENAME = "typeName"; + public static final String GUID = "GUID"; + public static final String ENTITIES = "entities"; + public static final String GUID_ASSIGNMENTS = "guidAssignments"; + + public static final String DEFINITION = "definition"; + public static final String ERROR = "error"; + public static final String STACKTRACE = "stackTrace"; + public static final String REQUEST_ID = "requestId"; + public static final String RESULTS = "results"; + public static final String COUNT = "count"; + public static final String ROWS = "rows"; + public static final String DATATYPE = "dataType"; + public static final String STATUS = "Status"; + + public static final String EVENTS = "events"; + public static final String START_KEY = "startKey"; + public static final String NUM_RESULTS = "count"; + + public static final String URI_ENTITY = "entities"; + public static final String URI_ENTITY_AUDIT = "audit"; + public static final String URI_SEARCH = "discovery/search"; + public static final String URI_NAME_LINEAGE = "lineage/hive/table"; + public static final String URI_LINEAGE = "lineage/"; + public static final String URI_TRAITS = "traits"; + public static final String TRAITS = "traits"; + public static final String TRAIT_DEFINITIONS = "traitDefinitions"; + + + public static final String QUERY_TYPE = "queryType"; + public static final String ATTRIBUTE_NAME = "property"; + public static final String ATTRIBUTE_VALUE = "value"; + + public static final String SUPERTYPE = "supertype"; + public static final String NOT_SUPERTYPE = "notsupertype"; + + public static final String ASSET_TYPE = "Asset"; + public static final String NAME = "name"; + public static final String DESCRIPTION = "description"; + public static final String OWNER = "owner"; + public static final String CREATE_TIME = "createTime"; + + public static final String INFRASTRUCTURE_SUPER_TYPE = "Infrastructure"; + public static final String DATA_SET_SUPER_TYPE = "DataSet"; + public static final String PROCESS_SUPER_TYPE = "Process"; + public static final String PROCESS_ATTRIBUTE_INPUTS = "inputs"; + public static final String PROCESS_ATTRIBUTE_OUTPUTS = "outputs"; + + public static final String REFERENCEABLE_SUPER_TYPE = "Referenceable"; + public static final String QUALIFIED_NAME = "qualifiedName"; + public static final String REFERENCEABLE_ATTRIBUTE_NAME = QUALIFIED_NAME; + + public static final String UNKNOWN_STATUS = "Unknown status"; + + /** + * Constructor for AtlasClient with cookie params as header + * @param baseUrl + * @param cookieName + * @param value + * @param path + * @param domain + */ + + public AtlasClient(String[] baseUrl, String cookieName, String value, String path, String domain) { + super(baseUrl, new Cookie(cookieName, value, path, domain)); + } + + /** + * Constructor for AtlasClient with cookie as header + * @param baseUrl + * @param cookie + */ + + public AtlasClient(String[] baseUrl, Cookie cookie) { + super(baseUrl, cookie); + } + + + // New constructor for Basic auth + public AtlasClient(String[] baseUrl, String[] basicAuthUserNamePassword) { + super(baseUrl, basicAuthUserNamePassword); + } + + /** + * Create a new Atlas client. + * @param baseUrls A list of URLs that point to an ensemble of Atlas servers working in + * High Availability mode. The client will automatically determine the + * active instance on startup and also when there is a scenario of + * failover. + */ + public AtlasClient(String... baseUrls) throws AtlasException { + this(getCurrentUGI(), baseUrls); + } + + /** + * Create a new Atlas client. + * @param ugi UserGroupInformation + * @param doAsUser + * @param baseUrls A list of URLs that point to an ensemble of Atlas servers working in + * High Availability mode. The client will automatically determine the + * active instance on startup and also when there is a scenario of + * failover. + */ + public AtlasClient(UserGroupInformation ugi, String doAsUser, String... baseUrls) { + initializeState(baseUrls, ugi, doAsUser); + } + + private AtlasClient(UserGroupInformation ugi, String[] baseUrls) { + this(ugi, ugi.getShortUserName(), baseUrls); + } + + //Used by LocalAtlasClient + protected AtlasClient() { + //Do nothing + } + + @VisibleForTesting + public AtlasClient(Configuration configuration, String[] baseUrl, String[] basicAuthUserNamePassword) { + super(configuration, baseUrl, basicAuthUserNamePassword); + } + + @Override + protected API formatPathParameters(final API api, final String... params) { + return new API(String.format(api.getPath(), params), api.getMethod(), api.getExpectedStatus()); + } + + @VisibleForTesting + public AtlasClient(Configuration configuration, String... baseUrls) throws AtlasException { + initializeState(configuration, baseUrls, getCurrentUGI(), getCurrentUGI().getShortUserName()); + } + + @VisibleForTesting + AtlasClient(WebResource service, Configuration configuration) { + super(service, configuration); + } + + public WebResource getResource() { + return service; + } + + public static class API_V1 extends API { + //Admin operations + public static final API_V1 VERSION = new API_V1(BASE_URI + ADMIN_VERSION, HttpMethod.GET, Response.Status.OK); + public static final API_V1 STATUS = new API_V1(BASE_URI + ADMIN_STATUS, HttpMethod.GET, Response.Status.OK); + + //Type operations + public static final API_V1 CREATE_TYPE = new API_V1(BASE_URI + TYPES, HttpMethod.POST, Response.Status.CREATED); + public static final API_V1 UPDATE_TYPE = new API_V1(BASE_URI + TYPES, HttpMethod.PUT, Response.Status.OK); + public static final API_V1 GET_TYPE = new API_V1(BASE_URI + TYPES, HttpMethod.GET, Response.Status.OK); + public static final API_V1 LIST_TYPES = new API_V1(BASE_URI + TYPES, HttpMethod.GET, Response.Status.OK); + public static final API_V1 LIST_TRAIT_TYPES = new API_V1(BASE_URI + TYPES + "?type=trait", HttpMethod.GET, Response.Status.OK); + + //Entity operations + public static final API_V1 CREATE_ENTITY = new API_V1(BASE_URI + URI_ENTITY, HttpMethod.POST, Response.Status.CREATED); + public static final API_V1 GET_ENTITY = new API_V1(BASE_URI + URI_ENTITY, HttpMethod.GET, Response.Status.OK); + public static final API_V1 UPDATE_ENTITY = new API_V1(BASE_URI + URI_ENTITY, HttpMethod.PUT, Response.Status.OK); + public static final API_V1 UPDATE_ENTITY_PARTIAL = new API_V1(BASE_URI + URI_ENTITY, HttpMethod.POST, Response.Status.OK); + public static final API_V1 LIST_ENTITIES = new API_V1(BASE_URI + URI_ENTITY, HttpMethod.GET, Response.Status.OK); + public static final API_V1 DELETE_ENTITIES = new API_V1(BASE_URI + URI_ENTITY, HttpMethod.DELETE, Response.Status.OK); + public static final API_V1 DELETE_ENTITY = new API_V1(BASE_URI + URI_ENTITY, HttpMethod.DELETE, Response.Status.OK); + + //audit operation + public static final API_V1 LIST_ENTITY_AUDIT = new API_V1(BASE_URI + URI_ENTITY, HttpMethod.GET, Response.Status.OK); + + //Trait operations + public static final API_V1 ADD_TRAITS = new API_V1(BASE_URI + URI_ENTITY, HttpMethod.POST, Response.Status.CREATED); + public static final API_V1 DELETE_TRAITS = new API_V1(BASE_URI + URI_ENTITY, HttpMethod.DELETE, Response.Status.OK); + public static final API_V1 LIST_TRAITS = new API_V1(BASE_URI + URI_ENTITY, HttpMethod.GET, Response.Status.OK); + public static final API_V1 GET_ALL_TRAIT_DEFINITIONS = new API_V1(BASE_URI + URI_ENTITY, HttpMethod.GET, Response.Status.OK); + public static final API_V1 GET_TRAIT_DEFINITION = new API_V1(BASE_URI + URI_ENTITY, HttpMethod.GET, Response.Status.OK); + + //Search operations + public static final API_V1 SEARCH = new API_V1(BASE_URI + URI_SEARCH, HttpMethod.GET, Response.Status.OK); + public static final API_V1 SEARCH_DSL = new API_V1(BASE_URI + URI_SEARCH + "/dsl", HttpMethod.GET, Response.Status.OK); + public static final API_V1 SEARCH_FULL_TEXT = new API_V1(BASE_URI + URI_SEARCH + "/fulltext", HttpMethod.GET, Response.Status.OK); + public static final API_V1 GREMLIN_SEARCH = new API_V1(BASE_URI + URI_SEARCH + "/gremlin", HttpMethod.GET, Response.Status.OK); + + //Lineage operations based on dataset name + public static final API_V1 NAME_LINEAGE_INPUTS_GRAPH = new API_V1(BASE_URI + URI_NAME_LINEAGE, HttpMethod.GET, Response.Status.OK); + public static final API_V1 NAME_LINEAGE_OUTPUTS_GRAPH = new API_V1(BASE_URI + URI_NAME_LINEAGE, HttpMethod.GET, Response.Status.OK); + public static final API_V1 NAME_LINEAGE_SCHEMA = new API_V1(BASE_URI + URI_NAME_LINEAGE, HttpMethod.GET, Response.Status.OK); + + //Lineage operations based on entity id of the dataset + public static final API_V1 LINEAGE_INPUTS_GRAPH = new API_V1(BASE_URI + URI_LINEAGE, HttpMethod.GET, Response.Status.OK); + public static final API_V1 LINEAGE_OUTPUTS_GRAPH = new API_V1(BASE_URI + URI_LINEAGE, HttpMethod.GET, Response.Status.OK); + public static final API_V1 LINEAGE_SCHEMA = new API_V1(BASE_URI + URI_LINEAGE, HttpMethod.GET, Response.Status.OK); + + private API_V1(String path, String method, Response.Status status) { + super(path, method, status); + } + } + + /** + * Register the given type(meta model) + * @param typeAsJson type definition a jaon + * @return result json object + * @throws AtlasServiceException + */ + public List<String> createType(String typeAsJson) throws AtlasServiceException { + LOG.debug("Creating type definition: {}", typeAsJson); + JSONObject response = callAPIWithBody(API_V1.CREATE_TYPE, typeAsJson); + List<String> results = extractResults(response, AtlasClient.TYPES, new ExtractOperation<String, JSONObject>() { + @Override + String extractElement(JSONObject element) throws JSONException { + return element.getString(AtlasClient.NAME); + } + }); + LOG.debug("Create type definition returned results: {}", results); + return results; + } + + /** + * Register the given type(meta model) + * @param typeDef type definition + * @return result json object + * @throws AtlasServiceException + */ + public List<String> createType(TypesDef typeDef) throws AtlasServiceException { + return createType(TypesSerialization.toJson(typeDef)); + } + + /** + * Creates trait type with specifiedName, superTraits and attributes + * @param traitName the name of the trait type + * @param superTraits the list of super traits from which this trait type inherits attributes + * @param attributeDefinitions the list of attributes of the trait type + * @return the list of types created + * @throws AtlasServiceException + */ + public List<String> createTraitType(String traitName, ImmutableSet<String> superTraits, AttributeDefinition... attributeDefinitions) throws AtlasServiceException { + HierarchicalTypeDefinition<TraitType> piiTrait = + TypesUtil.createTraitTypeDef(traitName, superTraits, attributeDefinitions); + + String traitDefinitionAsJSON = TypesSerialization.toJson(piiTrait, true); + LOG.debug("Creating trait type {} {}", traitName, traitDefinitionAsJSON); + return createType(traitDefinitionAsJSON); + } + + /** + * Creates simple trait type with specifiedName with no superTraits or attributes + * @param traitName the name of the trait type + * @return the list of types created + * @throws AtlasServiceException + */ + public List<String> createTraitType(String traitName) throws AtlasServiceException { + return createTraitType(traitName, null); + } + + /** + * Register the given type(meta model) + * @param typeAsJson type definition a jaon + * @return result json object + * @throws AtlasServiceException + */ + public List<String> updateType(String typeAsJson) throws AtlasServiceException { + LOG.debug("Updating type definition: {}", typeAsJson); + JSONObject response = callAPIWithBody(API_V1.UPDATE_TYPE, typeAsJson); + List<String> results = extractResults(response, AtlasClient.TYPES, new ExtractOperation<String, JSONObject>() { + @Override + String extractElement(JSONObject element) throws JSONException { + return element.getString(AtlasClient.NAME); + } + }); + LOG.debug("Update type definition returned results: {}", results); + return results; + } + + /** + * Register the given type(meta model) + * @param typeDef type definition + * @return result json object + * @throws AtlasServiceException + */ + public List<String> updateType(TypesDef typeDef) throws AtlasServiceException { + return updateType(TypesSerialization.toJson(typeDef)); + } + + /** + * Returns all type names in the system + * @return list of type names + * @throws AtlasServiceException + */ + public List<String> listTypes() throws AtlasServiceException { + final JSONObject jsonObject = callAPIWithQueryParams(API_V1.LIST_TYPES, null); + return extractResults(jsonObject, AtlasClient.RESULTS, new ExtractOperation<String, String>()); + } + + /** + * Returns all type names with the given category + * @param category + * @return list of type names + * @throws AtlasServiceException + */ + public List<String> listTypes(final DataTypes.TypeCategory category) throws AtlasServiceException { + final API api = API_V1.LIST_TYPES; + JSONObject response = callAPIWithRetries(api, null, new ResourceCreator() { + @Override + public WebResource createResource() { + WebResource resource = getResource(api.getPath()); + resource = resource.queryParam(TYPE, category.name()); + return resource; + } + }); + return extractResults(response, AtlasClient.RESULTS, new ExtractOperation<String, String>()); + } + + /** + * Return the list of type names in the type system which match the specified filter. + * + * @param category returns types whose category is the given typeCategory + * @param superType returns types which contain the given supertype + * @param notSupertype returns types which do not contain the given supertype + * + * Its possible to specify combination of these filters in one request and the conditions are combined with AND + * For example, typeCategory = TRAIT && supertype contains 'X' && supertype !contains 'Y' + * If there is no filter, all the types are returned + * @return list of type names + */ + public List<String> listTypes(final DataTypes.TypeCategory category, final String superType, + final String notSupertype) throws AtlasServiceException { + final API api = API_V1.LIST_TYPES; + JSONObject response = callAPIWithRetries(api, null, new ResourceCreator() { + @Override + public WebResource createResource() { + WebResource resource = getResource(api); + resource = resource.queryParam(TYPE, category.name()); + resource = resource.queryParam(SUPERTYPE, superType); + resource = resource.queryParam(NOT_SUPERTYPE, notSupertype); + return resource; + } + }); + return extractResults(response, AtlasClient.RESULTS, new ExtractOperation<String, String>()); + } + + public TypesDef getType(String typeName) throws AtlasServiceException { + try { + JSONObject response = callAPIWithBodyAndParams(API_V1.GET_TYPE, null, typeName); + String typeJson = response.getString(DEFINITION); + return TypesSerialization.fromJson(typeJson); + } catch (JSONException e) { + throw new AtlasServiceException(e); + } + } + + /** + * Create the given entity + * @param entities entity(type instance) as json + * @return json array of guids + * @throws AtlasServiceException + */ + protected List<String> createEntity(JSONArray entities) throws AtlasServiceException { + LOG.debug("Creating entities: {}", entities); + JSONObject response = callAPIWithBody(API_V1.CREATE_ENTITY, entities.toString()); + List<String> results = extractEntityResult(response).getCreatedEntities(); + LOG.debug("Create entities returned results: {}", results); + return results; + } + + protected EntityResult extractEntityResult(JSONObject response) throws AtlasServiceException { + return EntityResult.fromString(response.toString()); + } + + /** + * Create the given entity + * @param entitiesAsJson entity(type instance) as json + * @return json array of guids + * @throws AtlasServiceException + */ + public List<String> createEntity(String... entitiesAsJson) throws AtlasServiceException { + return createEntity(new JSONArray(Arrays.asList(entitiesAsJson))); + } + + public List<String> createEntity(Referenceable... entities) throws AtlasServiceException { + return createEntity(Arrays.asList(entities)); + } + + public List<String> createEntity(Collection<Referenceable> entities) throws AtlasServiceException { + JSONArray entityArray = getEntitiesArray(entities); + return createEntity(entityArray); + } + + private JSONArray getEntitiesArray(Collection<Referenceable> entities) { + JSONArray entityArray = new JSONArray(entities.size()); + for (Referenceable entity : entities) { + entityArray.put(InstanceSerialization.toJson(entity, true)); + } + return entityArray; + } + + /** + * Replaces entity definitions identified by their guid or unique attribute + * Updates properties set in the definition for the entity corresponding to guid + * @param entities entities to be updated + * @return json array of guids which were updated/created + * @throws AtlasServiceException + */ + public EntityResult updateEntities(Referenceable... entities) throws AtlasServiceException { + return updateEntities(Arrays.asList(entities)); + } + + protected EntityResult updateEntities(JSONArray entities) throws AtlasServiceException { + LOG.debug("Updating entities: {}", entities); + JSONObject response = callAPIWithBody(API_V1.UPDATE_ENTITY, entities.toString()); + EntityResult results = extractEntityResult(response); + LOG.debug("Update entities returned results: {}", results); + return results; + } + + public EntityResult updateEntities(Collection<Referenceable> entities) throws AtlasServiceException { + JSONArray entitiesArray = getEntitiesArray(entities); + return updateEntities(entitiesArray); + } + + /** + * Supports Partial updates + * Updates property for the entity corresponding to guid + * @param guid guid + * @param attribute property key + * @param value property value + */ + public EntityResult updateEntityAttribute(final String guid, final String attribute, String value) + throws AtlasServiceException { + LOG.debug("Updating entity id: {}, attribute name: {}, attribute value: {}", guid, attribute, value); + final API api = API_V1.UPDATE_ENTITY_PARTIAL; + JSONObject response = callAPIWithRetries(api, value, new ResourceCreator() { + @Override + public WebResource createResource() { + WebResource resource = getResource(api, guid); + resource = resource.queryParam(ATTRIBUTE_NAME, attribute); + return resource; + } + }); + return extractEntityResult(response); + } + + /** + * Supports Partial updates + * Updates properties set in the definition for the entity corresponding to guid + * @param guid guid + * @param entity entity definition + */ + public EntityResult updateEntity(String guid, Referenceable entity) throws AtlasServiceException { + String entityJson = InstanceSerialization.toJson(entity, true); + LOG.debug("Updating entity id {} with {}", guid, entityJson); + JSONObject response = callAPIWithBodyAndParams(API_V1.UPDATE_ENTITY_PARTIAL, entityJson, guid); + return extractEntityResult(response); + } + + /** + * Associate trait to an entity + * + * @param guid guid + * @param traitDefinition trait definition + */ + public void addTrait(String guid, Struct traitDefinition) throws AtlasServiceException { + String traitJson = InstanceSerialization.toJson(traitDefinition, true); + LOG.debug("Adding trait to entity with id {} {}", guid, traitJson); + callAPIWithBodyAndParams(API_V1.ADD_TRAITS, traitJson, guid, URI_TRAITS); + } + + /** + * Delete a trait from the given entity + * @param guid guid of the entity + * @param traitName trait to be deleted + * @throws AtlasServiceException + */ + public void deleteTrait(String guid, String traitName) throws AtlasServiceException { + callAPIWithBodyAndParams(API_V1.DELETE_TRAITS, null, guid, TRAITS, traitName); + } + + /** + * Supports Partial updates + * Updates properties set in the definition for the entity corresponding to guid + * @param entityType Type of the entity being updated + * @param uniqueAttributeName Attribute Name that uniquely identifies the entity + * @param uniqueAttributeValue Attribute Value that uniquely identifies the entity + * @param entity entity definition + */ + public EntityResult updateEntity(final String entityType, final String uniqueAttributeName, + final String uniqueAttributeValue, + Referenceable entity) throws AtlasServiceException { + final API api = API_V1.UPDATE_ENTITY_PARTIAL; + String entityJson = InstanceSerialization.toJson(entity, true); + LOG.debug("Updating entity type: {}, attributeName: {}, attributeValue: {}, entity: {}", entityType, + uniqueAttributeName, uniqueAttributeValue, entityJson); + JSONObject response = callAPIWithRetries(api, entityJson, new ResourceCreator() { + @Override + public WebResource createResource() { + WebResource resource = getResource(api, QUALIFIED_NAME); + resource = resource.queryParam(TYPE, entityType); + resource = resource.queryParam(ATTRIBUTE_NAME, uniqueAttributeName); + resource = resource.queryParam(ATTRIBUTE_VALUE, uniqueAttributeValue); + return resource; + } + }); + EntityResult result = extractEntityResult(response); + LOG.debug("Update entity returned result: {}", result); + return result; + } + + protected String getString(JSONObject jsonObject, String parameter) throws AtlasServiceException { + try { + return jsonObject.getString(parameter); + } catch (JSONException e) { + throw new AtlasServiceException(e); + } + } + + /** + * Delete the specified entities from the repository + * + * @param guids guids of entities to delete + * @return List of entity ids updated/deleted + * @throws AtlasServiceException + */ + public EntityResult deleteEntities(final String... guids) throws AtlasServiceException { + LOG.debug("Deleting entities: {}", guids); + final API api = API_V1.DELETE_ENTITIES; + JSONObject jsonResponse = callAPIWithRetries(api, null, new ResourceCreator() { + @Override + public WebResource createResource() { + WebResource resource = getResource(api); + for (String guid : guids) { + resource = resource.queryParam(GUID.toLowerCase(), guid); + } + return resource; + } + }); + EntityResult results = extractEntityResult(jsonResponse); + LOG.debug("Delete entities returned results: {}", results); + return results; + } + + /** + * Supports Deletion of an entity identified by its unique attribute value + * @param entityType Type of the entity being deleted + * @param uniqueAttributeName Attribute Name that uniquely identifies the entity + * @param uniqueAttributeValue Attribute Value that uniquely identifies the entity + * @return List of entity ids updated/deleted(including composite references from that entity) + */ + public EntityResult deleteEntity(String entityType, String uniqueAttributeName, String uniqueAttributeValue) + throws AtlasServiceException { + LOG.debug("Deleting entity type: {}, attributeName: {}, attributeValue: {}", entityType, uniqueAttributeName, + uniqueAttributeValue); + API api = API_V1.DELETE_ENTITIES; + WebResource resource = getResource(api); + resource = resource.queryParam(TYPE, entityType); + resource = resource.queryParam(ATTRIBUTE_NAME, uniqueAttributeName); + resource = resource.queryParam(ATTRIBUTE_VALUE, uniqueAttributeValue); + JSONObject jsonResponse = callAPIWithResource(api, resource); + EntityResult results = extractEntityResult(jsonResponse); + LOG.debug("Delete entities returned results: {}", results); + return results; + } + + /** + * Get an entity given the entity id + * @param guid entity id + * @return result object + * @throws AtlasServiceException + */ + public Referenceable getEntity(String guid) throws AtlasServiceException { + JSONObject jsonResponse = callAPIWithBodyAndParams(API_V1.GET_ENTITY, null, guid); + try { + String entityInstanceDefinition = jsonResponse.getString(AtlasClient.DEFINITION); + return InstanceSerialization.fromJsonReferenceable(entityInstanceDefinition, true); + } catch (JSONException e) { + throw new AtlasServiceException(API_V1.GET_ENTITY, e); + } + } + + public static String toString(JSONArray jsonArray) throws JSONException { + ArrayList<String> resultsList = new ArrayList<>(); + for (int index = 0; index < jsonArray.length(); index++) { + resultsList.add(jsonArray.getString(index)); + } + return StringUtils.join(resultsList, ","); + } + + /** + * Get an entity given the entity id + * @param entityType entity type name + * @param attribute qualified name of the entity + * @param value + * @return result object + * @throws AtlasServiceException + */ + public Referenceable getEntity(final String entityType, final String attribute, final String value) + throws AtlasServiceException { + final API api = API_V1.GET_ENTITY; + JSONObject jsonResponse = callAPIWithRetries(api, null, new ResourceCreator() { + @Override + public WebResource createResource() { + WebResource resource = getResource(api); + resource = resource.queryParam(TYPE, entityType); + resource = resource.queryParam(ATTRIBUTE_NAME, attribute); + resource = resource.queryParam(ATTRIBUTE_VALUE, value); + return resource; + } + }); + try { + String entityInstanceDefinition = jsonResponse.getString(AtlasClient.DEFINITION); + return InstanceSerialization.fromJsonReferenceable(entityInstanceDefinition, true); + } catch (JSONException e) { + throw new AtlasServiceException(api, e); + } + } + + /** + * List entities for a given entity type + * @param entityType + * @return + * @throws AtlasServiceException + */ + public List<String> listEntities(final String entityType) throws AtlasServiceException { + JSONObject jsonResponse = callAPIWithRetries(API_V1.LIST_ENTITIES, null, new ResourceCreator() { + @Override + public WebResource createResource() { + WebResource resource = getResource(API_V1.LIST_ENTITIES); + resource = resource.queryParam(TYPE, entityType); + return resource; + } + }); + return extractResults(jsonResponse, AtlasClient.RESULTS, new ExtractOperation<String, String>()); + } + + /** + * List traits for a given entity identified by its GUID + * @param guid GUID of the entity + * @return List<String> - traitnames associated with entity + * @throws AtlasServiceException + */ + public List<String> listTraits(final String guid) throws AtlasServiceException { + JSONObject jsonResponse = callAPIWithBodyAndParams(API_V1.LIST_TRAITS, null, guid, URI_TRAITS); + return extractResults(jsonResponse, AtlasClient.RESULTS, new ExtractOperation<String, String>()); + } + + /** + * Get all trait definitions for an entity + * @param guid GUID of the entity + * @return List<String> trait definitions of the traits associated to the entity + * @throws AtlasServiceException + */ + public List<Struct> listTraitDefinitions(final String guid) throws AtlasServiceException { + JSONObject jsonResponse = callAPIWithBodyAndParams(API_V1.GET_ALL_TRAIT_DEFINITIONS, null, guid, TRAIT_DEFINITIONS); + List<JSONObject> traitDefList = extractResults(jsonResponse, AtlasClient.RESULTS, new ExtractOperation<JSONObject, JSONObject>()); + ArrayList<Struct> traitStructList = new ArrayList<>(); + for (JSONObject traitDef : traitDefList) { + Struct traitStruct = InstanceSerialization.fromJsonStruct(traitDef.toString(), true); + traitStructList.add(traitStruct); + } + return traitStructList; + } + + /** + * Get trait definition for a given entity and traitname + * @param guid GUID of the entity + * @param traitName + * @return trait definition + * @throws AtlasServiceException + */ + public Struct getTraitDefinition(final String guid, final String traitName) throws AtlasServiceException { + JSONObject jsonResponse = callAPIWithBodyAndParams(API_V1.GET_TRAIT_DEFINITION, null, guid, TRAIT_DEFINITIONS, traitName); + + try { + return InstanceSerialization.fromJsonStruct(jsonResponse.getString(AtlasClient.RESULTS), false); + } catch (JSONException e) { + throw new AtlasServiceException(API_V1.GET_TRAIT_DEFINITION, e); + } + } + + protected class ExtractOperation<T, U> { + T extractElement(U element) throws JSONException { + return (T) element; + } + } + + protected <T, U> List<T> extractResults(JSONObject jsonResponse, String key, ExtractOperation<T, U> extractInterafce) + throws AtlasServiceException { + try { + JSONArray results = jsonResponse.getJSONArray(key); + ArrayList<T> resultsList = new ArrayList<>(); + for (int index = 0; index < results.length(); index++) { + Object element = results.get(index); + resultsList.add(extractInterafce.extractElement((U) element)); + } + return resultsList; + } catch (JSONException e) { + throw new AtlasServiceException(e); + } + } + + /** + * Get the latest numResults entity audit events in decreasing order of timestamp for the given entity id + * @param entityId entity id + * @param numResults number of results to be returned + * @return list of audit events for the entity id + * @throws AtlasServiceException + */ + public List<EntityAuditEvent> getEntityAuditEvents(String entityId, short numResults) + throws AtlasServiceException { + return getEntityAuditEvents(entityId, null, numResults); + } + + /** + * Get the entity audit events in decreasing order of timestamp for the given entity id + * @param entityId entity id + * @param startKey key for the first event to be returned, used for pagination + * @param numResults number of results to be returned + * @return list of audit events for the entity id + * @throws AtlasServiceException + */ + public List<EntityAuditEvent> getEntityAuditEvents(String entityId, String startKey, short numResults) + throws AtlasServiceException { + WebResource resource = getResource(API_V1.LIST_ENTITY_AUDIT, entityId, URI_ENTITY_AUDIT); + if (StringUtils.isNotEmpty(startKey)) { + resource = resource.queryParam(START_KEY, startKey); + } + resource = resource.queryParam(NUM_RESULTS, String.valueOf(numResults)); + + JSONObject jsonResponse = callAPIWithResource(API_V1.LIST_ENTITY_AUDIT, resource); + return extractResults(jsonResponse, AtlasClient.EVENTS, new ExtractOperation<EntityAuditEvent, JSONObject>() { + @Override + EntityAuditEvent extractElement(JSONObject element) throws JSONException { + return SerDe.GSON.fromJson(element.toString(), EntityAuditEvent.class); + } + }); + + } + + /** + * Search using dsl/full text + * @param searchQuery + * @param limit number of rows to be returned in the result, used for pagination. maxlimit > limit > 0. -1 maps to atlas.search.defaultlimit property value + * @param offset offset to the results returned, used for pagination. offset >= 0. -1 maps to offset 0 + * @return Query results + * @throws AtlasServiceException + */ + public JSONArray search(final String searchQuery, final int limit, final int offset) throws AtlasServiceException { + final API api = API_V1.SEARCH; + JSONObject result = callAPIWithRetries(api, null, new ResourceCreator() { + @Override + public WebResource createResource() { + WebResource resource = getResource(api); + resource = resource.queryParam(QUERY, searchQuery); + resource = resource.queryParam(LIMIT, String.valueOf(limit)); + resource = resource.queryParam(OFFSET, String.valueOf(offset)); + return resource; + } + }); + try { + return result.getJSONArray(RESULTS); + } catch (JSONException e) { + throw new AtlasServiceException(e); + } + + } + + /** + * Search given query DSL + * @param query DSL query + * @param limit number of rows to be returned in the result, used for pagination. maxlimit > limit > 0. -1 maps to atlas.search.defaultlimit property value + * @param offset offset to the results returned, used for pagination. offset >= 0. -1 maps to offset 0 + * @return result json object + * @throws AtlasServiceException + */ + public JSONArray searchByDSL(final String query, final int limit, final int offset) throws AtlasServiceException { + LOG.debug("DSL query: {}", query); + final API api = API_V1.SEARCH_DSL; + JSONObject result = callAPIWithRetries(api, null, new ResourceCreator() { + @Override + public WebResource createResource() { + WebResource resource = getResource(api); + resource = resource.queryParam(QUERY, query); + resource = resource.queryParam(LIMIT, String.valueOf(limit)); + resource = resource.queryParam(OFFSET, String.valueOf(offset)); + return resource; + } + }); + try { + return result.getJSONArray(RESULTS); + } catch (JSONException e) { + throw new AtlasServiceException(e); + } + } + + /** + * Search given full text search + * @param query Query + * @param limit number of rows to be returned in the result, used for pagination. maxlimit > limit > 0. -1 maps to atlas.search.defaultlimit property value + * @param offset offset to the results returned, used for pagination. offset >= 0. -1 maps to offset 0 + * @return result json object + * @throws AtlasServiceException + */ + public JSONObject searchByFullText(final String query, final int limit, final int offset) throws AtlasServiceException { + final API api = API_V1.SEARCH_FULL_TEXT; + return callAPIWithRetries(api, null, new ResourceCreator() { + @Override + public WebResource createResource() { + WebResource resource = getResource(api); + resource = resource.queryParam(QUERY, query); + resource = resource.queryParam(LIMIT, String.valueOf(limit)); + resource = resource.queryParam(OFFSET, String.valueOf(offset)); + return resource; + } + }); + } + + public JSONObject getInputGraph(String datasetName) throws AtlasServiceException { + JSONObject response = callAPIWithBodyAndParams(API_V1.NAME_LINEAGE_INPUTS_GRAPH, null, datasetName, "/inputs/graph"); + try { + return response.getJSONObject(AtlasClient.RESULTS); + } catch (JSONException e) { + throw new AtlasServiceException(e); + } + } + + public JSONObject getOutputGraph(String datasetName) throws AtlasServiceException { + JSONObject response = callAPIWithBodyAndParams(API_V1.NAME_LINEAGE_OUTPUTS_GRAPH, null, datasetName, "/outputs/graph"); + try { + return response.getJSONObject(AtlasClient.RESULTS); + } catch (JSONException e) { + throw new AtlasServiceException(e); + } + } + + public JSONObject getInputGraphForEntity(String entityId) throws AtlasServiceException { + JSONObject response = callAPIWithBodyAndParams(API_V1.LINEAGE_INPUTS_GRAPH, null, entityId, "/inputs/graph"); + try { + return response.getJSONObject(AtlasClient.RESULTS); + } catch (JSONException e) { + throw new AtlasServiceException(e); + } + } + + public JSONObject getOutputGraphForEntity(String datasetId) throws AtlasServiceException { + JSONObject response = callAPIWithBodyAndParams(API_V1.LINEAGE_OUTPUTS_GRAPH, null, datasetId, "/outputs/graph"); + try { + return response.getJSONObject(AtlasClient.RESULTS); + } catch (JSONException e) { + throw new AtlasServiceException(e); + } + } + + public JSONObject getSchemaForEntity(String datasetId) throws AtlasServiceException { + JSONObject response = callAPIWithBodyAndParams(API_V1.LINEAGE_OUTPUTS_GRAPH, null, datasetId, "/schema"); + try { + return response.getJSONObject(AtlasClient.RESULTS); + } catch (JSONException e) { + throw new AtlasServiceException(e); + } + } + + // Wrapper methods for compatibility + @VisibleForTesting + public JSONObject callAPIWithResource(API api, WebResource resource) throws AtlasServiceException { + return callAPIWithResource(api, resource, null, JSONObject.class); + } + + @VisibleForTesting + public JSONObject callAPIWithResource(API_V1 apiV1, WebResource resource) throws AtlasServiceException { + return callAPIWithResource(apiV1, resource, null, JSONObject.class); + } + + @VisibleForTesting + public WebResource getResource(API api, String... params) { + return getResource(api.getPath(), params); + } + + @VisibleForTesting + public WebResource getResource(API_V1 apiV1, String... params) { + return getResource(apiV1.getPath(), params); + } + + @VisibleForTesting + public JSONObject callAPIWithBody(API api, Object requestObject) throws AtlasServiceException { + return callAPI(api, JSONObject.class, requestObject, (String[]) null); + } + + @VisibleForTesting + public JSONObject callAPIWithBody(API_V1 apiV1, Object requestObject) throws AtlasServiceException { + return callAPI(apiV1, JSONObject.class, requestObject, (String[]) null); + } + + @VisibleForTesting + public JSONObject callAPIWithBodyAndParams(API api, Object requestObject, String... params) throws AtlasServiceException { + return callAPI(api, JSONObject.class, requestObject, params); + } + + @VisibleForTesting + public JSONObject callAPIWithBodyAndParams(API_V1 apiV1, Object requestObject, String... params) throws AtlasServiceException { + return callAPI(apiV1, JSONObject.class, requestObject, params); + } + + @VisibleForTesting + public JSONObject callAPIWithQueryParams(API api, MultivaluedMap<String, String> queryParams) throws AtlasServiceException { + return callAPI(api, JSONObject.class, queryParams); + } + + @VisibleForTesting + public JSONObject callAPIWithQueryParams(API_V1 apiV1, MultivaluedMap<String, String> queryParams) throws AtlasServiceException { + return callAPI(apiV1, JSONObject.class, queryParams); + } + + @VisibleForTesting + JSONObject callAPIWithRetries(API api, Object requestObject, ResourceCreator resourceCreator) throws AtlasServiceException { + return super.callAPIWithRetries(api, requestObject, resourceCreator); + } + + @VisibleForTesting + JSONObject callAPIWithRetries(API_V1 apiV1, Object requestObject, ResourceCreator resourceCreator) throws AtlasServiceException { + return super.callAPIWithRetries(apiV1, requestObject, resourceCreator); + } +} http://git-wip-us.apache.org/repos/asf/atlas/blob/187730dd/client/client-v1/src/main/java/org/apache/atlas/CreateUpdateEntitiesResult.java ---------------------------------------------------------------------- diff --git a/client/client-v1/src/main/java/org/apache/atlas/CreateUpdateEntitiesResult.java b/client/client-v1/src/main/java/org/apache/atlas/CreateUpdateEntitiesResult.java new file mode 100644 index 0000000..5e6d6db --- /dev/null +++ b/client/client-v1/src/main/java/org/apache/atlas/CreateUpdateEntitiesResult.java @@ -0,0 +1,124 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas; + +import org.apache.atlas.model.instance.GuidMapping; +import org.apache.atlas.model.legacy.EntityResult; +import org.apache.atlas.type.AtlasType; + +import java.util.Collections; +import java.util.List; + +/** + * Result from creating or updating entities. + */ +@Deprecated +public class CreateUpdateEntitiesResult { + + /** + * Guid mapping for the entities that were created/updated + */ + private GuidMapping guidMapping; + + /** + * Entity result + */ + private EntityResult entityResult; + + /** + * Gets the guid mapping + */ + public GuidMapping getGuidMapping() { + return guidMapping; + } + + /** + * Sets the guid mapping + */ + public void setGuidMapping(GuidMapping guidMapping) { + this.guidMapping = guidMapping; + } + + /** + * Gets the entity result + */ + public EntityResult getEntityResult() { + return entityResult; + } + + /** + * Sets the entity result + */ + public void setEntityResult(EntityResult entityResult) { + this.entityResult = entityResult; + } + + /** + * Deserializes the given json into an instance of + * CreateUpdateEntitiesResult. + * + * @param json + * the (unmodified) json that comes back from Atlas. + * @return + * @throws AtlasServiceException + */ + public static CreateUpdateEntitiesResult fromJson(String json) throws AtlasServiceException { + + GuidMapping guidMapping = AtlasType.fromJson(json, GuidMapping.class); + EntityResult entityResult = EntityResult.fromString(json); + CreateUpdateEntitiesResult result = new CreateUpdateEntitiesResult(); + result.setEntityResult(entityResult); + result.setGuidMapping(guidMapping); + return result; + } + + /** + * Convenience method to get the guids of the created entities from + * the EntityResult. + */ + public List<String> getCreatedEntities() { + if(entityResult == null) { + return Collections.emptyList(); + } + return getEntityResult().getCreatedEntities(); + } + + /** + * Convenience method to get the guids of the updated entities from + * the EntityResult. + */ + public List<String> getUpdatedEntities() { + if(entityResult == null) { + return Collections.emptyList(); + } + return getEntityResult().getUpdateEntities(); + } + + + /** + * Convenience method to get the guids of the deleted entities + * from the EntityResult. + */ + public List<String> getDeletedEntities() { + if (entityResult == null) { + return Collections.emptyList(); + } + return getEntityResult().getDeletedEntities(); + } + +} http://git-wip-us.apache.org/repos/asf/atlas/blob/187730dd/client/client-v1/src/main/java/org/apache/atlas/EntityAuditEvent.java ---------------------------------------------------------------------- diff --git a/client/client-v1/src/main/java/org/apache/atlas/EntityAuditEvent.java b/client/client-v1/src/main/java/org/apache/atlas/EntityAuditEvent.java new file mode 100644 index 0000000..904674d --- /dev/null +++ b/client/client-v1/src/main/java/org/apache/atlas/EntityAuditEvent.java @@ -0,0 +1,146 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas; + +import org.apache.atlas.typesystem.IReferenceableInstance; +import org.apache.atlas.typesystem.json.InstanceSerialization; + +import java.util.Objects; + +/** + * Structure of entity audit event + */ +public class EntityAuditEvent { + public enum EntityAuditAction { + ENTITY_CREATE, ENTITY_UPDATE, ENTITY_DELETE, TAG_ADD, TAG_DELETE, TAG_UPDATE, + ENTITY_IMPORT_CREATE, ENTITY_IMPORT_UPDATE, ENTITY_IMPORT_DELETE, + } + + private String entityId; + private long timestamp; + private String user; + private EntityAuditAction action; + private String details; + private String eventKey; + private IReferenceableInstance entityDefinition; + + public EntityAuditEvent() { + } + + public EntityAuditEvent(String entityId, Long ts, String user, EntityAuditAction action, String details, + IReferenceableInstance entityDefinition) throws AtlasException { + this.entityId = entityId; + this.timestamp = ts; + this.user = user; + this.action = action; + this.details = details; + this.entityDefinition = entityDefinition; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + EntityAuditEvent that = (EntityAuditEvent) o; + return timestamp == that.timestamp && + Objects.equals(entityId, that.entityId) && + Objects.equals(user, that.user) && + action == that.action && + Objects.equals(details, that.details) && + Objects.equals(eventKey, that.eventKey) && + Objects.equals(entityDefinition, that.entityDefinition); + } + + @Override + public int hashCode() { + return Objects.hash(entityId, timestamp, user, action, details, eventKey, entityDefinition); + } + + @Override + public String toString() { + return SerDe.GSON.toJson(this); + } + + public static EntityAuditEvent fromString(String eventString) { + return SerDe.GSON.fromJson(eventString, EntityAuditEvent.class); + } + + public String getEntityId() { + return entityId; + } + + public long getTimestamp() { + return timestamp; + } + + public String getUser() { + return user; + } + + public EntityAuditAction getAction() { + return action; + } + + public String getDetails() { + return details; + } + + public void setEntityId(String entityId) { + this.entityId = entityId; + } + + public void setTimestamp(long timestamp) { + this.timestamp = timestamp; + } + + public void setUser(String user) { + this.user = user; + } + + public void setAction(EntityAuditAction action) { + this.action = action; + } + + public void setDetails(String details) { + this.details = details; + } + + public String getEventKey() { + return eventKey; + } + + public void setEventKey(String eventKey) { + this.eventKey = eventKey; + } + + public IReferenceableInstance getEntityDefinition() { + return entityDefinition; + } + + public String getEntityDefinitionString() { + if (entityDefinition != null) { + return InstanceSerialization.toJson(entityDefinition, true); + } + return null; + } + + public void setEntityDefinition(String entityDefinition) { + this.entityDefinition = InstanceSerialization.fromJsonReferenceable(entityDefinition, true); + } +} http://git-wip-us.apache.org/repos/asf/atlas/blob/187730dd/client/client-v1/src/main/java/org/apache/atlas/SerDe.java ---------------------------------------------------------------------- diff --git a/client/client-v1/src/main/java/org/apache/atlas/SerDe.java b/client/client-v1/src/main/java/org/apache/atlas/SerDe.java new file mode 100644 index 0000000..cdc3509 --- /dev/null +++ b/client/client-v1/src/main/java/org/apache/atlas/SerDe.java @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonDeserializationContext; +import com.google.gson.JsonDeserializer; +import com.google.gson.JsonElement; +import com.google.gson.JsonParser; +import com.google.gson.JsonSerializationContext; +import com.google.gson.JsonSerializer; +import org.apache.atlas.typesystem.IReferenceableInstance; +import org.apache.atlas.typesystem.IStruct; +import org.apache.atlas.typesystem.Referenceable; +import org.apache.atlas.typesystem.Struct; +import org.apache.atlas.typesystem.json.InstanceSerialization; + +import java.lang.reflect.Type; + +@Deprecated +public class SerDe { + public static final Gson GSON = new GsonBuilder(). + registerTypeAdapter(IStruct.class, new StructDeserializer()). + registerTypeAdapter(IReferenceableInstance.class, new ReferenceableSerializerDeserializer()). + registerTypeAdapter(Referenceable.class, new ReferenceableSerializerDeserializer()). + create(); + + /** + * Serde for Struct used by AbstractNotificationConsumer.GSON. + */ + public static final class StructDeserializer implements JsonDeserializer<IStruct>, JsonSerializer<IStruct> { + @Override + public IStruct deserialize(final JsonElement json, final Type type, + final JsonDeserializationContext context) { + return context.deserialize(json, Struct.class); + } + + @Override + public JsonElement serialize(IStruct src, Type typeOfSrc, JsonSerializationContext context) { + String instanceJson = InstanceSerialization.toJson(src, true); + return new JsonParser().parse(instanceJson).getAsJsonObject(); + } + } + + /** + * Serde for Referenceable used by AbstractNotificationConsumer.GSON. + */ + public static final class ReferenceableSerializerDeserializer implements JsonDeserializer<IStruct>, + JsonSerializer<IReferenceableInstance> { + @Override + public IReferenceableInstance deserialize(final JsonElement json, final Type type, + final JsonDeserializationContext context) { + + return InstanceSerialization.fromJsonReferenceable(json.toString(), true); + } + + @Override + public JsonElement serialize(IReferenceableInstance src, Type typeOfSrc, JsonSerializationContext context) { + String instanceJson = InstanceSerialization.toJson(src, true); + return new JsonParser().parse(instanceJson).getAsJsonObject(); + } + } +}
