ATLAS-47 Entity mutations for complex types (sumasai via shwethags)
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/51656991 Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/51656991 Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/51656991 Branch: refs/heads/branch-0.6-incubating Commit: 51656991fc008e57936bfc12afa51b9e372ae6a5 Parents: 6c3f096 Author: Shwetha GS <[email protected]> Authored: Tue Dec 8 12:09:57 2015 +0530 Committer: Shwetha GS <[email protected]> Committed: Tue Dec 8 12:09:57 2015 +0530 ---------------------------------------------------------------------- .../atlas/hive/bridge/HiveMetaStoreBridge.java | 9 +- .../org/apache/atlas/hive/hook/HiveHookIT.java | 2 +- .../main/java/org/apache/atlas/AtlasClient.java | 141 ++- common/pom.xml | 14 +- .../org/apache/atlas/ApplicationProperties.java | 86 ++ .../java/org/apache/atlas/AtlasException.java | 44 + .../atlas/listener/EntityChangeListener.java | 69 -- .../atlas/listener/TypesChangeListener.java | 49 - .../java/org/apache/atlas/utils/MD5Utils.java | 59 + .../org/apache/atlas/utils/ParamChecker.java | 148 +++ docs/src/site/twiki/Configuration.twiki | 4 +- docs/src/site/twiki/InstallationSteps.twiki | 2 +- notification/pom.xml | 5 + .../NotificationEntityChangeListener.java | 4 +- pom.xml | 7 + release-log.txt | 1 + repository/pom.xml | 2 +- .../apache/atlas/RepositoryMetadataModule.java | 4 +- .../atlas/discovery/DiscoveryException.java | 74 -- .../atlas/discovery/DiscoveryService.java | 52 - .../atlas/discovery/HiveLineageService.java | 4 +- .../apache/atlas/discovery/LineageService.java | 67 -- .../graph/DefaultGraphPersistenceStrategy.java | 14 +- .../atlas/repository/EntityExistsException.java | 32 - .../repository/EntityNotFoundException.java | 44 - .../atlas/repository/MetadataRepository.java | 38 +- .../atlas/repository/graph/EntityProcessor.java | 81 ++ .../atlas/repository/graph/FullTextMapper.java | 146 +++ .../graph/GraphBackedMetadataRepository.java | 1094 +----------------- .../atlas/repository/graph/GraphHelper.java | 180 ++- .../graph/GraphToTypedInstanceMapper.java | 419 +++++++ .../graph/TypedInstanceToGraphMapper.java | 633 ++++++++++ .../memory/ReplaceIdWithInstance.java | 1 - .../atlas/services/DefaultMetadataService.java | 168 ++- .../apache/atlas/services/MetadataService.java | 157 --- .../query/GraphPersistenceStrategies.scala | 4 +- .../apache/atlas/BaseHiveRepositoryTest.java | 25 +- .../test/java/org/apache/atlas/TestUtils.java | 29 +- .../GraphBackedDiscoveryServiceTest.java | 4 +- .../atlas/discovery/HiveLineageServiceTest.java | 6 +- .../GraphBackedMetadataRepositoryTest.java | 32 +- .../graph/GraphRepoMapperScaleTest.java | 18 +- .../service/DefaultMetadataServiceTest.java | 509 +++++++- .../org/apache/atlas/query/GremlinTest.scala | 1 - server-api/pom.xml | 53 + .../atlas/discovery/DiscoveryException.java | 74 ++ .../atlas/discovery/DiscoveryService.java | 52 + .../apache/atlas/discovery/LineageService.java | 67 ++ .../atlas/listener/EntityChangeListener.java | 69 ++ .../atlas/listener/TypesChangeListener.java | 49 + .../apache/atlas/services/MetadataService.java | 187 +++ .../exception/EntityExistsException.java | 32 + .../exception/EntityNotFoundException.java | 46 + typesystem/pom.xml | 9 +- .../org/apache/atlas/ApplicationProperties.java | 86 -- .../java/org/apache/atlas/AtlasException.java | 44 - .../java/org/apache/atlas/ParamChecker.java | 148 --- .../org/apache/atlas/TypeExistsException.java | 25 - .../org/apache/atlas/TypeNotFoundException.java | 44 - .../org/apache/atlas/typesystem/IInstance.java | 2 + .../apache/atlas/typesystem/ITypedInstance.java | 5 +- .../apache/atlas/typesystem/Referenceable.java | 43 +- .../org/apache/atlas/typesystem/Struct.java | 46 + .../exception/TypeExistsException.java | 27 + .../exception/TypeNotFoundException.java | 46 + .../persistence/DownCastStructInstance.java | 5 + .../apache/atlas/typesystem/persistence/Id.java | 14 + .../persistence/ReferenceableInstance.java | 14 +- .../typesystem/persistence/StructInstance.java | 35 + .../typesystem/types/AttributeDefinition.java | 2 +- .../atlas/typesystem/types/ClassType.java | 28 +- .../atlas/typesystem/types/DataTypes.java | 44 +- .../apache/atlas/typesystem/types/EnumType.java | 11 +- .../typesystem/types/EnumTypeDefinition.java | 2 +- .../atlas/typesystem/types/EnumValue.java | 2 +- .../atlas/typesystem/types/IDataType.java | 4 + .../atlas/typesystem/types/Multiplicity.java | 4 +- .../typesystem/types/ObjectGraphWalker.java | 6 + .../atlas/typesystem/types/StructType.java | 20 + .../typesystem/types/StructTypeDefinition.java | 2 +- .../atlas/typesystem/types/TraitType.java | 20 + .../atlas/typesystem/types/TypeSystem.java | 4 +- .../atlas/typesystem/json/Serialization.scala | 41 +- .../apache/atlas/web/filters/AuditFilter.java | 6 +- .../atlas/web/resources/EntityResource.java | 213 +++- .../web/resources/HiveLineageResource.java | 4 +- .../resources/MetadataDiscoveryResource.java | 2 +- .../atlas/web/resources/TypesResource.java | 2 +- .../org/apache/atlas/web/util/Servlets.java | 2 +- .../notification/EntityNotificationIT.java | 2 +- .../NotificationHookConsumerIT.java | 2 +- .../atlas/web/resources/BaseResourceIT.java | 7 +- .../web/resources/EntityJerseyResourceIT.java | 116 +- 93 files changed, 4016 insertions(+), 2234 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java index 90f3d96..f367317 100755 --- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java +++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java @@ -479,12 +479,13 @@ public class HiveMetaStoreBridge { HiveDataModelGenerator dataModelGenerator = new HiveDataModelGenerator(); AtlasClient dgiClient = getAtlasClient(); - //Register hive data model if its not already registered - if (dgiClient.getType(HiveDataTypes.HIVE_PROCESS.getName()) == null) { + try { + dgiClient.getType(HiveDataTypes.HIVE_PROCESS.getName()); + LOG.info("Hive data model is already registered!"); + } catch(AtlasServiceException ase) { + //Expected in case types do not exist LOG.info("Registering Hive data model"); dgiClient.createType(dataModelGenerator.getModelAsJson()); - } else { - LOG.info("Hive data model is already registered!"); } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java index 643a29a..5447de5 100755 --- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java +++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java @@ -20,7 +20,7 @@ package org.apache.atlas.hive.hook; import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasClient; -import org.apache.atlas.ParamChecker; +import org.apache.atlas.utils.ParamChecker; import org.apache.atlas.hive.bridge.HiveMetaStoreBridge; import org.apache.atlas.hive.model.HiveDataModelGenerator; import org.apache.atlas.hive.model.HiveDataTypes; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/client/src/main/java/org/apache/atlas/AtlasClient.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/atlas/AtlasClient.java b/client/src/main/java/org/apache/atlas/AtlasClient.java index 2e8a268..114548c 100755 --- a/client/src/main/java/org/apache/atlas/AtlasClient.java +++ b/client/src/main/java/org/apache/atlas/AtlasClient.java @@ -133,40 +133,43 @@ public class AtlasClient { enum API { //Type operations - CREATE_TYPE(BASE_URI + TYPES, HttpMethod.POST), - UPDATE_TYPE(BASE_URI + TYPES, HttpMethod.PUT), - GET_TYPE(BASE_URI + TYPES, HttpMethod.GET), - LIST_TYPES(BASE_URI + TYPES, HttpMethod.GET), - LIST_TRAIT_TYPES(BASE_URI + TYPES + "?type=trait", HttpMethod.GET), + CREATE_TYPE(BASE_URI + TYPES, HttpMethod.POST, Response.Status.CREATED), + UPDATE_TYPE(BASE_URI + TYPES, HttpMethod.PUT, Response.Status.OK), + GET_TYPE(BASE_URI + TYPES, HttpMethod.GET, Response.Status.OK), + LIST_TYPES(BASE_URI + TYPES, HttpMethod.GET, Response.Status.OK), + LIST_TRAIT_TYPES(BASE_URI + TYPES + "?type=trait", HttpMethod.GET, Response.Status.OK), //Entity operations - CREATE_ENTITY(BASE_URI + URI_ENTITY, HttpMethod.POST), - GET_ENTITY(BASE_URI + URI_ENTITY, HttpMethod.GET), - UPDATE_ENTITY(BASE_URI + URI_ENTITY, HttpMethod.PUT), - LIST_ENTITIES(BASE_URI + URI_ENTITY, HttpMethod.GET), + CREATE_ENTITY(BASE_URI + URI_ENTITY, HttpMethod.POST, Response.Status.CREATED), + GET_ENTITY(BASE_URI + URI_ENTITY, HttpMethod.GET, Response.Status.OK), + UPDATE_ENTITY(BASE_URI + URI_ENTITY, HttpMethod.PUT, Response.Status.OK), + UPDATE_ENTITY_PARTIAL(BASE_URI + URI_ENTITY, HttpMethod.POST, Response.Status.OK), + LIST_ENTITIES(BASE_URI + URI_ENTITY, HttpMethod.GET, Response.Status.OK), //Trait operations - ADD_TRAITS(BASE_URI + URI_ENTITY, HttpMethod.POST), - DELETE_TRAITS(BASE_URI + URI_ENTITY, HttpMethod.DELETE), - LIST_TRAITS(BASE_URI + URI_ENTITY, HttpMethod.GET), + ADD_TRAITS(BASE_URI + URI_ENTITY, HttpMethod.POST, Response.Status.CREATED), + DELETE_TRAITS(BASE_URI + URI_ENTITY, HttpMethod.DELETE, Response.Status.OK), + LIST_TRAITS(BASE_URI + URI_ENTITY, HttpMethod.GET, Response.Status.OK), //Search operations - SEARCH(BASE_URI + URI_SEARCH, HttpMethod.GET), - SEARCH_DSL(BASE_URI + URI_SEARCH + "/dsl", HttpMethod.GET), - SEARCH_GREMLIN(BASE_URI + URI_SEARCH + "/gremlin", HttpMethod.GET), - SEARCH_FULL_TEXT(BASE_URI + URI_SEARCH + "/fulltext", HttpMethod.GET), + SEARCH(BASE_URI + URI_SEARCH, HttpMethod.GET, Response.Status.OK), + SEARCH_DSL(BASE_URI + URI_SEARCH + "/dsl", HttpMethod.GET, Response.Status.OK), + SEARCH_GREMLIN(BASE_URI + URI_SEARCH + "/gremlin", HttpMethod.GET, Response.Status.OK), + SEARCH_FULL_TEXT(BASE_URI + URI_SEARCH + "/fulltext", HttpMethod.GET, Response.Status.OK), //Lineage operations - LINEAGE_INPUTS_GRAPH(BASE_URI + URI_LINEAGE, HttpMethod.GET), - LINEAGE_OUTPUTS_GRAPH(BASE_URI + URI_LINEAGE, HttpMethod.GET), - LINEAGE_SCHEMA(BASE_URI + URI_LINEAGE, HttpMethod.GET); + LINEAGE_INPUTS_GRAPH(BASE_URI + URI_LINEAGE, HttpMethod.GET, Response.Status.OK), + LINEAGE_OUTPUTS_GRAPH(BASE_URI + URI_LINEAGE, HttpMethod.GET, Response.Status.OK), + LINEAGE_SCHEMA(BASE_URI + URI_LINEAGE, HttpMethod.GET, Response.Status.OK); private final String method; private final String path; + private final Response.Status status; - API(String path, String method) { + API(String path, String method, Response.Status status) { this.path = path; this.method = method; + this.status = status; } public String getMethod() { @@ -176,6 +179,8 @@ public class AtlasClient { public String getPath() { return path; } + + public Response.Status getExpectedStatus() { return status; } } /** @@ -231,7 +236,7 @@ public class AtlasClient { JSONObject response = callAPIWithResource(API.GET_TYPE, resource); return response.getString(DEFINITION); } catch (AtlasServiceException e) { - if (e.getStatus() == ClientResponse.Status.NOT_FOUND) { + if (Response.Status.NOT_FOUND.equals(e.getStatus())) { return null; } throw e; @@ -266,11 +271,82 @@ public class AtlasClient { } public JSONArray createEntity(Referenceable... entities) throws AtlasServiceException { + JSONArray entityArray = getEntitiesArray(entities); + return createEntity(entityArray); + } + + private JSONArray getEntitiesArray(Referenceable[] entities) { JSONArray entityArray = new JSONArray(entities.length); for (Referenceable entity : entities) { entityArray.put(InstanceSerialization.toJson(entity, true)); } - return createEntity(entityArray); + return entityArray; + } + + /** + * Replaces entity definitions identified by their guid or unique attribute + * Updates properties set in the definition for the entity corresponding to guid + * @param entities entities to be updated + * @return json array of guids which were updated/created + * @throws AtlasServiceException + */ + public JSONArray updateEntities(Referenceable... entities) throws AtlasServiceException { + JSONArray entitiesArray = getEntitiesArray(entities); + JSONObject response = callAPI(API.UPDATE_ENTITY, entitiesArray.toString()); + try { + return response.getJSONArray(GUID); + } catch (JSONException e) { + throw new AtlasServiceException(API.UPDATE_ENTITY, e); + } + } + + /** + * Supports Partial updates + * Updates property for the entity corresponding to guid + * @param guid guid + * @param attribute property key + * @param value property value + */ + public void updateEntityAttribute(String guid, String attribute, String value) throws AtlasServiceException { + API api = API.UPDATE_ENTITY_PARTIAL; + WebResource resource = getResource(api, guid); + resource = resource.queryParam(ATTRIBUTE_NAME, attribute); + callAPIWithResource(api, resource, value); + } + + /** + * Supports Partial updates + * Updates properties set in the definition for the entity corresponding to guid + * @param guid guid + * @param entity entity definition + */ + public void updateEntity(String guid, Referenceable entity) throws AtlasServiceException { + String entityJson = InstanceSerialization.toJson(entity, true); + callAPI(API.UPDATE_ENTITY_PARTIAL, entityJson, guid); + } + + /** + * Supports Partial updates + * Updates properties set in the definition for the entity corresponding to guid + * @param entityType Type of the entity being updated + * @param uniqueAttributeName Attribute Name that uniquely identifies the entity + * @param uniqueAttributeValue Attribute Value that uniquely identifies the entity + * @param entity entity definition + */ + public String updateEntity(String entityType, String uniqueAttributeName, String uniqueAttributeValue, + Referenceable entity) throws AtlasServiceException { + API api = API.UPDATE_ENTITY_PARTIAL; + WebResource resource = getResource(api, "qualifiedName"); + resource = resource.queryParam(TYPE, entityType); + resource = resource.queryParam(ATTRIBUTE_NAME, uniqueAttributeName); + resource = resource.queryParam(ATTRIBUTE_VALUE, uniqueAttributeValue); + String entityJson = InstanceSerialization.toJson(entity, true); + JSONObject response = callAPIWithResource(api, resource, entityJson); + try { + return response.getString(GUID); + } catch (JSONException e) { + throw new AtlasServiceException(api, e); + } } /** @@ -351,19 +427,6 @@ public class AtlasClient { } /** - * Updates property for the entity corresponding to guid - * @param guid guid - * @param property property key - * @param value property value - */ - public JSONObject updateEntity(String guid, String property, String value) throws AtlasServiceException { - WebResource resource = getResource(API.UPDATE_ENTITY, guid); - resource = resource.queryParam(ATTRIBUTE_NAME, property); - resource = resource.queryParam(ATTRIBUTE_VALUE, value); - return callAPIWithResource(API.UPDATE_ENTITY, resource); - } - - /** * Search using gremlin/dsl/full text * @param searchQuery * @return @@ -488,13 +551,11 @@ public class AtlasClient { } private JSONObject callAPIWithResource(API api, WebResource resource, Object requestObject) - throws AtlasServiceException { + throws AtlasServiceException { ClientResponse clientResponse = resource.accept(JSON_MEDIA_TYPE).type(JSON_MEDIA_TYPE) - .method(api.getMethod(), ClientResponse.class, requestObject); + .method(api.getMethod(), ClientResponse.class, requestObject); - Response.Status expectedStatus = - HttpMethod.POST.equals(api.getMethod()) ? Response.Status.CREATED : Response.Status.OK; - if (clientResponse.getStatus() == expectedStatus.getStatusCode()) { + if (clientResponse.getStatus() == api.getExpectedStatus().getStatusCode()) { String responseAsString = clientResponse.getEntity(String.class); try { return new JSONObject(responseAsString); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/common/pom.xml ---------------------------------------------------------------------- diff --git a/common/pom.xml b/common/pom.xml index 5498105..a7a5544 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -33,13 +33,19 @@ <dependencies> <dependency> - <groupId>org.apache.atlas</groupId> - <artifactId>atlas-typesystem</artifactId> + <groupId>org.testng</groupId> + <artifactId>testng</artifactId> </dependency> <dependency> - <groupId>org.testng</groupId> - <artifactId>testng</artifactId> + <groupId>com.google.inject</groupId> + <artifactId>guice</artifactId> </dependency> + + <dependency> + <groupId>commons-configuration</groupId> + <artifactId>commons-configuration</artifactId> + </dependency> + </dependencies> </project> http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/common/src/main/java/org/apache/atlas/ApplicationProperties.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/atlas/ApplicationProperties.java b/common/src/main/java/org/apache/atlas/ApplicationProperties.java new file mode 100644 index 0000000..738ec53 --- /dev/null +++ b/common/src/main/java/org/apache/atlas/ApplicationProperties.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas; + +import org.apache.commons.configuration.CompositeConfiguration; +import org.apache.commons.configuration.Configuration; +import org.apache.commons.configuration.ConfigurationException; +import org.apache.commons.configuration.PropertiesConfiguration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.net.URL; +import java.util.Arrays; +import java.util.Iterator; + +public class ApplicationProperties extends PropertiesConfiguration { + private static final Logger LOG = LoggerFactory.getLogger(ApplicationProperties.class); + + public static final String APPLICATION_PROPERTIES = "application.properties"; + public static final String CLIENT_PROPERTIES = "client.properties"; + + private static Configuration INSTANCE = null; + + private ApplicationProperties(URL url) throws ConfigurationException { + super(url); + } + + public static Configuration get() throws AtlasException { + if (INSTANCE == null) { + synchronized (ApplicationProperties.class) { + if (INSTANCE == null) { + Configuration applicationProperties = get(APPLICATION_PROPERTIES); + Configuration clientProperties = get(CLIENT_PROPERTIES); + INSTANCE = new CompositeConfiguration(Arrays.asList(applicationProperties, clientProperties)); + } + } + } + return INSTANCE; + } + + public static Configuration get(String fileName) throws AtlasException { + String confLocation = System.getProperty("atlas.conf"); + try { + URL url = confLocation == null ? ApplicationProperties.class.getResource("/" + fileName) + : new File(confLocation, fileName).toURI().toURL(); + LOG.info("Loading {} from {}", fileName, url); + + Configuration configuration = new ApplicationProperties(url).interpolatedConfiguration(); + logConfiguration(configuration); + return configuration; + } catch (Exception e) { + throw new AtlasException("Failed to load application properties", e); + } + } + + private static void logConfiguration(Configuration configuration) { + if (LOG.isDebugEnabled()) { + Iterator<String> keys = configuration.getKeys(); + LOG.debug("Configuration loaded:"); + while (keys.hasNext()) { + String key = keys.next(); + LOG.debug("{} = {}", key, configuration.getProperty(key)); + } + } + } + + public static final Configuration getSubsetConfiguration(Configuration inConf, String prefix) { + return inConf.subset(prefix); + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/common/src/main/java/org/apache/atlas/AtlasException.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/atlas/AtlasException.java b/common/src/main/java/org/apache/atlas/AtlasException.java new file mode 100644 index 0000000..2eb0658 --- /dev/null +++ b/common/src/main/java/org/apache/atlas/AtlasException.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas; + +/** + * Base Exception class for metadata API. + */ +public class AtlasException extends Exception { + + public AtlasException() { + } + + public AtlasException(String message) { + super(message); + } + + public AtlasException(String message, Throwable cause) { + super(message, cause); + } + + public AtlasException(Throwable cause) { + super(cause); + } + + public AtlasException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/common/src/main/java/org/apache/atlas/listener/EntityChangeListener.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/atlas/listener/EntityChangeListener.java b/common/src/main/java/org/apache/atlas/listener/EntityChangeListener.java deleted file mode 100644 index 08ed0d3..0000000 --- a/common/src/main/java/org/apache/atlas/listener/EntityChangeListener.java +++ /dev/null @@ -1,69 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.listener; - -import org.apache.atlas.AtlasException; -import org.apache.atlas.typesystem.IStruct; -import org.apache.atlas.typesystem.ITypedReferenceableInstance; - -import java.util.Collection; - -/** - * Entity (a Typed instance) change notification listener. - */ -public interface EntityChangeListener { - - /** - * This is upon adding new entities to the repository. - * - * @param entities the created entities - * - * @throws AtlasException if the listener notification fails - */ - void onEntitiesAdded(Collection<ITypedReferenceableInstance> entities) throws AtlasException; - - /** - * This is upon updating an entity. - * - * @param entity the updated entity - * - * @throws AtlasException if the listener notification fails - */ - void onEntityUpdated(ITypedReferenceableInstance entity) throws AtlasException; - - /** - * This is upon adding a new trait to a typed instance. - * - * @param entity the entity - * @param trait trait that needs to be added to entity - * - * @throws AtlasException if the listener notification fails - */ - void onTraitAdded(ITypedReferenceableInstance entity, IStruct trait) throws AtlasException; - - /** - * This is upon deleting a trait from a typed instance. - * - * @param entity the entity - * @param traitName trait name for the instance that needs to be deleted from entity - * - * @throws AtlasException if the listener notification fails - */ - void onTraitDeleted(ITypedReferenceableInstance entity, String traitName) throws AtlasException; -} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/common/src/main/java/org/apache/atlas/listener/TypesChangeListener.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/atlas/listener/TypesChangeListener.java b/common/src/main/java/org/apache/atlas/listener/TypesChangeListener.java deleted file mode 100644 index 5ff6d4a..0000000 --- a/common/src/main/java/org/apache/atlas/listener/TypesChangeListener.java +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.listener; - -import org.apache.atlas.AtlasException; -import org.apache.atlas.typesystem.types.IDataType; - -import java.util.Collection; - -/** - * Types change notification listener. - */ -public interface TypesChangeListener { - - /** - * This is upon adding new type(s) to Store. - * - * @param dataTypes data type - * @throws AtlasException - */ - void onAdd(Collection<? extends IDataType> dataTypes) throws AtlasException; - - /** - * This is upon removing an existing type from the Store. - * - * @param typeName type name - * @throws AtlasException - */ - // void onRemove(String typeName) throws MetadataException; - - //This is upon updating an existing type to the store - void onChange(Collection<? extends IDataType> dataTypes) throws AtlasException; -} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/common/src/main/java/org/apache/atlas/utils/MD5Utils.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/atlas/utils/MD5Utils.java b/common/src/main/java/org/apache/atlas/utils/MD5Utils.java new file mode 100644 index 0000000..35e4744 --- /dev/null +++ b/common/src/main/java/org/apache/atlas/utils/MD5Utils.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.utils; + +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; + +public class MD5Utils { + + private static final ThreadLocal<MessageDigest> DIGESTER_FACTORY = + new ThreadLocal<MessageDigest>() { + @Override + protected MessageDigest initialValue() { + try { + return MessageDigest.getInstance("MD5"); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } + } + }; + + /** + * Create a thread local MD5 digester + */ + public static MessageDigest getDigester() { + MessageDigest digester = DIGESTER_FACTORY.get(); + digester.reset(); + return digester; + } + + private static final char[] HEX_DIGITS = + {'0','1','2','3','4','5','6','7','8','9','a','b','c','d','e','f'}; + + public static String toString(byte[] digest) { + StringBuilder buf = new StringBuilder(MD5_LEN*2); + for (int i = 0; i < MD5_LEN; i++) { + int b = digest[i]; + buf.append(HEX_DIGITS[(b >> 4) & 0xf]); + buf.append(HEX_DIGITS[b & 0xf]); + } + return buf.toString(); + } + + public static final int MD5_LEN = 16; +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/common/src/main/java/org/apache/atlas/utils/ParamChecker.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/atlas/utils/ParamChecker.java b/common/src/main/java/org/apache/atlas/utils/ParamChecker.java new file mode 100644 index 0000000..ab543e6 --- /dev/null +++ b/common/src/main/java/org/apache/atlas/utils/ParamChecker.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.utils; + +import java.util.Arrays; +import java.util.Collection; + +public class ParamChecker { + + /** + * Check that a value is not null. If null throws an IllegalArgumentException. + * + * @param obj value. + * @param name parameter name for the exception message. + * @return the given value. + */ + public static <T> T notNull(T obj, String name) { + if (obj == null) { + throw new IllegalArgumentException(name + " cannot be null"); + } + return obj; + } + + /** + * Check that a list is not null and that none of its elements is null. If null or if the list has emtpy elements + * throws an IllegalArgumentException. + * @param list the list of T. + * @param name parameter name for the exception message. + */ + public static <T> Collection<T> notNullElements(Collection<T> list, String name) { + notEmpty(list, name); + for (T ele : list) { + notNull(ele, String.format("Collection %s element %s", name, ele)); + } + return list; + } + + /** + * Check that a list is not null and that none of its elements is null. If null or if the list has emtpy elements + * throws an IllegalArgumentException. + * @param array the array of T. + * @param name parameter name for the exception message. + */ + public static <T> T[] notNullElements(T[] array, String name) { + notEmpty(Arrays.asList(array), name); + for (T ele : array) { + notNull(ele, String.format("Collection %s element %s", name, ele)); + } + return array; + } + + /** + * Check that a list is not null and not empty. + * @param list the list of T. + * @param name parameter name for the exception message. + */ + public static <T> Collection<T> notEmpty(Collection<T> list, String name) { + notNull(list, name); + if (list.isEmpty()) { + throw new IllegalArgumentException(String.format("Collection %s is empty", name)); + } + return list; + } + + /** + * Check that a string is not null and not empty. If null or emtpy throws an IllegalArgumentException. + * + * @param value value. + * @param name parameter name for the exception message. + * @return the given value. + */ + public static String notEmpty(String value, String name) { + return notEmpty(value, name, null); + } + + /** + * Check that a string is not empty if its not null. + * + * @param value value. + * @param name parameter name for the exception message. + * @return the given value. + */ + public static String notEmptyIfNotNull(String value, String name) { + return notEmptyIfNotNull(value, name, null); + } + + /** + * Check that a string is not empty if its not null. + * + * @param value value. + * @param name parameter name for the exception message. + * @return the given value. + */ + public static String notEmptyIfNotNull(String value, String name, String info) { + if (value == null) { + return value; + } + + if (value.trim().length() == 0) { + throw new IllegalArgumentException(name + " cannot be empty" + (info == null ? "" : ", " + info)); + } + return value.trim(); + } + + /** + * Check that a string is not null and not empty. If null or emtpy throws an IllegalArgumentException. + * + * @param value value. + * @param name parameter name for the exception message. + * @param info additional information to be printed with the exception message + * @return the given value. + */ + public static String notEmpty(String value, String name, String info) { + if (value == null) { + throw new IllegalArgumentException(name + " cannot be null" + (info == null ? "" : ", " + info)); + } + return notEmptyIfNotNull(value, name, info); + } + + /** + * Check that a list is not null and that none of its elements is null. If null or if the list has emtpy elements + * throws an IllegalArgumentException. + * @param list the list of strings. + * @param name parameter name for the exception message. + */ + public static Collection<String> notEmptyElements(Collection<String> list, String name) { + notEmpty(list, name); + for (String ele : list) { + notEmpty(ele, String.format("list %s element %s", name, ele)); + } + return list; + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/docs/src/site/twiki/Configuration.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/Configuration.twiki b/docs/src/site/twiki/Configuration.twiki index 21d1c0e..526b300 100644 --- a/docs/src/site/twiki/Configuration.twiki +++ b/docs/src/site/twiki/Configuration.twiki @@ -60,8 +60,9 @@ Without Ranger, HBase shell can be used to set the permissions. </verbatim> ---++++ Graph Search Index + This section sets up the graph db - titan - to use an search indexing system. The example -configuration below setsup to use an embedded Elastic search indexing system. +configuration below sets up to use an embedded Elastic search indexing system. <verbatim> atlas.graph.index.search.backend=elasticsearch @@ -72,6 +73,7 @@ atlas.graph.index.search.elasticsearch.create.sleep=2000 </verbatim> ---++++ Graph Search Index - Solr +Please note that Solr installation in Cloud mode is a prerequisite before configuring Solr as the search indexing backend. Refer InstallationSteps section for Solr installation/configuration. <verbatim> atlas.graph.index.search.backend=solr5 http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/docs/src/site/twiki/InstallationSteps.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/InstallationSteps.twiki b/docs/src/site/twiki/InstallationSteps.twiki index 8d1f3d0..0eefb20 100644 --- a/docs/src/site/twiki/InstallationSteps.twiki +++ b/docs/src/site/twiki/InstallationSteps.twiki @@ -124,7 +124,7 @@ export METADATA_SERVER_OPTS="-Djava.awt.headless=true -Djava.security.krb5.realm * Hbase as the Storage Backend for the Graph Repository By default, Atlas uses Titan as the graph repository and is the only graph repository implementation available currently. -The HBase versions currently supported are 0.98.x, 1.0.x, 1.1.x. For configuring ATLAS graph persistence on HBase, please go through the "Configuration - Graph persistence engine - HBase" section +The HBase versions currently supported are 1.1.x. For configuring ATLAS graph persistence on HBase, please go through the "Configuration - Graph persistence engine - HBase" section for more details. Pre-requisites for running HBase as a distributed cluster http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/notification/pom.xml ---------------------------------------------------------------------- diff --git a/notification/pom.xml b/notification/pom.xml index 796ea17..125ef75 100644 --- a/notification/pom.xml +++ b/notification/pom.xml @@ -48,6 +48,11 @@ </dependency> <dependency> + <groupId>org.apache.atlas</groupId> + <artifactId>atlas-server-api</artifactId> + </dependency> + + <dependency> <groupId>org.apache.kafka</groupId> <artifactId>kafka-clients</artifactId> </dependency> http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/notification/src/main/java/org/apache/atlas/notification/entity/NotificationEntityChangeListener.java ---------------------------------------------------------------------- diff --git a/notification/src/main/java/org/apache/atlas/notification/entity/NotificationEntityChangeListener.java b/notification/src/main/java/org/apache/atlas/notification/entity/NotificationEntityChangeListener.java index 23a6d69..e2d16df 100644 --- a/notification/src/main/java/org/apache/atlas/notification/entity/NotificationEntityChangeListener.java +++ b/notification/src/main/java/org/apache/atlas/notification/entity/NotificationEntityChangeListener.java @@ -73,8 +73,8 @@ public class NotificationEntityChangeListener implements EntityChangeListener { } @Override - public void onEntityUpdated(ITypedReferenceableInstance entity) throws AtlasException { - notifyOfEntityEvent(Collections.singleton(entity), EntityNotification.OperationType.ENTITY_UPDATE); + public void onEntitiesUpdated(Collection<ITypedReferenceableInstance> entities) throws AtlasException { + notifyOfEntityEvent(entities, EntityNotification.OperationType.ENTITY_UPDATE); } @Override http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 2f7ee82..929d255 100755 --- a/pom.xml +++ b/pom.xml @@ -407,6 +407,7 @@ <modules> <module>common</module> <module>typesystem</module> + <module>server-api</module> <module>notification</module> <module>client</module> <module>titan</module> @@ -932,6 +933,12 @@ <dependency> <groupId>org.apache.atlas</groupId> + <artifactId>atlas-server-api</artifactId> + <version>${project.version}</version> + </dependency> + + <dependency> + <groupId>org.apache.atlas</groupId> <artifactId>atlas-repository</artifactId> <version>${project.version}</version> </dependency> http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 4ebba32..1dae6fb 100644 --- a/release-log.txt +++ b/release-log.txt @@ -14,6 +14,7 @@ ATLAS-54 Rename configs in hive hook (shwethags) ATLAS-3 Mixed Index creation fails with Date types (sumasai via shwethags) ALL CHANGES: +ATLAS-47 Entity mutations for complex types (sumasai via shwethags) ATLAS-345 UI: Should allow tag addition on any search result that returns a reference-able entity (darshankumar89 via shwethags) ATLAS-279 UI not displaying results for certain successful "select" search queries (anilsg via shwethags) ATLAS-242 The qualified name for hive entities should be backward compatible (shwethags) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/repository/pom.xml ---------------------------------------------------------------------- diff --git a/repository/pom.xml b/repository/pom.xml index 6e1baee..5810956 100755 --- a/repository/pom.xml +++ b/repository/pom.xml @@ -41,7 +41,7 @@ <dependency> <groupId>org.apache.atlas</groupId> - <artifactId>atlas-common</artifactId> + <artifactId>atlas-server-api</artifactId> </dependency> <dependency> http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java b/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java index b12c9a8..f77c237 100755 --- a/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java +++ b/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java @@ -49,11 +49,13 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule { @Override protected void configure() { // special wiring for Titan Graph + + + ThrowingProviderBinder.create(binder()).bind(GraphProvider.class, TitanGraph.class).to(TitanGraphProvider.class) .asEagerSingleton(); // allow for dynamic binding of the metadata repo & graph service - // bind the MetadataRepositoryService interface to an implementation bind(MetadataRepository.class).to(GraphBackedMetadataRepository.class).asEagerSingleton(); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/repository/src/main/java/org/apache/atlas/discovery/DiscoveryException.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/discovery/DiscoveryException.java b/repository/src/main/java/org/apache/atlas/discovery/DiscoveryException.java deleted file mode 100755 index ba69af7..0000000 --- a/repository/src/main/java/org/apache/atlas/discovery/DiscoveryException.java +++ /dev/null @@ -1,74 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.discovery; - -import org.apache.atlas.AtlasException; - -import java.security.PrivilegedActionException; - -public class DiscoveryException extends AtlasException { - - /** - * Constructs a new exception with the specified detail message. The - * cause is not initialized, and may subsequently be initialized by - * a call to {@link #initCause}. - * - * @param message the detail message. The detail message is saved for - * later retrieval by the {@link #getMessage()} method. - */ - public DiscoveryException(String message) { - super(message); - } - - /** - * Constructs a new exception with the specified detail message and - * cause. <p>Note that the detail message associated with - * {@code cause} is <i>not</i> automatically incorporated in - * this exception's detail message. - * - * @param message the detail message (which is saved for later retrieval - * by the {@link #getMessage()} method). - * @param cause the cause (which is saved for later retrieval by the - * {@link #getCause()} method). (A <tt>null</tt> value is - * permitted, and indicates that the cause is nonexistent or - * unknown.) - * @since 1.4 - */ - public DiscoveryException(String message, Throwable cause) { - super(message, cause); - } - - /** - * Constructs a new exception with the specified cause and a detail - * message of <tt>(cause==null ? null : cause.toString())</tt> (which - * typically contains the class and detail message of <tt>cause</tt>). - * This constructor is useful for exceptions that are little more than - * wrappers for other throwables (for example, {@link - * PrivilegedActionException}). - * - * @param cause the cause (which is saved for later retrieval by the - * {@link #getCause()} method). (A <tt>null</tt> value is - * permitted, and indicates that the cause is nonexistent or - * unknown.) - * @since 1.4 - */ - public DiscoveryException(Throwable cause) { - super(cause); - } -} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/repository/src/main/java/org/apache/atlas/discovery/DiscoveryService.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/discovery/DiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/DiscoveryService.java deleted file mode 100755 index e347c2c..0000000 --- a/repository/src/main/java/org/apache/atlas/discovery/DiscoveryService.java +++ /dev/null @@ -1,52 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.discovery; - -import java.util.List; -import java.util.Map; - -/** - * Metadata discovery service. - */ -public interface DiscoveryService { - - /** - * Full text search - */ - String searchByFullText(String query) throws DiscoveryException; - - /** - * Search using query DSL. - * - * @param dslQuery query in DSL format. - * @return JSON representing the type and results. - */ - String searchByDSL(String dslQuery) throws DiscoveryException; - - /** - * Assumes the User is familiar with the persistence structure of the Repository. - * The given query is run uninterpreted against the underlying Graph Store. - * The results are returned as a List of Rows. each row is a Map of Key,Value pairs. - * - * @param gremlinQuery query in gremlin dsl format - * @return List of Maps - * @throws org.apache.atlas.discovery.DiscoveryException - */ - List<Map<String, String>> searchByGremlin(String gremlinQuery) throws DiscoveryException; -} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/repository/src/main/java/org/apache/atlas/discovery/HiveLineageService.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/discovery/HiveLineageService.java b/repository/src/main/java/org/apache/atlas/discovery/HiveLineageService.java index 7a6ff55..00905d7 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/HiveLineageService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/HiveLineageService.java @@ -22,14 +22,14 @@ import com.thinkaurelius.titan.core.TitanGraph; import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasException; import org.apache.atlas.GraphTransaction; -import org.apache.atlas.ParamChecker; +import org.apache.atlas.typesystem.exception.EntityNotFoundException; +import org.apache.atlas.utils.ParamChecker; import org.apache.atlas.discovery.graph.DefaultGraphPersistenceStrategy; import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService; import org.apache.atlas.query.Expressions; import org.apache.atlas.query.GremlinQueryResult; import org.apache.atlas.query.HiveLineageQuery; import org.apache.atlas.query.HiveWhereUsedQuery; -import org.apache.atlas.repository.EntityNotFoundException; import org.apache.atlas.repository.MetadataRepository; import org.apache.atlas.repository.graph.GraphProvider; import org.apache.atlas.typesystem.persistence.ReferenceableInstance; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/repository/src/main/java/org/apache/atlas/discovery/LineageService.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/discovery/LineageService.java b/repository/src/main/java/org/apache/atlas/discovery/LineageService.java deleted file mode 100644 index 8dc36cd..0000000 --- a/repository/src/main/java/org/apache/atlas/discovery/LineageService.java +++ /dev/null @@ -1,67 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.discovery; - -import org.apache.atlas.AtlasException; - -/** - * Lineage service interface. - */ -public interface LineageService { - - /** - * Return the lineage outputs for the given tableName. - * - * @param tableName tableName - * @return Outputs as JSON - */ - String getOutputs(String tableName) throws AtlasException; - - /** - * Return the lineage outputs graph for the given tableName. - * - * @param tableName tableName - * @return Outputs Graph as JSON - */ - String getOutputsGraph(String tableName) throws AtlasException; - - /** - * Return the lineage inputs for the given tableName. - * - * @param tableName tableName - * @return Inputs as JSON - */ - String getInputs(String tableName) throws AtlasException; - - /** - * Return the lineage inputs graph for the given tableName. - * - * @param tableName tableName - * @return Inputs Graph as JSON - */ - String getInputsGraph(String tableName) throws AtlasException; - - /** - * Return the schema for the given tableName. - * - * @param tableName tableName - * @return Schema as JSON - */ - String getSchema(String tableName) throws AtlasException; -} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/repository/src/main/java/org/apache/atlas/discovery/graph/DefaultGraphPersistenceStrategy.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/discovery/graph/DefaultGraphPersistenceStrategy.java b/repository/src/main/java/org/apache/atlas/discovery/graph/DefaultGraphPersistenceStrategy.java index 65a46a4..90718ed 100755 --- a/repository/src/main/java/org/apache/atlas/discovery/graph/DefaultGraphPersistenceStrategy.java +++ b/repository/src/main/java/org/apache/atlas/discovery/graph/DefaultGraphPersistenceStrategy.java @@ -30,6 +30,7 @@ import org.apache.atlas.query.TypeUtils; import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.MetadataRepository; import org.apache.atlas.repository.graph.GraphBackedMetadataRepository; +import org.apache.atlas.repository.graph.GraphHelper; import org.apache.atlas.typesystem.ITypedReferenceableInstance; import org.apache.atlas.typesystem.ITypedStruct; import org.apache.atlas.typesystem.persistence.Id; @@ -43,7 +44,7 @@ import org.apache.atlas.typesystem.types.TypeSystem; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; +import javax.inject.Inject; import java.util.List; /** @@ -55,6 +56,7 @@ public class DefaultGraphPersistenceStrategy implements GraphPersistenceStrategi private final GraphBackedMetadataRepository metadataRepository; + @Inject public DefaultGraphPersistenceStrategy(MetadataRepository metadataRepository) { this.metadataRepository = (GraphBackedMetadataRepository) metadataRepository; } @@ -71,7 +73,11 @@ public class DefaultGraphPersistenceStrategy implements GraphPersistenceStrategi @Override public String edgeLabel(IDataType<?> dataType, AttributeInfo aInfo) { - return metadataRepository.getEdgeLabel(dataType, aInfo); + try { + return metadataRepository.getEdgeLabel(dataType, aInfo); + } catch (AtlasException e) { + throw new RuntimeException(e); + } } @Override @@ -90,7 +96,7 @@ public class DefaultGraphPersistenceStrategy implements GraphPersistenceStrategi @Override public List<String> traitNames(TitanVertex vertex) { - return metadataRepository.getTraitNames(vertex); + return GraphHelper.getTraitNames(vertex); } @Override @@ -100,7 +106,7 @@ public class DefaultGraphPersistenceStrategy implements GraphPersistenceStrategi @Override public Id getIdFromVertex(String dataTypeName, TitanVertex vertex) { - return metadataRepository.getIdFromVertex(dataTypeName, vertex); + return GraphHelper.getIdFromVertex(dataTypeName, vertex); } @Override http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/repository/src/main/java/org/apache/atlas/repository/EntityExistsException.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/EntityExistsException.java b/repository/src/main/java/org/apache/atlas/repository/EntityExistsException.java deleted file mode 100644 index 7ea7e41..0000000 --- a/repository/src/main/java/org/apache/atlas/repository/EntityExistsException.java +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p/> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p/> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.repository; - -import org.apache.atlas.AtlasException; -import org.apache.atlas.typesystem.IReferenceableInstance; - -public class EntityExistsException extends AtlasException { - public EntityExistsException(IReferenceableInstance typedInstance, Exception e) { - super("Model violation for type "+ typedInstance.getTypeName(), e); - } - - public EntityExistsException(IReferenceableInstance typedInstance) { - super("Model violation for type "+ typedInstance.getTypeName()); - } -} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/repository/src/main/java/org/apache/atlas/repository/EntityNotFoundException.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/EntityNotFoundException.java b/repository/src/main/java/org/apache/atlas/repository/EntityNotFoundException.java deleted file mode 100644 index db21bc7..0000000 --- a/repository/src/main/java/org/apache/atlas/repository/EntityNotFoundException.java +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.repository; - -/** - * A simple wrapper for 404. - */ -public class EntityNotFoundException extends RepositoryException { - public EntityNotFoundException() { - } - - public EntityNotFoundException(String message) { - super(message); - } - - public EntityNotFoundException(String message, Throwable cause) { - super(message, cause); - } - - public EntityNotFoundException(Throwable cause) { - super(cause); - } - - public EntityNotFoundException(String message, Throwable cause, boolean enableSuppression, - boolean writableStackTrace) { - super(message, cause, enableSuppression, writableStackTrace); - } -} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/repository/src/main/java/org/apache/atlas/repository/MetadataRepository.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/MetadataRepository.java b/repository/src/main/java/org/apache/atlas/repository/MetadataRepository.java index c11d9a0..2091e71 100755 --- a/repository/src/main/java/org/apache/atlas/repository/MetadataRepository.java +++ b/repository/src/main/java/org/apache/atlas/repository/MetadataRepository.java @@ -19,6 +19,8 @@ package org.apache.atlas.repository; import org.apache.atlas.AtlasException; +import org.apache.atlas.typesystem.exception.EntityExistsException; +import org.apache.atlas.typesystem.exception.EntityNotFoundException; import org.apache.atlas.typesystem.ITypedReferenceableInstance; import org.apache.atlas.typesystem.ITypedStruct; import org.apache.atlas.typesystem.types.AttributeInfo; @@ -70,7 +72,7 @@ public interface MetadataRepository { * @param aInfo attribute info * @return edge label for a given attribute */ - String getEdgeLabel(IDataType<?> dataType, AttributeInfo aInfo); + String getEdgeLabel(IDataType<?> dataType, AttributeInfo aInfo) throws AtlasException; /** * Creates an entity definition (instance) corresponding to a given type. @@ -89,7 +91,7 @@ public interface MetadataRepository { * @return entity (typed instance) definition * @throws RepositoryException */ - ITypedReferenceableInstance getEntityDefinition(String guid) throws RepositoryException; + ITypedReferenceableInstance getEntityDefinition(String guid) throws RepositoryException, EntityNotFoundException; /** * Gets the list of entities for a given entity type. @@ -108,20 +110,6 @@ public interface MetadataRepository { * @throws RepositoryException */ // boolean deleteEntity(String guid) throws RepositoryException; - - /** - * Updates an entity given its GUID with the attribute name and value. - * - * @param guid globally unique identifier for the entity - * @param attributeName name of the attribute - * @param attributeValue value of the attribute - * @return an entity instance with updated state - * @throws RepositoryException - */ - //ITypedReferenceableInstance updateEntity(String guid, String attributeName, - // String attributeValue) throws RepositoryException; - - // Trait management functions /** @@ -149,15 +137,19 @@ public interface MetadataRepository { * @param traitNameToBeDeleted name of the trait * @throws RepositoryException */ - void deleteTrait(String guid, String traitNameToBeDeleted) throws RepositoryException; + void deleteTrait(String guid, String traitNameToBeDeleted) throws EntityNotFoundException, RepositoryException; + + /** + * Adds/Updates the property to the entity that corresponds to the GUID + * Supports only primitive attribute/Class Id updations. + */ + void updatePartial(ITypedReferenceableInstance entity) throws RepositoryException; /** - * Adds/Updates the property to/in the entity that corresponds to the GUID - * @param guid entity id - * @param property property name - * @param value property value + * Adds the property to the entity that corresponds to the GUID + * @param entitiesToBeUpdated The entities to be updated */ - void updateEntity(String guid, String property, String value) throws RepositoryException; + String[] updateEntities(ITypedReferenceableInstance... entitiesToBeUpdated) throws RepositoryException; /** * Returns the entity for the given type and qualified name @@ -166,5 +158,5 @@ public interface MetadataRepository { * @param value * @return entity instance */ - ITypedReferenceableInstance getEntityDefinition(String entityType, String attribute, String value) throws AtlasException; + ITypedReferenceableInstance getEntityDefinition(String entityType, String attribute, Object value) throws AtlasException; } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/repository/src/main/java/org/apache/atlas/repository/graph/EntityProcessor.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/EntityProcessor.java b/repository/src/main/java/org/apache/atlas/repository/graph/EntityProcessor.java new file mode 100644 index 0000000..59472e4 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/graph/EntityProcessor.java @@ -0,0 +1,81 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.graph; + +import org.apache.atlas.AtlasException; +import org.apache.atlas.repository.RepositoryException; +import org.apache.atlas.typesystem.IReferenceableInstance; +import org.apache.atlas.typesystem.ITypedReferenceableInstance; +import org.apache.atlas.typesystem.persistence.Id; +import org.apache.atlas.typesystem.types.DataTypes; +import org.apache.atlas.typesystem.types.ObjectGraphWalker; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +public final class EntityProcessor implements ObjectGraphWalker.NodeProcessor { + + private final Map<Id, IReferenceableInstance> idToInstanceMap; + + public EntityProcessor() { + idToInstanceMap = new LinkedHashMap<>(); + } + + public Collection<IReferenceableInstance> getInstances() { + ArrayList<IReferenceableInstance> instances = new ArrayList<IReferenceableInstance>(idToInstanceMap.values()); + Collections.reverse(instances); + return instances; + } + + @Override + public void processNode(ObjectGraphWalker.Node nd) throws AtlasException { + IReferenceableInstance ref = null; + Id id = null; + + if (nd.attributeName == null) { + ref = (IReferenceableInstance) nd.instance; + id = ref.getId(); + } else if (nd.aInfo.dataType().getTypeCategory() == DataTypes.TypeCategory.CLASS) { + if (nd.value != null && (nd.value instanceof Id)) { + id = (Id) nd.value; + } + } + + if (id != null) { + if (id.isUnassigned()) { + if (ref != null) { + if (idToInstanceMap.containsKey(id)) { // Oops + throw new RepositoryException( + String.format("Unexpected internal error: Id %s processed again", id)); + } + + idToInstanceMap.put(id, ref); + } + } + } + } + + public void addInstanceIfNotExists(ITypedReferenceableInstance ref) { + if(!idToInstanceMap.containsKey(ref.getId())) { + idToInstanceMap.put(ref.getId(), ref); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapper.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapper.java b/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapper.java new file mode 100644 index 0000000..36d8034 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapper.java @@ -0,0 +1,146 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.graph; + +import com.tinkerpop.blueprints.Vertex; +import org.apache.atlas.AtlasException; +import org.apache.atlas.repository.Constants; +import org.apache.atlas.typesystem.ITypedInstance; +import org.apache.atlas.typesystem.ITypedReferenceableInstance; +import org.apache.atlas.typesystem.types.AttributeInfo; +import org.apache.atlas.typesystem.types.DataTypes; +import org.apache.atlas.typesystem.types.EnumValue; +import org.apache.atlas.typesystem.types.IDataType; +import org.apache.commons.lang.StringUtils; + +import java.util.List; +import java.util.Map; + +public class FullTextMapper { + + private final GraphToTypedInstanceMapper graphToTypedInstanceMapper; + + private static final GraphHelper graphHelper = GraphHelper.getInstance(); + + private static final String FULL_TEXT_DELIMITER = " "; + + FullTextMapper(GraphToTypedInstanceMapper graphToTypedInstanceMapper) { + this.graphToTypedInstanceMapper = graphToTypedInstanceMapper; + } + + public String mapRecursive(Vertex instanceVertex, boolean followReferences) throws AtlasException { + String guid = instanceVertex.getProperty(Constants.GUID_PROPERTY_KEY); + ITypedReferenceableInstance typedReference = + graphToTypedInstanceMapper.mapGraphToTypedInstance(guid, instanceVertex); + String fullText = forInstance(typedReference, followReferences); + StringBuilder fullTextBuilder = + new StringBuilder(typedReference.getTypeName()).append(FULL_TEXT_DELIMITER).append(fullText); + + List<String> traits = typedReference.getTraits(); + for (String traitName : traits) { + String traitText = forInstance((ITypedInstance) typedReference.getTrait(traitName), false); + fullTextBuilder.append(FULL_TEXT_DELIMITER).append(traitName).append(FULL_TEXT_DELIMITER) + .append(traitText); + } + return fullTextBuilder.toString(); + } + + private String forAttribute(IDataType type, Object value, boolean followReferences) + throws AtlasException { + if (value == null) { + return null; + } + switch (type.getTypeCategory()) { + case PRIMITIVE: + return String.valueOf(value); + case ENUM: + + return ((EnumValue) value).value; + + case ARRAY: + StringBuilder fullText = new StringBuilder(); + IDataType elemType = ((DataTypes.ArrayType) type).getElemType(); + List list = (List) value; + + for (Object element : list) { + String elemFullText = forAttribute(elemType, element, false); + if (StringUtils.isNotEmpty(elemFullText)) { + fullText = fullText.append(FULL_TEXT_DELIMITER).append(elemFullText); + } + } + return fullText.toString(); + + case MAP: + fullText = new StringBuilder(); + IDataType keyType = ((DataTypes.MapType) type).getKeyType(); + IDataType valueType = ((DataTypes.MapType) type).getValueType(); + Map map = (Map) value; + + for (Object entryObj : map.entrySet()) { + Map.Entry entry = (Map.Entry) entryObj; + String keyFullText = forAttribute(keyType, entry.getKey(), false); + if (StringUtils.isNotEmpty(keyFullText)) { + fullText = fullText.append(FULL_TEXT_DELIMITER).append(keyFullText); + } + String valueFullText = forAttribute(valueType, entry.getValue(), false); + if (StringUtils.isNotEmpty(valueFullText)) { + fullText = fullText.append(FULL_TEXT_DELIMITER).append(valueFullText); + } + } + return fullText.toString(); + + case CLASS: + if (followReferences) { + String refGuid = ((ITypedReferenceableInstance) value).getId()._getId(); + Vertex refVertex = graphHelper.getVertexForGUID(refGuid); + return mapRecursive(refVertex, false); + } + break; + + case STRUCT: + if (followReferences) { + return forInstance((ITypedInstance) value, true); + } + break; + + default: + throw new IllegalStateException("Unhandled type category " + type.getTypeCategory()); + + } + return null; + } + + private String forInstance(ITypedInstance typedInstance, boolean followReferences) + throws AtlasException { + StringBuilder fullText = new StringBuilder(); + for (AttributeInfo attributeInfo : typedInstance.fieldMapping().fields.values()) { + Object attrValue = typedInstance.get(attributeInfo.name); + if (attrValue == null) { + continue; + } + + String attrFullText = forAttribute(attributeInfo.dataType(), attrValue, followReferences); + if (StringUtils.isNotEmpty(attrFullText)) { + fullText = + fullText.append(FULL_TEXT_DELIMITER).append(attributeInfo.name).append(FULL_TEXT_DELIMITER) + .append(attrFullText); + } + } + return fullText.toString(); + } +}
