atlas git commit: ATLAS-2649: updated Hive Hook to create lineage between HBase table and Hive table
Repository: atlas Updated Branches: refs/heads/master f15995cc8 -> f62ed0926 ATLAS-2649: updated Hive Hook to create lineage between HBase table and Hive table Signed-off-by: Madhan Neethiraj Project: http://git-wip-us.apache.org/repos/asf/atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/f62ed092 Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/f62ed092 Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/f62ed092 Branch: refs/heads/master Commit: f62ed0926dc5a116729d4cb4d2f6d97fac3e8232 Parents: f15995c Author: rmani Authored: Thu May 10 15:10:20 2018 -0700 Committer: Madhan Neethiraj Committed: Thu May 10 22:54:03 2018 -0700 -- addons/hive-bridge/pom.xml | 5 + .../atlas/hive/hook/events/BaseHiveEvent.java | 97 +++- .../atlas/hive/hook/events/CreateTable.java | 27 +- 3 files changed, 122 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/atlas/blob/f62ed092/addons/hive-bridge/pom.xml -- diff --git a/addons/hive-bridge/pom.xml b/addons/hive-bridge/pom.xml index 0d4ab2c..5814854 100755 --- a/addons/hive-bridge/pom.xml +++ b/addons/hive-bridge/pom.xml @@ -295,6 +295,11 @@ ${hbase.version} +org.apache.hbase + hbase-server +${hbase.version} + + com.sun.jersey jersey-json ${jersey.version} http://git-wip-us.apache.org/repos/asf/atlas/blob/f62ed092/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java -- diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java index ca13812..fad53c0 100644 --- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java +++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java @@ -28,6 +28,7 @@ import org.apache.atlas.model.instance.AtlasStruct; import org.apache.atlas.model.notification.HookNotification; import org.apache.atlas.utils.HdfsNameServiceResolver; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.metastore.api.Database; @@ -75,6 +76,8 @@ public abstract class BaseHiveEvent { public static final String HIVE_TYPE_SERDE = "hive_serde"; public static final String HIVE_TYPE_ORDER = "hive_order"; public static final String HDFS_TYPE_PATH = "hdfs_path"; +public static final String HBASE_TYPE_TABLE = "hbase_table"; +public static final String HBASE_TYPE_NAMESPACE = "hbase_namespace"; public static final String ATTRIBUTE_QUALIFIED_NAME= "qualifiedName"; public static final String ATTRIBUTE_NAME = "name"; @@ -124,9 +127,15 @@ public abstract class BaseHiveEvent { public static final String ATTRIBUTE_DEPENDENCY_TYPE = "depenendencyType"; public static final String ATTRIBUTE_EXPRESSION= "expression"; public static final String ATTRIBUTE_ALIASES = "aliases"; +public static final String ATTRIBUTE_URI = "uri"; +public static final String ATTRIBUTE_STORAGE_HANDLER = "storage_handler"; +public static final String ATTRIBUTE_NAMESPACE = "namespace"; - -public static final long MILLIS_CONVERT_FACTOR = 1000; +public static final String HBASE_STORAGE_HANDLER_CLASS = "org.apache.hadoop.hive.hbase.HBaseStorageHandler"; +public static final String HBASE_DEFAULT_NAMESPACE = "default"; +public static final String HBASE_NAMESPACE_TABLE_DELIMITER = ":"; +public static final String HBASE_PARAM_TABLE_NAME = "hbase.table.name"; +public static final long MILLIS_CONVERT_FACTOR = 1000; public static final Map OWNER_TYPE_TO_ENUM_VALUE = new HashMap<>(); @@ -156,6 +165,10 @@ public abstract class BaseHiveEvent { return table.getTTable() != null ? (table.getTTable().getCreateTime() * MILLIS_CONVERT_FACTOR) : System.currentTimeMilli
atlas git commit: ATLAS-2670: Validate entity guid when processing term dissociation
Repository: atlas Updated Branches: refs/heads/master f85ff28e0 -> f15995cc8 ATLAS-2670: Validate entity guid when processing term dissociation Change-Id: I5e2db1b9968a37482b9ff97ba8602aa262a2db80 Project: http://git-wip-us.apache.org/repos/asf/atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/f15995cc Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/f15995cc Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/f15995cc Branch: refs/heads/master Commit: f15995cc8471e498e2a86c3e4790889ac84e Parents: f85ff28 Author: apoorvnaik Authored: Thu May 10 10:02:11 2018 -0700 Committer: apoorvnaik Committed: Thu May 10 12:10:38 2018 -0700 -- .../java/org/apache/atlas/AtlasErrorCode.java| 3 ++- .../apache/atlas/glossary/GlossaryTermUtils.java | 19 ++- 2 files changed, 20 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/atlas/blob/f15995cc/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java -- diff --git a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java index 5e3d818..7cf3fd1 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java +++ b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java @@ -142,7 +142,8 @@ public enum AtlasErrorCode { RELATIONSHIP_END_IS_NULL(400, "ATLAS-400-00-07D", "Relationship end is invalid. Expected {0} but is NULL"), INVALID_TERM_RELATION_TO_SELF(400, "ATLAS-400-00-07E", "Invalid Term relationship: Term can't have a relationship with self"), INVALID_CHILD_CATEGORY_DIFFERENT_GLOSSARY(400, "ATLAS-400-00-07F", "Invalid child category relationship: Child category (guid = {0}) belongs to different glossary"), -ATTRIBUTE_TYPE_INVALID(400, "ATLAS-400-00-080", "{0}.{1}: invalid attribute type. Attribute cannot be of type classification"), +INVALID_TERM_DISSOCIATION(400, "ATLAS-400-00-080", "Given term (guid={0}) is not associated to entity(guid={1})"), +ATTRIBUTE_TYPE_INVALID(400, "ATLAS-400-00-081", "{0}.{1}: invalid attribute type. Attribute cannot be of type classification"), UNAUTHORIZED_ACCESS(403, "ATLAS-403-00-001", "{0} is not authorized to perform {1}"), http://git-wip-us.apache.org/repos/asf/atlas/blob/f15995cc/repository/src/main/java/org/apache/atlas/glossary/GlossaryTermUtils.java -- diff --git a/repository/src/main/java/org/apache/atlas/glossary/GlossaryTermUtils.java b/repository/src/main/java/org/apache/atlas/glossary/GlossaryTermUtils.java index 5400b77..6977369 100644 --- a/repository/src/main/java/org/apache/atlas/glossary/GlossaryTermUtils.java +++ b/repository/src/main/java/org/apache/atlas/glossary/GlossaryTermUtils.java @@ -100,6 +100,14 @@ public class GlossaryTermUtils extends GlossaryUtils { } Objects.requireNonNull(glossaryTerm); +Set assignedEntities = glossaryTerm.getAssignedEntities(); +Map assignedEntityMap = new HashMap<>(); +if (CollectionUtils.isNotEmpty(assignedEntities)) { +for (AtlasRelatedObjectId relatedObjectId : assignedEntities) { +assignedEntityMap.put(relatedObjectId.getGuid(), relatedObjectId); +} +} + if (CollectionUtils.isNotEmpty(relatedObjectIds)) { for (AtlasRelatedObjectId relatedObjectId : relatedObjectIds) { if (DEBUG_ENABLED) { @@ -108,7 +116,12 @@ public class GlossaryTermUtils extends GlossaryUtils { if (Objects.isNull(relatedObjectId.getRelationshipGuid())) { throw new AtlasBaseException(AtlasErrorCode.TERM_DISSOCIATION_MISSING_RELATION_GUID); } - relationshipStore.deleteById(relatedObjectId.getRelationshipGuid()); +AtlasRelatedObjectId existingTermRelation = assignedEntityMap.get(relatedObjectId.getGuid()); +if (CollectionUtils.isNotEmpty(assignedEntities) && isRelationshipGuidSame(existingTermRelation, relatedObjectId)) { + relationshipStore.deleteById(relatedObjectId.getRelationshipGuid()); +} else { +throw new AtlasBaseException(AtlasErrorCode.INVALID_TERM_DISSOCIATION, glossaryTerm.getGuid(), relatedObjectId.getGuid()); +} } } @@ -117,6 +130,10 @@ public class GlossaryTermUtils extends GlossaryUtils { } } +private boolean isRelationshipGuidSame(final AtlasRelatedObjectId existing, final AtlasRelatedObjectId relatedObjectId) { +return StringUtils.equals(relatedObjectId.getRelationshipGuid(), existing.getRelationshipGuid()); +} + private void processTermAnchor(AtlasGlossaryTerm upda
atlas git commit: ATLAS-2669 : Glossary UI : Attribute UI improved while associating term to the related term.
Repository: atlas Updated Branches: refs/heads/master 479b9ab1c -> f85ff28e0 ATLAS-2669 : Glossary UI : Attribute UI improved while associating term to the related term. Signed-off-by: Madhan Neethiraj Project: http://git-wip-us.apache.org/repos/asf/atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/f85ff28e Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/f85ff28e Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/f85ff28e Branch: refs/heads/master Commit: f85ff28e05f92e75be3d663dd1a8c1c75f6130dd Parents: 479b9ab Author: kevalbhatt Authored: Thu May 10 20:13:03 2018 +0530 Committer: Madhan Neethiraj Committed: Thu May 10 09:16:28 2018 -0700 -- dashboardv2/gruntfile.js| 5 +- dashboardv2/package.json| 3 +- dashboardv2/public/css/scss/style.scss | 1 + dashboardv2/public/css/scss/wizard.scss | 341 +++ dashboardv2/public/js/main.js | 6 +- .../glossary/AssignTermLayoutView_tmpl.html | 33 +- .../TermRelationAttributeLayoutView_tmpl.html | 13 - .../js/views/glossary/AssignTermLayoutView.js | 44 ++- .../views/glossary/GlossaryDetailLayoutView.js | 31 +- .../js/views/glossary/GlossaryLayoutView.js | 10 +- .../js/views/search/SearchResultLayoutView.js | 3 + 11 files changed, 453 insertions(+), 37 deletions(-) -- http://git-wip-us.apache.org/repos/asf/atlas/blob/f85ff28e/dashboardv2/gruntfile.js -- diff --git a/dashboardv2/gruntfile.js b/dashboardv2/gruntfile.js index 2bcecfe..1d54a81 100644 --- a/dashboardv2/gruntfile.js +++ b/dashboardv2/gruntfile.js @@ -106,7 +106,8 @@ module.exports = function(grunt) { 'nv.d3.min.js': { 'nvd3/build': 'nvd3' }, 'jquery.sparkline.min.js': { 'jquery-sparkline': 'sparkline' }, 'table-dragger.js': { 'table-dragger/dist': 'table-dragger' }, -'jstree.min.js': { 'jstree/dist': 'jstree' } +'jstree.min.js': { 'jstree/dist': 'jstree' }, +'jquery.steps.min.js': { 'jquery-steps/build': 'jquery-steps' } } }, @@ -143,7 +144,7 @@ module.exports = function(grunt) { }], 'query-builder.default.min.css': { 'jQuery-QueryBuilder/dist/css': 'jQueryQueryBuilder/css' }, 'daterangepicker.css': { 'bootstrap-daterangepicker': 'bootstrap-daterangepicker/css' }, -'nv.d3.min.css': { 'nvd3/build': 'nvd3/css' }, +'nv.d3.min.css': { 'nvd3/build': 'nvd3/css' } } }, http://git-wip-us.apache.org/repos/asf/atlas/blob/f85ff28e/dashboardv2/package.json -- diff --git a/dashboardv2/package.json b/dashboardv2/package.json index e543468..7fbd1bf 100644 --- a/dashboardv2/package.json +++ b/dashboardv2/package.json @@ -37,7 +37,8 @@ "jquery-asBreadcrumbs": "0.2.2", "jquery-placeholder": "2.3.1", "jquery-sparkline": "2.4.0", -"jstree": "^3.3.5", +"jquery-steps": "1.1.0", +"jstree": "3.3.5", "moment": "2.21.0", "moment-timezone": "0.5.14", "npm": "^5.8.0", http://git-wip-us.apache.org/repos/asf/atlas/blob/f85ff28e/dashboardv2/public/css/scss/style.scss -- diff --git a/dashboardv2/public/css/scss/style.scss b/dashboardv2/public/css/scss/style.scss index 2b226cd..cd1781b 100644 --- a/dashboardv2/public/css/scss/style.scss +++ b/dashboardv2/public/css/scss/style.scss @@ -32,4 +32,5 @@ @import "search.scss"; @import "profile-table.scss"; @import "glossary.scss"; +@import "wizard.scss"; @import "override.scss"; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/atlas/blob/f85ff28e/dashboardv2/public/css/scss/wizard.scss -- diff --git a/dashboardv2/public/css/scss/wizard.scss b/dashboardv2/public/css/scss/wizard.scss new file mode 100644 index 000..6b3ceb4 --- /dev/null +++ b/dashboardv2/public/css/scss/wizard.scss @@ -0,0 +1,341 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + *
atlas git commit: ATLAS-2667: Enhance GraphTransactionInterceptor to deal with nested/inner commits
Repository: atlas Updated Branches: refs/heads/master 6e7aa6ed3 -> 479b9ab1c ATLAS-2667: Enhance GraphTransactionInterceptor to deal with nested/inner commits Change-Id: I9ea29deb9aea226f077f4d008d459fdb3ac6663f Project: http://git-wip-us.apache.org/repos/asf/atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/479b9ab1 Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/479b9ab1 Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/479b9ab1 Branch: refs/heads/master Commit: 479b9ab1c0d461e89d14d2d3426af64f3cb9acb9 Parents: 6e7aa6e Author: apoorvnaik Authored: Tue May 8 15:28:47 2018 -0700 Committer: apoorvnaik Committed: Thu May 10 07:58:06 2018 -0700 -- .../atlas/GraphTransactionInterceptor.java | 102 +++ 1 file changed, 83 insertions(+), 19 deletions(-) -- http://git-wip-us.apache.org/repos/asf/atlas/blob/479b9ab1/repository/src/main/java/org/apache/atlas/GraphTransactionInterceptor.java -- diff --git a/repository/src/main/java/org/apache/atlas/GraphTransactionInterceptor.java b/repository/src/main/java/org/apache/atlas/GraphTransactionInterceptor.java index b3e690f..4c43677 100644 --- a/repository/src/main/java/org/apache/atlas/GraphTransactionInterceptor.java +++ b/repository/src/main/java/org/apache/atlas/GraphTransactionInterceptor.java @@ -21,14 +21,15 @@ import com.google.common.annotations.VisibleForTesting; import org.aopalliance.intercept.MethodInterceptor; import org.aopalliance.intercept.MethodInvocation; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.exception.NotFoundException; +import org.apache.atlas.repository.graphdb.AtlasGraph; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; import javax.inject.Inject; import javax.ws.rs.core.Response; +import java.lang.reflect.Method; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -41,8 +42,10 @@ public class GraphTransactionInterceptor implements MethodInterceptor { private static final Logger LOG = LoggerFactory.getLogger(GraphTransactionInterceptor.class); @VisibleForTesting -private static final ObjectUpdateSynchronizer OBJECT_UPDATE_SYNCHRONIZER = new ObjectUpdateSynchronizer(); -private static final ThreadLocal> postTransactionHooks = new ThreadLocal<>(); +private static final ObjectUpdateSynchronizer OBJECT_UPDATE_SYNCHRONIZER = new ObjectUpdateSynchronizer(); +private static final ThreadLocal> postTransactionHooks = new ThreadLocal<>(); +private static final ThreadLocal isTxnOpen = ThreadLocal.withInitial(() -> Boolean.FALSE); +private static final ThreadLocal innerFailure = ThreadLocal.withInitial(() -> Boolean.FALSE); private final AtlasGraph graph; @@ -53,39 +56,72 @@ public class GraphTransactionInterceptor implements MethodInterceptor { @Override public Object invoke(MethodInvocation invocation) throws Throwable { +Methodmethod= invocation.getMethod(); +StringinvokingClass = method.getDeclaringClass().getSimpleName(); +StringinvokedMethodName = method.getName(); + +boolean isInnerTxn = isTxnOpen.get(); +// Outermost txn marks any subsequent transaction as inner +isTxnOpen.set(Boolean.TRUE); + +if (LOG.isDebugEnabled() && isInnerTxn) { +LOG.debug("Txn entry-point {}.{} is inner txn. Commit/Rollback will be ignored", invokingClass, invokedMethodName); +} + boolean isSuccess = false; try { try { Object response = invocation.proceed(); -graph.commit(); -isSuccess = true; -if (LOG.isDebugEnabled()) { -LOG.debug("graph commit"); +if (isInnerTxn) { +if (LOG.isDebugEnabled()) { +LOG.debug("Ignoring commit for nested/inner transaction {}.{}", invokingClass, invokedMethodName); +} +} else { +doCommitOrRollback(invokingClass, invokedMethodName); } +isSuccess = !innerFailure.get(); + return response; } catch (Throwable t) { -if (logException(t)) { -LOG.error("graph rollback due to exception ", t); +if (isInnerTxn) { +if (LOG.isDebugEnabled()) { +LOG.debug("Ignoring rollback for nested/inner transaction {}.{}", invokingClass, invokedMethodN