Repository: atlas Updated Branches: refs/heads/branch-0.8 8e1c60658 -> d359f3043
ATLAS-2524: fix Hive hook for incorrect handling of 'alter view as' operation (cherry picked from commit f42c1d9ff3b8bba53431520845ae371361b7e735) Project: http://git-wip-us.apache.org/repos/asf/atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/d359f304 Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/d359f304 Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/d359f304 Branch: refs/heads/branch-0.8 Commit: d359f3043a4905af81f752c60f9dc838c87801eb Parents: 8e1c606 Author: Madhan Neethiraj <[email protected]> Authored: Thu Mar 29 17:11:19 2018 -0700 Committer: Madhan Neethiraj <[email protected]> Committed: Fri Mar 30 11:36:09 2018 -0700 ---------------------------------------------------------------------- .../atlas/hive/hook/AtlasHiveHookContext.java | 52 ++++++++++++++++++++ .../hive/hook/events/AlterTableRename.java | 11 ----- .../atlas/hive/hook/events/BaseHiveEvent.java | 34 ++++++------- 3 files changed, 68 insertions(+), 29 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/atlas/blob/d359f304/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java index 9105ebe..c31d94c 100644 --- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java +++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java @@ -19,9 +19,16 @@ package org.apache.atlas.hive.hook; import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.RandomStringUtils; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.ql.hooks.Entity; import org.apache.hadoop.hive.ql.hooks.HookContext; +import org.apache.hadoop.hive.ql.hooks.WriteEntity; import org.apache.hadoop.hive.ql.metadata.Hive; +import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.plan.HiveOperation; +import org.apache.hadoop.hive.ql.session.SessionState; import java.util.Collection; import java.util.HashMap; @@ -29,6 +36,11 @@ import java.util.Map; public class AtlasHiveHookContext { + public static final char QNAME_SEP_CLUSTER_NAME = '@'; + public static final char QNAME_SEP_ENTITY_NAME = '.'; + public static final char QNAME_SEP_PROCESS = ':'; + public static final String TEMP_TABLE_PREFIX = "_temp-"; + private final HiveHook hook; private final HiveOperation hiveOperation; private final HookContext hiveContext; @@ -40,6 +52,8 @@ public class AtlasHiveHookContext { this.hiveOperation = hiveOperation; this.hiveContext = hiveContext; this.hive = Hive.get(hiveContext.getConf()); + + init(); } public HookContext getHiveContext() { @@ -69,6 +83,24 @@ public class AtlasHiveHookContext { return hook.getClusterName(); } + public String getQualifiedName(Database db) { + return (db.getName() + QNAME_SEP_CLUSTER_NAME).toLowerCase() + getClusterName(); + } + + public String getQualifiedName(Table table) { + String tableName = table.getTableName(); + + if (table.isTemporary()) { + if (SessionState.get() != null && SessionState.get().getSessionId() != null) { + tableName = tableName + TEMP_TABLE_PREFIX + SessionState.get().getSessionId(); + } else { + tableName = tableName + TEMP_TABLE_PREFIX + RandomStringUtils.random(10); + } + } + + return (table.getDbName() + QNAME_SEP_ENTITY_NAME + tableName + QNAME_SEP_CLUSTER_NAME).toLowerCase() + getClusterName(); + } + public boolean isKnownDatabase(String dbQualifiedName) { return hook.isKnownDatabase(dbQualifiedName); } @@ -88,4 +120,24 @@ public class AtlasHiveHookContext { public void removeFromKnownTable(String tblQualifiedName) { hook.removeFromKnownTable(tblQualifiedName); } + + private void init() { + String operationName = hiveContext.getOperationName(); + + if (operationName != null && operationName.startsWith("CREATE") || operationName.startsWith("ALTER")) { + if (CollectionUtils.isNotEmpty(hiveContext.getOutputs())) { + for (WriteEntity output : hiveContext.getOutputs()) { + switch (output.getType()) { + case DATABASE: + hook.removeFromKnownDatabase(getQualifiedName(output.getDatabase())); + break; + + case TABLE: + hook.removeFromKnownTable(getQualifiedName(output.getTable())); + break; + } + } + } + } + } } http://git-wip-us.apache.org/repos/asf/atlas/blob/d359f304/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/AlterTableRename.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/AlterTableRename.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/AlterTableRename.java index a64d768..d5b9fc3 100644 --- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/AlterTableRename.java +++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/AlterTableRename.java @@ -137,17 +137,6 @@ public class AlterTableRename extends BaseHiveEvent { } } - private String getColumnQualifiedName(String tblQualifiedName, String columnName) { - int sepPos = tblQualifiedName.lastIndexOf(QNAME_SEP_CLUSTER_NAME); - - if (sepPos == -1) { - return tblQualifiedName + QNAME_SEP_ENTITY_NAME + columnName.toLowerCase(); - } else { - return tblQualifiedName.substring(0, sepPos) + QNAME_SEP_ENTITY_NAME + columnName.toLowerCase() + tblQualifiedName.substring(sepPos); - } - - } - private void removeAttribute(AtlasEntityWithExtInfo entity, String attributeName) { Object attributeValue = entity.getEntity().getAttribute(attributeName); http://git-wip-us.apache.org/repos/asf/atlas/blob/d359f304/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java index 2a29bcd..196970b 100644 --- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java +++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java @@ -27,7 +27,6 @@ import org.apache.atlas.model.instance.AtlasObjectId; import org.apache.atlas.model.instance.AtlasStruct; import org.apache.atlas.notification.hook.HookNotification.HookNotificationMessage; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang.RandomStringUtils; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.metastore.api.Database; @@ -43,7 +42,6 @@ import org.apache.hadoop.hive.ql.hooks.WriteEntity; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.plan.HiveOperation; -import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -60,6 +58,9 @@ import java.util.List; import java.util.Map; import java.util.Set; +import static org.apache.atlas.hive.hook.AtlasHiveHookContext.QNAME_SEP_CLUSTER_NAME; +import static org.apache.atlas.hive.hook.AtlasHiveHookContext.QNAME_SEP_ENTITY_NAME; +import static org.apache.atlas.hive.hook.AtlasHiveHookContext.QNAME_SEP_PROCESS; public abstract class BaseHiveEvent { private static final Logger LOG = LoggerFactory.getLogger(BaseHiveEvent.class); @@ -124,10 +125,6 @@ public abstract class BaseHiveEvent { public static final String ATTRIBUTE_ALIASES = "aliases"; - public static final char QNAME_SEP_CLUSTER_NAME = '@'; - public static final char QNAME_SEP_ENTITY_NAME = '.'; - public static final char QNAME_SEP_PROCESS = ':'; - public static final String TEMP_TABLE_PREFIX = "_temp-"; public static final long MILLIS_CONVERT_FACTOR = 1000; public static final Map<Integer, String> OWNER_TYPE_TO_ENUM_VALUE = new HashMap<>(); @@ -580,21 +577,11 @@ public abstract class BaseHiveEvent { } protected String getQualifiedName(Database db) { - return (db.getName() + QNAME_SEP_CLUSTER_NAME).toLowerCase() + getClusterName(); + return context.getQualifiedName(db); } protected String getQualifiedName(Table table) { - String tableName = table.getTableName(); - - if (table.isTemporary()) { - if (SessionState.get() != null && SessionState.get().getSessionId() != null) { - tableName = tableName + TEMP_TABLE_PREFIX + SessionState.get().getSessionId(); - } else { - tableName = tableName + TEMP_TABLE_PREFIX + RandomStringUtils.random(10); - } - } - - return (table.getDbName() + QNAME_SEP_ENTITY_NAME + tableName + QNAME_SEP_CLUSTER_NAME).toLowerCase() + getClusterName(); + return context.getQualifiedName(table); } protected String getQualifiedName(Table table, StorageDescriptor sd) { @@ -647,6 +634,17 @@ public abstract class BaseHiveEvent { return path.toLowerCase(); } + protected String getColumnQualifiedName(String tblQualifiedName, String columnName) { + int sepPos = tblQualifiedName.lastIndexOf(QNAME_SEP_CLUSTER_NAME); + + if (sepPos == -1) { + return tblQualifiedName + QNAME_SEP_ENTITY_NAME + columnName.toLowerCase(); + } else { + return tblQualifiedName.substring(0, sepPos) + QNAME_SEP_ENTITY_NAME + columnName.toLowerCase() + tblQualifiedName.substring(sepPos); + } + + } + protected String getQualifiedName(List<AtlasEntity> inputs, List<AtlasEntity> outputs) throws Exception { HiveOperation operation = context.getHiveOperation();
