This is an automated email from the ASF dual-hosted git repository. madhan pushed a commit to branch branch-1.0 in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/branch-1.0 by this push: new 594c998 ATLAS-3033: Skip hive temp table while getting Table object details from hiveContext. 594c998 is described below commit 594c998037d55eb04b38c4c727d1f3419dfd4ec8 Author: nixonrodrigues <ni...@apache.org> AuthorDate: Thu May 2 20:37:08 2019 -0700 ATLAS-3033: Skip hive temp table while getting Table object details from hiveContext. Signed-off-by: Madhan Neethiraj <mad...@apache.org> (cherry picked from commit b5472ce313dcf24b0bffbe5201f0c95eec025848) (cherry picked from commit 0323ce3825c63b06d7b0032edd10b73b84078a4a) (cherry picked from commit 3e65639e2bf9155090a87bf05f16ef040ded14d1) --- .../atlas/hive/hook/AtlasHiveHookContext.java | 13 +++++++++++ .../java/org/apache/atlas/hive/hook/HiveHook.java | 26 ++++++++++++++++++++++ .../atlas/hive/hook/events/BaseHiveEvent.java | 20 +++++++++++++---- 3 files changed, 55 insertions(+), 4 deletions(-) diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java index 44c6437..58a73c3 100644 --- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java +++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java @@ -33,6 +33,7 @@ import org.apache.hadoop.hive.ql.session.SessionState; import java.util.Collection; import java.util.HashMap; +import java.util.List; import java.util.Map; @@ -102,6 +103,18 @@ public class AtlasHiveHookContext { return hook.getPreprocessActionForHiveTable(qualifiedName); } + public List getIgnoreDummyDatabaseName() { + return hook.getIgnoreDummyDatabaseName(); + } + + public List getIgnoreDummyTableName() { + return hook.getIgnoreDummyTableName(); + } + + public String getIgnoreValuesTmpTableNamePrefix() { + return hook.getIgnoreValuesTmpTableNamePrefix(); + } + public String getQualifiedName(Database db) { return (db.getName() + QNAME_SEP_CLUSTER_NAME).toLowerCase() + getClusterName(); } diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java index ee02285..3456970 100644 --- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java +++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java @@ -26,6 +26,7 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hive.ql.hooks.ExecuteWithHookContext; import org.apache.hadoop.hive.ql.hooks.HookContext; +import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer; import org.apache.hadoop.hive.ql.plan.HiveOperation; import org.apache.hadoop.hive.shims.Utils; import org.apache.hadoop.security.UserGroupInformation; @@ -81,6 +82,9 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext { private static final List<Pattern> hiveTablesToIgnore = new ArrayList<>(); private static final List<Pattern> hiveTablesToPrune = new ArrayList<>(); private static final Map<String, PreprocessAction> hiveTablesCache; + private static final List ignoreDummyDatabaseName; + private static final List ignoreDummyTableName; + private static final String ignoreValuesTmpTableNamePrefix; private static HiveHookObjectNamesCache knownObjects = null; @@ -134,6 +138,16 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext { } knownObjects = nameCacheEnabled ? new HiveHookObjectNamesCache(nameCacheDatabaseMaxCount, nameCacheTableMaxCount, nameCacheRebuildIntervalSeconds) : null; + + List<String> defaultDummyDatabase = new ArrayList<>(); + List<String> defaultDummyTable = new ArrayList<>(); + + defaultDummyDatabase.add(SemanticAnalyzer.DUMMY_DATABASE); + defaultDummyTable.add(SemanticAnalyzer.DUMMY_TABLE); + + ignoreDummyDatabaseName = atlasProperties.getList("atlas.hook.hive.ignore.dummy.database.name", defaultDummyDatabase); + ignoreDummyTableName = atlasProperties.getList("atlas.hook.hive.ignore.dummy.table.name", defaultDummyTable); + ignoreValuesTmpTableNamePrefix = atlasProperties.getString("atlas.hook.hive.ignore.values.tmp.table.name.prefix", "Values__Tmp__Table__"); } @@ -252,6 +266,18 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext { return skipHiveColumnLineageHive20633InputsThreshold; } + public List getIgnoreDummyDatabaseName() { + return ignoreDummyDatabaseName; + } + + public List getIgnoreDummyTableName() { + return ignoreDummyTableName; + } + + public String getIgnoreValuesTmpTableNamePrefix() { + return ignoreValuesTmpTableNamePrefix; + } + public PreprocessAction getPreprocessActionForHiveTable(String qualifiedName) { PreprocessAction ret = PreprocessAction.NONE; diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java index 31346d0..b6754f7 100644 --- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java +++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java @@ -238,17 +238,29 @@ public abstract class BaseHiveEvent { switch (entity.getType()) { case DATABASE: { - Database db = getHive().getDatabase(entity.getDatabase().getName()); + if (!context.getIgnoreDummyDatabaseName().contains(entity.getDatabase().getName())) { + Database db = getHive().getDatabase(entity.getDatabase().getName()); - ret = toDbEntity(db); + ret = toDbEntity(db); + } } break; case TABLE: case PARTITION: { - Table table = getHive().getTable(entity.getTable().getDbName(), entity.getTable().getTableName()); + String dbName = entity.getTable().getDbName(); + String tableName = entity.getTable().getTableName(); + boolean skipTable = StringUtils.isNotEmpty(context.getIgnoreValuesTmpTableNamePrefix()) && tableName.toLowerCase().startsWith(context.getIgnoreValuesTmpTableNamePrefix()); + + if (!skipTable) { + skipTable = context.getIgnoreDummyTableName().contains(tableName) && context.getIgnoreDummyDatabaseName().contains(dbName); + } - ret = toTableEntity(table, entityExtInfo); + if (!skipTable) { + Table table = getHive().getTable(dbName, tableName); + + ret = toTableEntity(table, entityExtInfo); + } } break;