This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 3e65639 ATLAS-3033: Skip hive temp table while getting Table object
details from hiveContext.
3e65639 is described below
commit 3e65639e2bf9155090a87bf05f16ef040ded14d1
Author: nixonrodrigues <ni...@apache.org>
AuthorDate: Thu May 2 20:37:08 2019 -0700
ATLAS-3033: Skip hive temp table while getting Table object details from
hiveContext.
Signed-off-by: Madhan Neethiraj <mad...@apache.org>
(cherry picked from commit b5472ce313dcf24b0bffbe5201f0c95eec025848)
(cherry picked from commit 0323ce3825c63b06d7b0032edd10b73b84078a4a)
---
.../atlas/hive/hook/AtlasHiveHookContext.java | 12 ++++++++++
.../java/org/apache/atlas/hive/hook/HiveHook.java | 26 ++++++++++++++++++++++
.../atlas/hive/hook/events/BaseHiveEvent.java | 20 +++++++++++++----
3 files changed, 54 insertions(+), 4 deletions(-)
diff --git
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java
index 52da710..5c56a39 100644
---
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java
+++
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java
@@ -149,6 +149,18 @@ public class AtlasHiveHookContext {
return hook.getPreprocessActionForHiveTable(qualifiedName);
}
+ public List getIgnoreDummyDatabaseName() {
+ return hook.getIgnoreDummyDatabaseName();
+ }
+
+ public List getIgnoreDummyTableName() {
+ return hook.getIgnoreDummyTableName();
+ }
+
+ public String getIgnoreValuesTmpTableNamePrefix() {
+ return hook.getIgnoreValuesTmpTableNamePrefix();
+ }
+
public String getQualifiedName(Database db) {
return (db.getName() + QNAME_SEP_CLUSTER_NAME).toLowerCase() +
getClusterName();
}
diff --git
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
index 6109297..21b53e9 100644
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
@@ -26,6 +26,7 @@ import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hive.ql.hooks.ExecuteWithHookContext;
import org.apache.hadoop.hive.ql.hooks.HookContext;
+import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer;
import org.apache.hadoop.hive.ql.plan.HiveOperation;
import org.apache.hadoop.hive.shims.Utils;
import org.apache.hadoop.security.UserGroupInformation;
@@ -81,6 +82,9 @@ public class HiveHook extends AtlasHook implements
ExecuteWithHookContext {
private static final List<Pattern> hiveTablesToIgnore =
new ArrayList<>();
private static final List<Pattern> hiveTablesToPrune =
new ArrayList<>();
private static final Map<String, PreprocessAction> hiveTablesCache;
+ private static final List ignoreDummyDatabaseName;
+ private static final List ignoreDummyTableName;
+ private static final String
ignoreValuesTmpTableNamePrefix;
private static HiveHookObjectNamesCache knownObjects = null;
@@ -134,6 +138,16 @@ public class HiveHook extends AtlasHook implements
ExecuteWithHookContext {
}
knownObjects = nameCacheEnabled ? new
HiveHookObjectNamesCache(nameCacheDatabaseMaxCount, nameCacheTableMaxCount,
nameCacheRebuildIntervalSeconds) : null;
+
+ List<String> defaultDummyDatabase = new ArrayList<>();
+ List<String> defaultDummyTable = new ArrayList<>();
+
+ defaultDummyDatabase.add(SemanticAnalyzer.DUMMY_DATABASE);
+ defaultDummyTable.add(SemanticAnalyzer.DUMMY_TABLE);
+
+ ignoreDummyDatabaseName =
atlasProperties.getList("atlas.hook.hive.ignore.dummy.database.name",
defaultDummyDatabase);
+ ignoreDummyTableName =
atlasProperties.getList("atlas.hook.hive.ignore.dummy.table.name",
defaultDummyTable);
+ ignoreValuesTmpTableNamePrefix =
atlasProperties.getString("atlas.hook.hive.ignore.values.tmp.table.name.prefix",
"Values__Tmp__Table__");
}
@@ -246,6 +260,18 @@ public class HiveHook extends AtlasHook implements
ExecuteWithHookContext {
return skipHiveColumnLineageHive20633InputsThreshold;
}
+ public List getIgnoreDummyDatabaseName() {
+ return ignoreDummyDatabaseName;
+ }
+
+ public List getIgnoreDummyTableName() {
+ return ignoreDummyTableName;
+ }
+
+ public String getIgnoreValuesTmpTableNamePrefix() {
+ return ignoreValuesTmpTableNamePrefix;
+ }
+
public PreprocessAction getPreprocessActionForHiveTable(String
qualifiedName) {
PreprocessAction ret = PreprocessAction.NONE;
diff --git
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
index 4127c61..bb39fdf 100644
---
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
+++
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
@@ -237,17 +237,29 @@ public abstract class BaseHiveEvent {
switch (entity.getType()) {
case DATABASE: {
- Database db =
getHive().getDatabase(entity.getDatabase().getName());
+ if
(!context.getIgnoreDummyDatabaseName().contains(entity.getDatabase().getName()))
{
+ Database db =
getHive().getDatabase(entity.getDatabase().getName());
- ret = toDbEntity(db);
+ ret = toDbEntity(db);
+ }
}
break;
case TABLE:
case PARTITION: {
- Table table =
getHive().getTable(entity.getTable().getDbName(),
entity.getTable().getTableName());
+ String dbName = entity.getTable().getDbName();
+ String tableName = entity.getTable().getTableName();
+ boolean skipTable =
StringUtils.isNotEmpty(context.getIgnoreValuesTmpTableNamePrefix()) &&
tableName.toLowerCase().startsWith(context.getIgnoreValuesTmpTableNamePrefix());
+
+ if (!skipTable) {
+ skipTable =
context.getIgnoreDummyTableName().contains(tableName) &&
context.getIgnoreDummyDatabaseName().contains(dbName);
+ }
- ret = toTableEntity(table, entityExtInfo);
+ if (!skipTable) {
+ Table table = getHive().getTable(dbName, tableName);
+
+ ret = toTableEntity(table, entityExtInfo);
+ }
}
break;
