[atlas] branch branch-2.0 updated: ATLAS-3590: Hive hook should ignore capturing lineage for temporary tables in CTAS queries

[sarath]

This is an automated email from the ASF dual-hosted git repository.

sarath pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git


The following commit(s) were added to refs/heads/branch-2.0 by this push:
     new 1d3bef6  ATLAS-3590: Hive hook should ignore capturing lineage for 
temporary tables in CTAS queries
1d3bef6 is described below

commit 1d3bef6319b76b804774452fd1ee8c6521c5b54e
Author: Sarath Subramanian <sar...@apache.org>
AuthorDate: Thu Jan 16 16:51:14 2020 -0800

    ATLAS-3590: Hive hook should ignore capturing lineage for temporary tables 
in CTAS queries
    
    (cherry picked from commit 47f2e77bf798547e652e16cf10adaa818ded1e3e)
---
 .../src/main/java/org/apache/atlas/hive/hook/HiveHook.java     |  2 +-
 .../java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java  | 10 +++++++---
 .../org/apache/atlas/hive/hook/events/CreateHiveProcess.java   |  9 ++++++---
 3 files changed, 14 insertions(+), 7 deletions(-)

diff --git 
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java 
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
index 5b1f61b..e886b9b 100644
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
@@ -201,7 +201,7 @@ public class HiveHook extends AtlasHook implements 
ExecuteWithHookContext {
                 case EXPORT:
                 case IMPORT:
                 case QUERY:
-                    event = new CreateHiveProcess(context);
+                    event = new CreateHiveProcess(context, true);
                 break;
 
                 case ALTERTABLE_FILEFORMAT:
diff --git 
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
 
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
index b3b2299..91accfa 100644
--- 
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
+++ 
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
@@ -233,14 +233,14 @@ public abstract class BaseHiveEvent {
         }
     }
 
-    protected AtlasEntity getInputOutputEntity(Entity entity, 
AtlasEntityExtInfo entityExtInfo) throws Exception {
+    protected AtlasEntity getInputOutputEntity(Entity entity, 
AtlasEntityExtInfo entityExtInfo, boolean skipTempTables) throws Exception {
         AtlasEntity ret = null;
 
         switch(entity.getType()) {
             case TABLE:
             case PARTITION:
             case DFS_DIR: {
-                ret = toAtlasEntity(entity, entityExtInfo);
+                ret = toAtlasEntity(entity, entityExtInfo, skipTempTables);
             }
             break;
         }
@@ -248,7 +248,7 @@ public abstract class BaseHiveEvent {
         return ret;
     }
 
-    protected AtlasEntity toAtlasEntity(Entity entity, AtlasEntityExtInfo 
entityExtInfo) throws Exception {
+    protected AtlasEntity toAtlasEntity(Entity entity, AtlasEntityExtInfo 
entityExtInfo, boolean skipTempTables) throws Exception {
         AtlasEntity ret = null;
 
         switch (entity.getType()) {
@@ -274,6 +274,10 @@ public abstract class BaseHiveEvent {
                 }
 
                 if (!skipTable) {
+                    skipTable = skipTempTables && 
entity.getTable().isTemporary();
+                }
+
+                if (!skipTable) {
                     Table table = getHive().getTable(dbName, tableName);
 
                     ret = toTableEntity(table, entityExtInfo);
diff --git 
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateHiveProcess.java
 
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateHiveProcess.java
index 82df576..659417d 100644
--- 
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateHiveProcess.java
+++ 
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateHiveProcess.java
@@ -49,9 +49,12 @@ import java.util.Set;
 
 public class CreateHiveProcess extends BaseHiveEvent {
     private static final Logger LOG = 
LoggerFactory.getLogger(CreateHiveProcess.class);
+    private final boolean skipTempTables;
 
-    public CreateHiveProcess(AtlasHiveHookContext context) {
+    public CreateHiveProcess(AtlasHiveHookContext context, boolean 
skipTempTables) {
         super(context);
+
+        this.skipTempTables = skipTempTables;
     }
 
     @Override
@@ -84,7 +87,7 @@ public class CreateHiveProcess extends BaseHiveEvent {
                         continue;
                     }
 
-                    AtlasEntity entity = getInputOutputEntity(input, ret);
+                    AtlasEntity entity = getInputOutputEntity(input, ret, 
skipTempTables);
 
                     if (!input.isDirect()) {
                         continue;
@@ -104,7 +107,7 @@ public class CreateHiveProcess extends BaseHiveEvent {
                         continue;
                     }
 
-                    AtlasEntity entity = getInputOutputEntity(output, ret);
+                    AtlasEntity entity = getInputOutputEntity(output, ret, 
skipTempTables);
 
                     if (entity != null) {
                         outputs.add(entity);