[atlas] branch master updated: ATLAS-4127: Optimize Hive table calls in Atlas Hook

[sarath]

This is an automated email from the ASF dual-hosted git repository.

sarath pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git


The following commit(s) were added to refs/heads/master by this push:
     new f848c97  ATLAS-4127: Optimize Hive table calls in Atlas Hook
f848c97 is described below

commit f848c9761bd340c9cbb790254420506d97c8e6b4
Author: Deep Singh <deepam1...@gmail.com>
AuthorDate: Wed Feb 3 20:05:28 2021 -0600

    ATLAS-4127: Optimize Hive table calls in Atlas Hook
    
    Signed-off-by: Sarath Subramanian <sar...@apache.org>
---
 .../java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java    | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git 
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
 
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
index 3b2bbe0..7c269ce 100644
--- 
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
+++ 
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
@@ -1035,6 +1035,7 @@ public abstract class BaseHiveEvent {
         Collections.sort(sortedEntities, entityComparator);
 
         Set<String> dataSetsProcessed = new HashSet<>();
+        Map<String, Table> tableMap   = new HashMap<>();
 
         for (Entity entity : sortedEntities) {
             if (ignoreHDFSPaths && 
(Entity.Type.DFS_DIR.equals(entity.getType()) || 
Entity.Type.LOCAL_DIR.equals(entity.getType()))) {
@@ -1046,8 +1047,14 @@ public abstract class BaseHiveEvent {
 
             try {
                 if (entity.getType() == Entity.Type.PARTITION || 
entity.getType() == Entity.Type.TABLE) {
-                    Table table = 
getHive().getTable(entity.getTable().getDbName(), 
entity.getTable().getTableName());
+                    String tableKey = entity.getTable().getDbName() + "." + 
entity.getTable().getTableName();
+                    Table  table    = tableMap.get(tableKey);
 
+                    if (table == null) {
+                        table = 
getHive().getTable(entity.getTable().getDbName(), 
entity.getTable().getTableName());
+
+                        tableMap.put(tableKey, table); //since there could be 
several partitions in a table, store it to avoid hive calls.
+                    }
                     if (table != null) {
                         createTime    = getTableCreateTime(table);
                         qualifiedName = getQualifiedName(table);