Repository: atlas
Updated Branches:
  refs/heads/branch-0.8 8d7e94db7 -> 4a72f2062


ATLAS-2491: updated Hive hook to recognize configuration 
'hdfs_path.convert_to_lowercase'


Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/4a72f206
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/4a72f206
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/4a72f206

Branch: refs/heads/branch-0.8
Commit: 4a72f20628405658d7a53710fd75ce63194ece7d
Parents: 8d7e94d
Author: Madhan Neethiraj <mad...@apache.org>
Authored: Fri Nov 9 09:00:13 2018 -0800
Committer: Madhan Neethiraj <mad...@apache.org>
Committed: Sat Nov 10 14:29:09 2018 -0800

----------------------------------------------------------------------
 .../atlas/hive/bridge/HiveMetaStoreBridge.java  |  4 ++--
 .../atlas/hive/hook/AtlasHiveHookContext.java   |  4 ++++
 .../org/apache/atlas/hive/hook/HiveHook.java    |  7 +++++++
 .../atlas/hive/hook/events/BaseHiveEvent.java   | 22 +++++++++++++++-----
 4 files changed, 30 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/atlas/blob/4a72f206/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
----------------------------------------------------------------------
diff --git 
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
 
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
index d6a1b37..2f6d1fa 100755
--- 
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
+++ 
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
@@ -204,7 +204,7 @@ public class HiveMetaStoreBridge {
      * @param hiveConf {@link HiveConf} for Hive component in the cluster
      */
     public HiveMetaStoreBridge(Configuration atlasProperties, HiveConf 
hiveConf, AtlasClientV2 atlasClientV2) throws Exception {
-        this(atlasProperties.getString(HIVE_CLUSTER_NAME, 
DEFAULT_CLUSTER_NAME), Hive.get(hiveConf), atlasClientV2, 
atlasProperties.getBoolean(HDFS_PATH_CONVERT_TO_LOWER_CASE, true));
+        this(atlasProperties.getString(HIVE_CLUSTER_NAME, 
DEFAULT_CLUSTER_NAME), Hive.get(hiveConf), atlasClientV2, 
atlasProperties.getBoolean(HDFS_PATH_CONVERT_TO_LOWER_CASE, false));
     }
 
     /**
@@ -690,7 +690,7 @@ public class HiveMetaStoreBridge {
         AtlasEntity ret           = new AtlasEntity(HDFS_PATH);
         Path        path          = new Path(pathUri);
 
-        ret.setAttribute(ATTRIBUTE_NAME, 
Path.getPathWithoutSchemeAndAuthority(path).toString().toLowerCase());
+        ret.setAttribute(ATTRIBUTE_NAME, 
Path.getPathWithoutSchemeAndAuthority(path).toString());
         ret.setAttribute(ATTRIBUTE_CLUSTER_NAME, clusterName);
         ret.setAttribute(ATTRIBUTE_PATH, pathUri);
 

http://git-wip-us.apache.org/repos/asf/atlas/blob/4a72f206/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java
----------------------------------------------------------------------
diff --git 
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java
 
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java
index b467f4c..23cb853 100644
--- 
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java
+++ 
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java
@@ -85,6 +85,10 @@ public class AtlasHiveHookContext {
         return hook.getClusterName();
     }
 
+    public boolean isConvertHdfsPathToLowerCase() {
+        return hook.isConvertHdfsPathToLowerCase();
+    }
+
     public boolean getSkipHiveColumnLineageHive20633() {
         return hook.getSkipHiveColumnLineageHive20633();
     }

http://git-wip-us.apache.org/repos/asf/atlas/blob/4a72f206/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
----------------------------------------------------------------------
diff --git 
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java 
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
index 501eb63..7b60553 100644
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
@@ -47,6 +47,7 @@ public class HiveHook extends AtlasHook implements 
ExecuteWithHookContext {
 
     public static final String CONF_PREFIX                         = 
"atlas.hook.hive.";
     public static final String CONF_CLUSTER_NAME                   = 
"atlas.cluster.name";
+    public static final String HDFS_PATH_CONVERT_TO_LOWER_CASE     = 
CONF_PREFIX + "hdfs_path.convert_to_lowercase";
     public static final String HOOK_NAME_CACHE_ENABLED             = 
CONF_PREFIX + "name.cache.enabled";
     public static final String HOOK_NAME_CACHE_DATABASE_COUNT      = 
CONF_PREFIX + "name.cache.database.count";
     public static final String HOOK_NAME_CACHE_TABLE_COUNT         = 
CONF_PREFIX + "name.cache.table.count";
@@ -59,6 +60,7 @@ public class HiveHook extends AtlasHook implements 
ExecuteWithHookContext {
     private static final Map<String, HiveOperation> OPERATION_MAP = new 
HashMap<>();
 
     private static final String  clusterName;
+    private static final boolean convertHdfsPathToLowerCase;
     private static final boolean nameCacheEnabled;
     private static final int     nameCacheDatabaseMaxCount;
     private static final int     nameCacheTableMaxCount;
@@ -75,6 +77,7 @@ public class HiveHook extends AtlasHook implements 
ExecuteWithHookContext {
         }
 
         clusterName                     = 
atlasProperties.getString(CONF_CLUSTER_NAME, DEFAULT_CLUSTER_NAME);
+        convertHdfsPathToLowerCase      = 
atlasProperties.getBoolean(HDFS_PATH_CONVERT_TO_LOWER_CASE, false);
         nameCacheEnabled                = 
atlasProperties.getBoolean(HOOK_NAME_CACHE_ENABLED, true);
         nameCacheDatabaseMaxCount       = 
atlasProperties.getInt(HOOK_NAME_CACHE_DATABASE_COUNT, 10000);
         nameCacheTableMaxCount          = 
atlasProperties.getInt(HOOK_NAME_CACHE_TABLE_COUNT, 10000);
@@ -189,6 +192,10 @@ public class HiveHook extends AtlasHook implements 
ExecuteWithHookContext {
         return clusterName;
     }
 
+    public boolean isConvertHdfsPathToLowerCase() {
+        return convertHdfsPathToLowerCase;
+    }
+
     public boolean getSkipHiveColumnLineageHive20633() {
         return skipHiveColumnLineageHive20633;
     }

http://git-wip-us.apache.org/repos/asf/atlas/blob/4a72f206/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
----------------------------------------------------------------------
diff --git 
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
 
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
index 9109a0f..2d90a15 100644
--- 
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
+++ 
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
@@ -124,6 +124,7 @@ public abstract class BaseHiveEvent {
     public static final String ATTRIBUTE_EXPRESSION                = 
"expression";
     public static final String ATTRIBUTE_ALIASES                   = "aliases";
 
+    public static final String HDFS_PATH_PREFIX = "hdfs://";
 
     public static final long   MILLIS_CONVERT_FACTOR  = 1000;
 
@@ -481,7 +482,14 @@ public abstract class BaseHiveEvent {
     }
 
     protected AtlasEntity getHDFSPathEntity(Path path) {
-        String      strPath           = path.toString().toLowerCase();
+        String strPath = path.toString();
+        String name    = 
Path.getPathWithoutSchemeAndAuthority(path).toString();
+
+        if (strPath.startsWith(HDFS_PATH_PREFIX) && 
context.isConvertHdfsPathToLowerCase()) {
+            strPath = strPath.toLowerCase();
+            name    = name.toLowerCase();
+        }
+
         String      pathQualifiedName = getQualifiedName(strPath);
         AtlasEntity ret               = context.getEntity(pathQualifiedName);
 
@@ -490,7 +498,7 @@ public abstract class BaseHiveEvent {
 
             ret.setAttribute(ATTRIBUTE_PATH, strPath);
             ret.setAttribute(ATTRIBUTE_QUALIFIED_NAME, pathQualifiedName);
-            ret.setAttribute(ATTRIBUTE_NAME, 
Path.getPathWithoutSchemeAndAuthority(path).toString().toLowerCase());
+            ret.setAttribute(ATTRIBUTE_NAME, name);
             ret.setAttribute(ATTRIBUTE_CLUSTER_NAME, getClusterName());
 
             context.putEntity(pathQualifiedName, ret);
@@ -625,14 +633,18 @@ public abstract class BaseHiveEvent {
     }
 
     protected String getQualifiedName(URI location) {
-        String strPath = new Path(location).toString().toLowerCase();
+        String strPath = new Path(location).toString();
+
+        if (strPath.startsWith(HDFS_PATH_PREFIX) && 
context.isConvertHdfsPathToLowerCase()) {
+            strPath = strPath.toLowerCase();
+        }
 
         return getQualifiedName(strPath);
     }
 
     protected String getQualifiedName(String path) {
-        if (path.startsWith("hdfs://")) {
-            return (path + QNAME_SEP_CLUSTER_NAME).toLowerCase() + 
getClusterName();
+        if (path.startsWith(HDFS_PATH_PREFIX)) {
+            return path + QNAME_SEP_CLUSTER_NAME + getClusterName();
         }
 
         return path.toLowerCase();

Reply via email to