Repository: atlas
Updated Branches:
  refs/heads/master 85280ddf0 -> 854208c1e


ATLAS-2961: updated Hive hook to recognize configuration 
'hdfs_path.convert_to_lowercase'

(cherry picked from commit a37aeda48aa4239cd110e665c837c730edb4b800)
(cherry picked from commit fea8acbc6790fa23fe5b3121b38a3c218c694c09)


Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/854208c1
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/854208c1
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/854208c1

Branch: refs/heads/master
Commit: 854208c1ebc2a8a643dfaef190c76a13041cc1ad
Parents: 85280dd
Author: Madhan Neethiraj <mad...@apache.org>
Authored: Fri Nov 9 09:00:13 2018 -0800
Committer: Madhan Neethiraj <mad...@apache.org>
Committed: Sat Nov 10 14:56:33 2018 -0800

----------------------------------------------------------------------
 .../atlas/hive/bridge/HiveMetaStoreBridge.java  |  4 ++--
 .../atlas/hive/hook/AtlasHiveHookContext.java   |  4 ++++
 .../org/apache/atlas/hive/hook/HiveHook.java    |  7 ++++++
 .../atlas/hive/hook/events/BaseHiveEvent.java   | 24 ++++++++++++++++----
 4 files changed, 33 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/atlas/blob/854208c1/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
----------------------------------------------------------------------
diff --git 
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
 
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
index dbb71ea..21ceda2 100755
--- 
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
+++ 
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
@@ -205,7 +205,7 @@ public class HiveMetaStoreBridge {
      * @param hiveConf {@link HiveConf} for Hive component in the cluster
      */
     public HiveMetaStoreBridge(Configuration atlasProperties, HiveConf 
hiveConf, AtlasClientV2 atlasClientV2) throws Exception {
-        this(atlasProperties.getString(HIVE_CLUSTER_NAME, 
DEFAULT_CLUSTER_NAME), Hive.get(hiveConf), atlasClientV2, 
atlasProperties.getBoolean(HDFS_PATH_CONVERT_TO_LOWER_CASE, true));
+        this(atlasProperties.getString(HIVE_CLUSTER_NAME, 
DEFAULT_CLUSTER_NAME), Hive.get(hiveConf), atlasClientV2, 
atlasProperties.getBoolean(HDFS_PATH_CONVERT_TO_LOWER_CASE, false));
     }
 
     /**
@@ -700,7 +700,7 @@ public class HiveMetaStoreBridge {
         String      nameServiceID = 
HdfsNameServiceResolver.getNameServiceIDForPath(pathUri);
         Path        path          = new Path(pathUri);
 
-        ret.setAttribute(ATTRIBUTE_NAME, 
Path.getPathWithoutSchemeAndAuthority(path).toString().toLowerCase());
+        ret.setAttribute(ATTRIBUTE_NAME, 
Path.getPathWithoutSchemeAndAuthority(path).toString());
         ret.setAttribute(ATTRIBUTE_CLUSTER_NAME, clusterName);
 
         if (StringUtils.isNotEmpty(nameServiceID)) {

http://git-wip-us.apache.org/repos/asf/atlas/blob/854208c1/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java
----------------------------------------------------------------------
diff --git 
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java
 
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java
index b467f4c..23cb853 100644
--- 
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java
+++ 
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java
@@ -85,6 +85,10 @@ public class AtlasHiveHookContext {
         return hook.getClusterName();
     }
 
+    public boolean isConvertHdfsPathToLowerCase() {
+        return hook.isConvertHdfsPathToLowerCase();
+    }
+
     public boolean getSkipHiveColumnLineageHive20633() {
         return hook.getSkipHiveColumnLineageHive20633();
     }

http://git-wip-us.apache.org/repos/asf/atlas/blob/854208c1/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
----------------------------------------------------------------------
diff --git 
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java 
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
index a9570e1..0f48578 100644
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
@@ -47,6 +47,7 @@ public class HiveHook extends AtlasHook implements 
ExecuteWithHookContext {
 
     public static final String CONF_PREFIX                         = 
"atlas.hook.hive.";
     public static final String CONF_CLUSTER_NAME                   = 
"atlas.cluster.name";
+    public static final String HDFS_PATH_CONVERT_TO_LOWER_CASE     = 
CONF_PREFIX + "hdfs_path.convert_to_lowercase";
     public static final String HOOK_NAME_CACHE_ENABLED             = 
CONF_PREFIX + "name.cache.enabled";
     public static final String HOOK_NAME_CACHE_DATABASE_COUNT      = 
CONF_PREFIX + "name.cache.database.count";
     public static final String HOOK_NAME_CACHE_TABLE_COUNT         = 
CONF_PREFIX + "name.cache.table.count";
@@ -59,6 +60,7 @@ public class HiveHook extends AtlasHook implements 
ExecuteWithHookContext {
     private static final Map<String, HiveOperation> OPERATION_MAP = new 
HashMap<>();
 
     private static final String  clusterName;
+    private static final boolean convertHdfsPathToLowerCase;
     private static final boolean nameCacheEnabled;
     private static final int     nameCacheDatabaseMaxCount;
     private static final int     nameCacheTableMaxCount;
@@ -75,6 +77,7 @@ public class HiveHook extends AtlasHook implements 
ExecuteWithHookContext {
         }
 
         clusterName                     = 
atlasProperties.getString(CONF_CLUSTER_NAME, DEFAULT_CLUSTER_NAME);
+        convertHdfsPathToLowerCase      = 
atlasProperties.getBoolean(HDFS_PATH_CONVERT_TO_LOWER_CASE, false);
         nameCacheEnabled                = 
atlasProperties.getBoolean(HOOK_NAME_CACHE_ENABLED, true);
         nameCacheDatabaseMaxCount       = 
atlasProperties.getInt(HOOK_NAME_CACHE_DATABASE_COUNT, 10000);
         nameCacheTableMaxCount          = 
atlasProperties.getInt(HOOK_NAME_CACHE_TABLE_COUNT, 10000);
@@ -189,6 +192,10 @@ public class HiveHook extends AtlasHook implements 
ExecuteWithHookContext {
         return clusterName;
     }
 
+    public boolean isConvertHdfsPathToLowerCase() {
+        return convertHdfsPathToLowerCase;
+    }
+
     public boolean getSkipHiveColumnLineageHive20633() {
         return skipHiveColumnLineageHive20633;
     }

http://git-wip-us.apache.org/repos/asf/atlas/blob/854208c1/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
----------------------------------------------------------------------
diff --git 
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
 
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
index 09f011c..e4537b4 100644
--- 
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
+++ 
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
@@ -145,6 +145,7 @@ public abstract class BaseHiveEvent {
     public static final String HBASE_NAMESPACE_TABLE_DELIMITER     = ":";
     public static final String HBASE_PARAM_TABLE_NAME              = 
"hbase.table.name";
     public static final long   MILLIS_CONVERT_FACTOR               = 1000;
+    public static final String HDFS_PATH_PREFIX                    = "hdfs://";
 
     public static final Map<Integer, String> OWNER_TYPE_TO_ENUM_VALUE = new 
HashMap<>();
 
@@ -505,7 +506,11 @@ public abstract class BaseHiveEvent {
 
     protected AtlasEntity getPathEntity(Path path, AtlasEntityExtInfo extInfo) 
{
         AtlasEntity ret;
-        String      strPath = path.toString().toLowerCase();
+        String strPath = path.toString();
+
+        if (strPath.startsWith(HDFS_PATH_PREFIX) && 
context.isConvertHdfsPathToLowerCase()) {
+            strPath = strPath.toLowerCase();
+        }
 
         if (isS3Path(strPath)) {
             String      bucketName          = path.toUri().getAuthority();
@@ -550,9 +555,15 @@ public abstract class BaseHiveEvent {
                     ret.setAttribute(ATTRIBUTE_NAMESERVICE_ID, nameServiceID);
                 }
 
+                String name = 
Path.getPathWithoutSchemeAndAuthority(path).toString();
+
+                if (strPath.startsWith(HDFS_PATH_PREFIX) && 
context.isConvertHdfsPathToLowerCase()) {
+                    name = name.toLowerCase();
+                }
+
                 ret.setAttribute(ATTRIBUTE_PATH, attrPath);
                 ret.setAttribute(ATTRIBUTE_QUALIFIED_NAME, pathQualifiedName);
-                ret.setAttribute(ATTRIBUTE_NAME, 
Path.getPathWithoutSchemeAndAuthority(path).toString().toLowerCase());
+                ret.setAttribute(ATTRIBUTE_NAME, name);
                 ret.setAttribute(ATTRIBUTE_CLUSTER_NAME, getClusterName());
 
                 context.putEntity(pathQualifiedName, ret);
@@ -688,7 +699,12 @@ public abstract class BaseHiveEvent {
     }
 
     protected String getQualifiedName(URI location) {
-        String strPath       = new Path(location).toString().toLowerCase();
+        String strPath = new Path(location).toString();
+
+        if (strPath.startsWith(HDFS_PATH_PREFIX) && 
context.isConvertHdfsPathToLowerCase()) {
+            strPath = strPath.toLowerCase();
+        }
+
         String nameServiceID = 
HdfsNameServiceResolver.getNameServiceIDForPath(strPath);
         String attrPath      = StringUtils.isEmpty(nameServiceID) ? strPath : 
HdfsNameServiceResolver.getPathWithNameServiceID(strPath);
 
@@ -697,7 +713,7 @@ public abstract class BaseHiveEvent {
 
     protected String getQualifiedName(String path) {
         if (path.startsWith(HdfsNameServiceResolver.HDFS_SCHEME)) {
-            return (path + QNAME_SEP_CLUSTER_NAME).toLowerCase() + 
getClusterName();
+            return path + QNAME_SEP_CLUSTER_NAME + getClusterName();
         }
 
         return path.toLowerCase();

Reply via email to