Repository: incubator-ranger Updated Branches: refs/heads/master 269617d5d -> 6194ea0c4
RANGER-780: update Atlas notification processing for recent changes to qualifiedName attribute Signed-off-by: Madhan Neethiraj <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/6194ea0c Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/6194ea0c Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/6194ea0c Branch: refs/heads/master Commit: 6194ea0c4d4059669f4f679741991fcfba3e495c Parents: 269617d Author: Abhay Kulkarni <[email protected]> Authored: Wed Dec 9 17:01:22 2015 -0800 Committer: Madhan Neethiraj <[email protected]> Committed: Sat Dec 12 09:30:25 2015 -0800 ---------------------------------------------------------------------- .../source/atlas/AtlasNotificationMapper.java | 117 ++++++++++++------- .../tagsync/process/TestTagSynchronizer.java | 32 +++++ 2 files changed, 106 insertions(+), 43 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/6194ea0c/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java index 7a2422b..666c2c8 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java @@ -36,7 +36,7 @@ import org.apache.ranger.tagsync.process.TagSyncConfig; import java.util.*; -class AtlasNotificationMapper { +public class AtlasNotificationMapper { private static final Log LOG = LogFactory.getLog(AtlasNotificationMapper.class); public static final String ENTITY_TYPE_HIVE_DB = "hive_db"; @@ -50,8 +50,7 @@ class AtlasNotificationMapper { public static final String ENTITY_ATTRIBUTE_QUALIFIED_NAME = "qualifiedName"; public static final String ENTITY_ATTRIBUTE_QUALIFIED_NAME_FOR_HIVE_TABLE = "name"; public static final String QUALIFIED_NAME_FORMAT_DELIMITER_STRING = "\\."; - - private static final int MAX_HIERARCHY_LEVELS = 5; + public static final String QUALIFIED_NAME_FORMAT_CLUSTER_DELIMITER_STRING = "@"; private static Properties properties = null; @@ -177,43 +176,45 @@ class AtlasNotificationMapper { elements = new HashMap<String, RangerPolicy.RangerPolicyResource>(); - String[] components = getQualifiedNameComponents(entity); - // components should contain qualifiedName, instanceName, dbName, tableName, columnName in that order + List<String> components = getQualifiedNameComponents(entity); + // components should contain qualifiedName, clusterName, dbName, tableName, columnName in that order String entityTypeName = entity.getTypeName(); - String instanceName, dbName, tableName, columnName; + String qualifiedName = components.get(0); + + String clusterName, dbName, tableName, columnName; - if (components.length > 1) { - instanceName = components[1]; - serviceName = getServiceName(instanceName, entityTypeName); + if (components.size() > 1) { + clusterName = components.get(1); + serviceName = getServiceName(clusterName, entityTypeName); } if (StringUtils.equals(entityTypeName, ENTITY_TYPE_HIVE_DB)) { - if (components.length > 2) { - dbName = components[2]; + if (components.size() > 2) { + dbName = components.get(2); RangerPolicy.RangerPolicyResource dbPolicyResource = new RangerPolicy.RangerPolicyResource(dbName); elements.put(RANGER_TYPE_HIVE_DB, dbPolicyResource); } else { - LOG.error("invalid qualifiedName for HIVE_DB, qualifiedName=" + components[0]); + LOG.error("invalid qualifiedName for HIVE_DB, qualifiedName=" + qualifiedName); } } else if (StringUtils.equals(entityTypeName, ENTITY_TYPE_HIVE_TABLE)) { - if (components.length > 3) { - dbName = components[2]; - tableName = components[3]; + if (components.size() > 3) { + dbName = components.get(2); + tableName = components.get(3); RangerPolicy.RangerPolicyResource dbPolicyResource = new RangerPolicy.RangerPolicyResource(dbName); elements.put(RANGER_TYPE_HIVE_DB, dbPolicyResource); RangerPolicy.RangerPolicyResource tablePolicyResource = new RangerPolicy.RangerPolicyResource(tableName); elements.put(RANGER_TYPE_HIVE_TABLE, tablePolicyResource); } else { - LOG.error("invalid qualifiedName for HIVE_TABLE, qualifiedName=" + components[0]); + LOG.error("invalid qualifiedName for HIVE_TABLE, qualifiedName=" + qualifiedName); } } else if (StringUtils.equals(entityTypeName, ENTITY_TYPE_HIVE_COLUMN)) { - if (components.length > 4) { - dbName = components[2]; - tableName = components[3]; - columnName = components[4]; + if (components.size() > 4) { + dbName = components.get(2); + tableName = components.get(3); + columnName = components.get(4); RangerPolicy.RangerPolicyResource dbPolicyResource = new RangerPolicy.RangerPolicyResource(dbName); elements.put(RANGER_TYPE_HIVE_DB, dbPolicyResource); RangerPolicy.RangerPolicyResource tablePolicyResource = new RangerPolicy.RangerPolicyResource(tableName); @@ -221,7 +222,7 @@ class AtlasNotificationMapper { RangerPolicy.RangerPolicyResource columnPolicyResource = new RangerPolicy.RangerPolicyResource(columnName); elements.put(RANGER_TYPE_HIVE_COLUMN, columnPolicyResource); } else { - LOG.error("invalid qualifiedName for HIVE_COLUMN, qualifiedName=" + components[0]); + LOG.error("invalid qualifiedName for HIVE_COLUMN, qualifiedName=" + qualifiedName); } } @@ -298,46 +299,76 @@ class AtlasNotificationMapper { return ret; } - static private String[] getQualifiedNameComponents(IReferenceableInstance entity) throws Exception { - String ret[] = new String[MAX_HIERARCHY_LEVELS]; - - String qualifiedNameAttributeName = StringUtils.equals(entity.getTypeName(), ENTITY_TYPE_HIVE_TABLE) ? + static private String getQualifiedNameAttributeName(String entityTypeName) { + String ret = StringUtils.equals(entityTypeName, ENTITY_TYPE_HIVE_TABLE) ? ENTITY_ATTRIBUTE_QUALIFIED_NAME_FOR_HIVE_TABLE : ENTITY_ATTRIBUTE_QUALIFIED_NAME; - String qualifiedName = getEntityAttribute(entity, qualifiedNameAttributeName, String.class); + return ret; + } - if (StringUtils.isBlank(qualifiedName)) { - throw new Exception("Could not get a valid value for " + qualifiedNameAttributeName + " attribute from entity notification."); - } + static private List<String> getQualifiedNameComponents(IReferenceableInstance entity) throws Exception { - String nameHierarchy[] = qualifiedName.split(QUALIFIED_NAME_FORMAT_DELIMITER_STRING); + List<String> ret = null; - int hierarchyLevels = nameHierarchy.length; + String qualifiedNameAttributeName = getQualifiedNameAttributeName(entity.getTypeName()); + + String qualifiedName = getEntityAttribute(entity, qualifiedNameAttributeName, String.class); + + ret = getQualifiedNameComponents(entity.getTypeName(), qualifiedName); if (LOG.isDebugEnabled()) { LOG.debug("----- Entity-Id:" + entity.getId()._getId()); LOG.debug("----- Entity-Type-Name:" + entity.getTypeName()); - LOG.debug("----- Entity-Qualified-Name:" + qualifiedName); - LOG.debug("----- Entity-Qualified-Name-Components -----"); - for (int i = 0; i < hierarchyLevels; i++) { - LOG.debug("----- Index:" + i + " Value:" + nameHierarchy[i]); + LOG.debug("----- Entity-Components -----"); + int i = 0; + for (String value : ret) { + LOG.debug("----- Index:" + i++ + " Value:" + value); } } + return ret; + } + + static public List<String> getQualifiedNameComponents(String entityTypeName, String qualifiedName) throws Exception { + + List<String> ret = null; + + String qualifiedNameAttributeName = getQualifiedNameAttributeName(entityTypeName); + + if (StringUtils.isBlank(qualifiedName)) { + throw new Exception("Could not get a valid value for " + qualifiedNameAttributeName + " attribute from entity notification."); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("Received .... " + qualifiedNameAttributeName + "=" + qualifiedName + " for entity type " + entityTypeName); + } + + String components[] = qualifiedName.split(QUALIFIED_NAME_FORMAT_CLUSTER_DELIMITER_STRING); - int i; - for (i = 0; i < ret.length; i++) { - ret[i] = null; + if (components == null || components.length != 2) { + throw new Exception("Qualified Name does not contain cluster-name, qualifiedName=" + qualifiedName); } - ret[0] = qualifiedName; - for (i = 0; i < hierarchyLevels; i++) { - ret[i + 1] = nameHierarchy[i]; + if (LOG.isDebugEnabled()) { + LOG.debug("name-hierarchy=" + components[0] + ", cluster-name=" + components[1]); + } + + String nameHierarchy[] = components[0].split(QUALIFIED_NAME_FORMAT_DELIMITER_STRING); + + int hierarchyLevels = nameHierarchy.length; + + ret = new ArrayList<String>(); + + ret.add(qualifiedName); + ret.add(components[1]); + + for (int i = 0; i < hierarchyLevels; i++) { + ret.add(nameHierarchy[i]); } return ret; } - static private String getServiceName(String instanceName, String entityTypeName) { + static private String getServiceName(String clusterName, String entityTypeName) { // Parse entityTypeName to get the Apache-component Name // Assumption: entityTypeName is <componentName>_<component_specific_type_name> // such as hive_table, hadoop_path, hbase_queue, etc. @@ -347,7 +378,7 @@ class AtlasNotificationMapper { apacheComponent = apacheComponents[0].toLowerCase(); } - return TagSyncConfig.getServiceName(apacheComponent, instanceName, properties); + return TagSyncConfig.getServiceName(apacheComponent, clusterName, properties); } static private <T> T getEntityAttribute(IReferenceableInstance entity, String name, Class<T> type) { http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/6194ea0c/tagsync/src/test/java/org/apache/ranger/tagsync/process/TestTagSynchronizer.java ---------------------------------------------------------------------- diff --git a/tagsync/src/test/java/org/apache/ranger/tagsync/process/TestTagSynchronizer.java b/tagsync/src/test/java/org/apache/ranger/tagsync/process/TestTagSynchronizer.java index 9d603d4..59d521c 100644 --- a/tagsync/src/test/java/org/apache/ranger/tagsync/process/TestTagSynchronizer.java +++ b/tagsync/src/test/java/org/apache/ranger/tagsync/process/TestTagSynchronizer.java @@ -20,15 +20,18 @@ package org.apache.ranger.tagsync.process; +import org.apache.commons.lang.StringUtils; import org.apache.ranger.tagsync.model.TagSource; import org.apache.ranger.tagsync.process.TagSyncConfig; import org.apache.ranger.tagsync.process.TagSynchronizer; +import org.apache.ranger.tagsync.source.atlas.AtlasNotificationMapper; import org.apache.ranger.tagsync.source.atlas.TagAtlasSource; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; import java.io.*; +import java.util.List; import java.util.Properties; import static org.junit.Assert.*; @@ -45,6 +48,7 @@ public class TestTagSynchronizer { TagSyncConfig config = TagSyncConfig.getInstance(); TagSyncConfig.dumpConfiguration(config, new BufferedWriter(new OutputStreamWriter(System.out))); + System.out.println(); Properties props = config.getProperties(); @@ -99,4 +103,32 @@ public class TestTagSynchronizer { System.out.println("Exiting testTagDownload()"); } + + @Test + public void testQualifiedNames() { + + List<String> components; + try { + components = AtlasNotificationMapper.getQualifiedNameComponents("hive_db", "database@cluster"); + printComponents(components); + + components = AtlasNotificationMapper.getQualifiedNameComponents("hive_table", "database.table@cluster"); + printComponents(components); + + components = AtlasNotificationMapper.getQualifiedNameComponents("hive_column", "database.table.column@cluster"); + printComponents(components); + + assert(true); + } catch (Exception e) { + System.out.println("Failed..."); + assert(false); + } + + } + private void printComponents(List<String> components) { + int i = 0; + for (String value : components) { + System.out.println("----- Index:" + i++ + " Value:" + value); + } + } }
