RANGER-660: TagSync updated to support mapping of Atlas instance-name to Ranger service-name via property
Signed-off-by: Madhan Neethiraj <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/892f6bf3 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/892f6bf3 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/892f6bf3 Branch: refs/heads/master Commit: 892f6bf3002ee18225bfcf758b9662513d3d1171 Parents: 2f64a68 Author: Abhay Kulkarni <[email protected]> Authored: Tue Oct 6 14:29:53 2015 -0700 Committer: Madhan Neethiraj <[email protected]> Committed: Wed Oct 14 09:52:31 2015 -0700 ---------------------------------------------------------------------- .../conf/templates/installprop2xml.properties | 2 ++ .../conf/templates/ranger-tagsync-template.xml | 4 +++ tagsync/scripts/install.properties | 7 ++++- tagsync/scripts/setup.py | 27 ++++++++++++++++++-- .../source/atlas/AtlasNotificationMapper.java | 2 ++ 5 files changed, 39 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/892f6bf3/tagsync/conf/templates/installprop2xml.properties ---------------------------------------------------------------------- diff --git a/tagsync/conf/templates/installprop2xml.properties b/tagsync/conf/templates/installprop2xml.properties index 9fe1cac..5d445ce 100644 --- a/tagsync/conf/templates/installprop2xml.properties +++ b/tagsync/conf/templates/installprop2xml.properties @@ -34,3 +34,5 @@ TAGSYNC_FILESOURCE_FILENAME = ranger.tagsync.filesource.filename TAGSYNC_ATLAS_KAFKA_ENDPOINTS = atlas.notification.kafka.bootstrap.servers TAGSYNC_ATLAS_ZOOKEEPER_ENDPOINT = atlas.notification.kafka.zookeeper.connect TAGSYNC_ATLAS_CONSUMER_GROUP = atlas.notification.kafka.group.id + +TAGSYNC_ATLAS_TO_RANGER_SERVICE_MAPPING = ranger.tagsync.atlas.to.service.mapping http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/892f6bf3/tagsync/conf/templates/ranger-tagsync-template.xml ---------------------------------------------------------------------- diff --git a/tagsync/conf/templates/ranger-tagsync-template.xml b/tagsync/conf/templates/ranger-tagsync-template.xml index 5f9f3b5..f9bc31b 100644 --- a/tagsync/conf/templates/ranger-tagsync-template.xml +++ b/tagsync/conf/templates/ranger-tagsync-template.xml @@ -51,4 +51,8 @@ <name>ranger.tagsync.source.impl.class</name> <value></value> </property> + <property> + <name>ranger.tagsync.atlas.to.service.mapping</name> + <value></value> + </property> </configuration> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/892f6bf3/tagsync/scripts/install.properties ---------------------------------------------------------------------- diff --git a/tagsync/scripts/install.properties b/tagsync/scripts/install.properties index 82477d3..fb11ede 100644 --- a/tagsync/scripts/install.properties +++ b/tagsync/scripts/install.properties @@ -55,4 +55,9 @@ SYNC_INTERVAL = 60000 TAGSYNC_ATLAS_KAFKA_ENDPOINTS = localhost:6667 TAGSYNC_ATLAS_ZOOKEEPER_ENDPOINT = localhost:2181 -TAGSYNC_ATLAS_CONSUMER_GROUP = entityConsumer \ No newline at end of file +TAGSYNC_ATLAS_CONSUMER_GROUP = entityConsumer + +# Mapping from Atlas hive instance-name to Ranger service-name +# this needs to be in format clusterName,componentType,serviceName;clusterName2,componentType2,serviceName2 +# Note that there are no blanks anywhere in the value-string +TAGSYNC_ATLAS_TO_RANGER_SERVICE_MAPPING = c0,hive,cl1_hive;c0,hbase,other_hbase http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/892f6bf3/tagsync/scripts/setup.py ---------------------------------------------------------------------- diff --git a/tagsync/scripts/setup.py b/tagsync/scripts/setup.py index 383964a..faa0af4 100755 --- a/tagsync/scripts/setup.py +++ b/tagsync/scripts/setup.py @@ -70,7 +70,10 @@ SYNC_INTERVAL_NEW_KEY = 'ranger.tagsync.sleeptimeinmillisbetweensynccycle' TAGSYNC_ATLAS_KAFKA_ENDPOINTS_KEY = 'TAGSYNC_ATLAS_KAFKA_ENDPOINTS' TAGSYNC_ATLAS_ZOOKEEPER_ENDPOINT_KEY = 'TAGSYNC_ATLAS_ZOOKEEPER_ENDPOINT' TAGSYNC_ATLAS_CONSUMER_GROUP_KEY = 'TAGSYNC_ATLAS_CONSUMER_GROUP' - +TAGSYNC_ATLAS_TO_RANGER_SERVICE_MAPPING = 'ranger.tagsync.atlas.to.service.mapping' +TAGSYNC_INSTALL_PROP_PREFIX_FOR_ATLAS_RANGER_MAPPING = 'ranger.tagsync.atlas.' +TAGSYNC_ATLAS_CLUSTER_IDENTIFIER = '.instance.' +TAGSYNC_INSTALL_PROP_SUFFIX_FOR_ATLAS_RANGER_MAPPING = '.ranger.service' TAG_SOURCE_ATLAS = 'atlas' TAG_SOURCE_FILE = 'file' TAG_SOURCE_LIST = [ TAG_SOURCE_ATLAS, TAG_SOURCE_FILE ] @@ -138,7 +141,27 @@ def writeXMLUsingProperties(xmlTemplateFileName,prop,xmlOutputFileName): config.find('value').text = "_" continue if (name in prop.keys()): - config.find('value').text = str(prop[name]) + if (name == TAGSYNC_ATLAS_TO_RANGER_SERVICE_MAPPING): + # Expected value is 'clusterName,componentName,serviceName;clusterName,componentName,serviceName' ... + # Blanks are not supported anywhere in the value. + valueString = str(prop[name]) + multiValues = valueString.split(';') + listLen = len(multiValues) + index = 0 + while index < listLen: + parts = multiValues[index].split(',') + if len(parts) == 3: + newConfig = ET.SubElement(root, 'property') + newName = ET.SubElement(newConfig, 'name') + newValue = ET.SubElement(newConfig, 'value') + newName.text = TAGSYNC_INSTALL_PROP_PREFIX_FOR_ATLAS_RANGER_MAPPING + str(parts[1]) + TAGSYNC_ATLAS_CLUSTER_IDENTIFIER + str(parts[0]) + TAGSYNC_INSTALL_PROP_SUFFIX_FOR_ATLAS_RANGER_MAPPING + newValue.text = str(parts[2]) + else: + print "ERROR: incorrect syntax for %s, value=%s" % (TAGSYNC_ATLAS_TO_RANGER_SERVICE_MAPPING, multiValues[index]) + index += 1 + root.remove(config) + else: + config.find('value').text = str(prop[name]) #else: # print "ERROR: key not found: %s" % (name) if isfile(xmlOutputFileName): http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/892f6bf3/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java index d5108a1..8046b68 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java @@ -340,6 +340,8 @@ class AtlasNotificationMapper { static private String getServiceName(String instanceName, String entityTypeName) { // Parse entityTypeName to get the Apache-component Name + // Assumption: entityTypeName is <componentName>_<component_specific_type_name> + // such as hive_table, hadoop_path, hbase_queue, etc. String apacheComponents[] = entityTypeName.split("_"); String apacheComponent = null; if (apacheComponents.length > 0) {
