Repository: incubator-ranger Updated Branches: refs/heads/master 97deb2b6f -> ce45eff83
RANGER-1032: Update TagSync installation to support kerberized mode configurations Signed-off-by: Madhan Neethiraj <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/ce45eff8 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/ce45eff8 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/ce45eff8 Branch: refs/heads/master Commit: ce45eff839544371c80f68bddf2ef47573cd210c Parents: 97deb2b Author: Abhay Kulkarni <[email protected]> Authored: Thu May 12 17:11:36 2016 -0700 Committer: Madhan Neethiraj <[email protected]> Committed: Tue Jun 14 06:20:02 2016 -0700 ---------------------------------------------------------------------- src/main/assembly/tagsync.xml | 3 - .../conf/templates/installprop2xml.properties | 13 +++ .../conf/templates/ranger-tagsync-template.xml | 16 ++++ tagsync/scripts/install.properties | 21 ++++- tagsync/scripts/ranger-tagsync-upload.sh | 2 +- tagsync/scripts/setup.py | 87 +++++++++++++++---- tagsync/scripts/updatetagadminpassword.py | 61 ++++++++++---- .../ranger/tagsync/process/TagSyncConfig.java | 89 ++++++++++++++++---- .../source/atlasrest/AtlasRESTTagSource.java | 58 +++++++++++-- .../tagsync/source/atlasrest/AtlasRESTUtil.java | 81 +++++++++++++++--- .../src/main/resources/ranger-tagsync-site.xml | 18 ++-- 11 files changed, 362 insertions(+), 87 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ce45eff8/src/main/assembly/tagsync.xml ---------------------------------------------------------------------- diff --git a/src/main/assembly/tagsync.xml b/src/main/assembly/tagsync.xml index fbcca8f..bb5b35c 100644 --- a/src/main/assembly/tagsync.xml +++ b/src/main/assembly/tagsync.xml @@ -93,9 +93,6 @@ <fileMode>644</fileMode> <outputDirectory>/conf.dist</outputDirectory> <directory>tagsync/conf.dist</directory> - <excludes> - <exclude>etc</exclude> - </excludes> </fileSet> <fileSet> <directoryMode>755</directoryMode> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ce45eff8/tagsync/conf/templates/installprop2xml.properties ---------------------------------------------------------------------- diff --git a/tagsync/conf/templates/installprop2xml.properties b/tagsync/conf/templates/installprop2xml.properties index aa0c568..510cd6f 100644 --- a/tagsync/conf/templates/installprop2xml.properties +++ b/tagsync/conf/templates/installprop2xml.properties @@ -26,11 +26,20 @@ TAG_SOURCE_ATLAS_KAFKA_BOOTSTRAP_SERVERS = atlas.kafka.bootstrap.servers TAG_SOURCE_ATLAS_KAFKA_ZOOKEEPER_CONNECT = atlas.kafka.zookeeper.connect TAG_SOURCE_ATLAS_KAFKA_ENTITIES_GROUP_ID = atlas.kafka.entities.group.id +TAG_SOURCE_ATLAS_KAFKA_SERVICE_NAME = atlas.kafka.sasl.kerberos.service.name +TAG_SOURCE_ATLAS_KAFKA_SECURITY_PROTOCOL = atlas.kafka.security.protocol + +TAG_SOURCE_ATLAS_KERBEROS_PRINCIPAL = atlas.jaas.kafkaClient.option.principal +TAG_SOURCE_ATLAS_KERBEROS_KEYTAB = atlas.jaas.kafkaClient.option.keyTab + TAG_SOURCE_ATLASREST_ENABLED = ranger.tagsync.source.atlasrest TAG_SOURCE_ATLASREST_ENDPOINT = ranger.tagsync.source.atlasrest.endpoint TAG_SOURCE_ATLASREST_DOWNLOAD_INTERVAL_IN_MILLIS = ranger.tagsync.source.atlasrest.download.interval.millis +TAG_SOURCE_ATLASREST_USERNAME = ranger.tagsync.source.atlasrest.username +TAG_SOURCE_ATLASREST_PASSWORD = ranger.tagsync.source.atlasrest.password + TAG_SOURCE_FILE_ENABLED = ranger.tagsync.source.file TAG_SOURCE_FILE_FILENAME = ranger.tagsync.source.file.filename @@ -40,6 +49,9 @@ TAGSYNC_ATLAS_TO_RANGER_SERVICE_MAPPING = ranger.tagsync.atlas.to.ranger.service TAGSYNC_ATLAS_CUSTOM_RESOURCE_MAPPERS = ranger.tagsync.atlas.custom.resource.mappers TAGSYNC_KEYSTORE_FILENAME = ranger.tagsync.keystore.filename +TAG_SOURCE_ATLASREST_KEYSTORE_FILENAME = ranger.tagsync.source.atlasrest.keystore.filename +TAG_SOURCE_ATLASREST_SSL_CONFIG_FILENAME = ranger.tagsync.source.atlasrest.ssl.config.filename + unix_user = unix_user unix_group = unix_group @@ -49,6 +61,7 @@ logdir = ranger.tagsync.logdir tagsync_principal = ranger.tagsync.kerberos.principal tagsync_keytab = ranger.tagsync.kerberos.keytab + # TODO - What property in ranger-tagsync-site.xml should hadoop_conf map to?? hadoop_conf = hadoop_conf http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ce45eff8/tagsync/conf/templates/ranger-tagsync-template.xml ---------------------------------------------------------------------- diff --git a/tagsync/conf/templates/ranger-tagsync-template.xml b/tagsync/conf/templates/ranger-tagsync-template.xml index 554a4e0..41aacbf 100644 --- a/tagsync/conf/templates/ranger-tagsync-template.xml +++ b/tagsync/conf/templates/ranger-tagsync-template.xml @@ -87,4 +87,20 @@ <name>ranger.tagsync.dest.ranger.username</name> <value></value> </property> + <property> + <name>ranger.tagsync.source.atlasrest.username</name> + <value></value> + </property> + <property> + <name>ranger.tagsync.source.atlasrest.password</name> + <value></value> + </property> + <property> + <name>ranger.tagsync.source.atlasrest.keystore.filename</name> + <value></value> + </property> + <property> + <name>ranger.tagsync.source.atlasrest.ssl.config.filename</name> + <value></value> + </property> </configuration> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ce45eff8/tagsync/scripts/install.properties ---------------------------------------------------------------------- diff --git a/tagsync/scripts/install.properties b/tagsync/scripts/install.properties index 90878d8..add1a26 100644 --- a/tagsync/scripts/install.properties +++ b/tagsync/scripts/install.properties @@ -32,18 +32,25 @@ TAG_SOURCE_ATLAS_KAFKA_BOOTSTRAP_SERVERS = localhost:6667 TAG_SOURCE_ATLAS_KAFKA_ZOOKEEPER_CONNECT = localhost:2181 TAG_SOURCE_ATLAS_KAFKA_ENTITIES_GROUP_ID = ranger_entities_consumer +TAG_SOURCE_ATLAS_KAFKA_SERVICE_NAME = kafka +TAG_SOURCE_ATLAS_KAFKA_SECURITY_PROTOCOL = SASL_PLAINTEXT + +TAG_SOURCE_ATLAS_KERBEROS_PRINCIPAL = +TAG_SOURCE_ATLAS_KERBEROS_KEYTAB = + TAG_SOURCE_ATLASREST_ENABLED = false TAG_SOURCE_ATLASREST_ENDPOINT = http://localhost:21000 TAG_SOURCE_ATLASREST_DOWNLOAD_INTERVAL_IN_MILLIS = 900000 +TAG_SOURCE_ATLASREST_USERNAME = +TAG_SOURCE_ATLASREST_PASSWORD = + TAG_SOURCE_FILE_ENABLED = false TAG_SOURCE_FILE_FILENAME = /etc/ranger/data/tags.json TAG_SOURCE_FILE_CHECK_INTERVAL_IN_MILLIS = 60000 -TAG_SOURCE_NONE_ENABLED = false - # Mapping from Atlas hive cluster-name to Ranger service-name # this needs to be in format clusterName,componentType,serviceName;clusterName2,componentType2,serviceName2 # Note that there are no blanks anywhere in the value-string @@ -74,6 +81,13 @@ TAGSYNC_ATLAS_CUSTOM_RESOURCE_MAPPERS= # TAGSYNC_KEYSTORE_FILENAME = /etc/ranger/tagsync/conf/rangertagsync.jceks +# File where Atlas credentials is kept in cryptic format + +TAG_SOURCE_ATLASREST_KEYSTORE_FILENAME = /etc/ranger/tagsync/conf/atlasuser.jceks + +# SSL config file name for HTTPS messages to tag source - Atlas-REST +TAG_SOURCE_ATLASREST_SSL_CONFIG_FILENAME = + #User and group for the tagsync process unix_user=ranger unix_group=ranger @@ -82,7 +96,10 @@ unix_group=ranger logdir = log #Set to run in kerberos environment +is_secure = false tagsync_principal= tagsync_keytab= + + hadoop_conf=/etc/hadoop/conf http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ce45eff8/tagsync/scripts/ranger-tagsync-upload.sh ---------------------------------------------------------------------- diff --git a/tagsync/scripts/ranger-tagsync-upload.sh b/tagsync/scripts/ranger-tagsync-upload.sh index 99f8dad..495903e 100755 --- a/tagsync/scripts/ranger-tagsync-upload.sh +++ b/tagsync/scripts/ranger-tagsync-upload.sh @@ -66,5 +66,5 @@ cp="${cdir}/conf:${cdir}/dist/*:${cdir}/lib/*" cd ${cdir} umask 0077 -java -Dproc_rangertagsync-${action} ${JAVA_OPTS} -Dlogdir="${logdir}" -cp "${cp}" ${className} $* > ${logdir}/tagsync.out 2>&1 +java -Dproc_rangertagsync-${action} ${JAVA_OPTS} -Dlogdir="${logdir}" -Dlog4j.configuration=file:/etc/ranger/tagsync/conf/log4j.properties -cp "${cp}" ${className} $* > ${logdir}/tagsync.out 2>&1 http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ce45eff8/tagsync/scripts/setup.py ---------------------------------------------------------------------- diff --git a/tagsync/scripts/setup.py b/tagsync/scripts/setup.py index ec35a10..ea47de7 100755 --- a/tagsync/scripts/setup.py +++ b/tagsync/scripts/setup.py @@ -71,7 +71,12 @@ initPrefixList = ['S99', 'K00'] TAGSYNC_ATLAS_KAFKA_ENDPOINTS_KEY = 'TAG_SOURCE_ATLAS_KAFKA_BOOTSTRAP_SERVERS' TAGSYNC_ATLAS_ZOOKEEPER_ENDPOINT_KEY = 'TAG_SOURCE_ATLAS_KAFKA_ZOOKEEPER_CONNECT' TAGSYNC_ATLAS_CONSUMER_GROUP_KEY = 'TAG_SOURCE_ATLAS_KAFKA_ENTITIES_GROUP_ID' -TAGSYNC_ATLAS_TO_RANGER_SERVICE_MAPPING = 'ranger.tagsync.atlas.to.service.mapping' + +TAG_SOURCE_ATLAS_KAKFA_SERVICE_NAME_KEY = 'TAG_SOURCE_ATLAS_KAFKA_SERVICE_NAME' +TAG_SOURCE_ATLAS_KAFKA_SECURITY_PROTOCOL_KEY = 'TAG_SOURCE_ATLAS_KAFKA_SECURITY_PROTOCOL' +TAG_SOURCE_ATLAS_KERBEROS_PRINCIPAL_KEY = 'TAG_SOURCE_ATLAS_KERBEROS_PRINCIPAL' +TAG_SOURCE_ATLAS_KERBEROS_KEYTAB_KEY = 'TAG_SOURCE_ATLAS_KERBEROS_KEYTAB' +TAGSYNC_ATLAS_TO_RANGER_SERVICE_MAPPING = 'ranger.tagsync.atlas.to.ranger.service.mapping' TAGSYNC_INSTALL_PROP_PREFIX_FOR_ATLAS_RANGER_MAPPING = 'ranger.tagsync.atlas.' TAGSYNC_ATLAS_CLUSTER_IDENTIFIER = '.instance.' TAGSYNC_INSTALL_PROP_SUFFIX_FOR_ATLAS_RANGER_MAPPING = '.ranger.service' @@ -88,6 +93,7 @@ TAG_SOURCE_FILE_ENABLED = 'ranger.tagsync.source.file' hadoopConfFileName = 'core-site.xml' ENV_HADOOP_CONF_FILE = "ranger-tagsync-env-hadoopconfdir.sh" globalDict = {} +configure_security = False RANGER_TAGSYNC_HOME = os.getenv("RANGER_TAGSYNC_HOME") if RANGER_TAGSYNC_HOME is None: @@ -209,6 +215,9 @@ def convertInstallPropsToXML(props): atlasOutFile = file(atlasOutFn, "w") + atlas_principal = '' + atlas_keytab = '' + for k,v in props.iteritems(): if (k in directKeyMap.keys()): newKey = directKeyMap[k] @@ -218,10 +227,27 @@ def convertInstallPropsToXML(props): atlasOutFile.write(newKey + "=" + v + "\n") elif (k == TAGSYNC_ATLAS_CONSUMER_GROUP_KEY): atlasOutFile.write(newKey + "=" + v + "\n") + elif (configure_security and k == TAG_SOURCE_ATLAS_KAKFA_SERVICE_NAME_KEY): + atlasOutFile.write(newKey + "=" + v + "\n") + elif (configure_security and k == TAG_SOURCE_ATLAS_KAFKA_SECURITY_PROTOCOL_KEY): + atlasOutFile.write(newKey + "=" + v + "\n") + elif (configure_security and k == TAG_SOURCE_ATLAS_KERBEROS_PRINCIPAL_KEY): + atlas_principal = v + elif (configure_security and k == TAG_SOURCE_ATLAS_KERBEROS_KEYTAB_KEY): + atlas_keytab = v else: ret[newKey] = v else: - print "Direct Key not found:%s" % (k) + print "INFO: Direct Key not found:%s" % (k) + + if (configure_security): + atlasOutFile.write("atlas.jaas.KafkaClient.loginModuleName = com.sun.security.auth.module.Krb5LoginModule" + "\n") + atlasOutFile.write("atlas.jaas.KafkaClient.loginModuleControlFlag = required" + "\n") + atlasOutFile.write("atlas.jaas.KafkaClient.option.useKeyTab = true" + "\n") + atlasOutFile.write("atlas.jaas.KafkaClient.option.storeKey = true" + "\n") + atlasOutFile.write("atlas.jaas.KafkaClient.option.serviceName = kafka" + "\n") + atlasOutFile.write("atlas.jaas.KafkaClient.option.keyTab = " + atlas_keytab + "\n") + atlasOutFile.write("atlas.jaas.KafkaClient.option.principal = " + atlas_principal + "\n") atlasOutFile.close() @@ -291,16 +317,27 @@ def initializeInitD(): def write_env_files(exp_var_name, log_path, file_name): final_path = "{0}/{1}".format(confBaseDirName,file_name) if not os.path.isfile(final_path): - print "Creating %s file" % file_name + print "INFO: Creating %s file" % file_name f = open(final_path, "w") f.write("export {0}={1}".format(exp_var_name,log_path)) f.close() def main(): + global configure_security + print "\nINFO: Installing ranger-tagsync .....\n" populate_global_dict() + + + kerberize = globalDict['is_secure'] + if kerberize != "": + kerberize = kerberize.lower() + if kerberize == "true" or kerberize == "enabled" or kerberize == "yes": + configure_security = True + + hadoop_conf = globalDict['hadoop_conf'] dirList = [ rangerBaseDirName, tagsyncBaseDirFullName, confFolderName ] @@ -393,15 +430,28 @@ def main(): if ('ranger.tagsync.dest.ranger.username' not in mergeProps): mergeProps['ranger.tagsync.dest.ranger.username'] = 'rangertagsync' - if (tagsyncKSPath == ''): - mergeProps['ranger.tagsync.dest.ranger.password'] = 'rangertagsync' - - else: + if (tagsyncKSPath != ''): tagadminPasswd = 'rangertagsync' tagadminAlias = 'tagadmin.user.password' updatePropertyInJCKSFile(tagsyncKSPath,tagadminAlias,tagadminPasswd) os.chown(tagsyncKSPath,ownerId,groupId) + tagsyncAtlasKSPath = mergeProps['ranger.tagsync.source.atlasrest.keystore.filename'] + + if ('ranger.tagsync.source.atlasrest.username' not in mergeProps): + mergeProps['ranger.tagsync.source.atlasrest.username'] = 'admin' + + if (tagsyncAtlasKSPath != ''): + if ('ranger.tagsync.source.atlasrest.password' not in mergeProps): + atlasPasswd = 'admin' + else: + atlasPasswd = mergeProps['ranger.tagsync.source.atlasrest.password'] + mergeProps.pop('ranger.tagsync.source.atlasrest.password') + + atlasAlias = 'atlas.user.password' + updatePropertyInJCKSFile(tagsyncAtlasKSPath,atlasAlias,atlasPasswd) + os.chown(tagsyncAtlasKSPath,ownerId,groupId) + writeXMLUsingProperties(fn, mergeProps, outfn) fixPermList = [ ".", tagsyncBaseDirName, confFolderName ] @@ -419,25 +469,26 @@ def main(): os.chown(fn, ownerId, groupId) os.chmod(fn, 0755) - write_env_files("RANGER_TAGSYNC_HADOOP_CONF_DIR", hadoop_conf, ENV_HADOOP_CONF_FILE); + write_env_files("RANGER_TAGSYNC_HADOOP_CONF_DIR", hadoop_conf, ENV_HADOOP_CONF_FILE) os.chown(os.path.join(confBaseDirName, ENV_HADOOP_CONF_FILE),ownerId,groupId) - os.chmod(os.path.join(confBaseDirName, ENV_HADOOP_CONF_FILE),0755) + os.chmod(os.path.join(confBaseDirName, ENV_HADOOP_CONF_FILE),0755) hadoop_conf_full_path = os.path.join(hadoop_conf, hadoopConfFileName) - tagsync_conf_full_path = os.path.join(tagsyncBaseDirFullName,confBaseDirName,hadoopConfFileName) - if not isfile(hadoop_conf_full_path): - print "WARN: core-site.xml file not found in provided hadoop conf path..." + tagsync_conf_full_path = os.path.join(tagsyncBaseDirFullName,confBaseDirName,hadoopConfFileName) + + if not isfile(hadoop_conf_full_path): + print "WARN: core-site.xml file not found in provided hadoop conf path..." f = open(tagsync_conf_full_path, "w") - f.write("<configuration></configuration>") - f.close() + f.write("<configuration></configuration>") + f.close() os.chown(tagsync_conf_full_path,ownerId,groupId) os.chmod(tagsync_conf_full_path,0750) else: - if os.path.islink(tagsync_conf_full_path): - os.remove(tagsync_conf_full_path) + if os.path.islink(tagsync_conf_full_path): + os.remove(tagsync_conf_full_path) - if isfile(hadoop_conf_full_path): - os.symlink(hadoop_conf_full_path, tagsync_conf_full_path) + if isfile(hadoop_conf_full_path) and not isfile(tagsync_conf_full_path): + os.symlink(hadoop_conf_full_path, tagsync_conf_full_path) print "\nINFO: Completed ranger-tagsync installation.....\n" http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ce45eff8/tagsync/scripts/updatetagadminpassword.py ---------------------------------------------------------------------- diff --git a/tagsync/scripts/updatetagadminpassword.py b/tagsync/scripts/updatetagadminpassword.py index d2efef5..0b1f2ca 100644 --- a/tagsync/scripts/updatetagadminpassword.py +++ b/tagsync/scripts/updatetagadminpassword.py @@ -109,32 +109,61 @@ def main(): log("[I] Using Java:" + str(JAVA_BIN),"info") globalDict=import_properties_from_xml(CFG_FILE,globalDict) - TAGSYNC_KEYSTORE_FILENAME=globalDict['ranger.tagsync.keystore.filename'] - log("[I] TAGSYNC_KEYSTORE_FILENAME:" + str(TAGSYNC_KEYSTORE_FILENAME),"info") - TAGSYNC_TAGADMIN_ALIAS="tagadmin.user.password" - TAGSYNC_TAGADMIN_PASSWORD = '' - TAGSYNC_TAGADMIN_USERNAME = '' + + ENDPOINT='' + KEYSTORE_FILENAME='' + KEYSTORE_FILENAME_PROMPT='' + ALIAS = '' + USERNAME='' + PASSWORD='' + USERNAME_PROPERTY_NAME='' + FILENAME_PROPERTY_NAME='' + + while ENDPOINT == "" or not (ENDPOINT == "ATLAS" or ENDPOINT == "RANGER"): + print "Enter Destination NAME (Ranger/Atlas):" + ENDPOINT=raw_input() + ENDPOINT = ENDPOINT.upper() + + if ENDPOINT == "ATLAS": + USERNAME_PROPERTY_NAME='ranger.tagsync.source.atlasrest.username' + FILENAME_PROPERTY_NAME='ranger.tagsync.source.atlasrest.keystore.filename' + ALIAS="atlas.user.password" + KEYSTORE_FILENAME_PROMPT='RANGER_TAGSYNC_ATLAS_KEYSTORE_FILENAME' + + elif ENDPOINT == "RANGER": + USERNAME_PROPERTY_NAME='ranger.tagsync.dest.ranger.username' + FILENAME_PROPERTY_NAME='ranger.tagsync.keystore.filename' + ALIAS="tagadmin.user.password" + KEYSTORE_FILENAME_PROMPT='RANGER_TAGSYNC_RANGER_KEYSTORE_FILENAME' + + else: + log("[E] Unsupported ENDPOINT[" + ENDPOINT + "]") + return + + KEYSTORE_FILENAME = globalDict[FILENAME_PROPERTY_NAME] + + log("[I] " + KEYSTORE_FILENAME_PROMPT + ":" + str(KEYSTORE_FILENAME),"info") unix_user = "ranger" unix_group = "ranger" - while TAGSYNC_TAGADMIN_USERNAME == "": - print "Enter tagadmin user name:" - TAGSYNC_TAGADMIN_USERNAME=raw_input() + while USERNAME == "": + print "Enter " + ENDPOINT + " user name:" + USERNAME=raw_input() - while TAGSYNC_TAGADMIN_PASSWORD == "": - TAGSYNC_TAGADMIN_PASSWORD=getpass.getpass("Enter tagadmin user password:") + while PASSWORD == "": + PASSWORD=getpass.getpass("Enter " + ENDPOINT + " user password:") - if TAGSYNC_KEYSTORE_FILENAME != "" or TAGSYNC_TAGADMIN_USERNAME != "" or TAGSYNC_TAGADMIN_PASSWORD != "": - log("[I] Storing tagadmin tagsync password in credential store:","info") - cmd="%s -cp lib/* org.apache.ranger.credentialapi.buildks create %s -value %s -provider jceks://file%s" %(JAVA_BIN,TAGSYNC_TAGADMIN_ALIAS,TAGSYNC_TAGADMIN_PASSWORD,TAGSYNC_KEYSTORE_FILENAME) + if KEYSTORE_FILENAME != "" or USERNAME != "" or PASSWORD != "": + log("[I] Storing " + ENDPOINT + " tagsync password in credential store:","info") + cmd="%s -cp lib/* org.apache.ranger.credentialapi.buildks create %s -value %s -provider jceks://file%s" %(JAVA_BIN,ALIAS,PASSWORD,KEYSTORE_FILENAME) ret=subprocess.call(shlex.split(cmd)) if ret == 0: - cmd="chown %s:%s %s" %(unix_user,unix_group,TAGSYNC_KEYSTORE_FILENAME) + cmd="chown %s:%s %s" %(unix_user,unix_group,KEYSTORE_FILENAME) ret=subprocess.call(shlex.split(cmd)) if ret == 0: if os.path.isfile(CFG_FILE): - write_properties_to_xml(CFG_FILE,"ranger.tagsync.dest.ranger.username",TAGSYNC_TAGADMIN_USERNAME) - write_properties_to_xml(CFG_FILE,"ranger.tagsync.keystore.filename",TAGSYNC_KEYSTORE_FILENAME) + write_properties_to_xml(CFG_FILE,USERNAME_PROPERTY_NAME,USERNAME) + write_properties_to_xml(CFG_FILE,FILENAME_PROPERTY_NAME,KEYSTORE_FILENAME) else: log("[E] Required file not found: ["+CFG_FILE+"]","error") else: http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ce45eff8/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncConfig.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncConfig.java b/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncConfig.java index 4cb8dea..c52e0d2 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncConfig.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncConfig.java @@ -24,7 +24,11 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.SecureClientLogin; import org.apache.log4j.Logger; -import java.io.*; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; import java.net.URL; import java.net.UnknownHostException; import java.util.Enumeration; @@ -52,10 +56,13 @@ public class TagSyncConfig extends Configuration { private static final String TAGSYNC_SINK_CLASS_PROP = "ranger.tagsync.dest.ranger.impl.class"; private static final String TAGSYNC_DEST_RANGER_PASSWORD_ALIAS = "tagadmin.user.password"; + private static final String TAGSYNC_SOURCE_ATLASREST_PASSWORD_ALIAS = "atlas.user.password"; private static final String TAGSYNC_TAGADMIN_USERNAME_PROP = "ranger.tagsync.dest.ranger.username"; + private static final String TAGSYNC_ATLASREST_USERNAME_PROP = "ranger.tagsync.source.atlasrest.username"; private static final String TAGSYNC_TAGADMIN_PASSWORD_PROP = "ranger.tagsync.dest.ranger.password"; + private static final String TAGSYNC_ATLASREST_PASSWORD_PROP = "ranger.tagsync.source.atlasrest.password"; private static final String TAGSYNC_TAGADMIN_CONNECTION_CHECK_INTERVAL_PROP = "ranger.tagsync.dest.ranger.connection.check.interval"; @@ -65,17 +72,22 @@ public class TagSyncConfig extends Configuration { private static final String TAGSYNC_ATLAS_REST_SOURCE_DOWNLOAD_INTERVAL_PROP = "ranger.tagsync.source.atlasrest.download.interval.millis"; + private static final String TAGSYNC_ATLAS_REST_SSL_CONFIG_FILE_PROP = "ranger.tagsync.source.atlasrest.ssl.config.filename"; + public static final String TAGSYNC_FILESOURCE_FILENAME_PROP = "ranger.tagsync.source.file.filename"; private static final String TAGSYNC_FILESOURCE_MOD_TIME_CHECK_INTERVAL_PROP = "ranger.tagsync.source.file.check.interval.millis"; private static final String TAGSYNC_TAGADMIN_KEYSTORE_PROP = "ranger.tagsync.keystore.filename"; + private static final String TAGSYNC_ATLASREST_KEYSTORE_PROP = "ranger.tagsync.source.atlasrest.keystore.filename"; private static final String DEFAULT_TAGADMIN_USERNAME = "rangertagsync"; private static final String DEFAULT_TAGADMIN_PASSWORD = "rangertagsync"; + private static final String DEFAULT_ATLASREST_USERNAME = "admin"; + private static final String DEFAULT_ATLASREST_PASSWORD = "admin"; private static final int DEFAULT_TAGSYNC_TAGADMIN_CONNECTION_CHECK_INTERVAL = 15000; - private static final long DEFAULT_TAGSYNC_REST_SOURCE_DOWNLOAD_INTERVAL = 900000; + private static final long DEFAULT_TAGSYNC_ATLASREST_SOURCE_DOWNLOAD_INTERVAL = 900000; private static final long DEFAULT_TAGSYNC_FILESOURCE_MOD_TIME_CHECK_INTERVAL = 60000; private static final String AUTH_TYPE = "hadoop.security.authentication"; @@ -210,7 +222,7 @@ public class TagSyncConfig extends Configuration { static public long getTagSourceAtlasDownloadIntervalInMillis(Properties prop) { String val = prop.getProperty(TAGSYNC_ATLAS_REST_SOURCE_DOWNLOAD_INTERVAL_PROP); - long ret = DEFAULT_TAGSYNC_REST_SOURCE_DOWNLOAD_INTERVAL; + long ret = DEFAULT_TAGSYNC_ATLASREST_SOURCE_DOWNLOAD_INTERVAL; if (StringUtils.isNotBlank(val)) { try { ret = Long.valueOf(val); @@ -233,18 +245,6 @@ public class TagSyncConfig extends Configuration { return prop.getProperty(TAGSYNC_TAGADMIN_REST_URL_PROP); } - static public String getTagAdminRESTSslConfigFile(Properties prop) { - return prop.getProperty(TAGSYNC_TAGADMIN_REST_SSL_CONFIG_FILE_PROP); - } - - static public String getTagSourceFileName(Properties prop) { - return prop.getProperty(TAGSYNC_FILESOURCE_FILENAME_PROP); - } - - static public String getAtlasEndpoint(Properties prop) { - return prop.getProperty(TAGSYNC_ATLASSOURCE_ENDPOINT_PROP); - } - static public String getTagAdminPassword(Properties prop) { //update credential from keystore String password = null; @@ -286,8 +286,61 @@ public class TagSyncConfig extends Configuration { return userName; } - static public String getAtlasSslConfigFileName(Properties prop) { - return ""; + static public String getTagAdminRESTSslConfigFile(Properties prop) { + return prop.getProperty(TAGSYNC_TAGADMIN_REST_SSL_CONFIG_FILE_PROP); + } + + static public String getTagSourceFileName(Properties prop) { + return prop.getProperty(TAGSYNC_FILESOURCE_FILENAME_PROP); + } + + static public String getAtlasRESTEndpoint(Properties prop) { + return prop.getProperty(TAGSYNC_ATLASSOURCE_ENDPOINT_PROP); + } + + static public String getAtlasRESTPassword(Properties prop) { + //update credential from keystore + String password = null; + if (prop != null && prop.containsKey(TAGSYNC_ATLASREST_PASSWORD_PROP)) { + password = prop.getProperty(TAGSYNC_ATLASREST_PASSWORD_PROP); + if (password != null && !password.isEmpty()) { + return password; + } + } + if (prop != null && prop.containsKey(TAGSYNC_ATLASREST_KEYSTORE_PROP)) { + String path = prop.getProperty(TAGSYNC_ATLASREST_KEYSTORE_PROP); + if (path != null) { + if (!path.trim().isEmpty()) { + try { + password = CredentialReader.getDecryptedString(path.trim(), TAGSYNC_SOURCE_ATLASREST_PASSWORD_ALIAS); + } catch (Exception ex) { + password = null; + } + if (password != null && !password.trim().isEmpty() && !password.trim().equalsIgnoreCase("none")) { + return password; + } + } + } + } + if(StringUtils.isBlank(password)){ + return DEFAULT_ATLASREST_PASSWORD; + } + return null; + } + + static public String getAtlasRESTUserName(Properties prop) { + String userName=null; + if(prop!=null && prop.containsKey(TAGSYNC_ATLASREST_USERNAME_PROP)){ + userName=prop.getProperty(TAGSYNC_ATLASREST_USERNAME_PROP); + } + if(StringUtils.isBlank(userName)){ + userName=DEFAULT_ATLASREST_USERNAME; + } + return userName; + } + + static public String getAtlasRESTSslConfigFile(Properties prop) { + return prop.getProperty(TAGSYNC_ATLAS_REST_SSL_CONFIG_FILE_PROP); } static public String getCustomAtlasResourceMappers(Properties prop) { @@ -339,7 +392,7 @@ public class TagSyncConfig extends Configuration { readConfigFile(CORE_SITE_FILE); readConfigFile(DEFAULT_CONFIG_FILE); - readConfigFile(CONFIG_FILE); + readConfigFile(CONFIG_FILE); props = getProps(); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ce45eff8/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java index 11ca2d8..3f29de5 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java @@ -29,6 +29,8 @@ import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.security.SecureClientLogin; +import org.apache.ranger.plugin.util.RangerRESTClient; import org.apache.ranger.tagsync.model.AbstractTagSource; import org.apache.ranger.plugin.util.ServiceTags; import org.apache.ranger.tagsync.model.TagSink; @@ -38,14 +40,24 @@ import org.apache.ranger.tagsync.source.atlas.AtlasEntityWithTraits; import org.apache.ranger.tagsync.source.atlas.AtlasNotificationMapper; import org.apache.ranger.tagsync.source.atlas.AtlasResourceMapperUtil; -import java.util.*; +import java.util.List; +import java.util.Map; +import java.util.Properties; public class AtlasRESTTagSource extends AbstractTagSource implements Runnable { private static final Log LOG = LogFactory.getLog(AtlasRESTTagSource.class); - private String atlasEndpoint; + static final String AUTH_TYPE_KERBEROS = "kerberos"; + private long sleepTimeBetweenCycleInMillis; + AtlasRESTUtil atlasRESTUtil = null; + + private String authenticationType; + private String principal; + private String keytab; + private String nameRules; + private Thread myThread = null; public static void main(String[] args) { @@ -90,11 +102,45 @@ public class AtlasRESTTagSource extends AbstractTagSource implements Runnable { boolean ret = AtlasResourceMapperUtil.initializeAtlasResourceMappers(properties); - atlasEndpoint = TagSyncConfig.getAtlasEndpoint(properties); sleepTimeBetweenCycleInMillis = TagSyncConfig.getTagSourceAtlasDownloadIntervalInMillis(properties); - if (StringUtils.isEmpty(atlasEndpoint)) { - LOG.info("No AtlasEndpoint specified, Initial download of Atlas-entities cannot be done."); + String restUrl = TagSyncConfig.getAtlasRESTEndpoint(properties); + String sslConfigFile = TagSyncConfig.getAtlasRESTSslConfigFile(properties); + String userName = TagSyncConfig.getAtlasRESTUserName(properties); + String password = TagSyncConfig.getAtlasRESTPassword(properties); + + authenticationType = TagSyncConfig.getAuthenticationType(properties); + nameRules = TagSyncConfig.getNameRules(properties); + principal = TagSyncConfig.getKerberosPrincipal(properties); + keytab = TagSyncConfig.getKerberosKeytab(properties); + + final boolean kerberized = StringUtils.isNotEmpty(authenticationType) + && authenticationType.trim().equalsIgnoreCase(AtlasRESTTagSource.AUTH_TYPE_KERBEROS) + && SecureClientLogin.isKerberosCredentialExists(principal, keytab); + + if (LOG.isDebugEnabled()) { + LOG.debug("restUrl=" + restUrl); + LOG.debug("sslConfigFile=" + sslConfigFile); + LOG.debug("userName=" + userName); + LOG.debug("authenticationType=" + authenticationType); + LOG.debug("principal=" + principal); + LOG.debug("keytab=" + keytab); + LOG.debug("nameRules=" + nameRules); + LOG.debug("kerberized=" + kerberized); + } + + if (StringUtils.isNotEmpty(restUrl)) { + if (!restUrl.endsWith("/")) { + restUrl += "/"; + } + RangerRESTClient atlasRESTClient = new RangerRESTClient(restUrl, sslConfigFile); + + if (!kerberized) { + atlasRESTClient.setBasicAuthInfo(userName, password); + } + atlasRESTUtil = new AtlasRESTUtil(atlasRESTClient, kerberized, authenticationType, principal, keytab, nameRules); + } else { + LOG.info("AtlasEndpoint not specified, Initial download of Atlas-entities cannot be done."); ret = false; } @@ -148,8 +194,6 @@ public class AtlasRESTTagSource extends AbstractTagSource implements Runnable { public void synchUp() { - AtlasRESTUtil atlasRESTUtil = new AtlasRESTUtil(atlasEndpoint); - List<AtlasEntityWithTraits> atlasEntitiesWithTraits = atlasRESTUtil.getEntitiesWithTraits(); if (CollectionUtils.isNotEmpty(atlasEntitiesWithTraits)) { http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ce45eff8/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTUtil.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTUtil.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTUtil.java index d7f983e..01aaec2 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTUtil.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTUtil.java @@ -28,13 +28,20 @@ import org.apache.atlas.typesystem.json.InstanceSerialization; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.security.SecureClientLogin; import org.apache.log4j.Logger; import org.apache.ranger.admin.client.datatype.RESTResponse; import org.apache.ranger.plugin.util.RangerRESTClient; import org.apache.ranger.tagsync.source.atlas.AtlasEntityWithTraits; import org.apache.ranger.tagsync.source.atlas.AtlasResourceMapperUtil; -import java.util.*; +import javax.security.auth.Subject; +import java.security.PrivilegedAction; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; @SuppressWarnings("unchecked") public class AtlasRESTUtil { @@ -58,23 +65,26 @@ public class AtlasRESTUtil { private final Gson gson = new Gson(); - private RangerRESTClient atlasRESTClient; + private final RangerRESTClient atlasRESTClient; + private final String principal; + private final String keytab; + private final String nameRules; + private final boolean kerberized; - public AtlasRESTUtil(String atlasEndpoint) { + public AtlasRESTUtil(RangerRESTClient atlasRESTClient, boolean kerberized, String authenticationType, String principal, String keytab, String nameRules) { if (LOG.isDebugEnabled()) { - LOG.debug("==> AtlasRESTUtil(" + atlasEndpoint + ")"); + LOG.debug("==> AtlasRESTUtil()"); } - if (!atlasEndpoint.endsWith("/")) { - atlasEndpoint += "/"; - } + this.kerberized = kerberized; - // This uses RangerRESTClient to invoke REST APIs on Atlas. It will work only if scheme of URL is http - atlasRESTClient = new RangerRESTClient(); - atlasRESTClient.setUrl(atlasEndpoint); + this.atlasRESTClient = atlasRESTClient; + this.principal = principal; + this.keytab = keytab; + this.nameRules = nameRules; if (LOG.isDebugEnabled()) { - LOG.debug("<== AtlasRESTUtil(" + atlasEndpoint + ")"); + LOG.debug("<== AtlasRESTUtil()"); } } @@ -231,7 +241,7 @@ public class AtlasRESTUtil { return ret; } - private Map<String, Object> atlasAPI(String endpoint) { + private Map<String, Object> atlasAPI(final String endpoint) { if (LOG.isDebugEnabled()) { LOG.debug("==> atlasAPI(" + endpoint + ")"); @@ -239,7 +249,49 @@ public class AtlasRESTUtil { Map<String, Object> ret = new HashMap<String, Object>(); try { - WebResource webResource = atlasRESTClient.getResource(endpoint); + if (kerberized) { + LOG.debug("Using kerberos authentication"); + Subject sub = SecureClientLogin.loginUserFromKeytab(principal, keytab, nameRules) ; + if(LOG.isDebugEnabled()) { + LOG.debug("Using Principal = "+ principal + ", keytab = "+keytab); + } + ret = Subject.doAs(sub, new PrivilegedAction<Map<String, Object>>() { + @Override + public Map<String, Object> run() { + try{ + return executeAtlasAPI(endpoint); + }catch (Exception e) { + LOG.error("Atlas API failed with message : ", e); + } + return null; + } + }); + } else { + LOG.debug("Using basic authentication"); + ret = executeAtlasAPI(endpoint); + } + } catch (Exception exception) { + LOG.error("Exception when fetching Atlas objects.", exception); + ret = null; + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== atlasAPI(" + endpoint + ")"); + } + return ret; + } + + private Map<String, Object> executeAtlasAPI(final String endpoint) { + + if (LOG.isDebugEnabled()) { + LOG.debug("==> executeAtlasAPI(" + endpoint + ")"); + } + + Map<String, Object> ret = new HashMap<String, Object>(); + + try { + final WebResource webResource = atlasRESTClient.getResource(endpoint); + ClientResponse response = webResource.accept(REST_MIME_TYPE_JSON).type(REST_MIME_TYPE_JSON).get(ClientResponse.class); if (response != null && response.getStatus() == 200) { @@ -255,8 +307,9 @@ public class AtlasRESTUtil { } if (LOG.isDebugEnabled()) { - LOG.debug("<== atlasAPI(" + endpoint + ")"); + LOG.debug("<== executeAtlasAPI(" + endpoint + ")"); } + return ret; } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ce45eff8/tagsync/src/main/resources/ranger-tagsync-site.xml ---------------------------------------------------------------------- diff --git a/tagsync/src/main/resources/ranger-tagsync-site.xml b/tagsync/src/main/resources/ranger-tagsync-site.xml index 30ba3d7..1effe55 100644 --- a/tagsync/src/main/resources/ranger-tagsync-site.xml +++ b/tagsync/src/main/resources/ranger-tagsync-site.xml @@ -70,22 +70,24 @@ <value /> </property> <property> - <name>ranger.tagsync.keystore.filename</name> - <value>/etc/ranger/tagsync/conf/rangertagsync.jceks</value> - </property> - <property> - <name>ranger.tagsync.kerberos.keytab</name> + <name>ranger.tagsync.kerberos.principal</name> <value /> </property> <property> - <name>ranger.tagsync.kerberos.principal</name> + <name>ranger.tagsync.kerberos.keytab</name> <value /> </property> <property> <name>ranger.tagsync.dest.ranger.username</name> <value>rangertagsync</value> </property> - - + <property> + <name>ranger.tagsync.keystore.filename</name> + <value>/etc/ranger/tagsync/conf/rangertagsync.jceks</value> + </property> + <property> + <name>ranger.tagsync.source.atlasrest.keystore.filename</name> + <value>/etc/ranger/tagsync/conf/atlasuser.jceks</value> + </property> </configuration>
