Repository: incubator-ranger Updated Branches: refs/heads/master 853a932b3 -> 46b5ecc33
RANGER-418: add upgrade scripts Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/46b5ecc3 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/46b5ecc3 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/46b5ecc3 Branch: refs/heads/master Commit: 46b5ecc33c02229d4c8721a8d616f2a788fef6f0 Parents: 853a932 Author: sneethiraj <[email protected]> Authored: Tue Apr 21 12:14:50 2015 -0400 Committer: sneethiraj <[email protected]> Committed: Tue Apr 21 14:43:32 2015 -0400 ---------------------------------------------------------------------- agents-common/scripts/upgrade-plugin.py | 160 +++++++++ agents-common/scripts/upgrade-plugin.sh | 29 ++ .../scripts/ranger-admin-site-template.xml | 223 +++++++++++++ security-admin/scripts/upgrade.sh | 33 ++ security-admin/scripts/upgrade_admin.py | 321 +++++++++++++++++++ src/main/assembly/admin-web.xml | 10 + src/main/assembly/hbase-agent.xml | 11 + src/main/assembly/hdfs-agent.xml | 11 + src/main/assembly/hive-agent.xml | 11 + src/main/assembly/knox-agent.xml | 11 + src/main/assembly/storm-agent.xml | 11 + 11 files changed, 831 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/46b5ecc3/agents-common/scripts/upgrade-plugin.py ---------------------------------------------------------------------- diff --git a/agents-common/scripts/upgrade-plugin.py b/agents-common/scripts/upgrade-plugin.py new file mode 100755 index 0000000..9c32dd1 --- /dev/null +++ b/agents-common/scripts/upgrade-plugin.py @@ -0,0 +1,160 @@ +#!/usr/bin/python +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import xml.etree.ElementTree as ET +import os,errno,sys +from os import listdir +from os.path import isfile, join, dirname +from urlparse import urlparse + +debugLevel = 1 + +SUPPORTED_COMPONENTS = [ "hdfs", "hive", "hbase", "knox", "storm" ] + +# +# xmlTemplateDirectory: directory where all of the xml templates are kept here +# + +xmlTemplateDirectory = './install/conf.templates/enable' + +def showUsage(): + print "This script must be run with a <componentName> as parameter" + print "USAGE: upgrade-plugin.py <componentName>" + print " <componentName> could be any one of the following: %s" % (SUPPORTED_COMPONENTS) + +if (len(sys.argv) == 1): + showUsage() + sys.exit(1) + +componentName = sys.argv[1] + +if (componentName not in SUPPORTED_COMPONENTS): + print "Invalid componentName passed as parameter: %s" % (componentName) + showUsage() + sys.exit(1) + +# +# For hdfs, the componentName is hadoop (for path calculation) +# + +if (componentName == 'hdfs'): + configPath = 'hadoop' +else: + configPath = componentName + +# +# configDirectory: where OLD (champlain) configuration exists and NEW (dal) configuration is written to +# +configDirectory = '/etc/' + configPath + '/conf' + + + +def getXMLConfigKeys(xmlFileName): + ret = [] + tree = ET.parse(xmlFileName) + root = tree.getroot() + for config in root.iter('property'): + name = config.find('name').text + ret.append(name) + return ret + +def getXMLConfigMap(xmlFileName): + ret = {} + tree = ET.parse(xmlFileName) + root = tree.getroot() + for config in root.iter('property'): + name = config.find('name').text + val = config.find('value').text + ret[name] = val + return ret + +def writeXMLUsingProperties(xmlTemplateFileName,prop,xmlOutputFileName): + tree = ET.parse(xmlTemplateFileName) + root = tree.getroot() + for config in root.iter('property'): + name = config.find('name').text + if (name in prop): + config.find('value').text = prop[name] + tree.write(xmlOutputFileName) + +def rewriteConfig(props,newProps): + if (debugLevel > 0): + for k,v in props.iteritems(): + print "old config[%s] = [%s]" % (k,v) + # + # Derived fields + # + pmUrl = props['xasecure.' + componentName + '.policymgr.url'] + url = urlparse(pmUrl) + restUrl = url[0] + "://" + url[1] + serviceName = url[2].split("/")[-1] + props['ranger.plugin.' + componentName + '.policy.rest.url'] = restUrl + props['ranger.plugin.' + componentName + '.service.name'] = serviceName + props['ranger.plugin.' + componentName + '.policy.pollIntervalMs'] = props['xasecure.' + componentName + '.policymgr.url.reloadIntervalInMillis'] + #props['ranger.plugin.' + componentName + '.policy.rest.ssl.config.file'] = props['y'] + fileLoc = props['xasecure.' + componentName + '.policymgr.url.laststoredfile'] + props['ranger.plugin.' + componentName + '.policy.cache.dir'] = dirname(fileLoc) + if ( 'xasecure.policymgr.sslconfig.filename' in props ): + props['ranger.plugin.' + componentName + '.policy.rest.ssl.config.file'] = props['xasecure.policymgr.sslconfig.filename'] + else: + sslConfigFileName = join(configDirectory,'ranger-policymgr-ssl.xml') + props['ranger.plugin.' + componentName + '.policy.rest.ssl.config.file'] = sslConfigFileName + # + # Fix for KNOX ssl (missing) configuration + # + if ('xasecure.policymgr.clientssl.keystore.password' not in props): + props['xasecure.policymgr.clientssl.keystore.password'] = 'none' + if ('xasecure.policymgr.clientssl.truststore.password' not in props): + props['xasecure.policymgr.clientssl.truststore.password'] = 'none' + if ('xasecure.policymgr.clientssl.keystore.credential.file' not in props): + props['xasecure.policymgr.clientssl.keystore.credential.file'] = 'jceks://file/tmp/keystore-' + serviceName + '-ssl.jceks' + if ( 'xasecure.policymgr.clientssl.truststore.credential.file' not in props): + props['xasecure.policymgr.clientssl.truststore.credential.file'] = 'jceks://file/tmp/keystore-' + serviceName + '-ssl.jceks' + + for fn in listdir(xmlTemplateDirectory): + file = join(xmlTemplateDirectory,fn) + if isfile(file) and fn.startswith("ranger-") and fn.endswith(".xml") : + newConfigFile = join(configDirectory, fn) + writeXMLUsingProperties(file, props, newConfigFile) + +def main(): + props = {} + newProps = {} + foundFiles = [] + for fn in listdir(configDirectory): + file = join(configDirectory,fn) + if isfile(file) and fn.startswith("xasecure-") and fn.endswith(".xml") : + foundFiles.append(file) + r = getXMLConfigMap(file) + props.update(r) + if (len(foundFiles) == 0): + print "INFO: Previous version of ranger is not enabled/configured for component [%s]" % (componentName) + sys.exit(0) + if (len(foundFiles) != 3): + print "ERROR: Expected to find three files matching xasecure-*.xml files under the folder (%s) - found %s" % (configDirectory,foundFiles) + sys.exit(1) + for fn in listdir(xmlTemplateDirectory): + file = join(xmlTemplateDirectory,fn) + if isfile(file) and fn.startswith("ranger-") and fn.endswith(".xml") : + r = getXMLConfigMap(file) + newProps.update(r) + newConfigFile = join(configDirectory,fn) + if isfile(newConfigFile): + print "ERROR: new config file [%s] already exists. Upgrade script can not overwrite an existing config file." % (newConfigFile) + sys.exit(1) + rewriteConfig(props,newProps) + +main() http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/46b5ecc3/agents-common/scripts/upgrade-plugin.sh ---------------------------------------------------------------------- diff --git a/agents-common/scripts/upgrade-plugin.sh b/agents-common/scripts/upgrade-plugin.sh new file mode 100755 index 0000000..8a3d7ab --- /dev/null +++ b/agents-common/scripts/upgrade-plugin.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +COMPONENT_NAME=`basename $0 | cut -d. -f1 | sed -e 's:^upgrade-::'` + +echo "${COMPONENT_NAME}" | grep -- '-plugin' > /dev/null 2>&1 + +if [ $? -ne 0 ] +then + echo "$0 : is not applicable for component [${COMPONENT_NAME}]. It is applicable only for ranger plugin component; Exiting ..." + exit 0 +fi + +HCOMPONENT_NAME=`echo ${COMPONENT_NAME} | sed -e 's:-plugin::'` + +./upgrade-plugin.py "${HCOMPONENT_NAME}" http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/46b5ecc3/security-admin/scripts/ranger-admin-site-template.xml ---------------------------------------------------------------------- diff --git a/security-admin/scripts/ranger-admin-site-template.xml b/security-admin/scripts/ranger-admin-site-template.xml new file mode 100644 index 0000000..2c0462d --- /dev/null +++ b/security-admin/scripts/ranger-admin-site-template.xml @@ -0,0 +1,223 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration> + <property> + <name>ranger.service.host</name> + <value></value> + </property> + <property> + <name>ranger.service.http.enabled</name> + <value></value> + </property> + <property> + <name>ranger.service.http.port</name> + <value></value> + </property> + <property> + <name>ranger.service.shutdown.port</name> + <value></value> + </property> + <property> + <name>ranger.service.shutdown.command</name> + <value></value> + </property> + <property> + <name>ranger.service.https.port</name> + <value></value> + </property> + <property> + <name>ranger.service.https.attrib.ssl.enabled</name> + <value></value> + </property> + <property> + <name>ranger.service.https.attrib.ssl.protocol</name> + <value></value> + </property> + <property> + <name>ranger.service.https.attrib.client.auth</name> + <value></value> + </property> + <property> + <name>ranger.service.https.attrib.keystore.keyalias</name> + <value></value> + </property> + <property> + <name>ranger.service.https.attrib.keystore.pass</name> + <value></value> + </property> + <property> + <name>ranger.https.attrib.keystore.file</name> + <value></value> + </property> + <property> + <name>ranger.accesslog.dateformat</name> + <value></value> + </property> + <property> + <name>ranger.accesslog.pattern</name> + <value></value> + </property> + <property> + <name>ranger.externalurl</name> + <value></value> + </property> + <property> + <name>ranger.contextName</name> + <value></value> + </property> + <property> + <name>ranger.jpa.showsql</name> + <value></value> + </property> + <property> + <name>ranger.env.local</name> + <value></value> + </property> + <property> + <name>ranger.jpa.jdbc.dialect</name> + <value></value> + </property> + <property> + <name>ranger.jpa.jdbc.driver</name> + <value></value> + </property> + <property> + <name>ranger.jpa.jdbc.url</name> + <value></value> + </property> + <property> + <name>ranger.jpa.jdbc.user</name> + <value></value> + </property> + <property> + <name>ranger.jpa.jdbc.password</name> + <value></value> + </property> + <property> + <name>ranger.jpa.jdbc.maxpoolsize</name> + <value></value> + </property> + <property> + <name>ranger.jpa.jdbc.minpoolsize</name> + <value></value> + </property> + <property> + <name>ranger.jpa.jdbc.initialpoolsize</name> + <value></value> + </property> + <property> + <name>ranger.jpa.jdbc.maxidletime</name> + <value></value> + </property> + <property> + <name>ranger.jpa.jdbc.maxstatements</name> + <value></value> + </property> + <property> + <name>ranger.jpa.jdbc.preferredtestquery</name> + <value></value> + </property> + <property> + <name>ranger.jpa.jdbc.idleconnectiontestperiod</name> + <value></value> + </property> + <property> + <name>ranger.jpa.jdbc.credential.alias</name> + <value></value> + </property> + <property> + <name>ranger.jpa.jdbc.credential.provider.path</name> + <value></value> + </property> + <property> + <name>ranger.logs.base.dir</name> + <value></value> + </property> + <property> + <name>ranger.scheduler.enabled</name> + <value></value> + </property> + <property> + <name>ranger.audit.source.type</name> + <value></value> + </property> + <property> + <name>ranger.solr.url</name> + <value></value> + </property> + <property> + <name>ranger.jpa.audit.jdbc.dialect</name> + <value></value> + </property> + <property> + <name>ranger.jpa.audit.jdbc.driver</name> + <value></value> + </property> + <property> + <name>ranger.jpa.audit.jdbc.url</name> + <value></value> + </property> + <property> + <name>ranger.jpa.audit.jdbc.user</name> + <value></value> + </property> + <property> + <name>ranger.jpa.audit.jdbc.password</name> + <value></value> + </property> + <property> + <name>ranger.jpa.audit.jdbc.credential.alias</name> + <value></value> + </property> + <property> + <name>ranger.jpa.audit.jdbc.credential.provider.path</name> + <value></value> + </property> + <property> + <name>ranger.authentication.method</name> + <value></value> + </property> + <property> + <name>ranger.ldap.url</name> + <value></value> + </property> + <property> + <name>ranger.ldap.user.dnpattern</name> + <value></value> + </property> + <property> + <name>ranger.ldap.group.searchbase</name> + <value></value> + </property> + <property> + <name>ranger.ldap.group.searchfilter</name> + <value></value> + </property> + <property> + <name>ranger.ldap.group.roleattribute</name> + <value></value> + </property> + <property> + <name>ranger.ldap.ad.domain</name> + <value></value> + </property> + <property> + <name>ranger.ldap.ad.url</name> + <value></value> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/46b5ecc3/security-admin/scripts/upgrade.sh ---------------------------------------------------------------------- diff --git a/security-admin/scripts/upgrade.sh b/security-admin/scripts/upgrade.sh new file mode 100644 index 0000000..a467298 --- /dev/null +++ b/security-admin/scripts/upgrade.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ------------------------------------------------------------------------------------- +# +# Ranger Admin Upgrade Script +# +# This script will generate install configuration based on the current installation and run setup to upgrade schema + +./upgrade_admin.py +if [ $? -eq 0 ] +then + trap 'rm -f ./install.properties ; exit 1' 2 3 15 + ./setup.sh + ec=$? + rm -f ./install.properties + exit $ec +else + echo "ERROR: unable to complete upgrade-admin.py" + exit 1 +fi http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/46b5ecc3/security-admin/scripts/upgrade_admin.py ---------------------------------------------------------------------- diff --git a/security-admin/scripts/upgrade_admin.py b/security-admin/scripts/upgrade_admin.py new file mode 100755 index 0000000..9c2f2dc --- /dev/null +++ b/security-admin/scripts/upgrade_admin.py @@ -0,0 +1,321 @@ +#!/usr/bin/python +import re +import StringIO +import xml.etree.ElementTree as ET +import ConfigParser +import os,errno,sys,getopt +from os import listdir +from os.path import isfile, join, dirname, basename +from urlparse import urlparse +from time import gmtime, strftime, localtime +import shutil + +debugLevel = 1 +generateXML = 0 +installPropFileName = 'install.properties' + +tempLibFolder = "./upgrade-temp" + +def showUsage(): + print "upgrade_admin.py [-g] [-h]" + print "This script will generate %s based on currently installed ranger (v0.4.*) configuration." % (installPropFileName) + print " -g option will generate ranger-admin-site.xml in the current directory." + print " -h will display help text." + +try: + opts, args = getopt.getopt(sys.argv[1:],"gh") +except getopt.GetoptError: + showUsage() + sys.exit(2) +for opt,arg in opts: + if (opt == '-g'): + generateXML = 1 + elif (opt == '-h'): + showUsage() + sys.exit(0) +# +# configDirectory: where OLD (champlain) configuration exists and NEW (dal) configuration is written to +# +configDirectory = '/etc/ranger/admin/conf' +rangerJAASDirectoryName = join(configDirectory,'ranger_jaas') + +xaSystemPropFile = 'xa_system.properties' +ldapPropFile = 'xa_ldap.properties' +rangerJAASPropFile = 'unixauth.properties' +securityContextFile = 'security-applicationContext.xml' +webserverConfigFile = 'ranger_webserver.properties' + +rangerSiteXMLFile = "ranger-admin-site.xml" + + +# +# xmlTemplateDirectory: directory where all of the xml templates are kept here +# +templateDirectoryName = './templates-upgrade' +rangerSiteTemplateXMLFile = "ranger-admin-site-template.xml" + +# +# Install Properties To Ranger Properties +# +config2xmlMAP = { + 'service.host':'ranger.service.host', + 'http.enabled':'ranger.service.http.enabled', + 'http.service.port':'ranger.service.http.port', + 'service.shutdownPort':'ranger.service.shutdown.port', + 'service.shutdownCommand':'ranger.service.shutdown.command', + 'https.service.port':'ranger.service.https.port', + 'https.attrib.SSLEnabled':'ranger.service.https.attrib.ssl.enabled', + 'https.attrib.sslProtocol':'ranger.service.https.attrib.ssl.protocol', + 'https.attrib.clientAuth':'ranger.service.https.attrib.client.auth', + 'https.attrib.keyAlias':'ranger.service.https.attrib.keystore.keyalias', + 'https.attrib.keystorePass':'ranger.service.https.attrib.keystore.pass', + 'https.attrib.keystoreFile':'ranger.https.attrib.keystore.file', + 'accesslog.dateformat':'ranger.accesslog.dateformat', + 'accesslog.pattern':'ranger.accesslog.pattern', + 'xa.webapp.url.root':'ranger.externalurl', + 'xa.webapp.contextName':'ranger.contextName', + 'xa.jpa.showsql':'ranger.jpa.showsql', + 'xa.env.local':'ranger.env.local', + 'jdbc.dialect':'ranger.jpa.jdbc.dialect', + 'jdbc.driver':'ranger.jpa.jdbc.driver', + 'jdbc.url':'ranger.jpa.jdbc.url', + 'jdbc.user':'ranger.jpa.jdbc.user', + 'jdbc.password':'ranger.jpa.jdbc.password', + 'jdbc.maxPoolSize':'ranger.jpa.jdbc.maxpoolsize', + 'jdbc.minPoolSize':'ranger.jpa.jdbc.minpoolsize', + 'jdbc.initialPoolSize':'ranger.jpa.jdbc.initialpoolsize', + 'jdbc.maxIdleTime':'ranger.jpa.jdbc.maxidletime', + 'jdbc.maxStatements':'ranger.jpa.jdbc.maxstatements', + 'jdbc.preferredTestQuery':'ranger.jpa.jdbc.preferredtestquery', + 'jdbc.idleConnectionTestPeriod':'ranger.jpa.jdbc.idleconnectiontestperiod', + 'xaDB.jdbc.credential.alias':'ranger.jpa.jdbc.credential.alias', + 'xaDB.jdbc.credential.provider.path':'ranger.jpa.jdbc.credential.provider.path', + 'xa.logs.base.dir':'ranger.logs.base.dir', + 'xa.scheduler.enabled':'ranger.scheduler.enabled', + 'xa.audit.store':'ranger.audit.source.type', + 'audit_solr_url':'ranger.solr.url', + 'auditDB.jdbc.dialect':'ranger.jpa.audit.jdbc.dialect', + 'auditDB.jdbc.driver':'ranger.jpa.audit.jdbc.driver', + 'auditDB.jdbc.url':'ranger.jpa.audit.jdbc.url', + 'auditDB.jdbc.user':'ranger.jpa.audit.jdbc.user', + 'auditDB.jdbc.password':'ranger.jpa.audit.jdbc.password', + 'auditDB.jdbc.credential.alias':'ranger.jpa.audit.jdbc.credential.alias', + 'auditDB.jdbc.credential.provider.path':'ranger.jpa.audit.jdbc.credential.provider.path', + 'authentication_method':'ranger.authentication.method', + 'xa_ldap_url':'ranger.ldap.url', + 'xa_ldap_userDNpattern':'ranger.ldap.user.dnpattern', + 'xa_ldap_groupSearchBase':'ranger.ldap.group.searchbase', + 'xa_ldap_groupSearchFilter':'ranger.ldap.group.searchfilter', + 'xa_ldap_groupRoleAttribute':'ranger.ldap.group.roleattribute', + 'xa_ldap_ad_domain':'ranger.ldap.ad.domain', + 'xa_ldap_ad_url':'ranger.ldap.ad.url' } + +def archiveFile(originalFileName): + archiveDir = dirname(originalFileName) + archiveFileName = "." + basename(originalFileName) + "." + (strftime("%d%m%Y%H%M%S", localtime())) + movedFileName = join(archiveDir,archiveFileName) + print "INFO: moving [%s] to [%s] ......." % (originalFileName,movedFileName) + os.rename(originalFileName, movedFileName) + +def getPropertiesConfigMap(configFileName): + ret = {} + config = StringIO.StringIO() + config.write('[dummysection]\n') + config.write(open(configFileName).read()) + config.seek(0,os.SEEK_SET) + fcp = ConfigParser.ConfigParser() + fcp.optionxform = str + fcp.readfp(config) + for k,v in fcp.items('dummysection'): + ret[k] = v + return ret + +def getPropertiesKeyList(configFileName): + ret = [] + config = StringIO.StringIO() + config.write('[dummysection]\n') + config.write(open(configFileName).read()) + config.seek(0,os.SEEK_SET) + fcp = ConfigParser.ConfigParser() + fcp.optionxform = str + fcp.readfp(config) + for k,v in fcp.items('dummysection'): + ret.append(k) + return ret + +def readFromJCKSFile(jcksFileName,propName): + fn = jcksFileName + cmd = "java -cp './cred/lib/*' org.apache.ranger.credentialapi.buildks get '" + propName + "' -provider jceks://file" + fn + " 2> /dev/null" + pwd = os.popen(cmd).read() + pwd = pwd.strip() + return pwd + +def writeXMLUsingProperties(xmlTemplateFileName,prop,xmlOutputFileName): + tree = ET.parse(xmlTemplateFileName) + root = tree.getroot() + for config in root.iter('property'): + name = config.find('name').text + if (name in prop.keys()): + config.find('value').text = prop[name] + else: + print "ERROR: key not found: %s" % (name) + if isfile(xmlOutputFileName): + archiveFile(xmlOutputFileName) + tree.write(xmlOutputFileName) + +def main(): + installFileName = join(templateDirectoryName, installPropFileName) + installProps = {} + rangerprops = {} + + xaSystemPropFileName = join(configDirectory, xaSystemPropFile) + xaSysProps = getPropertiesConfigMap(xaSystemPropFileName) + + ldapPropFileName = join(configDirectory, ldapPropFile) + xaLdapProps = getPropertiesConfigMap (ldapPropFileName) + + jaasPropFileName = join(rangerJAASDirectoryName, rangerJAASPropFile) + unixauthProps = getPropertiesConfigMap (jaasPropFileName) + + webserverConfigFileName = join(configDirectory, webserverConfigFile) + webconfig = getPropertiesConfigMap(webserverConfigFileName) + + for k in config2xmlMAP.keys(): + xmlKey = config2xmlMAP[k] + if (k in xaSysProps.keys()): + xmlVal = xaSysProps[k] + elif (k in xaLdapProps.keys()): + xmlVal = xaLdapProps[k] + elif (k in unixauthProps.keys()): + xmlVal = unixauthProps[k] + elif (k in webconfig.keys()): + xmlVal = webconfig[k] + else: + xmlVal = 'Unknown' + rangerprops[xmlKey] = xmlVal + + jdbcUrl = xaSysProps['jdbc.url'] + auditJcksFileName = xaSysProps['auditDB.jdbc.credential.provider.path'] + jcksFileName = xaSysProps['xaDB.jdbc.credential.provider.path'] + + + tokens = jdbcUrl.split(":") + hostTokens = jdbcUrl.split("//") + dbTokens = hostTokens[1].split("/") + + libFolderCmd='dirname `readlink -f /usr/bin/ranger-admin`' + libFolder = os.popen(libFolderCmd).read().strip() + '/webapp/WEB-INF/lib' + + if (tokens[2] == 'mysql'): + installProps['DB_FLAVOR'] = 'MYSQL' + installProps['SQL_COMMAND_INVOKER'] = 'mysql' + installProps['db_host'] = dbTokens[0] + installProps['db_name'] = dbTokens[1] + installProps['audit_db_name'] = dbTokens[1] + mysqlConnectorJarFileName = [ f for f in listdir(libFolder) if (isfile(join(libFolder,f)) and f.startswith("mysql") and f.endswith(".jar")) ] + if (len(mysqlConnectorJarFileName) > 0): + if not os.path.exists(tempLibFolder): + os.makedirs(tempLibFolder) + tempLibFile=join(tempLibFolder,mysqlConnectorJarFileName[0]) + shutil.copy(join(libFolder,mysqlConnectorJarFileName[0]), tempLibFile) + installProps['SQL_CONNECTOR_JAR'] = tempLibFile + elif (token[3] == 'odbc'): + installProps['DB_FLAVOR'] = 'ORACLE' + installProps['SQL_COMMAND_INVOKER'] = 'sqlplus' + installProps['db_host'] = dbTokens[0] + oraConnectorJarFileName = [ f for f in listdir(libFolder) if (isfile(join(libFolder,f)) and f.startswith("ojdbc") and f.endswith(".jar")) ] + if (len(oraConnectorJarFileName) > 0): + if not os.path.exists(tempLibFolder): + os.makedirs(tempLibFolder) + tempLibFile=join(tempLibFolder,oraConnectorJarFileName[0]) + shutil.copy(join(libFolder,oraConnectorJarFileName[0]), tempLibFile) + installProps['SQL_CONNECTOR_JAR'] = tempLibFile + # + # TODO: for oracle, need to find out as how to get these values + # + installProps['db_name'] = '' + installProps['audit_db_name'] = '' + else: + print "ERROR: Unable to determine the DB_FLAVOR from url [%]" % (jdbcUrl) + sys.exit(1) + + installProps['db_user'] = xaSysProps['jdbc.user'] + installProps['db_password'] = readFromJCKSFile(jcksFileName, 'policyDB.jdbc.password') + installProps['db_root_user'] = 'unknown' + installProps['db_root_password'] = 'unknown' + + installProps['audit_db_user']=xaSysProps['auditDB.jdbc.user'] + installProps['audit_db_password']= readFromJCKSFile(auditJcksFileName, 'auditDB.jdbc.password') + + installProps['policymgr_external_url'] = xaSysProps['xa.webapp.url.root'] + installProps['policymgr_http_enabled'] = xaSysProps['http.enabled'] + + securityContextFileName = join(configDirectory, securityContextFile) + tree = ET.parse(securityContextFileName) + root = tree.getroot() + ns = {'beans' : 'http://www.springframework.org/schema/beans'} + if ( len(root.findall(".//*[@id='activeDirectoryAuthenticationProvider']",ns)) > 0): + installProps['authentication_method'] = 'AD' + installProps['xa_ldap_ad_domain'] = xaLdapProps['xa_ldap_ad_domain'] + installProps['xa_ldap_ad_url'] = xaLdapProps['xa_ldap_ad_url'] + elif ( len(root.findall(".//*[@id='ldapAuthProvider']",ns)) > 0 ): + installProps['authentication_method'] = 'LDAP' + installProps['xa_ldap_url'] = xaLdapProps['xa_ldap_url'] + installProps['xa_ldap_userDNpattern'] = xaLdapProps['xa_ldap_userDNpattern'] + installProps['xa_ldap_groupSearchBase'] = xaLdapProps['xa_ldap_groupSearchBase'] + installProps['xa_ldap_groupSearchFilter'] = xaLdapProps['xa_ldap_groupSearchFilter'] + installProps['xa_ldap_groupRoleAttribute'] = xaLdapProps['xa_ldap_groupRoleAttribute'] + elif ( len(root.findall(".//*[@id='jaasAuthProvider']",ns)) > 0 ): + installProps['authentication_method'] = 'UNIX' + installProps['remoteLoginEnabled'] = unixauthProps['remoteLoginEnabled'] + installProps['authServiceHostName'] = unixauthProps['authServiceHostName'] + installProps['authServicePort'] = unixauthProps['authServicePort'] + else: + installProps['authentication_method'] = 'NONE' + + rangerprops['ranger.authentication.method'] = installProps['authentication_method'] + + installProps['cred_keystore_filename'] = jcksFileName + + keylist = getPropertiesKeyList(installFileName) + defValMap = getPropertiesConfigMap(installFileName) + + + for wk,wv in webconfig.iteritems(): + nk = "ranger." + wk + nk = nk.replace('.','_') + installProps[nk] = wv + keylist.append(nk) + + writeToFile(keylist,defValMap,installProps,installPropFileName) + + if (generateXML == 1): + writeXMLUsingProperties(join(templateDirectoryName,rangerSiteTemplateXMLFile), rangerprops, rangerSiteXMLFile) + +def writeToFile(keyList, defValMap, props, outFileName): + + if (isfile(outFileName)): + archiveFile(outFileName) + + outf = open(outFileName, 'w') + + print >> outf, "#" + print >> outf, "# -----------------------------------------------------------------------------------" + print >> outf, "# This file is generated as part of upgrade script and should be deleted after upgrade" + print >> outf, "# Generated at %s " % (strftime("%d/%m/%Y %H:%M:%S", localtime())) + print >> outf, "# -----------------------------------------------------------------------------------" + print >> outf, "#" + + for key in keyList: + if (key in props): + print >> outf, "%s=%s" % (key,props[key]) + val = props[key] + else: + print >> outf, "# Default value for [%s] is used\n%s=%s\n#---" % (key, key,defValMap[key]) + val = defValMap[key] + + outf.flush() + outf.close() + + +main() http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/46b5ecc3/src/main/assembly/admin-web.xml ---------------------------------------------------------------------- diff --git a/src/main/assembly/admin-web.xml b/src/main/assembly/admin-web.xml index 6e5b3c7..f984248 100644 --- a/src/main/assembly/admin-web.xml +++ b/src/main/assembly/admin-web.xml @@ -314,10 +314,20 @@ <include>db_setup.py</include> <include>dba_script.py</include> <include>restrict_permissions.py</include> + <include>upgrade_admin.py</include> + <include>upgrade.sh</include> </includes> <fileMode>544</fileMode> </fileSet> <fileSet> + <outputDirectory>/templates-upgrade</outputDirectory> + <directory>security-admin/scripts</directory> + <includes> + <include>install.properties</include> + <include>ranger-admin-site-template.xml</include> + </includes> + </fileSet> + <fileSet> <outputDirectory>/</outputDirectory> <directory>security-admin</directory> <includes> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/46b5ecc3/src/main/assembly/hbase-agent.xml ---------------------------------------------------------------------- diff --git a/src/main/assembly/hbase-agent.xml b/src/main/assembly/hbase-agent.xml index 62cef98..851ef6f 100644 --- a/src/main/assembly/hbase-agent.xml +++ b/src/main/assembly/hbase-agent.xml @@ -125,6 +125,17 @@ <fileMode>755</fileMode> </file> <file> + <source>agents-common/scripts/upgrade-plugin.sh</source> + <outputDirectory>/</outputDirectory> + <destName>upgrade-hbase-plugin.sh</destName> + <fileMode>755</fileMode> + </file> + <file> + <source>agents-common/scripts/upgrade-plugin.py</source> + <outputDirectory>/</outputDirectory> + <fileMode>755</fileMode> + </file> + <file> <source>agents-common/scripts/enable-agent.sh</source> <outputDirectory>/</outputDirectory> <destName>disable-hbase-plugin.sh</destName> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/46b5ecc3/src/main/assembly/hdfs-agent.xml ---------------------------------------------------------------------- diff --git a/src/main/assembly/hdfs-agent.xml b/src/main/assembly/hdfs-agent.xml index 593aedb..036c2bd 100644 --- a/src/main/assembly/hdfs-agent.xml +++ b/src/main/assembly/hdfs-agent.xml @@ -115,6 +115,17 @@ <fileMode>755</fileMode> </file> <file> + <source>agents-common/scripts/upgrade-plugin.sh</source> + <outputDirectory>/</outputDirectory> + <destName>upgrade-hdfs-plugin.sh</destName> + <fileMode>755</fileMode> + </file> + <file> + <source>agents-common/scripts/upgrade-plugin.py</source> + <outputDirectory>/</outputDirectory> + <fileMode>755</fileMode> + </file> + <file> <source>agents-common/scripts/enable-agent.sh</source> <outputDirectory>/</outputDirectory> <destName>disable-hdfs-plugin.sh</destName> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/46b5ecc3/src/main/assembly/hive-agent.xml ---------------------------------------------------------------------- diff --git a/src/main/assembly/hive-agent.xml b/src/main/assembly/hive-agent.xml index 6137110..c48b1b5 100644 --- a/src/main/assembly/hive-agent.xml +++ b/src/main/assembly/hive-agent.xml @@ -125,6 +125,17 @@ <fileMode>755</fileMode> </file> <file> + <source>agents-common/scripts/upgrade-plugin.sh</source> + <outputDirectory>/</outputDirectory> + <destName>upgrade-hive-plugin.sh</destName> + <fileMode>755</fileMode> + </file> + <file> + <source>agents-common/scripts/upgrade-plugin.py</source> + <outputDirectory>/</outputDirectory> + <fileMode>755</fileMode> + </file> + <file> <source>agents-common/scripts/enable-agent.sh</source> <outputDirectory>/</outputDirectory> <destName>disable-hive-plugin.sh</destName> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/46b5ecc3/src/main/assembly/knox-agent.xml ---------------------------------------------------------------------- diff --git a/src/main/assembly/knox-agent.xml b/src/main/assembly/knox-agent.xml index 10d3eeb..6552406 100644 --- a/src/main/assembly/knox-agent.xml +++ b/src/main/assembly/knox-agent.xml @@ -131,6 +131,17 @@ <fileMode>755</fileMode> </file> <file> + <source>agents-common/scripts/upgrade-plugin.sh</source> + <outputDirectory>/</outputDirectory> + <destName>upgrade-knox-plugin.sh</destName> + <fileMode>755</fileMode> + </file> + <file> + <source>agents-common/scripts/upgrade-plugin.py</source> + <outputDirectory>/</outputDirectory> + <fileMode>755</fileMode> + </file> + <file> <source>agents-common/scripts/enable-agent.sh</source> <outputDirectory>/</outputDirectory> <destName>disable-knox-plugin.sh</destName> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/46b5ecc3/src/main/assembly/storm-agent.xml ---------------------------------------------------------------------- diff --git a/src/main/assembly/storm-agent.xml b/src/main/assembly/storm-agent.xml index 5f32128..998b957 100644 --- a/src/main/assembly/storm-agent.xml +++ b/src/main/assembly/storm-agent.xml @@ -139,6 +139,17 @@ <fileMode>755</fileMode> </file> <file> + <source>agents-common/scripts/upgrade-plugin.sh</source> + <outputDirectory>/</outputDirectory> + <destName>upgrade-storm-plugin.sh</destName> + <fileMode>755</fileMode> + </file> + <file> + <source>agents-common/scripts/upgrade-plugin.py</source> + <outputDirectory>/</outputDirectory> + <fileMode>755</fileMode> + </file> + <file> <source>agents-common/scripts/enable-agent.sh</source> <outputDirectory>/</outputDirectory> <destName>disable-storm-plugin.sh</destName>
