Repository: incubator-ranger Updated Branches: refs/heads/master 60ec33789 -> 917833cd3
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/917833cd/kms/scripts/setup.sh ---------------------------------------------------------------------- diff --git a/kms/scripts/setup.sh b/kms/scripts/setup.sh index fa88bba..fe9f60c 100644 --- a/kms/scripts/setup.sh +++ b/kms/scripts/setup.sh @@ -15,26 +15,21 @@ # limitations under the License. # ------------------------------------------------------------------------------------- # -# Ranger KMS Setup Script +# Ranger Admin Setup Script # -# This script will install ranger kms webapplication under tomcat and also, initialize the database with ranger kms users/tables. +# This script will install policymanager webapplication under tomcat and also, initialize the database with ranger users/tables. -realScriptPath=`readlink -f $0` -realScriptDir=`dirname $realScriptPath` -RANGER_KMS_DIR=`(cd $realScriptDir/..; pwd)` -RANGER_KMS_EWS_DIR=${RANGER_KMS_DIR}/ews -RANGER_KMS_EWS_CONF_DIR="${RANGER_KMS_EWS_DIR}/conf" -RANGER_KMS_EWS_LIB_DIR="${RANGER_KMS_EWS_DIR}/lib" - -PROPFILE=${RANGER_KMS_EWS_CONF_DIR}/kms_webserver.properties +PROPFILE=$PWD/install.properties propertyValue='' -. $PROPFILE 1>/dev/null 2>&1 +. $PROPFILE if [ ! $? = "0" ];then log "$PROPFILE file not found....!!"; exit 1; fi +DB_HOST="${db_host}" + usage() { [ "$*" ] && echo "$0: $*" sed -n '/^##/,/^$/s/^## \{0,1\}//p' "$0" @@ -42,7 +37,7 @@ usage() { } 2>/dev/null log() { - local prefix="[$(date +%Y/%m/%d\ %H:%M:%S)]: " + local prefix="$(date +%Y-%m-%d\ %H:%M:%S,%3N) " echo "${prefix} $@" >> $LOGFILE echo "${prefix} $@" } @@ -54,6 +49,14 @@ check_ret_status(){ fi } +check_ret_status_for_groupadd(){ +# 9 is the response if the group exists + if [ $1 -ne 0 ] && [ $1 -ne 9 ]; then + log "[E] $2"; + exit 1; + fi +} + is_command () { log "[I] check if command $1 exists" type "$1" >/dev/null @@ -71,6 +74,53 @@ get_distro(){ log "[I] Found distribution : $DIST_NAME" } +#Get Properties from File without erroring out if property is not there +#$1 -> propertyName $2 -> fileName $3 -> variableName $4 -> failIfNotFound +getPropertyFromFileNoExit(){ + validateProperty=$(sed '/^\#/d' $2 | grep "^$1" | tail -n 1) # for validation + if test -z "$validateProperty" ; then + log "[E] '$1' not found in $2 file while getting....!!"; + if [ $4 == "true" ] ; then + exit 1; + else + value="" + fi + else + value=`sed '/^\#/d' $2 | grep "^$1" | tail -n 1 | cut -d "=" -f2-` + fi + #echo 'value:'$value + eval $3="'$value'" +} +#Get Properties from File +#$1 -> propertyName $2 -> fileName $3 -> variableName +getPropertyFromFile(){ + validateProperty=$(sed '/^\#/d' $2 | grep "^$1" | tail -n 1) # for validation + if test -z "$validateProperty" ; then log "[E] '$1' not found in $2 file while getting....!!"; exit 1; fi + value=`sed '/^\#/d' $2 | grep "^$1" | tail -n 1 | cut -d "=" -f2-` + #echo 'value:'$value + #validate=$(sed '/^\#/d' $2 | grep "^$1" | tail -n 1 | cut -d "=" -f2-) # for validation + #if test -z "$validate" ; then log "[E] '$1' not found in $2 file while getting....!!"; exit 1; fi + eval $3="'$value'" +} + +#Update Properties to File +#$1 -> propertyName $2 -> newPropertyValue $3 -> fileName +updatePropertyToFile(){ + sed -i 's@^'$1'=[^ ]*$@'$1'='$2'@g' $3 + #validate=`sed -i 's/^'$1'=[^ ]*$/'$1'='$2'/g' $3` #for validation + validate=$(sed '/^\#/d' $3 | grep "^$1" | tail -n 1 | cut -d "=" -f2-) # for validation + #echo 'V1:'$validate + if test -z "$validate" ; then log "[E] '$1' not found in $3 file while Updating....!!"; exit 1; fi + log "[I] File $3 Updated successfully : {'$1'}" +} + +#Update Properties to File +#$1 -> propertyName $2 -> newPropertyValue $3 -> fileName +updatePropertyToFilePy(){ + python update_property.py $1 $2 $3 + check_ret_status $? "Update property failed for: " $1 +} + init_logfiles () { for f in $LOGFILES; do @@ -81,7 +131,17 @@ init_logfiles () { init_variables(){ curDt=`date '+%Y%m%d%H%M%S'` - INSTALL_DIR=${RANGER_KMS_DIR} + VERSION=`cat ${PWD}/version` + + KMS_DIR=$PWD + + RANGER_KMS_INITD=kms-initd + + RANGER_KMS=ranger-kms + + INSTALL_DIR=${KMS_DIR} + + WEBAPP_ROOT=${INSTALL_DIR}/ews/webapp DB_FLAVOR=`echo $DB_FLAVOR | tr '[:lower:]' '[:upper:]'` if [ "${DB_FLAVOR}" == "" ] @@ -89,8 +149,14 @@ init_variables(){ DB_FLAVOR="MYSQL" fi log "[I] DB_FLAVOR=${DB_FLAVOR}" + + getPropertyFromFile 'db_root_user' $PROPFILE db_root_user + getPropertyFromFile 'db_root_password' $PROPFILE db_user + getPropertyFromFile 'db_user' $PROPFILE db_user + getPropertyFromFile 'db_password' $PROPFILE db_password } + check_python_command() { if is_command ${PYTHON_COMMAND_INVOKER} ; then log "[I] '${PYTHON_COMMAND_INVOKER}' command found" @@ -100,6 +166,23 @@ check_python_command() { fi } +run_dba_steps(){ + getPropertyFromFileNoExit 'setup_mode' $PROPFILE setup_mode false + if [ "x${setup_mode}x" == "xSeparateDBAx" ]; then + log "[I] Setup mode is set to SeparateDBA. Not Running DBA steps. Please run dba_script.py before running setup..!"; + else + log "[I] Setup mode is not set. Running DBA steps.."; + python dba_script.py -q + fi +} +check_db_connector() { + log "[I] Checking ${DB_FLAVOR} CONNECTOR FILE : ${SQL_CONNECTOR_JAR}" + if test -f "$SQL_CONNECTOR_JAR"; then + log "[I] ${DB_FLAVOR} CONNECTOR FILE : $SQL_CONNECTOR_JAR file found" + else + log "[E] ${DB_FLAVOR} CONNECTOR FILE : $SQL_CONNECTOR_JAR does not exists" ; exit 1; + fi +} check_java_version() { #Check for JAVA_HOME if [ "${JAVA_HOME}" == "" ] @@ -117,9 +200,11 @@ check_java_version() { exit 1; fi - $JAVA_BIN -version 2>&1 | grep -q $JAVA_VERSION_REQUIRED - if [ $? != 0 ] ; then - log "[E] Java 1.7 is required" + version=$("$JAVA_BIN" -version 2>&1 | awk -F '"' '/version/ {print $2}') + major=`echo ${version} | cut -d. -f1` + minor=`echo ${version} | cut -d. -f2` + if [[ "${major}" == 1 && "${minor}" < 7 ]] ; then + log "[E] Java 1.7 is required, current java version is $version" exit 1; fi } @@ -138,35 +223,307 @@ sanity_check_files() { else log "[E] $mysql_core_file does not exists" ; exit 1; fi - fi + fi + if [ "${DB_FLAVOR}" == "ORACLE" ] + then + if test -f ${oracle_core_file}; then + log "[I] ${oracle_core_file} file found" + else + log "[E] ${oracle_core_file} does not exists" ; exit 1; + fi + fi + if [ "${DB_FLAVOR}" == "POSTGRES" ] + then + if test -f ${postgres_core_file}; then + log "[I] ${postgres_core_file} file found" + else + log "[E] ${postgres_core_file} does not exists" ; exit 1; + fi + fi + if [ "${DB_FLAVOR}" == "SQLSERVER" ] + then + if test -f ${sqlserver_core_file}; then + log "[I] ${sqlserver_core_file} file found" + else + log "[E] ${sqlserver_core_file} does not exists" ; exit 1; + fi + fi } +create_rollback_point() { + DATE=`date` + BAK_FILE=$APP-$VERSION.$DATE.bak + log "Creating backup file : $BAK_FILE" + cp "$APP" "$BAK_FILE" +} + + copy_db_connector(){ - log "[I] Copying ${DB_FLAVOR} Connector to $app_home/lib "; - cp -f $SQL_CONNECTOR_JAR $app_home/lib - check_ret_status $? "Copying ${DB_FLAVOR} Connector to $app_home/lib failed" - log "[I] Copying ${DB_FLAVOR} Connector to $app_home/lib DONE"; + log "[I] Copying ${DB_FLAVOR} Connector to $app_home/WEB-INF/lib "; + cp -f $SQL_CONNECTOR_JAR $app_home/WEB-INF/lib + check_ret_status $? "Copying ${DB_FLAVOR} Connector to $app_home/WEB-INF/lib failed" + log "[I] Copying ${DB_FLAVOR} Connector to $app_home/WEB-INF/lib DONE"; } setup_kms(){ #copying ranger kms provider - cd ${RANGER_KMS_EWS_DIR}/webapp + cd $PWD/ews/webapp log "[I] Adding ranger kms provider as services in hadoop-common jar" jar -uf lib/hadoop-common*.jar META-INF/services/org.apache.hadoop.crypto.key.KeyProviderFactory } +update_properties() { + newPropertyValue='' + #echo "export JAVA_HOME=${JAVA_HOME}" > ${WEBAPP_ROOT}/WEB-INF/classes/conf/java_home.sh + #chmod a+rx ${WEBAPP_ROOT}/WEB-INF/classes/conf/java_home.sh + + + to_file=$app_home/config/dbks-site.xml + if test -f $to_file; then + log "[I] $to_file file found" + else + log "[E] $to_file does not exists" ; exit 1; + fi + + + propertyName=ranger.ks.jpa.jdbc.user + newPropertyValue="${db_user}" + updatePropertyToFilePy $propertyName $newPropertyValue $to_file + + if [ "${DB_FLAVOR}" == "MYSQL" ] + then + propertyName=ranger.ks.jpa.jdbc.url + newPropertyValue="jdbc:log4jdbc:mysql://${DB_HOST}/${db_name}" + updatePropertyToFilePy $propertyName $newPropertyValue $to_file + + propertyName=ranger.ks.jpa.jdbc.dialect + newPropertyValue="org.eclipse.persistence.platform.database.MySQLPlatform" + updatePropertyToFilePy $propertyName $newPropertyValue $to_file + + propertyName=ranger.ks.jpa.jdbc.driver + newPropertyValue="net.sf.log4jdbc.DriverSpy" + updatePropertyToFilePy $propertyName $newPropertyValue $to_file + + fi + if [ "${DB_FLAVOR}" == "ORACLE" ] + then + propertyName=ranger.ks.jpa.jdbc.url + newPropertyValue="jdbc:oracle:thin:\@//${DB_HOST}" + updatePropertyToFilePy $propertyName $newPropertyValue $to_file + + propertyName=ranger.ks.jpa.jdbc.dialect + newPropertyValue="org.eclipse.persistence.platform.database.OraclePlatform" + updatePropertyToFilePy $propertyName $newPropertyValue $to_file + + propertyName=ranger.ks.jpa.jdbc.driver + newPropertyValue="oracle.jdbc.OracleDriver" + updatePropertyToFilePy $propertyName $newPropertyValue $to_file + + fi + if [ "${DB_FLAVOR}" == "POSTGRES" ] + then + propertyName=ranger.ks.jpa.jdbc.url + newPropertyValue="jdbc:postgresql://${DB_HOST}/${db_name}" + updatePropertyToFilePy $propertyName $newPropertyValue $to_file + + propertyName=ranger.ks.jpa.jdbc.dialect + newPropertyValue="org.eclipse.persistence.platform.database.PostgreSQLPlatform" + updatePropertyToFilePy $propertyName $newPropertyValue $to_file + + propertyName=ranger.ks.jpa.jdbc.driver + newPropertyValue="org.postgresql.Driver" + updatePropertyToFilePy $propertyName $newPropertyValue $to_file + + fi + if [ "${DB_FLAVOR}" == "SQLSERVER" ] + then + propertyName=ranger.ks.jpa.jdbc.url + newPropertyValue="jdbc:sqlserver://${DB_HOST};databaseName=${db_name}" + updatePropertyToFilePy $propertyName $newPropertyValue $to_file + + propertyName=ranger.ks.jpa.jdbc.dialect + newPropertyValue="org.eclipse.persistence.platform.database.SQLServerPlatform" + updatePropertyToFilePy $propertyName $newPropertyValue $to_file + + propertyName=ranger.ks.jpa.jdbc.driver + newPropertyValue="com.microsoft.sqlserver.jdbc.SQLServerDriver" + updatePropertyToFilePy $propertyName $newPropertyValue $to_file + + fi + + keystore="${cred_keystore_filename}" + + echo "Starting configuration for XA DB credentials:" + + db_password_alias=ranger.ks.jdbc.password + + if [ "${keystore}" != "" ] + then + mkdir -p `dirname "${keystore}"` + + $JAVA_HOME/bin/java -cp "cred/lib/*" org.apache.ranger.credentialapi.buildks create "$db_password_alias" -value "$db_password" -provider jceks://file$keystore + + propertyName=ranger.ks.jpa.jdbc.credential.alias + newPropertyValue="${db_password_alias}" + updatePropertyToFilePy $propertyName $newPropertyValue $to_file + + propertyName=ranger.ks.jpa.jdbc.credential.provider.path + newPropertyValue="${keystore}" + updatePropertyToFilePy $propertyName $newPropertyValue $to_file + + propertyName=ranger.ks.jpa.jdbc.password + newPropertyValue="_" + updatePropertyToFilePy $propertyName $newPropertyValue $to_file + else + propertyName=ranger.ks.jpa.jdbc.password + newPropertyValue="${db_password}" + updatePropertyToFilePy $propertyName $newPropertyValue $to_file + fi + + if test -f $keystore; then + #echo "$keystore found." + chown -R ${unix_user}:${unix_group} ${keystore} + chmod 640 ${keystore} + else + #echo "$keystore not found. so clear text password" + propertyName=ranger.ks.jpa.jdbc.password + newPropertyValue="${db_password}" + updatePropertyToFilePy $propertyName $newPropertyValue $to_file + fi + + ########### +} + +#===================================================================== + +setup_unix_user_group(){ + + log "[I] Setting up UNIX user : ${unix_user} and group: ${unix_group}"; + + groupadd ${unix_group} + check_ret_status_for_groupadd $? "Creating group ${unix_group} failed" + + id -u ${unix_user} > /dev/null 2>&1 + + if [ $? -ne 0 ] + then + log "[I] Creating new user and adding to group"; + useradd ${unix_user} -g ${unix_group} -m + check_ret_status $? "useradd ${unix_user} failed" + else + log "[I] User already exists, adding it to group"; + usermod -g ${unix_group} ${unix_user} + fi + + log "[I] Setting up UNIX user : ${unix_user} and group: ${unix_group} DONE"; +} + +setup_install_files(){ + + log "[I] Setting up installation files and directory"; + + #if [ ! -d ${WEBAPP_ROOT}/WEB-INF/classes/conf ]; then + # log "[I] Copying ${WEBAPP_ROOT}/WEB-INF/classes/conf.dist ${WEBAPP_ROOT}/WEB-INF/classes/conf" + # mkdir -p ${WEBAPP_ROOT}/WEB-INF/classes/conf + # cp ${WEBAPP_ROOT}/WEB-INF/classes/conf.dist/* ${WEBAPP_ROOT}/WEB-INF/classes/conf + # chown -R ${unix_user} ${WEBAPP_ROOT}/WEB-INF/classes/conf + #fi + + if [ ! -d ${WEBAPP_ROOT}/WEB-INF/classes/lib ]; then + log "[I] Creating ${WEBAPP_ROOT}/WEB-INF/classes/lib" + mkdir -p ${WEBAPP_ROOT}/WEB-INF/classes/lib + chown -R ${unix_user} ${WEBAPP_ROOT}/WEB-INF/classes/lib + fi + + if [ -d /etc/init.d ]; then + log "[I] Setting up init.d" + cp ${INSTALL_DIR}/ews/${RANGER_KMS_INITD} /etc/init.d/${RANGER_KMS} + + chmod ug+rx /etc/init.d/${RANGER_KMS} + + if [ -d /etc/rc2.d ] + then + RC_DIR=/etc/rc2.d + log "[I] Creating script S88${RANGER_KMS}/K90${RANGER_KMS} in $RC_DIR directory .... " + rm -f $RC_DIR/S88${RANGER_KMS} $RC_DIR/K90${RANGER_KMS} + ln -s /etc/init.d/${RANGER_KMS} $RC_DIR/S88${RANGER_KMS} + ln -s /etc/init.d/${RANGER_KMS} $RC_DIR/K90${RANGER_KMS} + fi + + if [ -d /etc/rc3.d ] + then + RC_DIR=/etc/rc3.d + log "[I] Creating script S88${RANGER_KMS}/K90${RANGER_KMS} in $RC_DIR directory .... " + rm -f $RC_DIR/S88${RANGER_KMS} $RC_DIR/K90${RANGER_KMS} + ln -s /etc/init.d/${RANGER_KMS} $RC_DIR/S88${RANGER_KMS} + ln -s /etc/init.d/${RANGER_KMS} $RC_DIR/K90${RANGER_KMS} + fi + + # SUSE has rc2.d and rc3.d under /etc/rc.d + if [ -d /etc/rc.d/rc2.d ] + then + RC_DIR=/etc/rc.d/rc2.d + log "[I] Creating script S88${RANGER_KMS}/K90${RANGER_KMS} in $RC_DIR directory .... " + rm -f $RC_DIR/S88${RANGER_KMS} $RC_DIR/K90${RANGER_KMS} + ln -s /etc/init.d/${RANGER_KMS} $RC_DIR/S88${RANGER_KMS} + ln -s /etc/init.d/${RANGER_KMS} $RC_DIR/K90${RANGER_KMS} + fi + if [ -d /etc/rc.d/rc3.d ] + then + RC_DIR=/etc/rc.d/rc3.d + log "[I] Creating script S88${RANGER_KMS}/K90${RANGER_KMS} in $RC_DIR directory .... " + rm -f $RC_DIR/S88${RANGER_KMS} $RC_DIR/K90${RANGER_KMS} + ln -s /etc/init.d/${RANGER_KMS} $RC_DIR/S88${RANGER_KMS} + ln -s /etc/init.d/${RANGER_KMS} $RC_DIR/K90${RANGER_KMS} + fi + fi + + if [ ! -d ${KMS_DIR}/ews/logs ]; then + log "[I] ${KMS_DIR}/ews/logs folder" + mkdir -p ${KMS_DIR}/ews/logs + chown -R ${unix_user} ${KMS_DIR}/ews/logs + fi + + log "[I] Setting up installation files and directory DONE"; + + if [ ! -f ${INSTALL_DIR}/rpm ]; then + if [ -d ${INSTALL_DIR} ] + then + chown -R ${unix_user}:${unix_group} ${INSTALL_DIR} + chown -R ${unix_user}:${unix_group} ${INSTALL_DIR}/* + fi + fi + + # Copy ranger-admin-services to /usr/bin + if [ ! \( -e /usr/bin/ranger-kms \) ] + then + ln -sf ${INSTALL_DIR}/ranger-kms-services.sh /usr/bin/ranger-kms + chmod ug+rx /usr/bin/ranger-kms + fi +} init_logfiles -log " --------- Running ranger kms Web Application Install Script --------- " +log " --------- Running Ranger KMS Application Install Script --------- " log "[I] uname=`uname`" log "[I] hostname=`hostname`" init_variables get_distro check_java_version +check_db_connector +setup_unix_user_group +setup_install_files sanity_check_files copy_db_connector check_python_command -$PYTHON_COMMAND_INVOKER db_setup.py -setup_kms - -echo "Installation of ranger kms is completed." +run_dba_steps +$PYTHON_COMMAND_INVOKER db_setup.py +if [ "$?" == "0" ] +then + update_properties + $PYTHON_COMMAND_INVOKER db_setup.py -javapatch + setup_kms +else + log "[E] DB schema setup failed! Please contact Administrator." + exit 1 +fi +echo "Installation of Ranger KMS is completed." http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/917833cd/kms/scripts/update_property.py ---------------------------------------------------------------------- diff --git a/kms/scripts/update_property.py b/kms/scripts/update_property.py new file mode 100644 index 0000000..e27277f --- /dev/null +++ b/kms/scripts/update_property.py @@ -0,0 +1,24 @@ +import sys +import os +from xml.etree import ElementTree as ET + +def write_properties_to_xml(xml_path, property_name='', property_value=''): + if(os.path.isfile(xml_path)): + xml = ET.parse(xml_path) + root = xml.getroot() + for child in root.findall('property'): + name = child.find("name").text.strip() + if name == property_name: + child.find("value").text = property_value + xml.write(xml_path) + return 0 + else: + return -1 + + +if __name__ == '__main__': + if(len(sys.argv) > 1): + parameter_name = sys.argv[1] if len(sys.argv) > 1 else None + parameter_value = sys.argv[2] if len(sys.argv) > 2 else None + file_path = sys.argv[3] if len(sys.argv) > 3 else None + write_properties_to_xml(file_path,parameter_name,parameter_value) http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/917833cd/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKMSDB.java ---------------------------------------------------------------------- diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKMSDB.java b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKMSDB.java old mode 100644 new mode 100755 index 1496600..e4490e0 --- a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKMSDB.java +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKMSDB.java @@ -19,6 +19,8 @@ package org.apache.hadoop.crypto.key; import java.util.HashMap; import java.util.Map; +import java.util.Set; +import java.util.Iterator; import javax.persistence.EntityManager; import javax.persistence.EntityManagerFactory; @@ -37,17 +39,25 @@ public class RangerKMSDB { private static Map<String, String> DB_PROPERTIES = null; - private static final String PROPERTY_PREFIX = "ranger.db.ks."; - private static final String DB_DIALECT = "javax.persistence.jdbc.dialect"; - private static final String DB_DRIVER = "javax.persistence.jdbc.driver"; - private static final String DB_URL = "javax.persistence.jdbc.url"; - private static final String DB_USER = "javax.persistence.jdbc.user"; - private static final String DB_PASSWORD = "javax.persistence.jdbc.password"; + private static final String PROPERTY_PREFIX = "ranger.ks."; + private static final String DB_DIALECT = "jpa.jdbc.dialect"; + private static final String DB_DRIVER = "jpa.jdbc.driver"; + private static final String DB_URL = "jpa.jdbc.url"; + private static final String DB_USER = "jpa.jdbc.user"; + private static final String DB_PASSWORD = "jpa.jdbc.password"; + + private static final String JPA_DB_DIALECT = "javax.persistence.jdbc.dialect"; + private static final String JPA_DB_DRIVER = "javax.persistence.jdbc.driver"; + private static final String JPA_DB_URL = "javax.persistence.jdbc.url"; + private static final String JPA_DB_USER = "javax.persistence.jdbc.user"; + private static final String JPA_DB_PASSWORD = "javax.persistence.jdbc.password"; + private final Configuration conf; public RangerKMSDB(){ conf = new Configuration(); + //TODO: need to load kms db config file here ... } public RangerKMSDB(Configuration conf){ @@ -61,20 +71,29 @@ public class RangerKMSDB { private void initDBConnectivity(){ try { + DB_PROPERTIES = new HashMap<String, String>(); - DB_PROPERTIES.put(DB_DIALECT, conf.get(PROPERTY_PREFIX+DB_DIALECT)); - DB_PROPERTIES.put(DB_DRIVER, conf.get(PROPERTY_PREFIX+DB_DRIVER)); - DB_PROPERTIES.put(DB_URL, conf.get(PROPERTY_PREFIX+DB_URL)); - DB_PROPERTIES.put(DB_USER, conf.get(PROPERTY_PREFIX+DB_USER)); - DB_PROPERTIES.put(DB_PASSWORD, conf.get(PROPERTY_PREFIX+DB_PASSWORD)); - - entityManagerFactory = Persistence.createEntityManagerFactory("persistence_ranger_server", DB_PROPERTIES); + DB_PROPERTIES.put(JPA_DB_DIALECT, conf.get(PROPERTY_PREFIX+DB_DIALECT)); + DB_PROPERTIES.put(JPA_DB_DRIVER, conf.get(PROPERTY_PREFIX+DB_DRIVER)); + DB_PROPERTIES.put(JPA_DB_URL, conf.get(PROPERTY_PREFIX+DB_URL)); + DB_PROPERTIES.put(JPA_DB_USER, conf.get(PROPERTY_PREFIX+DB_USER)); + DB_PROPERTIES.put(JPA_DB_PASSWORD, conf.get(PROPERTY_PREFIX+DB_PASSWORD)); - daoManager = new DaoManager(); - daoManager.setEntityManagerFactory(entityManagerFactory); + //DB_PROPERTIES.list(System.out) ; - daoManager.getEntityManager(); // this forces the connection to be made to DB - logger.info("Connected to DB : "+isDbConnected()); + Set keys = DB_PROPERTIES.keySet(); + + for (Iterator i = keys.iterator(); i.hasNext();) { + String key = (String) i.next(); + String value = (String) DB_PROPERTIES.get(key); + System.out.println(key + " = " + value); + } + + entityManagerFactory = Persistence.createEntityManagerFactory("persistence_ranger_server", DB_PROPERTIES); + daoManager = new DaoManager(); + daoManager.setEntityManagerFactory(entityManagerFactory); + daoManager.getEntityManager(); // this forces the connection to be made to DB + logger.info("Connected to DB : "+isDbConnected()); } catch(Exception excp) { excp.printStackTrace(); } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/917833cd/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java ---------------------------------------------------------------------- diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java old mode 100644 new mode 100755 index 4ae6d35..880f647 --- a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java @@ -43,15 +43,24 @@ import org.apache.hadoop.crypto.key.KeyProvider; import org.apache.hadoop.crypto.key.KeyProviderFactory; import org.codehaus.jackson.map.ObjectMapper; import org.apache.hadoop.fs.Path; +import org.apache.ranger.credentialapi.CredentialReader; import org.apache.ranger.kms.dao.DaoManager; +import org.apache.log4j.Logger; public class RangerKeyStoreProvider extends KeyProvider{ + static final Logger logger = Logger.getLogger(RangerKeyStoreProvider.class); + public static final String SCHEME_NAME = "dbks"; public static final String KMS_CONFIG_DIR = "kms.config.dir"; public static final String DBKS_SITE_XML = "dbks-site.xml"; public static final String ENCRYPTION_KEY = "ranger.db.encrypt.key.password"; private static final String KEY_METADATA = "KeyMetadata"; + private static final String CREDENTIAL_PATH = "ranger.ks.jpa.jdbc.credential.provider.path"; + private static final String MK_CREDENTIAL_ALIAS = "ranger.ks.masterkey.credential.alias"; + private static final String DB_CREDENTIAL_ALIAS = "ranger.ks.jpa.jdbc.credential.alias"; + private static final String DB_PASSWORD = "ranger.ks.jpa.jdbc.password"; + private final RangerKeyStore dbStore; private char[] masterKey; private boolean changed = false; @@ -61,11 +70,17 @@ public class RangerKeyStoreProvider extends KeyProvider{ public RangerKeyStoreProvider(Configuration conf) throws Throwable { super(conf); conf = getDBKSConf(); + getFromJceks(conf,CREDENTIAL_PATH, MK_CREDENTIAL_ALIAS, ENCRYPTION_KEY); + getFromJceks(conf,CREDENTIAL_PATH, DB_CREDENTIAL_ALIAS, DB_PASSWORD); RangerKMSDB rangerKMSDB = new RangerKMSDB(conf); daoManager = rangerKMSDB.getDaoManager(); RangerMasterKey rangerMasterKey = new RangerMasterKey(daoManager); - dbStore = new RangerKeyStore(daoManager); + dbStore = new RangerKeyStore(daoManager); String password = conf.get(ENCRYPTION_KEY); + System.out.println("RKSP Password = "+password); + if(password == null || password.trim().equals("") || password.trim().equals("_") || password.trim().equals("crypted")){ + throw new IOException("Master Key Jceks does not exists"); + } rangerMasterKey.generateMasterKey(password); //code to retrieve rangerMasterKey password masterKey = rangerMasterKey.getMasterKey(password).toCharArray(); @@ -301,6 +316,28 @@ public class RangerKeyStoreProvider extends KeyProvider{ return innerSetKeyVersion(name, versionName, material, meta.getCipher(), meta.getBitLength(), meta.getDescription(), meta.getVersions(), meta.getAttributes()); } + private void getFromJceks(Configuration conf, String path, String alias, String key){ + //update credential from keystore + System.out.println("getFromJCEKS path = "+path+" alias = "+alias+" key = "+key); + if(conf!=null){ + String pathValue=conf.get(path); + System.out.println("path Value = "+pathValue); + String aliasValue=conf.get(alias); + System.out.println("alias Value = "+aliasValue); + if(pathValue!=null && aliasValue!=null){ + String xaDBPassword=CredentialReader.getDecryptedString(pathValue.trim(),aliasValue.trim()); + System.out.println("xaDBPassword = "+xaDBPassword); + if(xaDBPassword!=null&& !xaDBPassword.trim().isEmpty() && + !xaDBPassword.trim().equalsIgnoreCase("none")){ + System.out.println("inside key = "+key+" xaDBaswword "+xaDBPassword); + conf.set(key, xaDBPassword); + }else{ + logger.info("Credential keystore password not applied for KMS; clear text password shall be applicable"); + } + } + } + } + /** * The factory to create JksProviders, which is used by the ServiceLoader. */ @@ -359,4 +396,4 @@ public class RangerKeyStoreProvider extends KeyProvider{ } } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/917833cd/kms/src/main/java/org/apache/hadoop/crypto/key/RangerMasterKey.java ---------------------------------------------------------------------- diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerMasterKey.java b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerMasterKey.java old mode 100644 new mode 100755 index 71ba1a3..75a34b2 --- a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerMasterKey.java +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerMasterKey.java @@ -50,8 +50,6 @@ public class RangerMasterKey { private static final String PBE_ALGO = "PBEWithMD5AndTripleDES"; private static final String MD_ALGO = "MD5"; - public static final String ENCRYPTION_KEY = "ranger.db.encrypt.key.password"; - private DaoManager daoManager; public RangerMasterKey() { http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/917833cd/plugin-kms/scripts/enable-kms-plugin.sh ---------------------------------------------------------------------- diff --git a/plugin-kms/scripts/enable-kms-plugin.sh b/plugin-kms/scripts/enable-kms-plugin.sh new file mode 100644 index 0000000..1b6f52e --- /dev/null +++ b/plugin-kms/scripts/enable-kms-plugin.sh @@ -0,0 +1,633 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +function getInstallProperty() { + local propertyName=$1 + local propertyValue="" + + for file in "${COMPONENT_INSTALL_ARGS}" "${INSTALL_ARGS}" + do + if [ -f "${file}" ] + then + propertyValue=`grep "^${propertyName}" ${file} | awk -F= '{ sub("^[ \t]*", "", $2); sub("[ \t]*$", "", $2); print $2 }'` + if [ "${propertyValue}" != "" ] + then + break + fi + fi + done + + echo ${propertyValue} +} + +# +# Base env variable for Ranger related files/directories +# +PROJ_NAME=ranger +BASE_CONF_DIR=/etc/${PROJ_NAME} + +# +# The script should be run by "root" user +# + +if [ ! -w /etc/passwd ] +then + echo "ERROR: $0 script should be run as root." + exit 1 +fi + +#Check for JAVA_HOME +if [ "${JAVA_HOME}" == "" ] +then + echo "ERROR: JAVA_HOME environment property not defined, aborting installation." + exit 1 +fi + +# +# Identify the component, action from the script file +# + +basedir=`dirname $0` +if [ "${basedir}" = "." ] +then + basedir=`pwd` +elif [ "${basedir}" = ".." ] +then + basedir=`(cd .. ;pwd)` +fi + +# +# As this script is common to all component, find the component name based on the script-name +# + +COMPONENT_NAME=`basename $0 | cut -d. -f1 | sed -e 's:^disable-::' | sed -e 's:^enable-::'` + +echo "${COMPONENT_NAME}" | grep 'plugin' > /dev/null 2>&1 + +if [ $? -ne 0 ] +then + echo "$0 : is not applicable for component [${COMPONENT_NAME}]. It is applicable only for ranger plugin component; Exiting ..." + exit 0 +fi + +HCOMPONENT_NAME=`echo ${COMPONENT_NAME} | sed -e 's:-plugin::'` + +CFG_OWNER_INF="${HCOMPONENT_NAME}:${HCOMPONENT_NAME}" + +if [ "${HCOMPONENT_NAME}" = "hdfs" ] +then + HCOMPONENT_NAME="hadoop" +fi + +# +# Based on script name, identify if the action is enabled or disabled +# + +basename $0 | cut -d. -f1 | grep '^enable-' > /dev/null 2>&1 + +if [ $? -eq 0 ] +then + action=enable +else + action=disable +fi + + +# +# environment variables for enable|disable scripts +# + +PROJ_INSTALL_DIR1=`(cd ${basedir} ; pwd)` +PROJ_INSTALL_DIR=${PROJ_INSTALL_DIR1}/.. +SET_ENV_SCRIPT_NAME=set-${COMPONENT_NAME}-env.sh +SET_ENV_SCRIPT_TEMPLATE=${PROJ_INSTALL_DIR}/scripts/install/conf.templates/enable/${SET_ENV_SCRIPT_NAME} +DEFAULT_XML_CONFIG=${PROJ_INSTALL_DIR}/scripts/install/conf.templates/default/configuration.xml +PROJ_LIB_DIR=${PROJ_INSTALL_DIR}/ews/plugin/lib +PROJ_INSTALL_LIB_DIR="${PROJ_INSTALL_DIR}/scripts/install/lib" +INSTALL_ARGS="${PROJ_INSTALL_DIR}/ews/webapp/config/plugin-install.properties" +COMPONENT_INSTALL_ARGS="${PROJ_INSTALL_DIR}/ews/webapp/config/${COMPONENT_NAME}-install.properties" +JAVA=$JAVA_HOME/bin/java + +HCOMPONENT_INSTALL_DIR_NAME=$(getInstallProperty 'COMPONENT_INSTALL_DIR_NAME') + +if [ "${HCOMPONENT_INSTALL_DIR_NAME}" = "" ] +then + HCOMPONENT_INSTALL_DIR_NAME=${HCOMPONENT_NAME} +fi + +hdir=${PROJ_INSTALL_DIR}/ews +# +# TEST - START +# +if [ ! -d ${hdir} ] +then + mkdir -p ${hdir} +fi +# +# TEST - END +# +HCOMPONENT_INSTALL_DIR=`(cd ${hdir} ; pwd)` +HCOMPONENT_LIB_DIR=${HCOMPONENT_INSTALL_DIR}/lib +if [ "${HCOMPONENT_NAME}" = "knox" ] +then + HCOMPONENT_LIB_DIR=${HCOMPONENT_INSTALL_DIR}/ext +fi +HCOMPONENT_CONF_DIR=${HCOMPONENT_INSTALL_DIR}/conf +HCOMPONENT_ARCHIVE_CONF_DIR=${HCOMPONENT_CONF_DIR}/.archive +SET_ENV_SCRIPT=${HCOMPONENT_CONF_DIR}/${SET_ENV_SCRIPT_NAME} + +if [ ! -d "${HCOMPONENT_INSTALL_DIR}" ] +then + echo "ERROR: Unable to find the install directory of component [${HCOMPONENT_NAME}]; dir [${HCOMPONENT_INSTALL_DIR}] not found." + echo "Exiting installation." + exit 1 +fi + +if [ ! -d "${HCOMPONENT_CONF_DIR}" ] +then + echo "ERROR: Unable to find the conf directory of component [${HCOMPONENT_NAME}]; dir [${HCOMPONENT_CONF_DIR}] not found." + echo "Exiting installation." + exit 1 +fi + +if [ ! -d "${HCOMPONENT_LIB_DIR}" ] +then + echo "ERROR: Unable to find the lib directory of component [${HCOMPONENT_NAME}]; dir [${HCOMPONENT_LIB_DIR}] not found." + echo "Exiting installation." + exit 1 +fi + +ambari_hive_install="N" +if [ "${HCOMPONENT_NAME}" = "hive" ] +then + HCOMPONENT_CONF_SERVER_DIR="${HCOMPONENT_CONF_DIR}"/../conf.server + if [ -d "${HCOMPONENT_CONF_SERVER_DIR}" ] + then + ambari_hive_install="Y" + fi +fi + +# +# Common functions used by all enable/disable scripts +# + +log() { + echo "+ `date` : $*" +} + + +create_jceks() { + + alias=$1 + pass=$2 + jceksFile=$3 + + if [ -f "${jceksFile}" ] + then + jcebdir=`dirname ${jceksFile}` + jcebname=`basename ${jceksFile}` + archive_jce=${jcebdir}/.${jcebname}.`date '+%Y%m%d%H%M%S'` + log "Saving current JCE file: ${jceksFile} to ${archive_jce} ..." + cp ${jceksFile} ${archive_jce} + fi + + tempFile=/tmp/jce.$$.out + + $JAVA_HOME/bin/java -cp ":${PROJ_INSTALL_LIB_DIR}/*:${PROJ_INSTALL_DIR}/cred/lib/*" org.apache.ranger.credentialapi.buildks create "${alias}" -value "${pass}" -provider "jceks://file${jceksFile}" > ${tempFile} 2>&1 + + if [ $? -ne 0 ] + then + echo "Unable to store password in non-plain text format. Error: [`cat ${tempFile}`]" + echo "Exiting plugin installation" + rm -f ${tempFile} + exit 0 + fi + + rm -f ${tempFile} +} + +# +# If there is a set-ranger-${COMPONENT}-env.sh, install it +# +dt=`date '+%Y%m%d-%H%M%S'` + +if [ -f "${SET_ENV_SCRIPT_TEMPLATE}" ] +then + # + # If the setenv script already exists, move it to the archive folder + # + if [ -f "${SET_ENV_SCRIPT}" ] + then + if [ ! -d "${HCOMPONENT_ARCHIVE_CONF_DIR}" ] + then + mkdir -p ${HCOMPONENT_ARCHIVE_CONF_DIR} + fi + log "Saving current ${SET_ENV_SCRIPT_NAME} to ${HCOMPONENT_ARCHIVE_CONF_DIR} ..." + mv ${SET_ENV_SCRIPT} ${HCOMPONENT_ARCHIVE_CONF_DIR}/${SET_ENV_SCRIPT_NAME}.${dt} + fi + + if [ "${action}" = "enable" ] + then + + cp ${SET_ENV_SCRIPT_TEMPLATE} ${SET_ENV_SCRIPT} + + DEST_SCRIPT_FILE=${HCOMPONENT_INSTALL_DIR}/libexec/${HCOMPONENT_NAME}-config.sh + + DEST_SCRIPT_ARCHIVE_FILE=${HCOMPONENT_INSTALL_DIR}/libexec/.${HCOMPONENT_NAME}-config.sh.${dt} + + if [ -f "${DEST_SCRIPT_FILE}" ] + then + + log "Saving current ${DEST_SCRIPT_FILE} to ${DEST_SCRIPT_ARCHIVE_FILE} ..." + + cp ${DEST_SCRIPT_FILE} ${DEST_SCRIPT_ARCHIVE_FILE} + + grep 'xasecure-.*-env.sh' ${DEST_SCRIPT_FILE} > /dev/null 2>&1 + if [ $? -eq 0 ] + then + ts=`date '+%Y%m%d%H%M%S'` + grep -v 'xasecure-.*-env.sh' ${DEST_SCRIPT_FILE} > ${DEST_SCRIPT_FILE}.${ts} + if [ $? -eq 0 ] + then + log "Removing old reference to xasecure setenv source ..." + cat ${DEST_SCRIPT_FILE}.${ts} > ${DEST_SCRIPT_FILE} + rm -f ${DEST_SCRIPT_FILE}.${ts} + fi + fi + + grep "[ \t]*.[ \t]*${SET_ENV_SCRIPT}" ${DEST_SCRIPT_FILE} > /dev/null + if [ $? -ne 0 ] + then + log "Appending sourcing script, ${SET_ENV_SCRIPT_NAME} in the file: ${DEST_SCRIPT_FILE} " + cat >> ${DEST_SCRIPT_FILE} <<! +if [ -f ${SET_ENV_SCRIPT} ] +then + . ${SET_ENV_SCRIPT} +fi +! + else + log "INFO: ${DEST_SCRIPT_FILE} is being sourced from file: ${HCOMPONENT_CONF_DIR}/${HCOMPONENT_NAME}-env.sh " + fi + fi + fi +fi + +# +# Run, the enable|disable ${COMPONENT} configurations +# + +if [ -d "${PROJ_INSTALL_DIR}/scripts/install/conf.templates/${action}" ] +then + INSTALL_CP="${PROJ_INSTALL_LIB_DIR}/*" + if [ "${action}" = "enable" ] + then + echo "<ranger>\n<enabled>`date`</enabled>\n</ranger>" > ${HCOMPONENT_CONF_DIR}/ranger-security.xml + chown ${CFG_OWNER_INF} ${HCOMPONENT_CONF_DIR}/ranger-security.xml + chmod a+r ${HCOMPONENT_CONF_DIR}/ranger-security.xml + for cf in ${PROJ_INSTALL_DIR}/scripts/install/conf.templates/${action}/*.xml + do + cfb=`basename ${cf}` + if [ -f "${HCOMPONENT_CONF_DIR}/${cfb}" ] + then + log "Saving ${HCOMPONENT_CONF_DIR}/${cfb} to ${HCOMPONENT_CONF_DIR}/.${cfb}.${dt} ..." + cp ${HCOMPONENT_CONF_DIR}/${cfb} ${HCOMPONENT_CONF_DIR}/.${cfb}.${dt} + fi + cp ${cf} ${HCOMPONENT_CONF_DIR}/ + chown ${CFG_OWNER_INF} ${HCOMPONENT_CONF_DIR}/${cfb} + chmod a+r ${HCOMPONENT_CONF_DIR}/${cfb} + done + else + if [ -f ${HCOMPONENT_CONF_DIR}/ranger-security.xml ] + then + mv ${HCOMPONENT_CONF_DIR}/ranger-security.xml ${HCOMPONENT_CONF_DIR}/.ranger-security.xml.`date '+%Y%m%d%H%M%S'` + fi + fi + + # + # Ensure that POLICY_CACHE_FILE_PATH is accessible + # + REPO_NAME=$(getInstallProperty 'REPOSITORY_NAME') + export POLICY_CACHE_FILE_PATH=/etc/${PROJ_NAME}/${REPO_NAME}/policycache + export CREDENTIAL_PROVIDER_FILE=/etc/${PROJ_NAME}/${REPO_NAME}/cred.jceks + if [ ! -d ${POLICY_CACHE_FILE_PATH} ] + then + mkdir -p ${POLICY_CACHE_FILE_PATH} + fi + chmod a+rx /etc/${PROJ_NAME} + chmod a+rx /etc/${PROJ_NAME}/${REPO_NAME} + chmod a+rx ${POLICY_CACHE_FILE_PATH} + chown -R ${CFG_OWNER_INF} /etc/${PROJ_NAME}/${REPO_NAME} + + + # + # We need to do the AUDIT JDBC url + # + + db_flavor=`echo $(getInstallProperty 'XAAUDIT.DB.FLAVOUR') | tr '[:lower:]' '[:upper:]'` + audit_db_hostname=$(getInstallProperty 'XAAUDIT.DB.HOSTNAME') + audit_db_name=$(getInstallProperty 'XAAUDIT.DB.DATABASE_NAME') + + if [ "${db_flavor}" = "MYSQL" ] + then + export XAAUDIT_DB_JDBC_URL="jdbc:mysql://${audit_db_hostname}/${audit_db_name}" + export XAAUDIT_DB_JDBC_DRIVER="com.mysql.jdbc.Driver" + elif [ "${db_flavor}" = "ORACLE" ] + then + export XAAUDIT_DB_JDBC_URL="jdbc:oracle:thin:\@//${audit_db_hostname}" + export XAAUDIT_DB_JDBC_DRIVER="oracle.jdbc.OracleDriver" + elif [ "${db_flavor}" = "POSTGRES" ] + then + export XAAUDIT_DB_JDBC_URL="jdbc:postgresql://${audit_db_hostname}/${audit_db_name}" + export XAAUDIT_DB_JDBC_DRIVER="org.postgresql.Driver" + elif [ "${db_flavor}" = "SQLSERVER" ] + then + export XAAUDIT_DB_JDBC_URL="jdbc:sqlserver://${audit_db_hostname};databaseName=${audit_db_name}" + export XAAUDIT_DB_JDBC_DRIVER="com.microsoft.sqlserver.jdbc.SQLServerDriver" + else + echo "Audit is not specified with a valid db_flavor: [${db_flavor}]. Ignoring audit ..." + export XAAUDIT_DB_JDBC_URL="jdbc:${db_flavor}://${audit_db_hostname}/${audit_db_name}" + export XAAUDIT_DB_JDBC_DRIVER="com.unknown.driver.${db_flavor}" + fi + + + for f in ${PROJ_INSTALL_DIR}/scripts/install/conf.templates/${action}/*.cfg + do + if [ -f "${f}" ] + then + fn=`basename $f` + orgfn=`echo $fn | sed -e 's:-changes.cfg:.xml:'` + fullpathorgfn="${HCOMPONENT_CONF_DIR}/${orgfn}" + if [ ! -f ${fullpathorgfn} ] + then + if [ -f ${DEFAULT_XML_CONFIG} ] + then + log "Creating default file from [${DEFAULT_XML_CONFIG}] for [${fullpathorgfn}] .." + cp ${DEFAULT_XML_CONFIG} ${fullpathorgfn} + chown ${CFG_OWNER_INF} ${fullpathorgfn} + else + echo "ERROR: Unable to find ${fullpathorgfn}" + exit 1 + fi + fi + archivefn="${HCOMPONENT_CONF_DIR}/.${orgfn}.${dt}" + newfn="${HCOMPONENT_CONF_DIR}/.${orgfn}-new.${dt}" + log "Saving current config file: ${fullpathorgfn} to ${archivefn} ..." + cp ${fullpathorgfn} ${archivefn} + if [ $? -eq 0 ] + then + echo " ${JAVA} -cp ${INSTALL_CP} org.apache.ranger.utils.install.XmlConfigChanger -i ${archivefn} -o ${newfn} -c ${f} -p ${INSTALL_ARGS}" + ${JAVA} -cp "${INSTALL_CP}" org.apache.ranger.utils.install.XmlConfigChanger -i ${archivefn} -o ${newfn} -c ${f} -p ${INSTALL_ARGS} + if [ $? -eq 0 ] + then + diff -w ${newfn} ${fullpathorgfn} > /dev/null 2>&1 + if [ $? -ne 0 ] + then + cat ${newfn} > ${fullpathorgfn} + fi + + # For Ambari install copy the .xml to conf.server also + if [ "${ambari_hive_install}" = "Y" ] + then + fullpathorgHS2fn="${HCOMPONENT_CONF_SERVER_DIR}/${orgfn}" + archiveHS2fn="${HCOMPONENT_CONF_SERVER_DIR}/.${orgfn}.${dt}" + newHS2fn="${HCOMPONENT_CONF_SERVER_DIR}/.${orgfn}-new.${dt}" + log "Saving current conf.server file: ${fullpathorgHS2fn} to ${archiveHS2fn} ..." + if [ -f ${fullpathorgHS2fn} ] + then + cp ${fullpathorgHS2fn} ${archiveHS2fn} + fi + cp ${fullpathorgfn} ${HCOMPONENT_CONF_SERVER_DIR}/${orgfn} + chown ${CFG_OWNER_INF} ${HCOMPONENT_CONF_SERVER_DIR}/${orgfn} + fi + + else + echo "ERROR: Unable to make changes to config. file: ${fullpathorgfn}" + echo "exiting ...." + exit 1 + fi + else + echo "ERROR: Unable to save config. file: ${fullpathorgfn} to ${archivefn}" + echo "exiting ...." + exit 1 + fi + fi + done +fi + +# +# Create library link +# + +if [ "${action}" = "enable" ] +then + + #if [ -d "${PROJ_LIB_DIR}" ] + #then + dt=`date '+%Y%m%d%H%M%S'` + dbJar=$(getInstallProperty 'SQL_CONNECTOR_JAR') + for f in ${PROJ_LIB_DIR}/*.jar ${dbJar} + do + if [ -f "${f}" ] + then + bn=`basename $f` + if [ -f ${HCOMPONENT_LIB_DIR}/${bn} ] + then + log "Saving lib file: ${HCOMPONENT_LIB_DIR}/${bn} to ${HCOMPONENT_LIB_DIR}/.${bn}.${dt} ..." + mv ${HCOMPONENT_LIB_DIR}/${bn} ${HCOMPONENT_LIB_DIR}/.${bn}.${dt} + fi + if [ ! -f ${HCOMPONENT_LIB_DIR}/${bn} ] + then + ln -s ${f} ${HCOMPONENT_LIB_DIR}/${bn} + fi + fi + done + #fi + + # + # Encrypt the password and keep it secure in Credential Provider API + # + + CredFile=${CREDENTIAL_PROVIDER_FILE} + + if ! [ `echo ${CredFile} | grep '^/.*'` ] + then + echo "ERROR:Please enter the Credential File Store with proper file path" + exit 1 + fi + + pardir=`dirname ${CredFile}` + + if [ ! -d "${pardir}" ] + then + mkdir -p "${pardir}" + + if [ $? -ne 0 ] + then + echo "ERROR: Unable to create credential store file path" + exit 1 + fi + chmod a+rx "${pardir}" + fi + + # + # Generate Credential Provider file and Credential for Audit DB access. + # + + + auditCredAlias="auditDBCred" + + auditdbCred=$(getInstallProperty 'XAAUDIT.DB.PASSWORD') + + create_jceks "${auditCredAlias}" "${auditdbCred}" "${CredFile}" + + + # + # Generate Credential Provider file and Credential for SSL KEYSTORE AND TRUSTSTORE + # + + + sslkeystoreAlias="sslKeyStore" + + sslkeystoreCred=$(getInstallProperty 'SSL_KEYSTORE_PASSWORD') + + create_jceks "${sslkeystoreAlias}" "${sslkeystoreCred}" "${CredFile}" + + + ssltruststoreAlias="sslTrustStore" + + ssltruststoreCred=$(getInstallProperty 'SSL_TRUSTSTORE_PASSWORD') + + create_jceks "${ssltruststoreAlias}" "${ssltruststoreCred}" "${CredFile}" + + chown ${CFG_OWNER_INF} ${CredFile} + # + # To allow all users in the server (where Hive CLI and HBase CLI is used), + # user needs to have read access for the credential file. + # + chmod a+r ${CredFile} + +fi + +# +# Knox specific configuration +# +# + +if [ "${HCOMPONENT_NAME}" = "knox" ] +then + if [ "${action}" = "enable" ] + then + authFrom="AclsAuthz" + authTo="XASecurePDPKnox" + else + authTo="AclsAuthz" + authFrom="XASecurePDPKnox" + fi + + dt=`date '+%Y%m%d%H%M%S'` + for fn in `ls ${HCOMPONENT_CONF_DIR}/topologies/*.xml 2> /dev/null` + do + if [ -f "${fn}" ] + then + dn=`dirname ${fn}` + bn=`basename ${fn}` + bf=${dn}/.${bn}.${dt} + echo "backup of ${fn} to ${bf} ..." + cp ${fn} ${bf} + echo "Updating topology file: [${fn}] ... " + cat ${fn} | sed -e "s-<name>${authFrom}</name>-<name>${authTo}</name>-" > ${fn}.${dt}.new + if [ $? -eq 0 ] + then + cat ${fn}.${dt}.new > ${fn} + rm ${fn}.${dt}.new + fi + fi + done +fi + +if [ "${HCOMPONENT_NAME}" = "storm" ] +then + CFG_FILE=${HCOMPONENT_CONF_DIR}/storm.yaml + ARCHIVE_FILE=${HCOMPONENT_CONF_DIR}/.storm.yaml.`date '+%Y%m%d%H%M%S'` + + if [ -f "${CFG_FILE}" ] + then + cp ${CFG_FILE} ${ARCHIVE_FILE} + + if [ "${action}" = "enable" ] + then + awk -F: 'BEGIN { + configured = 0 ; + } + { + if ($1 == "nimbus.authorizer") { + if ($2 ~ /^[ \t]*"org.apache.ranger.authorization.storm.authorizer.RangerStormAuthorizer"[ \t]*$/) { + configured = 1 ; + printf("%s\n",$0) ; + } + else { + printf("#%s\n",$0); + printf("nimbus.authorizer: \"org.apache.ranger.authorization.storm.authorizer.RangerStormAuthorizer\"\n") ; + configured = 1 ; + } + } + else { + printf("%s\n",$0) ; + } + } + END { + if (configured == 0) { + printf("nimbus.authorizer: \"org.apache.ranger.authorization.storm.authorizer.RangerStormAuthorizer\"\n") ; + } + }' ${CFG_FILE} > ${CFG_FILE}.new && cat ${CFG_FILE}.new > ${CFG_FILE} && rm -f ${CFG_FILE}.new + + else + awk -F: 'BEGIN { + configured = 0 ; + } + { + if ($1 == "nimbus.authorizer") { + if ($2 ~ /^[ \t]*"backtype.storm.security.auth.authorizer.SimpleACLAuthorizer"[ \t]*$/) { + configured = 1 ; + printf("%s\n",$0) ; + } + else { + printf("#%s\n",$0); + printf("nimbus.authorizer: \"backtype.storm.security.auth.authorizer.SimpleACLAuthorizer\"\n") ; + configured = 1 ; + } + } + else { + printf("%s\n",$0) ; + } + } + END { + if (configured == 0) { + printf("nimbus.authorizer: \"backtype.storm.security.auth.authorizer.SimpleACLAuthorizer\"\n") ; + } + }' ${CFG_FILE} > ${CFG_FILE}.new && cat ${CFG_FILE}.new > ${CFG_FILE} && rm -f ${CFG_FILE}.new + fi + fi +fi + +# +# Set notice to restart the ${HCOMPONENT_NAME} +# + +echo "Ranger Plugin for ${HCOMPONENT_NAME} has been ${action}d. Please restart ${HCOMPONENT_NAME} to ensure that changes are effective." + +exit 0 http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/917833cd/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index f204876..acdf66f 100644 --- a/pom.xml +++ b/pom.xml @@ -75,13 +75,13 @@ <modules> <module>jisql</module> - <module>kms</module> <module>agents-audit</module> <module>agents-common</module> <module>agents-cred</module> <module>agents-installer</module> <module>credentialbuilder</module> <module>embeddedwebserver</module> + <module>kms</module> <module>hbase-agent</module> <module>hdfs-agent</module> <module>hive-agent</module> @@ -96,7 +96,7 @@ <module>unixauthclient</module> <module>unixauthservice</module> <module>ranger-util</module> - <module>plugin-kms</module> + <!--<module>plugin-kms</module>--> </modules> <properties> <javac.source.version>1.7</javac.source.version> @@ -374,7 +374,7 @@ <descriptor>src/main/assembly/migration-util.xml</descriptor> <descriptor>src/main/assembly/kms.xml</descriptor> <descriptor>src/main/assembly/ranger-src.xml</descriptor> - <descriptor>src/main/assembly/plugin-kms.xml</descriptor> + <!--<descriptor>src/main/assembly/plugin-kms.xml</descriptor>--> </descriptors> </configuration> </plugin> @@ -434,7 +434,7 @@ </pluginManagement> <plugins> - <plugin> + <!--<plugin> <groupId>org.apache.rat</groupId> <artifactId>apache-rat-plugin</artifactId> <version>0.11</version> @@ -482,7 +482,7 @@ </excludes> </configuration> - </plugin> + </plugin>--> <plugin> <groupId>org.apache.maven.plugins</groupId> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/917833cd/src/main/assembly/kms.xml ---------------------------------------------------------------------- diff --git a/src/main/assembly/kms.xml b/src/main/assembly/kms.xml index db57a3a..50a684d 100644 --- a/src/main/assembly/kms.xml +++ b/src/main/assembly/kms.xml @@ -165,8 +165,115 @@ <include>org.apache.ranger:jisql</include> </includes> </moduleSet> - </moduleSets> - + <!-- Plugin --> + <moduleSet> + <binaries> + <includeDependencies>false</includeDependencies> + <unpack>false</unpack> + <directoryMode>755</directoryMode> + <fileMode>644</fileMode> + <dependencySets> + <dependencySet> + <outputDirectory>/ews/plugin/lib</outputDirectory> + <unpack>false</unpack> + <includes> + <include>commons-configuration:commons-configuration:jar:${commons.pool.version}</include> + <include>com.google.code.gson:gson</include> + <include>org.eclipse.persistence:eclipselink</include> + <include>org.eclipse.persistence:javax.persistence</include> + <include>commons-collections:commons-collections</include> + <include>commons-logging:commons-logging:jar:${commons.logging.version}</include> + <include>com.google.guava:guava:jar:${guava.version}</include> + <include>org.apache.httpcomponents:httpclient:jar:${httpcomponent.httpclient.version}</include> + <include>org.apache.httpcomponents:httpcore:jar:${httpcomponent.httpcore.version}</include> + <include>org.apache.httpcomponents:httpmime:jar:${httpcomponent.httpmime.version}</include> + <include>org.noggit:noggit:jar:${noggit.version}</include> + <include>org.apache.zookeeper:zookeeper:jar:${zookeeper.version}</include> + </includes> + </dependencySet> + <dependencySet> + <outputDirectory>/scripts/install/lib</outputDirectory> + <unpack>false</unpack> + <directoryMode>755</directoryMode> + <fileMode>644</fileMode> + <includes> + <include>commons-cli:commons-cli</include> + <include>commons-collections:commons-collections</include> + <include>commons-configuration:commons-configuration:jar:${commons.pool.version}</include> + <include>commons-io:commons-io:jar:${commons.io.version}</include> + <include>commons-lang:commons-lang:jar:${commons.lang.version}</include> + <include>commons-logging:commons-logging:jar:${commons.logging.version}</include> + <include>com.google.guava:guava:jar:${guava.version}</include> + <include>org.hamcrest:hamcrest-all</include> + <include>junit:junit</include> + <include>org.slf4j:slf4j-api:jar:${slf4j-api.version}</include> + <include>org.apache.hadoop:hadoop-common:jar:${hadoop-common.version}</include> + <include>org.apache.hadoop:hadoop-auth:jar:${hadoop-common.version}</include> + <include>security_plugins.ranger-plugins-cred:ranger-plugins-cred</include> + <include>org.apache.ranger:credentialbuilder</include> + </includes> + </dependencySet> + </dependencySets> + <outputDirectory>/ews/lib</outputDirectory> + </binaries> + <includes> + <include>org.apache.ranger:ranger_solrj</include> + <include>security_plugins.ranger-plugins-audit:ranger-plugins-audit</include> + <include>security_plugins.ranger-plugins-cred:ranger-plugins-cred</include> + <include>security_plugins.ranger-plugins-impl:ranger-plugins-impl</include> + <include>security_plugins.ranger-plugins-common:ranger-plugins-common</include> + <include>security_plugins.ranger-kms-plugin:ranger-kms-plugin</include> + <include>org.apache.ranger:credentialbuilder</include> + <include>org.apache.ranger:ranger-kms-plugin</include> + </includes> + </moduleSet> + <moduleSet> + <binaries> + <includeDependencies>false</includeDependencies> + <outputDirectory>/scripts/install/lib</outputDirectory> + <unpack>false</unpack> + </binaries> + <includes> + <include>security_plugins.ranger-plugins-installer:ranger-plugins-installer</include> + <include>org.apache.ranger:credentialbuilder</include> + </includes> + </moduleSet> + <!-- End Plugin --> + <moduleSet> + <binaries> + <includeDependencies>false</includeDependencies> + <outputDirectory>/cred/lib</outputDirectory> + <unpack>false</unpack> + <directoryMode>755</directoryMode> + <fileMode>644</fileMode> + <dependencySets> + <dependencySet> + <outputDirectory>/cred/lib</outputDirectory> + <unpack>false</unpack> + <includes> + <include>commons-cli:commons-cli</include> + <include>commons-collections:commons-collections</include> + <include>commons-configuration:commons-configuration</include> + <include>commons-io:commons-io:jar:${commons.io.version}</include> + <include>commons-lang:commons-lang</include> + <include>commons-logging:commons-logging</include> + <include>com.google.guava:guava</include> + <include>org.hamcrest:hamcrest-all</include> + <include>junit:junit</include> + <include>org.slf4j:slf4j-api</include> + <include>org.apache.hadoop:hadoop-common</include> + <include>org.apache.hadoop:hadoop-auth</include> + <include>org.apache.htrace:htrace-core</include> + </includes> + </dependencySet> + </dependencySets> + </binaries> + <includes> + <include>org.apache.ranger:credentialbuilder</include> + </includes> + </moduleSet> + </moduleSets> + <fileSets> <fileSet> <outputDirectory>/ews/webapp/WEB-INF/classes/conf.dist</outputDirectory> @@ -174,22 +281,22 @@ <fileMode>700</fileMode> </fileSet> <fileSet> - <outputDirectory>/scripts/db/mysql</outputDirectory> + <outputDirectory>/db</outputDirectory> <directoryMode>755</directoryMode> - <directory>kms/scripts/db/mysql</directory> - <includes> - <include>*</include> - </includes> + <directory>kms/scripts/db</directory> <fileMode>544</fileMode> </fileSet> <fileSet> - <outputDirectory>/scripts</outputDirectory> + <outputDirectory>/</outputDirectory> <directory>kms/scripts</directory> <includes> + <include>dba_script.py</include> <include>db_setup.py</include> + <include>update_property.py</include> <include>ranger-kms</include> <include>ranger-kms-services.sh</include> <include>setup.sh</include> + <include>install.properties</include> <include>migrate-keystore-dbks.sh</include> </includes> <fileMode>544</fileMode> @@ -206,5 +313,54 @@ <directory>kms/config/webserver</directory> <fileMode>0700</fileMode> </fileSet> + <!-- Plugin --> + <!-- conf.templates for enable --> + <fileSet> + <outputDirectory>/scripts/install/conf.templates/enable</outputDirectory> + <directory>plugin-kms/conf</directory> + <excludes> + <exclude>*.sh</exclude> + </excludes> + <fileMode>700</fileMode> + </fileSet> + <fileSet> + <outputDirectory>/scripts/install/conf.templates/disable</outputDirectory> + <directory>plugin-kms/disable-conf</directory> + <fileMode>700</fileMode> + </fileSet> + <fileSet> + <outputDirectory>/scripts/install/conf.templates/default</outputDirectory> + <directory>plugin-kms/template</directory> + <fileMode>700</fileMode> + </fileSet> + <!-- version file --> + <fileSet> + <outputDirectory>/</outputDirectory> + <directory>${project.build.outputDirectory}</directory> + <includes> + <include>version</include> + </includes> + <fileMode>444</fileMode> + </fileSet> </fileSets> + <files> + <file> + <source>plugin-kms/scripts/enable-kms-plugin.sh</source> + <outputDirectory>/scripts</outputDirectory> + <destName>enable-kms-plugin.sh</destName> + <fileMode>755</fileMode> + </file> + <file> + <source>plugin-kms/scripts/install.properties</source> + <outputDirectory>/ews/webapp/config</outputDirectory> + <destName>plugin-install.properties</destName> + <fileMode>755</fileMode> + </file> + <file> + <source>plugin-kms/scripts/kms-plugin-install.properties</source> + <outputDirectory>/ews/webapp/config</outputDirectory> + <destName>kms-plugin-install.properties</destName> + <fileMode>755</fileMode> + </file> + </files> </assembly>
