Repository: incubator-ranger Updated Branches: refs/heads/master 575fc1b43 -> 0f3ace824
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/0f3ace82/migration-util/bin/ranger_knox_plugin_install.properties ---------------------------------------------------------------------- diff --git a/migration-util/bin/ranger_knox_plugin_install.properties b/migration-util/bin/ranger_knox_plugin_install.properties new file mode 100755 index 0000000..d821c5d --- /dev/null +++ b/migration-util/bin/ranger_knox_plugin_install.properties @@ -0,0 +1,109 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Location of Policy Manager URL +# +# Example: +# POLICY_MGR_URL=http://policymanager.xasecure.net:6080 +# +POLICY_MGR_URL= + +# +# Location of db client library (please check the location of the jar file) +# +# Example: +# SQL_CONNECTOR_JAR=/usr/share/java/mysql-connector-java.jar +# SQL_CONNECTOR_JAR=/usr/share/java/ojdbc6.jar +# +SQL_CONNECTOR_JAR=/usr/share/java/mysql-connector-java.jar + +# +# This is the repository name created within policy manager +# +# Example: +# REPOSITORY_NAME=knoxdev +# +REPOSITORY_NAME= + +# KNOX_HOME directory, would contain conf/, ext/ subdirectories +KNOX_HOME= + + +# AUDIT DB Configuration +# +# This information should match with the one you specified during the PolicyManager Installation +# +# Example: +# XAAUDIT.DB.IS_ENABLED=true +# XAAUDIT.DB.FLAVOUR=MYSQL +# XAAUDIT.DB.FLAVOUR=ORACLE +# XAAUDIT.DB.HOSTNAME=localhost +# XAAUDIT.DB.DATABASE_NAME=ranger_audit +# XAAUDIT.DB.USER_NAME=rangerlogger +# XAAUDIT.DB.PASSWORD=rangerlogger +# +XAAUDIT.DB.IS_ENABLED=false +XAAUDIT.DB.FLAVOUR=MYSQL +XAAUDIT.DB.HOSTNAME= +XAAUDIT.DB.DATABASE_NAME= +XAAUDIT.DB.USER_NAME= +XAAUDIT.DB.PASSWORD= + +# +# Audit to HDFS Configuration +# +# If XAAUDIT.HDFS.IS_ENABLED is set to true, please replace tokens +# that start with __REPLACE__ with appropriate values +# XAAUDIT.HDFS.IS_ENABLED=true +# XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://__REPLACE__NAME_NODE_HOST:8020/ranger/audit/%app-type%/%time:yyyyMMdd% +# XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=__REPLACE__LOG_DIR/knox/audit +# XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=__REPLACE__LOG_DIR/knox/audit/archive +# +# Example: +# XAAUDIT.HDFS.IS_ENABLED=true +# XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://namenode.example.com:8020/ranger/audit/%app-type%/%time:yyyyMMdd% +# XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=/var/log/knox/audit +# XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=/var/log/knox/audit/archive +# +XAAUDIT.HDFS.IS_ENABLED=false +XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://__REPLACE__NAME_NODE_HOST:8020/ranger/audit/%app-type%/%time:yyyyMMdd% +XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=__REPLACE__LOG_DIR/knox/audit +XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=__REPLACE__LOG_DIR/knox/audit/archive + +XAAUDIT.HDFS.DESTINTATION_FILE=%hostname%-audit.log +XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS=900 +XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS=86400 +XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS=60 +XAAUDIT.HDFS.LOCAL_BUFFER_FILE=%time:yyyyMMdd-HHmm.ss%.log +XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60 +XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600 +XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10 + +# +# SSL Client Certificate Information +# +# Example: +# SSL_KEYSTORE_FILE_PATH=/etc/knox/conf/ranger-plugin-keystore.jks +# SSL_KEYSTORE_PASSWORD=none +# SSL_TRUSTSTORE_FILE_PATH=/etc/knox/conf/ranger-plugin-truststore.jks +# SSL_TRUSTSTORE_PASSWORD=none +# +# You do not need use SSL between agent and security admin tool, please leave these sample value as it is. +# +SSL_KEYSTORE_FILE_PATH=/etc/knox/conf/ranger-plugin-keystore.jks +SSL_KEYSTORE_PASSWORD=myKeyFilePassword +SSL_TRUSTSTORE_FILE_PATH=/etc/knox/conf/ranger-plugin-truststore.jks +SSL_TRUSTSTORE_PASSWORD=changeit http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/0f3ace82/migration-util/bin/ranger_storm_plugin_install.properties ---------------------------------------------------------------------- diff --git a/migration-util/bin/ranger_storm_plugin_install.properties b/migration-util/bin/ranger_storm_plugin_install.properties new file mode 100755 index 0000000..28a0fd7 --- /dev/null +++ b/migration-util/bin/ranger_storm_plugin_install.properties @@ -0,0 +1,106 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Location of Policy Manager URL +# +# Example: +# POLICY_MGR_URL=http://policymanager.xasecure.net:6080 +# +POLICY_MGR_URL= + +# +# Location of db client library (please check the location of the jar file) +# +# Example: +# SQL_CONNECTOR_JAR=/usr/share/java/mysql-connector-java.jar +# SQL_CONNECTOR_JAR=/usr/share/java/ojdbc6.jar +# +SQL_CONNECTOR_JAR=/usr/share/java/mysql-connector-java.jar + +# +# This is the repository name created within policy manager +# +# Example: +# REPOSITORY_NAME=stormdev +# +REPOSITORY_NAME= + +# +# AUDIT DB Configuration +# +# This information should match with the one you specified during the PolicyManager Installation +# +# Example: +# XAAUDIT.DB.IS_ENABLED=true +# XAAUDIT.DB.FLAVOUR=MYSQL +# XAAUDIT.DB.FLAVOUR=ORACLE +# XAAUDIT.DB.HOSTNAME=localhost +# XAAUDIT.DB.DATABASE_NAME=ranger_audit +# XAAUDIT.DB.USER_NAME=rangerlogger +# XAAUDIT.DB.PASSWORD=rangerlogger +# +XAAUDIT.DB.IS_ENABLED=false +XAAUDIT.DB.FLAVOUR=MYSQL +XAAUDIT.DB.HOSTNAME= +XAAUDIT.DB.DATABASE_NAME= +XAAUDIT.DB.USER_NAME= +XAAUDIT.DB.PASSWORD= + +# +# Audit to HDFS Configuration +# +# If XAAUDIT.HDFS.IS_ENABLED is set to true, please replace tokens +# that start with __REPLACE__ with appropriate values +# XAAUDIT.HDFS.IS_ENABLED=true +# XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://__REPLACE__NAME_NODE_HOST:8020/ranger/audit/%app-type%/%time:yyyyMMdd% +# XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=__REPLACE__LOG_DIR/storm/audit +# XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=__REPLACE__LOG_DIR/storm/audit/archive +# +# Example: +# XAAUDIT.HDFS.IS_ENABLED=true +# XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://namenode.example.com:8020/ranger/audit/%app-type%/%time:yyyyMMdd% +# XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=/var/log/storm/audit +# XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=/var/log/storm/audit/archive +# +XAAUDIT.HDFS.IS_ENABLED=false +XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://__REPLACE__NAME_NODE_HOST:8020/ranger/audit/%app-type%/%time:yyyyMMdd% +XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=__REPLACE__LOG_DIR/storm/audit +XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=__REPLACE__LOG_DIR/storm/audit/archive + +XAAUDIT.HDFS.DESTINTATION_FILE=%hostname%-audit.log +XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS=900 +XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS=86400 +XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS=60 +XAAUDIT.HDFS.LOCAL_BUFFER_FILE=%time:yyyyMMdd-HHmm.ss%.log +XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60 +XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600 +XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10 + +# +# SSL Client Certificate Information +# +# Example: +# SSL_KEYSTORE_FILE_PATH=/etc/storm/conf/ranger-plugin-keystore.jks +# SSL_KEYSTORE_PASSWORD=none +# SSL_TRUSTSTORE_FILE_PATH=/etc/storm/conf/ranger-plugin-truststore.jks +# SSL_TRUSTSTORE_PASSWORD=none +# +# You do not need use SSL between agent and security admin tool, please leave these sample value as it is. +# +SSL_KEYSTORE_FILE_PATH=/etc/storm/conf/ranger-plugin-keystore.jks +SSL_KEYSTORE_PASSWORD=myKeyFilePassword +SSL_TRUSTSTORE_FILE_PATH=/etc/storm/conf/ranger-plugin-truststore.jks +SSL_TRUSTSTORE_PASSWORD=changeit http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/0f3ace82/migration-util/bin/ranger_usersync_install.properties ---------------------------------------------------------------------- diff --git a/migration-util/bin/ranger_usersync_install.properties b/migration-util/bin/ranger_usersync_install.properties new file mode 100755 index 0000000..63e4ffe --- /dev/null +++ b/migration-util/bin/ranger_usersync_install.properties @@ -0,0 +1,104 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# The following URL should be the base URL for connecting to the policy manager web application +# For example: +# +# POLICY_MGR_URL = http://policymanager.xasecure.net:6080 +# +POLICY_MGR_URL = + +# sync source, only unix and ldap are supported at present +# defaults to unix +SYNC_SOURCE = + + +# +# Minumum Unix User-id to start SYNC. +# This should avoid creating UNIX system-level users in the Policy Manager +# +MIN_UNIX_USER_ID_TO_SYNC = 1000 + +# sync interval in minutes +# user, groups would be synced again at the end of each sync interval +# defaults to 5 if SYNC_SOURCE is unix +# defaults to 360 if SYNC_SOURCE is ldap +SYNC_INTERVAL = + +#User and group for the usersync process +unix_user=ranger +unix_group=ranger + + +# --------------------------------------------------------------- +# The following properties are relevant only if SYNC_SOURCE = ldap +# --------------------------------------------------------------- + +# URL of source ldap +# a sample value would be: ldap://ldap.example.com:389 +# Must specify a value if SYNC_SOURCE is ldap +SYNC_LDAP_URL = + +# ldap bind dn used to connect to ldap and query for users and groups +# a sample value would be cn=admin,ou=users,dc=hadoop,dc=apache,dc-org +# Must specify a value if SYNC_SOURCE is ldap +SYNC_LDAP_BIND_DN = + +# ldap bind password for the bind dn specified above +# please ensure read access to this file is limited to root, to protect the password +# Must specify a value if SYNC_SOURCE is ldap +# unless anonymous search is allowed by the directory on users and group +SYNC_LDAP_BIND_PASSWORD = +CRED_KEYSTORE_FILENAME=/usr/lib/xausersync/.jceks/xausersync.jceks +# search base for users +# sample value would be ou=users,dc=hadoop,dc=apache,dc=org +SYNC_LDAP_USER_SEARCH_BASE = + +# search scope for the users, only base, one and sub are supported values +# please customize the value to suit your deployment +# default value: sub +SYNC_LDAP_USER_SEARCH_SCOPE = sub + +# objectclass to identify user entries +# please customize the value to suit your deployment +# default value: person +SYNC_LDAP_USER_OBJECT_CLASS = person + +# optional additional filter constraining the users selected for syncing +# a sample value would be (dept=eng) +# please customize the value to suit your deployment +# default value is empty +SYNC_LDAP_USER_SEARCH_FILTER = + +# attribute from user entry that would be treated as user name +# please customize the value to suit your deployment +# default value: cn +SYNC_LDAP_USER_NAME_ATTRIBUTE = cn + +# attribute from user entry whose values would be treated as +# group values to be pushed into Policy Manager database +# You could provide multiple attribute names separated by comma +# default value: memberof, ismemberof +SYNC_LDAP_USER_GROUP_NAME_ATTRIBUTE = memberof,ismemberof +# +# UserSync - Case Conversion Flags +# possible values: none, lower, upper +SYNC_LDAP_USERNAME_CASE_CONVERSION=lower +SYNC_LDAP_GROUPNAME_CASE_CONVERSION=lower + +#user sync log path +logdir=logs +#/var/log/ranger/usersync http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/0f3ace82/migration-util/doc/README.TXT ---------------------------------------------------------------------- diff --git a/migration-util/doc/README.TXT b/migration-util/doc/README.TXT new file mode 100644 index 0000000..2e04f02 --- /dev/null +++ b/migration-util/doc/README.TXT @@ -0,0 +1,76 @@ +Utility to Migrate Ranger Config into Ambari (from a previous manual install) +----------------------------------------------------------------------------- + +Pre-requisites: +---------------- + * Ambari Server (2.0.0) installed on one of the host of the cluster. + * Ranger service is manually installed outside Ambari. + * If user had Ambari 1.7.0 installed, Ambari needs to be upgraded to 2.0.0, before this script is run. + * Ranger service is expected to be up and running. + * Whichever component plugin needs to be imported, that plugin needs to be installed as well as enabled independently by the user. + * The installation folder paths for each component is as follows : + * For Ranger Admin service: /usr/hdp/<hdp_vers>/ranger-admin + * For Ranger Usersync service: /usr/hdp/<hdp_vers>/ranger-usersync + * For Ranger Hbase plugin: /usr/hdp/<hdp_vers>/ranger-hbase-plugin + * For Ranger Hdfs plugin: /usr/hdp/<hdp_vers>/ranger-hdfs-plugin + * For Ranger Hive plugin: /usr/hdp/<hdp_vers>/ranger-hive-plugin + * For Ranger Knox plugin: /usr/hdp/<hdp_vers>/ranger-knox-plugin + * For Ranger Storm plugin: /usr/hdp/<hdp_vers>/ranger-storm-plugin + +Pre Install: +------------ + * Please take a backup of existing ranger DB and configurations, in case required. + * Please take a backup of existing ambari DB and configurations, in case required. + * You must have an existing MySQL Server or Oracle Server database instance running to be used by Ranger. + * Ensure that the access for the DB Admin user (root in case of MySQL or SYS in case of Oracle) is enabled in DB server from any host. + * Execute the following command on the Ambari Server host. + * Replace database-type with mysql or oracle and /jdbc/driver/path based on the location of the MySQL or Oracle JDBC driver: + + ambari-server setup --jdbc-db={database-type} --jdbc-driver={/jdbc/driver/path} + +Steps to Add Ranger in a Ambari HDP cluster: +-------------------------------------------- + * In order to know about usage of the script, Execute the command: + python import_ranger_to_ambari.py + * Migration script locates the existing config in post-install properties files used by the service. + * This script will collect required configs from the independently installed Ranger service configurations + and post it to the specified Ambari Server. + * Set appropriate values in the respective install.properties template file, only if required. + For Ranger Admin service, ranger_admin_install.properties + For Ranger Usersync service, ranger_usersync_install.properties + For Ranger Hbase plugin, ranger_hbase_plugin_install.properties + For Ranger Hdfs plugin, ranger_hdfs_plugin_install.properties + For Ranger Hive plugin, ranger_hive_plugin_install.properties + For Ranger Knox plugin, ranger_knox_plugin_install.properties + For Ranger Storm plugin, ranger_storm_plugin_install.properties + +To import Ranger (Admin and User-Sync) service and plugins we need to run the python script using command line as shown below, +with valid input parameters, from the host where specific Ranger component has been installed / enabled : + +python import_ranger_to_ambari.py {install option} {ambari server url} {ambari server admin username:password} {cluster name} {FQDN of host having Ranger Admin or Ranger Usersync or plugins installed} + +for example the actual command will be as : + +python import_ranger_to_ambari.py 1 http://100.100.100.100:8080 admin:admin ambari_cluster rangerambari-feb09-rhel6-mp-sec-6.cs1cloud.internal + +First parameter (install option) is to mention the service type to be ported to Ambari that is, + 1 for adding Ranger service and Ranger Admin component to Ambari. + 2 for adding Ranger User sync component to Ambari. + 3 to import Ranger Hdfs Plugin configs to Ambari. + 4 to import Ranger Hive Plugin configs to Ambari. + 5 to import Ranger Hbase Plugin configs to Ambari. + 6 to import Ranger Knox Plugin configs to Ambari. + 7 to import Ranger Storm Plugin configs to Ambari. + + * After running the script with first parameter as 1, Ranger service should be visible as Ambari service and Ranger-Admin should be visible. + * After running with 2 Ranger-Usersync should also be visible as its component, like-wise for the respective plugins. + * After executing the script with options 3 to 7 - please visit Ambari UI and restart the individual component, after the UI reflects the changes. + +Debugging: +---------- + * Make sure to back up the ranger config and DB prior to running the scripts. + * It is possible that the service-components may be added to Ambari but the installation may fail. In that case, look for the logs from Ambari-UI. + * If any of the pre-requisite is not met this might happen, in that case, try installation from Ambari-UI itself. + * If the services are configured and not started it may show up as Install Pending, reinstall both Admin and User-sync from Ambari UI, and then start the services. + * In case if the services are installed and in stopped state, restart them from Ambari-UI. + http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/0f3ace82/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index d3f4ec8..62d448e 100644 --- a/pom.xml +++ b/pom.xml @@ -351,6 +351,7 @@ <descriptor>src/main/assembly/admin-web.xml</descriptor> <descriptor>src/main/assembly/usersync.xml</descriptor> <descriptor>src/main/assembly/ranger-src.xml</descriptor> + <descriptor>src/main/assembly/migration-util.xml</descriptor> </descriptors> </configuration> </plugin> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/0f3ace82/src/main/assembly/migration-util.xml ---------------------------------------------------------------------- diff --git a/src/main/assembly/migration-util.xml b/src/main/assembly/migration-util.xml new file mode 100644 index 0000000..f2eda12 --- /dev/null +++ b/src/main/assembly/migration-util.xml @@ -0,0 +1,43 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<assembly> + <id>migration-util</id> + <formats> + <format>tar.gz</format> + <format>zip</format> + </formats> + <baseDirectory>${project.name}-${project.version}-migration-util</baseDirectory> + <includeBaseDirectory>true</includeBaseDirectory> + <fileSets> + <fileSet> + <directoryMode>755</directoryMode> + <fileMode>644</fileMode> + <outputDirectory>/</outputDirectory> + <directory>migration-util</directory> + </fileSet> + <fileSet> + <directoryMode>755</directoryMode> + <outputDirectory>/</outputDirectory> + <directory>${project.build.directory}</directory> + <includes> + <include>version</include> + </includes> + <fileMode>444</fileMode> + </fileSet> + </fileSets> +</assembly>
