RANGER-242: Ranger config migration script for ambari Signed-off-by: sneethiraj <[email protected]>
Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/0f3ace82 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/0f3ace82 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/0f3ace82 Branch: refs/heads/stack Commit: 0f3ace824a88e5b90d509646339deee961b87066 Parents: 575fc1b Author: vperiasamy <[email protected]> Authored: Tue Feb 10 17:03:30 2015 -0500 Committer: sneethiraj <[email protected]> Committed: Tue Feb 10 22:47:10 2015 -0500 ---------------------------------------------------------------------- migration-util/bin/import_ranger_to_ambari.py | 950 +++++++++++++++++++ .../bin/ranger_admin_install.properties | 166 ++++ .../bin/ranger_hbase_plugin_install.properties | 115 +++ .../bin/ranger_hdfs_plugin_install.properties | 106 +++ .../bin/ranger_hive_plugin_install.properties | 115 +++ .../bin/ranger_knox_plugin_install.properties | 109 +++ .../bin/ranger_storm_plugin_install.properties | 106 +++ .../bin/ranger_usersync_install.properties | 104 ++ migration-util/doc/README.TXT | 76 ++ pom.xml | 1 + src/main/assembly/migration-util.xml | 43 + 11 files changed, 1891 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/0f3ace82/migration-util/bin/import_ranger_to_ambari.py ---------------------------------------------------------------------- diff --git a/migration-util/bin/import_ranger_to_ambari.py b/migration-util/bin/import_ranger_to_ambari.py new file mode 100644 index 0000000..b68becb --- /dev/null +++ b/migration-util/bin/import_ranger_to_ambari.py @@ -0,0 +1,950 @@ +import os +import sys +import urllib2 +import base64 +import httplib +import json +import time +from xml.etree import ElementTree as ET +import datetime +import logging +from subprocess import Popen,PIPE + + +def port_ranger_installation_to_ambari(): + print('Preparing advanged configurations for Ranger') + flag_ranger_admin_present,ranger_admin_properties_from_file = get_ranger_admin_install_properties() + if flag_ranger_admin_present: + print('Ranger Admin service is installed, making configurations as required by Ambari.') + if create_ranger_service_in_ambari(): + print('Ranger service is added sucessfully in Ambari') + if create_ranger_service_components_in_ambari('RANGER_ADMIN'): + print('Ranger service component is added successfully in Ambari') + if register_ranger_admin_host_in_ambari(): + print('Ranger admin host is registered successfully in Ambari') + if add_advanced_ranger_configurations(1,ranger_admin_properties_from_file): + print('Ranger advanced configurations added successfully in Ambari, kindly run Ranger-Usersync to complete Ranger service install') + else: + print('Ranger advanced configurations added failed in Ambari') + else: + print('Ranger admin host registration failed in Ambari') + else: + print('Ranger service component add failed in Ambari') + else: + print('Ranger service add failed in Ambari') + else: + print( 'Ranger Admin and Usersync services are not installed, not importing configurations to Ambari.') + + +def port_ranger_usersync_installation_to_ambari(): + print ('Preparing configurations for Ranger User-sync') + flag_ranger_usersync_present,ranger_usersync_properties_from_file = get_ranger_usersync_install_properties() + if flag_ranger_usersync_present: + print('Ranger Usersync service is installed, making configurations as required by Ambari.') + if create_ranger_service_components_in_ambari('RANGER_USERSYNC'): + print('Ranger service component is added successfully in Ambari') + if register_ranger_usersync_host_in_ambari(): + print('Ranger usersync host is registered successfully in Ambari') + if add_advanced_ranger_configurations(2, ranger_usersync_properties_from_file): + print('Ranger advanced configurations added successfully in Ambari') + if call_ranger_installed(): + print('Ranger service installed successfully in Ambari.') + else: + print('Ranger service install failed in Ambari') + else: + print('Ranger advanced configurations added failed in Ambari') + else: + print('Ranger usersync host registration failed in Ambari') + else: + print('Ranger service add failed in Ambari') + else: + print( 'Ranger Admin and Usersync services are not installed, not importing configurations to Ambari.') + + +def create_ranger_service_in_ambari(): + print('Creating ranger service in ambari') + ranger_create_url = ambari_service_url + '/' + ranger_service_name + request_result = call_ambari_api(ranger_create_url,'POST',ambari_username_password,'') + if request_result is not None: + response_code = request_result.getcode() + response = json.loads(json.JSONEncoder().encode(request_result.read())) + if (response_code == 201 and response is not None): + print('Ranger service created successfully in Ambari.') + return True + elif(response_code == 409 and response is not None): + print('Ranger service is alread created in Ambari.') + return False + else: + print('Ranger service creation failed in Ambari.') + return False + +def create_ranger_service_components_in_ambari(ranger_service_component_name): + print('adding Ranger servcie components in Ambari') + ranger_service_components = '{"components":[{"ServiceComponentInfo":{"component_name":"'+ ranger_service_component_name +'"}}]}' + print('creating ranger service in ambari') + ranger_service_component_create_url = ambari_service_url + '?ServiceInfo/service_name=' + ranger_service_name + request_result = call_ambari_api(ranger_service_component_create_url,'POST',ambari_username_password,ranger_service_components) + if request_result is not None: + response_code = request_result.getcode() + response = json.loads(json.JSONEncoder().encode(request_result.read())) + if (response_code == 201 and response is not None): + print('Ranger service component : ' + ranger_service_component_name + ', created successfully in Ambari.') + return True + elif (response_code == 409 and response is not None) : + print('Ranger service component : ' + ranger_service_component_name + ', is already present in Ambari.') + return False + else: + print('Ranger service component creation for : ' + ranger_service_component_name + ', failed in Ambari.') + return False + +def register_ranger_admin_host_in_ambari(): + print('adding Ranger servcie components in Ambari') + print('creating ranger admin service in ambari') + ranger_service_component_create_url = ambari_cluster_url + '/hosts/' + ranger_admin_fqdn + '/host_components/' + admin_component_name + request_result = call_ambari_api(ranger_service_component_create_url,'POST',ambari_username_password, '') + if request_result is not None: + response_code = request_result.getcode() + response = json.loads(json.JSONEncoder().encode(request_result.read())) + if response_code == 201 and response is not None: + print('Ranger admin host registered successfully in Ambari.') + return True + else: + print('Ranger admin host registration failed in Ambari.') + return False + +def register_ranger_usersync_host_in_ambari(): + print('adding Ranger servcie components in Ambari') + print('creating ranger admin service in ambari') + ranger_host_register_url = ambari_cluster_url + '/hosts/' + ranger_admin_fqdn + '/host_components/' + usersync_component_name + request_result = call_ambari_api(ranger_host_register_url,'POST',ambari_username_password, '') + if request_result is not None: + response_code = request_result.getcode() + response = json.loads(json.JSONEncoder().encode(request_result.read())) + if (response_code == 201 and response is not None): + print('Ranger usersync host registered successfully in Ambari.') + return True + elif(response_code == 409 and response is not None): + print('Ranger usersync host is already registered Ambari.') + return False + else: + print('Ranger usersync host registration failed in Ambari.') + return False + + +def add_advanced_ranger_configurations(add_admin_or_usersync, ranger_service_properties_from_file): + print( 'creating advanced configurations to be added to Ambari.') + ranger_config_data = '' + advanced_admin_properties = dict() + advanced_ranger_site_properties = dict() + advanced_ranger_env_properties = dict() + advanced_user_sync_properties = dict() + date_time_stamp = getDateTimeNow() + + if(add_admin_or_usersync == 1): + ranger_service_properties_from_file = get_db_flavour(ranger_service_properties_from_file) + advanced_admin_properties['DB_FLAVOR'] = ranger_service_properties_from_file['DB_FLAVOR'] + advanced_admin_properties['SQL_COMMAND_INVOKER'] = ranger_service_properties_from_file['SQL_COMMAND_INVOKER'] + advanced_admin_properties['SQL_CONNECTOR_JAR'] = ranger_service_properties_from_file['SQL_CONNECTOR_JAR'] + advanced_admin_properties['db_root_user'] = ranger_service_properties_from_file['db_root_user'] + advanced_admin_properties['db_root_password'] = ranger_service_properties_from_file['db_root_password'] + advanced_admin_properties['db_host'] = ranger_service_properties_from_file['db_host'] + advanced_admin_properties['db_name'] = ranger_service_properties_from_file['db_name'] + advanced_admin_properties['db_user'] = ranger_service_properties_from_file['db_user'] + advanced_admin_properties['db_password'] = ranger_service_properties_from_file['db_password'] + advanced_admin_properties['audit_db_name'] = ranger_service_properties_from_file['audit_db_name'] + advanced_admin_properties['audit_db_user'] = ranger_service_properties_from_file['audit_db_user'] + advanced_admin_properties['audit_db_password'] = ranger_service_properties_from_file['audit_db_password'] + advanced_admin_properties['policymgr_external_url'] = ranger_service_properties_from_file['xa.webapp.url.root'] + advanced_admin_properties['policymgr_http_enabled'] = ranger_service_properties_from_file['http.enabled'] + advanced_admin_properties['authentication_method'] = ranger_service_properties_from_file['authentication_method'] + advanced_admin_properties['remoteLoginEnabled'] = ranger_service_properties_from_file['remoteLoginEnabled'] + advanced_admin_properties['authServiceHostName'] = ranger_service_properties_from_file['authServiceHostName'] + advanced_admin_properties['authServicePort'] = ranger_service_properties_from_file['authServicePort'] + advanced_admin_properties['xa_ldap_url'] = ranger_service_properties_from_file['xa_ldap_url'] + advanced_admin_properties['xa_ldap_userDNpattern'] = ranger_service_properties_from_file['xa_ldap_userDNpattern'] + advanced_admin_properties['xa_ldap_groupSearchBase'] = ranger_service_properties_from_file['xa_ldap_groupSearchBase'] + advanced_admin_properties['xa_ldap_groupSearchFilter'] = ranger_service_properties_from_file['xa_ldap_groupSearchFilter'] + advanced_admin_properties['xa_ldap_groupRoleAttribute'] = ranger_service_properties_from_file['xa_ldap_groupRoleAttribute'] + advanced_admin_properties['xa_ldap_ad_domain'] = ranger_service_properties_from_file['xa_ldap_ad_domain'] + advanced_admin_properties['xa_ldap_ad_url'] = ranger_service_properties_from_file['xa_ldap_ad_url'] + + advanced_ranger_site_properties['HTTP_SERVICE_PORT'] = ranger_service_properties_from_file['http.service.port'] + advanced_ranger_site_properties['HTTPS_SERVICE_PORT'] = ranger_service_properties_from_file['https.service.port'] + advanced_ranger_site_properties['HTTPS_KEYSTORE_FILE'] = ranger_service_properties_from_file['https.attrib.keystoreFile'] + advanced_ranger_site_properties['HTTPS_KEYSTORE_PASS'] = ranger_service_properties_from_file['https.attrib.keystorePass'] + advanced_ranger_site_properties['HTTPS_KEY_ALIAS'] = ranger_service_properties_from_file['https.attrib.keyAlias'] + advanced_ranger_site_properties['HTTPS_CLIENT_AUTH'] = ranger_service_properties_from_file['https.attrib.clientAuth'] + advanced_ranger_site_properties['HTTP_ENABLED'] = ranger_service_properties_from_file['policymgr_http_enabled'] + + + advanced_ranger_env_properties['ranger_user'] = 'ranger' + advanced_ranger_env_properties['ranger_group'] = 'ranger' + advanced_ranger_env_properties['ranger_admin_log_dir'] = '/var/log/ranger/admin' + advanced_ranger_env_properties['ranger_usersync_log_dir'] = '/var/log/ranger/usersync' + advanced_ranger_env_properties['ranger_admin_username'] = 'amb_ranger_admin' + advanced_ranger_env_properties['ranger_admin_password'] = 'ambari123' + advanced_ranger_env_properties['admin_password'] = 'admin' + + + ranger_config_data = '[{"Clusters":{"desired_config":[{"type":"admin-properties","tag":"' + str(date_time_stamp) + '","properties":' + json.dumps(advanced_admin_properties) + ', "properties_attributes": {"final": "true"}},{"type":"ranger-site","tag":"' + str(date_time_stamp) + '","properties":' + json.dumps(advanced_ranger_site_properties) + ', "properties_attributes": {"final": "false"}},{"type":"ranger-env","tag":"' + str(date_time_stamp) + '","properties":' + json.dumps(advanced_ranger_env_properties) + ', "properties_attributes": {"final": "false"}}]}}]' + + + if(add_admin_or_usersync == 2): + advanced_user_sync_properties['SYNC_SOURCE'] = ranger_service_properties_from_file['SYNC_SOURCE'] + advanced_user_sync_properties['MIN_UNIX_USER_ID_TO_SYNC'] = ranger_service_properties_from_file['usergroupSync.unix.minUserId'] + advanced_user_sync_properties['SYNC_INTERVAL'] = ranger_service_properties_from_file['usergroupSync.sleepTimeInMillisBetweenSyncCycle'] + advanced_user_sync_properties['SYNC_LDAP_URL'] = ranger_service_properties_from_file['ldapGroupSync.ldapUrl'] + advanced_user_sync_properties['SYNC_LDAP_BIND_DN'] = ranger_service_properties_from_file['ldapGroupSync.ldapBindDn'] + advanced_user_sync_properties['SYNC_LDAP_BIND_PASSWORD'] = ranger_service_properties_from_file['ldapGroupSync.ldapBindPassword'] + advanced_user_sync_properties['CRED_KEYSTORE_FILENAME'] = ranger_service_properties_from_file['ldapGroupSync.ldapBindKeystore'] + advanced_user_sync_properties['SYNC_LDAP_USER_SEARCH_BASE'] = ranger_service_properties_from_file['ldapGroupSync.userSearchBase'] + advanced_user_sync_properties['SYNC_LDAP_USER_SEARCH_SCOPE'] = ranger_service_properties_from_file['ldapGroupSync.userSearchScope'] + advanced_user_sync_properties['SYNC_LDAP_USER_OBJECT_CLASS'] = ranger_service_properties_from_file['ldapGroupSync.userObjectClass'] + advanced_user_sync_properties['SYNC_LDAP_USER_SEARCH_FILTER'] = ranger_service_properties_from_file['ldapGroupSync.userSearchFilter'] + advanced_user_sync_properties['SYNC_LDAP_USER_NAME_ATTRIBUTE'] = ranger_service_properties_from_file['ldapGroupSync.userNameAttribute'] + advanced_user_sync_properties['SYNC_LDAP_USER_GROUP_NAME_ATTRIBUTE'] = ranger_service_properties_from_file['ldapGroupSync.userGroupNameAttribute'] + advanced_user_sync_properties['SYNC_LDAP_USERNAME_CASE_CONVERSION'] = ranger_service_properties_from_file['ldapGroupSync.username.caseConversion'] + advanced_user_sync_properties['SYNC_LDAP_GROUPNAME_CASE_CONVERSION'] = ranger_service_properties_from_file['ldapGroupSync.groupname.caseConversion'] + advanced_user_sync_properties['logdir'] = ranger_service_properties_from_file['logdir'] + + ranger_config_data = '[{"Clusters":{"desired_config":[{"type":"usersync-properties","tag":"' + str(date_time_stamp) + '","properties":' + json.dumps(advanced_user_sync_properties) + ', "properties_attributes": {"final": "false"}}]}}]' + + ranger_config_request_url = ambari_cluster_url + request_result = call_ambari_api(ranger_config_request_url,'PUT',ambari_username_password,str(ranger_config_data)) + if request_result is not None: + response_code = request_result.getcode() + response = json.loads(json.JSONEncoder().encode(request_result.read())) + if response_code == 200 and response is not None: + print('Ranger advanced configuration added successfully in Ambari.') + return True + else: + print('Ranger advanced configuration add failed in Ambari.') + return False + +def get_db_flavour(ranger_admin_properties_from_file): + print('inside get_db_flavour getting db flavor, library and command invoker') + install_properties_db_flavour = ranger_admin_properties_from_file['DB_FLAVOR'] + print( 'install_properties_db_flavour = ' + install_properties_db_flavour) + xa_system_properties_db_dialect = ranger_admin_properties_from_file['jdbc.dialect'] + print( 'xa_system_properties_db_dialect = ' + xa_system_properties_db_dialect) + xa_system_properties_url = ranger_admin_properties_from_file['jdbc.url'] + print( 'xa_system_properties_url = ' + xa_system_properties_url) + if('mysql'.lower() in xa_system_properties_db_dialect.lower() and 'mysql'.lower() in xa_system_properties_url.lower()): + print('db dialect and jdbc url are set as MYSQL setting db_flavour and sql command invoker as mysql') + ranger_admin_properties_from_file['DB_FLAVOR'] = 'MYSQL' + ranger_admin_properties_from_file['SQL_COMMAND_INVOKER'] = 'mysql' + ranger_admin_properties_from_file['SQL_CONNECTOR_JAR'] = '/usr/share/java/mysql-connector-java.jar' + if('oracle'.lower() in xa_system_properties_db_dialect and 'oracle'.lower() in xa_system_properties_url.lower()): + print('db dialect and jdbc url are set as Oracle setting db_flavour and sql command invoker as oracle') + ranger_admin_properties_from_file['DB_FLAVOR'] = 'ORACLE' + ranger_admin_properties_from_file['SQL_COMMAND_INVOKER'] = 'sqlplus' + ranger_admin_properties_from_file['SQL_CONNECTOR_JAR'] = '/usr/share/java/ojdbc6.jar' + return ranger_admin_properties_from_file + + +def call_ranger_installed(): + print('Changing state of ranger services from init to installed.') + ranger_state_change_request = '{"RequestInfo":{"context":"Install Ranger Service","operation_level":{"level":"CLUSTER","cluster_name":"' + str(cluster_name) + '"}},"Body":{"ServiceInfo":{"state":"INSTALLED"}}}' + ranger_state_change_url = ambari_service_url + '?ServiceInfo/state=INIT' + request_result = call_ambari_api(ranger_state_change_url,'PUT',ambari_username_password, ranger_state_change_request) + if request_result is not None: + response_code = request_result.getcode() + response = json.loads(json.JSONEncoder().encode(request_result.read())) + if (response_code == 200 and response is not None): + print('Ranger state changed to install successfully in Ambari.') + return True + if (response_code == 409 and response is not None): + print('Ranger is already installed in Ambari.') + return False + if response_code == 202 and response is not None: + print('Ranger state changed to install posted in Ambari, checking for updated status waiting for 30 seconds') + parsed_response = json.loads(response) + response_href_url = parsed_response['href'] + response_request_id = parsed_response['Requests']['id'] + response_status = parsed_response['Requests']['status'] + if response_status != 'Installed': + print('Received response but status is not installed, verifying installation to be successful.') + flag_ranger_installed = True + time.sleep(30) + while flag_ranger_installed: + print('checking request status') + ambari_request_url = ambari_cluster_url + '/requests/' + str(response_request_id) + request_status_result = call_ambari_api(ambari_request_url ,'GET',ambari_username_password,'') + if request_status_result is not None: + response_code = request_status_result.getcode() + response_status = json.loads(json.JSONEncoder().encode(request_status_result.read())) + if (response_code != 200 and 'FAILED' not in response_status): + print('Ranger install failed in ambari.') + flag_ranger_installed = False + if (response_code == 200 and 'PENDING' in response_status): + print('Ranger install is pending in ambari.') + flag_ranger_installed = True + if (response_code == 200 and 'COMPLETED' in response_status): + print('Ranger installed successfully in ambari.') + flag_ranger_installed = False + else: + flag_ranger_installed = True + return not flag_ranger_installed + else: + print('Ranger state changed to install failed in Ambari.') + return False + + +def get_ranger_usersync_install_properties(): + print('preparing advanced configurations for Ranger User-sync') + flag_hadoop_present = check_hadoop_dir_present(hdp_current_dir) + flag_ranger_usersync_present = False + ranger_usersync_current_dir = os.path.join(hdp_current_dir , 'ranger-usersync') + ranger_usersync_properties_from_file = None + print('checking for ranger-usersync service to be present') + flag_ranger_usersync_present,ranger_usersync__installed_version = check_ranger_usersync_install(ranger_usersync_current_dir) + if flag_ranger_usersync_present: + print('ranger usersync is installed, getting existing properties for ambari import') + ranger_usersync_properties_from_file = import_properties_from_file("ranger_usersync_install.properties") + if ranger_usersync_properties_from_file is not None and len(ranger_usersync_properties_from_file) != 0: + print('got ranger usersync values from install.properties file, need to configure ambari for ranger service.') + ranger_usersync_properties_from_file = get_additional_properties_for_usersync(ranger_usersync_properties_from_file) + return flag_ranger_usersync_present,ranger_usersync_properties_from_file + + + + +def get_ranger_admin_install_properties(): + ranger_admin_current_dir = os.path.join(hdp_current_dir , 'ranger-admin') + flag_hadoop_present = check_hadoop_dir_present(hdp_current_dir) + flag_ranger_admin_present = False + ranger_admin_properties_from_file = None + + + if flag_hadoop_present : + print('Hadoop directory is present, checking ranger admin installation.') + flag_ranger_admin_present , ranger_admin_installed_version = check_ranger_admin_install(ranger_admin_current_dir) + + if flag_ranger_admin_present: + print('Ranger Admin is present.getting existing configurations to port to Ambari.') + if os.path.isfile("ranger_admin_install.properties"): + print('ranger_install_properties_path exists, getting existing properties for ambari port') + ranger_admin_properties_from_file = import_properties_from_file("ranger_admin_install.properties") + + if ranger_admin_properties_from_file is not None and len(ranger_admin_properties_from_file) != 0: + print('got ranger admin values from ranger_admin_install.properties file, need to configure ambari for ranger service.') + print('getting additional properties required by Ranger services') + ranger_admin_properties_from_file = get_additional_properties_for_admin(ranger_admin_properties_from_file) + + + return flag_ranger_admin_present,ranger_admin_properties_from_file + + + +def check_hadoop_dir_present(hdp_current_dir_path): + flag_hadoop_dir_present = False + if os.path.isdir(hdp_current_dir_path): + print('hadoop is installed.') + flag_hadoop_dir_present = True + return flag_hadoop_dir_present + +def check_ranger_admin_install(ranger_admin_current_dir): + flag_ranger_dir_present = False + ranger_current_installed_version = '' + print('checking ranger service path folder') + if os.path.isdir(ranger_admin_current_dir): + print('ranger admin is installed.') + if os.path.islink(ranger_admin_current_dir): + flag_ranger_dir_present = True + print('ranger admin link found getting current version from link.') + ranger_home_path = os.path.realpath(ranger_admin_current_dir) + ranger_current_installed_version = ranger_home_path.split('/')[4] + return flag_ranger_dir_present, ranger_current_installed_version + + +def check_ranger_usersync_install(ranger_usersync_current_dir): + flag_ranger_dir_present = False + ranger_current_installed_version = '' + print('checking ranger service path folder') + if os.path.isdir(ranger_usersync_current_dir): + print('Ranger User-sync is installed.') + if os.path.islink(ranger_usersync_current_dir): + flag_ranger_dir_present = True + print('ranger admin link found getting current version from link.') + ranger_home_path = os.path.realpath(ranger_usersync_current_dir) + ranger_current_installed_version = ranger_home_path.split('/')[4] + return flag_ranger_dir_present, ranger_current_installed_version + + +def get_additional_properties_for_admin(ranger_admin_properties_from_file): + ranger_conf_path = '/etc/ranger/admin/conf' + ranger_webserver_properties_path = os.path.join(ranger_conf_path, 'ranger_webserver.properties') + ranger_ldap_properties_path = os.path.join(ranger_conf_path,'xa_ldap.properties') + ranger_system_properties_path = os.path.join(ranger_conf_path,'xa_system.properties') + ranger_admin_properties_from_file = import_properties_from_file(ranger_webserver_properties_path,ranger_admin_properties_from_file) + ranger_admin_properties_from_file = import_properties_from_file(ranger_ldap_properties_path,ranger_admin_properties_from_file) + ranger_admin_properties_from_file = import_properties_from_file(ranger_system_properties_path,ranger_admin_properties_from_file) + return ranger_admin_properties_from_file + +def get_additional_properties_for_usersync(ranger_usersync_properties_from_file): + ranger_conf_path = '/etc/ranger/usersync/conf' + unix_auth_properties_path = os.path.join(ranger_conf_path, 'unixauthservice.properties') + ranger_usersync_properties_from_file = import_properties_from_file(unix_auth_properties_path,ranger_usersync_properties_from_file) + return ranger_usersync_properties_from_file + + +def port_hdfs_plugin_to_ambari(): + print('Trying to add ranger hdfs plugin.') + flag_hdfs_plugin_installed,hdfs_plugin_install_properties,hdfs_site_xml_properties = get_hdfs_plugin_configuration() + if flag_hdfs_plugin_installed and hdfs_plugin_install_properties is not None and hdfs_site_xml_properties is not None: + hdfs_site_xml_properties['dfs.permissions.enabled'] = 'true' + print('hdfs plugin is installed and enabled, adding to configurations') + advanced_ranger_hdfs_plugin_properties = dict() + + advanced_ranger_hdfs_plugin_properties['policy_user'] = 'ambari-qa' + advanced_ranger_hdfs_plugin_properties['hadoop.rpc.protection'] = '-' + advanced_ranger_hdfs_plugin_properties['common.name.for.certificate'] = '-' + advanced_ranger_hdfs_plugin_properties['ranger-hdfs-plugin-enabled'] = 'Yes' + advanced_ranger_hdfs_plugin_properties['REPOSITORY_CONFIG_USERNAME'] = 'hadoop' + advanced_ranger_hdfs_plugin_properties['REPOSITORY_CONFIG_PASSWORD'] = 'hadoop' + advanced_ranger_hdfs_plugin_properties['XAAUDIT.DB.IS_ENABLED'] = hdfs_plugin_install_properties['xasecure.audit.db.is.enabled'] + advanced_ranger_hdfs_plugin_properties['XAAUDIT.HDFS.IS_ENABLED'] = hdfs_plugin_install_properties['xasecure.audit.hdfs.is.enabled'] + advanced_ranger_hdfs_plugin_properties['XAAUDIT.HDFS.DESTINATION_DIRECTORY'] = hdfs_plugin_install_properties['xasecure.audit.hdfs.config.destination.directory'] + advanced_ranger_hdfs_plugin_properties['XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY'] = hdfs_plugin_install_properties['xasecure.audit.hdfs.config.local.buffer.directory'] + advanced_ranger_hdfs_plugin_properties['XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY'] = hdfs_plugin_install_properties['xasecure.audit.hdfs.config.local.archive.directory'] + advanced_ranger_hdfs_plugin_properties['XAAUDIT.HDFS.DESTINTATION_FILE'] = hdfs_plugin_install_properties['xasecure.audit.hdfs.config.destination.file'] + advanced_ranger_hdfs_plugin_properties['XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS'] = hdfs_plugin_install_properties['xasecure.audit.hdfs.config.destination.flush.interval.seconds'] + advanced_ranger_hdfs_plugin_properties['XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS'] = hdfs_plugin_install_properties['xasecure.audit.hdfs.config.destination.rollover.interval.seconds'] + advanced_ranger_hdfs_plugin_properties['XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS'] = hdfs_plugin_install_properties['xasecure.audit.hdfs.config.destination.open.retry.interval.seconds'] + advanced_ranger_hdfs_plugin_properties['XAAUDIT.HDFS.LOCAL_BUFFER_FILE'] = hdfs_plugin_install_properties['xasecure.audit.hdfs.config.local.buffer.file'] + advanced_ranger_hdfs_plugin_properties['XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS'] = hdfs_plugin_install_properties['xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds'] + advanced_ranger_hdfs_plugin_properties['XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS'] = hdfs_plugin_install_properties['xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds'] + advanced_ranger_hdfs_plugin_properties['XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT'] = hdfs_plugin_install_properties['xasecure.audit.hdfs.config.local.archive.max.file.count'] + advanced_ranger_hdfs_plugin_properties['SSL_KEYSTORE_FILE_PATH'] = hdfs_plugin_install_properties['xasecure.policymgr.clientssl.keystore'] + advanced_ranger_hdfs_plugin_properties['SSL_KEYSTORE_PASSWORD'] = hdfs_plugin_install_properties['xasecure.policymgr.clientssl.keystore.password'] + advanced_ranger_hdfs_plugin_properties['SSL_TRUSTSTORE_FILE_PATH'] = hdfs_plugin_install_properties['xasecure.policymgr.clientssl.truststore'] + advanced_ranger_hdfs_plugin_properties['SSL_TRUSTSTORE_PASSWORD'] = hdfs_plugin_install_properties['xasecure.policymgr.clientssl.truststore.password'] + + date_time_stamp = getDateTimeNow() + plugin_configuration_data = '[{"Clusters":{"desired_config":[{"type":"hdfs-site","tag":"' + str(date_time_stamp) + '","properties":' + str(json.dumps(hdfs_site_xml_properties)) + ', "properties_attributes": {"final": "false"}},{"type": "ranger-hdfs-plugin-properties", "tag": "' + str(date_time_stamp) + '", "properties":'+json.dumps(advanced_ranger_hdfs_plugin_properties)+',"properties_attributes": {"final": "false"}}]}}]' + + ranger_config_request_url = ambari_cluster_url + request_result = call_ambari_api(ranger_config_request_url,'PUT',ambari_username_password,str(plugin_configuration_data)) + if request_result is not None: + response_code = request_result.getcode() + response = json.loads(json.JSONEncoder().encode(request_result.read())) + if response_code == 200 and response is not None: + print('Ranger hdfs plugin configuration added successfully in Ambari.') + return True + else: + print('Ranger hdfs plugin configuration add failed in Ambari.') + return False + +def port_ranger_hive_plugin_to_ambari(): + print('Trying to add ranger hive plugin.') + flag_hive_plugin_installed,hive_plugin_install_properties,hive_server2_xml_properties = get_hive_plugin_configuration() + if flag_hive_plugin_installed and hive_plugin_install_properties is not None and hive_server2_xml_properties is not None: + hive_server2_xml_properties['hive.security.authorization.enabled'] = 'true' + hive_server2_xml_properties['hive.security.authorization.manager'] = 'com.xasecure.authorization.hive.authorizer.XaSecureHiveAuthorizerFactory' + hive_server2_xml_properties['hive.security.authenticator.manager'] = 'org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator' + hive_server2_xml_properties['hive.conf.restricted.list'] = 'hive.security.authorization.enabled, hive.security.authorization.manager,hive.security.authenticator.manager' + print('hive plugin is installed and enabled, adding to configurations') + advanced_ranger_hive_plugin_properties = dict() + + advanced_ranger_hive_plugin_properties['policy_user'] = 'ambari-qa' + advanced_ranger_hive_plugin_properties['jdbc.driverClassName'] = 'org.apache.hive.jdbc.HiveDriver' + advanced_ranger_hive_plugin_properties['common.name.for.certificate'] = '-' + advanced_ranger_hive_plugin_properties['ranger-hive-plugin-enabled'] = 'Yes' + advanced_ranger_hive_plugin_properties['REPOSITORY_CONFIG_USERNAME'] = 'hive' + advanced_ranger_hive_plugin_properties['REPOSITORY_CONFIG_PASSWORD'] = 'hive' + advanced_ranger_hive_plugin_properties['XAAUDIT.DB.IS_ENABLED'] = hive_plugin_install_properties['xasecure.audit.db.is.enabled'] + advanced_ranger_hive_plugin_properties['XAAUDIT.HDFS.IS_ENABLED'] = hive_plugin_install_properties['xasecure.audit.hdfs.is.enabled'] + advanced_ranger_hive_plugin_properties['XAAUDIT.HDFS.DESTINATION_DIRECTORY'] = hive_plugin_install_properties['xasecure.audit.hdfs.config.destination.directory'] + advanced_ranger_hive_plugin_properties['XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY'] = hive_plugin_install_properties['xasecure.audit.hdfs.config.local.buffer.directory'] + advanced_ranger_hive_plugin_properties['XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY'] = hive_plugin_install_properties['xasecure.audit.hdfs.config.local.archive.directory'] + advanced_ranger_hive_plugin_properties['XAAUDIT.HDFS.DESTINTATION_FILE'] = hive_plugin_install_properties['xasecure.audit.hdfs.config.destination.file'] + advanced_ranger_hive_plugin_properties['XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS'] = hive_plugin_install_properties['xasecure.audit.hdfs.config.destination.flush.interval.seconds'] + advanced_ranger_hive_plugin_properties['XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS'] = hive_plugin_install_properties['xasecure.audit.hdfs.config.destination.rollover.interval.seconds'] + advanced_ranger_hive_plugin_properties['XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS'] = hive_plugin_install_properties['xasecure.audit.hdfs.config.destination.open.retry.interval.seconds'] + advanced_ranger_hive_plugin_properties['XAAUDIT.HDFS.LOCAL_BUFFER_FILE'] = hive_plugin_install_properties['xasecure.audit.hdfs.config.local.buffer.file'] + advanced_ranger_hive_plugin_properties['XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS'] = hive_plugin_install_properties['xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds'] + advanced_ranger_hive_plugin_properties['XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS'] = hive_plugin_install_properties['xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds'] + advanced_ranger_hive_plugin_properties['XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT'] = hive_plugin_install_properties['xasecure.audit.hdfs.config.local.archive.max.file.count'] + advanced_ranger_hive_plugin_properties['SSL_KEYSTORE_FILE_PATH'] = hive_plugin_install_properties['xasecure.policymgr.clientssl.keystore'] + advanced_ranger_hive_plugin_properties['SSL_KEYSTORE_PASSWORD'] = hive_plugin_install_properties['xasecure.policymgr.clientssl.keystore.password'] + advanced_ranger_hive_plugin_properties['SSL_TRUSTSTORE_FILE_PATH'] = hive_plugin_install_properties['xasecure.policymgr.clientssl.truststore'] + advanced_ranger_hive_plugin_properties['SSL_TRUSTSTORE_PASSWORD'] = hive_plugin_install_properties['xasecure.policymgr.clientssl.truststore.password'] + advanced_ranger_hive_plugin_properties['UPDATE_XAPOLICIES_ON_GRANT_REVOKE'] = hive_plugin_install_properties['UPDATE_XAPOLICIES_ON_GRANT_REVOKE'] + + + date_time_stamp = getDateTimeNow() + plugin_configuration_data = '[{"Clusters":{"desired_config":[{"type":"hiveserver2-site","tag":"' + str(date_time_stamp) + '","properties":' + str(json.dumps(hive_server2_xml_properties)) + ', "properties_attributes": {"final": "false"}},{"type": "ranger-hive-plugin-properties","tag":"' + str(date_time_stamp) + '", "properties":'+json.dumps(advanced_ranger_hive_plugin_properties)+',"properties_attributes": {"final": "false"}}]}}]' + + ranger_config_request_url = ambari_cluster_url + request_result = call_ambari_api(ranger_config_request_url,'PUT',ambari_username_password,str(plugin_configuration_data)) + if request_result is not None: + response_code = request_result.getcode() + response = json.loads(json.JSONEncoder().encode(request_result.read())) + if response_code == 200 and response is not None: + print('Ranger hive plugin configuration added successfully in Ambari.') + return True + else: + print('Ranger hive plugin configuration add failed in Ambari.') + return False + + + +def port_ranger_hbase_plugin_to_ambari(): + print('Trying to add Ranger Hbase plugin.') + flag_hbase_plugin_installed,hbase_plugin_install_properties,hbase_site_xml_properties = get_hbase_plugin_configuration() + if flag_hbase_plugin_installed and hbase_plugin_install_properties is not None and hbase_site_xml_properties is not None: + hbase_site_xml_properties['hive.security.authorization.enabled'] = 'true' + hbase_site_xml_properties['hive.security.authorization.manager'] = 'com.xasecure.authorization.hive.authorizer.XaSecureHiveAuthorizerFactory' + hbase_site_xml_properties['hive.security.authenticator.manager'] = 'org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator' + hbase_site_xml_properties['hive.conf.restricted.list'] = 'hive.security.authorization.enabled, hive.security.authorization.manager,hive.security.authenticator.manager' + print('Hbase plugin is installed and enabled, adding to configurations') + advanced_ranger_hbase_plugin_properties = dict() + + advanced_ranger_hbase_plugin_properties['policy_user'] = 'ambari-qa' + advanced_ranger_hbase_plugin_properties['ranger-hbase-plugin-enabled'] = 'Yes' + advanced_ranger_hbase_plugin_properties['REPOSITORY_CONFIG_USERNAME'] = 'hbase' + advanced_ranger_hbase_plugin_properties['REPOSITORY_CONFIG_PASSWORD'] = 'hbase' + advanced_ranger_hbase_plugin_properties['XAAUDIT.DB.IS_ENABLED'] = hbase_plugin_install_properties['xasecure.audit.db.is.enabled'] + advanced_ranger_hbase_plugin_properties['XAAUDIT.HDFS.IS_ENABLED'] = hbase_plugin_install_properties['xasecure.audit.hdfs.is.enabled'] + advanced_ranger_hbase_plugin_properties['XAAUDIT.HDFS.DESTINATION_DIRECTORY'] = hbase_plugin_install_properties['xasecure.audit.hdfs.config.destination.directory'] + advanced_ranger_hbase_plugin_properties['XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY'] = hbase_plugin_install_properties['xasecure.audit.hdfs.config.local.buffer.directory'] + advanced_ranger_hbase_plugin_properties['XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY'] = hbase_plugin_install_properties['xasecure.audit.hdfs.config.local.archive.directory'] + advanced_ranger_hbase_plugin_properties['XAAUDIT.HDFS.DESTINTATION_FILE'] = hbase_plugin_install_properties['xasecure.audit.hdfs.config.destination.file'] + advanced_ranger_hbase_plugin_properties['XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS'] = hbase_plugin_install_properties['xasecure.audit.hdfs.config.destination.flush.interval.seconds'] + advanced_ranger_hbase_plugin_properties['XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS'] = hbase_plugin_install_properties['xasecure.audit.hdfs.config.destination.rollover.interval.seconds'] + advanced_ranger_hbase_plugin_properties['XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS'] = hbase_plugin_install_properties['xasecure.audit.hdfs.config.destination.open.retry.interval.seconds'] + advanced_ranger_hbase_plugin_properties['XAAUDIT.HDFS.LOCAL_BUFFER_FILE'] = hbase_plugin_install_properties['xasecure.audit.hdfs.config.local.buffer.file'] + advanced_ranger_hbase_plugin_properties['XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS'] = hbase_plugin_install_properties['xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds'] + advanced_ranger_hbase_plugin_properties['XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS'] = hbase_plugin_install_properties['xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds'] + advanced_ranger_hbase_plugin_properties['XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT'] = hbase_plugin_install_properties['xasecure.audit.hdfs.config.local.archive.max.file.count'] + advanced_ranger_hbase_plugin_properties['SSL_KEYSTORE_FILE_PATH'] = hbase_plugin_install_properties['xasecure.policymgr.clientssl.keystore'] + advanced_ranger_hbase_plugin_properties['SSL_KEYSTORE_PASSWORD'] = hbase_plugin_install_properties['xasecure.policymgr.clientssl.keystore.password'] + advanced_ranger_hbase_plugin_properties['SSL_TRUSTSTORE_FILE_PATH'] = hbase_plugin_install_properties['xasecure.policymgr.clientssl.truststore'] + advanced_ranger_hbase_plugin_properties['SSL_TRUSTSTORE_PASSWORD'] = hbase_plugin_install_properties['xasecure.policymgr.clientssl.truststore.password'] + advanced_ranger_hbase_plugin_properties['UPDATE_XAPOLICIES_ON_GRANT_REVOKE'] = hbase_plugin_install_properties['UPDATE_XAPOLICIES_ON_GRANT_REVOKE'] + + date_time_stamp = getDateTimeNow() + plugin_configuration_data = '[{"Clusters":{"desired_config":[{"type":"hbase-site","tag":"' + str(date_time_stamp) + '","properties":' + str(json.dumps(hbase_site_xml_properties)) + ', "properties_attributes": {"final": "false"}},{"type": "ranger-hbase-plugin-properties","tag":"' + str(date_time_stamp) + '", "properties":'+json.dumps(advanced_ranger_hbase_plugin_properties)+',"properties_attributes": {"final": "false"}}]}}]' + + ranger_config_request_url = ambari_cluster_url + request_result = call_ambari_api(ranger_config_request_url,'PUT',ambari_username_password,str(plugin_configuration_data)) + if request_result is not None: + response_code = request_result.getcode() + response = json.loads(json.JSONEncoder().encode(request_result.read())) + if response_code == 200 and response is not None: + print('Ranger Hbase plugin configuration added successfully in Ambari.') + return True + else: + print('Ranger Hbase plugin configuration add failed in Ambari.') + return False + + +def port_ranger_knox_plugin_to_ambari(): + print('trying to add ranger knox plugin.') + flag_knox_plugin_installed,knox_plugin_install_properties = get_knox_plugin_configuration() + if flag_knox_plugin_installed and knox_plugin_install_properties is not None: + print('Knox plugin is installed and enabled, adding to configurations') + advanced_ranger_knox_plugin_properties = dict() + + advanced_ranger_knox_plugin_properties['policy_user'] = 'ambari-qa' + advanced_ranger_knox_plugin_properties['common.name.for.certificate'] = '-' + advanced_ranger_knox_plugin_properties['ranger-knox-plugin-enabled'] = 'Yes' + advanced_ranger_knox_plugin_properties['REPOSITORY_CONFIG_USERNAME'] = 'admin' + advanced_ranger_knox_plugin_properties['REPOSITORY_CONFIG_PASSWORD'] = 'admin-password' + advanced_ranger_knox_plugin_properties['KNOX_HOME'] = '/usr/hdp/current/knox-server' + advanced_ranger_knox_plugin_properties['XAAUDIT.DB.IS_ENABLED'] = knox_plugin_install_properties['xasecure.audit.db.is.enabled'] + advanced_ranger_knox_plugin_properties['XAAUDIT.HDFS.IS_ENABLED'] = knox_plugin_install_properties['xasecure.audit.hdfs.is.enabled'] + advanced_ranger_knox_plugin_properties['XAAUDIT.HDFS.DESTINATION_DIRECTORY'] = knox_plugin_install_properties['xasecure.audit.hdfs.config.destination.directory'] + advanced_ranger_knox_plugin_properties['XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY'] = knox_plugin_install_properties['xasecure.audit.hdfs.config.local.buffer.directory'] + advanced_ranger_knox_plugin_properties['XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY'] = knox_plugin_install_properties['xasecure.audit.hdfs.config.local.archive.directory'] + advanced_ranger_knox_plugin_properties['XAAUDIT.HDFS.DESTINTATION_FILE'] = knox_plugin_install_properties['xasecure.audit.hdfs.config.destination.file'] + advanced_ranger_knox_plugin_properties['XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS'] = knox_plugin_install_properties['xasecure.audit.hdfs.config.destination.flush.interval.seconds'] + advanced_ranger_knox_plugin_properties['XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS'] = knox_plugin_install_properties['xasecure.audit.hdfs.config.destination.rollover.interval.seconds'] + advanced_ranger_knox_plugin_properties['XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS'] = knox_plugin_install_properties['xasecure.audit.hdfs.config.destination.open.retry.interval.seconds'] + advanced_ranger_knox_plugin_properties['XAAUDIT.HDFS.LOCAL_BUFFER_FILE'] = knox_plugin_install_properties['xasecure.audit.hdfs.config.local.buffer.file'] + advanced_ranger_knox_plugin_properties['XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS'] = knox_plugin_install_properties['xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds'] + advanced_ranger_knox_plugin_properties['XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS'] = knox_plugin_install_properties['xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds'] + advanced_ranger_knox_plugin_properties['XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT'] = knox_plugin_install_properties['xasecure.audit.hdfs.config.local.archive.max.file.count'] + + + knox_ssl_keystore_password = '' + knox_ssl_truststore_password = '' + + libpath= os.path.join(hdp_version_dir,'ranger-knox-plugin','install','lib','*') + aliasKey='sslkeystore' + aliasValue = '' + filepath = os.path.join('/etc/ranger' , knox_plugin_install_properties['REPOSITORY_NAME'],'cred.jceks') + getorcreateorlist = 'get' + + statuscode,value = call_keystore(libpath, aliasKey, aliasValue, filepath, getorcreateorlist) + if statuscode == 0: + knox_ssl_keystore_password = value.strip() + + aliasKey='ssltruststore' + statuscode,value = call_keystore(libpath,aliasKey,aliasValue , filepath,getorcreateorlist) + if statuscode == 0: + knox_ssl_truststore_password = value.strip() + + + advanced_ranger_knox_plugin_properties['SSL_KEYSTORE_FILE_PATH'] = knox_plugin_install_properties['xasecure.policymgr.clientssl.keystore'] + advanced_ranger_knox_plugin_properties['SSL_KEYSTORE_PASSWORD'] = knox_ssl_keystore_password + advanced_ranger_knox_plugin_properties['SSL_TRUSTSTORE_FILE_PATH'] = knox_plugin_install_properties['xasecure.policymgr.clientssl.truststore'] + advanced_ranger_knox_plugin_properties['SSL_TRUSTSTORE_PASSWORD'] = knox_ssl_truststore_password + + + date_time_stamp = getDateTimeNow() + plugin_configuration_data = '[{"Clusters":{"desired_config":[{"type": "ranger-knox-plugin-properties","tag":"' + str(date_time_stamp) + '", "properties":'+json.dumps(advanced_ranger_knox_plugin_properties)+',"properties_attributes": {"final": "false"}}]}}]' + + ranger_config_request_url = ambari_cluster_url + request_result = call_ambari_api(ranger_config_request_url,'PUT',ambari_username_password,str(plugin_configuration_data)) + if request_result is not None: + response_code = request_result.getcode() + response = json.loads(json.JSONEncoder().encode(request_result.read())) + if response_code == 200 and response is not None: + print('Ranger Knox plugin configuration added successfully in Ambari.') + return True + else: + print('Ranger Knox plugin configuration add failed in Ambari.') + return False + + +def port_ranger_storm_plugin_to_ambari(): + print('Trying to add ranger storm plugin.') + flag_storm_plugin_installed,storm_plugin_install_properties = get_storm_plugin_configuration() + if flag_storm_plugin_installed and storm_plugin_install_properties is not None: + print('Storm plugin is installed and enabled, adding to configurations') + advanced_ranger_storm_plugin_properties = dict() + + advanced_ranger_storm_plugin_properties['policy_user'] = 'storm' + advanced_ranger_storm_plugin_properties['common.name.for.certificate'] = '-' + advanced_ranger_storm_plugin_properties['ranger-storm-plugin-enabled'] = 'Yes' + advanced_ranger_storm_plugin_properties['REPOSITORY_CONFIG_USERNAME'] = '[email protected]' + advanced_ranger_storm_plugin_properties['REPOSITORY_CONFIG_PASSWORD'] = 'stormtestuser' + advanced_ranger_storm_plugin_properties['XAAUDIT.DB.IS_ENABLED'] = storm_plugin_install_properties['xasecure.audit.db.is.enabled'] + advanced_ranger_storm_plugin_properties['XAAUDIT.HDFS.IS_ENABLED'] = storm_plugin_install_properties['xasecure.audit.hdfs.is.enabled'] + advanced_ranger_storm_plugin_properties['XAAUDIT.HDFS.DESTINATION_DIRECTORY'] = storm_plugin_install_properties['xasecure.audit.hdfs.config.destination.directory'] + advanced_ranger_storm_plugin_properties['XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY'] = storm_plugin_install_properties['xasecure.audit.hdfs.config.local.buffer.directory'] + advanced_ranger_storm_plugin_properties['XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY'] = storm_plugin_install_properties['xasecure.audit.hdfs.config.local.archive.directory'] + advanced_ranger_storm_plugin_properties['XAAUDIT.HDFS.DESTINTATION_FILE'] = storm_plugin_install_properties['xasecure.audit.hdfs.config.destination.file'] + advanced_ranger_storm_plugin_properties['XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS'] = storm_plugin_install_properties['xasecure.audit.hdfs.config.destination.flush.interval.seconds'] + advanced_ranger_storm_plugin_properties['XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS'] = storm_plugin_install_properties['xasecure.audit.hdfs.config.destination.rollover.interval.seconds'] + advanced_ranger_storm_plugin_properties['XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS'] = storm_plugin_install_properties['xasecure.audit.hdfs.config.destination.open.retry.interval.seconds'] + advanced_ranger_storm_plugin_properties['XAAUDIT.HDFS.LOCAL_BUFFER_FILE'] = storm_plugin_install_properties['xasecure.audit.hdfs.config.local.buffer.file'] + advanced_ranger_storm_plugin_properties['XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS'] = storm_plugin_install_properties['xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds'] + advanced_ranger_storm_plugin_properties['XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS'] = storm_plugin_install_properties['xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds'] + advanced_ranger_storm_plugin_properties['XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT'] = storm_plugin_install_properties['xasecure.audit.hdfs.config.local.archive.max.file.count'] + advanced_ranger_storm_plugin_properties['SSL_KEYSTORE_FILE_PATH'] = storm_plugin_install_properties['xasecure.policymgr.clientssl.keystore'] + advanced_ranger_storm_plugin_properties['SSL_KEYSTORE_PASSWORD'] = storm_plugin_install_properties['xasecure.policymgr.clientssl.keystore.password'] + advanced_ranger_storm_plugin_properties['SSL_TRUSTSTORE_FILE_PATH'] = storm_plugin_install_properties['xasecure.policymgr.clientssl.truststore'] + advanced_ranger_storm_plugin_properties['SSL_TRUSTSTORE_PASSWORD'] = storm_plugin_install_properties['xasecure.policymgr.clientssl.truststore.password'] + + date_time_stamp = getDateTimeNow() + plugin_configuration_data = '[{"Clusters":{"desired_config":[{"type": "ranger-storm-plugin-properties","tag":"' + str(date_time_stamp) + '", "properties":'+json.dumps(advanced_ranger_storm_plugin_properties)+',"properties_attributes": {"final": "false"}}]}}]' + + ranger_config_request_url = ambari_cluster_url + request_result = call_ambari_api(ranger_config_request_url,'PUT',ambari_username_password,str(plugin_configuration_data)) + if request_result is not None: + response_code = request_result.getcode() + response = json.loads(json.JSONEncoder().encode(request_result.read())) + if response_code == 200 and response is not None: + print('Ranger Storm plugin configuration added successfully in Ambari.') + return True + else: + print('Ranger Storm plugin configuration add failed in Ambari.') + return False + + + +def get_hdfs_plugin_configuration(): + flag_hdfs_plugin_installed,hdfs_plugin_install_properties = get_plugin_details('hdfs') + if flag_hdfs_plugin_installed: + print('hdfs plugin is present and installed to ranger,getting additional properties from installed files .') + base_hadoop_conf_path = '/etc/hadoop/conf/' + hdfs_site_xml_path = os.path.join(base_hadoop_conf_path,'hdfs-site.xml') + hdfs_site_xml_properties = import_properties_from_xml(hdfs_site_xml_path, hdfs_plugin_install_properties) + xasecure_audit_xml_path = os.path.join(base_hadoop_conf_path,'xasecure-audit.xml') + hdfs_plugin_install_properties = import_properties_from_xml(xasecure_audit_xml_path, hdfs_plugin_install_properties) + xasecure_hdfs_security_xml_path = os.path.join(base_hadoop_conf_path,'xasecure-hdfs-security.xml') + hdfs_plugin_install_properties = import_properties_from_xml(xasecure_hdfs_security_xml_path, hdfs_plugin_install_properties) + xasecure_policy_ssl_xml_path = os.path.join(base_hadoop_conf_path,'xasecure-policymgr-ssl.xml') + hdfs_plugin_install_properties = import_properties_from_xml(xasecure_policy_ssl_xml_path, hdfs_plugin_install_properties) + return flag_hdfs_plugin_installed,hdfs_plugin_install_properties,hdfs_site_xml_properties + else: + print('hdfs plugin is present but not installed to ranger not configuring plugin for the same.') + return False,None,None + +def get_hive_plugin_configuration(): + flag_hive_plugin_installed,hive_plugin_install_properties = get_plugin_details('hive') + if flag_hive_plugin_installed: + print('hive plugin is present and installed to ranger, configuring to setup in ambari.') + base_hive_conf_path = '/etc/hive/conf/' + hive_server2_xml_path = os.path.join(base_hive_conf_path,'hiveserver2-site.xml') + hive_server2_xml_properties = import_properties_from_xml(hive_server2_xml_path, hive_plugin_install_properties) + xasecure_audit_xml_path = os.path.join(base_hive_conf_path,'xasecure-audit.xml') + hive_plugin_install_properties = import_properties_from_xml(xasecure_audit_xml_path, hive_plugin_install_properties) + xasecure_hive_security_xml_path = os.path.join(base_hive_conf_path,'xasecure-hive-security.xml') + hive_plugin_install_properties = import_properties_from_xml(xasecure_hive_security_xml_path, hive_plugin_install_properties) + xasecure_policy_ssl_xml_path = os.path.join(base_hive_conf_path,'xasecure-policymgr-ssl.xml') + hive_plugin_install_properties = import_properties_from_xml(xasecure_policy_ssl_xml_path, hive_plugin_install_properties) + return flag_hive_plugin_installed,hive_plugin_install_properties,hive_server2_xml_properties + else: + print'hive plugin is present and installed to ranger, configuring to setup in ambari.' + return False,None,None + +def get_hbase_plugin_configuration(): + flag_hbase_plugin_installed,hbase_plugin_install_properties = get_plugin_details('hbase') + if flag_hbase_plugin_installed: + print'hbase plugin is present and installed to ranger, configuring to setup in ambari.' + base_hbase_conf_path = '/etc/hbase/conf/' + hbase_site_xml_path = os.path.join(base_hbase_conf_path,'hbase-site.xml') + hbase_site_xml_properties = import_properties_from_xml(hbase_site_xml_path, hbase_plugin_install_properties) + xasecure_audit_xml_path = os.path.join(base_hbase_conf_path,'xasecure-audit.xml') + hbase_plugin_install_properties = import_properties_from_xml(xasecure_audit_xml_path, hbase_plugin_install_properties) + xasecure_hbase_security_xml_path = os.path.join(base_hbase_conf_path,'xasecure-hbase-security.xml') + hbase_plugin_install_properties = import_properties_from_xml(xasecure_hbase_security_xml_path, hbase_plugin_install_properties) + xasecure_policy_ssl_xml_path = os.path.join(base_hbase_conf_path,'xasecure-policymgr-ssl.xml') + hbase_plugin_install_properties = import_properties_from_xml(xasecure_policy_ssl_xml_path, hbase_plugin_install_properties) + return hbase_plugin_install_properties,hbase_plugin_install_properties,hbase_site_xml_properties + else: + print'hbase plugin is present and installed to ranger, configuring to setup in ambari.' + return False,None,None + + +def get_knox_plugin_configuration(): + flag_knox_plugin_installed,knox_plugin_install_properties = get_plugin_details('knox') + if flag_knox_plugin_installed: + print('knox plugin is present and installed to ranger, configuring to setup in ambari.') + base_knox_conf_path = '/etc/knox/conf/' + xasecure_audit_xml_path = os.path.join(base_knox_conf_path,'xasecure-audit.xml') + knox_plugin_install_properties = import_properties_from_xml(xasecure_audit_xml_path, knox_plugin_install_properties) + xasecure_knox_security_xml_path = os.path.join(base_knox_conf_path,'xasecure-knox-security.xml') + knox_plugin_install_properties = import_properties_from_xml(xasecure_knox_security_xml_path, knox_plugin_install_properties) + xasecure_policy_ssl_xml_path = os.path.join(base_knox_conf_path,'xasecure-policymgr-ssl.xml') + knox_plugin_install_properties = import_properties_from_xml(xasecure_policy_ssl_xml_path, knox_plugin_install_properties) + return knox_plugin_install_properties,knox_plugin_install_properties + else: + print'knox plugin is present and installed to ranger, configuring to setup in ambari.' + return False,None + + +def get_storm_plugin_configuration(): + flag_storm_plugin_installed,storm_plugin_install_properties = get_plugin_details('storm') + if flag_storm_plugin_installed: + print'storm plugin is present and installed to ranger, configuring to setup in ambari.' + base_knox_conf_path = '/etc/storm/conf/' + xasecure_audit_xml_path = os.path.join(base_knox_conf_path,'xasecure-audit.xml') + storm_plugin_install_properties = import_properties_from_xml(xasecure_audit_xml_path, storm_plugin_install_properties) + xasecure_storm_security_xml_path = os.path.join(base_knox_conf_path,'xasecure-storm-security.xml') + storm_plugin_install_properties = import_properties_from_xml(xasecure_storm_security_xml_path, storm_plugin_install_properties) + xasecure_policy_ssl_xml_path = os.path.join(base_knox_conf_path,'xasecure-policymgr-ssl.xml') + storm_plugin_install_properties = import_properties_from_xml(xasecure_policy_ssl_xml_path, storm_plugin_install_properties) + + return storm_plugin_install_properties,storm_plugin_install_properties + else: + print'storm plugin is present and installed to ranger, configuring to setup in ambari.' + return False,None + + + + + +def get_plugin_details(component_name): + print'getting ranger plugin details for ' + component_name + flag_plugin_properties_present = False + plugin_install_properties = dict() + component_modified_name = 'ranger_' + component_name + '_plugin_install.properties' + if os.path.isfile(component_modified_name): + print component_name + ' is installed, reading properties' + plugin_install_properties = import_properties_from_file(component_modified_name) + if plugin_install_properties is not None and len(plugin_install_properties) != 0: + print'plugin properties imported adding to existing configuration.' + flag_plugin_installed = check_plugin_enabled(component_name, plugin_install_properties) + if flag_plugin_installed: + flag_plugin_properties_present = True + else: + flag_plugin_properties_present = False + else: + print 'plugin properties is None hence setting flag to false' + flag_plugin_properties_present = False + else: + print component_name + ' is not found installed not configuring properties for the same' + return flag_plugin_properties_present,plugin_install_properties + + + +def check_plugin_enabled(component_name,component_plugin_install_properties): + flag_plugin_installed = False + repo_base_path = os.path.join('/etc/ranger',component_plugin_install_properties['REPOSITORY_NAME']) + if os.path.exists(repo_base_path): + print 'Plugin is installed for component ' + component_name + flag_plugin_installed = True + return flag_plugin_installed + + + + +def call_ambari_api(ambari_url,method, username_password,data): + try: +# print('calling ambari api for given request') +# print('ambari_url = ' + str(ambari_url)) +# print('method = ' + str(method)) +# print('username_password = ' + str(username_password)) +# print('data = ' + str(data)) + url = ambari_url + base64string = base64.encodestring('{0}'.format(username_password)).replace('\n', '') + headers = {"X-Requested-By": "ambari"} + request = urllib2.Request(url, data, headers,'compressed') + request.get_method = lambda: method + + request.add_header("Authorization", "Basic {0}".format(base64string)) + result = urllib2.urlopen(request) + return result + except urllib2.URLError, e: + if isinstance(e, urllib2.HTTPError): + print("HTTP Code: {0}".format(e.code)) + print("HTTP Data: {0}".format(e.read())) + return None + else: + print("Error: {0}".format(e.reason)) + return None + except httplib.BadStatusLine: + print("Ambari service is not reachable, please restart the service and then try again") + return None + + +def import_properties_from_file(install_properties_path , properties_from_file = None): + if properties_from_file is None: + print 'properties_from_file is none initializing to dict' + properties_from_file = dict() + install_properties_file = open(install_properties_path) + for each_line in install_properties_file.read().split('\n'): + each_line = each_line.strip() + if len(each_line) == 0:continue + if '#https.service.port' in each_line: + each_line = each_line.strip('#') + if '#' in each_line: continue + key, value = each_line.strip().split("=",1) + key = key.strip() + value = value.strip() + properties_from_file[key] = value + return properties_from_file + + + +def import_properties_from_xml(xml_path,properties_from_xml = None): + print 'getting values from file : ' + str(xml_path) + xml = ET.parse(xml_path) + root = xml.getroot() + if properties_from_xml is None: + properties_from_xml = dict() + for child in root.findall('property'): + name = child.find("name").text.strip() + value = child.find("value").text.strip() if child.find("value").text is not None else "" + properties_from_xml[name] = value + return properties_from_xml + + + +def call_keystore(libpath,aliasKey,aliasValue , filepath,getorcreateorlist): + finalLibPath = libpath.replace('\\','/').replace('//','/') + finalFilePath = 'jceks://file/'+filepath.replace('\\','/').replace('//','/') + if getorcreateorlist == 'create': + commandtorun = ['java', '-cp', finalLibPath, 'com.hortonworks.credentialapi.buildks' ,'create', aliasKey, '-value', aliasValue, '-provider',finalFilePath] + p = Popen(commandtorun,stdin=PIPE, stdout=PIPE, stderr=PIPE) + output, error = p.communicate() + statuscode = p.returncode + return statuscode + elif getorcreateorlist == 'get': + commandtorun = ['java', '-cp', finalLibPath, 'com.hortonworks.credentialapi.buildks' ,'get', aliasKey, '-provider',finalFilePath] + p = Popen(commandtorun,stdin=PIPE, stdout=PIPE, stderr=PIPE) + output, error = p.communicate() + statuscode = p.returncode + return statuscode, output + elif getorcreateorlist == 'list': + commandtorun = ['java', '-cp', finalLibPath, 'org.apache.ranger.credentialapi.buildks' ,'list' , '-provider',finalFilePath] + p = Popen(commandtorun,stdin=PIPE, stdout=PIPE, stderr=PIPE) + output, error = p.communicate() + statuscode = p.returncode + return statuscode, output + else: + print 'proper command not received for input need get or create' + + + +def get_hdp_version(): + print('getting hdp version installed for ranger') + ranger_admin_current_path = os.path.join(hdp_current_dir,'ranger-admin') + ranger_admin_actual_path = os.path.realpath(ranger_admin_current_path) + hdp_version_obtained = ranger_admin_actual_path.split('/')[4] + return hdp_version_obtained + + +def getDateTimeNow(): + return datetime.datetime.now().strftime("%Y%m%d%H%M%S") + + + +if __name__ == '__main__': + + + if len(sys.argv) > 1: + function_to_call = sys.argv[1] if len(sys.argv) > 1 else None + base_url = sys.argv[2] if len(sys.argv) > 2 else None + print ('base url = ' + base_url) + ambari_username_password = sys.argv[3] if len(sys.argv) > 3 else None + print ('ambari_username_password = ' + ambari_username_password) + cluster_name = sys.argv[4] if len(sys.argv) > 4 else None + print ('cluster_name = ' + cluster_name) + ranger_admin_fqdn = sys.argv[5] if len(sys.argv) > 5 else None + print ('ranger_admin_fqdn = ' + ranger_admin_fqdn) + ranger_service_name = 'RANGER' + admin_component_name = 'RANGER_ADMIN' + usersync_component_name = 'RANGER_USERSYNC' + ambari_cluster_url = str(base_url) +'/api/v1/clusters/' + str(cluster_name) + ambari_service_url = str(ambari_cluster_url) + '/services' + hdp_dir = os.path.join('/usr','hdp') + hdp_current_dir = os.path.join(hdp_dir,'current') + hdp_version = get_hdp_version() + print('Found hdp_version = ' + str(hdp_version) ) + hdp_version_dir = os.path.join(hdp_dir,hdp_version) + if function_to_call is not None and len(function_to_call) > 0: + print'Found first argument as : ' + function_to_call + function_to_call = int(function_to_call) + if function_to_call == 1: + print('Porting Ranger Admin installation details to Ambari.') + port_ranger_installation_to_ambari() + elif function_to_call == 2: + print('Porting Ranger User-sync installation details to Ambari.') + port_ranger_usersync_installation_to_ambari() + elif function_to_call == 3: + print('Porting Ranger Hdfs plugin details to Ambari.') + port_hdfs_plugin_to_ambari() + elif function_to_call == 4: + print('Porting Ranger Hive plugin details to Ambari.') + port_ranger_hive_plugin_to_ambari() + elif function_to_call == 5: + print('Porting Ranger Hbase plugin details to Ambari.') + port_ranger_hbase_plugin_to_ambari() + elif function_to_call == 6: + print('Porting Ranger Knox plugin details to Ambari.') + port_ranger_knox_plugin_to_ambari() + elif function_to_call == 7: + print('Porting Ranger Storm plugin details to Ambari.') + port_ranger_storm_plugin_to_ambari() + else: + print ('Unsupported option passed for installation, please pass proper supported option') + + else: + print 'Usage :' \ + '\n python import_ranger_to_ambari.py {install option eg. 1} { ambari server url (eg. http://100.100.100.100:8080) } {ambari server username password (eg. demo_user:demo_pass) } {cluster name (eg. ambari_cluster)} {FQDN of host having Ranger Admin or Ranger Usersync or plugins installe (eg. ambari.server.com)} ' \ + '\n Actual call will be like : python ranger_port_script.py 1 http://100.100.100.100:8080 demo_user:demo_pass ambari_cluster ambari.server.com'\ + '\n Pass first parameter as 1 for Ranger integration with Ambari.' \ + '\n Pass first parameter as 2 for Ranger User-sync integration with Ambari.'\ + '\n Pass first parameter as 3 for Ranger Hdfs Plugin integration with Ambari.'\ + '\n Pass first parameter as 4 for Ranger Hive Plugin integration with Ambari.'\ + '\n Pass first parameter as 5 for Ranger Hbase Plugin integration with Ambari.'\ + '\n Pass first parameter as 6 for Ranger Knox Plugin integration with Ambari.'\ + '\n Pass first parameter as 7 for Ranger Storm Plugin integration with Ambari.' + + sys.exit(0) + http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/0f3ace82/migration-util/bin/ranger_admin_install.properties ---------------------------------------------------------------------- diff --git a/migration-util/bin/ranger_admin_install.properties b/migration-util/bin/ranger_admin_install.properties new file mode 100755 index 0000000..b835bba --- /dev/null +++ b/migration-util/bin/ranger_admin_install.properties @@ -0,0 +1,166 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# This file provides list of deployment variables for the Policy Manager Web Application +# + +#------------------------- DB CONFIG - BEGIN ---------------------------------- + +#DB_FLAVOR=ORACLE +#DB_FLAVOR=MYSQL +DB_FLAVOR=MYSQL +# +# The executable path to be used to invoke command-line MYSQL +# +#SQL_COMMAND_INVOKER='mysql' +#SQL_COMMAND_INVOKER='sqlplus' +SQL_COMMAND_INVOKER='mysql' + +# +# Location of DB client library (please check the location of the jar file) +# +#SQL_CONNECTOR_JAR=/usr/share/java/ojdbc6.jar +#SQL_CONNECTOR_JAR=/usr/share/java/mysql-connector-java.jar +SQL_CONNECTOR_JAR=/usr/share/java/mysql-connector-java.jar + + +# +# DB password for the DB admin user-id +# ************************************************************************** +# ** If the password is left empty or not-defined here, +# ** it will be prompted to enter the password during installation process +# ************************************************************************** +# +#db_root_user=root +#db_root_user=SYS +db_root_user=root +db_root_password= +db_host=localhost + +# +# DB UserId used for the XASecure schema +# +db_name=ranger +db_user=rangeradmin +db_password= + +# +# DB UserId for storing auditlog infromation +# +# * audit_db can be same as the XASecure schema db +# * audit_db must exists in the same ${db_host} as xaserver database ${db_name} +# * audit_user must be a different user than db_user (as audit user has access to only audit tables) +# +audit_db_name=ranger_audit +audit_db_user=rangerlogger +audit_db_password= + +#------------------------- DB CONFIG - END ---------------------------------- + +# +# ------- PolicyManager CONFIG ---------------- +# + +policymgr_external_url=http://localhost:6080 +policymgr_http_enabled=true + +# +# ------- PolicyManager CONFIG - END --------------- +# + + +# +# ------- UNIX User CONFIG ---------------- +# +unix_user=ranger +unix_group=ranger + +# +# ------- UNIX User CONFIG - END ---------------- +# + +# +# UNIX authentication service for Policy Manager +# +# PolicyManager can authenticate using UNIX username/password +# The UNIX server specified here as authServiceHostName needs to be installed with xasecure-unix-ugsync package. +# Once the service is installed on authServiceHostName, the UNIX username/password from the host <authServiceHostName> can be used to login into policy manager +# +# ** The installation of xasecure-unix-ugsync package can be installed after the policymanager installation is finished. +# +#LDAP|ACTIVE_DIRECTORY|UNIX|NONE +authentication_method=NONE +remoteLoginEnabled=true +authServiceHostName=localhost +authServicePort=5151 + +####LDAP settings - Required only if have selected LDAP authentication #### +# +# Sample Settings +# +#xa_ldap_url="ldap://71.127.43.33:389"; +#xa_ldap_userDNpattern="uid={0},ou=users,dc=xasecure,dc=net" +#xa_ldap_groupSearchBase="ou=groups,dc=xasecure,dc=net" +#xa_ldap_groupSearchFilter="(member=uid={0},ou=users,dc=xasecure,dc=net)" +#xa_ldap_groupRoleAttribute="cn" + +xa_ldap_url= +xa_ldap_userDNpattern= +xa_ldap_groupSearchBase= +xa_ldap_groupSearchFilter= +xa_ldap_groupRoleAttribute= + +####ACTIVE_DIRECTORY settings - Required only if have selected AD authentication #### +# +# Sample Settings +# +#xa_ldap_ad_domain="xasecure.net" +#xa_ldap_ad_url="ldap://ad.xasecure.net:389"; + +xa_ldap_ad_domain= +xa_ldap_ad_url= +# +# ----------------------------------------------------------- +# + +# ################# DO NOT MODIFY ANY VARIABLES BELOW ######################### +# +# --- These deployment variables are not to be modified unless you understand the full impact of the changes +# +################################################################################ + +XAPOLICYMGR_DIR=$PWD +app_home=$PWD/ews/webapp +TMPFILE=$PWD/.fi_tmp +LOGFILE=$PWD/logfile +LOGFILES="$LOGFILE" + +JAVA_BIN='java' +JAVA_VERSION_REQUIRED='1.7' +JAVA_ORACLE='Java(TM) SE Runtime Environment' + +mysql_create_user_file=${PWD}/db/create_dev_user.sql +mysql_core_file=${PWD}/db/xa_core_db.sql +mysql_audit_file=${PWD}/db/xa_audit_db.sql +mysql_asset_file=${PWD}/db/reset_asset.sql + +#oracle_create_user_file=${PWD}/db/oracle/create_dev_user_oracle.sql +oracle_core_file=${PWD}/db/oracle/xa_core_db_oracle.sql +oracle_audit_file=${PWD}/db/oracle/xa_audit_db_oracle.sql +#oracle_asset_file=${PWD}/db/oracle/reset_asset_oracle.sql +# + +cred_keystore_filename=$app_home/WEB-INF/classes/conf/.jceks/rangeradmin.jceks http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/0f3ace82/migration-util/bin/ranger_hbase_plugin_install.properties ---------------------------------------------------------------------- diff --git a/migration-util/bin/ranger_hbase_plugin_install.properties b/migration-util/bin/ranger_hbase_plugin_install.properties new file mode 100755 index 0000000..d5ccaae --- /dev/null +++ b/migration-util/bin/ranger_hbase_plugin_install.properties @@ -0,0 +1,115 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Location of Policy Manager URL +# +# Example: +# POLICY_MGR_URL=http://policymanager.xasecure.net:6080 +# +POLICY_MGR_URL=http://172.18.145.41:6080 + +# +# Location of db client library (please check the location of the jar file) +# +# Example: +# SQL_CONNECTOR_JAR=/usr/share/java/mysql-connector-java.jar +# SQL_CONNECTOR_JAR=/usr/share/java/ojdbc6.jar +# +SQL_CONNECTOR_JAR=/usr/share/java/mysql-connector-java.jar + +# +# This is the repository name created within policy manager +# +# Example: +# REPOSITORY_NAME=hbasedev +# +REPOSITORY_NAME=c1_hbase + +# +# AUDIT DB Configuration +# +# This information should match with the one you specified during the PolicyManager Installation +# +# Example: +# XAAUDIT.DB.IS_ENABLED=true +# XAAUDIT.DB.FLAVOUR=MYSQL +# XAAUDIT.DB.FLAVOUR=ORACLE +# XAAUDIT.DB.HOSTNAME=localhost +# XAAUDIT.DB.DATABASE_NAME=ranger_audit +# XAAUDIT.DB.USER_NAME=rangerlogger +# XAAUDIT.DB.PASSWORD=rangerlogger +# +XAAUDIT.DB.IS_ENABLED=True +XAAUDIT.DB.FLAVOUR=MYSQL +XAAUDIT.DB.HOSTNAME=localhost +XAAUDIT.DB.DATABASE_NAME=ranger_audit +XAAUDIT.DB.USER_NAME=rangerlogger +XAAUDIT.DB.PASSWORD=rangerlogger + +# +# Audit to HDFS Configuration +# +# If XAAUDIT.HDFS.IS_ENABLED is set to true, please replace tokens +# that start with __REPLACE__ with appropriate values +# XAAUDIT.HDFS.IS_ENABLED=true +# XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://__REPLACE__NAME_NODE_HOST:8020/ranger/audit/%app-type%/%time:yyyyMMdd% +# XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=__REPLACE__LOG_DIR/hbase/audit/%app-type% +# XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=__REPLACE__LOG_DIR/hbase/audit/archive/%app-type% +# +# Example: +# XAAUDIT.HDFS.IS_ENABLED=true +# XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://namenode.example.com:8020/ranger/audit/%app-type%/%time:yyyyMMdd% +# XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=/var/log/hbase/audit/%app-type% +# XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=/var/log/hbase/audit/archive/%app-type% +# +XAAUDIT.HDFS.IS_ENABLED=False +XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://__REPLACE__NAME_NODE_HOST:8020/ranger/audit/%app-type%/%time:yyyyMMdd% +XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=__REPLACE__LOG_DIR/hbase/audit/%app-type% +XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=__REPLACE__LOG_DIR/hbase/audit/archive/%app-type% + +XAAUDIT.HDFS.DESTINTATION_FILE=%hostname%-audit.log +XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS=900 +XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS=86400 +XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS=60 +XAAUDIT.HDFS.LOCAL_BUFFER_FILE=%time:yyyyMMdd-HHmm.ss%.log +XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60 +XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600 +XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10 + +# +# SSL Client Certificate Information +# +# Example: +# SSL_KEYSTORE_FILE_PATH=/etc/hbase/conf/ranger-plugin-keystore.jks +# SSL_KEYSTORE_PASSWORD=none +# SSL_TRUSTSTORE_FILE_PATH=/etc/hbase/conf/ranger-plugin-truststore.jks +# SSL_TRUSTSTORE_PASSWORD=none +# +# You do not need use SSL between agent and security admin tool, please leave these sample value as it is. +# +SSL_KEYSTORE_FILE_PATH=/etc/hbase/conf/ranger-plugin-keystore.jks +SSL_KEYSTORE_PASSWORD=myKeyFilePassword +SSL_TRUSTSTORE_FILE_PATH=/etc/hbase/conf/ranger-plugin-truststore.jks +SSL_TRUSTSTORE_PASSWORD=changeit + +# +# Should HBase GRANT/REVOKE update XA policies? +# +# Example: +# UPDATE_XAPOLICIES_ON_GRANT_REVOKE=true +# UPDATE_XAPOLICIES_ON_GRANT_REVOKE=false +# +UPDATE_XAPOLICIES_ON_GRANT_REVOKE=True http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/0f3ace82/migration-util/bin/ranger_hdfs_plugin_install.properties ---------------------------------------------------------------------- diff --git a/migration-util/bin/ranger_hdfs_plugin_install.properties b/migration-util/bin/ranger_hdfs_plugin_install.properties new file mode 100755 index 0000000..93790e3 --- /dev/null +++ b/migration-util/bin/ranger_hdfs_plugin_install.properties @@ -0,0 +1,106 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Location of Policy Manager URL +# +# Example: +# POLICY_MGR_URL=http://policymanager.xasecure.net:6080 +# +POLICY_MGR_URL= + +# +# Location of db client library (please check the location of the jar file) +# +# Example: +# SQL_CONNECTOR_JAR=/usr/share/java/mysql-connector-java.jar +# SQL_CONNECTOR_JAR=/usr/share/java/ojdbc6.jar +# +SQL_CONNECTOR_JAR=/usr/share/java/mysql-connector-java.jar + +# +# This is the repository name created within policy manager +# +# Example: +# REPOSITORY_NAME=hadoopdev +# +REPOSITORY_NAME= + +# +# AUDIT DB Configuration +# +# This information should match with the one you specified during the PolicyManager Installation +# +# Example: +# XAAUDIT.DB.IS_ENABLED=true +# XAAUDIT.DB.FLAVOUR=MYSQL +# XAAUDIT.DB.FLAVOUR=ORACLE +# XAAUDIT.DB.HOSTNAME=localhost +# XAAUDIT.DB.DATABASE_NAME=ranger_audit +# XAAUDIT.DB.USER_NAME=rangerlogger +# XAAUDIT.DB.PASSWORD=rangerlogger +# +XAAUDIT.DB.IS_ENABLED=false +XAAUDIT.DB.FLAVOUR=MYSQL +XAAUDIT.DB.HOSTNAME= +XAAUDIT.DB.DATABASE_NAME= +XAAUDIT.DB.USER_NAME= +XAAUDIT.DB.PASSWORD= + +# +# Audit to HDFS Configuration +# +# If XAAUDIT.HDFS.IS_ENABLED is set to true, please replace tokens +# that start with __REPLACE__ with appropriate values +# XAAUDIT.HDFS.IS_ENABLED=true +# XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://__REPLACE__NAME_NODE_HOST:8020/ranger/audit/%app-type%/%time:yyyyMMdd% +# XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=__REPLACE__LOG_DIR/hadoop/%app-type%/audit +# XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=__REPLACE__LOG_DIR/hadoop/%app-type%/audit/archive +# +# Example: +# XAAUDIT.HDFS.IS_ENABLED=true +# XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://namenode.example.com:8020/ranger/audit/%app-type%/%time:yyyyMMdd% +# XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=/var/log/hadoop/%app-type%/audit +# XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=/var/log/hadoop/%app-type%/audit/archive +# +XAAUDIT.HDFS.IS_ENABLED=false +XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://__REPLACE__NAME_NODE_HOST:8020/ranger/audit/%app-type%/%time:yyyyMMdd% +XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=__REPLACE__LOG_DIR/hadoop/%app-type%/audit +XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=__REPLACE__LOG_DIR/hadoop/%app-type%/audit/archive + +XAAUDIT.HDFS.DESTINTATION_FILE=%hostname%-audit.log +XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS=900 +XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS=86400 +XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS=60 +XAAUDIT.HDFS.LOCAL_BUFFER_FILE=%time:yyyyMMdd-HHmm.ss%.log +XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60 +XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600 +XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10 + +# +# SSL Client Certificate Information +# +# Example: +# SSL_KEYSTORE_FILE_PATH=/etc/hadoop/conf/ranger-plugin-keystore.jks +# SSL_KEYSTORE_PASSWORD=none +# SSL_TRUSTSTORE_FILE_PATH=/etc/hadoop/conf/ranger-plugin-truststore.jks +# SSL_TRUSTSTORE_PASSWORD=none +# +# You do not need use SSL between agent and security admin tool, please leave these sample value as it is. +# +SSL_KEYSTORE_FILE_PATH=/etc/hadoop/conf/ranger-plugin-keystore.jks +SSL_KEYSTORE_PASSWORD=myKeyFilePassword +SSL_TRUSTSTORE_FILE_PATH=/etc/hadoop/conf/ranger-plugin-truststore.jks +SSL_TRUSTSTORE_PASSWORD=changeit http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/0f3ace82/migration-util/bin/ranger_hive_plugin_install.properties ---------------------------------------------------------------------- diff --git a/migration-util/bin/ranger_hive_plugin_install.properties b/migration-util/bin/ranger_hive_plugin_install.properties new file mode 100755 index 0000000..bbd1849 --- /dev/null +++ b/migration-util/bin/ranger_hive_plugin_install.properties @@ -0,0 +1,115 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Location of Policy Manager URL +# +# Example: +# POLICY_MGR_URL=http://policymanager.xasecure.net:6080 +# +POLICY_MGR_URL= + +# +# Location of db client library (please check the location of the jar file) +# +# Example: +# SQL_CONNECTOR_JAR=/usr/share/java/mysql-connector-java.jar +# SQL_CONNECTOR_JAR=/usr/share/java/ojdbc6.jar +# +SQL_CONNECTOR_JAR=/usr/share/java/mysql-connector-java.jar + +# +# This is the repository name created within policy manager +# +# Example: +# REPOSITORY_NAME=hivedev +# +REPOSITORY_NAME= + +# +# AUDIT DB Configuration +# +# This information should match with the one you specified during the PolicyManager Installation +# +# Example: +# XAAUDIT.DB.IS_ENABLED=true +# XAAUDIT.DB.FLAVOUR=MYSQL +# XAAUDIT.DB.FLAVOUR=ORACLE +# XAAUDIT.DB.HOSTNAME=localhost +# XAAUDIT.DB.DATABASE_NAME=ranger_audit +# XAAUDIT.DB.USER_NAME=rangerlogger +# XAAUDIT.DB.PASSWORD=rangerlogger +# +XAAUDIT.DB.IS_ENABLED=false +XAAUDIT.DB.FLAVOUR=MYSQL +XAAUDIT.DB.HOSTNAME= +XAAUDIT.DB.DATABASE_NAME= +XAAUDIT.DB.USER_NAME= +XAAUDIT.DB.PASSWORD= + +# +# Audit to HDFS Configuration +# +# If XAAUDIT.HDFS.IS_ENABLED is set to true, please replace tokens +# that start with __REPLACE__ with appropriate values +# XAAUDIT.HDFS.IS_ENABLED=true +# XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://__REPLACE__NAME_NODE_HOST:8020/ranger/audit/%app-type%/%time:yyyyMMdd% +# XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=__REPLACE__LOG_DIR/hive/audit/%app-type% +# XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=__REPLACE__LOG_DIR/hive/audit/archive/%app-type% +# +# Example: +# XAAUDIT.HDFS.IS_ENABLED=true +# XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://namenode.example.com:8020/ranger/audit/%app-type%/%time:yyyyMMdd% +# XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=/var/log/hive/audit/%app-type% +# XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=/var/log/hive/audit/archive/%app-type% +# +XAAUDIT.HDFS.IS_ENABLED=false +XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://__REPLACE__NAME_NODE_HOST:8020/ranger/audit/%app-type%/%time:yyyyMMdd% +XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=__REPLACE__LOG_DIR/hive/audit/%app-type% +XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=__REPLACE__LOG_DIR/hive/audit/archive/%app-type% + +XAAUDIT.HDFS.DESTINTATION_FILE=%hostname%-audit.log +XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS=900 +XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS=86400 +XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS=60 +XAAUDIT.HDFS.LOCAL_BUFFER_FILE=%time:yyyyMMdd-HHmm.ss%.log +XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60 +XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600 +XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10 + +# +# SSL Client Certificate Information +# +# Example: +# SSL_KEYSTORE_FILE_PATH=/etc/hive/conf/ranger-plugin-keystore.jks +# SSL_KEYSTORE_PASSWORD=none +# SSL_TRUSTSTORE_FILE_PATH=/etc/hive/conf/ranger-plugin-truststore.jks +# SSL_TRUSTSTORE_PASSWORD=none +# +# You do not need use SSL between agent and security admin tool, please leave these sample value as it is. +# +SSL_KEYSTORE_FILE_PATH=/etc/hive/conf/ranger-plugin-keystore.jks +SSL_KEYSTORE_PASSWORD=myKeyFilePassword +SSL_TRUSTSTORE_FILE_PATH=/etc/hive/conf/ranger-plugin-truststore.jks +SSL_TRUSTSTORE_PASSWORD=changeit + +# +# Should Hive GRANT/REVOKE update XA policies? +# +# Example: +# UPDATE_XAPOLICIES_ON_GRANT_REVOKE=true +# UPDATE_XAPOLICIES_ON_GRANT_REVOKE=false +# +UPDATE_XAPOLICIES_ON_GRANT_REVOKE=true
