http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3770fbd5/migration-util/ambari2.1-hdp2.3-ranger0.50/bin/import_ranger_to_ambari.py ---------------------------------------------------------------------- diff --git a/migration-util/ambari2.1-hdp2.3-ranger0.50/bin/import_ranger_to_ambari.py b/migration-util/ambari2.1-hdp2.3-ranger0.50/bin/import_ranger_to_ambari.py new file mode 100755 index 0000000..bc06a65 --- /dev/null +++ b/migration-util/ambari2.1-hdp2.3-ranger0.50/bin/import_ranger_to_ambari.py @@ -0,0 +1,2253 @@ +import os +import sys +import urllib2 +import base64 +import httplib +import json +import time +from xml.etree import ElementTree as ET +import datetime +from subprocess import Popen, PIPE +import re + + +def port_ranger_admin_installation_to_ambari(): + print('preparing advanged configurations for ranger') + flag_ranger_admin_present, ranger_admin_properties_from_file = get_ranger_admin_install_properties() + if flag_ranger_admin_present: + print('ranger admin service is installed, making configurations as required by ambari.') + if create_ranger_service_in_ambari(): + print('ranger service is added sucessfully in ambari') + if create_ranger_service_components_in_ambari('RANGER_ADMIN'): + print('ranger service component is added successfully in ambari') + if register_ranger_admin_host_in_ambari(): + print('ranger admin host is registered successfully in ambari') + if add_advanced_ranger_configurations(1, ranger_admin_properties_from_file): + print('ranger-admin advanced configurations added successfully in ambari, kindly run ranger-usersync to complete ranger service install') + else: + print('ranger advanced configurations added failed in ambari') + else: + print('ranger admin host registration failed in ambari') + else: + print('ranger service component add failed in ambari') + else: + print('ranger service add failed in ambari') + else: + print('ranger admin and usersync services are not installed, not importing configurations to ambari.') + + +def port_ranger_usersync_installation_to_ambari(): + print ('preparing configurations for ranger user-sync') + flag_ranger_usersync_present, ranger_usersync_properties_from_file = get_ranger_usersync_install_properties() + if flag_ranger_usersync_present: + print('ranger usersync service is installed, making configurations as required by ambari.') + if create_ranger_service_components_in_ambari('RANGER_USERSYNC'): + print('ranger service component is added successfully in ambari') + if register_ranger_usersync_host_in_ambari(): + print('ranger usersync host is registered successfully in ambari') + if add_advanced_ranger_configurations(2, ranger_usersync_properties_from_file): + print('ranger advanced configurations added successfully in ambari') + if call_ranger_installed(): + print('ranger service installed successfully in ambari.') + else: + print('ranger service install failed in ambari') + else: + print('ranger advanced configurations added failed in ambari') + else: + print('ranger usersync host registration failed in ambari') + else: + print('ranger service add failed in ambari') + else: + print('ranger admin and usersync services are not installed, not importing configurations to ambari.') + + +def port_ranger_kms_installation_to_ambari(): + print('preparing advanged configurations for ranger-kms') + flag_ranger_kms_present, ranger_kms_properties_from_file = get_ranger_kms_install_properties() + if flag_ranger_kms_present: + print('ranger kms service is installed, making configurations as required by ambari.') + if create_ranger_kms_service_in_ambari(): + print('ranger kms service is added sucessfully in ambari') + if create_kms_service_components_in_ambari('RANGER_KMS_SERVER'): + print('ranger kms service component is added successfully in ambari') + if register_ranger_kms_host_in_ambari(): + print('ranger kms host is registered successfully in ambari') + if add_advanced_ranger_kms_configurations(ranger_kms_properties_from_file): + print('ranger kms advanced configurations added successfully in ambari') + if call_ranger_kms_installed(): + print('ranger kms service installed successfully in ambari.') + else: + print('ranger kms service install failed in ambari') + else: + print('ranger kms advanced configurations added failed in ambari') + else: + print('ranger kms host registration failed in ambari') + else: + print('ranger kms service component add failed in ambari') + else: + print('ranger kms service add failed in ambari') + else: + print('ranger kms service is not installed, not importing configurations to ambari.') + + +def create_ranger_service_in_ambari(): + print('creating ranger admin service in ambari') + ranger_create_url = ambari_service_url + '/' + ranger_service_name + request_result = call_ambari_api(ranger_create_url, 'POST', ambari_username_password, '') + if request_result is not None: + response_code = request_result.getcode() + response = json.loads(json.JSONEncoder().encode(request_result.read())) + if (response_code == 201 and response is not None): + print('ranger service created successfully in ambari.') + return True + elif (response_code == 409 and response is not None): + print('ranger service is already created in ambari.') + return True + else: + print('ranger service creation failed in ambari.') + return False + + +def create_ranger_kms_service_in_ambari(): + print('creating ranger kms service in ambari') + ranger_create_url = ambari_service_url + '/RANGER_KMS' + request_result = call_ambari_api(ranger_create_url, 'POST', ambari_username_password, '') + if request_result is not None: + response_code = request_result.getcode() + response = json.loads(json.JSONEncoder().encode(request_result.read())) + if (response_code == 201 and response is not None): + print('ranger service created successfully in ambari.') + return True + elif (response_code == 409 and response is not None): + print('ranger service is already created in ambari.') + return True + else: + print('ranger service creation failed in ambari.') + return False + +def create_ranger_service_components_in_ambari(ranger_service_component_name): + print('adding ranger service components in ambari') + ranger_service_components = '{"components":[{"ServiceComponentInfo":{"component_name":"' + ranger_service_component_name + '"}}]}' + print('creating ranger service in ambari') + ranger_service_component_create_url = ambari_service_url + '?ServiceInfo/service_name=' + ranger_service_name + request_result = call_ambari_api(ranger_service_component_create_url, 'POST', ambari_username_password, ranger_service_components) + if request_result is not None: + response_code = request_result.getcode() + response = json.loads(json.JSONEncoder().encode(request_result.read())) + if (response_code == 201 and response is not None): + print('ranger service component : ' + ranger_service_component_name + ', created successfully in ambari.') + return True + elif (response_code == 409 and response is not None): + print('ranger service component : ' + ranger_service_component_name + ', is already present in ambari.') + return True + else: + print('ranger service component creation for : ' + ranger_service_component_name + ', failed in ambari.') + return False + +def create_kms_service_components_in_ambari(ranger_service_component_name): + print('adding ranger service components in ambari') + ranger_service_components = '{"components":[{"ServiceComponentInfo":{"component_name":"' + ranger_service_component_name + '"}}]}' + print('creating ranger kms service in ambari -> ' + str(ranger_service_components)) + ranger_service_component_create_url = ambari_service_url + '?ServiceInfo/service_name=RANGER_KMS' + print('ranger_service_component_create_url -> ' + str(ranger_service_component_create_url)) + request_result = call_ambari_api(ranger_service_component_create_url, 'POST', ambari_username_password, ranger_service_components) + if request_result is not None: + response_code = request_result.getcode() + response = json.loads(json.JSONEncoder().encode(request_result.read())) + if (response_code == 201 and response is not None): + print('ranger service component : ' + ranger_service_component_name + ', created successfully in ambari.') + return True + elif (response_code == 409 and response is not None): + print('ranger service component : ' + ranger_service_component_name + ', is already present in ambari.') + return True + else: + print('ranger service component creation for : ' + ranger_service_component_name + ', failed in ambari.') + return False + +def register_ranger_admin_host_in_ambari(): + print('adding ranger servcie components in ambari') + print('creating ranger admin service in ambari') + ranger_service_component_create_url = ambari_cluster_url + '/hosts/' + ranger_admin_fqdn + '/host_components/' + admin_component_name + request_result = call_ambari_api(ranger_service_component_create_url, 'POST', ambari_username_password, '') + if request_result is not None: + response_code = request_result.getcode() + response = json.loads(json.JSONEncoder().encode(request_result.read())) + if response_code == 201 and response is not None: + print('ranger admin host registered successfully in ambari.') + return True + elif (response_code == 409 and response is not None): + print('ranger admin host is already registered ambari.') + return True + else: + print('ranger admin host registration failed in ambari.') + return False + + +def register_ranger_usersync_host_in_ambari(): + print('adding ranger servcie components in ambari') + print('creating ranger admin service in ambari') + ranger_host_register_url = ambari_cluster_url + '/hosts/' + ranger_admin_fqdn + '/host_components/' + usersync_component_name + request_result = call_ambari_api(ranger_host_register_url, 'POST', ambari_username_password, '') + if request_result is not None: + response_code = request_result.getcode() + response = json.loads(json.JSONEncoder().encode(request_result.read())) + if (response_code == 201 and response is not None): + print('ranger usersync host registered successfully in ambari.') + return True + elif (response_code == 409 and response is not None): + print('ranger usersync host is already registered ambari.') + return True + else: + print('ranger usersync host registration failed in ambari.') + return False + +def register_ranger_kms_host_in_ambari(): + print('adding ranger kms host in ambari') + print('creating ranger kms service in ambari') + ranger_service_component_create_url = ambari_cluster_url + '/hosts/' + ranger_admin_fqdn + '/host_components/' + "RANGER_KMS_SERVER" + request_result = call_ambari_api(ranger_service_component_create_url, 'POST', ambari_username_password, '') + if request_result is not None: + response_code = request_result.getcode() + response = json.loads(json.JSONEncoder().encode(request_result.read())) + if response_code == 201 and response is not None: + print('ranger kms host registered successfully in ambari.') + return True + elif (response_code == 409 and response is not None): + print('ranger kms host is already registered ambari.') + return True + else: + print('ranger kms host registration failed in ambari.') + return False + + +def add_advanced_ranger_configurations(add_admin_or_usersync, ranger_service_properties_from_file): + print('creating advanced configurations to be added to ambari.') + ranger_config_data = '' + advanced_admin_properties = dict() + advanced_ranger_site_properties = dict() + advanced_ranger_env_properties = dict() + advanced_user_sync_properties = dict() + advanced_ranger_admin_site_properties = dict() + date_time_stamp = getDateTimeNow() + + if (add_admin_or_usersync == 1): + if not ((str(ranger_service_properties_from_file['db_root_password']).strip() == '') or + (str(ranger_service_properties_from_file['db_root_user']).strip() == '' )) : + advanced_admin_properties['DB_FLAVOR'] = ranger_service_properties_from_file['DB_FLAVOR'] + advanced_admin_properties['SQL_CONNECTOR_JAR'] = ranger_service_properties_from_file['SQL_CONNECTOR_JAR'] + advanced_admin_properties['db_root_user'] = ranger_service_properties_from_file['db_root_user'] + advanced_admin_properties['db_root_password'] = ranger_service_properties_from_file['db_root_password'] + advanced_admin_properties['db_host'] = ranger_service_properties_from_file['db_host'] + advanced_admin_properties['db_name'] = ranger_service_properties_from_file['db_name'] + advanced_admin_properties['db_user'] = ranger_service_properties_from_file['ranger.jpa.jdbc.user'] + advanced_admin_properties['db_password'] = ranger_service_properties_from_file['ranger.jpa.jdbc.password'] + advanced_admin_properties['audit_db_name'] = ranger_service_properties_from_file['audit_db_name'] + advanced_admin_properties['audit_db_user'] = ranger_service_properties_from_file['ranger.jpa.audit.jdbc.user'] + advanced_admin_properties['audit_db_password'] = ranger_service_properties_from_file['ranger.jpa.audit.jdbc.password'] + advanced_admin_properties['policymgr_external_url'] = ranger_service_properties_from_file['ranger.externalurl'] + + advanced_ranger_env_properties['ranger_user'] = 'ranger' + advanced_ranger_env_properties['ranger_group'] = 'ranger' + advanced_ranger_env_properties['ranger_admin_log_dir'] = '/var/log/ranger/admin' + advanced_ranger_env_properties['ranger_usersync_log_dir'] = '/var/log/ranger/usersync' + advanced_ranger_env_properties['ranger_admin_username'] = 'amb_ranger_admin' + advanced_ranger_env_properties['ranger_admin_password'] = 'ambari123' + advanced_ranger_env_properties['admin_username'] = 'admin' + advanced_ranger_env_properties['admin_password'] = 'admin' + advanced_ranger_env_properties['ranger_pid_dir'] = '/var/run/ranger' + advanced_ranger_env_properties['create_db_dbuser'] = 'true' + advanced_ranger_env_properties['xml_configurations_supported'] = 'true' + + advanced_ranger_admin_site_properties['ranger.service.host'] = ranger_service_properties_from_file['ranger.service.host'] + advanced_ranger_admin_site_properties['ranger.service.http.enabled'] = ranger_service_properties_from_file.get('ranger.service.http.enabled','true') + advanced_ranger_admin_site_properties['ranger.service.http.port'] = ranger_service_properties_from_file.get('ranger.service.http.port','6080') + advanced_ranger_admin_site_properties['ranger.service.https.port'] = ranger_service_properties_from_file.get('ranger.service.https.port','6182') + advanced_ranger_admin_site_properties['ranger.service.https.attrib.ssl.enabled'] = ranger_service_properties_from_file.get('ranger.service.https.attrib.ssl.enabled','false') + advanced_ranger_admin_site_properties['ranger.service.https.attrib.clientAuth'] = ranger_service_properties_from_file.get('ranger.service.https.attrib.clientAuth','want') + advanced_ranger_admin_site_properties['ranger.service.https.attrib.keystore.keyalias'] = ranger_service_properties_from_file.get('ranger.service.https.attrib.keystore.keyalias','rangeradmin') + advanced_ranger_admin_site_properties['ranger.service.https.attrib.keystore.pass'] = ranger_service_properties_from_file.get('ranger.service.https.attrib.keystore.pass','xasecure') + advanced_ranger_admin_site_properties['ranger.https.attrib.keystore.file'] = ranger_service_properties_from_file.get('ranger.https.attrib.keystore.file','/etc/ranger/admin/conf/ranger-admin-keystore.jks') + advanced_ranger_admin_site_properties['ranger.externalurl'] = ranger_service_properties_from_file.get('ranger.externalurl','http://localhost:6080') + advanced_ranger_admin_site_properties['ranger.jpa.jdbc.driver'] = ranger_service_properties_from_file.get('ranger.jpa.jdbc.driver','com.mysql.jdbc.Driver') + advanced_ranger_admin_site_properties['ranger.jpa.jdbc.url'] = ranger_service_properties_from_file.get('ranger.jpa.jdbc.url','jdbc:mysql://localhost') + advanced_ranger_admin_site_properties['ranger.jpa.jdbc.user'] = ranger_service_properties_from_file['ranger.jpa.jdbc.user'] + advanced_ranger_admin_site_properties['ranger.jpa.jdbc.password'] = ranger_service_properties_from_file['ranger.jpa.jdbc.password'] + advanced_ranger_admin_site_properties['ranger.jpa.jdbc.credential.alias'] = ranger_service_properties_from_file.get('ranger.jpa.jdbc.credential.alias','rangeradmin') + advanced_ranger_admin_site_properties['ranger.credential.provider.path'] = ranger_service_properties_from_file.get('ranger.credential.provider.path','/etc/ranger/admin/rangeradmin.jceks') + advanced_ranger_admin_site_properties['ranger.audit.source.type'] = ranger_service_properties_from_file.get('ranger.audit.source.type','db') + advanced_ranger_admin_site_properties['ranger.audit.solr.urls'] = ranger_service_properties_from_file.get('ranger.audit.solr.urls','http://solr_host:6083/solr/ranger_audits') + advanced_ranger_admin_site_properties['ranger.authentication.method'] = ranger_service_properties_from_file.get('ranger.authentication.method','UNIX') + advanced_ranger_admin_site_properties['ranger.ldap.url'] = ranger_service_properties_from_file.get('ranger.ldap.url','ldap://71.127.43.33:389') + advanced_ranger_admin_site_properties['ranger.ldap.user.dnpattern'] = ranger_service_properties_from_file.get('ranger.ldap.user.dnpattern','uid={0},ou=users,dc=xasecure,dc=net') + advanced_ranger_admin_site_properties['ranger.ldap.group.searchbase'] = ranger_service_properties_from_file.get('ranger.ldap.group.searchbase','ou=groups,dc=xasecure,dc=net') + advanced_ranger_admin_site_properties['ranger.ldap.group.searchfilter'] = ranger_service_properties_from_file.get('ranger.ldap.group.searchfilter','(member=uid={0},ou=users,dc=xasecure,dc=net)') + advanced_ranger_admin_site_properties['ranger.ldap.group.roleattribute'] = ranger_service_properties_from_file.get('ranger.ldap.group.roleattribute','cn') + advanced_ranger_admin_site_properties['ranger.ldap.ad.domain'] = ranger_service_properties_from_file.get('ranger.ldap.ad.domain','localhost') + advanced_ranger_admin_site_properties['ranger.ldap.ad.url'] = ranger_service_properties_from_file.get('ranger.ldap.ad.url','ldap://ad.xasecure.net:389') + advanced_ranger_admin_site_properties['ranger.jpa.audit.jdbc.driver'] = ranger_service_properties_from_file.get('ranger.jpa.audit.jdbc.driver','{{ranger_jdbc_driver}}') + advanced_ranger_admin_site_properties['ranger.jpa.audit.jdbc.url'] = ranger_service_properties_from_file.get('ranger.jpa.audit.jdbc.url','{{audit_jdbc_url}}') + advanced_ranger_admin_site_properties['ranger.jpa.audit.jdbc.user'] = ranger_service_properties_from_file.get('ranger.jpa.audit.jdbc.user','{{ranger_audit_db_user}}') + advanced_ranger_admin_site_properties['ranger.jpa.audit.jdbc.password'] = ranger_service_properties_from_file.get('ranger.jpa.audit.jdbc.password','_') + advanced_ranger_admin_site_properties['ranger.jpa.audit.jdbc.credential.alias'] = ranger_service_properties_from_file.get('ranger.jpa.audit.jdbc.credential.alias','rangeraudit') + advanced_ranger_admin_site_properties['ranger.unixauth.remote.login.enabled'] = ranger_service_properties_from_file.get('ranger.unixauth.remote.login.enabled','true') + advanced_ranger_admin_site_properties['ranger.unixauth.service.hostname'] = ranger_service_properties_from_file.get('ranger.unixauth.service.hostname','localhost') + advanced_ranger_admin_site_properties['ranger.unixauth.service.port'] = ranger_service_properties_from_file.get('ranger.unixauth.service.port','5151') + advanced_ranger_admin_site_properties['ranger.jpa.jdbc.dialect'] = ranger_service_properties_from_file.get('ranger.jpa.jdbc.dialect','{{jdbc_dialect}}') + advanced_ranger_admin_site_properties['ranger.jpa.audit.jdbc.dialect'] = ranger_service_properties_from_file.get('ranger.jpa.audit.jdbc.dialect','{{jdbc_dialect}') + advanced_ranger_admin_site_properties['ranger.audit.solr.zookeepers'] = ranger_service_properties_from_file.get('ranger.audit.solr.zookeepers','NONE') + advanced_ranger_admin_site_properties['ranger.audit.solr.username'] = ranger_service_properties_from_file.get('ranger.audit.solr.username','ranger_solr') + advanced_ranger_admin_site_properties['ranger.audit.solr.password'] = ranger_service_properties_from_file.get('ranger.audit.solr.password','NONE') + + ranger_config_data = '[{"Clusters":{"desired_config":[{"type":"admin-properties", "service_config_version_note": "Initial configuration for Ranger Admin service" ,"tag":"' + str( + date_time_stamp) + '","properties":' + json.dumps( + advanced_admin_properties) + ', "properties_attributes": {"final": "true"}},{"type":"ranger-site", "service_config_version_note": "Initial configuration for Ranger Admin service" ,"tag":"' + str( + date_time_stamp) + '","properties":' + json.dumps( + advanced_ranger_site_properties) + ', "properties_attributes": {"final": "false"}},{"type":"ranger-env", "service_config_version_note": "Initial configuration for Ranger Admin service" ,"tag":"' + str( + date_time_stamp) + '","properties":' + json.dumps(advanced_ranger_env_properties) + ', "properties_attributes": {"final": "false"}},{"type":"ranger-admin-site", "service_config_version_note": "Initial configuration for Ranger Admin service" ,"tag":"' + str( + date_time_stamp) + '","properties":' + json.dumps( + advanced_ranger_admin_site_properties) + ', "properties_attributes": {"final": "false"}}]}}]' + + + print ('####################### admin_properties configuration :') + for each_key in advanced_admin_properties: + print str(each_key) + ' = ' + str(advanced_admin_properties[each_key]) + + print ('####################### ranger_site_properties configuration :') + for each_key in advanced_ranger_site_properties: + print str(each_key) + ' = ' + str(advanced_ranger_site_properties[each_key]) + + print ('####################### ranger_env_properties configuration :') + for each_key in advanced_ranger_env_properties: + print str(each_key) + ' = ' + str(advanced_ranger_env_properties[each_key]) + + print ('####################### ranger_admin_site_properties configuration :') + for each_key in advanced_ranger_env_properties: + print str(each_key) + ' = ' + str(advanced_ranger_env_properties[each_key]) + + + else: + print('either db_root_user or db_root_password value is missing from ranger_admin_install.properties file, please set appropriate value and run the script again.') + sys.exit(1) + + elif (add_admin_or_usersync == 2): + advanced_user_sync_properties['ranger.usersync.port'] = ranger_service_properties_from_file['ranger.usersync.port'] + advanced_user_sync_properties['ranger.usersync.ssl'] = ranger_service_properties_from_file['ranger.usersync.ssl'] + advanced_user_sync_properties['ranger.usersync.keystore.file'] = ranger_service_properties_from_file['ranger.usersync.keystore.file'] + advanced_user_sync_properties['ranger.usersync.keystore.password'] = ranger_service_properties_from_file.get('ranger.usersync.keystore.password','UnIx529p') + advanced_user_sync_properties['ranger.usersync.truststore.file'] = ranger_service_properties_from_file.get('ranger.usersync.truststore.file','/usr/hdp/current/ranger-usersync/conf/mytruststore.jks') + advanced_user_sync_properties['ranger.usersync.truststore.password'] = ranger_service_properties_from_file.get('ranger.usersync.truststore.password','changeit') + advanced_user_sync_properties['ranger.usersync.passwordvalidator.path'] = ranger_service_properties_from_file['ranger.usersync.passwordvalidator.path'] + advanced_user_sync_properties['ranger.usersync.sink.impl.class'] = ranger_service_properties_from_file['ranger.usersync.sink.impl.class'] + advanced_user_sync_properties['ranger.usersync.policymanager.baseURL'] = ranger_service_properties_from_file['ranger.usersync.policymanager.baseURL'] + advanced_user_sync_properties['ranger.usersync.policymanager.maxrecordsperapicall'] = ranger_service_properties_from_file['ranger.usersync.policymanager.maxrecordsperapicall'] + advanced_user_sync_properties['ranger.usersync.policymanager.mockrun'] = ranger_service_properties_from_file['ranger.usersync.policymanager.mockrun'] + advanced_user_sync_properties['ranger.usersync.unix.minUserId'] = ranger_service_properties_from_file['ranger.usersync.unix.minUserId'] + advanced_user_sync_properties['ranger.usersync.sleeptimeinmillisbetweensynccycle'] = ranger_service_properties_from_file['ranger.usersync.sleeptimeinmillisbetweensynccycle'] + advanced_user_sync_properties['ranger.usersync.source.impl.class'] = ranger_service_properties_from_file['ranger.usersync.source.impl.class'] + advanced_user_sync_properties['ranger.usersync.filesource.file'] = ranger_service_properties_from_file.get('ranger.usersync.filesource.file','/tmp/usergroup.txt') + advanced_user_sync_properties['ranger.usersync.filesource.text.delimiter'] = ranger_service_properties_from_file.get('ranger.usersync.filesource.text.delimiter',',') + advanced_user_sync_properties['ranger.usersync.ldap.url'] = ranger_service_properties_from_file.get('ranger.usersync.ldap.url','ldap://localhost:389') + advanced_user_sync_properties['ranger.usersync.ldap.binddn'] = ranger_service_properties_from_file.get('ranger.usersync.ldap.binddn','cn=admin,dc=xasecure,dc=net') + advanced_user_sync_properties['ranger.usersync.ldap.ldapbindpassword'] = ranger_service_properties_from_file['ranger.usersync.ldap.ldapbindpassword'] + advanced_user_sync_properties['ranger.usersync.ldap.bindalias'] = ranger_service_properties_from_file.get('ranger.usersync.ldap.bindalias','testldapalias') + advanced_user_sync_properties['ranger.usersync.ldap.bindkeystore'] = ranger_service_properties_from_file.get('ranger.usersync.ldap.bindkeystore','') + advanced_user_sync_properties['ranger.usersync.ldap.searchBase'] = ranger_service_properties_from_file.get('ranger.usersync.ldap.searchBase','dc=hadoop,dc=apache,dc=org') + advanced_user_sync_properties['ranger.usersync.ldap.user.searchbase'] = ranger_service_properties_from_file.get('ranger.usersync.ldap.user.searchbase','ou=users,dc=xasecure,dc=net') + advanced_user_sync_properties['ranger.usersync.ldap.user.searchscope'] = ranger_service_properties_from_file.get('ranger.usersync.ldap.user.searchscope','sub') + advanced_user_sync_properties['ranger.usersync.ldap.user.objectclass'] = ranger_service_properties_from_file.get('ranger.usersync.ldap.user.objectclass','person') + advanced_user_sync_properties['ranger.usersync.ldap.user.searchfilter'] = ranger_service_properties_from_file.get('ranger.usersync.ldap.user.searchfilter','empty') + advanced_user_sync_properties['ranger.usersync.ldap.user.nameattribute'] = ranger_service_properties_from_file.get('ranger.usersync.ldap.user.nameattribute','cn') + advanced_user_sync_properties['ranger.usersync.ldap.user.groupnameattribute'] = ranger_service_properties_from_file.get('ranger.usersync.ldap.user.groupnameattribute','memberof, ismemberof') + advanced_user_sync_properties['ranger.usersync.ldap.username.caseconversion'] = ranger_service_properties_from_file.get('ranger.usersync.ldap.username.caseconversion','lower') + advanced_user_sync_properties['ranger.usersync.ldap.groupname.caseconversion'] = ranger_service_properties_from_file.get('ranger.usersync.ldap.groupname.caseconversion','lower') + advanced_user_sync_properties['ranger.usersync.logdir'] = ranger_service_properties_from_file.get('ranger.usersync.logdir','/var/log/ranger/usersync') + advanced_user_sync_properties['ranger.usersync.group.searchenabled'] = ranger_service_properties_from_file.get('ranger.usersync.group.searchenabled','false') + advanced_user_sync_properties['ranger.usersync.group.usermapsyncenabled'] = ranger_service_properties_from_file.get('ranger.usersync.group.usermapsyncenabled','false') + advanced_user_sync_properties['ranger.usersync.group.searchbase'] = ranger_service_properties_from_file.get('ranger.usersync.group.searchbase',' ') + advanced_user_sync_properties['ranger.usersync.group.searchscope'] = ranger_service_properties_from_file.get('ranger.usersync.group.searchscope',' ') + advanced_user_sync_properties['ranger.usersync.group.objectclass'] = ranger_service_properties_from_file.get('ranger.usersync.group.objectclass',' ') + advanced_user_sync_properties['ranger.usersync.group.searchfilter'] = ranger_service_properties_from_file.get('ranger.usersync.group.searchfilter',' ') + advanced_user_sync_properties['ranger.usersync.group.nameattribute'] = ranger_service_properties_from_file.get('ranger.usersync.group.nameattribute',' ') + advanced_user_sync_properties['ranger.usersync.group.memberattributename'] = ranger_service_properties_from_file.get('ranger.usersync.group.memberattributename',' ') + advanced_user_sync_properties['ranger.usersync.pagedresultsenabled'] = ranger_service_properties_from_file.get('ranger.usersync.pagedresultsenabled','true') + advanced_user_sync_properties['ranger.usersync.pagedresultssize'] = ranger_service_properties_from_file.get('ranger.usersync.pagedresultssize','500') + advanced_user_sync_properties['ranger.usersync.credstore.filename'] = ranger_service_properties_from_file.get('ranger.usersync.credstore.filename','/usr/hdp/current/ranger-usersync/conf/ugsync.jceks') + + ranger_config_data = '[{"Clusters":{"desired_config":[{"type":"ranger-ugsync-site", "service_config_version_note": "Initial configuration for Ranger Usersync service" ,"tag":"' + str( + date_time_stamp) + '","properties":' + json.dumps(advanced_user_sync_properties) + ', "properties_attributes": {"final": "false"}}]}}]' + + + print ('####################### user_sync_properties configuration :') + for each_key in advanced_user_sync_properties: + print str(each_key) + ' = ' + str(advanced_user_sync_properties[each_key]) + + else: + print ('invalid option for to add configuration to ranger.') + sys.exit(1) + + + confirm_configurations = raw_input('please confirm the above configuration values y/n (n) : ') + if(confirm_configurations == ''): + confirm_configurations = 'n' + print ('input registered as ' + str(confirm_configurations)) + + if(confirm_configurations.lower() == 'y'): + ranger_config_request_url = ambari_cluster_url + request_result = call_ambari_api(ranger_config_request_url, 'PUT', ambari_username_password, str(ranger_config_data)) + if request_result is not None: + response_code = request_result.getcode() + response = json.loads(json.JSONEncoder().encode(request_result.read())) + if response_code == 200 and response is not None: + print('ranger advanced configuration added successfully in ambari.') + return True + else: + print('ranger advanced configuration add failed in ambari.') + return False + else: + print('ranger advanced configuration add failed in ambari.') + return False + else: + print ('exiting installation without configuration !') + sys.exit(0) + + +def add_advanced_ranger_kms_configurations(ranger_kms_properties_from_file): + print('creating advanced configurations to be added to ') + advanced_kms_env_properties = dict() + advanced_kms_properties = dict() + advanced_dbks_site_properties = dict() + advanced_kms_site_properties = dict() + advanced_ranger_kms_site_properties = dict() + advanced_ranger_kms_audit_properties = dict() + advanced_ranger_kms_policymgr_ssl_properties = dict() + advanced_ranger_kms_security_properties = dict() + advanced_kms_log4j_properties = dict() + date_time_stamp = getDateTimeNow() + + advanced_kms_env_properties['kms_user'] = 'kms' + advanced_kms_env_properties['kms_group'] = 'kms' + advanced_kms_env_properties['kms_log_dir'] = '/var/log/ranger/kms' + advanced_kms_env_properties['kms_port'] = '9292' + + advanced_kms_properties['REPOSITORY_CONFIG_USERNAME'] = 'keyadmin' + advanced_kms_properties['REPOSITORY_CONFIG_PASSWORD'] = 'keyadmin' + advanced_kms_properties['DB_FLAVOR'] = ranger_kms_properties_from_file.get('DB_FLAVOR','MYSQL') + advanced_kms_properties['SQL_CONNECTOR_JAR'] = ranger_kms_properties_from_file.get('SQL_CONNECTOR_JAR','/usr/share/java/mysql-connector-java.jar') + advanced_kms_properties['db_root_user'] = ranger_kms_properties_from_file.get('db_root_user','root') + advanced_kms_properties['db_root_password'] = ranger_kms_properties_from_file.get('db_root_password','') + advanced_kms_properties['db_host'] = ranger_kms_properties_from_file.get('db_host','localhost') + advanced_kms_properties['db_name'] = ranger_kms_properties_from_file.get('db_name','rangerkms') + advanced_kms_properties['db_user'] = ranger_kms_properties_from_file.get('ranger.ks.jpa.jdbc.user','rangerkms') + advanced_kms_properties['db_password'] = ranger_kms_properties_from_file.get('ranger.ks.jpa.jdbc.password','') + advanced_kms_properties['KMS_MASTER_KEY_PASSWD'] = ranger_kms_properties_from_file.get('KMS_MASTER_KEY_PASSWD','') + + + advanced_dbks_site_properties['hadoop.kms.blacklist.DECRYPT_EEK'] = ranger_kms_properties_from_file.get('hadoop.kms.blacklist.DECRYPT_EEK','hdfs') + advanced_dbks_site_properties['ranger.db.encrypt.key.password'] = ranger_kms_properties_from_file.get('ranger.db.encrypt.key.password','_') + advanced_dbks_site_properties['ranger.ks.jpa.jdbc.url'] = ranger_kms_properties_from_file.get('ranger.ks.jpa.jdbc.url','{{db_jdbc_url}}') + advanced_dbks_site_properties['ranger.ks.jpa.jdbc.user'] = ranger_kms_properties_from_file.get('ranger.ks.jpa.jdbc.user','{{db_user}}') + advanced_dbks_site_properties['ranger.ks.jpa.jdbc.password'] = ranger_kms_properties_from_file.get('ranger.ks.jpa.jdbc.password','_') + advanced_dbks_site_properties['ranger.ks.jpa.jdbc.credential.provider.path'] = ranger_kms_properties_from_file.get('ranger.ks.jpa.jdbc.credential.provider.path','/etc/ranger/kms/rangerkms.jceks') + advanced_dbks_site_properties['ranger.ks.jpa.jdbc.credential.alias'] = ranger_kms_properties_from_file.get('ranger.ks.jpa.jdbc.credential.alias','ranger.ks.jdbc.password') + advanced_dbks_site_properties['ranger.ks.masterkey.credential.alias'] = ranger_kms_properties_from_file.get('ranger.ks.masterkey.credential.alias','ranger.ks.masterkey.password') + advanced_dbks_site_properties['ranger.ks.jpa.jdbc.dialect'] = ranger_kms_properties_from_file.get('ranger.ks.jpa.jdbc.dialect','{{jdbc_dialect}}') + advanced_dbks_site_properties['ranger.ks.jpa.jdbc.driver'] = ranger_kms_properties_from_file.get('ranger.ks.jpa.jdbc.driver','{{db_jdbc_driver}}') + advanced_dbks_site_properties['ranger.ks.jdbc.sqlconnectorjar'] = ranger_kms_properties_from_file.get('ranger.ks.jdbc.sqlconnectorjar','{{driver_curl_target}}') + + + advanced_kms_site_properties['hadoop.kms.key.provider.uri'] = ranger_kms_properties_from_file.get('hadoop.kms.key.provider.uri','dbks://http@localhost:9292/kms') + advanced_kms_site_properties['hadoop.security.keystore.JavaKeyStoreProvider.password'] = ranger_kms_properties_from_file.get('hadoop.security.keystore.JavaKeyStoreProvider.password','none') + advanced_kms_site_properties['hadoop.kms.cache.enable'] = ranger_kms_properties_from_file.get('hadoop.kms.cache.enable','true') + advanced_kms_site_properties['hadoop.kms.cache.timeout.ms'] = ranger_kms_properties_from_file.get('hadoop.kms.cache.timeout.ms','600000') + advanced_kms_site_properties['hadoop.kms.current.key.cache.timeout.ms'] = ranger_kms_properties_from_file.get('hadoop.kms.current.key.cache.timeout.ms','30000') + advanced_kms_site_properties['hadoop.kms.audit.aggregation.window.ms'] = ranger_kms_properties_from_file.get('hadoop.kms.audit.aggregation.window.ms','10000') + advanced_kms_site_properties['hadoop.kms.authentication.type'] = ranger_kms_properties_from_file.get('hadoop.kms.authentication.type','simple') + advanced_kms_site_properties['hadoop.kms.authentication.kerberos.keytab'] = ranger_kms_properties_from_file.get('hadoop.kms.authentication.kerberos.keytab','${user.home}/kms.keytab') + advanced_kms_site_properties['hadoop.kms.authentication.kerberos.principal'] = ranger_kms_properties_from_file.get('hadoop.kms.authentication.kerberos.principal','HTTP/localhost') + advanced_kms_site_properties['hadoop.kms.authentication.kerberos.name.rules'] = ranger_kms_properties_from_file.get('hadoop.kms.authentication.kerberos.name.rules','DEFAULT') + advanced_kms_site_properties['hadoop.kms.authentication.signer.secret.provider'] = ranger_kms_properties_from_file.get('hadoop.kms.authentication.signer.secret.provider','random') + advanced_kms_site_properties['hadoop.kms.authentication.signer.secret.provider.zookeeper.path'] = ranger_kms_properties_from_file.get('hadoop.kms.authentication.signer.secret.provider.zookeeper.path','/hadoop-kms/hadoop-auth-signature-secret') + advanced_kms_site_properties['hadoop.kms.authentication.signer.secret.provider.zookeeper.auth.type'] = ranger_kms_properties_from_file.get('hadoop.kms.authentication.signer.secret.provider.zookeeper.auth.type','kerberos') + advanced_kms_site_properties['hadoop.kms.authentication.signer.secret.provider.zookeeper.kerberos.keytab'] = ranger_kms_properties_from_file.get('/etc/hadoop/conf/kms.keytab','/etc/hadoop/conf/kms.keytab') + advanced_kms_site_properties['hadoop.kms.authentication.signer.secret.provider.zookeeper.kerberos.principal'] = ranger_kms_properties_from_file.get('hadoop.kms.authentication.signer.secret.provider.zookeeper.kerberos.principal','kms/#HOSTNAME#') + advanced_kms_site_properties['hadoop.kms.security.authorization.manager'] = ranger_kms_properties_from_file.get('hadoop.kms.security.authorization.manager','org.apache.ranger.authorization.kms.authorizer.RangerKmsAuthorizer') + + + advanced_ranger_kms_site_properties['ranger.service.host'] = ranger_kms_properties_from_file.get('ranger.service.host','{{kms_host}}') + advanced_ranger_kms_site_properties['ranger.service.http.port'] = ranger_kms_properties_from_file.get('ranger.service.http.port','{{kms_port}}') + advanced_ranger_kms_site_properties['ranger.service.https.port'] = ranger_kms_properties_from_file.get('ranger.service.https.port','9393') + advanced_ranger_kms_site_properties['ranger.service.shutdown.port'] = ranger_kms_properties_from_file.get('ranger.service.shutdown.port','7085') + advanced_ranger_kms_site_properties['ranger.contextName'] = ranger_kms_properties_from_file.get('ranger.contextName','/kms') + advanced_ranger_kms_site_properties['xa.webapp.dir'] = ranger_kms_properties_from_file.get('xa.webapp.dir','./webapp') + advanced_ranger_kms_site_properties['ranger.service.https.attrib.ssl.enabled'] = ranger_kms_properties_from_file.get('ranger.service.https.attrib.ssl.enabled','false') + + + advanced_ranger_kms_audit_properties['xasecure.audit.is.enabled'] = ranger_kms_properties_from_file.get('xasecure.audit.is.enabled','true') + advanced_ranger_kms_audit_properties['xasecure.audit.destination.db'] = ranger_kms_properties_from_file.get('xasecure.audit.db.is.enabled','false') + advanced_ranger_kms_audit_properties['xasecure.audit.destination.db.jdbc.url'] = ranger_kms_properties_from_file.get('xasecure.audit.jpa.javax.persistence.jdbc.url','{{audit_jdbc_url}}') + advanced_ranger_kms_audit_properties['xasecure.audit.destination.db.user'] = ranger_kms_properties_from_file.get('xasecure.audit.jpa.javax.persistence.jdbc.user','{{xa_audit_db_user}}') + advanced_ranger_kms_audit_properties['xasecure.audit.destination.db.password'] = ranger_kms_properties_from_file.get('xasecure.audit.jpa.javax.persistence.jdbc.password','crypted') + advanced_ranger_kms_audit_properties['xasecure.audit.destination.db.jdbc.driver'] = ranger_kms_properties_from_file.get('xasecure.audit.jpa.javax.persistence.jdbc.driver','{{jdbc_driver}}') + advanced_ranger_kms_audit_properties['xasecure.audit.credential.provider.file'] = ranger_kms_properties_from_file.get('xasecure.audit.credential.provider.file','jceks://file{{credential_file}}') + advanced_ranger_kms_audit_properties['xasecure.audit.destination.db.batch.filespool.dir'] = ranger_kms_properties_from_file.get('xasecure.audit.destination.db.batch.filespool.dir','/var/log/ranger/kms/audit/db/spool') + advanced_ranger_kms_audit_properties['xasecure.audit.destination.hdfs'] = ranger_kms_properties_from_file.get('xasecure.audit.destination.hdfs','true') + advanced_ranger_kms_audit_properties['xasecure.audit.destination.hdfs.dir'] = ranger_kms_properties_from_file.get('xasecure.audit.destination.hdfs.dir','hdfs://NAMENODE_HOSTNAME:8020/ranger/audit') + advanced_ranger_kms_audit_properties['xasecure.audit.destination.hdfs.batch.filespool.dir'] = ranger_kms_properties_from_file.get('xasecure.audit.destination.hdfs.batch.filespool.dir','/var/log/ranger/kms/audit/hdfs/spool') + advanced_ranger_kms_audit_properties['xasecure.audit.destination.solr'] = ranger_kms_properties_from_file.get('xasecure.audit.destination.solr','true') + advanced_ranger_kms_audit_properties['xasecure.audit.destination.solr.urls'] = ranger_kms_properties_from_file.get('xasecure.audit.destination.solr.urls','{{ranger_audit_solr_urls}}') + advanced_ranger_kms_audit_properties['xasecure.audit.destination.solr.zookeepers'] = ranger_kms_properties_from_file.get('xasecure.audit.destination.solr.zookeepers','none') + advanced_ranger_kms_audit_properties['xasecure.audit.destination.solr.batch.filespool.dir'] = ranger_kms_properties_from_file.get('xasecure.audit.destination.solr.batch.filespool.dir','/var/log/ranger/kms/audit/solr/spool') + advanced_ranger_kms_audit_properties['xasecure.audit.provider.summary.enabled'] = ranger_kms_properties_from_file.get('xasecure.audit.provider.summary.enabled','false') + + + advanced_ranger_kms_policymgr_ssl_properties['xasecure.policymgr.clientssl.keystore'] = ranger_kms_properties_from_file.get('xasecure.policymgr.clientssl.keystore','/usr/hdp/current/ranger-kms/conf/ranger-plugin-keystore.jks') + advanced_ranger_kms_policymgr_ssl_properties['xasecure.policymgr.clientssl.keystore.password'] = ranger_kms_properties_from_file.get('xasecure.policymgr.clientssl.keystore.password','myKeyFilePassword') + advanced_ranger_kms_policymgr_ssl_properties['xasecure.policymgr.clientssl.truststore'] = ranger_kms_properties_from_file.get('xasecure.policymgr.clientssl.truststore','/usr/hdp/current/ranger-kms/conf/ranger-plugin-truststore.jks') + advanced_ranger_kms_policymgr_ssl_properties['xasecure.policymgr.clientssl.truststore.password'] = ranger_kms_properties_from_file.get('xasecure.policymgr.clientssl.truststore.password','changeit') + advanced_ranger_kms_policymgr_ssl_properties['xasecure.policymgr.clientssl.keystore.credential.file'] = ranger_kms_properties_from_file.get('xasecure.policymgr.clientssl.keystore.credential.file','jceks://file{{credential_file}}') + advanced_ranger_kms_policymgr_ssl_properties['xasecure.policymgr.clientssl.truststore.credential.file'] = ranger_kms_properties_from_file.get('xasecure.policymgr.clientssl.truststore.credential.file','jceks://file{{credential_file}}') + + + advanced_ranger_kms_security_properties['ranger.plugin.kms.service.name'] = ranger_kms_properties_from_file.get('ranger.plugin.kms.service.name','{{repo_name}}') + advanced_ranger_kms_security_properties['ranger.plugin.kms.policy.source.impl'] = ranger_kms_properties_from_file.get('ranger.plugin.kms.policy.source.impl','org.apache.ranger.admin.client.RangerAdminRESTClient') + advanced_ranger_kms_security_properties['ranger.plugin.kms.policy.rest.url'] = ranger_kms_properties_from_file.get('ranger.plugin.kms.policy.rest.url','{{policymgr_mgr_url}}') + advanced_ranger_kms_security_properties['ranger.plugin.kms.policy.rest.ssl.config.file'] = ranger_kms_properties_from_file.get('ranger.plugin.kms.policy.rest.ssl.config.file','/etc/ranger/kms/conf/ranger-policymgr-ssl.xml') + advanced_ranger_kms_security_properties['ranger.plugin.kms.policy.pollIntervalMs'] = ranger_kms_properties_from_file.get('ranger.plugin.kms.policy.pollIntervalMs','30000') + advanced_ranger_kms_security_properties['ranger.plugin.kms.policy.cache.dir'] = ranger_kms_properties_from_file.get('ranger.plugin.kms.policy.cache.dir','/etc/ranger/{{repo_name}}/policycache') + + advanced_kms_log4j_properties['content'] = ranger_kms_properties_from_file.get('kms.log4j.properties','') + + + kms_config_data = '[{"Clusters":{"desired_config":[{"type":"kms-properties", "service_config_version_note": "Initial configuration for Ranger KMS service" ,"tag":"' + str( + date_time_stamp) + '","properties":' + json.dumps( + advanced_kms_properties) + ', "properties_attributes": {"final": "true"}},{"type":"kms-site", "service_config_version_note": "Initial configuration for Ranger KMS service" ,"tag":"' + str( + date_time_stamp) + '","properties":' + json.dumps( + advanced_kms_site_properties) + ', "properties_attributes": {"final": "false"}},{"type":"kms-env", "service_config_version_note": "Initial configuration for Ranger KMS service" ,"tag":"' + str( + date_time_stamp) + '","properties":' + json.dumps(advanced_kms_env_properties) + ', "properties_attributes": {"final": "false"}},{"type":"dbks-site", "service_config_version_note": "Initial configuration for Ranger KMS service" ,"tag":"' + str( + date_time_stamp) + '","properties":' + json.dumps(advanced_dbks_site_properties) + ', "properties_attributes": {"final": "false"}},{"type":"ranger-kms-site", "service_config_version_note": "Initial configuration for Ranger KMS service" ,"tag":"' + str( + date_time_stamp) + '","properties":' + json.dumps(advanced_ranger_kms_site_properties) + ', "properties_attributes": {"final": "false"}},{"type":"ranger-kms-audit", "service_config_version_note": "Initial configuration for Ranger KMS service" ,"tag":"' + str( + date_time_stamp) + '","properties":' + json.dumps(advanced_ranger_kms_audit_properties) + ', "properties_attributes": {"final": "false"}},{"type":"ranger-kms-policymgr-ssl", "service_config_version_note": "Initial configuration for Ranger KMS service" ,"tag":"' + str( + date_time_stamp) + '","properties":' + json.dumps(advanced_ranger_kms_policymgr_ssl_properties) + ', "properties_attributes": {"final": "false"}},{"type":"kms-log4j", "service_config_version_note": "Initial configuration for Ranger KMS service" ,"tag":"' + str( + date_time_stamp) + '","properties":' + json.dumps(advanced_kms_log4j_properties) + ', "properties_attributes": {"final": "false"}},{"type":"ranger-kms-security", "service_config_version_note": "Initial configuration for Ranger KMS service" ,"tag":"' + str( + date_time_stamp) + '","properties":' + json.dumps(advanced_ranger_kms_security_properties) + ', "properties_attributes": {"final": "false"}}]}}]' + + + print ('####################### kms_properties configuration :') + for each_key in advanced_kms_properties: + print str(each_key) + ' = ' + str(advanced_kms_properties[each_key]) + + print ('####################### kms_site_properties configuration :') + for each_key in advanced_kms_site_properties: + print str(each_key) + ' = ' + str(advanced_kms_site_properties[each_key]) + + print ('####################### kms_env_properties configuration :') + for each_key in advanced_kms_env_properties: + print str(each_key) + ' = ' + str(advanced_kms_env_properties[each_key]) + + print ('####################### ranger_kms_site_properties configuration :') + for each_key in advanced_ranger_kms_site_properties: + print str(each_key) + ' = ' + str(advanced_ranger_kms_site_properties[each_key]) + + print ('####################### kms_dbks_site_properties configuration :') + for each_key in advanced_dbks_site_properties: + print str(each_key) + ' = ' + str(advanced_dbks_site_properties[each_key]) + + print ('####################### ranger_kms_audit_properties configuration :') + for each_key in advanced_ranger_kms_audit_properties: + print str(each_key) + ' = ' + str(advanced_ranger_kms_audit_properties[each_key]) + + print ('####################### ranger_kms_policymgr_ssl_properties configuration :') + for each_key in advanced_ranger_kms_policymgr_ssl_properties: + print str(each_key) + ' = ' + str(advanced_ranger_kms_policymgr_ssl_properties[each_key]) + + + print ('####################### ranger_kms_security_properties configuration :') + for each_key in advanced_ranger_kms_security_properties: + print str(each_key) + ' = ' + str(advanced_ranger_kms_security_properties[each_key]) + + print ('####################### ranger_kms_log4j_properties configuration :') + for each_key in advanced_kms_log4j_properties: + print str(each_key) + ' = ' + str(advanced_kms_log4j_properties[each_key]) + + + confirm_configurations = raw_input('please confirm the above configuration values y/n (n) : ') + if(confirm_configurations == ''): + confirm_configurations = 'n' + print ('input registered as ' + str(confirm_configurations)) + + if(confirm_configurations.lower() == 'y'): + ranger_config_request_url = ambari_cluster_url + request_result = call_ambari_api(ranger_config_request_url, 'PUT', ambari_username_password, str(kms_config_data)) + if request_result is not None: + response_code = request_result.getcode() + response = json.loads(json.JSONEncoder().encode(request_result.read())) + if response_code == 200 and response is not None: + print('ranger kms advanced configuration added successfully in ambari.') + return True + else: + print('ranger kms advanced configuration add failed in ambari.') + return False + else: + print('ranger kms advanced configuration add failed in ambari.') + return False + else: + print ('exiting installation without configuration !') + sys.exit(0) + + + +def call_ranger_installed(): + print('changing state of ranger services from init to installed.') + ranger_state_change_request = '{"RequestInfo":{"context":"Install Ranger Service","operation_level":{"level":"CLUSTER","cluster_name":"' + str( + cluster_name) + '"}},"Body":{"ServiceInfo":{"state":"INSTALLED"}}}' + ranger_state_change_url = ambari_service_url + '?ServiceInfo/state=INIT' + request_result = call_ambari_api(ranger_state_change_url, 'PUT', ambari_username_password, ranger_state_change_request) + if request_result is not None: + response_code = request_result.getcode() + response = json.loads(json.JSONEncoder().encode(request_result.read())) + if (response_code == 200 and response is not None): + print('ranger state changed to install successfully in ambari.') + return True + if (response_code == 409 and response is not None): + print('ranger is already installed in ambari.') + return True + if response_code == 202 and response is not None: + print('ranger state changed to install posted in ambari, checking for updated status waiting for 30 seconds') + parsed_response = json.loads(response) + response_href_url = parsed_response['href'] + response_request_id = parsed_response['Requests']['id'] + response_status = parsed_response['Requests']['status'] + if response_status != 'Installed': + print('Received response but status is not installed, verifying installation to be successful.') + flag_ranger_installed = True + time.sleep(30) + while flag_ranger_installed: + print('checking request status') + ambari_request_url = ambari_cluster_url + '/requests/' + str(response_request_id) + request_status_result = call_ambari_api(ambari_request_url, 'GET', ambari_username_password, '') + if request_status_result is not None: + response_code = request_status_result.getcode() + response_status = json.loads(json.JSONEncoder().encode(request_status_result.read())) + if (response_code == 200 and 'FAILED' in response_status): + print('ranger install failed in ambari.') + flag_ranger_installed = True + if (response_code == 200 ): + print('ranger install is pending in ambari.') + flag_ranger_installed = True + if (response_code == 200 and ('PENDING' in response_status or 'INTERNAL_REQUEST' in response_status)): + print('ranger install is pending in ambari.') + flag_ranger_installed = True + if (response_code == 200 and 'COMPLETED' in response_status): + print('ranger installed successfully in ambari.') + flag_ranger_installed = False + else: + flag_ranger_installed = True + return not flag_ranger_installed + else: + print('ranger state changed to install failed in ambari.') + return False + +def call_ranger_kms_installed(): + print('changing state of ranger services from init to installed.') + ranger_state_change_request = '{"RequestInfo":{"context":"Install Ranger KMS Service","operation_level":{"level":"CLUSTER","cluster_name":"' + str( + cluster_name) + '"}},"Body":{"ServiceInfo":{"state":"INSTALLED"}}}' + ranger_state_change_url = ambari_service_url + '?ServiceInfo/state=INIT' + request_result = call_ambari_api(ranger_state_change_url, 'PUT', ambari_username_password, ranger_state_change_request) + if request_result is not None: + response_code = request_result.getcode() + response = json.loads(json.JSONEncoder().encode(request_result.read())) + if (response_code == 200 and response is not None): + print('ranger state changed to install successfully in ambari.') + return True + if (response_code == 409 and response is not None): + print('ranger is already installed in ambari.') + return True + if response_code == 202 and response is not None: + print('ranger state changed to install posted in ambari, checking for updated status waiting for 30 seconds') + parsed_response = json.loads(response) + response_href_url = parsed_response['href'] + response_request_id = parsed_response['Requests']['id'] + response_status = parsed_response['Requests']['status'] + if response_status != 'Installed': + print('Received response but status is not installed, verifying installation to be successful.') + flag_ranger_installed = True + time.sleep(30) + while flag_ranger_installed: + print('checking request status') + ambari_request_url = ambari_cluster_url + '/requests/' + str(response_request_id) + request_status_result = call_ambari_api(ambari_request_url, 'GET', ambari_username_password, '') + if request_status_result is not None: + response_code = request_status_result.getcode() + response_status = json.loads(json.JSONEncoder().encode(request_status_result.read())) + if (response_code == 200 and 'FAILED' in response_status): + print('ranger install failed in ambari.') + flag_ranger_installed = True + if (response_code == 200 ): + print('ranger kms install is pending in ambari.') + flag_ranger_installed = True + if (response_code == 200 and ('PENDING' in response_status or 'INTERNAL_REQUEST' in response_status)): + print('ranger kms install is pending in ambari.') + flag_ranger_installed = True + if (response_code == 200 and 'COMPLETED' in response_status): + print('ranger kms installed successfully in ambari.') + flag_ranger_installed = False + else: + flag_ranger_installed = True + return not flag_ranger_installed + else: + print('ranger kms state changed to install failed in ambari.') + return False + + +def get_ranger_usersync_install_properties(): + print('preparing advanced configurations for ranger User-sync') + flag_hadoop_present = check_hadoop_dir_present(hdp_current_dir) + flag_ranger_usersync_present = False + ranger_usersync_current_dir = os.path.join(hdp_current_dir, 'ranger-usersync') + ranger_usersync_properties_from_file = dict() + print('checking for ranger-usersync service to be present') + flag_ranger_usersync_present, ranger_usersync__installed_version = check_ranger_usersync_install(ranger_usersync_current_dir) + if flag_ranger_usersync_present: + print('ranger usersync is installed, getting existing properties for ambari import') + print('got ranger usersync values from install.properties file, need to configure ambari for ranger service.') + ranger_usersync_properties_from_file = get_additional_properties_for_usersync(ranger_usersync_properties_from_file) + return flag_ranger_usersync_present, ranger_usersync_properties_from_file + + +def get_ranger_admin_install_properties(): + ranger_admin_current_dir = os.path.join(hdp_current_dir, 'ranger-admin') + flag_hadoop_present = check_hadoop_dir_present(hdp_current_dir) + flag_ranger_admin_present = False + ranger_admin_properties_from_file = dict() + + if flag_hadoop_present: + print('hadoop directory is present, checking ranger admin installation.') + flag_ranger_admin_present, ranger_admin_installed_version = check_ranger_admin_install(ranger_admin_current_dir) + + if flag_ranger_admin_present: + print('ranger admin is present.getting existing configurations to port to ambari.') + if os.path.isfile("ranger_admin_install.properties"): + print('ranger_install_properties_path exists, getting existing properties for ambari port') + ranger_admin_properties_from_file = import_properties_from_file("ranger_admin_install.properties") + if not (ranger_admin_properties_from_file['db_root_user'] == '' or ranger_admin_properties_from_file['db_root_password'] == ''): + print 'db_root_username and db_root_password are not blank.' + print 'value for db_root_user = ' + str(ranger_admin_properties_from_file['db_root_user']) + print 'value for db_root_password = ' + str(ranger_admin_properties_from_file['db_root_password']) + else: + print 'db_root_username or db_root_password are blank, please provide proper values in ranger_admin_install.properties. exiting installation without any changes.' + sys.exit(1) + + print('got ranger admin values from ranger_admin_install.properties file, need to configure ambari for ranger service.') + print('getting additional properties required by ranger services') + ranger_admin_properties_from_file = get_additional_properties_for_admin(ranger_admin_properties_from_file) + + return flag_ranger_admin_present, ranger_admin_properties_from_file + +def get_ranger_kms_install_properties(): + ranger_kms_current_dir = os.path.join(hdp_current_dir, 'ranger-kms') + flag_hadoop_present = check_hadoop_dir_present(hdp_current_dir) + flag_ranger_kms_present = False + ranger_kms_properties_from_file = dict() + + if flag_hadoop_present: + print('hadoop directory is present, checking ranger kms installation.') + flag_ranger_kms_present, ranger_kms_installed_version = check_ranger_kms_install(ranger_kms_current_dir) + + if flag_ranger_kms_present: + print('ranger kms is present.getting existing configurations to port to ambari.') + if os.path.isfile("ranger_admin_install.properties"): + print('ranger_install_properties_path exists, getting existing properties for ambari port') + ranger_kms_properties_from_file = import_properties_from_file("ranger_admin_install.properties") + if not (ranger_kms_properties_from_file['db_root_user'] == '' or ranger_kms_properties_from_file['db_root_password'] == ''): + print 'db_root_username and db_root_password are not blank.' + print 'value for db_root_user = ' + str(ranger_kms_properties_from_file['db_root_user']) + print 'value for db_root_password = ' + str(ranger_kms_properties_from_file['db_root_password']) + else: + print 'db_root_username or db_root_password are blank, please provide proper values in ranger_admin_install.properties. exiting installation without any changes.' + sys.exit(1) + + print('got ranger admin values from ranger_admin_install.properties file, need to configure ambari for ranger service.') + print('getting additional properties required by ranger services') + ranger_kms_properties_from_file = get_additional_properties_for_kms(ranger_kms_properties_from_file) + + return flag_ranger_kms_present, ranger_kms_properties_from_file + + +def check_hadoop_dir_present(hdp_current_dir_path): + flag_hadoop_dir_present = False + if os.path.isdir(hdp_current_dir_path): + print('hadoop is installed.') + flag_hadoop_dir_present = True + return flag_hadoop_dir_present + + +def check_ranger_admin_install(ranger_admin_current_dir): + flag_ranger_dir_present = False + ranger_current_installed_version = '' + print('checking ranger service path folder') + if os.path.isdir(ranger_admin_current_dir): + print('ranger admin is installed.') + if os.path.islink(ranger_admin_current_dir): + flag_ranger_dir_present = True + print('ranger admin link found getting current version from link.') + ranger_home_path = os.path.realpath(ranger_admin_current_dir) + ranger_current_installed_version = ranger_home_path.split('/')[4] + return flag_ranger_dir_present, ranger_current_installed_version + + +def check_ranger_usersync_install(ranger_usersync_current_dir): + flag_ranger_dir_present = False + ranger_current_installed_version = '' + print('checking ranger service path folder') + if os.path.isdir(ranger_usersync_current_dir): + print('ranger user-sync is installed.') + if os.path.islink(ranger_usersync_current_dir): + flag_ranger_dir_present = True + print('ranger admin link found getting current version from link.') + ranger_home_path = os.path.realpath(ranger_usersync_current_dir) + ranger_current_installed_version = ranger_home_path.split('/')[4] + return flag_ranger_dir_present, ranger_current_installed_version + + +def check_ranger_kms_install(ranger_kms_current_dir): + flag_ranger_dir_present = False + ranger_current_installed_version = '' + print('checking ranger kms service path folder') + if os.path.isdir(ranger_kms_current_dir): + print('ranger kms is installed.') + if os.path.islink(ranger_kms_current_dir): + flag_ranger_dir_present = True + print('ranger kms link found getting current version from link.') + ranger_home_path = os.path.realpath(ranger_kms_current_dir) + ranger_current_installed_version = ranger_home_path.split('/')[4] + return flag_ranger_dir_present, ranger_current_installed_version + + +def get_additional_properties_for_admin(ranger_admin_properties_from_file): + ranger_conf_path = '/etc/ranger/admin/conf' + ranger_admin_default_site_xml_properties = os.path.join(ranger_conf_path,'ranger-admin-default-site.xml') + ranger_admin_site_xml_properties = os.path.join(ranger_conf_path,'ranger-admin-site.xml') + try: + + ranger_admin_properties_from_file = import_properties_from_xml(ranger_admin_default_site_xml_properties, ranger_admin_properties_from_file) + ranger_admin_properties_from_file = import_properties_from_xml(ranger_admin_site_xml_properties, ranger_admin_properties_from_file) + except Exception, e: + print "Error loading ranger-admin properties from xml files : ", str(e) + + print('getting db flavor, library and command invoker') + ranger_jpa_jdbc_dialect = ranger_admin_properties_from_file['ranger.jpa.jdbc.dialect'] + print('ranger_jpa_jdbc_dialect = ' + ranger_jpa_jdbc_dialect) + ranger_jpa_jdbc_url = ranger_admin_properties_from_file['ranger.jpa.jdbc.url'] + print('ranger_jpa_jdbc_url = ' + ranger_jpa_jdbc_url) + if ('mysql'.lower() in ranger_jpa_jdbc_dialect.lower() and 'mysql'.lower() in ranger_jpa_jdbc_url.lower()): + print('db dialect and jdbc url are set as MYSQL setting db_flavour and sql command invoker as mysql') + ranger_admin_properties_from_file['DB_FLAVOR'] = 'MYSQL' + ranger_admin_properties_from_file['SQL_CONNECTOR_JAR'] = '/usr/share/java/mysql-connector-java.jar' + elif ('oracle'.lower() in ranger_jpa_jdbc_dialect and 'oracle'.lower() in ranger_jpa_jdbc_url.lower()): + print('db dialect and jdbc url are set as Oracle setting db_flavour and sql command invoker as oracle') + ranger_admin_properties_from_file['DB_FLAVOR'] = 'ORACLE' + ranger_admin_properties_from_file['SQL_CONNECTOR_JAR'] = '/usr/share/java/ojdbc6.jar' + elif ('postgres'.lower() in ranger_jpa_jdbc_dialect and 'postgres'.lower() in ranger_jpa_jdbc_url.lower()): + print('db dialect and jdbc url are set as postgres setting db_flavour and sql command invoker as postgres') + ranger_admin_properties_from_file['DB_FLAVOR'] = 'POSTGRES' + ranger_admin_properties_from_file['SQL_CONNECTOR_JAR'] = '/usr/share/java/postgresql.jar' + else: + print('found unsupported DB_FLAVOUR, please configure as MYSQL, ORACLE or Postgres which are supported for now.exitting for now') + sys.exit(1) + + ranger_jpa_jdbc_url = ranger_admin_properties_from_file['ranger.jpa.jdbc.url'] + print('found jdbc url configured as : ' + str(ranger_jpa_jdbc_url) + ' , getting db host from configured jdbc url') + ranger_database_host_name = ranger_jpa_jdbc_url.split(':') + ranger_database_host = ranger_database_host_name[3].split('/')[2] + ranger_database_name = ranger_database_host_name[3].split('/')[3] + print('found db host as : ' + str(ranger_database_host)) + print('found db name as : ' + str(ranger_database_name)) + ranger_admin_properties_from_file['db_host'] = ranger_database_host + ranger_admin_properties_from_file['db_name'] = ranger_database_name + + ranger_audit_jdbc_url = ranger_admin_properties_from_file['ranger.jpa.audit.jdbc.url'] + print('found audit jdbc url configured as : ' + str(ranger_audit_jdbc_url) + ' , getting db host from configured jdbc url') + ranger_audit_database_host_name = ranger_audit_jdbc_url.split(':') + + ranger_audit_database_host = ranger_audit_database_host_name[3].split('/')[2] + ranger_audit_database_name = ranger_audit_database_host_name[3].split('/')[3] + print('found ranger_audit_database_name as : ' + str(ranger_audit_database_name)) + ranger_admin_properties_from_file['audit_db_host'] = ranger_audit_database_host + ranger_admin_properties_from_file['audit_db_name'] = ranger_audit_database_name + + ranger_db_password = '' + ranger_audit_db_password = '' + + libpath = os.path.join(hdp_version_dir, 'ranger-admin', 'cred', 'lib', '*') + aliasKey = 'ranger.db.password' + aliasValue = '' + filepath = ranger_admin_properties_from_file['ranger.credential.provider.path'] + if (filepath is None or len(filepath) == 0): + filepath = os.path.join(hdp_version_dir, 'ranger-admin', 'ews', 'webapp', 'WEB-INF', 'classes', 'conf', '.jceks', 'rangeradmin.jceks') + + getorcreateorlist = 'get' + + statuscode, value = call_keystore(libpath, aliasKey, aliasValue, filepath, getorcreateorlist) + if statuscode == 0: + ranger_db_password = value.strip() + + aliasKey = 'ranger.auditdb.password' + statuscode, value = call_keystore(libpath, aliasKey, aliasValue, filepath, getorcreateorlist) + if statuscode == 0: + ranger_audit_db_password = value.strip() + + if str(ranger_admin_properties_from_file['ranger.jpa.jdbc.password']) == '_' or str(ranger_admin_properties_from_file['ranger.jpa.jdbc.password']).lower() == 'crypted': + ranger_admin_properties_from_file['ranger.jpa.jdbc.password'] = ranger_db_password + if str(ranger_admin_properties_from_file['ranger.jpa.audit.jdbc.password']) == '_' or str(ranger_admin_properties_from_file['ranger.jpa.audit.jdbc.password']).lower() == 'crypted': + ranger_admin_properties_from_file['ranger.jpa.audit.jdbc.password'] = ranger_audit_db_password + + return ranger_admin_properties_from_file + + +def get_additional_properties_for_kms(ranger_kms_properties_from_file): + ranger_conf_path = '/etc/ranger/kms/conf' + kms_dbks_site_properties = os.path.join(ranger_conf_path,'dbks-site.xml') + ranger_kms_properties_from_file = import_properties_from_xml(kms_dbks_site_properties, ranger_kms_properties_from_file) + kms_site_properties = os.path.join(ranger_conf_path,'kms-site.xml') + ranger_kms_properties_from_file = import_properties_from_xml(kms_site_properties, ranger_kms_properties_from_file) + ranger_kms_site_properties = os.path.join(ranger_conf_path,'ranger-kms-site.xml') + ranger_kms_properties_from_file = import_properties_from_xml(ranger_kms_site_properties, ranger_kms_properties_from_file) + ranger_kms_audit_properties = os.path.join(ranger_conf_path,'ranger-kms-audit.xml') + ranger_kms_properties_from_file = import_properties_from_xml(ranger_kms_audit_properties, ranger_kms_properties_from_file) + ranger_kms_security_properties = os.path.join(ranger_conf_path,'ranger-kms-security.xml') + ranger_kms_properties_from_file = import_properties_from_xml(ranger_kms_security_properties, ranger_kms_properties_from_file) + ranger_policy_mgr_ssl_properties = os.path.join(ranger_conf_path,'ranger-policymgr-ssl.xml') + ranger_kms_properties_from_file = import_properties_from_xml(ranger_policy_mgr_ssl_properties, ranger_kms_properties_from_file) + kms_log4j_properties = os.path.join(ranger_conf_path,'kms-log4j.properties') + ranger_kms_properties_from_file['kms.log4j.properties'] = read_properties_file(kms_log4j_properties) + + kms_jpa_jdbc_dialect = ranger_kms_properties_from_file['ranger.ks.jpa.jdbc.dialect'] + print('kms_jpa_jdbc_dialect = ' + str(kms_jpa_jdbc_dialect)) + kms_jpa_jdbc_url = ranger_kms_properties_from_file['ranger.ks.jpa.jdbc.url'] + if ('mysql'.lower() in kms_jpa_jdbc_dialect.lower() and 'mysql'.lower() in kms_jpa_jdbc_url.lower()): + print('db dialect and jdbc url are set as MYSQL setting db_flavour and sql command invoker as mysql') + ranger_kms_properties_from_file['DB_FLAVOR'] = 'MYSQL' + ranger_kms_properties_from_file['SQL_CONNECTOR_JAR'] = '/usr/share/java/mysql-connector-java.jar' + elif ('oracle'.lower() in kms_jpa_jdbc_dialect and 'oracle'.lower() in kms_jpa_jdbc_url.lower()): + print('db dialect and jdbc url are set as Oracle setting db_flavour and sql command invoker as oracle') + ranger_kms_properties_from_file['DB_FLAVOR'] = 'ORACLE' + ranger_kms_properties_from_file['SQL_CONNECTOR_JAR'] = '/usr/share/java/ojdbc6.jar' + elif ('postgres'.lower() in kms_jpa_jdbc_dialect and 'postgres'.lower() in kms_jpa_jdbc_url.lower()): + print('db dialect and jdbc url are set as postgres setting db_flavour and sql command invoker as postgres') + ranger_kms_properties_from_file['DB_FLAVOR'] = 'POSTGRES' + ranger_kms_properties_from_file['SQL_CONNECTOR_JAR'] = '/usr/share/java/postgresql.jar' + else: + print('found unsupported DB_FLAVOUR, please configure as MYSQL, ORACLE or Postgres which are supported for now.exitting for now') + sys.exit(1) + + kms_jpa_db_hostname = kms_jpa_jdbc_url.split(':') + + kms_jpa_jdbc_hostname = kms_jpa_db_hostname[3].split('/')[2] + kms_jpa_jdbc_db_name = kms_jpa_db_hostname[3].split('/')[3] + print('found db hostname = ' + kms_jpa_jdbc_hostname ) + print('found db name = ' + kms_jpa_jdbc_db_name ) + + ranger_kms_properties_from_file['db_host'] = kms_jpa_jdbc_hostname + ranger_kms_properties_from_file['db_name'] = kms_jpa_jdbc_db_name + + kms_database_password = '' + kms_keystore_masterkey_password = '' + + libpath = os.path.join(hdp_version_dir, 'ranger-kms', 'cred', 'lib', '*') + aliasKey = ranger_kms_properties_from_file['ranger.ks.jpa.jdbc.credential.alias'] + if(aliasKey is None or len(aliasKey) == 0): + aliasKey = 'ranger.ks.jdbc.password' + aliasValue = '' + filepath = ranger_kms_properties_from_file['ranger.ks.jpa.jdbc.credential.provider.path'] + if (filepath is None or len(filepath) == 0) : + filepath = os.path.join(hdp_version_dir, 'ranger-kms', 'ews', 'webapp', 'WEB-INF', 'classes', 'conf', '.jceks', 'rangerkms.jceks') + + getorcreateorlist = 'get' + + statuscode, value = call_keystore(libpath, aliasKey, aliasValue, filepath, getorcreateorlist) + if statuscode == 0: + kms_database_password = value.strip() + print('kms_database_password = ' + kms_database_password) + aliasKey = ranger_kms_properties_from_file['ranger.ks.masterkey.credential.alias'] + if(aliasKey is None or len(aliasKey) == 0): + aliasKey = 'ranger.ks.masterkey.password' + statuscode, value = call_keystore(libpath, aliasKey, aliasValue, filepath, getorcreateorlist) + + if statuscode == 0: + kms_keystore_masterkey_password = value.strip() + + print('kms_keystore_masterkey_password = ' + kms_keystore_masterkey_password) + + if str(ranger_kms_properties_from_file['ranger.ks.jpa.jdbc.password']) == '_' or str(ranger_kms_properties_from_file['ranger.ks.jpa.jdbc.password']).lower() == 'crypted': + ranger_kms_properties_from_file['ranger.ks.jpa.jdbc.password'] = kms_database_password + ranger_kms_properties_from_file['KMS_MASTER_KEY_PASSWD'] = kms_keystore_masterkey_password + ranger_kms_properties_from_file['ranger.db.encrypt.key.password'] = kms_keystore_masterkey_password + + return ranger_kms_properties_from_file + + +def get_additional_properties_for_usersync(ranger_usersync_properties_from_file): + ranger_conf_path = '/etc/ranger/usersync/conf' + ranger_ugsync_default_site_xml_properties = os.path.join(ranger_conf_path, 'ranger-ugsync-default.xml') + ranger_ugsync_site_xml_properties = os.path.join(ranger_conf_path, 'ranger-ugsync-site.xml') + ranger_usersync_properties_from_file = import_properties_from_xml(ranger_ugsync_default_site_xml_properties, ranger_usersync_properties_from_file) + ranger_usersync_properties_from_file = import_properties_from_xml(ranger_ugsync_site_xml_properties, ranger_usersync_properties_from_file) + if (('unix'.lower()) in str(ranger_usersync_properties_from_file['ranger.usersync.source.impl.class']).lower()): + print('sync_source is unix') + ranger_usersync_properties_from_file['SYNC_SOURCE'] = 'unix' + if (('ldap'.lower()) in str(ranger_usersync_properties_from_file['ranger.usersync.source.impl.class']).lower()): + print('sync source is ldap') + ranger_usersync_properties_from_file['SYNC_SOURCE'] = 'ldap' + return ranger_usersync_properties_from_file + + +def port_ranger_hdfs_plugin_to_ambari(): + print('Trying to add ranger hdfs plugin.') + flag_hdfs_plugin_installed, hdfs_plugin_install_properties, hdfs_site_xml_properties = get_hdfs_plugin_configuration() + if flag_hdfs_plugin_installed and hdfs_plugin_install_properties is not None and hdfs_site_xml_properties is not None: + hdfs_site_xml_properties['dfs.permissions.enabled'] = 'true' + print('hdfs plugin is installed and enabled, adding to configurations') + advanced_ranger_hdfs_plugin_properties = dict() + advanced_ranger_hdfs_audit_properties = dict() + advanced_ranger_hdfs_policymgr_ssl_properties = dict() + advanced_ranger_hdfs_security_properties = dict() + + advanced_ranger_hdfs_plugin_properties['policy_user'] = 'ambari-qa' + advanced_ranger_hdfs_plugin_properties['hadoop.rpc.protection'] = '' + advanced_ranger_hdfs_plugin_properties['common.name.for.certificate'] = '' + advanced_ranger_hdfs_plugin_properties['ranger-hdfs-plugin-enabled'] = 'Yes' + advanced_ranger_hdfs_plugin_properties['REPOSITORY_CONFIG_USERNAME'] = 'hadoop' + advanced_ranger_hdfs_plugin_properties['REPOSITORY_CONFIG_PASSWORD'] = 'hadoop' + + advanced_ranger_hdfs_audit_properties['xasecure.audit.is.enabled'] = hdfs_plugin_install_properties.get('xasecure.audit.is.enabled','true') + advanced_ranger_hdfs_audit_properties['xasecure.audit.destination.db'] = hdfs_plugin_install_properties.get('xasecure.audit.db.is.enabled','false') + advanced_ranger_hdfs_audit_properties['xasecure.audit.destination.db.jdbc.url'] = hdfs_plugin_install_properties.get('xasecure.audit.jpa.javax.persistence.jdbc.url','{{audit_jdbc_url}}') + advanced_ranger_hdfs_audit_properties['xasecure.audit.destination.db.user'] = hdfs_plugin_install_properties.get('xasecure.audit.jpa.javax.persistence.jdbc.user','{{xa_audit_db_user}}') + advanced_ranger_hdfs_audit_properties['xasecure.audit.destination.db.password'] = hdfs_plugin_install_properties.get('xasecure.audit.jpa.javax.persistence.jdbc.password','crypted') + advanced_ranger_hdfs_audit_properties['xasecure.audit.destination.db.jdbc.driver'] = hdfs_plugin_install_properties.get('xasecure.audit.jpa.javax.persistence.jdbc.driver','{{jdbc_driver}}') + advanced_ranger_hdfs_audit_properties['xasecure.audit.credential.provider.file'] = hdfs_plugin_install_properties.get('xasecure.audit.credential.provider.file','jceks://file{{credential_file}}') + advanced_ranger_hdfs_audit_properties['xasecure.audit.destination.db.batch.filespool.dir'] = hdfs_plugin_install_properties.get('xasecure.audit.destination.db.batch.filespool.dir','/var/log/hadoop/hdfs/audit/db/spool') + advanced_ranger_hdfs_audit_properties['xasecure.audit.destination.hdfs'] = hdfs_plugin_install_properties.get('xasecure.audit.destination.hdfs','true') + advanced_ranger_hdfs_audit_properties['xasecure.audit.destination.hdfs.dir'] = hdfs_plugin_install_properties.get('xasecure.audit.destination.hdfs.dir','hdfs://NAMENODE_HOSTNAME:8020/ranger/audit') + advanced_ranger_hdfs_audit_properties['xasecure.audit.destination.hdfs.batch.filespool.dir'] = hdfs_plugin_install_properties.get('xasecure.audit.destination.hdfs.batch.filespool.dir','/var/log/hadoop/hdfs/audit/hdfs/spool') + advanced_ranger_hdfs_audit_properties['xasecure.audit.destination.solr'] = hdfs_plugin_install_properties.get('xasecure.audit.destination.solr','false') + advanced_ranger_hdfs_audit_properties['xasecure.audit.destination.solr.urls'] = hdfs_plugin_install_properties.get('xasecure.audit.destination.solr.urls','{{ranger_audit_solr_urls}}') + advanced_ranger_hdfs_audit_properties['xasecure.audit.destination.solr.zookeepers'] = hdfs_plugin_install_properties.get('xasecure.audit.is.enabled','none') + advanced_ranger_hdfs_audit_properties['xasecure.audit.destination.solr.batch.filespool.dir'] = hdfs_plugin_install_properties.get('xasecure.audit.destination.solr.batch.filespool.dir','/var/log/hadoop/hdfs/audit/solr/spool') + advanced_ranger_hdfs_audit_properties['xasecure.audit.provider.summary.enabled'] = hdfs_plugin_install_properties.get('xasecure.audit.provider.summary.enabled','false') + + + advanced_ranger_hdfs_policymgr_ssl_properties['xasecure.policymgr.clientssl.keystore'] = hdfs_plugin_install_properties.get('xasecure.policymgr.clientssl.keystore','/usr/hdp/current/hadoop-client/conf/ranger-plugin-keystore.jks') + advanced_ranger_hdfs_policymgr_ssl_properties['xasecure.policymgr.clientssl.keystore.password'] = hdfs_plugin_install_properties.get('xasecure.policymgr.clientssl.keystore.password','myKeyFilePassword') + advanced_ranger_hdfs_policymgr_ssl_properties['xasecure.policymgr.clientssl.truststore'] = hdfs_plugin_install_properties.get('xasecure.policymgr.clientssl.truststore','/usr/hdp/current/hadoop-client/conf/ranger-plugin-truststore.jks') + advanced_ranger_hdfs_policymgr_ssl_properties['xasecure.policymgr.clientssl.truststore.password'] = hdfs_plugin_install_properties.get('xasecure.policymgr.clientssl.truststore.password','changeit') + advanced_ranger_hdfs_policymgr_ssl_properties['xasecure.policymgr.clientssl.keystore.credential.file'] = hdfs_plugin_install_properties.get('xasecure.policymgr.clientssl.keystore.credential.file','jceks://file{{credential_file}}') + advanced_ranger_hdfs_policymgr_ssl_properties['xasecure.policymgr.clientssl.truststore.credential.file'] = hdfs_plugin_install_properties.get('xasecure.policymgr.clientssl.truststore.credential.file','jceks://file{{credential_file}}') + + + + advanced_ranger_hdfs_security_properties['ranger.plugin.hdfs.service.name'] = hdfs_plugin_install_properties.get('ranger.plugin.hdfs.service.name','{{repo_name}}') + advanced_ranger_hdfs_security_properties['ranger.plugin.hdfs.policy.source.impl'] = hdfs_plugin_install_properties.get('ranger.plugin.hdfs.policy.source.impl','org.apache.ranger.admin.client.RangerAdminRESTClient') + advanced_ranger_hdfs_security_properties['ranger.plugin.hdfs.policy.rest.url'] = hdfs_plugin_install_properties.get('ranger.plugin.hdfs.policy.rest.url','{{policymgr_mgr_url}}') + advanced_ranger_hdfs_security_properties['ranger.plugin.hdfs.policy.rest.ssl.config.file'] = hdfs_plugin_install_properties.get('ranger.plugin.hdfs.policy.rest.ssl.config.file','/etc/hadoop/conf/ranger-policymgr-ssl.xml') + advanced_ranger_hdfs_security_properties['ranger.plugin.hdfs.policy.pollIntervalMs'] = hdfs_plugin_install_properties.get('ranger.plugin.hdfs.policy.pollIntervalMs','30000') + advanced_ranger_hdfs_security_properties['ranger.plugin.hdfs.policy.cache.dir'] = hdfs_plugin_install_properties.get('ranger.plugin.hdfs.policy.cache.dir','/etc/ranger/{{repo_name}}/policycache') + advanced_ranger_hdfs_security_properties['xasecure.add-hadoop-authorization'] = hdfs_plugin_install_properties.get('xasecure.add-hadoop-authorization','true') + + + date_time_stamp = getDateTimeNow() + + plugin_configuration_data = '[{"Clusters":{"desired_config":[{"type":"hdfs-site", "service_config_version_note": "Initial configuration for Ranger HDFS plugin" ,"tag":"' + str(date_time_stamp) + '","properties":' + str( + json.dumps(hdfs_site_xml_properties)) + ', "properties_attributes": {"final": "false"}},{"type": "ranger-hdfs-plugin-properties", "service_config_version_note": "Initial configuration for Ranger HDFS plugin" , "tag": "' + str( + date_time_stamp) + '", "properties":' + json.dumps(advanced_ranger_hdfs_plugin_properties) + ',"properties_attributes": {"final": "false"}},{"type": "ranger-hdfs-audit", "service_config_version_note": "Initial configuration for Ranger HDFS plugin" , "tag": "' + str( + date_time_stamp) + '", "properties":' + json.dumps(advanced_ranger_hdfs_audit_properties) + ',"properties_attributes": {"final": "false"}},{"type": "ranger-hdfs-policymgr-ssl", "service_config_version_note": "Initial configuration for Ranger HDFS plugin" , "tag": "' + str( + date_time_stamp) + '", "properties":' + json.dumps(advanced_ranger_hdfs_policymgr_ssl_properties) + ',"properties_attributes": {"final": "false"}},{"type": "ranger-hdfs-security", "service_config_version_note": "Initial configuration for Ranger HDFS plugin" , "tag": "' + str( + date_time_stamp) + '", "properties":' + json.dumps(advanced_ranger_hdfs_security_properties) + ',"properties_attributes": {"final": "false"}}]}}]' + + print ('####################### hdfs_site_xml configuration :') + for each_key in hdfs_site_xml_properties: + print str(each_key) + ' = ' + str(hdfs_site_xml_properties[each_key]) + + print ('####################### ranger_hdfs_plugin_properties configuration :') + for each_key in advanced_ranger_hdfs_plugin_properties: + print str(each_key) + ' = ' + str(advanced_ranger_hdfs_plugin_properties[each_key]) + + print ('####################### ranger_hdfs_audit_properties configuration :') + for each_key in advanced_ranger_hdfs_audit_properties: + print str(each_key) + ' = ' + str(advanced_ranger_hdfs_audit_properties[each_key]) + + print ('####################### ranger_hdfs_policymgr_ssl_properties configuration :') + for each_key in advanced_ranger_hdfs_policymgr_ssl_properties: + print str(each_key) + ' = ' + str(advanced_ranger_hdfs_policymgr_ssl_properties[each_key]) + + print ('####################### ranger_hdfs_security_properties configuration :') + for each_key in advanced_ranger_hdfs_security_properties: + print str(each_key) + ' = ' + str(advanced_ranger_hdfs_security_properties[each_key]) + + confirm_configurations = raw_input('please confirm the above configuration values y/n (n) : ') + if(confirm_configurations == ''): + confirm_configurations = 'n' + print ('Input registered as ' + str(confirm_configurations)) + + + if(confirm_configurations.lower() == 'y'): + ranger_config_request_url = ambari_cluster_url + request_result = call_ambari_api(ranger_config_request_url, 'PUT', ambari_username_password, str(plugin_configuration_data)) + if request_result is not None: + response_code = request_result.getcode() + response = json.loads(json.JSONEncoder().encode(request_result.read())) + if response_code == 200 and response is not None: + print('ranger hdfs plugin configuration added successfully in ambari.') + return True + else: + print('ranger hdfs plugin configuration add failed in ambari.') + return False + else: + print('ranger hdfs plugin configuration add failed in ambari.') + return False + else: + print ('exiting installation without configuration !') + sys.exit(0) + else: + print('ranger hdfs plugin configuration add failed in ambari.') + return False + +def port_ranger_yarn_plugin_to_ambari(): + print('Trying to add ranger yarn plugin.') + flag_yarn_plugin_installed,yarn_plugin_installed_properties,yarn_site_xml_properties = get_yarn_plugin_configuration() + if( flag_yarn_plugin_installed and yarn_plugin_installed_properties is not None and yarn_site_xml_properties is not None): + print('yarn plugin is installed and enabled, adding to configurations') + advanced_ranger_yarn_plugin_properties = dict() + advanced_ranger_yarn_audit_properties = dict() + advanced_ranger_yarn_policymgr_ssl_properties = dict() + advanced_ranger_yarn_security_properties = dict() + + advanced_ranger_yarn_plugin_properties['policy_user'] = 'ambari-qa' + advanced_ranger_yarn_plugin_properties['hadoop.rpc.protection'] = '' + advanced_ranger_yarn_plugin_properties['common.name.for.certificate'] = '' + advanced_ranger_yarn_plugin_properties['ranger-yarn-plugin-enabled'] = 'Yes' + advanced_ranger_yarn_plugin_properties['REPOSITORY_CONFIG_USERNAME'] = 'yarn' + advanced_ranger_yarn_plugin_properties['REPOSITORY_CONFIG_PASSWORD'] = 'yarn' + + + advanced_ranger_yarn_audit_properties['xasecure.audit.is.enabled'] = yarn_plugin_installed_properties.get('xasecure.audit.is.enabled','true') + advanced_ranger_yarn_audit_properties['xasecure.audit.destination.db'] = yarn_plugin_installed_properties.get('xasecure.audit.db.is.enabled','false') + advanced_ranger_yarn_audit_properties['xasecure.audit.destination.db.jdbc.url'] = yarn_plugin_installed_properties.get('xasecure.audit.jpa.javax.persistence.jdbc.url','{{audit_jdbc_url}}') + advanced_ranger_yarn_audit_properties['xasecure.audit.destination.db.user'] = yarn_plugin_installed_properties.get('xasecure.audit.jpa.javax.persistence.jdbc.user','{{xa_audit_db_user}}') + advanced_ranger_yarn_audit_properties['xasecure.audit.destination.db.password'] = yarn_plugin_installed_properties.get('xasecure.audit.jpa.javax.persistence.jdbc.password','crypted') + advanced_ranger_yarn_audit_properties['xasecure.audit.destination.db.jdbc.driver'] = yarn_plugin_installed_properties.get('xasecure.audit.jpa.javax.persistence.jdbc.driver','{{jdbc_driver}}') + advanced_ranger_yarn_audit_properties['xasecure.audit.credential.provider.file'] = yarn_plugin_installed_properties.get('xasecure.audit.credential.provider.file','jceks://file{{credential_file}}') + advanced_ranger_yarn_audit_properties['xasecure.audit.destination.db.batch.filespool.dir'] = yarn_plugin_installed_properties.get('xasecure.audit.destination.db.batch.filespool.dir','/var/log/hadoop/yarn/audit/db/spool') + advanced_ranger_yarn_audit_properties['xasecure.audit.destination.hdfs'] = yarn_plugin_installed_properties.get('xasecure.audit.destination.hdfs','true') + advanced_ranger_yarn_audit_properties['xasecure.audit.destination.hdfs.dir'] = yarn_plugin_installed_properties.get('xasecure.audit.destination.hdfs.dir','hdfs://NAMENODE_HOSTNAME:8020/ranger/audit') + advanced_ranger_yarn_audit_properties['xasecure.audit.destination.hdfs.batch.filespool.dir'] = yarn_plugin_installed_properties.get('xasecure.audit.destination.hdfs.batch.filespool.dir','/var/log/hadoop/yarn/audit/hdfs/spool') + advanced_ranger_yarn_audit_properties['xasecure.audit.destination.solr'] = yarn_plugin_installed_properties.get('xasecure.audit.destination.solr','false') + advanced_ranger_yarn_audit_properties['xasecure.audit.destination.solr.urls'] = yarn_plugin_installed_properties.get('xasecure.audit.destination.solr.urls','{{ranger_audit_solr_urls}}') + advanced_ranger_yarn_audit_properties['xasecure.audit.destination.solr.zookeepers'] = yarn_plugin_installed_properties.get('xasecure.audit.destination.solr.zookeepers','none') + advanced_ranger_yarn_audit_properties['xasecure.audit.destination.solr.batch.filespool.dir'] = yarn_plugin_installed_properties.get('xasecure.audit.destination.solr.batch.filespool.dir','/var/log/hadoop/yarn/audit/solr/spool') + advanced_ranger_yarn_audit_properties['xasecure.audit.provider.summary.enabled'] = yarn_plugin_installed_properties.get('xasecure.audit.provider.summary.enabled','false') + + advanced_ranger_yarn_policymgr_ssl_properties['xasecure.policymgr.clientssl.keystore'] = yarn_plugin_installed_properties.get('xasecure.policymgr.clientssl.keystore','/usr/hdp/current/hadoop-client/conf/ranger-yarn-plugin-keystore.jks') + advanced_ranger_yarn_policymgr_ssl_properties['xasecure.policymgr.clientssl.keystore.password'] = yarn_plugin_installed_properties.get('xasecure.policymgr.clientssl.keystore.password','myKeyFilePassword') + advanced_ranger_yarn_policymgr_ssl_properties['xasecure.policymgr.clientssl.truststore'] = yarn_plugin_installed_properties.get('xasecure.policymgr.clientssl.truststore','/usr/hdp/current/hadoop-client/conf/ranger-yarn-plugin-truststore.jks') + advanced_ranger_yarn_policymgr_ssl_properties['xasecure.policymgr.clientssl.truststore.password'] = yarn_plugin_installed_properties.get('xasecure.policymgr.clientssl.truststore.password','changeit') + advanced_ranger_yarn_policymgr_ssl_properties['xasecure.policymgr.clientssl.keystore.credential.file'] = yarn_plugin_installed_properties.get('xasecure.policymgr.clientssl.keystore.credential.file','jceks://file{{credential_file}
<TRUNCATED>
