Repository: ambari Updated Branches: refs/heads/trunk 71305effa -> efa35f49b
AMBARI-16182. Add new component level identity for RANGER_KMS in kerberos.json for stack 2.5(Mugdha Varadkar via gautam) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/efa35f49 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/efa35f49 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/efa35f49 Branch: refs/heads/trunk Commit: efa35f49b82a4db88bc49830d63df41c3ae33ccc Parents: 71305ef Author: Gautam Borad <[email protected]> Authored: Mon May 2 18:04:22 2016 +0530 Committer: Gautam Borad <[email protected]> Committed: Wed May 4 13:34:56 2016 +0530 ---------------------------------------------------------------------- .../libraries/functions/ranger_functions_v2.py | 2 +- .../RANGER_KMS/0.5.0.2.3/package/scripts/kms.py | 66 ++++++++++++-------- .../0.5.0.2.3/package/scripts/params.py | 11 ++++ .../RANGER/configuration/ranger-admin-site.xml | 2 +- .../RANGER_KMS/configuration/dbks-site.xml | 18 ++++++ .../HDP/2.5/services/RANGER_KMS/kerberos.json | 65 +++++++++++++++++++ 6 files changed, 135 insertions(+), 29 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/efa35f49/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions_v2.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions_v2.py b/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions_v2.py index 9709713..4f319ea 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions_v2.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions_v2.py @@ -371,7 +371,7 @@ class RangeradminV2: response,error_message,time_in_millis = self.call_curl_request(component_user,component_user_keytab,component_user_principal,search_repo_url,False,method,data,header) if response and len(response) > 0: response_json = json.loads(response) - if response_json['name'].lower() == name.lower(): + if 'name' in response_json and response_json['name'].lower() == name.lower(): Logger.info('Repository created Successfully') service_name = response_json['name'] service_type = response_json['type'] http://git-wip-us.apache.org/repos/asf/ambari/blob/efa35f49/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py index 9e54649..5ec5525 100755 --- a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py +++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py @@ -33,6 +33,8 @@ from resource_management.core.exceptions import Fail from resource_management.core.logger import Logger from resource_management.libraries.functions.format import format from resource_management.libraries.functions.ranger_functions import Rangeradmin +from resource_management.libraries.functions.ranger_functions_v2 import RangeradminV2 +from resource_management.libraries.functions.decorator import safe_retry from resource_management.core.utils import PasswordString from resource_management.core.shell import as_sudo import re @@ -343,16 +345,14 @@ def enable_kms_plugin(): import params if params.has_ranger_admin: - count = 0 - while count < 5: - ranger_flag = check_ranger_service() - if ranger_flag: - break - else: - time.sleep(5) # delay for 5 seconds - count = count + 1 + + if params.stack_supports_ranger_kerberos and params.security_enabled: + ranger_flag = check_ranger_service_support_kerberos() else: - Logger.error("Ranger service is not reachable after {0} tries".format(count)) + ranger_flag = check_ranger_service() + + if not ranger_flag: + Logger.error('Error in Get/Create service for Ranger Kms.') current_datetime = datetime.now() @@ -458,20 +458,16 @@ def check_ranger_service(): if user_resp_code is not None and user_resp_code == 200: get_repo_flag = get_repo(params.policymgr_mgr_url, params.repo_name, ambari_username_password_for_ranger) if not get_repo_flag: - create_repo_flag = create_repo(params.policymgr_mgr_url, json.dumps(params.kms_ranger_plugin_repo), ambari_username_password_for_ranger) - if create_repo_flag: - return True - else: - return False + return create_repo(params.policymgr_mgr_url, json.dumps(params.kms_ranger_plugin_repo), ambari_username_password_for_ranger) else: return True else: - Logger.error('Ambari admin user creation failed') return False else: - Logger.error('Ranger service is not reachable host') + Logger.error('Ranger service is not reachable') return False +@safe_retry(times=5, sleep_time=8, backoff_factor=1.5, err_class=Fail, return_on_fail=False) def create_repo(url, data, usernamepassword): try: base_url = url + '/service/public/v2/api/service' @@ -493,15 +489,13 @@ def create_repo(url, data, usernamepassword): return False except urllib2.URLError, e: if isinstance(e, urllib2.HTTPError): - Logger.error("Error creating service. Http status code - {0}. \n {1}".format(e.code, e.read())) - return False + raise Fail("Error creating service. Http status code - {0}. \n {1}".format(e.code, e.read())) else: - Logger.error("Error creating service. Reason - {0}.".format(e.reason)) - return False + raise Fail("Error creating service. Reason - {0}.".format(e.reason)) except socket.timeout as e: - Logger.error("Error creating service. Reason - {0}".format(e)) - return False + raise Fail("Error creating service. Reason - {0}".format(e)) +@safe_retry(times=5, sleep_time=8, backoff_factor=1.5, err_class=Fail, return_on_fail=False) def get_repo(url, name, usernamepassword): try: base_url = url + '/service/public/v2/api/service?serviceName=' + name + '&serviceType=kms&isEnabled=true' @@ -526,11 +520,29 @@ def get_repo(url, name, usernamepassword): return False except urllib2.URLError, e: if isinstance(e, urllib2.HTTPError): - Logger.error("Error getting {0} service. Http status code - {1}. \n {2}".format(name, e.code, e.read())) - return False + raise Fail("Error getting {0} service. Http status code - {1}. \n {2}".format(name, e.code, e.read())) else: - Logger.error("Error getting {0} service. Reason - {1}.".format(name, e.reason)) - return False + raise Fail("Error getting {0} service. Reason - {1}.".format(name, e.reason)) except socket.timeout as e: - Logger.error("Error creating service. Reason - {0}".format(e)) + raise Fail("Error creating service. Reason - {0}".format(e)) + +def check_ranger_service_support_kerberos(): + import params + + ranger_adm_obj = RangeradminV2(url=params.policymgr_mgr_url) + response_code = ranger_adm_obj.check_ranger_login_curl(params.kms_user, params.rangerkms_keytab, params.rangerkms_principal, params.policymgr_mgr_url, True) + + if response_code is not None and response_code[0] == 200: + get_repo_name_response = ranger_adm_obj.get_repository_by_name_curl(params.kms_user, params.rangerkms_keytab, params.rangerkms_principal, params.repo_name, 'kms', 'true') + if get_repo_name_response is not None: + Logger.info('KMS repository {0} exist'.format(get_repo_name_response['name'])) + return True + else: + create_repo_response = ranger_adm_obj.create_repository_curl(params.kms_user, params.rangerkms_keytab, params.rangerkms_principal, params.repo_name, json.dumps(params.kms_ranger_plugin_repo), None) + if create_repo_response is not None and len(create_repo_response) > 0: + return True + else: + return False + else: + Logger.error('Ranger service is not reachable') return False http://git-wip-us.apache.org/repos/asf/ambari/blob/efa35f49/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/params.py b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/params.py index ce136b2..c33d9da 100755 --- a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/params.py @@ -38,6 +38,7 @@ stack_version_formatted = format_stack_version(stack_version_unformatted) stack_supports_config_versioning = stack_version_formatted and check_stack_feature(StackFeature.CONFIG_VERSIONING, stack_version_formatted) stack_support_kms_hsm = stack_version_formatted and check_stack_feature(StackFeature.RANGER_KMS_HSM_SUPPORT, stack_version_formatted) +stack_supports_ranger_kerberos = stack_version_formatted and check_stack_feature(StackFeature.RANGER_KERBEROS_SUPPORT, stack_version_formatted) hadoop_conf_dir = conf_select.get_hadoop_conf_dir() security_enabled = config['configurations']['cluster-env']['security_enabled'] @@ -177,6 +178,9 @@ kms_plugin_config = { 'provider' : format('kms://http@{kms_host}:{kms_port}/kms') } +if stack_supports_ranger_kerberos: + kms_plugin_config['policy.download.auth.users'] = 'keyadmin' + kms_ranger_plugin_repo = { 'isEnabled' : 'true', 'configs' : kms_plugin_config, @@ -212,3 +216,10 @@ jce_source_dir = format('{tmp_dir}/jce_dir') enable_kms_hsm = default("/configurations/dbks-site/ranger.ks.hsm.enabled", False) hms_partition_alias = default("/configurations/dbks-site/ranger.ks.hsm.partition.password.alias", "ranger.kms.hsm.partition.password") hms_partition_passwd = default("/configurations/kms-env/hsm_partition_password", None) + +# kms kerberos from stack 2.5 onward +rangerkms_keytab = config['configurations']['dbks-site']['ranger.ks.kerberos.keytab'] +if stack_supports_ranger_kerberos and security_enabled: + rangerkms_principal = default("/configurations/dbks-site/ranger.ks.kerberos.principal", None) + if rangerkms_principal is not None: + rangerkms_principal = rangerkms_principal.replace('_HOST', kms_host.lower()) http://git-wip-us.apache.org/repos/asf/ambari/blob/efa35f49/ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER/configuration/ranger-admin-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER/configuration/ranger-admin-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER/configuration/ranger-admin-site.xml index 1a5dd7c..ef05604 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER/configuration/ranger-admin-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER/configuration/ranger-admin-site.xml @@ -49,7 +49,7 @@ </property> <property> - <name>ranger.admin.kerberos.token.valid</name> + <name>ranger.admin.kerberos.token.valid.seconds</name> <value>30</value> <description></description> </property> http://git-wip-us.apache.org/repos/asf/ambari/blob/efa35f49/ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER_KMS/configuration/dbks-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER_KMS/configuration/dbks-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER_KMS/configuration/dbks-site.xml index b652574..fca42b3 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER_KMS/configuration/dbks-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER_KMS/configuration/dbks-site.xml @@ -75,4 +75,22 @@ <description>HSM partition password alias</description> </property> + <property> + <name>ranger.ks.kerberos.principal</name> + <value></value> + <description></description> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + </property> + + <property> + <name>ranger.ks.kerberos.keytab</name> + <value></value> + <description></description> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + </property> + </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/efa35f49/ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER_KMS/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER_KMS/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER_KMS/kerberos.json new file mode 100644 index 0000000..bfd142a --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER_KMS/kerberos.json @@ -0,0 +1,65 @@ +{ + "services": [ + { + "name": "RANGER_KMS", + "identities": [ + { + "name": "/spnego", + "keytab": { + "configuration": "kms-site/hadoop.kms.authentication.kerberos.keytab" + } + }, + { + "name": "/smokeuser" + } + ], + "auth_to_local_properties" : [ + "kms-site/hadoop.kms.authentication.kerberos.name.rules" + ], + "configurations": [ + { + "kms-site": { + "hadoop.kms.authentication.type": "kerberos", + "hadoop.kms.authentication.kerberos.principal": "*" + } + } + ], + "components": [ + { + "name": "RANGER_KMS_SERVER", + "identities": [ + { + "name": "/spnego", + "principal": { + "configuration": "kms-site/hadoop.kms.authentication.signer.secret.provider.zookeeper.kerberos.principal" + }, + "keytab": { + "configuration": "kms-site/hadoop.kms.authentication.signer.secret.provider.zookeeper.kerberos.keytab" + } + }, + { + "name": "/smokeuser" + }, + { + "name": "rangerkms", + "principal": { + "value": "rangerkms/_HOST@${realm}", + "type" : "service", + "configuration": "dbks-site/ranger.ks.kerberos.principal", + "local_username" : "keyadmin" + }, + "keytab": { + "file": "${keytab_dir}/rangerkms.service.keytab", + "owner": { + "name": "${kms-env/kms_user}", + "access": "r" + }, + "configuration": "dbks-site/ranger.ks.kerberos.keytab" + } + } + ] + } + ] + } + ] +} \ No newline at end of file
