AMBARI-18338. Microsoft-R client should work in a secured cluster (Balazs bence Sari via magyari_sandor)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/cc684be1 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/cc684be1 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/cc684be1 Branch: refs/heads/branch-dev-patch-upgrade Commit: cc684be15b1fa5404ed86918f8b13d8ae2b0f1ae Parents: 561dc8e Author: Balazs Bence Sari <bs...@hortonworks.com> Authored: Thu Sep 15 14:18:38 2016 +0200 Committer: Sandor Magyari <smagy...@hortonworks.com> Committed: Thu Sep 15 14:18:38 2016 +0200 ---------------------------------------------------------------------- .../MICROSOFT_R/8.0.0/kerberos.json | 15 ++++++++++ .../MICROSOFT_R/8.0.0/package/scripts/params.py | 1 + .../8.0.0/package/scripts/params_linux.py | 29 ++++++++++++++++++++ .../8.0.0/package/scripts/service_check.py | 11 +++++++- 4 files changed, 55 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/cc684be1/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/kerberos.json ---------------------------------------------------------------------- diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/kerberos.json b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/kerberos.json new file mode 100644 index 0000000..86fab2d --- /dev/null +++ b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/kerberos.json @@ -0,0 +1,15 @@ +{ + "services": [ + { + "name": "MICROSOFT_R", + "identities": [ + { + "name": "/HDFS/NAMENODE/hdfs" + }, + { + "name": "/smokeuser" + } + ] + } + ] +} http://git-wip-us.apache.org/repos/asf/ambari/blob/cc684be1/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/scripts/params.py b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/scripts/params.py index 36541e7..aa20797 100644 --- a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/scripts/params.py +++ b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/scripts/params.py @@ -28,3 +28,4 @@ else: host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False) +revo_share_hdfs_folder = "/user/RevoShare" \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/cc684be1/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/scripts/params_linux.py b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/scripts/params_linux.py index d72c42e..71f17b6 100644 --- a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/scripts/params_linux.py +++ b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/scripts/params_linux.py @@ -44,7 +44,36 @@ smoke_hdfs_user_mode = 0770 user_group = config['configurations']['cluster-env']['user_group'] security_enabled = config['configurations']['cluster-env']['security_enabled'] smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] +hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] +hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None)) # not supporting 32 bit jdk. java64_home = config['hostLevelParams']['java_home'] + +hadoop_bin_dir = stack_select.get_hadoop_dir("bin") +hdfs_user_principal = default('/configurations/hadoop-env/hdfs_principal_name', None) +hdfs_site = config['configurations']['hdfs-site'] +default_fs = config['configurations']['core-site']['fs.defaultFS'] +dfs_type = default("/commandParams/dfs_type", "") +hadoop_conf_dir = "/etc/hadoop/conf" + + +import functools +#create partial functions with common arguments for every HdfsResource call +#to create/delete/copyfromlocal hdfs directories/files we need to call params.HdfsResource in code +HdfsResource = functools.partial( + HdfsResource, + user=hdfs_user, + hdfs_resource_ignore_file = "/var/lib/ambari-agent/data/.hdfs_resource_ignore", + security_enabled = security_enabled, + keytab = hdfs_user_keytab, + kinit_path_local = kinit_path_local, + hadoop_bin_dir = hadoop_bin_dir, + hadoop_conf_dir = hadoop_conf_dir, + principal_name = hdfs_user_principal, + hdfs_site = hdfs_site, + default_fs = default_fs, + immutable_paths = get_not_managed_resources(), + dfs_type = dfs_type +) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/cc684be1/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/scripts/service_check.py b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/scripts/service_check.py index 8a425ca..547476b 100644 --- a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/scripts/service_check.py +++ b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/scripts/service_check.py @@ -27,9 +27,9 @@ from resource_management.core.resources.system import Execute, File from resource_management.core.source import InlineTemplate, StaticFile from resource_management.libraries.functions.format import format from resource_management.libraries.script.script import Script -from ambari_commons import OSConst from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl + class MicrosoftRServiceCheck(Script): pass @@ -39,6 +39,15 @@ class MicrosoftRServiceCheckLinux(MicrosoftRServiceCheck): import params env.set_params(params) + try: + params.HdfsResource(params.revo_share_hdfs_folder, + type="directory", + action="create_on_execute", + owner=params.hdfs_user, + mode=0777) + except Exception as exception: + Logger.warning("Could not check the existence of /user/RevoShare on HDFS, exception: {0}".format(str(exception))) + if params.security_enabled: kinit_cmd = format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser_principal};") Execute(kinit_cmd,
