Repository: ambari Updated Branches: refs/heads/branch-2.0.maint 4342a0602 -> 113eb2362
AMBARI-13073. Backport Request - Ambari is overwriting ownership of yarn.local.dir with yarn:hadoop in Kerberized cluster (dlysnichenko) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/113eb236 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/113eb236 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/113eb236 Branch: refs/heads/branch-2.0.maint Commit: 113eb2362101cbca8d0f7bcd7ce8e94d307b08a8 Parents: 4342a06 Author: Lisnichenko Dmitro <[email protected]> Authored: Mon Sep 14 15:03:56 2015 +0300 Committer: Lisnichenko Dmitro <[email protected]> Committed: Mon Sep 14 15:03:56 2015 +0300 ---------------------------------------------------------------------- .../YARN/2.1.0.2.0/package/scripts/params.py | 7 ++ .../YARN/2.1.0.2.0/package/scripts/yarn.py | 69 ++++++++++++++------ .../stacks/2.0.6/YARN/test_nodemanager.py | 31 ++++----- .../stacks/2.0.6/YARN/test_resourcemanager.py | 8 ++- 4 files changed, 77 insertions(+), 38 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/113eb236/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params.py index 29e8dfb..c396650 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params.py @@ -91,6 +91,12 @@ hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] smokeuser = config['configurations']['cluster-env']['smokeuser'] smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name'] security_enabled = config['configurations']['cluster-env']['security_enabled'] +nm_security_marker = '/var/lib/hadoop-yarn/nm_security_enabled' +current_nm_security_state = os.path.isfile(nm_security_marker) +toggle_nm_security = (current_nm_security_state and not security_enabled) or (not current_nm_security_state and security_enabled) +rm_security_marker = "/var/lib/hadoop-yarn/rm_security_enabled" +current_rm_security_state = os.path.isfile(rm_security_marker) +toggle_rm_security = (current_rm_security_state and not security_enabled) or (not current_rm_security_state and security_enabled) smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] yarn_executor_container_group = config['configurations']['yarn-site']['yarn.nodemanager.linux-container-executor.group'] kinit_path_local = functions.get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None)) @@ -115,6 +121,7 @@ mapred_pid_dir_prefix = status_params.mapred_pid_dir_prefix mapred_log_dir_prefix = config['configurations']['mapred-env']['mapred_log_dir_prefix'] mapred_env_sh_template = config['configurations']['mapred-env']['content'] yarn_env_sh_template = config['configurations']['yarn-env']['content'] +yarn_nodemanager_recovery_dir = default('/configurations/yarn-site/yarn.nodemanager.recovery.dir', None) if len(rm_hosts) > 1: additional_rm_host = rm_hosts[1] http://git-wip-us.apache.org/repos/asf/ambari/blob/113eb236/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py index 0736663..bdea2d7 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py @@ -61,28 +61,47 @@ def yarn(name = None): params.HdfsDirectory(None, action="create") if name == "nodemanager": - Directory(params.nm_local_dirs.split(',') + params.nm_log_dirs.split(','), - owner=params.yarn_user, - group=params.user_group, - recursive=True, - cd_access="a", - ignore_failures=True, - mode=0775 - ) - - Execute(('chown', '-R', params.yarn_user, params.nm_local_dirs), - only_if=format("test -d {nm_local_dirs}"), - sudo=True) + # First start after enabling/disabling security + if params.toggle_nm_security: + Directory(params.nm_local_dirs.split(',') + params.nm_log_dirs.split(','), + action='delete' + ) + # If yarn.nodemanager.recovery.dir exists, remove this dir + if params.yarn_nodemanager_recovery_dir: + Directory(InlineTemplate(params.yarn_nodemanager_recovery_dir).get_content(), + action='delete' + ) - if params.security_enabled: - smokeuser_directories = [os.path.join(dir, 'usercache' ,params.smokeuser) - for dir in params.nm_local_dirs.split(',')] - for directory in smokeuser_directories: - Execute(('chown', '-R', params.smokeuser, directory), - only_if=format("test -d {directory}"), - sudo=True, + # Setting NM marker file + if params.security_enabled: + File(params.nm_security_marker, + content="Marker file to track first start after enabling/disabling security. " + "During first start yarn local, log dirs are removed and recreated" ) + elif not params.security_enabled: + File(params.nm_security_marker, action="delete") + + + if not params.security_enabled or params.toggle_nm_security: + Directory(params.nm_local_dirs.split(',') + params.nm_log_dirs.split(','), + owner=params.yarn_user, + group=params.user_group, + recursive=True, + cd_access="a", + ignore_failures=True, + mode=0775 + ) + + if params.yarn_nodemanager_recovery_dir: + Directory(InlineTemplate(params.yarn_nodemanager_recovery_dir).get_content(), + owner=params.yarn_user, + group=params.user_group, + recursive=True, + mode=0755, + cd_access = 'a', + ) + Directory([params.yarn_pid_dir_prefix, params.yarn_pid_dir, params.yarn_log_dir], owner=params.yarn_user, group=params.user_group, @@ -151,6 +170,18 @@ def yarn(name = None): group=params.user_group, mode=0700 ) + if params.toggle_rm_security: + Execute('yarn resourcemanager -format-state-store', user = params.yarn_user, + ) + # Setting RM marker file + if params.security_enabled: + File(params.rm_security_marker, + content="Marker file to track first start after enabling/disabling security. " + "During first start ResourceManager state store is formatted" + ) + elif not params.security_enabled: + File(params.rm_security_marker, action="delete") + elif name == 'apptimelineserver': Directory(params.ats_leveldb_dir, owner=params.yarn_user, http://git-wip-us.apache.org/repos/asf/ambari/blob/113eb236/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py index 0956565..9ac71e7 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py +++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py @@ -230,11 +230,6 @@ class TestNodeManager(RMFTestCase): mode = 0775, cd_access='a' ) - self.assertResourceCalled('Execute', ('chown', '-R', u'yarn', u'/hadoop/yarn/local,/hadoop/yarn/local1'), - sudo = True, - only_if = 'test -d /hadoop/yarn/local,/hadoop/yarn/local1', - ) - self.assertResourceCalled('Directory', '/var/run/hadoop-yarn', owner = 'yarn', group = 'hadoop', @@ -436,30 +431,30 @@ class TestNodeManager(RMFTestCase): action = ['create'], ) self.assertResourceCalled('Directory', '/hadoop/yarn/local', - owner = 'yarn', + action = ['delete'], + ) + self.assertResourceCalled('Directory', '/hadoop/yarn/log', + action = ['delete'], + ) + self.assertResourceCalled('File', '/var/lib/hadoop-yarn/nm_security_enabled', + content = 'Marker file to track first start after enabling/disabling security. During first start yarn local, log dirs are removed and recreated', + ) + self.assertResourceCalled('Directory', '/hadoop/yarn/local', group = 'hadoop', recursive = True, + cd_access = 'a', ignore_failures = True, mode = 0775, - cd_access='a' + owner = 'yarn', ) self.assertResourceCalled('Directory', '/hadoop/yarn/log', - owner = 'yarn', group = 'hadoop', recursive = True, + cd_access = 'a', ignore_failures = True, mode = 0775, - cd_access='a' - ) - self.assertResourceCalled('Execute', ('chown', '-R', u'yarn', u'/hadoop/yarn/local'), - sudo = True, - only_if = 'test -d /hadoop/yarn/local', + owner = 'yarn', ) - - self.assertResourceCalled('Execute', ('chown', '-R', u'ambari-qa', u'/hadoop/yarn/local/usercache/ambari-qa'), - sudo = True, - only_if = 'test -d /hadoop/yarn/local/usercache/ambari-qa', - ) self.assertResourceCalled('Directory', '/var/run/hadoop-yarn', owner = 'yarn', group = 'hadoop', http://git-wip-us.apache.org/repos/asf/ambari/blob/113eb236/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py index d32fc1a..ffef460 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py +++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py @@ -396,6 +396,12 @@ class TestResourceManager(RMFTestCase): owner = 'yarn', group = 'hadoop', ) + self.assertResourceCalled('Execute', 'yarn resourcemanager -format-state-store', + user = 'yarn', + ) + self.assertResourceCalled('File', '/var/lib/hadoop-yarn/rm_security_enabled', + content = 'Marker file to track first start after enabling/disabling security. During first start ResourceManager state store is formatted', + ) self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude', owner = 'yarn', group = 'hadoop', @@ -571,4 +577,4 @@ class TestResourceManager(RMFTestCase): hdp_stack_version = self.STACK_VERSION, target = RMFTestCase.TARGET_COMMON_SERVICES ) - put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"}) \ No newline at end of file + put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"})
