Repository: ambari Updated Branches: refs/heads/trunk 1d2e6e667 -> 0e675ff28
AMBARI-12935 Ambari is overwriting ownership of yarn.local.dir with yarn:hadoop in Kerberized cluster (dsen) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/0e675ff2 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/0e675ff2 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/0e675ff2 Branch: refs/heads/trunk Commit: 0e675ff28914f461ec19b0b98539c5e6583cb33e Parents: 1d2e6e6 Author: Dmytro Sen <[email protected]> Authored: Mon Aug 31 15:13:48 2015 +0300 Committer: Dmytro Sen <[email protected]> Committed: Mon Aug 31 15:13:48 2015 +0300 ---------------------------------------------------------------------- .../2.1.0.2.0/package/scripts/params_linux.py | 7 ++ .../YARN/2.1.0.2.0/package/scripts/yarn.py | 72 ++++++++++++++------ .../stacks/2.0.6/YARN/test_nodemanager.py | 22 +++--- .../stacks/2.0.6/YARN/test_resourcemanager.py | 6 ++ 4 files changed, 73 insertions(+), 34 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/0e675ff2/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py index 3d45d21..70ae4b8 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py @@ -118,6 +118,12 @@ hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] smokeuser = config['configurations']['cluster-env']['smokeuser'] smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name'] security_enabled = config['configurations']['cluster-env']['security_enabled'] +nm_security_marker = '/var/lib/hadoop-yarn/nm_security_enabled' +current_nm_security_state = os.path.isfile(nm_security_marker) +toggle_nm_security = (current_nm_security_state and not security_enabled) or (not current_nm_security_state and security_enabled) +rm_security_marker = "/var/lib/hadoop-yarn/rm_security_enabled" +current_rm_security_state = os.path.isfile(rm_security_marker) +toggle_rm_security = (current_rm_security_state and not security_enabled) or (not current_rm_security_state and security_enabled) smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] yarn_executor_container_group = config['configurations']['yarn-site']['yarn.nodemanager.linux-container-executor.group'] yarn_nodemanager_container_executor_class = config['configurations']['yarn-site']['yarn.nodemanager.container-executor.class'] @@ -145,6 +151,7 @@ mapred_pid_dir_prefix = status_params.mapred_pid_dir_prefix mapred_log_dir_prefix = config['configurations']['mapred-env']['mapred_log_dir_prefix'] mapred_env_sh_template = config['configurations']['mapred-env']['content'] yarn_env_sh_template = config['configurations']['yarn-env']['content'] +yarn_nodemanager_recovery_dir = default('/configurations/yarn-site/yarn.nodemanager.recovery.dir', None) if len(rm_hosts) > 1: additional_rm_host = rm_hosts[1] http://git-wip-us.apache.org/repos/asf/ambari/blob/0e675ff2/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py index 30629be..bea6954 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py @@ -98,31 +98,47 @@ def yarn(name = None): ) if name == "nodemanager": - Directory(params.nm_local_dirs.split(',') + params.nm_log_dirs.split(','), - owner=params.yarn_user, - group=params.user_group, - recursive=True, - cd_access="a", - ignore_failures=True, - mode=0775 - ) - smokeuser_directories = [os.path.join(dir, 'usercache' ,params.smokeuser) - for dir in params.nm_local_dirs.split(',')] + # First start after enabling/disabling security + if params.toggle_nm_security: + Directory(params.nm_local_dirs.split(',') + params.nm_log_dirs.split(','), + action='delete' + ) - if not params.security_enabled: - for directory in smokeuser_directories: - Execute(('chown', '-R', params.yarn_user, directory), - only_if=format("test -d {directory}"), - sudo=True) + # If yarn.nodemanager.recovery.dir exists, remove this dir + if params.yarn_nodemanager_recovery_dir: + Directory(InlineTemplate(params.yarn_nodemanager_recovery_dir).get_content(), + action='delete' + ) + # Setting NM marker file + if params.security_enabled: + File(params.nm_security_marker, + content="Marker file to track first start after enabling/disabling security. " + "During first start yarn local, log dirs are removed and recreated" + ) + elif not params.security_enabled: + File(params.nm_security_marker, action="delete") + + + if not params.security_enabled or params.toggle_nm_security: + Directory(params.nm_local_dirs.split(',') + params.nm_log_dirs.split(','), + owner=params.yarn_user, + group=params.user_group, + recursive=True, + cd_access="a", + ignore_failures=True, + mode=0775 + ) + + if params.yarn_nodemanager_recovery_dir: + Directory(InlineTemplate(params.yarn_nodemanager_recovery_dir).get_content(), + owner=params.yarn_user, + group=params.user_group, + recursive=True, + mode=0755 + ) - if params.security_enabled: - for directory in smokeuser_directories: - Execute(('chown', '-R', params.smokeuser, directory), - only_if=format("test -d {directory}"), - sudo=True, - ) Directory([params.yarn_pid_dir_prefix, params.yarn_pid_dir, params.yarn_log_dir], owner=params.yarn_user, group=params.user_group, @@ -206,6 +222,20 @@ def yarn(name = None): mode=0700 ) params.HdfsResource(None, action="execute") + if params.toggle_rm_security: + Execute('yarn resourcemanager -format-state-store', user = params.yarn_user, + ) + # Setting RM marker file + if params.security_enabled: + File(params.rm_security_marker, + content="Marker file to track first start after enabling/disabling security. " + "During first start ResourceManager state store is formatted" + ) + elif not params.security_enabled: + File(params.rm_security_marker, action="delete") + + + elif name == 'apptimelineserver': Directory(params.ats_leveldb_dir, owner=params.yarn_user, http://git-wip-us.apache.org/repos/asf/ambari/blob/0e675ff2/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py index 613ab69..6794097 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py +++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py @@ -167,15 +167,6 @@ class TestNodeManager(RMFTestCase): mode = 0775, cd_access='a' ) - self.assertResourceCalled('Execute', ('chown', '-R', u'yarn', u'/hadoop/yarn/local/usercache/ambari-qa'), - sudo = True, - only_if = 'test -d /hadoop/yarn/local/usercache/ambari-qa', - ) - - self.assertResourceCalled('Execute', ('chown', '-R', u'yarn', u'/hadoop/yarn/local1/usercache/ambari-qa'), - sudo = True, - only_if = 'test -d /hadoop/yarn/local1/usercache/ambari-qa', - ) self.assertResourceCalled('Directory', '/var/run/hadoop-yarn', owner = 'yarn', group = 'hadoop', @@ -334,6 +325,15 @@ class TestNodeManager(RMFTestCase): def assert_configure_secured(self): self.assertResourceCalled('Directory', '/hadoop/yarn/local', + action = ['delete'] + ) + self.assertResourceCalled('Directory', '/hadoop/yarn/log', + action = ['delete'] + ) + self.assertResourceCalled('File', '/var/lib/hadoop-yarn/nm_security_enabled', + content= 'Marker file to track first start after enabling/disabling security. During first start yarn local, log dirs are removed and recreated' + ) + self.assertResourceCalled('Directory', '/hadoop/yarn/local', owner = 'yarn', group = 'hadoop', recursive = True, @@ -350,10 +350,6 @@ class TestNodeManager(RMFTestCase): cd_access='a' ) - self.assertResourceCalled('Execute', ('chown', '-R', u'ambari-qa', u'/hadoop/yarn/local/usercache/ambari-qa'), - sudo = True, - only_if = 'test -d /hadoop/yarn/local/usercache/ambari-qa', - ) self.assertResourceCalled('Directory', '/var/run/hadoop-yarn', owner = 'yarn', group = 'hadoop', http://git-wip-us.apache.org/repos/asf/ambari/blob/0e675ff2/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py index 428655b..fb7d847 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py +++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py @@ -434,6 +434,12 @@ class TestResourceManager(RMFTestCase): owner = 'yarn', group = 'hadoop', ) + self.assertResourceCalled('Execute', 'yarn resourcemanager -format-state-store', + user = 'yarn', + ) + self.assertResourceCalled('File', '/var/lib/hadoop-yarn/rm_security_enabled', + content = 'Marker file to track first start after enabling/disabling security. During first start ResourceManager state store is formatted', + ) self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude', owner = 'yarn', group = 'hadoop',
