AMBARI-21729 Owner of yarn.include stays to be root:root where as dfs.include gets changed to the correct one upon restart services (dsen)
(cherry picked from commit 640fefb21cc73b7ccbd4dc714c0415dcbb311700) Change-Id: I102293ceaf8e9815d3a5efb7222449835f64ee52 Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6d2bab16 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6d2bab16 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6d2bab16 Branch: refs/heads/2.5-maint Commit: 6d2bab1615598753b642e35ae2f1148641eac16b Parents: fd3379f Author: Dmytro Sen <[email protected]> Authored: Wed Aug 16 12:35:45 2017 +0300 Committer: Zuul <[email protected]> Committed: Wed Aug 16 11:31:41 2017 -0700 ---------------------------------------------------------------------- .../2.1.0.2.0/package/scripts/params_linux.py | 5 +- .../YARN/2.1.0.2.0/package/scripts/yarn.py | 14 +- .../stacks/2.0.6/YARN/test_resourcemanager.py | 160 ++- .../default_yarn_include_file_dont_manage.json | 1260 ++++++++++++++++++ .../default_yarn_include_file_manage.json | 1260 ++++++++++++++++++ .../secured_yarn_include_file_dont_manage.json | 1078 +++++++++++++++ .../secured_yarn_include_file_manage.json | 1078 +++++++++++++++ .../YARN/package/scripts/params_linux.py | 5 +- .../2.0/services/YARN/package/scripts/yarn.py | 16 +- 9 files changed, 4865 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/6d2bab16/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py index fad1fcb..f217192 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py @@ -169,9 +169,6 @@ rm_hosts = config['clusterHostInfo']['rm_host'] rm_host = rm_hosts[0] rm_port = config['configurations']['yarn-site']['yarn.resourcemanager.webapp.address'].split(':')[-1] rm_https_port = default('/configurations/yarn-site/yarn.resourcemanager.webapp.https.address', ":8090").split(':')[-1] -# TODO UPGRADE default, update site during upgrade -rm_nodes_exclude_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.exclude-path","/etc/hadoop/conf/yarn.exclude") -rm_nodes_exclude_dir = os.path.dirname(rm_nodes_exclude_path) java64_home = config['hostLevelParams']['java_home'] java_exec = format("{java64_home}/bin/java") @@ -234,6 +231,7 @@ user_group = config['configurations']['cluster-env']['user_group'] #exclude file exclude_hosts = default("/clusterHostInfo/decom_nm_hosts", []) exclude_file_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.exclude-path","/etc/hadoop/conf/yarn.exclude") +rm_nodes_exclude_dir = os.path.dirname(exclude_file_path) nm_hosts = default("/clusterHostInfo/nm_hosts", []) #incude file @@ -241,6 +239,7 @@ include_file_path = default("/configurations/yarn-site/yarn.resourcemanager.node include_hosts = None manage_include_files = default("/configurations/yarn-site/manage.include.files", False) if include_file_path and manage_include_files: + rm_nodes_include_dir = os.path.dirname(include_file_path) include_hosts = list(set(nm_hosts) - set(exclude_hosts)) ats_host = set(default("/clusterHostInfo/app_timeline_server_hosts", [])) http://git-wip-us.apache.org/repos/asf/ambari/blob/6d2bab16/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py index 3300875..f6e8391 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py @@ -296,10 +296,22 @@ def yarn(name=None, config_dir=None): create_parents=True, cd_access='a', ) - File(params.rm_nodes_exclude_path, + File(params.exclude_file_path, + content=Template("exclude_hosts_list.j2"), owner=params.yarn_user, group=params.user_group ) + if params.include_hosts: + Directory(params.rm_nodes_include_dir, + mode=0755, + create_parents=True, + cd_access='a', + ) + File(params.include_file_path, + content=Template("include_hosts_list.j2"), + owner=params.yarn_user, + group=params.user_group + ) File(params.yarn_job_summary_log, owner=params.yarn_user, group=params.user_group http://git-wip-us.apache.org/repos/asf/ambari/blob/6d2bab16/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py index e8b5f78..01b5282 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py +++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py @@ -48,6 +48,28 @@ class TestResourceManager(RMFTestCase): self.assert_configure_default() self.assertNoMoreResources() + def test_configure_default_with_include_file_dont_manage(self): + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py", + classname="Resourcemanager", + command="configure", + config_file="default_yarn_include_file_dont_manage.json", + stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + self.assert_configure_default(is_include_file_configured=True, manage_include_files=False) + self.assertNoMoreResources() + + def test_configure_default_with_include_file_manage(self): + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py", + classname="Resourcemanager", + command="configure", + config_file="default_yarn_include_file_manage.json", + stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + self.assert_configure_default(is_include_file_configured=True, manage_include_files=True) + self.assertNoMoreResources() + def test_start_default(self): self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py", classname="Resourcemanager", @@ -98,6 +120,28 @@ class TestResourceManager(RMFTestCase): ) self.assert_configure_secured() + def test_configure_secured_with_include_file_dont_manage(self): + + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py", + classname="Resourcemanager", + command="configure", + config_file="secured_yarn_include_file_dont_manage.json", + stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + self.assert_configure_secured(is_include_file_configured=True, manage_include_files=False) + + def test_configure_secured_with_include_file_manage(self): + + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py", + classname="Resourcemanager", + command="configure", + config_file="secured_yarn_include_file_manage.json", + stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + self.assert_configure_secured(is_include_file_configured=True, manage_include_files=True) + def test_start_secured(self): self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py", classname="Resourcemanager", @@ -177,7 +221,95 @@ class TestResourceManager(RMFTestCase): self.assertNoMoreResources() - def assert_configure_default(self): + def test_decommission_default_with_include_file_dont_manage(self): + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py", + classname = "Resourcemanager", + command = "decommission", + config_file="default_yarn_include_file_dont_manage.json", + stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude', + owner = 'yarn', + content = Template('exclude_hosts_list.j2'), + group = 'hadoop', + ) + self.assertResourceCalled('Execute', ' yarn --config /etc/hadoop/conf rmadmin -refreshNodes', + environment = {'PATH': "/bin:/usr/bin:/usr/lib/hadoop-yarn/bin"}, + user = 'yarn', + ) + self.assertNoMoreResources() + + def test_decommission_default_with_include_file_manage(self): + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py", + classname = "Resourcemanager", + command = "decommission", + config_file="default_yarn_include_file_manage.json", + stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude', + owner = 'yarn', + content = Template('exclude_hosts_list.j2'), + group = 'hadoop', + ) + self.assertResourceCalled('File', '/etc/hadoop/conf_for_include/yarn.include', + owner = 'yarn', + content = Template('include_hosts_list.j2'), + group = 'hadoop', + ) + self.assertResourceCalled('Execute', ' yarn --config /etc/hadoop/conf rmadmin -refreshNodes', + environment = {'PATH': "/bin:/usr/bin:/usr/lib/hadoop-yarn/bin"}, + user = 'yarn', + ) + self.assertNoMoreResources() + + def test_decommission_secured_with_include_file_dont_manage(self): + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py", + classname = "Resourcemanager", + command = "decommission", + config_file="secured_yarn_include_file_dont_manage.json", + stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude', + owner = 'yarn', + content = Template('exclude_hosts_list.j2'), + group = 'hadoop', + ) + self.assertResourceCalled('Execute', '/usr/bin/kinit -kt /etc/security/keytabs/rm.service.keytab rm/[email protected]; yarn --config /etc/hadoop/conf rmadmin -refreshNodes', + environment = {'PATH': "/bin:/usr/bin:/usr/lib/hadoop-yarn/bin"}, + user = 'yarn', + ) + + self.assertNoMoreResources() + + def test_decommission_secured_with_include_file_manage(self): + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py", + classname = "Resourcemanager", + command = "decommission", + config_file="secured_yarn_include_file_manage.json", + stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude', + owner = 'yarn', + content = Template('exclude_hosts_list.j2'), + group = 'hadoop', + ) + self.assertResourceCalled('File', '/etc/hadoop/conf_for_include/yarn.include', + owner = 'yarn', + content = Template('include_hosts_list.j2'), + group = 'hadoop', + ) + self.assertResourceCalled('Execute', '/usr/bin/kinit -kt /etc/security/keytabs/rm.service.keytab rm/[email protected]; yarn --config /etc/hadoop/conf rmadmin -refreshNodes', + environment = {'PATH': "/bin:/usr/bin:/usr/lib/hadoop-yarn/bin"}, + user = 'yarn', + ) + + self.assertNoMoreResources() + + def assert_configure_default(self, is_include_file_configured = False, manage_include_files=False): self.assertResourceCalled('Directory', '/var/run/hadoop-yarn', owner = 'yarn', @@ -274,9 +406,21 @@ class TestResourceManager(RMFTestCase): cd_access = 'a', ) self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude', + content = Template('exclude_hosts_list.j2'), owner = 'yarn', group = 'hadoop', ) + if is_include_file_configured and manage_include_files: + self.assertResourceCalled('Directory', '/etc/hadoop/conf_for_include', + mode = 0755, + create_parents = True, + cd_access = 'a', + ) + self.assertResourceCalled('File', '/etc/hadoop/conf_for_include/yarn.include', + content = Template('include_hosts_list.j2'), + owner = 'yarn', + group = 'hadoop', + ) self.assertResourceCalled('File', '/var/log/hadoop-yarn/yarn/hadoop-mapreduce.jobsummary.log', owner = 'yarn', group = 'hadoop', @@ -346,7 +490,7 @@ class TestResourceManager(RMFTestCase): group = 'hadoop', ) - def assert_configure_secured(self): + def assert_configure_secured(self, is_include_file_configured = False, manage_include_files=False): self.assertResourceCalled('Directory', '/var/run/hadoop-yarn', owner = 'yarn', @@ -443,9 +587,21 @@ class TestResourceManager(RMFTestCase): cd_access = 'a', ) self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude', + content = Template('exclude_hosts_list.j2'), owner = 'yarn', group = 'hadoop', ) + if is_include_file_configured and manage_include_files: + self.assertResourceCalled('Directory', '/etc/hadoop/conf_for_include', + mode = 0755, + create_parents = True, + cd_access = 'a', + ) + self.assertResourceCalled('File', '/etc/hadoop/conf_for_include/yarn.include', + content = Template('include_hosts_list.j2'), + owner = 'yarn', + group = 'hadoop', + ) self.assertResourceCalled('File', '/var/log/hadoop-yarn/yarn/hadoop-mapreduce.jobsummary.log', owner = 'yarn', group = 'hadoop',
