AMBARI-21729 Owner of yarn.include stays to be root:root where as dfs.include gets changed to the correct one upon restart services (dsen)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/5c78bd75 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/5c78bd75 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/5c78bd75 Branch: refs/heads/branch-feature-logsearch-ui Commit: 5c78bd752aa5a064966ecee9fcee374cf12eb7a8 Parents: ab8d58b Author: Dmytro Sen <d...@apache.org> Authored: Wed Aug 16 12:34:02 2017 +0300 Committer: Dmytro Sen <d...@apache.org> Committed: Wed Aug 16 12:34:02 2017 +0300 ---------------------------------------------------------------------- .../2.1.0.2.0/package/scripts/params_linux.py | 5 +- .../YARN/2.1.0.2.0/package/scripts/yarn.py | 14 +- .../3.0.0.3.0/package/scripts/params_linux.py | 5 +- .../YARN/3.0.0.3.0/package/scripts/yarn.py | 14 +- .../stacks/2.0.6/YARN/test_resourcemanager.py | 160 ++- .../default_yarn_include_file_dont_manage.json | 1260 ++++++++++++++++++ .../default_yarn_include_file_manage.json | 1260 ++++++++++++++++++ .../secured_yarn_include_file_dont_manage.json | 1078 +++++++++++++++ .../secured_yarn_include_file_manage.json | 1078 +++++++++++++++ .../YARN/package/scripts/params_linux.py | 5 +- .../2.0/services/YARN/package/scripts/yarn.py | 16 +- 11 files changed, 4880 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/5c78bd75/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py index 0b1215b..2ab779f 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py @@ -169,9 +169,6 @@ rm_hosts = config['clusterHostInfo']['rm_host'] rm_host = rm_hosts[0] rm_port = config['configurations']['yarn-site']['yarn.resourcemanager.webapp.address'].split(':')[-1] rm_https_port = default('/configurations/yarn-site/yarn.resourcemanager.webapp.https.address', ":8090").split(':')[-1] -# TODO UPGRADE default, update site during upgrade -rm_nodes_exclude_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.exclude-path","/etc/hadoop/conf/yarn.exclude") -rm_nodes_exclude_dir = os.path.dirname(rm_nodes_exclude_path) java64_home = config['hostLevelParams']['java_home'] java_exec = format("{java64_home}/bin/java") @@ -241,6 +238,7 @@ user_group = config['configurations']['cluster-env']['user_group'] #exclude file exclude_hosts = default("/clusterHostInfo/decom_nm_hosts", []) exclude_file_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.exclude-path","/etc/hadoop/conf/yarn.exclude") +rm_nodes_exclude_dir = os.path.dirname(exclude_file_path) nm_hosts = default("/clusterHostInfo/nm_hosts", []) #incude file @@ -248,6 +246,7 @@ include_file_path = default("/configurations/yarn-site/yarn.resourcemanager.node include_hosts = None manage_include_files = default("/configurations/yarn-site/manage.include.files", False) if include_file_path and manage_include_files: + rm_nodes_include_dir = os.path.dirname(include_file_path) include_hosts = list(set(nm_hosts) - set(exclude_hosts)) ats_host = set(default("/clusterHostInfo/app_timeline_server_hosts", [])) http://git-wip-us.apache.org/repos/asf/ambari/blob/5c78bd75/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py index 1177107..363d464 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py @@ -390,10 +390,22 @@ def setup_resourcemanager(): create_parents=True, cd_access='a', ) - File(params.rm_nodes_exclude_path, + File(params.exclude_file_path, + content=Template("exclude_hosts_list.j2"), owner=params.yarn_user, group=params.user_group ) + if params.include_hosts: + Directory(params.rm_nodes_include_dir, + mode=0755, + create_parents=True, + cd_access='a', + ) + File(params.include_file_path, + content=Template("include_hosts_list.j2"), + owner=params.yarn_user, + group=params.user_group + ) File(params.yarn_job_summary_log, owner=params.yarn_user, group=params.user_group http://git-wip-us.apache.org/repos/asf/ambari/blob/5c78bd75/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py index 68d17f0..e66ec3c 100644 --- a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py @@ -166,9 +166,6 @@ rm_hosts = config['clusterHostInfo']['rm_host'] rm_host = rm_hosts[0] rm_port = config['configurations']['yarn-site']['yarn.resourcemanager.webapp.address'].split(':')[-1] rm_https_port = default('/configurations/yarn-site/yarn.resourcemanager.webapp.https.address', ":8090").split(':')[-1] -# TODO UPGRADE default, update site during upgrade -rm_nodes_exclude_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.exclude-path","/etc/hadoop/conf/yarn.exclude") -rm_nodes_exclude_dir = os.path.dirname(rm_nodes_exclude_path) java64_home = config['hostLevelParams']['java_home'] java_exec = format("{java64_home}/bin/java") @@ -238,6 +235,7 @@ user_group = config['configurations']['cluster-env']['user_group'] #exclude file exclude_hosts = default("/clusterHostInfo/decom_nm_hosts", []) exclude_file_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.exclude-path","/etc/hadoop/conf/yarn.exclude") +rm_nodes_exclude_dir = os.path.dirname(exclude_file_path) nm_hosts = default("/clusterHostInfo/nm_hosts", []) #incude file @@ -245,6 +243,7 @@ include_file_path = default("/configurations/yarn-site/yarn.resourcemanager.node include_hosts = None manage_include_files = default("/configurations/yarn-site/manage.include.files", False) if include_file_path and manage_include_files: + rm_nodes_include_dir = os.path.dirname(include_file_path) include_hosts = list(set(nm_hosts) - set(exclude_hosts)) ats_host = set(default("/clusterHostInfo/app_timeline_server_hosts", [])) http://git-wip-us.apache.org/repos/asf/ambari/blob/5c78bd75/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/yarn.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/yarn.py b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/yarn.py index 0591511..ef314f3 100644 --- a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/yarn.py +++ b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/yarn.py @@ -391,10 +391,22 @@ def setup_resourcemanager(): create_parents=True, cd_access='a', ) - File(params.rm_nodes_exclude_path, + File(params.exclude_file_path, + content=Template("exclude_hosts_list.j2"), owner=params.yarn_user, group=params.user_group ) + if params.include_hosts: + Directory(params.rm_nodes_include_dir, + mode=0755, + create_parents=True, + cd_access='a', + ) + File(params.include_file_path, + content=Template("include_hosts_list.j2"), + owner=params.yarn_user, + group=params.user_group + ) # This depends on the parent directory already existing. File(params.yarn_job_summary_log, owner=params.yarn_user, http://git-wip-us.apache.org/repos/asf/ambari/blob/5c78bd75/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py index 8e92116..8a703df 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py +++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py @@ -50,6 +50,28 @@ class TestResourceManager(RMFTestCase): self.assert_configure_default() self.assertNoMoreResources() + def test_configure_default_with_include_file_dont_manage(self): + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py", + classname="Resourcemanager", + command="configure", + config_file="default_yarn_include_file_dont_manage.json", + stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + self.assert_configure_default(is_include_file_configured=True, manage_include_files=False) + self.assertNoMoreResources() + + def test_configure_default_with_include_file_manage(self): + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py", + classname="Resourcemanager", + command="configure", + config_file="default_yarn_include_file_manage.json", + stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + self.assert_configure_default(is_include_file_configured=True, manage_include_files=True) + self.assertNoMoreResources() + def test_start_default(self): self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py", classname="Resourcemanager", @@ -100,6 +122,28 @@ class TestResourceManager(RMFTestCase): ) self.assert_configure_secured() + def test_configure_secured_with_include_file_dont_manage(self): + + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py", + classname="Resourcemanager", + command="configure", + config_file="secured_yarn_include_file_dont_manage.json", + stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + self.assert_configure_secured(is_include_file_configured=True, manage_include_files=False) + + def test_configure_secured_with_include_file_manage(self): + + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py", + classname="Resourcemanager", + command="configure", + config_file="secured_yarn_include_file_manage.json", + stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + self.assert_configure_secured(is_include_file_configured=True, manage_include_files=True) + def test_start_secured(self): self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py", classname="Resourcemanager", @@ -179,7 +223,95 @@ class TestResourceManager(RMFTestCase): self.assertNoMoreResources() - def assert_configure_default(self): + def test_decommission_default_with_include_file_dont_manage(self): + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py", + classname = "Resourcemanager", + command = "decommission", + config_file="default_yarn_include_file_dont_manage.json", + stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude', + owner = 'yarn', + content = Template('exclude_hosts_list.j2'), + group = 'hadoop', + ) + self.assertResourceCalled('Execute', ' yarn --config /etc/hadoop/conf rmadmin -refreshNodes', + environment = {'PATH': "/bin:/usr/bin:/usr/lib/hadoop-yarn/bin"}, + user = 'yarn', + ) + self.assertNoMoreResources() + + def test_decommission_default_with_include_file_manage(self): + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py", + classname = "Resourcemanager", + command = "decommission", + config_file="default_yarn_include_file_manage.json", + stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude', + owner = 'yarn', + content = Template('exclude_hosts_list.j2'), + group = 'hadoop', + ) + self.assertResourceCalled('File', '/etc/hadoop/conf_for_include/yarn.include', + owner = 'yarn', + content = Template('include_hosts_list.j2'), + group = 'hadoop', + ) + self.assertResourceCalled('Execute', ' yarn --config /etc/hadoop/conf rmadmin -refreshNodes', + environment = {'PATH': "/bin:/usr/bin:/usr/lib/hadoop-yarn/bin"}, + user = 'yarn', + ) + self.assertNoMoreResources() + + def test_decommission_secured_with_include_file_dont_manage(self): + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py", + classname = "Resourcemanager", + command = "decommission", + config_file="secured_yarn_include_file_dont_manage.json", + stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude', + owner = 'yarn', + content = Template('exclude_hosts_list.j2'), + group = 'hadoop', + ) + self.assertResourceCalled('Execute', '/usr/bin/kinit -kt /etc/security/keytabs/rm.service.keytab rm/c6401.ambari.apache....@example.com; yarn --config /etc/hadoop/conf rmadmin -refreshNodes', + environment = {'PATH': "/bin:/usr/bin:/usr/lib/hadoop-yarn/bin"}, + user = 'yarn', + ) + + self.assertNoMoreResources() + + def test_decommission_secured_with_include_file_manage(self): + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py", + classname = "Resourcemanager", + command = "decommission", + config_file="secured_yarn_include_file_manage.json", + stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude', + owner = 'yarn', + content = Template('exclude_hosts_list.j2'), + group = 'hadoop', + ) + self.assertResourceCalled('File', '/etc/hadoop/conf_for_include/yarn.include', + owner = 'yarn', + content = Template('include_hosts_list.j2'), + group = 'hadoop', + ) + self.assertResourceCalled('Execute', '/usr/bin/kinit -kt /etc/security/keytabs/rm.service.keytab rm/c6401.ambari.apache....@example.com; yarn --config /etc/hadoop/conf rmadmin -refreshNodes', + environment = {'PATH': "/bin:/usr/bin:/usr/lib/hadoop-yarn/bin"}, + user = 'yarn', + ) + + self.assertNoMoreResources() + + def assert_configure_default(self, is_include_file_configured = False, manage_include_files=False): self.assertResourceCalled('Directory', '/var/run/hadoop-yarn', owner = 'yarn', group = 'hadoop', @@ -235,9 +367,21 @@ class TestResourceManager(RMFTestCase): cd_access = 'a', ) self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude', + content = Template('exclude_hosts_list.j2'), owner = 'yarn', group = 'hadoop', ) + if is_include_file_configured and manage_include_files: + self.assertResourceCalled('Directory', '/etc/hadoop/conf_for_include', + mode = 0755, + create_parents = True, + cd_access = 'a', + ) + self.assertResourceCalled('File', '/etc/hadoop/conf_for_include/yarn.include', + content = Template('include_hosts_list.j2'), + owner = 'yarn', + group = 'hadoop', + ) self.assertResourceCalled('File', '/var/log/hadoop-yarn/yarn/hadoop-mapreduce.jobsummary.log', owner = 'yarn', group = 'hadoop', @@ -347,7 +491,7 @@ class TestResourceManager(RMFTestCase): group = 'hadoop', ) - def assert_configure_secured(self): + def assert_configure_secured(self, is_include_file_configured = False, manage_include_files=False): self.assertResourceCalled('Directory', '/var/run/hadoop-yarn', owner = 'yarn', group = 'hadoop', @@ -403,9 +547,21 @@ class TestResourceManager(RMFTestCase): cd_access = 'a', ) self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude', + content = Template('exclude_hosts_list.j2'), owner = 'yarn', group = 'hadoop', ) + if is_include_file_configured and manage_include_files: + self.assertResourceCalled('Directory', '/etc/hadoop/conf_for_include', + mode = 0755, + create_parents = True, + cd_access = 'a', + ) + self.assertResourceCalled('File', '/etc/hadoop/conf_for_include/yarn.include', + content = Template('include_hosts_list.j2'), + owner = 'yarn', + group = 'hadoop', + ) self.assertResourceCalled('File', '/var/log/hadoop-yarn/yarn/hadoop-mapreduce.jobsummary.log', owner = 'yarn', group = 'hadoop',
