AMBARI-18471. Refactor yarn() function in YARN service. Part 1. (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6fb1ceef Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6fb1ceef Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6fb1ceef Branch: refs/heads/branch-feature-AMBARI-18456 Commit: 6fb1ceef66b8d63c66603eebf1cd9e6d91c815aa Parents: c265ae6 Author: Andrew Onishuk <[email protected]> Authored: Wed Sep 28 08:44:44 2016 +0300 Committer: Andrew Onishuk <[email protected]> Committed: Wed Sep 28 08:44:44 2016 +0300 ---------------------------------------------------------------------- .../2.1.0.2.0/package/scripts/params_linux.py | 7 + .../YARN/2.1.0.2.0/package/scripts/yarn.py | 548 +++++++++---------- .../stacks/2.0.6/YARN/test_historyserver.py | 1 - .../stacks/2.0.6/YARN/test_resourcemanager.py | 54 +- .../stacks/2.1/YARN/test_apptimelineserver.py | 40 +- .../test/python/stacks/2.3/YARN/test_ats_1_5.py | 188 +++---- 6 files changed, 421 insertions(+), 417 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/6fb1ceef/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py index 4d42861..0d46069 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py @@ -193,6 +193,13 @@ else: rm_webui_address = format("{rm_host}:{rm_port}") rm_webui_https_address = format("{rm_host}:{rm_https_port}") +if security_enabled: + tc_mode = 0644 + tc_owner = "root" +else: + tc_mode = None + tc_owner = hdfs_user + nm_webui_address = config['configurations']['yarn-site']['yarn.nodemanager.webapp.address'] hs_webui_address = config['configurations']['mapred-site']['mapreduce.jobhistory.webapp.address'] nm_address = config['configurations']['yarn-site']['yarn.nodemanager.address'] # still contains 0.0.0.0 http://git-wip-us.apache.org/repos/asf/ambari/blob/6fb1ceef/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py index a3a3a06..70ed5b3 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py @@ -37,63 +37,6 @@ from ambari_commons import OSConst from resource_management.libraries.functions.mounted_dirs_helper import handle_mounted_dirs -# Local Imports - - -@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY) -def yarn(name = None): - import params - XmlConfig("mapred-site.xml", - conf_dir=params.config_dir, - configurations=params.config['configurations']['mapred-site'], - owner=params.yarn_user, - mode='f' - ) - XmlConfig("yarn-site.xml", - conf_dir=params.config_dir, - configurations=params.config['configurations']['yarn-site'], - owner=params.yarn_user, - mode='f', - configuration_attributes=params.config['configuration_attributes']['yarn-site'] - ) - XmlConfig("capacity-scheduler.xml", - conf_dir=params.config_dir, - configurations=params.config['configurations']['capacity-scheduler'], - owner=params.yarn_user, - mode='f' - ) - - if params.service_map.has_key(name): - service_name = params.service_map[name] - - ServiceConfig(service_name, - action="change_user", - username = params.yarn_user, - password = Script.get_password(params.yarn_user)) - -def create_log_dir(dir_name): - import params - Directory(dir_name, - create_parents = True, - cd_access="a", - mode=0775, - owner=params.yarn_user, - group=params.user_group, - ignore_failures=True, - ) - -def create_local_dir(dir_name): - import params - Directory(dir_name, - create_parents = True, - cd_access="a", - mode=0755, - owner=params.yarn_user, - group=params.user_group, - ignore_failures=True, - recursive_mode_flags = {'f': 'a+rw', 'd': 'a+rwx'}, - ) - @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) def yarn(name=None, config_dir=None): """ @@ -102,107 +45,18 @@ def yarn(name=None, config_dir=None): """ import params + if name == 'resourcemanager': + setup_resourcemanager() + elif name == 'nodemanager': + setup_nodemanager() + elif name == 'apptimelineserver': + setup_ats() + elif name == 'historyserver': + setup_historyserver() + if config_dir is None: config_dir = params.hadoop_conf_dir - if name == "historyserver": - if params.yarn_log_aggregation_enabled: - params.HdfsResource(params.yarn_nm_app_log_dir, - action="create_on_execute", - type="directory", - owner=params.yarn_user, - group=params.user_group, - mode=01777, - recursive_chmod=True - ) - - # create the /tmp folder with proper permissions if it doesn't exist yet - if params.entity_file_history_directory.startswith('/tmp'): - params.HdfsResource(params.hdfs_tmp_dir, - action="create_on_execute", - type="directory", - owner=params.hdfs_user, - mode=0777, - ) - - params.HdfsResource(params.entity_file_history_directory, - action="create_on_execute", - type="directory", - owner=params.yarn_user, - group=params.user_group - ) - params.HdfsResource("/mapred", - type="directory", - action="create_on_execute", - owner=params.mapred_user - ) - params.HdfsResource("/mapred/system", - type="directory", - action="create_on_execute", - owner=params.hdfs_user - ) - params.HdfsResource(params.mapreduce_jobhistory_done_dir, - type="directory", - action="create_on_execute", - owner=params.mapred_user, - group=params.user_group, - change_permissions_for_parents=True, - mode=0777 - ) - params.HdfsResource(None, action="execute") - Directory(params.jhs_leveldb_state_store_dir, - owner=params.mapred_user, - group=params.user_group, - create_parents = True, - cd_access="a", - recursive_ownership = True, - ) - - #<editor-fold desc="Node Manager Section"> - if name == "nodemanager": - - # First start after enabling/disabling security - if params.toggle_nm_security: - Directory(params.nm_local_dirs_list + params.nm_log_dirs_list, - action='delete' - ) - - # If yarn.nodemanager.recovery.dir exists, remove this dir - if params.yarn_nodemanager_recovery_dir: - Directory(InlineTemplate(params.yarn_nodemanager_recovery_dir).get_content(), - action='delete' - ) - - # Setting NM marker file - if params.security_enabled: - Directory(params.nm_security_marker_dir) - File(params.nm_security_marker, - content="Marker file to track first start after enabling/disabling security. " - "During first start yarn local, log dirs are removed and recreated" - ) - elif not params.security_enabled: - File(params.nm_security_marker, action="delete") - - - if not params.security_enabled or params.toggle_nm_security: - # handle_mounted_dirs ensures that we don't create dirs which are temporary unavailable (unmounted), and intended to reside on a different mount. - nm_log_dir_to_mount_file_content = handle_mounted_dirs(create_log_dir, params.nm_log_dirs, params.nm_log_dir_to_mount_file, params) - # create a history file used by handle_mounted_dirs - File(params.nm_log_dir_to_mount_file, - owner=params.hdfs_user, - group=params.user_group, - mode=0644, - content=nm_log_dir_to_mount_file_content - ) - nm_local_dir_to_mount_file_content = handle_mounted_dirs(create_local_dir, params.nm_local_dirs, params.nm_local_dir_to_mount_file, params) - File(params.nm_local_dir_to_mount_file, - owner=params.hdfs_user, - group=params.user_group, - mode=0644, - content=nm_local_dir_to_mount_file_content - ) - #</editor-fold> - if params.yarn_nodemanager_recovery_dir: Directory(InlineTemplate(params.yarn_nodemanager_recovery_dir).get_content(), owner=params.yarn_user, @@ -245,15 +99,14 @@ def yarn(name=None, config_dir=None): # During RU, Core Masters and Slaves need hdfs-site.xml # TODO, instead of specifying individual configs, which is susceptible to breaking when new configs are added, # RU should rely on all available in <stack-root>/<version>/hadoop/conf - if 'hdfs-site' in params.config['configurations']: - XmlConfig("hdfs-site.xml", - conf_dir=config_dir, - configurations=params.config['configurations']['hdfs-site'], - configuration_attributes=params.config['configuration_attributes']['hdfs-site'], - owner=params.hdfs_user, - group=params.user_group, - mode=0644 - ) + XmlConfig("hdfs-site.xml", + conf_dir=config_dir, + configurations=params.config['configurations']['hdfs-site'], + configuration_attributes=params.config['configuration_attributes']['hdfs-site'], + owner=params.hdfs_user, + group=params.user_group, + mode=0644 + ) XmlConfig("mapred-site.xml", conf_dir=config_dir, @@ -282,85 +135,6 @@ def yarn(name=None, config_dir=None): mode=0644 ) - if name == 'resourcemanager': - Directory(params.rm_nodes_exclude_dir, - mode=0755, - create_parents=True, - cd_access='a', - ) - File(params.rm_nodes_exclude_path, - owner=params.yarn_user, - group=params.user_group - ) - File(params.yarn_job_summary_log, - owner=params.yarn_user, - group=params.user_group - ) - if not is_empty(params.node_label_enable) and params.node_label_enable or is_empty(params.node_label_enable) and params.node_labels_dir: - params.HdfsResource(params.node_labels_dir, - type="directory", - action="create_on_execute", - change_permissions_for_parents=True, - owner=params.yarn_user, - group=params.user_group, - mode=0700 - ) - params.HdfsResource(None, action="execute") - - - elif name == 'apptimelineserver': - Directory(params.ats_leveldb_dir, - owner=params.yarn_user, - group=params.user_group, - create_parents = True, - cd_access="a", - ) - - # if stack support application timeline-service state store property (timeline_state_store stack feature) - if params.stack_supports_timeline_state_store: - Directory(params.ats_leveldb_state_store_dir, - owner=params.yarn_user, - group=params.user_group, - create_parents = True, - cd_access="a", - ) - # app timeline server 1.5 directories - if not is_empty(params.entity_groupfs_store_dir): - parent_path = os.path.dirname(params.entity_groupfs_store_dir) - params.HdfsResource(parent_path, - type="directory", - action="create_on_execute", - change_permissions_for_parents=True, - owner=params.yarn_user, - group=params.user_group, - mode=0755 - ) - params.HdfsResource(params.entity_groupfs_store_dir, - type="directory", - action="create_on_execute", - owner=params.yarn_user, - group=params.user_group, - mode=params.entity_groupfs_store_dir_mode - ) - if not is_empty(params.entity_groupfs_active_dir): - parent_path = os.path.dirname(params.entity_groupfs_active_dir) - params.HdfsResource(parent_path, - type="directory", - action="create_on_execute", - change_permissions_for_parents=True, - owner=params.yarn_user, - group=params.user_group, - mode=0755 - ) - params.HdfsResource(params.entity_groupfs_active_dir, - type="directory", - action="create_on_execute", - owner=params.yarn_user, - group=params.user_group, - mode=params.entity_groupfs_active_dir_mode - ) - params.HdfsResource(None, action="execute") - File(format("{limits_conf_dir}/yarn.conf"), mode=0644, content=Template('yarn.conf.j2') @@ -378,8 +152,7 @@ def yarn(name=None, config_dir=None): content=InlineTemplate(params.yarn_env_sh_template) ) - container_executor = format("{yarn_container_bin}/container-executor") - File(container_executor, + File(format("{yarn_container_bin}/container-executor"), group=params.yarn_executor_container_group, mode=params.container_executor_mode ) @@ -396,15 +169,8 @@ def yarn(name=None, config_dir=None): mode=0755, cd_access="a") - if params.security_enabled: - tc_mode = 0644 - tc_owner = "root" - else: - tc_mode = None - tc_owner = params.hdfs_user - File(os.path.join(config_dir, "mapred-env.sh"), - owner=tc_owner, + owner=params.tc_owner, mode=0755, content=InlineTemplate(params.mapred_env_sh_template) ) @@ -416,35 +182,34 @@ def yarn(name=None, config_dir=None): mode=06050 ) File(os.path.join(config_dir, 'taskcontroller.cfg'), - owner = tc_owner, - mode = tc_mode, + owner = params.tc_owner, + mode = params.tc_mode, group = params.mapred_tt_group, content=Template("taskcontroller.cfg.j2") ) else: File(os.path.join(config_dir, 'taskcontroller.cfg'), - owner=tc_owner, + owner=params.tc_owner, content=Template("taskcontroller.cfg.j2") ) - if "mapred-site" in params.config['configurations']: - XmlConfig("mapred-site.xml", - conf_dir=config_dir, - configurations=params.config['configurations']['mapred-site'], - configuration_attributes=params.config['configuration_attributes']['mapred-site'], - owner=params.mapred_user, - group=params.user_group - ) + XmlConfig("mapred-site.xml", + conf_dir=config_dir, + configurations=params.config['configurations']['mapred-site'], + configuration_attributes=params.config['configuration_attributes']['mapred-site'], + owner=params.mapred_user, + group=params.user_group + ) + + XmlConfig("capacity-scheduler.xml", + conf_dir=config_dir, + configurations=params.config['configurations'][ + 'capacity-scheduler'], + configuration_attributes=params.config['configuration_attributes']['capacity-scheduler'], + owner=params.hdfs_user, + group=params.user_group + ) - if "capacity-scheduler" in params.config['configurations']: - XmlConfig("capacity-scheduler.xml", - conf_dir=config_dir, - configurations=params.config['configurations'][ - 'capacity-scheduler'], - configuration_attributes=params.config['configuration_attributes']['capacity-scheduler'], - owner=params.hdfs_user, - group=params.user_group - ) if "ssl-client" in params.config['configurations']: XmlConfig("ssl-client.xml", conf_dir=config_dir, @@ -495,4 +260,239 @@ def yarn(name=None, config_dir=None): File(os.path.join(config_dir, 'ssl-server.xml.example'), owner=params.mapred_user, group=params.user_group - ) \ No newline at end of file + ) + +def setup_historyserver(): + import params + + if params.yarn_log_aggregation_enabled: + params.HdfsResource(params.yarn_nm_app_log_dir, + action="create_on_execute", + type="directory", + owner=params.yarn_user, + group=params.user_group, + mode=01777, + recursive_chmod=True + ) + + # create the /tmp folder with proper permissions if it doesn't exist yet + if params.entity_file_history_directory.startswith('/tmp'): + params.HdfsResource(params.hdfs_tmp_dir, + action="create_on_execute", + type="directory", + owner=params.hdfs_user, + mode=0777, + ) + + params.HdfsResource(params.entity_file_history_directory, + action="create_on_execute", + type="directory", + owner=params.yarn_user, + group=params.user_group + ) + params.HdfsResource("/mapred", + type="directory", + action="create_on_execute", + owner=params.mapred_user + ) + params.HdfsResource("/mapred/system", + type="directory", + action="create_on_execute", + owner=params.hdfs_user + ) + params.HdfsResource(params.mapreduce_jobhistory_done_dir, + type="directory", + action="create_on_execute", + owner=params.mapred_user, + group=params.user_group, + change_permissions_for_parents=True, + mode=0777 + ) + params.HdfsResource(None, action="execute") + Directory(params.jhs_leveldb_state_store_dir, + owner=params.mapred_user, + group=params.user_group, + create_parents = True, + cd_access="a", + recursive_ownership = True, + ) + +def setup_nodemanager(): + import params + + # First start after enabling/disabling security + if params.toggle_nm_security: + Directory(params.nm_local_dirs_list + params.nm_log_dirs_list, + action='delete' + ) + + # If yarn.nodemanager.recovery.dir exists, remove this dir + if params.yarn_nodemanager_recovery_dir: + Directory(InlineTemplate(params.yarn_nodemanager_recovery_dir).get_content(), + action='delete' + ) + + # Setting NM marker file + if params.security_enabled: + Directory(params.nm_security_marker_dir) + File(params.nm_security_marker, + content="Marker file to track first start after enabling/disabling security. " + "During first start yarn local, log dirs are removed and recreated" + ) + elif not params.security_enabled: + File(params.nm_security_marker, action="delete") + + + if not params.security_enabled or params.toggle_nm_security: + # handle_mounted_dirs ensures that we don't create dirs which are temporary unavailable (unmounted), and intended to reside on a different mount. + nm_log_dir_to_mount_file_content = handle_mounted_dirs(create_log_dir, params.nm_log_dirs, params.nm_log_dir_to_mount_file, params) + # create a history file used by handle_mounted_dirs + File(params.nm_log_dir_to_mount_file, + owner=params.hdfs_user, + group=params.user_group, + mode=0644, + content=nm_log_dir_to_mount_file_content + ) + nm_local_dir_to_mount_file_content = handle_mounted_dirs(create_local_dir, params.nm_local_dirs, params.nm_local_dir_to_mount_file, params) + File(params.nm_local_dir_to_mount_file, + owner=params.hdfs_user, + group=params.user_group, + mode=0644, + content=nm_local_dir_to_mount_file_content + ) + +def setup_resourcemanager(): + import params + + Directory(params.rm_nodes_exclude_dir, + mode=0755, + create_parents=True, + cd_access='a', + ) + File(params.rm_nodes_exclude_path, + owner=params.yarn_user, + group=params.user_group + ) + File(params.yarn_job_summary_log, + owner=params.yarn_user, + group=params.user_group + ) + if not is_empty(params.node_label_enable) and params.node_label_enable or is_empty(params.node_label_enable) and params.node_labels_dir: + params.HdfsResource(params.node_labels_dir, + type="directory", + action="create_on_execute", + change_permissions_for_parents=True, + owner=params.yarn_user, + group=params.user_group, + mode=0700 + ) + params.HdfsResource(None, action="execute") + +def setup_ats(): + import params + + Directory(params.ats_leveldb_dir, + owner=params.yarn_user, + group=params.user_group, + create_parents = True, + cd_access="a", + ) + + # if stack support application timeline-service state store property (timeline_state_store stack feature) + if params.stack_supports_timeline_state_store: + Directory(params.ats_leveldb_state_store_dir, + owner=params.yarn_user, + group=params.user_group, + create_parents = True, + cd_access="a", + ) + # app timeline server 1.5 directories + if not is_empty(params.entity_groupfs_store_dir): + parent_path = os.path.dirname(params.entity_groupfs_store_dir) + params.HdfsResource(parent_path, + type="directory", + action="create_on_execute", + change_permissions_for_parents=True, + owner=params.yarn_user, + group=params.user_group, + mode=0755 + ) + params.HdfsResource(params.entity_groupfs_store_dir, + type="directory", + action="create_on_execute", + owner=params.yarn_user, + group=params.user_group, + mode=params.entity_groupfs_store_dir_mode + ) + if not is_empty(params.entity_groupfs_active_dir): + parent_path = os.path.dirname(params.entity_groupfs_active_dir) + params.HdfsResource(parent_path, + type="directory", + action="create_on_execute", + change_permissions_for_parents=True, + owner=params.yarn_user, + group=params.user_group, + mode=0755 + ) + params.HdfsResource(params.entity_groupfs_active_dir, + type="directory", + action="create_on_execute", + owner=params.yarn_user, + group=params.user_group, + mode=params.entity_groupfs_active_dir_mode + ) + params.HdfsResource(None, action="execute") + +def create_log_dir(dir_name): + import params + Directory(dir_name, + create_parents = True, + cd_access="a", + mode=0775, + owner=params.yarn_user, + group=params.user_group, + ignore_failures=True, + ) + +def create_local_dir(dir_name): + import params + Directory(dir_name, + create_parents = True, + cd_access="a", + mode=0755, + owner=params.yarn_user, + group=params.user_group, + ignore_failures=True, + recursive_mode_flags = {'f': 'a+rw', 'd': 'a+rwx'}, + ) + +@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY) +def yarn(name = None): + import params + XmlConfig("mapred-site.xml", + conf_dir=params.config_dir, + configurations=params.config['configurations']['mapred-site'], + owner=params.yarn_user, + mode='f' + ) + XmlConfig("yarn-site.xml", + conf_dir=params.config_dir, + configurations=params.config['configurations']['yarn-site'], + owner=params.yarn_user, + mode='f', + configuration_attributes=params.config['configuration_attributes']['yarn-site'] + ) + XmlConfig("capacity-scheduler.xml", + conf_dir=params.config_dir, + configurations=params.config['configurations']['capacity-scheduler'], + owner=params.yarn_user, + mode='f' + ) + + if params.service_map.has_key(name): + service_name = params.service_map[name] + + ServiceConfig(service_name, + action="change_user", + username = params.yarn_user, + password = Script.get_password(params.yarn_user)) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/6fb1ceef/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py index 643f946..119dcf0 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py +++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py @@ -464,7 +464,6 @@ class TestHistoryServer(RMFTestCase): ) def assert_configure_secured(self): - self.assertResourceCalled('HdfsResource', '/app-logs', immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, security_enabled = True, http://git-wip-us.apache.org/repos/asf/ambari/blob/6fb1ceef/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py index 7b5ce18..c98a64d 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py +++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py @@ -178,7 +178,19 @@ class TestResourceManager(RMFTestCase): self.assertNoMoreResources() def assert_configure_default(self): - + self.assertResourceCalled('Directory', '/etc/hadoop/conf', + mode = 0755, + create_parents = True, + cd_access = 'a', + ) + self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude', + owner = 'yarn', + group = 'hadoop', + ) + self.assertResourceCalled('File', '/var/log/hadoop-yarn/yarn/hadoop-mapreduce.jobsummary.log', + owner = 'yarn', + group = 'hadoop', + ) self.assertResourceCalled('Directory', '/var/run/hadoop-yarn', owner = 'yarn', group = 'hadoop', @@ -268,19 +280,6 @@ class TestResourceManager(RMFTestCase): configurations = self.getConfig()['configurations']['capacity-scheduler'], configuration_attributes = self.getConfig()['configuration_attributes']['capacity-scheduler'] ) - self.assertResourceCalled('Directory', '/etc/hadoop/conf', - mode = 0755, - create_parents = True, - cd_access = 'a', - ) - self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude', - owner = 'yarn', - group = 'hadoop', - ) - self.assertResourceCalled('File', '/var/log/hadoop-yarn/yarn/hadoop-mapreduce.jobsummary.log', - owner = 'yarn', - group = 'hadoop', - ) self.assertResourceCalled('File', '/etc/security/limits.d/yarn.conf', content = Template('yarn.conf.j2'), mode = 0644, @@ -347,7 +346,19 @@ class TestResourceManager(RMFTestCase): ) def assert_configure_secured(self): - + self.assertResourceCalled('Directory', '/etc/hadoop/conf', + mode = 0755, + create_parents = True, + cd_access = 'a', + ) + self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude', + owner = 'yarn', + group = 'hadoop', + ) + self.assertResourceCalled('File', '/var/log/hadoop-yarn/yarn/hadoop-mapreduce.jobsummary.log', + owner = 'yarn', + group = 'hadoop', + ) self.assertResourceCalled('Directory', '/var/run/hadoop-yarn', owner = 'yarn', group = 'hadoop', @@ -437,19 +448,6 @@ class TestResourceManager(RMFTestCase): configurations = self.getConfig()['configurations']['capacity-scheduler'], configuration_attributes = self.getConfig()['configuration_attributes']['capacity-scheduler'] ) - self.assertResourceCalled('Directory', '/etc/hadoop/conf', - mode = 0755, - create_parents = True, - cd_access = 'a', - ) - self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude', - owner = 'yarn', - group = 'hadoop', - ) - self.assertResourceCalled('File', '/var/log/hadoop-yarn/yarn/hadoop-mapreduce.jobsummary.log', - owner = 'yarn', - group = 'hadoop', - ) self.assertResourceCalled('File', '/etc/security/limits.d/yarn.conf', content = Template('yarn.conf.j2'), mode = 0644, http://git-wip-us.apache.org/repos/asf/ambari/blob/6fb1ceef/ambari-server/src/test/python/stacks/2.1/YARN/test_apptimelineserver.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.1/YARN/test_apptimelineserver.py b/ambari-server/src/test/python/stacks/2.1/YARN/test_apptimelineserver.py index dc11ba9..40db813 100644 --- a/ambari-server/src/test/python/stacks/2.1/YARN/test_apptimelineserver.py +++ b/ambari-server/src/test/python/stacks/2.1/YARN/test_apptimelineserver.py @@ -92,6 +92,26 @@ class TestAppTimelineServer(RMFTestCase): self.assertNoMoreResources() def assert_configure_default(self): + self.assertResourceCalled('Directory', '/var/log/hadoop-yarn/timeline', + owner = 'yarn', + group = 'hadoop', + create_parents = True, + cd_access='a' + ) + self.assertResourceCalled('HdfsResource', None, + immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, + security_enabled = False, + hadoop_bin_dir = '/usr/bin', + keytab = UnknownConfigurationMock(), + default_fs = 'hdfs://c6401.ambari.apache.org:8020', + dfs_type = '', + hdfs_site = self.getConfig()['configurations']['hdfs-site'], + kinit_path_local = '/usr/bin/kinit', + principal_name = UnknownConfigurationMock(), + user = 'hdfs', + action = ['execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', + hadoop_conf_dir = '/etc/hadoop/conf', + ) self.assertResourceCalled('Directory', '/var/run/hadoop-yarn', owner = 'yarn', group = 'hadoop', @@ -181,26 +201,6 @@ class TestAppTimelineServer(RMFTestCase): configurations = self.getConfig()['configurations']['capacity-scheduler'], configuration_attributes = self.getConfig()['configuration_attributes']['capacity-scheduler'] ) - self.assertResourceCalled('Directory', '/var/log/hadoop-yarn/timeline', - owner = 'yarn', - group = 'hadoop', - create_parents = True, - cd_access='a' - ) - self.assertResourceCalled('HdfsResource', None, - immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, - security_enabled = False, - hadoop_bin_dir = '/usr/bin', - keytab = UnknownConfigurationMock(), - default_fs = 'hdfs://c6401.ambari.apache.org:8020', - dfs_type = '', - hdfs_site = self.getConfig()['configurations']['hdfs-site'], - kinit_path_local = '/usr/bin/kinit', - principal_name = UnknownConfigurationMock(), - user = 'hdfs', - action = ['execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', - hadoop_conf_dir = '/etc/hadoop/conf', - ) self.assertResourceCalled('File', '/etc/security/limits.d/yarn.conf', content = Template('yarn.conf.j2'), mode = 0644, http://git-wip-us.apache.org/repos/asf/ambari/blob/6fb1ceef/ambari-server/src/test/python/stacks/2.3/YARN/test_ats_1_5.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/YARN/test_ats_1_5.py b/ambari-server/src/test/python/stacks/2.3/YARN/test_ats_1_5.py index abfe756..b523412 100644 --- a/ambari-server/src/test/python/stacks/2.3/YARN/test_ats_1_5.py +++ b/ambari-server/src/test/python/stacks/2.3/YARN/test_ats_1_5.py @@ -50,6 +50,100 @@ class TestAts(RMFTestCase): self.assertNoMoreResources() def assert_configure_default(self): + self.assertResourceCalled('Directory', '/var/log/hadoop-yarn/timeline', + owner = 'yarn', + group = 'hadoop', + create_parents = True, + cd_access = 'a', + ) + self.assertResourceCalled('HdfsResource', '/ats', + immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, + security_enabled = False, + hadoop_bin_dir = '/usr/bin', + keytab = UnknownConfigurationMock(), + default_fs = 'hdfs://c6401.ambari.apache.org:8020', + dfs_type = '', + hdfs_site = self.getConfig()['configurations']['hdfs-site'], + kinit_path_local = '/usr/bin/kinit', + principal_name = UnknownConfigurationMock(), + user = 'hdfs', + change_permissions_for_parents = True, + owner = 'yarn', + group = 'hadoop', + hadoop_conf_dir = '/etc/hadoop/conf', + type = 'directory', + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', + mode = 0755, + ) + self.assertResourceCalled('HdfsResource', '/ats/done', + immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, + security_enabled = False, + hadoop_bin_dir = '/usr/bin', + keytab = UnknownConfigurationMock(), + default_fs = 'hdfs://c6401.ambari.apache.org:8020', + dfs_type = '', + hdfs_site = self.getConfig()['configurations']['hdfs-site'], + kinit_path_local = '/usr/bin/kinit', + principal_name = UnknownConfigurationMock(), + user = 'hdfs', + owner = 'yarn', + group = 'hadoop', + hadoop_conf_dir = '/etc/hadoop/conf', + type = 'directory', + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', + mode = 0700, + ) + self.assertResourceCalled('HdfsResource', '/ats', + immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, + security_enabled = False, + hadoop_bin_dir = '/usr/bin', + keytab = UnknownConfigurationMock(), + default_fs = 'hdfs://c6401.ambari.apache.org:8020', + dfs_type = '', + hdfs_site = self.getConfig()['configurations']['hdfs-site'], + kinit_path_local = '/usr/bin/kinit', + principal_name = UnknownConfigurationMock(), + user = 'hdfs', + change_permissions_for_parents = True, + owner = 'yarn', + group = 'hadoop', + hadoop_conf_dir = '/etc/hadoop/conf', + type = 'directory', + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', + mode = 0755, + ) + self.assertResourceCalled('HdfsResource', '/ats/active', + immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, + security_enabled = False, + hadoop_bin_dir = '/usr/bin', + keytab = UnknownConfigurationMock(), + dfs_type = '', + default_fs = 'hdfs://c6401.ambari.apache.org:8020', + hdfs_site = self.getConfig()['configurations']['hdfs-site'], + kinit_path_local = '/usr/bin/kinit', + principal_name = UnknownConfigurationMock(), + user = 'hdfs', + owner = 'yarn', + group = 'hadoop', + hadoop_conf_dir = '/etc/hadoop/conf', + type = 'directory', + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', + mode = 01777, + ) + self.assertResourceCalled('HdfsResource', None, + immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, + security_enabled = False, + hadoop_bin_dir = '/usr/bin', + keytab = UnknownConfigurationMock(), + default_fs = 'hdfs://c6401.ambari.apache.org:8020', + dfs_type = '', + hdfs_site = self.getConfig()['configurations']['hdfs-site'], + kinit_path_local = '/usr/bin/kinit', + principal_name = UnknownConfigurationMock(), + user = 'hdfs', + action = ['execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', + hadoop_conf_dir = '/etc/hadoop/conf', + ) self.assertResourceCalled('Directory', '/var/run/hadoop-yarn', owner = 'yarn', group = 'hadoop', @@ -146,100 +240,6 @@ class TestAts(RMFTestCase): owner = 'yarn', configurations = self.getConfig()['configurations']['capacity-scheduler'], ) - self.assertResourceCalled('Directory', '/var/log/hadoop-yarn/timeline', - owner = 'yarn', - group = 'hadoop', - create_parents = True, - cd_access = 'a', - ) - self.assertResourceCalled('HdfsResource', '/ats', - immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, - security_enabled = False, - hadoop_bin_dir = '/usr/bin', - keytab = UnknownConfigurationMock(), - default_fs = 'hdfs://c6401.ambari.apache.org:8020', - dfs_type = '', - hdfs_site = self.getConfig()['configurations']['hdfs-site'], - kinit_path_local = '/usr/bin/kinit', - principal_name = UnknownConfigurationMock(), - user = 'hdfs', - change_permissions_for_parents = True, - owner = 'yarn', - group = 'hadoop', - hadoop_conf_dir = '/etc/hadoop/conf', - type = 'directory', - action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', - mode = 0755, - ) - self.assertResourceCalled('HdfsResource', '/ats/done', - immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, - security_enabled = False, - hadoop_bin_dir = '/usr/bin', - keytab = UnknownConfigurationMock(), - default_fs = 'hdfs://c6401.ambari.apache.org:8020', - dfs_type = '', - hdfs_site = self.getConfig()['configurations']['hdfs-site'], - kinit_path_local = '/usr/bin/kinit', - principal_name = UnknownConfigurationMock(), - user = 'hdfs', - owner = 'yarn', - group = 'hadoop', - hadoop_conf_dir = '/etc/hadoop/conf', - type = 'directory', - action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', - mode = 0700, - ) - self.assertResourceCalled('HdfsResource', '/ats', - immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, - security_enabled = False, - hadoop_bin_dir = '/usr/bin', - keytab = UnknownConfigurationMock(), - default_fs = 'hdfs://c6401.ambari.apache.org:8020', - dfs_type = '', - hdfs_site = self.getConfig()['configurations']['hdfs-site'], - kinit_path_local = '/usr/bin/kinit', - principal_name = UnknownConfigurationMock(), - user = 'hdfs', - change_permissions_for_parents = True, - owner = 'yarn', - group = 'hadoop', - hadoop_conf_dir = '/etc/hadoop/conf', - type = 'directory', - action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', - mode = 0755, - ) - self.assertResourceCalled('HdfsResource', '/ats/active', - immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, - security_enabled = False, - hadoop_bin_dir = '/usr/bin', - keytab = UnknownConfigurationMock(), - dfs_type = '', - default_fs = 'hdfs://c6401.ambari.apache.org:8020', - hdfs_site = self.getConfig()['configurations']['hdfs-site'], - kinit_path_local = '/usr/bin/kinit', - principal_name = UnknownConfigurationMock(), - user = 'hdfs', - owner = 'yarn', - group = 'hadoop', - hadoop_conf_dir = '/etc/hadoop/conf', - type = 'directory', - action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', - mode = 01777, - ) - self.assertResourceCalled('HdfsResource', None, - immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, - security_enabled = False, - hadoop_bin_dir = '/usr/bin', - keytab = UnknownConfigurationMock(), - default_fs = 'hdfs://c6401.ambari.apache.org:8020', - dfs_type = '', - hdfs_site = self.getConfig()['configurations']['hdfs-site'], - kinit_path_local = '/usr/bin/kinit', - principal_name = UnknownConfigurationMock(), - user = 'hdfs', - action = ['execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', - hadoop_conf_dir = '/etc/hadoop/conf', - ) self.assertResourceCalled('File', '/etc/security/limits.d/yarn.conf', content = Template('yarn.conf.j2'), mode = 0644,
