Repository: ambari Updated Branches: refs/heads/AMBARI-13364 3e02731ab -> 7a77bfe52
AMBARI-15053: Parameterize distro-specific stack information for YARN/MR (Juanjo Marron via dili) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/7a77bfe5 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/7a77bfe5 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/7a77bfe5 Branch: refs/heads/AMBARI-13364 Commit: 7a77bfe52060920678d50aebc6d7f1e24ef01858 Parents: 3e02731 Author: Di Li <[email protected]> Authored: Mon Feb 29 10:28:27 2016 -0500 Committer: Di Li <[email protected]> Committed: Mon Feb 29 10:28:27 2016 -0500 ---------------------------------------------------------------------- .../scripts/application_timeline_server.py | 5 ++-- .../2.1.0.2.0/package/scripts/historyserver.py | 9 ++++---- .../package/scripts/mapreduce2_client.py | 5 ++-- .../2.1.0.2.0/package/scripts/nodemanager.py | 5 ++-- .../2.1.0.2.0/package/scripts/params_linux.py | 24 +++++++++++--------- .../package/scripts/resourcemanager.py | 5 ++-- .../2.1.0.2.0/package/scripts/service_check.py | 6 ++--- .../YARN/2.1.0.2.0/package/scripts/yarn.py | 6 ++--- .../2.1.0.2.0/package/scripts/yarn_client.py | 5 ++-- 9 files changed, 39 insertions(+), 31 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/7a77bfe5/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/application_timeline_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/application_timeline_server.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/application_timeline_server.py index 7644225..61e4d66 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/application_timeline_server.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/application_timeline_server.py @@ -63,14 +63,15 @@ class ApplicationTimelineServerWindows(ApplicationTimelineServer): @OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT) class ApplicationTimelineServerDefault(ApplicationTimelineServer): def get_stack_to_component(self): - return {"HDP": "hadoop-yarn-timelineserver"} + import params + return {params.stack_name: "hadoop-yarn-timelineserver"} def pre_upgrade_restart(self, env, upgrade_type=None): Logger.info("Executing Stack Upgrade pre-restart") import params env.set_params(params) - if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0: + if params.version and compare_versions(format_hdp_stack_version(params.version), params.stack_version_ru_support) >= 0: conf_select.select(params.stack_name, "hadoop", params.version) hdp_select.select("hadoop-yarn-timelineserver", params.version) http://git-wip-us.apache.org/repos/asf/ambari/blob/7a77bfe5/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/historyserver.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/historyserver.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/historyserver.py index 5d95c5c..c8b1bcd 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/historyserver.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/historyserver.py @@ -70,14 +70,15 @@ class HistoryserverWindows(HistoryServer): @OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT) class HistoryServerDefault(HistoryServer): def get_stack_to_component(self): - return {"HDP": "hadoop-mapreduce-historyserver"} + import params + return {params.stack_name: "hadoop-mapreduce-historyserver"} def pre_upgrade_restart(self, env, upgrade_type=None): Logger.info("Executing Stack Upgrade pre-restart") import params env.set_params(params) - if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0: + if params.version and compare_versions(format_hdp_stack_version(params.version), params.stack_version_ru_support) >= 0: conf_select.select(params.stack_name, "hadoop", params.version) hdp_select.select("hadoop-mapreduce-historyserver", params.version) # MC Hammer said, "Can't touch this" @@ -91,7 +92,7 @@ class HistoryServerDefault(HistoryServer): env.set_params(params) self.configure(env) # FOR SECURITY - if params.hdp_stack_version_major and compare_versions(params.hdp_stack_version_major, '2.2.0.0') >= 0: + if params.stack_version_formatted and compare_versions(params.stack_version_formatted, params.stack_version_ru_support) >= 0: # MC Hammer said, "Can't touch this" resource_created = copy_to_hdfs( "mapreduce", @@ -111,7 +112,7 @@ class HistoryServerDefault(HistoryServer): if resource_created: params.HdfsResource(None, action="execute") else: - # In HDP 2.1, tez.tar.gz was copied to a different folder in HDFS. + # In version < stack_version_ru_support tez.tar.gz was copied to a different folder in HDFS. install_tez_jars() service('historyserver', action='start', serviceName='mapreduce') http://git-wip-us.apache.org/repos/asf/ambari/blob/7a77bfe5/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/mapreduce2_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/mapreduce2_client.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/mapreduce2_client.py index 7ceadf0..085ad10 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/mapreduce2_client.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/mapreduce2_client.py @@ -50,13 +50,14 @@ class MapReduce2ClientWindows(MapReduce2Client): @OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT) class MapReduce2ClientDefault(MapReduce2Client): def get_stack_to_component(self): - return {"HDP": "hadoop-client"} + import params + return {params.stack_name: "hadoop-client"} def pre_upgrade_restart(self, env, upgrade_type=None): import params env.set_params(params) - if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0: + if params.version and compare_versions(format_hdp_stack_version(params.version), params.stack_version_ru_support) >= 0: conf_select.select(params.stack_name, "hadoop", params.version) hdp_select.select("hadoop-client", params.version) http://git-wip-us.apache.org/repos/asf/ambari/blob/7a77bfe5/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager.py index d508d55..9ea7713 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager.py @@ -65,14 +65,15 @@ class NodemanagerWindows(Nodemanager): @OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT) class NodemanagerDefault(Nodemanager): def get_stack_to_component(self): - return {"HDP": "hadoop-yarn-nodemanager"} + import params + return {params.stack_name: "hadoop-yarn-nodemanager"} def pre_upgrade_restart(self, env, upgrade_type=None): Logger.info("Executing NodeManager Stack Upgrade pre-restart") import params env.set_params(params) - if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0: + if params.version and compare_versions(format_hdp_stack_version(params.version), params.stack_version_ru_support) >= 0: conf_select.select(params.stack_name, "hadoop", params.version) hdp_select.select("hadoop-yarn-nodemanager", params.version) http://git-wip-us.apache.org/repos/asf/ambari/blob/7a77bfe5/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py index f0b6927..c57a4d9 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py @@ -33,7 +33,7 @@ from resource_management.libraries import functions import status_params # a map of the Ambari role to the component name -# for use with /usr/hdp/current/<component> +# for use with <stack_dir>/current/<component> MAPR_SERVER_ROLE_DIRECTORY_MAP = { 'HISTORYSERVER' : 'hadoop-mapreduce-historyserver', 'MAPREDUCE2_CLIENT' : 'hadoop-mapreduce-client', @@ -51,11 +51,13 @@ config = Script.get_config() tmp_dir = Script.get_tmp_dir() stack_name = default("/hostLevelParams/stack_name", None) +stack_dir = config['configurations']['cluster-env']['stack_dir'] # This is expected to be of the form #.#.#.# stack_version_unformatted = str(config['hostLevelParams']['stack_version']) -hdp_stack_version_major = format_hdp_stack_version(stack_version_unformatted) -hdp_stack_version = functions.get_hdp_version('hadoop-yarn-resourcemanager') +stack_version_formatted = format_hdp_stack_version(stack_version_unformatted) +stack_version = functions.get_hdp_version('hadoop-yarn-resourcemanager') +stack_version_ru_support = config['configurations']['cluster-env']['stack_version_ru_support'] # New Cluster Stack Version that is defined during the RESTART of a Stack Upgrade. # It cannot be used during the initial Cluser Install because the version is not yet known. @@ -75,8 +77,8 @@ yarn_bin = "/usr/lib/hadoop-yarn/sbin" yarn_container_bin = "/usr/lib/hadoop-yarn/bin" hadoop_java_io_tmpdir = os.path.join(tmp_dir, "hadoop_java_io_tmpdir") -# hadoop parameters for 2.2+ -if Script.is_hdp_stack_greater_or_equal("2.2"): +# hadoop parameters for stack_version_ru_support+ +if Script.is_hdp_stack_greater_or_equal(stack_version_ru_support): # MapR directory root mapred_role_root = "hadoop-mapreduce-client" command_role = default("/role", "") @@ -88,14 +90,14 @@ if Script.is_hdp_stack_greater_or_equal("2.2"): if command_role in YARN_SERVER_ROLE_DIRECTORY_MAP: yarn_role_root = YARN_SERVER_ROLE_DIRECTORY_MAP[command_role] - hadoop_mapred2_jar_location = format("/usr/hdp/current/{mapred_role_root}") - mapred_bin = format("/usr/hdp/current/{mapred_role_root}/sbin") + hadoop_mapred2_jar_location = format("{stack_dir}/current/{mapred_role_root}") + mapred_bin = format("{stack_dir}/current/{mapred_role_root}/sbin") - hadoop_yarn_home = format("/usr/hdp/current/{yarn_role_root}") - yarn_bin = format("/usr/hdp/current/{yarn_role_root}/sbin") - yarn_container_bin = format("/usr/hdp/current/{yarn_role_root}/bin") + hadoop_yarn_home = format("{stack_dir}/current/{yarn_role_root}") + yarn_bin = format("{stack_dir}/current/{yarn_role_root}/sbin") + yarn_container_bin = format("{stack_dir}/current/{yarn_role_root}/bin") - # Timeline Service property that was added in 2.2 + # Timeline Service property that was added in tack_version_ru_support ats_leveldb_state_store_dir = config['configurations']['yarn-site']['yarn.timeline-service.leveldb-state-store.path'] # ats 1.5 properties http://git-wip-us.apache.org/repos/asf/ambari/blob/7a77bfe5/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py index ec7799e..27ed804 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py @@ -95,14 +95,15 @@ class ResourcemanagerWindows(Resourcemanager): @OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT) class ResourcemanagerDefault(Resourcemanager): def get_stack_to_component(self): - return {"HDP": "hadoop-yarn-resourcemanager"} + import params + return {params.stack_name: "hadoop-yarn-resourcemanager"} def pre_upgrade_restart(self, env, upgrade_type=None): Logger.info("Executing Stack Upgrade post-restart") import params env.set_params(params) - if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0: + if params.version and compare_versions(format_hdp_stack_version(params.version), params.stack_version_ru_support) >= 0: conf_select.select(params.stack_name, "hadoop", params.version) hdp_select.select("hadoop-yarn-resourcemanager", params.version) http://git-wip-us.apache.org/repos/asf/ambari/blob/7a77bfe5/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service_check.py index cd4d558..c25c8f4 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service_check.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service_check.py @@ -85,10 +85,10 @@ class ServiceCheckDefault(ServiceCheck): import params env.set_params(params) - if params.hdp_stack_version_major != "" and compare_versions(params.hdp_stack_version_major, '2.2') >= 0: - path_to_distributed_shell_jar = "/usr/hdp/current/hadoop-yarn-client/hadoop-yarn-applications-distributedshell.jar" + if params.stack_version_formatted != "" and compare_versions(params.stack_version_formatted, params.stack_version_ru_support) >= 0: + path_to_distributed_shell_jar = format("{stack_dir}/current/hadoop-yarn-client/hadoop-yarn-applications-distributedshell.jar") else: - path_to_distributed_shell_jar = "/usr/lib/hadoop-yarn/hadoop-yarn-applications-distributedshell*.jar" + path_to_distributed_shell_jar = format("{stack_dir}/hadoop-yarn/hadoop-yarn-applications-distributedshell*.jar") yarn_distrubuted_shell_check_cmd = format("yarn org.apache.hadoop.yarn.applications.distributedshell.Client " "-shell_command ls -num_containers {number_of_nm} -jar {path_to_distributed_shell_jar}") http://git-wip-us.apache.org/repos/asf/ambari/blob/7a77bfe5/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py index edfbe44..97891de 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py @@ -189,7 +189,7 @@ def yarn(name = None): # During RU, Core Masters and Slaves need hdfs-site.xml # TODO, instead of specifying individual configs, which is susceptible to breaking when new configs are added, - # RU should rely on all available in /usr/hdp/<version>/hadoop/conf + # RU should rely on all available in {stack_dir}/<version>/hadoop/conf if 'hdfs-site' in params.config['configurations']: XmlConfig("hdfs-site.xml", conf_dir=params.hadoop_conf_dir, @@ -252,8 +252,8 @@ def yarn(name = None): cd_access="a", ) - # if HDP stack is greater than/equal to 2.2, mkdir for state store property (added in 2.2) - if (Script.is_hdp_stack_greater_or_equal("2.2")): + # if stack is greater than/equal to stack_version_ru_support, mkdir for state store property (added in stack_version_ru_support) + if (Script.is_hdp_stack_greater_or_equal(params.stack_version_ru_support)): Directory(params.ats_leveldb_state_store_dir, owner=params.yarn_user, group=params.user_group, http://git-wip-us.apache.org/repos/asf/ambari/blob/7a77bfe5/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn_client.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn_client.py index 0c6115f..8c4c063 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn_client.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn_client.py @@ -50,13 +50,14 @@ class YarnClientWindows(YarnClient): @OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT) class YarnClientDefault(YarnClient): def get_stack_to_component(self): - return {"HDP": "hadoop-client"} + import params + return {params.stack_name: "hadoop-client"} def pre_upgrade_restart(self, env, upgrade_type=None): import params env.set_params(params) - if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0: + if params.version and compare_versions(format_hdp_stack_version(params.version), params.stack_version_ru_support) >= 0: conf_select.select(params.stack_name, "hadoop", params.version) hdp_select.select("hadoop-client", params.version)
