Repository: ambari Updated Branches: refs/heads/AMBARI-13364 e9741e0e6 -> f83a07e36
AMBARI-15137: Parameterize distro-specific stack information for TEZ (Juanjo Marron via dili) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f83a07e3 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f83a07e3 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f83a07e3 Branch: refs/heads/AMBARI-13364 Commit: f83a07e360a9dd6717ce03a7b5d718e1fe8433c4 Parents: e9741e0 Author: Di Li <[email protected]> Authored: Mon Feb 29 10:34:46 2016 -0500 Committer: Di Li <[email protected]> Committed: Mon Feb 29 10:34:46 2016 -0500 ---------------------------------------------------------------------- .../TEZ/0.4.0.2.1/package/scripts/params_linux.py | 18 +++++++++++------- .../0.4.0.2.1/package/scripts/params_windows.py | 10 +++++----- .../TEZ/0.4.0.2.1/package/scripts/pre_upgrade.py | 4 ++-- .../0.4.0.2.1/package/scripts/service_check.py | 4 ++-- .../TEZ/0.4.0.2.1/package/scripts/tez_client.py | 5 +++-- .../HDP/2.0.6/configuration/cluster-env.xml | 5 +++++ 6 files changed, 28 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/f83a07e3/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_linux.py index 25f867e..05bf5e0 100644 --- a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_linux.py @@ -32,10 +32,13 @@ config = Script.get_config() tmp_dir = Script.get_tmp_dir() stack_name = default("/hostLevelParams/stack_name", None) +stack_dir = config['configurations']['cluster-env']['stack_dir'] # This is expected to be of the form #.#.#.# stack_version_unformatted = str(config['hostLevelParams']['stack_version']) -hdp_stack_version = format_hdp_stack_version(stack_version_unformatted) +stack_version_formatted = format_hdp_stack_version(stack_version_unformatted) +stack_version_ru_support = config['configurations']['cluster-env']['stack_version_ru_support'] +stack_version_tez_symlink_support = config['configurations']['cluster-env']['stack_version_tez_symlink_support'] # New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade version = default("/commandParams/version", None) @@ -48,15 +51,16 @@ tez_etc_dir = "/etc/tez" config_dir = "/etc/tez/conf" tez_examples_jar = "/usr/lib/tez/tez-mapreduce-examples*.jar" -# hadoop parameters for 2.2+ -if Script.is_hdp_stack_greater_or_equal("2.2"): - tez_examples_jar = "/usr/hdp/current/tez-client/tez-examples*.jar" +# hadoop parameters for stack_version_ru_support+ +if Script.is_hdp_stack_greater_or_equal(stack_version_ru_support): + tez_examples_jar = "{stack_dir}/current/tez-client/tez-examples*.jar" -# tez only started linking /usr/hdp/x.x.x.x/tez-client/conf in HDP 2.3+ -if Script.is_hdp_stack_greater_or_equal("2.3"): +# tez only started linking <stack_dir>/x.x.x.x/tez-client/conf in stack_version_tez_symlink_support+ +if Script.is_hdp_stack_greater_or_equal(stack_version_tez_symlink_support): # !!! use realpath for now since the symlink exists but is broken and a # broken symlink messes with the DirectoryProvider class - config_dir = os.path.realpath("/usr/hdp/current/tez-client/conf") + config_path = os.path.join(stack_dir, "current/tez-client/conf") + config_dir = os.path.realpath(config_path) kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None)) security_enabled = config['configurations']['cluster-env']['security_enabled'] http://git-wip-us.apache.org/repos/asf/ambari/blob/f83a07e3/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_windows.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_windows.py b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_windows.py index 636d092..31d37f0 100644 --- a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_windows.py +++ b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_windows.py @@ -34,20 +34,20 @@ try: except KeyError: hadoop_classpath_prefix_template = "" -hdp_stack_version = "" +stack_version_formatted = "" -hdp_root = None +stack_root = None try: - hdp_root = os.path.abspath(os.path.join(os.environ["HADOOP_HOME"], "..")) + stack_root = os.path.abspath(os.path.join(os.environ["HADOOP_HOME"], "..")) except: pass def refresh_tez_state_dependent_params(): - global tez_home_dir, tez_conf_dir, hdp_stack_version + global tez_home_dir, tez_conf_dir, stack_version_formatted tez_home_dir = os.environ["TEZ_HOME"] tez_conf_dir = os.path.join(tez_home_dir, "conf") # this is not available on INSTALL action because hdp-select is not available - hdp_stack_version = get_hdp_version("tez") + stack_version_formatted = get_hdp_version("tez") if os.environ.has_key("TEZ_HOME"): http://git-wip-us.apache.org/repos/asf/ambari/blob/f83a07e3/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/pre_upgrade.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/pre_upgrade.py b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/pre_upgrade.py index a38d917..68df679 100644 --- a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/pre_upgrade.py +++ b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/pre_upgrade.py @@ -38,8 +38,8 @@ class TezPreUpgrade(Script): Logger.info("Before starting Stack Upgrade, check if tez tarball has been copied to HDFS.") - if params.hdp_stack_version and compare_versions(params.hdp_stack_version, '2.2.0.0') >= 0: - Logger.info("Stack version {0} is sufficient to check if need to copy tez.tar.gz to HDFS.".format(params.hdp_stack_version)) + if params.stack_version_formatted and compare_versions(params.stack_version_formatted, params.stack_version_ru_support) >= 0: + Logger.info("Stack version {0} is sufficient to check if need to copy tez.tar.gz to HDFS.".format(params.stack_version_formatted)) # Force it to copy the current version of the tez tarball, rather than the version the RU will go to. resource_created = copy_to_hdfs( http://git-wip-us.apache.org/repos/asf/ambari/blob/f83a07e3/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/service_check.py index 1747cf3..e581d8b 100644 --- a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/service_check.py +++ b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/service_check.py @@ -67,7 +67,7 @@ class TezServiceCheckLinux(TezServiceCheck): source = format("{tmp_dir}/sample-tez-test"), ) - if params.hdp_stack_version and compare_versions(params.hdp_stack_version, '2.2.0.0') >= 0: + if params.stack_version_formatted and compare_versions(params.stack_version_formatted, params.stack_version_ru_support) >= 0: copy_to_hdfs("tez", params.user_group, params.hdfs_user, host_sys_prepped=params.host_sys_prepped) params.HdfsResource(None, action = "execute") @@ -100,7 +100,7 @@ class TezServiceCheckWindows(TezServiceCheck): def service_check(self, env): import params env.set_params(params) - smoke_cmd = os.path.join(params.hdp_root,"Run-SmokeTests.cmd") + smoke_cmd = os.path.join(params.stack_root,"Run-SmokeTests.cmd") service = "TEZ" Execute(format("cmd /C {smoke_cmd} {service}"), logoutput=True, user=params.tez_user) http://git-wip-us.apache.org/repos/asf/ambari/blob/f83a07e3/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez_client.py b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez_client.py index 8bdabf1..0015d69 100644 --- a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez_client.py +++ b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez_client.py @@ -49,13 +49,14 @@ class TezClient(Script): class TezClientLinux(TezClient): def get_stack_to_component(self): - return {"HDP": "hadoop-client"} + import params + return {params.stack_name: "hadoop-client"} def pre_upgrade_restart(self, env, upgrade_type=None): import params env.set_params(params) - if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0: + if params.version and compare_versions(format_hdp_stack_version(params.version), params.stack_version_ru_support) >= 0: conf_select.select(params.stack_name, "tez", params.version) conf_select.select(params.stack_name, "hadoop", params.version) hdp_select.select("hadoop-client", params.version) http://git-wip-us.apache.org/repos/asf/ambari/blob/f83a07e3/ambari-server/src/main/resources/stacks/HDP/2.0.6/configuration/cluster-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/configuration/cluster-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.6/configuration/cluster-env.xml index 4c4b4a2..8fb9194 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/configuration/cluster-env.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/configuration/cluster-env.xml @@ -72,6 +72,11 @@ <description>Stack version from which oozie-server special case in the PRE-UPGRADE phase is supported</description> </property> <property> + <name>stack_version_tez_symlink_support</name> + <value>2.3.0.0</value> + <description>Stack version from which tez-client symlink for configuration is supported</description> + </property> + <property> <name>security_enabled</name> <value>false</value> <description>Hadoop Security</description>
