Repository: ambari Updated Branches: refs/heads/trunk 2831bdb3f -> 2dbb7b284
AMBARI-11513 [WinTP2] HDP 2.3 stack: Tez service fails to start +Fixed Tez settings generation +Fixed Tez service check dependency order Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2dbb7b28 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2dbb7b28 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2dbb7b28 Branch: refs/heads/trunk Commit: 2dbb7b2842807359937b5d15fb4a35635333622f Parents: 2831bdb Author: Florian Barca <[email protected]> Authored: Fri May 29 09:43:46 2015 -0700 Committer: Florian Barca <[email protected]> Committed: Fri May 29 09:43:46 2015 -0700 ---------------------------------------------------------------------- .../src/main/python/ambari_commons/os_utils.py | 21 ++++++++- .../libraries/functions/get_hdp_version.py | 32 +++++++++++++- .../0.4.0.2.1/package/scripts/params_windows.py | 22 +++++++++- .../TEZ/0.4.0.2.1/package/scripts/tez_client.py | 46 +++++++++++++++++--- .../stacks/HDPWIN/2.1/role_command_order.json | 1 + .../2.1/services/TEZ/configuration/tez-env.xml | 3 +- .../2.2/services/TEZ/configuration/tez-site.xml | 11 +---- .../services/HIVE/configuration/hive-site.xml | 9 ---- .../2.3/services/TEZ/configuration/tez-site.xml | 15 ++++++- 9 files changed, 130 insertions(+), 30 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/2dbb7b28/ambari-common/src/main/python/ambari_commons/os_utils.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/ambari_commons/os_utils.py b/ambari-common/src/main/python/ambari_commons/os_utils.py index 5ee34a8..d496530 100644 --- a/ambari-common/src/main/python/ambari_commons/os_utils.py +++ b/ambari-common/src/main/python/ambari_commons/os_utils.py @@ -123,4 +123,23 @@ def find_in_path(file): full_path = _search_file(file, os.environ["PATH"], os.pathsep) if full_path is None: raise Exception("File {0} not found in PATH".format(file)) - return full_path \ No newline at end of file + return full_path + +def extract_path_component(path, path_fragment): + iFragment = path.find(path_fragment) + if iFragment != -1: + iComponentStart = 0 + while iComponentStart < iFragment: + iComponentStartTemp = path.find(os.pathsep, iComponentStart) + if iComponentStartTemp == -1 or iComponentStartTemp > iFragment: + break + iComponentStart = iComponentStartTemp + + iComponentEnd = path.find(os.pathsep, iFragment) + if iComponentEnd == -1: + iComponentEnd = len(path) + + path_component = path[iComponentStart:iComponentEnd] + return path_component + else: + return None http://git-wip-us.apache.org/repos/asf/ambari/blob/2dbb7b28/ambari-common/src/main/python/resource_management/libraries/functions/get_hdp_version.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/get_hdp_version.py b/ambari-common/src/main/python/resource_management/libraries/functions/get_hdp_version.py index ec30dc4..e8fdbb6 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/get_hdp_version.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/get_hdp_version.py @@ -19,15 +19,45 @@ limitations under the License. Ambari Agent """ - __all__ = ["get_hdp_version"] + import os import re + +from ambari_commons import OSConst +from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl + from resource_management.core.logger import Logger from resource_management.core.exceptions import Fail from resource_management.core import shell +@OsFamilyFuncImpl(OSConst.WINSRV_FAMILY) +def get_hdp_version(package_name): + """ + @param package_name, name of the package, from which, function will try to get hdp version + """ + try: + component_home_dir = os.environ[package_name.upper() + "_HOME"] + except KeyError: + Logger.info('Skipping get_hdp_version since the component {0} is not yet available'.format(package_name)) + return None # lazy fail + + #As a rule, component_home_dir is of the form <hdp_root_dir>\[\]<component_versioned_subdir>[\] + home_dir_split = os.path.split(component_home_dir) + iSubdir = len(home_dir_split) - 1 + while not home_dir_split[iSubdir]: + iSubdir -= 1 + + #The component subdir is expected to be of the form <package_name>-<package_version>.<hdp_stack_version> + # with package_version = #.#.# and hdp_stack_version=#.#.#.#-<build_number> + match = re.findall('[0-9]+.[0-9]+.[0-9]+.[0-9]+-[0-9]+', home_dir_split[iSubdir]) + if not match: + Logger.info('Failed to get extracted version for component {0}. Home dir not in expected format.'.format(package_name)) + return None # lazy fail + + return match[0] +@OsFamilyFuncImpl(OsFamilyImpl.DEFAULT) def get_hdp_version(package_name): """ @param package_name, name of the package, from which, function will try to get hdp version http://git-wip-us.apache.org/repos/asf/ambari/blob/2dbb7b28/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_windows.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_windows.py b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_windows.py index 2a7bd8b..2f9ee30 100644 --- a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_windows.py +++ b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_windows.py @@ -17,17 +17,35 @@ limitations under the License. """ -from resource_management import * import os +from resource_management.libraries.functions.get_hdp_version import get_hdp_version +from resource_management.libraries.script.script import Script + config = Script.get_config() hadoop_user = config["configurations"]["cluster-env"]["hadoop.user.name"] +user_group = config["configurations"]["cluster-env"]["user_group"] tez_user = hadoop_user tez_home_dir = None tez_conf_dir = "conf" +try: + hadoop_classpath_prefix_template = config["configurations"]["tez-site"]["tez.cluster.additional.classpath.prefix"] +except KeyError: + hadoop_classpath_prefix_template = "" + +hdp_stack_version = "" + hdp_root = os.path.abspath(os.path.join(os.environ["HADOOP_HOME"], "..")) -if os.environ.has_key("TEZ_HOME"): + +def refresh_tez_state_dependent_params(): + global tez_home_dir, tez_conf_dir, hdp_stack_version tez_home_dir = os.environ["TEZ_HOME"] tez_conf_dir = os.path.join(tez_home_dir, "conf") + # this is not available on INSTALL action because hdp-select is not available + hdp_stack_version = get_hdp_version("tez") + + +if os.environ.has_key("TEZ_HOME"): + refresh_tez_state_dependent_params() http://git-wip-us.apache.org/repos/asf/ambari/blob/2dbb7b28/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez_client.py b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez_client.py index e6c92e1..1bd82ae 100644 --- a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez_client.py +++ b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez_client.py @@ -18,14 +18,23 @@ limitations under the License. Ambari Agent """ +import os +import urlparse -import sys -from resource_management import * +from ambari_commons import OSConst +from ambari_commons.inet_utils import download_file +from ambari_commons.os_family_impl import OsFamilyImpl +from ambari_commons.os_utils import copy_file, extract_path_component + +from resource_management.core.exceptions import ClientComponentHasNoStatus +from resource_management.core.source import InlineTemplate from resource_management.libraries.functions import conf_select from resource_management.libraries.functions import hdp_select +from resource_management.libraries.functions.get_hdp_version import get_hdp_version +from resource_management.libraries.functions.version import compare_versions, format_hdp_stack_version +from resource_management.libraries.script.script import Script + from tez import tez -from ambari_commons import OSConst -from ambari_commons.os_family_impl import OsFamilyImpl class TezClient(Script): def configure(self, env): @@ -56,12 +65,39 @@ class TezClientLinux(TezClient): @OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY) class TezClientWindows(TezClient): - def install(self, env): import params if params.tez_home_dir is None: self.install_packages(env) + params.refresh_tez_state_dependent_params() + env.set_params(params) + self._install_lzo_support_if_needed(params) self.configure(env) + def _install_lzo_support_if_needed(self, params): + hadoop_classpath_prefix = self._expand_hadoop_classpath_prefix(params.hadoop_classpath_prefix_template, params.config['configurations']['tez-site']) + + hadoop_lzo_dest_path = extract_path_component(hadoop_classpath_prefix, "hadoop-lzo-") + if hadoop_lzo_dest_path: + hadoop_lzo_file = os.path.split(hadoop_lzo_dest_path)[1] + + config = Script.get_config() + file_url = urlparse.urljoin(config['hostLevelParams']['jdk_location'], hadoop_lzo_file) + hadoop_lzo_dl_path = os.path.join(config["hostLevelParams"]["agentCacheDir"], hadoop_lzo_file) + download_file(file_url, hadoop_lzo_dl_path) + #This is for protection against configuration changes. It will infect every new destination with the lzo jar, + # but since the classpath points to the jar directly we're getting away with it. + if not os.path.exists(hadoop_lzo_dest_path): + copy_file(hadoop_lzo_dl_path, hadoop_lzo_dest_path) + + def _expand_hadoop_classpath_prefix(self, hadoop_classpath_prefix_template, configurations): + import resource_management + + hadoop_classpath_prefix_obj = InlineTemplate(hadoop_classpath_prefix_template, configurations_dict=configurations, + extra_imports=[resource_management, resource_management.core, + resource_management.core.source]) + hadoop_classpath_prefix = hadoop_classpath_prefix_obj.get_content() + return hadoop_classpath_prefix + if __name__ == "__main__": TezClient().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/2dbb7b28/ambari-server/src/main/resources/stacks/HDPWIN/2.1/role_command_order.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/role_command_order.json b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/role_command_order.json index dc55533..daa7f80 100644 --- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/role_command_order.json +++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/role_command_order.json @@ -72,6 +72,7 @@ "YARN_SERVICE_CHECK-SERVICE_CHECK": ["NODEMANAGER-START", "RESOURCEMANAGER-START"], "RESOURCEMANAGER_SERVICE_CHECK-SERVICE_CHECK": ["RESOURCEMANAGER-START"], "PIG_SERVICE_CHECK-SERVICE_CHECK": ["RESOURCEMANAGER-START", "NODEMANAGER-START"], + "TEZ_SERVICE_CHECK-SERVICE_CHECK": ["NODEMANAGER-START", "RESOURCEMANAGER-START", "HISTORYSERVER-START"], "NAMENODE-STOP": ["JOBTRACKER-STOP", "TASKTRACKER-STOP", "RESOURCEMANAGER-STOP", "NODEMANAGER-STOP", "HISTORYSERVER-STOP", "HBASE_MASTER-STOP"], "DATANODE-STOP": ["JOBTRACKER-STOP", "TASKTRACKER-STOP", "RESOURCEMANAGER-STOP", http://git-wip-us.apache.org/repos/asf/ambari/blob/2dbb7b28/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/TEZ/configuration/tez-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/TEZ/configuration/tez-env.xml b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/TEZ/configuration/tez-env.xml index fd7e7b0..55f3eaf 100644 --- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/TEZ/configuration/tez-env.xml +++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/TEZ/configuration/tez-env.xml @@ -30,7 +30,6 @@ <property> <name>content</name> <description>This is the jinja template for tez-env.cmd file</description> - <value> - </value> + <value></value> </property> </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/2dbb7b28/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/TEZ/configuration/tez-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/TEZ/configuration/tez-site.xml b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/TEZ/configuration/tez-site.xml index 50345fd..3f8c368 100644 --- a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/TEZ/configuration/tez-site.xml +++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/TEZ/configuration/tez-site.xml @@ -18,7 +18,6 @@ --> <configuration supports_final="true" supports_do_not_extend="true"> - <property> <name>tez.lib.uris</name> <value>/hdp/apps/${hdp.version}/tez/tez.tar.gz</value> @@ -47,12 +46,6 @@ </property> <property> - <name>tez.staging-dir</name> - <value>/tmp/${user.name}/staging</value> - <description>The staging dir used while submitting DAGs</description> - </property> - - <property> <name>tez.am.resource.memory.mb</name> <value>1536</value> <description>The amount of memory to be used by the AppMaster. @@ -76,7 +69,7 @@ <property> <name>tez.am.launch.env</name> - <value>LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-amd64-64</value> + <value>LD_LIBRARY_PATH=C:\hdp\hadoop\bin;C:\hdp\hadoop\share\hadoop\common\lib</value> <description> Additional execution environment entries for tez. This is not an additive property. You must preserve the original value if you want to have access to native libraries. @@ -108,7 +101,7 @@ <property> <name>tez.task.launch.env</name> - <value>LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-amd64-64</value> + <value>LD_LIBRARY_PATH=C:\hdp\hadoop\bin;C:\hdp\hadoop\share\hadoop\common\lib</value> <description> Additional execution environment entries for tez. This is not an additive property. You must preserve the original value if you want to have access to native libraries. http://git-wip-us.apache.org/repos/asf/ambari/blob/2dbb7b28/ambari-server/src/main/resources/stacks/HDPWIN/2.3/services/HIVE/configuration/hive-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.3/services/HIVE/configuration/hive-site.xml b/ambari-server/src/main/resources/stacks/HDPWIN/2.3/services/HIVE/configuration/hive-site.xml index e55b1de..a16994a 100644 --- a/ambari-server/src/main/resources/stacks/HDPWIN/2.3/services/HIVE/configuration/hive-site.xml +++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.3/services/HIVE/configuration/hive-site.xml @@ -205,15 +205,6 @@ limitations under the License. </property> <property> - <name>hive.execution.engine</name> - <value>mr</value> - <description> - Expects one of [mr, tez]. - Chooses execution engine. Options are: mr (Map reduce, default) or tez (hadoop 2 only) - </description> - </property> - - <property> <name>hive.compute.query.using.stats</name> <value>true</value> <description> http://git-wip-us.apache.org/repos/asf/ambari/blob/2dbb7b28/ambari-server/src/main/resources/stacks/HDPWIN/2.3/services/TEZ/configuration/tez-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.3/services/TEZ/configuration/tez-site.xml b/ambari-server/src/main/resources/stacks/HDPWIN/2.3/services/TEZ/configuration/tez-site.xml index be534f1..8f4ff63 100644 --- a/ambari-server/src/main/resources/stacks/HDPWIN/2.3/services/TEZ/configuration/tez-site.xml +++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.3/services/TEZ/configuration/tez-site.xml @@ -27,6 +27,20 @@ </property> <property> + <name>tez.cluster.additional.classpath.prefix</name> + <value>C:\hdp\hadoop\share\hadoop\common\lib\hadoop-lzo-0.4.19.{{hdp_stack_version}}.jar</value> + </property> + + <property> + <name>tez.lib.uris</name> + <value>file:///C:/hdp/tez-0.7.0.{{hdp_stack_version}}/tez-0.7.0.{{hdp_stack_version}}.tar.gz</value> + <description>Comma-delimited list of the location of the Tez libraries which will be localized for DAGs. + Specifying a single .tar.gz or .tgz assumes that a compressed version of the tez libs is being used. This is uncompressed into a tezlibs directory when running containers, and tezlibs/;tezlibs/lib/ are added to the classpath (after . and .*). + If multiple files are specified - files are localized as regular files, contents of directories are localized as regular files (non-recursive). + </description> + </property> + + <property> <name>tez.runtime.optimize.local.fetch</name> <value>true</value> <description>If the shuffle input is on the local host bypass the http fetch and access the files directly.</description> @@ -94,5 +108,4 @@ </property> </depends-on> </property> - </configuration>
