Repository: ambari Updated Branches: refs/heads/branch-2.6 8c5007469 -> e7e58bbf4 refs/heads/trunk 252e9898c -> 001f290be
AMBARI-21928 In case if component package doesn't advertise his version, Ambari silently failing on deploy stack steps (dgrinenko) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/001f290b Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/001f290b Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/001f290b Branch: refs/heads/trunk Commit: 001f290be2622375786f27cc054727adb7be8905 Parents: 252e989 Author: Dmytro Grinenko <[email protected]> Authored: Tue Sep 12 15:12:44 2017 +0300 Committer: Dmytro Grinenko <[email protected]> Committed: Tue Sep 12 15:12:44 2017 +0300 ---------------------------------------------------------------------- .../libraries/functions/stack_tools.py | 12 ++--- .../libraries/functions/version_select_util.py | 57 +++++++++----------- .../libraries/script/script.py | 2 + .../scripts/shared_initialization.py | 19 +++---- .../hooks/after-INSTALL/test_after_install.py | 2 +- 5 files changed, 42 insertions(+), 50 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/001f290b/ambari-common/src/main/python/resource_management/libraries/functions/stack_tools.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/stack_tools.py b/ambari-common/src/main/python/resource_management/libraries/functions/stack_tools.py index fa97fd1..d9233a3 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/stack_tools.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/stack_tools.py @@ -32,6 +32,7 @@ from resource_management.core.utils import pad STACK_SELECTOR_NAME = "stack_selector" CONF_SELECTOR_NAME = "conf_selector" + def get_stack_tool(name): """ Give a tool selector name get the stack-specific tool name, tool path, tool package @@ -43,7 +44,7 @@ def get_stack_tool(name): stack_name = default("/hostLevelParams/stack_name", None) if stack_name is None: Logger.warning("Cannot find the stack name in the command. Stack tools cannot be loaded") - return (None, None, None) + return None, None, None stack_tools = None stack_tools_config = default("/configurations/cluster-env/stack_tools", None) @@ -52,23 +53,22 @@ def get_stack_tool(name): if stack_tools is None: Logger.warning("The stack tools could not be found in cluster-env") - return (None, None, None) + return None, None, None if stack_name not in stack_tools: Logger.warning("Cannot find stack tools for the stack named {0}".format(stack_name)) - return (None, None, None) + return None, None, None # load the stack tooks keyed by the stack name stack_tools = stack_tools[stack_name] if not stack_tools or not name or name.lower() not in stack_tools: Logger.warning("Cannot find config for {0} stack tool in {1}".format(str(name), str(stack_tools))) - return (None, None, None) - + return None, None, None tool_config = stack_tools[name.lower()] - # Return fixed length (tool_name, tool_path tool_package) tuple + # Return fixed length (tool_name, tool_path, tool_package) tuple return tuple(pad(tool_config[:3], 3)) def get_stack_tool_name(name): http://git-wip-us.apache.org/repos/asf/ambari/blob/001f290b/ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py b/ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py index 79dc874..9fbb42b 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py @@ -26,7 +26,6 @@ import tempfile from resource_management.core.logger import Logger from resource_management.core import shell from resource_management.libraries.functions import stack_tools -import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set. def get_component_version(stack_name, component_name): @@ -39,42 +38,36 @@ def get_component_version(stack_name, component_name): :return: Returns a string if found, e.g., 2.2.1.0-2175, otherwise, returns None """ version = None - if stack_name is None or component_name is None: + if not stack_name or not component_name: Logger.error("Could not determine component version because of the parameters is empty. " \ "stack_name: %s, component_name: %s" % (str(stack_name), str(component_name))) return version - out = None - code = -1 - if not stack_name: - Logger.error("Stack name not provided") - elif not component_name: - Logger.error("Component name not provided") + stack_selector_name, stack_selector_path, stack_selector_package = stack_tools.get_stack_tool(stack_tools.STACK_SELECTOR_NAME) + if stack_selector_name and stack_selector_path and os.path.exists(stack_selector_path): + tmpfile = tempfile.NamedTemporaryFile(delete=True) + out, code = None, -1 + get_stack_comp_version_cmd = '%s status %s > %s' % (stack_selector_path, component_name, tmpfile.name) + + try: + # This is necessary because Ubuntu returns "stdin: is not a tty", see AMBARI-8088 + # ToDo: original problem looks strange + with open(tmpfile.name, 'r') as f: + code = shell.call(get_stack_comp_version_cmd, quiet=True)[0] + out = f.read() + + if code != 0 or out is None: + raise ValueError("Code is nonzero or output is empty") + + Logger.debug("Command: %s\nOutput: %s" % (get_stack_comp_version_cmd, str(out))) + matches = re.findall(r"( [\d\.]+(\-\d+)?)", out) + version = matches[0][0].strip() if matches and len(matches) > 0 and len(matches[0]) > 0 else None + Logger.debug("Version for component %s: %s" % (component_name, str(version))) + except Exception, e: + Logger.error("Could not determine stack version for component %s by calling '%s'. Return Code: %s, Output: %s." % + (component_name, get_stack_comp_version_cmd, str(code), str(out))) else: - (stack_selector_name, stack_selector_path, stack_selector_package) = stack_tools.get_stack_tool(stack_tools.STACK_SELECTOR_NAME) - if stack_selector_name and stack_selector_path and os.path.exists(stack_selector_path): - tmpfile = tempfile.NamedTemporaryFile() - - get_stack_comp_version_cmd = "" - try: - # This is necessary because Ubuntu returns "stdin: is not a tty", see AMBARI-8088 - with open(tmpfile.name, 'r') as file: - get_stack_comp_version_cmd = '%s status %s > %s' % (stack_selector_path, component_name, tmpfile.name) - code, stdoutdata = shell.call(get_stack_comp_version_cmd, quiet=True) - out = file.read() - - if code != 0 or out is None: - raise Exception("Code is nonzero or output is empty") - - Logger.debug("Command: %s\nOutput: %s" % (get_stack_comp_version_cmd, str(out))) - matches = re.findall(r"( [\d\.]+(\-\d+)?)", out) - version = matches[0][0].strip() if matches and len(matches) > 0 and len(matches[0]) > 0 else None - Logger.debug("Version for component %s: %s" % (component_name, str(version))) - except Exception, e: - Logger.error("Could not determine stack version for component %s by calling '%s'. Return Code: %s, Output: %s." % - (component_name, get_stack_comp_version_cmd, str(code), str(out))) - else: - Logger.error("Could not find stack selector for stack: %s" % str(stack_name)) + Logger.error("Could not find stack selector for stack: %s" % str(stack_name)) return version http://git-wip-us.apache.org/repos/asf/ambari/blob/001f290b/ambari-common/src/main/python/resource_management/libraries/script/script.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py b/ambari-common/src/main/python/resource_management/libraries/script/script.py index fcfac2b..0e352d1 100644 --- a/ambari-common/src/main/python/resource_management/libraries/script/script.py +++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py @@ -228,6 +228,8 @@ class Script(object): repo_version_id = default("/hostLevelParams/repository_version_id", None) if repo_version_id: self.put_structured_out({"repository_version_id": repo_version_id}) + else: + Logger.error("Component '{0}' did not advertise a version. This may indicate a problem with the component packaging.".format(stack_select_package_name)) def should_expose_component_version(self, command_name): http://git-wip-us.apache.org/repos/asf/ambari/blob/001f290b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/shared_initialization.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/shared_initialization.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/shared_initialization.py index daaffd8..67c3ba8 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/shared_initialization.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/shared_initialization.py @@ -46,7 +46,7 @@ def setup_stack_symlinks(struct_out_file): return if params.host_sys_prepped: - Logger.warning("Skipping running stack-selector-tool becase this is a sys_prepped host. This may cause symlink pointers not to be created for HDP componets installed later on top of an already sys_prepped host.") + Logger.warning("Skipping running stack-selector-tool because this is a sys_prepped host. This may cause symlink pointers not to be created for HDP components installed later on top of an already sys_prepped host") return # get the packages which the stack-select tool should be used on @@ -105,17 +105,14 @@ def load_version(struct_out_file): """ Load version from file. Made a separate method for testing """ - json_version = None try: - if os.path.exists(struct_out_file): - with open(struct_out_file, 'r') as fp: - json_info = json.load(fp) - json_version = json_info['version'] - except: - pass - - return json_version - + with open(struct_out_file, 'r') as fp: + json_info = json.load(fp) + + return json_info['version'] + except (IOError, KeyError, TypeError): + return None + def link_configs(struct_out_file): """ http://git-wip-us.apache.org/repos/asf/ambari/blob/001f290b/ambari-server/src/test/python/stacks/2.0.6/hooks/after-INSTALL/test_after_install.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/hooks/after-INSTALL/test_after_install.py b/ambari-server/src/test/python/stacks/2.0.6/hooks/after-INSTALL/test_after_install.py index d6740cb..f905cf9 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/hooks/after-INSTALL/test_after_install.py +++ b/ambari-server/src/test/python/stacks/2.0.6/hooks/after-INSTALL/test_after_install.py @@ -344,4 +344,4 @@ class TestHookAfterInstall(RMFTestCase): config_dict = json_content, config_overrides = self.CONFIG_OVERRIDES) - logger_warning_mock.assert_any_call('Skipping running stack-selector-tool becase this is a sys_prepped host. This may cause symlink pointers not to be created for HDP componets installed later on top of an already sys_prepped host.') + logger_warning_mock.assert_any_call('Skipping running stack-selector-tool because this is a sys_prepped host. This may cause symlink pointers not to be created for HDP components installed later on top of an already sys_prepped host')
