Repository: ambari Updated Branches: refs/heads/trunk b511b5b22 -> 1d3dce261
AMBARI-20148 - "Set Version on All Hosts" Fails For Hosts Without Any Stack Components (jonathanhurley) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/1d3dce26 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/1d3dce26 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/1d3dce26 Branch: refs/heads/trunk Commit: 1d3dce26190cd9105e47d1c18472075fdda75ef6 Parents: b511b5b Author: Jonathan Hurley <[email protected]> Authored: Thu Feb 23 14:59:01 2017 -0500 Committer: Jonathan Hurley <[email protected]> Committed: Fri Feb 24 08:51:54 2017 -0500 ---------------------------------------------------------------------- .../custom_actions/scripts/ru_set_all.py | 65 ++++++++++++++++---- .../python/custom_actions/test_ru_set_all.py | 61 +++++++++++++++++- 2 files changed, 110 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/1d3dce26/ambari-server/src/main/resources/custom_actions/scripts/ru_set_all.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/custom_actions/scripts/ru_set_all.py b/ambari-server/src/main/resources/custom_actions/scripts/ru_set_all.py index b7fcc51..a7732d9 100644 --- a/ambari-server/src/main/resources/custom_actions/scripts/ru_set_all.py +++ b/ambari-server/src/main/resources/custom_actions/scripts/ru_set_all.py @@ -22,6 +22,7 @@ Ambari Agent import os import shutil +import socket from ambari_commons.os_check import OSCheck from resource_management.libraries.script import Script from resource_management.libraries.functions import conf_select @@ -38,15 +39,12 @@ from resource_management.libraries.functions import StackFeature class UpgradeSetAll(Script): """ - This script is a part of Rolling Upgrade workflow and is used to set the - component versions as a final step in the upgrade process + This script is a part of stack upgrade workflow and is used to set the + all of the component versions as a final step in the upgrade process """ def actionexecute(self, env): - config = Script.get_config() - version = default('/commandParams/version', None) - stack_name = default('/hostLevelParams/stack_name', "") if not version: raise Fail("Value is required for '/commandParams/version'") @@ -56,15 +54,25 @@ class UpgradeSetAll(Script): cmd = ('/usr/bin/yum', 'clean', 'all') code, out = shell.call(cmd, sudo=True) - real_ver = format_stack_version(version) - if real_ver and check_stack_feature(StackFeature.ROLLING_UPGRADE, real_ver): - stack_selector_path = stack_tools.get_stack_tool_path(stack_tools.STACK_SELECTOR_NAME) - cmd = ('ambari-python-wrap', stack_selector_path, 'set', 'all', version) - code, out = shell.call(cmd, sudo=True) - if code != 0: - raise Exception("Command '{0}' exit code is nonzero".format(cmd)) + formatted_version = format_stack_version(version) + if not formatted_version: + raise Fail("Unable to determine a properly formatted stack version from {0}".format(version)) + + stack_selector_path = stack_tools.get_stack_tool_path(stack_tools.STACK_SELECTOR_NAME) + + # this script runs on all hosts; if this host doesn't have stack components, + # then don't invoke the stack tool + # (no need to log that it's skipped - the function will do that) + if is_host_skippable(stack_selector_path, formatted_version): + return + + # invoke "set all" + cmd = ('ambari-python-wrap', stack_selector_path, 'set', 'all', version) + code, out = shell.call(cmd, sudo=True) + if code != 0: + raise Exception("Command '{0}' exit code is nonzero".format(cmd)) - if real_ver and check_stack_feature(StackFeature.CONFIG_VERSIONING, real_ver): + if check_stack_feature(StackFeature.CONFIG_VERSIONING, formatted_version): # backup the old and symlink /etc/[component]/conf to <stack-root>/current/[component] for k, v in conf_select.get_package_dirs().iteritems(): for dir_def in v: @@ -148,6 +156,37 @@ class UpgradeSetAll(Script): Logger.info(" Skipping restoring config from backup {0} since it does not exist".format(backup_conf_directory)) +def is_host_skippable(stack_selector_path, formatted_version): + """ + Gets whether this host should not have the stack select tool called. + :param stack_selector_path the path to the stack selector tool. + :param formatted_version: the version to use with the stack selector tool. + :return: True if this host should be skipped, False otherwise. + """ + if not os.path.exists(stack_selector_path): + Logger.info("{0} does not have any stack components installed and will not invoke {1}".format( + socket.gethostname(), stack_selector_path)) + + return True + + # invoke the tool, checking its output + cmd = ('ambari-python-wrap', stack_selector_path, "versions") + code, out = shell.call(cmd, sudo=True) + + if code != 0: + Logger.info("{0} is unable to determine which stack versions are available using {1}".format( + socket.gethostname(), stack_selector_path)) + + return True + + # check to see if the output is empty, indicating no versions installed + if not out.strip(): + Logger.info("{0} has no stack versions installed".format(socket.gethostname())) + return True + + return False + + def link_config(old_conf, link_conf): """ Creates a config link following: http://git-wip-us.apache.org/repos/asf/ambari/blob/1d3dce26/ambari-server/src/test/python/custom_actions/test_ru_set_all.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/custom_actions/test_ru_set_all.py b/ambari-server/src/test/python/custom_actions/test_ru_set_all.py index b018605..e1a89a8 100644 --- a/ambari-server/src/test/python/custom_actions/test_ru_set_all.py +++ b/ambari-server/src/test/python/custom_actions/test_ru_set_all.py @@ -46,7 +46,7 @@ def fake_call(command, **kwargs): :param command: Command that will be echoed. :return: Returns a tuple of (process output code, output) """ - return (0, command) + return (0, str(command)) class TestRUSetAll(RMFTestCase): @@ -69,10 +69,11 @@ class TestRUSetAll(RMFTestCase): def tearDown(self): Logger.logger = None + @patch("os.path.exists") @patch("resource_management.core.shell.call") @patch.object(Script, 'get_config') @patch.object(OSCheck, 'is_redhat_family') - def test_execution(self, family_mock, get_config_mock, call_mock): + def test_execution(self, family_mock, get_config_mock, call_mock, exists_mock): # Mock the config objects json_file_path = os.path.join(self.get_custom_actions_dir(), "ru_execute_tasks_namenode_prepare.json") self.assertTrue(os.path.isfile(json_file_path)) @@ -88,6 +89,7 @@ class TestRUSetAll(RMFTestCase): family_mock.return_value = True get_config_mock.return_value = config_dict call_mock.side_effect = fake_call # echo the command + exists_mock.return_value = True # Ensure that the json file was actually read. stack_name = default("/hostLevelParams/stack_name", None) @@ -104,11 +106,12 @@ class TestRUSetAll(RMFTestCase): call_mock.assert_called_with(('ambari-python-wrap', '/usr/bin/hdp-select', 'set', 'all', u'2.2.1.0-2260'), sudo=True) + @patch("os.path.exists") @patch("resource_management.core.shell.call") @patch.object(Script, 'get_config') @patch.object(OSCheck, 'is_redhat_family') @patch("ru_set_all.link_config") - def test_execution_23(self, link_mock, family_mock, get_config_mock, call_mock): + def test_execution_23(self, link_mock, family_mock, get_config_mock, call_mock, exists_mock): # Mock the config objects json_file_path = os.path.join(self.get_custom_actions_dir(), "ru_execute_tasks_namenode_prepare.json") self.assertTrue(os.path.isfile(json_file_path)) @@ -125,6 +128,7 @@ class TestRUSetAll(RMFTestCase): family_mock.return_value = True get_config_mock.return_value = config_dict call_mock.side_effect = fake_call # echo the command + exists_mock.return_value = True # Ensure that the json file was actually read. stack_name = default("/hostLevelParams/stack_name", None) @@ -142,6 +146,57 @@ class TestRUSetAll(RMFTestCase): self.assertTrue(link_mock.called) call_mock.assert_called_with(('ambari-python-wrap', '/usr/bin/hdp-select', 'set', 'all', '2.3.0.0-1234'), sudo=True) + @patch("os.path.exists") + @patch("resource_management.core.shell.call") + @patch.object(Script, 'get_config') + @patch.object(OSCheck, 'is_redhat_family') + def test_skippable_hosts(self, family_mock, get_config_mock, call_mock, exists_mock): + """ + Tests that hosts are skippable if they don't have stack components installed + :return: + """ + # Mock the config objects + json_file_path = os.path.join(self.get_custom_actions_dir(), + "ru_execute_tasks_namenode_prepare.json") + self.assertTrue(os.path.isfile(json_file_path)) + + with open(json_file_path, "r") as json_file: + json_payload = json.load(json_file) + + json_payload["configurations"]["cluster-env"]["stack_tools"] = self.get_stack_tools() + json_payload["configurations"]["cluster-env"]["stack_features"] = self.get_stack_features() + + config_dict = ConfigDictionary(json_payload) + + family_mock.return_value = False + get_config_mock.return_value = config_dict + exists_mock.return_value = True + + def hdp_select_call(command, **kwargs): + # return no versions + if "versions" in command: + return (0,"") + + return (0,command) + + call_mock.side_effect = hdp_select_call + + # Ensure that the json file was actually read. + stack_name = default("/hostLevelParams/stack_name", None) + stack_version = default("/hostLevelParams/stack_version", None) + service_package_folder = default('/roleParams/service_package_folder', None) + + self.assertEqual(stack_name, "HDP") + self.assertEqual(stack_version, '2.2') + self.assertEqual(service_package_folder, "common-services/HDFS/2.1.0.2.0/package") + + # Begin the test + ru_execute = UpgradeSetAll() + ru_execute.actionexecute(None) + + call_mock.assert_called_with(('ambari-python-wrap', u'/usr/bin/hdp-select', 'versions'), sudo = True) + self.assertEqual(call_mock.call_count, 1) + @patch("os.path.islink") @patch("os.path.isdir")
