AMBARI-13210: RU - Install version stuck (jluniya)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/23bf111a Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/23bf111a Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/23bf111a Branch: refs/heads/branch-dev-patch-upgrade Commit: 23bf111a00391dcd3cee4e92d19a136fbdc3ca3c Parents: 93f86a4 Author: Jayush Luniya <[email protected]> Authored: Thu Sep 24 15:55:36 2015 -0700 Committer: Jayush Luniya <[email protected]> Committed: Thu Sep 24 15:55:36 2015 -0700 ---------------------------------------------------------------------- .../libraries/functions/hdp_select.py | 19 ++- .../libraries/functions/version_select_util.py | 20 ++- .../DistributeRepositoriesActionListener.java | 13 +- .../custom_actions/scripts/install_packages.py | 128 +++++++++++++++---- .../custom_actions/TestInstallPackages.py | 36 ++++-- 5 files changed, 171 insertions(+), 45 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/23bf111a/ambari-common/src/main/python/resource_management/libraries/functions/hdp_select.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/hdp_select.py b/ambari-common/src/main/python/resource_management/libraries/functions/hdp_select.py index 3113c86..5efc07e 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/hdp_select.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/hdp_select.py @@ -18,6 +18,7 @@ limitations under the License. """ +import os import sys from resource_management.core.logger import Logger from resource_management.core.exceptions import Fail @@ -27,6 +28,7 @@ from resource_management.libraries.functions.get_hdp_version import get_hdp_vers from resource_management.libraries.script.script import Script from resource_management.core.shell import call from resource_management.libraries.functions.version import format_hdp_stack_version +from resource_management.libraries.functions.version_select_util import get_versions_from_stack_root HDP_SELECT_PREFIX = ('ambari-python-wrap', 'hdp-select') # hdp-select set oozie-server 2.2.0.0-1234 @@ -240,12 +242,19 @@ def _get_upgrade_stack(): return None -def get_hdp_versions(): +def get_hdp_versions(stack_root): + """ + Gets list of stack versions installed on the host. + Be default a call to hdp-select versions is made to get the list of installed stack versions. + As a fallback list of installed versions is collected from stack version directories in stack install root. + :param stack_root: Stack install root + :return: Returns list of installed stack versions. + """ code, out = call(HDP_SELECT_PREFIX + ('versions',)) + versions = [] if 0 == code: - versions = [] for line in out.splitlines(): versions.append(line.rstrip('\n')) - return versions - else: - return [] + if not versions: + versions = get_versions_from_stack_root(stack_root) + return versions http://git-wip-us.apache.org/repos/asf/ambari/blob/23bf111a/ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py b/ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py index d1649df..f1a484b 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py @@ -19,6 +19,7 @@ limitations under the License. Ambari Agent """ +import os import re import tempfile @@ -74,4 +75,21 @@ def get_component_version(stack_name, component_name): else: Logger.error("Could not find a stack for stack name: %s" % str(stack_name)) - return version \ No newline at end of file + return version + + +def get_versions_from_stack_root(stack_root): + """ + Given a stack install root (/usr/hdp), returns a list of stack versions currently installed. + The list of installed stack versions is determined purely based on the stack version directories + found in the stack install root. + Because each stack name may have different logic, the input is a generic dictionary. + :param stack_root: Stack install root directory + :return: Returns list of installed stack versions + """ + if stack_root is None or not os.path.exists(stack_root): + return [] + + installed_stack_versions = [f for f in os.listdir(stack_root) if os.path.isdir(os.path.join(stack_root, f)) + and re.match("([\d\.]+(-\d+)?)", f)] + return installed_stack_versions http://git-wip-us.apache.org/repos/asf/ambari/blob/23bf111a/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/DistributeRepositoriesActionListener.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/DistributeRepositoriesActionListener.java b/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/DistributeRepositoriesActionListener.java index 2c56861..cd82957 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/DistributeRepositoriesActionListener.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/DistributeRepositoriesActionListener.java @@ -99,20 +99,23 @@ public class DistributeRepositoriesActionListener { String repositoryVersion = null; if (event.getCommandReport() == null) { - LOG.error("Command report is null, will set all INSTALLING versions for host {} to INSTALL_FAILED.", event.getHostname()); + LOG.error( + "Command report is null, will set all INSTALLING versions for host {} to INSTALL_FAILED.", + event.getHostname()); + } else if (!event.getCommandReport().getStatus().equals(HostRoleStatus.COMPLETED.toString())) { + LOG.warn( + "Distribute repositories did not complete, will set all INSTALLING versions for host {} to INSTALL_FAILED.", + event.getHostname()); } else { // Parse structured output try { + newHostState = RepositoryVersionState.INSTALLED; DistributeRepositoriesStructuredOutput structuredOutput = StageUtils.getGson().fromJson( event.getCommandReport().getStructuredOut(), DistributeRepositoriesStructuredOutput.class); repositoryVersion = structuredOutput.getInstalledRepositoryVersion(); - if (event.getCommandReport().getStatus().equals(HostRoleStatus.COMPLETED.toString())) { - newHostState = RepositoryVersionState.INSTALLED; - } - // Handle the case in which the version to install did not contain the build number, // but the structured output does contain the build number. if (null != structuredOutput.getActualVersion() && !structuredOutput.getActualVersion().isEmpty() && http://git-wip-us.apache.org/repos/asf/ambari/blob/23bf111a/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py b/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py index a5fd9f6..bec1c39 100644 --- a/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py +++ b/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py @@ -88,9 +88,12 @@ class InstallPackages(Script): self.stack_root_folder = self.STACK_TO_ROOT_FOLDER[stack_name] if self.stack_root_folder is None: raise Fail("Cannot determine the stack's root directory by parsing the stack_id property, {0}".format(str(stack_id))) + if self.repository_version is None: + raise Fail("Cannot determine the repository version to install") self.repository_version = self.repository_version.strip() + # Install/update repositories installed_repositories = [] self.current_repositories = [] @@ -130,24 +133,10 @@ class InstallPackages(Script): if num_errors > 0: raise Fail("Failed to distribute repositories/install packages") - # If the repo contains a build number, optimistically assume it to be the actual_version. It will get changed - # to correct value if it is not - self.actual_version = None - if self.repository_version: - m = re.search("[\d\.]+-\d+", self.repository_version) - if m: - # Contains a build number - self.repo_version_with_build_number = self.repository_version - self.structured_output['actual_version'] = self.repo_version_with_build_number # This is the best value known so far. - self.put_structured_out(self.structured_output) - else: - self.repo_version_with_build_number = None - # Initial list of versions, used to compute the new version installed - self.old_versions = get_hdp_versions() + self.old_versions = get_hdp_versions(self.stack_root_folder) try: - # It's possible for the process to receive a SIGTERM while installing the packages ret_code = self.install_packages(package_list) if ret_code == 0: self.structured_output['package_installation_result'] = 'SUCCESS' @@ -169,18 +158,31 @@ class InstallPackages(Script): def compute_actual_version(self): """ - After packages are installed, determine what the new actual version is, in order to save it. + After packages are installed, determine what the new actual version is. """ + + # If the repo contains a build number, optimistically assume it to be the actual_version. It will get changed + # to correct value if it is not + self.actual_version = None + self.repo_version_with_build_number = None + if self.repository_version: + m = re.search("[\d\.]+-\d+", self.repository_version) + if m: + # Contains a build number + self.repo_version_with_build_number = self.repository_version + self.structured_output['actual_version'] = self.repo_version_with_build_number # This is the best value known so far. + self.put_structured_out(self.structured_output) + Logger.info("Attempting to determine actual version with build number.") Logger.info("Old versions: {0}".format(self.old_versions)) - new_versions = get_hdp_versions() + new_versions = get_hdp_versions(self.stack_root_folder) Logger.info("New versions: {0}".format(new_versions)) deltas = set(new_versions) - set(self.old_versions) Logger.info("Deltas: {0}".format(deltas)) - # Get HDP version without build number + # Get version without build number normalized_repo_version = self.repository_version.split('-')[0] if 1 == len(deltas): @@ -188,21 +190,92 @@ class InstallPackages(Script): self.structured_output['actual_version'] = self.actual_version self.put_structured_out(self.structured_output) write_actual_version_to_history_file(normalized_repo_version, self.actual_version) + Logger.info( + "Found actual version {0} by checking the delta between versions before and after installing packages".format( + self.actual_version)) else: - Logger.info("Cannot determine a new actual version installed by using the delta method.") # If the first install attempt does a partial install and is unable to report this to the server, - # then a subsequent attempt will report an empty delta. For this reason, it is important to search the - # repo version history file to determine if we previously did write an actual_version. - self.actual_version = read_actual_version_from_history_file(normalized_repo_version) + # then a subsequent attempt will report an empty delta. For this reason, we search for a best fit version for the repo version + Logger.info("Cannot determine actual version installed by checking the delta between versions " + "before and after installing package") + Logger.info("Will try to find for the actual version by searching for best possible match in the list of versions installed") + self.actual_version = self.find_best_fit_version(new_versions, self.repository_version) if self.actual_version is not None: self.actual_version = self.actual_version.strip() self.structured_output['actual_version'] = self.actual_version self.put_structured_out(self.structured_output) - Logger.info("Found actual version {0} by parsing file {1}".format(self.actual_version, REPO_VERSION_HISTORY_FILE)) - elif self.repo_version_with_build_number is None: + Logger.info("Found actual version {0} by searching for best possible match".format(self.actual_version)) + else: msg = "Could not determine actual version installed. Try reinstalling packages again." raise Fail(msg) + def check_partial_install(self): + """ + If an installation did not complete successfully, check if installation was partially complete and + log the partially completed version to REPO_VERSION_HISTORY_FILE. + :return: + """ + Logger.info("Installation of packages failed. Checking if installation was partially complete") + Logger.info("Old versions: {0}".format(self.old_versions)) + + new_versions = get_hdp_versions(self.stack_root_folder) + Logger.info("New versions: {0}".format(new_versions)) + + deltas = set(new_versions) - set(self.old_versions) + Logger.info("Deltas: {0}".format(deltas)) + + # Get version without build number + normalized_repo_version = self.repository_version.split('-')[0] + + if 1 == len(deltas): + # Some packages were installed successfully. Log this version to REPO_VERSION_HISTORY_FILE + partial_install_version = next(iter(deltas)).strip() + write_actual_version_to_history_file(normalized_repo_version, partial_install_version) + Logger.info("Version {0} was partially installed. ".format(partial_install_version)) + + def find_best_fit_version(self, versions, repo_version): + """ + Given a list of installed versions and a repo version, search for a version that best fits the repo version + If the repo version is found in the list of installed versions, return the repo version itself. + If the repo version is not found in the list of installed versions + normalize the repo version and use the REPO_VERSION_HISTORY_FILE file to search the list. + + :param versions: List of versions installed + :param repo_version: Repo version to search + :return: Matching version, None if no match was found. + """ + if versions is None or repo_version is None: + return None + + build_num_match = re.search("[\d\.]+-\d+", repo_version) + if build_num_match and repo_version in versions: + # If repo version has build number and is found in the list of versions, return it as the matching version + Logger.info("Best Fit Version: Resolved from repo version with valid build number: {0}".format(repo_version)) + return repo_version + + # Get version without build number + normalized_repo_version = repo_version.split('-')[0] + + # Find all versions that match the normalized repo version + match_versions = filter(lambda x: x.startswith(normalized_repo_version), versions) + if match_versions: + + if len(match_versions) == 1: + # Resolved without conflicts + Logger.info("Best Fit Version: Resolved from normalized repo version without conflicts: {0}".format(match_versions[0])) + return match_versions[0] + + # Resolve conflicts using REPO_VERSION_HISTORY_FILE + history_version = read_actual_version_from_history_file(normalized_repo_version) + + # Validate history version retrieved is valid + if history_version in match_versions: + Logger.info("Best Fit Version: Resolved from normalized repo version using {0}: {1}".format(REPO_VERSION_HISTORY_FILE, history_version)) + return history_version + + # No matching version + return None + def install_packages(self, package_list): """ @@ -245,7 +318,10 @@ class InstallPackages(Script): Package(package, action="remove") # Compute the actual version in order to save it in structured out try: - self.compute_actual_version() + if ret_code == 0: + self.compute_actual_version() + else: + self.check_partial_install() except Fail, err: ret_code = 1 Logger.logger.exception("Failure while computing actual version. Error: {0}".format(str(err))) @@ -297,7 +373,7 @@ class InstallPackages(Script): def abort_handler(self, signum, frame): Logger.error("Caught signal {0}, will handle it gracefully. Compute the actual version if possible before exiting.".format(signum)) - self.compute_actual_version() + self.check_partial_install() if __name__ == "__main__": http://git-wip-us.apache.org/repos/asf/ambari/blob/23bf111a/ambari-server/src/test/python/custom_actions/TestInstallPackages.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/custom_actions/TestInstallPackages.py b/ambari-server/src/test/python/custom_actions/TestInstallPackages.py index 5b2a148..83b6bb5 100644 --- a/ambari-server/src/test/python/custom_actions/TestInstallPackages.py +++ b/ambari-server/src/test/python/custom_actions/TestInstallPackages.py @@ -71,7 +71,10 @@ class TestInstallPackages(RMFTestCase): read_actual_version_from_history_file_mock, hdp_versions_mock, put_structured_out_mock, allInstalledPackages_mock, list_ambari_managed_repos_mock): - read_actual_version_from_history_file_mock.return_value = VERSION_STUB + hdp_versions_mock.side_effect = [ + [], # before installation attempt + [VERSION_STUB] + ] allInstalledPackages_mock.side_effect = TestInstallPackages._add_packages list_ambari_managed_repos_mock.return_value=[] self.executeScript("scripts/install_packages.py", @@ -126,7 +129,10 @@ class TestInstallPackages(RMFTestCase): read_actual_version_from_history_file_mock, hdp_versions_mock, put_structured_out_mock, allInstalledPackages_mock, list_ambari_managed_repos_mock, is_suse_family_mock): is_suse_family_mock = True - read_actual_version_from_history_file_mock.return_value = VERSION_STUB + hdp_versions_mock.side_effect = [ + [], # before installation attempt + [VERSION_STUB] + ] allInstalledPackages_mock.side_effect = TestInstallPackages._add_packages list_ambari_managed_repos_mock.return_value=[] self.executeScript("scripts/install_packages.py", @@ -183,7 +189,10 @@ class TestInstallPackages(RMFTestCase): hdp_versions_mock, allInstalledPackages_mock, put_structured_out_mock, is_redhat_family_mock, list_ambari_managed_repos_mock): - read_actual_version_from_history_file_mock.return_value = VERSION_STUB + hdp_versions_mock.side_effect = [ + [], # before installation attempt + [VERSION_STUB] + ] allInstalledPackages_mock.side_effect = TestInstallPackages._add_packages list_ambari_managed_repos_mock.return_value=["HDP-UTILS-2.2.0.1-885"] is_redhat_family_mock.return_value = True @@ -274,7 +283,6 @@ class TestInstallPackages(RMFTestCase): self.assertTrue(put_structured_out_mock.called) self.assertEquals(put_structured_out_mock.call_args[0][0], {'stack_id': 'HDP-2.2', - 'actual_version': VERSION_STUB, 'installed_repository_version': VERSION_STUB, 'ambari_repositories': [], 'package_installation_result': 'FAIL'}) @@ -313,6 +321,10 @@ class TestInstallPackages(RMFTestCase): hdp_versions_mock, allInstalledPackages_mock, put_structured_out_mock, package_mock, is_suse_family_mock): + hdp_versions_mock.side_effect = [ + [], # before installation attempt + [VERSION_STUB] + ] read_actual_version_from_history_file_mock.return_value = VERSION_STUB allInstalledPackages_mock = MagicMock(side_effect = TestInstallPackages._add_packages) is_suse_family_mock.return_value = True @@ -562,17 +574,21 @@ class TestInstallPackages(RMFTestCase): allInstalledPackages_mock.side_effect = TestInstallPackages._add_packages list_ambari_managed_repos_mock.return_value = [] - self.executeScript("scripts/install_packages.py", + try: + self.executeScript("scripts/install_packages.py", classname="InstallPackages", command="actionexecute", config_dict=command_json, target=RMFTestCase.TARGET_CUSTOM_ACTIONS, os_type=('Redhat', '6.4', 'Final'), ) + self.fail("Should throw exception") + except Fail: + pass # Expected self.assertTrue(put_structured_out_mock.called) self.assertEquals(put_structured_out_mock.call_args[0][0], - {'package_installation_result': 'SUCCESS', + {'package_installation_result': 'FAIL', 'stack_id': u'HDP-2.2', 'installed_repository_version': VERSION_STUB, 'actual_version': VERSION_STUB, @@ -806,17 +822,21 @@ class TestInstallPackages(RMFTestCase): allInstalledPackages_mock.side_effect = TestInstallPackages._add_packages list_ambari_managed_repos_mock.return_value = [] - self.executeScript("scripts/install_packages.py", + try: + self.executeScript("scripts/install_packages.py", classname="InstallPackages", command="actionexecute", config_dict=command_json, target=RMFTestCase.TARGET_CUSTOM_ACTIONS, os_type=('Redhat', '6.4', 'Final'), ) + self.fail("Should throw exception") + except Fail: + pass # Expected self.assertTrue(put_structured_out_mock.called) self.assertEquals(put_structured_out_mock.call_args[0][0], - {'package_installation_result': 'SUCCESS', + {'package_installation_result': 'FAIL', 'stack_id': u'HDP-2.2', 'installed_repository_version': VERSION_STUB, 'actual_version': VERSION_STUB,
