AMBARI-22563. Packages Cannot Be Installed When Yum Transactions Fail (Dmytro Grinenko via ncole)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/959ad900 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/959ad900 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/959ad900 Branch: refs/heads/branch-feature-AMBARI-21674 Commit: 959ad900e40c0d04c57145cd84414983175a629a Parents: 158c94a Author: Nate Cole <[email protected]> Authored: Fri Dec 1 16:39:01 2017 -0500 Committer: Nate Cole <[email protected]> Committed: Fri Dec 1 16:39:01 2017 -0500 ---------------------------------------------------------------------- .../core/providers/package/__init__.py | 15 +++ .../core/providers/package/yumrpm.py | 108 +++++++++++++++++++ .../custom_actions/scripts/install_packages.py | 12 ++- 3 files changed, 133 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/959ad900/ambari-common/src/main/python/resource_management/core/providers/package/__init__.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/providers/package/__init__.py b/ambari-common/src/main/python/resource_management/core/providers/package/__init__.py index fc695a7..f2a375f 100644 --- a/ambari-common/src/main/python/resource_management/core/providers/package/__init__.py +++ b/ambari-common/src/main/python/resource_management/core/providers/package/__init__.py @@ -66,6 +66,21 @@ class PackageProvider(Provider): else: return self.resource.package_name + def check_uncompleted_transactions(self): + """ + Check package manager against uncompleted transactions. + + :rtype bool + """ + return False + + def print_uncompleted_transaction_hint(self): + """ + Print friendly message about they way to fix the issue + + """ + pass + def get_available_packages_in_repos(self, repositories): """ Gets all (both installed and available) packages that are available at given repositories. http://git-wip-us.apache.org/repos/asf/ambari/blob/959ad900/ambari-common/src/main/python/resource_management/core/providers/package/yumrpm.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/providers/package/yumrpm.py b/ambari-common/src/main/python/resource_management/core/providers/package/yumrpm.py index 367e2af..c83a3ce 100644 --- a/ambari-common/src/main/python/resource_management/core/providers/package/yumrpm.py +++ b/ambari-common/src/main/python/resource_management/core/providers/package/yumrpm.py @@ -28,6 +28,9 @@ from resource_management.core import shell from resource_management.core.shell import string_cmd_from_args_list from resource_management.core.logger import Logger from resource_management.core.utils import suppress_stdout +from resource_management.core import sudo + +from StringIO import StringIO import re import os @@ -44,6 +47,9 @@ REMOVE_CMD = { REMOVE_WITHOUT_DEPENDENCIES_CMD = ['rpm', '-e', '--nodeps'] +YUM_LIB_DIR = "/var/lib/yum" +YUM_TR_PREFIX = "transaction-" + YUM_REPO_LOCATION = "/etc/yum.repos.d" REPO_UPDATE_CMD = ['/usr/bin/yum', 'clean', 'metadata'] ALL_INSTALLED_PACKAGES_CMD = [AMBARI_SUDO_BINARY, "yum", "list", "installed", "--noplugins"] @@ -359,3 +365,105 @@ class YumProvider(RPMBasedPackageProvider): repo_ids.append(section) return set(repo_ids) + + def __extract_transaction_id(self, filename): + """ + :type filename str + """ + return filename.split(".", 1)[1] + + def __transaction_file_parser(self, f): + """ + :type f file|BinaryIO|StringIO + :rtype collections.Iterable(str) + """ + for line in f: + yield line.split(":", 1)[1].strip() + + def uncomplete_transactions(self): + """ + Transactions reader + + :rtype collections.Iterable(YumTransactionItem) + """ + transactions = {} + + prefix_len = len(YUM_TR_PREFIX) + for item in sudo.listdir(YUM_LIB_DIR): + if YUM_TR_PREFIX == item[:prefix_len]: + tr_id = self.__extract_transaction_id(item) + + f = StringIO(sudo.read_file(os.path.join(YUM_LIB_DIR, item))) + pkgs_in_transaction = list(self.__transaction_file_parser(f)) + + if tr_id not in transactions: + transactions[tr_id] = YumTransactionItem(tr_id) + + if RPMTransactions.all in item: + transactions[tr_id].pkgs_all = pkgs_in_transaction + elif RPMTransactions.done in item: + transactions[tr_id].pkgs_done = pkgs_in_transaction + + for tr in transactions.values(): + if len(tr.pkgs_all) == 0: + continue + + if isinstance(tr, YumTransactionItem): + yield tr + + def check_uncompleted_transactions(self): + """ + Check package manager against uncompleted transactions. + + :rtype bool + """ + + transactions = list(self.uncomplete_transactions()) + + if len(transactions) > 0: + Logger.info("Yum non-completed transactions check failed, found {0} non-completed transaction(s):".format(len(transactions))) + for tr in transactions: + Logger.info("[{0}] Packages broken: {1}; Packages not-installed {2}".format( + tr.transaction_id, + ", ".join(tr.pkgs_done), + ", ".join(tr.pkgs_aborted) + )) + + return True + + Logger.info("Yum non-completed transactions check passed") + return False + + def print_uncompleted_transaction_hint(self): + """ + Print friendly message about they way to fix the issue + + """ + help_msg = """*** Incomplete Yum Transactions *** + +Ambari has detected that there are incomplete Yum transactions on this host. This will interfere with the installation process and must be resolved before continuing. + +- Identify the pending transactions with the command 'yum history list <packages failed>' +- Revert each pending transaction with the command 'yum history undo' +- Flush the transaction log with 'yum-complete-transaction --cleanup-only' +""" + + for line in help_msg.split("\n"): + Logger.error(line) + + +class YumTransactionItem(object): + def __init__(self, transaction_id, pkgs_done=None, pkgs_all=None): + self.transaction_id = transaction_id + self.pkgs_done = pkgs_done if pkgs_done else [] + self.pkgs_all = pkgs_all if pkgs_all else [] + + @property + def pkgs_aborted(self): + return set(self.pkgs_all) ^ set(self.pkgs_done) + + +class RPMTransactions(object): + all = "all" + done = "done" + aborted = "aborted" # custom one http://git-wip-us.apache.org/repos/asf/ambari/blob/959ad900/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py b/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py index fff18bb..862f205 100644 --- a/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py +++ b/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py @@ -112,7 +112,7 @@ class InstallPackages(Script): "Will install packages for repository version {0}".format(self.repository_version)) new_repo_files = create_repo_files(template, command_repository) self.repo_files.update(new_repo_files) - except Exception, err: + except Exception as err: Logger.logger.exception("Cannot install repository files. Error: {0}".format(str(err))) num_errors += 1 @@ -124,6 +124,14 @@ class InstallPackages(Script): self.put_structured_out(self.structured_output) + try: + # check package manager non-completed transactions + if self.pkg_provider.check_uncompleted_transactions(): + self.pkg_provider.print_uncompleted_transaction_hint() + num_errors += 1 + except Exception as e: # we need to ignore any exception + Logger.warning("Failed to check for uncompleted package manager transactions: " + str(e)) + if num_errors > 0: raise Fail("Failed to distribute repositories/install packages") @@ -139,7 +147,7 @@ class InstallPackages(Script): is_package_install_successful = True else: num_errors += 1 - except Exception, err: + except Exception as err: num_errors += 1 Logger.logger.exception("Could not install packages. Error: {0}".format(str(err)))
