AMBARI-21853 - Packages Already Installed Cause Problems When Scoping By New Repo ID Schema (jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/96037358 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/96037358 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/96037358 Branch: refs/heads/feature-branch-AMBARI-21307 Commit: 960373587b8ace28277ef15edcdda72d9b62f71f Parents: d22a671 Author: Jonathan Hurley <jhur...@hortonworks.com> Authored: Wed Aug 30 10:14:57 2017 -0400 Committer: Jonathan Hurley <jhur...@hortonworks.com> Committed: Wed Aug 30 10:21:36 2017 -0400 ---------------------------------------------------------------------- .../core/providers/package/yumrpm.py | 41 +++++++++++++++++++- .../libraries/script/script.py | 1 + 2 files changed, 41 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/96037358/ambari-common/src/main/python/resource_management/core/providers/package/yumrpm.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/providers/package/yumrpm.py b/ambari-common/src/main/python/resource_management/core/providers/package/yumrpm.py index 8c02992..78c77ca 100644 --- a/ambari-common/src/main/python/resource_management/core/providers/package/yumrpm.py +++ b/ambari-common/src/main/python/resource_management/core/providers/package/yumrpm.py @@ -27,9 +27,12 @@ from resource_management.core.shell import string_cmd_from_args_list from resource_management.core.logger import Logger from resource_management.core.utils import suppress_stdout +import glob import re import os +import ConfigParser + INSTALL_CMD = { True: ['/usr/bin/yum', '-y', 'install'], False: ['/usr/bin/yum', '-d', '0', '-e', '0', '-y', 'install'], @@ -58,6 +61,7 @@ VERIFY_DEPENDENCY_CMD = ['/usr/bin/yum', '-d', '0', '-e', '0', 'check', 'depende LIST_ALL_SELECT_TOOL_PACKAGES_CMD = "yum list all --showduplicates|grep -v '@' |grep '^{pkg_name}'|awk '{print $2}'" SELECT_TOOL_VERSION_PATTERN = re.compile("(\d{1,2}\.\d{1,2}\.\d{1,2}\.\d{1,2}-*\d*).*") # xx.xx.xx.xx(-xxxx) +YUM_REPO_LOCATION = "/etc/yum.repos.d" class YumProvider(RPMBasedPackageProvider): @@ -70,7 +74,9 @@ class YumProvider(RPMBasedPackageProvider): available_packages = [] installed_packages = [] available_packages_in_repos = [] - repo_ids = [repository['repoId'] for repository in repositories] + + repo_ids = self._build_repos_ids(repositories) + Logger.info("Looking for matching packages in the following repositories: {0}".format(", ".join(repo_ids))) for repo in repo_ids: available_packages.extend(self._lookup_packages( @@ -283,3 +289,36 @@ class YumProvider(RPMBasedPackageProvider): return REPO_UPDATE_CMD + + @staticmethod + def _build_repos_ids(repositories): + """ + Gets a set of repository identifiers based on the supplied repository JSON structure as + well as any matching repos defined in /etc/yum.repos.d. + :param repositories: the repositories defined on the command + :return: the list of repo IDs from both the command and any matches found on the system + with the same URLs. + """ + repo_ids = [repository['repoId'] for repository in repositories] + base_urls = [repository['baseUrl'] for repository in repositories if 'baseUrl' in repository] + mirrors = [repository['mirrorsList'] for repository in repositories if 'mirrorsList' in repository] + + # for every repo file, find any which match the base URLs we're trying to write out + # if there are any matches, it means the repo already exists and we should use it to search + # for packages to install + for repo_file in glob.glob(os.path.join(YUM_REPO_LOCATION, "*.repo")): + config_parser = ConfigParser.ConfigParser() + config_parser.read(repo_file) + sections = config_parser.sections() + for section in sections: + if config_parser.has_option(section, "baseurl"): + base_url = config_parser.get(section, "baseurl") + if base_url in base_urls: + repo_ids.append(section) + + if config_parser.has_option(section, "mirrorlist"): + mirror = config_parser.get(section, "mirrorlist") + if mirror in mirrors: + repo_ids.append(section) + + return set(repo_ids) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/96037358/ambari-common/src/main/python/resource_management/libraries/script/script.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py b/ambari-common/src/main/python/resource_management/libraries/script/script.py index 57b6f92..fcfac2b 100644 --- a/ambari-common/src/main/python/resource_management/libraries/script/script.py +++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py @@ -745,6 +745,7 @@ class Script(object): try: available_packages_in_repos = pkg_provider.get_available_packages_in_repos(config['repositoryFile']['repositories']) except Exception as err: + Logger.exception("Unable to load available packages") available_packages_in_repos = [] if isinstance(package_list_str, basestring) and len(package_list_str) > 0: package_list = json.loads(package_list_str)
