http://git-wip-us.apache.org/repos/asf/ambari/blob/33879743/ambari-common/src/main/python/resource_management/libraries/script/script.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py b/ambari-common/src/main/python/resource_management/libraries/script/script.py index 54a7e6d..313b666 100644 --- a/ambari-common/src/main/python/resource_management/libraries/script/script.py +++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py @@ -51,13 +51,13 @@ from resource_management.libraries.functions.version import compare_versions from resource_management.libraries.functions.version import format_stack_version from resource_management.libraries.functions import stack_tools from resource_management.libraries.functions.constants import Direction -from resource_management.libraries.functions import packages_analyzer from resource_management.libraries.script.config_dictionary import ConfigDictionary, UnknownConfiguration from resource_management.core.resources.system import Execute from contextlib import closing from resource_management.libraries.functions.stack_features import check_stack_feature from resource_management.libraries.functions.constants import StackFeature from resource_management.libraries.functions.show_logs import show_logs +from resource_management.core.providers import get_provider from resource_management.libraries.functions.fcntl_based_process_lock import FcntlBasedProcessLock import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set. @@ -435,7 +435,9 @@ class Script(object): if not Script.stack_version_from_distro_select or '*' in Script.stack_version_from_distro_select: # FIXME: this method is not reliable to get stack-selector-version # as if there are multiple versions installed with different <stack-selector-tool>, we won't detect the older one (if needed). - Script.stack_version_from_distro_select = packages_analyzer.getInstalledPackageVersion( + pkg_provider = get_provider("Package") + + Script.stack_version_from_distro_select = pkg_provider.get_installed_package_version( stack_tools.get_stack_tool_package(stack_tools.STACK_SELECTOR_NAME)) return Script.stack_version_from_distro_select @@ -456,12 +458,19 @@ class Script(object): return package - def format_package_name(self, name): + def format_package_name(self, name, repo_version=None): from resource_management.libraries.functions.default import default """ This function replaces ${stack_version} placeholder with actual version. If the package version is passed from the server, use that as an absolute truth. + + :param name name of the package + :param repo_version actual version of the repo currently installing """ + stack_version_package_formatted = "" + + if not repo_version: + repo_version = self.get_stack_version_before_packages_installed() package_delimiter = '-' if OSCheck.is_ubuntu_family() else '_' @@ -497,7 +506,7 @@ class Script(object): # Wildcards cause a lot of troubles with installing packages, if the version contains wildcards we try to specify it. if not package_version or '*' in package_version: - stack_version_package_formatted = self.get_stack_version_before_packages_installed().replace('.', package_delimiter).replace('-', package_delimiter) if STACK_VERSION_PLACEHOLDER in name else name + stack_version_package_formatted = repo_version.replace('.', package_delimiter).replace('-', package_delimiter) if STACK_VERSION_PLACEHOLDER in name else name package_name = name.replace(STACK_VERSION_PLACEHOLDER, stack_version_package_formatted) @@ -703,7 +712,7 @@ class Script(object): if 'host_sys_prepped' in config['hostLevelParams']: # do not install anything on sys-prepped host - if config['hostLevelParams']['host_sys_prepped'] == True: + if config['hostLevelParams']['host_sys_prepped'] is True: Logger.info("Node has all packages pre-installed. Skipping.") return pass @@ -711,8 +720,9 @@ class Script(object): package_list_str = config['hostLevelParams']['package_list'] agent_stack_retry_on_unavailability = bool(config['hostLevelParams']['agent_stack_retry_on_unavailability']) agent_stack_retry_count = int(config['hostLevelParams']['agent_stack_retry_count']) + pkg_provider = get_provider("Package") try: - available_packages_in_repos = packages_analyzer.get_available_packages_in_repos(config['repositoryFile']['repositories']) + available_packages_in_repos = pkg_provider.get_available_packages_in_repos(config['repositoryFile']['repositories']) except Exception as err: available_packages_in_repos = [] if isinstance(package_list_str, basestring) and len(package_list_str) > 0:
http://git-wip-us.apache.org/repos/asf/ambari/blob/33879743/ambari-server/src/main/resources/custom_actions/scripts/check_host.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/custom_actions/scripts/check_host.py b/ambari-server/src/main/resources/custom_actions/scripts/check_host.py index 358c104..a485415 100644 --- a/ambari-server/src/main/resources/custom_actions/scripts/check_host.py +++ b/ambari-server/src/main/resources/custom_actions/scripts/check_host.py @@ -27,7 +27,6 @@ import socket import getpass import tempfile -from resource_management.libraries.functions import packages_analyzer from resource_management.libraries.functions.default import default from ambari_commons import os_utils from ambari_commons.os_check import OSCheck, OSConst @@ -40,6 +39,7 @@ from resource_management.core.exceptions import Fail from ambari_commons.constants import AMBARI_SUDO_BINARY from resource_management.core import shell from resource_management.core.logger import Logger +from resource_management.core.providers import get_provider # WARNING. If you are adding a new host check that is used by cleanup, add it to BEFORE_CLEANUP_HOST_CHECKS # It is used by HostCleanup.py @@ -113,6 +113,7 @@ class CheckHost(Script): def __init__(self): self.reportFileHandler = HostCheckReportFileHandler() + self.pkg_provider = get_provider("Package") def actionexecute(self, env): Logger.info("Host checks started.") @@ -229,21 +230,18 @@ class CheckHost(Script): def execute_existing_repos_and_installed_packages_check(self, config): Logger.info("Installed packages and existing repos checks started.") - installedPackages = [] - availablePackages = [] - packages_analyzer.allInstalledPackages(installedPackages) - packages_analyzer.allAvailablePackages(availablePackages) - - repos = [] - packages_analyzer.getInstalledRepos(self.PACKAGES, installedPackages + availablePackages, - self.IGNORE_PACKAGES_FROM_REPOS, repos) - packagesInstalled = packages_analyzer.getInstalledPkgsByRepo(repos, self.IGNORE_PACKAGES, installedPackages) - additionalPkgsInstalled = packages_analyzer.getInstalledPkgsByNames( - self.ADDITIONAL_PACKAGES, installedPackages) + installedPackages = self.pkg_provider.all_installed_packages() + availablePackages = self.pkg_provider.all_available_packages() + + repos = self.pkg_provider.get_installed_repos(self.PACKAGES, installedPackages + availablePackages, + self.IGNORE_PACKAGES_FROM_REPOS) + + packagesInstalled = self.pkg_provider.get_installed_pkgs_by_repo(repos, self.IGNORE_PACKAGES, installedPackages) + additionalPkgsInstalled = self.pkg_provider.get_installed_pkgs_by_names(self.ADDITIONAL_PACKAGES, installedPackages) allPackages = list(set(packagesInstalled + additionalPkgsInstalled)) - installedPackages = packages_analyzer.getPackageDetails(installedPackages, allPackages) - repos = packages_analyzer.getReposToRemove(repos, self.IGNORE_REPOS) + installedPackages = self.pkg_provider.get_package_details(installedPackages, allPackages) + repos = self.pkg_provider.get_repos_to_remove(repos, self.IGNORE_REPOS) Logger.info("Installed packages and existing repos checks completed.") return installedPackages, repos http://git-wip-us.apache.org/repos/asf/ambari/blob/33879743/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py b/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py index a86a71d..189043b 100644 --- a/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py +++ b/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py @@ -28,20 +28,21 @@ from ambari_commons.str_utils import cbool, cint from resource_management.core.exceptions import Fail from resource_management.core.logger import Logger from resource_management.core.resources import Package -from resource_management.libraries.functions.packages_analyzer import allInstalledPackages, verifyDependencies from resource_management.libraries.functions import conf_select from resource_management.libraries.functions import stack_tools from resource_management.libraries.functions.stack_select import get_stack_versions from resource_management.libraries.functions.version import format_stack_version from resource_management.libraries.functions.repo_version_history \ import read_actual_version_from_history_file, write_actual_version_to_history_file, REPO_VERSION_HISTORY_FILE +from resource_management.core.providers import get_provider from resource_management.core.resources.system import Link from resource_management.libraries.functions import StackFeature -from resource_management.libraries.functions import packages_analyzer from resource_management.libraries.functions.repository_util import create_repo_files, CommandRepository from resource_management.libraries.functions.stack_features import check_stack_feature from resource_management.libraries.resources.repository import Repository from resource_management.libraries.script.script import Script +from resource_management.core import sudo + class InstallPackages(Script): """ @@ -53,6 +54,11 @@ class InstallPackages(Script): UBUNTU_REPO_COMPONENTS_POSTFIX = ["main"] + def __init__(self): + super(InstallPackages, self).__init__() + + self.pkg_provider = get_provider("Package") + def actionexecute(self, env): num_errors = 0 @@ -83,8 +89,6 @@ class InstallPackages(Script): except KeyError: pass - - self.stack_name = Script.get_stack_name() if self.stack_name is None: raise Fail("Cannot determine the stack name") @@ -99,7 +103,7 @@ class InstallPackages(Script): self.repository_version = self.repository_version.strip() try: - if 0 == len(command_repository.repositories): + if not command_repository.repositories: Logger.warning( "Repository list is empty. Ambari may not be managing the repositories for {0}.".format( self.repository_version)) @@ -171,13 +175,12 @@ class InstallPackages(Script): # After upgrading hdf-select package from HDF-2.X to HDF-3.Y, we need to create this symlink if self.stack_name.upper() == "HDF" \ - and not os.path.exists("/usr/bin/conf-select") and os.path.exists("/usr/bin/hdfconf-select"): - Link("/usr/bin/conf-select", to = "/usr/bin/hdfconf-select") + and not sudo.path_exists("/usr/bin/conf-select") and sudo.path_exists("/usr/bin/hdfconf-select"): + Link("/usr/bin/conf-select", to="/usr/bin/hdfconf-select") for package_name, directories in conf_select.get_package_dirs().iteritems(): conf_select.select(self.stack_name, package_name, stack_version, ignore_errors = True) - def compute_actual_version(self): """ After packages are installed, determine what the new actual version is. @@ -313,6 +316,7 @@ class InstallPackages(Script): # Install packages packages_were_checked = False + packages_installed_before = [] stack_selector_package = stack_tools.get_stack_tool_package(stack_tools.STACK_SELECTOR_NAME) try: Package(stack_selector_package, @@ -321,13 +325,12 @@ class InstallPackages(Script): retry_count=agent_stack_retry_count ) - packages_installed_before = [] - allInstalledPackages(packages_installed_before) + packages_installed_before = self.pkg_provider.all_installed_packages() packages_installed_before = [package[0] for package in packages_installed_before] packages_were_checked = True filtered_package_list = self.filter_package_list(package_list) try: - available_packages_in_repos = packages_analyzer.get_available_packages_in_repos(config['repositoryFile']['repositories']) + available_packages_in_repos = self.pkg_provider.get_available_packages_in_repos(config['repositoryFile']['repositories']) except Exception: available_packages_in_repos = [] for package in filtered_package_list: @@ -343,8 +346,7 @@ class InstallPackages(Script): # Remove already installed packages in case of fail if packages_were_checked and packages_installed_before: - packages_installed_after = [] - allInstalledPackages(packages_installed_after) + packages_installed_after = self.pkg_provider.all_installed_packages() packages_installed_after = [package[0] for package in packages_installed_after] packages_installed_before = set(packages_installed_before) new_packages_installed = [package for package in packages_installed_after if package not in packages_installed_before] @@ -361,7 +363,7 @@ class InstallPackages(Script): if self.ignore_package_dependencies: Logger.info("Ignoring package dependencies") - elif not verifyDependencies(): + elif not self.pkg_provider.verify_dependencies(): ret_code = 1 Logger.logger.error("Failure while verifying dependencies") Logger.logger.error("*******************************************************************************") @@ -379,36 +381,6 @@ class InstallPackages(Script): Logger.logger.exception("Failure while computing actual version. Error: {0}".format(str(err))) return ret_code - def install_repository(self, url_info, append_to_file, template): - - repo = { - 'repoName': "{0}-{1}".format(url_info['name'], self.repository_version) - } - - if not 'baseUrl' in url_info: - repo['baseurl'] = None - else: - repo['baseurl'] = url_info['baseUrl'] - - if not 'mirrorsList' in url_info: - repo['mirrorsList'] = None - else: - repo['mirrorsList'] = url_info['mirrorsList'] - - ubuntu_components = [url_info['name']] + self.UBUNTU_REPO_COMPONENTS_POSTFIX - file_name = self.stack_name + "-" + self.repository_version - - Repository(repo['repoName'], - action = "create", - base_url = repo['baseurl'], - mirror_list = repo['mirrorsList'], - repo_file_name = file_name, - repo_template = template, - append_to_file = append_to_file, - components = ubuntu_components, # ubuntu specific - ) - return repo['repoName'], file_name - def abort_handler(self, signum, frame): Logger.error("Caught signal {0}, will handle it gracefully. Compute the actual version if possible before exiting.".format(signum)) self.check_partial_install() http://git-wip-us.apache.org/repos/asf/ambari/blob/33879743/ambari-server/src/main/resources/custom_actions/scripts/remove_previous_stacks.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/custom_actions/scripts/remove_previous_stacks.py b/ambari-server/src/main/resources/custom_actions/scripts/remove_previous_stacks.py index 958b800..2b50171 100644 --- a/ambari-server/src/main/resources/custom_actions/scripts/remove_previous_stacks.py +++ b/ambari-server/src/main/resources/custom_actions/scripts/remove_previous_stacks.py @@ -25,8 +25,8 @@ import os from resource_management import Script, format, Package, Execute, Fail from resource_management.core.logger import Logger from resource_management.libraries.functions import stack_tools -from resource_management.libraries.functions.packages_analyzer import allInstalledPackages from resource_management.libraries.functions.stack_select import get_stack_versions +from resource_management.core.providers import get_provider CURRENT_ = "/current/" stack_root = Script.get_stack_root() @@ -43,6 +43,7 @@ class RemovePreviousStacks(Script): self.stack_tool_package = stack_tools.get_stack_tool_package(stack_tools.STACK_SELECTOR_NAME) versions_to_remove = self.get_lower_versions(version) + self.pkg_provider = get_provider("Package") for low_version in versions_to_remove: self.remove_stack_version(structured_output, low_version) @@ -73,8 +74,8 @@ class RemovePreviousStacks(Script): def get_packages_to_remove(self, version): packages = [] formated_version = version.replace('.', '_').replace('-', '_') - all_installed_packages = [] - allInstalledPackages(all_installed_packages) + all_installed_packages = self.pkg_provider.all_installed_packages() + all_installed_packages = [package[0] for package in all_installed_packages] for package in all_installed_packages: if formated_version in package and self.stack_tool_package not in package:
