Repository: ambari Updated Branches: refs/heads/trunk 5bdede509 -> 467e34487
AMBARI-22291 Adapt Repository Files For Existing Deployments for trunk (dgrinenko) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/467e3448 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/467e3448 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/467e3448 Branch: refs/heads/trunk Commit: 467e34487f56cc8ad700a11e1a4d5a0385b2be6a Parents: 5bdede5 Author: Dmytro Grinenko <hapyles...@apache.org> Authored: Mon Oct 23 21:55:29 2017 +0300 Committer: Dmytro Grinenko <hapyles...@apache.org> Committed: Mon Oct 23 21:55:29 2017 +0300 ---------------------------------------------------------------------- .../core/providers/package/__init__.py | 2 +- .../core/providers/package/apt.py | 23 ++++-- .../core/providers/package/yumrpm.py | 79 +++++++++++++----- .../core/providers/package/zypper.py | 37 +++++++-- .../libraries/functions/repository_util.py | 79 ++++++++++-------- .../libraries/script/script.py | 4 +- .../ambari/server/agent/CommandRepository.java | 85 ++++++++++++++++++++ .../ambari/server/agent/ExecutionCommand.java | 1 + .../server/configuration/Configuration.java | 13 +++ .../controller/AmbariActionExecutionHelper.java | 1 - .../AmbariCustomCommandExecutionHelper.java | 17 +++- .../orm/entities/RepositoryVersionEntity.java | 27 +++++++ .../stack/upgrade/RepositoryVersionHelper.java | 21 ++++- .../server/upgrade/UpgradeCatalog260.java | 13 +++ .../main/resources/Ambari-DDL-Derby-CREATE.sql | 1 + .../main/resources/Ambari-DDL-MySQL-CREATE.sql | 1 + .../main/resources/Ambari-DDL-Oracle-CREATE.sql | 1 + .../resources/Ambari-DDL-Postgres-CREATE.sql | 1 + .../resources/Ambari-DDL-SQLAnywhere-CREATE.sql | 1 + .../resources/Ambari-DDL-SQLServer-CREATE.sql | 1 + .../custom_actions/scripts/install_packages.py | 5 +- ...ClusterStackVersionResourceProviderTest.java | 2 + .../configs/install_packages_config.json | 1 + .../install_packages_repository_file.json | 1 + 24 files changed, 337 insertions(+), 80 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/467e3448/ambari-common/src/main/python/resource_management/core/providers/package/__init__.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/providers/package/__init__.py b/ambari-common/src/main/python/resource_management/core/providers/package/__init__.py index 8728b5e..fc695a7 100644 --- a/ambari-common/src/main/python/resource_management/core/providers/package/__init__.py +++ b/ambari-common/src/main/python/resource_management/core/providers/package/__init__.py @@ -69,7 +69,7 @@ class PackageProvider(Provider): def get_available_packages_in_repos(self, repositories): """ Gets all (both installed and available) packages that are available at given repositories. - :param repositories: from command configs like config['repositoryFile']['repositories'] + :type repositories resource_management.libraries.functions.repository_util.CommandRepository :return: installed and available packages from these repositories """ raise NotImplementedError() http://git-wip-us.apache.org/repos/asf/ambari/blob/467e3448/ambari-common/src/main/python/resource_management/core/providers/package/apt.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/providers/package/apt.py b/ambari-common/src/main/python/resource_management/core/providers/package/apt.py index e236697..12631fa 100644 --- a/ambari-common/src/main/python/resource_management/core/providers/package/apt.py +++ b/ambari-common/src/main/python/resource_management/core/providers/package/apt.py @@ -274,24 +274,31 @@ class AptProvider(PackageProvider): return packages - def get_available_packages_in_repos(self, repositories): + def get_available_packages_in_repos(self, repos): """ Gets all (both installed and available) packages that are available at given repositories. - :param repositories: from command configs like config['repositoryFile']['repositories'] + :type repos resource_management.libraries.functions.repository_util.CommandRepository :return: installed and available packages from these repositories """ filtered_packages = [] packages = self.all_available_packages() + repo_ids = [] - for repo in repositories: - repo_url_part = repo['baseUrl'].replace("http://", "").replace("/", "_") + for repo in repos.items: + repo_ids.append(repo.base_url.replace("http://", "").replace("/", "_")) - for package in packages: - if repo_url_part in package[2]: - filtered_packages.append(package[0]) + if repos.feat.scoped: + Logger.info("Looking for matching packages in the following repositories: {0}".format(", ".join(repo_ids))) + for repo_id in repo_ids: + for package in packages: + if repo_id in package[2]: + filtered_packages.append(package[0]) - return filtered_packages + return filtered_packages + else: + Logger.info("Packages will be queried using all available repositories on the system.") + return [package[0] for package in packages] def get_all_package_versions(self, pkg_name): """ http://git-wip-us.apache.org/repos/asf/ambari/blob/467e3448/ambari-common/src/main/python/resource_management/core/providers/package/yumrpm.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/providers/package/yumrpm.py b/ambari-common/src/main/python/resource_management/core/providers/package/yumrpm.py index b9b6792..9117510 100644 --- a/ambari-common/src/main/python/resource_management/core/providers/package/yumrpm.py +++ b/ambari-common/src/main/python/resource_management/core/providers/package/yumrpm.py @@ -19,6 +19,8 @@ limitations under the License. Ambari Agent """ +import ConfigParser +import glob from ambari_commons.constants import AMBARI_SUDO_BINARY from resource_management.core.providers.package import RPMBasedPackageProvider @@ -27,12 +29,9 @@ from resource_management.core.shell import string_cmd_from_args_list from resource_management.core.logger import Logger from resource_management.core.utils import suppress_stdout -import glob import re import os -import ConfigParser - INSTALL_CMD = { True: ['/usr/bin/yum', '-y', 'install'], False: ['/usr/bin/yum', '-d', '0', '-e', '0', '-y', 'install'], @@ -45,6 +44,7 @@ REMOVE_CMD = { REMOVE_WITHOUT_DEPENDENCIES_CMD = ['rpm', '-e', '--nodeps'] +YUM_REPO_LOCATION = "/etc/yum.repos.d" REPO_UPDATE_CMD = ['/usr/bin/yum', 'clean', 'metadata'] ALL_INSTALLED_PACKAGES_CMD = [AMBARI_SUDO_BINARY, "yum", "list", "installed"] ALL_AVAILABLE_PACKAGES_CMD = [AMBARI_SUDO_BINARY, "yum", "list", "available"] @@ -61,30 +61,42 @@ VERIFY_DEPENDENCY_CMD = ['/usr/bin/yum', '-d', '0', '-e', '0', 'check', 'depende LIST_ALL_SELECT_TOOL_PACKAGES_CMD = "yum list all --showduplicates|grep -v '@' |grep '^{pkg_name}'|awk '{print $2}'" SELECT_TOOL_VERSION_PATTERN = re.compile("(\d{1,2}\.\d{1,2}\.\d{1,2}\.\d{1,2}-*\d*).*") # xx.xx.xx.xx(-xxxx) -YUM_REPO_LOCATION = "/etc/yum.repos.d" class YumProvider(RPMBasedPackageProvider): - def get_available_packages_in_repos(self, repositories): + def get_available_packages_in_repos(self, repos): """ Gets all (both installed and available) packages that are available at given repositories. - :param repositories: from command configs like config['repositoryFile']['repositories'] + + :type repos resource_management.libraries.functions.repository_util.CommandRepository :return: installed and available packages from these repositories """ available_packages = [] installed_packages = [] - available_packages_in_repos = [] + repo_ids = [repo.repo_id for repo in repos.items] - repo_ids = self._build_repos_ids(repositories) - Logger.info("Looking for matching packages in the following repositories: {0}".format(", ".join(repo_ids))) + if repos.feat.scoped: + Logger.info("Looking for matching packages in the following repositories: {0}".format(", ".join(repo_ids))) + else: + Logger.info("Packages will be queried using all available repositories on the system.") for repo in repo_ids: - available_packages.extend(self._lookup_packages( - [AMBARI_SUDO_BINARY, "yum", "list", "available", "--disablerepo=*", "--enablerepo=" + repo], 'Available Packages')) + repo = repo if repos.feat.scoped else None + available_packages.extend(self._get_available_packages(repo)) installed_packages.extend(self._get_installed_packages(repo)) - available_packages_in_repos += [package[0] for package in available_packages + installed_packages] - return available_packages_in_repos + # fallback logic + + if repos.feat.scoped: + fallback_repo_ids = set(repo_ids) ^ self._build_repos_ids(repos) # no reason to scan the same repos again + if fallback_repo_ids: + Logger.info("Adding fallback repositories: {0}".format(", ".join(fallback_repo_ids))) + + for repo in fallback_repo_ids: + available_packages.extend(self._get_available_packages(repo)) + installed_packages.extend(self._get_installed_packages(repo)) + + return [package[0] for package in available_packages + installed_packages] def get_all_package_versions(self, pkg_name): """ @@ -117,6 +129,22 @@ class YumProvider(RPMBasedPackageProvider): return [self.__parse_select_tool_version(i) for i in versions] + def _get_available_packages(self, repo_filter=None): + """ + Returning list of available packages with possibility to filter them by name + :param repo_filter: repository name + + :type repo_filter str|None + :rtype list[list,] + """ + + cmd = [AMBARI_SUDO_BINARY, "yum", "list", "available"] + + if repo_filter: + cmd.extend(["--disablerepo=*", "--enablerepo=" + repo_filter]) + + return self._lookup_packages(cmd, 'Available Packages') + def _get_installed_packages(self, repo_filter=None): """ Returning list of the installed packages with possibility to filter them by name @@ -179,7 +207,7 @@ class YumProvider(RPMBasedPackageProvider): :rtype list|dict """ # ToDo: move to iterative package lookup (check apt provider for details) - return self._lookup_packages(ALL_AVAILABLE_PACKAGES_CMD, "Available Packages") + return self._get_available_packages(None) def all_installed_packages(self, from_unknown_repo=False): """ @@ -191,7 +219,7 @@ class YumProvider(RPMBasedPackageProvider): :return result_type formatted list of packages """ # ToDo: move to iterative package lookup (check apt provider for details) - return self._lookup_packages(ALL_INSTALLED_PACKAGES_CMD, "Installed Packages") + return self._get_installed_packages(None) def verify_dependencies(self): """ @@ -289,19 +317,28 @@ class YumProvider(RPMBasedPackageProvider): return REPO_UPDATE_CMD - @staticmethod - def _build_repos_ids(repositories): + def _build_repos_ids(repos): """ Gets a set of repository identifiers based on the supplied repository JSON structure as well as any matching repos defined in /etc/yum.repos.d. - :param repositories: the repositories defined on the command + :type repos resource_management.libraries.functions.repository_util.CommandRepository :return: the list of repo IDs from both the command and any matches found on the system with the same URLs. """ - repo_ids = [repository['repoId'] for repository in repositories] - base_urls = [repository['baseUrl'] for repository in repositories if 'baseUrl' in repository] - mirrors = [repository['mirrorsList'] for repository in repositories if 'mirrorsList' in repository] + + repo_ids = [] + base_urls = [] + mirrors = [] + + for repo in repos.items: + repo_ids.append(repo.repo_id) + + if repo.base_url: + base_urls.append(repo.base_url) + + if repo.mirrors_list: + mirrors.append(repo.mirrors_list) # for every repo file, find any which match the base URLs we're trying to write out # if there are any matches, it means the repo already exists and we should use it to search http://git-wip-us.apache.org/repos/asf/ambari/blob/467e3448/ambari-common/src/main/python/resource_management/core/providers/package/zypper.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/providers/package/zypper.py b/ambari-common/src/main/python/resource_management/core/providers/package/zypper.py index 6fc4b59..504ec0d 100644 --- a/ambari-common/src/main/python/resource_management/core/providers/package/zypper.py +++ b/ambari-common/src/main/python/resource_management/core/providers/package/zypper.py @@ -59,23 +59,28 @@ SELECT_TOOL_VERSION_PATTERN = re.compile("(\d{1,2}\.\d{1,2}\.\d{1,2}\.\d{1,2}-*\ class ZypperProvider(RPMBasedPackageProvider): - def get_available_packages_in_repos(self, repositories): + def get_available_packages_in_repos(self, repos): """ Gets all (both installed and available) packages that are available at given repositories. - :param repositories: from command configs like config['repositoryFile']['repositories'] + :type repos resource_management.libraries.functions.repository_util.CommandRepository :return: installed and available packages from these repositories """ available_packages = [] - available_packages_in_repos = [] - repo_ids = [repository['repoId'] for repository in repositories] + repo_ids = [repository.repo_id for repository in repos.items] - for repo in repo_ids: - available_packages.extend(self._lookup_packages([AMBARI_SUDO_BINARY, "zypper", "--no-gpg-checks", "search", "--details", "--repo", repo])) + # zypper cant tell from which repository were installed package, as repo r matching by pkg_name + # as result repository would be matched if it contains package with same meta info + if repos.feat.scoped: + Logger.info("Looking for matching packages in the following repositories: {0}".format(", ".join(repo_ids))) + else: + Logger.info("Packages will be queried using all available repositories on the system.") - available_packages_in_repos += [package[0] for package in available_packages] + for repo in repo_ids: + repo = repo if repos.feat.scoped else None + available_packages.extend(self._get_available_packages(repo)) - return available_packages_in_repos + return [package[0] for package in available_packages] def get_all_package_versions(self, pkg_name): """ @@ -96,6 +101,22 @@ class ZypperProvider(RPMBasedPackageProvider): matches = SELECT_TOOL_VERSION_PATTERN.findall(v.strip()) return matches[0] if matches else None + def _get_available_packages(self, repo_filter=None): + """ + Returning list of available packages with possibility to filter them by name + :param repo_filter: repository name + + :type repo_filter str|None + :rtype list[list,] + """ + + cmd = [AMBARI_SUDO_BINARY, "zypper", "--no-gpg-checks", "search", "--details"] + + if repo_filter: + cmd.extend(["--repo=" + repo_filter]) + + return self._lookup_packages(cmd) + def normalize_select_tool_versions(self, versions): """ Function expect output from get_all_package_versions http://git-wip-us.apache.org/repos/asf/ambari/blob/467e3448/ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py b/ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py index 7ad7df0..ef54f40 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py @@ -33,27 +33,23 @@ def create_repo_files(template, command_repository): """ Creates repositories in a consistent manner for all types :param command_repository: a CommandRepository instance + :type command_repository CommandRepository :return: a dictionary with repo ID => repo file name mapping """ if command_repository.version_id is None: raise Fail("The command repository was not parsed correctly") - if 0 == len(command_repository.repositories): + if 0 == len(command_repository.items): Logger.warning( "Repository for {0}/{1} has no repositories. Ambari may not be managing this version.".format( command_repository.stack_name, command_repository.version_string)) return {} - # add the stack name to the file name just to make it a little easier to debug - # version_id is the primary id of the repo_version table in the database - file_name = "ambari-{0}-{1}".format(command_repository.stack_name.lower(), - command_repository.version_id) - append_to_file = False # initialize to False to create the file anew. repo_files = {} - for repository in command_repository.repositories: + for repository in command_repository.items: if repository.repo_id is None: raise Fail("Repository with url {0} has no id".format(repository.base_url)) @@ -64,50 +60,63 @@ def create_repo_files(template, command_repository): command_repository.stack_name, command_repository.version_string, repository.repo_id)) else: Repository(repository.repo_id, - action = "create", - base_url = repository.base_url, - mirror_list = repository.mirrors_list, - repo_file_name = file_name, - repo_template = template, - components = repository.ubuntu_components, - append_to_file = append_to_file) + action="create", + base_url=repository.base_url, + mirror_list=repository.mirrors_list, + repo_file_name=command_repository.repo_filename, + repo_template=template, + components=repository.ubuntu_components, + append_to_file=append_to_file) append_to_file = True - repo_files[repository.repo_id] = file_name + repo_files[repository.repo_id] = command_repository.repo_filename return repo_files -def _find_value(dictionary, key): +def _find_value(dictionary, key, default=None): """ Helper to find a value in a dictionary """ if key not in dictionary: - return None + return default return dictionary[key] +class CommandRepositoryFeature(object): + def __init__(self, feat_dict): + """ + :type feat_dict dict + """ + self.pre_installed = _find_value(feat_dict, "preInstalled", default=False) + self.scoped = _find_value(feat_dict, "scoped", default=True) + + class CommandRepository(object): """ Class that encapsulates the representation of repositories passed in a command. This class should match the CommandRepository class. """ - def __init__(self, jsonvalue): - - if isinstance(jsonvalue, dict): - json_dict = jsonvalue - elif isinstance(jsonvalue, basestring): - json_dict = json.loads(jsonvalue) + def __init__(self, repo_object): + """ + :type repo_object dict|basestring + """ - if json_dict is None: - raise Fail("Cannot deserialize command repository {0}".format(str(jsonvalue))) + if isinstance(repo_object, dict): + json_dict = dict(repo_object) # strict dict(from ConfigDict) to avoid hidden type conversions + elif isinstance(repo_object, basestring): + json_dict = json.loads(repo_object) + else: + raise Fail("Cannot deserialize command repository {0}".format(str(repo_object))) # version_id is the primary id of the repo_version table in the database self.version_id = _find_value(json_dict, 'repoVersionId') self.stack_name = _find_value(json_dict, 'stackName') self.version_string = _find_value(json_dict, 'repoVersion') - self.repositories = [] + self.repo_filename = _find_value(json_dict, 'repoFileName') + self.feat = CommandRepositoryFeature(_find_value(json_dict, "feature", default={})) + self.items = [] repos_def = _find_value(json_dict, 'repositories') if repos_def is not None: @@ -115,25 +124,29 @@ class CommandRepository(object): repos_def = [repos_def] for repo_def in repos_def: - self.repositories.append(_CommandRepositoryEntry(repo_def)) + self.items.append(CommandRepositoryItem(self, repo_def)) -class _CommandRepositoryEntry(object): +class CommandRepositoryItem(object): """ Class that represents the entries of a CommandRepository. This isn't meant to be instantiated outside a CommandRepository """ - def __init__(self, json_dict): + + def __init__(self, repo, json_dict): + """ + :type repo CommandRepository + :type json_dict dict + """ + self._repo = repo + self.repo_id = _find_value(json_dict, 'repoId') # this is the id within the repo file, not an Ambari artifact self.repo_name = _find_value(json_dict, 'repoName') self.distribution = _find_value(json_dict, 'distribution') self.components = _find_value(json_dict, 'components') self.base_url = _find_value(json_dict, 'baseUrl') self.mirrors_list = _find_value(json_dict, 'mirrorsList') - self.ambari_managed = _find_value(json_dict, 'ambariManaged') - - if self.ambari_managed is None: - self.ambari_managed = True + self.ambari_managed = _find_value(json_dict, 'ambariManaged', default=True) self.ubuntu_components = [self.distribution if self.distribution else self.repo_name] + \ [self.components.replace(",", " ") if self.components else UBUNTU_REPO_COMPONENTS_POSTFIX] http://git-wip-us.apache.org/repos/asf/ambari/blob/467e3448/ambari-common/src/main/python/resource_management/libraries/script/script.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py b/ambari-common/src/main/python/resource_management/libraries/script/script.py index 12e6f98..88e5336 100644 --- a/ambari-common/src/main/python/resource_management/libraries/script/script.py +++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py @@ -53,6 +53,7 @@ from resource_management.libraries.functions.version import format_stack_version from resource_management.libraries.functions import stack_tools from resource_management.libraries.functions.constants import Direction from resource_management.libraries.script.config_dictionary import ConfigDictionary, UnknownConfiguration +from resource_management.libraries.functions.repository_util import CommandRepository from resource_management.core.resources.system import Execute from contextlib import closing from resource_management.libraries.functions.stack_features import check_stack_feature @@ -775,10 +776,9 @@ class Script(object): if self.available_packages_in_repos: return self.available_packages_in_repos - pkg_provider = get_provider("Package") try: - self.available_packages_in_repos = pkg_provider.get_available_packages_in_repos(self.get_config()['repositoryFile']['repositories']) + self.available_packages_in_repos = pkg_provider.get_available_packages_in_repos(CommandRepository(self.get_config()['repositoryFile'])) except Exception as err: Logger.exception("Unable to load available packages") self.available_packages_in_repos = [] http://git-wip-us.apache.org/repos/asf/ambari/blob/467e3448/ambari-server/src/main/java/org/apache/ambari/server/agent/CommandRepository.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/CommandRepository.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/CommandRepository.java index 301f475..a70326e 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/agent/CommandRepository.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/CommandRepository.java @@ -21,6 +21,8 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; +import org.apache.ambari.annotations.Experimental; +import org.apache.ambari.annotations.ExperimentalFeature; import org.apache.ambari.server.orm.entities.RepositoryEntity; import org.apache.ambari.server.state.RepositoryInfo; import org.apache.commons.lang.builder.ToStringBuilder; @@ -45,6 +47,21 @@ public class CommandRepository { @SerializedName("stackName") private String m_stackName; + @SerializedName("repoFileName") + private String m_repoFileName; + + @SerializedName("feature") + private CommandRepositoryFeature feature = new CommandRepositoryFeature(); + + /** + * Provides {@link CommandRepository} feature + * + * @return {@link CommandRepositoryFeature} + */ + public CommandRepositoryFeature getFeature(){ + return feature; + } + /** * {@code true} if Ambari believes that this repository has reported back it's * version after distribution. @@ -147,6 +164,70 @@ public class CommandRepository { } /** + * Update repository id to be consistent with old format + * + * @param repoVersion + */ + @Deprecated + @Experimental(feature= ExperimentalFeature.PATCH_UPGRADES) + public void setLegacyRepoId(String repoVersion){ + for (Repository repo : m_repositories) { + repo.m_repoId = String.format("%s-%s", repo.getRepoName(), repoVersion); + } + } + + /** + * Sets filename for the repo + * + * @param stackName name of the stack + * @param repoVersion repository version + */ + @Deprecated + @Experimental(feature= ExperimentalFeature.PATCH_UPGRADES) + public void setLegacyRepoFileName(String stackName, String repoVersion) { + this.m_repoFileName = String.format("%s-%s", stackName, repoVersion); + } + + /** + * Sets filename for the repo + * + * @param stackName name of the stack + * @param repoVersionId repository version id + */ + public void setRepoFileName(String stackName, Long repoVersionId) { + this.m_repoFileName = String.format("ambari-%s-%s", stackName.toLowerCase(), repoVersionId.toString()); + } + + /** + * Minimal information about repository feature + */ + public static class CommandRepositoryFeature { + + /** + * Repository is pre-installed on the host + */ + @SerializedName("preInstalled") + private Boolean m_isPreInstalled = false; + + /** + * Indicates if any operation with the packages should be scoped to this repository only. + * + * Currently affecting: getting available packages from the repository + */ + @SerializedName("scoped") + private boolean m_isScoped = true; + + public void setIsScoped(boolean isScoped){ + this.m_isScoped = isScoped; + } + + public void setPreInstalled(String isPreInstalled) { + this.m_isPreInstalled = isPreInstalled.equalsIgnoreCase("true"); + } + + } + + /** * Minimal information required to generate repo files on the agent. These are copies * of the repository objects from repo versions that can be changed for URL overrides, etc. */ @@ -196,6 +277,10 @@ public class CommandRepository { m_osType = osType; } + public void setRepoId(String repoId){ + m_repoId = repoId; + } + public void setBaseUrl(String url) { m_baseUrl = url; } http://git-wip-us.apache.org/repos/asf/ambari/blob/467e3448/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java index fd27169..9198164 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java @@ -450,6 +450,7 @@ public class ExecutionCommand extends AgentCommand { @Deprecated @Experimental(feature=ExperimentalFeature.PATCH_UPGRADES) String REPO_INFO = "repo_info"; + String DB_NAME = "db_name"; String GLOBAL = "global"; String AMBARI_DB_RCA_URL = "ambari_db_rca_url"; http://git-wip-us.apache.org/repos/asf/ambari/blob/467e3448/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java index 1b4d741..3b2baad 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java @@ -2068,6 +2068,10 @@ public class Configuration { public static final ConfigurationProperty<String> SYS_PREPPED_HOSTS = new ConfigurationProperty<>( "packages.pre.installed", "false"); + @Markdown(description = "This property is used in specific testing circumstances only. Its use otherwise will lead to very unpredictable results with repository management and package installation") + public static final ConfigurationProperty<String> LEGACY_OVERRIDE = new ConfigurationProperty<>( + "repositories.legacy-override.enabled", "false"); + private static final String LDAP_ADMIN_GROUP_MAPPING_MEMBER_ATTR_DEFAULT = ""; /** @@ -3434,6 +3438,15 @@ public class Configuration { return getProperty(SYS_PREPPED_HOSTS); } + /** + * Return {@code true} if we forced to work with legacy repositories + * + * @return {@link Boolean} + */ + public boolean arePackagesLegacyOverridden(){ + return getProperty(LEGACY_OVERRIDE).equalsIgnoreCase("true"); + } + public CommandExecutionType getStageExecutionType(){ return CommandExecutionType.valueOf(getProperty(COMMAND_EXECUTION_TYPE)); } http://git-wip-us.apache.org/repos/asf/ambari/blob/467e3448/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java index 0f6eb90..0a43732 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java @@ -572,7 +572,6 @@ public class AmbariActionExecutionHelper { rootJsonObject.add("repositories", repositories); } } - hostLevelParams.put(REPO_INFO, rootJsonObject.toString()); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/467e3448/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java index e12477e..044e6ef 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java @@ -1286,9 +1286,12 @@ public class AmbariCustomCommandExecutionHelper { public CommandRepository getCommandRepository(final Cluster cluster, ServiceComponent component, final Host host) throws AmbariException { final CommandRepository command = new CommandRepository(); + boolean sysPreppedHost = configs.areHostsSysPrepped().equalsIgnoreCase("true"); StackId stackId = component.getDesiredStackId(); command.setRepositories(Collections.<RepositoryInfo>emptyList()); command.setStackName(stackId.getStackName()); + command.getFeature().setPreInstalled(configs.areHostsSysPrepped()); + command.getFeature().setIsScoped(!sysPreppedHost); final BaseUrlUpdater<Void> updater = new BaseUrlUpdater<Void>(null) { @Override @@ -1308,7 +1311,14 @@ public class AmbariCustomCommandExecutionHelper { if (!osEntity.isAmbariManagedRepos()) { command.setNonManaged(); } else { - command.setUniqueSuffix(String.format("-repo-%s", rve.getId())); + if (rve.isLegacy()){ + command.setLegacyRepoId(rve.getVersion()); + command.setLegacyRepoFileName(rve.getStackName(), rve.getVersion()); + command.getFeature().setIsScoped(false); + } else { + command.setRepoFileName(rve.getStackName(), rve.getId()); + command.setUniqueSuffix(String.format("-repo-%s", rve.getId())); + } } } } @@ -1319,6 +1329,11 @@ public class AmbariCustomCommandExecutionHelper { updateBaseUrls(cluster, component, updater); + if (configs.arePackagesLegacyOverridden()) { + LOG.warn("Legacy override option is turned on, disabling CommandRepositoryFeature.scoped feature"); + command.getFeature().setIsScoped(false); + } + return command; } http://git-wip-us.apache.org/repos/asf/ambari/blob/467e3448/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryVersionEntity.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryVersionEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryVersionEntity.java index 7eedc4d..ceb35e5 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryVersionEntity.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryVersionEntity.java @@ -43,6 +43,8 @@ import javax.persistence.TableGenerator; import javax.persistence.Transient; import javax.persistence.UniqueConstraint; +import org.apache.ambari.annotations.Experimental; +import org.apache.ambari.annotations.ExperimentalFeature; import org.apache.ambari.server.StaticallyInject; import org.apache.ambari.server.state.RepositoryType; import org.apache.ambari.server.state.StackId; @@ -153,6 +155,9 @@ public class RepositoryVersionEntity { @Column(name = "resolved", nullable = false) private short resolved = 0; + @Column(name = "legacy", nullable = false) + private short isLegacy = 0; + @ManyToOne @JoinColumn(name = "parent_id") private RepositoryVersionEntity parent; @@ -489,6 +494,28 @@ public class RepositoryVersionEntity { } /** + * Gets whether this repository is legacy + * + * @return + */ + @Deprecated + @Experimental(feature= ExperimentalFeature.PATCH_UPGRADES) + public boolean isLegacy(){ + return isLegacy == 1; + } + + /** + * Sets whether this repository is legacy. Scoped for moving from old-style repository naming to new + * + * @param isLegacy + */ + @Deprecated + @Experimental(feature= ExperimentalFeature.PATCH_UPGRADES) + public void setLegacy(boolean isLegacy){ + this.isLegacy = isLegacy ? (short) 1 : (short) 0; + } + + /** * Sets whether this repository has been installed and has reported back its * actual version. * http://git-wip-us.apache.org/repos/asf/ambari/blob/467e3448/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java index f540d8d..8f34613 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java @@ -309,24 +309,39 @@ public class RepositoryVersionHelper { /** * Adds a command repository to the action context * @param context the context - * @param osFamily the OS family + * @param osEntity the OS family * @param repoVersion the repository version entity - * @param repos the repository entities */ public void addCommandRepository(ActionExecutionContext context, RepositoryVersionEntity repoVersion, OperatingSystemEntity osEntity) { final CommandRepository commandRepo = new CommandRepository(); + boolean sysPreppedHost = configuration.get().areHostsSysPrepped().equalsIgnoreCase("true"); + commandRepo.setRepositories(osEntity.getOsType(), osEntity.getRepositories()); commandRepo.setRepositoryVersion(repoVersion.getVersion()); commandRepo.setRepositoryVersionId(repoVersion.getId()); commandRepo.setResolved(repoVersion.isResolved()); commandRepo.setStackName(repoVersion.getStackId().getStackName()); + commandRepo.getFeature().setPreInstalled(configuration.get().areHostsSysPrepped()); + commandRepo.getFeature().setIsScoped(!sysPreppedHost); if (!osEntity.isAmbariManagedRepos()) { commandRepo.setNonManaged(); } else { - commandRepo.setUniqueSuffix(String.format("-repo-%s", repoVersion.getId())); + if (repoVersion.isLegacy()){ + commandRepo.setLegacyRepoFileName(repoVersion.getStackName(), repoVersion.getVersion()); + commandRepo.setLegacyRepoId(repoVersion.getVersion()); + commandRepo.getFeature().setIsScoped(false); + } else { + commandRepo.setRepoFileName(repoVersion.getStackName(), repoVersion.getId()); + commandRepo.setUniqueSuffix(String.format("-repo-%s", repoVersion.getId())); + } + } + + if (configuration.get().arePackagesLegacyOverridden()) { + LOG.warn("Legacy override option is turned on, disabling CommandRepositoryFeature.scoped feature"); + commandRepo.getFeature().setIsScoped(false); } context.addVisitor(new ExecutionCommandVisitor() { http://git-wip-us.apache.org/repos/asf/ambari/blob/467e3448/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog260.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog260.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog260.java index ffcf668..11f79fe 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog260.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog260.java @@ -88,6 +88,7 @@ public class UpgradeCatalog260 extends AbstractUpgradeCatalog { public static final String REPO_VERSION_ID_COLUMN = "repo_version_id"; public static final String REPO_VERSION_RESOLVED_COLUMN = "resolved"; public static final String REPO_VERSION_HIDDEN_COLUMN = "hidden"; + public static final String REPO_VERSION_LEGACY_COLUMN = "legacy"; public static final String HOST_COMPONENT_DESIRED_STATE_TABLE = "hostcomponentdesiredstate"; public static final String FK_HCDS_DESIRED_STACK_ID = "FK_hcds_desired_stack_id"; @@ -204,6 +205,7 @@ public class UpgradeCatalog260 extends AbstractUpgradeCatalog { createUpgradeHistoryTable(); updateRepositoryVersionTable(); renameServiceDeletedColumn(); + addLegacyColumn(); expandUpgradeItemItemTextColumn(); addViewUrlPKConstraint(); removeStaleConstraints(); @@ -255,6 +257,17 @@ public class UpgradeCatalog260 extends AbstractUpgradeCatalog { String.class, char[].class); } + private void addLegacyColumn() throws AmbariException, SQLException { + Boolean isLegacyColumnExists = dbAccessor.tableHasColumn(REPO_VERSION_TABLE, REPO_VERSION_LEGACY_COLUMN); + if (!isLegacyColumnExists) { + DBAccessor.DBColumnInfo legacyColumn = new DBAccessor.DBColumnInfo(REPO_VERSION_LEGACY_COLUMN, Short.class, null, 1, false); + dbAccessor.addColumn(REPO_VERSION_TABLE, legacyColumn); + + legacyColumn.setDefaultValue(0); + dbAccessor.alterColumn(REPO_VERSION_TABLE, legacyColumn); + } + } + private void renameServiceDeletedColumn() throws AmbariException, SQLException { if (dbAccessor.tableHasColumn(CLUSTER_CONFIG_TABLE, SERVICE_DELETED_COLUMN)) { dbAccessor.renameColumn(CLUSTER_CONFIG_TABLE, SERVICE_DELETED_COLUMN, new DBAccessor.DBColumnInfo(UNMAPPED_COLUMN, Short.class, null, 0, false)); http://git-wip-us.apache.org/repos/asf/ambari/blob/467e3448/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql index 614af1e..015ec0a 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql @@ -156,6 +156,7 @@ CREATE TABLE repo_version ( repo_type VARCHAR(255) DEFAULT 'STANDARD' NOT NULL, hidden SMALLINT NOT NULL DEFAULT 0, resolved SMALLINT NOT NULL DEFAULT 0, + legacy SMALLINT NOT NULL DEFAULT 0, version_url VARCHAR(1024), version_xml CLOB, version_xsd VARCHAR(512), http://git-wip-us.apache.org/repos/asf/ambari/blob/467e3448/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql index 530411a..eb9ca96 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql @@ -176,6 +176,7 @@ CREATE TABLE repo_version ( repo_type VARCHAR(255) DEFAULT 'STANDARD' NOT NULL, hidden SMALLINT NOT NULL DEFAULT 0, resolved TINYINT(1) NOT NULL DEFAULT 0, + legacy TINYINT(1) NOT NULL DEFAULT 0, version_url VARCHAR(1024), version_xml MEDIUMTEXT, version_xsd VARCHAR(512), http://git-wip-us.apache.org/repos/asf/ambari/blob/467e3448/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql index ebe5f12..dac3f28 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql @@ -157,6 +157,7 @@ CREATE TABLE repo_version ( repo_type VARCHAR2(255) DEFAULT 'STANDARD' NOT NULL, hidden NUMBER(1) DEFAULT 0 NOT NULL, resolved NUMBER(1) DEFAULT 0 NOT NULL, + legacy NUMBER(1) DEFAULT 0 NOT NULL, version_url VARCHAR(1024), version_xml CLOB, version_xsd VARCHAR(512), http://git-wip-us.apache.org/repos/asf/ambari/blob/467e3448/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql index 634db95..c321a38 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql @@ -160,6 +160,7 @@ CREATE TABLE repo_version ( version_xsd VARCHAR(512), parent_id BIGINT, resolved SMALLINT NOT NULL DEFAULT 0, + legacy SMALLINT NOT NULL DEFAULT 0, CONSTRAINT PK_repo_version PRIMARY KEY (repo_version_id), CONSTRAINT FK_repoversion_stack_id FOREIGN KEY (stack_id) REFERENCES stack(stack_id), CONSTRAINT UQ_repo_version_display_name UNIQUE (display_name), http://git-wip-us.apache.org/repos/asf/ambari/blob/467e3448/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql index f64ff80..8740ed7 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql @@ -155,6 +155,7 @@ CREATE TABLE repo_version ( repo_type VARCHAR(255) DEFAULT 'STANDARD' NOT NULL, hidden SMALLINT NOT NULL DEFAULT 0, resolved BIT NOT NULL DEFAULT 0, + legacy BIT NOT NULL DEFAULT 0, version_url VARCHAR(1024), version_xml TEXT, version_xsd VARCHAR(512), http://git-wip-us.apache.org/repos/asf/ambari/blob/467e3448/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql index 7a3feaf..415589d 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql @@ -169,6 +169,7 @@ CREATE TABLE repo_version ( repo_type VARCHAR(255) DEFAULT 'STANDARD' NOT NULL, hidden SMALLINT NOT NULL DEFAULT 0, resolved BIT NOT NULL DEFAULT 0, + legacy BIT NOT NULL DEFAULT 0, version_url VARCHAR(1024), version_xml VARCHAR(MAX), version_xsd VARCHAR(512), http://git-wip-us.apache.org/repos/asf/ambari/blob/467e3448/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py b/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py index 872d55e..47f9a74 100644 --- a/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py +++ b/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py @@ -103,7 +103,7 @@ class InstallPackages(Script): self.repository_version = self.repository_version.strip() try: - if not command_repository.repositories: + if not command_repository.items: Logger.warning( "Repository list is empty. Ambari may not be managing the repositories for {0}.".format( self.repository_version)) @@ -328,6 +328,7 @@ class InstallPackages(Script): # specific repo that the stack-select tools are coming out of in case there are multiple # patches installed repositories = config['repositoryFile']['repositories'] + command_repos = CommandRepository(config['repositoryFile']) repository_ids = [repository['repoId'] for repository in repositories] repos_to_use = {} for repo_id in repository_ids: @@ -345,7 +346,7 @@ class InstallPackages(Script): packages_were_checked = True filtered_package_list = self.filter_package_list(package_list) try: - available_packages_in_repos = self.pkg_provider.get_available_packages_in_repos(repositories) + available_packages_in_repos = self.pkg_provider.get_available_packages_in_repos(command_repos) except Exception: available_packages_in_repos = [] for package in filtered_package_list: http://git-wip-us.apache.org/repos/asf/ambari/blob/467e3448/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProviderTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProviderTest.java index a1415703..9840106 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProviderTest.java @@ -1447,6 +1447,8 @@ public class ClusterStackVersionResourceProviderTest { RepositoryVersionEntity repoVersion = createNiceMock(RepositoryVersionEntity.class); expect(repoVersion.getId()).andReturn(1L).anyTimes(); expect(repoVersion.getStackId()).andReturn(new StackId("HDP-2.1.1")).anyTimes(); + expect(repoVersion.isLegacy()).andReturn(false).anyTimes(); + expect(repoVersion.getStackName()).andReturn("HDP").anyTimes(); String os_json = "[\n" + http://git-wip-us.apache.org/repos/asf/ambari/blob/467e3448/ambari-server/src/test/python/custom_actions/configs/install_packages_config.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/custom_actions/configs/install_packages_config.json b/ambari-server/src/test/python/custom_actions/configs/install_packages_config.json index 4bebe99..8029959 100644 --- a/ambari-server/src/test/python/custom_actions/configs/install_packages_config.json +++ b/ambari-server/src/test/python/custom_actions/configs/install_packages_config.json @@ -54,6 +54,7 @@ "stackName": "HDP", "repoVersionId": 1, "repoVersion": "2.2.0.1-885", + "repoFileName": "ambari-hdp-1", "repositories": [ { "repoName": "HDP-UTILS", http://git-wip-us.apache.org/repos/asf/ambari/blob/467e3448/ambari-server/src/test/python/custom_actions/configs/install_packages_repository_file.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/custom_actions/configs/install_packages_repository_file.json b/ambari-server/src/test/python/custom_actions/configs/install_packages_repository_file.json index 761dced..e7abedd 100644 --- a/ambari-server/src/test/python/custom_actions/configs/install_packages_repository_file.json +++ b/ambari-server/src/test/python/custom_actions/configs/install_packages_repository_file.json @@ -28,6 +28,7 @@ "stackName": "HDP", "repoVersionId": 4, "repoVersion": "2.2.0.1-885", + "repoFileName": "ambari-hdp-4", "repositories": [ { "repoName": "HDP-UTILS",