Repository: ambari Updated Branches: refs/heads/branch-2.6 f64d027db -> 412c6cac4
AMBARI-21854. Adapt Repository Files For Existing Deployments (dgrinenko via dlysnichenko) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/412c6cac Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/412c6cac Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/412c6cac Branch: refs/heads/branch-2.6 Commit: 412c6cac4416cb676046dd6792ccb962441febf5 Parents: f64d027 Author: Lisnichenko Dmitro <dlysniche...@hortonworks.com> Authored: Wed Sep 20 12:45:25 2017 +0300 Committer: Lisnichenko Dmitro <dlysniche...@hortonworks.com> Committed: Wed Sep 20 16:20:54 2017 +0300 ---------------------------------------------------------------------- .../core/providers/package/__init__.py | 2 +- .../core/providers/package/apt.py | 23 ++++-- .../core/providers/package/yumrpm.py | 82 +++++++++++++------ .../core/providers/package/zypper.py | 37 +++++++-- .../libraries/functions/repository_util.py | 77 ++++++++++-------- .../libraries/script/script.py | 3 +- .../ambari/server/agent/CommandRepository.java | 85 ++++++++++++++++++++ .../ambari/server/agent/ExecutionCommand.java | 1 + .../server/configuration/Configuration.java | 13 +++ .../controller/AmbariActionExecutionHelper.java | 1 - .../AmbariCustomCommandExecutionHelper.java | 17 +++- .../orm/entities/RepositoryVersionEntity.java | 27 +++++++ .../stack/upgrade/RepositoryVersionHelper.java | 21 ++++- .../server/upgrade/UpgradeCatalog260.java | 14 ++++ .../main/resources/Ambari-DDL-Derby-CREATE.sql | 1 + .../main/resources/Ambari-DDL-MySQL-CREATE.sql | 1 + .../main/resources/Ambari-DDL-Oracle-CREATE.sql | 1 + .../resources/Ambari-DDL-Postgres-CREATE.sql | 1 + .../resources/Ambari-DDL-SQLAnywhere-CREATE.sql | 1 + .../resources/Ambari-DDL-SQLServer-CREATE.sql | 1 + .../custom_actions/scripts/install_packages.py | 4 +- 21 files changed, 333 insertions(+), 80 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/412c6cac/ambari-common/src/main/python/resource_management/core/providers/package/__init__.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/providers/package/__init__.py b/ambari-common/src/main/python/resource_management/core/providers/package/__init__.py index 84ff9cc..fba16cf 100644 --- a/ambari-common/src/main/python/resource_management/core/providers/package/__init__.py +++ b/ambari-common/src/main/python/resource_management/core/providers/package/__init__.py @@ -69,7 +69,7 @@ class PackageProvider(Provider): def get_available_packages_in_repos(self, repositories): """ Gets all (both installed and available) packages that are available at given repositories. - :param repositories: from command configs like config['repositoryFile']['repositories'] + :type repositories resource_management.libraries.functions.repository_util.CommandRepository :return: installed and available packages from these repositories """ raise NotImplementedError() http://git-wip-us.apache.org/repos/asf/ambari/blob/412c6cac/ambari-common/src/main/python/resource_management/core/providers/package/apt.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/providers/package/apt.py b/ambari-common/src/main/python/resource_management/core/providers/package/apt.py index 65caeff..6517da2 100644 --- a/ambari-common/src/main/python/resource_management/core/providers/package/apt.py +++ b/ambari-common/src/main/python/resource_management/core/providers/package/apt.py @@ -274,24 +274,31 @@ class AptProvider(PackageProvider): return packages - def get_available_packages_in_repos(self, repositories): + def get_available_packages_in_repos(self, repos): """ Gets all (both installed and available) packages that are available at given repositories. - :param repositories: from command configs like config['repositoryFile']['repositories'] + :type repos resource_management.libraries.functions.repository_util.CommandRepository :return: installed and available packages from these repositories """ filtered_packages = [] packages = self.all_available_packages() + repo_ids = [] - for repo in repositories: - repo_url_part = repo['baseUrl'].replace("http://";, "").replace("/", "_") + for repo in repos.items: + repo_ids.append(repo.base_url.replace("http://";, "").replace("/", "_")) - for package in packages: - if repo_url_part in package[2]: - filtered_packages.append(package[0]) + if repos.feat.scoped: + Logger.info("Looking for matching packages in the following repositories: {0}".format(", ".join(repo_ids))) + for repo_id in repo_ids: + for package in packages: + if repo_id in package[2]: + filtered_packages.append(package[0]) - return filtered_packages + return filtered_packages + else: + Logger.info("Packages will be queried using all available repositories on the system.") + return [package[0] for package in packages] def get_all_package_versions(self, pkg_name): """ http://git-wip-us.apache.org/repos/asf/ambari/blob/412c6cac/ambari-common/src/main/python/resource_management/core/providers/package/yumrpm.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/providers/package/yumrpm.py b/ambari-common/src/main/python/resource_management/core/providers/package/yumrpm.py index e39a4c3..63faaab 100644 --- a/ambari-common/src/main/python/resource_management/core/providers/package/yumrpm.py +++ b/ambari-common/src/main/python/resource_management/core/providers/package/yumrpm.py @@ -19,6 +19,8 @@ limitations under the License. Ambari Agent """ +import ConfigParser +import glob from ambari_commons.constants import AMBARI_SUDO_BINARY from resource_management.core.providers.package import RPMBasedPackageProvider @@ -27,12 +29,9 @@ from resource_management.core.shell import string_cmd_from_args_list from resource_management.core.logger import Logger from resource_management.core.utils import suppress_stdout -import glob import re import os -import ConfigParser - INSTALL_CMD = { True: ['/usr/bin/yum', '-y', 'install'], False: ['/usr/bin/yum', '-d', '0', '-e', '0', '-y', 'install'], @@ -45,6 +44,7 @@ REMOVE_CMD = { REMOVE_WITHOUT_DEPENDENCIES_CMD = ['rpm', '-e', '--nodeps'] +YUM_REPO_LOCATION = "/etc/yum.repos.d" REPO_UPDATE_CMD = ['/usr/bin/yum', 'clean', 'metadata'] ALL_INSTALLED_PACKAGES_CMD = [AMBARI_SUDO_BINARY, "yum", "list", "installed"] ALL_AVAILABLE_PACKAGES_CMD = [AMBARI_SUDO_BINARY, "yum", "list", "available"] @@ -61,30 +61,41 @@ VERIFY_DEPENDENCY_CMD = ['/usr/bin/yum', '-d', '0', '-e', '0', 'check', 'depende LIST_ALL_SELECT_TOOL_PACKAGES_CMD = "yum list all --showduplicates|grep -v '@' |grep '^{pkg_name}'|awk '{print $2}'" SELECT_TOOL_VERSION_PATTERN = re.compile("(\d{1,2}\.\d{1,2}\.\d{1,2}\.\d{1,2}-*\d*).*") # xx.xx.xx.xx(-xxxx) -YUM_REPO_LOCATION = "/etc/yum.repos.d" class YumProvider(RPMBasedPackageProvider): - def get_available_packages_in_repos(self, repositories): + def get_available_packages_in_repos(self, repos): """ Gets all (both installed and available) packages that are available at given repositories. - :param repositories: from command configs like config['repositoryFile']['repositories'] + + :type repos resource_management.libraries.functions.repository_util.CommandRepository :return: installed and available packages from these repositories """ available_packages = [] installed_packages = [] - available_packages_in_repos = [] + repo_ids = [repo.repo_id for repo in repos.items] - repo_ids = self._build_repos_ids(repositories) - Logger.info("Looking for matching packages in the following repositories: {0}".format(", ".join(repo_ids))) + if repos.feat.scoped: + Logger.info("Looking for matching packages in the following repositories: {0}".format(", ".join(repo_ids))) + else: + Logger.info("Packages will be queried using all available repositories on the system.") for repo in repo_ids: - available_packages.extend(self._lookup_packages( - [AMBARI_SUDO_BINARY, "yum", "list", "available", "--disablerepo=*", "--enablerepo=" + repo], 'Available Packages')) + repo = repo if repos.feat.scoped else None + available_packages.extend(self._get_available_packages(repo)) installed_packages.extend(self._get_installed_packages(repo)) - available_packages_in_repos += [package[0] for package in available_packages + installed_packages] - return available_packages_in_repos + # fallback logic + if len(installed_packages) == 0 or len(available_packages) == 0: + repo_ids = self._build_repos_ids(repos) + for repo in repo_ids: + available_packages.extend(self._get_available_packages(repo)) + installed_packages.extend(self._get_installed_packages(repo)) + + available_packages = set(available_packages) + installed_packages = set(installed_packages) + + return [package[0] for package in available_packages + installed_packages] def get_all_package_versions(self, pkg_name): """ @@ -117,6 +128,22 @@ class YumProvider(RPMBasedPackageProvider): return [self.__parse_select_tool_version(i) for i in versions] + def _get_available_packages(self, repo_filter=None): + """ + Returning list of available packages with possibility to filter them by name + :param repo_filter: repository name + + :type repo_filter str|None + :rtype list[list,] + """ + + cmd = [AMBARI_SUDO_BINARY, "yum", "list", "available"] + + if repo_filter: + cmd.extend(["--disablerepo=*", "--enablerepo=" + repo_filter]) + + return self._lookup_packages(cmd, 'Available Packages') + def _get_installed_packages(self, repo_filter=None): """ Returning list of the installed packages with possibility to filter them by name @@ -179,7 +206,7 @@ class YumProvider(RPMBasedPackageProvider): :rtype list|dict """ # ToDo: move to iterative package lookup (check apt provider for details) - return self._lookup_packages(ALL_AVAILABLE_PACKAGES_CMD, "Available Packages") + return self._get_available_packages(None) def all_installed_packages(self, from_unknown_repo=False): """ @@ -191,7 +218,7 @@ class YumProvider(RPMBasedPackageProvider): :return result_type formatted list of packages """ # ToDo: move to iterative package lookup (check apt provider for details) - return self._lookup_packages(ALL_INSTALLED_PACKAGES_CMD, "Installed Packages") + return self._get_installed_packages(None) def verify_dependencies(self): """ @@ -269,7 +296,6 @@ class YumProvider(RPMBasedPackageProvider): else: return self.rpm_check_package_available(name) - def yum_check_package_available(self, name): """ Does the same as rpm_check_package_avaiable, but faster. @@ -289,19 +315,28 @@ class YumProvider(RPMBasedPackageProvider): return False - @staticmethod - def _build_repos_ids(repositories): + def _build_repos_ids(repos): """ Gets a set of repository identifiers based on the supplied repository JSON structure as well as any matching repos defined in /etc/yum.repos.d. - :param repositories: the repositories defined on the command + :type repos resource_management.libraries.functions.repository_util.CommandRepository :return: the list of repo IDs from both the command and any matches found on the system with the same URLs. """ - repo_ids = [repository['repoId'] for repository in repositories] - base_urls = [repository['baseUrl'] for repository in repositories if 'baseUrl' in repository] - mirrors = [repository['mirrorsList'] for repository in repositories if 'mirrorsList' in repository] + + repo_ids = [] + base_urls = [] + mirrors = [] + + for repo in repos.items: + repo_ids.append(repo.repo_id) + + if repo.base_url: + base_urls.append(repo.base_url) + + if repo.mirrors_list: + mirrors.append(repo.mirrors_list) # for every repo file, find any which match the base URLs we're trying to write out # if there are any matches, it means the repo already exists and we should use it to search @@ -321,4 +356,5 @@ class YumProvider(RPMBasedPackageProvider): if mirror in mirrors: repo_ids.append(section) - return set(repo_ids) \ No newline at end of file + return set(repo_ids) + http://git-wip-us.apache.org/repos/asf/ambari/blob/412c6cac/ambari-common/src/main/python/resource_management/core/providers/package/zypper.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/providers/package/zypper.py b/ambari-common/src/main/python/resource_management/core/providers/package/zypper.py index 1e5fb90..125b7b7 100644 --- a/ambari-common/src/main/python/resource_management/core/providers/package/zypper.py +++ b/ambari-common/src/main/python/resource_management/core/providers/package/zypper.py @@ -61,23 +61,28 @@ SELECT_TOOL_VERSION_PATTERN = re.compile("(\d{1,2}\.\d{1,2}\.\d{1,2}\.\d{1,2}-*\ class ZypperProvider(RPMBasedPackageProvider): - def get_available_packages_in_repos(self, repositories): + def get_available_packages_in_repos(self, repos): """ Gets all (both installed and available) packages that are available at given repositories. - :param repositories: from command configs like config['repositoryFile']['repositories'] + :type repos resource_management.libraries.functions.repository_util.CommandRepository :return: installed and available packages from these repositories """ available_packages = [] - available_packages_in_repos = [] - repo_ids = [repository['repoId'] for repository in repositories] + repo_ids = [repository.repo_id for repository in repos.items] - for repo in repo_ids: - available_packages.extend(self._lookup_packages([AMBARI_SUDO_BINARY, "zypper", "--no-gpg-checks", "search", "--details", "--repo", repo])) + # zypper cant tell from which repository were installed package, as repo r matching by pkg_name + # as result repository would be matched if it contains package with same meta info + if repos.feat.scoped: + Logger.info("Looking for matching packages in the following repositories: {0}".format(", ".join(repo_ids))) + else: + Logger.info("Packages will be queried using all available repositories on the system.") - available_packages_in_repos += [package[0] for package in available_packages] + for repo in repo_ids: + repo = repo if repos.feat.scoped else None + available_packages.extend(self._get_available_packages(repo)) - return available_packages_in_repos + return [package[0] for package in available_packages] def get_all_package_versions(self, pkg_name): """ @@ -98,6 +103,22 @@ class ZypperProvider(RPMBasedPackageProvider): matches = SELECT_TOOL_VERSION_PATTERN.findall(v.strip()) return matches[0] if matches else None + def _get_available_packages(self, repo_filter=None): + """ + Returning list of available packages with possibility to filter them by name + :param repo_filter: repository name + + :type repo_filter str|None + :rtype list[list,] + """ + + cmd = [AMBARI_SUDO_BINARY, "zypper", "--no-gpg-checks", "search", "--details"] + + if repo_filter: + cmd.extend(["--repo=" + repo_filter]) + + return self._lookup_packages(cmd) + def normalize_select_tool_versions(self, versions): """ Function expect output from get_all_package_versions http://git-wip-us.apache.org/repos/asf/ambari/blob/412c6cac/ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py b/ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py index b2f65d8..b224998 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py @@ -33,26 +33,22 @@ def create_repo_files(template, command_repository): """ Creates repositories in a consistent manner for all types :param command_repository: a CommandRepository instance + :type command_repository CommandRepository :return: """ if command_repository.version_id is None: raise Fail("The command repository was not parsed correctly") - if 0 == len(command_repository.repositories): + if 0 == len(command_repository.items): Logger.warning( "Repository for {0}/{1} has no repositories. Ambari may not be managing this version.".format( command_repository.stack_name, command_repository.version_string)) return - # add the stack name to the file name just to make it a little easier to debug - # version_id is the primary id of the repo_version table in the database - file_name = "ambari-{0}-{1}".format(command_repository.stack_name.lower(), - command_repository.version_id) - append_to_file = False # initialize to False to create the file anew. - for repository in command_repository.repositories: + for repository in command_repository.items: if repository.repo_id is None: raise Fail("Repository with url {0} has no id".format(repository.base_url)) @@ -63,47 +59,60 @@ def create_repo_files(template, command_repository): command_repository.stack_name, command_repository.version_string, repository.repo_id)) else: Repository(repository.repo_id, - action = "create", - base_url = repository.base_url, - mirror_list = repository.mirrors_list, - repo_file_name = file_name, - repo_template = template, - components = repository.ubuntu_components, - append_to_file = append_to_file) + action="create", + base_url=repository.base_url, + mirror_list=repository.mirrors_list, + repo_file_name=command_repository.repo_filename, + repo_template=template, + components=repository.ubuntu_components, + append_to_file=append_to_file) append_to_file = True -def _find_value(dictionary, key): +def _find_value(dictionary, key, default=None): """ Helper to find a value in a dictionary """ if key not in dictionary: - return None + return default return dictionary[key] +class CommandRepositoryFeature(object): + def __init__(self, feat_dict): + """ + :type feat_dict dict + """ + self.pre_installed = _find_value(feat_dict, "preInstalled", default=False) + self.scoped = _find_value(feat_dict, "scoped", default=True) + + class CommandRepository(object): """ Class that encapsulates the representation of repositories passed in a command. This class should match the CommandRepository class. """ - def __init__(self, jsonvalue): - - if isinstance(jsonvalue, dict): - json_dict = jsonvalue - elif isinstance(jsonvalue, basestring): - json_dict = json.loads(jsonvalue) + def __init__(self, repo_object): + """ + :type repo_object dict|basestring + """ - if json_dict is None: - raise Fail("Cannot deserialize command repository {0}".format(str(jsonvalue))) + if isinstance(repo_object, dict): + json_dict = dict(repo_object) # strict dict(from ConfigDict) to avoid hidden type conversions + elif isinstance(repo_object, basestring): + json_dict = json.loads(repo_object) + else: + raise Fail("Cannot deserialize command repository {0}".format(str(repo_object))) # version_id is the primary id of the repo_version table in the database self.version_id = _find_value(json_dict, 'repoVersionId') self.stack_name = _find_value(json_dict, 'stackName') self.version_string = _find_value(json_dict, 'repoVersion') - self.repositories = [] + self.repo_filename = _find_value(json_dict, 'repoFileName') + self.feat = CommandRepositoryFeature(_find_value(json_dict, "feature", default={})) + self.items = [] repos_def = _find_value(json_dict, 'repositories') if repos_def is not None: @@ -111,25 +120,29 @@ class CommandRepository(object): repos_def = [repos_def] for repo_def in repos_def: - self.repositories.append(_CommandRepositoryEntry(repo_def)) + self.items.append(CommandRepositoryItem(self, repo_def)) -class _CommandRepositoryEntry(object): +class CommandRepositoryItem(object): """ Class that represents the entries of a CommandRepository. This isn't meant to be instantiated outside a CommandRepository """ - def __init__(self, json_dict): + + def __init__(self, repo, json_dict): + """ + :type repo CommandRepository + :type json_dict dict + """ + self._repo = repo + self.repo_id = _find_value(json_dict, 'repoId') # this is the id within the repo file, not an Ambari artifact self.repo_name = _find_value(json_dict, 'repoName') self.distribution = _find_value(json_dict, 'distribution') self.components = _find_value(json_dict, 'components') self.base_url = _find_value(json_dict, 'baseUrl') self.mirrors_list = _find_value(json_dict, 'mirrorsList') - self.ambari_managed = _find_value(json_dict, 'ambariManaged') - - if self.ambari_managed is None: - self.ambari_managed = True + self.ambari_managed = _find_value(json_dict, 'ambariManaged', default=True) self.ubuntu_components = [self.distribution if self.distribution else self.repo_name] + \ [self.components.replace(",", " ") if self.components else UBUNTU_REPO_COMPONENTS_POSTFIX] http://git-wip-us.apache.org/repos/asf/ambari/blob/412c6cac/ambari-common/src/main/python/resource_management/libraries/script/script.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py b/ambari-common/src/main/python/resource_management/libraries/script/script.py index 5940166..99184d1 100644 --- a/ambari-common/src/main/python/resource_management/libraries/script/script.py +++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py @@ -52,6 +52,7 @@ from resource_management.libraries.functions.version import format_stack_version from resource_management.libraries.functions import stack_tools from resource_management.libraries.functions.constants import Direction from resource_management.libraries.script.config_dictionary import ConfigDictionary, UnknownConfiguration +from resource_management.libraries.functions.repository_util import CommandRepository from resource_management.core.resources.system import Execute from contextlib import closing from resource_management.libraries.functions.stack_features import check_stack_feature @@ -726,7 +727,7 @@ class Script(object): agent_stack_retry_count = int(config['hostLevelParams']['agent_stack_retry_count']) pkg_provider = get_provider("Package") try: - available_packages_in_repos = pkg_provider.get_available_packages_in_repos(config['repositoryFile']['repositories']) + available_packages_in_repos = pkg_provider.get_available_packages_in_repos(CommandRepository(config['repositoryFile'])) except Exception as err: Logger.exception("Unable to load available packages") available_packages_in_repos = [] http://git-wip-us.apache.org/repos/asf/ambari/blob/412c6cac/ambari-server/src/main/java/org/apache/ambari/server/agent/CommandRepository.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/CommandRepository.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/CommandRepository.java index 1351713..1b04d62 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/agent/CommandRepository.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/CommandRepository.java @@ -21,6 +21,8 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; +import org.apache.ambari.annotations.Experimental; +import org.apache.ambari.annotations.ExperimentalFeature; import org.apache.ambari.server.orm.entities.RepositoryEntity; import org.apache.ambari.server.state.RepositoryInfo; import org.apache.commons.lang.builder.ToStringBuilder; @@ -45,6 +47,21 @@ public class CommandRepository { @SerializedName("stackName") private String m_stackName; + @SerializedName("repoFileName") + private String m_repoFileName; + + @SerializedName("feature") + private CommandRepositoryFeature feature = new CommandRepositoryFeature(); + + /** + * Provides {@link CommandRepository} feature + * + * @return {@link CommandRepositoryFeature} + */ + public CommandRepositoryFeature getFeature(){ + return feature; + } + /** * @param version the repo version */ @@ -119,6 +136,70 @@ public class CommandRepository { } /** + * Update repository id to be consistent with old format + * + * @param repoVersion + */ + @Deprecated + @Experimental(feature= ExperimentalFeature.PATCH_UPGRADES) + public void setLegacyRepoId(String repoVersion){ + for (Repository repo : m_repositories) { + repo.m_repoId = String.format("%s-%s", repo.getRepoName(), repoVersion); + } + } + + /** + * Sets filename for the repo + * + * @param stackName name of the stack + * @param repoVersion repository version + */ + @Deprecated + @Experimental(feature= ExperimentalFeature.PATCH_UPGRADES) + public void setLegacyRepoFileName(String stackName, String repoVersion) { + this.m_repoFileName = String.format("%s-%s", stackName, repoVersion); + } + + /** + * Sets filename for the repo + * + * @param stackName name of the stack + * @param repoVersionId repository version id + */ + public void setRepoFileName(String stackName, Long repoVersionId) { + this.m_repoFileName = String.format("ambari-%s-%s", stackName.toLowerCase(), repoVersionId.toString()); + } + + /** + * Minimal information about repository feature + */ + public static class CommandRepositoryFeature { + + /** + * Repository is pre-installed on the host + */ + @SerializedName("preInstalled") + private Boolean m_isPreInstalled = false; + + /** + * Indicates if any operation with the packages should be scoped to this repository only. + * + * Currently affecting: getting available packages from the repository + */ + @SerializedName("scoped") + private boolean m_isScoped = true; + + public void setIsScoped(boolean isScoped){ + this.m_isScoped = isScoped; + } + + public void setPreInstalled(String isPreInstalled) { + this.m_isPreInstalled = isPreInstalled.equalsIgnoreCase("true"); + } + + } + + /** * Minimal information required to generate repo files on the agent. These are copies * of the repository objects from repo versions that can be changed for URL overrides, etc. */ @@ -168,6 +249,10 @@ public class CommandRepository { m_osType = osType; } + public void setRepoId(String repoId){ + m_repoId = repoId; + } + public void setBaseUrl(String url) { m_baseUrl = url; } http://git-wip-us.apache.org/repos/asf/ambari/blob/412c6cac/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java index 0906044..6ffc9c1 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java @@ -445,6 +445,7 @@ public class ExecutionCommand extends AgentCommand { @Deprecated @Experimental(feature=ExperimentalFeature.PATCH_UPGRADES) String REPO_INFO = "repo_info"; + String DB_NAME = "db_name"; String GLOBAL = "global"; String AMBARI_DB_RCA_URL = "ambari_db_rca_url"; http://git-wip-us.apache.org/repos/asf/ambari/blob/412c6cac/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java index e51a9b2..0bd6af6 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java @@ -2022,6 +2022,10 @@ public class Configuration { public static final ConfigurationProperty<String> SYS_PREPPED_HOSTS = new ConfigurationProperty<>( "packages.pre.installed", "false"); + @Markdown(description = "This property is used in specific testing circumstances only. Its use otherwise will lead to very unpredictable results with repository management and package installation") + public static final ConfigurationProperty<String> LEGACY_OVERRIDE = new ConfigurationProperty<>( + "repositories.legacy-override.enabled", "false"); + private static final String LDAP_ADMIN_GROUP_MAPPING_MEMBER_ATTR_DEFAULT = ""; /** @@ -3369,6 +3373,15 @@ public class Configuration { return getProperty(SYS_PREPPED_HOSTS); } + /** + * Return {@code true} if we forced to work with legacy repositories + * + * @return {@link Boolean} + */ + public boolean arePackagesLegacyOverridden(){ + return getProperty(LEGACY_OVERRIDE).equalsIgnoreCase("true"); + } + public CommandExecutionType getStageExecutionType(){ return CommandExecutionType.valueOf(getProperty(COMMAND_EXECUTION_TYPE)); } http://git-wip-us.apache.org/repos/asf/ambari/blob/412c6cac/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java index a644eda..80677fd 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java @@ -570,7 +570,6 @@ public class AmbariActionExecutionHelper { rootJsonObject.add("repositories", repositories); } } - hostLevelParams.put(REPO_INFO, rootJsonObject.toString()); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/412c6cac/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java index 3c15d92..d14b92f 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java @@ -1275,9 +1275,12 @@ public class AmbariCustomCommandExecutionHelper { public CommandRepository getCommandRepository(final Cluster cluster, ServiceComponent component, final Host host) throws AmbariException { final CommandRepository command = new CommandRepository(); + boolean sysPreppedHost = configs.areHostsSysPrepped().equalsIgnoreCase("true"); StackId stackId = component.getDesiredStackId(); command.setRepositories(Collections.<RepositoryInfo>emptyList()); command.setStackName(stackId.getStackName()); + command.getFeature().setPreInstalled(configs.areHostsSysPrepped()); + command.getFeature().setIsScoped(!sysPreppedHost); final BaseUrlUpdater<Void> updater = new BaseUrlUpdater<Void>(null) { @Override @@ -1296,7 +1299,14 @@ public class AmbariCustomCommandExecutionHelper { if (!osEntity.isAmbariManagedRepos()) { command.setNonManaged(); } else { - command.setUniqueSuffix(String.format("-repo-%s", rve.getId())); + if (rve.isLegacy()){ + command.setLegacyRepoId(rve.getVersion()); + command.setLegacyRepoFileName(rve.getStackName(), rve.getVersion()); + command.getFeature().setIsScoped(false); + } else { + command.setRepoFileName(rve.getStackName(), rve.getId()); + command.setUniqueSuffix(String.format("-repo-%s", rve.getId())); + } } } } @@ -1307,6 +1317,11 @@ public class AmbariCustomCommandExecutionHelper { updateBaseUrls(cluster, component, updater); + if (configs.arePackagesLegacyOverridden()) { + LOG.warn("Legacy override option is turned on, disabling CommandRepositoryFeature.scoped feature"); + command.getFeature().setIsScoped(false); + } + return command; } http://git-wip-us.apache.org/repos/asf/ambari/blob/412c6cac/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryVersionEntity.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryVersionEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryVersionEntity.java index 72269f0..04dfe7a 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryVersionEntity.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryVersionEntity.java @@ -45,6 +45,8 @@ import javax.persistence.TableGenerator; import javax.persistence.Transient; import javax.persistence.UniqueConstraint; +import org.apache.ambari.annotations.Experimental; +import org.apache.ambari.annotations.ExperimentalFeature; import org.apache.ambari.server.StaticallyInject; import org.apache.ambari.server.state.RepositoryType; import org.apache.ambari.server.state.StackId; @@ -146,6 +148,9 @@ public class RepositoryVersionEntity { @Column(name = "hidden", nullable = false, insertable = true, updatable = true) private short isHidden = 0; + @Column(name = "legacy", nullable = false) + private boolean isLegacy = false; + @ManyToOne @JoinColumn(name = "parent_id") private RepositoryVersionEntity parent; @@ -473,4 +478,26 @@ public class RepositoryVersionEntity { this.isHidden = (short) (isHidden ? 1 : 0); } + /** + * Gets whether this repository is legacy + * + * @return + */ + @Deprecated + @Experimental(feature= ExperimentalFeature.PATCH_UPGRADES) + public Boolean isLegacy(){ + return this.isLegacy; + } + + /** + * Sets whether this repository is legacy. Scoped for moving from old-style repository naming to new + * + * @param isLegacy + */ + @Deprecated + @Experimental(feature= ExperimentalFeature.PATCH_UPGRADES) + public void setLegacy(boolean isLegacy){ + this.isLegacy = isLegacy; + } + } http://git-wip-us.apache.org/repos/asf/ambari/blob/412c6cac/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java index 3ab28e9..c6a85d0 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java @@ -309,23 +309,38 @@ public class RepositoryVersionHelper { /** * Adds a command repository to the action context * @param context the context - * @param osFamily the OS family + * @param osEntity the OS family * @param repoVersion the repository version entity - * @param repos the repository entities */ public void addCommandRepository(ActionExecutionContext context, RepositoryVersionEntity repoVersion, OperatingSystemEntity osEntity) { final CommandRepository commandRepo = new CommandRepository(); + boolean sysPreppedHost = configuration.get().areHostsSysPrepped().equalsIgnoreCase("true"); + commandRepo.setRepositories(osEntity.getOsType(), osEntity.getRepositories()); commandRepo.setRepositoryVersion(repoVersion.getVersion()); commandRepo.setRepositoryVersionId(repoVersion.getId()); commandRepo.setStackName(repoVersion.getStackId().getStackName()); + commandRepo.getFeature().setPreInstalled(configuration.get().areHostsSysPrepped()); + commandRepo.getFeature().setIsScoped(!sysPreppedHost); if (!osEntity.isAmbariManagedRepos()) { commandRepo.setNonManaged(); } else { - commandRepo.setUniqueSuffix(String.format("-repo-%s", repoVersion.getId())); + if (repoVersion.isLegacy()){ + commandRepo.setLegacyRepoFileName(repoVersion.getStackName(), repoVersion.getVersion()); + commandRepo.setLegacyRepoId(repoVersion.getVersion()); + commandRepo.getFeature().setIsScoped(false); + } else { + commandRepo.setRepoFileName(repoVersion.getStackName(), repoVersion.getId()); + commandRepo.setUniqueSuffix(String.format("-repo-%s", repoVersion.getId())); + } + } + + if (configuration.get().arePackagesLegacyOverridden()) { + LOG.warn("Legacy override option is turned on, disabling CommandRepositoryFeature.scoped feature"); + commandRepo.getFeature().setIsScoped(false); } context.addVisitor(new ExecutionCommandVisitor() { http://git-wip-us.apache.org/repos/asf/ambari/blob/412c6cac/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog260.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog260.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog260.java index afd3d1d..6b78dd6 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog260.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog260.java @@ -79,6 +79,7 @@ public class UpgradeCatalog260 extends AbstractUpgradeCatalog { public static final String REPO_VERSION_TABLE = "repo_version"; public static final String REPO_VERSION_ID_COLUMN = "repo_version_id"; public static final String REPO_VERSION_HIDDEN_COLUMN = "hidden"; + public static final String REPO_VERSION_LEGACY_COLUMN = "legacy"; public static final String HOST_COMPONENT_DESIRED_STATE_TABLE = "hostcomponentdesiredstate"; public static final String FK_HCDS_DESIRED_STACK_ID = "FK_hcds_desired_stack_id"; @@ -180,6 +181,7 @@ public class UpgradeCatalog260 extends AbstractUpgradeCatalog { createUpgradeHistoryTable(); updateRepositoryVersionTable(); renameServiceDeletedColumn(); + addLegacyColumn(); expandUpgradeItemItemTextColumn(); } @@ -191,6 +193,18 @@ public class UpgradeCatalog260 extends AbstractUpgradeCatalog { String.class, char[].class); } + private void addLegacyColumn() throws AmbariException, SQLException { + Boolean isLegacyColumnExists = dbAccessor.tableHasColumn(REPO_VERSION_TABLE, REPO_VERSION_LEGACY_COLUMN); + if (!isLegacyColumnExists) { + DBAccessor.DBColumnInfo legacyColumn = new DBAccessor.DBColumnInfo(REPO_VERSION_LEGACY_COLUMN, Boolean.class, null, 1, false); + dbAccessor.addColumn(REPO_VERSION_TABLE, legacyColumn); + + legacyColumn.setDefaultValue(0); + dbAccessor.alterColumn(REPO_VERSION_TABLE, legacyColumn); + } + } + + private void renameServiceDeletedColumn() throws AmbariException, SQLException { if (dbAccessor.tableHasColumn(CLUSTER_CONFIG_TABLE, SERVICE_DELETED_COLUMN)) { dbAccessor.renameColumn(CLUSTER_CONFIG_TABLE, SERVICE_DELETED_COLUMN, new DBAccessor.DBColumnInfo(UNMAPPED_COLUMN, Short.class, null, 0, false)); http://git-wip-us.apache.org/repos/asf/ambari/blob/412c6cac/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql index 796d9b2..cea350c 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql @@ -155,6 +155,7 @@ CREATE TABLE repo_version ( repositories VARCHAR(3000) NOT NULL, repo_type VARCHAR(255) DEFAULT 'STANDARD' NOT NULL, hidden SMALLINT NOT NULL DEFAULT 0, + legacy SMALLINT NOT NULL DEFAULT 0, version_url VARCHAR(1024), version_xml CLOB, version_xsd VARCHAR(512), http://git-wip-us.apache.org/repos/asf/ambari/blob/412c6cac/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql index c002601..6bb903f 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql @@ -175,6 +175,7 @@ CREATE TABLE repo_version ( repositories MEDIUMTEXT NOT NULL, repo_type VARCHAR(255) DEFAULT 'STANDARD' NOT NULL, hidden SMALLINT NOT NULL DEFAULT 0, + legacy TINYINT(1) NOT NULL DEFAULT 0, version_url VARCHAR(1024), version_xml MEDIUMTEXT, version_xsd VARCHAR(512), http://git-wip-us.apache.org/repos/asf/ambari/blob/412c6cac/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql index d78c1c7..543aee1 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql @@ -156,6 +156,7 @@ CREATE TABLE repo_version ( repositories CLOB NOT NULL, repo_type VARCHAR2(255) DEFAULT 'STANDARD' NOT NULL, hidden NUMBER(1) DEFAULT 0 NOT NULL, + legacy NUMBER(1) DEFAULT 0 NOT NULL, version_url VARCHAR(1024), version_xml CLOB, version_xsd VARCHAR(512), http://git-wip-us.apache.org/repos/asf/ambari/blob/412c6cac/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql index 90f1397..276cd2a 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql @@ -159,6 +159,7 @@ CREATE TABLE repo_version ( version_xml TEXT, version_xsd VARCHAR(512), parent_id BIGINT, + legacy BOOLEAN NOT NULL DEFAULT false, CONSTRAINT PK_repo_version PRIMARY KEY (repo_version_id), CONSTRAINT FK_repoversion_stack_id FOREIGN KEY (stack_id) REFERENCES stack(stack_id), CONSTRAINT UQ_repo_version_display_name UNIQUE (display_name), http://git-wip-us.apache.org/repos/asf/ambari/blob/412c6cac/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql index c4bb763..ea2c927 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql @@ -154,6 +154,7 @@ CREATE TABLE repo_version ( repositories TEXT NOT NULL, repo_type VARCHAR(255) DEFAULT 'STANDARD' NOT NULL, hidden SMALLINT NOT NULL DEFAULT 0, + legacy BIT NOT NULL DEFAULT 0, version_url VARCHAR(1024), version_xml TEXT, version_xsd VARCHAR(512), http://git-wip-us.apache.org/repos/asf/ambari/blob/412c6cac/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql index 9f7aa91..90014b6 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql @@ -168,6 +168,7 @@ CREATE TABLE repo_version ( repositories VARCHAR(MAX) NOT NULL, repo_type VARCHAR(255) DEFAULT 'STANDARD' NOT NULL, hidden SMALLINT NOT NULL DEFAULT 0, + legacy BIT NOT NULL DEFAULT 0, version_url VARCHAR(1024), version_xml VARCHAR(MAX), version_xsd VARCHAR(512), http://git-wip-us.apache.org/repos/asf/ambari/blob/412c6cac/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py b/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py index 189043b..5987adf 100644 --- a/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py +++ b/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py @@ -103,7 +103,7 @@ class InstallPackages(Script): self.repository_version = self.repository_version.strip() try: - if not command_repository.repositories: + if not command_repository.items: Logger.warning( "Repository list is empty. Ambari may not be managing the repositories for {0}.".format( self.repository_version)) @@ -330,7 +330,7 @@ class InstallPackages(Script): packages_were_checked = True filtered_package_list = self.filter_package_list(package_list) try: - available_packages_in_repos = self.pkg_provider.get_available_packages_in_repos(config['repositoryFile']['repositories']) + available_packages_in_repos = self.pkg_provider.get_available_packages_in_repos(CommandRepository(config['repositoryFile'])) except Exception: available_packages_in_repos = [] for package in filtered_package_list:
