Repository: ambari Updated Branches: refs/heads/branch-2.6 2c181ac20 -> 83cbde7d7
Revert "AMBARI-22561. Need to address HDP-GPL repo update after user accepts license in post-install scenario. Breaks HDF deploy. (aonishuk)" This reverts commit 3ac717a497efc60ca9b51f8ca7bef21f3df26fb3. Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/613dccbd Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/613dccbd Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/613dccbd Branch: refs/heads/branch-2.6 Commit: 613dccbd202374bae1664c5f09c41e1b97d77410 Parents: 2c181ac Author: Siddharth Wagle <swa...@hortonworks.com> Authored: Mon Dec 4 10:35:29 2017 -0800 Committer: Siddharth Wagle <swa...@hortonworks.com> Committed: Mon Dec 4 10:35:29 2017 -0800 ---------------------------------------------------------------------- .../libraries/functions/lzo_utils.py | 9 +- .../libraries/functions/repository_util.py | 92 ++++++++------------ .../libraries/script/script.py | 10 +-- .../ambari/server/agent/CommandRepository.java | 6 -- .../custom_actions/scripts/install_packages.py | 8 +- .../scripts/repo_initialization.py | 9 +- .../src/test/python/stacks/utils/RMFTestCase.py | 2 - 7 files changed, 51 insertions(+), 85 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/613dccbd/ambari-common/src/main/python/resource_management/libraries/functions/lzo_utils.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/lzo_utils.py b/ambari-common/src/main/python/resource_management/libraries/functions/lzo_utils.py index c505969..68ee607 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/lzo_utils.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/lzo_utils.py @@ -54,9 +54,6 @@ def get_lzo_packages(): return lzo_packages -def is_gpl_license_accepted(): - return default("/hostLevelParams/gpl_license_accepted", False) - def should_install_lzo(): """ Return true if lzo is enabled via core-site.xml and GPL license (required for lzo) is accepted. @@ -68,7 +65,8 @@ def should_install_lzo(): if not lzo_enabled: return False - if not is_gpl_license_accepted(): + is_gpl_license_accepted = default("/hostLevelParams/gpl_license_accepted", False) + if not is_gpl_license_accepted: Logger.warning(INSTALLING_LZO_WITHOUT_GPL) return False @@ -81,9 +79,6 @@ def install_lzo_if_needed(): if not should_install_lzo(): return - # If user has just accepted GPL license. GPL repository can not yet be present. - Script.repository_util.create_repo_files() - lzo_packages = get_lzo_packages() config = Script.get_config() http://git-wip-us.apache.org/repos/asf/ambari/blob/613dccbd/ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py b/ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py index 5d73b5d..f1c8ef1 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py @@ -17,75 +17,60 @@ limitations under the License. """ -from ambari_commons.os_check import OSCheck from resource_management.core.exceptions import Fail from resource_management.core.logger import Logger from resource_management.libraries.resources.repository import Repository -from resource_management.libraries.functions.is_empty import is_empty import ambari_simplejson as json -__all__ = ["RepositoryUtil", "CommandRepository"] +__all__ = ["create_repo_files", "CommandRepository"] # components_lits = repoName + postfix UBUNTU_REPO_COMPONENTS_POSTFIX = "main" -class RepositoryUtil: - def __init__(self, config, tags_to_skip): - self.tags_to_skip = tags_to_skip - # repo templates - repo_file = config['repositoryFile'] - repo_rhel_suse = config['configurations']['cluster-env']['repo_suse_rhel_template'] - repo_ubuntu = config['configurations']['cluster-env']['repo_ubuntu_template'] +def create_repo_files(template, command_repository): + """ + Creates repositories in a consistent manner for all types + :param command_repository: a CommandRepository instance + :type command_repository CommandRepository + :return: a dictionary with repo ID => repo file name mapping + """ - if is_empty(repo_file): - return + if command_repository.version_id is None: + raise Fail("The command repository was not parsed correctly") - self.template = repo_rhel_suse if OSCheck.is_redhat_family() or OSCheck.is_suse_family() else repo_ubuntu - self.command_repository = CommandRepository(repo_file) + if 0 == len(command_repository.items): + Logger.warning( + "Repository for {0}/{1} has no repositories. Ambari may not be managing this version.".format( + command_repository.stack_name, command_repository.version_string)) + return {} - def create_repo_files(self): - """ - Creates repositories in a consistent manner for all types - :return: a dictionary with repo ID => repo file name mapping - """ - if self.command_repository.version_id is None: - raise Fail("The command repository was not parsed correctly") + append_to_file = False # initialize to False to create the file anew. + repo_files = {} - if 0 == len(self.command_repository.items): + for repository in command_repository.items: + + if repository.repo_id is None: + raise Fail("Repository with url {0} has no id".format(repository.base_url)) + + if not repository.ambari_managed: Logger.warning( - "Repository for {0}/{1} has no repositories. Ambari may not be managing this version.".format( - self.command_repository.stack_name, self.command_repository.version_string)) - return {} - - append_to_file = False # initialize to False to create the file anew. - repo_files = {} - for repository in self.command_repository.items: - if repository.repo_id is None: - raise Fail("Repository with url {0} has no id".format(repository.base_url)) - - if self.tags_to_skip & repository.tags: - Logger.info("Repository with url {0} is not created due to its tags: {1}".format(repository.base_url, repository.tags)) - continue - - if not repository.ambari_managed: - Logger.warning( - "Repository for {0}/{1}/{2} is not managed by Ambari".format( - self.command_repository.stack_name, self.command_repository.version_string, repository.repo_id)) - else: - Repository(repository.repo_id, - action="create", - base_url=repository.base_url, - mirror_list=repository.mirrors_list, - repo_file_name=self.command_repository.repo_filename, - repo_template=self.template, - components=repository.ubuntu_components, - append_to_file=append_to_file) - append_to_file = True - repo_files[repository.repo_id] = self.command_repository.repo_filename - - return repo_files + "Repository for {0}/{1}/{2} is not managed by Ambari".format( + command_repository.stack_name, command_repository.version_string, repository.repo_id)) + else: + Repository(repository.repo_id, + action="create", + base_url=repository.base_url, + mirror_list=repository.mirrors_list, + repo_file_name=command_repository.repo_filename, + repo_template=template, + components=repository.ubuntu_components, + append_to_file=append_to_file) + append_to_file = True + repo_files[repository.repo_id] = command_repository.repo_filename + + return repo_files def _find_value(dictionary, key, default=None): @@ -161,7 +146,6 @@ class CommandRepositoryItem(object): self.components = _find_value(json_dict, 'components') self.base_url = _find_value(json_dict, 'baseUrl') self.mirrors_list = _find_value(json_dict, 'mirrorsList') - self.tags = set(_find_value(json_dict, 'tags', default=[])) self.ambari_managed = _find_value(json_dict, 'ambariManaged', default=True) self.ubuntu_components = [self.distribution if self.distribution else self.repo_name] + \ http://git-wip-us.apache.org/repos/asf/ambari/blob/613dccbd/ambari-common/src/main/python/resource_management/libraries/script/script.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py b/ambari-common/src/main/python/resource_management/libraries/script/script.py index fbb90a0..ccb8a54 100644 --- a/ambari-common/src/main/python/resource_management/libraries/script/script.py +++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py @@ -52,7 +52,7 @@ from resource_management.libraries.functions.version import format_stack_version from resource_management.libraries.functions import stack_tools from resource_management.libraries.functions.constants import Direction from resource_management.libraries.script.config_dictionary import ConfigDictionary, UnknownConfiguration -from resource_management.libraries.functions.repository_util import CommandRepository, RepositoryUtil +from resource_management.libraries.functions.repository_util import CommandRepository from resource_management.core.resources.system import Execute from contextlib import closing from resource_management.libraries.functions.stack_features import check_stack_feature @@ -355,14 +355,6 @@ class Script(object): Logger.logger.exception("Can not read json file with command parameters: ") sys.exit(1) - from resource_management.libraries.functions import lzo_utils - - repo_tags_to_skip = set() - if not lzo_utils.is_gpl_license_accepted(): - repo_tags_to_skip.add("GPL") - - Script.repository_util = RepositoryUtil(Script.config, repo_tags_to_skip) - # Run class method depending on a command type try: method = self.choose_method_to_execute(self.command_name) http://git-wip-us.apache.org/repos/asf/ambari/blob/613dccbd/ambari-server/src/main/java/org/apache/ambari/server/agent/CommandRepository.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/CommandRepository.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/CommandRepository.java index 0777edd..e207ac5 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/agent/CommandRepository.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/CommandRepository.java @@ -20,13 +20,11 @@ package org.apache.ambari.server.agent; import java.util.ArrayList; import java.util.Collection; import java.util.List; -import java.util.Set; import org.apache.ambari.annotations.Experimental; import org.apache.ambari.annotations.ExperimentalFeature; import org.apache.ambari.server.orm.entities.RepositoryEntity; import org.apache.ambari.server.state.RepositoryInfo; -import org.apache.ambari.server.state.stack.RepoTag; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.builder.ToStringBuilder; @@ -258,9 +256,6 @@ public class CommandRepository { @SerializedName("mirrorsList") private String m_mirrorsList; - @SerializedName("tags") - private Set<RepoTag> m_tags; - @SerializedName("applicableServices") @Experimental(feature = ExperimentalFeature.CUSTOM_SERVICE_REPOS, comment = "Remove logic for handling custom service repos after enabling multi-mpack cluster deployment") @@ -288,7 +283,6 @@ public class CommandRepository { m_mirrorsList = entity.getMirrorsList(); m_osType = osType; m_applicableServices = entity.getApplicableServices(); - m_tags = entity.getTags(); } public void setRepoId(String repoId){ http://git-wip-us.apache.org/repos/asf/ambari/blob/613dccbd/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py b/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py index c8497cd..f2218e5 100644 --- a/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py +++ b/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py @@ -37,7 +37,7 @@ from resource_management.libraries.functions.repo_version_history \ from resource_management.core.providers import get_provider from resource_management.core.resources.system import Link from resource_management.libraries.functions import StackFeature -from resource_management.libraries.functions.repository_util import CommandRepository +from resource_management.libraries.functions.repository_util import create_repo_files, CommandRepository from resource_management.libraries.functions.stack_features import check_stack_feature from resource_management.libraries.resources.repository import Repository from resource_management.libraries.script.script import Script @@ -72,6 +72,10 @@ class InstallPackages(Script): except KeyError: raise Fail("The command repository indicated by 'repositoryFile' was not found") + repo_rhel_suse = config['configurations']['cluster-env']['repo_suse_rhel_template'] + repo_ubuntu = config['configurations']['cluster-env']['repo_ubuntu_template'] + template = repo_rhel_suse if OSCheck.is_redhat_family() or OSCheck.is_suse_family() else repo_ubuntu + # Handle a SIGTERM and SIGINT gracefully signal.signal(signal.SIGTERM, self.abort_handler) signal.signal(signal.SIGINT, self.abort_handler) @@ -108,7 +112,7 @@ class InstallPackages(Script): else: Logger.info( "Will install packages for repository version {0}".format(self.repository_version)) - new_repo_files = Script.repository_util.create_repo_files() + new_repo_files = create_repo_files(template, command_repository) self.repo_files.update(new_repo_files) except Exception as err: Logger.logger.exception("Cannot install repository files. Error: {0}".format(str(err))) http://git-wip-us.apache.org/repos/asf/ambari/blob/613dccbd/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/repo_initialization.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/repo_initialization.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/repo_initialization.py index a329167..82e57aa 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/repo_initialization.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/repo_initialization.py @@ -19,8 +19,7 @@ limitations under the License. from ambari_commons.os_check import OSCheck from resource_management.libraries.resources.repository import Repository -from resource_management.libraries.functions.repository_util import CommandRepository, UBUNTU_REPO_COMPONENTS_POSTFIX -from resource_management.libraries.script.script import Script +from resource_management.libraries.functions.repository_util import create_repo_files, CommandRepository, UBUNTU_REPO_COMPONENTS_POSTFIX from resource_management.core.logger import Logger import ambari_simplejson as json @@ -63,13 +62,13 @@ def install_repos(): if params.host_sys_prepped: return + template = params.repo_rhel_suse if OSCheck.is_suse_family() or OSCheck.is_redhat_family() else params.repo_ubuntu + # use this newer way of specifying repositories, if available if params.repo_file is not None: - Script.repository_util.create_repo_files() + create_repo_files(template, CommandRepository(params.repo_file)) return - template = params.repo_rhel_suse if OSCheck.is_suse_family() or OSCheck.is_redhat_family() else params.repo_ubuntu - _alter_repo("create", params.repo_info, template) if params.service_repo_info: http://git-wip-us.apache.org/repos/asf/ambari/blob/613dccbd/ambari-server/src/test/python/stacks/utils/RMFTestCase.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/utils/RMFTestCase.py b/ambari-server/src/test/python/stacks/utils/RMFTestCase.py index 5e7f41e..80712c5 100644 --- a/ambari-server/src/test/python/stacks/utils/RMFTestCase.py +++ b/ambari-server/src/test/python/stacks/utils/RMFTestCase.py @@ -38,7 +38,6 @@ with patch("platform.linux_distribution", return_value = ('Suse','11','Final')): from resource_management.libraries.script.config_dictionary import ConfigDictionary from resource_management.libraries.script.script import Script from resource_management.libraries.script.config_dictionary import UnknownConfiguration - from resource_management.libraries.functions.repository_util import RepositoryUtil PATH_TO_STACKS = "main/resources/stacks/HDP" PATH_TO_STACK_TESTS = "test/python/stacks/" @@ -124,7 +123,6 @@ class RMFTestCase(TestCase): script_class_inst = RMFTestCase._get_attr(script_module, classname)() script_class_inst.log_out_files = log_out_files script_class_inst.available_packages_in_repos = available_packages_in_repos - Script.repository_util = RepositoryUtil(self.config_dict, set()) method = RMFTestCase._get_attr(script_class_inst, command) except IOError, err: raise RuntimeError("Cannot load class %s from %s: %s" % (classname, norm_path, err.message))
