This is an automated email from the ASF dual-hosted git repository. tvb pushed a commit to branch jonathan/mirror-client in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit b74966f66ca46ad1b5787879b23f53de30e3f198 Author: Jonathan Maw <[email protected]> AuthorDate: Thu May 24 15:37:25 2018 +0100 TIDY: Store mirrors as objects, not weird dicts --- buildstream/_mirror.py | 79 ++++++++++++++++++++++++++++++++++++++ buildstream/_project.py | 63 +++++++++++++++--------------- buildstream/plugins/sources/git.py | 1 - buildstream/source.py | 10 +++-- 4 files changed, 118 insertions(+), 35 deletions(-) diff --git a/buildstream/_mirror.py b/buildstream/_mirror.py new file mode 100644 index 0000000..d98a5b0 --- /dev/null +++ b/buildstream/_mirror.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2018 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Jonathan Maw <[email protected]> + + +from . import utils +from . import _yaml +from ._exceptions import ImplError, LoadError, LoadErrorReason + + +# MirrorKind() +# +# Defines the kind of mirrors that buildstream is capable of handling. +class MirrorKind(): + + # The default type of mirror, replace the alias + DEFAULT = 'default' + + # A mirror generated by buildstream + BST_GENERATED = 'bst-generated' + + +class Mirror(): + def __init__(self, node): + self.location = _yaml.node_get(node, str, "location-name") + + def get_mirror_uris(self, uri, source): + raise ImplError("Base mirror class does not implement get_mirror_uri") + + +class DefaultMirror(Mirror): + def __init__(self, node): + super().__init__(node) + allowed_fields = ['location-name', 'aliases', 'mirror-kind'] + _yaml.node_validate(node, allowed_fields) + + self.aliases = {} + for alias_mapping, uris in _yaml.node_items(node['aliases']): + assert isinstance(uris, list) + self.aliases[alias_mapping] = list(uris) + + def get_mirror_uris(self, uri, source): + url_prefix, url_body = uri.split(utils._ALIAS_SEPARATOR, 1) + for alias_uri in self.aliases.get(url_prefix, []): + yield alias_uri + url_body + + +class BstGeneratedMirror(Mirror): + def __init__(self, node): + super().__init__(node) + allowed_fields = [ + 'location-name', 'mirror-kind', 'site', 'aliases-covered' + ] + _yaml.node_validate(node, allowed_fields) + self.site = _yaml.node_get(node, str, 'site') + if '://' in self.site: + provenance = _yaml.node_get_provenance(node, key='site') + raise LoadError(LoadErrorReason.INVALID_DATA, + '{}: Site should not contain a URI prefix'.format(provenance)) + self.aliases_covered = _yaml.node_get(node, list, 'aliases-covered') + + def get_mirror_uris(self, uri, source): + yield source.get_normalised_mirror_path(uri, prefix=self.site) diff --git a/buildstream/_project.py b/buildstream/_project.py index 05e15c7..52fd36e 100644 --- a/buildstream/_project.py +++ b/buildstream/_project.py @@ -35,6 +35,7 @@ from ._sourcefactory import SourceFactory from ._projectrefs import ProjectRefs, ProjectRefStorage from ._versions import BST_FORMAT_VERSION from ._workspaces import Workspaces +from ._mirror import MirrorKind, DefaultMirror, BstGeneratedMirror # HostMount() @@ -245,30 +246,21 @@ class Project(): # We numerically address urls url = list(urls) - # Flatten the mirrors and put them in the right order - flattened_mirrors = {} - for alias in aliases: - flattened_mirrors[alias] = [] - for mirror_location, alias_mappings in self.mirrors.items(): - if alias in alias_mappings: - mapping_list = list(alias_mappings[alias]) - if mirror_location == self.default_mirror: - # The default mirror goes first - flattened_mirrors[alias] = mapping_list + flattened_mirrors[alias] - else: - flattened_mirrors[alias].extend(mapping_list) - flattened_mirrors[alias].append(self._aliases[alias]) + reordered_mirrors = OrderedDict(self.mirrors) + reordered_mirrors.move_to_end(self.default_mirror, last=False) # pylint: disable=no-member combinations = [[]] for url in urls: new_combinations = [] for combination in combinations: alias = urls_to_aliases[url] - for mirror_uri in flattened_mirrors[alias]: - # TODO: MAKE NICE - _, url_body = url.split(utils._ALIAS_SEPARATOR, 1) - new_url = mirror_uri + url_body - + for mirror in reordered_mirrors.values(): + for uri in mirror.get_mirror_uris(url, self): + new_combinations.append(combination + [uri]) + if alias in self._aliases: + default_alias = self._aliases[alias] + _, body = url.split(utils._ALIAS_SEPARATOR, 1) + new_url = default_alias + body new_combinations.append(combination + [new_url]) combinations = new_combinations @@ -493,20 +485,29 @@ class Project(): self._shell_host_files.append(mount) - mirrors = _yaml.node_get(config, list, 'mirrors', default_value=[]) - for mirror in mirrors: - allowed_mirror_fields = [ - 'location-name', 'aliases' - ] - _yaml.node_validate(mirror, allowed_mirror_fields) - mirror_location = _yaml.node_get(mirror, str, 'location-name') - alias_mappings = {} - for alias_mapping, uris in _yaml.node_items(mirror['aliases']): - assert isinstance(uris, list) - alias_mappings[alias_mapping] = list(uris) - self.mirrors[mirror_location] = alias_mappings + self._load_mirrors(config) + + def _load_mirrors(self, config): + mirrors_node = _yaml.node_get(config, list, 'mirrors', default_value=[]) + for mirror_node in mirrors_node: + # different kinds of mirror expect different fields + mirror_kind = _yaml.node_get(mirror_node, str, 'mirror-kind', + default_value=MirrorKind.DEFAULT) + if mirror_kind == MirrorKind.DEFAULT: + mirror = DefaultMirror(mirror_node) + self.mirrors[mirror.location] = mirror + + elif mirror_kind == MirrorKind.BST_GENERATED: + mirror = BstGeneratedMirror(mirror_node) + self.mirrors[mirror.location] = mirror + + else: + provenance = _yaml.node_get_provenance(mirror_node, key='mirror-kind') + raise LoadError(LoadErrorReason.INVALID_DATA, + "{}: Unexpected mirror-kind: {}".format(provenance, mirror_kind)) + if not self.default_mirror: - self.default_mirror = mirror_location + self.default_mirror = mirror.location # _assert_plugin_format() # diff --git a/buildstream/plugins/sources/git.py b/buildstream/plugins/sources/git.py index 42c5f7e..20393d3 100644 --- a/buildstream/plugins/sources/git.py +++ b/buildstream/plugins/sources/git.py @@ -107,7 +107,6 @@ class GitMirror(): # system configured tmpdir is not on the same partition. # with self.source.tempdir() as tmpdir: - self.source.info("*** url is '{}'".format(self.url)) self.source.call([self.source.host_git, 'clone', '--mirror', '-n', self.url, tmpdir], fail="Failed to clone git repository {}".format(self.url)) diff --git a/buildstream/source.py b/buildstream/source.py index 0428473..5f9f02a 100644 --- a/buildstream/source.py +++ b/buildstream/source.py @@ -304,7 +304,7 @@ class Source(Plugin): os.makedirs(directory, exist_ok=True) return directory - def get_normalised_mirror_path(self, upstream_url, *, prefix="", suffix=""): + def get_normalised_mirror_path(self, upstream_url, *, prefix=""): """Constructs a path for the mirror from the given URL Returns: @@ -313,7 +313,7 @@ class Source(Plugin): kind = self.get_kind() normalised_url = utils.url_directory_name(upstream_url) - return os.path.join(self.__protocol_prefix, prefix, kind, normalised_url, suffix) + return os.path.join(self.__protocol_prefix, prefix, kind, normalised_url) def translate_url(self, url): """Translates the given url which may be specified with an alias @@ -634,8 +634,12 @@ class Source(Plugin): if not self._used_urls: return False - context = self._get_context() + # Skip if we have no mirrors defined to fetch from project = self._get_project() + if not project.mirrors: + return False + + context = self._get_context() source_kind = type(self) for combination in project.generate_alias_combinations(self._used_urls): new_source = source_kind(context, project, self.__meta, uri_overrides=combination)
