This is an automated email from the ASF dual-hosted git repository. tvb pushed a commit to branch jonathan/mirror-client-sourcedownloader-tidy in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 105e10200140629203b76a34fbb743e823bf8b74 Author: Jonathan Maw <[email protected]> AuthorDate: Mon Apr 9 15:48:50 2018 +0100 project: Parse and store mirrors with default mirror set via config or command-line In user config (buildstream.conf), it is set with the "default-mirror" field. On the command-line, it is set with "--default-mirror" --- buildstream/_context.py | 6 ++++++ buildstream/_frontend/app.py | 3 ++- buildstream/_frontend/cli.py | 2 ++ buildstream/_project.py | 41 +++++++++++++++++++++++++++++++++++++--- tests/completions/completions.py | 1 + 5 files changed, 49 insertions(+), 4 deletions(-) diff --git a/buildstream/_context.py b/buildstream/_context.py index 114ac9e..bd9e458 100644 --- a/buildstream/_context.py +++ b/buildstream/_context.py @@ -102,6 +102,9 @@ class Context(): # What to do when a build fails in non interactive mode self.sched_error_action = 'continue' + # The default mirror to fetch from + self.default_mirror = None + # Whether elements must be rebuilt when their dependencies have changed self._strict_build_plan = None @@ -154,6 +157,7 @@ class Context(): _yaml.node_validate(defaults, [ 'sourcedir', 'builddir', 'artifactdir', 'logdir', 'scheduler', 'artifacts', 'logging', 'projects', + 'default-mirror', ]) for directory in ['sourcedir', 'builddir', 'artifactdir', 'logdir']: @@ -199,6 +203,8 @@ class Context(): # Load per-projects overrides self._project_overrides = _yaml.node_get(defaults, Mapping, 'projects', default_value={}) + self.default_mirror = _yaml.node_get(defaults, str, "default-mirror", default_value=None) + # Shallow validation of overrides, parts of buildstream which rely # on the overrides are expected to validate elsewhere. for _, overrides in _yaml.node_items(self._project_overrides): diff --git a/buildstream/_frontend/app.py b/buildstream/_frontend/app.py index d30b592..8f6dea7 100644 --- a/buildstream/_frontend/app.py +++ b/buildstream/_frontend/app.py @@ -203,7 +203,8 @@ class App(): # Load the Project # try: - self.project = Project(directory, self.context, cli_options=self._main_options['option']) + self.project = Project(directory, self.context, cli_options=self._main_options['option'], + default_mirror=self._main_options.get('default_mirror')) except LoadError as e: # Let's automatically start a `bst init` session in this case diff --git a/buildstream/_frontend/cli.py b/buildstream/_frontend/cli.py index 4651245..b52053c 100644 --- a/buildstream/_frontend/cli.py +++ b/buildstream/_frontend/cli.py @@ -217,6 +217,8 @@ def print_version(ctx, param, value): help="Elements must be rebuilt when their dependencies have changed") @click.option('--option', '-o', type=click.Tuple([str, str]), multiple=True, metavar='OPTION VALUE', help="Specify a project option") [email protected]('--default-mirror', default=None, + help="The mirror to fetch from first, before attempting other mirrors") @click.pass_context def cli(context, **kwargs): """Build and manipulate BuildStream projects diff --git a/buildstream/_project.py b/buildstream/_project.py index 9f42bf6..188bf27 100644 --- a/buildstream/_project.py +++ b/buildstream/_project.py @@ -20,7 +20,7 @@ import os import multiprocessing # for cpu_count() -from collections import Mapping +from collections import Mapping, OrderedDict from pluginbase import PluginBase from . import utils from . import _cachekey @@ -71,7 +71,7 @@ class HostMount(): # class Project(): - def __init__(self, directory, context, *, junction=None, cli_options=None): + def __init__(self, directory, context, *, junction=None, cli_options=None, default_mirror=None): # The project name self.name = None @@ -95,6 +95,9 @@ class Project(): self.base_env_nocache = None # The base nocache mask (list) for the environment self.element_overrides = {} # Element specific configurations self.source_overrides = {} # Source specific configurations + self.mirrors = OrderedDict() # contains dicts of alias-mappings to URIs. + + self.default_mirror = default_mirror or context.default_mirror # The name of the preferred mirror. # # Private Members @@ -203,6 +206,23 @@ class Project(): self._assert_plugin_format(source, version) return source + # get_alias_uris() + # + # Yields every URI to replace a given alias with + def get_alias_uris(self, alias): + if not alias or alias not in self._aliases: + return [None] + + mirror_list = [] + for key, alias_mapping in self.mirrors.items(): + if alias in alias_mapping: + if key == self.default_mirror: + mirror_list = alias_mapping[alias] + mirror_list + else: + mirror_list += alias_mapping[alias] + mirror_list.append(self._aliases[alias]) + return mirror_list + # _load(): # # Loads the project configuration file in the project directory. @@ -250,7 +270,7 @@ class Project(): 'aliases', 'name', 'artifacts', 'options', 'fail-on-overlap', 'shell', - 'ref-storage', 'sandbox' + 'ref-storage', 'sandbox', 'mirrors', ]) # The project name, element path and option declarations @@ -415,6 +435,21 @@ class Project(): self._shell_host_files.append(mount) + mirrors = _yaml.node_get(config, list, 'mirrors', default_value=[]) + for mirror in mirrors: + allowed_mirror_fields = [ + 'location-name', 'aliases' + ] + _yaml.node_validate(mirror, allowed_mirror_fields) + mirror_location = _yaml.node_get(mirror, str, 'location-name') + alias_mappings = {} + for alias_mapping, uris in _yaml.node_items(mirror['aliases']): + assert isinstance(uris, list) + alias_mappings[alias_mapping] = list(uris) + self.mirrors[mirror_location] = alias_mappings + if not self.default_mirror: + self.default_mirror = mirror_location + # _assert_plugin_format() # # Helper to raise a PluginError if the loaded plugin is of a lesser version then diff --git a/tests/completions/completions.py b/tests/completions/completions.py index 7c169c2..1ff026e 100644 --- a/tests/completions/completions.py +++ b/tests/completions/completions.py @@ -27,6 +27,7 @@ MAIN_OPTIONS = [ "--colors ", "--config ", "--debug ", + "--default-mirror ", "--directory ", "--error-lines ", "--fetchers ",
