This is an automated email from the ASF dual-hosted git repository. github-bot pushed a commit to branch jonathan/mirror-client in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 3ce7919f68f0ec427764298dbc279e1ceb421f21 Author: Jonathan Maw <[email protected]> AuthorDate: Thu Apr 12 16:37:28 2018 +0100 When fetching, try to fetch from mirrors first **KLUDGE WARNING**: This involves making the source store its "meta" object so that it's possible to create a copy of the source inside the fetch queue, instead of back when the pipeline was being loaded. --- buildstream/_scheduler/fetchqueue.py | 6 +++++- buildstream/source.py | 27 +++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/buildstream/_scheduler/fetchqueue.py b/buildstream/_scheduler/fetchqueue.py index 6105572..33e36f3 100644 --- a/buildstream/_scheduler/fetchqueue.py +++ b/buildstream/_scheduler/fetchqueue.py @@ -41,7 +41,11 @@ class FetchQueue(Queue): def process(self, element): for source in element.sources(): - source._fetch() + # Try to fetch from the mirrors first + source_fetched = source._mirrored_fetch() + # Fall back to the default + if not source_fetched: + source._fetch() def status(self, element): # state of dependencies may have changed, recalculate element state diff --git a/buildstream/source.py b/buildstream/source.py index 48803ee..9c70b00 100644 --- a/buildstream/source.py +++ b/buildstream/source.py @@ -137,6 +137,7 @@ class Source(Plugin): self.__consistency = Consistency.INCONSISTENT # Cached consistency state self.__alias_overrides = alias_overrides # Aliases to use instead of the one from the project self._expected_aliases = set() # A hacky way to store which aliases the source used + self.__meta = meta # MetaSource stored so we can copy this source later. # Collect the composited element configuration and # ask the element to configure itself. @@ -610,6 +611,32 @@ class Source(Plugin): return new_ref + # _mirrored_fetch(): + # + # Tries to fetch from every mirror, stopping once it succeeds + # + # Returns: + # (bool): True if it successfully fetched from a mirror. + # + def _mirrored_fetch(self): + # Mirrors can't do anything if this source doesn't use aliases + if not self._expected_aliases: + return False + + context = self._get_context() + project = self._get_project() + source_kind = type(self) + for combination in project.generate_alias_combinations(self._expected_aliases): + new_source = source_kind(context, project, self.__meta, alias_overrides=combination) + new_source._preflight() + try: + new_source._fetch() + except SourceError: + # SourceErrors from fetch are caused by network error + # or ref not found + continue + return True + ############################################################# # Local Private Methods # #############################################################
