This is an automated email from the ASF dual-hosted git repository. tvb pushed a commit to branch tristan/mirror-plugins in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 66f26c2ece5ddc64f35c13adefc5db1a578c7caf Author: Tristan van Berkom <[email protected]> AuthorDate: Mon Oct 2 14:16:04 2023 +0900 _project.py: Use SourceMirror objects --- src/buildstream/_pluginfactory/pluginfactory.py | 2 + src/buildstream/_project.py | 75 +++++++++++++++------- src/buildstream/_site.py | 3 + src/buildstream/source.py | 83 ++++++++++++++++++++----- 4 files changed, 124 insertions(+), 39 deletions(-) diff --git a/src/buildstream/_pluginfactory/pluginfactory.py b/src/buildstream/_pluginfactory/pluginfactory.py index fbcf544cc..101c45273 100644 --- a/src/buildstream/_pluginfactory/pluginfactory.py +++ b/src/buildstream/_pluginfactory/pluginfactory.py @@ -83,6 +83,8 @@ class PluginFactory: self._site_plugins_path = _site.source_plugins elif self._plugin_type == PluginType.ELEMENT: self._site_plugins_path = _site.element_plugins + elif self._plugin_type == PluginType.SOURCE_MIRROR: + self._site_plugins_path = _site.source_mirror_plugins self._site_source = self._plugin_base.make_plugin_source( searchpath=[self._site_plugins_path], diff --git a/src/buildstream/_project.py b/src/buildstream/_project.py index 1a1054898..2ed6fb348 100644 --- a/src/buildstream/_project.py +++ b/src/buildstream/_project.py @@ -15,7 +15,7 @@ # Tristan Van Berkom <[email protected]> # Tiago Gomes <[email protected]> -from typing import TYPE_CHECKING, Optional, Dict, Union, List +from typing import TYPE_CHECKING, Optional, Dict, Union, List, Tuple import os import urllib.parse @@ -31,14 +31,15 @@ from ._exceptions import LoadError from .exceptions import LoadErrorReason from ._options import OptionPool from .node import ScalarNode, MappingNode, ProvenanceInformation, _assert_symbol_name -from ._pluginfactory import ElementFactory, SourceFactory, load_plugin_origin -from .types import CoreWarnings, _HostMount, _SourceMirror, _SourceUriPolicy +from ._pluginfactory import ElementFactory, SourceFactory, SourceMirrorFactory, load_plugin_origin +from .types import CoreWarnings, _HostMount, _SourceUriPolicy from ._projectrefs import ProjectRefs, ProjectRefStorage from ._loader import Loader, LoadContext from .element import Element from ._includes import Includes from ._workspaces import WORKSPACE_PROJECT_FILE from ._remotespec import RemoteSpec +from .sourcemirror import SourceMirror if TYPE_CHECKING: @@ -56,7 +57,7 @@ class ProjectConfig: self.base_variables = {} # The base set of variables self.element_overrides = {} # Element specific configurations self.source_overrides = {} # Source specific configurations - self.mirrors = {} # Dictionary of _SourceAlias objects + self.mirrors = {} # Dictionary of SourceMirror objects self.default_mirror = None # The name of the preferred mirror. self._aliases = None # Aliases dictionary @@ -115,6 +116,7 @@ class Project: self.element_factory: Optional[ElementFactory] = None # ElementFactory for loading elements self.source_factory: Optional[SourceFactory] = None # SourceFactory for loading sources + self.source_mirror_factory: Optional[SourceMirrorFactory] = None # SourceMirrorFactory self.sandbox: Optional[MappingNode] = None self.splits: Optional[MappingNode] = None @@ -195,6 +197,28 @@ class Project: # Public Methods # ######################################################## + # get_alias_url(): + # + # Fetch the value of a URL alias + # + # Args: + # alias: The alias + # first_pass: Whether to use first pass configuration (for junctions) + # + # Returns: + # The alias substitution + # + def get_alias_url(self, alias: str, *, first_pass: bool = False) -> Optional[str]: + if first_pass: + config = self.first_pass_config + else: + config = self.config + + if config._aliases: + return config._aliases.get_str(alias, default=None) + + return None + # translate_url(): # # Translates the given url which may be specified with an alias @@ -211,14 +235,10 @@ class Project: # fully qualified urls based on the shorthand which is allowed # to be specified in the YAML def translate_url(self, url, *, first_pass=False): - if first_pass: - config = self.first_pass_config - else: - config = self.config if url and utils._ALIAS_SEPARATOR in url: url_alias, url_body = url.split(utils._ALIAS_SEPARATOR, 1) - alias_url = config._aliases.get_str(url_alias, default=None) + alias_url = self.get_alias_url(url_alias) if alias_url: url = alias_url + url_body @@ -382,35 +402,45 @@ class Project: # get_alias_uris() # # Args: - # alias (str): The alias. - # first_pass (bool): Whether to use first pass configuration (for junctions) - # tracking (bool): Whether we want the aliases for tracking (otherwise assume fetching) + # alias: The alias. + # first_pass: Whether to use first pass configuration (for junctions) + # tracking: Whether we want the aliases for tracking (otherwise assume fetching) + # + # Returns: + # A list of (SourceMirror, string) tuples with each alias substitution found along + # with the SourceMirror object which produced it. # - # Returns a list of every URI to replace an alias with - def get_alias_uris(self, alias, *, first_pass=False, tracking=False): + def get_alias_uris( + self, alias: str, *, first_pass: bool = False, tracking: bool = False + ) -> List[Tuple[Optional[SourceMirror], str]]: + if first_pass: config = self.first_pass_config else: config = self.config if not alias or alias not in config._aliases: # pylint: disable=unsupported-membership-test - return [None] + return [] - uri_list = [] + uri_list: List[Tuple[Optional[SourceMirror], str]] = [] policy = self._context.track_source if tracking else self._context.fetch_source if policy in (_SourceUriPolicy.ALL, _SourceUriPolicy.MIRRORS) or ( policy == _SourceUriPolicy.USER and self._mirror_override ): for mirror_name, mirror in config.mirrors.items(): - if alias in mirror.aliases: + mirror_uri_list = mirror._get_alias_uris(alias) + if mirror_uri_list: + + list_to_add = [(mirror, uri) for uri in mirror_uri_list] + if mirror_name == config.default_mirror: - uri_list = mirror.aliases[alias] + uri_list + uri_list = list_to_add + uri_list else: - uri_list += mirror.aliases[alias] + uri_list += list_to_add if policy in (_SourceUriPolicy.ALL, _SourceUriPolicy.ALIASES): - uri_list.append(config._aliases.get_str(alias)) + uri_list.append((None, config._aliases.get_str(alias))) return uri_list @@ -1025,9 +1055,9 @@ class Project: # even if the mirrors are specified in user configuration. variables.expand(mirrors_node) - # Collect _SourceMirror objects + # Collect SourceMirror objects for mirror_node in mirrors_node: - mirror = _SourceMirror.new_from_node(mirror_node) + mirror = self.source_mirror_factory.create(self._context, self, mirror_node) output.mirrors[mirror.name] = mirror if not output.default_mirror: output.default_mirror = mirror.name @@ -1087,6 +1117,7 @@ class Project: pluginbase = PluginBase(package="buildstream.plugins") self.element_factory = ElementFactory(pluginbase) self.source_factory = SourceFactory(pluginbase) + self.source_mirror_factory = SourceMirrorFactory(pluginbase) # Load the plugin origins and register them to their factories origins = config.get_sequence("plugins", default=[]) diff --git a/src/buildstream/_site.py b/src/buildstream/_site.py index a75faaa7d..c548d7ad9 100644 --- a/src/buildstream/_site.py +++ b/src/buildstream/_site.py @@ -30,6 +30,9 @@ element_plugins = os.path.join(root, "plugins", "elements") # The Source plugin directory source_plugins = os.path.join(root, "plugins", "sources") +# The SourceMirror plugin directory +source_mirror_plugins = os.path.join(root, "plugins", "sourcemirrors") + # Default user configuration default_user_config = os.path.join(root, "data", "userconfig.yaml") diff --git a/src/buildstream/source.py b/src/buildstream/source.py index 0278c5a32..509be7445 100644 --- a/src/buildstream/source.py +++ b/src/buildstream/source.py @@ -232,6 +232,7 @@ from .storage import CasBasedDirectory from .storage import FileBasedDirectory from .storage.directory import Directory from ._variables import Variables +from .sourcemirror import SourceMirror if TYPE_CHECKING: from typing import Any, Dict, Set @@ -415,6 +416,9 @@ class Source(Plugin): # Set of marked download URLs self.__marked_urls = set() # type: Set[str] + # The active SourceMirror in context of fetch/track + self.__active_mirror: Optional[SourceMirror] = None + # Collect the composited element configuration and # ask the element to configure itself. self.__init_defaults(project, meta) @@ -714,28 +718,55 @@ class Source(Plugin): :func:`Source.mark_download_url() <buildstream.source.Source.mark_download_url>` is not called. """ + project = self._get_project() + # Ensure that the download URL is also marked self.mark_download_url(url, primary=primary) + # XXX + # + # Here we are called either: + # * From a source fetcher with "alias_override" specified + # * From a cloned source with self.__alias_override specified + # * From the default source, with a possibly aliased source + # * In this last case, mirrors are ignored + # + # In both relevant cases, we need to have the SourceMirror in context, and + # then we can delegate the translation to the SourceMirror while providing + # the original "url" to the SourceMirror method. + # + # Use the `self.__active_mirror` object + # Alias overriding can happen explicitly (by command-line) or # implicitly (the Source being constructed with an __alias_override). + # if alias_override or self.__alias_override: + + assert self.__active_mirror is not None + url_alias, url_body = url.split(utils._ALIAS_SEPARATOR, 1) - if url_alias: - if alias_override: - url = alias_override + url_body - else: - # Implicit alias overrides may only be done for one - # specific alias, so that sources that fetch from multiple - # URLs and use different aliases default to only overriding - # one alias, rather than getting confused. - override_alias = self.__alias_override[0] # type: ignore - override_url = self.__alias_override[1] # type: ignore - if url_alias == override_alias: - url = override_url + url_body - return url + project_alias_url = project.get_alias_url(url_alias, first_pass=self.__first_pass) + + if self.__alias_override is not None: + override_alias = self.__alias_override[0] # type: ignore + override_url = self.__alias_override[1] # type: ignore + + # Implicit alias overrides may only be done for one + # specific alias, so that sources that fetch from multiple + # URLs and use different aliases default to only overriding + # one alias, rather than getting confused. + # + if url_alias != override_alias: + return url + + alias_override = override_url + + # + # Delegate the URL translation to the SourceMirror plugin + # + return self.__active_mirror.translate_url(project.name, url_alias, project_alias_url, alias_override, url_body) + else: - project = self._get_project() return project.translate_url(url, first_pass=self.__first_pass) def mark_download_url(self, url: str, *, primary: bool = True) -> None: @@ -1396,7 +1427,10 @@ class Source(Plugin): alias = fetcher._get_alias() last_error = None - for uri in project.get_alias_uris(alias, first_pass=self.__first_pass, tracking=False): + for mirror, uri in project.get_alias_uris(alias, first_pass=self.__first_pass, tracking=False): + + self.__active_mirror = mirror + try: fetcher.fetch(uri) # FIXME: Need to consider temporary vs. permanent failures, @@ -1406,10 +1440,12 @@ class Source(Plugin): continue # No error, we're done with this fetcher + self.__active_mirror = None break else: # No break occurred, raise the last detected error + self.__active_mirror = None raise last_error # Default codepath is to reinstantiate the Source @@ -1425,7 +1461,10 @@ class Source(Plugin): return last_error = None - for uri in project.get_alias_uris(alias, first_pass=self.__first_pass, tracking=False): + for mirror, uri in project.get_alias_uris(alias, first_pass=self.__first_pass, tracking=False): + + self.__active_mirror = mirror + new_source = self.__clone_for_uri(uri) try: new_source.fetch(**kwargs) @@ -1436,8 +1475,11 @@ class Source(Plugin): continue # No error, we're done here + self.__active_mirror = None return + self.__active_mirror = None + # Re raise the last detected error raise last_error @@ -1456,7 +1498,10 @@ class Source(Plugin): # NOTE: We are assuming here that tracking only requires substituting the # first alias used last_error = None - for uri in reversed(project.get_alias_uris(alias, first_pass=self.__first_pass, tracking=True)): + for mirror, uri in reversed(project.get_alias_uris(alias, first_pass=self.__first_pass, tracking=True)): + + self.__active_mirror = mirror + new_source = self.__clone_for_uri(uri) try: ref = new_source.track(**kwargs) # pylint: disable=assignment-from-none @@ -1465,7 +1510,11 @@ class Source(Plugin): except BstError as e: last_error = e continue + + self.__active_mirror = None return ref + + self.__active_mirror = None raise last_error @classmethod
