This is an automated email from the ASF dual-hosted git repository. akitouni pushed a commit to branch abderrahim/buildstream-mirrors-merge in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 2b6fa5037dfe6112fe3deded86ea37a731fa87b0 Author: Tristan van Berkom <[email protected]> AuthorDate: Fri Sep 22 13:13:39 2023 +0900 Adding source mirror plugins --- src/buildstream/__init__.py | 1 + src/buildstream/_pluginfactory/__init__.py | 1 + src/buildstream/_pluginfactory/pluginfactory.py | 3 + src/buildstream/_pluginfactory/pluginorigin.py | 9 +- .../_pluginfactory/sourcemirrorfactory.py | 71 ++++++++ src/buildstream/sourcemirror.py | 193 +++++++++++++++++++++ 6 files changed, 277 insertions(+), 1 deletion(-) diff --git a/src/buildstream/__init__.py b/src/buildstream/__init__.py index 1c51bd647..24177d58e 100644 --- a/src/buildstream/__init__.py +++ b/src/buildstream/__init__.py @@ -32,6 +32,7 @@ if "_BST_COMPLETION" not in os.environ: from .node import MappingNode, Node, ProvenanceInformation, ScalarNode, SequenceNode from .plugin import Plugin from .source import Source, SourceError, SourceFetcher + from .sourcemirror import SourceMirror from .downloadablefilesource import DownloadableFileSource from .element import Element, ElementError, DependencyConfiguration from .buildelement import BuildElement diff --git a/src/buildstream/_pluginfactory/__init__.py b/src/buildstream/_pluginfactory/__init__.py index 28b4f3d7d..bc3f798b8 100644 --- a/src/buildstream/_pluginfactory/__init__.py +++ b/src/buildstream/_pluginfactory/__init__.py @@ -18,6 +18,7 @@ from .pluginoriginpip import PluginOriginPip from .pluginoriginjunction import PluginOriginJunction from .sourcefactory import SourceFactory from .elementfactory import ElementFactory +from .sourcemirrorfactory import SourceMirrorFactory # load_plugin_origin() diff --git a/src/buildstream/_pluginfactory/pluginfactory.py b/src/buildstream/_pluginfactory/pluginfactory.py index fbcf544cc..de3223085 100644 --- a/src/buildstream/_pluginfactory/pluginfactory.py +++ b/src/buildstream/_pluginfactory/pluginfactory.py @@ -22,6 +22,7 @@ from .. import utils from .. import _site from ..plugin import Plugin from ..source import Source +from ..sourcemirror import SourceMirror from ..element import Element from ..node import Node from ..utils import UtilError @@ -314,6 +315,8 @@ class PluginFactory: base_type = Source elif self._plugin_type == PluginType.ELEMENT: base_type = Element + elif self._plugin_type == PluginType.SOURCE_MIRROR: + base_type = SourceMirror try: if not issubclass(plugin_type, base_type): diff --git a/src/buildstream/_pluginfactory/pluginorigin.py b/src/buildstream/_pluginfactory/pluginorigin.py index 9dfaf55d5..fd0e42892 100644 --- a/src/buildstream/_pluginfactory/pluginorigin.py +++ b/src/buildstream/_pluginfactory/pluginorigin.py @@ -30,6 +30,9 @@ class PluginType(FastEnum): # An Element plugin ELEMENT = "element" + # A SourceMirror plugin + SOURCE_MIRROR = "source-mirror" + def __str__(self): return str(self.value) @@ -68,7 +71,7 @@ class PluginConfiguration: class PluginOrigin: # Common fields valid for all plugin origins - _COMMON_CONFIG_KEYS = ["origin", "sources", "elements", "allow-deprecated"] + _COMMON_CONFIG_KEYS = ["origin", "sources", "elements", "source-mirrors", "allow-deprecated"] def __init__(self, origin_type): @@ -76,6 +79,7 @@ class PluginOrigin: self.origin_type = origin_type # The PluginOriginType self.elements = {} # A dictionary of PluginConfiguration self.sources = {} # A dictionary of PluginConfiguration objects + self.source_mirrors = {} # A dictionary of PluginConfiguration objects self.provenance_node = None self.project = None @@ -112,6 +116,9 @@ class PluginOrigin: source_sequence = origin_node.get_sequence("sources", []) self._load_plugin_configurations(source_sequence, self.sources) + source_mirror_sequence = origin_node.get_sequence("source-mirrors", []) + self._load_plugin_configurations(source_mirror_sequence, self.source_mirrors) + ############################################## # Abstract methods # ############################################## diff --git a/src/buildstream/_pluginfactory/sourcemirrorfactory.py b/src/buildstream/_pluginfactory/sourcemirrorfactory.py new file mode 100644 index 000000000..5060fd71d --- /dev/null +++ b/src/buildstream/_pluginfactory/sourcemirrorfactory.py @@ -0,0 +1,71 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Authors: +# Tristan Van Berkom <[email protected]> + +from typing import TYPE_CHECKING, Type, cast + +from .pluginfactory import PluginFactory +from .pluginorigin import PluginType + +from ..node import MappingNode +from ..plugin import Plugin +from ..sourcemirror import SourceMirror + +if TYPE_CHECKING: + from .._context import Context + from .._project import Project + + +# A SourceMirrorFactory creates SourceMirror instances +# in the context of a given factory +# +# Args: +# plugin_base (PluginBase): The main PluginBase object to work with +# +class SourceMirrorFactory(PluginFactory): + def __init__(self, plugin_base): + super().__init__(plugin_base, PluginType.SOURCE_MIRROR) + + # create(): + # + # Create a SourceMirror object. + # + # Args: + # context (object): The Context object for processing + # project (object): The project object + # node (MappingNode): The node where the mirror was defined + # + # Returns: + # A newly created SourceMirror object of the appropriate kind + # + # Raises: + # PluginError (if the kind lookup failed) + # LoadError (if the source mirror itself took issue with the config) + # + def create(self, context: "Context", project: "Project", node: MappingNode) -> SourceMirror: + plugin_type: Type[Plugin] + + # Shallow parsing to get the custom plugin type, delegate the remainder + # of the parsing to SourceMirror + # + kind = node.get_str("kind", None) + if kind is None: + plugin_type = SourceMirror + else: + plugin_type, _ = self.lookup(context.messenger, kind, node) + + source_mirror_type = cast(Type[SourceMirror], plugin_type) + source_mirror = source_mirror_type(context, project, node) + return source_mirror diff --git a/src/buildstream/sourcemirror.py b/src/buildstream/sourcemirror.py new file mode 100644 index 000000000..a535fe982 --- /dev/null +++ b/src/buildstream/sourcemirror.py @@ -0,0 +1,193 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Authors: +# Tristan Van Berkom <[email protected]> +""" +SourceMirror - Base source mirror class +======================================= +The SourceMirror plugin allows one to customize how +:func:`Source.translate_url() <buildstream.source.Source.translate_url>` will +behave when looking up mirrors, allowing some additional flexibility in the +implementation of source mirrors. + + +.. _core_source_mirror_abstract_methods: + +Abstract Methods +---------------- +For loading and configuration purposes, SourceMirrors may optionally implement +the :func:`Plugin base class Plugin.configure() method <buildstream.plugin.Plugin.configure>` +in order to load any custom configuration in the `config` dictionary. + +The remaining :ref:`Plugin base class abstract methods <core_plugin_abstract_methods>` are +not relevant to the SourceMirror plugin object and need not be implemented. + +Sources expose the following abstract methods. Unless explicitly mentioned, +these methods are mandatory to implement. + +* :func:`SourceMirror.translate_url() <buildstream.source.SourceMirror.translate_url>` + + Produce an appropriate URL for the given URL and alias. + + +Class Reference +--------------- +""" + +from typing import Optional, Dict, List, TYPE_CHECKING + +from .node import MappingNode, SequenceNode +from .plugin import Plugin +from ._exceptions import BstError, LoadError +from .exceptions import ErrorDomain, LoadErrorReason + +if TYPE_CHECKING: + + # pylint: disable=cyclic-import + from ._context import Context + from ._project import Project + + # pylint: enable=cyclic-import + + +class SourceMirrorError(BstError): + """This exception should be raised by :class:`.SourceMirror` implementations + to report errors to the user. + + Args: + message: The breif error description to report to the user + detail: A possibly multiline, more detailed error message + reason: An optional machine readable reason string, used for test cases + + *Since: 2.2* + """ + + def __init__( + self, message: str, *, detail: Optional[str] = None, reason: Optional[str] = None, temporary: bool = False + ): + super().__init__(message, detail=detail, domain=ErrorDomain.SOURCE, reason=reason) + + +class SourceMirror(Plugin): + """SourceMirror() + + Base SourceMirror class. + + All SourceMirror plugins derive from this class, this interface defines how + the core will be interacting with SourceMirror plugins. + + *Since: 2.2* + """ + + # The SourceMirror plugin type is only supported since BuildStream 2.2 + BST_MIN_VERSION = "2.2" + + def __init__( + self, + context: "Context", + project: "Project", + node: MappingNode, + ): + # Note: the MappingNode passed here is already expanded with + # the project level base variables, so there is no need + # to expand them redundantly here. + # + node.validate_keys(["name", "kind", "config", "aliases"]) + + # Do local base class parsing first + name: str = node.get_str("name") + self.__aliases: Dict[str, List[str]] = self.__load_aliases(node) + + # Chain up to Plugin + super().__init__(name, context, project, node, "source-mirror") + + # Plugin specific parsing + config = node.get_mapping("config", default={}) + self._configure(config) + + ########################################################## + # Public API # + ########################################################## + def translate_url( + self, project_name: str, alias: str, alias_url: str, alias_substitute_url: Optional[str], source_url: str + ) -> str: + """Produce an alternative url for `url` for the given alias. + + Args: + project_name: The name of the project this URL comes from + alias: The alias to translate for + alias_url: The default URL configured for this alias in the originating project + alias_substitute_url: The alias substitute URL configured in the mirror configuration, or None + source_url: The URL as specified by original source YAML, excluding the alias + """ + # + # Default implementation behaves in the same way we behaved before + # introducing the SourceMirror plugin. + # + assert alias_substitute_url is not None + + return alias_substitute_url + source_url + + ############################################################# + # Plugin API implementation # + ############################################################# + + # + # Provide a dummy implementation as the base class is used as a default + # + def configure(self, node: MappingNode) -> None: + pass + + ########################################################## + # Internal API # + ########################################################## + + # _get_alias_uris(): + # + # Get a list of URIs for the specified alias + # + # Args: + # alias: The alias to fetch URIs for + # + # Returns: + # A list of URIs for the given alias + # + def _get_alias_uris(self, alias: str) -> List[str]: + + aliases: List[str] + try: + aliases = self.__aliases[alias] + except KeyError: + aliases = [] + + return aliases + + ########################################################## + # Private API # + ########################################################## + def __load_aliases(self, node: MappingNode) -> Dict[str, List[str]]: + + aliases: Dict[str, List[str]] = {} + alias_node: MappingNode = node.get_mapping("aliases") + + for alias, uris in alias_node.items(): + if not isinstance(uris, SequenceNode): + raise LoadError( + "{}: Value of '{}' expected to be a list of strings".format(uris, alias), + LoadErrorReason.INVALID_DATA, + ) + + aliases[alias] = uris.as_str_list() + + return aliases
