abderrahim commented on code in PR #1890: URL: https://github.com/apache/buildstream/pull/1890#discussion_r1509989058
########## src/buildstream/sourcemirror.py: ########## @@ -0,0 +1,203 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Authors: +# Tristan Van Berkom <[email protected]> +""" +SourceMirror - Base source mirror class +======================================= +The SourceMirror plugin allows one to customize how +:func:`Source.translate_url() <buildstream.source.Source.translate_url>` will +behave when looking up mirrors, allowing some additional flexibility in the +implementation of source mirrors. + + +.. _core_source_mirror_abstract_methods: + +Abstract Methods +---------------- +For loading and configuration purposes, SourceMirrors may optionally implement +the :func:`Plugin base class Plugin.configure() method <buildstream.plugin.Plugin.configure>` +in order to load any custom configuration in the `config` dictionary. + +The remaining :ref:`Plugin base class abstract methods <core_plugin_abstract_methods>` are +not relevant to the SourceMirror plugin object and need not be implemented. + +Sources expose the following abstract methods. Unless explicitly mentioned, +these methods are mandatory to implement. + +* :func:`SourceMirror.translate_url() <buildstream.source.SourceMirror.translate_url>` + + Produce an appropriate URL for the given URL and alias. + + +Class Reference +--------------- +""" + +from typing import Optional, Dict, List, Any, TYPE_CHECKING + +from .node import MappingNode, SequenceNode +from .plugin import Plugin +from ._exceptions import BstError, LoadError +from .exceptions import ErrorDomain, LoadErrorReason + +if TYPE_CHECKING: + + # pylint: disable=cyclic-import + from ._context import Context + from ._project import Project + + # pylint: enable=cyclic-import + + +class SourceMirrorError(BstError): + """This exception should be raised by :class:`.SourceMirror` implementations + to report errors to the user. + + Args: + message: The breif error description to report to the user + detail: A possibly multiline, more detailed error message + reason: An optional machine readable reason string, used for test cases + + *Since: 2.2* + """ + + def __init__( + self, message: str, *, detail: Optional[str] = None, reason: Optional[str] = None, temporary: bool = False + ): + super().__init__(message, detail=detail, domain=ErrorDomain.SOURCE, reason=reason) + + +class SourceMirror(Plugin): + """SourceMirror() + + Base SourceMirror class. + + All SourceMirror plugins derive from this class, this interface defines how + the core will be interacting with SourceMirror plugins. + + *Since: 2.2* + """ + + # The SourceMirror plugin type is only supported since BuildStream 2.2 + BST_MIN_VERSION = "2.2" + + def __init__( + self, + context: "Context", + project: "Project", + node: MappingNode, + ): + # Note: the MappingNode passed here is already expanded with + # the project level base variables, so there is no need + # to expand them redundantly here. + # + node.validate_keys(["name", "kind", "config", "aliases"]) + + # Do local base class parsing first + name: str = node.get_str("name") + self.__aliases: Dict[str, List[str]] = self.__load_aliases(node) Review Comment: Can we maybe not make this mandatory? Having this be mandatory gives weird source mirror plugins (like the one in tests). Can I suggest the following: * Move this alias logic to a core plugin * Have the plugin factory check and require that either `kind` or `aliases` is provided, but not both * if `aliases` is provided, create the default bst implementation (it could be special-cased to get "aliases" instead of "config") * if `kind` is provided, create the specified plugin. * Have some API to allow the plugins to request the aliases they can handle (This could probably be extended in the future to handle unaliased urls) ########## tests/frontend/mirror.py: ########## @@ -807,3 +807,93 @@ def test_mirror_expand_project_and_toplevel_root(cli, tmpdir): # Success if the expanded %{project-root} is found assert foo_str in contents assert bar_str in contents + + +# Test a simple SourceMirror implementation which reads +# plugin configuration and behaves in the same way as default +# mirrors but using data in the plugin configuration instead. +# [email protected](DATA_DIR) [email protected]("datafiles") +def test_source_mirror_plugin(cli, tmpdir): + output_file = os.path.join(str(tmpdir), "output.txt") + project_dir = str(tmpdir) + element_dir = os.path.join(project_dir, "elements") + os.makedirs(element_dir, exist_ok=True) + element_name = "test.bst" + element_path = os.path.join(element_dir, element_name) + element = generate_element(output_file) + _yaml.roundtrip_dump(element, element_path) + + project_file = os.path.join(project_dir, "project.conf") + project = { + "name": "test", + "min-version": "2.0", + "element-path": "elements", + "aliases": { + "foo": "FOO/", + "bar": "BAR/", + }, + "mirrors": [ + { + "name": "middle-earth", + "kind": "mirror", + "aliases": { + "foo": ["<invalid>"], Review Comment: ^ This is what I am referring to. We shouldn't have to do this when implementing source mirrors. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
