This is an automated email from the ASF dual-hosted git repository. tvb pushed a commit to branch tristan/mirror-plugins in repository https://gitbox.apache.org/repos/asf/buildstream-plugins.git
commit a312599ee45ed0b11a896eefc392848d091bb75c Author: Tristan van Berkom <[email protected]> AuthorDate: Mon Oct 14 18:28:56 2024 +0900 Adding gitlab_lfs_mirror source mirror plugin Since we've added the SourceMirror feature in BuildStream 2.2, it makes sense to add this mirror plugin upstream. This is especially helpful as we can rely on loading stable buildstream-plugins via the `pip` plugin origin for loading source mirror plugins, avoiding a cyclic dependency issue if you want to use mirror plugins to load plugins via the `junction` plugin origin. --- doc/source/index.rst | 1 + project.conf | 1 + setup.py | 1 + .../sourcemirrors/gitlab_lfs_mirror.py | 76 ++++++++++++++++++++++ 4 files changed, 79 insertions(+) diff --git a/doc/source/index.rst b/doc/source/index.rst index 95b6f8c..88c5bc5 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -48,3 +48,4 @@ To these plugins in your project, follow the :caption: Source Mirror Plugins sourcemirrors/simple_mirror + sourcemirrors/gitlab_lfs_mirror diff --git a/project.conf b/project.conf index 301bb08..bad78fe 100644 --- a/project.conf +++ b/project.conf @@ -43,4 +43,5 @@ plugins: - origin: local path: src/buildstream_plugins/sourcemirrors source-mirrors: + - gitlab_lfs_mirror - simple_mirror diff --git a/setup.py b/setup.py index b7de25b..5417244 100755 --- a/setup.py +++ b/setup.py @@ -93,6 +93,7 @@ setup( "zip = buildstream_plugins.sources.zip", ], "buildstream.plugins.sourcemirrors": [ + "gitlab_lfs_mirror = buildstream_plugins.sourcemirrors.gitlab_lfs_mirror", "simple_mirror = buildstream_plugins.sourcemirrors.simple_mirror", ], }, diff --git a/src/buildstream_plugins/sourcemirrors/gitlab_lfs_mirror.py b/src/buildstream_plugins/sourcemirrors/gitlab_lfs_mirror.py new file mode 100644 index 0000000..0d30632 --- /dev/null +++ b/src/buildstream_plugins/sourcemirrors/gitlab_lfs_mirror.py @@ -0,0 +1,76 @@ +""" +gitlab_lfs_mirror - plugin for accessing files stored in git-lfs on GitLab +========================================================================== + +.. note:: + + The ``gitlab_lfs_mirror`` plugin is available *Since 2.4.0* + +If you store files in a git-lfs repository on gitlab.com, you can access +them using a URL like + +https://gitlab.com/path/to/repo/-/raw/master/path/to/file + +which you can then use as a mirror for buildstream. However this only works for +public repositories. For internal (on self-hosted GitLab instances) and private +repositories, the above doesn't work since buildstream cannot authenticate with +the GitLab web UI. + +This plugin solves this by going through the GitLab REST API. You need an +access token that access the Repository Files API (i.e. with read_api or +read_repository scope). As of this writing, the GitLab CI/CD job token doesn't +allow access to this API endpoint, so you need a dedicated access token. + +**Usage:** + +.. code:: yaml + +- name: my-mirror + kind: gitlab_lfs_mirror + config: + url: https://gitlab.example.com/ + project: mirrors/{alias} + ref: main # optional, defaults to master + aliases: + - my-alias + - another-alias +""" + +from posixpath import join +from urllib.parse import quote + +from buildstream import SourceMirror + + +class GitlabLFSMirror(SourceMirror): + BST_MIN_VERSION = "2.2" + + def configure(self, node): + node.validate_keys(["aliases", "url", "project", "ref"]) + self.set_supported_aliases(node.get_str_list("aliases")) + + self.url = node.get_str("url") + self.project = node.get_str("project") + self.ref = node.get_str("ref", "master") + + def translate_url(self, alias, alias_url, source_url, extra_data): + project_id = quote(self.project.format(alias=alias), safe="") + filename = quote(source_url, safe="") + + translated_url = join( + self.url, + "api/v4/projects", + project_id, + "repository/files", + filename, + f"raw?ref={self.ref}&lfs=true", + ) + + if extra_data is not None: + extra_data["http-auth"] = "bearer" + + return translated_url + + +def setup(): + return GitlabLFSMirror
