This is an automated email from the ASF dual-hosted git repository. tvb pushed a commit to branch tristan/remove-plugins in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 9901b7fd6afb8c20148888a80d03425fb7148c3b Author: Tristan van Berkom <[email protected]> AuthorDate: Wed Mar 30 14:39:38 2022 +0900 plugins/sources/git.py: Removing git source And removing the git source from the documentation plugins index --- doc/source/core_plugins.rst | 1 - src/buildstream/_gitsourcebase.py | 831 --------------------------------- src/buildstream/plugins/sources/git.py | 164 ------- 3 files changed, 996 deletions(-) diff --git a/doc/source/core_plugins.rst b/doc/source/core_plugins.rst index 15cd701..79e4d56 100644 --- a/doc/source/core_plugins.rst +++ b/doc/source/core_plugins.rst @@ -51,7 +51,6 @@ information. sources/local sources/remote sources/tar - sources/git .. _plugins_external: diff --git a/src/buildstream/_gitsourcebase.py b/src/buildstream/_gitsourcebase.py deleted file mode 100644 index 901069b..0000000 --- a/src/buildstream/_gitsourcebase.py +++ /dev/null @@ -1,831 +0,0 @@ -# -# Copyright (C) 2016 Codethink Limited -# Copyright (C) 2018 Bloomberg Finance LP -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Authors: -# Tristan Van Berkom <[email protected]> -# Chandan Singh <[email protected]> -# Tom Mewett <[email protected]> - -"""Abstract base class for source implementations that work with a Git repository""" - -import os -import re -import shutil -from io import StringIO -from tempfile import TemporaryFile - -from configparser import RawConfigParser - -from .source import Source, SourceError, SourceFetcher -from .types import CoreWarnings -from . import utils -from .types import FastEnum -from .utils import move_atomic, DirectoryExistsError - -GIT_MODULES = ".gitmodules" -EXACT_TAG_PATTERN = r"(?P<tag>.*)-0-g(?P<commit>.*)" - -# Warnings -WARN_INCONSISTENT_SUBMODULE = "inconsistent-submodule" -WARN_UNLISTED_SUBMODULE = "unlisted-submodule" -WARN_INVALID_SUBMODULE = "invalid-submodule" - - -class _RefFormat(FastEnum): - SHA1 = "sha1" - GIT_DESCRIBE = "git-describe" - - -def _strip_tag(rev): - return rev.split("-g")[-1] - - -# This class represents a single Git repository. The Git source needs to account for -# submodules, but we don't want to cache them all under the umbrella of the -# superproject - so we use this class which caches them independently, according -# to their URL. Instances keep reference to their "parent" GitSourceBase, -# and if applicable, where in the superproject they are found. -# -# Args: -# source (_GitSourceBase or subclass): The parent source -# path (str): The relative location of the submodule in the superproject; -# the empty string for the superproject itself -# url (str): Where to clone the repo from -# ref (str): Specified 'ref' from the source configuration -# primary (bool): Whether this is the primary URL for the source -# tags (list): Tag configuration; see _GitSourceBase._load_tags -# -class _GitMirror(SourceFetcher): - def __init__(self, source, path, url, ref, *, primary=False, tags=None): - - super().__init__() - self.source = source - self.path = path - self.url = url - self.ref = ref - self.tags = tags or [] - self.primary = primary - self.mirror = os.path.join(source.get_mirror_directory(), utils.url_directory_name(url)) - - # _ensure_repo(): - # - # Ensures that the Git repository exists at the mirror location and is configured - # to fetch from the given URL - # - def _ensure_repo(self): - if not os.path.exists(self.mirror): - with self.source.tempdir() as tmpdir: - self.source.call( - [self.source.host_git, "init", "--bare", tmpdir], - fail="Failed to initialise repository", - ) - - try: - move_atomic(tmpdir, self.mirror) - except DirectoryExistsError: - # Another process was quicker to download this repository. - # Let's discard our own - self.source.status("{}: Discarding duplicate repository".format(self.source)) - except OSError as e: - raise SourceError( - "{}: Failed to move created repository from '{}' to '{}': {}".format( - self.source, tmpdir, self.mirror, e - ) - ) from e - - def _fetch(self, url, fetch_all=False): - self._ensure_repo() - - # Work out whether we can fetch a specific tag: are we given a ref which - # 1. is in git-describe format - # 2. refers to an exact tag (is "...-0-g...") - # 3. is available on the remote and tags the specified commit? - # And lastly: are we on a new-enough Git which allows cloning from our potentially shallow cache? - if fetch_all: - pass - # Fetching from a shallow-cloned repo was first supported in v1.9.0 - elif not self.ref or self.source.host_git_version is not None and self.source.host_git_version < (1, 9, 0): - fetch_all = True - else: - m = re.match(EXACT_TAG_PATTERN, self.ref) - if m is None: - fetch_all = True - else: - tag = m.group("tag") - commit = m.group("commit") - - if not self.remote_has_tag(url, tag, commit): - self.source.status( - "{}: {} is not advertised on {}. Fetching all Git refs".format(self.source, self.ref, url) - ) - fetch_all = True - else: - exit_code = self.source.call( - [ - self.source.host_git, - "fetch", - "--depth=1", - url, - "+refs/tags/{tag}:refs/tags/{tag}".format(tag=tag), - ], - cwd=self.mirror, - ) - if exit_code != 0: - self.source.status( - "{}: Failed to fetch tag '{}' from {}. Fetching all Git refs".format(self.source, tag, url) - ) - fetch_all = True - - if fetch_all: - self.source.call( - [ - self.source.host_git, - "fetch", - "--prune", - url, - "+refs/heads/*:refs/heads/*", - "+refs/tags/*:refs/tags/*", - ], - fail="Failed to fetch from remote git repository: {}".format(url), - fail_temporarily=True, - cwd=self.mirror, - ) - - def fetch(self, alias_override=None): # pylint: disable=arguments-differ - resolved_url = self.source.translate_url(self.url, alias_override=alias_override, primary=self.primary) - - with self.source.timed_activity("Fetching from {}".format(resolved_url), silent_nested=True): - if not self.has_ref(): - self._fetch(resolved_url) - self.assert_ref() - - def has_ref(self): - if not self.ref: - return False - - # If the mirror doesnt exist, we also dont have the ref - if not os.path.exists(self.mirror): - return False - - # Check if the ref is really there - rc = self.source.call([self.source.host_git, "cat-file", "-t", self.ref], cwd=self.mirror) - return rc == 0 - - def assert_ref(self): - if not self.has_ref(): - raise SourceError( - "{}: expected ref '{}' was not found in git repository: '{}'".format(self.source, self.ref, self.url) - ) - - # remote_has_tag(): - # - # Args: - # url (str) - # tag (str) - # commit (str) - # - # Returns: - # (bool): Whether the remote at `url` has the tag `tag` attached to `commit` - # - def remote_has_tag(self, url, tag, commit): - _, ls_remote = self.source.check_output( - [self.source.host_git, "ls-remote", url], - cwd=self.mirror, - fail="Failed to list advertised remote refs from git repository {}".format(url), - ) - - line = "{commit}\trefs/tags/{tag}".format(commit=commit, tag=tag) - return line in ls_remote or line + "^{}" in ls_remote - - # to_commit(): - # - # Args: - # rev (str): A Git "commit-ish" rev - # - # Returns: - # (str): The full revision ID of the commit - # - def to_commit(self, rev): - _, output = self.source.check_output( - [self.source.host_git, "rev-list", "-n", "1", rev], - fail="Unable to find revision {}".format(rev), - cwd=self.mirror, - ) - - return output.strip() - - # describe(): - # - # Args: - # rev (str): A Git "commit-ish" rev - # - # Returns: - # (str): The full revision ID of the commit given by rev, prepended - # with tag information as given by git-describe (where available) - # - def describe(self, rev): - _, output = self.source.check_output( - [self.source.host_git, "describe", "--tags", "--abbrev=40", "--long", "--always", rev], - fail="Unable to find revision {}".format(rev), - cwd=self.mirror, - ) - - return output.strip() - - # reachable_tags(): - # - # Args: - # rev (str): A Git "commit-ish" rev - # - # Returns: - # (list): A list of tags in the ancestry of rev. Each entry is a triple of the form - # (tag name (str), commit ref (str), annotated (bool)) describing a tag, - # its tagged commit and whether it's annotated - # - def reachable_tags(self, rev): - tags = set() - for options in [[], ["--first-parent"], ["--tags"], ["--tags", "--first-parent"]]: - exit_code, output = self.source.check_output( - [self.source.host_git, "describe", "--abbrev=0", rev, *options], cwd=self.mirror - ) - if exit_code == 0: - tag = output.strip() - _, commit_ref = self.source.check_output( - [self.source.host_git, "rev-parse", tag + "^{commit}"], - fail="Unable to resolve tag '{}'".format(tag), - cwd=self.mirror, - ) - exit_code = self.source.call([self.source.host_git, "cat-file", "tag", tag], cwd=self.mirror) - annotated = exit_code == 0 - - tags.add((tag, commit_ref.strip(), annotated)) - - return list(tags) - - def stage(self, directory): - fullpath = os.path.join(directory, self.path) - - # Using --shared here avoids copying the objects into the checkout, in any - # case we're just checking out a specific commit and then removing the .git/ - # directory. - self.source.call( - [self.source.host_git, "clone", "--no-checkout", "--shared", self.mirror, fullpath], - fail="Failed to create git mirror {} in directory: {}".format(self.mirror, fullpath), - fail_temporarily=True, - ) - - self.source.call( - [self.source.host_git, "checkout", "--force", self.ref], - fail="Failed to checkout git ref {}".format(self.ref), - cwd=fullpath, - ) - - # Remove .git dir - shutil.rmtree(os.path.join(fullpath, ".git")) - - self._rebuild_git(fullpath) - - def init_workspace(self, directory): - fullpath = os.path.join(directory, self.path) - url = self.source.translate_url(self.url) - - self.source.call( - [self.source.host_git, "clone", "--no-checkout", self.mirror, fullpath], - fail="Failed to clone git mirror {} in directory: {}".format(self.mirror, fullpath), - fail_temporarily=True, - ) - - self.source.call( - [self.source.host_git, "remote", "set-url", "origin", url], - fail='Failed to add remote origin "{}"'.format(url), - cwd=fullpath, - ) - - self.source.call( - [self.source.host_git, "checkout", "--force", self.ref], - fail="Failed to checkout git ref {}".format(self.ref), - cwd=fullpath, - ) - - # get_submodule_mirrors(): - # - # Returns: - # An iterator through new instances of this class, one of each submodule - # in the repo - # - def get_submodule_mirrors(self): - for path, url in self.submodule_list(): - ref = self.submodule_ref(path) - if ref is not None: - mirror = self.__class__(self.source, os.path.join(self.path, path), url, ref) - yield mirror - - # List the submodules (path/url tuples) present at the given ref of this repo - def submodule_list(self): - modules = "{}:{}".format(self.ref, GIT_MODULES) - exit_code, output = self.source.check_output([self.source.host_git, "show", modules], cwd=self.mirror) - - # If git show reports error code 128 here, we take it to mean there is - # no .gitmodules file to display for the given revision. - if exit_code == 128: - return - elif exit_code != 0: - raise SourceError("{plugin}: Failed to show gitmodules at ref {ref}".format(plugin=self, ref=self.ref)) - - content = "\n".join([l.strip() for l in output.splitlines()]) - - io = StringIO(content) - parser = RawConfigParser() - parser.read_file(io) - - for section in parser.sections(): - # validate section name against the 'submodule "foo"' pattern - if re.match(r'submodule "(.*)"', section): - path = parser.get(section, "path") - url = parser.get(section, "url") - - yield (path, url) - - # Fetch the ref which this mirror requires its submodule to have, - # at the given ref of this mirror. - def submodule_ref(self, submodule, ref=None): - if not ref: - ref = self.ref - - # list objects in the parent repo tree to find the commit - # object that corresponds to the submodule - _, output = self.source.check_output( - [self.source.host_git, "ls-tree", ref, submodule], - fail="ls-tree failed for commit {} and submodule: {}".format(ref, submodule), - cwd=self.mirror, - ) - - # read the commit hash from the output - fields = output.split() - if len(fields) >= 2 and fields[1] == "commit": - submodule_commit = output.split()[2] - - # fail if the commit hash is invalid - if len(submodule_commit) != 40: - raise SourceError( - "{}: Error reading commit information for submodule '{}'".format(self.source, submodule) - ) - - return submodule_commit - - else: - detail = ( - "The submodule '{}' is defined either in the BuildStream source\n".format(submodule) - + "definition, or in a .gitmodules file. But the submodule was never added to the\n" - + "underlying git repository with `git submodule add`." - ) - - self.source.warn( - "{}: Ignoring inconsistent submodule '{}'".format(self.source, submodule), - detail=detail, - warning_token=WARN_INCONSISTENT_SUBMODULE, - ) - - return None - - def _rebuild_git(self, fullpath): - if not self.tags: - return - - with self.source.tempdir() as tmpdir: - included = set() - shallow = set() - for _, commit_ref, _ in self.tags: - - if commit_ref == self.ref: - # rev-list does not work in case of same rev - shallow.add(self.ref) - else: - _, out = self.source.check_output( - [ - self.source.host_git, - "rev-list", - "--ancestry-path", - "--boundary", - "{}..{}".format(commit_ref, self.ref), - ], - fail="Failed to get git history {}..{} in directory: {}".format( - commit_ref, self.ref, fullpath - ), - fail_temporarily=True, - cwd=self.mirror, - ) - self.source.warn("refs {}..{}: {}".format(commit_ref, self.ref, out.splitlines())) - for line in out.splitlines(): - rev = line.lstrip("-") - if line[0] == "-": - shallow.add(rev) - else: - included.add(rev) - - shallow -= included - included |= shallow - - self.source.call( - [self.source.host_git, "init"], - fail="Cannot initialize git repository: {}".format(fullpath), - cwd=fullpath, - ) - - for rev in included: - with TemporaryFile(dir=tmpdir) as commit_file: - self.source.call( - [self.source.host_git, "cat-file", "commit", rev], - stdout=commit_file, - fail="Failed to get commit {}".format(rev), - cwd=self.mirror, - ) - commit_file.seek(0, 0) - self.source.call( - [self.source.host_git, "hash-object", "-w", "-t", "commit", "--stdin"], - stdin=commit_file, - fail="Failed to add commit object {}".format(rev), - cwd=fullpath, - ) - - with open(os.path.join(fullpath, ".git", "shallow"), "w", encoding="utf-8") as shallow_file: - for rev in shallow: - shallow_file.write("{}\n".format(rev)) - - for tag, commit_ref, annotated in self.tags: - if annotated: - with TemporaryFile(dir=tmpdir) as tag_file: - tag_data = "object {}\ntype commit\ntag {}\n".format(commit_ref, tag) - tag_file.write(tag_data.encode("ascii")) - tag_file.seek(0, 0) - _, tag_ref = self.source.check_output( - [self.source.host_git, "hash-object", "-w", "-t", "tag", "--stdin"], - stdin=tag_file, - fail="Failed to add tag object {}".format(tag), - cwd=fullpath, - ) - - self.source.call( - [self.source.host_git, "tag", tag, tag_ref.strip()], - fail="Failed to tag: {}".format(tag), - cwd=fullpath, - ) - else: - self.source.call( - [self.source.host_git, "tag", tag, commit_ref], - fail="Failed to tag: {}".format(tag), - cwd=fullpath, - ) - - with open(os.path.join(fullpath, ".git", "HEAD"), "w", encoding="utf-8") as head: - self.source.call( - [self.source.host_git, "rev-parse", self.ref], - stdout=head, - fail="Failed to parse commit {}".format(self.ref), - cwd=self.mirror, - ) - - -class _GitSourceBase(Source): - # pylint: disable=attribute-defined-outside-init - - # The GitMirror class which this plugin uses. This may be - # overridden in derived plugins as long as the replacement class - # follows the same interface used by the _GitMirror class - BST_MIRROR_CLASS = _GitMirror - - def configure(self, node): - ref = node.get_str("ref", None) - - config_keys = ["url", "track", "ref", "submodules", "checkout-submodules", "ref-format", "track-tags", "tags"] - node.validate_keys(config_keys + Source.COMMON_CONFIG_KEYS) - - tags_node = node.get_sequence("tags", []) - for tag_node in tags_node: - tag_node.validate_keys(["tag", "commit", "annotated"]) - - tags = self._load_tags(node) - self.track_tags = node.get_bool("track-tags", default=False) - - self.original_url = node.get_str("url") - self.mirror = self.BST_MIRROR_CLASS(self, "", self.original_url, ref, tags=tags, primary=True) - self.tracking = node.get_str("track", None) - - self.ref_format = node.get_enum("ref-format", _RefFormat, _RefFormat.SHA1) - - # At this point we now know if the source has a ref and/or a track. - # If it is missing both then we will be unable to track or build. - if self.mirror.ref is None and self.tracking is None: - raise SourceError( - "{}: Git sources require a ref and/or track".format(self), reason="missing-track-and-ref" - ) - - self.checkout_submodules = node.get_bool("checkout-submodules", default=True) - - # Parse a dict of submodule overrides, stored in the submodule_overrides - # and submodule_checkout_overrides dictionaries. - self.submodule_overrides = {} - self.submodule_checkout_overrides = {} - modules = node.get_mapping("submodules", {}) - for path in modules.keys(): - submodule = modules.get_mapping(path) - url = submodule.get_str("url", None) - - # Make sure to mark all URLs that are specified in the configuration - if url: - self.mark_download_url(url, primary=False) - - self.submodule_overrides[path] = url - if "checkout" in submodule: - checkout = submodule.get_bool("checkout") - self.submodule_checkout_overrides[path] = checkout - - self.mark_download_url(self.original_url) - - def preflight(self): - # Check if git is installed, get the binary at the same time - self.host_git = utils.get_host_tool("git") - - rc, version_str = self.check_output([self.host_git, "--version"]) - # e.g. on Git for Windows we get "git version 2.21.0.windows.1". - # e.g. on Mac via Homebrew we get "git version 2.19.0". - if rc == 0: - self.host_git_version = tuple(int(x) for x in version_str.split(" ")[2].split(".")[:3]) - else: - self.host_git_version = None - - def get_unique_key(self): - ref = self.mirror.ref - if ref is not None: - # Strip any (arbitary) tag information, leaving just the commit ID - ref = _strip_tag(ref) - - # Here we want to encode the local name of the repository and - # the ref, if the user changes the alias to fetch the same sources - # from another location, it should not affect the cache key. - key = [self.original_url, ref] - if self.mirror.tags: - tags = {tag: (commit, annotated) for tag, commit, annotated in self.mirror.tags} - key.append({"tags": tags}) - - # Only modify the cache key with checkout_submodules if it's something - # other than the default behaviour. - if self.checkout_submodules is False: - key.append({"checkout_submodules": self.checkout_submodules}) - - # We want the cache key to change if the source was - # configured differently, and submodules count. - if self.submodule_overrides: - key.append(self.submodule_overrides) - - if self.submodule_checkout_overrides: - key.append({"submodule_checkout_overrides": self.submodule_checkout_overrides}) - - return key - - def is_resolved(self): - return self.mirror.ref is not None - - def is_cached(self): - return self._have_all_refs() - - def load_ref(self, node): - self.mirror.ref = node.get_str("ref", None) - self.mirror.tags = self._load_tags(node) - - def get_ref(self): - if self.mirror.ref is None: - return None - return self.mirror.ref, self.mirror.tags - - def set_ref(self, ref, node): - if not ref: - self.mirror.ref = None - if "ref" in node: - del node["ref"] - self.mirror.tags = [] - if "tags" in node: - del node["tags"] - else: - actual_ref, tags = ref - node["ref"] = self.mirror.ref = actual_ref - self.mirror.tags = tags - if tags: - node["tags"] = [] - for tag, commit_ref, annotated in tags: - data = {"tag": tag, "commit": commit_ref, "annotated": annotated} - node["tags"].append(data) - else: - if "tags" in node: - del node["tags"] - - def track(self): # pylint: disable=arguments-differ - - # If self.tracking is not specified it's not an error, just silently return - if not self.tracking: - # Is there a better way to check if a ref is given. - if self.mirror.ref is None: - detail = "Without a tracking branch ref can not be updated. Please " + "provide a ref or a track." - raise SourceError("{}: No track or ref".format(self), detail=detail, reason="track-attempt-no-track") - return None - - # Resolve the URL for the message - resolved_url = self.translate_url(self.mirror.url) - with self.timed_activity("Tracking {} from {}".format(self.tracking, resolved_url), silent_nested=True): - self.mirror._fetch(resolved_url, fetch_all=True) - - ref = self.mirror.to_commit(self.tracking) - tags = self.mirror.reachable_tags(ref) if self.track_tags else [] - - if self.ref_format == _RefFormat.GIT_DESCRIBE: - ref = self.mirror.describe(ref) - - return ref, tags - - def init_workspace(self, directory): - with self.timed_activity('Setting up workspace "{}"'.format(directory), silent_nested=True): - self.mirror.init_workspace(directory) - for mirror in self._recurse_submodules(configure=True): - mirror.init_workspace(directory) - - def stage(self, directory): - # Stage the main repo in the specified directory - # - with self.timed_activity("Staging {}".format(self.mirror.url), silent_nested=True): - self.mirror.stage(directory) - for mirror in self._recurse_submodules(configure=True): - mirror.stage(directory) - - def get_source_fetchers(self): - self.mirror.mark_download_url(self.mirror.url) - yield self.mirror - # _recurse_submodules only iterates those which are known at the current - # cached state - but fetch is called on each result as we go, so this will - # yield all configured submodules - for submodule in self._recurse_submodules(configure=True): - submodule.mark_download_url(submodule.url) - yield submodule - - def validate_cache(self): - discovered_submodules = {} - unlisted_submodules = [] - invalid_submodules = [] - - for submodule in self._recurse_submodules(configure=False): - discovered_submodules[submodule.path] = submodule.url - if self._ignoring_submodule(submodule.path): - continue - - if submodule.path not in self.submodule_overrides: - unlisted_submodules.append((submodule.path, submodule.url)) - - # Warn about submodules which are explicitly configured but do not exist - for path, url in self.submodule_overrides.items(): - if path not in discovered_submodules: - invalid_submodules.append((path, url)) - - if invalid_submodules: - detail = [] - for path, url in invalid_submodules: - detail.append(" Submodule URL '{}' at path '{}'".format(url, path)) - - self.warn( - "{}: Invalid submodules specified".format(self), - warning_token=WARN_INVALID_SUBMODULE, - detail="The following submodules are specified in the source " - "description but do not exist according to the repository\n\n" + "\n".join(detail), - ) - - # Warn about submodules which exist but have not been explicitly configured - if unlisted_submodules: - detail = [] - for path, url in unlisted_submodules: - detail.append(" Submodule URL '{}' at path '{}'".format(url, path)) - - self.warn( - "{}: Unlisted submodules exist".format(self), - warning_token=WARN_UNLISTED_SUBMODULE, - detail="The following submodules exist but are not specified " - + "in the source description\n\n" - + "\n".join(detail), - ) - - # Assert that the ref exists in the track tag/branch, if track has been specified. - # Also don't do this check if an exact tag ref is given, as we probably didn't fetch - # any branch refs - ref_in_track = False - if not re.match(EXACT_TAG_PATTERN, self.mirror.ref) and self.tracking: - _, branch = self.check_output( - [self.host_git, "branch", "--list", self.tracking, "--contains", self.mirror.ref], - cwd=self.mirror.mirror, - ) - if branch: - ref_in_track = True - else: - _, tag = self.check_output( - [self.host_git, "tag", "--list", self.tracking, "--contains", self.mirror.ref], - cwd=self.mirror.mirror, - ) - if tag: - ref_in_track = True - - if not ref_in_track: - detail = ( - "The ref provided for the element does not exist locally " - + "in the provided track branch / tag '{}'.\n".format(self.tracking) - + "You may wish to track the element to update the ref from '{}' ".format(self.tracking) - + "with `bst source track`,\n" - + "or examine the upstream at '{}' for the specific ref.".format(self.mirror.url) - ) - - self.warn( - "{}: expected ref '{}' was not found in given track '{}' for staged repository: '{}'\n".format( - self, self.mirror.ref, self.tracking, self.mirror.url - ), - detail=detail, - warning_token=CoreWarnings.REF_NOT_IN_TRACK, - ) - - ########################################################### - # Local Functions # - ########################################################### - - def _have_all_refs(self): - return self.mirror.has_ref() and all( - submodule.has_ref() for submodule in self._recurse_submodules(configure=True) - ) - - # _configure_submodules(): - # - # Args: - # submodules: An iterator of _GitMirror (or similar) objects for submodules - # - # Returns: - # An iterator through `submodules` but filtered of any ignored submodules - # and modified to use any custom URLs configured in the source - # - def _configure_submodules(self, submodules): - for submodule in submodules: - if self._ignoring_submodule(submodule.path): - continue - # Allow configuration to override the upstream location of the submodules. - submodule.url = self.submodule_overrides.get(submodule.path, submodule.url) - yield submodule - - # _recurse_submodules(): - # - # Recursively iterates through GitMirrors for submodules of the main repo. Only - # submodules that are cached are recursed into - but this is decided at - # iteration time, so you can fetch in a for loop over this function to fetch - # all submodules. - # - # Args: - # configure (bool): Whether to apply the 'submodule' config while recursing - # (URL changing and 'checkout' overrides) - # - def _recurse_submodules(self, configure): - def recurse(mirror): - submodules = mirror.get_submodule_mirrors() - if configure: - submodules = self._configure_submodules(submodules) - - for submodule in submodules: - yield submodule - if submodule.has_ref(): - yield from recurse(submodule) - - yield from recurse(self.mirror) - - def _load_tags(self, node): - tags = [] - tags_node = node.get_sequence("tags", []) - for tag_node in tags_node: - tag = tag_node.get_str("tag") - commit_ref = tag_node.get_str("commit") - annotated = tag_node.get_bool("annotated") - tags.append((tag, commit_ref, annotated)) - return tags - - # _ignoring_submodule(): - # - # Args: - # path (str): The path of a submodule in the superproject - # - # Returns: - # (bool): Whether to not clone/checkout this submodule - # - def _ignoring_submodule(self, path): - return not self.submodule_checkout_overrides.get(path, self.checkout_submodules) diff --git a/src/buildstream/plugins/sources/git.py b/src/buildstream/plugins/sources/git.py deleted file mode 100644 index 0bdcea5..0000000 --- a/src/buildstream/plugins/sources/git.py +++ /dev/null @@ -1,164 +0,0 @@ -# -# Copyright (C) 2016 Codethink Limited -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Authors: -# Tristan Van Berkom <[email protected]> - -""" -git - stage files from a git repository -======================================= - -**Host dependencies:** - - * git - -.. attention:: - - Note that this plugin **will checkout git submodules by default**; even if - they are not specified in the `.bst` file. - -**Usage:** - -.. code:: yaml - - # Specify the git source kind - kind: git - - # Specify the repository url, using an alias defined - # in your project configuration is recommended. - url: upstream:foo.git - - # Optionally specify a symbolic tracking branch or tag, this - # will be used to update the 'ref' when refreshing the pipeline. - track: master - - # Optionally specify the ref format used for tracking. - # The default is 'sha1' for the raw commit hash. - # If you specify 'git-describe', the commit hash will be prefixed - # with the closest tag. - ref-format: sha1 - - # Specify the commit ref, this must be specified in order to - # checkout sources and build, but can be automatically updated - # if the 'track' attribute was specified. - ref: d63cbb6fdc0bbdadc4a1b92284826a6d63a7ebcd - - # Optionally specify whether submodules should be checked-out. - # This is done recursively, as with `git clone --recurse-submodules`. - # If not set, this will default to 'True' - checkout-submodules: True - - # If your repository has submodules, explicitly specifying the - # url from which they are to be fetched allows you to easily - # rebuild the same sources from a different location. This is - # especially handy when used with project defined aliases which - # can be redefined at a later time. - # You may also explicitly specify whether to check out this - # submodule. If 'checkout' is set, it will control whether to - # checkout that submodule and recurse into it. It defaults to the - # value of 'checkout-submodules'. - submodules: - plugins/bar: - url: upstream:bar.git - checkout: True - plugins/bar/quux: - checkout: False - plugins/baz: - url: upstream:baz.git - checkout: False - - # Enable tag tracking. - # - # This causes the `tags` metadata to be populated automatically - # as a result of tracking the git source. - # - # By default this is 'False'. - # - track-tags: True - - # If the list of tags below is set, then a lightweight dummy - # git repository will be staged along with the content at - # build time. - # - # This is useful for a growing number of modules which use - # `git describe` at build time in order to determine the version - # which will be encoded into the built software. - # - # The 'tags' below is considered as a part of the git source - # reference and will be stored in the 'project.refs' file if - # that has been selected as your project's ref-storage. - # - # Migration notes: - # - # If you are upgrading from BuildStream 1.2, which used to - # stage the entire repository by default, you will notice that - # some modules which use `git describe` are broken, and will - # need to enable this feature in order to fix them. - # - # If you need to enable this feature without changing the - # the specific commit that you are building, then we recommend - # the following migration steps for any git sources where - # `git describe` is required: - # - # o Enable `track-tags` feature - # o Set the `track` parameter to the desired commit sha which - # the current `ref` points to - # o Run `bst source track` for these elements, this will result in - # populating the `tags` portion of the refs without changing - # the refs - # o Restore the `track` parameter to the branches which you have - # previously been tracking afterwards. - # - tags: - - tag: lightweight-example - commit: 04ad0dc656cb7cc6feb781aa13bdbf1d67d0af78 - annotated: false - - tag: annotated-example - commit: 10abe77fe8d77385d86f225b503d9185f4ef7f3a - annotated: true - -See :ref:`built-in functionality doumentation <core_source_builtins>` for -details on common configuration options for sources. - -**Configurable Warnings:** - -This plugin provides the following :ref:`configurable warnings <configurable_warnings>`: - -- ``git:inconsistent-submodule`` - A submodule present in the git repository's .gitmodules was never - added with `git submodule add`. - -- ``git:unlisted-submodule`` - A submodule is present in the git repository but was not specified in - the source configuration and was not disabled for checkout. - -- ``git:invalid-submodule`` - A submodule is specified in the source configuration but does not exist - in the repository. - -This plugin also utilises the following configurable :class:`core warnings <buildstream.types.CoreWarnings>`: - -- :attr:`ref-not-in-track <buildstream.types.CoreWarnings.REF_NOT_IN_TRACK>` - The provided ref was not - found in the provided track in the element's git repository. -""" - -from buildstream import _GitSourceBase - - -class GitSource(_GitSourceBase): - - BST_MIN_VERSION = "2.0" - - -# Plugin entry point -def setup(): - return GitSource
