This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch tristan/change-remote-config
in repository https://gitbox.apache.org/repos/asf/buildstream.git
The following commit(s) were added to refs/heads/tristan/change-remote-config
by this push:
new 82eac25 WIP: Change it all
82eac25 is described below
commit 82eac251f4c64c036d3606038551731932d2c766
Author: Tristan van Berkom <[email protected]>
AuthorDate: Tue Jan 26 22:22:15 2021 +0900
WIP: Change it all
---
src/buildstream/_artifactcache.py | 49 ++--
src/buildstream/_assetcache.py | 385 ++++++++++----------------------
src/buildstream/_context.py | 169 +++++++++++---
src/buildstream/_elementsourcescache.py | 13 +-
src/buildstream/_sourcecache.py | 15 +-
src/buildstream/_stream.py | 115 +++-------
src/buildstream/element.py | 1 +
tests/artifactcache/capabilities.py | 4 +-
tests/artifactcache/config.py | 16 +-
tests/artifactcache/pull.py | 11 +-
tests/artifactcache/push.py | 10 +-
tests/frontend/pull.py | 16 +-
tests/sourcecache/capabilities.py | 6 +-
13 files changed, 339 insertions(+), 471 deletions(-)
diff --git a/src/buildstream/_artifactcache.py
b/src/buildstream/_artifactcache.py
index d73c7f0..a6bdc99 100644
--- a/src/buildstream/_artifactcache.py
+++ b/src/buildstream/_artifactcache.py
@@ -37,10 +37,6 @@ REMOTE_ASSET_ARTIFACT_URN_TEMPLATE =
"urn:fdc:buildstream.build:2020:artifact:{}
# context (Context): The BuildStream context
#
class ArtifactCache(AssetCache):
-
- spec_name = "artifact_cache_specs"
- config_node_name = "artifacts"
-
def __init__(self, context):
super().__init__(context)
@@ -48,12 +44,6 @@ class ArtifactCache(AssetCache):
self._basedir = context.artifactdir
os.makedirs(self._basedir, exist_ok=True)
- def update_mtime(self, ref):
- try:
- os.utime(os.path.join(self._basedir, ref))
- except FileNotFoundError as e:
- raise ArtifactError("Couldn't find artifact: {}".format(ref)) from
e
-
# preflight():
#
# Preflight check.
@@ -88,7 +78,7 @@ class ArtifactCache(AssetCache):
# ([str]) - A list of artifact names as generated in LRU order
#
def list_artifacts(self, *, glob=None):
- return [ref for _, ref in
sorted(list(self._list_refs_mtimes(self._basedir, glob_expr=glob)))]
+ return [ref for _, ref in
sorted(list(self.list_refs_mtimes(self._basedir, glob_expr=glob)))]
# remove():
#
@@ -101,7 +91,7 @@ class ArtifactCache(AssetCache):
#
def remove(self, ref):
try:
- self._remove_ref(ref)
+ self.remove_ref(ref)
except AssetCacheError as e:
raise ArtifactError("{}".format(e)) from e
@@ -123,9 +113,7 @@ class ArtifactCache(AssetCache):
project = element._get_project()
display_key = element._get_display_key()
- index_remotes = [r for r in self._index_remotes[project] if
r.spec.push]
- storage_remotes = [r for r in self._storage_remotes[project] if
r.spec.push]
-
+ index_remotes, storage_remotes = self.get_remotes(project.name, True)
artifact_proto = artifact._get_proto()
artifact_digest =
self.cas.add_object(buffer=artifact_proto.SerializeToString())
@@ -176,10 +164,12 @@ class ArtifactCache(AssetCache):
artifact_name = element.get_artifact_name(key=key)
uri = REMOTE_ASSET_ARTIFACT_URN_TEMPLATE.format(artifact_name)
+ index_remotes, storage_remotes = self.get_remotes(project.name, False)
+
errors = []
# Start by pulling our artifact proto, so that we know which
# blobs to pull
- for remote in self._index_remotes[project]:
+ for remote in index_remotes:
remote.init()
try:
element.status("Pulling artifact {} <- {}".format(display_key,
remote))
@@ -205,7 +195,7 @@ class ArtifactCache(AssetCache):
errors = []
# If we do, we can pull it!
- for remote in self._storage_remotes[project]:
+ for remote in storage_remotes:
remote.init()
try:
element.status("Pulling data for artifact {} <-
{}".format(display_key, remote))
@@ -240,7 +230,8 @@ class ArtifactCache(AssetCache):
# digest (Digest): The digest of the tree
#
def pull_tree(self, project, digest):
- for remote in self._storage_remotes[project]:
+ _, storage_remotes = self.get_remotes(project.name, False)
+ for remote in storage_remotes:
digest = self.cas.pull_tree(remote, digest)
if digest:
@@ -261,12 +252,7 @@ class ArtifactCache(AssetCache):
# (ArtifactError): if there was an error
#
def push_message(self, project, message):
-
- if self._has_push_remotes:
- push_remotes = [r for r in self._storage_remotes[project] if
r.spec.push]
- else:
- push_remotes = []
-
+ _, push_remotes = self.get_remotes(project.name, True)
if not push_remotes:
raise ArtifactError(
"push_message was called, but no remote artifact " + "servers
are configured as push remotes."
@@ -305,7 +291,9 @@ class ArtifactCache(AssetCache):
# missing_blobs (list): The Digests of the blobs to fetch
#
def fetch_missing_blobs(self, project, missing_blobs):
- for remote in self._index_remotes[project]:
+
+ index_remotes, _ = self.get_remotes(project.name, False)
+ for remote in index_remotes:
if not missing_blobs:
break
@@ -332,8 +320,7 @@ class ArtifactCache(AssetCache):
if not missing_blobs:
return []
- push_remotes = [r for r in self._storage_remotes[project] if
r.spec.push]
-
+ _, push_remotes = self.get_remotes(project.name, True)
remote_missing_blobs_list = []
for remote in push_remotes:
@@ -358,13 +345,15 @@ class ArtifactCache(AssetCache):
# (bool): True if the element is available remotely
#
def check_remotes_for_element(self, element):
+ project = element._get_project()
+ index_remotes, _ = self.get_remotes(project.name, False)
+
# If there are no remotes
- if not self._index_remotes:
+ if not index_remotes:
return False
- project = element._get_project()
ref = element.get_artifact_name()
- for remote in self._index_remotes[project]:
+ for remote in index_remotes:
remote.init()
if self._query_remote(ref, remote):
diff --git a/src/buildstream/_assetcache.py b/src/buildstream/_assetcache.py
index eff9de9..66e2182 100644
--- a/src/buildstream/_assetcache.py
+++ b/src/buildstream/_assetcache.py
@@ -18,13 +18,12 @@
#
import os
import re
-from itertools import chain
-from typing import TYPE_CHECKING
+from typing import List, Dict, Tuple, Iterable, Optional
import grpc
from . import utils
from .node import MappingNode
-from ._cas import CASRemote
+from ._cas import CASRemote, CASCache
from ._exceptions import AssetCacheError, RemoteError
from ._remotespec import RemoteSpec, RemoteType
from ._remote import BaseRemote
@@ -32,14 +31,9 @@ from ._protos.build.bazel.remote.asset.v1 import
remote_asset_pb2, remote_asset_
from ._protos.google.rpc import code_pb2
-if TYPE_CHECKING:
- from typing import Optional, Type
- from ._exceptions import BstError
-
-
class AssetRemote(BaseRemote):
- def __init__(self, *args, **kwargs):
- super().__init__(*args, **kwargs)
+ def __init__(self, spec):
+ super().__init__(spec)
self.fetch_service = None
self.push_service = None
@@ -257,161 +251,137 @@ class AssetRemote(BaseRemote):
raise AssetCacheError("PushDirectory failed with status {}:
{}".format(e.code().name, e.details())) from e
-# Base Asset Cache for Caches to derive from
+# RemotePair()
#
-class AssetCache:
+# A pair of remotes which corresponds to a RemoteSpec, we
+# need separate remote objects for the index and the storage so
+# we store them together for each RemoteSpec here.
+#
+# Either members of the RemotePair may be None, in case that
+# the user specified a diffrerent RemoteSpec for indexing and
+# for storage.
+#
+# Both members may also be None, in the case that we were unable
+# to establish a connection to this remote at initialization time.
+#
+class RemotePair:
+ def __init__(self, cas: CASCache, spec: RemoteSpec):
+ self.index: Optional[AssetRemote] = None
+ self.storage: Optional[CASRemote] = None
+ self.error: Optional[str] = None
- # None of these should ever be called in the base class, but this appeases
- # pylint to some degree
- spec_name = None # type: str
- config_node_name = None # type: str
+ try:
+ if spec.remote_type in [RemoteType.INDEX, RemoteType.ALL]:
+ index = AssetRemote(spec)
+ index.check()
+ self.index = index
+ if spec.remote_type in [RemoteType.STORAGE, RemoteType.ALL]:
+ storage = CASRemote(spec, cas)
+ storage.check()
+ self.storage = storage
+ except RemoteError as e:
+ self.error = str(e)
+
+# Base Asset Cache for Caches to derive from
+#
+class AssetCache:
def __init__(self, context):
self.context = context
- self.cas = context.get_cascache()
+ self.cas: CASCache = context.get_cascache()
- self._remotes_setup = False # Check to prevent double-setup of remotes
- # Per-project list of Remote instances.
- self._storage_remotes = {}
- self._index_remotes = {}
+ # Table of RemotePair objects
+ self._remotes: Dict[RemoteSpec, RemotePair] = {}
- self.global_remote_specs = []
- self.project_remote_specs = {}
+ # Table of prioritized RemoteSpecs which are valid for each project
+ self._project_specs: Dict[str, List[RemoteSpec]] = {}
- self._has_fetch_remotes = False
- self._has_push_remotes = False
+ self._has_fetch_remotes: bool = False
+ self._has_push_remotes: bool = False
self._basedir = None
- # close_grpc_channels():
- #
- # Close open gRPC channels.
- #
- def close_grpc_channels(self):
- # Close all remotes and their gRPC channels
- for project_remotes in chain(self._index_remotes.values(),
self._storage_remotes.values()):
- for remote in project_remotes:
- remote.close()
-
# release_resources():
#
# Release resources used by AssetCache.
#
def release_resources(self):
- self.close_grpc_channels()
- # specs_from_config_node()
+ # Close all remotes and their gRPC channels
+ for remote in self._remotes.values():
+ if remote.index:
+ remote.index.close()
+ if remote.storage:
+ remote.storage.close()
+
+ # setup_remotes():
#
- # Parses the configuration of remote artifact caches from a config block.
+ # Sets up which remotes to use
#
# Args:
- # config_node (dict): The config block, which may contain a key defined
by cls.config_node_name
- # basedir (str): The base directory for relative paths
+ # specs: The active remote specs
+ # project_specs: List of specs for each project
#
- # Returns:
- # A list of RemoteSpec instances.
- #
- # Raises:
- # LoadError, if the config block contains invalid keys.
- #
- @classmethod
- def specs_from_config_node(cls, config_node, basedir=None):
- cache_specs = []
- artifacts = config_node.get_sequence(cls.config_node_name, default=[],
allowed_types=[MappingNode])
+ def setup_remotes(self, specs: Iterable[RemoteSpec], project_specs:
Dict[str, List[RemoteSpec]]):
+
+ # Hold on to the project specs
+ self._project_specs = project_specs
- for spec_node in artifacts:
- cache_specs.append(RemoteSpec.new_from_node(spec_node))
+ with self.context.messenger.timed_activity("Initializing remote
caches", silent_nested=True):
+ for spec in specs:
+ # This can be called multiple times, ensure that we only try
+ # to instantiate each remote once.
+ #
+ if spec in self._remotes:
+ continue
- return cache_specs
+ remote = RemotePair(self.cas, spec)
+ if remote.error:
+ self.context.messenger.warn("Failed to initialize remote
{}: {}".format(spec.url, remote.error))
- # setup_remotes():
+ self._remotes[spec] = remote
+
+ # Determine overall existance of push or fetch remotes
+ self._has_fetch_remotes = any(remote.storage for _, remote in
self._remotes.items()) and any(
+ remote.index for _, remote in self._remotes.items()
+ )
+ self._has_push_remotes = any(spec.push and remote.storage for spec,
remote in self._remotes.items()) and any(
+ spec.push and remote.index for spec, remote in
self._remotes.items()
+ )
+
+ # get_remotes():
#
- # Sets up which remotes to use
+ # List the index remotes and storage remotes available for fetching
#
# Args:
- # use_config (bool): Whether to use project configuration
- # remote_url (str): Remote cache URL
- # reinitialize (bool): Whether to re-initialize the remotes a second
time
- #
- # This requires that all of the projects which are to be processed in the
session
- # have already been loaded and are observable in the Context.
- #
- def setup_remotes(self, *, use_config=False, remote_url=None,
reinitialize=False):
-
- # In some edge cases we need to reinitialize
- if reinitialize:
- self._storage_remotes = {}
- self._index_remotes = {}
- self._remotes_setup = False
-
- # Ensure we do not double-initialise since this can be expensive
- if self._remotes_setup:
- return
-
- self._remotes_setup = True
-
- # Initialize remote caches. We allow the commandline to override
- # the user config in some cases (for example `bst artifact push
--remote=...`).
- has_remote_caches = False
- if remote_url:
- self._set_remotes([RemoteSpec(RemoteType.ALL, remote_url,
push=True)])
- has_remote_caches = True
- if use_config:
- for project in self.context.get_projects():
- caches = self._configured_remote_cache_specs(self.context,
project)
- if caches: # caches is a list of RemoteSpec instances
- self._set_remotes(caches, project=project)
- has_remote_caches = True
-
- def remote_failed(remote, error):
- self.context.messenger.warn("Failed to initialize remote {}:
{}".format(remote.url, error))
-
- if has_remote_caches:
- with self.context.messenger.timed_activity("Initializing remote
caches", silent_nested=True):
- self.initialize_remotes(on_failure=remote_failed)
-
- # Notify remotes that forking is disabled
- def notify_fork_disabled(self):
- for project in self._index_remotes:
- for remote in self._index_remotes[project]:
- remote.notify_fork_disabled()
- for project in self._storage_remotes:
- for remote in self._storage_remotes[project]:
- remote.notify_fork_disabled()
-
- # initialize_remotes():
- #
- # This will contact each remote cache.
+ # project_name: The project name
+ # push: Whether pushing is required for this remote
#
- # Args:
- # on_failure (callable): Called if we fail to contact one of the
caches.
+ # Returns:
+ # index_remotes: The index remotes
+ # storage_remotes: The storage remotes
#
- def initialize_remotes(self, *, on_failure=None):
- index_remotes, storage_remotes =
self._create_remote_instances(on_failure=on_failure)
-
- # Assign remote instances to their respective projects
- for project in self.context.get_projects():
- # Get the list of specs that should be considered for this
- # project
- remote_specs = self.global_remote_specs.copy()
- if project in self.project_remote_specs:
- remote_specs.extend(self.project_remote_specs[project])
+ def get_remotes(self, project_name: str, push: bool) ->
Tuple[List[AssetRemote], List[CASRemote]]:
+ try:
+ project_specs = self._project_specs[project_name]
+ except KeyError:
+ # Technically this shouldn't happen, but here is a defensive
return none the less.
+ return [], []
- # De-duplicate the list
- remote_specs = list(utils._deduplicate(remote_specs))
+ index_remotes = []
+ storage_remotes = []
+ for spec in project_specs:
- def get_remotes(remote_list, remote_specs):
- for remote_spec in remote_specs:
- # If a remote_spec didn't make it into the remotes
- # dict, that means we can't access it, and it has been
- # disabled for this session.
- if remote_spec not in remote_list:
- continue
+ if push and not spec.push:
+ continue
- yield remote_list[remote_spec]
+ remote = self._remotes[spec]
+ if remote.index:
+ index_remotes.append(remote.index)
+ if remote.storage:
+ storage_remotes.append(remote.storage)
- self._index_remotes[project] = list(get_remotes(index_remotes,
remote_specs))
- self._storage_remotes[project] = list(get_remotes(storage_remotes,
remote_specs))
+ return index_remotes, storage_remotes
# has_fetch_remotes():
#
@@ -430,9 +400,10 @@ class AssetCache:
# At least one (sub)project has fetch remotes
return True
else:
+ project = plugin._get_project()
+ index_remotes, storage_remotes = self.get_remotes(project.name,
False)
+
# Check whether the specified element's project has fetch remotes
- index_remotes = self._index_remotes[plugin._get_project()]
- storage_remotes = self._storage_remotes[plugin._get_project()]
return index_remotes and storage_remotes
# has_push_remotes():
@@ -452,143 +423,13 @@ class AssetCache:
# At least one (sub)project has push remotes
return True
else:
- # Check whether the specified element's project has push remotes
- index_remotes = self._index_remotes[plugin._get_project()]
- storage_remotes = self._storage_remotes[plugin._get_project()]
- return any(remote.spec.push for remote in index_remotes) and any(
- remote.spec.push for remote in storage_remotes
- )
-
- ################################################
- # Local Private Methods #
- ################################################
+ project = plugin._get_project()
+ index_remotes, storage_remotes = self.get_remotes(project.name,
True)
- # _configured_remote_cache_specs():
- #
- # Return the list of configured remotes for a given project, in priority
- # order. This takes into account the user and project configuration.
- #
- # Args:
- # context (Context): The BuildStream context
- # project (Project): The BuildStream project
- #
- # Returns:
- # A list of RemoteSpec instances describing the remote caches.
- #
- @classmethod
- def _configured_remote_cache_specs(cls, context, project):
- project_overrides = context.get_overrides(project.name)
- project_extra_specs = cls.specs_from_config_node(project_overrides)
-
- project_specs = getattr(project, cls.spec_name)
- context_specs = getattr(context, cls.spec_name)
-
- return list(utils._deduplicate(project_extra_specs + project_specs +
context_specs))
-
- # _create_remote_instances():
- #
- # Create the global set of Remote instances, including
- # project-specific and global instances, ensuring that all of them
- # are accessible.
- #
- # Args:
- # on_failure (Callable[[Remote,Exception],None]):
- # What do do when a remote doesn't respond.
- #
- # Returns:
- # (Dict[RemoteSpec, AssetRemote], Dict[RemoteSpec, CASRemote]) -
- # The created remote instances, index first, storage last.
- #
- def _create_remote_instances(self, *, on_failure=None):
- # Create a flat list of all remote specs, global or
- # project-specific
- remote_specs = self.global_remote_specs.copy()
- for project in self.project_remote_specs:
- remote_specs.extend(self.project_remote_specs[project])
-
- # By de-duplicating it after we flattened the list, we ensure
- # that we never instantiate the same remote twice. This
- # de-duplication also preserves their order.
- remote_specs = list(utils._deduplicate(remote_specs))
-
- # Now let's create a dict of this, indexed by their specs, so
- # that we can later assign them to the right projects.
- index_remotes = {}
- storage_remotes = {}
- for remote_spec in remote_specs:
- try:
- index, storage = self._instantiate_remote(remote_spec)
- except RemoteError as err:
- if on_failure:
- on_failure(remote_spec, str(err))
- continue
-
- raise
-
- # Finally, we can instantiate the remote. Note that
- # NamedTuples are hashable, so we can use them as pretty
- # low-overhead keys.
- if index:
- index_remotes[remote_spec] = index
- if storage:
- storage_remotes[remote_spec] = storage
-
- self._has_fetch_remotes = storage_remotes and index_remotes
- self._has_push_remotes = any(spec.push for spec in storage_remotes)
and any(
- spec.push for spec in index_remotes
- )
-
- return index_remotes, storage_remotes
-
- # _instantiate_remote()
- #
- # Instantiate a remote given its spec, asserting that it is
- # reachable - this may produce two remote instances (a storage and
- # an index remote as specified by the class variables).
- #
- # Args:
- #
- # remote_spec (RemoteSpec): The spec of the remote to
- # instantiate.
- #
- # Returns:
- #
- # (Tuple[Remote|None, Remote|None]) - The remotes, index remote
- # first, storage remote second. One must always be specified,
- # the other may be None.
- #
- def _instantiate_remote(self, remote_spec):
- # Our remotes can be index, storage or both. In either case,
- # we need to use a different type of Remote for our calls, so
- # we create two objects here
- index = None
- storage = None
- if remote_spec.remote_type in [RemoteType.INDEX, RemoteType.ALL]:
- index = AssetRemote(remote_spec) # pylint: disable=not-callable
- index.check()
- if remote_spec.remote_type in [RemoteType.STORAGE, RemoteType.ALL]:
- storage = CASRemote(remote_spec, self.cas)
- storage.check()
-
- return (index, storage)
-
- # _set_remotes():
- #
- # Set the list of remote caches. If project is None, the global list of
- # remote caches will be set, which is used by all projects. If a project is
- # specified, the per-project list of remote caches will be set.
- #
- # Args:
- # remote_specs (list): List of ArtifactCacheSpec instances, in
priority order.
- # project (Project): The Project instance for project-specific remotes
- def _set_remotes(self, remote_specs, *, project=None):
- if project is None:
- # global remotes
- self.global_remote_specs = remote_specs
- else:
- self.project_remote_specs[project] = remote_specs
+ # Check whether the specified element's project has fetch remotes
+ return bool(index_remotes and storage_remotes)
- # _list_refs_mtimes()
+ # list_refs_mtimes()
#
# List refs in a directory, given a base path. Also returns the
# associated mtimes
@@ -600,7 +441,7 @@ class AssetCache:
# Returns:
# (iter (mtime, filename)]): iterator of tuples of mtime and refs
#
- def _list_refs_mtimes(self, base_path, *, glob_expr=None):
+ def list_refs_mtimes(self, base_path, *, glob_expr=None):
path = base_path
if glob_expr is not None:
globdir = os.path.dirname(glob_expr)
@@ -622,7 +463,7 @@ class AssetCache:
# Obtain the mtime (the time a file was last modified)
yield (os.path.getmtime(ref_path), relative_path)
- # _remove_ref()
+ # remove_ref()
#
# Removes a ref.
#
@@ -636,7 +477,7 @@ class AssetCache:
# (AssetCacheError): If the ref didnt exist, or a system error
# occurred while removing it
#
- def _remove_ref(self, ref):
+ def remove_ref(self, ref):
try:
utils._remove_path_with_parents(self._basedir, ref)
except FileNotFoundError as e:
diff --git a/src/buildstream/_context.py b/src/buildstream/_context.py
index e5df670..7d2def6 100644
--- a/src/buildstream/_context.py
+++ b/src/buildstream/_context.py
@@ -19,6 +19,9 @@
import os
import shutil
+
+from typing import List, Dict, Set, Optional
+
from . import utils
from . import _site
from . import _yaml
@@ -83,12 +86,6 @@ class Context:
# Default root location for workspaces
self.workspacedir = None
- # specs for source cache remotes
- self.source_cache_specs = None
-
- # The locations from which to push and pull prebuilt artifacts
- self.artifact_cache_specs = None
-
# The global remote execution configuration
self.remote_execution_specs = None
@@ -161,15 +158,27 @@ class Context:
# Whether file contents are required for all artifacts in the local
cache
self.require_artifact_files = True
- # Whether elements must be rebuilt when their dependencies have changed
- self._strict_build_plan = None
+ # Don't shoot the messenger
+ self.messenger = Messenger()
# Make sure the XDG vars are set in the environment before loading
anything
self._init_xdg()
- self.messenger = Messenger()
-
+ #
# Private variables
+ #
+
+ # Whether elements must be rebuilt when their dependencies have changed
+ self._strict_build_plan = None
+
+ # Lists of globally configured cache specs
+ self._global_artifact_cache_specs: List[RemoteSpec] = []
+ self._global_source_cache_specs: List[RemoteSpec] = []
+
+ # Set of all actively configured remote specs
+ self._active_artifact_cache_specs: Set[RemoteSpec] = set()
+ self._active_source_cache_specs: Set[RemoteSpec] = set()
+
self._platform = None
self._artifactcache = None
self._elementsourcescache = None
@@ -327,18 +336,13 @@ class Context:
LoadErrorReason.INVALID_DATA,
) from e
-
# Load artifact remote specs
- caches = config.get_sequence("artifacts", default=[],
allowed_types=[MappingNode])
- for node in caches:
- spec = RemoteSpec.new_from_node(node)
- self.artifact_cache_specs.append(spec)
+ caches = defaults.get_sequence("artifacts", default=[],
allowed_types=[MappingNode])
+ self._global_artifact_cache_specs = [RemoteSpec.new_from_node(node)
for node in caches]
# Load source cache remote specs
- caches = config.get_sequence("source-caches", default=[],
allowed_types=[MappingNode])
- for node in caches:
- spec = RemoteSpec.new_from_node(node)
- self.source_cache_specs.append(spec)
+ caches = defaults.get_sequence("source-caches", default=[],
allowed_types=[MappingNode])
+ self._global_source_cache_specs = [RemoteSpec.new_from_node(node) for
node in caches]
# Load the global remote execution config including
pull-artifact-files setting
remote_execution = defaults.get_mapping("remote-execution",
default=None)
@@ -446,19 +450,6 @@ class Context:
def add_project(self, project):
if not self._projects:
self._workspaces = Workspaces(project,
self._workspace_project_cache)
-
- #
- # While loading the first, toplevel project, we can adjust some
- # global settings which can be overridden on a per toplevel
project basis.
- #
- override_node = self.get_overrides(project.name)
- if override_node:
- remote_execution =
override_node.get_mapping("remote-execution", default=None)
- if remote_execution:
- self.pull_artifact_files, self.remote_execution_specs =
self._load_remote_execution(
- remote_execution
- )
-
self._projects.append(project)
# get_projects():
@@ -477,10 +468,116 @@ class Context:
# invoked with as opposed to a junctioned subproject.
#
# Returns:
- # (Project): The Project object
+ # (Project): The toplevel Project object, or None
#
def get_toplevel_project(self):
- return self._projects[0]
+ try:
+ return self._projects[0]
+ except IndexError:
+ return None
+
+ # initialize_remotes()
+ #
+ # This will resolve what remotes each loaded project will interact
+ # with an initialize the underlying asset cache modules.
+ #
+ # Note that this can be called more than once, in the case that
+ # Stream() has loaded additional projects during the load cycle
+ # and some state needs to be recalculated.
+ #
+ # Args:
+ # connect_artifact_cache: Whether to try to contact remote artifact
caches
+ # connect_source_cache: Whether to try to contact remote source caches
+ # artifact_remote: An overriding artifact cache remote, or None
+ # source_remote: An overriding source cache remote, or None
+ #
+ def initialize_remotes(
+ self,
+ connect_artifact_cache: bool,
+ connect_source_cache: bool,
+ artifact_remote: Optional[RemoteSpec],
+ source_remote: Optional[RemoteSpec],
+ ) -> None:
+
+ # Ensure all projects are fully loaded.
+ for project in self._projects:
+ project.ensure_fully_loaded()
+
+ #
+ # If the global remote execution specs have been overridden by the
+ # toplevel project, then adjust them now that we're all loaded.
+ #
+ project = self.get_toplevel_project()
+ if project:
+ override_node = self.get_overrides(project.name)
+ if override_node:
+ remote_execution =
override_node.get_mapping("remote-execution", default=None)
+ if remote_execution:
+ self.pull_artifact_files, self.remote_execution_specs =
self._load_remote_execution(
+ remote_execution
+ )
+
+ # Collect a table of which specs apply to each project, these
+ # are calculated here and handed over to the asset caches.
+ #
+ project_artifact_cache_specs: Dict[str, List[RemoteSpec]] = {}
+ project_source_cache_specs: Dict[str, List[RemoteSpec]] = {}
+
+ cli_artifact_remotes = [artifact_remote] if artifact_remote else []
+ cli_source_remotes = [source_remote] if source_remote else []
+
+ #
+ # Maintain our list of remote specs for artifact and source caches
+ #
+ for project in self._projects:
+
+ artifact_specs: List[RemoteSpec] = []
+ source_specs: List[RemoteSpec] = []
+
+ override_node = self.get_overrides(project.name)
+
+ # Resolve which remote specs to use, CLI -> Override -> Global ->
Project recommendation
+ if connect_artifact_cache:
+ caches = override_node.get_sequence("artifacts", default=[],
allowed_types=[MappingNode])
+ override_artifact_specs: List[RemoteSpec] =
[RemoteSpec.new_from_node(node) for node in caches]
+ artifact_specs = (
+ cli_artifact_remotes
+ or override_artifact_specs
+ or self._global_artifact_cache_specs
+ or project.artifact_cache_specs
+ )
+ artifact_specs = list(utils._deduplicate(artifact_specs))
+
+ if connect_source_cache:
+ caches = override_node.get_sequence("source-caches",
default=[], allowed_types=[MappingNode])
+ override_source_specs: List[RemoteSpec] =
[RemoteSpec.new_from_node(node) for node in caches]
+ source_specs = (
+ cli_source_remotes
+ or override_source_specs
+ or self._global_source_cache_specs
+ or project.source_cache_specs
+ )
+ source_specs = list(utils._deduplicate(source_specs))
+
+ # Store them for lookups later on
+ project_artifact_cache_specs[project.name] = artifact_specs
+ project_source_cache_specs[project.name] = source_specs
+
+ #
+ # Now that we know which remote specs are going to be used,
maintain
+ # our total set of overall active remote specs, this helps the
asset cache
+ # modules to maintain a remote connection for the required remotes.
+ #
+ for spec in artifact_specs:
+ self._active_artifact_cache_specs.add(spec)
+ for spec in source_specs:
+ self._active_source_cache_specs.add(spec)
+
+ # Now initialize the underlying asset caches
+ #
+ self.artifactcache.setup_remotes(self._active_artifact_cache_specs,
project_artifact_cache_specs)
+
self.elementsourcescache.setup_remotes(self._active_source_cache_specs,
project_source_cache_specs)
+ self.sourcecache.setup_remotes(self._active_source_cache_specs,
project_source_cache_specs)
# get_workspaces():
#
@@ -542,6 +639,10 @@ class Context:
# does not require file contents of all artifacts to be available in the
# local cache.
#
+ # XXX FIXME: This API can be removed, it is determined by configuration
+ # belonging to the Context anyway.
+ #
+ #
def set_artifact_files_optional(self):
self.require_artifact_files = False
diff --git a/src/buildstream/_elementsourcescache.py
b/src/buildstream/_elementsourcescache.py
index 873806e..c80ad4b 100644
--- a/src/buildstream/_elementsourcescache.py
+++ b/src/buildstream/_elementsourcescache.py
@@ -32,10 +32,6 @@ REMOTE_ASSET_SOURCE_URN_TEMPLATE =
"urn:fdc:buildstream.build:2020:source:{}"
# context (Context): The Buildstream context
#
class ElementSourcesCache(AssetCache):
-
- spec_name = "source_cache_specs"
- config_node_name = "source-caches"
-
def __init__(self, context):
super().__init__(context)
@@ -86,11 +82,13 @@ class ElementSourcesCache(AssetCache):
uri = REMOTE_ASSET_SOURCE_URN_TEMPLATE.format(ref)
+ index_remotes, storage_remotes = self.get_remotes(project.name, False)
+
source_digest = None
errors = []
# Start by pulling our source proto, so that we know which
# blobs to pull
- for remote in self._index_remotes[project]:
+ for remote in index_remotes:
remote.init()
try:
plugin.status("Pulling source {} <- {}".format(display_key,
remote))
@@ -114,7 +112,7 @@ class ElementSourcesCache(AssetCache):
return False
errors = []
- for remote in self._storage_remotes[project]:
+ for remote in storage_remotes:
remote.init()
try:
plugin.status("Pulling data for source {} <-
{}".format(display_key, remote))
@@ -160,8 +158,7 @@ class ElementSourcesCache(AssetCache):
uri = REMOTE_ASSET_SOURCE_URN_TEMPLATE.format(ref)
- index_remotes = [r for r in self._index_remotes[project] if
r.spec.push]
- storage_remotes = [r for r in self._storage_remotes[project] if
r.spec.push]
+ index_remotes, storage_remotes = self.get_remotes(project.name, True)
source_proto = self.load_proto(sources)
source_digest =
self.cas.add_object(buffer=source_proto.SerializeToString())
diff --git a/src/buildstream/_sourcecache.py b/src/buildstream/_sourcecache.py
index 475a166..2284d0d 100644
--- a/src/buildstream/_sourcecache.py
+++ b/src/buildstream/_sourcecache.py
@@ -36,10 +36,6 @@ REMOTE_ASSET_SOURCE_URN_TEMPLATE =
"urn:fdc:buildstream.build:2020:source:{}"
# context (Context): The Buildstream context
#
class SourceCache(AssetCache):
-
- spec_name = "source_cache_specs"
- config_node_name = "source-caches"
-
def __init__(self, context):
super().__init__(context)
@@ -117,8 +113,7 @@ class SourceCache(AssetCache):
project = source._get_project()
display_key = source._get_brief_display_key()
- index_remotes = self._index_remotes[project]
- storage_remotes = self._storage_remotes[project]
+ index_remotes, storage_remotes = self.get_remotes(project.name, False)
# First fetch the source directory digest so we know what to pull
source_digest = None
@@ -172,13 +167,7 @@ class SourceCache(AssetCache):
ref = source._get_source_name()
project = source._get_project()
- index_remotes = []
- storage_remotes = []
-
- # find configured push remotes for this source
- if self._has_push_remotes:
- index_remotes = [r for r in self._index_remotes[project] if
r.spec.push]
- storage_remotes = [r for r in self._storage_remotes[project] if
r.spec.push]
+ index_remotes, storage_remotes = self.get_remotes(project.name, True)
pushed_storage = False
pushed_index = False
diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py
index fa2421e..1728bd7 100644
--- a/src/buildstream/_stream.py
+++ b/src/buildstream/_stream.py
@@ -47,6 +47,7 @@ from ._scheduler import (
from .element import Element
from ._profile import Topics, PROFILER
from ._project import ProjectRefStorage
+from ._remotespec import RemoteType, RemoteSpec
from ._state import State
from .types import _KeyStrength, _PipelineSelection, _Scope
from .plugin import Plugin
@@ -137,7 +138,7 @@ class Stream:
# targets (list of str): Targets to pull
# selection (_PipelineSelection): The selection mode for the specified
targets
# except_targets (list of str): Specified targets to except from
fetching
- # use_artifact_config (bool): If artifact remote configs should be
loaded
+ # connect_artifact_cache (bool): Whether to try to contact remote
artifact caches
# load_artifacts (bool): Whether to load artifacts with artifact names
#
# Returns:
@@ -148,7 +149,7 @@ class Stream:
*,
selection=_PipelineSelection.NONE,
except_targets=(),
- use_artifact_config=False,
+ connect_artifact_cache=False,
load_artifacts=False,
):
with PROFILER.profile(Topics.LOAD_SELECTION,
"_".join(t.replace(os.sep, "-") for t in targets)):
@@ -156,7 +157,7 @@ class Stream:
targets,
selection=selection,
except_targets=except_targets,
- use_artifact_config=use_artifact_config,
+ connect_artifact_cache=connect_artifact_cache,
load_artifacts=load_artifacts,
)
@@ -200,7 +201,7 @@ class Stream:
else:
selection = _PipelineSelection.BUILD if scope == _Scope.BUILD else
_PipelineSelection.RUN
- elements = self.load_selection((element,), selection=selection,
use_artifact_config=True)
+ elements = self.load_selection((element,), selection=selection,
connect_artifact_cache=True)
# Get element to stage from `targets` list.
# If scope is BUILD, it will not be in the `elements` list.
@@ -269,17 +270,13 @@ class Stream:
#
def build(self, targets, *, selection=_PipelineSelection.PLAN,
ignore_junction_targets=False, remote=None):
- use_config = True
- if remote:
- use_config = False
-
elements = self._load(
targets,
selection=selection,
ignore_junction_targets=ignore_junction_targets,
- use_artifact_config=use_config,
+ connect_artifact_cache=True,
artifact_remote_url=remote,
- use_source_config=True,
+ connect_source_cache=True,
dynamic_plan=True,
)
@@ -330,15 +327,11 @@ class Stream:
#
def fetch(self, targets, *, selection=_PipelineSelection.PLAN,
except_targets=None, remote=None):
- use_source_config = True
- if remote:
- use_source_config = False
-
elements = self._load(
targets,
selection=selection,
except_targets=except_targets,
- use_source_config=use_source_config,
+ connect_source_cache=True,
source_remote_url=remote,
)
@@ -394,16 +387,8 @@ class Stream:
#
def source_push(self, targets, *, selection=_PipelineSelection.NONE,
remote=None):
- use_source_config = True
- if remote:
- use_source_config = False
-
elements = self._load(
- targets,
- selection=selection,
- use_source_config=use_source_config,
- source_remote_url=remote,
- load_artifacts=True,
+ targets, selection=selection, connect_source_cache=True,
source_remote_url=remote, load_artifacts=True,
)
if not self._sourcecache.has_push_remotes():
@@ -433,15 +418,11 @@ class Stream:
#
def pull(self, targets, *, selection=_PipelineSelection.NONE,
ignore_junction_targets=False, remote=None):
- use_config = True
- if remote:
- use_config = False
-
elements = self._load(
targets,
selection=selection,
ignore_junction_targets=ignore_junction_targets,
- use_artifact_config=use_config,
+ connect_artifact_cache=True,
artifact_remote_url=remote,
load_artifacts=True,
attempt_artifact_metadata=True,
@@ -475,15 +456,11 @@ class Stream:
#
def push(self, targets, *, selection=_PipelineSelection.NONE,
ignore_junction_targets=False, remote=None):
- use_config = True
- if remote:
- use_config = False
-
elements = self._load(
targets,
selection=selection,
ignore_junction_targets=ignore_junction_targets,
- use_artifact_config=use_config,
+ connect_artifact_cache=True,
artifact_remote_url=remote,
load_artifacts=True,
)
@@ -536,7 +513,7 @@ class Stream:
elements = self._load(
(target,),
selection=selection,
- use_artifact_config=True,
+ connect_artifact_cache=True,
load_artifacts=True,
attempt_artifact_metadata=True,
)
@@ -624,7 +601,7 @@ class Stream:
def artifact_show(self, targets, *, selection=_PipelineSelection.NONE):
# Obtain list of Element and/or ArtifactElement objects
target_objects = self.load_selection(
- targets, selection=selection, use_artifact_config=True,
load_artifacts=True
+ targets, selection=selection, connect_artifact_cache=True,
load_artifacts=True
)
if self._artifacts.has_fetch_remotes():
@@ -1255,38 +1232,6 @@ class Stream:
return elements, except_elements, artifacts
- # _connect_remotes()
- #
- # Connect to the source and artifact remotes.
- #
- # Args:
- # artifact_url: The url of the artifact server to connect to.
- # source_url: The url of the source server to connect to.
- # use_artifact_config: Whether to use the artifact config.
- # use_source_config: Whether to use the source config.
- # reinitialize: Whether to reinitialize from scratch
- #
- def _connect_remotes(
- self,
- artifact_url: str,
- source_url: str,
- use_artifact_config: bool,
- use_source_config: bool,
- reinitialize: bool = False,
- ):
- # ArtifactCache.setup_remotes expects all projects to be fully loaded
- for project in self._context.get_projects():
- project.ensure_fully_loaded()
-
- # Connect to remote caches, this needs to be done before resolving
element state
- self._artifacts.setup_remotes(
- use_config=use_artifact_config, remote_url=artifact_url,
reinitialize=reinitialize
- )
- self._elementsourcescache.setup_remotes(
- use_config=use_source_config, remote_url=source_url,
reinitialize=reinitialize
- )
- self._sourcecache.setup_remotes(use_config=use_source_config,
remote_url=source_url, reinitialize=reinitialize)
-
# _resolve_cached_remotely()
#
# Checks whether the listed elements are currently cached in
@@ -1411,8 +1356,8 @@ class Stream:
# selection (_PipelineSelection): The selection mode for the specified
targets
# except_targets (list of str): Specified targets to except from
fetching
# ignore_junction_targets (bool): Whether junction targets should be
filtered out
- # use_artifact_config (bool): Whether to initialize artifacts with the
config
- # use_source_config (bool): Whether to initialize remote source caches
with the config
+ # connect_artifact_cache (bool): Whether to try to contact remote
artifact caches
+ # connect_source_cache (bool): Whether to try to contact remote source
caches
# artifact_remote_url (str): A remote url for initializing the artifacts
# source_remote_url (str): A remote url for initializing source caches
# dynamic_plan (bool): Require artifacts as needed during the build
@@ -1430,8 +1375,8 @@ class Stream:
selection=_PipelineSelection.NONE,
except_targets=(),
ignore_junction_targets=False,
- use_artifact_config=False,
- use_source_config=False,
+ connect_artifact_cache=False,
+ connect_source_cache=False,
artifact_remote_url=None,
source_remote_url=None,
dynamic_plan=False,
@@ -1455,8 +1400,20 @@ class Stream:
# Hold on to the targets
self.targets = elements
+ # FIXME: Instead of converting the URL to a RemoteSpec here, the CLI
needs to
+ # be enhanced to parse a fully qualified RemoteSpec (including
certs etc)
+ # from the command line, the CLI should be feeding the
RemoteSpec through
+ # the Stream API directly.
+ #
+ artifact_remote = None
+ if artifact_remote_url:
+ artifact_remote = RemoteSpec(RemoteType.ALL, artifact_remote_url,
push=True)
+ source_remote = None
+ if source_remote_url:
+ source_remote = RemoteSpec(RemoteType.ALL, source_remote_url,
push=True)
+
# Connect to remote caches, this needs to be done before resolving
element state
- self._connect_remotes(artifact_remote_url, source_remote_url,
use_artifact_config, use_source_config)
+ self._context.initialize_remotes(connect_artifact_cache,
connect_source_cache, artifact_remote, source_remote)
# In some cases we need to have an actualized artifact, with all of
# it's metadata, such that we can derive attributes about the artifact
@@ -1482,14 +1439,12 @@ class Stream:
artifact_targets, [], rewritable=False,
valid_artifact_names=True
)
- # FIXME:
- #
- # Sadly, we need to reinitialize just because we
re-instantiated new projects due to
- # downloading artifacts - this could be fixed by addressing the
awkward structure
- # of remotes in the asset caches.
+ # It can be that new remotes have been added by way of loading new
+ # projects referenced by the new artifact elements, so we need to
+ # ensure those remotes are also initialized.
#
- self._connect_remotes(
- artifact_remote_url, source_remote_url, use_artifact_config,
use_source_config, reinitialize=True
+ self._context.initialize_remotes(
+ connect_artifact_cache, connect_source_cache, artifact_remote,
source_remote
)
self.targets += artifacts
diff --git a/src/buildstream/element.py b/src/buildstream/element.py
index 503361e..c8c5807 100644
--- a/src/buildstream/element.py
+++ b/src/buildstream/element.py
@@ -1959,6 +1959,7 @@ class Element(Plugin):
# (bool): True if this element does not need a push job to be created
#
def _skip_push(self, *, skip_uncached):
+
if not self.__artifacts.has_push_remotes(plugin=self):
# No push remotes for this element's project
return True
diff --git a/tests/artifactcache/capabilities.py
b/tests/artifactcache/capabilities.py
index 0f2f1aa..a74ed6c 100644
--- a/tests/artifactcache/capabilities.py
+++ b/tests/artifactcache/capabilities.py
@@ -40,8 +40,8 @@ def
test_artifact_cache_with_missing_capabilities_is_skipped(cli, tmpdir, datafi
# Create a local artifact cache handle
artifactcache = context.artifactcache
- # Manually setup the CAS remote
- artifactcache.setup_remotes(use_config=True)
+ # Initialize remotes
+ context.initialize_remotes(True, True, None, None)
assert (
not artifactcache.has_fetch_remotes()
diff --git a/tests/artifactcache/config.py b/tests/artifactcache/config.py
index 1b8cbad..3bf853a 100644
--- a/tests/artifactcache/config.py
+++ b/tests/artifactcache/config.py
@@ -1,13 +1,11 @@
# Pylint doesn't play well with fixtures and dependency injection from pytest
# pylint: disable=redefined-outer-name
-import itertools
import os
import pytest
from buildstream._remotespec import RemoteSpec, RemoteType
-from buildstream._artifactcache import ArtifactCache
from buildstream._project import Project
from buildstream.utils import _deduplicate
from buildstream import _yaml
@@ -102,11 +100,13 @@ def test_artifact_cache_precedence(tmpdir,
override_caches, project_caches, user
project = Project(str(project_dir), context)
project.ensure_fully_loaded()
- # Use the helper from the artifactcache module to parse our
configuration.
- parsed_cache_specs =
ArtifactCache._configured_remote_cache_specs(context, project)
+ # Check the specs which the artifact cache thinks are configured
+ context.initialize_remotes(True, True, None, None)
+ artifactcache = context.artifactcache
+ parsed_cache_specs = artifactcache._project_specs[project.name]
# Verify that it was correctly read.
- expected_cache_specs =
list(_deduplicate(itertools.chain(override_caches, project_caches,
user_caches)))
+ expected_cache_specs = list(_deduplicate(override_caches or
user_caches or project_caches))
assert parsed_cache_specs == expected_cache_specs
@@ -220,8 +220,10 @@ def test_paths_for_artifact_config_are_expanded(tmpdir,
monkeypatch, artifacts_c
project = Project(str(project_dir), context)
project.ensure_fully_loaded()
- # Use the helper from the artifactcache module to parse our
configuration.
- parsed_cache_specs =
ArtifactCache._configured_remote_cache_specs(context, project)
+ # Check the specs which the artifact cache thinks are configured
+ context.initialize_remotes(True, True, None, None)
+ artifactcache = context.artifactcache
+ parsed_cache_specs = artifactcache._project_specs[project.name]
if isinstance(artifacts_config, dict):
artifacts_config = [artifacts_config]
diff --git a/tests/artifactcache/pull.py b/tests/artifactcache/pull.py
index fc27e75..502c350 100644
--- a/tests/artifactcache/pull.py
+++ b/tests/artifactcache/pull.py
@@ -86,8 +86,8 @@ def test_pull(cli, tmpdir, datafiles):
# Create a local artifact cache handle
artifactcache = context.artifactcache
- # Manually setup the CAS remote
- artifactcache.setup_remotes(use_config=True)
+ # Initialize remotes
+ context.initialize_remotes(True, True, None, None)
assert artifactcache.has_push_remotes(plugin=element), "No remote
configured for element target.bst"
assert artifactcache.pull(element, element_key), "Pull operation
failed"
@@ -138,9 +138,10 @@ def test_pull_tree(cli, tmpdir, datafiles):
# Retrieve the Directory object from the cached artifact
artifact_digest = cli.artifact.get_digest(rootcache_dir, element,
element_key)
+ # Initialize remotes
+ context.initialize_remotes(True, True, None, None)
+
artifactcache = context.artifactcache
- # Manually setup the CAS remote
- artifactcache.setup_remotes(use_config=True)
assert artifactcache.has_push_remotes()
directory = remote_execution_pb2.Directory()
@@ -161,7 +162,7 @@ def test_pull_tree(cli, tmpdir, datafiles):
cli.remove_artifact_from_cache(project_dir, "target.bst")
# Assert that we are not cached locally anymore
- artifactcache.close_grpc_channels()
+ artifactcache.release_resources()
cas._casd_channel.request_shutdown()
cas.close_grpc_channels()
assert cli.get_element_state(project_dir, "target.bst") != "cached"
diff --git a/tests/artifactcache/push.py b/tests/artifactcache/push.py
index 02ff259..48985df 100644
--- a/tests/artifactcache/push.py
+++ b/tests/artifactcache/push.py
@@ -39,9 +39,8 @@ def _push(cli, cache_dir, project_dir, config_file, target):
for e in element._dependencies(_Scope.ALL):
e._initialize_state()
- # Manually setup the CAS remotes
- artifactcache.setup_remotes(use_config=True)
- artifactcache.initialize_remotes()
+ # Initialize remotes
+ context.initialize_remotes(True, True, None, None)
assert artifactcache.has_push_remotes(plugin=element), "No remote
configured for element target.bst"
assert element._push(), "Push operation failed"
@@ -139,9 +138,8 @@ def test_push_message(tmpdir, datafiles):
# Create a local artifact cache handle
artifactcache = context.artifactcache
- # Manually setup the artifact remote
- artifactcache.setup_remotes(use_config=True)
- artifactcache.initialize_remotes()
+ # Initialize remotes
+ context.initialize_remotes(True, True, None, None)
assert artifactcache.has_push_remotes()
command = remote_execution_pb2.Command(
diff --git a/tests/frontend/pull.py b/tests/frontend/pull.py
index b721ca8..6733584 100644
--- a/tests/frontend/pull.py
+++ b/tests/frontend/pull.py
@@ -391,18 +391,14 @@ def test_build_remote_option(caplog, cli, tmpdir,
datafiles):
with create_artifact_share(os.path.join(str(tmpdir), "artifactshare1")) as
shareuser, create_artifact_share(
os.path.join(str(tmpdir), "artifactshare2")
- ) as shareproject, create_artifact_share(os.path.join(str(tmpdir),
"artifactshare3")) as sharecli:
-
- # Add shareproject repo url to project.conf
- with open(os.path.join(project, "project.conf"), "a") as projconf:
- projconf.write("artifacts:\n- url: {}\n push:
True".format(shareproject.repo))
+ ) as sharecli:
# Configure shareuser remote in user conf
cli.configure({"artifacts": [{"url": shareuser.repo, "push": True}]})
- # Push the artifacts to the shareuser and shareproject remotes.
- # Assert that shareuser and shareproject have the artfifacts cached,
- # but sharecli doesn't, then delete locally cached elements
+ # Push the artifacts to the shareuser remote.
+ # Assert that shareuser has the artfifacts cached, but sharecli
doesn't,
+ # then delete locally cached elements
result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
all_elements = ["target.bst", "import-bin.bst", "compose-all.bst"]
@@ -410,11 +406,10 @@ def test_build_remote_option(caplog, cli, tmpdir,
datafiles):
assert element_name in result.get_pushed_elements()
assert_not_shared(cli, sharecli, project, element_name)
assert_shared(cli, shareuser, project, element_name)
- assert_shared(cli, shareproject, project, element_name)
cli.remove_artifact_from_cache(project, element_name)
# Now check that a build with cli set as sharecli results in nothing
being pulled,
- # as it doesn't have them cached and shareuser/shareproject should be
ignored. This
+ # as it doesn't have them cached and shareuser should be ignored. This
# will however result in the artifacts being built and pushed to it
result = cli.run(project=project, args=["build", "--remote",
sharecli.repo, "target.bst"])
result.assert_success()
@@ -430,7 +425,6 @@ def test_build_remote_option(caplog, cli, tmpdir,
datafiles):
for element_name in all_elements:
assert cli.get_element_state(project, element_name) == "cached"
assert element_name in result.get_pulled_elements()
- assert shareproject.repo not in result.stderr
assert shareuser.repo not in result.stderr
assert sharecli.repo in result.stderr
diff --git a/tests/sourcecache/capabilities.py
b/tests/sourcecache/capabilities.py
index 964b458..67be749 100644
--- a/tests/sourcecache/capabilities.py
+++ b/tests/sourcecache/capabilities.py
@@ -33,12 +33,12 @@ def
test_artifact_cache_with_missing_capabilities_is_skipped(cli, tmpdir, datafi
project = Project(project_dir, context)
project.ensure_fully_loaded()
+ # Initialize remotes
+ context.initialize_remotes(True, True, None, None)
+
# Create a local artifact cache handle
sourcecache = context.sourcecache
- # Manually setup the CAS remote
- sourcecache.setup_remotes(use_config=True)
-
assert (
not sourcecache.has_fetch_remotes()
), "System didn't realize the source cache didn't support
BuildStream"
