This is an automated email from the ASF dual-hosted git repository.
akitouni pushed a commit to branch abderrahim/cascache-cleanup
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit bbbe9ddcf97f19fcf84533d020db897c6d33ffb6
Author: Abderrahim Kitouni <akito...@gnome.org>
AuthorDate: Sun Apr 27 19:51:52 2025 +0100
Use CASDProcessManager directly to access REAPI stubs
This removes some indirections through CASCache, now that
CASDProcessManager is directly available from the Context
---
src/buildstream/_assetcache.py | 8 ++---
src/buildstream/_cas/cascache.py | 45 ++++++--------------------
src/buildstream/_cas/casremote.py | 10 +++---
src/buildstream/_stream.py | 2 +-
src/buildstream/sandbox/_reremote.py | 13 ++++----
src/buildstream/sandbox/_sandboxbuildboxrun.py | 7 ++--
src/buildstream/sandbox/_sandboxremote.py | 4 +--
7 files changed, 30 insertions(+), 59 deletions(-)
diff --git a/src/buildstream/_assetcache.py b/src/buildstream/_assetcache.py
index df70fc2ff..8e883fdfd 100644
--- a/src/buildstream/_assetcache.py
+++ b/src/buildstream/_assetcache.py
@@ -20,7 +20,7 @@ from typing import List, Dict, Tuple, Iterable, Optional
import grpc
from . import utils
-from ._cas import CASRemote, CASCache, CASDProcessManager
+from ._cas import CASRemote, CASDProcessManager
from ._exceptions import AssetCacheError, RemoteError
from ._remotespec import RemoteSpec, RemoteType
from ._remote import BaseRemote
@@ -264,7 +264,7 @@ class AssetRemote(BaseRemote):
# to establish a connection to this remote at initialization time.
#
class RemotePair:
- def __init__(self, casd: CASDProcessManager, cas: CASCache, spec:
RemoteSpec):
+ def __init__(self, casd: CASDProcessManager, spec: RemoteSpec):
self.index: Optional[AssetRemote] = None
self.storage: Optional[CASRemote] = None
self.error: Optional[str] = None
@@ -275,7 +275,7 @@ class RemotePair:
index.check()
self.index = index
if spec.remote_type in [RemoteType.STORAGE, RemoteType.ALL]:
- storage = CASRemote(spec, cas)
+ storage = CASRemote(spec, casd)
storage.check()
self.storage = storage
except RemoteError as e:
@@ -322,7 +322,7 @@ class AssetCache:
if spec in self._remotes:
continue
- remote = RemotePair(casd, self.cas, spec)
+ remote = RemotePair(casd, spec)
if remote.error:
self.context.messenger.warn("Failed to initialize remote {}:
{}".format(spec.url, remote.error))
diff --git a/src/buildstream/_cas/cascache.py b/src/buildstream/_cas/cascache.py
index fbfc150f3..68fd4b610 100644
--- a/src/buildstream/_cas/cascache.py
+++ b/src/buildstream/_cas/cascache.py
@@ -71,25 +71,9 @@ class CASCache:
else:
assert not self._remote_cache
- self._default_remote = CASRemote(None, self)
+ self._default_remote = CASRemote(None, casd)
self._default_remote.init()
- # get_cas():
- #
- # Return ContentAddressableStorage stub for buildbox-casd channel.
- #
- def get_cas(self):
- assert self._casd, "CASCache was created without buildbox-casd"
- return self._casd.get_cas()
-
- # get_local_cas():
- #
- # Return LocalCAS stub for buildbox-casd channel.
- #
- def get_local_cas(self):
- assert self._casd, "CASCache was created without buildbox-casd"
- return self._casd.get_local_cas()
-
# preflight():
#
# Preflight check.
@@ -133,7 +117,7 @@ class CASCache:
# Returns: True if the directory is available in the local cache
#
def contains_directory(self, digest):
- local_cas = self.get_local_cas()
+ local_cas = self._casd.get_local_cas()
# Without a remote cache, `FetchTree` simply checks the local cache.
request = local_cas_pb2.FetchTreeRequest()
@@ -241,7 +225,7 @@ class CASCache:
#
def ensure_tree(self, tree):
if self._remote_cache:
- local_cas = self.get_local_cas()
+ local_cas = self._casd.get_local_cas()
request = local_cas_pb2.FetchTreeRequest()
request.root_digest.CopyFrom(tree)
@@ -260,7 +244,7 @@ class CASCache:
# dir_digest (Digest): Digest object for the directory to fetch.
#
def fetch_directory(self, remote, dir_digest):
- local_cas = self.get_local_cas()
+ local_cas = self._casd.get_local_cas()
request = local_cas_pb2.FetchTreeRequest()
request.instance_name = remote.local_cas_instance_name
@@ -399,7 +383,7 @@ class CASCache:
for path in paths:
request.path.append(path)
- local_cas = self.get_local_cas()
+ local_cas = self._casd.get_local_cas()
response = local_cas.CaptureFiles(request)
@@ -432,7 +416,7 @@ class CASCache:
# (Digest): The digest of the imported directory
#
def import_directory(self, path: str, properties: Optional[List[str]] =
None) -> SourceRef:
- local_cas = self.get_local_cas()
+ local_cas = self._casd.get_local_cas()
request = local_cas_pb2.CaptureTreeRequest()
request.path.append(path)
@@ -478,7 +462,7 @@ class CASCache:
#
@contextlib.contextmanager
def stage_directory(self, directory_digest):
- local_cas = self.get_local_cas()
+ local_cas = self._casd.get_local_cas()
request = local_cas_pb2.StageTreeRequest()
request.root_digest.CopyFrom(directory_digest)
@@ -535,7 +519,7 @@ class CASCache:
# Returns: List of missing Digest objects
#
def missing_blobs(self, blobs, *, remote=None):
- cas = self.get_cas()
+ cas = self._casd.get_cas()
if remote:
instance_name = remote.local_cas_instance_name
@@ -576,7 +560,7 @@ class CASCache:
if self._remote_cache and _fetch_tree:
# Ensure we have the directory protos in the local cache
- local_cas = self.get_local_cas()
+ local_cas = self._casd.get_local_cas()
request = local_cas_pb2.FetchTreeRequest()
request.root_digest.CopyFrom(directory_digest)
@@ -719,17 +703,6 @@ class CASCache:
def get_cache_usage(self):
return self._cache_usage_monitor.get_cache_usage()
- # get_casd()
- #
- # Get the underlying buildbox-casd process
- #
- # Returns:
- # (subprocess.Process): The casd process that is used for the current
cascache
- #
- def get_casd(self):
- assert self._casd is not None, "Only call this with a running
buildbox-casd process"
- return self._casd
-
# _CASCacheUsage
#
diff --git a/src/buildstream/_cas/casremote.py
b/src/buildstream/_cas/casremote.py
index 2db8033e4..3fefb9d47 100644
--- a/src/buildstream/_cas/casremote.py
+++ b/src/buildstream/_cas/casremote.py
@@ -37,10 +37,10 @@ class BlobNotFound(CASRemoteError):
# Represents a single remote CAS cache.
#
class CASRemote(BaseRemote):
- def __init__(self, spec, cascache, **kwargs):
+ def __init__(self, spec, casd, **kwargs):
super().__init__(spec, **kwargs)
- self.cascache = cascache
+ self.casd = casd
self.local_cas_instance_name = None
# check_remote
@@ -55,7 +55,7 @@ class CASRemote(BaseRemote):
self.local_cas_instance_name = ""
return
- local_cas = self.cascache.get_local_cas()
+ local_cas = self.casd.get_local_cas()
request = local_cas_pb2.GetInstanceNameForRemotesRequest()
self.spec.to_localcas_remote(request.content_addressable_storage)
response = local_cas.GetInstanceNameForRemotes(request)
@@ -89,7 +89,7 @@ class _CASBatchRead:
if not self._requests:
return
- local_cas = self._remote.cascache.get_local_cas()
+ local_cas = self._remote.casd.get_local_cas()
for request in self._requests:
batch_response = local_cas.FetchMissingBlobs(request)
@@ -143,7 +143,7 @@ class _CASBatchUpdate:
if not self._requests:
return
- local_cas = self._remote.cascache.get_local_cas()
+ local_cas = self._remote.casd.get_local_cas()
for request in self._requests:
batch_response = local_cas.UploadMissingBlobs(request)
diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py
index da8d48550..5b7de307e 100644
--- a/src/buildstream/_stream.py
+++ b/src/buildstream/_stream.py
@@ -1810,7 +1810,7 @@ class Stream:
self._session_start_callback()
self._running = True
- status = self._scheduler.run(self.queues,
self._context.get_cascache().get_casd())
+ status = self._scheduler.run(self.queues, self._context.get_casd())
self._running = False
if status == SchedStatus.ERROR:
diff --git a/src/buildstream/sandbox/_reremote.py
b/src/buildstream/sandbox/_reremote.py
index 79853fa34..923a93efd 100644
--- a/src/buildstream/sandbox/_reremote.py
+++ b/src/buildstream/sandbox/_reremote.py
@@ -22,8 +22,8 @@ from .._exceptions import RemoteError
class RERemote(CASRemote):
- def __init__(self, cas_spec, remote_execution_specs, cascache):
- super().__init__(cas_spec, cascache)
+ def __init__(self, cas_spec, remote_execution_specs, casd):
+ super().__init__(cas_spec, casd)
self.remote_execution_specs = remote_execution_specs
self.exec_service = None
@@ -31,7 +31,7 @@ class RERemote(CASRemote):
self.ac_service = None
def _configure_protocols(self):
- local_cas = self.cascache.get_local_cas()
+ local_cas = self.casd.get_local_cas()
request = local_cas_pb2.GetInstanceNameForRemotesRequest()
if self.remote_execution_specs.storage_spec:
self.remote_execution_specs.storage_spec.to_localcas_remote(request.content_addressable_storage)
@@ -50,10 +50,9 @@ class RERemote(CASRemote):
response = local_cas.GetInstanceNameForRemotes(request)
self.local_cas_instance_name = response.instance_name
- casd = self.cascache.get_casd()
- self.exec_service = casd.get_exec_service()
- self.operations_service = casd.get_operations_service()
- self.ac_service = casd.get_ac_service()
+ self.exec_service = self.casd.get_exec_service()
+ self.operations_service = self.casd.get_operations_service()
+ self.ac_service = self.casd.get_ac_service()
def _check(self):
super()._check()
diff --git a/src/buildstream/sandbox/_sandboxbuildboxrun.py
b/src/buildstream/sandbox/_sandboxbuildboxrun.py
index c87642f6b..007ae95b4 100644
--- a/src/buildstream/sandbox/_sandboxbuildboxrun.py
+++ b/src/buildstream/sandbox/_sandboxbuildboxrun.py
@@ -38,11 +38,11 @@ class SandboxBuildBoxRun(SandboxREAPI):
super().__init__(*args, **kwargs)
context = self._get_context()
- cascache = context.get_cascache()
+ casd = context.get_casd()
re_specs = context.remote_execution_specs
if re_specs and re_specs.action_spec:
- self.re_remote = RERemote(context.remote_cache_spec, re_specs,
cascache)
+ self.re_remote = RERemote(context.remote_cache_spec, re_specs,
casd)
try:
self.re_remote.init()
self.re_remote.check()
@@ -110,8 +110,7 @@ class SandboxBuildBoxRun(SandboxREAPI):
stdout, stderr = self._get_output()
context = self._get_context()
- cascache = context.get_cascache()
- casd = cascache.get_casd()
+ casd = context.get_casd()
config = self._get_config()
if config.remote_apis_socket_path and context.remote_cache_spec and
not self.re_remote:
diff --git a/src/buildstream/sandbox/_sandboxremote.py
b/src/buildstream/sandbox/_sandboxremote.py
index 8072938bf..28d6fcb32 100644
--- a/src/buildstream/sandbox/_sandboxremote.py
+++ b/src/buildstream/sandbox/_sandboxremote.py
@@ -37,7 +37,7 @@ class SandboxRemote(SandboxREAPI):
super().__init__(*args, **kwargs)
context = self._get_context()
- cascache = context.get_cascache()
+ casd = context.get_casd()
specs = context.remote_execution_specs
if specs is None or specs.exec_spec is None:
@@ -48,7 +48,7 @@ class SandboxRemote(SandboxREAPI):
self.action_spec = specs.action_spec
self.operation_name = None
- self.re_remote = RERemote(context.remote_cache_spec, specs, cascache)
+ self.re_remote = RERemote(context.remote_cache_spec, specs, casd)
try:
self.re_remote.init()
except grpc.RpcError as e:
