This is an automated email from the ASF dual-hosted git repository. akitouni pushed a commit to branch abderrahim/ensure-blobs in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit ebb6e55adef918b75906b6eb8bfbc45585aae833 Author: Abderrahim Kitouni <[email protected]> AuthorDate: Thu Aug 1 14:28:06 2024 +0100 storage: ensure blobs are available when using CasBasedDirectory with storage service --- src/buildstream/storage/_casbaseddirectory.py | 4 ++++ src/buildstream/storage/_filebaseddirectory.py | 3 +++ src/buildstream/storage/directory.py | 7 +++++++ 3 files changed, 14 insertions(+) diff --git a/src/buildstream/storage/_casbaseddirectory.py b/src/buildstream/storage/_casbaseddirectory.py index 9c2e83a7b..de65cf848 100644 --- a/src/buildstream/storage/_casbaseddirectory.py +++ b/src/buildstream/storage/_casbaseddirectory.py @@ -185,6 +185,7 @@ class CasBasedDirectory(Directory): return result def export_to_tar(self, tarfile: TarFile, destination_dir: str, mtime: int = BST_ARBITRARY_TIMESTAMP) -> None: + self._ensure() for filename, entry in sorted(self.__index.items()): arcname = os.path.join(destination_dir, filename) if entry.type == FileType.DIRECTORY: @@ -396,6 +397,9 @@ class CasBasedDirectory(Directory): def _set_deterministic_user(self) -> None: pass + def _ensure(self): + self.__cas_cache.ensure_tree(self._get_digest()) + def _get_underlying_path(self, filename) -> str: try: entry = self.__index[filename] diff --git a/src/buildstream/storage/_filebaseddirectory.py b/src/buildstream/storage/_filebaseddirectory.py index 93fc17768..215dc96d3 100644 --- a/src/buildstream/storage/_filebaseddirectory.py +++ b/src/buildstream/storage/_filebaseddirectory.py @@ -267,6 +267,9 @@ class FileBasedDirectory(Directory): # assert isinstance(external_pathspec, Directory) + # Ensure blobs are available locally + external_pathspec._ensure() + def copy_action(src_path, dest_path, mtime, result): utils.safe_copy(src_path, dest_path, result=result) utils._set_file_mtime(dest_path, mtime) diff --git a/src/buildstream/storage/directory.py b/src/buildstream/storage/directory.py index fd19393d0..206930b33 100644 --- a/src/buildstream/storage/directory.py +++ b/src/buildstream/storage/directory.py @@ -466,6 +466,13 @@ class Directory: def _export_files(self, to_directory: str, *, can_link: bool = False, can_destroy: bool = False) -> None: raise NotImplementedError() + # _ensure() + # + # Makes sure the files for the directory are available locally. Should be called before + # using _get_underlying_path() + def _ensure(self): + pass + # _get_underlying_path() # # Args:
