gtristan commented on a change in pull request #1609:
URL: https://github.com/apache/buildstream/pull/1609#discussion_r827683525
##########
File path: src/buildstream/storage/_casbaseddirectory.py
##########
@@ -114,136 +128,337 @@ def get_equivalency_properties(e: IndexEntry):
# CasBasedDirectory intentionally doesn't call its superclass constuctor,
# which is meant to be unimplemented.
# pylint: disable=super-init-not-called
+class CasBasedDirectory(Directory):
+ def __init__(
+ self,
+ cas_cache: CASCache,
+ *,
+ digest=None,
+ parent: Optional["CasBasedDirectory"] = None,
+ filename: Optional[str] = None
+ ) -> None:
+ # The CAS cache
+ self.__cas_cache: CASCache = cas_cache
+ # The name of this directory
+ self.__filename: Optional[str] = filename
-class CasBasedDirectory(Directory):
- """
- CAS-based directories can have two names; one is a 'common name' which has
no effect
- on functionality, and the 'filename'. If a CasBasedDirectory has a parent,
then 'filename'
- must be the name of an entry in the parent directory's index which points
to this object.
- This is used to inform a parent directory that it must update the given
hash for this
- object when this object changes.
-
- Typically a top-level CasBasedDirectory will have a common_name and no
filename, and
- subdirectories wil have a filename and no common_name. common_name can
used to identify
- CasBasedDirectory objects in a log file, since they have no unique
position in a file
- system.
- """
-
- # Two constants which define the separators used by the remote execution
API.
- _pb2_path_sep = "/"
- _pb2_absolute_path_prefix = "/"
-
- def __init__(self, cas_cache, *, digest=None, parent=None,
common_name="untitled", filename=None):
- self.filename = filename
- self.common_name = common_name
- self.cas_cache = cas_cache
+ # The remote_execution_pb2.Digest of this directory
self.__digest = digest
- self.index = {}
- self.parent = parent
- self.__subtree_read_only = None
- if digest:
- self._populate_index(digest)
+ # The parent directory
+ self.__parent: Optional["CasBasedDirectory"] = parent
- # _clear():
- #
- # Remove all entries from this directory.
- #
- def _clear(self):
- self.__invalidate_digest()
- self.index = {}
+ # An index of directory entries
+ self.__index: Dict[str, _IndexEntry] = {}
- # _reset():
- #
- # Replace the contents of this directory with the entries from the
specified
- # directory digest.
- #
- # Args:
- # digest (Digest): The digest of the replacement directory
- #
- def _reset(self, *, digest=None):
- self._clear()
+ # Whether this directory and it's subdirectories should be read-only
+ self.__subtree_read_only: Optional[bool] = None
if digest:
- self.__digest = digest
- self._populate_index(digest)
+ self.__populate_index(digest)
- def _populate_index(self, digest):
- try:
- pb2_directory = remote_execution_pb2.Directory()
- with open(self.cas_cache.objpath(digest), "rb") as f:
- pb2_directory.ParseFromString(f.read())
- except FileNotFoundError as e:
- raise VirtualDirectoryError("Directory not found in local cache:
{}".format(e)) from e
+ def __iter__(self) -> Iterator[str]:
+ yield from self.__index.keys()
- for prop in pb2_directory.node_properties.properties:
- if prop.name == "SubtreeReadOnly":
- self.__subtree_read_only = prop.value == "true"
+ def __str__(self) -> str:
+ return "[CAS:{}]".format(self.__get_identifier())
- for entry in pb2_directory.directories:
- self.index[entry.name] = IndexEntry(entry.name,
_FileType.DIRECTORY, digest=entry.digest)
- for entry in pb2_directory.files:
- if entry.node_properties.HasField("mtime"):
- mtime = entry.node_properties.mtime
- else:
- mtime = None
+ #############################################################
+ # Implementation of Public API #
+ #############################################################
- self.index[entry.name] = IndexEntry(
- entry.name,
- _FileType.REGULAR_FILE,
- digest=entry.digest,
- is_executable=entry.is_executable,
- mtime=mtime,
+ def descend(self, path: str, *, create: bool = False, follow_symlinks:
bool = False) -> "CasBasedDirectory":
+ self._validate_path(path)
+ paths = path.split("/")
+ return self.__descend(paths, create=create,
follow_symlinks=follow_symlinks)
+
+ def import_single_file(self, external_pathspec: str) -> FileListResult:
+ result = FileListResult()
+ if self.__check_replacement(os.path.basename(external_pathspec),
os.path.dirname(external_pathspec), result):
+ self.__add_file(
+ os.path.basename(external_pathspec), external_pathspec,
properties=None,
)
- for entry in pb2_directory.symlinks:
- self.index[entry.name] = IndexEntry(entry.name, _FileType.SYMLINK,
target=entry.target)
+ result.files_written.append(external_pathspec)
+ return result
+
+ def export_to_tar(self, tarfile: TarFile, destination_dir: str, mtime: int
= BST_ARBITRARY_TIMESTAMP) -> None:
+ for filename, entry in sorted(self.__index.items()):
+ arcname = os.path.join(destination_dir, filename)
+ if entry.type == FileType.DIRECTORY:
+ tarinfo = tarfilelib.TarInfo(arcname)
+ tarinfo.mtime = mtime
+ tarinfo.type = tarfilelib.DIRTYPE
+ tarinfo.mode = 0o755
+ tarfile.addfile(tarinfo)
+ self.descend(filename).export_to_tar(tarfile, arcname, mtime)
+ elif entry.type == FileType.REGULAR_FILE:
+ source_name = self.__cas_cache.objpath(entry.digest)
+ tarinfo = tarfilelib.TarInfo(arcname)
+ tarinfo.mtime = mtime
+ if entry.is_executable:
+ tarinfo.mode |= stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH
+ tarinfo.size = os.path.getsize(source_name)
+ with open(source_name, "rb") as f:
+ tarfile.addfile(tarinfo, f)
+ elif entry.type == FileType.SYMLINK:
+ assert entry.target is not None
+ tarinfo = tarfilelib.TarInfo(arcname)
+ tarinfo.mtime = mtime
+ tarinfo.mode = 0o777
+ tarinfo.linkname = entry.target
+ tarinfo.type = tarfilelib.SYMTYPE
+ sio = StringIO(entry.target)
+ bio = BytesIO(sio.read().encode("utf8"))
+ tarfile.addfile(tarinfo, bio)
+ else:
+ raise DirectoryError("can not export file type {} to
tar".format(entry.type))
+
+ def is_empty(self) -> bool:
+ return len(self.__index) == 0
+
+ def listdir(self) -> Iterator[str]:
Review comment:
oops listdir() is an artifact of something I've already backed out of
the branch.
This is handled by way of the `Directory` being iterable instead, I'll
remove this from wherever it was added.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
