This is an automated email from the ASF dual-hosted git repository. github-bot pushed a commit to branch abderrahim/source-cache in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 829e30b1f50ff70bdd7ffa24fb73549e31c6a460 Author: Abderrahim Kitouni <[email protected]> AuthorDate: Sun Jun 28 09:20:39 2020 +0100 move caching sources from Element to SourceCache --- src/buildstream/_sourcecache.py | 34 ++++++++++++++++++++++++++----- src/buildstream/element.py | 45 ++--------------------------------------- 2 files changed, 31 insertions(+), 48 deletions(-) diff --git a/src/buildstream/_sourcecache.py b/src/buildstream/_sourcecache.py index dcde0b4..41efa5d 100644 --- a/src/buildstream/_sourcecache.py +++ b/src/buildstream/_sourcecache.py @@ -161,15 +161,24 @@ class SourceCache(BaseCache): # dependent on previous sources, such as the patch source. # # Args: - # source: last source - # previous_sources: rest of the sources. - def commit(self, source, previous_sources): + # sources: list of the sources of an element + def commit(self, sources): + last_requires_previous = 0 + # commit all other sources by themselves + for ix, source in enumerate(sources): + if source.BST_REQUIRES_PREVIOUS_SOURCES_STAGE: + self._commit_one(source, sources[last_requires_previous:ix]) + last_requires_previous = ix + else: + self._commit_one(source, []) + + def _commit_one(self, source, previous_sources): ref = source._get_source_name() # Use tmpdir for now vdir = CasBasedDirectory(self.cas) for previous_source in previous_sources: - vdir.import_files(self.export(previous_source)) + vdir.import_files(self._export_one(previous_source)) if not source.BST_STAGE_VIRTUAL_DIRECTORY: with utils._tempdir(dir=self.context.tmpdir, prefix="staging-temp") as tmpdir: @@ -191,7 +200,22 @@ class SourceCache(BaseCache): # # Returns: # CASBasedDirectory - def export(self, source): + def export(self, sources): + # find last required source + last_requires_previous_ix = 0 + for ix, source in enumerate(sources): + if source.BST_REQUIRES_PREVIOUS_SOURCES_STAGE: + last_requires_previous_ix = ix + + import_dir = CasBasedDirectory(self.cas) + + for source in sources[last_requires_previous_ix:]: + source_dir = self._export_one(source) + import_dir.import_files(source_dir) + + return import_dir + + def _export_one(self, source): ref = source._get_source_name() source = self._get_source(ref) return CasBasedDirectory(self.cas, digest=source.files) diff --git a/src/buildstream/element.py b/src/buildstream/element.py index 6a0fa5f..0a3f41c 100644 --- a/src/buildstream/element.py +++ b/src/buildstream/element.py @@ -106,7 +106,6 @@ from ._artifact import Artifact from .storage.directory import Directory from .storage._filebaseddirectory import FileBasedDirectory -from .storage._casbaseddirectory import CasBasedDirectory from .storage.directory import VirtualDirectoryError if TYPE_CHECKING: @@ -1352,16 +1351,8 @@ class Element(Plugin): if self.__sources: - sourcecache = context.sourcecache - # find last required source - last_required_previous_ix = self.__last_source_requires_previous() - import_dir = CasBasedDirectory(context.get_cascache()) - try: - for source in self.__sources[last_required_previous_ix:]: - source_dir = sourcecache.export(source) - import_dir.import_files(source_dir) - + import_dir = context.sourcecache.export(self.__sources) except SourceCacheError as e: raise ElementError("Error trying to export source for {}: {}".format(self.name, e)) except VirtualDirectoryError as e: @@ -2056,7 +2047,7 @@ class Element(Plugin): source._fetch(previous_sources) previous_sources.append(source) - self.__cache_sources() + self.__sourcecache.commit(self.__sources) # _calculate_cache_key(): # @@ -2889,38 +2880,6 @@ class Element(Plugin): return True - # __cache_sources(): - # - # Caches the sources into the local CAS - # - def __cache_sources(self): - if self.__sources and not self._has_all_sources_in_source_cache(): - last_requires_previous = 0 - # commit all other sources by themselves - for ix, source in enumerate(self.__sources): - if source.BST_REQUIRES_PREVIOUS_SOURCES_STAGE: - self.__sourcecache.commit(source, self.__sources[last_requires_previous:ix]) - last_requires_previous = ix - else: - self.__sourcecache.commit(source, []) - - # __last_source_requires_previous - # - # This is the last source that requires previous sources to be cached. - # Sources listed after this will be cached separately. - # - # Returns: - # (int): index of last source that requires previous sources - # - def __last_source_requires_previous(self): - if self.__last_source_requires_previous_ix is None: - last_requires_previous = 0 - for ix, source in enumerate(self.__sources): - if source.BST_REQUIRES_PREVIOUS_SOURCES_STAGE: - last_requires_previous = ix - self.__last_source_requires_previous_ix = last_requires_previous - return self.__last_source_requires_previous_ix - # __update_cache_keys() # # Updates weak and strict cache keys
