This is an automated email from the ASF dual-hosted git repository. juergbi pushed a commit to branch juerg/cache-query in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 2787576dac8216440ca6d78cbfe848eb28c96b89 Author: Jürg Billeter <[email protected]> AuthorDate: Thu Oct 8 17:09:00 2020 +0200 Call _initialize_state() in Element._new_from_load_element() With the cache queries moved out, `_initialize_state()` is fairly lightweight and can be called earlier. --- src/buildstream/_loader/loadelement.pyx | 1 - src/buildstream/_loader/loader.py | 1 - src/buildstream/_stream.py | 35 --------------------------------- src/buildstream/element.py | 3 +++ tests/artifactcache/push.py | 7 ------- tests/sourcecache/fetch.py | 6 ------ tests/sourcecache/push.py | 2 -- tests/sourcecache/staging.py | 3 --- 8 files changed, 3 insertions(+), 55 deletions(-) diff --git a/src/buildstream/_loader/loadelement.pyx b/src/buildstream/_loader/loadelement.pyx index 210869e..f69e138 100644 --- a/src/buildstream/_loader/loadelement.pyx +++ b/src/buildstream/_loader/loadelement.pyx @@ -286,7 +286,6 @@ cdef class LoadElement: from ..element import Element element = Element._new_from_load_element(self) - element._initialize_state() # Custom error for link dependencies, since we don't completely # parse their dependencies we cannot rely on the built-in ElementError. diff --git a/src/buildstream/_loader/loader.py b/src/buildstream/_loader/loader.py index 6db2017..df4df84 100644 --- a/src/buildstream/_loader/loader.py +++ b/src/buildstream/_loader/loader.py @@ -816,7 +816,6 @@ class Loader: ) element = Element._new_from_load_element(load_element) - element._initialize_state() # Handle the case where a subproject has no ref # diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py index 87bfeed..cef605e 100644 --- a/src/buildstream/_stream.py +++ b/src/buildstream/_stream.py @@ -1628,7 +1628,6 @@ class Stream: # Now move on to loading primary selection. # - self._resolve_elements(self.targets) selected = _pipeline.get_selection(self._context, self.targets, selection, silent=False) selected = _pipeline.except_elements(self.targets, selected, except_elements) @@ -1644,40 +1643,6 @@ class Stream: return selected - # _resolve_elements() - # - # Resolve element state and cache keys. - # - # Args: - # targets (list of Element): The list of toplevel element targets - # - def _resolve_elements(self, targets): - with self._context.messenger.simple_task("Resolving cached state", silent_nested=True) as task: - # We need to go through the project to access the loader - # - # FIXME: We need to calculate the total elements to resolve differently so that - # it can include artifact elements - # - if task and self._project: - task.set_maximum_progress(self._project.loader.loaded) - - # XXX: Now that Element._update_state() can trigger recursive update_state calls - # it is possible that we could get a RecursionError. However, this is unlikely - # to happen, even for large projects (tested with the Debian stack). Although, - # if it does become a problem we may have to set the recursion limit to a - # greater value. - for element in _pipeline.dependencies(targets, _Scope.ALL): - # Determine initial element state. - element._initialize_state() - - # We may already have Elements which are cached and have their runtimes - # cached, if this is the case, we should immediately notify their reverse - # dependencies. - element._update_ready_for_runtime_and_cached() - - if task: - task.add_current_progress() - # _reset() # # Resets the internal state related to a given scheduler run. diff --git a/src/buildstream/element.py b/src/buildstream/element.py index 943852f..3695a42 100644 --- a/src/buildstream/element.py +++ b/src/buildstream/element.py @@ -1135,6 +1135,8 @@ class Element(Plugin): element.__preflight() + element._initialize_state() + if task: task.add_current_progress() @@ -2884,6 +2886,7 @@ class Element(Plugin): self._mimic_artifact() if not artifact.cached(): self.__pull_pending = True + self._initialize_state() @classmethod def __compose_default_splits(cls, project, defaults, first_pass): diff --git a/tests/artifactcache/push.py b/tests/artifactcache/push.py index 6b31542..24a1ee1 100644 --- a/tests/artifactcache/push.py +++ b/tests/artifactcache/push.py @@ -6,7 +6,6 @@ import os import pytest from buildstream import _yaml -from buildstream.types import _Scope from buildstream._project import Project from buildstream._protos.build.bazel.remote.execution.v2 import remote_execution_pb2 from buildstream.testing import cli # pylint: disable=unused-import @@ -33,12 +32,6 @@ def _push(cli, cache_dir, project_dir, config_file, target): # Create a local artifact cache handle artifactcache = context.artifactcache - # Ensure the element's artifact memeber is initialised - # This is duplicated from Pipeline.resolve_elements() - # as this test does not use the cli frontend. - for e in element._dependencies(_Scope.ALL): - e._initialize_state() - # Initialize remotes context.initialize_remotes(True, True, None, None) diff --git a/tests/sourcecache/fetch.py b/tests/sourcecache/fetch.py index 46f39b0..f37242f 100644 --- a/tests/sourcecache/fetch.py +++ b/tests/sourcecache/fetch.py @@ -74,7 +74,6 @@ def test_source_fetch(cli, tmpdir, datafiles): project.ensure_fully_loaded() element = project.load_elements([element_name])[0] - element._initialize_state() element._query_source_cache() assert not element._cached_sources() source = list(element.sources())[0] @@ -115,7 +114,6 @@ def test_source_fetch(cli, tmpdir, datafiles): project.ensure_fully_loaded() element = project.load_elements([element_name])[0] - element._initialize_state() # check that we have the source in the cas now and it's not fetched element._query_source_cache() @@ -136,7 +134,6 @@ def test_fetch_fallback(cli, tmpdir, datafiles): project.ensure_fully_loaded() element = project.load_elements([element_name])[0] - element._initialize_state() element._query_source_cache() assert not element._cached_sources() source = list(element.sources())[0] @@ -155,7 +152,6 @@ def test_fetch_fallback(cli, tmpdir, datafiles): # Check that the source in both in the source dir and the local CAS element = project.load_elements([element_name])[0] - element._initialize_state() element._query_source_cache() assert element._cached_sources() @@ -172,7 +168,6 @@ def test_pull_fail(cli, tmpdir, datafiles): project.ensure_fully_loaded() element = project.load_elements([element_name])[0] - element._initialize_state() element._query_source_cache() assert not element._cached_sources() source = list(element.sources())[0] @@ -205,7 +200,6 @@ def test_source_pull_partial_fallback_fetch(cli, tmpdir, datafiles): project.ensure_fully_loaded() element = project.load_elements([element_name])[0] - element._initialize_state() element._query_source_cache() assert not element._cached_sources() source = list(element.sources())[0] diff --git a/tests/sourcecache/push.py b/tests/sourcecache/push.py index 76333ca..47e845b 100644 --- a/tests/sourcecache/push.py +++ b/tests/sourcecache/push.py @@ -86,7 +86,6 @@ def test_source_push_split(cli, tmpdir, datafiles): project.ensure_fully_loaded() element = project.load_elements(["push.bst"])[0] - element._initialize_state() element._query_source_cache() assert not element._cached_sources() source = list(element.sources())[0] @@ -137,7 +136,6 @@ def test_source_push(cli, tmpdir, datafiles): project.ensure_fully_loaded() element = project.load_elements(["push.bst"])[0] - element._initialize_state() element._query_source_cache() assert not element._cached_sources() source = list(element.sources())[0] diff --git a/tests/sourcecache/staging.py b/tests/sourcecache/staging.py index e0e7002..6671c79 100644 --- a/tests/sourcecache/staging.py +++ b/tests/sourcecache/staging.py @@ -64,7 +64,6 @@ def test_source_staged(tmpdir, cli, datafiles): # now check that the source is in the refs file, this is pretty messy but # seems to be the only way to get the sources? element = project.load_elements(["import-bin.bst"])[0] - element._initialize_state() element._query_source_cache() source = list(element.sources())[0] assert element._cached_sources() @@ -100,7 +99,6 @@ def test_source_fetch(tmpdir, cli, datafiles): sourcecache = context.sourcecache element = project.load_elements(["import-dev.bst"])[0] - element._initialize_state() element._query_source_cache() source = list(element.sources())[0] assert element._cached_sources() @@ -135,7 +133,6 @@ def test_staged_source_build(tmpdir, datafiles, cli): project.ensure_fully_loaded() element = project.load_elements(["import-dev.bst"])[0] - element._initialize_state() # check consistency of the source element._query_source_cache()
