This is an automated email from the ASF dual-hosted git repository. striker pushed a commit to branch striker/speculative-actions in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 4ae530c22884f51c179ea4203835a70204bef5ac Author: Sander Striker <[email protected]> AuthorDate: Sat Mar 21 21:48:35 2026 +0100 speculative-actions: Add tiered mode system Replace the boolean speculative-actions config with tiered modes that let users control the cost/benefit trade-off: - none: disabled entirely - prime-only: use existing Speculative Actions to prime, don't generate new ones - source-artifact: generate SOURCE and ARTIFACT overlays (no AC calls) - intra-element: also generate intra-element ACTION overlays - full: also generate cross-element ACTION overlays Each mode includes all capabilities of the previous modes. Boolean values (True/False) are accepted for backward compatibility. The mode gates which queues are enabled (priming for all except none, generation for source-artifact and above) and which overlay types the generator produces (ACTION overlay logic skipped in source-artifact mode, cross-element seeding skipped in intra-element mode). Co-Authored-By: Claude Opus 4.6 (1M context) <[email protected]> --- src/buildstream/_context.py | 16 +++-- .../queues/speculativeactiongenerationqueue.py | 12 +++- src/buildstream/_speculative_actions/generator.py | 78 ++++++++++++---------- src/buildstream/_stream.py | 8 ++- src/buildstream/data/userconfig.yaml | 14 ++-- src/buildstream/element.py | 4 +- src/buildstream/types.py | 24 +++++++ 7 files changed, 106 insertions(+), 50 deletions(-) diff --git a/src/buildstream/_context.py b/src/buildstream/_context.py index 9cf89854f..3ea819358 100644 --- a/src/buildstream/_context.py +++ b/src/buildstream/_context.py @@ -31,7 +31,7 @@ from ._elementsourcescache import ElementSourcesCache from ._remotespec import RemoteSpec, RemoteExecutionSpec from ._sourcecache import SourceCache from ._cas import CASCache, CASDProcessManager, CASLogLevel -from .types import _CacheBuildTrees, _PipelineSelection, _SchedulerErrorAction, _SourceUriPolicy +from .types import _CacheBuildTrees, _PipelineSelection, _SchedulerErrorAction, _SourceUriPolicy, _SpeculativeActionMode from ._workspaces import Workspaces, WorkspaceProjectCache from .node import Node, MappingNode @@ -164,8 +164,8 @@ class Context: # What to do when a build fails in non interactive mode self.sched_error_action: Optional[str] = None - # Whether speculative actions are enabled - self.speculative_actions: bool = False + # Speculative actions mode + self.speculative_actions_mode: _SpeculativeActionMode = _SpeculativeActionMode.NONE # Maximum jobs per build self.build_max_jobs: Optional[int] = None @@ -462,7 +462,15 @@ class Context: self.sched_network_retries = scheduler.get_int("network-retries") # Load speculative actions config - self.speculative_actions = scheduler.get_bool("speculative-actions") + # Accepts mode string (none/prime-only/source-artifact/intra-element/full) + # or boolean for backward compatibility (True → full, False → none) + try: + self.speculative_actions_mode = scheduler.get_enum("speculative-actions", _SpeculativeActionMode) + except Exception: + self.speculative_actions_mode = ( + _SpeculativeActionMode.FULL if scheduler.get_bool("speculative-actions") + else _SpeculativeActionMode.NONE + ) # Load build config build = defaults.get_mapping("build") diff --git a/src/buildstream/_scheduler/queues/speculativeactiongenerationqueue.py b/src/buildstream/_scheduler/queues/speculativeactiongenerationqueue.py index 225cc6d93..3c0fc57a6 100644 --- a/src/buildstream/_scheduler/queues/speculativeactiongenerationqueue.py +++ b/src/buildstream/_scheduler/queues/speculativeactiongenerationqueue.py @@ -91,12 +91,20 @@ class SpeculativeActionGenerationQueue(Queue): dependencies = list(element._dependencies(_Scope.BUILD, recurse=False)) # Get action cache service for ACTION overlay generation + # (only needed for intra-element and full modes) + from ...types import _SpeculativeActionMode + + mode = context.speculative_actions_mode casd = context.get_casd() - ac_service = casd.get_ac_service() if casd else None + ac_service = None + if mode in (_SpeculativeActionMode.INTRA_ELEMENT, _SpeculativeActionMode.FULL): + ac_service = casd.get_ac_service() if casd else None # Generate overlays generator = SpeculativeActionsGenerator(cas, ac_service=ac_service, artifactcache=artifactcache) - spec_actions = generator.generate_speculative_actions(element, subaction_digests, dependencies) + spec_actions = generator.generate_speculative_actions( + element, subaction_digests, dependencies, mode=mode + ) if not spec_actions or not spec_actions.actions: return 0 diff --git a/src/buildstream/_speculative_actions/generator.py b/src/buildstream/_speculative_actions/generator.py index 427e1cbde..9c6be7308 100644 --- a/src/buildstream/_speculative_actions/generator.py +++ b/src/buildstream/_speculative_actions/generator.py @@ -59,7 +59,7 @@ class SpeculativeActionsGenerator: # SOURCE overlays are tried first, then ACTION, then ARTIFACT. self._digest_cache: Dict[str, list] = {} - def generate_speculative_actions(self, element, subaction_digests, dependencies): + def generate_speculative_actions(self, element, subaction_digests, dependencies, mode=None): """ Generate SpeculativeActions for an element build. @@ -71,6 +71,8 @@ class SpeculativeActionsGenerator: element: The element that was built subaction_digests: List of Action digests from the build (from ActionResult.subactions) dependencies: List of dependency elements (for resolving artifact overlays) + mode: _SpeculativeActionMode controlling which overlay types to generate. + None defaults to FULL for backward compatibility. Returns: A SpeculativeActions message containing: @@ -79,6 +81,10 @@ class SpeculativeActionsGenerator: """ from .._protos.buildstream.v2 import speculative_actions_pb2 from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2 + from ..types import _SpeculativeActionMode + + if mode is None: + mode = _SpeculativeActionMode.FULL spec_actions = speculative_actions_pb2.SpeculativeActions() @@ -90,43 +96,44 @@ class SpeculativeActionsGenerator: prior_outputs = {} # Seed prior_outputs with dependency subaction outputs for - # cross-element ACTION overlays. Dependencies have already been - # built and had their generation queue run, so their SAs and - # ActionResults are available. - if self._ac_service and self._artifactcache: - self._seed_dependency_outputs(dependencies, prior_outputs) + # cross-element ACTION overlays (full mode only). + if mode == _SpeculativeActionMode.FULL: + if self._ac_service and self._artifactcache: + self._seed_dependency_outputs(dependencies, prior_outputs) # Generate overlays for each subaction for subaction_digest in subaction_digests: spec_action = self._generate_action_overlays(element, subaction_digest) # Generate ACTION overlays for digests that match prior subaction outputs - # but weren't already resolved as SOURCE or ARTIFACT - if self._ac_service and prior_outputs: - action = self._cas.fetch_action(subaction_digest) - if action: - input_digests = self._extract_digests_from_action(action) - # Collect hashes already covered by SOURCE/ARTIFACT overlays - already_overlaid = set() - if spec_action: - for overlay in spec_action.overlays: - already_overlaid.add(overlay.target_digest.hash) - - for digest_hash, digest_size in input_digests: - if digest_hash in prior_outputs and digest_hash not in already_overlaid: - source_element, producing_action_digest, output_path = prior_outputs[digest_hash] - # Create ACTION overlay - if spec_action is None: - spec_action = speculative_actions_pb2.SpeculativeActions.SpeculativeAction() - spec_action.base_action_digest.CopyFrom(subaction_digest) - overlay = speculative_actions_pb2.SpeculativeActions.Overlay() - overlay.type = speculative_actions_pb2.SpeculativeActions.Overlay.ACTION - overlay.source_element = source_element - overlay.source_action_digest.CopyFrom(producing_action_digest) - overlay.source_path = output_path - overlay.target_digest.hash = digest_hash - overlay.target_digest.size_bytes = digest_size - spec_action.overlays.append(overlay) + # but weren't already resolved as SOURCE or ARTIFACT. + # Requires intra-element or full mode. + if mode in (_SpeculativeActionMode.INTRA_ELEMENT, _SpeculativeActionMode.FULL): + if self._ac_service and prior_outputs: + action = self._cas.fetch_action(subaction_digest) + if action: + input_digests = self._extract_digests_from_action(action) + # Collect hashes already covered by SOURCE/ARTIFACT overlays + already_overlaid = set() + if spec_action: + for overlay in spec_action.overlays: + already_overlaid.add(overlay.target_digest.hash) + + for digest_hash, digest_size in input_digests: + if digest_hash in prior_outputs and digest_hash not in already_overlaid: + source_element, producing_action_digest, output_path = prior_outputs[digest_hash] + # Create ACTION overlay + if spec_action is None: + spec_action = speculative_actions_pb2.SpeculativeActions.SpeculativeAction() + spec_action.base_action_digest.CopyFrom(subaction_digest) + overlay = speculative_actions_pb2.SpeculativeActions.Overlay() + overlay.type = speculative_actions_pb2.SpeculativeActions.Overlay.ACTION + overlay.source_element = source_element + overlay.source_action_digest.CopyFrom(producing_action_digest) + overlay.source_path = output_path + overlay.target_digest.hash = digest_hash + overlay.target_digest.size_bytes = digest_size + spec_action.overlays.append(overlay) # Sort overlays: SOURCE > ACTION > ARTIFACT # This ensures the instantiator tries SOURCE first, then @@ -141,9 +148,10 @@ class SpeculativeActionsGenerator: spec_actions.actions.append(spec_action) # Fetch this subaction's ActionResult and record its outputs - # for subsequent subactions - if self._ac_service: - self._record_subaction_outputs(subaction_digest, prior_outputs) + # for subsequent subactions (intra-element and full modes) + if mode in (_SpeculativeActionMode.INTRA_ELEMENT, _SpeculativeActionMode.FULL): + if self._ac_service: + self._record_subaction_outputs(subaction_digest, prior_outputs) # Generate artifact overlays for the element's output files artifact_overlays = self._generate_artifact_overlays(element) diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py index 36d67b5ed..cba21ed84 100644 --- a/src/buildstream/_stream.py +++ b/src/buildstream/_stream.py @@ -49,7 +49,7 @@ from ._profile import Topics, PROFILER from ._project import ProjectRefStorage from ._remotespec import RemoteSpec from ._state import State -from .types import _KeyStrength, _PipelineSelection, _Scope, _HostMount +from .types import _KeyStrength, _PipelineSelection, _Scope, _HostMount, _SpeculativeActionMode from .plugin import Plugin from . import utils, node, _yaml, _site, _pipeline @@ -431,7 +431,9 @@ class Stream: self._add_queue(FetchQueue(self._scheduler, skip_cached=True)) - if self._context.speculative_actions: + sa_mode = self._context.speculative_actions_mode + + if sa_mode != _SpeculativeActionMode.NONE: # Priming queue: For each element, instantiate and submit its speculative # actions to warm the remote ActionCache BEFORE the element reaches BuildQueue. # Must come after FetchQueue so sources are available for resolving SOURCE overlays. @@ -439,7 +441,7 @@ class Stream: self._add_queue(BuildQueue(self._scheduler, imperative=True)) - if self._context.speculative_actions: + if sa_mode not in (_SpeculativeActionMode.NONE, _SpeculativeActionMode.PRIME_ONLY): # Generation queue: After each build, extract subactions and generate # overlays so future builds can benefit from cache priming. self._add_queue(SpeculativeActionGenerationQueue(self._scheduler)) diff --git a/src/buildstream/data/userconfig.yaml b/src/buildstream/data/userconfig.yaml index b510fcd71..3155fe80a 100644 --- a/src/buildstream/data/userconfig.yaml +++ b/src/buildstream/data/userconfig.yaml @@ -74,11 +74,17 @@ scheduler: # on-error: quit - # Enable speculative actions for cache priming. - # When enabled, subactions from builds are recorded and used to - # speculatively prime the remote ActionCache in future builds. + # Speculative actions mode for cache priming. + # Controls which overlay types are generated and whether priming is enabled. + # Modes (each includes capabilities of previous modes): + # none - Disabled entirely + # prime-only - Use existing SAs to prime, don't generate new ones + # source-artifact - Generate SOURCE and ARTIFACT overlays (no AC calls) + # intra-element - Also generate intra-element ACTION overlays + # full - Also generate cross-element ACTION overlays + # Also accepts True (= full) or False (= none) for backward compatibility. # - speculative-actions: False + speculative-actions: none # diff --git a/src/buildstream/element.py b/src/buildstream/element.py index 972709121..b4dcd1434 100644 --- a/src/buildstream/element.py +++ b/src/buildstream/element.py @@ -90,7 +90,7 @@ from .plugin import Plugin from .sandbox import _SandboxFlags, SandboxCommandError from .sandbox._config import SandboxConfig from .sandbox._sandboxremote import SandboxRemote -from .types import _Scope, _CacheBuildTrees, _KeyStrength, OverlapAction, _DisplayKey +from .types import _Scope, _CacheBuildTrees, _KeyStrength, OverlapAction, _DisplayKey, _SpeculativeActionMode from ._artifact import Artifact from ._elementproxy import ElementProxy from ._elementsources import ElementSources @@ -1973,7 +1973,7 @@ class Element(Plugin): if ( pull and not artifact.cached() - and context.speculative_actions + and context.speculative_actions_mode != _SpeculativeActionMode.NONE and self.__weak_cache_key and not self.__artifacts.contains(self, self.__weak_cache_key) ): diff --git a/src/buildstream/types.py b/src/buildstream/types.py index 0cc2106b3..6ee4ec921 100644 --- a/src/buildstream/types.py +++ b/src/buildstream/types.py @@ -241,6 +241,30 @@ class _SchedulerErrorAction(FastEnum): TERMINATE = "terminate" +# _SpeculativeActionMode() +# +# Graduated modes for speculative actions, controlling which overlay +# types are generated and whether priming is enabled. Each mode +# includes all capabilities of the previous modes. +# +class _SpeculativeActionMode(FastEnum): + + # Speculative actions disabled entirely + NONE = "none" + + # Use existing SAs to prime the cache, but don't generate new ones + PRIME_ONLY = "prime-only" + + # Generate SOURCE and ARTIFACT overlays only (no AC calls during generation) + SOURCE_ARTIFACT = "source-artifact" + + # Also generate intra-element ACTION overlays (AC calls for own subactions) + INTRA_ELEMENT = "intra-element" + + # Full mode: also generate cross-element ACTION overlays + FULL = "full" + + # _CacheBuildTrees() # # When to cache build trees
