This is an automated email from the ASF dual-hosted git repository. tvb pushed a commit to branch tristan/optimize-show in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit ca83b8085d25f6053ad65e01b32c64d55e2b807d Author: Tristan van Berkom <[email protected]> AuthorDate: Mon Jun 10 13:10:48 2024 +0800 Optimize bst show when no cached state is needed. This avoids interrogating the local cache and resolving artifact remotes when running `bst show` without `%{state}` in the format. This significantly optimizes the `bst show` scripting interface. * _frontend/cli.py: Determine if %{state} is specified and dont query the cache in that case. * _stream.py: Optionally avoid resolving remotes when loading the pipeline. --- src/buildstream/_frontend/cli.py | 21 ++++++++++++++++----- src/buildstream/_stream.py | 22 ++++++++++++++-------- 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/src/buildstream/_frontend/cli.py b/src/buildstream/_frontend/cli.py index 207e157e0..884a7b77a 100644 --- a/src/buildstream/_frontend/cli.py +++ b/src/buildstream/_frontend/cli.py @@ -12,6 +12,7 @@ # limitations under the License. # import os +import re import sys from functools import partial @@ -627,19 +628,29 @@ def show(app, elements, deps, except_, order, format_): $'---------- %{name} ----------\\n%{vars}' """ with app.initialized(): + + if not format_: + format_ = app.context.log_element_format + + # First determine whether we need to go about querying the local cache + # and spending time setting up remotes. + state_match = re.search(r"%(\{(state)[^%]*?\})", format_) + need_state = bool(state_match is not None) + if not elements: elements = app.project.get_default_targets() - dependencies = app.stream.load_selection(elements, selection=deps, except_targets=except_) + dependencies = app.stream.load_selection( + elements, selection=deps, except_targets=except_, need_state=need_state + ) - app.stream.query_cache(dependencies) + # Don't spend time interrogating the cache if we don't need to show element state + if need_state: + app.stream.query_cache(dependencies) if order == "alpha": dependencies = sorted(dependencies) - if not format_: - format_ = app.context.log_element_format - report = app.logger.show_pipeline(dependencies, format_) click.echo(report) diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py index 159436914..7e4570495 100644 --- a/src/buildstream/_stream.py +++ b/src/buildstream/_stream.py @@ -147,6 +147,7 @@ class Stream: # source_remotes: Source cache remotes specified on the commmand line # ignore_project_artifact_remotes: Whether to ignore artifact remotes specified by projects # ignore_project_source_remotes: Whether to ignore source remotes specified by projects + # need_state: Whether resolving element state is required # # Returns: # (list of Element): The selected elements @@ -163,6 +164,7 @@ class Stream: source_remotes: Iterable[RemoteSpec] = (), ignore_project_artifact_remotes: bool = False, ignore_project_source_remotes: bool = False, + need_state: bool = True, ): with PROFILER.profile(Topics.LOAD_SELECTION, "_".join(t.replace(os.sep, "-") for t in targets)): target_objects = self._load( @@ -176,6 +178,7 @@ class Stream: source_remotes=source_remotes, ignore_project_artifact_remotes=ignore_project_artifact_remotes, ignore_project_source_remotes=ignore_project_source_remotes, + need_state=need_state, ) return target_objects @@ -1617,6 +1620,7 @@ class Stream: # source_remotes: Source cache remotes specified on the commmand line # ignore_project_artifact_remotes: Whether to ignore artifact remotes specified by projects # ignore_project_source_remotes: Whether to ignore source remotes specified by projects + # need_state: Whether resolving element state is required # # Returns: # (list of Element): The primary element selection @@ -1637,6 +1641,7 @@ class Stream: source_remotes: Iterable[RemoteSpec] = (), ignore_project_artifact_remotes: bool = False, ignore_project_source_remotes: bool = False, + need_state: bool = True, ): elements, except_elements, artifacts = self._load_elements_from_targets( targets, except_targets, rewritable=False, valid_artifact_names=load_artifacts @@ -1656,14 +1661,15 @@ class Stream: self.targets = elements # Connect to remote caches, this needs to be done before resolving element state - self._context.initialize_remotes( - connect_artifact_cache, - connect_source_cache, - artifact_remotes, - source_remotes, - ignore_project_artifact_remotes=ignore_project_artifact_remotes, - ignore_project_source_remotes=ignore_project_source_remotes, - ) + if need_state: + self._context.initialize_remotes( + connect_artifact_cache, + connect_source_cache, + artifact_remotes, + source_remotes, + ignore_project_artifact_remotes=ignore_project_artifact_remotes, + ignore_project_source_remotes=ignore_project_source_remotes, + ) # In some cases we need to have an actualized artifact, with all of # it's metadata, such that we can derive attributes about the artifact
