This is an automated email from the ASF dual-hosted git repository. tvb pushed a commit to branch jonathan/progress in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 51f612acb5bb4c7e42f07c0569695d6dd3661c1f Author: Jonathan Maw <[email protected]> AuthorDate: Thu May 23 18:06:37 2019 +0100 Add Progress reporting for element loading This is very primitive and does not have a very good way to find the total number of elements while loading. --- src/buildstream/_context.py | 13 ++++++++++ src/buildstream/_loader/loader.py | 16 +++++++++++- src/buildstream/_progress.py | 53 +++++++++++++++++++++++++++++++++++++++ src/buildstream/_project.py | 4 +-- 4 files changed, 83 insertions(+), 3 deletions(-) diff --git a/src/buildstream/_context.py b/src/buildstream/_context.py index 151ea63..f01df7a 100644 --- a/src/buildstream/_context.py +++ b/src/buildstream/_context.py @@ -34,6 +34,7 @@ from ._profile import Topics, PROFILER from ._artifactcache import ArtifactCache from ._sourcecache import SourceCache from ._cas import CASCache, CASQuota, CASCacheUsage +from ._progress import Progress from ._workspaces import Workspaces, WorkspaceProjectCache from .plugin import Plugin from .sandbox import SandboxRemote @@ -554,6 +555,18 @@ class Context(): self._pop_message_depth() self.message(message) + @contextmanager + def progress_activity(self, activity_name, *, total=None, unique_id=None, + detail=None, silent_nested=False): + with self.timed_activity(activity_name, unique_id=unique_id, + detail=detail, silent_nested=silent_nested): + progress = Progress(self, activity_name, total=total, unique_id=unique_id) + yield progress + + def report_progress(self, message_text, unique_id=None): + message = Message(unique_id, MessageType.INFO, message_text) + self.message(message) + # recorded_messages() # # Records all messages in a log file while the context manager diff --git a/src/buildstream/_loader/loader.py b/src/buildstream/_loader/loader.py index 5bd0487..d68c4b3 100644 --- a/src/buildstream/_loader/loader.py +++ b/src/buildstream/_loader/loader.py @@ -74,6 +74,10 @@ class Loader(): self._elements = {} # Dict of elements self._loaders = {} # Dict of junction loaders + # Progress object stored in the loader during element loading because + # passing it through function args would be a huge mess + self._progress = None + self._includes = Includes(self, copy_tree=True) # load(): @@ -86,11 +90,14 @@ class Loader(): # ticker (callable): An optional function for tracking load progress # targets (list of str): Target, element-path relative bst filenames in the project # fetch_subprojects (bool): Whether to fetch subprojects while loading + # progress (Progress): The object to report progress to # # Raises: LoadError # # Returns: The toplevel LoadElement - def load(self, targets, rewritable=False, ticker=None, fetch_subprojects=False): + def load(self, targets, progress, rewritable=False, ticker=None, fetch_subprojects=False): + + self._progress = progress for filename in targets: if os.path.isabs(filename): @@ -143,6 +150,7 @@ class Loader(): ret.append(loader._collect_element(element)) self._clean_caches() + self._progress = None return ret @@ -405,6 +413,9 @@ class Loader(): if meta_element: return meta_element + if self._progress: + self._progress.add_total(1) + node = element.node elt_provenance = _yaml.node_get_provenance(node) meta_sources = [] @@ -450,6 +461,9 @@ class Loader(): if dep.dep_type != 'build': meta_element.dependencies.append(meta_dep) + if self._progress: + self._progress.add_progress(1) + return meta_element # _get_loader(): diff --git a/src/buildstream/_progress.py b/src/buildstream/_progress.py new file mode 100644 index 0000000..2ad9e6a --- /dev/null +++ b/src/buildstream/_progress.py @@ -0,0 +1,53 @@ +# Copyright (C) 2019 Bloomberg Finance LP +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Jonathan Maw <[email protected]> + +import time + +class Progress(): + def __init__(self, context, activity_name, *, total=None, unique_id=None): + self._context = context + self._activity_name = activity_name + self._last_reported = time.monotonic() + self._interval = 3.0 # seconds + self._count = 0 + self._total = total + self._unique_id = unique_id + + def add_total(self, count): + if self._total is None: + self._total = count + else: + self._total += count + self._check_report_progress() + + def add_progress(self, count): + self._count += count + self._check_report_progress() + + def _check_report_progress(self): + # It would be more efficient to have a time-based poll rather than + # a regular check whether enough time has passed + now = time.monotonic() + if now >= self._last_reported + self._interval: + self._last_reported = now + message_text = self._activity_name + ": " + str(self._count) + if self._total is not None: + message_text += "/" + str(self._total) + self._context.report_progress(message_text, self._unique_id) + + diff --git a/src/buildstream/_project.py b/src/buildstream/_project.py index c40321c..3d5714f 100644 --- a/src/buildstream/_project.py +++ b/src/buildstream/_project.py @@ -449,8 +449,8 @@ class Project(): # def load_elements(self, targets, *, rewritable=False, fetch_subprojects=False): - with self._context.timed_activity("Loading elements", silent_nested=True): - meta_elements = self.loader.load(targets, rewritable=rewritable, + with self._context.progress_activity("Loading elements", silent_nested=True) as progress: + meta_elements = self.loader.load(targets, progress, rewritable=rewritable, ticker=None, fetch_subprojects=fetch_subprojects)
