This is an automated email from the ASF dual-hosted git repository. juergbi pushed a commit to branch juerg/remote-cache-ci in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 1bd00f1e64bc96aaf67f8edc198b45dbf3c717e7 Author: Jürg Billeter <[email protected]> AuthorDate: Tue Mar 2 11:36:33 2021 +0100 _scheduler: Add CacheQueryQueue to support cache query in job threads This prepares parallelization of cache queries but does not enable it yet. --- src/buildstream/_scheduler/__init__.py | 1 + .../_scheduler/queues/cachequeryqueue.py | 66 ++++++++++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/src/buildstream/_scheduler/__init__.py b/src/buildstream/_scheduler/__init__.py index d2f458f..fcde00d 100644 --- a/src/buildstream/_scheduler/__init__.py +++ b/src/buildstream/_scheduler/__init__.py @@ -25,6 +25,7 @@ from .queues.trackqueue import TrackQueue from .queues.buildqueue import BuildQueue from .queues.artifactpushqueue import ArtifactPushQueue from .queues.pullqueue import PullQueue +from .queues.cachequeryqueue import CacheQueryQueue from .scheduler import Scheduler, SchedStatus from .jobs import ElementJob, JobStatus diff --git a/src/buildstream/_scheduler/queues/cachequeryqueue.py b/src/buildstream/_scheduler/queues/cachequeryqueue.py new file mode 100644 index 0000000..b650a91 --- /dev/null +++ b/src/buildstream/_scheduler/queues/cachequeryqueue.py @@ -0,0 +1,66 @@ +# +# Copyright (C) 2020 Bloomberg Finance LP +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. + +from . import Queue, QueueStatus +from ..resources import ResourceType +from ..jobs import JobStatus +from ...types import _KeyStrength + + +# A queue which queries the cache for artifacts and sources +# +class CacheQueryQueue(Queue): + + action_name = "Cache-query" + complete_name = "Cache queried" + resources = [ResourceType.PROCESS, ResourceType.CACHE] + + def __init__(self, scheduler, *, sources=False): + super().__init__(scheduler) + + self._sources = sources + + def get_process_func(self): + if not self._sources: + return CacheQueryQueue._query_artifacts_or_sources + else: + return CacheQueryQueue._query_sources + + def status(self, element): + if not element._get_cache_key(strength=_KeyStrength.WEAK): + # Strict and weak cache keys are unavailable if the element or + # a dependency has an unresolved source + return QueueStatus.SKIP + + return QueueStatus.READY + + def done(self, _, element, result, status): + if status is JobStatus.FAIL: + return + + if not self._sources: + if not element._pull_pending(): + element._load_artifact_done() + + @staticmethod + def _query_artifacts_or_sources(element): + element._load_artifact(pull=False) + if not element._can_query_cache() or not element._cached_success(): + element._query_source_cache() + + @staticmethod + def _query_sources(element): + element._query_source_cache()
