Hello community, here is the log from the commit of package buildstream for openSUSE:Factory checked in at 2019-03-28 22:49:25 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/buildstream (Old) and /work/SRC/openSUSE:Factory/.buildstream.new.25356 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "buildstream" Thu Mar 28 22:49:25 2019 rev:10 rq:689297 version:1.2.5 Changes: -------- --- /work/SRC/openSUSE:Factory/buildstream/buildstream.changes 2019-02-28 21:39:36.613617411 +0100 +++ /work/SRC/openSUSE:Factory/.buildstream.new.25356/buildstream.changes 2019-03-28 22:49:32.811045391 +0100 @@ -1,0 +2,12 @@ +Mon Mar 25 19:29:19 UTC 2019 - Bjørn Lie <[email protected]> + +- Update to version 1.2.5: + + Fixed failure to process some elements when workspaces are + open. + + Better error reporting when files are missing, or when + encountering errors in sub projects. + + Do not require exact versions of dependencies for running + tests. + + Fail on overlap policy no longer inherited from subprojects. + +------------------------------------------------------------------- Old: ---- BuildStream-1.2.4.tar.xz New: ---- BuildStream-1.2.5.tar.xz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ buildstream.spec ++++++ --- /var/tmp/diff_new_pack.YHxPGq/_old 2019-03-28 22:49:33.531045261 +0100 +++ /var/tmp/diff_new_pack.YHxPGq/_new 2019-03-28 22:49:33.535045260 +0100 @@ -18,13 +18,14 @@ %define real_name BuildStream Name: buildstream -Version: 1.2.4 +Version: 1.2.5 Release: 0 Summary: A framework for modelling build pipelines in YAML License: LGPL-2.1-or-later Group: Development/Tools/Building URL: https://wiki.gnome.org/Projects/BuildStream -Source: https://download.gnome.org/sources/BuildStream/1.2/%{real_name}-%{version}.tar.xz +Source0: https://download.gnome.org/sources/BuildStream/1.2/%{real_name}-%{version}.tar.xz + BuildRequires: bubblewrap BuildRequires: fdupes BuildRequires: python3-base >= 3.4 ++++++ BuildStream-1.2.4.tar.xz -> BuildStream-1.2.5.tar.xz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/BuildStream.egg-info/PKG-INFO new/BuildStream-1.2.5/BuildStream.egg-info/PKG-INFO --- old/BuildStream-1.2.4/BuildStream.egg-info/PKG-INFO 2019-02-14 14:09:13.000000000 +0100 +++ new/BuildStream-1.2.5/BuildStream.egg-info/PKG-INFO 2019-03-25 12:33:21.000000000 +0100 @@ -1,6 +1,6 @@ Metadata-Version: 1.2 Name: BuildStream -Version: 1.2.4 +Version: 1.2.5 Summary: A framework for modelling build pipelines in YAML Home-page: https://gitlab.com/BuildStream/buildstream Author: BuildStream Developers diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/BuildStream.egg-info/SOURCES.txt new/BuildStream-1.2.5/BuildStream.egg-info/SOURCES.txt --- old/BuildStream-1.2.4/BuildStream.egg-info/SOURCES.txt 2019-02-14 14:09:13.000000000 +0100 +++ new/BuildStream-1.2.5/BuildStream.egg-info/SOURCES.txt 2019-03-25 12:33:21.000000000 +0100 @@ -43,6 +43,7 @@ buildstream/plugin.py buildstream/scriptelement.py buildstream/source.py +buildstream/types.py buildstream/utils.py buildstream/_artifactcache/__init__.py buildstream/_artifactcache/artifactcache.py @@ -445,6 +446,10 @@ tests/format/project/emptyname/project.conf tests/format/project/invalidname/project.conf tests/format/project/local-plugin/project.conf +tests/format/project/missing-element/manual.bst +tests/format/project/missing-element/project.conf +tests/format/project/missing-junction/manual.bst +tests/format/project/missing-junction/project.conf tests/format/project/missingname/project.conf tests/format/project/overridepath/manual.bst tests/format/project/overridepath/project.conf @@ -517,6 +522,9 @@ tests/frontend/overlaps/collect-whitelisted.bst tests/frontend/overlaps/collect.bst tests/frontend/overlaps/script.bst +tests/frontend/overlaps/sub-collect.bst +tests/frontend/overlaps/sub-project/a-sub.bst +tests/frontend/overlaps/sub-project/z-sub.bst tests/frontend/project/project.conf tests/frontend/project/elements/checkout-deps.bst tests/frontend/project/elements/compose-all.bst @@ -552,6 +560,20 @@ tests/frontend/track-optional-inline/target.bst tests/frontend/track-optional-project-refs/project.conf tests/frontend/track-optional-project-refs/target.bst +tests/frontend/workspaced-build-dep/project.conf +tests/frontend/workspaced-build-dep/elements/elem1.bst +tests/frontend/workspaced-build-dep/elements/elem2.bst +tests/frontend/workspaced-build-dep/elements/elem3.bst +tests/frontend/workspaced-build-dep/elements/elem4.bst +tests/frontend/workspaced-build-dep/elements/elem5.bst +tests/frontend/workspaced-build-dep/elements/stack.bst +tests/frontend/workspaced-runtime-dep/project.conf +tests/frontend/workspaced-runtime-dep/elements/elem1.bst +tests/frontend/workspaced-runtime-dep/elements/elem2.bst +tests/frontend/workspaced-runtime-dep/elements/elem3.bst +tests/frontend/workspaced-runtime-dep/elements/elem4.bst +tests/frontend/workspaced-runtime-dep/elements/elem5.bst +tests/frontend/workspaced-runtime-dep/elements/stack.bst tests/integration/autotools.py tests/integration/build-uid.py tests/integration/cmake.py @@ -652,6 +674,16 @@ tests/loader/junctions/invalid/junction-with-deps.bst tests/loader/junctions/invalid/missing.bst tests/loader/junctions/invalid/project.conf +tests/loader/junctions/missing-element/junction-A.bst +tests/loader/junctions/missing-element/project.conf +tests/loader/junctions/missing-element/sub-target-bad-junction.bst +tests/loader/junctions/missing-element/sub-target.bst +tests/loader/junctions/missing-element/target.bst +tests/loader/junctions/missing-element/junctionA/bad-junction-target.bst +tests/loader/junctions/missing-element/junctionA/junction-B.bst +tests/loader/junctions/missing-element/junctionA/project.conf +tests/loader/junctions/missing-element/junctionA/target.bst +tests/loader/junctions/missing-element/junctionA/junctionB/project.conf tests/loader/junctions/nested/foo.bst tests/loader/junctions/nested/project.conf tests/loader/junctions/nested/target.bst diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/NEWS new/BuildStream-1.2.5/NEWS --- old/BuildStream-1.2.4/NEWS 2019-02-14 13:55:14.000000000 +0100 +++ new/BuildStream-1.2.5/NEWS 2019-03-25 12:27:59.000000000 +0100 @@ -1,4 +1,17 @@ ================= +buildstream 1.2.5 +================= + + o Fixed failure to process some elements when workspaces are open (#919) + + o Better error reporting when files are missing, or when encountering + errors in sub projects (#947) + + o Do not require exact versions of dependencies for running tests (#916) + + o Fail on overlap policy no longer inherited from subprojects (#926) + +================= buildstream 1.2.4 ================= diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/PKG-INFO new/BuildStream-1.2.5/PKG-INFO --- old/BuildStream-1.2.4/PKG-INFO 2019-02-14 14:09:13.000000000 +0100 +++ new/BuildStream-1.2.5/PKG-INFO 2019-03-25 12:33:22.000000000 +0100 @@ -1,6 +1,6 @@ Metadata-Version: 1.2 Name: BuildStream -Version: 1.2.4 +Version: 1.2.5 Summary: A framework for modelling build pipelines in YAML Home-page: https://gitlab.com/BuildStream/buildstream Author: BuildStream Developers diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/buildstream/_context.py new/BuildStream-1.2.5/buildstream/_context.py --- old/BuildStream-1.2.4/buildstream/_context.py 2019-02-14 13:55:14.000000000 +0100 +++ new/BuildStream-1.2.5/buildstream/_context.py 2019-03-25 12:27:37.000000000 +0100 @@ -31,7 +31,7 @@ from ._profile import Topics, profile_start, profile_end from ._artifactcache import ArtifactCache from ._workspaces import Workspaces -from .plugin import _plugin_lookup +from .plugin import Plugin # Context() @@ -524,7 +524,7 @@ plugin_name = "" if message.unique_id: template += " {plugin}" - plugin = _plugin_lookup(message.unique_id) + plugin = Plugin._lookup(message.unique_id) plugin_name = plugin.name template += ": {message}" diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/buildstream/_frontend/app.py new/BuildStream-1.2.5/buildstream/_frontend/app.py --- old/BuildStream-1.2.4/buildstream/_frontend/app.py 2019-02-14 13:55:14.000000000 +0100 +++ new/BuildStream-1.2.5/buildstream/_frontend/app.py 2019-03-25 12:27:37.000000000 +0100 @@ -531,7 +531,7 @@ queue = job.queue # Get the last failure message for additional context - failure = self._fail_messages.get(element._get_unique_id()) + failure = self._fail_messages.get(element._unique_id) # XXX This is dangerous, sometimes we get the job completed *before* # the failure message reaches us ?? diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/buildstream/_frontend/widget.py new/BuildStream-1.2.5/buildstream/_frontend/widget.py --- old/BuildStream-1.2.4/buildstream/_frontend/widget.py 2019-02-14 13:55:14.000000000 +0100 +++ new/BuildStream-1.2.5/buildstream/_frontend/widget.py 2019-03-25 12:27:37.000000000 +0100 @@ -32,7 +32,7 @@ from .. import __version__ as bst_version from .._exceptions import ImplError from .._message import MessageType -from ..plugin import _plugin_lookup +from ..plugin import Plugin # These messages are printed a bit differently @@ -187,7 +187,7 @@ if element_id is None: return "" - plugin = _plugin_lookup(element_id) + plugin = Plugin._lookup(element_id) name = plugin._get_full_name() # Sneak the action name in with the element name @@ -224,7 +224,7 @@ missing = False key = ' ' * self._key_length - plugin = _plugin_lookup(element_id) + plugin = Plugin._lookup(element_id) if isinstance(plugin, Element): _, key, missing = plugin._get_display_key() @@ -586,7 +586,7 @@ # Track logfiles for later use element_id = message.task_id or message.unique_id if message.message_type in ERROR_MESSAGES and element_id is not None: - plugin = _plugin_lookup(element_id) + plugin = Plugin._lookup(element_id) self._failure_messages[plugin].append(message) return self._render(message) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/buildstream/_loader/loader.py new/BuildStream-1.2.5/buildstream/_loader/loader.py --- old/BuildStream-1.2.4/buildstream/_loader/loader.py 2019-02-14 13:55:14.000000000 +0100 +++ new/BuildStream-1.2.5/buildstream/_loader/loader.py 2019-03-24 13:24:09.000000000 +0100 @@ -202,11 +202,12 @@ # rewritable (bool): Whether we should load in round trippable mode # ticker (callable): A callback to report loaded filenames to the frontend # fetch_subprojects (bool): Whether to fetch subprojects while loading + # provenance (Provenance): The location from where the file was referred to, or None # # Returns: # (LoadElement): A loaded LoadElement # - def _load_file(self, filename, rewritable, ticker, fetch_subprojects): + def _load_file(self, filename, rewritable, ticker, fetch_subprojects, provenance=None): # Silently ignore already loaded files if filename in self._elements: @@ -222,21 +223,34 @@ node = _yaml.load(fullpath, shortname=filename, copy_tree=rewritable, project=self.project) except LoadError as e: if e.reason == LoadErrorReason.MISSING_FILE: + + if self.project.junction: + message = "Could not find element '{}' in project referred to by junction element '{}'" \ + .format(filename, self.project.junction.name) + else: + message = "Could not find element '{}' in elements directory '{}'".format(filename, self._basedir) + + if provenance: + message = "{}: {}".format(provenance, message) + # If we can't find the file, try to suggest plausible # alternatives by stripping the element-path from the given # filename, and verifying that it exists. - message = "Could not find element '{}' in elements directory '{}'".format(filename, self._basedir) detail = None elements_dir = os.path.relpath(self._basedir, self.project.directory) element_relpath = os.path.relpath(filename, elements_dir) if filename.startswith(elements_dir) and os.path.exists(os.path.join(self._basedir, element_relpath)): detail = "Did you mean '{}'?".format(element_relpath) + raise LoadError(LoadErrorReason.MISSING_FILE, message, detail=detail) from e + elif e.reason == LoadErrorReason.LOADING_DIRECTORY: # If a <directory>.bst file exists in the element path, # let's suggest this as a plausible alternative. message = str(e) + if provenance: + message = "{}: {}".format(provenance, message) detail = None if os.path.exists(os.path.join(self._basedir, filename + '.bst')): element_name = filename + '.bst' @@ -262,13 +276,14 @@ # Load all dependency files for the new LoadElement for dep in element.deps: if dep.junction: - self._load_file(dep.junction, rewritable, ticker, fetch_subprojects) + self._load_file(dep.junction, rewritable, ticker, fetch_subprojects, dep.provenance) loader = self._get_loader(dep.junction, rewritable=rewritable, ticker=ticker, fetch_subprojects=fetch_subprojects) else: loader = self - dep_element = loader._load_file(dep.name, rewritable, ticker, fetch_subprojects) + dep_element = loader._load_file(dep.name, rewritable, ticker, + fetch_subprojects, dep.provenance) if _yaml.node_get(dep_element.node, str, Symbol.KIND) == 'junction': raise LoadError(LoadErrorReason.INVALID_DATA, diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/buildstream/_scheduler/jobs/elementjob.py new/BuildStream-1.2.5/buildstream/_scheduler/jobs/elementjob.py --- old/BuildStream-1.2.4/buildstream/_scheduler/jobs/elementjob.py 2019-02-14 13:55:14.000000000 +0100 +++ new/BuildStream-1.2.5/buildstream/_scheduler/jobs/elementjob.py 2019-03-25 12:27:37.000000000 +0100 @@ -73,7 +73,7 @@ self._complete_cb = complete_cb # The complete callable function # Set the task wide ID for logging purposes - self.set_task_id(element._get_unique_id()) + self.set_task_id(element._unique_id) @property def element(self): @@ -100,7 +100,7 @@ args = dict(kwargs) args['scheduler'] = True self._scheduler.context.message( - Message(self._element._get_unique_id(), + Message(self._element._unique_id, message_type, message, **args)) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/buildstream/_scheduler/queues/buildqueue.py new/BuildStream-1.2.5/buildstream/_scheduler/queues/buildqueue.py --- old/BuildStream-1.2.4/buildstream/_scheduler/queues/buildqueue.py 2019-02-14 13:55:14.000000000 +0100 +++ new/BuildStream-1.2.5/buildstream/_scheduler/queues/buildqueue.py 2019-03-25 12:27:37.000000000 +0100 @@ -35,9 +35,6 @@ return element._assemble() def status(self, element): - # state of dependencies may have changed, recalculate element state - element._update_state() - if not element._is_required(): # Artifact is not currently required but it may be requested later. # Keep it in the queue. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/buildstream/_scheduler/queues/fetchqueue.py new/BuildStream-1.2.5/buildstream/_scheduler/queues/fetchqueue.py --- old/BuildStream-1.2.4/buildstream/_scheduler/queues/fetchqueue.py 2019-02-14 13:55:14.000000000 +0100 +++ new/BuildStream-1.2.5/buildstream/_scheduler/queues/fetchqueue.py 2019-03-25 12:27:37.000000000 +0100 @@ -44,9 +44,6 @@ source._fetch() def status(self, element): - # state of dependencies may have changed, recalculate element state - element._update_state() - if not element._is_required(): # Artifact is not currently required but it may be requested later. # Keep it in the queue. @@ -72,7 +69,7 @@ if not success: return - element._update_state() + element._fetch_done() # Successful fetch, we must be CACHED now assert element._get_consistency() == Consistency.CACHED diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/buildstream/_scheduler/queues/pullqueue.py new/BuildStream-1.2.5/buildstream/_scheduler/queues/pullqueue.py --- old/BuildStream-1.2.4/buildstream/_scheduler/queues/pullqueue.py 2019-02-14 13:55:14.000000000 +0100 +++ new/BuildStream-1.2.5/buildstream/_scheduler/queues/pullqueue.py 2019-03-25 12:27:37.000000000 +0100 @@ -38,9 +38,6 @@ raise SkipJob(self.action_name) def status(self, element): - # state of dependencies may have changed, recalculate element state - element._update_state() - if not element._is_required(): # Artifact is not currently required but it may be requested later. # Keep it in the queue. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/buildstream/_scheduler/queues/queue.py new/BuildStream-1.2.5/buildstream/_scheduler/queues/queue.py --- old/BuildStream-1.2.4/buildstream/_scheduler/queues/queue.py 2019-02-14 13:55:14.000000000 +0100 +++ new/BuildStream-1.2.5/buildstream/_scheduler/queues/queue.py 2019-03-25 12:27:37.000000000 +0100 @@ -348,7 +348,7 @@ # a message for the element they are processing def _message(self, element, message_type, brief, **kwargs): context = element._get_context() - message = Message(element._get_unique_id(), message_type, brief, **kwargs) + message = Message(element._unique_id, message_type, brief, **kwargs) context.message(message) def _element_log_path(self, element): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/buildstream/_scheduler/queues/trackqueue.py new/BuildStream-1.2.5/buildstream/_scheduler/queues/trackqueue.py --- old/BuildStream-1.2.4/buildstream/_scheduler/queues/trackqueue.py 2019-02-14 13:55:14.000000000 +0100 +++ new/BuildStream-1.2.5/buildstream/_scheduler/queues/trackqueue.py 2019-03-25 12:27:37.000000000 +0100 @@ -19,8 +19,7 @@ # Jürg Billeter <[email protected]> # BuildStream toplevel imports -from ...plugin import _plugin_lookup -from ... import SourceError +from ...plugin import Plugin # Local imports from . import Queue, QueueStatus @@ -55,7 +54,7 @@ # Set the new refs in the main process one by one as they complete for unique_id, new_ref in result: - source = _plugin_lookup(unique_id) + source = Plugin._lookup(unique_id) source._save_ref(new_ref) element._tracking_done() diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/buildstream/_stream.py new/BuildStream-1.2.5/buildstream/_stream.py --- old/BuildStream-1.2.4/buildstream/_stream.py 2019-02-14 13:55:14.000000000 +0100 +++ new/BuildStream-1.2.5/buildstream/_stream.py 2019-03-25 12:27:37.000000000 +0100 @@ -28,7 +28,7 @@ from contextlib import contextmanager from tempfile import TemporaryDirectory -from ._exceptions import StreamError, ImplError, BstError, set_last_task_error +from ._exceptions import StreamError, ImplError, BstError from ._message import Message, MessageType from ._scheduler import Scheduler, SchedStatus, TrackQueue, FetchQueue, BuildQueue, PullQueue, PushQueue from ._pipeline import Pipeline, PipelineSelection @@ -1007,17 +1007,6 @@ _, status = self._scheduler.run(self.queues) - # Force update element states after a run, such that the summary - # is more coherent - try: - for element in self.total_elements: - element._update_state() - except BstError as e: - self._message(MessageType.ERROR, "Error resolving final state", detail=str(e)) - set_last_task_error(e.domain, e.reason) - except Exception as e: # pylint: disable=broad-except - self._message(MessageType.BUG, "Unhandled exception while resolving final state", detail=str(e)) - if status == SchedStatus.ERROR: raise StreamError() elif status == SchedStatus.TERMINATED: diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/buildstream/_version.py new/BuildStream-1.2.5/buildstream/_version.py --- old/BuildStream-1.2.4/buildstream/_version.py 2019-02-14 14:09:13.000000000 +0100 +++ new/BuildStream-1.2.5/buildstream/_version.py 2019-03-25 12:33:22.000000000 +0100 @@ -8,11 +8,11 @@ version_json = ''' { - "date": "2019-02-14T21:27:11+0900", + "date": "2019-03-25T20:28:51+0900", "dirty": false, "error": null, - "full-revisionid": "c270d09a5bb473a3a7f2f7243440a306b792bc89", - "version": "1.2.4" + "full-revisionid": "5e638aa386238c5f5562e8b0562127d28f722252", + "version": "1.2.5" } ''' # END VERSION_JSON diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/buildstream/_yaml.py new/BuildStream-1.2.5/buildstream/_yaml.py --- old/BuildStream-1.2.4/buildstream/_yaml.py 2019-02-14 13:55:14.000000000 +0100 +++ new/BuildStream-1.2.5/buildstream/_yaml.py 2019-03-24 13:24:09.000000000 +0100 @@ -70,7 +70,11 @@ # Convert a Provenance to a string for error reporting def __str__(self): - return "{} [line {:d} column {:d}]".format(self.filename.shortname, self.line, self.col) + filename = self.filename.shortname + if self.filename.project and self.filename.project.junction: + filename = "{}:{}".format(self.filename.project.junction.name, self.filename.shortname) + + return "{} [line {:d} column {:d}]".format(filename, self.line, self.col) # Abstract method def clone(self): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/buildstream/element.py new/BuildStream-1.2.5/buildstream/element.py --- old/BuildStream-1.2.4/buildstream/element.py 2019-02-14 13:55:14.000000000 +0100 +++ new/BuildStream-1.2.5/buildstream/element.py 2019-03-25 12:27:37.000000000 +0100 @@ -88,6 +88,7 @@ from ._versions import BST_CORE_ARTIFACT_VERSION from ._exceptions import BstError, LoadError, LoadErrorReason, ImplError, ErrorDomain from .utils import UtilError +from .types import _UniquePriorityQueue from . import Plugin, Consistency from . import SandboxFlags from . import utils @@ -214,6 +215,8 @@ self.__runtime_dependencies = [] # Direct runtime dependency Elements self.__build_dependencies = [] # Direct build dependency Elements + self.__reverse_dependencies = set() # Direct reverse dependency Elements + self.__ready_for_runtime = False # Wether the element has all its dependencies ready and has a cache key self.__sources = [] # List of Sources self.__weak_cache_key = None # Our cached weak cache key self.__strict_cache_key = None # Our cached cache key for strict builds @@ -685,6 +688,7 @@ files_written = {} old_dep_keys = {} workspace = self._get_workspace() + project = self._get_project() if self.__can_build_incrementally() and workspace.last_successful: old_dep_keys = self.__get_artifact_metadata_dependencies(workspace.last_successful) @@ -746,9 +750,8 @@ overlapping_elements = elements[1:] for elm in overlapping_elements: element = self.search(scope, elm) - element_project = element._get_project() if not element.__file_is_whitelisted(f): - if element_project.fail_on_overlap: + if project.fail_on_overlap: overlap_error_elements.append(elm) overlap_error = True else: @@ -924,9 +927,12 @@ for meta_dep in meta.dependencies: dependency = Element._new_from_meta(meta_dep, artifacts) element.__runtime_dependencies.append(dependency) + dependency.__reverse_dependencies.add(element) + for meta_dep in meta.build_dependencies: dependency = Element._new_from_meta(meta_dep, artifacts) element.__build_dependencies.append(dependency) + dependency.__reverse_dependencies.add(element) return element @@ -1143,6 +1149,10 @@ # Strong cache key could not be calculated yet return + if not self.__ready_for_runtime and self.__cache_key is not None: + self.__ready_for_runtime = all( + dep.__ready_for_runtime for dep in self.__runtime_dependencies) + # _get_display_key(): # # Returns cache keys for display purposes @@ -1245,7 +1255,7 @@ self.__tracking_scheduled = False self.__tracking_done = True - self._update_state() + self.__update_state_recursively() # _track(): # @@ -1262,7 +1272,7 @@ for source in self.__sources: old_ref = source.get_ref() new_ref = source._track() - refs.append((source._get_unique_id(), new_ref)) + refs.append((source._unique_id, new_ref)) # Complimentary warning that the new ref will be unused. if old_ref != new_ref and self._get_workspace(): @@ -1421,7 +1431,7 @@ self.__assemble_scheduled = False self.__assemble_done = True - self._update_state() + self.__update_state_recursively() if self._get_workspace() and self._cached(): # @@ -1592,6 +1602,15 @@ return artifact_size + # _fetch_done() + # + # Indicates that fetching the sources for this element has been done. + # + def _fetch_done(self): + # We are not updating the state recursively here since fetching can + # never end up in updating them. + self._update_state() + # _pull_pending() # # Check whether the artifact will be pulled. @@ -1625,7 +1644,7 @@ def _pull_done(self): self.__pull_done = True - self._update_state() + self.__update_state_recursively() def _pull_strong(self, *, progress=None): weak_key = self._get_cache_key(strength=_KeyStrength.WEAK) @@ -2504,6 +2523,24 @@ return utils._deduplicate(keys) + # __update_state_recursively() + # + # Update the state of all reverse dependencies, recursively. + # + def __update_state_recursively(self): + queue = _UniquePriorityQueue() + queue.push(self._unique_id, self) + + while queue: + element = queue.pop() + + old_ready_for_runtime = element.__ready_for_runtime + element._update_state() + + if element.__ready_for_runtime != old_ready_for_runtime: + for rdep in element.__reverse_dependencies: + queue.push(rdep._unique_id, rdep) + def _overlap_error_detail(f, forbidden_overlap_elements, elements): if forbidden_overlap_elements: diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/buildstream/plugin.py new/BuildStream-1.2.5/buildstream/plugin.py --- old/BuildStream-1.2.4/buildstream/plugin.py 2019-02-14 13:55:14.000000000 +0100 +++ new/BuildStream-1.2.5/buildstream/plugin.py 2019-03-25 12:27:37.000000000 +0100 @@ -92,6 +92,7 @@ --------------- """ +import itertools import os import subprocess from contextlib import contextmanager @@ -145,6 +146,23 @@ core format version :ref:`core format version <project_format_version>`. """ + # Unique id generator for Plugins + # + # Each plugin gets a unique id at creation. + # Ids are a monotically increasing integer + __id_generator = itertools.count() + + # Hold on to a lookup table by counter of all instantiated plugins. + # We use this to send the id back from child processes so we can lookup + # corresponding element/source in the master process. + # + # Use WeakValueDictionary() so the map we use to lookup objects does not + # keep the plugins alive after pipeline destruction. + # + # Note that Plugins can only be instantiated in the main process before + # scheduling tasks. + __TABLE = WeakValueDictionary() + def __init__(self, name, context, project, provenance, type_tag): self.name = name @@ -157,11 +175,24 @@ For sources this is for display purposes only. """ + # Unique ID + # + # This id allows to uniquely identify a plugin. + # + # /!\ the unique id must be an increasing value /!\ + # This is because we are depending on it in buildstream.element.Element + # to give us a topological sort over all elements. + # Modifying how we handle ids here will modify the behavior of the + # Element's state handling. + self._unique_id = next(self.__id_generator) + + # register ourself in the table containing all existing plugins + self.__TABLE[self._unique_id] = self + self.__context = context # The Context object self.__project = project # The Project object self.__provenance = provenance # The Provenance information self.__type_tag = type_tag # The type of plugin (element or source) - self.__unique_id = _plugin_register(self) # Unique ID self.__configuring = False # Whether we are currently configuring # Infer the kind identifier @@ -519,7 +550,7 @@ self.call(... command which takes time ...) """ with self.__context.timed_activity(activity_name, - unique_id=self.__unique_id, + unique_id=self._unique_id, detail=detail, silent_nested=silent_nested): yield @@ -611,6 +642,23 @@ # Private Methods used in BuildStream # ############################################################# + # _lookup(): + # + # Fetch a plugin in the current process by its + # unique identifier + # + # Args: + # unique_id: The unique identifier as returned by + # plugin._unique_id + # + # Returns: + # (Plugin): The plugin for the given ID, or None + # + @classmethod + def _lookup(cls, unique_id): + assert unique_id in cls.__TABLE, "Could not find plugin with ID {}".format(unique_id) + return cls.__TABLE[unique_id] + # _get_context() # # Fetches the invocation context @@ -625,13 +673,6 @@ def _get_project(self): return self.__project - # _get_unique_id(): - # - # Fetch the plugin's unique identifier - # - def _get_unique_id(self): - return self.__unique_id - # _get_provenance(): # # Fetch bst file, line and column of the entity @@ -716,7 +757,7 @@ return (exit_code, output) def __message(self, message_type, brief, **kwargs): - message = Message(self.__unique_id, message_type, brief, **kwargs) + message = Message(self._unique_id, message_type, brief, **kwargs) self.__context.message(message) def __note_command(self, output, *popenargs, **kwargs): @@ -734,42 +775,3 @@ return '{}:{}'.format(project.junction.name, self.name) else: return self.name - - -# Hold on to a lookup table by counter of all instantiated plugins. -# We use this to send the id back from child processes so we can lookup -# corresponding element/source in the master process. -# -# Use WeakValueDictionary() so the map we use to lookup objects does not -# keep the plugins alive after pipeline destruction. -# -# Note that Plugins can only be instantiated in the main process before -# scheduling tasks. -__PLUGINS_UNIQUE_ID = 0 -__PLUGINS_TABLE = WeakValueDictionary() - - -# _plugin_lookup(): -# -# Fetch a plugin in the current process by its -# unique identifier -# -# Args: -# unique_id: The unique identifier as returned by -# plugin._get_unique_id() -# -# Returns: -# (Plugin): The plugin for the given ID, or None -# -def _plugin_lookup(unique_id): - assert unique_id in __PLUGINS_TABLE, "Could not find plugin with ID {}".format(unique_id) - return __PLUGINS_TABLE[unique_id] - - -# No need for unregister, WeakValueDictionary() will remove entries -# in itself when the referenced plugins are garbage collected. -def _plugin_register(plugin): - global __PLUGINS_UNIQUE_ID # pylint: disable=global-statement - __PLUGINS_UNIQUE_ID += 1 - __PLUGINS_TABLE[__PLUGINS_UNIQUE_ID] = plugin - return __PLUGINS_UNIQUE_ID diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/buildstream/types.py new/BuildStream-1.2.5/buildstream/types.py --- old/BuildStream-1.2.4/buildstream/types.py 1970-01-01 01:00:00.000000000 +0100 +++ new/BuildStream-1.2.5/buildstream/types.py 2019-03-25 12:27:37.000000000 +0100 @@ -0,0 +1,177 @@ +# +# Copyright (C) 2018 Bloomberg LP +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Tristan Van Berkom <[email protected]> +# Jim MacArthur <[email protected]> +# Benjamin Schubert <[email protected]> + +""" +Foundation types +================ + +""" + +from enum import Enum +import heapq + + +class Scope(Enum): + """Defines the scope of dependencies to include for a given element + when iterating over the dependency graph in APIs like + :func:`Element.dependencies() <buildstream.element.Element.dependencies>` + """ + + ALL = 1 + """All elements which the given element depends on, following + all elements required for building. Including the element itself. + """ + + BUILD = 2 + """All elements required for building the element, including their + respective run dependencies. Not including the given element itself. + """ + + RUN = 3 + """All elements required for running the element. Including the element + itself. + """ + + NONE = 4 + """Just the element itself, no dependencies. + + *Since: 1.4* + """ + + +class Consistency(): + """Defines the various consistency states of a :class:`.Source`. + """ + + INCONSISTENT = 0 + """Inconsistent + + Inconsistent sources have no explicit reference set. They cannot + produce a cache key, be fetched or staged. They can only be tracked. + """ + + RESOLVED = 1 + """Resolved + + Resolved sources have a reference and can produce a cache key and + be fetched, however they cannot be staged. + """ + + CACHED = 2 + """Cached + + Sources have a cached unstaged copy in the source directory. + """ + + +class CoreWarnings(): + """CoreWarnings() + + Some common warnings which are raised by core functionalities within BuildStream are found in this class. + """ + + OVERLAPS = "overlaps" + """ + This warning will be produced when buildstream detects an overlap on an element + which is not whitelisted. See :ref:`Overlap Whitelist <public_overlap_whitelist>` + """ + + REF_NOT_IN_TRACK = "ref-not-in-track" + """ + This warning will be produced when a source is configured with a reference + which is found to be invalid based on the configured track + """ + + BAD_ELEMENT_SUFFIX = "bad-element-suffix" + """ + This warning will be produced when an element whose name does not end in .bst + is referenced either on the command line or by another element + """ + + BAD_CHARACTERS_IN_NAME = "bad-characters-in-name" + """ + This warning will be produces when filename for a target contains invalid + characters in its name. + """ + + +# _KeyStrength(): +# +# Strength of cache key +# +class _KeyStrength(Enum): + + # Includes strong cache keys of all build dependencies and their + # runtime dependencies. + STRONG = 1 + + # Includes names of direct build dependencies but does not include + # cache keys of dependencies. + WEAK = 2 + + +# _UniquePriorityQueue(): +# +# Implements a priority queue that adds only each key once. +# +# The queue will store and priority based on a tuple (key, item). +# +class _UniquePriorityQueue: + + def __init__(self): + self._items = set() + self._heap = [] + + # push(): + # + # Push a new item in the queue. + # + # If the item is already present in the queue as identified by the key, + # this is a noop. + # + # Args: + # key (hashable, comparable): unique key to use for checking for + # the object's existence and used for + # ordering + # item (any): item to push to the queue + # + def push(self, key, item): + if key not in self._items: + self._items.add(key) + heapq.heappush(self._heap, (key, item)) + + # pop(): + # + # Pop the next item from the queue, by priority order. + # + # Returns: + # (any): the next item + # + # Throw: + # IndexError: when the list is empty + # + def pop(self): + key, item = heapq.heappop(self._heap) + self._items.remove(key) + return item + + def __len__(self): + return len(self._heap) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/requirements/dev-requirements.in new/BuildStream-1.2.5/requirements/dev-requirements.in --- old/BuildStream-1.2.4/requirements/dev-requirements.in 2019-02-14 13:55:14.000000000 +0100 +++ new/BuildStream-1.2.5/requirements/dev-requirements.in 2019-03-24 13:24:09.000000000 +0100 @@ -1,8 +1,6 @@ -coverage == 4.4.0 pep8 -pylint == 2.1.1 +pylint pytest >= 3.7 -pytest-cov >= 2.5.0 pytest-datafiles pytest-env pytest-pep8 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/requirements/dev-requirements.txt new/BuildStream-1.2.5/requirements/dev-requirements.txt --- old/BuildStream-1.2.4/requirements/dev-requirements.txt 2019-02-14 13:55:14.000000000 +0100 +++ new/BuildStream-1.2.5/requirements/dev-requirements.txt 2019-03-24 13:24:09.000000000 +0100 @@ -1,27 +1,27 @@ -coverage==4.4 pep8==1.7.1 -pylint==2.1.1 -pytest==4.1.1 -pytest-cov==2.6.1 +pylint==2.3.1 +pytest==4.3.0 pytest-datafiles==2.0 pytest-env==0.6.2 pytest-pep8==1.0.6 pytest-pylint==0.14.0 -pytest-xdist==1.26.0 +pytest-xdist==1.26.1 pytest-timeout==1.3.3 ## The following requirements were added by pip freeze: apipkg==1.5 -astroid==2.1.0 -atomicwrites==1.2.1 -attrs==18.2.0 +astroid==2.2.3 +atomicwrites==1.3.0 +attrs==19.1.0 execnet==1.5.0 -isort==4.3.4 +isort==4.3.10 lazy-object-proxy==1.3.1 mccabe==0.6.1 -more-itertools==5.0.0 -pluggy==0.8.1 -py==1.7.0 +more-itertools==6.0.0 +pathlib2==2.3.3 +pluggy==0.9.0 +py==1.8.0 pytest-cache==1.0 -pytest-forked==1.0.1 +pytest-forked==1.0.2 six==1.12.0 -wrapt==1.11.0 +typed-ast==1.2.0 +wrapt==1.11.1 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/setup.cfg new/BuildStream-1.2.5/setup.cfg --- old/BuildStream-1.2.4/setup.cfg 2019-02-14 14:09:13.000000000 +0100 +++ new/BuildStream-1.2.5/setup.cfg 2019-03-25 12:33:22.000000000 +0100 @@ -11,7 +11,7 @@ test = pytest [tool:pytest] -addopts = --verbose --basetemp ./tmp --pep8 --pylint --pylint-rcfile=.pylintrc --cov=buildstream --cov-config .coveragerc --durations=20 +addopts = --verbose --basetemp ./tmp --pep8 --pylint --pylint-rcfile=.pylintrc --durations=20 norecursedirs = integration-cache tmp __pycache__ .eggs python_files = tests/*/*.py pep8maxlinelength = 119 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/tests/format/project/missing-element/manual.bst new/BuildStream-1.2.5/tests/format/project/missing-element/manual.bst --- old/BuildStream-1.2.4/tests/format/project/missing-element/manual.bst 1970-01-01 01:00:00.000000000 +0100 +++ new/BuildStream-1.2.5/tests/format/project/missing-element/manual.bst 2019-03-14 07:56:54.000000000 +0100 @@ -0,0 +1,4 @@ +kind: manual + +depends: +- missing.bst diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/tests/format/project/missing-element/project.conf new/BuildStream-1.2.5/tests/format/project/missing-element/project.conf --- old/BuildStream-1.2.4/tests/format/project/missing-element/project.conf 1970-01-01 01:00:00.000000000 +0100 +++ new/BuildStream-1.2.5/tests/format/project/missing-element/project.conf 2019-03-14 07:56:54.000000000 +0100 @@ -0,0 +1 @@ +name: test diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/tests/format/project/missing-junction/manual.bst new/BuildStream-1.2.5/tests/format/project/missing-junction/manual.bst --- old/BuildStream-1.2.4/tests/format/project/missing-junction/manual.bst 1970-01-01 01:00:00.000000000 +0100 +++ new/BuildStream-1.2.5/tests/format/project/missing-junction/manual.bst 2019-03-14 07:56:54.000000000 +0100 @@ -0,0 +1,5 @@ +kind: manual + +depends: +- filename: element.bst + junction: missing.bst diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/tests/format/project/missing-junction/project.conf new/BuildStream-1.2.5/tests/format/project/missing-junction/project.conf --- old/BuildStream-1.2.4/tests/format/project/missing-junction/project.conf 1970-01-01 01:00:00.000000000 +0100 +++ new/BuildStream-1.2.5/tests/format/project/missing-junction/project.conf 2019-03-14 07:56:54.000000000 +0100 @@ -0,0 +1 @@ +name: test diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/tests/format/project.py new/BuildStream-1.2.5/tests/format/project.py --- old/BuildStream-1.2.4/tests/format/project.py 2019-02-14 13:55:14.000000000 +0100 +++ new/BuildStream-1.2.5/tests/format/project.py 2019-03-24 13:24:09.000000000 +0100 @@ -27,6 +27,26 @@ @pytest.mark.datafiles(os.path.join(DATA_DIR)) +def test_missing_element(cli, datafiles): + project = os.path.join(datafiles.dirname, datafiles.basename, "missing-element") + result = cli.run(project=project, args=['show', 'manual.bst']) + result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.MISSING_FILE) + + # Assert that we have the expected provenance encoded into the error + assert "manual.bst [line 4 column 2]" in result.stderr + + [email protected](os.path.join(DATA_DIR)) +def test_missing_junction(cli, datafiles): + project = os.path.join(datafiles.dirname, datafiles.basename, "missing-junction") + result = cli.run(project=project, args=['show', 'manual.bst']) + result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.MISSING_FILE) + + # Assert that we have the expected provenance encoded into the error + assert "manual.bst [line 4 column 2]" in result.stderr + + [email protected](os.path.join(DATA_DIR)) def test_empty_project_name(cli, datafiles): project = os.path.join(datafiles.dirname, datafiles.basename, "emptyname") result = cli.run(project=project, args=['workspace', 'list']) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/tests/frontend/overlaps/sub-collect.bst new/BuildStream-1.2.5/tests/frontend/overlaps/sub-collect.bst --- old/BuildStream-1.2.4/tests/frontend/overlaps/sub-collect.bst 1970-01-01 01:00:00.000000000 +0100 +++ new/BuildStream-1.2.5/tests/frontend/overlaps/sub-collect.bst 2019-02-24 10:22:46.000000000 +0100 @@ -0,0 +1,11 @@ +kind: compose + +depends: +- filename: c.bst + type: build +- filename: a-sub.bst + junction: sub-project.bst + type: build +- filename: z-sub.bst + junction: sub-project.bst + type: build diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/tests/frontend/overlaps/sub-project/a-sub.bst new/BuildStream-1.2.5/tests/frontend/overlaps/sub-project/a-sub.bst --- old/BuildStream-1.2.4/tests/frontend/overlaps/sub-project/a-sub.bst 1970-01-01 01:00:00.000000000 +0100 +++ new/BuildStream-1.2.5/tests/frontend/overlaps/sub-project/a-sub.bst 2019-02-24 10:22:46.000000000 +0100 @@ -0,0 +1,7 @@ +kind: import +config: + source: / + target: / +sources: +- kind: local + path: "files/a" diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/tests/frontend/overlaps/sub-project/z-sub.bst new/BuildStream-1.2.5/tests/frontend/overlaps/sub-project/z-sub.bst --- old/BuildStream-1.2.4/tests/frontend/overlaps/sub-project/z-sub.bst 1970-01-01 01:00:00.000000000 +0100 +++ new/BuildStream-1.2.5/tests/frontend/overlaps/sub-project/z-sub.bst 2019-02-24 10:22:46.000000000 +0100 @@ -0,0 +1,7 @@ +kind: import +config: + source: / + target: / +sources: +- kind: local + path: "files/z" diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/tests/frontend/overlaps.py new/BuildStream-1.2.5/tests/frontend/overlaps.py --- old/BuildStream-1.2.4/tests/frontend/overlaps.py 2019-02-14 13:55:14.000000000 +0100 +++ new/BuildStream-1.2.5/tests/frontend/overlaps.py 2019-03-24 13:24:09.000000000 +0100 @@ -1,6 +1,7 @@ import os import pytest from tests.testutils.runcli import cli +from tests.testutils import generate_junction from buildstream._exceptions import ErrorDomain from buildstream import _yaml @@ -10,15 +11,12 @@ "overlaps" ) -project_template = { - "name": "test", - "element-path": "." -} - -def gen_project(project_dir, fail_on_overlap): - template = dict(project_template) - template["fail-on-overlap"] = fail_on_overlap +def gen_project(project_dir, fail_on_overlap, project_name="test"): + template = { + "name": project_name, + "fail-on-overlap": fail_on_overlap + } projectfile = os.path.join(project_dir, "project.conf") _yaml.dump(template, projectfile) @@ -82,3 +80,30 @@ result = cli.run(project=project_dir, silent=True, args=[ 'build', 'script.bst']) result.assert_success() + + [email protected](DATA_DIR) [email protected]("project_policy", [('fail'), ('warn')]) [email protected]("subproject_policy", [('fail'), ('warn')]) +def test_overlap_subproject(cli, tmpdir, datafiles, project_policy, subproject_policy): + project_dir = str(datafiles) + subproject_dir = os.path.join(project_dir, 'sub-project') + junction_path = os.path.join(project_dir, 'sub-project.bst') + + gen_project(project_dir, bool(project_policy == 'fail'), project_name='test') + gen_project(subproject_dir, bool(subproject_policy == 'fail'), project_name='subtest') + generate_junction(tmpdir, subproject_dir, junction_path) + + # Here we have a dependency chain where the project element + # always overlaps with the subproject element. + # + # Test that overlap error vs warning policy for this overlap + # is always controlled by the project and not the subproject. + # + result = cli.run(project=project_dir, silent=True, args=['build', 'sub-collect.bst']) + if project_policy == 'fail': + result.assert_main_error(ErrorDomain.STREAM, None) + result.assert_task_error(ErrorDomain.ELEMENT, "overlap-error") + else: + result.assert_success() + assert "WARNING Non-whitelisted overlaps detected" in result.stderr diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/tests/frontend/workspace.py new/BuildStream-1.2.5/tests/frontend/workspace.py --- old/BuildStream-1.2.4/tests/frontend/workspace.py 2019-02-14 13:55:14.000000000 +0100 +++ new/BuildStream-1.2.5/tests/frontend/workspace.py 2019-03-25 12:27:37.000000000 +0100 @@ -782,3 +782,54 @@ # Check that the original /usr/bin/hello is not in the checkout assert not os.path.exists(os.path.join(checkout, 'usr', 'bin', 'hello')) + + +# This strange test tests against a regression raised in issue #919, +# where opening a workspace on a runtime dependency of a build only +# dependency causes `bst build` to not build the specified target +# but just successfully builds the workspaced element and happily +# exits without completing the build. +# +TEST_DIR = os.path.join( + os.path.dirname(os.path.realpath(__file__)) +) + + [email protected](TEST_DIR) [email protected]( + ["case", "non_workspaced_elements_state"], + [ + ("workspaced-build-dep", ["waiting", "waiting", "waiting", "waiting", "waiting"]), + ("workspaced-runtime-dep", ["buildable", "buildable", "waiting", "waiting", "waiting"]) + ], +) [email protected]("strict", [("strict"), ("non-strict")]) +def test_build_all(cli, tmpdir, datafiles, case, strict, non_workspaced_elements_state): + project = os.path.join(str(datafiles), case) + workspace = os.path.join(str(tmpdir), 'workspace') + non_leaf_elements = ["elem2.bst", "elem3.bst", "stack.bst", "elem4.bst", "elem5.bst"] + all_elements = ["elem1.bst", *non_leaf_elements] + + # Configure strict mode + strict_mode = True + if strict != 'strict': + strict_mode = False + cli.configure({ + 'projects': { + 'test': { + 'strict': strict_mode + } + } + }) + + # First open the workspace + result = cli.run(project=project, args=['workspace', 'open', 'elem1.bst', workspace]) + result.assert_success() + + # Now build the targets elem4.bst and elem5.bst + result = cli.run(project=project, args=['build', 'elem4.bst', 'elem5.bst']) + result.assert_success() + + # Assert that the target is built + for element in all_elements: + assert cli.get_element_state(project, element) == 'cached' diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/tests/frontend/workspaced-build-dep/elements/elem1.bst new/BuildStream-1.2.5/tests/frontend/workspaced-build-dep/elements/elem1.bst --- old/BuildStream-1.2.4/tests/frontend/workspaced-build-dep/elements/elem1.bst 1970-01-01 01:00:00.000000000 +0100 +++ new/BuildStream-1.2.5/tests/frontend/workspaced-build-dep/elements/elem1.bst 2019-03-25 12:27:37.000000000 +0100 @@ -0,0 +1,5 @@ +kind: import + +sources: +- kind: local + path: files/file1 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/tests/frontend/workspaced-build-dep/elements/elem2.bst new/BuildStream-1.2.5/tests/frontend/workspaced-build-dep/elements/elem2.bst --- old/BuildStream-1.2.4/tests/frontend/workspaced-build-dep/elements/elem2.bst 1970-01-01 01:00:00.000000000 +0100 +++ new/BuildStream-1.2.5/tests/frontend/workspaced-build-dep/elements/elem2.bst 2019-03-25 12:27:37.000000000 +0100 @@ -0,0 +1,9 @@ +kind: import + +depends: +- filename: elem1.bst + type: build + +sources: +- kind: local + path: files/file2 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/tests/frontend/workspaced-build-dep/elements/elem3.bst new/BuildStream-1.2.5/tests/frontend/workspaced-build-dep/elements/elem3.bst --- old/BuildStream-1.2.4/tests/frontend/workspaced-build-dep/elements/elem3.bst 1970-01-01 01:00:00.000000000 +0100 +++ new/BuildStream-1.2.5/tests/frontend/workspaced-build-dep/elements/elem3.bst 2019-03-25 12:27:37.000000000 +0100 @@ -0,0 +1,9 @@ +kind: import + +depends: +- filename: elem2.bst + type: build + +sources: +- kind: local + path: files/file3 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/tests/frontend/workspaced-build-dep/elements/elem4.bst new/BuildStream-1.2.5/tests/frontend/workspaced-build-dep/elements/elem4.bst --- old/BuildStream-1.2.4/tests/frontend/workspaced-build-dep/elements/elem4.bst 1970-01-01 01:00:00.000000000 +0100 +++ new/BuildStream-1.2.5/tests/frontend/workspaced-build-dep/elements/elem4.bst 2019-03-25 12:27:37.000000000 +0100 @@ -0,0 +1,9 @@ +kind: import + +depends: +- filename: stack.bst + type: build + +sources: +- kind: local + path: files/file4 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/tests/frontend/workspaced-build-dep/elements/elem5.bst new/BuildStream-1.2.5/tests/frontend/workspaced-build-dep/elements/elem5.bst --- old/BuildStream-1.2.4/tests/frontend/workspaced-build-dep/elements/elem5.bst 1970-01-01 01:00:00.000000000 +0100 +++ new/BuildStream-1.2.5/tests/frontend/workspaced-build-dep/elements/elem5.bst 2019-03-25 12:27:37.000000000 +0100 @@ -0,0 +1,9 @@ +kind: import + +depends: +- filename: elem3.bst + type: build + +sources: +- kind: local + path: files/file4 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/tests/frontend/workspaced-build-dep/elements/stack.bst new/BuildStream-1.2.5/tests/frontend/workspaced-build-dep/elements/stack.bst --- old/BuildStream-1.2.4/tests/frontend/workspaced-build-dep/elements/stack.bst 1970-01-01 01:00:00.000000000 +0100 +++ new/BuildStream-1.2.5/tests/frontend/workspaced-build-dep/elements/stack.bst 2019-03-25 12:27:37.000000000 +0100 @@ -0,0 +1,4 @@ +kind: stack + +depends: +- elem3.bst diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/tests/frontend/workspaced-build-dep/project.conf new/BuildStream-1.2.5/tests/frontend/workspaced-build-dep/project.conf --- old/BuildStream-1.2.4/tests/frontend/workspaced-build-dep/project.conf 1970-01-01 01:00:00.000000000 +0100 +++ new/BuildStream-1.2.5/tests/frontend/workspaced-build-dep/project.conf 2019-03-25 12:27:37.000000000 +0100 @@ -0,0 +1,8 @@ +# Unique project name +name: test + +# Required BuildStream format version +format-version: 12 + +# Subdirectory where elements are stored +element-path: elements diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/tests/frontend/workspaced-runtime-dep/elements/elem1.bst new/BuildStream-1.2.5/tests/frontend/workspaced-runtime-dep/elements/elem1.bst --- old/BuildStream-1.2.4/tests/frontend/workspaced-runtime-dep/elements/elem1.bst 1970-01-01 01:00:00.000000000 +0100 +++ new/BuildStream-1.2.5/tests/frontend/workspaced-runtime-dep/elements/elem1.bst 2019-03-25 12:27:37.000000000 +0100 @@ -0,0 +1,5 @@ +kind: import + +sources: +- kind: local + path: files/file1 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/tests/frontend/workspaced-runtime-dep/elements/elem2.bst new/BuildStream-1.2.5/tests/frontend/workspaced-runtime-dep/elements/elem2.bst --- old/BuildStream-1.2.4/tests/frontend/workspaced-runtime-dep/elements/elem2.bst 1970-01-01 01:00:00.000000000 +0100 +++ new/BuildStream-1.2.5/tests/frontend/workspaced-runtime-dep/elements/elem2.bst 2019-03-25 12:27:37.000000000 +0100 @@ -0,0 +1,9 @@ +kind: import + +depends: +- filename: elem1.bst + type: runtime + +sources: +- kind: local + path: files/file2 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/tests/frontend/workspaced-runtime-dep/elements/elem3.bst new/BuildStream-1.2.5/tests/frontend/workspaced-runtime-dep/elements/elem3.bst --- old/BuildStream-1.2.4/tests/frontend/workspaced-runtime-dep/elements/elem3.bst 1970-01-01 01:00:00.000000000 +0100 +++ new/BuildStream-1.2.5/tests/frontend/workspaced-runtime-dep/elements/elem3.bst 2019-03-25 12:27:37.000000000 +0100 @@ -0,0 +1,9 @@ +kind: import + +depends: +- filename: elem2.bst + type: runtime + +sources: +- kind: local + path: files/file3 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/tests/frontend/workspaced-runtime-dep/elements/elem4.bst new/BuildStream-1.2.5/tests/frontend/workspaced-runtime-dep/elements/elem4.bst --- old/BuildStream-1.2.4/tests/frontend/workspaced-runtime-dep/elements/elem4.bst 1970-01-01 01:00:00.000000000 +0100 +++ new/BuildStream-1.2.5/tests/frontend/workspaced-runtime-dep/elements/elem4.bst 2019-03-25 12:27:37.000000000 +0100 @@ -0,0 +1,9 @@ +kind: import + +depends: +- filename: stack.bst + type: build + +sources: +- kind: local + path: files/file4 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/tests/frontend/workspaced-runtime-dep/elements/elem5.bst new/BuildStream-1.2.5/tests/frontend/workspaced-runtime-dep/elements/elem5.bst --- old/BuildStream-1.2.4/tests/frontend/workspaced-runtime-dep/elements/elem5.bst 1970-01-01 01:00:00.000000000 +0100 +++ new/BuildStream-1.2.5/tests/frontend/workspaced-runtime-dep/elements/elem5.bst 2019-03-25 12:27:37.000000000 +0100 @@ -0,0 +1,9 @@ +kind: import + +depends: +- filename: elem3.bst + type: build + +sources: +- kind: local + path: files/file4 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/tests/frontend/workspaced-runtime-dep/elements/stack.bst new/BuildStream-1.2.5/tests/frontend/workspaced-runtime-dep/elements/stack.bst --- old/BuildStream-1.2.4/tests/frontend/workspaced-runtime-dep/elements/stack.bst 1970-01-01 01:00:00.000000000 +0100 +++ new/BuildStream-1.2.5/tests/frontend/workspaced-runtime-dep/elements/stack.bst 2019-03-25 12:27:37.000000000 +0100 @@ -0,0 +1,4 @@ +kind: stack + +depends: +- elem3.bst diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/tests/frontend/workspaced-runtime-dep/project.conf new/BuildStream-1.2.5/tests/frontend/workspaced-runtime-dep/project.conf --- old/BuildStream-1.2.4/tests/frontend/workspaced-runtime-dep/project.conf 1970-01-01 01:00:00.000000000 +0100 +++ new/BuildStream-1.2.5/tests/frontend/workspaced-runtime-dep/project.conf 2019-03-25 12:27:37.000000000 +0100 @@ -0,0 +1,8 @@ +# Unique project name +name: test + +# Required BuildStream format version +format-version: 12 + +# Subdirectory where elements are stored +element-path: elements diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/tests/loader/junctions/missing-element/junction-A.bst new/BuildStream-1.2.5/tests/loader/junctions/missing-element/junction-A.bst --- old/BuildStream-1.2.4/tests/loader/junctions/missing-element/junction-A.bst 1970-01-01 01:00:00.000000000 +0100 +++ new/BuildStream-1.2.5/tests/loader/junctions/missing-element/junction-A.bst 2019-03-24 13:24:09.000000000 +0100 @@ -0,0 +1,4 @@ +kind: junction +sources: +- kind: local + path: junctionA diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/tests/loader/junctions/missing-element/junctionA/bad-junction-target.bst new/BuildStream-1.2.5/tests/loader/junctions/missing-element/junctionA/bad-junction-target.bst --- old/BuildStream-1.2.4/tests/loader/junctions/missing-element/junctionA/bad-junction-target.bst 1970-01-01 01:00:00.000000000 +0100 +++ new/BuildStream-1.2.5/tests/loader/junctions/missing-element/junctionA/bad-junction-target.bst 2019-03-24 13:24:09.000000000 +0100 @@ -0,0 +1,5 @@ +kind: manual + +depends: +- filename: noelement.bst + junction: missing-junction.bst diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/tests/loader/junctions/missing-element/junctionA/junction-B.bst new/BuildStream-1.2.5/tests/loader/junctions/missing-element/junctionA/junction-B.bst --- old/BuildStream-1.2.4/tests/loader/junctions/missing-element/junctionA/junction-B.bst 1970-01-01 01:00:00.000000000 +0100 +++ new/BuildStream-1.2.5/tests/loader/junctions/missing-element/junctionA/junction-B.bst 2019-03-24 13:24:09.000000000 +0100 @@ -0,0 +1,4 @@ +kind: junction +sources: +- kind: local + path: junctionB diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/tests/loader/junctions/missing-element/junctionA/junctionB/project.conf new/BuildStream-1.2.5/tests/loader/junctions/missing-element/junctionA/junctionB/project.conf --- old/BuildStream-1.2.4/tests/loader/junctions/missing-element/junctionA/junctionB/project.conf 1970-01-01 01:00:00.000000000 +0100 +++ new/BuildStream-1.2.5/tests/loader/junctions/missing-element/junctionA/junctionB/project.conf 2019-03-24 13:24:09.000000000 +0100 @@ -0,0 +1 @@ +name: projectB diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/tests/loader/junctions/missing-element/junctionA/project.conf new/BuildStream-1.2.5/tests/loader/junctions/missing-element/junctionA/project.conf --- old/BuildStream-1.2.4/tests/loader/junctions/missing-element/junctionA/project.conf 1970-01-01 01:00:00.000000000 +0100 +++ new/BuildStream-1.2.5/tests/loader/junctions/missing-element/junctionA/project.conf 2019-03-24 13:24:09.000000000 +0100 @@ -0,0 +1 @@ +name: projectA diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/tests/loader/junctions/missing-element/junctionA/target.bst new/BuildStream-1.2.5/tests/loader/junctions/missing-element/junctionA/target.bst --- old/BuildStream-1.2.4/tests/loader/junctions/missing-element/junctionA/target.bst 1970-01-01 01:00:00.000000000 +0100 +++ new/BuildStream-1.2.5/tests/loader/junctions/missing-element/junctionA/target.bst 2019-03-24 13:24:09.000000000 +0100 @@ -0,0 +1,5 @@ +kind: stack + +depends: +- filename: missing.bst + junction: junction-B.bst diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/tests/loader/junctions/missing-element/project.conf new/BuildStream-1.2.5/tests/loader/junctions/missing-element/project.conf --- old/BuildStream-1.2.4/tests/loader/junctions/missing-element/project.conf 1970-01-01 01:00:00.000000000 +0100 +++ new/BuildStream-1.2.5/tests/loader/junctions/missing-element/project.conf 2019-03-24 13:24:09.000000000 +0100 @@ -0,0 +1 @@ +name: test diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/tests/loader/junctions/missing-element/sub-target-bad-junction.bst new/BuildStream-1.2.5/tests/loader/junctions/missing-element/sub-target-bad-junction.bst --- old/BuildStream-1.2.4/tests/loader/junctions/missing-element/sub-target-bad-junction.bst 1970-01-01 01:00:00.000000000 +0100 +++ new/BuildStream-1.2.5/tests/loader/junctions/missing-element/sub-target-bad-junction.bst 2019-03-24 13:24:09.000000000 +0100 @@ -0,0 +1,5 @@ +kind: stack + +depends: +- filename: bad-junction-target.bst + junction: junction-A.bst diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/tests/loader/junctions/missing-element/sub-target.bst new/BuildStream-1.2.5/tests/loader/junctions/missing-element/sub-target.bst --- old/BuildStream-1.2.4/tests/loader/junctions/missing-element/sub-target.bst 1970-01-01 01:00:00.000000000 +0100 +++ new/BuildStream-1.2.5/tests/loader/junctions/missing-element/sub-target.bst 2019-03-24 13:24:09.000000000 +0100 @@ -0,0 +1,5 @@ +kind: stack + +depends: +- filename: target.bst + junction: junction-A.bst diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/tests/loader/junctions/missing-element/target.bst new/BuildStream-1.2.5/tests/loader/junctions/missing-element/target.bst --- old/BuildStream-1.2.4/tests/loader/junctions/missing-element/target.bst 1970-01-01 01:00:00.000000000 +0100 +++ new/BuildStream-1.2.5/tests/loader/junctions/missing-element/target.bst 2019-03-24 13:24:09.000000000 +0100 @@ -0,0 +1,5 @@ +kind: stack + +depends: +- filename: missing.bst + junction: junction-A.bst diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/tests/loader/junctions.py new/BuildStream-1.2.5/tests/loader/junctions.py --- old/BuildStream-1.2.4/tests/loader/junctions.py 2019-02-14 13:55:14.000000000 +0100 +++ new/BuildStream-1.2.5/tests/loader/junctions.py 2019-03-24 13:24:09.000000000 +0100 @@ -3,7 +3,7 @@ import shutil from buildstream import _yaml, ElementError -from buildstream._exceptions import LoadError, LoadErrorReason +from buildstream._exceptions import ErrorDomain, LoadError, LoadErrorReason from tests.testutils import cli, create_repo from tests.testutils.site import HAVE_GIT @@ -48,6 +48,36 @@ @pytest.mark.datafiles(DATA_DIR) +def test_missing_file_in_subproject(cli, datafiles): + project = os.path.join(str(datafiles), 'missing-element') + result = cli.run(project=project, args=['show', 'target.bst']) + result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.MISSING_FILE) + + # Assert that we have the expected provenance encoded into the error + assert "target.bst [line 4 column 2]" in result.stderr + + [email protected](DATA_DIR) +def test_missing_file_in_subsubproject(cli, datafiles): + project = os.path.join(str(datafiles), 'missing-element') + result = cli.run(project=project, args=['show', 'sub-target.bst']) + result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.MISSING_FILE) + + # Assert that we have the expected provenance encoded into the error + assert "junction-A.bst:target.bst [line 4 column 2]" in result.stderr + + [email protected](DATA_DIR) +def test_missing_junction_in_subproject(cli, datafiles): + project = os.path.join(str(datafiles), 'missing-element') + result = cli.run(project=project, args=['show', 'sub-target-bad-junction.bst']) + result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.MISSING_FILE) + + # Assert that we have the expected provenance encoded into the error + assert "junction-A.bst:bad-junction-target.bst [line 4 column 2]" in result.stderr + + [email protected](DATA_DIR) def test_nested_simple(cli, tmpdir, datafiles): foo = os.path.join(str(datafiles), 'foo') copy_subprojects(foo, datafiles, ['base']) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/BuildStream-1.2.4/tests/testutils/artifactshare.py new/BuildStream-1.2.5/tests/testutils/artifactshare.py --- old/BuildStream-1.2.4/tests/testutils/artifactshare.py 2019-02-14 13:55:14.000000000 +0100 +++ new/BuildStream-1.2.5/tests/testutils/artifactshare.py 2019-03-24 13:24:09.000000000 +0100 @@ -8,7 +8,6 @@ from contextlib import contextmanager from multiprocessing import Process, Queue -import pytest_cov from buildstream import _yaml from buildstream._artifactcache.cascache import CASCache @@ -75,7 +74,13 @@ # Run the artifact server. # def run(self, q): - pytest_cov.embed.cleanup_on_sigterm() + + try: + import pytest_cov + except ImportError: + pass + else: + pytest_cov.embed.cleanup_on_sigterm() # Optionally mock statvfs if self.total_space:
