Hello community, here is the log from the commit of package python-distributed for openSUSE:Factory checked in at 2020-09-14 12:33:34 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/python-distributed (Old) and /work/SRC/openSUSE:Factory/.python-distributed.new.4249 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "python-distributed" Mon Sep 14 12:33:34 2020 rev:34 rq:834241 version:2.26.0 Changes: -------- --- /work/SRC/openSUSE:Factory/python-distributed/python-distributed.changes 2020-08-31 16:51:52.248401564 +0200 +++ /work/SRC/openSUSE:Factory/.python-distributed.new.4249/python-distributed.changes 2020-09-14 12:35:43.701381788 +0200 @@ -1,0 +2,19 @@ +Sat Sep 12 19:58:20 UTC 2020 - Arun Persaud <[email protected]> + +- update to version 2.26.0: + * Add logging for adaptive start and stop (GH#4101) Matthew Rocklin + * Don’t close a nannied worker if it hasn’t yet started (GH#4093) + Matthew Rocklin + * Respect timeouts when closing clients synchronously (GH#4096) + Matthew Rocklin + * Log when downloading a preload script (GH#4094) Matthew Rocklin + * dask-worker --nprocs accepts negative values (GH#4089) Dror + Speiser + * Support zero-worker clients (GH#4090) Matthew Rocklin + * Exclude fire-and-forget client from metrics (GH#4078) Tom + Augspurger + * Drop Serialized.deserialize() method (GH#4073) jakirkham + * Add timeout= keyword to Client.wait_for_workers method (GH#4087) + Matthew Rocklin + +------------------------------------------------------------------- Old: ---- distributed-2.25.0.tar.gz New: ---- distributed-2.26.0.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ python-distributed.spec ++++++ --- /var/tmp/diff_new_pack.nTN62A/_old 2020-09-14 12:35:45.913383199 +0200 +++ /var/tmp/diff_new_pack.nTN62A/_new 2020-09-14 12:35:45.913383199 +0200 @@ -21,7 +21,7 @@ # Test requires network connection %bcond_with test Name: python-distributed -Version: 2.25.0 +Version: 2.26.0 Release: 0 Summary: Library for distributed computing with Python License: BSD-3-Clause ++++++ distributed-2.25.0.tar.gz -> distributed-2.26.0.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/distributed-2.25.0/PKG-INFO new/distributed-2.26.0/PKG-INFO --- old/distributed-2.25.0/PKG-INFO 2020-08-29 00:37:02.176320000 +0200 +++ new/distributed-2.26.0/PKG-INFO 2020-09-11 23:28:19.954074000 +0200 @@ -1,6 +1,6 @@ Metadata-Version: 1.2 Name: distributed -Version: 2.25.0 +Version: 2.26.0 Summary: Distributed scheduler for Dask Home-page: https://distributed.dask.org Maintainer: Matthew Rocklin diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/distributed-2.25.0/distributed/_version.py new/distributed-2.26.0/distributed/_version.py --- old/distributed-2.25.0/distributed/_version.py 2020-08-29 00:37:02.177928400 +0200 +++ new/distributed-2.26.0/distributed/_version.py 2020-09-11 23:28:19.955338200 +0200 @@ -8,11 +8,11 @@ version_json = ''' { - "date": "2020-08-28T17:36:26-0500", + "date": "2020-09-11T16:27:23-0500", "dirty": false, "error": null, - "full-revisionid": "b862f14cf338cac1c31d9f5ee604aea9c59c9935", - "version": "2.25.0" + "full-revisionid": "73d381a29906bf6e08fad013c921922bd73fd9d4", + "version": "2.26.0" } ''' # END VERSION_JSON diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/distributed-2.25.0/distributed/cli/dask_worker.py new/distributed-2.26.0/distributed/cli/dask_worker.py --- old/distributed-2.25.0/distributed/cli/dask_worker.py 2020-08-25 19:17:40.000000000 +0200 +++ new/distributed-2.26.0/distributed/cli/dask_worker.py 2020-09-08 06:05:25.000000000 +0200 @@ -127,7 +127,8 @@ type=int, default=1, show_default=True, - help="Number of worker processes to launch.", + help="Number of worker processes to launch. " + "If negative, then (CPU_COUNT + 1 + nprocs) is used.", ) @click.option( "--name", @@ -288,6 +289,15 @@ if v is not None } + if nprocs < 0: + nprocs = CPU_COUNT + 1 + nprocs + + if nprocs <= 0: + logger.error( + "Failed to launch worker. Must specify --nprocs so that there's at least one process." + ) + sys.exit(1) + if nprocs > 1 and not nanny: logger.error( "Failed to launch worker. You cannot use the --no-nanny argument when nprocs > 1." diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/distributed-2.25.0/distributed/cli/tests/test_dask_worker.py new/distributed-2.26.0/distributed/cli/tests/test_dask_worker.py --- old/distributed-2.25.0/distributed/cli/tests/test_dask_worker.py 2020-08-25 19:17:40.000000000 +0200 +++ new/distributed-2.26.0/distributed/cli/tests/test_dask_worker.py 2020-09-08 06:05:25.000000000 +0200 @@ -8,6 +8,7 @@ import sys import os from time import sleep +from multiprocessing import cpu_count import distributed.cli.dask_worker from distributed import Client, Scheduler @@ -238,6 +239,13 @@ ) +def test_nprocs_negative(loop): + with popen(["dask-scheduler", "--no-dashboard"]) as sched: + with popen(["dask-worker", "127.0.0.1:8786", "--nprocs=-1"]) as worker: + with Client("tcp://127.0.0.1:8786", loop=loop) as c: + c.wait_for_workers(cpu_count(), timeout="10 seconds") + + def test_nprocs_expands_name(loop): with popen(["dask-scheduler", "--no-dashboard"]) as sched: with popen( diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/distributed-2.25.0/distributed/client.py new/distributed-2.26.0/distributed/client.py --- old/distributed-2.25.0/distributed/client.py 2020-08-29 00:26:28.000000000 +0200 +++ new/distributed-2.26.0/distributed/client.py 2020-09-09 05:39:40.000000000 +0200 @@ -1028,13 +1028,6 @@ **self._startup_kwargs, ) - # Wait for all workers to be ready - # XXX should be a LocalCluster method instead - while not self.cluster.workers or len(self.cluster.scheduler.workers) < len( - self.cluster.workers - ): - await asyncio.sleep(0.01) - address = self.cluster.scheduler_address self._gather_semaphore = asyncio.Semaphore(5) @@ -1157,15 +1150,24 @@ except EnvironmentError: logger.debug("Not able to query scheduler for identity") - async def _wait_for_workers(self, n_workers=0): + async def _wait_for_workers(self, n_workers=0, timeout=None): info = await self.scheduler.identity() + if timeout: + deadline = time() + parse_timedelta(timeout) + else: + deadline = None while n_workers and len(info["workers"]) < n_workers: + if deadline and time() > deadline: + raise TimeoutError( + "Only %d/%d workers arrived after %s" + % (len(info["workers"]), n_workers, timeout) + ) await asyncio.sleep(0.1) info = await self.scheduler.identity() - def wait_for_workers(self, n_workers=0): + def wait_for_workers(self, n_workers=0, timeout=None): """Blocking call to wait for n workers before continuing""" - return self.sync(self._wait_for_workers, n_workers) + return self.sync(self._wait_for_workers, n_workers, timeout=timeout) def _heartbeat(self): if self.scheduler_comm: @@ -1427,7 +1429,7 @@ self.sync(_) - sync(self.loop, self._close, fast=True) + sync(self.loop, self._close, fast=True, callback_timeout=timeout) assert self.status == "closed" @@ -4815,7 +4817,8 @@ c = _get_global_client() if c is not None: c._should_close_loop = False - c.close(timeout=2) + with suppress(TimeoutError): + c.close(timeout=2) atexit.register(_close_global_client) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/distributed-2.25.0/distributed/deploy/adaptive.py new/distributed-2.26.0/distributed/deploy/adaptive.py --- old/distributed-2.25.0/distributed/deploy/adaptive.py 2020-08-25 19:13:46.000000000 +0200 +++ new/distributed-2.26.0/distributed/deploy/adaptive.py 2020-09-11 23:15:38.000000000 +0200 @@ -103,6 +103,8 @@ self.target_duration = parse_timedelta(target_duration) + logger.info("Adaptive scaling started: minimum=%s maximum=%s", minimum, maximum) + super().__init__( minimum=minimum, maximum=maximum, wait_count=wait_count, interval=interval ) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/distributed-2.25.0/distributed/deploy/adaptive_core.py new/distributed-2.26.0/distributed/deploy/adaptive_core.py --- old/distributed-2.25.0/distributed/deploy/adaptive_core.py 2020-08-25 19:17:40.000000000 +0200 +++ new/distributed-2.26.0/distributed/deploy/adaptive_core.py 2020-09-11 23:15:38.000000000 +0200 @@ -1,4 +1,5 @@ import collections +import logging import math from tornado.ioloop import IOLoop, PeriodicCallback @@ -8,6 +9,9 @@ from ..utils import parse_timedelta +logger = logging.getLogger(__name__) + + class AdaptiveCore: """ The core logic for adaptive deployments, with none of the cluster details @@ -103,6 +107,8 @@ self.log = collections.deque(maxlen=10000) def stop(self): + logger.info("Adaptive stop") + if self.periodic_callback: self.periodic_callback.stop() self.periodic_callback = None diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/distributed-2.25.0/distributed/deploy/tests/test_local.py new/distributed-2.26.0/distributed/deploy/tests/test_local.py --- old/distributed-2.25.0/distributed/deploy/tests/test_local.py 2020-08-29 00:26:28.000000000 +0200 +++ new/distributed-2.26.0/distributed/deploy/tests/test_local.py 2020-09-08 06:05:25.000000000 +0200 @@ -1050,3 +1050,11 @@ assert w assert not w + + [email protected] +async def test_no_workers(cleanup): + async with Client( + n_workers=0, silence_logs=False, dashboard_address=None, asynchronous=True + ) as c: + pass diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/distributed-2.25.0/distributed/http/scheduler/prometheus/__init__.py new/distributed-2.26.0/distributed/http/scheduler/prometheus/__init__.py --- old/distributed-2.25.0/distributed/http/scheduler/prometheus/__init__.py 2020-08-25 19:17:41.000000000 +0200 +++ new/distributed-2.26.0/distributed/http/scheduler/prometheus/__init__.py 2020-09-08 06:05:25.000000000 +0200 @@ -15,7 +15,7 @@ yield GaugeMetricFamily( "dask_scheduler_clients", "Number of clients connected.", - value=len(self.server.clients), + value=len([k for k in self.server.clients if k != "fire-and-forget"]), ) yield GaugeMetricFamily( diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/distributed-2.25.0/distributed/http/scheduler/tests/test_scheduler_http.py new/distributed-2.26.0/distributed/http/scheduler/tests/test_scheduler_http.py --- old/distributed-2.25.0/distributed/http/scheduler/tests/test_scheduler_http.py 2020-08-25 19:17:41.000000000 +0200 +++ new/distributed-2.26.0/distributed/http/scheduler/tests/test_scheduler_http.py 2020-09-08 06:05:25.000000000 +0200 @@ -92,9 +92,14 @@ assert response.headers["Content-Type"] == "text/plain; version=0.0.4" txt = response.body.decode("utf8") - families = {familiy.name for familiy in text_string_to_metric_families(txt)} + families = { + family.name: family for family in text_string_to_metric_families(txt) + } assert "dask_scheduler_workers" in families + client = families["dask_scheduler_clients"] + assert client.samples[0].value == 1.0 + @gen_cluster(client=True, clean_kwargs={"threads": False}) async def test_prometheus_collect_task_states(c, s, a, b): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/distributed-2.25.0/distributed/nanny.py new/distributed-2.26.0/distributed/nanny.py --- old/distributed-2.25.0/distributed/nanny.py 2020-08-29 00:26:28.000000000 +0200 +++ new/distributed-2.26.0/distributed/nanny.py 2020-09-11 23:15:38.000000000 +0200 @@ -438,7 +438,7 @@ self.loop.add_callback(self._on_exit, exitcode) async def _on_exit(self, exitcode): - if self.status not in (Status.closing, Status.closed): + if self.status not in (Status.init, Status.closing, Status.closed): try: await self.scheduler.unregister(address=self.worker_address) except (EnvironmentError, CommClosedError): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/distributed-2.25.0/distributed/preloading.py new/distributed-2.26.0/distributed/preloading.py --- old/distributed-2.25.0/distributed/preloading.py 2020-08-29 00:26:28.000000000 +0200 +++ new/distributed-2.26.0/distributed/preloading.py 2020-09-08 06:05:25.000000000 +0200 @@ -120,6 +120,7 @@ async def _download_module(url: str) -> ModuleType: + logger.info("Downloading preload at %s", url) assert is_webaddress(url) client = AsyncHTTPClient() diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/distributed-2.25.0/distributed/protocol/serialize.py new/distributed-2.26.0/distributed/protocol/serialize.py --- old/distributed-2.25.0/distributed/protocol/serialize.py 2020-08-29 00:26:28.000000000 +0200 +++ new/distributed-2.26.0/distributed/protocol/serialize.py 2020-09-08 06:05:25.000000000 +0200 @@ -381,12 +381,6 @@ self.header = header self.frames = frames - def deserialize(self): - from .core import decompress - - frames = decompress(self.header, self.frames) - return deserialize(self.header, frames) - def __eq__(self, other): return ( isinstance(other, Serialized) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/distributed-2.25.0/distributed/tests/test_client.py new/distributed-2.26.0/distributed/tests/test_client.py --- old/distributed-2.25.0/distributed/tests/test_client.py 2020-08-29 00:26:28.000000000 +0200 +++ new/distributed-2.26.0/distributed/tests/test_client.py 2020-09-08 06:05:25.000000000 +0200 @@ -5807,6 +5807,12 @@ assert time() < start + 1 await w.close() + with pytest.raises(TimeoutError) as info: + await c.wait_for_workers(n_workers=10, timeout="1 ms") + + assert "2/10" in str(info.value).replace(" ", "") + assert "1 ms" in str(info.value) + @pytest.mark.skipif(WINDOWS, reason="num_fds not supported on windows") @pytest.mark.asyncio diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/distributed-2.25.0/distributed/tests/test_preload.py new/distributed-2.26.0/distributed/tests/test_preload.py --- old/distributed-2.25.0/distributed/tests/test_preload.py 2020-08-29 00:26:28.000000000 +0200 +++ new/distributed-2.26.0/distributed/tests/test_preload.py 2020-09-08 06:05:25.000000000 +0200 @@ -9,7 +9,7 @@ import dask from distributed import Client, Scheduler, Worker, Nanny -from distributed.utils_test import cluster +from distributed.utils_test import cluster, captured_logger from distributed.utils_test import loop, cleanup # noqa F401 @@ -139,7 +139,9 @@ app = web.Application([(r"/preload", MyHandler)]) server = app.listen(12345) try: - async with Scheduler(preload=["http://localhost:12345/preload";]) as s: - assert s.foo == 1 + with captured_logger("distributed.preloading") as log: + async with Scheduler(preload=["http://localhost:12345/preload";]) as s: + assert s.foo == 1 + assert "12345/preload" in log.getvalue() finally: server.stop() diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/distributed-2.25.0/distributed/tests/test_worker.py new/distributed-2.26.0/distributed/tests/test_worker.py --- old/distributed-2.25.0/distributed/tests/test_worker.py 2020-08-29 00:26:28.000000000 +0200 +++ new/distributed-2.26.0/distributed/tests/test_worker.py 2020-09-11 23:15:38.000000000 +0200 @@ -1647,3 +1647,20 @@ else: assert w.status == Status.closed assert "Heartbeat to scheduler failed" in logger.getvalue() + + [email protected] +async def test_bad_local_directory(cleanup): + async with await Scheduler() as s: + try: + async with Worker(s.address, local_directory="/not/a/valid-directory"): + pass + except PermissionError: + pass + else: + if WINDOWS: + pass + else: + assert False + + assert not any("error" in log for log in s.get_logs()) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/distributed-2.25.0/distributed.egg-info/PKG-INFO new/distributed-2.26.0/distributed.egg-info/PKG-INFO --- old/distributed-2.25.0/distributed.egg-info/PKG-INFO 2020-08-29 00:37:01.000000000 +0200 +++ new/distributed-2.26.0/distributed.egg-info/PKG-INFO 2020-09-11 23:28:19.000000000 +0200 @@ -1,6 +1,6 @@ Metadata-Version: 1.2 Name: distributed -Version: 2.25.0 +Version: 2.26.0 Summary: Distributed scheduler for Dask Home-page: https://distributed.dask.org Maintainer: Matthew Rocklin diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/distributed-2.25.0/docs/source/changelog.rst new/distributed-2.26.0/docs/source/changelog.rst --- old/distributed-2.25.0/docs/source/changelog.rst 2020-08-29 00:36:04.000000000 +0200 +++ new/distributed-2.26.0/docs/source/changelog.rst 2020-09-11 23:27:00.000000000 +0200 @@ -1,6 +1,20 @@ Changelog ========= +2.26.0 - 2020-09-11 +------------------- + +- Add logging for adaptive start and stop (:pr:`4101`) `Matthew Rocklin`_ +- Don't close a nannied worker if it hasn't yet started (:pr:`4093`) `Matthew Rocklin`_ +- Respect timeouts when closing clients synchronously (:pr:`4096`) `Matthew Rocklin`_ +- Log when downloading a preload script (:pr:`4094`) `Matthew Rocklin`_ +- ``dask-worker --nprocs`` accepts negative values (:pr:`4089`) `Dror Speiser`_ +- Support zero-worker clients (:pr:`4090`) `Matthew Rocklin`_ +- Exclude ``fire-and-forget`` client from metrics (:pr:`4078`) `Tom Augspurger`_ +- Drop ``Serialized.deserialize()`` method (:pr:`4073`) `jakirkham`_ +- Add ``timeout=`` keyword to ``Client.wait_for_workers`` method (:pr:`4087`) `Matthew Rocklin`_ + + 2.25.0 - 2020-08-28 ------------------- @@ -1939,3 +1953,4 @@ .. _`Jack Xiaosong Xu`: https://github.com/jackxxu .. _`Willi Rath`: https://github.com/willirath .. _`Roberto Panai`: https://github.com/rpanai +.. _`Dror Speiser`: https://github.com/drorspei
