Script 'mail_helper' called by obssrc Hello community, here is the log from the commit of package python-kombu for openSUSE:Factory checked in at 2022-01-05 13:39:26 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/python-kombu (Old) and /work/SRC/openSUSE:Factory/.python-kombu.new.1896 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "python-kombu" Wed Jan 5 13:39:26 2022 rev:73 rq:943612 version:5.2.3 Changes: -------- --- /work/SRC/openSUSE:Factory/python-kombu/python-kombu.changes 2021-12-13 20:45:52.080495895 +0100 +++ /work/SRC/openSUSE:Factory/.python-kombu.new.1896/python-kombu.changes 2022-01-05 13:39:47.649527540 +0100 @@ -1,0 +2,18 @@ +Mon Jan 3 14:16:01 UTC 2022 - Dirk M??ller <dmuel...@suse.com> + +- update to 5.2.3: + * Allow redis >= 4.0.2. + * Fix PyPy CI jobs. + * SQS transport: detect FIFO queue properly by checking queue URL (#1450). + * Ensure that restore is atomic in redis transport (#1444). + * Restrict setuptools>=59.1.1,<59.7.0. + * Bump minimum py-amqp to v5.0.9 (#1462). + * Reduce memory usage of Transport (#1470). + * Prevent event loop polling on closed redis transports (and causing leak). + * Respect connection timeout (#1458) + * prevent redis event loop stopping on ???consumer: Cannot connect??? (#1477). + * Bump redis version to >= 3.4.1. + * try latest sqs dependencies to fix security warning. + * Tests & dependency updates + +------------------------------------------------------------------- Old: ---- kombu-5.2.0.tar.gz New: ---- kombu-5.2.3.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ python-kombu.spec ++++++ --- /var/tmp/diff_new_pack.rcUqEA/_old 2022-01-05 13:39:48.149527933 +0100 +++ /var/tmp/diff_new_pack.rcUqEA/_new 2022-01-05 13:39:48.153527936 +0100 @@ -1,7 +1,7 @@ # # spec file for package python-kombu # -# Copyright (c) 2021 SUSE LLC +# Copyright (c) 2022 SUSE LLC # # All modifications and additions to the file contributed by third parties # remain the property of their copyright owners, unless otherwise agreed @@ -19,7 +19,7 @@ %{?!python_module:%define python_module() python-%{**} python3-%{**}} %define skip_python2 1 Name: python-kombu -Version: 5.2.0 +Version: 5.2.3 Release: 0 Summary: AMQP Messaging Framework for Python License: BSD-3-Clause @@ -29,8 +29,8 @@ BuildRequires: %{python_module PyYAML >= 3.10} BuildRequires: %{python_module Pyro4} BuildRequires: %{python_module SQLAlchemy} -BuildRequires: %{python_module amqp >= 5.0.6} -BuildRequires: %{python_module boto3 >= 1.4.4} +BuildRequires: %{python_module amqp >= 5.0.9} +BuildRequires: %{python_module boto3 >= 1.9.12} BuildRequires: %{python_module cached-property} BuildRequires: %{python_module case >= 1.5.2} BuildRequires: %{python_module importlib-metadata >= 0.18} @@ -38,11 +38,11 @@ BuildRequires: %{python_module pycurl >= 7.43.0.2} BuildRequires: %{python_module pytest} BuildRequires: %{python_module pytz} -BuildRequires: %{python_module redis >= 3.3.11} +BuildRequires: %{python_module redis >= 3.4.1} BuildRequires: %{python_module setuptools >= 20.6.7} BuildRequires: fdupes BuildRequires: python-rpm-macros -Requires: python-amqp >= 5.0.6 +Requires: python-amqp >= 5.0.9 Requires: python-cached-property Requires: python-importlib-metadata >= 0.18 Requires: python-setuptools ++++++ kombu-5.2.0.tar.gz -> kombu-5.2.3.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/kombu-5.2.0/AUTHORS new/kombu-5.2.3/AUTHORS --- old/kombu-5.2.0/AUTHORS 2021-11-02 16:36:25.000000000 +0100 +++ new/kombu-5.2.3/AUTHORS 2021-11-08 01:17:43.000000000 +0100 @@ -18,6 +18,7 @@ Antoine Legrand <antoine.legr...@smartjog.com> Anton Gyllenberg <an...@iki.fi> Ask Solem <a...@celeryproject.org> +Asif Saif Uddin <auv...@gmail.com> Basil Mironenko <bmirone...@ddn.com> Bobby Beever <bobby.bee...@yahoo.com> Brian Bernstein diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/kombu-5.2.0/PKG-INFO new/kombu-5.2.3/PKG-INFO --- old/kombu-5.2.0/PKG-INFO 2021-11-02 17:37:47.842602700 +0100 +++ new/kombu-5.2.3/PKG-INFO 2021-12-29 05:59:22.182823400 +0100 @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: kombu -Version: 5.2.0 +Version: 5.2.3 Summary: Messaging library for Python. Home-page: https://kombu.readthedocs.io Author: Ask Solem @@ -17,6 +17,7 @@ Classifier: Programming Language :: Python :: 3.7 Classifier: Programming Language :: Python :: 3.8 Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 Classifier: Programming Language :: Python :: Implementation :: CPython Classifier: Programming Language :: Python :: Implementation :: PyPy Classifier: Intended Audience :: Developers diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/kombu-5.2.0/README.rst new/kombu-5.2.3/README.rst --- old/kombu-5.2.0/README.rst 2021-11-02 17:35:32.000000000 +0100 +++ new/kombu-5.2.3/README.rst 2021-12-29 05:55:15.000000000 +0100 @@ -4,7 +4,7 @@ |build-status| |coverage| |license| |wheel| |pyversion| |pyimp| |downloads| -:Version: 5.2.0 +:Version: 5.2.3 :Documentation: https://kombu.readthedocs.io/ :Download: https://pypi.org/project/kombu/ :Source: https://github.com/celery/kombu/ @@ -305,7 +305,7 @@ Join the `celery-users`_ mailing list. -.. _`celery-users`: https://groups.google.com/group/celery-users/ +.. _`kombu forum`: https://github.com/celery/kombu/discussions Bug tracker =========== diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/kombu-5.2.0/docs/includes/introduction.txt new/kombu-5.2.3/docs/includes/introduction.txt --- old/kombu-5.2.0/docs/includes/introduction.txt 2021-11-02 17:35:32.000000000 +0100 +++ new/kombu-5.2.3/docs/includes/introduction.txt 2021-12-29 05:54:42.000000000 +0100 @@ -1,4 +1,4 @@ -:Version: 5.2.0 +:Version: 5.2.3 :Web: https://kombu.readthedocs.io/ :Download: https://pypi.org/project/kombu/ :Source: https://github.com/celery/kombu/ @@ -23,15 +23,7 @@ * Allows application authors to support several message server solutions by using pluggable transports. - * AMQP transport using the `py-amqp`_, `librabbitmq`_, or `qpid-python`_ libraries. - - * High performance AMQP transport written in C - when using `librabbitmq`_ - - This is automatically enabled if librabbitmq is installed: - - .. code-block:: console - - $ pip install librabbitmq + * AMQP transport using the `py-amqp`_, `redis`_, or `SQS`_ libraries. * Virtual transports makes it really easy to add support for non-AMQP transports. There is already built-in support for `Redis`_, diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/kombu-5.2.0/extra/requirements/default.txt new/kombu-5.2.3/extra/requirements/default.txt --- old/kombu-5.2.0/extra/requirements/default.txt 2021-11-02 16:36:25.000000000 +0100 +++ new/kombu-5.2.3/extra/requirements/default.txt 2021-12-29 05:15:42.000000000 +0100 @@ -1,4 +1,4 @@ importlib-metadata>=0.18; python_version<"3.8" cached_property; python_version<"3.8" -amqp>=5.0.6,<6.0.0 +amqp>=5.0.9,<6.0.0 vine diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/kombu-5.2.0/extra/requirements/extras/librabbitmq.txt new/kombu-5.2.3/extra/requirements/extras/librabbitmq.txt --- old/kombu-5.2.0/extra/requirements/extras/librabbitmq.txt 2021-11-02 16:36:25.000000000 +0100 +++ new/kombu-5.2.3/extra/requirements/extras/librabbitmq.txt 2021-11-08 01:17:43.000000000 +0100 @@ -1 +1 @@ -librabbitmq>=1.5.2 +librabbitmq>=2.0.0 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/kombu-5.2.0/extra/requirements/extras/mongodb.txt new/kombu-5.2.3/extra/requirements/extras/mongodb.txt --- old/kombu-5.2.0/extra/requirements/extras/mongodb.txt 2021-11-02 16:36:25.000000000 +0100 +++ new/kombu-5.2.3/extra/requirements/extras/mongodb.txt 2021-11-08 01:17:43.000000000 +0100 @@ -1 +1 @@ -pymongo>=3.3.0 +pymongo>=3.3.0,<3.12.1 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/kombu-5.2.0/extra/requirements/extras/redis.txt new/kombu-5.2.3/extra/requirements/extras/redis.txt --- old/kombu-5.2.0/extra/requirements/extras/redis.txt 2021-11-02 16:36:25.000000000 +0100 +++ new/kombu-5.2.3/extra/requirements/extras/redis.txt 2021-12-19 06:12:32.000000000 +0100 @@ -1 +1 @@ -redis>=3.3.11 +redis>=3.4.1,!=4.0.0,!=4.0.1 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/kombu-5.2.0/extra/requirements/extras/sqs.txt new/kombu-5.2.3/extra/requirements/extras/sqs.txt --- old/kombu-5.2.0/extra/requirements/extras/sqs.txt 2021-11-02 16:36:25.000000000 +0100 +++ new/kombu-5.2.3/extra/requirements/extras/sqs.txt 2021-11-08 01:17:43.000000000 +0100 @@ -1,3 +1,3 @@ -boto3>=1.4.4 -pycurl==7.43.0.2 # Latest build with wheels provided -urllib3<1.26 # Unittests are faiing with urllib3>=1.28 +boto3>=1.9.12 +pycurl~=7.44.1 # Latest build with wheels provided +urllib3>=1.26.7 # Unittests are faiing with urllib3>=1.28 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/kombu-5.2.0/extra/requirements/pkgutils.txt new/kombu-5.2.3/extra/requirements/pkgutils.txt --- old/kombu-5.2.0/extra/requirements/pkgutils.txt 2021-11-02 16:36:25.000000000 +0100 +++ new/kombu-5.2.3/extra/requirements/pkgutils.txt 2021-12-22 04:35:30.000000000 +0100 @@ -1,4 +1,4 @@ -setuptools>=20.6.7 +setuptools>=59.1.1,<59.7.0 wheel>=0.29.0 flake8>=2.5.4 tox>=2.3.1 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/kombu-5.2.0/extra/requirements/test-ci.txt new/kombu-5.2.3/extra/requirements/test-ci.txt --- old/kombu-5.2.0/extra/requirements/test-ci.txt 2021-11-02 16:36:25.000000000 +0100 +++ new/kombu-5.2.3/extra/requirements/test-ci.txt 2021-12-19 06:12:32.000000000 +0100 @@ -1,5 +1,4 @@ pytest-cov -pytest-travis-fold codecov -r extras/redis.txt -r extras/yaml.txt diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/kombu-5.2.0/kombu/__init__.py new/kombu-5.2.3/kombu/__init__.py --- old/kombu-5.2.0/kombu/__init__.py 2021-11-02 17:34:56.000000000 +0100 +++ new/kombu-5.2.3/kombu/__init__.py 2021-12-29 05:53:28.000000000 +0100 @@ -5,7 +5,7 @@ import sys from collections import namedtuple -__version__ = '5.2.0' +__version__ = '5.2.3' __author__ = 'Ask Solem' __contact__ = 'auv...@gmail.com, a...@celeryproject.org' __homepage__ = 'https://kombu.readthedocs.io' diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/kombu-5.2.0/kombu/connection.py new/kombu-5.2.3/kombu/connection.py --- old/kombu-5.2.0/kombu/connection.py 2021-11-02 16:36:25.000000000 +0100 +++ new/kombu-5.2.3/kombu/connection.py 2021-12-29 05:15:42.000000000 +0100 @@ -837,7 +837,7 @@ return self.transport.qos_semantics_matches_spec(self.connection) def _extract_failover_opts(self): - conn_opts = {} + conn_opts = {'timeout': self.connect_timeout} transport_opts = self.transport_options if transport_opts: if 'max_retries' in transport_opts: diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/kombu-5.2.0/kombu/transport/SQS.py new/kombu-5.2.3/kombu/transport/SQS.py --- old/kombu-5.2.0/kombu/transport/SQS.py 2021-11-02 16:36:25.000000000 +0100 +++ new/kombu-5.2.3/kombu/transport/SQS.py 2021-12-19 06:12:32.000000000 +0100 @@ -59,8 +59,8 @@ 'backoff_policy': {1: 10, 2: 20, 3: 40, 4: 80, 5: 320, 6: 640}, # optional 'backoff_tasks': ['svc.tasks.tasks.task1'] # optional }, - 'queue-2': { - 'url': 'https://sqs.us-east-1.amazonaws.com/xxx/bbb', + 'queue-2.fifo': { + 'url': 'https://sqs.us-east-1.amazonaws.com/xxx/bbb.fifo', 'access_key_id': 'c', 'secret_access_key': 'd', 'backoff_policy': {1: 10, 2: 20, 3: 40, 4: 80, 5: 320, 6: 640}, # optional @@ -71,6 +71,9 @@ 'sts_token_timeout': 900 # optional } +Note that FIFO and standard queues must be named accordingly (the name of +a FIFO queue must end with the .fifo suffix). + backoff_policy & backoff_tasks are optional arguments. These arguments automatically change the message visibility timeout, in order to have different times between specific task retries. This would apply after @@ -167,6 +170,10 @@ """Predefined queues are being used and an undefined queue was used.""" +class InvalidQueueException(Exception): + """Predefined queues are being used and configuration is not valid.""" + + class QoS(virtual.QoS): """Quality of Service guarantees implementation for SQS.""" @@ -237,6 +244,7 @@ if boto3 is None: raise ImportError('boto3 is not installed') super().__init__(*args, **kwargs) + self._validate_predifined_queues() # SQS blows up if you try to create a new queue when one already # exists but with a different visibility_timeout. This prepopulates @@ -246,6 +254,26 @@ self.hub = kwargs.get('hub') or get_event_loop() + def _validate_predifined_queues(self): + """Check that standard and FIFO queues are named properly. + + AWS requires FIFO queues to have a name + that ends with the .fifo suffix. + """ + for queue_name, q in self.predefined_queues.items(): + fifo_url = q['url'].endswith('.fifo') + fifo_name = queue_name.endswith('.fifo') + if fifo_url and not fifo_name: + raise InvalidQueueException( + "Queue with url '{}' must have a name " + "ending with .fifo".format(q['url']) + ) + elif not fifo_url and fifo_name: + raise InvalidQueueException( + "Queue with name '{}' is not a FIFO queue: " + "'{}'".format(queue_name, q['url']) + ) + def _update_queue_cache(self, queue_name_prefix): if self.predefined_queues: for queue_name, q in self.predefined_queues.items(): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/kombu-5.2.0/kombu/transport/redis.py new/kombu-5.2.3/kombu/transport/redis.py --- old/kombu-5.2.0/kombu/transport/redis.py 2021-11-02 16:36:25.000000000 +0100 +++ new/kombu-5.2.3/kombu/transport/redis.py 2021-12-29 05:15:42.000000000 +0100 @@ -353,13 +353,17 @@ pass def restore_by_tag(self, tag, client=None, leftmost=False): - with self.channel.conn_or_acquire(client) as client: - with client.pipeline() as pipe: - p, _, _ = self._remove_from_indices( - tag, pipe.hget(self.unacked_key, tag)).execute() + + def restore_transaction(pipe): + p = pipe.hget(self.unacked_key, tag) + pipe.multi() + self._remove_from_indices(tag, pipe) if p: M, EX, RK = loads(bytes_to_str(p)) # json is unicode - self.channel._do_restore_message(M, EX, RK, client, leftmost) + self.channel._do_restore_message(M, EX, RK, pipe, leftmost) + + with self.channel.conn_or_acquire(client) as client: + client.transaction(restore_transaction, self.unacked_key) @cached_property def unacked_key(self): @@ -709,32 +713,34 @@ self.connection.cycle._on_connection_disconnect(connection) def _do_restore_message(self, payload, exchange, routing_key, - client=None, leftmost=False): - with self.conn_or_acquire(client) as client: + pipe, leftmost=False): + try: try: - try: - payload['headers']['redelivered'] = True - except KeyError: - pass - for queue in self._lookup(exchange, routing_key): - (client.lpush if leftmost else client.rpush)( - queue, dumps(payload), - ) - except Exception: - crit('Could not restore message: %r', payload, exc_info=True) + payload['headers']['redelivered'] = True + except KeyError: + pass + for queue in self._lookup(exchange, routing_key): + (pipe.lpush if leftmost else pipe.rpush)( + queue, dumps(payload), + ) + except Exception: + crit('Could not restore message: %r', payload, exc_info=True) def _restore(self, message, leftmost=False): if not self.ack_emulation: return super()._restore(message) tag = message.delivery_tag - with self.conn_or_acquire() as client: - with client.pipeline() as pipe: - P, _ = pipe.hget(self.unacked_key, tag) \ - .hdel(self.unacked_key, tag) \ - .execute() + + def restore_transaction(pipe): + P = pipe.hget(self.unacked_key, tag) + pipe.multi() + pipe.hdel(self.unacked_key, tag) if P: M, EX, RK = loads(bytes_to_str(P)) # json is unicode - self._do_restore_message(M, EX, RK, client, leftmost) + self._do_restore_message(M, EX, RK, pipe, leftmost) + + with self.conn_or_acquire() as client: + client.transaction(restore_transaction, self.unacked_key) def _restore_at_beginning(self, message): return self._restore(message, leftmost=True) @@ -1231,6 +1237,14 @@ def _on_disconnect(connection): if connection._sock: loop.remove(connection._sock) + + # must have started polling or this will break reconnection + if cycle.fds: + # stop polling in the event loop + try: + loop.on_tick.remove(on_poll_start) + except KeyError: + pass cycle._on_connection_disconnect = _on_disconnect def on_poll_start(): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/kombu-5.2.0/kombu/transport/virtual/base.py new/kombu-5.2.3/kombu/transport/virtual/base.py --- old/kombu-5.2.0/kombu/transport/virtual/base.py 2021-11-02 16:36:25.000000000 +0100 +++ new/kombu-5.2.3/kombu/transport/virtual/base.py 2021-12-29 05:15:42.000000000 +0100 @@ -462,14 +462,7 @@ typ: cls(self) for typ, cls in self.exchange_types.items() } - try: - self.channel_id = self.connection._avail_channel_ids.pop() - except IndexError: - raise ResourceError( - 'No free channel ids, current={}, channel_max={}'.format( - len(self.connection.channels), - self.connection.channel_max), (20, 10), - ) + self.channel_id = self._get_free_channel_id() topts = self.connection.client.transport_options for opt_name in self.from_transport_options: @@ -844,6 +837,22 @@ return (self.max_priority - priority) if reverse else priority + def _get_free_channel_id(self): + # Cast to a set for fast lookups, and keep stored as an array + # for lower memory usage. + used_channel_ids = set(self.connection._used_channel_ids) + + for channel_id in range(1, self.connection.channel_max + 1): + if channel_id not in used_channel_ids: + self.connection._used_channel_ids.append(channel_id) + return channel_id + + raise ResourceError( + 'No free channel ids, current={}, channel_max={}'.format( + len(self.connection.channels), + self.connection.channel_max), (20, 10), + ) + class Management(base.Management): """Base class for the AMQP management API.""" @@ -907,9 +916,7 @@ polling_interval = client.transport_options.get('polling_interval') if polling_interval is not None: self.polling_interval = polling_interval - self._avail_channel_ids = array( - ARRAY_TYPE_H, range(self.channel_max, 0, -1), - ) + self._used_channel_ids = array(ARRAY_TYPE_H) def create_channel(self, connection): try: @@ -921,7 +928,11 @@ def close_channel(self, channel): try: - self._avail_channel_ids.append(channel.channel_id) + try: + self._used_channel_ids.remove(channel.channel_id) + except ValueError: + # channel id already removed + pass try: self.channels.remove(channel) except ValueError: diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/kombu-5.2.0/kombu.egg-info/PKG-INFO new/kombu-5.2.3/kombu.egg-info/PKG-INFO --- old/kombu-5.2.0/kombu.egg-info/PKG-INFO 2021-11-02 17:37:47.000000000 +0100 +++ new/kombu-5.2.3/kombu.egg-info/PKG-INFO 2021-12-29 05:59:21.000000000 +0100 @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: kombu -Version: 5.2.0 +Version: 5.2.3 Summary: Messaging library for Python. Home-page: https://kombu.readthedocs.io Author: Ask Solem @@ -17,6 +17,7 @@ Classifier: Programming Language :: Python :: 3.7 Classifier: Programming Language :: Python :: 3.8 Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 Classifier: Programming Language :: Python :: Implementation :: CPython Classifier: Programming Language :: Python :: Implementation :: PyPy Classifier: Intended Audience :: Developers diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/kombu-5.2.0/kombu.egg-info/requires.txt new/kombu-5.2.3/kombu.egg-info/requires.txt --- old/kombu-5.2.0/kombu.egg-info/requires.txt 2021-11-02 17:37:47.000000000 +0100 +++ new/kombu-5.2.3/kombu.egg-info/requires.txt 2021-12-29 05:59:21.000000000 +0100 @@ -1,4 +1,4 @@ -amqp<6.0.0,>=5.0.6 +amqp<6.0.0,>=5.0.9 vine [:python_version < "3.8"] @@ -15,10 +15,10 @@ python-consul>=0.6.0 [librabbitmq] -librabbitmq>=1.5.2 +librabbitmq>=2.0.0 [mongodb] -pymongo>=3.3.0 +pymongo<3.12.1,>=3.3.0 [msgpack] msgpack @@ -31,7 +31,7 @@ qpid-tools>=0.26 [redis] -redis>=3.3.11 +redis!=4.0.0,!=4.0.1,>=3.4.1 [slmq] softlayer_messaging>=1.0.3 @@ -40,9 +40,9 @@ sqlalchemy [sqs] -boto3>=1.4.4 -pycurl==7.43.0.2 -urllib3<1.26 +boto3>=1.9.12 +pycurl~=7.44.1 +urllib3>=1.26.7 [yaml] PyYAML>=3.10 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/kombu-5.2.0/requirements/default.txt new/kombu-5.2.3/requirements/default.txt --- old/kombu-5.2.0/requirements/default.txt 2021-11-02 16:36:25.000000000 +0100 +++ new/kombu-5.2.3/requirements/default.txt 2021-12-29 05:15:42.000000000 +0100 @@ -1,4 +1,4 @@ importlib-metadata>=0.18; python_version<"3.8" cached_property; python_version<"3.8" -amqp>=5.0.6,<6.0.0 +amqp>=5.0.9,<6.0.0 vine diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/kombu-5.2.0/requirements/extras/librabbitmq.txt new/kombu-5.2.3/requirements/extras/librabbitmq.txt --- old/kombu-5.2.0/requirements/extras/librabbitmq.txt 2021-11-02 16:36:25.000000000 +0100 +++ new/kombu-5.2.3/requirements/extras/librabbitmq.txt 2021-11-08 01:17:43.000000000 +0100 @@ -1 +1 @@ -librabbitmq>=1.5.2 +librabbitmq>=2.0.0 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/kombu-5.2.0/requirements/extras/mongodb.txt new/kombu-5.2.3/requirements/extras/mongodb.txt --- old/kombu-5.2.0/requirements/extras/mongodb.txt 2021-11-02 16:36:25.000000000 +0100 +++ new/kombu-5.2.3/requirements/extras/mongodb.txt 2021-11-08 01:17:43.000000000 +0100 @@ -1 +1 @@ -pymongo>=3.3.0 +pymongo>=3.3.0,<3.12.1 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/kombu-5.2.0/requirements/extras/redis.txt new/kombu-5.2.3/requirements/extras/redis.txt --- old/kombu-5.2.0/requirements/extras/redis.txt 2021-11-02 16:36:25.000000000 +0100 +++ new/kombu-5.2.3/requirements/extras/redis.txt 2021-12-19 06:12:32.000000000 +0100 @@ -1 +1 @@ -redis>=3.3.11 +redis>=3.4.1,!=4.0.0,!=4.0.1 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/kombu-5.2.0/requirements/extras/sqs.txt new/kombu-5.2.3/requirements/extras/sqs.txt --- old/kombu-5.2.0/requirements/extras/sqs.txt 2021-11-02 16:36:25.000000000 +0100 +++ new/kombu-5.2.3/requirements/extras/sqs.txt 2021-11-08 01:17:43.000000000 +0100 @@ -1,3 +1,3 @@ -boto3>=1.4.4 -pycurl==7.43.0.2 # Latest build with wheels provided -urllib3<1.26 # Unittests are faiing with urllib3>=1.28 +boto3>=1.9.12 +pycurl~=7.44.1 # Latest build with wheels provided +urllib3>=1.26.7 # Unittests are faiing with urllib3>=1.28 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/kombu-5.2.0/requirements/pkgutils.txt new/kombu-5.2.3/requirements/pkgutils.txt --- old/kombu-5.2.0/requirements/pkgutils.txt 2021-11-02 16:36:25.000000000 +0100 +++ new/kombu-5.2.3/requirements/pkgutils.txt 2021-12-22 04:35:30.000000000 +0100 @@ -1,4 +1,4 @@ -setuptools>=20.6.7 +setuptools>=59.1.1,<59.7.0 wheel>=0.29.0 flake8>=2.5.4 tox>=2.3.1 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/kombu-5.2.0/requirements/test-ci.txt new/kombu-5.2.3/requirements/test-ci.txt --- old/kombu-5.2.0/requirements/test-ci.txt 2021-11-02 16:36:25.000000000 +0100 +++ new/kombu-5.2.3/requirements/test-ci.txt 2021-12-19 06:12:32.000000000 +0100 @@ -1,5 +1,4 @@ pytest-cov -pytest-travis-fold codecov -r extras/redis.txt -r extras/yaml.txt diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/kombu-5.2.0/setup.py new/kombu-5.2.3/setup.py --- old/kombu-5.2.0/setup.py 2021-11-02 16:36:25.000000000 +0100 +++ new/kombu-5.2.3/setup.py 2021-11-08 00:46:52.000000000 +0100 @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import os import re import sys @@ -141,6 +141,7 @@ 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', 'Intended Audience :: Developers', diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/kombu-5.2.0/t/integration/common.py new/kombu-5.2.3/t/integration/common.py --- old/kombu-5.2.0/t/integration/common.py 2021-11-02 16:36:25.000000000 +0100 +++ new/kombu-5.2.3/t/integration/common.py 2021-12-19 06:12:32.000000000 +0100 @@ -398,6 +398,47 @@ assert msg.payload == data +class BaseMessage: + + def test_ack(self, connection): + with connection as conn: + with closing(conn.SimpleQueue('test_ack')) as queue: + queue.put({'Hello': 'World'}, headers={'k1': 'v1'}) + message = queue.get_nowait() + message.ack() + with pytest.raises(queue.Empty): + queue.get_nowait() + + def test_reject_no_requeue(self, connection): + with connection as conn: + with closing(conn.SimpleQueue('test_reject_no_requeue')) as queue: + queue.put({'Hello': 'World'}, headers={'k1': 'v1'}) + message = queue.get_nowait() + message.reject(requeue=False) + with pytest.raises(queue.Empty): + queue.get_nowait() + + def test_reject_requeue(self, connection): + with connection as conn: + with closing(conn.SimpleQueue('test_reject_requeue')) as queue: + queue.put({'Hello': 'World'}, headers={'k1': 'v1'}) + message = queue.get_nowait() + message.reject(requeue=True) + message2 = queue.get_nowait() + assert message.body == message2.body + message2.ack() + + def test_requeue(self, connection): + with connection as conn: + with closing(conn.SimpleQueue('test_requeue')) as queue: + queue.put({'Hello': 'World'}, headers={'k1': 'v1'}) + message = queue.get_nowait() + message.requeue() + message2 = queue.get_nowait() + assert message.body == message2.body + message2.ack() + + class BaseFailover(BasicFunctionality): def test_connect(self, failover_connection): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/kombu-5.2.0/t/integration/test_py_amqp.py new/kombu-5.2.3/t/integration/test_py_amqp.py --- old/kombu-5.2.0/t/integration/test_py_amqp.py 2021-11-02 16:36:25.000000000 +0100 +++ new/kombu-5.2.3/t/integration/test_py_amqp.py 2021-12-19 06:12:32.000000000 +0100 @@ -4,8 +4,8 @@ import kombu -from .common import (BaseExchangeTypes, BaseFailover, BasePriority, - BaseTimeToLive, BasicFunctionality) +from .common import (BaseExchangeTypes, BaseFailover, BaseMessage, + BasePriority, BaseTimeToLive, BasicFunctionality) def get_connection(hostname, port, vhost): @@ -73,3 +73,9 @@ @pytest.mark.flaky(reruns=5, reruns_delay=2) class test_PyAMQPFailover(BaseFailover): pass + + +@pytest.mark.env('py-amqp') +@pytest.mark.flaky(reruns=5, reruns_delay=2) +class test_PyAMQPMessage(BaseMessage): + pass diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/kombu-5.2.0/t/integration/test_redis.py new/kombu-5.2.3/t/integration/test_redis.py --- old/kombu-5.2.0/t/integration/test_redis.py 2021-11-02 16:36:25.000000000 +0100 +++ new/kombu-5.2.3/t/integration/test_redis.py 2021-12-19 06:12:32.000000000 +0100 @@ -2,16 +2,24 @@ from time import sleep import pytest +import redis import kombu -from .common import BaseExchangeTypes, BasePriority, BasicFunctionality +from .common import (BaseExchangeTypes, BaseMessage, BasePriority, + BasicFunctionality) def get_connection( - hostname, port, vhost, transport_options=None): + hostname, port, vhost, user_name=None, password=None, + transport_options=None): + + credentials = f'{user_name}:{password}@' if user_name else '' + return kombu.Connection( - f'redis://{hostname}:{port}', transport_options=transport_options) + f'redis://{credentials}{hostname}:{port}', + transport_options=transport_options + ) @pytest.fixture(params=[None, {'global_keyprefix': '_prefixed_'}]) @@ -33,6 +41,19 @@ @pytest.mark.env('redis') +def test_failed_credentials(): + """Tests denied connection when wrong credentials were provided""" + with pytest.raises(redis.exceptions.ResponseError): + get_connection( + hostname=os.environ.get('REDIS_HOST', 'localhost'), + port=os.environ.get('REDIS_6379_TCP', '6379'), + vhost=None, + user_name='wrong_redis_user', + password='wrong_redis_password' + ).connect() + + +@pytest.mark.env('redis') @pytest.mark.flaky(reruns=5, reruns_delay=2) class test_RedisBasicFunctionality(BasicFunctionality): pass @@ -100,3 +121,9 @@ assert received_messages[0] == {'msg': 'second'} assert received_messages[1] == {'msg': 'first'} assert received_messages[2] == {'msg': 'third'} + + +@pytest.mark.env('redis') +@pytest.mark.flaky(reruns=5, reruns_delay=2) +class test_RedisMessage(BaseMessage): + pass diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/kombu-5.2.0/t/unit/test_connection.py new/kombu-5.2.3/t/unit/test_connection.py --- old/kombu-5.2.0/t/unit/test_connection.py 2021-11-02 16:36:25.000000000 +0100 +++ new/kombu-5.2.3/t/unit/test_connection.py 2021-12-29 05:15:42.000000000 +0100 @@ -587,6 +587,16 @@ conn = Connection('example.com;example.com;') assert conn.as_uri() == 'amqp://guest:**@example.com:5672//' + def test_connection_respect_its_timeout(self): + invalid_port = 1222 + with Connection( + f'amqp://guest:guest@localhost:{invalid_port}//', + transport_options={'max_retries': 2}, + connect_timeout=1 + ) as conn: + with pytest.raises(OperationalError): + conn.default_channel + class test_Connection_with_transport_options: diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/kombu-5.2.0/t/unit/transport/test_SQS.py new/kombu-5.2.3/t/unit/transport/test_SQS.py --- old/kombu-5.2.0/t/unit/transport/test_SQS.py 2021-11-02 16:36:25.000000000 +0100 +++ new/kombu-5.2.3/t/unit/transport/test_SQS.py 2021-12-19 06:12:32.000000000 +0100 @@ -38,6 +38,11 @@ 'access_key_id': 'c', 'secret_access_key': 'd', }, + 'queue-3.fifo': { + 'url': 'https://sqs.us-east-1.amazonaws.com/xxx/queue-3.fifo', + 'access_key_id': 'e', + 'secret_access_key': 'f', + } } @@ -151,6 +156,7 @@ predefined_queues_sqs_conn_mocks = { 'queue-1': SQSClientMock(QueueName='queue-1'), 'queue-2': SQSClientMock(QueueName='queue-2'), + 'queue-3.fifo': SQSClientMock(QueueName='queue-3.fifo') } def mock_sqs(): @@ -738,6 +744,53 @@ QueueUrl='https://sqs.us-east-1.amazonaws.com/xxx/queue-1', ReceiptHandle='test_message_id', VisibilityTimeout=20) + def test_predefined_queues_put_to_fifo_queue(self): + connection = Connection(transport=SQS.Transport, transport_options={ + 'predefined_queues': example_predefined_queues, + }) + channel = connection.channel() + + queue_name = 'queue-3.fifo' + + exchange = Exchange('test_SQS', type='direct') + p = messaging.Producer(channel, exchange, routing_key=queue_name) + + queue = Queue(queue_name, exchange, queue_name) + queue(channel).declare() + + channel.sqs = Mock() + sqs_queue_mock = Mock() + channel.sqs.return_value = sqs_queue_mock + p.publish('message') + + sqs_queue_mock.send_message.assert_called_once() + assert 'MessageGroupId' in sqs_queue_mock.send_message.call_args[1] + assert 'MessageDeduplicationId' in \ + sqs_queue_mock.send_message.call_args[1] + + @pytest.mark.parametrize('predefined_queues', ( + { + 'invalid-fifo-queue-name': { + 'url': 'https://sqs.us-east-1.amazonaws.com/xxx/queue.fifo', + 'access_key_id': 'a', + 'secret_access_key': 'b' + } + }, + { + 'standard-queue.fifo': { + 'url': 'https://sqs.us-east-1.amazonaws.com/xxx/queue', + 'access_key_id': 'a', + 'secret_access_key': 'b' + } + } + )) + def test_predefined_queues_invalid_configuration(self, predefined_queues): + connection = Connection(transport=SQS.Transport, transport_options={ + 'predefined_queues': predefined_queues, + }) + with pytest.raises(SQS.InvalidQueueException): + connection.channel() + def test_sts_new_session(self): # Arrange connection = Connection(transport=SQS.Transport, transport_options={ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/kombu-5.2.0/t/unit/transport/test_consul.py new/kombu-5.2.3/t/unit/transport/test_consul.py --- old/kombu-5.2.0/t/unit/transport/test_consul.py 2021-11-02 16:36:25.000000000 +0100 +++ new/kombu-5.2.3/t/unit/transport/test_consul.py 2021-12-29 05:15:42.000000000 +0100 @@ -1,3 +1,4 @@ +from array import array from queue import Empty from unittest.mock import Mock @@ -12,6 +13,8 @@ def setup(self): self.connection = Mock() + self.connection._used_channel_ids = array('H') + self.connection.channel_max = 65535 self.connection.client.transport_options = {} self.connection.client.port = 303 self.consul = self.patching('consul.Consul').return_value diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/kombu-5.2.0/t/unit/transport/test_redis.py new/kombu-5.2.3/t/unit/transport/test_redis.py --- old/kombu-5.2.0/t/unit/transport/test_redis.py 2021-11-02 16:36:25.000000000 +0100 +++ new/kombu-5.2.3/t/unit/transport/test_redis.py 2021-12-29 05:15:42.000000000 +0100 @@ -13,7 +13,6 @@ from kombu.transport import virtual from kombu.utils import eventio # patch poll from kombu.utils.json import dumps -from t.mocks import ContextMock def _redis_modules(): @@ -401,38 +400,56 @@ ) crit.assert_called() - def test_restore(self): + def test_restore_no_messages(self): message = Mock(name='message') + with patch('kombu.transport.redis.loads') as loads: - loads.return_value = 'M', 'EX', 'RK' + def transaction_handler(restore_transaction, unacked_key): + assert unacked_key == self.channel.unacked_key + pipe = Mock(name='pipe') + pipe.hget.return_value = None + + restore_transaction(pipe) + + pipe.multi.assert_called_once_with() + pipe.hdel.assert_called_once_with( + unacked_key, message.delivery_tag) + loads.assert_not_called() + client = self.channel._create_client = Mock(name='client') client = client() - client.pipeline = ContextMock() - restore = self.channel._do_restore_message = Mock( - name='_do_restore_message', - ) - pipe = client.pipeline.return_value - pipe_hget = Mock(name='pipe.hget') - pipe.hget.return_value = pipe_hget - pipe_hget_hdel = Mock(name='pipe.hget.hdel') - pipe_hget.hdel.return_value = pipe_hget_hdel - result = Mock(name='result') - pipe_hget_hdel.execute.return_value = None, None - + client.transaction.side_effect = transaction_handler self.channel._restore(message) - client.pipeline.assert_called_with() - unacked_key = self.channel.unacked_key - loads.assert_not_called() - - tag = message.delivery_tag - pipe.hget.assert_called_with(unacked_key, tag) - pipe_hget.hdel.assert_called_with(unacked_key, tag) - pipe_hget_hdel.execute.assert_called_with() + client.transaction.assert_called() + + def test_restore_messages(self): + message = Mock(name='message') + + with patch('kombu.transport.redis.loads') as loads: - pipe_hget_hdel.execute.return_value = result, None + def transaction_handler(restore_transaction, unacked_key): + assert unacked_key == self.channel.unacked_key + restore = self.channel._do_restore_message = Mock( + name='_do_restore_message', + ) + result = Mock(name='result') + loads.return_value = 'M', 'EX', 'RK' + pipe = Mock(name='pipe') + pipe.hget.return_value = result + + restore_transaction(pipe) + + loads.assert_called_with(result) + pipe.multi.assert_called_once_with() + pipe.hdel.assert_called_once_with( + unacked_key, message.delivery_tag) + loads.assert_called() + restore.assert_called_with('M', 'EX', 'RK', pipe, False) + + client = self.channel._create_client = Mock(name='client') + client = client() + client.transaction.side_effect = transaction_handler self.channel._restore(message) - loads.assert_called_with(result) - restore.assert_called_with('M', 'EX', 'RK', client, False) def test_qos_restore_visible(self): client = self.channel._create_client = Mock(name='client') @@ -837,6 +854,26 @@ call(13, transport.on_readable, 13), ]) + @pytest.mark.parametrize('fds', [{12: 'LISTEN', 13: 'BRPOP'}, {}]) + def test_register_with_event_loop__on_disconnect__loop_cleanup(self, fds): + """Ensure event loop polling stops on disconnect (if started).""" + transport = self.connection.transport + self.connection._sock = None + transport.cycle = Mock(name='cycle') + transport.cycle.fds = fds + conn = Mock(name='conn') + conn.client = Mock(name='client', transport_options={}) + loop = Mock(name='loop') + loop.on_tick = set() + redis.Transport.register_with_event_loop(transport, conn, loop) + assert len(loop.on_tick) == 1 + transport.cycle._on_connection_disconnect(self.connection) + if fds: + assert len(loop.on_tick) == 0 + else: + # on_tick shouldn't be cleared when polling hasn't started + assert len(loop.on_tick) == 1 + def test_configurable_health_check(self): transport = self.connection.transport transport.cycle = Mock(name='cycle') diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/kombu-5.2.0/t/unit/transport/virtual/test_base.py new/kombu-5.2.3/t/unit/transport/virtual/test_base.py --- old/kombu-5.2.0/t/unit/transport/virtual/test_base.py 2021-11-02 16:36:25.000000000 +0100 +++ new/kombu-5.2.3/t/unit/transport/virtual/test_base.py 2021-12-29 05:15:42.000000000 +0100 @@ -1,6 +1,7 @@ import io import socket import warnings +from array import array from time import monotonic from unittest.mock import MagicMock, Mock, patch @@ -178,13 +179,19 @@ if self.channel._qos is not None: self.channel._qos._on_collect.cancel() - def test_exceeds_channel_max(self): - c = client() - t = c.transport - avail = t._avail_channel_ids = Mock(name='_avail_channel_ids') - avail.pop.side_effect = IndexError() + def test_get_free_channel_id(self): + conn = client() + channel = conn.channel() + assert channel.channel_id == 1 + assert channel._get_free_channel_id() == 2 + + def test_get_free_channel_id__exceeds_channel_max(self): + conn = client() + conn.transport.channel_max = 2 + channel = conn.channel() + channel._get_free_channel_id() with pytest.raises(ResourceError): - virtual.Channel(t) + channel._get_free_channel_id() def test_exchange_bind_interface(self): with pytest.raises(NotImplementedError): @@ -577,6 +584,23 @@ del(c1) # so pyflakes doesn't complain del(c2) + def test_create_channel(self): + """Ensure create_channel can create channels successfully.""" + assert self.transport.channels == [] + created_channel = self.transport.create_channel(self.transport) + assert self.transport.channels == [created_channel] + + def test_close_channel(self): + """Ensure close_channel actually removes the channel and updates + _used_channel_ids. + """ + assert self.transport._used_channel_ids == array('H') + created_channel = self.transport.create_channel(self.transport) + assert self.transport._used_channel_ids == array('H', (1,)) + self.transport.close_channel(created_channel) + assert self.transport.channels == [] + assert self.transport._used_channel_ids == array('H') + def test_drain_channel(self): channel = self.transport.create_channel(self.transport) with pytest.raises(virtual.Empty):