Hello community, here is the log from the commit of package salt for openSUSE:Factory checked in at 2020-05-09 19:49:42 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/salt (Old) and /work/SRC/openSUSE:Factory/.salt.new.2738 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "salt" Sat May 9 19:49:42 2020 rev:104 rq:801830 version:3000.2 Changes: -------- --- /work/SRC/openSUSE:Factory/salt/salt.changes 2020-05-01 11:07:22.511067410 +0200 +++ /work/SRC/openSUSE:Factory/.salt.new.2738/salt.changes 2020-05-09 19:50:08.188552210 +0200 @@ -1,0 +2,22 @@ +Fri May 8 14:24:19 UTC 2020 - Jochen Breuer <[email protected]> + +- Python 3.8 compatibility changes +- msgpack support for version >= 1.0.0 (bsc#1171257) + +- Added: + * python3.8-compatibility-pr-s-235.patch + * msgpack-support-versions-1.0.0.patch + +------------------------------------------------------------------- +Thu May 7 15:36:38 UTC 2020 - Pablo Suárez Hernández <[email protected]> + +- Prevent sporious "salt-api" stuck processes when managing SSH minions + because of logging deadlock (bsc#1159284) +- Avoid segfault from "salt-api" under certain conditions of heavy load + managing SSH minions (bsc#1169604) + +- Added: + * prevent-logging-deadlock-on-salt-api-subprocesses-bs.patch + * make-lazyloader.__init__-call-to-_refresh_file_mappi.patch + +------------------------------------------------------------------- New: ---- make-lazyloader.__init__-call-to-_refresh_file_mappi.patch msgpack-support-versions-1.0.0.patch prevent-logging-deadlock-on-salt-api-subprocesses-bs.patch python3.8-compatibility-pr-s-235.patch ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ salt.spec ++++++ --- /var/tmp/diff_new_pack.oY6vHV/_old 2020-05-09 19:50:11.620559577 +0200 +++ /var/tmp/diff_new_pack.oY6vHV/_new 2020-05-09 19:50:11.624559586 +0200 @@ -300,6 +300,14 @@ Patch108: remove-vendored-backports-abc-from-requirements.patch # PATCH-FIX_OPENSUSE: https://github.com/openSUSE/salt/commit/a5ef829408685d9e65eaa24bba40d221adffaa95 Patch109: fix-typo-in-minion_runner-for-aesfuncs-exposed-metho.patch +# PATCH-FIX_UPSTREAM: https://github.com/saltstack/salt/pull/57119 +Patch110: make-lazyloader.__init__-call-to-_refresh_file_mappi.patch +# PATCH-FIX_UPSTREAM: https://github.com/saltstack/salt/pull/57123 +Patch111: prevent-logging-deadlock-on-salt-api-subprocesses-bs.patch +# PATCH-FIX_UPSTREAM: https://github.com/saltstack/salt/pull/57122 +Patch112: msgpack-support-versions-1.0.0.patch +# PATCH-FIX_OPENSUSE: https://github.com/openSUSE/salt/pull/235 +Patch113: python3.8-compatibility-pr-s-235.patch BuildRoot: %{_tmppath}/%{name}-%{version}-build BuildRequires: logrotate @@ -916,6 +924,10 @@ %patch107 -p1 %patch108 -p1 %patch109 -p1 +%patch110 -p1 +%patch111 -p1 +%patch112 -p1 +%patch113 -p1 %build %if 0%{?build_py2} ++++++ _lastrevision ++++++ --- /var/tmp/diff_new_pack.oY6vHV/_old 2020-05-09 19:50:11.688559723 +0200 +++ /var/tmp/diff_new_pack.oY6vHV/_new 2020-05-09 19:50:11.688559723 +0200 @@ -1 +1 @@ -52475d78d58db61df54486af13c5eb4055c2d12e \ No newline at end of file +4b7e8cbf000c33224e9cd7a7eb581954ca3dbbec \ No newline at end of file ++++++ make-lazyloader.__init__-call-to-_refresh_file_mappi.patch ++++++ >From 6af6a52165c70c3be7c8d339a3dd5e539f3c1772 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20Su=C3=A1rez=20Hern=C3=A1ndez?= <[email protected]> Date: Thu, 23 Apr 2020 09:54:53 +0100 Subject: [PATCH] Make LazyLoader.__init__ call to _refresh_file_mapping thread-safe (bsc#1169604) --- salt/loader.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/loader.py b/salt/loader.py index 5bd4773645c77a133701982e19d19739be00a38f..54dadb0b513dbaa4914b0d4b1d343dde709699ad 100644 --- a/salt/loader.py +++ b/salt/loader.py @@ -1251,7 +1251,8 @@ class LazyLoader(salt.utils.lazy.LazyDict): self.suffix_order.append(suffix) self._lock = threading.RLock() - self._refresh_file_mapping() + with self._lock: + self._refresh_file_mapping() super(LazyLoader, self).__init__() # late init the lazy loader # create all of the import namespaces -- 2.23.0 ++++++ msgpack-support-versions-1.0.0.patch ++++++ >From ef23c1d53e99e19e5b03658aa62b67cfef9adce5 Mon Sep 17 00:00:00 2001 From: Alberto Planas <[email protected]> Date: Thu, 7 May 2020 12:40:55 +0200 Subject: [PATCH] msgpack: support versions >= 1.0.0 A recent change in msgpack >= 1.0.0, update the default value for the parameter `raw` to False. This change breaks Salt for those versions. This patch add the parameter `raw=True` to all the unpack operations, restoring the old default. Fix bsc#1171257 (cherry picked from commit 1b3939fb01fc3405d8d222f118617220aecee092) --- salt/utils/msgpack.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/salt/utils/msgpack.py b/salt/utils/msgpack.py index 4b5a256513..027fe81a18 100644 --- a/salt/utils/msgpack.py +++ b/salt/utils/msgpack.py @@ -69,12 +69,26 @@ def _sanitize_msgpack_kwargs(kwargs): return kwargs +def _sanitize_msgpack_unpack_kwargs(kwargs): + """ + Clean up msgpack keyword arguments for unpack operations, based on + the version + https://github.com/msgpack/msgpack-python/blob/master/ChangeLog.rst + """ + assert isinstance(kwargs, dict) + if version >= (1, 0, 0) and kwargs.get("raw", None) is None: + log.info("adding `raw=True` argument to msgpack call") + kwargs["raw"] = True + + return _sanitize_msgpack_kwargs(kwargs) + + class Unpacker(msgpack.Unpacker): ''' Wraps the msgpack.Unpacker and removes non-relevant arguments ''' def __init__(self, *args, **kwargs): - msgpack.Unpacker.__init__(self, *args, **_sanitize_msgpack_kwargs(kwargs)) + msgpack.Unpacker.__init__(self, *args, **_sanitize_msgpack_unpack_kwargs(kwargs)) def pack(o, stream, **kwargs): @@ -113,7 +127,7 @@ def unpack(stream, **kwargs): By default, this function uses the msgpack module and falls back to msgpack_pure, if the msgpack is not available. ''' - return msgpack.unpack(stream, **_sanitize_msgpack_kwargs(kwargs)) + return msgpack.unpack(stream, **_sanitize_msgpack_unpack_kwargs(kwargs)) def unpackb(packed, **kwargs): @@ -125,7 +139,7 @@ def unpackb(packed, **kwargs): By default, this function uses the msgpack module and falls back to msgpack_pure. ''' - return msgpack.unpackb(packed, **_sanitize_msgpack_kwargs(kwargs)) + return msgpack.unpackb(packed, **_sanitize_msgpack_unpack_kwargs(kwargs)) # alias for compatibility to simplejson/marshal/pickle. -- 2.26.1 ++++++ prevent-logging-deadlock-on-salt-api-subprocesses-bs.patch ++++++ >From 217c5ba75b5de813ddf769e7eeebe4027c1c9a70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20Su=C3=A1rez=20Hern=C3=A1ndez?= <[email protected]> Date: Wed, 22 Jan 2020 08:19:55 +0000 Subject: [PATCH] Prevent logging deadlock on salt-api subprocesses (bsc#1159284) --- salt/_logging/impl.py | 60 ++++++++++++++++++++--------------- salt/client/ssh/__init__.py | 16 +++++++--- salt/client/ssh/client.py | 9 +++++- salt/client/ssh/wrapper/cp.py | 2 +- salt/loader.py | 2 +- salt/utils/lazy.py | 5 ++- 6 files changed, 61 insertions(+), 33 deletions(-) diff --git a/salt/_logging/impl.py b/salt/_logging/impl.py index 347259bcf506705e2ea1a24da030a7132eb8a527..fdfabf6d3b16619350107bd01f2c3a606fb93262 100644 --- a/salt/_logging/impl.py +++ b/salt/_logging/impl.py @@ -19,6 +19,7 @@ PROFILE = logging.PROFILE = 15 TRACE = logging.TRACE = 5 GARBAGE = logging.GARBAGE = 1 QUIET = logging.QUIET = 1000 +DEBUG = logging.DEBUG = 10 # Import Salt libs from salt._logging.handlers import StreamHandler @@ -187,11 +188,11 @@ class SaltLoggingClass(six.with_metaclass(LoggingMixinMeta, LOGGING_LOGGER_CLASS ''' instance = super(SaltLoggingClass, cls).__new__(cls) - try: - max_logger_length = len(max( - list(logging.Logger.manager.loggerDict), key=len - )) - for handler in logging.root.handlers: + max_logger_length = len(max( + list(logging.Logger.manager.loggerDict), key=len + )) + for handler in logging.root.handlers: + try: if handler in (LOGGING_NULL_HANDLER, LOGGING_STORE_HANDLER, LOGGING_TEMP_HANDLER): @@ -210,18 +211,15 @@ class SaltLoggingClass(six.with_metaclass(LoggingMixinMeta, LOGGING_LOGGER_CLASS match = MODNAME_PATTERN.search(fmt) if not match: # Not matched. Release handler and return. - handler.release() return instance if 'digits' not in match.groupdict(): # No digits group. Release handler and return. - handler.release() return instance digits = match.group('digits') if not digits or not (digits and digits.isdigit()): # No valid digits. Release handler and return. - handler.release() return instance if int(digits) < max_logger_length: @@ -233,9 +231,14 @@ class SaltLoggingClass(six.with_metaclass(LoggingMixinMeta, LOGGING_LOGGER_CLASS ) handler.setFormatter(formatter) handler.release() - except ValueError: - # There are no registered loggers yet - pass + except ValueError: + # There are no registered loggers yet + pass + finally: + try: + handler.release() + except: + pass return instance def _log(self, level, msg, args, exc_info=None, @@ -278,20 +281,26 @@ class SaltLoggingClass(six.with_metaclass(LoggingMixinMeta, LOGGING_LOGGER_CLASS else: extra['exc_info_on_loglevel'] = exc_info_on_loglevel - if sys.version_info < (3,): - LOGGING_LOGGER_CLASS._log( - self, level, msg, args, exc_info=exc_info, extra=extra - ) - elif sys.version_info < (3, 8): - LOGGING_LOGGER_CLASS._log( - self, level, msg, args, exc_info=exc_info, extra=extra, - stack_info=stack_info - ) - else: - LOGGING_LOGGER_CLASS._log( - self, level, msg, args, exc_info=exc_info, extra=extra, - stack_info=stack_info, stacklevel=stacklevel - ) + try: + logging._acquireLock() + if sys.version_info < (3,): + LOGGING_LOGGER_CLASS._log( + self, level, msg, args, exc_info=exc_info, extra=extra + ) + elif sys.version_info < (3, 8): + LOGGING_LOGGER_CLASS._log( + self, level, msg, args, exc_info=exc_info, extra=extra, + stack_info=stack_info + ) + else: + LOGGING_LOGGER_CLASS._log( + self, level, msg, args, exc_info=exc_info, extra=extra, + stack_info=stack_info, stacklevel=stacklevel + ) + except: + pass + finally: + logging._releaseLock() def makeRecord(self, name, level, fn, lno, msg, args, exc_info, func=None, extra=None, sinfo=None): @@ -393,6 +402,7 @@ if logging.getLoggerClass() is not SaltLoggingClass: logging.addLevelName(PROFILE, 'PROFILE') logging.addLevelName(TRACE, 'TRACE') logging.addLevelName(GARBAGE, 'GARBAGE') + logging.addLevelName(DEBUG, 'DEBUG') # ----- REMOVE ON REFACTORING COMPLETE --------------------------------------------------------------------------> if not logging.root.handlers: diff --git a/salt/client/ssh/__init__.py b/salt/client/ssh/__init__.py index d9e91b0f50bfaa76d519fcaa4bdc868bce80f554..e8aad093e0f6df32faa16a838f1db2c6746e1b8e 100644 --- a/salt/client/ssh/__init__.py +++ b/salt/client/ssh/__init__.py @@ -520,7 +520,9 @@ class SSH(object): mine=mine, **target) ret = {'id': single.id} + logging._acquireLock() stdout, stderr, retcode = single.run() + logging._releaseLock() # This job is done, yield try: data = salt.utils.json.find_json(stdout) @@ -586,10 +588,16 @@ class SSH(object): self.targets[host], mine, ) - routine = Process( - target=self.handle_routine, - args=args) - routine.start() + try: + logging._acquireLock() + routine = Process( + target=self.handle_routine, + args=args) + routine.start() + except: + pass + finally: + logging._releaseLock() running[host] = {'thread': routine} continue ret = {} diff --git a/salt/client/ssh/client.py b/salt/client/ssh/client.py index e8e634ca12d85f1e1a9e047f43eac8c041cc5666..d4a89cf4fbbde5282597dc6b82c66dde4288edf1 100644 --- a/salt/client/ssh/client.py +++ b/salt/client/ssh/client.py @@ -6,6 +6,8 @@ import os import copy import logging import random +import time +import multiprocessing # Import Salt libs import salt.config @@ -15,6 +17,7 @@ from salt.exceptions import SaltClientError # Temporary log = logging.getLogger(__name__) +_LOCK = multiprocessing.Lock() class SSHClient(object): ''' @@ -61,7 +64,11 @@ class SSHClient(object): opts['selected_target_option'] = tgt_type opts['tgt'] = tgt opts['arg'] = arg - return salt.client.ssh.SSH(opts) + _LOCK.acquire() + ret = salt.client.ssh.SSH(opts) + time.sleep(0.01) + _LOCK.release() + return ret def cmd_iter( self, diff --git a/salt/client/ssh/wrapper/cp.py b/salt/client/ssh/wrapper/cp.py index 894e62f94c87ae5b68c1f82fc3e80ec8f25ac118..9bf0c150a071b4bfa780fe0293e3d8e93ab8e6ef 100644 --- a/salt/client/ssh/wrapper/cp.py +++ b/salt/client/ssh/wrapper/cp.py @@ -4,7 +4,7 @@ Wrap the cp module allowing for managed ssh file transfers ''' # Import Python libs from __future__ import absolute_import, print_function -import logging +import salt.log.setup as logging import os # Import salt libs diff --git a/salt/loader.py b/salt/loader.py index 54dadb0b513dbaa4914b0d4b1d343dde709699ad..b824a70a0cc40128f3271f70f676f1551194236c 100644 --- a/salt/loader.py +++ b/salt/loader.py @@ -11,7 +11,7 @@ import os import re import sys import time -import logging +import salt.log.setup as logging import inspect import tempfile import functools diff --git a/salt/utils/lazy.py b/salt/utils/lazy.py index 3cd6489d2d8c50ec4e6eb70c50407f1084db377b..bb4b38e1a3cfa05945cd438fc9d30e7c47c3391b 100644 --- a/salt/utils/lazy.py +++ b/salt/utils/lazy.py @@ -5,7 +5,8 @@ Lazily-evaluated data structures, primarily used by Salt's loader # Import Python Libs from __future__ import absolute_import, unicode_literals -import logging +import salt.log.setup as logging +import time import salt.exceptions try: @@ -102,9 +103,11 @@ class LazyDict(MutableMapping): # load the item if self._load(key): log.debug('LazyLoaded %s', key) + time.sleep(0.0001) return self._dict[key] else: log.debug('Could not LazyLoad %s: %s', key, self.missing_fun_string(key)) + time.sleep(0.0001) raise KeyError(key) else: return self._dict[key] -- 2.23.0 ++++++ python3.8-compatibility-pr-s-235.patch ++++++ ++++ 1995 lines (skipped)
