Hello community, here is the log from the commit of package salt for openSUSE:Factory checked in at 2018-05-16 18:43:20 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/salt (Old) and /work/SRC/openSUSE:Factory/.salt.new (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "salt" Wed May 16 18:43:20 2018 rev:77 rq:609095 version:2018.3.0 Changes: -------- --- /work/SRC/openSUSE:Factory/salt/salt.changes 2018-05-10 15:50:34.214690205 +0200 +++ /work/SRC/openSUSE:Factory/.salt.new/salt.changes 2018-05-16 18:44:50.932235527 +0200 @@ -1,0 +2,26 @@ +Wed May 16 10:57:17 UTC 2018 - Jochen Breuer <[email protected]> + +- No more AWS EC2 rate limitations in salt-cloud (bsc#1088888) +- MySQL returner now also allows to use Unix sockets (bsc#1091371) + +- Added: + * fix-for-ec2-rate-limit-failures.patch + * enable-passing-a-unix_socket-for-mysql-returners-bsc.patch + +------------------------------------------------------------------- +Thu May 10 09:12:24 UTC 2018 - Pablo Suárez Hernández <[email protected]> + +- Do not override jid on returners, only sending back to master (bsc#1092373) +- Fixes for salt-ssh: + - Option --extra-filerefs doesn't add all files to the state + archive + - Pillar completely overwritten (not merged) when doing + module.run + state.apply with pillar in kwargs +- Remove minion/thin/version if exists to force thin regeneration (bsc#1092161) + +- Added: + * extra-filerefs-include-files-even-if-no-refs-in-stat.patch + * do-not-override-jid-on-returners-only-sending-back-t.patch + * option-to-merge-current-pillar-with-opts-pillar-duri.patch + +------------------------------------------------------------------- New: ---- do-not-override-jid-on-returners-only-sending-back-t.patch enable-passing-a-unix_socket-for-mysql-returners-bsc.patch extra-filerefs-include-files-even-if-no-refs-in-stat.patch fix-for-ec2-rate-limit-failures.patch option-to-merge-current-pillar-with-opts-pillar-duri.patch ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ salt.spec ++++++ --- /var/tmp/diff_new_pack.nO75HZ/_old 2018-05-16 18:44:52.148191585 +0200 +++ /var/tmp/diff_new_pack.nO75HZ/_new 2018-05-16 18:44:52.148191585 +0200 @@ -98,7 +98,16 @@ Patch19: initialize-__context__-retcode-for-functions-handled.patch # PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/47232 Patch20: fixed-usage-of-ipaddress.patch - +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/47497 +Patch21: extra-filerefs-include-files-even-if-no-refs-in-stat.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/47504 +Patch22: option-to-merge-current-pillar-with-opts-pillar-duri.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/47471 +Patch23: do-not-override-jid-on-returners-only-sending-back-t.patch +# PATCH-FIX_OPENSUSE bsc#1091371 +Patch24: enable-passing-a-unix_socket-for-mysql-returners-bsc.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/47211 +Patch25: fix-for-ec2-rate-limit-failures.patch # BuildRoot: %{_tmppath}/%{name}-%{version}-build BuildRoot: %{_tmppath}/%{name}-%{version}-build @@ -577,6 +586,11 @@ %patch18 -p1 %patch19 -p1 %patch20 -p1 +%patch21 -p1 +%patch22 -p1 +%patch23 -p1 +%patch24 -p1 +%patch25 -p1 %build %if 0%{?build_py2} @@ -1071,11 +1085,13 @@ %posttrans -n python2-salt # force re-generate a new thin.tgz rm -f %{_localstatedir}/cache/salt/master/thin/version +rm -f %{_localstatedir}/cache/salt/minion/thin/version %if 0%{?build_py3} %posttrans -n python3-salt # force re-generate a new thin.tgz rm -f %{_localstatedir}/cache/salt/master/thin/version +rm -f %{_localstatedir}/cache/salt/minion/thin/version %endif %files api ++++++ _lastrevision ++++++ --- /var/tmp/diff_new_pack.nO75HZ/_old 2018-05-16 18:44:52.204189561 +0200 +++ /var/tmp/diff_new_pack.nO75HZ/_new 2018-05-16 18:44:52.204189561 +0200 @@ -1 +1 @@ -191f12d1f56ce5b5645ae420da0392941bcb2de4 \ No newline at end of file +09133c4494ccc374b1ec76de50772e1efee3f00e \ No newline at end of file ++++++ do-not-override-jid-on-returners-only-sending-back-t.patch ++++++ >From c49a9aca3519d1baef2f9d82963a6e80eabb26d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20Su=C3=A1rez=20Hern=C3=A1ndez?= <[email protected]> Date: Fri, 4 May 2018 09:34:13 +0100 Subject: [PATCH] Do not override jid on returners, only sending back to master (bsc#1092373) --- salt/utils/schedule.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/salt/utils/schedule.py b/salt/utils/schedule.py index 6cb3ce0ef8..e81ae143bd 100644 --- a/salt/utils/schedule.py +++ b/salt/utils/schedule.py @@ -753,11 +753,13 @@ class Schedule(object): else: # Send back to master so the job is included in the job list mret = ret.copy() - mret['jid'] = 'req' - if data.get('return_job') == 'nocache': - # overwrite 'req' to signal to master that - # this job shouldn't be stored - mret['jid'] = 'nocache' + # No returners defined, so we're only sending back to the master + if not data_returner and not self.schedule_returner: + mret['jid'] = 'req' + if data.get('return_job') == 'nocache': + # overwrite 'req' to signal to master that + # this job shouldn't be stored + mret['jid'] = 'nocache' load = {'cmd': '_return', 'id': self.opts['id']} for key, value in six.iteritems(mret): load[key] = value -- 2.15.1 ++++++ enable-passing-a-unix_socket-for-mysql-returners-bsc.patch ++++++ >From cdfb19b6c2801a7d5a883492a0ccc57c803dcd7f Mon Sep 17 00:00:00 2001 From: Maximilian Meister <[email protected]> Date: Thu, 3 May 2018 15:52:23 +0200 Subject: [PATCH] enable passing a unix_socket for mysql returners (bsc#1091371) quick fix for: https://bugzilla.suse.com/show_bug.cgi?id=1091371 the upstream patch will go through some bigger refactoring of the mysql drivers to be cleaner this patch should only be temporary and can be dropped again once the refactor is done upstream Signed-off-by: Maximilian Meister <[email protected]> --- salt/returners/mysql.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/salt/returners/mysql.py b/salt/returners/mysql.py index 85892cb06c..a286731d5c 100644 --- a/salt/returners/mysql.py +++ b/salt/returners/mysql.py @@ -18,6 +18,7 @@ config. These are the defaults: mysql.pass: 'salt' mysql.db: 'salt' mysql.port: 3306 + mysql.unix_socket: '/tmp/mysql.sock' SSL is optional. The defaults are set to None. If you do not want to use SSL, either exclude these options or set them to None. @@ -43,6 +44,7 @@ optional. The following ssl options are simply for illustration purposes: alternative.mysql.ssl_ca: '/etc/pki/mysql/certs/localhost.pem' alternative.mysql.ssl_cert: '/etc/pki/mysql/certs/localhost.crt' alternative.mysql.ssl_key: '/etc/pki/mysql/certs/localhost.key' + alternative.mysql.unix_socket: '/tmp/mysql.sock' Should you wish the returner data to be cleaned out every so often, set `keep_jobs` to the number of hours for the jobs to live in the tables. @@ -198,7 +200,8 @@ def _get_options(ret=None): 'port': 3306, 'ssl_ca': None, 'ssl_cert': None, - 'ssl_key': None} + 'ssl_key': None, + 'unix_socket': '/tmp/mysql.sock'} attrs = {'host': 'host', 'user': 'user', @@ -207,7 +210,8 @@ def _get_options(ret=None): 'port': 'port', 'ssl_ca': 'ssl_ca', 'ssl_cert': 'ssl_cert', - 'ssl_key': 'ssl_key'} + 'ssl_key': 'ssl_key', + 'unix_socket': 'unix_socket'} _options = salt.returners.get_returner_options(__virtualname__, ret, @@ -261,7 +265,8 @@ def _get_serv(ret=None, commit=False): passwd=_options.get('pass'), db=_options.get('db'), port=_options.get('port'), - ssl=ssl_options) + ssl=ssl_options, + unix_socket=_options.get('unix_socket')) try: __context__['mysql_returner_conn'] = conn -- 2.13.6 ++++++ extra-filerefs-include-files-even-if-no-refs-in-stat.patch ++++++ >From 544dfd7dbaa1c837b75976f15ad67159b1bdedbb Mon Sep 17 00:00:00 2001 From: Matei Albu <[email protected]> Date: Sun, 6 May 2018 11:56:18 +0200 Subject: [PATCH] --extra-filerefs include files even if no refs in states to apply Fixes #47496 (cherry picked from commit d67239a) --- salt/client/ssh/state.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/client/ssh/state.py b/salt/client/ssh/state.py index 8fa11d031e..08d4846bb6 100644 --- a/salt/client/ssh/state.py +++ b/salt/client/ssh/state.py @@ -135,9 +135,9 @@ def lowstate_file_refs(chunks, extras=''): elif state.startswith('__'): continue crefs.extend(salt_refs(chunk[state])) + if saltenv not in refs: + refs[saltenv] = [] if crefs: - if saltenv not in refs: - refs[saltenv] = [] refs[saltenv].append(crefs) if extras: extra_refs = extras.split(',') -- 2.13.6 ++++++ fix-for-ec2-rate-limit-failures.patch ++++++ >From b0273e150b29fb44d72246918792117b485441de Mon Sep 17 00:00:00 2001 From: Daniel Wallace <[email protected]> Date: Wed, 25 Apr 2018 11:13:15 -0500 Subject: [PATCH] Fix for EC2 Rate Limit Failures Fix for ec2 rate limit failures described here: https://bugzilla.suse.com/show_bug.cgi?id=1088888 --- salt/utils/aws.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/salt/utils/aws.py b/salt/utils/aws.py index 059450e7ca..912f1466ba 100644 --- a/salt/utils/aws.py +++ b/salt/utils/aws.py @@ -20,6 +20,7 @@ import hmac import logging import salt.config import re +import random from salt.ext import six # Import Salt libs @@ -442,8 +443,9 @@ def query(params=None, setname=None, requesturl=None, location=None, ) headers = {} - attempts = 5 - while attempts > 0: + MAX_RETRIES = 6 + attempts = 0 + while attempts < MAX_RETRIES: log.debug('AWS Request: %s', requesturl) log.trace('AWS Request Parameters: %s', params_with_headers) try: @@ -461,15 +463,23 @@ def query(params=None, setname=None, requesturl=None, location=None, # check to see if we should retry the query err_code = data.get('Errors', {}).get('Error', {}).get('Code', '') - if attempts > 0 and err_code and err_code in AWS_RETRY_CODES: - attempts -= 1 + if attempts < MAX_RETRIES and err_code and err_code in AWS_RETRY_CODES: + attempts += 1 log.error( 'AWS Response Status Code and Error: [%s %s] %s; ' 'Attempts remaining: %s', exc.response.status_code, exc, data, attempts ) - # Wait a bit before continuing to prevent throttling - time.sleep(2) + # backoff an exponential amount of time to throttle requests + # during "API Rate Exceeded" failures as suggested by the AWS documentation here: + # https://docs.aws.amazon.com/AWSEC2/latest/APIReference/query-api-troubleshooting.html + # and also here: + # https://docs.aws.amazon.com/general/latest/gr/api-retries.html + # Failure to implement this approach results in a failure rate of >30% when using salt-cloud with + # "--parallel" when creating 50 or more instances with a fixed delay of 2 seconds. + # A failure rate of >10% is observed when using the salt-api with an asyncronous client + # specified (runner_async). + time.sleep(random.uniform(1, 2**attempts)) continue log.error( -- 2.13.6 ++++++ option-to-merge-current-pillar-with-opts-pillar-duri.patch ++++++ >From 0cfa4f2a1cf559f87286069691a2766cb24f6076 Mon Sep 17 00:00:00 2001 From: Matei Albu <[email protected]> Date: Sun, 6 May 2018 21:15:58 +0200 Subject: [PATCH] Option to merge current pillar with opts['pillar'] during pillar compile Fixes #47501 (cherry picked from commit 2f1485e) --- doc/ref/configuration/minion.rst | 28 ++++++++++++++++++++++++++++ salt/config/__init__.py | 4 +++- salt/pillar/__init__.py | 7 +++++++ 3 files changed, 38 insertions(+), 1 deletion(-) diff --git a/doc/ref/configuration/minion.rst b/doc/ref/configuration/minion.rst index 9683a0a20a..75ad26c723 100644 --- a/doc/ref/configuration/minion.rst +++ b/doc/ref/configuration/minion.rst @@ -3219,3 +3219,31 @@ URL of the repository: Replace ``<commit_id>`` with the SHA1 hash of a commit ID. Specifying a commit ID is useful in that it allows one to revert back to a previous version in the event that an error is introduced in the latest revision of the repo. + +``ssh_merge_pillar`` +-------------------- + +.. versionadded:: 2018.3.2 + +Default: ``True`` + +Merges the compiled pillar data with the pillar data already available globally. +This is useful when using ``salt-ssh`` or ``salt-call --local`` and overriding the pillar +data in a state file: + +.. code-block:: yaml + + apply_showpillar: + module.run: + - name: state.apply + - mods: + - showpillar + - kwargs: + pillar: + test: "foo bar" + +If set to ``True`` the ``showpillar`` state will have access to the +global pillar data. + +If set to ``False`` only the overriding pillar data will be available +to the ``showpillar`` state. diff --git a/salt/config/__init__.py b/salt/config/__init__.py index b3de3820b0..82d3dfa07f 100644 --- a/salt/config/__init__.py +++ b/salt/config/__init__.py @@ -983,6 +983,7 @@ VALID_OPTS = { 'ssh_identities_only': bool, 'ssh_log_file': six.string_types, 'ssh_config_file': six.string_types, + 'ssh_merge_pillar': bool, # Enable ioflo verbose logging. Warning! Very verbose! 'ioflo_verbose': int, @@ -1476,6 +1477,7 @@ DEFAULT_MINION_OPTS = { }, 'discovery': False, 'schedule': {}, + 'ssh_merge_pillar': True } DEFAULT_MASTER_OPTS = { @@ -2078,7 +2080,7 @@ def _validate_ssh_minion_opts(opts): for opt_name in list(ssh_minion_opts): if re.match('^[a-z0-9]+fs_', opt_name, flags=re.IGNORECASE) \ - or 'pillar' in opt_name \ + or ('pillar' in opt_name and not 'ssh_merge_pillar' == opt_name) \ or opt_name in ('fileserver_backend',): log.warning( '\'%s\' is not a valid ssh_minion_opts parameter, ignoring', diff --git a/salt/pillar/__init__.py b/salt/pillar/__init__.py index 388b774434..5940b7c105 100644 --- a/salt/pillar/__init__.py +++ b/salt/pillar/__init__.py @@ -993,6 +993,13 @@ class Pillar(object): mopts['file_roots'] = self.actual_file_roots mopts['saltversion'] = __version__ pillar['master'] = mopts + if 'pillar' in self.opts and self.opts.get('ssh_merge_pillar', False): + pillar = merge( + self.opts['pillar'], + pillar, + self.merge_strategy, + self.opts.get('renderer', 'yaml'), + self.opts.get('pillar_merge_lists', False)) if errors: for error in errors: log.critical('Pillar render error: %s', error) -- 2.13.6
