Script 'mail_helper' called by obssrc Hello community, here is the log from the commit of package salt for openSUSE:Factory checked in at 2024-01-29 22:27:10 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/salt (Old) and /work/SRC/openSUSE:Factory/.salt.new.1815 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "salt" Mon Jan 29 22:27:10 2024 rev:142 rq:1142448 version:3006.0 Changes: -------- --- /work/SRC/openSUSE:Factory/salt/salt.changes 2023-09-22 21:48:04.170287902 +0200 +++ /work/SRC/openSUSE:Factory/.salt.new.1815/salt.changes 2024-01-29 22:27:18.413728257 +0100 @@ -1,0 +2,104 @@ +Mon Jan 29 13:50:44 UTC 2024 - Pablo Suárez Hernández <[email protected]> + +- Provide user(salt)/group(salt) capabilities for RPM 4.19 + +------------------------------------------------------------------- +Tue Jan 23 15:39:14 UTC 2024 - Pablo Suárez Hernández <[email protected]> + +- Prevent exceptions with fileserver.update when called via state (bsc#1218482) + +- Added: + * allow-kwargs-for-fileserver-roots-update-bsc-1218482.patch + +------------------------------------------------------------------- +Thu Jan 18 16:10:04 UTC 2024 - Pablo Suárez Hernández <[email protected]> + +- Improve pip target override condition with VENV_PIP_TARGET + environment variable (bsc#1216850) +- Fixed KeyError in logs when running a state that fails + +- Added: + * improve-pip-target-override-condition-with-venv_pip_.patch + * fixed-keyerror-in-logs-when-running-a-state-that-fai.patch + +------------------------------------------------------------------- +Thu Jan 18 13:06:21 UTC 2024 - Marek Czernek <[email protected]> + +- Ensure that pillar refresh loads beacons from pillar without restart +- Fix the aptpkg.py unit test failure +- Prefer unittest.mock to python-mock in test suite + +- Added: + * update-__pillar__-during-pillar_refresh.patch + * fix-the-aptpkg.py-unit-test-failure.patch + * prefer-unittest.mock-for-python-versions-that-are-su.patch + +------------------------------------------------------------------- +Fri Dec 1 11:04:02 UTC 2023 - Pablo Suárez Hernández <[email protected]> + +- Enable "KeepAlive" probes for Salt SSH executions (bsc#1211649) + +- Added: + * enable-keepalive-probes-for-salt-ssh-executions-bsc-.patch + +------------------------------------------------------------------- +Thu Nov 16 10:33:00 UTC 2023 - Pablo Suárez Hernández <[email protected]> + +- Revert changes to set Salt configured user early in the stack (bsc#1216284) + +- Added: + * revert-make-sure-configured-user-is-properly-set-by-.patch + +------------------------------------------------------------------- +Mon Nov 13 16:02:35 UTC 2023 - Pablo Suárez Hernández <[email protected]> + +- Align behavior of some modules when using salt-call via symlink (bsc#1215963) +- Fix gitfs "__env__" and improve cache cleaning (bsc#1193948) + +- Added: + * fix-gitfs-__env__-and-improve-cache-cleaning-bsc-119.patch + * dereference-symlinks-to-set-proper-__cli-opt-bsc-121.patch + +------------------------------------------------------------------- +Tue Oct 31 11:51:17 UTC 2023 - Alexander Graul <[email protected]> + +- Randomize pre_flight_script path (CVE-2023-34049 bsc#1215157) + +- Added: + * fix-cve-2023-34049-bsc-1215157.patch + +------------------------------------------------------------------- +Tue Oct 17 15:28:22 UTC 2023 - Marek Czernek <[email protected]> + +- Allow all primitive grain types for autosign_grains (bsc#1214477) + +- Added: + * allow-all-primitive-grain-types-for-autosign_grains-.patch + +------------------------------------------------------------------- +Mon Oct 2 12:49:41 UTC 2023 - Victor Zhestkov <[email protected]> + +- Fix optimization_order opt to prevent testsuite fails +- Improve salt.utils.json.find_json to avoid fails + with transactional salt salt-ssh managed clients (bsc#1213293) +- Use salt-call from salt bundle with transactional_update +- Only call native_str on curl_debug message in tornado when needed +- Implement the calling for batch async from the salt CLI + +- Added: + * implement-the-calling-for-batch-async-from-the-salt-.patch + * improve-salt.utils.json.find_json-bsc-1213293.patch + * only-call-native_str-on-curl_debug-message-in-tornad.patch + * use-salt-call-from-salt-bundle-with-transactional_up.patch + * fix-optimization_order-opt-to-prevent-test-fails.patch + +------------------------------------------------------------------- +Mon Oct 2 09:50:00 UTC 2023 - Pablo Suárez Hernández <[email protected]> + +- Fix calculation of SLS context vars when trailing dots + on targetted sls/state (bsc#1213518) + +- Added: + * fix-calculation-of-sls-context-vars-when-trailing-do.patch + +------------------------------------------------------------------- New: ---- allow-all-primitive-grain-types-for-autosign_grains-.patch allow-kwargs-for-fileserver-roots-update-bsc-1218482.patch dereference-symlinks-to-set-proper-__cli-opt-bsc-121.patch enable-keepalive-probes-for-salt-ssh-executions-bsc-.patch fix-calculation-of-sls-context-vars-when-trailing-do.patch fix-cve-2023-34049-bsc-1215157.patch fix-gitfs-__env__-and-improve-cache-cleaning-bsc-119.patch fix-optimization_order-opt-to-prevent-test-fails.patch fix-the-aptpkg.py-unit-test-failure.patch fixed-keyerror-in-logs-when-running-a-state-that-fai.patch implement-the-calling-for-batch-async-from-the-salt-.patch improve-pip-target-override-condition-with-venv_pip_.patch improve-salt.utils.json.find_json-bsc-1213293.patch only-call-native_str-on-curl_debug-message-in-tornad.patch prefer-unittest.mock-for-python-versions-that-are-su.patch revert-make-sure-configured-user-is-properly-set-by-.patch update-__pillar__-during-pillar_refresh.patch use-salt-call-from-salt-bundle-with-transactional_up.patch BETA DEBUG BEGIN: New:- Added: * allow-all-primitive-grain-types-for-autosign_grains-.patch New:- Added: * allow-kwargs-for-fileserver-roots-update-bsc-1218482.patch New: * fix-gitfs-__env__-and-improve-cache-cleaning-bsc-119.patch * dereference-symlinks-to-set-proper-__cli-opt-bsc-121.patch New:- Added: * enable-keepalive-probes-for-salt-ssh-executions-bsc-.patch New:- Added: * fix-calculation-of-sls-context-vars-when-trailing-do.patch New:- Added: * fix-cve-2023-34049-bsc-1215157.patch New:- Added: * fix-gitfs-__env__-and-improve-cache-cleaning-bsc-119.patch * dereference-symlinks-to-set-proper-__cli-opt-bsc-121.patch New: * use-salt-call-from-salt-bundle-with-transactional_up.patch * fix-optimization_order-opt-to-prevent-test-fails.patch New: * update-__pillar__-during-pillar_refresh.patch * fix-the-aptpkg.py-unit-test-failure.patch * prefer-unittest.mock-for-python-versions-that-are-su.patch New: * improve-pip-target-override-condition-with-venv_pip_.patch * fixed-keyerror-in-logs-when-running-a-state-that-fai.patch New:- Added: * implement-the-calling-for-batch-async-from-the-salt-.patch * improve-salt.utils.json.find_json-bsc-1213293.patch New:- Added: * improve-pip-target-override-condition-with-venv_pip_.patch * fixed-keyerror-in-logs-when-running-a-state-that-fai.patch New: * implement-the-calling-for-batch-async-from-the-salt-.patch * improve-salt.utils.json.find_json-bsc-1213293.patch * only-call-native_str-on-curl_debug-message-in-tornad.patch New: * improve-salt.utils.json.find_json-bsc-1213293.patch * only-call-native_str-on-curl_debug-message-in-tornad.patch * use-salt-call-from-salt-bundle-with-transactional_up.patch New: * fix-the-aptpkg.py-unit-test-failure.patch * prefer-unittest.mock-for-python-versions-that-are-su.patch New:- Added: * revert-make-sure-configured-user-is-properly-set-by-.patch New:- Added: * update-__pillar__-during-pillar_refresh.patch * fix-the-aptpkg.py-unit-test-failure.patch New: * only-call-native_str-on-curl_debug-message-in-tornad.patch * use-salt-call-from-salt-bundle-with-transactional_up.patch * fix-optimization_order-opt-to-prevent-test-fails.patch BETA DEBUG END: ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ salt.spec ++++++ --- /var/tmp/diff_new_pack.rUnAZz/_old 2024-01-29 22:27:21.125826560 +0100 +++ /var/tmp/diff_new_pack.rUnAZz/_new 2024-01-29 22:27:21.129826705 +0100 @@ -296,7 +296,7 @@ # PATCH-FIX_UPSTREAM: https://github.com/saltstack/salt/commit/f82860b8ad3ee786762fa02fa1a6eaf6e24dc8d4 # PATCH-FIX_UPSTREAM: https://github.com/saltstack/salt/pull/65020 Patch76: do-not-fail-on-bad-message-pack-message-bsc-1213441-.patch -# PATCH-FIX_UPSTREAM: https://github.com/saltstack/salt/pull/64510 +# PATCH-FIX_UPSTREAM: https://github.com/saltstack/salt/pull/64510 (dropped at patch 91) Patch77: make-sure-configured-user-is-properly-set-by-salt-bs.patch # PATCH-FIX_UPSTREAM: https://github.com/saltstack/salt/pull/64959 Patch78: fixed-gitfs-cachedir_basename-to-avoid-hash-collisio.patch @@ -304,6 +304,44 @@ Patch79: revert-usage-of-long-running-req-channel-bsc-1213960.patch # PATCH-FIX_UPSTREAM: https://github.com/saltstack/salt/pull/65238 Patch80: write-salt-version-before-building-when-using-with-s.patch +# PATCH-FIX_UPSTREAM: https://github.com/saltstack/salt/pull/65036 +Patch81: fix-calculation-of-sls-context-vars-when-trailing-do.patch +# PATCH-FIX_OPENSUSE: https://github.com/openSUSE/salt/pull/594 +Patch82: implement-the-calling-for-batch-async-from-the-salt-.patch +# PATCH-FIX_UPSTREAM: https://github.com/tornadoweb/tornado/pull/2277 +Patch83: only-call-native_str-on-curl_debug-message-in-tornad.patch +# PATCH-FIX_UPSTREAM: https://github.com/saltstack/salt/pull/65204 +Patch84: use-salt-call-from-salt-bundle-with-transactional_up.patch +# PATCH-FIX_UPSTREAM: https://github.com/saltstack/salt/pull/65181 +Patch85: improve-salt.utils.json.find_json-bsc-1213293.patch +# PATCH-FIX_UPSTREAM: https://github.com/saltstack/salt/pull/65266 +Patch86: fix-optimization_order-opt-to-prevent-test-fails.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/65232 +Patch87: allow-all-primitive-grain-types-for-autosign_grains-.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/65482 +Patch88: fix-cve-2023-34049-bsc-1215157.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/65017 +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/65136 +Patch89: fix-gitfs-__env__-and-improve-cache-cleaning-bsc-119.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/65435 +Patch90: dereference-symlinks-to-set-proper-__cli-opt-bsc-121.patch +# PATCH-FIX_OPENSUSE: https://github.com/openSUSE/salt/pull/614 (revert patch 77) +Patch91: revert-make-sure-configured-user-is-properly-set-by-.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/65488 +Patch92: enable-keepalive-probes-for-salt-ssh-executions-bsc-.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/65644 +Patch93: prefer-unittest.mock-for-python-versions-that-are-su.patch +# PATCH-FIX_OPENSUSE: https://github.com/openSUSE/salt/pull/620 +Patch94: fix-the-aptpkg.py-unit-test-failure.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/65092 +Patch95: update-__pillar__-during-pillar_refresh.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/65009 +Patch96: fixed-keyerror-in-logs-when-running-a-state-that-fai.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/65562 +Patch97: improve-pip-target-override-condition-with-venv_pip_.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/65819 +Patch98: allow-kwargs-for-fileserver-roots-update-bsc-1218482.patch + ### IMPORTANT: The line below is used as a snippet marker. Do not touch it. ### SALT PATCHES LIST END @@ -319,6 +357,8 @@ Requires(pre): %{_sbindir}/groupadd Requires(pre): %{_sbindir}/useradd +Provides: user(salt) +Provides: group(salt) %if 0%{?suse_version} Requires(pre): %fillup_prereq ++++++ _lastrevision ++++++ --- /var/tmp/diff_new_pack.rUnAZz/_old 2024-01-29 22:27:21.209829605 +0100 +++ /var/tmp/diff_new_pack.rUnAZz/_new 2024-01-29 22:27:21.213829750 +0100 @@ -1,3 +1,3 @@ -3becea2e5b00beff724c22a8ae320d4567031c7b +d02a7d8eb7fdb302ed603f2c45525b41e812beb7 (No newline at EOF) ++++++ allow-all-primitive-grain-types-for-autosign_grains-.patch ++++++ >From ae4e1d1cc15b3c510bdd774a1dfeff67c522324a Mon Sep 17 00:00:00 2001 From: Marek Czernek <[email protected]> Date: Tue, 17 Oct 2023 13:05:00 +0200 Subject: [PATCH] Allow all primitive grain types for autosign_grains (#607) * Allow all primitive grain types for autosign_grains Signed-off-by: Marek Czernek <[email protected]> * blacken daemons/masterapi.py and its test_auto_key Signed-off-by: Marek Czernek <[email protected]> --------- Signed-off-by: Marek Czernek <[email protected]> Co-authored-by: Alexander Graul <[email protected]> --- changelog/61416.fixed.md | 1 + changelog/63708.fixed.md | 1 + salt/daemons/masterapi.py | 2 +- .../pytests/unit/daemons/masterapi/test_auto_key.py | 13 +++++++------ 4 files changed, 10 insertions(+), 7 deletions(-) create mode 100644 changelog/61416.fixed.md create mode 100644 changelog/63708.fixed.md diff --git a/changelog/61416.fixed.md b/changelog/61416.fixed.md new file mode 100644 index 0000000000..3203a0a1c6 --- /dev/null +++ b/changelog/61416.fixed.md @@ -0,0 +1 @@ +Allow all primitive grain types for autosign_grains diff --git a/changelog/63708.fixed.md b/changelog/63708.fixed.md new file mode 100644 index 0000000000..3203a0a1c6 --- /dev/null +++ b/changelog/63708.fixed.md @@ -0,0 +1 @@ +Allow all primitive grain types for autosign_grains diff --git a/salt/daemons/masterapi.py b/salt/daemons/masterapi.py index 3716c63d99..54aca64a76 100644 --- a/salt/daemons/masterapi.py +++ b/salt/daemons/masterapi.py @@ -366,7 +366,7 @@ class AutoKey: line = salt.utils.stringutils.to_unicode(line).strip() if line.startswith("#"): continue - if autosign_grains[grain] == line: + if str(autosign_grains[grain]) == line: return True return False diff --git a/tests/pytests/unit/daemons/masterapi/test_auto_key.py b/tests/pytests/unit/daemons/masterapi/test_auto_key.py index b3657b7f1b..54c3f22d2a 100644 --- a/tests/pytests/unit/daemons/masterapi/test_auto_key.py +++ b/tests/pytests/unit/daemons/masterapi/test_auto_key.py @@ -17,11 +17,11 @@ def gen_permissions(owner="", group="", others=""): """ ret = 0 for c in owner: - ret |= getattr(stat, "S_I{}USR".format(c.upper()), 0) + ret |= getattr(stat, f"S_I{c.upper()}USR", 0) for c in group: - ret |= getattr(stat, "S_I{}GRP".format(c.upper()), 0) + ret |= getattr(stat, f"S_I{c.upper()}GRP", 0) for c in others: - ret |= getattr(stat, "S_I{}OTH".format(c.upper()), 0) + ret |= getattr(stat, f"S_I{c.upper()}OTH", 0) return ret @@ -256,16 +256,17 @@ def test_check_autosign_grains_no_autosign_grains_dir(auto_key): _test_check_autosign_grains(test_func, auto_key, autosign_grains_dir=None) -def test_check_autosign_grains_accept(auto_key): [email protected]("grain_value", ["test_value", 123, True]) +def test_check_autosign_grains_accept(grain_value, auto_key): """ Asserts that autosigning from grains passes when a matching grain value is in an autosign_grain file. """ def test_func(*args): - assert auto_key.check_autosign_grains({"test_grain": "test_value"}) is True + assert auto_key.check_autosign_grains({"test_grain": grain_value}) is True - file_content = "#test_ignore\ntest_value" + file_content = f"#test_ignore\n{grain_value}" _test_check_autosign_grains(test_func, auto_key, file_content=file_content) -- 2.42.0 ++++++ allow-kwargs-for-fileserver-roots-update-bsc-1218482.patch ++++++ >From 8ae54e8a0e12193507f1936f363c3438b4a006ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yeray=20Guti=C3=A9rrez=20Cedr=C3=A9s?= <[email protected]> Date: Tue, 23 Jan 2024 15:33:28 +0000 Subject: [PATCH] Allow kwargs for fileserver roots update (bsc#1218482) (#618) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Allow kwargs for fileserver roots update (bsc#1218482) * Prevent exceptions with fileserver.update when called via state * Fix wrong logic and enhance tests around fileserver.update * Remove test which is not longer valid --------- Co-authored-by: Pablo Suárez Hernández <[email protected]> --- changelog/65819.fixed.md | 1 + salt/fileserver/roots.py | 8 ++-- salt/runners/fileserver.py | 6 +++ tests/integration/runners/test_fileserver.py | 40 ++++++++++++++++++-- tests/pytests/unit/fileserver/test_roots.py | 2 +- 5 files changed, 47 insertions(+), 10 deletions(-) create mode 100644 changelog/65819.fixed.md diff --git a/changelog/65819.fixed.md b/changelog/65819.fixed.md new file mode 100644 index 0000000000..432f5c791c --- /dev/null +++ b/changelog/65819.fixed.md @@ -0,0 +1 @@ +Prevent exceptions with fileserver.update when called via state diff --git a/salt/fileserver/roots.py b/salt/fileserver/roots.py index 4880cbab9b..a02b597c6f 100644 --- a/salt/fileserver/roots.py +++ b/salt/fileserver/roots.py @@ -193,9 +193,7 @@ def update(): os.makedirs(mtime_map_path_dir) with salt.utils.files.fopen(mtime_map_path, "wb") as fp_: for file_path, mtime in new_mtime_map.items(): - fp_.write( - salt.utils.stringutils.to_bytes("{}:{}\n".format(file_path, mtime)) - ) + fp_.write(salt.utils.stringutils.to_bytes(f"{file_path}:{mtime}\n")) if __opts__.get("fileserver_events", False): # if there is a change, fire an event @@ -326,11 +324,11 @@ def _file_lists(load, form): return [] list_cache = os.path.join( list_cachedir, - "{}.p".format(salt.utils.files.safe_filename_leaf(actual_saltenv)), + f"{salt.utils.files.safe_filename_leaf(actual_saltenv)}.p", ) w_lock = os.path.join( list_cachedir, - ".{}.w".format(salt.utils.files.safe_filename_leaf(actual_saltenv)), + f".{salt.utils.files.safe_filename_leaf(actual_saltenv)}.w", ) cache_match, refresh_cache, save_cache = salt.fileserver.check_file_list_cache( __opts__, form, list_cache, w_lock diff --git a/salt/runners/fileserver.py b/salt/runners/fileserver.py index d75d7de0cf..1ed05b68ca 100644 --- a/salt/runners/fileserver.py +++ b/salt/runners/fileserver.py @@ -350,6 +350,12 @@ def update(backend=None, **kwargs): salt-run fileserver.update backend=git remotes=myrepo,yourrepo """ fileserver = salt.fileserver.Fileserver(__opts__) + + # Remove possible '__pub_user' in kwargs as it is not expected + # on "update" function for the different fileserver backends. + if "__pub_user" in kwargs: + del kwargs["__pub_user"] + fileserver.update(back=backend, **kwargs) return True diff --git a/tests/integration/runners/test_fileserver.py b/tests/integration/runners/test_fileserver.py index ae8ab766aa..62f0da0c4a 100644 --- a/tests/integration/runners/test_fileserver.py +++ b/tests/integration/runners/test_fileserver.py @@ -202,15 +202,31 @@ class FileserverTest(ShellCase): fileserver.update """ ret = self.run_run_plus(fun="fileserver.update") - self.assertTrue(ret["return"]) + self.assertTrue(ret["return"] is True) # Backend submitted as a string ret = self.run_run_plus(fun="fileserver.update", backend="roots") - self.assertTrue(ret["return"]) + self.assertTrue(ret["return"] is True) # Backend submitted as a list ret = self.run_run_plus(fun="fileserver.update", backend=["roots"]) - self.assertTrue(ret["return"]) + self.assertTrue(ret["return"] is True) + + # Possible '__pub_user' is removed from kwargs + ret = self.run_run_plus( + fun="fileserver.update", backend=["roots"], __pub_user="foo" + ) + self.assertTrue(ret["return"] is True) + + # Unknown arguments + ret = self.run_run_plus( + fun="fileserver.update", backend=["roots"], unknown_arg="foo" + ) + self.assertIn( + "Passed invalid arguments: update() got an unexpected keyword argument" + " 'unknown_arg'", + ret["return"], + ) # Other arguments are passed to backend def mock_gitfs_update(remotes=None): @@ -225,7 +241,23 @@ class FileserverTest(ShellCase): ret = self.run_run_plus( fun="fileserver.update", backend="gitfs", remotes="myrepo,yourrepo" ) - self.assertTrue(ret["return"]) + self.assertTrue(ret["return"] is True) + mock_backend_func.assert_called_once_with(remotes="myrepo,yourrepo") + + # Possible '__pub_user' arguments are removed from kwargs + mock_backend_func = create_autospec(mock_gitfs_update) + mock_return_value = { + "gitfs.envs": None, # This is needed to activate the backend + "gitfs.update": mock_backend_func, + } + with patch("salt.loader.fileserver", MagicMock(return_value=mock_return_value)): + ret = self.run_run_plus( + fun="fileserver.update", + backend="gitfs", + remotes="myrepo,yourrepo", + __pub_user="foo", + ) + self.assertTrue(ret["return"] is True) mock_backend_func.assert_called_once_with(remotes="myrepo,yourrepo") # Unknown arguments are passed to backend diff --git a/tests/pytests/unit/fileserver/test_roots.py b/tests/pytests/unit/fileserver/test_roots.py index a8a80eea17..96bceb0fd3 100644 --- a/tests/pytests/unit/fileserver/test_roots.py +++ b/tests/pytests/unit/fileserver/test_roots.py @@ -236,7 +236,7 @@ def test_update_mtime_map(): # between Python releases. lines_written = sorted(mtime_map_mock.write_calls()) expected = sorted( - salt.utils.stringutils.to_bytes("{key}:{val}\n".format(key=key, val=val)) + salt.utils.stringutils.to_bytes(f"{key}:{val}\n") for key, val in new_mtime_map.items() ) assert lines_written == expected, lines_written -- 2.43.0 ++++++ dereference-symlinks-to-set-proper-__cli-opt-bsc-121.patch ++++++ >From 9942c488b1e74f2c6f187fcef3556fe53382bb4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20Su=C3=A1rez=20Hern=C3=A1ndez?= <[email protected]> Date: Mon, 13 Nov 2023 15:04:14 +0000 Subject: [PATCH] Dereference symlinks to set proper __cli opt (bsc#1215963) (#611) * Dereference symlinks to set proper __cli * Add changelog entry * Add unit tests to check path is expanded --------- Co-authored-by: vzhestkov <[email protected]> --- changelog/65435.fixed.md | 1 + salt/config/__init__.py | 8 ++++++-- tests/pytests/unit/config/test_master_config.py | 13 +++++++++++++ tests/pytests/unit/config/test_minion_config.py | 13 +++++++++++++ 4 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 changelog/65435.fixed.md create mode 100644 tests/pytests/unit/config/test_master_config.py create mode 100644 tests/pytests/unit/config/test_minion_config.py diff --git a/changelog/65435.fixed.md b/changelog/65435.fixed.md new file mode 100644 index 0000000000..5fa532891d --- /dev/null +++ b/changelog/65435.fixed.md @@ -0,0 +1 @@ +Dereference symlinks to set proper __cli opt diff --git a/salt/config/__init__.py b/salt/config/__init__.py index 43182f3f92..d8258a4dbc 100644 --- a/salt/config/__init__.py +++ b/salt/config/__init__.py @@ -3747,7 +3747,9 @@ def apply_minion_config( ) opts["fileserver_backend"][idx] = new_val - opts["__cli"] = salt.utils.stringutils.to_unicode(os.path.basename(sys.argv[0])) + opts["__cli"] = salt.utils.stringutils.to_unicode( + os.path.basename(salt.utils.path.expand(sys.argv[0])) + ) # No ID provided. Will getfqdn save us? using_ip_for_id = False @@ -3949,7 +3951,9 @@ def apply_master_config(overrides=None, defaults=None): ) opts["keep_acl_in_token"] = True - opts["__cli"] = salt.utils.stringutils.to_unicode(os.path.basename(sys.argv[0])) + opts["__cli"] = salt.utils.stringutils.to_unicode( + os.path.basename(salt.utils.path.expand(sys.argv[0])) + ) if "environment" in opts: if opts["saltenv"] is not None: diff --git a/tests/pytests/unit/config/test_master_config.py b/tests/pytests/unit/config/test_master_config.py new file mode 100644 index 0000000000..c9de8a7892 --- /dev/null +++ b/tests/pytests/unit/config/test_master_config.py @@ -0,0 +1,13 @@ +import salt.config +from tests.support.mock import MagicMock, patch + + +def test___cli_path_is_expanded(): + defaults = salt.config.DEFAULT_MASTER_OPTS.copy() + overrides = {} + with patch( + "salt.utils.path.expand", MagicMock(return_value="/path/to/testcli") + ) as expand_mock: + opts = salt.config.apply_master_config(overrides, defaults) + assert expand_mock.called + assert opts["__cli"] == "testcli" diff --git a/tests/pytests/unit/config/test_minion_config.py b/tests/pytests/unit/config/test_minion_config.py new file mode 100644 index 0000000000..34aa84daa7 --- /dev/null +++ b/tests/pytests/unit/config/test_minion_config.py @@ -0,0 +1,13 @@ +import salt.config +from tests.support.mock import MagicMock, patch + + +def test___cli_path_is_expanded(): + defaults = salt.config.DEFAULT_MINION_OPTS.copy() + overrides = {} + with patch( + "salt.utils.path.expand", MagicMock(return_value="/path/to/testcli") + ) as expand_mock: + opts = salt.config.apply_minion_config(overrides, defaults) + assert expand_mock.called + assert opts["__cli"] == "testcli" -- 2.42.0 ++++++ enable-keepalive-probes-for-salt-ssh-executions-bsc-.patch ++++++ >From 5303cc612bcbdb1ec45ede397ca1e2ca12ba3bd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20Su=C3=A1rez=20Hern=C3=A1ndez?= <[email protected]> Date: Fri, 1 Dec 2023 10:59:30 +0000 Subject: [PATCH] Enable "KeepAlive" probes for Salt SSH executions (bsc#1211649) (#610) * Enable KeepAlive probes for Salt SSH connections (bsc#1211649) * Add tests for Salt SSH keepalive options * Add changelog file * Make changes suggested by pre-commit --- changelog/65488.added.md | 1 + salt/client/ssh/__init__.py | 32 +++++++++--- salt/client/ssh/client.py | 13 ++++- salt/client/ssh/shell.py | 12 +++++ salt/config/__init__.py | 6 +++ salt/utils/parsers.py | 19 +++++++ tests/pytests/unit/client/ssh/test_single.py | 55 ++++++++++++++++++++ tests/pytests/unit/client/ssh/test_ssh.py | 3 ++ 8 files changed, 133 insertions(+), 8 deletions(-) create mode 100644 changelog/65488.added.md diff --git a/changelog/65488.added.md b/changelog/65488.added.md new file mode 100644 index 0000000000..78476cec11 --- /dev/null +++ b/changelog/65488.added.md @@ -0,0 +1 @@ +Enable "KeepAlive" probes for Salt SSH executions diff --git a/salt/client/ssh/__init__.py b/salt/client/ssh/__init__.py index 1e143f9e30..1d8426b7c2 100644 --- a/salt/client/ssh/__init__.py +++ b/salt/client/ssh/__init__.py @@ -50,8 +50,8 @@ import salt.utils.thin import salt.utils.url import salt.utils.verify from salt._logging import LOG_LEVELS -from salt._logging.mixins import MultiprocessingStateMixin from salt._logging.impl import LOG_LOCK +from salt._logging.mixins import MultiprocessingStateMixin from salt.template import compile_template from salt.utils.process import Process from salt.utils.zeromq import zmq @@ -307,6 +307,18 @@ class SSH(MultiprocessingStateMixin): "ssh_timeout", salt.config.DEFAULT_MASTER_OPTS["ssh_timeout"] ) + self.opts.get("timeout", salt.config.DEFAULT_MASTER_OPTS["timeout"]), + "keepalive": self.opts.get( + "ssh_keepalive", + salt.config.DEFAULT_MASTER_OPTS["ssh_keepalive"], + ), + "keepalive_interval": self.opts.get( + "ssh_keepalive_interval", + salt.config.DEFAULT_MASTER_OPTS["ssh_keepalive_interval"], + ), + "keepalive_count_max": self.opts.get( + "ssh_keepalive_count_max", + salt.config.DEFAULT_MASTER_OPTS["ssh_keepalive_count_max"], + ), "sudo": self.opts.get( "ssh_sudo", salt.config.DEFAULT_MASTER_OPTS["ssh_sudo"] ), @@ -557,7 +569,7 @@ class SSH(MultiprocessingStateMixin): mods=self.mods, fsclient=self.fsclient, thin=self.thin, - **target + **target, ) if salt.utils.path.which("ssh-copy-id"): # we have ssh-copy-id, use it! @@ -573,7 +585,7 @@ class SSH(MultiprocessingStateMixin): mods=self.mods, fsclient=self.fsclient, thin=self.thin, - **target + **target, ) stdout, stderr, retcode = single.cmd_block() try: @@ -601,7 +613,7 @@ class SSH(MultiprocessingStateMixin): fsclient=self.fsclient, thin=self.thin, mine=mine, - **target + **target, ) ret = {"id": single.id} stdout, stderr, retcode = single.run() @@ -1022,7 +1034,10 @@ class Single: remote_port_forwards=None, winrm=False, ssh_options=None, - **kwargs + keepalive=True, + keepalive_interval=60, + keepalive_count_max=3, + **kwargs, ): # Get mine setting and mine_functions if defined in kwargs (from roster) self.mine = mine @@ -1081,6 +1096,9 @@ class Single: "priv": priv, "priv_passwd": priv_passwd, "timeout": timeout, + "keepalive": keepalive, + "keepalive_interval": keepalive_interval, + "keepalive_count_max": keepalive_count_max, "sudo": sudo, "tty": tty, "mods": self.mods, @@ -1302,7 +1320,7 @@ class Single: self.id, fsclient=self.fsclient, minion_opts=self.minion_opts, - **self.target + **self.target, ) opts_pkg = pre_wrapper["test.opts_pkg"]() # pylint: disable=E1102 @@ -1388,7 +1406,7 @@ class Single: self.id, fsclient=self.fsclient, minion_opts=self.minion_opts, - **self.target + **self.target, ) wrapper.fsclient.opts["cachedir"] = opts["cachedir"] self.wfuncs = salt.loader.ssh_wrapper(opts, wrapper, self.context) diff --git a/salt/client/ssh/client.py b/salt/client/ssh/client.py index 0b67598fc6..a00f5de423 100644 --- a/salt/client/ssh/client.py +++ b/salt/client/ssh/client.py @@ -52,6 +52,9 @@ class SSHClient: ("ssh_priv_passwd", str), ("ssh_identities_only", bool), ("ssh_remote_port_forwards", str), + ("ssh_keepalive", bool), + ("ssh_keepalive_interval", int), + ("ssh_keepalive_count_max", int), ("ssh_options", list), ("ssh_max_procs", int), ("ssh_askpass", bool), @@ -108,7 +111,15 @@ class SSHClient: return sane_kwargs def _prep_ssh( - self, tgt, fun, arg=(), timeout=None, tgt_type="glob", kwarg=None, context=None, **kwargs + self, + tgt, + fun, + arg=(), + timeout=None, + tgt_type="glob", + kwarg=None, + context=None, + **kwargs ): """ Prepare the arguments diff --git a/salt/client/ssh/shell.py b/salt/client/ssh/shell.py index bc1ad034df..182e2c19e3 100644 --- a/salt/client/ssh/shell.py +++ b/salt/client/ssh/shell.py @@ -85,6 +85,9 @@ class Shell: remote_port_forwards=None, winrm=False, ssh_options=None, + keepalive=True, + keepalive_interval=None, + keepalive_count_max=None, ): self.opts = opts # ssh <ipv6>, but scp [<ipv6]:/path @@ -95,6 +98,9 @@ class Shell: self.priv = priv self.priv_passwd = priv_passwd self.timeout = timeout + self.keepalive = keepalive + self.keepalive_interval = keepalive_interval + self.keepalive_count_max = keepalive_count_max self.sudo = sudo self.tty = tty self.mods = mods @@ -130,6 +136,9 @@ class Shell: if self.opts.get("_ssh_version", (0,)) > (4, 9): options.append("GSSAPIAuthentication=no") options.append("ConnectTimeout={}".format(self.timeout)) + if self.keepalive: + options.append(f"ServerAliveInterval={self.keepalive_interval}") + options.append(f"ServerAliveCountMax={self.keepalive_count_max}") if self.opts.get("ignore_host_keys"): options.append("StrictHostKeyChecking=no") if self.opts.get("no_host_keys"): @@ -165,6 +174,9 @@ class Shell: if self.opts["_ssh_version"] > (4, 9): options.append("GSSAPIAuthentication=no") options.append("ConnectTimeout={}".format(self.timeout)) + if self.keepalive: + options.append(f"ServerAliveInterval={self.keepalive_interval}") + options.append(f"ServerAliveCountMax={self.keepalive_count_max}") if self.opts.get("ignore_host_keys"): options.append("StrictHostKeyChecking=no") if self.opts.get("no_host_keys"): diff --git a/salt/config/__init__.py b/salt/config/__init__.py index d8258a4dbc..68f2b0f674 100644 --- a/salt/config/__init__.py +++ b/salt/config/__init__.py @@ -822,6 +822,9 @@ VALID_OPTS = immutabletypes.freeze( "ssh_scan_ports": str, "ssh_scan_timeout": float, "ssh_identities_only": bool, + "ssh_keepalive": bool, + "ssh_keepalive_interval": int, + "ssh_keepalive_count_max": int, "ssh_log_file": str, "ssh_config_file": str, "ssh_merge_pillar": bool, @@ -1592,6 +1595,9 @@ DEFAULT_MASTER_OPTS = immutabletypes.freeze( "ssh_scan_ports": "22", "ssh_scan_timeout": 0.01, "ssh_identities_only": False, + "ssh_keepalive": True, + "ssh_keepalive_interval": 60, + "ssh_keepalive_count_max": 3, "ssh_log_file": os.path.join(salt.syspaths.LOGS_DIR, "ssh"), "ssh_config_file": os.path.join(salt.syspaths.HOME_DIR, ".ssh", "config"), "cluster_mode": False, diff --git a/salt/utils/parsers.py b/salt/utils/parsers.py index dc125de7d7..6c7f9f2f66 100644 --- a/salt/utils/parsers.py +++ b/salt/utils/parsers.py @@ -3383,6 +3383,25 @@ class SaltSSHOptionParser( "-R parameters." ), ) + ssh_group.add_option( + "--disable-keepalive", + default=True, + action="store_false", + dest="ssh_keepalive", + help=( + "Disable KeepAlive probes (ServerAliveInterval) for the SSH connection." + ), + ) + ssh_group.add_option( + "--keepalive-interval", + dest="ssh_keepalive_interval", + help=("Define the value for ServerAliveInterval option."), + ) + ssh_group.add_option( + "--keepalive-count-max", + dest="ssh_keepalive_count_max", + help=("Define the value for ServerAliveCountMax option."), + ) ssh_group.add_option( "--ssh-option", dest="ssh_options", diff --git a/tests/pytests/unit/client/ssh/test_single.py b/tests/pytests/unit/client/ssh/test_single.py index c88a1c2127..8d87da8700 100644 --- a/tests/pytests/unit/client/ssh/test_single.py +++ b/tests/pytests/unit/client/ssh/test_single.py @@ -63,6 +63,61 @@ def test_single_opts(opts, target): **target, ) + assert single.shell._ssh_opts() == "" + expected_cmd = ( + "ssh login1 " + "-o KbdInteractiveAuthentication=no -o " + "PasswordAuthentication=yes -o ConnectTimeout=65 -o ServerAliveInterval=60 " + "-o ServerAliveCountMax=3 -o Port=22 " + "-o IdentityFile=/etc/salt/pki/master/ssh/salt-ssh.rsa " + "-o User=root date +%s" + ) + assert single.shell._cmd_str("date +%s") == expected_cmd + + +def test_single_opts_custom_keepalive_options(opts, target): + """Sanity check for ssh.Single options with custom keepalive""" + + single = ssh.Single( + opts, + opts["argv"], + "localhost", + mods={}, + fsclient=None, + thin=salt.utils.thin.thin_path(opts["cachedir"]), + mine=False, + keepalive_interval=15, + keepalive_count_max=5, + **target, + ) + + assert single.shell._ssh_opts() == "" + expected_cmd = ( + "ssh login1 " + "-o KbdInteractiveAuthentication=no -o " + "PasswordAuthentication=yes -o ConnectTimeout=65 -o ServerAliveInterval=15 " + "-o ServerAliveCountMax=5 -o Port=22 " + "-o IdentityFile=/etc/salt/pki/master/ssh/salt-ssh.rsa " + "-o User=root date +%s" + ) + assert single.shell._cmd_str("date +%s") == expected_cmd + + +def test_single_opts_disable_keepalive(opts, target): + """Sanity check for ssh.Single options with custom keepalive""" + + single = ssh.Single( + opts, + opts["argv"], + "localhost", + mods={}, + fsclient=None, + thin=salt.utils.thin.thin_path(opts["cachedir"]), + mine=False, + keepalive=False, + **target, + ) + assert single.shell._ssh_opts() == "" expected_cmd = ( "ssh login1 " diff --git a/tests/pytests/unit/client/ssh/test_ssh.py b/tests/pytests/unit/client/ssh/test_ssh.py index cece16026c..23223ba8ec 100644 --- a/tests/pytests/unit/client/ssh/test_ssh.py +++ b/tests/pytests/unit/client/ssh/test_ssh.py @@ -78,6 +78,9 @@ def roster(): ("ssh_scan_ports", "test", True), ("ssh_scan_timeout", 1.0, True), ("ssh_timeout", 1, False), + ("ssh_keepalive", True, True), + ("ssh_keepalive_interval", 30, True), + ("ssh_keepalive_count_max", 3, True), ("ssh_log_file", "/tmp/test", True), ("raw_shell", True, True), ("refresh_cache", True, True), -- 2.42.0 ++++++ fix-calculation-of-sls-context-vars-when-trailing-do.patch ++++++ >From 3403a7391df785be31b6fbe401a8229c2007ac19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20Su=C3=A1rez=20Hern=C3=A1ndez?= <[email protected]> Date: Mon, 2 Oct 2023 10:44:05 +0100 Subject: [PATCH] Fix calculation of SLS context vars when trailing dots on targetted sls/state (bsc#1213518) (#598) * Fix calculation of SLS context vars when trailing dots on targetted state * Add changelog file --- changelog/63411.fixed.md | 1 + salt/utils/templates.py | 5 +++-- tests/unit/utils/test_templates.py | 14 ++++++++++++++ 3 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 changelog/63411.fixed.md diff --git a/changelog/63411.fixed.md b/changelog/63411.fixed.md new file mode 100644 index 0000000000..65340e3652 --- /dev/null +++ b/changelog/63411.fixed.md @@ -0,0 +1 @@ +Fix calculation of SLS context vars when trailing dots on targetted state diff --git a/salt/utils/templates.py b/salt/utils/templates.py index 4a8adf2a14..8639ea703e 100644 --- a/salt/utils/templates.py +++ b/salt/utils/templates.py @@ -113,8 +113,9 @@ def generate_sls_context(tmplpath, sls): sls_context = {} - # Normalize SLS as path. - slspath = sls.replace(".", "/") + # Normalize SLS as path and remove possible trailing slashes + # to prevent matching issues and wrong vars calculation + slspath = sls.replace(".", "/").rstrip("/") if tmplpath: # Normalize template path diff --git a/tests/unit/utils/test_templates.py b/tests/unit/utils/test_templates.py index 4ba2f52d7b..264b4ae801 100644 --- a/tests/unit/utils/test_templates.py +++ b/tests/unit/utils/test_templates.py @@ -320,6 +320,20 @@ class WrapRenderTestCase(TestCase): slspath="foo", ) + def test_generate_sls_context__one_level_init_implicit_with_trailing_dot(self): + """generate_sls_context - Basic one level with implicit init.sls with trailing dot""" + self._test_generated_sls_context( + "/tmp/foo/init.sls", + "foo.", + tplfile="foo/init.sls", + tpldir="foo", + tpldot="foo", + slsdotpath="foo", + slscolonpath="foo", + sls_path="foo", + slspath="foo", + ) + def test_generate_sls_context__one_level_init_explicit(self): """generate_sls_context - Basic one level with explicit init.sls""" self._test_generated_sls_context( -- 2.42.0 ++++++ fix-cve-2023-34049-bsc-1215157.patch ++++++ ++++ 1164 lines (skipped) ++++++ fix-gitfs-__env__-and-improve-cache-cleaning-bsc-119.patch ++++++ ++++ 2025 lines (skipped) ++++++ fix-optimization_order-opt-to-prevent-test-fails.patch ++++++ >From aaf593d17f51a517e0adb6e9ec1c0d768ab5f855 Mon Sep 17 00:00:00 2001 From: Victor Zhestkov <[email protected]> Date: Mon, 2 Oct 2023 14:24:27 +0200 Subject: [PATCH] Fix optimization_order opt to prevent test fails --- tests/pytests/unit/grains/test_core.py | 4 ++-- tests/pytests/unit/loader/test_loader.py | 2 +- tests/pytests/unit/test_config.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/pytests/unit/grains/test_core.py b/tests/pytests/unit/grains/test_core.py index 993c723950..36545287b9 100644 --- a/tests/pytests/unit/grains/test_core.py +++ b/tests/pytests/unit/grains/test_core.py @@ -156,7 +156,7 @@ def test_network_grains_secondary_ip(tmp_path): opts = { "cachedir": str(cache_dir), "extension_modules": str(extmods), - "optimization_order": [0], + "optimization_order": [0, 1, 2], } with patch("salt.utils.network.interfaces", side_effect=[data]): grains = salt.loader.grain_funcs(opts) @@ -243,7 +243,7 @@ def test_network_grains_cache(tmp_path): opts = { "cachedir": str(cache_dir), "extension_modules": str(extmods), - "optimization_order": [0], + "optimization_order": [0, 1, 2], } with patch( "salt.utils.network.interfaces", side_effect=[call_1, call_2] diff --git a/tests/pytests/unit/loader/test_loader.py b/tests/pytests/unit/loader/test_loader.py index f4a4b51a58..86348749db 100644 --- a/tests/pytests/unit/loader/test_loader.py +++ b/tests/pytests/unit/loader/test_loader.py @@ -57,7 +57,7 @@ def test_raw_mod_functions(): "Ensure functions loaded by raw_mod are LoaderFunc instances" opts = { "extension_modules": "", - "optimization_order": [0], + "optimization_order": [0, 1, 2], } ret = salt.loader.raw_mod(opts, "grains", "get") for k, v in ret.items(): diff --git a/tests/pytests/unit/test_config.py b/tests/pytests/unit/test_config.py index cb343cb75e..76d5605360 100644 --- a/tests/pytests/unit/test_config.py +++ b/tests/pytests/unit/test_config.py @@ -16,7 +16,7 @@ def test_call_id_function(tmp_path): "cachedir": str(cache_dir), "extension_modules": str(extmods), "grains": {"osfinger": "meh"}, - "optimization_order": [0], + "optimization_order": [0, 1, 2], } ret = salt.config.call_id_function(opts) assert ret == "meh" -- 2.42.0 ++++++ fix-the-aptpkg.py-unit-test-failure.patch ++++++ >From 4bc3be7814daf5365d63b88f164f791ea53b418f Mon Sep 17 00:00:00 2001 From: Marek Czernek <[email protected]> Date: Wed, 17 Jan 2024 15:04:53 +0100 Subject: [PATCH] Fix the aptpkg.py unit test failure --- salt/modules/aptpkg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/aptpkg.py b/salt/modules/aptpkg.py index 9885e9fb60..ad5450c415 100644 --- a/salt/modules/aptpkg.py +++ b/salt/modules/aptpkg.py @@ -3128,7 +3128,7 @@ def expand_repo_def(**kwargs): NOT USABLE IN THE CLI """ warn_until_date( - "20240101", + "20250101", "The pkg.expand_repo_def function is deprecated and set for removal " "after {date}. This is only unsed internally by the apt pkg state " "module. If that's not the case, please file an new issue requesting " -- 2.43.0 ++++++ fixed-keyerror-in-logs-when-running-a-state-that-fai.patch ++++++ >From f41a8e2a142a8487e13af481990928e0afb5f15e Mon Sep 17 00:00:00 2001 From: Victor Zhestkov <[email protected]> Date: Thu, 18 Jan 2024 17:02:03 +0100 Subject: [PATCH] Fixed KeyError in logs when running a state that fails. (#615) Co-authored-by: Megan Wilhite <[email protected]> --- changelog/64231.fixed.md | 1 + salt/master.py | 2 +- salt/minion.py | 4 ++ salt/utils/event.py | 3 +- .../integration/states/test_state_test.py | 38 +++++++++++++++++++ 5 files changed, 46 insertions(+), 2 deletions(-) create mode 100644 changelog/64231.fixed.md create mode 100644 tests/pytests/integration/states/test_state_test.py diff --git a/changelog/64231.fixed.md b/changelog/64231.fixed.md new file mode 100644 index 0000000000..0991c5a8b9 --- /dev/null +++ b/changelog/64231.fixed.md @@ -0,0 +1 @@ +Fixed KeyError in logs when running a state that fails. diff --git a/salt/master.py b/salt/master.py index fc243ef674..3d2ba1e29d 100644 --- a/salt/master.py +++ b/salt/master.py @@ -1790,7 +1790,7 @@ class AESFuncs(TransportMethods): def pub_ret(self, load): """ Request the return data from a specific jid, only allowed - if the requesting minion also initialted the execution. + if the requesting minion also initiated the execution. :param dict load: The minion payload diff --git a/salt/minion.py b/salt/minion.py index 4db0d31bd4..2ccd0cd5a9 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -2022,6 +2022,8 @@ class Minion(MinionBase): ret["jid"] = data["jid"] ret["fun"] = data["fun"] ret["fun_args"] = data["arg"] + if "user" in data: + ret["user"] = data["user"] if "master_id" in data: ret["master_id"] = data["master_id"] if "metadata" in data: @@ -2141,6 +2143,8 @@ class Minion(MinionBase): ret["jid"] = data["jid"] ret["fun"] = data["fun"] ret["fun_args"] = data["arg"] + if "user" in data: + ret["user"] = data["user"] if "metadata" in data: ret["metadata"] = data["metadata"] if minion_instance.connected: diff --git a/salt/utils/event.py b/salt/utils/event.py index 869e12a140..e6d7b00520 100644 --- a/salt/utils/event.py +++ b/salt/utils/event.py @@ -902,7 +902,8 @@ class SaltEvent: data["success"] = False data["return"] = "Error: {}.{}".format(tags[0], tags[-1]) data["fun"] = fun - data["user"] = load["user"] + if "user" in load: + data["user"] = load["user"] self.fire_event( data, tagify([load["jid"], "sub", load["id"], "error", fun], "job"), diff --git a/tests/pytests/integration/states/test_state_test.py b/tests/pytests/integration/states/test_state_test.py new file mode 100644 index 0000000000..b2328a4c2b --- /dev/null +++ b/tests/pytests/integration/states/test_state_test.py @@ -0,0 +1,38 @@ +def test_failing_sls(salt_master, salt_minion, salt_cli, caplog): + """ + Test when running state.sls and the state fails. + When the master stores the job and attempts to send + an event a KeyError was previously being logged. + This test ensures we do not log an error when + attempting to send an event about a failing state. + """ + statesls = """ + test_state: + test.fail_without_changes: + - name: "bla" + """ + with salt_master.state_tree.base.temp_file("test_failure.sls", statesls): + ret = salt_cli.run("state.sls", "test_failure", minion_tgt=salt_minion.id) + for message in caplog.messages: + assert "Event iteration failed with" not in message + + +def test_failing_sls_compound(salt_master, salt_minion, salt_cli, caplog): + """ + Test when running state.sls in a compound command and the state fails. + When the master stores the job and attempts to send + an event a KeyError was previously being logged. + This test ensures we do not log an error when + attempting to send an event about a failing state. + """ + statesls = """ + test_state: + test.fail_without_changes: + - name: "bla" + """ + with salt_master.state_tree.base.temp_file("test_failure.sls", statesls): + ret = salt_cli.run( + "state.sls,cmd.run", "test_failure,ls", minion_tgt=salt_minion.id + ) + for message in caplog.messages: + assert "Event iteration failed with" not in message -- 2.43.0 ++++++ implement-the-calling-for-batch-async-from-the-salt-.patch ++++++ >From 7ab208fd2d23eaa582cdbba912d4538d8c87e5f4 Mon Sep 17 00:00:00 2001 From: Victor Zhestkov <[email protected]> Date: Mon, 2 Oct 2023 13:24:15 +0200 Subject: [PATCH] Implement the calling for batch async from the salt CLI * Implement calling batch async with salt CLI * Add the test for calling batch async with salt CLI --- salt/cli/salt.py | 53 ++++++++++++++++++++++++++++- tests/pytests/unit/cli/test_salt.py | 50 +++++++++++++++++++++++++++ 2 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 tests/pytests/unit/cli/test_salt.py diff --git a/salt/cli/salt.py b/salt/cli/salt.py index f90057f668..e19cfa5ce6 100644 --- a/salt/cli/salt.py +++ b/salt/cli/salt.py @@ -47,7 +47,12 @@ class SaltCMD(salt.utils.parsers.SaltCMDOptionParser): self.exit(2, "{}\n".format(exc)) return - if self.options.batch or self.options.static: + if self.options.batch and self.config["async"]: + # _run_batch_async() will just return the jid and exit + # Execution will not continue past this point + # in batch async mode. Batch async is handled by the master. + self._run_batch_async() + elif self.options.batch or self.options.static: # _run_batch() will handle all output and # exit with the appropriate error condition # Execution will not continue past this point @@ -296,6 +301,52 @@ class SaltCMD(salt.utils.parsers.SaltCMDOptionParser): retcode = job_retcode sys.exit(retcode) + def _run_batch_async(self): + kwargs = { + "tgt": self.config["tgt"], + "fun": self.config["fun"], + "arg": self.config["arg"], + "timeout": self.options.timeout, + "show_timeout": self.options.show_timeout, + "show_jid": self.options.show_jid, + "batch": self.config["batch"], + } + tgt = kwargs.pop("tgt", "") + fun = kwargs.pop("fun", "") + + if self.config.get("eauth", ""): + kwargs.update( + { + "eauth": self.config["eauth"], + } + ) + for opt in ("username", "password"): + if opt in self.config: + kwargs[opt] = self.config[opt] + + try: + ret = self.local_client.run_job(tgt, fun, **kwargs) + except ( + AuthenticationError, + AuthorizationError, + SaltInvocationError, + EauthAuthenticationError, + SaltClientError, + ) as exc: + ret = str(exc) + self.exit(2, "ERROR: {}\n".format(exc)) + if "jid" in ret and "error" not in ret: + salt.utils.stringutils.print_cli( + "Executed command with job ID: {}".format(ret["jid"]) + ) + else: + self._output_ret(ret, self.config.get("output", "nested")) + + if "error" in ret: + sys.exit(1) + + sys.exit(0) + def _print_errors_summary(self, errors): if errors: salt.utils.stringutils.print_cli("\n") diff --git a/tests/pytests/unit/cli/test_salt.py b/tests/pytests/unit/cli/test_salt.py new file mode 100644 index 0000000000..d9f4b5b097 --- /dev/null +++ b/tests/pytests/unit/cli/test_salt.py @@ -0,0 +1,50 @@ +import pytest + +from tests.support.mock import MagicMock, patch + + +def test_saltcmd_batch_async_call(): + """ + Test calling batch async with salt CLI + """ + import salt.cli.salt + + local_client = MagicMock() + local_client.run_job = MagicMock(return_value={"jid": 123456}) + with pytest.raises(SystemExit) as exit_info, patch( + "sys.argv", + [ + "salt", + "--batch=10", + "--async", + "*", + "test.arg", + "arg1", + "arg2", + "kwarg1=val1", + ], + ), patch("salt.cli.salt.SaltCMD.process_config_dir", MagicMock), patch( + "salt.output.display_output", MagicMock() + ), patch( + "salt.client.get_local_client", return_value=local_client + ), patch( + "salt.utils.stringutils.print_cli", MagicMock() + ) as print_cli: + salt_cmd = salt.cli.salt.SaltCMD() + salt_cmd.config = { + "async": True, + "batch": 10, + "tgt": "*", + "fun": "test.arg", + "arg": ["arg1", "arg2", {"__kwarg__": True, "kwarg1": "val1"}], + } + salt_cmd._mixin_after_parsed_funcs = [] + salt_cmd.run() + + local_client.run_job.assert_called_once() + assert local_client.run_job.mock_calls[0].args[0] == "*" + assert local_client.run_job.mock_calls[0].args[1] == "test.arg" + assert local_client.run_job.mock_calls[0].kwargs["arg"] == ["arg1", "arg2", {"__kwarg__": True, "kwarg1": "val1"}] + assert local_client.run_job.mock_calls[0].kwargs["batch"] == 10 + print_cli.assert_called_once_with("Executed command with job ID: 123456") + assert exit_info.value.code == 0 -- 2.42.0 ++++++ improve-pip-target-override-condition-with-venv_pip_.patch ++++++ >From da938aa8a572138b5b9b1535c5c3d69326e5194e Mon Sep 17 00:00:00 2001 From: Victor Zhestkov <[email protected]> Date: Thu, 18 Jan 2024 17:02:23 +0100 Subject: [PATCH] Improve pip target override condition with VENV_PIP_TARGET environment variable (bsc#1216850) (#613) * Improve pip target override condition * Improve pip test with different condition of overriding the target * Add changelog entry --- changelog/65562.fixed.md | 1 + salt/modules/pip.py | 6 ++-- tests/pytests/unit/modules/test_pip.py | 50 +++++++++++++++++--------- 3 files changed, 38 insertions(+), 19 deletions(-) create mode 100644 changelog/65562.fixed.md diff --git a/changelog/65562.fixed.md b/changelog/65562.fixed.md new file mode 100644 index 0000000000..ba483b4b77 --- /dev/null +++ b/changelog/65562.fixed.md @@ -0,0 +1 @@ +Improve the condition of overriding target for pip with VENV_PIP_TARGET environment variable. diff --git a/salt/modules/pip.py b/salt/modules/pip.py index a60bdca0bb..68a2a442a1 100644 --- a/salt/modules/pip.py +++ b/salt/modules/pip.py @@ -857,9 +857,11 @@ def install( cmd.extend(["--build", build]) # Use VENV_PIP_TARGET environment variable value as target - # if set and no target specified on the function call + # if set and no target specified on the function call. + # Do not set target if bin_env specified, use default + # for specified binary environment or expect explicit target specification. target_env = os.environ.get("VENV_PIP_TARGET", None) - if target is None and target_env is not None: + if target is None and target_env is not None and bin_env is None: target = target_env if target: diff --git a/tests/pytests/unit/modules/test_pip.py b/tests/pytests/unit/modules/test_pip.py index b7ad1ea3fd..c03e6ed292 100644 --- a/tests/pytests/unit/modules/test_pip.py +++ b/tests/pytests/unit/modules/test_pip.py @@ -1738,28 +1738,44 @@ def test_when_version_is_called_with_a_user_it_should_be_passed_to_undelying_run ) -def test_install_target_from_VENV_PIP_TARGET_in_resulting_command(python_binary): [email protected]( + "bin_env,target,target_env,expected_target", + [ + (None, None, None, None), + (None, "/tmp/foo", None, "/tmp/foo"), + (None, None, "/tmp/bar", "/tmp/bar"), + (None, "/tmp/foo", "/tmp/bar", "/tmp/foo"), + ("/tmp/venv", "/tmp/foo", None, "/tmp/foo"), + ("/tmp/venv", None, "/tmp/bar", None), + ("/tmp/venv", "/tmp/foo", "/tmp/bar", "/tmp/foo"), + ], +) +def test_install_target_from_VENV_PIP_TARGET_in_resulting_command( + python_binary, bin_env, target, target_env, expected_target +): pkg = "pep8" - target = "/tmp/foo" - target_env = "/tmp/bar" mock = MagicMock(return_value={"retcode": 0, "stdout": ""}) environment = os.environ.copy() - environment["VENV_PIP_TARGET"] = target_env + real_get_pip_bin = pip._get_pip_bin + + def mock_get_pip_bin(bin_env): + if not bin_env: + return real_get_pip_bin(bin_env) + return [f"{bin_env}/bin/pip"] + + if target_env is not None: + environment["VENV_PIP_TARGET"] = target_env with patch.dict(pip.__salt__, {"cmd.run_all": mock}), patch.object( os, "environ", environment - ): - pip.install(pkg) - expected = [*python_binary, "install", "--target", target_env, pkg] - mock.assert_called_with( - expected, - saltenv="base", - runas=None, - use_vt=False, - python_shell=False, - ) - mock.reset_mock() - pip.install(pkg, target=target) - expected = [*python_binary, "install", "--target", target, pkg] + ), patch.object(pip, "_get_pip_bin", mock_get_pip_bin): + pip.install(pkg, bin_env=bin_env, target=target) + expected_binary = python_binary + if bin_env is not None: + expected_binary = [f"{bin_env}/bin/pip"] + if expected_target is not None: + expected = [*expected_binary, "install", "--target", expected_target, pkg] + else: + expected = [*expected_binary, "install", pkg] mock.assert_called_with( expected, saltenv="base", -- 2.43.0 ++++++ improve-salt.utils.json.find_json-bsc-1213293.patch ++++++ >From 4e6b445f2dbe8a79d220c697abff946e00b2e57b Mon Sep 17 00:00:00 2001 From: Victor Zhestkov <[email protected]> Date: Mon, 2 Oct 2023 13:26:20 +0200 Subject: [PATCH] Improve salt.utils.json.find_json (bsc#1213293) * Improve salt.utils.json.find_json * Move tests of find_json to pytest --- salt/utils/json.py | 39 +++++++- tests/pytests/unit/utils/test_json.py | 122 ++++++++++++++++++++++++++ 2 files changed, 158 insertions(+), 3 deletions(-) create mode 100644 tests/pytests/unit/utils/test_json.py diff --git a/salt/utils/json.py b/salt/utils/json.py index 33cdbf401d..0845b64694 100644 --- a/salt/utils/json.py +++ b/salt/utils/json.py @@ -32,18 +32,51 @@ def find_json(raw): """ ret = {} lines = __split(raw) + lengths = list(map(len, lines)) + starts = [] + ends = [] + + # Search for possible starts end ends of the json fragments for ind, _ in enumerate(lines): + line = lines[ind].lstrip() + if line == "{" or line == "[": + starts.append((ind, line)) + if line == "}" or line == "]": + ends.append((ind, line)) + + # List all the possible pairs of starts and ends, + # and fill the length of each block to sort by size after + starts_ends = [] + for start, start_br in starts: + for end, end_br in reversed(ends): + if end > start and ( + (start_br == "{" and end_br == "}") + or (start_br == "[" and end_br == "]") + ): + starts_ends.append((start, end, sum(lengths[start : end + 1]))) + + # Iterate through all the possible pairs starting from the largest + starts_ends.sort(key=lambda x: (x[2], x[1] - x[0], x[0]), reverse=True) + for start, end, _ in starts_ends: + working = "\n".join(lines[start : end + 1]) try: - working = "\n".join(lines[ind:]) - except UnicodeDecodeError: - working = "\n".join(salt.utils.data.decode(lines[ind:])) + ret = json.loads(working) + except ValueError: + continue + if ret: + return ret + # Fall back to old implementation for backward compatibility + # excpecting json after the text + for ind, _ in enumerate(lines): + working = "\n".join(lines[ind:]) try: ret = json.loads(working) except ValueError: continue if ret: return ret + if not ret: # Not json, raise an error raise ValueError diff --git a/tests/pytests/unit/utils/test_json.py b/tests/pytests/unit/utils/test_json.py new file mode 100644 index 0000000000..72b1023003 --- /dev/null +++ b/tests/pytests/unit/utils/test_json.py @@ -0,0 +1,122 @@ +""" +Tests for salt.utils.json +""" + +import textwrap + +import pytest + +import salt.utils.json + + +def test_find_json(): + some_junk_text = textwrap.dedent( + """ + Just some junk text + with multiline + """ + ) + some_warning_message = textwrap.dedent( + """ + [WARNING] Test warning message + """ + ) + test_small_json = textwrap.dedent( + """ + { + "local": true + } + """ + ) + test_sample_json = """ + { + "glossary": { + "title": "example glossary", + "GlossDiv": { + "title": "S", + "GlossList": { + "GlossEntry": { + "ID": "SGML", + "SortAs": "SGML", + "GlossTerm": "Standard Generalized Markup Language", + "Acronym": "SGML", + "Abbrev": "ISO 8879:1986", + "GlossDef": { + "para": "A meta-markup language, used to create markup languages such as DocBook.", + "GlossSeeAlso": ["GML", "XML"] + }, + "GlossSee": "markup" + } + } + } + } + } + """ + expected_ret = { + "glossary": { + "GlossDiv": { + "GlossList": { + "GlossEntry": { + "GlossDef": { + "GlossSeeAlso": ["GML", "XML"], + "para": ( + "A meta-markup language, used to create markup" + " languages such as DocBook." + ), + }, + "GlossSee": "markup", + "Acronym": "SGML", + "GlossTerm": "Standard Generalized Markup Language", + "SortAs": "SGML", + "Abbrev": "ISO 8879:1986", + "ID": "SGML", + } + }, + "title": "S", + }, + "title": "example glossary", + } + } + + # First test the valid JSON + ret = salt.utils.json.find_json(test_sample_json) + assert ret == expected_ret + + # Now pre-pend some garbage and re-test + garbage_prepend_json = f"{some_junk_text}{test_sample_json}" + ret = salt.utils.json.find_json(garbage_prepend_json) + assert ret == expected_ret + + # Now post-pend some garbage and re-test + garbage_postpend_json = f"{test_sample_json}{some_junk_text}" + ret = salt.utils.json.find_json(garbage_postpend_json) + assert ret == expected_ret + + # Now pre-pend some warning and re-test + warning_prepend_json = f"{some_warning_message}{test_sample_json}" + ret = salt.utils.json.find_json(warning_prepend_json) + assert ret == expected_ret + + # Now post-pend some warning and re-test + warning_postpend_json = f"{test_sample_json}{some_warning_message}" + ret = salt.utils.json.find_json(warning_postpend_json) + assert ret == expected_ret + + # Now put around some garbage and re-test + garbage_around_json = f"{some_junk_text}{test_sample_json}{some_junk_text}" + ret = salt.utils.json.find_json(garbage_around_json) + assert ret == expected_ret + + # Now pre-pend small json and re-test + small_json_pre_json = f"{test_small_json}{test_sample_json}" + ret = salt.utils.json.find_json(small_json_pre_json) + assert ret == expected_ret + + # Now post-pend small json and re-test + small_json_post_json = f"{test_sample_json}{test_small_json}" + ret = salt.utils.json.find_json(small_json_post_json) + assert ret == expected_ret + + # Test to see if a ValueError is raised if no JSON is passed in + with pytest.raises(ValueError): + ret = salt.utils.json.find_json(some_junk_text) -- 2.42.0 ++++++ only-call-native_str-on-curl_debug-message-in-tornad.patch ++++++ >From b76b74bd9640adf3b6798e4de4b89aaa7af62c9f Mon Sep 17 00:00:00 2001 From: Victor Zhestkov <[email protected]> Date: Mon, 2 Oct 2023 13:24:43 +0200 Subject: [PATCH] Only call native_str on curl_debug message in tornado when needed Co-authored-by: Ben Darnell <[email protected]> --- salt/ext/tornado/curl_httpclient.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/ext/tornado/curl_httpclient.py b/salt/ext/tornado/curl_httpclient.py index 8652343cf7..9e4133fd13 100644 --- a/salt/ext/tornado/curl_httpclient.py +++ b/salt/ext/tornado/curl_httpclient.py @@ -494,10 +494,11 @@ class CurlAsyncHTTPClient(AsyncHTTPClient): def _curl_debug(self, debug_type, debug_msg): debug_types = ('I', '<', '>', '<', '>') - debug_msg = native_str(debug_msg) if debug_type == 0: + debug_msg = native_str(debug_msg) curl_log.debug('%s', debug_msg.strip()) elif debug_type in (1, 2): + debug_msg = native_str(debug_msg) for line in debug_msg.splitlines(): curl_log.debug('%s %s', debug_types[debug_type], line) elif debug_type == 4: -- 2.42.0 ++++++ prefer-unittest.mock-for-python-versions-that-are-su.patch ++++++ >From 107de57586f0b0f784771543b942dfb6bb70453a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yeray=20Guti=C3=A9rrez=20Cedr=C3=A9s?= <[email protected]> Date: Wed, 13 Dec 2023 11:03:45 +0000 Subject: [PATCH] Prefer unittest.mock for Python versions that are sufficient --- requirements/pytest.txt | 2 +- .../unit/cloud/clouds/test_dimensiondata.py | 4 +- tests/pytests/unit/cloud/clouds/test_gce.py | 4 +- tests/support/mock.py | 48 +++++++++---------- 4 files changed, 25 insertions(+), 33 deletions(-) diff --git a/requirements/pytest.txt b/requirements/pytest.txt index 5b67583a3d..0bead83f5b 100644 --- a/requirements/pytest.txt +++ b/requirements/pytest.txt @@ -1,4 +1,4 @@ -mock >= 3.0.0 +mock >= 3.0.0; python_version < '3.8' # PyTest pytest >= 7.0.1; python_version <= "3.6" pytest >= 7.2.0; python_version > "3.6" diff --git a/tests/pytests/unit/cloud/clouds/test_dimensiondata.py b/tests/pytests/unit/cloud/clouds/test_dimensiondata.py index e196805004..aab2e686f2 100644 --- a/tests/pytests/unit/cloud/clouds/test_dimensiondata.py +++ b/tests/pytests/unit/cloud/clouds/test_dimensiondata.py @@ -11,7 +11,6 @@ from salt.cloud.clouds import dimensiondata from salt.exceptions import SaltCloudSystemExit from salt.utils.versions import Version from tests.support.mock import MagicMock -from tests.support.mock import __version__ as mock_version from tests.support.mock import patch try: @@ -144,8 +143,7 @@ def test_import(): with patch("salt.config.check_driver_dependencies", return_value=True) as p: get_deps = dimensiondata.get_dependencies() assert get_deps is True - if Version(mock_version) >= Version("2.0.0"): - assert p.call_count >= 1 + assert p.call_count >= 1 def test_provider_matches(): diff --git a/tests/pytests/unit/cloud/clouds/test_gce.py b/tests/pytests/unit/cloud/clouds/test_gce.py index 265818016e..ec1346a978 100644 --- a/tests/pytests/unit/cloud/clouds/test_gce.py +++ b/tests/pytests/unit/cloud/clouds/test_gce.py @@ -13,7 +13,6 @@ from salt.cloud.clouds import gce from salt.exceptions import SaltCloudSystemExit from salt.utils.versions import Version from tests.support.mock import MagicMock -from tests.support.mock import __version__ as mock_version from tests.support.mock import call, patch @@ -281,8 +280,7 @@ def test_import(): with patch("salt.config.check_driver_dependencies", return_value=True) as p: get_deps = gce.get_dependencies() assert get_deps is True - if Version(mock_version) >= Version("2.0.0"): - p.assert_called_once() + p.assert_called_once() @pytest.mark.parametrize( diff --git a/tests/support/mock.py b/tests/support/mock.py index 2256ad8f5d..59e5fcbc8e 100644 --- a/tests/support/mock.py +++ b/tests/support/mock.py @@ -18,37 +18,33 @@ import copy import errno import fnmatch import sys - -# By these days, we should blowup if mock is not available -import mock # pylint: disable=blacklisted-external-import - -# pylint: disable=no-name-in-module,no-member -from mock import ( - ANY, - DEFAULT, - FILTER_DIR, - MagicMock, - Mock, - NonCallableMagicMock, - NonCallableMock, - PropertyMock, - __version__, - call, - create_autospec, - patch, - sentinel, -) +import importlib + +current_version = (sys.version_info.major, sys.version_info.minor) + +# Prefer unittest.mock for Python versions that are sufficient +if current_version >= (3,8): + mock = importlib.import_module('unittest.mock') +else: + mock = importlib.import_module('mock') + +ANY = mock.ANY +DEFAULT = mock.DEFAULT +FILTER_DIR = mock.FILTER_DIR +MagicMock = mock.MagicMock +Mock = mock.Mock +NonCallableMagicMock = mock.NonCallableMagicMock +NonCallableMock = mock.NonCallableMock +PropertyMock = mock.PropertyMock +call = mock.call +create_autospec = mock.create_autospec +patch = mock.patch +sentinel = mock.sentinel import salt.utils.stringutils # pylint: disable=no-name-in-module,no-member - -__mock_version = tuple( - int(part) for part in mock.__version__.split(".") if part.isdigit() -) # pylint: disable=no-member - - class MockFH: def __init__(self, filename, read_data, *args, **kwargs): self.filename = filename -- 2.41.0 ++++++ revert-make-sure-configured-user-is-properly-set-by-.patch ++++++ >From d9980c8d2cfedfd6f08543face6ee7e34e9d1b54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20Su=C3=A1rez=20Hern=C3=A1ndez?= <[email protected]> Date: Thu, 16 Nov 2023 09:23:58 +0000 Subject: [PATCH] Revert "Make sure configured user is properly set by Salt (bsc#1210994) (#596)" (#614) This reverts commit 5ea4add5c8e2bed50b9825edfff7565e5f6124f3. --- pkg/common/salt-master.service | 1 - pkg/old/deb/salt-master.service | 1 - pkg/old/suse/salt-master.service | 1 - salt/cli/daemons.py | 27 ------------------- salt/cli/ssh.py | 8 ------ salt/utils/verify.py | 4 +-- .../integration/cli/test_salt_minion.py | 4 +-- 7 files changed, 4 insertions(+), 42 deletions(-) diff --git a/pkg/common/salt-master.service b/pkg/common/salt-master.service index 257ecc283f..377c87afeb 100644 --- a/pkg/common/salt-master.service +++ b/pkg/common/salt-master.service @@ -8,7 +8,6 @@ LimitNOFILE=100000 Type=notify NotifyAccess=all ExecStart=/usr/bin/salt-master -User=salt [Install] WantedBy=multi-user.target diff --git a/pkg/old/deb/salt-master.service b/pkg/old/deb/salt-master.service index f9dca296b4..b5d0cdd22c 100644 --- a/pkg/old/deb/salt-master.service +++ b/pkg/old/deb/salt-master.service @@ -7,7 +7,6 @@ LimitNOFILE=16384 Type=notify NotifyAccess=all ExecStart=/usr/bin/salt-master -User=salt [Install] WantedBy=multi-user.target diff --git a/pkg/old/suse/salt-master.service b/pkg/old/suse/salt-master.service index caabca511c..9e002d16ca 100644 --- a/pkg/old/suse/salt-master.service +++ b/pkg/old/suse/salt-master.service @@ -8,7 +8,6 @@ LimitNOFILE=100000 Type=simple ExecStart=/usr/bin/salt-master TasksMax=infinity -User=salt [Install] WantedBy=multi-user.target diff --git a/salt/cli/daemons.py b/salt/cli/daemons.py index c9ee9ced91..ecc05c919e 100644 --- a/salt/cli/daemons.py +++ b/salt/cli/daemons.py @@ -7,7 +7,6 @@ import logging import os import warnings -import salt.defaults.exitcodes import salt.utils.kinds as kinds from salt.exceptions import SaltClientError, SaltSystemExit, get_error_message from salt.utils import migrations @@ -74,16 +73,6 @@ class DaemonsMixin: # pylint: disable=no-init self.__class__.__name__, ) - def verify_user(self): - """ - Verify Salt configured user for Salt and shutdown daemon if not valid. - - :return: - """ - if not check_user(self.config["user"]): - self.action_log_info("Cannot switch to configured user for Salt. Exiting") - self.shutdown(salt.defaults.exitcodes.EX_NOUSER) - def action_log_info(self, action): """ Say daemon starting. @@ -189,10 +178,6 @@ class Master( self.config["interface"] = ip_bracket(self.config["interface"]) migrations.migrate_paths(self.config) - # Ensure configured user is valid and environment is properly set - # before initializating rest of the stack. - self.verify_user() - # Late import so logging works correctly import salt.master @@ -305,10 +290,6 @@ class Minion( transport = self.config.get("transport").lower() - # Ensure configured user is valid and environment is properly set - # before initializating rest of the stack. - self.verify_user() - try: # Late import so logging works correctly import salt.minion @@ -497,10 +478,6 @@ class ProxyMinion( self.action_log_info("An instance is already running. Exiting") self.shutdown(1) - # Ensure configured user is valid and environment is properly set - # before initializating rest of the stack. - self.verify_user() - # TODO: AIO core is separate from transport # Late import so logging works correctly import salt.minion @@ -599,10 +576,6 @@ class Syndic( self.action_log_info('Setting up "{}"'.format(self.config["id"])) - # Ensure configured user is valid and environment is properly set - # before initializating rest of the stack. - self.verify_user() - # Late import so logging works correctly import salt.minion diff --git a/salt/cli/ssh.py b/salt/cli/ssh.py index 672f32b8c0..6048cb5f58 100644 --- a/salt/cli/ssh.py +++ b/salt/cli/ssh.py @@ -1,9 +1,7 @@ import sys import salt.client.ssh -import salt.defaults.exitcodes import salt.utils.parsers -from salt.utils.verify import check_user class SaltSSH(salt.utils.parsers.SaltSSHOptionParser): @@ -17,11 +15,5 @@ class SaltSSH(salt.utils.parsers.SaltSSHOptionParser): # that won't be used anyways with -H or --hosts self.parse_args() - if not check_user(self.config["user"]): - self.exit( - salt.defaults.exitcodes.EX_NOUSER, - "Cannot switch to configured user for Salt. Exiting", - ) - ssh = salt.client.ssh.SSH(self.config) ssh.run() diff --git a/salt/utils/verify.py b/salt/utils/verify.py index 7899fbe538..879128f231 100644 --- a/salt/utils/verify.py +++ b/salt/utils/verify.py @@ -335,8 +335,8 @@ def check_user(user): # We could just reset the whole environment but let's just override # the variables we can get from pwuser - # We ensure HOME is always present and set according to pwuser - os.environ["HOME"] = pwuser.pw_dir + if "HOME" in os.environ: + os.environ["HOME"] = pwuser.pw_dir if "SHELL" in os.environ: os.environ["SHELL"] = pwuser.pw_shell diff --git a/tests/pytests/integration/cli/test_salt_minion.py b/tests/pytests/integration/cli/test_salt_minion.py index bde2dd51d7..c0d6013474 100644 --- a/tests/pytests/integration/cli/test_salt_minion.py +++ b/tests/pytests/integration/cli/test_salt_minion.py @@ -41,7 +41,7 @@ def test_exit_status_unknown_user(salt_master, minion_id): factory = salt_master.salt_minion_daemon( minion_id, overrides={"user": "unknown-user"} ) - factory.start(start_timeout=30, max_start_attempts=1) + factory.start(start_timeout=10, max_start_attempts=1) assert exc.value.process_result.returncode == salt.defaults.exitcodes.EX_NOUSER assert "The user is not available." in exc.value.process_result.stderr @@ -53,7 +53,7 @@ def test_exit_status_unknown_argument(salt_master, minion_id): """ with pytest.raises(FactoryNotStarted) as exc: factory = salt_master.salt_minion_daemon(minion_id) - factory.start("--unknown-argument", start_timeout=30, max_start_attempts=1) + factory.start("--unknown-argument", start_timeout=10, max_start_attempts=1) assert exc.value.process_result.returncode == salt.defaults.exitcodes.EX_USAGE assert "Usage" in exc.value.process_result.stderr -- 2.42.0 ++++++ update-__pillar__-during-pillar_refresh.patch ++++++ >From 3e7c5d95423491f83d0016eb7c02285cd0b1bcf4 Mon Sep 17 00:00:00 2001 From: Marek Czernek <[email protected]> Date: Wed, 17 Jan 2024 15:39:41 +0100 Subject: [PATCH] Update __pillar__ during pillar_refresh --- changelog/63583.fixed.md | 1 + salt/minion.py | 1 + .../integration/modules/test_pillar.py | 110 +++++++++++++++++- 3 files changed, 111 insertions(+), 1 deletion(-) create mode 100644 changelog/63583.fixed.md diff --git a/changelog/63583.fixed.md b/changelog/63583.fixed.md new file mode 100644 index 0000000000..f1b6e32507 --- /dev/null +++ b/changelog/63583.fixed.md @@ -0,0 +1 @@ +Need to make sure we update __pillar__ during a pillar refresh to ensure that process_beacons has the updated beacons loaded from pillar. diff --git a/salt/minion.py b/salt/minion.py index 9597d6e63a..4db0d31bd4 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -2498,6 +2498,7 @@ class Minion(MinionBase): current_schedule, new_schedule ) self.opts["pillar"] = new_pillar + self.functions.pack["__pillar__"] = self.opts["pillar"] finally: async_pillar.destroy() self.matchers_refresh() diff --git a/tests/pytests/integration/modules/test_pillar.py b/tests/pytests/integration/modules/test_pillar.py index 66f7b9e47b..5db9a1630a 100644 --- a/tests/pytests/integration/modules/test_pillar.py +++ b/tests/pytests/integration/modules/test_pillar.py @@ -1,9 +1,14 @@ +import logging import pathlib import time +import types import attr import pytest +log = logging.getLogger(__name__) + + pytestmark = [ pytest.mark.slow_test, pytest.mark.windows_whitelisted, @@ -210,7 +215,7 @@ class PillarRefresh: "top.sls", top_file_contents ) self.minion_1_pillar = self.master.pillar_tree.base.temp_file( - "minion-1-pillar.sls", "{}: true".format(self.pillar_key) + "minion-1-pillar.sls", f"{self.pillar_key}: true" ) self.top_file.__enter__() self.minion_1_pillar.__enter__() @@ -588,3 +593,106 @@ def test_pillar_ext_59975(salt_call_cli): """ ret = salt_call_cli.run("pillar.ext", '{"libvert": _}') assert "ext_pillar_opts" in ret.data + + [email protected] +def event_listerner_timeout(grains): + if grains["os"] == "Windows": + if grains["osrelease"].startswith("2019"): + return types.SimpleNamespace(catch=120, miss=30) + return types.SimpleNamespace(catch=90, miss=10) + return types.SimpleNamespace(catch=60, miss=10) + + [email protected]_test +def test_pillar_refresh_pillar_beacons( + base_env_pillar_tree_root_dir, + salt_cli, + salt_minion, + salt_master, + event_listener, + event_listerner_timeout, +): + """ + Ensure beacons jobs in pillar are started after + a pillar refresh and then not running when pillar + is cleared. + """ + + top_sls = """ + base: + '{}': + - test_beacons + """.format( + salt_minion.id + ) + + test_beacons_sls_empty = "" + + test_beacons_sls = """ + beacons: + status: + - loadavg: + - 1-min + """ + + assert salt_minion.is_running() + + top_tempfile = pytest.helpers.temp_file( + "top.sls", top_sls, base_env_pillar_tree_root_dir + ) + beacon_tempfile = pytest.helpers.temp_file( + "test_beacons.sls", test_beacons_sls_empty, base_env_pillar_tree_root_dir + ) + + with top_tempfile, beacon_tempfile: + # Calling refresh_pillar to update in-memory pillars + salt_cli.run("saltutil.refresh_pillar", wait=True, minion_tgt=salt_minion.id) + + # Ensure beacons start when pillar is refreshed + with salt_master.pillar_tree.base.temp_file( + "test_beacons.sls", test_beacons_sls + ): + # Calling refresh_pillar to update in-memory pillars + salt_cli.run( + "saltutil.refresh_pillar", wait=True, minion_tgt=salt_minion.id + ) + + # Give the beacons a chance to start + time.sleep(5) + + event_tag = f"salt/beacon/*/status/*" + start_time = time.time() + + event_pattern = (salt_master.id, event_tag) + matched_events = event_listener.wait_for_events( + [event_pattern], + after_time=start_time, + timeout=event_listerner_timeout.catch, + ) + + assert matched_events.found_all_events + + # Ensure beacons sttop when pillar is refreshed + with salt_master.pillar_tree.base.temp_file( + "test_beacons.sls", test_beacons_sls_empty + ): + # Calling refresh_pillar to update in-memory pillars + salt_cli.run( + "saltutil.refresh_pillar", wait=True, minion_tgt=salt_minion.id + ) + + # Give the beacons a chance to stop + time.sleep(5) + + event_tag = f"salt/beacon/*/status/*" + start_time = time.time() + + event_pattern = (salt_master.id, event_tag) + matched_events = event_listener.wait_for_events( + [event_pattern], + after_time=start_time, + timeout=event_listerner_timeout.miss, + ) + + assert not matched_events.found_all_events -- 2.43.0 ++++++ use-salt-call-from-salt-bundle-with-transactional_up.patch ++++++ >From 0459d3f711eb9898f56a97d0bf0eb66fd1421a56 Mon Sep 17 00:00:00 2001 From: Victor Zhestkov <[email protected]> Date: Mon, 2 Oct 2023 13:25:52 +0200 Subject: [PATCH] Use salt-call from salt bundle with transactional_update * Use salt-call from the bundle with transactional_update * Add test checking which salt-call is selected by executable --- salt/modules/transactional_update.py | 13 +++++- .../unit/modules/test_transactional_update.py | 44 +++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/salt/modules/transactional_update.py b/salt/modules/transactional_update.py index 658ebccc6b..d6915475f5 100644 --- a/salt/modules/transactional_update.py +++ b/salt/modules/transactional_update.py @@ -276,6 +276,9 @@ transaction. """ import logging +import os.path +import pathlib +import sys import salt.client.ssh.state import salt.client.ssh.wrapper.state @@ -941,10 +944,18 @@ def call(function, *args, **kwargs): activate_transaction = kwargs.pop("activate_transaction", False) try: + # Set default salt-call command + salt_call_cmd = "salt-call" + python_exec_dir = os.path.dirname(sys.executable) + if "venv-salt-minion" in pathlib.Path(python_exec_dir).parts: + # If the module is executed with the Salt Bundle, + # use salt-call from the Salt Bundle + salt_call_cmd = os.path.join(python_exec_dir, "salt-call") + safe_kwargs = salt.utils.args.clean_kwargs(**kwargs) salt_argv = ( [ - "salt-call", + salt_call_cmd, "--out", "json", "-l", diff --git a/tests/pytests/unit/modules/test_transactional_update.py b/tests/pytests/unit/modules/test_transactional_update.py index 5d9294c49b..dbd72fd74b 100644 --- a/tests/pytests/unit/modules/test_transactional_update.py +++ b/tests/pytests/unit/modules/test_transactional_update.py @@ -670,3 +670,47 @@ def test_single_queue_true(): "salt.modules.transactional_update.call", MagicMock(return_value="result") ): assert tu.single("pkg.installed", name="emacs", queue=True) == "result" + + [email protected]( + "executable,salt_call_cmd", + [ + ("/usr/bin/python3", "salt-call"), + ( + "/usr/lib/venv-salt-minion/bin/python", + "/usr/lib/venv-salt-minion/bin/salt-call", + ), + ], +) +def test_call_which_salt_call_selected_with_executable(executable, salt_call_cmd): + """Test transactional_update.chroot which salt-call used""" + utils_mock = { + "json.find_json": MagicMock(return_value={"return": "result"}), + } + salt_mock = { + "cmd.run_all": MagicMock(return_value={"retcode": 0, "stdout": ""}), + } + with patch("sys.executable", executable), patch.dict( + tu.__utils__, utils_mock + ), patch.dict(tu.__salt__, salt_mock): + assert tu.call("test.ping") == "result" + + salt_mock["cmd.run_all"].assert_called_with( + [ + "transactional-update", + "--non-interactive", + "--drop-if-no-change", + "--no-selfupdate", + "--continue", + "--quiet", + "run", + salt_call_cmd, + "--out", + "json", + "-l", + "quiet", + "--no-return-event", + "--", + "test.ping", + ] + ) -- 2.42.0
