Script 'mail_helper' called by obssrc Hello community, here is the log from the commit of package salt for openSUSE:Factory checked in at 2025-02-25 16:41:02 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/salt (Old) and /work/SRC/openSUSE:Factory/.salt.new.1873 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "salt" Tue Feb 25 16:41:02 2025 rev:167 rq:1248212 version:3006.0 Changes: -------- --- /work/SRC/openSUSE:Factory/salt/salt.changes 2025-02-18 19:08:45.283617126 +0100 +++ /work/SRC/openSUSE:Factory/.salt.new.1873/salt.changes 2025-02-25 16:41:48.636428996 +0100 @@ -1,0 +2,38 @@ +Mon Feb 24 16:17:55 UTC 2025 - Pablo Suárez Hernández <[email protected]> + +- Fix issue of using update-alternatives with alts + +------------------------------------------------------------------- +Fri Feb 21 12:46:01 UTC 2025 - Pablo Suárez Hernández <[email protected]> + +- Fix virt_query outputter and add support for block devices +- Make _auth calls visible with master stats +- Repair mount.fstab_present always returning pending changes +- Set virtual grain in Podman systemd container +- Fix crash due wrong client reference on `SaltMakoTemplateLookup` +- Enhace batch async and fix some detected issues + +- Added: + * repair-virt_query-outputter-655.patch + * make-_auth-calls-visible-with-master-stats-696.patch + * repair-fstab_present-test-mode-702.patch + * set-virtual-grain-in-podman-systemd-container-703.patch + * fixed-file-client-private-attribute-reference-on-sal.patch + * backport-batch-async-fixes-and-improvements-701.patch + +------------------------------------------------------------------- +Wed Feb 19 16:06:43 UTC 2025 - Pablo Suárez Hernández <[email protected]> + +- Enhacement of Salt packaging + * Use update-alternatives for all salt scripts + * Use flexible dependencies for the subpackages + * Make salt-minion to require flavored zypp-plugin + * Make zyppnotify to use update-alternatives + * Drop unused yumnotify plugin + * Add dependency to python3-dnf-plugins-core for RHEL based +- Fix tests failures after "repo.saltproject.io" deprecation + +- Added: + * fix-tests-failures-after-repo.saltproject.io-depreca.patch + +------------------------------------------------------------------- New: ---- backport-batch-async-fixes-and-improvements-701.patch fix-tests-failures-after-repo.saltproject.io-depreca.patch fixed-file-client-private-attribute-reference-on-sal.patch make-_auth-calls-visible-with-master-stats-696.patch repair-fstab_present-test-mode-702.patch repair-virt_query-outputter-655.patch set-virtual-grain-in-podman-systemd-container-703.patch BETA DEBUG BEGIN: New: * fixed-file-client-private-attribute-reference-on-sal.patch * backport-batch-async-fixes-and-improvements-701.patch New:- Added: * fix-tests-failures-after-repo.saltproject.io-depreca.patch New: * set-virtual-grain-in-podman-systemd-container-703.patch * fixed-file-client-private-attribute-reference-on-sal.patch * backport-batch-async-fixes-and-improvements-701.patch New: * repair-virt_query-outputter-655.patch * make-_auth-calls-visible-with-master-stats-696.patch * repair-fstab_present-test-mode-702.patch New: * make-_auth-calls-visible-with-master-stats-696.patch * repair-fstab_present-test-mode-702.patch * set-virtual-grain-in-podman-systemd-container-703.patch New:- Added: * repair-virt_query-outputter-655.patch * make-_auth-calls-visible-with-master-stats-696.patch New: * repair-fstab_present-test-mode-702.patch * set-virtual-grain-in-podman-systemd-container-703.patch * fixed-file-client-private-attribute-reference-on-sal.patch BETA DEBUG END: ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ salt.spec ++++++ --- /var/tmp/diff_new_pack.Yswh2k/_old 2025-02-25 16:41:51.160534449 +0100 +++ /var/tmp/diff_new_pack.Yswh2k/_new 2025-02-25 16:41:51.164534617 +0100 @@ -34,7 +34,7 @@ %define psuffix %{nil} %endif -%if 0%{?suse_version} > 1210 || 0%{?rhel} >= 7 || 0%{?fedora} >=28 +%if 0%{?suse_version} > 1210 || 0%{?rhel} >= 7 || 0%{?fedora} >= 28 %bcond_without systemd %else %bcond_with systemd @@ -58,6 +58,7 @@ %{?sle15allpythons} %define skip_python2 1 %if 0%{?rhel} == 8 || (0%{?suse_version} == 1500 && 0%{?sle_version} < 150400) +%define singlespec_compat 1 %define __python3_bin_suffix 3.6 %if 0%{?rhel} == 8 %define __python3 /usr/libexec/platform-python @@ -77,7 +78,6 @@ args = args:gsub("$python", python_bin)\ print(rpm.expand(args .. "\\n"))\ } -%define _nosinglespec 1 %endif Name: salt%{psuffix} Version: 3006.0 @@ -509,6 +509,21 @@ Patch152: update-for-deprecation-of-hex-in-pygit2-1.15.0-and-a.patch # PATCH-FIX_UPSTREAM: https://github.com/saltstack/salt/pull/67186 Patch153: fix-failed-to-stat-root-.gitconfig-issue-on-gitfs-bs.patch +# PATCH-FIX_UPSTREAM: https://github.com/saltstack/salt/commit/58f448405b7f46505b2047ecda72abb42b6df9d1 +# PATCH-FIX_UPSTREAM: https://github.com/saltstack/salt/commit/79d4ff772a162b5b8e602e3437c13b90a25bc190 +Patch154: fix-tests-failures-after-repo.saltproject.io-depreca.patch +# PATCH-FIX_UPSTREAM: https://github.com/saltstack/salt/pull/60269 +Patch155: backport-batch-async-fixes-and-improvements-701.patch +# PATCH-FIX_UPSTREAM: https://github.com/saltstack/salt/commit/560ab52ccf94c7974d5a418dfbba7409e0493066 +Patch156: fixed-file-client-private-attribute-reference-on-sal.patch +# PATCH-FIX_UPSTREAM: https://github.com/saltstack/salt/pull/67734 +Patch157: set-virtual-grain-in-podman-systemd-container-703.patch +# PATCH-FIX_UPSTREAM: https://github.com/saltstack/salt/pull/67066 +Patch158: repair-fstab_present-test-mode-702.patch +# PATCH-FIX_UPSTREAM: https://github.com/saltstack/salt/pull/67746 +Patch159: make-_auth-calls-visible-with-master-stats-696.patch +# PATCH-FIX_UPSTREAM: https://github.com/saltstack/salt/pull/65843 +Patch160: repair-virt_query-outputter-655.patch ### IMPORTANT: The line below is used as a snippet marker. Do not touch it. ### SALT PATCHES LIST END @@ -576,10 +591,6 @@ BuildRequires: zsh %endif -%if 0%{?rhel} || 0%{?fedora} -BuildRequires: yum -%endif - %define python_subpackage_only 1 %python_subpackages @@ -593,7 +604,7 @@ %if "%{flavor}" != "testsuite" -%if 0%{?_nosinglespec} +%if 0%{?singlespec_compat} %package -n python3-salt %else %package -n python-salt @@ -650,7 +661,7 @@ %if 0%{?rhel} == 8 Requires: platform-python %else -%if 0%{?_nosinglespec} +%if 0%{?singlespec_compat} Requires: %{python_module base} %else Requires: python-base @@ -674,16 +685,13 @@ Requires: python3-markupsafe Requires: python3-msgpack > 0.3 Requires: python3-zmq >= 2.2.0 -Requires: yum -%if 0%{?rhel} == 8 || 0%{?fedora} >= 30 +%if 0%{?rhel} >= 8 || 0%{?fedora} >= 30 Requires: dnf -%endif -%if 0%{?rhel} == 6 -Requires: yum-plugin-security +Requires: python3-dnf-plugins-core %endif %else # SUSE -%if 0%{?_nosinglespec} +%if 0%{?singlespec_compat} Requires: %{python_module Jinja2} Requires: %{python_module MarkupSafe} Requires: %{python_module msgpack-python > 0.3} @@ -705,7 +713,7 @@ Requires: python-pyzmq >= 2.2.0 %endif %endif # end of RHEL / SUSE specific section -%if 0%{?_nosinglespec} +%if 0%{?singlespec_compat} Recommends: %{python_module jmespath} Requires: %{python_module PyYAML} Requires: %{python_module psutil} @@ -744,17 +752,20 @@ # %if 0%{?suse_version} # python-xml is part of python-base in all rhel versions -%if 0%{?_nosinglespec} +%if 0%{?singlespec_compat} Requires: %{python_module xml} +Requires: %{python_module zypp-plugin} Suggests: %{python_module Mako} Recommends: %{python_module netaddr} Recommends: %{python_module pyinotify} %else Requires: python-xml +Requires: python-zypp-plugin Suggests: python-Mako Recommends: python-netaddr Recommends: python-pyinotify %endif +Requires(pre): libzypp(plugin:system) >= 0 %endif # Required by Salt modules @@ -764,7 +775,7 @@ Recommends: man Recommends: python3-passlib -%if 0%{?_nosinglespec} +%if 0%{?singlespec_compat} Provides: bundled(%{python_module tornado}) = 4.5.3 %else Provides: bundled(python-tornado) = 4.5.3 @@ -772,7 +783,7 @@ Provides: %{name}-call = %{version}-%{release} -%if 0%{?_nosinglespec} +%if 0%{?singlespec_compat} %description -n python3-salt %else %description -n python-salt @@ -785,7 +796,11 @@ Requires: %{name} = %{version}-%{release} Requires: %{name}-master = %{version}-%{release} %if 0%{?suse_version} +%if 0%{?sle_version} >= 150400 +Requires: %{python_module CherryPy >= 3.2.2 if %python-salt} +%else Requires: python3-CherryPy >= 3.2.2 +%endif %else Requires: python3-cherrypy >= 3.2.2 %endif @@ -798,11 +813,19 @@ Group: System/Management Requires: %{name} = %{version}-%{release} Requires: %{name}-master = %{version}-%{release} -Requires: python3-apache-libcloud %if 0%{?suse_version} +%if 0%{?sle_version} >= 150400 +Requires: %{python_module apache-libcloud if %python-salt} +Recommends: %{python_module botocore if %python-salt} +Recommends: %{python_module netaddr if %python-salt} +%else +Requires: python3-apache-libcloud Recommends: python3-botocore Recommends: python3-netaddr %endif +%else +Requires: python3-apache-libcloud +%endif %description cloud public cloud VM management system @@ -824,8 +847,12 @@ Group: System/Management Requires: %{name} = %{version}-%{release} %if 0%{?suse_version} +%if 0%{?sle_version} >= 150400 +Recommends: %{python_module pygit2 >= 0.20.3 if %python-salt} +%else Recommends: python3-pygit2 >= 0.20.3 %endif +%endif %ifarch %{ix86} x86_64 %if 0%{?suse_version} %if 0%{?suse_version} > 1110 @@ -852,10 +879,6 @@ %if 0%{?suse_version} > 1500 || 0%{?sle_version} > 150000 Requires: (%{name}-transactional-update = %{version}-%{release} if read-only-root-fs) %endif -%if 0%{?suse_version} -Requires: python3-zypp-plugin -Requires(pre): libzypp(plugin:system) >= 0 -%endif %if %{with systemd} %{?systemd_requires} @@ -980,8 +1003,8 @@ %if "%{flavor}" == "testsuite" -%if 0%{?_nosinglespec} -%package -n %{python_module salt-testsuite} +%if 0%{?singlespec_compat} +%package -n python3-salt-testsuite %else %package -n python-salt-testsuite %endif @@ -995,7 +1018,7 @@ BuildRequires: %{python_module setuptools} Requires: salt = %{version} -%if 0%{?_nosinglespec} +%if 0%{?singlespec_compat} Recommends: %{python_module CherryPy} Requires: %{python_module Genshi} Requires: %{python_module Mako} @@ -1042,7 +1065,7 @@ Obsoletes: %{name}-tests -%if 0%{?_nosinglespec} +%if 0%{?singlespec_compat} %description -n python3-salt-testsuite %else %description -n python-salt-testsuite @@ -1161,11 +1184,9 @@ %if 0%{?suse_version} install -Dd -m 0750 %{buildroot}%{_prefix}/lib/zypp/plugins/commit %{__install} scripts/suse/zypper/plugins/commit/zyppnotify %{buildroot}%{_prefix}/lib/zypp/plugins/commit/zyppnotify -sed -i '1s=^#!/usr/bin/\(python\|env python\)[0-9.]*=#!/usr/bin/python3=' %{buildroot}%{_prefix}/lib/zypp/plugins/commit/zyppnotify %endif -# Install Yum plugins only on RH machines -%if 0%{?fedora} || 0%{?rhel} +# Install DNF plugin only on RH machines %if 0%{?fedora} >= 22 || 0%{?rhel} >= 8 install -Dd %{buildroot}%{python3_sitelib}/dnf-plugins install -Dd %{buildroot}%{python3_sitelib}/dnf-plugins/__pycache__ @@ -1174,14 +1195,6 @@ %{__install} scripts/suse/dnf/plugins/dnfnotify.conf %{buildroot}%{_sysconfdir}/dnf/plugins %{__python3} -m compileall -d %{python3_sitelib}/dnf-plugins %{buildroot}%{python3_sitelib}/dnf-plugins/dnfnotify.py %{__python3} -O -m compileall -d %{python3_sitelib}/dnf-plugins %{buildroot}%{python3_sitelib}/dnf-plugins/dnfnotify.py -%else -install -Dd %{buildroot}%{_prefix}/share/yum-plugins -install -Dd %{buildroot}%{_sysconfdir}/yum/pluginconf.d -%{__install} scripts/suse/yum/plugins/yumnotify.py %{buildroot}%{_prefix}/share/yum-plugins -%{__install} scripts/suse/yum/plugins/yumnotify.conf %{buildroot}%{_sysconfdir}/yum/pluginconf.d -%{__python} -m compileall -d %{_prefix}/share/yum-plugins %{buildroot}%{_prefix}/share/yum-plugins/yumnotify.py -%{__python} -O -m compileall -d %{_prefix}/share/yum-plugins %{buildroot}%{_prefix}/share/yum-plugins/yumnotify.py -%endif %endif ## install init and systemd scripts @@ -1252,6 +1265,17 @@ %if 0%{?_alternatives} %python_clone -a %{buildroot}%{_bindir}/salt-call +%python_clone -a %{buildroot}%{_bindir}/salt-support +%python_clone -a %{buildroot}%{_bindir}/spm +install -Dd -m 0750 %{buildroot}%{_exec_prefix}/libexec/salt +for SALT_SCRIPT in salt salt-api salt-cloud salt-cp salt-key salt-master salt-minion salt-proxy salt-run salt-ssh salt-syndic; do + mv "%{buildroot}%{_bindir}/${SALT_SCRIPT}" "%{buildroot}%{_exec_prefix}/libexec/salt/" +%python_clone -a %{buildroot}%{_exec_prefix}/libexec/salt/${SALT_SCRIPT} + ln -s "%{_exec_prefix}/libexec/salt/${SALT_SCRIPT}" "%{buildroot}%{_bindir}/${SALT_SCRIPT}" +done +mv "%{buildroot}%{_prefix}/lib/zypp/plugins/commit/zyppnotify" "%{buildroot}%{_exec_prefix}/libexec/salt/" +%python_clone -a %{buildroot}%{_exec_prefix}/libexec/salt/zyppnotify +ln -s "%{_exec_prefix}/libexec/salt/zyppnotify" "%{buildroot}%{_prefix}/lib/zypp/plugins/commit/zyppnotify" %endif %endif @@ -1274,10 +1298,6 @@ if [[ -d "$S_PHOME/.ssh" ]]; then mv $S_PHOME/.ssh $S_HOME fi -%if 0%{?_alternatives} -[ -h %{_bindir}/salt-call ] || rm -f %{_bindir}/salt-call -%python_libalternatives_reset_alternative salt-call -%endif %post %if %{with systemd} @@ -1486,17 +1506,49 @@ %if 0%{?_alternatives} %pre -n python-salt -[ -h %{_bindir}/salt-call ] || rm -f %{_bindir}/salt-call -%python_libalternatives_reset_alternative salt-call +for SALT_SCRIPT in salt-call salt-support spm; do + [ -h "%{_bindir}/${SALT_SCRIPT}" ] || rm -f "%{_bindir}/${SALT_SCRIPT}" + if [ "$1" -gt 0 ] && [ -f /usr/sbin/update-alternatives ]; then + update-alternatives --quiet --remove "${SALT_SCRIPT}" "%{_bindir}/${SALT_SCRIPT}-%{python_bin_suffix}" + fi +done +for SALT_SCRIPT in salt salt-api salt-cloud salt-cp salt-key salt-master salt-minion salt-proxy salt-run salt-ssh salt-syndic zyppnotify; do + [ -h "%{_exec_prefix}/libexec/salt/${SALT_SCRIPT}" ] || rm -f "%{_exec_prefix}/libexec/salt/${SALT_SCRIPT}" + if [ "$1" -gt 0 ] && [ -f /usr/sbin/update-alternatives ]; then + update-alternatives --quiet --remove "${SALT_SCRIPT}" "%{_exec_prefix}/libexec/salt/${SALT_SCRIPT}-%{python_bin_suffix}" + fi +done +%if ! %{with libalternatives} %post -n python-salt -%python_install_alternative salt-call +if [ -f /usr/sbin/update-alternatives ]; then + for SALT_SCRIPT in salt-call salt-support spm; do + update-alternatives --quiet --install "%{_bindir}/${SALT_SCRIPT}" "${SALT_SCRIPT}" \ + "%{_bindir}/${SALT_SCRIPT}-%{python_bin_suffix}" %{python_version_nodots} + done + for SALT_SCRIPT in salt salt-api salt-cloud salt-cp salt-key salt-master salt-minion salt-proxy salt-run salt-ssh salt-syndic zyppnotify; do + update-alternatives --quiet --install "%{_exec_prefix}/libexec/salt/${SALT_SCRIPT}" "${SALT_SCRIPT}" \ + "%{_exec_prefix}/libexec/salt/${SALT_SCRIPT}-%{python_bin_suffix}" %{python_version_nodots} + done +fi %postun -n python-salt -%python_uninstall_alternative salt-call +if [ -f /usr/sbin/update-alternatives ]; then + for SALT_SCRIPT in salt-call salt-support spm; do + if [ ! -e "%{_bindir}/${SALT_SCRIPT}-%{python_bin_suffix}" ]; then + update-alternatives --quiet --remove "${SALT_SCRIPT}" "%{_bindir}/${SALT_SCRIPT}-%{python_bin_suffix}" + fi + done + for SALT_SCRIPT in salt salt-api salt-cloud salt-cp salt-key salt-master salt-minion salt-proxy salt-run salt-ssh salt-syndic zyppnotify; do + if [ ! -e "%{_exec_prefix}/libexec/salt/${SALT_SCRIPT}-%{python_bin_suffix}" ]; then + update-alternatives --quiet --remove "${SALT_SCRIPT}" "%{_exec_prefix}/libexec/salt/${SALT_SCRIPT}-%{python_bin_suffix}" + fi + done +fi +%endif %endif -%if 0%{?_nosinglespec} +%if 0%{?singlespec_compat} %posttrans -n %{python_module salt} %else %posttrans -n python-salt @@ -1559,16 +1611,11 @@ %{_prefix}/lib/zypp/plugins/commit/zyppnotify %endif -# Install Yum plugins only on RH machines -%if 0%{?fedora} || 0%{?rhel} +# Install DNF plugin only on RH machines %if 0%{?fedora} >= 22 || 0%{?rhel} >= 8 %{python3_sitelib}/dnf-plugins/dnfnotify.py %{python3_sitelib}/dnf-plugins/__pycache__/dnfnotify.* %{_sysconfdir}/dnf/plugins/dnfnotify.conf -%else -%{_prefix}/share/yum-plugins/yumnotify.* -%{_sysconfdir}/yum/pluginconf.d/yumnotify.conf -%endif %endif %if %{with systemd} @@ -1651,7 +1698,24 @@ %defattr(-,root,root,-) %if 0%{?_alternatives} %python_alternative %{_bindir}/salt-call +%python_alternative %{_bindir}/salt-support +%python_alternative %{_bindir}/spm +%dir %{_exec_prefix}/libexec +%dir %attr(0755, root, root) %{_exec_prefix}/libexec/salt +%python_alternative %{_exec_prefix}/libexec/salt/salt +%python_alternative %{_exec_prefix}/libexec/salt/salt-api +%python_alternative %{_exec_prefix}/libexec/salt/salt-cloud +%python_alternative %{_exec_prefix}/libexec/salt/salt-cp +%python_alternative %{_exec_prefix}/libexec/salt/salt-key +%python_alternative %{_exec_prefix}/libexec/salt/salt-master +%python_alternative %{_exec_prefix}/libexec/salt/salt-minion +%python_alternative %{_exec_prefix}/libexec/salt/salt-proxy +%python_alternative %{_exec_prefix}/libexec/salt/salt-run +%python_alternative %{_exec_prefix}/libexec/salt/salt-ssh +%python_alternative %{_exec_prefix}/libexec/salt/salt-syndic +%python_alternative %{_exec_prefix}/libexec/salt/zyppnotify %endif + %dir %{python_sitelib}/salt %dir %{python_sitelib}/salt-*.egg-info %{python_sitelib}/salt/* ++++++ _lastrevision ++++++ --- /var/tmp/diff_new_pack.Yswh2k/_old 2025-02-25 16:41:51.264538794 +0100 +++ /var/tmp/diff_new_pack.Yswh2k/_new 2025-02-25 16:41:51.268538962 +0100 @@ -1,3 +1,3 @@ -a9b3f5bf8fd572965c589d95a7f99d36f7f37b62 +4e81748d5e88d323e700a458ca0e9680acc81927 (No newline at EOF) ++++++ backport-batch-async-fixes-and-improvements-701.patch ++++++ >From 4fe7231fa99de8edc848367386f1a6a5192a0f58 Mon Sep 17 00:00:00 2001 From: Victor Zhestkov <[email protected]> Date: Fri, 21 Feb 2025 11:15:42 +0100 Subject: [PATCH] Backport batch async fixes and improvements (#701) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Backport batch async fixes and improvements Co-authored-by: Pablo Suárez Hernández <[email protected]> * Align batch_async tests --------- Co-authored-by: Pablo Suárez Hernández <[email protected]> --- salt/cli/batch_async.py | 60 ++++++++++++++++----- tests/pytests/unit/cli/test_batch_async.py | 63 ++++++++-------------- 2 files changed, 69 insertions(+), 54 deletions(-) diff --git a/salt/cli/batch_async.py b/salt/cli/batch_async.py index 5d49993faa..92215d0e04 100644 --- a/salt/cli/batch_async.py +++ b/salt/cli/batch_async.py @@ -35,7 +35,7 @@ def batch_async_required(opts, minions, extra): Check opts to identify if batch async is required for the operation. """ if not isinstance(minions, list): - False + return False batch_async_opts = opts.get("batch_async", {}) batch_async_threshold = ( batch_async_opts.get("threshold", 1) @@ -179,6 +179,7 @@ class SharedEventsChannel: self._used_by.discard(subscriber_id) def destroy_unused(self): + log.trace("SharedEventsChannel.destroy_unused called") if self._used_by: return False self.master_event.remove_event_handler(self.__handle_event) @@ -267,6 +268,7 @@ class BatchAsync: self.ended = False self.event = self.events_channel.master_event self.scheduled = False + self._start_batch_on_timeout = None def __set_event_handler(self): self.events_channel.subscribe( @@ -278,6 +280,8 @@ class BatchAsync: @salt.ext.tornado.gen.coroutine def __event_handler(self, tag, data, op): + # IMPORTANT: This function must run fast and not wait for any other task, + # otherwise it would cause events to be stuck. if not self.event: return try: @@ -285,7 +289,9 @@ class BatchAsync: if op == "ping_return": self.minions.add(minion) if self.targeted_minions == self.minions: - yield self.start_batch() + # call start_batch and do not wait for timeout as we received + # the responses from all the targets + self.io_loop.add_callback(self.start_batch) elif op == "find_job_return": if data.get("return", None): self.find_job_returned.add(minion) @@ -293,7 +299,8 @@ class BatchAsync: if minion in self.active: self.active.remove(minion) self.done_minions.add(minion) - yield self.schedule_next() + if not self.active: + self.io_loop.add_callback(self.schedule_next) except Exception as ex: # pylint: disable=W0703 log.error( "Exception occured while processing event: %s: %s", @@ -333,7 +340,7 @@ class BatchAsync: ) if timedout_minions: - yield self.schedule_next() + self.io_loop.add_callback(self.schedule_next) if self.event and running: self.find_job_returned = self.find_job_returned.difference(running) @@ -344,6 +351,9 @@ class BatchAsync: """ Find if the job was finished on the minions """ + log.trace( + "[%s] BatchAsync.find_job called for minions: %s", self.batch_jid, minions + ) if not self.event: return not_done = minions.difference(self.done_minions).difference( @@ -386,6 +396,7 @@ class BatchAsync: if not self.event: return self.__set_event_handler() + # call test.ping for all the targets in async way ping_return = yield self.events_channel.local_client.run_job_async( self.opts["tgt"], "test.ping", @@ -398,19 +409,24 @@ class BatchAsync: listen=False, **self.eauth, ) + # ping_return contains actual targeted minions and no actual responses + # from the minions as it's async and intended to populate targeted_minions set self.targeted_minions = set(ping_return["minions"]) - # start batching even if not all minions respond to ping - yield salt.ext.tornado.gen.sleep( - self.batch_presence_ping_timeout or self.opts["gather_job_timeout"] + # schedule start_batch to perform even if not all the minions responded + # self.__event_handler can push start_batch in case if all targets responded + self._start_batch_on_timeout = self.io_loop.call_later( + self.batch_presence_ping_timeout or self.opts["gather_job_timeout"], + self.start_batch, ) - if self.event: - yield self.start_batch() @salt.ext.tornado.gen.coroutine def start_batch(self): """ Fire `salt/batch/*/start` and continue batch with `run_next` """ + if self._start_batch_on_timeout is not None: + self.io_loop.remove_timeout(self._start_batch_on_timeout) + self._start_batch_on_timeout = None if self.initialized: return self.batch_size = get_bnum(self.opts, self.minions, True) @@ -431,6 +447,7 @@ class BatchAsync: """ End the batch and call safe closing """ + log.trace("[%s] BatchAsync.end_batch called", self.batch_jid) left = self.minions.symmetric_difference( self.done_minions.union(self.timedout_minions) ) @@ -452,10 +469,11 @@ class BatchAsync: # release to the IOLoop to allow the event to be published # before closing batch async execution - yield salt.ext.tornado.gen.sleep(1) + yield salt.ext.tornado.gen.sleep(0.03) self.close_safe() def close_safe(self): + log.trace("[%s] BatchAsync.close_safe called", self.batch_jid) if self.events_channel is not None: self.events_channel.unsubscribe(None, None, id(self)) self.events_channel.unuse(id(self)) @@ -465,11 +483,22 @@ class BatchAsync: @salt.ext.tornado.gen.coroutine def schedule_next(self): + log.trace("[%s] BatchAsync.schedule_next called", self.batch_jid) if self.scheduled: + log.trace( + "[%s] BatchAsync.schedule_next -> Batch already scheduled, nothing to do.", + self.batch_jid, + ) return self.scheduled = True - # call later so that we maybe gather more returns - yield salt.ext.tornado.gen.sleep(self.batch_delay) + if self._get_next(): + # call later so that we maybe gather more returns + log.trace( + "[%s] BatchAsync.schedule_next delaying batch %s second(s).", + self.batch_jid, + self.batch_delay, + ) + yield salt.ext.tornado.gen.sleep(self.batch_delay) if self.event: yield self.run_next() @@ -480,6 +509,11 @@ class BatchAsync: """ self.scheduled = False next_batch = self._get_next() + log.trace( + "[%s] BatchAsync.run_next called. Next Batch -> %s", + self.batch_jid, + next_batch, + ) if not next_batch: yield self.end_batch() return @@ -504,7 +538,7 @@ class BatchAsync: yield salt.ext.tornado.gen.sleep(self.opts["timeout"]) # The batch can be done already at this point, which means no self.event - if self.event: + if self.event and self.active.intersection(next_batch): yield self.find_job(set(next_batch)) except Exception as ex: # pylint: disable=W0703 log.error( diff --git a/tests/pytests/unit/cli/test_batch_async.py b/tests/pytests/unit/cli/test_batch_async.py index bc871aba54..be8de692e6 100644 --- a/tests/pytests/unit/cli/test_batch_async.py +++ b/tests/pytests/unit/cli/test_batch_async.py @@ -85,11 +85,17 @@ def test_batch_start_on_batch_presence_ping_timeout(batch): future.set_result({}) with patch.object(batch, "events_channel", MagicMock()), patch( "salt.ext.tornado.gen.sleep", return_value=future - ), patch.object(batch, "start_batch", return_value=future) as start_batch_mock: + ), patch.object(batch, "io_loop", MagicMock()), patch.object( + batch, "start_batch", return_value=future + ) as start_batch_mock: batch.events_channel.local_client.run_job_async.return_value = future_ret ret = batch.start() - # assert start_batch is called - start_batch_mock.assert_called_once() + # start_batch is scheduled to be called later + assert batch.io_loop.call_later.call_args[0] == ( + batch.batch_presence_ping_timeout, + batch.start_batch, + ) + assert batch._start_batch_on_timeout is not None # assert test.ping called assert batch.events_channel.local_client.run_job_async.call_args[0] == ( "*", @@ -109,16 +115,21 @@ def test_batch_start_on_gather_job_timeout(batch): batch.batch_presence_ping_timeout = None with patch.object(batch, "events_channel", MagicMock()), patch( "salt.ext.tornado.gen.sleep", return_value=future + ), patch.object(batch, "io_loop", MagicMock()), patch.object( + batch, "start_batch", return_value=future ), patch.object( batch, "start_batch", return_value=future ) as start_batch_mock, patch.object( batch, "batch_presence_ping_timeout", None ): batch.events_channel.local_client.run_job_async.return_value = future_ret - # ret = batch_async.start(batch) ret = batch.start() - # assert start_batch is called - start_batch_mock.assert_called_once() + # start_batch is scheduled to be called later + assert batch.io_loop.call_later.call_args[0] == ( + batch.opts["gather_job_timeout"], + batch.start_batch, + ) + assert batch._start_batch_on_timeout is not None def test_batch_fire_start_event(batch): @@ -271,34 +282,10 @@ def test_batch__event_handler_ping_return(batch): assert batch.done_minions == set() -def test_batch__event_handler_call_start_batch_when_all_pings_return(batch): - batch.targeted_minions = {"foo"} - future = salt.ext.tornado.gen.Future() - future.set_result({}) - with patch.object(batch, "start_batch", return_value=future) as start_batch_mock: - batch.start() - batch._BatchAsync__event_handler( - "salt/job/1234/ret/foo", {"id": "foo"}, "ping_return" - ) - start_batch_mock.assert_called_once() - - -def test_batch__event_handler_not_call_start_batch_when_not_all_pings_return(batch): - batch.targeted_minions = {"foo", "bar"} - future = salt.ext.tornado.gen.Future() - future.set_result({}) - with patch.object(batch, "start_batch", return_value=future) as start_batch_mock: - batch.start() - batch._BatchAsync__event_handler( - "salt/job/1234/ret/foo", {"id": "foo"}, "ping_return" - ) - start_batch_mock.assert_not_called() - - def test_batch__event_handler_batch_run_return(batch): future = salt.ext.tornado.gen.Future() future.set_result({}) - with patch.object( + with patch.object(batch, "io_loop", MagicMock()), patch.object( batch, "schedule_next", return_value=future ) as schedule_next_mock: batch.start() @@ -308,7 +295,7 @@ def test_batch__event_handler_batch_run_return(batch): ) assert batch.active == set() assert batch.done_minions == {"foo"} - schedule_next_mock.assert_called_once() + batch.io_loop.add_callback.call_args[0] == (batch.schedule_next) def test_batch__event_handler_find_job_return(batch): @@ -322,9 +309,7 @@ def test_batch__event_handler_find_job_return(batch): def test_batch_run_next_end_batch_when_no_next(batch): future = salt.ext.tornado.gen.Future() future.set_result({}) - with patch.object( - batch, "_get_next", return_value={} - ), patch.object( + with patch.object(batch, "_get_next", return_value={}), patch.object( batch, "end_batch", return_value=future ) as end_batch_mock: batch.run_next() @@ -337,9 +322,7 @@ def test_batch_find_job(batch): batch.minions = {"foo", "bar"} with patch("salt.ext.tornado.gen.sleep", return_value=future), patch.object( batch, "check_find_job", return_value=future - ) as check_find_job_mock, patch.object( - batch, "jid_gen", return_value="1236" - ): + ) as check_find_job_mock, patch.object(batch, "jid_gen", return_value="1236"): batch.events_channel.local_client.run_job_async.return_value = future batch.find_job({"foo", "bar"}) assert check_find_job_mock.call_args[0] == ( @@ -355,9 +338,7 @@ def test_batch_find_job_with_done_minions(batch): batch.minions = {"foo", "bar"} with patch("salt.ext.tornado.gen.sleep", return_value=future), patch.object( batch, "check_find_job", return_value=future - ) as check_find_job_mock, patch.object( - batch, "jid_gen", return_value="1236" - ): + ) as check_find_job_mock, patch.object(batch, "jid_gen", return_value="1236"): batch.events_channel.local_client.run_job_async.return_value = future batch.find_job({"foo", "bar"}) assert check_find_job_mock.call_args[0] == ( -- 2.48.1 ++++++ fix-tests-failures-after-repo.saltproject.io-depreca.patch ++++++ >From a630c6a707a1d5227b4a1fa8f0f751fefd3ef47f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20Su=C3=A1rez=20Hern=C3=A1ndez?= <[email protected]> Date: Wed, 19 Feb 2025 13:56:01 +0000 Subject: [PATCH] Fix tests failures after "repo.saltproject.io" deprecation (#704) * Use broadcom.com instead of repo.saltproject.io for test_cp * Change repo.saltproject.io to new url --------- Co-authored-by: Daniel A. Wozniak <[email protected]> Co-authored-by: twangboy <[email protected]> --- README.rst | 4 +- doc/_themes/saltstack2/layout.html | 15 +-- doc/conf.py | 12 +- doc/ref/configuration/delta_proxy.rst | 6 +- doc/topics/cloud/windows.rst | 2 +- pkg/tests/support/helpers.py | 115 ++++++++++++------ salt/modules/saltutil.py | 4 +- salt/runners/manage.py | 5 +- salt/states/pkgrepo.py | 6 +- tests/integration/modules/test_cp.py | 41 ++++--- tests/pytests/functional/modules/test_pkg.py | 8 +- .../functional/states/pkgrepo/test_debian.py | 6 +- .../integration/netapi/test_ssh_client.py | 3 +- tests/support/win_installer.py | 1 + 14 files changed, 135 insertions(+), 93 deletions(-) diff --git a/README.rst b/README.rst index f5121f1a74..77806aa14a 100644 --- a/README.rst +++ b/README.rst @@ -93,7 +93,9 @@ for more information. To download and install Salt, see: * `The Salt install guide <https://docs.saltproject.io/salt/install-guide/en/latest/index.html>`_ -* `Salt Project repository <https://repo.saltproject.io/>`_ +* `Salt Project repository <https://packages.broadcom.com/artifactory/saltproject-generic/>`_ +* `Salt Project debian repository <https://packages.broadcom.com/artifactory/saltproject-deb/>`_ +* `Salt Project redhat repository <https://packages.broadcom.com/artifactory/saltproject-rpm/>`_ Technical support diff --git a/doc/_themes/saltstack2/layout.html b/doc/_themes/saltstack2/layout.html index 04bff89e1f..83918a7fb3 100644 --- a/doc/_themes/saltstack2/layout.html +++ b/doc/_themes/saltstack2/layout.html @@ -157,16 +157,11 @@ <!-- Collect the nav links, forms, and other content for toggling --> <div class="collapse navbar-collapse" id="navbarCollapse"> <ul class="nav navbar-nav"> - <li><a href="/en/latest/">Overview</a></li> - <li><a href="https://docs.saltproject.io/salt/user-guide/en/latest/";>Salt User Guide</a></li> - <li><a href="/en/latest/contents.html">Documentation</a></li> - <li><a href="https://repo.saltproject.io";>Downloads</a></li> - <li><a href="/en/latest/topics/development/">Develop</a></li> - <!--<li><a href="/en/2016.3/faq/">FAQ</a></li> - <li><a href="/en/2016.3/samples/">Code Samples</a></li>--> - <!-- <li><a href="https://repo.saltproject.io"; target="_blank">Downloads</a></li>--> - <!--<li><a href="http://saltstack.com/training"; target="_blank">Training</a></li> - <li><a href="http://saltstack.com/support"; target="_blank">Support</a></li>--> + <li><a href="/en/latest/">Overview</a></li> + <li><a href="https://docs.saltproject.io/salt/user-guide/en/latest/";>Salt User Guide</a></li> + <li><a href="/en/latest/contents.html">Documentation</a></li> + <li><a href="https://packages.broadcom.com/artifactory/saltproject-generic/";>Downloads</a></li> + <li><a href="/en/latest/topics/development/">Develop</a></li> </ul> </div> </div> diff --git a/doc/conf.py b/doc/conf.py index 653d912c20..24420d402e 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -178,17 +178,17 @@ rst_prolog = """\ .. |windownload| raw:: html <p>Python3 x86: <a - href="https://repo.saltproject.io/windows/Salt-Minion-{release}-Py3-x86-Setup.exe";><strong>Salt-Minion-{release}-x86-Setup.exe</strong></a> - | <a href="https://repo.saltproject.io/windows/Salt-Minion-{release}-Py3-x86-Setup.exe.md5";><strong>md5</strong></a></p> + href="https://packages.broadcom.com/artifactory/saltproject-generic/windows/{release}/Salt-Minion-{release}-Py3-x86-Setup.exe";><strong>Salt-Minion-{release}-x86-Setup.exe</strong></a> + | <a href="https://packages.broadcom.com/artifactory/saltproject-generic/windows/{release}/Salt-Minion-{release}-Py3-x86-Setup.exe.md5";><strong>md5</strong></a></p> <p>Python3 AMD64: <a - href="https://repo.saltproject.io/windows/Salt-Minion-{release}-Py3-AMD64-Setup.exe";><strong>Salt-Minion-{release}-AMD64-Setup.exe</strong></a> - | <a href="https://repo.saltproject.io/windows/Salt-Minion-{release}-Py3-AMD64-Setup.exe.md5";><strong>md5</strong></a></p> + href="https://packages.broadcom.com/artifactory/saltproject-generic/windows/{release}/Salt-Minion-{release}-Py3-AMD64-Setup.exe";><strong>Salt-Minion-{release}-AMD64-Setup.exe</strong></a> + | <a href="https://packages.broadcom.com/artifactory/saltproject-generic/windows/{release}/Salt-Minion-{release}-Py3-AMD64-Setup.exe.md5";><strong>md5</strong></a></p> .. |osxdownloadpy3| raw:: html - <p>x86_64: <a href="https://repo.saltproject.io/osx/salt-{release}-py3-x86_64.pkg";><strong>salt-{release}-py3-x86_64.pkg</strong></a> - | <a href="https://repo.saltproject.io/osx/salt-{release}-py3-x86_64.pkg.md5";><strong>md5</strong></a></p> + <p>x86_64: <a href="https://packages.broadcom.com/artifactory/saltproject-generic/macos/{release}/salt-{release}-py3-x86_64.pkg";><strong>salt-{release}-py3-x86_64.pkg</strong></a> + | <a href="https://packages.broadcom.com/artifactory/saltproject-generic/macos/{release}/salt-{release}-py3-x86_64.pkg.md5";><strong>md5</strong></a></p> """.format( release=stripped_release diff --git a/doc/ref/configuration/delta_proxy.rst b/doc/ref/configuration/delta_proxy.rst index be1831da39..bce5f821c9 100644 --- a/doc/ref/configuration/delta_proxy.rst +++ b/doc/ref/configuration/delta_proxy.rst @@ -146,10 +146,8 @@ Before installing the delta proxy minion, ensure that: Install or upgrade Salt ----------------------- Ensure your Salt masters are running at least Salt version 3004. For instructions -on installing or upgrading Salt, see `repo.saltproject.io -<http://repo.saltproject.io/>`_. For RedHat systems, see `Install or Upgrade Salt -<https://enterprise.saltproject.io/en/latest/docs/install-salt.html>`_. - +on installing or upgrading Salt, see the `Salt install guide +<https://docs.saltproject.io/salt/install-guide>`_. .. _delta-proxy-install: diff --git a/doc/topics/cloud/windows.rst b/doc/topics/cloud/windows.rst index 9dfdde6db5..79d6665a5a 100644 --- a/doc/topics/cloud/windows.rst +++ b/doc/topics/cloud/windows.rst @@ -62,7 +62,7 @@ from saltstack.com: * `SaltStack Download Area`__ -.. __: https://repo.saltproject.io/windows/ +.. __: https://packages.broadcom.com/artifactory/saltproject-generic/windows/ .. _new-pywinrm: diff --git a/pkg/tests/support/helpers.py b/pkg/tests/support/helpers.py index 90abf8b88e..ce23f699b6 100644 --- a/pkg/tests/support/helpers.py +++ b/pkg/tests/support/helpers.py @@ -636,8 +636,7 @@ class SaltPkgInstall: def install_previous(self): """ - Install previous version. This is used for - upgrade tests. + Install previous version. This is used for upgrade tests. """ major_ver = self.major minor_ver = self.minor @@ -648,16 +647,12 @@ class SaltPkgInstall: distro_name = self.distro_name if distro_name == "centos" or distro_name == "fedora": distro_name = "redhat" - root_url = "salt/py3/" - if self.classic: - root_url = "py3/" + root_url = "https://packages.broadcom.com/artifactory"; if self.distro_name in ["redhat", "centos", "amazon", "fedora", "vmware"]: for fp in pathlib.Path("/etc", "yum.repos.d").glob("epel*"): fp.unlink() - gpg_key = "SALTSTACK-GPG-KEY.pub" - if self.distro_version == "9": - gpg_key = "SALTSTACK-GPG-KEY2.pub" + if platform.is_aarch64(): arch = "aarch64" else: @@ -694,46 +689,86 @@ class SaltPkgInstall: arch = "arm64" else: arch = "amd64" + pathlib.Path("/etc/apt/keyrings").mkdir(parents=True, exist_ok=True) + gpg_full_path = "/etc/apt/keyrings/salt-archive-keyring.gpg" + + # download the gpg pub key download_file( - f"https://repo.saltproject.io/{root_url}{distro_name}/{self.distro_version}/{arch}/{major_ver}/salt-archive-keyring.gpg";, - "/etc/apt/keyrings/salt-archive-keyring.gpg", + f"{root_url}/api/security/keypair/SaltProjectKey/public", + f"{gpg_full_path}", ) - with open( + with salt.utils.files.fopen( pathlib.Path("/etc", "apt", "sources.list.d", "salt.list"), "w" ) as fp: fp.write( - f"deb [signed-by=/etc/apt/keyrings/salt-archive-keyring.gpg arch={arch}] " - f"https://repo.saltproject.io/{root_url}{distro_name}/{self.distro_version}/{arch}/{major_ver} {self.distro_codename} main" + f"deb [signed-by={gpg_full_path} arch={arch}] " + f"{root_url}/saltproject-deb/ {self.distro_codename} main" ) - ret = self.proc.run(self.pkg_mngr, "update") - self._check_retcode(ret) - ret = self.proc.run( - self.pkg_mngr, - "install", - *self.salt_pkgs, - "-y", - ) self._check_retcode(ret) + + cmd = [self.pkg_mngr, "install", *self.salt_pkgs, "-y"] + + if downgrade: + pref_file = pathlib.Path("/etc", "apt", "preferences.d", "salt.pref") + pref_file.parent.mkdir(exist_ok=True) + # TODO: There's probably something I should put in here to say what version + # TODO: But maybe that's done elsewhere, hopefully in self.salt_pkgs + pref_file.write_text( + textwrap.dedent( + f"""\ + Package: salt* + Pin: origin "{root_url}/saltproject-deb" + Pin-Priority: 1001 + """ + ), + encoding="utf-8", + ) + cmd.append("--allow-downgrades") + env = os.environ.copy() + env["DEBIAN_FRONTEND"] = "noninteractive" + extra_args = [ + "-o", + "DPkg::Options::=--force-confdef", + "-o", + "DPkg::Options::=--force-confold", + ] + self.proc.run(self.pkg_mngr, "update", *extra_args, env=env) + + cmd.extend(extra_args) + + ret = self.proc.run(*cmd, env=env) + # Pre-relenv packages down get downgraded to cleanly programmatically + # They work manually, and the install tests after downgrades will catch problems with the install + # Let's not check the returncode if this is the case + if not ( + downgrade + and packaging.version.parse(self.prev_version) + < packaging.version.parse("3006.0") + ): + self._check_retcode(ret) + if downgrade: + pref_file.unlink() self.stop_services() elif platform.is_windows(): self.onedir = True self.installer_pkg = True self.bin_dir = self.install_dir / "bin" - self.run_root = self.bin_dir / f"salt.exe" - self.ssm_bin = self.bin_dir / "ssm.exe" - if self.file_ext == "msi": - self.ssm_bin = self.install_dir / "ssm.exe" + self.run_root = self.bin_dir / "salt.exe" + self.ssm_bin = self.install_dir / "ssm.exe" - if not self.classic: - win_pkg = f"salt-{full_version}-windows-amd64.{self.file_ext}" - win_pkg_url = f"https://repo.saltproject.io/salt/py3/windows/{full_version}/{win_pkg}"; + if self.file_ext == "exe": + win_pkg = ( + f"Salt-Minion-{self.prev_version}-Py3-AMD64-Setup.{self.file_ext}" + ) + elif self.file_ext == "msi": + win_pkg = f"Salt-Minion-{self.prev_version}-Py3-AMD64.{self.file_ext}" else: - if self.file_ext == "msi": - win_pkg = f"Salt-Minion-{min_ver}-1-Py3-AMD64.{self.file_ext}" - elif self.file_ext == "exe": - win_pkg = f"Salt-Minion-{min_ver}-1-Py3-AMD64-Setup.{self.file_ext}" - win_pkg_url = f"https://repo.saltproject.io/windows/{win_pkg}"; + log.debug(f"Unknown windows file extension: {self.file_ext}") + + win_pkg_url = ( + f"{root_url}/saltproject-generic/windows/{major_ver}/{win_pkg}" + ) pkg_path = pathlib.Path(r"C:\TEMP", win_pkg) pkg_path.parent.mkdir(exist_ok=True) ret = requests.get(win_pkg_url) @@ -763,12 +798,16 @@ class SaltPkgInstall: self._install_system_service() elif platform.is_darwin(): - if self.classic: - mac_pkg = f"salt-{min_ver}.{minor_ver}-1-py3-x86_64.pkg" - mac_pkg_url = f"https://repo.saltproject.io/osx/{mac_pkg}"; + if relenv and platform.is_aarch64(): + arch = "arm64" + elif platform.is_aarch64() and self.classic: + arch = "arm64" else: - mac_pkg = f"salt-{min_ver}.{minor_ver}-1-macos-x86_64.pkg" - mac_pkg_url = f"https://repo.saltproject.io/salt/py3/macos/{major_ver}.{minor_ver}-1/{mac_pkg}"; + arch = "x86_64" + + mac_pkg = f"salt-{self.prev_version}-py3-{arch}.pkg" + mac_pkg_url = f"{root_url}/saltproject-generic/macos/{major_ver}/{mac_pkg}" + mac_pkg_path = f"/tmp/{mac_pkg}" if not os.path.exists(mac_pkg_path): download_file( diff --git a/salt/modules/saltutil.py b/salt/modules/saltutil.py index a692c3f34d..320b9c34fa 100644 --- a/salt/modules/saltutil.py +++ b/salt/modules/saltutil.py @@ -128,8 +128,8 @@ def _sync(form, saltenv=None, extmod_whitelist=None, extmod_blacklist=None): def update(version=None): """ Update the salt minion from the URL defined in opts['update_url'] - VMware, Inc provides the latest builds here: - update_url: https://repo.saltproject.io/windows/ + Broadcom, Inc provides the latest builds here: + update_url: https://packages.broadcom.com/artifactory/saltproject-generic/windows/ Be aware that as of 2014-8-11 there's a bug in esky such that only the latest version available in the update_url can be downloaded and installed. diff --git a/salt/runners/manage.py b/salt/runners/manage.py index 9dc67ed728..81197ca41f 100644 --- a/salt/runners/manage.py +++ b/salt/runners/manage.py @@ -772,7 +772,7 @@ def bootstrap_psexec( installer_url URL of minion installer executable. Defaults to the latest version from - https://repo.saltproject.io/windows/ + https://packages.broadcom.com/artifactory/saltproject-generic/windows/ username Optional user name for login on remote computer. @@ -790,6 +790,9 @@ def bootstrap_psexec( salt-run manage.bootstrap_psexec hosts='host1,host2' installer_url='http://exampledomain/salt-installer.exe' """ + # TODO: Need to make this gets the latest version from the new repo location + # TODO: Similar to tests/support/win_installer.py + # TODO: Maybe need to move that ^^^^ to a salt util if not installer_url: base_url = "https://repo.saltproject.io/windows/"; source = urllib.request.urlopen(base_url).read() diff --git a/salt/states/pkgrepo.py b/salt/states/pkgrepo.py index f041644287..4ef5fd9c2f 100644 --- a/salt/states/pkgrepo.py +++ b/salt/states/pkgrepo.py @@ -99,17 +99,17 @@ Using ``aptkey: False`` with ``key_url`` example: .. code-block:: yaml - deb [signed-by=/etc/apt/keyrings/salt-archive-keyring.gpg arch=amd64] https://repo.saltproject.io/py3/ubuntu/18.04/amd64/latest bionic main: + deb [signed-by=/etc/apt/keyrings/salt-archive-keyring.gpg arch=amd64] https://packages.broadcom.com/artifactory/saltproject-deb/ bionic main: pkgrepo.managed: - file: /etc/apt/sources.list.d/salt.list - - key_url: https://repo.saltproject.io/py3/ubuntu/18.04/amd64/latest/salt-archive-keyring.gpg + - key_url: https://packages.broadcom.com/artifactory/api/security/keypair/SaltProjectKey/public - aptkey: False Using ``aptkey: False`` with ``keyserver`` and ``keyid``: .. code-block:: yaml - deb [signed-by=/etc/apt/keyrings/salt-archive-keyring.gpg arch=amd64] https://repo.saltproject.io/py3/ubuntu/18.04/amd64/latest bionic main: + deb [signed-by=/etc/apt/keyrings/salt-archive-keyring.gpg arch=amd64] https://packages.broadcom.com/artifactory/saltproject-deb/ bionic main: pkgrepo.managed: - file: /etc/apt/sources.list.d/salt.list - keyserver: keyserver.ubuntu.com diff --git a/tests/integration/modules/test_cp.py b/tests/integration/modules/test_cp.py index cd3e4c2f5a..d417f90ddc 100644 --- a/tests/integration/modules/test_cp.py +++ b/tests/integration/modules/test_cp.py @@ -231,12 +231,15 @@ class CPModuleTest(ModuleCase): """ cp.get_url with https:// source given """ - self.run_function("cp.get_url", ["https://repo.saltproject.io/index.html";, tgt]) + self.run_function( + "cp.get_url", + ["https://packages.broadcom.com/artifactory/saltproject-generic/";, tgt], + ) with salt.utils.files.fopen(tgt, "r") as instructions: data = salt.utils.stringutils.to_unicode(instructions.read()) - self.assertIn("Salt Project", data) - self.assertIn("Package", data) - self.assertIn("Repo", data) + self.assertIn("Index of saltproject", data) + self.assertIn("onedir", data) + self.assertIn("Artifactory Online Server", data) self.assertNotIn("AYBABTU", data) @pytest.mark.slow_test @@ -245,14 +248,15 @@ class CPModuleTest(ModuleCase): cp.get_url with https:// source given and destination omitted. """ ret = self.run_function( - "cp.get_url", ["https://repo.saltproject.io/index.html";] + "cp.get_url", + ["https://packages.broadcom.com/artifactory/saltproject-generic/";], ) with salt.utils.files.fopen(ret, "r") as instructions: data = salt.utils.stringutils.to_unicode(instructions.read()) - self.assertIn("Salt Project", data) - self.assertIn("Package", data) - self.assertIn("Repo", data) + self.assertIn("Index of saltproject", data) + self.assertIn("onedir", data) + self.assertIn("Artifactory Online Server", data) self.assertNotIn("AYBABTU", data) @pytest.mark.slow_test @@ -266,16 +270,19 @@ class CPModuleTest(ModuleCase): tgt = None while time.time() - start <= timeout: ret = self.run_function( - "cp.get_url", ["https://repo.saltproject.io/index.html";, tgt] + "cp.get_url", + ["https://packages.broadcom.com/artifactory/saltproject-generic/";, tgt], ) if ret.find("HTTP 599") == -1: break time.sleep(sleep) if ret.find("HTTP 599") != -1: - raise Exception("https://repo.saltproject.io/index.html returned 599 error") - self.assertIn("Salt Project", ret) - self.assertIn("Package", ret) - self.assertIn("Repo", ret) + raise Exception( + "https://packages.broadcom.com/artifactory/saltproject-generic/ returned 599 error" + ) + self.assertIn("Index of saltproject", ret) + self.assertIn("onedir", ret) + self.assertIn("Artifactory Online Server", ret) self.assertNotIn("AYBABTU", ret) @pytest.mark.slow_test @@ -344,11 +351,11 @@ class CPModuleTest(ModuleCase): """ cp.get_file_str with https:// source given """ - src = "https://repo.saltproject.io/index.html"; + src = "https://packages.broadcom.com/artifactory/saltproject-generic/"; ret = self.run_function("cp.get_file_str", [src]) - self.assertIn("Salt Project", ret) - self.assertIn("Package", ret) - self.assertIn("Repo", ret) + self.assertIn("Index of saltproject", ret) + self.assertIn("onedir", ret) + self.assertIn("Artifactory Online Server", ret) self.assertNotIn("AYBABTU", ret) @pytest.mark.slow_test diff --git a/tests/pytests/functional/modules/test_pkg.py b/tests/pytests/functional/modules/test_pkg.py index 82d0801965..addb3da3d1 100644 --- a/tests/pytests/functional/modules/test_pkg.py +++ b/tests/pytests/functional/modules/test_pkg.py @@ -130,12 +130,8 @@ def test_mod_del_repo(grains, modules, refresh_db): elif grains["os_family"] == "RedHat": repo = "saltstack" name = "SaltStack repo for RHEL/CentOS {}".format(grains["osmajorrelease"]) - baseurl = "https://repo.saltproject.io/py3/redhat/{}/x86_64/latest/".format( - grains["osmajorrelease"] - ) - gpgkey = "https://repo.saltproject.io/py3/redhat/{}/x86_64/latest/SALTSTACK-GPG-KEY.pub".format( - grains["osmajorrelease"] - ) + baseurl = "https://packages.broadcom.com/artifactory/saltproject-rpm/"; + gpgkey = "https://packages.broadcom.com/artifactory/api/security/keypair/SaltProjectKey/public"; gpgcheck = 1 enabled = 1 ret = modules.pkg.mod_repo( diff --git a/tests/pytests/functional/states/pkgrepo/test_debian.py b/tests/pytests/functional/states/pkgrepo/test_debian.py index 7bda100b63..307fcb5819 100644 --- a/tests/pytests/functional/states/pkgrepo/test_debian.py +++ b/tests/pytests/functional/states/pkgrepo/test_debian.py @@ -616,8 +616,8 @@ class Repo: @alt_repo.default def _default_alt_repo(self): """ - Use an alternative repo, packages do not - exist for the OS on repo.saltproject.io + Use an alternative repo, packages do not exist for the OS on + packages.broadcom.com """ if ( self.grains["osfullname"] == "Ubuntu" @@ -777,7 +777,7 @@ def test_adding_repo_file_signedby_alt_file(pkgrepo, states, repo): assert repo.repo_content in ret.comment key_file = repo.key_file.parent / "salt-alt-key.gpg" - repo_content = "deb [arch=amd64 signed-by={}] https://repo.saltproject.io/py3/debian/10/amd64/latest buster main".format( + repo_content = "deb [arch=amd64 signed-by={}] https://packages.broadcom.com/artifactory/saltproject-deb/ buster main".format( str(key_file) ) ret = states.pkgrepo.managed( diff --git a/tests/pytests/integration/netapi/test_ssh_client.py b/tests/pytests/integration/netapi/test_ssh_client.py index 457c151c94..7dd540d9b9 100644 --- a/tests/pytests/integration/netapi/test_ssh_client.py +++ b/tests/pytests/integration/netapi/test_ssh_client.py @@ -149,7 +149,8 @@ def test_shell_inject_ssh_priv( """ # ZDI-CAN-11143 path = tmp_path / "test-11143" - tgts = ["repo.saltproject.io", "www.zerodayinitiative.com"] + tgts = ["packages.broadcom.com", "www.zerodayinitiative.com"] + ret = None for tgt in tgts: low = { "roster": "cache", diff --git a/tests/support/win_installer.py b/tests/support/win_installer.py index 6a2f387dc8..d67105e8a0 100644 --- a/tests/support/win_installer.py +++ b/tests/support/win_installer.py @@ -10,6 +10,7 @@ """ import hashlib +from html.parser import HTMLParser import requests -- 2.48.1 ++++++ fixed-file-client-private-attribute-reference-on-sal.patch ++++++ >From 1772da828f40e36d2a9eceb7055a1fa1a2257830 Mon Sep 17 00:00:00 2001 From: Georg <[email protected]> Date: Fri, 21 Feb 2025 10:23:38 +0000 Subject: [PATCH] Fixed file client private attribute reference on `SaltMakoTemplateLookup` (#694) Fixes #64280 Signed-off-by: Pedro Algarvio <[email protected]> (cherry picked from commit 560ab52ccf94c7974d5a418dfbba7409e0493066) Co-authored-by: Pedro Algarvio <[email protected]> --- changelog/64280.fixed.md | 1 + salt/utils/mako.py | 6 ++++-- tests/pytests/unit/utils/test_mako.py | 28 +++++++++++++++++++++++++++ 3 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 changelog/64280.fixed.md create mode 100644 tests/pytests/unit/utils/test_mako.py diff --git a/changelog/64280.fixed.md b/changelog/64280.fixed.md new file mode 100644 index 0000000000..5a9b905dd0 --- /dev/null +++ b/changelog/64280.fixed.md @@ -0,0 +1 @@ +Fixed file client private attribute reference on `SaltMakoTemplateLookup` diff --git a/salt/utils/mako.py b/salt/utils/mako.py index 037d5d86de..4397ae8cc7 100644 --- a/salt/utils/mako.py +++ b/salt/utils/mako.py @@ -99,8 +99,10 @@ if HAS_MAKO: ) def destroy(self): - if self.client: + if self._file_client: + file_client = self._file_client + self._file_client = None try: - self.client.destroy() + file_client.destroy() except AttributeError: pass diff --git a/tests/pytests/unit/utils/test_mako.py b/tests/pytests/unit/utils/test_mako.py new file mode 100644 index 0000000000..952cf44652 --- /dev/null +++ b/tests/pytests/unit/utils/test_mako.py @@ -0,0 +1,28 @@ +import pytest + +from tests.support.mock import Mock, call, patch + +pytest.importorskip("mako") + +# This import needs to be after the above importorskip so that no ImportError +# is raised if Mako is not installed +from salt.utils.mako import SaltMakoTemplateLookup + + +def test_mako_template_lookup(minion_opts): + """ + The shudown method can be called without raising an exception when the + file_client does not have a destroy method + """ + # Test SaltCacheLoader creating and destroying the file client created + file_client = Mock() + with patch("salt.fileclient.get_file_client", return_value=file_client): + loader = SaltMakoTemplateLookup(minion_opts) + assert loader._file_client is None + assert loader.file_client() is file_client + assert loader._file_client is file_client + try: + loader.destroy() + except AttributeError: + pytest.fail("Regression when calling SaltMakoTemplateLookup.destroy()") + assert file_client.mock_calls == [call.destroy()] -- 2.48.1 ++++++ make-_auth-calls-visible-with-master-stats-696.patch ++++++ >From 32099b97c2fa549cb050d3ae618b5200c07328c8 Mon Sep 17 00:00:00 2001 From: Victor Zhestkov <[email protected]> Date: Fri, 21 Feb 2025 11:59:00 +0100 Subject: [PATCH] Make `_auth` calls visible with master stats (#696) * Put _auth calls to the master stats * Add _auth master stats tests * test small fix --- salt/channel/server.py | 9 ++++-- salt/master.py | 5 ++++ tests/pytests/unit/channel/__init__.py | 0 tests/pytests/unit/channel/test_server.py | 34 +++++++++++++++++++++++ tests/pytests/unit/test_master.py | 25 +++++++++++++++++ 5 files changed, 70 insertions(+), 3 deletions(-) create mode 100644 tests/pytests/unit/channel/__init__.py create mode 100644 tests/pytests/unit/channel/test_server.py diff --git a/salt/channel/server.py b/salt/channel/server.py index f1b6f701a9..59da3a2dc2 100644 --- a/salt/channel/server.py +++ b/salt/channel/server.py @@ -9,6 +9,7 @@ import hashlib import logging import os import shutil +import time import salt.crypt import salt.ext.tornado.gen @@ -149,9 +150,11 @@ class ReqServerChannel: # intercept the "_auth" commands, since the main daemon shouldn't know # anything about our key auth if payload["enc"] == "clear" and payload.get("load", {}).get("cmd") == "_auth": - raise salt.ext.tornado.gen.Return( - self._auth(payload["load"], sign_messages) - ) + start = time.time() + ret = self._auth(payload["load"], sign_messages) + if self.opts.get("master_stats", False): + yield self.payload_handler({"cmd": "_auth", "_start": start}) + raise salt.ext.tornado.gen.Return(ret) nonce = None if version > 1: diff --git a/salt/master.py b/salt/master.py index 49cfb68860..c0cd9a366b 100644 --- a/salt/master.py +++ b/salt/master.py @@ -1036,6 +1036,11 @@ class MWorker(salt.utils.process.SignalHandlingProcess): :param dict payload: The payload route to the appropriate handler """ + if payload.get("cmd") == "_auth": + if self.opts["master_stats"]: + self.stats["_auth"]["runs"] += 1 + self._post_stats(payload["_start"], "_auth") + return key = payload["enc"] load = payload["load"] if key == "aes": diff --git a/tests/pytests/unit/channel/__init__.py b/tests/pytests/unit/channel/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/pytests/unit/channel/test_server.py b/tests/pytests/unit/channel/test_server.py new file mode 100644 index 0000000000..3fa5d94bea --- /dev/null +++ b/tests/pytests/unit/channel/test_server.py @@ -0,0 +1,34 @@ +import time + +import pytest + +import salt.channel.server as server +import salt.ext.tornado.gen +from tests.support.mock import MagicMock, patch + + +def test__auth_cmd_stats_passing(): + req_server_channel = server.ReqServerChannel({"master_stats": True}, None) + + fake_ret = {"enc": "clear", "load": b"FAKELOAD"} + + def _auth_mock(*_, **__): + time.sleep(0.03) + return fake_ret + + future = salt.ext.tornado.gen.Future() + future.set_result({}) + + with patch.object(req_server_channel, "_auth", _auth_mock): + req_server_channel.payload_handler = MagicMock(return_value=future) + req_server_channel.handle_message( + {"enc": "clear", "load": {"cmd": "_auth", "id": "minion"}} + ) + cur_time = time.time() + req_server_channel.payload_handler.assert_called_once() + assert req_server_channel.payload_handler.call_args[0][0]["cmd"] == "_auth" + auth_call_duration = ( + cur_time - req_server_channel.payload_handler.call_args[0][0]["_start"] + ) + assert auth_call_duration >= 0.03 + assert auth_call_duration < 0.05 diff --git a/tests/pytests/unit/test_master.py b/tests/pytests/unit/test_master.py index 679229066d..7fccb24d73 100644 --- a/tests/pytests/unit/test_master.py +++ b/tests/pytests/unit/test_master.py @@ -282,3 +282,28 @@ def test_syndic_return_cache_dir_creation_traversal(encrypted_requests): ) assert not (cachedir / "syndics").exists() assert not (cachedir / "mamajama").exists() + + +def test_collect__auth_to_master_stats(): + """ + Check if master stats is collecting _auth calls while not calling neither _handle_aes nor _handle_clear + """ + opts = { + "master_stats": True, + "master_stats_event_iter": 10, + } + req_channel_mock = MagicMock() + mworker = salt.master.MWorker(opts, {}, {}, [req_channel_mock]) + with patch.object(mworker, "_handle_aes") as handle_aes_mock, patch.object( + mworker, "_handle_clear" + ) as handle_clear_mock: + mworker._handle_payload({"cmd": "_auth", "_start": time.time() - 0.02}) + assert mworker.stats["_auth"]["runs"] == 1 + assert mworker.stats["_auth"]["mean"] >= 0.02 + assert mworker.stats["_auth"]["mean"] < 0.04 + mworker._handle_payload({"cmd": "_auth", "_start": time.time() - 0.02}) + assert mworker.stats["_auth"]["runs"] == 2 + assert mworker.stats["_auth"]["mean"] >= 0.02 + assert mworker.stats["_auth"]["mean"] < 0.04 + handle_aes_mock.assert_not_called() + handle_clear_mock.assert_not_called() -- 2.48.1 ++++++ repair-fstab_present-test-mode-702.patch ++++++ >From 73d18711314738796e802a6d929f4b609cee1f67 Mon Sep 17 00:00:00 2001 From: Georg <[email protected]> Date: Fri, 21 Feb 2025 10:26:25 +0000 Subject: [PATCH] Repair fstab_present test mode (#702) Return no pending changes if the configuration already matches. Signed-off-by: Georg Pfuetzenreuter <[email protected]> (cherry picked from commit fc7ed2b53152ab255d7763f200e8d28d526c5e52) --- changelog/67065.fixed.md | 1 + salt/states/mount.py | 1 + tests/pytests/unit/states/test_mount.py | 6 +++--- 3 files changed, 5 insertions(+), 3 deletions(-) create mode 100644 changelog/67065.fixed.md diff --git a/changelog/67065.fixed.md b/changelog/67065.fixed.md new file mode 100644 index 0000000000..7b210dc297 --- /dev/null +++ b/changelog/67065.fixed.md @@ -0,0 +1 @@ +Repaired mount.fstab_present always returning pending changes diff --git a/salt/states/mount.py b/salt/states/mount.py index 36b9a16b5d..97dddbe3b0 100644 --- a/salt/states/mount.py +++ b/salt/states/mount.py @@ -1228,6 +1228,7 @@ def fstab_present( if out == "present": msg = "{} entry is already in {}." ret["comment"].append(msg.format(fs_file, config)) + ret["result"] = True elif out == "new": msg = "{} entry will be written in {}." ret["comment"].append(msg.format(fs_file, config)) diff --git a/tests/pytests/unit/states/test_mount.py b/tests/pytests/unit/states/test_mount.py index 5e4d5274e8..382fe6d0e8 100644 --- a/tests/pytests/unit/states/test_mount.py +++ b/tests/pytests/unit/states/test_mount.py @@ -701,7 +701,7 @@ def test_fstab_present_macos_test_present(): """ ret = { "name": "/dev/sda1", - "result": None, + "result": True, "changes": {}, "comment": ["/home entry is already in /etc/auto_salt."], } @@ -730,7 +730,7 @@ def test_fstab_present_aix_test_present(): """ ret = { "name": "/dev/sda1", - "result": None, + "result": True, "changes": {}, "comment": ["/home entry is already in /etc/filesystems."], } @@ -761,7 +761,7 @@ def test_fstab_present_test_present(): """ ret = { "name": "/dev/sda1", - "result": None, + "result": True, "changes": {}, "comment": ["/home entry is already in /etc/fstab."], } -- 2.48.1 ++++++ repair-virt_query-outputter-655.patch ++++++ >From 325506774381cc8edadee9b2f43fd6733d4f9edb Mon Sep 17 00:00:00 2001 From: Georg <[email protected]> Date: Fri, 21 Feb 2025 12:40:45 +0000 Subject: [PATCH] Repair virt_query outputter (#655) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Repair virt_query outputter Existing code was not functional. Only return if a dictionary is passed and reference the correct data. Signed-off-by: Georg Pfuetzenreuter <[email protected]> (cherry picked from commit e3c365ad8f385121aa878950e13892d986d79656) * Facilitate block devices in virt_query outputter Disk data in Libvirt VMs does not contain a disk size if the disk references a block device. Skip the field for such disks instead of failing with a key error. Signed-off-by: Georg Pfuetzenreuter <[email protected]> (cherry picked from commit ed73abd44117ad745e9c91f2b33caf04402b117c) * Add unit tests for virt_query outputter --------- Co-authored-by: Pablo Suárez Hernández <[email protected]> --- changelog/65841.fixed.md | 1 + salt/output/virt_query.py | 64 +++---- tests/pytests/unit/output/test_virt_query.py | 176 +++++++++++++++++++ 3 files changed, 210 insertions(+), 31 deletions(-) create mode 100644 changelog/65841.fixed.md create mode 100644 tests/pytests/unit/output/test_virt_query.py diff --git a/changelog/65841.fixed.md b/changelog/65841.fixed.md new file mode 100644 index 0000000000..7fb6336ea1 --- /dev/null +++ b/changelog/65841.fixed.md @@ -0,0 +1 @@ +Restore functionality of virt_query outputter and add support for block devices. diff --git a/salt/output/virt_query.py b/salt/output/virt_query.py index d20e6357e6..0f989fedfa 100644 --- a/salt/output/virt_query.py +++ b/salt/output/virt_query.py @@ -12,35 +12,37 @@ def output(data, **kwargs): # pylint: disable=unused-argument Display output for the salt-run virt.query function """ out = "" - for id_ in data["data"]: - out += "{}\n".format(id_) - for vm_ in data["data"][id_]["vm_info"]: - out += " {}\n".format(vm_) - vm_data = data[id_]["vm_info"][vm_] - if "cpu" in vm_data: - out += " CPU: {}\n".format(vm_data["cpu"]) - if "mem" in vm_data: - out += " Memory: {}\n".format(vm_data["mem"]) - if "state" in vm_data: - out += " State: {}\n".format(vm_data["state"]) - if "graphics" in vm_data: - if vm_data["graphics"].get("type", "") == "vnc": - out += " Graphics: vnc - {}:{}\n".format( - id_, vm_data["graphics"]["port"] - ) - if "disks" in vm_data: - for disk, d_data in vm_data["disks"].items(): - out += " Disk - {}:\n".format(disk) - out += " Size: {}\n".format(d_data["disk size"]) - out += " File: {}\n".format(d_data["file"]) - out += " File Format: {}\n".format(d_data["file format"]) - if "nics" in vm_data: - for mac in vm_data["nics"]: - out += " Nic - {}:\n".format(mac) - out += " Source: {}\n".format( - vm_data["nics"][mac]["source"][ - next(iter(vm_data["nics"][mac]["source"].keys())) - ] - ) - out += " Type: {}\n".format(vm_data["nics"][mac]["type"]) + if isinstance(data, dict) and "event" in data: + for id_ in data["event"]["data"]: + out += "{}\n".format(id_) + for vm_ in data["event"]["data"][id_]["vm_info"]: + out += " {}\n".format(vm_) + vm_data = data["event"]["data"][id_]["vm_info"][vm_] + if "cpu" in vm_data: + out += " CPU: {}\n".format(vm_data["cpu"]) + if "mem" in vm_data: + out += " Memory: {}\n".format(vm_data["mem"]) + if "state" in vm_data: + out += " State: {}\n".format(vm_data["state"]) + if "graphics" in vm_data: + if vm_data["graphics"].get("type", "") == "vnc": + out += " Graphics: vnc - {}:{}\n".format( + id_, vm_data["graphics"]["port"] + ) + if "disks" in vm_data: + for disk, d_data in vm_data["disks"].items(): + out += " Disk - {}:\n".format(disk) + if "disk size" in d_data: + out += " Size: {}\n".format(d_data["disk size"]) + out += " File: {}\n".format(d_data["file"]) + out += " File Format: {}\n".format(d_data["file format"]) + if "nics" in vm_data: + for mac in vm_data["nics"]: + out += " NIC - {}:\n".format(mac) + out += " Source: {}\n".format( + vm_data["nics"][mac]["source"][ + next(iter(vm_data["nics"][mac]["source"].keys())) + ] + ) + out += " Type: {}\n".format(vm_data["nics"][mac]["type"]) return out diff --git a/tests/pytests/unit/output/test_virt_query.py b/tests/pytests/unit/output/test_virt_query.py new file mode 100644 index 0000000000..3f8814ee26 --- /dev/null +++ b/tests/pytests/unit/output/test_virt_query.py @@ -0,0 +1,176 @@ +""" +unittests for virt_query outputter +""" + +import pytest + +import salt.output.virt_query as virt_query +from tests.support.mock import patch + + [email protected] +def configure_loader_modules(): + return {virt_query: {}} + + [email protected] +def data(): + return { + "suffix": "progress", + "event": { + "data": { + "mysystem": { + "freecpu": 14, + "freemem": 29566.0, + "node_info": { + "cpucores": 8, + "cpumhz": 1089, + "cpumodel": "x86_64", + "cpus": 16, + "cputhreads": 2, + "numanodes": 1, + "phymemory": 30846, + "sockets": 1, + }, + "vm_info": { + "vm1": { + "cpu": 2, + "cputime": 1214270000000, + "disks": { + "vda": { + "file": "default/vm1-main-disk", + "type": "disk", + "file format": "qcow2", + "virtual size": 214748364800, + "disk size": 1831731200, + "backing file": { + "file": "/var/lib/libvirt/images/sles15sp4o", + "file format": "qcow2", + }, + }, + "hdd": { + "file": "default/vm1-cloudinit-disk", + "type": "cdrom", + "file format": "raw", + "virtual size": 374784, + "disk size": 376832, + }, + }, + "graphics": { + "autoport": "yes", + "keymap": "None", + "listen": "0.0.0.0", + "port": "5900", + "type": "spice", + }, + "nics": { + "aa:bb:cc:dd:ee:ff": { + "type": "network", + "mac": "aa:bb:cc:dd:ee:ff", + "source": {"network": "default"}, + "model": "virtio", + "address": { + "type": "pci", + "domain": "0x0000", + "bus": "0x00", + "slot": "0x03", + "function": "0x0", + }, + } + }, + "uuid": "yyyyyy", + "loader": {"path": "None"}, + "on_crash": "destroy", + "on_reboot": "restart", + "on_poweroff": "destroy", + "maxMem": 1048576, + "mem": 1048576, + "state": "running", + }, + "uyuni-proxy": { + "cpu": 2, + "cputime": 0, + "disks": { + "vda": { + "file": "default/uyuni-proxy-main-disk", + "type": "disk", + "file format": "qcow2", + "virtual size": 214748364800, + "disk size": 4491255808, + "backing file": { + "file": "/var/lib/libvirt/images/leapmicro55o", + "file format": "qcow2", + }, + } + }, + "graphics": { + "autoport": "yes", + "keymap": "None", + "listen": "0.0.0.0", + "port": "None", + "type": "spice", + }, + "nics": { + "aa:bb:cc:dd:ee:aa": { + "type": "network", + "mac": "aa:bb:cc:dd:ee:aa", + "source": {"network": "default"}, + "model": "virtio", + "address": { + "type": "pci", + "domain": "0x0000", + "bus": "0x00", + "slot": "0x03", + "function": "0x0", + }, + } + }, + "uuid": "xxxxx", + "loader": {"path": "None"}, + "on_crash": "destroy", + "on_reboot": "restart", + "on_poweroff": "destroy", + "maxMem": 2097152, + "mem": 2097152, + "state": "shutdown", + }, + }, + } + }, + "outputter": "virt_query", + "_stamp": "2025-02-21T11:28:04.406561", + }, + } + + +def test_default_output(data): + ret = virt_query.output(data) + expected = """mysystem + vm1 + CPU: 2 + Memory: 1048576 + State: running + Disk - vda: + Size: 1831731200 + File: default/vm1-main-disk + File Format: qcow2 + Disk - hdd: + Size: 376832 + File: default/vm1-cloudinit-disk + File Format: raw + NIC - aa:bb:cc:dd:ee:ff: + Source: default + Type: network + uyuni-proxy + CPU: 2 + Memory: 2097152 + State: shutdown + Disk - vda: + Size: 4491255808 + File: default/uyuni-proxy-main-disk + File Format: qcow2 + NIC - aa:bb:cc:dd:ee:aa: + Source: default + Type: network +""" + assert expected == ret -- 2.48.1 ++++++ set-virtual-grain-in-podman-systemd-container-703.patch ++++++ >From dde665763bd2f043022f9601dd25d0ca8aa716be Mon Sep 17 00:00:00 2001 From: Georg <[email protected]> Date: Fri, 21 Feb 2025 10:24:51 +0000 Subject: [PATCH] Set virtual grain in Podman systemd container (#703) Correctly handle the systemd-detect-virt output to identify a Podman container running systemd as what it is instead of as a physical machine. Signed-off-by: Georg Pfuetzenreuter <[email protected]> (cherry picked from commit cf504a06859fb4a4fe9b8ebdd76380697f1f0c25) --- changelog/67733.fixed.md | 1 + salt/grains/core.py | 4 ++++ tests/pytests/unit/grains/test_core.py | 31 ++++++++++++++++++++++++++ 3 files changed, 36 insertions(+) create mode 100644 changelog/67733.fixed.md diff --git a/changelog/67733.fixed.md b/changelog/67733.fixed.md new file mode 100644 index 0000000000..242f65ec76 --- /dev/null +++ b/changelog/67733.fixed.md @@ -0,0 +1 @@ +Set correct virtual grain in systemd based Podman containers diff --git a/salt/grains/core.py b/salt/grains/core.py index 84d5b179dd..ceb142a7b8 100644 --- a/salt/grains/core.py +++ b/salt/grains/core.py @@ -911,6 +911,10 @@ def _virtual(osdata): grains["virtual"] = "container" grains["virtual_subtype"] = "LXC" break + elif "podman" in output: + grains["virtual"] = "container" + grains["virtual_subtype"] = "Podman" + break elif "amazon" in output: grains["virtual"] = "Nitro" grains["virtual_subtype"] = "Amazon EC2" diff --git a/tests/pytests/unit/grains/test_core.py b/tests/pytests/unit/grains/test_core.py index 3d2beaa2c9..072287248f 100644 --- a/tests/pytests/unit/grains/test_core.py +++ b/tests/pytests/unit/grains/test_core.py @@ -1752,6 +1752,37 @@ def test_lxc_virtual_with_virt_what(): assert ret["virtual_subtype"] == "LXC" [email protected]_on_windows +def test_podman_virtual_with_systemd_detect_virt(): + """ + Test if virtual grains are parsed correctly in Podman using systemd-detect-virt. + """ + + def _which_side_effect(path): + if path == "systemd-detect-virt": + return "/usr/bin/systemd-detect-virt" + return None + + with patch.object( + salt.utils.platform, "is_windows", MagicMock(return_value=False) + ), patch.object( + salt.utils.path, + "which", + MagicMock(return_value=True, side_effect=_which_side_effect), + ), patch.dict( + core.__salt__, + { + "cmd.run_all": MagicMock( + return_value={"pid": 78, "retcode": 0, "stderr": "", "stdout": "podman"} + ) + }, + ): + osdata = {"kernel": "test"} + ret = core._virtual(osdata) + assert ret["virtual"] == "container" + assert ret["virtual_subtype"] == "Podman" + + @pytest.mark.skip_on_windows def test_container_inside_virtual_machine(): """ -- 2.48.1
