This is an automated email from the ASF dual-hosted git repository.
nvazquez pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/cloudstack.git
The following commit(s) were added to refs/heads/main by this push:
new a95eb5970c0 systemvm: fix subprocess deadlock in router health check
scripts (#13737)
a95eb5970c0 is described below
commit a95eb5970c0b23887f369336d35264cc0ccc0451
Author: The Viking <[email protected]>
AuthorDate: Mon Nov 9 12:15:23 2026 -0500
systemvm: fix subprocess deadlock in router health check scripts (#13737)
The health check scripts run shell commands with Popen(stdout=PIPE) and
call wait() before reading the pipe. When the command output exceeds the
OS pipe capacity (64 KiB on Linux), the child blocks writing to the full
pipe and the parent blocks in wait() forever. This is the deadlock warned
about in the Python subprocess documentation.
On routers with large iptables rule sets this makes iptables_check.py
hang on every run of the advanced health check. A new stuck process chain
accumulates each advanced-check interval (10 minutes by default) until
the router exhausts memory and swap, keepalived's track script starves,
and the router enters FAULT and reboots. On a 512 MB router this produced
a reboot roughly every 9 hours, and redundant pairs failed together
because their timers are synchronized at creation.
Drain the pipe with communicate() and check returncode instead of calling
wait() with an unread pipe. The same latent pattern is also fixed in
memory_usage_check.py, cpu_usage_check.py and gateways_check.py.
Fixes: #13735
---
systemvm/debian/root/health_checks/cpu_usage_check.py | 5 +++--
systemvm/debian/root/health_checks/gateways_check.py | 3 ++-
systemvm/debian/root/health_checks/iptables_check.py | 5 +++--
systemvm/debian/root/health_checks/memory_usage_check.py | 5 +++--
4 files changed, 11 insertions(+), 7 deletions(-)
diff --git a/systemvm/debian/root/health_checks/cpu_usage_check.py
b/systemvm/debian/root/health_checks/cpu_usage_check.py
index 270ea208610..c753b86fbbc 100644
--- a/systemvm/debian/root/health_checks/cpu_usage_check.py
+++ b/systemvm/debian/root/health_checks/cpu_usage_check.py
@@ -37,8 +37,9 @@ def main():
"'{ split($1, vs, \",\"); idle=vs[length(vs)]; " \
"sub(\"%\", \"\", idle); printf \"%.2f\", 100 - idle }'"
pout = Popen(cmd, shell=True, stdout=PIPE)
- if pout.wait() == 0:
- currentUsage = float(pout.communicate()[0].decode().strip())
+ stdout, _ = pout.communicate()
+ if pout.returncode == 0:
+ currentUsage = float(stdout.decode().strip())
if currentUsage > maxCpuUsage:
print("CPU Usage " + str(currentUsage) +
"% has crossed threshold of " + str(maxCpuUsage) + "%")
diff --git a/systemvm/debian/root/health_checks/gateways_check.py
b/systemvm/debian/root/health_checks/gateways_check.py
index e3b661b7498..19841c97b1d 100644
--- a/systemvm/debian/root/health_checks/gateways_check.py
+++ b/systemvm/debian/root/health_checks/gateways_check.py
@@ -36,7 +36,8 @@ def main():
for i in range(5):
pingCmd = "ping " + gw + " -c 5 -w 10"
pout = Popen(pingCmd, shell=True, stdout=PIPE)
- if pout.wait() == 0:
+ pout.communicate()
+ if pout.returncode == 0:
reachableGw = True
break
diff --git a/systemvm/debian/root/health_checks/iptables_check.py
b/systemvm/debian/root/health_checks/iptables_check.py
index 27e06f8352b..ebf7b4bd825 100644
--- a/systemvm/debian/root/health_checks/iptables_check.py
+++ b/systemvm/debian/root/health_checks/iptables_check.py
@@ -41,13 +41,14 @@ def main():
fetchIpTableEntriesCmd = "iptables-save | grep " + destIp
pout = Popen(fetchIpTableEntriesCmd, shell=True, stdout=PIPE)
- if pout.wait() != 0:
+ stdout, _ = pout.communicate()
+ if pout.returncode != 0:
failedCheck = True
failureMessage = failureMessage + "Unable to execute iptables-save
command " \
"for fetching rules by " +
fetchIpTableEntriesCmd + "\n"
continue
- ipTablesMatchingEntries =
pout.communicate()[0].decode().strip().split('\n')
+ ipTablesMatchingEntries = stdout.decode().strip().split('\n')
for pfEntryListExpected in entriesExpected:
foundPfEntryList = False
for ipTableEntry in ipTablesMatchingEntries:
diff --git a/systemvm/debian/root/health_checks/memory_usage_check.py
b/systemvm/debian/root/health_checks/memory_usage_check.py
index eba0d5e49dd..f468160648f 100644
--- a/systemvm/debian/root/health_checks/memory_usage_check.py
+++ b/systemvm/debian/root/health_checks/memory_usage_check.py
@@ -35,9 +35,10 @@ def main():
maxMemoryUsage = float(data["maxMemoryUsage"])
cmd = "free | awk 'FNR == 2 { print $3 * 100 / $2 }'"
pout = Popen(cmd, shell=True, stdout=PIPE)
+ stdout, _ = pout.communicate()
- if pout.wait() == 0:
- currentUsage = float(pout.communicate()[0].decode().strip())
+ if pout.returncode == 0:
+ currentUsage = float(stdout.decode().strip())
if currentUsage > maxMemoryUsage:
print("Memory Usage " + str(currentUsage) +
"% has crossed threshold of " + str(maxMemoryUsage) + "%")
