This is an automated email from the ASF dual-hosted git repository.
rohit pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/cloudstack.git
The following commit(s) were added to refs/heads/master by this push:
new 0331999 systemvm: improve SystemVM startup and memory usage (#3126)
0331999 is described below
commit 033199972ecf0c83975f2ae43c266e92c62ec1a0
Author: Paul Angus <[email protected]>
AuthorDate: Wed Jun 26 10:10:59 2019 +0100
systemvm: improve SystemVM startup and memory usage (#3126)
In order to reduce memory footprint and improve boot speed/predictability.
The following changes have been made:
- add vm.min_free_kbytes to sysctl
- periodically clear disk cache (depending on memory size)
- only start guest services specific to hypervisor
- use systemvm code to determine hypervisor type (not systemd)
- start cloud service at end of post init rather than through systemd
- reduce initial threads started for httpd
- fix vmtools config file
Fixes #3039
Signed-off-by: Rohit Yadav <[email protected]>
---
.../conf-enabled/mods-enabled/mpm_event.conf | 18 ++++++
systemvm/debian/etc/cron.daily/clear_cache | 5 ++
systemvm/debian/etc/cron.hourly/clear_cache | 8 +++
systemvm/debian/etc/sysctl.conf | 6 ++
systemvm/debian/etc/vmware-tools/tools.conf | 17 ++++++
.../systemd/system/baremetal-vr.service | 0
.../{etc => lib}/systemd/system/cloud.service | 2 +-
.../system/hyperv-daemons.hv-fcopy-daemon.service | 9 +++
.../system/hyperv-daemons.hv-kvp-daemon.service | 8 +++
.../system/hyperv-daemons.hv-vss-daemon.service | 8 +++
.../lib/systemd/system/open-vm-tools.service | 12 ++++
.../debian/lib/systemd/system/xe-daemon.service | 15 +++++
systemvm/debian/opt/cloud/bin/setup/bootstrap.sh | 67 ++++++++++++++--------
.../debian/opt/cloud/bin/setup/consoleproxy.sh | 13 +----
systemvm/debian/opt/cloud/bin/setup/postinit.sh | 15 +++--
systemvm/debian/opt/cloud/bin/setup/secstorage.sh | 17 ++----
.../scripts/configure_systemvm_services.sh | 15 ++++-
17 files changed, 181 insertions(+), 54 deletions(-)
diff --git
a/systemvm/debian/etc/apache2/conf-enabled/mods-enabled/mpm_event.conf
b/systemvm/debian/etc/apache2/conf-enabled/mods-enabled/mpm_event.conf
new file mode 100644
index 0000000..ea06042
--- /dev/null
+++ b/systemvm/debian/etc/apache2/conf-enabled/mods-enabled/mpm_event.conf
@@ -0,0 +1,18 @@
+# event MPM
+# StartServers: initial number of server processes to start
+# MinSpareThreads: minimum number of worker threads which are kept spare
+# MaxSpareThreads: maximum number of worker threads which are kept spare
+# ThreadsPerChild: constant number of worker threads in each server process
+# MaxRequestWorkers: maximum number of worker threads
+# MaxConnectionsPerChild: maximum number of requests a server process serves
+<IfModule mpm_event_module>
+ StartServers 1
+ MinSpareThreads 25
+ MaxSpareThreads 75
+ ThreadLimit 64
+ ThreadsPerChild 25
+ MaxRequestWorkers 30
+ MaxConnectionsPerChild 1000
+</IfModule>
+
+# vim: syntax=apache ts=4 sw=4 sts=4 sr noet
diff --git a/systemvm/debian/etc/cron.daily/clear_cache
b/systemvm/debian/etc/cron.daily/clear_cache
new file mode 100755
index 0000000..e2662f9
--- /dev/null
+++ b/systemvm/debian/etc/cron.daily/clear_cache
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+# clear memory cache to ultimately reduce swapping
+
+sync && echo 1 > /proc/sys/vm/drop_caches
diff --git a/systemvm/debian/etc/cron.hourly/clear_cache
b/systemvm/debian/etc/cron.hourly/clear_cache
new file mode 100755
index 0000000..5daa167
--- /dev/null
+++ b/systemvm/debian/etc/cron.hourly/clear_cache
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+# clear memory cache to ultimately reduce swapping
+
+phymem=$(free|awk '/^Mem:/{print $2}')
+if [ $phymem -lt 513000 ]; then
+ sync && echo 1 > /proc/sys/vm/drop_caches
+fi
diff --git a/systemvm/debian/etc/sysctl.conf b/systemvm/debian/etc/sysctl.conf
index 5b5f818..5dd1ae4 100644
--- a/systemvm/debian/etc/sysctl.conf
+++ b/systemvm/debian/etc/sysctl.conf
@@ -63,3 +63,9 @@ net.ipv6.conf.all.autoconf = 0
# Minimum swappiness without disabling it
vm.swappiness=1
+
+# make the kernel more aggressive in reclaiming RAM from the disk and swap
caches
+vm.vfs_cache_pressure = 200
+
+# try to maintain 'free' memory thereby reducing the size of disk cache, hence
reducing swapping.
+vm.min_free_kbytes = 20480
diff --git a/systemvm/debian/etc/vmware-tools/tools.conf
b/systemvm/debian/etc/vmware-tools/tools.conf
new file mode 100644
index 0000000..26f96ff
--- /dev/null
+++ b/systemvm/debian/etc/vmware-tools/tools.conf
@@ -0,0 +1,17 @@
+[logging]
+# Turns on logging globally. It can still be disabled for each domain.
+# log = true
+
+# Disables core dumps on fatal errors; they're enabled by default.
+enableCoreDump = false
+
+# Defines the "vmsvc" domain, logging to file
+# vmsvc.level = message
+vmsvc.handler = file
+# Setup file rotation - keep 3 files
+vmsvc.maxOldLogFiles = 2
+# Max log file size kept: 1 MB
+vmsvc.maxLogSize = 1
+
+# Defines the "vmtoolsd" domain, and disable logging for it.
+# vmtoolsd.level = none
diff --git a/systemvm/debian/etc/systemd/system/baremetal-vr.service
b/systemvm/debian/lib/systemd/system/baremetal-vr.service
similarity index 100%
rename from systemvm/debian/etc/systemd/system/baremetal-vr.service
rename to systemvm/debian/lib/systemd/system/baremetal-vr.service
diff --git a/systemvm/debian/etc/systemd/system/cloud.service
b/systemvm/debian/lib/systemd/system/cloud.service
similarity index 70%
rename from systemvm/debian/etc/systemd/system/cloud.service
rename to systemvm/debian/lib/systemd/system/cloud.service
index 2243923..e3824bf 100644
--- a/systemvm/debian/etc/systemd/system/cloud.service
+++ b/systemvm/debian/lib/systemd/system/cloud.service
@@ -1,6 +1,6 @@
[Unit]
Description=CloudStack Agent service
-After=cloud-early-config.service network.target local-fs.target
+After=cloud-early-config.service network.target cloud-postinit.service
local-fs.target
[Install]
WantedBy=multi-user.target
diff --git
a/systemvm/debian/lib/systemd/system/hyperv-daemons.hv-fcopy-daemon.service
b/systemvm/debian/lib/systemd/system/hyperv-daemons.hv-fcopy-daemon.service
new file mode 100644
index 0000000..12a0b63
--- /dev/null
+++ b/systemvm/debian/lib/systemd/system/hyperv-daemons.hv-fcopy-daemon.service
@@ -0,0 +1,9 @@
+[Unit]
+Description=Hyper-V file copy service (FCOPY) daemon
+ConditionPathExists=/dev/vmbus/hv_fcopy
+
+[Service]
+ExecStart=/usr/sbin/hv_fcopy_daemon -n
+
+[Install]
+WantedBy=multi-user.target
\ No newline at end of file
diff --git
a/systemvm/debian/lib/systemd/system/hyperv-daemons.hv-kvp-daemon.service
b/systemvm/debian/lib/systemd/system/hyperv-daemons.hv-kvp-daemon.service
new file mode 100644
index 0000000..534a25a
--- /dev/null
+++ b/systemvm/debian/lib/systemd/system/hyperv-daemons.hv-kvp-daemon.service
@@ -0,0 +1,8 @@
+[Unit]
+Description=Hyper-V key-value pair (KVP) daemon
+
+[Service]
+ExecStart=/usr/sbin/hv_kvp_daemon -n
+
+[Install]
+WantedBy=multi-user.target
\ No newline at end of file
diff --git
a/systemvm/debian/lib/systemd/system/hyperv-daemons.hv-vss-daemon.service
b/systemvm/debian/lib/systemd/system/hyperv-daemons.hv-vss-daemon.service
new file mode 100644
index 0000000..d4fd675
--- /dev/null
+++ b/systemvm/debian/lib/systemd/system/hyperv-daemons.hv-vss-daemon.service
@@ -0,0 +1,8 @@
+[Unit]
+Description=Hyper-V volume shadow copy service (VSS) daemon
+
+[Service]
+ExecStart=/usr/sbin/hv_vss_daemon -n
+
+[Install]
+WantedBy=multi-user.target
diff --git a/systemvm/debian/lib/systemd/system/open-vm-tools.service
b/systemvm/debian/lib/systemd/system/open-vm-tools.service
new file mode 100644
index 0000000..e7cdde7
--- /dev/null
+++ b/systemvm/debian/lib/systemd/system/open-vm-tools.service
@@ -0,0 +1,12 @@
+[Unit]
+Description=Service for virtual machines hosted on VMware
+Documentation=http://open-vm-tools.sourceforge.net/about.php
+DefaultDependencies=no
+Before=cloud-early-config.service
+
+[Service]
+ExecStart=/usr/bin/vmtoolsd
+TimeoutStopSec=5
+
+[Install]
+WantedBy=multi-user.target
diff --git a/systemvm/debian/lib/systemd/system/xe-daemon.service
b/systemvm/debian/lib/systemd/system/xe-daemon.service
new file mode 100644
index 0000000..6dfef7b
--- /dev/null
+++ b/systemvm/debian/lib/systemd/system/xe-daemon.service
@@ -0,0 +1,15 @@
+[Unit]
+Description=Xen Guest Monitoring Agent
+DefaultDependencies=no
+After=local-fs.target
+Requires=proc-xen.mount
+Before=network.target cloud-early-config.service
+ConditionPathExists=/proc/xen/capabilities
+
+[Service]
+ExecStartPre=/usr/sbin/xe-linux-distribution /var/cache/xe-linux-distribution
+ExecStart=/usr/sbin/xe-daemon
+StandardOutput=journal+console
+
+[Install]
+WantedBy=multi-user.target
diff --git a/systemvm/debian/opt/cloud/bin/setup/bootstrap.sh
b/systemvm/debian/opt/cloud/bin/setup/bootstrap.sh
index 0208b36..5df59a4 100755
--- a/systemvm/debian/opt/cloud/bin/setup/bootstrap.sh
+++ b/systemvm/debian/opt/cloud/bin/setup/bootstrap.sh
@@ -19,6 +19,9 @@
PATH="/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin"
CMDLINE=/var/cache/cloud/cmdline
+rm -f /var/cache/cloud/enabled_svcs
+rm -f /var/cache/cloud/disabled_svcs
+
. /lib/lsb/init-functions
log_it() {
@@ -56,25 +59,25 @@ hypervisor() {
}
config_guest() {
- if [ "$HYPERVISOR" == "kvm" ]
- then
- # Configure hot-plug
- modprobe acpiphp || true
- modprobe pci_hotplug || true
- sed -i -e "/^s0:2345:respawn.*/d" /etc/inittab
- sed -i -e "/6:23:respawn/a\s0:2345:respawn:/sbin/getty -L 115200 ttyS0
vt102" /etc/inittab
- fi
[ ! -d /proc/xen ] && sed -i 's/^vc/#vc/' /etc/inittab && telinit q
- [ -d /proc/xen ] && sed -i 's/^#vc/vc/' /etc/inittab && telinit q
-}
+ [ -d /proc/xen ] && sed -i 's/^#vc/vc/' /etc/inittab && telinit q
+
+ systemctl daemon-reload
-get_boot_params() {
case $HYPERVISOR in
xen-pv|xen-domU)
+ systemctl stop ntpd
+ systemctl disable ntpd
+ systemctl start xe-daemon
+
cat /proc/cmdline > $CMDLINE
sed -i "s/%/ /g" $CMDLINE
;;
xen-hvm)
+ systemctl stop ntpd
+ systemctl disable ntpd
+ systemctl start xe-daemon
+
if [ ! -f /usr/bin/xenstore-read ]; then
log_it "ERROR: xentools not installed, cannot found xenstore-read"
&& exit 5
fi
@@ -82,7 +85,13 @@ get_boot_params() {
sed -i "s/%/ /g" $CMDLINE
;;
kvm)
+ # Configure hot-plug
+ modprobe acpiphp || true
+ modprobe pci_hotplug || true
+ sed -i -e "/^s0:2345:respawn.*/d" /etc/inittab
+ sed -i -e "/6:23:respawn/a\s0:2345:respawn:/sbin/getty -L 115200
ttyS0 vt102" /etc/inittab
systemctl enable --now qemu-guest-agent
+
# Wait for $CMDLINE file to be written by the qemu-guest-agent
for i in {1..60}; do
if [ -s $CMDLINE ]; then
@@ -96,13 +105,16 @@ get_boot_params() {
fi
;;
vmware)
+ # system time sync'd with host via vmware tools
+ systemctl stop ntpd
+ systemctl disable ntpd
+ systemctl start open-vm-tools
+
vmtoolsd --cmd 'machine.id.get' > $CMDLINE
;;
virtualpc|hyperv)
# Hyper-V is recognized as virtualpc hypervisor type. Boot args are
passed using KVP Daemon
- #waiting for the hv_kvp_daemon to start up
- #sleep need to fix the race condition of hv_kvp_daemon and
cloud-early-config
- [ -f /usr/sbin/hv_kvp_daemon ] && /usr/sbin/hv_kvp_daemon
+ systemctl start hyperv-daemons.hv-fcopy-daemon.service
hyperv-daemons.hv-kvp-daemon.service hyperv-daemons.hv-vss-daemon.service
sleep 5
cp -f /var/opt/hyperv/.kvp_pool_0 $CMDLINE
cat /dev/null > /var/opt/hyperv/.kvp_pool_0
@@ -117,13 +129,11 @@ get_boot_params() {
fi
;;
esac
-}
-get_systemvm_type() {
+ # Find and export guest type
export TYPE=$(grep -Po 'type=\K[a-zA-Z]*' $CMDLINE)
}
-
patch_systemvm() {
local patchfile=$1
local backupfolder="/tmp/.conf.backup"
@@ -172,19 +182,29 @@ patch() {
return 0
}
+config_sysctl() {
+ # When there is more memory reset the cache back pressure to default 100
+ physmem=$(free|awk '/^Mem:/{print $2}')
+ if [ $((physmem)) -lt 409600 ]; then
+ sed -i "/^vm.vfs_cache_pressure/ c\vm.vfs_cache_pressure = 200"
/etc/sysctl.conf
+ else
+ sed -i "/^vm.vfs_cache_pressure/ c\vm.vfs_cache_pressure = 100"
/etc/sysctl.conf
+ fi
+
+ sync
+ sysctl -p
+}
+
bootstrap() {
log_it "Bootstrapping systemvm appliance"
export HYPERVISOR=$(hypervisor)
- [ $? -ne 0 ] && log_it "Failed to detect hypervisor type, bailing out of
early init" && exit 10
- log_it "Detected that we are running inside $HYPERVISOR"
+ [ $? -ne 0 ] && log_it "Failed to detect hypervisor type, bailing out" &&
exit 10
+ log_it "Starting guest services for $HYPERVISOR"
config_guest
- get_boot_params
- get_systemvm_type
patch
- sync
- sysctl -p
+ config_sysctl
log_it "Configuring systemvm type=$TYPE"
if [ -f "/opt/cloud/bin/setup/$TYPE.sh" ]; then
@@ -192,6 +212,7 @@ bootstrap() {
else
/opt/cloud/bin/setup/default.sh
fi
+
log_it "Finished setting up systemvm"
exit 0
}
diff --git a/systemvm/debian/opt/cloud/bin/setup/consoleproxy.sh
b/systemvm/debian/opt/cloud/bin/setup/consoleproxy.sh
index 225dc6f..3f00f3d 100755
--- a/systemvm/debian/opt/cloud/bin/setup/consoleproxy.sh
+++ b/systemvm/debian/opt/cloud/bin/setup/consoleproxy.sh
@@ -18,14 +18,13 @@
. /opt/cloud/bin/setup/common.sh
-consoleproxy_svcs() {
+setup_console_proxy() {
+ log_it "Setting up console proxy system vm"
+
echo "cloud" > /var/cache/cloud/enabled_svcs
echo "haproxy dnsmasq apache2 nfs-common portmap" >
/var/cache/cloud/disabled_svcs
mkdir -p /var/log/cloud
-}
-setup_console_proxy() {
- log_it "Setting up console proxy system vm"
setup_common eth0 eth1 eth2
setup_system_rfc1918_internal
@@ -51,10 +50,4 @@ setup_console_proxy() {
rm -f /etc/logrotate.d/cloud
}
-consoleproxy_svcs
-if [ $? -gt 0 ]
-then
- log_it "Failed to execute consoleproxy_svcs"
- exit 1
-fi
setup_console_proxy
diff --git a/systemvm/debian/opt/cloud/bin/setup/postinit.sh
b/systemvm/debian/opt/cloud/bin/setup/postinit.sh
index fc92b7e..5e7e4c0 100755
--- a/systemvm/debian/opt/cloud/bin/setup/postinit.sh
+++ b/systemvm/debian/opt/cloud/bin/setup/postinit.sh
@@ -29,36 +29,39 @@ if [ "$TYPE" == "router" ] || [ "$TYPE" == "vpcrouter" ] ||
[ "$TYPE" == "dhcpsr
then
if [ -x /opt/cloud/bin/update_config.py ]
then
- /opt/cloud/bin/update_config.py cmd_line.json || true
+ /opt/cloud/bin/update_config.py cmd_line.json || true
fi
fi
[ ! -f /var/cache/cloud/enabled_svcs ] && touch /var/cache/cloud/enabled_svcs
for svc in $(cat /var/cache/cloud/enabled_svcs)
do
- systemctl enable --now --no-block $svc
+ systemctl enable --now --no-block $svc
done
[ ! -f /var/cache/cloud/disabled_svcs ] && touch /var/cache/cloud/disabled_svcs
for svc in $(cat /var/cache/cloud/disabled_svcs)
do
- systemctl disable --now --no-block $svc
+ systemctl disable --now --no-block $svc
done
# Restore the persistent iptables nat, rules and filters for IPv4 and IPv6 if
they exist
ipv4="/etc/iptables/rules.v4"
if [ -e $ipv4 ]
then
- iptables-restore < $ipv4
+ iptables-restore < $ipv4
fi
ipv6="/etc/iptables/rules.v6"
if [ -e $ipv6 ]
then
- ip6tables-restore < $ipv6
+ ip6tables-restore < $ipv6
fi
-# Enable SSH
+# Patch known systemd/sshd memory leak -
https://github.com/systemd/systemd/issues/8015#issuecomment-476160981
+echo '@include null' >> /etc/pam.d/systemd-user
+
+# Enable and Start SSH
systemctl enable --now --no-block ssh
date > /var/cache/cloud/boot_up_done
diff --git a/systemvm/debian/opt/cloud/bin/setup/secstorage.sh
b/systemvm/debian/opt/cloud/bin/setup/secstorage.sh
index 8b6d4ee..d3a6d21 100755
--- a/systemvm/debian/opt/cloud/bin/setup/secstorage.sh
+++ b/systemvm/debian/opt/cloud/bin/setup/secstorage.sh
@@ -18,15 +18,12 @@
. /opt/cloud/bin/setup/common.sh
-secstorage_svcs() {
- echo "apache2 cloud nfs-common portmap" > /var/cache/cloud/enabled_svcs
- echo "conntrackd keepalived haproxy dnsmasq" > /var/cache/cloud/disabled_svcs
- mkdir -p /var/log/cloud
-}
-
setup_secstorage() {
log_it "Setting up secondary storage system vm"
- sysctl vm.min_free_kbytes=8192
+
+ echo "cloud apache2 nfs-common portmap" > /var/cache/cloud/enabled_svcs
+ echo "conntrackd keepalived haproxy dnsmasq" > /var/cache/cloud/disabled_svcs
+ mkdir -p /var/log/cloud
setup_common eth0 eth1 eth2
setup_storage_network
@@ -80,10 +77,4 @@ CORS
rm -f /etc/logrotate.d/cloud
}
-secstorage_svcs
-if [ $? -gt 0 ]
-then
- log_it "Failed to execute secstorage_svcs"
- exit 1
-fi
setup_secstorage
diff --git
a/tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh
b/tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh
index 78d868d..56406b7 100644
--- a/tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh
+++ b/tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh
@@ -46,7 +46,7 @@ function install_cloud_scripts() {
rsync -av ./cloud_scripts/ /
chmod +x /opt/cloud/bin/* /opt/cloud/bin/setup/* \
/root/{clearUsageRules.sh,reconfigLB.sh,monitorServices.py} \
- /etc/profile.d/cloud.sh
+ /etc/profile.d/cloud.sh /etc/cron.daily/* /etc/cron.hourly/*
chmod -x /etc/systemd/system/*
@@ -64,6 +64,7 @@ function do_signature() {
function configure_issue() {
cat > /etc/issue <<EOF
+ESC [ 2J
__?.o/ Apache CloudStack SystemVM $CLOUDSTACK_RELEASE
( )# https://cloudstack.apache.org
(___(_) Debian GNU/Linux 9 \n \l
@@ -108,6 +109,18 @@ function configure_services() {
systemctl disable strongswan
systemctl disable x11-common
systemctl disable xl2tpd
+ systemctl disable vgauth
+ systemctl disable sshd
+ systemctl disable nfs-common
+ systemctl disable portmap
+
+ # Disable guest services which will selectively be started based on
hypervisor
+ systemctl disable open-vm-tools
+ systemctl disable xe-daemon
+ systemctl disable hyperv-daemons.hv-fcopy-daemon.service
+ systemctl disable hyperv-daemons.hv-kvp-daemon.service
+ systemctl disable hyperv-daemons.hv-vss-daemon.service
+ systemctl disable qemu-guest-agent
configure_apache2
configure_strongswan
