Ori.livneh has uploaded a new change for review.
https://gerrit.wikimedia.org/r/225211
Change subject: Add a script for rolling restart of HHVM servers
......................................................................
Add a script for rolling restart of HHVM servers
This is a hack, but it works reasonably well in practice and it solves a real
problem. I am only adding it to my dotfiles for now, but if anyone notices this
commit and asks for it, I'll move it somewhere public.
Change-Id: Icceb900d02745b2a5c93772068424e67b46a1ead
---
M modules/admin/files/home/ori/.hosts/palladium
1 file changed, 35 insertions(+), 0 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/11/225211/1
diff --git a/modules/admin/files/home/ori/.hosts/palladium
b/modules/admin/files/home/ori/.hosts/palladium
index 2620d3a..c5037c4 100755
--- a/modules/admin/files/home/ori/.hosts/palladium
+++ b/modules/admin/files/home/ori/.hosts/palladium
@@ -57,3 +57,38 @@
_pybal_query "$host"
}
+
+
+hhvm_rolling_restart() {
+ set -e
+ sudo -v # Acquire and/or extend sudo up-front.
+
+ local base_dir="/srv/pybal-config/pybal/eqiad"
+ local temp_dir="$(/bin/mktemp --directory --tmpdir="$HOME"
pybal.XXXXXXXXXX)"
+ local pools=( apaches api rendering )
+
+ echo "* Creating backups..."
+
+ pushd "$base_dir" &>/dev/null
+ cp "${pools[@]}" "$temp_dir"
+ md5sum "${pools[@]}" > "${temp_dir}/checksum"
+
+ for i in {0..1}; do
+ echo "---"
+ echo "* Doing mwXXX${i}.eqiad.wmnet..."
+ md5sum --quiet --check --strict "${temp_dir}/checksum"
+ for pool in "${pools[@]}"; do
+ sudo sed -i'' -e "s/\(mw...${i}.eqiad.wmnet.*\)True/\1False/"
"$pool"
+ done
+ echo "* Sleeping for 30 seconds to drain connections..."
+ sleep 30
+ sudo salt "mw*${i}.eqiad.wmnet" cmd.run "status hhvm 2>/dev/null |
grep -q running && restart hhvm"
+ pushd "$temp_dir" &>/dev/null
+ sudo cp "${pools[@]}" "${base_dir}"
+ popd &>/dev/null
+ md5sum --check --strict "${temp_dir}/checksum"
+ done
+
+ popd &>/dev/null
+ rm -rf "$temp_dir"
+}
--
To view, visit https://gerrit.wikimedia.org/r/225211
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Icceb900d02745b2a5c93772068424e67b46a1ead
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ori.livneh <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
