Rush has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/403308 )
Change subject: tools: ferm pre hook to stop kube-proxy
......................................................................
tools: ferm pre hook to stop kube-proxy
There is a dangerous race condition here between
kube-proxy and ferm. The only sane thing to do is
have one updating at a time. This needs to be
revisited and reworked.
Bug: T182722
Change-Id: Icca8d25948451b31e3c0781c67906e93281939fa
---
M modules/role/manifests/toollabs/k8s/worker.pp
M modules/role/manifests/toollabs/proxy.pp
R modules/toollabs/files/ferm_post_handler.sh
A modules/toollabs/files/ferm_pre_handler.sh
A modules/toollabs/manifests/ferm_handlers.pp
D modules/toollabs/manifests/ferm_restart_handler.pp
6 files changed, 55 insertions(+), 23 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/08/403308/1
diff --git a/modules/role/manifests/toollabs/k8s/worker.pp
b/modules/role/manifests/toollabs/k8s/worker.pp
index 928d82a..dd88f04 100644
--- a/modules/role/manifests/toollabs/k8s/worker.pp
+++ b/modules/role/manifests/toollabs/k8s/worker.pp
@@ -2,7 +2,7 @@
class role::toollabs::k8s::worker {
include ::toollabs::infrastructure
include ::base::firewall
- include ::toollabs::ferm_restart_handler
+ include ::toollabs::ferm_handlers
$flannel_etcd_url = join(prefix(suffix(hiera('flannel::etcd_hosts'),
':2379'), 'https://'), ',')
diff --git a/modules/role/manifests/toollabs/proxy.pp
b/modules/role/manifests/toollabs/proxy.pp
index c82cfef..4490ab9 100644
--- a/modules/role/manifests/toollabs/proxy.pp
+++ b/modules/role/manifests/toollabs/proxy.pp
@@ -3,7 +3,7 @@
include ::toollabs::proxy
include ::role::toollabs::k8s::webproxy
include ::base::firewall
- include ::toollabs::ferm_restart_handler
+ include ::toollabs::ferm_handlers
ferm::service { 'proxymanager':
proto => 'tcp',
diff --git a/modules/toollabs/files/ferm_restart_handler.sh
b/modules/toollabs/files/ferm_post_handler.sh
similarity index 97%
rename from modules/toollabs/files/ferm_restart_handler.sh
rename to modules/toollabs/files/ferm_post_handler.sh
index 692219d..e324bf4 100644
--- a/modules/toollabs/files/ferm_restart_handler.sh
+++ b/modules/toollabs/files/ferm_post_handler.sh
@@ -1,4 +1,4 @@
-#/bin/bash
+#!/bin/bash
/usr/bin/logger -i -t ${0} "restart firewall components post ferm management"
diff --git a/modules/toollabs/files/ferm_pre_handler.sh
b/modules/toollabs/files/ferm_pre_handler.sh
new file mode 100644
index 0000000..0a3301a
--- /dev/null
+++ b/modules/toollabs/files/ferm_pre_handler.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# https://kubernetes.io/docs/reference/generated/kube-proxy/
+# kube-proxy does its own competing state dump and restore
+# we stop kube-proxy here for the duration.
+# Ferm seems to handle these pre-hooks intelligently in that
+# a bad config or an unresolvable host in a rule is checked
+# before any prehooks. In that case Ferm itself will stop
+# but kube-proxy will never be touched.
+/usr/bin/logger -i -t ${0} "stop kube-proxy"
+service kube-proxy stop
diff --git a/modules/toollabs/manifests/ferm_handlers.pp
b/modules/toollabs/manifests/ferm_handlers.pp
new file mode 100644
index 0000000..a8c789a
--- /dev/null
+++ b/modules/toollabs/manifests/ferm_handlers.pp
@@ -0,0 +1,41 @@
+# tldr; hook post ferm updates to let other interested
+# parties resync their iptables state.
+# See: T182722
+# http://ferm.foo-projects.org/download/2.1/ferm.html#hooks
+
+class toollabs::ferm_handlers{
+
+ file {'/usr/local/sbin/ferm_restart_handler':
+ ensure => 'absent',
+ source => 'puppet:///modules/toollabs/ferm_restart_handler.sh',
+ owner => 'root',
+ group => 'root',
+ mode => '0555',
+ }
+
+ file {'/usr/local/sbin/ferm_pre_handler':
+ source => 'puppet:///modules/toollabs/ferm_pre_handler.sh',
+ owner => 'root',
+ group => 'root',
+ mode => '0555',
+ }
+
+ file {'/usr/local/sbin/ferm_post_handler':
+ source => 'puppet:///modules/toollabs/ferm_post_handler.sh',
+ owner => 'root',
+ group => 'root',
+ mode => '0555',
+ }
+
+ ferm::conf{'ferm_pre_handler':
+ prio => 00,
+ content => '@hook post "/usr/local/sbin/ferm_pre_handler";',
+ subscribe => File['/usr/local/sbin/ferm_pre_handler'],
+ }
+
+ ferm::conf{'ferm_post_handler':
+ prio => 00,
+ content => '@hook post "/usr/local/sbin/ferm_post_handler";',
+ subscribe => File['/usr/local/sbin/ferm_post_handler'],
+ }
+}
diff --git a/modules/toollabs/manifests/ferm_restart_handler.pp
b/modules/toollabs/manifests/ferm_restart_handler.pp
deleted file mode 100644
index 58a4437..0000000
--- a/modules/toollabs/manifests/ferm_restart_handler.pp
+++ /dev/null
@@ -1,20 +0,0 @@
-# tldr; hook post ferm updates to let other interested
-# parties resync their iptables state.
-# See: T182722
-class toollabs::ferm_restart_handler{
-
- file {'/usr/local/sbin/ferm_restart_handler':
- source => 'puppet:///modules/toollabs/ferm_restart_handler.sh',
- owner => 'root',
- group => 'root',
- mode => '0555',
- }
-
- # http://ferm.foo-projects.org/download/2.1/ferm.html#hooks
- # https://phabricator.wikimedia.org/T182722
- ferm::conf{'ferm_restart_handler':
- prio => 00,
- content => '@hook post "/usr/local/sbin/ferm_restart_handler";',
- subscribe => File['/usr/local/sbin/ferm_restart_handler'],
- }
-}
--
To view, visit https://gerrit.wikimedia.org/r/403308
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Icca8d25948451b31e3c0781c67906e93281939fa
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Rush <r...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
