Rush has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/403308 )

Change subject: tools: ferm pre hook to stop kube-proxy
......................................................................

tools: ferm pre hook to stop kube-proxy

There is a dangerous race condition here between
kube-proxy and ferm.  The only sane thing to do is
have one updating at a time.  This needs to be
revisited and reworked.

Bug: T182722
Change-Id: Icca8d25948451b31e3c0781c67906e93281939fa
---
M modules/role/manifests/toollabs/k8s/worker.pp
M modules/role/manifests/toollabs/proxy.pp
R modules/toollabs/files/ferm_post_handler.sh
A modules/toollabs/files/ferm_pre_handler.sh
A modules/toollabs/manifests/ferm_handlers.pp
D modules/toollabs/manifests/ferm_restart_handler.pp
6 files changed, 55 insertions(+), 23 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/08/403308/1

diff --git a/modules/role/manifests/toollabs/k8s/worker.pp 
b/modules/role/manifests/toollabs/k8s/worker.pp
index 928d82a..dd88f04 100644
--- a/modules/role/manifests/toollabs/k8s/worker.pp
+++ b/modules/role/manifests/toollabs/k8s/worker.pp
@@ -2,7 +2,7 @@
 class role::toollabs::k8s::worker {
     include ::toollabs::infrastructure
     include ::base::firewall
-    include ::toollabs::ferm_restart_handler
+    include ::toollabs::ferm_handlers
 
     $flannel_etcd_url = join(prefix(suffix(hiera('flannel::etcd_hosts'), 
':2379'), 'https://'), ',')
 
diff --git a/modules/role/manifests/toollabs/proxy.pp 
b/modules/role/manifests/toollabs/proxy.pp
index c82cfef..4490ab9 100644
--- a/modules/role/manifests/toollabs/proxy.pp
+++ b/modules/role/manifests/toollabs/proxy.pp
@@ -3,7 +3,7 @@
     include ::toollabs::proxy
     include ::role::toollabs::k8s::webproxy
     include ::base::firewall
-    include ::toollabs::ferm_restart_handler
+    include ::toollabs::ferm_handlers
 
     ferm::service { 'proxymanager':
         proto  => 'tcp',
diff --git a/modules/toollabs/files/ferm_restart_handler.sh 
b/modules/toollabs/files/ferm_post_handler.sh
similarity index 97%
rename from modules/toollabs/files/ferm_restart_handler.sh
rename to modules/toollabs/files/ferm_post_handler.sh
index 692219d..e324bf4 100644
--- a/modules/toollabs/files/ferm_restart_handler.sh
+++ b/modules/toollabs/files/ferm_post_handler.sh
@@ -1,4 +1,4 @@
-#/bin/bash
+#!/bin/bash
 
 /usr/bin/logger -i -t ${0} "restart firewall components post ferm management"
 
diff --git a/modules/toollabs/files/ferm_pre_handler.sh 
b/modules/toollabs/files/ferm_pre_handler.sh
new file mode 100644
index 0000000..0a3301a
--- /dev/null
+++ b/modules/toollabs/files/ferm_pre_handler.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# https://kubernetes.io/docs/reference/generated/kube-proxy/
+# kube-proxy does its own competing state dump and restore
+# we stop kube-proxy here for the duration.
+# Ferm seems to handle these pre-hooks intelligently in that
+# a bad config or an unresolvable host in a rule is checked
+# before any prehooks.  In that case Ferm itself will stop
+# but kube-proxy will never be touched.
+/usr/bin/logger -i -t ${0} "stop kube-proxy"
+service kube-proxy stop
diff --git a/modules/toollabs/manifests/ferm_handlers.pp 
b/modules/toollabs/manifests/ferm_handlers.pp
new file mode 100644
index 0000000..a8c789a
--- /dev/null
+++ b/modules/toollabs/manifests/ferm_handlers.pp
@@ -0,0 +1,41 @@
+# tldr; hook post ferm updates to let other interested
+#       parties resync their iptables state.
+# See: T182722
+# http://ferm.foo-projects.org/download/2.1/ferm.html#hooks
+
+class toollabs::ferm_handlers{
+
+    file {'/usr/local/sbin/ferm_restart_handler':
+        ensure => 'absent',
+        source => 'puppet:///modules/toollabs/ferm_restart_handler.sh',
+        owner  => 'root',
+        group  => 'root',
+        mode   => '0555',
+    }
+
+    file {'/usr/local/sbin/ferm_pre_handler':
+        source => 'puppet:///modules/toollabs/ferm_pre_handler.sh',
+        owner  => 'root',
+        group  => 'root',
+        mode   => '0555',
+    }
+
+    file {'/usr/local/sbin/ferm_post_handler':
+        source => 'puppet:///modules/toollabs/ferm_post_handler.sh',
+        owner  => 'root',
+        group  => 'root',
+        mode   => '0555',
+    }
+
+    ferm::conf{'ferm_pre_handler':
+        prio      => 00,
+        content   => '@hook post "/usr/local/sbin/ferm_pre_handler";',
+        subscribe => File['/usr/local/sbin/ferm_pre_handler'],
+    }
+
+    ferm::conf{'ferm_post_handler':
+        prio      => 00,
+        content   => '@hook post "/usr/local/sbin/ferm_post_handler";',
+        subscribe => File['/usr/local/sbin/ferm_post_handler'],
+    }
+}
diff --git a/modules/toollabs/manifests/ferm_restart_handler.pp 
b/modules/toollabs/manifests/ferm_restart_handler.pp
deleted file mode 100644
index 58a4437..0000000
--- a/modules/toollabs/manifests/ferm_restart_handler.pp
+++ /dev/null
@@ -1,20 +0,0 @@
-# tldr; hook post ferm updates to let other interested
-#       parties resync their iptables state.
-# See: T182722
-class toollabs::ferm_restart_handler{
-
-    file {'/usr/local/sbin/ferm_restart_handler':
-        source => 'puppet:///modules/toollabs/ferm_restart_handler.sh',
-        owner  => 'root',
-        group  => 'root',
-        mode   => '0555',
-    }
-
-    # http://ferm.foo-projects.org/download/2.1/ferm.html#hooks
-    # https://phabricator.wikimedia.org/T182722
-    ferm::conf{'ferm_restart_handler':
-        prio      => 00,
-        content   => '@hook post "/usr/local/sbin/ferm_restart_handler";',
-        subscribe => File['/usr/local/sbin/ferm_restart_handler'],
-    }
-}

-- 
To view, visit https://gerrit.wikimedia.org/r/403308
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Icca8d25948451b31e3c0781c67906e93281939fa
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Rush <r...@wikimedia.org>

_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

Reply via email to