Rush has uploaded a new change for review. ( https://gerrit.wikimedia.org/r/403308 )
Change subject: tools: ferm pre hook to stop kube-proxy ...................................................................... tools: ferm pre hook to stop kube-proxy There is a dangerous race condition here between kube-proxy and ferm. The only sane thing to do is have one updating at a time. This needs to be revisited and reworked. Bug: T182722 Change-Id: Icca8d25948451b31e3c0781c67906e93281939fa --- M modules/role/manifests/toollabs/k8s/worker.pp M modules/role/manifests/toollabs/proxy.pp R modules/toollabs/files/ferm_post_handler.sh A modules/toollabs/files/ferm_pre_handler.sh A modules/toollabs/manifests/ferm_handlers.pp D modules/toollabs/manifests/ferm_restart_handler.pp 6 files changed, 55 insertions(+), 23 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/operations/puppet refs/changes/08/403308/1 diff --git a/modules/role/manifests/toollabs/k8s/worker.pp b/modules/role/manifests/toollabs/k8s/worker.pp index 928d82a..dd88f04 100644 --- a/modules/role/manifests/toollabs/k8s/worker.pp +++ b/modules/role/manifests/toollabs/k8s/worker.pp @@ -2,7 +2,7 @@ class role::toollabs::k8s::worker { include ::toollabs::infrastructure include ::base::firewall - include ::toollabs::ferm_restart_handler + include ::toollabs::ferm_handlers $flannel_etcd_url = join(prefix(suffix(hiera('flannel::etcd_hosts'), ':2379'), 'https://'), ',') diff --git a/modules/role/manifests/toollabs/proxy.pp b/modules/role/manifests/toollabs/proxy.pp index c82cfef..4490ab9 100644 --- a/modules/role/manifests/toollabs/proxy.pp +++ b/modules/role/manifests/toollabs/proxy.pp @@ -3,7 +3,7 @@ include ::toollabs::proxy include ::role::toollabs::k8s::webproxy include ::base::firewall - include ::toollabs::ferm_restart_handler + include ::toollabs::ferm_handlers ferm::service { 'proxymanager': proto => 'tcp', diff --git a/modules/toollabs/files/ferm_restart_handler.sh b/modules/toollabs/files/ferm_post_handler.sh similarity index 97% rename from modules/toollabs/files/ferm_restart_handler.sh rename to modules/toollabs/files/ferm_post_handler.sh index 692219d..e324bf4 100644 --- a/modules/toollabs/files/ferm_restart_handler.sh +++ b/modules/toollabs/files/ferm_post_handler.sh @@ -1,4 +1,4 @@ -#/bin/bash +#!/bin/bash /usr/bin/logger -i -t ${0} "restart firewall components post ferm management" diff --git a/modules/toollabs/files/ferm_pre_handler.sh b/modules/toollabs/files/ferm_pre_handler.sh new file mode 100644 index 0000000..0a3301a --- /dev/null +++ b/modules/toollabs/files/ferm_pre_handler.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +# https://kubernetes.io/docs/reference/generated/kube-proxy/ +# kube-proxy does its own competing state dump and restore +# we stop kube-proxy here for the duration. +# Ferm seems to handle these pre-hooks intelligently in that +# a bad config or an unresolvable host in a rule is checked +# before any prehooks. In that case Ferm itself will stop +# but kube-proxy will never be touched. +/usr/bin/logger -i -t ${0} "stop kube-proxy" +service kube-proxy stop diff --git a/modules/toollabs/manifests/ferm_handlers.pp b/modules/toollabs/manifests/ferm_handlers.pp new file mode 100644 index 0000000..a8c789a --- /dev/null +++ b/modules/toollabs/manifests/ferm_handlers.pp @@ -0,0 +1,41 @@ +# tldr; hook post ferm updates to let other interested +# parties resync their iptables state. +# See: T182722 +# http://ferm.foo-projects.org/download/2.1/ferm.html#hooks + +class toollabs::ferm_handlers{ + + file {'/usr/local/sbin/ferm_restart_handler': + ensure => 'absent', + source => 'puppet:///modules/toollabs/ferm_restart_handler.sh', + owner => 'root', + group => 'root', + mode => '0555', + } + + file {'/usr/local/sbin/ferm_pre_handler': + source => 'puppet:///modules/toollabs/ferm_pre_handler.sh', + owner => 'root', + group => 'root', + mode => '0555', + } + + file {'/usr/local/sbin/ferm_post_handler': + source => 'puppet:///modules/toollabs/ferm_post_handler.sh', + owner => 'root', + group => 'root', + mode => '0555', + } + + ferm::conf{'ferm_pre_handler': + prio => 00, + content => '@hook post "/usr/local/sbin/ferm_pre_handler";', + subscribe => File['/usr/local/sbin/ferm_pre_handler'], + } + + ferm::conf{'ferm_post_handler': + prio => 00, + content => '@hook post "/usr/local/sbin/ferm_post_handler";', + subscribe => File['/usr/local/sbin/ferm_post_handler'], + } +} diff --git a/modules/toollabs/manifests/ferm_restart_handler.pp b/modules/toollabs/manifests/ferm_restart_handler.pp deleted file mode 100644 index 58a4437..0000000 --- a/modules/toollabs/manifests/ferm_restart_handler.pp +++ /dev/null @@ -1,20 +0,0 @@ -# tldr; hook post ferm updates to let other interested -# parties resync their iptables state. -# See: T182722 -class toollabs::ferm_restart_handler{ - - file {'/usr/local/sbin/ferm_restart_handler': - source => 'puppet:///modules/toollabs/ferm_restart_handler.sh', - owner => 'root', - group => 'root', - mode => '0555', - } - - # http://ferm.foo-projects.org/download/2.1/ferm.html#hooks - # https://phabricator.wikimedia.org/T182722 - ferm::conf{'ferm_restart_handler': - prio => 00, - content => '@hook post "/usr/local/sbin/ferm_restart_handler";', - subscribe => File['/usr/local/sbin/ferm_restart_handler'], - } -} -- To view, visit https://gerrit.wikimedia.org/r/403308 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Icca8d25948451b31e3c0781c67906e93281939fa Gerrit-PatchSet: 1 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: Rush <r...@wikimedia.org> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits