Andrew Bogott has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/397879 )
Change subject: tools k8s workers: add a mostly-permissive firewall
......................................................................
tools k8s workers: add a mostly-permissive firewall
Bug: T180055
Change-Id: Icb9fc1a1f9d12708b8dc71282b4b4082e0652b8a
---
M hieradata/labs.yaml
M modules/profile/manifests/kubernetes/node.pp
M modules/role/manifests/toollabs/k8s/worker.pp
3 files changed, 58 insertions(+), 20 deletions(-)
Approvals:
Andrew Bogott: Verified; Looks good to me, approved
Madhuvishy: Looks good to me, but someone else must approve
diff --git a/hieradata/labs.yaml b/hieradata/labs.yaml
index ce53a17..f91ad7e 100644
--- a/hieradata/labs.yaml
+++ b/hieradata/labs.yaml
@@ -135,3 +135,6 @@
jenkins_agent_username: 'jenkins-deploy'
role::logging::mediawiki::udp2log::logstash_host:
'deployment-logstash2.deployment-prep.eqiad.wmflabs'
+
+# Don't use prod-style firewalls for k8s nodes
+profile::kubernetes::node::prod_firewalls: false
diff --git a/modules/profile/manifests/kubernetes/node.pp
b/modules/profile/manifests/kubernetes/node.pp
index 9498460..a772e9b 100644
--- a/modules/profile/manifests/kubernetes/node.pp
+++ b/modules/profile/manifests/kubernetes/node.pp
@@ -8,6 +8,7 @@
$prometheus_nodes = hiera('prometheus_nodes', []),
$kubelet_config = hiera('profile::kubernetes::node::kubelet_config',
'/etc/kubernetes/kubeconfig'),
$kubeproxy_config = hiera('profile::kubernetes::node::kubeproxy_config',
'/etc/kubernetes/kubeconfig'),
+ $prod_firewalls = hiera('profile::kubernetes::node::prod_firewalls', true),
) {
base::expose_puppet_certs { '/etc/kubernetes':
@@ -36,19 +37,22 @@
kubeconfig => $kubeproxy_config,
}
- $master_hosts_ferm = join($master_hosts, ' ')
- ferm::service { 'kubelet-http':
- proto => 'tcp',
- port => '10250',
- srange => "(@resolve((${master_hosts_ferm}))
@resolve((${master_hosts_ferm}), AAAA))",
- }
-
- if !empty($prometheus_nodes) {
- $prometheus_ferm_nodes = join($prometheus_nodes, ' ')
- ferm::service { 'kubelet-http-readonly-prometheus':
+ # We can't use this for VMs because of the AAAA lookups
+ if $prod_firewalls {
+ $master_hosts_ferm = join($master_hosts, ' ')
+ ferm::service { 'kubelet-http':
proto => 'tcp',
- port => '10255',
- srange => "(@resolve((${prometheus_ferm_nodes}))
@resolve((${prometheus_ferm_nodes}), AAAA))"
+ port => '10250',
+ srange => "(@resolve((${master_hosts_ferm}))
@resolve((${master_hosts_ferm}), AAAA))",
+ }
+
+ if !empty($prometheus_nodes) {
+ $prometheus_ferm_nodes = join($prometheus_nodes, ' ')
+ ferm::service { 'kubelet-http-readonly-prometheus':
+ proto => 'tcp',
+ port => '10255',
+ srange => "(@resolve((${prometheus_ferm_nodes}))
@resolve((${prometheus_ferm_nodes}), AAAA))"
+ }
}
}
# Alert us if kubelet operational latencies exceed a certain threshold.
TODO: reevaluate
diff --git a/modules/role/manifests/toollabs/k8s/worker.pp
b/modules/role/manifests/toollabs/k8s/worker.pp
index 52c6cc4..76f34fb 100644
--- a/modules/role/manifests/toollabs/k8s/worker.pp
+++ b/modules/role/manifests/toollabs/k8s/worker.pp
@@ -4,11 +4,6 @@
$flannel_etcd_url = join(prefix(suffix(hiera('flannel::etcd_hosts'),
':2379'), 'https://'), ',')
- ferm::service { 'flannel-vxlan':
- proto => udp,
- port => 8472,
- }
-
class { '::k8s::flannel':
etcd_endpoints => $flannel_etcd_url,
}
@@ -38,8 +33,44 @@
class { '::profile::kubernetes::node':
- use_cni => false,
- infra_pod => 'docker-registry.tools.wmflabs.org/pause:2.0',
- require => Class[::profile::docker::flannel],
+ use_cni => false,
+ infra_pod => 'docker-registry.tools.wmflabs.org/pause:2.0',
+ require => Class[::profile::docker::flannel],
+ prod_firewalls => false,
}
+
+ # Firewall! Kubelet opens some scary ports to the outside world,
+ # so this class just closes those particular ports whilst leaving
everything
+ # else in the hands of the OpenStack security groups.
+ $master_hosts = hiera('k8s::master_hosts')
+ $master_hosts_ferm = join($master_hosts, ' ')
+
+ ferm::service { 'tools-kubelet-http':
+ proto => 'tcp',
+ port => '10250',
+ srange => "@resolve((${master_hosts_ferm}))",
+ }
+ ferm::service { 'tools-kubelet-http-readonly-prometheus':
+ proto => 'tcp',
+ port => '10255',
+ srange => "@resolve((${master_hosts_ferm}))",
+ }
+
+ ferm::service { 'flannel-vxlan':
+ proto => udp,
+ port => 8472,
+ }
+
+ # We really only want to be this permissive for other tools hosts.
+ # Fortunately there's a nova-network security rule overlaying this
+ # one which limits this permissive policy to things within the tools
+ # project.
+ #
+ # Ideally this will get winnowed down as time passes, but for the
+ # moment I just really want to get the above things properly closed off
+ ferm::rule {'rest-of-everything':
+ rule => 'saddr 10.0.0.0/8 proto tcp dport (1:8472 8473:10249
10251:10254 10256:65535) ACCEPT;'
+ }
+
+ include profile::base::firewall
}
--
To view, visit https://gerrit.wikimedia.org/r/397879
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Icb9fc1a1f9d12708b8dc71282b4b4082e0652b8a
Gerrit-PatchSet: 8
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Andrew Bogott <abog...@wikimedia.org>
Gerrit-Reviewer: Andrew Bogott <abog...@wikimedia.org>
Gerrit-Reviewer: Giuseppe Lavagetto <glavage...@wikimedia.org>
Gerrit-Reviewer: Madhuvishy <mviswanat...@wikimedia.org>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
