Alexandros Kosiaris has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/374795 )
Change subject: kubernetes: Refactor/add admission controllers
......................................................................
kubernetes: Refactor/add admission controllers
Migrate away from the disjoint structure of an array for the admission
controllers and separate parameters for their configuration to a more
tightly coupled structure where admission controllers are specified as a
Hash with keys the admission controller names and values their
respective parameters. That allows for greater flexibility and decouples
the definition of the class from the implementation details of the
controllers and their parameters.
Amend the roles for production, staging as well as labs to keep the
status quo.
There are two glaring differences in production/staging, that
is we no longer enforce the registry. That is expected to be enforced
later on if possible via the dynamic admission controllers, but even now
it is not an immediate issue due to the networking policies we have in
place. The second is that we add the ServiceAccount controller per the
recommendedation of the kubernetes docs
Bug: T170119
Change-Id: Ie960a72da62b12beca2408e1b5028ea50ef3d5d5
---
M hieradata/role/common/kubernetes/master.yaml
M hieradata/role/common/kubernetes/staging/master.yaml
M modules/k8s/manifests/apiserver.pp
M modules/k8s/templates/kube-apiserver.default.erb
M modules/profile/manifests/kubernetes/master.pp
M modules/role/manifests/toollabs/k8s/master.pp
6 files changed, 61 insertions(+), 79 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/95/374795/1
diff --git a/hieradata/role/common/kubernetes/master.yaml
b/hieradata/role/common/kubernetes/master.yaml
index 8891dc8..35e22c0 100644
--- a/hieradata/role/common/kubernetes/master.yaml
+++ b/hieradata/role/common/kubernetes/master.yaml
@@ -2,22 +2,19 @@
debdeploy-kubernetes-eqiad:
value: standard
cluster: kubernetes
-profile::kubernetes::master::docker_registry: darmstadtium.eqiad.wmnet
profile::kubernetes::master::accessible_to: all
profile::kubernetes::master::apiserver_count: 2
profile::kubernetes::master::admission_controllers:
- - NamespaceLifecycle
- - ResourceQuota
- - LimitRanger
- - RegistryEnforcer
- - DefaultStorageClass
+ NamespaceLifecycle: ''
+ LimitRanger: ''
+ ServiceAccount: ''
+ DefaultStorageClass: ''
+ ResourceQuota: ''
profile::kubernetes::master::expose_puppet_certs: true
profile::kubernetes::master::service_cert: "kubemaster.svc.%{::site}.wmnet"
profile::kubernetes::master::ssl_cert_path:
"/etc/ssl/localcerts/kubemaster.svc.%{::site}.wmnet.crt"
profile::kubernetes::master::ssl_key_path:
"/etc/ssl/private/kubemaster.svc.%{::site}.wmnet.key"
profile::kubernetes::master::authz_mode: ''
-profile::kubernetes::master::host_automounts: []
-profile::kubernetes::master::host_path_prefixes_allowed: []
# TODO: This needs to become a profile
role::lvs::realserver::pools:
kubemaster: {}
diff --git a/hieradata/role/common/kubernetes/staging/master.yaml
b/hieradata/role/common/kubernetes/staging/master.yaml
index 8161129..53883de 100644
--- a/hieradata/role/common/kubernetes/staging/master.yaml
+++ b/hieradata/role/common/kubernetes/staging/master.yaml
@@ -2,23 +2,20 @@
debdeploy-kubernetes-eqiad:
value: standard
cluster: kubernetes
-profile::kubernetes::master::docker_registry: darmstadtium.eqiad.wmnet
profile::kubernetes::master::accessible_to:
- kubestage1001.eqiad.wmnet
- kubestage1002.eqiad.wmnet
profile::kubernetes::master::apiserver_count: 1
profile::kubernetes::master::admission_controllers:
- - NamespaceLifecycle
- - ResourceQuota
- - LimitRanger
- - RegistryEnforcer
- - DefaultStorageClass
+ NamespaceLifecycle: ''
+ LimitRanger: ''
+ ServiceAccount: ''
+ DefaultStorageClass: ''
+ ResourceQuota: ''
profile::kubernetes::master::expose_puppet_certs: true
profile::kubernetes::master::ssl_cert_path: "/etc/kubernetes/ssl/cert.pem"
profile::kubernetes::master::ssl_key_path: "/etc/kubernetes/ssl/server.key"
profile::kubernetes::master::authz_mode: ''
-profile::kubernetes::master::host_automounts: []
-profile::kubernetes::master::host_path_prefixes_allowed: []
profile::kubernetes::master::service_cluster_ip_range: 10.64.76.0/24
profile::kubernetes::master::etcd_urls:
- https://kubestagetcd1001.eqiad.wmnet:2379
diff --git a/modules/k8s/manifests/apiserver.pp
b/modules/k8s/manifests/apiserver.pp
index 9f401ad..2a7c550 100644
--- a/modules/k8s/manifests/apiserver.pp
+++ b/modules/k8s/manifests/apiserver.pp
@@ -1,23 +1,17 @@
class k8s::apiserver(
$etcd_servers,
- $docker_registry,
$ssl_cert_path=undef,
$ssl_key_path=undef,
$kube_api_port = undef,
$kubelet_port = undef,
$service_cluster_ip_range = '192.168.0.0/17',
- $admission_controllers = [
- 'NamespaceLifecycle',
- 'ResourceQuota',
- 'LimitRanger',
- 'UidEnforcer',
- 'RegistryEnforcer',
- 'HostAutomounter',
- 'HostPathEnforcer',
- ],
- $host_automounts = [],
- $host_paths_allowed = [],
- $host_path_prefixes_allowed = [],
+ $admission_controllers = {
+ 'NamespaceLifecycle' => '',
+ 'LimitRanger' => '',
+ 'ServiceAccount' => '',
+ 'DefaultStorageClass' => '',
+ 'ResourceQuota' => '',
+ },
$authz_mode = 'abac',
$apiserver_count = undef,
) {
@@ -31,10 +25,8 @@
require_package('kubernetes-master')
require_package('kubernetes-client')
- $host_automounts_string = join($host_automounts, ',')
- $host_paths_allowed_string = join(concat($host_paths_allowed,
$host_automounts), ',')
- $host_path_prefixes_allowed_string = join($host_path_prefixes_allowed, ',')
- $admission_control = join($admission_controllers, ',')
+ $admission_control = join(keys($admission_controllers), ',')
+ $admission_control_params = join(values($admission_controllers), ' ')
$users = hiera('k8s_infrastructure_users')
file { '/etc/kubernetes/infrastructure-users':
diff --git a/modules/k8s/templates/kube-apiserver.default.erb
b/modules/k8s/templates/kube-apiserver.default.erb
index 2ebf8ec..fee203d 100644
--- a/modules/k8s/templates/kube-apiserver.default.erb
+++ b/modules/k8s/templates/kube-apiserver.default.erb
@@ -40,7 +40,4 @@
--runtime-config=batch/v2alpha1 \
--tls-cert-file=<%= @ssl_cert_path %> \
--tls-private-key-file=<%= @ssl_key_path %> \
---enforced-docker-registry=<%= @docker_registry %> \
---host-automounts=<%= @host_automounts_string %> \
---host-paths-allowed=<%= @host_paths_allowed_string %> \
---host-path-prefixes-allowed=<%= @host_path_prefixes_allowed_string %>"
+<%= @admission_control_params %>
diff --git a/modules/profile/manifests/kubernetes/master.pp
b/modules/profile/manifests/kubernetes/master.pp
index cd0c312..421cc1c 100644
--- a/modules/profile/manifests/kubernetes/master.pp
+++ b/modules/profile/manifests/kubernetes/master.pp
@@ -3,7 +3,6 @@
# List of hosts this is accessible to.
# SPECIAL VALUE: use 'all' to have this port be open to the world
$accessible_to=hiera('profile::kubernetes::master::accessible_to'),
- $docker_registry=hiera('profile::kubernetes::master::docker_registry'),
$service_cluster_ip_range=hiera('profile::kubernetes::master::service_cluster_ip_range'),
$apiserver_count=hiera('profile::kubernetes::master::apiserver_count'),
$admission_controllers=hiera('profile::kubernetes::master::admission_controllers'),
@@ -12,8 +11,6 @@
$ssl_cert_path=hiera('profile::kubernetes::master::ssl_cert_path'),
$ssl_key_path=hiera('profile::kubernetes::master::ssl_cert_path'),
$authz_mode=hiera('profile::kubernetes::master::authz_mode'),
- $host_automounts=hiera('profile::kubernetes::master::host_automounts'),
-
$host_path_prefixes_allowed=hiera('profile::kubernetes::master::host_path_prefixes_allowed'),
){
if $expose_puppet_certs {
base::expose_puppet_certs { '/etc/kubernetes':
@@ -34,16 +31,13 @@
$etcd_servers = join($etcd_urls, ',')
class { '::k8s::apiserver':
- etcd_servers => $etcd_servers,
- docker_registry => $docker_registry,
- ssl_cert_path => $ssl_cert_path,
- ssl_key_path => $ssl_key_path,
- authz_mode => $authz_mode,
- service_cluster_ip_range => $service_cluster_ip_range,
- apiserver_count => $apiserver_count,
- admission_controllers => $admission_controllers,
- host_path_prefixes_allowed => $host_path_prefixes_allowed,
- host_automounts => $host_automounts,
+ etcd_servers => $etcd_servers,
+ ssl_cert_path => $ssl_cert_path,
+ ssl_key_path => $ssl_key_path,
+ authz_mode => $authz_mode,
+ service_cluster_ip_range => $service_cluster_ip_range,
+ apiserver_count => $apiserver_count,
+ admission_controllers => $admission_controllers,
}
class { '::k8s::scheduler': }
diff --git a/modules/role/manifests/toollabs/k8s/master.pp
b/modules/role/manifests/toollabs/k8s/master.pp
index 4fbd7d7..2514ef2 100644
--- a/modules/role/manifests/toollabs/k8s/master.pp
+++ b/modules/role/manifests/toollabs/k8s/master.pp
@@ -22,36 +22,41 @@
$ssl_key_path = "/etc/ssl/private/${ssl_certificate_name}.key"
}
+ # Set our host allowed paths
+ $host_automounts = [
+ '/etc/ldap.conf',
+ '/etc/ldap.yaml',
+ '/etc/novaobserver.yaml',
+ '/var/run/nslcd/socket',
+ ]
+ $host_path_prefixes_allowed = [
+ '/data/project/',
+ '/public/dumps/',
+ '/data/scratch/',
+ ]
+ $host_automounts_string = join($host_automounts, ',')
+ $host_path_prefixes_allowed_string = join($host_path_prefixes_allowed, ',')
+
+ $docker_registry = hiera('docker::registry')
+
class { '::profile::kubernetes::master':
- etcd_urls => $etcd_url,
- service_cluster_ip_range => '192.168.0.0/17',
- apiserver_count => 1,
- accessible_to => 'all',
- expose_puppet_certs => $use_puppet_certs,
- ssl_cert_path => $ssl_cert_path,
- ssl_key_path => $ssl_key_path,
- host_path_prefixes_allowed => [
- '/data/project/',
- '/public/dumps/',
- '/data/scratch/',
- ],
- docker_registry => hiera('docker::registry'),
- host_automounts => [
- '/etc/ldap.conf',
- '/etc/ldap.yaml',
- '/etc/novaobserver.yaml',
- '/var/run/nslcd/socket',
- ],
- authz_mode => 'abac',
- admission_controllers => [
- 'NamespaceLifecycle',
- 'ResourceQuota',
- 'LimitRanger',
- 'UidEnforcer',
- 'RegistryEnforcer',
- 'HostAutomounter',
- 'HostPathEnforcer',
- ],
+ etcd_urls => $etcd_url,
+ service_cluster_ip_range => '192.168.0.0/17',
+ apiserver_count => 1,
+ accessible_to => 'all',
+ expose_puppet_certs => $use_puppet_certs,
+ ssl_cert_path => $ssl_cert_path,
+ ssl_key_path => $ssl_key_path,
+ authz_mode => 'abac',
+ admission_controllers => {
+ 'NamespaceLifecycle' => '',
+ 'ResourceQuota' => '',
+ 'LimitRanger' => '',
+ 'UidEnforcer' => '',
+ 'RegistryEnforcer' =>
"--enforced-docker-registry=${docker_registry}",
+ 'HostAutomounter' =>
"--host-automounts=${host_automounts_string}",
+ 'HostPathEnforcer' =>
"--host-path-prefixes-allowed=${host_path_prefixes_allowed_string}",
+ },
}
class { '::toollabs::maintain_kubeusers':
--
To view, visit https://gerrit.wikimedia.org/r/374795
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ie960a72da62b12beca2408e1b5028ea50ef3d5d5
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Alexandros Kosiaris <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
