Alexandros Kosiaris has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/374795 )
Change subject: kubernetes: Refactor/add admission controllers
......................................................................
kubernetes: Refactor/add admission controllers
Migrate away from the disjoint structure of an array for the admission
controllers and separate parameters for their configuration to a more
tightly coupled structure where admission controllers are specified as a
Hash with keys the admission controller names and values their
respective parameters. That allows for greater flexibility and decouples
the definition of the class from the implementation details of the
controllers and their parameters.
Amend the roles for production, staging as well as labs to keep the
status quo.
There is one glaring difference in production/staging, that
is we no longer enforce the registry. That is expected to be enforced
later on if possible via the dynamic admission controllers, but even now
it is not an immediate issue due to the networking policies we have in
place.
Bug: T170119
Change-Id: Ie960a72da62b12beca2408e1b5028ea50ef3d5d5
---
M hieradata/role/common/kubernetes/master.yaml
M hieradata/role/common/kubernetes/staging/master.yaml
M modules/k8s/manifests/apiserver.pp
M modules/k8s/templates/kube-apiserver.default.erb
M modules/profile/manifests/kubernetes/master.pp
M modules/role/manifests/toollabs/k8s/master.pp
6 files changed, 63 insertions(+), 79 deletions(-)
Approvals:
Alexandros Kosiaris: Verified; Looks good to me, approved
diff --git a/hieradata/role/common/kubernetes/master.yaml
b/hieradata/role/common/kubernetes/master.yaml
index 8891dc8..d4b9233 100644
--- a/hieradata/role/common/kubernetes/master.yaml
+++ b/hieradata/role/common/kubernetes/master.yaml
@@ -2,22 +2,18 @@
debdeploy-kubernetes-eqiad:
value: standard
cluster: kubernetes
-profile::kubernetes::master::docker_registry: darmstadtium.eqiad.wmnet
profile::kubernetes::master::accessible_to: all
profile::kubernetes::master::apiserver_count: 2
profile::kubernetes::master::admission_controllers:
- - NamespaceLifecycle
- - ResourceQuota
- - LimitRanger
- - RegistryEnforcer
- - DefaultStorageClass
+ NamespaceLifecycle: ''
+ LimitRanger: ''
+ DefaultStorageClass: ''
+ ResourceQuota: ''
profile::kubernetes::master::expose_puppet_certs: true
profile::kubernetes::master::service_cert: "kubemaster.svc.%{::site}.wmnet"
profile::kubernetes::master::ssl_cert_path:
"/etc/ssl/localcerts/kubemaster.svc.%{::site}.wmnet.crt"
profile::kubernetes::master::ssl_key_path:
"/etc/ssl/private/kubemaster.svc.%{::site}.wmnet.key"
profile::kubernetes::master::authz_mode: ''
-profile::kubernetes::master::host_automounts: []
-profile::kubernetes::master::host_path_prefixes_allowed: []
# TODO: This needs to become a profile
role::lvs::realserver::pools:
kubemaster: {}
diff --git a/hieradata/role/common/kubernetes/staging/master.yaml
b/hieradata/role/common/kubernetes/staging/master.yaml
index 8161129..3d4cab6 100644
--- a/hieradata/role/common/kubernetes/staging/master.yaml
+++ b/hieradata/role/common/kubernetes/staging/master.yaml
@@ -2,23 +2,19 @@
debdeploy-kubernetes-eqiad:
value: standard
cluster: kubernetes
-profile::kubernetes::master::docker_registry: darmstadtium.eqiad.wmnet
profile::kubernetes::master::accessible_to:
- kubestage1001.eqiad.wmnet
- kubestage1002.eqiad.wmnet
profile::kubernetes::master::apiserver_count: 1
profile::kubernetes::master::admission_controllers:
- - NamespaceLifecycle
- - ResourceQuota
- - LimitRanger
- - RegistryEnforcer
- - DefaultStorageClass
+ NamespaceLifecycle: ''
+ LimitRanger: ''
+ DefaultStorageClass: ''
+ ResourceQuota: ''
profile::kubernetes::master::expose_puppet_certs: true
profile::kubernetes::master::ssl_cert_path: "/etc/kubernetes/ssl/cert.pem"
profile::kubernetes::master::ssl_key_path: "/etc/kubernetes/ssl/server.key"
profile::kubernetes::master::authz_mode: ''
-profile::kubernetes::master::host_automounts: []
-profile::kubernetes::master::host_path_prefixes_allowed: []
profile::kubernetes::master::service_cluster_ip_range: 10.64.76.0/24
profile::kubernetes::master::etcd_urls:
- https://kubestagetcd1001.eqiad.wmnet:2379
diff --git a/modules/k8s/manifests/apiserver.pp
b/modules/k8s/manifests/apiserver.pp
index 9f401ad..4307259 100644
--- a/modules/k8s/manifests/apiserver.pp
+++ b/modules/k8s/manifests/apiserver.pp
@@ -1,23 +1,17 @@
class k8s::apiserver(
$etcd_servers,
- $docker_registry,
$ssl_cert_path=undef,
$ssl_key_path=undef,
$kube_api_port = undef,
$kubelet_port = undef,
$service_cluster_ip_range = '192.168.0.0/17',
- $admission_controllers = [
- 'NamespaceLifecycle',
- 'ResourceQuota',
- 'LimitRanger',
- 'UidEnforcer',
- 'RegistryEnforcer',
- 'HostAutomounter',
- 'HostPathEnforcer',
- ],
- $host_automounts = [],
- $host_paths_allowed = [],
- $host_path_prefixes_allowed = [],
+ $admission_controllers = {
+ 'NamespaceLifecycle' => '',
+ 'LimitRanger' => '',
+ 'ServiceAccount' => '',
+ 'DefaultStorageClass' => '',
+ 'ResourceQuota' => '',
+ },
$authz_mode = 'abac',
$apiserver_count = undef,
) {
@@ -31,10 +25,8 @@
require_package('kubernetes-master')
require_package('kubernetes-client')
- $host_automounts_string = join($host_automounts, ',')
- $host_paths_allowed_string = join(concat($host_paths_allowed,
$host_automounts), ',')
- $host_path_prefixes_allowed_string = join($host_path_prefixes_allowed, ',')
- $admission_control = join($admission_controllers, ',')
+ $admission_control = join(keys($admission_controllers), ',')
+ $admission_control_params = lstrip(join(values($admission_controllers), '
'))
$users = hiera('k8s_infrastructure_users')
file { '/etc/kubernetes/infrastructure-users':
diff --git a/modules/k8s/templates/kube-apiserver.default.erb
b/modules/k8s/templates/kube-apiserver.default.erb
index 2ebf8ec..2825e73 100644
--- a/modules/k8s/templates/kube-apiserver.default.erb
+++ b/modules/k8s/templates/kube-apiserver.default.erb
@@ -40,7 +40,7 @@
--runtime-config=batch/v2alpha1 \
--tls-cert-file=<%= @ssl_cert_path %> \
--tls-private-key-file=<%= @ssl_key_path %> \
---enforced-docker-registry=<%= @docker_registry %> \
---host-automounts=<%= @host_automounts_string %> \
---host-paths-allowed=<%= @host_paths_allowed_string %> \
---host-path-prefixes-allowed=<%= @host_path_prefixes_allowed_string %>"
+<%- if not @admission_control_params.empty? -%>
+<%= @admission_control_params %> \
+<%- end -%>
+"
diff --git a/modules/profile/manifests/kubernetes/master.pp
b/modules/profile/manifests/kubernetes/master.pp
index cd0c312..421cc1c 100644
--- a/modules/profile/manifests/kubernetes/master.pp
+++ b/modules/profile/manifests/kubernetes/master.pp
@@ -3,7 +3,6 @@
# List of hosts this is accessible to.
# SPECIAL VALUE: use 'all' to have this port be open to the world
$accessible_to=hiera('profile::kubernetes::master::accessible_to'),
- $docker_registry=hiera('profile::kubernetes::master::docker_registry'),
$service_cluster_ip_range=hiera('profile::kubernetes::master::service_cluster_ip_range'),
$apiserver_count=hiera('profile::kubernetes::master::apiserver_count'),
$admission_controllers=hiera('profile::kubernetes::master::admission_controllers'),
@@ -12,8 +11,6 @@
$ssl_cert_path=hiera('profile::kubernetes::master::ssl_cert_path'),
$ssl_key_path=hiera('profile::kubernetes::master::ssl_cert_path'),
$authz_mode=hiera('profile::kubernetes::master::authz_mode'),
- $host_automounts=hiera('profile::kubernetes::master::host_automounts'),
-
$host_path_prefixes_allowed=hiera('profile::kubernetes::master::host_path_prefixes_allowed'),
){
if $expose_puppet_certs {
base::expose_puppet_certs { '/etc/kubernetes':
@@ -34,16 +31,13 @@
$etcd_servers = join($etcd_urls, ',')
class { '::k8s::apiserver':
- etcd_servers => $etcd_servers,
- docker_registry => $docker_registry,
- ssl_cert_path => $ssl_cert_path,
- ssl_key_path => $ssl_key_path,
- authz_mode => $authz_mode,
- service_cluster_ip_range => $service_cluster_ip_range,
- apiserver_count => $apiserver_count,
- admission_controllers => $admission_controllers,
- host_path_prefixes_allowed => $host_path_prefixes_allowed,
- host_automounts => $host_automounts,
+ etcd_servers => $etcd_servers,
+ ssl_cert_path => $ssl_cert_path,
+ ssl_key_path => $ssl_key_path,
+ authz_mode => $authz_mode,
+ service_cluster_ip_range => $service_cluster_ip_range,
+ apiserver_count => $apiserver_count,
+ admission_controllers => $admission_controllers,
}
class { '::k8s::scheduler': }
diff --git a/modules/role/manifests/toollabs/k8s/master.pp
b/modules/role/manifests/toollabs/k8s/master.pp
index 4fbd7d7..81647b4 100644
--- a/modules/role/manifests/toollabs/k8s/master.pp
+++ b/modules/role/manifests/toollabs/k8s/master.pp
@@ -22,36 +22,42 @@
$ssl_key_path = "/etc/ssl/private/${ssl_certificate_name}.key"
}
+ # Set our host allowed paths
+ $host_automounts = [
+ '/etc/ldap.conf',
+ '/etc/ldap.yaml',
+ '/etc/novaobserver.yaml',
+ '/var/run/nslcd/socket',
+ ]
+ $host_path_prefixes_allowed = [
+ '/data/project/',
+ '/public/dumps/',
+ '/data/scratch/',
+ ]
+ $host_automounts_string = join($host_automounts, ',')
+ $host_path_prefixes_allowed_string = join($host_path_prefixes_allowed, ',')
+
+
+ $docker_registry = hiera('docker::registry')
+
class { '::profile::kubernetes::master':
- etcd_urls => $etcd_url,
- service_cluster_ip_range => '192.168.0.0/17',
- apiserver_count => 1,
- accessible_to => 'all',
- expose_puppet_certs => $use_puppet_certs,
- ssl_cert_path => $ssl_cert_path,
- ssl_key_path => $ssl_key_path,
- host_path_prefixes_allowed => [
- '/data/project/',
- '/public/dumps/',
- '/data/scratch/',
- ],
- docker_registry => hiera('docker::registry'),
- host_automounts => [
- '/etc/ldap.conf',
- '/etc/ldap.yaml',
- '/etc/novaobserver.yaml',
- '/var/run/nslcd/socket',
- ],
- authz_mode => 'abac',
- admission_controllers => [
- 'NamespaceLifecycle',
- 'ResourceQuota',
- 'LimitRanger',
- 'UidEnforcer',
- 'RegistryEnforcer',
- 'HostAutomounter',
- 'HostPathEnforcer',
- ],
+ etcd_urls => $etcd_url,
+ service_cluster_ip_range => '192.168.0.0/17',
+ apiserver_count => 1,
+ accessible_to => 'all',
+ expose_puppet_certs => $use_puppet_certs,
+ ssl_cert_path => $ssl_cert_path,
+ ssl_key_path => $ssl_key_path,
+ authz_mode => 'abac',
+ admission_controllers => {
+ 'NamespaceLifecycle' => '',
+ 'ResourceQuota' => '',
+ 'LimitRanger' => '',
+ 'UidEnforcer' => '',
+ 'RegistryEnforcer' =>
"--enforced-docker-registry=${docker_registry}",
+ 'HostAutomounter' =>
"--host-automounts=${host_automounts_string}",
+ 'HostPathEnforcer' =>
"--host-paths-allowed=${host_automounts_string}
--host-path-prefixes-allowed=${host_path_prefixes_allowed_string}",
+ },
}
class { '::toollabs::maintain_kubeusers':
--
To view, visit https://gerrit.wikimedia.org/r/374795
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ie960a72da62b12beca2408e1b5028ea50ef3d5d5
Gerrit-PatchSet: 8
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Alexandros Kosiaris <[email protected]>
Gerrit-Reviewer: Alexandros Kosiaris <[email protected]>
Gerrit-Reviewer: Gehel <[email protected]>
Gerrit-Reviewer: Giuseppe Lavagetto <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
