Hello community, here is the log from the commit of package kubernetes-salt for openSUSE:Factory checked in at 2018-04-27 16:10:18 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/kubernetes-salt (Old) and /work/SRC/openSUSE:Factory/.kubernetes-salt.new (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "kubernetes-salt" Fri Apr 27 16:10:18 2018 rev:15 rq:601903 version:3.0.0+git_r742_8508870 Changes: -------- --- /work/SRC/openSUSE:Factory/kubernetes-salt/kubernetes-salt.changes 2018-04-24 15:34:13.207854421 +0200 +++ /work/SRC/openSUSE:Factory/.kubernetes-salt.new/kubernetes-salt.changes 2018-04-27 16:10:20.752038484 +0200 @@ -1,0 +2,84 @@ +Thu Apr 26 09:56:06 UTC 2018 - [email protected] + +- Commit c3b81a6 by Flavio Castelli [email protected] + Ensure swap is disabled before kubelet is started + + We have to ensure the swap state is executed before the kubelet service is + started, otherwise kubelt won't run and this will lead to issues like the + ones causing bsc#1090337 + + Signed-off-by: Flavio Castelli <[email protected]> + + +------------------------------------------------------------------- +Wed Apr 25 12:10:02 UTC 2018 - [email protected] + +- Commit 24bea3d by Nirmoy Das [email protected] + cni: add cilium as alternate to flannel plugin + + +------------------------------------------------------------------- +Tue Apr 24 15:58:27 UTC 2018 - [email protected] + +- Commit 1fd2a98 by Alvaro Saurin [email protected] + Remove leftover file + + feature#node_removal + + +------------------------------------------------------------------- +Tue Apr 24 09:21:14 UTC 2018 - [email protected] + +- Commit e1b9c75 by Kiall Mac Innes [email protected] + Update tiller tag to 2.8.2 + + This matches the tag used in the updated image via SR#162727. + + +------------------------------------------------------------------- +Tue Apr 24 08:42:18 UTC 2018 - [email protected] + +- Commit 3e70e4f by Alvaro Saurin [email protected] + Use get_with_expr() + + feature#node_removal + + Commit b4d09dd by Alvaro Saurin [email protected] + Convert integers in the pillar to real integers. Unit tests for the + get_pillar() function. + + See https://trello.com/c/O7daOErL + + feature#node_removal + + Commit 0d65d79 by Alvaro Saurin [email protected] + Fix: do not include the current node in the list + of endpoints when adding a new member. Unit tests for the etcd modoule. + + See https://trello.com/c/O7daOErL + + feature#node_removal + + Commit 399f7ea by Alvaro Saurin [email protected] + Try to resist unresponsive nodes when removing a node. + * the replacement will not be chosen from + the unresponsive nodes + * affected nodes will exclude them too. Possibility to skip any action on + the target (with the `skip` pillar), so we can remove unresponsive targets + while still looking for replacements. + + See https://trello.com/c/O7daOErL + + feature#node_removal + + +------------------------------------------------------------------- +Tue Apr 24 07:58:31 UTC 2018 - [email protected] + +- Commit f80f752 by Alvaro Saurin [email protected] + Don't to remove some things that are not so important. + + feature#node_removal + + +------------------------------------------------------------------- ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ kubernetes-salt.spec ++++++ --- /var/tmp/diff_new_pack.UxiAiM/_old 2018-04-27 16:10:21.396014867 +0200 +++ /var/tmp/diff_new_pack.UxiAiM/_new 2018-04-27 16:10:21.400014720 +0200 @@ -32,7 +32,7 @@ Name: kubernetes-salt %define gitrepo salt -Version: 3.0.0+git_r727_68280fa +Version: 3.0.0+git_r742_8508870 Release: 0 BuildArch: noarch Summary: Production-Grade Container Scheduling and Management ++++++ master.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/pillar/certificates.sls new/salt-master/pillar/certificates.sls --- old/salt-master/pillar/certificates.sls 2018-04-23 14:03:55.000000000 +0200 +++ new/salt-master/pillar/certificates.sls 2018-04-26 11:56:10.000000000 +0200 @@ -50,3 +50,6 @@ kube_proxy_key: '/etc/pki/kube-proxy.key' kube_proxy_crt: '/etc/pki/kube-proxy.crt' + + cilium_key: '/etc/pki/cilium.key' + cilium_crt: '/etc/pki/cilium.crt' diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/_modules/caasp_etcd.py new/salt-master/salt/_modules/caasp_etcd.py --- old/salt-master/salt/_modules/caasp_etcd.py 2018-04-23 14:03:55.000000000 +0200 +++ new/salt-master/salt/_modules/caasp_etcd.py 2018-04-26 11:56:10.000000000 +0200 @@ -173,6 +173,7 @@ error('no etcd members available!!') raise NoEtcdServersException() + etcd_members_lst.sort() return sep.join(etcd_members_lst) @@ -221,7 +222,7 @@ return member_line.split(':')[0] except Exception as e: - error("cannot get member ID: %s", e) - error("output: %s", members_output) + error('cannot get member ID for "%s": %s', e, this_nodename) + error('output: %s', members_output) return '' diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/_modules/caasp_nodes.py new/salt-master/salt/_modules/caasp_nodes.py --- old/salt-master/salt/_modules/caasp_nodes.py 2018-04-23 14:03:55.000000000 +0200 +++ new/salt-master/salt/_modules/caasp_nodes.py 2018-04-26 11:56:10.000000000 +0200 @@ -100,9 +100,10 @@ } -# filter out empty/None and sort a list +# for a list `lst`, filter out empty/None, remove duplicates and sort it def _sanitize_list(lst): res = [x for x in lst if x] + res = list(set(res)) res.sort() return res @@ -240,6 +241,9 @@ elif replacement_provided and replacement in forbidden: abort('%s cannot be replaced by %s: the replacement has a "ca" or "admin" role', target, replacement) + elif replacement_provided and replacement in excluded: + abort('%s cannot be replaced by %s: the replacement is in the list of nodes excluded', + target, replacement) masters = get_from_args_or_with_expr( 'masters', kwargs, 'G@roles:kube-master') diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/_modules/caasp_pillar.py new/salt-master/salt/_modules/caasp_pillar.py --- old/salt-master/salt/_modules/caasp_pillar.py 2018-04-23 14:03:55.000000000 +0200 +++ new/salt-master/salt/_modules/caasp_pillar.py 2018-04-26 11:56:10.000000000 +0200 @@ -20,9 +20,16 @@ res = __salt__['pillar.get'](name, None) if res is None: res = default + if isinstance(res, basestring): + try: + return int(res) + except ValueError: + pass + if res.lower() in ["true", "yes", "on"]: return True elif res.lower() in ["false", "no", "off"]: return False + return res diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/_modules/tests/test_caasp_etcd.py new/salt-master/salt/_modules/tests/test_caasp_etcd.py --- old/salt-master/salt/_modules/tests/test_caasp_etcd.py 1970-01-01 01:00:00.000000000 +0100 +++ new/salt-master/salt/_modules/tests/test_caasp_etcd.py 2018-04-26 11:56:10.000000000 +0200 @@ -0,0 +1,55 @@ +from __future__ import absolute_import + +import unittest + +import caasp_etcd +from caasp_etcd import ETCD_CLIENT_PORT, get_endpoints +from caasp_log import ExecutionAborted + +try: + from mock import patch, MagicMock +except ImportError: + _mocking_lib_available = False +else: + _mocking_lib_available = True + + +caasp_etcd.__salt__ = {} + + +class TestGetEndpoints(unittest.TestCase): + ''' + Some basic tests for get_from_args_or_with_expr() + ''' + + def test_get_endpoints(self): + nodes = { + 'AAA': 'node1', + 'BBB': 'node2', + 'CCC': 'node3' + } + + mock = MagicMock(return_value=nodes) + with patch.dict(caasp_etcd.__salt__, {'caasp_grains.get': mock}): + res = get_endpoints() + mock.assert_called_once_with('G@roles:etcd') + + for i in nodes.values(): + self.assertIn('https://{}:{}'.format(i, ETCD_CLIENT_PORT), res, + 'did not get the expected list of etcd endpoints: {}'.format(res)) + + mock.reset_mock() + + res = get_endpoints(with_id=True) + mock.assert_called_once_with('G@roles:etcd') + + for (j, k) in nodes.items(): + self.assertIn('{}=https://{}:{}'.format(j, k, ETCD_CLIENT_PORT), res, + 'did not get the expected list of etcd endpoints: {}'.format(res)) + + mock.reset_mock() + + res = get_endpoints(skip_removed=True) + mock.assert_called_once_with('G@roles:etcd and not G@removal_in_progress:true') + + mock.reset_mock() diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/_modules/tests/test_caasp_nodes.py new/salt-master/salt/_modules/tests/test_caasp_nodes.py --- old/salt-master/salt/_modules/tests/test_caasp_nodes.py 2018-04-23 14:03:55.000000000 +0200 +++ new/salt-master/salt/_modules/tests/test_caasp_nodes.py 2018-04-26 11:56:10.000000000 +0200 @@ -212,6 +212,13 @@ self.assertIn('kube-minion', roles, 'kube-minion role not found in replacement') + # check we cannot use an excluded node + with self.assertRaises(ExecutionAborted): + replacement, roles = get_replacement_for(self.minion_1, + replacement=self.minion_3, + excluded=[self.minion_3], + **self.get_replacement_for_kwargs) + def test_invalid_etcd_replacement(self): ''' Check get_replacement_for() realizes a minion diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/_modules/tests/test_caasp_pillar.py new/salt-master/salt/_modules/tests/test_caasp_pillar.py --- old/salt-master/salt/_modules/tests/test_caasp_pillar.py 1970-01-01 01:00:00.000000000 +0100 +++ new/salt-master/salt/_modules/tests/test_caasp_pillar.py 2018-04-26 11:56:10.000000000 +0200 @@ -0,0 +1,49 @@ +from __future__ import absolute_import + +import unittest + +import caasp_pillar +from caasp_log import ExecutionAborted +from caasp_pillar import get as get_pillar + +try: + from mock import patch, MagicMock +except ImportError: + _mocking_lib_available = False +else: + _mocking_lib_available = True + + +caasp_pillar.__salt__ = {} + + +class TestGetPillar(unittest.TestCase): + ''' + Some basic tests for caasp_pillar.get() + ''' + + def test_get_pillar(self): + + mock = MagicMock() + with patch.dict(caasp_pillar.__salt__, {'pillar.get': mock}): + # check we get a integer + mock.return_value = '123' + res = get_pillar('some_int_pillar') + self.assertTrue(isinstance(res, int), + 'expected to get a integer: {}'.format(res)) + mock.reset_mock() + + # check we get a boolean + for value in ['true', 'on', 'TRUE']: + mock.return_value = value + res = get_pillar('some_bool_pillar') + self.assertTrue(isinstance(res, bool), + 'expected to get a bool: {}'.format(res)) + mock.reset_mock() + + # check we get a string + mock.return_value = 'something' + res = get_pillar('some_str_pillar') + self.assertTrue(isinstance(res, str), + 'expected to get a string: {}'.format(res)) + mock.reset_mock() diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/_states/caasp_etcd.py new/salt-master/salt/_states/caasp_etcd.py --- old/salt-master/salt/_states/caasp_etcd.py 2018-04-23 14:03:55.000000000 +0200 +++ new/salt-master/salt/_states/caasp_etcd.py 2018-04-26 11:56:10.000000000 +0200 @@ -21,13 +21,22 @@ def etcdctl(name, retry={}, **kwargs): ''' Run an etcdctl command + + Arguments: + + In addition to all the arguments supported by the `caasp_cmd.run` state. + + * `skip_this`: (optional) skip current node when calculating the list of etcd endpoints. + ''' retry_ = {'attempts': DEFAULT_ATTEMPTS, 'interval': DEFAULT_ATTEMPTS_INTERVAL, 'until': None} retry_.update(retry) - args = __salt__['caasp_etcd.get_etcdctl_args_str']() + skip_this = kwargs.pop('skip_this', False) + + args = __salt__['caasp_etcd.get_etcdctl_args_str'](skip_this=skip_this) cmd = 'etcdctl {} {}'.format(args, name) log.debug('CaaS: running etcdctl as: %s', cmd) @@ -53,7 +62,7 @@ name = 'member add {} {}'.format(this_id, this_peer_url) log.debug('CaaS: adding etcd member') - return etcdctl(name=name, **kwargs) + return etcdctl(name=name, skip_this=True, **kwargs) # once the member has been added to the cluster, we # must make sure etcd joins an "existing" cluster. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/addons/remove-pre-reboot.sls new/salt-master/salt/addons/remove-pre-reboot.sls --- old/salt-master/salt/addons/remove-pre-reboot.sls 2018-04-23 14:03:55.000000000 +0200 +++ new/salt-master/salt/addons/remove-pre-reboot.sls 1970-01-01 01:00:00.000000000 +0100 @@ -1,3 +0,0 @@ - -/etc/kubernetes/addons/namespace.yaml: - file.absent diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/addons/tiller/manifests/20-deployment.yaml new/salt-master/salt/addons/tiller/manifests/20-deployment.yaml --- old/salt-master/salt/addons/tiller/manifests/20-deployment.yaml 2018-04-23 14:03:55.000000000 +0200 +++ new/salt-master/salt/addons/tiller/manifests/20-deployment.yaml 2018-04-26 11:56:10.000000000 +0200 @@ -34,7 +34,7 @@ - env: - name: TILLER_NAMESPACE value: kube-system - image: sles12/tiller:2.7.2 + image: sles12/tiller:2.8.2 imagePullPolicy: IfNotPresent livenessProbe: httpGet: diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/cert/remove-pre-reboot.sls new/salt-master/salt/cert/remove-pre-reboot.sls --- old/salt-master/salt/cert/remove-pre-reboot.sls 1970-01-01 01:00:00.000000000 +0100 +++ new/salt-master/salt/cert/remove-pre-reboot.sls 2018-04-26 11:56:10.000000000 +0200 @@ -0,0 +1,6 @@ + +{{ pillar['ssl']['crt_file'] }}: + file.absent + +{{ pillar['ssl']['key_file'] }}: + file.absent diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/cleanup/remove-post-orchestration.sls new/salt-master/salt/cleanup/remove-post-orchestration.sls --- old/salt-master/salt/cleanup/remove-post-orchestration.sls 2018-04-23 14:03:55.000000000 +0200 +++ new/salt-master/salt/cleanup/remove-post-orchestration.sls 2018-04-26 11:56:10.000000000 +0200 @@ -10,7 +10,7 @@ # k8s cluster ############### -{%- set k8s_nodes = salt['mine.get']('roles:(kube-master|kube-minion)', 'nodename', expr_form='grain_pcre').keys() %} +{%- set k8s_nodes = salt.caasp_nodes.get_with_expr('G@roles:kube-master', booted=True) %} {%- if forced or target in k8s_nodes %} {%- from '_macros/kubectl.jinja' import kubectl with context %} @@ -24,7 +24,7 @@ # etcd node ############### -{%- set etcd_members = salt['mine.get']('roles:etcd', 'nodename', expr_form='grain').keys() %} +{%- set etcd_members = salt.caasp_nodes.get_with_expr('G@roles:etcd', booted=True) %} {%- if forced or target in etcd_members %} etcd-remove-member: diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/cleanup/remove-pre-reboot.sls new/salt-master/salt/cleanup/remove-pre-reboot.sls --- old/salt-master/salt/cleanup/remove-pre-reboot.sls 2018-04-23 14:03:55.000000000 +0200 +++ new/salt-master/salt/cleanup/remove-pre-reboot.sls 2018-04-26 11:56:10.000000000 +0200 @@ -2,24 +2,10 @@ # try to remove some dirs that could contain sensitive # information, even when they were not directly managed by us -wipe-etc-kubernetes: - cmd.run: - - name: rm -rf /etc/kubernetes/* - wipe-certificates: cmd.run: - name: rm -rf /var/lib/ca-certificates/* -# remove some logs that could contain sensitive information -wipe-var-log: - cmd.run: - - name: |- - for f in apparmor audit containers faillog firewall localmessages pods zypper.log YaST2 ; do - rm -rf /var/log/$f - done - # NOTE: do not try to remove /var/log/salt - # or the Salt minion will crash... - # cleanup all the Salt things we can # NOTE: we must be careful (or Salt will stop working) cleanup-salt: diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/cni/cilium/init.sls new/salt-master/salt/cni/cilium/init.sls --- old/salt-master/salt/cni/cilium/init.sls 1970-01-01 01:00:00.000000000 +0100 +++ new/salt-master/salt/cni/cilium/init.sls 2018-04-26 11:56:10.000000000 +0200 @@ -0,0 +1,21 @@ +{% set plugin = salt['pillar.get']('cni:plugin', 'cilium').lower() %} +{% if plugin == "cilium" %} + +include: + - ca-cert + - cert + - crypto + +{% from '_macros/certs.jinja' import certs with context %} +{{ certs("cilium", + pillar['ssl']['cilium_crt'], + pillar['ssl']['cilium_key'], + cn = grains['nodename'], + o = 'system:nodes') }} + +{% else %} +{# See https://github.com/saltstack/salt/issues/14553 #} +dummy_step: + cmd.run: + - name: "echo saltstack bug 14553" +{% endif %} diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/cni/cilium-config.yaml.jinja new/salt-master/salt/cni/cilium-config.yaml.jinja --- old/salt-master/salt/cni/cilium-config.yaml.jinja 1970-01-01 01:00:00.000000000 +0100 +++ new/salt-master/salt/cni/cilium-config.yaml.jinja 2018-04-26 11:56:10.000000000 +0200 @@ -0,0 +1,30 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: cilium-config + namespace: kube-system +data: + # This etcd-config contains the etcd endpoints of your cluster. If you use + # TLS please make sure you uncomment the ca-file line and add the respective + # certificate has a k8s secret, see explanation below in the comment labeled + # "ETCD-CERT" + etcd-config: |- + --- + endpoints: +{%- for endpoints in salt.caasp_etcd.get_endpoints().split(',') %} + - {{ endpoints }} +{%- endfor %} + # + # In case you want to use TLS in etcd, uncomment the following line + # and add the certificate as explained in the comment labeled "ETCD-CERT" + ca-file: '/etc/pki/trust/anchors/SUSE_CaaSP_CA.crt' + # + # In case you want client to server authentication, uncomment the following + # lines and add the certificate and key in cilium-etcd-secrets below + key-file: '{{ cilium_key }}' + cert-file: '{{ cilium_certificate }}' + + # If you want to run cilium in debug mode change this value to true + debug: "false" + disable-ipv4: "false" + diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/cni/cilium-ds.yaml.jinja new/salt-master/salt/cni/cilium-ds.yaml.jinja --- old/salt-master/salt/cni/cilium-ds.yaml.jinja 1970-01-01 01:00:00.000000000 +0100 +++ new/salt-master/salt/cni/cilium-ds.yaml.jinja 2018-04-26 11:56:10.000000000 +0200 @@ -0,0 +1,171 @@ +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: cilium + namespace: kube-system +spec: + updateStrategy: + type: "RollingUpdate" + rollingUpdate: + # Specifies the maximum number of Pods that can be unavailable during the update process. + # The current default value is 1 or 100% for daemonsets; Adding an explicit value here + # to avoid confusion, as the default value is specific to the type (daemonset/deployment). + maxUnavailable: "100%" + selector: + matchLabels: + k8s-app: cilium + kubernetes.io/cluster-service: "true" + template: + metadata: + labels: + k8s-app: cilium + kubernetes.io/cluster-service: "true" + annotations: + # This annotation plus the CriticalAddonsOnly toleration makes + # cilium to be a critical pod in the cluster, which ensures cilium + # gets priority scheduling. + # https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/ + scheduler.alpha.kubernetes.io/critical-pod: '' + scheduler.alpha.kubernetes.io/tolerations: >- + [{"key":"dedicated","operator":"Equal","value":"master","effect":"NoSchedule"}] + spec: + serviceAccountName: cilium + initContainers: + - name: install-cni-conf + image: {{ pillar['cilium']['image'] }} + command: + - /bin/sh + - "-c" + - "cp -f /etc/cni/net.d/10-cilium-cni.conf /host/etc/cni/net.d/10-cilium-cni.conf" + volumeMounts: + - name: host-cni-conf + mountPath: /host/etc/cni/net.d + - name: install-cni-bin + image: {{ pillar['cilium']['image'] }} + command: + - /bin/sh + - "-c" + - "cp -f /usr/lib/cni/* /host/opt/cni/bin/" + volumeMounts: + - name: host-cni-bin + mountPath: /host/opt/cni/bin/ + + containers: + - image: {{ pillar['cilium']['image'] }} + imagePullPolicy: IfNotPresent + name: cilium-agent + command: [ "cilium-agent" ] + args: + - "--debug=$(CILIUM_DEBUG)" + - "--disable-envoy-version-check" + - "-t=vxlan" + - "--kvstore=etcd" + - "--kvstore-opt=etcd.config=/var/lib/etcd-config/etcd.config" + - "--disable-ipv4=$(DISABLE_IPV4)" + ports: + - name: prometheus + containerPort: 9090 + lifecycle: + preStop: + exec: + command: + - "rm -f /host/etc/cni/net.d/10-cilium-cni.conf /host/opt/cni/bin/cilium-cni" + env: + - name: "K8S_NODE_NAME" + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: "CILIUM_DEBUG" + valueFrom: + configMapKeyRef: + name: cilium-config + key: debug + - name: "DISABLE_IPV4" + valueFrom: + configMapKeyRef: + name: cilium-config + key: disable-ipv4 + livenessProbe: + exec: + command: + - cilium + - status + # The initial delay for the liveness probe is intentionally large to + # avoid an endless kill & restart cycle if in the event that the initial + # bootstrapping takes longer than expected. + initialDelaySeconds: 120 + failureThreshold: 10 + periodSeconds: 10 + readinessProbe: + exec: + command: + - cilium + - status + initialDelaySeconds: 5 + periodSeconds: 5 + volumeMounts: + - name: bpf-maps + mountPath: /sys/fs/bpf + - name: cilium-run + mountPath: /var/run/cilium + - name: host-cni-bin + mountPath: /host/opt/cni/bin/ + - name: host-cni-conf + mountPath: /host/etc/cni/net.d + - name: docker-socket + mountPath: /var/run/docker.sock + readOnly: true + - name: etcd-config-path + mountPath: /var/lib/etcd-config + readOnly: true + - name: etcd-certs + mountPath: /etc/pki + readOnly: true + securityContext: + capabilities: + add: + - "NET_ADMIN" + privileged: true + hostNetwork: true + volumes: + # To keep state between restarts / upgrades + - name: cilium-run + hostPath: + path: /var/run/cilium + # To keep state between restarts / upgrades + - name: bpf-maps + hostPath: + path: /sys/fs/bpf + # To read docker events from the node + - name: docker-socket + hostPath: + path: /var/run/docker.sock + # To install cilium cni plugin in the host + - name: host-cni-bin + hostPath: + path: {{ pillar['cni']['dirs']['bin'] }} + # To install cilium cni configuration in the host + - name: host-cni-conf + hostPath: + path: {{ pillar['cni']['dirs']['conf'] }} + # To read the etcd config stored in config maps + - name: etcd-config-path + configMap: + name: cilium-config + items: + - key: etcd-config + path: etcd.config + - name: etcd-certs + hostPath: + path: /etc/pki + restartPolicy: Always + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + - effect: NoSchedule + key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + # Mark cilium's pod as critical for rescheduling + - key: CriticalAddonsOnly + operator: "Exists" + diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/cni/cilium-rbac.yaml.jinja new/salt-master/salt/cni/cilium-rbac.yaml.jinja --- old/salt-master/salt/cni/cilium-rbac.yaml.jinja 1970-01-01 01:00:00.000000000 +0100 +++ new/salt-master/salt/cni/cilium-rbac.yaml.jinja 2018-04-26 11:56:10.000000000 +0200 @@ -0,0 +1,98 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: cilium + namespace: kube-system +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cilium +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cilium +subjects: +- kind: ServiceAccount + name: cilium + namespace: kube-system +- kind: Group + name: system:nodes +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: suse:caasp:psp:cilium +roleRef: + kind: ClusterRole + name: suse:caasp:psp:privileged + apiGroup: rbac.authorization.k8s.io +subjects: +- kind: ServiceAccount + name: cilium + namespace: kube-system +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cilium +rules: +- apiGroups: + - "networking.k8s.io" + resources: + - networkpolicies + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - namespaces + - services + - nodes + - endpoints + - componentstatuses + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - pods + - nodes + verbs: + - get + - list + - watch + - update +- apiGroups: + - extensions + resources: + - networkpolicies #FIXME remove this when we drop support for k8s NP-beta GH-1202 + - thirdpartyresources + - ingresses + verbs: + - create + - get + - list + - watch +- apiGroups: + - "apiextensions.k8s.io" + resources: + - customresourcedefinitions + verbs: + - create + - get + - list + - watch + - update +- apiGroups: + - cilium.io + resources: + - ciliumnetworkpolicies + - ciliumendpoints + verbs: + - "*" diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/cni/init.sls new/salt-master/salt/cni/init.sls --- old/salt-master/salt/cni/init.sls 2018-04-23 14:03:55.000000000 +0200 +++ new/salt-master/salt/cni/init.sls 2018-04-26 11:56:10.000000000 +0200 @@ -3,11 +3,12 @@ - addons - kubectl-config +{% set plugin = salt['pillar.get']('cni:plugin', 'flannel').lower() %} + ####################### # flannel CNI plugin ####################### -{% set plugin = salt['pillar.get']('cni:plugin', 'flannel').lower() %} {% if plugin == "flannel" %} /etc/kubernetes/addons/kube-flannel-rbac.yaml: @@ -48,3 +49,66 @@ - file: /etc/kubernetes/addons/kube-flannel-rbac.yaml {% endif %} + +{% if plugin == "cilium" %} +/etc/kubernetes/addons/cilium-config.yaml: + file.managed: + - source: salt://cni/cilium-config.yaml.jinja + - template: jinja + - makedirs: true + - require: + - file: /etc/kubernetes/addons + - defaults: + user: 'cluster-admin' + cilium_certificate: {{ pillar['ssl']['cilium_crt'] }} + cilium_key: {{ pillar['ssl']['cilium_key'] }} + + cmd.run: + - name: | + kubectl apply --namespace kube-system -f /etc/kubernetes/addons/cilium-config.yaml + - env: + - KUBECONFIG: {{ pillar['paths']['kubeconfig'] }} + - require: + - kube-apiserver + - file: {{ pillar['paths']['kubeconfig'] }} + - watch: + - file: /etc/kubernetes/addons/cilium-config.yaml + +/etc/kubernetes/addons/cilium-rbac.yaml: + file.managed: + - source: salt://cni/cilium-rbac.yaml.jinja + - template: jinja + - makedirs: true + - require: + - file: /etc/kubernetes/addons + cmd.run: + - name: | + kubectl apply --namespace kube-system -f /etc/kubernetes/addons/cilium-rbac.yaml + - env: + - KUBECONFIG: {{ pillar['paths']['kubeconfig'] }} + - require: + - kube-apiserver + - file: {{ pillar['paths']['kubeconfig'] }} + - watch: + - file: /etc/kubernetes/addons/cilium-rbac.yaml + +/etc/kubernetes/addons/cilium-ds.yaml: + file.managed: + - source: salt://cni/cilium-ds.yaml.jinja + - template: jinja + - makedirs: true + - require: + - file: /etc/kubernetes/addons + cmd.run: + - name: | + kubectl apply --namespace kube-system -f /etc/kubernetes/addons/cilium-ds.yaml + - env: + - KUBECONFIG: {{ pillar['paths']['kubeconfig'] }} + - require: + - kube-apiserver + - file: {{ pillar['paths']['kubeconfig'] }} + - watch: + - /etc/kubernetes/addons/cilium-config.yaml + - file: /etc/kubernetes/addons/cilium-config.yaml + +{% endif %} diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/cni/remove-pre-reboot.sls new/salt-master/salt/cni/remove-pre-reboot.sls --- old/salt-master/salt/cni/remove-pre-reboot.sls 2018-04-23 14:03:55.000000000 +0200 +++ new/salt-master/salt/cni/remove-pre-reboot.sls 1970-01-01 01:00:00.000000000 +0100 @@ -1,7 +0,0 @@ -# cleanup all the things we have created - -/etc/kubernetes/addons/kube-flannel-rbac.yaml: - file.absent - -/etc/kubernetes/addons/kube-flannel.yaml: - file.absent diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/crio/remove-pre-reboot.sls new/salt-master/salt/crio/remove-pre-reboot.sls --- old/salt-master/salt/crio/remove-pre-reboot.sls 2018-04-23 14:03:55.000000000 +0200 +++ new/salt-master/salt/crio/remove-pre-reboot.sls 1970-01-01 01:00:00.000000000 +0100 @@ -1,13 +0,0 @@ -# cleanup all the things we have created - -/etc/systemd/system/kubelet.service.d/kubelet.conf: - file.absent - -/var/lib/containers/storage: - cmd.run: - - name: |- - for subvolume in {{pillar['cri']['crio']['dirs']['root']}}/btrfs/subvolumes/* ; do - btrfs subvolume delete $subvolume - done - rm -rf {{pillar['cri']['crio']['dirs']['root']}}* - rm -rf {{pillar['cri']['crio']['dirs']['runroot']}}* diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/docker/remove-pre-reboot.sls new/salt-master/salt/docker/remove-pre-reboot.sls --- old/salt-master/salt/docker/remove-pre-reboot.sls 2018-04-23 14:03:55.000000000 +0200 +++ new/salt-master/salt/docker/remove-pre-reboot.sls 2018-04-26 11:56:10.000000000 +0200 @@ -1,21 +1,4 @@ -# cleanup all the things we have created -/etc/systemd/system/docker.service.d/proxy.conf: - file.absent - -/etc/docker/daemon.json: - file.absent - -/etc/sysconfig/docker: - file.absent /etc/docker/certs.d: file.absent - -/var/lib/docker: - cmd.run: - - name: |- - for subvolume in /var/lib/docker/btrfs/subvolumes/* ; do - btrfs subvolume delete $subvolume - done - rm -rf /var/lib/docker/* diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/etc-hosts/remove-pre-reboot.sls new/salt-master/salt/etc-hosts/remove-pre-reboot.sls --- old/salt-master/salt/etc-hosts/remove-pre-reboot.sls 2018-04-23 14:03:55.000000000 +0200 +++ new/salt-master/salt/etc-hosts/remove-pre-reboot.sls 1970-01-01 01:00:00.000000000 +0100 @@ -1,4 +0,0 @@ -# cleanup all the things we have created - -/etc/hosts: - file.absent diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/etcd/init.sls new/salt-master/salt/etcd/init.sls --- old/salt-master/salt/etcd/init.sls 2018-04-23 14:03:55.000000000 +0200 +++ new/salt-master/salt/etcd/init.sls 2018-04-26 11:56:10.000000000 +0200 @@ -14,6 +14,9 @@ - require: - file: /etc/zypp/repos.d/containers.repo caasp_etcd.member_add: + - retry: + interval: 4 + attempts: 15 - require: - {{ pillar['ssl']['crt_file'] }} - {{ pillar['ssl']['key_file'] }} diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/etcd/remove-pre-reboot.sls new/salt-master/salt/etcd/remove-pre-reboot.sls --- old/salt-master/salt/etcd/remove-pre-reboot.sls 2018-04-23 14:03:55.000000000 +0200 +++ new/salt-master/salt/etcd/remove-pre-reboot.sls 1970-01-01 01:00:00.000000000 +0100 @@ -1,23 +0,0 @@ - -# cleanup all the things we have created for etcd - -/etc/sysconfig/etcd: - file.absent - -/etc/sysconfig/etcdctl: - file.absent - -/etc/systemd/system/etcd.service.d/etcd.conf: - file.absent - -etcd-user-removal: - user.absent: - - name: etcd - -etcd-group-removal: - group.absent: - - name: etcd - -etcd-wipe-var-lib: - cmd.run: - - name: rm -rf /var/lib/etcd/* diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/etcd/remove-pre-stop-services.sls new/salt-master/salt/etcd/remove-pre-stop-services.sls --- old/salt-master/salt/etcd/remove-pre-stop-services.sls 2018-04-23 14:03:55.000000000 +0200 +++ new/salt-master/salt/etcd/remove-pre-stop-services.sls 1970-01-01 01:00:00.000000000 +0100 @@ -1,11 +0,0 @@ - -# this node is being removed from the cluster, -# but etcd is still running: -# we explicitly remove the node from the etcd cluster, -# so it is not considered a node suffering some -# transient failure... -etcd-remove-member: - caasp_etcd.member_remove: - - nodename: {{ salt['pillar.get']('nodename', grains['nodename']) }} - # NOTE: we are not requiring /etc/hosts or the certificates - # because we are assuming this node was on high state diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/kube-apiserver/remove-pre-reboot.sls new/salt-master/salt/kube-apiserver/remove-pre-reboot.sls --- old/salt-master/salt/kube-apiserver/remove-pre-reboot.sls 2018-04-23 14:03:55.000000000 +0200 +++ new/salt-master/salt/kube-apiserver/remove-pre-reboot.sls 2018-04-26 11:56:10.000000000 +0200 @@ -8,6 +8,3 @@ {{ pillar['ssl']['kube_apiserver_key'] }}: file.absent - -/etc/kubernetes/apiserver: - file.absent diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/kube-controller-manager/remove-pre-reboot.sls new/salt-master/salt/kube-controller-manager/remove-pre-reboot.sls --- old/salt-master/salt/kube-controller-manager/remove-pre-reboot.sls 2018-04-23 14:03:55.000000000 +0200 +++ new/salt-master/salt/kube-controller-manager/remove-pre-reboot.sls 2018-04-26 11:56:10.000000000 +0200 @@ -11,6 +11,3 @@ {{ pillar['paths']['service_account_key'] }}: file.absent - -{{ pillar['paths']['kube_controller_mgr_config'] }}: - file.absent diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/kube-proxy/remove-pre-reboot.sls new/salt-master/salt/kube-proxy/remove-pre-reboot.sls --- old/salt-master/salt/kube-proxy/remove-pre-reboot.sls 2018-04-23 14:03:55.000000000 +0200 +++ new/salt-master/salt/kube-proxy/remove-pre-reboot.sls 2018-04-26 11:56:10.000000000 +0200 @@ -8,6 +8,3 @@ {{ pillar['ssl']['kube_proxy_key'] }}: file.absent - -{{ pillar['paths']['kube_proxy_config'] }}: - file.absent diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/kube-scheduler/remove-pre-reboot.sls new/salt-master/salt/kube-scheduler/remove-pre-reboot.sls --- old/salt-master/salt/kube-scheduler/remove-pre-reboot.sls 2018-04-23 14:03:55.000000000 +0200 +++ new/salt-master/salt/kube-scheduler/remove-pre-reboot.sls 2018-04-26 11:56:10.000000000 +0200 @@ -8,6 +8,3 @@ {{ pillar['ssl']['kube_scheduler_key'] }}: file.absent - -{{ pillar['paths']['kube_scheduler_config'] }}: - file.absent diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/kubectl-config/remove-pre-reboot.sls new/salt-master/salt/kubectl-config/remove-pre-reboot.sls --- old/salt-master/salt/kubectl-config/remove-pre-reboot.sls 2018-04-23 14:03:55.000000000 +0200 +++ new/salt-master/salt/kubectl-config/remove-pre-reboot.sls 2018-04-26 11:56:10.000000000 +0200 @@ -5,9 +5,3 @@ {{ pillar['ssl']['kubectl_key'] }}: file.absent - -{{ pillar['paths']['kubeconfig'] }}: - file.absent - -/root/.kube: - file.absent diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/kubelet/init.sls new/salt-master/salt/kubelet/init.sls --- old/salt-master/salt/kubelet/init.sls 2018-04-23 14:03:55.000000000 +0200 +++ new/salt-master/salt/kubelet/init.sls 2018-04-26 11:56:10.000000000 +0200 @@ -4,6 +4,7 @@ - cert - kubernetes-common - kubectl-config + - swap /etc/kubernetes/kubelet-initial: file.managed: @@ -82,6 +83,7 @@ - file: /etc/kubernetes/manifests - file: /etc/kubernetes/kubelet-initial - kubelet-config + - cmd: unmount-swaps caasp_retriable.retry: - name: iptables-kubelet - target: iptables.append diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/kubelet/remove-pre-reboot.sls new/salt-master/salt/kubelet/remove-pre-reboot.sls --- old/salt-master/salt/kubelet/remove-pre-reboot.sls 2018-04-23 14:03:55.000000000 +0200 +++ new/salt-master/salt/kubelet/remove-pre-reboot.sls 2018-04-26 11:56:10.000000000 +0200 @@ -10,20 +10,10 @@ {{ pillar['ssl']['kubelet_key'] }}: file.absent -/etc/kubernetes/kubelet-initial: - file.absent - +# this file can contain sensitive information, so it must be removed too {{ pillar['paths']['kubelet_config'] }}: file.absent +# and this one too /etc/kubernetes/openstack-config: file.absent - -wipe-var-lib-kubelet: - cmd.run: - - name: |- - mnts=`mount | grep kubelet | grep tmpfs | cut -f3 -d" "` - for i in $mnts ; do - umount $i - done - rm -rf /var/lib/kubelet/* diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/kubelet/stop.sls new/salt-master/salt/kubelet/stop.sls --- old/salt-master/salt/kubelet/stop.sls 2018-04-23 14:03:55.000000000 +0200 +++ new/salt-master/salt/kubelet/stop.sls 2018-04-26 11:56:10.000000000 +0200 @@ -23,23 +23,6 @@ - force: True {%- endif %} -{%- if node_removal_in_progress %} - -# we must run the `delete node` when haproxy is still running. -# * in pre-stop-services, we have not cordoned the node yet -# * in pre-reboot, haproxy has been stopped -# so we have to do it here... - -delete-node-from-kubernetes: - cmd.run: - - name: |- - kubectl --kubeconfig={{ pillar['paths']['kubeconfig'] }} delete node {{ grains['nodename'] }} - - require: - - file: {{ pillar['paths']['kubeconfig'] }} - - drain-kubelet - -{%- endif %} - kubelet: service.dead: - enable: False diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/kubernetes-common/remove-pre-reboot.sls new/salt-master/salt/kubernetes-common/remove-pre-reboot.sls --- old/salt-master/salt/kubernetes-common/remove-pre-reboot.sls 2018-04-23 14:03:55.000000000 +0200 +++ new/salt-master/salt/kubernetes-common/remove-pre-reboot.sls 2018-04-26 11:56:10.000000000 +0200 @@ -1,10 +1,8 @@ -# cleanup all the things we have created + +# some files that could contain sensitive information: /etc/kubernetes/config: file.absent /etc/kubernetes/openstack-config: file.absent - -/var/lib/kubernetes: - file.absent diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/motd/remove-pre-reboot.sls new/salt-master/salt/motd/remove-pre-reboot.sls --- old/salt-master/salt/motd/remove-pre-reboot.sls 2018-04-23 14:03:55.000000000 +0200 +++ new/salt-master/salt/motd/remove-pre-reboot.sls 1970-01-01 01:00:00.000000000 +0100 @@ -1,4 +0,0 @@ -/etc/motd: - file.managed: - - contents: |- - This machine has been removed from the CaaS cluster. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/orch/removal.sls new/salt-master/salt/orch/removal.sls --- old/salt-master/salt/orch/removal.sls 2018-04-23 14:03:55.000000000 +0200 +++ new/salt-master/salt/orch/removal.sls 2018-04-26 11:56:10.000000000 +0200 @@ -1,23 +1,44 @@ -# must provide the node (id) to be removed in the 'target' pillar +{#- must provide the node (id) to be removed in the 'target' pillar #} {%- set target = salt['pillar.get']('target') %} +{#- ... and we can provide an optional replacement node #} +{%- set replacement = salt['pillar.get']('replacement', '') %} + +{#- Get a list of nodes seem to be down or unresponsive #} +{#- This sends a "are you still there?" message to all #} +{#- the nodes and wait for a response, so it takes some time. #} +{#- Hopefully this list will not be too long... #} +{%- set nodes_down = salt.saltutil.runner('manage.down') %} +{%- if not nodes_down %} + {%- do salt.caasp_log.debug('all nodes seem to be up') %} + {%- set all_responsive_nodes_tgt = 'P@roles:(etcd|kube-master|kube-minion)' %} +{%- else %} + {%- do salt.caasp_log.debug('nodes "%s" seem to be down', nodes_down|join(',')) %} + {%- set all_responsive_nodes_tgt = 'not L@' + nodes_down|join(',') + + ' and P@roles:(etcd|kube-master|kube-minion)' %} + + {%- if target in nodes_down %} + {%- do salt.caasp_log.abort('target is unresponsive, forced removal must be used') %} + {%- endif %} +{%- endif %} + {%- set etcd_members = salt.saltutil.runner('mine.get', tgt='G@roles:etcd', fun='network.interfaces', tgt_type='compound').keys() %} {%- set masters = salt.saltutil.runner('mine.get', tgt='G@roles:kube-master', fun='network.interfaces', tgt_type='compound').keys() %} {%- set minions = salt.saltutil.runner('mine.get', tgt='G@roles:kube-minion', fun='network.interfaces', tgt_type='compound').keys() %} -{#- ... and we can provide an optional replacement node #} -{%- set replacement = salt['pillar.get']('replacement', '') %} +{%- set super_master_tgt = salt.caasp_nodes.get_super_master(masters=masters, + excluded=[target] + nodes_down) %} +{%- if not super_master_tgt %} + {%- do salt.caasp_log.abort('(after removing %s) no masters are reachable', target) %} +{%- endif %} {#- try to use the user-provided replacement or find a replacement by ourselves #} {#- if no valid replacement can be used/found, `replacement` will be '' #} {%- set replacement, replacement_roles = salt.caasp_nodes.get_replacement_for(target, replacement, masters=masters, minions=minions, - etcd_members=etcd_members) %} - -{############################## - # set grains - #############################} + etcd_members=etcd_members, + excluded=nodes_down) %} # Ensure we mark all nodes with the "as node is being removed" grain. # This will ensure the update-etc-hosts orchestration is not run. @@ -30,6 +51,23 @@ - removal_in_progress - true +# make sure we have a solid ground before starting the removal +# (ie, expired certs produce really funny errors) +update-config: + salt.state: + - tgt: '{{ all_responsive_nodes_tgt }}' + - tgt_type: compound + - sls: + - etc-hosts + - ca-cert + - cert + - require: + - set-cluster-wide-removal-grain + +{############################## + # set grains + #############################} + assign-removal-grain: salt.function: - tgt: {{ target }} @@ -38,47 +76,47 @@ - node_removal_in_progress - true - require: - - set-cluster-wide-removal-grain + - update-config {%- if replacement %} assign-addition-grain: salt.function: - - tgt: {{ replacement }} + - tgt: '{{ replacement }}' - name: grains.setval - arg: - node_addition_in_progress - true - require: - - set-cluster-wide-removal-grain - - assign-removal-grain + - update-config {#- and then we can assign these (new) roles to the replacement #} {% for role in replacement_roles %} assign-{{ role }}-role-to-replacement: salt.function: - - tgt: {{ replacement }} + - tgt: '{{ replacement }}' - name: grains.append - arg: - roles - {{ role }} - require: - - assign-removal-grain + - update-config - assign-addition-grain - {%- endfor %} + {% endfor %} {%- endif %} {# replacement #} sync-all: salt.function: - - tgt: '*' + - tgt: '{{ all_responsive_nodes_tgt }}' + - tgt_type: compound - names: - saltutil.refresh_pillar - saltutil.refresh_grains - mine.update - saltutil.sync_all - require: - - set-cluster-wide-removal-grain + - update-config - assign-removal-grain {%- for role in replacement_roles %} - assign-{{ role }}-role-to-replacement @@ -92,7 +130,7 @@ highstate-replacement: salt.state: - - tgt: {{ replacement }} + - tgt: '{{ replacement }}' - highstate: True - require: - sync-all @@ -108,7 +146,7 @@ set-bootstrap-complete-flag-in-replacement: salt.function: - - tgt: {{ replacement }} + - tgt: '{{ replacement }}' - name: grains.setval - arg: - bootstrap_complete @@ -119,7 +157,7 @@ # remove the we-are-adding-this-node grain remove-addition-grain: salt.function: - - tgt: {{ replacement }} + - tgt: '{{ replacement }}' - name: grains.delval - arg: - node_addition_in_progress @@ -138,21 +176,6 @@ # the replacement should be ready at this point: # we can remove the old node running in {{ target }} -{%- if target in etcd_members %} {# we are only doing this for etcd at the moment... #} -prepare-target-removal: - salt.state: - - tgt: {{ target }} - - sls: - {%- if target in etcd_members %} - - etcd.remove-pre-stop-services - {%- endif %} - - require: - - sync-all - {%- if replacement %} - - set-bootstrap-complete-flag-in-replacement - {%- endif %} -{%- endif %} - stop-services-in-target: salt.state: - tgt: {{ target }} @@ -171,9 +194,9 @@ {%- endif %} - require: - sync-all - {%- if target in etcd_members %} - - prepare-target-removal - {%- endif %} + {%- if replacement %} + - remove-addition-grain + {%- endif %} # remove any other configuration in the machines cleanups-in-target-before-rebooting: @@ -184,7 +207,6 @@ - kube-apiserver.remove-pre-reboot - kube-controller-manager.remove-pre-reboot - kube-scheduler.remove-pre-reboot - - addons.remove-pre-reboot - addons.dns.remove-pre-reboot - addons.tiller.remove-pre-reboot - addons.dex.remove-pre-reboot @@ -193,12 +215,7 @@ - kubelet.remove-pre-reboot - kubectl-config.remove-pre-reboot - cri.remove-pre-reboot - - cni.remove-pre-reboot - {%- if target in etcd_members %} - - etcd.remove-pre-reboot - {%- endif %} - - etc-hosts.remove-pre-reboot - - motd.remove-pre-reboot + - cert.remove-pre-reboot - cleanup.remove-pre-reboot - require: - stop-services-in-target @@ -217,14 +234,30 @@ # (we don't need to wait for the node: # just forget about it...) -# remove the Salt key and the mine for the target +# do any cluster-scope removals in the super_master +remove-from-cluster-in-super-master: + salt.state: + - tgt: '{{ super_master_tgt }}' + - pillar: + target: {{ target }} + - sls: + - cleanup.remove-post-orchestration + - require: + - sync-all + - shutdown-target + {%- if replacement %} + - remove-addition-grain + {%- endif %} + +# remove the Salt key +# (it will appear as "unaccepted") remove-target-salt-key: salt.wheel: - name: key.reject - include_accepted: True - match: {{ target }} - require: - - shutdown-target + - remove-from-cluster-in-super-master # remove target's data in the Salt Master's cache remove-target-mine: @@ -244,18 +277,18 @@ # the etcd server we have just removed (but they would # keep working fine as long as we had >1 etcd servers) -{%- set affected_expr = salt.caasp_nodes.get_expr_affected_by(target, - excluded=[replacement], - masters=masters, - minions=minions, - etcd_members=etcd_members) %} - -{%- do salt.caasp_log.debug('will high-state machines affected by removal: %s', affected_expr) %} +{%- set affected_tgt = salt.caasp_nodes.get_expr_affected_by(target, + excluded=[replacement] + nodes_down, + masters=masters, + minions=minions, + etcd_members=etcd_members) %} +{%- do salt.caasp_log.debug('will high-state machines affected by removal: "%s"', affected_tgt) %} # make sure the cluster has up-to-date state sync-after-removal: salt.function: - - tgt: '*' + - tgt: '{{ all_responsive_nodes_tgt }}' + - tgt_type: compound - names: - saltutil.clear_cache - mine.update @@ -264,7 +297,7 @@ highstate-affected: salt.state: - - tgt: {{ affected_expr }} + - tgt: '{{ affected_tgt }}' - tgt_type: compound - highstate: True - batch: 1 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/top.sls new/salt-master/salt/top.sls --- old/salt-master/salt/top.sls 2018-04-23 14:03:55.000000000 +0200 +++ new/salt-master/salt/top.sls 2018-04-26 11:56:10.000000000 +0200 @@ -37,3 +37,4 @@ - cert - kubelet - kube-proxy + - cni/cilium
