This is an automated email from the ASF dual-hosted git repository. pearl11594 pushed a commit to branch cks-enhancements-part2 in repository https://gitbox.apache.org/repos/asf/cloudstack.git
commit 8dac16aa359faa6500ea1e1ce548169cfd08331a Author: Pearl Dsilva <[email protected]> AuthorDate: Wed Jan 29 17:09:46 2025 -0500 Fix CKS HA clusters --- .../cluster/actionworkers/KubernetesClusterStartWorker.java | 12 ++++++++++-- .../src/main/resources/conf/k8s-control-node-add.yml | 7 +++++++ .../src/main/resources/conf/k8s-control-node.yml | 7 +++++++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java index 68bec58d462..69d493e85fe 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java @@ -161,6 +161,8 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif final String certSans = "{{ k8s_control.server_ips }}"; final String k8sCertificate = "{{ k8s_control.certificate_key }}"; final String externalCniPlugin = "{{ k8s.external.cni.plugin }}"; + final String isHaCluster = "{{ k8s.ha.cluster }}"; + final String publicIP = "{{ k8s.public.ip }}"; final List<String> addresses = new ArrayList<>(); addresses.add(controlNodeIp); @@ -202,7 +204,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif CLUSTER_API_PORT, KubernetesClusterUtil.generateClusterHACertificateKey(kubernetesCluster)); } - initArgs += String.format("--apiserver-cert-extra-sans=%s", controlNodeIp); + initArgs += String.format("--apiserver-cert-extra-sans=%s", String.join(",", addresses)); initArgs += String.format(" --kubernetes-version=%s", getKubernetesClusterVersion().getSemanticVersion()); k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterInitArgsKey, initArgs); k8sControlNodeConfig = k8sControlNodeConfig.replace(ejectIsoKey, String.valueOf(ejectIso)); @@ -212,6 +214,8 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif k8sControlNodeConfig = k8sControlNodeConfig.replace(certSans, String.format("- %s", serverIp)); k8sControlNodeConfig = k8sControlNodeConfig.replace(k8sCertificate, KubernetesClusterUtil.generateClusterHACertificateKey(kubernetesCluster)); k8sControlNodeConfig = k8sControlNodeConfig.replace(externalCniPlugin, String.valueOf(externalCni)); + k8sControlNodeConfig = k8sControlNodeConfig.replace(isHaCluster, String.valueOf(kubernetesCluster.getControlNodeCount() > 1)); + k8sControlNodeConfig = k8sControlNodeConfig.replace(publicIP, publicIpAddress); k8sControlNodeConfig = updateKubeConfigWithRegistryDetails(k8sControlNodeConfig); @@ -309,6 +313,8 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif final String ejectIsoKey = "{{ k8s.eject.iso }}"; final String installWaitTime = "{{ k8s.install.wait.time }}"; final String installReattemptsCount = "{{ k8s.install.reattempts.count }}"; + final String isHaCluster = "{{ k8s.ha.cluster }}"; + final String publicIP = "{{ k8s.public.ip }}"; final Long waitTime = KubernetesClusterService.KubernetesControlNodeInstallAttemptWait.value(); final Long reattempts = KubernetesClusterService.KubernetesControlNodeInstallReattempts.value(); @@ -328,6 +334,8 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterTokenKey, KubernetesClusterUtil.generateClusterToken(kubernetesCluster)); k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterHACertificateKey, KubernetesClusterUtil.generateClusterHACertificateKey(kubernetesCluster)); k8sControlNodeConfig = k8sControlNodeConfig.replace(ejectIsoKey, String.valueOf(ejectIso)); + k8sControlNodeConfig = k8sControlNodeConfig.replace(isHaCluster, String.valueOf(kubernetesCluster.getControlNodeCount() > 1)); + k8sControlNodeConfig = k8sControlNodeConfig.replace(publicIP, publicIpAddress); k8sControlNodeConfig = updateKubeConfigWithRegistryDetails(k8sControlNodeConfig); return k8sControlNodeConfig; @@ -426,7 +434,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif String hostName = String.format("%s-control-%s", kubernetesClusterNodeNamePrefix, suffix); String k8sControlNodeConfig = null; try { - k8sControlNodeConfig = getKubernetesAdditionalControlNodeConfig(joinIp, Hypervisor.HypervisorType.VMware.equals(clusterTemplate.getHypervisorType())); + k8sControlNodeConfig = getKubernetesAdditionalControlNodeConfig(publicIpAddress, Hypervisor.HypervisorType.VMware.equals(clusterTemplate.getHypervisorType())); } catch (IOException e) { logAndThrow(Level.ERROR, "Failed to read Kubernetes control configuration file", e); } diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml index 38f217f403c..e1d021dd078 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml @@ -225,6 +225,9 @@ write_files: exit 0 fi + HA_CLUSTER={{ k8s.ha.cluster }} + CLUSTER_PUBLIC_IP={{ k8s.public.ip }} + if [[ $(systemctl is-active setup-kube-system) != "inactive" ]]; then echo "setup-kube-system is running!" exit 1 @@ -242,6 +245,10 @@ write_files: cp -i /etc/kubernetes/admin.conf /root/.kube/config chown $(id -u):$(id -g) /root/.kube/config + if [[ "$HA_CLUSTER" == "true" ]]; then + sed -i -E "s|(server:\\s*).*|\\1https://${CLUSTER_PUBLIC_IP}:6443|" /root/.kube/config + fi + sudo touch /home/cloud/success echo "true" > /home/cloud/success diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml index dc066e10d06..d76dbe3f1f9 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml @@ -275,6 +275,9 @@ write_files: echo "Already provisioned!" exit 0 fi + + HA_CLUSTER={{ k8s.ha.cluster }} + CLUSTER_PUBLIC_IP={{ k8s.public.ip }} if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then export PATH=$PATH:/opt/bin @@ -319,6 +322,10 @@ write_files: cp -i /etc/kubernetes/admin.conf /root/.kube/config chown $(id -u):$(id -g) /root/.kube/config echo export PATH=\$PATH:/opt/bin >> /root/.bashrc + + if [[ "$HA_CLUSTER" == "true" ]]; then + sed -i -E "s|(server:\\s*).*|\\1https://${CLUSTER_PUBLIC_IP}:6443|" /root/.kube/config + fi if [ -d "$K8S_CONFIG_SCRIPTS_COPY_DIR" ]; then ### Network, dashboard configs available offline ###
