This is an automated email from the ASF dual-hosted git repository.
pearl11594 pushed a commit to branch nsx-integration-fixes
in repository https://gitbox.apache.org/repos/asf/cloudstack.git
commit a89964854e30d8d703e72d956684e254a77d9d79
Author: Pearl Dsilva <pearl1...@gmail.com>
AuthorDate: Wed Apr 10 10:06:40 2024 -0400
CKS: Externalize control and worker node setup wait time and installation
attempts (#38)
---
.../cluster/KubernetesClusterManagerImpl.java | 6 +++++-
.../cluster/KubernetesClusterService.java | 24 ++++++++++++++++++++++
...ernetesClusterResourceModifierActionWorker.java | 9 +++++++-
.../KubernetesClusterStartWorker.java | 14 +++++++++++++
.../main/resources/conf/k8s-control-node-add.yml | 10 +++++++--
.../src/main/resources/conf/k8s-control-node.yml | 10 +++++++--
.../src/main/resources/conf/k8s-node.yml | 10 +++++++--
7 files changed, 75 insertions(+), 8 deletions(-)
diff --git
a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java
b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java
index 834d6d33330..6fff17e3428 100644
---
a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java
+++
b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java
@@ -1984,7 +1984,11 @@ public class KubernetesClusterManagerImpl extends
ManagerBase implements Kuberne
KubernetesClusterUpgradeTimeout,
KubernetesClusterUpgradeRetries,
KubernetesClusterExperimentalFeaturesEnabled,
- KubernetesMaxClusterSize
+ KubernetesMaxClusterSize,
+ KubernetesControlNodeInstallAttemptWait,
+ KubernetesControlNodeInstallReattempts,
+ KubernetesWorkerNodeInstallAttemptWait,
+ KubernetesWorkerNodeInstallReattempts
};
}
}
diff --git
a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java
b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java
index 39b926537f5..5d5e188fa45 100644
---
a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java
+++
b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java
@@ -91,6 +91,30 @@ public interface KubernetesClusterService extends
PluggableService, Configurable
true,
ConfigKey.Scope.Account,
KubernetesServiceEnabled.key());
+ static final ConfigKey<Long> KubernetesControlNodeInstallAttemptWait = new
ConfigKey<Long>("Advanced", Long.class,
+ "cloud.kubernetes.control.node.install.attempt.wait.duration",
+ "15",
+ "Time in seconds for the installation process to wait before it
re-attempts",
+ true,
+ KubernetesServiceEnabled.key());
+ static final ConfigKey<Long> KubernetesControlNodeInstallReattempts = new
ConfigKey<Long>("Advanced", Long.class,
+ "cloud.kubernetes.control.node.install.reattempt.count",
+ "100",
+ "Number of times the offline installation of K8S will be
re-attempted",
+ true,
+ KubernetesServiceEnabled.key());
+ final ConfigKey<Long> KubernetesWorkerNodeInstallAttemptWait = new
ConfigKey<Long>("Advanced", Long.class,
+ "cloud.kubernetes.worker.node.install.attempt.wait.duration",
+ "30",
+ "Time in seconds for the installation process to wait before it
re-attempts",
+ true,
+ KubernetesServiceEnabled.key());
+ static final ConfigKey<Long> KubernetesWorkerNodeInstallReattempts = new
ConfigKey<Long>("Advanced", Long.class,
+ "cloud.kubernetes.worker.node.install.reattempt.count",
+ "40",
+ "Number of times the offline installation of K8S will be
re-attempted",
+ true,
+ KubernetesServiceEnabled.key());
KubernetesCluster findById(final Long id);
diff --git
a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java
b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java
index d0f9cf842c5..cf4bb07f58b 100644
---
a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java
+++
b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java
@@ -31,6 +31,7 @@ import java.util.stream.Collectors;
import javax.inject.Inject;
+import com.cloud.kubernetes.cluster.KubernetesClusterService;
import com.cloud.network.rules.FirewallManager;
import com.cloud.offering.NetworkOffering;
import com.cloud.offerings.dao.NetworkOfferingDao;
@@ -174,6 +175,11 @@ public class KubernetesClusterResourceModifierActionWorker
extends KubernetesClu
final String joinIpKey = "{{ k8s_control_node.join_ip }}";
final String clusterTokenKey = "{{ k8s_control_node.cluster.token }}";
final String ejectIsoKey = "{{ k8s.eject.iso }}";
+ final String installWaitTime = "{{ k8s.install.wait.time }}";
+ final String installReattemptsCount = "{{ k8s.install.reattempts.count
}}";
+
+ final Long waitTime =
KubernetesClusterService.KubernetesWorkerNodeInstallAttemptWait.value();
+ final Long reattempts =
KubernetesClusterService.KubernetesWorkerNodeInstallReattempts.value();
String pubKey = "- \"" + configurationDao.getValue("ssh.publickey") +
"\"";
String sshKeyPair = kubernetesCluster.getKeyPair();
if (StringUtils.isNotEmpty(sshKeyPair)) {
@@ -186,7 +192,8 @@ public class KubernetesClusterResourceModifierActionWorker
extends KubernetesClu
k8sNodeConfig = k8sNodeConfig.replace(joinIpKey, joinIp);
k8sNodeConfig = k8sNodeConfig.replace(clusterTokenKey,
KubernetesClusterUtil.generateClusterToken(kubernetesCluster));
k8sNodeConfig = k8sNodeConfig.replace(ejectIsoKey,
String.valueOf(ejectIso));
-
+ k8sNodeConfig = k8sNodeConfig.replace(installWaitTime,
String.valueOf(waitTime));
+ k8sNodeConfig = k8sNodeConfig.replace(installReattemptsCount,
String.valueOf(reattempts));
k8sNodeConfig = updateKubeConfigWithRegistryDetails(k8sNodeConfig);
return k8sNodeConfig;
diff --git
a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java
b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java
index a7cea8093c8..c1fdc6d2b45 100644
---
a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java
+++
b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java
@@ -139,6 +139,9 @@ public class KubernetesClusterStartWorker extends
KubernetesClusterResourceModif
final String clusterToken = "{{ k8s_control_node.cluster.token }}";
final String clusterInitArgsKey = "{{
k8s_control_node.cluster.initargs }}";
final String ejectIsoKey = "{{ k8s.eject.iso }}";
+ final String installWaitTime = "{{ k8s.install.wait.time }}";
+ final String installReattemptsCount = "{{ k8s.install.reattempts.count
}}";
+
final List<String> addresses = new ArrayList<>();
addresses.add(controlNodeIp);
if (!serverIp.equals(controlNodeIp)) {
@@ -150,6 +153,8 @@ public class KubernetesClusterStartWorker extends
KubernetesClusterResourceModif
final String tlsClientCert =
CertUtils.x509CertificateToPem(certificate.getClientCertificate());
final String tlsPrivateKey =
CertUtils.privateKeyToPem(certificate.getPrivateKey());
final String tlsCaCert =
CertUtils.x509CertificatesToPem(certificate.getCaCertificates());
+ final Long waitTime =
KubernetesClusterService.KubernetesControlNodeInstallAttemptWait.value();
+ final Long reattempts =
KubernetesClusterService.KubernetesControlNodeInstallReattempts.value();
k8sControlNodeConfig = k8sControlNodeConfig.replace(apiServerCert,
tlsClientCert.replace("\n", "\n "));
k8sControlNodeConfig = k8sControlNodeConfig.replace(apiServerKey,
tlsPrivateKey.replace("\n", "\n "));
k8sControlNodeConfig = k8sControlNodeConfig.replace(caCert,
tlsCaCert.replace("\n", "\n "));
@@ -161,6 +166,8 @@ public class KubernetesClusterStartWorker extends
KubernetesClusterResourceModif
pubKey += "\n - \"" + sshkp.getPublicKey() + "\"";
}
}
+ k8sControlNodeConfig = k8sControlNodeConfig.replace(installWaitTime,
String.valueOf(waitTime));
+ k8sControlNodeConfig =
k8sControlNodeConfig.replace(installReattemptsCount,
String.valueOf(reattempts));
k8sControlNodeConfig = k8sControlNodeConfig.replace(sshPubKey, pubKey);
k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterToken,
KubernetesClusterUtil.generateClusterToken(kubernetesCluster));
String initArgs = "";
@@ -241,6 +248,11 @@ public class KubernetesClusterStartWorker extends
KubernetesClusterResourceModif
final String sshPubKey = "{{ k8s.ssh.pub.key }}";
final String clusterHACertificateKey = "{{
k8s_control_node.cluster.ha.certificate.key }}";
final String ejectIsoKey = "{{ k8s.eject.iso }}";
+ final String installWaitTime = "{{ k8s.install.wait.time }}";
+ final String installReattemptsCount = "{{ k8s.install.reattempts.count
}}";
+
+ final Long waitTime =
KubernetesClusterService.KubernetesControlNodeInstallAttemptWait.value();
+ final Long reattempts =
KubernetesClusterService.KubernetesControlNodeInstallReattempts.value();
String pubKey = "- \"" + configurationDao.getValue("ssh.publickey") +
"\"";
String sshKeyPair = kubernetesCluster.getKeyPair();
if (StringUtils.isNotEmpty(sshKeyPair)) {
@@ -249,6 +261,8 @@ public class KubernetesClusterStartWorker extends
KubernetesClusterResourceModif
pubKey += "\n - \"" + sshkp.getPublicKey() + "\"";
}
}
+ k8sControlNodeConfig = k8sControlNodeConfig.replace(installWaitTime,
String.valueOf(waitTime));
+ k8sControlNodeConfig =
k8sControlNodeConfig.replace(installReattemptsCount,
String.valueOf(reattempts));
k8sControlNodeConfig = k8sControlNodeConfig.replace(sshPubKey, pubKey);
k8sControlNodeConfig = k8sControlNodeConfig.replace(joinIpKey, joinIp);
k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterTokenKey,
KubernetesClusterUtil.generateClusterToken(kubernetesCluster));
diff --git
a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml
b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml
index 2c18efa0189..35134bd1e16 100644
---
a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml
+++
b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml
@@ -42,8 +42,14 @@ write_files:
ATTEMPT_ONLINE_INSTALL=false
setup_complete=false
- OFFLINE_INSTALL_ATTEMPT_SLEEP=15
- MAX_OFFLINE_INSTALL_ATTEMPTS=100
+ OFFLINE_INSTALL_ATTEMPT_SLEEP={{ k8s.install.wait.time }}
+ MAX_OFFLINE_INSTALL_ATTEMPTS={{ k8s.install.reattempts.count }}
+ if [[ -z $OFFLINE_INSTALL_ATTEMPT_SLEEP ||
$OFFLINE_INSTALL_ATTEMPT_SLEEP -eq 0 ]]; then
+ OFFLINE_INSTALL_ATTEMPT_SLEEP=15
+ fi
+ if [[ -z $MAX_OFFLINE_INSTALL_ATTEMPTS || $MAX_OFFLINE_INSTALL_ATTEMPTS
-eq 0 ]]; then
+ MAX_OFFLINE_INSTALL_ATTEMPTS=100
+ fi
offline_attempts=1
MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3
EJECT_ISO_FROM_OS={{ k8s.eject.iso }}
diff --git
a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml
b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml
index aa7eec97ac8..3154fb20251 100644
---
a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml
+++
b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml
@@ -62,8 +62,14 @@ write_files:
ATTEMPT_ONLINE_INSTALL=false
setup_complete=false
- OFFLINE_INSTALL_ATTEMPT_SLEEP=15
- MAX_OFFLINE_INSTALL_ATTEMPTS=100
+ OFFLINE_INSTALL_ATTEMPT_SLEEP={{ k8s.install.wait.time }}
+ MAX_OFFLINE_INSTALL_ATTEMPTS={{ k8s.install.reattempts.count }}
+ if [[ -z $OFFLINE_INSTALL_ATTEMPT_SLEEP ||
$OFFLINE_INSTALL_ATTEMPT_SLEEP -eq 0 ]]; then
+ OFFLINE_INSTALL_ATTEMPT_SLEEP=15
+ fi
+ if [[ -z $MAX_OFFLINE_INSTALL_ATTEMPTS || $MAX_OFFLINE_INSTALL_ATTEMPTS
-eq 0 ]]; then
+ MAX_OFFLINE_INSTALL_ATTEMPTS=100
+ fi
offline_attempts=1
MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3
EJECT_ISO_FROM_OS={{ k8s.eject.iso }}
diff --git
a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml
b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml
index de1f4c9ffc7..298236da0df 100644
---
a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml
+++
b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml
@@ -42,8 +42,14 @@ write_files:
ATTEMPT_ONLINE_INSTALL=false
setup_complete=false
- OFFLINE_INSTALL_ATTEMPT_SLEEP=30
- MAX_OFFLINE_INSTALL_ATTEMPTS=40
+ OFFLINE_INSTALL_ATTEMPT_SLEEP={{ k8s.install.wait.time }}
+ MAX_OFFLINE_INSTALL_ATTEMPTS={{ k8s.install.reattempts.count }}
+ if [[ -z $OFFLINE_INSTALL_ATTEMPT_SLEEP ||
$OFFLINE_INSTALL_ATTEMPT_SLEEP -eq 0 ]]; then
+ OFFLINE_INSTALL_ATTEMPT_SLEEP=30
+ fi
+ if [[ -z $MAX_OFFLINE_INSTALL_ATTEMPTS || $MAX_OFFLINE_INSTALL_ATTEMPTS
-eq 0 ]]; then
+ MAX_OFFLINE_INSTALL_ATTEMPTS=40
+ fi
offline_attempts=1
MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3
EJECT_ISO_FROM_OS={{ k8s.eject.iso }}
