This is an automated email from the ASF dual-hosted git repository. pearl11594 pushed a commit to branch cks-enhancements-part2 in repository https://gitbox.apache.org/repos/asf/cloudstack.git
commit 249d7106df4cb62c097568da089884c4fac5ced3 Author: Pearl Dsilva <[email protected]> AuthorDate: Tue Jan 28 15:21:38 2025 -0500 Externalize the CKS Configuration, so that end users can tweak the configuration before deploying the cluster * Add new directory to c8 packaging for CKS config * Remove k8s configuration from resources and make it configurable * Revert "Remove k8s configuration from resources and make it configurable" This reverts commit d5997033ebe4ba559e6478a64578b894f8e7d3db. * copy conf to mgmt server and consume them from there --- debian/rules | 8 ++++++ packaging/el8/cloud.spec | 9 ++++++- .../KubernetesClusterActionWorker.java | 31 ++++++++++++++-------- .../KubernetesClusterStartWorker.java | 26 +++++++++--------- 4 files changed, 49 insertions(+), 25 deletions(-) diff --git a/debian/rules b/debian/rules index e7ff6759d44..4d431b044fc 100755 --- a/debian/rules +++ b/debian/rules @@ -70,6 +70,7 @@ override_dh_auto_install: mkdir -p $(DESTDIR)/usr/share/$(PACKAGE)-management/lib mkdir -p $(DESTDIR)/usr/share/$(PACKAGE)-management/setup mkdir -p $(DESTDIR)/usr/share/$(PACKAGE)-management/templates/systemvm + mkdir -p $(DESTDIR)/usr/share/$(PACKAGE)-management/cks/conf mkdir $(DESTDIR)/var/log/$(PACKAGE)/management mkdir $(DESTDIR)/var/cache/$(PACKAGE)/management mkdir $(DESTDIR)/var/log/$(PACKAGE)/ipallocator @@ -83,6 +84,7 @@ override_dh_auto_install: cp client/target/cloud-client-ui-$(VERSION).jar $(DESTDIR)/usr/share/$(PACKAGE)-management/lib/cloudstack-$(VERSION).jar cp client/target/lib/*jar $(DESTDIR)/usr/share/$(PACKAGE)-management/lib/ cp -r engine/schema/dist/systemvm-templates/* $(DESTDIR)/usr/share/$(PACKAGE)-management/templates/systemvm/ + cp -r plugins/integrations/kubernetes-service/src/main/resources/conf/* $(DESTDIR)/usr/share/$(PACKAGE)-management/cks/conf/ rm -rf $(DESTDIR)/usr/share/$(PACKAGE)-management/templates/systemvm/md5sum.txt # Bundle cmk in cloudstack-management @@ -95,6 +97,12 @@ override_dh_auto_install: chmod 0440 $(DESTDIR)/$(SYSCONFDIR)/sudoers.d/$(PACKAGE) install -D client/target/utilities/bin/cloud-update-xenserver-licenses $(DESTDIR)/usr/bin/cloudstack-update-xenserver-licenses + + install -D plugins/integrations/kubernetes-service/src/main/resources/conf/etcd-node.yml $(DESTDIR)/usr/share/$(PACKAGE)-management/cks/conf/etcd-node.yml + install -D plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml $(DESTDIR)/usr/share/$(PACKAGE)-management/cks/conf/k8s-control-node.yml + install -D plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml $(DESTDIR)/usr/share/$(PACKAGE)-management/cks/conf/k8s-control-node-add.yml + install -D plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml $(DESTDIR)/usr/share/$(PACKAGE)-management/cks/conf/k8s-node.yml + # Remove configuration in /ur/share/cloudstack-management/webapps/client/WEB-INF # This should all be in /etc/cloudstack/management ln -s ../../..$(SYSCONFDIR)/$(PACKAGE)/management $(DESTDIR)/usr/share/$(PACKAGE)-management/conf diff --git a/packaging/el8/cloud.spec b/packaging/el8/cloud.spec index 22fede6fb85..2c6898cac7c 100644 --- a/packaging/el8/cloud.spec +++ b/packaging/el8/cloud.spec @@ -248,6 +248,7 @@ cp -r plugins/network-elements/cisco-vnmc/src/main/scripts/network/cisco/* ${RPM mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/ mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/lib mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/setup +mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cks/conf mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/log/%{name}/management mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/management mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/systemd/system/%{name}-management.service.d @@ -273,7 +274,7 @@ wget https://github.com/apache/cloudstack-cloudmonkey/releases/download/$CMK_REL chmod +x ${RPM_BUILD_ROOT}%{_bindir}/cmk cp -r client/target/utilities/scripts/db/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/setup - +cp -r plugins/integrations/kubernetes-service/src/main/resources/conf/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cks/conf cp -r client/target/cloud-client-ui-%{_maventag}.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/ cp -r client/target/classes/META-INF/webapp ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/webapp cp ui/dist/config.json ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/management/ @@ -308,6 +309,11 @@ touch ${RPM_BUILD_ROOT}%{_localstatedir}/run/%{name}-management.pid #install -D server/target/conf/cloudstack-catalina.logrotate ${RPM_BUILD_ROOT}%{_sysconfdir}/logrotate.d/%{name}-catalina install -D server/target/conf/cloudstack-management.logrotate ${RPM_BUILD_ROOT}%{_sysconfdir}/logrotate.d/%{name}-management +install -D plugins/integrations/kubernetes-service/src/main/resources/conf/etcd-node.yml ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cks/conf/etcd-node.yml +install -D plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cks/conf/k8s-control-node.yml +install -D plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cks/conf/k8s-control-node-add.yml +install -D plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cks/conf/k8s-node.yml + # SystemVM template mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/templates/systemvm cp -r engine/schema/dist/systemvm-templates/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/templates/systemvm @@ -608,6 +614,7 @@ pip3 install --upgrade /usr/share/cloudstack-marvin/Marvin-*.tar.gz %attr(0755,root,root) %{_bindir}/%{name}-sysvmadm %attr(0755,root,root) %{_bindir}/%{name}-setup-encryption %attr(0755,root,root) %{_bindir}/cmk +%{_datadir}/%{name}-management/cks/conf/*.yml %{_datadir}/%{name}-management/setup/*.sql %{_datadir}/%{name}-management/setup/*.sh %{_datadir}/%{name}-management/setup/server-setup.xml diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java index bc5af08a9c1..864c68c96e7 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java @@ -21,6 +21,9 @@ import java.io.File; import java.io.FileWriter; import java.io.IOException; import java.lang.reflect.Field; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -143,6 +146,7 @@ public class KubernetesClusterActionWorker { public static final String CKS_CLUSTER_SECURITY_GROUP_NAME = "CKSSecurityGroup"; public static final String CKS_SECURITY_GROUP_DESCRIPTION = "Security group for CKS nodes"; + public static final String CKS_CONFIG_PATH = "/usr/share/cloudstack-management/cks"; protected Logger logger = LogManager.getLogger(getClass()); @@ -264,6 +268,11 @@ public class KubernetesClusterActionWorker { return IOUtils.toString(Objects.requireNonNull(Thread.currentThread().getContextClassLoader().getResourceAsStream(resource)), com.cloud.utils.StringUtils.getPreferredCharset()); } + protected String readK8sConfigFile(String resource) throws IOException { + Path path = Paths.get(String.format("%s%s", CKS_CONFIG_PATH, resource)); + return Files.readString(path); + } + protected String getControlNodeLoginUser() { List<KubernetesClusterVmMapVO> vmMapVOList = getKubernetesClusterVMMaps(); if (!vmMapVOList.isEmpty()) { @@ -316,7 +325,7 @@ public class KubernetesClusterActionWorker { } protected void logTransitStateDetachIsoAndThrow(final Level logLevel, final String message, final KubernetesCluster kubernetesCluster, - final List<UserVm> clusterVMs, final KubernetesCluster.Event event, final Exception e) throws CloudRuntimeException { + final List<UserVm> clusterVMs, final KubernetesCluster.Event event, final Exception e) throws CloudRuntimeException { logMessage(logLevel, message, e); stateTransitTo(kubernetesCluster.getId(), event); detachIsoKubernetesVMs(clusterVMs); @@ -670,14 +679,14 @@ public class KubernetesClusterActionWorker { try { String command = String.format("sudo %s/%s -u '%s' -k '%s' -s '%s'", - scriptPath, deploySecretsScriptFilename, ApiServiceConfiguration.ApiServletPath.value(), keys[0], keys[1]); + scriptPath, deploySecretsScriptFilename, ApiServiceConfiguration.ApiServletPath.value(), keys[0], keys[1]); Account account = accountDao.findById(kubernetesCluster.getAccountId()); if (account != null && account.getType() == Account.Type.PROJECT) { String projectId = projectService.findByProjectAccountId(account.getId()).getUuid(); command = String.format("%s -p '%s'", command, projectId); } Pair<Boolean, String> result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(), - pkFile, null, command, 10000, 10000, 60000); + pkFile, null, command, 10000, 10000, 60000); return result.first(); } catch (Exception e) { String msg = String.format("Failed to add cloudstack-secret to Kubernetes cluster: %s", kubernetesCluster.getName()); @@ -696,7 +705,7 @@ public class KubernetesClusterActionWorker { writer.close(); } catch (IOException e) { logAndThrow(Level.ERROR, String.format("Kubernetes Cluster %s : Failed to fetch script %s", - kubernetesCluster.getName(), filename), e); + kubernetesCluster.getName(), filename), e); } return file; } @@ -719,11 +728,11 @@ public class KubernetesClusterActionWorker { sshKeyFile = getManagementServerSshPublicKeyFile(); } SshHelper.scpTo(nodeAddress, sshPort, getControlNodeLoginUser(), sshKeyFile, null, - "~/", file.getAbsolutePath(), "0755", 20000, 30 * 60 * 1000); + "~/", file.getAbsolutePath(), "0755", 20000, 30 * 60 * 1000); // Ensure destination dir scriptPath exists and copy file to destination String cmdStr = String.format("sudo mkdir -p %s ; sudo mv ~/%s %s/%s", scriptPath, file.getName(), scriptPath, destination); SshHelper.sshExecute(nodeAddress, sshPort, getControlNodeLoginUser(), sshKeyFile, null, - cmdStr, 10000, 10000, 10 * 60 * 1000); + cmdStr, 10000, 10000, 10 * 60 * 1000); } catch (Exception e) { throw new CloudRuntimeException(e); } @@ -771,7 +780,7 @@ public class KubernetesClusterActionWorker { // Since the provider creates IP addresses, don't deploy it unless the underlying network supports it if (manager.isDirectAccess(network)) { logMessage(Level.INFO, String.format("Skipping adding the provider as %s is not on an isolated network", - kubernetesCluster.getName()), null); + kubernetesCluster.getName()), null); return true; } File pkFile = getManagementServerSshPublicKeyFile(); @@ -782,7 +791,7 @@ public class KubernetesClusterActionWorker { try { String command = String.format("sudo %s/%s", scriptPath, deployProviderScriptFilename); Pair<Boolean, String> result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(), - pkFile, null, command, 10000, 10000, 60000); + pkFile, null, command, 10000, 10000, 60000); // Maybe the file isn't present. Try and copy it if (!result.first()) { @@ -792,12 +801,12 @@ public class KubernetesClusterActionWorker { if (!createCloudStackSecret(keys)) { logTransitStateAndThrow(Level.ERROR, String.format("Failed to setup keys for Kubernetes cluster %s", - kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); } // If at first you don't succeed ... result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(), - pkFile, null, command, 10000, 10000, 60000); + pkFile, null, command, 10000, 10000, 60000); if (!result.first()) { throw new CloudRuntimeException(result.second()); } @@ -871,7 +880,7 @@ public class KubernetesClusterActionWorker { } public String getKubernetesNodeConfig(final String joinIp, final boolean ejectIso, final boolean mountCksIsoOnVR) throws IOException { - String k8sNodeConfig = readResourceFile("/conf/k8s-node.yml"); + String k8sNodeConfig = readK8sConfigFile("/conf/k8s-node.yml"); final String sshPubKey = "{{ k8s.ssh.pub.key }}"; final String joinIpKey = "{{ k8s_control_node.join_ip }}"; final String clusterTokenKey = "{{ k8s_control_node.cluster.token }}"; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java index 5b0214f2095..68bec58d462 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java @@ -142,9 +142,9 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif } private Pair<String, String> getKubernetesControlNodeConfig(final String controlNodeIp, final String serverIp, - final List<Network.IpAddresses> etcdIps, final String hostName, final boolean haSupported, - final boolean ejectIso, final boolean externalCni) throws IOException { - String k8sControlNodeConfig = readResourceFile("/conf/k8s-control-node.yml"); + final List<Network.IpAddresses> etcdIps, final String hostName, final boolean haSupported, + final boolean ejectIso, final boolean externalCni) throws IOException { + String k8sControlNodeConfig = readK8sConfigFile("/conf/k8s-control-node.yml"); final String apiServerCert = "{{ k8s_control_node.apiserver.crt }}"; final String apiServerKey = "{{ k8s_control_node.apiserver.key }}"; final String caCert = "{{ k8s_control_node.ca.crt }}"; @@ -170,7 +170,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif boolean externalEtcd = !etcdIps.isEmpty(); final Certificate certificate = caManager.issueCertificate(null, Arrays.asList(hostName, "kubernetes", - "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster", "kubernetes.default.svc.cluster.local"), + "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster", "kubernetes.default.svc.cluster.local"), addresses, 3650, null); final String tlsClientCert = CertUtils.x509CertificateToPem(certificate.getClientCertificate()); final String tlsPrivateKey = CertUtils.privateKeyToPem(certificate.getPrivateKey()); @@ -301,7 +301,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif } private String getKubernetesAdditionalControlNodeConfig(final String joinIp, final boolean ejectIso) throws IOException { - String k8sControlNodeConfig = readResourceFile("/conf/k8s-control-node-add.yml"); + String k8sControlNodeConfig = readK8sConfigFile("/conf/k8s-control-node-add.yml"); final String joinIpKey = "{{ k8s_control_node.join_ip }}"; final String clusterTokenKey = "{{ k8s_control_node.cluster.token }}"; final String sshPubKey = "{{ k8s.ssh.pub.key }}"; @@ -336,13 +336,13 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif private String getInitialEtcdClusterDetails(List<String> ipAddresses, List<String> hostnames) { String initialCluster = "%s=http://%s:%s";; StringBuilder clusterInfo = new StringBuilder(); - for (int i = 0; i < ipAddresses.size(); i++) { - clusterInfo.append(String.format(initialCluster, hostnames.get(i), ipAddresses.get(i), KubernetesClusterActionWorker.ETCD_NODE_PEER_COMM_PORT)); - if (i < ipAddresses.size()-1) { - clusterInfo.append(","); - } + for (int i = 0; i < ipAddresses.size(); i++) { + clusterInfo.append(String.format(initialCluster, hostnames.get(i), ipAddresses.get(i), KubernetesClusterActionWorker.ETCD_NODE_PEER_COMM_PORT)); + if (i < ipAddresses.size()-1) { + clusterInfo.append(","); } - return clusterInfo.toString(); + } + return clusterInfo.toString(); } /** @@ -373,7 +373,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif private String getEtcdNodeConfig(final List<String> ipAddresses, final List<String> hostnames, final int etcdNodeIndex, final boolean ejectIso) throws IOException { - String k8sEtcdNodeConfig = readResourceFile("/conf/etcd-node.yml"); + String k8sEtcdNodeConfig = readK8sConfigFile("/conf/etcd-node.yml"); final String sshPubKey = "{{ k8s.ssh.pub.key }}"; final String ejectIsoKey = "{{ k8s.eject.iso }}"; final String installWaitTime = "{{ k8s.install.wait.time }}"; @@ -576,7 +576,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif } private List<Network.IpAddresses> getEtcdNodeGuestIps(final Network network, final long etcdNodeCount) { - List<Network.IpAddresses> guestIps = new ArrayList<>(); + List<Network.IpAddresses> guestIps = new ArrayList<>(); for (int i = 1; i <= etcdNodeCount; i++) { guestIps.add(new Network.IpAddresses(ipAddressManager.acquireGuestIpAddress(network, null), null)); }
