This is an automated email from the ASF dual-hosted git repository. pearl11594 pushed a commit to branch test-opt in repository https://gitbox.apache.org/repos/asf/cloudstack.git
commit 71aea5e0c08883b68f351c9808156a18b584a50d Author: Pearl Dsilva <[email protected]> AuthorDate: Fri Dec 24 12:20:41 2021 +0530 Refactor ca/cert patching logic --- .../main/java/com/cloud/vm/VirtualMachineGuru.java | 25 ++++++++++++ .../com/cloud/vm/VirtualMachineManagerImpl.java | 17 -------- scripts/util/keystore-cert-import | 45 ++++++++++++++-------- .../consoleproxy/ConsoleProxyManagerImpl.java | 21 ++++------ .../java/com/cloud/consoleproxy/ConsoleProxy.java | 27 +++++++------ .../SecondaryStorageManagerImpl.java | 11 ++++++ .../debian/opt/cloud/bin/setup/cloud-early-config | 2 +- systemvm/debian/opt/cloud/bin/setup/common.sh | 22 ++++++++++- .../debian/opt/cloud/bin/setup/consoleproxy.sh | 2 + systemvm/debian/opt/cloud/bin/setup/secstorage.sh | 1 + 10 files changed, 112 insertions(+), 61 deletions(-) diff --git a/engine/api/src/main/java/com/cloud/vm/VirtualMachineGuru.java b/engine/api/src/main/java/com/cloud/vm/VirtualMachineGuru.java index 7611df8..d6d7513 100644 --- a/engine/api/src/main/java/com/cloud/vm/VirtualMachineGuru.java +++ b/engine/api/src/main/java/com/cloud/vm/VirtualMachineGuru.java @@ -20,7 +20,14 @@ import com.cloud.agent.api.Answer; import com.cloud.agent.manager.Commands; import com.cloud.deploy.DeployDestination; import com.cloud.exception.ResourceUnavailableException; +import com.cloud.utils.PasswordGenerator; +import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.cloudstack.ca.CAManager; +import org.apache.cloudstack.framework.ca.Certificate; +import org.apache.cloudstack.utils.security.CertUtils; +import org.apache.cloudstack.utils.security.KeyStoreUtils; +import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.Base64; @@ -72,4 +79,22 @@ public interface VirtualMachineGuru { } return base64EncodedPublicKey; } + + private static String getEncodedString(String certificate) { + return Base64.getEncoder().encodeToString(certificate.replace("\n", KeyStoreUtils.CERT_NEWLINE_ENCODER).replace(" ", KeyStoreUtils.CERT_SPACE_ENCODER).getBytes(StandardCharsets.UTF_8)); + } + + static void appendCertificateDetails(StringBuilder buf, Certificate certificate) { + try { + buf.append(" certificate=").append(getEncodedString(CertUtils.x509CertificateToPem(certificate.getClientCertificate()))); + buf.append(" cacertificate=").append(getEncodedString(CertUtils.x509CertificatesToPem(certificate.getCaCertificates()))); + if (certificate.getPrivateKey() != null) { + buf.append(" privatekey=").append(getEncodedString(CertUtils.privateKeyToPem(certificate.getPrivateKey()))); + } + } catch (IOException e) { + throw new CloudRuntimeException("Failed to transform X509 cert to PEM format", e); + } + buf.append(" keystore_password=").append(getEncodedString(PasswordGenerator.generateRandomPassword(16))); + buf.append(" validity=").append(CAManager.CertValidityPeriod.value()); + } } diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index da70408..4e4a25d 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -1008,9 +1008,6 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac final String csr = caManager.generateKeyStoreAndCsr(vmHost, sshAccessDetails); if (!Strings.isNullOrEmpty(csr)) { final Map<String, String> ipAddressDetails = new HashMap<>(sshAccessDetails); - for (Map.Entry<String,String> e : ipAddressDetails.entrySet()) { - s_logger.info("PEARL - k = " + e.getKey() + " v: "+ e.getValue()); - } ipAddressDetails.remove(NetworkElementCommand.ROUTER_NAME); final Certificate certificate = caManager.issueCertificate(csr, Arrays.asList(vm.getHostName(), vm.getInstanceName()), new ArrayList<>(ipAddressDetails.values()), CAManager.CertValidityPeriod.value(), null); @@ -1278,20 +1275,6 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac if (s_logger.isDebugEnabled()) { s_logger.debug("Start completed for VM " + vm); } - final Host vmHost = _hostDao.findById(destHostId); - if (vmHost != null && (VirtualMachine.Type.ConsoleProxy.equals(vm.getType()) || - VirtualMachine.Type.SecondaryStorageVm.equals(vm.getType())) && caManager.canProvisionCertificates()) { - final Map<String, String> sshAccessDetails = _networkMgr.getSystemVMAccessDetails(vm); - for (int retries = 3; retries > 0; retries--) { - try { - setupAgentSecurity(vmHost, sshAccessDetails, vm); - return; - } catch (final Exception e) { - s_logger.error("Retrying after catching exception while trying to secure agent for systemvm id=" + vm.getId(), e); - } - } - throw new CloudRuntimeException("Failed to setup and secure agent for systemvm id=" + vm.getId()); - } return; } else { if (s_logger.isDebugEnabled()) { diff --git a/scripts/util/keystore-cert-import b/scripts/util/keystore-cert-import index a2b57bf..8e63e95 100755 --- a/scripts/util/keystore-cert-import +++ b/scripts/util/keystore-cert-import @@ -17,19 +17,39 @@ # under the License. PROPS_FILE="$1" -KS_FILE="$2" -MODE="$3" -CERT_FILE="$4" -CERT=$(echo "$5" | tr '^' '\n' | tr '~' ' ') -CACERT_FILE="$6" -CACERT=$(echo "$7" | tr '^' '\n' | tr '~' ' ') -PRIVKEY_FILE="$8" -PRIVKEY=$(echo "$9" | tr '^' '\n' | tr '~' ' ') +KS_PASS="$2" +KS_VALIDITY="$3" +KS_FILE="$4" +MODE="$5" +CERT_FILE="$6" +CERT=$(echo "$7" | tr '^' '\n' | tr '~' ' ') +CACERT_FILE="$8" +CACERT=$(echo "$9" | tr '^' '\n' | tr '~' ' ') +PRIVKEY_FILE="${10}" +PRIVKEY=$(echo "${11}" | tr '^' '\n' | tr '~' ' ') ALIAS="cloud" SYSTEM_FILE="/var/cache/cloud/cmdline" LIBVIRTD_FILE="/etc/libvirt/libvirtd.conf" +# Re-use existing password or use the one provided +if [ -f "$PROPS_FILE" ]; then + OLD_PASS=$(sed -n '/keystore.passphrase/p' "$PROPS_FILE" 2>/dev/null | sed 's/keystore.passphrase=//g' 2>/dev/null) + if [ ! -z "${OLD_PASS// }" ]; then + KS_PASS="$OLD_PASS" + else + sed -i "/keystore.passphrase.*/d" $PROPS_FILE 2> /dev/null || true + echo "keystore.passphrase=$KS_PASS" >> $PROPS_FILE + fi +fi + +if [ -f "$KS_FILE" ]; then + keytool -delete -noprompt -alias "$ALIAS" -keystore "$KS_FILE" -storepass "$KS_PASS" > /dev/null 2>&1 || true +fi + +CN=$(hostname --fqdn) +keytool -genkey -storepass "$KS_PASS" -keypass "$KS_PASS" -alias "$ALIAS" -keyalg RSA -validity "$KS_VALIDITY" -dname cn="$CN",ou="cloudstack",o="cloudstack",c="cloudstack" -keystore "$KS_FILE" > /dev/null 2>&1 + # Find keystore password KS_PASS=$(sed -n '/keystore.passphrase/p' "$PROPS_FILE" 2>/dev/null | sed 's/keystore.passphrase=//g' 2>/dev/null) @@ -56,11 +76,6 @@ for caChain in $(ls cloudca.*); do done rm -f cloudca.* -# Stop cloud service in systemvm -if [ "$MODE" == "ssh" ] && [ -f $SYSTEM_FILE ]; then - systemctl stop cloud > /dev/null 2>&1 -fi - # Import private key if available if [ ! -z "${PRIVKEY// }" ]; then echo "$PRIVKEY" > "$PRIVKEY_FILE" @@ -99,10 +114,6 @@ if [ -f "$SYSTEM_FILE" ]; then chmod 644 /usr/local/share/ca-certificates/cloudstack/ca.crt update-ca-certificates > /dev/null 2>&1 || true - # Ensure cloud service is running in systemvm - if [ "$MODE" == "ssh" ]; then - systemctl start cloud > /dev/null 2>&1 - fi fi # Fix file permission diff --git a/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java b/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java index 1dcaf7b..40c6363 100644 --- a/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java +++ b/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java @@ -29,11 +29,11 @@ import java.util.Map; import javax.inject.Inject; import javax.naming.ConfigurationException; -import com.cloud.utils.PasswordGenerator; import org.apache.cloudstack.agent.lb.IndirectAgentLB; import org.apache.cloudstack.ca.CAManager; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; +import org.apache.cloudstack.framework.ca.Certificate; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; @@ -1210,8 +1210,11 @@ public class ConsoleProxyManagerImpl extends ManagerBase implements ConsoleProxy @Override public boolean finalizeVirtualMachineProfile(VirtualMachineProfile profile, DeployDestination dest, ReservationContext context) { -// final Certificate certificate = caManager.issueCertificate(null, Arrays.asList(profile.getHostName(), profile.getInstanceName()), -// new ArrayList<>(ipAddressDetails.values()), CAManager.CertValidityPeriod.value(), null); + final Map<String, String> sshAccessDetails = networkMgr.getSystemVMAccessDetails(profile.getVirtualMachine()); + final Map<String, String> ipAddressDetails = new HashMap<>(sshAccessDetails); + ipAddressDetails.remove("router.name"); + final Certificate certificate = caManager.issueCertificate(null, Arrays.asList(profile.getHostName(), profile.getInstanceName()), + new ArrayList<>(ipAddressDetails.values()), CAManager.CertValidityPeriod.value(), null); ConsoleProxyVO vm = consoleProxyDao.findById(profile.getId()); Map<String, String> details = userVmDetailsDao.listDetailsKeyPairs(vm.getId()); vm.setDetails(details); @@ -1281,17 +1284,7 @@ public class ConsoleProxyManagerImpl extends ManagerBase implements ConsoleProxy buf.append(" dns2=").append(dc.getDns2()); } -// try { -// buf.append(" certificate=").append(CertUtils.x509CertificateToPem(certificate.getClientCertificate())); -// buf.append(" cacertificate=").append(CertUtils.x509CertificatesToPem(certificate.getCaCertificates())); -// if (certificate.getPrivateKey() != null) { -// buf.append(" privatekey=").append(CertUtils.privateKeyToPem(certificate.getPrivateKey())); -// } -// } catch (IOException e) { -// throw new CloudRuntimeException("Failed to transform X509 cert to PEM format", e); -// } - buf.append(" keystore_password=").append(PasswordGenerator.generateRandomPassword(16)); - buf.append(" validity=").append(CAManager.CertValidityPeriod.value()); + VirtualMachineGuru.appendCertificateDetails(buf, certificate); String bootArgs = buf.toString(); if (s_logger.isDebugEnabled()) { s_logger.debug("Boot Args for " + profile + ": " + bootArgs); diff --git a/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxy.java b/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxy.java index 702e9a8..53b2b99 100644 --- a/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxy.java +++ b/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxy.java @@ -16,6 +16,15 @@ // under the License. package com.cloud.consoleproxy; +import com.cloud.consoleproxy.util.Logger; +import com.cloud.utils.PropertiesUtil; +import com.cloud.utils.StringUtils; +import com.google.gson.Gson; +import com.sun.net.httpserver.HttpServer; +import org.apache.commons.lang3.ArrayUtils; +import org.apache.log4j.xml.DOMConfigurator; +import org.eclipse.jetty.websocket.api.Session; + import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; @@ -31,15 +40,6 @@ import java.util.Map; import java.util.Properties; import java.util.concurrent.Executor; -import com.cloud.utils.StringUtils; -import org.apache.log4j.xml.DOMConfigurator; -import org.eclipse.jetty.websocket.api.Session; - -import com.cloud.consoleproxy.util.Logger; -import com.cloud.utils.PropertiesUtil; -import com.google.gson.Gson; -import com.sun.net.httpserver.HttpServer; - /** * * ConsoleProxy, singleton class that manages overall activities in console proxy process. To make legacy code work, we still @@ -74,6 +74,7 @@ public class ConsoleProxy { static boolean standaloneStart = false; static String encryptorPassword = "Dummy"; + static final String[] skipProperties = new String[]{"certificate", "cacertificate", "keystore_password", "privatekey"}; private static void configLog4j() { final ClassLoader loader = Thread.currentThread().getContextClassLoader(); @@ -108,7 +109,9 @@ public class ConsoleProxy { private static void configProxy(Properties conf) { s_logger.info("Configure console proxy..."); for (Object key : conf.keySet()) { - s_logger.info("Property " + (String)key + ": " + conf.getProperty((String)key)); + if (!ArrayUtils.contains(skipProperties, key)) { + s_logger.info("Property " + (String)key + ": " + conf.getProperty((String)key)); + } } String s = conf.getProperty("consoleproxy.httpListenPort"); @@ -247,7 +250,9 @@ public class ConsoleProxy { if (conf != null) { for (Object key : conf.keySet()) { - s_logger.info("Context property " + (String)key + ": " + conf.getProperty((String)key)); + if (!ArrayUtils.contains(skipProperties, key)) { + s_logger.info("Context property " + (String) key + ": " + conf.getProperty((String) key)); + } } } diff --git a/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java b/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java index 99539a2..ad74838 100644 --- a/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java +++ b/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java @@ -31,11 +31,13 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; import org.apache.cloudstack.agent.lb.IndirectAgentLB; +import org.apache.cloudstack.ca.CAManager; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope; +import org.apache.cloudstack.framework.ca.Certificate; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; @@ -245,6 +247,8 @@ public class SecondaryStorageManagerImpl extends ManagerBase implements Secondar private ImageStoreDetailsUtil imageStoreDetailsUtil; @Inject private IndirectAgentLB indirectAgentLB; + @Inject + private CAManager caManager; private long _capacityScanInterval = DEFAULT_CAPACITY_SCAN_INTERVAL_IN_MILLISECONDS; private int _secStorageVmMtuSize; @@ -1072,6 +1076,12 @@ public class SecondaryStorageManagerImpl extends ManagerBase implements Secondar return false; } + final Map<String, String> sshAccessDetails = _networkMgr.getSystemVMAccessDetails(profile.getVirtualMachine()); + final Map<String, String> ipAddressDetails = new HashMap<>(sshAccessDetails); + ipAddressDetails.remove("router.name"); + final Certificate certificate = caManager.issueCertificate(null, Arrays.asList(profile.getHostName(), profile.getInstanceName()), + new ArrayList<>(ipAddressDetails.values()), CAManager.CertValidityPeriod.value(), null); + StringBuilder buf = profile.getBootArgsBuilder(); buf.append(" template=domP type=secstorage"); buf.append(" host=").append(StringUtils.toCSVList(indirectAgentLB.getManagementServerList(dest.getHost().getId(), dest.getDataCenter().getId(), null))); @@ -1157,6 +1167,7 @@ public class SecondaryStorageManagerImpl extends ManagerBase implements Secondar String nfsVersion = imageStoreDetailsUtil != null ? imageStoreDetailsUtil.getNfsVersion(secStore.getId()) : null; buf.append(" nfsVersion=").append(nfsVersion); + VirtualMachineGuru.appendCertificateDetails(buf, certificate); String bootArgs = buf.toString(); if (s_logger.isDebugEnabled()) { s_logger.debug(String.format("Boot args for machine profile [%s]: [%s].", profile.toString(), bootArgs)); diff --git a/systemvm/debian/opt/cloud/bin/setup/cloud-early-config b/systemvm/debian/opt/cloud/bin/setup/cloud-early-config index 5a50fe8..15da63e 100755 --- a/systemvm/debian/opt/cloud/bin/setup/cloud-early-config +++ b/systemvm/debian/opt/cloud/bin/setup/cloud-early-config @@ -55,7 +55,7 @@ patch() { eval $(validate_checksums $md5file $oldpatchfile) if [ "$oldmd5" == "$newmd5" ] && [ -d /usr/local/cloud/systemvm ] && [ "$(ls -A /usr/local/cloud/systemvm)" ]; then - log_it "Checksum matches, do need to patch" + log_it "Checksum matches, no need to patch" return 0 fi diff --git a/systemvm/debian/opt/cloud/bin/setup/common.sh b/systemvm/debian/opt/cloud/bin/setup/common.sh index 75c8f3c..be0b1c4 100755 --- a/systemvm/debian/opt/cloud/bin/setup/common.sh +++ b/systemvm/debian/opt/cloud/bin/setup/common.sh @@ -593,6 +593,17 @@ routing_svcs() { fi } +setup_certificates() { + certificate=$(echo "$CERTIFICATE" | base64 -d) + cacertificate=$(echo "$CACERTIFICATE" | base64 -d) + privatekey=$(echo "$PRIVATEKEY" | base64 -d) + kspass=$(echo "$KEYSTORE_PSSWD"| base64 -d) + ksvalidity="$KS_VALIDITY" + /opt/cloud/bin/keystore-cert-import /usr/local/cloud/systemvm/conf/agent.properties $kspass $ksvalidity \ + /usr/local/cloud/systemvm/conf/cloud.jks ssh /usr/local/cloud/systemvm/conf/cloud.crt \ + $certificate /usr/local/cloud/systemvm/conf/cloud.ca.crt $cacertificate /usr/local/cloud/systemvm/conf/cloud.key $privatekey +} + parse_cmd_line() { CMDLINE=$(cat /var/cache/cloud/cmdline) TYPE="unknown" @@ -766,7 +777,16 @@ parse_cmd_line() { export KEYSTORE_PSSWD=$VALUE ;; validity) - export VALIDITY=$VALUE + export KS_VALIDITY=$VALUE + ;; + certificate) + export CERTIFICATE=$VALUE + ;; + cacertificate) + export CACERTIFICATE=$VALUE + ;; + privatekey) + export PRIVATEKEY=$VALUE ;; esac done diff --git a/systemvm/debian/opt/cloud/bin/setup/consoleproxy.sh b/systemvm/debian/opt/cloud/bin/setup/consoleproxy.sh index ec45b7f..324f92e 100755 --- a/systemvm/debian/opt/cloud/bin/setup/consoleproxy.sh +++ b/systemvm/debian/opt/cloud/bin/setup/consoleproxy.sh @@ -36,6 +36,8 @@ setup_console_proxy() { enable_fwding 0 enable_irqbalance 0 rm -f /etc/logrotate.d/cloud + + setup_certificates } setup_console_proxy diff --git a/systemvm/debian/opt/cloud/bin/setup/secstorage.sh b/systemvm/debian/opt/cloud/bin/setup/secstorage.sh index 3b21ed5..a34e671 100755 --- a/systemvm/debian/opt/cloud/bin/setup/secstorage.sh +++ b/systemvm/debian/opt/cloud/bin/setup/secstorage.sh @@ -66,6 +66,7 @@ CORS setup_ntp rm -f /etc/logrotate.d/cloud + setup_certificates } setup_secstorage
