This is an automated email from the ASF dual-hosted git repository. pearl11594 pushed a commit to branch test-opt in repository https://gitbox.apache.org/repos/asf/cloudstack.git
commit ca4750d537cf4e0dc51041028e2f35f8718014c1 Author: Pearl Dsilva <[email protected]> AuthorDate: Tue Dec 21 14:24:32 2021 +0530 Commit comprises of: - remove docker from systemvm template - use containerd as container runtime - update create-k8s-binaries script to use ctr for all docker operations - Update userdata sent to the k8s nodes - update cksnode script, run during patching of the cks/k8s nodes --- .../kvm/resource/LibvirtComputingResource.java | 26 ++++--------------- .../main/resources/conf/k8s-control-node-add.yml | 10 ++++---- .../src/main/resources/conf/k8s-control-node.yml | 12 ++++----- .../src/main/resources/conf/k8s-node.yml | 10 ++++---- .../main/resources/script/upgrade-kubernetes.sh | 2 +- scripts/util/create-kubernetes-binaries-iso.sh | 22 ++++++++-------- .../consoleproxy/ConsoleProxyManagerImpl.java | 8 ++++++ systemvm/debian/opt/cloud/bin/setup/cksnode.sh | 9 ++++--- .../debian/opt/cloud/bin/setup/cloud-early-config | 29 +++++++++++++++------- systemvm/debian/opt/cloud/bin/setup/common.sh | 6 +++++ systemvm/patch-sysvms.sh | 3 ++- .../scripts/configure_systemvm_services.sh | 4 --- .../scripts/install_systemvm_packages.sh | 2 +- 13 files changed, 77 insertions(+), 66 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 0b2db81..8ab8396 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -1169,20 +1169,6 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv _storagePoolMgr = new KVMStoragePoolManager(_storage, _monitor); - _sysvmISOPath = (String)params.get("systemvm.iso.path"); - if (_sysvmISOPath == null) { - final String[] isoPaths = {"/usr/share/cloudstack-common/vms/systemvm.iso"}; - for (final String isoPath : isoPaths) { - if (_storage.exists(isoPath)) { - _sysvmISOPath = isoPath; - break; - } - } - if (_sysvmISOPath == null) { - s_logger.debug("Can't find system vm ISO"); - } - } - final Map<String, String> bridges = new HashMap<String, String>(); params.put("libvirt.host.bridges", bridges); @@ -2903,14 +2889,12 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv } if (vmSpec.getType() != VirtualMachine.Type.User) { - if (_sysvmISOPath != null) { - final DiskDef iso = new DiskDef(); - // iso.defISODisk(_sysvmISOPath); - if (_guestCpuArch != null && _guestCpuArch.equals("aarch64")) { - iso.setBusType(DiskDef.DiskBus.SCSI); - } - vm.getDevices().addDevice(iso); + final DiskDef iso = new DiskDef(); + iso.defISODisk(_sysvmISOPath); + if (_guestCpuArch != null && _guestCpuArch.equals("aarch64")) { + iso.setBusType(DiskDef.DiskBus.SCSI); } + vm.getDevices().addDevice(iso); } // For LXC, find and add the root filesystem, rbd data disks diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml index 0891d20..87c5924 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml @@ -118,7 +118,7 @@ write_files: fi retval=0 set +e - docker load < "${BINARIES_DIR}/docker/$line" + ctr image import "${BINARIES_DIR}/docker/$line" retval=$? set -e if [ $retval -eq 0 ]; then @@ -165,7 +165,7 @@ write_files: fi systemctl enable kubelet && systemctl start kubelet - modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1 + modprobe overlay && modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1 if [ -d "$BINARIES_DIR" ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then crucial_cmd_attempts=1 @@ -176,7 +176,7 @@ write_files: fi retval=0 set +e - kubeadm config images pull + kubeadm config images pull --cri-socket /run/containerd/containerd.sock retval=$? set -e if [ $retval -eq 0 ]; then @@ -218,8 +218,8 @@ write_files: owner: root:root content: | [Unit] - Requires=docker.service - After=docker.service + Requires=containerd.service + After=containerd.service [Service] Type=simple diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml index 322f080..55daf7f 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml @@ -138,7 +138,7 @@ write_files: fi retval=0 set +e - docker load < "${BINARIES_DIR}/docker/$line" + ctr image import "${BINARIES_DIR}/docker/$line" retval=$? set -e if [ $retval -eq 0 ]; then @@ -187,7 +187,7 @@ write_files: fi systemctl enable kubelet && systemctl start kubelet - modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1 + modprobe overlay && modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1 if [ -d "$BINARIES_DIR" ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then crucial_cmd_attempts=1 @@ -198,7 +198,7 @@ write_files: fi retval=0 set +e - kubeadm config images pull + kubeadm config images pull --cri-socket /run/containerd/containerd.sock retval=$? set -e if [ $retval -eq 0 ]; then @@ -216,7 +216,7 @@ write_files: fi retval=0 set +e - kubeadm init --token {{ k8s_control_node.cluster.token }} --token-ttl 0 {{ k8s_control_node.cluster.initargs }} + kubeadm init --token {{ k8s_control_node.cluster.token }} --token-ttl 0 {{ k8s_control_node.cluster.initargs }} --cri-socket /run/containerd/containerd.sock retval=$? set -e if [ $retval -eq 0 ]; then @@ -275,8 +275,8 @@ write_files: owner: root:root content: | [Unit] - Requires=docker.service - After=docker.service + Requires=containerd.service + After=containerd.service [Service] Type=simple diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml index 28ba43e..ac05c60 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml @@ -118,7 +118,7 @@ write_files: fi retval=0 set +e - docker load < "${BINARIES_DIR}/docker/$line" + ctr image import "${BINARIES_DIR}/docker/$line" retval=$? set -e if [ $retval -eq 0 ]; then @@ -165,7 +165,7 @@ write_files: fi systemctl enable kubelet && systemctl start kubelet - modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1 + modprobe overlay && modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1 if [ -d "$BINARIES_DIR" ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then crucial_cmd_attempts=1 @@ -176,7 +176,7 @@ write_files: fi retval=0 set +e - kubeadm config images pull + kubeadm config images pull --cri-socket /run/containerd/containerd.sock retval=$? set -e if [ $retval -eq 0 ]; then @@ -218,8 +218,8 @@ write_files: owner: root:root content: | [Unit] - Requires=docker.service - After=docker.service + Requires=containerd.service + After=containerd.service [Service] Type=simple diff --git a/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh b/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh index 7e0c3c0..b85ea00 100755 --- a/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh +++ b/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh @@ -93,7 +93,7 @@ if [ -d "$BINARIES_DIR" ]; then output=`ls ${BINARIES_DIR}/docker/` if [ "$output" != "" ]; then while read -r line; do - docker load < "${BINARIES_DIR}/docker/$line" + ctr image import "${BINARIES_DIR}/docker/$line" done <<< "$output" fi if [ -e "${BINARIES_DIR}/provider.yaml" ]; then diff --git a/scripts/util/create-kubernetes-binaries-iso.sh b/scripts/util/create-kubernetes-binaries-iso.sh index ba3dca7..ce7626c 100755 --- a/scripts/util/create-kubernetes-binaries-iso.sh +++ b/scripts/util/create-kubernetes-binaries-iso.sh @@ -98,19 +98,18 @@ provider_conf_file="${working_dir}/provider.yaml" curl -sSL ${PROVIDER_URL} -o ${provider_conf_file} echo "Fetching k8s docker images..." -docker -v +ctr -v if [ $? -ne 0 ]; then - echo "Installing docker..." + echo "Installing containerd..." if [ -f /etc/redhat-release ]; then sudo yum -y remove docker-common docker container-selinux docker-selinux docker-engine sudo yum -y install lvm2 device-mapper device-mapper-persistent-data device-mapper-event device-mapper-libs device-mapper-event-libs sudo yum install -y http://mirror.centos.org/centos/7/extras/x86_64/Packages/container-selinux-2.107-3.el7.noarch.rpm - sudo wget https://download.docker.com/linux/centos/docker-ce.repo -O /etc/yum.repos.d/docker-ce.repo && sudo yum -y install docker-ce - sudo systemctl enable docker && sudo systemctl start docker + sudo yum install -y containerd.io elif [ -f /etc/lsb-release ]; then - sudo apt update && sudo apt install docker.io -y - sudo systemctl enable docker && sudo systemctl start docker + sudo apt update && sudo apt install containerd.io -y fi + sudo systemctl enable containerd && sudo systemctl start containerd fi mkdir -p "${working_dir}/docker" output=`${k8s_dir}/kubeadm config images list --kubernetes-version=${RELEASE}` @@ -130,11 +129,14 @@ provider_image=`grep "image:" ${provider_conf_file} | cut -d ':' -f2- | tr -d ' output=`printf "%s\n" ${output} ${provider_image}` while read -r line; do - echo "Downloading docker image $line ---" - sudo docker pull "$line" + echo "Downloading image $line ---" + if [[ $line == kubernetesui* ]] || [[ $line == apache* ]]; then + line="docker.io/${line}" + fi + sudo ctr image pull "$line" image_name=`echo "$line" | grep -oE "[^/]+$"` - sudo docker save "$line" > "${working_dir}/docker/$image_name.tar" - sudo docker image rm "$line" + sudo ctr image export "${working_dir}/docker/$image_name.tar" "$line" + sudo ctr image rm "$line" done <<< "$output" echo "Restore kubeadm permissions..." diff --git a/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java b/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java index a3177fa..7434ea8 100644 --- a/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java +++ b/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java @@ -29,7 +29,9 @@ import java.util.Map; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.utils.PasswordGenerator; import org.apache.cloudstack.agent.lb.IndirectAgentLB; +import org.apache.cloudstack.ca.CAManager; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; import org.apache.cloudstack.framework.config.ConfigKey; @@ -221,6 +223,10 @@ public class ConsoleProxyManagerImpl extends ManagerBase implements ConsoleProxy private VirtualMachineManager virtualMachineManager; @Inject private IndirectAgentLB indirectAgentLB; + @Inject + private CAManager caManager; + @Inject + private NetworkOrchestrationService networkMgr; private ConsoleProxyListener consoleProxyListener; @@ -1274,6 +1280,8 @@ public class ConsoleProxyManagerImpl extends ManagerBase implements ConsoleProxy buf.append(" dns2=").append(dc.getDns2()); } + buf.append(" keystore_password=").append(PasswordGenerator.generateRandomPassword(16)); + buf.append(" validity=").append(CAManager.CertValidityPeriod.value()); String bootArgs = buf.toString(); if (s_logger.isDebugEnabled()) { s_logger.debug("Boot Args for " + profile + ": " + bootArgs); diff --git a/systemvm/debian/opt/cloud/bin/setup/cksnode.sh b/systemvm/debian/opt/cloud/bin/setup/cksnode.sh index 612fdd4..7f580ac 100755 --- a/systemvm/debian/opt/cloud/bin/setup/cksnode.sh +++ b/systemvm/debian/opt/cloud/bin/setup/cksnode.sh @@ -28,18 +28,23 @@ setup_k8s_node() { # set default ssh port and restart sshd service sed -i 's/3922/22/g' /etc/ssh/sshd_config + systemctl restart ssh # Prevent root login > /root/.ssh/authorized_keys passwd -l root #sed -i 's#root:x:0:0:root:/root:/bin/bash#root:x:0:0:root:/root:/sbin/nologin#' /etc/passwd + # Update containerd configuration + mkdir -p /etc/containerd + containerd config default>/etc/containerd/config.toml + systemctl restart containerd + swapoff -a sudo sed -i '/ swap / s/^/#/' /etc/fstab log_it "Swap disabled" log_it "Setting up interfaces" -# setup_common eth0 setup_system_rfc1918_internal log_it "Setting up entry in hosts" @@ -61,8 +66,6 @@ setup_k8s_node() { log_it "Starting cloud-init services" systemctl enable --now --no-block containerd - systemctl enable --now --no-block docker.socket - systemctl enable --now --no-block docker.service if [ -f /home/core/success ]; then systemctl stop cloud-init cloud-config cloud-final systemctl disable cloud-init cloud-config cloud-final diff --git a/systemvm/debian/opt/cloud/bin/setup/cloud-early-config b/systemvm/debian/opt/cloud/bin/setup/cloud-early-config index 9695b18..370dfc0 100755 --- a/systemvm/debian/opt/cloud/bin/setup/cloud-early-config +++ b/systemvm/debian/opt/cloud/bin/setup/cloud-early-config @@ -31,29 +31,40 @@ log_it() { log_action_msg "$@" } +validate_checksums() { + local oldmd5= + [ -f ${1} ] && oldmd5=$(cat ${1}) + local newmd5= + [ -f ${2} ] && newmd5=$(md5sum ${2} | awk '{print $1}') + log_it "Scripts checksum detected: oldmd5=$oldmd5 newmd5=$newmd5" >> /dev/null 2>&1 + echo "oldmd5='${oldmd5}'; newmd5='${newmd5}'" +} + patch() { local PATCH_MOUNT=/home/cloud - local patchfile=$PATCH_MOUNT/cloud-scripts.tgz + local PATCH_SCRIPTS=cloud-scripts.tgz + local oldpatchfile=/usr/share/cloud/$PATCH_SCRIPTS + local patchfile=$PATCH_MOUNT/$PATCH_SCRIPTS local privkey=$PATCH_MOUNT/authorized_keys local md5file=/var/cache/cloud/cloud-scripts-signature - local cdrom_dev= mkdir -p $PATCH_MOUNT if [ -f /var/cache/cloud/authorized_keys ]; then privkey=/var/cache/cloud/authorized_keys fi + eval $(validate_checksums $md5file $oldpatchfile) + if [ "$oldmd5" == "$newmd5" ] && [ ! -f ${patchfile} ]; then + log_it "Checksum matches, do need to patch" + return 0 + fi + retry=60 local patched=false while [ $retry -gt 0 ] do if [ -f $patchfile ]; then - local oldmd5= - [ -f ${md5file} ] && oldmd5=$(cat ${md5file}) - local newmd5= - [ -f ${patchfile} ] && newmd5=$(md5sum ${patchfile} | awk '{print $1}') - log_it "Scripts checksum detected: oldmd5=$oldmd5 newmd5=$newmd5" - log_it ls -lrt $PATCH_MOUNT + eval $(validate_checksums $md5file $patchfile) if [ "$oldmd5" != "$newmd5" ] && [ -f ${patchfile} ] && [ "$newmd5" != "" ] then tar xzf $patchfile -C / @@ -84,7 +95,7 @@ patch() { cleanup() { rm -rf /home/cloud/agent.zip - rm -rf /home/cloud/cloud-scripts.tgz + mv /home/cloud/cloud-scripts.tgz /usr/share/cloud/cloud-scripts.tgz } start() { diff --git a/systemvm/debian/opt/cloud/bin/setup/common.sh b/systemvm/debian/opt/cloud/bin/setup/common.sh index 7f3d857..75c8f3c 100755 --- a/systemvm/debian/opt/cloud/bin/setup/common.sh +++ b/systemvm/debian/opt/cloud/bin/setup/common.sh @@ -762,6 +762,12 @@ parse_cmd_line() { authorized_key) export AUTHORIZED_KEYS=$VALUE ;; + keystore_password) + export KEYSTORE_PSSWD=$VALUE + ;; + validity) + export VALIDITY=$VALUE + ;; esac done echo -e "\n\t}\n}" >> ${CHEF_TMP_FILE} diff --git a/systemvm/patch-sysvms.sh b/systemvm/patch-sysvms.sh index cf0b452..31478d7 100644 --- a/systemvm/patch-sysvms.sh +++ b/systemvm/patch-sysvms.sh @@ -76,7 +76,8 @@ restart_services() { cleanup_systemVM() { rm -rf $backupfolder - rm -rf "$newpath""cloud-scripts.tgz" "$newpath""agent.zip" "$newpath""patch-sysvms.sh" + mv "$newpath"cloud-scripts.tgz /usr/share/cloud/cloud-scripts.tgz + rm -rf "$newpath""agent.zip" "$newpath""patch-sysvms.sh" } patch_systemvm() { diff --git a/tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh b/tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh index 8cdfce7..4f97b1b 100644 --- a/tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh +++ b/tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh @@ -127,10 +127,6 @@ function configure_services() { # Disable container services systemctl disable containerd - systemctl disable docker.service - systemctl stop docker.service - systemctl disable docker.socket - systemctl stop docker.socket # Disable cloud init by default cat <<EOF > /etc/cloud/cloud.cfg.d/cloudstack.cfg diff --git a/tools/appliance/systemvmtemplate/scripts/install_systemvm_packages.sh b/tools/appliance/systemvmtemplate/scripts/install_systemvm_packages.sh index 555a00c..3b4ef65 100644 --- a/tools/appliance/systemvmtemplate/scripts/install_systemvm_packages.sh +++ b/tools/appliance/systemvmtemplate/scripts/install_systemvm_packages.sh @@ -98,7 +98,7 @@ function install_packages() { apt-key fingerprint 0EBFCD88 add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian $(lsb_release -cs) stable" apt-get update - ${apt_get} install docker-ce docker-ce-cli containerd.io + ${apt_get} install containerd.io apt_clean
