This is an automated email from the ASF dual-hosted git repository. pearl11594 pushed a commit to branch test-opt in repository https://gitbox.apache.org/repos/asf/cloudstack.git
commit 80915815f78a7f40fb0819a1a59e81fb10b5c052 Author: Pearl Dsilva <[email protected]> AuthorDate: Tue Dec 7 13:09:16 2021 +0530 Support for live patching systemVMs and deprecating systemVM.iso. Includes: - fix systemVM template version - Include agent.zip, cloud-scripts.tgz to the commons package - Support for live-patching systemVMs - CPVM, SSVM, Routers - Fix Unit test - Remove systemvm.iso dependency --- api/src/main/java/com/cloud/event/EventTypes.java | 4 + .../java/com/cloud/server/ManagementService.java | 2 + .../command/admin/systemvm/PatchSystemVMCmd.java | 107 +++++++++++++++ .../com/cloud/agent/api/PatchSystemVmAnswer.java | 44 ++++++ .../com/cloud/agent/api/PatchSystemVmCommand.java | 29 ++++ debian/rules | 3 +- engine/schema/pom.xml | 9 +- .../com/cloud/upgrade/DatabaseUpgradeChecker.java | 9 +- .../upgrade/SystemVmTemplateRegistration.java | 22 +-- .../resources/META-INF/db/schema-41600to41610.sql | 4 +- engine/schema/templateConfig.sh | 20 +-- packaging/centos7/cloud.spec | 7 +- packaging/centos8/cloud.spec | 7 +- packaging/suse15/cloud.spec | 7 +- .../kvm/resource/LibvirtComputingResource.java | 10 +- .../LibvirtPatchSystemVmCommandWrapper.java | 132 ++++++++++++++++++ .../wrapper/LibvirtStartCommandWrapper.java | 21 +++ .../kvm/resource/LibvirtComputingResourceTest.java | 11 +- pom.xml | 1 + scripts/vm/hypervisor/xenserver/xcpserver/patch | 2 + .../com/cloud/api/ResponseObjectTypeAdapter.java | 4 + .../com/cloud/server/ManagementServerImpl.java | 90 ++++++++++++- .../etc/systemd/system/cloud-early-config.service | 7 +- .../etc/systemd/system/cloud-postinit.service | 2 +- ...-early-config.service => cloud-preinit.service} | 4 +- systemvm/debian/opt/cloud/bin/setup/bootstrap.sh | 134 +------------------ systemvm/debian/opt/cloud/bin/setup/cksnode.sh | 2 +- .../debian/opt/cloud/bin/setup/cloud-early-config | 73 +++++----- systemvm/debian/opt/cloud/bin/setup/common.sh | 2 +- .../debian/opt/cloud/bin/setup/consoleproxy.sh | 12 -- systemvm/debian/opt/cloud/bin/setup/dhcpsrvr.sh | 16 +-- systemvm/debian/opt/cloud/bin/setup/elbvm.sh | 9 -- systemvm/debian/opt/cloud/bin/setup/ilbvm.sh | 2 +- .../opt/cloud/bin/setup/{bootstrap.sh => init.sh} | 147 +++++++++------------ systemvm/debian/opt/cloud/bin/setup/postinit.sh | 16 +-- systemvm/debian/opt/cloud/bin/setup/router.sh | 19 --- systemvm/debian/opt/cloud/bin/setup/secstorage.sh | 9 -- systemvm/debian/opt/cloud/bin/setup/vpcrouter.sh | 3 +- systemvm/patch-sysvms.sh | 108 +++++++++++++++ systemvm/pom.xml | 6 + tools/appliance/build.sh | 8 +- .../scripts/configure_systemvm_services.sh | 1 + tools/appliance/systemvmtemplate/template.json | 4 +- .../main/java/com/cloud/utils/EncryptionUtil.java | 16 +++ .../main/java/com/cloud/utils/ssh/SshHelper.java | 42 ++++++ 45 files changed, 803 insertions(+), 384 deletions(-) diff --git a/api/src/main/java/com/cloud/event/EventTypes.java b/api/src/main/java/com/cloud/event/EventTypes.java index 7533e58..16aaba0 100644 --- a/api/src/main/java/com/cloud/event/EventTypes.java +++ b/api/src/main/java/com/cloud/event/EventTypes.java @@ -635,6 +635,9 @@ public class EventTypes { // Storage Policies public static final String EVENT_IMPORT_VCENTER_STORAGE_POLICIES = "IMPORT.VCENTER.STORAGE.POLICIES"; + // SystemVM + public static final String EVENT_LIVE_PATCH_SYSTEMVM = "LIVE.PATCH.SYSTEM.VM"; + static { // TODO: need a way to force author adding event types to declare the entity details as well, with out braking @@ -1045,6 +1048,7 @@ public class EventTypes { entityEventDetails.put(EVENT_IMPORT_VCENTER_STORAGE_POLICIES, "StoragePolicies"); entityEventDetails.put(EVENT_IMAGE_STORE_DATA_MIGRATE, ImageStore.class); + entityEventDetails.put(EVENT_LIVE_PATCH_SYSTEMVM, "SystemVMs"); } public static String getEntityForEvent(String eventName) { diff --git a/api/src/main/java/com/cloud/server/ManagementService.java b/api/src/main/java/com/cloud/server/ManagementService.java index 56f36a8..9c09833 100644 --- a/api/src/main/java/com/cloud/server/ManagementService.java +++ b/api/src/main/java/com/cloud/server/ManagementService.java @@ -39,6 +39,7 @@ import org.apache.cloudstack.api.command.admin.resource.ListCapacityCmd; import org.apache.cloudstack.api.command.admin.resource.UploadCustomCertificateCmd; import org.apache.cloudstack.api.command.admin.systemvm.DestroySystemVmCmd; import org.apache.cloudstack.api.command.admin.systemvm.ListSystemVMsCmd; +import org.apache.cloudstack.api.command.admin.systemvm.PatchSystemVMCmd; import org.apache.cloudstack.api.command.admin.systemvm.RebootSystemVmCmd; import org.apache.cloudstack.api.command.admin.systemvm.ScaleSystemVMCmd; import org.apache.cloudstack.api.command.admin.systemvm.StopSystemVmCmd; @@ -424,5 +425,6 @@ public interface ManagementService { void cleanupVMReservations(); + Pair<Boolean, String> patchSystemVM(PatchSystemVMCmd cmd); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/systemvm/PatchSystemVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/systemvm/PatchSystemVMCmd.java new file mode 100644 index 0000000..481e62c --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/systemvm/PatchSystemVMCmd.java @@ -0,0 +1,107 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.systemvm; + +import com.cloud.event.EventTypes; +import com.cloud.user.Account; +import com.cloud.utils.Pair; +import com.cloud.vm.VirtualMachine; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.cloudstack.api.response.SystemVmResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.log4j.Logger; + +@APICommand(name = "patchSystemVm", description = "Attempts to live patch systemVMs - CPVM, SSVM, Routers ", + responseObject = SuccessResponse.class, requestHasSensitiveInfo = false, + responseHasSensitiveInfo = false, authorized = { RoleType.Admin }) +public class PatchSystemVMCmd extends BaseAsyncCmd { + public static final Logger s_logger = Logger.getLogger(PatchSystemVMCmd.class.getName()); + private static final String s_name = "patchsystemvmresponse"; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = SystemVmResponse.class, + description = "patches systemVM - CPVM/SSVM/Router with the specified ID") + private Long id; + + @Parameter(name = ApiConstants.FORCED, type = CommandType.BOOLEAN, + description = "If true, initiates copy of scripts and restart of the agent if if the template version is the latest." + + "To be used with ID parameter only") + private Boolean force; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + + public Long getId() { + return id; + } + + public boolean isForced() { + return force != null ? force : false; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public String getEventType() { + return EventTypes.EVENT_LIVE_PATCH_SYSTEMVM; + } + + @Override + public String getEventDescription() { + return String.format("Attempting to live patch System VM with Id: %s ", this._uuidMgr.getUuid(VirtualMachine.class, getId())); + } + + @Override + public String getCommandName() { + return s_name; + } + + @Override + public long getEntityOwnerId() { + Account account = CallContext.current().getCallingAccount(); + if (account != null) { + return account.getId(); + } + + return Account.ACCOUNT_ID_SYSTEM; + } + + @Override + public void execute() { + Pair<Boolean, String> patched = _mgr.patchSystemVM(this); + if (patched.first()) { + final SuccessResponse response = new SuccessResponse(getCommandName()); + response.setDisplayText(patched.second()); + setResponseObject(response); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, patched.second()); + } + } +} diff --git a/core/src/main/java/com/cloud/agent/api/PatchSystemVmAnswer.java b/core/src/main/java/com/cloud/agent/api/PatchSystemVmAnswer.java new file mode 100644 index 0000000..1065768 --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/PatchSystemVmAnswer.java @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.api; + +public class PatchSystemVmAnswer extends Answer { + + String templateVersion; + String scriptsVersion; + + public PatchSystemVmAnswer() { + } + + public PatchSystemVmAnswer(PatchSystemVmCommand cmd, String details, String templateVersion, String scriptsVersion) { + super(cmd, true, details); + this.templateVersion = templateVersion; + this.scriptsVersion = scriptsVersion; + } + + public PatchSystemVmAnswer(PatchSystemVmCommand cmd, String details) { + super(cmd, false, details); + } + + public String getTemplateVersion() { + return this.templateVersion; + } + + public String getScriptsVersion() { + return this.scriptsVersion; + } +} diff --git a/core/src/main/java/com/cloud/agent/api/PatchSystemVmCommand.java b/core/src/main/java/com/cloud/agent/api/PatchSystemVmCommand.java new file mode 100644 index 0000000..b9ea2d6 --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/PatchSystemVmCommand.java @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.api; + +public class PatchSystemVmCommand extends GetDomRVersionCmd { + boolean forced; + + public boolean isForced() { + return forced; + } + + public void setForced(boolean forced) { + this.forced = forced; + } +} diff --git a/debian/rules b/debian/rules index ed1559a..1e7f119 100755 --- a/debian/rules +++ b/debian/rules @@ -128,7 +128,8 @@ override_dh_auto_install: install -D client/target/utilities/bin/cloud-setup-management $(DESTDIR)/usr/bin/cloudstack-setup-management install -D client/target/utilities/bin/cloud-setup-encryption $(DESTDIR)/usr/bin/cloudstack-setup-encryption install -D client/target/utilities/bin/cloud-sysvmadm $(DESTDIR)/usr/bin/cloudstack-sysvmadm - install -D systemvm/dist/systemvm.iso $(DESTDIR)/usr/share/$(PACKAGE)-common/vms/systemvm.iso + #install -D systemvm/dist/systemvm.iso $(DESTDIR)/usr/share/$(PACKAGE)-common/vms/systemvm.iso + install -D systemvm/dist/* $(DESTDIR)/usr/share/$(PACKAGE)-common/vms/ # We need jasypt for cloud-install-sys-tmplt, so this is a nasty hack to get it into the right place install -D agent/target/dependencies/jasypt-1.9.3.jar $(DESTDIR)/usr/share/$(PACKAGE)-common/lib diff --git a/engine/schema/pom.xml b/engine/schema/pom.xml index 8675352..3b1bba2 100644 --- a/engine/schema/pom.xml +++ b/engine/schema/pom.xml @@ -73,10 +73,11 @@ </goals> <configuration> <source> - def projectVersion = project.version + def projectVersion = project.properties['project.systemvm.template.version'] + println(projectVersion) String[] versionParts = projectVersion.tokenize('.') - pom.properties['cs.version'] = "4.16" - pom.properties['patch.version'] = "0" + pom.properties['cs.version'] = versionParts[0] + "." + versionParts[1] + pom.properties['patch.version'] = versionParts[2] </source> </configuration> </execution> @@ -146,7 +147,7 @@ <executable>bash</executable> <arguments> <argument>templateConfig.sh</argument> - <armument>${project.version}</armument> + <armument>${project.systemvm.template.version}</armument> </arguments> </configuration> </execution> diff --git a/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java b/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java index cf3f728..3c6abbc 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java @@ -367,10 +367,11 @@ public class DatabaseUpgradeChecker implements SystemIntegrityChecker { return; } - SystemVmTemplateRegistration.parseMetadataFile(); - final CloudStackVersion currentVersion = CloudStackVersion.parse(currentVersionValue); - SystemVmTemplateRegistration.CS_MAJOR_VERSION = String.valueOf(currentVersion.getMajorRelease()) + "." + String.valueOf(currentVersion.getMinorRelease()); - SystemVmTemplateRegistration.CS_TINY_VERSION = String.valueOf(currentVersion.getPatchRelease()); + String csVersion = SystemVmTemplateRegistration.parseMetadataFile(); + final CloudStackVersion sysVmVersion = CloudStackVersion.parse(csVersion); + final CloudStackVersion currentVersion = CloudStackVersion.parse(currentVersionValue); + SystemVmTemplateRegistration.CS_MAJOR_VERSION = String.valueOf(sysVmVersion.getMajorRelease()) + "." + String.valueOf(sysVmVersion.getMinorRelease()); + SystemVmTemplateRegistration.CS_TINY_VERSION = String.valueOf(sysVmVersion.getPatchRelease()); s_logger.info("DB version = " + dbVersion + " Code Version = " + currentVersion); diff --git a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java index d788ada..4119b11 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java @@ -36,6 +36,7 @@ import com.cloud.template.VirtualMachineTemplate; import com.cloud.upgrade.dao.BasicTemplateDataStoreDaoImpl; import com.cloud.user.Account; import com.cloud.utils.DateUtil; +import com.cloud.utils.EncryptionUtil; import com.cloud.utils.Pair; import com.cloud.utils.UriUtils; import com.cloud.utils.db.GlobalLock; @@ -54,7 +55,6 @@ import org.apache.cloudstack.storage.datastore.db.ImageStoreDaoImpl; import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; -import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; import org.ini4j.Ini; @@ -64,7 +64,6 @@ import java.io.BufferedReader; import java.io.File; import java.io.FileReader; import java.io.IOException; -import java.io.InputStream; import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; @@ -351,16 +350,6 @@ public class SystemVmTemplateRegistration { } } - private String calculateChecksum(File file) { - try (InputStream is = Files.newInputStream(Paths.get(file.getPath()))) { - return DigestUtils.md5Hex(is); - } catch (IOException e) { - String errMsg = "Failed to calculate template checksum"; - LOGGER.error(errMsg, e); - throw new CloudRuntimeException(errMsg, e); - } - } - public Long getRegisteredTemplateId(Pair<Hypervisor.HypervisorType, String> hypervisorAndTemplateName) { VMTemplateVO vmTemplate = vmTemplateDao.findLatestTemplateByName(hypervisorAndTemplateName.second()); Long templateId = null; @@ -690,7 +679,7 @@ public class SystemVmTemplateRegistration { } } - public static void parseMetadataFile() { + public static String parseMetadataFile() { try { Ini ini = new Ini(); ini.load(new FileReader(METADATA_FILE)); @@ -702,6 +691,8 @@ public class SystemVmTemplateRegistration { NewTemplateChecksum.put(hypervisorType, section.get("checksum")); NewTemplateUrl.put(hypervisorType, section.get("downloadurl")); } + Ini.Section section = ini.get("default"); + return section.get("version"); } catch (Exception e) { String errMsg = String.format("Failed to parse systemVM template metadata file: %s", METADATA_FILE); LOGGER.error(errMsg, e); @@ -735,7 +726,7 @@ public class SystemVmTemplateRegistration { } File tempFile = new File(TEMPLATES_PATH + matchedTemplate); - String templateChecksum = calculateChecksum(tempFile); + String templateChecksum = EncryptionUtil.calculateChecksum(tempFile); if (!templateChecksum.equals(NewTemplateChecksum.get(getHypervisorType(hypervisor)))) { LOGGER.error(String.format("Checksum mismatch: %s != %s ", templateChecksum, NewTemplateChecksum.get(getHypervisorType(hypervisor)))); templatesFound = false; @@ -812,9 +803,6 @@ public class SystemVmTemplateRegistration { private void updateRegisteredTemplateDetails(Long templateId, Map.Entry<Hypervisor.HypervisorType, String> hypervisorAndTemplateName) { VMTemplateVO templateVO = vmTemplateDao.findById(templateId); templateVO.setTemplateType(Storage.TemplateType.SYSTEM); - if (Hypervisor.HypervisorType.VMware == templateVO.getHypervisorType()) { - templateVO.setDeployAsIs(true); - } boolean updated = vmTemplateDao.update(templateVO.getId(), templateVO); if (!updated) { String errMsg = String.format("updateSystemVmTemplates:Exception while updating template with id %s to be marked as 'system'", templateId); diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41600to41610.sql b/engine/schema/src/main/resources/META-INF/db/schema-41600to41610.sql index 24c5b79..2fd0e5e 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41600to41610.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41600to41610.sql @@ -17,4 +17,6 @@ --; -- Schema upgrade from 4.16.0.0 to 4.16.1.0 ---; \ No newline at end of file +--; + +UPDATE `cloud`.`vm_template` set deploy_as_is = 0 where id = 8; \ No newline at end of file diff --git a/engine/schema/templateConfig.sh b/engine/schema/templateConfig.sh index c309353..891c73d 100644 --- a/engine/schema/templateConfig.sh +++ b/engine/schema/templateConfig.sh @@ -23,8 +23,10 @@ function getTemplateVersion() { subversion1="$(cut -d'.' -f1 <<<"$version")" subversion2="$(cut -d'.' -f2 <<<"$version")" minorversion="$(cut -d'.' -f3 <<<"$version")" + securityversion="$(cut -d'.' -f4 <<<"$version")" export CS_VERSION="${subversion1}"."${subversion2}" export CS_MINOR_VERSION="${minorversion}" + export VERSION="${CS_VERSION}.${CS_MINOR_VERSION}" } function getGenericName() { @@ -52,12 +54,14 @@ function getChecksum() { function createMetadataFile() { local fileData=$(cat $SOURCEFILE) + echo -e "["default"]\nversion = $VERSION.${securityversion}\n" >> $METADATAFILE for i in "${!templates[@]}" do section="$i" hvName=$(getGenericName $i) - templatename="systemvm-${i}-${CS_VERSION}.${CS_MINOR_VERSION}" - checksum=$(getChecksum "$fileData" $hvName) + + templatename="systemvm-${i}-${VERSION}" + checksum=$(getChecksum "$fileData" "$VERSION-$hvName") downloadurl="${templates[$i]}" filename=$(echo ${downloadurl##*'/'}) echo -e "["$section"]\ntemplatename = $templatename\nchecksum = $checksum\ndownloadurl = $downloadurl\nfilename = $filename\n" >> $METADATAFILE @@ -66,12 +70,12 @@ function createMetadataFile() { declare -A templates getTemplateVersion $1 -templates=( ["kvm"]="https://download.cloudstack.org/systemvm/${CS_VERSION}/systemvmtemplate-${CS_VERSION}.${CS_MINOR_VERSION}-kvm.qcow2.bz2"; - ["vmware"]="https://download.cloudstack.org/systemvm/${CS_VERSION}/systemvmtemplate-${CS_VERSION}.${CS_MINOR_VERSION}-vmware.ova"; - ["xenserver"]="https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$CS_VERSION.$CS_MINOR_VERSION-xen.vhd.bz2"; - ["hyperv"]="https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$CS_VERSION.$CS_MINOR_VERSION-hyperv.vhd.zip"; - ["lxc"]="https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$CS_VERSION.$CS_MINOR_VERSION-kvm.qcow2.bz2"; - ["ovm3"]="https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$CS_VERSION.$CS_MINOR_VERSION-ovm.raw.bz2"; ) +templates=( ["kvm"]="https://download.cloudstack.org/systemvm/${CS_VERSION}/systemvmtemplate-$VERSION-kvm.qcow2.bz2"; + ["vmware"]="https://download.cloudstack.org/systemvm/${CS_VERSION}/systemvmtemplate-$VERSION-vmware.ova"; + ["xenserver"]="https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-xen.vhd.bz2"; + ["hyperv"]="https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-hyperv.vhd.zip"; + ["lxc"]="https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-kvm.qcow2.bz2"; + ["ovm3"]="https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-ovm.raw.bz2"; ) PARENTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )/dist/systemvm-templates/" diff --git a/packaging/centos7/cloud.spec b/packaging/centos7/cloud.spec index 0728f58..a932c94 100644 --- a/packaging/centos7/cloud.spec +++ b/packaging/centos7/cloud.spec @@ -231,7 +231,8 @@ mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/vms mkdir -p ${RPM_BUILD_ROOT}%{python_sitearch}/ mkdir -p ${RPM_BUILD_ROOT}/usr/bin cp -r scripts/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/scripts -install -D systemvm/dist/systemvm.iso ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/vms/systemvm.iso +#install -D systemvm/dist/systemvm.iso ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/vms/systemvm.iso +install -D systemvm/dist/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/vms/ install python/lib/cloud_utils.py ${RPM_BUILD_ROOT}%{python_sitearch}/cloud_utils.py cp -r python/lib/cloudutils ${RPM_BUILD_ROOT}%{python_sitearch}/ python3 -m py_compile ${RPM_BUILD_ROOT}%{python_sitearch}/cloud_utils.py @@ -600,7 +601,11 @@ pip3 install --upgrade urllib3 %dir %attr(0755,root,root) %{_datadir}/%{name}-common/vms %attr(0755,root,root) %{_datadir}/%{name}-common/scripts %attr(0755,root,root) /usr/bin/cloudstack-sccs +# TODO: Remove systemvm.iso %attr(0644, root, root) %{_datadir}/%{name}-common/vms/systemvm.iso +%attr(0644, root, root) %{_datadir}/%{name}-common/vms/agent.zip +%attr(0644, root, root) %{_datadir}/%{name}-common/vms/cloud-scripts.tgz +%attr(0644, root, root) %{_datadir}/%{name}-common/vms/patch-sysvms.sh %attr(0644,root,root) %{python_sitearch}/cloud_utils.py %attr(0644,root,root) %{python_sitearch}/__pycache__/* %attr(0644,root,root) %{python_sitearch}/cloudutils/* diff --git a/packaging/centos8/cloud.spec b/packaging/centos8/cloud.spec index 31d85dd..d19e4fd 100644 --- a/packaging/centos8/cloud.spec +++ b/packaging/centos8/cloud.spec @@ -224,7 +224,8 @@ mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/vms mkdir -p ${RPM_BUILD_ROOT}%{python_sitearch}/ mkdir -p ${RPM_BUILD_ROOT}/usr/bin cp -r scripts/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/scripts -install -D systemvm/dist/systemvm.iso ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/vms/systemvm.iso +#install -D systemvm/dist/systemvm.iso ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/vms/systemvm.iso +install -D systemvm/dist/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/vms/ install python/lib/cloud_utils.py ${RPM_BUILD_ROOT}%{python_sitearch}/cloud_utils.py cp -r python/lib/cloudutils ${RPM_BUILD_ROOT}%{python_sitearch}/ python3 -m py_compile ${RPM_BUILD_ROOT}%{python_sitearch}/cloud_utils.py @@ -588,7 +589,11 @@ pip install --upgrade /usr/share/cloudstack-marvin/Marvin-*.tar.gz %dir %attr(0755,root,root) %{_datadir}/%{name}-common/vms %attr(0755,root,root) %{_datadir}/%{name}-common/scripts %attr(0755,root,root) /usr/bin/cloudstack-sccs +# TODO: Remove systemvm.iso %attr(0644, root, root) %{_datadir}/%{name}-common/vms/systemvm.iso +%attr(0644, root, root) %{_datadir}/%{name}-common/vms/agent.zip +%attr(0644, root, root) %{_datadir}/%{name}-common/vms/cloud-scripts.tgz +%attr(0644, root, root) %{_datadir}/%{name}-common/vms/patch-sysvms.sh %attr(0644,root,root) %{python_sitearch}/cloud_utils.py %attr(0644,root,root) %{python_sitearch}/__pycache__/* %attr(0644,root,root) %{python_sitearch}/cloudutils/* diff --git a/packaging/suse15/cloud.spec b/packaging/suse15/cloud.spec index 30300c6..cc75a96 100644 --- a/packaging/suse15/cloud.spec +++ b/packaging/suse15/cloud.spec @@ -226,7 +226,8 @@ mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/vms mkdir -p ${RPM_BUILD_ROOT}%{python_sitearch}/ mkdir -p ${RPM_BUILD_ROOT}/usr/bin cp -r scripts/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/scripts -install -D systemvm/dist/systemvm.iso ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/vms/systemvm.iso +#install -D systemvm/dist/systemvm.iso ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/vms/systemvm.iso +install -D systemvm/dist/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/vms/ install python/lib/cloud_utils.py ${RPM_BUILD_ROOT}%{python_sitearch}/cloud_utils.py cp -r python/lib/cloudutils ${RPM_BUILD_ROOT}%{python_sitearch}/ python3 -m py_compile ${RPM_BUILD_ROOT}%{python_sitearch}/cloud_utils.py @@ -582,7 +583,11 @@ pip install --upgrade /usr/share/cloudstack-marvin/Marvin-*.tar.gz %dir %attr(0755,root,root) %{_datadir}/%{name}-common/vms %attr(0755,root,root) %{_datadir}/%{name}-common/scripts %attr(0755,root,root) /usr/bin/cloudstack-sccs +# TODO: Remove systemvm.iso %attr(0644, root, root) %{_datadir}/%{name}-common/vms/systemvm.iso +%attr(0644, root, root) %{_datadir}/%{name}-common/vms/agent.zip +%attr(0644, root, root) %{_datadir}/%{name}-common/vms/cloud-scripts.tgz +%attr(0644, root, root) %{_datadir}/%{name}-common/vms/patch-sysvms.sh %attr(0644,root,root) %{python_sitearch}/cloud_utils.py %attr(0644,root,root) %{python_sitearch}/__pycache__/* %attr(0644,root,root) %{python_sitearch}/cloudutils/* diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 9684b7e..4a3f778 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -280,6 +280,10 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv private static final String AARCH64 = "aarch64"; public static final String RESIZE_NOTIFY_ONLY = "NOTIFYONLY"; + public static final String BASEPATH = "/usr/share/cloudstack-common/vms/"; + + public static String[] srcFiles = new String[] { "agent.zip", "cloud-scripts.tgz" }; + public static String[] newSrcFiles = new String[] { "agent.zip", "cloud-scripts.tgz", "patch-sysvms.sh" }; private String _modifyVlanPath; private String _versionstringpath; @@ -403,7 +407,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv s_powerStatesTable.put(DomainState.VIR_DOMAIN_SHUTDOWN, PowerState.PowerOff); } - private VirtualRoutingResource _virtRouterResource; + public VirtualRoutingResource _virtRouterResource; private String _pingTestPath; @@ -463,7 +467,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv try { SshHelper.scpTo(routerIp, 3922, "root", permKey, null, path, content.getBytes(), filename, null); } catch (final Exception e) { - s_logger.warn("Fail to create file " + path + filename + " in VR " + routerIp, e); + s_logger.warn("Failed to create file " + path + filename + " in VR " + routerIp, e); details = e.getMessage(); success = false; } @@ -2906,7 +2910,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv if (vmSpec.getType() != VirtualMachine.Type.User) { if (_sysvmISOPath != null) { final DiskDef iso = new DiskDef(); - iso.defISODisk(_sysvmISOPath); + // iso.defISODisk(_sysvmISOPath); if (_guestCpuArch != null && _guestCpuArch.equals("aarch64")) { iso.setBusType(DiskDef.DiskBus.SCSI); } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPatchSystemVmCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPatchSystemVmCommandWrapper.java new file mode 100644 index 0000000..392ccb6 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPatchSystemVmCommandWrapper.java @@ -0,0 +1,132 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.hypervisor.kvm.resource.wrapper; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.PatchSystemVmAnswer; +import com.cloud.agent.api.PatchSystemVmCommand; +import com.cloud.agent.api.routing.NetworkElementCommand; +import com.cloud.agent.resource.virtualnetwork.VRScripts; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.utils.EncryptionUtil; +import com.cloud.utils.ExecutionResult; +import com.cloud.utils.Pair; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.script.Script; +import com.cloud.utils.ssh.SshHelper; +import org.apache.commons.lang3.StringUtils; +import org.apache.log4j.Logger; + +import java.io.File; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +@ResourceWrapper(handles = PatchSystemVmCommand.class) +public class LibvirtPatchSystemVmCommandWrapper extends CommandWrapper<PatchSystemVmCommand, Answer, LibvirtComputingResource> { + private static final Logger s_logger = Logger.getLogger(LibvirtPatchSystemVmCommandWrapper.class); + private static int sshPort = Integer.parseInt(LibvirtComputingResource.DEFAULTDOMRSSHPORT); + private static File pemFile = new File(LibvirtComputingResource.SSHPRVKEYPATH); + + @Override + public Answer execute(PatchSystemVmCommand cmd, LibvirtComputingResource serverResource) { + final String controlIp = cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP); + final String sysVMName = cmd.getAccessDetail(NetworkElementCommand.ROUTER_NAME); + ExecutionResult result; + try { + result = getSystemVmVersionAndChecksum(serverResource, controlIp); + scpPatchFiles(controlIp); + } catch (CloudRuntimeException e) { + return new PatchSystemVmAnswer(cmd, e.getMessage()); + } + + final String[] lines = result.getDetails().split("&"); + // TODO: do we fail, or patch anyway?? + if (lines.length != 2) { + return new PatchSystemVmAnswer(cmd, result.getDetails()); + } + + String scriptChecksum = lines[1].trim(); + String checksum = calculateCurrentChecksum(sysVMName).trim(); + + if (!StringUtils.isEmpty(checksum) && checksum.equals(scriptChecksum)) { + if (!cmd.isForced()) { + String msg = String.format("No change in the scripts checksum, not patching systemVM %s", sysVMName); + s_logger.info(msg); + return new PatchSystemVmAnswer(cmd, msg, lines[0], lines[1]); + } + } + + Pair<Boolean, String> patchResult = null; + try { + patchResult = SshHelper.sshExecute(controlIp, sshPort, "root", + pemFile, null, "/home/cloud/patch-sysvms.sh", 10000, 10000, 60000); + } catch (Exception e) { + return new PatchSystemVmAnswer(cmd, e.getMessage()); + } + + if (patchResult.first()) { + return new PatchSystemVmAnswer(cmd, String.format("Successfully patched systemVM %s ", sysVMName), lines[0], lines[1]); + } + return new PatchSystemVmAnswer(cmd, patchResult.second()); + } + + private String calculateCurrentChecksum(String name) { + String cloudScriptsPath = Script.findScript("", "vms/cloud-scripts.tgz"); + if (cloudScriptsPath == null) { + throw new CloudRuntimeException(String.format("Unable to find cloudScripts path, cannot update SystemVM %s", name)); + } + String md5sum = EncryptionUtil.calculateChecksum(new File(cloudScriptsPath)); + return md5sum; + } + + private ExecutionResult getSystemVmVersionAndChecksum(LibvirtComputingResource serverResource, String controlIp) { + ExecutionResult result; + try { + result = serverResource.executeInVR(controlIp, VRScripts.VERSION, null); + if (!result.isSuccess()) { + String errMsg = String.format("GetSystemVMVersionCmd on %s failed, message %s", controlIp, result.getDetails()); + s_logger.error(errMsg); + throw new CloudRuntimeException(errMsg); + } + } catch (final Exception e) { + final String msg = "GetSystemVMVersionCmd failed due to " + e; + s_logger.error(msg, e); + throw new CloudRuntimeException(msg, e); + } + return result; + } + + private void scpPatchFiles(String controlIp) { + try { + List<String> srcFiles = Arrays.asList(LibvirtComputingResource.newSrcFiles); + srcFiles = srcFiles.stream() + .map(file -> LibvirtComputingResource.BASEPATH + file) // Using Lambda notation to update the entries + .collect(Collectors.toList()); + String[] newSrcFiles = srcFiles.toArray(new String[0]); + SshHelper.scpTo(controlIp, sshPort, "root", pemFile, null, + "/home/cloud/", newSrcFiles, "0755"); + } catch (Exception e) { + String errMsg = "Failed to scp files to system VM"; + s_logger.error(errMsg, e); + throw new CloudRuntimeException(errMsg, e); + } + } +} + diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartCommandWrapper.java index f151255..b95c163 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartCommandWrapper.java @@ -19,8 +19,13 @@ package com.cloud.hypervisor.kvm.resource.wrapper; +import java.io.File; import java.net.URISyntaxException; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import com.cloud.utils.ssh.SshHelper; import org.apache.log4j.Logger; import org.libvirt.Connect; import org.libvirt.DomainInfo.DomainState; @@ -115,6 +120,22 @@ public final class LibvirtStartCommandWrapper extends CommandWrapper<StartComman break; } } + + try { + List<String> srcFiles = Arrays.asList(LibvirtComputingResource.srcFiles); + srcFiles = srcFiles.stream() + .map(file -> LibvirtComputingResource.BASEPATH + file) + .collect(Collectors.toList()); + File pemFile = new File(LibvirtComputingResource.SSHPRVKEYPATH); + SshHelper.scpTo(controlIp, 3922, "root", pemFile, null, + "/home/cloud/", srcFiles.toArray(new String[0]), "0755"); + // TODO: May want to remove this when cert patching logic is moved + Thread.sleep(10000); + } catch (Exception e) { + String errMsg = "Failed to scp files to system VM. Patching of systemVM failed"; + s_logger.error(errMsg, e); + return new StartAnswer(command, String.format("%s due to: %s", errMsg, e.getMessage())); + } } } diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java index c744299..c1bb5bb 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java @@ -54,6 +54,7 @@ import javax.xml.xpath.XPathConstants; import javax.xml.xpath.XPathExpressionException; import javax.xml.xpath.XPathFactory; +import com.cloud.utils.ssh.SshHelper; import org.apache.cloudstack.storage.command.AttachAnswer; import org.apache.cloudstack.storage.command.AttachCommand; import org.apache.cloudstack.utils.linux.CPUStat; @@ -210,7 +211,7 @@ import org.apache.cloudstack.utils.bytescale.ByteScaleUtils; import org.libvirt.VcpuInfo; @RunWith(PowerMockRunner.class) -@PrepareForTest(value = {MemStat.class}) +@PrepareForTest(value = {MemStat.class, SshHelper.class}) @PowerMockIgnore({"javax.xml.*", "org.w3c.dom.*", "org.apache.xerces.*"}) public class LibvirtComputingResourceTest { @@ -5278,7 +5279,9 @@ public class LibvirtComputingResourceTest { } @Test - public void testStartCommand() { + public void testStartCommand() throws Exception { + PowerMockito.mockStatic(SshHelper.class); + PowerMockito.doNothing().when(SshHelper.class, "scpTo", Mockito.anyString(), Mockito.anyInt(), Mockito.anyString(), Mockito.any(File.class), nullable(String.class), Mockito.anyString(), Mockito.any(String[].class), Mockito.anyString()); final VirtualMachineTO vmSpec = Mockito.mock(VirtualMachineTO.class); final com.cloud.host.Host host = Mockito.mock(com.cloud.host.Host.class); final boolean executeInSequence = false; @@ -5352,7 +5355,9 @@ public class LibvirtComputingResourceTest { } @Test - public void testStartCommandIsolationEc2() { + public void testStartCommandIsolationEc2() throws Exception { + PowerMockito.mockStatic(SshHelper.class); + PowerMockito.doNothing().when(SshHelper.class, "scpTo", Mockito.anyString(), Mockito.anyInt(), Mockito.anyString(), Mockito.any(File.class), nullable(String.class), Mockito.anyString(), Mockito.any(String[].class), Mockito.anyString()); final VirtualMachineTO vmSpec = Mockito.mock(VirtualMachineTO.class); final com.cloud.host.Host host = Mockito.mock(com.cloud.host.Host.class); final boolean executeInSequence = false; diff --git a/pom.xml b/pom.xml index 67cbc6e..69f520b 100644 --- a/pom.xml +++ b/pom.xml @@ -49,6 +49,7 @@ <!-- keep in alphabetic order --> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> + <project.systemvm.template.version>4.16.0.0</project.systemvm.template.version> <!-- Build properties --> <cs.jdk.version>11</cs.jdk.version> diff --git a/scripts/vm/hypervisor/xenserver/xcpserver/patch b/scripts/vm/hypervisor/xenserver/xcpserver/patch index 862aa2e..32c7c46 100644 --- a/scripts/vm/hypervisor/xenserver/xcpserver/patch +++ b/scripts/vm/hypervisor/xenserver/xcpserver/patch @@ -32,6 +32,8 @@ vmops=..,0755,/etc/xapi.d/plugins ovstunnel=..,0755,/etc/xapi.d/plugins vmopsSnapshot=..,0755,/etc/xapi.d/plugins systemvm.iso=../../../../../vms,0644,/opt/xensource/packages/iso +agent.zip=../../../../../vms,0644,/opt/xensource/packages/iso +cloud-scripts.tgz=../../../../../vms,0644,/opt/xensource/packages/iso id_rsa.cloud=../../../systemvm,0600,/root/.ssh network_info.sh=..,0755,/opt/cloud/bin setupxenserver.sh=..,0755,/opt/cloud/bin diff --git a/server/src/main/java/com/cloud/api/ResponseObjectTypeAdapter.java b/server/src/main/java/com/cloud/api/ResponseObjectTypeAdapter.java index 44baedc..f6f777e 100644 --- a/server/src/main/java/com/cloud/api/ResponseObjectTypeAdapter.java +++ b/server/src/main/java/com/cloud/api/ResponseObjectTypeAdapter.java @@ -22,6 +22,7 @@ import java.lang.reflect.Type; import org.apache.cloudstack.api.ResponseObject; import org.apache.cloudstack.api.response.ExceptionResponse; import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; import com.google.gson.JsonElement; @@ -38,6 +39,9 @@ public class ResponseObjectTypeAdapter implements JsonSerializer<ResponseObject> if (responseObj instanceof SuccessResponse) { obj.addProperty("success", ((SuccessResponse)responseObj).getSuccess()); + if (!StringUtils.isEmpty(((SuccessResponse) responseObj).getDisplayText())) { + obj.addProperty("details", ((SuccessResponse)responseObj).getDisplayText()); + } return obj; } else if (responseObj instanceof ExceptionResponse) { obj.addProperty("errorcode", ((ExceptionResponse)responseObj).getErrorCode()); diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index 08ae0ad..78dd3fb 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -44,8 +44,15 @@ import javax.crypto.spec.SecretKeySpec; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.agent.api.PatchSystemVmAnswer; +import com.cloud.agent.api.PatchSystemVmCommand; +import com.cloud.agent.api.routing.NetworkElementCommand; import com.cloud.dc.DomainVlanMapVO; import com.cloud.dc.dao.DomainVlanMapDao; +import com.cloud.exception.AgentUnavailableException; +import com.cloud.network.Networks; +import com.cloud.vm.NicVO; +import com.cloud.vm.dao.NicDao; import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.affinity.AffinityGroupProcessor; import org.apache.cloudstack.affinity.dao.AffinityGroupVMMapDao; @@ -224,6 +231,7 @@ import org.apache.cloudstack.api.command.admin.swift.ListSwiftsCmd; import org.apache.cloudstack.api.command.admin.systemvm.DestroySystemVmCmd; import org.apache.cloudstack.api.command.admin.systemvm.ListSystemVMsCmd; import org.apache.cloudstack.api.command.admin.systemvm.MigrateSystemVMCmd; +import org.apache.cloudstack.api.command.admin.systemvm.PatchSystemVMCmd; import org.apache.cloudstack.api.command.admin.systemvm.RebootSystemVmCmd; import org.apache.cloudstack.api.command.admin.systemvm.ScaleSystemVMCmd; import org.apache.cloudstack.api.command.admin.systemvm.StartSystemVMCmd; @@ -761,6 +769,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe static final ConfigKey<Integer> sshKeyLength = new ConfigKey<Integer>("Advanced", Integer.class, "ssh.key.length", "2048", "Specifies custom SSH key length (bit)", true, ConfigKey.Scope.Global); static final ConfigKey<Boolean> humanReadableSizes = new ConfigKey<Boolean>("Advanced", Boolean.class, "display.human.readable.sizes", "true", "Enables outputting human readable byte sizes to logs and usage records.", false, ConfigKey.Scope.Global); public static final ConfigKey<String> customCsIdentifier = new ConfigKey<String>("Advanced", String.class, "custom.cs.identifier", UUID.randomUUID().toString().split("-")[0].substring(4), "Custom identifier for the cloudstack installation", true, ConfigKey.Scope.Global); + private static final VirtualMachine.Type []systemVmTypes = { VirtualMachine.Type.SecondaryStorageVm, VirtualMachine.Type.ConsoleProxy, VirtualMachine.Type.DomainRouter }; @Inject public AccountManager _accountMgr; @@ -823,7 +832,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe @Inject private StoragePoolJoinDao _poolJoinDao; @Inject - private NetworkDao _networkDao; + private NetworkDao networkDao; @Inject private StorageManager _storageMgr; @Inject @@ -896,6 +905,8 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe private AnnotationDao annotationDao; @Inject private DomainVlanMapDao _domainVlanMapDao; + @Inject + private NicDao nicDao; private LockControllerListener _lockControllerListener; private final ScheduledExecutorService _eventExecutor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("EventChecker")); @@ -2133,9 +2144,9 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe if (ip == null) { throw new InvalidParameterValueException("Please specify a valid ipaddress id"); } - network = _networkDao.findById(ip.getSourceNetworkId()); + network = networkDao.findById(ip.getSourceNetworkId()); } else { - network = _networkDao.findById(networkId); + network = networkDao.findById(networkId); } if (network == null || network.getGuestType() != Network.GuestType.Shared) { throw new InvalidParameterValueException("Please specify a valid network id"); @@ -2197,7 +2208,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe } if (associatedNetworkId != null) { - _accountMgr.checkAccess(caller, null, false, _networkDao.findById(associatedNetworkId)); + _accountMgr.checkAccess(caller, null, false, networkDao.findById(associatedNetworkId)); sc.setParameters("associatedNetworkIdEq", associatedNetworkId); } if (vpcId != null) { @@ -2213,7 +2224,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe Long zoneId = zone; Account owner = _accountMgr.finalizeOwner(CallContext.current().getCallingAccount(), cmd.getAccountName(), cmd.getDomainId(), cmd.getProjectId()); if (associatedNetworkId != null) { - NetworkVO guestNetwork = _networkDao.findById(associatedNetworkId); + NetworkVO guestNetwork = networkDao.findById(associatedNetworkId); if (zoneId == null) { zoneId = guestNetwork.getDataCenterId(); } else if (zoneId != guestNetwork.getDataCenterId()) { @@ -3486,6 +3497,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe cmdList.add(UploadResourceIconCmd.class); cmdList.add(DeleteResourceIconCmd.class); cmdList.add(ListResourceIconCmd.class); + cmdList.add(PatchSystemVMCmd.class); // Out-of-band management APIs for admins cmdList.add(EnableOutOfBandManagementForHostCmd.class); @@ -3891,7 +3903,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe boolean elasticLoadBalancerEnabled = false; boolean KVMSnapshotEnabled = false; String supportELB = "false"; - final List<NetworkVO> networks = _networkDao.listSecurityGroupEnabledNetworks(); + final List<NetworkVO> networks = networkDao.listSecurityGroupEnabledNetworks(); if (networks != null && !networks.isEmpty()) { securityGroupsEnabled = true; final String elbEnabled = _configDao.getValue(Config.ElasticLoadBalancerEnabled.key()); @@ -4592,6 +4604,72 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe _dpMgr.cleanupVMReservations(); } + @Override + public Pair<Boolean, String> patchSystemVM(PatchSystemVMCmd cmd) { + Long systemVmId = cmd.getId(); + boolean forced = cmd.isForced(); + + if (systemVmId == null) { + throw new InvalidParameterValueException("Please provide a valid ID of a system VM to be patched"); + } + + final VMInstanceVO systemVm = _vmInstanceDao.findByIdTypes(systemVmId, systemVmTypes); + if (systemVm == null) { + throw new InvalidParameterValueException("Unable to find SystemVm with id " + systemVmId); + } + + return updateSystemVM(systemVm, forced); + } + + + private String getControlIp(final long systemVmId) { + String controlIpAddress = null; + final List<NicVO> nics = nicDao.listByVmId(systemVmId); + for (final NicVO n : nics) { + final NetworkVO nc = networkDao.findById(n.getNetworkId()); + if (nc != null && nc.getTrafficType() == Networks.TrafficType.Control) { + controlIpAddress = n.getIPv4Address(); + // router will have only one control IP + break; + } + } + + if (controlIpAddress == null) { + s_logger.warn("Unable to find systemVm's control ip in its attached NICs!. systemVmId: " + systemVmId); + VMInstanceVO systemVM = _vmInstanceDao.findById(systemVmId); + return systemVM.getPrivateIpAddress(); + } + + return controlIpAddress; + } + + private Pair<Boolean, String> updateSystemVM(VMInstanceVO systemVM, boolean forced) { + return patchSystemVm(systemVM, forced); + } + + private Pair<Boolean, String> patchSystemVm(VMInstanceVO systemVM, boolean forced) { + PatchSystemVmAnswer answer = new PatchSystemVmAnswer(); + final PatchSystemVmCommand command = new PatchSystemVmCommand(); + command.setAccessDetail(NetworkElementCommand.ROUTER_IP, getControlIp(systemVM.getId())); + command.setAccessDetail(NetworkElementCommand.ROUTER_NAME, systemVM.getInstanceName()); + command.setForced(forced); + try { + answer = (PatchSystemVmAnswer) _agentMgr.send(systemVM.getHostId(), command); + if (!answer.getResult()) { + String errMsg = String.format("Failed to patch systemVM %s due to %s", systemVM.getInstanceName(), answer.getDetails()); + s_logger.error(errMsg); + return new Pair<>(false, errMsg); + } + + } catch (AgentUnavailableException | OperationTimedoutException e) { + String errMsg = "SystemVM live patch failed"; + s_logger.error(errMsg, e); + return new Pair<>(false, String.format("%s due to: %s", errMsg, e.getMessage())); + } + s_logger.info(String.format("Successfully patch system VM %s", systemVM.getInstanceName())); + return new Pair<>(true, answer.getDetails()); + } + public List<StoragePoolAllocator> getStoragePoolAllocators() { return _storagePoolAllocators; } diff --git a/systemvm/debian/etc/systemd/system/cloud-early-config.service b/systemvm/debian/etc/systemd/system/cloud-early-config.service index 2af5276..cfaf5e7 100644 --- a/systemvm/debian/etc/systemd/system/cloud-early-config.service +++ b/systemvm/debian/etc/systemd/system/cloud-early-config.service @@ -2,11 +2,8 @@ Description=CloudStack post-boot patching service using cmdline DefaultDependencies=no -Before=network-pre.target -Wants=network-pre.target - -Requires=local-fs.target -After=local-fs.target +Requires=local-fs.target cloud-preinit.service +After=local-fs.target cloud-preinit.service [Install] WantedBy=multi-user.target diff --git a/systemvm/debian/etc/systemd/system/cloud-postinit.service b/systemvm/debian/etc/systemd/system/cloud-postinit.service index cb20aaf..f5b23e2 100644 --- a/systemvm/debian/etc/systemd/system/cloud-postinit.service +++ b/systemvm/debian/etc/systemd/system/cloud-postinit.service @@ -1,7 +1,7 @@ [Unit] Description=CloudStack post-patching init script After=cloud-early-config.service network.target local-fs.target -Before=ssh.service +#Before=ssh.service [Install] WantedBy=multi-user.target diff --git a/systemvm/debian/etc/systemd/system/cloud-early-config.service b/systemvm/debian/etc/systemd/system/cloud-preinit.service similarity index 67% copy from systemvm/debian/etc/systemd/system/cloud-early-config.service copy to systemvm/debian/etc/systemd/system/cloud-preinit.service index 2af5276..373cd9e 100644 --- a/systemvm/debian/etc/systemd/system/cloud-early-config.service +++ b/systemvm/debian/etc/systemd/system/cloud-preinit.service @@ -1,5 +1,5 @@ [Unit] -Description=CloudStack post-boot patching service using cmdline +Description=CloudStack service to initialize interfaces DefaultDependencies=no Before=network-pre.target @@ -13,6 +13,6 @@ WantedBy=multi-user.target [Service] Type=oneshot -ExecStart=/opt/cloud/bin/setup/cloud-early-config +ExecStart=/opt/cloud/bin/setup/init.sh RemainAfterExit=true TimeoutStartSec=5min diff --git a/systemvm/debian/opt/cloud/bin/setup/bootstrap.sh b/systemvm/debian/opt/cloud/bin/setup/bootstrap.sh index 2335d64..3f64be7 100755 --- a/systemvm/debian/opt/cloud/bin/setup/bootstrap.sh +++ b/systemvm/debian/opt/cloud/bin/setup/bootstrap.sh @@ -15,7 +15,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - +set -x PATH="/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin" CMDLINE=/var/cache/cloud/cmdline @@ -29,124 +29,6 @@ log_it() { log_action_msg "$@" } -hypervisor() { - if [ -d /proc/xen ]; then - mount -t xenfs none /proc/xen - $(dmesg | grep -q "Xen HVM") - if [ $? -eq 0 ]; then # 1=PV,0=HVM - echo "xen-hvm" && return 0 - else - echo "xen-pv" && return 0 - fi - fi - - [ -x /usr/sbin/virt-what ] && local facts=( $(virt-what) ) - if [ "$facts" != "" ]; then - # Xen HVM is recognized as Hyperv when Viridian extensions are enabled - if [ "${facts[-1]}" == "xen-domU" ] && [ "${facts[0]}" == "hyperv" ]; then - echo "xen-hvm" && return 0 - else - echo ${facts[-1]} && return 0 - fi - fi - - grep -q QEMU /proc/cpuinfo && echo "kvm" && return 0 - grep -q QEMU /var/log/messages && echo "kvm" && return 0 - - vmware-checkvm &> /dev/null && echo "vmware" && return 0 - - echo "unknown" && return 1 -} - -config_guest() { - [ ! -d /proc/xen ] && sed -i 's/^vc/#vc/' /etc/inittab && telinit q - [ -d /proc/xen ] && sed -i 's/^#vc/vc/' /etc/inittab && telinit q - - systemctl daemon-reload - - case $HYPERVISOR in - xen-pv|xen-domU) - systemctl stop ntpd - systemctl disable ntpd - systemctl enable xe-daemon - systemctl start xe-daemon - - cat /proc/cmdline > $CMDLINE - sed -i "s/%/ /g" $CMDLINE - ;; - xen-hvm) - systemctl stop ntpd - systemctl disable ntpd - systemctl enable xe-daemon - systemctl start xe-daemon - - if [ ! -f /usr/bin/xenstore-read ]; then - log_it "ERROR: xentools not installed, cannot found xenstore-read" && exit 5 - fi - /usr/bin/xenstore-read vm-data/cloudstack/init > $CMDLINE - sed -i "s/%/ /g" $CMDLINE - ;; - kvm) - # Configure kvm hotplug support - if grep -E 'CONFIG_HOTPLUG_PCI=y|CONFIG_HOTPLUG_PCI_ACPI=y' /boot/config-`uname -r`; then - log_it "acpiphp and pci_hotplug module already compiled in" - else - modprobe acpiphp 2> /dev/null && log_it "acpiphp module loaded" || true - modprobe pci_hotplug 2> /dev/null && log_it "pci_hotplug module loaded" || true - fi - - sed -i -e "/^s0:2345:respawn.*/d" /etc/inittab - sed -i -e "/6:23:respawn/a\s0:2345:respawn:/sbin/getty -L 115200 ttyS0 vt102" /etc/inittab - systemctl enable qemu-guest-agent - systemctl start qemu-guest-agent - - # Wait for $CMDLINE file to be written by the qemu-guest-agent - for i in {1..60}; do - if [ -s $CMDLINE ]; then - log_it "Received a new non-empty cmdline file from qemu-guest-agent" - # Remove old configuration files in /etc/cloudstack if VR is booted from cloudstack - rm -rf /etc/cloudstack/*.json - log_it "Booting from cloudstack, remove old configuration files in /etc/cloudstack/" - break - fi - sleep 1 - done - if [ ! -s $CMDLINE ]; then - log_it "Failed to receive the cmdline file via the qemu-guest-agent" - fi - ;; - vmware) - # system time sync'd with host via vmware tools - systemctl stop ntpd - systemctl disable ntpd - systemctl enable open-vm-tools - systemctl start open-vm-tools - - vmtoolsd --cmd 'machine.id.get' > $CMDLINE - ;; - virtualpc|hyperv) - # Hyper-V is recognized as virtualpc hypervisor type. Boot args are passed using KVP Daemon - systemctl enable hyperv-daemons.hv-fcopy-daemon.service hyperv-daemons.hv-kvp-daemon.service hyperv-daemons.hv-vss-daemon.service - systemctl start hyperv-daemons.hv-fcopy-daemon.service hyperv-daemons.hv-kvp-daemon.service hyperv-daemons.hv-vss-daemon.service - sleep 5 - cp -f /var/opt/hyperv/.kvp_pool_0 $CMDLINE - cat /dev/null > /var/opt/hyperv/.kvp_pool_0 - ;; - virtualbox) - # Virtualbox is used to test the virtual router - # get the commandline from a dmistring (yes, hacky!) - dmidecode | grep cmdline | sed 's/^.*cmdline://' > $CMDLINE - RV=$? - if [ $RV -ne 0 ] ; then - log_it "Failed to get cmdline from a virtualbox dmi property" - fi - ;; - esac - - # Find and export guest type - export TYPE=$(grep -Po 'type=\K[a-zA-Z]*' $CMDLINE) -} - patch_systemvm() { local patchfile=$1 local backupfolder="/tmp/.conf.backup" @@ -158,6 +40,8 @@ patch_systemvm() { fi rm /usr/local/cloud/systemvm -rf mkdir -p /usr/local/cloud/systemvm + ls -lrt $patchfile + echo "All" | unzip $patchfile -d /usr/local/cloud/systemvm >$logfile 2>&1 find /usr/local/cloud/systemvm/ -name \*.sh | xargs chmod 555 if [ -f $backupfolder/cloud.jks ]; then @@ -171,7 +55,7 @@ patch_systemvm() { } patch() { - local PATCH_MOUNT=/media/cdrom + local PATCH_MOUNT=/home/cloud local logfile="/var/log/patchsystemvm.log" if [ "$TYPE" == "consoleproxy" ] || [ "$TYPE" == "secstorage" ] && [ -f ${PATCH_MOUNT}/agent.zip ] && [ -f /var/cache/cloud/patch.required ] @@ -188,11 +72,7 @@ patch() { rm -f /var/cache/cloud/patch.required chmod -x /etc/systemd/system/cloud*.service systemctl daemon-reload - umount $PATCH_MOUNT || true - if [ -f /mnt/cmdline ]; then - cat /mnt/cmdline > $CMDLINE - fi return 0 } @@ -212,11 +92,7 @@ config_sysctl() { bootstrap() { log_it "Bootstrapping systemvm appliance" - export HYPERVISOR=$(hypervisor) - [ $? -ne 0 ] && log_it "Failed to detect hypervisor type, bailing out" && exit 10 - log_it "Starting guest services for $HYPERVISOR" - - config_guest + export TYPE=$(grep -Po 'type=\K[a-zA-Z]*' $CMDLINE) patch config_sysctl diff --git a/systemvm/debian/opt/cloud/bin/setup/cksnode.sh b/systemvm/debian/opt/cloud/bin/setup/cksnode.sh index a864d18..612fdd4 100755 --- a/systemvm/debian/opt/cloud/bin/setup/cksnode.sh +++ b/systemvm/debian/opt/cloud/bin/setup/cksnode.sh @@ -39,7 +39,7 @@ setup_k8s_node() { log_it "Swap disabled" log_it "Setting up interfaces" - setup_common eth0 +# setup_common eth0 setup_system_rfc1918_internal log_it "Setting up entry in hosts" diff --git a/systemvm/debian/opt/cloud/bin/setup/cloud-early-config b/systemvm/debian/opt/cloud/bin/setup/cloud-early-config index d0ebd0b..9695b18 100755 --- a/systemvm/debian/opt/cloud/bin/setup/cloud-early-config +++ b/systemvm/debian/opt/cloud/bin/setup/cloud-early-config @@ -15,7 +15,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - +set -x PATH="/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin" # Clear boot up flag, it would be created by rc.local after boot up done @@ -32,53 +32,61 @@ log_it() { } patch() { - local PATCH_MOUNT=/media/cdrom + local PATCH_MOUNT=/home/cloud local patchfile=$PATCH_MOUNT/cloud-scripts.tgz local privkey=$PATCH_MOUNT/authorized_keys local md5file=/var/cache/cloud/cloud-scripts-signature local cdrom_dev= mkdir -p $PATCH_MOUNT - if [ -e /dev/xvdd ]; then - cdrom_dev=/dev/xvdd - elif [ -e /dev/cdrom ]; then - cdrom_dev=/dev/cdrom - elif [ -e /dev/cdrom1 ]; then - cdrom_dev=/dev/cdrom1 - elif [ -e /dev/cdrom2 ]; then - cdrom_dev=/dev/cdrom2 - elif [ -e /dev/cdrom3 ]; then - cdrom_dev=/dev/cdrom3 - fi - if [ -f /var/cache/cloud/authorized_keys ]; then privkey=/var/cache/cloud/authorized_keys fi - if [ -n "$cdrom_dev" ]; then - mount -o ro $cdrom_dev $PATCH_MOUNT - local oldmd5= - [ -f ${md5file} ] && oldmd5=$(cat ${md5file}) - local newmd5= - [ -f ${patchfile} ] && newmd5=$(md5sum ${patchfile} | awk '{print $1}') - log_it "Scripts checksum detected: oldmd5=$oldmd5 newmd5=$newmd5" - if [ "$oldmd5" != "$newmd5" ] && [ -f ${patchfile} ] && [ "$newmd5" != "" ] - then - tar xzf $patchfile -C / - echo ${newmd5} > ${md5file} - log_it "Patched scripts using $patchfile" - touch /var/cache/cloud/patch.required + retry=60 + local patched=false + while [ $retry -gt 0 ] + do + if [ -f $patchfile ]; then + local oldmd5= + [ -f ${md5file} ] && oldmd5=$(cat ${md5file}) + local newmd5= + [ -f ${patchfile} ] && newmd5=$(md5sum ${patchfile} | awk '{print $1}') + log_it "Scripts checksum detected: oldmd5=$oldmd5 newmd5=$newmd5" + log_it ls -lrt $PATCH_MOUNT + if [ "$oldmd5" != "$newmd5" ] && [ -f ${patchfile} ] && [ "$newmd5" != "" ] + then + tar xzf $patchfile -C / + ls -lrt /opt/cloud/bin/keystore* + echo ${newmd5} > ${md5file} + log_it "Patched scripts using $patchfile" + touch /var/cache/cloud/patch.required + fi + + if [ -f $privkey ]; then + cp -f $privkey /root/.ssh/ + chmod go-rwx /root/.ssh/authorized_keys + fi + patched=true + break fi - if [ -f $privkey ]; then - cp -f $privkey /root/.ssh/ - chmod go-rwx /root/.ssh/authorized_keys - fi - fi + sleep 2 + retry=$(($retry-1)) + log_it "Could not find patch file, retrying" + done + if [ $retry -eq 0 ] && [ "$patched" == "false" ]; then + return 2 + fi return 0 } +cleanup() { + rm -rf /home/cloud/agent.zip + rm -rf /home/cloud/cloud-scripts.tgz +} + start() { log_it "Executing cloud-early-config" @@ -99,6 +107,7 @@ start() { patch sync /opt/cloud/bin/setup/bootstrap.sh + cleanup log_it "Finished setting up systemvm" exit 0 diff --git a/systemvm/debian/opt/cloud/bin/setup/common.sh b/systemvm/debian/opt/cloud/bin/setup/common.sh index 60b8875..7f3d857 100755 --- a/systemvm/debian/opt/cloud/bin/setup/common.sh +++ b/systemvm/debian/opt/cloud/bin/setup/common.sh @@ -15,7 +15,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - +set -x PATH="/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin" . /lib/lsb/init-functions diff --git a/systemvm/debian/opt/cloud/bin/setup/consoleproxy.sh b/systemvm/debian/opt/cloud/bin/setup/consoleproxy.sh index 3f00f3d..ec45b7f 100755 --- a/systemvm/debian/opt/cloud/bin/setup/consoleproxy.sh +++ b/systemvm/debian/opt/cloud/bin/setup/consoleproxy.sh @@ -25,7 +25,6 @@ setup_console_proxy() { echo "haproxy dnsmasq apache2 nfs-common portmap" > /var/cache/cloud/disabled_svcs mkdir -p /var/log/cloud - setup_common eth0 eth1 eth2 setup_system_rfc1918_internal log_it "Setting up entry in hosts" @@ -33,17 +32,6 @@ setup_console_proxy() { public_ip=`getPublicIp` echo "$public_ip $NAME" >> /etc/hosts - log_it "Applying iptables rules" - cp /etc/iptables/iptables-consoleproxy /etc/iptables/rules.v4 - - log_it "Configuring sshd" - local hyp=$HYPERVISOR - if [ "$hyp" == "vmware" ] || [ "$hyp" == "hyperv" ]; then - setup_sshd $ETH1_IP "eth1" - else - setup_sshd $ETH0_IP "eth0" - fi - disable_rpfilter enable_fwding 0 enable_irqbalance 0 diff --git a/systemvm/debian/opt/cloud/bin/setup/dhcpsrvr.sh b/systemvm/debian/opt/cloud/bin/setup/dhcpsrvr.sh index 9161aeb..0b9e8a7 100755 --- a/systemvm/debian/opt/cloud/bin/setup/dhcpsrvr.sh +++ b/systemvm/debian/opt/cloud/bin/setup/dhcpsrvr.sh @@ -25,7 +25,7 @@ dhcpsrvr_svcs() { setup_dhcpsrvr() { log_it "Setting up dhcp server system vm" - setup_common eth0 eth1 +# setup_common eth0 eth1 setup_dnsmasq setup_apache2 $ETH0_IP @@ -36,18 +36,16 @@ setup_dhcpsrvr() { enable_irqbalance 0 enable_fwding 0 - cp /etc/iptables/iptables-router /etc/iptables/rules.v4 - #Only allow DNS service for current network sed -i "s/-A INPUT -i eth0 -p udp -m udp --dport 53 -j ACCEPT/-A INPUT -i eth0 -p udp -m udp --dport 53 -s $DHCP_RANGE\/$CIDR_SIZE -j ACCEPT/g" /etc/iptables/rules.v4 sed -i "s/-A INPUT -i eth0 -p tcp -m tcp --dport 53 -j ACCEPT/-A INPUT -i eth0 -p tcp -m tcp --dport 53 -s $DHCP_RANGE\/$CIDR_SIZE -j ACCEPT/g" /etc/iptables/rules.v4 - if [ "$SSHONGUEST" == "true" ] - then - setup_sshd $ETH0_IP "eth0" - else - setup_sshd $ETH1_IP "eth1" - fi +# if [ "$SSHONGUEST" == "true" ] +# then +# setup_sshd $ETH0_IP "eth0" +# else +# setup_sshd $ETH1_IP "eth1" +# fi } dhcpsrvr_svcs diff --git a/systemvm/debian/opt/cloud/bin/setup/elbvm.sh b/systemvm/debian/opt/cloud/bin/setup/elbvm.sh index ae16b4b..52132cc 100755 --- a/systemvm/debian/opt/cloud/bin/setup/elbvm.sh +++ b/systemvm/debian/opt/cloud/bin/setup/elbvm.sh @@ -25,20 +25,11 @@ elbvm_svcs() { setup_elbvm() { log_it "Setting up Elastic Load Balancer system vm" - setup_common eth0 eth1 sed -i /$NAME/d /etc/hosts public_ip=$ETH2_IP [ "$ETH2_IP" == "0.0.0.0" ] || [ "$ETH2_IP" == "" ] && public_ip=$ETH0_IP echo "$public_ip $NAME" >> /etc/hosts - cp /etc/iptables/iptables-elbvm /etc/iptables/rules.v4 - if [ "$SSHONGUEST" == "true" ] - then - setup_sshd $ETH0_IP "eth0" - else - setup_sshd $ETH1_IP "eth1" - fi - enable_fwding 0 enable_irqbalance 0 } diff --git a/systemvm/debian/opt/cloud/bin/setup/ilbvm.sh b/systemvm/debian/opt/cloud/bin/setup/ilbvm.sh index ac801b2..83cc855 100755 --- a/systemvm/debian/opt/cloud/bin/setup/ilbvm.sh +++ b/systemvm/debian/opt/cloud/bin/setup/ilbvm.sh @@ -25,7 +25,7 @@ ilbvm_svcs() { setup_ilbvm() { log_it "Setting up Internal Load Balancer system vm" - setup_common eth0 eth1 +# setup_common eth0 eth1 #eth0 = guest network, eth1=control network sed -i /$NAME/d /etc/hosts diff --git a/systemvm/debian/opt/cloud/bin/setup/bootstrap.sh b/systemvm/debian/opt/cloud/bin/setup/init.sh old mode 100755 new mode 100644 similarity index 67% copy from systemvm/debian/opt/cloud/bin/setup/bootstrap.sh copy to systemvm/debian/opt/cloud/bin/setup/init.sh index 2335d64..5923b35 --- a/systemvm/debian/opt/cloud/bin/setup/bootstrap.sh +++ b/systemvm/debian/opt/cloud/bin/setup/init.sh @@ -16,19 +16,10 @@ # specific language governing permissions and limitations # under the License. +set -x PATH="/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin" CMDLINE=/var/cache/cloud/cmdline -rm -f /var/cache/cloud/enabled_svcs -rm -f /var/cache/cloud/disabled_svcs - -. /lib/lsb/init-functions - -log_it() { - echo "$(date) $@" >> /var/log/cloud.log - log_action_msg "$@" -} - hypervisor() { if [ -d /proc/xen ]; then mount -t xenfs none /proc/xen @@ -143,92 +134,84 @@ config_guest() { ;; esac + if [ -f /mnt/cmdline ]; then + cat /mnt/cmdline > $CMDLINE + fi + # Find and export guest type export TYPE=$(grep -Po 'type=\K[a-zA-Z]*' $CMDLINE) } -patch_systemvm() { - local patchfile=$1 - local backupfolder="/tmp/.conf.backup" - local logfile="/var/log/patchsystemvm.log" - if [ -f /usr/local/cloud/systemvm/conf/cloud.jks ]; then - rm -fr $backupfolder - mkdir -p $backupfolder - cp -r /usr/local/cloud/systemvm/conf/* $backupfolder/ - fi - rm /usr/local/cloud/systemvm -rf - mkdir -p /usr/local/cloud/systemvm - echo "All" | unzip $patchfile -d /usr/local/cloud/systemvm >$logfile 2>&1 - find /usr/local/cloud/systemvm/ -name \*.sh | xargs chmod 555 - if [ -f $backupfolder/cloud.jks ]; then - cp -r $backupfolder/* /usr/local/cloud/systemvm/conf/ - echo "Restored keystore file and certs using backup" >> $logfile - fi - rm -fr $backupfolder - # Import global cacerts into 'cloud' service's keystore - keytool -importkeystore -srckeystore /etc/ssl/certs/java/cacerts -destkeystore /usr/local/cloud/systemvm/certs/realhostip.keystore -srcstorepass changeit -deststorepass vmops.com -noprompt || true - return 0 -} +setup_interface_sshd() { -patch() { - local PATCH_MOUNT=/media/cdrom - local logfile="/var/log/patchsystemvm.log" - - if [ "$TYPE" == "consoleproxy" ] || [ "$TYPE" == "secstorage" ] && [ -f ${PATCH_MOUNT}/agent.zip ] && [ -f /var/cache/cloud/patch.required ] - then - echo "Patching systemvm for cloud service with mount=$PATCH_MOUNT for type=$TYPE" >> $logfile - patch_systemvm ${PATCH_MOUNT}/agent.zip - if [ $? -gt 0 ] - then - echo "Failed to apply patch systemvm\n" >> $logfile - exit 1 + if [ "$TYPE" != "cksnode" ]; then + log_it "Applying iptables rules" + if [ "$TYPE" != "dhcpsrvr" ]; then + cp /etc/iptables/iptables-$TYPE /etc/iptables/rules.v4 + else + cp /etc/iptables/iptables-router /etc/iptables/rules.v4 fi fi - rm -f /var/cache/cloud/patch.required - chmod -x /etc/systemd/system/cloud*.service - systemctl daemon-reload - umount $PATCH_MOUNT || true - - if [ -f /mnt/cmdline ]; then - cat /mnt/cmdline > $CMDLINE - fi - return 0 -} - -config_sysctl() { - # When there is more memory reset the cache back pressure to default 100 - physmem=$(free|awk '/^Mem:/{print $2}') - if [ $((physmem)) -lt 409600 ]; then - sed -i "/^vm.vfs_cache_pressure/ c\vm.vfs_cache_pressure = 200" /etc/sysctl.conf - else - sed -i "/^vm.vfs_cache_pressure/ c\vm.vfs_cache_pressure = 100" /etc/sysctl.conf - fi + if [ "$TYPE" == "consoleproxy" ] || [ "$TYPE" == "secstorage" ]; then + setup_common eth0 eth1 eth2 + log_it "Configuring sshd" + local hyp=$HYPERVISOR + if [ "$hyp" == "vmware" ] || [ "$hyp" == "hyperv" ]; then + setup_sshd $ETH1_IP "eth1" + else + setup_sshd $ETH0_IP "eth0" + fi - sync - sysctl -p -} + elif [ "$TYPE" == "router" ]; then + if [ -n "$ETH2_IP" ]; then + setup_common eth0 eth1 eth2 -bootstrap() { - log_it "Bootstrapping systemvm appliance" + if [ -n "$EXTRA_PUBNICS" ]; then + for ((i = 3; i < 3 + $EXTRA_PUBNICS; i++)); do + setup_interface "$i" "0.0.0.0" "255.255.255.255" $GW "force" + done + fi + else + setup_common eth0 eth1 + if [ -n "$EXTRA_PUBNICS" ]; then + for ((i = 2; i < 2 + $EXTRA_PUBNICS; i++)); do + setup_interface "$i" "0.0.0.0" "255.255.255.255" $GW "force" + done + fi + fi + setup_sshd $ETH1_IP "eth1" - export HYPERVISOR=$(hypervisor) - [ $? -ne 0 ] && log_it "Failed to detect hypervisor type, bailing out" && exit 10 - log_it "Starting guest services for $HYPERVISOR" + elif [ "$TYPE" == "vpcrouter" ]; then + setup_interface "0" $ETH0_IP $ETH0_MASK $GW + setup_sshd $ETH0_IP "eth0" - config_guest - patch - config_sysctl + elif [ "$TYPE" == "ilbvm" ]; then + setup_common eth0 eth1 + setup_sshd $ETH1_IP "eth1" - log_it "Configuring systemvm type=$TYPE" - if [ -f "/opt/cloud/bin/setup/$TYPE.sh" ]; then - /opt/cloud/bin/setup/$TYPE.sh - else - /opt/cloud/bin/setup/default.sh + elif [ "$TYPE" == "elbvm" ] || [ "$TYPE" == "dhcpsrvr"]; then + setup_common eth0 eth1 + if [ "$SSHONGUEST" == "true" ]; then + setup_sshd $ETH0_IP "eth0" + else + setup_sshd $ETH1_IP "eth1" + fi + elif [ "$TYPE" == "cksnode" ]; then + setup_common eth0 fi - log_it "Finished setting up systemvm" - exit 0 + systemctl restart systemd-journald + # Patch known systemd/sshd memory leak - https://github.com/systemd/systemd/issues/8015#issuecomment-476160981 + echo '@include null' >> /etc/pam.d/systemd-user + # Enable and Start SSH + systemctl enable --now --no-block ssh } -bootstrap +export HYPERVISOR=$(hypervisor) +[ $? -ne 0 ] && log_it "Failed to detect hypervisor type, bailing out" && exit 10 +log_it "Starting guest services for $HYPERVISOR" + +config_guest +source /opt/cloud/bin/setup/common.sh +setup_interface_sshd \ No newline at end of file diff --git a/systemvm/debian/opt/cloud/bin/setup/postinit.sh b/systemvm/debian/opt/cloud/bin/setup/postinit.sh index 0492930..ba5c394 100755 --- a/systemvm/debian/opt/cloud/bin/setup/postinit.sh +++ b/systemvm/debian/opt/cloud/bin/setup/postinit.sh @@ -23,17 +23,11 @@ log_it() { log_action_msg "$@" } -# Eject cdrom if any -CMDLINE=/var/cache/cloud/cmdline -export TYPE=$(grep -Po 'type=\K[a-zA-Z]*' $CMDLINE) -if [ "$TYPE" != "cksnode" ]; then - eject || true -fi - # Restart journald for setting changes to apply systemctl restart systemd-journald -TYPE=$(grep -Po 'type=\K[a-zA-Z]*' /var/cache/cloud/cmdline) +CMDLINE=/var/cache/cloud/cmdline +TYPE=$(grep -Po 'type=\K[a-zA-Z]*' $CMDLINE) if [ "$TYPE" == "router" ] || [ "$TYPE" == "vpcrouter" ] || [ "$TYPE" == "dhcpsrvr" ] then if [ -x /opt/cloud/bin/update_config.py ] @@ -71,10 +65,4 @@ then ip6tables-restore < $ipv6 fi -# Patch known systemd/sshd memory leak - https://github.com/systemd/systemd/issues/8015#issuecomment-476160981 -echo '@include null' >> /etc/pam.d/systemd-user - -# Enable and Start SSH -systemctl enable --now --no-block ssh - date > /var/cache/cloud/boot_up_done diff --git a/systemvm/debian/opt/cloud/bin/setup/router.sh b/systemvm/debian/opt/cloud/bin/setup/router.sh index e8f6edf..d7113c4 100755 --- a/systemvm/debian/opt/cloud/bin/setup/router.sh +++ b/systemvm/debian/opt/cloud/bin/setup/router.sh @@ -43,23 +43,6 @@ setup_router() { oldmd5= [ -f "/etc/udev/rules.d/70-persistent-net.rules" ] && oldmd5=$(md5sum "/etc/udev/rules.d/70-persistent-net.rules" | awk '{print $1}') - if [ -n "$ETH2_IP" ]; then - setup_common eth0 eth1 eth2 - - if [ -n "$EXTRA_PUBNICS" ]; then - for ((i = 3; i < 3 + $EXTRA_PUBNICS; i++)); do - setup_interface "$i" "0.0.0.0" "255.255.255.255" $GW "force" - done - fi - else - setup_common eth0 eth1 - if [ -n "$EXTRA_PUBNICS" ]; then - for ((i = 2; i < 2 + $EXTRA_PUBNICS; i++)); do - setup_interface "$i" "0.0.0.0" "255.255.255.255" $GW "force" - done - fi - fi - log_it "Checking udev NIC assignment order changes" if [ "$NIC_MACS" != "" ] then @@ -88,8 +71,6 @@ setup_router() { enable_fwding 1 enable_rpsrfs 1 enable_passive_ftp 1 - cp /etc/iptables/iptables-router /etc/iptables/rules.v4 - setup_sshd $ETH1_IP "eth1" # Only allow DNS service for current network sed -i "s/-A INPUT -i eth0 -p udp -m udp --dport 53 -j ACCEPT/-A INPUT -i eth0 -p udp -m udp --dport 53 -s $DHCP_RANGE\/$CIDR_SIZE -j ACCEPT/g" /etc/iptables/rules.v4 diff --git a/systemvm/debian/opt/cloud/bin/setup/secstorage.sh b/systemvm/debian/opt/cloud/bin/setup/secstorage.sh index 13ed5c5..3b21ed5 100755 --- a/systemvm/debian/opt/cloud/bin/setup/secstorage.sh +++ b/systemvm/debian/opt/cloud/bin/setup/secstorage.sh @@ -25,7 +25,6 @@ setup_secstorage() { echo "conntrackd keepalived haproxy dnsmasq" > /var/cache/cloud/disabled_svcs mkdir -p /var/log/cloud - setup_common eth0 eth1 eth2 setup_storage_network setup_system_rfc1918_internal @@ -37,14 +36,6 @@ setup_secstorage() { log_it "Applying iptables rules" cp /etc/iptables/iptables-secstorage /etc/iptables/rules.v4 - log_it "Configuring sshd" - local hyp=$HYPERVISOR - if [ "$hyp" == "vmware" ] || [ "$hyp" == "hyperv" ]; then - setup_sshd $ETH1_IP "eth1" - else - setup_sshd $ETH0_IP "eth0" - fi - log_it "Configuring apache2" setup_apache2 $ETH2_IP diff --git a/systemvm/debian/opt/cloud/bin/setup/vpcrouter.sh b/systemvm/debian/opt/cloud/bin/setup/vpcrouter.sh index f97fb16..ba4af90 100755 --- a/systemvm/debian/opt/cloud/bin/setup/vpcrouter.sh +++ b/systemvm/debian/opt/cloud/bin/setup/vpcrouter.sh @@ -29,7 +29,6 @@ setup_vpcrouter() { auto lo eth0 iface lo inet loopback EOF - setup_interface "0" $ETH0_IP $ETH0_MASK $GW echo $NAME > /etc/hostname echo 'AVAHI_DAEMON_DETECT_LOCAL=0' > /etc/default/avahi-daemon @@ -86,7 +85,7 @@ EOF enable_fwding 1 enable_passive_ftp 1 cp /etc/iptables/iptables-vpcrouter /etc/iptables/rules.v4 - setup_sshd $ETH0_IP "eth0" +# setup_sshd $ETH0_IP "eth0" cp /etc/vpcdnsmasq.conf /etc/dnsmasq.conf cp /etc/cloud-nic.rules /etc/udev/rules.d/cloud-nic.rules echo "" > /etc/dnsmasq.d/dhcphosts.txt diff --git a/systemvm/patch-sysvms.sh b/systemvm/patch-sysvms.sh new file mode 100644 index 0000000..cf0b452 --- /dev/null +++ b/systemvm/patch-sysvms.sh @@ -0,0 +1,108 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +PATH="/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin" +backupfolder=/tmp/bkpup_live_patch +logfile="/var/log/livepatchsystemvm.log" +newpath="/home/cloud/" +CMDLINE=/var/cache/cloud/cmdline +md5file=/var/cache/cloud/cloud-scripts-signature +svcfile=/var/cache/cloud/enabled_svcs +TYPE=$(grep -Po 'type=\K[a-zA-Z]*' $CMDLINE) +patchfailed=0 + + +backup_old_package() { + mkdir -p $backupfolder + echo "Backing up keystore file and certificates" > $logfile + mkdir -p $backupfolder/conf + cp -r /usr/local/cloud/systemvm/conf/* $backupfolder/conf + echo "Backing up agent package" >> $logfile + zip -r $backupfolder/agent.zip /usr/local/cloud/systemvm/* >> $logfile 2>&1 + cp $md5file $backupfolder + echo "Backing up cloud-scripts file" >> $logfile + tar -zcvf $backupfolder/cloud-scripts.tgz /etc/ /var/ /opt/ /root/ >> $logfile 2>&1 +} + +restore_backup() { + echo "Restoring cloud scripts" >> $logfile + tar -xvf $backupfolder/cloud-scripts.tar -C / >> $logfile 2>&1 + echo "Restoring agent package" >> $logfile + unzip $backupfolder/agent.zip -d /usr/local/cloud/systemvm/ >> $logfile 2>&1 + echo "Restore keystore file and certificates" + mkdir -p "/usr/local/cloud/systemvm/conf/" + cp -r $backupfolder/conf/* /usr/local/cloud/systemvm/conf/ + restart_services + cp $backupfolder/cloud-scripts-signature $md5file +} + +update_checksum() { + newmd5=$(md5sum $1 | awk '{print $1}') + echo "checksum: " ${newmd5} >> $logfile + echo ${newmd5} > ${md5file} +} + +restart_services() { + systemctl daemon-reload + while IFS= read -r line + do + echo "$line" + systemctl restart "$line" + sleep 5 + systemctl is-active --quiet "$line" + if [ $? -gt 0 ]; then + echo "Failed to start "$line" service. Patch Failed. Restoring backup" >> $logfile + restore_backup + patchfailed=1 + break + fi + done < "$svcfile" +} + +cleanup_systemVM() { + rm -rf $backupfolder + rm -rf "$newpath""cloud-scripts.tgz" "$newpath""agent.zip" "$newpath""patch-sysvms.sh" +} + +patch_systemvm() { + rm -rf /usr/local/cloud/systemvm + mkdir -p /usr/local/cloud/systemvm + echo "All" | unzip $newpath/agent.zip -d /usr/local/cloud/systemvm >> $logfile 2>&1 + find /usr/local/cloud/systemvm/ -name \*.sh | xargs chmod 555 + + echo "Extracting cloud scripts" >> $logfile + tar -xvf $newpath/cloud-scripts.tgz -C / >> $logfile 2>&1 + + if [ -f $backupfolder/conf/cloud.jks ]; then + cp -r $backupfolder/conf/* /usr/local/cloud/systemvm/conf/ + echo "Restored keystore file and certs using backup" >> $logfile + fi + + update_checksum $newpath/cloud-scripts.tgz + + if [ "$TYPE" == "consoleproxy" ] || [ "$TYPE" == "secstorage" ] || [[ "$TYPE" == *router ]]; then + restart_services + fi +} + + +backup_old_package +patch_systemvm +cleanup_systemVM + +exit $patchfailed diff --git a/systemvm/pom.xml b/systemvm/pom.xml index 21b95d2..ef2e540 100644 --- a/systemvm/pom.xml +++ b/systemvm/pom.xml @@ -88,6 +88,12 @@ <include>agent.zip</include> </includes> </resource> + <resource> + <directory>${basedir}</directory> + <includes> + <include>patch-sysvms.sh</include> + </includes> + </resource> </resources> </configuration> </execution> diff --git a/tools/appliance/build.sh b/tools/appliance/build.sh index 1c83f9a..79de31a 100755 --- a/tools/appliance/build.sh +++ b/tools/appliance/build.sh @@ -349,10 +349,10 @@ function main() { # process the disk at dist kvm_export - ovm_export - xen_server_export - vmware_export - hyperv_export +# ovm_export +# xen_server_export +# vmware_export +# hyperv_export rm -f "dist/${appliance}" cd dist && chmod +r * && cd .. cd dist && md5sum * > md5sum.txt && cd .. diff --git a/tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh b/tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh index db3eec5..8cdfce7 100644 --- a/tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh +++ b/tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh @@ -68,6 +68,7 @@ function install_cloud_scripts() { chmod -x /etc/systemd/system/* || true systemctl daemon-reload + systemctl enable cloud-preinit systemctl enable cloud-early-config systemctl enable cloud-postinit } diff --git a/tools/appliance/systemvmtemplate/template.json b/tools/appliance/systemvmtemplate/template.json index bd932bf..46fdbc9 100644 --- a/tools/appliance/systemvmtemplate/template.json +++ b/tools/appliance/systemvmtemplate/template.json @@ -27,8 +27,8 @@ "format": "qcow2", "headless": true, "http_directory": "http", - "iso_checksum": "sha512:5f6aed67b159d7ccc1a90df33cc8a314aa278728a6f50707ebf10c02e46664e383ca5fa19163b0a1c6a4cb77a39587881584b00b45f512b4a470f1138eaa1801", - "iso_url": "https://cdimage.debian.org/debian-cd/11.0.0/amd64/iso-cd/debian-11.0.0-amd64-netinst.iso";, + "iso_checksum": "sha512:02257c3ec27e45d9f022c181a69b59da67e5c72871cdb4f9a69db323a1fad58093f2e69702d29aa98f5f65e920e0b970d816475a5a936e1f3bf33832257b7e92", + "iso_url": "https://cdimage.debian.org/debian-cd/11.1.0/amd64/iso-cd/debian-11.1.0-amd64-netinst.iso";, "net_device": "virtio-net", "output_directory": "../dist", "qemuargs": [ diff --git a/utils/src/main/java/com/cloud/utils/EncryptionUtil.java b/utils/src/main/java/com/cloud/utils/EncryptionUtil.java index b82842e..ff791a3 100644 --- a/utils/src/main/java/com/cloud/utils/EncryptionUtil.java +++ b/utils/src/main/java/com/cloud/utils/EncryptionUtil.java @@ -18,7 +18,12 @@ */ package com.cloud.utils; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; import java.io.UnsupportedEncodingException; +import java.nio.file.Files; +import java.nio.file.Paths; import java.security.InvalidKeyException; import java.security.NoSuchAlgorithmException; @@ -26,6 +31,7 @@ import javax.crypto.Mac; import javax.crypto.spec.SecretKeySpec; import org.apache.commons.codec.binary.Base64; +import org.apache.commons.codec.digest.DigestUtils; import org.apache.log4j.Logger; import org.jasypt.encryption.pbe.PBEStringEncryptor; import org.jasypt.encryption.pbe.StandardPBEStringEncryptor; @@ -70,4 +76,14 @@ public class EncryptionUtil { throw new CloudRuntimeException("unable to generate signature", e); } } + + public static String calculateChecksum(File file) { + try (InputStream is = Files.newInputStream(Paths.get(file.getPath()))) { + return DigestUtils.md5Hex(is); + } catch (IOException e) { + String errMsg = "Failed to calculate template checksum"; + s_logger.error(errMsg, e); + throw new CloudRuntimeException(errMsg, e); + } + } } diff --git a/utils/src/main/java/com/cloud/utils/ssh/SshHelper.java b/utils/src/main/java/com/cloud/utils/ssh/SshHelper.java index d5cd91af..6625864 100644 --- a/utils/src/main/java/com/cloud/utils/ssh/SshHelper.java +++ b/utils/src/main/java/com/cloud/utils/ssh/SshHelper.java @@ -52,6 +52,12 @@ public class SshHelper { scpTo(host, port, user, pemKeyFile, password, remoteTargetDirectory, localFile, fileMode, DEFAULT_CONNECT_TIMEOUT, DEFAULT_KEX_TIMEOUT); } + public static void scpTo(String host, int port, String user, File pemKeyFile, String password, String remoteTargetDirectory, String[] localFiles, String fileMode) + throws Exception { + + scpTo(host, port, user, pemKeyFile, password, remoteTargetDirectory, localFiles, fileMode, DEFAULT_CONNECT_TIMEOUT, DEFAULT_KEX_TIMEOUT); + } + public static void scpTo(String host, int port, String user, File pemKeyFile, String password, String remoteTargetDirectory, byte[] data, String remoteFileName, String fileMode) throws Exception { @@ -118,6 +124,42 @@ public class SshHelper { } } + public static void scpTo(String host, int port, String user, File pemKeyFile, String password, String remoteTargetDirectory, String[] localFiles, String fileMode, + int connectTimeoutInMs, int kexTimeoutInMs) throws Exception { + + com.trilead.ssh2.Connection conn = null; + com.trilead.ssh2.SCPClient scpClient = null; + + try { + conn = new com.trilead.ssh2.Connection(host, port); + conn.connect(null, connectTimeoutInMs, kexTimeoutInMs); + + if (pemKeyFile == null) { + if (!conn.authenticateWithPassword(user, password)) { + String msg = "Failed to authentication SSH user " + user + " on host " + host; + s_logger.error(msg); + throw new Exception(msg); + } + } else { + if (!conn.authenticateWithPublicKey(user, pemKeyFile, password)) { + String msg = "Failed to authentication SSH user " + user + " on host " + host; + s_logger.error(msg); + throw new Exception(msg); + } + } + + scpClient = conn.createSCPClient(); + + if (fileMode != null) + scpClient.put(localFiles, remoteTargetDirectory, fileMode); + else + scpClient.put(localFiles, remoteTargetDirectory); + } finally { + if (conn != null) + conn.close(); + } + } + public static void scpTo(String host, int port, String user, File pemKeyFile, String password, String remoteTargetDirectory, byte[] data, String remoteFileName, String fileMode, int connectTimeoutInMs, int kexTimeoutInMs) throws Exception {
