Pearl1594 commented on a change in pull request #4329: URL: https://github.com/apache/cloudstack/pull/4329#discussion_r695797473
########## File path: engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java ########## @@ -0,0 +1,801 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.upgrade; + +import com.cloud.hypervisor.Hypervisor; +import com.cloud.storage.Storage.ImageFormat; +import com.cloud.storage.VMTemplateVO; +import com.cloud.utils.DateUtil; +import com.cloud.utils.Pair; +import com.cloud.utils.UriUtils; +import com.cloud.utils.db.GlobalLock; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.script.Script; +import org.apache.log4j.Logger; +import org.ini4j.Ini; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileReader; +import java.io.IOException; +import java.net.URI; +import java.security.MessageDigest; +import java.sql.Connection; +import java.sql.Date; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; + +public class SystemVmTemplateRegistration { + private static final Logger LOGGER = Logger.getLogger(SystemVmTemplateRegistration.class); + private static final String MOUNT_COMMAND = "sudo mount -t nfs %s %s"; + private static final String UMOUNT_COMMAND = "sudo umount %s"; + private static final String HASH_ALGORITHM = "MD5"; + private static final String RELATIVE_TEMPLATE_PATH = "./engine/schema/dist/systemvm-templates/"; + private static final String ABSOLUTE_TEMPLATE_PATH = "/usr/share/cloudstack-management/templates/"; + private static final String TEMPLATES_PATH = fetchTemplatesPath(); + private static final String METADATA_FILE_NAME = "metadata.ini"; + private static final String METADATA_FILE = TEMPLATES_PATH + METADATA_FILE_NAME; + private static final String TEMPORARY_SECONDARY_STORE = "/tmp/tmpSecStorage"; + private static final String PARENT_TEMPLATE_FOLDER = TEMPORARY_SECONDARY_STORE; + private static final String PARTIAL_TEMPLATE_FOLDER = "/template/tmpl/1/"; + private static final String FETCH_DISTINCT_ELIGIBLE_ZONES = "SELECT DISTINCT(data_center_id) FROM `cloud`.`image_store` WHERE protocol = \"nfs\" AND role = \"Image\" AND removed is null"; + private static final String FETCH_DISTINCT_HYPERVISORS_IN_ZONE = "SELECT DISTINCT(hypervisor_type) FROM `cloud`.`cluster` where removed is null AND data_center_id=?"; + private static final String FETCH_IMAGE_STORE_PER_ZONE = "SELECT url,id FROM `cloud`.`image_store` WHERE data_center_id=? AND role = \"Image\" AND image_provider_name = \"NFS\" AND removed IS NULL LIMIT 1"; + private static final String INSERT_VM_TEMPLATE_TABLE = "INSERT INTO `cloud`.`vm_template` (uuid, unique_name, name, public, featured, created, type, hvm, bits, account_id, url, checksum, enable_password, display_text, format, guest_os_id, cross_zones, hypervisor_type, state, deploy_as_is)" + + "VALUES (?, ?, ?, 0, 0, ?, 'SYSTEM', 0, 64, 1, ?, ?, 0, ?, ?, ?, 1, ?, 'Inactive', ?)"; + private static final String INSERT_TEMPLATE_STORE_REF_TABLE = "INSERT INTO `cloud`.`template_store_ref` (store_id, template_id, created, last_updated, job_id, download_pct, download_state, error_str, local_path, install_path, url, state, destroyed, is_copy," + + " update_count, ref_cnt, store_role) VALUES (?, ?, ?, ?, NULL, 0, 'NOT_DOWNLOADED', NULL, NULL, ?, ?, 'Allocated', 0, 0, 0, 0, 'Image')"; + private static final String UPDATE_TEMPLATE_STORE_REF_TABLE = "UPDATE `cloud`.`template_store_ref` SET download_pct=100, download_state='DOWNLOADED', " + + "state='Ready', size=?, physical_size=?, last_updated=?, updated=? where template_id=?"; + private static final String UPDATE_VM_TEMPLATE_ENTRY = "UPDATE `cloud`.`vm_template` set size = ?, state = 'Active' where id = ?"; + private static final String UPDATE_CONFIGURATION_TABLE = "UPDATE `cloud`.`configuration` SET value = ? WHERE name = ?"; + private static final String UPDATE_TEMPLATE_TABLE_ON_FAILURE = "UPDATE `cloud`.`vm_template` set removed = ?, state = 'Inactive' where id = ?"; + private static final String DELETE_TEMPLATE_REF_RECORD_ON_FAILURE = "DELETE from `cloud`.`template_store_ref` where template_id = ?"; + private static final Integer SCRIPT_TIMEOUT = 1800000; + private static final Integer LOCK_WAIT_TIMEOUT = 1200; + public static String CS_MAJOR_VERSION = "4.16"; + public static String CS_TINY_VERSION = "0"; + + private static class SystemVMTemplateDetails { + Long id; + String uuid; + String name; + String uniqueName; + Date created; + String url; + String checksum; + ImageFormat format; + Integer guestOsId; + Hypervisor.HypervisorType hypervisorType; + Long storeId; + Long size; + Long physicalSize; + String installPath; + boolean deployAsIs; + Date updated; + + SystemVMTemplateDetails() { + } + + SystemVMTemplateDetails(String uuid, String name, Date created, String url, String checksum, + ImageFormat format, Integer guestOsId, Hypervisor.HypervisorType hypervisorType, + Long storeId) { + this.uuid = uuid; + this.name = name; + this.created = created; + this.url = url; + this.checksum = checksum; + this.format = format; + this.guestOsId = guestOsId; + this.hypervisorType = hypervisorType; + this.storeId = storeId; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getId() { + return id; + } + + public String getUuid() { + return uuid; + } + + public String getName() { + return name; + } + + public Date getCreated() { + return created; + } + + public String getUrl() { + return url; + } + + public String getChecksum() { + return checksum; + } + + public ImageFormat getFormat() { + return format; + } + + public Integer getGuestOsId() { + return guestOsId; + } + + public Hypervisor.HypervisorType getHypervisorType() { + return hypervisorType; + } + + public Long getStoreId() { + return storeId; + } + + public Long getSize() { + return size; + } + + public void setSize(Long size) { + this.size = size; + } + + public Long getPhysicalSize() { + return physicalSize; + } + + public void setPhysicalSize(Long physicalSize) { + this.physicalSize = physicalSize; + } + + public String getInstallPath() { + return installPath; + } + + public void setInstallPath(String installPath) { + this.installPath = installPath; + } + + public String getUniqueName() { + return uniqueName; + } + + public void setUniqueName(String uniqueName) { + this.uniqueName = uniqueName; + } + + public boolean isDeployAsIs() { + return deployAsIs; + } + + public void setDeployAsIs(boolean deployAsIs) { + this.deployAsIs = deployAsIs; + } + + public Date getUpdated() { + return updated; + } + + public void setUpdated(Date updated) { + this.updated = updated; + } + } + + public static final List<Hypervisor.HypervisorType> hypervisorList = Arrays.asList(Hypervisor.HypervisorType.KVM, + Hypervisor.HypervisorType.VMware, + Hypervisor.HypervisorType.XenServer, + Hypervisor.HypervisorType.Hyperv, + Hypervisor.HypervisorType.LXC, + Hypervisor.HypervisorType.Ovm3 + ); + + public static final Map<Hypervisor.HypervisorType, String> NewTemplateNameList = new HashMap<Hypervisor.HypervisorType, String>(); + public static final Map<Hypervisor.HypervisorType, String> fileNames = new HashMap<Hypervisor.HypervisorType, String>(); + public static final Map<Hypervisor.HypervisorType, String> newTemplateUrl = new HashMap<Hypervisor.HypervisorType, String>(); + public static final Map<Hypervisor.HypervisorType, String> newTemplateChecksum = new HashMap<Hypervisor.HypervisorType, String>(); + + public static final Map<Hypervisor.HypervisorType, String> routerTemplateConfigurationNames = new HashMap<Hypervisor.HypervisorType, String>() { + { + put(Hypervisor.HypervisorType.KVM, "router.template.kvm"); + put(Hypervisor.HypervisorType.VMware, "router.template.vmware"); + put(Hypervisor.HypervisorType.XenServer, "router.template.xenserver"); + put(Hypervisor.HypervisorType.Hyperv, "router.template.hyperv"); + put(Hypervisor.HypervisorType.LXC, "router.template.lxc"); + put(Hypervisor.HypervisorType.Ovm3, "router.template.ovm3"); + } + }; + + public static final Map<Hypervisor.HypervisorType, Integer> hypervisorGuestOsMap = new HashMap<Hypervisor.HypervisorType, Integer>() { + { + put(Hypervisor.HypervisorType.KVM, 15); + put(Hypervisor.HypervisorType.XenServer, 99); + put(Hypervisor.HypervisorType.VMware, 99); + put(Hypervisor.HypervisorType.Hyperv, 15); + put(Hypervisor.HypervisorType.LXC, 15); + put(Hypervisor.HypervisorType.Ovm3, 183); + } + }; + + public static final Map<Hypervisor.HypervisorType, ImageFormat> hypervisorImageFormat = new HashMap<Hypervisor.HypervisorType, ImageFormat>() { + { + put(Hypervisor.HypervisorType.KVM, ImageFormat.QCOW2); + put(Hypervisor.HypervisorType.XenServer, ImageFormat.VHD); + put(Hypervisor.HypervisorType.VMware, ImageFormat.OVA); + put(Hypervisor.HypervisorType.Hyperv, ImageFormat.VHD); + put(Hypervisor.HypervisorType.LXC, ImageFormat.QCOW2); + put(Hypervisor.HypervisorType.Ovm3, ImageFormat.RAW); + } + }; + + public static boolean validateIfSeeded(String url, String path) { + try { + mountStore(url); + int lastIdx = path.lastIndexOf(File.separator); + String partialDirPath = path.substring(0, lastIdx); + String templatePath = TEMPORARY_SECONDARY_STORE + File.separator + partialDirPath; + File templateProps = new File(templatePath + "/template.properties"); + if (templateProps.exists()) { + LOGGER.info("SystemVM template already seeded, skipping registration"); + return true; + } + LOGGER.info("SystemVM template not seeded"); + return false; + } catch (Exception e) { + throw new CloudRuntimeException("Failed to verify if the template is seeded", e); + } finally { + unmountStore(); + } + } + + private static String calculateChecksum(MessageDigest digest, File file) { + try { + FileInputStream fis = new FileInputStream(file); + byte[] byteArray = new byte[1024]; + int bytesCount = 0; + + while ((bytesCount = fis.read(byteArray)) != -1) { + digest.update(byteArray, 0, bytesCount); + } + + fis.close(); + byte[] bytes = digest.digest(); + + StringBuilder sb = new StringBuilder(); + for (byte aByte : bytes) { + sb.append(Integer + .toString((aByte & 0xff) + 0x100, 16) + .substring(1)); + } + return sb.toString(); + } catch (IOException e) { + String errMsg = String.format("Failed to calculate Checksum of template file: %s ", file.getName()); + LOGGER.error(errMsg, e); + throw new CloudRuntimeException(errMsg, e); + } + } + + public static long isTemplateAlreadyRegistered(Connection conn, Pair<Hypervisor.HypervisorType, String> hypervisorAndTemplateName) { + long templateId = -1; + try { + PreparedStatement pstmt = conn.prepareStatement("select id from `cloud`.`vm_template` where name = ? and removed is null order by id desc limit 1"); + // Get systemvm template id for corresponding hypervisor + pstmt.setString(1, hypervisorAndTemplateName.second()); + try (ResultSet rs = pstmt.executeQuery()) { + if (rs.next()) { + templateId = rs.getLong(1); + } + } catch (final SQLException e) { + String errMsg = "updateSystemVmTemplates: Exception caught while getting ids of SystemVM templates"; + LOGGER.error(errMsg, e); + throw new CloudRuntimeException(errMsg, e); + } + } catch (SQLException e) { + String errorMessage = "Unable to upgrade the database"; + LOGGER.error(errorMessage, e); + throw new CloudRuntimeException(errorMessage, e); + } + return templateId; + } + + private static String fetchTemplatesPath() { + String filePath = RELATIVE_TEMPLATE_PATH + METADATA_FILE_NAME; Review comment: The check for `./engine/schema/dist/systemvm-templates/"` path is for developer workflow - similar to this: https://github.com/apache/cloudstack/blob/main/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java#L7053 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@cloudstack.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
