Pearl1594 commented on a change in pull request #5264: URL: https://github.com/apache/cloudstack/pull/5264#discussion_r690963279
########## File path: engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java ########## @@ -0,0 +1,769 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.upgrade; + +import com.cloud.hypervisor.Hypervisor; +import com.cloud.utils.DateUtil; +import com.cloud.utils.Pair; +import com.cloud.utils.db.GlobalLock; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.script.Script; +import org.apache.log4j.Logger; +import org.ini4j.Ini; + +import javax.naming.ConfigurationException; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileReader; +import java.io.IOException; +import java.security.MessageDigest; +import java.sql.Connection; +import java.sql.Date; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; + +public class SystemVmTemplateRegistration { + private static final Logger LOGGER = Logger.getLogger(SystemVmTemplateRegistration.class); + private static final String mountCommand = "sudo mount -t nfs %s %s"; + private static final String umountCommand = "sudo umount %s"; + private static final String hashAlgorithm = "MD5"; + private static final String relativeTemplatePath = "./engine/schema/dist/systemvm-templates/"; + private static final String AbsolutetemplatesPath = "/usr/share/cloudstack-management/templates/"; + private static final String templatesPath = fetchTemplatesPath(); + private static final String metadataFileName = "metadata.ini"; + private static final String metadataFile = templatesPath + metadataFileName; + private static final String TEMPORARY_SECONDARY_STORE = "/tmp/tmpSecStorage"; + private static final String PARENT_TEMPLATE_FOLDER = TEMPORARY_SECONDARY_STORE; + private static final String PARTIAL_TEMPLATE_FOLDER = "/template/tmpl/1/"; + private static final String FETCH_FOLDER_NAME = "SELECT id FROM vm_template ORDER BY id DESC LIMIT 1;"; + // TODO: filter out only zones with NFS based 'Image' stores - to rule out image cache scenario + private static final String FETCH_DISTINCT_ELIGIBLE_ZONES = "SELECT DISTINCT(data_center_id) FROM `cloud`.`image_store` WHERE protocol = \"nfs\" AND removed is null"; + private static final String FETCH_DISTINCT_HYPERVISORS_IN_ZONE = "SELECT DISTINCT(hypervisor_type) FROM `cloud`.`cluster` where removed is null and data_center_id=?"; + private static final String FETCH_IMAGE_STORE_PER_ZONE = "SELECT url,id FROM `cloud`.`image_store` WHERE data_center_id=? AND removed IS NULL LIMIT 1"; + private static final String INSERT_VM_TEMPLATE_TABLE = "INSERT INTO `cloud`.`vm_template` (uuid, unique_name, name, public, featured, created, type, hvm, bits, account_id, url, checksum, enable_password, display_text, format, guest_os_id, cross_zones, hypervisor_type, state, deploy_as_is)" + + "VALUES (?, ?, ?, 0, 0, ?, 'SYSTEM', 0, 64, 1, ?, ?, 0, ?, ?, ?, 1, ?, 'Inactive', ?)"; + private static final String INSERT_TEMPLATE_STORE_REF_TABLE = "INSERT INTO `cloud`.`template_store_ref` (store_id, template_id, created, last_updated, job_id, download_pct, download_state, error_str, local_path, install_path, url, state, destroyed, is_copy," + + " update_count, ref_cnt, store_role) VALUES (?, ?, ?, ?, NULL, 0, 'NOT_DOWNLOADED', NULL, NULL, ?, ?, 'Allocated', 0, 0, 0, 0, 'Image')"; + private static final String UPDATE_TEMPLATE_STORE_REF_TABLE = "UPDATE template_store_ref SET download_pct=100, download_state='DOWNLOADED', " + + "state='Ready', size=?, physical_size=?, last_updated=?, updated=? where template_id=?"; + private static final String UPDATE_VM_TEMPLATE_ENTRY = "UPDATE vm_template set size = ?, state = 'Active' where id = ?"; + private static final String UPDATE_CONFIGURATION_TABLE = "UPDATE `cloud`.`configuration` SET value = ? WHERE name = ?"; + private static final String UPDATE_TEMPLATE_TABLE_ON_FAILURE = "UPDATE vm_template set removed = ?, state = 'Inactive' where id = ?"; + private static final String DELETE_TEMPLATE_REF_RECORD_ON_FAILURE = "DELETE from template_store_ref where template_id = ?"; + private static final Integer SCRIPT_TIMEOUT = 1800000; + public static String CS_MAJOR_VERSION = "4.16"; + public static String CS_TINY_VERSION = "0"; + + + private static class SystemVMTemplateDetails { + Long id; + String uuid; + String name; + String uniqueName; + Date created; + String url; + String checksum; + TemplateFormat format; + Integer guestOsId; + Hypervisor.HypervisorType hypervisorType; + Long storeId; + Long size; + Long physicalSize; + String installPath; + boolean deployAsIs; + Date updated; + + SystemVMTemplateDetails() { + } + + SystemVMTemplateDetails(String uuid, String name, Date created, String url, String checksum, + TemplateFormat format, Integer guestOsId, Hypervisor.HypervisorType hypervisorType, + Long storeId) { + this.uuid = uuid; + this.name = name; + this.created = created; + this.url = url; + this.checksum = checksum; + this.format = format; + this.guestOsId = guestOsId; + this.hypervisorType = hypervisorType; + this.storeId = storeId; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getId() { + return id; + } + + public String getUuid() { + return uuid; + } + + public String getName() { + return name; + } + + public Date getCreated() { + return created; + } + + public String getUrl() { + return url; + } + + public String getChecksum() { + return checksum; + } + + public TemplateFormat getFormat() { + return format; + } + + public Integer getGuestOsId() { + return guestOsId; + } + + public Hypervisor.HypervisorType getHypervisorType() { + return hypervisorType; + } + + public Long getStoreId() { + return storeId; + } + + public Long getSize() { + return size; + } + + public void setSize(Long size) { + this.size = size; + } + + public Long getPhysicalSize() { + return physicalSize; + } + + public void setPhysicalSize(Long physicalSize) { + this.physicalSize = physicalSize; + } + + public String getInstallPath() { + return installPath; + } + + public void setInstallPath(String installPath) { + this.installPath = installPath; + } + + public String getUniqueName() { + return uniqueName; + } + + public void setUniqueName(String uniqueName) { + this.uniqueName = uniqueName; + } + + public boolean isDeployAsIs() { + return deployAsIs; + } + + public void setDeployAsIs(boolean deployAsIs) { + this.deployAsIs = deployAsIs; + } + + public Date getUpdated() { + return updated; + } + + public void setUpdated(Date updated) { + this.updated = updated; + } + } + + public static final List<Hypervisor.HypervisorType> hypervisorList = Arrays.asList(Hypervisor.HypervisorType.KVM, + Hypervisor.HypervisorType.VMware, + Hypervisor.HypervisorType.XenServer, + Hypervisor.HypervisorType.Hyperv, + Hypervisor.HypervisorType.LXC, + Hypervisor.HypervisorType.Ovm3 + ); + + public static final Map<Hypervisor.HypervisorType, String> NewTemplateNameList = new HashMap<Hypervisor.HypervisorType, String>(); + public static final Map<Hypervisor.HypervisorType, String> fileNames = new HashMap<Hypervisor.HypervisorType, String>(); + public static final Map<Hypervisor.HypervisorType, String> newTemplateUrl = new HashMap<Hypervisor.HypervisorType, String>(); + public static final Map<Hypervisor.HypervisorType, String> newTemplateChecksum = new HashMap<Hypervisor.HypervisorType, String>(); + + public static final Map<Hypervisor.HypervisorType, String> routerTemplateConfigurationNames = new HashMap<Hypervisor.HypervisorType, String>() { + { + put(Hypervisor.HypervisorType.KVM, "router.template.kvm"); + put(Hypervisor.HypervisorType.VMware, "router.template.vmware"); + put(Hypervisor.HypervisorType.XenServer, "router.template.xenserver"); + put(Hypervisor.HypervisorType.Hyperv, "router.template.hyperv"); + put(Hypervisor.HypervisorType.LXC, "router.template.lxc"); + put(Hypervisor.HypervisorType.Ovm3, "router.template.ovm3"); + } + }; + + public static final Map<Hypervisor.HypervisorType, Integer> hypervisorGuestOsMap = new HashMap<Hypervisor.HypervisorType, Integer>() { + { + put(Hypervisor.HypervisorType.KVM, 15); + put(Hypervisor.HypervisorType.XenServer, 99); + put(Hypervisor.HypervisorType.VMware, 99); + put(Hypervisor.HypervisorType.Hyperv, 15); + put(Hypervisor.HypervisorType.LXC, 15); + put(Hypervisor.HypervisorType.Ovm3, 183); + } + }; + + public static enum TemplateFormat{ + QCOW2("qcow2"), + RAW("raw"), + VHD("vhd"), + OVA("ova"); + + private final String fileExtension; + + TemplateFormat(String fileExtension) { + this.fileExtension = fileExtension; + } + } + + public static final Map<Hypervisor.HypervisorType, TemplateFormat> hypervisorImageFormat = new HashMap<Hypervisor.HypervisorType, TemplateFormat>() { + { + put(Hypervisor.HypervisorType.KVM, TemplateFormat.QCOW2); + put(Hypervisor.HypervisorType.XenServer, TemplateFormat.VHD); + put(Hypervisor.HypervisorType.VMware, TemplateFormat.OVA); + put(Hypervisor.HypervisorType.Hyperv, TemplateFormat.VHD); + put(Hypervisor.HypervisorType.LXC, TemplateFormat.QCOW2); + put(Hypervisor.HypervisorType.Ovm3, TemplateFormat.RAW); + } + }; + + public static boolean validateIfSeeded(String url, String path) { + try { + mountStore(url); + int lastIdx = path.lastIndexOf('/'); + String partialDirPath = path.substring(0, lastIdx); + String templatePath = TEMPORARY_SECONDARY_STORE + "/" + partialDirPath; + File templateProps = new File(templatePath + "/template.properties"); + if (templateProps.exists()) { + LOGGER.info("SystemVM template already seeded, skipping registration"); + return true; + } + LOGGER.info("SystemVM template not seeded"); + return false; + } catch (Exception e) { + throw new CloudRuntimeException("Failed to verify if the template is seeded"); + } finally { + unmountStore(); + } + } + + private static String calculateChecksum(MessageDigest digest, File file) { + try { + FileInputStream fis = new FileInputStream(file); + byte[] byteArray = new byte[1024]; + int bytesCount = 0; + + while ((bytesCount = fis.read(byteArray)) != -1) { + digest.update(byteArray, 0, bytesCount); + } + + fis.close(); + byte[] bytes = digest.digest(); + + StringBuilder sb = new StringBuilder(); + for (byte aByte : bytes) { + sb.append(Integer + .toString((aByte & 0xff) + 0x100, 16) + .substring(1)); + } + return sb.toString(); + } catch (IOException e) { + String errMsg = String.format("Failed to calculate Checksum of template file: %s due to: %s ", file.getName(), e); + LOGGER.error(errMsg); + throw new CloudRuntimeException(errMsg); + } + } + + public static long isTemplateAlreadyRegistered(Connection conn, Pair<Hypervisor.HypervisorType, String> hypervisorAndTemplateName) { + long templateId = -1; + try { + PreparedStatement pstmt = conn.prepareStatement("select id from `cloud`.`vm_template` where name = ? and removed is null order by id desc limit 1"); + // Get systemvm template id for corresponding hypervisor + pstmt.setString(1, hypervisorAndTemplateName.second()); + try (ResultSet rs = pstmt.executeQuery()) { + if (rs.next()) { + templateId = rs.getLong(1); + } + } catch (final SQLException e) { + String errMsg = String.format("updateSystemVmTemplates: Exception caught while getting ids of SystemVM templates: %s ", e.getMessage()); + LOGGER.error(errMsg); + throw new CloudRuntimeException(errMsg); + } + } catch (SQLException e) { + String errorMessage = "Unable to upgrade the database"; + LOGGER.error(errorMessage, e); + throw new CloudRuntimeException(errorMessage, e); + } + return templateId; + } + + private static String fetchTemplatesPath() { + String filePath = relativeTemplatePath + metadataFileName; + LOGGER.debug("Looking for file [" + filePath + "] in the classpath."); + File metaFile = new File(filePath); + String templatePath = null; + if (metaFile.exists()) { + templatePath = relativeTemplatePath; + } + if (templatePath == null) { + filePath = AbsolutetemplatesPath + metadataFileName; + metaFile = new File(filePath); + templatePath = AbsolutetemplatesPath; + LOGGER.debug("Looking for file [" + filePath + "] in the classpath."); + if (!metaFile.exists()) { + String errMsg = String.format("Unable to locate metadata file in your setup at %s", filePath.toString()); + LOGGER.error(errMsg); + throw new CloudRuntimeException(errMsg); + } + } + return templatePath; + } + + private static String getHypervisorName(String name) { + if (name.equals("xenserver")) { + return "xen"; + } + if (name.equals("ovm3")) { + return "ovm"; + } + return name; + + } + + private static Hypervisor.HypervisorType getHypervisorType(String hypervisor) { + if (hypervisor.equalsIgnoreCase("xen")) { + hypervisor = "xenserver"; + } else if (hypervisor.equalsIgnoreCase("ovm")) { + hypervisor = "ovm3"; + } + return Hypervisor.HypervisorType.getType(hypervisor); + } + + private static List<Long> getEligibleZoneIds(Connection conn) { + List<Long> zones = new ArrayList<Long>(); + try { + PreparedStatement pstmt = conn.prepareStatement(FETCH_DISTINCT_ELIGIBLE_ZONES); + ResultSet rs = pstmt.executeQuery(); + while (rs.next()) { + zones.add(rs.getLong(1)); + } + } catch (SQLException e) { + String errMsg = "Failed to fetch eligible zones for SystemVM template registration due to: %s"; + LOGGER.error(String.format(errMsg, e.getMessage())); + throw new CloudRuntimeException("Failed to fetch eligible zones for SystemVM template registration"); + } + return zones; + } + + private static Pair<String, Long> getNfsStoreInZone(Connection conn, Long zoneId) { + String url = null; + Long storeId = null; + try { + PreparedStatement pstmt = conn.prepareStatement(FETCH_IMAGE_STORE_PER_ZONE); + if(pstmt != null) { + pstmt.setLong(1, zoneId); + ResultSet resultSet = pstmt.executeQuery(); + while (resultSet.next()) { + url = resultSet.getString(1); + storeId = resultSet.getLong(2); + } + } + } catch (SQLException e) { + String errMsg = String.format("Failed to fetch NFS store in zone = %s for SystemVM template registration", zoneId); + LOGGER.error(errMsg + String.format("due to: %s", e.getMessage())); + throw new CloudRuntimeException(errMsg); + } + return new Pair<>(url, storeId); + } + + public static void mountStore(String storeUrl) { + try { + if (storeUrl != null) { + String path = storeUrl.split("://")[1]; Review comment: Addressed with https://github.com/apache/cloudstack/pull/4329/commits/c91c39652d301c9bfc9dfd227531e3b262d87218#diff-a9c9a38684718059c060de404bf9529de96e502f1e81b30793e6a32f725042a9L409-R412 ########## File path: engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java ########## @@ -0,0 +1,769 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.upgrade; + +import com.cloud.hypervisor.Hypervisor; +import com.cloud.utils.DateUtil; +import com.cloud.utils.Pair; +import com.cloud.utils.db.GlobalLock; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.script.Script; +import org.apache.log4j.Logger; +import org.ini4j.Ini; + +import javax.naming.ConfigurationException; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileReader; +import java.io.IOException; +import java.security.MessageDigest; +import java.sql.Connection; +import java.sql.Date; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; + +public class SystemVmTemplateRegistration { + private static final Logger LOGGER = Logger.getLogger(SystemVmTemplateRegistration.class); + private static final String mountCommand = "sudo mount -t nfs %s %s"; + private static final String umountCommand = "sudo umount %s"; + private static final String hashAlgorithm = "MD5"; + private static final String relativeTemplatePath = "./engine/schema/dist/systemvm-templates/"; + private static final String AbsolutetemplatesPath = "/usr/share/cloudstack-management/templates/"; + private static final String templatesPath = fetchTemplatesPath(); + private static final String metadataFileName = "metadata.ini"; + private static final String metadataFile = templatesPath + metadataFileName; + private static final String TEMPORARY_SECONDARY_STORE = "/tmp/tmpSecStorage"; + private static final String PARENT_TEMPLATE_FOLDER = TEMPORARY_SECONDARY_STORE; + private static final String PARTIAL_TEMPLATE_FOLDER = "/template/tmpl/1/"; Review comment: The plan was to align with the approach followed when we perform seeding via cloud-install-sys-tmplt, where we'd always use system account to be the owner of the template ########## File path: engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java ########## @@ -0,0 +1,769 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.upgrade; + +import com.cloud.hypervisor.Hypervisor; +import com.cloud.utils.DateUtil; +import com.cloud.utils.Pair; +import com.cloud.utils.db.GlobalLock; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.script.Script; +import org.apache.log4j.Logger; +import org.ini4j.Ini; + +import javax.naming.ConfigurationException; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileReader; +import java.io.IOException; +import java.security.MessageDigest; +import java.sql.Connection; +import java.sql.Date; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; + +public class SystemVmTemplateRegistration { + private static final Logger LOGGER = Logger.getLogger(SystemVmTemplateRegistration.class); + private static final String mountCommand = "sudo mount -t nfs %s %s"; + private static final String umountCommand = "sudo umount %s"; + private static final String hashAlgorithm = "MD5"; + private static final String relativeTemplatePath = "./engine/schema/dist/systemvm-templates/"; + private static final String AbsolutetemplatesPath = "/usr/share/cloudstack-management/templates/"; + private static final String templatesPath = fetchTemplatesPath(); + private static final String metadataFileName = "metadata.ini"; + private static final String metadataFile = templatesPath + metadataFileName; + private static final String TEMPORARY_SECONDARY_STORE = "/tmp/tmpSecStorage"; Review comment: Is there a need to have randomness in the naming of the temporary directory where we mount the secondary stores? ########## File path: engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java ########## @@ -0,0 +1,769 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.upgrade; + +import com.cloud.hypervisor.Hypervisor; +import com.cloud.utils.DateUtil; +import com.cloud.utils.Pair; +import com.cloud.utils.db.GlobalLock; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.script.Script; +import org.apache.log4j.Logger; +import org.ini4j.Ini; + +import javax.naming.ConfigurationException; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileReader; +import java.io.IOException; +import java.security.MessageDigest; +import java.sql.Connection; +import java.sql.Date; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; + +public class SystemVmTemplateRegistration { + private static final Logger LOGGER = Logger.getLogger(SystemVmTemplateRegistration.class); + private static final String mountCommand = "sudo mount -t nfs %s %s"; + private static final String umountCommand = "sudo umount %s"; + private static final String hashAlgorithm = "MD5"; + private static final String relativeTemplatePath = "./engine/schema/dist/systemvm-templates/"; + private static final String AbsolutetemplatesPath = "/usr/share/cloudstack-management/templates/"; + private static final String templatesPath = fetchTemplatesPath(); + private static final String metadataFileName = "metadata.ini"; + private static final String metadataFile = templatesPath + metadataFileName; + private static final String TEMPORARY_SECONDARY_STORE = "/tmp/tmpSecStorage"; + private static final String PARENT_TEMPLATE_FOLDER = TEMPORARY_SECONDARY_STORE; + private static final String PARTIAL_TEMPLATE_FOLDER = "/template/tmpl/1/"; + private static final String FETCH_FOLDER_NAME = "SELECT id FROM vm_template ORDER BY id DESC LIMIT 1;"; + // TODO: filter out only zones with NFS based 'Image' stores - to rule out image cache scenario + private static final String FETCH_DISTINCT_ELIGIBLE_ZONES = "SELECT DISTINCT(data_center_id) FROM `cloud`.`image_store` WHERE protocol = \"nfs\" AND removed is null"; + private static final String FETCH_DISTINCT_HYPERVISORS_IN_ZONE = "SELECT DISTINCT(hypervisor_type) FROM `cloud`.`cluster` where removed is null and data_center_id=?"; + private static final String FETCH_IMAGE_STORE_PER_ZONE = "SELECT url,id FROM `cloud`.`image_store` WHERE data_center_id=? AND removed IS NULL LIMIT 1"; + private static final String INSERT_VM_TEMPLATE_TABLE = "INSERT INTO `cloud`.`vm_template` (uuid, unique_name, name, public, featured, created, type, hvm, bits, account_id, url, checksum, enable_password, display_text, format, guest_os_id, cross_zones, hypervisor_type, state, deploy_as_is)" + + "VALUES (?, ?, ?, 0, 0, ?, 'SYSTEM', 0, 64, 1, ?, ?, 0, ?, ?, ?, 1, ?, 'Inactive', ?)"; + private static final String INSERT_TEMPLATE_STORE_REF_TABLE = "INSERT INTO `cloud`.`template_store_ref` (store_id, template_id, created, last_updated, job_id, download_pct, download_state, error_str, local_path, install_path, url, state, destroyed, is_copy," + + " update_count, ref_cnt, store_role) VALUES (?, ?, ?, ?, NULL, 0, 'NOT_DOWNLOADED', NULL, NULL, ?, ?, 'Allocated', 0, 0, 0, 0, 'Image')"; + private static final String UPDATE_TEMPLATE_STORE_REF_TABLE = "UPDATE template_store_ref SET download_pct=100, download_state='DOWNLOADED', " + + "state='Ready', size=?, physical_size=?, last_updated=?, updated=? where template_id=?"; + private static final String UPDATE_VM_TEMPLATE_ENTRY = "UPDATE vm_template set size = ?, state = 'Active' where id = ?"; + private static final String UPDATE_CONFIGURATION_TABLE = "UPDATE `cloud`.`configuration` SET value = ? WHERE name = ?"; + private static final String UPDATE_TEMPLATE_TABLE_ON_FAILURE = "UPDATE vm_template set removed = ?, state = 'Inactive' where id = ?"; + private static final String DELETE_TEMPLATE_REF_RECORD_ON_FAILURE = "DELETE from template_store_ref where template_id = ?"; + private static final Integer SCRIPT_TIMEOUT = 1800000; + public static String CS_MAJOR_VERSION = "4.16"; + public static String CS_TINY_VERSION = "0"; + + + private static class SystemVMTemplateDetails { + Long id; + String uuid; + String name; + String uniqueName; + Date created; + String url; + String checksum; + TemplateFormat format; + Integer guestOsId; + Hypervisor.HypervisorType hypervisorType; + Long storeId; + Long size; + Long physicalSize; + String installPath; + boolean deployAsIs; + Date updated; + + SystemVMTemplateDetails() { + } + + SystemVMTemplateDetails(String uuid, String name, Date created, String url, String checksum, + TemplateFormat format, Integer guestOsId, Hypervisor.HypervisorType hypervisorType, + Long storeId) { + this.uuid = uuid; + this.name = name; + this.created = created; + this.url = url; + this.checksum = checksum; + this.format = format; + this.guestOsId = guestOsId; + this.hypervisorType = hypervisorType; + this.storeId = storeId; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getId() { + return id; + } + + public String getUuid() { + return uuid; + } + + public String getName() { + return name; + } + + public Date getCreated() { + return created; + } + + public String getUrl() { + return url; + } + + public String getChecksum() { + return checksum; + } + + public TemplateFormat getFormat() { + return format; + } + + public Integer getGuestOsId() { + return guestOsId; + } + + public Hypervisor.HypervisorType getHypervisorType() { + return hypervisorType; + } + + public Long getStoreId() { + return storeId; + } + + public Long getSize() { + return size; + } + + public void setSize(Long size) { + this.size = size; + } + + public Long getPhysicalSize() { + return physicalSize; + } + + public void setPhysicalSize(Long physicalSize) { + this.physicalSize = physicalSize; + } + + public String getInstallPath() { + return installPath; + } + + public void setInstallPath(String installPath) { + this.installPath = installPath; + } + + public String getUniqueName() { + return uniqueName; + } + + public void setUniqueName(String uniqueName) { + this.uniqueName = uniqueName; + } + + public boolean isDeployAsIs() { + return deployAsIs; + } + + public void setDeployAsIs(boolean deployAsIs) { + this.deployAsIs = deployAsIs; + } + + public Date getUpdated() { + return updated; + } + + public void setUpdated(Date updated) { + this.updated = updated; + } + } + + public static final List<Hypervisor.HypervisorType> hypervisorList = Arrays.asList(Hypervisor.HypervisorType.KVM, + Hypervisor.HypervisorType.VMware, + Hypervisor.HypervisorType.XenServer, + Hypervisor.HypervisorType.Hyperv, + Hypervisor.HypervisorType.LXC, + Hypervisor.HypervisorType.Ovm3 + ); + + public static final Map<Hypervisor.HypervisorType, String> NewTemplateNameList = new HashMap<Hypervisor.HypervisorType, String>(); + public static final Map<Hypervisor.HypervisorType, String> fileNames = new HashMap<Hypervisor.HypervisorType, String>(); + public static final Map<Hypervisor.HypervisorType, String> newTemplateUrl = new HashMap<Hypervisor.HypervisorType, String>(); + public static final Map<Hypervisor.HypervisorType, String> newTemplateChecksum = new HashMap<Hypervisor.HypervisorType, String>(); + + public static final Map<Hypervisor.HypervisorType, String> routerTemplateConfigurationNames = new HashMap<Hypervisor.HypervisorType, String>() { + { + put(Hypervisor.HypervisorType.KVM, "router.template.kvm"); + put(Hypervisor.HypervisorType.VMware, "router.template.vmware"); + put(Hypervisor.HypervisorType.XenServer, "router.template.xenserver"); + put(Hypervisor.HypervisorType.Hyperv, "router.template.hyperv"); + put(Hypervisor.HypervisorType.LXC, "router.template.lxc"); + put(Hypervisor.HypervisorType.Ovm3, "router.template.ovm3"); + } + }; + + public static final Map<Hypervisor.HypervisorType, Integer> hypervisorGuestOsMap = new HashMap<Hypervisor.HypervisorType, Integer>() { + { + put(Hypervisor.HypervisorType.KVM, 15); + put(Hypervisor.HypervisorType.XenServer, 99); + put(Hypervisor.HypervisorType.VMware, 99); + put(Hypervisor.HypervisorType.Hyperv, 15); + put(Hypervisor.HypervisorType.LXC, 15); + put(Hypervisor.HypervisorType.Ovm3, 183); + } + }; + + public static enum TemplateFormat{ + QCOW2("qcow2"), + RAW("raw"), + VHD("vhd"), + OVA("ova"); + + private final String fileExtension; + + TemplateFormat(String fileExtension) { + this.fileExtension = fileExtension; + } + } + + public static final Map<Hypervisor.HypervisorType, TemplateFormat> hypervisorImageFormat = new HashMap<Hypervisor.HypervisorType, TemplateFormat>() { + { + put(Hypervisor.HypervisorType.KVM, TemplateFormat.QCOW2); + put(Hypervisor.HypervisorType.XenServer, TemplateFormat.VHD); + put(Hypervisor.HypervisorType.VMware, TemplateFormat.OVA); + put(Hypervisor.HypervisorType.Hyperv, TemplateFormat.VHD); + put(Hypervisor.HypervisorType.LXC, TemplateFormat.QCOW2); + put(Hypervisor.HypervisorType.Ovm3, TemplateFormat.RAW); + } + }; + + public static boolean validateIfSeeded(String url, String path) { + try { + mountStore(url); + int lastIdx = path.lastIndexOf('/'); + String partialDirPath = path.substring(0, lastIdx); + String templatePath = TEMPORARY_SECONDARY_STORE + "/" + partialDirPath; + File templateProps = new File(templatePath + "/template.properties"); + if (templateProps.exists()) { + LOGGER.info("SystemVM template already seeded, skipping registration"); + return true; + } + LOGGER.info("SystemVM template not seeded"); + return false; + } catch (Exception e) { + throw new CloudRuntimeException("Failed to verify if the template is seeded"); + } finally { + unmountStore(); + } + } + + private static String calculateChecksum(MessageDigest digest, File file) { + try { + FileInputStream fis = new FileInputStream(file); + byte[] byteArray = new byte[1024]; + int bytesCount = 0; + + while ((bytesCount = fis.read(byteArray)) != -1) { + digest.update(byteArray, 0, bytesCount); + } + + fis.close(); + byte[] bytes = digest.digest(); + + StringBuilder sb = new StringBuilder(); + for (byte aByte : bytes) { + sb.append(Integer + .toString((aByte & 0xff) + 0x100, 16) + .substring(1)); + } + return sb.toString(); + } catch (IOException e) { + String errMsg = String.format("Failed to calculate Checksum of template file: %s due to: %s ", file.getName(), e); + LOGGER.error(errMsg); + throw new CloudRuntimeException(errMsg); + } + } + + public static long isTemplateAlreadyRegistered(Connection conn, Pair<Hypervisor.HypervisorType, String> hypervisorAndTemplateName) { + long templateId = -1; + try { + PreparedStatement pstmt = conn.prepareStatement("select id from `cloud`.`vm_template` where name = ? and removed is null order by id desc limit 1"); + // Get systemvm template id for corresponding hypervisor + pstmt.setString(1, hypervisorAndTemplateName.second()); + try (ResultSet rs = pstmt.executeQuery()) { + if (rs.next()) { + templateId = rs.getLong(1); + } + } catch (final SQLException e) { + String errMsg = String.format("updateSystemVmTemplates: Exception caught while getting ids of SystemVM templates: %s ", e.getMessage()); + LOGGER.error(errMsg); + throw new CloudRuntimeException(errMsg); + } + } catch (SQLException e) { + String errorMessage = "Unable to upgrade the database"; + LOGGER.error(errorMessage, e); + throw new CloudRuntimeException(errorMessage, e); + } + return templateId; + } + + private static String fetchTemplatesPath() { + String filePath = relativeTemplatePath + metadataFileName; + LOGGER.debug("Looking for file [" + filePath + "] in the classpath."); + File metaFile = new File(filePath); + String templatePath = null; + if (metaFile.exists()) { + templatePath = relativeTemplatePath; + } + if (templatePath == null) { + filePath = AbsolutetemplatesPath + metadataFileName; + metaFile = new File(filePath); + templatePath = AbsolutetemplatesPath; + LOGGER.debug("Looking for file [" + filePath + "] in the classpath."); + if (!metaFile.exists()) { + String errMsg = String.format("Unable to locate metadata file in your setup at %s", filePath.toString()); + LOGGER.error(errMsg); + throw new CloudRuntimeException(errMsg); + } + } + return templatePath; + } + + private static String getHypervisorName(String name) { + if (name.equals("xenserver")) { + return "xen"; + } + if (name.equals("ovm3")) { + return "ovm"; + } + return name; + + } + + private static Hypervisor.HypervisorType getHypervisorType(String hypervisor) { + if (hypervisor.equalsIgnoreCase("xen")) { + hypervisor = "xenserver"; + } else if (hypervisor.equalsIgnoreCase("ovm")) { + hypervisor = "ovm3"; + } + return Hypervisor.HypervisorType.getType(hypervisor); + } + + private static List<Long> getEligibleZoneIds(Connection conn) { + List<Long> zones = new ArrayList<Long>(); + try { + PreparedStatement pstmt = conn.prepareStatement(FETCH_DISTINCT_ELIGIBLE_ZONES); + ResultSet rs = pstmt.executeQuery(); + while (rs.next()) { + zones.add(rs.getLong(1)); + } + } catch (SQLException e) { + String errMsg = "Failed to fetch eligible zones for SystemVM template registration due to: %s"; + LOGGER.error(String.format(errMsg, e.getMessage())); + throw new CloudRuntimeException("Failed to fetch eligible zones for SystemVM template registration"); + } + return zones; + } + + private static Pair<String, Long> getNfsStoreInZone(Connection conn, Long zoneId) { + String url = null; + Long storeId = null; + try { + PreparedStatement pstmt = conn.prepareStatement(FETCH_IMAGE_STORE_PER_ZONE); + if(pstmt != null) { + pstmt.setLong(1, zoneId); + ResultSet resultSet = pstmt.executeQuery(); + while (resultSet.next()) { + url = resultSet.getString(1); + storeId = resultSet.getLong(2); + } + } + } catch (SQLException e) { + String errMsg = String.format("Failed to fetch NFS store in zone = %s for SystemVM template registration", zoneId); + LOGGER.error(errMsg + String.format("due to: %s", e.getMessage())); + throw new CloudRuntimeException(errMsg); + } + return new Pair<>(url, storeId); + } + + public static void mountStore(String storeUrl) { + try { + if (storeUrl != null) { + String path = storeUrl.split("://")[1]; + int index = path.indexOf('/'); + String host = path.substring(0, index); + String mountPath = path.substring(index); + Script.runSimpleBashScript("mkdir -p " + TEMPORARY_SECONDARY_STORE); + String mount = String.format(mountCommand, host + ":" + mountPath, TEMPORARY_SECONDARY_STORE); + Script.runSimpleBashScript(mount); + } + } catch (Exception e) { + String msg = "NFS Store URL is not in the correct format"; + LOGGER.error(msg); + throw new CloudRuntimeException(msg); + + } + } + + private static List<String> fetchAllHypervisors(Connection conn, Long zoneId) { + List<String> hypervisorList = new ArrayList<>(); + try { + PreparedStatement pstmt = conn.prepareStatement(FETCH_DISTINCT_HYPERVISORS_IN_ZONE); + if(pstmt != null) { + pstmt.setLong(1, zoneId); + ResultSet resultSet = pstmt.executeQuery(); + while (resultSet.next()) { + hypervisorList.add(resultSet.getString(1)); + } + } + } catch (SQLException e) { + String errMsg = String.format("Failed to fetch distinct hypervisors in zone: %s for SystemVM template registration", zoneId); + LOGGER.error(errMsg + String.format("due to: %s", e.getMessage())); + throw new CloudRuntimeException(errMsg); + } + return hypervisorList; + } + + private static Long createTemplateObjectInDB(Connection conn, SystemVMTemplateDetails details) { + Long id = null; + try { + PreparedStatement pstmt = conn.prepareStatement(INSERT_VM_TEMPLATE_TABLE); + if (pstmt != null) { + int i = 1; + pstmt.setString(i++, details.getUuid()); + pstmt.setString(i++, details.getUuid()); + pstmt.setString(i++, details.getName()); + pstmt.setDate(i++, details.getCreated()); + pstmt.setString(i++, details.getUrl()); + pstmt.setString(i++, details.getChecksum()); + pstmt.setString(i++, details.getName()); + pstmt.setString(i++, details.getFormat().toString()); + pstmt.setLong(i++, details.getGuestOsId()); + pstmt.setString(i++, details.getHypervisorType().toString()); + pstmt.setBoolean(i++, details.getHypervisorType() == Hypervisor.HypervisorType.VMware); + pstmt.executeUpdate(); + + pstmt = conn.prepareStatement("SELECT id FROM vm_template ORDER BY id DESC LIMIT 1"); + try (ResultSet rs = pstmt.executeQuery()) { + if (rs.next()) { + id = rs.getLong(1); + } + } catch (final SQLException e) { + String errMsg = String.format("Failed to fetch template id %s", e.getMessage()); + LOGGER.error(errMsg); + throw new CloudRuntimeException(errMsg); + } + } + } catch (Exception e) { Review comment: I thought it seemed appropriate, do you see any adverse effect with this approach? ########## File path: debian/rules ########## @@ -66,17 +66,21 @@ override_dh_auto_install: mkdir -p $(DESTDIR)/usr/share/$(PACKAGE)-management mkdir -p $(DESTDIR)/usr/share/$(PACKAGE)-management/lib mkdir -p $(DESTDIR)/usr/share/$(PACKAGE)-management/setup + mkdir -p $(DESTDIR)/usr/share/$(PACKAGE)-management/templates mkdir $(DESTDIR)/var/log/$(PACKAGE)/management mkdir $(DESTDIR)/var/cache/$(PACKAGE)/management mkdir $(DESTDIR)/var/log/$(PACKAGE)/ipallocator mkdir $(DESTDIR)/var/lib/$(PACKAGE)/management mkdir $(DESTDIR)/var/lib/$(PACKAGE)/mnt + cp -r client/target/utilities/scripts/db/* $(DESTDIR)/usr/share/$(PACKAGE)-management/setup/ cp -r client/target/classes/META-INF/webapp $(DESTDIR)/usr/share/$(PACKAGE)-management/webapp cp server/target/conf/* $(DESTDIR)/$(SYSCONFDIR)/$(PACKAGE)/server/ cp client/target/conf/* $(DESTDIR)/$(SYSCONFDIR)/$(PACKAGE)/management/ cp client/target/cloud-client-ui-$(VERSION).jar $(DESTDIR)/usr/share/$(PACKAGE)-management/lib/cloudstack-$(VERSION).jar cp client/target/lib/*jar $(DESTDIR)/usr/share/$(PACKAGE)-management/lib/ + cp -r engine/schema/dist/systemvm-templates/* $(DESTDIR)/usr/share/$(PACKAGE)-management/templates/ Review comment: yes, the size of cloudstack-management pkg has increased to 1.2GB , I presume this was a known side-effect of this feature. ########## File path: engine/schema/templateConfig.sh ########## @@ -0,0 +1,84 @@ +#!/bin/bash Review comment: placed it here, as it the files generated by this script is consumed by SystemVMTemplateRegistration to populate details such as checksum, download path, template name, etc during upgrade -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
