This is an automated email from the ASF dual-hosted git repository. harikrishna pushed a commit to branch LiveStoragMigrationScaleIOMain in repository https://gitbox.apache.org/repos/asf/cloudstack.git
commit 61b451a20cf0148340e802196fc42d559497b130 Author: Harikrishna Patnala <[email protected]> AuthorDate: Wed Feb 22 11:04:19 2023 +0530 Recent changes of migration across clusters --- .../LibvirtMigrateVolumeCommandWrapper.java | 121 ++++++++++++++++++++- .../datastore/client/ScaleIOGatewayClientImpl.java | 2 +- .../driver/ScaleIOPrimaryDataStoreDriver.java | 68 +++++++++++- 3 files changed, 188 insertions(+), 3 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java index 688bcce48c3..420df975838 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java @@ -23,19 +23,35 @@ import com.cloud.agent.api.Answer; import com.cloud.agent.api.storage.MigrateVolumeAnswer; import com.cloud.agent.api.storage.MigrateVolumeCommand; import com.cloud.agent.api.to.DiskTO; +import com.cloud.exception.InternalErrorException; import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.hypervisor.kvm.resource.LibvirtDomainXMLParser; +import com.cloud.hypervisor.kvm.resource.LibvirtVMDef; import com.cloud.hypervisor.kvm.storage.KVMPhysicalDisk; import com.cloud.hypervisor.kvm.storage.KVMStoragePool; import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; import com.cloud.resource.CommandWrapper; import com.cloud.resource.ResourceWrapper; +import java.io.File; +import java.util.List; import java.util.Map; import java.util.UUID; +import com.cloud.storage.Storage; +import org.apache.cloudstack.storage.datastore.client.ScaleIOGatewayClient; +import org.apache.cloudstack.storage.datastore.util.ScaleIOUtil; import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; import org.apache.cloudstack.storage.to.VolumeObjectTO; import org.apache.log4j.Logger; +import org.libvirt.Connect; +import org.libvirt.Domain; +import org.libvirt.DomainInfo; +import org.libvirt.TypedParameter; +import org.libvirt.LibvirtException; +import org.libvirt.event.BlockJobListener; +import org.libvirt.event.BlockJobStatus; +import org.libvirt.event.BlockJobType; @ResourceWrapper(handles = MigrateVolumeCommand.class) public final class LibvirtMigrateVolumeCommandWrapper extends CommandWrapper<MigrateVolumeCommand, Answer, LibvirtComputingResource> { @@ -44,13 +60,116 @@ public final class LibvirtMigrateVolumeCommandWrapper extends CommandWrapper<Mig @Override public Answer execute(final MigrateVolumeCommand command, final LibvirtComputingResource libvirtComputingResource) { LOGGER.info("I'm here HARIIIII"); + + VolumeObjectTO srcVolumeObjectTO = (VolumeObjectTO)command.getSrcData(); + PrimaryDataStoreTO srcPrimaryDataStore = (PrimaryDataStoreTO)srcVolumeObjectTO.getDataStore(); + + MigrateVolumeAnswer answer; + if (srcPrimaryDataStore.getPoolType().equals(Storage.StoragePoolType.PowerFlex)) { + answer = migrateVolumeInternal(command, libvirtComputingResource); + } else { + answer = migrateRegularVolume(command, libvirtComputingResource); + } + + return answer; + } + + private MigrateVolumeAnswer migrateVolumeInternal (final MigrateVolumeCommand command, final LibvirtComputingResource libvirtComputingResource) { + VolumeObjectTO srcVolumeObjectTO = (VolumeObjectTO)command.getSrcData(); + PrimaryDataStoreTO srcPrimaryDataStore = (PrimaryDataStoreTO)srcVolumeObjectTO.getDataStore(); + final String vmName = srcVolumeObjectTO.getVmName(); + LOGGER.info("HARI VM name: "+ vmName); + + VolumeObjectTO destVolumeObjectTO = (VolumeObjectTO)command.getDestData(); + PrimaryDataStoreTO destPrimaryDataStore = (PrimaryDataStoreTO)destVolumeObjectTO.getDataStore(); + + String srcPath = srcVolumeObjectTO.getPath(); + String destPath = destVolumeObjectTO.getPath(); + + Map<String, String> destDetails = command.getDestDetails(); + + final String srcVolumeId = ScaleIOUtil.getVolumePath(srcVolumeObjectTO.getPath()); + LOGGER.info("HARI Source volume ID: "+ srcVolumeId); + + final String destVolumeId = ScaleIOUtil.getVolumePath(destVolumeObjectTO.getPath()); + LOGGER.info("HARI destination volume ID: "+ destVolumeId); + + final String destSystemId = destDetails.get(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID); + LOGGER.info("HARI destination system ID: "+ destSystemId); + + + final String destDiskFileName = ScaleIOUtil.DISK_NAME_PREFIX + destSystemId + "-" + destVolumeId; + final String diskFilePath = ScaleIOUtil.DISK_PATH + File.separator + destDiskFileName; + + Domain dm = null; + try { + final LibvirtUtilitiesHelper libvirtUtilitiesHelper = libvirtComputingResource.getLibvirtUtilitiesHelper(); + Connect conn = libvirtUtilitiesHelper.getConnection(); + dm = libvirtComputingResource.getDomain(conn, vmName); + + if (dm == null) { + return new MigrateVolumeAnswer(command, false, + "Migrate volume failed due to can not find vm: " + vmName, null); + } + + DomainInfo.DomainState domainState = dm.getInfo().state ; + if (domainState != DomainInfo.DomainState.VIR_DOMAIN_RUNNING) { + return new MigrateVolumeAnswer(command, false, + "Migrate volume failed due to VM is not running: " + vmName + " with domainState = " + domainState, null); + } + + final LibvirtDomainXMLParser parser = new LibvirtDomainXMLParser(); + final String domXml = dm.getXMLDesc(0); + parser.parseDomainXML(domXml); + LOGGER.info(String.format("VM [%s] with XML configuration [%s] will be migrated to host.", vmName, domXml)); + + List<LibvirtVMDef.DiskDef> disks = parser.getDisks(); + LibvirtVMDef.DiskDef diskdef = null; + for (final LibvirtVMDef.DiskDef disk : disks) { + final String file = disk.getDiskPath(); + LOGGER.info("HARIIII : " + file); + if (file != null && file.contains(srcVolumeId)) { + diskdef = disk; + break; + } + } + if (diskdef == null) { + throw new InternalErrorException("disk: " + srcPath + " is not attached before"); + } + diskdef.setDiskPath(diskFilePath); + LOGGER.info("HARIIII Destination xml : " + diskdef.toString()); + dm.blockCopy(srcPath, diskdef.toString(), new TypedParameter[]{}, 0); + BlockJobListener listener = new BlockJobListener() { + @Override + public void onEvent(Domain domain, String diskPath, BlockJobType type, BlockJobStatus status) { + + } + }; + + return new MigrateVolumeAnswer(command, true, null, destPath); + } catch (LibvirtException e) { + String msg = "Migrate volume failed due to " + e.toString(); + LOGGER.warn(msg, e); + return new MigrateVolumeAnswer(command, false, msg, null); + } catch (InternalErrorException e) { + throw new RuntimeException(e); + } finally { + if (dm != null) { + try { + dm.free(); + } catch (LibvirtException l) { + LOGGER.trace("Ignoring libvirt error.", l); + }; + } + } + } + private MigrateVolumeAnswer migrateRegularVolume(final MigrateVolumeCommand command, final LibvirtComputingResource libvirtComputingResource) { KVMStoragePoolManager storagePoolManager = libvirtComputingResource.getStoragePoolMgr(); VolumeObjectTO srcVolumeObjectTO = (VolumeObjectTO)command.getSrcData(); PrimaryDataStoreTO srcPrimaryDataStore = (PrimaryDataStoreTO)srcVolumeObjectTO.getDataStore(); Map<String, String> srcDetails = command.getSrcDetails(); - String srcPath = srcDetails != null ? srcDetails.get(DiskTO.IQN) : srcVolumeObjectTO.getPath(); VolumeObjectTO destVolumeObjectTO = (VolumeObjectTO)command.getDestData(); diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/client/ScaleIOGatewayClientImpl.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/client/ScaleIOGatewayClientImpl.java index ad1279b5221..61e190d5239 100644 --- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/client/ScaleIOGatewayClientImpl.java +++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/client/ScaleIOGatewayClientImpl.java @@ -761,7 +761,7 @@ public class ScaleIOGatewayClientImpl implements ScaleIOGatewayClient { } String srcPoolId = volume.getStoragePoolId(); - LOG.debug("Migrating the volume: " + srcVolumeId + " on the src pool: " + srcPoolId + " to the dest pool: " + destPoolId + + LOG.info("Migrating the volume: " + srcVolumeId + " on the src pool: " + srcPoolId + " to the dest pool: " + destPoolId + " in the same PowerFlex cluster"); post("/instances/Volume::" + srcVolumeId + "/action/migrateVTree", diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java index ad97e7e7269..98b27939ca9 100644 --- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java +++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java @@ -19,6 +19,7 @@ package org.apache.cloudstack.storage.datastore.driver; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.UUID; import javax.inject.Inject; @@ -42,6 +43,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreDriver import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeService; import org.apache.cloudstack.framework.async.AsyncCompletionCallback; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.storage.RemoteHostEndPoint; @@ -71,6 +73,7 @@ import com.cloud.agent.api.Answer; import com.cloud.agent.api.to.DataObjectType; import com.cloud.agent.api.to.DataStoreTO; import com.cloud.agent.api.to.DataTO; +import com.cloud.agent.api.to.DiskTO; import com.cloud.alert.AlertManager; import com.cloud.configuration.Config; import com.cloud.host.Host; @@ -127,6 +130,8 @@ public class ScaleIOPrimaryDataStoreDriver implements PrimaryDataStoreDriver { private HostDao hostDao; @Inject private VMInstanceDao vmInstanceDao; + @Inject + private VolumeService volumeService; public ScaleIOPrimaryDataStoreDriver() { @@ -768,10 +773,23 @@ public class ScaleIOPrimaryDataStoreDriver implements PrimaryDataStoreDriver { final String srcVolumeId = ScaleIOUtil.getVolumePath(srcVolumePath); final StoragePoolVO destStoragePool = storagePoolDao.findById(destPoolId); final String destStoragePoolId = destStoragePool.getPath(); + //CreateObjectAnswer createAnswer = createVolume((VolumeInfo) destData, destStore.getId()); + //String destVolumePath = createAnswer.getData().getPath(); + final String destVolumeId = UUID.randomUUID().toString(); + final String destScaleIOVolumeName = String.format("%s-%s-%s-%s", ScaleIOUtil.VOLUME_PREFIX, srcData.getId(), + srcData.getUuid().split("-")[0].substring(4), ManagementServerImpl.customCsIdentifier.value()); + String destVolumePath = String.format("%s:%s", destVolumeId, destScaleIOVolumeName); + + VolumeObjectTO destVolTO = (VolumeObjectTO) destData.getTO(); + destVolTO.setPath(destVolumePath); + + Map<String, String> srcDetails = getVolumeDetails((VolumeInfo) srcData); + Map<String, String> destDetails = getVolumeDetails((VolumeInfo) destData); String value = configDao.getValue(Config.MigrateWait.key()); int waitInterval = NumbersUtil.parseInt(value, Integer.parseInt(Config.MigrateWait.getDefaultValue())); - MigrateVolumeCommand migrateVolumeCommand = new MigrateVolumeCommand(srcData.getId(), srcVolumePath, destStoragePool, ((VolumeInfo) srcData).getAttachedVmName(), ((VolumeInfo) srcData).getVolumeType(), waitInterval); + MigrateVolumeCommand migrateVolumeCommand = new MigrateVolumeCommand(srcData.getTO(), destVolTO, + srcDetails, destDetails, waitInterval); long hostId = 0; VMInstanceVO instance = vmInstanceDao.findVMByInstanceName(((VolumeInfo) srcData).getAttachedVmName()); @@ -860,6 +878,52 @@ public class ScaleIOPrimaryDataStoreDriver implements PrimaryDataStoreDriver { return answer; } + private Map<String, String> getVolumeDetails(VolumeInfo volumeInfo) { + long storagePoolId = volumeInfo.getPoolId(); + StoragePoolVO storagePoolVO = storagePoolDao.findById(storagePoolId); + + if (!storagePoolVO.isManaged()) { + return null; + } + + Map<String, String> volumeDetails = new HashMap<>(); + + VolumeVO volumeVO = volumeDao.findById(volumeInfo.getId()); + + volumeDetails.put(DiskTO.STORAGE_HOST, storagePoolVO.getHostAddress()); + volumeDetails.put(DiskTO.STORAGE_PORT, String.valueOf(storagePoolVO.getPort())); + volumeDetails.put(DiskTO.IQN, volumeVO.get_iScsiName()); + volumeDetails.put(DiskTO.PROTOCOL_TYPE, (volumeVO.getPoolType() != null) ? volumeVO.getPoolType().toString() : null); + volumeDetails.put(StorageManager.STORAGE_POOL_DISK_WAIT.toString(), String.valueOf(StorageManager.STORAGE_POOL_DISK_WAIT.valueIn(storagePoolVO.getId()))); + + volumeDetails.put(DiskTO.VOLUME_SIZE, String.valueOf(volumeVO.getSize())); + volumeDetails.put(DiskTO.SCSI_NAA_DEVICE_ID, getVolumeProperty(volumeInfo.getId(), DiskTO.SCSI_NAA_DEVICE_ID)); + + ChapInfo chapInfo = volumeService.getChapInfo(volumeInfo, volumeInfo.getDataStore()); + + if (chapInfo != null) { + volumeDetails.put(DiskTO.CHAP_INITIATOR_USERNAME, chapInfo.getInitiatorUsername()); + volumeDetails.put(DiskTO.CHAP_INITIATOR_SECRET, chapInfo.getInitiatorSecret()); + volumeDetails.put(DiskTO.CHAP_TARGET_USERNAME, chapInfo.getTargetUsername()); + volumeDetails.put(DiskTO.CHAP_TARGET_SECRET, chapInfo.getTargetSecret()); + } + + String systemId = storagePoolDetailsDao.findDetail(storagePoolId, ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID).getValue(); + volumeDetails.put(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID, systemId); + + return volumeDetails; + } + + private String getVolumeProperty(long volumeId, String property) { + VolumeDetailVO volumeDetails = volumeDetailsDao.findDetail(volumeId, property); + + if (volumeDetails != null) { + return volumeDetails.getValue(); + } + + return null; + } + private Answer migrateVolume(DataObject srcData, DataObject destData) { // Volume migration within same PowerFlex/ScaleIO cluster (with same System ID) DataStore srcStore = srcData.getDataStore(); @@ -875,6 +939,8 @@ public class ScaleIOPrimaryDataStoreDriver implements PrimaryDataStoreDriver { final StoragePoolVO destStoragePool = storagePoolDao.findById(destPoolId); final String destStoragePoolId = destStoragePool.getPath(); int migrationTimeout = StorageManager.KvmStorageOfflineMigrationWait.value(); + LOGGER.info("HARI source volume " + srcVolumeId); + LOGGER.info("HARI destination volume " + destStoragePoolId); boolean migrateStatus = client.migrateVolume(srcVolumeId, destStoragePoolId, migrationTimeout); if (migrateStatus) { String newVolumeName = String.format("%s-%s-%s-%s", ScaleIOUtil.VOLUME_PREFIX, destData.getId(),
