Updated Branches: refs/heads/object_store ef03d5a12 -> e92cd6d63
Move data store specific extract volume logic from VolumeManager to data store driver. Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/e92cd6d6 Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/e92cd6d6 Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/e92cd6d6 Branch: refs/heads/object_store Commit: e92cd6d632ca355edbd22c86cdc9228be9abf8db Parents: ef03d5a1 Author: Min Chen <min.c...@citrix.com> Authored: Tue Jun 4 13:54:14 2013 -0700 Committer: Min Chen <min.c...@citrix.com> Committed: Tue Jun 4 13:54:14 2013 -0700 ---------------------------------------------------------------------- api/src/com/cloud/storage/VolumeApiService.java | 2 +- .../api/command/user/volume/ExtractVolumeCmd.java | 49 ++--- server/src/com/cloud/api/ApiResponseHelper.java | 4 +- server/src/com/cloud/storage/VolumeManager.java | 12 +- .../src/com/cloud/storage/VolumeManagerImpl.java | 188 ++++----------- 5 files changed, 76 insertions(+), 179 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cloudstack/blob/e92cd6d6/api/src/com/cloud/storage/VolumeApiService.java ---------------------------------------------------------------------- diff --git a/api/src/com/cloud/storage/VolumeApiService.java b/api/src/com/cloud/storage/VolumeApiService.java index 58bd0fd..95f962d 100644 --- a/api/src/com/cloud/storage/VolumeApiService.java +++ b/api/src/com/cloud/storage/VolumeApiService.java @@ -97,5 +97,5 @@ public interface VolumeApiService { * @throws PermissionDeniedException * */ - Long extractVolume(ExtractVolumeCmd cmd); + String extractVolume(ExtractVolumeCmd cmd); } http://git-wip-us.apache.org/repos/asf/cloudstack/blob/e92cd6d6/api/src/org/apache/cloudstack/api/command/user/volume/ExtractVolumeCmd.java ---------------------------------------------------------------------- diff --git a/api/src/org/apache/cloudstack/api/command/user/volume/ExtractVolumeCmd.java b/api/src/org/apache/cloudstack/api/command/user/volume/ExtractVolumeCmd.java index d2beeff..5fbe106 100644 --- a/api/src/org/apache/cloudstack/api/command/user/volume/ExtractVolumeCmd.java +++ b/api/src/org/apache/cloudstack/api/command/user/volume/ExtractVolumeCmd.java @@ -126,33 +126,26 @@ public class ExtractVolumeCmd extends BaseAsyncCmd { @Override public void execute(){ - // try { - UserContext.current().setEventDetails("Volume Id: "+getId()); - Long uploadId = _volumeService.extractVolume(this); - if (uploadId != null){ - Upload uploadInfo = _entityMgr.findById(Upload.class, uploadId); - ExtractResponse response = new ExtractResponse(); - response.setResponseName(getCommandName()); - response.setObjectName("volume"); - Volume vol = _entityMgr.findById(Volume.class, id); - response.setId(vol.getUuid()); - response.setName(vol.getName()); - DataCenter zone = _entityMgr.findById(DataCenter.class, zoneId); - response.setZoneId(zone.getUuid()); - response.setZoneName(zone.getName()); - response.setMode(mode); - response.setUploadId(uploadInfo.getUuid()); - response.setState(uploadInfo.getUploadState().toString()); - Account account = _entityMgr.findById(Account.class, getEntityOwnerId()); - response.setAccountId(account.getUuid()); - response.setUrl(uploadInfo.getUploadUrl()); - this.setResponseObject(response); - } else { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to extract volume"); - } - // } catch (URISyntaxException ex) { - // s_logger.info(ex); - // throw new ServerApiException(ApiErrorCode.PARAM_ERROR, ex.getMessage()); - // } + UserContext.current().setEventDetails("Volume Id: " + getId()); + String uploadUrl = _volumeService.extractVolume(this); + if (uploadUrl != null) { + ExtractResponse response = new ExtractResponse(); + response.setResponseName(getCommandName()); + response.setObjectName("volume"); + Volume vol = _entityMgr.findById(Volume.class, id); + response.setId(vol.getUuid()); + response.setName(vol.getName()); + DataCenter zone = _entityMgr.findById(DataCenter.class, zoneId); + response.setZoneId(zone.getUuid()); + response.setZoneName(zone.getName()); + response.setMode(mode); + response.setState(Upload.Status.DOWNLOAD_URL_CREATED.toString()); + Account account = _entityMgr.findById(Account.class, getEntityOwnerId()); + response.setAccountId(account.getUuid()); + response.setUrl(uploadUrl); + this.setResponseObject(response); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to extract volume"); + } } } http://git-wip-us.apache.org/repos/asf/cloudstack/blob/e92cd6d6/server/src/com/cloud/api/ApiResponseHelper.java ---------------------------------------------------------------------- diff --git a/server/src/com/cloud/api/ApiResponseHelper.java b/server/src/com/cloud/api/ApiResponseHelper.java index 310b353..cf11b41 100755 --- a/server/src/com/cloud/api/ApiResponseHelper.java +++ b/server/src/com/cloud/api/ApiResponseHelper.java @@ -1605,8 +1605,8 @@ public class ApiResponseHelper implements ResponseGenerator { response.setZoneName(zone.getName()); } response.setMode(mode); - response.setUrl(url); - response.setState(Upload.Status.DOWNLOAD_URL_CREATED.toString()); + response.setUrl(url); + response.setState(Upload.Status.DOWNLOAD_URL_CREATED.toString()); Account account = ApiDBUtils.findAccountById(accountId); response.setAccountId(account.getUuid()); http://git-wip-us.apache.org/repos/asf/cloudstack/blob/e92cd6d6/server/src/com/cloud/storage/VolumeManager.java ---------------------------------------------------------------------- diff --git a/server/src/com/cloud/storage/VolumeManager.java b/server/src/com/cloud/storage/VolumeManager.java index 65c56d5..5f533ca 100644 --- a/server/src/com/cloud/storage/VolumeManager.java +++ b/server/src/com/cloud/storage/VolumeManager.java @@ -18,13 +18,11 @@ */ package com.cloud.storage; -import java.net.URISyntaxException; import java.util.Map; import org.apache.cloudstack.api.command.user.volume.AttachVolumeCmd; import org.apache.cloudstack.api.command.user.volume.CreateVolumeCmd; import org.apache.cloudstack.api.command.user.volume.DetachVolumeCmd; -import org.apache.cloudstack.api.command.user.volume.ExtractVolumeCmd; import org.apache.cloudstack.api.command.user.volume.MigrateVolumeCmd; import org.apache.cloudstack.api.command.user.volume.ResizeVolumeCmd; import org.apache.cloudstack.api.command.user.volume.UploadVolumeCmd; @@ -35,8 +33,6 @@ import com.cloud.agent.api.to.VirtualMachineTO; import com.cloud.deploy.DeployDestination; import com.cloud.exception.ConcurrentOperationException; import com.cloud.exception.InsufficientStorageCapacityException; -import com.cloud.exception.InternalErrorException; -import com.cloud.exception.PermissionDeniedException; import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.StorageUnavailableException; import com.cloud.host.Host; @@ -54,6 +50,7 @@ public interface VolumeManager extends VolumeApiService { Long destPoolClusterId, HypervisorType dataDiskHyperType) throws ConcurrentOperationException; + @Override VolumeVO uploadVolume(UploadVolumeCmd cmd) throws ResourceAllocationException; @@ -65,28 +62,35 @@ public interface VolumeManager extends VolumeApiService { String getVmNameOnVolume(Volume volume); + @Override VolumeVO allocVolume(CreateVolumeCmd cmd) throws ResourceAllocationException; + @Override VolumeVO createVolume(CreateVolumeCmd cmd); + @Override VolumeVO resizeVolume(ResizeVolumeCmd cmd) throws ResourceAllocationException; + @Override boolean deleteVolume(long volumeId, Account caller) throws ConcurrentOperationException; void destroyVolume(VolumeVO volume); DiskProfile allocateRawVolume(Type type, String name, DiskOfferingVO offering, Long size, VMInstanceVO vm, Account owner); + @Override Volume attachVolumeToVM(AttachVolumeCmd command); + @Override Volume detachVolumeFromVM(DetachVolumeCmd cmmd); void release(VirtualMachineProfile<? extends VMInstanceVO> profile); void cleanupVolumes(long vmId) throws ConcurrentOperationException; + @Override Volume migrateVolume(MigrateVolumeCmd cmd); <T extends VMInstanceVO> void migrateVolumes(T vm, VirtualMachineTO vmTo, Host srcHost, Host destHost, http://git-wip-us.apache.org/repos/asf/cloudstack/blob/e92cd6d6/server/src/com/cloud/storage/VolumeManagerImpl.java ---------------------------------------------------------------------- diff --git a/server/src/com/cloud/storage/VolumeManagerImpl.java b/server/src/com/cloud/storage/VolumeManagerImpl.java index fcd714d..5abd432 100644 --- a/server/src/com/cloud/storage/VolumeManagerImpl.java +++ b/server/src/com/cloud/storage/VolumeManagerImpl.java @@ -43,10 +43,8 @@ import org.apache.cloudstack.api.command.user.volume.MigrateVolumeCmd; import org.apache.cloudstack.api.command.user.volume.ResizeVolumeCmd; import org.apache.cloudstack.api.command.user.volume.UploadVolumeCmd; -import com.amazonaws.services.s3.model.CannedAccessControlList; import com.cloud.storage.dao.*; import org.apache.cloudstack.api.command.user.volume.*; -import org.apache.cloudstack.api.response.ExtractResponse; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProviderManager; @@ -72,23 +70,20 @@ import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreDao; import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO; +import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity; import org.apache.commons.lang.StringUtils; import com.cloud.agent.AgentManager; import com.cloud.agent.api.Answer; import com.cloud.agent.api.storage.CreateVolumeOVAAnswer; import com.cloud.agent.api.storage.CreateVolumeOVACommand; -import com.cloud.agent.api.to.DataStoreTO; import com.cloud.agent.api.to.DataTO; import com.cloud.agent.api.to.DiskTO; -import com.cloud.agent.api.to.S3TO; -import com.cloud.agent.api.to.SwiftTO; import com.cloud.agent.api.to.VirtualMachineTO; import com.cloud.alert.AlertManager; import com.cloud.api.ApiDBUtils; import com.cloud.async.AsyncJobExecutor; import com.cloud.async.AsyncJobManager; -import com.cloud.async.AsyncJobResult; import com.cloud.async.AsyncJobVO; import com.cloud.async.BaseAsyncJobExecutor; import com.cloud.capacity.CapacityManager; @@ -119,7 +114,6 @@ import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.PermissionDeniedException; import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.StorageUnavailableException; -import com.cloud.exception.UnsupportedServiceException; import com.cloud.host.Host; import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; @@ -134,7 +128,6 @@ import com.cloud.server.ManagementServer; import com.cloud.service.ServiceOfferingVO; import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.storage.Storage.ImageFormat; -import com.cloud.storage.Upload.Status; import com.cloud.storage.Volume.Type; import com.cloud.storage.dao.DiskOfferingDao; import com.cloud.storage.dao.SnapshotDao; @@ -166,7 +159,6 @@ import com.cloud.uservm.UserVm; import com.cloud.utils.EnumUtils; import com.cloud.utils.NumbersUtil; import com.cloud.utils.Pair; -import com.cloud.utils.S3Utils; import com.cloud.utils.UriUtils; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.db.DB; @@ -2615,11 +2607,9 @@ public class VolumeManagerImpl extends ManagerBase implements VolumeManager { @Override @ActionEvent(eventType = EventTypes.EVENT_VOLUME_EXTRACT, eventDescription = "extracting volume", async = true) - public Long extractVolume(ExtractVolumeCmd cmd) { + public String extractVolume(ExtractVolumeCmd cmd) { Long volumeId = cmd.getId(); - String url = cmd.getUrl(); Long zoneId = cmd.getZoneId(); - AsyncJobVO job = null; // FIXME: cmd.getJob(); String mode = cmd.getMode(); Account account = UserContext.current().getCaller(); @@ -2654,18 +2644,16 @@ public class VolumeManagerImpl extends ManagerBase implements VolumeManager { throw ex; } - if (volume.getVolumeType() != Volume.Type.DATADISK) { // Datadisk dont - // have any - // template - // dependence. + if (volume.getVolumeType() != Volume.Type.DATADISK) { + // Datadisk dont have any template dependence. VMTemplateVO template = ApiDBUtils.findTemplateById(volume.getTemplateId()); if (template != null) { // For ISO based volumes template = null and // we allow extraction of all ISO based // volumes boolean isExtractable = template.isExtractable() && template.getTemplateType() != Storage.TemplateType.SYSTEM; - if (!isExtractable && account != null && account.getType() != Account.ACCOUNT_TYPE_ADMIN) { // Global - // admins are always allowed to extract + if (!isExtractable && account != null && account.getType() != Account.ACCOUNT_TYPE_ADMIN) { + // Global admins are always allowed to extract PermissionDeniedException ex = new PermissionDeniedException("The volume with specified volumeId is not allowed to be extracted"); ex.addProxyObject(volume, volumeId, "volumeId"); throw ex; @@ -2680,139 +2668,51 @@ public class VolumeManagerImpl extends ManagerBase implements VolumeManager { extractMode = mode.equals(Upload.Mode.FTP_UPLOAD.toString()) ? Upload.Mode.FTP_UPLOAD : Upload.Mode.HTTP_DOWNLOAD; } - long accountId = volume.getAccountId(); + // Clean up code to remove all those previous uploadVO and uploadMonitor code. Previous code is trying to fake an async operation purely in + // db table with uploadVO and async_job entry, but internal implementation is actually synchronous. StoragePool srcPool = (StoragePool) this.dataStoreMgr.getPrimaryDataStore(volume.getPoolId()); - DataStore secStore = this.dataStoreMgr.getImageStore(zoneId); + ImageStoreEntity secStore = (ImageStoreEntity) this.dataStoreMgr.getImageStore(zoneId); String secondaryStorageURL = secStore.getUri(); - List<UploadVO> extractURLList = _uploadDao.listByTypeUploadStatus(volumeId, Upload.Type.VOLUME, UploadVO.Status.DOWNLOAD_URL_CREATED); + String value = this._configDao.getValue(Config.CopyVolumeWait.toString()); + int copyvolumewait = NumbersUtil.parseInt(value, Integer.parseInt(Config.CopyVolumeWait.getDefaultValue())); + // Copy volume from primary to secondary storage + VolumeInfo srcVol = this.volFactory.getVolume(volume.getId()); + AsyncCallFuture<VolumeApiResult> cvAnswer = this.volService.copyVolume(srcVol, secStore); + // Check if you got a valid answer. + VolumeApiResult cvResult = null; + try { + cvResult = cvAnswer.get(); + } catch (InterruptedException e1) { + s_logger.debug("failed copy volume", e1); + throw new CloudRuntimeException("Failed to copy volume", e1); + } catch (ExecutionException e1) { + s_logger.debug("failed copy volume", e1); + throw new CloudRuntimeException("Failed to copy volume", e1); + } + if (cvResult == null || cvResult.isFailed()) { + String errorString = "Failed to copy the volume from the source primary storage pool to secondary storage."; + throw new CloudRuntimeException(errorString); + } + + VolumeInfo vol = cvResult.getVolume(); + String volumeLocalPath = vol.getPath(); + String volumeName = StringUtils.substringBeforeLast(StringUtils.substringAfterLast(volumeLocalPath, "/"), "."); + // volss, handle the ova special case; + if (getFormatForPool(srcPool) == "ova") { + // TODO: need to handle this for S3 as secondary storage + CreateVolumeOVACommand cvOVACmd = new CreateVolumeOVACommand(secondaryStorageURL, volumeLocalPath, volumeName, srcPool, copyvolumewait); + CreateVolumeOVAAnswer OVAanswer = null; - if (extractMode == Upload.Mode.HTTP_DOWNLOAD && extractURLList.size() > 0) { - return extractURLList.get(0).getId(); // If download url already - // Note: volss - // exists then return - } else { - UploadVO uploadJob = _uploadMonitor.createNewUploadEntry(secStore.getId(), volumeId, UploadVO.Status.COPY_IN_PROGRESS, - Upload.Type.VOLUME, url, extractMode); - s_logger.debug("Extract Mode - " + uploadJob.getMode()); - uploadJob = _uploadDao.createForUpdate(uploadJob.getId()); - - // Update the async Job - - ExtractResponse resultObj = new ExtractResponse(ApiDBUtils.findVolumeById(volumeId).getUuid(), volume.getName(), ApiDBUtils - .findAccountById(accountId).getUuid(), UploadVO.Status.COPY_IN_PROGRESS.toString(), uploadJob.getUuid()); - resultObj.setResponseName(cmd.getCommandName()); - AsyncJobExecutor asyncExecutor = BaseAsyncJobExecutor.getCurrentExecutor(); - if (asyncExecutor != null) { - job = asyncExecutor.getJob(); - _asyncMgr.updateAsyncJobAttachment(job.getId(), Upload.Type.VOLUME.toString(), volumeId); - _asyncMgr.updateAsyncJobStatus(job.getId(), AsyncJobResult.STATUS_IN_PROGRESS, resultObj); - } - //TODO: AncientDataMotionStrategy.copyObject should use different timeout parent for different objects - String value = this._configDao.getValue(Config.CopyVolumeWait.toString()); - int copyvolumewait = NumbersUtil.parseInt(value, Integer.parseInt(Config.CopyVolumeWait.getDefaultValue())); - // Copy volume from primary to secondary storage - VolumeInfo srcVol = this.volFactory.getVolume(volume.getId()); - AsyncCallFuture<VolumeApiResult> cvAnswer = this.volService.copyVolume(srcVol, secStore); - // Check if you got a valid answer. - VolumeApiResult cvResult = null; try { - cvResult = cvAnswer.get(); - } catch (InterruptedException e1) { - s_logger.debug("failed copy volume", e1); - throw new CloudRuntimeException("Failed to copy volume" , e1); - } catch (ExecutionException e1) { - s_logger.debug("failed copy volume", e1); - throw new CloudRuntimeException("Failed to copy volume" , e1); - } - if (cvResult == null || cvResult.isFailed()) { - String errorString = "Failed to copy the volume from the source primary storage pool to secondary storage."; - - // Update the async job. - resultObj.setResultString(errorString); - resultObj.setUploadStatus(UploadVO.Status.COPY_ERROR.toString()); - if (asyncExecutor != null) { - _asyncMgr.completeAsyncJob(job.getId(), AsyncJobResult.STATUS_FAILED, 0, resultObj); - } - - // Update the DB that volume couldn't be copied - uploadJob.setUploadState(UploadVO.Status.COPY_ERROR); - uploadJob.setErrorString(errorString); - uploadJob.setLastUpdated(new Date()); - _uploadDao.update(uploadJob.getId(), uploadJob); - - throw new CloudRuntimeException(errorString); - } - - VolumeInfo vol = cvResult.getVolume(); - String volumeLocalPath = vol.getPath(); - String volumeName = StringUtils.substringBeforeLast(StringUtils.substringAfterLast(volumeLocalPath, "/"), "."); - // volss, handle the ova special case; - if (getFormatForPool(srcPool) == "ova") { - //TODO: need to handle this for S3 as secondary storage - CreateVolumeOVACommand cvOVACmd = new CreateVolumeOVACommand(secondaryStorageURL, volumeLocalPath, volumeName, srcPool, - copyvolumewait); - CreateVolumeOVAAnswer OVAanswer = null; - - try { - cvOVACmd.setContextParam("hypervisor", HypervisorType.VMware.toString()); - OVAanswer = (CreateVolumeOVAAnswer) storageMgr.sendToPool(srcPool, cvOVACmd); // Fang: - // for - // extract - // volume, - // create - // the - // ova - // file - // here; - - } catch (StorageUnavailableException e) { - s_logger.debug("Storage unavailable"); - } - } - // Update the DB that volume is copied and volumePath - uploadJob.setUploadState(UploadVO.Status.COPY_COMPLETE); - uploadJob.setLastUpdated(new Date()); - uploadJob.setInstallPath(volumeLocalPath); - _uploadDao.update(uploadJob.getId(), uploadJob); - - DataStoreTO volStore = secStore.getTO(); - if (volStore instanceof SwiftTO) { - throw new UnsupportedServiceException("ExtractVolume is not yet supported for Swift image store provider"); - } - - if (volStore instanceof S3TO) { - // for S3, no need to do anything, just return volume url for - // extract template. but we need to set object acl as public_read to - // make the url accessible - S3TO s3 = (S3TO) volStore; - String key = vol.getPath(); - try { - S3Utils.setObjectAcl(s3, s3.getBucketName(), key, CannedAccessControlList.PublicRead); - } catch (Exception ex) { - s_logger.error("Failed to set ACL on S3 object " + key + " to PUBLIC_READ", ex); - throw new CloudRuntimeException("Failed to set ACL on S3 object " + key + " to PUBLIC_READ"); - } - // construct the url from s3 - StringBuffer s3url = new StringBuffer(); - s3url.append(s3.isHttps() ? "https://"; : "http://";); - s3url.append(s3.getEndPoint()); - s3url.append("/"); - s3url.append(s3.getBucketName()); - s3url.append("/"); - s3url.append(key); - - UploadVO vo = _uploadDao.createForUpdate(); - vo.setLastUpdated(new Date()); - vo.setUploadUrl(s3url.toString()); - vo.setUploadState(Status.DOWNLOAD_URL_CREATED); - _uploadDao.update(uploadJob.getId(), vo); - } else { - // create a URL. - _uploadMonitor.createVolumeDownloadURL(volumeId, volumeLocalPath, Upload.Type.VOLUME, zoneId, uploadJob.getId(), volume.getFormat()); + cvOVACmd.setContextParam("hypervisor", HypervisorType.VMware.toString()); + // for extract volume, create the ova file here; + OVAanswer = (CreateVolumeOVAAnswer) storageMgr.sendToPool(srcPool, cvOVACmd); + } catch (StorageUnavailableException e) { + s_logger.debug("Storage unavailable"); } - return uploadJob.getId(); } + return secStore.createEntityExtractUrl(vol.getPath(), vol.getFormat()); } private String getFormatForPool(StoragePool pool) {
