Support for backend snapshots with XenServer
Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/2bd035d1 Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/2bd035d1 Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/2bd035d1 Branch: refs/heads/master Commit: 2bd035d1991e97dfd94cc5c9b936c94aed2d3bc4 Parents: 2b4b8aa Author: Mike Tutkowski <[email protected]> Authored: Mon Nov 16 12:18:25 2015 -0700 Committer: Mike Tutkowski <[email protected]> Committed: Fri May 13 01:02:04 2016 -0600 ---------------------------------------------------------------------- .../cloud/agent/api/StartupRoutingCommand.java | 10 +- .../agent/api/storage/ResizeVolumeCommand.java | 37 +- .../storage/resource/StorageProcessor.java | 3 + .../StorageSubsystemCommandHandlerBase.java | 3 + .../storage/command/ResignatureAnswer.java | 60 + .../storage/command/ResignatureCommand.java | 48 + .../storage/to/PrimaryDataStoreTO.java | 1 + .../api/storage/DataStoreCapabilities.java | 18 +- .../subsystem/api/storage/PrimaryDataStore.java | 2 + .../api/storage/PrimaryDataStoreDriver.java | 35 +- .../api/storage/PrimaryDataStoreInfo.java | 1 + .../subsystem/api/storage/TemplateService.java | 3 + .../subsystem/api/storage/VolumeService.java | 4 +- .../src/com/cloud/vm/VmWorkResizeVolume.java | 9 +- .../orchestration/VolumeOrchestrator.java | 12 +- .../schema/src/com/cloud/dc/dao/ClusterDao.java | 2 + .../src/com/cloud/dc/dao/ClusterDaoImpl.java | 42 +- .../schema/src/com/cloud/host/dao/HostDao.java | 3 + .../src/com/cloud/host/dao/HostDaoImpl.java | 33 + .../src/com/cloud/host/dao/HostDetailsDao.java | 2 + .../com/cloud/host/dao/HostDetailsDaoImpl.java | 30 + .../storage/dao/VMTemplatePoolDaoImpl.java | 4 +- .../motion/StorageSystemDataMotionStrategy.java | 687 +++++++++--- .../storage/image/TemplateServiceImpl.java | 19 + .../storage/image/store/TemplateObject.java | 65 +- .../test/FakePrimaryDataStoreDriver.java | 27 +- .../snapshot/StorageSystemSnapshotStrategy.java | 213 +++- .../snapshot/XenserverSnapshotStrategy.java | 4 +- .../allocator/AbstractStoragePoolAllocator.java | 2 +- engine/storage/volume/pom.xml | 5 + .../storage/datastore/PrimaryDataStoreImpl.java | 30 +- .../storage/volume/VolumeServiceImpl.java | 431 +++++++- plugins/api/solidfire-intg-test/pom.xml | 2 +- .../spring-solidfire-intg-test-context.xml | 4 +- .../admin/solidfire/GetPathForVolumeCmd.java | 67 ++ .../solidfire/GetSolidFireAccountIdCmd.java | 72 ++ .../GetSolidFireVolumeAccessGroupIdCmd.java | 81 ++ .../solidfire/GetSolidFireVolumeSizeCmd.java | 70 ++ .../solidfire/GetVolumeSnapshotDetailsCmd.java | 73 ++ .../admin/solidfire/GetVolumeiScsiNameCmd.java | 68 ++ .../solidfire/GetSolidFireAccountIdCmd.java | 83 -- .../GetSolidFireVolumeAccessGroupIdCmd.java | 84 -- .../GetSolidFireVolumeIscsiNameCmd.java | 78 -- .../solidfire/GetSolidFireVolumeSizeCmd.java | 84 -- .../response/ApiSolidFireAccountIdResponse.java | 37 - ...ApiSolidFireVolumeAccessGroupIdResponse.java | 37 - .../ApiSolidFireVolumeIscsiNameResponse.java | 37 - .../ApiSolidFireVolumeSizeResponse.java | 37 - .../solidfire/ApiPathForVolumeResponse.java | 33 + .../ApiSolidFireAccountIdResponse.java | 33 + ...ApiSolidFireVolumeAccessGroupIdResponse.java | 33 + .../ApiSolidFireVolumeSizeResponse.java | 33 + .../ApiVolumeSnapshotDetailsResponse.java | 43 + .../solidfire/ApiVolumeiScsiNameResponse.java | 33 + .../ApiSolidFireIntegrationTestService.java | 22 + .../ApiSolidFireIntegrationTestServiceImpl.java | 48 + .../solidfire/ApiSolidFireService.java | 37 - .../solidfire/ApiSolidFireServiceImpl.java | 126 --- .../SolidFireIntegrationTestManager.java | 23 + .../SolidFireIntegrationTestManagerImpl.java | 78 ++ .../solidfire/SolidFireIntegrationTestUtil.java | 112 ++ .../kvm/storage/KVMStorageProcessor.java | 9 + .../ovm3/resources/Ovm3StorageProcessor.java | 16 +- .../resource/SimulatorStorageProcessor.java | 9 + .../resource/VmwareStorageProcessor.java | 9 + .../xenserver/resource/CitrixResourceBase.java | 119 +- .../resource/XenServerStorageProcessor.java | 61 +- .../resource/Xenserver625StorageProcessor.java | 14 +- .../CitrixResizeVolumeCommandWrapper.java | 70 +- .../xenbase/CitrixStartCommandWrapper.java | 4 +- .../ElastistorPrimaryDataStoreDriver.java | 8 +- .../CloudStackPrimaryDataStoreDriverImpl.java | 13 +- .../driver/NexentaPrimaryDataStoreDriver.java | 13 +- .../SamplePrimaryDataStoreDriverImpl.java | 17 +- .../driver/SolidFirePrimaryDataStoreDriver.java | 1022 ++++++++++++++---- .../SolidFirePrimaryDataStoreLifeCycle.java | 67 +- .../provider/SolidFireSharedHostListener.java | 64 +- .../storage/datastore/util/SolidFireUtil.java | 508 +++++---- .../com/cloud/capacity/CapacityManagerImpl.java | 25 +- .../deploy/DeploymentPlanningManagerImpl.java | 3 +- .../hypervisor/CloudZonesStartupProcessor.java | 23 + .../com/cloud/resource/ResourceManagerImpl.java | 24 + .../com/cloud/storage/ResizeVolumePayload.java | 7 +- .../src/com/cloud/storage/StorageManager.java | 24 + .../com/cloud/storage/StorageManagerImpl.java | 75 +- .../com/cloud/storage/VolumeApiServiceImpl.java | 64 +- .../storage/snapshot/SnapshotManagerImpl.java | 32 +- .../com/cloud/template/TemplateManagerImpl.java | 52 +- ui/scripts/storage.js | 2 + utils/src/main/java/com/cloud/utils/Utils.java | 38 + 90 files changed, 4243 insertions(+), 1562 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cloudstack/blob/2bd035d1/core/src/com/cloud/agent/api/StartupRoutingCommand.java ---------------------------------------------------------------------- diff --git a/core/src/com/cloud/agent/api/StartupRoutingCommand.java b/core/src/com/cloud/agent/api/StartupRoutingCommand.java index c413836..b459f88 100644 --- a/core/src/com/cloud/agent/api/StartupRoutingCommand.java +++ b/core/src/com/cloud/agent/api/StartupRoutingCommand.java @@ -35,7 +35,7 @@ public class StartupRoutingCommand extends StartupCommand { long memory; long dom0MinMemory; boolean poolSync; - + private boolean supportsClonedVolumes; String caps; String pool; @@ -180,4 +180,12 @@ public class StartupRoutingCommand extends StartupCommand { public void setGpuGroupDetails(HashMap<String, HashMap<String, VgpuTypesInfo>> groupDetails) { this.groupDetails = groupDetails; } + + public boolean getSupportsClonedVolumes() { + return supportsClonedVolumes; + } + + public void setSupportsClonedVolumes(boolean supportsClonedVolumes) { + this.supportsClonedVolumes = supportsClonedVolumes; + } } http://git-wip-us.apache.org/repos/asf/cloudstack/blob/2bd035d1/core/src/com/cloud/agent/api/storage/ResizeVolumeCommand.java ---------------------------------------------------------------------- diff --git a/core/src/com/cloud/agent/api/storage/ResizeVolumeCommand.java b/core/src/com/cloud/agent/api/storage/ResizeVolumeCommand.java index 3b121e1..22cff13 100644 --- a/core/src/com/cloud/agent/api/storage/ResizeVolumeCommand.java +++ b/core/src/com/cloud/agent/api/storage/ResizeVolumeCommand.java @@ -25,22 +25,34 @@ import com.cloud.agent.api.to.StorageFilerTO; public class ResizeVolumeCommand extends Command { private String path; private StorageFilerTO pool; - private String vmInstance; - private Long newSize; private Long currentSize; + private Long newSize; private boolean shrinkOk; + private String vmInstance; - protected ResizeVolumeCommand() { + /* For managed storage */ + private boolean managed; + private String iScsiName; + protected ResizeVolumeCommand() { } public ResizeVolumeCommand(String path, StorageFilerTO pool, Long currentSize, Long newSize, boolean shrinkOk, String vmInstance) { this.path = path; this.pool = pool; - this.vmInstance = vmInstance; this.currentSize = currentSize; this.newSize = newSize; this.shrinkOk = shrinkOk; + this.vmInstance = vmInstance; + this.managed = false; + } + + public ResizeVolumeCommand(String path, StorageFilerTO pool, Long currentSize, Long newSize, boolean shrinkOk, String vmInstance, + boolean isManaged, String iScsiName) { + this(path, pool, currentSize, newSize, shrinkOk, vmInstance); + + this.iScsiName = iScsiName; + this.managed = isManaged; } public String getPath() { @@ -55,22 +67,20 @@ public class ResizeVolumeCommand extends Command { return pool; } - public long getNewSize() { - return newSize; - } + public long getCurrentSize() { return currentSize; } - public long getCurrentSize() { - return currentSize; - } + public long getNewSize() { return newSize; } - public boolean getShrinkOk() { - return shrinkOk; - } + public boolean getShrinkOk() { return shrinkOk; } public String getInstanceName() { return vmInstance; } + public boolean isManaged() { return managed; } + + public String get_iScsiName() {return iScsiName; } + /** * {@inheritDoc} */ @@ -78,5 +88,4 @@ public class ResizeVolumeCommand extends Command { public boolean executeInSequence() { return false; } - } http://git-wip-us.apache.org/repos/asf/cloudstack/blob/2bd035d1/core/src/com/cloud/storage/resource/StorageProcessor.java ---------------------------------------------------------------------- diff --git a/core/src/com/cloud/storage/resource/StorageProcessor.java b/core/src/com/cloud/storage/resource/StorageProcessor.java index e2bf1b7..e5832cc 100644 --- a/core/src/com/cloud/storage/resource/StorageProcessor.java +++ b/core/src/com/cloud/storage/resource/StorageProcessor.java @@ -26,6 +26,7 @@ import org.apache.cloudstack.storage.command.DeleteCommand; import org.apache.cloudstack.storage.command.DettachCommand; import org.apache.cloudstack.storage.command.ForgetObjectCmd; import org.apache.cloudstack.storage.command.IntroduceObjectCmd; +import org.apache.cloudstack.storage.command.ResignatureCommand; import org.apache.cloudstack.storage.command.SnapshotAndCopyCommand; import com.cloud.agent.api.Answer; @@ -68,4 +69,6 @@ public interface StorageProcessor { public Answer forgetObject(ForgetObjectCmd cmd); public Answer snapshotAndCopy(SnapshotAndCopyCommand cmd); + + public Answer resignature(ResignatureCommand cmd); } http://git-wip-us.apache.org/repos/asf/cloudstack/blob/2bd035d1/core/src/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java ---------------------------------------------------------------------- diff --git a/core/src/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java b/core/src/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java index fc771e0..d9d2993 100644 --- a/core/src/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java +++ b/core/src/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java @@ -28,6 +28,7 @@ import org.apache.cloudstack.storage.command.CreateObjectCommand; import org.apache.cloudstack.storage.command.DeleteCommand; import org.apache.cloudstack.storage.command.DettachCommand; import org.apache.cloudstack.storage.command.IntroduceObjectCmd; +import org.apache.cloudstack.storage.command.ResignatureCommand; import org.apache.cloudstack.storage.command.SnapshotAndCopyCommand; import org.apache.cloudstack.storage.command.StorageSubSystemCommand; @@ -64,6 +65,8 @@ public class StorageSubsystemCommandHandlerBase implements StorageSubsystemComma return processor.introduceObject((IntroduceObjectCmd)command); } else if (command instanceof SnapshotAndCopyCommand) { return processor.snapshotAndCopy((SnapshotAndCopyCommand)command); + } else if (command instanceof ResignatureCommand) { + return processor.resignature((ResignatureCommand)command); } return new Answer((Command)command, false, "not implemented yet"); http://git-wip-us.apache.org/repos/asf/cloudstack/blob/2bd035d1/core/src/org/apache/cloudstack/storage/command/ResignatureAnswer.java ---------------------------------------------------------------------- diff --git a/core/src/org/apache/cloudstack/storage/command/ResignatureAnswer.java b/core/src/org/apache/cloudstack/storage/command/ResignatureAnswer.java new file mode 100644 index 0000000..071f6a9 --- /dev/null +++ b/core/src/org/apache/cloudstack/storage/command/ResignatureAnswer.java @@ -0,0 +1,60 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.storage.command; + +import com.cloud.agent.api.Answer; +import com.cloud.storage.Storage.ImageFormat; + +public class ResignatureAnswer extends Answer { + private long size; + private String path; + private ImageFormat format; + + public ResignatureAnswer() { + } + + public ResignatureAnswer(String errMsg) { + super(null, false, errMsg); + } + + public void setSize(long size) { + this.size = size; + } + + public long getSize() { + return size; + } + + public void setPath(String path) { + this.path = path; + } + + public String getPath() { + return path; + } + + public void setFormat(ImageFormat format) { + this.format = format; + } + + public ImageFormat getFormat() { + return format; + } +} http://git-wip-us.apache.org/repos/asf/cloudstack/blob/2bd035d1/core/src/org/apache/cloudstack/storage/command/ResignatureCommand.java ---------------------------------------------------------------------- diff --git a/core/src/org/apache/cloudstack/storage/command/ResignatureCommand.java b/core/src/org/apache/cloudstack/storage/command/ResignatureCommand.java new file mode 100644 index 0000000..beb4b65 --- /dev/null +++ b/core/src/org/apache/cloudstack/storage/command/ResignatureCommand.java @@ -0,0 +1,48 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.storage.command; + +import com.cloud.utils.Utils; + +import java.util.Map; + +public final class ResignatureCommand extends StorageSubSystemCommand { + private final Map<String, String> details; + + private boolean executeInSequence = true; + + public ResignatureCommand(final Map<String, String> details) { + this.details = Utils.getImmutableMap(details); + } + + public Map<String, String> getDetails() { + return details; + } + + @Override + public void setExecuteInSequence(final boolean executeInSequence) { + this.executeInSequence = executeInSequence; + } + + @Override + public boolean executeInSequence() { + return executeInSequence; + } +} http://git-wip-us.apache.org/repos/asf/cloudstack/blob/2bd035d1/core/src/org/apache/cloudstack/storage/to/PrimaryDataStoreTO.java ---------------------------------------------------------------------- diff --git a/core/src/org/apache/cloudstack/storage/to/PrimaryDataStoreTO.java b/core/src/org/apache/cloudstack/storage/to/PrimaryDataStoreTO.java index a69f357..9b711bc 100644 --- a/core/src/org/apache/cloudstack/storage/to/PrimaryDataStoreTO.java +++ b/core/src/org/apache/cloudstack/storage/to/PrimaryDataStoreTO.java @@ -37,6 +37,7 @@ public class PrimaryDataStoreTO implements DataStoreTO { public static final String CHAP_INITIATOR_SECRET = PrimaryDataStore.CHAP_INITIATOR_SECRET; public static final String CHAP_TARGET_USERNAME = PrimaryDataStore.CHAP_TARGET_USERNAME; public static final String CHAP_TARGET_SECRET = PrimaryDataStore.CHAP_TARGET_SECRET; + public static final String REMOVE_AFTER_COPY = PrimaryDataStore.REMOVE_AFTER_COPY; public static final String VOLUME_SIZE = PrimaryDataStore.VOLUME_SIZE; private final String uuid; http://git-wip-us.apache.org/repos/asf/cloudstack/blob/2bd035d1/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreCapabilities.java ---------------------------------------------------------------------- diff --git a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreCapabilities.java b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreCapabilities.java index 09883c6..2cde5bd 100644 --- a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreCapabilities.java +++ b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreCapabilities.java @@ -18,7 +18,23 @@ */ package org.apache.cloudstack.engine.subsystem.api.storage; +/** + * enumerates different capabilities storage drivers may have + */ public enum DataStoreCapabilities { VOLUME_SNAPSHOT_QUIESCEVM, - STORAGE_SYSTEM_SNAPSHOT // indicates to the StorageSystemSnapshotStrategy that this driver takes snapshots on its own system + /** + * indicates that this driver takes CloudStack volume snapshots on its own system (as either back-end snapshots or back-end clones) + */ + STORAGE_SYSTEM_SNAPSHOT, + /** + * indicates that this driver supports the "cloneOfSnapshot" property of cloud.snapshot_details (for creating a back-end volume + * from a back-end snapshot or a back-end clone) and that it supports the invocation of the createAsync method where a SnapshotInfo is passed in while using + * the "tempVolume" property of snapshot_details + */ + CAN_CREATE_VOLUME_FROM_SNAPSHOT, + /** + * indicates that this driver supports the "cloneOfSnapshot" property of cloud.snapshot_details (for creating a volume from a volume) + */ + CAN_CREATE_VOLUME_FROM_VOLUME } http://git-wip-us.apache.org/repos/asf/cloudstack/blob/2bd035d1/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStore.java ---------------------------------------------------------------------- diff --git a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStore.java b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStore.java index 465b7eb..a399758 100644 --- a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStore.java +++ b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStore.java @@ -23,6 +23,8 @@ import java.util.List; import org.apache.cloudstack.engine.subsystem.api.storage.disktype.DiskFormat; public interface PrimaryDataStore extends DataStore, PrimaryDataStoreInfo { + DataObject create(DataObject dataObject, boolean createEntryInTempSpoolRef); + VolumeInfo getVolume(long id); List<VolumeInfo> getVolumes(); http://git-wip-us.apache.org/repos/asf/cloudstack/blob/2bd035d1/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreDriver.java ---------------------------------------------------------------------- diff --git a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreDriver.java b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreDriver.java index e0c0d28..6dcdf4f 100644 --- a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreDriver.java +++ b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreDriver.java @@ -23,27 +23,38 @@ import org.apache.cloudstack.storage.command.CommandResult; import com.cloud.host.Host; import com.cloud.storage.StoragePool; -import com.cloud.storage.Volume; public interface PrimaryDataStoreDriver extends DataStoreDriver { - ChapInfo getChapInfo(VolumeInfo volumeInfo); + ChapInfo getChapInfo(DataObject dataObject); boolean grantAccess(DataObject dataObject, Host host, DataStore dataStore); void revokeAccess(DataObject dataObject, Host host, DataStore dataStore); - // intended for managed storage (cloud.storage_pool.managed = true) - // if not managed, return volume.getSize() - long getVolumeSizeIncludingHypervisorSnapshotReserve(Volume volume, StoragePool storagePool); - - // intended for managed storage (cloud.storage_pool.managed = true) - // if managed storage, return the total number of bytes currently in use for the storage pool in question - // if not managed storage, return 0 + /** + * intended for managed storage (cloud.storage_pool.managed = true) + * if not managed, return volume.getSize() + */ + long getDataObjectSizeIncludingHypervisorSnapshotReserve(DataObject dataObject, StoragePool storagePool); + + /** + * intended for zone-wide primary storage that is capable of storing a template once and using it in multiple clusters + * if not this kind of storage, return 0 + */ + long getBytesRequiredForTemplate(TemplateInfo templateInfo, StoragePool storagePool); + + /** + * intended for managed storage (cloud.storage_pool.managed = true) + * if managed storage, return the total number of bytes currently in use for the storage pool in question + * if not managed storage, return 0 + */ long getUsedBytes(StoragePool storagePool); - // intended for managed storage (cloud.storage_pool.managed = true) - // if managed storage, return the total number of IOPS currently in use for the storage pool in question - // if not managed storage, return 0 + /** + * intended for managed storage (cloud.storage_pool.managed = true) + * if managed storage, return the total number of IOPS currently in use for the storage pool in question + * if not managed storage, return 0 + */ long getUsedIops(StoragePool storagePool); void takeSnapshot(SnapshotInfo snapshot, AsyncCompletionCallback<CreateCmdResult> callback); http://git-wip-us.apache.org/repos/asf/cloudstack/blob/2bd035d1/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreInfo.java ---------------------------------------------------------------------- diff --git a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreInfo.java b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreInfo.java index f08d9a4..7f2f4dc 100644 --- a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreInfo.java +++ b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreInfo.java @@ -36,6 +36,7 @@ public interface PrimaryDataStoreInfo extends StoragePool { static final String CHAP_INITIATOR_SECRET = "chapInitiatorSecret"; static final String CHAP_TARGET_USERNAME = "chapTargetUsername"; static final String CHAP_TARGET_SECRET = "chapTargetSecret"; + static final String REMOVE_AFTER_COPY = "removeAfterCopy"; static final String VOLUME_SIZE = "volumeSize"; boolean isHypervisorSupported(HypervisorType hypervisor); http://git-wip-us.apache.org/repos/asf/cloudstack/blob/2bd035d1/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java ---------------------------------------------------------------------- diff --git a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java index 88ce932..ff204c6 100644 --- a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java +++ b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java @@ -32,6 +32,7 @@ public interface TemplateService { public TemplateApiResult(TemplateInfo template) { super(); + this.template = template; } @@ -52,6 +53,8 @@ public interface TemplateService { AsyncCallFuture<TemplateApiResult> prepareTemplateOnPrimary(TemplateInfo srcTemplate, StoragePool pool); + AsyncCallFuture<TemplateApiResult> deleteTemplateOnPrimary(TemplateInfo template, StoragePool pool); + void syncTemplateToRegionStore(long templateId, DataStore store); void handleSysTemplateDownload(HypervisorType hostHyper, Long dcId); http://git-wip-us.apache.org/repos/asf/cloudstack/blob/2bd035d1/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/VolumeService.java ---------------------------------------------------------------------- diff --git a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/VolumeService.java b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/VolumeService.java index 8352682..75a7ad9 100644 --- a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/VolumeService.java +++ b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/VolumeService.java @@ -45,7 +45,7 @@ public interface VolumeService { } } - ChapInfo getChapInfo(VolumeInfo volumeInfo, DataStore dataStore); + ChapInfo getChapInfo(DataObject dataObject, DataStore dataStore); boolean grantAccess(DataObject dataObject, Host host, DataStore dataStore); @@ -81,7 +81,7 @@ public interface VolumeService { VolumeEntity getVolumeEntity(long volumeId); - AsyncCallFuture<VolumeApiResult> createManagedStorageAndVolumeFromTemplateAsync(VolumeInfo volumeInfo, long destDataStoreId, + AsyncCallFuture<VolumeApiResult> createManagedStorageVolumeFromTemplateAsync(VolumeInfo volumeInfo, long destDataStoreId, TemplateInfo srcTemplateInfo, long destHostId); AsyncCallFuture<VolumeApiResult> createVolumeFromTemplateAsync(VolumeInfo volume, long dataStoreId, http://git-wip-us.apache.org/repos/asf/cloudstack/blob/2bd035d1/engine/components-api/src/com/cloud/vm/VmWorkResizeVolume.java ---------------------------------------------------------------------- diff --git a/engine/components-api/src/com/cloud/vm/VmWorkResizeVolume.java b/engine/components-api/src/com/cloud/vm/VmWorkResizeVolume.java index d269112..de049b3 100644 --- a/engine/components-api/src/com/cloud/vm/VmWorkResizeVolume.java +++ b/engine/components-api/src/com/cloud/vm/VmWorkResizeVolume.java @@ -24,12 +24,12 @@ public class VmWorkResizeVolume extends VmWork { private long newSize; private Long newMinIops; private Long newMaxIops; + private Integer newHypervisorSnapshotReserve; private Long newServiceOfferingId; private boolean shrinkOk; - public VmWorkResizeVolume(long userId, long accountId, long vmId, String handlerName, - long volumeId, long currentSize, long newSize, Long newMinIops, Long newMaxIops, Long newServiceOfferingId, boolean shrinkOk) { - + public VmWorkResizeVolume(long userId, long accountId, long vmId, String handlerName, long volumeId, long currentSize, long newSize, + Long newMinIops, Long newMaxIops, Integer newHypervisorSnapshotReserve, Long newServiceOfferingId, boolean shrinkOk) { super(userId, accountId, vmId, handlerName); this.volumeId = volumeId; @@ -37,6 +37,7 @@ public class VmWorkResizeVolume extends VmWork { this.newSize = newSize; this.newMinIops = newMinIops; this.newMaxIops = newMaxIops; + this.newHypervisorSnapshotReserve = newHypervisorSnapshotReserve; this.newServiceOfferingId = newServiceOfferingId; this.shrinkOk = shrinkOk; } @@ -68,4 +69,6 @@ public class VmWorkResizeVolume extends VmWork { public boolean isShrinkOk() { return shrinkOk; } + + public Integer getNewHypervisorSnapshotReserve() { return newHypervisorSnapshotReserve; } } http://git-wip-us.apache.org/repos/asf/cloudstack/blob/2bd035d1/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java ---------------------------------------------------------------------- diff --git a/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java b/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java index d407bb1..166210a 100644 --- a/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java +++ b/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java @@ -1242,10 +1242,11 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati future = volService.createVolumeAsync(volume, destPool); } else { - TemplateInfo templ = tmplFactory.getReadyTemplateOnImageStore(templateId, dest.getDataCenter().getId()); + if (templ == null) { s_logger.debug("can't find ready template: " + templateId + " for data center " + dest.getDataCenter().getId()); + throw new CloudRuntimeException("can't find ready template: " + templateId + " for data center " + dest.getDataCenter().getId()); } @@ -1260,13 +1261,13 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati long hostId = vm.getVirtualMachine().getHostId(); - future = volService.createManagedStorageAndVolumeFromTemplateAsync(volume, destPool.getId(), templ, hostId); + future = volService.createManagedStorageVolumeFromTemplateAsync(volume, destPool.getId(), templ, hostId); } else { future = volService.createVolumeFromTemplateAsync(volume, destPool.getId(), templ); } } - VolumeApiResult result = null; + VolumeApiResult result; try { result = future.get(); if (result.isFailed()) { @@ -1290,10 +1291,7 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati newVol = _volsDao.findById(newVol.getId()); break; //break out of template-redeploy retry loop - } catch (InterruptedException e) { - s_logger.error("Unable to create " + newVol, e); - throw new StorageUnavailableException("Unable to create " + newVol + ":" + e.toString(), destPool.getId()); - } catch (ExecutionException e) { + } catch (InterruptedException | ExecutionException e) { s_logger.error("Unable to create " + newVol, e); throw new StorageUnavailableException("Unable to create " + newVol + ":" + e.toString(), destPool.getId()); } http://git-wip-us.apache.org/repos/asf/cloudstack/blob/2bd035d1/engine/schema/src/com/cloud/dc/dao/ClusterDao.java ---------------------------------------------------------------------- diff --git a/engine/schema/src/com/cloud/dc/dao/ClusterDao.java b/engine/schema/src/com/cloud/dc/dao/ClusterDao.java index 50c234c..8e02822 100644 --- a/engine/schema/src/com/cloud/dc/dao/ClusterDao.java +++ b/engine/schema/src/com/cloud/dc/dao/ClusterDao.java @@ -45,4 +45,6 @@ public interface ClusterDao extends GenericDao<ClusterVO, Long> { List<ClusterVO> listClustersByDcId(long zoneId); List<Long> listAllCusters(long zoneId); + + boolean computeWhetherClusterSupportsResigning(long clusterId); } http://git-wip-us.apache.org/repos/asf/cloudstack/blob/2bd035d1/engine/schema/src/com/cloud/dc/dao/ClusterDaoImpl.java ---------------------------------------------------------------------- diff --git a/engine/schema/src/com/cloud/dc/dao/ClusterDaoImpl.java b/engine/schema/src/com/cloud/dc/dao/ClusterDaoImpl.java index 3459e51..aa0b21b 100644 --- a/engine/schema/src/com/cloud/dc/dao/ClusterDaoImpl.java +++ b/engine/schema/src/com/cloud/dc/dao/ClusterDaoImpl.java @@ -30,6 +30,9 @@ import org.springframework.stereotype.Component; import com.cloud.dc.ClusterVO; import com.cloud.dc.HostPodVO; +import com.cloud.host.HostVO; +import com.cloud.host.dao.HostDao; +import com.cloud.host.dao.HostDetailsDao; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.org.Grouping; import com.cloud.utils.db.GenericDaoBase; @@ -57,7 +60,11 @@ public class ClusterDaoImpl extends GenericDaoBase<ClusterVO, Long> implements C private static final String GET_POD_CLUSTER_MAP_PREFIX = "SELECT pod_id, id FROM cloud.cluster WHERE cluster.id IN( "; private static final String GET_POD_CLUSTER_MAP_SUFFIX = " )"; @Inject - protected HostPodDao _hostPodDao; + private HostDao hostDao; + @Inject + private HostDetailsDao hostDetailsDao; + @Inject + protected HostPodDao hostPodDao; public ClusterDaoImpl() { super(); @@ -214,7 +221,7 @@ public class ClusterDaoImpl extends GenericDaoBase<ClusterVO, Long> implements C @Override public List<Long> listClustersWithDisabledPods(long zoneId) { - GenericSearchBuilder<HostPodVO, Long> disabledPodIdSearch = _hostPodDao.createSearchBuilder(Long.class); + GenericSearchBuilder<HostPodVO, Long> disabledPodIdSearch = hostPodDao.createSearchBuilder(Long.class); disabledPodIdSearch.selectFields(disabledPodIdSearch.entity().getId()); disabledPodIdSearch.and("dataCenterId", disabledPodIdSearch.entity().getDataCenterId(), Op.EQ); disabledPodIdSearch.and("allocationState", disabledPodIdSearch.entity().getAllocationState(), Op.EQ); @@ -260,4 +267,35 @@ public class ClusterDaoImpl extends GenericDaoBase<ClusterVO, Long> implements C sc.setParameters("dataCenterId", zoneId); return customSearch(sc, null); } + + @Override + public boolean computeWhetherClusterSupportsResigning(long clusterId) { + ClusterVO cluster = findById(clusterId); + + if (cluster == null || cluster.getAllocationState() != Grouping.AllocationState.Enabled) { + return false; + } + + List<HostVO> hosts = hostDao.findByClusterId(clusterId); + + if (hosts == null) { + return false; + } + + Map<Long, String> mapSupportsResign = hostDetailsDao.findDetails("supportsResign"); + + for (HostVO host : hosts) { + if (host == null) { + return false; + } + + String value = mapSupportsResign.get(host.getId()); + + if (Boolean.parseBoolean(value) == false) { + return false; + } + } + + return true; + } } http://git-wip-us.apache.org/repos/asf/cloudstack/blob/2bd035d1/engine/schema/src/com/cloud/host/dao/HostDao.java ---------------------------------------------------------------------- diff --git a/engine/schema/src/com/cloud/host/dao/HostDao.java b/engine/schema/src/com/cloud/host/dao/HostDao.java index 26e0644..3cfdc94 100644 --- a/engine/schema/src/com/cloud/host/dao/HostDao.java +++ b/engine/schema/src/com/cloud/host/dao/HostDao.java @@ -23,6 +23,7 @@ import com.cloud.host.Host; import com.cloud.host.Host.Type; import com.cloud.host.HostVO; import com.cloud.host.Status; +import com.cloud.hypervisor.Hypervisor; import com.cloud.info.RunningHostCountInfo; import com.cloud.resource.ResourceState; import com.cloud.utils.db.GenericDao; @@ -89,6 +90,8 @@ public interface HostDao extends GenericDao<HostVO, Long>, StateDao<Status, Stat List<HostVO> listByDataCenterId(long id); + List<HostVO> listByDataCenterIdAndHypervisorType(long zoneId, Hypervisor.HypervisorType hypervisorType); + List<Long> listAllHosts(long zoneId); List<HostVO> listAllHostsByType(Host.Type type); http://git-wip-us.apache.org/repos/asf/cloudstack/blob/2bd035d1/engine/schema/src/com/cloud/host/dao/HostDaoImpl.java ---------------------------------------------------------------------- diff --git a/engine/schema/src/com/cloud/host/dao/HostDaoImpl.java b/engine/schema/src/com/cloud/host/dao/HostDaoImpl.java index 09d9d40..54133b9 100644 --- a/engine/schema/src/com/cloud/host/dao/HostDaoImpl.java +++ b/engine/schema/src/com/cloud/host/dao/HostDaoImpl.java @@ -47,7 +47,9 @@ import com.cloud.host.HostTagVO; import com.cloud.host.HostVO; import com.cloud.host.Status; import com.cloud.host.Status.Event; +import com.cloud.hypervisor.Hypervisor; import com.cloud.info.RunningHostCountInfo; +import com.cloud.org.Grouping; import com.cloud.org.Managed; import com.cloud.resource.ResourceState; import com.cloud.utils.DateUtil; @@ -422,6 +424,37 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao } @Override + public List<HostVO> listByDataCenterIdAndHypervisorType(long zoneId, Hypervisor.HypervisorType hypervisorType) { + SearchBuilder<ClusterVO> clusterSearch = _clusterDao.createSearchBuilder(); + + clusterSearch.and("allocationState", clusterSearch.entity().getAllocationState(), SearchCriteria.Op.EQ); + clusterSearch.and("hypervisorType", clusterSearch.entity().getHypervisorType(), SearchCriteria.Op.EQ); + + SearchBuilder<HostVO> hostSearch = createSearchBuilder(); + + hostSearch.and("dc", hostSearch.entity().getDataCenterId(), SearchCriteria.Op.EQ); + hostSearch.and("type", hostSearch.entity().getType(), Op.EQ); + hostSearch.and("status", hostSearch.entity().getStatus(), Op.EQ); + hostSearch.and("resourceState", hostSearch.entity().getResourceState(), Op.EQ); + + hostSearch.join("clusterSearch", clusterSearch, hostSearch.entity().getClusterId(), clusterSearch.entity().getId(), JoinBuilder.JoinType.INNER); + + hostSearch.done(); + + SearchCriteria<HostVO> sc = hostSearch.create(); + + sc.setParameters("dc", zoneId); + sc.setParameters("type", Host.Type.Routing); + sc.setParameters("status", Status.Up); + sc.setParameters("resourceState", ResourceState.Enabled); + + sc.setJoinParameters("clusterSearch", "allocationState", Grouping.AllocationState.Enabled); + sc.setJoinParameters("clusterSearch", "hypervisorType", hypervisorType.toString()); + + return listBy(sc); + } + + @Override public HostVO findByGuid(String guid) { SearchCriteria<HostVO> sc = GuidSearch.create("guid", guid); return findOneBy(sc); http://git-wip-us.apache.org/repos/asf/cloudstack/blob/2bd035d1/engine/schema/src/com/cloud/host/dao/HostDetailsDao.java ---------------------------------------------------------------------- diff --git a/engine/schema/src/com/cloud/host/dao/HostDetailsDao.java b/engine/schema/src/com/cloud/host/dao/HostDetailsDao.java index 7f1a618..77e45dd 100644 --- a/engine/schema/src/com/cloud/host/dao/HostDetailsDao.java +++ b/engine/schema/src/com/cloud/host/dao/HostDetailsDao.java @@ -24,6 +24,8 @@ import com.cloud.utils.db.GenericDao; public interface HostDetailsDao extends GenericDao<DetailVO, Long> { Map<String, String> findDetails(long hostId); + Map<Long, String> findDetails(String name); + void persist(long hostId, Map<String, String> details); DetailVO findDetail(long hostId, String name); http://git-wip-us.apache.org/repos/asf/cloudstack/blob/2bd035d1/engine/schema/src/com/cloud/host/dao/HostDetailsDaoImpl.java ---------------------------------------------------------------------- diff --git a/engine/schema/src/com/cloud/host/dao/HostDetailsDaoImpl.java b/engine/schema/src/com/cloud/host/dao/HostDetailsDaoImpl.java index 6a8ff56..525ee9a 100644 --- a/engine/schema/src/com/cloud/host/dao/HostDetailsDaoImpl.java +++ b/engine/schema/src/com/cloud/host/dao/HostDetailsDaoImpl.java @@ -37,6 +37,7 @@ import com.cloud.utils.exception.CloudRuntimeException; public class HostDetailsDaoImpl extends GenericDaoBase<DetailVO, Long> implements HostDetailsDao { protected final SearchBuilder<DetailVO> HostSearch; protected final SearchBuilder<DetailVO> DetailSearch; + protected final SearchBuilder<DetailVO> NameSearch; public HostDetailsDaoImpl() { HostSearch = createSearchBuilder(); @@ -47,6 +48,10 @@ public class HostDetailsDaoImpl extends GenericDaoBase<DetailVO, Long> implement DetailSearch.and("hostId", DetailSearch.entity().getHostId(), SearchCriteria.Op.EQ); DetailSearch.and("name", DetailSearch.entity().getName(), SearchCriteria.Op.EQ); DetailSearch.done(); + + NameSearch = createSearchBuilder(); + NameSearch.and("name", NameSearch.entity().getName(), SearchCriteria.Op.EQ); + NameSearch.done(); } @Override @@ -65,10 +70,13 @@ public class HostDetailsDaoImpl extends GenericDaoBase<DetailVO, Long> implement @Override public Map<String, String> findDetails(long hostId) { SearchCriteria<DetailVO> sc = HostSearch.create(); + sc.setParameters("hostId", hostId); List<DetailVO> results = search(sc, null); + Map<String, String> details = new HashMap<String, String>(results.size()); + for (DetailVO result : results) { if ("password".equals(result.getName())) { details.put(result.getName(), DBEncryptionUtil.decrypt(result.getValue())); @@ -76,6 +84,28 @@ public class HostDetailsDaoImpl extends GenericDaoBase<DetailVO, Long> implement details.put(result.getName(), result.getValue()); } } + + return details; + } + + @Override + public Map<Long, String> findDetails(String name) { + SearchCriteria<DetailVO> sc = NameSearch.create(); + + sc.setParameters("name", name); + + List<DetailVO> results = search(sc, null); + + Map<Long, String> details = new HashMap<>(results.size()); + + for (DetailVO result : results) { + if ("password".equals(result.getName())) { + details.put(result.getHostId(), DBEncryptionUtil.decrypt(result.getValue())); + } else { + details.put(result.getHostId(), result.getValue()); + } + } + return details; } http://git-wip-us.apache.org/repos/asf/cloudstack/blob/2bd035d1/engine/schema/src/com/cloud/storage/dao/VMTemplatePoolDaoImpl.java ---------------------------------------------------------------------- diff --git a/engine/schema/src/com/cloud/storage/dao/VMTemplatePoolDaoImpl.java b/engine/schema/src/com/cloud/storage/dao/VMTemplatePoolDaoImpl.java index 770e673..93aad15 100644 --- a/engine/schema/src/com/cloud/storage/dao/VMTemplatePoolDaoImpl.java +++ b/engine/schema/src/com/cloud/storage/dao/VMTemplatePoolDaoImpl.java @@ -118,9 +118,9 @@ public class VMTemplatePoolDaoImpl extends GenericDaoBase<VMTemplateStoragePoolV } @Override - public VMTemplateStoragePoolVO findByPoolTemplate(long hostId, long templateId) { + public VMTemplateStoragePoolVO findByPoolTemplate(long poolId, long templateId) { SearchCriteria<VMTemplateStoragePoolVO> sc = PoolTemplateSearch.create(); - sc.setParameters("pool_id", hostId); + sc.setParameters("pool_id", poolId); sc.setParameters("template_id", templateId); return findOneIncludingRemovedBy(sc); } http://git-wip-us.apache.org/repos/asf/cloudstack/blob/2bd035d1/engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java ---------------------------------------------------------------------- diff --git a/engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java b/engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java index cdcab75..adb1720 100644 --- a/engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java +++ b/engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java @@ -18,18 +18,27 @@ */ package org.apache.cloudstack.storage.motion; +import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Random; +import java.util.concurrent.ExecutionException; import javax.inject.Inject; +import com.cloud.dc.dao.ClusterDao; +import com.cloud.exception.AgentUnavailableException; +import com.cloud.exception.OperationTimedoutException; + import org.apache.cloudstack.engine.subsystem.api.storage.ChapInfo; import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult; import org.apache.cloudstack.engine.subsystem.api.storage.DataMotionStrategy; import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreCapabilities; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine.Event; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; import org.apache.cloudstack.engine.subsystem.api.storage.StrategyPriority; @@ -43,8 +52,12 @@ import org.apache.cloudstack.framework.async.AsyncCompletionCallback; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.storage.command.CopyCmdAnswer; import org.apache.cloudstack.storage.command.CopyCommand; +import org.apache.cloudstack.storage.command.ResignatureAnswer; +import org.apache.cloudstack.storage.command.ResignatureCommand; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.storage.to.VolumeObjectTO; +import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -55,65 +68,98 @@ import com.cloud.configuration.Config; import com.cloud.host.Host; import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; +import com.cloud.host.dao.HostDetailsDao; import com.cloud.hypervisor.Hypervisor.HypervisorType; -import com.cloud.org.Cluster; -import com.cloud.org.Grouping.AllocationState; -import com.cloud.resource.ResourceState; import com.cloud.server.ManagementService; import com.cloud.storage.DataStoreRole; import com.cloud.storage.DiskOfferingVO; import com.cloud.storage.SnapshotVO; import com.cloud.storage.Storage.ImageFormat; +import com.cloud.storage.VolumeDetailVO; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.DiskOfferingDao; import com.cloud.storage.dao.SnapshotDao; import com.cloud.storage.dao.SnapshotDetailsDao; import com.cloud.storage.dao.SnapshotDetailsVO; import com.cloud.storage.dao.VolumeDao; +import com.cloud.storage.dao.VolumeDetailsDao; import com.cloud.utils.NumbersUtil; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VirtualMachineManager; +import com.google.common.base.Preconditions; + @Component public class StorageSystemDataMotionStrategy implements DataMotionStrategy { - private static final Logger s_logger = Logger.getLogger(StorageSystemDataMotionStrategy.class); + private static final Logger LOGGER = Logger.getLogger(StorageSystemDataMotionStrategy.class); + private static final Random RANDOM = new Random(System.nanoTime()); @Inject private AgentManager _agentMgr; @Inject private ConfigurationDao _configDao; + @Inject private DataStoreManager dataStoreMgr; @Inject private DiskOfferingDao _diskOfferingDao; + @Inject private ClusterDao clusterDao; @Inject private HostDao _hostDao; + @Inject private HostDetailsDao hostDetailsDao; @Inject private ManagementService _mgr; @Inject private PrimaryDataStoreDao _storagePoolDao; @Inject private SnapshotDao _snapshotDao; @Inject private SnapshotDetailsDao _snapshotDetailsDao; @Inject private VolumeDao _volumeDao; @Inject private VolumeDataFactory _volumeDataFactory; + @Inject private VolumeDetailsDao volumeDetailsDao; @Inject private VolumeService _volumeService; @Override public StrategyPriority canHandle(DataObject srcData, DataObject destData) { if (srcData instanceof SnapshotInfo) { - if (canHandle(srcData.getDataStore()) || canHandle(destData.getDataStore())) { + if (canHandle(srcData) || canHandle(destData)) { return StrategyPriority.HIGHEST; } } + if (srcData instanceof TemplateInfo && destData instanceof VolumeInfo && + (srcData.getDataStore().getId() == destData.getDataStore().getId()) && + (canHandle(srcData) || canHandle(destData))) { + // Both source and dest are on the same storage, so just clone them. + return StrategyPriority.HIGHEST; + } + return StrategyPriority.CANT_HANDLE; } - private boolean canHandle(DataStore dataStore) { + private boolean canHandle(DataObject dataObject) { + Preconditions.checkArgument(dataObject != null, "Passing 'null' to dataObject of canHandle(DataObject) is not supported."); + + DataStore dataStore = dataObject.getDataStore(); + if (dataStore.getRole() == DataStoreRole.Primary) { Map<String, String> mapCapabilities = dataStore.getDriver().getCapabilities(); - if (mapCapabilities != null) { + if (mapCapabilities == null) { + return false; + } + + if (dataObject instanceof VolumeInfo || dataObject instanceof SnapshotInfo) { String value = mapCapabilities.get(DataStoreCapabilities.STORAGE_SYSTEM_SNAPSHOT.toString()); - Boolean supportsStorageSystemSnapshots = new Boolean(value); + Boolean supportsStorageSystemSnapshots = Boolean.valueOf(value); if (supportsStorageSystemSnapshots) { - s_logger.info("Using 'StorageSystemDataMotionStrategy'"); + LOGGER.info("Using 'StorageSystemDataMotionStrategy' (dataObject is a volume or snapshot and the storage system supports snapshots)"); + + return true; + } + } else if (dataObject instanceof TemplateInfo) { + // If the storage system can clone volumes, we can cache templates on it. + String value = mapCapabilities.get(DataStoreCapabilities.CAN_CREATE_VOLUME_FROM_VOLUME.toString()); + Boolean canCloneVolume = Boolean.valueOf(value); + + if (canCloneVolume) { + LOGGER.info("Using 'StorageSystemDataMotionStrategy' (dataObject is a template and the storage system can create a volume from a volume)"); return true; } + } } @@ -132,36 +178,92 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { validate(snapshotInfo); - boolean canHandleSrc = canHandle(srcData.getDataStore()); + boolean canHandleSrc = canHandle(srcData); if (canHandleSrc && destData instanceof TemplateInfo && (destData.getDataStore().getRole() == DataStoreRole.Image || destData.getDataStore().getRole() == DataStoreRole.ImageCache)) { handleCreateTemplateFromSnapshot(snapshotInfo, (TemplateInfo)destData, callback); + return; } if (destData instanceof VolumeInfo) { VolumeInfo volumeInfo = (VolumeInfo)destData; - boolean canHandleDest = canHandle(destData.getDataStore()); + + boolean canHandleDest = canHandle(destData); if (canHandleSrc && canHandleDest) { - handleCreateVolumeFromSnapshotBothOnStorageSystem(snapshotInfo, volumeInfo, callback); - return; + if (snapshotInfo.getDataStore().getId() == volumeInfo.getDataStore().getId()) { + handleCreateVolumeFromSnapshotBothOnStorageSystem(snapshotInfo, volumeInfo, callback); + return; + } + else { + String errMsg = "This operation is not supported (DataStoreCapabilities.STORAGE_SYSTEM_SNAPSHOT " + + "not supported by source or destination storage plug-in). " + getSrcDestDataStoreMsg(srcData, destData); + + LOGGER.warn(errMsg); + + throw new UnsupportedOperationException(errMsg); + } } + if (canHandleSrc) { - throw new UnsupportedOperationException("This operation is not supported (DataStoreCapabilities.STORAGE_SYSTEM_SNAPSHOT " + - "not supported by destination storage plug-in)."); + String errMsg = "This operation is not supported (DataStoreCapabilities.STORAGE_SYSTEM_SNAPSHOT " + + "not supported by destination storage plug-in). " + getDestDataStoreMsg(destData); + + LOGGER.warn(errMsg); + + throw new UnsupportedOperationException(errMsg); } + if (canHandleDest) { - throw new UnsupportedOperationException("This operation is not supported (DataStoreCapabilities.STORAGE_SYSTEM_SNAPSHOT " + - "not supported by source storage plug-in)."); + String errMsg = "This operation is not supported (DataStoreCapabilities.STORAGE_SYSTEM_SNAPSHOT " + + "not supported by source storage plug-in). " + getSrcDataStoreMsg(srcData); + + LOGGER.warn(errMsg); + + throw new UnsupportedOperationException(errMsg); } } + } else if (srcData instanceof TemplateInfo && destData instanceof VolumeInfo) { + boolean canHandleSrc = canHandle(srcData); + + if (!canHandleSrc) { + String errMsg = "This operation is not supported (DataStoreCapabilities.STORAGE_CAN_CREATE_VOLUME_FROM_VOLUME " + + "not supported by destination storage plug-in). " + getDestDataStoreMsg(destData); + + LOGGER.warn(errMsg); + + throw new UnsupportedOperationException(errMsg); + } + + handleCreateVolumeFromTemplateBothOnStorageSystem((TemplateInfo)srcData, (VolumeInfo)destData, callback); + + return; } throw new UnsupportedOperationException("This operation is not supported."); } + private String getSrcDestDataStoreMsg(DataObject srcData, DataObject destData) { + Preconditions.checkArgument(srcData != null, "Passing 'null' to srcData of getSrcDestDataStoreMsg(DataObject, DataObject) is not supported."); + Preconditions.checkArgument(destData != null, "Passing 'null' to destData of getSrcDestDataStoreMsg(DataObject, DataObject) is not supported."); + + return "Source data store = " + srcData.getDataStore().getName() + "; " + "Destination data store = " + destData.getDataStore().getName() + "."; + } + + private String getSrcDataStoreMsg(DataObject srcData) { + Preconditions.checkArgument(srcData != null, "Passing 'null' to srcData of getSrcDataStoreMsg(DataObject) is not supported."); + + return "Source data store = " + srcData.getDataStore().getName() + "."; + } + + private String getDestDataStoreMsg(DataObject destData) { + Preconditions.checkArgument(destData != null, "Passing 'null' to destData of getDestDataStoreMsg(DataObject) is not supported."); + + return "Destination data store = " + destData.getDataStore().getName() + "."; + } + private void validate(SnapshotInfo snapshotInfo) { long volumeId = snapshotInfo.getVolumeId(); @@ -172,7 +274,13 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { } } - private Void handleCreateTemplateFromSnapshot(SnapshotInfo snapshotInfo, TemplateInfo templateInfo, AsyncCompletionCallback<CopyCommandResult> callback) { + private boolean usingBackendSnapshotFor(SnapshotInfo snapshotInfo) { + String property = getProperty(snapshotInfo.getId(), "takeSnapshot"); + + return Boolean.parseBoolean(property); + } + + private void handleCreateTemplateFromSnapshot(SnapshotInfo snapshotInfo, TemplateInfo templateInfo, AsyncCompletionCallback<CopyCommandResult> callback) { try { snapshotInfo.processEvent(Event.CopyingRequested); } @@ -180,57 +288,168 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { throw new CloudRuntimeException("This snapshot is not currently in a state where it can be used to create a template."); } - HostVO hostVO = getHost(snapshotInfo.getDataStore().getId()); - DataStore srcDataStore = snapshotInfo.getDataStore(); + HostVO hostVO = getHost(snapshotInfo); - String value = _configDao.getValue(Config.PrimaryStorageDownloadWait.toString()); - int primaryStorageDownloadWait = NumbersUtil.parseInt(value, Integer.parseInt(Config.PrimaryStorageDownloadWait.getDefaultValue())); - CopyCommand copyCommand = new CopyCommand(snapshotInfo.getTO(), templateInfo.getTO(), primaryStorageDownloadWait, VirtualMachineManager.ExecuteInSequence.value()); + boolean usingBackendSnapshot = usingBackendSnapshotFor(snapshotInfo); + boolean computeClusterSupportsResign = clusterDao.computeWhetherClusterSupportsResigning(hostVO.getClusterId()); - String errMsg = null; + if (usingBackendSnapshot && !computeClusterSupportsResign) { + String noSupportForResignErrMsg = "Unable to locate an applicable host with which to perform a resignature operation : Cluster ID = " + hostVO.getClusterId(); - CopyCmdAnswer copyCmdAnswer = null; + LOGGER.warn(noSupportForResignErrMsg); + + throw new CloudRuntimeException(noSupportForResignErrMsg); + } try { - _volumeService.grantAccess(snapshotInfo, hostVO, srcDataStore); + if (usingBackendSnapshot) { + createVolumeFromSnapshot(hostVO, snapshotInfo, true); + } - Map<String, String> srcDetails = getSnapshotDetails(_storagePoolDao.findById(srcDataStore.getId()), snapshotInfo); + DataStore srcDataStore = snapshotInfo.getDataStore(); - copyCommand.setOptions(srcDetails); + String value = _configDao.getValue(Config.PrimaryStorageDownloadWait.toString()); + int primaryStorageDownloadWait = NumbersUtil.parseInt(value, Integer.parseInt(Config.PrimaryStorageDownloadWait.getDefaultValue())); + CopyCommand copyCommand = new CopyCommand(snapshotInfo.getTO(), templateInfo.getTO(), primaryStorageDownloadWait, VirtualMachineManager.ExecuteInSequence.value()); + + String errMsg = null; + + CopyCmdAnswer copyCmdAnswer = null; - copyCmdAnswer = (CopyCmdAnswer)_agentMgr.send(hostVO.getId(), copyCommand); - } - catch (Exception ex) { - throw new CloudRuntimeException(ex.getMessage()); - } - finally { try { - _volumeService.revokeAccess(snapshotInfo, hostVO, srcDataStore); + // If we are using a back-end snapshot, then we should still have access to it from the hosts in the cluster that hostVO is in + // (because we passed in true as the third parameter to createVolumeFromSnapshot above). + if (usingBackendSnapshot == false) { + _volumeService.grantAccess(snapshotInfo, hostVO, srcDataStore); + } + + Map<String, String> srcDetails = getSnapshotDetails(snapshotInfo); + + copyCommand.setOptions(srcDetails); + + copyCmdAnswer = (CopyCmdAnswer)_agentMgr.send(hostVO.getId(), copyCommand); } - catch (Exception ex) { - s_logger.debug(ex.getMessage(), ex); + catch (CloudRuntimeException | AgentUnavailableException | OperationTimedoutException ex) { + String msg = "Failed to create template from snapshot (Snapshot ID = " + snapshotInfo.getId() + ") : "; + + LOGGER.warn(msg, ex); + + throw new CloudRuntimeException(msg + ex.getMessage()); } + finally { + try { + _volumeService.revokeAccess(snapshotInfo, hostVO, srcDataStore); + } + catch (Exception ex) { + LOGGER.warn("Error revoking access to snapshot (Snapshot ID = " + snapshotInfo.getId() + "): " + ex.getMessage(), ex); + } - if (copyCmdAnswer == null || !copyCmdAnswer.getResult()) { - if (copyCmdAnswer != null && copyCmdAnswer.getDetails() != null && !copyCmdAnswer.getDetails().isEmpty()) { - errMsg = copyCmdAnswer.getDetails(); + if (copyCmdAnswer == null || !copyCmdAnswer.getResult()) { + if (copyCmdAnswer != null && !StringUtils.isEmpty(copyCmdAnswer.getDetails())) { + errMsg = copyCmdAnswer.getDetails(); + } + else { + errMsg = "Unable to create template from snapshot"; + } } - else { - errMsg = "Unable to perform host-side operation"; + + try { + if (StringUtils.isEmpty(errMsg)) { + snapshotInfo.processEvent(Event.OperationSuccessed); + } + else { + snapshotInfo.processEvent(Event.OperationFailed); + } + } + catch (Exception ex) { + LOGGER.warn("Error processing snapshot event: " + ex.getMessage(), ex); } } - try { - if (errMsg == null) { - snapshotInfo.processEvent(Event.OperationSuccessed); + CopyCommandResult result = new CopyCommandResult(null, copyCmdAnswer); + + result.setResult(errMsg); + + callback.complete(result); + } + finally { + if (usingBackendSnapshot) { + deleteVolumeFromSnapshot(snapshotInfo); + } + } + } + + /** + * Clones a template present on the storage to a new volume and resignatures it. + * + * @param templateInfo source template + * @param volumeInfo destination ROOT volume + * @param callback for async + */ + private void handleCreateVolumeFromTemplateBothOnStorageSystem(TemplateInfo templateInfo, VolumeInfo volumeInfo, AsyncCompletionCallback<CopyCommandResult> callback) { + Preconditions.checkArgument(templateInfo != null, "Passing 'null' to templateInfo of handleCreateVolumeFromTemplateBothOnStorageSystem is not supported."); + Preconditions.checkArgument(volumeInfo != null, "Passing 'null' to volumeInfo of handleCreateVolumeFromTemplateBothOnStorageSystem is not supported."); + + CopyCmdAnswer copyCmdAnswer = null; + String errMsg = null; + + HostVO hostVO = getHost(volumeInfo.getDataCenterId(), true); + + if (hostVO == null) { + throw new CloudRuntimeException("Unable to locate a host capable of resigning in the zone with the following ID: " + volumeInfo.getDataCenterId()); + } + + boolean computeClusterSupportsResign = clusterDao.computeWhetherClusterSupportsResigning(hostVO.getClusterId()); + + if (!computeClusterSupportsResign) { + String noSupportForResignErrMsg = "Unable to locate an applicable host with which to perform a resignature operation : Cluster ID = " + hostVO.getClusterId(); + + LOGGER.warn(noSupportForResignErrMsg); + + throw new CloudRuntimeException(noSupportForResignErrMsg); + } + + try { + VolumeDetailVO volumeDetail = new VolumeDetailVO(volumeInfo.getId(), + "cloneOfTemplate", + String.valueOf(templateInfo.getId()), + false); + + volumeDetail = volumeDetailsDao.persist(volumeDetail); + + AsyncCallFuture<VolumeApiResult> future = _volumeService.createVolumeAsync(volumeInfo, volumeInfo.getDataStore()); + VolumeApiResult result = future.get(); + + if (volumeDetail != null) { + volumeDetailsDao.remove(volumeDetail.getId()); + } + + if (result.isFailed()) { + LOGGER.warn("Failed to create a volume: " + result.getResult()); + + throw new CloudRuntimeException(result.getResult()); + } + + volumeInfo = _volumeDataFactory.getVolume(volumeInfo.getId(), volumeInfo.getDataStore()); + + volumeInfo.processEvent(Event.MigrationRequested); + + volumeInfo = _volumeDataFactory.getVolume(volumeInfo.getId(), volumeInfo.getDataStore()); + + copyCmdAnswer = performResignature(volumeInfo, hostVO); + + if (copyCmdAnswer == null || !copyCmdAnswer.getResult()) { + if (copyCmdAnswer != null && !StringUtils.isEmpty(copyCmdAnswer.getDetails())) { + throw new CloudRuntimeException(copyCmdAnswer.getDetails()); } else { - snapshotInfo.processEvent(Event.OperationFailed); + throw new CloudRuntimeException("Unable to create a volume from a template"); } } - catch (Exception ex) { - s_logger.debug(ex.getMessage(), ex); - } + } catch (InterruptedException | ExecutionException ex) { + volumeInfo.getDataStore().getDriver().deleteAsync(volumeInfo.getDataStore(), volumeInfo, null); + + throw new CloudRuntimeException("Create volume from template (ID = " + templateInfo.getId() + ") failed " + ex.getMessage()); } CopyCommandResult result = new CopyCommandResult(null, copyCmdAnswer); @@ -238,12 +457,40 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { result.setResult(errMsg); callback.complete(result); - - return null; } - private Void handleCreateVolumeFromSnapshotBothOnStorageSystem(SnapshotInfo snapshotInfo, VolumeInfo volumeInfo, AsyncCompletionCallback<CopyCommandResult> callback) { + private void handleCreateVolumeFromSnapshotBothOnStorageSystem(SnapshotInfo snapshotInfo, VolumeInfo volumeInfo, AsyncCompletionCallback<CopyCommandResult> callback) { + CopyCmdAnswer copyCmdAnswer = null; + String errMsg = null; + try { + HostVO hostVO = getHost(snapshotInfo); + + boolean usingBackendSnapshot = usingBackendSnapshotFor(snapshotInfo); + boolean computeClusterSupportsResign = clusterDao.computeWhetherClusterSupportsResigning(hostVO.getClusterId()); + + if (usingBackendSnapshot && !computeClusterSupportsResign) { + String noSupportForResignErrMsg = "Unable to locate an applicable host with which to perform a resignature operation : Cluster ID = " + hostVO.getClusterId(); + + LOGGER.warn(noSupportForResignErrMsg); + + throw new CloudRuntimeException(noSupportForResignErrMsg); + } + + boolean canStorageSystemCreateVolumeFromVolume = canStorageSystemCreateVolumeFromVolume(snapshotInfo); + boolean useCloning = usingBackendSnapshot || (canStorageSystemCreateVolumeFromVolume && computeClusterSupportsResign); + + VolumeDetailVO volumeDetail = null; + + if (useCloning) { + volumeDetail = new VolumeDetailVO(volumeInfo.getId(), + "cloneOfSnapshot", + String.valueOf(snapshotInfo.getId()), + false); + + volumeDetail = volumeDetailsDao.persist(volumeDetail); + } + // at this point, the snapshotInfo and volumeInfo should have the same disk offering ID (so either one should be OK to get a DiskOfferingVO instance) DiskOfferingVO diskOffering = _diskOfferingDao.findByIdIncludingRemoved(volumeInfo.getDiskOfferingId()); SnapshotVO snapshot = _snapshotDao.findById(snapshotInfo.getId()); @@ -255,99 +502,123 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { VolumeApiResult result = future.get(); + if (volumeDetail != null) { + volumeDetailsDao.remove(volumeDetail.getId()); + } + if (result.isFailed()) { - s_logger.debug("Failed to create a volume: " + result.getResult()); + LOGGER.warn("Failed to create a volume: " + result.getResult()); throw new CloudRuntimeException(result.getResult()); } - } - catch (Exception ex) { - throw new CloudRuntimeException(ex.getMessage()); - } - volumeInfo = _volumeDataFactory.getVolume(volumeInfo.getId(), volumeInfo.getDataStore()); + volumeInfo = _volumeDataFactory.getVolume(volumeInfo.getId(), volumeInfo.getDataStore()); - volumeInfo.processEvent(Event.MigrationRequested); + volumeInfo.processEvent(Event.MigrationRequested); - volumeInfo = _volumeDataFactory.getVolume(volumeInfo.getId(), volumeInfo.getDataStore()); + volumeInfo = _volumeDataFactory.getVolume(volumeInfo.getId(), volumeInfo.getDataStore()); - HostVO hostVO = getHost(snapshotInfo.getDataStore().getId()); - - String value = _configDao.getValue(Config.PrimaryStorageDownloadWait.toString()); - int primaryStorageDownloadWait = NumbersUtil.parseInt(value, Integer.parseInt(Config.PrimaryStorageDownloadWait.getDefaultValue())); - CopyCommand copyCommand = new CopyCommand(snapshotInfo.getTO(), volumeInfo.getTO(), primaryStorageDownloadWait, VirtualMachineManager.ExecuteInSequence.value()); + if (useCloning) { + copyCmdAnswer = performResignature(volumeInfo, hostVO); + } + else { + // asking for a XenServer host here so we don't always prefer to use XenServer hosts that support resigning + // even when we don't need those hosts to do this kind of copy work + hostVO = getHost(snapshotInfo.getDataCenterId(), false); - CopyCmdAnswer copyCmdAnswer = null; + copyCmdAnswer = performCopyOfVdi(volumeInfo, snapshotInfo, hostVO); + } - try { - _volumeService.grantAccess(snapshotInfo, hostVO, snapshotInfo.getDataStore()); - _volumeService.grantAccess(volumeInfo, hostVO, volumeInfo.getDataStore()); + if (copyCmdAnswer == null || !copyCmdAnswer.getResult()) { + if (copyCmdAnswer != null && !StringUtils.isEmpty(copyCmdAnswer.getDetails())) { + errMsg = copyCmdAnswer.getDetails(); + } + else { + errMsg = "Unable to create volume from snapshot"; + } + } + } + catch (Exception ex) { + errMsg = ex.getMessage() != null ? ex.getMessage() : "Copy operation failed in 'StorageSystemDataMotionStrategy.handleCreateVolumeFromSnapshotBothOnStorageSystem'"; + } - Map<String, String> srcDetails = getSnapshotDetails(_storagePoolDao.findById(snapshotInfo.getDataStore().getId()), snapshotInfo); + CopyCommandResult result = new CopyCommandResult(null, copyCmdAnswer); - copyCommand.setOptions(srcDetails); + result.setResult(errMsg); - Map<String, String> destDetails = getVolumeDetails(volumeInfo); + callback.complete(result); + } - copyCommand.setOptions2(destDetails); + /** + * If the underlying storage system is making use of read-only snapshots, this gives the storage system the opportunity to + * create a volume from the snapshot so that we can copy the VHD file that should be inside of the snapshot to secondary storage. + * + * The resultant volume must be writable because we need to resign the SR and the VDI that should be inside of it before we copy + * the VHD file to secondary storage. + * + * If the storage system is using writable snapshots, then nothing need be done by that storage system here because we can just + * resign the SR and the VDI that should be inside of the snapshot before copying the VHD file to secondary storage. + */ + private void createVolumeFromSnapshot(HostVO hostVO, SnapshotInfo snapshotInfo, boolean keepGrantedAccess) { + SnapshotDetailsVO snapshotDetails = handleSnapshotDetails(snapshotInfo.getId(), "tempVolume", "create"); - copyCmdAnswer = (CopyCmdAnswer)_agentMgr.send(hostVO.getId(), copyCommand); - } - catch (Exception ex) { - throw new CloudRuntimeException(ex.getMessage()); + try { + snapshotInfo.getDataStore().getDriver().createAsync(snapshotInfo.getDataStore(), snapshotInfo, null); } finally { - try { - _volumeService.revokeAccess(snapshotInfo, hostVO, snapshotInfo.getDataStore()); - } - catch (Exception ex) { - s_logger.debug(ex.getMessage(), ex); - } - - try { - _volumeService.revokeAccess(volumeInfo, hostVO, volumeInfo.getDataStore()); - } - catch (Exception ex) { - s_logger.debug(ex.getMessage(), ex); - } + _snapshotDetailsDao.remove(snapshotDetails.getId()); } - String errMsg = null; + CopyCmdAnswer copyCmdAnswer = performResignature(snapshotInfo, hostVO, keepGrantedAccess); if (copyCmdAnswer == null || !copyCmdAnswer.getResult()) { - if (copyCmdAnswer != null && copyCmdAnswer.getDetails() != null && !copyCmdAnswer.getDetails().isEmpty()) { - errMsg = copyCmdAnswer.getDetails(); + if (copyCmdAnswer != null && !StringUtils.isEmpty(copyCmdAnswer.getDetails())) { + throw new CloudRuntimeException(copyCmdAnswer.getDetails()); } else { - errMsg = "Unable to perform host-side operation"; + throw new CloudRuntimeException("Unable to create volume from snapshot"); } } + } - CopyCommandResult result = new CopyCommandResult(null, copyCmdAnswer); + /** + * If the underlying storage system needed to create a volume from a snapshot for createVolumeFromSnapshot(HostVO, SnapshotInfo), then + * this is its opportunity to delete that temporary volume and restore properties in snapshot_details to the way they were before the + * invocation of createVolumeFromSnapshot(HostVO, SnapshotInfo). + */ + private void deleteVolumeFromSnapshot(SnapshotInfo snapshotInfo) { + SnapshotDetailsVO snapshotDetails = handleSnapshotDetails(snapshotInfo.getId(), "tempVolume", "delete"); - result.setResult(errMsg); + try { + snapshotInfo.getDataStore().getDriver().createAsync(snapshotInfo.getDataStore(), snapshotInfo, null); + } + finally { + _snapshotDetailsDao.remove(snapshotDetails.getId()); + } + } - callback.complete(result); + private SnapshotDetailsVO handleSnapshotDetails(long csSnapshotId, String name, String value) { + _snapshotDetailsDao.removeDetail(csSnapshotId, name); - return null; + SnapshotDetailsVO snapshotDetails = new SnapshotDetailsVO(csSnapshotId, name, value, false); + + return _snapshotDetailsDao.persist(snapshotDetails); } - private Map<String, String> getSnapshotDetails(StoragePoolVO storagePoolVO, SnapshotInfo snapshotInfo) { - Map<String, String> details = new HashMap<String, String>(); + private boolean canStorageSystemCreateVolumeFromVolume(SnapshotInfo snapshotInfo) { + boolean supportsCloningVolumeFromVolume = false; - details.put(DiskTO.STORAGE_HOST, storagePoolVO.getHostAddress()); - details.put(DiskTO.STORAGE_PORT, String.valueOf(storagePoolVO.getPort())); + DataStore dataStore = dataStoreMgr.getDataStore(snapshotInfo.getDataStore().getId(), DataStoreRole.Primary); - long snapshotId = snapshotInfo.getId(); + Map<String, String> mapCapabilities = dataStore.getDriver().getCapabilities(); - details.put(DiskTO.IQN, getProperty(snapshotId, DiskTO.IQN)); + if (mapCapabilities != null) { + String value = mapCapabilities.get(DataStoreCapabilities.CAN_CREATE_VOLUME_FROM_VOLUME.toString()); - details.put(DiskTO.CHAP_INITIATOR_USERNAME, getProperty(snapshotId, DiskTO.CHAP_INITIATOR_USERNAME)); - details.put(DiskTO.CHAP_INITIATOR_SECRET, getProperty(snapshotId, DiskTO.CHAP_INITIATOR_SECRET)); - details.put(DiskTO.CHAP_TARGET_USERNAME, getProperty(snapshotId, DiskTO.CHAP_TARGET_USERNAME)); - details.put(DiskTO.CHAP_TARGET_SECRET, getProperty(snapshotId, DiskTO.CHAP_TARGET_SECRET)); + supportsCloningVolumeFromVolume = Boolean.valueOf(value); + } - return details; + return supportsCloningVolumeFromVolume; } private String getProperty(long snapshotId, String property) { @@ -361,59 +632,209 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { } private Map<String, String> getVolumeDetails(VolumeInfo volumeInfo) { - Map<String, String> sourceDetails = new HashMap<String, String>(); + Map<String, String> volumeDetails = new HashMap<String, String>(); VolumeVO volumeVO = _volumeDao.findById(volumeInfo.getId()); long storagePoolId = volumeVO.getPoolId(); StoragePoolVO storagePoolVO = _storagePoolDao.findById(storagePoolId); - sourceDetails.put(DiskTO.STORAGE_HOST, storagePoolVO.getHostAddress()); - sourceDetails.put(DiskTO.STORAGE_PORT, String.valueOf(storagePoolVO.getPort())); - sourceDetails.put(DiskTO.IQN, volumeVO.get_iScsiName()); + volumeDetails.put(DiskTO.STORAGE_HOST, storagePoolVO.getHostAddress()); + volumeDetails.put(DiskTO.STORAGE_PORT, String.valueOf(storagePoolVO.getPort())); + volumeDetails.put(DiskTO.IQN, volumeVO.get_iScsiName()); ChapInfo chapInfo = _volumeService.getChapInfo(volumeInfo, volumeInfo.getDataStore()); if (chapInfo != null) { - sourceDetails.put(DiskTO.CHAP_INITIATOR_USERNAME, chapInfo.getInitiatorUsername()); - sourceDetails.put(DiskTO.CHAP_INITIATOR_SECRET, chapInfo.getInitiatorSecret()); - sourceDetails.put(DiskTO.CHAP_TARGET_USERNAME, chapInfo.getTargetUsername()); - sourceDetails.put(DiskTO.CHAP_TARGET_SECRET, chapInfo.getTargetSecret()); + volumeDetails.put(DiskTO.CHAP_INITIATOR_USERNAME, chapInfo.getInitiatorUsername()); + volumeDetails.put(DiskTO.CHAP_INITIATOR_SECRET, chapInfo.getInitiatorSecret()); + volumeDetails.put(DiskTO.CHAP_TARGET_USERNAME, chapInfo.getTargetUsername()); + volumeDetails.put(DiskTO.CHAP_TARGET_SECRET, chapInfo.getTargetSecret()); } - return sourceDetails; + return volumeDetails; } - public HostVO getHost(long dataStoreId) { - StoragePoolVO storagePoolVO = _storagePoolDao.findById(dataStoreId); + private Map<String, String> getSnapshotDetails(SnapshotInfo snapshotInfo) { + Map<String, String> snapshotDetails = new HashMap<String, String>(); + + long storagePoolId = snapshotInfo.getDataStore().getId(); + StoragePoolVO storagePoolVO = _storagePoolDao.findById(storagePoolId); - List<? extends Cluster> clusters = _mgr.searchForClusters(storagePoolVO.getDataCenterId(), new Long(0), Long.MAX_VALUE, HypervisorType.XenServer.toString()); + snapshotDetails.put(DiskTO.STORAGE_HOST, storagePoolVO.getHostAddress()); + snapshotDetails.put(DiskTO.STORAGE_PORT, String.valueOf(storagePoolVO.getPort())); - if (clusters == null) { - throw new CloudRuntimeException("Unable to locate an applicable cluster"); + long snapshotId = snapshotInfo.getId(); + + snapshotDetails.put(DiskTO.IQN, getProperty(snapshotId, DiskTO.IQN)); + + snapshotDetails.put(DiskTO.CHAP_INITIATOR_USERNAME, getProperty(snapshotId, DiskTO.CHAP_INITIATOR_USERNAME)); + snapshotDetails.put(DiskTO.CHAP_INITIATOR_SECRET, getProperty(snapshotId, DiskTO.CHAP_INITIATOR_SECRET)); + snapshotDetails.put(DiskTO.CHAP_TARGET_USERNAME, getProperty(snapshotId, DiskTO.CHAP_TARGET_USERNAME)); + snapshotDetails.put(DiskTO.CHAP_TARGET_SECRET, getProperty(snapshotId, DiskTO.CHAP_TARGET_SECRET)); + + return snapshotDetails; + } + + private HostVO getHost(SnapshotInfo snapshotInfo) { + HostVO hostVO = getHost(snapshotInfo.getDataCenterId(), true); + + if (hostVO == null) { + hostVO = getHost(snapshotInfo.getDataCenterId(), false); + + if (hostVO == null) { + throw new CloudRuntimeException("Unable to locate an applicable host in data center with ID = " + snapshotInfo.getDataCenterId()); + } } - for (Cluster cluster : clusters) { - if (cluster.getAllocationState() == AllocationState.Enabled) { - List<HostVO> hosts = _hostDao.findByClusterId(cluster.getId()); + return hostVO; + } - if (hosts != null) { - for (HostVO host : hosts) { - if (host.getResourceState() == ResourceState.Enabled) { - return host; - } - } + private HostVO getHost(Long zoneId, boolean computeClusterMustSupportResign) { + Preconditions.checkArgument(zoneId != null, "Zone ID cannot be null."); + + List<HostVO> hosts = _hostDao.listByDataCenterIdAndHypervisorType(zoneId, HypervisorType.XenServer); + + if (hosts == null) { + return null; + } + + List<Long> clustersToSkip = new ArrayList<>(); + + Collections.shuffle(hosts, RANDOM); + + for (HostVO host : hosts) { + if (computeClusterMustSupportResign) { + long clusterId = host.getClusterId(); + + if (clustersToSkip.contains(clusterId)) { + continue; + } + + if (clusterDao.computeWhetherClusterSupportsResigning(clusterId)) { + return host; + } + else { + clustersToSkip.add(clusterId); } } + else { + return host; + } } - throw new CloudRuntimeException("Unable to locate an applicable cluster"); + return null; } @Override public void copyAsync(Map<VolumeInfo, DataStore> volumeMap, VirtualMachineTO vmTo, Host srcHost, Host destHost, AsyncCompletionCallback<CopyCommandResult> callback) { CopyCommandResult result = new CopyCommandResult(null, null); + result.setResult("Unsupported operation requested for copying data."); + callback.complete(result); } + + private Map<String, String> getDetails(DataObject dataObj) { + if (dataObj instanceof VolumeInfo) { + return getVolumeDetails((VolumeInfo)dataObj); + } + else if (dataObj instanceof SnapshotInfo) { + return getSnapshotDetails((SnapshotInfo)dataObj); + } + + throw new CloudRuntimeException("'dataObj' must be of type 'VolumeInfo' or 'SnapshotInfo'."); + } + + private CopyCmdAnswer performResignature(DataObject dataObj, HostVO hostVO) { + return performResignature(dataObj, hostVO, false); + } + + private CopyCmdAnswer performResignature(DataObject dataObj, HostVO hostVO, boolean keepGrantedAccess) { + long storagePoolId = dataObj.getDataStore().getId(); + DataStore dataStore = dataStoreMgr.getDataStore(storagePoolId, DataStoreRole.Primary); + + Map<String, String> details = getDetails(dataObj); + + ResignatureCommand command = new ResignatureCommand(details); + + ResignatureAnswer answer = null; + + try { + _volumeService.grantAccess(dataObj, hostVO, dataStore); + + answer = (ResignatureAnswer)_agentMgr.send(hostVO.getId(), command); + } + catch (CloudRuntimeException | AgentUnavailableException | OperationTimedoutException ex) { + keepGrantedAccess = false; + + String msg = "Failed to resign the DataObject with the following ID: " + dataObj.getId(); + + LOGGER.warn(msg, ex); + + throw new CloudRuntimeException(msg + ex.getMessage()); + } + finally { + if (keepGrantedAccess == false) { + _volumeService.revokeAccess(dataObj, hostVO, dataStore); + } + } + + if (answer == null || !answer.getResult()) { + final String errMsg; + + if (answer != null && answer.getDetails() != null && !answer.getDetails().isEmpty()) { + errMsg = answer.getDetails(); + } + else { + errMsg = "Unable to perform resignature operation in 'StorageSystemDataMotionStrategy.performResignature'"; + } + + throw new CloudRuntimeException(errMsg); + } + + VolumeObjectTO newVolume = new VolumeObjectTO(); + + newVolume.setSize(answer.getSize()); + newVolume.setPath(answer.getPath()); + newVolume.setFormat(answer.getFormat()); + + return new CopyCmdAnswer(newVolume); + } + + private CopyCmdAnswer performCopyOfVdi(VolumeInfo volumeInfo, SnapshotInfo snapshotInfo, HostVO hostVO) { + String value = _configDao.getValue(Config.PrimaryStorageDownloadWait.toString()); + int primaryStorageDownloadWait = NumbersUtil.parseInt(value, Integer.parseInt(Config.PrimaryStorageDownloadWait.getDefaultValue())); + CopyCommand copyCommand = new CopyCommand(snapshotInfo.getTO(), volumeInfo.getTO(), primaryStorageDownloadWait, VirtualMachineManager.ExecuteInSequence.value()); + + CopyCmdAnswer copyCmdAnswer = null; + + try { + _volumeService.grantAccess(snapshotInfo, hostVO, snapshotInfo.getDataStore()); + _volumeService.grantAccess(volumeInfo, hostVO, volumeInfo.getDataStore()); + + Map<String, String> srcDetails = getSnapshotDetails(snapshotInfo); + + copyCommand.setOptions(srcDetails); + + Map<String, String> destDetails = getVolumeDetails(volumeInfo); + + copyCommand.setOptions2(destDetails); + + copyCmdAnswer = (CopyCmdAnswer)_agentMgr.send(hostVO.getId(), copyCommand); + } + catch (CloudRuntimeException | AgentUnavailableException | OperationTimedoutException ex) { + String msg = "Failed to perform VDI copy : "; + + LOGGER.warn(msg, ex); + + throw new CloudRuntimeException(msg + ex.getMessage()); + } + finally { + _volumeService.revokeAccess(snapshotInfo, hostVO, snapshotInfo.getDataStore()); + _volumeService.revokeAccess(volumeInfo, hostVO, volumeInfo.getDataStore()); + } + + return copyCmdAnswer; + } }
