This is an automated email from the ASF dual-hosted git repository.

sureshanaparti pushed a commit to branch 4.19
in repository https://gitbox.apache.org/repos/asf/cloudstack.git


The following commit(s) were added to refs/heads/4.19 by this push:
     new 6c06e85c803 Temporarily backup StorPool volume before expunge (#8843)
6c06e85c803 is described below

commit 6c06e85c803a72b23149b09587823d05bce6b444
Author: slavkap <[email protected]>
AuthorDate: Wed Jun 26 11:28:04 2024 +0300

    Temporarily backup StorPool volume before expunge (#8843)
    
    * Temporarily backup StorPool volume before expunge
    
    Sometimes the users delete the volumes by mistake. This enhancment
    provides a solution to backup the volume before it's deleted. The user
    will be able to see the snapshot in CloudStack UI/CLI and create only a
    volume from it.
    A task will check (by default on every 5mins) if the snapshots are
    deleted from StorPool
    
    Global settings to enable the delay delete option:
    
    `storpool.delete.after.interval` - The interval (in seconds) after the 
StorPool snapshot will be deleted
    `storpool.list.snapshots.delete.after.interval` - The interval (in seconds) 
to fetch the StorPool snapshots with deleteAfter flag
    
    Minor fix when deleting snapshots
    
    * added Apache licence
    
    * addressed comments
---
 .../com/cloud/storage/dao/SnapshotDetailsDao.java  |   3 +
 .../cloud/storage/dao/SnapshotDetailsDaoImpl.java  |  33 ++++++
 .../storage/volume/VolumeServiceImpl.java          |   4 +-
 .../storage/datastore/api/StorPoolSnapshotDef.java |  98 ++++++++++++++++++
 .../driver/StorPoolPrimaryDataStoreDriver.java     |  87 +++++++++++++---
 .../datastore/driver/StorPoolStatsCollector.java   | 111 ++++++++++++++++++++-
 .../storage/datastore/util/StorPoolUtil.java       |  41 ++++++--
 .../storage/motion/StorPoolDataMotionStrategy.java |   9 +-
 .../snapshot/StorPoolConfigurationManager.java     |  12 ++-
 .../storage/snapshot/StorPoolSnapshotStrategy.java |  26 ++++-
 10 files changed, 393 insertions(+), 31 deletions(-)

diff --git 
a/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDetailsDao.java 
b/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDetailsDao.java
index 43bb5b3d4d5..02a0355d92d 100644
--- a/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDetailsDao.java
+++ b/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDetailsDao.java
@@ -18,9 +18,12 @@
  */
 package com.cloud.storage.dao;
 
+import java.util.List;
+
 import org.apache.cloudstack.resourcedetail.ResourceDetailsDao;
 
 import com.cloud.utils.db.GenericDao;
 
 public interface SnapshotDetailsDao extends GenericDao<SnapshotDetailsVO, 
Long>, ResourceDetailsDao<SnapshotDetailsVO> {
+    public List<SnapshotDetailsVO> findDetailsByZoneAndKey(long dcId, String 
key);
 }
diff --git 
a/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDetailsDaoImpl.java 
b/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDetailsDaoImpl.java
index e4ae22cd021..584a2481726 100644
--- 
a/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDetailsDaoImpl.java
+++ 
b/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDetailsDaoImpl.java
@@ -18,11 +18,44 @@
  */
 package com.cloud.storage.dao;
 
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+
 import org.apache.cloudstack.resourcedetail.ResourceDetailsDaoBase;
 
+import com.cloud.utils.db.TransactionLegacy;
+import com.cloud.utils.exception.CloudRuntimeException;
+
 public class SnapshotDetailsDaoImpl extends 
ResourceDetailsDaoBase<SnapshotDetailsVO> implements SnapshotDetailsDao {
+    private static final String GET_SNAPSHOT_DETAILS_ON_ZONE = "SELECT s.* 
FROM snapshot_details s LEFT JOIN snapshots ss ON ss.id=s.snapshot_id WHERE 
ss.data_center_id = ? AND s.name = ?";
+
     @Override
     public void addDetail(long resourceId, String key, String value, boolean 
display) {
         super.addDetail(new SnapshotDetailsVO(resourceId, key, value, 
display));
     }
+
+    public List<SnapshotDetailsVO> findDetailsByZoneAndKey(long dcId, String 
key) {
+        StringBuilder sql = new StringBuilder(GET_SNAPSHOT_DETAILS_ON_ZONE);
+        TransactionLegacy txn = TransactionLegacy.currentTxn();
+        List<SnapshotDetailsVO> snapshotDetailsOnZone = new 
ArrayList<SnapshotDetailsVO>();
+        try (PreparedStatement pstmt = txn.prepareStatement(sql.toString());) {
+            if (pstmt != null) {
+                pstmt.setLong(1, dcId);
+                pstmt.setString(2, key);
+                try (ResultSet rs = pstmt.executeQuery();) {
+                    while (rs.next()) {
+                        snapshotDetailsOnZone.add(toEntityBean(rs, false));
+                    }
+                } catch (SQLException e) {
+                    throw new CloudRuntimeException("Could not find details by 
given zone and key due to:" + e.getMessage(), e);
+                }
+            }
+            return snapshotDetailsOnZone;
+        } catch (SQLException e) {
+            throw new CloudRuntimeException("Could not find details by given 
zone and key due to:" + e.getMessage(), e);
+        }
+    }
 }
diff --git 
a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java
 
b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java
index a47cb41a323..63726b40093 100644
--- 
a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java
+++ 
b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java
@@ -504,7 +504,9 @@ public class VolumeServiceImpl implements VolumeService {
                             _snapshotStoreDao.remove(snapStoreVo.getId());
                         }
                     } else {
-                        _snapshotStoreDao.remove(snapStoreVo.getId());
+                        if 
(!StoragePoolType.StorPool.equals(storagePoolVO.getPoolType())) {
+                            _snapshotStoreDao.remove(snapStoreVo.getId());
+                        }
                     }
                 }
                 snapshotApiService.markVolumeSnapshotsAsDestroyed(vo);
diff --git 
a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/api/StorPoolSnapshotDef.java
 
b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/api/StorPoolSnapshotDef.java
new file mode 100644
index 00000000000..26004205709
--- /dev/null
+++ 
b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/api/StorPoolSnapshotDef.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.cloudstack.storage.datastore.api;
+
+import java.io.Serializable;
+import java.util.Map;
+
+public class StorPoolSnapshotDef implements Serializable {
+    private static final long serialVersionUID = 1L;
+    private String name;
+    private Integer deleteAfter;
+    private Map<String, String> tags;
+    private Boolean bind;
+    private Integer iops;
+    private String rename;
+    private transient String volumeName;
+
+    public StorPoolSnapshotDef(String volumeName, Integer deleteAfter, 
Map<String, String> tags) {
+        super();
+        this.volumeName = volumeName;
+        this.deleteAfter = deleteAfter;
+        this.tags = tags;
+    }
+
+    public StorPoolSnapshotDef(String name, Integer deleteAfter, Map<String, 
String> tags, Boolean bind, Integer iops,
+            String rename) {
+        super();
+        this.name = name;
+        this.deleteAfter = deleteAfter;
+        this.tags = tags;
+        this.bind = bind;
+        this.iops = iops;
+        this.rename = rename;
+    }
+
+    public String getName() {
+        return name;
+    }
+    public void setName(String name) {
+        this.name = name;
+    }
+    public Integer getDeleteAfter() {
+        return deleteAfter;
+    }
+    public void setDeleteAfter(Integer deleteAfter) {
+        this.deleteAfter = deleteAfter;
+    }
+    public Map<String, String> getTags() {
+        return tags;
+    }
+    public void setTags(Map<String, String> tags) {
+        this.tags = tags;
+    }
+    public Boolean getBind() {
+        return bind;
+    }
+    public void setBind(Boolean bind) {
+        this.bind = bind;
+    }
+    public Integer getIops() {
+        return iops;
+    }
+    public void setIops(Integer iops) {
+        this.iops = iops;
+    }
+    public String getRename() {
+        return rename;
+    }
+    public void setRename(String rename) {
+        this.rename = rename;
+    }
+
+    public String getVolumeName() {
+        return volumeName;
+    }
+
+    public void setVolumeName(String volumeName) {
+        this.volumeName = volumeName;
+    }
+
+}
diff --git 
a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/driver/StorPoolPrimaryDataStoreDriver.java
 
b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/driver/StorPoolPrimaryDataStoreDriver.java
index 08a3252d869..f5fdc96cb48 100644
--- 
a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/driver/StorPoolPrimaryDataStoreDriver.java
+++ 
b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/driver/StorPoolPrimaryDataStoreDriver.java
@@ -18,6 +18,7 @@
  */
 package org.apache.cloudstack.storage.datastore.driver;
 
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -31,6 +32,7 @@ import 
org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
 import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint;
 import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector;
+import 
org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine;
 import 
org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreDriver;
 import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
 import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo;
@@ -42,6 +44,7 @@ import org.apache.cloudstack.storage.command.CommandResult;
 import org.apache.cloudstack.storage.command.CopyCmdAnswer;
 import org.apache.cloudstack.storage.command.CreateObjectAnswer;
 import org.apache.cloudstack.storage.command.StorageSubSystemCommand;
+import org.apache.cloudstack.storage.datastore.api.StorPoolSnapshotDef;
 import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO;
@@ -87,6 +90,8 @@ import com.cloud.server.ResourceTag;
 import com.cloud.server.ResourceTag.ResourceObjectType;
 import com.cloud.storage.DataStoreRole;
 import com.cloud.storage.ResizeVolumePayload;
+import com.cloud.storage.Snapshot;
+import com.cloud.storage.SnapshotVO;
 import com.cloud.storage.Storage.StoragePoolType;
 import com.cloud.storage.StorageManager;
 import com.cloud.storage.StoragePool;
@@ -95,6 +100,7 @@ import com.cloud.storage.VMTemplateStoragePoolVO;
 import com.cloud.storage.Volume;
 import com.cloud.storage.VolumeDetailVO;
 import com.cloud.storage.VolumeVO;
+import com.cloud.storage.dao.SnapshotDao;
 import com.cloud.storage.dao.SnapshotDetailsDao;
 import com.cloud.storage.dao.SnapshotDetailsVO;
 import com.cloud.storage.dao.StoragePoolHostDao;
@@ -132,9 +138,11 @@ public class StorPoolPrimaryDataStoreDriver implements 
PrimaryDataStoreDriver {
     @Inject
     private HostDao hostDao;
     @Inject
-    private ResourceTagDao _resourceTagDao;
+    private ResourceTagDao resourceTagDao;
     @Inject
-    private SnapshotDetailsDao _snapshotDetailsDao;
+    private SnapshotDetailsDao snapshotDetailsDao;
+    @Inject
+    private SnapshotDao snapshotDao;
     @Inject
     private SnapshotDataStoreDao snapshotDataStoreDao;
     @Inject
@@ -401,7 +409,7 @@ public class StorPoolPrimaryDataStoreDriver implements 
PrimaryDataStoreDriver {
             }
             try {
                 SpConnectionDesc conn = 
StorPoolUtil.getSpConnection(dataStore.getUuid(), dataStore.getId(), 
storagePoolDetailsDao, primaryStoreDao);
-
+                tryToSnapshotVolumeBeforeDelete(vinfo, dataStore, name, conn);
                 SpApiResponse resp = StorPoolUtil.volumeDelete(name, conn);
                 if (resp.getError() == null) {
                     updateStoragePool(dataStore.getId(), - vinfo.getSize());
@@ -431,6 +439,54 @@ public class StorPoolPrimaryDataStoreDriver implements 
PrimaryDataStoreDriver {
         callback.complete(res);
     }
 
+    private void tryToSnapshotVolumeBeforeDelete(VolumeInfo vinfo, DataStore 
dataStore, String name, SpConnectionDesc conn) {
+        Integer deleteAfter = 
StorPoolConfigurationManager.DeleteAfterInterval.valueIn(dataStore.getId());
+        if (deleteAfter != null && deleteAfter > 0 && vinfo.getPassphraseId() 
== null) {
+            createTemporarySnapshot(vinfo, name, deleteAfter, conn);
+        } else {
+            StorPoolUtil.spLog("The volume [%s] is not marked to be snapshot. 
Check the global setting `storpool.delete.after.interval` or the volume is 
encrypted [%s]", name, deleteAfter, vinfo.getPassphraseId() != null);
+        }
+    }
+
+    private void createTemporarySnapshot(VolumeInfo vinfo, String name, 
Integer deleteAfter, SpConnectionDesc conn) {
+        Map<String, String> tags = new HashMap<>();
+        tags.put("cs", StorPoolUtil.DELAY_DELETE);
+        StorPoolSnapshotDef snapshot = new StorPoolSnapshotDef(name, 
deleteAfter, tags);
+        StorPoolUtil.spLog("Creating backup snapshot before delete the volume 
[%s]", vinfo.getName());
+        SpApiResponse snapshotResponse = StorPoolUtil.volumeSnapshot(snapshot, 
conn);
+        if (snapshotResponse.getError() == null) {
+            String snapshotName = 
StorPoolUtil.getSnapshotNameFromResponse(snapshotResponse, false, 
StorPoolUtil.GLOBAL_ID);
+            String snapshotPath = StorPoolUtil.devPath(snapshotName);
+            SnapshotVO snapshotVo = createSnapshotVo(vinfo, snapshotName);
+            createSnapshotOnPrimaryVo(vinfo, snapshotVo, snapshotPath);
+            SnapshotDetailsVO snapshotDetails = new 
SnapshotDetailsVO(snapshotVo.getId(), StorPoolUtil.SP_DELAY_DELETE, "~" + 
snapshotName, true);
+            snapshotDetailsDao.persist(snapshotDetails);
+        }
+    }
+
+    private void createSnapshotOnPrimaryVo(VolumeInfo vinfo, SnapshotVO 
snapshotVo, String snapshotPath) {
+        SnapshotDataStoreVO snapshotOnPrimaryVo = new SnapshotDataStoreVO();
+        snapshotOnPrimaryVo.setSnapshotId(snapshotVo.getId());
+        snapshotOnPrimaryVo.setDataStoreId(vinfo.getDataCenterId());
+        snapshotOnPrimaryVo.setRole(vinfo.getDataStore().getRole());
+        snapshotOnPrimaryVo.setVolumeId(vinfo.getId());
+        snapshotOnPrimaryVo.setSize(vinfo.getSize());
+        snapshotOnPrimaryVo.setPhysicalSize(vinfo.getSize());
+        snapshotOnPrimaryVo.setInstallPath(snapshotPath);
+        
snapshotOnPrimaryVo.setState(ObjectInDataStoreStateMachine.State.Ready);
+        snapshotDataStoreDao.persist(snapshotOnPrimaryVo);
+    }
+
+    private SnapshotVO createSnapshotVo(VolumeInfo vinfo, String snapshotName) 
{
+        SnapshotVO snapshotVo = new SnapshotVO(vinfo.getDataCenterId(), 
vinfo.getAccountId(), vinfo.getDomainId(), vinfo.getId(),
+                vinfo.getDiskOfferingId(), snapshotName,
+                (short)Snapshot.Type.RECURRING.ordinal(), 
Snapshot.Type.RECURRING.name(),
+                vinfo.getSize(), vinfo.getMinIops(), vinfo.getMaxIops(), 
vinfo.getHypervisorType(), Snapshot.LocationType.PRIMARY);
+        snapshotVo.setState(com.cloud.storage.Snapshot.State.BackedUp);
+        snapshotVo = snapshotDao.persist(snapshotVo);
+        return snapshotVo;
+    }
+
     private void logDataObject(final String pref, DataObject data) {
         final DataStore dstore = data.getDataStore();
         String name = null;
@@ -473,7 +529,7 @@ public class StorPoolPrimaryDataStoreDriver implements 
PrimaryDataStoreDriver {
         try {
             if (srcType == DataObjectType.SNAPSHOT && dstType == 
DataObjectType.VOLUME) {
                 SnapshotInfo sinfo = (SnapshotInfo)srcData;
-                final String snapshotName = 
StorPoolHelper.getSnapshotName(srcData.getId(), srcData.getUuid(), 
snapshotDataStoreDao, _snapshotDetailsDao);
+                final String snapshotName = 
StorPoolHelper.getSnapshotName(srcData.getId(), srcData.getUuid(), 
snapshotDataStoreDao, snapshotDetailsDao);
 
                 VolumeInfo vinfo = (VolumeInfo)dstData;
                 final String volumeName = vinfo.getUuid();
@@ -491,9 +547,12 @@ public class StorPoolPrimaryDataStoreDriver implements 
PrimaryDataStoreDriver {
                     StorPoolUtil.spLog("Created volume=%s with uuid=%s from 
snapshot=%s with uuid=%s", StorPoolUtil.getNameFromResponse(resp, false), 
to.getUuid(), snapshotName, sinfo.getUuid());
                 } else if 
(resp.getError().getName().equals("objectDoesNotExist")) {
                     //check if snapshot is on secondary storage
-                    StorPoolUtil.spLog("Snapshot %s does not exists on 
StorPool, will try to create a volume from a snopshot on secondary storage", 
snapshotName);
+                    StorPoolUtil.spLog("Snapshot %s does not exists on 
StorPool, will try to create a volume from a snapshot on secondary storage", 
snapshotName);
                     SnapshotDataStoreVO snap = 
getSnapshotImageStoreRef(sinfo.getId(), vinfo.getDataCenterId());
-                    if (snap != null && 
StorPoolStorageAdaptor.getVolumeNameFromPath(snap.getInstallPath(), false) == 
null) {
+                    SnapshotDetailsVO snapshotDetail = 
snapshotDetailsDao.findDetail(sinfo.getId(), StorPoolUtil.SP_DELAY_DELETE);
+                    if (snapshotDetail != null) {
+                        err = String.format("Could not create volume from 
snapshot due to: %s", resp.getError());
+                    } else if (snap != null && 
StorPoolStorageAdaptor.getVolumeNameFromPath(snap.getInstallPath(), false) == 
null) {
                         resp = StorPoolUtil.volumeCreate(srcData.getUuid(), 
null, size, null, "no", "snapshot", sinfo.getBaseVolume().getMaxIops(), conn);
                         if (resp.getError() == null) {
                             VolumeObjectTO dstTO = (VolumeObjectTO) 
dstData.getTO();
@@ -514,11 +573,11 @@ public class StorPoolPrimaryDataStoreDriver implements 
PrimaryDataStoreDriver {
                                     err = String.format("Could not freeze 
Storpool volume %s. Error: %s", srcData.getUuid(), resp2.getError());
                                 } else {
                                     String name = 
StorPoolUtil.getNameFromResponse(resp, false);
-                                    SnapshotDetailsVO snapshotDetails = 
_snapshotDetailsDao.findDetail(sinfo.getId(), sinfo.getUuid());
+                                    SnapshotDetailsVO snapshotDetails = 
snapshotDetailsDao.findDetail(sinfo.getId(), sinfo.getUuid());
                                     if (snapshotDetails != null) {
                                         
StorPoolHelper.updateSnapshotDetailsValue(snapshotDetails.getId(), 
StorPoolUtil.devPath(name), "snapshot");
                                     }else {
-                                        
StorPoolHelper.addSnapshotDetails(sinfo.getId(), sinfo.getUuid(), 
StorPoolUtil.devPath(name), _snapshotDetailsDao);
+                                        
StorPoolHelper.addSnapshotDetails(sinfo.getId(), sinfo.getUuid(), 
StorPoolUtil.devPath(name), snapshotDetailsDao);
                                     }
                                     resp = 
StorPoolUtil.volumeCreate(volumeName, StorPoolUtil.getNameFromResponse(resp, 
true), size, null, null, "volume", sinfo.getBaseVolume().getMaxIops(), conn);
                                     if (resp.getError() == null) {
@@ -548,8 +607,10 @@ public class StorPoolPrimaryDataStoreDriver implements 
PrimaryDataStoreDriver {
                     err = String.format("Could not create Storpool volume %s 
from snapshot %s. Error: %s", volumeName, snapshotName, resp.getError());
                 }
             } else if (srcType == DataObjectType.SNAPSHOT && dstType == 
DataObjectType.SNAPSHOT) {
+                SnapshotInfo sinfo = (SnapshotInfo)srcData;
+                SnapshotDetailsVO snapshotDetail = 
snapshotDetailsDao.findDetail(sinfo.getId(), StorPoolUtil.SP_DELAY_DELETE);
                 // bypass secondary storage
-                if 
(StorPoolConfigurationManager.BypassSecondaryStorage.value()) {
+                if 
(StorPoolConfigurationManager.BypassSecondaryStorage.value() || snapshotDetail 
!= null) {
                     SnapshotObjectTO snapshot = (SnapshotObjectTO) 
srcData.getTO();
                     answer = new CopyCmdAnswer(snapshot);
                 } else {
@@ -986,9 +1047,9 @@ public class StorPoolPrimaryDataStoreDriver implements 
PrimaryDataStoreDriver {
                 SnapshotObjectTO snapTo = (SnapshotObjectTO)snapshot.getTO();
                 snapTo.setPath(StorPoolUtil.devPath(name.split("~")[1]));
                 answer = new CreateObjectAnswer(snapTo);
-                StorPoolHelper.addSnapshotDetails(snapshot.getId(), 
snapshot.getUuid(), snapTo.getPath(), _snapshotDetailsDao);
+                StorPoolHelper.addSnapshotDetails(snapshot.getId(), 
snapshot.getUuid(), snapTo.getPath(), snapshotDetailsDao);
                 //add primary storage of snapshot
-                StorPoolHelper.addSnapshotDetails(snapshot.getId(), 
StorPoolUtil.SP_STORAGE_POOL_ID, 
String.valueOf(snapshot.getDataStore().getId()), _snapshotDetailsDao);
+                StorPoolHelper.addSnapshotDetails(snapshot.getId(), 
StorPoolUtil.SP_STORAGE_POOL_ID, 
String.valueOf(snapshot.getDataStore().getId()), snapshotDetailsDao);
                 
StorPoolUtil.spLog("StorpoolPrimaryDataStoreDriverImpl.takeSnapshot: snapshot: 
name=%s, uuid=%s, volume: name=%s, uuid=%s", name, snapshot.getUuid(), 
volumeName, vinfo.getUuid());
             }
         } catch (Exception e) {
@@ -1003,7 +1064,7 @@ public class StorPoolPrimaryDataStoreDriver implements 
PrimaryDataStoreDriver {
     @Override
     public void revertSnapshot(final SnapshotInfo snapshot, final SnapshotInfo 
snapshotOnPrimaryStore, final AsyncCompletionCallback<CommandResult> callback) {
         final VolumeInfo vinfo = snapshot.getBaseVolume();
-        final String snapshotName = 
StorPoolHelper.getSnapshotName(snapshot.getId(), snapshot.getUuid(), 
snapshotDataStoreDao, _snapshotDetailsDao);
+        final String snapshotName = 
StorPoolHelper.getSnapshotName(snapshot.getId(), snapshot.getUuid(), 
snapshotDataStoreDao, snapshotDetailsDao);
         final String volumeName = 
StorPoolStorageAdaptor.getVolumeNameFromPath(vinfo.getPath(), true);
         StorPoolUtil.spLog("StorpoolPrimaryDataStoreDriverImpl.revertSnapshot: 
snapshot: name=%s, uuid=%s, volume: name=%s, uuid=%s", snapshotName, 
snapshot.getUuid(), volumeName, vinfo.getUuid());
         String err = null;
@@ -1058,7 +1119,7 @@ public class StorPoolPrimaryDataStoreDriver implements 
PrimaryDataStoreDriver {
     }
 
     private String getVcPolicyTag(Long vmId) {
-        ResourceTag resourceTag = vmId != null ? 
_resourceTagDao.findByKey(vmId, ResourceObjectType.UserVm, 
StorPoolUtil.SP_VC_POLICY) : null;
+        ResourceTag resourceTag = vmId != null ? 
resourceTagDao.findByKey(vmId, ResourceObjectType.UserVm, 
StorPoolUtil.SP_VC_POLICY) : null;
         return resourceTag != null ? resourceTag.getValue() : "";
     }
 
diff --git 
a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/driver/StorPoolStatsCollector.java
 
b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/driver/StorPoolStatsCollector.java
index 359b11d491e..44acd1eab75 100644
--- 
a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/driver/StorPoolStatsCollector.java
+++ 
b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/driver/StorPoolStatsCollector.java
@@ -28,19 +28,29 @@ import java.util.concurrent.TimeUnit;
 
 import javax.inject.Inject;
 
+import 
org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine.State;
 import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
 import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
+import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao;
+import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO;
 import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao;
 import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
 import org.apache.cloudstack.storage.datastore.util.StorPoolUtil;
 import org.apache.cloudstack.storage.snapshot.StorPoolConfigurationManager;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.log4j.Logger;
+import org.apache.commons.lang3.StringUtils;
 
+import com.cloud.storage.DataStoreRole;
+import com.cloud.storage.SnapshotVO;
+import com.cloud.storage.dao.SnapshotDao;
+import com.cloud.storage.dao.SnapshotDetailsDao;
+import com.cloud.storage.dao.SnapshotDetailsVO;
 import com.cloud.utils.NumbersUtil;
 import com.cloud.utils.Pair;
 import com.cloud.utils.component.ManagerBase;
 import com.cloud.utils.concurrency.NamedThreadFactory;
+import com.cloud.utils.exception.CloudRuntimeException;
 import com.google.gson.JsonArray;
 import com.google.gson.JsonElement;
 import com.google.gson.JsonObject;
@@ -55,6 +65,12 @@ public class StorPoolStatsCollector extends ManagerBase {
     private StoragePoolDetailsDao storagePoolDetailsDao;
     @Inject
     private ConfigurationDao configurationDao;
+    @Inject
+    private SnapshotDao snapshotDao;
+    @Inject
+    private SnapshotDataStoreDao snapshotDataStoreDao;
+    @Inject
+    private SnapshotDetailsDao snapshotDetailsDao;
 
     private ScheduledExecutorService executor;
 
@@ -70,7 +86,7 @@ public class StorPoolStatsCollector extends ManagerBase {
     public boolean start() {
         List<StoragePoolVO> spPools = 
storagePoolDao.findPoolsByProvider(StorPoolUtil.SP_PROVIDER_NAME);
         if (CollectionUtils.isNotEmpty(spPools)) {
-            executor = Executors.newScheduledThreadPool(2,new 
NamedThreadFactory("StorPoolStatsCollector"));
+            executor = Executors.newScheduledThreadPool(3, new 
NamedThreadFactory("StorPoolStatsCollector"));
             long storageStatsInterval = 
NumbersUtil.parseLong(configurationDao.getValue("storage.stats.interval"), 
60000L);
             long volumeStatsInterval = 
NumbersUtil.parseLong(configurationDao.getValue("volume.stats.interval"), 
60000L);
 
@@ -80,6 +96,13 @@ public class StorPoolStatsCollector extends ManagerBase {
             if (StorPoolConfigurationManager.StorageStatsInterval.value() > 0 
&& storageStatsInterval > 0) {
                 executor.scheduleAtFixedRate(new 
StorPoolStorageStatsMonitorTask(), 120, 
StorPoolConfigurationManager.StorageStatsInterval.value(), TimeUnit.SECONDS);
             }
+            for (StoragePoolVO pool: spPools) {
+                Integer deleteAfter = 
StorPoolConfigurationManager.DeleteAfterInterval.valueIn(pool.getId());
+                if (deleteAfter != null && deleteAfter > 0) {
+                    executor.scheduleAtFixedRate(new 
StorPoolSnapshotsWithDelayDelete(), 120, 
StorPoolConfigurationManager.ListSnapshotsWithDeleteAfterInterval.value(), 
TimeUnit.SECONDS);
+                    break;
+                }
+            }
         }
 
         return true;
@@ -185,4 +208,90 @@ public class StorPoolStatsCollector extends ManagerBase {
             }
         }
     }
+
+    class StorPoolSnapshotsWithDelayDelete implements Runnable {
+
+        @Override
+        public void run() {
+            List<StoragePoolVO> spPools = 
storagePoolDao.findPoolsByProvider(StorPoolUtil.SP_PROVIDER_NAME);
+            if (CollectionUtils.isNotEmpty(spPools)) {
+                Map<Long, StoragePoolVO> onePoolForZone = new HashMap<>();
+                for (StoragePoolVO storagePoolVO : spPools) {
+                    onePoolForZone.put(storagePoolVO.getDataCenterId(), 
storagePoolVO);
+                }
+                for (StoragePoolVO storagePool : onePoolForZone.values()) {
+                    List<SnapshotDetailsVO> snapshotsDetails = 
snapshotDetailsDao.findDetailsByZoneAndKey(storagePool.getDataCenterId(), 
StorPoolUtil.SP_DELAY_DELETE);
+                    if (CollectionUtils.isEmpty(snapshotsDetails)) {
+                        return;
+                    }
+                    Map<String, String> snapshotsWithDelayDelete = new 
HashMap<>();
+
+                    try {
+                        log.debug(String.format("Collecting snapshots marked 
to be deleted for zone [%s]", storagePool.getDataCenterId()));
+                        JsonArray arr = 
StorPoolUtil.snapshotsListAllClusters(StorPoolUtil.getSpConnection(storagePool.getUuid(),
+                                storagePool.getId(), storagePoolDetailsDao, 
storagePoolDao));
+                         
snapshotsWithDelayDelete.putAll(getSnapshotsMarkedForDeletion(arr));
+                         log.debug(String.format("Found snapshot details [%s] 
and snapshots on StorPool with delay delete flag [%s]", snapshotsDetails, 
snapshotsWithDelayDelete));
+                         syncSnapshots(snapshotsDetails, 
snapshotsWithDelayDelete);
+                    } catch (Exception e) {
+                        log.debug("Could not fetch the snapshots with delay 
delete flag " + e.getMessage());
+                    }
+                }
+            }
+        }
+
+        private void syncSnapshots(List<SnapshotDetailsVO> snapshotsDetails,
+                Map<String, String> snapshotsWithDelayDelete) {
+            for (SnapshotDetailsVO snapshotDetailsVO : snapshotsDetails) {
+                 if 
(!snapshotsWithDelayDelete.containsKey(snapshotDetailsVO.getValue())) {
+                     StorPoolUtil.spLog("The snapshot [%s] with delayDelete 
flag is no longer on StorPool. Removing it from CloudStack", 
snapshotDetailsVO.getValue());
+                     SnapshotDataStoreVO ss = snapshotDataStoreDao
+                             
.findBySourceSnapshot(snapshotDetailsVO.getResourceId(), DataStoreRole.Primary);
+                     if (ss != null) {
+                         ss.setState(State.Destroyed);
+                         snapshotDataStoreDao.update(ss.getId(), ss);
+                     }
+                     SnapshotVO snap = 
snapshotDao.findById(snapshotDetailsVO.getResourceId());
+                     if (snap != null) {
+                         
snap.setState(com.cloud.storage.Snapshot.State.Destroyed);
+                         snapshotDao.update(snap.getId(), snap);
+                     }
+                     snapshotDetailsDao.remove(snapshotDetailsVO.getId());
+                 }
+             }
+        }
+
+        private  Map<String, String> getSnapshotsMarkedForDeletion(JsonArray 
arr) {
+            for (JsonElement jsonElement : arr) {
+                JsonObject error = 
jsonElement.getAsJsonObject().getAsJsonObject("error");
+                if (error != null) {
+                    throw new CloudRuntimeException(String.format("Could not 
collect the snapshots marked for deletion from all storage nodes due to: [%s]", 
error));
+                }
+            }
+            Map<String, String> snapshotsWithDelayDelete = new HashMap<>();
+            for (JsonElement jsonElement : arr) {
+                JsonObject response = 
jsonElement.getAsJsonObject().getAsJsonObject("response");
+                if (response == null) {
+                    return snapshotsWithDelayDelete;
+                }
+                collectSnapshots(snapshotsWithDelayDelete, response);
+            }
+            log.debug("Found snapshots on StorPool" + 
snapshotsWithDelayDelete);
+            return snapshotsWithDelayDelete;
+        }
+
+        private void collectSnapshots(Map<String, String> 
snapshotsWithDelayDelete, JsonObject response) {
+            JsonArray snapshots = 
response.getAsJsonObject().getAsJsonArray("data");
+            for (JsonElement snapshot : snapshots) {
+                String name = 
snapshot.getAsJsonObject().get("name").getAsString();
+                JsonObject tags = 
snapshot.getAsJsonObject().get("tags").getAsJsonObject();
+                if (!StringUtils.startsWith(name, "*") && 
StringUtils.containsNone(name, "@") && tags != null && 
!tags.entrySet().isEmpty()) {
+                    String tag = tags.getAsJsonPrimitive("cs").getAsString();
+                    if (tag != null && tag.equals(StorPoolUtil.DELAY_DELETE)) {
+                        snapshotsWithDelayDelete.put(name, tag);
+                    }
+                }
+            }
+        }
+    }
 }
diff --git 
a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/util/StorPoolUtil.java
 
b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/util/StorPoolUtil.java
index 675dffbda5b..58e87b2fba6 100644
--- 
a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/util/StorPoolUtil.java
+++ 
b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/util/StorPoolUtil.java
@@ -28,6 +28,8 @@ import com.google.gson.JsonElement;
 import com.google.gson.JsonObject;
 import com.google.gson.JsonParser;
 import com.google.gson.JsonPrimitive;
+
+import org.apache.cloudstack.storage.datastore.api.StorPoolSnapshotDef;
 import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailVO;
 import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao;
@@ -124,6 +126,15 @@ public class StorPoolUtil {
 
     public static final String SP_VOLUME_ON_CLUSTER = "SP_VOLUME_ON_CLUSTER";
 
+    private static final String DATA = "data";
+
+    private static final String CLUSTERS = "clusters";
+
+    public static final String SP_DELAY_DELETE = "SP_DELAY_DELETE";
+
+    public static final String DELAY_DELETE = "delayDelete";
+
+
     public static enum StorpoolRights {
         RO("ro"), RW("rw"), DETACH("detach");
 
@@ -416,27 +427,31 @@ public class StorPoolUtil {
     public static JsonArray snapshotsList(SpConnectionDesc conn) {
         SpApiResponse resp = GET("MultiCluster/SnapshotsList", conn);
         JsonObject obj = resp.fullJson.getAsJsonObject();
-        JsonArray data = obj.getAsJsonArray("data");
-        return data;
+        return obj.getAsJsonArray(DATA);
+    }
+
+    public static JsonArray snapshotsListAllClusters(SpConnectionDesc conn) {
+        SpApiResponse resp = GET("MultiCluster/AllClusters/SnapshotsList", 
conn);
+        JsonObject obj = resp.fullJson.getAsJsonObject();
+        return obj.getAsJsonObject(DATA).getAsJsonArray(CLUSTERS);
     }
 
     public static JsonArray volumesList(SpConnectionDesc conn) {
         SpApiResponse resp = GET("MultiCluster/VolumesList", conn);
         JsonObject obj = resp.fullJson.getAsJsonObject();
-        JsonArray data = obj.getAsJsonArray("data");
-        return data;
+        return obj.getAsJsonArray(DATA);
     }
 
     public static JsonArray volumesSpace(SpConnectionDesc conn) {
         SpApiResponse resp = GET("MultiCluster/AllClusters/VolumesSpace", 
conn);
         JsonObject obj = resp.fullJson.getAsJsonObject();
-        return obj.getAsJsonObject("data").getAsJsonArray("clusters");
+        return obj.getAsJsonObject(DATA).getAsJsonArray(CLUSTERS);
     }
 
     public static JsonArray templatesStats(SpConnectionDesc conn) {
         SpApiResponse resp = 
GET("MultiCluster/AllClusters/VolumeTemplatesStatus", conn);
         JsonObject obj = resp.fullJson.getAsJsonObject();
-        return obj.getAsJsonObject("data").getAsJsonArray("clusters");
+        return obj.getAsJsonObject(DATA).getAsJsonArray(CLUSTERS);
     }
 
     private static boolean objectExists(SpApiError err) {
@@ -453,7 +468,7 @@ public class StorPoolUtil {
         if (resp.getError() != null && !objectExists(resp.getError())) {
             return null;
         }
-        JsonObject data = obj.getAsJsonArray("data").get(0).getAsJsonObject();
+        JsonObject data = obj.getAsJsonArray(DATA).get(0).getAsJsonObject();
         return data.getAsJsonPrimitive("size").getAsLong();
     }
 
@@ -461,7 +476,7 @@ public class StorPoolUtil {
         SpApiResponse resp = GET("MultiCluster/Snapshot/" + name, conn);
         JsonObject obj = resp.fullJson.getAsJsonObject();
 
-        JsonObject data = obj.getAsJsonArray("data").get(0).getAsJsonObject();
+        JsonObject data = obj.getAsJsonArray(DATA).get(0).getAsJsonObject();
         JsonPrimitive clusterId = data.getAsJsonPrimitive("clusterId");
         return clusterId != null ? clusterId.getAsString() : null;
     }
@@ -470,7 +485,7 @@ public class StorPoolUtil {
         SpApiResponse resp = GET("MultiCluster/Volume/" + name, conn);
         JsonObject obj = resp.fullJson.getAsJsonObject();
 
-        JsonObject data = obj.getAsJsonArray("data").get(0).getAsJsonObject();
+        JsonObject data = obj.getAsJsonArray(DATA).get(0).getAsJsonObject();
         JsonPrimitive clusterId = data.getAsJsonPrimitive("clusterId");
         return clusterId != null ? clusterId.getAsString() : null;
     }
@@ -579,6 +594,10 @@ public class StorPoolUtil {
         return POST("MultiCluster/VolumeSnapshot/" + volumeName, json, conn);
     }
 
+    public static SpApiResponse volumeSnapshot(StorPoolSnapshotDef snapshot, 
SpConnectionDesc conn) {
+        return POST("MultiCluster/VolumeSnapshot/" + snapshot.getVolumeName(), 
snapshot, conn);
+    }
+
     public static SpApiResponse volumesGroupSnapshot(final 
List<VolumeObjectTO> volumeTOs, final String vmUuid,
             final String snapshotName, String csTag, SpConnectionDesc conn) {
         Map<String, Object> json = new LinkedHashMap<>();
@@ -638,7 +657,7 @@ public class StorPoolUtil {
 
     public static String getSnapshotNameFromResponse(SpApiResponse resp, 
boolean tildeNeeded, String globalIdOrRemote) {
         JsonObject obj = resp.fullJson.getAsJsonObject();
-        JsonPrimitive data = 
obj.getAsJsonObject("data").getAsJsonPrimitive(globalIdOrRemote);
+        JsonPrimitive data = 
obj.getAsJsonObject(DATA).getAsJsonPrimitive(globalIdOrRemote);
         String name = data != null ? data.getAsString() : null;
         name = name != null ? !tildeNeeded ? name : "~" + name : name;
         return name;
@@ -646,7 +665,7 @@ public class StorPoolUtil {
 
     public static String getNameFromResponse(SpApiResponse resp, boolean 
tildeNeeded) {
         JsonObject obj = resp.fullJson.getAsJsonObject();
-        JsonPrimitive data = 
obj.getAsJsonObject("data").getAsJsonPrimitive("name");
+        JsonPrimitive data = 
obj.getAsJsonObject(DATA).getAsJsonPrimitive("name");
         String name = data != null ? data.getAsString() : null;
         name = name != null ? name.startsWith("~") && !tildeNeeded ? 
name.split("~")[1] : name : name;
         return name;
diff --git 
a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/motion/StorPoolDataMotionStrategy.java
 
b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/motion/StorPoolDataMotionStrategy.java
index a735b0fe918..6cfa5ec14a2 100644
--- 
a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/motion/StorPoolDataMotionStrategy.java
+++ 
b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/motion/StorPoolDataMotionStrategy.java
@@ -148,18 +148,21 @@ public class StorPoolDataMotionStrategy implements 
DataMotionStrategy {
     public StrategyPriority canHandle(DataObject srcData, DataObject destData) 
{
         DataObjectType srcType = srcData.getType();
         DataObjectType dstType = destData.getType();
-        if (srcType == DataObjectType.SNAPSHOT && dstType == 
DataObjectType.TEMPLATE
-                && 
StorPoolConfigurationManager.BypassSecondaryStorage.value()) {
+        if (srcType == DataObjectType.SNAPSHOT && dstType == 
DataObjectType.TEMPLATE) {
             SnapshotInfo sinfo = (SnapshotInfo) srcData;
             VolumeInfo volume = sinfo.getBaseVolume();
             StoragePoolVO storagePool = 
_storagePool.findById(volume.getPoolId());
             if 
(!storagePool.getStorageProviderName().equals(StorPoolUtil.SP_PROVIDER_NAME)) {
                 return StrategyPriority.CANT_HANDLE;
             }
+            SnapshotDetailsVO snapshotDetail = 
_snapshotDetailsDao.findDetail(sinfo.getId(), StorPoolUtil.SP_DELAY_DELETE);
+            if (snapshotDetail != null) {
+                throw new CloudRuntimeException("Cannot create a template from 
the last snapshot of deleted volume. You can only restore the volume.");
+            }
             String snapshotName = 
StorPoolHelper.getSnapshotName(sinfo.getId(), sinfo.getUuid(), 
_snapshotStoreDao,
                     _snapshotDetailsDao);
             StorPoolUtil.spLog("StorPoolDataMotionStrategy.canHandle snapshot 
name=%s", snapshotName);
-            if (snapshotName != null) {
+            if (snapshotName != null && 
StorPoolConfigurationManager.BypassSecondaryStorage.value()) {
                 return StrategyPriority.HIGHEST;
             }
         }
diff --git 
a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/snapshot/StorPoolConfigurationManager.java
 
b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/snapshot/StorPoolConfigurationManager.java
index dcb2b226467..e4e930c8dee 100644
--- 
a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/snapshot/StorPoolConfigurationManager.java
+++ 
b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/snapshot/StorPoolConfigurationManager.java
@@ -44,6 +44,16 @@ public class StorPoolConfigurationManager implements 
Configurable {
             "The interval in seconds to get StorPool template statistics",
             false);
 
+    public static final ConfigKey<Integer> DeleteAfterInterval = new 
ConfigKey<>("Advanced", Integer.class,
+            "storpool.delete.after.interval", "0",
+            "The interval (in seconds) after the StorPool snapshot will be 
deleted",
+            false, ConfigKey.Scope.StoragePool);
+
+    public static final ConfigKey<Integer> 
ListSnapshotsWithDeleteAfterInterval = new ConfigKey<>("Advanced", 
Integer.class,
+            "storpool.list.snapshots.delete.after.interval", "360",
+            "The interval (in seconds) to fetch the StorPool snapshots with 
deleteAfter flag",
+            false);
+
     @Override
     public String getConfigComponentName() {
         return StorPoolConfigurationManager.class.getSimpleName();
@@ -51,6 +61,6 @@ public class StorPoolConfigurationManager implements 
Configurable {
 
     @Override
     public ConfigKey<?>[] getConfigKeys() {
-        return new ConfigKey<?>[] { BypassSecondaryStorage, StorPoolClusterId, 
AlternativeEndPointEnabled, AlternativeEndpoint, VolumesStatsInterval, 
StorageStatsInterval };
+        return new ConfigKey<?>[] { BypassSecondaryStorage, StorPoolClusterId, 
AlternativeEndPointEnabled, AlternativeEndpoint, VolumesStatsInterval, 
StorageStatsInterval, DeleteAfterInterval, ListSnapshotsWithDeleteAfterInterval 
};
     }
 }
diff --git 
a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/snapshot/StorPoolSnapshotStrategy.java
 
b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/snapshot/StorPoolSnapshotStrategy.java
index 55d691f33e0..23361bd1e5e 100644
--- 
a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/snapshot/StorPoolSnapshotStrategy.java
+++ 
b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/snapshot/StorPoolSnapshotStrategy.java
@@ -116,6 +116,8 @@ public class StorPoolSnapshotStrategy implements 
SnapshotStrategy {
                 if (resp.getError() != null) {
                     final String err = String.format("Failed to clean-up 
Storpool snapshot %s. Error: %s", name, resp.getError());
                     StorPoolUtil.spLog(err);
+                    markSnapshotAsDestroyedIfAlreadyRemoved(snapshotId, resp);
+                    throw new CloudRuntimeException(err);
                 } else {
                     res = deleteSnapshotFromDbIfNeeded(snapshotVO, zoneId);
                     
StorPoolUtil.spLog("StorpoolSnapshotStrategy.deleteSnapshot: executed 
successfully=%s, snapshot uuid=%s, name=%s", res, snapshotVO.getUuid(), name);
@@ -129,6 +131,16 @@ public class StorPoolSnapshotStrategy implements 
SnapshotStrategy {
         return res;
     }
 
+    private void markSnapshotAsDestroyedIfAlreadyRemoved(Long snapshotId, 
SpApiResponse resp) {
+        if (resp.getError().getName().equals("objectDoesNotExist")) {
+            SnapshotDataStoreVO snapshotOnPrimary = 
_snapshotStoreDao.findBySourceSnapshot(snapshotId, DataStoreRole.Primary);
+            if (snapshotOnPrimary != null) {
+                snapshotOnPrimary.setState(State.Destroyed);
+                _snapshotStoreDao.update(snapshotOnPrimary.getId(), 
snapshotOnPrimary);
+            }
+        }
+    }
+
     @Override
     public StrategyPriority canHandle(Snapshot snapshot, Long zoneId, 
SnapshotOperation op) {
         log.debug(String.format("StorpoolSnapshotStrategy.canHandle: 
snapshot=%s, uuid=%s, op=%s", snapshot.getName(), snapshot.getUuid(), op));
@@ -166,7 +178,7 @@ public class StorPoolSnapshotStrategy implements 
SnapshotStrategy {
         boolean resultIsSet = false;
         try {
             while (snapshot != null &&
-                (snapshot.getState() == Snapshot.State.Destroying || 
snapshot.getState() == Snapshot.State.Destroyed || snapshot.getState() == 
Snapshot.State.Error)) {
+                (snapshot.getState() == Snapshot.State.Destroying || 
snapshot.getState() == Snapshot.State.Destroyed || snapshot.getState() == 
Snapshot.State.Error || snapshot.getState() == Snapshot.State.BackedUp)) {
                 SnapshotInfo child = snapshot.getChild();
 
                 if (child != null) {
@@ -330,9 +342,21 @@ public class StorPoolSnapshotStrategy implements 
SnapshotStrategy {
         } else {
             snapshotZoneDao.removeSnapshotFromZones(snapshotVO.getId());
         }
+        if (CollectionUtils.isNotEmpty(retrieveSnapshotEntries(snapshotId, 
null))) {
+            return true;
+        }
+        updateSnapshotToDestroyed(snapshotVO);
         return true;
     }
 
+    private List<SnapshotInfo> retrieveSnapshotEntries(long snapshotId, Long 
zoneId) {
+        return snapshotDataFactory.getSnapshots(snapshotId, zoneId);
+    }
+
+    private void updateSnapshotToDestroyed(SnapshotVO snapshotVo) {
+        snapshotVo.setState(Snapshot.State.Destroyed);
+        _snapshotDao.update(snapshotVo.getId(), snapshotVo);
+    }
 
     @Override
     public SnapshotInfo takeSnapshot(SnapshotInfo snapshot) {


Reply via email to