This is an automated email from the ASF dual-hosted git repository. dahn pushed a commit to branch 4.20 in repository https://gitbox.apache.org/repos/asf/cloudstack.git
commit 81a8ac8e1ffa4438349eb5901e798428a6a744dd Author: Abhisar Sinha <[email protected]> AuthorDate: Mon Mar 9 10:19:56 2026 +0530 secondary storage resource limit for upload --- .../storage/command/UploadStatusCommand.java | 10 ++ .../cloud/storage/ImageStoreUploadMonitorImpl.java | 169 ++++++++++++++++----- .../storage/resource/HttpUploadServerHandler.java | 2 + .../resource/NfsSecondaryStorageResource.java | 33 +++- 4 files changed, 173 insertions(+), 41 deletions(-) diff --git a/core/src/main/java/org/apache/cloudstack/storage/command/UploadStatusCommand.java b/core/src/main/java/org/apache/cloudstack/storage/command/UploadStatusCommand.java index 9e6b76e467f..f78744046f7 100644 --- a/core/src/main/java/org/apache/cloudstack/storage/command/UploadStatusCommand.java +++ b/core/src/main/java/org/apache/cloudstack/storage/command/UploadStatusCommand.java @@ -28,6 +28,7 @@ public class UploadStatusCommand extends Command { } private String entityUuid; private EntityType entityType; + private Boolean abort; protected UploadStatusCommand() { } @@ -37,6 +38,11 @@ public class UploadStatusCommand extends Command { this.entityType = entityType; } + public UploadStatusCommand(String entityUuid, EntityType entityType, Boolean abort) { + this(entityUuid, entityType); + this.abort = abort; + } + public String getEntityUuid() { return entityUuid; } @@ -45,6 +51,10 @@ public class UploadStatusCommand extends Command { return entityType; } + public Boolean getAbort() { + return abort; + } + @Override public boolean executeInSequence() { return false; diff --git a/server/src/main/java/com/cloud/storage/ImageStoreUploadMonitorImpl.java b/server/src/main/java/com/cloud/storage/ImageStoreUploadMonitorImpl.java index 408eb69917a..b56e5b56213 100755 --- a/server/src/main/java/com/cloud/storage/ImageStoreUploadMonitorImpl.java +++ b/server/src/main/java/com/cloud/storage/ImageStoreUploadMonitorImpl.java @@ -26,10 +26,10 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; import com.cloud.agent.api.to.OVFInformationTO; -import com.cloud.exception.ResourceAllocationException; import com.cloud.resourcelimit.CheckedReservation; import com.cloud.user.Account; import com.cloud.user.dao.AccountDao; +import com.cloud.user.AccountManager; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; @@ -126,6 +126,8 @@ public class ImageStoreUploadMonitorImpl extends ManagerBase implements ImageSto private ReservationDao reservationDao; @Inject private AccountDao accountDao; + @Inject + private AccountManager _accountMgr; private long _nodeId; private ScheduledExecutorService _executor = null; @@ -214,6 +216,36 @@ public class ImageStoreUploadMonitorImpl extends ManagerBase implements ImageSto public UploadStatusCheck() { } + private Answer sendUploadStatusCommandForVolume(EndPoint ep, UploadStatusCommand cmd, VolumeVO volume) { + Answer answer = null; + try { + answer = ep.sendMessage(cmd); + } catch (CloudRuntimeException e) { + logger.warn("Unable to get upload status for volume {}. Error details: {}", volume, e.getMessage()); + answer = new UploadStatusAnswer(cmd, UploadStatus.UNKNOWN, e.getMessage()); + } + if (answer == null || !(answer instanceof UploadStatusAnswer)) { + logger.warn("No or invalid answer corresponding to UploadStatusCommand for volume {}", volume); + return null; + } + return answer; + } + + private Answer sendUploadStatusCommandForTemplate(EndPoint ep, UploadStatusCommand cmd, VMTemplateVO template) { + Answer answer = null; + try { + answer = ep.sendMessage(cmd); + } catch (CloudRuntimeException e) { + logger.warn("Unable to get upload status for template {}. Error details: {}", template, e.getMessage()); + answer = new UploadStatusAnswer(cmd, UploadStatus.UNKNOWN, e.getMessage()); + } + if (answer == null || !(answer instanceof UploadStatusAnswer)) { + logger.warn("No or invalid answer corresponding to UploadStatusCommand for template {}", template); + return null; + } + return answer; + } + @Override protected void runInContext() { // 1. Select all entries with download_state = Not_Downloaded or Download_In_Progress @@ -240,18 +272,17 @@ public class ImageStoreUploadMonitorImpl extends ManagerBase implements ImageSto UploadStatusCommand cmd = new UploadStatusCommand(volume.getUuid(), EntityType.Volume); if (host != null && host.getManagementServerId() != null) { if (_nodeId == host.getManagementServerId().longValue()) { - Answer answer = null; - try { - answer = ep.sendMessage(cmd); - } catch (CloudRuntimeException e) { - logger.warn("Unable to get upload status for volume {}. Error details: {}", volume, e.getMessage()); - answer = new UploadStatusAnswer(cmd, UploadStatus.UNKNOWN, e.getMessage()); - } - if (answer == null || !(answer instanceof UploadStatusAnswer)) { - logger.warn("No or invalid answer corresponding to UploadStatusCommand for volume {}", volume); + Answer answer = sendUploadStatusCommandForVolume(ep, cmd, volume); + if (answer == null) { continue; } - handleVolumeStatusResponse((UploadStatusAnswer)answer, volume, volumeDataStore); + if (!handleVolumeStatusResponse((UploadStatusAnswer)answer, volume, volumeDataStore)) { + cmd = new UploadStatusCommand(volume.getUuid(), EntityType.Volume, true); + answer = sendUploadStatusCommandForVolume(ep, cmd, volume); + if (answer == null) { + logger.warn("Unable to abort upload for volume {}", volume); + } + } } } else { String error = "Volume " + volume.getUuid() + " failed to upload as SSVM is either destroyed or SSVM agent not in 'Up' state"; @@ -284,18 +315,17 @@ public class ImageStoreUploadMonitorImpl extends ManagerBase implements ImageSto UploadStatusCommand cmd = new UploadStatusCommand(template.getUuid(), EntityType.Template); if (host != null && host.getManagementServerId() != null) { if (_nodeId == host.getManagementServerId().longValue()) { - Answer answer = null; - try { - answer = ep.sendMessage(cmd); - } catch (CloudRuntimeException e) { - logger.warn("Unable to get upload status for template {}. Error details: {}", template, e.getMessage()); - answer = new UploadStatusAnswer(cmd, UploadStatus.UNKNOWN, e.getMessage()); - } - if (answer == null || !(answer instanceof UploadStatusAnswer)) { - logger.warn("No or invalid answer corresponding to UploadStatusCommand for template {}", template); + Answer answer = sendUploadStatusCommandForTemplate(ep, cmd, template); + if (answer == null) { continue; } - handleTemplateStatusResponse((UploadStatusAnswer)answer, template, templateDataStore); + if (!handleTemplateStatusResponse((UploadStatusAnswer) answer, template, templateDataStore)) { + cmd = new UploadStatusCommand(template.getUuid(), EntityType.Template, true); + answer = sendUploadStatusCommandForTemplate(ep, cmd, template); + if (answer == null) { + logger.warn("Unable to abort upload for template {}", template); + } + } } } else { String error = String.format( @@ -312,7 +342,41 @@ public class ImageStoreUploadMonitorImpl extends ManagerBase implements ImageSto } } - private void handleVolumeStatusResponse(final UploadStatusAnswer answer, final VolumeVO volume, final VolumeDataStoreVO volumeDataStore) { + private Boolean checkAndUpdateSecondaryStorageResourceLimit(Long accountId, Long lastSize, Long currentSize) { + if (lastSize >= currentSize) { + return true; + } + Long usage = currentSize - lastSize; + try (CheckedReservation secStorageReservation = new CheckedReservation(_accountMgr.getAccount(accountId), Resource.ResourceType.secondary_storage, null, null, usage, reservationDao, _resourceLimitMgr)) { + _resourceLimitMgr.incrementResourceCount(accountId, Resource.ResourceType.secondary_storage, usage); + return true; + } catch (Exception e) { + _resourceLimitMgr.decrementResourceCount(accountId, Resource.ResourceType.secondary_storage, lastSize); + return false; + } + } + + private Boolean checkAndUpdateVolumeResourceLimit(VolumeVO volume, VolumeDataStoreVO volumeDataStore, UploadStatusAnswer answer) { + boolean success = true; + Long currentSize = answer.getVirtualSize() != 0 ? answer.getVirtualSize() : answer.getPhysicalSize(); + Long lastSize = volume.getSize() != null ? volume.getSize() : 0L; + if (!checkAndUpdateSecondaryStorageResourceLimit(volume.getAccountId(), volume.getSize(), currentSize)) { + volumeDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR); + volumeDataStore.setState(State.Failed); + volumeDataStore.setErrorString("Storage Limit Reached"); + Account owner = accountDao.findById(volume.getAccountId()); + String msg = String.format("Upload of volume [%s] failed because its owner [%s] does not have enough secondary storage space available.", volume.getUuid(), owner.getUuid()); + logger.error(msg); + success = false; + } + VolumeVO volumeUpdate = _volumeDao.findById(volume.getId()); + volumeUpdate.setSize(currentSize); + _volumeDao.update(volumeUpdate.getId(), volumeUpdate); + return success; + } + + private boolean handleVolumeStatusResponse(final UploadStatusAnswer answer, final VolumeVO volume, final VolumeDataStoreVO volumeDataStore) { + final boolean[] needAbort = new boolean[]{false}; final StateMachine2<Volume.State, Event, Volume> stateMachine = Volume.State.getStateMachine(); Transaction.execute(new TransactionCallbackNoReturn() { @Override @@ -324,6 +388,11 @@ public class ImageStoreUploadMonitorImpl extends ManagerBase implements ImageSto try { switch (answer.getStatus()) { case COMPLETED: + if (!checkAndUpdateVolumeResourceLimit(tmpVolume, tmpVolumeDataStore, answer)) { + stateMachine.transitTo(tmpVolume, Event.OperationFailed, null, _volumeDao); + sendAlert = true; + break; + } tmpVolumeDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOADED); tmpVolumeDataStore.setState(State.Ready); tmpVolumeDataStore.setInstallPath(answer.getInstallPath()); @@ -335,7 +404,6 @@ public class ImageStoreUploadMonitorImpl extends ManagerBase implements ImageSto volumeUpdate.setSize(answer.getVirtualSize()); _volumeDao.update(tmpVolume.getId(), volumeUpdate); stateMachine.transitTo(tmpVolume, Event.OperationSucceeded, null, _volumeDao); - _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), Resource.ResourceType.secondary_storage, answer.getVirtualSize()); // publish usage events UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_UPLOAD, tmpVolume.getAccountId(), @@ -348,6 +416,12 @@ public class ImageStoreUploadMonitorImpl extends ManagerBase implements ImageSto } break; case IN_PROGRESS: + if (!checkAndUpdateVolumeResourceLimit(tmpVolume, tmpVolumeDataStore, answer)) { + stateMachine.transitTo(tmpVolume, Event.OperationFailed, null, _volumeDao); + sendAlert = true; + needAbort[0] = true; + break; + } if (tmpVolume.getState() == Volume.State.NotUploaded) { tmpVolumeDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOAD_IN_PROGRESS); tmpVolumeDataStore.setDownloadPercent(answer.getDownloadPercent()); @@ -396,10 +470,29 @@ public class ImageStoreUploadMonitorImpl extends ManagerBase implements ImageSto } } }); + return !needAbort[0]; + } + + private Boolean checkAndUpdateTemplateResourceLimit(VMTemplateVO template, TemplateDataStoreVO templateDataStore, UploadStatusAnswer answer) { + boolean success = true; + Long currentSize = answer.getVirtualSize() != 0 ? answer.getVirtualSize() : answer.getPhysicalSize(); + Long lastSize = template.getSize() != null ? template.getSize() : 0L; + if (!checkAndUpdateSecondaryStorageResourceLimit(template.getAccountId(), lastSize, currentSize)) { + templateDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR); + templateDataStore.setErrorString("Storage Limit Reached"); + templateDataStore.setState(State.Failed); + Account owner = accountDao.findById(template.getAccountId()); + String msg = String.format("Upload of template [%s] failed because its owner [%s] does not have enough secondary storage space available.", template.getUuid(), owner.getUuid()); + logger.error(msg); + success = false; + } + templateDataStore.setSize(currentSize); + return success; } - private void handleTemplateStatusResponse(final UploadStatusAnswer answer, final VMTemplateVO template, final TemplateDataStoreVO templateDataStore) { + private boolean handleTemplateStatusResponse(final UploadStatusAnswer answer, final VMTemplateVO template, final TemplateDataStoreVO templateDataStore) { final StateMachine2<VirtualMachineTemplate.State, VirtualMachineTemplate.Event, VirtualMachineTemplate> stateMachine = VirtualMachineTemplate.State.getStateMachine(); + final boolean[] needAbort = new boolean[]{false}; Transaction.execute(new TransactionCallbackNoReturn() { @Override public void doInTransactionWithoutResult(TransactionStatus status) { @@ -410,6 +503,11 @@ public class ImageStoreUploadMonitorImpl extends ManagerBase implements ImageSto try { switch (answer.getStatus()) { case COMPLETED: + if (!checkAndUpdateTemplateResourceLimit(tmpTemplate, tmpTemplateDataStore, answer)) { + stateMachine.transitTo(tmpTemplate, VirtualMachineTemplate.Event.OperationFailed, null, _templateDao); + sendAlert = true; + break; + } tmpTemplateDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOADED); tmpTemplateDataStore.setState(State.Ready); tmpTemplateDataStore.setInstallPath(answer.getInstallPath()); @@ -445,22 +543,6 @@ public class ImageStoreUploadMonitorImpl extends ManagerBase implements ImageSto break; } } - - Account owner = accountDao.findById(template.getAccountId()); - long templateSize = answer.getVirtualSize(); - - try (CheckedReservation secondaryStorageReservation = new CheckedReservation(owner, Resource.ResourceType.secondary_storage, null, null, templateSize, reservationDao, _resourceLimitMgr)) { - _resourceLimitMgr.incrementResourceCount(owner.getId(), Resource.ResourceType.secondary_storage, templateSize); - } catch (ResourceAllocationException e) { - tmpTemplateDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.UPLOAD_ERROR); - tmpTemplateDataStore.setState(State.Failed); - stateMachine.transitTo(tmpTemplate, VirtualMachineTemplate.Event.OperationFailed, null, _templateDao); - msg = String.format("Upload of template [%s] failed because its owner [%s] does not have enough secondary storage space available.", template.getUuid(), owner.getUuid()); - logger.warn(msg); - sendAlert = true; - break; - } - stateMachine.transitTo(tmpTemplate, VirtualMachineTemplate.Event.OperationSucceeded, null, _templateDao); //publish usage event String etype = EventTypes.EVENT_TEMPLATE_CREATE; @@ -477,6 +559,12 @@ public class ImageStoreUploadMonitorImpl extends ManagerBase implements ImageSto } break; case IN_PROGRESS: + if (!checkAndUpdateTemplateResourceLimit(tmpTemplate, tmpTemplateDataStore, answer)) { + stateMachine.transitTo(tmpTemplate, VirtualMachineTemplate.Event.OperationFailed, null, _templateDao); + sendAlert = true; + needAbort[0] = true; + break; + } if (tmpTemplate.getState() == VirtualMachineTemplate.State.NotUploaded) { tmpTemplateDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOAD_IN_PROGRESS); stateMachine.transitTo(tmpTemplate, VirtualMachineTemplate.Event.UploadRequested, null, _templateDao); @@ -526,6 +614,7 @@ public class ImageStoreUploadMonitorImpl extends ManagerBase implements ImageSto } } }); + return !needAbort[0]; } } diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/HttpUploadServerHandler.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/HttpUploadServerHandler.java index a580105d52a..605749649bb 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/HttpUploadServerHandler.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/HttpUploadServerHandler.java @@ -130,6 +130,7 @@ public class HttpUploadServerHandler extends SimpleChannelInboundHandler<HttpObj if (decoder != null) { decoder.cleanFiles(); } + storageResource.deregisterUploadChannel(uuid); requestProcessed = false; } @@ -182,6 +183,7 @@ public class HttpUploadServerHandler extends SimpleChannelInboundHandler<HttpObj requestProcessed = true; return; } + storageResource.registerUploadChannel(uuid, ctx.channel()); //set the base directory to download the file DiskFileUpload.baseDirectory = uploadEntity.getInstallPathPrefix(); this.processTimeout = uploadEntity.getProcessTimeout(); diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java index 9b50666258e..dc27f74bf3b 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java @@ -49,6 +49,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -254,7 +255,8 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S protected String _parent = "/mnt/SecStorage"; final private String _tmpltpp = "template.properties"; protected String createTemplateFromSnapshotXenScript; - private HashMap<String, UploadEntity> uploadEntityStateMap = new HashMap<>(); + private final Map<String, UploadEntity> uploadEntityStateMap = new ConcurrentHashMap<>(); + private final Map<String, Channel> uploadChannelMap = new ConcurrentHashMap<>(); private String _ssvmPSK = null; private long processTimeout; @@ -2374,6 +2376,20 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S String entityUuid = cmd.getEntityUuid(); if (uploadEntityStateMap.containsKey(entityUuid)) { UploadEntity uploadEntity = uploadEntityStateMap.get(entityUuid); + if (Boolean.TRUE.equals(cmd.getAbort())) { + updateStateMapWithError(entityUuid, "Upload Entity aborted"); + String errorMsg = uploadEntity.getErrorMessage(); + if (errorMsg == null) { + errorMsg = "Upload aborted by management server"; + } + Channel channel = uploadChannelMap.remove(entityUuid); + if (channel != null && channel.isActive()) { + logger.info("Closing upload channel for entity {}", entityUuid); + channel.close(); + } + uploadEntityStateMap.remove(entityUuid); + return new UploadStatusAnswer(cmd, UploadStatus.ERROR, errorMsg); + } if (uploadEntity.getUploadState() == UploadEntity.Status.ERROR) { uploadEntityStateMap.remove(entityUuid); return new UploadStatusAnswer(cmd, UploadStatus.ERROR, uploadEntity.getErrorMessage()); @@ -2392,6 +2408,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S UploadStatusAnswer answer = new UploadStatusAnswer(cmd, UploadStatus.IN_PROGRESS); long downloadedSize = FileUtils.sizeOfDirectory(new File(uploadEntity.getInstallPathPrefix())); int downloadPercent = (int)(100 * downloadedSize / uploadEntity.getContentLength()); + answer.setPhysicalSize(downloadedSize); answer.setDownloadPercent(Math.min(downloadPercent, 100)); return answer; } @@ -3421,6 +3438,10 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S public String postUpload(String uuid, String filename, long processTimeout) { UploadEntity uploadEntity = uploadEntityStateMap.get(uuid); + if (uploadEntity == null) { + logger.warn("Upload entity not found for uuid: {}. Upload may have been aborted.", uuid); + return "Upload entity not found. Upload may have been aborted."; + } int installTimeoutPerGig = 180 * 60 * 1000; String resourcePath = uploadEntity.getInstallPathPrefix(); @@ -3571,6 +3592,16 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S return _ssvmPSK; } + public void registerUploadChannel(String uuid, Channel channel) { + uploadChannelMap.put(uuid, channel); + } + + public void deregisterUploadChannel(String uuid) { + if (uuid != null) { + uploadChannelMap.remove(uuid); + } + } + public void updateStateMapWithError(String uuid, String errorMessage) { UploadEntity uploadEntity = null; if (uploadEntityStateMap.get(uuid) != null) {
