This is an automated email from the ASF dual-hosted git repository.
shwstppr pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/cloudstack.git
The following commit(s) were added to refs/heads/main by this push:
new 80bbb29abfb CleanUp Async Jobs after mgmt server maintenance (#8394)
80bbb29abfb is described below
commit 80bbb29abfb90476cec155c2f831e626710bf420
Author: kishankavala <[email protected]>
AuthorDate: Fri Jan 19 13:26:25 2024 +0530
CleanUp Async Jobs after mgmt server maintenance (#8394)
This PR fixes moves resources stuck in transition state during async job
cleanup
Problem:
During maintenance of the management server, other servers in the cluster
or the same server after a restart initiate async job cleanup. However, this
process leaves resources in a transitional state. The only recovery option
currently available is to make direct database changes.
Solution:
This PR introduces a resolution by changing Volume, Virtual Machine, and
Network resources from their transitional states. This adjustment enables the
reattempt of failed operations without the need for manual database
modifications.
---
api/src/main/java/com/cloud/storage/Volume.java | 54 +++++---
.../service/NetworkOrchestrationService.java | 3 +
.../subsystem/api/storage/VolumeService.java | 5 +-
.../engine/orchestration/NetworkOrchestrator.java | 3 +-
.../cloudstack/storage/volume/VolumeObject.java | 2 +-
.../storage/volume/VolumeServiceImpl.java | 12 +-
.../framework/jobs/impl/AsyncJobManagerImpl.java | 143 ++++++++++++++++++---
.../jobs/impl/AsyncJobManagerImplTest.java | 96 ++++++++++++++
.../java/com/cloud/vpc/MockNetworkManagerImpl.java | 6 +
9 files changed, 274 insertions(+), 50 deletions(-)
diff --git a/api/src/main/java/com/cloud/storage/Volume.java
b/api/src/main/java/com/cloud/storage/Volume.java
index 4a14197bd30..308ed2544ed 100644
--- a/api/src/main/java/com/cloud/storage/Volume.java
+++ b/api/src/main/java/com/cloud/storage/Volume.java
@@ -39,30 +39,38 @@ public interface Volume extends ControlledEntity, Identity,
InternalIdentity, Ba
};
enum State {
- Allocated("The volume is allocated but has not been created yet."),
- Creating("The volume is being created. getPoolId() should reflect the
pool where it is being created."),
- Ready("The volume is ready to be used."),
- Migrating("The volume is migrating to other storage pool"),
- Snapshotting("There is a snapshot created on this volume, not backed
up to secondary storage yet"),
- RevertSnapshotting("There is a snapshot created on this volume, the
volume is being reverting from snapshot"),
- Resizing("The volume is being resized"),
- Expunging("The volume is being expunging"),
- Expunged("The volume has been expunged, and can no longer be
recovered"),
- Destroy("The volume is destroyed, and can be recovered."),
- Destroying("The volume is destroying, and can't be recovered."),
- UploadOp("The volume upload operation is in progress or in short the
volume is on secondary storage"),
- Copying("Volume is copying from image store to primary, in case it's
an uploaded volume"),
- Uploaded("Volume is uploaded"),
- NotUploaded("The volume entry is just created in DB, not yet
uploaded"),
- UploadInProgress("Volume upload is in progress"),
- UploadError("Volume upload encountered some error"),
- UploadAbandoned("Volume upload is abandoned since the upload was never
initiated within a specified time"),
- Attaching("The volume is attaching to a VM from Ready state."),
- Restoring("The volume is being restored from backup.");
+ Allocated(false, "The volume is allocated but has not been created
yet."),
+ Creating(true, "The volume is being created. getPoolId() should
reflect the pool where it is being created."),
+ Ready(false, "The volume is ready to be used."),
+ Migrating(true, "The volume is migrating to other storage pool"),
+ Snapshotting(true, "There is a snapshot created on this volume, not
backed up to secondary storage yet"),
+ RevertSnapshotting(true, "There is a snapshot created on this volume,
the volume is being reverting from snapshot"),
+ Resizing(true, "The volume is being resized"),
+ Expunging(true, "The volume is being expunging"),
+ Expunged(false, "The volume has been expunged, and can no longer be
recovered"),
+ Destroy(false, "The volume is destroyed, and can be recovered."),
+ Destroying(false, "The volume is destroying, and can't be recovered."),
+ UploadOp(true, "The volume upload operation is in progress or in short
the volume is on secondary storage"),
+ Copying(true, "Volume is copying from image store to primary, in case
it's an uploaded volume"),
+ Uploaded(false, "Volume is uploaded"),
+ NotUploaded(true, "The volume entry is just created in DB, not yet
uploaded"),
+ UploadInProgress(true, "Volume upload is in progress"),
+ UploadError(false, "Volume upload encountered some error"),
+ UploadAbandoned(false, "Volume upload is abandoned since the upload
was never initiated within a specified time"),
+ Attaching(true, "The volume is attaching to a VM from Ready state."),
+ Restoring(true, "The volume is being restored from backup.");
+
+ boolean _transitional;
String _description;
- private State(String description) {
+ /**
+ * Volume State
+ * @param transitional true for transition/non-final state, otherwise
false
+ * @param description description of the state
+ */
+ private State(boolean transitional, String description) {
+ _transitional = transitional;
_description = description;
}
@@ -70,6 +78,10 @@ public interface Volume extends ControlledEntity, Identity,
InternalIdentity, Ba
return s_fsm;
}
+ public boolean isTransitional() {
+ return _transitional;
+ }
+
public String getDescription() {
return _description;
}
diff --git
a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java
b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java
index c691b8b0942..2005b70b439 100644
---
a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java
+++
b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java
@@ -50,6 +50,7 @@ import com.cloud.network.rules.LoadBalancerContainer.Scheme;
import com.cloud.offering.NetworkOffering;
import com.cloud.user.Account;
import com.cloud.user.User;
+import com.cloud.utils.fsm.NoTransitionException;
import com.cloud.utils.Pair;
import com.cloud.vm.Nic;
import com.cloud.vm.NicProfile;
@@ -268,6 +269,8 @@ public interface NetworkOrchestrationService {
Map<String, String> finalizeServicesAndProvidersForNetwork(NetworkOffering
offering, Long physicalNetworkId);
+ boolean stateTransitTo(Network network, Network.Event e) throws
NoTransitionException;
+
List<Provider> getProvidersForServiceInNetwork(Network network, Service
service);
StaticNatServiceProvider getStaticNatProviderForNetwork(Network network);
diff --git
a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeService.java
b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeService.java
index 2c12b70e9eb..50aee83f497 100644
---
a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeService.java
+++
b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeService.java
@@ -18,13 +18,13 @@
*/
package org.apache.cloudstack.engine.subsystem.api.storage;
-import com.cloud.agent.api.Answer;
import java.util.Map;
import org.apache.cloudstack.engine.cloud.entity.api.VolumeEntity;
import org.apache.cloudstack.framework.async.AsyncCallFuture;
import org.apache.cloudstack.storage.command.CommandResult;
+import com.cloud.agent.api.Answer;
import com.cloud.agent.api.to.VirtualMachineTO;
import com.cloud.exception.StorageAccessException;
import com.cloud.host.Host;
@@ -35,6 +35,9 @@ import com.cloud.user.Account;
import com.cloud.utils.Pair;
public interface VolumeService {
+
+ String SNAPSHOT_ID = "SNAPSHOT_ID";
+
class VolumeApiResult extends CommandResult {
private final VolumeInfo volume;
diff --git
a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java
b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java
index 0f6dfe60866..6c10a02abba 100644
---
a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java
+++
b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java
@@ -4424,7 +4424,8 @@ public class NetworkOrchestrator extends ManagerBase
implements NetworkOrchestra
return accessDetails;
}
- protected boolean stateTransitTo(final NetworkVO network, final
Network.Event e) throws NoTransitionException {
+ @Override
+ public boolean stateTransitTo(final Network network, final Network.Event
e) throws NoTransitionException {
return _stateMachine.transitTo(network, e, null, _networksDao);
}
diff --git
a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java
b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java
index 5ebee87acd4..9e41e0d4d0e 100644
---
a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java
+++
b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java
@@ -851,7 +851,7 @@ public class VolumeObject implements VolumeInfo {
@Override
public boolean delete() {
- return dataStore == null ? true : dataStore.delete(this);
+ return dataStore == null || dataStore.delete(this);
}
@Override
diff --git
a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java
b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java
index c0ef227251c..8a3cd39ecbf 100644
---
a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java
+++
b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java
@@ -32,10 +32,6 @@ import java.util.concurrent.ExecutionException;
import javax.inject.Inject;
-import org.apache.cloudstack.secret.dao.PassphraseDao;
-import com.cloud.storage.VMTemplateVO;
-import com.cloud.storage.dao.VMTemplateDao;
-import com.cloud.storage.resource.StorageProcessor;
import org.apache.cloudstack.annotation.AnnotationService;
import org.apache.cloudstack.annotation.dao.AnnotationDao;
import org.apache.cloudstack.engine.cloud.entity.api.VolumeEntity;
@@ -66,6 +62,7 @@ import
org.apache.cloudstack.framework.async.AsyncCallbackDispatcher;
import org.apache.cloudstack.framework.async.AsyncCompletionCallback;
import org.apache.cloudstack.framework.async.AsyncRpcContext;
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
+import org.apache.cloudstack.secret.dao.PassphraseDao;
import org.apache.cloudstack.storage.RemoteHostEndPoint;
import org.apache.cloudstack.storage.command.CommandResult;
import org.apache.cloudstack.storage.command.CopyCmdAnswer;
@@ -83,6 +80,7 @@ import
org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO;
import org.apache.cloudstack.storage.image.store.TemplateObject;
import org.apache.cloudstack.storage.to.TemplateObjectTO;
import org.apache.cloudstack.storage.to.VolumeObjectTO;
+import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;
@@ -122,13 +120,16 @@ import com.cloud.storage.StoragePool;
import com.cloud.storage.VMTemplateStoragePoolVO;
import com.cloud.storage.VMTemplateStorageResourceAssoc;
import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
+import com.cloud.storage.VMTemplateVO;
import com.cloud.storage.Volume;
import com.cloud.storage.Volume.State;
import com.cloud.storage.VolumeDetailVO;
import com.cloud.storage.VolumeVO;
+import com.cloud.storage.dao.VMTemplateDao;
import com.cloud.storage.dao.VMTemplatePoolDao;
import com.cloud.storage.dao.VolumeDao;
import com.cloud.storage.dao.VolumeDetailsDao;
+import com.cloud.storage.resource.StorageProcessor;
import com.cloud.storage.snapshot.SnapshotApiService;
import com.cloud.storage.snapshot.SnapshotManager;
import com.cloud.storage.template.TemplateConstants;
@@ -142,7 +143,6 @@ import com.cloud.utils.db.DB;
import com.cloud.utils.db.GlobalLock;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.vm.VirtualMachine;
-import org.apache.commons.lang3.StringUtils;
@Component
public class VolumeServiceImpl implements VolumeService {
@@ -206,8 +206,6 @@ public class VolumeServiceImpl implements VolumeService {
@Inject
private PassphraseDao passphraseDao;
- private final static String SNAPSHOT_ID = "SNAPSHOT_ID";
-
public VolumeServiceImpl() {
}
diff --git
a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java
b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java
index 9100ee5e34b..3c0f81d0bc1 100644
---
a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java
+++
b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java
@@ -38,9 +38,13 @@ import javax.naming.ConfigurationException;
import org.apache.cloudstack.api.ApiCommandResourceType;
import org.apache.cloudstack.api.ApiErrorCode;
import org.apache.cloudstack.context.CallContext;
+import
org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService;
import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory;
import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotService;
+import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory;
+import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo;
+import org.apache.cloudstack.engine.subsystem.api.storage.VolumeService;
import org.apache.cloudstack.framework.config.ConfigKey;
import org.apache.cloudstack.framework.config.Configurable;
import org.apache.cloudstack.framework.jobs.AsyncJob;
@@ -65,7 +69,12 @@ import org.apache.log4j.MDC;
import org.apache.log4j.NDC;
import com.cloud.cluster.ClusterManagerListener;
+import com.cloud.network.Network;
+import com.cloud.network.dao.NetworkDao;
+import com.cloud.network.dao.NetworkVO;
import com.cloud.storage.Snapshot;
+import com.cloud.storage.Volume;
+import com.cloud.storage.VolumeDetailVO;
import com.cloud.storage.dao.SnapshotDao;
import com.cloud.storage.dao.SnapshotDetailsDao;
import com.cloud.storage.dao.SnapshotDetailsVO;
@@ -93,7 +102,11 @@ import com.cloud.utils.db.TransactionCallbackNoReturn;
import com.cloud.utils.db.TransactionStatus;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.exception.ExceptionUtil;
+import com.cloud.utils.fsm.NoTransitionException;
import com.cloud.utils.mgmt.JmxUtil;
+import com.cloud.vm.VMInstanceVO;
+import com.cloud.vm.VirtualMachine;
+import com.cloud.vm.VirtualMachineManager;
import com.cloud.vm.dao.VMInstanceDao;
public class AsyncJobManagerImpl extends ManagerBase implements
AsyncJobManager, ClusterManagerListener, Configurable {
@@ -148,6 +161,15 @@ public class AsyncJobManagerImpl extends ManagerBase
implements AsyncJobManager,
@Inject
private SnapshotDetailsDao _snapshotDetailsDao;
+ @Inject
+ private VolumeDataFactory volFactory;
+ @Inject
+ private VirtualMachineManager virtualMachineManager;
+ @Inject
+ private NetworkDao networkDao;
+ @Inject
+ private NetworkOrchestrationService networkOrchestrationService;
+
private volatile long _executionRunNumber = 1;
private final ScheduledExecutorService _heartbeatScheduler =
Executors.newScheduledThreadPool(1, new
NamedThreadFactory("AsyncJobMgr-Heartbeat"));
@@ -1089,6 +1111,7 @@ public class AsyncJobManagerImpl extends ManagerBase
implements AsyncJobManager,
if (s_logger.isDebugEnabled()) {
s_logger.debug("Cancel left-over job-" +
job.getId());
}
+ cleanupResources(job);
job.setStatus(JobInfo.Status.FAILED);
job.setResultCode(ApiErrorCode.INTERNAL_ERROR.getHttpCode());
job.setResult("job cancelled because of management
server restart or shutdown");
@@ -1101,26 +1124,8 @@ public class AsyncJobManagerImpl extends ManagerBase
implements AsyncJobManager,
s_logger.debug("Purge queue item for cancelled
job-" + job.getId());
}
_queueMgr.purgeAsyncJobQueueItemId(job.getId());
- if
(ApiCommandResourceType.Volume.toString().equals(job.getInstanceType())) {
-
- try {
-
_volumeDetailsDao.removeDetail(job.getInstanceId(), "SNAPSHOT_ID");
- _volsDao.remove(job.getInstanceId());
- } catch (Exception e) {
- s_logger.error("Unexpected exception while
removing concurrent request meta data :" + e.getLocalizedMessage());
- }
- }
- }
- final List<SnapshotDetailsVO> snapshotList =
_snapshotDetailsDao.findDetails(AsyncJob.Constants.MS_ID, Long.toString(msid),
false);
- for (final SnapshotDetailsVO snapshotDetailsVO :
snapshotList) {
- SnapshotInfo snapshot =
snapshotFactory.getSnapshotOnPrimaryStore(snapshotDetailsVO.getResourceId());
- if (snapshot == null) {
-
_snapshotDetailsDao.remove(snapshotDetailsVO.getId());
- continue;
- }
- snapshotSrv.processEventOnSnapshotObject(snapshot,
Snapshot.Event.OperationFailed);
-
_snapshotDetailsDao.removeDetail(snapshotDetailsVO.getResourceId(),
AsyncJob.Constants.MS_ID);
}
+ cleanupFailedSnapshotsCreatedWithDefaultStrategy(msid);
}
});
} catch (Throwable e) {
@@ -1128,6 +1133,106 @@ public class AsyncJobManagerImpl extends ManagerBase
implements AsyncJobManager,
}
}
+ /*
+ Cleanup Resources in transition state and move them to appropriate state
+ This will allow other operation on the resource, instead of being stuck in
transition state
+ */
+ protected boolean cleanupResources(AsyncJobVO job) {
+ try {
+ ApiCommandResourceType resourceType =
ApiCommandResourceType.fromString(job.getInstanceType());
+ if (resourceType == null) {
+ s_logger.warn("Unknown ResourceType. Skip Cleanup: " +
job.getInstanceType());
+ return true;
+ }
+ switch (resourceType) {
+ case Volume:
+ return cleanupVolume(job.getInstanceId());
+ case VirtualMachine:
+ return cleanupVirtualMachine(job.getInstanceId());
+ case Network:
+ return cleanupNetwork(job.getInstanceId());
+ }
+ } catch (Exception e) {
+ s_logger.warn("Error while cleaning up resource: [" +
job.getInstanceType().toString() + "] with Id: " + job.getInstanceId(), e);
+ return false;
+ }
+ return true;
+ }
+
+ private boolean cleanupVolume(final long volumeId) {
+ VolumeInfo vol = volFactory.getVolume(volumeId);
+ if (vol == null) {
+ s_logger.warn("Volume not found. Skip Cleanup. VolumeId: " +
volumeId);
+ return true;
+ }
+ if (vol.getState().isTransitional()) {
+ s_logger.debug("Cleaning up volume with Id: " + volumeId);
+ boolean status = vol.stateTransit(Volume.Event.OperationFailed);
+ cleanupFailedVolumesCreatedFromSnapshots(volumeId);
+ return status;
+ }
+ s_logger.debug("Volume not in transition state. Skip cleanup.
VolumeId: " + volumeId);
+ return true;
+ }
+
+ private boolean cleanupVirtualMachine(final long vmId) throws Exception {
+ VMInstanceVO vmInstanceVO = _vmInstanceDao.findById(vmId);
+ if (vmInstanceVO == null) {
+ s_logger.warn("Instance not found. Skip Cleanup. InstanceId: " +
vmId);
+ return true;
+ }
+ if (vmInstanceVO.getState().isTransitional()) {
+ s_logger.debug("Cleaning up Instance with Id: " + vmId);
+ return virtualMachineManager.stateTransitTo(vmInstanceVO,
VirtualMachine.Event.OperationFailed, vmInstanceVO.getHostId());
+ }
+ s_logger.debug("Instance not in transition state. Skip cleanup.
InstanceId: " + vmId);
+ return true;
+ }
+
+ private boolean cleanupNetwork(final long networkId) throws Exception {
+ NetworkVO networkVO = networkDao.findById(networkId);
+ if (networkVO == null) {
+ s_logger.warn("Network not found. Skip Cleanup. NetworkId: " +
networkId);
+ return true;
+ }
+ if (Network.State.Implementing.equals(networkVO.getState())) {
+ try {
+ s_logger.debug("Cleaning up Network with Id: " + networkId);
+ return networkOrchestrationService.stateTransitTo(networkVO,
Network.Event.OperationFailed);
+ } catch (final NoTransitionException e) {
+ networkVO.setState(Network.State.Shutdown);
+ networkDao.update(networkVO.getId(), networkVO);
+ }
+ }
+ s_logger.debug("Network not in transition state. Skip cleanup.
NetworkId: " + networkId);
+ return true;
+ }
+
+ private void cleanupFailedVolumesCreatedFromSnapshots(final long volumeId)
{
+ try {
+ VolumeDetailVO volumeDetail =
_volumeDetailsDao.findDetail(volumeId, VolumeService.SNAPSHOT_ID);
+ if (volumeDetail != null) {
+ _volumeDetailsDao.removeDetail(volumeId,
VolumeService.SNAPSHOT_ID);
+ _volsDao.remove(volumeId);
+ }
+ } catch (Exception e) {
+ s_logger.error("Unexpected exception while removing concurrent
request meta data :" + e.getLocalizedMessage());
+ }
+ }
+
+ private void cleanupFailedSnapshotsCreatedWithDefaultStrategy(final long
msid) {
+ final List<SnapshotDetailsVO> snapshotList =
_snapshotDetailsDao.findDetails(AsyncJob.Constants.MS_ID, Long.toString(msid),
false);
+ for (final SnapshotDetailsVO snapshotDetailsVO : snapshotList) {
+ SnapshotInfo snapshot =
snapshotFactory.getSnapshotOnPrimaryStore(snapshotDetailsVO.getResourceId());
+ if (snapshot == null) {
+ _snapshotDetailsDao.remove(snapshotDetailsVO.getId());
+ continue;
+ }
+ snapshotSrv.processEventOnSnapshotObject(snapshot,
Snapshot.Event.OperationFailed);
+
_snapshotDetailsDao.removeDetail(snapshotDetailsVO.getResourceId(),
AsyncJob.Constants.MS_ID);
+ }
+ }
+
@Override
public void onManagementNodeJoined(List<? extends ManagementServerHost>
nodeList, long selfNodeId) {
}
diff --git
a/framework/jobs/src/test/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImplTest.java
b/framework/jobs/src/test/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImplTest.java
new file mode 100644
index 00000000000..0be5dbc01cb
--- /dev/null
+++
b/framework/jobs/src/test/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImplTest.java
@@ -0,0 +1,96 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.framework.jobs.impl;
+
+import com.cloud.network.Network;
+import com.cloud.network.dao.NetworkDao;
+import com.cloud.network.dao.NetworkVO;
+import com.cloud.storage.Volume;
+import com.cloud.utils.fsm.NoTransitionException;
+import com.cloud.vm.VMInstanceVO;
+import com.cloud.vm.VirtualMachine;
+import com.cloud.vm.VirtualMachineManager;
+import com.cloud.vm.dao.VMInstanceDao;
+import org.apache.cloudstack.api.ApiCommandResourceType;
+import
org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService;
+import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory;
+import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.Spy;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class AsyncJobManagerImplTest {
+ @Spy
+ @InjectMocks
+ AsyncJobManagerImpl asyncJobManager;
+ @Mock
+ VolumeDataFactory volFactory;
+ @Mock
+ VMInstanceDao vmInstanceDao;
+ @Mock
+ VirtualMachineManager virtualMachineManager;
+ @Mock
+ NetworkDao networkDao;
+ @Mock
+ NetworkOrchestrationService networkOrchestrationService;
+
+ @Test
+ public void testCleanupVolumeResource() {
+ AsyncJobVO job = new AsyncJobVO();
+ job.setInstanceType(ApiCommandResourceType.Volume.toString());
+ job.setInstanceId(1L);
+ VolumeInfo volumeInfo = Mockito.mock(VolumeInfo.class);
+ when(volFactory.getVolume(Mockito.anyLong())).thenReturn(volumeInfo);
+ when(volumeInfo.getState()).thenReturn(Volume.State.Attaching);
+ asyncJobManager.cleanupResources(job);
+ Mockito.verify(volumeInfo,
Mockito.times(1)).stateTransit(Volume.Event.OperationFailed);
+ }
+
+ @Test
+ public void testCleanupVmResource() throws NoTransitionException {
+ AsyncJobVO job = new AsyncJobVO();
+ job.setInstanceType(ApiCommandResourceType.VirtualMachine.toString());
+ job.setInstanceId(1L);
+ VMInstanceVO vmInstanceVO = Mockito.mock(VMInstanceVO.class);
+
when(vmInstanceDao.findById(Mockito.anyLong())).thenReturn(vmInstanceVO);
+
when(vmInstanceVO.getState()).thenReturn(VirtualMachine.State.Starting);
+ when(vmInstanceVO.getHostId()).thenReturn(1L);
+ asyncJobManager.cleanupResources(job);
+ Mockito.verify(virtualMachineManager,
Mockito.times(1)).stateTransitTo(vmInstanceVO,
VirtualMachine.Event.OperationFailed, 1L);
+ }
+
+ @Test
+ public void testCleanupNetworkResource() throws NoTransitionException {
+ AsyncJobVO job = new AsyncJobVO();
+ job.setInstanceType(ApiCommandResourceType.Network.toString());
+ job.setInstanceId(1L);
+ NetworkVO networkVO = Mockito.mock(NetworkVO.class);
+ when(networkDao.findById(Mockito.anyLong())).thenReturn(networkVO);
+ when(networkVO.getState()).thenReturn(Network.State.Implementing);
+ asyncJobManager.cleanupResources(job);
+ Mockito.verify(networkOrchestrationService,
Mockito.times(1)).stateTransitTo(networkVO,
+ Network.Event.OperationFailed);
+ }
+}
diff --git a/server/src/test/java/com/cloud/vpc/MockNetworkManagerImpl.java
b/server/src/test/java/com/cloud/vpc/MockNetworkManagerImpl.java
index 04556c49ab2..288211c4330 100644
--- a/server/src/test/java/com/cloud/vpc/MockNetworkManagerImpl.java
+++ b/server/src/test/java/com/cloud/vpc/MockNetworkManagerImpl.java
@@ -26,6 +26,7 @@ import javax.naming.ConfigurationException;
import com.cloud.dc.DataCenter;
import com.cloud.network.PublicIpQuarantine;
+import com.cloud.utils.fsm.NoTransitionException;
import org.apache.cloudstack.acl.ControlledEntity.ACLType;
import org.apache.cloudstack.api.command.admin.address.ReleasePodIpCmdByAdmin;
import
org.apache.cloudstack.api.command.admin.network.DedicateGuestVlanRangeCmd;
@@ -824,6 +825,11 @@ public class MockNetworkManagerImpl extends ManagerBase
implements NetworkOrches
return null;
}
+ @Override
+ public boolean stateTransitTo(Network network, Network.Event e) throws
NoTransitionException {
+ return true;
+ }
+
@Override
public boolean isNetworkInlineMode(Network network) {
// TODO Auto-generated method stub
