This is an automated email from the ASF dual-hosted git repository.
weizhou pushed a commit to branch 4.18-vm-autoscaling
in repository https://gitbox.apache.org/repos/asf/cloudstack.git
The following commit(s) were added to refs/heads/4.18-vm-autoscaling by this
push:
new fe92a04b04e AS server: use try-catch-finally in scale down to avoid
wrong state in edge cases (e.g. VR is Starting)
fe92a04b04e is described below
commit fe92a04b04eb595dd48a8253b79ca71d23fdf3c6
Author: Wei Zhou <[email protected]>
AuthorDate: Tue Oct 18 13:54:46 2022 +0200
AS server: use try-catch-finally in scale down to avoid wrong state in edge
cases (e.g. VR is Starting)
---
.../com/cloud/network/as/AutoScaleManagerImpl.java | 87 ++++++++++++----------
1 file changed, 49 insertions(+), 38 deletions(-)
diff --git
a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java
b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java
index 60c86d0c959..597aa34baf7 100644
--- a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java
+++ b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java
@@ -2043,51 +2043,62 @@ public class AutoScaleManagerImpl extends ManagerBase
implements AutoScaleManage
ActionEventUtils.onStartedActionEvent(User.UID_SYSTEM,
asGroup.getAccountId(), EventTypes.EVENT_AUTOSCALEVMGROUP_SCALEDOWN,
"Scaling down AutoScale VM group " + groupId, groupId,
ApiCommandResourceType.AutoScaleVmGroup.toString(),
true, 0);
- final long vmId = removeLBrule(asGroup);
- if (vmId != -1) {
- long profileId = asGroup.getProfileId();
-
- // update group-vm mapping
- autoScaleVmGroupVmMapDao.remove(groupId, vmId);
- // update last_quietTime
- List<AutoScaleVmGroupPolicyMapVO> groupPolicyVOs =
autoScaleVmGroupPolicyMapDao.listByVmGroupId(groupId);
- for (AutoScaleVmGroupPolicyMapVO groupPolicyVO : groupPolicyVOs) {
- AutoScalePolicyVO vo =
autoScalePolicyDao.findById(groupPolicyVO.getPolicyId());
- if (vo.getAction().equals(AutoScalePolicy.Action.SCALEUP)) {
- vo.setLastQuietTime(new Date());
- autoScalePolicyDao.persist(vo);
- break;
+ try {
+ long vmId = -1;
+ try {
+ vmId = removeLBrule(asGroup);
+ } catch (Exception ex) {
+ s_logger.info("Got exception when remove LB rule for a VM in
AutoScale VM group %d: " + groupId, ex);
+ ActionEventUtils.onCompletedActionEvent(User.UID_SYSTEM,
asGroup.getAccountId(), EventVO.LEVEL_ERROR,
EventTypes.EVENT_AUTOSCALEVMGROUP_SCALEDOWN,
+ String.format("Failed to remove LB rule for a VM in
AutoScale VM group %d", groupId), groupId,
ApiCommandResourceType.AutoScaleVmGroup.toString(), 0);
+ throw ex;
+ }
+ if (vmId != -1) {
+ long profileId = asGroup.getProfileId();
+
+ // update group-vm mapping
+ autoScaleVmGroupVmMapDao.remove(groupId, vmId);
+ // update last_quietTime
+ List<AutoScaleVmGroupPolicyMapVO> groupPolicyVOs =
autoScaleVmGroupPolicyMapDao.listByVmGroupId(groupId);
+ for (AutoScaleVmGroupPolicyMapVO groupPolicyVO :
groupPolicyVOs) {
+ AutoScalePolicyVO vo =
autoScalePolicyDao.findById(groupPolicyVO.getPolicyId());
+ if (vo.getAction().equals(AutoScalePolicy.Action.SCALEUP))
{
+ vo.setLastQuietTime(new Date());
+ autoScalePolicyDao.persist(vo);
+ break;
+ }
}
- }
- // Add an Inactive-dummy record to statistics table
- createInactiveDummyRecord(asGroup.getId());
+ // Add an Inactive-dummy record to statistics table
+ createInactiveDummyRecord(asGroup.getId());
- // get expungeVmGracePeriod param
- AutoScaleVmProfileVO asProfile =
autoScaleVmProfileDao.findById(profileId);
- Integer expungeVmGracePeriod = asProfile.getExpungeVmGracePeriod();
- if (expungeVmGracePeriod > 0) {
- try {
- Thread.sleep(expungeVmGracePeriod * 1000L);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- throw new CloudRuntimeException(String.format("Error while
waiting %s seconds to destroy the VM %s", expungeVmGracePeriod, vmId));
+ // get expungeVmGracePeriod param
+ AutoScaleVmProfileVO asProfile =
autoScaleVmProfileDao.findById(profileId);
+ Integer expungeVmGracePeriod =
asProfile.getExpungeVmGracePeriod();
+ if (expungeVmGracePeriod > 0) {
+ try {
+ Thread.sleep(expungeVmGracePeriod * 1000L);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new CloudRuntimeException(String.format("Error
while waiting %s seconds to destroy the VM %s", expungeVmGracePeriod, vmId));
+ }
+ }
+ if (destroyVm(vmId)) {
+ ActionEventUtils.onCompletedActionEvent(User.UID_SYSTEM,
asGroup.getAccountId(), EventVO.LEVEL_INFO,
EventTypes.EVENT_AUTOSCALEVMGROUP_SCALEDOWN,
+ String.format("Destroyed VM %d in AutoScale VM
group %d", vmId, groupId), groupId,
ApiCommandResourceType.AutoScaleVmGroup.toString(), 0);
+ } else {
+ ActionEventUtils.onCompletedActionEvent(User.UID_SYSTEM,
asGroup.getAccountId(), EventVO.LEVEL_ERROR,
EventTypes.EVENT_AUTOSCALEVMGROUP_SCALEDOWN,
+ String.format("Failed to destroy VM %d in
AutoScale VM group %d", vmId, groupId), groupId,
ApiCommandResourceType.AutoScaleVmGroup.toString(), 0);
}
- }
- if (destroyVm(vmId)) {
- ActionEventUtils.onCompletedActionEvent(User.UID_SYSTEM,
asGroup.getAccountId(), EventVO.LEVEL_INFO,
EventTypes.EVENT_AUTOSCALEVMGROUP_SCALEDOWN,
- String.format("Destroyed VM %d in AutoScale VM group
%d", vmId, groupId), groupId,
ApiCommandResourceType.AutoScaleVmGroup.toString(), 0);
} else {
+ s_logger.error("Can not remove LB rule for the VM being
destroyed. Do nothing more.");
ActionEventUtils.onCompletedActionEvent(User.UID_SYSTEM,
asGroup.getAccountId(), EventVO.LEVEL_ERROR,
EventTypes.EVENT_AUTOSCALEVMGROUP_SCALEDOWN,
- String.format("Failed to destroy VM %d in AutoScale VM
group %d", vmId, groupId), groupId,
ApiCommandResourceType.AutoScaleVmGroup.toString(), 0);
+ String.format("Failed to remove LB rule for a VM in
AutoScale VM group %d", groupId), groupId,
ApiCommandResourceType.AutoScaleVmGroup.toString(), 0);
+ }
+ } finally {
+ if (!autoScaleVmGroupDao.updateState(groupId, newState, oldState))
{
+ s_logger.error(String.format("Can not update vmgroup state
from %s back to %s, groupId: %s", newState, oldState, groupId));
}
- } else {
- s_logger.error("Can not remove LB rule for the VM being destroyed.
Do nothing more.");
- ActionEventUtils.onCompletedActionEvent(User.UID_SYSTEM,
asGroup.getAccountId(), EventVO.LEVEL_ERROR,
EventTypes.EVENT_AUTOSCALEVMGROUP_SCALEDOWN,
- String.format("Failed to remove LB rule for VM %d in
AutoScale VM group %d", vmId, groupId), groupId,
ApiCommandResourceType.AutoScaleVmGroup.toString(), 0);
- }
- if (!autoScaleVmGroupDao.updateState(groupId, newState, oldState)) {
- s_logger.error(String.format("Can not update vmgroup state from %s
back to %s, groupId: %s", newState, oldState, groupId));
}
}
