Give graceful state transition period to live with race-condition on VM startup time
Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/5a75a3e1 Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/5a75a3e1 Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/5a75a3e1 Branch: refs/heads/resize-root Commit: 5a75a3e1f92dc8f6352f1e27797c44d40565454b Parents: 3123c30 Author: Kelven Yang <[email protected]> Authored: Wed Mar 12 14:30:18 2014 -0700 Committer: Kelven Yang <[email protected]> Committed: Thu Mar 13 16:59:56 2014 -0700 ---------------------------------------------------------------------- .../vm/VirtualMachinePowerStateSyncImpl.java | 44 +++++++++++++++++--- .../vmware/resource/VmwareResource.java | 2 - 2 files changed, 38 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cloudstack/blob/5a75a3e1/engine/orchestration/src/com/cloud/vm/VirtualMachinePowerStateSyncImpl.java ---------------------------------------------------------------------- diff --git a/engine/orchestration/src/com/cloud/vm/VirtualMachinePowerStateSyncImpl.java b/engine/orchestration/src/com/cloud/vm/VirtualMachinePowerStateSyncImpl.java index 6332958..9edecac 100644 --- a/engine/orchestration/src/com/cloud/vm/VirtualMachinePowerStateSyncImpl.java +++ b/engine/orchestration/src/com/cloud/vm/VirtualMachinePowerStateSyncImpl.java @@ -16,6 +16,7 @@ // under the License. package com.cloud.vm; +import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -24,10 +25,12 @@ import javax.inject.Inject; import org.apache.log4j.Logger; +import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.messagebus.MessageBus; import org.apache.cloudstack.framework.messagebus.PublishScope; import com.cloud.agent.api.HostVmStateReportEntry; +import com.cloud.utils.DateUtil; import com.cloud.vm.dao.VMInstanceDao; public class VirtualMachinePowerStateSyncImpl implements VirtualMachinePowerStateSync { @@ -37,6 +40,9 @@ public class VirtualMachinePowerStateSyncImpl implements VirtualMachinePowerStat @Inject VMInstanceDao _instanceDao; @Inject VirtualMachineManager _vmMgr; + protected final ConfigKey<Integer> PingInterval = new ConfigKey<Integer>(Integer.class, "ping.interval", "Advanced", "60", + "Interval to send application level pings to make sure the connection is still working", false); + public VirtualMachinePowerStateSyncImpl() { } @@ -96,15 +102,41 @@ public class VirtualMachinePowerStateSyncImpl implements VirtualMachinePowerStat } if (vmsThatAreMissingReport.size() > 0) { + Date currentTime = DateUtil.currentGMTTime(); + if (s_logger.isDebugEnabled()) + s_logger.debug("Run missing VM report. current time: " + currentTime.getTime()); + + // 2 times of sync-update interval for graceful period + long milliSecondsGracefullPeriod = PingInterval.value() * 2000; + for (VMInstanceVO instance : vmsThatAreMissingReport) { - if (_instanceDao.updatePowerState(instance.getId(), hostId, VirtualMachine.PowerState.PowerReportMissing)) { - if (s_logger.isDebugEnabled()) - s_logger.debug("VM state report is updated. host: " + hostId + ", vm id: " + instance.getId() + ", power state: PowerReportMissing "); - _messageBus.publish(null, VirtualMachineManager.Topics.VM_POWER_STATE, PublishScope.GLOBAL, instance.getId()); + Date vmStateUpdateTime = instance.getUpdateTime(); + if (vmStateUpdateTime == null) { + s_logger.warn("VM state was updated but update time is null?! vm id: " + instance.getId()); + vmStateUpdateTime = currentTime; + } + + if (s_logger.isDebugEnabled()) + s_logger.debug("Detected missing VM. host: " + hostId + ", vm id: " + instance.getId() + + ", power state: PowerReportMissing, last state update: " + vmStateUpdateTime.getTime()); + + long milliSecondsSinceLastStateUpdate = currentTime.getTime() - vmStateUpdateTime.getTime(); + + if (milliSecondsSinceLastStateUpdate > milliSecondsGracefullPeriod) { + s_logger.debug("vm id: " + instance.getId() + " - time since last state update(" + milliSecondsSinceLastStateUpdate + "ms) has passed graceful period"); + + if (_instanceDao.updatePowerState(instance.getId(), hostId, VirtualMachine.PowerState.PowerReportMissing)) { + if (s_logger.isDebugEnabled()) + s_logger.debug("VM state report is updated. host: " + hostId + ", vm id: " + instance.getId() + ", power state: PowerReportMissing "); + + _messageBus.publish(null, VirtualMachineManager.Topics.VM_POWER_STATE, PublishScope.GLOBAL, instance.getId()); + } else { + if (s_logger.isDebugEnabled()) + s_logger.debug("VM power state does not change, skip DB writing. vm id: " + instance.getId()); + } } else { - if (s_logger.isDebugEnabled()) - s_logger.debug("VM power state does not change, skip DB writing. vm id: " + instance.getId()); + s_logger.debug("vm id: " + instance.getId() + " - time since last state update(" + milliSecondsSinceLastStateUpdate + "ms) has not passed graceful period yet"); } } } http://git-wip-us.apache.org/repos/asf/cloudstack/blob/5a75a3e1/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java ---------------------------------------------------------------------- diff --git a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 4693f86..0a2cad4 100755 --- a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -41,7 +41,6 @@ import java.util.UUID; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; - import javax.naming.ConfigurationException; import org.apache.log4j.Logger; @@ -2681,7 +2680,6 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa try { VirtualMachineMO vmMo = hyperHost.findVmOnHyperHost(cmd.getVmName()); if (vmMo != null) { - State state = null; synchronized (_vms) { state = _vms.get(cmd.getVmName());
