Handle a race-condition in VirtualNetworkManagerImpl.java in the process of starting it parallelly.
Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/42d63737 Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/42d63737 Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/42d63737 Branch: refs/heads/resize-root Commit: 42d637376bba7a975830d695f14be7678dc6e110 Parents: 5a75a3e Author: Kelven Yang <[email protected]> Authored: Wed Mar 12 16:09:51 2014 -0700 Committer: Kelven Yang <[email protected]> Committed: Thu Mar 13 16:59:56 2014 -0700 ---------------------------------------------------------------------- .../VirtualNetworkApplianceManagerImpl.java | 117 ++++++++++++------- 1 file changed, 76 insertions(+), 41 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cloudstack/blob/42d63737/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java ---------------------------------------------------------------------- diff --git a/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java b/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java index 74cfd74..eeab91d 100755 --- a/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java +++ b/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java @@ -17,6 +17,44 @@ package com.cloud.network.router; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Calendar; +import java.util.Collections; +import java.util.Comparator; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TimeZone; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import javax.ejb.Local; +import javax.inject.Inject; +import javax.naming.ConfigurationException; +import org.apache.log4j.Logger; +import org.apache.cloudstack.api.command.admin.router.RebootRouterCmd; +import org.apache.cloudstack.api.command.admin.router.UpgradeRouterCmd; +import org.apache.cloudstack.api.command.admin.router.UpgradeRouterTemplateCmd; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; +import org.apache.cloudstack.framework.config.ConfigDepot; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.framework.jobs.AsyncJobManager; +import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO; +import org.apache.cloudstack.managed.context.ManagedContextRunnable; +import org.apache.cloudstack.utils.identity.ManagementServerNode; import com.cloud.agent.AgentManager; import com.cloud.agent.Listener; import com.cloud.agent.api.AgentControlAnswer; @@ -238,45 +276,7 @@ import com.cloud.vm.dao.NicIpAliasVO; import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.UserVmDetailsDao; import com.cloud.vm.dao.VMInstanceDao; -import org.apache.cloudstack.api.command.admin.router.RebootRouterCmd; -import org.apache.cloudstack.api.command.admin.router.UpgradeRouterCmd; -import org.apache.cloudstack.api.command.admin.router.UpgradeRouterTemplateCmd; import org.apache.cloudstack.config.ApiServiceConfiguration; -import org.apache.cloudstack.context.CallContext; -import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; -import org.apache.cloudstack.framework.config.ConfigDepot; -import org.apache.cloudstack.framework.config.ConfigKey; -import org.apache.cloudstack.framework.config.Configurable; -import org.apache.cloudstack.framework.config.dao.ConfigurationDao; -import org.apache.cloudstack.framework.jobs.AsyncJobManager; -import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO; -import org.apache.cloudstack.managed.context.ManagedContextRunnable; -import org.apache.cloudstack.utils.identity.ManagementServerNode; -import org.apache.log4j.Logger; - -import javax.ejb.Local; -import javax.inject.Inject; -import javax.naming.ConfigurationException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Calendar; -import java.util.Collections; -import java.util.Comparator; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TimeZone; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; /** * VirtualNetworkApplianceManagerImpl manages the different types of virtual network appliances available in the Cloud Stack. @@ -1885,8 +1885,35 @@ public class VirtualNetworkApplianceManagerImpl extends ManagerBase implements V return new Pair<DeploymentPlan, List<DomainRouterVO>>(plan, routers); } - private DomainRouterVO startVirtualRouter(final DomainRouterVO router, final User user, final Account caller, final Map<Param, Object> params) throws StorageUnavailableException, - InsufficientCapacityException, ConcurrentOperationException, ResourceUnavailableException { + private DomainRouterVO waitRouter(DomainRouterVO router) { + DomainRouterVO vm = _routerDao.findById(router.getId()); + + if (s_logger.isDebugEnabled()) + s_logger.debug("Router " + router.getInstanceName() + " is not fully up yet, we will wait"); + while (vm.getState() == State.Starting) { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + } + + // reload to get the latest state info + vm = _routerDao.findById(router.getId()); + } + + if (vm.getState() == State.Running) { + if (s_logger.isDebugEnabled()) + s_logger.debug("Router " + router.getInstanceName() + " is now fully up"); + + return router; + } + + s_logger.warn("Router " + router.getInstanceName() + " failed to start. current state: " + vm.getState()); + return null; + } + + private DomainRouterVO startVirtualRouter(DomainRouterVO router, User user, Account caller, Map<Param, Object> params) + throws StorageUnavailableException, InsufficientCapacityException, + ConcurrentOperationException, ResourceUnavailableException { if (router.getRole() != Role.VIRTUAL_ROUTER || !router.getIsRedundantRouter()) { return this.start(router, user, caller, params, null); @@ -1897,7 +1924,15 @@ public class VirtualNetworkApplianceManagerImpl extends ManagerBase implements V return router; } - final DataCenterDeployment plan = new DataCenterDeployment(0, null, null, null, null, null); + // + // If another thread has already requested a VR start, there is a transition period for VR to transit from + // Starting to Running, there exist a race conditioning window here + // We will wait until VR is up or fail + if (router.getState() == State.Starting) { + return waitRouter(router); + } + + DataCenterDeployment plan = new DataCenterDeployment(0, null, null, null, null, null); DomainRouterVO result = null; assert router.getIsRedundantRouter(); final List<Long> networkIds = _routerDao.getRouterNetworks(router.getId());
