CS-15544: recreate router if router filesystem is crashed
Project: http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/commit/8b7b838c Tree: http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/tree/8b7b838c Diff: http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/diff/8b7b838c Branch: refs/heads/vpc Commit: 8b7b838c8a689e777a9c1ed321b225631762283e Parents: ed4b6a5 Author: Edison Su <[email protected]> Authored: Wed Jul 11 16:14:28 2012 -0700 Committer: Edison Su <[email protected]> Committed: Wed Jul 11 16:14:28 2012 -0700 ---------------------------------------------------------------------- .../network/lb/ElasticLoadBalancerManagerImpl.java | 9 ++++ .../AgentBasedConsoleProxyManager.java | 8 ++++ .../consoleproxy/ConsoleProxyManagerImpl.java | 8 ++++ .../router/VirtualNetworkApplianceManagerImpl.java | 13 ++++++ server/src/com/cloud/storage/StorageManager.java | 2 +- .../src/com/cloud/storage/StorageManagerImpl.java | 30 ++++++++++++--- .../secondary/SecondaryStorageManagerImpl.java | 8 ++++ server/src/com/cloud/vm/UserVmManagerImpl.java | 6 +++ server/src/com/cloud/vm/VirtualMachineGuru.java | 2 + .../com/cloud/vm/VirtualMachineManagerImpl.java | 11 ++++- .../test/com/cloud/vm/MockUserVmManagerImpl.java | 6 +++ 11 files changed, 94 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/8b7b838c/plugins/network-elements/elastic-loadbalancer/src/com/cloud/network/lb/ElasticLoadBalancerManagerImpl.java ---------------------------------------------------------------------- diff --git a/plugins/network-elements/elastic-loadbalancer/src/com/cloud/network/lb/ElasticLoadBalancerManagerImpl.java b/plugins/network-elements/elastic-loadbalancer/src/com/cloud/network/lb/ElasticLoadBalancerManagerImpl.java index 3985c51..8d1963d 100644 --- a/plugins/network-elements/elastic-loadbalancer/src/com/cloud/network/lb/ElasticLoadBalancerManagerImpl.java +++ b/plugins/network-elements/elastic-loadbalancer/src/com/cloud/network/lb/ElasticLoadBalancerManagerImpl.java @@ -987,4 +987,13 @@ public class ElasticLoadBalancerManagerImpl implements return VirtualMachineName.getSystemVmId(vmName); } + + + @Override + public boolean recreateNeeded( + VirtualMachineProfile<DomainRouterVO> profile, long hostId, + Commands cmds, ReservationContext context) { + // TODO Auto-generated method stub + return false; + } } http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/8b7b838c/server/src/com/cloud/consoleproxy/AgentBasedConsoleProxyManager.java ---------------------------------------------------------------------- diff --git a/server/src/com/cloud/consoleproxy/AgentBasedConsoleProxyManager.java b/server/src/com/cloud/consoleproxy/AgentBasedConsoleProxyManager.java index d892957..6d20fb2 100755 --- a/server/src/com/cloud/consoleproxy/AgentBasedConsoleProxyManager.java +++ b/server/src/com/cloud/consoleproxy/AgentBasedConsoleProxyManager.java @@ -345,4 +345,12 @@ public class AgentBasedConsoleProxyManager implements ConsoleProxyManager, Virtu @Override public void finalizeExpunge(ConsoleProxyVO proxy) { } + + @Override + public boolean recreateNeeded( + VirtualMachineProfile<ConsoleProxyVO> profile, long hostId, + Commands cmds, ReservationContext context) { + // TODO Auto-generated method stub + return false; + } } http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/8b7b838c/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java ---------------------------------------------------------------------- diff --git a/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java b/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java index 6fef240..ec4f685 100755 --- a/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java +++ b/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java @@ -2006,4 +2006,12 @@ public class ConsoleProxyManagerImpl implements ConsoleProxyManager, ConsoleProx } return _hashKey; } + + @Override + public boolean recreateNeeded( + VirtualMachineProfile<ConsoleProxyVO> profile, long hostId, + Commands cmds, ReservationContext context) { + // TODO Auto-generated method stub + return false; + } } http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/8b7b838c/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java ---------------------------------------------------------------------- diff --git a/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java b/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java index dea4114..766c769 100755 --- a/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java +++ b/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java @@ -2972,4 +2972,17 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian return routerControlIpAddress; } + + @Override + public boolean recreateNeeded( + VirtualMachineProfile<DomainRouterVO> profile, long hostId, + Commands cmds, ReservationContext context) { + //asssume that if failed to ssh into router, meaning router is crashed + CheckSshAnswer answer = (CheckSshAnswer) cmds.getAnswer("checkSsh"); + if (answer == null || !answer.getResult()) { + return true; + } + + return false; + } } http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/8b7b838c/server/src/com/cloud/storage/StorageManager.java ---------------------------------------------------------------------- diff --git a/server/src/com/cloud/storage/StorageManager.java b/server/src/com/cloud/storage/StorageManager.java index 9acc702..67ad97c 100755 --- a/server/src/com/cloud/storage/StorageManager.java +++ b/server/src/com/cloud/storage/StorageManager.java @@ -180,7 +180,7 @@ public interface StorageManager extends StorageService, Manager { void createCapacityEntry(StoragePoolVO storagePool, short capacityType, long allocated); - void prepare(VirtualMachineProfile<? extends VirtualMachine> vm, DeployDestination dest) throws StorageUnavailableException, InsufficientStorageCapacityException, ConcurrentOperationException; + void prepare(VirtualMachineProfile<? extends VirtualMachine> vm, DeployDestination dest, boolean recreate) throws StorageUnavailableException, InsufficientStorageCapacityException, ConcurrentOperationException; void release(VirtualMachineProfile<? extends VMInstanceVO> profile); http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/8b7b838c/server/src/com/cloud/storage/StorageManagerImpl.java ---------------------------------------------------------------------- diff --git a/server/src/com/cloud/storage/StorageManagerImpl.java b/server/src/com/cloud/storage/StorageManagerImpl.java index baea54c..27d8cb7 100755 --- a/server/src/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/com/cloud/storage/StorageManagerImpl.java @@ -3173,7 +3173,7 @@ public class StorageManagerImpl implements StorageManager, Manager, ClusterManag } @Override - public void prepare(VirtualMachineProfile<? extends VirtualMachine> vm, DeployDestination dest) throws StorageUnavailableException, InsufficientStorageCapacityException { + public void prepare(VirtualMachineProfile<? extends VirtualMachine> vm, DeployDestination dest, boolean recreate) throws StorageUnavailableException, InsufficientStorageCapacityException { if (dest == null) { if (s_logger.isDebugEnabled()) { @@ -3193,7 +3193,11 @@ public class StorageManagerImpl implements StorageManager, Manager, ClusterManag if (dest.getStorageForDisks() != null) { assignedPool = dest.getStorageForDisks().get(vol); } - if (assignedPool != null) { + if (assignedPool == null && recreate) { + assignedPool = _storagePoolDao.findById(vol.getPoolId()); + + } + if (assignedPool != null || recreate) { Volume.State state = vol.getState(); if (state == Volume.State.Allocated || state == Volume.State.Creating) { recreateVols.add(vol); @@ -3231,6 +3235,12 @@ public class StorageManagerImpl implements StorageManager, Manager, ClusterManag for (VolumeVO vol : recreateVols) { VolumeVO newVol; + StoragePool existingPool = null; + if (recreate && (dest.getStorageForDisks() == null || dest.getStorageForDisks().get(vol) == null)) { + existingPool = _storagePoolDao.findById(vol.getPoolId()); + s_logger.debug("existing pool: " + existingPool.getId()); + } + if (vol.getState() == Volume.State.Allocated || vol.getState() == Volume.State.Creating) { newVol = vol; } else { @@ -3251,7 +3261,9 @@ public class StorageManagerImpl implements StorageManager, Manager, ClusterManag } catch (NoTransitionException e) { throw new CloudRuntimeException("Unable to create " + e.toString()); } - Pair<VolumeTO, StoragePool> created = createVolume(newVol, _diskOfferingDao.findById(newVol.getDiskOfferingId()), vm, vols, dest); + + Pair<VolumeTO, StoragePool> created = createVolume(newVol, _diskOfferingDao.findById(newVol.getDiskOfferingId()), vm, vols, dest, existingPool); + if (created == null) { Long poolId = newVol.getPoolId(); newVol.setPoolId(null); @@ -3307,7 +3319,7 @@ public class StorageManagerImpl implements StorageManager, Manager, ClusterManag } public Pair<VolumeTO, StoragePool> createVolume(VolumeVO toBeCreated, DiskOfferingVO offering, VirtualMachineProfile<? extends VirtualMachine> vm, List<? extends Volume> alreadyCreated, - DeployDestination dest) throws StorageUnavailableException { + DeployDestination dest, StoragePool sPool) throws StorageUnavailableException { if (s_logger.isDebugEnabled()) { s_logger.debug("Creating volume: " + toBeCreated); } @@ -3317,9 +3329,15 @@ public class StorageManagerImpl implements StorageManager, Manager, ClusterManag if (toBeCreated.getTemplateId() != null) { template = _templateDao.findById(toBeCreated.getTemplateId()); } + + StoragePool pool = null; + if (sPool != null) { + pool = sPool; + } else { + pool = dest.getStorageForDisks().get(toBeCreated); + } - if (dest.getStorageForDisks() != null) { - StoragePool pool = dest.getStorageForDisks().get(toBeCreated); + if (pool != null) { if (s_logger.isDebugEnabled()) { s_logger.debug("Trying to create in " + pool); } http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/8b7b838c/server/src/com/cloud/storage/secondary/SecondaryStorageManagerImpl.java ---------------------------------------------------------------------- diff --git a/server/src/com/cloud/storage/secondary/SecondaryStorageManagerImpl.java b/server/src/com/cloud/storage/secondary/SecondaryStorageManagerImpl.java index bb7fa02..1121bb1 100755 --- a/server/src/com/cloud/storage/secondary/SecondaryStorageManagerImpl.java +++ b/server/src/com/cloud/storage/secondary/SecondaryStorageManagerImpl.java @@ -1450,4 +1450,12 @@ public class SecondaryStorageManagerImpl implements SecondaryStorageVmManager, V } return null; } + + @Override + public boolean recreateNeeded( + VirtualMachineProfile<SecondaryStorageVmVO> profile, long hostId, + Commands cmds, ReservationContext context) { + // TODO Auto-generated method stub + return false; + } } http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/8b7b838c/server/src/com/cloud/vm/UserVmManagerImpl.java ---------------------------------------------------------------------- diff --git a/server/src/com/cloud/vm/UserVmManagerImpl.java b/server/src/com/cloud/vm/UserVmManagerImpl.java index 635a943..649e215 100755 --- a/server/src/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/com/cloud/vm/UserVmManagerImpl.java @@ -3534,6 +3534,12 @@ public class UserVmManagerImpl implements UserVmManager, UserVmService, Manager s_logger.info("AssignVM: vm " + vm.getInstanceName() + " now belongs to account " + cmd.getAccountName()); return vm; } + @Override + public boolean recreateNeeded(VirtualMachineProfile<UserVmVO> profile, + long hostId, Commands cmds, ReservationContext context) { + // TODO Auto-generated method stub + return false; + } @Override http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/8b7b838c/server/src/com/cloud/vm/VirtualMachineGuru.java ---------------------------------------------------------------------- diff --git a/server/src/com/cloud/vm/VirtualMachineGuru.java b/server/src/com/cloud/vm/VirtualMachineGuru.java index d779485..b92982c 100644 --- a/server/src/com/cloud/vm/VirtualMachineGuru.java +++ b/server/src/com/cloud/vm/VirtualMachineGuru.java @@ -63,6 +63,8 @@ public interface VirtualMachineGuru<T extends VirtualMachine> { void finalizeExpunge(T vm); + boolean recreateNeeded(VirtualMachineProfile<T> profile, long hostId, Commands cmds, ReservationContext context); + /** * Returns the id parsed from the name. If it cannot parse the name, * then return null. This method is used to determine if this is http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/8b7b838c/server/src/com/cloud/vm/VirtualMachineManagerImpl.java ---------------------------------------------------------------------- diff --git a/server/src/com/cloud/vm/VirtualMachineManagerImpl.java b/server/src/com/cloud/vm/VirtualMachineManagerImpl.java index 561e5b2..ac8fd2a 100755 --- a/server/src/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/server/src/com/cloud/vm/VirtualMachineManagerImpl.java @@ -651,6 +651,7 @@ public class VirtualMachineManagerImpl implements VirtualMachineManager, Listene DataCenterDeployment originalPlan = plan; int retry = _retry; + boolean recreate = false; while (retry-- != 0) { // It's != so that it can match -1. if(reuseVolume){ @@ -745,7 +746,8 @@ public class VirtualMachineManagerImpl implements VirtualMachineManager, Listene } _networkMgr.prepare(vmProfile, dest, ctx); if (vm.getHypervisorType() != HypervisorType.BareMetal) { - _storageMgr.prepare(vmProfile, dest); + _storageMgr.prepare(vmProfile, dest, recreate); + recreate = false; } //since StorageMgr succeeded in volume creation, reuse Volume for further tries until current cluster has capacity if(!reuseVolume){ @@ -798,6 +800,7 @@ public class VirtualMachineManagerImpl implements VirtualMachineManager, Listene if (s_logger.isDebugEnabled()) { s_logger.info("The guru did not like the answers so stopping " + vm); } + StopCommand cmd = new StopCommand(vm.getInstanceName()); StopAnswer answer = (StopAnswer) _agentMgr.easySend(destHostId, cmd); if (answer == null || !answer.getResult()) { @@ -805,7 +808,11 @@ public class VirtualMachineManagerImpl implements VirtualMachineManager, Listene _haMgr.scheduleStop(vm, destHostId, WorkType.ForceStop); throw new ExecutionException("Unable to stop " + vm + " so we are unable to retry the start operation"); } - throw new ExecutionException("Unable to start " + vm + " due to error in finalizeStart, not retrying"); + if (vmGuru.recreateNeeded(vmProfile, destHostId, cmds, ctx)) { + recreate = true; + } else { + throw new ExecutionException("Unable to start " + vm + " due to error in finalizeStart, not retrying"); + } } } s_logger.info("Unable to start VM on " + dest.getHost() + " due to " + (startAnswer == null ? " no start answer" : startAnswer.getDetails())); http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/8b7b838c/server/test/com/cloud/vm/MockUserVmManagerImpl.java ---------------------------------------------------------------------- diff --git a/server/test/com/cloud/vm/MockUserVmManagerImpl.java b/server/test/com/cloud/vm/MockUserVmManagerImpl.java index 5241eb2..31065fe 100644 --- a/server/test/com/cloud/vm/MockUserVmManagerImpl.java +++ b/server/test/com/cloud/vm/MockUserVmManagerImpl.java @@ -290,6 +290,12 @@ public class MockUserVmManagerImpl implements UserVmManager, UserVmService, Mana } + @Override + public boolean recreateNeeded(VirtualMachineProfile<UserVmVO> profile, + long hostId, Commands cmds, ReservationContext context) { + // TODO Auto-generated method stub + return false; + } @Override public UserVm startVirtualMachine(DeployVMCmd cmd) throws InsufficientCapacityException, ConcurrentOperationException, ResourceUnavailableException {
