Hello,
Any help or pointers would be greatly appreciated.
After upgrading CS Virtual routers to 4.15.1 The VM does actually get
built and powers on (visible via console proxy), and then is suddenly
powered off. and this error is displayed in the interface
'(r-340-VM) Resource [Host:10] is unreachable: Host 10: Unable to start
instance due to Unable to start VM:f308d9b5-632c-47d2-9b24-01f02bb257a5 due
to error in finalizeStart, not retrying'
Looking through the logs I cannot find anything other than the below snip
that seem interesting:
:"java.lang.NoSuchMethodError: 'boolean
com.cloud.agent.api.to.IpAddressTO.isPrivateGateway()'
Here are the full logs:
2021-07-15 19:45:43,250 DEBUG [c.c.a.t.Request]
(AgentManager-Handler-5:null) (logid:) Seq 10-6995779071166649120:
Processing: { Ans: , MgmtId: 345050527765, via: 10, Ver: v1, Flags:
10000110,
[{"com.cloud.agent.api.StartAnswer":{"vm":{"id":"340","name":"r-340-VM","state":"Starting","type":"DomainRouter","cpus":"1","minSpeed":"500","maxSpeed":"500","minRam":"(256.00
MB) 268435456","maxRam":"(256.00 MB)
268435456","arch":"x86_64","os":"Debian GNU/Linux 9
(64-bit)","platformEmulator":"Debian GNU/Linux 9 (64-bit)","bootArgs":"
vpccidr=10.0.0.0/16 domain=cs9cloud.internal dns1=81.19.54.209 dns2=
template=domP name=r-340-VM eth0ip=169.254.146.220
eth0mask=255.2","enableHA":"true","limitCpuUse":"false","enableDynamicallyScaleVm":"false","vncPassword":"M_cgfURA4I7i0BNoy2pmYw","vncAddr":"10.100.6.201","params":{},"uuid":"f308d9b5-632c-47d2-9b24-01f02bb257a5","enterHardwareSetup":"false","disks":[{"data":{"org.apache.cloudstack.storage.to.VolumeObjectTO":{"uuid":"8a7e0ad0-69b1-46ed-a75c-e984b836a989","volumeType":"ROOT","dataStore":{"org.apache.cloudstack.storage.to.PrimaryDataStoreTO":{"uuid":"658c4937-b7bc-3aa7-a0d3-aa144224dd52","id":"7","poolType":"RBD","host":"ceph-mon.cloudstack","path":"cloudstack2019","port":"6789","url":"RBD://ceph-mon.cloudstack/cloudstack2019/?ROLE=Primary&STOREUUID=658c4937-b7bc-3aa7-a0d3-aa144224dd52","isManaged":"false"}},"name":"ROOT-340","size":"(2.44
GB)
2621440000","path":"8a7e0ad0-69b1-46ed-a75c-e984b836a989","volumeId":"468","vmName":"r-340-VM","accountId":"9","format":"RAW","provisioningType":"THIN","id":"468","deviceId":"0","bytesReadRate":"(0
bytes) 0","bytesWriteRate":"(0 bytes) 0","iopsReadRate":"(0 bytes)
0","iopsWriteRate":"(0 bytes)
0","hypervisorType":"KVM","directDownload":"false","deployAsIs":"false"}},"diskSeq":"0","path":"8a7e0ad0-69b1-46ed-a75c-e984b836a989","type":"ROOT","_details":{"storageHost":"ceph-mon.cloudstack","managed":"false","storagePort":"6789","volumeSize":"(2.44
GB)
2621440000"}}],"nics":[{"deviceId":"0","networkRateMbps":"-1","defaultNic":"false","pxeDisable":"true","nicUuid":"efeacfd4-95d1-4cdb-897d-a2f79bc25782","details":{"PromiscuousMode":"false","MacAddressChanges":"true","ForgedTransmits":"true"},"dpdkEnabled":"false","uuid":"bae5a165-83f2-4134-8ae0-6dc7a149a32a","ip":"169.254.146.220","netmask":"255.255.0.0","gateway":"169.254.0.1","mac":"0e:00:a9:fe:92:dc","broadcastType":"LinkLocal","type":"Control","isSecurityGroupEnabled":"false"}],"guestOsDetails":{},"extraConfig":{}},"result":"true","wait":"0","bypassHostMaintenance":"false"}},{"com.cloud.agent.api.check.CheckSshAnswer":{"result":"true","wait":"0","bypassHostMaintenance":"false"}},{"com.cloud.agent.api.GetDomRVersionAnswer":{"templateVersion":"Cloudstack
Release 4.15.1 Wed 10 Mar 2021 05:38:45 AM
UTC","scriptsVersion":"ac3f3efc5ffe5dbab9616395c32e0d3d
","result":"true","details":"Cloudstack Release 4.15.1 Wed 10 Mar 2021
05:38:45 AM UTC&ac3f3efc5ffe5dbab9616395c32e0d3d
","wait":"0","bypassHostMaintenance":"false"}},{"com.cloud.agent.api.PlugNicAnswer":{"result":"true","details":"success","wait":"0","bypassHostMaintenance":"false"}},{"com.cloud.agent.api.Answer":{"result":"false","details"
at
com.cloud.agent.resource.virtualnetwork.facade.IpAssociationConfigItem.generateConfig(IpAssociationConfigItem.java:45)
at
com.cloud.agent.resource.virtualnetwork.VirtualRoutingResource.generateCommandCfg(VirtualRoutingResource.java:489)
at
com.cloud.agent.resource.virtualnetwork.VirtualRoutingResource.executeRequest(VirtualRoutingResource.java:142)
at
com.cloud.hypervisor.kvm.resource.wrapper.LibvirtNetworkElementCommandWrapper.execute(LibvirtNetworkElementCommandWrapper.java:35)
at
com.cloud.hypervisor.kvm.resource.wrapper.LibvirtNetworkElementCommandWrapper.execute(LibvirtNetworkElementCommandWrapper.java:29)
at
com.cloud.hypervisor.kvm.resource.wrapper.LibvirtRequestWrapper.execute(LibvirtRequestWrapper.java:78)
at
com.cloud.hypervisor.kvm.resource.LibvirtComputingResource.executeRequest(LibvirtComputingResource.java:1476)
at com.cloud.agent.Agent.processRequest(Agent.java:661)
at com.cloud.agent.Agent$AgentRequestHandler.doTask(Agent.java:1079)
at com.cloud.utils.nio.Task.call(Task.java:83)
at com.cloud.utils.nio.Task.call(Task.java:29)
at
java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base/java.lang.Thread.run(Thread.java:834)
","wait":"0","bypassHostMaintenance":"false"}},{"com.cloud.agent.api.Answer":{"result":"false","details":"Stopped
by previous
failure","wait":"0","bypassHostMaintenance":"false"}},{"com.cloud.agent.api.Answer":{"result":"false","details":"Stopped
by previous
failure","wait":"0","bypassHostMaintenance":"false"}},{"com.cloud.agent.api.Answer":{"result":"false","details":"Stopped
by previous
failure","wait":"0","bypassHostMaintenance":"false"}},{"com.cloud.agent.api.Answer":{"result":"false","details":"Stopped
by previous
failure","wait":"0","bypassHostMaintenance":"false"}},{"com.cloud.agent.api.Answer":{"result":"false","details":"Stopped
by previous
failure","wait":"0","bypassHostMaintenance":"false"}},{"com.cloud.agent.api.Answer":{"result":"false","details":"Stopped
by previous
failure","wait":"0","bypassHostMaintenance":"false"}},{"com.cloud.agent.api.Answer":{"result":"false","details":"Stopped
by previous
failure","wait":"0","bypassHostMaintenance":"false"}},{"com.cloud.agent.api.Answer":{"result":"false","details":"Stopped
by previous
failure","wait":"0","bypassHostMaintenance":"false"}},{"com.cloud.agent.api.Answer":{"result":"false","details":"Stopped
by previous
failure","wait":"0","bypassHostMaintenance":"false"}},{"com.cloud.agent.api.Answer":{"result":"false","details":"Stopped
by previous
failure","wait":"0","bypassHostMaintenance":"false"}},{"com.cloud.agent.api.Answer":{"result":"false","details":"Stopped
by previous
failure","wait":"0","bypassHostMaintenance":"false"}},{"com.cloud.agent.api.Answer":{"result":"false","details":"Stopped
by previous
failure","wait":"0","bypassHostMaintenance":"false"}},{"com.cloud.agent.api.Answer":{"result":"false","details":"Stopped
by previous
failure","wait":"0","bypassHostMaintenance":"false"}},{"com.cloud.agent.api.Answer":{"result":"false","details":"Stopped
by previous
failure","wait":"0","bypassHostMaintenance":"false"}},{"com.cloud.agent.api.Answer":{"result":"false","details":"Stopped
by previous
failure","wait":"0","bypassHostMaintenance":"false"}},{"com.cloud.agent.api.Answer":{"result":"false","details":"Stopped
by previous
failure","wait":"0","bypassHostMaintenance":"false"}},{"com.cloud.agent.api.Answer":{"result":"false","details":"Stopped
by previous
failure","wait":"0","bypassHostMaintenance":"false"}},{"com.cloud.agent.api.Answer":{"result":"false","details":"Stopped
by previous
failure","wait":"0","bypassHostMaintenance":"false"}},{"com.cloud.agent.api.Answer":{"result":"false","details":"Stopped
by previous
failure","wait":"0","bypassHostMaintenance":"false"}},{"com.cloud.agent.api.Answer":{"result":"false","details":"Stopped
by previous
failure","wait":"0","bypassHostMaintenance":"false"}},{"com.cloud.agent.api.Answer":{"result":"false","details":"Stopped
by previous
failure","wait":"0","bypassHostMaintenance":"false"}},{"com.cloud.agent.api.Answer":{"result":"false","details":"Stopped
by previous
failure","wait":"0","bypassHostMaintenance":"false"}},{"com.cloud.agent.api.Answer":{"result":"false","details":"Stopped
by previous
failure","wait":"0","bypassHostMaintenance":"false"}},{"com.cloud.agent.api.Answer":{"result":"false","details":"Stopped
by previous
failure","wait":"0","bypassHostMaintenance":"false"}},{"com.cloud.agent.api.Answer":{"result":"false","details":"Stopped
by previous
failure","wait":"0","bypassHostMaintenance":"false"}},{"com.cloud.agent.api.Answer":{"result":"false","details":"Stopped
by previous
failure","wait":"0","bypassHostMaintenance":"false"}},{"com.cloud.agent.api.Answer":{"result":"false","details":"Stopped
by previous
failure","wait":"0","bypassHostMaintenance":"false"}},{"com.cloud.agent.api.Answer":{"result":"false","details":"Stopped
by previous
failure","wait":"0","bypassHostMaintenance":"false"}},{"com.cloud.agent.api.Answer":{"result":"false","details":"Stopped
by previous
failure","wait":"0","bypassHostMaintenance":"false"}},{"com.cloud.agent.api.Answer":{"result":"false","details":"Stopped
by previous
failure","wait":"0","bypassHostMaintenance":"false"}},{"com.cloud.agent.api.Answer":{"result":"false","details":"Stopped
by previous
failure","wait":"0","bypassHostMaintenance":"false"}},{"com.cloud.agent.api.Answer":{"result":"false","details":"Stopped
by previous
failure","wait":"0","bypassHostMaintenance":"false"}},{"com.cloud.agent.api.Answer":{"result":"false","details":"Stopped
by previous
failure","wait":"0","bypassHostMaintenance":"false"}},{"com.cloud.agent.api.Answer":{"result":"false","details":"Stopped
by previous
failure","wait":"0","bypassHostMaintenance":"false"}},{"com.cloud.agent.api.Answer":{"result":"false","details":"Stopped
by previous
failure","wait":"0","bypassHostMaintenance":"false"}},{"com.cloud.agent.api.Answer":{"result":"false","details":"Stopped
by previous
failure","wait":"0","bypassHostMaintenance":"false"}},{"com.cloud.agent.api.Answer":{"result":"false","details":"Stopped
by previous
failure","wait":"0","bypassHostMaintenance":"false"}},{"com.cloud.agent.api.Answer":{"result":"false","details":"Stopped
by previous
failure","wait":"0","bypassHostMaintenance":"false"}},{"com.cloud.agent.api.Answer":{"result":"false","details":"Stopped
by previous
failure","wait":"0","bypassHostMaintenance":"false"}},{"com.cloud.agent.api.Answer":{"result":"false","details":"Stopped
by previous
failure","wait":"0","bypassHostMaintenance":"false"}},{"com.cloud.agent.api.Answer":{"result":"false","details":"Stopped
by previous failure","wait":"0","bypassHostMaintenance":"false"}}] }
2021-07-15 19:45:43,250 DEBUG [c.c.a.m.AgentAttache]
(AgentManager-Handler-5:null) (logid:) Seq 10-6995779071166649120: No more
commands found
2021-07-15 19:45:43,251 DEBUG [c.c.a.t.Request]
(Work-Job-Executor-62:ctx-03a2fc9a job-6737/job-6738 ctx-6355d615)
(logid:1dbb1d9a) Seq 10-6995779071166649120: Received: { Ans: , MgmtId:
345050527765, via: 10(chst01.vxtream.com), Ver: v1, Flags: 110, {
StartAnswer, CheckSshAnswer, GetDomRVersionAnswer, PlugNicAnswer, Answer,
Answer, Answer, Answer, Answer, Answer, Answer, Answer, Answer, Answer,
Answer, Answer, Answer, Answer, Answer, Answer, Answer, Answer, Answer,
Answer, Answer, Answer, Answer, Answer, Answer, Answer, Answer, Answer,
Answer, Answer, Answer, Answer, Answer, Answer, Answer, Answer, Answer,
Answer, Answer, Answer, Answer, Answer } }
2021-07-15 19:45:43,256 WARN [c.c.a.AlertManagerImpl]
(Work-Job-Executor-62:ctx-03a2fc9a job-6737/job-6738 ctx-6355d615)
(logid:1dbb1d9a) AlertType:: 9 | dataCenterId:: 1 | podId:: 1 | clusterId::
null | message:: Command: com.cloud.agent.api.Command failed while starting
virtual router
2021-07-15 19:45:43,261 ERROR [c.c.n.r.VirtualNetworkApplianceManagerImpl]
(Work-Job-Executor-62:ctx-03a2fc9a job-6737/job-6738 ctx-6355d615)
(logid:1dbb1d9a) java.lang.NoSuchMethodError: 'boolean
com.cloud.agent.api.to.IpAddressTO.isPrivateGateway()'
at
com.cloud.agent.resource.virtualnetwork.facade.IpAssociationConfigItem.generateConfig(IpAssociationConfigItem.java:45)
at
com.cloud.agent.resource.virtualnetwork.VirtualRoutingResource.generateCommandCfg(VirtualRoutingResource.java:489)
at
com.cloud.agent.resource.virtualnetwork.VirtualRoutingResource.executeRequest(VirtualRoutingResource.java:142)
at
com.cloud.hypervisor.kvm.resource.wrapper.LibvirtNetworkElementCommandWrapper.execute(LibvirtNetworkElementCommandWrapper.java:35)
at
com.cloud.hypervisor.kvm.resource.wrapper.LibvirtNetworkElementCommandWrapper.execute(LibvirtNetworkElementCommandWrapper.java:29)
at
com.cloud.hypervisor.kvm.resource.wrapper.LibvirtRequestWrapper.execute(LibvirtRequestWrapper.java:78)
at
com.cloud.hypervisor.kvm.resource.LibvirtComputingResource.executeRequest(LibvirtComputingResource.java:1476)
at com.cloud.agent.Agent.processRequest(Agent.java:661)
at com.cloud.agent.Agent$AgentRequestHandler.doTask(Agent.java:1079)
at com.cloud.utils.nio.Task.call(Task.java:83)
at com.cloud.utils.nio.Task.call(Task.java:29)
at
java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base/java.lang.Thread.run(Thread.java:834)
2021-07-15 19:45:43,261 WARN [c.c.n.r.VirtualNetworkApplianceManagerImpl]
(Work-Job-Executor-62:ctx-03a2fc9a job-6737/job-6738 ctx-6355d615)
(logid:1dbb1d9a) Command: com.cloud.agent.api.Command failed while starting
virtual router
2021-07-15 19:45:43,261 INFO [c.c.v.VirtualMachineManagerImpl]
(Work-Job-Executor-62:ctx-03a2fc9a job-6737/job-6738 ctx-6355d615)
(logid:1dbb1d9a) The guru did not like the answers so stopping
VM[DomainRouter|r-340-VM]
2021-07-15 19:45:43,276 DEBUG [c.c.a.t.Request]
(Work-Job-Executor-62:ctx-03a2fc9a job-6737/job-6738 ctx-6355d615)
(logid:1dbb1d9a) Seq 10-6995779071166649139: Sending { Cmd , MgmtId:
345050527765, via: 10(chst01.vxtream.com), Ver: v1, Flags: 100011,
[{"com.cloud.agent.api.StopCommand":{"isProxy":"false","checkBeforeCleanup":"false","controlIp":"169.254.48.102","forceStop":"false","volumesToDisconnect":[],"vmName":"r-340-VM","executeInSequence":"false","wait":"0","bypassHostMaintenance":"false"}}]
}
2021-07-15 19:45:44,113 DEBUG [c.c.a.ApiServlet]
(qtp1620890840-526:ctx-bb91cb6a) (logid:f057b73d) ===START===
188.31.54.242 -- GET
jobId=1dbb1d9a-be80-4909-bd8c-67c2c07cb61d&command=queryAsyncJobResult&response=json
2021-07-15 19:45:44,114 DEBUG [c.c.a.ApiServlet]
(qtp1620890840-581:ctx-7913c7a8) (logid:1cf14db0) ===START===
188.31.54.242 -- GET
jobid=1dbb1d9a-be80-4909-bd8c-67c2c07cb61d&command=queryAsyncJobResult&response=json
2021-07-15 19:45:44,127 DEBUG [c.c.a.ApiServer]
(qtp1620890840-581:ctx-7913c7a8 ctx-d2d0d4c3) (logid:1cf14db0) CIDRs from
which account 'Acct[400de9e2-ee6a-11e9-8f54-0050569d3815-admin]' is allowed
to perform API calls: 0.0.0.0/0,::/0
2021-07-15 19:45:44,128 DEBUG [c.c.a.ApiServer]
(qtp1620890840-526:ctx-bb91cb6a ctx-270980a8) (logid:f057b73d) CIDRs from
which account 'Acct[400de9e2-ee6a-11e9-8f54-0050569d3815-admin]' is allowed
to perform API calls: 0.0.0.0/0,::/0
2021-07-15 19:45:44,150 DEBUG [c.c.a.ApiServlet]
(qtp1620890840-526:ctx-bb91cb6a ctx-270980a8) (logid:f057b73d) ===END===
188.31.54.242 -- GET
jobId=1dbb1d9a-be80-4909-bd8c-67c2c07cb61d&command=queryAsyncJobResult&response=json
2021-07-15 19:45:44,152 DEBUG [c.c.a.ApiServlet]
(qtp1620890840-581:ctx-7913c7a8 ctx-d2d0d4c3) (logid:1cf14db0) ===END===
188.31.54.242 -- GET
jobid=1dbb1d9a-be80-4909-bd8c-67c2c07cb61d&command=queryAsyncJobResult&response=json
2021-07-15 19:45:45,487 DEBUG [c.c.a.t.Request]
(AgentManager-Handler-2:null) (logid:) Seq 10-6995779071166649139:
Processing: { Ans: , MgmtId: 345050527765, via: 10, Ver: v1, Flags: 10,
[{"com.cloud.agent.api.StopAnswer":{"result":"true","wait":"0","bypassHostMaintenance":"false"}}]
}
2021-07-15 19:45:45,487 DEBUG [c.c.a.t.Request]
(Work-Job-Executor-62:ctx-03a2fc9a job-6737/job-6738 ctx-6355d615)
(logid:1dbb1d9a) Seq 10-6995779071166649139: Received: { Ans: , MgmtId:
345050527765, via: 10(chst01.vxtream.com), Ver: v1, Flags: 10, { StopAnswer
} }
2021-07-15 19:45:45,487 ERROR [c.c.v.VirtualMachineManagerImpl]
(Work-Job-Executor-62:ctx-03a2fc9a job-6737/job-6738 ctx-6355d615)
(logid:1dbb1d9a) Failed to start instance VM[DomainRouter|r-340-VM]
com.cloud.utils.exception.ExecutionException: Unable to start
VM:f308d9b5-632c-47d2-9b24-01f02bb257a5 due to error in finalizeStart, not
retrying
at
com.cloud.vm.VirtualMachineManagerImpl.orchestrateStart(VirtualMachineManagerImpl.java:1275)
at
com.cloud.vm.VirtualMachineManagerImpl.orchestrateStart(VirtualMachineManagerImpl.java:5502)
at jdk.internal.reflect.GeneratedMethodAccessor292.invoke(Unknown
Source)
at
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:566)
at
com.cloud.vm.VmWorkJobHandlerProxy.handleVmWorkJob(VmWorkJobHandlerProxy.java:107)
at
com.cloud.vm.VirtualMachineManagerImpl.handleVmWorkJob(VirtualMachineManagerImpl.java:5669)
at
com.cloud.vm.VmWorkJobDispatcher.runJob(VmWorkJobDispatcher.java:102)
at
org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.runInContext(AsyncJobManagerImpl.java:620)
at
org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(ManagedContextRunnable.java:48)
at
org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:55)
at
org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:102)
at
org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:52)
at
org.apache.cloudstack.managed.context.ManagedContextRunnable.run(ManagedContextRunnable.java:45)
at
org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.run(AsyncJobManagerImpl.java:568)
at
java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
at
java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base/java.lang.Thread.run(Thread.java:834)
