canghai908 commented on issue #6463:
URL: https://github.com/apache/cloudstack/issues/6463#issuecomment-1214177323
@rohityadavcloud
this is cloudstack-agent logs
```
2022-08-13 04:28:47,358 WARN [cloud.agent.Agent]
(agentRequest-Handler-1:null) (logid:ede576d7) Caught:
com.cloud.utils.exception.CloudRuntimeException: Failed to create storage
pool: a67e3adf-8acf-3ec8-9342-634bd09dc951
at
com.cloud.hypervisor.kvm.storage.LibvirtStorageAdaptor.createStoragePool(LibvirtStorageAdaptor.java:672)
at
com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager.createStoragePool(KVMStoragePoolManager.java:350)
at
com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager.createStoragePool(KVMStoragePoolManager.java:344)
at
com.cloud.hypervisor.kvm.resource.wrapper.LibvirtModifyStoragePoolCommandWrapper.execute(LibvirtModifyStoragePoolCommandWrapper.java:42)
at
com.cloud.hypervisor.kvm.resource.wrapper.LibvirtModifyStoragePoolCommandWrapper.execute(LibvirtModifyStoragePoolCommandWrapper.java:35)
at
com.cloud.hypervisor.kvm.resource.wrapper.LibvirtRequestWrapper.execute(LibvirtRequestWrapper.java:78)
at
com.cloud.hypervisor.kvm.resource.LibvirtComputingResource.executeRequest(LibvirtComputingResource.java:1830)
at com.cloud.agent.Agent.processRequest(Agent.java:660)
at com.cloud.agent.Agent$AgentRequestHandler.doTask(Agent.java:1080)
at com.cloud.utils.nio.Task.call(Task.java:83)
at com.cloud.utils.nio.Task.call(Task.java:29)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base/java.lang.Thread.run(Thread.java:829)
2022-08-13 04:32:10,136 INFO [kvm.storage.LibvirtStorageAdaptor]
(agentRequest-Handler-2:null) (logid:0bf60b72) Attempting to create storage
pool a67e3adf-8acf-3ec8-9342-634bd09dc951 (RBD) in libvirt
2022-08-13 04:32:10,138 WARN [kvm.storage.LibvirtStorageAdaptor]
(agentRequest-Handler-2:null) (logid:0bf60b72) Storage pool
a67e3adf-8acf-3ec8-9342-634bd09dc951 was not found running in libvirt. Need to
create it.
2022-08-13 04:32:10,138 INFO [kvm.storage.LibvirtStorageAdaptor]
(agentRequest-Handler-2:null) (logid:0bf60b72) Didn't find an existing storage
pool a67e3adf-8acf-3ec8-9342-634bd09dc951 by UUID, checking for pools with
duplicate paths
2022-08-13 04:32:10,206 ERROR [kvm.storage.LibvirtStorageAdaptor]
(agentRequest-Handler-2:null) (logid:0bf60b72) Failed to create RBD storage
pool: org.libvirt.LibvirtException: failed to connect to the RADOS monitor on:
rbd.cloud.priv,: Operation not supported
2022-08-13 04:32:10,206 ERROR [kvm.storage.LibvirtStorageAdaptor]
(agentRequest-Handler-2:null) (logid:0bf60b72) Failed to create the RBD storage
pool, cleaning up the libvirt secret
2022-08-13 04:32:10,207 WARN [cloud.agent.Agent]
(agentRequest-Handler-2:null) (logid:0bf60b72) Caught:
com.cloud.utils.exception.CloudRuntimeException: Failed to create storage
pool: a67e3adf-8acf-3ec8-9342-634bd09dc951
at
com.cloud.hypervisor.kvm.storage.LibvirtStorageAdaptor.createStoragePool(LibvirtStorageAdaptor.java:672)
at
com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager.createStoragePool(KVMStoragePoolManager.java:350)
at
com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager.createStoragePool(KVMStoragePoolManager.java:344)
at
com.cloud.hypervisor.kvm.resource.wrapper.LibvirtModifyStoragePoolCommandWrapper.execute(LibvirtModifyStoragePoolCommandWrapper.java:42)
at
com.cloud.hypervisor.kvm.resource.wrapper.LibvirtModifyStoragePoolCommandWrapper.execute(LibvirtModifyStoragePoolCommandWrapper.java:35)
at
com.cloud.hypervisor.kvm.resource.wrapper.LibvirtRequestWrapper.execute(LibvirtRequestWrapper.java:78)
at
com.cloud.hypervisor.kvm.resource.LibvirtComputingResource.executeRequest(LibvirtComputingResource.java:1830)
at com.cloud.agent.Agent.processRequest(Agent.java:660)
at com.cloud.agent.Agent$AgentRequestHandler.doTask(Agent.java:1080)
at com.cloud.utils.nio.Task.call(Task.java:83)
at com.cloud.utils.nio.Task.call(Task.java:29)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base/java.lang.Thread.run(Thread.java:829)
2022-08-13 04:32:12,114 INFO [kvm.storage.LibvirtStorageAdaptor]
(agentRequest-Handler-3:null) (logid:0d441e25) Attempting to create storage
pool a67e3adf-8acf-3ec8-9342-634bd09dc951 (RBD) in libvirt
2022-08-13 04:32:12,115 WARN [kvm.storage.LibvirtStorageAdaptor]
(agentRequest-Handler-3:null) (logid:0d441e25) Storage pool
a67e3adf-8acf-3ec8-9342-634bd09dc951 was not found running in libvirt. Need to
create it.
2022-08-13 04:32:12,115 INFO [kvm.storage.LibvirtStorageAdaptor]
(agentRequest-Handler-3:null) (logid:0d441e25) Didn't find an existing storage
pool a67e3adf-8acf-3ec8-9342-634bd09dc951 by UUID, checking for pools with
duplicate paths
2022-08-13 04:32:12,183 ERROR [kvm.storage.LibvirtStorageAdaptor]
(agentRequest-Handler-3:null) (logid:0d441e25) Failed to create RBD storage
pool: org.libvirt.LibvirtException: failed to connect to the RADOS monitor on:
rbd.cloud.priv,: Operation not supported
2022-08-13 04:32:12,184 ERROR [kvm.storage.LibvirtStorageAdaptor]
(agentRequest-Handler-3:null) (logid:0d441e25) Failed to create the RBD storage
pool, cleaning up the libvirt secret
2022-08-13 04:32:12,184 WARN [cloud.agent.Agent]
(agentRequest-Handler-3:null) (logid:0d441e25) Caught:
com.cloud.utils.exception.CloudRuntimeException: Failed to create storage
pool: a67e3adf-8acf-3ec8-9342-634bd09dc951
at
com.cloud.hypervisor.kvm.storage.LibvirtStorageAdaptor.createStoragePool(LibvirtStorageAdaptor.java:672)
at
com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager.createStoragePool(KVMStoragePoolManager.java:350)
at
com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager.createStoragePool(KVMStoragePoolManager.java:344)
at
com.cloud.hypervisor.kvm.resource.wrapper.LibvirtModifyStoragePoolCommandWrapper.execute(LibvirtModifyStoragePoolCommandWrapper.java:42)
at
com.cloud.hypervisor.kvm.resource.wrapper.LibvirtModifyStoragePoolCommandWrapper.execute(LibvirtModifyStoragePoolCommandWrapper.java:35)
at
com.cloud.hypervisor.kvm.resource.wrapper.LibvirtRequestWrapper.execute(LibvirtRequestWrapper.java:78)
at
com.cloud.hypervisor.kvm.resource.LibvirtComputingResource.executeRequest(LibvirtComputingResource.java:1830)=
```
mgmt server log
```
2022-08-13 04:32:10,225 DEBUG [c.c.s.StorageManagerImpl]
(qtp1462044018-366:ctx-3e5ccaa7 ctx-e2a6c0d8) (logid:0bf60b72) Failed to clean
up storage pool: null
2022-08-13 04:32:10,225 INFO [c.c.a.ApiServer]
(qtp1462044018-366:ctx-3e5ccaa7 ctx-e2a6c0d8) (logid:0bf60b72) Failed to add
data store: Failed to access storage pool
2022-08-13 04:32:10,226 DEBUG [c.c.a.ApiServlet]
(qtp1462044018-366:ctx-3e5ccaa7 ctx-e2a6c0d8) (logid:0bf60b72) ===END===
172.16.3.252 -- POST command=createStoragePool&response=json
2022-08-13 04:32:10,485 DEBUG [o.a.c.h.HAManagerImpl]
(BackgroundTaskPollManager-2:ctx-fcf8bbe0) (logid:1c6fda81) HA health check
task is running...
2022-08-13 04:32:12,027 DEBUG [c.c.a.ApiServlet]
(qtp1462044018-18:ctx-7f80a5be) (logid:0d441e25) ===START=== 172.16.3.252 --
POST command=createStoragePool&response=json
2022-08-13 04:32:12,037 DEBUG [c.c.a.ApiServer]
(qtp1462044018-18:ctx-7f80a5be ctx-75c25dfd) (logid:0d441e25) CIDRs from which
account 'Acct[c5268ccf-1ac5-11ed-8fd2-525400dfc23b-admin] -- Account {"id": 2,
"name": "admin", "uuid": "c5268ccf-1ac5-11ed-8fd2-525400dfc23b"}' is allowed to
perform API calls: 0.0.0.0/0,::/0
2022-08-13 04:32:12,048 DEBUG
[o.a.c.s.d.l.CloudStackPrimaryDataStoreLifeCycleImpl]
(qtp1462044018-18:ctx-7f80a5be ctx-75c25dfd) (logid:0d441e25) createPool Params
@ scheme - rbd storageHost - rbd.cloud.priv hostPath - /cloudstack port - -1
2022-08-13 04:32:12,058 DEBUG
[o.a.c.s.d.l.CloudStackPrimaryDataStoreLifeCycleImpl]
(qtp1462044018-18:ctx-7f80a5be ctx-75c25dfd) (logid:0d441e25) creating pool rv
on host 1
2022-08-13 04:32:12,062 DEBUG [c.c.a.t.Request]
(qtp1462044018-18:ctx-7f80a5be ctx-75c25dfd) (logid:0d441e25) Seq
1-7861877574505333217: Sending { Cmd , MgmtId: 90520745394747, via:
1(kvm27.cloud.priv), Ver: v1, Flags: 100011,
[{"com.cloud.agent.api.CreateStoragePoolCommand":{"_createDatastore":"false","add":"true","pool":{"id":"48","uuid":"a67e3adf-8acf-3ec8-9342-634bd09dc951","host":"rbd.cloud.priv","path":"cloudstack","userInfo":"cloudstack:AQDqdPdiP4rJEBAAttog6uNyIy9plWg0c77e_A==","port":"0","type":"RBD"},"localPath":"/mnt//0b735f43-6986-350d-b809-2d1592a7ffac","wait":"0","bypassHostMaintenance":"false"}}]
}
2022-08-13 04:32:12,064 DEBUG [c.c.a.t.Request]
(AgentManager-Handler-2:null) (logid:) Seq 1-7861877574505333217: Processing:
{ Ans: , MgmtId: 90520745394747, via: 1, Ver: v1, Flags: 10,
[{"com.cloud.agent.api.Answer":{"result":"true","details":"success","wait":"0","bypassHostMaintenance":"false"}}]
}
2022-08-13 04:32:12,064 DEBUG [c.c.a.t.Request]
(qtp1462044018-18:ctx-7f80a5be ctx-75c25dfd) (logid:0d441e25) Seq
1-7861877574505333217: Received: { Ans: , MgmtId: 90520745394747, via:
1(kvm27.cloud.priv), Ver: v1, Flags: 10, { Answer } }
2022-08-13 04:32:12,064 DEBUG [c.c.a.m.AgentManagerImpl]
(qtp1462044018-18:ctx-7f80a5be ctx-75c25dfd) (logid:0d441e25) Details from
executing class com.cloud.agent.api.CreateStoragePoolCommand: success
2022-08-13 04:32:12,066 DEBUG
[o.a.c.s.d.l.CloudStackPrimaryDataStoreLifeCycleImpl]
(qtp1462044018-18:ctx-7f80a5be ctx-75c25dfd) (logid:0d441e25) In createPool
Adding the pool to each of the hosts
2022-08-13 04:32:12,067 DEBUG [c.c.s.StorageManagerImpl]
(qtp1462044018-18:ctx-7f80a5be ctx-75c25dfd) (logid:0d441e25) Adding pool rv to
host 1
2022-08-13 04:32:12,072 DEBUG [c.c.a.t.Request]
(qtp1462044018-18:ctx-7f80a5be ctx-75c25dfd) (logid:0d441e25) Seq
1-7861877574505333218: Sending { Cmd , MgmtId: 90520745394747, via:
1(kvm27.cloud.priv), Ver: v1, Flags: 100011,
[{"com.cloud.agent.api.ModifyStoragePoolCommand":{"add":"true","pool":{"id":"48","uuid":"a67e3adf-8acf-3ec8-9342-634bd09dc951","host":"rbd.cloud.priv","path":"cloudstack","userInfo":"cloudstack:AQDqdPdiP4rJEBAAttog6uNyIy9plWg0c77e_A==","port":"0","type":"RBD"},"localPath":"/mnt//0b735f43-6986-350d-b809-2d1592a7ffac","wait":"0","bypassHostMaintenance":"false"}}]
}
2022-08-13 04:32:12,185 DEBUG [c.c.a.t.Request]
(AgentManager-Handler-3:null) (logid:) Seq 1-7861877574505333218: Processing:
{ Ans: , MgmtId: 90520745394747, via: 1, Ver: v1, Flags: 10,
[{"com.cloud.agent.api.Answer":{"result":"false","details":"com.cloud.utils.exception.CloudRuntimeException:
Failed to create storage pool: a67e3adf-8acf-3ec8-9342-634bd09dc951
at
com.cloud.hypervisor.kvm.storage.LibvirtStorageAdaptor.createStoragePool(LibvirtStorageAdaptor.java:672)
at
com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager.createStoragePool(KVMStoragePoolManager.java:350)
at
com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager.createStoragePool(KVMStoragePoolManager.java:344)
at
com.cloud.hypervisor.kvm.resource.wrapper.LibvirtModifyStoragePoolCommandWrapper.execute(LibvirtModifyStoragePoolCommandWrapper.java:42)
at
com.cloud.hypervisor.kvm.resource.wrapper.LibvirtModifyStoragePoolCommandWrapper.execute(LibvirtModifyStoragePoolCommandWrapper.java:35)
at
com.cloud.hypervisor.kvm.resource.wrapper.LibvirtRequestWrapper.execute(LibvirtRequestWrapper.java:78)
at
com.cloud.hypervisor.kvm.resource.LibvirtComputingResource.executeRequest(LibvirtComputingResource.java:1830)
at com.cloud.agent.Agent.processRequest(Agent.java:660)
at com.cloud.agent.Agent$AgentRequestHandler.doTask(Agent.java:1080)
at com.cloud.utils.nio.Task.call(Task.java:83)
at com.cloud.utils.nio.Task.call(Task.java:29)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base/java.lang.Thread.run(Thread.java:829)
","wait":"0","bypassHostMaintenance":"false"}}] }
2022-08-13 04:32:12,185 DEBUG [c.c.a.t.Request]
(qtp1462044018-18:ctx-7f80a5be ctx-75c25dfd) (logid:0d441e25) Seq
1-7861877574505333218: Received: { Ans: , MgmtId: 90520745394747, via:
1(kvm27.cloud.priv), Ver: v1, Flags: 10, { Answer } }
2022-08-13 04:32:12,185 DEBUG [c.c.a.m.AgentManagerImpl]
(qtp1462044018-18:ctx-7f80a5be ctx-75c25dfd) (logid:0d441e25) Details from
executing class com.cloud.agent.api.ModifyStoragePoolCommand:
com.cloud.utils.exception.CloudRuntimeException: Failed to create storage pool:
a67e3adf-8acf-3ec8-9342-634bd09dc951
at
com.cloud.hypervisor.kvm.storage.LibvirtStorageAdaptor.createStoragePool(LibvirtStorageAdaptor.java:672)
at
com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager.createStoragePool(KVMStoragePoolManager.java:350)
at
com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager.createStoragePool(KVMStoragePoolManager.java:344)
at
com.cloud.hypervisor.kvm.resource.wrapper.LibvirtModifyStoragePoolCommandWrapper.execute(LibvirtModifyStoragePoolCommandWrapper.java:42)
at
com.cloud.hypervisor.kvm.resource.wrapper.LibvirtModifyStoragePoolCommandWrapper.execute(LibvirtModifyStoragePoolCommandWrapper.java:35)
at
com.cloud.hypervisor.kvm.resource.wrapper.LibvirtRequestWrapper.execute(LibvirtRequestWrapper.java:78)
at
com.cloud.hypervisor.kvm.resource.LibvirtComputingResource.executeRequest(LibvirtComputingResource.java:1830)
at com.cloud.agent.Agent.processRequest(Agent.java:660)
at com.cloud.agent.Agent$AgentRequestHandler.doTask(Agent.java:1080)
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
