we added retry logic around this
** Changed in: nova
Status: Incomplete => Fix Released
--
You received this bug notification because you are a member of Yahoo!
Engineering Team, which is subscribed to OpenStack Compute (nova).
https://bugs.launchpad.net/bugs/1306432
Title:
n-cpu with devstack is down caused by libvirt io error sometimes
Status in OpenStack Compute (Nova):
Fix Released
Bug description:
OS: Ubuntu 12.04
Kernel: 3.8.0-37-generic
libvirt: 0.9.8-2ubuntu17.18
2014-04-11 16:35:03.980 12081 AUDIT nova.service [-] Starting compute node
(version 2014.2)
2014-04-11 16:35:04.004 12081 DEBUG nova.virt.libvirt.driver [-] Connecting
to libvirt: qemu:///system _get_new_connection
/opt/stack/nova/nova/virt/libvirt/driver.py:670
2014-04-11 16:35:18.664 WARNING nova.virt.libvirt.driver
[req-6e97f1b4-a832-4521-881d-c91196220556 None None] Cannot update service
status on host: Router,since it is not registered.
2014-04-11 16:35:18.664 12081 DEBUG nova.virt.libvirt.driver [-] Registering
for lifecycle events <nova.virt.libvirt.driver.LibvirtDriver object at
0x2b5ce10> _get_new_connection /opt/stack/nova/nova/virt/libvirt/driver.p
y:686
2014-04-11 16:35:19.145 12081 DEBUG nova.virt.libvirt.driver [-] Registering
for connection events: <nova.virt.libvirt.driver.LibvirtDriver object at
0x2b5ce10> _get_new_connection /opt/stack/nova/nova/virt/libvirt/driver
.py:698
2014-04-11 16:35:19.145 12081 DEBUG nova.virt.libvirt.driver [-] The version
of python-libvirt does not support registerCloseCallback or is too old:
virConnect instance has no attribute 'registerCloseCallback' _get_new_co
nnection /opt/stack/nova/nova/virt/libvirt/driver.py:708
Traceback (most recent call last):
File "/usr/local/lib/python2.7/dist-packages/eventlet/hubs/hub.py", line
346, in fire_timers
timer()
File "/usr/local/lib/python2.7/dist-packages/eventlet/hubs/timer.py", line
56, in __call__
cb(*args, **kw)
File "/usr/local/lib/python2.7/dist-packages/eventlet/event.py", line 163,
in _do_send
waiter.switch(result)
File "/usr/local/lib/python2.7/dist-packages/eventlet/greenthread.py", line
194, in main
result = function(*args, **kwargs)
File "/opt/stack/nova/nova/openstack/common/service.py", line 490, in
run_service
service.start()
File "/opt/stack/nova/nova/service.py", line 163, in start
self.manager.init_host()
File "/opt/stack/nova/nova/compute/manager.py", line 1051, in init_host
self.driver.init_host(host=self.host)
File "/opt/stack/nova/nova/virt/libvirt/driver.py", line 656, in init_host
self._do_quality_warnings()
File "/opt/stack/nova/nova/virt/libvirt/driver.py", line 639, in
_do_quality_warnings
caps = self.get_host_capabilities()
File "/opt/stack/nova/nova/virt/libvirt/driver.py", line 2842, in
get_host_capabilities
xmlstr = self._conn.getCapabilities()
File "/usr/local/lib/python2.7/dist-packages/eventlet/tpool.py", line 179,
in doit
result = proxy_call(self._autowrap, f, *args, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/eventlet/tpool.py", line 139,
in proxy_call
rv = execute(f,*args,**kwargs)
File "/usr/local/lib/python2.7/dist-packages/eventlet/tpool.py", line 77,
in tworker
rv = meth(*args,**kwargs)
File "/usr/lib/python2.7/dist-packages/libvirt.py", line 2236, in
getCapabilities
if ret is None: raise libvirtError ('virConnectGetCapabilities() failed',
conn=self)
libvirtError: End of file while reading data: Input/output error
2014-04-11 16:35:50.023 12081 ERROR nova.openstack.common.threadgroup [-] End
of file while reading data: Input/output error
2014-04-11 16:35:50.023 12081 TRACE nova.openstack.common.threadgroup
Traceback (most recent call last):
2014-04-11 16:35:50.023 12081 TRACE nova.openstack.common.threadgroup File
"/opt/stack/nova/nova/openstack/common/threadgroup.py", line 125, in wait
2014-04-11 16:35:50.023 12081 TRACE nova.openstack.common.threadgroup
x.wait()
2014-04-11 16:35:50.023 12081 TRACE nova.openstack.common.threadgroup File
"/opt/stack/nova/nova/openstack/common/threadgroup.py", line 47, in wait
2014-04-11 16:35:50.023 12081 TRACE nova.openstack.common.threadgroup
return self.thread.wait()
2014-04-11 16:35:50.023 12081 TRACE nova.openstack.common.threadgroup File
"/usr/local/lib/python2.7/dist-packages/eventlet/greenthread.py", line 168, in
wait
2014-04-11 16:35:50.023 12081 TRACE nova.openstack.common.threadgroup
return self._exit_event.wait()
2014-04-11 16:35:50.023 12081 TRACE nova.openstack.common.threadgroup File
"/usr/local/lib/python2.7/dist-packages/eventlet/event.py", line 116, in wait
2014-04-11 16:35:50.023 12081 TRACE nova.openstack.common.threadgroup
return hubs.get_hub().switch()
2014-04-11 16:35:50.023 12081 TRACE nova.openstack.common.threadgroup File
"/usr/local/lib/python2.7/dist-packages/eventlet/hubs/hub.py", line 187, in
switch
2014-04-11 16:35:50.023 12081 TRACE nova.openstack.common.threadgroup
return self.greenlet.switch()
2014-04-11 16:35:50.023 12081 TRACE nova.openstack.common.threadgroup File
"/usr/local/lib/python2.7/dist-packages/eventlet/greenthread.py", line 194, in
main
2014-04-11 16:35:50.023 12081 TRACE nova.openstack.common.threadgroup
result = function(*args, **kwargs)
2014-04-11 16:35:50.023 12081 TRACE nova.openstack.common.threadgroup File
"/opt/stack/nova/nova/openstack/common/service.py", line 490, in run_service
2014-04-11 16:35:50.023 12081 TRACE nova.openstack.common.threadgroup
service.start()
2014-04-11 16:35:50.023 12081 TRACE nova.openstack.common.threadgroup File
"/opt/stack/nova/nova/service.py", line 163, in start
2014-04-11 16:35:50.023 12081 TRACE nova.openstack.common.threadgroup
self.manager.init_host()
2014-04-11 16:35:50.023 12081 TRACE nova.openstack.common.threadgroup File
"/opt/stack/nova/nova/compute/manager.py", line 1051, in init_host
2014-04-11 16:35:50.023 12081 TRACE nova.openstack.common.threadgroup
self.driver.init_host(host=self.host)
2014-04-11 16:35:50.023 12081 TRACE nova.openstack.common.threadgroup File
"/opt/stack/nova/nova/virt/libvirt/driver.py", line 656, in init_host
2014-04-11 16:35:50.023 12081 TRACE nova.openstack.common.threadgroup
self._do_quality_warnings()
2014-04-11 16:35:50.023 12081 TRACE nova.openstack.common.threadgroup File
"/opt/stack/nova/nova/virt/libvirt/driver.py", line 639, in _do_quality_warnings
2014-04-11 16:35:50.023 12081 TRACE nova.openstack.common.threadgroup
caps = self.get_host_capabilities()
2014-04-11 16:35:50.023 12081 TRACE nova.openstack.common.threadgroup File
"/opt/stack/nova/nova/virt/libvirt/driver.py", line 2842, in
get_host_capabilities
2014-04-11 16:35:50.023 12081 TRACE nova.openstack.common.threadgroup
xmlstr = self._conn.getCapabilities()
2014-04-11 16:35:50.023 12081 TRACE nova.openstack.common.threadgroup File
"/usr/local/lib/python2.7/dist-packages/eventlet/tpool.py", line 179, in doit
2014-04-11 16:35:50.023 12081 TRACE nova.openstack.common.threadgroup
result = proxy_call(self._autowrap, f, *args, **kwargs)
2014-04-11 16:35:50.023 12081 TRACE nova.openstack.common.threadgroup File
"/usr/local/lib/python2.7/dist-packages/eventlet/tpool.py", line 139, in
proxy_call
2014-04-11 16:35:50.023 12081 TRACE nova.openstack.common.threadgroup rv
= execute(f,*args,**kwargs)
2014-04-11 16:35:50.023 12081 TRACE nova.openstack.common.threadgroup File
"/usr/local/lib/python2.7/dist-packages/eventlet/tpool.py", line 77, in tworker
2014-04-11 16:35:50.023 12081 TRACE nova.openstack.common.threadgroup rv
= meth(*args,**kwargs)
2014-04-11 16:35:50.023 12081 TRACE nova.openstack.common.threadgroup File
"/usr/lib/python2.7/dist-packages/libvirt.py", line 2236, in getCapabilities
2014-04-11 16:35:50.023 12081 TRACE nova.openstack.common.threadgroup if
ret is None: raise libvirtError ('virConnectGetCapabilities() failed',
conn=self)
2014-04-11 16:35:50.023 12081 TRACE nova.openstack.common.threadgroup
libvirtError: End of file while reading data: Input/output error
2014-04-11 16:35:50.023 12081 TRACE nova.openstack.common.threadgroup
To manage notifications about this bug go to:
https://bugs.launchpad.net/nova/+bug/1306432/+subscriptions
--
Mailing list: https://launchpad.net/~yahoo-eng-team
Post to : [email protected]
Unsubscribe : https://launchpad.net/~yahoo-eng-team
More help : https://help.launchpad.net/ListHelp
