So I emailed yesterday about a strange problem where I was seeing that I
could create cinder volumes just fine, but when I try to attach them to
an instance, they get stuck in the "attaching" state and kind of hang
there forever, it never really attaches.
Again, Icehouse RC1 on RedHat 6.5.
The cinder logs (all types) show nothing. The only evidence of
strangeness I see in any logs are in the nova compute log on the server
that the instance I'm trying to attach to is running on. I see this:
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher File
"/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 266, in
decorated_function
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher
return function(self, context, *args, **kwargs)
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher File
"/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 309, in
decorated_function
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher e,
sys.exc_info())
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher File
"/usr/lib/python2.6/site-packages/nova/openstack/common/excutils.py",
line 68, in __exit__
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher
six.reraise(self.type_, self.value, self.tb)
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher File
"/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 296, in
decorated_function
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher
return function(self, context, *args, **kwargs)
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher File
"/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 4130,
in attach_volume
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher
bdm.destroy(context)
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher File
"/usr/lib/python2.6/site-packages/nova/openstack/common/excutils.py",
line 68, in __exit__
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher
six.reraise(self.type_, self.value, self.tb)
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher File
"/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 4127,
in attach_volume
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher
return self._attach_volume(context, instance, driver_bdm)
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher File
"/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 4148,
in _attach_volume
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher
self.volume_api.unreserve_volume(context, bdm.volume_id)
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher File
"/usr/lib/python2.6/site-packages/nova/volume/cinder.py", line 173, in
wrapper
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher res
= method(self, ctx, volume_id, *args, **kwargs)
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher File
"/usr/lib/python2.6/site-packages/nova/volume/cinder.py", line 249, in
unreserve_volume
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher
cinderclient(context).volumes.unreserve(volume_id)
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher File
"/usr/lib/python2.6/site-packages/cinderclient/v1/volumes.py", line 293,
in unreserve
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher
return self._action('os-unreserve', volume)
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher File
"/usr/lib/python2.6/site-packages/cinderclient/v1/volumes.py", line 250,
in _action
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher
return self.api.client.post(url, body=body)
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher File
"/usr/lib/python2.6/site-packages/cinderclient/client.py", line 210, in post
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher
return self._cs_request(url, 'POST', **kwargs)
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher File
"/usr/lib/python2.6/site-packages/cinderclient/client.py", line 199, in
_cs_request
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher
raise exceptions.ConnectionError(msg)
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher
ConnectionError: Unable to establish connection: [Errno 101] ENETUNREACH
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher
2014-04-09 22:08:29.998 6547 ERROR oslo.messaging._drivers.common [-]
Returning exception Unable to establish connection: [Errno 101]
ENETUNREACH to caller
2014-04-09 22:08:29.998 6547 ERROR oslo.messaging._drivers.common [-]
['Traceback (most recent call last):\n', ' File
"/usr/lib/python2.6/site-packages/oslo/messaging/rpc/dispatcher.py",
line 133, in _dispatch_and_reply\n incoming.message))\n', ' File
"/usr/lib/python2.6/site-packages/oslo/messaging/rpc/dispatcher.py",
line 176, in _dispatch\n return self._do_dispatch(endpoint, method,
ctxt, args)\n', ' File
"/usr/lib/python2.6/site-packages/oslo/messaging/rpc/dispatcher.py",
line 122, in _do_dispatch\n result = getattr(endpoint, method)(ctxt,
**new_args)\n', ' File
"/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 399, in
decorated_function\n return function(self, context, *args,
**kwargs)\n', ' File
"/usr/lib/python2.6/site-packages/nova/exception.py", line 88, in
wrapped\n payload)\n', ' File
"/usr/lib/python2.6/site-packages/nova/openstack/common/excutils.py",
line 68, in __exit__\n six.reraise(self.type_, self.value,
self.tb)\n', ' File
"/usr/lib/python2.6/site-packages/nova/exception.py", line 71, in
wrapped\n return f(self, context, *args, **kw)\n', ' File
"/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 280, in
decorated_function\n pass\n', ' File
"/usr/lib/python2.6/site-packages/nova/openstack/common/excutils.py",
line 68, in __exit__\n six.reraise(self.type_, self.value,
self.tb)\n', ' File
"/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 266, in
decorated_function\n return function(self, context, *args,
**kwargs)\n', ' File
"/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 309, in
decorated_function\n e, sys.exc_info())\n', ' File
"/usr/lib/python2.6/site-packages/nova/openstack/common/excutils.py",
line 68, in __exit__\n six.reraise(self.type_, self.value,
self.tb)\n', ' File
"/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 296, in
decorated_function\n return function(self, context, *args,
**kwargs)\n', ' File
"/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 4130,
in attach_volume\n bdm.destroy(context)\n', ' File
"/usr/lib/python2.6/site-packages/nova/openstack/common/excutils.py",
line 68, in __exit__\n six.reraise(self.type_, self.value,
self.tb)\n', ' File
"/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 4127,
in attach_volume\n return self._attach_volume(context, instance,
driver_bdm)\n', ' File
"/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 4148,
in _attach_volume\n self.volume_api.unreserve_volume(context,
bdm.volume_id)\n', ' File
"/usr/lib/python2.6/site-packages/nova/volume/cinder.py", line 173, in
wrapper\n res = method(self, ctx, volume_id, *args, **kwargs)\n', '
File "/usr/lib/python2.6/site-packages/nova/volume/cinder.py", line 249,
in unreserve_volume\n
cinderclient(context).volumes.unreserve(volume_id)\n', ' File
"/usr/lib/python2.6/site-packages/cinderclient/v1/volumes.py", line 293,
in unreserve\n return self._action(\'os-unreserve\', volume)\n', '
File "/usr/lib/python2.6/site-packages/cinderclient/v1/volumes.py", line
250, in _action\n return self.api.client.post(url, body=body)\n', '
File "/usr/lib/python2.6/site-packages/cinderclient/client.py", line
210, in post\n return self._cs_request(url, \'POST\', **kwargs)\n', '
File "/usr/lib/python2.6/site-packages/cinderclient/client.py", line
199, in _cs_request\n raise exceptions.ConnectionError(msg)\n',
'ConnectionError: Unable to establish connection: [Errno 101]
ENETUNREACH\n']
So clearly nova is having trouble attaching the volume, and I see
several "Unable to establish connection" messages. Does that mean it
can't connect to the iSCSI port on the cinder server? No firewall
there. Or something else it can't connect to?
Thanks again,
erich
_______________________________________________
Mailing list: http://lists.openstack.org/cgi-bin/mailman/listinfo/openstack
Post to : [email protected]
Unsubscribe : http://lists.openstack.org/cgi-bin/mailman/listinfo/openstack
