Hi All,
I recently upgraded a small three node cluster to Kraken and noticed some
unusual traffic from the monitor nodes to each of the other nodes with a
destination TCP port of zero. I tracked this down to one of the ceph-mgr
threads with strace.
root@vhost10:~# ps -eLf | grep ceph-mgr
ceph 17443 1 17443 0 17 17:00 ? 00:00:00
/usr/bin/ceph-mgr -f --cluster ceph --id a --setuser ceph --setgroup ceph
ceph 17443 1 17444 0 17 17:00 ? 00:00:00
/usr/bin/ceph-mgr -f --cluster ceph --id a --setuser ceph --setgroup ceph
ceph 17443 1 17445 0 17 17:00 ? 00:00:00
/usr/bin/ceph-mgr -f --cluster ceph --id a --setuser ceph --setgroup ceph
ceph 17443 1 17446 0 17 17:00 ? 00:00:00
/usr/bin/ceph-mgr -f --cluster ceph --id a --setuser ceph --setgroup ceph
ceph 17443 1 17447 0 17 17:00 ? 00:00:00
/usr/bin/ceph-mgr -f --cluster ceph --id a --setuser ceph --setgroup ceph
ceph 17443 1 17448 0 17 17:00 ? 00:00:00
/usr/bin/ceph-mgr -f --cluster ceph --id a --setuser ceph --setgroup ceph
ceph 17443 1 17449 0 17 17:00 ? 00:00:00
/usr/bin/ceph-mgr -f --cluster ceph --id a --setuser ceph --setgroup ceph
ceph 17443 1 17450 0 17 17:00 ? 00:00:00
/usr/bin/ceph-mgr -f --cluster ceph --id a --setuser ceph --setgroup ceph
ceph 17443 1 17451 0 17 17:00 ? 00:00:00
/usr/bin/ceph-mgr -f --cluster ceph --id a --setuser ceph --setgroup ceph
ceph 17443 1 17452 0 17 17:00 ? 00:00:00
/usr/bin/ceph-mgr -f --cluster ceph --id a --setuser ceph --setgroup ceph
ceph 17443 1 17453 0 17 17:00 ? 00:00:00
/usr/bin/ceph-mgr -f --cluster ceph --id a --setuser ceph --setgroup ceph
ceph 17443 1 17454 0 17 17:00 ? 00:00:00
/usr/bin/ceph-mgr -f --cluster ceph --id a --setuser ceph --setgroup ceph
ceph 17443 1 17455 0 17 17:00 ? 00:00:00
/usr/bin/ceph-mgr -f --cluster ceph --id a --setuser ceph --setgroup ceph
ceph 17443 1 17456 0 17 17:00 ? 00:00:00
/usr/bin/ceph-mgr -f --cluster ceph --id a --setuser ceph --setgroup ceph
ceph 17443 1 17457 0 17 17:00 ? 00:00:00
/usr/bin/ceph-mgr -f --cluster ceph --id a --setuser ceph --setgroup ceph
ceph 17443 1 17458 0 17 17:00 ? 00:00:02
/usr/bin/ceph-mgr -f --cluster ceph --id a --setuser ceph --setgroup ceph
ceph 17443 1 17459 0 17 17:00 ? 00:00:00
/usr/bin/ceph-mgr -f --cluster ceph --id a --setuser ceph --setgroup ceph
root 17755 17649 17755 0 1 17:16 pts/3 00:00:00 grep ceph-mgr
root@vhost10:~# strace -p 17446 -e connect
Process 17446 attached
connect(24, {sa_family=AF_INET6, sin6_port=htons(0), inet_pton(AF_INET6,
"fdd3:62f9:a63:1:ec4:7aff:fe74:17ae", &sin6_addr), sin6_flowinfo=0,
sin6_scope_id=0}, 28) = -1 EINPROGRESS (Operation now in progress)
connect(24, {sa_family=AF_INET6, sin6_port=htons(0), inet_pton(AF_INET6,
"fdd3:62f9:a63:1:ec4:7aff:fe74:17ae", &sin6_addr), sin6_flowinfo=0,
sin6_scope_id=0}, 28) = -1 ECONNREFUSED (Connection refused)
connect(24, {sa_family=AF_INET6, sin6_port=htons(0), inet_pton(AF_INET6,
"fdd3:62f9:a63:1:ec4:7aff:fe74:17ae", &sin6_addr), sin6_flowinfo=0,
sin6_scope_id=0}, 28) = -1 EINPROGRESS (Operation now in progress)
connect(24, {sa_family=AF_INET6, sin6_port=htons(0), inet_pton(AF_INET6,
"fdd3:62f9:a63:1:ec4:7aff:fe74:17ae", &sin6_addr), sin6_flowinfo=0,
sin6_scope_id=0}, 28) = -1 ECONNREFUSED (Connection refused)
This installation is using IPv6, so this could be a contributing factor.
Additionally there is a Python error loading cherrypy, in the ceph-mgr log:
2017-02-01 17:00:32.480292 7f0e25697700 1 mgr init Loading python module
'rest'
2017-02-01 17:00:32.520284 7f0e25697700 -1 mgr load Module not found: 'rest'
2017-02-01 17:00:32.520302 7f0e25697700 -1 mgr init Error loading module
'rest': (2) No such file or directory
2017-02-01 17:00:32.520316 7f0e25697700 -1 mgr init Traceback (most recent
call last):
File "/usr/lib/ceph/mgr/rest/__init__.py", line 2, in <module>
from module import * # NOQA
File "/usr/lib/ceph/mgr/rest/module.py", line 23, in <module>
import cherrypy
ImportError: No module named cherrypy
Thanks,
Dustin Lundquist
_______________________________________________
ceph-users mailing list
[email protected]
http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
