Hello. After upgrade to 10.2.2-0ubuntu1, they are still crashing on the
same command:
-1> 2016-06-30 14:43:11.973392 7f4f48fc2700 0 mon.pp6@1(leader).osd e10230
moving crush item name 'pp1' to location {root=fast2500}
0> 2016-06-30 14:43:11.975532 7f4f48fc2700 -1 crush/CrushWrapper.h: In
function 'int CrushWrapper::detach_bucket(CephContext*, int)' thread
7f4f48fc2700 time 2016-06-30 14:43:11.973438
crush/CrushWrapper.h: 940: FAILED assert(successful_detach)
ceph version 10.2.2 (45107e21c568dd033c2f0a3107dec8f0b0e58374)
1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x80)
[0x559f04e3d7c0]
2: (()+0x5680f3) [0x559f04d4b0f3]
3: (CrushWrapper::move_bucket(CephContext*, int,
std::map<std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> >, std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> >,
std::less<std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > >,
std::allocator<std::pair<std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const,
std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >
> > > const&)+0xda) [0x559f04d48d0a]
4: (OSDMonitor::prepare_command_impl(std::shared_ptr<MonOpRequest>,
std::map<std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> >, boost::variant<std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> >, bool, long, double,
std::vector<std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > > > >,
std::less<std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > >,
std::allocator<std::pair<std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const,
boost::variant<std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> >, bool, long, double,
std::vector<std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > > > > > > >&)+0x2cf
e) [0x559f04b0a4be]
5: (OSDMonitor::prepare_command(std::shared_ptr<MonOpRequest>)+0x2ff)
[0x559f04b1c51f]
6: (OSDMonitor::prepare_update(std::shared_ptr<MonOpRequest>)+0x24b)
[0x559f04b1ca6b]
7: (PaxosService::dispatch(std::shared_ptr<MonOpRequest>)+0xb4f)
[0x559f04acf31f]
8: (PaxosService::C_RetryMessage::_finish(int)+0x58) [0x559f04ad0908]
9: (C_MonOp::finish(int)+0x82) [0x559f04a98ab2]
10: (Context::complete(int)+0x9) [0x559f04a97b99]
11: (void finish_contexts<Context>(CephContext*, std::__cxx11::list<Context*,
std::allocator<Context*> >&, int)+0x1fb) [0x559f04a9e4cb]
12: (Paxos::finish_round()+0x287) [0x559f04ac4d87]
13: (Paxos::handle_last(std::shared_ptr<MonOpRequest>)+0xe19) [0x559f04ac5f69]
14: (Paxos::dispatch(std::shared_ptr<MonOpRequest>)+0x250) [0x559f04ac6790]
15: (Monitor::dispatch_op(std::shared_ptr<MonOpRequest>)+0xa38)
[0x559f04a920b8]
16: (Monitor::_ms_dispatch(Message*)+0x554) [0x559f04a928b4]
17: (Monitor::ms_dispatch(Message*)+0x23) [0x559f04ab5963]
18: (DispatchQueue::entry()+0xf2b) [0x559f04f328fb]
19: (DispatchQueue::DispatchThread::entry()+0xd) [0x559f04e23b0d]
20: (()+0x76fa) [0x7f4f50d066fa]
21: (clone()+0x6d) [0x7f4f4efbab5d]
NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to
interpret this.
--- logging levels ---
0/ 5 none
0/ 1 lockdep
0/ 1 context
1/ 1 crush
1/ 5 mds
1/ 5 mds_balancer
1/ 5 mds_locker
1/ 5 mds_log
1/ 5 mds_log_expire
1/ 5 mds_migrator
0/ 1 buffer
0/ 1 timer
0/ 1 filer
0/ 1 striper
0/ 1 objecter
0/ 5 rados
0/ 5 rbd
0/ 5 rbd_mirror
0/ 5 rbd_replay
0/ 5 journaler
0/ 5 objectcacher
0/ 5 client
0/ 5 osd
0/ 5 optracker
0/ 5 objclass
1/ 3 filestore
1/ 3 journal
0/ 5 ms
1/ 5 mon
0/10 monc
1/ 5 paxos
0/ 5 tp
1/ 5 auth
1/ 5 crypto
1/ 1 finisher
1/ 5 heartbeatmap
1/ 5 perfcounter
1/ 5 rgw
1/10 civetweb
1/ 5 javaclient
1/ 5 asok
1/ 1 throttle
0/ 0 refs
1/ 5 xio
1/ 5 compressor
1/ 5 newstore
1/ 5 bluestore
1/ 5 bluefs
1/ 3 bdev
1/ 5 kstore
4/ 5 rocksdb
4/ 5 leveldb
1/ 5 kinetic
1/ 5 fuse
-2/-2 (syslog threshold)
-1/-1 (stderr threshold)
max_recent 10000
max_new 1000
log_file /var/log/ceph/ceph-mon.pp6.log
--- end dump of recent events ---
2016-06-30 14:43:11.978616 7f4f48fc2700 -1 *** Caught signal (Aborted) **
in thread 7f4f48fc2700 thread_name:ms_dispatch
ceph version 10.2.2 (45107e21c568dd033c2f0a3107dec8f0b0e58374)
1: (()+0x52ac7e) [0x559f04d0dc7e]
2: (()+0x113d0) [0x7f4f50d103d0]
3: (gsignal()+0x38) [0x7f4f4eee9418]
4: (abort()+0x16a) [0x7f4f4eeeb01a]
5: (ceph::__ceph_assert_fail(char const*, char const*, int, char
const*)+0x26b) [0x559f04e3d9ab]
6: (()+0x5680f3) [0x559f04d4b0f3]
7: (CrushWrapper::move_bucket(CephContext*, int,
std::map<std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> >, std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> >,
std::less<std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > >,
std::allocator<std::pair<std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const,
std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >
> > > const&)+0xda) [0x559f04d48d0a]
8: (OSDMonitor::prepare_command_impl(std::shared_ptr<MonOpRequest>,
std::map<std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> >, boost::variant<std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> >, bool, long, double,
std::vector<std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > > > >,
std::less<std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > >,
std::allocator<std::pair<std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const,
boost::variant<std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> >, bool, long, double,
std::vector<std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > > > > > > >&)+0x2cf
e) [0x559f04b0a4be]
9: (OSDMonitor::prepare_command(std::shared_ptr<MonOpRequest>)+0x2ff)
[0x559f04b1c51f]
10: (OSDMonitor::prepare_update(std::shared_ptr<MonOpRequest>)+0x24b)
[0x559f04b1ca6b]
11: (PaxosService::dispatch(std::shared_ptr<MonOpRequest>)+0xb4f)
[0x559f04acf31f]
12: (PaxosService::C_RetryMessage::_finish(int)+0x58) [0x559f04ad0908]
13: (C_MonOp::finish(int)+0x82) [0x559f04a98ab2]
14: (Context::complete(int)+0x9) [0x559f04a97b99]
15: (void finish_contexts<Context>(CephContext*, std::__cxx11::list<Context*,
std::allocator<Context*> >&, int)+0x1fb) [0x559f04a9e4cb]
16: (Paxos::finish_round()+0x287) [0x559f04ac4d87]
17: (Paxos::handle_last(std::shared_ptr<MonOpRequest>)+0xe19) [0x559f04ac5f69]
18: (Paxos::dispatch(std::shared_ptr<MonOpRequest>)+0x250) [0x559f04ac6790]
19: (Monitor::dispatch_op(std::shared_ptr<MonOpRequest>)+0xa38)
[0x559f04a920b8]
20: (Monitor::_ms_dispatch(Message*)+0x554) [0x559f04a928b4]
21: (Monitor::ms_dispatch(Message*)+0x23) [0x559f04ab5963]
22: (DispatchQueue::entry()+0xf2b) [0x559f04f328fb]
23: (DispatchQueue::DispatchThread::entry()+0xd) [0x559f04e23b0d]
24: (()+0x76fa) [0x7f4f50d066fa]
25: (clone()+0x6d) [0x7f4f4efbab5d]
NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to
interpret this.
--- begin dump of recent events ---
0> 2016-06-30 14:43:11.978616 7f4f48fc2700 -1 *** Caught signal (Aborted)
**
in thread 7f4f48fc2700 thread_name:ms_dispatch
ceph version 10.2.2 (45107e21c568dd033c2f0a3107dec8f0b0e58374)
1: (()+0x52ac7e) [0x559f04d0dc7e]
2: (()+0x113d0) [0x7f4f50d103d0]
3: (gsignal()+0x38) [0x7f4f4eee9418]
4: (abort()+0x16a) [0x7f4f4eeeb01a]
5: (ceph::__ceph_assert_fail(char const*, char const*, int, char
const*)+0x26b) [0x559f04e3d9ab]
6: (()+0x5680f3) [0x559f04d4b0f3]
7: (CrushWrapper::move_bucket(CephContext*, int,
std::map<std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> >, std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> >,
std::less<std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > >,
std::allocator<std::pair<std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const,
std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >
> > > const&)+0xda) [0x559f04d48d0a]
8: (OSDMonitor::prepare_command_impl(std::shared_ptr<MonOpRequest>,
std::map<std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> >, boost::variant<std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> >, bool, long, double,
std::vector<std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > > > >,
std::less<std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > >,
std::allocator<std::pair<std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const,
boost::variant<std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> >, bool, long, double,
std::vector<std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > > > > > > >&)+0x2cf
e) [0x559f04b0a4be]
9: (OSDMonitor::prepare_command(std::shared_ptr<MonOpRequest>)+0x2ff)
[0x559f04b1c51f]
10: (OSDMonitor::prepare_update(std::shared_ptr<MonOpRequest>)+0x24b)
[0x559f04b1ca6b]
11: (PaxosService::dispatch(std::shared_ptr<MonOpRequest>)+0xb4f)
[0x559f04acf31f]
12: (PaxosService::C_RetryMessage::_finish(int)+0x58) [0x559f04ad0908]
13: (C_MonOp::finish(int)+0x82) [0x559f04a98ab2]
14: (Context::complete(int)+0x9) [0x559f04a97b99]
15: (void finish_contexts<Context>(CephContext*, std::__cxx11::list<Context*,
std::allocator<Context*> >&, int)+0x1fb) [0x559f04a9e4cb]
16: (Paxos::finish_round()+0x287) [0x559f04ac4d87]
17: (Paxos::handle_last(std::shared_ptr<MonOpRequest>)+0xe19) [0x559f04ac5f69]
18: (Paxos::dispatch(std::shared_ptr<MonOpRequest>)+0x250) [0x559f04ac6790]
19: (Monitor::dispatch_op(std::shared_ptr<MonOpRequest>)+0xa38)
[0x559f04a920b8]
20: (Monitor::_ms_dispatch(Message*)+0x554) [0x559f04a928b4]
21: (Monitor::ms_dispatch(Message*)+0x23) [0x559f04ab5963]
22: (DispatchQueue::entry()+0xf2b) [0x559f04f328fb]
23: (DispatchQueue::DispatchThread::entry()+0xd) [0x559f04e23b0d]
24: (()+0x76fa) [0x7f4f50d066fa]
25: (clone()+0x6d) [0x7f4f4efbab5d]
NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to
interpret this.
--- logging levels ---
0/ 5 none
0/ 1 lockdep
0/ 1 context
1/ 1 crush
1/ 5 mds
1/ 5 mds_balancer
1/ 5 mds_locker
1/ 5 mds_log
1/ 5 mds_log_expire
1/ 5 mds_migrator
0/ 1 buffer
0/ 1 timer
0/ 1 filer
0/ 1 striper
0/ 1 objecter
0/ 5 rados
0/ 5 rbd
0/ 5 rbd_mirror
0/ 5 rbd_replay
0/ 5 journaler
0/ 5 objectcacher
0/ 5 client
0/ 5 osd
0/ 5 optracker
0/ 5 objclass
1/ 3 filestore
1/ 3 journal
0/ 5 ms
1/ 5 mon
0/10 monc
1/ 5 paxos
0/ 5 tp
1/ 5 auth
1/ 5 crypto
1/ 1 finisher
1/ 5 heartbeatmap
1/ 5 perfcounter
1/ 5 rgw
1/10 civetweb
1/ 5 javaclient
1/ 5 asok
1/ 1 throttle
0/ 0 refs
1/ 5 xio
1/ 5 compressor
1/ 5 newstore
1/ 5 bluestore
1/ 5 bluefs
1/ 3 bdev
1/ 5 kstore
4/ 5 rocksdb
4/ 5 leveldb
1/ 5 kinetic
1/ 5 fuse
-2/-2 (syslog threshold)
-1/-1 (stderr threshold)
max_recent 10000
max_new 1000
log_file /var/log/ceph/ceph-mon.pp6.log
--- end dump of recent events ---
** Changed in: ceph (Ubuntu)
Status: Incomplete => New
--
You received this bug notification because you are a member of Ubuntu
Bugs, which is subscribed to Ubuntu.
https://bugs.launchpad.net/bugs/1597411
Title:
host move inside CRUSH cause permanent crash for all mons
To manage notifications about this bug go to:
https://bugs.launchpad.net/ubuntu/+source/ceph/+bug/1597411/+subscriptions
--
ubuntu-bugs mailing list
[email protected]
https://lists.ubuntu.com/mailman/listinfo/ubuntu-bugs
