[
https://issues.apache.org/jira/browse/MESOS-303?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=13489083#comment-13489083
]
Erich Nachbar commented on MESOS-303:
-------------------------------------
Thanks, Ben for looking into this! Beers are on me.
I attached the requested full backtrace below:
Thread 36 (Thread 0x7fffdbfff700 (LWP 4187)):
#0 0x00007ffff5200bac in pthread_cond_wait@@GLIBC_2.3.2 ()
from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007ffff7836a35 in do_completion (v=0x686a10) at src/mt_adaptor.c:330
#2 0x00007ffff51fbd8c in start_thread () from
/lib/x86_64-linux-gnu/libpthread.so.0
#3 0x00007ffff4f45fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#4 0x0000000000000000 in ?? ()
Thread 35 (Thread 0x7fffe0ac5700 (LWP 4186)):
#0 0x00007ffff4f38e93 in poll () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007ffff7836841 in do_io (v=0x686a10) at src/mt_adaptor.c:299
#2 0x00007ffff51fbd8c in start_thread () from
/lib/x86_64-linux-gnu/libpthread.so.0
#3 0x00007ffff4f45fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#4 0x0000000000000000 in ?? ()
Thread 34 (Thread 0x7fffe18f5700 (LWP 4185)):
#0 0x00007ffff4f46633 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007ffff783f707 in epoll_poll (loop=0x7ffff7ddb200,
timeout=<value optimized out>) at ev_epoll.c:127
#2 0x00007ffff78424c0 in ev_loop (loop=0x7ffff7ddb200, flags=<value optimized
out>)
at ev.c:2311
#3 0x00007ffff76cf65f in process::serve(void*) () from
/usr/local/lib/libmesos-0.9.0.so
#4 0x00007ffff51fbd8c in start_thread () from
/lib/x86_64-linux-gnu/libpthread.so.0
#5 0x00007ffff4f45fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#6 0x0000000000000000 in ?? ()
Thread 33 (Thread 0x7fffe270e700 (LWP 4184)):
#0 0x00007ffff5200bac in pthread_cond_wait@@GLIBC_2.3.2 ()
from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007ffff76e7949 in Gate::arrive(long) () from
/usr/local/lib/libmesos-0.9.0.so
#2 0x00007ffff76cf6cf in process::schedule(void*) ()
from /usr/local/lib/libmesos-0.9.0.so
#3 0x00007ffff51fbd8c in start_thread () from
/lib/x86_64-linux-gnu/libpthread.so.0
#4 0x00007ffff4f45fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#5 0x0000000000000000 in ?? ()
Thread 32 (Thread 0x7fffe2f0f700 (LWP 4183)):
#0 0x00007ffff5200bac in pthread_cond_wait@@GLIBC_2.3.2 ()
from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007ffff76e7949 in Gate::arrive(long) () from
/usr/local/lib/libmesos-0.9.0.so
#2 0x00007ffff76cf6cf in process::schedule(void*) ()
from /usr/local/lib/libmesos-0.9.0.so
#3 0x00007ffff51fbd8c in start_thread () from
/lib/x86_64-linux-gnu/libpthread.so.0
#4 0x00007ffff4f45fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#5 0x0000000000000000 in ?? ()
Thread 31 (Thread 0x7fffe3710700 (LWP 4182)):
#0 0x00007ffff5200bac in pthread_cond_wait@@GLIBC_2.3.2 ()
from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007ffff76e7949 in Gate::arrive(long) () from
/usr/local/lib/libmesos-0.9.0.so
#2 0x00007ffff76cf6cf in process::schedule(void*) ()
from /usr/local/lib/libmesos-0.9.0.so
#3 0x00007ffff51fbd8c in start_thread () from
/lib/x86_64-linux-gnu/libpthread.so.0
#4 0x00007ffff4f45fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#5 0x0000000000000000 in ?? ()
Thread 30 (Thread 0x7fffe3f11700 (LWP 4181)):
#0 0x00007ffff5200bac in pthread_cond_wait@@GLIBC_2.3.2 ()
from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007ffff76e7949 in Gate::arrive(long) () from
/usr/local/lib/libmesos-0.9.0.so
#2 0x00007ffff76cf6cf in process::schedule(void*) ()
from /usr/local/lib/libmesos-0.9.0.so
#3 0x00007ffff51fbd8c in start_thread () from
/lib/x86_64-linux-gnu/libpthread.so.0
#4 0x00007ffff4f45fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#5 0x0000000000000000 in ?? ()
Thread 29 (Thread 0x7fffe4712700 (LWP 4180)):
#0 0x00007ffff5200bac in pthread_cond_wait@@GLIBC_2.3.2 ()
from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007ffff76e7949 in Gate::arrive(long) () from
/usr/local/lib/libmesos-0.9.0.so
#2 0x00007ffff76cf6cf in process::schedule(void*) ()
from /usr/local/lib/libmesos-0.9.0.so
#3 0x00007ffff51fbd8c in start_thread () from
/lib/x86_64-linux-gnu/libpthread.so.0
#4 0x00007ffff4f45fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#5 0x0000000000000000 in ?? ()
Thread 28 (Thread 0x7fffe4f13700 (LWP 4179)):
#0 0x00007ffff5200bac in pthread_cond_wait@@GLIBC_2.3.2 ()
from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007ffff76e7949 in Gate::arrive(long) () from
/usr/local/lib/libmesos-0.9.0.so
#2 0x00007ffff76cf6cf in process::schedule(void*) ()
from /usr/local/lib/libmesos-0.9.0.so
#3 0x00007ffff51fbd8c in start_thread () from
/lib/x86_64-linux-gnu/libpthread.so.0
#4 0x00007ffff4f45fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#5 0x0000000000000000 in ?? ()
Thread 27 (Thread 0x7fffe5714700 (LWP 4178)):
#0 0x00007ffff5200bac in pthread_cond_wait@@GLIBC_2.3.2 ()
from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007ffff76e7949 in Gate::arrive(long) () from
/usr/local/lib/libmesos-0.9.0.so
#2 0x00007ffff76cf6cf in process::schedule(void*) ()
from /usr/local/lib/libmesos-0.9.0.so
#3 0x00007ffff51fbd8c in start_thread () from
/lib/x86_64-linux-gnu/libpthread.so.0
#4 0x00007ffff4f45fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#5 0x0000000000000000 in ?? ()
Thread 26 (Thread 0x7fffe5f15700 (LWP 4177)):
#0 0x00007ffff5200bac in pthread_cond_wait@@GLIBC_2.3.2 ()
from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007ffff76e7949 in Gate::arrive(long) () from
/usr/local/lib/libmesos-0.9.0.so
#2 0x00007ffff76cf6cf in process::schedule(void*) ()
from /usr/local/lib/libmesos-0.9.0.so
#3 0x00007ffff51fbd8c in start_thread () from
/lib/x86_64-linux-gnu/libpthread.so.0
#4 0x00007ffff4f45fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#5 0x0000000000000000 in ?? ()
Thread 25 (Thread 0x7fffe6716700 (LWP 4176)):
#0 0x00007ffff4f372ed in read () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007ffff4ed2798 in _IO_file_underflow () from
/lib/x86_64-linux-gnu/libc.so.6
#2 0x00007ffff4ed37be in _IO_default_uflow () from
/lib/x86_64-linux-gnu/libc.so.6
#3 0x00007ffff4ec78fa in _IO_getline_info () from
/lib/x86_64-linux-gnu/libc.so.6
#4 0x00007ffff4ec67ca in fgets () from /lib/x86_64-linux-gnu/libc.so.6
#5 0x00007ffff75100ea in os::shell(std::basic_ostream<char,
std::char_traits<char> >*, std::basic_string<char, std::char_traits<char>,
std::allocator<char> > const&, ...) ()
from /usr/local/lib/libmesos-0.9.0.so
#6 0x00007ffff75108d0 in mesos::internal::utils::process::killtree(int, int,
bool, bool, bool) () from /usr/local/lib/libmesos-0.9.0.so
#7 0x00007ffff750ed4e in
mesos::internal::slave::ProcessBasedIsolationModule::killExecutor(mesos::FrameworkID
const&, mesos::ExecutorID const&) ()
from /usr/local/lib/libmesos-0.9.0.so
#8 0x00007ffff7503946 in std::tr1::_Mem_fn<void
(mesos::internal::slave::IsolationModule::*)(mesos::FrameworkID const&,
mesos::ExecutorID
const&)>::operator()(mesos::internal::slave::IsolationModule*,
mesos::FrameworkID const&, mesos::ExecutorID const&) const ()
from /usr/local/lib/libmesos-0.9.0.so
#9 0x00007ffff7501442 in std::tr1::result_of<std::tr1::_Mem_fn<void
(mesos::internal::slave::IsolationModule::*)(mesos::FrameworkID const&,
mesos::ExecutorID const&)>
()(std::tr1::result_of<std::tr1::_Mu<std::tr1::_Placeholder<1>, false, true>
()(std::tr1::_Placeholder<1>,
std::tr1::tuple<mesos::internal::slave::IsolationModule*&>)>::type,
std::tr1::result_of<std::tr1::_Mu<mesos::FrameworkID, false, false>
()(mesos::FrameworkID, std::tr1::_Mu<std::tr1::_Placeholder<1>, false, true>
()(std::tr1::_Placeholder<1>,
std::tr1::tuple<mesos::internal::slave::IsolationModule*&>))>::type,
std::tr1::result_of<std::tr1::_Mu<mesos::ExecutorID, false, false>
()(mesos::ExecutorID, std::tr1::_Mu<std::tr1::_Placeholder<1>, false, true>
()(std::tr1::_Placeholder<1>,
std::tr1::tuple<mesos::internal::slave::IsolationModule*&>))>::type)>::type
std::tr1::_Bind<std::tr1::_Mem_fn<void
(mesos::internal::slave::IsolationModule::*)(mesos::FrameworkID const&,
mesos::ExecutorID const&)> ()(std::tr1::_Placeholder<1>, mesos::FrameworkID,
mesos::ExecutorID)>::__call<mesos::internal::slave::IsolationModule*&, 0, 1,
2>(std::tr1::_Mu<std::tr1::_Placeholder<1>, false, true> (
const&)(std::tr1::_Placeholder<1>,
std::tr1::tuple<mesos::internal::slave::IsolationModule*&>),
std::tr1::_Index_tuple<0, 1, 2>) () from /usr/local/lib/libmesos-0.9.0.so
#10 0x00007ffff74fd562 in std::tr1::result_of<std::tr1::_Mem_fn<void
(mesos::internal::slave::IsolationModule::*)(mesos::FrameworkID const&,
mesos::ExecutorID const&)>
()(std::tr1::result_of<std::tr1::_Mu<std::tr1::_Placeholder<1>, false, true>
()(std::tr1::_Placeholder<1>,
std::tr1::tuple<mesos::internal::slave::IsolationModule*>)>::type,
std::tr1::result_of<std::tr1::_Mu<mesos::FrameworkID, false, false>
()(mesos::FrameworkID, std::tr1::_Mu<std::tr1::_Placeholder<1>, false, true>
()(std::tr1::_Placeholder<1>,
std::tr1::tuple<mesos::internal::slave::IsolationModule*>))>::type,
std::tr1::result_of<std::tr1::_Mu<mesos::ExecutorID, false, false>
()(mesos::ExecutorID, std::tr1::_Mu<std::tr1::_Placeholder<1>, false, true>
()(std::tr1::_Placeholder<1>,
std::tr1::tuple<mesos::internal::slave::IsolationModule*>))>::type)>::type
std::tr1::_Bind<std::tr1::_Mem_fn<void
(mesos::internal::slave::IsolationModule::*)(mesos::FrameworkID const&,
mesos::ExecutorID const&)> ()(std::tr1::_Placeholder<1>, mesos::FrameworkID,
mesos::ExecutorID)>::operator()<mesos::internal::slave::IsolationModule*>(mesos::internal::slave::IsolationModule*&)
()
from /usr/local/lib/libmesos-0.9.0.so
#11 0x00007ffff74f7bd0 in std::tr1::_Function_handler<void
()(mesos::internal::slave::IsolationModule*),
std::tr1::_Bind<std::tr1::_Mem_fn<void
(mesos::internal::slave::IsolationModule::*)(mesos::FrameworkID const&,
mesos::ExecutorID const&)> ()(std::tr1::_Placeholder<1>, mesos::FrameworkID,
mesos::ExecutorID)> >::_M_invoke(std::tr1::_Any_data const&,
mesos::internal::slave::IsolationModule*) () from
/usr/local/lib/libmesos-0.9.0.so
#12 0x00007ffff74f3800 in std::tr1::function<void
()(mesos::internal::slave::IsolationModule*)>::operator()(mesos::internal::slave::IsolationModule*)
const ()
from /usr/local/lib/libmesos-0.9.0.so
#13 0x00007ffff74ed1a7 in void
process::internal::vdispatcher<mesos::internal::slave::IsolationModule>(process::ProcessBase*,
std::tr1::shared_ptr<std::tr1::function<void
()(mesos::internal::slave::IsolationModule*)> >) () from
/usr/local/lib/libmesos-0.9.0.so
#14 0x00007ffff74feb9b in std::tr1::result_of<void
(*()(std::tr1::result_of<std::tr1::_Mu<std::tr1::_Placeholder<1>, false, true>
()(std::tr1::_Placeholder<1>, std::tr1::tuple<process::ProcessBase*&>)>::type,
std::tr1::result_of<std::tr1::_Mu<std::tr1::shared_ptr<std::tr1::function<void
()(mesos::internal::slave::IsolationModule*)> >, false, false>
()(std::tr1::shared_ptr<std::tr1::function<void
()(mesos::internal::slave::IsolationModule*)> >,
std::tr1::_Mu<std::tr1::_Placeholder<1>, false, true>
()(std::tr1::_Placeholder<1>,
std::tr1::tuple<process::ProcessBase*&>))>::type))(process::ProcessBase*,
std::tr1::shared_ptr<std::tr1::function<void
()(mesos::internal::slave::IsolationModule*)> >)>::type std::tr1::_Bind<void
(*()(std::tr1::_Placeholder<1>, std::tr1::shared_ptr<std::tr1::function<void
()(mesos::internal::slave::IsolationModule*)> >))(process::ProcessBase*,
std::tr1::shared_ptr<std::tr1::function<void
()(mesos::internal::slave::IsolationModule*)>
>)>::__call<process::ProcessBase*&, 0,
1>(std::tr1::_Mu<std::tr1::_Placeholder<1>, false, true> (
const&)(std::tr1::_Placeholder<1>, std::tr1::tuple<process::ProcessBase*&>),
std::tr1::_Index_tuple<0, 1>) () from /usr/local/lib/libmesos-0.9.0.so
#15 0x00007ffff74f9916 in std::tr1::result_of<void
(*()(std::tr1::result_of<std::tr1::_Mu<std::tr1::_Placeholder<1>, false, true>
()(std::tr1::_Placeholder<1>, std::tr1::tuple<process::ProcessBase*>)>::type,
std::tr1::result_of<std::tr1::_Mu<std::tr1::shared_ptr<std::tr1::function<void
()(mesos::internal::slave::IsolationModule*)> >, false, false>
()(std::tr1::shared_ptr<std::tr1::function<void
()(mesos::internal::slave::IsolationModule*)> >,
std::tr1::_Mu<std::tr1::_Placeholder<1>, false, true>
()(std::tr1::_Placeholder<1>,
std::tr1::tuple<process::ProcessBase*>))>::type))(process::ProcessBase*,
std::tr1::shared_ptr<std::tr1::function<void
()(mesos::internal::slave::IsolationModule*)> >)>::type std::tr1::_Bind<void
(*()(std::tr1::_Placeholder<1>, std::tr1::shared_ptr<std::tr1::function<void
()(mesos::internal::slave::IsolationModule*)> >))(process::ProcessBase*,
std::tr1::shared_ptr<std::tr1::function<void
()(mesos::internal::slave::IsolationModule*)>
>)>::operator()<process::ProcessBase*>(process::ProcessBase*&) ()
from /usr/local/lib/libmesos-0.9.0.so
#16 0x00007ffff74f38ec in std::tr1::_Function_handler<void
()(process::ProcessBase*), std::tr1::_Bind<void (*()(std::tr1::_Placeholder<1>,
std::tr1::shared_ptr<std::tr1::function<void
()(mesos::internal::slave::IsolationModule*)> >))(process::ProcessBase*,
std::tr1::shared_ptr<std::tr1::function<void
()(mesos::internal::slave::IsolationModule*)> >)>
>::_M_invoke(std::tr1::_Any_data const&, process::ProcessBase*) ()
from /usr/local/lib/libmesos-0.9.0.so
#17 0x00007ffff76eecd0 in std::tr1::function<void
()(process::ProcessBase*)>::operator()(process::ProcessBase*) const () from
/usr/local/lib/libmesos-0.9.0.so
#18 0x00007ffff76da56b in process::ProcessBase::visit(process::DispatchEvent
const&) ()
from /usr/local/lib/libmesos-0.9.0.so
#19 0x00007ffff76df1a4 in process::DispatchEvent::visit(process::EventVisitor*)
const ()
from /usr/local/lib/libmesos-0.9.0.so
#20 0x00007ffff738a85e in process::ProcessBase::serve(process::Event const&) ()
from /usr/local/lib/libmesos-0.9.0.so
#21 0x00007ffff76d7ccb in
process::ProcessManager::resume(process::ProcessBase*) ()
from /usr/local/lib/libmesos-0.9.0.so
#22 0x00007ffff76cf6f7 in process::schedule(void*) ()
from /usr/local/lib/libmesos-0.9.0.so
#23 0x00007ffff51fbd8c in start_thread () from
/lib/x86_64-linux-gnu/libpthread.so.0
#24 0x00007ffff4f45fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#25 0x0000000000000000 in ?? ()
Thread 24 (Thread 0x7fffe6f17700 (LWP 4175)):
#0 0x00007ffff5200bac in pthread_cond_wait@@GLIBC_2.3.2 ()
from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007ffff76e7949 in Gate::arrive(long) () from
/usr/local/lib/libmesos-0.9.0.so
#2 0x00007ffff76cf6cf in process::schedule(void*) ()
from /usr/local/lib/libmesos-0.9.0.so
#3 0x00007ffff51fbd8c in start_thread () from
/lib/x86_64-linux-gnu/libpthread.so.0
#4 0x00007ffff4f45fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#5 0x0000000000000000 in ?? ()
Thread 23 (Thread 0x7fffe7718700 (LWP 4174)):
#0 0x00007ffff5200bac in pthread_cond_wait@@GLIBC_2.3.2 ()
from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007ffff76e7949 in Gate::arrive(long) () from
/usr/local/lib/libmesos-0.9.0.so
#2 0x00007ffff76cf6cf in process::schedule(void*) ()
from /usr/local/lib/libmesos-0.9.0.so
#3 0x00007ffff51fbd8c in start_thread () from
/lib/x86_64-linux-gnu/libpthread.so.0
#4 0x00007ffff4f45fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#5 0x0000000000000000 in ?? ()
Thread 22 (Thread 0x7fffe7f19700 (LWP 4173)):
#0 0x00007ffff5200bac in pthread_cond_wait@@GLIBC_2.3.2 ()
from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007ffff76e7949 in Gate::arrive(long) () from
/usr/local/lib/libmesos-0.9.0.so
#2 0x00007ffff76cf6cf in process::schedule(void*) ()
from /usr/local/lib/libmesos-0.9.0.so
#3 0x00007ffff51fbd8c in start_thread () from
/lib/x86_64-linux-gnu/libpthread.so.0
#4 0x00007ffff4f45fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#5 0x0000000000000000 in ?? ()
Thread 21 (Thread 0x7fffe871a700 (LWP 4172)):
#0 0x00007ffff5200bac in pthread_cond_wait@@GLIBC_2.3.2 ()
from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007ffff76e7949 in Gate::arrive(long) () from
/usr/local/lib/libmesos-0.9.0.so
#2 0x00007ffff76cf6cf in process::schedule(void*) ()
from /usr/local/lib/libmesos-0.9.0.so
#3 0x00007ffff51fbd8c in start_thread () from
/lib/x86_64-linux-gnu/libpthread.so.0
#4 0x00007ffff4f45fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#5 0x0000000000000000 in ?? ()
Thread 20 (Thread 0x7fffe8f1b700 (LWP 4171)):
#0 0x00007ffff5200bac in pthread_cond_wait@@GLIBC_2.3.2 ()
from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007ffff76e7949 in Gate::arrive(long) () from
/usr/local/lib/libmesos-0.9.0.so
#2 0x00007ffff76cf6cf in process::schedule(void*) ()
from /usr/local/lib/libmesos-0.9.0.so
#3 0x00007ffff51fbd8c in start_thread () from
/lib/x86_64-linux-gnu/libpthread.so.0
#4 0x00007ffff4f45fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#5 0x0000000000000000 in ?? ()
Thread 19 (Thread 0x7fffe971c700 (LWP 4170)):
#0 0x00007ffff5200bac in pthread_cond_wait@@GLIBC_2.3.2 ()
from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007ffff76e7949 in Gate::arrive(long) () from
/usr/local/lib/libmesos-0.9.0.so
#2 0x00007ffff76cf6cf in process::schedule(void*) ()
from /usr/local/lib/libmesos-0.9.0.so
#3 0x00007ffff51fbd8c in start_thread () from
/lib/x86_64-linux-gnu/libpthread.so.0
#4 0x00007ffff4f45fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#5 0x0000000000000000 in ?? ()
Thread 18 (Thread 0x7fffe9f1d700 (LWP 4169)):
#0 0x00007ffff5200bac in pthread_cond_wait@@GLIBC_2.3.2 ()
from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007ffff76e7949 in Gate::arrive(long) () from
/usr/local/lib/libmesos-0.9.0.so
#2 0x00007ffff76cf6cf in process::schedule(void*) ()
from /usr/local/lib/libmesos-0.9.0.so
#3 0x00007ffff51fbd8c in start_thread () from
/lib/x86_64-linux-gnu/libpthread.so.0
#4 0x00007ffff4f45fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#5 0x0000000000000000 in ?? ()
Thread 17 (Thread 0x7fffea71e700 (LWP 4168)):
#0 0x00007ffff5200bac in pthread_cond_wait@@GLIBC_2.3.2 ()
from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007ffff76e7949 in Gate::arrive(long) () from
/usr/local/lib/libmesos-0.9.0.so
#2 0x00007ffff76cf6cf in process::schedule(void*) ()
from /usr/local/lib/libmesos-0.9.0.so
#3 0x00007ffff51fbd8c in start_thread () from
/lib/x86_64-linux-gnu/libpthread.so.0
#4 0x00007ffff4f45fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#5 0x0000000000000000 in ?? ()
Thread 16 (Thread 0x7fffeaf1f700 (LWP 4167)):
#0 0x00007ffff74dbaea in mesos::internal::slave::Executor::~Executor() ()
from /usr/local/lib/libmesos-0.9.0.so
#1 0x00007ffff74ec00c in
__gnu_cxx::new_allocator<mesos::internal::slave::Executor>::destroy(mesos::internal::slave::Executor*)
() from /usr/local/lib/libmesos-0.9.0.so
#2 0x00007ffff74e3bd5 in std::_List_base<mesos::internal::slave::Executor,
std::allocator<mesos::internal::slave::Executor> >::_M_clear() ()
from /usr/local/lib/libmesos-0.9.0.so
#3 0x00007ffff74de3df in std::_List_base<mesos::internal::slave::Executor,
std::allocator<mesos::internal::slave::Executor> >::~_List_base() ()
from /usr/local/lib/libmesos-0.9.0.so
#4 0x00007ffff74dc670 in std::list<mesos::internal::slave::Executor,
std::allocator<mesos::internal::slave::Executor> >::~list() () from
/usr/local/lib/libmesos-0.9.0.so
#5 0x00007ffff74dc7fb in mesos::internal::slave::Framework::~Framework() ()
from /usr/local/lib/libmesos-0.9.0.so
#6 0x00007ffff74d87d5 in
mesos::internal::slave::Slave::shutdownExecutorTimeout(mesos::FrameworkID
const&, mesos::ExecutorID const&, UUID const&) ()
from /usr/local/lib/libmesos-0.9.0.so
#7 0x00007ffff7501313 in std::tr1::_Mem_fn<void
(mesos::internal::slave::Slave::*)(mesos::FrameworkID const&, mesos::ExecutorID
const&, UUID const&)>::operator()(mesos::internal::slave::Slave*,
mesos::FrameworkID const&, mesos::ExecutorID const&, UUID const&) const
() from /usr/local/lib/libmesos-0.9.0.so
#8 0x00007ffff74fd404 in std::tr1::result_of<std::tr1::_Mem_fn<void
(mesos::internal::slave::Slave::*)(mesos::FrameworkID const&, mesos::ExecutorID
const&, UUID const&)>
()(std::tr1::result_of<std::tr1::_Mu<std::tr1::_Placeholder<1>, false, true>
()(std::tr1::_Placeholder<1>,
std::tr1::tuple<mesos::internal::slave::Slave*&>)>::type,
std::tr1::result_of<std::tr1::_Mu<mesos::FrameworkID, false, false>
()(mesos::FrameworkID, std::tr1::_Mu<std::tr1::_Placeholder<1>, false, true>
()(std::tr1::_Placeholder<1>,
std::tr1::tuple<mesos::internal::slave::Slave*&>))>::type,
std::tr1::result_of<std::tr1::_Mu<mesos::ExecutorID, false, false>
()(mesos::ExecutorID, std::tr1::_Mu<std::tr1::_Placeholder<1>, false, true>
()(std::tr1::_Placeholder<1>,
std::tr1::tuple<mesos::internal::slave::Slave*&>))>::type,
std::tr1::result_of<std::tr1::_Mu<UUID, false, false> ()(UUID,
std::tr1::_Mu<std::tr1::_Placeholder<1>, false, true>
()(std::tr1::_Placeholder<1>,
std::tr1::tuple<mesos::internal::slave::Slave*&>))>::type)>::type
std::tr1::_Bind<std::tr1::_Mem_fn<void
(mesos::internal::slave::Slave::*)(mesos::FrameworkID const&, mesos::ExecutorID
const&, UUID const&)> ()(std::tr1::_Placeholder<1>, mesos::FrameworkID,
mesos::ExecutorID, UUID)>::__call<mesos::internal::slave::Slave*&, 0, 1, 2,
3>(std::tr1::_Mu<std::tr1::_Placeholder<1>, false, true> (
const&)(std::tr1::_Placeholder<1>,
std::tr1::tuple<mesos::internal::slave::Slave*&>), std::tr1::_Index_tuple<0, 1,
2, 3>) () from /usr/local/lib/libmesos-0.9.0.so
#9 0x00007ffff74f7956 in std::tr1::result_of<std::tr1::_Mem_fn<void
(mesos::internal::slave::Slave::*)(mesos::FrameworkID const&, mesos::ExecutorID
const&, UUID const&)>
()(std::tr1::result_of<std::tr1::_Mu<std::tr1::_Placeholder<1>, false, true>
()(std::tr1::_Placeholder<1>,
std::tr1::tuple<mesos::internal::slave::Slave*>)>::type,
std::tr1::result_of<std::tr1::_Mu<mesos::FrameworkID, false, false>
()(mesos::FrameworkID, std::tr1::_Mu<std::tr1::_Placeholder<1>, false, true>
()(std::tr1::_Placeholder<1>,
std::tr1::tuple<mesos::internal::slave::Slave*>))>::type,
std::tr1::result_of<std::tr1::_Mu<mesos::ExecutorID, false, false>
()(mesos::ExecutorID, std::tr1::_Mu<std::tr1::_Placeholder<1>, false, true>
()(std::tr1::_Placeholder<1>,
std::tr1::tuple<mesos::internal::slave::Slave*>))>::type,
std::tr1::result_of<std::tr1::_Mu<UUID, false, false> ()(UUID,
std::tr1::_Mu<std::tr1::_Placeholder<1>, false, true>
()(std::tr1::_Placeholder<1>,
std::tr1::tuple<mesos::internal::slave::Slave*>))>::type)>::type
std::tr1::_Bind<std::tr1::_Mem_fn<void
(mesos::internal::slave::Slave::*)(mesos::FrameworkID const&, mesos::ExecutorID
const&, UUID const&)> ()(std::tr1::_Placeholder<1>, mesos::FrameworkID,
mesos::ExecutorID,
UUID)>::operator()<mesos::internal::slave::Slave*>(mesos::internal::slave::Slave*&)
()
from /usr/local/lib/libmesos-0.9.0.so
#10 0x00007ffff74f12dc in std::tr1::_Function_handler<void
()(mesos::internal::slave::Slave*), std::tr1::_Bind<std::tr1::_Mem_fn<void
(mesos::internal::slave::Slave::*)(mesos::FrameworkID const&, mesos::ExecutorID
const&, UUID const&)> ()(std::tr1::_Placeholder<1>, mesos::FrameworkID,
mesos::ExecutorID, UUID)> >::_M_invoke(std::tr1::_Any_data const&,
mesos::internal::slave::Slave*) () from /usr/local/lib/libmesos-0.9.0.so
#11 0x00007ffff74ed58a in std::tr1::function<void
()(mesos::internal::slave::Slave*)>::operator()(mesos::internal::slave::Slave*)
const () from /usr/local/lib/libmesos-0.9.0.so
#12 0x00007ffff74e508d in void
process::internal::vdispatcher<mesos::internal::slave::Slave>(process::ProcessBase*,
std::tr1::shared_ptr<std::tr1::function<void
()(mesos::internal::slave::Slave*)> >) () from /usr/local/lib/libmesos-0.9.0.so
#13 0x00007ffff74f9be9 in std::tr1::result_of<void
(*()(std::tr1::result_of<std::tr1::_Mu<std::tr1::_Placeholder<1>, false, true>
()(std::tr1::_Placeholder<1>, std::tr1::tuple<process::ProcessBase*&>)>::type,
std::tr1::result_of<std::tr1::_Mu<std::tr1::shared_ptr<std::tr1::function<void
()(mesos::internal::slave::Slave*)> >, false, false>
()(std::tr1::shared_ptr<std::tr1::function<void
()(mesos::internal::slave::Slave*)> >, std::tr1::_Mu<std::tr1::_Placeholder<1>,
false, true> ()(std::tr1::_Placeholder<1>,
std::tr1::tuple<process::ProcessBase*&>))>::type))(process::ProcessBase*,
std::tr1::shared_ptr<std::tr1::function<void
()(mesos::internal::slave::Slave*)> >)>::type std::tr1::_Bind<void
(*()(std::tr1::_Placeholder<1>, std::tr1::shared_ptr<std::tr1::function<void
()(mesos::internal::slave::Slave*)> >))(process::ProcessBase*,
std::tr1::shared_ptr<std::tr1::function<void
()(mesos::internal::slave::Slave*)> >)>::__call<process::ProcessBase*&, 0,
1>(std::tr1::_Mu<std::tr1::_Placeholder<1>, false, true> (
const&)(std::tr1::_Placeholder<1>, std::tr1::tuple<process::ProcessBase*&>),
std::tr1::_Index_tuple<0, 1>) ()
from /usr/local/lib/libmesos-0.9.0.so
#14 0x00007ffff74f3ce4 in std::tr1::result_of<void
(*()(std::tr1::result_of<std::tr1::_Mu<std::tr1::_Placeholder<1>, false, true>
()(std::tr1::_Placeholder<1>, std::tr1::tuple<process::ProcessBase*>)>::type,
std::tr1::result_of<std::tr1::_Mu<std::tr1::shared_ptr<std::tr1::function<void
()(mesos::internal::slave::Slave*)> >, false, false>
()(std::tr1::shared_ptr<std::tr1::function<void
()(mesos::internal::slave::Slave*)> >, std::tr1::_Mu<std::tr1::_Placeholder<1>,
false, true> ()(std::tr1::_Placeholder<1>,
std::tr1::tuple<process::ProcessBase*>))>::type))(process::ProcessBase*,
std::tr1::shared_ptr<std::tr1::function<void
()(mesos::internal::slave::Slave*)> >)>::type std::tr1::_Bind<void
(*()(std::tr1::_Placeholder<1>, std::tr1::shared_ptr<std::tr1::function<void
()(mesos::internal::slave::Slave*)> >))(process::ProcessBase*,
std::tr1::shared_ptr<std::tr1::function<void
()(mesos::internal::slave::Slave*)>
>)>::operator()<process::ProcessBase*>(process::ProcessBase*&) () from
/usr/local/lib/libmesos-0.9.0.so
#15 0x00007ffff74ed676 in std::tr1::_Function_handler<void
()(process::ProcessBase*), std::tr1::_Bind<void (*()(std::tr1::_Placeholder<1>,
std::tr1::shared_ptr<std::tr1::function<void
()(mesos::internal::slave::Slave*)> >))(process::ProcessBase*,
std::tr1::shared_ptr<std::tr1::function<void
()(mesos::internal::slave::Slave*)> >)> >::_M_invoke(std::tr1::_Any_data
const&, process::ProcessBase*) () from /usr/local/lib/libmesos-0.9.0.so
#16 0x00007ffff76eecd0 in std::tr1::function<void
()(process::ProcessBase*)>::operator()(process::ProcessBase*) const () from
/usr/local/lib/libmesos-0.9.0.so
#17 0x00007ffff76da56b in process::ProcessBase::visit(process::DispatchEvent
const&) ()
from /usr/local/lib/libmesos-0.9.0.so
#18 0x00007ffff76df1a4 in process::DispatchEvent::visit(process::EventVisitor*)
const ()
from /usr/local/lib/libmesos-0.9.0.so
#19 0x00007ffff738a85e in process::ProcessBase::serve(process::Event const&) ()
from /usr/local/lib/libmesos-0.9.0.so
#20 0x00007ffff76d7ccb in
process::ProcessManager::resume(process::ProcessBase*) ()
from /usr/local/lib/libmesos-0.9.0.so
#21 0x00007ffff76cf6f7 in process::schedule(void*) ()
from /usr/local/lib/libmesos-0.9.0.so
#22 0x00007ffff51fbd8c in start_thread () from
/lib/x86_64-linux-gnu/libpthread.so.0
#23 0x00007ffff4f45fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#24 0x0000000000000000 in ?? ()
Thread 15 (Thread 0x7fffeb720700 (LWP 4166)):
#0 0x00007ffff5200bac in pthread_cond_wait@@GLIBC_2.3.2 ()
from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007ffff76e7949 in Gate::arrive(long) () from
/usr/local/lib/libmesos-0.9.0.so
#2 0x00007ffff76cf6cf in process::schedule(void*) ()
from /usr/local/lib/libmesos-0.9.0.so
#3 0x00007ffff51fbd8c in start_thread () from
/lib/x86_64-linux-gnu/libpthread.so.0
#4 0x00007ffff4f45fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#5 0x0000000000000000 in ?? ()
Thread 14 (Thread 0x7fffebf21700 (LWP 4165)):
#0 0x00007ffff5200bac in pthread_cond_wait@@GLIBC_2.3.2 ()
from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007ffff76e7949 in Gate::arrive(long) () from
/usr/local/lib/libmesos-0.9.0.so
#2 0x00007ffff76cf6cf in process::schedule(void*) ()
from /usr/local/lib/libmesos-0.9.0.so
#3 0x00007ffff51fbd8c in start_thread () from
/lib/x86_64-linux-gnu/libpthread.so.0
#4 0x00007ffff4f45fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#5 0x0000000000000000 in ?? ()
Thread 13 (Thread 0x7fffec722700 (LWP 4164)):
#0 0x00007ffff5200bac in pthread_cond_wait@@GLIBC_2.3.2 ()
from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007ffff76e7949 in Gate::arrive(long) () from
/usr/local/lib/libmesos-0.9.0.so
#2 0x00007ffff76cf6cf in process::schedule(void*) ()
from /usr/local/lib/libmesos-0.9.0.so
#3 0x00007ffff51fbd8c in start_thread () from
/lib/x86_64-linux-gnu/libpthread.so.0
#4 0x00007ffff4f45fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#5 0x0000000000000000 in ?? ()
Thread 12 (Thread 0x7fffecf23700 (LWP 4163)):
#0 0x00007ffff5200bac in pthread_cond_wait@@GLIBC_2.3.2 ()
from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007ffff76e7949 in Gate::arrive(long) () from
/usr/local/lib/libmesos-0.9.0.so
#2 0x00007ffff76cf6cf in process::schedule(void*) ()
from /usr/local/lib/libmesos-0.9.0.so
#3 0x00007ffff51fbd8c in start_thread () from
/lib/x86_64-linux-gnu/libpthread.so.0
#4 0x00007ffff4f45fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#5 0x0000000000000000 in ?? ()
Thread 11 (Thread 0x7fffed724700 (LWP 4162)):
#0 0x00007ffff5200bac in pthread_cond_wait@@GLIBC_2.3.2 ()
from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007ffff76e7949 in Gate::arrive(long) () from
/usr/local/lib/libmesos-0.9.0.so
#2 0x00007ffff76cf6cf in process::schedule(void*) ()
from /usr/local/lib/libmesos-0.9.0.so
#3 0x00007ffff51fbd8c in start_thread () from
/lib/x86_64-linux-gnu/libpthread.so.0
#4 0x00007ffff4f45fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#5 0x0000000000000000 in ?? ()
Thread 10 (Thread 0x7fffedf25700 (LWP 4161)):
#0 0x00007ffff5200bac in pthread_cond_wait@@GLIBC_2.3.2 ()
from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007ffff76e7949 in Gate::arrive(long) () from
/usr/local/lib/libmesos-0.9.0.so
#2 0x00007ffff76cf6cf in process::schedule(void*) ()
from /usr/local/lib/libmesos-0.9.0.so
#3 0x00007ffff51fbd8c in start_thread () from
/lib/x86_64-linux-gnu/libpthread.so.0
#4 0x00007ffff4f45fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#5 0x0000000000000000 in ?? ()
Thread 9 (Thread 0x7fffee726700 (LWP 4160)):
#0 0x00007ffff5200bac in pthread_cond_wait@@GLIBC_2.3.2 ()
from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007ffff76e7949 in Gate::arrive(long) () from
/usr/local/lib/libmesos-0.9.0.so
#2 0x00007ffff76cf6cf in process::schedule(void*) ()
from /usr/local/lib/libmesos-0.9.0.so
#3 0x00007ffff51fbd8c in start_thread () from
/lib/x86_64-linux-gnu/libpthread.so.0
#4 0x00007ffff4f45fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#5 0x0000000000000000 in ?? ()
Thread 8 (Thread 0x7fffeef27700 (LWP 4159)):
#0 0x00007ffff5200bac in pthread_cond_wait@@GLIBC_2.3.2 ()
from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007ffff76e7949 in Gate::arrive(long) () from
/usr/local/lib/libmesos-0.9.0.so
#2 0x00007ffff76cf6cf in process::schedule(void*) ()
from /usr/local/lib/libmesos-0.9.0.so
#3 0x00007ffff51fbd8c in start_thread () from
/lib/x86_64-linux-gnu/libpthread.so.0
#4 0x00007ffff4f45fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#5 0x0000000000000000 in ?? ()
Thread 7 (Thread 0x7fffef728700 (LWP 4158)):
#0 0x00007ffff5200bac in pthread_cond_wait@@GLIBC_2.3.2 ()
from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007ffff76e7949 in Gate::arrive(long) () from
/usr/local/lib/libmesos-0.9.0.so
#2 0x00007ffff76cf6cf in process::schedule(void*) ()
from /usr/local/lib/libmesos-0.9.0.so
#3 0x00007ffff51fbd8c in start_thread () from
/lib/x86_64-linux-gnu/libpthread.so.0
#4 0x00007ffff4f45fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#5 0x0000000000000000 in ?? ()
Thread 6 (Thread 0x7fffeff29700 (LWP 4157)):
#0 0x00007ffff5200bac in pthread_cond_wait@@GLIBC_2.3.2 ()
from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007ffff76e7949 in Gate::arrive(long) () from
/usr/local/lib/libmesos-0.9.0.so
#2 0x00007ffff76cf6cf in process::schedule(void*) ()
from /usr/local/lib/libmesos-0.9.0.so
#3 0x00007ffff51fbd8c in start_thread () from
/lib/x86_64-linux-gnu/libpthread.so.0
#4 0x00007ffff4f45fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#5 0x0000000000000000 in ?? ()
Thread 5 (Thread 0x7ffff072a700 (LWP 4156)):
#0 0x00007ffff5200bac in pthread_cond_wait@@GLIBC_2.3.2 ()
from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007ffff76e7949 in Gate::arrive(long) () from
/usr/local/lib/libmesos-0.9.0.so
#2 0x00007ffff76cf6cf in process::schedule(void*) ()
from /usr/local/lib/libmesos-0.9.0.so
#3 0x00007ffff51fbd8c in start_thread () from
/lib/x86_64-linux-gnu/libpthread.so.0
#4 0x00007ffff4f45fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#5 0x0000000000000000 in ?? ()
Thread 4 (Thread 0x7ffff0f2b700 (LWP 4155)):
#0 0x00007ffff5200bac in pthread_cond_wait@@GLIBC_2.3.2 ()
from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007ffff76e7949 in Gate::arrive(long) () from
/usr/local/lib/libmesos-0.9.0.so
#2 0x00007ffff76cf6cf in process::schedule(void*) ()
from /usr/local/lib/libmesos-0.9.0.so
#3 0x00007ffff51fbd8c in start_thread () from
/lib/x86_64-linux-gnu/libpthread.so.0
#4 0x00007ffff4f45fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#5 0x0000000000000000 in ?? ()
Thread 3 (Thread 0x7ffff172c700 (LWP 4154)):
#0 0x00007ffff5200bac in pthread_cond_wait@@GLIBC_2.3.2 ()
from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007ffff76e7949 in Gate::arrive(long) () from
/usr/local/lib/libmesos-0.9.0.so
#2 0x00007ffff76cf6cf in process::schedule(void*) ()
from /usr/local/lib/libmesos-0.9.0.so
#3 0x00007ffff51fbd8c in start_thread () from
/lib/x86_64-linux-gnu/libpthread.so.0
#4 0x00007ffff4f45fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#5 0x0000000000000000 in ?? ()
Thread 2 (Thread 0x7ffff1f2d700 (LWP 4153)):
#0 0x00007ffff5200bac in pthread_cond_wait@@GLIBC_2.3.2 ()
from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007ffff76e7949 in Gate::arrive(long) () from
/usr/local/lib/libmesos-0.9.0.so
#2 0x00007ffff76cf6cf in process::schedule(void*) ()
from /usr/local/lib/libmesos-0.9.0.so
#3 0x00007ffff51fbd8c in start_thread () from
/lib/x86_64-linux-gnu/libpthread.so.0
#4 0x00007ffff4f45fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#5 0x0000000000000000 in ?? ()
Thread 1 (Thread 0x7ffff7fe2760 (LWP 4149)):
#0 0x00007ffff5200bac in pthread_cond_wait@@GLIBC_2.3.2 ()
from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007ffff76e7949 in Gate::arrive(long) () from
/usr/local/lib/libmesos-0.9.0.so
#2 0x00007ffff76d8b04 in process::ProcessManager::wait(process::UPID const&) ()
from /usr/local/lib/libmesos-0.9.0.so
#3 0x00007ffff76db730 in process::wait(process::UPID const&, Duration const&)
()
from /usr/local/lib/libmesos-0.9.0.so
#4 0x0000000000434f7d in main ()
A debugging session is active.
Inferior 1 [process 4149] will be killed.
Quit anyway? (y or n)
> mesos slave crashes during framework termination
> ------------------------------------------------
>
> Key: MESOS-303
> URL: https://issues.apache.org/jira/browse/MESOS-303
> Project: Mesos
> Issue Type: Bug
> Environment: Ubuntu 11.04
> Reporter: Erich Nachbar
> Priority: Critical
>
> Hi,
> I'm running Spark 0.6.0 on Mesos trunk (5230fea125b0b) and see my mesos
> slaves terminating when a Spark job is aborted (CTRL-C).
> The logs only show a Segfault message, but I obtained a backtrace through gdb
> to give a little more context.
> Mesos passes all checks (make check) except for the linux container.
> Mesos was built using: ./configure.ubuntu-natty-64 --with-zookeeper
> --with-webui
> Mesos slave command: mesos-slave --master=zk://szk0:2181/mesos
> Here are the last few lines leading up to the segfault using gdb:
> 2012-10-31
> 22:15:35,698:29485(0x7fffe0ac5700):ZOO_DEBUG@zookeeper_process@1983: Got ping
> response in 0 ms
> 2012-10-31
> 22:15:39,047:29485(0x7fffe0ac5700):ZOO_DEBUG@zookeeper_process@1983: Got ping
> response in 13 ms
> 2012-10-31
> 22:15:42,385:29485(0x7fffe0ac5700):ZOO_DEBUG@zookeeper_process@1983: Got ping
> response in 15 ms
> I1031 22:15:45.434877 29511 slave.cpp:652] Asked to shut down framework
> 201210312057-1560611338-5050-24091-0009
> I1031 22:15:45.435017 29511 slave.cpp:656] Shutting down framework
> 201210312057-1560611338-5050-24091-0009
> I1031 22:15:45.435387 29511 slave.cpp:1102] Shutting down executor 'default'
> of framework 201210312057-1560611338-5050-24091-0009
> 2012-10-31
> 22:15:45,707:29485(0x7fffe0ac5700):ZOO_DEBUG@zookeeper_process@1983: Got ping
> response in 0 ms
> 2012-10-31
> 22:15:49,044:29485(0x7fffe0ac5700):ZOO_DEBUG@zookeeper_process@1983: Got ping
> response in 0 ms
> I1031 22:15:50.437018 29495 slave.cpp:1131] Killing executor 'default' of
> framework 201210312057-1560611338-5050-24091-0009
> I1031 22:15:50.439749 29502 gc.cpp:97] Scheduling
> /tmp/mesos/slaves/201210312057-1560611338-5050-24091-22/frameworks/201210312057-1560611338-5050-24091-0009/executors/default/runs/74aa6767-e45c-40db-8bfd-5aaf9960fabe
> for removal
> /usr/local/libexec/mesos/killtree.sh: line 229: echo: write error: Broken pipe
> /usr/local/libexec/mesos/killtree.sh: line 135: echo: write error: Broken pipe
> root@shd0:~/mesos_git# /usr/local/libexec/mesos/killtree.sh: line 124:
> printf: write error: Broken pipe
> /usr/local/libexec/mesos/killtree.sh: line 124: printf: write error: Broken
> pipe
> /usr/local/libexec/mesos/killtree.sh: line 229: echo: write error: Broken pipe
> -------------------------------------------------------------------------------------------
> Here is the backtrace from gdb:
> #0 0x0000000000000000 in ?? ()
> #1 0x00007ffff74dbaf6 in mesos::internal::slave::Executor::~Executor() ()
> from /usr/local/lib/libmesos-0.9.0.so
> #2 0x00007ffff74ec00c in
> __gnu_cxx::new_allocator<mesos::internal::slave::Executor>::destroy(mesos::internal::slave::Executor*)
> () from /usr/local/lib/libmesos-0.9.0.so
> #3 0x00007ffff74e3bd5 in std::_List_base<mesos::internal::slave::Executor,
> std::allocator<mesos::internal::slave::Executor> >::_M_clear() ()
> from /usr/local/lib/libmesos-0.9.0.so
> #4 0x00007ffff74de3df in std::_List_base<mesos::internal::slave::Executor,
> std::allocator<mesos::internal::slave::Executor> >::~_List_base() ()
> from /usr/local/lib/libmesos-0.9.0.so
> #5 0x00007ffff74dc670 in std::list<mesos::internal::slave::Executor,
> std::allocator<mesos::internal::slave::Executor> >::~list() () from
> /usr/local/lib/libmesos-0.9.0.so
> #6 0x00007ffff74dc7fb in mesos::internal::slave::Framework::~Framework() ()
> from /usr/local/lib/libmesos-0.9.0.so
> #7 0x00007ffff74d87d5 in
> mesos::internal::slave::Slave::shutdownExecutorTimeout(mesos::FrameworkID
> const&, mesos::ExecutorID const&, UUID const&) ()
> from /usr/local/lib/libmesos-0.9.0.so
> #8 0x00007ffff7501313 in std::tr1::_Mem_fn<void
> (mesos::internal::slave::Slave::*)(mesos::FrameworkID const&,
> mesos::ExecutorID const&, UUID
> const&)>::operator()(mesos::internal::slave::Slave*, mesos::FrameworkID
> const&, mesos::ExecutorID const&, UUID const&) const
> () from /usr/local/lib/libmesos-0.9.0.so
> #9 0x00007ffff74fd404 in std::tr1::result_of<std::tr1::_Mem_fn<void
> (mesos::internal::slave::Slave::*)(mesos::FrameworkID const&,
> mesos::ExecutorID const&, UUID const&)>
> ()(std::tr1::result_of<std::tr1::_Mu<std::tr1::_Placeholder<1>, false, true>
> ()(std::tr1::_Placeholder<1>,
> std::tr1::tuple<mesos::internal::slave::Slave*&>)>::type,
> std::tr1::result_of<std::tr1::_Mu<mesos::FrameworkID, false, false>
> ()(mesos::FrameworkID, std::tr1::_Mu<std::tr1::_Placeholder<1>, false, true>
> ()(std::tr1::_Placeholder<1>,
> std::tr1::tuple<mesos::internal::slave::Slave*&>))>::type,
> std::tr1::result_of<std::tr1::_Mu<mesos::ExecutorID, false, false>
> ()(mesos::ExecutorID, std::tr1::_Mu<std::tr1::_Placeholder<1>, false, true>
> ()(std::tr1::_Placeholder<1>,
> std::tr1::tuple<mesos::internal::slave::Slave*&>))>::type,
> std::tr1::result_of<std::tr1::_Mu<UUID, false, false> ()(UUID,
> std::tr1::_Mu<std::tr1::_Placeholder<1>, false, true>
> ()(std::tr1::_Placeholder<1>,
> std::tr1::tuple<mesos::internal::slave::Slave*&>))>::type)>::type
> std::tr1::_Bind<std::tr1::_Mem_fn<void
> (mesos::internal::slave::Slave::*)(mesos::FrameworkID const&,
> mesos::ExecutorID const&, UUID const&)> ()(std::tr1::_Placeholder<1>,
> mesos::FrameworkID, mesos::ExecutorID,
> UUID)>::__call<mesos::internal::slave::Slave*&, 0, 1, 2,
> 3>(std::tr1::_Mu<std::tr1::_Placeholder<1>, false, true> (
> const&)(std::tr1::_Placeholder<1>,
> std::tr1::tuple<mesos::internal::slave::Slave*&>), std::tr1::_Index_tuple<0,
> 1, 2, 3>) () from /usr/local/lib/libmesos-0.9.0.so
> #10 0x00007ffff74f7956 in std::tr1::result_of<std::tr1::_Mem_fn<void
> (mesos::internal::slave::Slave::*)(mesos::FrameworkID const&,
> mesos::ExecutorID const&, UUID const&)>
> ()(std::tr1::result_of<std::tr1::_Mu<std::tr1::_Placeholder<1>, false, true>
> ()(std::tr1::_Placeholder<1>,
> std::tr1::tuple<mesos::internal::slave::Slave*>)>::type,
> std::tr1::result_of<std::tr1::_Mu<mesos::FrameworkID, false, false>
> ()(mesos::FrameworkID, std::tr1::_Mu<std::tr1::_Placeholder<1>, false, true>
> ()(std::tr1::_Placeholder<1>,
> std::tr1::tuple<mesos::internal::slave::Slave*>))>::type,
> std::tr1::result_of<std::tr1::_Mu<mesos::ExecutorID, false, false>
> ()(mesos::ExecutorID, std::tr1::_Mu<std::tr1::_Placeholder<1>, false, true>
> ()(std::tr1::_Placeholder<1>,
> std::tr1::tuple<mesos::internal::slave::Slave*>))>::type,
> std::tr1::result_of<std::tr1::_Mu<UUID, false, false> ()(UUID,
> std::tr1::_Mu<std::tr1::_Placeholder<1>, false, true>
> ()(std::tr1::_Placeholder<1>,
> std::tr1::tuple<mesos::internal::slave::Slave*>))>::type)>::type
> std::tr1::_Bind<std::tr1::_Mem_fn<void
> (mesos::internal::slave::Slave::*)(mesos::FrameworkID const&,
> mesos::ExecutorID const&, UUID const&)> ()(std::tr1::_Placeholder<1>,
> mesos::FrameworkID, mesos::ExecutorID,
> UUID)>::operator()<mesos::internal::slave::Slave*>(mesos::internal::slave::Slave*&)
> ()
> from /usr/local/lib/libmesos-0.9.0.so
> #11 0x00007ffff74f12dc in std::tr1::_Function_handler<void
> ()(mesos::internal::slave::Slave*), std::tr1::_Bind<std::tr1::_Mem_fn<void
> (mesos::internal::slave::Slave::*)(mesos::FrameworkID const&,
> mesos::ExecutorID const&, UUID const&)> ()(std::tr1::_Placeholder<1>,
> mesos::FrameworkID, mesos::ExecutorID, UUID)>
> >::_M_invoke(std::tr1::_Any_data const&, mesos::internal::slave::Slave*) ()
> from /usr/local/lib/libmesos-0.9.0.so
> #12 0x00007ffff74ed58a in std::tr1::function<void
> ()(mesos::internal::slave::Slave*)>::operator()(mesos::internal::slave::Slave*)
> const () from /usr/local/lib/libmesos-0.9.0.so
> #13 0x00007ffff74e508d in void
> process::internal::vdispatcher<mesos::internal::slave::Slave>(process::ProcessBase*,
> std::tr1::shared_ptr<std::tr1::function<void
> ()(mesos::internal::slave::Slave*)> >) () from
> /usr/local/lib/libmesos-0.9.0.so
> #14 0x00007ffff74f9be9 in std::tr1::result_of<void
> (*()(std::tr1::result_of<std::tr1::_Mu<std::tr1::_Placeholder<1>, false,
> true> ()(std::tr1::_Placeholder<1>,
> std::tr1::tuple<process::ProcessBase*&>)>::type,
> std::tr1::result_of<std::tr1::_Mu<std::tr1::shared_ptr<std::tr1::function<void
> ()(mesos::internal::slave::Slave*)> >, false, false>
> ()(std::tr1::shared_ptr<std::tr1::function<void
> ()(mesos::internal::slave::Slave*)> >,
> std::tr1::_Mu<std::tr1::_Placeholder<1>, false, true>
> ()(std::tr1::_Placeholder<1>,
> std::tr1::tuple<process::ProcessBase*&>))>::type))(process::ProcessBase*,
> std::tr1::shared_ptr<std::tr1::function<void
> ()(mesos::internal::slave::Slave*)> >)>::type std::tr1::_Bind<void
> (*()(std::tr1::_Placeholder<1>, std::tr1::shared_ptr<std::tr1::function<void
> ()(mesos::internal::slave::Slave*)> >))(process::ProcessBase*,
> std::tr1::shared_ptr<std::tr1::function<void
> ()(mesos::internal::slave::Slave*)> >)>::__call<process::ProcessBase*&, 0,
> 1>(std::tr1::_Mu<std::tr1::_Placeholder<1>, false, true> (
> const&)(std::tr1::_Placeholder<1>, std::tr1::tuple<process::ProcessBase*&>),
> std::tr1::_Index_tuple<0, 1>) ()
> from /usr/local/lib/libmesos-0.9.0.so
> #15 0x00007ffff74f3ce4 in std::tr1::result_of<void
> (*()(std::tr1::result_of<std::tr1::_Mu<std::tr1::_Placeholder<1>, false,
> true> ()(std::tr1::_Placeholder<1>,
> std::tr1::tuple<process::ProcessBase*>)>::type,
> std::tr1::result_of<std::tr1::_Mu<std::tr1::shared_ptr<std::tr1::function<void
> ()(mesos::internal::slave::Slave*)> >, false, false>
> ()(std::tr1::shared_ptr<std::tr1::function<void
> ()(mesos::internal::slave::Slave*)> >,
> std::tr1::_Mu<std::tr1::_Placeholder<1>, false, true>
> ()(std::tr1::_Placeholder<1>,
> std::tr1::tuple<process::ProcessBase*>))>::type))(process::ProcessBase*,
> std::tr1::shared_ptr<std::tr1::function<void
> ()(mesos::internal::slave::Slave*)> >)>::type std::tr1::_Bind<void
> (*()(std::tr1::_Placeholder<1>, std::tr1::shared_ptr<std::tr1::function<void
> ()(mesos::internal::slave::Slave*)> >))(process::ProcessBase*,
> std::tr1::shared_ptr<std::tr1::function<void
> ()(mesos::internal::slave::Slave*)>
> >)>::operator()<process::ProcessBase*>(process::ProcessBase*&) () from
> /usr/local/lib/libmesos-0.9.0.so
> #16 0x00007ffff74ed676 in std::tr1::_Function_handler<void
> ()(process::ProcessBase*), std::tr1::_Bind<void
> (*()(std::tr1::_Placeholder<1>, std::tr1::shared_ptr<std::tr1::function<void
> ()(mesos::internal::slave::Slave*)> >))(process::ProcessBase*,
> std::tr1::shared_ptr<std::tr1::function<void
> ()(mesos::internal::slave::Slave*)> >)> >::_M_invoke(std::tr1::_Any_data
> const&, process::ProcessBase*) () from /usr/local/lib/libmesos-0.9.0.so
> #17 0x00007ffff76eecd0 in std::tr1::function<void
> ()(process::ProcessBase*)>::operator()(process::ProcessBase*) const () from
> /usr/local/lib/libmesos-0.9.0.so
> #18 0x00007ffff76da56b in process::ProcessBase::visit(process::DispatchEvent
> const&) ()
> from /usr/local/lib/libmesos-0.9.0.so
> #19 0x00007ffff76df1a4 in
> process::DispatchEvent::visit(process::EventVisitor*) const ()
> from /usr/local/lib/libmesos-0.9.0.so
> #20 0x00007ffff738a85e in process::ProcessBase::serve(process::Event const&)
> ()
> from /usr/local/lib/libmesos-0.9.0.so
> #21 0x00007ffff76d7ccb in
> process::ProcessManager::resume(process::ProcessBase*) ()
> from /usr/local/lib/libmesos-0.9.0.so
> #22 0x00007ffff76cf6f7 in process::schedule(void*) ()
> from /usr/local/lib/libmesos-0.9.0.so
> #23 0x00007ffff51fbd8c in start_thread () from
> /lib/x86_64-linux-gnu/libpthread.so.0
> #24 0x00007ffff4f45fdd in clone () from /lib/x86_64-linux-gnu/libc.so.6
> #25 0x0000000000000000 in ?? ()
> A debugging session is active.
> I discussed with Florian the issue and did some investigations into the code.
> It seems that the problematic section of the code has received some fairly
> major patch:
> diff --git a/src/slave/process_based_isolation_module.cpp
> b/src/slave/process_based_isolation_module.cpp
> index 7448326..b0b6a81 100644
> --- a/src/slave/process_based_isolation_module.cpp
> +++ b/src/slave/process_based_isolation_module.cpp
> @@ -18,6 +18,7 @@
> #include <errno.h>
> #include <signal.h>
> +#include <stdio.h> // For perror.
> #include <string.h>
> #include <map>
> @@ -150,29 +151,33 @@ void ProcessBasedIsolationModule::launchExecutor(
> dispatch(slave, &Slave::executorStarted,
> frameworkId, executorId, pid);
> } else {
> - // In child process, make cleanup easier.
> + // In child process, we make cleanup easier by putting process
> + // into it's own session. DO NOT USE GLOG!
> + close(pipes[0]);
> +
> // NOTE: We setsid() in a loop because setsid() might fail if another
> // process has the same process group id as the calling process.
> - close(pipes[0]);
> while ((pid = setsid()) == -1) {
> - PLOG(ERROR) << "Could not put executor in own session, "
> - << "forking another process and retrying";
> + perror("Could not put executor in own session");
> +
> + std::cerr << "Forking another process and retrying ..." << std::endl;
> if ((pid = fork()) == -1) {
> - LOG(ERROR) << "Failed to fork to launch executor";
> - exit(-1);
> + perror("Failed to fork to launch executor");
> + abort();
> }
> if (pid) {
> // In parent process.
> // It is ok to suicide here, though process reaper signals the exit,
> // because the process isolation module ignores unknown processes.
> - exit(-1);
> + exit(0);
> }
> }
> if (write(pipes[1], &pid, sizeof(pid)) != sizeof(pid)) {
> - PLOG(FATAL) << "Failed to write PID on pipe";
> + perror("Failed to write PID on pipe");
> + abort();
> }
> close(pipes[1]);
> @@ -182,7 +187,8 @@ void ProcessBasedIsolationModule::launchExecutor(
> executorInfo, directory);
> if
> -----------------------------------------
> We are a bit with our backs against the wall due to the fact that the old
> released Mesos 0.9 requires restarting the whole cluster in case of a master
> failure (which we have had a few) losing all running jobs.
--
This message is automatically generated by JIRA.
If you think it was sent incorrectly, please contact your JIRA administrators
For more information on JIRA, see: http://www.atlassian.com/software/jira
