[
https://issues.apache.org/jira/browse/IMPALA-7775?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16669060#comment-16669060
]
Tim Armstrong commented on IMPALA-7775:
---------------------------------------
It looks like the heartbeat thread crashed while the statestore was being
initialised. Unclear if it's the same statestore instance or a use-after-free
(the this pointer matches).
{noformat}
Thread 1 (Thread 0x7f30fbc60700 (LWP 3021)):
#0 0x00007f310af09428 in raise () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f310af0b02a in abort () from /lib/x86_64-linux-gnu/libc.so.6
#2 0x00007f310ba6984d in __gnu_cxx::__verbose_terminate_handler() () from
/usr/lib/x86_64-linux-gnu/libstdc++.so.6
#3 0x00007f310ba676b6 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#4 0x00007f310ba67701 in std::terminate() () from
/usr/lib/x86_64-linux-gnu/libstdc++.so.6
#5 0x00007f310ba67919 in __cxa_throw () from
/usr/lib/x86_64-linux-gnu/libstdc++.so.6
#6 0x0000000001972534 in boost::throw_exception<boost::lock_error> (e=...) at
/opt/Impala-Toolchain/boost-1.57.0-p3/include/boost/throw_exception.hpp:69
#7 0x000000000197117c in boost::mutex::lock (this=0x72fbdf0) at
/opt/Impala-Toolchain/boost-1.57.0-p3/include/boost/thread/pthread/mutex.hpp:119
#8 0x0000000001a85faa in boost::lock_guard<boost::mutex>::lock_guard
(this=0x7f30fbc5eff0, m_=...) at
/opt/Impala-Toolchain/boost-1.57.0-p3/include/boost/thread/lock_guard.hpp:38
#9 0x0000000001ba42aa in impala::StatsMetric<double, 31>::Update
(this=0x72fbdd0, value=@0x7f30fbc5f120: 0.0012073819999999999) at
be/src/util/collection-metrics.h:150
#10 0x0000000001ce1164 in impala::Statestore::SendHeartbeat (this=0x7456000,
subscriber=0x7370b60) at be/src/statestore/statestore.cc:864
#11 0x0000000001ce1809 in impala::Statestore::DoSubscriberUpdate
(this=0x7456000, update_kind=impala::Statestore::UpdateKind::HEARTBEAT,
thread_id=0, update=...) at be/src/statestore/statestore.cc:920
#12 0x0000000001cfb07f in boost::_mfi::mf3<void, impala::Statestore,
impala::Statestore::UpdateKind, int,
impala::Statestore::ScheduledSubscriberUpdate const&>::operator()
(this=0x7432d20, p=0x7456000,
a1=impala::Statestore::UpdateKind::HEARTBEAT, a2=0, a3=...) at
/opt/Impala-Toolchain/boost-1.57.0-p3/include/boost/bind/mem_fn_template.hpp:393
#13 0x0000000001cf99a1 in
boost::_bi::list4<boost::_bi::value<impala::Statestore*>,
boost::_bi::value<impala::Statestore::UpdateKind>, boost::arg<1>, boost::arg<2>
>::operator()<boost::_mfi::mf3<void, impala::Statestore,
impala::Statestore::UpdateKind, int,
impala::Statestore::ScheduledSubscriberUpdate const&>, boost::_bi::list2<int&,
impala::Statestore::ScheduledSubscriberUpdate const&> > (this=0x7432d30, f=...,
a=...)
at /opt/Impala-Toolchain/boost-1.57.0-p3/include/boost/bind/bind.hpp:457
#14 0x0000000001cf7362 in boost::_bi::bind_t<void, boost::_mfi::mf3<void,
impala::Statestore, impala::Statestore::UpdateKind, int,
impala::Statestore::ScheduledSubscriberUpdate const&>,
boost::_bi::list4<boost::_bi::value<impala::Statestore*>,
boost::_bi::value<impala::Statestore::UpdateKind>, boost::arg<1>, boost::arg<2>
> >::operator()<int, impala::Statestore::ScheduledSubscriberUpdate>
(this=0x7432d20,
a1=@0x7f30fbc5f704: 0, a2=...) at
/opt/Impala-Toolchain/boost-1.57.0-p3/include/boost/bind/bind_template.hpp:89
#15 0x0000000001cf4621 in
boost::detail::function::void_function_obj_invoker2<boost::_bi::bind_t<void,
boost::_mfi::mf3<void, impala::Statestore, impala::Statestore::UpdateKind, int,
impala::Statestore::ScheduledSubscriberUpdate const&>,
boost::_bi::list4<boost::_bi::value<impala::Statestore*>,
boost::_bi::value<impala::Statestore::UpdateKind>, boost::arg<1>, boost::arg<2>
> >, void, int, impala::Statestore::ScheduledSubscriberUpdate const&>::invoke
(function_obj_ptr=..., a0=0, a1=...) at
/opt/Impala-Toolchain/boost-1.57.0-p3/include/boost/function/function_template.hpp:153
#16 0x0000000001cf0cff in boost::function2<void, int,
impala::Statestore::ScheduledSubscriberUpdate const&>::operator()
(this=0x7456720, a0=0, a1=...)
at
/opt/Impala-Toolchain/boost-1.57.0-p3/include/boost/function/function_template.hpp:767
#17 0x0000000001ceb4d3 in
impala::ThreadPool<impala::Statestore::ScheduledSubscriberUpdate>::WorkerThread
(this=0x7456700, thread_id=0) at be/src/util/thread-pool.h:153
#18 0x0000000001cfcc1f in boost::_mfi::mf1<void,
impala::ThreadPool<impala::Statestore::ScheduledSubscriberUpdate>,
int>::operator() (this=0x744f9a0, p=0x7456700, a1=0)
at
/opt/Impala-Toolchain/boost-1.57.0-p3/include/boost/bind/mem_fn_template.hpp:165
#19 0x0000000001cfc3e3 in
boost::_bi::list2<boost::_bi::value<impala::ThreadPool<impala::Statestore::ScheduledSubscriberUpdate>*>,
boost::_bi::value<int> >::operator()<boost::_mfi::mf1<void,
impala::ThreadPool<impala::Statestore::ScheduledSubscriberUpdate>, int>,
boost::_bi::list0> (this=0x744f9b0, f=..., a=...) at
/opt/Impala-Toolchain/boost-1.57.0-p3/include/boost/bind/bind.hpp:313
#20 0x0000000001cfb17f in boost::_bi::bind_t<void, boost::_mfi::mf1<void,
impala::ThreadPool<impala::Statestore::ScheduledSubscriberUpdate>, int>,
boost::_bi::list2<boost::_bi::value<impala::ThreadPool<impala::Statestore::ScheduledSubscriberUpdate>*>,
boost::_bi::value<int> > >::operator() (this=0x744f9a0) at
/opt/Impala-Toolchain/boost-1.57.0-p3/include/boost/bind/bind_template.hpp:20
#21 0x0000000001cf9c96 in
boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void,
boost::_mfi::mf1<void,
impala::ThreadPool<impala::Statestore::ScheduledSubscriberUpdate>, int>,
boost::_bi::list2<boost::_bi::value<impala::ThreadPool<impala::Statestore::ScheduledSubscriberUpdate>*>,
boost::_bi::value<int> > >, void>::invoke (function_obj_ptr=...)
at
/opt/Impala-Toolchain/boost-1.57.0-p3/include/boost/function/function_template.hpp:153
#22 0x00000000019a4f0a in boost::function0<void>::operator()
(this=0x7f30fbc5fca0) at
/opt/Impala-Toolchain/boost-1.57.0-p3/include/boost/function/function_template.hpp:767
#23 0x0000000001dc6bf3 in impala::Thread::SuperviseThread(std::string const&,
std::string const&, boost::function<void ()>, impala::ThreadDebugInfo const*,
impala::Promise<long, (impala::PromiseMode)0>*) (
name=..., category=..., functor=..., parent_thread_info=0x0,
thread_started=0x7ffcaa2e2520) at be/src/util/thread.cc:359
#24 0x0000000001dcfdcb in boost::_bi::list5<boost::_bi::value<std::string>,
boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >,
boost::_bi::value<impala::ThreadDebugInfo*>,
boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*>
>::operator()<void (*)(std::string const&, std::string const&,
boost::function<void ()>, impala::ThreadDebugInfo const*, impala::Promise<long,
(impala::Promise---Type <return> to continue, or q <return> to quit---
Mode)0>*), boost::_bi::list0>(boost::_bi::type<void>, void (*&)(std::string
const&, std::string const&, boost::function<void ()>, impala::ThreadDebugInfo
const*, impala::Promise<long, (impala::PromiseMode)0>*), boost::_bi::list0&,
int) (this=0x735b9c0,
f=@0x735b9b8: 0x1dc688c <impala::Thread::SuperviseThread(std::string
const&, std::string const&, boost::function<void ()>, impala::ThreadDebugInfo
const*, impala::Promise<long, (impala::PromiseMode)0>*)>,
a=...) at
/opt/Impala-Toolchain/boost-1.57.0-p3/include/boost/bind/bind.hpp:525
#25 0x0000000001dcfcef in boost::_bi::bind_t<void, void (*)(std::string const&,
std::string const&, boost::function<void ()>, impala::ThreadDebugInfo const*,
impala::Promise<long, (impala::PromiseMode)0>*),
boost::_bi::list5<boost::_bi::value<std::string>,
boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >,
boost::_bi::value<impala::Threa
{noformat}
{noformat}
Thread 2 (Thread 0x7f310ed448c0 (LWP 2994)):
#0 0x00007f310b0433b5 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f310baa89e8 in std::string::compare(std::string const&) const ()
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#2 0x00000000019a780d in std::operator< <char, std::char_traits<char>,
std::allocator<char> > (__lhs=..., __rhs=...)
at
/home/tarmstrong/Impala/incubator-impala/toolchain/gcc-4.9.2/include/c++/4.9.2/bits/basic_string.h:2590
#3 0x00000000019a5cef in std::less<std::string>::operator() (this=0x66711c0
<impala::MetricDefs::GetInstance()::instance>, __x=..., __y=...)
at
/home/tarmstrong/Impala/incubator-impala/toolchain/gcc-4.9.2/include/c++/4.9.2/bits/stl_function.h:371
#4 0x0000000001d6e0ba in std::_Rb_tree<std::string, std::pair<std::string
const, impala::TMetricDef>, std::_Select1st<std::pair<std::string const,
impala::TMetricDef> >, std::less<std::string>, std::allocator<st
d::pair<std::string const, impala::TMetricDef> > >::_M_lower_bound
(this=0x66711c0 <impala::MetricDefs::GetInstance()::instance>, __x=0x745de30,
__y=0x745df80, __k=...)
at
/home/tarmstrong/Impala/incubator-impala/toolchain/gcc-4.9.2/include/c++/4.9.2/bits/stl_tree.h:1261
#5 0x0000000001d6d0bb in std::_Rb_tree<std::string, std::pair<std::string
const, impala::TMetricDef>, std::_Select1st<std::pair<std::string const,
impala::TMetricDef> >, std::less<std::string>, std::allocator<st
d::pair<std::string const, impala::TMetricDef> > >::find (this=0x66711c0
<impala::MetricDefs::GetInstance()::instance>, __k=...)
at
/home/tarmstrong/Impala/incubator-impala/toolchain/gcc-4.9.2/include/c++/4.9.2/bits/stl_tree.h:1913
#6 0x0000000001d6c521 in std::map<std::string, impala::TMetricDef,
std::less<std::string>, std::allocator<std::pair<std::string const,
impala::TMetricDef> > >::find (
this=0x66711c0 <impala::MetricDefs::GetInstance()::instance>, __x=...) at
/home/tarmstrong/Impala/incubator-impala/toolchain/gcc-4.9.2/include/c++/4.9.2/bits/stl_map.h:860
#7 0x0000000001d6a65c in impala::MetricDefs::Get (key=..., arg=...) at
be/src/util/metrics.cc:70
---Type <return> to continue, or q <return> to quit---
#8 0x0000000001aa367c in impala::MetricGroup::AddGauge (this=0x7322000,
key=..., value=0, metric_def_arg=...) at be/src/util/metrics.h:324
#9 0x0000000003250fce in impala::ThriftServer::ThriftServer (this=0x7339320,
name=..., processor=..., port=0, auth_provider=0x0, metrics=0x7322000,
max_concurrent_connections=0)
at be/src/rpc/thrift-server.cc:342
#10 0x0000000001c1c5dc in impala::ThriftServerBuilder::Build
(this=0x7ffcaa2e2620, server=0x7ffcaa2e2688) at be/src/rpc/thrift-server.h:304
#11 0x0000000001cdd17c in impala::Statestore::Init (this=0x7457600,
state_store_port=0) at be/src/statestore/statestore.cc:471
#12 0x0000000001966f38 in impala::StatestoreSslTest_SmokeTest_Test::TestBody
(this=0x7438030) at be/src/statestore/statestore-test.cc:81
#13 0x000000000459d1d3 in void
testing::internal::HandleExceptionsInMethodIfSupported<testing::Test,
void>(testing::Test*, void (testing::Test::*)(), char const*) ()
#14 0x0000000004593f6a in testing::Test::Run() ()
#15 0x00000000045940b8 in testing::TestInfo::Run() ()
#16 0x0000000004594195 in testing::TestCase::Run() ()
#17 0x0000000004595418 in testing::internal::UnitTestImpl::RunAllTests() ()
#18 0x00000000045956f3 in testing::UnitTest::Run() ()
#19 0x0000000001967da9 in main (argc=2, argv=0x7ffcaa2e32d8) at
be/src/statestore/statestore-test.cc:105
{noformat}
> StatestoreSslTest crashed with mutex lock failed in pthread_mutex_lock:
> Invalid argument
> ----------------------------------------------------------------------------------------
>
> Key: IMPALA-7775
> URL: https://issues.apache.org/jira/browse/IMPALA-7775
> Project: IMPALA
> Issue Type: Bug
> Components: Backend
> Affects Versions: Impala 3.1.0
> Reporter: Tim Armstrong
> Priority: Critical
> Labels: broken-build
>
> {noformat}
> 20:17:28 [==========] Running 2 tests from 2 test cases.
> 20:17:28 [----------] Global test environment set-up.
> 20:17:28 [----------] 1 test from StatestoreTest
> 20:17:28 [ RUN ] StatestoreTest.SmokeTest
> 20:17:28 [ OK ] StatestoreTest.SmokeTest (24 ms)
> 20:17:28 [----------] 1 test from StatestoreTest (24 ms total)
> 20:17:28
> 20:17:28 [----------] 1 test from StatestoreSslTest
> 20:17:28 [ RUN ] StatestoreSslTest.SmokeTest
> 20:17:28 terminate called after throwing an instance of
> 'boost::exception_detail::clone_impl<boost::exception_detail::error_info_injector<boost::lock_error>
> >'
> 20:17:28 what(): boost: mutex lock failed in pthread_mutex_lock: Invalid
> argument
> 20:17:28 Wrote minidump to
> /home/ubuntu/Impala/logs/be_tests/minidumps/statestore-test/63ff46ee-a127-4ef6-5bccb5ba-dc73c28a.dmp
> 20:17:28 Wrote minidump to
> /home/ubuntu/Impala/logs/be_tests/minidumps/statestore-test/63ff46ee-a127-4ef6-5bccb5ba-dc73c28a.dmp
> 20:17:28
> {noformat}
> https://jenkins.impala.io/job/ubuntu-16.04-from-scratch/3441
> This smells like a lifecycle bug in the backend test.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
