Peter Rozsa created KUDU-3735:
---------------------------------
Summary: Data race in stack trace collection in debug-util.cc
Key: KUDU-3735
URL: https://issues.apache.org/jira/browse/KUDU-3735
Project: Kudu
Issue Type: Bug
Components: util
Reporter: Peter Rozsa
ThreadSanitizer detects a data race in kudu::stack_trace_internal::SignalData
during concurrent stack trace collection. The race occurs between atomic
variable initialization in one thread and atomic operations on the same
variable in another thread.
The actual call is made from Impala, but it can occur in different environments
too.
{code:java}
WARNING: ThreadSanitizer: data race (pid=747181) Atomic write of size 8 at
0x7b080011e0e8 by thread T54 (mutexes: write M1007816000152797856, write
M1013445224811605968): #0 __tsan_atomic64_compare_exchange_val <null>
(data-stream-test+0x24458a6) #1
std::__atomic_base<long>::compare_exchange_strong(long&, long,
std::memory_order, std::memory_order)
/gcc-10.4.0/lib/gcc/x86_64-pc-linux-gnu/10.4.0/../../../../include/c++/10.4.0/bits/atomic_base.h:509:9
(data-stream-test+0x2bf1bb1) #2
std::__atomic_base<long>::compare_exchange_strong(long&, long,
std::memory_order)
/gcc-10.4.0/lib/gcc/x86_64-pc-linux-gnu/10.4.0/../../../../include/c++/10.4.0/bits/atomic_base.h:533
(data-stream-test+0x2bf1bb1) #3 kudu::(anonymous
namespace)::HandleStackTraceSignal(int, siginfo_t*, void*)
repos/Impala/be/src/kudu/util/debug-util.cc:277 (data-stream-test+0x2bf1bb1)
#4 __tsan::ProcessPendingSignals(__tsan::ThreadState*) <null>
(data-stream-test+0x23f2810) #5 __gthread_mutex_lock(pthread_mutex_t*)
/gcc-10.4.0/lib/gcc/x86_64-pc-linux-gnu/10.4.0/../../../../include/c++/10.4.0/x86_64-pc-linux-gnu/bits/gthr-default.h:749:12
(data-stream-test+0x2483276) #6 std::mutex::lock()
/gcc-10.4.0/lib/gcc/x86_64-pc-linux-gnu/10.4.0/../../../../include/c++/10.4.0/bits/std_mutex.h:100:17
(data-stream-test+0x248bda9) #7
std::lock_guard<std::mutex>::lock_guard(std::mutex&)
/gcc-10.4.0/lib/gcc/x86_64-pc-linux-gnu/10.4.0/../../../../include/c++/10.4.0/bits/std_mutex.h:159:19
(data-stream-test+0x248b488) #8
std::_V2::condition_variable_any::notify_one()
/gcc-10.4.0/lib/gcc/x86_64-pc-linux-gnu/10.4.0/../../../../include/c++/10.4.0/condition_variable:303:25
(data-stream-test+0x2ee7766) #9
impala::KrpcDataStreamSender::Channel::MarkDone(impala::Status const&)
repos/Impala/be/src/runtime/krpc-data-stream-sender.cc:406:16
(data-stream-test+0x2eeed88) #10
impala::KrpcDataStreamSender::Channel::TransmitDataCompleteCb()
repos/Impala/be/src/runtime/krpc-data-stream-sender.cc:543:5
(data-stream-test+0x2eeff17) #11 boost::_mfi::mf0<void,
impala::KrpcDataStreamSender::Channel>::operator()(impala::KrpcDataStreamSender::Channel*)
const /boost-1.74.0-p1/include/boost/bind/mem_fn_template.hpp:49:29
(data-stream-test+0x2f056a6) #12 void
boost::_bi::list1<boost::_bi::value<impala::KrpcDataStreamSender::Channel*>
>::operator()<boost::_mfi::mf0<void, impala::KrpcDataStreamSender::Channel>,
boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf0<void,
impala::KrpcDataStreamSender::Channel>&, boost::_bi::list0&, int)
/boost-1.74.0-p1/include/boost/bind/bind.hpp:259:9 (data-stream-test+0x2f055fa)
#13 boost::_bi::bind_t<void, boost::_mfi::mf0<void,
impala::KrpcDataStreamSender::Channel>,
boost::_bi::list1<boost::_bi::value<impala::KrpcDataStreamSender::Channel*> >
>::operator()() /boost-1.74.0-p1/include/boost/bind/bind.hpp:1294:16
(data-stream-test+0x2f05583) #14 void std::__invoke_impl<void,
boost::_bi::bind_t<void, boost::_mfi::mf0<void,
impala::KrpcDataStreamSender::Channel>,
boost::_bi::list1<boost::_bi::value<impala::KrpcDataStreamSender::Channel*> >
>&>(std::__invoke_other, boost::_bi::bind_t<void, boost::_mfi::mf0<void,
impala::KrpcDataStreamSender::Channel>,
boost::_bi::list1<boost::_bi::value<impala::KrpcDataStreamSender::Channel*> >
>&)
/gcc-10.4.0/lib/gcc/x86_64-pc-linux-gnu/10.4.0/../../../../include/c++/10.4.0/bits/invoke.h:60:14
(data-stream-test+0x2f05501) #15 std::enable_if<is_invocable_r_v<void,
boost::_bi::bind_t<void, boost::_mfi::mf0<void,
impala::KrpcDataStreamSender::Channel>,
boost::_bi::list1<boost::_bi::value<impala::KrpcDataStreamSender::Channel*> >
>&>, void>::type std::__invoke_r<void, boost::_bi::bind_t<void,
boost::_mfi::mf0<void, impala::KrpcDataStreamSender::Channel>,
boost::_bi::list1<boost::_bi::value<impala::KrpcDataStreamSender::Channel*> >
>&>(boost::_bi::bind_t<void, boost::_mfi::mf0<void,
impala::KrpcDataStreamSender::Channel>,
boost::_bi::list1<boost::_bi::value<impala::KrpcDataStreamSender::Channel*> >
>&)
/gcc-10.4.0/lib/gcc/x86_64-pc-linux-gnu/10.4.0/../../../../include/c++/10.4.0/bits/invoke.h:110:2
(data-stream-test+0x2f05481) #16 std::_Function_handler<void (),
boost::_bi::bind_t<void, boost::_mfi::mf0<void,
impala::KrpcDataStreamSender::Channel>,
boost::_bi::list1<boost::_bi::value<impala::KrpcDataStreamSender::Channel*> > >
>::_M_invoke(std::_Any_data const&)
/gcc-10.4.0/lib/gcc/x86_64-pc-linux-gnu/10.4.0/../../../../include/c++/10.4.0/bits/std_function.h:291:9
(data-stream-test+0x2f052f1) #17 std::function<void ()>::operator()() const
/gcc-10.4.0/lib/gcc/x86_64-pc-linux-gnu/10.4.0/../../../../include/c++/10.4.0/bits/std_function.h:622:14
(data-stream-test+0x2c3eedc) #18 kudu::rpc::OutboundCall::CallCallback()
repos/Impala/be/src/kudu/rpc/outbound_call.cc:309:5
(data-stream-test+0x2d1b644) #19
kudu::rpc::OutboundCall::SetResponse(std::unique_ptr<kudu::rpc::CallResponse,
std::default_delete<kudu::rpc::CallResponse> >)
repos/Impala/be/src/kudu/rpc/outbound_call.cc:341:5
(data-stream-test+0x2d1b85e) #20
kudu::rpc::Connection::HandleCallResponse(std::unique_ptr<kudu::rpc::InboundTransfer,
std::default_delete<kudu::rpc::InboundTransfer> >)
repos/Impala/be/src/kudu/rpc/connection.cc:744:14 (data-stream-test+0x2cfa1de)
#21 kudu::rpc::Connection::ReadHandler(ev::io&, int)
repos/Impala/be/src/kudu/rpc/connection.cc:682:7 (data-stream-test+0x2cf9a2f)
#22 void ev::base<ev_io, ev::io>::method_thunk<kudu::rpc::Connection,
&kudu::rpc::Connection::ReadHandler>(ev_loop*, ev_io*, int)
/libev-4.20-p1/include/ev++.h:479:7 (data-stream-test+0x2d04e1a) #23
ev_invoke_pending /mnt/source/libev/libev-4.20-p1/ev.c:3155:11
(data-stream-test+0x571c222) #24 ev_run
/mnt/source/libev/libev-4.20-p1/ev.c:3555:7 (data-stream-test+0x571f574) #25
ev_run /mnt/source/libev/libev-4.20-p1/ev.c:3402:1 (data-stream-test+0x571f574)
#26 kudu::rpc::ReactorThread::RunThread()
repos/Impala/be/src/kudu/rpc/reactor.cc:503:9 (data-stream-test+0x2d232b4)
#27 kudu::rpc::ReactorThread::Init()::$_0::operator()() const
repos/Impala/be/src/kudu/rpc/reactor.cc:194:48 (data-stream-test+0x2d253c1)
#28 void std::__invoke_impl<void,
kudu::rpc::ReactorThread::Init()::$_0&>(std::__invoke_other,
kudu::rpc::ReactorThread::Init()::$_0&)
/gcc-10.4.0/lib/gcc/x86_64-pc-linux-gnu/10.4.0/../../../../include/c++/10.4.0/bits/invoke.h:60:14
(data-stream-test+0x2d25351) #29 std::enable_if<is_invocable_r_v<void,
kudu::rpc::ReactorThread::Init()::$_0&>, void>::type std::__invoke_r<void,
kudu::rpc::ReactorThread::Init()::$_0&>(kudu::rpc::ReactorThread::Init()::$_0&)
/gcc-10.4.0/lib/gcc/x86_64-pc-linux-gnu/10.4.0/../../../../include/c++/10.4.0/bits/invoke.h:110:2
(data-stream-test+0x2d252d1) #30 std::_Function_handler<void (),
kudu::rpc::ReactorThread::Init()::$_0>::_M_invoke(std::_Any_data const&)
/gcc-10.4.0/lib/gcc/x86_64-pc-linux-gnu/10.4.0/../../../../include/c++/10.4.0/bits/std_function.h:291:9
(data-stream-test+0x2d251a1) #31 std::function<void ()>::operator()() const
/gcc-10.4.0/lib/gcc/x86_64-pc-linux-gnu/10.4.0/../../../../include/c++/10.4.0/bits/std_function.h:622:14
(data-stream-test+0x2c3eedc) #32 kudu::Thread::SuperviseThread(void*)
repos/Impala/be/src/kudu/util/thread.cc:691:3 (data-stream-test+0x2cb0e56)
Previous write of size 8 at 0x7b080011e0e8 by thread T44 (mutexes: write
M936321304528812448): #0 std::__atomic_base<long>::__atomic_base(long)
/gcc-10.4.0/lib/gcc/x86_64-pc-linux-gnu/10.4.0/../../../../include/c++/10.4.0/bits/atomic_base.h:286:58
(data-stream-test+0x2bf415d) #1 std::atomic<long>::atomic(long)
/gcc-10.4.0/lib/gcc/x86_64-pc-linux-gnu/10.4.0/../../../../include/c++/10.4.0/atomic:816:56
(data-stream-test+0x2bf4100) #2
kudu::stack_trace_internal::SignalData::SignalData()
repos/Impala/be/src/kudu/util/debug-util.cc:243:38 (data-stream-test+0x2bf331c)
#3 kudu::StackTraceCollector::TriggerAsync(long, kudu::StackTrace*)
repos/Impala/be/src/kudu/util/debug-util.cc:436:40 (data-stream-test+0x2bf0151)
#4 kudu::GetThreadStack(long, kudu::StackTrace*)
repos/Impala/be/src/kudu/util/debug-util.cc:507:3 (data-stream-test+0x2bf05fe)
#5 kudu::DumpThreadStack[abi:cxx11](long)
repos/Impala/be/src/kudu/util/debug-util.cc:514:14 (data-stream-test+0x2bf071c)
#6 kudu::KernelStackWatchdog::RunThread()
repos/Impala/be/src/kudu/util/kernel_stack_watchdog.cc:185:31
(data-stream-test+0x2cc5e3e) #7
kudu::KernelStackWatchdog::KernelStackWatchdog()::$_0::operator()() const
repos/Impala/be/src/kudu/util/kernel_stack_watchdog.cc:71:3
(data-stream-test+0x2cc6631) #8 void std::__invoke_impl<void,
kudu::KernelStackWatchdog::KernelStackWatchdog()::$_0&>(std::__invoke_other,
kudu::KernelStackWatchdog::KernelStackWatchdog()::$_0&)
/gcc-10.4.0/lib/gcc/x86_64-pc-linux-gnu/10.4.0/../../../../include/c++/10.4.0/bits/invoke.h:60:14
(data-stream-test+0x2cc65c1) #9 std::enable_if<is_invocable_r_v<void,
kudu::KernelStackWatchdog::KernelStackWatchdog()::$_0&>, void>::type
std::__invoke_r<void,
kudu::KernelStackWatchdog::KernelStackWatchdog()::$_0&>(kudu::KernelStackWatchdog::KernelStackWatchdog()::$_0&)
/gcc-10.4.0/lib/gcc/x86_64-pc-linux-gnu/10.4.0/../../../../include/c++/10.4.0/bits/invoke.h:110:2
(data-stream-test+0x2cc6541) #10 std::_Function_handler<void (),
kudu::KernelStackWatchdog::KernelStackWatchdog()::$_0>::_M_invoke(std::_Any_data
const&)
/gcc-10.4.0/lib/gcc/x86_64-pc-linux-gnu/10.4.0/../../../../include/c++/10.4.0/bits/std_function.h:291:9
(data-stream-test+0x2cc6411) #11 std::function<void ()>::operator()() const
/gcc-10.4.0/lib/gcc/x86_64-pc-linux-gnu/10.4.0/../../../../include/c++/10.4.0/bits/std_function.h:622:14
(data-stream-test+0x2c3eedc) #12 kudu::Thread::SuperviseThread(void*)
repos/Impala/be/src/kudu/util/thread.cc:691:3 (data-stream-test+0x2cb0e56){code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
