[
https://issues.apache.org/jira/browse/KUDU-2916?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Adar Dembo resolved KUDU-2916.
------------------------------
Resolution: Duplicate
Fix Version/s: n/a
> Admin.TestDumpMemTrackers is flaky in tsan
> ------------------------------------------
>
> Key: KUDU-2916
> URL: https://issues.apache.org/jira/browse/KUDU-2916
> Project: Kudu
> Issue Type: Bug
> Affects Versions: 1.10.0
> Reporter: Hao Hao
> Priority: Major
> Fix For: n/a
>
>
> I saw a tsan failure for AdminCliTest.TestDumpMemTrackers with the following
> log:
> {noformat}
> /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/kudu-admin-test.cc:2162:
> Failure
> Value of: s.ok()
> Actual: false
> Expected: true
> Runtime error: /tmp/dist-test-taskUWtx7r/build/tsan/bin/kudu: process exited
> with non-zero status 66
> stdout:
> {"id":"root","limit":-1,"current_consumption":481,"peak_consumption":481,"child_trackers":[{"id":"server","parent_id":"root","limit":-1,"current_consumption":313,"peak_consumption":313,"child_trackers":[{"id":"result-tracker","parent_id":"server","limit":-1,"current_consumption":0,"peak_consumption":0},{"id":"log_block_manager","parent_id":"server","limit":-1,"current_consumption":48,"peak_consumption":48},{"id":"tablet-9276b163452b4b0399ff2cae579f7251","parent_id":"server","limit":-1,"current_consumption":265,"peak_consumption":265,"child_trackers":[{"id":"DeltaMemStores","parent_id":"tablet-9276b163452b4b0399ff2cae579f7251","limit":-1,"current_consumption":0,"peak_consumption":0},{"id":"MemRowSet-0","parent_id":"tablet-9276b163452b4b0399ff2cae579f7251","limit":-1,"current_consumption":265,"peak_consumption":265},{"id":"txn_tracker","parent_id":"tablet-9276b163452b4b0399ff2cae579f7251","limit":67108864,"current_consumption":0,"peak_consumption":0}]}]},{"id":"ttl-cache-sharded_fifo_cache","parent_id":"root","limit":-1,"current_consumption":0,"peak_consumption":0},{"id":"code_cache-sharded_lru_cache","parent_id":"root","limit":-1,"current_consumption":0,"peak_consumption":0},{"id":"block_cache-sharded_lru_cache","parent_id":"root","limit":-1,"current_consumption":0,"peak_consumption":0},{"id":"lbm-sharded_lru_cache","parent_id":"root","limit":-1,"current_consumption":0,"peak_consumption":0},{"id":"log_cache","parent_id":"root","limit":1073741824,"current_consumption":168,"peak_consumption":168,"child_trackers":[{"id":"log_cache:457a3168758d4f4f8f4c59e8dd179cd3:9276b163452b4b0399ff2cae579f7251","parent_id":"log_cache","limit":10485760,"current_consumption":168,"peak_consumption":168}]}]}
> stderr: W0803 14:03:00.206982 8443 flags.cc:404] Enabled unsafe flag:
> --never_fsync=true
> W0803 14:03:00.849385 8443 thread.cc:599] rpc reactor (reactor) Time spent
> creating pthread: real 0.590s user 0.230s sys 0.360s
> W0803 14:03:00.849658 8443 thread.cc:566] rpc reactor (reactor) Time spent
> starting thread: real 0.591s user 0.230s sys 0.360s
> ==================
> WARNING: ThreadSanitizer: destroy of a locked mutex (pid=8443)
> #0 pthread_rwlock_destroy
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:1313
> (kudu+0x4bbb24)
> #1 glog_internal_namespace_::Mutex::~Mutex()
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/glog-0.3.5/src/base/mutex.h:249:30
> (libglog.so.0+0x16488)
> #2 cxa_at_exit_wrapper(void*)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:386
> (kudu+0x48beb3)
> and:
> #0 operator new(unsigned long)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cc:57
> (kudu+0x52ae83)
> #1 __allocate
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/libcxx/include/new:228:10
> (libc++.so.1+0xd63f3)
> #2 allocate
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/libcxx/include/memory:1793
> (libc++.so.1+0xd63f3)
> #3 allocate
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/libcxx/include/memory:1547
> (libc++.so.1+0xd63f3)
> #4 __init
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/libcxx/include/string:1591
> (libc++.so.1+0xd63f3)
> #5 std::__1::basic_string<char, std::__1::char_traits<char>,
> std::__1::allocator<char> >::basic_string(std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> > const&)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/libcxx/include/string:1653
> (libc++.so.1+0xd63f3)
> #6 std::__1::pair<std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> > const,
> std::__1::pair<void const*, int>
> >::pair(std::__1::pair<std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> > const,
> std::__1::pair<void const*, int> > const&)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/utility:324:5
> (libprotobuf.so.14+0x188711)
> #7 void
> std::__1::allocator<std::__1::__tree_node<std::__1::__value_type<std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> >,
> std::__1::pair<void const*, int> >, void*>
> >::construct<std::__1::pair<std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> > const,
> std::__1::pair<void const*, int> >,
> std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>,
> std::__1::allocator<char> > const, std::__1::pair<void const*, int> >
> const&>(std::__1::pair<std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> > const,
> std::__1::pair<void const*, int> >*,
> std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>,
> std::__1::allocator<char> > const, std::__1::pair<void const*, int> > const&)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:1805
> (libprotobuf.so.14+0x188711)
> #8 void
> std::__1::allocator_traits<std::__1::allocator<std::__1::__tree_node<std::__1::__value_type<std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> >,
> std::__1::pair<void const*, int> >, void*> >
> >::__construct<std::__1::pair<std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> > const,
> std::__1::pair<void const*, int> >,
> std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>,
> std::__1::allocator<char> > const, std::__1::pair<void const*, int> >
> const&>(std::__1::integral_constant<bool, true>,
> std::__1::allocator<std::__1::__tree_node<std::__1::__value_type<std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> >,
> std::__1::pair<void const*, int> >, void*> >&,
> std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>,
> std::__1::allocator<char> > const, std::__1::pair<void const*, int> >*,
> std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>,
> std::__1::allocator<char> > const, std::__1::pair<void const*, int> > const&)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:1715
> (libprotobuf.so.14+0x188711)
> #9 void
> std::__1::allocator_traits<std::__1::allocator<std::__1::__tree_node<std::__1::__value_type<std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> >,
> std::__1::pair<void const*, int> >, void*> >
> >::construct<std::__1::pair<std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> > const,
> std::__1::pair<void const*, int> >,
> std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>,
> std::__1::allocator<char> > const, std::__1::pair<void const*, int> >
> const&>(std::__1::allocator<std::__1::__tree_node<std::__1::__value_type<std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> >,
> std::__1::pair<void const*, int> >, void*> >&,
> std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>,
> std::__1::allocator<char> > const, std::__1::pair<void const*, int> >*,
> std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>,
> std::__1::allocator<char> > const, std::__1::pair<void const*, int> > const&)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:1561
> (libprotobuf.so.14+0x188711)
> #10
> std::__1::unique_ptr<std::__1::__tree_node<std::__1::__value_type<std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> >,
> std::__1::pair<void const*, int> >, void*>,
> std::__1::__tree_node_destructor<std::__1::allocator<std::__1::__tree_node<std::__1::__value_type<std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> >,
> std::__1::pair<void const*, int> >, void*> > > >
> std::__1::__tree<std::__1::__value_type<std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void
> const*, int> >, std::__1::__map_value_compare<std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> >,
> std::__1::__value_type<std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void
> const*, int> >, std::__1::less<std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> > >, true>,
> std::__1::allocator<std::__1::__value_type<std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void
> const*, int> > >
> >::__construct_node<std::__1::pair<std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> > const,
> std::__1::pair<void const*, int> >
> const&>(std::__1::pair<std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> > const,
> std::__1::pair<void const*, int> > const&)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/__tree:2192
> (libprotobuf.so.14+0x188711)
> #11
> std::__1::pair<std::__1::__tree_iterator<std::__1::__value_type<std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> >,
> std::__1::pair<void const*, int> >,
> std::__1::__tree_node<std::__1::__value_type<std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void
> const*, int> >, void*>*, long>, bool>
> std::__1::__tree<std::__1::__value_type<std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void
> const*, int> >, std::__1::__map_value_compare<std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> >,
> std::__1::__value_type<std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void
> const*, int> >, std::__1::less<std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> > >, true>,
> std::__1::allocator<std::__1::__value_type<std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void
> const*, int> > > >::__emplace_unique_key_args<std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> >,
> std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>,
> std::__1::allocator<char> > const, std::__1::pair<void const*, int> >
> const&>(std::__1::basic_string<char, std::__1::char_traits<char>,
> std::__1::allocator<char> > const&,
> std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>,
> std::__1::allocator<char> > const, std::__1::pair<void const*, int> > const&)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/__tree:2137
> (libprotobuf.so.14+0x188711)
> #12 bool
> google::protobuf::InsertIfNotPresent<std::__1::map<std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> >,
> std::__1::pair<void const*, int>, std::__1::less<std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> > >,
> std::__1::allocator<std::__1::pair<std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> > const,
> std::__1::pair<void const*, int> > > >
> >(std::__1::map<std::__1::basic_string<char, std::__1::char_traits<char>,
> std::__1::allocator<char> >, std::__1::pair<void const*, int>,
> std::__1::less<std::__1::basic_string<char, std::__1::char_traits<char>,
> std::__1::allocator<char> > >,
> std::__1::allocator<std::__1::pair<std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> > const,
> std::__1::pair<void const*, int> > > >*,
> std::__1::map<std::__1::basic_string<char, std::__1::char_traits<char>,
> std::__1::allocator<char> >, std::__1::pair<void const*, int>,
> std::__1::less<std::__1::basic_string<char, std::__1::char_traits<char>,
> std::__1::allocator<char> > >,
> std::__1::allocator<std::__1::pair<std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> > const,
> std::__1::pair<void const*, int> > > >::value_type::first_type const&,
> std::__1::map<std::__1::basic_string<char, std::__1::char_traits<char>,
> std::__1::allocator<char> >, std::__1::pair<void const*, int>,
> std::__1::less<std::__1::basic_string<char, std::__1::char_traits<char>,
> std::__1::allocator<char> > >,
> std::__1::allocator<std::__1::pair<std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> > const,
> std::__1::pair<void const*, int> > > >::value_type::second_type const&)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/__tree:1273:16
> (libprotobuf.so.14+0x184c18)
> #13
> google::protobuf::SimpleDescriptorDatabase::DescriptorIndex<std::__1::pair<void
> const*, int> >::AddFile(google::protobuf::FileDescriptorProto const&,
> std::__1::pair<void const*, int>)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/descriptor_database.cc:57
> (libprotobuf.so.14+0x184c18)
> #14 google::protobuf::EncodedDescriptorDatabase::Add(void const*, int)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/descriptor_database.cc:316:19
> (libprotobuf.so.14+0x182d7c)
> #15 google::protobuf::DescriptorPool::InternalAddGeneratedFile(void
> const*, int)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/descriptor.cc:1315:3
> (libprotobuf.so.14+0x13b705)
> #16
> google::protobuf::protobuf_google_2fprotobuf_2ftype_2eproto::(anonymous
> namespace)::AddDescriptorsImpl()
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/type.pb.cc:240:3
> (libprotobuf.so.14+0x237c10)
> #17 google::protobuf::internal::FunctionClosure0::Run()
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/stubs/callback.h:129:5
> (libprotobuf.so.14+0xd330b)
> #18 google::protobuf::GoogleOnceInitImpl(long*,
> google::protobuf::Closure*)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/stubs/once.cc:83:14
> (libprotobuf.so.14+0xd5d6a)
> #19 google::protobuf::GoogleOnceInit(long*, void (*)())
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/stubs/once.h:128:5
> (libprotobuf.so.14+0x237b66)
> #20
> google::protobuf::protobuf_google_2fprotobuf_2ftype_2eproto::AddDescriptors()
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/type.pb.cc:251
> (libprotobuf.so.14+0x237b66)
> #21
> google::protobuf::protobuf_google_2fprotobuf_2fapi_2eproto::(anonymous
> namespace)::AddDescriptorsImpl()
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/api.pb.cc:179:3
> (libprotobuf.so.14+0x12ca1d)
> #22 google::protobuf::internal::FunctionClosure0::Run()
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/stubs/callback.h:129:5
> (libprotobuf.so.14+0xd330b)
> #23 google::protobuf::GoogleOnceInitImpl(long*,
> google::protobuf::Closure*)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/stubs/once.cc:83:14
> (libprotobuf.so.14+0xd5d6a)
> #24 google::protobuf::GoogleOnceInit(long*, void (*)())
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/stubs/once.h:128:5
> (libprotobuf.so.14+0xcf0a6)
> #25
> google::protobuf::protobuf_google_2fprotobuf_2fapi_2eproto::AddDescriptors()
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/api.pb.cc:185
> (libprotobuf.so.14+0xcf0a6)
> #26
> google::protobuf::protobuf_google_2fprotobuf_2fapi_2eproto::StaticDescriptorInitializer::StaticDescriptorInitializer()
>
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/api.pb.cc:190
> (libprotobuf.so.14+0xcf0a6)
> #27 __cxx_global_var_init
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/api.pb.cc:192
> (libprotobuf.so.14+0xcf0a6)
> #28 _GLOBAL__sub_I_api.pb.cc
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/api.pb.cc
> (libprotobuf.so.14+0xcf0a6)
> #29 call_init /build/eglibc-SvCtMH/eglibc-2.19/elf/dl-init.c:78
> (ld-linux-x86-64.so.2+0x102d9)
> Location is global 'google::vmodule_lock' of size 64 at 0x7f67b632f878
> (libglog.so.0+0x00000023c878)
> Mutex M134 (0x7f67b632f878) created at:
> #0 pthread_rwlock_init
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:1304
> (kudu+0x4bbd94)
> #1 glog_internal_namespace_::Mutex::Mutex()
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/glog-0.3.5/src/base/mutex.h:247:19
> (libglog.so.0+0xcca6)
> #2 __cxx_global_var_init.15
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/glog-0.3.5/src/vlog_is_on.cc:120
> (libglog.so.0+0xcca6)
> #3 _GLOBAL__sub_I_vlog_is_on.cc
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/glog-0.3.5/src/vlog_is_on.cc
> (libglog.so.0+0xcca6)
> #4 call_init /build/eglibc-SvCtMH/eglibc-2.19/elf/dl-init.c:78
> (ld-linux-x86-64.so.2+0x102d9)
> SUMMARY: ThreadSanitizer: destroy of a locked mutex
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/glog-0.3.5/src/base/mutex.h:249:30
> in glog_internal_namespace_::Mutex::~Mutex()
> ==================
> ==================
> WARNING: ThreadSanitizer: lock-order-inversion (potential deadlock) (pid=8443)
> Cycle in lock order graph: M1860 (0x7b14000167b8) => M37857730294380664
> (0x000000000000) => M1860
> Mutex M37857730294380664 acquired here while holding mutex M1860 in main
> thread:
> #0 pthread_rwlock_wrlock
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:1352
> (kudu+0x4bb154)
> #1 glog_internal_namespace_::Mutex::Lock()
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/glog-0.3.5/src/base/mutex.h:250:30
> (libglog.so.0+0x1abb7)
> #2
> glog_internal_namespace_::MutexLock::MutexLock(glog_internal_namespace_::Mutex*)
>
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/glog-0.3.5/src/base/mutex.h:290
> (libglog.so.0+0x1abb7)
> #3 google::InitVLOG3__(int**, int*, char const*, int)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/glog-0.3.5/src/vlog_is_on.cc:199
> (libglog.so.0+0x1abb7)
> #4
> kudu::rpc::Messenger::ShutdownInternal(kudu::rpc::Messenger::ShutdownMode)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/messenger.cc:283:5
> (libkrpc.so+0xa9471)
> #5 kudu::rpc::Messenger::AllExternalReferencesDropped()
> /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/messenger.cc:249:3
> (libkrpc.so+0xa9227)
> #6 std::__1::mem_fun_t<void,
> kudu::rpc::Messenger>::operator()(kudu::rpc::Messenger*) const
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/functional:1120:17
> (libkrpc.so+0xad755)
> #7 std::__1::__shared_ptr_pointer<kudu::rpc::Messenger*,
> std::__1::mem_fun_t<void, kudu::rpc::Messenger>,
> std::__1::allocator<kudu::rpc::Messenger> >::__on_zero_shared()
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:3586
> (libkrpc.so+0xad755)
> #8 std::__1::__shared_count::__release_shared()
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:3490:9
> (kudu+0x576f2e)
> #9 std::__1::__shared_weak_count::__release_shared()
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:3532
> (kudu+0x576f2e)
> #10 std::__1::shared_ptr<kudu::rpc::Messenger>::~shared_ptr()
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:4468
> (kudu+0x576f2e)
> #11 kudu::rpc::Proxy::~Proxy()
> /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/proxy.cc:72:1
> (libkrpc.so+0xbd0f3)
> #12 kudu::server::GenericServiceProxy::~GenericServiceProxy()
> /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/src/kudu/server/server_base.proxy.cc:26:1
> (libserver_base_proto.so+0x4bd79)
> #13
> std::__1::default_delete<kudu::server::GenericServiceProxy>::operator()(kudu::server::GenericServiceProxy*)
> const
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:2285:5
> (libkudu_tools_util.so+0x74acf)
> #14 std::__1::unique_ptr<kudu::server::GenericServiceProxy,
> std::__1::default_delete<kudu::server::GenericServiceProxy>
> >::reset(kudu::server::GenericServiceProxy*)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:2598
> (libkudu_tools_util.so+0x74acf)
> #15 std::__1::unique_ptr<kudu::server::GenericServiceProxy,
> std::__1::default_delete<kudu::server::GenericServiceProxy> >::~unique_ptr()
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:2552
> (libkudu_tools_util.so+0x74acf)
> #16 kudu::tools::DumpMemTrackers(std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> > const&, unsigned
> short)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_action_common.cc:629
> (libkudu_tools_util.so+0x74acf)
> #17 kudu::tools::(anonymous
> namespace)::TserverDumpMemTrackers(kudu::tools::RunnerContext const&)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_action_tserver.cc:161:10
> (kudu+0x5b8599)
> #18
> _ZNSt3__18__invokeIRPFN4kudu6StatusERKNS1_5tools13RunnerContextEEJS6_EEEDTclclsr3std3__1E7forwardIT_Efp_Espclsr3std3__1E7forwardIT0_Efp0_EEEOSA_DpOSB_
>
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/type_traits:4482:1
> (kudu+0x53956b)
> #19 kudu::Status
> std::__1::__invoke_void_return_wrapper<kudu::Status>::__call<kudu::Status
> (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext
> const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&),
> kudu::tools::RunnerContext const&)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/__functional_base:318
> (kudu+0x53956b)
> #20 std::__1::__function::__func<kudu::Status
> (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status
> (*)(kudu::tools::RunnerContext const&)>, kudu::Status
> (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext
> const&)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/functional:1562:12
> (kudu+0x53949d)
> #21 std::__1::function<kudu::Status (kudu::tools::RunnerContext
> const&)>::operator()(kudu::tools::RunnerContext const&) const
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/functional:1916:12
> (libkudu_tools_util.so+0x6b2a4)
> #22 kudu::tools::Action::Run(std::__1::vector<kudu::tools::Mode*,
> std::__1::allocator<kudu::tools::Mode*> > const&,
> std::__1::unordered_map<std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> >,
> std::__1::basic_string<char, std::__1::char_traits<char>,
> std::__1::allocator<char> >, std::__1::hash<std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> > >,
> std::__1::equal_to<std::__1::basic_string<char, std::__1::char_traits<char>,
> std::__1::allocator<char> > >,
> std::__1::allocator<std::__1::pair<std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> > const,
> std::__1::basic_string<char, std::__1::char_traits<char>,
> std::__1::allocator<char> > > > > const&,
> std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>,
> std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) const
> /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_action.cc:258:10
> (libkudu_tools_util.so+0x69974)
> #23 kudu::tools::DispatchCommand(std::__1::vector<kudu::tools::Mode*,
> std::__1::allocator<kudu::tools::Mode*> > const&, kudu::tools::Action*,
> std::__1::deque<std::__1::basic_string<char, std::__1::char_traits<char>,
> std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> > > > const&)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_main.cc:132:15
> (kudu+0x5bbcf6)
> #24 kudu::tools::RunTool(int, char**, bool)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_main.cc:204:16
> (kudu+0x5bcc51)
> #25 main
> /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_main.cc:265:10
> (kudu+0x5bcfbe)
> Hint: use TSAN_OPTIONS=second_deadlock_stack=1 to get more informative
> warning message
> Mutex M1860 acquired here while holding mutex M37857730294380664 in thread
> T7:
> #0 AnnotateRWLockAcquired
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interface_ann.cc:271
> (kudu+0x4dbd0c)
> #1 kudu::rw_spinlock::lock()
> /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/locks.h:112:5
> (libkudu_client.so+0x16d842)
> #2 kudu::percpu_rwlock::lock()
> /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/locks.h:222:22
> (libkudu_client.so+0x16d7d2)
> #3
> std::__1::lock_guard<kudu::percpu_rwlock>::lock_guard(kudu::percpu_rwlock&)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/__mutex_base:104:27
> (libkrpc.so+0xaad39)
> #4 kudu::rpc::Messenger::~Messenger()
> /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/messenger.cc:430
> (libkrpc.so+0xaad39)
> #5
> std::__1::default_delete<kudu::rpc::Messenger>::operator()(kudu::rpc::Messenger*)
> const
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:2285:5
> (libkrpc.so+0xb081b)
> #6 std::__1::__shared_ptr_pointer<kudu::rpc::Messenger*,
> std::__1::default_delete<kudu::rpc::Messenger>,
> std::__1::allocator<kudu::rpc::Messenger> >::__on_zero_shared()
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:3586
> (libkrpc.so+0xb081b)
> #7 std::__1::__shared_count::__release_shared()
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:3490:9
> (kudu+0x576f2e)
> #8 std::__1::__shared_weak_count::__release_shared()
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:3532
> (kudu+0x576f2e)
> #9 std::__1::shared_ptr<kudu::rpc::Messenger>::~shared_ptr()
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:4468
> (kudu+0x576f2e)
> #10 std::__1::shared_ptr<kudu::rpc::Messenger>::reset()
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:4603:5
> (libkrpc.so+0xbf151)
> #11 kudu::rpc::ReactorThread::RunThread()
> /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:499
> (libkrpc.so+0xbf151)
> #12 boost::_mfi::mf0<void,
> kudu::rpc::ReactorThread>::operator()(kudu::rpc::ReactorThread*) const
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/bind/mem_fn_template.hpp:49:29
> (libkrpc.so+0xc9539)
> #13 void boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*>
> >::operator()<boost::_mfi::mf0<void, kudu::rpc::ReactorThread>,
> boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf0<void,
> kudu::rpc::ReactorThread>&, boost::_bi::list0&, int)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/bind/bind.hpp:259:9
> (libkrpc.so+0xc948a)
> #14 boost::_bi::bind_t<void, boost::_mfi::mf0<void,
> kudu::rpc::ReactorThread>,
> boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*> >
> >::operator()()
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/bind/bind.hpp:1222:16
> (libkrpc.so+0xc9413)
> #15
> boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void,
> boost::_mfi::mf0<void, kudu::rpc::ReactorThread>,
> boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*> > >,
> void>::invoke(boost::detail::function::function_buffer&)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/function/function_template.hpp:159:11
> (libkrpc.so+0xc9209)
> #16 boost::function0<void>::operator()() const
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/function/function_template.hpp:770:14
> (libkrpc.so+0xb8a11)
> #17 kudu::Thread::SuperviseThread(void*)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:657:3
> (libkudu_util.so+0x1e1824)
> Thread T7 'rpc reactor-845' (tid=8451, running) created by main thread at:
> #0 pthread_create
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:992
> (kudu+0x4bd8eb)
> #1 kudu::Thread::StartThread(std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> > const&,
> std::__1::basic_string<char, std::__1::char_traits<char>,
> std::__1::allocator<char> > const&, boost::function<void ()> const&, unsigned
> long, scoped_refptr<kudu::Thread>*)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:601:15
> (libkudu_util.so+0x1e100b)
> #2 kudu::Status kudu::Thread::Create<void
> (kudu::rpc::ReactorThread::*)(),
> kudu::rpc::ReactorThread*>(std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> > const&,
> std::__1::basic_string<char, std::__1::char_traits<char>,
> std::__1::allocator<char> > const&, void (kudu::rpc::ReactorThread::*
> const&)(), kudu::rpc::ReactorThread* const&, scoped_refptr<kudu::Thread>*)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.h:164:12
> (libkrpc.so+0xc48e5)
> #3 kudu::rpc::ReactorThread::Init()
> /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:185:10
> (libkrpc.so+0xbec4e)
> #4 kudu::rpc::Reactor::Init()
> /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:759:18
> (libkrpc.so+0xc32f1)
> #5 kudu::rpc::Messenger::Init()
> /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/messenger.cc:446:5
> (libkrpc.so+0xa90e2)
> #6
> kudu::rpc::MessengerBuilder::Build(std::__1::shared_ptr<kudu::rpc::Messenger>*)
>
> /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/messenger.cc:205:3
> (libkrpc.so+0xa8b3d)
> #7 kudu::Status
> kudu::tools::BuildProxy<kudu::server::GenericServiceProxy>(std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> > const&, unsigned
> short, std::__1::unique_ptr<kudu::server::GenericServiceProxy,
> std::__1::default_delete<kudu::server::GenericServiceProxy> >*)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_action_common.cc:328:3
> (libkudu_tools_util.so+0x7c55e)
> #8 kudu::tools::DumpMemTrackers(std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> > const&, unsigned
> short)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_action_common.cc:591:3
> (libkudu_tools_util.so+0x747e3)
> #9 kudu::tools::(anonymous
> namespace)::TserverDumpMemTrackers(kudu::tools::RunnerContext const&)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_action_tserver.cc:161:10
> (kudu+0x5b8599)
> #10
> _ZNSt3__18__invokeIRPFN4kudu6StatusERKNS1_5tools13RunnerContextEEJS6_EEEDTclclsr3std3__1E7forwardIT_Efp_Espclsr3std3__1E7forwardIT0_Efp0_EEEOSA_DpOSB_
>
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/type_traits:4482:1
> (kudu+0x53956b)
> #11 kudu::Status
> std::__1::__invoke_void_return_wrapper<kudu::Status>::__call<kudu::Status
> (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext
> const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&),
> kudu::tools::RunnerContext const&)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/__functional_base:318
> (kudu+0x53956b)
> #12 std::__1::__function::__func<kudu::Status
> (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status
> (*)(kudu::tools::RunnerContext const&)>, kudu::Status
> (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext
> const&)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/functional:1562:12
> (kudu+0x53949d)
> #13 std::__1::function<kudu::Status (kudu::tools::RunnerContext
> const&)>::operator()(kudu::tools::RunnerContext const&) const
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/functional:1916:12
> (libkudu_tools_util.so+0x6b2a4)
> #14 kudu::tools::Action::Run(std::__1::vector<kudu::tools::Mode*,
> std::__1::allocator<kudu::tools::Mode*> > const&,
> std::__1::unordered_map<std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> >,
> std::__1::basic_string<char, std::__1::char_traits<char>,
> std::__1::allocator<char> >, std::__1::hash<std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> > >,
> std::__1::equal_to<std::__1::basic_string<char, std::__1::char_traits<char>,
> std::__1::allocator<char> > >,
> std::__1::allocator<std::__1::pair<std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> > const,
> std::__1::basic_string<char, std::__1::char_traits<char>,
> std::__1::allocator<char> > > > > const&,
> std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>,
> std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) const
> /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_action.cc:258:10
> (libkudu_tools_util.so+0x69974)
> #15 kudu::tools::DispatchCommand(std::__1::vector<kudu::tools::Mode*,
> std::__1::allocator<kudu::tools::Mode*> > const&, kudu::tools::Action*,
> std::__1::deque<std::__1::basic_string<char, std::__1::char_traits<char>,
> std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> > > > const&)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_main.cc:132:15
> (kudu+0x5bbcf6)
> #16 kudu::tools::RunTool(int, char**, bool)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_main.cc:204:16
> (kudu+0x5bcc51)
> #17 main
> /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_main.cc:265:10
> (kudu+0x5bcfbe)
> SUMMARY: ThreadSanitizer: lock-order-inversion (potential deadlock)
> /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/glog-0.3.5/src/base/mutex.h:250:30
> in glog_internal_namespace_::Mutex::Lock()
> ==================
> ThreadSanitizer: reported 2 warnings
> {noformat}
--
This message was sent by Atlassian JIRA
(v7.6.14#76016)
