lidavidm commented on pull request #9941:
URL: https://github.com/apache/arrow/pull/9941#issuecomment-819506928
ThreadSanitizer found issues still :/
<details>
<summary>arrow-threading-utility-test</summary>
```
[ RUN ]
TestRunSynchronously/TestRunSynchronously.ContinueAfterExternal/0
==================
WARNING: ThreadSanitizer: data race (pid=5400)
Write of size 8 at 0x7b2c00000498 by main thread:
#0 pthread_cond_destroy <null> (arrow-threading-utility-test+0x71ae5)
#1 arrow::internal::SerialExecutor::State::~State()
/arrow/cpp/src/arrow/util/thread_pool.cc:48:24 (libarrow.so.400+0x102b639)
#2
std::default_delete<arrow::internal::SerialExecutor::State>::operator()(arrow::internal::SerialExecutor::State*)
const
/usr/bin/../lib/gcc/x86_64-linux-gnu/9/../../../../include/c++/9/bits/unique_ptr.h:81:2
(libarrow.so.400+0x102b466)
#3 std::unique_ptr<arrow::internal::SerialExecutor::State,
std::default_delete<arrow::internal::SerialExecutor::State> >::~unique_ptr()
/usr/bin/../lib/gcc/x86_64-linux-gnu/9/../../../../include/c++/9/bits/unique_ptr.h:292:4
(libarrow.so.400+0x1028823)
#4 arrow::internal::SerialExecutor::~SerialExecutor()
/arrow/cpp/src/arrow/util/thread_pool.cc:56:36 (libarrow.so.400+0x1020aa5)
#5 arrow::Result<arrow::detail::Empty>
arrow::internal::SerialExecutor::RunInSerialExecutor<arrow::detail::Empty>(arrow::internal::FnOnce<arrow::Future<arrow::detail::Empty>
(arrow::internal::Executor*)>) /arrow/cpp/src/arrow/util/thread_pool.h:220:5
(libarrow.so.400+0x1030965)
#6 arrow::Result<arrow::detail::Empty>
arrow::internal::RunSynchronously<arrow::detail::Empty>(arrow::internal::FnOnce<arrow::Future<arrow::detail::Empty>
(arrow::internal::Executor*)>, bool)
/arrow/cpp/src/arrow/util/thread_pool.h:337:12 (libarrow.so.400+0x10295aa)
#7
arrow::internal::RunSynchronouslyVoid(arrow::internal::FnOnce<arrow::Future<arrow::detail::Empty>
(arrow::internal::Executor*)>, bool)
/arrow/cpp/src/arrow/util/thread_pool.cc:420:10 (libarrow.so.400+0x10237fb)
#8
arrow::internal::TestRunSynchronously::RunVoid(arrow::internal::FnOnce<arrow::Future<arrow::detail::Empty>
(arrow::internal::Executor*)>)
/arrow/cpp/src/arrow/util/thread_pool_test.cc:136:12
(arrow-threading-utility-test+0x2506df)
#9
arrow::internal::TestRunSynchronously_ContinueAfterExternal_Test::TestBody()
/arrow/cpp/src/arrow/util/thread_pool_test.cc:229:3
(arrow-threading-utility-test+0x22d9ec)
#10 void
testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test,
void>(testing::Test*, void (testing::Test::*)(), char const*)
/build/cpp/googletest_ep-prefix/src/googletest_ep/googletest/src/gtest.cc:2433:10
(libgtestd.so+0xabc73)
Previous read of size 8 at 0x7b2c00000498 by thread T5:
#0 pthread_cond_signal <null> (arrow-threading-utility-test+0x717c2)
#1 std::condition_variable::notify_one() <null> (libstdc++.so.6+0xd0e6c)
#2 arrow::Status
arrow::internal::Executor::Spawn<arrow::Future<arrow::detail::Empty>
arrow::internal::Executor::Transfer<arrow::detail::Empty>(arrow::Future<arrow::detail::Empty>)::'lambda'(arrow::Result<arrow::detail::Empty>
const&)::operator()(arrow::Result<arrow::detail::Empty>
const&)::'lambda'()>(arrow::detail::Empty&&, arrow::StopToken)
/arrow/cpp/src/arrow/util/thread_pool.h:86:12
(arrow-threading-utility-test+0x2763d0)
#3 arrow::Future<arrow::detail::Empty>
arrow::internal::Executor::Transfer<arrow::detail::Empty>(arrow::Future<arrow::detail::Empty>)::'lambda'(arrow::Result<arrow::detail::Empty>
const&)::operator()(arrow::Result<arrow::detail::Empty> const&)
/arrow/cpp/src/arrow/util/thread_pool.h:109:27
(arrow-threading-utility-test+0x276138)
#4
arrow::Future<arrow::detail::Empty>::Callback<arrow::Future<arrow::detail::Empty>
arrow::internal::Executor::Transfer<arrow::detail::Empty>(arrow::Future<arrow::detail::Empty>)::'lambda'(arrow::Result<arrow::detail::Empty>
const&)>::operator()() && /arrow/cpp/src/arrow/util/future.h:526:7
(arrow-threading-utility-test+0x275ffb)
#5 arrow::internal::FnOnce<void
()>::FnImpl<arrow::Future<arrow::detail::Empty>::Callback<arrow::Future<arrow::detail::Empty>
arrow::internal::Executor::Transfer<arrow::detail::Empty>(arrow::Future<arrow::detail::Empty>)::'lambda'(arrow::Result<arrow::detail::Empty>
const&)> >::invoke() /arrow/cpp/src/arrow/util/functional.h:122:42
(arrow-threading-utility-test+0x275f67)
#6 arrow::internal::FnOnce<void ()>::operator()() &&
/arrow/cpp/src/arrow/util/functional.h:110:17 (libarrow.so.400+0xf9cb80)
#7 arrow::ConcreteFutureImpl::DoMarkFinishedOrFailed(arrow::FutureState)
/arrow/cpp/src/arrow/util/future.cc:276:7 (libarrow.so.400+0xf9c86c)
#8 arrow::ConcreteFutureImpl::DoMarkFinished()
/arrow/cpp/src/arrow/util/future.cc:230:27 (libarrow.so.400+0xf98edd)
#9 arrow::FutureImpl::MarkFinished()
/arrow/cpp/src/arrow/util/future.cc:333:60 (libarrow.so.400+0xf952fa)
#10
arrow::Future<arrow::detail::Empty>::DoMarkFinished(arrow::Result<arrow::detail::Empty>)
/arrow/cpp/src/arrow/util/future.h:546:14
(arrow-threading-utility-test+0x186475)
#11 void
arrow::Future<arrow::detail::Empty>::MarkFinished<arrow::detail::Empty,
void>(arrow::Status) /arrow/cpp/src/arrow/util/future.h:336:12
(arrow-threading-utility-test+0x16f4ad)
#12 std::enable_if<(!(std::is_void<arrow::Status>::value)) &&
(!(is_future<arrow::Status>::value)), void>::type
arrow::detail::ContinueFuture::operator()<arrow::internal::TestRunSynchronously_ContinueAfterExternal_Test::TestBody()::$_6::operator()(arrow::internal::Executor*)
const::'lambda'()&, arrow::Status, arrow::Future<arrow::detail::Empty>
>(arrow::Future<arrow::detail::Empty>,
arrow::internal::TestRunSynchronously_ContinueAfterExternal_Test::TestBody()::$_6::operator()(arrow::internal::Executor*)
const::'lambda'()&) const /arrow/cpp/src/arrow/util/future.h:84:10
(arrow-threading-utility-test+0x24c6f6)
#13 void std::__invoke_impl<void, arrow::detail::ContinueFuture&,
arrow::Future<arrow::detail::Empty>&,
arrow::internal::TestRunSynchronously_ContinueAfterExternal_Test::TestBody()::$_6::operator()(arrow::internal::Executor*)
const::'lambda'()&>(std::__invoke_other, arrow::detail::ContinueFuture&,
arrow::Future<arrow::detail::Empty>&,
arrow::internal::TestRunSynchronously_ContinueAfterExternal_Test::TestBody()::$_6::operator()(arrow::internal::Executor*)
const::'lambda'()&)
/usr/bin/../lib/gcc/x86_64-linux-gnu/9/../../../../include/c++/9/bits/invoke.h:60:14
(arrow-threading-utility-test+0x24c619)
#14 std::__invoke_result<arrow::detail::ContinueFuture&,
arrow::Future<arrow::detail::Empty>&,
arrow::internal::TestRunSynchronously_ContinueAfterExternal_Test::TestBody()::$_6::operator()(arrow::internal::Executor*)
const::'lambda'()&>::type std::__invoke<arrow::detail::ContinueFuture&,
arrow::Future<arrow::detail::Empty>&,
arrow::internal::TestRunSynchronously_ContinueAfterExternal_Test::TestBody()::$_6::operator()(arrow::internal::Executor*)
const::'lambda'()&>(arrow::detail::ContinueFuture&,
arrow::Future<arrow::detail::Empty>&,
arrow::internal::TestRunSynchronously_ContinueAfterExternal_Test::TestBody()::$_6::operator()(arrow::internal::Executor*)
const::'lambda'()&)
/usr/bin/../lib/gcc/x86_64-linux-gnu/9/../../../../include/c++/9/bits/invoke.h:95:14
(arrow-threading-utility-test+0x24c49a)
#15 void std::_Bind<arrow::detail::ContinueFuture
(arrow::Future<arrow::detail::Empty>,
arrow::internal::TestRunSynchronously_ContinueAfterExternal_Test::TestBody()::$_6::operator()(arrow::internal::Executor*)
const::'lambda'())>::__call<void, 0ul, 1ul>(std::tuple<>&&,
std::_Index_tuple<0ul, 1ul>)
/usr/bin/../lib/gcc/x86_64-linux-gnu/9/../../../../include/c++/9/functional:400:11
(arrow-threading-utility-test+0x24c403)
#16 void std::_Bind<arrow::detail::ContinueFuture
(arrow::Future<arrow::detail::Empty>,
arrow::internal::TestRunSynchronously_ContinueAfterExternal_Test::TestBody()::$_6::operator()(arrow::internal::Executor*)
const::'lambda'())>::operator()<void>()
/usr/bin/../lib/gcc/x86_64-linux-gnu/9/../../../../include/c++/9/functional:482:17
(arrow-threading-utility-test+0x24c359)
#17 arrow::internal::FnOnce<void
()>::FnImpl<std::_Bind<arrow::detail::ContinueFuture
(arrow::Future<arrow::detail::Empty>,
arrow::internal::TestRunSynchronously_ContinueAfterExternal_Test::TestBody()::$_6::operator()(arrow::internal::Executor*)
const::'lambda'())> >::invoke() /arrow/cpp/src/arrow/util/functional.h:122:42
(arrow-threading-utility-test+0x24c2f7)
#18 arrow::internal::FnOnce<void ()>::operator()() &&
/arrow/cpp/src/arrow/util/functional.h:110:17 (libarrow.so.400+0xf9cb80)
#19
arrow::internal::WorkerLoop(std::shared_ptr<arrow::internal::ThreadPool::State>,
std::_List_iterator<std::thread>)
/arrow/cpp/src/arrow/util/thread_pool.cc:167:11 (libarrow.so.400+0x1026cfb)
#20
arrow::internal::ThreadPool::LaunchWorkersUnlocked(int)::$_2::operator()()
const /arrow/cpp/src/arrow/util/thread_pool.cc:321:37
(libarrow.so.400+0x1026924)
#21 void std::__invoke_impl<void,
arrow::internal::ThreadPool::LaunchWorkersUnlocked(int)::$_2>(std::__invoke_other,
arrow::internal::ThreadPool::LaunchWorkersUnlocked(int)::$_2&&)
/usr/bin/../lib/gcc/x86_64-linux-gnu/9/../../../../include/c++/9/bits/invoke.h:60:14
(libarrow.so.400+0x1026890)
#22
std::__invoke_result<arrow::internal::ThreadPool::LaunchWorkersUnlocked(int)::$_2>::type
std::__invoke<arrow::internal::ThreadPool::LaunchWorkersUnlocked(int)::$_2>(arrow::internal::ThreadPool::LaunchWorkersUnlocked(int)::$_2&&)
/usr/bin/../lib/gcc/x86_64-linux-gnu/9/../../../../include/c++/9/bits/invoke.h:95:14
(libarrow.so.400+0x10267a0)
#23 void
std::thread::_Invoker<std::tuple<arrow::internal::ThreadPool::LaunchWorkersUnlocked(int)::$_2>
>::_M_invoke<0ul>(std::_Index_tuple<0ul>)
/usr/bin/../lib/gcc/x86_64-linux-gnu/9/../../../../include/c++/9/thread:244:13
(libarrow.so.400+0x1026748)
#24
std::thread::_Invoker<std::tuple<arrow::internal::ThreadPool::LaunchWorkersUnlocked(int)::$_2>
>::operator()()
/usr/bin/../lib/gcc/x86_64-linux-gnu/9/../../../../include/c++/9/thread:251:11
(libarrow.so.400+0x10266e8)
#25
std::thread::_State_impl<std::thread::_Invoker<std::tuple<arrow::internal::ThreadPool::LaunchWorkersUnlocked(int)::$_2>
> >::_M_run()
/usr/bin/../lib/gcc/x86_64-linux-gnu/9/../../../../include/c++/9/thread:195:13
(libarrow.so.400+0x102646f)
#26 <null> <null> (libstdc++.so.6+0xd6d83)
Location is heap block of size 176 at 0x7b2c00000420 allocated by main
thread:
#0 operator new(unsigned long) <null>
(arrow-threading-utility-test+0xff3a7)
#1 arrow::internal::SerialExecutor::SerialExecutor()
/arrow/cpp/src/arrow/util/thread_pool.cc:55:43 (libarrow.so.400+0x10209a7)
#2 arrow::Result<arrow::detail::Empty>
arrow::internal::SerialExecutor::RunInSerialExecutor<arrow::detail::Empty>(arrow::internal::FnOnce<arrow::Future<arrow::detail::Empty>
(arrow::internal::Executor*)>) /arrow/cpp/src/arrow/util/thread_pool.h:220:12
(libarrow.so.400+0x1030906)
#3 arrow::Result<arrow::detail::Empty>
arrow::internal::RunSynchronously<arrow::detail::Empty>(arrow::internal::FnOnce<arrow::Future<arrow::detail::Empty>
(arrow::internal::Executor*)>, bool)
/arrow/cpp/src/arrow/util/thread_pool.h:337:12 (libarrow.so.400+0x10295aa)
#4
arrow::internal::RunSynchronouslyVoid(arrow::internal::FnOnce<arrow::Future<arrow::detail::Empty>
(arrow::internal::Executor*)>, bool)
/arrow/cpp/src/arrow/util/thread_pool.cc:420:10 (libarrow.so.400+0x10237fb)
#5
arrow::internal::TestRunSynchronously::RunVoid(arrow::internal::FnOnce<arrow::Future<arrow::detail::Empty>
(arrow::internal::Executor*)>)
/arrow/cpp/src/arrow/util/thread_pool_test.cc:136:12
(arrow-threading-utility-test+0x2506df)
#6
arrow::internal::TestRunSynchronously_ContinueAfterExternal_Test::TestBody()
/arrow/cpp/src/arrow/util/thread_pool_test.cc:229:3
(arrow-threading-utility-test+0x22d9ec)
#7 void
testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test,
void>(testing::Test*, void (testing::Test::*)(), char const*)
/build/cpp/googletest_ep-prefix/src/googletest_ep/googletest/src/gtest.cc:2433:10
(libgtestd.so+0xabc73)
Thread T5 (tid=7374, running) created by main thread at:
#0 pthread_create <null> (arrow-threading-utility-test+0x707cb)
#1 std::thread::_M_start_thread(std::unique_ptr<std::thread::_State,
std::default_delete<std::thread::_State> >, void (*)()) <null>
(libstdc++.so.6+0xd7048)
#2 arrow::internal::ThreadPool::LaunchWorkersUnlocked(int)
/arrow/cpp/src/arrow/util/thread_pool.cc:321:11 (libarrow.so.400+0x1022311)
#3 arrow::internal::ThreadPool::SpawnReal(arrow::internal::TaskHints,
arrow::internal::FnOnce<void ()>, arrow::StopToken,
arrow::internal::FnOnce<void (arrow::Status const&)>&&)
/arrow/cpp/src/arrow/util/thread_pool.cc:338:7 (libarrow.so.400+0x1022ae4)
#4 arrow::Result<arrow::Future<arrow::detail::Empty> >
arrow::internal::Executor::Submit<arrow::internal::TestRunSynchronously_ContinueAfterExternal_Test::TestBody()::$_6::operator()(arrow::internal::Executor*)
const::'lambda'(), arrow::Future<arrow::detail::Empty>
>(arrow::internal::TaskHints, arrow::StopToken,
arrow::internal::TestRunSynchronously_ContinueAfterExternal_Test::TestBody()::$_6::operator()(arrow::internal::Executor*)
const::'lambda'()&&) /arrow/cpp/src/arrow/util/thread_pool.h:149:5
(arrow-threading-utility-test+0x24b483)
#5 arrow::Result<arrow::Future<arrow::detail::Empty> >
arrow::internal::Executor::Submit<arrow::internal::TestRunSynchronously_ContinueAfterExternal_Test::TestBody()::$_6::operator()(arrow::internal::Executor*)
const::'lambda'(), arrow::Future<arrow::detail::Empty>
>(arrow::internal::TestRunSynchronously_ContinueAfterExternal_Test::TestBody()::$_6::operator()(arrow::internal::Executor*)
const::'lambda'()&&) /arrow/cpp/src/arrow/util/thread_pool.h:175:12
(arrow-threading-utility-test+0x24b173)
#6
arrow::internal::TestRunSynchronously_ContinueAfterExternal_Test::TestBody()::$_6::operator()(arrow::internal::Executor*)
const /arrow/cpp/src/arrow/util/thread_pool_test.cc:223:45
(arrow-threading-utility-test+0x24af53)
#7 arrow::internal::FnOnce<arrow::Future<arrow::detail::Empty>
(arrow::internal::Executor*)>::FnImpl<arrow::internal::TestRunSynchronously_ContinueAfterExternal_Test::TestBody()::$_6>::invoke(arrow::internal::Executor*&&)
/arrow/cpp/src/arrow/util/functional.h:122:42
(arrow-threading-utility-test+0x24ae8b)
#8 arrow::internal::FnOnce<arrow::Future<arrow::detail::Empty>
(arrow::internal::Executor*)>::operator()(arrow::internal::Executor*) &&
/arrow/cpp/src/arrow/util/functional.h:110:17 (libarrow.so.400+0x1030818)
#9 arrow::Result<arrow::detail::Empty>
arrow::internal::SerialExecutor::Run<arrow::detail::Empty>(arrow::internal::FnOnce<arrow::Future<arrow::detail::Empty>
(arrow::internal::Executor*)>) /arrow/cpp/src/arrow/util/thread_pool.h:232:22
(libarrow.so.400+0x1030b8a)
#10 arrow::Result<arrow::detail::Empty>
arrow::internal::SerialExecutor::RunInSerialExecutor<arrow::detail::Empty>(arrow::internal::FnOnce<arrow::Future<arrow::detail::Empty>
(arrow::internal::Executor*)>) /arrow/cpp/src/arrow/util/thread_pool.h:220:29
(libarrow.so.400+0x1030938)
#11 arrow::Result<arrow::detail::Empty>
arrow::internal::RunSynchronously<arrow::detail::Empty>(arrow::internal::FnOnce<arrow::Future<arrow::detail::Empty>
(arrow::internal::Executor*)>, bool)
/arrow/cpp/src/arrow/util/thread_pool.h:337:12 (libarrow.so.400+0x10295aa)
#12
arrow::internal::RunSynchronouslyVoid(arrow::internal::FnOnce<arrow::Future<arrow::detail::Empty>
(arrow::internal::Executor*)>, bool)
/arrow/cpp/src/arrow/util/thread_pool.cc:420:10 (libarrow.so.400+0x10237fb)
#13
arrow::internal::TestRunSynchronously::RunVoid(arrow::internal::FnOnce<arrow::Future<arrow::detail::Empty>
(arrow::internal::Executor*)>)
/arrow/cpp/src/arrow/util/thread_pool_test.cc:136:12
(arrow-threading-utility-test+0x2506df)
#14
arrow::internal::TestRunSynchronously_ContinueAfterExternal_Test::TestBody()
/arrow/cpp/src/arrow/util/thread_pool_test.cc:229:3
(arrow-threading-utility-test+0x22d9ec)
#15 void
testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test,
void>(testing::Test*, void (testing::Test::*)(), char const*)
/build/cpp/googletest_ep-prefix/src/googletest_ep/googletest/src/gtest.cc:2433:10
(libgtestd.so+0xabc73)
SUMMARY: ThreadSanitizer: data race
(/build/cpp/debug/arrow-threading-utility-test+0x71ae5) in pthread_cond_destroy
==================
[ OK ]
TestRunSynchronously/TestRunSynchronously.ContinueAfterExternal/0 (763 ms)
```
</details>
also in arrow-dataset-file-csv-test (too much to paste here)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
