lidavidm edited a comment on pull request #9656:
URL: https://github.com/apache/arrow/pull/9656#issuecomment-813632358
The integration failure is real & reproducible. Sometimes the global
allocator gets destructed before all threads join, and if a thread holds a
Future which holds a Buffer, when that Buffer gets destructed, it calls the
global allocator's Free()…but by then, the vtable for the constructor points
only to pure virtual methods, causing a crash. Still trying to figure out where
the future/thread came from in the first place.
<details>
<summary>Stack trace</summary>
```
#0 0x00007f79101b77f5 in raise (sig=19) at
../sysdeps/unix/sysv/linux/raise.c:46
#1 0x00005556cffaad9b in handler (sig=6) at
/arrow/cpp/src/arrow/testing/json_integration_test.cc:1190
#2 <signal handler called>
#3 __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51
#4 0x00007f790fc7f921 in __GI_abort () at abort.c:79
#5 0x00007f79100de892 in __gnu_cxx::__verbose_terminate_handler () at
/home/conda/feedstock_root/build_artifacts/ctng-compilers_1610729750655/work/.build/x86_64-conda-linux-gnu/src/gcc/libstdc++-v3/libsupc++/vterminate.cc:95
#6 0x00007f79100dcf69 in __cxxabiv1::__terminate (handler=<optimized out>)
at
/home/conda/feedstock_root/build_artifacts/ctng-compilers_1610729750655/work/.build/x86_64-conda-linux-gnu/src/gcc/libstdc++-v3/libsupc++/eh_terminate.cc:48
#7 0x00007f79100dcfab in std::terminate () at
/home/conda/feedstock_root/build_artifacts/ctng-compilers_1610729750655/work/.build/x86_64-conda-linux-gnu/src/gcc/libstdc++-v3/libsupc++/eh_terminate.cc:58
#8 0x00007f79100dd9d0 in __cxxabiv1::__cxa_pure_virtual () at
/home/conda/feedstock_root/build_artifacts/ctng-compilers_1610729750655/work/.build/x86_64-conda-linux-gnu/src/gcc/libstdc++-v3/libsupc++/pure.cc:50
#9 0x00007f7910eb8c4c in arrow::PoolBuffer::~PoolBuffer
(this=0x7f7908000b80, __in_chrg=<optimized out>) at
/arrow/cpp/src/arrow/buffer.cc:185
#10 0x00007f7910eb8c74 in arrow::PoolBuffer::~PoolBuffer
(this=0x7f7908000b80, __in_chrg=<optimized out>) at
/arrow/cpp/src/arrow/buffer.cc:187
#11 0x00005556cffb8f7c in
std::default_delete<arrow::ResizableBuffer>::operator() (this=0x5556d0288350,
__ptr=0x7f7908000b80) at
/opt/conda/envs/arrow/x86_64-conda-linux-gnu/include/c++/9.3.0/bits/unique_ptr.h:81
#12 0x00005556cffe142e in std::_Sp_counted_deleter<arrow::ResizableBuffer*,
std::default_delete<arrow::ResizableBuffer>, std::allocator<void>,
(__gnu_cxx::_Lock_policy)2>::_M_dispose (this=0x5556d0288340)
at
/opt/conda/envs/arrow/x86_64-conda-linux-gnu/include/c++/9.3.0/bits/shared_ptr_base.h:471
#13 0x00007f7910a73764 in
std::_Sp_counted_base<(__gnu_cxx::_Lock_policy)2>::_M_release
(this=0x5556d0288340) at
/opt/conda/envs/arrow/x86_64-conda-linux-gnu/include/c++/9.3.0/bits/shared_ptr_base.h:155
#14 0x00007f7910a6b063 in
std::__shared_count<(__gnu_cxx::_Lock_policy)2>::~__shared_count
(this=0x7f7908000cb0, __in_chrg=<optimized out>) at
/opt/conda/envs/arrow/x86_64-conda-linux-gnu/include/c++/9.3.0/bits/shared_ptr_base.h:730
#15 0x00007f7910a607fe in std::__shared_ptr<arrow::Buffer,
(__gnu_cxx::_Lock_policy)2>::~__shared_ptr (this=0x7f7908000ca8,
__in_chrg=<optimized out>) at
/opt/conda/envs/arrow/x86_64-conda-linux-gnu/include/c++/9.3.0/bits/shared_ptr_base.h:1169
#16 0x00007f7910a6081a in std::shared_ptr<arrow::Buffer>::~shared_ptr
(this=0x7f7908000ca8, __in_chrg=<optimized out>) at
/opt/conda/envs/arrow/x86_64-conda-linux-gnu/include/c++/9.3.0/bits/shared_ptr.h:103
#17 0x00007f7910a7473f in arrow::Result<std::shared_ptr<arrow::Buffer>
>::Destroy (this=0x7f7908000ca0) at /arrow/cpp/src/arrow/result.h:453
#18 0x00007f7910a6bf72 in arrow::Result<std::shared_ptr<arrow::Buffer>
>::~Result (this=0x7f7908000ca0, __in_chrg=<optimized out>) at
/arrow/cpp/src/arrow/result.h:125
#19 0x00007f7910a99842 in arrow::Future<std::shared_ptr<arrow::Buffer>
>::SetResult(arrow::Result<std::shared_ptr<arrow::Buffer>
>)::{lambda(void*)#1}::operator()(void*) const (this=0x0, p=0x7f7908000ca0) at
/arrow/cpp/src/arrow/util/future.h:539
#20 0x00007f7910a9986f in arrow::Future<std::shared_ptr<arrow::Buffer>
>::SetResult(arrow::Result<std::shared_ptr<arrow::Buffer>
>)::{lambda(void*)#1}::_FUN(void*) () at /arrow/cpp/src/arrow/util/future.h:539
#21 0x00007f7910a713ea in std::unique_ptr<void, void
(*)(void*)>::~unique_ptr (this=0x7f7908000cd0, __in_chrg=<optimized out>) at
/opt/conda/envs/arrow/x86_64-conda-linux-gnu/include/c++/9.3.0/bits/unique_ptr.h:292
#22 0x00007f7910c88412 in arrow::FutureImpl::~FutureImpl
(this=0x7f7908000cc0, __in_chrg=<optimized out>) at
/arrow/cpp/src/arrow/util/future.h:142
#23 0x00007f7910c94bac in arrow::ConcreteFutureImpl::~ConcreteFutureImpl
(this=0x7f7908000cc0, __in_chrg=<optimized out>) at
/arrow/cpp/src/arrow/util/future.cc:209
#24 0x00007f7910c94bc8 in arrow::ConcreteFutureImpl::~ConcreteFutureImpl
(this=0x7f7908000cc0, __in_chrg=<optimized out>) at
/arrow/cpp/src/arrow/util/future.cc:209
#25 0x00007f79114f57fe in std::default_delete<arrow::FutureImpl>::operator()
(this=0x7f7908000d80, __ptr=0x7f7908000cc0) at
/opt/conda/envs/arrow/x86_64-conda-linux-gnu/include/c++/9.3.0/bits/unique_ptr.h:81
#26 0x00007f791153c7f4 in std::_Sp_counted_deleter<arrow::FutureImpl*,
std::default_delete<arrow::FutureImpl>, std::allocator<void>,
(__gnu_cxx::_Lock_policy)2>::_M_dispose (this=0x7f7908000d70)
at
/opt/conda/envs/arrow/x86_64-conda-linux-gnu/include/c++/9.3.0/bits/shared_ptr_base.h:471
#27 0x00007f7910a73764 in
std::_Sp_counted_base<(__gnu_cxx::_Lock_policy)2>::_M_release
(this=0x7f7908000d70) at
/opt/conda/envs/arrow/x86_64-conda-linux-gnu/include/c++/9.3.0/bits/shared_ptr_base.h:155
#28 0x00007f7910a6b063 in
std::__shared_count<(__gnu_cxx::_Lock_policy)2>::~__shared_count
(this=0x7f7908000c88, __in_chrg=<optimized out>) at
/opt/conda/envs/arrow/x86_64-conda-linux-gnu/include/c++/9.3.0/bits/shared_ptr_base.h:730
#29 0x00007f7910a68372 in std::__shared_ptr<arrow::FutureImpl,
(__gnu_cxx::_Lock_policy)2>::~__shared_ptr (this=0x7f7908000c80,
__in_chrg=<optimized out>)
at
/opt/conda/envs/arrow/x86_64-conda-linux-gnu/include/c++/9.3.0/bits/shared_ptr_base.h:1169
#30 0x00007f7910a6838e in std::shared_ptr<arrow::FutureImpl>::~shared_ptr
(this=0x7f7908000c80, __in_chrg=<optimized out>) at
/opt/conda/envs/arrow/x86_64-conda-linux-gnu/include/c++/9.3.0/bits/shared_ptr.h:103
#31 0x00007f7910a715aa in arrow::Future<std::shared_ptr<arrow::Buffer>
>::~Future (this=0x7f7908000c80, __in_chrg=<optimized out>) at
/arrow/cpp/src/arrow/util/future.h:245
#32 0x00007f7910cd87ec in std::_Head_base<0ul,
arrow::Future<std::shared_ptr<arrow::Buffer> >, false>::~_Head_base
(this=0x7f7908000c80, __in_chrg=<optimized out>) at
/opt/conda/envs/arrow/x86_64-conda-linux-gnu/include/c++/9.3.0/tuple:120
#33 0x00007f7910cd0fe0 in std::_Tuple_impl<0,
arrow::Future<std::shared_ptr<arrow::Buffer> >,
arrow::io::RandomAccessFile::ReadAsync(const arrow::io::IOContext&, int64_t,
int64_t)::<lambda()> >::~_Tuple_impl(void) (this=0x7f7908000c60,
__in_chrg=<optimized out>) at
/opt/conda/envs/arrow/x86_64-conda-linux-gnu/include/c++/9.3.0/tuple:185
#34 0x00007f7910cd1008 in
std::tuple<arrow::Future<std::shared_ptr<arrow::Buffer> >,
arrow::io::RandomAccessFile::ReadAsync(const arrow::io::IOContext&, int64_t,
int64_t)::<lambda()> >::~tuple(void) (this=0x7f7908000c60,
__in_chrg=<optimized out>) at
/opt/conda/envs/arrow/x86_64-conda-linux-gnu/include/c++/9.3.0/tuple:893
#35 0x00007f7910cd1028 in
std::_Bind<arrow::detail::ContinueFuture(arrow::Future<std::shared_ptr<arrow::Buffer>
>, arrow::io::RandomAccessFile::ReadAsync(const arrow::io::IOContext&,
int64_t, int64_t)::<lambda()>)>::~_Bind(void) (
this=0x7f7908000c58, __in_chrg=<optimized out>) at
/opt/conda/envs/arrow/x86_64-conda-linux-gnu/include/c++/9.3.0/functional:386
#36 0x00007f7910cd3538 in
arrow::internal::FnOnce<void()>::FnImpl<std::_Bind<arrow::detail::ContinueFuture(arrow::Future<std::shared_ptr<arrow::Buffer>
>, arrow::io::RandomAccessFile::ReadAsync(const arrow::io::IOContext&,
int64_t, int64_t)::<lambda()>)> >::~FnImpl(void) (this=0x7f7908000c50,
__in_chrg=<optimized out>) at /arrow/cpp/src/arrow/util/functional.h:120
#37 0x00007f7910cd3560 in
arrow::internal::FnOnce<void()>::FnImpl<std::_Bind<arrow::detail::ContinueFuture(arrow::Future<std::shared_ptr<arrow::Buffer>
>, arrow::io::RandomAccessFile::ReadAsync(const arrow::io::IOContext&,
int64_t, int64_t)::<lambda()>)> >::~FnImpl(void) (this=0x7f7908000c50,
__in_chrg=<optimized out>) at /arrow/cpp/src/arrow/util/functional.h:120
#38 0x00007f7910a9a492 in std::default_delete<arrow::internal::FnOnce<void
()>::Impl>::operator()(arrow::internal::FnOnce<void ()>::Impl*) const
(this=0x7f790f5dcca8, __ptr=0x7f7908000c50)
at
/opt/conda/envs/arrow/x86_64-conda-linux-gnu/include/c++/9.3.0/bits/unique_ptr.h:81
#39 0x00007f7910a99342 in std::unique_ptr<arrow::internal::FnOnce<void
()>::Impl, std::default_delete<arrow::internal::FnOnce<void ()>::Impl>
>::~unique_ptr() (this=0x7f790f5dcca8, __in_chrg=<optimized out>)
at
/opt/conda/envs/arrow/x86_64-conda-linux-gnu/include/c++/9.3.0/bits/unique_ptr.h:292
#40 0x00007f7910c58bf9 in arrow::internal::FnOnce<void ()>::operator()() &&
(this=0x7f790f5dcce0) at /arrow/cpp/src/arrow/util/functional.h:109
#41 0x00007f7910c4d67c in arrow::internal::WorkerLoop (state=..., it=...) at
/arrow/cpp/src/arrow/util/thread_pool.cc:110
#42 0x00007f7910c4e15f in
arrow::internal::ThreadPool::<lambda()>::operator()(void) const
(__closure=0x5556d0287f88) at /arrow/cpp/src/arrow/util/thread_pool.cc:258
#43 0x00007f7910c54b28 in std::__invoke_impl<void,
arrow::internal::ThreadPool::LaunchWorkersUnlocked(int)::<lambda()>
>(std::__invoke_other, arrow::internal::ThreadPool::<lambda()> &&) (__f=...)
at
/opt/conda/envs/arrow/x86_64-conda-linux-gnu/include/c++/9.3.0/bits/invoke.h:60
#44 0x00007f7910c54add in
std::__invoke<arrow::internal::ThreadPool::LaunchWorkersUnlocked(int)::<lambda()>
>(arrow::internal::ThreadPool::<lambda()> &&) (__fn=...)
at
/opt/conda/envs/arrow/x86_64-conda-linux-gnu/include/c++/9.3.0/bits/invoke.h:95
#45 0x00007f7910c54a8a in
std::thread::_Invoker<std::tuple<arrow::internal::ThreadPool::LaunchWorkersUnlocked(int)::<lambda()>
> >::_M_invoke<0>(std::_Index_tuple<0>) (this=0x5556d0287f88)
at
/opt/conda/envs/arrow/x86_64-conda-linux-gnu/include/c++/9.3.0/thread:244
--Type <RET> for more, q to quit, c to continue without paging--
#46 0x00007f7910c54a60 in
std::thread::_Invoker<std::tuple<arrow::internal::ThreadPool::LaunchWorkersUnlocked(int)::<lambda()>
> >::operator()(void) (this=0x5556d0287f88)
at
/opt/conda/envs/arrow/x86_64-conda-linux-gnu/include/c++/9.3.0/thread:251
#47 0x00007f7910c54a1c in
std::thread::_State_impl<std::thread::_Invoker<std::tuple<arrow::internal::ThreadPool::LaunchWorkersUnlocked(int)::<lambda()>
> > >::_M_run(void) (this=0x5556d0287f80)
at
/opt/conda/envs/arrow/x86_64-conda-linux-gnu/include/c++/9.3.0/thread:195
#48 0x00007f79100f9067 in std::execute_native_thread_routine
(__p=0x5556d0287f80) at
/home/conda/feedstock_root/build_artifacts/ctng-compilers_1610729750655/work/.build/x86_64-conda-linux-gnu/src/gcc/libstdc++-v3/src/c++11/thread.cc:80
#49 0x00007f79101ac6db in start_thread (arg=0x7f790f5dd700) at
pthread_create.c:463
#50 0x00007f790fd6071f in clone () at
../sysdeps/unix/sysv/linux/x86_64/clone.S:95
```
</details>
<details>
<summary>The global allocator is already destructed</summary>
```
(gdb) info vtbl arrow::system_pool
vtable for 'arrow::SystemMemoryPool' @ 0x7f79112d6da8 (subobject @
0x7f7911315ea0):
[0]: 0x0
[1]: 0x0
[2]: 0x7f79100dd9b4 <__cxxabiv1::__cxa_pure_virtual()>
[3]: 0x7f79100dd9b4 <__cxxabiv1::__cxa_pure_virtual()>
[4]: 0x7f79100dd9b4 <__cxxabiv1::__cxa_pure_virtual()>
[5]: 0x7f79100dd9b4 <__cxxabiv1::__cxa_pure_virtual()>
[6]: 0x7f7910dc17f2 <arrow::MemoryPool::max_memory() const>
[7]: 0x7f79100dd9b4 <__cxxabiv1::__cxa_pure_virtual()>
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
