[
https://issues.apache.org/jira/browse/IMPALA-7402?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16586530#comment-16586530
]
Tim Armstrong commented on IMPALA-7402:
---------------------------------------
There were three threads in the scan node at that point in time.
{noformat}
Thread 246 (LWP 23465):
#4 0x0000000001912955 in base::subtle::NoBarrier_AtomicExchange
(ptr=0x14f577b8, new_value=0) at
/data/jenkins/workspace/impala-asf-master-exhaustive-centos6/repos/Impala/be/src/gutil/atomicops-internals-x86.h:91
#5 0x000000000192a294 in boost::lock_guard<boost::mutex>::lock_guard
(this=0x7f782f074a50, m_=...) at
/data/jenkins/workspace/impala-asf-master-exhaustive-centos6/Impala-Toolchain/boost-1.57.0-p3/include/boost/thread/lock_guard.hpp:38
#6 0x00000000020a1d75 in impala::BufferPool::Client::FreedBuffer
(this=0x18b08e1c0, len=8192) at
/data/jenkins/workspace/impala-asf-master-exhaustive-centos6/repos/Impala/be/src/runtime/bufferpool/buffer-pool-internal.h:262
#7 0x000000000209b44d in impala::BufferPool::FreeBuffer (this=0xa7b7a00,
client=0xc54a6990, handle=0x14f577b8) at
/data/jenkins/workspace/impala-asf-master-exhaustive-centos6/repos/Impala/be/src/runtime/bufferpool/buffer-pool.cc:255
#8 0x000000000315b246 in impala::io::BufferDescriptor::Free (this=0x14f57780)
at
/data/jenkins/workspace/impala-asf-master-exhaustive-centos6/repos/Impala/be/src/runtime/io/scan-range.cc:803
#9 0x0000000003155364 in impala::io::ScanRange::CleanUpBuffer
(this=0x11862840, scan_range_lock=..., buffer_desc=...) at
/data/jenkins/workspace/impala-asf-master-exhaustive-centos6/repos/Impala/be/src/runtime/io/scan-range.cc:253
#10 0x00000000031542a2 in
impala::io::ScanRange::AddUnusedBuffers(std::vector<std::unique_ptr<impala::io::BufferDescriptor,
std::default_delete<impala::io::BufferDescriptor> >,
std::allocator<std::unique_ptr<impala::io::BufferDescriptor,
std::default_delete<impala::io::BufferDescriptor> > > >&&, bool)
(this=0x11862840, buffers=..., returned=true) at
/data/jenkins/workspace/impala-asf-master-exhaustive-centos6/repos/Impala/be/src/runtime/io/scan-range.cc:137
#11 0x0000000003154020 in impala::io::ScanRange::ReturnBuffer (this=0x11862840,
buffer_desc=...) at
/data/jenkins/workspace/impala-asf-master-exhaustive-centos6/repos/Impala/be/src/runtime/io/scan-range.cc:111
#12 0x0000000002eba8f0 in impala::ScannerContext::Stream::ReturnIoBuffer
(this=0xa2783c0) at
/data/jenkins/workspace/impala-asf-master-exhaustive-centos6/repos/Impala/be/src/exec/scanner-context.cc:355
#13 0x0000000002eb75b9 in
impala::ScannerContext::Stream::ReleaseCompletedResources (this=0xa2783c0,
done=true) at
/data/jenkins/workspace/impala-asf-master-exhaustive-centos6/repos/Impala/be/src/exec/scanner-context.cc:110
#14 0x0000000002eb7185 in impala::ScannerContext::ReleaseCompletedResources
(this=0x7f782f075010, done=true) at
/data/jenkins/workspace/impala-asf-master-exhaustive-centos6/repos/Impala/be/src/exec/scanner-context.cc:70
#15 0x000000000218d7bb in impala::HdfsTextScanner::Close (this=0xd43a580,
row_batch=0x13d368a0) at
/data/jenkins/workspace/impala-asf-master-exhaustive-centos6/repos/Impala/be/src/exec/hdfs-text-scanner.cc:181
#16 0x000000000214b2bc in impala::HdfsScanner::Close (this=0xd43a580) at
/data/jenkins/workspace/impala-asf-master-exhaustive-centos6/repos/Impala/be/src/exec/hdfs-scanner.cc:149
#17 0x0000000002121b4c in impala::HdfsScanNode::ProcessSplit (this=0xc54a6800,
filter_ctxs=..., expr_results_pool=0x7f782f0753f0, scan_range=0x11862840,
scanner_thread_reservation=0x7f782f075368) at
/data/jenkins/workspace/impala-asf-master-exhaustive-centos6/repos/Impala/be/src/exec/hdfs-scan-node.cc:510
#18 0x00000000021207b7 in impala::HdfsScanNode::ScannerThread (this=0xc54a6800,
first_thread=false, scanner_thread_reservation=8192) at
/data/jenkins/workspace/impala-asf-master-exhaustive-centos6/repos/Impala/be/src/exec/hdfs-scan-node.cc:381
Thread 1 (LWP 23473):
#0 0x0000003d1a8328e5 in ?? ()
#1 0x0000003d1a8340c5 in ?? ()
#2 0x0000000002120d08 in impala::HdfsScanNode::ScannerThread (this=0xffffffff,
first_thread=false, scanner_thread_reservation=262438073157) at
/data/jenkins/workspace/impala-asf-master-exhaustive-centos6/repos/Impala/be/src/exec/hdfs-scan-node.cc:433
#3 0x00000000043d2d87 in google::LogMessage::Flush() ()
#4 0x00000000043d634e in google::LogMessageFatal::~LogMessageFatal() ()
#5 0x000000000209f35e in impala::BufferPool::Client::WriteDirtyPagesAsync
(this=0x18b08e1c0, min_bytes_to_write=8192) at
/data/jenkins/workspace/impala-asf-master-exhaustive-centos6/repos/Impala/be/src/runtime/bufferpool/buffer-pool.cc:645
#6 0x000000000209ee31 in impala::BufferPool::Client::CleanPages
(this=0x18b08e1c0, client_lock=0x7f7827869190, len=8192) at
/data/jenkins/workspace/impala-asf-master-exhaustive-centos6/repos/Impala/be/src/runtime/bufferpool/buffer-pool.cc:625
#7 0x000000000209ec11 in impala::BufferPool::Client::DecreaseReservationTo
(this=0x18b08e1c0, max_decrease=8192, target_bytes=8192) at
/data/jenkins/workspace/impala-asf-master-exhaustive-centos6/repos/Impala/be/src/runtime/bufferpool/buffer-pool.cc:609
#8 0x000000000209ba95 in
impala::BufferPool::ClientHandle::DecreaseReservationTo (this=0xc54a6990,
max_decrease=8192, target_bytes=8192) at
/data/jenkins/workspace/impala-asf-master-exhaustive-centos6/repos/Impala/be/src/runtime/bufferpool/buffer-pool.cc:319
#9 0x000000000211f8cb in
impala::HdfsScanNode::ReturnReservationFromScannerThread (this=0xc54a6800,
lock=..., bytes=8192) at
/data/jenkins/workspace/impala-asf-master-exhaustive-centos6/repos/Impala/be/src/exec/hdfs-scan-node.cc:233
#10 0x0000000002120d09 in impala::HdfsScanNode::ScannerThread (this=0xc54a6800,
first_thread=false, scanner_thread_reservation=8192) at
/data/jenkins/workspace/impala-asf-master-exhaustive-centos6/repos/Impala/be/src/exec/hdfs-scan-node.cc:433
Thread 4 (LWP 23521):
#5 0x000000000192a524 in boost::unique_lock<boost::mutex>::lock
(this=0x7f7830476df0) at
/data/jenkins/workspace/impala-asf-master-exhaustive-centos6/Impala-Toolchain/boost-1.57.0-p3/include/boost/thread/lock_types.hpp:346
#6 0x000000000192a2e3 in boost::unique_lock<boost::mutex>::unique_lock
(this=0x7f7830476df0, m_=...) at
/data/jenkins/workspace/impala-asf-master-exhaustive-centos6/Impala-Toolchain/boost-1.57.0-p3/include/boost/thread/lock_types.hpp:124
#7 0x000000000211fbf4 in impala::HdfsScanNode::ThreadTokenAvailableCb
(this=0xc54a6800, pool=0x18dae220) at
/data/jenkins/workspace/impala-asf-master-exhaustive-centos6/repos/Impala/be/src/exec/hdfs-scan-node.cc:269
#8 0x0000000002124b44 in boost::_mfi::mf1<void, impala::HdfsScanNode,
impala::ThreadResourcePool*>::operator() (this=0x7f78304770c8, p=0xc54a6800,
a1=0x18dae220) at
/data/jenkins/workspace/impala-asf-master-exhaustive-centos6/Impala-Toolchain/boost-1.57.0-p3/include/boost/bind/mem_fn_template.hpp:165
#9 0x0000000002124822 in
boost::_bi::list2<boost::_bi::value<impala::HdfsScanNode*>, boost::arg<1>
>::operator()<boost::_mfi::mf1<void, impala::HdfsScanNode,
impala::ThreadResourcePool*>, boost::_bi::list1<impala::ThreadResourcePool*&> >
(this=0x7f78304770d8, f=..., a=...) at
/data/jenkins/workspace/impala-asf-master-exhaustive-centos6/Impala-Toolchain/boost-1.57.0-p3/include/boost/bind/bind.hpp:313
#10 0x0000000002124516 in boost::_bi::bind_t<void, boost::_mfi::mf1<void,
impala::HdfsScanNode, impala::ThreadResourcePool*>,
boost::_bi::list2<boost::_bi::value<impala::HdfsScanNode*>, boost::arg<1> >
>::operator()<impala::ThreadResourcePool*> (this=0x7f78304770c8,
a1=@0x7f7830477040: 0x18dae220) at
/data/jenkins/workspace/impala-asf-master-exhaustive-centos6/Impala-Toolchain/boost-1.57.0-p3/include/boost/bind/bind_template.hpp:32
#11 0x00000000021240ca in
boost::detail::function::void_function_obj_invoker1<boost::_bi::bind_t<void,
boost::_mfi::mf1<void, impala::HdfsScanNode, impala::ThreadResourcePool*>,
boost::_bi::list2<boost::_bi::value<impala::HdfsScanNode*>, boost::arg<1> > >,
void, impala::ThreadResourcePool*>::invoke (function_obj_ptr=...,
a0=0x18dae220) at
/data/jenkins/workspace/impala-asf-master-exhaustive-centos6/Impala-Toolchain/boost-1.57.0-p3/include/boost/function/function_template.hpp:153
#12 0x0000000001dd8adf in boost::function1<void,
impala::ThreadResourcePool*>::operator() (this=0x7f78304770c0, a0=0x18dae220)
at
/data/jenkins/workspace/impala-asf-master-exhaustive-centos6/Impala-Toolchain/boost-1.57.0-p3/include/boost/function/function_template.hpp:767
#13 0x0000000001dd8030 in impala::ThreadResourcePool::InvokeCallbacks
(this=0x18dae220) at
/data/jenkins/workspace/impala-asf-master-exhaustive-centos6/repos/Impala/be/src/runtime/thread-resource-mgr.cc:105
#14 0x0000000001dd8227 in impala::ThreadResourceMgr::UpdatePoolQuotas
(this=0xbb808c0, new_pool=0x0) at
/data/jenkins/workspace/impala-asf-master-exhaustive-centos6/repos/Impala/be/src/runtime/thread-resource-mgr.cc:120
#15 0x0000000001dd7b26 in impala::ThreadResourceMgr::DestroyPool
(this=0xbb808c0, pool=...) at
/data/jenkins/workspace/impala-asf-master-exhaustive-centos6/repos/Impala/be/src/runtime/thread-resource-mgr.cc:71
#16 0x0000000001dd18ce in impala::RuntimeState::ReleaseResources
(this=0xdc32680) at
/data/jenkins/workspace/impala-asf-master-exhaustive-centos6/repos/Impala/be/src/runtime/runtime-state.cc:268
#17 0x0000000001e2031b in impala::FragmentInstanceState::Close (this=0xf063000)
at
/data/jenkins/workspace/impala-asf-master-exhaustive-centos6/repos/Impala/be/src/runtime/fragment-instance-state.cc:345
#18 0x0000000001e1ca71 in impala::FragmentInstanceState::Exec (this=0xf063000)
at
/data/jenkins/workspace/impala-asf-master-exhaustive-centos6/repos/Impala/be/src/runtime/fragment-instance-state.cc:118
#19 0x0000000001e2dab5 in impala::QueryState::ExecFInstance (this=0x119e0800,
fis=0xf063000) at
/data/jenkins/workspace/impala-asf-master-exhaustive-centos6/repos/Impala/be/src/runtime/query-state.cc:485
{noformat}
> DCHECK failed min_bytes_to_write <= dirty_unpinned_pages_ in buffer-pool
> ------------------------------------------------------------------------
>
> Key: IMPALA-7402
> URL: https://issues.apache.org/jira/browse/IMPALA-7402
> Project: IMPALA
> Issue Type: Bug
> Components: Backend
> Affects Versions: Impala 3.1.0
> Reporter: Vuk Ercegovac
> Assignee: Tim Armstrong
> Priority: Blocker
> Labels: broken-build
>
> One of the impalad's crashed with the following DCHECK failure:
> {noformat}
> F0806 01:26:21.905500 5101 buffer-pool.cc:645] Check failed:
> min_bytes_to_write <= dirty_unpinned_pages_.bytes() (8192 vs. 0)
> Here is the backtrace:{noformat}
> {noformat}
> #0 0x0000003af1e328e5 in raise () from /lib64/libc.so.6
> #1 0x0000003af1e340c5 in abort () from /lib64/libc.so.6
> #2 0x000000000437f454 in google::DumpStackTraceAndExit() ()
> #3 0x0000000004375ead in google::LogMessage::Fail() ()
> #4 0x0000000004377752 in google::LogMessage::SendToLog() ()
> #5 0x0000000004375887 in google::LogMessage::Flush() ()
> #6 0x0000000004378e4e in google::LogMessageFatal::~LogMessageFatal() ()
> #7 0x000000000205ad16 in impala::BufferPool::Client::WriteDirtyPagesAsync
> (this=0x17d03f0e0, min_bytes_to_write=8192) at
> /data/jenkins/workspace/impala-cdh6.x-exhaustive-centos6/repos/Impala/be/src/runtime/bufferpool/buffer-pool.cc:645
>
> #8 0x000000000205a835 in impala::BufferPool::Client::CleanPages
> (this=0x17d03f0e0, client_lock=0x7f324cb12220, len=8192) at
> /data/jenkins/workspace/impala-cdh6.x-exhaustive-centos6/repos/Impala/be/src/runtime/bufferpool/buffer-pool.cc:625
>
> #9 0x000000000205a646 in impala::BufferPool::Client::DecreaseReservationTo
> (this=0x17d03f0e0, max_decrease=8192, target_bytes=8192) at
> /data/jenkins/workspace/impala-cdh6.x-exhaustive-centos6/repos/Impala/be/src/runtime/bufferpool/buffer-pool.cc:609
>
> #10 0x0000000002057583 in
> impala::BufferPool::ClientHandle::DecreaseReservationTo (this=0x181c0a990,
> max_decrease=8192, target_bytes=8192) at
> /data/jenkins/workspace/impala-cdh6.x-exhaustive-centos6/repos/Impala/be/src/runtime/bufferpool/buffer-pool.cc:319
>
> #11 0x00000000020d9419 in
> impala::HdfsScanNode::ReturnReservationFromScannerThread (this=0x181c0a800,
> lock=..., bytes=8192) at
> /data/jenkins/workspace/impala-cdh6.x-exhaustive-centos6/repos/Impala/be/src/exec/hdfs-scan-node.cc:194
>
> #12 0x00000000020da485 in impala::HdfsScanNode::ScannerThread
> (this=0x181c0a800, first_thread=false, scanner_thread_reservation=8192) at
> /data/jenkins/workspace/impala-cdh6.x-exhaustive-centos6/repos/Impala/be/src/exec/hdfs-scan-node.cc:367
>
> #13 0x00000000020d96b0 in impala::HdfsScanNode::<lambda()>::operator()(void)
> const (__closure=0x7f324cb12b88) at
> /data/jenkins/workspace/impala-cdh6.x-exhaustive-centos6/repos/Impala/be/src/exec/hdfs-scan-node.cc:261
>
> #14 0x00000000020db6d6 in
> boost::detail::function::void_function_obj_invoker0<impala::HdfsScanNode::ThreadTokenAvailableCb(impala::ThreadResourcePool*)::<lambda()>,
> void>::invoke(boost::detail::function::function_buffer &)
> (function_obj_ptr=...) at
> /data/jenkins/workspace/impala-cdh6.x-exhaustive-centos6/Impala-Toolchain/boost-1.57.0-p3/include/boost/function/function_template.hpp:153
> ...
> {noformat}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
