[
https://issues.apache.org/jira/browse/KUDU-777?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Jean-Daniel Cryans updated KUDU-777:
------------------------------------
Fix Version/s: (was: 0.8.0)
0.9.0
Updated the Fix Version since 0.8.0 is practically out.
> heap-use-after-free in mt-tablet-test
> -------------------------------------
>
> Key: KUDU-777
> URL: https://issues.apache.org/jira/browse/KUDU-777
> Project: Kudu
> Issue Type: Bug
> Components: tablet
> Affects Versions: M5
> Reporter: Adar Dembo
> Assignee: Todd Lipcon
> Priority: Critical
> Fix For: 0.9.0
>
>
> From http://sandbox.jenkins.cloudera.com/job/kudu-gerrit/8319:
> {noformat}
> ==25094==ERROR: AddressSanitizer: heap-use-after-free on address
> 0x603000120028 at pc 0xb03430 bp 0x7f52d0237360 sp 0x7f52d0236b20
> READ of size 4 at 0x603000120028 thread T307 (test0-25406)
> #0 0xb0342f in memcpy
> /data1/jenkins-workspace/kudu-gerrit/BUILD_TYPE/ASAN/label/kudu-gerrit-slaves/thirdparty/llvm-3.4.2.src/projects/compiler-rt/lib/asan/asan_interceptors.cc:377
> #0 0x362d49cdff in ?? ??:0
> #1 0x362d49cf6a in ?? ??:0
> #3 0x1a30525 in kudu::Slice::ToString() const
> /data1/jenkins-workspace/kudu-gerrit/BUILD_TYPE/ASAN/label/kudu-gerrit-slaves/src/kudu/util/slice.cc:21
> #4 0xd17366 in
> kudu::tablet::RowSetTree::Reset(std::vector<std::tr1::shared_ptr<kudu::tablet::RowSet>,
> std::allocator<std::tr1::shared_ptr<kudu::tablet::RowSet> > > const&)
> /data1/jenkins-workspace/kudu-gerrit/BUILD_TYPE/ASAN/label/kudu-gerrit-slaves/src/kudu/tablet/rowset_tree.cc:103
> #5 0xbc13b6 in
> kudu::tablet::Tablet::ModifyRowSetTree(kudu::tablet::RowSetTree const&,
> std::vector<std::tr1::shared_ptr<kudu::tablet::RowSet>,
> std::allocator<std::tr1::shared_ptr<kudu::tablet::RowSet> > > const&,
> std::vector<std::tr1::shared_ptr<kudu::tablet::RowSet>,
> std::allocator<std::tr1::shared_ptr<kudu::tablet::RowSet> > > const&,
> kudu::tablet::RowSetTree*)
> /data1/jenkins-workspace/kudu-gerrit/BUILD_TYPE/ASAN/label/kudu-gerrit-slaves/src/kudu/tablet/tablet.cc:494
> #6 0xbc1ac7 in
> kudu::tablet::Tablet::AtomicSwapRowSetsUnlocked(std::vector<std::tr1::shared_ptr<kudu::tablet::RowSet>,
> std::allocator<std::tr1::shared_ptr<kudu::tablet::RowSet> > > const&,
> std::vector<std::tr1::shared_ptr<kudu::tablet::RowSet>,
> std::allocator<std::tr1::shared_ptr<kudu::tablet::RowSet> > > const&)
> /data1/jenkins-workspace/kudu-gerrit/BUILD_TYPE/ASAN/label/kudu-gerrit-slaves/src/kudu/tablet/tablet.cc:508
> #7 0xbc18b0 in
> kudu::tablet::Tablet::AtomicSwapRowSets(std::vector<std::tr1::shared_ptr<kudu::tablet::RowSet>,
> std::allocator<std::tr1::shared_ptr<kudu::tablet::RowSet> > > const&,
> std::vector<std::tr1::shared_ptr<kudu::tablet::RowSet>,
> std::allocator<std::tr1::shared_ptr<kudu::tablet::RowSet> > > const&)
> /data1/jenkins-workspace/kudu-gerrit/BUILD_TYPE/ASAN/label/kudu-gerrit-slaves/src/kudu/tablet/tablet.cc:500
> #8 0xbc5a84 in kudu::tablet::Tablet::DoCompactionOrFlush(kudu::Schema
> const&, kudu::tablet::RowSetsInCompaction const&, long)
> /data1/jenkins-workspace/kudu-gerrit/BUILD_TYPE/ASAN/label/kudu-gerrit-slaves/src/kudu/tablet/tablet.cc:1167
> #9 0xbc31d3 in
> kudu::tablet::Tablet::FlushInternal(kudu::tablet::RowSetsInCompaction const&,
> std::tr1::shared_ptr<kudu::tablet::MemRowSet> const&, kudu::Schema const&)
> /data1/jenkins-workspace/kudu-gerrit/BUILD_TYPE/ASAN/label/kudu-gerrit-slaves/src/kudu/tablet/tablet.cc:603
> #10 0xbc23f4 in kudu::tablet::Tablet::FlushUnlocked()
> /data1/jenkins-workspace/kudu-gerrit/BUILD_TYPE/ASAN/label/kudu-gerrit-slaves/src/kudu/tablet/tablet.cc:544
> #11 0xbc1eb9 in kudu::tablet::Tablet::Flush()
> /data1/jenkins-workspace/kudu-gerrit/BUILD_TYPE/ASAN/label/kudu-gerrit-slaves/src/kudu/tablet/tablet.cc:524
> #12 0xb30a48 in
> kudu::tablet::MultiThreadedTabletTest<kudu::tablet::NullableValueTestSetup>::FlushThread(int)
>
> /data1/jenkins-workspace/kudu-gerrit/BUILD_TYPE/ASAN/label/kudu-gerrit-slaves/src/kudu/tablet/mt-tablet-test.cc:256
> #13 0xb41f0a in boost::_bi::bind_t<void, boost::_mfi::mf1<void,
> kudu::tablet::MultiThreadedTabletTest<kudu::tablet::NullableValueTestSetup>,
> int>,
> boost::_bi::list2<boost::_bi::value<kudu::tablet::MultiThreadedTabletTest<kudu::tablet::NullableValueTestSetup>*>,
> boost::_bi::value<int> > >::operator()()
> /usr/include/boost/bind/bind_template.hpp:20
> #14 0x19d8e3c in boost::function0<void>::operator()() const
> /usr/include/boost/function/function_template.hpp:1012
> #15 0x1a4759f in kudu::Thread::SuperviseThread(void*)
> /data1/jenkins-workspace/kudu-gerrit/BUILD_TYPE/ASAN/label/kudu-gerrit-slaves/src/kudu/util/thread.cc:478
> #16 0x7f52f4216850 in start_thread (/lib64/libpthread.so.0+0x7850)
> #17 0x7f52f324794c in clone (/lib64/libc.so.6+0xe894c)
> 0x603000120028 is located 24 bytes inside of 29-byte region
> [0x603000120010,0x60300012002d)
> freed by thread T311 (test0-25410) here:
> #0 0xb13c2e in operator delete(void*)
> /data1/jenkins-workspace/kudu-gerrit/BUILD_TYPE/ASAN/label/kudu-gerrit-slaves/thirdparty/llvm-3.4.2.src/projects/compiler-rt/lib/asan/asan_new_delete.cc:83
> #2 0x362d49d4c8 in ?? ??:0
> #2 0xdc56d2 in kudu::tablet::CFileSet::~CFileSet()
> /data1/jenkins-workspace/kudu-gerrit/BUILD_TYPE/ASAN/label/kudu-gerrit-slaves/src/kudu/tablet/cfile_set.cc:55
> #3 0xb393f5 in
> std::tr1::_Sp_counted_base<(__gnu_cxx::_Lock_policy)2>::_M_release()
> /usr/lib/gcc/x86_64-redhat-linux/4.4.7/../../../../include/c++/4.4.7/tr1_impl/boost_sp_counted_base.h:140
> #4 0xc9079d in void std::tr1::__shared_ptr<kudu::tablet::CFileSet,
> (__gnu_cxx::_Lock_policy)2>::reset<kudu::tablet::CFileSet>(kudu::tablet::CFileSet*)
>
> /usr/lib/gcc/x86_64-redhat-linux/4.4.7/../../../../include/c++/4.4.7/tr1/shared_ptr.h:505
> #5 0xc879c6 in
> kudu::tablet::DiskRowSet::MajorCompactDeltaStoresWithColumns(std::vector<unsigned
> long, std::allocator<unsigned long> > const&)
> /data1/jenkins-workspace/kudu-gerrit/BUILD_TYPE/ASAN/label/kudu-gerrit-slaves/src/kudu/tablet/diskrowset.cc:492
> #6 0xc87443 in kudu::tablet::DiskRowSet::MajorCompactDeltaStores()
> /data1/jenkins-workspace/kudu-gerrit/BUILD_TYPE/ASAN/label/kudu-gerrit-slaves/src/kudu/tablet/diskrowset.cc:466
> #7 0xbd2537 in
> kudu::tablet::Tablet::CompactWorstDeltas(kudu::tablet::RowSet::DeltaCompactionType)
>
> /data1/jenkins-workspace/kudu-gerrit/BUILD_TYPE/ASAN/label/kudu-gerrit-slaves/src/kudu/tablet/tablet.cc:1480
> #8 0xb3ebe9 in
> kudu::tablet::MultiThreadedTabletTest<kudu::tablet::NullableValueTestSetup>::CompactDeltas(kudu::tablet::RowSet::DeltaCompactionType)
>
> /data1/jenkins-workspace/kudu-gerrit/BUILD_TYPE/ASAN/label/kudu-gerrit-slaves/src/kudu/tablet/mt-tablet-test.cc:292
> #9 0xb41f0a in boost::_bi::bind_t<void, boost::_mfi::mf1<void,
> kudu::tablet::MultiThreadedTabletTest<kudu::tablet::NullableValueTestSetup>,
> int>,
> boost::_bi::list2<boost::_bi::value<kudu::tablet::MultiThreadedTabletTest<kudu::tablet::NullableValueTestSetup>*>,
> boost::_bi::value<int> > >::operator()()
> /usr/include/boost/bind/bind_template.hpp:20
> #10 0x19d8e3c in boost::function0<void>::operator()() const
> /usr/include/boost/function/function_template.hpp:1012
> #11 0x1a4759f in kudu::Thread::SuperviseThread(void*)
> /data1/jenkins-workspace/kudu-gerrit/BUILD_TYPE/ASAN/label/kudu-gerrit-slaves/src/kudu/util/thread.cc:478
> #12 0x7f52f4216850 in start_thread (/lib64/libpthread.so.0+0x7850)
> previously allocated by thread T307 (test0-25406) here:
> #0 0xb1392e in operator new(unsigned long)
> /data1/jenkins-workspace/kudu-gerrit/BUILD_TYPE/ASAN/label/kudu-gerrit-slaves/thirdparty/llvm-3.4.2.src/projects/compiler-rt/lib/asan/asan_new_delete.cc:52
> #3 0x362d49c3c8 in ?? ??:0
> #2 0xdc724b in kudu::tablet::CFileSet::LoadMinMaxKeys()
> /data1/jenkins-workspace/kudu-gerrit/BUILD_TYPE/ASAN/label/kudu-gerrit-slaves/src/kudu/tablet/cfile_set.cc:131
> #3 0xdc5dae in kudu::tablet::CFileSet::Open()
> /data1/jenkins-workspace/kudu-gerrit/BUILD_TYPE/ASAN/label/kudu-gerrit-slaves/src/kudu/tablet/cfile_set.cc:88
> #4 0xc867eb in kudu::tablet::DiskRowSet::Open()
> /data1/jenkins-workspace/kudu-gerrit/BUILD_TYPE/ASAN/label/kudu-gerrit-slaves/src/kudu/tablet/diskrowset.cc:435
> #5 0xc864f7 in
> kudu::tablet::DiskRowSet::Open(std::tr1::shared_ptr<kudu::tablet::RowSetMetadata>
> const&, kudu::log::LogAnchorRegistry*,
> std::tr1::shared_ptr<kudu::tablet::DiskRowSet>*,
> std::tr1::shared_ptr<kudu::MemTracker> const&)
> /data1/jenkins-workspace/kudu-gerrit/BUILD_TYPE/ASAN/label/kudu-gerrit-slaves/src/kudu/tablet/diskrowset.cc:417
> #6 0xbc4968 in kudu::tablet::Tablet::DoCompactionOrFlush(kudu::Schema
> const&, kudu::tablet::RowSetsInCompaction const&, long)
> /data1/jenkins-workspace/kudu-gerrit/BUILD_TYPE/ASAN/label/kudu-gerrit-slaves/src/kudu/tablet/tablet.cc:1036
> #7 0xbc31d3 in
> kudu::tablet::Tablet::FlushInternal(kudu::tablet::RowSetsInCompaction const&,
> std::tr1::shared_ptr<kudu::tablet::MemRowSet> const&, kudu::Schema const&)
> /data1/jenkins-workspace/kudu-gerrit/BUILD_TYPE/ASAN/label/kudu-gerrit-slaves/src/kudu/tablet/tablet.cc:603
> #8 0xbc23f4 in kudu::tablet::Tablet::FlushUnlocked()
> /data1/jenkins-workspace/kudu-gerrit/BUILD_TYPE/ASAN/label/kudu-gerrit-slaves/src/kudu/tablet/tablet.cc:544
> #9 0xbc1eb9 in kudu::tablet::Tablet::Flush()
> /data1/jenkins-workspace/kudu-gerrit/BUILD_TYPE/ASAN/label/kudu-gerrit-slaves/src/kudu/tablet/tablet.cc:524
> #10 0xb30a48 in
> kudu::tablet::MultiThreadedTabletTest<kudu::tablet::NullableValueTestSetup>::FlushThread(int)
>
> /data1/jenkins-workspace/kudu-gerrit/BUILD_TYPE/ASAN/label/kudu-gerrit-slaves/src/kudu/tablet/mt-tablet-test.cc:256
> #11 0xb41f0a in boost::_bi::bind_t<void, boost::_mfi::mf1<void,
> kudu::tablet::MultiThreadedTabletTest<kudu::tablet::NullableValueTestSetup>,
> int>,
> boost::_bi::list2<boost::_bi::value<kudu::tablet::MultiThreadedTabletTest<kudu::tablet::NullableValueTestSetup>*>,
> boost::_bi::value<int> > >::operator()()
> /usr/include/boost/bind/bind_template.hpp:20
> #12 0x19d8e3c in boost::function0<void>::operator()() const
> /usr/include/boost/function/function_template.hpp:1012
> #13 0x1a4759f in kudu::Thread::SuperviseThread(void*)
> /data1/jenkins-workspace/kudu-gerrit/BUILD_TYPE/ASAN/label/kudu-gerrit-slaves/src/kudu/util/thread.cc:478
> #14 0x7f52f4216850 in start_thread (/lib64/libpthread.so.0+0x7850)
> Thread T307 (test0-25406) created by T0 here:
> #0 0xb0284f in __interceptor_pthread_create
> /data1/jenkins-workspace/kudu-gerrit/BUILD_TYPE/ASAN/label/kudu-gerrit-slaves/thirdparty/llvm-3.4.2.src/projects/compiler-rt/lib/asan/asan_interceptors.cc:185
> #1 0x1a46bb1 in kudu::Thread::StartThread(std::string const&, std::string
> const&, boost::function<void ()()> const&, scoped_refptr<kudu::Thread>*)
> /data1/jenkins-workspace/kudu-gerrit/BUILD_TYPE/ASAN/label/kudu-gerrit-slaves/src/kudu/util/thread.cc:406
> #2 0xb3f2a6 in kudu::Status kudu::Thread::Create<void
> (kudu::tablet::MultiThreadedTabletTest<kudu::tablet::NullableValueTestSetup>::*)(int),
>
> kudu::tablet::MultiThreadedTabletTest<kudu::tablet::NullableValueTestSetup>*,
> int>(std::string const&, std::string const&, void
> (kudu::tablet::MultiThreadedTabletTest<kudu::tablet::NullableValueTestSetup>::*
> const&)(int),
> kudu::tablet::MultiThreadedTabletTest<kudu::tablet::NullableValueTestSetup>*
> const&, int const&, scoped_refptr<kudu::Thread>*)
> /data1/jenkins-workspace/kudu-gerrit/BUILD_TYPE/ASAN/label/kudu-gerrit-slaves/src/kudu/util/thread.h:132
> #3 0xb303f2 in void
> kudu::tablet::MultiThreadedTabletTest<kudu::tablet::NullableValueTestSetup>::StartThreads<void
>
> (kudu::tablet::MultiThreadedTabletTest<kudu::tablet::NullableValueTestSetup>::*)(int)>(int,
> void
> (kudu::tablet::MultiThreadedTabletTest<kudu::tablet::NullableValueTestSetup>::*
> const&)(int))
> /data1/jenkins-workspace/kudu-gerrit/BUILD_TYPE/ASAN/label/kudu-gerrit-slaves/src/kudu/tablet/mt-tablet-test.cc:363
> #4 0xb2fae6 in
> kudu::tablet::MultiThreadedTabletTest_DeleteAndReinsert_Test<kudu::tablet::NullableValueTestSetup>::TestBody()
>
> /data1/jenkins-workspace/kudu-gerrit/BUILD_TYPE/ASAN/label/kudu-gerrit-slaves/src/kudu/tablet/mt-tablet-test.cc:431
> #5 0x1bb05dc in HandleSehExceptionsInMethodIfSupported<testing::Test,
> void>
> /data1/jenkins-workspace/kudu-gerrit/BUILD_TYPE/ASAN/label/kudu-gerrit-slaves/thirdparty/gmock-1.7.0/gtest/src/gtest.cc:2078
> #6 0x1bb05dc in void
> testing::internal::HandleExceptionsInMethodIfSupported<testing::Test,
> void>(testing::Test*, void (testing::Test::*)(), char const*)
> /data1/jenkins-workspace/kudu-gerrit/BUILD_TYPE/ASAN/label/kudu-gerrit-slaves/thirdparty/gmock-1.7.0/gtest/src/gtest.cc:2114
> Thread T311 (test0-25410) created by T0 here:
> #0 0xb0284f in __interceptor_pthread_create
> /data1/jenkins-workspace/kudu-gerrit/BUILD_TYPE/ASAN/label/kudu-gerrit-slaves/thirdparty/llvm-3.4.2.src/projects/compiler-rt/lib/asan/asan_interceptors.cc:185
> #1 0x1a46bb1 in kudu::Thread::StartThread(std::string const&, std::string
> const&, boost::function<void ()()> const&, scoped_refptr<kudu::Thread>*)
> /data1/jenkins-workspace/kudu-gerrit/BUILD_TYPE/ASAN/label/kudu-gerrit-slaves/src/kudu/util/thread.cc:406
> #2 0xb3f2a6 in kudu::Status kudu::Thread::Create<void
> (kudu::tablet::MultiThreadedTabletTest<kudu::tablet::NullableValueTestSetup>::*)(int),
>
> kudu::tablet::MultiThreadedTabletTest<kudu::tablet::NullableValueTestSetup>*,
> int>(std::string const&, std::string const&, void
> (kudu::tablet::MultiThreadedTabletTest<kudu::tablet::NullableValueTestSetup>::*
> const&)(int),
> kudu::tablet::MultiThreadedTabletTest<kudu::tablet::NullableValueTestSetup>*
> const&, int const&, scoped_refptr<kudu::Thread>*)
> /data1/jenkins-workspace/kudu-gerrit/BUILD_TYPE/ASAN/label/kudu-gerrit-slaves/src/kudu/util/thread.h:132
> #3 0xb303f2 in void
> kudu::tablet::MultiThreadedTabletTest<kudu::tablet::NullableValueTestSetup>::StartThreads<void
>
> (kudu::tablet::MultiThreadedTabletTest<kudu::tablet::NullableValueTestSetup>::*)(int)>(int,
> void
> (kudu::tablet::MultiThreadedTabletTest<kudu::tablet::NullableValueTestSetup>::*
> const&)(int))
> /data1/jenkins-workspace/kudu-gerrit/BUILD_TYPE/ASAN/label/kudu-gerrit-slaves/src/kudu/tablet/mt-tablet-test.cc:363
> #4 0xb2fc26 in
> kudu::tablet::MultiThreadedTabletTest_DeleteAndReinsert_Test<kudu::tablet::NullableValueTestSetup>::TestBody()
>
> /data1/jenkins-workspace/kudu-gerrit/BUILD_TYPE/ASAN/label/kudu-gerrit-slaves/src/kudu/tablet/mt-tablet-test.cc:436
> #5 0x1bb05dc in HandleSehExceptionsInMethodIfSupported<testing::Test,
> void>
> /data1/jenkins-workspace/kudu-gerrit/BUILD_TYPE/ASAN/label/kudu-gerrit-slaves/thirdparty/gmock-1.7.0/gtest/src/gtest.cc:2078
> #6 0x1bb05dc in void
> testing::internal::HandleExceptionsInMethodIfSupported<testing::Test,
> void>(testing::Test*, void (testing::Test::*)(), char const*)
> /data1/jenkins-workspace/kudu-gerrit/BUILD_TYPE/ASAN/label/kudu-gerrit-slaves/thirdparty/gmock-1.7.0/gtest/src/gtest.cc:2114
> SUMMARY: AddressSanitizer: heap-use-after-free
> /data1/jenkins-workspace/kudu-gerrit/BUILD_TYPE/ASAN/label/kudu-gerrit-slaves/thirdparty/llvm-3.4.2.src/projects/compiler-rt/lib/asan/asan_interceptors.cc:377
> memcpy
> Shadow bytes around the buggy address:
> 0x0c068001bfb0: fa fa fa fa fd fd fd fa fa fa fd fd fd fa fa fa
> 0x0c068001bfc0: fd fd fd fa fa fa fd fd fd fa fa fa fd fd fd fa
> 0x0c068001bfd0: fa fa fd fd fd fd fa fa fd fd fd fa fa fa fd fd
> 0x0c068001bfe0: fd fd fa fa fa fa fa fa fa fa fd fd fd fa fa fa
> 0x0c068001bff0: fd fd fd fd fa fa fd fd fd fd fa fa fd fd fd fa
> =>0x0c068001c000: fa fa fd fd fd[fd]fa fa fd fd fd fa fa fa fd fd
> 0x0c068001c010: fd fd fa fa fd fd fd fa fa fa fa fa fa fa fa fa
> 0x0c068001c020: fa fa fa fa fa fa fd fd fd fa fa fa fd fd fd fa
> 0x0c068001c030: fa fa fd fd fd fa fa fa fd fd fd fa fa faI0517
> 00:47:56.611583 25405 test_graph.cc:76] metrics: { "scope":
> "MultiThreadedTabletTest/5", "time": 2.253}
> fd fd
> 0x0c068001c040: fd fa fa fa fd fd fd fa fa fa fd fd fd fd fa fa
> 0x0c068001c050: fd fd fd fa fa fa fd fd fd fd fa fa fd fd fd fa
> Shadow byte legend (one shadow byte represents 8 application bytes):
> Addressable: 00
> Partially addressable: 01 02 03 04 05 06 07
> Heap left redzone: fa
> Heap right redzone: fb
> Freed heap region: fd
> Stack left redzone: f1
> Stack mid redzone: f2
> Stack right redzone: f3
> Stack partial redzone: f4
> Stack after return: f5
> Stack use after scope: f8
> Global redzone: f9
> Global init order: f6
> Poisoned by user: f7
> ASan internal: fe
> ==25094==ABORTING
> {noformat}
> I've done a little bit of analysis, will post my findings shortly.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
