[
https://issues.apache.org/jira/browse/KUDU-3569?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Alexey Serbin updated KUDU-3569:
--------------------------------
Description:
Running {{alter_table-randomized-test}} under TSAN produced data race warnings
like below, indicating a race in
{{CFileSet::Iterator::OptimizePKPredicates()}}. One actor was
{{tablet::AlterSchemaOp::Apply()}} initiated by AlterTable, the other
concurrent actor was the maintenance thread running major delta compaction.
Apparently, the same data race might happen if the other concurrent actor was a
thread handling a scan request containing IN-list predicates optimized at the
DRS level.
{noformat}
WARNING: ThreadSanitizer: data race (pid=3919595)
Write of size 8 at 0x7b44000f4a20 by thread T7:
#0 std::__1::__vector_base<unsigned long, std::__1::allocator<unsigned
long>>::__destruct_at_end(unsigned long*)
/root/Projects/kudu/thirdparty/installed/tsan/include/c++/v1/vector:429:12
(kudu+0x4d4080)
#1 std::__1::__vector_base<unsigned long, std::__1::allocator<unsigned
long>>::clear()
/root/Projects/kudu/thirdparty/installed/tsan/include/c++/v1/vector:371:29
(kudu+0x4d3f94)
#2 std::__1::__vector_base<unsigned long, std::__1::allocator<unsigned
long>>::~__vector_base()
/root/Projects/kudu/thirdparty/installed/tsan/include/c++/v1/vector:465:9
(kudu+0x4d3d4b)
#3 std::__1::vector<unsigned long, std::__1::allocator<unsigned long>
>::~vector()
/root/Projects/kudu/thirdparty/installed/tsan/include/c++/v1/vector:557:5
(kudu+0x4d1261)
#4 kudu::Schema::~Schema()
/root/Projects/kudu/src/kudu/common/schema.h:491:7 (kudu+0x4cc40f)
#5 std::__1::__shared_ptr_emplace<kudu::Schema,
std::__1::allocator<kudu::Schema>>::__on_zero_shared()
/root/Projects/kudu/thirdparty/installed/tsan/include/c++/v1/memory:3503:23
(libtablet.so+0x389d45)
#6 std::__1::__shared_count::__release_shared()
/root/Projects/kudu/thirdparty/installed/tsan/include/c++/v1/memory:3341:9
(kudu+0x4d4d05)
#7 std::__1::__shared_weak_count::__release_shared()
/root/Projects/kudu/thirdparty/installed/tsan/include/c++/v1/memory:3383:27
(kudu+0x4d4ca9)
#8 std::__1::shared_ptr<kudu::Schema>::~shared_ptr()
/root/Projects/kudu/thirdparty/installed/tsan/include/c++/v1/memory:4098:19
(kudu+0x5303e8)
#9
kudu::tablet::TabletMetadata::SetSchema(std::__1::shared_ptr<kudu::Schema>
const&, unsigned int)
/root/Projects/kudu/src/kudu/tablet/tablet_metadata.cc:957:1
(libtablet.so+0x4d8882)
#10 kudu::tablet::Tablet::AlterSchema(kudu::tablet::AlterSchemaOpState*)
/root/Projects/kudu/src/kudu/tablet/tablet.cc:1727:14 (libtablet.so+0x32720a)
#11 kudu::tablet::AlterSchemaOp::Apply(kudu::consensus::CommitMsg**)
/root/Projects/kudu/src/kudu/tablet/ops/alter_schema_op.cc:127:3
(libtablet.so+0x4013f8)
#12 kudu::tablet::OpDriver::ApplyTask()
/root/Projects/kudu/src/kudu/tablet/ops/op_driver.cc:527:21
(libtablet.so+0x40873a)
...
Previous read of size 8 at 0x7b44000f4a20 by thread T22 (mutexes: write
M799524414306809968, write M765184518688777856):
#0 std::__1::vector<unsigned long, std::__1::allocator<unsigned long>
>::empty() const
/root/Projects/kudu/thirdparty/installed/tsan/include/c++/v1/vector:664:41
(kudu+0x5ca926)
#1 kudu::Schema::initialized() const
/root/Projects/kudu/src/kudu/common/schema.h:676:26 (kudu+0x5ca3fd)
#2 kudu::Schema::key_byte_size() const
/root/Projects/kudu/src/kudu/common/schema.h:572:5 (libkudu_common.so+0x171eae)
#3 kudu::EncodedKey::DecodeEncodedString(kudu::Schema const&, kudu::Arena*,
kudu::Slice const&, kudu::EncodedKey**)
/root/Projects/kudu/src/kudu/common/encoded_key.cc:60:76
(libkudu_common.so+0x171091)
#4 kudu::tablet::CFileSet::Iterator::OptimizePKPredicates(kudu::ScanSpec*)
/root/Projects/kudu/src/kudu/tablet/cfile_set.cc:444:5 (libtablet.so+0x428934)
#5 kudu::tablet::CFileSet::Iterator::Init(kudu::ScanSpec*)
/root/Projects/kudu/src/kudu/tablet/cfile_set.cc:410:3 (libtablet.so+0x4285d7)
#6 kudu::MaterializingIterator::Init(kudu::ScanSpec*)
/root/Projects/kudu/src/kudu/common/generic_iterators.cc:1176:3
(libkudu_common.so+0x178872)
#7
kudu::tablet::MajorDeltaCompaction::FlushRowSetAndDeltas(kudu::fs::IOContext
const*) /root/Projects/kudu/src/kudu/tablet/delta_compaction.cc:130:3
(libtablet.so+0x54ca30)
#8 kudu::tablet::MajorDeltaCompaction::Compact(kudu::fs::IOContext const*)
/root/Projects/kudu/src/kudu/tablet/delta_compaction.cc:340:3
(libtablet.so+0x54ead0)
#9
kudu::tablet::DiskRowSet::MajorCompactDeltaStoresWithColumnIds(std::__1::vector<kudu::ColumnId,
std::__1::allocator<kudu::ColumnId> > const&, kudu::fs::IOContext const*,
kudu::tablet::HistoryGcOpts)
/root/Projects/kudu/src/kudu/tablet/diskrowset.cc:588:3 (libtablet.so+0x46b38c)
#10 kudu::tablet::DiskRowSet::MajorCompactDeltaStores(kudu::fs::IOContext
const*, kudu::tablet::HistoryGcOpts)
/root/Projects/kudu/src/kudu/tablet/diskrowset.cc:572:10 (libtablet.so+0x46b033)
#11
kudu::tablet::Tablet::CompactWorstDeltas(kudu::tablet::RowSet::DeltaCompactionType)
/root/Projects/kudu/src/kudu/tablet/tablet.cc:2881:5 (libtablet.so+0x32d832)
#12 kudu::tablet::MajorDeltaCompactionOp::Perform()
/root/Projects/kudu/src/kudu/tablet/tablet_mm_ops.cc:364:3
(libtablet.so+0x3c0846)
#13 kudu::MaintenanceManager::LaunchOp(kudu::MaintenanceOp*)
/root/Projects/kudu/src/kudu/util/maintenance_manager.cc:640:9
(libkudu_util.so+0x37f5f6)
{noformat}
was:
Running {{alter_table-randomized-test}} under TSAN produced data race warnings
like below, indicating a race in
{{CFileSet::Iterator::OptimizePKPredicates()}}. One actor was
{{tablet::AlterSchemaOp::Apply()}} initiated by AlterTable, the other
concurrent actor was the maintenance thread running major delta compaction.
Apparently, the same data race might happen if the other concurrent actor was a
thread handling a scan request containing IN-list predicates optimized at the
DRS level.
{noformat}
WARNING: ThreadSanitizer: data race (pid=3919595)
Write of size 8 at 0x7b44000f4a20 by thread T7:
#0 std::__1::__vector_base<unsigned long, std::__1::allocator<unsigned long>
>::__destruct_at_end(unsigned long*) /root/Projects/kudu/thirdparty/installed/t
san/include/c++/v1/vector:429:12 (kudu+0x4d4080)
#1 std::__1::__vector_base<unsigned long, std::__1::allocator<unsigned long>
>::clear() /root/Projects/kudu/thirdparty/installed/tsan/include/c++/v1/vector:
371:29 (kudu+0x4d3f94)
#2 std::__1::__vector_base<unsigned long, std::__1::allocator<unsigned long>
>::~__vector_base() /root/Projects/kudu/thirdparty/installed/tsan/include/c++/v
1/vector:465:9 (kudu+0x4d3d4b)
#3 std::__1::vector<unsigned long, std::__1::allocator<unsigned long> >::~ve
ctor() /root/Projects/kudu/thirdparty/installed/tsan/include/c++/v1/vector:557:5
(kudu+0x4d1261)
#4 kudu::Schema::~Schema() /root/Projects/kudu/src/kudu/common/schema.h:491:
7 (kudu+0x4cc40f)
#5 std::__1::__shared_ptr_emplace<kudu::Schema, std::__1::allocator<kudu::Sc
hema> >::__on_zero_shared() /root/Projects/kudu/thirdparty/installed/tsan/includ
e/c++/v1/memory:3503:23 (libtablet.so+0x389d45)
#6 std::__1::__shared_count::__release_shared()
/root/Projects/kudu/thirdparty/installed/tsan/include/c++/v1/memory:3341:9
(kudu+0x4d4d05)
#7 std::__1::__shared_weak_count::__release_shared()
/root/Projects/kudu/thirdparty/installed/tsan/include/c++/v1/memory:3383:27
(kudu+0x4d4ca9)
#8 std::__1::shared_ptr<kudu::Schema>::~shared_ptr()
/root/Projects/kudu/thirdparty/installed/tsan/include/c++/v1/memory:4098:19
(kudu+0x5303e8)
#9
kudu::tablet::TabletMetadata::SetSchema(std::__1::shared_ptr<kudu::Schema>
const&, unsigned int)
/root/Projects/kudu/src/kudu/tablet/tablet_metadata.cc:957:1
(libtablet.so+0x4d8882)
#10 kudu::tablet::Tablet::AlterSchema(kudu::tablet::AlterSchemaOpState*)
/root/Projects/kudu/src/kudu/tablet/tablet.cc:1727:14 (libtablet.so+0x32720a)
#11 kudu::tablet::AlterSchemaOp::Apply(kudu::consensus::CommitMsg**)
/root/Projects/kudu/src/kudu/tablet/ops/alter_schema_op.cc:127:3
(libtablet.so+0x4013f8)
#12 kudu::tablet::OpDriver::ApplyTask()
/root/Projects/kudu/src/kudu/tablet/ops/op_driver.cc:527:21
(libtablet.so+0x40873a)
...
Previous read of size 8 at 0x7b44000f4a20 by thread T22 (mutexes: write
M799524414306809968, write M765184518688777856):
#0 std::__1::vector<unsigned long, std::__1::allocator<unsigned long>
>::empty() const
/root/Projects/kudu/thirdparty/installed/tsan/include/c++/v1/vector:664:41
(kudu+0x5ca926)
#1 kudu::Schema::initialized() const
/root/Projects/kudu/src/kudu/common/schema.h:676:26 (kudu+0x5ca3fd)
#2 kudu::Schema::key_byte_size() const
/root/Projects/kudu/src/kudu/common/schema.h:572:5 (libkudu_common.so+0x171eae)
#3 kudu::EncodedKey::DecodeEncodedString(kudu::Schema const&, kudu::Arena*,
kudu::Slice const&, kudu::EncodedKey**)
/root/Projects/kudu/src/kudu/common/encoded_key.cc:60:76
(libkudu_common.so+0x171091)
#4 kudu::tablet::CFileSet::Iterator::OptimizePKPredicates(kudu::ScanSpec*)
/root/Projects/kudu/src/kudu/tablet/cfile_set.cc:444:5 (libtablet.so+0x428934)
#5 kudu::tablet::CFileSet::Iterator::Init(kudu::ScanSpec*)
/root/Projects/kudu/src/kudu/tablet/cfile_set.cc:410:3 (libtablet.so+0x4285d7)
#6 kudu::MaterializingIterator::Init(kudu::ScanSpec*)
/root/Projects/kudu/src/kudu/common/generic_iterators.cc:1176:3
(libkudu_common.so+0x178872)
#7
kudu::tablet::MajorDeltaCompaction::FlushRowSetAndDeltas(kudu::fs::IOContext
const*) /root/Projects/kudu/src/kudu/tablet/delta_compaction.cc:130:3
(libtablet.so+0x54ca30)
#8 kudu::tablet::MajorDeltaCompaction::Compact(kudu::fs::IOContext const*)
/root/Projects/kudu/src/kudu/tablet/delta_compaction.cc:340:3
(libtablet.so+0x54ead0)
#9
kudu::tablet::DiskRowSet::MajorCompactDeltaStoresWithColumnIds(std::__1::vector<kudu::ColumnId,
std::__1::allocator<kudu::ColumnId> > const&, kudu::fs::IOContext const*,
kudu::tablet::HistoryGcOpts)
/root/Projects/kudu/src/kudu/tablet/diskrowset.cc:588:3 (libtablet.so+0x46b38c)
#10 kudu::tablet::DiskRowSet::MajorCompactDeltaStores(kudu::fs::IOContext
const*, kudu::tablet::HistoryGcOpts)
/root/Projects/kudu/src/kudu/tablet/diskrowset.cc:572:10 (libtablet.so+0x46b033)
#11
kudu::tablet::Tablet::CompactWorstDeltas(kudu::tablet::RowSet::DeltaCompactionType)
/root/Projects/kudu/src/kudu/tablet/tablet.cc:2881:5 (libtablet.so+0x32d832)
#12 kudu::tablet::MajorDeltaCompactionOp::Perform()
/root/Projects/kudu/src/kudu/tablet/tablet_mm_ops.cc:364:3
(libtablet.so+0x3c0846)
#13 kudu::MaintenanceManager::LaunchOp(kudu::MaintenanceOp*)
/root/Projects/kudu/src/kudu/util/maintenance_manager.cc:640:9
(libkudu_util.so+0x37f5f6)
{noformat}
> Data race in CFileSet::Iterator::OptimizePKPredicates()
> -------------------------------------------------------
>
> Key: KUDU-3569
> URL: https://issues.apache.org/jira/browse/KUDU-3569
> Project: Kudu
> Issue Type: Bug
> Components: tserver
> Affects Versions: 1.17.0
> Reporter: Alexey Serbin
> Priority: Major
>
> Running {{alter_table-randomized-test}} under TSAN produced data race
> warnings like below, indicating a race in
> {{CFileSet::Iterator::OptimizePKPredicates()}}. One actor was
> {{tablet::AlterSchemaOp::Apply()}} initiated by AlterTable, the other
> concurrent actor was the maintenance thread running major delta compaction.
> Apparently, the same data race might happen if the other concurrent actor was
> a thread handling a scan request containing IN-list predicates optimized at
> the DRS level.
> {noformat}
> WARNING: ThreadSanitizer: data race (pid=3919595)
> Write of size 8 at 0x7b44000f4a20 by thread T7:
> #0 std::__1::__vector_base<unsigned long, std::__1::allocator<unsigned
> long>>::__destruct_at_end(unsigned long*)
> /root/Projects/kudu/thirdparty/installed/tsan/include/c++/v1/vector:429:12
> (kudu+0x4d4080)
> #1 std::__1::__vector_base<unsigned long, std::__1::allocator<unsigned
> long>>::clear()
> /root/Projects/kudu/thirdparty/installed/tsan/include/c++/v1/vector:371:29
> (kudu+0x4d3f94)
> #2 std::__1::__vector_base<unsigned long, std::__1::allocator<unsigned
> long>>::~__vector_base()
> /root/Projects/kudu/thirdparty/installed/tsan/include/c++/v1/vector:465:9
> (kudu+0x4d3d4b)
> #3 std::__1::vector<unsigned long, std::__1::allocator<unsigned long>
> >::~vector()
> /root/Projects/kudu/thirdparty/installed/tsan/include/c++/v1/vector:557:5
> (kudu+0x4d1261)
> #4 kudu::Schema::~Schema()
> /root/Projects/kudu/src/kudu/common/schema.h:491:7 (kudu+0x4cc40f)
> #5 std::__1::__shared_ptr_emplace<kudu::Schema,
> std::__1::allocator<kudu::Schema>>::__on_zero_shared()
> /root/Projects/kudu/thirdparty/installed/tsan/include/c++/v1/memory:3503:23
> (libtablet.so+0x389d45)
> #6 std::__1::__shared_count::__release_shared()
> /root/Projects/kudu/thirdparty/installed/tsan/include/c++/v1/memory:3341:9
> (kudu+0x4d4d05)
> #7 std::__1::__shared_weak_count::__release_shared()
> /root/Projects/kudu/thirdparty/installed/tsan/include/c++/v1/memory:3383:27
> (kudu+0x4d4ca9)
> #8 std::__1::shared_ptr<kudu::Schema>::~shared_ptr()
> /root/Projects/kudu/thirdparty/installed/tsan/include/c++/v1/memory:4098:19
> (kudu+0x5303e8)
> #9
> kudu::tablet::TabletMetadata::SetSchema(std::__1::shared_ptr<kudu::Schema>
> const&, unsigned int)
> /root/Projects/kudu/src/kudu/tablet/tablet_metadata.cc:957:1
> (libtablet.so+0x4d8882)
> #10 kudu::tablet::Tablet::AlterSchema(kudu::tablet::AlterSchemaOpState*)
> /root/Projects/kudu/src/kudu/tablet/tablet.cc:1727:14 (libtablet.so+0x32720a)
> #11 kudu::tablet::AlterSchemaOp::Apply(kudu::consensus::CommitMsg**)
> /root/Projects/kudu/src/kudu/tablet/ops/alter_schema_op.cc:127:3
> (libtablet.so+0x4013f8)
> #12 kudu::tablet::OpDriver::ApplyTask()
> /root/Projects/kudu/src/kudu/tablet/ops/op_driver.cc:527:21
> (libtablet.so+0x40873a)
> ...
> Previous read of size 8 at 0x7b44000f4a20 by thread T22 (mutexes: write
> M799524414306809968, write M765184518688777856):
> #0 std::__1::vector<unsigned long, std::__1::allocator<unsigned long>
> >::empty() const
> /root/Projects/kudu/thirdparty/installed/tsan/include/c++/v1/vector:664:41
> (kudu+0x5ca926)
> #1 kudu::Schema::initialized() const
> /root/Projects/kudu/src/kudu/common/schema.h:676:26 (kudu+0x5ca3fd)
> #2 kudu::Schema::key_byte_size() const
> /root/Projects/kudu/src/kudu/common/schema.h:572:5
> (libkudu_common.so+0x171eae)
> #3 kudu::EncodedKey::DecodeEncodedString(kudu::Schema const&,
> kudu::Arena*, kudu::Slice const&, kudu::EncodedKey**)
> /root/Projects/kudu/src/kudu/common/encoded_key.cc:60:76
> (libkudu_common.so+0x171091)
> #4
> kudu::tablet::CFileSet::Iterator::OptimizePKPredicates(kudu::ScanSpec*)
> /root/Projects/kudu/src/kudu/tablet/cfile_set.cc:444:5 (libtablet.so+0x428934)
> #5 kudu::tablet::CFileSet::Iterator::Init(kudu::ScanSpec*)
> /root/Projects/kudu/src/kudu/tablet/cfile_set.cc:410:3 (libtablet.so+0x4285d7)
> #6 kudu::MaterializingIterator::Init(kudu::ScanSpec*)
> /root/Projects/kudu/src/kudu/common/generic_iterators.cc:1176:3
> (libkudu_common.so+0x178872)
> #7
> kudu::tablet::MajorDeltaCompaction::FlushRowSetAndDeltas(kudu::fs::IOContext
> const*) /root/Projects/kudu/src/kudu/tablet/delta_compaction.cc:130:3
> (libtablet.so+0x54ca30)
> #8 kudu::tablet::MajorDeltaCompaction::Compact(kudu::fs::IOContext
> const*) /root/Projects/kudu/src/kudu/tablet/delta_compaction.cc:340:3
> (libtablet.so+0x54ead0)
> #9
> kudu::tablet::DiskRowSet::MajorCompactDeltaStoresWithColumnIds(std::__1::vector<kudu::ColumnId,
> std::__1::allocator<kudu::ColumnId> > const&, kudu::fs::IOContext const*,
> kudu::tablet::HistoryGcOpts)
> /root/Projects/kudu/src/kudu/tablet/diskrowset.cc:588:3
> (libtablet.so+0x46b38c)
> #10 kudu::tablet::DiskRowSet::MajorCompactDeltaStores(kudu::fs::IOContext
> const*, kudu::tablet::HistoryGcOpts)
> /root/Projects/kudu/src/kudu/tablet/diskrowset.cc:572:10
> (libtablet.so+0x46b033)
> #11
> kudu::tablet::Tablet::CompactWorstDeltas(kudu::tablet::RowSet::DeltaCompactionType)
> /root/Projects/kudu/src/kudu/tablet/tablet.cc:2881:5 (libtablet.so+0x32d832)
> #12 kudu::tablet::MajorDeltaCompactionOp::Perform()
> /root/Projects/kudu/src/kudu/tablet/tablet_mm_ops.cc:364:3
> (libtablet.so+0x3c0846)
> #13 kudu::MaintenanceManager::LaunchOp(kudu::MaintenanceOp*)
> /root/Projects/kudu/src/kudu/util/maintenance_manager.cc:640:9
> (libkudu_util.so+0x37f5f6)
> {noformat}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
