This is an automated email from the ASF dual-hosted git repository. adar pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/kudu.git
commit 62c2fddcdc65f2092ee335ac1a35288dd3f7e4a5 Author: Andrew Wong <[email protected]> AuthorDate: Tue Oct 1 13:30:05 2019 -0700 cfile: track memory for CFileReaders and BloomFileReaders This lumps a tablet's CFileReader tracked memory and BloomFileReader tracked memory into the ids "CFileReaders" and "BloomFileReaders" respectively. I test this locally and verified via the web UI that I could see CFileReader and BloomFileReader entries once they had been flushed. Change-Id: Ic62c89079edffb41d1f5b40d7b699c525fa447a7 Reviewed-on: http://gerrit.cloudera.org:8080/14322 Reviewed-by: Adar Dembo <[email protected]> Tested-by: Kudu Jenkins --- src/kudu/tablet/cfile_set-test.cc | 15 ++++++++++----- src/kudu/tablet/cfile_set.cc | 22 +++++++++++++--------- src/kudu/tablet/cfile_set.h | 9 ++++++--- src/kudu/tablet/diskrowset.cc | 6 ++++-- src/kudu/tablet/tablet_mem_trackers.h | 8 +++++++- 5 files changed, 40 insertions(+), 20 deletions(-) diff --git a/src/kudu/tablet/cfile_set-test.cc b/src/kudu/tablet/cfile_set-test.cc index e3fd517..c074e9d 100644 --- a/src/kudu/tablet/cfile_set-test.cc +++ b/src/kudu/tablet/cfile_set-test.cc @@ -278,7 +278,8 @@ TEST_F(TestCFileSet, TestPartiallyMaterialize) { WriteTestRowSet(kNumRows); shared_ptr<CFileSet> fileset; - ASSERT_OK(CFileSet::Open(rowset_meta_, MemTracker::GetRootTracker(), nullptr, &fileset)); + ASSERT_OK(CFileSet::Open(rowset_meta_, MemTracker::GetRootTracker(), MemTracker::GetRootTracker(), + nullptr, &fileset)); unique_ptr<CFileSet::Iterator> iter(fileset->NewIterator(&schema_, nullptr)); ASSERT_OK(iter->Init(nullptr)); @@ -358,7 +359,8 @@ TEST_F(TestCFileSet, TestIteratePartialSchema) { WriteTestRowSet(kNumRows); shared_ptr<CFileSet> fileset; - ASSERT_OK(CFileSet::Open(rowset_meta_, MemTracker::GetRootTracker(), nullptr, &fileset)); + ASSERT_OK(CFileSet::Open(rowset_meta_, MemTracker::GetRootTracker(), MemTracker::GetRootTracker(), + nullptr, &fileset)); Schema new_schema; ASSERT_OK(schema_.CreateProjectionByNames({ "c0", "c2" }, &new_schema)); @@ -391,7 +393,8 @@ TEST_F(TestCFileSet, TestRangeScan) { WriteTestRowSet(kNumRows); shared_ptr<CFileSet> fileset; - ASSERT_OK(CFileSet::Open(rowset_meta_, MemTracker::GetRootTracker(), nullptr, &fileset)); + ASSERT_OK(CFileSet::Open(rowset_meta_, MemTracker::GetRootTracker(), MemTracker::GetRootTracker(), + nullptr, &fileset)); // Create iterator. unique_ptr<CFileSet::Iterator> cfile_iter(fileset->NewIterator(&schema_, nullptr)); @@ -445,7 +448,8 @@ TEST_F(TestCFileSet, TestRangePredicates2) { WriteTestRowSet(kNumRows); shared_ptr<CFileSet> fileset; - ASSERT_OK(CFileSet::Open(rowset_meta_, MemTracker::GetRootTracker(), nullptr, &fileset)); + ASSERT_OK(CFileSet::Open(rowset_meta_, MemTracker::GetRootTracker(), MemTracker::GetRootTracker(), + nullptr, &fileset)); // Range scan where rows match on both ends DoTestRangeScan(fileset, 2000, 2010); @@ -501,7 +505,8 @@ TEST_F(TestCFileSet, TestBloomFilterPredicates) { &ret1_contain, &ret1_exclude, &ret2_contain, &ret2_exclude); shared_ptr<CFileSet> fileset; - ASSERT_OK(CFileSet::Open(rowset_meta_, MemTracker::GetRootTracker(), nullptr, &fileset)); + ASSERT_OK(CFileSet::Open(rowset_meta_, MemTracker::GetRootTracker(), MemTracker::GetRootTracker(), + nullptr, &fileset)); vector<ColumnPredicate::BloomFilterInner> bfs; // BloomFilter of column 0 contain. diff --git a/src/kudu/tablet/cfile_set.cc b/src/kudu/tablet/cfile_set.cc index 52a892b..e1b7881 100644 --- a/src/kudu/tablet/cfile_set.cc +++ b/src/kudu/tablet/cfile_set.cc @@ -88,7 +88,7 @@ using strings::Substitute; //////////////////////////////////////////////////////////// static Status OpenReader(FsManager* fs, - shared_ptr<MemTracker> parent_mem_tracker, + shared_ptr<MemTracker> cfile_reader_tracker, const BlockId& block_id, const IOContext* io_context, unique_ptr<CFileReader>* new_reader) { @@ -96,7 +96,7 @@ static Status OpenReader(FsManager* fs, RETURN_NOT_OK(fs->OpenBlock(block_id, &block)); ReaderOptions opts; - opts.parent_mem_tracker = std::move(parent_mem_tracker); + opts.parent_mem_tracker = std::move(cfile_reader_tracker); opts.io_context = io_context; return CFileReader::OpenNoInit(std::move(block), std::move(opts), @@ -108,20 +108,24 @@ static Status OpenReader(FsManager* fs, //////////////////////////////////////////////////////////// CFileSet::CFileSet(shared_ptr<RowSetMetadata> rowset_metadata, - shared_ptr<MemTracker> parent_mem_tracker) + shared_ptr<MemTracker> bloomfile_tracker, + shared_ptr<MemTracker> cfile_reader_tracker) : rowset_metadata_(std::move(rowset_metadata)), - parent_mem_tracker_(std::move(parent_mem_tracker)) { + bloomfile_tracker_(std::move(bloomfile_tracker)), + cfile_reader_tracker_(std::move(cfile_reader_tracker)) { } CFileSet::~CFileSet() { } Status CFileSet::Open(shared_ptr<RowSetMetadata> rowset_metadata, - shared_ptr<MemTracker> parent_mem_tracker, + shared_ptr<MemTracker> bloomfile_tracker, + shared_ptr<MemTracker> cfile_reader_tracker, const IOContext* io_context, shared_ptr<CFileSet>* cfile_set) { shared_ptr<CFileSet> cfs(new CFileSet(std::move(rowset_metadata), - std::move(parent_mem_tracker))); + std::move(bloomfile_tracker), + std::move(cfile_reader_tracker))); RETURN_NOT_OK(cfs->DoOpen(io_context)); cfile_set->swap(cfs); @@ -140,7 +144,7 @@ Status CFileSet::DoOpen(const IOContext* io_context) { unique_ptr<CFileReader> reader; RETURN_NOT_OK(OpenReader(rowset_metadata_->fs_manager(), - parent_mem_tracker_, + cfile_reader_tracker_, rowset_metadata_->column_data_block_for_col_id(col_id), io_context, &reader)); @@ -152,7 +156,7 @@ Status CFileSet::DoOpen(const IOContext* io_context) { if (rowset_metadata_->has_adhoc_index_block()) { RETURN_NOT_OK(OpenReader(rowset_metadata_->fs_manager(), - parent_mem_tracker_, + cfile_reader_tracker_, rowset_metadata_->adhoc_index_block(), io_context, &ad_hoc_idx_reader_)); @@ -183,8 +187,8 @@ Status CFileSet::OpenBloomReader(const IOContext* io_context) { RETURN_NOT_OK(fs->OpenBlock(rowset_metadata_->bloom_block(), &block)); ReaderOptions opts; - opts.parent_mem_tracker = parent_mem_tracker_; opts.io_context = io_context; + opts.parent_mem_tracker = bloomfile_tracker_; Status s = BloomFileReader::OpenNoInit(std::move(block), std::move(opts), &bloom_reader_); diff --git a/src/kudu/tablet/cfile_set.h b/src/kudu/tablet/cfile_set.h index 4721e95..a34dd21 100644 --- a/src/kudu/tablet/cfile_set.h +++ b/src/kudu/tablet/cfile_set.h @@ -74,7 +74,8 @@ class CFileSet : public std::enable_shared_from_this<CFileSet> { class Iterator; static Status Open(std::shared_ptr<RowSetMetadata> rowset_metadata, - std::shared_ptr<MemTracker> parent_mem_tracker, + std::shared_ptr<MemTracker> bloomfile_tracker, + std::shared_ptr<MemTracker> cfile_reader_tracker, const fs::IOContext* io_context, std::shared_ptr<CFileSet>* cfile_set); @@ -133,7 +134,8 @@ class CFileSet : public std::enable_shared_from_this<CFileSet> { DISALLOW_COPY_AND_ASSIGN(CFileSet); CFileSet(std::shared_ptr<RowSetMetadata> rowset_metadata, - std::shared_ptr<MemTracker> parent_mem_tracker); + std::shared_ptr<MemTracker> bloomfile_tracker, + std::shared_ptr<MemTracker> cfile_reader_tracker); Status DoOpen(const fs::IOContext* io_context); Status OpenBloomReader(const fs::IOContext* io_context); @@ -153,7 +155,8 @@ class CFileSet : public std::enable_shared_from_this<CFileSet> { const Schema &tablet_schema() const { return rowset_metadata_->tablet_schema(); } std::shared_ptr<RowSetMetadata> rowset_metadata_; - std::shared_ptr<MemTracker> parent_mem_tracker_; + std::shared_ptr<MemTracker> bloomfile_tracker_; + std::shared_ptr<MemTracker> cfile_reader_tracker_; std::string min_encoded_key_; std::string max_encoded_key_; diff --git a/src/kudu/tablet/diskrowset.cc b/src/kudu/tablet/diskrowset.cc index b2395a9..f4dd37d 100644 --- a/src/kudu/tablet/diskrowset.cc +++ b/src/kudu/tablet/diskrowset.cc @@ -535,7 +535,8 @@ DiskRowSet::DiskRowSet(shared_ptr<RowSetMetadata> rowset_metadata, Status DiskRowSet::Open(const IOContext* io_context) { TRACE_EVENT0("tablet", "DiskRowSet::Open"); RETURN_NOT_OK(CFileSet::Open(rowset_metadata_, - mem_trackers_.tablet_tracker, + mem_trackers_.bloomfile_tracker, + mem_trackers_.cfile_reader_tracker, io_context, &base_data_)); @@ -607,7 +608,8 @@ Status DiskRowSet::MajorCompactDeltaStoresWithColumnIds(const vector<ColumnId>& // appropriate blocks to match the update. shared_ptr<CFileSet> new_base; RETURN_NOT_OK(CFileSet::Open(rowset_metadata_, - mem_trackers_.tablet_tracker, + mem_trackers_.bloomfile_tracker, + mem_trackers_.cfile_reader_tracker, io_context, &new_base)); { diff --git a/src/kudu/tablet/tablet_mem_trackers.h b/src/kudu/tablet/tablet_mem_trackers.h index 4882318..405b4cb 100644 --- a/src/kudu/tablet/tablet_mem_trackers.h +++ b/src/kudu/tablet/tablet_mem_trackers.h @@ -30,6 +30,8 @@ struct TabletMemTrackers { // Intended for unit tests where the tracker hierarchy doesn't matter. TabletMemTrackers() : tablet_tracker(MemTracker::GetRootTracker()), + bloomfile_tracker(MemTracker::GetRootTracker()), + cfile_reader_tracker(MemTracker::GetRootTracker()), dms_tracker(MemTracker::GetRootTracker()) { } @@ -38,13 +40,17 @@ struct TabletMemTrackers { : tablet_tracker(MemTracker::CreateTracker( -1, strings::Substitute("tablet-$0", tablet_id), - parent_mem_tracker)), + std::move(parent_mem_tracker))), + bloomfile_tracker(MemTracker::CreateTracker(-1, "BloomFileReaders", tablet_tracker)), + cfile_reader_tracker(MemTracker::CreateTracker(-1, "CFileReaders", tablet_tracker)), dms_tracker(MemTracker::CreateTracker(-1, "DeltaMemStores", tablet_tracker)) { } std::shared_ptr<MemTracker> tablet_tracker; // All of the below are children of tablet_tracker; + std::shared_ptr<MemTracker> bloomfile_tracker; + std::shared_ptr<MemTracker> cfile_reader_tracker; std::shared_ptr<MemTracker> dms_tracker; };
