This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new d57c234 [MemTracker] Refactored the hierarchical structure of
memtracker (#5956)
d57c234 is described below
commit d57c2344e1a2cac20b37e9f00a060b5027a3f3e1
Author: Mingyu Chen <[email protected]>
AuthorDate: Wed Jun 16 09:44:24 2021 +0800
[MemTracker] Refactored the hierarchical structure of memtracker (#5956)
To avoid showing too many memtracker on BE web pages.
The MemTracker level now has 3 levels: OVERVIEW, TASK and VERBOSE.
OVERVIEW Mainly used for main memory consumption module such as
Query/Load/Metadata.
TASK is mainly used to record the memory overhead of a single task such as
a single query, load, and compaction task.
VERBOSE is used for other more detailed memtrackers.
---
be/src/exec/olap_scanner.cpp | 2 +-
be/src/http/action/compaction_action.h | 2 +-
be/src/http/default_path_handlers.cpp | 12 +++--
be/src/olap/compaction.cpp | 4 +-
be/src/olap/fs/file_block_manager.cpp | 4 +-
be/src/olap/lru_cache.cpp | 2 +-
be/src/olap/page_cache.cpp | 2 +-
be/src/olap/rowset/beta_rowset.cpp | 2 +-
be/src/olap/rowset/beta_rowset_reader.cpp | 2 +-
be/src/olap/rowset/segment_reader.cpp | 2 +-
be/src/olap/rowset/segment_v2/segment.cpp | 22 ++++++---
be/src/olap/rowset/segment_v2/segment.h | 9 ++--
be/src/olap/rowset/segment_v2/segment_iterator.cpp | 6 +--
be/src/olap/schema_change.cpp | 4 +-
be/src/olap/storage_engine.cpp | 8 ++--
be/src/olap/storage_engine.h | 4 ++
be/src/olap/tablet_manager.cpp | 2 +-
be/src/runtime/buffered_block_mgr2.cc | 2 -
be/src/runtime/exec_env.h | 3 ++
be/src/runtime/exec_env_init.cpp | 4 +-
be/src/runtime/load_channel.cpp | 2 +-
be/src/runtime/load_channel_mgr.cpp | 2 +-
be/src/runtime/mem_tracker.cpp | 4 +-
be/src/runtime/mem_tracker.h | 6 +--
be/src/runtime/plan_fragment_executor.cpp | 2 +-
be/test/olap/rowset/segment_v2/segment_test.cpp | 52 +++++++++++-----------
26 files changed, 94 insertions(+), 72 deletions(-)
diff --git a/be/src/exec/olap_scanner.cpp b/be/src/exec/olap_scanner.cpp
index 1f719c9..e97d3ce 100644
--- a/be/src/exec/olap_scanner.cpp
+++ b/be/src/exec/olap_scanner.cpp
@@ -53,7 +53,7 @@ OlapScanner::OlapScanner(RuntimeState* runtime_state,
OlapScanNode* parent, bool
_version(-1),
_mem_tracker(MemTracker::CreateTracker(
runtime_state->fragment_mem_tracker()->limit(),
"OlapScanner",
- runtime_state->fragment_mem_tracker(), true, true,
MemTrackerLevel::DEBUG)) {
+ runtime_state->fragment_mem_tracker(), true, true,
MemTrackerLevel::VERBOSE)) {
_rows_read_counter = parent->rows_read_counter();
_rows_pushed_cond_filtered_counter =
parent->_rows_pushed_cond_filtered_counter;
}
diff --git a/be/src/http/action/compaction_action.h
b/be/src/http/action/compaction_action.h
index 5a7b936..57513de 100644
--- a/be/src/http/action/compaction_action.h
+++ b/be/src/http/action/compaction_action.h
@@ -42,7 +42,7 @@ public:
CompactionAction(CompactionActionType type)
: _type(type) {
_compaction_mem_tracker = type == RUN_COMPACTION ?
- MemTracker::CreateTracker(-1, "ManualCompaction") : nullptr;
+ MemTracker::CreateTracker(-1, "ManualCompaction", nullptr,
false, false, MemTrackerLevel::TASK) : nullptr;
}
virtual ~CompactionAction() {}
diff --git a/be/src/http/default_path_handlers.cpp
b/be/src/http/default_path_handlers.cpp
index d11991d..d841697 100644
--- a/be/src/http/default_path_handlers.cpp
+++ b/be/src/http/default_path_handlers.cpp
@@ -129,7 +129,8 @@ void mem_tracker_handler(const WebPageHandler::ArgumentMap&
args, std::stringstr
" data-search='true' "
" class='table table-striped'>\n";
(*output) << "<thead><tr>"
- "<th>Id</th>"
+ "<th data-sortable='true' "
+ ">Id</th>"
"<th>Parent</th>"
"<th>Limit</th>"
"<th data-sorter='bytesSorter' "
@@ -137,7 +138,9 @@ void mem_tracker_handler(const WebPageHandler::ArgumentMap&
args, std::stringstr
">Current Consumption</th>"
"<th data-sorter='bytesSorter' "
" data-sortable='true' "
- ">Peak Consumption</th>";
+ ">Peak Consumption</th>"
+ "<th data-sortable='true' "
+ ">Use Count</th></tr></thead>";
(*output) << "<tbody>\n";
std::vector<shared_ptr<MemTracker>> trackers;
@@ -147,10 +150,11 @@ void mem_tracker_handler(const
WebPageHandler::ArgumentMap& args, std::stringstr
string limit_str = tracker->limit() == -1 ? "none" :
ItoaKMGT(tracker->limit());
string current_consumption_str = ItoaKMGT(tracker->consumption());
string peak_consumption_str = ItoaKMGT(tracker->peak_consumption());
+ int64_t use_count = tracker.use_count();
(*output) << strings::Substitute(
"<tr><td>$0</td><td>$1</td><td>$2</td>" // id, parent, limit
- "<td>$3</td><td>$4</td></tr>\n", // current, peak
- tracker->label(), parent, limit_str, current_consumption_str,
peak_consumption_str);
+ "<td>$3</td><td>$4</td><td>$5</td></tr>\n", // current,
peak
+ tracker->label(), parent, limit_str, current_consumption_str,
peak_consumption_str, use_count);
}
(*output) << "</tbody></table>\n";
}
diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp
index 383ae9c..f735be8 100644
--- a/be/src/olap/compaction.cpp
+++ b/be/src/olap/compaction.cpp
@@ -28,7 +28,7 @@ namespace doris {
Compaction::Compaction(TabletSharedPtr tablet, const std::string& label,
const std::shared_ptr<MemTracker>& parent_tracker)
- : _mem_tracker(MemTracker::CreateTracker(-1, label, parent_tracker,
true, false)),
+ : _mem_tracker(MemTracker::CreateTracker(-1, label, parent_tracker,
true, false, MemTrackerLevel::TASK)),
_readers_tracker(MemTracker::CreateTracker(-1,
"CompactionReaderTracker:" + std::to_string(tablet->tablet_id()), _mem_tracker,
true, false)),
_writer_tracker(MemTracker::CreateTracker(-1,
"CompationWriterTracker:" + std::to_string(tablet->tablet_id()), _mem_tracker,
@@ -177,7 +177,7 @@ OLAPStatus Compaction::construct_input_rowset_readers() {
RETURN_NOT_OK(rowset->create_reader(
MemTracker::CreateTracker(
-1, "Compaction:RowsetReader:" +
rowset->rowset_id().to_string(),
- _readers_tracker, true, true, MemTrackerLevel::DEBUG),
+ _readers_tracker, true, true),
&rs_reader));
_input_rs_readers.push_back(std::move(rs_reader));
}
diff --git a/be/src/olap/fs/file_block_manager.cpp
b/be/src/olap/fs/file_block_manager.cpp
index 2f49136..f62f95d 100644
--- a/be/src/olap/fs/file_block_manager.cpp
+++ b/be/src/olap/fs/file_block_manager.cpp
@@ -368,8 +368,8 @@ Status FileReadableBlock::readv(uint64_t offset, const
Slice* results, size_t re
FileBlockManager::FileBlockManager(Env* env, BlockManagerOptions opts)
: _env(DCHECK_NOTNULL(env)),
_opts(std::move(opts)),
- _mem_tracker(
- MemTracker::CreateTracker(-1, "FileBlockManager",
_opts.parent_mem_tracker)) {
+ _mem_tracker(MemTracker::CreateTracker(-1, "FileBlockManager",
_opts.parent_mem_tracker,
+ false, false, MemTrackerLevel::OVERVIEW)) {
if (_opts.enable_metric) {
_metrics.reset(new internal::BlockManagerMetrics());
}
diff --git a/be/src/olap/lru_cache.cpp b/be/src/olap/lru_cache.cpp
index f420cc5..b591909 100644
--- a/be/src/olap/lru_cache.cpp
+++ b/be/src/olap/lru_cache.cpp
@@ -412,7 +412,7 @@ uint32_t ShardedLRUCache::_shard(uint32_t hash) {
ShardedLRUCache::ShardedLRUCache(const std::string& name, size_t
total_capacity, std::shared_ptr<MemTracker> parent)
: _name(name), _last_id(1),
- _mem_tracker(MemTracker::CreateTracker(-1, name, parent, true, false))
{
+ _mem_tracker(MemTracker::CreateTracker(-1, name, parent, true, false,
MemTrackerLevel::OVERVIEW)) {
const size_t per_shard = (total_capacity + (kNumShards - 1)) / kNumShards;
for (int s = 0; s < kNumShards; s++) {
_shards[s].set_capacity(per_shard);
diff --git a/be/src/olap/page_cache.cpp b/be/src/olap/page_cache.cpp
index 6ce9fab..056ad63 100644
--- a/be/src/olap/page_cache.cpp
+++ b/be/src/olap/page_cache.cpp
@@ -29,7 +29,7 @@ void StoragePageCache::create_global_cache(size_t capacity,
int32_t index_cache_
StoragePageCache::StoragePageCache(size_t capacity, int32_t
index_cache_percentage)
: _index_cache_percentage(index_cache_percentage),
- _mem_tracker(MemTracker::CreateTracker(capacity, "StorgePageCache"))
{
+ _mem_tracker(MemTracker::CreateTracker(capacity, "StorgePageCache",
nullptr, true, true, MemTrackerLevel::OVERVIEW)) {
if (index_cache_percentage == 0) {
_data_page_cache =
std::unique_ptr<Cache>(new_lru_cache("DataPageCache", capacity, _mem_tracker));
} else if (index_cache_percentage == 100) {
diff --git a/be/src/olap/rowset/beta_rowset.cpp
b/be/src/olap/rowset/beta_rowset.cpp
index fc03a7c..a7a7dcb 100644
--- a/be/src/olap/rowset/beta_rowset.cpp
+++ b/be/src/olap/rowset/beta_rowset.cpp
@@ -50,7 +50,7 @@ OLAPStatus BetaRowset::do_load(bool /*use_cache*/,
std::shared_ptr<MemTracker> p
for (int seg_id = 0; seg_id < num_segments(); ++seg_id) {
std::string seg_path = segment_file_path(_rowset_path, rowset_id(),
seg_id);
std::shared_ptr<segment_v2::Segment> segment;
- auto s = segment_v2::Segment::open(seg_path, seg_id, _schema, parent,
&segment);
+ auto s = segment_v2::Segment::open(seg_path, seg_id, _schema,
&segment);
if (!s.ok()) {
LOG(WARNING) << "failed to open segment " << seg_path << " under
rowset " << unique_id()
<< " : " << s.to_string();
diff --git a/be/src/olap/rowset/beta_rowset_reader.cpp
b/be/src/olap/rowset/beta_rowset_reader.cpp
index d0e7496..ee13dad 100644
--- a/be/src/olap/rowset/beta_rowset_reader.cpp
+++ b/be/src/olap/rowset/beta_rowset_reader.cpp
@@ -91,7 +91,7 @@ OLAPStatus BetaRowsetReader::init(RowsetReaderContext*
read_context) {
std::vector<std::unique_ptr<RowwiseIterator>> seg_iterators;
for (auto& seg_ptr : _rowset->_segments) {
std::unique_ptr<RowwiseIterator> iter;
- auto s = seg_ptr->new_iterator(schema, read_options, &iter);
+ auto s = seg_ptr->new_iterator(schema, read_options, _parent_tracker,
&iter);
if (!s.ok()) {
LOG(WARNING) << "failed to create iterator[" << seg_ptr->id() <<
"]: " << s.to_string();
return OLAP_ERR_ROWSET_READER_INIT;
diff --git a/be/src/olap/rowset/segment_reader.cpp
b/be/src/olap/rowset/segment_reader.cpp
index 38d3d1f..8314d0a 100644
--- a/be/src/olap/rowset/segment_reader.cpp
+++ b/be/src/olap/rowset/segment_reader.cpp
@@ -58,7 +58,7 @@ SegmentReader::SegmentReader(const std::string file,
SegmentGroup* segment_group
_is_using_mmap(false),
_is_data_loaded(false),
_buffer_size(0),
- _tracker(MemTracker::CreateTracker(-1, "SegmentReader:" + file,
parent_tracker)),
+ _tracker(MemTracker::CreateTracker(-1, "SegmentReader:" + file,
parent_tracker, false)),
_mem_pool(new MemPool(_tracker.get())),
_shared_buffer(NULL),
_lru_cache(lru_cache),
diff --git a/be/src/olap/rowset/segment_v2/segment.cpp
b/be/src/olap/rowset/segment_v2/segment.cpp
index 992d8c4..fbfd52f 100644
--- a/be/src/olap/rowset/segment_v2/segment.cpp
+++ b/be/src/olap/rowset/segment_v2/segment.cpp
@@ -27,6 +27,7 @@
#include "olap/rowset/segment_v2/page_io.h"
#include "olap/rowset/segment_v2/segment_iterator.h"
#include "olap/rowset/segment_v2/segment_writer.h" // k_segment_magic_length
+#include "olap/storage_engine.h"
#include "olap/tablet_schema.h"
#include "util/crc32c.h"
#include "util/slice.h" // Slice
@@ -37,16 +38,22 @@ namespace segment_v2 {
using strings::Substitute;
Status Segment::open(std::string filename, uint32_t segment_id, const
TabletSchema* tablet_schema,
- std::shared_ptr<MemTracker> parent,
std::shared_ptr<Segment>* output) {
- std::shared_ptr<Segment> segment(new Segment(std::move(filename),
segment_id, tablet_schema, std::move(parent)));
+ std::shared_ptr<Segment>* output) {
+ std::shared_ptr<Segment> segment(new Segment(std::move(filename),
segment_id, tablet_schema));
RETURN_IF_ERROR(segment->_open());
output->swap(segment);
return Status::OK();
}
-Segment::Segment(std::string fname, uint32_t segment_id, const TabletSchema*
tablet_schema, std::shared_ptr<MemTracker> parent)
+Segment::Segment(std::string fname, uint32_t segment_id, const TabletSchema*
tablet_schema)
: _fname(std::move(fname)), _segment_id(segment_id),
- _tablet_schema(tablet_schema),
_mem_tracker(MemTracker::CreateTracker(-1, "Segment", std::move(parent), false,
true)) {}
+ _tablet_schema(tablet_schema) {
+#ifndef BE_TEST
+ _mem_tracker = MemTracker::CreateTracker(-1, "Segment",
StorageEngine::instance()->tablet_mem_tracker(), false);
+#else
+ _mem_tracker = MemTracker::CreateTracker(-1, "Segment", nullptr, false);
+#endif
+}
Segment::~Segment() {
_mem_tracker->Release(_mem_tracker->consumption());
@@ -59,6 +66,7 @@ Status Segment::_open() {
}
Status Segment::new_iterator(const Schema& schema, const StorageReadOptions&
read_options,
+ std::shared_ptr<MemTracker> parent,
std::unique_ptr<RowwiseIterator>* iter) {
DCHECK_NOTNULL(read_options.stats);
read_options.stats->total_segment_number++;
@@ -80,7 +88,7 @@ Status Segment::new_iterator(const Schema& schema, const
StorageReadOptions& rea
}
RETURN_IF_ERROR(_load_index());
- iter->reset(new SegmentIterator(this->shared_from_this(), schema,
_mem_tracker));
+ iter->reset(new SegmentIterator(this->shared_from_this(), schema, parent));
iter->get()->init(read_options);
return Status::OK();
}
@@ -188,7 +196,7 @@ Status Segment::_create_column_readers() {
return Status::OK();
}
-Status Segment::new_column_iterator(uint32_t cid, ColumnIterator** iter) {
+Status Segment::new_column_iterator(uint32_t cid, std::shared_ptr<MemTracker>
parent, ColumnIterator** iter) {
if (_column_readers[cid] == nullptr) {
const TabletColumn& tablet_column = _tablet_schema->column(cid);
if (!tablet_column.has_default_value() &&
!tablet_column.is_nullable()) {
@@ -200,7 +208,7 @@ Status Segment::new_column_iterator(uint32_t cid,
ColumnIterator** iter) {
tablet_column.has_default_value(),
tablet_column.default_value(),
tablet_column.is_nullable(), type_info,
tablet_column.length()));
ColumnIteratorOptions iter_opts;
- iter_opts.mem_tracker = MemTracker::CreateTracker(-1,
"DefaultColumnIterator", _mem_tracker, false);
+ iter_opts.mem_tracker = MemTracker::CreateTracker(-1,
"DefaultColumnIterator", parent, false);
RETURN_IF_ERROR(default_value_iter->init(iter_opts));
*iter = default_value_iter.release();
diff --git a/be/src/olap/rowset/segment_v2/segment.h
b/be/src/olap/rowset/segment_v2/segment.h
index cc883b1..0cfd408 100644
--- a/be/src/olap/rowset/segment_v2/segment.h
+++ b/be/src/olap/rowset/segment_v2/segment.h
@@ -60,18 +60,19 @@ using SegmentSharedPtr = std::shared_ptr<Segment>;
class Segment : public std::enable_shared_from_this<Segment> {
public:
static Status open(std::string filename, uint32_t segment_id, const
TabletSchema* tablet_schema,
- std::shared_ptr<MemTracker> parent,
std::shared_ptr<Segment>* output);
+ std::shared_ptr<Segment>* output);
~Segment();
Status new_iterator(const Schema& schema, const StorageReadOptions&
read_options,
+ std::shared_ptr<MemTracker> parent,
std::unique_ptr<RowwiseIterator>* iter);
uint64_t id() const { return _segment_id; }
uint32_t num_rows() const { return _footer.num_rows(); }
- Status new_column_iterator(uint32_t cid, ColumnIterator** iter);
+ Status new_column_iterator(uint32_t cid, std::shared_ptr<MemTracker>
parent, ColumnIterator** iter);
Status new_bitmap_index_iterator(uint32_t cid, BitmapIndexIterator** iter);
@@ -104,7 +105,7 @@ public:
private:
DISALLOW_COPY_AND_ASSIGN(Segment);
- Segment(std::string fname, uint32_t segment_id, const TabletSchema*
tablet_schema, std::shared_ptr<MemTracker> parent);
+ Segment(std::string fname, uint32_t segment_id, const TabletSchema*
tablet_schema);
// open segment file and read the minimum amount of necessary information
(footer)
Status _open();
Status _parse_footer();
@@ -119,6 +120,8 @@ private:
uint32_t _segment_id;
const TabletSchema* _tablet_schema;
+ // This mem tracker is only for tracking memory use by segment meta data
such as footer or index page.
+ // The memory consumed by querying is tracked in segment iterator.
std::shared_ptr<MemTracker> _mem_tracker;
SegmentFooterPB _footer;
diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
index 859cca5..50e8feb 100644
--- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
@@ -99,7 +99,7 @@ SegmentIterator::SegmentIterator(std::shared_ptr<Segment>
segment, const Schema&
_lazy_materialization_read(false),
_inited(false) {
// use for count the mem use of ColumnIterator
- _mem_tracker = MemTracker::CreateTracker(-1, "SegmentIterator",
std::move(parent), 0);
+ _mem_tracker = MemTracker::CreateTracker(-1, "SegmentIterator",
std::move(parent), false);
}
SegmentIterator::~SegmentIterator() {
@@ -194,7 +194,7 @@ Status SegmentIterator::_prepare_seek(const
StorageReadOptions::KeyRange& key_ra
// create used column iterator
for (auto cid : _seek_schema->column_ids()) {
if (_column_iterators[cid] == nullptr) {
- RETURN_IF_ERROR(_segment->new_column_iterator(cid,
&_column_iterators[cid]));
+ RETURN_IF_ERROR(_segment->new_column_iterator(cid, _mem_tracker,
&_column_iterators[cid]));
ColumnIteratorOptions iter_opts;
iter_opts.stats = _opts.stats;
iter_opts.rblock = _rblock.get();
@@ -322,7 +322,7 @@ Status SegmentIterator::_init_return_column_iterators() {
}
for (auto cid : _schema.column_ids()) {
if (_column_iterators[cid] == nullptr) {
- RETURN_IF_ERROR(_segment->new_column_iterator(cid,
&_column_iterators[cid]));
+ RETURN_IF_ERROR(_segment->new_column_iterator(cid, _mem_tracker,
&_column_iterators[cid]));
ColumnIteratorOptions iter_opts;
iter_opts.stats = _opts.stats;
iter_opts.use_page_cache = _opts.use_page_cache;
diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp
index 7d69f60..70b5d15 100644
--- a/be/src/olap/schema_change.cpp
+++ b/be/src/olap/schema_change.cpp
@@ -1361,7 +1361,7 @@ bool
SchemaChangeWithSorting::_external_sorting(vector<RowsetSharedPtr>& src_row
}
SchemaChangeHandler::SchemaChangeHandler()
- : _mem_tracker(MemTracker::CreateTracker(-1, "SchemaChange")) {
+ : _mem_tracker(MemTracker::CreateTracker(-1, "SchemaChange",
StorageEngine::instance()->schema_change_mem_tracker())) {
REGISTER_HOOK_METRIC(schema_change_mem_consumption,
[this]() { return _mem_tracker->consumption(); });
}
@@ -1474,7 +1474,7 @@ OLAPStatus
SchemaChangeHandler::_do_process_alter_tablet_v2(const TAlterTabletRe
reader_context.seek_columns = &return_columns;
auto mem_tracker = MemTracker::CreateTracker(-1, "AlterTablet:" +
std::to_string(base_tablet->tablet_id()) + "-"
- + std::to_string(new_tablet->tablet_id()), _mem_tracker, true, false,
MemTrackerLevel::DEBUG);
+ + std::to_string(new_tablet->tablet_id()), _mem_tracker, true, false,
MemTrackerLevel::TASK);
do {
// get history data to be converted and it will check if there is hold
in base tablet
diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp
index e5eaf71..fd20d31 100644
--- a/be/src/olap/storage_engine.cpp
+++ b/be/src/olap/storage_engine.cpp
@@ -113,7 +113,9 @@ StorageEngine::StorageEngine(const EngineOptions& options)
_index_stream_lru_cache(NULL),
_file_cache(nullptr),
_compaction_mem_tracker(
- MemTracker::CreateTracker(-1, "AutoCompaction")),
+ MemTracker::CreateTracker(-1, "AutoCompaction", nullptr,
false, false, MemTrackerLevel::OVERVIEW)),
+ _tablet_mem_tracker(
+ MemTracker::CreateTracker(-1, "TabletHeader", nullptr,
false, false, MemTrackerLevel::OVERVIEW)),
_stop_background_threads_latch(1),
_tablet_manager(new TabletManager(config::tablet_map_shard_size)),
_txn_manager(new TxnManager(config::txn_map_shard_size,
config::txn_shard_size)),
@@ -1028,14 +1030,14 @@ bool
StorageEngine::check_rowset_id_in_unused_rowsets(const RowsetId& rowset_id)
void StorageEngine::create_cumulative_compaction(
TabletSharedPtr best_tablet, std::shared_ptr<CumulativeCompaction>&
cumulative_compaction) {
- std::string tracker_label = "StorageEngine:CumulativeCompaction:" +
std::to_string(syscall(__NR_gettid));
+ std::string tracker_label = "StorageEngine:CumulativeCompaction:" +
std::to_string(best_tablet->tablet_id());
cumulative_compaction.reset(
new CumulativeCompaction(best_tablet, tracker_label,
_compaction_mem_tracker));
}
void StorageEngine::create_base_compaction(TabletSharedPtr best_tablet,
std::shared_ptr<BaseCompaction>&
base_compaction) {
- std::string tracker_label = "StorageEngine:BaseCompaction:" +
std::to_string(syscall(__NR_gettid));
+ std::string tracker_label = "StorageEngine:BaseCompaction:" +
std::to_string(best_tablet->tablet_id());
base_compaction.reset(new BaseCompaction(best_tablet, tracker_label,
_compaction_mem_tracker));
}
diff --git a/be/src/olap/storage_engine.h b/be/src/olap/storage_engine.h
index 8980e82..26afb53 100644
--- a/be/src/olap/storage_engine.h
+++ b/be/src/olap/storage_engine.h
@@ -179,6 +179,9 @@ public:
Status get_compaction_status_json(std::string* result);
+ std::shared_ptr<MemTracker> tablet_mem_tracker() { return
_tablet_mem_tracker; }
+ std::shared_ptr<MemTracker> schema_change_mem_tracker() { return
_schema_change_mem_tracker; }
+
private:
// Instance should be inited from `static open()`
// MUST NOT be called in other circumstances.
@@ -306,6 +309,7 @@ private:
std::unordered_map<std::string, RowsetSharedPtr> _unused_rowsets;
std::shared_ptr<MemTracker> _compaction_mem_tracker;
+ std::shared_ptr<MemTracker> _tablet_mem_tracker;
std::shared_ptr<MemTracker> _schema_change_mem_tracker;
CountDownLatch _stop_background_threads_latch;
diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp
index b1e7cf3..09a9e87 100644
--- a/be/src/olap/tablet_manager.cpp
+++ b/be/src/olap/tablet_manager.cpp
@@ -71,7 +71,7 @@ static bool _cmp_tablet_by_create_time(const TabletSharedPtr&
a, const TabletSha
}
TabletManager::TabletManager(int32_t tablet_map_lock_shard_size)
- : _mem_tracker(MemTracker::CreateTracker(-1, "TabletManager", nullptr,
false)),
+ : _mem_tracker(MemTracker::CreateTracker(-1, "TabletMeta", nullptr,
false, false, MemTrackerLevel::OVERVIEW)),
_tablets_shards_size(tablet_map_lock_shard_size),
_tablets_shards_mask(tablet_map_lock_shard_size - 1),
_last_update_stat_ms(0) {
diff --git a/be/src/runtime/buffered_block_mgr2.cc
b/be/src/runtime/buffered_block_mgr2.cc
index 9621092..48d4a76 100644
--- a/be/src/runtime/buffered_block_mgr2.cc
+++ b/be/src/runtime/buffered_block_mgr2.cc
@@ -1288,8 +1288,6 @@ void BufferedBlockMgr2::init(DiskIoMgr* io_mgr,
RuntimeProfile* parent_profile,
_integrity_check_timer = ADD_TIMER(_profile.get(),
"TotalIntegrityCheckTime");
// Create a new mem_tracker and allocate buffers.
- // _mem_tracker.reset(new MemTracker(
- // profile(), mem_limit, -1, "Block Manager", parent_tracker));
_mem_tracker = MemTracker::CreateTracker(mem_limit, "BufferedBlockMgr2",
parent_tracker);
_initialized = true;
diff --git a/be/src/runtime/exec_env.h b/be/src/runtime/exec_env.h
index 5968a8e..2fcf901 100644
--- a/be/src/runtime/exec_env.h
+++ b/be/src/runtime/exec_env.h
@@ -140,6 +140,9 @@ public:
PluginMgr* plugin_mgr() { return _plugin_mgr; }
+ // The root tracker should be set before calling ExecEnv::init();
+ void set_root_mem_tracker(std::shared_ptr<MemTracker> root_tracker);
+
private:
Status _init(const std::vector<StorePath>& store_paths);
void _destroy();
diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp
index 7bd31a3..63095d5 100644
--- a/be/src/runtime/exec_env_init.cpp
+++ b/be/src/runtime/exec_env_init.cpp
@@ -185,8 +185,8 @@ Status ExecEnv::_init_mem_tracker() {
return Status::InternalError(ss.str());
}
- _mem_tracker = MemTracker::CreateTracker(bytes_limit, "Query",
MemTracker::GetRootTracker(),
- false, false);
+ _mem_tracker = MemTracker::CreateTracker(bytes_limit, "Process",
MemTracker::GetRootTracker(),
+ false, false, MemTrackerLevel::OVERVIEW);
LOG(INFO) << "Using global memory limit: " <<
PrettyPrinter::print(bytes_limit, TUnit::BYTES);
RETURN_IF_ERROR(_disk_io_mgr->init(_mem_tracker));
diff --git a/be/src/runtime/load_channel.cpp b/be/src/runtime/load_channel.cpp
index d6ade09..43f8a88 100644
--- a/be/src/runtime/load_channel.cpp
+++ b/be/src/runtime/load_channel.cpp
@@ -27,7 +27,7 @@ LoadChannel::LoadChannel(const UniqueId& load_id, int64_t
mem_limit, int64_t tim
const std::shared_ptr<MemTracker>& mem_tracker)
: _load_id(load_id), _timeout_s(timeout_s) {
_mem_tracker = MemTracker::CreateTracker(
- mem_limit, "LoadChannel:" + _load_id.to_string(), mem_tracker,
true, false);
+ mem_limit, "LoadChannel:" + _load_id.to_string(), mem_tracker,
true, false, MemTrackerLevel::TASK);
// _last_updated_time should be set before being inserted to
// _load_channels in load_channel_mgr, or it may be erased
// immediately by gc thread.
diff --git a/be/src/runtime/load_channel_mgr.cpp
b/be/src/runtime/load_channel_mgr.cpp
index 15e2c08..f0110aa 100644
--- a/be/src/runtime/load_channel_mgr.cpp
+++ b/be/src/runtime/load_channel_mgr.cpp
@@ -85,7 +85,7 @@ LoadChannelMgr::~LoadChannelMgr() {
Status LoadChannelMgr::init(int64_t process_mem_limit) {
int64_t load_mem_limit = calc_process_max_load_memory(process_mem_limit);
- _mem_tracker = MemTracker::CreateTracker(load_mem_limit, "Load");
+ _mem_tracker = MemTracker::CreateTracker(load_mem_limit, "LoadChannelMgr",
nullptr, true, false, MemTrackerLevel::OVERVIEW);
REGISTER_HOOK_METRIC(load_mem_consumption, [this]() {
return _mem_tracker->consumption();
});
diff --git a/be/src/runtime/mem_tracker.cpp b/be/src/runtime/mem_tracker.cpp
index 9177e72..ea0a169 100644
--- a/be/src/runtime/mem_tracker.cpp
+++ b/be/src/runtime/mem_tracker.cpp
@@ -72,7 +72,7 @@ static std::shared_ptr<MemTracker> root_tracker;
static GoogleOnceType root_tracker_once = GOOGLE_ONCE_INIT;
void MemTracker::CreateRootTracker() {
- root_tracker.reset(new MemTracker(-1, "Root"));
+ root_tracker.reset(new MemTracker(nullptr, -1, "Root", nullptr, true,
MemTrackerLevel::OVERVIEW));
root_tracker->Init();
}
@@ -143,7 +143,7 @@ std::shared_ptr<MemTracker>
MemTracker::CreateTracker(int64_t byte_limit, const
}
MemTracker::MemTracker(int64_t byte_limit, const std::string& label)
- : MemTracker(nullptr, byte_limit, label,
std::shared_ptr<MemTracker>(), true, MemTrackerLevel::RELEASE) {}
+ : MemTracker(nullptr, byte_limit, label,
std::shared_ptr<MemTracker>(), true, MemTrackerLevel::VERBOSE) {}
MemTracker::MemTracker(RuntimeProfile* profile, int64_t byte_limit, const
string& label,
const std::shared_ptr<MemTracker>& parent, bool
log_usage_if_zero, MemTrackerLevel level)
diff --git a/be/src/runtime/mem_tracker.h b/be/src/runtime/mem_tracker.h
index 142e611..b938f6e 100644
--- a/be/src/runtime/mem_tracker.h
+++ b/be/src/runtime/mem_tracker.h
@@ -42,7 +42,7 @@ enum class MemLimit { HARD, SOFT };
/// The Level use to decide whether to show it in web page
/// each MemTracker have a Level equals to parent, only be set explicit
-enum class MemTrackerLevel {RELEASE = 0, DEBUG};
+enum class MemTrackerLevel {OVERVIEW = 0, TASK, VERBOSE};
class ObjectPool;
class MemTracker;
@@ -92,11 +92,11 @@ public:
static std::shared_ptr<MemTracker> CreateTracker(
int64_t byte_limit = -1, const std::string& label = std::string(),
std::shared_ptr<MemTracker> parent = std::shared_ptr<MemTracker>(),
- bool log_usage_if_zero = true, bool reset_label_name = true,
MemTrackerLevel level = MemTrackerLevel::RELEASE);
+ bool log_usage_if_zero = true, bool reset_label_name = true,
MemTrackerLevel level = MemTrackerLevel::VERBOSE);
static std::shared_ptr<MemTracker> CreateTracker(
RuntimeProfile* profile, int64_t byte_limit, const std::string&
label = std::string(),
- const std::shared_ptr<MemTracker>& parent =
std::shared_ptr<MemTracker>(), bool reset_label_name = true, MemTrackerLevel
level = MemTrackerLevel::RELEASE);
+ const std::shared_ptr<MemTracker>& parent =
std::shared_ptr<MemTracker>(), bool reset_label_name = true, MemTrackerLevel
level = MemTrackerLevel::VERBOSE);
// this is used for creating an orphan mem tracker, or for unit test.
// If a mem tracker has parent, it should be created by `CreateTracker()`
diff --git a/be/src/runtime/plan_fragment_executor.cpp
b/be/src/runtime/plan_fragment_executor.cpp
index 3bd8a1b..2df18a5 100644
--- a/be/src/runtime/plan_fragment_executor.cpp
+++ b/be/src/runtime/plan_fragment_executor.cpp
@@ -132,7 +132,7 @@ Status PlanFragmentExecutor::prepare(const
TExecPlanFragmentParams& request,
_mem_tracker = MemTracker::CreateTracker(bytes_limit,
"PlanFragmentExecutor:" +
print_id(_query_id) + ":" +
print_id(params.fragment_instance_id),
- _exec_env->process_mem_tracker(),
true, false);
+ _exec_env->process_mem_tracker(),
true, false, MemTrackerLevel::TASK);
_runtime_state->set_fragment_mem_tracker(_mem_tracker);
LOG(INFO) << "Using query memory limit: " <<
PrettyPrinter::print(bytes_limit, TUnit::BYTES);
diff --git a/be/test/olap/rowset/segment_v2/segment_test.cpp
b/be/test/olap/rowset/segment_v2/segment_test.cpp
index 2efb213..cbb3ac8 100644
--- a/be/test/olap/rowset/segment_v2/segment_test.cpp
+++ b/be/test/olap/rowset/segment_v2/segment_test.cpp
@@ -136,7 +136,7 @@ protected:
ASSERT_TRUE(st.ok());
ASSERT_TRUE(wblock->close().ok());
- st = Segment::open(filename, 0, &query_schema,
MemTracker::CreateTracker(-1, "Segment", nullptr, false), res);
+ st = Segment::open(filename, 0, &query_schema, res);
ASSERT_TRUE(st.ok());
ASSERT_EQ(nrows, (*res)->num_rows());
}
@@ -164,7 +164,7 @@ TEST_F(SegmentReaderWriterTest, normal) {
StorageReadOptions read_opts;
read_opts.stats = &stats;
std::unique_ptr<RowwiseIterator> iter;
- segment->new_iterator(schema, read_opts, &iter);
+ segment->new_iterator(schema, read_opts, nullptr, &iter);
RowBlockV2 block(schema, 1024);
@@ -220,7 +220,7 @@ TEST_F(SegmentReaderWriterTest, normal) {
read_opts.stats = &stats;
read_opts.key_ranges.emplace_back(lower_bound.get(), false,
upper_bound.get(), true);
std::unique_ptr<RowwiseIterator> iter;
- segment->new_iterator(schema, read_opts, &iter);
+ segment->new_iterator(schema, read_opts, nullptr, &iter);
RowBlockV2 block(schema, 100);
ASSERT_TRUE(iter->next_batch(&block).ok());
@@ -246,7 +246,7 @@ TEST_F(SegmentReaderWriterTest, normal) {
read_opts.stats = &stats;
read_opts.key_ranges.emplace_back(lower_bound.get(), false,
nullptr, false);
std::unique_ptr<RowwiseIterator> iter;
- segment->new_iterator(schema, read_opts, &iter);
+ segment->new_iterator(schema, read_opts, nullptr, &iter);
RowBlockV2 block(schema, 100);
ASSERT_TRUE(iter->next_batch(&block).is_end_of_file());
@@ -275,7 +275,7 @@ TEST_F(SegmentReaderWriterTest, normal) {
read_opts.stats = &stats;
read_opts.key_ranges.emplace_back(lower_bound.get(), false,
upper_bound.get(), false);
std::unique_ptr<RowwiseIterator> iter;
- segment->new_iterator(schema, read_opts, &iter);
+ segment->new_iterator(schema, read_opts, nullptr, &iter);
RowBlockV2 block(schema, 100);
ASSERT_TRUE(iter->next_batch(&block).is_end_of_file());
@@ -312,7 +312,7 @@ TEST_F(SegmentReaderWriterTest, LazyMaterialization) {
read_opts.stats = &stats;
std::unique_ptr<RowwiseIterator> iter;
- ASSERT_TRUE(segment->new_iterator(read_schema, read_opts,
&iter).ok());
+ ASSERT_TRUE(segment->new_iterator(read_schema, read_opts, nullptr,
&iter).ok());
RowBlockV2 block(read_schema, 1024);
ASSERT_TRUE(iter->next_batch(&block).ok());
@@ -336,7 +336,7 @@ TEST_F(SegmentReaderWriterTest, LazyMaterialization) {
read_opts.stats = &stats;
std::unique_ptr<RowwiseIterator> iter;
- ASSERT_TRUE(segment->new_iterator(read_schema, read_opts,
&iter).ok());
+ ASSERT_TRUE(segment->new_iterator(read_schema, read_opts, nullptr,
&iter).ok());
RowBlockV2 block(read_schema, 1024);
ASSERT_TRUE(iter->next_batch(&block).ok());
@@ -356,7 +356,7 @@ TEST_F(SegmentReaderWriterTest, LazyMaterialization) {
read_opts.stats = &stats;
std::unique_ptr<RowwiseIterator> iter;
- ASSERT_TRUE(segment->new_iterator(read_schema, read_opts,
&iter).ok());
+ ASSERT_TRUE(segment->new_iterator(read_schema, read_opts, nullptr,
&iter).ok());
RowBlockV2 block(read_schema, 1024);
ASSERT_TRUE(iter->next_batch(&block).ok());
@@ -388,7 +388,7 @@ TEST_F(SegmentReaderWriterTest, LazyMaterialization) {
read_opts.stats = &stats;
std::unique_ptr<RowwiseIterator> iter;
- ASSERT_TRUE(segment->new_iterator(read_schema, read_opts,
&iter).ok());
+ ASSERT_TRUE(segment->new_iterator(read_schema, read_opts, nullptr,
&iter).ok());
RowBlockV2 block(read_schema, 1024);
ASSERT_TRUE(iter->next_batch(&block).ok());
@@ -450,7 +450,7 @@ TEST_F(SegmentReaderWriterTest, TestIndex) {
read_opts.conditions = conditions.get();
std::unique_ptr<RowwiseIterator> iter;
- segment->new_iterator(schema, read_opts, &iter);
+ segment->new_iterator(schema, read_opts, nullptr, &iter);
RowBlockV2 block(schema, 1);
@@ -473,7 +473,7 @@ TEST_F(SegmentReaderWriterTest, TestIndex) {
read_opts.conditions = conditions.get();
std::unique_ptr<RowwiseIterator> iter;
- segment->new_iterator(schema, read_opts, &iter);
+ segment->new_iterator(schema, read_opts, nullptr, &iter);
RowBlockV2 block(schema, 1024);
@@ -533,7 +533,7 @@ TEST_F(SegmentReaderWriterTest, TestIndex) {
read_opts.delete_conditions.push_back(delete_conditions.get());
std::unique_ptr<RowwiseIterator> iter;
- segment->new_iterator(schema, read_opts, &iter);
+ segment->new_iterator(schema, read_opts, nullptr, &iter);
RowBlockV2 block(schema, 1024);
@@ -581,7 +581,7 @@ TEST_F(SegmentReaderWriterTest, TestIndex) {
ASSERT_EQ(OLAP_SUCCESS, conditions->append_condition(condition));
read_opts.conditions = conditions.get();
std::unique_ptr<RowwiseIterator> iter;
- segment->new_iterator(schema, read_opts, &iter);
+ segment->new_iterator(schema, read_opts, nullptr, &iter);
RowBlockV2 block(schema, 1024);
ASSERT_TRUE(iter->next_batch(&block).is_end_of_file());
@@ -674,7 +674,7 @@ TEST_F(SegmentReaderWriterTest, TestDefaultValueColumn) {
StorageReadOptions read_opts;
read_opts.stats = &stats;
std::unique_ptr<RowwiseIterator> iter;
- segment->new_iterator(schema, read_opts, &iter);
+ segment->new_iterator(schema, read_opts, nullptr, &iter);
RowBlockV2 block(schema, 1024);
@@ -724,7 +724,7 @@ TEST_F(SegmentReaderWriterTest, TestDefaultValueColumn) {
StorageReadOptions read_opts;
read_opts.stats = &stats;
std::unique_ptr<RowwiseIterator> iter;
- segment->new_iterator(schema, read_opts, &iter);
+ segment->new_iterator(schema, read_opts, nullptr, &iter);
RowBlockV2 block(schema, 1024);
@@ -817,7 +817,7 @@ TEST_F(SegmentReaderWriterTest, TestStringDict) {
{
std::shared_ptr<Segment> segment;
- st = Segment::open(fname, 0, tablet_schema.get(),
MemTracker::CreateTracker(-1, "Segment", nullptr, false), &segment);
+ st = Segment::open(fname, 0, tablet_schema.get(), &segment);
ASSERT_TRUE(st.ok());
ASSERT_EQ(4096, segment->num_rows());
Schema schema(*tablet_schema);
@@ -827,7 +827,7 @@ TEST_F(SegmentReaderWriterTest, TestStringDict) {
StorageReadOptions read_opts;
read_opts.stats = &stats;
std::unique_ptr<RowwiseIterator> iter;
- segment->new_iterator(schema, read_opts, &iter);
+ segment->new_iterator(schema, read_opts, nullptr, &iter);
RowBlockV2 block(schema, 1024);
@@ -880,7 +880,7 @@ TEST_F(SegmentReaderWriterTest, TestStringDict) {
read_opts.stats = &stats;
read_opts.key_ranges.emplace_back(lower_bound.get(), false,
nullptr, false);
std::unique_ptr<RowwiseIterator> iter;
- segment->new_iterator(schema, read_opts, &iter);
+ segment->new_iterator(schema, read_opts, nullptr, &iter);
RowBlockV2 block(schema, 100);
st = iter->next_batch(&block);
@@ -913,7 +913,7 @@ TEST_F(SegmentReaderWriterTest, TestStringDict) {
read_opts.stats = &stats;
read_opts.key_ranges.emplace_back(lower_bound.get(), false,
upper_bound.get(), false);
std::unique_ptr<RowwiseIterator> iter;
- segment->new_iterator(schema, read_opts, &iter);
+ segment->new_iterator(schema, read_opts, nullptr, &iter);
RowBlockV2 block(schema, 100);
st = iter->next_batch(&block);
@@ -937,7 +937,7 @@ TEST_F(SegmentReaderWriterTest, TestStringDict) {
read_opts.conditions = conditions.get();
std::unique_ptr<RowwiseIterator> iter;
- segment->new_iterator(schema, read_opts, &iter);
+ segment->new_iterator(schema, read_opts, nullptr, &iter);
RowBlockV2 block(schema, 1024);
int left = 4 * 1024;
@@ -994,7 +994,7 @@ TEST_F(SegmentReaderWriterTest, TestStringDict) {
read_opts.conditions = conditions.get();
std::unique_ptr<RowwiseIterator> iter;
- segment->new_iterator(schema, read_opts, &iter);
+ segment->new_iterator(schema, read_opts, nullptr, &iter);
RowBlockV2 block(schema, 1024);
@@ -1033,7 +1033,7 @@ TEST_F(SegmentReaderWriterTest, TestBitmapPredicate) {
read_opts.stats = &stats;
std::unique_ptr<RowwiseIterator> iter;
- segment->new_iterator(schema, read_opts, &iter);
+ segment->new_iterator(schema, read_opts, nullptr, &iter);
RowBlockV2 block(schema, 1024);
ASSERT_TRUE(iter->next_batch(&block).ok());
@@ -1055,7 +1055,7 @@ TEST_F(SegmentReaderWriterTest, TestBitmapPredicate) {
read_opts.stats = &stats;
std::unique_ptr<RowwiseIterator> iter;
- segment->new_iterator(schema, read_opts, &iter);
+ segment->new_iterator(schema, read_opts, nullptr, &iter);
RowBlockV2 block(schema, 1024);
ASSERT_TRUE(iter->next_batch(&block).ok());
@@ -1077,7 +1077,7 @@ TEST_F(SegmentReaderWriterTest, TestBitmapPredicate) {
read_opts.stats = &stats;
std::unique_ptr<RowwiseIterator> iter;
- segment->new_iterator(schema, read_opts, &iter);
+ segment->new_iterator(schema, read_opts, nullptr, &iter);
RowBlockV2 block(schema, 1024);
ASSERT_FALSE(iter->next_batch(&block).ok());
@@ -1101,7 +1101,7 @@ TEST_F(SegmentReaderWriterTest, TestBitmapPredicate) {
read_opts.stats = &stats;
std::unique_ptr<RowwiseIterator> iter;
- segment->new_iterator(schema, read_opts, &iter);
+ segment->new_iterator(schema, read_opts, nullptr, &iter);
RowBlockV2 block(schema, 1024);
ASSERT_TRUE(iter->next_batch(&block).ok());
@@ -1124,7 +1124,7 @@ TEST_F(SegmentReaderWriterTest, TestBitmapPredicate) {
read_opts.stats = &stats;
std::unique_ptr<RowwiseIterator> iter;
- segment->new_iterator(schema, read_opts, &iter);
+ segment->new_iterator(schema, read_opts, nullptr, &iter);
RowBlockV2 block(schema, 1024);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
