This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 1a81b9e [MemTracker] Some enchance of MemTracker (#5783)
1a81b9e is described below
commit 1a81b9e1609370923472ab5af7db8036046df137
Author: HappenLee <[email protected]>
AuthorDate: Tue May 18 20:27:50 2021 -0500
[MemTracker] Some enchance of MemTracker (#5783)
1 Make some MemTracker have reasonable parent MemTracker not the root
tracker
2 Make each MemTracker can be easily to trace.
3 Add show level of MemTracker to reduce the MemTracker show in the web
page to have a way to control show how many tracker in web page.
---
be/src/common/config.h | 6 ++
be/src/exec/olap_scanner.cpp | 14 ++---
be/src/exec/olap_scanner.h | 2 +
be/src/exec/tablet_sink.cpp | 2 +-
be/src/olap/compaction.cpp | 10 ++--
be/src/olap/lru_cache.cpp | 2 +-
be/src/olap/memtable.cpp | 2 +-
be/src/olap/row_block2.cpp | 3 +-
be/src/olap/rowset/beta_rowset_reader.cpp | 21 ++++---
be/src/olap/rowset/beta_rowset_reader.h | 2 +-
be/src/olap/rowset/segment_v2/segment.cpp | 2 +-
be/src/olap/rowset/segment_v2/segment_iterator.cpp | 2 +-
be/src/olap/schema_change.cpp | 9 ++-
be/src/olap/tablet.cpp | 10 ++--
be/src/olap/tablet.h | 7 ++-
be/src/runtime/data_stream_recvr.cc | 2 +-
be/src/runtime/data_stream_sender.cpp | 3 +-
be/src/runtime/exec_env_init.cpp | 4 +-
be/src/runtime/export_sink.cpp | 3 +-
be/src/runtime/load_channel.cpp | 2 +-
be/src/runtime/mem_tracker.cpp | 64 +++++++++++++++++-----
be/src/runtime/mem_tracker.h | 12 +++-
be/src/runtime/plan_fragment_executor.cpp | 2 +-
be/src/runtime/runtime_state.cpp | 4 +-
be/src/runtime/tablets_channel.cpp | 2 +-
docs/en/administrator-guide/config/be_config.md | 14 ++++-
docs/zh-CN/administrator-guide/config/be_config.md | 10 ++++
27 files changed, 150 insertions(+), 66 deletions(-)
diff --git a/be/src/common/config.h b/be/src/common/config.h
index c9a8790..d70f935 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -598,6 +598,12 @@ CONF_Int32(aws_log_level, "3");
// the buffer size when read data from remote storage like s3
CONF_mInt32(remote_storage_read_buffer_mb, "256");
+// Default level of MemTracker to show in web page
+// now MemTracker support two level:
+// RELEASE: 0
+// DEBUG: 1
+// the level equal or lower than mem_tracker_level will show in web page
+CONF_Int16(mem_tracker_level, "0");
} // namespace config
} // namespace doris
diff --git a/be/src/exec/olap_scanner.cpp b/be/src/exec/olap_scanner.cpp
index 76273d7..10ce02d 100644
--- a/be/src/exec/olap_scanner.cpp
+++ b/be/src/exec/olap_scanner.cpp
@@ -47,7 +47,11 @@ OlapScanner::OlapScanner(RuntimeState* runtime_state,
OlapScanNode* parent, bool
_aggregation(aggregation),
_need_agg_finalize(need_agg_finalize),
_tuple_idx(parent->_tuple_idx),
- _direct_conjunct_size(parent->_direct_conjunct_size) {
+ _direct_conjunct_size(parent->_direct_conjunct_size),
+
_mem_tracker(MemTracker::CreateTracker(runtime_state->fragment_mem_tracker()->limit(),
+ "OlapScanner",
+
runtime_state->fragment_mem_tracker(),
+ true, true,
MemTrackerLevel::DEBUG)) {
_reader.reset(new Reader());
DCHECK(_reader.get() != NULL);
@@ -90,7 +94,7 @@ Status OlapScanner::prepare(const TPaloScanRange& scan_range,
// the rowsets maybe compacted when the last olap scanner starts
Version rd_version(0, _version);
OLAPStatus acquire_reader_st =
- _tablet->capture_rs_readers(rd_version,
&_params.rs_readers);
+ _tablet->capture_rs_readers(rd_version,
&_params.rs_readers, _mem_tracker);
if (acquire_reader_st != OLAP_SUCCESS) {
LOG(WARNING) << "fail to init reader.res=" <<
acquire_reader_st;
std::stringstream ss;
@@ -248,11 +252,7 @@ Status OlapScanner::get_batch(RuntimeState* state,
RowBatch* batch, bool* eof) {
bzero(tuple_buf, state->batch_size() * _tuple_desc->byte_size());
Tuple* tuple = reinterpret_cast<Tuple*>(tuple_buf);
- auto tracker =
MemTracker::CreateTracker(state->fragment_mem_tracker()->limit(),
- "OlapScanner:" +
print_id(state->query_id()),
- state->fragment_mem_tracker());
- std::unique_ptr<MemPool> mem_pool(new MemPool(tracker.get()));
-
+ std::unique_ptr<MemPool> mem_pool(new MemPool(_mem_tracker.get()));
int64_t raw_rows_threshold = raw_rows_read() +
config::doris_scanner_row_num;
{
SCOPED_TIMER(_parent->_scan_timer);
diff --git a/be/src/exec/olap_scanner.h b/be/src/exec/olap_scanner.h
index 08b6035..15cf13b 100644
--- a/be/src/exec/olap_scanner.h
+++ b/be/src/exec/olap_scanner.h
@@ -145,6 +145,8 @@ private:
bool _is_closed = false;
MonotonicStopWatch _watcher;
+
+ std::shared_ptr<MemTracker> _mem_tracker;
};
} // namespace doris
diff --git a/be/src/exec/tablet_sink.cpp b/be/src/exec/tablet_sink.cpp
index 5208941..60b1706 100644
--- a/be/src/exec/tablet_sink.cpp
+++ b/be/src/exec/tablet_sink.cpp
@@ -534,7 +534,7 @@ Status OlapTableSink::prepare(RuntimeState* state) {
// profile must add to state's object pool
_profile = state->obj_pool()->add(new RuntimeProfile("OlapTableSink"));
_mem_tracker = MemTracker::CreateTracker(-1, "OlapTableSink:" +
std::to_string(state->load_job_id()),
- state->instance_mem_tracker());
+ state->instance_mem_tracker(),
true, false);
SCOPED_TIMER(_profile->total_time_counter());
diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp
index 6a47c89..ee57e39 100644
--- a/be/src/olap/compaction.cpp
+++ b/be/src/olap/compaction.cpp
@@ -28,9 +28,11 @@ namespace doris {
Compaction::Compaction(TabletSharedPtr tablet, const std::string& label,
const std::shared_ptr<MemTracker>& parent_tracker)
- : _mem_tracker(MemTracker::CreateTracker(-1, label, parent_tracker)),
- _readers_tracker(MemTracker::CreateTracker(-1,
"CompactionReaderTracker:" + std::to_string(tablet->tablet_id()),
_mem_tracker)),
- _writer_tracker(MemTracker::CreateTracker(-1,
"CompationWriterTracker:" + std::to_string(tablet->tablet_id()), _mem_tracker)),
+ : _mem_tracker(MemTracker::CreateTracker(-1, label, parent_tracker,
true, false)),
+ _readers_tracker(MemTracker::CreateTracker(-1,
"CompactionReaderTracker:" + std::to_string(tablet->tablet_id()), _mem_tracker,
+ true, false)),
+ _writer_tracker(MemTracker::CreateTracker(-1,
"CompationWriterTracker:" + std::to_string(tablet->tablet_id()), _mem_tracker,
+ true, false)),
_tablet(tablet),
_input_rowsets_size(0),
_input_row_num(0),
@@ -171,7 +173,7 @@ OLAPStatus Compaction::construct_input_rowset_readers() {
RETURN_NOT_OK(rowset->create_reader(
MemTracker::CreateTracker(
-1, "Compaction:RowsetReader:" +
rowset->rowset_id().to_string(),
- _readers_tracker),
+ _readers_tracker, true, true, MemTrackerLevel::DEBUG),
&rs_reader));
_input_rs_readers.push_back(std::move(rs_reader));
}
diff --git a/be/src/olap/lru_cache.cpp b/be/src/olap/lru_cache.cpp
index 81da1b8..0abf428 100644
--- a/be/src/olap/lru_cache.cpp
+++ b/be/src/olap/lru_cache.cpp
@@ -400,7 +400,7 @@ uint32_t ShardedLRUCache::_shard(uint32_t hash) {
ShardedLRUCache::ShardedLRUCache(const std::string& name, size_t
total_capacity, std::shared_ptr<MemTracker> parent)
: _name(name), _last_id(1),
- _mem_tracker(MemTracker::CreateTracker(-1, name, parent, true)) {
+ _mem_tracker(MemTracker::CreateTracker(-1, name, parent, true, false))
{
const size_t per_shard = (total_capacity + (kNumShards - 1)) / kNumShards;
for (int s = 0; s < kNumShards; s++) {
_shards[s].set_capacity(per_shard);
diff --git a/be/src/olap/memtable.cpp b/be/src/olap/memtable.cpp
index 4ee8b4a..f728af5 100644
--- a/be/src/olap/memtable.cpp
+++ b/be/src/olap/memtable.cpp
@@ -40,7 +40,7 @@ MemTable::MemTable(int64_t tablet_id, Schema* schema, const
TabletSchema* tablet
_slot_descs(slot_descs),
_keys_type(keys_type),
_row_comparator(_schema),
- _mem_tracker(MemTracker::CreateTracker(-1, "MemTable:" +
std::to_string(tablet_id), parent_tracker)),
+ _mem_tracker(MemTracker::CreateTracker(-1, "MemTable",
parent_tracker)),
_buffer_mem_pool(new MemPool(_mem_tracker.get())),
_table_mem_pool(new MemPool(_mem_tracker.get())),
_schema_size(_schema->schema_size()),
diff --git a/be/src/olap/row_block2.cpp b/be/src/olap/row_block2.cpp
index 2b940bd..94d600d 100644
--- a/be/src/olap/row_block2.cpp
+++ b/be/src/olap/row_block2.cpp
@@ -18,6 +18,7 @@
#include "olap/row_block2.h"
#include <sstream>
+#include <utility>
#include "gutil/strings/substitute.h"
#include "olap/row_cursor.h"
@@ -33,7 +34,7 @@ RowBlockV2::RowBlockV2(const Schema& schema, uint16_t
capacity, std::shared_ptr<
: _schema(schema),
_capacity(capacity),
_column_vector_batches(_schema.num_columns()),
- _tracker(MemTracker::CreateTracker(-1, "RowBlockV2", parent)),
+ _tracker(MemTracker::CreateTracker(-1, "RowBlockV2",
std::move(parent))),
_pool(new MemPool(_tracker.get())),
_selection_vector(nullptr) {
for (auto cid : _schema.column_ids()) {
diff --git a/be/src/olap/rowset/beta_rowset_reader.cpp
b/be/src/olap/rowset/beta_rowset_reader.cpp
index 0f5899c..7a3c49b 100644
--- a/be/src/olap/rowset/beta_rowset_reader.cpp
+++ b/be/src/olap/rowset/beta_rowset_reader.cpp
@@ -17,6 +17,8 @@
#include "beta_rowset_reader.h"
+#include <utility>
+
#include "olap/delete_handler.h"
#include "olap/generic_iterators.h"
#include "olap/row_block.h"
@@ -28,12 +30,17 @@
namespace doris {
BetaRowsetReader::BetaRowsetReader(BetaRowsetSharedPtr rowset,
- const std::shared_ptr<MemTracker>&
parent_tracker)
- : _rowset(std::move(rowset)), _stats(&_owned_stats),
_parent_tracker(parent_tracker) {
+ std::shared_ptr<MemTracker> parent_tracker)
+ : _rowset(std::move(rowset)), _stats(&_owned_stats),
_parent_tracker(std::move(parent_tracker)) {
_rowset->aquire();
}
OLAPStatus BetaRowsetReader::init(RowsetReaderContext* read_context) {
+ // If do not init the RowsetReader with a parent_tracker, use the
runtime_state instance_mem_tracker
+ if (_parent_tracker == nullptr && read_context->runtime_state != nullptr) {
+ _parent_tracker = read_context->runtime_state->instance_mem_tracker();
+ }
+
RETURN_NOT_OK(_rowset->load(true, _parent_tracker));
_context = read_context;
if (_context->stats != nullptr) {
@@ -110,13 +117,9 @@ OLAPStatus BetaRowsetReader::init(RowsetReaderContext*
read_context) {
// init input block
_input_block.reset(new RowBlockV2(schema, 1024, _parent_tracker));
- // init output block and row
- if (_parent_tracker == nullptr && read_context->runtime_state != nullptr) {
- _output_block.reset(new RowBlock(read_context->tablet_schema,
-
read_context->runtime_state->instance_mem_tracker()));
- } else {
- _output_block.reset(new RowBlock(read_context->tablet_schema,
_parent_tracker));
- }
+ // init input/output block and row
+ _output_block.reset(new RowBlock(read_context->tablet_schema,
_parent_tracker));
+
RowBlockInfo output_block_info;
output_block_info.row_num = 1024;
output_block_info.null_supported = true;
diff --git a/be/src/olap/rowset/beta_rowset_reader.h
b/be/src/olap/rowset/beta_rowset_reader.h
index 1318d89..6d8a3f1 100644
--- a/be/src/olap/rowset/beta_rowset_reader.h
+++ b/be/src/olap/rowset/beta_rowset_reader.h
@@ -30,7 +30,7 @@ namespace doris {
class BetaRowsetReader : public RowsetReader {
public:
BetaRowsetReader(BetaRowsetSharedPtr rowset,
- const std::shared_ptr<MemTracker>& parent_tracker =
nullptr);
+ std::shared_ptr<MemTracker> parent_tracker = nullptr);
~BetaRowsetReader() override { _rowset->release(); }
diff --git a/be/src/olap/rowset/segment_v2/segment.cpp
b/be/src/olap/rowset/segment_v2/segment.cpp
index f8e6024..992d8c4 100644
--- a/be/src/olap/rowset/segment_v2/segment.cpp
+++ b/be/src/olap/rowset/segment_v2/segment.cpp
@@ -46,7 +46,7 @@ Status Segment::open(std::string filename, uint32_t
segment_id, const TabletSche
Segment::Segment(std::string fname, uint32_t segment_id, const TabletSchema*
tablet_schema, std::shared_ptr<MemTracker> parent)
: _fname(std::move(fname)), _segment_id(segment_id),
- _tablet_schema(tablet_schema),
_mem_tracker(MemTracker::CreateTracker(-1, "Segment", std::move(parent),
false)) {}
+ _tablet_schema(tablet_schema),
_mem_tracker(MemTracker::CreateTracker(-1, "Segment", std::move(parent), false,
true)) {}
Segment::~Segment() {
_mem_tracker->Release(_mem_tracker->consumption());
diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
index 67c5adf..859cca5 100644
--- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
@@ -189,7 +189,7 @@ Status SegmentIterator::_prepare_seek(const
StorageReadOptions::KeyRange& key_ra
}
}
_seek_schema.reset(new Schema(key_fields, key_fields.size()));
- _seek_block.reset(new RowBlockV2(*_seek_schema, 1));
+ _seek_block.reset(new RowBlockV2(*_seek_schema, 1, _mem_tracker));
// create used column iterator
for (auto cid : _seek_schema->column_ids()) {
diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp
index e738897..7d69f60 100644
--- a/be/src/olap/schema_change.cpp
+++ b/be/src/olap/schema_change.cpp
@@ -1338,7 +1338,7 @@ bool
SchemaChangeWithSorting::_external_sorting(vector<RowsetSharedPtr>& src_row
std::vector<RowsetReaderSharedPtr> rs_readers;
for (auto& rowset : src_rowsets) {
RowsetReaderSharedPtr rs_reader;
- auto res = rowset->create_reader(&rs_reader);
+ auto res = rowset->create_reader(_mem_tracker, &rs_reader);
if (res != OLAP_SUCCESS) {
LOG(WARNING) << "failed to create rowset reader.";
return false;
@@ -1473,6 +1473,9 @@ OLAPStatus
SchemaChangeHandler::_do_process_alter_tablet_v2(const TAlterTabletRe
// for schema change, seek_columns is the same to return_columns
reader_context.seek_columns = &return_columns;
+ auto mem_tracker = MemTracker::CreateTracker(-1, "AlterTablet:" +
std::to_string(base_tablet->tablet_id()) + "-"
+ + std::to_string(new_tablet->tablet_id()), _mem_tracker, true, false,
MemTrackerLevel::DEBUG);
+
do {
// get history data to be converted and it will check if there is hold
in base tablet
res = _get_versions_to_be_changed(base_tablet,
&versions_to_be_changed);
@@ -1535,7 +1538,7 @@ OLAPStatus
SchemaChangeHandler::_do_process_alter_tablet_v2(const TAlterTabletRe
}
// acquire data sources correspond to history versions
- base_tablet->capture_rs_readers(versions_to_be_changed, &rs_readers);
+ base_tablet->capture_rs_readers(versions_to_be_changed, &rs_readers,
mem_tracker);
if (rs_readers.size() < 1) {
LOG(WARNING) << "fail to acquire all data sources. "
<< "version_num=" << versions_to_be_changed.size()
@@ -1688,7 +1691,7 @@ OLAPStatus
SchemaChangeHandler::schema_version_convert(TabletSharedPtr base_tabl
reader_context.seek_columns = &return_columns;
RowsetReaderSharedPtr rowset_reader;
- RETURN_NOT_OK((*base_rowset)->create_reader(&rowset_reader));
+ RETURN_NOT_OK((*base_rowset)->create_reader(_mem_tracker, &rowset_reader));
rowset_reader->init(&reader_context);
RowsetWriterContext writer_context;
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index 7866f0f..7921d04 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -628,15 +628,17 @@ OLAPStatus Tablet::_capture_consistent_rowsets_unlocked(
}
OLAPStatus Tablet::capture_rs_readers(const Version& spec_version,
- std::vector<RowsetReaderSharedPtr>*
rs_readers) const {
+ std::vector<RowsetReaderSharedPtr>*
rs_readers,
+ std::shared_ptr<MemTracker>
parent_tracker) const {
std::vector<Version> version_path;
RETURN_NOT_OK(capture_consistent_versions(spec_version, &version_path));
- RETURN_NOT_OK(capture_rs_readers(version_path, rs_readers));
+ RETURN_NOT_OK(capture_rs_readers(version_path, rs_readers,
parent_tracker));
return OLAP_SUCCESS;
}
OLAPStatus Tablet::capture_rs_readers(const std::vector<Version>& version_path,
- std::vector<RowsetReaderSharedPtr>*
rs_readers) const {
+ std::vector<RowsetReaderSharedPtr>*
rs_readers,
+ std::shared_ptr<MemTracker>
parent_tracker) const {
DCHECK(rs_readers != nullptr && rs_readers->empty());
for (auto version : version_path) {
auto it = _rs_version_map.find(version);
@@ -653,7 +655,7 @@ OLAPStatus Tablet::capture_rs_readers(const
std::vector<Version>& version_path,
}
}
RowsetReaderSharedPtr rs_reader;
- auto res = it->second->create_reader(&rs_reader);
+ auto res = it->second->create_reader(parent_tracker, &rs_reader);
if (res != OLAP_SUCCESS) {
LOG(WARNING) << "failed to create reader for rowset:" <<
it->second->rowset_id();
return OLAP_ERR_CAPTURE_ROWSET_READER_ERROR;
diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h
index 1102fb5..7da632e 100644
--- a/be/src/olap/tablet.h
+++ b/be/src/olap/tablet.h
@@ -121,9 +121,12 @@ public:
OLAPStatus capture_consistent_rowsets(const Version& spec_version,
vector<RowsetSharedPtr>* rowsets)
const;
OLAPStatus capture_rs_readers(const Version& spec_version,
- vector<RowsetReaderSharedPtr>* rs_readers)
const;
+ vector<RowsetReaderSharedPtr>* rs_readers,
+ std::shared_ptr<MemTracker> parent_tracker =
nullptr) const;
+
OLAPStatus capture_rs_readers(const vector<Version>& version_path,
- vector<RowsetReaderSharedPtr>* rs_readers)
const;
+ vector<RowsetReaderSharedPtr>* rs_readers,
+ std::shared_ptr<MemTracker> parent_tracker =
nullptr) const;
DelPredicateArray delete_predicates() { return
_tablet_meta->delete_predicates(); }
void add_delete_predicate(const DeletePredicatePB& delete_predicate,
int64_t version);
diff --git a/be/src/runtime/data_stream_recvr.cc
b/be/src/runtime/data_stream_recvr.cc
index fd3a536..cba524e 100644
--- a/be/src/runtime/data_stream_recvr.cc
+++ b/be/src/runtime/data_stream_recvr.cc
@@ -447,7 +447,7 @@ DataStreamRecvr::DataStreamRecvr(
_profile(profile),
_sub_plan_query_statistics_recvr(sub_plan_query_statistics_recvr) {
_mem_tracker = MemTracker::CreateTracker(
- _profile, -1, "DataStreamRecvr:" +
print_id(_fragment_instance_id), parent_tracker);
+ _profile, -1, "DataStreamRecvr", parent_tracker);
// Create one queue per sender if is_merging is true.
int num_queues = is_merging ? num_senders : 1;
diff --git a/be/src/runtime/data_stream_sender.cpp
b/be/src/runtime/data_stream_sender.cpp
index 343baf5..a8a500b 100644
--- a/be/src/runtime/data_stream_sender.cpp
+++ b/be/src/runtime/data_stream_sender.cpp
@@ -469,8 +469,7 @@ Status DataStreamSender::prepare(RuntimeState* state) {
_profile = _pool->add(new RuntimeProfile(title.str()));
SCOPED_TIMER(_profile->total_time_counter());
_mem_tracker = MemTracker::CreateTracker(
- _profile, -1, "DataStreamSender:" +
print_id(state->fragment_instance_id()),
- state->instance_mem_tracker());
+ _profile, -1, "DataStreamSender", state->instance_mem_tracker());
if (_part_type == TPartitionType::UNPARTITIONED || _part_type ==
TPartitionType::RANDOM) {
// Randomize the order we open/transmit to channels to avoid
thundering herd problems.
diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp
index 39b9a2a..7bd31a3 100644
--- a/be/src/runtime/exec_env_init.cpp
+++ b/be/src/runtime/exec_env_init.cpp
@@ -185,8 +185,8 @@ Status ExecEnv::_init_mem_tracker() {
return Status::InternalError(ss.str());
}
- _mem_tracker =
- MemTracker::CreateTracker(bytes_limit, "Query",
MemTracker::GetRootTracker());
+ _mem_tracker = MemTracker::CreateTracker(bytes_limit, "Query",
MemTracker::GetRootTracker(),
+ false, false);
LOG(INFO) << "Using global memory limit: " <<
PrettyPrinter::print(bytes_limit, TUnit::BYTES);
RETURN_IF_ERROR(_disk_io_mgr->init(_mem_tracker));
diff --git a/be/src/runtime/export_sink.cpp b/be/src/runtime/export_sink.cpp
index 0e88bb4..5e4d2cc 100644
--- a/be/src/runtime/export_sink.cpp
+++ b/be/src/runtime/export_sink.cpp
@@ -72,8 +72,7 @@ Status ExportSink::prepare(RuntimeState* state) {
_mem_tracker = MemTracker::CreateTracker(
-1,
- "ExportSink:" + print_id(state->fragment_instance_id()),
- state->instance_mem_tracker());
+ "ExportSink", state->instance_mem_tracker());
// Prepare the exprs to run.
RETURN_IF_ERROR(Expr::prepare(_output_expr_ctxs, state, _row_desc,
_mem_tracker));
diff --git a/be/src/runtime/load_channel.cpp b/be/src/runtime/load_channel.cpp
index 30c2f2f..d6ade09 100644
--- a/be/src/runtime/load_channel.cpp
+++ b/be/src/runtime/load_channel.cpp
@@ -27,7 +27,7 @@ LoadChannel::LoadChannel(const UniqueId& load_id, int64_t
mem_limit, int64_t tim
const std::shared_ptr<MemTracker>& mem_tracker)
: _load_id(load_id), _timeout_s(timeout_s) {
_mem_tracker = MemTracker::CreateTracker(
- mem_limit, "LoadChannel:" + _load_id.to_string(), mem_tracker);
+ mem_limit, "LoadChannel:" + _load_id.to_string(), mem_tracker,
true, false);
// _last_updated_time should be set before being inserted to
// _load_channels in load_channel_mgr, or it may be erased
// immediately by gc thread.
diff --git a/be/src/runtime/mem_tracker.cpp b/be/src/runtime/mem_tracker.cpp
index 2c5fac4..bdb8030 100644
--- a/be/src/runtime/mem_tracker.cpp
+++ b/be/src/runtime/mem_tracker.cpp
@@ -17,9 +17,11 @@
#include "runtime/mem_tracker.h"
-#include <stdint.h>
+#include <cstdint>
#include <boost/algorithm/string/join.hpp>
+#include <boost/algorithm/string.hpp>
+
#include <limits>
#include <memory>
@@ -75,15 +77,32 @@ void MemTracker::CreateRootTracker() {
}
std::shared_ptr<MemTracker> MemTracker::CreateTracker(RuntimeProfile* profile,
int64_t byte_limit,
- const std::string& label,
- const
std::shared_ptr<MemTracker>& parent) {
- shared_ptr<MemTracker> real_parent;
+ const std::string&
label, const std::shared_ptr<MemTracker>& parent,
+ bool reset_label_name,
MemTrackerLevel level) {
+ std::shared_ptr<MemTracker> real_parent;
+ std::string label_name;
+ // if parent is not null, reset label name to query id.
+ // The parent label always:
RuntimeState:instance:8ca5a59e3aa84f74-84bb0d0466193736
+ // we just need the last id of it: 8ca5a59e3aa84f74-84bb0d0466193736
+ // to build the new label name of tracker: `label`:
8ca5a59e3aa84f74-84bb0d0466193736
+ // else if parent is null
+ // just use the root is parent and keep the label_name as label
if (parent) {
- real_parent = std::move(parent);
+ real_parent = parent;
+ if (reset_label_name) {
+ std::vector<string> tmp_result;
+ boost::split(tmp_result, parent->label(), boost::is_any_of(":"));
+ label_name = label + ":" + tmp_result[tmp_result.size() - 1];
+ } else {
+ label_name = label;
+ }
} else {
real_parent = GetRootTracker();
+ label_name = label;
}
- shared_ptr<MemTracker> tracker(new MemTracker(profile, byte_limit, label,
real_parent, true));
+
+ shared_ptr<MemTracker> tracker(new MemTracker(profile, byte_limit,
label_name, real_parent, true,
+ level > real_parent->_level ? level : real_parent->_level));
real_parent->AddChildTracker(tracker);
tracker->Init();
@@ -91,16 +110,32 @@ std::shared_ptr<MemTracker>
MemTracker::CreateTracker(RuntimeProfile* profile, i
}
std::shared_ptr<MemTracker> MemTracker::CreateTracker(int64_t byte_limit,
const std::string& label,
- std::shared_ptr<MemTracker> parent,
- bool log_usage_if_zero) {
- shared_ptr<MemTracker> real_parent;
+ std::shared_ptr<MemTracker> parent, bool log_usage_if_zero, bool
reset_label_name, MemTrackerLevel level) {
+ std::shared_ptr<MemTracker> real_parent;
+ std::string label_name;
+ // if parent is not null, reset label name to query id.
+ // The parent label always:
RuntimeState:instance:8ca5a59e3aa84f74-84bb0d0466193736
+ // we just need the last id of it: 8ca5a59e3aa84f74-84bb0d0466193736
+ // to build the new label name of tracker: `label`:
8ca5a59e3aa84f74-84bb0d0466193736
+ // else if parent is null
+ // just use the root is parent and keep the label_name as label
if (parent) {
- real_parent = std::move(parent);
+ real_parent = parent;
+ if (reset_label_name) {
+ std::vector<string> tmp_result;
+ boost::split(tmp_result, parent->label(), boost::is_any_of(":"));
+ label_name = label + ":" + tmp_result[tmp_result.size() - 1];
+ } else {
+ label_name = label;
+ }
} else {
real_parent = GetRootTracker();
+ label_name = label;
}
+
shared_ptr<MemTracker> tracker(
- new MemTracker(nullptr, byte_limit, label, real_parent,
log_usage_if_zero));
+ new MemTracker(nullptr, byte_limit, label_name, real_parent,
log_usage_if_zero,
+ level > real_parent->_level ? level :
real_parent->_level));
real_parent->AddChildTracker(tracker);
tracker->Init();
@@ -108,16 +143,17 @@ std::shared_ptr<MemTracker>
MemTracker::CreateTracker(int64_t byte_limit, const
}
MemTracker::MemTracker(int64_t byte_limit, const std::string& label)
- : MemTracker(nullptr, byte_limit, label,
std::shared_ptr<MemTracker>(), true) {}
+ : MemTracker(nullptr, byte_limit, label,
std::shared_ptr<MemTracker>(), true, MemTrackerLevel::RELEASE) {}
MemTracker::MemTracker(RuntimeProfile* profile, int64_t byte_limit, const
string& label,
- const std::shared_ptr<MemTracker>& parent, bool
log_usage_if_zero)
+ const std::shared_ptr<MemTracker>& parent, bool
log_usage_if_zero, MemTrackerLevel level)
: limit_(byte_limit),
soft_limit_(CalcSoftLimit(byte_limit)),
label_(label),
parent_(parent),
consumption_metric_(nullptr),
log_usage_if_zero_(log_usage_if_zero),
+ _level(level),
num_gcs_metric_(nullptr),
bytes_freed_by_last_gc_metric_(nullptr),
bytes_over_limit_metric_(nullptr),
@@ -259,7 +295,7 @@ void
MemTracker::ListTrackers(vector<shared_ptr<MemTracker>>* trackers) {
}
for (const auto& child_weak : children) {
shared_ptr<MemTracker> child = child_weak.lock();
- if (child) {
+ if (child &&
static_cast<decltype(config::mem_tracker_level)>(child->_level) <=
config::mem_tracker_level) {
to_process.emplace_back(std::move(child));
}
}
diff --git a/be/src/runtime/mem_tracker.h b/be/src/runtime/mem_tracker.h
index 94f40fa..142e611 100644
--- a/be/src/runtime/mem_tracker.h
+++ b/be/src/runtime/mem_tracker.h
@@ -40,6 +40,10 @@ namespace doris {
/// limit should be used.
enum class MemLimit { HARD, SOFT };
+/// The Level use to decide whether to show it in web page
+/// each MemTracker have a Level equals to parent, only be set explicit
+enum class MemTrackerLevel {RELEASE = 0, DEBUG};
+
class ObjectPool;
class MemTracker;
struct ReservationTrackerCounters;
@@ -88,11 +92,11 @@ public:
static std::shared_ptr<MemTracker> CreateTracker(
int64_t byte_limit = -1, const std::string& label = std::string(),
std::shared_ptr<MemTracker> parent = std::shared_ptr<MemTracker>(),
- bool log_usage_if_zero = true);
+ bool log_usage_if_zero = true, bool reset_label_name = true,
MemTrackerLevel level = MemTrackerLevel::RELEASE);
static std::shared_ptr<MemTracker> CreateTracker(
RuntimeProfile* profile, int64_t byte_limit, const std::string&
label = std::string(),
- const std::shared_ptr<MemTracker>& parent =
std::shared_ptr<MemTracker>());
+ const std::shared_ptr<MemTracker>& parent =
std::shared_ptr<MemTracker>(), bool reset_label_name = true, MemTrackerLevel
level = MemTrackerLevel::RELEASE);
// this is used for creating an orphan mem tracker, or for unit test.
// If a mem tracker has parent, it should be created by `CreateTracker()`
@@ -420,7 +424,7 @@ private:
/// If 'log_usage_if_zero' is false, this tracker (and its children) will
not be
/// included in LogUsage() output if consumption is 0.
MemTracker(RuntimeProfile* profile, int64_t byte_limit, const std::string&
label,
- const std::shared_ptr<MemTracker>& parent, bool
log_usage_if_zero);
+ const std::shared_ptr<MemTracker>& parent, bool
log_usage_if_zero, MemTrackerLevel);
private:
friend class PoolMemTrackerRegistry;
@@ -553,6 +557,8 @@ private:
/// if consumption is 0.
bool log_usage_if_zero_;
+ MemTrackerLevel _level;
+
/// The number of times the GcFunctions were called.
IntCounter* num_gcs_metric_;
diff --git a/be/src/runtime/plan_fragment_executor.cpp
b/be/src/runtime/plan_fragment_executor.cpp
index 815d0d1..3bd8a1b 100644
--- a/be/src/runtime/plan_fragment_executor.cpp
+++ b/be/src/runtime/plan_fragment_executor.cpp
@@ -132,7 +132,7 @@ Status PlanFragmentExecutor::prepare(const
TExecPlanFragmentParams& request,
_mem_tracker = MemTracker::CreateTracker(bytes_limit,
"PlanFragmentExecutor:" +
print_id(_query_id) + ":" +
print_id(params.fragment_instance_id),
- _exec_env->process_mem_tracker());
+ _exec_env->process_mem_tracker(),
true, false);
_runtime_state->set_fragment_mem_tracker(_mem_tracker);
LOG(INFO) << "Using query memory limit: " <<
PrettyPrinter::print(bytes_limit, TUnit::BYTES);
diff --git a/be/src/runtime/runtime_state.cpp b/be/src/runtime/runtime_state.cpp
index b8c0c26..5237646 100644
--- a/be/src/runtime/runtime_state.cpp
+++ b/be/src/runtime/runtime_state.cpp
@@ -215,9 +215,9 @@ Status RuntimeState::init_mem_trackers(const TUniqueId&
query_id) {
_query_mem_tracker =
MemTracker::CreateTracker(bytes_limit, "RuntimeState:query:" +
print_id(query_id),
- _exec_env->process_mem_tracker());
+ _exec_env->process_mem_tracker(), true,
false);
_instance_mem_tracker = MemTracker::CreateTracker(
- &_profile, -1, "RuntimeState:instance:" + print_id(query_id),
_query_mem_tracker);
+ &_profile, -1, "RuntimeState:instance:", _query_mem_tracker);
/*
// TODO: this is a stopgap until we implement ExprContext
diff --git a/be/src/runtime/tablets_channel.cpp
b/be/src/runtime/tablets_channel.cpp
index 5e647bb..531a3bd 100644
--- a/be/src/runtime/tablets_channel.cpp
+++ b/be/src/runtime/tablets_channel.cpp
@@ -34,7 +34,7 @@ std::atomic<uint64_t> TabletsChannel::_s_tablet_writer_count;
TabletsChannel::TabletsChannel(const TabletsChannelKey& key,
const std::shared_ptr<MemTracker>& mem_tracker)
: _key(key), _state(kInitialized), _closed_senders(64) {
- _mem_tracker = MemTracker::CreateTracker(-1, "TabletsChannel:" +
key.to_string(), mem_tracker);
+ _mem_tracker = MemTracker::CreateTracker(-1, "TabletsChannel",
mem_tracker);
static std::once_flag once_flag;
std::call_once(once_flag, [] {
REGISTER_HOOK_METRIC(tablet_writer_count, [&]() { return
_s_tablet_writer_count.load(); });
diff --git a/docs/en/administrator-guide/config/be_config.md
b/docs/en/administrator-guide/config/be_config.md
index 8dd31c1..38ea221 100644
--- a/docs/en/administrator-guide/config/be_config.md
+++ b/docs/en/administrator-guide/config/be_config.md
@@ -874,6 +874,7 @@ If the parameter is `THREAD_POOL`, the model is a blocking
I/O model.
### `aws_log_level`
* Type: int32
+
* Description: log level of AWS SDK,
```
Off = 0,
@@ -884,4 +885,15 @@ If the parameter is `THREAD_POOL`, the model is a blocking
I/O model.
Debug = 5,
Trace = 6
```
-* Default: 3
\ No newline at end of file
+
+* Default: 3
+
+### `mem_tracker_level`
+
+* Type: int16
+* Description: The level at which MemTracker is displayed on the Web page
equal or lower than this level will be displayed on the Web page
+ ```
+ RELEASE = 0
+ DEBUG = 1
+ ```
+* Default: 0
\ No newline at end of file
diff --git a/docs/zh-CN/administrator-guide/config/be_config.md
b/docs/zh-CN/administrator-guide/config/be_config.md
index 8f97567..2b1f6ff 100644
--- a/docs/zh-CN/administrator-guide/config/be_config.md
+++ b/docs/zh-CN/administrator-guide/config/be_config.md
@@ -886,3 +886,13 @@ Stream Load 一般适用于导入几个GB以内的数据，不适合导入过大
```
* 默认值: 3
+
+### `mem_tracker_level`
+
+* 类型: int16
+* 描述: MemTracker在Web页面上展示的级别，等于或低于这个级别的MemTracker会在Web页面上展示
+ ```
+ RELEASE = 0
+ DEBUG = 1
+ ```
+* 默认值: 0
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
