This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 8726bfa121 [enhancement](memory) Add tablet schema cache metrics
(#14742)
8726bfa121 is described below
commit 8726bfa121bd138453d4ff7285c338e182390f4f
Author: Xinyi Zou <[email protected]>
AuthorDate: Mon Dec 5 18:19:13 2022 +0800
[enhancement](memory) Add tablet schema cache metrics (#14742)
---
be/src/olap/tablet_schema.cpp | 26 +-------------------------
be/src/olap/tablet_schema.h | 4 ++--
be/src/olap/tablet_schema_cache.h | 8 ++++++++
be/src/runtime/memory/mem_tracker_limiter.cpp | 1 +
be/src/util/doris_metrics.cpp | 6 ++++++
be/src/util/doris_metrics.h | 3 +++
6 files changed, 21 insertions(+), 27 deletions(-)
diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp
index d9c46888c1..d684806454 100644
--- a/be/src/olap/tablet_schema.cpp
+++ b/be/src/olap/tablet_schema.cpp
@@ -441,18 +441,6 @@ void TabletColumn::to_schema_pb(ColumnPB* column) const {
}
}
-uint32_t TabletColumn::mem_size() const {
- auto size = sizeof(TabletColumn);
- size += _col_name.size();
- if (_has_default_value) {
- size += _default_value.size();
- }
- for (auto& sub_column : _sub_columns) {
- size += sub_column.mem_size();
- }
- return size;
-}
-
void TabletColumn::add_sub_column(TabletColumn& sub_column) {
_sub_columns.push_back(sub_column);
sub_column._parent = this;
@@ -557,6 +545,7 @@ void TabletSchema::clear_columns() {
}
void TabletSchema::init_from_pb(const TabletSchemaPB& schema) {
+ SCOPED_MEM_COUNT(&_mem_size);
_keys_type = schema.keys_type();
_num_columns = 0;
_num_key_columns = 0;
@@ -729,19 +718,6 @@ void TabletSchema::to_schema_pb(TabletSchemaPB*
tablet_schema_pb) const {
tablet_schema_pb->set_compression_type(_compression_type);
}
-uint32_t TabletSchema::mem_size() const {
- auto size = sizeof(TabletSchema);
- for (auto& col : _cols) {
- size += col.mem_size();
- }
-
- for (auto& pair : _field_name_to_index) {
- size += pair.first.size();
- size += sizeof(pair.second);
- }
- return size;
-}
-
size_t TabletSchema::row_size() const {
size_t size = 0;
for (auto& column : _cols) {
diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h
index 6702901e83..20b05d1801 100644
--- a/be/src/olap/tablet_schema.h
+++ b/be/src/olap/tablet_schema.h
@@ -45,7 +45,6 @@ public:
void init_from_pb(const ColumnPB& column);
void init_from_thrift(const TColumn& column);
void to_schema_pb(ColumnPB* column) const;
- uint32_t mem_size() const;
int32_t unique_id() const { return _unique_id; }
std::string name() const { return _col_name; }
@@ -150,7 +149,7 @@ public:
void append_column(TabletColumn column, bool is_dropped_column = false);
void copy_from(const TabletSchema& tablet_schema);
std::string to_key() const;
- uint32_t mem_size() const;
+ int64_t mem_size() const { return _mem_size; };
size_t row_size() const;
int32_t field_index(const std::string& field_name) const;
@@ -243,6 +242,7 @@ private:
int32_t _sequence_col_idx = -1;
int32_t _schema_version = -1;
bool _disable_auto_compaction = false;
+ int64_t _mem_size = 0;
};
bool operator==(const TabletSchema& a, const TabletSchema& b);
diff --git a/be/src/olap/tablet_schema_cache.h
b/be/src/olap/tablet_schema_cache.h
index 4dbdd93c6a..47bfcb16f8 100644
--- a/be/src/olap/tablet_schema_cache.h
+++ b/be/src/olap/tablet_schema_cache.h
@@ -24,6 +24,7 @@
#include <unordered_map>
#include "olap/tablet_schema.h"
+#include "util/doris_metrics.h"
namespace doris {
@@ -49,6 +50,9 @@ public:
pb.ParseFromString(key);
tablet_schema_ptr->init_from_pb(pb);
_cache[key] = tablet_schema_ptr;
+ DorisMetrics::instance()->tablet_schema_cache_count->increment(1);
+
DorisMetrics::instance()->tablet_schema_cache_memory_bytes->increment(
+ tablet_schema_ptr->mem_size());
return tablet_schema_ptr;
}
return iter->second;
@@ -63,8 +67,12 @@ private:
for (;;) {
std::this_thread::sleep_for(std::chrono::seconds(tablet_schema_cache_recycle_interval));
std::lock_guard guard(_mtx);
+ LOG(INFO) << "Tablet Schema Cache Capacity " << _cache.size();
for (auto iter = _cache.begin(), last = _cache.end(); iter !=
last;) {
if (iter->second.unique()) {
+
DorisMetrics::instance()->tablet_schema_cache_memory_bytes->increment(
+ -iter->second->mem_size());
+
DorisMetrics::instance()->tablet_schema_cache_count->increment(-1);
iter = _cache.erase(iter);
} else {
++iter;
diff --git a/be/src/runtime/memory/mem_tracker_limiter.cpp
b/be/src/runtime/memory/mem_tracker_limiter.cpp
index 6bac6a8203..16f74267a3 100644
--- a/be/src/runtime/memory/mem_tracker_limiter.cpp
+++ b/be/src/runtime/memory/mem_tracker_limiter.cpp
@@ -68,6 +68,7 @@ MemTrackerLimiter::MemTrackerLimiter(Type type, const
std::string& label, int64_
}
MemTrackerLimiter::~MemTrackerLimiter() {
+ if (_type == Type::GLOBAL) return;
consume(_untracked_mem);
// mem hook record tracker cannot guarantee that the final consumption is
0,
// nor can it guarantee that the memory alloc and free are recorded in a
one-to-one correspondence.
diff --git a/be/src/util/doris_metrics.cpp b/be/src/util/doris_metrics.cpp
index 15be14b06e..51c52d4ab1 100644
--- a/be/src/util/doris_metrics.cpp
+++ b/be/src/util/doris_metrics.cpp
@@ -161,6 +161,9 @@
DEFINE_GAUGE_CORE_METRIC_PROTOTYPE_2ARG(query_cache_memory_total_byte, MetricUni
DEFINE_GAUGE_CORE_METRIC_PROTOTYPE_2ARG(query_cache_sql_total_count,
MetricUnit::NOUNIT);
DEFINE_GAUGE_CORE_METRIC_PROTOTYPE_2ARG(query_cache_partition_total_count,
MetricUnit::NOUNIT);
+DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(tablet_schema_cache_count,
MetricUnit::NOUNIT);
+DEFINE_GAUGE_CORE_METRIC_PROTOTYPE_2ARG(tablet_schema_cache_memory_bytes,
MetricUnit::BYTES);
+
DEFINE_GAUGE_CORE_METRIC_PROTOTYPE_2ARG(upload_total_byte, MetricUnit::BYTES);
DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(upload_rowset_count, MetricUnit::ROWSETS);
DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(upload_fail_count, MetricUnit::ROWSETS);
@@ -290,6 +293,9 @@ DorisMetrics::DorisMetrics() :
_metric_registry(_s_registry_name) {
INT_UGAUGE_METRIC_REGISTER(_server_metric_entity,
query_cache_sql_total_count);
INT_UGAUGE_METRIC_REGISTER(_server_metric_entity,
query_cache_partition_total_count);
+ INT_COUNTER_METRIC_REGISTER(_server_metric_entity,
tablet_schema_cache_count);
+ INT_UGAUGE_METRIC_REGISTER(_server_metric_entity,
tablet_schema_cache_memory_bytes);
+
INT_COUNTER_METRIC_REGISTER(_server_metric_entity,
local_file_reader_total);
INT_COUNTER_METRIC_REGISTER(_server_metric_entity, s3_file_reader_total);
INT_COUNTER_METRIC_REGISTER(_server_metric_entity,
local_file_writer_total);
diff --git a/be/src/util/doris_metrics.h b/be/src/util/doris_metrics.h
index 7536be7372..a14922f6e1 100644
--- a/be/src/util/doris_metrics.h
+++ b/be/src/util/doris_metrics.h
@@ -199,6 +199,9 @@ public:
UIntGauge* query_cache_sql_total_count;
UIntGauge* query_cache_partition_total_count;
+ IntCounter* tablet_schema_cache_count;
+ UIntGauge* tablet_schema_cache_memory_bytes;
+
UIntGauge* scanner_thread_pool_queue_size;
UIntGauge* add_batch_task_queue_size;
UIntGauge* send_batch_thread_pool_thread_num;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
