This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new ab5ee818110 [fix](memory) Fix page cache memory tracker consumption in
prune (#34320)
ab5ee818110 is described below
commit ab5ee8181100d8cbee381b2697c5b6874f86b0d1
Author: Xinyi Zou <[email protected]>
AuthorDate: Mon May 6 12:48:23 2024 +0800
[fix](memory) Fix page cache memory tracker consumption in prune (#34320)
---
be/src/olap/page_cache.h | 6 +++---
be/src/runtime/memory/cache_policy.h | 31 ++++++++++++++++++++++++-------
be/src/runtime/memory/lru_cache_policy.h | 12 ------------
3 files changed, 27 insertions(+), 22 deletions(-)
diff --git a/be/src/olap/page_cache.h b/be/src/olap/page_cache.h
index 9799b06b1a6..e1bb8be8d6f 100644
--- a/be/src/olap/page_cache.h
+++ b/be/src/olap/page_cache.h
@@ -111,7 +111,7 @@ public:
: LRUCachePolicy(CachePolicy::CacheType::DATA_PAGE_CACHE,
capacity,
LRUCacheType::SIZE,
config::data_page_cache_stale_sweep_time_sec,
num_shards) {
- init_mem_tracker_by_allocator();
+
init_mem_tracker_by_allocator(lru_cache_type_string(LRUCacheType::SIZE));
}
};
@@ -121,7 +121,7 @@ public:
: LRUCachePolicy(CachePolicy::CacheType::INDEXPAGE_CACHE,
capacity,
LRUCacheType::SIZE,
config::index_page_cache_stale_sweep_time_sec,
num_shards) {
- init_mem_tracker_by_allocator();
+
init_mem_tracker_by_allocator(lru_cache_type_string(LRUCacheType::SIZE));
}
};
@@ -131,7 +131,7 @@ public:
: LRUCachePolicy(CachePolicy::CacheType::PK_INDEX_PAGE_CACHE,
capacity,
LRUCacheType::SIZE,
config::pk_index_page_cache_stale_sweep_time_sec, num_shards) {
- init_mem_tracker_by_allocator();
+
init_mem_tracker_by_allocator(lru_cache_type_string(LRUCacheType::SIZE));
}
};
diff --git a/be/src/runtime/memory/cache_policy.h
b/be/src/runtime/memory/cache_policy.h
index ceec3bd4350..a7c421ed9a7 100644
--- a/be/src/runtime/memory/cache_policy.h
+++ b/be/src/runtime/memory/cache_policy.h
@@ -95,8 +95,30 @@ public:
virtual void prune_all(bool force) = 0;
CacheType type() { return _type; }
+ void init_mem_tracker(const std::string& type_name) {
+ _mem_tracker =
+ std::make_unique<MemTracker>(fmt::format("{}[{}]",
type_string(_type), type_name),
+
ExecEnv::GetInstance()->details_mem_tracker_set());
+ }
MemTracker* mem_tracker() { return _mem_tracker.get(); }
- int64_t mem_consumption() { return _mem_tracker->consumption(); }
+ void init_mem_tracker_by_allocator(const std::string& type_name) {
+ _mem_tracker_by_allocator = MemTrackerLimiter::create_shared(
+ MemTrackerLimiter::Type::GLOBAL,
+ fmt::format("{}[{}](AllocByAllocator)", type_string(_type),
type_name));
+ }
+ std::shared_ptr<MemTrackerLimiter> mem_tracker_by_allocator() const {
+ DCHECK(_mem_tracker_by_allocator != nullptr);
+ return _mem_tracker_by_allocator;
+ }
+ int64_t mem_consumption() {
+ if (_mem_tracker_by_allocator != nullptr) {
+ return _mem_tracker_by_allocator->consumption();
+ } else if (_mem_tracker != nullptr) {
+ return _mem_tracker->consumption();
+ }
+ LOG(FATAL) << "__builtin_unreachable";
+ __builtin_unreachable();
+ }
bool enable_prune() const { return _enable_prune; }
RuntimeProfile* profile() { return _profile.get(); }
@@ -111,15 +133,10 @@ protected:
_cost_timer = ADD_TIMER(_profile, "CostTime");
}
- void init_mem_tracker(const std::string& type_name) {
- _mem_tracker =
- std::make_unique<MemTracker>(fmt::format("{}[{}]",
type_string(_type), type_name),
-
ExecEnv::GetInstance()->details_mem_tracker_set());
- }
-
CacheType _type;
std::unique_ptr<MemTracker> _mem_tracker;
+ std::shared_ptr<MemTrackerLimiter> _mem_tracker_by_allocator;
std::unique_ptr<RuntimeProfile> _profile;
RuntimeProfile::Counter* _prune_stale_number_counter = nullptr;
diff --git a/be/src/runtime/memory/lru_cache_policy.h
b/be/src/runtime/memory/lru_cache_policy.h
index 0d9524f2212..8f0806e9da9 100644
--- a/be/src/runtime/memory/lru_cache_policy.h
+++ b/be/src/runtime/memory/lru_cache_policy.h
@@ -91,17 +91,6 @@ public:
}
}
- void init_mem_tracker_by_allocator() {
- _mem_tracker_by_allocator = MemTrackerLimiter::create_shared(
- MemTrackerLimiter::Type::GLOBAL,
- fmt::format("{}[{}](AllocByAllocator)", type_string(_type),
- lru_cache_type_string(_lru_cache_type)));
- }
- std::shared_ptr<MemTrackerLimiter> mem_tracker_by_allocator() const {
- DCHECK(_mem_tracker_by_allocator != nullptr);
- return _mem_tracker_by_allocator;
- }
-
// Insert and cache value destroy will be manually consume tracking_bytes
to mem tracker.
// If lru cache is LRUCacheType::SIZE, tracking_bytes usually equal to
charge.
Cache::Handle* insert(const CacheKey& key, void* value, size_t charge,
size_t tracking_bytes,
@@ -214,7 +203,6 @@ private:
// compatible with ShardedLRUCache usage, but will not actually cache.
std::shared_ptr<Cache> _cache;
LRUCacheType _lru_cache_type;
- std::shared_ptr<MemTrackerLimiter> _mem_tracker_by_allocator;
};
} // namespace doris
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
