This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 31c3806a7d9 [fix](memory) Fix BE memory info compatible with Cgroup
(#35412)
31c3806a7d9 is described below
commit 31c3806a7d9a42a6894afe13471937cc1083e45d
Author: Xinyi Zou <[email protected]>
AuthorDate: Mon May 27 11:44:09 2024 +0800
[fix](memory) Fix BE memory info compatible with Cgroup (#35412)
1. `memory.usage_in_bytes ~= free.used + free.(buff/cache) - (buff)`, free
cache can be reused,
so, modify cgroup_memory_usage = memory.usage_in_bytes -
memory.meminfo["Cached"].
2. If system not configured with cgroup, find cgroup file path will failed,
refactor refresh cgroup memory info, compatible with find failed.
---
be/src/common/config.cpp | 2 +
be/src/common/config.h | 3 ++
be/src/util/cgroup_util.cpp | 10 ++++
be/src/util/cgroup_util.h | 1 +
be/src/util/mem_info.cpp | 108 +++++++++++++++++++++++++++++++++++---------
be/src/util/mem_info.h | 7 ++-
6 files changed, 108 insertions(+), 23 deletions(-)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 0e2680a7949..7bc2d1ee1ab 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -78,6 +78,8 @@ DEFINE_String(priority_networks, "");
// performance or compact
DEFINE_String(memory_mode, "moderate");
+DEFINE_mBool(enable_use_cgroup_memory_info, "true");
+
// process memory limit specified as number of bytes
// ('<int>[bB]?'), megabytes ('<float>[mM]'), gigabytes ('<float>[gG]'),
// or percentage of the physical memory ('<int>%').
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 4cbb9d14c3f..eedb9e21950 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -118,6 +118,9 @@ DECLARE_String(priority_networks);
// performance moderate or compact, only tcmalloc compile
DECLARE_String(memory_mode);
+// if true, process memory limit and memory usage based on cgroup memory info.
+DECLARE_mBool(enable_use_cgroup_memory_info);
+
// process memory limit specified as number of bytes
// ('<int>[bB]?'), megabytes ('<float>[mM]'), gigabytes ('<float>[gG]'),
// or percentage of the physical memory ('<int>%').
diff --git a/be/src/util/cgroup_util.cpp b/be/src/util/cgroup_util.cpp
index 02337392253..a2c3e294e66 100644
--- a/be/src/util/cgroup_util.cpp
+++ b/be/src/util/cgroup_util.cpp
@@ -178,6 +178,16 @@ Status CGroupUtil::find_cgroup_mem_usage(int64_t* bytes) {
return read_cgroup_value(usage_file_path, bytes);
}
+Status CGroupUtil::find_cgroup_mem_info(std::string* file_path) {
+ if (!enable()) {
+ return Status::InvalidArgument("cgroup is not enabled!");
+ }
+ string cgroup_path;
+ RETURN_IF_ERROR(find_abs_cgroup_path("memory", &cgroup_path));
+ *file_path = cgroup_path + "/memory.meminfo";
+ return Status::OK();
+}
+
Status CGroupUtil::find_cgroup_cpu_limit(float* cpu_count) {
if (!enable()) {
return Status::InvalidArgument("cgroup is not enabled!");
diff --git a/be/src/util/cgroup_util.h b/be/src/util/cgroup_util.h
index 015da392f7b..2152720ccdd 100644
--- a/be/src/util/cgroup_util.h
+++ b/be/src/util/cgroup_util.h
@@ -34,6 +34,7 @@ public:
// memory.usage_in_bytes ~= free.used + free.(buff/cache) - (buff)
//
https://serverfault.com/questions/902009/the-memory-usage-reported-in-cgroup-differs-from-the-free-command
static Status find_cgroup_mem_usage(int64_t* bytes);
+ static Status find_cgroup_mem_info(std::string* file_path);
// Determines the CGroup cpu cores limit from the current processes'
cgroup.
static Status find_cgroup_cpu_limit(float* cpu_count);
diff --git a/be/src/util/mem_info.cpp b/be/src/util/mem_info.cpp
index 62ddbcdda4d..d8a2ffbbf6c 100644
--- a/be/src/util/mem_info.cpp
+++ b/be/src/util/mem_info.cpp
@@ -59,8 +59,6 @@ bvar::PassiveStatus<int64_t> g_sys_mem_avail(
bool MemInfo::_s_initialized = false;
std::atomic<int64_t> MemInfo::_s_physical_mem =
std::numeric_limits<int64_t>::max();
-int64_t MemInfo::_s_cgroup_mem_limit = std::numeric_limits<int64_t>::max();
-int64_t MemInfo::_s_cgroup_mem_limit_refresh_wait_times = 0;
std::atomic<int64_t> MemInfo::_s_mem_limit =
std::numeric_limits<int64_t>::max();
std::atomic<int64_t> MemInfo::_s_soft_mem_limit =
std::numeric_limits<int64_t>::max();
@@ -69,6 +67,12 @@ std::string MemInfo::_s_allocator_cache_mem_str = "";
std::atomic<int64_t> MemInfo::_s_virtual_memory_used = 0;
std::atomic<int64_t> MemInfo::refresh_interval_memory_growth = 0;
+int64_t MemInfo::_s_cgroup_mem_limit = std::numeric_limits<int64_t>::max();
+int64_t MemInfo::_s_cgroup_mem_usage = std::numeric_limits<int64_t>::min();
+static std::unordered_map<std::string, int64_t> _s_cgroup_mem_info_bytes;
+bool MemInfo::_s_cgroup_mem_refresh_state = false;
+int64_t MemInfo::_s_cgroup_mem_refresh_wait_times = 0;
+
static std::unordered_map<std::string, int64_t> _mem_info_bytes;
std::atomic<int64_t> MemInfo::_s_sys_mem_available = -1;
int64_t MemInfo::_s_sys_mem_available_low_water_mark = -1;
@@ -397,27 +401,91 @@ void MemInfo::refresh_proc_meminfo() {
meminfo.close();
}
- // 1. calculate physical_mem
- int64_t physical_mem = -1;
- int64_t cgroup_mem_limit = -1;
- physical_mem = _mem_info_bytes["MemTotal"];
- if (_s_cgroup_mem_limit_refresh_wait_times >= 0) {
+ // refresh cgroup memory
+ if (_s_cgroup_mem_refresh_wait_times >= 0 &&
config::enable_use_cgroup_memory_info) {
+ int64_t cgroup_mem_limit = -1;
+ int64_t cgroup_mem_usage = -1;
+ std::string cgroup_mem_info_file_path;
+ _s_cgroup_mem_refresh_state = true;
Status status = CGroupUtil::find_cgroup_mem_limit(&cgroup_mem_limit);
- if (status.ok() && cgroup_mem_limit > 0) {
+ if (!status.ok() || cgroup_mem_limit <= 0) {
+ _s_cgroup_mem_refresh_state = false;
+ }
+ status = CGroupUtil::find_cgroup_mem_usage(&cgroup_mem_usage);
+ if (!status.ok() || cgroup_mem_usage <= 0) {
+ _s_cgroup_mem_refresh_state = false;
+ }
+ status = CGroupUtil::find_cgroup_mem_info(&cgroup_mem_info_file_path);
+ if (status.ok()) {
+ std::ifstream cgroup_meminfo(cgroup_mem_info_file_path,
std::ios::in);
+ std::string line;
+
+ while (cgroup_meminfo.good() && !cgroup_meminfo.eof()) {
+ getline(cgroup_meminfo, line);
+ std::vector<std::string> fields =
+ strings::Split(line, " ", strings::SkipWhitespace());
+ if (fields.size() < 2) {
+ continue;
+ }
+ std::string key = fields[0].substr(0, fields[0].size() - 1);
+
+ StringParser::ParseResult result;
+ auto mem_value =
StringParser::string_to_int<int64_t>(fields[1].data(),
+
fields[1].size(), &result);
+
+ if (result == StringParser::PARSE_SUCCESS) {
+ if (fields.size() == 2) {
+ _s_cgroup_mem_info_bytes[key] = mem_value;
+ } else if (fields[2] == "kB") {
+ _s_cgroup_mem_info_bytes[key] = mem_value * 1024L;
+ }
+ }
+ }
+ if (cgroup_meminfo.is_open()) {
+ cgroup_meminfo.close();
+ }
+ } else {
+ _s_cgroup_mem_refresh_state = false;
+ }
+
+ if (_s_cgroup_mem_refresh_state) {
_s_cgroup_mem_limit = cgroup_mem_limit;
- _s_cgroup_mem_limit_refresh_wait_times =
- -1000; // wait 10s, 1000 * 100ms, avoid too frequently.
+ //
https://serverfault.com/questions/902009/the-memory-usage-reported-in-cgroup-differs-from-the-free-command
+ // memory.usage_in_bytes ~= free.used + free.(buff/cache) - (buff)
+ // so, memory.usage_in_bytes - memory.meminfo["Cached"]
+ _s_cgroup_mem_usage = cgroup_mem_usage -
_s_cgroup_mem_info_bytes["Cached"];
+ // wait 10s, 100 * 100ms, avoid too frequently.
+ _s_cgroup_mem_refresh_wait_times = -100;
+ LOG(INFO) << "Refresh cgroup memory win, refresh again after 10s,
cgroup mem limit: "
+ << _s_cgroup_mem_limit << ", cgroup mem usage: " <<
_s_cgroup_mem_usage
+ << ", cgroup mem info cached: " <<
_s_cgroup_mem_info_bytes["Cached"];
} else {
- _s_cgroup_mem_limit = std::numeric_limits<int64_t>::max();
- _s_cgroup_mem_limit_refresh_wait_times =
- -6000; // find cgroup failed, wait 60s, 6000 * 100ms.
+ // find cgroup failed, wait 300s, 1000 * 100ms.
+ _s_cgroup_mem_refresh_wait_times = -3000;
+ LOG(INFO)
+ << "Refresh cgroup memory failed, refresh again after
300s, cgroup mem limit: "
+ << _s_cgroup_mem_limit << ", cgroup mem usage: " <<
_s_cgroup_mem_usage
+ << ", cgroup mem info cached: " <<
_s_cgroup_mem_info_bytes["Cached"];
}
} else {
- _s_cgroup_mem_limit_refresh_wait_times++;
+ if (config::enable_use_cgroup_memory_info) {
+ _s_cgroup_mem_refresh_wait_times++;
+ } else {
+ _s_cgroup_mem_refresh_state = false;
+ }
}
- if (_s_cgroup_mem_limit > 0) {
+
+ // 1. calculate physical_mem
+ int64_t physical_mem = -1;
+
+ physical_mem = _mem_info_bytes["MemTotal"];
+ if (_s_cgroup_mem_refresh_state) {
// In theory, always cgroup_mem_limit < physical_mem
- physical_mem = std::min(physical_mem, _s_cgroup_mem_limit);
+ if (physical_mem < 0) {
+ physical_mem = _s_cgroup_mem_limit;
+ } else {
+ physical_mem = std::min(physical_mem, _s_cgroup_mem_limit);
+ }
}
if (physical_mem <= 0) {
@@ -453,16 +521,14 @@ void MemInfo::refresh_proc_meminfo() {
// 3. refresh process available memory
int64_t mem_available = -1;
- int64_t cgroup_mem_usage = 0;
if (_mem_info_bytes.find("MemAvailable") != _mem_info_bytes.end()) {
mem_available = _mem_info_bytes["MemAvailable"];
}
- auto status = CGroupUtil::find_cgroup_mem_usage(&cgroup_mem_usage);
- if (status.ok() && cgroup_mem_usage > 0 && cgroup_mem_limit > 0) {
+ if (_s_cgroup_mem_refresh_state) {
if (mem_available < 0) {
- mem_available = cgroup_mem_limit - cgroup_mem_usage;
+ mem_available = _s_cgroup_mem_limit - _s_cgroup_mem_usage;
} else {
- mem_available = std::min(mem_available, cgroup_mem_limit -
cgroup_mem_usage);
+ mem_available = std::min(mem_available, _s_cgroup_mem_limit -
_s_cgroup_mem_usage);
}
}
if (mem_available < 0) {
diff --git a/be/src/util/mem_info.h b/be/src/util/mem_info.h
index 3032148115d..e44646bf328 100644
--- a/be/src/util/mem_info.h
+++ b/be/src/util/mem_info.h
@@ -211,8 +211,6 @@ public:
private:
static bool _s_initialized;
static std::atomic<int64_t> _s_physical_mem;
- static int64_t _s_cgroup_mem_limit;
- static int64_t _s_cgroup_mem_limit_refresh_wait_times;
static std::atomic<int64_t> _s_mem_limit;
static std::atomic<int64_t> _s_soft_mem_limit;
@@ -220,6 +218,11 @@ private:
static std::string _s_allocator_cache_mem_str;
static std::atomic<int64_t> _s_virtual_memory_used;
+ static int64_t _s_cgroup_mem_limit;
+ static int64_t _s_cgroup_mem_usage;
+ static bool _s_cgroup_mem_refresh_state;
+ static int64_t _s_cgroup_mem_refresh_wait_times;
+
static std::atomic<int64_t> _s_sys_mem_available;
static int64_t _s_sys_mem_available_low_water_mark;
static int64_t _s_sys_mem_available_warning_water_mark;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
