morningman commented on code in PR #16084:
URL: https://github.com/apache/doris/pull/16084#discussion_r1082727190
##########
be/src/common/daemon.cpp:
##########
@@ -203,29 +205,42 @@ void Daemon::memory_maintenance_thread() {
doris::MemInfo::refresh_allocator_mem();
#endif
doris::MemInfo::refresh_proc_mem_no_allocator_cache();
- LOG_EVERY_N(INFO, 10) << MemTrackerLimiter::process_mem_log_str();
// Refresh mem tracker each type metrics.
doris::MemTrackerLimiter::refresh_global_counter();
- if (doris::config::memory_debug) {
- doris::MemTrackerLimiter::print_log_process_usage("memory_debug",
false);
- }
doris::MemTrackerLimiter::enable_print_log_process_usage();
// If system available memory is not enough, or the process memory
exceeds the limit, reduce refresh interval.
if (doris::MemInfo::sys_mem_available() <
doris::MemInfo::sys_mem_available_low_water_mark() ||
doris::MemInfo::proc_mem_no_allocator_cache() >=
doris::MemInfo::mem_limit()) {
- interval_milliseconds = 100;
+ doris::MemTrackerLimiter::print_log_process_usage("process full
gc", false);
Review Comment:
Looks like it will print too many logs when memory is insuffient?
Because the interval is only 500ms?
##########
be/src/common/daemon.cpp:
##########
@@ -203,29 +205,42 @@ void Daemon::memory_maintenance_thread() {
doris::MemInfo::refresh_allocator_mem();
#endif
doris::MemInfo::refresh_proc_mem_no_allocator_cache();
- LOG_EVERY_N(INFO, 10) << MemTrackerLimiter::process_mem_log_str();
// Refresh mem tracker each type metrics.
doris::MemTrackerLimiter::refresh_global_counter();
- if (doris::config::memory_debug) {
- doris::MemTrackerLimiter::print_log_process_usage("memory_debug",
false);
- }
doris::MemTrackerLimiter::enable_print_log_process_usage();
// If system available memory is not enough, or the process memory
exceeds the limit, reduce refresh interval.
if (doris::MemInfo::sys_mem_available() <
doris::MemInfo::sys_mem_available_low_water_mark() ||
doris::MemInfo::proc_mem_no_allocator_cache() >=
doris::MemInfo::mem_limit()) {
- interval_milliseconds = 100;
+ doris::MemTrackerLimiter::print_log_process_usage("process full
gc", false);
+ interval_milliseconds = std::min(100,
config::memory_maintenance_sleep_time_ms);
doris::MemInfo::process_full_gc();
+ cache_gc_interval_ms = config::cache_gc_interval_s * 1000;
} else if (doris::MemInfo::sys_mem_available() <
doris::MemInfo::sys_mem_available_warning_water_mark() ||
doris::MemInfo::proc_mem_no_allocator_cache() >=
doris::MemInfo::soft_mem_limit()) {
- interval_milliseconds = 200;
+ doris::MemTrackerLimiter::print_log_process_usage("process minor
gc", false);
+ interval_milliseconds = std::min(200,
config::memory_maintenance_sleep_time_ms);
doris::MemInfo::process_minor_gc();
+ cache_gc_interval_ms = config::cache_gc_interval_s * 1000;
} else {
interval_milliseconds = config::memory_maintenance_sleep_time_ms;
+ if (doris::config::memory_debug) {
+ doris::MemTrackerLimiter::print_log_process_usage("memory
debug", false);
+ } else {
+ LOG_EVERY_N(INFO, 10) <<
MemTrackerLimiter::process_mem_log_str();
+ }
+ cache_gc_interval_ms -= interval_milliseconds;
+ if (cache_gc_interval_ms < 0) {
+ cache_gc_freed_mem = 0;
+ doris::MemInfo::process_cache_gc(cache_gc_freed_mem);
+ LOG(INFO) << fmt::format("Process regular GC Cache, Free
Memory {} Bytes",
Review Comment:
ditto
##########
be/src/common/daemon.cpp:
##########
@@ -203,29 +205,42 @@ void Daemon::memory_maintenance_thread() {
doris::MemInfo::refresh_allocator_mem();
#endif
doris::MemInfo::refresh_proc_mem_no_allocator_cache();
- LOG_EVERY_N(INFO, 10) << MemTrackerLimiter::process_mem_log_str();
// Refresh mem tracker each type metrics.
doris::MemTrackerLimiter::refresh_global_counter();
- if (doris::config::memory_debug) {
- doris::MemTrackerLimiter::print_log_process_usage("memory_debug",
false);
- }
doris::MemTrackerLimiter::enable_print_log_process_usage();
// If system available memory is not enough, or the process memory
exceeds the limit, reduce refresh interval.
if (doris::MemInfo::sys_mem_available() <
doris::MemInfo::sys_mem_available_low_water_mark() ||
doris::MemInfo::proc_mem_no_allocator_cache() >=
doris::MemInfo::mem_limit()) {
- interval_milliseconds = 100;
+ doris::MemTrackerLimiter::print_log_process_usage("process full
gc", false);
+ interval_milliseconds = std::min(100,
config::memory_maintenance_sleep_time_ms);
doris::MemInfo::process_full_gc();
+ cache_gc_interval_ms = config::cache_gc_interval_s * 1000;
} else if (doris::MemInfo::sys_mem_available() <
doris::MemInfo::sys_mem_available_warning_water_mark() ||
doris::MemInfo::proc_mem_no_allocator_cache() >=
doris::MemInfo::soft_mem_limit()) {
- interval_milliseconds = 200;
+ doris::MemTrackerLimiter::print_log_process_usage("process minor
gc", false);
Review Comment:
ditto
##########
be/src/common/daemon.cpp:
##########
@@ -203,29 +205,42 @@ void Daemon::memory_maintenance_thread() {
doris::MemInfo::refresh_allocator_mem();
#endif
doris::MemInfo::refresh_proc_mem_no_allocator_cache();
- LOG_EVERY_N(INFO, 10) << MemTrackerLimiter::process_mem_log_str();
// Refresh mem tracker each type metrics.
doris::MemTrackerLimiter::refresh_global_counter();
- if (doris::config::memory_debug) {
- doris::MemTrackerLimiter::print_log_process_usage("memory_debug",
false);
- }
doris::MemTrackerLimiter::enable_print_log_process_usage();
// If system available memory is not enough, or the process memory
exceeds the limit, reduce refresh interval.
if (doris::MemInfo::sys_mem_available() <
doris::MemInfo::sys_mem_available_low_water_mark() ||
doris::MemInfo::proc_mem_no_allocator_cache() >=
doris::MemInfo::mem_limit()) {
- interval_milliseconds = 100;
+ doris::MemTrackerLimiter::print_log_process_usage("process full
gc", false);
+ interval_milliseconds = std::min(100,
config::memory_maintenance_sleep_time_ms);
doris::MemInfo::process_full_gc();
+ cache_gc_interval_ms = config::cache_gc_interval_s * 1000;
} else if (doris::MemInfo::sys_mem_available() <
doris::MemInfo::sys_mem_available_warning_water_mark() ||
doris::MemInfo::proc_mem_no_allocator_cache() >=
doris::MemInfo::soft_mem_limit()) {
- interval_milliseconds = 200;
+ doris::MemTrackerLimiter::print_log_process_usage("process minor
gc", false);
+ interval_milliseconds = std::min(200,
config::memory_maintenance_sleep_time_ms);
doris::MemInfo::process_minor_gc();
+ cache_gc_interval_ms = config::cache_gc_interval_s * 1000;
} else {
interval_milliseconds = config::memory_maintenance_sleep_time_ms;
+ if (doris::config::memory_debug) {
+ doris::MemTrackerLimiter::print_log_process_usage("memory
debug", false);
+ } else {
+ LOG_EVERY_N(INFO, 10) <<
MemTrackerLimiter::process_mem_log_str();
Review Comment:
ditto
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
