This is an automated email from the ASF dual-hosted git repository. yiguolei pushed a commit to branch spill_and_reserve in repository https://gitbox.apache.org/repos/asf/doris.git
commit 260d0374d290612fdbee6c0d79a0cab4da28a811 Author: yiguolei <[email protected]> AuthorDate: Mon Sep 23 10:46:15 2024 +0800 [opt](cache elimation) using step decending method to elimiate cache (#41086) ## Proposed changes The cache elimation logic should not use a Linear continuity double value, should use a step by step value. Or the cache limitation value will be changed often. --------- Co-authored-by: yiguolei <[email protected]> --- be/src/common/daemon.cpp | 12 +++------ be/src/util/algorithm_util.h | 55 +++++++++++++++++++++++++++++++++++++++++ be/test/util/algo_util_test.cpp | 46 ++++++++++++++++++++++++++++++++++ 3 files changed, 105 insertions(+), 8 deletions(-) diff --git a/be/src/common/daemon.cpp b/be/src/common/daemon.cpp index 6e500ef6d59..210dbe8008e 100644 --- a/be/src/common/daemon.cpp +++ b/be/src/common/daemon.cpp @@ -58,6 +58,7 @@ #include "runtime/memory/memory_reclamation.h" #include "runtime/runtime_query_statistics_mgr.h" #include "runtime/workload_group/workload_group_manager.h" +#include "util/algorithm_util.h" #include "util/cpu_info.h" #include "util/debug_util.h" #include "util/disk_info.h" @@ -242,7 +243,7 @@ void refresh_memory_state_after_memory_change() { void refresh_cache_capacity() { if (refresh_cache_capacity_sleep_time_ms <= 0) { - auto cache_capacity_reduce_mem_limit = uint64_t( + auto cache_capacity_reduce_mem_limit = int64_t( doris::MemInfo::soft_mem_limit() * config::cache_capacity_reduce_mem_limit_frac); int64_t process_memory_usage = doris::GlobalMemoryArbitrator::process_memory_usage(); // the rule is like this: @@ -250,13 +251,8 @@ void refresh_cache_capacity() { // 2. if the process mem usage > soft memlimit * 0.6 and process mem usage < soft memlimit, then it will be adjusted to a lower value. // 3. if the process mem usage > soft memlimit, then the capacity is adjusted to 0. double new_cache_capacity_adjust_weighted = - process_memory_usage <= cache_capacity_reduce_mem_limit - ? 1 - : std::max<double>( - 1 - (process_memory_usage - cache_capacity_reduce_mem_limit) / - (doris::MemInfo::soft_mem_limit() - - cache_capacity_reduce_mem_limit), - 0); + AlgoUtil::descent_by_step(10, cache_capacity_reduce_mem_limit, + doris::MemInfo::soft_mem_limit(), process_memory_usage); if (new_cache_capacity_adjust_weighted != doris::GlobalMemoryArbitrator::last_cache_capacity_adjust_weighted) { doris::GlobalMemoryArbitrator::last_cache_capacity_adjust_weighted = diff --git a/be/src/util/algorithm_util.h b/be/src/util/algorithm_util.h new file mode 100644 index 00000000000..acddd3be3a3 --- /dev/null +++ b/be/src/util/algorithm_util.h @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include <cstdint> +#include <utility> + +#include "common/status.h" +namespace doris { +class AlgoUtil { +public: + // descent the value step by step not linear continuity + // If the result is linear continuity, then the value will changed very quickly and will cost + // a lot of CPU and cache will not stable and will hold some lock. + // Its better to use step num to be 10, do not use 3, the divide value is not stable. + // For example, if step num is 3, then the result will be 0.33333... 0.66666..., the double value + // is not stable. + static double descent_by_step(int step_num, int64_t low_bound, int64_t high_bound, + int64_t current) { + if (current <= low_bound) { + return 1; + } + if (current >= high_bound) { + return 0; + } + if (high_bound <= low_bound) { + // Invalid + return 0; + } + // Use floor value, so that the step size is a little smaller than the actual value. + // And then the used step will be a little larger than the actual value. + int64_t step_size = (int64_t)std::floor((high_bound - low_bound) / (step_num * 1.0)); + int64_t used_step = (int64_t)std::ceil((current - low_bound) / (step_size * 1.0)); + // Then the left step is smaller than actual value. + // This elimation algo will elimate more cache than actual. + int64_t left_step = step_num - used_step; + return left_step / (step_num * 1.0); + } +}; +} // namespace doris \ No newline at end of file diff --git a/be/test/util/algo_util_test.cpp b/be/test/util/algo_util_test.cpp new file mode 100644 index 00000000000..cdf3316f1c9 --- /dev/null +++ b/be/test/util/algo_util_test.cpp @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include <gtest/gtest-message.h> +#include <gtest/gtest-test-part.h> + +#include <boost/utility/binary.hpp> +#include <memory> + +#include "gtest/gtest_pred_impl.h" +#include "util/algorithm_util.h" + +namespace doris { + +TEST(TestAlgoUtil, DescentByStep) { + // double descent_by_step(int step_num, int64_t low_bound, int64_t high_bound, int64_t current) + EXPECT_EQ(AlgoUtil::descent_by_step(10, 100, 200, 101), 0.9); + EXPECT_EQ(AlgoUtil::descent_by_step(10, 100, 200, 99), 1); + EXPECT_EQ(AlgoUtil::descent_by_step(10, 200, 100, 101), 1); + EXPECT_EQ(AlgoUtil::descent_by_step(10, 100, 200, 111), 0.8); + EXPECT_EQ(AlgoUtil::descent_by_step(10, 100, 200, 188), 0.1); + EXPECT_EQ(AlgoUtil::descent_by_step(10, 100, 200, 100), 1); + EXPECT_EQ(AlgoUtil::descent_by_step(10, 100, 200, 200), 0); + EXPECT_EQ(AlgoUtil::descent_by_step(10, 100, 200, 300), 0); + + EXPECT_EQ(AlgoUtil::descent_by_step(4, 100, 200, 133), 0.5); + EXPECT_EQ(AlgoUtil::descent_by_step(4, 100, 200, 110), 0.75); + EXPECT_EQ(AlgoUtil::descent_by_step(4, 100, 200, 125), 0.75); + EXPECT_EQ(AlgoUtil::descent_by_step(4, 100, 200, 126), 0.5); +} + +} // namespace doris --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
