This is an automated email from the ASF dual-hosted git repository. yiguolei pushed a commit to branch 2.1-tmp in repository https://gitbox.apache.org/repos/asf/doris.git
commit 1b3a11a02b79a33a8d66a0a48bc27fb26ddd253b Author: abmdocrt <[email protected]> AuthorDate: Mon Apr 1 22:53:21 2024 +0800 [Enhancement](merge-on-write) Support dynamic delete bitmap cache (#32991) * The default delete bitmap cache is set to 100MB, which can be insufficient and cause performance issues when the amount of user data is large. To mitigate the problem of an inadequate cache, we will take the larger of 5% of the total memory and 100MB as the delete bitmap cache size. --- be/src/common/config.cpp | 5 +++++ be/src/common/config.h | 1 + be/src/olap/tablet_meta.cpp | 15 ++++++++++++++- 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index f3bde7dd36b..17a5587f073 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -797,6 +797,11 @@ DEFINE_mInt32(jdbc_connection_pool_cache_clear_time_sec, "28800"); // Global bitmap cache capacity for aggregation cache, size in bytes DEFINE_Int64(delete_bitmap_agg_cache_capacity, "104857600"); +// The default delete bitmap cache is set to 100MB, +// which can be insufficient and cause performance issues when the amount of user data is large. +// To mitigate the problem of an inadequate cache, +// we will take the larger of 0.5% of the total memory and 100MB as the delete bitmap cache size. +DEFINE_String(delete_bitmap_dynamic_agg_cache_limit, "0.5%"); DEFINE_mInt32(delete_bitmap_agg_cache_stale_sweep_time_sec, "1800"); // reference https://github.com/edenhill/librdkafka/blob/master/INTRODUCTION.md#broker-version-compatibility diff --git a/be/src/common/config.h b/be/src/common/config.h index a0845d043de..6076ce79433 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -851,6 +851,7 @@ DECLARE_mInt32(jdbc_connection_pool_cache_clear_time_sec); // Global bitmap cache capacity for aggregation cache, size in bytes DECLARE_Int64(delete_bitmap_agg_cache_capacity); +DECLARE_String(delete_bitmap_dynamic_agg_cache_limit); DECLARE_mInt32(delete_bitmap_agg_cache_stale_sweep_time_sec); // A common object cache depends on an Sharded LRU Cache. diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp index d8235191b93..f8a3bb9e97a 100644 --- a/be/src/olap/tablet_meta.cpp +++ b/be/src/olap/tablet_meta.cpp @@ -38,6 +38,8 @@ #include "olap/tablet_meta_manager.h" #include "olap/utils.h" #include "util/debug_points.h" +#include "util/mem_info.h" +#include "util/parse_util.h" #include "util/string_util.h" #include "util/time.h" #include "util/uid_util.h" @@ -924,7 +926,18 @@ bool operator!=(const TabletMeta& a, const TabletMeta& b) { } DeleteBitmap::DeleteBitmap(int64_t tablet_id) : _tablet_id(tablet_id) { - _agg_cache.reset(new AggCache(config::delete_bitmap_agg_cache_capacity)); + // The default delete bitmap cache is set to 100MB, + // which can be insufficient and cause performance issues when the amount of user data is large. + // To mitigate the problem of an inadequate cache, + // we will take the larger of 0.5% of the total memory and 100MB as the delete bitmap cache size. + bool is_percent = false; + int64_t delete_bitmap_agg_cache_cache_limit = + ParseUtil::parse_mem_spec(config::delete_bitmap_dynamic_agg_cache_limit, + MemInfo::mem_limit(), MemInfo::physical_mem(), &is_percent); + _agg_cache.reset(new AggCache(delete_bitmap_agg_cache_cache_limit > + config::delete_bitmap_agg_cache_capacity + ? delete_bitmap_agg_cache_cache_limit + : config::delete_bitmap_agg_cache_capacity)); } DeleteBitmap::DeleteBitmap(const DeleteBitmap& o) { --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
