This is an automated email from the ASF dual-hosted git repository. yiguolei pushed a commit to branch 2.1-tmp in repository https://gitbox.apache.org/repos/asf/doris.git
commit 5aba0524acff15ab9b8746888c53154b71b02a3e Author: abmdocrt <[email protected]> AuthorDate: Mon Apr 1 22:53:21 2024 +0800 [Enhancement](merge-on-write) Support dynamic delete bitmap cache (#32991) * The default delete bitmap cache is set to 100MB, which can be insufficient and cause performance issues when the amount of user data is large. To mitigate the problem of an inadequate cache, we will take the larger of 5% of the total memory and 100MB as the delete bitmap cache size. --- be/src/common/config.cpp | 5 +++++ be/src/common/config.h | 1 + be/src/olap/tablet_meta.cpp | 15 ++++++++++++++- 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 9364303a3ea..2c228f7cf63 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -794,6 +794,11 @@ DEFINE_mInt32(jdbc_connection_pool_cache_clear_time_sec, "28800"); // Global bitmap cache capacity for aggregation cache, size in bytes DEFINE_Int64(delete_bitmap_agg_cache_capacity, "104857600"); +// The default delete bitmap cache is set to 100MB, +// which can be insufficient and cause performance issues when the amount of user data is large. +// To mitigate the problem of an inadequate cache, +// we will take the larger of 0.5% of the total memory and 100MB as the delete bitmap cache size. +DEFINE_String(delete_bitmap_dynamic_agg_cache_limit, "0.5%"); DEFINE_mInt32(delete_bitmap_agg_cache_stale_sweep_time_sec, "1800"); // reference https://github.com/edenhill/librdkafka/blob/master/INTRODUCTION.md#broker-version-compatibility diff --git a/be/src/common/config.h b/be/src/common/config.h index 01d8a123a45..9ce6b242248 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -848,6 +848,7 @@ DECLARE_mInt32(jdbc_connection_pool_cache_clear_time_sec); // Global bitmap cache capacity for aggregation cache, size in bytes DECLARE_Int64(delete_bitmap_agg_cache_capacity); +DECLARE_String(delete_bitmap_dynamic_agg_cache_limit); DECLARE_mInt32(delete_bitmap_agg_cache_stale_sweep_time_sec); // A common object cache depends on an Sharded LRU Cache. diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp index d8235191b93..f8a3bb9e97a 100644 --- a/be/src/olap/tablet_meta.cpp +++ b/be/src/olap/tablet_meta.cpp @@ -38,6 +38,8 @@ #include "olap/tablet_meta_manager.h" #include "olap/utils.h" #include "util/debug_points.h" +#include "util/mem_info.h" +#include "util/parse_util.h" #include "util/string_util.h" #include "util/time.h" #include "util/uid_util.h" @@ -924,7 +926,18 @@ bool operator!=(const TabletMeta& a, const TabletMeta& b) { } DeleteBitmap::DeleteBitmap(int64_t tablet_id) : _tablet_id(tablet_id) { - _agg_cache.reset(new AggCache(config::delete_bitmap_agg_cache_capacity)); + // The default delete bitmap cache is set to 100MB, + // which can be insufficient and cause performance issues when the amount of user data is large. + // To mitigate the problem of an inadequate cache, + // we will take the larger of 0.5% of the total memory and 100MB as the delete bitmap cache size. + bool is_percent = false; + int64_t delete_bitmap_agg_cache_cache_limit = + ParseUtil::parse_mem_spec(config::delete_bitmap_dynamic_agg_cache_limit, + MemInfo::mem_limit(), MemInfo::physical_mem(), &is_percent); + _agg_cache.reset(new AggCache(delete_bitmap_agg_cache_cache_limit > + config::delete_bitmap_agg_cache_capacity + ? delete_bitmap_agg_cache_cache_limit + : config::delete_bitmap_agg_cache_capacity)); } DeleteBitmap::DeleteBitmap(const DeleteBitmap& o) { --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
