This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 2517ceac5d8 [Enhancement](merge-on-write) Support dynamic delete
bitmap cache (#32991)
2517ceac5d8 is described below
commit 2517ceac5d8494a706820254a7ce93d5cf10523b
Author: abmdocrt <[email protected]>
AuthorDate: Mon Apr 1 22:53:21 2024 +0800
[Enhancement](merge-on-write) Support dynamic delete bitmap cache (#32991)
* The default delete bitmap cache is set to 100MB, which can be
insufficient and cause performance issues when the amount of user data is
large. To mitigate the problem of an inadequate cache, we will take the larger
of 5% of the total memory and 100MB as the delete bitmap cache size.
---
be/src/common/config.cpp | 5 +++++
be/src/common/config.h | 1 +
be/src/olap/tablet_meta.cpp | 15 ++++++++++++++-
3 files changed, 20 insertions(+), 1 deletion(-)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 2ac6bb06917..0bd7a767abb 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -800,6 +800,11 @@ DEFINE_mInt32(jdbc_connection_pool_cache_clear_time_sec,
"28800");
// Global bitmap cache capacity for aggregation cache, size in bytes
DEFINE_Int64(delete_bitmap_agg_cache_capacity, "104857600");
+// The default delete bitmap cache is set to 100MB,
+// which can be insufficient and cause performance issues when the amount of
user data is large.
+// To mitigate the problem of an inadequate cache,
+// we will take the larger of 0.5% of the total memory and 100MB as the delete
bitmap cache size.
+DEFINE_String(delete_bitmap_dynamic_agg_cache_limit, "0.5%");
DEFINE_mInt32(delete_bitmap_agg_cache_stale_sweep_time_sec, "1800");
// reference
https://github.com/edenhill/librdkafka/blob/master/INTRODUCTION.md#broker-version-compatibility
diff --git a/be/src/common/config.h b/be/src/common/config.h
index e3c404463a1..00b1f1a43b3 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -856,6 +856,7 @@ DECLARE_mInt32(jdbc_connection_pool_cache_clear_time_sec);
// Global bitmap cache capacity for aggregation cache, size in bytes
DECLARE_Int64(delete_bitmap_agg_cache_capacity);
+DECLARE_String(delete_bitmap_dynamic_agg_cache_limit);
DECLARE_mInt32(delete_bitmap_agg_cache_stale_sweep_time_sec);
// A common object cache depends on an Sharded LRU Cache.
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index a15e809b8b8..434fbaaf2e8 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -41,6 +41,8 @@
#include "olap/tablet_meta_manager.h"
#include "olap/utils.h"
#include "util/debug_points.h"
+#include "util/mem_info.h"
+#include "util/parse_util.h"
#include "util/string_util.h"
#include "util/time.h"
#include "util/uid_util.h"
@@ -930,7 +932,18 @@ bool operator!=(const TabletMeta& a, const TabletMeta& b) {
}
DeleteBitmap::DeleteBitmap(int64_t tablet_id) : _tablet_id(tablet_id) {
- _agg_cache.reset(new AggCache(config::delete_bitmap_agg_cache_capacity));
+ // The default delete bitmap cache is set to 100MB,
+ // which can be insufficient and cause performance issues when the amount
of user data is large.
+ // To mitigate the problem of an inadequate cache,
+ // we will take the larger of 0.5% of the total memory and 100MB as the
delete bitmap cache size.
+ bool is_percent = false;
+ int64_t delete_bitmap_agg_cache_cache_limit =
+
ParseUtil::parse_mem_spec(config::delete_bitmap_dynamic_agg_cache_limit,
+ MemInfo::mem_limit(),
MemInfo::physical_mem(), &is_percent);
+ _agg_cache.reset(new AggCache(delete_bitmap_agg_cache_cache_limit >
+
config::delete_bitmap_agg_cache_capacity
+ ? delete_bitmap_agg_cache_cache_limit
+ :
config::delete_bitmap_agg_cache_capacity));
}
DeleteBitmap::DeleteBitmap(const DeleteBitmap& o) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
