This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 4c30059f3d0 branch-3.1: [Opt](mow) Opt mow load performance and CPU
usage #55073 #55733 #55771 (#55767)
4c30059f3d0 is described below
commit 4c30059f3d0ad2754e223ad49cdb0565899d69e0
Author: bobhan1 <[email protected]>
AuthorDate: Sun Sep 14 00:22:36 2025 +0800
branch-3.1: [Opt](mow) Opt mow load performance and CPU usage #55073 #55733
#55771 (#55767)
pick #55073 #55733 #55771
---
be/src/cloud/cloud_base_compaction.cpp | 1 +
be/src/cloud/cloud_cumulative_compaction.cpp | 2 +
be/src/cloud/cloud_full_compaction.cpp | 1 +
be/src/common/config.cpp | 3 +
be/src/common/config.h | 3 +
be/src/http/action/delete_bitmap_action.cpp | 34 +++++
be/src/http/action/delete_bitmap_action.h | 3 +-
be/src/olap/base_tablet.cpp | 27 +++-
be/src/olap/base_tablet.h | 3 +
be/src/olap/compaction.cpp | 1 +
be/src/olap/full_compaction.cpp | 2 +
be/src/olap/lru_cache.cpp | 16 +++
be/src/olap/lru_cache.h | 5 +
be/src/olap/tablet_meta.cpp | 138 ++++++++++++++++++---
be/src/olap/tablet_meta.h | 8 +-
be/src/runtime/memory/lru_cache_policy.h | 4 +
be/src/service/http_service.cpp | 10 ++
.../pipeline/cloud_p0/conf/be_custom.conf | 2 +
.../pipeline/cloud_p1/conf/be_custom.conf | 1 +
regression-test/pipeline/p0/conf/be.conf | 2 +
regression-test/pipeline/p1/conf/be.conf | 2 +
.../metrics_p0/test_delete_bitmap_metrics.groovy | 31 +++++
22 files changed, 278 insertions(+), 21 deletions(-)
diff --git a/be/src/cloud/cloud_base_compaction.cpp
b/be/src/cloud/cloud_base_compaction.cpp
index 175abc04430..5e00721bfc7 100644
--- a/be/src/cloud/cloud_base_compaction.cpp
+++ b/be/src/cloud/cloud_base_compaction.cpp
@@ -435,6 +435,7 @@ Status CloudBaseCompaction::modify_rowsets() {
stats.num_rows(),
stats.data_size());
}
}
+ _tablet->prefill_dbm_agg_cache_after_compaction(_output_rowset);
return Status::OK();
}
diff --git a/be/src/cloud/cloud_cumulative_compaction.cpp
b/be/src/cloud/cloud_cumulative_compaction.cpp
index 64d63aee025..d2ea6ac0e8a 100644
--- a/be/src/cloud/cloud_cumulative_compaction.cpp
+++ b/be/src/cloud/cloud_cumulative_compaction.cpp
@@ -435,6 +435,8 @@ Status CloudCumulativeCompaction::modify_rowsets() {
LOG(INFO) << "delete_expired_stale_rowsets for tablet=" <<
_tablet->tablet_id();
_engine.tablet_mgr().vacuum_stale_rowsets(CountDownLatch(1));
});
+
+ _tablet->prefill_dbm_agg_cache_after_compaction(_output_rowset);
return Status::OK();
}
diff --git a/be/src/cloud/cloud_full_compaction.cpp
b/be/src/cloud/cloud_full_compaction.cpp
index 08e43ab2142..ac60c869154 100644
--- a/be/src/cloud/cloud_full_compaction.cpp
+++ b/be/src/cloud/cloud_full_compaction.cpp
@@ -292,6 +292,7 @@ Status CloudFullCompaction::modify_rowsets() {
stats.num_rows(),
stats.data_size());
}
}
+ _tablet->prefill_dbm_agg_cache_after_compaction(_output_rowset);
return Status::OK();
}
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index c2f69661741..aba7561d1a3 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1564,6 +1564,9 @@ DEFINE_mInt32(max_segment_partial_column_cache_size,
"500");
DEFINE_mBool(enable_wal_tde, "false");
+DEFINE_mBool(enable_prefill_output_dbm_agg_cache_after_compaction, "true");
+DEFINE_mBool(enable_prefill_all_dbm_agg_cache_after_compaction, "true");
+
// clang-format off
#ifdef BE_TEST
// test s3
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 8784bd71f1d..55591fc36c7 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1630,6 +1630,9 @@ DECLARE_mInt32(max_segment_partial_column_cache_size);
DECLARE_mBool(enable_wal_tde);
+DECLARE_mBool(enable_prefill_output_dbm_agg_cache_after_compaction);
+DECLARE_mBool(enable_prefill_all_dbm_agg_cache_after_compaction);
+
#ifdef BE_TEST
// test s3
DECLARE_String(test_s3_resource);
diff --git a/be/src/http/action/delete_bitmap_action.cpp
b/be/src/http/action/delete_bitmap_action.cpp
index 5fc4d0f4388..9e152e5af1f 100644
--- a/be/src/http/action/delete_bitmap_action.cpp
+++ b/be/src/http/action/delete_bitmap_action.cpp
@@ -188,6 +188,32 @@ Status
DeleteBitmapAction::_handle_show_ms_delete_bitmap_count(HttpRequest* req,
return Status::OK();
}
+Status
DeleteBitmapAction::_handle_show_agg_cache_delete_bitmap_count(HttpRequest* req,
+
std::string* json_result) {
+ uint64_t tablet_id = 0;
+ bool verbose = false;
+ RETURN_NOT_OK_STATUS_WITH_WARN(_check_param(req, &tablet_id, &verbose),
"check param failed");
+ BaseTabletSPtr tablet = nullptr;
+ if (config::is_cloud_mode()) {
+ tablet =
DORIS_TRY(_engine.to_cloud().tablet_mgr().get_tablet(tablet_id));
+ DBUG_EXECUTE_IF(
+
"DeleteBitmapAction._handle_show_local_delete_bitmap_count.vacuum_stale_rowsets",
+ {
_engine.to_cloud().tablet_mgr().vacuum_stale_rowsets(CountDownLatch(1)); });
+ } else {
+ tablet = _engine.to_local().tablet_manager()->get_tablet(tablet_id);
+ DBUG_EXECUTE_IF(
+
"DeleteBitmapAction._handle_show_local_delete_bitmap_count.start_delete_unused_"
+ "rowset",
+ { _engine.to_local().start_delete_unused_rowset(); });
+ }
+ if (tablet == nullptr) {
+ return Status::NotFound("Tablet not found. tablet_id={}", tablet_id);
+ }
+ auto dbm = tablet->tablet_meta()->delete_bitmap()->agg_cache_snapshot();
+ _show_delete_bitmap(dbm, verbose, json_result);
+ return Status::OK();
+}
+
void DeleteBitmapAction::handle(HttpRequest* req) {
req->add_output_header(HttpHeaders::CONTENT_TYPE, HEADER_JSON.data());
if (_delete_bitmap_action_type == DeleteBitmapActionType::COUNT_LOCAL) {
@@ -206,6 +232,14 @@ void DeleteBitmapAction::handle(HttpRequest* req) {
} else {
HttpChannel::send_reply(req, HttpStatus::OK, json_result);
}
+ } else if (_delete_bitmap_action_type ==
DeleteBitmapActionType::COUNT_AGG_CACHE) {
+ std::string json_result;
+ Status st = _handle_show_agg_cache_delete_bitmap_count(req,
&json_result);
+ if (!st.ok()) {
+ HttpChannel::send_reply(req, HttpStatus::OK, st.to_json());
+ } else {
+ HttpChannel::send_reply(req, HttpStatus::OK, json_result);
+ }
}
}
diff --git a/be/src/http/action/delete_bitmap_action.h
b/be/src/http/action/delete_bitmap_action.h
index 284e8dbcf57..db3ab3bcd89 100644
--- a/be/src/http/action/delete_bitmap_action.h
+++ b/be/src/http/action/delete_bitmap_action.h
@@ -32,7 +32,7 @@ class HttpRequest;
class ExecEnv;
-enum class DeleteBitmapActionType { COUNT_LOCAL = 1, COUNT_MS = 2 };
+enum class DeleteBitmapActionType { COUNT_LOCAL = 1, COUNT_MS = 2,
COUNT_AGG_CACHE = 3 };
/// This action is used for viewing the delete bitmap status
class DeleteBitmapAction : public HttpHandlerWithAuth {
@@ -47,6 +47,7 @@ public:
private:
Status _handle_show_local_delete_bitmap_count(HttpRequest* req,
std::string* json_result);
Status _handle_show_ms_delete_bitmap_count(HttpRequest* req, std::string*
json_result);
+ Status _handle_show_agg_cache_delete_bitmap_count(HttpRequest* req,
std::string* json_result);
private:
BaseStorageEngine& _engine;
diff --git a/be/src/olap/base_tablet.cpp b/be/src/olap/base_tablet.cpp
index cd22b88a4c8..b56c424348a 100644
--- a/be/src/olap/base_tablet.cpp
+++ b/be/src/olap/base_tablet.cpp
@@ -545,7 +545,7 @@ Status BaseTablet::lookup_row_key(const Slice& encoded_key,
TabletSchema* latest
if (!s.ok() && !s.is<KEY_ALREADY_EXISTS>()) {
return s;
}
- if (s.ok() &&
_tablet_meta->delete_bitmap()->contains_agg_without_cache(
+ if (s.ok() &&
_tablet_meta->delete_bitmap()->contains_agg_with_cache_if_eligible(
{loc.rowset_id, loc.segment_id, version},
loc.row_id)) {
// if has sequence col, we continue to compare the sequence_id
of
// all rowsets, util we find an existing key.
@@ -2135,4 +2135,29 @@ TabletSchemaSPtr
BaseTablet::calculate_variant_extended_schema() const {
return vectorized::schema_util::calculate_variant_extended_schema(rowsets,
_max_version_schema);
}
+void BaseTablet::prefill_dbm_agg_cache(const RowsetSharedPtr& rowset, int64_t
version) {
+ for (std::size_t i = 0; i < rowset->num_segments(); i++) {
+ tablet_meta()->delete_bitmap()->get_agg({rowset->rowset_id(), i,
version});
+ }
+}
+
+void BaseTablet::prefill_dbm_agg_cache_after_compaction(const RowsetSharedPtr&
output_rowset) {
+ if (keys_type() == KeysType::UNIQUE_KEYS &&
enable_unique_key_merge_on_write() &&
+ (config::enable_prefill_output_dbm_agg_cache_after_compaction ||
+ config::enable_prefill_all_dbm_agg_cache_after_compaction)) {
+ int64_t cur_max_version {-1};
+ {
+ std::shared_lock rlock(get_header_lock());
+ cur_max_version = max_version_unlocked();
+ }
+ if (config::enable_prefill_all_dbm_agg_cache_after_compaction) {
+ traverse_rowsets(
+ [&](const RowsetSharedPtr& rs) { prefill_dbm_agg_cache(rs,
cur_max_version); },
+ false);
+ } else if
(config::enable_prefill_output_dbm_agg_cache_after_compaction) {
+ prefill_dbm_agg_cache(output_rowset, cur_max_version);
+ }
+ }
+}
+
} // namespace doris
diff --git a/be/src/olap/base_tablet.h b/be/src/olap/base_tablet.h
index 38dba97829d..cb61252c6fe 100644
--- a/be/src/olap/base_tablet.h
+++ b/be/src/olap/base_tablet.h
@@ -344,6 +344,9 @@ public:
Result<CaptureRowsetResult> _remote_capture_rowsets(const Version&
version_range) const;
+ void prefill_dbm_agg_cache(const RowsetSharedPtr& rowset, int64_t version);
+ void prefill_dbm_agg_cache_after_compaction(const RowsetSharedPtr&
output_rowset);
+
protected:
// Find the missed versions until the spec_version.
//
diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp
index f4930c9f3da..a3ccc407078 100644
--- a/be/src/olap/compaction.cpp
+++ b/be/src/olap/compaction.cpp
@@ -1252,6 +1252,7 @@ Status CompactionMixin::modify_rowsets() {
}
DBUG_EXECUTE_IF("CumulativeCompaction.modify_rowsets.delete_expired_stale_rowset",
{ tablet()->delete_expired_stale_rowset(); });
+ _tablet->prefill_dbm_agg_cache_after_compaction(_output_rowset);
return Status::OK();
}
diff --git a/be/src/olap/full_compaction.cpp b/be/src/olap/full_compaction.cpp
index 7d3c497f2a2..c859bc70cbd 100644
--- a/be/src/olap/full_compaction.cpp
+++ b/be/src/olap/full_compaction.cpp
@@ -178,6 +178,8 @@ Status FullCompaction::modify_rowsets() {
DBUG_EXECUTE_IF("FullCompaction.modify_rowsets.sleep", { sleep(5); })
tablet()->save_meta();
}
+
+ _tablet->prefill_dbm_agg_cache_after_compaction(_output_rowset);
return Status::OK();
}
diff --git a/be/src/olap/lru_cache.cpp b/be/src/olap/lru_cache.cpp
index 673a13b5392..0b46a2f1494 100644
--- a/be/src/olap/lru_cache.cpp
+++ b/be/src/olap/lru_cache.cpp
@@ -618,6 +618,16 @@ PrunedInfo LRUCache::prune_if(CachePrunePredicate pred,
bool lazy_mode) {
return {pruned_count, pruned_size};
}
+void LRUCache::for_each_entry(const std::function<void(const LRUHandle*)>&
visitor) {
+ std::lock_guard l(_mutex);
+ for (LRUHandle* p = _lru_normal.next; p != &_lru_normal; p = p->next) {
+ visitor(p);
+ }
+ for (LRUHandle* p = _lru_durable.next; p != &_lru_durable; p = p->next) {
+ visitor(p);
+ }
+}
+
void LRUCache::set_cache_value_time_extractor(CacheValueTimeExtractor
cache_value_time_extractor) {
_cache_value_time_extractor = cache_value_time_extractor;
}
@@ -764,6 +774,12 @@ PrunedInfo ShardedLRUCache::prune_if(CachePrunePredicate
pred, bool lazy_mode) {
return pruned_info;
}
+void ShardedLRUCache::for_each_entry(const std::function<void(const
LRUHandle*)>& visitor) {
+ for (int s = 0; s < _num_shards; s++) {
+ _shards[s]->for_each_entry(visitor);
+ }
+}
+
int64_t ShardedLRUCache::get_usage() {
size_t total_usage = 0;
for (int i = 0; i < _num_shards; i++) {
diff --git a/be/src/olap/lru_cache.h b/be/src/olap/lru_cache.h
index 0b7da8754ff..9f4c95a4b9c 100644
--- a/be/src/olap/lru_cache.h
+++ b/be/src/olap/lru_cache.h
@@ -206,6 +206,8 @@ public:
// may hold lock for a long time to execute predicate.
virtual PrunedInfo prune_if(CachePrunePredicate pred, bool lazy_mode =
false) { return {0, 0}; }
+ virtual void for_each_entry(const std::function<void(const LRUHandle*)>&
visitor) = 0;
+
virtual int64_t get_usage() = 0;
virtual PrunedInfo set_capacity(size_t capacity) = 0;
@@ -328,6 +330,7 @@ public:
void erase(const CacheKey& key, uint32_t hash);
PrunedInfo prune();
PrunedInfo prune_if(CachePrunePredicate pred, bool lazy_mode = false);
+ void for_each_entry(const std::function<void(const LRUHandle*)>& visitor);
void set_cache_value_time_extractor(CacheValueTimeExtractor
cache_value_time_extractor);
void set_cache_value_check_timestamp(bool cache_value_check_timestamp);
@@ -396,6 +399,7 @@ public:
virtual uint64_t new_id() override;
PrunedInfo prune() override;
PrunedInfo prune_if(CachePrunePredicate pred, bool lazy_mode = false)
override;
+ void for_each_entry(const std::function<void(const LRUHandle*)>& visitor)
override;
int64_t get_usage() override;
size_t get_element_count() override;
PrunedInfo set_capacity(size_t capacity) override;
@@ -459,6 +463,7 @@ public:
PrunedInfo prune_if(CachePrunePredicate pred, bool lazy_mode = false)
override {
return {0, 0};
};
+ void for_each_entry(const std::function<void(const LRUHandle*)>& visitor)
override {}
int64_t get_usage() override { return 0; };
PrunedInfo set_capacity(size_t capacity) override { return {0, 0}; };
size_t get_capacity() override { return 0; };
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index 0a246482788..da1aae33780 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -17,6 +17,7 @@
#include "olap/tablet_meta.h"
+#include <bvar/bvar.h>
#include <gen_cpp/Descriptors_types.h>
#include <gen_cpp/FrontendService_types.h>
#include <gen_cpp/Types_types.h>
@@ -43,6 +44,7 @@
#include "io/fs/local_file_system.h"
#include "olap/data_dir.h"
#include "olap/file_header.h"
+#include "olap/lru_cache.h"
#include "olap/olap_common.h"
#include "olap/olap_define.h"
#include "olap/rowset/rowset.h"
@@ -66,6 +68,22 @@ namespace doris {
#include "common/compile_check_begin.h"
using namespace ErrorCode;
+bvar::Adder<uint64_t> g_contains_agg_with_cache_if_eligible_total(
+ "g_contains_agg_with_cache_if_eligible_total");
+bvar::Adder<uint64_t> g_contains_agg_with_cache_if_eligible_partial_hit(
+ "g_contains_agg_with_cache_if_eligible_partial_hit");
+bvar::Adder<uint64_t> g_contains_agg_with_cache_if_eligible_full_hit(
+ "g_contains_agg_with_cache_if_eligible_full_hit");
+bvar::Window<bvar::Adder<uint64_t>>
g_contains_agg_with_cache_if_eligible_total_minute(
+ "g_contains_agg_with_cache_if_eligible_total_1m",
+ &g_contains_agg_with_cache_if_eligible_total, 60);
+bvar::Window<bvar::Adder<uint64_t>>
g_contains_agg_with_cache_if_eligible_partial_hit_minute(
+ "g_contains_agg_with_cache_if_eligible_partial_hit_1m",
+ &g_contains_agg_with_cache_if_eligible_partial_hit, 60);
+bvar::Window<bvar::Adder<uint64_t>>
g_contains_agg_with_cache_if_eligible_full_hit_minute(
+ "g_contains_agg_with_cache_if_eligible_full_hit_1m",
+ &g_contains_agg_with_cache_if_eligible_full_hit, 60);
+
TabletMetaSharedPtr TabletMeta::create(
const TCreateTabletReq& request, const TabletUid& tablet_uid, uint64_t
shard_id,
uint32_t next_unique_id,
@@ -722,6 +740,7 @@ void TabletMeta::init_from_pb(const TabletMetaPB&
tablet_meta_pb) {
if (tablet_meta_pb.has_enable_unique_key_merge_on_write()) {
_enable_unique_key_merge_on_write =
tablet_meta_pb.enable_unique_key_merge_on_write();
+ _delete_bitmap->set_tablet_id(_tablet_id);
}
// init _rs_metas
@@ -1147,6 +1166,39 @@ bool operator!=(const TabletMeta& a, const TabletMeta&
b) {
return !(a == b);
}
+// We cannot just copy the underlying memory to construct a string
+// due to equivalent objects may have different padding bytes.
+// Reading padding bytes is undefined behavior, neither copy nor
+// placement new will help simplify the code.
+// Refer to C11 standards §6.2.6.1/6 and §6.7.9/21 for more info.
+static std::string agg_cache_key(int64_t tablet_id, const
DeleteBitmap::BitmapKey& bmk) {
+ std::string ret(sizeof(tablet_id) + sizeof(bmk), '\0');
+ *reinterpret_cast<int64_t*>(ret.data()) = tablet_id;
+ auto t = reinterpret_cast<DeleteBitmap::BitmapKey*>(ret.data() +
sizeof(tablet_id));
+ std::get<RowsetId>(*t).version = std::get<RowsetId>(bmk).version;
+ std::get<RowsetId>(*t).hi = std::get<RowsetId>(bmk).hi;
+ std::get<RowsetId>(*t).mi = std::get<RowsetId>(bmk).mi;
+ std::get<RowsetId>(*t).lo = std::get<RowsetId>(bmk).lo;
+ std::get<1>(*t) = std::get<1>(bmk);
+ std::get<2>(*t) = std::get<2>(bmk);
+ return ret;
+}
+
+// decode cache key info from a agg_cache_key
+static void decode_agg_cache_key(const std::string& key_str, int64_t&
tablet_id,
+ DeleteBitmap::BitmapKey& bmk) {
+ const char* ptr = key_str.data();
+ tablet_id = *reinterpret_cast<const int64_t*>(ptr);
+ ptr += sizeof(tablet_id);
+ auto* t =
reinterpret_cast<DeleteBitmap::BitmapKey*>(const_cast<char*>(ptr));
+ std::get<RowsetId>(bmk).version = std::get<RowsetId>(*t).version;
+ std::get<RowsetId>(bmk).hi = std::get<RowsetId>(*t).hi;
+ std::get<RowsetId>(bmk).mi = std::get<RowsetId>(*t).mi;
+ std::get<RowsetId>(bmk).lo = std::get<RowsetId>(*t).lo;
+ std::get<1>(bmk) = std::get<1>(*t);
+ std::get<2>(bmk) = std::get<2>(*t);
+}
+
DeleteBitmapAggCache::DeleteBitmapAggCache(size_t capacity)
: LRUCachePolicy(CachePolicy::CacheType::DELETE_BITMAP_AGG_CACHE,
capacity,
LRUCacheType::SIZE,
config::delete_bitmap_agg_cache_stale_sweep_time_sec,
@@ -1160,6 +1212,22 @@ DeleteBitmapAggCache*
DeleteBitmapAggCache::create_instance(size_t capacity) {
return new DeleteBitmapAggCache(capacity);
}
+DeleteBitmap DeleteBitmapAggCache::snapshot(int64_t tablet_id) {
+ DeleteBitmap ret(tablet_id);
+ auto collector = [&](const LRUHandle* handle) {
+ auto key = handle->key().to_string();
+ int64_t key_tablet_id;
+ DeleteBitmap::BitmapKey bmk;
+ decode_agg_cache_key(key, key_tablet_id, bmk);
+ if (key_tablet_id == tablet_id) {
+ const auto& dbm =
reinterpret_cast<DeleteBitmapAggCache::Value*>(handle->value)->bitmap;
+ ret.set(bmk, dbm);
+ }
+ };
+ DeleteBitmapAggCache::instance()->for_each_entry(collector);
+ return ret;
+}
+
DeleteBitmap::DeleteBitmap(int64_t tablet_id) : _tablet_id(tablet_id) {}
DeleteBitmap::DeleteBitmap(const DeleteBitmap& o) {
@@ -1318,9 +1386,53 @@ uint64_t DeleteBitmap::get_size() const {
return charge;
}
-bool DeleteBitmap::contains_agg_without_cache(const BitmapKey& bmk, uint32_t
row_id) const {
+bool DeleteBitmap::contains_agg_with_cache_if_eligible(const BitmapKey& bmk,
+ uint32_t row_id) const {
+ g_contains_agg_with_cache_if_eligible_total << 1;
+ int64_t start_version {0};
+ if (config::enable_mow_get_agg_by_cache) {
+ auto deleter = [&](Cache::Handle* handle) {
+ DeleteBitmapAggCache::instance()->release(handle);
+ };
+ std::unique_ptr<Cache::Handle, decltype(deleter)> dbm_handle(nullptr,
deleter);
+ int64_t cached_version = 0;
+ // 1. try to lookup the desired key directly
+
dbm_handle.reset(DeleteBitmapAggCache::instance()->lookup(agg_cache_key(_tablet_id,
bmk)));
+ if (dbm_handle != nullptr) {
+ cached_version = std::get<2>(bmk);
+ } else {
+ // 2. if not found, try to lookup with cached version
+ cached_version = _get_rowset_cache_version(bmk);
+ if (cached_version > 0) {
+ if (cached_version > std::get<2>(bmk)) {
+ cached_version = 0;
+ } else {
+
dbm_handle.reset(DeleteBitmapAggCache::instance()->lookup(agg_cache_key(
+ _tablet_id, {std::get<0>(bmk), std::get<1>(bmk),
cached_version})));
+ }
+ }
+ }
+ if (dbm_handle != nullptr) {
+ const auto& cached_dbm =
+ reinterpret_cast<DeleteBitmapAggCache::Value*>(
+
DeleteBitmapAggCache::instance()->value(dbm_handle.get()))
+ ->bitmap;
+ if (cached_version == std::get<2>(bmk)) {
+ g_contains_agg_with_cache_if_eligible_full_hit << 1;
+ } else {
+ g_contains_agg_with_cache_if_eligible_partial_hit << 1;
+ }
+ if (cached_dbm.contains(row_id)) {
+ return true;
+ }
+ if (cached_version == std::get<2>(bmk)) {
+ return false;
+ }
+ start_version = cached_version + 1;
+ }
+ }
+ DeleteBitmap::BitmapKey start {std::get<0>(bmk), std::get<1>(bmk),
start_version};
std::shared_lock l(lock);
- DeleteBitmap::BitmapKey start {std::get<0>(bmk), std::get<1>(bmk), 0};
for (auto it = delete_bitmap.lower_bound(start); it !=
delete_bitmap.end(); ++it) {
auto& [k, bm] = *it;
if (std::get<0>(k) != std::get<0>(bmk) || std::get<1>(k) !=
std::get<1>(bmk) ||
@@ -1479,22 +1591,12 @@ DeleteBitmap::Version
DeleteBitmap::_get_rowset_cache_version(const BitmapKey& b
return 0;
}
-// We cannot just copy the underlying memory to construct a string
-// due to equivalent objects may have different padding bytes.
-// Reading padding bytes is undefined behavior, neither copy nor
-// placement new will help simplify the code.
-// Refer to C11 standards §6.2.6.1/6 and §6.7.9/21 for more info.
-static std::string agg_cache_key(int64_t tablet_id, const
DeleteBitmap::BitmapKey& bmk) {
- std::string ret(sizeof(tablet_id) + sizeof(bmk), '\0');
- *reinterpret_cast<int64_t*>(ret.data()) = tablet_id;
- auto t = reinterpret_cast<DeleteBitmap::BitmapKey*>(ret.data() +
sizeof(tablet_id));
- std::get<RowsetId>(*t).version = std::get<RowsetId>(bmk).version;
- std::get<RowsetId>(*t).hi = std::get<RowsetId>(bmk).hi;
- std::get<RowsetId>(*t).mi = std::get<RowsetId>(bmk).mi;
- std::get<RowsetId>(*t).lo = std::get<RowsetId>(bmk).lo;
- std::get<1>(*t) = std::get<1>(bmk);
- std::get<2>(*t) = std::get<2>(bmk);
- return ret;
+DeleteBitmap DeleteBitmap::agg_cache_snapshot() {
+ return DeleteBitmapAggCache::instance()->snapshot(_tablet_id);
+}
+
+void DeleteBitmap::set_tablet_id(int64_t tablet_id) {
+ _tablet_id = tablet_id;
}
std::shared_ptr<roaring::Roaring> DeleteBitmap::get_agg(const BitmapKey& bmk)
const {
diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h
index ef57add9d01..68f6d323bbc 100644
--- a/be/src/olap/tablet_meta.h
+++ b/be/src/olap/tablet_meta.h
@@ -380,6 +380,8 @@ public:
static DeleteBitmapAggCache* create_instance(size_t capacity);
+ DeleteBitmap snapshot(int64_t tablet_id);
+
class Value : public LRUCacheValueBase {
public:
roaring::Roaring bitmap;
@@ -564,7 +566,7 @@ public:
*/
bool contains_agg(const BitmapKey& bitmap, uint32_t row_id) const;
- bool contains_agg_without_cache(const BitmapKey& bmk, uint32_t row_id)
const;
+ bool contains_agg_with_cache_if_eligible(const BitmapKey& bmk, uint32_t
row_id) const;
/**
* Gets aggregated delete_bitmap on rowset_id and version, the same effect:
* `select sum(roaring::Roaring) where RowsetId=rowset_id and
SegmentId=seg_id and Version <= version`
@@ -599,6 +601,10 @@ public:
*/
DeleteBitmap diffset(const std::set<BitmapKey>& key_set) const;
+ DeleteBitmap agg_cache_snapshot();
+
+ void set_tablet_id(int64_t tablet_id);
+
private:
DeleteBitmap::Version _get_rowset_cache_version(const BitmapKey& bmk)
const;
diff --git a/be/src/runtime/memory/lru_cache_policy.h
b/be/src/runtime/memory/lru_cache_policy.h
index 63b2abe2118..c06a6a03982 100644
--- a/be/src/runtime/memory/lru_cache_policy.h
+++ b/be/src/runtime/memory/lru_cache_policy.h
@@ -131,6 +131,10 @@ public:
return _cache->insert(key, value, charge, priority);
}
+ void for_each_entry(const std::function<void(const LRUHandle*)>& visitor) {
+ _cache->for_each_entry(visitor);
+ }
+
Cache::Handle* lookup(const CacheKey& key) { return _cache->lookup(key); }
void release(Cache::Handle* handle) { _cache->release(handle); }
diff --git a/be/src/service/http_service.cpp b/be/src/service/http_service.cpp
index 429da49dd35..3fae307b7ad 100644
--- a/be/src/service/http_service.cpp
+++ b/be/src/service/http_service.cpp
@@ -397,6 +397,11 @@ void HttpService::register_local_handler(StorageEngine&
engine) {
TPrivilegeHier::GLOBAL,
TPrivilegeType::ADMIN));
_ev_http_server->register_handler(HttpMethod::GET,
"/api/delete_bitmap/count_local",
count_delete_bitmap_action);
+ DeleteBitmapAction* count_agg_cache_delete_bitmap_action =
+ _pool.add(new
DeleteBitmapAction(DeleteBitmapActionType::COUNT_AGG_CACHE, _env, engine,
+ TPrivilegeHier::GLOBAL,
TPrivilegeType::ADMIN));
+ _ev_http_server->register_handler(HttpMethod::GET,
"/api/delete_bitmap/count_agg_cache",
+ count_agg_cache_delete_bitmap_action);
CheckTabletSegmentAction* check_tablet_segment_action = _pool.add(new
CheckTabletSegmentAction(
_env, engine, TPrivilegeHier::GLOBAL, TPrivilegeType::ADMIN));
@@ -460,6 +465,11 @@ void
HttpService::register_cloud_handler(CloudStorageEngine& engine) {
TPrivilegeHier::GLOBAL,
TPrivilegeType::ADMIN));
_ev_http_server->register_handler(HttpMethod::GET,
"/api/delete_bitmap/count_ms",
count_ms_delete_bitmap_action);
+ DeleteBitmapAction* count_agg_cache_delete_bitmap_action =
+ _pool.add(new
DeleteBitmapAction(DeleteBitmapActionType::COUNT_AGG_CACHE, _env, engine,
+ TPrivilegeHier::GLOBAL,
TPrivilegeType::ADMIN));
+ _ev_http_server->register_handler(HttpMethod::GET,
"/api/delete_bitmap/count_agg_cache",
+ count_agg_cache_delete_bitmap_action);
#ifdef ENABLE_INJECTION_POINT
InjectionPointAction* injection_point_action = _pool.add(new
InjectionPointAction);
_ev_http_server->register_handler(HttpMethod::GET,
"/api/injection_point/{op}",
diff --git a/regression-test/pipeline/cloud_p0/conf/be_custom.conf
b/regression-test/pipeline/cloud_p0/conf/be_custom.conf
index 024c7eb24e3..946f05cbb19 100644
--- a/regression-test/pipeline/cloud_p0/conf/be_custom.conf
+++ b/regression-test/pipeline/cloud_p0/conf/be_custom.conf
@@ -46,3 +46,5 @@
sys_log_verbose_modules=query_context,runtime_query_statistics_mgr
# So feature has bug, so by default is false, only open it in pipeline to
observe
enable_parquet_page_index=true
+
+enable_prefill_all_dbm_agg_cache_after_compaction=true
diff --git a/regression-test/pipeline/cloud_p1/conf/be_custom.conf
b/regression-test/pipeline/cloud_p1/conf/be_custom.conf
index 92b32480863..307cf47a201 100644
--- a/regression-test/pipeline/cloud_p1/conf/be_custom.conf
+++ b/regression-test/pipeline/cloud_p1/conf/be_custom.conf
@@ -34,3 +34,4 @@ crash_in_memory_tracker_inaccurate = true
enable_table_size_correctness_check=true
enable_write_index_searcher_cache=true
large_cumu_compaction_task_min_thread_num=3
+enable_prefill_all_dbm_agg_cache_after_compaction=true
diff --git a/regression-test/pipeline/p0/conf/be.conf
b/regression-test/pipeline/p0/conf/be.conf
index 0aecdd140d1..09dddd3c2d0 100644
--- a/regression-test/pipeline/p0/conf/be.conf
+++ b/regression-test/pipeline/p0/conf/be.conf
@@ -85,3 +85,5 @@ enable_graceful_exit_check=true
enable_fetch_rowsets_from_peer_replicas = true
enable_write_index_searcher_cache=true
+
+enable_prefill_all_dbm_agg_cache_after_compaction=true
diff --git a/regression-test/pipeline/p1/conf/be.conf
b/regression-test/pipeline/p1/conf/be.conf
index 9e82774c0df..b57c3abca74 100644
--- a/regression-test/pipeline/p1/conf/be.conf
+++ b/regression-test/pipeline/p1/conf/be.conf
@@ -70,3 +70,5 @@ large_cumu_compaction_task_min_thread_num=3
# This feature has bug, so by default is false, only open it in pipeline to
observe
enable_parquet_page_index=true
enable_graceful_exit_check=true
+
+enable_prefill_all_dbm_agg_cache_after_compaction=true
diff --git
a/regression-test/suites/metrics_p0/test_delete_bitmap_metrics.groovy
b/regression-test/suites/metrics_p0/test_delete_bitmap_metrics.groovy
index 0f090eb14a9..11bbaad712e 100644
--- a/regression-test/suites/metrics_p0/test_delete_bitmap_metrics.groovy
+++ b/regression-test/suites/metrics_p0/test_delete_bitmap_metrics.groovy
@@ -102,6 +102,27 @@ suite("test_delete_bitmap_metrics", "p0") {
return deleteBitmapStatus
}
+ def getAggCacheDeleteBitmapStatus = { be_host, be_http_port, tablet_id,
boolean verbose=false ->
+ boolean running = true
+ StringBuilder sb = new StringBuilder();
+ sb.append("curl -X GET http://${be_host}:${be_http_port}";)
+ sb.append("/api/delete_bitmap/count_agg_cache?tablet_id=")
+ sb.append(tablet_id)
+ if (verbose) {
+ sb.append("&verbose=true")
+ }
+
+ String command = sb.toString()
+ logger.info(command)
+ def process = command.execute()
+ def code = process.waitFor()
+ def out = process.getText()
+ logger.info("Get agg cache delete bitmap count status: =" + code + ",
out=" + out)
+ assertEquals(code, 0)
+ def deleteBitmapStatus = parseJson(out.trim())
+ return deleteBitmapStatus
+ }
+
String[][] backends = sql """ show backends """
assertTrue(backends.size() > 0)
String backendId;
@@ -190,6 +211,16 @@ suite("test_delete_bitmap_metrics", "p0") {
assertTrue(ms_delete_bitmap_count == 7)
assertTrue(ms_delete_bitmap_cardinality == 7)
}
+
+ def status =
getAggCacheDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id],
backendId_to_backendHttpPort[trigger_backend_id], tablet_id)
+ logger.info("agg cache status: ${status}")
+ assert status.delete_bitmap_count == 8
+ assert status.cardinality == 7
+ assert status.size > 0
+
+ status =
getAggCacheDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id],
backendId_to_backendHttpPort[trigger_backend_id], tablet_id, true)
+ logger.info("agg cache verbose status: ${status}")
+
def tablet_delete_bitmap_count = 0;
def base_rowset_delete_bitmap_count = 0;
int retry_time = 0;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
