This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new d71e2d91eeb branch-4.1: [improvement](be) Add queue time for delete
bitmap tasks (#65523)
d71e2d91eeb is described below
commit d71e2d91eeb2edb69dfb4f9d835a7b3ee0144e8c
Author: bobhan1 <[email protected]>
AuthorDate: Wed Jul 15 12:11:44 2026 +0800
branch-4.1: [improvement](be) Add queue time for delete bitmap tasks
(#65523)
### What problem does this PR solve?
Issue Number: None
Related PR: #65248
Problem Summary: Backport #65248 to branch-4.1. MOW memtable flush
calculates delete bitmap asynchronously, but its log did not show how
long the task waited in the executor queue. The load delete-bitmap
executor also used the same thread-pool metric name as the normal
delete-bitmap executor. This change records queue time for flush,
segment-level, between-segment, and cloud tablet-level async
delete-bitmap tasks, and gives the load delete-bitmap executor a
distinct thread-pool name. The branch-4.1 conflict in
`beta_rowset_writer.cpp` was resolved while preserving the
branch-specific debug point.
### Release note
None
### Check List (For Author)
- Test
- [x] Manual test (add detailed scripts or steps below)
- `./build.sh --be -j100`
- `build-support/check-format.sh`
- Behavior changed:
- [x] No.
- Does this need documentation?
- [x] No.
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label
---
.../cloud/cloud_engine_calc_delete_bitmap_task.cpp | 25 ++++++++++++----------
.../cloud/cloud_engine_calc_delete_bitmap_task.h | 2 +-
be/src/cloud/cloud_storage_engine.cpp | 4 +++-
.../storage/delete/calc_delete_bitmap_executor.cpp | 12 +++++++----
.../storage/delete/calc_delete_bitmap_executor.h | 3 ++-
be/src/storage/rowset/beta_rowset_writer.cpp | 7 ++++--
be/src/storage/storage_engine.cpp | 4 +++-
be/src/storage/tablet/base_tablet.cpp | 20 +++++++++--------
be/src/storage/tablet/base_tablet.h | 5 +++--
9 files changed, 50 insertions(+), 32 deletions(-)
diff --git a/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp
b/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp
index 66510d01d18..ffa7529d172 100644
--- a/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp
+++ b/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp
@@ -94,15 +94,18 @@ Status CloudEngineCalcDeleteBitmapTask::execute() {
if (has_tablet_states) {
tablet_calc_delete_bitmap_ptr->set_tablet_state(partition.tablet_states[i]);
}
- auto submit_st = token->submit_func([tablet_id,
tablet_calc_delete_bitmap_ptr, this]() {
- auto st = tablet_calc_delete_bitmap_ptr->handle();
- if (st.ok()) {
- add_succ_tablet_id(tablet_id);
- } else {
- LOG(WARNING) << "handle calc delete bitmap fail, st=" <<
st.to_string();
- add_error_tablet_id(tablet_id, st);
- }
- });
+ const auto submit_time_us = MonotonicMicros();
+ auto submit_st = token->submit_func(
+ [tablet_id, tablet_calc_delete_bitmap_ptr, this,
submit_time_us]() {
+ const auto queue_time_us = MonotonicMicros() -
submit_time_us;
+ auto st =
tablet_calc_delete_bitmap_ptr->handle(queue_time_us);
+ if (st.ok()) {
+ add_succ_tablet_id(tablet_id);
+ } else {
+ LOG(WARNING) << "handle calc delete bitmap fail,
st=" << st.to_string();
+ add_error_tablet_id(tablet_id, st);
+ }
+ });
VLOG_DEBUG << "submit TabletCalcDeleteBitmapTask for tablet=" <<
tablet_id;
if (!submit_st.ok()) {
_res = submit_st;
@@ -144,7 +147,7 @@ void
CloudTabletCalcDeleteBitmapTask::set_tablet_state(int64_t tablet_state) {
_ms_tablet_state = tablet_state;
}
-Status CloudTabletCalcDeleteBitmapTask::handle() const {
+Status CloudTabletCalcDeleteBitmapTask::handle(int64_t queue_time_us) const {
VLOG_DEBUG << "start calculate delete bitmap on tablet " << _tablet_id
<< ", txn_id=" << _transaction_id;
SCOPED_ATTACH_TASK(_mem_tracker);
@@ -285,7 +288,7 @@ Status CloudTabletCalcDeleteBitmapTask::handle() const {
auto total_update_delete_bitmap_time_us = MonotonicMicros() - t3;
LOG(INFO) << "finish calculate delete bitmap on tablet"
<< ", table_id=" << tablet->table_id() << ", transaction_id=" <<
_transaction_id
- << ", tablet_id=" << tablet->tablet_id()
+ << ", tablet_id=" << tablet->tablet_id() << ", queue_time_us="
<< queue_time_us
<< ", get_tablet_time_us=" << get_tablet_time_us
<< ", sync_rowset_time_us=" << sync_rowset_time_us
<< ", total_update_delete_bitmap_time_us=" <<
total_update_delete_bitmap_time_us
diff --git a/be/src/cloud/cloud_engine_calc_delete_bitmap_task.h
b/be/src/cloud/cloud_engine_calc_delete_bitmap_task.h
index 5ea11e8d4c3..04b085b5ff6 100644
--- a/be/src/cloud/cloud_engine_calc_delete_bitmap_task.h
+++ b/be/src/cloud/cloud_engine_calc_delete_bitmap_task.h
@@ -43,7 +43,7 @@ public:
int64_t ms_cumulative_point);
void set_tablet_state(int64_t tablet_state);
- Status handle() const;
+ Status handle(int64_t queue_time_us) const;
private:
Status _handle_rowset(std::shared_ptr<CloudTablet> tablet, int64_t version,
diff --git a/be/src/cloud/cloud_storage_engine.cpp
b/be/src/cloud/cloud_storage_engine.cpp
index 6ba92830f99..07ea98920e6 100644
--- a/be/src/cloud/cloud_storage_engine.cpp
+++ b/be/src/cloud/cloud_storage_engine.cpp
@@ -207,10 +207,12 @@ Status CloudStorageEngine::open() {
cast_set<int32_t>(io::FileCacheFactory::instance()->get_cache_instance_size()));
_calc_delete_bitmap_executor =
std::make_unique<CalcDeleteBitmapExecutor>();
- _calc_delete_bitmap_executor->init(config::calc_delete_bitmap_max_thread);
+ _calc_delete_bitmap_executor->init("TabletCalcDeleteBitmapThreadPool",
+ config::calc_delete_bitmap_max_thread);
_calc_delete_bitmap_executor_for_load =
std::make_unique<CalcDeleteBitmapExecutor>();
_calc_delete_bitmap_executor_for_load->init(
+ "LoadCalcDeleteBitmapThreadPool",
config::calc_delete_bitmap_for_load_max_thread > 0
? config::calc_delete_bitmap_for_load_max_thread
: std::max(1, CpuInfo::num_cores() / 2));
diff --git a/be/src/storage/delete/calc_delete_bitmap_executor.cpp
b/be/src/storage/delete/calc_delete_bitmap_executor.cpp
index 1867dc54a9e..af1b4bfb23a 100644
--- a/be/src/storage/delete/calc_delete_bitmap_executor.cpp
+++ b/be/src/storage/delete/calc_delete_bitmap_executor.cpp
@@ -41,11 +41,13 @@ Status CalcDeleteBitmapToken::submit(BaseTabletSPtr tablet,
RowsetSharedPtr cur_
_resource_ctx = thread_context()->resource_ctx();
}
+ const auto submit_time_us = MonotonicMicros();
return _thread_token->submit_func([=, this]() {
+ const auto queue_time_us = MonotonicMicros() - submit_time_us;
SCOPED_ATTACH_TASK(_resource_ctx);
auto st = tablet->calc_segment_delete_bitmap(cur_rowset, cur_segment,
target_rowsets,
delete_bitmap,
end_version, rowset_writer,
- tablet_delete_bitmap);
+ tablet_delete_bitmap,
queue_time_us);
if (!st.ok()) {
LOG(WARNING) << "failed to calc segment delete bitmap, tablet_id: "
<< tablet->tablet_id() << " rowset: " <<
cur_rowset->rowset_id()
@@ -68,10 +70,12 @@ Status CalcDeleteBitmapToken::submit(BaseTabletSPtr tablet,
TabletSchemaSPtr sch
RETURN_IF_ERROR(_status);
_resource_ctx = thread_context()->resource_ctx();
}
+ const auto submit_time_us = MonotonicMicros();
return _thread_token->submit_func([=, this]() {
+ const auto queue_time_us = MonotonicMicros() - submit_time_us;
SCOPED_ATTACH_TASK(_resource_ctx);
auto st = tablet->calc_delete_bitmap_between_segments(schema,
rowset_id, segments,
- delete_bitmap);
+ delete_bitmap,
queue_time_us);
if (!st.ok()) {
LOG(WARNING) << "failed to calc delete bitmap between segments,
tablet_id: "
<< tablet->tablet_id() << " rowset: " << rowset_id
@@ -90,8 +94,8 @@ Status CalcDeleteBitmapToken::wait() {
return _status;
}
-void CalcDeleteBitmapExecutor::init(int max_threads) {
- static_cast<void>(ThreadPoolBuilder("TabletCalcDeleteBitmapThreadPool")
+void CalcDeleteBitmapExecutor::init(const std::string& name, int max_threads) {
+ static_cast<void>(ThreadPoolBuilder(name)
.set_min_threads(1)
.set_max_threads(max_threads)
.build(&_thread_pool));
diff --git a/be/src/storage/delete/calc_delete_bitmap_executor.h
b/be/src/storage/delete/calc_delete_bitmap_executor.h
index be97505e038..4f0f0fd2c7f 100644
--- a/be/src/storage/delete/calc_delete_bitmap_executor.h
+++ b/be/src/storage/delete/calc_delete_bitmap_executor.h
@@ -23,6 +23,7 @@
#include <iosfwd>
#include <memory>
#include <shared_mutex>
+#include <string>
#include <utility>
#include <vector>
@@ -105,7 +106,7 @@ public:
~CalcDeleteBitmapExecutor() { _thread_pool->shutdown(); }
// init should be called after storage engine is opened,
- void init(int max_threads);
+ void init(const std::string& name, int max_threads);
std::unique_ptr<CalcDeleteBitmapToken> create_token();
diff --git a/be/src/storage/rowset/beta_rowset_writer.cpp
b/be/src/storage/rowset/beta_rowset_writer.cpp
index 614a8de9071..708e7f9bfe8 100644
--- a/be/src/storage/rowset/beta_rowset_writer.cpp
+++ b/be/src/storage/rowset/beta_rowset_writer.cpp
@@ -400,9 +400,11 @@ Status
BaseBetaRowsetWriter::_generate_delete_bitmap(int32_t segment_id) {
// Submit the entire delete bitmap calculation process to thread pool for
async execution
// This avoids blocking memtable flush thread while waiting for file
upload to complete
// The process includes: file_writer->close(), _build_tmp, load_segments,
and calc_delete_bitmap
+ const auto submit_time_us = MonotonicMicros();
return _calc_delete_bitmap_token->submit_func([this, segment_id,
- specified_rowsets =
std::move(
-
specified_rowsets)]() -> Status {
+ specified_rowsets =
std::move(specified_rowsets),
+ submit_time_us]() -> Status
{
+ const auto queue_time_us = MonotonicMicros() - submit_time_us;
Status st = Status::OK();
// Step 1: Close file_writer (must be done before load_segments)
auto* file_writer = _seg_files.get(segment_id);
@@ -465,6 +467,7 @@ Status
BaseBetaRowsetWriter::_generate_delete_bitmap(int32_t segment_id) {
<<
_context.mow_context->delete_bitmap->get_delete_bitmap_count()
<< ", delete_bitmap_cardinality: "
<< _context.mow_context->delete_bitmap->cardinality()
+ << ", queue_time_us: " << queue_time_us
<< ", cost: " << watch.get_elapse_time_us() << "(us), total
rows: " << total_rows;
return Status::OK();
});
diff --git a/be/src/storage/storage_engine.cpp
b/be/src/storage/storage_engine.cpp
index 212a2833fb6..d3891ff0bcf 100644
--- a/be/src/storage/storage_engine.cpp
+++ b/be/src/storage/storage_engine.cpp
@@ -326,10 +326,12 @@ Status StorageEngine::_open() {
_memtable_flush_executor->init(_disk_num);
_calc_delete_bitmap_executor =
std::make_unique<CalcDeleteBitmapExecutor>();
- _calc_delete_bitmap_executor->init(config::calc_delete_bitmap_max_thread);
+ _calc_delete_bitmap_executor->init("TabletCalcDeleteBitmapThreadPool",
+ config::calc_delete_bitmap_max_thread);
_calc_delete_bitmap_executor_for_load =
std::make_unique<CalcDeleteBitmapExecutor>();
_calc_delete_bitmap_executor_for_load->init(
+ "LoadCalcDeleteBitmapThreadPool",
config::calc_delete_bitmap_for_load_max_thread > 0
? config::calc_delete_bitmap_for_load_max_thread
: std::max(1, CpuInfo::num_cores() / 2));
diff --git a/be/src/storage/tablet/base_tablet.cpp
b/be/src/storage/tablet/base_tablet.cpp
index a9945012f5f..628d3a6aa81 100644
--- a/be/src/storage/tablet/base_tablet.cpp
+++ b/be/src/storage/tablet/base_tablet.cpp
@@ -446,7 +446,8 @@ void
BaseTablet::generate_tablet_meta_copy_unlocked(TabletMeta& new_tablet_meta,
Status BaseTablet::calc_delete_bitmap_between_segments(
TabletSchemaSPtr schema, RowsetId rowset_id,
- const std::vector<segment_v2::SegmentSharedPtr>& segments,
DeleteBitmapPtr delete_bitmap) {
+ const std::vector<segment_v2::SegmentSharedPtr>& segments,
DeleteBitmapPtr delete_bitmap,
+ int64_t queue_time_us) {
size_t const num_segments = segments.size();
if (num_segments < 2) {
return Status::OK();
@@ -474,9 +475,9 @@ Status BaseTablet::calc_delete_bitmap_between_segments(
LOG(INFO) << fmt::format(
"construct delete bitmap between segments, "
"tablet: {}, rowset: {}, number of segments: {}, bitmap count: {},
bitmap cardinality: "
- "{}, cost {} (us)",
+ "{}, queue_time_us: {}, cost {} (us)",
tablet_id(), rowset_id.to_string(), num_segments,
- delete_bitmap->get_delete_bitmap_count(),
delete_bitmap->cardinality(),
+ delete_bitmap->get_delete_bitmap_count(),
delete_bitmap->cardinality(), queue_time_us,
watch.get_elapse_time_us());
return Status::OK();
}
@@ -663,7 +664,8 @@ Status
BaseTablet::calc_segment_delete_bitmap(RowsetSharedPtr rowset,
const
std::vector<RowsetSharedPtr>& specified_rowsets,
DeleteBitmapPtr delete_bitmap,
int64_t end_version,
RowsetWriter* rowset_writer,
- DeleteBitmapPtr
tablet_delete_bitmap) {
+ DeleteBitmapPtr
tablet_delete_bitmap,
+ int64_t queue_time_us) {
OlapStopWatch watch;
auto rowset_id = rowset->rowset_id();
Version dummy_version(end_version + 1, end_version + 1);
@@ -906,7 +908,7 @@ Status
BaseTablet::calc_segment_delete_bitmap(RowsetSharedPtr rowset,
RETURN_IF_ERROR(sort_block(block, ordered_block));
RETURN_IF_ERROR(rowset_writer->flush_single_block(&ordered_block));
auto cost_us = watch.get_elapse_time_us();
- if (config::enable_mow_verbose_log || cost_us > 10 * 1000) {
+ if (config::enable_mow_verbose_log || cost_us > 10 * 1000 ||
queue_time_us > 10 * 1000) {
LOG(INFO) << "calc segment delete bitmap for "
<< partial_update_info->partial_update_mode_str()
<< ", tablet: " << tablet_id() << " rowset: " <<
rowset_id
@@ -915,19 +917,19 @@ Status
BaseTablet::calc_segment_delete_bitmap(RowsetSharedPtr rowset,
<< " new generated rows: " << new_generated_rows
<< " bitmap num: " <<
delete_bitmap->get_delete_bitmap_count()
<< " bitmap cardinality: " <<
delete_bitmap->cardinality()
- << " cost: " << cost_us << "(us)";
+ << " queue_time_us: " << queue_time_us << ", cost: " <<
cost_us << "(us)";
}
return Status::OK();
}
auto cost_us = watch.get_elapse_time_us();
- if (config::enable_mow_verbose_log || cost_us > 10 * 1000) {
+ if (config::enable_mow_verbose_log || cost_us > 10 * 1000 || queue_time_us
> 10 * 1000) {
LOG(INFO) << "calc segment delete bitmap, tablet: " << tablet_id()
<< " rowset: " << rowset_id << " seg_id: " << seg->id()
<< " dummy_version: " << end_version + 1 << " rows: " <<
seg->num_rows()
<< " conflict rows: " << conflict_rows
<< " bitmap num: " <<
delete_bitmap->get_delete_bitmap_count()
- << " bitmap cardinality: " << delete_bitmap->cardinality()
<< " cost: " << cost_us
- << "(us)";
+ << " bitmap cardinality: " << delete_bitmap->cardinality()
+ << " queue_time_us: " << queue_time_us << ", cost: " <<
cost_us << "(us)";
}
return Status::OK();
}
diff --git a/be/src/storage/tablet/base_tablet.h
b/be/src/storage/tablet/base_tablet.h
index 7e75e320aca..4d73879600d 100644
--- a/be/src/storage/tablet/base_tablet.h
+++ b/be/src/storage/tablet/base_tablet.h
@@ -190,12 +190,13 @@ public:
const std::vector<RowsetSharedPtr>&
specified_rowsets,
DeleteBitmapPtr delete_bitmap, int64_t
end_version,
RowsetWriter* rowset_writer,
- DeleteBitmapPtr tablet_delete_bitmap =
nullptr);
+ DeleteBitmapPtr tablet_delete_bitmap =
nullptr,
+ int64_t queue_time_us = 0);
Status calc_delete_bitmap_between_segments(
TabletSchemaSPtr schema, RowsetId rowset_id,
const std::vector<segment_v2::SegmentSharedPtr>& segments,
- DeleteBitmapPtr delete_bitmap);
+ DeleteBitmapPtr delete_bitmap, int64_t queue_time_us = 0);
static Status commit_phase_update_delete_bitmap(
const BaseTabletSPtr& tablet, const RowsetSharedPtr& rowset,
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
