This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new afaab7ff98c branch-2.1-pick: [Opt](log) Add and fix logs for delete
bitmaps (#50060)
afaab7ff98c is described below
commit afaab7ff98c5583e77e296d0d0df51200e365ba4
Author: bobhan1 <[email protected]>
AuthorDate: Wed Apr 16 16:45:11 2025 +0800
branch-2.1-pick: [Opt](log) Add and fix logs for delete bitmaps (#50060)
pick https://github.com/apache/doris/pull/48523,
https://github.com/apache/doris/pull/48930
---
be/src/olap/data_dir.cpp | 39 ++++++++++++++++++----
be/src/olap/rowset/beta_rowset_writer.cpp | 5 ++-
be/src/olap/rowset_builder.cpp | 10 +++---
be/src/olap/tablet.cpp | 8 ++---
be/src/olap/tablet_meta.cpp | 11 ++++++
be/src/olap/tablet_meta.h | 2 ++
.../unique/test_unique_auto_inc_concurrent.groovy | 1 -
7 files changed, 57 insertions(+), 19 deletions(-)
diff --git a/be/src/olap/data_dir.cpp b/be/src/olap/data_dir.cpp
index 13238a6dc1e..cab61925ec4 100644
--- a/be/src/olap/data_dir.cpp
+++ b/be/src/olap/data_dir.cpp
@@ -422,12 +422,15 @@ Status DataDir::load() {
dir_rowset_metas.push_back(rowset_meta);
return true;
};
+ MonotonicStopWatch rs_timer;
+ rs_timer.start();
Status load_rowset_status =
RowsetMetaManager::traverse_rowset_metas(_meta, load_rowset_func);
-
+ rs_timer.stop();
if (!load_rowset_status) {
LOG(WARNING) << "errors when load rowset meta from meta env, skip this
data dir:" << _path;
} else {
- LOG(INFO) << "load rowset from meta finished, data dir: " << _path;
+ LOG(INFO) << "load rowset from meta finished, cost: "
+ << rs_timer.elapsed_time_milliseconds() << " ms, data dir: "
<< _path;
}
// load tablet
@@ -464,8 +467,11 @@ Status DataDir::load() {
}
return true;
};
+ MonotonicStopWatch tablet_timer;
+ tablet_timer.start();
Status load_tablet_status = TabletMetaManager::traverse_headers(_meta,
load_tablet_func);
- if (failed_tablet_ids.size() != 0) {
+ tablet_timer.stop();
+ if (!failed_tablet_ids.empty()) {
LOG(WARNING) << "load tablets from header failed"
<< ", loaded tablet: " << tablet_ids.size()
<< ", error tablet: " << failed_tablet_ids.size() << ",
path: " << _path;
@@ -480,7 +486,9 @@ Status DataDir::load() {
} else {
LOG(INFO) << "load tablet from meta finished"
<< ", loaded tablet: " << tablet_ids.size()
- << ", error tablet: " << failed_tablet_ids.size() << ",
path: " << _path;
+ << ", error tablet: " << failed_tablet_ids.size()
+ << ", cost: " << tablet_timer.elapsed_time_milliseconds()
+ << " ms, path: " << _path;
}
for (int64_t tablet_id : tablet_ids) {
@@ -504,8 +512,13 @@ Status DataDir::load() {
pending_publish_info_pb.transaction_id(), true);
return true;
};
+ MonotonicStopWatch pending_publish_timer;
+ pending_publish_timer.start();
RETURN_IF_ERROR(
TabletMetaManager::traverse_pending_publish(_meta,
load_pending_publish_info_func));
+ pending_publish_timer.stop();
+ LOG(INFO) << "load pending publish task from meta finished, cost: "
+ << pending_publish_timer.elapsed_time_milliseconds() << " ms,
data dir: " << _path;
int64_t rowset_partition_id_eq_0_num = 0;
for (auto rowset_meta : dir_rowset_metas) {
@@ -612,7 +625,11 @@ Status DataDir::load() {
}
}
- auto load_delete_bitmap_func = [this](int64_t tablet_id, int64_t version,
const string& val) {
+ int64_t dbm_cnt {0};
+ int64_t unknown_dbm_cnt {0};
+
+ auto load_delete_bitmap_func = [this, &dbm_cnt, &unknown_dbm_cnt](
+ int64_t tablet_id, int64_t version,
const string& val) {
TabletSharedPtr tablet = _tablet_manager->get_tablet(tablet_id);
if (!tablet) {
return true;
@@ -635,8 +652,10 @@ Status DataDir::load() {
rst_id.init(delete_bitmap_pb.rowset_ids(i));
// only process the rowset in _rs_metas
if (rowset_ids.find(rst_id) == rowset_ids.end()) {
+ ++unknown_dbm_cnt;
continue;
}
+ ++dbm_cnt;
auto seg_id = delete_bitmap_pb.segment_ids(i);
auto iter =
tablet->tablet_meta()->delete_bitmap().delete_bitmap.find(
{rst_id, seg_id, version});
@@ -650,14 +669,22 @@ Status DataDir::load() {
}
return true;
};
+ MonotonicStopWatch dbm_timer;
+ dbm_timer.start();
RETURN_IF_ERROR(TabletMetaManager::traverse_delete_bitmap(_meta,
load_delete_bitmap_func));
+ dbm_timer.stop();
+
+ LOG(INFO) << "load delete bitmap from meta finished, cost: "
+ << dbm_timer.elapsed_time_milliseconds() << " ms, data dir: " <<
_path;
// At startup, we only count these invalid rowset, but do not actually
delete it.
// The actual delete operation is in
StorageEngine::_clean_unused_rowset_metas,
// which is cleaned up uniformly by the background cleanup thread.
LOG(INFO) << "finish to load tablets from " << _path
<< ", total rowset meta: " << dir_rowset_metas.size()
- << ", invalid rowset num: " << invalid_rowset_counter;
+ << ", invalid rowset num: " << invalid_rowset_counter
+ << ", visible/stale rowsets' delete bitmap count: " << dbm_cnt
+ << ", invalid rowsets' delete bitmap count: " << unknown_dbm_cnt;
return Status::OK();
}
diff --git a/be/src/olap/rowset/beta_rowset_writer.cpp
b/be/src/olap/rowset/beta_rowset_writer.cpp
index e2e0255d9a1..f64afad9b22 100644
--- a/be/src/olap/rowset/beta_rowset_writer.cpp
+++ b/be/src/olap/rowset/beta_rowset_writer.cpp
@@ -192,7 +192,10 @@ Status BetaRowsetWriter::_generate_delete_bitmap(int32_t
segment_id) {
LOG(INFO) << "[Memtable Flush] construct delete bitmap tablet: " <<
_context.tablet->tablet_id()
<< ", rowset_ids: " << _context.mow_context->rowset_ids.size()
<< ", cur max_version: " << _context.mow_context->max_version
- << ", transaction_id: " << _context.mow_context->txn_id
+ << ", transaction_id: " << _context.mow_context->txn_id << ",
delete_bitmap_count: "
+ <<
_context.tablet->tablet_meta()->delete_bitmap().get_delete_bitmap_count()
+ << ", delete_bitmap_cardinality: "
+ << _context.tablet->tablet_meta()->delete_bitmap().cardinality()
<< ", cost: " << watch.get_elapse_time_us() << "(us), total
rows: " << total_rows;
return Status::OK();
}
diff --git a/be/src/olap/rowset_builder.cpp b/be/src/olap/rowset_builder.cpp
index ddb55775965..cc56e2f78a7 100644
--- a/be/src/olap/rowset_builder.cpp
+++ b/be/src/olap/rowset_builder.cpp
@@ -280,13 +280,13 @@ Status RowsetBuilder::submit_calc_delete_bitmap_task() {
// we print it's summarize logs here before commit.
LOG(INFO) << fmt::format(
"partial update calc delete bitmap summary before commit:
tablet({}), txn_id({}), "
- "rowset_ids({}), cur max_version({}), bitmap num({}), num rows
updated({}), num "
- "rows new added({}), num rows deleted({}), total rows({})",
+ "rowset_ids({}), cur max_version({}), bitmap num({}),
bitmap_cardinality({}), num "
+ "rows updated({}), num rows new added({}), num rows
deleted({}), total rows({})",
tablet()->tablet_id(), _req.txn_id, _rowset_ids.size(),
rowset_writer()->context().mow_context->max_version,
- _delete_bitmap->delete_bitmap.size(),
rowset_writer()->num_rows_updated(),
- rowset_writer()->num_rows_new_added(),
rowset_writer()->num_rows_deleted(),
- rowset_writer()->num_rows());
+ _delete_bitmap->get_delete_bitmap_count(),
_delete_bitmap->cardinality(),
+ rowset_writer()->num_rows_updated(),
rowset_writer()->num_rows_new_added(),
+ rowset_writer()->num_rows_deleted(),
rowset_writer()->num_rows());
return Status::OK();
}
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index 0bb318bdabb..674006d99df 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -3179,11 +3179,6 @@ Status
Tablet::calc_segment_delete_bitmap(RowsetSharedPtr rowset,
rsid_to_rowset, &block));
RETURN_IF_ERROR(sort_block(block, ordered_block));
RETURN_IF_ERROR(rowset_writer->flush_single_block(&ordered_block));
- if (new_generated_rows != rowset_writer->num_rows()) {
- LOG(WARNING) << "partial update correctness warning: conflict new
generated rows ("
- << new_generated_rows << ") not equal to the new
flushed rows ("
- << rowset_writer->num_rows() << "), tablet: " <<
tablet_id();
- }
LOG(INFO) << "calc segment delete bitmap for partial update, tablet: "
<< tablet_id()
<< " rowset: " << rowset_id << " seg_id: " << seg->id()
<< " dummy_version: " << end_version + 1 << " rows: " <<
seg->num_rows()
@@ -3724,7 +3719,8 @@ Status Tablet::update_delete_bitmap(TabletTxnInfo*
txn_info, int64_t txn_id) {
segments.begin(), segments.end(), 0,
[](size_t sum, const segment_v2::SegmentSharedPtr& s) { return sum
+= s->num_rows(); });
LOG(INFO) << "[Publish] construct delete bitmap tablet: " << tablet_id()
- << ", rowset_ids to add: " << rowset_ids_to_add.size()
+ << ", rowset_ids to add: "
+ << (specified_rowsets.size() + rowsets_skip_alignment.size())
<< ", rowset_ids to del: " << rowset_ids_to_del.size()
<< ", cur version: " << cur_version << ", transaction_id: " <<
txn_id << ","
<< ss.str() << " , total rows: " << total_rows;
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index bfeb9dbac84..b3360537e9a 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -1074,6 +1074,17 @@ uint64_t DeleteBitmap::cardinality() const {
return res;
}
+uint64_t DeleteBitmap::get_delete_bitmap_count() const {
+ std::shared_lock l(lock);
+ uint64_t count = 0;
+ for (auto it = delete_bitmap.begin(); it != delete_bitmap.end(); it++) {
+ if (std::get<1>(it->first) != DeleteBitmap::INVALID_SEGMENT_ID) {
+ count++;
+ }
+ }
+ return count;
+}
+
bool DeleteBitmap::contains_agg_without_cache(const BitmapKey& bmk, uint32_t
row_id) const {
std::shared_lock l(lock);
DeleteBitmap::BitmapKey start {std::get<0>(bmk), std::get<1>(bmk), 0};
diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h
index 77581b4f4ef..7767e6d2184 100644
--- a/be/src/olap/tablet_meta.h
+++ b/be/src/olap/tablet_meta.h
@@ -439,6 +439,8 @@ public:
*/
uint64_t cardinality() const;
+ uint64_t get_delete_bitmap_count() const;
+
/**
* Sets the bitmap of specific segment, it's may be insertion or
replacement
*
diff --git
a/regression-test/suites/data_model_p0/unique/test_unique_auto_inc_concurrent.groovy
b/regression-test/suites/data_model_p0/unique/test_unique_auto_inc_concurrent.groovy
index 4793fb52343..66a87707a6f 100644
---
a/regression-test/suites/data_model_p0/unique/test_unique_auto_inc_concurrent.groovy
+++
b/regression-test/suites/data_model_p0/unique/test_unique_auto_inc_concurrent.groovy
@@ -55,6 +55,5 @@ suite("test_unique_table_auto_inc_concurrent") {
run_test(15, 100000, 1)
run_test(5, 30000, 10)
- sql "drop table if exists ${table1};"
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
