zhannngchen commented on code in PR #40204:
URL: https://github.com/apache/doris/pull/40204#discussion_r1756816089
##########
be/src/cloud/cloud_cumulative_compaction.cpp:
##########
@@ -340,9 +341,125 @@ Status CloudCumulativeCompaction::modify_rowsets() {
stats.num_rows(),
stats.data_size());
}
}
+ if (_tablet->keys_type() == KeysType::UNIQUE_KEYS &&
+ _tablet->enable_unique_key_merge_on_write()) {
+ process_old_version_delete_bitmap();
+ }
return Status::OK();
}
+void CloudCumulativeCompaction::process_old_version_delete_bitmap() {
+ // agg previously rowset old version delete bitmap
+ std::vector<RowsetSharedPtr> pre_rowsets {};
+ std::vector<std::string> pre_rowset_ids {};
+ for (const auto& it : cloud_tablet()->rowset_map()) {
+ if (it.first.second < _input_rowsets.front()->start_version()) {
+ pre_rowsets.emplace_back(it.second);
+ pre_rowset_ids.emplace_back(it.second->rowset_id().to_string());
+ }
+ }
+ std::sort(pre_rowsets.begin(), pre_rowsets.end(), Rowset::comparator);
+ pre_rowsets.erase(pre_rowsets.begin());
+ if (!pre_rowsets.empty()) {
+ auto pre_max_version = _output_rowset->version().second;
+ DeleteBitmapPtr new_delete_bitmap =
+
std::make_shared<DeleteBitmap>(_tablet->tablet_meta()->tablet_id());
+ std::vector<std::tuple<int64_t, DeleteBitmap::BitmapKey,
DeleteBitmap::BitmapKey>>
+ to_remove_vec;
+ for (auto& rowset : pre_rowsets) {
+ for (uint32_t seg_id = 0; seg_id < rowset->num_segments();
++seg_id) {
+ rowset->rowset_id().to_string();
+ DeleteBitmap::BitmapKey start {rowset->rowset_id(), seg_id, 0};
+ DeleteBitmap::BitmapKey end {rowset->rowset_id(), seg_id,
pre_max_version};
+ DeleteBitmap::BitmapKey before_end {rowset->rowset_id(),
seg_id,
+ pre_max_version - 1};
+ auto d = _tablet->tablet_meta()->delete_bitmap().get_agg(
+ {rowset->rowset_id(), seg_id, pre_max_version});
+ to_remove_vec.emplace_back(
+ std::make_tuple(_tablet->tablet_id(), start,
before_end));
+ if (d->isEmpty()) {
+ continue;
+ }
+ new_delete_bitmap->set(end, *d);
+ }
+ }
+ Status get_st;
+ Status update_st;
+ Status remove_st;
+ do {
+ if (!new_delete_bitmap->empty()) {
+ // store agg delete bitmap
+ int64_t initiator = boost::hash_range(_uuid.begin(),
_uuid.end()) &
+ std::numeric_limits<int64_t>::max();
+
DBUG_EXECUTE_IF("CloudCumulativeCompaction.modify_rowsets.get_mow_lock_failed",
{
+ get_st = Status::InternalError(
+ "test fail to get_delete_bitmap_update_lock for "
+ "tablet_id {}",
+ cloud_tablet()->tablet_id());
+ });
+ if (get_st.ok()) {
+ get_st =
_engine.meta_mgr().get_delete_bitmap_update_lock(*cloud_tablet(), -1,
Review Comment:
as we discussed, the lock is unnecessary.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
