gavinchou commented on code in PR #55712:
URL: https://github.com/apache/doris/pull/55712#discussion_r2324758756
##########
be/src/cloud/cloud_meta_mgr.cpp:
##########
@@ -822,6 +822,106 @@ bool
CloudMetaMgr::sync_tablet_delete_bitmap_by_cache(CloudTablet* tablet, int64
return true;
}
+Status CloudMetaMgr::_get_delete_bitmap_from_ms(GetDeleteBitmapRequest& req,
+ GetDeleteBitmapResponse& res) {
+ VLOG_DEBUG << "send GetDeleteBitmapRequest: " << req.ShortDebugString();
+
+ auto st = retry_rpc("get delete bitmap", req, &res,
&MetaService_Stub::get_delete_bitmap);
+ if (st.code() == ErrorCode::THRIFT_RPC_ERROR) {
+ return st;
+ }
+
+ if (res.status().code() == MetaServiceCode::TABLET_NOT_FOUND) {
+ return Status::NotFound("failed to get delete bitmap: {}",
res.status().msg());
+ }
+ // The delete bitmap of stale rowsets will be removed when commit
compaction job,
+ // then delete bitmap of stale rowsets cannot be obtained. But the rowsets
obtained
+ // by sync_tablet_rowsets may include these stale rowsets. When this case
happend, the
+ // error code of ROWSETS_EXPIRED will be returned, we need to retry sync
rowsets again.
+ //
+ // Be query thread meta-service Be compaction thread
+ // | | |
+ // | get rowset | |
+ // |--------------------------->| |
+ // | return get rowset | |
+ // |<---------------------------| |
+ // | | commit job |
+ // | |<------------------------|
+ // | | return commit job |
+ // | |------------------------>|
+ // | get delete bitmap | |
+ // |--------------------------->| |
+ // | return get delete bitmap | |
+ // |<---------------------------| |
+ // | | |
+ if (res.status().code() == MetaServiceCode::ROWSETS_EXPIRED) {
+ return Status::Error<ErrorCode::ROWSETS_EXPIRED, false>("failed to get
delete bitmap: {}",
+
res.status().msg());
+ }
+ if (res.status().code() != MetaServiceCode::OK) {
+ return Status::Error<ErrorCode::INTERNAL_ERROR, false>("failed to get
delete bitmap: {}",
+
res.status().msg());
+ }
+ return Status::OK();
+}
+
+Status
CloudMetaMgr::_get_delete_bitmap_from_ms_by_batch(GetDeleteBitmapRequest& req,
+
GetDeleteBitmapResponse& res,
+ int64_t
bytes_threadhold) {
+ std::unordered_set<std::string> finished_rowset_ids {};
+ int count = 0;
+ do {
+ GetDeleteBitmapRequest cur_req;
+ GetDeleteBitmapResponse cur_res;
+
+ cur_req.set_cloud_unique_id(config::cloud_unique_id);
+ cur_req.set_tablet_id(req.tablet_id());
+ cur_req.set_base_compaction_cnt(req.base_compaction_cnt());
+ cur_req.set_cumulative_compaction_cnt(req.cumulative_compaction_cnt());
+ cur_req.set_cumulative_point(req.cumulative_point());
+ *(cur_req.mutable_idx()) = req.idx();
+ cur_req.set_store_version(req.store_version());
+ if (bytes_threadhold > 0) {
+ cur_req.set_dbm_bytes_threshold(bytes_threadhold);
+ }
+ for (int i = 0; i < req.rowset_ids_size(); i++) {
+ if (!finished_rowset_ids.contains(req.rowset_ids(i))) {
+ cur_req.add_rowset_ids(req.rowset_ids(i));
+ cur_req.add_begin_versions(req.begin_versions(i));
+ cur_req.add_end_versions(req.end_versions(i));
+ }
+ }
+
+ RETURN_IF_ERROR(_get_delete_bitmap_from_ms(cur_req, cur_res));
+ ++count;
+
+ // v1 delete bitmap
+ res.mutable_rowset_ids()->MergeFrom(cur_res.rowset_ids());
+ res.mutable_segment_ids()->MergeFrom(cur_res.segment_ids());
+ res.mutable_versions()->MergeFrom(cur_res.versions());
+
res.mutable_segment_delete_bitmaps()->MergeFrom(cur_res.segment_delete_bitmaps());
+
+ // v2 delete bitmap
+ res.mutable_delta_rowset_ids()->MergeFrom(cur_res.delta_rowset_ids());
+
res.mutable_delete_bitmap_storages()->MergeFrom(cur_res.delete_bitmap_storages());
+
+ for (const auto& rowset_id : cur_res.returned_rowset_ids()) {
+ finished_rowset_ids.insert(rowset_id);
+ }
+
+ bool has_more = cur_res.has_has_more() && cur_res.has_more();
+ if (!has_more) {
+ break;
+ }
+ LOG_INFO("batch get delete bitmap, progress={}/{}",
finished_rowset_ids.size(),
Review Comment:
also print current bytes of deletebitmap fetched?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
