dataroaring commented on code in PR #53908:
URL: https://github.com/apache/doris/pull/53908#discussion_r2285507975
##########
be/src/cloud/cloud_meta_mgr.cpp:
##########
@@ -1384,15 +1690,108 @@ Status CloudMetaMgr::update_delete_bitmap(const
CloudTablet& tablet, int64_t loc
if (next_visible_version > 0) {
req.set_next_visible_version(next_visible_version);
}
- for (auto& [key, bitmap] : delete_bitmap->delete_bitmap) {
- req.add_rowset_ids(std::get<0>(key).to_string());
- req.add_segment_ids(std::get<1>(key));
- req.add_versions(std::get<2>(key));
- // To save space, convert array and bitmap containers to run containers
- bitmap.runOptimize();
- std::string bitmap_data(bitmap.getSizeInBytes(), '\0');
- bitmap.write(bitmap_data.data());
- *(req.add_segment_delete_bitmaps()) = std::move(bitmap_data);
+
+ bool write_v1 =
+ config::delete_bitmap_store_version == 1 ||
config::delete_bitmap_store_version == 3;
+ bool write_v2 = config::delete_bitmap_store_version >= 2;
+ // write v1 kvs
+ if (write_v1) {
+ for (auto& [key, bitmap] : delete_bitmap->delete_bitmap) {
+ req.add_rowset_ids(std::get<0>(key).to_string());
+ req.add_segment_ids(std::get<1>(key));
+ req.add_versions(std::get<2>(key));
+ // To save space, convert array and bitmap containers to run
containers
+ bitmap.runOptimize();
+ std::string bitmap_data(bitmap.getSizeInBytes(), '\0');
+ bitmap.write(bitmap_data.data());
+ *(req.add_segment_delete_bitmaps()) = std::move(bitmap_data);
+ }
+ }
+
+ // write v2 kvs
+ if (write_v2) {
+ // TODO skip update if delete bitmap is empty
+ // it's ok for compaction and schema change, but for load, ms check
partition version
+ auto add_delete_bitmap = [](DeleteBitmapPB& delete_bitmap_pb,
+ const DeleteBitmap::BitmapKey& key,
roaring::Roaring& bitmap) {
+ delete_bitmap_pb.add_rowset_ids(std::get<0>(key).to_string());
+ delete_bitmap_pb.add_segment_ids(std::get<1>(key));
+ delete_bitmap_pb.add_versions(std::get<2>(key));
+ // To save space, convert array and bitmap containers to run
containers
+ bitmap.runOptimize();
+ std::string bitmap_data(bitmap.getSizeInBytes(), '\0');
+ bitmap.write(bitmap_data.data());
+ *(delete_bitmap_pb.add_segment_delete_bitmaps()) =
std::move(bitmap_data);
+ };
+ auto handle_rowset_delete_bitmap = [&](std::string& rowset_id,
+ DeleteBitmapPB&
delete_bitmap_pb) {
+ std::stringstream ss;
+ DeleteBitmapStoragePB delete_bitmap_storage;
+ for (int i = 0; i < delete_bitmap_pb.rowset_ids_size(); i++) {
+ ss << "{rid=" << delete_bitmap_pb.rowset_ids(i)
+ << ", sid=" << delete_bitmap_pb.segment_ids(i)
+ << ", ver=" << delete_bitmap_pb.versions(i) << "}, ";
+ }
+ if (config::delete_bitmap_max_bytes_store_in_fdb >= 0 &&
+ delete_bitmap_pb.ByteSizeLong() >
config::delete_bitmap_max_bytes_store_in_fdb) {
+ DeleteBitmapFileWriter file_writer(tablet.tablet_id(),
rowset_id, storage_resource);
+ RETURN_IF_ERROR(file_writer.init());
+ RETURN_IF_ERROR(file_writer.write(delete_bitmap_pb));
+ RETURN_IF_ERROR(file_writer.close());
+ delete_bitmap_pb.Clear();
+ delete_bitmap_storage.set_store_in_fdb(false);
+ } else {
+ delete_bitmap_storage.set_store_in_fdb(true);
+ *(delete_bitmap_storage.mutable_delete_bitmap()) =
std::move(delete_bitmap_pb);
+ }
+ LOG(INFO) << "handle one rowset delete bitmap for tablet_id: " <<
tablet.tablet_id()
+ << ", rowset_id: " << rowset_id
+ << ", delete_bitmap num: " <<
delete_bitmap_pb.rowset_ids_size()
+ << ", store_in_fdb=" <<
delete_bitmap_storage.store_in_fdb()
+ << ", size: " << delete_bitmap_pb.ByteSizeLong() << ",
keys=[" << ss.str()
+ << "]";
+ req.add_v2_rowset_ids(rowset_id);
+ *(req.add_delete_bitmap_storages()) =
std::move(delete_bitmap_storage);
+ return Status::OK();
+ };
+ req.set_store_version(2);
+ LOG(INFO) << "update delete bitmap for tablet_id: " <<
tablet.tablet_id()
+ << ", rowset_id: " << rowset_id
+ << ", delete_bitmap num: " <<
delete_bitmap_v2->delete_bitmap.size()
+ << ", lock_id=" << lock_id << ", initiator=" << initiator;
+ if (rowset_id.empty()) {
+ std::string pre_rowset_id = "";
+ DeleteBitmapPB delete_bitmap_pb;
+ for (auto it = delete_bitmap_v2->delete_bitmap.begin();
+ it != delete_bitmap_v2->delete_bitmap.end(); ++it) {
+ auto& key = it->first;
+ auto& bitmap = it->second;
+ auto cur_rowset_id = std::get<0>(key).to_string();
+ if (cur_rowset_id != pre_rowset_id) {
+ if (!pre_rowset_id.empty() &&
delete_bitmap_pb.rowset_ids_size() > 0) {
+ RETURN_IF_ERROR(
+ handle_rowset_delete_bitmap(pre_rowset_id,
delete_bitmap_pb));
+ }
+ pre_rowset_id = cur_rowset_id;
+ DCHECK_EQ(delete_bitmap_pb.rowset_ids_size(), 0);
+ DCHECK_EQ(delete_bitmap_pb.segment_ids_size(), 0);
+ DCHECK_EQ(delete_bitmap_pb.versions_size(), 0);
+ DCHECK_EQ(delete_bitmap_pb.segment_delete_bitmaps_size(),
0);
+ }
+ add_delete_bitmap(delete_bitmap_pb, key, bitmap);
+ if (it == std::prev(delete_bitmap_v2->delete_bitmap.end()) &&
+ delete_bitmap_pb.rowset_ids_size() > 0) {
+ RETURN_IF_ERROR(handle_rowset_delete_bitmap(cur_rowset_id,
delete_bitmap_pb));
+ }
+ }
+ } else {
+ DeleteBitmapPB delete_bitmap_pb;
+ for (auto& [key, bitmap] : delete_bitmap_v2->delete_bitmap) {
+ add_delete_bitmap(delete_bitmap_pb, key, bitmap);
Review Comment:
victim_indexed_bitmap —— 方式 1（记录别人对自己的删除）
deleter_indexed_bitmap —— 方式 2（记录自己删除别人）
这个应该比 v1 v2 要清晰.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
