dataroaring commented on code in PR #53908:
URL: https://github.com/apache/doris/pull/53908#discussion_r2282302102
##########
cloud/src/meta-service/meta_service.cpp:
##########
@@ -3393,147 +3564,33 @@ void
MetaServiceImpl::update_delete_bitmap(google::protobuf::RpcController* cont
<< " value_size: " << pending_val.size();
}
- // 5. Update delete bitmap for curent txn
- size_t current_key_count = 0;
- size_t current_value_count = 0;
- size_t total_key_count = 0;
- size_t total_value_count = 0;
- size_t total_txn_put_keys = 0;
- size_t total_txn_put_bytes = 0;
- size_t total_txn_size = 0;
- size_t total_txn_count = 0;
+ // 5. Update delete bitmap for current txn
+ UpdateDeleteBitmapTxnStats txn_stats;
std::set<std::string> non_exist_rowset_ids;
- for (size_t i = 0; i < request->rowset_ids_size(); ++i) {
- auto& key = delete_bitmap_keys.delete_bitmap_keys(i);
+ // write v1 kvs
+ for (size_t i = 0; i < delete_bitmap_keys_v1.size(); ++i) {
+ auto& key = delete_bitmap_keys_v1[i];
auto& val = request->segment_delete_bitmaps(i);
-
- // Split into multiple fdb transactions, because the size of one fdb
- // transaction can't exceed 10MB.
- if (txn->approximate_bytes() + key.size() * 3 + val.size() >
config::max_txn_commit_byte) {
- LOG(INFO) << "fdb txn size more than " <<
config::max_txn_commit_byte
- << ", current size: " << txn->approximate_bytes()
- << ", tablet_id: " << tablet_id << " lock_id=" <<
request->lock_id()
- << " initiator=" << request->initiator() << ", need to
commit";
- err = txn->commit();
- TEST_SYNC_POINT_CALLBACK("update_delete_bitmap:commit:err",
request->initiator(), i,
- &err);
- total_txn_put_keys += txn->num_put_keys();
- total_txn_put_bytes += txn->put_bytes();
- total_txn_size += txn->approximate_bytes();
- total_txn_count++;
- if (err != TxnErrorCode::TXN_OK) {
- code = cast_as<ErrCategory::COMMIT>(err);
- ss << "failed to update delete bitmap, err=" << err << "
tablet_id=" << tablet_id
- << " lock_id=" << request->lock_id() << " initiator=" <<
request->initiator()
- << " delete_bitmap_key=" << current_key_count
- << " delete_bitmap_value=" << current_value_count
- << " put_size=" << txn->put_bytes() << " num_put_keys=" <<
txn->num_put_keys()
- << " txn_size=" << txn->approximate_bytes();
- msg = ss.str();
- g_bvar_update_delete_bitmap_fail_counter << 1;
- return;
- }
- stats.get_bytes += txn->get_bytes();
- stats.put_bytes += txn->put_bytes();
- stats.del_bytes += txn->delete_bytes();
- stats.get_counter += txn->num_get_keys();
- stats.put_counter += txn->num_put_keys();
- stats.del_counter += txn->num_del_keys();
- current_key_count = 0;
- current_value_count = 0;
- TxnErrorCode err = txn_kv_->create_txn(&txn);
- if (err != TxnErrorCode::TXN_OK) {
- code = cast_as<ErrCategory::CREATE>(err);
- msg = "failed to init txn";
- return;
- }
- if (!without_lock) {
- std::string lock_key =
- meta_delete_bitmap_update_lock_key({instance_id,
table_id, -1});
- DeleteBitmapUpdateLockPB lock_info;
- if (!check_delete_bitmap_lock(code, msg, ss, txn, instance_id,
table_id,
- request->lock_id(),
request->initiator(), lock_key,
- lock_info, use_version, log)) {
- LOG(WARNING) << "failed to check delete bitmap lock,
table_id=" << table_id
- << " request lock_id=" << request->lock_id()
- << " request initiator=" <<
request->initiator() << " msg " << msg;
- return;
- }
- }
- }
-
-#ifdef ENABLE_INJECTION_POINT
- if (config::enable_update_delete_bitmap_kv_check) {
- if (!check_delete_bitmap_kv_exists(code, msg, txn, key,
instance_id, tablet_id,
- request->lock_id())) {
- return;
- }
- }
-#endif
-
- if (without_lock && request->has_pre_rowset_agg_end_version() &&
- request->pre_rowset_agg_end_version() > 0) {
- // check the rowset exists
- if (non_exist_rowset_ids.contains(request->rowset_ids(i))) {
- LOG(INFO) << "skip update delete bitmap, rowset_id=" <<
request->rowset_ids(i)
- << " version=" << request->pre_rowset_versions(i)
- << " tablet_id=" << tablet_id << " because the
rowset does not exist";
- continue;
- }
- auto rowset_key =
- meta_rowset_key({instance_id, tablet_id,
request->pre_rowset_versions(i)});
- std::string rowset_val;
- err = txn->get(rowset_key, &rowset_val);
- if (err != TxnErrorCode::TXN_OK && TxnErrorCode::TXN_KEY_NOT_FOUND
!= err) {
- ss << "failed to get rowset, instance_id=" << instance_id
- << " tablet_id=" << tablet_id << " version=" <<
request->pre_rowset_versions(i)
- << " err=" << err;
- msg = ss.str();
- code = cast_as<ErrCategory::READ>(err);
- return;
- }
- if (err == TxnErrorCode::TXN_KEY_NOT_FOUND) {
- non_exist_rowset_ids.emplace(request->rowset_ids(i));
- LOG(INFO) << "skip update delete bitmap, rowset_id=" <<
request->rowset_ids(i)
- << " version=" << request->pre_rowset_versions(i)
- << " tablet_id=" << tablet_id << " because the
rowset is not exist";
- continue;
- }
- doris::RowsetMetaCloudPB rs;
- if (!rs.ParseFromArray(rowset_val.data(), rowset_val.size())) {
- code = MetaServiceCode::PROTOBUF_PARSE_ERR;
- ss << "malformed rowset meta, unable to deserialize,
tablet_id=" << tablet_id
- << " key=" << hex(rowset_key);
- msg = ss.str();
- return;
- }
- if (rs.rowset_id_v2() != request->rowset_ids(i)) {
- LOG(INFO) << "skip update delete bitmap, rowset_id=" <<
request->rowset_ids(i)
- << " version=" << request->pre_rowset_versions(i)
- << " tablet_id=" << tablet_id << " because the
rowset is not exist";
- non_exist_rowset_ids.emplace(request->rowset_ids(i));
- continue;
- }
+ _write_delete_bitmap_kvs(code, msg, ss, txn_kv_, txn, use_version,
non_exist_rowset_ids, i,
+ log, stats, request, instance_id, key, val,
txn_stats);
+ if (code != MetaServiceCode::OK) return;
+ }
+ // write v2 kvs
+ for (size_t i = 0; i < delete_bitmap_keys_v2.size(); ++i) {
+ auto& key = delete_bitmap_keys_v2[i];
+ std::string val;
+ if (!request->delete_bitmap_storages(i).SerializeToString(&val)) {
Review Comment:
I can find where val is used.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
