gavinchou commented on code in PR #41782:
URL: https://github.com/apache/doris/pull/41782#discussion_r1811090204
##########
cloud/src/meta-service/meta_service_tablet_stats.cpp:
##########
@@ -156,4 +163,517 @@ void internal_get_tablet_stats(MetaServiceCode& code,
std::string& msg, Transact
merge_tablet_stats(stats, detached_stats);
}
+MetaServiceResponseStatus fix_tablet_stats_parse_param(
+ std::shared_ptr<ResourceManager> resource_mgr, const std::string&
table_id_str,
+ const std::string& cloud_unique_id_str, int64_t& table_id,
std::string& instance_id) {
+ MetaServiceCode code = MetaServiceCode::OK;
+ std::string msg;
+ MetaServiceResponseStatus st;
+ st.set_code(MetaServiceCode::OK);
+
+ // parse params
+ try {
+ table_id = std::stoll(table_id_str);
+ } catch (...) {
+ st.set_code(MetaServiceCode::INVALID_ARGUMENT);
+ st.set_msg("Invalid table_id, table_id: " + table_id_str);
+ return st;
+ }
+
+ instance_id = get_instance_id(resource_mgr, cloud_unique_id_str);
+ if (instance_id.empty()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ msg = "empty instance_id";
+ LOG(INFO) << msg << ", cloud_unique_id=" << cloud_unique_id_str;
+ st.set_code(code);
+ st.set_msg(msg);
+ return st;
+ }
+ return st;
+}
+
+MetaServiceResponseStatus read_range_tablet_stats(std::shared_ptr<TxnKv>
txn_kv,
+ const std::string&
instance_id, int64_t table_id,
+ const std::string& begin_key,
+ const std::string& end_key,
+
std::unique_ptr<RangeGetIterator>& it) {
+ MetaServiceCode code = MetaServiceCode::OK;
+ std::string msg;
+ MetaServiceResponseStatus st;
+ st.set_code(MetaServiceCode::OK);
+ std::unique_ptr<Transaction> txn;
+
+ TxnErrorCode err = txn_kv->create_txn(&txn);
+ if (err != TxnErrorCode::TXN_OK) {
+ code = cast_as<ErrCategory::CREATE>(err);
+ msg = fmt::format("[fix tablet stat] failed to create txn");
+ st.set_code(code);
+ st.set_msg(msg);
+ return st;
+ }
+
+ err = txn->get(begin_key, end_key, &it, true);
+ if (err != TxnErrorCode::TXN_OK) {
+ st.set_code(cast_as<ErrCategory::READ>(err));
+ st.set_msg(fmt::format("failed to get tablet stats, err={}
instance_id={} table_id={} ",
+ err, instance_id, table_id));
+ return st;
+ }
+
+ err = txn->commit();
+ if (err != TxnErrorCode::TXN_OK) {
+ st.set_code(cast_as<ErrCategory::COMMIT>(err));
+ std::string msg = fmt::format("[fix tablet stats]: failed to commit
txn)");
+ st.set_msg(msg);
+ return st;
+ }
+ return st;
+}
+
+MetaServiceResponseStatus create_txn_if_committed(bool& committed,
std::shared_ptr<TxnKv> txn_kv,
+ const std::string& msg,
+
std::unique_ptr<Transaction>& txn) {
+ // read data and write data use one txn
+ // check data use one txn
+ MetaServiceCode code = MetaServiceCode::OK;
+ TxnErrorCode err;
+ MetaServiceResponseStatus st;
+ st.set_code(MetaServiceCode::OK);
+ if (committed) {
+ err = txn_kv->create_txn(&txn);
+ committed = false;
+ if (err != TxnErrorCode::TXN_OK) {
+ code = cast_as<ErrCategory::CREATE>(err);
+ st.set_code(code);
+ st.set_msg(msg);
+ return st;
+ }
+ }
+ return st;
+}
+
+MetaServiceResponseStatus read_and_accumulate_rowset_data(
+ const std::vector<std::tuple<std::variant<int64_t, std::string>, int,
int>>& out,
+ const std::string_view& v, int64_t tablet_cnt,
std::unique_ptr<Transaction>& txn,
+ const std::string& instance_id, int64_t& total_disk_size,
TabletStatsPB& tablet_stat) {
+ MetaServiceCode code = MetaServiceCode::OK;
+ MetaServiceResponseStatus st;
+ st.set_code(MetaServiceCode::OK);
+
+ auto tablet_id = std::get<int64_t>(std::get<0>(out[6]));
+ tablet_stat.ParseFromArray(v.data(), v.size());
+ LOG(INFO) << fmt::format(
+ "[Sub txn id {}, Tablet id {} fix tablet stats phase 2]: read
original "
+ "tabletPB, tabletPB info: {}",
+ tablet_cnt, tablet_id, tablet_stat.DebugString());
+
+ GetRowsetResponse resp;
+ std::string msg;
+ internal_get_rowset(txn.get(), 0, std::numeric_limits<int64_t>::max() - 1,
instance_id,
+ tablet_id, code, msg, &resp, true);
+ if (code != MetaServiceCode::OK) {
+ st.set_code(code);
+ st.set_msg(msg);
+ return st;
+ }
+ total_disk_size = 0;
+ for (const auto& rs_meta : resp.rowset_meta()) {
+ rs_meta.rowset_id();
+ total_disk_size += rs_meta.total_disk_size();
+ LOG(INFO) << fmt::format(
+ "[Sub txn id {}, Tablet id {} fix tablet stats phase 2]: read "
+ "rowsetsPB, total disk size: {}, rowsetPB info: {}",
+ tablet_cnt, tablet_id, total_disk_size, rs_meta.DebugString());
+ }
+ return st;
+}
+
+MetaServiceResponseStatus write_tablet_stats_back(
+ TabletStatsPB& tablet_stat, int64_t total_disk_size, const
std::string& instance_id,
+ std::unique_ptr<Transaction>& txn, int64_t tablet_cnt,
+ std::vector<std::shared_ptr<TabletStatsPB>>&
tablet_stat_shared_ptr_vec) {
+ MetaServiceResponseStatus st;
+ st.set_code(MetaServiceCode::OK);
+
+ tablet_stat.set_data_size(total_disk_size);
+ std::string tablet_stat_key;
+ std::string tablet_stat_value;
+ tablet_stat_key = stats_tablet_key(
+ {instance_id, tablet_stat.idx().table_id(),
tablet_stat.idx().index_id(),
+ tablet_stat.idx().partition_id(), tablet_stat.idx().tablet_id()});
+ if (!tablet_stat.SerializeToString(&tablet_stat_value)) {
+ st.set_code(MetaServiceCode::PROTOBUF_SERIALIZE_ERR);
+ st.set_msg("failed to serialize tablet stat");
+ return st;
+ }
+ txn->put(tablet_stat_key, tablet_stat_value);
+
tablet_stat_shared_ptr_vec.emplace_back(std::make_shared<TabletStatsPB>(tablet_stat));
+ LOG(INFO) << fmt::format(
+ "[Sub txn id{}, Tablet id {} fix tablet stats phase 3]: write new "
+ "tabletPB, tabletPB info: {}",
+ tablet_cnt, tablet_stat.idx().tablet_id(),
tablet_stat.DebugString());
+ return st;
+}
+
+void set_tablet_data_size_to_zero(TabletStatsPB tablet_stat,
std::unique_ptr<Transaction>& txn,
+ const std::string& instance_id, int64_t
tablet_cnt) {
+ // 0x01 "stats" ${instance_id} "tablet" ${table_id} ${index_id}
${partition_id} ${tablet_id} "data_size" -> int64
+ std::string tablet_stat_data_size_key;
+ stats_tablet_data_size_key(
+ {instance_id, tablet_stat.idx().table_id(),
tablet_stat.idx().index_id(),
+ tablet_stat.idx().partition_id(), tablet_stat.idx().tablet_id()},
+ &tablet_stat_data_size_key);
+ // set tablet stats data size = 0
+ int64_t tablet_stat_data_size = 0;
+ std::string tablet_stat_data_size_value(sizeof(tablet_stat_data_size),
'\0');
+ memcpy(tablet_stat_data_size_value.data(), &tablet_stat_data_size,
+ sizeof(tablet_stat_data_size));
+ txn->put(tablet_stat_data_size_key, tablet_stat_data_size_value);
+ LOG(INFO) << fmt::format(
+ "[Sub txn id {} Tablet id {} fix tablet stats phase 4]: set tablet
stats "
+ "data size = 0, data size : {}",
+ tablet_cnt, tablet_stat.idx().tablet_id(),
tablet_stat_data_size_value);
+}
+
+MetaServiceResponseStatus commit_batch(std::unique_ptr<Transaction>& txn,
bool& committed,
+ int64_t tablet_cnt) {
+ MetaServiceResponseStatus st;
+ st.set_code(MetaServiceCode::OK);
+ TxnErrorCode err;
+ if (tablet_cnt % 50 == 0) {
+ err = txn->commit();
+ committed = true;
+ if (err != TxnErrorCode::TXN_OK) {
+ st.set_code(cast_as<ErrCategory::COMMIT>(err));
+ std::string msg = fmt::format("failed to commit 50 sub txns, sub
txn id {}-{}",
+ tablet_cnt - 50, tablet_cnt);
+ st.set_msg(msg);
+ return st;
+ } else {
+ LOG(INFO) << fmt::format(
+ "[fix tablet stats phase ?]: success to commit 50 sub
txns, sub txn id "
+ "{}-{}",
+ tablet_cnt - 50, tablet_cnt);
+ }
+ }
+ return st;
+}
+
+MetaServiceResponseStatus commit_batch_final(std::unique_ptr<Transaction>&
txn, bool& committed,
+ int64_t tablet_cnt) {
+ MetaServiceResponseStatus st;
+ st.set_code(MetaServiceCode::OK);
+ TxnErrorCode err;
+ if (!committed) {
+ err = txn->commit();
+ committed = true;
+ if (err != TxnErrorCode::TXN_OK) {
+ st.set_code(cast_as<ErrCategory::COMMIT>(err));
+ std::string msg = fmt::format(
+ "[fix tablet stats final commit]: failed to commit remain
txns, final sub txn "
+ "id {}",
+ tablet_cnt);
+ st.set_msg(msg);
+ return st;
+ } else {
+ LOG(INFO) << fmt::format(
+ "[fix tablet stats final commit]: success to commit remain
sub txns, final sub "
+ "txn id {}",
+ tablet_cnt);
+ };
+ }
+ return st;
+}
+
+MetaServiceResponseStatus fix_tablet_stats_data(
+ std::shared_ptr<TxnKv> txn_kv, const std::string& instance_id, int64_t
table_id,
+ std::vector<std::shared_ptr<TabletStatsPB>>&
tablet_stat_shared_ptr_vec) {
+ // fix tablet stats code
+ std::string msg;
+ MetaServiceResponseStatus st;
+ st.set_code(MetaServiceCode::OK);
+
+ std::unique_ptr<Transaction> txn;
+ std::string key, val;
+ int64_t start = 0;
+ int64_t end = std::numeric_limits<int64_t>::max() - 1;
+ auto begin_key = stats_tablet_key({instance_id, table_id, start, start,
start});
+ auto end_key = stats_tablet_key({instance_id, table_id, end, end, end});
Review Comment:
```suggestion
auto begin_key = stats_tablet_key({instance_id, table_id, 0, 0, 0});
auto end_key = stats_tablet_key({instance_id, table_id+1, 0, 0, 0});
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
