This is an automated email from the ASF dual-hosted git repository.
gavinchou pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new 074b86dce5f branch-4.0: [fix](be) Avoid rowset map race in backend
tablets scanner #65288 (#65301)
074b86dce5f is described below
commit 074b86dce5f4c33b953115bb7c8983c5508bfc08
Author: zclllyybb <[email protected]>
AuthorDate: Tue Jul 7 21:00:03 2026 +0800
branch-4.0: [fix](be) Avoid rowset map race in backend tablets scanner
#65288 (#65301)
Related PR: #65288
---
.../exec/schema_scanner/schema_tablets_scanner.cpp | 163 ++++++++++++++-------
1 file changed, 108 insertions(+), 55 deletions(-)
diff --git a/be/src/exec/schema_scanner/schema_tablets_scanner.cpp
b/be/src/exec/schema_scanner/schema_tablets_scanner.cpp
index 957ce370b8b..e9100e7c38b 100644
--- a/be/src/exec/schema_scanner/schema_tablets_scanner.cpp
+++ b/be/src/exec/schema_scanner/schema_tablets_scanner.cpp
@@ -25,8 +25,10 @@
#include <cstdint>
#include <memory>
#include <numeric>
+#include <shared_mutex>
#include <string>
#include <utility>
+#include <vector>
#include "cloud/cloud_storage_engine.h"
#include "cloud/cloud_tablet.h"
@@ -35,6 +37,7 @@
#include "common/status.h"
#include "exec/schema_scanner.h"
#include "exec/schema_scanner/schema_scanner_helper.h"
+#include "olap/rowset/rowset_fwd.h"
#include "olap/storage_engine.h"
#include "olap/tablet_fwd.h"
#include "olap/tablet_manager.h"
@@ -50,6 +53,30 @@ class Block;
#include "common/compile_check_begin.h"
+namespace {
+struct TabletInfoSnapshot {
+ int64_t tablet_id = 0;
+ int64_t replica_id = 0;
+ int64_t partition_id = 0;
+ std::string tablet_path;
+ int64_t tablet_local_size = 0;
+ int64_t tablet_remote_size = 0;
+ int64_t version_count = 0;
+ int64_t segment_count = 0;
+ int64_t num_columns = 0;
+ int64_t row_size = 0;
+ int32_t compaction_score = 0;
+ std::string compress_kind;
+ bool is_used = false;
+ bool is_alter_failed = false;
+ int64_t create_time = 0;
+ int64_t update_time = 0;
+ bool is_overlap = false;
+ std::vector<RowsetMetaSharedPtr> rowset_metas;
+ RowsetSharedPtr max_version_rowset;
+};
+} // namespace
+
std::vector<SchemaScanner::ColumnDesc> SchemaTabletsScanner::_s_tbls_columns =
{
// name, type, size, is_null
{"BE_ID", TYPE_BIGINT, sizeof(int64_t), true},
@@ -128,97 +155,123 @@ Status
SchemaTabletsScanner::_fill_block_impl(vectorized::Block* block) {
for (int i = 0; i < _tablets.size(); i++) {
BaseTabletSPtr tablet = _tablets[i];
+ TabletInfoSnapshot snapshot;
+ {
+ // Snapshot rowset maps under the tablet header lock because
compaction rewrites them
+ // under the same lock. Heavy aggregation and block filling run
after releasing it.
+ std::shared_lock rlock(tablet->get_header_lock());
+ const auto& tablet_meta = tablet->tablet_meta();
+ const auto& rs_metas = tablet_meta->all_rs_metas();
+
+ snapshot.tablet_id = tablet_meta->tablet_id();
+ snapshot.replica_id = tablet_meta->replica_id();
+ snapshot.partition_id = tablet_meta->partition_id();
+ snapshot.tablet_path = tablet->tablet_path();
+ snapshot.num_columns =
static_cast<int64_t>(tablet_meta->tablet_columns_num());
+ snapshot.row_size = static_cast<int64_t>(tablet->row_size());
+ snapshot.compress_kind =
CompressKind_Name(tablet->compress_kind());
+ snapshot.is_used = [&tablet]() {
+ if (config::is_cloud_mode()) {
+ return true;
+ }
+ return std::static_pointer_cast<Tablet>(tablet)->is_used();
+ }();
+ snapshot.is_alter_failed = tablet->is_alter_failed();
+ snapshot.create_time = tablet_meta->creation_time();
+ snapshot.rowset_metas.reserve(rs_metas.size());
+ for (const auto& [_, rs_meta] : rs_metas) {
+ snapshot.rowset_metas.emplace_back(rs_meta);
+ }
+ snapshot.max_version_rowset =
tablet->get_rowset_with_max_version();
+ }
+
+ snapshot.tablet_local_size = std::accumulate(
+ snapshot.rowset_metas.begin(), snapshot.rowset_metas.end(),
int64_t {0},
+ [](int64_t total_size, const auto& rs_meta) {
+ if (rs_meta->is_local()) {
+ total_size +=
static_cast<int64_t>(rs_meta->total_disk_size());
+ }
+ return total_size;
+ });
+ snapshot.tablet_remote_size = std::accumulate(
+ snapshot.rowset_metas.begin(), snapshot.rowset_metas.end(),
int64_t {0},
+ [](int64_t total_size, const auto& rs_meta) {
+ if (!rs_meta->is_local()) {
+ total_size +=
static_cast<int64_t>(rs_meta->total_disk_size());
+ }
+ return total_size;
+ });
+ snapshot.version_count =
static_cast<int64_t>(snapshot.rowset_metas.size());
+ snapshot.segment_count =
+ std::accumulate(snapshot.rowset_metas.begin(),
snapshot.rowset_metas.end(),
+ int64_t {0}, [](int64_t val, const auto&
rs_meta) {
+ return val +
static_cast<int64_t>(rs_meta->num_segments());
+ });
+ snapshot.compaction_score = static_cast<int32_t>(std::accumulate(
+ snapshot.rowset_metas.begin(), snapshot.rowset_metas.end(),
uint32_t {0},
+ [](uint32_t score, const auto& rs_meta) {
+ return score +
static_cast<uint32_t>(rs_meta->get_compaction_score());
+ }));
+ snapshot.update_time = snapshot.max_version_rowset == nullptr
+ ? 0
+ :
snapshot.max_version_rowset->newest_write_timestamp();
+ snapshot.is_overlap =
+ std::any_of(snapshot.rowset_metas.begin(),
snapshot.rowset_metas.end(),
+ [](const auto& rs_meta) { return
rs_meta->is_segments_overlapping(); });
+
// BE_ID
SchemaScannerHelper::insert_int64_value(0, _backend_id, block);
// TABLET_ID
- SchemaScannerHelper::insert_int64_value(1,
tablet->tablet_meta()->tablet_id(), block);
+ SchemaScannerHelper::insert_int64_value(1, snapshot.tablet_id, block);
// REPLICA_ID
- SchemaScannerHelper::insert_int64_value(2,
tablet->tablet_meta()->replica_id(), block);
+ SchemaScannerHelper::insert_int64_value(2, snapshot.replica_id, block);
// PARTITION_ID
- SchemaScannerHelper::insert_int64_value(3,
tablet->tablet_meta()->partition_id(), block);
+ SchemaScannerHelper::insert_int64_value(3, snapshot.partition_id,
block);
// TABLET_PATH
- SchemaScannerHelper::insert_string_value(4, tablet->tablet_path(),
block);
+ SchemaScannerHelper::insert_string_value(4, snapshot.tablet_path,
block);
// TABLET_LOCAL_SIZE
- SchemaScannerHelper::insert_int64_value(5,
tablet->tablet_meta()->tablet_local_size(),
- block);
+ SchemaScannerHelper::insert_int64_value(5, snapshot.tablet_local_size,
block);
// TABLET_REMOTE_SIZE
- SchemaScannerHelper::insert_int64_value(6,
tablet->tablet_meta()->tablet_remote_size(),
- block);
+ SchemaScannerHelper::insert_int64_value(6,
snapshot.tablet_remote_size, block);
// VERSION_COUNT
- SchemaScannerHelper::insert_int64_value(
- 7,
static_cast<int64_t>(tablet->tablet_meta()->version_count()), block);
+ SchemaScannerHelper::insert_int64_value(7, snapshot.version_count,
block);
// SEGMENT_COUNT
- SchemaScannerHelper::insert_int64_value(
- 8,
- [&tablet]() {
- auto rs_metas = tablet->tablet_meta()->all_rs_metas();
- return std::accumulate(rs_metas.begin(), rs_metas.end(), 0,
- [](int64_t val, const auto& it) {
- return val +
it.second->num_segments();
- });
- }(),
- block);
+ SchemaScannerHelper::insert_int64_value(8, snapshot.segment_count,
block);
// NUM_COLUMNS
- SchemaScannerHelper::insert_int64_value(9,
tablet->tablet_meta()->tablet_columns_num(),
- block);
+ SchemaScannerHelper::insert_int64_value(9, snapshot.num_columns,
block);
// ROW_SIZE
- SchemaScannerHelper::insert_int64_value(10,
static_cast<int64_t>(tablet->row_size()),
- block);
+ SchemaScannerHelper::insert_int64_value(10, snapshot.row_size, block);
// COMPACTION_SCORE
- SchemaScannerHelper::insert_int32_value(11,
tablet->get_real_compaction_score(), block);
+ SchemaScannerHelper::insert_int32_value(11, snapshot.compaction_score,
block);
// COMPRESS_KIND
- SchemaScannerHelper::insert_string_value(12,
CompressKind_Name(tablet->compress_kind()),
- block);
+ SchemaScannerHelper::insert_string_value(12, snapshot.compress_kind,
block);
// IS_USED
- SchemaScannerHelper::insert_bool_value(
- 13,
- [&tablet]() {
- if (config::is_cloud_mode()) {
- return true;
- }
- return std::static_pointer_cast<Tablet>(tablet)->is_used();
- }(),
- block);
+ SchemaScannerHelper::insert_bool_value(13, snapshot.is_used, block);
// IS_ALTER_FAILED
- SchemaScannerHelper::insert_bool_value(14, tablet->is_alter_failed(),
block);
+ SchemaScannerHelper::insert_bool_value(14, snapshot.is_alter_failed,
block);
// CREATE_TIME
- SchemaScannerHelper::insert_datetime_value(15,
tablet->tablet_meta()->creation_time(),
- _timezone_obj, block);
+ SchemaScannerHelper::insert_datetime_value(15, snapshot.create_time,
_timezone_obj, block);
// UPDATE_TIME
- SchemaScannerHelper::insert_datetime_value(
- 16,
- [&tablet]() {
- auto rowset = tablet->get_rowset_with_max_version();
- return rowset == nullptr ? 0 :
rowset->newest_write_timestamp();
- }(),
- _timezone_obj, block);
+ SchemaScannerHelper::insert_datetime_value(16, snapshot.update_time,
_timezone_obj, block);
// IS_OVERLAP
- SchemaScannerHelper::insert_bool_value(
- 17,
- [&tablet]() {
- const auto& rs_metas =
tablet->tablet_meta()->all_rs_metas();
- return std::any_of(rs_metas.begin(), rs_metas.end(),
[](const auto& it) {
- return it.second->is_segments_overlapping();
- });
- }(),
- block);
+ SchemaScannerHelper::insert_bool_value(17, snapshot.is_overlap, block);
}
return Status::OK();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
