This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 631dbda2f79 [Fix](compaction) Fix single compaction to get all local
versions 33480 (#33524)
631dbda2f79 is described below
commit 631dbda2f7973c2222c99a4af22122fe14959827
Author: Sun Chenyang <[email protected]>
AuthorDate: Mon Apr 15 08:49:58 2024 +0800
[Fix](compaction) Fix single compaction to get all local versions 33480
(#33524)
---
be/src/olap/olap_server.cpp | 2 +-
be/src/olap/single_replica_compaction.cpp | 2 +-
be/src/olap/tablet.cpp | 11 +++++-----
be/src/olap/tablet.h | 4 +++-
be/test/olap/tablet_test.cpp | 35 +++++++++++++++++++++++++++++++
5 files changed, 46 insertions(+), 8 deletions(-)
diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp
index 2a9f911ed96..7ce25e7016d 100644
--- a/be/src/olap/olap_server.cpp
+++ b/be/src/olap/olap_server.cpp
@@ -788,7 +788,7 @@ void StorageEngine::get_tablet_rowset_versions(const
PGetTabletVersionsRequest*
response->mutable_status()->set_status_code(TStatusCode::CANCELLED);
return;
}
- std::vector<Version> local_versions = tablet->get_all_versions();
+ std::vector<Version> local_versions = tablet->get_all_local_versions();
for (const auto& local_version : local_versions) {
auto version = response->add_versions();
version->set_first(local_version.first);
diff --git a/be/src/olap/single_replica_compaction.cpp
b/be/src/olap/single_replica_compaction.cpp
index ddd6e08bfa2..9e3c1843aaa 100644
--- a/be/src/olap/single_replica_compaction.cpp
+++ b/be/src/olap/single_replica_compaction.cpp
@@ -193,7 +193,7 @@ Status
SingleReplicaCompaction::_get_rowset_verisons_from_peer(
bool SingleReplicaCompaction::_find_rowset_to_fetch(const
std::vector<Version>& peer_versions,
Version* proper_version) {
// already sorted
- std::vector<Version> local_versions = _tablet->get_all_versions();
+ std::vector<Version> local_versions = _tablet->get_all_local_versions();
for (const auto& v : local_versions) {
VLOG_CRITICAL << _tablet->tablet_id() << " tablet local version: " <<
v.first << " - "
<< v.second;
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index a8f23a760f6..ab7653684e9 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -1889,13 +1889,14 @@ void
Tablet::execute_single_replica_compaction(SingleReplicaCompaction& compacti
set_last_failure_time(this, compaction, 0);
}
-std::vector<Version> Tablet::get_all_versions() {
+std::vector<Version> Tablet::get_all_local_versions() {
std::vector<Version> local_versions;
{
- std::lock_guard<std::shared_mutex> wrlock(_meta_lock);
- SCOPED_SIMPLE_TRACE_IF_TIMEOUT(TRACE_TABLET_LOCK_THRESHOLD);
- for (const auto& it : _rs_version_map) {
- local_versions.emplace_back(it.first);
+ std::shared_lock rlock(_meta_lock);
+ for (const auto& [version, rs] : _rs_version_map) {
+ if (rs->is_local()) {
+ local_versions.emplace_back(version);
+ }
}
}
std::sort(local_versions.begin(), local_versions.end(),
diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h
index 6c3bcb0e615..b6c01e84d5c 100644
--- a/be/src/olap/tablet.h
+++ b/be/src/olap/tablet.h
@@ -274,7 +274,9 @@ public:
std::vector<RowsetSharedPtr>
pick_candidate_rowsets_to_build_inverted_index(
const std::set<int32_t>& alter_index_uids, bool is_drop_op);
- std::vector<Version> get_all_versions();
+ // used for single compaction to get the local versions
+ // Single compaction does not require remote rowsets and cannot violate
the cooldown semantics
+ std::vector<Version> get_all_local_versions();
std::vector<RowsetSharedPtr> pick_first_consecutive_empty_rowsets(int
limit);
diff --git a/be/test/olap/tablet_test.cpp b/be/test/olap/tablet_test.cpp
index 838db9b0d6d..951d98b1e05 100644
--- a/be/test/olap/tablet_test.cpp
+++ b/be/test/olap/tablet_test.cpp
@@ -142,6 +142,20 @@ public:
pb1->set_tablet_schema(_tablet_meta->tablet_schema());
}
+ void init_rs_meta_resource(RowsetMetaSharedPtr& pb1, int64_t start,
int64_t end,
+ bool is_local) {
+ RowsetMetaPB rowset_meta_pb;
+ json2pb::JsonToProtoMessage(_json_rowset_meta, &rowset_meta_pb);
+ rowset_meta_pb.set_start_version(start);
+ rowset_meta_pb.set_end_version(end);
+ rowset_meta_pb.set_creation_time(10000);
+ if (!is_local) {
+ rowset_meta_pb.set_resource_id("100");
+ }
+ pb1->init_from_pb(rowset_meta_pb);
+ pb1->set_tablet_schema(_tablet_meta->tablet_schema());
+ }
+
void init_all_rs_meta(std::vector<RowsetMetaSharedPtr>* rs_metas) {
RowsetMetaSharedPtr ptr1(new RowsetMeta());
init_rs_meta(ptr1, 0, 0);
@@ -392,4 +406,25 @@ TEST_F(TestTablet, cooldown_policy) {
}
}
+TEST_F(TestTablet, get_local_versions) {
+ // 10 remote rowsets
+ for (int i = 1; i <= 10; i++) {
+ auto ptr = std::make_shared<RowsetMeta>();
+ init_rs_meta_resource(ptr, i, i, false);
+ static_cast<void>(_tablet_meta->add_rs_meta(ptr));
+ }
+
+ // 20 local rowsets
+ for (int i = 11; i <= 30; i++) {
+ auto ptr = std::make_shared<RowsetMeta>();
+ init_rs_meta_resource(ptr, i, i, true);
+ static_cast<void>(_tablet_meta->add_rs_meta(ptr));
+ }
+
+ TabletSharedPtr _tablet(new Tablet(_tablet_meta, nullptr));
+ _tablet->init();
+ const auto& local_versions = _tablet->get_all_local_versions();
+ ASSERT_EQ(local_versions.size(), 20);
+}
+
} // namespace doris
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
