This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new cd9137a0eef [improvement](drop tablet) impr gc shutdown tablet lock
(#26151) (#27478)
cd9137a0eef is described below
commit cd9137a0eef27d37604e4871dc2dd3d0c90f81e4
Author: yujun <[email protected]>
AuthorDate: Thu Nov 23 23:51:03 2023 +0800
[improvement](drop tablet) impr gc shutdown tablet lock (#26151) (#27478)
---
be/src/olap/tablet_manager.cpp | 44 ++++++++++++++++++++++++++++++------------
be/src/olap/tablet_manager.h | 4 ++++
2 files changed, 36 insertions(+), 12 deletions(-)
diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp
index 28137309c0d..f6cfb6b6dc4 100644
--- a/be/src/olap/tablet_manager.cpp
+++ b/be/src/olap/tablet_manager.cpp
@@ -638,12 +638,24 @@ TabletSharedPtr
TabletManager::_get_tablet_unlocked(TTabletId tablet_id, bool in
TabletSharedPtr tablet;
tablet = _get_tablet_unlocked(tablet_id);
if (tablet == nullptr && include_deleted) {
- std::shared_lock rdlock(_shutdown_tablets_lock);
- for (auto& deleted_tablet : _shutdown_tablets) {
- CHECK(deleted_tablet != nullptr) << "deleted tablet is nullptr";
- if (deleted_tablet->tablet_id() == tablet_id) {
- tablet = deleted_tablet;
- break;
+ {
+ std::shared_lock rdlock(_shutdown_tablets_lock);
+ for (auto& deleted_tablet : _shutdown_tablets) {
+ CHECK(deleted_tablet != nullptr) << "deleted tablet is
nullptr";
+ if (deleted_tablet->tablet_id() == tablet_id) {
+ tablet = deleted_tablet;
+ break;
+ }
+ }
+ }
+ if (tablet == nullptr) {
+ std::shared_lock rdlock(_shutdown_deleting_tablets_lock);
+ for (auto& deleted_tablet : _shutdown_deleting_tablets) {
+ CHECK(deleted_tablet != nullptr) << "deleted tablet is
nullptr";
+ if (deleted_tablet->tablet_id() == tablet_id) {
+ tablet = deleted_tablet;
+ break;
+ }
}
}
}
@@ -1065,9 +1077,17 @@ Status TabletManager::start_trash_sweep() {
clean_num = 0;
// should get write lock here, because it will remove tablet from
shut_down_tablets
// and get tablet will access shut_down_tablets
- std::lock_guard<std::shared_mutex> wrlock(_shutdown_tablets_lock);
- auto it = _shutdown_tablets.begin();
- while (it != _shutdown_tablets.end()) {
+ {
+ std::lock_guard<std::shared_mutex> wrlock1(_shutdown_tablets_lock);
+ std::lock_guard<std::shared_mutex>
wrlock2(_shutdown_deleting_tablets_lock);
+ for (const auto& tablet : _shutdown_tablets) {
+ _shutdown_deleting_tablets.push_back(tablet);
+ }
+ _shutdown_tablets.clear();
+ }
+ std::lock_guard<std::shared_mutex>
wrlock(_shutdown_deleting_tablets_lock);
+ auto it = _shutdown_deleting_tablets.begin();
+ while (it != _shutdown_deleting_tablets.end()) {
// check if the meta has the tablet info and its state is shutdown
if (it->use_count() > 1) {
// it means current tablet is referenced by other thread
@@ -1086,7 +1106,7 @@ Status TabletManager::start_trash_sweep() {
<< " old tablet_uid=" << (*it)->tablet_uid()
<< " cur tablet_uid=" <<
tablet_meta->tablet_uid();
// remove it from list
- it = _shutdown_tablets.erase(it);
+ it = _shutdown_deleting_tablets.erase(it);
continue;
}
// move data to trash
@@ -1115,7 +1135,7 @@ Status TabletManager::start_trash_sweep() {
<< "tablet_id=" << (*it)->tablet_id()
<< ", schema_hash=" << (*it)->schema_hash()
<< ", tablet_path=" << tablet_path;
- it = _shutdown_tablets.erase(it);
+ it = _shutdown_deleting_tablets.erase(it);
++clean_num;
} else {
// if could not find tablet info in meta store, then check if
dir existed
@@ -1135,7 +1155,7 @@ Status TabletManager::start_trash_sweep() {
<< "tablet_id=" << (*it)->tablet_id()
<< ", schema_hash=" << (*it)->schema_hash()
<< ", tablet_path=" << tablet_path;
- it = _shutdown_tablets.erase(it);
+ it = _shutdown_deleting_tablets.erase(it);
}
}
diff --git a/be/src/olap/tablet_manager.h b/be/src/olap/tablet_manager.h
index 4f6b3278194..aa763502195 100644
--- a/be/src/olap/tablet_manager.h
+++ b/be/src/olap/tablet_manager.h
@@ -242,6 +242,10 @@ private:
std::map<int64_t, std::set<TabletInfo>> _partition_tablet_map;
std::vector<TabletSharedPtr> _shutdown_tablets;
+ // gc thread will move _shutdown_tablets to _shutdown_deleting_tablets
+ std::shared_mutex _shutdown_deleting_tablets_lock;
+ std::list<TabletSharedPtr> _shutdown_deleting_tablets;
+
std::mutex _tablet_stat_cache_mutex;
std::shared_ptr<std::vector<TTabletStat>> _tablet_stat_list_cache =
std::make_shared<std::vector<TTabletStat>>();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
