This is an automated email from the ASF dual-hosted git repository.

chenBright pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/brpc.git


The following commit(s) were added to refs/heads/master by this push:
     new 0678a4977 fix(bthread): Destroy pooled KeyTables outside locks (#3682)
0678a4977 is described below

commit 0678a49771d3c4a50d83673084771ada2fde21a8
Author: darion-yaphet <[email protected]>
AuthorDate: Sat Nov 14 11:45:43 2026 +0800

    fix(bthread): Destroy pooled KeyTables outside locks (#3682)
    
    Detach cached KeyTables while holding the pool and cache locks, then 
destroy them after releasing the locks. Add regression coverage for TLS 
destructor reentrancy.
---
 src/bthread/key.cpp           | 52 ++++++++++++++++++++++++++++++++++++-------
 test/bthread_key_unittest.cpp | 51 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 95 insertions(+), 8 deletions(-)

diff --git a/src/bthread/key.cpp b/src/bthread/key.cpp
index 802086d6f..c071f8a1b 100644
--- a/src/bthread/key.cpp
+++ b/src/bthread/key.cpp
@@ -255,6 +255,24 @@ public:
         _length++;
     }
 
+    // Transfer the entire chain without invoking KeyTable destructors. The
+    // caller uses this while holding the cache-management locks, then destroys
+    // the receiving list after releasing those locks.
+    void transfer_list_from(KeyTableList* other) {
+        if (other == nullptr || other->_head == nullptr) {
+            return;
+        }
+        if (_head == nullptr) {
+            _head = other->_head;
+        } else {
+            _tail->next = other->_head;
+        }
+        _tail = other->_tail;
+        _length += other->_length;
+        other->_head = other->_tail = nullptr;
+        other->_length = 0;
+    }
+
     KeyTable* remove_front() {
         if (_head == nullptr) {
             return nullptr;
@@ -476,14 +494,32 @@ int 
bthread_keytable_pool_destroy(bthread_keytable_pool_t* pool) {
         return EINVAL;
     }
     bthread::KeyTable* saved_free_keytables = nullptr;
-    pthread_rwlock_wrlock(&pool->rwlock);
-    pool->destroyed = 1;
-    pool->size = 0;
-    delete (butil::ThreadLocal<bthread::KeyTableList>*)pool->list;
-    saved_free_keytables = (bthread::KeyTable*)pool->free_keytables;
-    pool->list = nullptr;
-    pool->free_keytables = nullptr;
-    pthread_rwlock_unlock(&pool->rwlock);
+    butil::ThreadLocal<bthread::KeyTableList>* lists = nullptr;
+    {
+        bthread::KeyTableList detached;
+        pthread_rwlock_wrlock(&pool->rwlock);
+        pool->destroyed = 1;
+        pool->size = 0;
+        lists = (butil::ThreadLocal<bthread::KeyTableList>*)pool->list;
+        if (lists) {
+            // for_each() holds ThreadLocal's internal mutex. Only detach list
+            // nodes here: deleting a KeyTable can invoke a user TLS 
destructor.
+            lists->for_each([&detached](bthread::KeyTableList* list) {
+                detached.transfer_list_from(list);
+            });
+        }
+        saved_free_keytables = (bthread::KeyTable*)pool->free_keytables;
+        // After destroyed is set, borrowers and returners stop using these
+        // cache entries. Detaching their roots while holding the pool lock
+        // makes it safe to release both locks before running destructors.
+        pool->list = nullptr;
+        pool->free_keytables = nullptr;
+        pthread_rwlock_unlock(&pool->rwlock);
+        // Destroy the detached tables after releasing both cache locks. This
+        // scope ends before lists is deleted, so its destructor sees only
+        // empty KeyTableLists and cannot invoke user TLS destructors.
+    }
+    delete lists;
 
     // Cheat get/setspecific and destroy the keytables.
     bthread::TaskGroup* g =
diff --git a/test/bthread_key_unittest.cpp b/test/bthread_key_unittest.cpp
index a883ad1ec..18cb99294 100644
--- a/test/bthread_key_unittest.cpp
+++ b/test/bthread_key_unittest.cpp
@@ -33,6 +33,7 @@ namespace bthread {
 DECLARE_uint32(key_table_list_size);
 DECLARE_uint32(borrow_from_globle_size);
 class KeyTable;
+class KeyTableList;
 // defined in bthread/key.cpp
 extern void return_keytable(bthread_keytable_pool_t*, KeyTable*);
 extern KeyTable* borrow_keytable(bthread_keytable_pool_t*);
@@ -428,6 +429,56 @@ TEST(KeyTest, using_pool) {
     ASSERT_EQ(0, bthread_key_delete(key));
 }
 
+struct PoolDestructorLockCheck {
+    bthread_keytable_pool_t* pool;
+    butil::ThreadLocal<KeyTableList>* lists;
+    int pool_lock_result;
+    int list_lock_result;
+    int calls;
+};
+
+static void check_pool_destructor_locks(void* arg) {
+    PoolDestructorLockCheck* check = 
static_cast<PoolDestructorLockCheck*>(arg);
+    ++check->calls;
+    // A successful try-lock proves that the TLS destructor is not running
+    // under either cache-management lock.
+    check->pool_lock_result = pthread_rwlock_trywrlock(&check->pool->rwlock);
+    if (check->pool_lock_result == 0) {
+        pthread_rwlock_unlock(&check->pool->rwlock);
+    }
+    check->list_lock_result = pthread_mutex_trylock(&check->lists->_mutex);
+    if (check->list_lock_result == 0) {
+        pthread_mutex_unlock(&check->lists->_mutex);
+    }
+}
+
+static void* create_pool_destructor_lock_check(const void* arg) {
+    return const_cast<void*>(arg);
+}
+
+TEST(KeyTest, pooled_destructor_runs_outside_cache_locks) {
+    GFLAGS_NAMESPACE::FlagSaver flag_saver;
+    bthread::FLAGS_borrow_from_globle_size = 1;
+    bthread_keytable_pool_t pool;
+    ASSERT_EQ(0, bthread_keytable_pool_init(&pool));
+    PoolDestructorLockCheck check = {
+        &pool, static_cast<butil::ThreadLocal<KeyTableList>*>(pool.list), -1, 
-1, 0
+    };
+    bthread_key_t key;
+    ASSERT_EQ(0, bthread_key_create(&key, check_pool_destructor_locks));
+    bthread_keytable_pool_reserve(&pool, 1, key,
+                                  create_pool_destructor_lock_check, &check);
+    KeyTable* kt = borrow_keytable(&pool);
+    ASSERT_NE(nullptr, kt);
+    return_keytable(&pool, kt);
+    ASSERT_EQ(0, bthread_keytable_pool_destroy(&pool));
+    EXPECT_EQ(1, check.calls);
+    EXPECT_EQ(0, check.pool_lock_result);
+    EXPECT_EQ(0, check.list_lock_result);
+    EXPECT_EQ(0, bthread_key_delete(key));
+    EXPECT_EQ(0, pthread_rwlock_destroy(&pool.rwlock));
+}
+
 bthread_keytable_pool_t test_pool;
 struct PoolData2 {
     bthread_key_t key;


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to