This is an automated email from the ASF dual-hosted git repository.
chenBright pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/brpc.git
The following commit(s) were added to refs/heads/master by this push:
new d54f0cee Fix TLS access across bthread migration (#3713)
d54f0cee is described below
commit d54f0cee8615d8c1b7c46787fcf65a088063749e
Author: Bright Chen <[email protected]>
AuthorDate: Fri Nov 6 14:17:45 2026 +0800
Fix TLS access across bthread migration (#3713)
---
docs/cn/thread_local.md | 26 +++
src/brpc/controller.cpp | 6 +-
src/brpc/http_status_code.cpp | 7 +-
src/brpc/parallel_channel.cpp | 11 +-
src/brpc/rdma/block_pool.cpp | 56 +++----
src/brpc/span.cpp | 18 +-
src/bthread/bthread.cpp | 67 ++++----
src/bthread/butex.cpp | 16 +-
src/bthread/fd.cpp | 8 +-
src/bthread/key.cpp | 20 +--
src/bthread/mutex.cpp | 41 ++---
src/bthread/task_control.cpp | 7 +-
src/bthread/task_group.cpp | 44 ++---
src/bthread/task_group.h | 4 +
src/bthread/task_meta.h | 4 +-
src/bthread/task_tracer.cpp | 4 +-
src/butil/containers/doubly_buffered_data.h | 31 ++--
src/butil/containers/flat_map.h | 8 +-
src/butil/errno.cpp | 18 +-
src/butil/fast_rand.cpp | 44 ++---
src/butil/find_cstr.cpp | 2 +-
src/butil/find_cstr.h | 18 +-
src/butil/iobuf.cpp | 16 +-
src/butil/logging.cc | 12 +-
src/butil/object_pool_inl.h | 18 +-
src/butil/resource_pool_inl.h | 16 +-
src/butil/thread_key.cpp | 36 ++--
src/butil/thread_local.h | 42 +++--
src/butil/thread_local_inl.h | 19 ++-
src/bvar/detail/agent_group.h | 40 ++---
test/baidu_thread_local_unittest.cpp | 13 +-
test/bthread_setconcurrency_unittest.cpp | 10 +-
test/bthread_tls_unittest.cpp | 251 ++++++++++++++++++++++++++++
33 files changed, 612 insertions(+), 321 deletions(-)
diff --git a/docs/cn/thread_local.md b/docs/cn/thread_local.md
index 41e02474..94241575 100644
--- a/docs/cn/thread_local.md
+++ b/docs/cn/thread_local.md
@@ -62,3 +62,29 @@ Use *p ... - still the errno of original
pthread, undefined b
把`__const__`定义为`__unused__`对程序其他部分的影响几乎为0。另外如果你没有**直接**使用errno(即你的项目中没有出现errno),或使用的是gcc
3.4,即使没有定义`-D__const__=__unused__`,程序的正确性也不会受影响,但为了防止未来可能的问题,我们强烈建议加上。
需要说明的是,和errno类似,pthread_self也有类似的问题,不过一般pthread_self除了打日志没有其他用途,影响面较小,在`-D__const__=__unused__`后pthread_self也会正常。
+
+# 迁移安全的 pthread-local 访问
+
+仓库内部使用 `butil/thread_local.h` 的 `BUTIL_THREAD_LOCAL`,将 TLS
存储封装在不可内联的访问函数中。每次调用都重新取得当前 pthread 的 TLS 地址,并通过编译器屏障防止 LTO
将返回地址视为不变值。即使源码没有显式保存 TLS 指针,直接使用 `__thread` 或 `thread_local`
也可能被编译器优化成跨挂起点复用旧地址,见 [issue #3520](https://github.com/apache/brpc/issues/3520)。
+
+```cpp
+#include "butil/thread_local.h"
+
+BUTIL_THREAD_LOCAL(int, tls_count, 0);
+
+void count_twice() {
+ ++tls_count();
+ bthread_yield();
+ ++tls_count(); // 重新访问当前 pthread 的计数器
+}
+
+class Cache {
+ static BUTIL_THREAD_LOCAL(void*, value, nullptr);
+}; // 静态成员不再需要在类外定义 TLS 变量
+
+using Buffer = char[64];
+static BUTIL_THREAD_LOCAL(Buffer, scratch, {});
+```
+
+- 读取、赋值、取地址分别使用 `tls_count()`、`tls_count() = n`、`&tls_count()`。
+- 跨源文件声明使用普通函数声明 `int& tls_count();`。
diff --git a/src/brpc/controller.cpp b/src/brpc/controller.cpp
index ccfb8392..714dd216 100644
--- a/src/brpc/controller.cpp
+++ b/src/brpc/controller.cpp
@@ -50,10 +50,6 @@
#include "brpc/selective_channel.h"
#include "bthread/task_group.h"
-namespace bthread {
-EXTERN_BAIDU_VOLATILE_THREAD_LOCAL(TaskGroup*, tls_task_group);
-}
-
// This is the only place that both client/server must link, so we put
// registrations of errno here.
BAIDU_REGISTER_ERRNO(brpc::ENOSERVICE, "No such service");
@@ -948,7 +944,7 @@ void Controller::OnVersionedRPCReturned(const
CompletionInfo& info,
response_attachment().clear();
// Retry backoff.
- bthread::TaskGroup* g =
bthread::BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_task_group);
+ bthread::TaskGroup* g = bthread::tls_task_group();
int64_t backoff_time_us = retry_policy->GetBackoffTimeMs(this) *
1000L;
if (backoff_time_us > 0 &&
backoff_time_us < _deadline_us - butil::gettimeofday_us()) {
diff --git a/src/brpc/http_status_code.cpp b/src/brpc/http_status_code.cpp
index 01f132b7..a26d931b 100644
--- a/src/brpc/http_status_code.cpp
+++ b/src/brpc/http_status_code.cpp
@@ -100,7 +100,8 @@ static void InitReasonPhrases() {
}
}
-static BAIDU_THREAD_LOCAL char tls_phrase_cache[64];
+using PhraseCache = char[64];
+static BUTIL_THREAD_LOCAL(PhraseCache, tls_phrase_cache, {});
const char *HttpReasonPhrase(int status_code) {
pthread_once(&init_reason_phrases_once, InitReasonPhrases);
@@ -110,9 +111,9 @@ const char *HttpReasonPhrase(int status_code) {
(desc = phrases[status_code])) {
return desc;
}
- snprintf(tls_phrase_cache, sizeof(tls_phrase_cache),
+ snprintf(tls_phrase_cache(), sizeof(tls_phrase_cache()),
"Unknown status code (%d)", status_code);
- return tls_phrase_cache;
+ return tls_phrase_cache();
}
int ErrorCodeToStatusCode(int error_code) {
diff --git a/src/brpc/parallel_channel.cpp b/src/brpc/parallel_channel.cpp
index 5ae115c0..d331937a 100644
--- a/src/brpc/parallel_channel.cpp
+++ b/src/brpc/parallel_channel.cpp
@@ -21,6 +21,7 @@
#include "butil/atomicops.h"
#include "butil/time.h"
#include "butil/macros.h"
+#include "butil/thread_local.h"
#include "brpc/details/controller_private_accessor.h"
#include "brpc/parallel_channel.h"
@@ -34,7 +35,7 @@ struct Memory {
int size;
void* ptr;
};
-static __thread Memory tls_cached_pchan_mem = { 0, nullptr };
+static BUTIL_THREAD_LOCAL(Memory, tls_cached_pchan_mem, { 0, nullptr });
#endif
class ParallelChannelDone : public google::protobuf::Closure {
@@ -105,13 +106,13 @@ public:
void* mem = nullptr;
int memsize = 0;
#ifdef BRPC_CACHE_PCHAN_MEM
- Memory pchan_mem = tls_cached_pchan_mem;
+ Memory pchan_mem = tls_cached_pchan_mem();
if (pchan_mem.size >= req_size) { // use tls if it's big enough
mem = pchan_mem.ptr;
memsize = pchan_mem.size;
pchan_mem.size = 0;
pchan_mem.ptr = nullptr;
- tls_cached_pchan_mem = pchan_mem;
+ tls_cached_pchan_mem() = pchan_mem;
} else {
mem = malloc(req_size);
memsize = req_size;
@@ -161,7 +162,7 @@ public:
d->sub_done(i)->~SubDone();
}
#ifdef BRPC_CACHE_PCHAN_MEM
- Memory pchan_mem = tls_cached_pchan_mem;
+ Memory pchan_mem = tls_cached_pchan_mem();
if (pchan_mem.size != 0) {
// free the memory if tls already has sth.
d->~ParallelChannelDone();
@@ -170,7 +171,7 @@ public:
pchan_mem.size = d->_memsize;
pchan_mem.ptr = d;
d->~ParallelChannelDone();
- tls_cached_pchan_mem = pchan_mem;
+ tls_cached_pchan_mem() = pchan_mem;
}
#else
d->~ParallelChannelDone();
diff --git a/src/brpc/rdma/block_pool.cpp b/src/brpc/rdma/block_pool.cpp
index 3a332a91..7861a44b 100644
--- a/src/brpc/rdma/block_pool.cpp
+++ b/src/brpc/rdma/block_pool.cpp
@@ -85,9 +85,9 @@ static bool g_dump_enable = false;
static butil::Mutex* g_dump_mutex = nullptr;
// Only for default block size
-static __thread IdleNode* tls_idle_list = nullptr;
-static __thread size_t tls_idle_num = 0;
-static __thread bool tls_inited = false;
+static BUTIL_THREAD_LOCAL(IdleNode*, tls_idle_list, nullptr);
+static BUTIL_THREAD_LOCAL(size_t, tls_idle_num, 0);
+static BUTIL_THREAD_LOCAL(bool, tls_inited, false);
static butil::Mutex* g_tls_info_mutex = nullptr;
static size_t g_tls_info_cnt = 0;
static size_t* g_tls_info[1024];
@@ -349,13 +349,13 @@ static void* AllocBlockFrom(int block_type) {
};
void* ptr = nullptr;
- if (0 == block_type && nullptr != tls_idle_list) {
- CHECK(tls_idle_num > 0);
- IdleNode* n = tls_idle_list;
- tls_idle_list = n->next;
+ if (0 == block_type && nullptr != tls_idle_list()) {
+ CHECK(tls_idle_num() > 0);
+ IdleNode* n = tls_idle_list();
+ tls_idle_list() = n->next;
ptr = n->start;
butil::return_object<IdleNode>(n);
- tls_idle_num--;
+ tls_idle_num()--;
return ptr;
}
@@ -399,19 +399,19 @@ static void* AllocBlockFrom(int block_type) {
// Move more blocks from global list to tls list
if (block_type == 0) {
node = g_info->idle_list[0][index];
- tls_idle_list = node;
+ tls_idle_list() = node;
IdleNode* last_node = nullptr;
while (node) {
- if (tls_idle_num > (uint32_t)FLAGS_rdma_memory_pool_tls_cache_num
/ 2
+ if (tls_idle_num() >
(uint32_t)FLAGS_rdma_memory_pool_tls_cache_num / 2
|| node->len > g_block_size[0]) {
break;
}
- tls_idle_num++;
+ tls_idle_num()++;
last_node = node;
node = node->next;
}
- if (tls_idle_num == 0) {
- tls_idle_list = nullptr;
+ if (tls_idle_num() == 0) {
+ tls_idle_list() = nullptr;
} else {
g_info->idle_list[0][index] = node;
}
@@ -438,9 +438,9 @@ void* AllocBlock(size_t size) {
void RecycleAll() {
// Only block_type == 0 needs recycle
- while (tls_idle_list) {
- IdleNode* node = tls_idle_list;
- tls_idle_list = node->next;
+ while (tls_idle_list()) {
+ IdleNode* node = tls_idle_list();
+ tls_idle_list() = node->next;
Region* r = GetRegion(node->start);
if (!r) {
continue;
@@ -450,7 +450,7 @@ void RecycleAll() {
node->next = g_info->idle_list[0][index];
g_info->idle_list[0][index] = node;
}
- tls_idle_num = 0;
+ tls_idle_num() = 0;
}
int DeallocBlock(void* buf) {
@@ -488,18 +488,18 @@ int DeallocBlock(void* buf) {
}
};
- if (block_type == 0 && tls_idle_num <
(uint32_t)FLAGS_rdma_memory_pool_tls_cache_num) {
- if (!tls_inited) {
- tls_inited = true;
+ if (block_type == 0 && tls_idle_num() <
(uint32_t)FLAGS_rdma_memory_pool_tls_cache_num) {
+ if (!tls_inited()) {
+ tls_inited() = true;
butil::thread_atexit(RecycleAll);
BAIDU_SCOPED_LOCK(*g_tls_info_mutex);
if (g_tls_info_cnt < 1024) {
- g_tls_info[g_tls_info_cnt++] = &tls_idle_num;
+ g_tls_info[g_tls_info_cnt++] = &tls_idle_num();
}
}
- tls_idle_num++;
- node->next = tls_idle_list;
- tls_idle_list = node;
+ tls_idle_num()++;
+ node->next = tls_idle_list();
+ tls_idle_list() = node;
return 0;
}
@@ -508,7 +508,7 @@ int DeallocBlock(void* buf) {
size_t len = 0;
// Recycle half the cached blocks in tls for default block size
int num = FLAGS_rdma_memory_pool_tls_cache_num / 2;
- IdleNode* new_head = tls_idle_list;
+ IdleNode* new_head = tls_idle_list();
IdleNode* recycle_tail = nullptr;
for (int i = 0; i < num; ++i) {
recycle_tail = new_head;
@@ -519,11 +519,11 @@ int DeallocBlock(void* buf) {
BAIDU_SCOPED_LOCK(*g_info->lock[0][index]);
recycle_tail->next = node;
node->next = g_info->idle_list[0][index];
- g_info->idle_list[0][index] = tls_idle_list;
+ g_info->idle_list[0][index] = tls_idle_list();
g_info->idle_size[0][index] += len;
}
- tls_idle_list = new_head;
- tls_idle_num -= num;
+ tls_idle_list() = new_head;
+ tls_idle_num() -= num;
} else {
BAIDU_SCOPED_LOCK(*g_info->lock[block_type][index]);
node->next = g_info->idle_list[block_type][index];
diff --git a/src/brpc/span.cpp b/src/brpc/span.cpp
index b94caa38..874b197d 100644
--- a/src/brpc/span.cpp
+++ b/src/brpc/span.cpp
@@ -70,18 +70,18 @@ void EndBthreadSpan() {
void SetTlsParentSpan(std::shared_ptr<Span> span) {
using namespace bthread;
- LocalStorage ls = BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_bls);
+ LocalStorage ls = tls_bls();
if (ls.rpcz_parent_span) {
*static_cast<std::weak_ptr<Span>*>(ls.rpcz_parent_span) = span;
} else {
ls.rpcz_parent_span = new std::weak_ptr<Span>(span);
- BAIDU_SET_VOLATILE_THREAD_LOCAL(tls_bls, ls);
+ tls_bls() = ls;
}
}
std::shared_ptr<Span> GetTlsParentSpan() {
using namespace bthread;
- LocalStorage ls = BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_bls);
+ LocalStorage ls = tls_bls();
if (!ls.rpcz_parent_span) {
return nullptr;
}
@@ -92,7 +92,7 @@ std::shared_ptr<Span> GetTlsParentSpan() {
void ClearTlsParentSpan() {
using namespace bthread;
- LocalStorage ls = BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_bls);
+ LocalStorage ls = tls_bls();
if (ls.rpcz_parent_span) {
static_cast<std::weak_ptr<Span>*>(ls.rpcz_parent_span)->reset();
}
@@ -100,7 +100,7 @@ void ClearTlsParentSpan() {
bool HasTlsParentSpan() {
using namespace bthread;
- LocalStorage ls = BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_bls);
+ LocalStorage ls = tls_bls();
if (!ls.rpcz_parent_span) {
return false;
}
@@ -154,8 +154,8 @@ struct IdGen {
butil::FastRandSeed seed;
};
-static __thread IdGen tls_trace_id_gen = { false, 0, 0, { { 0, 0 } } };
-static __thread IdGen tls_span_id_gen = { false, 0, 0, { { 0, 0 } } };
+static BUTIL_THREAD_LOCAL(IdGen, tls_trace_id_gen, { false, 0, 0, { { 0, 0 } }
});
+static BUTIL_THREAD_LOCAL(IdGen, tls_span_id_gen, { false, 0, 0, { { 0, 0 } }
});
inline uint64_t UpdateTLSRandom64(IdGen* g) {
if (!g->init) {
@@ -169,7 +169,7 @@ inline uint64_t UpdateTLSRandom64(IdGen* g) {
inline uint64_t GenerateSpanId() {
// 0 is an invalid Id
- IdGen* g = &tls_span_id_gen;
+ IdGen* g = &tls_span_id_gen();
if (g->seq == 0) {
UpdateTLSRandom64(g);
g->seq = 1;
@@ -179,7 +179,7 @@ inline uint64_t GenerateSpanId() {
inline uint64_t GenerateTraceId() {
// 0 is an invalid Id
- IdGen* g = &tls_trace_id_gen;
+ IdGen* g = &tls_trace_id_gen();
if (g->seq == 0) {
UpdateTLSRandom64(g);
g->seq = 1;
diff --git a/src/bthread/bthread.cpp b/src/bthread/bthread.cpp
index bd863a8f..7ec7eb8b 100644
--- a/src/bthread/bthread.cpp
+++ b/src/bthread/bthread.cpp
@@ -87,7 +87,6 @@ pthread_mutex_t g_task_control_mutex =
PTHREAD_MUTEX_INITIALIZER;
// are not constructed before main().
TaskControl* g_task_control = nullptr;
-EXTERN_BAIDU_VOLATILE_THREAD_LOCAL(TaskGroup*, tls_task_group);
extern void (*g_worker_startfn)();
extern void (*g_tagged_worker_startfn)(bthread_tag_t);
@@ -121,11 +120,11 @@ inline TaskControl* get_or_new_task_control() {
}
#ifdef BRPC_BTHREAD_TRACER
-BAIDU_THREAD_LOCAL TaskMeta* pthread_fake_meta = nullptr;
+BUTIL_THREAD_LOCAL(TaskMeta*, pthread_fake_meta, nullptr);
bthread_t init_for_pthread_stack_trace() {
- if (nullptr != pthread_fake_meta) {
- return pthread_fake_meta->tid;
+ if (nullptr != pthread_fake_meta()) {
+ return pthread_fake_meta()->tid;
}
TaskControl* c = get_task_control();
@@ -136,31 +135,31 @@ bthread_t init_for_pthread_stack_trace() {
}
butil::ResourceId<TaskMeta> slot;
- pthread_fake_meta = butil::get_resource(&slot);
- if (BAIDU_UNLIKELY(nullptr == pthread_fake_meta)) {
+ pthread_fake_meta() = butil::get_resource(&slot);
+ if (BAIDU_UNLIKELY(nullptr == pthread_fake_meta())) {
LOG(ERROR) << "Fail to get TaskMeta";
return INVALID_BTHREAD;
}
- pthread_fake_meta->attr = BTHREAD_ATTR_PTHREAD;
+ pthread_fake_meta()->attr = BTHREAD_ATTR_PTHREAD;
auto* version = reinterpret_cast<butil::atomic<int>*>(
- pthread_fake_meta->version_butex);
- pthread_fake_meta->tid = make_tid(static_cast<uint32_t>(
+ pthread_fake_meta()->version_butex);
+ pthread_fake_meta()->tid = make_tid(static_cast<uint32_t>(
version->load(butil::memory_order_relaxed)), slot);
// Make TaskTracer use signal trace mode for pthread.
- c->_task_tracer.set_running_status(syscall(SYS_gettid), pthread_fake_meta);
+ c->_task_tracer.set_running_status(syscall(SYS_gettid),
pthread_fake_meta());
// Release the TaskMeta at exit of pthread.
butil::thread_atexit([]() {
// Similar to TaskGroup::task_runner.
bool tracing;
{
- BAIDU_SCOPED_LOCK(pthread_fake_meta->version_lock);
- tracing = TaskTracer::set_end_status_unsafe(pthread_fake_meta);
+ BAIDU_SCOPED_LOCK(pthread_fake_meta()->version_lock);
+ tracing = TaskTracer::set_end_status_unsafe(pthread_fake_meta());
// Publish the version atomically, just like task_runner(), since
// lock-free readers may still access this TaskMeta.
auto* version = reinterpret_cast<butil::atomic<int>*>(
- pthread_fake_meta->version_butex);
+ pthread_fake_meta()->version_butex);
uint32_t next_version = static_cast<uint32_t>(
version->load(butil::memory_order_relaxed)) + 1;
// Skip zero to make bthread_t never be 0.
@@ -172,16 +171,16 @@ bthread_t init_for_pthread_stack_trace() {
if (tracing) {
// Wait for tracing completion.
- get_task_control()->_task_tracer.WaitForTracing(pthread_fake_meta);
+
get_task_control()->_task_tracer.WaitForTracing(pthread_fake_meta());
}
get_task_control()->_task_tracer.set_status(
- TASK_STATUS_UNKNOWN, pthread_fake_meta);
+ TASK_STATUS_UNKNOWN, pthread_fake_meta());
- butil::return_resource(get_slot(pthread_fake_meta->tid));
- pthread_fake_meta = nullptr;
+ butil::return_resource(get_slot(pthread_fake_meta()->tid));
+ pthread_fake_meta() = nullptr;
});
- return pthread_fake_meta->tid;
+ return pthread_fake_meta()->tid;
}
void stack_trace(std::ostream& os, bthread_t tid) {
@@ -270,7 +269,7 @@ static bool validate_bthread_concurrency_by_tag(const
char*, int32_t val) {
return bthread_setconcurrency_by_tag(val, FLAGS_bthread_current_tag) == 0;
}
-BAIDU_VOLATILE_THREAD_LOCAL(TaskGroup*, tls_task_group_nosignal, nullptr);
+BUTIL_THREAD_LOCAL(TaskGroup*, tls_task_group_nosignal, nullptr);
BUTIL_FORCE_INLINE int
start_from_non_worker(bthread_t* __restrict tid,
@@ -290,10 +289,10 @@ start_from_non_worker(bthread_t* __restrict tid,
// 1. NOSIGNAL is often for creating many bthreads in batch,
// inserting into the same TaskGroup maximizes the batch.
// 2. bthread_flush() needs to know which TaskGroup to flush.
- auto g = BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_task_group_nosignal);
+ auto g = tls_task_group_nosignal();
if (nullptr == g) {
g = c->choose_one_group(tag);
- BAIDU_SET_VOLATILE_THREAD_LOCAL(tls_task_group_nosignal, g);
+ tls_task_group_nosignal() = g;
} else {
// tls_task_group_nosignal is not tag-aware: mixing different
// tags with BTHREAD_NOSIGNAL on the same non-worker thread is
@@ -314,7 +313,7 @@ start_from_non_worker(bthread_t* __restrict tid,
// tag equal to BTHREAD_TAG_INVALID
BUTIL_FORCE_INLINE bool can_run_thread_local(const bthread_attr_t* __restrict
attr) {
return attr == nullptr ||
- attr->tag == BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_task_group)->tag()
||
+ attr->tag == tls_task_group()->tag() ||
attr->tag == BTHREAD_TAG_INVALID;
}
@@ -347,7 +346,7 @@ int bthread_start_urgent(bthread_t* __restrict tid,
const bthread_attr_t* __restrict attr,
void * (*fn)(void*),
void* __restrict arg) {
- bthread::TaskGroup* g =
bthread::BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_task_group);
+ bthread::TaskGroup* g = bthread::tls_task_group();
if (g) {
// if attribute is null use thread local task group
if (bthread::can_run_thread_local(attr)) {
@@ -361,7 +360,7 @@ int bthread_start_background(bthread_t* __restrict tid,
const bthread_attr_t* __restrict attr,
void * (*fn)(void*),
void* __restrict arg) {
- bthread::TaskGroup* g =
bthread::BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_task_group);
+ bthread::TaskGroup* g = bthread::tls_task_group();
if (g) {
// if attribute is null use thread local task group
if (bthread::can_run_thread_local(attr)) {
@@ -372,14 +371,14 @@ int bthread_start_background(bthread_t* __restrict tid,
}
void bthread_flush() {
- bthread::TaskGroup* g =
bthread::BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_task_group);
+ bthread::TaskGroup* g = bthread::tls_task_group();
if (g) {
return g->flush_nosignal_tasks();
}
- g = bthread::BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_task_group_nosignal);
+ g = bthread::tls_task_group_nosignal();
if (g) {
// NOSIGNAL tasks were created in this non-worker.
- bthread::BAIDU_SET_VOLATILE_THREAD_LOCAL(tls_task_group_nosignal,
nullptr);
+ bthread::tls_task_group_nosignal() = nullptr;
return g->flush_nosignal_tasks_remote();
}
}
@@ -398,7 +397,7 @@ int bthread_stopped(bthread_t tid) {
}
bthread_t bthread_self(void) {
- bthread::TaskGroup* g =
bthread::BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_task_group);
+ bthread::TaskGroup* g = bthread::tls_task_group();
// note: return 0 for main tasks now, which include main thread and
// all work threads. So that we can identify main tasks from logs
// more easily. This is probably questionable in the future.
@@ -413,7 +412,7 @@ int bthread_equal(bthread_t t1, bthread_t t2) {
}
void bthread_exit(void* retval) {
- bthread::TaskGroup* g =
bthread::BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_task_group);
+ bthread::TaskGroup* g = bthread::tls_task_group();
if (g != nullptr && !g->is_current_main_task()) {
throw bthread::ExitException(retval);
} else {
@@ -554,7 +553,7 @@ int bthread_add_reserved_concurrency_by_tag(int num,
bthread_tag_t tag) {
}
int bthread_about_to_quit() {
- bthread::TaskGroup* g =
bthread::BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_task_group);
+ bthread::TaskGroup* g = bthread::tls_task_group();
if (g != nullptr) {
bthread::TaskMeta* current_task = g->current_task();
if(!(current_task->attr.flags & BTHREAD_NEVER_QUIT)) {
@@ -599,7 +598,7 @@ int bthread_timer_del(bthread_timer_t id) {
}
int bthread_usleep(uint64_t microseconds) {
- bthread::TaskGroup* g =
bthread::BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_task_group);
+ bthread::TaskGroup* g = bthread::tls_task_group();
if (nullptr != g && !g->is_current_pthread_task()) {
return bthread::TaskGroup::usleep(&g, microseconds);
}
@@ -607,7 +606,7 @@ int bthread_usleep(uint64_t microseconds) {
}
int bthread_yield(void) {
- bthread::TaskGroup* g =
bthread::BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_task_group);
+ bthread::TaskGroup* g = bthread::tls_task_group();
if (nullptr != g && !g->is_current_pthread_task()) {
bthread::TaskGroup::yield(&g);
return 0;
@@ -680,12 +679,12 @@ int bthread_list_join(bthread_list_t* list) {
}
bthread_tag_t bthread_self_tag(void) {
- bthread::TaskGroup* g =
bthread::BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_task_group);
+ bthread::TaskGroup* g = bthread::tls_task_group();
return g != nullptr ? g->tag() : BTHREAD_TAG_DEFAULT;
}
uint64_t bthread_cpu_clock_ns(void) {
- bthread::TaskGroup* g =
bthread::BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_task_group);
+ bthread::TaskGroup* g = bthread::tls_task_group();
if (g != nullptr && !g->is_current_main_task()) {
return g->current_task_cpu_clock_ns();
}
diff --git a/src/bthread/butex.cpp b/src/bthread/butex.cpp
index df0562bd..df362f71 100644
--- a/src/bthread/butex.cpp
+++ b/src/bthread/butex.cpp
@@ -187,8 +187,6 @@ int wait_pthread(ButexPthreadWaiter& pw, const timespec*
abstime) {
}
}
-EXTERN_BAIDU_VOLATILE_THREAD_LOCAL(TaskGroup*, tls_task_group);
-
// Returns 0 when no need to unschedule or successfully unscheduled,
// -1 otherwise.
inline int unsleep_if_necessary(ButexBthreadWaiter* w,
@@ -280,7 +278,7 @@ void butex_destroy(void* butex) {
// if TaskGroup tls_task_group is belong to tag
inline bool is_same_tag(bthread_tag_t tag) {
- auto g = BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_task_group);
+ auto g = tls_task_group();
return g && g->tag() == tag;
}
@@ -291,7 +289,7 @@ inline bool check_nosignal(bool nosignal, bthread_tag_t
tag) {
// if tag is same return tls_task_group else choose one group with tag
inline TaskGroup* get_task_group(TaskControl* c, bthread_tag_t tag) {
- return is_same_tag(tag) ? BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_task_group)
+ return is_same_tag(tag) ? tls_task_group()
: c->choose_one_group(tag);
}
@@ -322,7 +320,7 @@ int butex_wake(void* arg, bool nosignal) {
ButexBthreadWaiter* bbw = static_cast<ButexBthreadWaiter*>(front);
unsleep_if_necessary(bbw, get_global_timer_thread());
TaskGroup* g = get_task_group(bbw->control, bbw->task_meta->attr.tag);
- if (g == BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_task_group)) {
+ if (g == tls_task_group()) {
run_in_local_task_group(g, bbw->task_meta, nosignal);
} else {
g->ready_to_run_remote(bbw->task_meta, check_nosignal(nosignal,
g->tag()));
@@ -386,7 +384,7 @@ int butex_wake_n(void* arg, size_t n, bool nosignal) {
}
}
auto g = get_task_group(next->control, next->task_meta->attr.tag);
- if (g == BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_task_group)) {
+ if (g == tls_task_group()) {
run_in_local_task_group(g, next->task_meta, nosignal);
} else {
g->ready_to_run_remote(next->task_meta, check_nosignal(nosignal,
g->tag()));
@@ -491,7 +489,7 @@ int butex_requeue(void* arg, void* arg2) {
ButexBthreadWaiter* bbw = static_cast<ButexBthreadWaiter*>(front);
unsleep_if_necessary(bbw, get_global_timer_thread());
auto g = is_same_tag(bbw->task_meta->attr.tag)
- ? BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_task_group)
+ ? tls_task_group()
: nullptr;
if (g) {
TaskGroup::exchange(&g, bbw->task_meta);
@@ -637,7 +635,7 @@ void wait_for_butex(void* arg) {
// the two functions. The on-stack ButexBthreadWaiter is safe to use and
// bw->waiter_state will not change again.
// unsleep_if_necessary(bw, get_global_timer_thread());
-
BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_task_group)->ready_to_run(bw->task_meta);
+ tls_task_group()->ready_to_run(bw->task_meta);
// FIXME: jump back to original thread is buggy.
// // Value unmatched or waiter is already woken up by TimerThread, jump
@@ -759,7 +757,7 @@ int butex_wait(void* arg, int expected_value, const
timespec* abstime, bool prep
butil::atomic_thread_fence(butil::memory_order_acquire);
return -1;
}
- TaskGroup* g = BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_task_group);
+ TaskGroup* g = tls_task_group();
if (nullptr == g || g->is_current_pthread_task()) {
return butex_wait_from_pthread(g, b, expected_value, abstime, prepend);
}
diff --git a/src/bthread/fd.cpp b/src/bthread/fd.cpp
index 5795b1dd..51c824e0 100644
--- a/src/bthread/fd.cpp
+++ b/src/bthread/fd.cpp
@@ -44,8 +44,6 @@ extern int pthread_fd_wait(int fd, unsigned events, const
timespec* abstime);
namespace bthread {
-EXTERN_BAIDU_VOLATILE_THREAD_LOCAL(TaskGroup*, tls_task_group);
-
template <typename T, size_t NBLOCK, size_t BLOCK_SIZE>
class LazyArray {
struct Block {
@@ -452,7 +450,7 @@ int bthread_fd_wait(int fd, unsigned events) {
errno = EINVAL;
return -1;
}
- bthread::TaskGroup* g =
bthread::BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_task_group);
+ bthread::TaskGroup* g = bthread::tls_task_group();
if (nullptr != g && !g->is_current_pthread_task()) {
return bthread::get_epoll_thread(fd).fd_wait(
fd, events, nullptr);
@@ -469,7 +467,7 @@ int bthread_fd_timedwait(int fd, unsigned events,
errno = EINVAL;
return -1;
}
- bthread::TaskGroup* g =
bthread::BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_task_group);
+ bthread::TaskGroup* g = bthread::tls_task_group();
if (nullptr != g && !g->is_current_pthread_task()) {
return bthread::get_epoll_thread(fd).fd_wait(
fd, events, abstime);
@@ -479,7 +477,7 @@ int bthread_fd_timedwait(int fd, unsigned events,
int bthread_connect(int sockfd, const sockaddr* serv_addr,
socklen_t addrlen) {
- bthread::TaskGroup* g =
bthread::BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_task_group);
+ bthread::TaskGroup* g = bthread::tls_task_group();
if (nullptr == g || g->is_current_pthread_task()) {
return ::connect(sockfd, serv_addr, addrlen);
}
diff --git a/src/bthread/key.cpp b/src/bthread/key.cpp
index 74e718a4..802086d6 100644
--- a/src/bthread/key.cpp
+++ b/src/bthread/key.cpp
@@ -45,11 +45,9 @@ DEFINE_uint32(borrow_from_globle_size, 200,
"from the global free_keytables when no KeyTable exists in the "
"current thread's keytable_list.");
-EXTERN_BAIDU_VOLATILE_THREAD_LOCAL(TaskGroup*, tls_task_group);
-
class KeyTable;
-BAIDU_VOLATILE_THREAD_LOCAL(bool, tls_ever_created_keytable, false);
+BUTIL_THREAD_LOCAL(bool, tls_ever_created_keytable, false);
// We keep thread specific data in a two-level array. The top-level array
// contains at most KEY_1STLEVEL_SIZE pointers to dynamically allocated
@@ -221,7 +219,7 @@ public:
_head(nullptr), _tail(nullptr), _length(0) {}
~KeyTableList() {
- TaskGroup* g = BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_task_group);
+ TaskGroup* g = tls_task_group();
KeyTable* old_kt = tls_bls_ptr()->keytable;
KeyTable* keytable = _head;
while (keytable) {
@@ -235,7 +233,7 @@ public:
if (old_kt == kt) {
old_kt = nullptr;
}
- g = BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_task_group);
+ g = tls_task_group();
}
tls_bls_ptr()->keytable = old_kt;
if (g) {
@@ -489,7 +487,7 @@ int bthread_keytable_pool_destroy(bthread_keytable_pool_t*
pool) {
// Cheat get/setspecific and destroy the keytables.
bthread::TaskGroup* g =
- bthread::BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_task_group);
+ bthread::tls_task_group();
bthread::KeyTable* old_kt = bthread::tls_bls_ptr()->keytable;
while (saved_free_keytables) {
bthread::KeyTable* kt = saved_free_keytables;
@@ -499,7 +497,7 @@ int bthread_keytable_pool_destroy(bthread_keytable_pool_t*
pool) {
g->current_task()->local_storage.keytable = kt;
}
delete kt;
- g = bthread::BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_task_group);
+ g = bthread::tls_task_group();
}
bthread::tls_bls_ptr()->keytable = old_kt;
if (g) {
@@ -646,15 +644,15 @@ int bthread_setspecific(bthread_key_t key, void* data) {
if (nullptr == kt) {
kt = new bthread::KeyTable;
bthread::tls_bls_ptr()->keytable = kt;
- bthread::TaskGroup* const g =
bthread::BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_task_group);
+ bthread::TaskGroup* const g = bthread::tls_task_group();
if (g) {
g->current_task()->local_storage.keytable = kt;
} else {
// Only cleanup keytable created by pthread.
// keytable created by bthread will be deleted
// in `return_keytable' or `bthread_keytable_pool_destroy'.
- if
(!bthread::BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_ever_created_keytable)) {
-
bthread::BAIDU_SET_VOLATILE_THREAD_LOCAL(tls_ever_created_keytable, true);
+ if (!bthread::tls_ever_created_keytable()) {
+ bthread::tls_ever_created_keytable() = true;
CHECK_EQ(0, butil::thread_atexit(bthread::cleanup_pthread,
kt));
}
}
@@ -667,7 +665,7 @@ void* bthread_getspecific(bthread_key_t key) {
if (kt) {
return kt->get_data(key);
}
- bthread::TaskGroup* const g =
bthread::BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_task_group);
+ bthread::TaskGroup* const g = bthread::tls_task_group();
if (g) {
bthread::TaskMeta* const task = g->current_task();
kt = bthread::borrow_keytable(task->attr.keytable_pool);
diff --git a/src/bthread/mutex.cpp b/src/bthread/mutex.cpp
index ac8b1d0e..3cd7334d 100644
--- a/src/bthread/mutex.cpp
+++ b/src/bthread/mutex.cpp
@@ -54,8 +54,6 @@ __END_DECLS
namespace bthread {
-EXTERN_BAIDU_VOLATILE_THREAD_LOCAL(TaskGroup*, tls_task_group);
-
// Warm up backtrace before main().
const butil::debug::StackTrace ALLOW_UNUSED dummy_bt;
@@ -524,11 +522,11 @@ inline uint64_t hash_mutex_ptr(const Mutex* m) {
// Mark being inside locking so that pthread_mutex calls inside collecting
// code are never sampled, otherwise deadlock may occur.
-BAIDU_VOLATILE_THREAD_LOCAL(bool, tls_inside_lock, false);
+BUTIL_THREAD_LOCAL(bool, tls_inside_lock, false);
// Warn up some singleton objects used in contention profiler
// to avoid deadlock in malloc call stack.
-BAIDU_VOLATILE_THREAD_LOCAL(bool, tls_warn_up, false);
+BUTIL_THREAD_LOCAL(bool, tls_warn_up, false);
#if BRPC_DEBUG_BTHREAD_SCHE_SAFETY
// ++tls_pthread_lock_count when pthread locking,
@@ -536,24 +534,24 @@ BAIDU_VOLATILE_THREAD_LOCAL(bool, tls_warn_up, false);
// Only when it is equal to 0, it is safe for the bthread to be scheduled.
// Note: If a mutex is locked/unlocked in different thread,
// `tls_pthread_lock_count' is inaccurate, so this feature cannot be used.
-BAIDU_VOLATILE_THREAD_LOCAL(int, tls_pthread_lock_count, 0);
+BUTIL_THREAD_LOCAL(int, tls_pthread_lock_count, 0);
inline void AddTlsPthreadLockCount() {
- ++(*BAIDU_GET_PTR_VOLATILE_THREAD_LOCAL(tls_pthread_lock_count));
+ ++tls_pthread_lock_count();
}
inline void SubTlsPthreadLockCount() {
- --(*BAIDU_GET_PTR_VOLATILE_THREAD_LOCAL(tls_pthread_lock_count));
+ --tls_pthread_lock_count();
}
void CheckBthreadScheSafety() {
- if (BAIDU_LIKELY(0 ==
BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_pthread_lock_count))) {
+ if (BAIDU_LIKELY(0 == tls_pthread_lock_count())) {
return;
}
// It can only be checked once because the counter is messed up.
LOG_BACKTRACE_ONCE(ERROR) << "bthread is suspended while holding "
- <<
BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_pthread_lock_count)
+ << tls_pthread_lock_count()
<< " pthread locks.";
}
#else
@@ -582,8 +580,7 @@ struct TLSPthreadContentionSites {
uint64_t cp_version;
MutexAndContentionSite list[TLS_MAX_COUNT];
};
-BAIDU_VOLATILE_THREAD_LOCAL(TLSPthreadContentionSites, tls_csites,
- TLSPthreadContentionSites());
+BUTIL_THREAD_LOCAL(TLSPthreadContentionSites, tls_csites, {});
#endif // DONT_SPEEDUP_PTHREAD_CONTENTION_PROFILER_WITH_TLS
// Guaranteed in linux/win.
@@ -632,9 +629,9 @@ inline bool remove_pthread_contention_site(const Mutex*
mutex,
// Submit the contention along with the callsite('s stacktrace)
void submit_contention(const bthread_contention_site_t& csite, int64_t now_ns)
{
- BAIDU_SET_VOLATILE_THREAD_LOCAL(tls_inside_lock, true);
+ tls_inside_lock() = true;
BRPC_SCOPE_EXIT {
- BAIDU_SET_VOLATILE_THREAD_LOCAL(tls_inside_lock, false);
+ tls_inside_lock() = false;
};
butil::debug::StackTrace stack(true); // May lock.
@@ -646,7 +643,7 @@ void submit_contention(const bthread_contention_site_t&
csite, int64_t now_ns) {
// 1. Warn up some singleton objects used in `submit_contention'
// to avoid deadlock in malloc call stack.
// 2. LocalPool is empty, GlobalPool may allocate memory by malloc.
- if (!BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_warn_up) ||
+ if (!tls_warn_up() ||
butil::local_pool_free_empty<SampledContention>()) {
// In malloc call stack, can not submit contention.
if (stack.FindSymbol((void*)malloc)) {
@@ -664,7 +661,7 @@ void submit_contention(const bthread_contention_site_t&
csite, int64_t now_ns) {
sc->nframes = stack.CopyAddressTo(sc->stack, arraysize(sc->stack));
sc->submit(now_ns / 1000); // may lock
// Once submit a contention, complete warn up.
- BAIDU_SET_VOLATILE_THREAD_LOCAL(tls_warn_up, true);
+ tls_warn_up() = true;
}
#if BRPC_DEBUG_LOCK
@@ -880,7 +877,7 @@ BUTIL_FORCE_INLINE int pthread_mutex_lock_impl(Mutex*
mutex, const struct timesp
if (!g_cp ||
// collecting code including backtrace() and submit() may call
// pthread_mutex_lock and cause deadlock. Don't sample.
- BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_inside_lock)) {
+ tls_inside_lock()) {
return pthread_mutex_lock_internal(mutex, abstime);
}
// Don't slow down non-contended locks.
@@ -893,8 +890,7 @@ BUTIL_FORCE_INLINE int pthread_mutex_lock_impl(Mutex*
mutex, const struct timesp
bthread_contention_site_t* csite = nullptr;
#ifndef DONT_SPEEDUP_PTHREAD_CONTENTION_PROFILER_WITH_TLS
- TLSPthreadContentionSites& fast_alt =
- *BAIDU_GET_PTR_VOLATILE_THREAD_LOCAL(tls_csites);
+ TLSPthreadContentionSites& fast_alt = tls_csites();
if (fast_alt.cp_version != g_cp_version) {
fast_alt.cp_version = g_cp_version;
fast_alt.count = 0;
@@ -936,7 +932,7 @@ BUTIL_FORCE_INLINE int pthread_mutex_trylock_impl(Mutex*
mutex) {
template <typename Mutex>
BUTIL_FORCE_INLINE int pthread_mutex_unlock_impl(Mutex* mutex) {
// Don't change behavior of unlock when profiler is off.
- if (!g_cp || BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_inside_lock)) {
+ if (!g_cp || tls_inside_lock()) {
// This branch brings an issue that an entry created by
// add_pthread_contention_site may not be cleared. Thus we add a
// 16-bit rolling version in the entry to find out such entry.
@@ -946,8 +942,7 @@ BUTIL_FORCE_INLINE int pthread_mutex_unlock_impl(Mutex*
mutex) {
bool miss_in_tls = true;
bthread_contention_site_t saved_csite = {0,0};
#ifndef DONT_SPEEDUP_PTHREAD_CONTENTION_PROFILER_WITH_TLS
- TLSPthreadContentionSites& fast_alt =
- *BAIDU_GET_PTR_VOLATILE_THREAD_LOCAL(tls_csites);
+ TLSPthreadContentionSites& fast_alt = tls_csites();
for (int i = fast_alt.count - 1; i >= 0; --i) {
if (fast_alt.list[i].mutex == mutex) {
if (is_contention_site_valid(fast_alt.list[i].csite)) {
@@ -1021,7 +1016,7 @@ BAIDU_CASSERT(sizeof(unsigned) == sizeof(MutexInternal),
#define BTHREAD_MUTEX_SET_OWNER
\
do {
\
- TaskGroup* task_group =
BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_task_group); \
+ TaskGroup* task_group = tls_task_group();
\
if (nullptr != task_group && !task_group->is_current_main_task()) {
\
m->owner.id = bthread_self();
\
} else {
\
@@ -1064,7 +1059,7 @@ inline int mutex_lock_contended_impl(bthread_mutex_t*
__restrict m,
BTHREAD_MUTEX_CHECK_OWNER;
// When a bthread first contends for a lock, active spinning makes sense.
// Spin only few times and only if local `rq' is empty.
- TaskGroup* g = BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_task_group);
+ TaskGroup* g = tls_task_group();
if (BAIDU_UNLIKELY(nullptr == g || g->rq_size() == 0)) {
for (int i = 0; i < MAX_SPIN_ITER; ++i) {
cpu_relax();
diff --git a/src/bthread/task_control.cpp b/src/bthread/task_control.cpp
index 320f8676..c409c9b1 100644
--- a/src/bthread/task_control.cpp
+++ b/src/bthread/task_control.cpp
@@ -71,7 +71,6 @@ DECLARE_int32(bthread_min_concurrency);
DECLARE_int32(bthread_parking_lot_of_each_tag);
extern pthread_mutex_t g_task_control_mutex;
-EXTERN_BAIDU_VOLATILE_THREAD_LOCAL(TaskGroup*, tls_task_group);
void (*g_worker_startfn)() = nullptr;
void (*g_tagged_worker_startfn)(bthread_tag_t) = nullptr;
@@ -129,7 +128,7 @@ void* TaskControl::worker_thread(void* arg) {
}
BT_VLOG << "Created worker=" << pthread_self() << " tid=" << g->_tid
<< " bthread=" << g->main_tid() << " tag=" << g->tag();
- BAIDU_SET_VOLATILE_THREAD_LOCAL(tls_task_group, g);
+ tls_task_group() = g;
c->_nworkers << 1;
c->tag_nworkers(g->tag()) << 1;
@@ -139,7 +138,7 @@ void* TaskControl::worker_thread(void* arg) {
BT_VLOG << "Destroying worker=" << pthread_self() << " bthread="
<< g->main_tid() << " idle=" << stat.cputime_ns / 1000000.0
<< "ms uptime=" << g->current_uptime_ns() / 1000000.0 << "ms";
- BAIDU_SET_VOLATILE_THREAD_LOCAL(tls_task_group, nullptr);
+ tls_task_group() = nullptr;
g->destroy_self();
c->_nworkers << -1;
c->tag_nworkers(g->tag()) << -1;
@@ -666,7 +665,7 @@ int TaskControl::_destroy_group(TaskGroup* g) {
}
bool TaskControl::steal_task(bthread_t* tid, size_t* seed, size_t offset) {
- auto tag = BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_task_group)->tag();
+ auto tag = tls_task_group()->tag();
if (_enable_priority_queue) {
for (int i = 0;
diff --git a/src/bthread/task_group.cpp b/src/bthread/task_group.cpp
index b2aefc26..dbeee4ee 100644
--- a/src/bthread/task_group.cpp
+++ b/src/bthread/task_group.cpp
@@ -69,18 +69,18 @@ DEFINE_bool(bthread_enable_cpu_clock_stat, false,
"Enable CPU clock statistics for bthread");
BUTIL_VALIDATE_GFLAG(bthread_enable_cpu_clock_stat, butil::PassValidate);
-BAIDU_VOLATILE_THREAD_LOCAL(TaskGroup*, tls_task_group, nullptr);
+BUTIL_THREAD_LOCAL(TaskGroup*, tls_task_group, nullptr);
// Sync with TaskMeta::local_storage when a bthread is created or destroyed.
// During running, the two fields may be inconsistent, use tls_bls as the
// groundtruth.
-BAIDU_VOLATILE_THREAD_LOCAL(LocalStorage, tls_bls,
BTHREAD_LOCAL_STORAGE_INITIALIZER);
+BUTIL_THREAD_LOCAL(LocalStorage, tls_bls, BTHREAD_LOCAL_STORAGE_INITIALIZER);
// defined in bthread/key.cpp
extern void return_keytable(bthread_keytable_pool_t*, KeyTable*);
// [Hacky] This is a special TLS set by bthread-rpc privately... to save
// overhead of creation keytable, may be removed later.
-BAIDU_VOLATILE_THREAD_LOCAL(void*, tls_unique_user_ptr, nullptr);
+BUTIL_THREAD_LOCAL(void*, tls_unique_user_ptr, nullptr);
const TaskStatistics EMPTY_STAT = { 0, 0, 0 };
@@ -320,7 +320,7 @@ void TaskGroup::asan_task_runner(intptr_t) {
void TaskGroup::task_runner(intptr_t skip_remained) {
// NOTE: tls_task_group is volatile since tasks are moved around
// different groups.
- TaskGroup* g = BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_task_group);
+ TaskGroup* g = tls_task_group();
#ifdef BRPC_BTHREAD_TRACER
TaskTracer::set_running_status(g->tid(), g->_cur_meta);
#endif // BRPC_BTHREAD_TRACER
@@ -330,7 +330,7 @@ void TaskGroup::task_runner(intptr_t skip_remained) {
RemainedFn fn = g->_last_context_remained;
g->_last_context_remained = nullptr;
fn(g->_last_context_remained_arg);
- g = BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_task_group);
+ g = tls_task_group();
}
#ifndef NDEBUG
@@ -413,7 +413,7 @@ void TaskGroup::task_runner(intptr_t skip_remained) {
// During running the function in TaskMeta and deleting the KeyTable in
// return_KeyTable, the group is probably changed.
- g = BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_task_group);
+ g = tls_task_group();
// Increase the version and wake up all joiners, if resulting version
// is 0, change it to 1 to make bthread_t never be 0. Any access
@@ -500,7 +500,7 @@ int TaskGroup::start_foreground(TaskGroup** pg,
if (g_create_bthread_span) {
m->local_storage.rpcz_parent_span = g_create_bthread_span();
} else {
- m->local_storage.rpcz_parent_span =
BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_bls).rpcz_parent_span;
+ m->local_storage.rpcz_parent_span = tls_bls().rpcz_parent_span;
}
}
m->cpuwide_start_ns = start_ns;
@@ -574,7 +574,7 @@ int TaskGroup::start_background(bthread_t* __restrict th,
if (g_create_bthread_span) {
m->local_storage.rpcz_parent_span = g_create_bthread_span();
} else {
- m->local_storage.rpcz_parent_span =
BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_bls).rpcz_parent_span;
+ m->local_storage.rpcz_parent_span = tls_bls().rpcz_parent_span;
}
}
m->cpuwide_start_ns = start_ns;
@@ -622,7 +622,7 @@ int TaskGroup::join(bthread_t tid, void** return_value) {
// The bthread is not created yet, this join is definitely wrong.
return EINVAL;
}
- TaskGroup* g = BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_task_group);
+ TaskGroup* g = tls_task_group();
if (g != nullptr && g->current_tid() == tid) {
// joining self causes indefinite waiting.
return EINVAL;
@@ -741,7 +741,7 @@ void TaskGroup::sched_to(TaskGroup** pg, TaskMeta*
next_meta) {
#endif
// Save errno so that errno is bthread-specific.
int saved_errno = errno;
- void* saved_unique_user_ptr =
BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_unique_user_ptr);
+ void* saved_unique_user_ptr = tls_unique_user_ptr();
TaskMeta* const cur_meta = g->_cur_meta;
int64_t now = butil::cpuwide_time_ns();
@@ -769,8 +769,8 @@ void TaskGroup::sched_to(TaskGroup** pg, TaskMeta*
next_meta) {
if (__builtin_expect(next_meta != cur_meta, 1)) {
g->_cur_meta = next_meta;
// Switch tls_bls
- cur_meta->local_storage = BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_bls);
- BAIDU_SET_VOLATILE_THREAD_LOCAL(tls_bls, next_meta->local_storage);
+ cur_meta->local_storage = tls_bls();
+ tls_bls() = next_meta->local_storage;
// Logging must be done after switching the local storage, since the
logging lib
// use bthread local storage internally, or will cause memory leak.
@@ -792,7 +792,7 @@ void TaskGroup::sched_to(TaskGroup** pg, TaskMeta*
next_meta) {
jump_stack(cur_meta->stack, next_meta->stack);
}
// probably went to another group, need to assign g again.
- g = BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_task_group);
+ g = tls_task_group();
#ifdef BRPC_BTHREAD_TRACER
TaskTracer::set_running_status(g->tid(), g->_cur_meta);
#endif // BRPC_BTHREAD_TRACER
@@ -819,13 +819,13 @@ void TaskGroup::sched_to(TaskGroup** pg, TaskMeta*
next_meta) {
RemainedFn fn = g->_last_context_remained;
g->_last_context_remained = nullptr;
fn(g->_last_context_remained_arg);
- g = BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_task_group);
+ g = tls_task_group();
}
// Restore errno
errno = saved_errno;
// tls_unique_user_ptr probably changed.
- BAIDU_SET_VOLATILE_THREAD_LOCAL(tls_unique_user_ptr,
saved_unique_user_ptr);
+ tls_unique_user_ptr() = saved_unique_user_ptr;
#ifndef NDEBUG
--g->_sched_recursive_guard;
@@ -904,14 +904,14 @@ void
TaskGroup::flush_nosignal_tasks_remote_locked(butil::Mutex& locked_mutex) {
}
void TaskGroup::ready_to_run_general(TaskMeta* meta, bool nosignal) {
- if (BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_task_group) == this) {
+ if (tls_task_group() == this) {
return ready_to_run(meta, nosignal);
}
return ready_to_run_remote(meta, nosignal);
}
void TaskGroup::flush_nosignal_tasks_general() {
- if (BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_task_group) == this) {
+ if (tls_task_group() == this) {
return flush_nosignal_tasks();
}
return flush_nosignal_tasks_remote();
@@ -919,13 +919,13 @@ void TaskGroup::flush_nosignal_tasks_general() {
void TaskGroup::ready_to_run_in_worker(void* args_in) {
ReadyToRunArgs* args = static_cast<ReadyToRunArgs*>(args_in);
- return BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_task_group)->
+ return tls_task_group()->
ready_to_run(args->meta, args->nosignal);
}
void TaskGroup::ready_to_run_in_worker_ignoresignal(void* args_in) {
ReadyToRunArgs* args = static_cast<ReadyToRunArgs*>(args_in);
- TaskGroup* g = BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_task_group);
+ TaskGroup* g = tls_task_group();
#ifdef BRPC_BTHREAD_TRACER
g->_control->_task_tracer.set_status(TASK_STATUS_READY, args->meta);
@@ -937,7 +937,7 @@ void TaskGroup::priority_to_run(void* args_in) {
// Publishing the parent lets another worker resume it immediately and
// reuse its stack. Copy the on-stack arguments before enqueueing it.
const ReadyToRunArgs args = *static_cast<ReadyToRunArgs*>(args_in);
- TaskGroup* g = BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_task_group);
+ TaskGroup* g = tls_task_group();
#ifdef BRPC_BTHREAD_TRACER
g->_control->_task_tracer.set_status(TASK_STATUS_READY, args.meta);
#endif // BRPC_BTHREAD_TRACER
@@ -959,7 +959,7 @@ struct SleepArgs {
};
static void ready_to_run_from_timer_thread(void* arg) {
- CHECK(BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_task_group) == nullptr);
+ CHECK(tls_task_group() == nullptr);
const SleepArgs* e = static_cast<const SleepArgs*>(arg);
TaskGroup* g = e->group;
bthread_tag_t tag = g->tag();
@@ -1109,7 +1109,7 @@ int TaskGroup::interrupt(bthread_t tid, TaskControl* c) {
}
} else if (sleep_id != 0) {
if (get_global_timer_thread()->unschedule(sleep_id) == 0) {
- TaskGroup* g = BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_task_group);
+ TaskGroup* g = tls_task_group();
TaskMeta* m = address_meta(tid);
if (g) {
g->ready_to_run(m);
diff --git a/src/bthread/task_group.h b/src/bthread/task_group.h
index bbd76ae7..d7373dd8 100644
--- a/src/bthread/task_group.h
+++ b/src/bthread/task_group.h
@@ -34,6 +34,10 @@
namespace bthread {
+class TaskGroup;
+// Resolve the current worker again after any operation that may suspend.
+TaskGroup*& tls_task_group();
+
// For exiting a bthread.
class ExitException : public std::exception {
public:
diff --git a/src/bthread/task_meta.h b/src/bthread/task_meta.h
index 7ca52e04..2b02bb15 100644
--- a/src/bthread/task_meta.h
+++ b/src/bthread/task_meta.h
@@ -51,10 +51,10 @@ struct LocalStorage {
const static LocalStorage LOCAL_STORAGE_INIT =
BTHREAD_LOCAL_STORAGE_INITIALIZER;
-EXTERN_BAIDU_VOLATILE_THREAD_LOCAL(LocalStorage, tls_bls);
+LocalStorage& tls_bls();
inline LocalStorage* tls_bls_ptr() {
- return BAIDU_GET_PTR_VOLATILE_THREAD_LOCAL(tls_bls);
+ return &tls_bls();
}
enum TaskStatus {
diff --git a/src/bthread/task_tracer.cpp b/src/bthread/task_tracer.cpp
index afb1642d..b3fd29be 100644
--- a/src/bthread/task_tracer.cpp
+++ b/src/bthread/task_tracer.cpp
@@ -59,7 +59,7 @@ BUTIL_VALIDATE_GFLAG(signal_trace_timeout_ms,
butil::PositiveInteger<uint32_t>);
DEFINE_int32(signal_number_for_trace, SIGURG,
"signal number used for stack trace, default to SIGURG");
-extern BAIDU_THREAD_LOCAL TaskMeta* pthread_fake_meta;
+TaskMeta*& pthread_fake_meta();
TaskTracer::SignalSync::~SignalSync() {
if (pipe_fds[0] >= 0) {
@@ -226,7 +226,7 @@ TaskTracer::Result TaskTracer::TraceImpl(bthread_t tid) {
};
if (tid == bthread_self() ||
- (nullptr != pthread_fake_meta && tid == pthread_fake_meta->tid)) {
+ (nullptr != pthread_fake_meta() && tid == pthread_fake_meta()->tid)) {
return Result::MakeErrorResult("Forbid to trace self=%d", tid);
}
diff --git a/src/butil/containers/doubly_buffered_data.h
b/src/butil/containers/doubly_buffered_data.h
index 410da3f0..53fae0b4 100644
--- a/src/butil/containers/doubly_buffered_data.h
+++ b/src/butil/containers/doubly_buffered_data.h
@@ -239,33 +239,36 @@ public:
CHECK(false) << "Invalid id=" << id;
return nullptr;
}
- if (_s_tls_blocks == nullptr) {
- _s_tls_blocks = new std::vector<ThreadBlock*>;
+ auto* blocks = _s_tls_blocks();
+ if (blocks == nullptr) {
+ blocks = new std::vector<ThreadBlock*>;
+ _s_tls_blocks() = blocks;
butil::thread_atexit(_destroy_tls_blocks);
}
const size_t block_id = (size_t)id / ELEMENTS_PER_BLOCK;
- if (block_id >= _s_tls_blocks->size()) {
+ if (block_id >= blocks->size()) {
// The 32ul avoid pointless small resizes.
- _s_tls_blocks->resize(std::max(block_id + 1, 32ul));
+ blocks->resize(std::max(block_id + 1, 32ul));
}
- ThreadBlock* tb = (*_s_tls_blocks)[block_id];
+ ThreadBlock* tb = (*blocks)[block_id];
if (tb == nullptr) {
tb = new ThreadBlock;
- (*_s_tls_blocks)[block_id] = tb;
+ (*blocks)[block_id] = tb;
}
return tb->at(id - block_id * ELEMENTS_PER_BLOCK);
}
private:
static void _destroy_tls_blocks() {
- if (!_s_tls_blocks) {
+ auto* blocks = _s_tls_blocks();
+ if (!blocks) {
return;
}
- for (size_t i = 0; i < _s_tls_blocks->size(); ++i) {
- delete (*_s_tls_blocks)[i];
+ for (size_t i = 0; i < blocks->size(); ++i) {
+ delete (*blocks)[i];
}
- delete _s_tls_blocks;
- _s_tls_blocks = nullptr;
+ delete blocks;
+ _s_tls_blocks() = nullptr;
}
inline static std::deque<WrapperTLSId>& _get_free_ids() {
@@ -280,7 +283,7 @@ private:
static pthread_mutex_t _s_mutex;
static WrapperTLSId _s_id;
static std::deque<WrapperTLSId>* _s_free_ids;
- static __thread std::vector<ThreadBlock*>* _s_tls_blocks;
+ static BUTIL_THREAD_LOCAL(std::vector<ThreadBlock*>*, _s_tls_blocks,
nullptr);
};
template <typename T, typename TLS, bool AllowBthreadSuspended>
@@ -294,10 +297,6 @@ template <typename T, typename TLS, bool
AllowBthreadSuspended>
typename DoublyBufferedData<T, TLS, AllowBthreadSuspended>::WrapperTLSId
DoublyBufferedData<T, TLS,
AllowBthreadSuspended>::WrapperTLSGroup::_s_id = 0;
-template <typename T, typename TLS, bool AllowBthreadSuspended>
-__thread std::vector<typename DoublyBufferedData<T, TLS,
AllowBthreadSuspended>::WrapperTLSGroup::ThreadBlock*>*
- DoublyBufferedData<T, TLS,
AllowBthreadSuspended>::WrapperTLSGroup::_s_tls_blocks = nullptr;
-
template <typename T, typename TLS, bool AllowBthreadSuspended>
class BAIDU_CACHELINE_ALIGNMENT DoublyBufferedData<T, TLS,
AllowBthreadSuspended>::Wrapper
: public DoublyBufferedDataWrapperBase<T, TLS> {
diff --git a/src/butil/containers/flat_map.h b/src/butil/containers/flat_map.h
index ca86dfd7..bf7c75b8 100644
--- a/src/butil/containers/flat_map.h
+++ b/src/butil/containers/flat_map.h
@@ -611,26 +611,26 @@ template <typename _T, typename _Hash, typename _Equal,
bool _Sparse>
const _T* find_lowered_cstr(
const FlatMap<std::string, _T, _Hash, _Equal, _Sparse>& m,
const char* key) {
- return m.seek(*tls_stringmap_temp.get_lowered_string(key));
+ return m.seek(*tls_stringmap_temp().get_lowered_string(key));
}
template <typename _T, typename _Hash, typename _Equal, bool _Sparse>
_T* find_lowered_cstr(FlatMap<std::string, _T, _Hash, _Equal, _Sparse>& m,
const char* key) {
- return m.seek(*tls_stringmap_temp.get_lowered_string(key));
+ return m.seek(*tls_stringmap_temp().get_lowered_string(key));
}
template <typename _T, typename _Hash, typename _Equal, bool _Sparse>
const _T* find_lowered_cstr(
const FlatMap<std::string, _T, _Hash, _Equal, _Sparse>& m,
const char* key, size_t length) {
- return m.seek(*tls_stringmap_temp.get_lowered_string(key, length));
+ return m.seek(*tls_stringmap_temp().get_lowered_string(key, length));
}
template <typename _T, typename _Hash, typename _Equal, bool _Sparse>
_T* find_lowered_cstr(FlatMap<std::string, _T, _Hash, _Equal, _Sparse>& m,
const char* key, size_t length) {
- return m.seek(*tls_stringmap_temp.get_lowered_string(key, length));
+ return m.seek(*tls_stringmap_temp().get_lowered_string(key, length));
}
} // namespace butil
diff --git a/src/butil/errno.cpp b/src/butil/errno.cpp
index 9b964e11..8afcf20a 100644
--- a/src/butil/errno.cpp
+++ b/src/butil/errno.cpp
@@ -25,6 +25,7 @@
#include <pthread.h> // pthread_mutex_t
#include <unistd.h> // _exit
#include "butil/scoped_lock.h" // BAIDU_SCOPED_LOCK
+#include "butil/thread_local.h"
namespace butil {
@@ -34,7 +35,8 @@ static const char* errno_desc[ERRNO_END - ERRNO_BEGIN] = {};
static pthread_mutex_t modify_desc_mutex = PTHREAD_MUTEX_INITIALIZER;
const size_t ERROR_BUFSIZE = 64;
-__thread char tls_error_buf[ERROR_BUFSIZE];
+using ErrorBuffer = char[ERROR_BUFSIZE];
+static BUTIL_THREAD_LOCAL(ErrorBuffer, tls_error_buf, {});
int DescribeCustomizedErrno(
int error_code, const char* error_name, const char* description) {
@@ -53,10 +55,10 @@ int DescribeCustomizedErrno(
}
} else {
#if defined(OS_MACOSX)
- const int rc = strerror_r(error_code, tls_error_buf, ERROR_BUFSIZE);
+ const int rc = strerror_r(error_code, tls_error_buf(), ERROR_BUFSIZE);
if (rc != EINVAL)
#else
- desc = strerror_r(error_code, tls_error_buf, ERROR_BUFSIZE);
+ desc = strerror_r(error_code, tls_error_buf(), ERROR_BUFSIZE);
if (desc && strncmp(desc, "Unknown error", 13) != 0)
#endif
{
@@ -80,20 +82,20 @@ const char* berror(int error_code) {
return s;
}
#if defined(OS_MACOSX)
- const int rc = strerror_r(error_code, butil::tls_error_buf,
butil::ERROR_BUFSIZE);
+ const int rc = strerror_r(error_code, butil::tls_error_buf(),
butil::ERROR_BUFSIZE);
if (rc == 0 || rc == ERANGE/*bufsize is not long enough*/) {
- return butil::tls_error_buf;
+ return butil::tls_error_buf();
}
#else
- s = strerror_r(error_code, butil::tls_error_buf, butil::ERROR_BUFSIZE);
+ s = strerror_r(error_code, butil::tls_error_buf(),
butil::ERROR_BUFSIZE);
if (s) {
return s;
}
#endif
}
- snprintf(butil::tls_error_buf, butil::ERROR_BUFSIZE,
+ snprintf(butil::tls_error_buf(), butil::ERROR_BUFSIZE,
"Unknown error %d", error_code);
- return butil::tls_error_buf;
+ return butil::tls_error_buf();
}
const char* berror() {
diff --git a/src/butil/fast_rand.cpp b/src/butil/fast_rand.cpp
index cef45854..e3220f30 100644
--- a/src/butil/fast_rand.cpp
+++ b/src/butil/fast_rand.cpp
@@ -23,6 +23,7 @@
#include "butil/macros.h"
#include "butil/time.h" // gettimeofday_us()
#include "butil/fast_rand.h"
+#include "butil/thread_local.h"
#include "butil/numerics/safe_conversions.h" // safe_abs
namespace butil {
@@ -78,7 +79,7 @@ inline uint64_t fast_rand_impl(uint64_t range, FastRandSeed*
seed) {
}
// Seeds for different threads are stored separately in thread-local storage.
-static __thread FastRandSeed _tls_seed = { { 0, 0 } };
+static BUTIL_THREAD_LOCAL(FastRandSeed, tls_seed, { { 0, 0 } });
// True if the seed is (probably) uninitialized. There's definitely false
// positive, but it's OK for us.
@@ -87,10 +88,11 @@ inline bool need_init(const FastRandSeed& seed) {
}
uint64_t fast_rand() {
- if (need_init(_tls_seed)) {
- init_fast_rand_seed(&_tls_seed);
+ FastRandSeed& seed = tls_seed();
+ if (need_init(seed)) {
+ init_fast_rand_seed(&seed);
}
- return xorshift128_next(&_tls_seed);
+ return xorshift128_next(&seed);
}
uint64_t fast_rand(FastRandSeed* seed) {
@@ -101,15 +103,17 @@ uint64_t fast_rand_less_than(uint64_t range) {
if (range == 0) {
return 0;
}
- if (need_init(_tls_seed)) {
- init_fast_rand_seed(&_tls_seed);
+ FastRandSeed& seed = tls_seed();
+ if (need_init(seed)) {
+ init_fast_rand_seed(&seed);
}
- return fast_rand_impl(range, &_tls_seed);
+ return fast_rand_impl(range, &seed);
}
int64_t fast_rand_in_64(int64_t min, int64_t max) {
- if (need_init(_tls_seed)) {
- init_fast_rand_seed(&_tls_seed);
+ FastRandSeed& seed = tls_seed();
+ if (need_init(seed)) {
+ init_fast_rand_seed(&seed);
}
if (BAIDU_UNLIKELY(min >= max)) {
if (min == max) {
@@ -121,7 +125,7 @@ int64_t fast_rand_in_64(int64_t min, int64_t max) {
if (min >= 0) {
// Always safe to do subtraction.
range = (uint64_t)(max - min) + 1;
- return min + (int64_t)fast_rand_impl(range, &_tls_seed);
+ return min + (int64_t)fast_rand_impl(range, &seed);
}
uint64_t abs_min = safe_abs(min);
@@ -132,15 +136,16 @@ int64_t fast_rand_in_64(int64_t min, int64_t max) {
}
if (range == 0) {
// max = INT64_MAX, min = INT64_MIN
- return (int64_t)xorshift128_next(&_tls_seed);
+ return (int64_t)xorshift128_next(&seed);
}
- uint64_t r = fast_rand_impl(range, &_tls_seed);
+ uint64_t r = fast_rand_impl(range, &seed);
return r >= abs_min ? (int64_t)(r - abs_min) : -((int64_t)(abs_min - r));
}
uint64_t fast_rand_in_u64(uint64_t min, uint64_t max) {
- if (need_init(_tls_seed)) {
- init_fast_rand_seed(&_tls_seed);
+ FastRandSeed& seed = tls_seed();
+ if (need_init(seed)) {
+ init_fast_rand_seed(&seed);
}
if (min >= max) {
if (min == max) {
@@ -153,9 +158,9 @@ uint64_t fast_rand_in_u64(uint64_t min, uint64_t max) {
uint64_t range = max - min + 1;
if (range == 0) {
// max = UINT64_MAX, min = UINT64_MIN
- return xorshift128_next(&_tls_seed);
+ return xorshift128_next(&seed);
}
- return min + fast_rand_impl(range, &_tls_seed);
+ return min + fast_rand_impl(range, &seed);
}
inline double fast_rand_double(FastRandSeed* seed) {
@@ -168,10 +173,11 @@ inline double fast_rand_double(FastRandSeed* seed) {
}
double fast_rand_double() {
- if (need_init(_tls_seed)) {
- init_fast_rand_seed(&_tls_seed);
+ FastRandSeed& seed = tls_seed();
+ if (need_init(seed)) {
+ init_fast_rand_seed(&seed);
}
- return fast_rand_double(&_tls_seed);
+ return fast_rand_double(&seed);
}
void fast_rand_bytes(void* output, size_t output_length) {
diff --git a/src/butil/find_cstr.cpp b/src/butil/find_cstr.cpp
index 82b2db95..5d30b388 100644
--- a/src/butil/find_cstr.cpp
+++ b/src/butil/find_cstr.cpp
@@ -21,6 +21,6 @@
namespace butil {
-thread_local StringMapThreadLocalTemp tls_stringmap_temp = { false, {} };
+BUTIL_THREAD_LOCAL(StringMapThreadLocalTemp, tls_stringmap_temp, { false, {}
});
} // namespace butil
diff --git a/src/butil/find_cstr.h b/src/butil/find_cstr.h
index 103f9e70..75a09054 100644
--- a/src/butil/find_cstr.h
+++ b/src/butil/find_cstr.h
@@ -103,58 +103,58 @@ struct StringMapThreadLocalTemp {
}
};
-extern thread_local StringMapThreadLocalTemp tls_stringmap_temp;
+StringMapThreadLocalTemp& tls_stringmap_temp();
template <typename T, typename C, typename A>
typename std::map<std::string, T, C, A>::const_iterator
find_cstr(const std::map<std::string, T, C, A>& m, const char* key) {
- return m.find(*tls_stringmap_temp.get_string(key));
+ return m.find(*tls_stringmap_temp().get_string(key));
}
template <typename T, typename C, typename A>
typename std::map<std::string, T, C, A>::iterator
find_cstr(std::map<std::string, T, C, A>& m, const char* key) {
- return m.find(*tls_stringmap_temp.get_string(key));
+ return m.find(*tls_stringmap_temp().get_string(key));
}
template <typename T, typename C, typename A>
typename std::map<std::string, T, C, A>::const_iterator
find_cstr(const std::map<std::string, T, C, A>& m,
const char* key, size_t length) {
- return m.find(*tls_stringmap_temp.get_string(key, length));
+ return m.find(*tls_stringmap_temp().get_string(key, length));
}
template <typename T, typename C, typename A>
typename std::map<std::string, T, C, A>::iterator
find_cstr(std::map<std::string, T, C, A>& m,
const char* key, size_t length) {
- return m.find(*tls_stringmap_temp.get_string(key, length));
+ return m.find(*tls_stringmap_temp().get_string(key, length));
}
template <typename T, typename C, typename A>
typename std::map<std::string, T, C, A>::const_iterator
find_lowered_cstr(const std::map<std::string, T, C, A>& m, const char* key) {
- return m.find(*tls_stringmap_temp.get_lowered_string(key));
+ return m.find(*tls_stringmap_temp().get_lowered_string(key));
}
template <typename T, typename C, typename A>
typename std::map<std::string, T, C, A>::iterator
find_lowered_cstr(std::map<std::string, T, C, A>& m, const char* key) {
- return m.find(*tls_stringmap_temp.get_lowered_string(key));
+ return m.find(*tls_stringmap_temp().get_lowered_string(key));
}
template <typename T, typename C, typename A>
typename std::map<std::string, T, C, A>::const_iterator
find_lowered_cstr(const std::map<std::string, T, C, A>& m,
const char* key, size_t length) {
- return m.find(*tls_stringmap_temp.get_lowered_string(key, length));
+ return m.find(*tls_stringmap_temp().get_lowered_string(key, length));
}
template <typename T, typename C, typename A>
typename std::map<std::string, T, C, A>::iterator
find_lowered_cstr(std::map<std::string, T, C, A>& m,
const char* key, size_t length) {
- return m.find(*tls_stringmap_temp.get_lowered_string(key, length));
+ return m.find(*tls_stringmap_temp().get_lowered_string(key, length));
}
} // namespace butil
diff --git a/src/butil/iobuf.cpp b/src/butil/iobuf.cpp
index 685a7a49..b0b5d449 100644
--- a/src/butil/iobuf.cpp
+++ b/src/butil/iobuf.cpp
@@ -296,13 +296,13 @@ inline IOBuf::Block* create_block_aligned(size_t
block_size, size_t alignment) {
// === Share TLS blocks between appending operations ===
-static __thread TLSData g_tls_data = { nullptr, 0, false };
+static BUTIL_THREAD_LOCAL(TLSData, g_tls_data, { nullptr, 0, false });
// Used in release_tls_block()
-TLSData* get_g_tls_data() { return &g_tls_data; }
+TLSData* get_g_tls_data() { return &g_tls_data(); }
// Used in UT
-IOBuf::Block* get_tls_block_head() { return g_tls_data.block_head; }
-int get_tls_block_count() { return g_tls_data.num_blocks; }
+IOBuf::Block* get_tls_block_head() { return g_tls_data().block_head; }
+int get_tls_block_count() { return g_tls_data().num_blocks; }
// Number of blocks that can't be returned to TLS which has too many block
// already. This counter should be 0 in most scenarios, otherwise performance
@@ -319,7 +319,7 @@ void dec_g_num_hit_tls_threshold() {
// Called in UT.
void remove_tls_block_chain() {
- TLSData& tls_data = g_tls_data;
+ TLSData& tls_data = g_tls_data();
IOBuf::Block* b = tls_data.block_head;
if (!b) {
return;
@@ -339,7 +339,7 @@ void remove_tls_block_chain() {
// Get a (non-full) block from TLS.
// Notice that the block is not removed from TLS.
IOBuf::Block* share_tls_block() {
- TLSData& tls_data = g_tls_data;
+ TLSData& tls_data = g_tls_data();
IOBuf::Block* const b = tls_data.block_head;
if (b != nullptr && !b->full()) {
return b;
@@ -371,7 +371,7 @@ IOBuf::Block* share_tls_block() {
// Return chained blocks to TLS.
// NOTE: b MUST be non-nullptr and all blocks linked SHOULD not be full.
void release_tls_block_chain(IOBuf::Block* b) {
- TLSData& tls_data = g_tls_data;
+ TLSData& tls_data = g_tls_data();
size_t n = 0;
if (tls_data.num_blocks >= max_blocks_per_thread()) {
do {
@@ -405,7 +405,7 @@ void release_tls_block_chain(IOBuf::Block* b) {
// Get and remove one (non-full) block from TLS. If TLS is empty, create one.
IOBuf::Block* acquire_tls_block() {
- TLSData& tls_data = g_tls_data;
+ TLSData& tls_data = g_tls_data();
IOBuf::Block* b = tls_data.block_head;
if (!b) {
return create_block();
diff --git a/src/butil/logging.cc b/src/butil/logging.cc
index 4d02a25d..159b8eb5 100644
--- a/src/butil/logging.cc
+++ b/src/butil/logging.cc
@@ -196,8 +196,8 @@ bool show_error_dialogs = false;
// the debug message dialog and process termination.
LogAssertHandler log_assert_handler = nullptr;
-BAIDU_VOLATILE_THREAD_LOCAL(int32_t, tls_log_pid, 0);
-BAIDU_VOLATILE_THREAD_LOCAL(butil::PlatformThreadId, tls_log_tid, 0);
+BUTIL_THREAD_LOCAL(int32_t, tls_log_pid, 0);
+BUTIL_THREAD_LOCAL(butil::PlatformThreadId, tls_log_tid, 0);
// Helper functions to wrap platform differences.
@@ -868,17 +868,17 @@ void PrintLogPrefix(std::ostream& os, int severity,
os << '.' << std::setw(6) << tv.tv_usec;
#endif
if (FLAGS_log_pid) {
- int32_t pid = BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_log_pid);
+ int32_t pid = tls_log_pid();
if (pid == 0) {
pid = CurrentProcessId();
- BAIDU_SET_VOLATILE_THREAD_LOCAL(tls_log_pid, pid);
+ tls_log_pid() = pid;
}
os << ' ' << std::setfill(' ') << std::setw(5) << pid;
}
- butil::PlatformThreadId tid = BAIDU_GET_VOLATILE_THREAD_LOCAL(tls_log_tid);
+ butil::PlatformThreadId tid = tls_log_tid();
if (tid == 0) {
tid = butil::PlatformThread::CurrentId();
- BAIDU_SET_VOLATILE_THREAD_LOCAL(tls_log_tid, tid);
+ tls_log_tid() = tid;
}
os << ' ' << std::setfill(' ') << std::setw(5) << tid << std::setfill('0');
if (FLAGS_log_bid && bthread_self) {
diff --git a/src/butil/object_pool_inl.h b/src/butil/object_pool_inl.h
index 15a95870..97a63d2c 100644
--- a/src/butil/object_pool_inl.h
+++ b/src/butil/object_pool_inl.h
@@ -30,7 +30,7 @@
#include "butil/atomicops.h" // butil::atomic
#include "butil/macros.h" // BAIDU_CACHELINE_ALIGNMENT
#include "butil/scoped_lock.h" // BAIDU_SCOPED_LOCK
-#include "butil/thread_local.h" // BAIDU_THREAD_LOCAL
+#include "butil/thread_local.h" // BUTIL_THREAD_LOCAL
#include "butil/time.h" // monotonic_time_us
#include "butil/memory/aligned_memory.h" // butil::AlignedMemory
#include "butil/debug/address_annotations.h"
@@ -299,9 +299,9 @@ public:
}
void clear_objects() {
- LocalPool* lp = _local_pool;
+ LocalPool* lp = _local_pool();
if (lp) {
- _local_pool = nullptr;
+ _local_pool() = nullptr;
butil::thread_atexit_cancel(LocalPool::delete_local_pool, lp);
delete lp;
}
@@ -509,13 +509,13 @@ private:
}
inline LocalPool* get_or_new_local_pool() {
- LocalPool* lp = BAIDU_GET_VOLATILE_THREAD_LOCAL(_local_pool);
+ LocalPool* lp = _local_pool();
if (BAIDU_LIKELY(lp != nullptr)) {
return lp;
}
lp = new LocalPool(this);
BAIDU_SCOPED_LOCK(_change_thread_mutex); //avoid race with clear()
- BAIDU_SET_VOLATILE_THREAD_LOCAL(_local_pool, lp);
+ _local_pool() = lp;
butil::thread_atexit(LocalPool::delete_local_pool, lp);
_nlocal.fetch_add(1, butil::memory_order_relaxed);
return lp;
@@ -523,7 +523,7 @@ private:
void clear_from_destructor_of_local_pool() {
// Remove tls
- _local_pool = nullptr;
+ _local_pool() = nullptr;
// Do nothing if there're active threads.
if (_nlocal.fetch_sub(1, butil::memory_order_relaxed) != 1) {
@@ -629,7 +629,7 @@ private:
static butil::static_atomic<ObjectPool*> _singleton;
static pthread_mutex_t _singleton_mutex;
- STATIC_MEMBER_BAIDU_VOLATILE_THREAD_LOCAL(LocalPool*, _local_pool);
+ static BUTIL_THREAD_LOCAL(LocalPool*, _local_pool, nullptr);
static butil::static_atomic<long> _nlocal;
static butil::static_atomic<size_t> _ngroup;
static pthread_mutex_t _block_group_mutex;
@@ -652,10 +652,6 @@ private:
template <typename T>
const size_t ObjectPool<T>::FREE_CHUNK_NITEM;
-template <typename T>
-BAIDU_THREAD_LOCAL typename ObjectPool<T>::LocalPool*
-ObjectPool<T>::_local_pool = nullptr;
-
template <typename T>
butil::static_atomic<ObjectPool<T>*> ObjectPool<T>::_singleton =
BUTIL_STATIC_ATOMIC_INIT(nullptr);
diff --git a/src/butil/resource_pool_inl.h b/src/butil/resource_pool_inl.h
index 1c75a086..321daa19 100644
--- a/src/butil/resource_pool_inl.h
+++ b/src/butil/resource_pool_inl.h
@@ -303,9 +303,9 @@ public:
}
void clear_resources() {
- LocalPool* lp = _local_pool;
+ LocalPool* lp = _local_pool();
if (lp) {
- _local_pool = nullptr;
+ _local_pool() = nullptr;
butil::thread_atexit_cancel(LocalPool::delete_local_pool, lp);
delete lp;
}
@@ -450,7 +450,7 @@ private:
}
inline LocalPool* get_or_new_local_pool() {
- LocalPool* lp = BAIDU_GET_VOLATILE_THREAD_LOCAL(_local_pool);
+ LocalPool* lp = _local_pool();
if (lp != nullptr) {
return lp;
}
@@ -459,7 +459,7 @@ private:
return nullptr;
}
BAIDU_SCOPED_LOCK(_change_thread_mutex); //avoid race with clear()
- BAIDU_SET_VOLATILE_THREAD_LOCAL(_local_pool, lp);
+ _local_pool() = lp;
butil::thread_atexit(LocalPool::delete_local_pool, lp);
_nlocal.fetch_add(1, butil::memory_order_relaxed);
return lp;
@@ -467,7 +467,7 @@ private:
void clear_from_destructor_of_local_pool() {
// Remove tls
- _local_pool = nullptr;
+ _local_pool() = nullptr;
if (_nlocal.fetch_sub(1, butil::memory_order_relaxed) != 1) {
return;
@@ -558,7 +558,7 @@ private:
static butil::static_atomic<ResourcePool*> _singleton;
static pthread_mutex_t _singleton_mutex;
- STATIC_MEMBER_BAIDU_VOLATILE_THREAD_LOCAL(LocalPool*, _local_pool);
+ static BUTIL_THREAD_LOCAL(LocalPool*, _local_pool, nullptr);
static butil::static_atomic<long> _nlocal;
static butil::static_atomic<size_t> _ngroup;
static pthread_mutex_t _block_group_mutex;
@@ -579,10 +579,6 @@ private:
template <typename T>
const size_t ResourcePool<T>::FREE_CHUNK_NITEM;
-template <typename T>
-BAIDU_THREAD_LOCAL typename ResourcePool<T>::LocalPool*
-ResourcePool<T>::_local_pool = nullptr;
-
template <typename T>
butil::static_atomic<ResourcePool<T>*> ResourcePool<T>::_singleton =
BUTIL_STATIC_ATOMIC_INIT(nullptr);
diff --git a/src/butil/thread_key.cpp b/src/butil/thread_key.cpp
index 22c57216..621cf061 100644
--- a/src/butil/thread_key.cpp
+++ b/src/butil/thread_key.cpp
@@ -38,7 +38,7 @@ pthread_mutex_t g_thread_key_mutex =
PTHREAD_MUTEX_INITIALIZER;
static size_t g_id = 0;
static std::deque<size_t>* g_free_ids = nullptr;
static std::vector<ThreadKeyInfo>* g_thread_keys = nullptr;
-static __thread std::vector<ThreadKeyTLS>* thread_key_tls_data = nullptr;
+static BUTIL_THREAD_LOCAL(std::vector<ThreadKeyTLS>*, thread_key_tls_data,
nullptr);
void detail::DestroyThreadKeyData(const std::vector<ThreadKeyInfo>& keys,
std::vector<ThreadKeyTLS>* tls_data) {
@@ -67,7 +67,8 @@ bool ThreadKey::Valid() const {
}
static void DestroyTlsData() {
- if (!thread_key_tls_data) {
+ auto* tls_data = thread_key_tls_data();
+ if (!tls_data) {
return;
}
std::vector<ThreadKeyInfo> dummy_keys;
@@ -77,9 +78,9 @@ static void DestroyTlsData() {
g_thread_keys->begin(),
g_thread_keys->end());
}
- detail::DestroyThreadKeyData(dummy_keys, thread_key_tls_data);
- delete thread_key_tls_data;
- thread_key_tls_data = nullptr;
+ detail::DestroyThreadKeyData(dummy_keys, tls_data);
+ delete tls_data;
+ thread_key_tls_data() = nullptr;
}
int thread_key_create(ThreadKey& thread_key, DtorFunction dtor) {
@@ -143,19 +144,21 @@ int thread_setspecific(ThreadKey& thread_key, void* data)
{
}
size_t id = thread_key._id;
size_t seq = thread_key._seq;
- if (BAIDU_UNLIKELY(!thread_key_tls_data)) {
- thread_key_tls_data = new std::vector<ThreadKeyTLS>;
- thread_key_tls_data->reserve(THREAD_KEY_RESERVE);
+ auto* tls_data = thread_key_tls_data();
+ if (BAIDU_UNLIKELY(!tls_data)) {
+ tls_data = new std::vector<ThreadKeyTLS>;
+ thread_key_tls_data() = tls_data;
+ tls_data->reserve(THREAD_KEY_RESERVE);
// Register the destructor of tls_data in this thread.
butil::thread_atexit(DestroyTlsData);
}
- if (id >= thread_key_tls_data->size()) {
- thread_key_tls_data->resize(id + 1);
+ if (id >= tls_data->size()) {
+ tls_data->resize(id + 1);
}
- (*thread_key_tls_data)[id].seq = seq;
- (*thread_key_tls_data)[id].data = data;
+ (*tls_data)[id].seq = seq;
+ (*tls_data)[id].data = data;
return 0;
}
@@ -166,13 +169,14 @@ void* thread_getspecific(ThreadKey& thread_key) {
}
size_t id = thread_key._id;
size_t seq = thread_key._seq;
- if (BAIDU_UNLIKELY(!thread_key_tls_data ||
- id >= thread_key_tls_data->size() ||
- (*thread_key_tls_data)[id].seq != seq)){
+ auto* tls_data = thread_key_tls_data();
+ if (BAIDU_UNLIKELY(!tls_data ||
+ id >= tls_data->size() ||
+ (*tls_data)[id].seq != seq)){
return nullptr;
}
- return (*thread_key_tls_data)[id].data;
+ return (*tls_data)[id].data;
}
} // namespace butil
\ No newline at end of file
diff --git a/src/butil/thread_local.h b/src/butil/thread_local.h
index 71c65796..19fb0bcb 100644
--- a/src/butil/thread_local.h
+++ b/src/butil/thread_local.h
@@ -20,6 +20,7 @@
#ifndef BUTIL_THREAD_LOCAL_H
#define BUTIL_THREAD_LOCAL_H
+#include <memory> // std::addressof
#include <new> // std::nothrow
#include "butil/macros.h"
@@ -29,6 +30,34 @@
#define BAIDU_THREAD_LOCAL __thread
#endif // _MSC_VER
+// Define a pthread-local variable accessed through name():
+// BUTIL_THREAD_LOCAL(int, counter, 0);
+// ++counter();
+// counter() = 10;
+// Prefix with static for internal linkage or a static class member, or inline
+// for a definition in a header. A declaration in another translation unit is
+// simply `type& name();`. Use a type alias for arrays or types containing
commas.
+//
+// A bthread can resume on a different pthread. Keep TLS address resolution in
+// a non-inlined function so the compiler cannot hoist it across a suspension.
+// The opaque asm output also prevents interprocedural optimization (including
+// LTO) from treating the returned address as invariant. A memory barrier
alone,
+// or an inline function taking &tls as its argument, does not achieve this.
+//
+// The returned reference/pointer is only valid for the current pthread: never
+// retain it across a call that may suspend. In particular, evaluate a possibly
+// suspending RHS before assigning it to name(). Initializers must not suspend.
+// Constant-initialized trivial types have no dynamic initialization/exit hook.
+#define BUTIL_THREAD_LOCAL(type, name, ...) \
+ __attribute__((noinline, unused)) type& name() { \
+ static thread_local type tls_value = __VA_ARGS__; \
+ type* tls_ptr = std::addressof(tls_value); \
+ asm volatile("" : "+r"(tls_ptr) : : "memory"); \
+ return *tls_ptr; \
+ }
+
+// Legacy interfaces retained for source compatibility. Prefer
BUTIL_THREAD_LOCAL
+// for new code: it hides the storage and needs no separate get/set/ptr macros.
#define BAIDU_VOLATILE_THREAD_LOCAL(type, var_name, default_value)
\
BAIDU_THREAD_LOCAL type var_name = default_value;
\
__attribute__((noinline, unused)) type get_##var_name(void) {
\
@@ -61,10 +90,8 @@
var_name = v;
\
}
-#if (defined (__aarch64__) && defined (__GNUC__)) || defined(__clang__)
-// GNU compiler under aarch and Clang compiler is incorrectly caching the
-// address of thread_local variables across a suspend-point. The following
-// macros used to disable the volatile thread local access optimization.
+// Always use the accessors to prevent TLS address caching across suspension,
+// regardless of compiler or architecture.
#define BAIDU_GET_VOLATILE_THREAD_LOCAL(var_name) get_##var_name()
#define BAIDU_GET_PTR_VOLATILE_THREAD_LOCAL(var_name) get_ptr_##var_name()
#define BAIDU_SET_VOLATILE_THREAD_LOCAL(var_name, value) set_##var_name(value)
@@ -73,13 +100,6 @@
extern type get_##var_name(void);
\
extern type *get_ptr_##var_name(void);
\
extern void set_##var_name(type v)
-#else
-#define BAIDU_GET_VOLATILE_THREAD_LOCAL(var_name) var_name
-#define BAIDU_GET_PTR_VOLATILE_THREAD_LOCAL(var_name) &var_name
-#define BAIDU_SET_VOLATILE_THREAD_LOCAL(var_name, value) var_name = value
-#define EXTERN_BAIDU_VOLATILE_THREAD_LOCAL(type, var_name)
\
- extern BAIDU_THREAD_LOCAL type var_name
-#endif
namespace butil {
diff --git a/src/butil/thread_local_inl.h b/src/butil/thread_local_inl.h
index 89f4cad8..6b20d99b 100644
--- a/src/butil/thread_local_inl.h
+++ b/src/butil/thread_local_inl.h
@@ -28,20 +28,21 @@ template <typename T>
class ThreadLocalHelper {
public:
inline static T* get() {
- if (__builtin_expect(value != nullptr, 1)) {
- return value;
+ T* ptr = value();
+ if (__builtin_expect(ptr != nullptr, 1)) {
+ return ptr;
}
- value = new (std::nothrow) T;
- if (value != nullptr) {
- butil::thread_atexit(delete_object<T>, value);
+ ptr = new (std::nothrow) T;
+ if (ptr != nullptr) {
+ value() = ptr;
+ butil::thread_atexit(delete_object<T>, ptr);
}
- return value;
+ return ptr;
}
- static BAIDU_THREAD_LOCAL T* value;
+private:
+ static BUTIL_THREAD_LOCAL(T*, value, nullptr);
};
-template <typename T> BAIDU_THREAD_LOCAL T* ThreadLocalHelper<T>::value =
nullptr;
-
} // namespace detail
template <typename T> inline T* get_thread_local() {
diff --git a/src/bvar/detail/agent_group.h b/src/bvar/detail/agent_group.h
index 192ca88a..1123606a 100644
--- a/src/bvar/detail/agent_group.h
+++ b/src/bvar/detail/agent_group.h
@@ -113,10 +113,11 @@ public:
// We need this function to be as fast as possible.
inline static Agent* get_tls_agent(AgentId id) {
if (__builtin_expect(id >= 0, 1)) {
- if (_s_tls_blocks) {
+ auto* blocks = _s_tls_blocks();
+ if (blocks) {
const size_t block_id = (size_t)id / ELEMENTS_PER_BLOCK;
- if (block_id < _s_tls_blocks->size()) {
- ThreadBlock* const tb = (*_s_tls_blocks)[block_id];
+ if (block_id < blocks->size()) {
+ ThreadBlock* const tb = (*blocks)[block_id];
if (tb) {
return tb->at(id - block_id * ELEMENTS_PER_BLOCK);
}
@@ -132,41 +133,44 @@ public:
CHECK(false) << "Invalid id=" << id;
return nullptr;
}
- if (_s_tls_blocks == nullptr) {
- _s_tls_blocks = new (std::nothrow) std::vector<ThreadBlock *>;
- if (__builtin_expect(_s_tls_blocks == nullptr, 0)) {
+ auto* blocks = _s_tls_blocks();
+ if (blocks == nullptr) {
+ blocks = new (std::nothrow) std::vector<ThreadBlock*>;
+ if (__builtin_expect(blocks == nullptr, 0)) {
LOG(FATAL) << "Fail to create vector, " << berror();
return nullptr;
}
+ _s_tls_blocks() = blocks;
butil::thread_atexit(_destroy_tls_blocks);
}
const size_t block_id = (size_t)id / ELEMENTS_PER_BLOCK;
- if (block_id >= _s_tls_blocks->size()) {
+ if (block_id >= blocks->size()) {
// The 32ul avoid pointless small resizes.
- _s_tls_blocks->resize(std::max(block_id + 1, 32ul));
+ blocks->resize(std::max(block_id + 1, 32ul));
}
- ThreadBlock* tb = (*_s_tls_blocks)[block_id];
+ ThreadBlock* tb = (*blocks)[block_id];
if (tb == nullptr) {
ThreadBlock *new_block = new (std::nothrow) ThreadBlock;
if (__builtin_expect(new_block == nullptr, 0)) {
return nullptr;
}
tb = new_block;
- (*_s_tls_blocks)[block_id] = new_block;
+ (*blocks)[block_id] = new_block;
}
return tb->at(id - block_id * ELEMENTS_PER_BLOCK);
}
private:
static void _destroy_tls_blocks() {
- if (!_s_tls_blocks) {
+ auto* blocks = _s_tls_blocks();
+ if (!blocks) {
return;
}
- for (size_t i = 0; i < _s_tls_blocks->size(); ++i) {
- delete (*_s_tls_blocks)[i];
+ for (size_t i = 0; i < blocks->size(); ++i) {
+ delete (*blocks)[i];
}
- delete _s_tls_blocks;
- _s_tls_blocks = nullptr;
+ delete blocks;
+ _s_tls_blocks() = nullptr;
}
inline static std::deque<AgentId> &_get_free_ids() {
@@ -180,7 +184,7 @@ private:
static pthread_mutex_t _s_mutex;
static AgentId _s_agent_kinds;
static std::deque<AgentId> *_s_free_ids;
- static __thread std::vector<ThreadBlock *> *_s_tls_blocks;
+ static BUTIL_THREAD_LOCAL(std::vector<ThreadBlock*>*, _s_tls_blocks,
nullptr);
};
template <typename Agent>
@@ -192,10 +196,6 @@ std::deque<AgentId>* AgentGroup<Agent>::_s_free_ids =
nullptr;
template <typename Agent>
AgentId AgentGroup<Agent>::_s_agent_kinds = 0;
-template <typename Agent>
-__thread std::vector<typename AgentGroup<Agent>::ThreadBlock *>
-*AgentGroup<Agent>::_s_tls_blocks = nullptr;
-
} // namespace detail
} // namespace bvar
diff --git a/test/baidu_thread_local_unittest.cpp
b/test/baidu_thread_local_unittest.cpp
index 2d242648..0eb09bb3 100644
--- a/test/baidu_thread_local_unittest.cpp
+++ b/test/baidu_thread_local_unittest.cpp
@@ -22,7 +22,7 @@
namespace {
-BAIDU_THREAD_LOCAL int * dummy = nullptr;
+BUTIL_THREAD_LOCAL(int*, dummy, nullptr);
const size_t NTHREAD = 8;
static bool processed[NTHREAD+1];
static bool deleted[NTHREAD+1];
@@ -64,6 +64,7 @@ protected:
};
+// Test the legacy native TLS keyword on pthreads (no bthread suspension).
BAIDU_THREAD_LOCAL void* x;
void* foo(void* arg) {
@@ -111,9 +112,9 @@ TEST_F(BaiduThreadLocalTest, get_thread_local) {
void delete_dummy(void* arg) {
*(bool*)arg = true;
- if (dummy) {
- delete dummy;
- dummy = nullptr;
+ if (dummy()) {
+ delete dummy();
+ dummy() = nullptr;
} else {
printf("dummy is NULL\n");
}
@@ -122,8 +123,8 @@ void delete_dummy(void* arg) {
void* proc_dummy(void* arg) {
bool *p = (bool*)arg;
*p = true;
- EXPECT_TRUE(dummy == nullptr);
- dummy = new int(p - processed);
+ EXPECT_TRUE(dummy() == nullptr);
+ dummy() = new int(p - processed);
butil::thread_atexit(delete_dummy, deleted + (p - processed));
return nullptr;
}
diff --git a/test/bthread_setconcurrency_unittest.cpp
b/test/bthread_setconcurrency_unittest.cpp
index 8e98a030..2ab6229c 100644
--- a/test/bthread_setconcurrency_unittest.cpp
+++ b/test/bthread_setconcurrency_unittest.cpp
@@ -63,14 +63,14 @@ static butil::atomic<int> *even;
static butil::atomic<int> nbthreads(0);
static butil::atomic<int> npthreads(0);
-static BAIDU_THREAD_LOCAL bool counted = false;
+static BUTIL_THREAD_LOCAL(bool, counted, false);
static butil::atomic<bool> stop (false);
static void *odd_thread(void *) {
nbthreads.fetch_add(1);
while (!stop) {
- if (!counted) {
- counted = true;
+ if (!counted()) {
+ counted() = true;
npthreads.fetch_add(1);
}
bthread::butex_wake_all(even);
@@ -82,8 +82,8 @@ static void *odd_thread(void *) {
static void *even_thread(void *) {
nbthreads.fetch_add(1);
while (!stop) {
- if (!counted) {
- counted = true;
+ if (!counted()) {
+ counted() = true;
npthreads.fetch_add(1);
}
bthread::butex_wake_all(odd);
diff --git a/test/bthread_tls_unittest.cpp b/test/bthread_tls_unittest.cpp
new file mode 100644
index 00000000..ade5e2ba
--- /dev/null
+++ b/test/bthread_tls_unittest.cpp
@@ -0,0 +1,251 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <gtest/gtest.h>
+
+#include <atomic>
+#include <condition_variable>
+#include <cstdlib>
+#include <memory>
+#include <mutex>
+#include <thread>
+
+#include "bthread/context.h"
+#include "bthread/task_meta.h"
+#include "butil/containers/doubly_buffered_data.h"
+#include "butil/debug/address_annotations.h"
+#include "butil/find_cstr.h"
+#include "butil/iobuf.h"
+#include "butil/object_pool.h"
+#include "butil/resource_pool.h"
+#include "butil/thread_local.h"
+#include "bvar/detail/agent_group.h"
+
+namespace {
+
+BUTIL_THREAD_LOCAL(int, tls_number, 7);
+using Buffer = char[16];
+BUTIL_THREAD_LOCAL(Buffer, tls_buffer, {});
+
+struct Noncopyable {
+ Noncopyable() = default;
+ Noncopyable(const Noncopyable&) = delete;
+ Noncopyable* operator&() { return nullptr; }
+ ~Noncopyable() { ++destroyed; }
+ int value = 17;
+ static std::atomic<int> destroyed;
+};
+std::atomic<int> Noncopyable::destroyed{0};
+BUTIL_THREAD_LOCAL(Noncopyable, tls_object, {});
+
+struct Agent {
+ int owner = -1;
+};
+using Group = bvar::detail::AgentGroup<Agent>;
+using Data = butil::DoublyBufferedData<int, int>;
+struct Helper {};
+struct Object {};
+struct Resource {};
+
+// Exercise a static member of a class template without an out-of-class TLS
+// definition. Each specialization must have its own storage.
+template <typename T>
+struct Member {
+ static BUTIL_THREAD_LOCAL(T, value, {});
+};
+
+struct Snapshot {
+ int* number;
+ Buffer* buffer;
+ int member;
+ Agent* agent;
+ int agent_owner;
+ int* wrapper;
+ int wrapper_owner;
+ Helper* helper;
+ Object* object;
+ Resource* resource;
+ bthread::LocalStorage* bls;
+ butil::iobuf::TLSData* iobuf;
+ butil::StringMapThreadLocalTemp* stringmap;
+};
+
+BUTIL_FORCE_INLINE Snapshot snapshot(Data& data) {
+ Data::ScopedPtr ptr;
+ if (data.Read(&ptr) != 0) {
+ std::abort();
+ }
+ Object* object = butil::get_object<Object>();
+ butil::return_object(object);
+ butil::ResourceId<Resource> id;
+ Resource* resource = butil::get_resource(&id);
+ butil::return_resource(id);
+ Agent* agent = Group::get_tls_agent(0);
+ return {&tls_number(), &tls_buffer(), Member<int>::value(),
+ agent, agent ? agent->owner : -1, &ptr.tls(), ptr.tls(),
+ butil::get_thread_local<Helper>(), object, resource,
+ &bthread::tls_bls(), butil::iobuf::get_g_tls_data(),
+ &butil::tls_stringmap_temp()};
+}
+
+// Use the same context switch as the scheduler, but explicitly hand the stack
+// between two pthreads. This guarantees migration on EVERY suspension, without
+// sleeps or depending on the scheduler's work-stealing decisions.
+constexpr int kRounds = 128;
+struct AsanStack {
+ void* fake = nullptr;
+ const void* bottom = nullptr;
+ size_t size = 0;
+};
+struct Migration {
+ std::mutex mutex;
+ std::condition_variable changed;
+ int ready = 0;
+ int worker = 0;
+ bool done = false;
+ bthread_fcontext_t fiber = nullptr;
+ bthread_fcontext_t native[2] = {};
+ AsanStack fiber_stack;
+ AsanStack native_stack[2];
+ Data data;
+ Snapshot expected[2];
+ Snapshot actual[kRounds];
+ int numbers[kRounds] = {};
+};
+
+void migrating_fiber(intptr_t arg) {
+ auto* state = reinterpret_cast<Migration*>(arg);
+ BUTIL_ASAN_FINISH_SWITCH_FIBER(nullptr,
+ &state->native_stack[state->worker].bottom,
+ &state->native_stack[state->worker].size);
+ for (int i = 0; i < kRounds; ++i) {
+ state->actual[i] = snapshot(state->data);
+ state->numbers[i] = ++tls_number();
+ tls_buffer()[0] = static_cast<char>(state->worker + 1);
+ BUTIL_ASAN_START_SWITCH_FIBER(&state->fiber_stack.fake,
+ state->native_stack[state->worker].bottom,
+ state->native_stack[state->worker].size);
+ bthread_jump_fcontext(&state->fiber, state->native[state->worker], 0);
+ BUTIL_ASAN_FINISH_SWITCH_FIBER(state->fiber_stack.fake,
+ &state->native_stack[state->worker].bottom,
+ &state->native_stack[state->worker].size);
+ }
+ state->done = true;
+ BUTIL_ASAN_START_SWITCH_FIBER(nullptr,
+ state->native_stack[state->worker].bottom,
+ state->native_stack[state->worker].size);
+ bthread_jump_fcontext(&state->fiber, state->native[state->worker], 0);
+ std::abort(); // A completed context must never be resumed.
+}
+
+TEST(ThreadLocalMigrationTest, reacquire_after_every_suspend) {
+ Migration state;
+ constexpr size_t stack_size = 128 * 1024;
+ std::unique_ptr<char[]> stack(new char[stack_size]);
+ state.fiber_stack.bottom = stack.get();
+ state.fiber_stack.size = stack_size;
+ state.fiber = bthread_make_fcontext(stack.get() + stack_size,
+ stack_size, migrating_fiber);
+ ASSERT_NE(nullptr, state.fiber);
+ auto run = [&state](int worker) {
+ std::unique_lock<std::mutex> lock(state.mutex);
+ tls_number() = worker * 1000;
+ Member<int>::value() = worker;
+ Group::get_or_create_tls_agent(0)->owner = worker;
+ {
+ Data::ScopedPtr ptr;
+ if (state.data.Read(&ptr) != 0) {
+ std::abort();
+ }
+ ptr.tls() = worker;
+ }
+ state.expected[worker] = snapshot(state.data);
+ ++state.ready;
+ state.changed.notify_all();
+ int block = 32;
+ for (;;) {
+ state.changed.wait(lock, [&state, worker] {
+ return state.done || (state.ready == 2 && state.worker ==
worker);
+ });
+ if (state.done) {
+ break;
+ }
+ // Grow the source pthread's vector while the fiber is suspended.
+ // A cached TLS slot must not make the next destination access it.
+ Group::get_or_create_tls_agent(block++ *
Group::ELEMENTS_PER_BLOCK);
+ BUTIL_ASAN_START_SWITCH_FIBER(&state.native_stack[worker].fake,
+ state.fiber_stack.bottom, state.fiber_stack.size);
+ bthread_jump_fcontext(&state.native[worker], state.fiber,
+ reinterpret_cast<intptr_t>(&state));
+ BUTIL_ASAN_FINISH_SWITCH_FIBER(state.native_stack[worker].fake,
+ nullptr, nullptr);
+ state.worker = 1 - worker;
+ state.changed.notify_all();
+ }
+ };
+ std::thread first(run, 0);
+ std::thread second(run, 1);
+ first.join();
+ second.join();
+
+ ASSERT_NE(state.expected[0].number, state.expected[1].number);
+ ASSERT_NE(state.expected[0].agent, state.expected[1].agent);
+ ASSERT_NE(state.expected[0].wrapper, state.expected[1].wrapper);
+ for (int i = 0; i < kRounds; ++i) {
+ SCOPED_TRACE(i);
+ const Snapshot& expected = state.expected[i % 2];
+ const Snapshot& actual = state.actual[i];
+ EXPECT_EQ(expected.number, actual.number);
+ EXPECT_EQ(expected.buffer, actual.buffer);
+ EXPECT_EQ(expected.member, actual.member);
+ EXPECT_EQ(expected.agent, actual.agent);
+ EXPECT_EQ(i % 2, actual.agent_owner);
+ EXPECT_EQ(expected.wrapper, actual.wrapper);
+ EXPECT_EQ(i % 2, actual.wrapper_owner);
+ EXPECT_EQ(expected.helper, actual.helper);
+ EXPECT_EQ(expected.object, actual.object);
+ EXPECT_EQ(expected.resource, actual.resource);
+ EXPECT_EQ(expected.bls, actual.bls);
+ EXPECT_EQ(expected.iobuf, actual.iobuf);
+ EXPECT_EQ(expected.stringmap, actual.stringmap);
+ EXPECT_EQ((i % 2) * 1000 + i / 2 + 1, state.numbers[i]);
+ }
+}
+
+TEST(ThreadLocalMigrationTest, accessor_initialization_and_types) {
+ EXPECT_EQ(7, tls_number());
+ EXPECT_EQ(0, tls_buffer()[0]);
+ EXPECT_EQ(0, Member<int>::value());
+ EXPECT_EQ(0L, Member<long>::value());
+ Member<int>::value() = 42;
+ EXPECT_EQ(0L, Member<long>::value());
+ Member<int>::value() = 0;
+}
+
+TEST(ThreadLocalMigrationTest, noncopyable_object_destroyed_on_thread_exit) {
+ const int before = Noncopyable::destroyed.load();
+ std::thread worker([] {
+ EXPECT_EQ(17, tls_object().value);
+ tls_object().value = 42;
+ EXPECT_EQ(42, tls_object().value);
+ EXPECT_NE(nullptr, std::addressof(tls_object()));
+ });
+ worker.join();
+ EXPECT_EQ(before + 1, Noncopyable::destroyed.load());
+}
+
+} // namespace
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]