This is an automated email from the ASF dual-hosted git repository.
chenBright pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/brpc.git
The following commit(s) were added to refs/heads/master by this push:
new f7295044d Fix Collector use-after-free during process exit (#3721)
f7295044d is described below
commit f7295044da124c46055e35658b3bd4ce51d52cdd
Author: Bright Chen <[email protected]>
AuthorDate: Tue Nov 10 16:11:59 2026 +0800
Fix Collector use-after-free during process exit (#3721)
* Fix Collector use-after-free during process exit
* Cover pre-main collector initialization
---
BUILD.bazel | 6 +
src/brpc/rpc_dump.cpp | 8 +
src/bvar/collector.cpp | 125 +++++++++--
src/bvar/collector.h | 15 ++
test/BUILD.bazel | 23 +++
test/CMakeLists.txt | 8 +
test/Makefile | 14 ++
test/brpc_collector_premain_unittest.cpp | 147 +++++++++++++
test/brpc_collector_shutdown_unittest.cpp | 333 ++++++++++++++++++++++++++++++
9 files changed, 658 insertions(+), 21 deletions(-)
diff --git a/BUILD.bazel b/BUILD.bazel
index 092b827af..5e5eb49d5 100644
--- a/BUILD.bazel
+++ b/BUILD.bazel
@@ -20,6 +20,12 @@ licenses(["notice"]) # Apache v2
exports_files(["LICENSE"])
+# Included by the pre-main shutdown test to control static initialization
order.
+exports_files(
+ ["src/bvar/collector.cpp", "src/brpc/rpc_dump.cpp"],
+ visibility = ["//test:__pkg__"],
+)
+
config_setting(
name = "brpc_with_flatbuffers",
define_values = {"BRPC_WITH_FLATBUFFERS": "true"},
diff --git a/src/brpc/rpc_dump.cpp b/src/brpc/rpc_dump.cpp
index a9dca57ca..c5455f5b8 100644
--- a/src/brpc/rpc_dump.cpp
+++ b/src/brpc/rpc_dump.cpp
@@ -62,6 +62,14 @@ BRPC_VALIDATE_GFLAG(rpc_dump, PassValidate);
BRPC_VALIDATE_GFLAG(rpc_dump_max_requests_in_one_file, PositiveInteger);
BRPC_VALIDATE_GFLAG(rpc_dump_max_files, PositiveInteger);
+// Declare this after the flags so both collector threads finish before the
+// string flag is destroyed, even if the collector was created before main().
+// This also prevents the first RPC dump from starting during later teardown.
+struct RpcDumpExitGuard {
+ ~RpcDumpExitGuard() { bvar::detail::StopCollector(); }
+};
+static RpcDumpExitGuard g_rpc_dump_exit_guard;
+
static const size_t UNWRITTEN_BUFSIZE = 1024 * 1024;
static const int64_t FLUSH_TIMEOUT = 2000000L; // 2s
diff --git a/src/bvar/collector.cpp b/src/bvar/collector.cpp
index 1fcf0db1a..9c221496d 100644
--- a/src/bvar/collector.cpp
+++ b/src/bvar/collector.cpp
@@ -17,6 +17,7 @@
// Date: Mon Dec 14 19:12:30 CST 2015
+#include <cstdlib>
#include <map>
#include <gflags/gflags.h>
#include "butil/memory/singleton_on_pthread_once.h"
@@ -64,6 +65,9 @@ public:
void wakeup_grab_thread();
+ // Called once, after singleton creation has finished.
+ void stop_and_join();
+
private:
// The thread for collecting TLS submissions.
void grab_thread();
@@ -99,7 +103,7 @@ private:
int64_t _last_active_cpuwide_us;
bool _created; // Mark validness of _grab_thread.
- bool _stop; // Set to true in dtor.
+ butil::atomic<bool> _stop;
pthread_t _grab_thread; // For joining.
pthread_t _dump_thread;
int64_t _ngrab BAIDU_CACHELINE_ALIGNMENT;
@@ -112,6 +116,51 @@ private:
pthread_cond_t _sleep_cond;
};
+// These survive static destruction, just like the collector itself. Only the
+// cold creation path and shutdown take the mutex; submissions remain TLS
based.
+static pthread_mutex_t g_collector_mutex = PTHREAD_MUTEX_INITIALIZER;
+static pthread_once_t g_collector_stop_once = PTHREAD_ONCE_INIT;
+static butil::static_atomic<bool> g_collector_stopped =
BUTIL_STATIC_ATOMIC_INIT(false);
+
+static Collector* GetCollector() {
+ if (g_collector_stopped.load(butil::memory_order_acquire)) {
+ return nullptr;
+ }
+ Collector* collector = butil::has_leaky_singleton<Collector>();
+ if (collector != nullptr) {
+ return collector;
+ }
+ BAIDU_SCOPED_LOCK(g_collector_mutex);
+ if (g_collector_stopped.load(butil::memory_order_relaxed)) {
+ return nullptr;
+ }
+ return butil::get_leaky_singleton<Collector>();
+}
+
+static void StopCollectorOnce() {
+ Collector* collector;
+ {
+ BAIDU_SCOPED_LOCK(g_collector_mutex);
+ g_collector_stopped.store(true, butil::memory_order_release);
+ collector = butil::has_leaky_singleton<Collector>();
+ }
+ // A worker may have been waiting in GetCollector() during initialization.
+ // Let it observe shutdown instead of holding the creation lock while
joining.
+ if (collector != nullptr) {
+ collector->stop_and_join();
+ }
+}
+
+void detail::StopCollector() {
+ pthread_once(&g_collector_stop_once, StopCollectorOnce);
+}
+
+// Also forbid first-time creation during later static destructors.
+struct CollectorExitGuard {
+ ~CollectorExitGuard() { detail::StopCollector(); }
+};
+static CollectorExitGuard g_collector_exit_guard;
+
Collector::Collector()
: _last_active_cpuwide_us(butil::cpuwide_time_us())
, _created(false)
@@ -125,6 +174,9 @@ Collector::Collector()
pthread_cond_init(&_dump_thread_cond, nullptr);
pthread_mutex_init(&_sleep_mutex, nullptr);
pthread_cond_init(&_sleep_cond, nullptr);
+ // The singleton is intentionally leaked. Register thread shutdown
explicitly
+ // so runtime initialization stops callbacks before static destruction.
+ CHECK_EQ(0, atexit(detail::StopCollector));
int rc = pthread_create(&_grab_thread, nullptr, run_grab_thread, this);
if (rc != 0) {
LOG(ERROR) << "Fail to create Collector, " << berror(rc);
@@ -134,17 +186,38 @@ Collector::Collector()
}
Collector::~Collector() {
- if (_created) {
- _stop = true;
- pthread_join(_grab_thread, nullptr);
- _created = false;
- }
+ stop_and_join();
pthread_mutex_destroy(&_dump_thread_mutex);
pthread_cond_destroy(&_dump_thread_cond);
pthread_mutex_destroy(&_sleep_mutex);
pthread_cond_destroy(&_sleep_cond);
}
+static void DestroySamples(butil::LinkNode<Collected>* root) {
+ while (root->next() != root) {
+ butil::LinkNode<Collected>* node = root->next();
+ node->RemoveFromList();
+ node->value()->destroy();
+ }
+}
+
+void Collector::stop_and_join() {
+ if (_created) {
+ _stop.store(true, butil::memory_order_relaxed);
+ wakeup_grab_thread();
+ // The grabber wakes and joins the dumper before it exits.
+ CHECK_EQ(0, pthread_join(_grab_thread, nullptr));
+ _created = false;
+ }
+ DestroySamples(&_dump_root);
+ butil::LinkNode<Collected> root;
+ Collected* head = reset();
+ if (head != nullptr) {
+ head->InsertBeforeAsList(&root);
+ DestroySamples(&root);
+ }
+}
+
template <typename T>
static T deref_value(void* arg) {
return *(T*)arg;
@@ -185,7 +258,7 @@ void Collector::grab_thread() {
PreprocessorMap prep_map;
// The main loop.
- while (!_stop) {
+ while (!_stop.load(butil::memory_order_relaxed)) {
const int64_t abstime = _last_active_cpuwide_us +
COLLECTOR_GRAB_INTERVAL_US;
// Clear and reuse vectors in prep_map, don't clear prep_map directly.
@@ -266,10 +339,12 @@ void Collector::grab_thread() {
_last_active_cpuwide_us = now;
// sleep for the next round.
- if (!_stop && abstime > now) {
+ if (abstime > now) {
timespec abstimespec = butil::microseconds_from_now(abstime - now);
pthread_mutex_lock(&_sleep_mutex);
- pthread_cond_timedwait(&_sleep_cond, &_sleep_mutex, &abstimespec);
+ if (!_stop.load(butil::memory_order_relaxed)) {
+ pthread_cond_timedwait(&_sleep_cond, &_sleep_mutex,
&abstimespec);
+ }
pthread_mutex_unlock(&_sleep_mutex);
}
_last_active_cpuwide_us = butil::cpuwide_time_us();
@@ -277,7 +352,7 @@ void Collector::grab_thread() {
// make sure _stop is true, we may have other reasons to quit above loop
{
BAIDU_SCOPED_LOCK(_dump_thread_mutex);
- _stop = true;
+ _stop.store(true, butil::memory_order_relaxed);
pthread_cond_signal(&_dump_thread_cond);
}
CHECK_EQ(0, pthread_join(_dump_thread, nullptr));
@@ -352,7 +427,10 @@ size_t
is_collectable_before_first_time_grabbed(CollectorSpeedLimit* sl) {
if (before_add == 0) {
sl->first_sample_us = butil::cpuwide_time_us();
} else if (before_add >= FLAGS_bvar_collector_expected_per_second) {
- butil::get_leaky_singleton<Collector>()->wakeup_grab_thread();
+ Collector* collector = GetCollector();
+ if (collector != nullptr) {
+ collector->wakeup_grab_thread();
+ }
}
}
return sl->sampling_range;
@@ -376,19 +454,20 @@ void Collector::dump_thread() {
size_t round = 0;
// The main loop
- while (!_stop) {
+ while (!_stop.load(butil::memory_order_relaxed)) {
++round;
// Get new samples set by grab_thread.
butil::LinkNode<Collected>* newhead = nullptr;
{
BAIDU_SCOPED_LOCK(_dump_thread_mutex);
- while (!_stop && _dump_root.next() == &_dump_root) {
+ while (!_stop.load(butil::memory_order_relaxed) &&
+ _dump_root.next() == &_dump_root) {
const int64_t now_ns = butil::cpuwide_time_ns();
busy_seconds += (now_ns - last_ns) / 1000000000.0;
pthread_cond_wait(&_dump_thread_cond, &_dump_thread_mutex);
last_ns = butil::cpuwide_time_ns();
}
- if (_stop) {
+ if (_stop.load(butil::memory_order_relaxed)) {
break;
}
newhead = _dump_root.next();
@@ -398,7 +477,8 @@ void Collector::dump_thread() {
newhead->InsertBeforeAsList(&root);
// Call callbacks.
- for (butil::LinkNode<Collected>* p = root.next(); !_stop && p !=
&root;) {
+ for (butil::LinkNode<Collected>* p = root.next();
+ !_stop.load(butil::memory_order_relaxed) && p != &root;) {
// We remove p from the list, save next first.
butil::LinkNode<Collected>* saved_next = p->next();
p->RemoveFromList();
@@ -408,15 +488,18 @@ void Collector::dump_thread() {
p = saved_next;
}
}
+ DestroySamples(&root);
}
-// Submit a sample for asynchronous dumping. The Collector holds only the
Collected*
-// pointer (e.g., SpanContainer*). Regardless of which branch is taken below,
the
-// sample will eventually be destroyed via either dump_and_destroy() or
destroy(),
-// both of which call 'delete this' to release the container and decrement the
-// reference count of any managed resources (e.g., shared_ptr<Span>).
+// Submit a sample for asynchronous dumping. Shutdown rejects new submissions.
+// A submission already racing shutdown may still enter the leaked reducer
after
+// its final reset; such samples are reclaimed along with the process.
void Collected::submit(int64_t cpuwide_us) {
- Collector* d = butil::get_leaky_singleton<Collector>();
+ Collector* d = GetCollector();
+ if (d == nullptr) {
+ destroy();
+ return;
+ }
// Destroy the sample in-place if the grab_thread did not run for twice
// of the normal interval. This also applies to the situation that
// grab_thread aborts due to severe errors.
diff --git a/src/bvar/collector.h b/src/bvar/collector.h
index 1dadf49c6..d526b288d 100644
--- a/src/bvar/collector.h
+++ b/src/bvar/collector.h
@@ -28,6 +28,18 @@
namespace bvar {
+namespace detail {
+// Permanently stop the process-wide collector and wait for its callbacks.
+// Discard pending samples without dumping. This is not a barrier for submit():
+// a submission overlapping shutdown may miss the final cleanup and remain
+// allocated until process termination, without another dump callback.
+// Internal exit hook, not a supported user API. May be called repeatedly, but
+// never from a collector callback. Does not create or delete the singleton.
+// Owners of destructible callback dependencies must call this before teardown;
+// the collector may have been initialized before those dependencies.
+void StopCollector();
+} // namespace detail
+
static const size_t INVALID_SAMPLING_RANGE = 0;
inline bool is_sampling_range_valid(size_t sampling_range) {
@@ -75,6 +87,9 @@ public:
// from the timestamp updated by collecting thread(which basically means
// the thread is not scheduled by OS in time), this sample is directly
// destroy()-ed to avoid memory explosion.
+ // Calls started after collector shutdown are destroy()-ed in-place.
+ // Calls overlapping shutdown may leave the sample allocated until process
+ // termination without calling destroy(), as described below.
void submit(int64_t cpuwide_us);
void submit() { submit(butil::cpuwide_time_us()); }
diff --git a/test/BUILD.bazel b/test/BUILD.bazel
index 268941839..594e16800 100644
--- a/test/BUILD.bazel
+++ b/test/BUILD.bazel
@@ -281,11 +281,34 @@ cc_test(
],
)
+# Textual inclusion fixes the two implementations' initialization order. Static
+# linking supplies the remaining brpc objects without duplicate flag
definitions.
+cc_library(
+ name = "collector_premain_sources",
+ textual_hdrs = [
+ "//:src/bvar/collector.cpp",
+ "//:src/brpc/rpc_dump.cpp",
+ ],
+ deps = ["//:brpc"],
+)
+
+cc_test(
+ name = "brpc_collector_premain_unittest",
+ srcs = ["brpc_collector_premain_unittest.cpp"],
+ copts = COPTS,
+ linkstatic = True,
+ deps = [
+ ":collector_premain_sources",
+ "@com_google_googletest//:gtest_main",
+ ] + TCMALLOC_DEP_UNLESS_ASAN,
+)
+
generate_unittests(
name = "brpc_unittests",
srcs = glob(
["brpc_*_unittest.cpp"],
exclude = [
+ "brpc_collector_premain_unittest.cpp",
"brpc_flatbuffers_unittest.cpp",
"brpc_flatbuffers_protocol_unittest.cpp",
],
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 20becf6ca..25b107cf1 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -340,6 +340,7 @@ endif()
# brpc tests
file(GLOB BRPC_UNITTESTS "brpc_*_unittest.cpp")
list(REMOVE_ITEM BRPC_UNITTESTS
+ "${CMAKE_CURRENT_SOURCE_DIR}/brpc_collector_premain_unittest.cpp"
"${CMAKE_CURRENT_SOURCE_DIR}/brpc_flatbuffers_unittest.cpp"
"${CMAKE_CURRENT_SOURCE_DIR}/brpc_flatbuffers_protocol_unittest.cpp")
foreach(BRPC_UT ${BRPC_UNITTESTS})
@@ -353,6 +354,13 @@ foreach(BRPC_UT ${BRPC_UNITTESTS})
add_test(NAME ${BRPC_UT_WE} COMMAND ${BRPC_UT_WE})
endforeach()
+# This test includes collector.cpp and rpc_dump.cpp to control initialization
+# order. A static archive supplies the other objects without duplicate flags.
+add_executable(brpc_collector_premain_unittest
brpc_collector_premain_unittest.cpp)
+target_link_libraries(brpc_collector_premain_unittest PRIVATE
+ brpc-static brpc_test_config gtest_main ${GPERFTOOLS_LIBRARIES})
+add_test(NAME brpc_collector_premain_unittest COMMAND
brpc_collector_premain_unittest)
+
if(BUILD_FUZZ_TESTS)
add_library(brpc-static-debug STATIC $<TARGET_OBJECTS:BUTIL_DEBUG_LIB>
$<TARGET_OBJECTS:SOURCES_DEBUG_LIB>
diff --git a/test/Makefile b/test/Makefile
index ccad5483d..7ee0ce33a 100644
--- a/test/Makefile
+++ b/test/Makefile
@@ -262,6 +262,20 @@ else ifeq ($(SYSTEM),Darwin)
endif
endif
+# The pre-main test includes both implementations and needs a static archive to
+# avoid loading duplicate flags from the shared test library.
+../libbrpc.a:FORCE
+ $(MAKE) -C.. libbrpc.a
+
+brpc_collector_premain_unittest.o:../src/bvar/collector.cpp
../src/brpc/rpc_dump.cpp
+
+brpc_collector_premain_unittest:brpc_collector_premain_unittest.o ../libbrpc.a
+ifeq ($(SYSTEM),Linux)
+ $(CXX) -o $@ $(LIBPATHS) $(SOPATHS) -Xlinker "-(" $^ $(STATIC_LINKINGS)
-Xlinker "-)" $(DYNAMIC_LINKINGS)
+else ifeq ($(SYSTEM),Darwin)
+ $(CXX) -o $@ $(LIBPATHS) $(SOPATHS) $^ $(STATIC_LINKINGS)
$(DYNAMIC_LINKINGS)
+endif
+
brpc_%_unittest:$(TEST_PROTO_OBJS) brpc_%_unittest.o | libbrpc.dbg.$(SOEXT)
@echo "> Linking $@"
ifeq ($(SYSTEM),Linux)
diff --git a/test/brpc_collector_premain_unittest.cpp
b/test/brpc_collector_premain_unittest.cpp
new file mode 100644
index 000000000..d55b0d1ba
--- /dev/null
+++ b/test/brpc_collector_premain_unittest.cpp
@@ -0,0 +1,147 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <atomic>
+#include <cstdlib>
+#include <cstring>
+#include <memory>
+#include <unistd.h>
+#include <gtest/gtest.h>
+#include "butil/files/scoped_temp_dir.h"
+
+// Compile these implementations in this translation unit to fix initialization
+// order independently of the linker and shared-library loading order. Link the
+// remaining dependencies statically so there is only one copy of each flag and
+// singleton. The CollectorExitGuard must be registered before the RPC flags.
+#include "bvar/collector.cpp"
+
+namespace {
+
+const char kDirectoryEnv[] = "BRPC_COLLECTOR_PREMAIN_TEST_DIR";
+std::atomic<bool> g_dump_started{false};
+std::atomic<bool> g_dump_finished{false};
+bool g_initialized_before_flags = false;
+
+void Require(bool condition, const char* message) {
+ if (!condition) {
+ (void)write(STDERR_FILENO, message, strlen(message));
+ (void)write(STDERR_FILENO, "\n", 1);
+ std::_Exit(1);
+ }
+}
+
+template <typename Predicate>
+bool WaitUntil(Predicate predicate) {
+ const int64_t deadline = butil::monotonic_time_ms() + 10000;
+ while (!predicate() && butil::monotonic_time_ms() < deadline) {
+ usleep(1000);
+ }
+ return predicate();
+}
+
+void VerifyAfterFlagDestruction() {
+ Require(g_dump_finished.load(),
+ "RPC flags were destroyed before the pre-main collector stopped");
+}
+
+struct InitCollectorBeforeFlags {
+ InitCollectorBeforeFlags() {
+ // Only the re-executed death-test child creates an early collector.
+ if (getenv(kDirectoryEnv) == nullptr) {
+ return;
+ }
+ alarm(15);
+ Require(butil::has_leaky_singleton<bvar::Collector>() == nullptr,
+ "collector was already initialized");
+ butil::get_leaky_singleton<bvar::Collector>();
+ g_initialized_before_flags = true;
+ // At exit: RpcDumpExitGuard -> RPC flags -> verification ->
Collector's
+ // atexit hook -> CollectorExitGuard. Neither fallback can mask
failure.
+ Require(atexit(VerifyAfterFlagDestruction) == 0,
+ "cannot register exit verification");
+ }
+};
+InitCollectorBeforeFlags g_init_collector_before_flags;
+
+} // namespace
+
+// Keep this below the initializer and its exit verification registration.
+#include "brpc/rpc_dump.cpp"
+
+namespace {
+
+class Request : public brpc::SampledRequest {
+public:
+ Request() {
+ request.append("pre-main collector request");
+ meta.set_protocol_type(brpc::PROTOCOL_BAIDU_STD);
+ }
+
+ void dump_and_destroy(size_t round) override {
+ g_dump_started.store(true);
+ // Resume only when shutdown starts, so completion cannot race ahead of
+ // exit verification if RpcDumpExitGuard is removed. SaveFlags must
+ // still be able to read rpc_dump_dir at this point.
+ Require(WaitUntil([] {
+ return bvar::g_collector_stopped.load(butil::memory_order_acquire);
+ }), "collector shutdown did not start");
+ brpc::SampledRequest::dump_and_destroy(round);
+ }
+
+ void destroy() override {
+ delete this;
+ g_dump_finished.store(true);
+ }
+};
+
+void RunExit() {
+ Require(g_initialized_before_flags,
+ "collector was not initialized before the RPC flags");
+ brpc::FLAGS_rpc_dump_dir = getenv(kDirectoryEnv);
+ brpc::FLAGS_rpc_dump_max_requests_in_one_file = 1;
+ (new Request)->submit();
+ Require(WaitUntil([] { return g_dump_started.load(); }),
+ "RPC dump did not start");
+ std::exit(0);
+}
+
+TEST(CollectorPremainDeathTest, stops_before_rpc_flags_are_destroyed) {
+#ifdef GTEST_FLAG_SET
+ GTEST_FLAG_SET(death_test_style, "threadsafe");
+#else
+ ::testing::GTEST_FLAG(death_test_style) = "threadsafe";
+#endif
+ butil::ScopedTempDir directory;
+ const bool parent = getenv(kDirectoryEnv) == nullptr;
+ if (parent) {
+ ASSERT_TRUE(directory.CreateUniqueTempDir());
+ ASSERT_EQ(0, setenv(kDirectoryEnv,
+ directory.path().value().c_str(), 1));
+ }
+ EXPECT_EXIT(RunExit(), ::testing::ExitedWithCode(0), "");
+ if (parent) {
+ EXPECT_EQ(0, unsetenv(kDirectoryEnv));
+ brpc::SampleIterator it(directory.path().value());
+ std::unique_ptr<brpc::SampledRequest> request(it.Next());
+ ASSERT_TRUE(request);
+ EXPECT_EQ("pre-main collector request", request->request.to_string());
+ std::unique_ptr<brpc::SampledRequest> extra(it.Next());
+ EXPECT_FALSE(extra);
+ }
+}
+
+} // namespace
diff --git a/test/brpc_collector_shutdown_unittest.cpp
b/test/brpc_collector_shutdown_unittest.cpp
new file mode 100644
index 000000000..08f558125
--- /dev/null
+++ b/test/brpc_collector_shutdown_unittest.cpp
@@ -0,0 +1,333 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <atomic>
+#include <cstdlib>
+#include <cstring>
+#include <memory>
+#include <thread>
+#include <unistd.h>
+#include <gtest/gtest.h>
+#include "butil/files/scoped_temp_dir.h"
+#include "butil/memory/singleton_on_pthread_once.h"
+#include "brpc/rpc_dump.h"
+
+namespace bvar {
+class Collector;
+}
+
+namespace brpc {
+DECLARE_string(rpc_dump_dir);
+DECLARE_int32(rpc_dump_max_requests_in_one_file);
+}
+
+namespace {
+
+const char kDirectoryEnv[] = "BRPC_COLLECTOR_SHUTDOWN_TEST_DIR";
+
+void Require(bool condition, const char* message) {
+ if (!condition) {
+ (void)write(STDERR_FILENO, message, strlen(message));
+ (void)write(STDERR_FILENO, "\n", 1);
+ std::_Exit(1);
+ }
+}
+
+template <typename Predicate>
+bool WaitUntil(Predicate predicate, int64_t timeout_ms = 10000) {
+ const int64_t deadline = butil::monotonic_time_ms() + timeout_ms;
+ while (!predicate() && butil::monotonic_time_ms() < deadline) {
+ usleep(1000);
+ }
+ return predicate();
+}
+
+struct SampleState {
+ std::atomic<int> entered{0};
+ std::atomic<int> destroyed{0};
+ std::atomic<bool> release{true};
+};
+
+class Sample : public bvar::Collected {
+public:
+ explicit Sample(SampleState* state,
+ bvar::CollectorPreprocessor* preprocessor = nullptr)
+ : _state(state), _preprocessor(preprocessor) {}
+
+ void dump_and_destroy(size_t) override {
+ _state->entered.fetch_add(1);
+ Require(WaitUntil([this] { return _state->release.load(); }),
+ "sample was not released");
+ destroy();
+ }
+
+ void destroy() override {
+ SampleState* state = _state;
+ delete this;
+ state->destroyed.fetch_add(1);
+ }
+
+ bvar::CollectorSpeedLimit* speed_limit() override { return nullptr; }
+ bvar::CollectorPreprocessor* preprocessor() override { return
_preprocessor; }
+
+private:
+ SampleState* _state;
+ bvar::CollectorPreprocessor* _preprocessor;
+};
+
+// Put another sample behind the blocker in the very same dump batch.
+class AppendSample : public bvar::CollectorPreprocessor {
+public:
+ explicit AppendSample(SampleState* state) : _state(state) {}
+ void process(std::vector<bvar::Collected*>& samples) override {
+ samples.push_back(new Sample(_state));
+ }
+
+private:
+ SampleState* _state;
+};
+
+class CollectorShutdownDeathTest : public ::testing::Test {
+protected:
+ void SetUp() override {
+ // Re-exec children so inherited collectors never lose their workers.
+#ifdef GTEST_FLAG_SET
+ GTEST_FLAG_SET(death_test_style, "threadsafe");
+#else
+ ::testing::GTEST_FLAG(death_test_style) = "threadsafe";
+#endif
+ if (getenv(kDirectoryEnv) == nullptr) {
+ ASSERT_TRUE(_directory.CreateUniqueTempDir());
+ ASSERT_EQ(0, setenv(kDirectoryEnv,
_directory.path().value().c_str(), 1));
+ _owns_directory = true;
+ }
+ }
+
+ void TearDown() override {
+ if (_owns_directory) {
+ EXPECT_EQ(0, unsetenv(kDirectoryEnv));
+ }
+ }
+
+private:
+ butil::ScopedTempDir _directory;
+ bool _owns_directory = false;
+};
+
+TEST_F(CollectorShutdownDeathTest, stop_before_first_sample_prevents_creation)
{
+ EXPECT_EXIT({
+ alarm(15);
+ Require(butil::has_leaky_singleton<bvar::Collector>() == nullptr,
+ "collector already exists");
+ bvar::detail::StopCollector();
+ bvar::detail::StopCollector();
+ SampleState state;
+ (new Sample(&state))->submit();
+ Require(state.destroyed.load() == 1 && state.entered.load() == 0,
+ "late sample was not discarded");
+ bvar::CollectorSpeedLimit limit = {};
+ limit.sampling_range = bvar::COLLECTOR_SAMPLING_BASE;
+ limit.count_before_grabbed.store(1000000, butil::memory_order_relaxed);
+ bvar::is_collectable(&limit);
+ Require(butil::has_leaky_singleton<bvar::Collector>() == nullptr,
+ "shutdown created a collector");
+ std::exit(0);
+ }, ::testing::ExitedWithCode(0), "");
+}
+
+TEST_F(CollectorShutdownDeathTest,
concurrent_stops_wait_and_discard_pending_samples) {
+ EXPECT_EXIT({
+ alarm(15);
+ SampleState blocked;
+ blocked.release.store(false);
+ SampleState queued;
+ AppendSample append(&queued);
+ (new Sample(&blocked, &append))->submit();
+ Require(WaitUntil([&] { return blocked.entered.load() == 1; }),
+ "dump did not start");
+ // Exercise submissions from a thread whose TLS has already been freed.
+ std::thread producer([&] {
+ for (int i = 0; i < 16; ++i) {
+ (new Sample(&queued))->submit();
+ }
+ });
+ producer.join();
+
+ std::atomic<int> stopping{0};
+ std::atomic<int> stopped{0};
+ auto stop = [&] {
+ stopping.fetch_add(1);
+ bvar::detail::StopCollector();
+ stopped.fetch_add(1);
+ };
+ std::thread stopper1(stop);
+ std::thread stopper2(stop);
+ Require(WaitUntil([&] { return stopping.load() == 2; }),
+ "stop callers did not start");
+ Require(!WaitUntil([&] { return stopped.load() != 0; }, 100),
+ "stop returned before the in-flight dump finished");
+ blocked.release.store(true);
+ stopper1.join();
+ stopper2.join();
+ Require(blocked.destroyed.load() == 1, "in-flight sample was not
released");
+ Require(queued.destroyed.load() == 17 && queued.entered.load() == 0,
+ "pending samples were dumped or not released");
+ (new Sample(&queued))->submit();
+ Require(queued.destroyed.load() == 18 && queued.entered.load() == 0,
+ "late submission was not discarded");
+ std::exit(0);
+ }, ::testing::ExitedWithCode(0), "");
+}
+
+TEST_F(CollectorShutdownDeathTest, stop_races_first_submissions) {
+ EXPECT_EXIT({
+ alarm(15);
+ SampleState state;
+ std::atomic<bool> start{false};
+ std::atomic<int> ready{0};
+ std::thread producers[8];
+ for (auto& producer : producers) {
+ producer = std::thread([&] {
+ ready.fetch_add(1);
+ while (!start.load()) {
+ std::this_thread::yield();
+ }
+ (new Sample(&state))->submit();
+ });
+ }
+ Require(WaitUntil([&] { return ready.load() == 8; }),
+ "producers did not start");
+ start.store(true);
+ bvar::detail::StopCollector();
+ const int dumped = state.entered.load();
+ for (auto& producer : producers) {
+ producer.join();
+ }
+ bvar::detail::StopCollector();
+ Require(state.entered.load() == dumped, "dump started after stop
returned");
+ SampleState late;
+ (new Sample(&late))->submit();
+ Require(late.destroyed.load() == 1 && late.entered.load() == 0,
+ "racing creation restarted the collector");
+ std::exit(0);
+ }, ::testing::ExitedWithCode(0), "");
+}
+
+TEST_F(CollectorShutdownDeathTest, stop_wakes_idle_workers) {
+ EXPECT_EXIT({
+ alarm(15);
+ SampleState state;
+ (new Sample(&state))->submit();
+ Require(WaitUntil([&] { return state.destroyed.load() == 1; }),
+ "sample was not dumped");
+ bvar::detail::StopCollector();
+ bvar::detail::StopCollector();
+ Require(butil::has_leaky_singleton<bvar::Collector>() != nullptr,
+ "shutdown destroyed the singleton");
+ (new Sample(&state))->submit();
+ Require(state.destroyed.load() == 2 && state.entered.load() == 1,
+ "idle collector restarted after shutdown");
+ std::exit(0);
+ }, ::testing::ExitedWithCode(0), "");
+}
+
+class Request : public brpc::SampledRequest {
+public:
+ explicit Request(SampleState* state) : _state(state) {
+ request.append("collector shutdown request");
+ meta.set_protocol_type(brpc::PROTOCOL_BAIDU_STD);
+ }
+
+ void dump_and_destroy(size_t round) override {
+ _state->entered.fetch_add(1);
+ Require(WaitUntil([this] { return _state->release.load(); }),
+ "RPC dump was not released");
+ // Exercise the actual RpcDumpContext::SaveFlags(), including first
use.
+ brpc::SampledRequest::dump_and_destroy(round);
+ }
+
+ void destroy() override {
+ SampleState* state = _state;
+ delete this;
+ state->destroyed.fetch_add(1);
+ }
+
+private:
+ SampleState* _state;
+};
+
+static SampleState* g_exit_sample = nullptr;
+static std::atomic<bool> g_exiting{false};
+static int g_expected_requests = 0;
+
+void VerifyExit() {
+ Require(g_exit_sample->destroyed.load() == 1,
+ "exit did not wait for the RPC dump");
+ SampleState late;
+ (new Request(&late))->submit();
+ Require(late.destroyed.load() == 1 && late.entered.load() == 0,
+ "exit allowed a late RPC dump");
+ brpc::SampleIterator it(getenv(kDirectoryEnv));
+ for (int i = 0; i < g_expected_requests; ++i) {
+ std::unique_ptr<brpc::SampledRequest> request(it.Next());
+ Require(request &&
+ request->request.to_string() == "collector shutdown request",
+ "in-flight RPC dump was not persisted before exit");
+ }
+ std::unique_ptr<brpc::SampledRequest> extra(it.Next());
+ Require(!extra, "unexpected extra RPC dump");
+}
+
+void RunRpcDumpExit(bool prime_context) {
+ alarm(15);
+ const char* directory = getenv(kDirectoryEnv);
+ Require(directory != nullptr, "missing dump directory");
+ brpc::FLAGS_rpc_dump_dir = directory;
+ brpc::FLAGS_rpc_dump_max_requests_in_one_file = 1;
+ // Registered before Collector creation: its exit hook must finish first.
+ Require(atexit(VerifyExit) == 0, "cannot register exit verification");
+ if (prime_context) {
+ SampleState prime;
+ (new Request(&prime))->submit();
+ Require(WaitUntil([&] { return prime.destroyed.load() == 1; }),
+ "initial RPC dump did not finish");
+ }
+ g_expected_requests = prime_context ? 2 : 1;
+ g_exit_sample = new SampleState;
+ g_exit_sample->release.store(false);
+ (new Request(g_exit_sample))->submit();
+ Require(WaitUntil([] { return g_exit_sample->entered.load() == 1; }),
+ "RPC dump did not start");
+ std::thread([] {
+ Require(WaitUntil([] { return g_exiting.load(); }), "exit did not
start");
+ usleep(100000);
+ g_exit_sample->release.store(true);
+ }).detach();
+ Require(atexit([] { g_exiting.store(true); }) == 0,
+ "cannot register exit notification");
+ std::exit(0);
+}
+
+TEST_F(CollectorShutdownDeathTest, process_exit_waits_for_first_rpc_dump) {
+ EXPECT_EXIT(RunRpcDumpExit(false), ::testing::ExitedWithCode(0), "");
+}
+
+TEST_F(CollectorShutdownDeathTest, process_exit_waits_for_new_rpc_dump_round) {
+ EXPECT_EXIT(RunRpcDumpExit(true), ::testing::ExitedWithCode(0), "");
+}
+
+} // namespace
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]