This is an automated email from the ASF dual-hosted git repository.

chenBright pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/brpc.git


The following commit(s) were added to refs/heads/master by this push:
     new f7295044d Fix Collector use-after-free during process exit (#3721)
f7295044d is described below

commit f7295044da124c46055e35658b3bd4ce51d52cdd
Author: Bright Chen <[email protected]>
AuthorDate: Tue Nov 10 16:11:59 2026 +0800

    Fix Collector use-after-free during process exit (#3721)
    
    * Fix Collector use-after-free during process exit
    
    * Cover pre-main collector initialization
---
 BUILD.bazel                               |   6 +
 src/brpc/rpc_dump.cpp                     |   8 +
 src/bvar/collector.cpp                    | 125 +++++++++--
 src/bvar/collector.h                      |  15 ++
 test/BUILD.bazel                          |  23 +++
 test/CMakeLists.txt                       |   8 +
 test/Makefile                             |  14 ++
 test/brpc_collector_premain_unittest.cpp  | 147 +++++++++++++
 test/brpc_collector_shutdown_unittest.cpp | 333 ++++++++++++++++++++++++++++++
 9 files changed, 658 insertions(+), 21 deletions(-)

diff --git a/BUILD.bazel b/BUILD.bazel
index 092b827af..5e5eb49d5 100644
--- a/BUILD.bazel
+++ b/BUILD.bazel
@@ -20,6 +20,12 @@ licenses(["notice"])  # Apache v2
 
 exports_files(["LICENSE"])
 
+# Included by the pre-main shutdown test to control static initialization 
order.
+exports_files(
+    ["src/bvar/collector.cpp", "src/brpc/rpc_dump.cpp"],
+    visibility = ["//test:__pkg__"],
+)
+
 config_setting(
     name = "brpc_with_flatbuffers",
     define_values = {"BRPC_WITH_FLATBUFFERS": "true"},
diff --git a/src/brpc/rpc_dump.cpp b/src/brpc/rpc_dump.cpp
index a9dca57ca..c5455f5b8 100644
--- a/src/brpc/rpc_dump.cpp
+++ b/src/brpc/rpc_dump.cpp
@@ -62,6 +62,14 @@ BRPC_VALIDATE_GFLAG(rpc_dump, PassValidate);
 BRPC_VALIDATE_GFLAG(rpc_dump_max_requests_in_one_file, PositiveInteger);
 BRPC_VALIDATE_GFLAG(rpc_dump_max_files, PositiveInteger);
 
+// Declare this after the flags so both collector threads finish before the
+// string flag is destroyed, even if the collector was created before main().
+// This also prevents the first RPC dump from starting during later teardown.
+struct RpcDumpExitGuard {
+    ~RpcDumpExitGuard() { bvar::detail::StopCollector(); }
+};
+static RpcDumpExitGuard g_rpc_dump_exit_guard;
+
 static const size_t UNWRITTEN_BUFSIZE = 1024 * 1024;
 static const int64_t FLUSH_TIMEOUT = 2000000L; // 2s
 
diff --git a/src/bvar/collector.cpp b/src/bvar/collector.cpp
index 1fcf0db1a..9c221496d 100644
--- a/src/bvar/collector.cpp
+++ b/src/bvar/collector.cpp
@@ -17,6 +17,7 @@
 
 // Date: Mon Dec 14 19:12:30 CST 2015
 
+#include <cstdlib>
 #include <map>
 #include <gflags/gflags.h>
 #include "butil/memory/singleton_on_pthread_once.h"
@@ -64,6 +65,9 @@ public:
 
     void wakeup_grab_thread();
 
+    // Called once, after singleton creation has finished.
+    void stop_and_join();
+
 private:
     // The thread for collecting TLS submissions.
     void grab_thread();
@@ -99,7 +103,7 @@ private:
     int64_t _last_active_cpuwide_us;
     
     bool _created;      // Mark validness of _grab_thread.
-    bool _stop;         // Set to true in dtor.
+    butil::atomic<bool> _stop;
     pthread_t _grab_thread;     // For joining.
     pthread_t _dump_thread;
     int64_t _ngrab BAIDU_CACHELINE_ALIGNMENT;
@@ -112,6 +116,51 @@ private:
     pthread_cond_t _sleep_cond;
 };
 
+// These survive static destruction, just like the collector itself. Only the
+// cold creation path and shutdown take the mutex; submissions remain TLS 
based.
+static pthread_mutex_t g_collector_mutex = PTHREAD_MUTEX_INITIALIZER;
+static pthread_once_t g_collector_stop_once = PTHREAD_ONCE_INIT;
+static butil::static_atomic<bool> g_collector_stopped = 
BUTIL_STATIC_ATOMIC_INIT(false);
+
+static Collector* GetCollector() {
+    if (g_collector_stopped.load(butil::memory_order_acquire)) {
+        return nullptr;
+    }
+    Collector* collector = butil::has_leaky_singleton<Collector>();
+    if (collector != nullptr) {
+        return collector;
+    }
+    BAIDU_SCOPED_LOCK(g_collector_mutex);
+    if (g_collector_stopped.load(butil::memory_order_relaxed)) {
+        return nullptr;
+    }
+    return butil::get_leaky_singleton<Collector>();
+}
+
+static void StopCollectorOnce() {
+    Collector* collector;
+    {
+        BAIDU_SCOPED_LOCK(g_collector_mutex);
+        g_collector_stopped.store(true, butil::memory_order_release);
+        collector = butil::has_leaky_singleton<Collector>();
+    }
+    // A worker may have been waiting in GetCollector() during initialization.
+    // Let it observe shutdown instead of holding the creation lock while 
joining.
+    if (collector != nullptr) {
+        collector->stop_and_join();
+    }
+}
+
+void detail::StopCollector() {
+    pthread_once(&g_collector_stop_once, StopCollectorOnce);
+}
+
+// Also forbid first-time creation during later static destructors.
+struct CollectorExitGuard {
+    ~CollectorExitGuard() { detail::StopCollector(); }
+};
+static CollectorExitGuard g_collector_exit_guard;
+
 Collector::Collector()
     : _last_active_cpuwide_us(butil::cpuwide_time_us())
     , _created(false)
@@ -125,6 +174,9 @@ Collector::Collector()
     pthread_cond_init(&_dump_thread_cond, nullptr);
     pthread_mutex_init(&_sleep_mutex, nullptr);
     pthread_cond_init(&_sleep_cond, nullptr);
+    // The singleton is intentionally leaked. Register thread shutdown 
explicitly
+    // so runtime initialization stops callbacks before static destruction.
+    CHECK_EQ(0, atexit(detail::StopCollector));
     int rc = pthread_create(&_grab_thread, nullptr, run_grab_thread, this);
     if (rc != 0) {
         LOG(ERROR) << "Fail to create Collector, " << berror(rc);
@@ -134,17 +186,38 @@ Collector::Collector()
 }
 
 Collector::~Collector() {
-    if (_created) {
-        _stop = true;
-        pthread_join(_grab_thread, nullptr);
-        _created = false;
-    }
+    stop_and_join();
     pthread_mutex_destroy(&_dump_thread_mutex);
     pthread_cond_destroy(&_dump_thread_cond);
     pthread_mutex_destroy(&_sleep_mutex);
     pthread_cond_destroy(&_sleep_cond);
 }
 
+static void DestroySamples(butil::LinkNode<Collected>* root) {
+    while (root->next() != root) {
+        butil::LinkNode<Collected>* node = root->next();
+        node->RemoveFromList();
+        node->value()->destroy();
+    }
+}
+
+void Collector::stop_and_join() {
+    if (_created) {
+        _stop.store(true, butil::memory_order_relaxed);
+        wakeup_grab_thread();
+        // The grabber wakes and joins the dumper before it exits.
+        CHECK_EQ(0, pthread_join(_grab_thread, nullptr));
+        _created = false;
+    }
+    DestroySamples(&_dump_root);
+    butil::LinkNode<Collected> root;
+    Collected* head = reset();
+    if (head != nullptr) {
+        head->InsertBeforeAsList(&root);
+        DestroySamples(&root);
+    }
+}
+
 template <typename T>
 static T deref_value(void* arg) {
     return *(T*)arg;
@@ -185,7 +258,7 @@ void Collector::grab_thread() {
     PreprocessorMap prep_map;
 
     // The main loop.
-    while (!_stop) {
+    while (!_stop.load(butil::memory_order_relaxed)) {
         const int64_t abstime = _last_active_cpuwide_us + 
COLLECTOR_GRAB_INTERVAL_US;
 
         // Clear and reuse vectors in prep_map, don't clear prep_map directly.
@@ -266,10 +339,12 @@ void Collector::grab_thread() {
         _last_active_cpuwide_us = now;
 
         // sleep for the next round.
-        if (!_stop && abstime > now) {
+        if (abstime > now) {
             timespec abstimespec = butil::microseconds_from_now(abstime - now);
             pthread_mutex_lock(&_sleep_mutex);
-            pthread_cond_timedwait(&_sleep_cond, &_sleep_mutex, &abstimespec);
+            if (!_stop.load(butil::memory_order_relaxed)) {
+                pthread_cond_timedwait(&_sleep_cond, &_sleep_mutex, 
&abstimespec);
+            }
             pthread_mutex_unlock(&_sleep_mutex);
         }
         _last_active_cpuwide_us = butil::cpuwide_time_us();
@@ -277,7 +352,7 @@ void Collector::grab_thread() {
     // make sure _stop is true, we may have other reasons to quit above loop
     {
         BAIDU_SCOPED_LOCK(_dump_thread_mutex);
-        _stop = true; 
+        _stop.store(true, butil::memory_order_relaxed);
         pthread_cond_signal(&_dump_thread_cond);
     }
     CHECK_EQ(0, pthread_join(_dump_thread, nullptr));
@@ -352,7 +427,10 @@ size_t 
is_collectable_before_first_time_grabbed(CollectorSpeedLimit* sl) {
         if (before_add == 0) {
             sl->first_sample_us = butil::cpuwide_time_us();
         } else if (before_add >= FLAGS_bvar_collector_expected_per_second) {
-            butil::get_leaky_singleton<Collector>()->wakeup_grab_thread();
+            Collector* collector = GetCollector();
+            if (collector != nullptr) {
+                collector->wakeup_grab_thread();
+            }
         }
     }
     return sl->sampling_range;
@@ -376,19 +454,20 @@ void Collector::dump_thread() {
     size_t round = 0;
 
     // The main loop
-    while (!_stop) {
+    while (!_stop.load(butil::memory_order_relaxed)) {
         ++round;
         // Get new samples set by grab_thread.
         butil::LinkNode<Collected>* newhead = nullptr;
         {
             BAIDU_SCOPED_LOCK(_dump_thread_mutex);
-            while (!_stop && _dump_root.next() == &_dump_root) {
+            while (!_stop.load(butil::memory_order_relaxed) &&
+                   _dump_root.next() == &_dump_root) {
                 const int64_t now_ns = butil::cpuwide_time_ns();
                 busy_seconds += (now_ns - last_ns) / 1000000000.0;
                 pthread_cond_wait(&_dump_thread_cond, &_dump_thread_mutex);
                 last_ns = butil::cpuwide_time_ns();
             }
-            if (_stop) {
+            if (_stop.load(butil::memory_order_relaxed)) {
                 break;
             }
             newhead = _dump_root.next();
@@ -398,7 +477,8 @@ void Collector::dump_thread() {
         newhead->InsertBeforeAsList(&root);
 
         // Call callbacks.
-        for (butil::LinkNode<Collected>* p = root.next(); !_stop && p != 
&root;) {
+        for (butil::LinkNode<Collected>* p = root.next();
+             !_stop.load(butil::memory_order_relaxed) && p != &root;) {
             // We remove p from the list, save next first.
             butil::LinkNode<Collected>* saved_next = p->next();
             p->RemoveFromList();
@@ -408,15 +488,18 @@ void Collector::dump_thread() {
             p = saved_next;
         }
     }
+    DestroySamples(&root);
 }
 
-// Submit a sample for asynchronous dumping. The Collector holds only the 
Collected*
-// pointer (e.g., SpanContainer*). Regardless of which branch is taken below, 
the
-// sample will eventually be destroyed via either dump_and_destroy() or 
destroy(),
-// both of which call 'delete this' to release the container and decrement the
-// reference count of any managed resources (e.g., shared_ptr<Span>).
+// Submit a sample for asynchronous dumping. Shutdown rejects new submissions.
+// A submission already racing shutdown may still enter the leaked reducer 
after
+// its final reset; such samples are reclaimed along with the process.
 void Collected::submit(int64_t cpuwide_us) {
-    Collector* d = butil::get_leaky_singleton<Collector>();
+    Collector* d = GetCollector();
+    if (d == nullptr) {
+        destroy();
+        return;
+    }
     // Destroy the sample in-place if the grab_thread did not run for twice
     // of the normal interval. This also applies to the situation that
     // grab_thread aborts due to severe errors.
diff --git a/src/bvar/collector.h b/src/bvar/collector.h
index 1dadf49c6..d526b288d 100644
--- a/src/bvar/collector.h
+++ b/src/bvar/collector.h
@@ -28,6 +28,18 @@
 
 namespace bvar {
 
+namespace detail {
+// Permanently stop the process-wide collector and wait for its callbacks.
+// Discard pending samples without dumping. This is not a barrier for submit():
+// a submission overlapping shutdown may miss the final cleanup and remain
+// allocated until process termination, without another dump callback.
+// Internal exit hook, not a supported user API. May be called repeatedly, but
+// never from a collector callback. Does not create or delete the singleton.
+// Owners of destructible callback dependencies must call this before teardown;
+// the collector may have been initialized before those dependencies.
+void StopCollector();
+}  // namespace detail
+
 static const size_t INVALID_SAMPLING_RANGE = 0;
 
 inline bool is_sampling_range_valid(size_t sampling_range) {
@@ -75,6 +87,9 @@ public:
     // from the timestamp updated by collecting thread(which basically means
     // the thread is not scheduled by OS in time), this sample is directly
     // destroy()-ed to avoid memory explosion.
+    // Calls started after collector shutdown are destroy()-ed in-place.
+    // Calls overlapping shutdown may leave the sample allocated until process
+    // termination without calling destroy(), as described below.
     void submit(int64_t cpuwide_us);
     void submit() { submit(butil::cpuwide_time_us()); }
 
diff --git a/test/BUILD.bazel b/test/BUILD.bazel
index 268941839..594e16800 100644
--- a/test/BUILD.bazel
+++ b/test/BUILD.bazel
@@ -281,11 +281,34 @@ cc_test(
     ],
 )
 
+# Textual inclusion fixes the two implementations' initialization order. Static
+# linking supplies the remaining brpc objects without duplicate flag 
definitions.
+cc_library(
+    name = "collector_premain_sources",
+    textual_hdrs = [
+        "//:src/bvar/collector.cpp",
+        "//:src/brpc/rpc_dump.cpp",
+    ],
+    deps = ["//:brpc"],
+)
+
+cc_test(
+    name = "brpc_collector_premain_unittest",
+    srcs = ["brpc_collector_premain_unittest.cpp"],
+    copts = COPTS,
+    linkstatic = True,
+    deps = [
+        ":collector_premain_sources",
+        "@com_google_googletest//:gtest_main",
+    ] + TCMALLOC_DEP_UNLESS_ASAN,
+)
+
 generate_unittests(
     name = "brpc_unittests",
     srcs = glob(
         ["brpc_*_unittest.cpp"],
         exclude = [
+            "brpc_collector_premain_unittest.cpp",
             "brpc_flatbuffers_unittest.cpp",
             "brpc_flatbuffers_protocol_unittest.cpp",
         ],
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 20becf6ca..25b107cf1 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -340,6 +340,7 @@ endif()
 # brpc tests
 file(GLOB BRPC_UNITTESTS "brpc_*_unittest.cpp")
 list(REMOVE_ITEM BRPC_UNITTESTS
+    "${CMAKE_CURRENT_SOURCE_DIR}/brpc_collector_premain_unittest.cpp"
     "${CMAKE_CURRENT_SOURCE_DIR}/brpc_flatbuffers_unittest.cpp"
     "${CMAKE_CURRENT_SOURCE_DIR}/brpc_flatbuffers_protocol_unittest.cpp")
 foreach(BRPC_UT ${BRPC_UNITTESTS})
@@ -353,6 +354,13 @@ foreach(BRPC_UT ${BRPC_UNITTESTS})
     add_test(NAME ${BRPC_UT_WE} COMMAND ${BRPC_UT_WE})
 endforeach()
 
+# This test includes collector.cpp and rpc_dump.cpp to control initialization
+# order. A static archive supplies the other objects without duplicate flags.
+add_executable(brpc_collector_premain_unittest 
brpc_collector_premain_unittest.cpp)
+target_link_libraries(brpc_collector_premain_unittest PRIVATE
+    brpc-static brpc_test_config gtest_main ${GPERFTOOLS_LIBRARIES})
+add_test(NAME brpc_collector_premain_unittest COMMAND 
brpc_collector_premain_unittest)
+
 if(BUILD_FUZZ_TESTS)
     add_library(brpc-static-debug STATIC $<TARGET_OBJECTS:BUTIL_DEBUG_LIB>
                                         $<TARGET_OBJECTS:SOURCES_DEBUG_LIB>
diff --git a/test/Makefile b/test/Makefile
index ccad5483d..7ee0ce33a 100644
--- a/test/Makefile
+++ b/test/Makefile
@@ -262,6 +262,20 @@ else ifeq ($(SYSTEM),Darwin)
 endif
 endif
 
+# The pre-main test includes both implementations and needs a static archive to
+# avoid loading duplicate flags from the shared test library.
+../libbrpc.a:FORCE
+       $(MAKE) -C.. libbrpc.a
+
+brpc_collector_premain_unittest.o:../src/bvar/collector.cpp 
../src/brpc/rpc_dump.cpp
+
+brpc_collector_premain_unittest:brpc_collector_premain_unittest.o ../libbrpc.a
+ifeq ($(SYSTEM),Linux)
+       $(CXX) -o $@ $(LIBPATHS) $(SOPATHS) -Xlinker "-(" $^ $(STATIC_LINKINGS) 
-Xlinker "-)" $(DYNAMIC_LINKINGS)
+else ifeq ($(SYSTEM),Darwin)
+       $(CXX) -o $@ $(LIBPATHS) $(SOPATHS) $^ $(STATIC_LINKINGS) 
$(DYNAMIC_LINKINGS)
+endif
+
 brpc_%_unittest:$(TEST_PROTO_OBJS) brpc_%_unittest.o | libbrpc.dbg.$(SOEXT)
        @echo "> Linking $@"
 ifeq ($(SYSTEM),Linux)
diff --git a/test/brpc_collector_premain_unittest.cpp 
b/test/brpc_collector_premain_unittest.cpp
new file mode 100644
index 000000000..d55b0d1ba
--- /dev/null
+++ b/test/brpc_collector_premain_unittest.cpp
@@ -0,0 +1,147 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <atomic>
+#include <cstdlib>
+#include <cstring>
+#include <memory>
+#include <unistd.h>
+#include <gtest/gtest.h>
+#include "butil/files/scoped_temp_dir.h"
+
+// Compile these implementations in this translation unit to fix initialization
+// order independently of the linker and shared-library loading order. Link the
+// remaining dependencies statically so there is only one copy of each flag and
+// singleton. The CollectorExitGuard must be registered before the RPC flags.
+#include "bvar/collector.cpp"
+
+namespace {
+
+const char kDirectoryEnv[] = "BRPC_COLLECTOR_PREMAIN_TEST_DIR";
+std::atomic<bool> g_dump_started{false};
+std::atomic<bool> g_dump_finished{false};
+bool g_initialized_before_flags = false;
+
+void Require(bool condition, const char* message) {
+    if (!condition) {
+        (void)write(STDERR_FILENO, message, strlen(message));
+        (void)write(STDERR_FILENO, "\n", 1);
+        std::_Exit(1);
+    }
+}
+
+template <typename Predicate>
+bool WaitUntil(Predicate predicate) {
+    const int64_t deadline = butil::monotonic_time_ms() + 10000;
+    while (!predicate() && butil::monotonic_time_ms() < deadline) {
+        usleep(1000);
+    }
+    return predicate();
+}
+
+void VerifyAfterFlagDestruction() {
+    Require(g_dump_finished.load(),
+            "RPC flags were destroyed before the pre-main collector stopped");
+}
+
+struct InitCollectorBeforeFlags {
+    InitCollectorBeforeFlags() {
+        // Only the re-executed death-test child creates an early collector.
+        if (getenv(kDirectoryEnv) == nullptr) {
+            return;
+        }
+        alarm(15);
+        Require(butil::has_leaky_singleton<bvar::Collector>() == nullptr,
+                "collector was already initialized");
+        butil::get_leaky_singleton<bvar::Collector>();
+        g_initialized_before_flags = true;
+        // At exit: RpcDumpExitGuard -> RPC flags -> verification -> 
Collector's
+        // atexit hook -> CollectorExitGuard. Neither fallback can mask 
failure.
+        Require(atexit(VerifyAfterFlagDestruction) == 0,
+                "cannot register exit verification");
+    }
+};
+InitCollectorBeforeFlags g_init_collector_before_flags;
+
+}  // namespace
+
+// Keep this below the initializer and its exit verification registration.
+#include "brpc/rpc_dump.cpp"
+
+namespace {
+
+class Request : public brpc::SampledRequest {
+public:
+    Request() {
+        request.append("pre-main collector request");
+        meta.set_protocol_type(brpc::PROTOCOL_BAIDU_STD);
+    }
+
+    void dump_and_destroy(size_t round) override {
+        g_dump_started.store(true);
+        // Resume only when shutdown starts, so completion cannot race ahead of
+        // exit verification if RpcDumpExitGuard is removed. SaveFlags must
+        // still be able to read rpc_dump_dir at this point.
+        Require(WaitUntil([] {
+            return bvar::g_collector_stopped.load(butil::memory_order_acquire);
+        }), "collector shutdown did not start");
+        brpc::SampledRequest::dump_and_destroy(round);
+    }
+
+    void destroy() override {
+        delete this;
+        g_dump_finished.store(true);
+    }
+};
+
+void RunExit() {
+    Require(g_initialized_before_flags,
+            "collector was not initialized before the RPC flags");
+    brpc::FLAGS_rpc_dump_dir = getenv(kDirectoryEnv);
+    brpc::FLAGS_rpc_dump_max_requests_in_one_file = 1;
+    (new Request)->submit();
+    Require(WaitUntil([] { return g_dump_started.load(); }),
+            "RPC dump did not start");
+    std::exit(0);
+}
+
+TEST(CollectorPremainDeathTest, stops_before_rpc_flags_are_destroyed) {
+#ifdef GTEST_FLAG_SET
+    GTEST_FLAG_SET(death_test_style, "threadsafe");
+#else
+    ::testing::GTEST_FLAG(death_test_style) = "threadsafe";
+#endif
+    butil::ScopedTempDir directory;
+    const bool parent = getenv(kDirectoryEnv) == nullptr;
+    if (parent) {
+        ASSERT_TRUE(directory.CreateUniqueTempDir());
+        ASSERT_EQ(0, setenv(kDirectoryEnv,
+                           directory.path().value().c_str(), 1));
+    }
+    EXPECT_EXIT(RunExit(), ::testing::ExitedWithCode(0), "");
+    if (parent) {
+        EXPECT_EQ(0, unsetenv(kDirectoryEnv));
+        brpc::SampleIterator it(directory.path().value());
+        std::unique_ptr<brpc::SampledRequest> request(it.Next());
+        ASSERT_TRUE(request);
+        EXPECT_EQ("pre-main collector request", request->request.to_string());
+        std::unique_ptr<brpc::SampledRequest> extra(it.Next());
+        EXPECT_FALSE(extra);
+    }
+}
+
+}  // namespace
diff --git a/test/brpc_collector_shutdown_unittest.cpp 
b/test/brpc_collector_shutdown_unittest.cpp
new file mode 100644
index 000000000..08f558125
--- /dev/null
+++ b/test/brpc_collector_shutdown_unittest.cpp
@@ -0,0 +1,333 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <atomic>
+#include <cstdlib>
+#include <cstring>
+#include <memory>
+#include <thread>
+#include <unistd.h>
+#include <gtest/gtest.h>
+#include "butil/files/scoped_temp_dir.h"
+#include "butil/memory/singleton_on_pthread_once.h"
+#include "brpc/rpc_dump.h"
+
+namespace bvar {
+class Collector;
+}
+
+namespace brpc {
+DECLARE_string(rpc_dump_dir);
+DECLARE_int32(rpc_dump_max_requests_in_one_file);
+}
+
+namespace {
+
+const char kDirectoryEnv[] = "BRPC_COLLECTOR_SHUTDOWN_TEST_DIR";
+
+void Require(bool condition, const char* message) {
+    if (!condition) {
+        (void)write(STDERR_FILENO, message, strlen(message));
+        (void)write(STDERR_FILENO, "\n", 1);
+        std::_Exit(1);
+    }
+}
+
+template <typename Predicate>
+bool WaitUntil(Predicate predicate, int64_t timeout_ms = 10000) {
+    const int64_t deadline = butil::monotonic_time_ms() + timeout_ms;
+    while (!predicate() && butil::monotonic_time_ms() < deadline) {
+        usleep(1000);
+    }
+    return predicate();
+}
+
+struct SampleState {
+    std::atomic<int> entered{0};
+    std::atomic<int> destroyed{0};
+    std::atomic<bool> release{true};
+};
+
+class Sample : public bvar::Collected {
+public:
+    explicit Sample(SampleState* state,
+                    bvar::CollectorPreprocessor* preprocessor = nullptr)
+        : _state(state), _preprocessor(preprocessor) {}
+
+    void dump_and_destroy(size_t) override {
+        _state->entered.fetch_add(1);
+        Require(WaitUntil([this] { return _state->release.load(); }),
+                "sample was not released");
+        destroy();
+    }
+
+    void destroy() override {
+        SampleState* state = _state;
+        delete this;
+        state->destroyed.fetch_add(1);
+    }
+
+    bvar::CollectorSpeedLimit* speed_limit() override { return nullptr; }
+    bvar::CollectorPreprocessor* preprocessor() override { return 
_preprocessor; }
+
+private:
+    SampleState* _state;
+    bvar::CollectorPreprocessor* _preprocessor;
+};
+
+// Put another sample behind the blocker in the very same dump batch.
+class AppendSample : public bvar::CollectorPreprocessor {
+public:
+    explicit AppendSample(SampleState* state) : _state(state) {}
+    void process(std::vector<bvar::Collected*>& samples) override {
+        samples.push_back(new Sample(_state));
+    }
+
+private:
+    SampleState* _state;
+};
+
+class CollectorShutdownDeathTest : public ::testing::Test {
+protected:
+    void SetUp() override {
+        // Re-exec children so inherited collectors never lose their workers.
+#ifdef GTEST_FLAG_SET
+        GTEST_FLAG_SET(death_test_style, "threadsafe");
+#else
+        ::testing::GTEST_FLAG(death_test_style) = "threadsafe";
+#endif
+        if (getenv(kDirectoryEnv) == nullptr) {
+            ASSERT_TRUE(_directory.CreateUniqueTempDir());
+            ASSERT_EQ(0, setenv(kDirectoryEnv, 
_directory.path().value().c_str(), 1));
+            _owns_directory = true;
+        }
+    }
+
+    void TearDown() override {
+        if (_owns_directory) {
+            EXPECT_EQ(0, unsetenv(kDirectoryEnv));
+        }
+    }
+
+private:
+    butil::ScopedTempDir _directory;
+    bool _owns_directory = false;
+};
+
+TEST_F(CollectorShutdownDeathTest, stop_before_first_sample_prevents_creation) 
{
+    EXPECT_EXIT({
+        alarm(15);
+        Require(butil::has_leaky_singleton<bvar::Collector>() == nullptr,
+                "collector already exists");
+        bvar::detail::StopCollector();
+        bvar::detail::StopCollector();
+        SampleState state;
+        (new Sample(&state))->submit();
+        Require(state.destroyed.load() == 1 && state.entered.load() == 0,
+                "late sample was not discarded");
+        bvar::CollectorSpeedLimit limit = {};
+        limit.sampling_range = bvar::COLLECTOR_SAMPLING_BASE;
+        limit.count_before_grabbed.store(1000000, butil::memory_order_relaxed);
+        bvar::is_collectable(&limit);
+        Require(butil::has_leaky_singleton<bvar::Collector>() == nullptr,
+                "shutdown created a collector");
+        std::exit(0);
+    }, ::testing::ExitedWithCode(0), "");
+}
+
+TEST_F(CollectorShutdownDeathTest, 
concurrent_stops_wait_and_discard_pending_samples) {
+    EXPECT_EXIT({
+        alarm(15);
+        SampleState blocked;
+        blocked.release.store(false);
+        SampleState queued;
+        AppendSample append(&queued);
+        (new Sample(&blocked, &append))->submit();
+        Require(WaitUntil([&] { return blocked.entered.load() == 1; }),
+                "dump did not start");
+        // Exercise submissions from a thread whose TLS has already been freed.
+        std::thread producer([&] {
+            for (int i = 0; i < 16; ++i) {
+                (new Sample(&queued))->submit();
+            }
+        });
+        producer.join();
+
+        std::atomic<int> stopping{0};
+        std::atomic<int> stopped{0};
+        auto stop = [&] {
+            stopping.fetch_add(1);
+            bvar::detail::StopCollector();
+            stopped.fetch_add(1);
+        };
+        std::thread stopper1(stop);
+        std::thread stopper2(stop);
+        Require(WaitUntil([&] { return stopping.load() == 2; }),
+                "stop callers did not start");
+        Require(!WaitUntil([&] { return stopped.load() != 0; }, 100),
+                "stop returned before the in-flight dump finished");
+        blocked.release.store(true);
+        stopper1.join();
+        stopper2.join();
+        Require(blocked.destroyed.load() == 1, "in-flight sample was not 
released");
+        Require(queued.destroyed.load() == 17 && queued.entered.load() == 0,
+                "pending samples were dumped or not released");
+        (new Sample(&queued))->submit();
+        Require(queued.destroyed.load() == 18 && queued.entered.load() == 0,
+                "late submission was not discarded");
+        std::exit(0);
+    }, ::testing::ExitedWithCode(0), "");
+}
+
+TEST_F(CollectorShutdownDeathTest, stop_races_first_submissions) {
+    EXPECT_EXIT({
+        alarm(15);
+        SampleState state;
+        std::atomic<bool> start{false};
+        std::atomic<int> ready{0};
+        std::thread producers[8];
+        for (auto& producer : producers) {
+            producer = std::thread([&] {
+                ready.fetch_add(1);
+                while (!start.load()) {
+                    std::this_thread::yield();
+                }
+                (new Sample(&state))->submit();
+            });
+        }
+        Require(WaitUntil([&] { return ready.load() == 8; }),
+                "producers did not start");
+        start.store(true);
+        bvar::detail::StopCollector();
+        const int dumped = state.entered.load();
+        for (auto& producer : producers) {
+            producer.join();
+        }
+        bvar::detail::StopCollector();
+        Require(state.entered.load() == dumped, "dump started after stop 
returned");
+        SampleState late;
+        (new Sample(&late))->submit();
+        Require(late.destroyed.load() == 1 && late.entered.load() == 0,
+                "racing creation restarted the collector");
+        std::exit(0);
+    }, ::testing::ExitedWithCode(0), "");
+}
+
+TEST_F(CollectorShutdownDeathTest, stop_wakes_idle_workers) {
+    EXPECT_EXIT({
+        alarm(15);
+        SampleState state;
+        (new Sample(&state))->submit();
+        Require(WaitUntil([&] { return state.destroyed.load() == 1; }),
+                "sample was not dumped");
+        bvar::detail::StopCollector();
+        bvar::detail::StopCollector();
+        Require(butil::has_leaky_singleton<bvar::Collector>() != nullptr,
+                "shutdown destroyed the singleton");
+        (new Sample(&state))->submit();
+        Require(state.destroyed.load() == 2 && state.entered.load() == 1,
+                "idle collector restarted after shutdown");
+        std::exit(0);
+    }, ::testing::ExitedWithCode(0), "");
+}
+
+class Request : public brpc::SampledRequest {
+public:
+    explicit Request(SampleState* state) : _state(state) {
+        request.append("collector shutdown request");
+        meta.set_protocol_type(brpc::PROTOCOL_BAIDU_STD);
+    }
+
+    void dump_and_destroy(size_t round) override {
+        _state->entered.fetch_add(1);
+        Require(WaitUntil([this] { return _state->release.load(); }),
+                "RPC dump was not released");
+        // Exercise the actual RpcDumpContext::SaveFlags(), including first 
use.
+        brpc::SampledRequest::dump_and_destroy(round);
+    }
+
+    void destroy() override {
+        SampleState* state = _state;
+        delete this;
+        state->destroyed.fetch_add(1);
+    }
+
+private:
+    SampleState* _state;
+};
+
+static SampleState* g_exit_sample = nullptr;
+static std::atomic<bool> g_exiting{false};
+static int g_expected_requests = 0;
+
+void VerifyExit() {
+    Require(g_exit_sample->destroyed.load() == 1,
+            "exit did not wait for the RPC dump");
+    SampleState late;
+    (new Request(&late))->submit();
+    Require(late.destroyed.load() == 1 && late.entered.load() == 0,
+            "exit allowed a late RPC dump");
+    brpc::SampleIterator it(getenv(kDirectoryEnv));
+    for (int i = 0; i < g_expected_requests; ++i) {
+        std::unique_ptr<brpc::SampledRequest> request(it.Next());
+        Require(request &&
+                request->request.to_string() == "collector shutdown request",
+                "in-flight RPC dump was not persisted before exit");
+    }
+    std::unique_ptr<brpc::SampledRequest> extra(it.Next());
+    Require(!extra, "unexpected extra RPC dump");
+}
+
+void RunRpcDumpExit(bool prime_context) {
+    alarm(15);
+    const char* directory = getenv(kDirectoryEnv);
+    Require(directory != nullptr, "missing dump directory");
+    brpc::FLAGS_rpc_dump_dir = directory;
+    brpc::FLAGS_rpc_dump_max_requests_in_one_file = 1;
+    // Registered before Collector creation: its exit hook must finish first.
+    Require(atexit(VerifyExit) == 0, "cannot register exit verification");
+    if (prime_context) {
+        SampleState prime;
+        (new Request(&prime))->submit();
+        Require(WaitUntil([&] { return prime.destroyed.load() == 1; }),
+                "initial RPC dump did not finish");
+    }
+    g_expected_requests = prime_context ? 2 : 1;
+    g_exit_sample = new SampleState;
+    g_exit_sample->release.store(false);
+    (new Request(g_exit_sample))->submit();
+    Require(WaitUntil([] { return g_exit_sample->entered.load() == 1; }),
+            "RPC dump did not start");
+    std::thread([] {
+        Require(WaitUntil([] { return g_exiting.load(); }), "exit did not 
start");
+        usleep(100000);
+        g_exit_sample->release.store(true);
+    }).detach();
+    Require(atexit([] { g_exiting.store(true); }) == 0,
+            "cannot register exit notification");
+    std::exit(0);
+}
+
+TEST_F(CollectorShutdownDeathTest, process_exit_waits_for_first_rpc_dump) {
+    EXPECT_EXIT(RunRpcDumpExit(false), ::testing::ExitedWithCode(0), "");
+}
+
+TEST_F(CollectorShutdownDeathTest, process_exit_waits_for_new_rpc_dump_round) {
+    EXPECT_EXIT(RunRpcDumpExit(true), ::testing::ExitedWithCode(0), "");
+}
+
+}  // namespace


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to