Copilot commented on code in PR #3761:
URL: https://github.com/apache/brpc/pull/3761#discussion_r4625558238


##########
test/brpc_rdma_unittest.cpp:
##########
@@ -3995,6 +4004,193 @@ TEST(RdmaInlineDataTest, should_post_inline) {
     EXPECT_FALSE(rdma::ShouldPostInline(true, 1, 0));
 }
 
+
+// Regression tests for the endpoint lifetime: the endpoint must outlive the
+// CQ Socket, and PollCq() must not dereference a recycled main Socket.
+namespace {
+
+static int FakeGetCqEvent(ibv_comp_channel*, ibv_cq**, void**) {
+    errno = EAGAIN;
+    return -1;
+}
+static void FakeAckCqEvents(ibv_cq*, unsigned int) {}
+static int FakeDestroyCq(ibv_cq*) { return 0; }
+static int FakeDestroyQp(ibv_qp*) { return 0; }
+static int FakeDestroyCompChannel(ibv_comp_channel*) { return 0; }
+static int FakeModifyQp(ibv_qp*, ibv_qp_attr*, ibv_qp_attr_mask) { return 0; }
+
+
+struct IbvMockGuard {
+    decltype(rdma::IbvGetCqEvent) get_cq_event;
+    decltype(rdma::IbvAckCqEvents) ack_cq_events;
+    decltype(rdma::IbvDestroyCq) destroy_cq;
+    decltype(rdma::IbvDestroyQp) destroy_qp;
+    decltype(rdma::IbvDestroyCompChannel) destroy_comp_channel;
+    decltype(rdma::IbvModifyQp) modify_qp;
+    int32_t prepared_qp_cnt;
+    bool use_polling;
+
+    IbvMockGuard()
+        : get_cq_event(rdma::IbvGetCqEvent)
+        , ack_cq_events(rdma::IbvAckCqEvents)
+        , destroy_cq(rdma::IbvDestroyCq)
+        , destroy_qp(rdma::IbvDestroyQp)
+        , destroy_comp_channel(rdma::IbvDestroyCompChannel)
+        , modify_qp(rdma::IbvModifyQp)
+        , prepared_qp_cnt(rdma::FLAGS_rdma_prepared_qp_cnt)
+        , use_polling(rdma::FLAGS_rdma_use_polling) {
+        rdma::IbvGetCqEvent = FakeGetCqEvent;
+        rdma::IbvAckCqEvents = FakeAckCqEvents;
+        rdma::IbvDestroyCq = FakeDestroyCq;
+        rdma::IbvDestroyQp = FakeDestroyQp;
+        rdma::IbvDestroyCompChannel = FakeDestroyCompChannel;
+        rdma::IbvModifyQp = FakeModifyQp;
+        rdma::FLAGS_rdma_prepared_qp_cnt = 0;
+        rdma::FLAGS_rdma_use_polling = false;
+    }
+
+    ~IbvMockGuard() {
+        rdma::IbvGetCqEvent = get_cq_event;
+        rdma::IbvAckCqEvents = ack_cq_events;
+        rdma::IbvDestroyCq = destroy_cq;
+        rdma::IbvDestroyQp = destroy_qp;
+        rdma::IbvDestroyCompChannel = destroy_comp_channel;
+        rdma::IbvModifyQp = modify_qp;
+        rdma::FLAGS_rdma_prepared_qp_cnt = prepared_qp_cnt;
+        rdma::FLAGS_rdma_use_polling = use_polling;
+    }
+};
+
+// A main RDMA Socket with a mocked resource, plus its CQ Socket.
+struct FakeRdmaConnection {
+    IbvMockGuard mock_guard;
+    ibv_context fake_ctx;
+    ibv_cq fake_recv_cq;
+    ibv_cq fake_send_cq;
+    ibv_comp_channel fake_comp_channel;
+    int comp_fd;
+    SocketId main_sid;
+    SocketUniquePtr main_socket;
+    RdmaTransport* transport;
+    rdma::RdmaEndpoint* ep;
+
+    FakeRdmaConnection()
+        : fake_ctx()
+        , fake_recv_cq()
+        , fake_send_cq()
+        , fake_comp_channel()
+        , comp_fd(eventfd(0, 0))
+        , transport(nullptr)
+        , ep(nullptr) {
+        fake_recv_cq.context = &fake_ctx;
+        fake_send_cq.context = &fake_ctx;
+        fake_comp_channel.fd = comp_fd;
+    }
+    ~FakeRdmaConnection() {
+        if (comp_fd >= 0) {
+            close(comp_fd);
+        }
+    }
+
+    int Setup() {
+        SocketOptions main_options;
+        main_options.socket_mode = SOCKET_MODE_RDMA;
+        if (Socket::Create(main_options, &main_sid) != 0) {
+            return -1;
+        }
+        if (Socket::Address(main_sid, &main_socket) != 0) {
+            return -1;
+        }
+        transport = static_cast<RdmaTransport*>(main_socket->_transport.get());
+        ep = transport->_rdma_ep;
+        if (ep == nullptr) {
+            return -1;
+        }
+        ep->_state.store(rdma::RdmaEndpoint::ESTABLISHED,
+                         butil::memory_order_release);
+        rdma::RdmaResource* res = new rdma::RdmaResource();
+        res->recv_cq = &fake_recv_cq;
+        res->send_cq = &fake_send_cq;
+        res->comp_channel = &fake_comp_channel;
+        ep->_resource = res;
+        return ep->StartCqEvents();
+    }
+};
+
+}  // namespace
+
+TEST_F(RdmaTest, endpoint_outlives_cq_socket_and_reset) {
+    const int live_before = rdma::g_live_endpoint_count.load();
+
+    // Case 1: the main Socket is recycled while a PollCq callback still holds
+    // a CQ Socket reference. The endpoint must outlive it, and a late callback
+    // must return safely.
+    {
+        FakeRdmaConnection conn;
+        ASSERT_GE(conn.comp_fd, 0);
+        ASSERT_EQ(0, conn.Setup());
+        rdma::RdmaEndpoint* ep = conn.ep;
+        ASSERT_NE(INVALID_SOCKET_ID, ep->_cq_sid);
+
+        // Hold a CQ Socket ref, as an in-flight PollCq would.
+        SocketUniquePtr cq_socket;
+        ASSERT_EQ(0, Socket::Address(ep->_cq_sid, &cq_socket));
+
+        // Recycle the main Socket as at exit: the drain must wait for that 
ref.
+        butil::atomic<bool> main_recycled(false);
+        std::thread recycler([&] {
+            conn.main_socket->ReleaseAdditionalReference();
+            conn.main_socket.reset();
+            main_recycled.store(true);
+        });
+        ASSERT_TRUE(WaitUntil([&] { return cq_socket->Failed(); }));
+
+        rdma::RdmaEndpoint::PollCq(cq_socket.get());

Review Comment:
   This call reaches `ibv_poll_cq` directly; unlike the mocked 
`IbvGetCqEvent`/destroy hooks, it is not redirected through a test seam. The 
test supplies zero-initialized stack `ibv_context`/`ibv_cq` objects, so 
libibverbs can dereference an uninitialized provider callback here and crash 
before the lifetime assertions run. Add an injectable poll wrapper and mock it, 
or drive this test with a real CQ.



##########
src/brpc/rdma/rdma_endpoint.cpp:
##########
@@ -1475,6 +1482,9 @@ int RdmaEndpoint::StartCqEvents() {
         return -1;
     }
 
+    // Before Create(): OnCreated() may call BeforeRecycle() before returning.
+    _nref.fetch_add(1, butil::memory_order_relaxed);

Review Comment:
   The extra endpoint reference is acquired before `Socket::Create`, but this 
failure path does not balance it. `Socket::OnCreated()` can fail before it 
assigns `options.user` (for example, `_io_event.Init()` fails), so 
`BeforeRecycle()` is never called to execute `DropRef()`. `StartCqEvents()` 
then returns `-1` with `_nref` still one higher; later transport release drops 
only the transport reference and leaves the endpoint leaked (and the new 
live-endpoint counter never returns to baseline). Track whether the CQ socket 
took ownership of this reference and release it on create failure when it did 
not, while avoiding a second drop when `BeforeRecycle()` already ran.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to