Copilot commented on code in PR #3720:
URL: https://github.com/apache/brpc/pull/3720#discussion_r4571004723


##########
src/brpc/rdma/srq_receive_pool.cpp:
##########
@@ -0,0 +1,624 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#if BRPC_WITH_RDMA
+#include "brpc/rdma/srq_receive_pool.h"
+
+#include <algorithm>
+#include <errno.h>
+#include <stdlib.h>
+#include <sys/eventfd.h>
+#include <sys/mman.h>
+#include <unistd.h>
+#include <map>
+#include <new>
+#include <vector>
+#include <gflags/gflags.h>
+#include "bthread/bthread.h"
+#include "bthread/mutex.h"
+#include "butil/containers/doubly_buffered_data.h"
+#include "butil/iobuf_inl.h"
+#include "butil/logging.h"
+#include "butil/scoped_lock.h"
+
+namespace brpc {
+namespace rdma {
+extern ibv_mr* (*IbvRegMr)(ibv_pd*, void*, size_t, int);
+extern int (*IbvDeregMr)(ibv_mr*);
+
+DEFINE_int32(rdma_srq_pool_grow_blocks, 256,
+             "Receive blocks per runtime SRQ pool growth step");
+DEFINE_int32(rdma_srq_pool_growth_retry_ms, 100,
+             "Delay before retrying failed SRQ pool allocation/registration");
+DEFINE_bool(rdma_srq_pool_hugepages, false,
+            "Align SRQ receive regions to 2 MiB and request transparent huge 
pages");
+
+// Each region owns an MR, page-aligned memory and a descriptor array. Growth
+// adds regions without relocating existing storage, keeping posted receive
+// buffers and application-held zero-copy addresses valid.
+struct SrqReceivePool::Region {
+    // Registration handle. Destruction retains pages if deregistration fails.
+    ibv_mr* mr{nullptr};
+    // Page-aligned allocation, moved to global quarantine if deregistration
+    // fails.
+    void* memory{nullptr};
+    // Receive descriptors with stable addresses throughout the region 
lifetime.
+    std::unique_ptr<Buffer[]> buffers;
+    // Number of physical blocks in this region.
+    uint32_t count{0};
+    // Registered extent (alignment padding, if any, is not registered).
+    size_t bytes{0};
+    // Owner-only allocation order and usage. Returns remain in use until the
+    // owner detaches them, even if the final application reference is gone.
+    size_t index{0};
+    uint32_t in_use{0};
+    Buffer* free{nullptr};
+    // Whether destruction must first remove the address index entry.
+    bool published{false};
+
+    static bool Later(const Region* lhs, const Region* rhs) {
+        return lhs->index > rhs->index;
+    }
+
+    ~Region();
+};
+
+struct SrqReceivePool::Growth {
+    // Retained by the task through result publication and eventfd 
notification.
+    SrqReceivePool* pool{nullptr};
+    // Reserved against the owner's remaining capacity before task launch.
+    uint32_t count{0};
+    // Null after failure. Even a failure publishes a result to end _growing.
+    std::unique_ptr<Region> region;
+};
+
+namespace {
+struct PoolRegistry {
+    struct LiveRegion {
+        // The outstanding block keeps the pool alive until Return releases its
+        // reference.
+        SrqReceivePool* pool;
+        // Descriptor array, indexed by the returned block offset within the
+        // region.
+        SrqReceivePool::Buffer* buffers;
+        // Region extent. Excludes adjacent addresses from pool lookups.
+        size_t bytes;
+        // Registration key used by the IOBuf send path.
+        uint32_t lkey;
+    };
+
+    struct RetiredRegion {
+        // MR whose failed deregistration requires an explicit retry.
+        ibv_mr* mr;
+        // Pages retained until the provider confirms successful 
deregistration.
+        void* memory;
+        // Bytes still registered with the provider.
+        size_t bytes;
+    };
+
+    // Lookups may point inside a region, so upper_bound must find its
+    // predecessor. butil::FlatMap supports exact keys only and cannot directly
+    // replace this ordered interval index.
+    using RegionMap = std::map<uintptr_t, LiveRegion>;
+    // Protects only byte accounting and failed deregistrations, not index
+    // access or verbs calls.
+    bthread::Mutex mutex;
+    // Maps region bases to lkeys and descriptors. Readers lock only their own
+    // pthread TLS wrapper. Removing an index entry waits for old readers 
before
+    // descriptors or MRs can be reclaimed. Never suspend a bthread inside a
+    // read section. Modify serializes writers internally and does not need the
+    // accounting mutex above.
+    butil::DoublyBufferedData<RegionMap> regions;
+    // Quarantined allocations retained after failed deregistration.
+    std::vector<RetiredRegion> retired;
+    // Live pool count. Zero lets RegionId/Recycle skip the address lookup.
+    butil::atomic<size_t> count{0};
+    // Registered bytes, including live regions and failed deregistrations.
+    size_t bytes{0};
+};
+
+PoolRegistry& Registry() {
+    // IOBufs may be released during static destruction. The registry must
+    // outlive them.
+    static auto registry = new PoolRegistry;
+    return *registry;
+}
+
+PoolRegistry::RegionMap::const_iterator
+FindRegion(const void* address, const PoolRegistry::RegionMap& regions) {
+    uintptr_t ptr = reinterpret_cast<uintptr_t>(address);
+    auto it = regions.upper_bound(ptr);
+    if (it == regions.begin()) {
+        return regions.end();
+    }
+    --it;
+    return ptr - it->first < it->second.bytes ? it : regions.end();
+}
+}  // namespace
+
+SrqReceivePool::Region::~Region() {
+    auto& registry = Registry();
+    if (published) {
+        registry.regions.Modify([&](PoolRegistry::RegionMap& regions) {
+            return regions.erase(reinterpret_cast<uintptr_t>(memory));
+        });
+    }
+    if (mr != nullptr) {
+        int rc = IbvDeregMr(mr);
+        BAIDU_SCOPED_LOCK(registry.mutex);
+        if (rc == 0) {
+            registry.bytes -= bytes;
+        } else {
+            // A failed deregistration still owns the pages. Keep accounting
+            // and allocation ownership until RetryDeregistrations succeeds.
+            LOG(ERROR) << "Cannot deregister SRQ receive region, mr=" << mr
+                       << ", bytes=" << bytes << ", rc=" << rc << ": " << 
berror(rc)
+                       << ", retaining its pages";
+            registry.retired.push_back({mr, memory, bytes});
+            memory = nullptr;
+        }
+    }
+    ::free(memory);
+}
+
+void* SrqReceivePool::operator new(size_t bytes) {
+    void* memory = nullptr;
+    if (posix_memalign(&memory, alignof(SrqReceivePool), bytes) != 0) {
+        throw std::bad_alloc();
+    }
+    return memory;
+}
+
+void SrqReceivePool::operator delete(void* memory) {
+    free(memory);
+}
+
+SrqReceivePool* SrqReceivePool::Create(ibv_pd* pd, size_t block_size, uint32_t 
block_count,
+                                       uint32_t grow_blocks, uint32_t 
initial_blocks) {
+    if (block_size <= sizeof(butil::IOBuf::Block) || block_size > UINT32_MAX ||
+        block_count == 0 || block_size % alignof(butil::IOBuf::Block) != 0 ||
+        block_size > SIZE_MAX / block_count || grow_blocks > block_count ||
+        initial_blocks > block_count || FLAGS_rdma_srq_pool_growth_retry_ms <= 
0 ||
+        FLAGS_rdma_srq_pool_growth_retry_ms > 60000 ||
+        IbvRegMr == nullptr || IbvDeregMr == nullptr) {
+        errno = EINVAL;
+        return nullptr;
+    }
+
+    // The constructor and aligned allocator are private to the factory, so
+    // std::make_unique cannot construct this reference-counted pool.
+    std::unique_ptr<SrqReceivePool> pool(new SrqReceivePool);
+    pool->_pd = pd;
+    pool->_block_size = block_size;
+    pool->_block_count = block_count;
+    pool->_grow_blocks = grow_blocks == 0 ? block_count : grow_blocks;
+    pool->_growth_retry_us = int64_t(FLAGS_rdma_srq_pool_growth_retry_ms) * 
1000;
+    pool->_hugepages = FLAGS_rdma_srq_pool_hugepages;
+    pool->_growth_fd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC);
+    if (pool->_growth_fd < 0) {
+        int error = errno;
+        PLOG(ERROR) << "Fail to create SRQ receive pool eventfd, pool=" << 
pool.get();
+        errno = error;
+        return nullptr;
+    }
+
+    auto region = pool->PrepareRegion(
+        initial_blocks == 0 ? pool->_grow_blocks : initial_blocks);
+    if (region == nullptr) {
+        return nullptr;
+    }
+
+    pool->AddRegion(std::move(region));
+    Registry().count.fetch_add(1, butil::memory_order_release);
+    pool->_registered = true;
+    return pool.release();
+}
+
+std::unique_ptr<SrqReceivePool::Region> SrqReceivePool::PrepareRegion(uint32_t 
count) {
+    auto region = std::make_unique<Region>();
+    region->count = count;
+    region->bytes = _block_size * count;
+    constexpr size_t hugepage_size = 2 * 1024 * 1024;
+    int err = ENOMEM;
+    if (_hugepages && region->bytes >= hugepage_size &&
+        region->bytes <= SIZE_MAX - (hugepage_size - 1)) {
+        size_t allocation = (region->bytes + hugepage_size - 1) & 
~(hugepage_size - 1);
+        err = posix_memalign(&region->memory, hugepage_size, allocation);
+        if (err == 0) {
+            // THP is a hint, not a requirement. Ordinary pages remain valid if
+            // madvise or huge-page allocation fails. Both paths use free(),
+            // including the delayed deregistration quarantine.
+            madvise(region->memory, allocation, MADV_HUGEPAGE);
+        }
+    }
+    if (err != 0) {
+        err = posix_memalign(&region->memory, 4096, region->bytes);
+    }
+    if (err != 0) {
+        LOG(ERROR) << "Fail to allocate SRQ receive region, pool=" << this
+                   << ", bytes=" << region->bytes << ", rc=" << err << ": " << 
berror(err);
+        errno = err;
+        return nullptr;
+    }
+
+    region->buffers = std::make_unique<Buffer[]>(region->count);
+    // Register receive memory independently of the ordinary IOBuf pool growth
+    // policy and its pthread TLS caches.
+    int access = IBV_ACCESS_LOCAL_WRITE;
+    if (_try_relaxed_order) {
+        access |= IBV_ACCESS_RELAXED_ORDERING;
+    }
+    region->mr = IbvRegMr(_pd, region->memory, region->bytes, access);
+    if (region->mr == nullptr && _try_relaxed_order) {
+        int error = errno;
+        region->mr = IbvRegMr(_pd, region->memory, region->bytes, 
IBV_ACCESS_LOCAL_WRITE);
+        // Only cache a capability-like rejection followed by a successful
+        // fallback. Resource exhaustion must not permanently disable RO.
+        if (region->mr != nullptr && (error == EINVAL || error == EOPNOTSUPP)) 
{
+            _try_relaxed_order = false;
+        }
+    }
+    if (region->mr == nullptr) {
+        int error = errno;
+        PLOG(ERROR) << "Fail to register SRQ receive region, pool=" << this
+                    << ", bytes=" << region->bytes << ", blocks=" << count;
+        errno = error;
+        return nullptr;
+    }
+
+    auto& registry = Registry();
+    {
+        BAIDU_SCOPED_LOCK(registry.mutex);
+        registry.bytes += region->bytes;
+    }
+    for (uint32_t i = 0; i < region->count; ++i) {
+        auto* buffer = &region->buffers[i];
+        buffer->_pool = this;
+        buffer->_region = region.get();
+        buffer->_memory = static_cast<char*>(region->memory) + i * _block_size;
+        buffer->_lkey = region->mr->lkey;
+        buffer->_next = region->free;
+        region->free = buffer;
+    }
+    // Modify invokes this callback for both index copies. Keep accounting and
+    // other one-time side effects outside the callback.
+    registry.regions.Modify([&](PoolRegistry::RegionMap& regions) {
+        regions.emplace(reinterpret_cast<uintptr_t>(region->memory),
+            PoolRegistry::LiveRegion{this, region->buffers.get(), 
region->bytes,
+                                     region->mr->lkey});
+        return 1;
+    });
+    region->published = true;
+    return region;
+}
+
+void SrqReceivePool::AddRegion(std::unique_ptr<Region> region) {
+    Region* added = region.get();
+    added->index = _regions.size();
+    _regions.push_back(std::move(region));
+    // Reserve during adoption so returning a formerly full region never needs
+    // a heap allocation. _regions grows geometrically, not once per block.
+    _free_regions.reserve(_regions.capacity());
+    _free_regions.push_back(added);
+    std::push_heap(_free_regions.begin(), _free_regions.end(), Region::Later);
+    _allocated_blocks += added->count;
+    _available += added->count;
+}
+
+void SrqReceivePool::AdoptGrowth() {
+    if (_completed_growth.load(butil::memory_order_acquire) == nullptr) {
+        return;
+    }
+
+    std::unique_ptr<Growth> growth(
+        _completed_growth.exchange(nullptr, butil::memory_order_acquire));
+    _growing = false;
+    if (growth->region != nullptr) {
+        AddRegion(std::move(growth->region));
+    }
+}
+
+void SrqReceivePool::RequestGrowth(uint32_t reusable) {
+    if (_growing || _allocated_blocks == _block_count ||
+        uint64_t(_available) + reusable > std::max(1u, _grow_blocks / 4)) {
+        return;
+    }
+    auto growth = std::make_unique<Growth>();
+    growth->pool = this;
+    growth->count = std::min(_grow_blocks, _block_count - _allocated_blocks);
+    _growing = true;
+    _references.fetch_add(1, butil::memory_order_relaxed);
+    bthread_t tid;
+    CHECK_EQ(bthread_start_background(&tid, nullptr, RunGrowth, growth.get()), 
0)
+        << "Cannot start SRQ receive pool growth";
+    growth.release();

Review Comment:
   A failed `bthread_start_background()` is treated as a fatal invariant here, 
even though this is a runtime growth/retry path. The code has already set 
`_growing` and incremented `_references`, so a scheduler/resource failure 
aborts the process instead of allowing the owner to keep using existing blocks 
and retrying growth; if the check is changed to return, those state changes 
also need to be rolled back and `growth` must be destroyed. Handle the return 
code explicitly, clear `_growing`, release the growth-task reference, and leave 
the pool usable for a later retry.



##########
src/brpc/rdma/srq_worker.cpp:
##########
@@ -0,0 +1,2717 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+#if BRPC_WITH_RDMA
+#include "brpc/rdma/srq_worker.h"
+#include <algorithm>
+#include <arpa/inet.h>
+#include <iterator>
+#include <map>
+#include <mutex>
+#include <sys/eventfd.h>
+#include <unistd.h>
+#include <utility>
+#include <vector>
+#include <gflags/gflags.h>
+#include "bthread/bthread.h"
+#include "butil/containers/flat_map.h"
+#include "butil/containers/mpsc_queue.h"
+#include "butil/fd_utility.h"
+#include "butil/macros.h"
+#include "butil/memory/scope_guard.h"
+#include "butil/object_pool.h"
+#include "brpc/errno.pb.h"
+#include "brpc/event_dispatcher.h"
+#include "brpc/input_messenger.h"
+#include "brpc/rdma_transport.h"
+#include "brpc/rdma/block_pool.h"
+#include "brpc/rdma/rdma_endpoint.h"
+#include "brpc/rdma/rdma_handshake.h"
+#include "brpc/rdma/rdma_helper.h"
+#include "brpc/rdma/srq_flow_control.h"
+#include "brpc/rdma/srq_receive_pool.h"
+
+namespace brpc {
+namespace rdma {
+
+extern int (*IbvQueryDevice)(ibv_context*, ibv_device_attr*);
+extern ibv_cq* (*IbvCreateCq)(ibv_context*, int, void*, ibv_comp_channel*, 
int);
+extern int (*IbvDestroyCq)(ibv_cq*);
+extern ibv_srq* (*IbvCreateSrq)(ibv_pd*, ibv_srq_init_attr*);
+extern int (*IbvDestroySrq)(ibv_srq*);
+extern ibv_qp* (*IbvCreateQp)(ibv_pd*, ibv_qp_init_attr*);
+extern int (*IbvModifyQp)(ibv_qp*, ibv_qp_attr*, ibv_qp_attr_mask);
+extern int (*IbvDestroyQp)(ibv_qp*);
+extern int (*IbvQueryEce)(ibv_qp*, ibv_ece*);
+extern ibv_comp_channel* (*IbvCreateCompChannel)(ibv_context*);
+extern int (*IbvDestroyCompChannel)(ibv_comp_channel*);
+extern int (*IbvGetCqEvent)(ibv_comp_channel*, ibv_cq**, void**);
+extern void (*IbvAckCqEvents)(ibv_cq*, unsigned int);
+// Keep verbs entry points replaceable, like resource-creation calls.
+// Fault-injection tests may replace them only after all SRQ workers have
+// stopped.
+int (*IbvPostSrqSend)(ibv_qp*, ibv_send_wr*, ibv_send_wr**) = ibv_post_send;
+int (*IbvPostSrqRecv)(ibv_srq*, ibv_recv_wr*, ibv_recv_wr**) = 
ibv_post_srq_recv;
+int (*IbvPollSrqCq)(ibv_cq*, int, ibv_wc*) = ibv_poll_cq;
+int (*IbvReqNotifySrqCq)(ibv_cq*, int) = ibv_req_notify_cq;
+
+#if defined(UNIT_TEST)
+// Test-only pause around the exit decision to exercise external Fail and
+// event-ownership handoff races deterministically.
+butil::atomic<void (*)(SrqWorker*, bool, butil::atomic<uint64_t>*, 
SrqReceivePool*)>
+    SrqWorkerStopHookForTest{nullptr};
+// Pause between the first failed try_lock and notification registration to
+// exercise an unlock that precedes registration.
+butil::atomic<void (*)()> SrqPostLockRetryHookForTest{nullptr};
+// Pause deferred notifications independently of CQ progress and endpoint 
reset.
+butil::atomic<void (*)(bool)> SrqFailureHookForTest{nullptr};
+// Observe software wakeups independently of CQ and async event timing.
+butil::atomic<void (*)(SrqWorker*)> SrqWorkerKickHookForTest{nullptr};
+// Inject task-start failures without exhausting the process scheduler.
+butil::atomic<int (*)(bool, bthread_t*, const bthread_attr_t*,
+                     void* (*)(void*), void*)> 
SrqStartTaskHookForTest{nullptr};
+#endif
+
+DECLARE_bool(rdma_recv_zerocopy);
+DECLARE_int32(rdma_zerocopy_min_size);
+DECLARE_int32(rdma_srq_pool_grow_blocks);
+DECLARE_int32(rdma_srq_pool_growth_retry_ms);
+
+DEFINE_bool(rdma_use_srq, false,
+            "Enable event-driven shared receive queues with v3 handshake");
+DEFINE_int32(rdma_srq_workers, 4, "SRQ workers per bthread tag in event mode");
+DEFINE_int32(rdma_srq_window, 512, "Initial and restored virtual SRQ credit 
window");
+DEFINE_int32(rdma_srq_sq_size, 128, "Per-connection SQ depth in SRQ mode");
+DEFINE_int32(rdma_srq_max_sge, 4, "Maximum SGEs per SRQ-mode send");
+DEFINE_int32(rdma_srq_inline_size, 128, "Requested inline bytes in SRQ mode");
+DEFINE_int32(rdma_srq_depth, 2048, "Posted receives per SRQ worker");
+DEFINE_int32(rdma_srq_pool_blocks, 16384,
+             "Maximum receive blocks per SRQ worker; registered on demand");
+DEFINE_int32(rdma_srq_tx_cq_size, 8192, "Shared TX CQ capacity per SRQ 
worker");
+DEFINE_int32(rdma_srq_control_reserve, 64, "TX CQ slots reserved for pure 
grants");
+
+namespace {
+using ConnectionPtr = std::shared_ptr<SrqConnection>;
+using ProgressCallback = std::function<ProgressRound()>;
+using ProgressQueue = butil::MPSCQueue<
+    ProgressCallback, butil::ObjectPoolAllocator<ProgressCallback>>;
+
+using SocketFailure = std::function<void()>;
+
+constexpr uint32_t kControlSlots = 2;
+constexpr int kBatch = 32;
+butil::atomic<size_t> g_live_connections{0};
+butil::atomic<size_t> g_live_workers{0};
+}  // namespace
+
+// Results local to one progress or Stop round. Notifications and delivery run
+// after releasing `_progress_mutex`. Lists retain references through every 
merge
+// and are deduplicated once outside the lock, without connection membership 
flags.
+struct ProgressRound {
+    // Writers to recheck after TX completion, grants or shared-CQ capacity
+    // becomes available.
+    std::vector<ConnectionPtr> writers;
+    // Connections with new input. Each `_incoming` queue preserves byte order.
+    std::vector<ConnectionPtr> deliveries;
+    // Connections ready for normal retirement after a fresh RX drain. Retained
+    // until outside the lock.
+    std::vector<ConnectionPtr> retiring;
+    // Callbacks dequeued this round, including those drained during shutdown.
+    // Retain their captured contexts so final references are released outside
+    // `_progress_mutex`.
+    std::vector<ProgressCallback> callbacks;
+    // Failure notifications collected under `_progress_mutex` by callbacks or
+    // worker shutdown.
+    // `RunEvent` and `Stop` dispatch their batches after unlocking.
+    std::vector<SocketFailure> failures;
+    // User payloads transferred by the owner for deferred release after
+    // unlocking.
+    butil::IOBuf released;
+    // A CQE was processed or a callback published new close work. `RunEvent`
+    // uses only the post-arm result to decide whether another pass is needed.
+    // Requeueing a resource waiter alone must not set this flag.
+    bool progressed{false};
+    // This poll filled its batch, or normal progress filled either CQ batch.
+    // `Append` does not merge this flag. Extra retirement polls must not
+    // influence the normal event loop's decision to skip arming.
+    bool full_batch{false};
+
+    ProgressRound() = default;
+
+    // IOBuf has no standard move constructor. Transfer its references
+    // explicitly so moving a result cannot leave a hidden payload owner that
+    // outlives deferred release and runs a user deleter on the progress task.
+    ProgressRound(ProgressRound&& other) noexcept
+        : writers(std::move(other.writers))
+        , deliveries(std::move(other.deliveries))
+        , retiring(std::move(other.retiring))
+        , callbacks(std::move(other.callbacks))
+        , failures(std::move(other.failures))
+        , released(other.released.movable())
+        , progressed(other.progressed)
+        , full_batch(other.full_batch) {}
+
+    // Move retained contexts between rounds without copying references or
+    // hashing connections. Deduplication and final releases belong to
+    // `FinishRound()`, after the owner lock has been released.
+    template <typename T>
+    static void AppendValues(std::vector<T>& target, std::vector<T>& source) {
+        if (target.empty()) {
+            target.swap(source);
+        } else {
+            target.insert(target.end(),
+                          std::make_move_iterator(source.begin()),
+                          std::make_move_iterator(source.end()));
+        }
+    }
+
+    void Append(ProgressRound other) {
+        AppendValues(writers, other.writers);
+        AppendValues(deliveries, other.deliveries);
+        AppendValues(retiring, other.retiring);
+        AppendValues(callbacks, other.callbacks);
+        AppendValues(failures, other.failures);
+        released.append(other.released.movable());
+        progressed = progressed || other.progressed;
+    }
+
+    static void Deduplicate(std::vector<ConnectionPtr>& connections) {
+        std::sort(connections.begin(), connections.end());
+        connections.erase(std::unique(connections.begin(), connections.end()),
+                          connections.end());
+    }
+};
+
+namespace {
+// Actions from one receive CQE: return a connection and flags without
+// allocating per-packet sets.
+struct ReceiveResult {
+    // Connection retained by the caller when input or a grant requires
+    // notification.
+    ConnectionPtr connection;
+    // A grant arrived. Recheck writability after releasing the owner lock.
+    bool wake_writer{false};
+    // Payload was queued. Attempt to start a delivery task.
+    bool deliver{false};
+};
+
+void DrainEventFd(int fd) {
+    uint64_t value;
+    while (read(fd, &value, sizeof(value)) == sizeof(value)) {}
+}
+
+struct DeliveryTask {
+    // Keeps connection state alive until this consumer task finishes.
+    ConnectionPtr connection;
+    // Keeps the Socket and its parser alive during delivery.
+    SocketUniquePtr socket;
+    // First block dequeued by the starter, consumed before later queue 
entries.
+    butil::IOBuf data;
+};
+
+int StartTask(bool urgent, bthread_t* tid,
+              const bthread_attr_t* attr,
+              void* (*fn)(void*), void* arg) {
+#if defined(UNIT_TEST)
+    auto hook = SrqStartTaskHookForTest.load(butil::memory_order_acquire);
+    if (hook != nullptr) {
+        return hook(urgent, tid, attr, fn, arg);
+    }
+#endif
+    return urgent ? bthread_start_urgent(tid, attr, fn, arg) :
+                    bthread_start_background(tid, attr, fn, arg);
+}
+
+void* RunPayloadRelease(void* arg) {
+    // DMA has stopped using these references. Their final deleters may wait
+    // for RPC failures, so release them independently of failure callbacks.
+    std::unique_ptr<butil::IOBuf> released(static_cast<butil::IOBuf*>(arg));
+    return nullptr;
+}
+
+void* RunSocketFailure(void* arg) {
+    std::unique_ptr<SocketFailure> failure(static_cast<SocketFailure*>(arg));
+#if defined(UNIT_TEST)
+    auto hook = SrqFailureHookForTest.load(butil::memory_order_acquire);
+    if (hook != nullptr) {
+        hook(true);
+    }
+#endif
+    (*failure)();
+#if defined(UNIT_TEST)
+    if (hook != nullptr) {
+        hook(false);
+    }
+#endif
+    return nullptr;
+}
+}  // namespace
+
+// Logical owner of shared transport resources, progressed by event-driven
+// bthreads without a dedicated pthread. Handles QPN/CQ dispatch, receive 
refill
+// and pending callbacks. Callbacks advance connection state. Protocol parsing,
+// Socket callbacks and user deleters run after the relevant transport locks 
are
+// released.
+// _events elects one event-loop owner. _progress_mutex also serializes that
+// owner with Add and Stop. It remains necessary even with one RunEvent task.
+class SrqWorker : public RdmaAsyncEventContext,
+                  public std::enable_shared_from_this<SrqWorker> {
+    // Lifecycle only advances. BROKEN covers both pending destruction and
+    // failed destruction awaiting retry.
+    enum class State : uint8_t {
+        RUNNING,
+        BROKEN,
+        DESTROYED,
+    };
+
+public:
+    explicit SrqWorker(bthread_tag_t tag)
+        : _tag(tag) { ++g_live_workers; }
+
+    ~SrqWorker() override {
+        CHECK(_connections.empty());
+        CHECK(_srq == nullptr && _tx_cq == nullptr && _rx_cq == nullptr);
+        if (_kick_fd >= 0) {
+            close(_kick_fd);
+        }
+        --g_live_workers;
+    }
+
+    bool Initialize();
+    bool Add(const ConnectionPtr& connection, uint32_t sq_size);
+    void Stop();
+    void OnEvent(const bthread_attr_t& attr);
+    // Signal eventfd only. Do not acquire progress ownership or run callbacks
+    // inline. OnEvent coalesces the notification into an existing event task 
or
+    // starts one after the previous task has relinquished ownership.
+    void Kick() {
+#if defined(UNIT_TEST)
+        auto hook = SrqWorkerKickHookForTest.load(butil::memory_order_acquire);
+        if (hook != nullptr) {
+            hook(this);
+        }
+#endif
+        const uint64_t one = 1;
+        if (_kick_fd >= 0) {
+            butil::ignore_result(write(_kick_fd, &one, sizeof(one)));
+        }
+    }
+
+    // Callbacks capture and retain their own context and are responsible for
+    // requeueing or handing off progress. Each enqueue executes once. Callers
+    // coalesce requests and synchronize publication with close. Late async
+    // events capture only weak references because teardown may have finished
+    // draining the queue. Callbacks run under `_progress_mutex` and return
+    // Socket notifications and user payloads through `ProgressRound` for
+    // execution or release outside the lock.
+    // External producers must Kick after enqueue has fully published, either
+    // here with `wake` or after a batch of enqueues. `TryDequeue` may return
+    // while a producer is still connecting a node. Worker-side requeues can
+    // omit the wake because their resource event supplies the next attempt.
+    void Schedule(ProgressCallback callback, bool wake) {
+        _commands.Enqueue(std::move(callback));
+        if (wake) {
+            Kick();
+        }
+    }
+
+    // Reserve for a batch before posting any WR. Each accepted WR retains one
+    // reservation until completion/shutdown. The submitter returns only unused
+    // or definitely unposted reservations. Control can use the full capacity,
+    // while data must leave room for grants that unblock exhausted FC windows.
+    uint32_t ReserveTx(uint32_t requested, bool control) {
+        uint32_t limit = control ? _tx_capacity : DataTxLimit();
+        uint32_t used = _tx_used.load(butil::memory_order_relaxed);
+        while (used < limit && !broken()) {
+            uint32_t reserved = std::min(requested, limit - used);
+            if (_tx_used.compare_exchange_weak(used, used + reserved,
+                                               butil::memory_order_acq_rel)) {
+                return reserved;
+            }
+        }
+        return 0;
+    }
+
+    // Availability snapshot only. Actual submission still requires an atomic
+    // ReserveTx reservation.
+    bool DataTxAvailable() const {
+        return !broken() && _tx_used.load(butil::memory_order_relaxed) < 
DataTxLimit();
+    }
+
+    void ReleaseTx(uint32_t count) {
+        if (count != 0) {
+            CHECK_GE(_tx_used.fetch_sub(count, butil::memory_order_release), 
count);
+        }
+    }
+
+    void Fail() {
+        MarkBroken();
+        // External and mixed-context callers must always notify progress.
+        // An active owner may be exiting, and repeated failures must not lose
+        // their wakeup just because another caller already marked BROKEN.
+        Kick();
+    }
+
+    std::function<void()> PrepareAsyncEvent(ibv_event_type type) override {
+        auto self = shared_from_this();
+        return [self, type] {
+            LOG(ERROR) << "Fatal RDMA async event for SRQ worker=" << 
self.get()
+                       << ", event_type=" << type;
+            self->Fail();
+        };
+    }
+
+    bool broken() const {
+        return _state.load(butil::memory_order_acquire) != State::RUNNING;
+    }
+
+    bool retired() const {
+        return _state.load(butil::memory_order_acquire) == State::DESTROYED;
+    }
+
+    void Debug(std::ostream& out, butil::StringPiece connector) {
+        State state = _state.load(butil::memory_order_acquire);
+        out << connector << "rdma_srq_worker_broken=" << (state != 
State::RUNNING)
+            << connector << "rdma_srq_worker_retired=" << (state == 
State::DESTROYED)
+            << connector << "rdma_srq_tx_outstanding=" << _tx_used.load()
+            << connector << "rdma_srq_tx_capacity=" << _tx_capacity;
+        // Read owner state only for diagnostics. If the worker is busy, omit
+        // receive counts rather than block the query. Pool availability is an
+        // O(1) owner count and excludes returns not yet absorbed by refill.
+        std::unique_lock<bthread::Mutex> lock(_progress_mutex, 
std::try_to_lock);
+        if (!lock.owns_lock()) {
+            return;
+        }
+        const size_t posted = RxOutstanding();
+        const size_t available = _pool ? _pool->available() + 
_rx_reusable.size() : 0;
+        const size_t refill = _rx_capacity - posted;
+        out << connector << "rdma_srq_pool_free="
+            << (available > refill ? available - refill : 0)
+            << connector << "rdma_srq_posted=" << posted;
+    }
+
+private:
+friend class SrqConnection;
+#if defined(UNIT_TEST)
+friend bool SrqWorkerIdleForTest(const SrqWorker* worker);
+friend bool InitializeSrqWorkerForTest(const std::shared_ptr<SrqWorker>& 
worker, ibv_cq* tx,
+                                       ibv_cq* rx, ibv_srq* srq, 
SrqReceivePool* pool, uint32_t depth);
+#endif
+
+    // Mark failure without scheduling another event. Use only when the caller
+    // guarantees cleanup through current progress, failed initialization or
+    // Stop. Holding a transport lock alone does not provide that guarantee.
+    void MarkBroken() {
+        State expected = State::RUNNING;
+        // Async callbacks may arrive after destruction. Never move DESTROYED
+        // back to BROKEN, even when a late Fail still requests a wakeup.
+        _state.compare_exchange_strong(expected, State::BROKEN,
+                                       butil::memory_order_release,
+                                       butil::memory_order_relaxed);
+    }
+
+    // Maximum total shared TX CQ occupancy allowed for data WRs, excluding the
+    // pure-grant reserve.
+    uint32_t DataTxLimit() const {
+        return _tx_capacity - FLAGS_rdma_srq_control_reserve;
+    }
+
+    static void* RunEvent(void* arg);
+    ProgressRound ProgressLocked();
+    void DetachEvents();
+    // Normal progress, failure exits and Stop transfer their completed result
+    // here after unlocking, preserving all wakeups and deferred releases.
+    void FinishRound(ProgressRound round);
+    void DispatchNotifications(butil::IOBuf released,
+                               std::vector<SocketFailure> failures);
+    size_t RxOutstanding() const { return _rx_buffers.size() - 
_rx_free.size(); }
+
+    // Both progress and Initialize check broken() after refill. Initialization
+    // failure is cleaned up synchronously by Stop before returning a worker.
+    void Refill();
+    // Poll one batch from either CQ so normal progress reaches RX, refill and
+    // callbacks even under continuous TX load. Report failures via `broken()`.
+    ProgressRound Poll(bool receive);
+    // Dequeue a batch, then invoke each callback once. Callbacks manage their
+    // own requeueing.
+    ProgressRound RunPending();
+    // The caller has observed LAST_WQE and TX reclamation for each candidate.
+    // Drain RX again before destruction. Candidates belong only to this round.
+    // Return the candidate references with any RX results for release outside
+    // the lock. No close phase or refill-pause state persists between rounds.
+    ProgressRound RetireConnections(std::vector<ConnectionPtr> connections);
+    ReceiveResult HandleReceive(const ibv_wc& wc);
+    ConnectionPtr HandleSend(const ibv_wc& wc, butil::IOBuf& released);
+    void ConsumeEvents();
+    // Stop DMA before releasing references even if a broken CQ cannot be
+    // polled. Retain the worker and memory after failed verbs destruction for
+    // an explicit retry. The caller drains pending notification commands even
+    // on failure. `retired()` reports whether cleanup finished.
+    ProgressRound DestroyAll();
+
+    // Scheduling tag shared by event and deferred tasks.
+    bthread_tag_t _tag;
+    // Serializes CQ handling, refill, connection lookup and teardown. Data
+    // submission uses only each connection _post_mutex, allowing parallel 
sends
+    // across QPs. Lock order is _progress_mutex then connection _post_mutex,
+    // never the reverse. Normal maintenance uses try_lock. Failure teardown 
may
+    // wait, so both are bthread mutexes.
+    bthread::Mutex _progress_mutex;
+    // Completion-notification channel shared by both CQs.
+    ibv_comp_channel* _channel{nullptr};
+    // Send CQ shared by all attached QPs.
+    ibv_cq* _tx_cq{nullptr};
+    // Shared receive CQ. CQEs identify the QP and receive slot.
+    ibv_cq* _rx_cq{nullptr};
+    // Hardware receive queue shared by all attached RC QPs.
+    ibv_srq* _srq{nullptr};
+    // Worker reference to the receive pool. Exported zero-copy blocks retain 
it
+    // independently.
+    SrqReceivePool* _pool{nullptr};
+    // Nonblocking eventfd for software work and failures.
+    int _kick_fd{-1};
+    // Target posted-receive depth, bounded by configured and actual SRQ/RX CQ
+    // capacities.
+    uint32_t _rx_capacity{0};
+    // Actual TX CQ capacity returned by verbs, fixed after initialization.
+    uint32_t _tx_capacity{0};
+    // Shared CQ reservations for prepared or unreclaimed data/control WRs.
+    // Concurrent submitters reserve atomically. Completion or rollback returns
+    // each reservation exactly once.
+    butil::atomic<uint32_t> _tx_used{0};
+    // Any thread may publish RUNNING -> BROKEN, causing submitters to stop
+    // reserving capacity once observed. The owner publishes DESTROYED after
+    // reclaiming QPs, SRQ, CQs and receive accounting, allowing registry
+    // replacement.
+    butil::atomic<State> _state{State::RUNNING};
+    // Versioned event-source ID borrowing completion, software-wakeup and
+    // growth fds.
+    VRefId _event_source{INVALID_VREF_ID};
+    // Event notification generation. Zero means no task owns the event loop.
+    // Coalesces notifications while preserving the handoff to the next owner.
+    butil::atomic<uint64_t> _events{0};
+    // Multi-producer queue for progress, async errors and Socket 
notifications.
+    // Each callback captures its own context and returns a `ProgressRound`.
+    ProgressQueue _commands;
+    // QPN-to-connection ownership table. Remove entries only after safe QP
+    // teardown. Pointers returned by `seek()` stay inside `_progress_mutex`
+    // and must not survive a table mutation. Deferred work copies the value.
+    butil::FlatMap<uint32_t, ConnectionPtr> _connections;
+    // Owner-only receive slots. Nonzero wr_id encodes the index plus one.
+    // Completion requires neither a hash-node allocation nor dereferencing a
+    // NIC-returned pointer.
+    std::vector<SrqReceivePool::Buffer*> _rx_buffers;
+    // Small-message copies export no BlockRef, so retain the block lease and
+    // initialized descriptor for refill. The worker owns these blocks between
+    // CQ progress and refill, including failure cleanup.
+    std::vector<SrqReceivePool::Buffer*> _rx_reusable;
+    // Free indices in _rx_buffers available for the next receive refill.
+    std::vector<uint32_t> _rx_free;
+};
+
+#if defined(UNIT_TEST)
+// Exercise connection synchronization without creating verbs resources or
+// attaching an event source. Tests drive delivery and observe software kicks.
+std::shared_ptr<SrqWorker> CreateSrqWorkerForTest() {
+    return std::make_shared<SrqWorker>(BTHREAD_TAG_DEFAULT);
+}
+
+// `FinishRound` publishes writer wakeups before releasing event ownership.
+// Tests use this barrier to exclude notifications left over from warmup RPCs.
+bool SrqWorkerIdleForTest(const SrqWorker* worker) {
+    return worker->_events.load(butil::memory_order_acquire) == 0;
+}
+#endif
+
+namespace {
+// Borrows three worker fds without transferring ownership to Socket. 
Dispatcher
+// IDs include a generation. In-flight callbacks retain the event source and
+// worker instead of touching a recycled replacement.
+class SrqEventSource : public VersionedRefWithId<SrqEventSource> {
+public:
+    explicit SrqEventSource(Forbidden f) : VersionedRefWithId(f) {}
+
+    int OnCreated(const std::shared_ptr<SrqWorker>& worker, bthread_tag_t tag) 
{
+        _worker = worker;
+        _io.set_bthread_tag(tag);
+        return _io.Init(reinterpret_cast<void*>(id()));
+    }
+
+    void BeforeRecycled() {
+        _io.Reset();
+        _worker.reset();
+    }
+
+    int Add(int fd) { return _io.AddConsumer(fd); }
+
+    void Remove(int fd) { _io.RemoveConsumer(fd); }
+
+    static int OnInputEvent(void* arg, uint32_t, const bthread_attr_t& attr) {
+        VersionedRefWithIdUniquePtr<SrqEventSource> source;
+        if (Address(reinterpret_cast<uintptr_t>(arg), &source) != 0) {
+            return -1;
+        }
+        source->_worker->OnEvent(attr);
+        return 0;
+    }
+
+    static int OnOutputEvent(void*, uint32_t, const bthread_attr_t&) { return 
0; }
+
+private:
+    // Retains the worker while fd callbacks may still access this event 
source.
+    std::shared_ptr<SrqWorker> _worker;
+    // EventDispatcher registration context shared by the three borrowed fds.
+    IOEvent<SrqEventSource> _io;
+};
+
+struct WorkerRegistry {
+    // Protects worker lookup, generation replacement and failed-destruction
+    // quarantine.
+    bthread::Mutex mutex;
+    // Current worker for each (bthread tag, configured shard).
+    std::map<std::pair<bthread_tag_t, int>, std::shared_ptr<SrqWorker>> 
workers;
+    // Workers retaining DMA resources after failed destruction, retried by
+    // explicit Stop.
+    std::vector<std::shared_ptr<SrqWorker>> quarantine;
+};
+
+// The registry outlives static destruction and late application IOBuf 
releases.
+WorkerRegistry& Registry() {
+    static auto* registry = new WorkerRegistry;
+    return *registry;
+}
+
+std::shared_ptr<SrqWorker> GetWorker(bthread_tag_t tag, int index) {
+    auto& registry = Registry();
+    std::unique_lock<bthread::Mutex> lock(registry.mutex);
+    auto key = std::make_pair(tag, index);
+    auto it = registry.workers.find(key);
+    if (it != registry.workers.end()) {
+        if (!it->second->retired()) {
+            return it->second;
+        }
+        // Replace only after all old-generation DMA resources have been
+        // reclaimed. Late endpoint/event references may retain the old context
+        // without affecting lookup of the replacement.
+        registry.workers.erase(it);
+    }
+    auto worker = std::make_shared<SrqWorker>(tag);
+    if (!worker->Initialize()) {
+        const int saved_errno = errno;
+        lock.unlock();
+        worker->Stop();
+        worker.reset();
+        errno = saved_errno;
+        return {};
+    }
+    registry.workers.emplace(key, worker);
+    return worker;
+}
+}  // namespace
+
+bool ValidateSrqOptions() {
+    if (!FLAGS_rdma_use_srq) {
+        return true;
+    }
+    if (FLAGS_rdma_use_polling) {
+        LOG(ERROR) << "rdma_use_srq requires event mode 
(rdma_use_polling=false)";
+        return false;
+    }
+    auto check = [](bool valid, const char* name, int value,
+                    const std::string& range) {
+        if (!valid) {
+            LOG(ERROR) << name << '=' << value << " is invalid; expected " << 
range;
+        }
+        return valid;
+    };
+    return check(SrqFlowControl::ValidWindow(FLAGS_rdma_srq_window),
+                 "rdma_srq_window", FLAGS_rdma_srq_window, "[4, 65535]") &&
+           check(FLAGS_rdma_srq_workers > 0,
+                 "rdma_srq_workers", FLAGS_rdma_srq_workers, "> 0") &&
+           check(FLAGS_rdma_srq_sq_size >= 16 && FLAGS_rdma_srq_sq_size <= 
4096,
+                 "rdma_srq_sq_size", FLAGS_rdma_srq_sq_size, "[16, 4096]") &&
+           check(FLAGS_rdma_srq_max_sge > 0 && FLAGS_rdma_srq_max_sge <= 32,
+                 "rdma_srq_max_sge", FLAGS_rdma_srq_max_sge, "[1, 32]") &&
+           check(FLAGS_rdma_srq_inline_size >= 0 && FLAGS_rdma_srq_inline_size 
<= 236,
+                 "rdma_srq_inline_size", FLAGS_rdma_srq_inline_size, "[0, 
236]") &&
+           check(FLAGS_rdma_srq_depth >= 32,
+                 "rdma_srq_depth", FLAGS_rdma_srq_depth, ">= 32") &&
+           check(FLAGS_rdma_srq_pool_blocks > FLAGS_rdma_srq_depth,
+                 "rdma_srq_pool_blocks", FLAGS_rdma_srq_pool_blocks,
+                 "> rdma_srq_depth=" + std::to_string(FLAGS_rdma_srq_depth)) &&
+           check(FLAGS_rdma_srq_pool_grow_blocks > 0,
+                 "rdma_srq_pool_grow_blocks", FLAGS_rdma_srq_pool_grow_blocks, 
"> 0") &&
+           check(FLAGS_rdma_srq_pool_growth_retry_ms > 0 &&
+                 FLAGS_rdma_srq_pool_growth_retry_ms <= 60000,
+                 "rdma_srq_pool_growth_retry_ms", 
FLAGS_rdma_srq_pool_growth_retry_ms,
+                 "[1, 60000]") &&
+           check(FLAGS_rdma_srq_control_reserve > 0,
+                 "rdma_srq_control_reserve", FLAGS_rdma_srq_control_reserve, 
"> 0") &&
+           check(FLAGS_rdma_srq_tx_cq_size > FLAGS_rdma_srq_control_reserve,
+                 "rdma_srq_tx_cq_size", FLAGS_rdma_srq_tx_cq_size,
+                 "> rdma_srq_control_reserve=" + 
std::to_string(FLAGS_rdma_srq_control_reserve));
+}
+
+// Check reported capabilities independently of vendor and device name.
+// Actual CQ/SRQ/RC QP creation remains the final
+// resource-availability check.
+bool ValidateSrqDevice(ibv_context* context) {
+    ibv_device_attr attr = {};
+    const int rc = IbvQueryDevice(context, &attr);
+    if (rc != 0) {
+        errno = rc;
+        PLOG(ERROR) << "Fail to query RDMA device for SRQ";
+        return false;
+    }
+
+    if (attr.max_srq <= 0 || attr.max_srq_sge < 1) {
+        LOG(ERROR) << "RDMA device does not support SRQ receives: max_srq="
+                   << attr.max_srq << ", max_srq_sge=" << attr.max_srq_sge;
+        errno = ENOTSUP;
+        return false;
+    }
+
+    if (FLAGS_rdma_srq_depth > attr.max_srq_wr ||
+        FLAGS_rdma_srq_depth > attr.max_cqe ||
+        FLAGS_rdma_srq_tx_cq_size > attr.max_cqe) {
+        LOG(ERROR) << "SRQ configuration exceeds RDMA device capacity: depth="
+                   << FLAGS_rdma_srq_depth << ", tx_cq_size="
+                   << FLAGS_rdma_srq_tx_cq_size << ", max_srq_wr="
+                   << attr.max_srq_wr << ", max_cqe=" << attr.max_cqe;
+        errno = EINVAL;
+        return false;
+    }
+    return true;
+}
+
+bool SrqWorker::Initialize() {
+    std::lock_guard<bthread::Mutex> owner(_progress_mutex);
+    if (!ValidateSrqDevice(GetRdmaContext())) {
+        return false;
+    }
+
+    _kick_fd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC);
+    if (_kick_fd < 0) {
+        int error = errno;
+        PLOG(ERROR) << "Fail to create kick eventfd for SRQ worker=" << this;
+        errno = error;
+        return false;
+    }
+
+    _channel = IbvCreateCompChannel(GetRdmaContext());
+    if (_channel == nullptr || butil::make_non_blocking(_channel->fd) < 0 ||
+        butil::make_close_on_exec(_channel->fd) < 0) {
+        int error = errno;
+        PLOG(ERROR) << "Fail to create or configure completion channel for SRQ 
worker";
+        errno = error;
+        return false;
+    }
+
+    auto* context = static_cast<RdmaAsyncEventContext*>(this);
+    _tx_cq = IbvCreateCq(GetRdmaContext(), FLAGS_rdma_srq_tx_cq_size,
+                         context, _channel, GetRdmaCompVector());
+    if (_tx_cq == nullptr) {
+        int error = errno;
+        PLOG(ERROR) << "Fail to create TX CQ for SRQ worker=" << this
+                    << ", capacity=" << FLAGS_rdma_srq_tx_cq_size;
+        errno = error;
+        return false;
+    }
+
+    RegisterRdmaAsyncEventContext(_tx_cq, context);
+    _rx_cq = IbvCreateCq(GetRdmaContext(), FLAGS_rdma_srq_depth,
+                         context, _channel, GetRdmaCompVector());
+    if (_rx_cq == nullptr) {
+        int error = errno;
+        PLOG(ERROR) << "Fail to create RX CQ for SRQ worker=" << this
+                    << ", capacity=" << FLAGS_rdma_srq_depth;
+        errno = error;
+        return false;
+    }
+
+    RegisterRdmaAsyncEventContext(_rx_cq, context);
+    _tx_capacity = _tx_cq->cqe;
+    ibv_srq_init_attr attr = {};
+    attr.srq_context = context;
+    attr.attr.max_wr = FLAGS_rdma_srq_depth;
+    attr.attr.max_sge = 1;
+    _srq = IbvCreateSrq(GetRdmaPd(), &attr);
+    if (!_srq) {
+        int error = errno;
+        PLOG(ERROR) << "Fail to create SRQ for worker=" << this
+                    << ", depth=" << FLAGS_rdma_srq_depth;
+        errno = error;
+        return false;
+    }
+
+    RegisterRdmaAsyncEventContext(_srq, context);
+    _rx_capacity = std::min(static_cast<uint32_t>(_rx_cq->cqe),
+                           std::min(attr.attr.max_wr,
+                                       
static_cast<uint32_t>(FLAGS_rdma_srq_depth)));
+    uint32_t limit = FLAGS_rdma_srq_pool_blocks;
+    uint32_t grow_blocks = std::min(
+        limit, static_cast<uint32_t>(FLAGS_rdma_srq_pool_grow_blocks));
+    // Retain spare blocks beyond the posted depth so the first zero-copy
+    // receive need not wait for registration. Initialize with one MR. Runtime
+    // growth has its own, smaller granularity.
+    uint32_t spare = std::min(limit - _rx_capacity,
+                              std::max(grow_blocks, (_rx_capacity + 9) / 10));
+    _pool = SrqReceivePool::Create(GetRdmaPd(), GetRdmaBlockSize(), limit,
+                                   grow_blocks, _rx_capacity + spare);
+    if (_pool == nullptr) {
+        int error = errno;
+        PLOG(ERROR) << "Fail to create receive pool for SRQ worker=" << this
+                    << ", initial_blocks=" << _rx_capacity + spare;
+        errno = error;
+        return false;
+    }
+
+    _rx_buffers.resize(_rx_capacity, nullptr);
+    _rx_free.reserve(_rx_capacity);
+    _rx_reusable.reserve(_rx_capacity);
+    for (uint32_t i = 0; i < _rx_capacity; ++i) {
+        _rx_free.push_back(i);
+    }
+    Refill();
+    if (broken() || RxOutstanding() == 0) {
+        return false;
+    }
+
+    if (SrqEventSource::Create(&_event_source, shared_from_this(), _tag) != 0) 
{
+        int error = errno;
+        PLOG(ERROR) << "Fail to create event source for SRQ worker=" << this;
+        errno = error;
+        return false;
+    }
+
+    VersionedRefWithIdUniquePtr<SrqEventSource> source;
+    CHECK_EQ(0, SrqEventSource::Address(_event_source, &source));
+    if (source->Add(_channel->fd) != 0 || source->Add(_kick_fd) != 0 ||
+        source->Add(_pool->growth_fd()) != 0) {
+        int error = errno;
+        PLOG(ERROR) << "Fail to register event fds for SRQ worker=" << this;
+        errno = error;
+        return false;
+    }
+
+    // `GetWorker()` publishes this worker only after initialization succeeds.
+    // No QP can produce a completion yet, so arm both CQs here and let the
+    // first completion start `RunEvent()` without an initial `Kick()`.
+    ibv_cq* cq = _tx_cq;
+    int rc = IbvReqNotifySrqCq(cq, 0);
+    if (rc == 0) {
+        cq = _rx_cq;
+        rc = IbvReqNotifySrqCq(cq, 0);
+    }
+    if (rc != 0) {
+        LOG(ERROR) << "Fail to arm " << (cq == _tx_cq ? "TX" : "RX")
+                   << " CQ for SRQ worker=" << this << ", rc=" << rc
+                   << ": " << berror(rc);
+        errno = rc;
+        return false;
+    }
+    return true;
+}
+
+#if defined(UNIT_TEST)
+// Fake verbs supply resources while the real event loop, command queue, CQ
+// handlers and refill run unchanged. The worker takes ownership of the pool
+// and handles, just as after normal initialization.
+bool InitializeSrqWorkerForTest(const std::shared_ptr<SrqWorker>& worker, 
ibv_cq* tx,
+                                ibv_cq* rx, ibv_srq* srq, SrqReceivePool* 
pool, uint32_t depth) {
+    worker->_tx_cq = tx;
+    worker->_rx_cq = rx;
+    worker->_srq = srq;
+    worker->_pool = pool;
+    worker->_tx_capacity = tx->cqe;
+    worker->_rx_capacity = depth;
+    worker->_rx_buffers.resize(depth, nullptr);
+    worker->_rx_free.reserve(depth);
+    worker->_rx_reusable.reserve(depth);
+    for (uint32_t i = 0; i < depth; ++i) {
+        worker->_rx_free.push_back(i);
+    }
+    if (worker->_connections.init(16) != 0) {
+        return false;
+    }
+    worker->Refill();
+    worker->_kick_fd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC);
+    if (worker->broken() || worker->_kick_fd < 0 ||
+        SrqEventSource::Create(&worker->_event_source, worker, worker->_tag) 
!= 0) {
+        return false;
+    }
+    VersionedRefWithIdUniquePtr<SrqEventSource> source;
+    CHECK_EQ(0, SrqEventSource::Address(worker->_event_source, &source));
+    return source->Add(worker->_kick_fd) == 0 &&
+        IbvReqNotifySrqCq(tx, 0) == 0 && IbvReqNotifySrqCq(rx, 0) == 0;
+}
+
+bool AddSrqConnectionForTest(const std::shared_ptr<SrqWorker>& worker,
+                             const ConnectionPtr& connection, uint32_t 
sq_size) {
+    return worker->Add(connection, sq_size);
+}
+
+uint32_t ReserveSrqTxForTest(const std::shared_ptr<SrqWorker>& worker, 
uint32_t count) {
+    return worker->ReserveTx(count, false);
+}
+
+void ScheduleSrqWorkerForTest(const std::shared_ptr<SrqWorker>& worker,
+                              std::function<void()> callback, bool wake) {
+    worker->Schedule([callback = std::move(callback)] {
+        callback();
+        return ProgressRound();
+    }, wake);
+}
+
+void WakeSrqWorkerForTest(const std::shared_ptr<SrqWorker>& worker) {
+    worker->Kick();
+}
+
+void StopSrqWorkerForTest(const std::shared_ptr<SrqWorker>& worker) {
+    worker->Stop();
+}
+#endif
+
+bool SrqWorker::Add(const ConnectionPtr& connection, uint32_t sq_size) {
+    std::lock_guard<bthread::Mutex> owner(_progress_mutex);
+    if (broken()) {
+        errno = ERDMA;
+        return false;
+    }
+
+    ibv_qp_init_attr attr = {};
+    // Store the RdmaAsyncEventContext base address expected by async dispatch.
+    // With multiple inheritance, it may differ from the most-derived object
+    // address.
+    attr.qp_context = static_cast<RdmaAsyncEventContext*>(connection.get());
+    attr.send_cq = _tx_cq;
+    attr.recv_cq = _rx_cq;
+    attr.srq = _srq;
+    attr.qp_type = IBV_QPT_RC;
+    attr.cap.max_send_wr = sq_size;
+    attr.cap.max_send_sge = std::min(GetRdmaMaxSge(), FLAGS_rdma_srq_max_sge);
+    attr.cap.max_inline_data = FLAGS_rdma_srq_inline_size;
+    connection->_qp = IbvCreateQp(GetRdmaPd(), &attr);
+    // Inline support is optional. Retry without requesting it if the device
+    // rejects the requested size.
+    if (connection->_qp == nullptr && attr.cap.max_inline_data) {
+        attr.cap.max_inline_data = 0;
+        connection->_qp = IbvCreateQp(GetRdmaPd(), &attr);
+    }
+    if (connection->_qp == nullptr) {
+        int error = errno;
+        PLOG(ERROR) << "Fail to create QP for SRQ worker=" << this
+                    << ", socket_id=" << connection->_sid << ", sq_size=" << 
sq_size;
+        errno = error;
+        return false;
+    }
+
+    RegisterRdmaAsyncEventContext(connection->_qp, connection.get());
+    connection->_qpn = connection->_qp->qp_num;
+    connection->_sq_size = std::min(sq_size, attr.cap.max_send_wr);
+    connection->_max_sge = std::min(attr.cap.max_send_sge,
+                                    
static_cast<uint32_t>(FLAGS_rdma_srq_max_sge));
+    connection->_inline_size = 
std::min(static_cast<uint32_t>(FLAGS_rdma_srq_inline_size),
+                                        attr.cap.max_inline_data);
+    connection->_records = 
std::make_unique<SrqConnection::TxRecord[]>(connection->_sq_size);
+    CHECK(_connections.seek(connection->_qpn) == nullptr);
+    _connections[connection->_qpn] = connection;
+    ++g_live_connections;
+    return true;
+}
+
+void SrqWorker::Refill() {
+    if (broken() || _pool == nullptr) {
+        return;
+    }
+
+    _pool->Prepare(_rx_reusable.size());
+    // Initialization reserves enough blocks for all receive slots. Afterwards,
+    // pressure copying in `HandleReceive()` preserves a replacement for every
+    // completed slot, so replenishment never waits for exported blocks to 
return.
+    DCHECK_GE(uint64_t(_pool->available()) + _rx_reusable.size(), 
_rx_free.size());

Review Comment:
   The invariant checked here is not preserved by the zero-copy path. 
`HandleReceive()` increments `_rx_free` for every completion, but when it 
transfers a block with `TakeReceived()` that block is not returned to 
`_pool->available()` and `_rx_reusable` is unchanged. Thus a batch can start 
with `available + reusable == rx_free`, export one zero-copy block, and enter 
`Refill()` with `available + reusable < rx_free`; sustained pool pressure makes 
this assertion fail even though the code deliberately allows zero-copy blocks 
to remain application-held. Remove or revise this assertion to account for 
leased zero-copy blocks, and keep the refill logic's actual shortage handling 
as the source of truth.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to