chenBright commented on code in PR #3145:
URL: https://github.com/apache/brpc/pull/3145#discussion_r2634300379
##########
src/brpc/rdma/rdma_endpoint.cpp:
##########
@@ -924,26 +944,55 @@ int RdmaEndpoint::SendImm(uint32_t imm) {
memset(&wr, 0, sizeof(wr));
wr.opcode = IBV_WR_SEND_WITH_IMM;
wr.imm_data = butil::HostToNet32(imm);
- wr.send_flags |= IBV_SEND_SOLICITED;
- wr.send_flags |= IBV_SEND_SIGNALED;
+ wr.send_flags |= IBV_SEND_SOLICITED | IBV_SEND_SIGNALED;
+ wr.wr_id = 0;
ibv_send_wr* bad = NULL;
int err = ibv_post_send(_resource->qp, &wr, &bad);
if (err != 0) {
+ std::ostringstream oss;
+ DebugInfo(oss, ", ");
// We use other way to guarantee the Send Queue is not full.
// So we just consider this error as an unrecoverable error.
- LOG(WARNING) << "Fail to ibv_post_send: " << berror(err);
+ LOG(WARNING) << "Fail to ibv_post_send: " << berror(err) << " " <<
oss.str();
return -1;
}
+
+ // `_sq_imm_window_size' will never be negative.
+ // Because IMM can only be sent if
+ // `_sq_imm_window_size` is greater than 0.
+ _sq_imm_window_size -= 1;
return 0;
}
ssize_t RdmaEndpoint::HandleCompletion(ibv_wc& wc) {
bool zerocopy = FLAGS_rdma_recv_zerocopy;
switch (wc.opcode) {
case IBV_WC_SEND: { // send completion
- // Do nothing
- break;
+ if (0 == wc.wr_id) {
+ _sq_imm_window_size += 1;
+ // If there are any unacknowledged recvs, send an ack.
+ SendAck(0);
+ return 0;
+ }
+ // Update SQ window.
+ uint16_t wnd_to_update = wc.wr_id;
Review Comment:
> I saw in the documentation that sbuf cleanup should be done after the
other end has finished receiving. How can this be guaranteed when this part is
moved to IBV_WC_SEND?
Previously, because there was only an application-layer sliding window, the
sbuf could only be released by relying on the application-layer ACK.
Now, when the local device receives a send WC, it means that it has received
an ACK from the remote device, the SQ has been released, the data has been sent
to the remote device, and the sbuf can be safely released.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
