Allow the user to override the default inline data size. We still require a minimum size in order to transfer receive buffer update message.
Signed-off-by: Sean Hefty <[email protected]> --- We can eliminate the need for inline entirely by reserving some of the send buffer space for control messages, but that work is deferred for a later patch. include/rdma/rsocket.h | 3 ++- src/rsocket.c | 21 +++++++++++++++++---- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/include/rdma/rsocket.h b/include/rdma/rsocket.h index 9fd9929..65feda9 100644 --- a/include/rdma/rsocket.h +++ b/include/rdma/rsocket.h @@ -75,7 +75,8 @@ int rgetsockname(int socket, struct sockaddr *addr, socklen_t *addrlen); #define SOL_RDMA 0x10000 enum { RDMA_SQSIZE, - RDMA_RQSIZE + RDMA_RQSIZE, + RDMA_INLINE }; int rsetsockopt(int socket, int level, int optname, diff --git a/src/rsocket.c b/src/rsocket.c index f94e48a..ef070a8 100644 --- a/src/rsocket.c +++ b/src/rsocket.c @@ -105,6 +105,7 @@ struct rs_sge { uint32_t length; }; +#define RS_MIN_INLINE (sizeof(struct rs_sge)) #define rs_host_is_net() (1 == htonl(1)) #define RS_CONN_FLAG_NET 1 @@ -163,6 +164,7 @@ struct rsocket { uint16_t sseq_no; uint16_t sseq_comp; uint16_t sq_size; + uint16_t sq_inline; uint16_t rq_size; uint16_t rseq_no; @@ -229,11 +231,13 @@ static struct rsocket *rs_alloc(struct rsocket *inherited_rs) if (inherited_rs) { rs->sbuf_size = inherited_rs->sbuf_size; rs->rbuf_size = inherited_rs->rbuf_size; + rs->sq_inline = inherited_rs->sq_inline; rs->sq_size = inherited_rs->sq_size; rs->rq_size = inherited_rs->rq_size; rs->ctrl_avail = inherited_rs->ctrl_avail; } else { rs->sbuf_size = rs->rbuf_size = RS_BUF_SIZE; + rs->sq_inline = RS_INLINE; rs->sq_size = rs->rq_size = RS_QP_SIZE; rs->ctrl_avail = RS_QP_CTRL_SIZE; } @@ -367,7 +371,7 @@ static int rs_create_ep(struct rsocket *rs) qp_attr.cap.max_recv_wr = rs->rq_size; qp_attr.cap.max_send_sge = 2; qp_attr.cap.max_recv_sge = 1; - qp_attr.cap.max_inline_data = RS_INLINE; + qp_attr.cap.max_inline_data = rs->sq_inline; ret = rdma_create_qp(rs->cm_id, NULL, &qp_attr); if (ret) @@ -1156,7 +1160,7 @@ ssize_t rsend(int socket, const void *buf, size_t len, int flags) if (xfer_size > rs->target_sgl[rs->target_sge].length) xfer_size = rs->target_sgl[rs->target_sge].length; - if (xfer_size <= RS_INLINE) { + if (xfer_size <= rs->sq_inline) { sge.addr = (uintptr_t) buf; sge.length = xfer_size; sge.lkey = 0; @@ -1277,7 +1281,7 @@ static ssize_t rsendv(int socket, const struct iovec *iov, int iovcnt, int flags ret = rs_write_data(rs, rs_wrid(1, xfer_size), rs->ssgl, 1, rs_msg_set(RS_OP_DATA, xfer_size), - xfer_size <= RS_INLINE ? IBV_SEND_INLINE : 0); + xfer_size <= rs->sq_inline ? IBV_SEND_INLINE : 0); if (xfer_size < rs_sbuf_left(rs)) rs->ssgl[0].addr += xfer_size; else @@ -1291,7 +1295,7 @@ static ssize_t rsendv(int socket, const struct iovec *iov, int iovcnt, int flags ret = rs_write_data(rs, rs_wrid(1, xfer_size), rs->ssgl, 2, rs_msg_set(RS_OP_DATA, xfer_size), - xfer_size <= RS_INLINE ? IBV_SEND_INLINE : 0); + xfer_size <= rs->sq_inline ? IBV_SEND_INLINE : 0); rs->ssgl[0].addr = (uintptr_t) rs->sbuf + rs->ssgl[1].length; } if (ret) @@ -1730,6 +1734,11 @@ int rsetsockopt(int socket, int level, int optname, case RDMA_RQSIZE: rs->rq_size = min((*(uint32_t *) optval), RS_QP_MAX_SIZE); break; + case RDMA_INLINE: + rs->sq_inline = min(*(uint32_t *) optval, RS_QP_MAX_SIZE); + if (rs->sq_inline < RS_MIN_INLINE) + rs->sq_inline = RS_MIN_INLINE; + break; default: break; } @@ -1809,6 +1818,10 @@ int rgetsockopt(int socket, int level, int optname, *((int *) optval) = rs->rq_size; *optlen = sizeof(int); break; + case RDMA_INLINE: + *((int *) optval) = rs->sq_inline; + *optlen = sizeof(int); + break; default: ret = ENOTSUP; break; -- To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
