Michael,
The patch adds command line options for RDMA reads and starting PSN. I used
these modifications to
help isolate the RDMA read performance degradation with 4.6.2 firmware.
-arlin
Signed-off by: Arlin Davis <[EMAIL PROTECTED]>
Index: rdma_bw.c
===================================================================
--- rdma_bw.c (revision 3768)
+++ rdma_bw.c (working copy)
@@ -304,7 +304,9 @@ static struct pingpong_context *pp_init_
* The Consumer is not allowed to assign Remote Write or Remote Atomic
to
* a Memory Region that has not been assigned Local Write. */
ctx->mr = ibv_reg_mr(ctx->pd, ctx->buf, size * 2,
- IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_LOCAL_WRITE);
+ IBV_ACCESS_REMOTE_WRITE |
+ IBV_ACCESS_REMOTE_READ |
+ IBV_ACCESS_LOCAL_WRITE);
if (!ctx->mr) {
fprintf(stderr, "Couldn't allocate MR\n");
return NULL;
@@ -345,7 +347,9 @@ static struct pingpong_context *pp_init_
attr.qp_state = IBV_QPS_INIT;
attr.pkey_index = 0;
attr.port_num = port;
- attr.qp_access_flags = IBV_ACCESS_REMOTE_WRITE;
+ attr.qp_access_flags = IBV_ACCESS_REMOTE_WRITE |
+ IBV_ACCESS_REMOTE_READ |
+ IBV_ACCESS_LOCAL_WRITE;
if (ibv_modify_qp(ctx->qp, &attr,
IBV_QP_STATE |
@@ -370,7 +374,7 @@ static int pp_connect_ctx(struct pingpon
attr.path_mtu = IBV_MTU_2048;
attr.dest_qp_num = dest->qpn;
attr.rq_psn = dest->psn;
- attr.max_dest_rd_atomic = 1;
+ attr.max_dest_rd_atomic = 4;
attr.min_rnr_timer = 12;
attr.ah_attr.is_global = 0;
attr.ah_attr.dlid = dest->lid;
@@ -394,7 +398,7 @@ static int pp_connect_ctx(struct pingpon
attr.retry_cnt = 7;
attr.rnr_retry = 7;
attr.sq_psn = my_psn;
- attr.max_rd_atomic = 1;
+ attr.max_rd_atomic = 4;
if (ibv_modify_qp(ctx->qp, &attr,
IBV_QP_STATE |
IBV_QP_TIMEOUT |
@@ -417,6 +421,7 @@ static void usage(const char *argv0)
printf("\n");
printf("Options:\n");
printf(" -p, --port=<port> listen on/connect to port <port>
(default 18515)\n");
+ printf(" -P, --starting_psn starting sequence on QP (default
random)\n");
printf(" -d, --ib-dev=<dev> use IB device <dev> (default first
device found)\n");
printf(" -i, --ib-port=<port> use port <port> of IB device (default
1)\n");
printf(" -s, --size=<size> size of message to exchange (default
65536)\n");
@@ -487,6 +492,8 @@ int main(int argc, char *argv[])
int scnt, ccnt;
int sockfd;
int duplex = 0;
+ int rdma_read = 0;
+ int starting_psn = 0;
struct ibv_qp *qp;
cycles_t *tposted;
@@ -498,16 +505,18 @@ int main(int argc, char *argv[])
static struct option long_options[] = {
{ .name = "port", .has_arg = 1, .val = 'p' },
+ { .name = "starting_psn", .has_arg = 1, .val = 'P' },
{ .name = "ib-dev", .has_arg = 1, .val = 'd' },
{ .name = "ib-port", .has_arg = 1, .val = 'i' },
{ .name = "size", .has_arg = 1, .val = 's' },
{ .name = "iters", .has_arg = 1, .val = 'n' },
{ .name = "tx-depth", .has_arg = 1, .val = 't' },
{ .name = "bidirectional", .has_arg = 0, .val = 'b' },
+ { .name = "rdma_read", .has_arg = 0, .val = 'r' },
{ 0 }
};
- c = getopt_long(argc, argv, "p:d:i:s:n:t:b", long_options,
NULL);
+ c = getopt_long(argc, argv, "p:P:d:i:s:n:t:br", long_options,
NULL);
if (c == -1)
break;
@@ -520,6 +529,14 @@ int main(int argc, char *argv[])
}
break;
+ case 'P':
+ starting_psn = strtol(optarg, NULL, 0);
+ if (port <= 0) {
+ usage(argv[0]);
+ return 1;
+ }
+ break;
+
case 'd':
ib_devname = strdupa(optarg);
break;
@@ -567,6 +584,10 @@ int main(int argc, char *argv[])
duplex = 1;
break;
+ case 'r':
+ rdma_read = 1;
+ break;
+
default:
usage(argv[0]);
return 1;
@@ -615,7 +636,11 @@ int main(int argc, char *argv[])
my_dest.lid = pp_get_local_lid(ctx, ib_port);
my_dest.qpn = ctx->qp->qp_num;
- my_dest.psn = lrand48() & 0xffffff;
+ if (!starting_psn)
+ my_dest.psn = lrand48() & 0xffffff;
+ else
+ my_dest.psn = starting_psn;
+
if (!my_dest.lid) {
fprintf(stderr, "Local lid 0x0 detected. Is an SM running?\n");
return 1;
@@ -624,9 +649,10 @@ int main(int argc, char *argv[])
my_dest.vaddr = (uintptr_t)ctx->buf + ctx->size;
printf(" local address: LID %#04x, QPN %#06x, PSN %#06x "
- "RKey %#08x VAddr %#016Lx\n",
+ "RKey %#08x VAddr %#016Lx %s\n",
my_dest.lid, my_dest.qpn, my_dest.psn,
- my_dest.rkey, my_dest.vaddr);
+ my_dest.rkey, my_dest.vaddr,
+ rdma_read ? "RDMA_READ":"RDMA_WRITE");
if (servername) {
sockfd = pp_client_connect(servername, port);
@@ -643,10 +669,11 @@ int main(int argc, char *argv[])
if (!rem_dest)
return 1;
- printf(" remote address: LID %#04x, QPN %#06x, PSN %#06x, "
- "RKey %#08x VAddr %#016Lx\n",
+ printf(" remote address: LID %#04x, QPN %#06x, PSN %#06x "
+ "RKey %#08x VAddr %#016Lx %s\n",
rem_dest->lid, rem_dest->qpn, rem_dest->psn,
- rem_dest->rkey, rem_dest->vaddr);
+ rem_dest->rkey, rem_dest->vaddr,
+ rdma_read ? "RDMA_READ":"RDMA_WRITE");
if (pp_connect_ctx(ctx, ib_port, my_dest.psn, rem_dest))
return 1;
@@ -675,7 +702,11 @@ int main(int argc, char *argv[])
ctx->wr.wr_id = PINGPONG_RDMA_WRID;
ctx->wr.sg_list = &ctx->list;
ctx->wr.num_sge = 1;
- ctx->wr.opcode = IBV_WR_RDMA_WRITE;
+ if (rdma_read)
+ ctx->wr.opcode = IBV_WR_RDMA_READ;
+ else
+ ctx->wr.opcode = IBV_WR_RDMA_WRITE;
+
ctx->wr.send_flags = IBV_SEND_SIGNALED;
ctx->wr.next = NULL;
_______________________________________________
openib-general mailing list
[email protected]
http://openib.org/mailman/listinfo/openib-general
To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general
