We discovered a strange problem running OFED; We're not sure if it is a
OFED problem but we post it here anyway.
Short description:
We have a program that allocates a set of buffers with valloc, sends
them with ibv_post_send and free them.
This is run in loop;
We have a "caching"-algorithm so that we register memory only the first
time we come across a buffer address.
We starts getting wrong data for parts of sends after a couple of
iterations
There are a few things worth mentioning:
- We must use valloc; the test works with malloc
- We must have a malloc allocating a too large chunk before starting the
loop (the malloc fails)
We have modified the "rdma_lat.c" program to show the error (attached)
Regards
Asmund
/*
* Copyright (c) 2005 Topspin Communications. All rights reserved.
* Copyright (c) 2005 Mellanox Technologies Ltd. All rights reserved.
* Copyright (c) 2005 Hewlett Packard, Inc (Grant Grundler)
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
* $Id$
*/
#if HAVE_CONFIG_H
# include <config.h>
#endif /* HAVE_CONFIG_H */
#define _GNU_SOURCE
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/time.h>
#include <netdb.h>
#include <malloc.h>
#include <getopt.h>
#include <arpa/inet.h>
#include <byteswap.h>
#include <time.h>
#include <assert.h>
#include <infiniband/verbs.h>
#include <rdma/rdma_cma.h>
#include "get_clock.h"
/* -------------------------------------------------------------------------*/
/* -------------------------------------------------------------------------*/
#define PMPI_SIZE 40000
#define PMPI_BUFFER_SIZE (40000+8192)
/* -------------------------------------------------------------------------*/
/* -------------------------------------------------------------------------*/
#define PINGPONG_RDMA_WRID 3
#define MAX_INLINE 400
static int inline_size = MAX_INLINE;
static int page_size;
static pid_t pid;
struct report_options {
int unsorted;
int histogram;
int cycles; /* report delta's in cycles, not microsec's */
};
struct pingpong_context {
struct ibv_context *context;
struct ibv_pd *pd;
struct ibv_mr *mr;
struct ibv_cq *rcq;
struct ibv_cq *scq;
struct ibv_qp *qp;
void *buf;
volatile char *post_buf;
volatile char *poll_buf;
int size;
int tx_depth;
struct ibv_sge list;
struct ibv_send_wr wr;
};
struct pingpong_dest {
int lid;
int qpn;
int psn;
unsigned rkey;
unsigned long long vaddr;
};
struct pp_data {
int port;
int ib_port;
unsigned size;
int tx_depth;
int use_cma;
int sockfd;
char *servername;
struct pingpong_dest my_dest;
struct pingpong_dest *rem_dest;
struct ibv_device *ib_dev;
struct rdma_event_channel *cm_channel;
struct rdma_cm_id *cm_id;
};
static struct pingpong_context *pp_init_ctx(void *, struct pp_data *);
static uint16_t pp_get_local_lid(struct pingpong_context *ctx, int port)
{
struct ibv_port_attr attr;
if (ibv_query_port(ctx->context, port, &attr))
return 0;
return attr.lid;
}
static struct ibv_device *pp_find_dev(const char *ib_devname)
{
struct ibv_device **dev_list;
struct ibv_device *ib_dev = NULL;
dev_list = ibv_get_device_list(NULL);
if (!ib_devname) {
ib_dev = dev_list[0];
if (!ib_dev)
fprintf(stderr, "No IB devices found\n");
} else {
for (; (ib_dev = *dev_list); ++dev_list) {
if (!strcmp(ibv_get_device_name(ib_dev), ib_devname))
break;
}
if (!ib_dev)
fprintf(stderr, "IB device %s not found\n", ib_devname);
}
return ib_dev;
}
#define KEY_MSG_SIZE (sizeof "0000:000000:000000:00000000:0000000000000000")
#define KEY_PRINT_FMT "%04x:%06x:%06x:%08x:%016Lx"
static int pp_write_keys(int sockfd, const struct pingpong_dest *my_dest)
{
char msg[KEY_MSG_SIZE];
sprintf(msg, KEY_PRINT_FMT, my_dest->lid, my_dest->qpn,
my_dest->psn, my_dest->rkey, my_dest->vaddr);
if (write(sockfd, msg, sizeof msg) != sizeof msg) {
perror("client write");
fprintf(stderr, "Couldn't send local address\n");
return -1;
}
return 0;
}
static int pp_read_keys(int sockfd, const struct pingpong_dest *my_dest,
struct pingpong_dest *rem_dest)
{
int parsed;
char msg[KEY_MSG_SIZE];
if (read(sockfd, msg, sizeof msg) != sizeof msg) {
perror("pp_read_keys");
fprintf(stderr, "Couldn't read remote address\n");
return -1;
}
parsed = sscanf(msg, KEY_PRINT_FMT, &rem_dest->lid, &rem_dest->qpn,
&rem_dest->psn, &rem_dest->rkey, &rem_dest->vaddr);
if (parsed != 5) {
fprintf(stderr, "Couldn't parse line <%.*s>\n",
(int)sizeof msg, msg);
return -1;
}
return 0;
}
static struct pingpong_context *pp_client_connect(struct pp_data *data)
{
struct addrinfo *res, *t;
struct addrinfo hints = {
.ai_family = AF_UNSPEC,
.ai_socktype = SOCK_STREAM
};
char *service;
int n;
int sockfd = -1;
struct pingpong_context *ctx = NULL;
if (asprintf(&service, "%d", data->port) < 0)
goto err4;
n = getaddrinfo(data->servername, service, &hints, &res);
if (n < 0) {
fprintf(stderr, "%d:%s: %s for %s:%d\n",
pid, __func__, gai_strerror(n),
data->servername, data->port);
goto err4;
}
for (t = res; t; t = t->ai_next) {
sockfd = socket(t->ai_family, t->ai_socktype,
t->ai_protocol);
if (sockfd >= 0) {
if (!connect(sockfd, t->ai_addr, t->ai_addrlen))
break;
close(sockfd);
sockfd = -1;
}
}
if (sockfd < 0) {
fprintf(stderr, "%d:%s: Couldn't connect to %s:%d\n",
pid, __func__, data->servername, data->port);
goto err3;
}
ctx = pp_init_ctx(data->ib_dev, data);
if (!ctx)
goto err3;
data->sockfd = sockfd;
freeaddrinfo(res);
return ctx;
err3:
freeaddrinfo(res);
err4:
return NULL;
}
static int pp_client_exch_dest(struct pp_data *data)
{
if (data->rem_dest != NULL)
free(data->rem_dest);
data->rem_dest = malloc(sizeof *data->rem_dest);
if (!data->rem_dest)
return -1;
if (pp_write_keys(data->sockfd, &data->my_dest))
return -1;
return pp_read_keys(data->sockfd, &data->my_dest, data->rem_dest);
}
static struct pingpong_context *pp_server_connect(struct pp_data *data)
{
struct addrinfo *res, *t;
struct addrinfo hints = {
.ai_flags = AI_PASSIVE,
.ai_family = AF_UNSPEC,
.ai_socktype = SOCK_STREAM
};
char *service;
int sockfd = -1, connfd;
int n;
struct pingpong_context *ctx = NULL;
if (asprintf(&service, "%d", data->port) < 0)
goto err5;
if ( (n = getaddrinfo(NULL, service, &hints, &res)) < 0 ) {
fprintf(stderr, "%d:%s: %s for port %d\n", pid, __func__,
gai_strerror(n), data->port);
goto err5;
}
for (t = res; t; t = t->ai_next) {
sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol);
if (sockfd >= 0) {
n = 1;
setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &n, sizeof n);
if (!bind(sockfd, t->ai_addr, t->ai_addrlen))
break;
close(sockfd);
sockfd = -1;
}
}
if (sockfd < 0) {
fprintf(stderr, "%d:%s: Couldn't listen to port %d\n", pid,
__func__, data->port);
goto err4;
}
listen(sockfd, 1);
connfd = accept(sockfd, NULL, 0);
if (connfd < 0) {
perror("server accept");
fprintf(stderr, "%d:%s: accept() failed\n", pid, __func__);
close(sockfd);
goto err4;
}
close(sockfd);
ctx = pp_init_ctx(data->ib_dev, data);
if (!ctx)
goto err4;
data->sockfd = connfd;
freeaddrinfo(res);
return ctx;
err4:
freeaddrinfo(res);
err5:
return NULL;
}
static int pp_server_exch_dest(struct pp_data *data)
{
if (data->rem_dest != NULL)
free(data->rem_dest);
data->rem_dest = malloc(sizeof *data->rem_dest);
if (!data->rem_dest)
return -1;
if (pp_read_keys(data->sockfd, &data->my_dest, data->rem_dest))
return -1;
return pp_write_keys(data->sockfd, &data->my_dest);
}
static struct pingpong_context *pp_init_ctx(void *ptr, struct pp_data *data)
{
struct pingpong_context *ctx;
struct ibv_device *ib_dev;
ctx = malloc(sizeof *ctx);
if (!ctx)
return NULL;
ctx->size = 4096 + 12 * 40960;
ctx->tx_depth = data->tx_depth;
ctx->buf = memalign(page_size, ctx->size );
if (!ctx->buf) {
fprintf(stderr, "%d:%s: Couldn't allocate work buf.\n",
pid, __func__);
return NULL;
}
memset(ctx->buf, 0, ctx->size);
ctx->post_buf = (char *)ctx->buf;
ctx->poll_buf = (char *)ctx->buf + 4;
ib_dev = (struct ibv_device *)ptr;
ctx->context = ibv_open_device(ib_dev);
if (!ctx->context) {
fprintf(stderr, "%d:%s: Couldn't get context for %s\n",
pid, __func__, ibv_get_device_name(ib_dev));
return NULL;
}
ctx->pd = ibv_alloc_pd(ctx->context);
if (!ctx->pd) {
fprintf(stderr, "%d:%s: Couldn't allocate PD\n", pid, __func__);
return NULL;
}
/* We dont really want IBV_ACCESS_LOCAL_WRITE, but IB spec says:
* The Consumer is not allowed to assign Remote Write or Remote Atomic
to
* a Memory Region that has not been assigned Local Write. */
ctx->mr = ibv_reg_mr(ctx->pd, ctx->buf, ctx->size,
IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_LOCAL_WRITE);
if (!ctx->mr) {
fprintf(stderr, "%d:%s: Couldn't allocate MR\n", pid, __func__);
return NULL;
}
ctx->rcq = ibv_create_cq(ctx->context, 1, NULL, NULL, 0);
if (!ctx->rcq) {
fprintf(stderr, "%d:%s: Couldn't create recv CQ\n", pid,
__func__);
return NULL;
}
ctx->scq = ibv_create_cq(ctx->context, ctx->tx_depth, ctx, NULL, 0);
if (!ctx->scq) {
fprintf(stderr, "%d:%s: Couldn't create send CQ\n", pid,
__func__);
return NULL;
}
struct ibv_qp_init_attr attr = {
.send_cq = ctx->scq,
.recv_cq = ctx->rcq,
.cap = {
.max_send_wr = ctx->tx_depth,
/* Work around: driver doesnt support
* recv_wr = 0 */
.max_recv_wr = 1,
.max_send_sge = 1,
.max_recv_sge = 1,
.max_inline_data = inline_size,
},
.qp_type = IBV_QPT_RC
};
ctx->qp = ibv_create_qp(ctx->pd, &attr);
if (!ctx->qp) {
fprintf(stderr, "%d:%s: Couldn't create QP\n", pid, __func__);
return NULL;
}
{
struct ibv_qp_attr attr;
attr.qp_state = IBV_QPS_INIT;
attr.pkey_index = 0;
attr.port_num = data->ib_port;
attr.qp_access_flags = IBV_ACCESS_REMOTE_WRITE;
if (ibv_modify_qp(ctx->qp, &attr,
IBV_QP_STATE |
IBV_QP_PKEY_INDEX |
IBV_QP_PORT |
IBV_QP_ACCESS_FLAGS)) {
fprintf(stderr, "%d:%s: Failed to modify QP to INIT\n",
pid, __func__);
return NULL;
}
}
return ctx;
}
static int pp_connect_ctx(struct pingpong_context *ctx, struct pp_data *data)
{
struct ibv_qp_attr attr = {
.qp_state = IBV_QPS_RTR,
.path_mtu = IBV_MTU_256,
.dest_qp_num = data->rem_dest->qpn,
.rq_psn = data->rem_dest->psn,
.max_dest_rd_atomic = 1,
.min_rnr_timer = 12,
.ah_attr.is_global = 0,
.ah_attr.dlid = data->rem_dest->lid,
.ah_attr.sl = 0,
.ah_attr.src_path_bits = 0,
.ah_attr.port_num = data->ib_port
};
if (ibv_modify_qp(ctx->qp, &attr,
IBV_QP_STATE |
IBV_QP_AV |
IBV_QP_PATH_MTU |
IBV_QP_DEST_QPN |
IBV_QP_RQ_PSN |
IBV_QP_MAX_DEST_RD_ATOMIC |
IBV_QP_MIN_RNR_TIMER)) {
fprintf(stderr, "%s: Failed to modify QP to RTR\n", __func__);
return 1;
}
attr.qp_state = IBV_QPS_RTS;
attr.timeout = 14;
attr.retry_cnt = 7;
attr.rnr_retry = 7;
attr.sq_psn = data->my_dest.psn;
attr.max_rd_atomic = 1;
if (ibv_modify_qp(ctx->qp, &attr,
IBV_QP_STATE |
IBV_QP_TIMEOUT |
IBV_QP_RETRY_CNT |
IBV_QP_RNR_RETRY |
IBV_QP_SQ_PSN |
IBV_QP_MAX_QP_RD_ATOMIC)) {
fprintf(stderr, "%s: Failed to modify QP to RTS\n", __func__);
return 1;
}
return 0;
}
static int pp_open_port(struct pingpong_context *ctx, struct pp_data *data )
{
char addr_fmt[] = "%8s address: LID %#04x QPN %#06x PSN %#06x RKey
%#08x VAddr %#016Lx\n";
/* Create connection between client and server.
* We do it by exchanging data over a TCP socket connection. */
data->my_dest.lid = pp_get_local_lid(ctx, data->ib_port);
data->my_dest.qpn = ctx->qp->qp_num;
data->my_dest.psn = lrand48() & 0xffffff;
if (!data->my_dest.lid) {
fprintf(stderr, "Local lid 0x0 detected. Is an SM running?\n");
return -1;
}
data->my_dest.rkey = ctx->mr->rkey;
data->my_dest.vaddr = (uintptr_t)ctx->buf;
printf(addr_fmt, "local", data->my_dest.lid, data->my_dest.qpn,
data->my_dest.psn,
data->my_dest.rkey, data->my_dest.vaddr);
if (data->servername) {
if (pp_client_exch_dest(data))
return 1;
} else {
if (pp_server_exch_dest(data))
return 1;
}
printf(addr_fmt, "remote", data->rem_dest->lid, data->rem_dest->qpn,
data->rem_dest->psn, data->rem_dest->rkey,
data->rem_dest->vaddr);
if (pp_connect_ctx(ctx, data))
return 1;
/* An additional handshake is required *after* moving qp to RTR.
Arbitrarily reuse exch_dest for this purpose. */
if (data->servername) {
if (pp_client_exch_dest(data))
return -1;
} else {
if (pp_server_exch_dest(data))
return -1;
}
if (write(data->sockfd, "done", sizeof "done") != sizeof "done"){
perror("write");
fprintf(stderr, "Couldn't write to socket\n");
return 1;
}
close(data->sockfd);
return 0;
}
static void usage(const char *argv0)
{
printf("Usage:\n");
printf(" %s start a server and wait for connection\n",
argv0);
printf(" %s <host> connect to server at <host>\n", argv0);
printf("\n");
printf("Options:\n");
printf(" -p, --port=<port> listen on/connect to port <port>
(default 18515)\n");
printf(" -d, --ib-dev=<dev> use IB device <dev> (default first
device found)\n");
printf(" -i, --ib-port=<port> use port <port> of IB device (default
1)\n");
printf(" -s, --size=<size> size of message to exchange (default
1)\n");
printf(" -t, --tx-depth=<dep> size of tx queue (default 50)\n");
printf(" -n, --iters=<iters> number of exchanges (at least 2,
default 1000)\n");
printf(" -I, --inline_size=<size> max size of message to be sent in
inline mode (default 400)\n");
printf(" -C, --report-cycles report times in cpu cycle units
(default microseconds)\n");
printf(" -H, --report-histogram print out all results (default print
summary only)\n");
printf(" -U, --report-unsorted (implies -H) print out unsorted
results (default sorted)\n");
printf(" -c, --cma Use the RDMA CMA to setup the RDMA
connection\n");
}
/* -------------------------------------------------------------------------*/
/* Code added to show problem */
/* Start */
/* -------------------------------------------------------------------------*/
void
post_send_and_poll(struct ibv_qp *qp,
struct ibv_send_wr *wr,
struct ibv_send_wr **bad_wr,
struct ibv_cq *cq)
{
int ne;
struct ibv_wc wc;
if (ibv_post_send(qp, wr, bad_wr)) {
fprintf(stderr, "Couldn't post send:\n");
assert(0);
}
do {
ne = ibv_poll_cq(cq, 1, &wc);
} while (ne == 0);
if (ne < 0) {
fprintf(stderr, "poll CQ failed %d\n", ne);
assert(0);
}
if (wc.status != IBV_WC_SUCCESS) {
fprintf(stderr, "Failed status %d: wr_id %d\n",
wc.status, (int) wc.wr_id);
assert(0);
}
}
/* -------------------------------------------------------------------------*/
static char* pin_addr[12];
static struct ibv_mr* pin_mr[12];
static int pinlist_init_done = 0;
/* -------------------------------------------------------------------------*/
void
send_buffer(struct pingpong_context* ctx, struct pp_data data, char* buffer,
int tag, int loop)
{
struct ibv_sge list;
struct ibv_send_wr wr;
struct ibv_send_wr *bad_wr;
struct ibv_mr* mr;
int i;
int found = 0;
int displ = (((loop *3) + tag - 11) * 40960) +4096;
if (pinlist_init_done == 0) {
for (i = 0; i < 12; i++) {
pin_addr[i] = NULL;
}
pinlist_init_done = 1;
}
for (i = 0; i < 12; i++) {
if (pin_addr[i] == buffer) {
mr = pin_mr[i];
found = 1;
break;
}
}
if (found == 0) {
printf(" %p Must be pinned\n", buffer);
mr = ibv_reg_mr(ctx->pd, buffer, 40000,
IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_LOCAL_WRITE);
if (mr == NULL) {
fprintf(stderr, "%d:%s: Couldn't allocate MR\n", pid, __func__);
assert(0);
}
for (i = 0; i < 12; i++) {
if (pin_addr[i] == 0) {
pin_addr[i] =buffer;
pin_mr[i] = mr;
break;
}
}
} else {
printf(" %p Already pinned\n", buffer);
}
list.addr = (uintptr_t) buffer;
list.length = 40000;
list.lkey = mr->lkey;
wr.wr.rdma.remote_addr = data.rem_dest->vaddr + displ;
wr.wr.rdma.rkey = data.rem_dest->rkey;
wr.wr_id = PINGPONG_RDMA_WRID;
wr.sg_list = &list;
wr.num_sge = 1;
wr.opcode = IBV_WR_RDMA_WRITE;
// wr.send_flags = IBV_SEND_SIGNALED| IBV_SEND_INLINE;
wr.send_flags = IBV_SEND_SIGNALED;
wr.next = NULL;
post_send_and_poll(ctx->qp, &wr, &bad_wr, ctx->scq);
}
/* -------------------------------------------------------------------------*/
/* -------------------------------------------------------------------------*/
/* -------------------------------------------------------------------------*/
void
init(char* b, int tag, int l)
{
int i;
int* p = (int*) b;
for (i = 0; i < 10000; i++) {
p[i] = l << 24 | tag << 16 | i;
}
}
/* -------------------------------------------------------------------------*/
void
do_sender(struct pingpong_context* ctx, struct pp_data data)
{
int i;
char* rbuf1 = 0;
char* rbuf2 = 0;
char* rbuf3 = 0;
char* rbuf4 = 0;
char* sbuf1 = 0;
char* sbuf2 = 0;
char* sbuf3 = 0;
char* sbuf4 = 0;
i = 0;
rbuf1 = (char*)valloc(88192);
sbuf1 = (char*)valloc(48192);
rbuf2 = (char*)valloc(88192);
sbuf2 = (char*)valloc(48192);
rbuf3 = (char*)valloc(88192);
sbuf3 = (char*)valloc(48192);
rbuf4 = (char*)valloc(88192);
sbuf4 = (char*)valloc(48192);
init(sbuf1,11,i);
init(sbuf2,12,i);
init(sbuf3,13,i);
init(sbuf4,14,i);
printf("sbuf %p %p %p %p\n", sbuf1, sbuf2, sbuf3, sbuf4);
send_buffer(ctx, data, sbuf1, 11, i);
send_buffer(ctx, data, sbuf2, 12, i);
send_buffer(ctx, data, sbuf3, 13, i);
free(sbuf1);
free(sbuf2);
free(sbuf3);
i = 1;
sbuf1 = (char*)valloc(48192);
sbuf2 = (char*)valloc(48192);
sbuf3 = (char*)valloc(48192);
init(sbuf1,11,i);
init(sbuf2,12,i);
init(sbuf3,13,i);
init(sbuf4,14,i);
printf("sbuf %p %p %p %p\n", sbuf1, sbuf2, sbuf3, sbuf4);
send_buffer(ctx, data, sbuf1, 11, i);
send_buffer(ctx, data, sbuf2, 12, i);
send_buffer(ctx, data, sbuf3, 13, i);
free(sbuf1);
free(sbuf2);
free(sbuf3);
i = 2;
sbuf1 = (char*)valloc(48192);
sbuf2 = (char*)valloc(48192);
sbuf3 = (char*)valloc(48192);
init(sbuf1,11,i);
init(sbuf2,12,i);
init(sbuf3,13,i);
init(sbuf4,14,i);
printf("sbuf %p %p %p %p\n", sbuf1, sbuf2, sbuf3, sbuf4);
send_buffer(ctx, data, sbuf1, 11, i);
send_buffer(ctx, data, sbuf2, 12, i);
send_buffer(ctx, data, sbuf3, 13, i);
free(sbuf1);
free(sbuf2);
free(sbuf3);
}
/* -------------------------------------------------------------------------*/
void
check(char* b, int tag, int l)
{
int i;
int count = 0;
int* p = (int*) b;
for (i = 0; i < 10000; i++) {
int exp = l << 24 | tag << 16 | i;
if (p[i] != exp) {
int pl = (p[i] >> 24) & 0xff;
int pt = (p[i] >> 16) & 0xff;
int pi = p[i]& 0xffff;
if (count ++ < 10) {
printf("----- check buff loop: %d tag: %d buff[%d] exp: %x(%d %d
%d) act: %x(%d %d %d)\n", l, tag, i, exp, l,tag,i, p[i], pl,pt,pi);
}
}
}
if (count != 0) {
printf("----- check buff loop: %d tag: %d --- found %d errors (but just
10 printed)\n", l, tag, count);
}
}
/* -------------------------------------------------------------------------*/
void
do_receiver(struct pingpong_context* ctx, struct pp_data data)
{
char* rbuf = (char*)ctx->buf + 4096;
check(rbuf,11,0); rbuf += 40960;
check(rbuf,12,0); rbuf += 40960;
check(rbuf,13,0); rbuf += 40960;
check(rbuf,11,1); rbuf += 40960;
check(rbuf,12,1); rbuf += 40960;
check(rbuf,13,1); rbuf += 40960;
check(rbuf,11,2); rbuf += 40960;
check(rbuf,12,2); rbuf += 40960;
check(rbuf,13,2);
}
/* -------------------------------------------------------------------------*/
/* End */
/* Code added to show problem */
/* -------------------------------------------------------------------------*/
/* -------------------------------------------------------------------------*/
/* -------------------------------------------------------------------------*/
/* -------------------------------------------------------------------------*/
int main(int argc, char *argv[])
{
const char *ib_devname = NULL;
int iters = 1000;
struct report_options report = {};
struct pingpong_context *ctx;
struct ibv_qp *qp;
struct ibv_send_wr *wr;
volatile char *poll_buf;
volatile char *post_buf;
struct ibv_send_wr *bad_wr;
int scnt, rcnt, ccnt;
cycles_t *tstamp;
struct pp_data data = {
.port = 18515,
.ib_port = 1,
.size = 1,
.tx_depth = 50,
.use_cma = 0,
.servername = NULL,
.rem_dest = NULL,
.ib_dev = NULL,
.cm_channel = NULL,
.cm_id = NULL
};
/* Parameter parsing. */
while (1) {
int c;
static struct option long_options[] = {
{ .name = "port", .has_arg = 1, .val = 'p' },
{ .name = "ib-dev", .has_arg = 1, .val = 'd' },
{ .name = "ib-port", .has_arg = 1, .val = 'i' },
{ .name = "size", .has_arg = 1, .val = 's' },
{ .name = "iters", .has_arg = 1, .val = 'n' },
{ .name = "tx-depth", .has_arg = 1, .val = 't' },
{ .name = "inline_size", .has_arg = 1, .val = 'I' },
{ .name = "report-cycles", .has_arg = 0, .val = 'C' },
{ .name = "report-histogram",.has_arg = 0, .val = 'H' },
{ .name = "report-unsorted",.has_arg = 0, .val = 'U' },
{ .name = "cma", .has_arg = 0, .val = 'c' },
{ 0 }
};
c = getopt_long(argc, argv, "p:d:i:s:n:t:I:CHUc", long_options,
NULL);
if (c == -1)
break;
switch (c) {
case 'p':
data.port = strtol(optarg, NULL, 0);
if (data.port < 0 || data.port > 65535) {
usage(argv[0]);
return 1;
}
break;
case 'd':
ib_devname = strdup(optarg);
break;
case 'i':
data.ib_port = strtol(optarg, NULL, 0);
if (data.ib_port < 0) {
usage(argv[0]);
return 2;
}
break;
case 's':
data.size = strtol(optarg, NULL, 0);
if (data.size < 1) { usage(argv[0]); return 3; }
break;
case 't':
data.tx_depth = strtol(optarg, NULL, 0);
if (data.tx_depth < 1) { usage(argv[0]); return
4; }
break;
case 'n':
iters = strtol(optarg, NULL, 0);
if (iters < 2) {
usage(argv[0]);
return 5;
}
break;
case 'I':
inline_size = strtol(optarg, NULL, 0);
break;
case 'C':
report.cycles = 1;
break;
case 'H':
report.histogram = 1;
break;
case 'U':
report.unsorted = 1;
break;
case 'c':
data.use_cma = 1;
break;
default:
usage(argv[0]);
return 5;
}
}
if (optind == argc - 1)
data.servername = strdup(argv[optind]);
else if (optind < argc) {
usage(argv[0]);
return 6;
}
/*
* Done with parameter parsing. Perform setup.
*/
pid = getpid();
srand48(pid * time(NULL));
page_size = sysconf(_SC_PAGESIZE);
data.ib_dev = pp_find_dev(ib_devname);
if (!data.ib_dev)
return 7;
if (data.servername) {
ctx = pp_client_connect(&data);
if (!ctx)
return 8;
} else {
ctx = pp_server_connect(&data);
if (!ctx)
return 8;
}
if (pp_open_port(ctx, &data))
return 9;
wr = &ctx->wr;
ctx->list.addr = (uintptr_t) ctx->buf;
ctx->list.length = 1;
ctx->list.lkey = ctx->mr->lkey;
wr->wr.rdma.remote_addr = data.rem_dest->vaddr + 4;
wr->wr.rdma.rkey = data.rem_dest->rkey;
ctx->wr.wr_id = PINGPONG_RDMA_WRID;
ctx->wr.sg_list = &ctx->list;
ctx->wr.num_sge = 1;
ctx->wr.opcode = IBV_WR_RDMA_WRITE;
if (ctx->size > inline_size || ctx->size == 0) {
ctx->wr.send_flags = IBV_SEND_SIGNALED;
} else {
ctx->wr.send_flags = IBV_SEND_SIGNALED | IBV_SEND_INLINE;
}
ctx->wr.next = NULL;
scnt = 0;
rcnt = 0;
ccnt = 0;
poll_buf = ctx->poll_buf;
post_buf = ctx->post_buf;
qp = ctx->qp;
tstamp = malloc(iters * sizeof *tstamp);
if (!tstamp) {
perror("malloc");
return 10;
}
/* -------------------------------------------------------------------------*/
/* -------------------------------------------------------------------------*/
/* -------------------------------------------------------------------------*/
/* -------------------------------------------------------------------------*/
/* Code added to show problem */
/* Start */
/* -------------------------------------------------------------------------*/
if (data.servername == 0) {
char* dummy = (char*) malloc(39999000000);
assert(dummy == NULL);
do_sender(ctx, data);
*post_buf = (char)123; /* Send "data-ready to receiver */
post_send_and_poll(qp, wr, &bad_wr, ctx->scq);
} else {
while (*poll_buf == 0) { /* Wait for data-ready from sender */
}
do_receiver(ctx, data);
}
/* -------------------------------------------------------------------------*/
/* End */
/* Code added to show problem */
/* -------------------------------------------------------------------------*/
return 0;
}
_______________________________________________
general mailing list
general@lists.openfabrics.org
http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general
To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general
