Hi Sean, I have succefully done the latency test on NetEffect NE020 cards (see attached xls file with graph). I worked with librdmacm version 1.0.12 (of June 2010) and your new APIs rdma_reg_msgs rdma_post_recv rdma_post_send
Is librdmacm version 1.0.12 stable? The flag IBV_SEND_INLINE supports buffer size until 64 bytes (cards' feature). I know that RDMA is a protocol connected with 10 Gigabit/s Ethernet. How can I see this trasfer speed? Can you suggest the "path"? I attached the starting latency_RDMA.cpp code, which works with librdmacm version 1.0.12 with BUF_SIZE < = 64 bytes (thanks to Chien). I would like to explore transfer speed in range 1 bytes < BUF_SIZE < 4 Mbytes. I try changing the flag IBV_SEND_INLINE - as you suggested in the mail on July 1st - but the code does not work. All contributions are welcome about RDMA transfer speed code. Thank you very much. Regards, Andrea Andrea Gozzelino INFN - Laboratori Nazionali di Legnaro (LNL) Viale dell'Universita' 2 -I-35020 - Legnaro (PD)- ITALIA Office: E-101 Tel: +39 049 8068346 Fax: +39 049 641925 Mail: [email protected] Cell: +39 3488245552
/*
------------------------------------------------------------------
Program: RDMA wrapper2
Use: Reduce RDMA to socket model
Author: Damiano Bortolato - Andrea Gozzelino (INFN - LNL)
Date: June 2010
Compile: g++ -c wrapper_RDMA2.cpp -lrdmacm
Note: use librdmacm-1.0.12 by Sean Hefty
Note: help from Chien Tin Tung (Intel)
------------------------------------------------------------------
*/
//Include
#include <rdma/rdma_cma.h>
#include <rdma/rdma_verbs.h>
#include <infiniband/arch.h>
#include <netdb.h>
#include <errno.h>
#include <string.h>
#include <stdio.h>
#include <stdlib.h>
#include <inttypes.h>
#include <unistd.h>
#include <time.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <sys/times.h>
#include <iostream>
#include <fstream>
#include <assert.h>
#include <sys/param.h>
#include <iomanip> // format manipulation
#include <vector>
using namespace std;
// header file
#include "header_RDMA.h"
//******************************************************
//********************** RDMA SERVER *******************
//******************************************************
int asServer(TestMessage &TM){
printf("\nRDMA server machine redigo-01 is starting. \n");
// Information from TCP socket
int BUF_SIZE = TM.data.message.tbuffer_size;
printf("BUF_SIZE = %i \n", BUF_SIZE);
int NTX=TM.data.message.loops;
printf("NTX = %i \n", NTX);
// Variables
static char *port = "7471";
static char *server = "100.168.0.56";
struct rdma_cm_id *listen_id, *id;
struct ibv_mr *mr;
char *send_msg=new char[BUF_SIZE];
char *recv_msg=new char[BUF_SIZE];
struct rdma_addrinfo hints, *res;
struct ibv_qp_init_attr attr;
struct ibv_wc wc;
int ret;// function return value
int i; // loop counter
// RDMA_getaddrinfo
memset(&hints, 0, sizeof hints);
hints.ai_flags = RAI_PASSIVE;
hints.ai_port_space = RDMA_PS_TCP;
ret = rdma_getaddrinfo(NULL, port, &hints, &res);
if (ret) {
printf("rdma_getaddrinfo %d\n", errno);
return ret;
}
else printf("OK rdma_getaddrinfo function. \n");
// RDMA_create_ep
memset(&attr, 0, sizeof attr);
attr.cap.max_send_wr = attr.cap.max_recv_wr = 128;
attr.cap.max_send_sge = attr.cap.max_recv_sge = 1;
attr.cap.max_inline_data = BUF_SIZE;
attr.sq_sig_all = 1;
ret = rdma_create_ep(&listen_id, res, NULL, &attr);
rdma_freeaddrinfo(res);
if (ret) {
printf("rdma_create_ep %d\n", errno);
return ret;
}
else printf("OK rdma_create_ep funtion. \n");
// RDMA_listen
ret = rdma_listen(listen_id, 0);
if (ret) {
printf("rdma_listen %d\n", errno);
return ret;
}
else printf("OK rdma_listen function. \n");
// RDMA_get_request
ret = rdma_get_request(listen_id, &id);
if (ret) {
printf("rdma_get_request %d\n", errno);
return ret;
}
else printf("OK rdma_get_request function \n");
// RDMA_reg_msgs
mr = rdma_reg_msgs(id, recv_msg, BUF_SIZE);
if (!mr) {
printf("rdma_reg_msgs %d\n", errno);
return ret;
}
else printf("OK rdma_reg_msgs function. \n");
// Pre post receive: RDMA_post_recv
ret = rdma_post_recv(id, NULL, recv_msg, BUF_SIZE, mr);
if (ret) {
printf("rdma_post_recv %d\n", errno);
return ret;
}
else printf("OK PRE POST rdma_post_recv. \n");
// RDMA_accept on server side
ret = rdma_accept(id, NULL);
if (ret) {
printf("rdma_connect %d\n", errno);
return ret;
}
else printf("OK rdma_connect: server accepts connection. \n");
//---------------------------------------------------------
//--------- SERVER RDMA: data movement --------------------
//---------------------------------------------------------
// Loop
for (i=0; i < NTX; i++){
// ********** Server is receiving ***************
// RDMA_post_recv
ret = rdma_post_recv(id, NULL, recv_msg, BUF_SIZE, mr);
if (ret) {
printf("rdma_post_recv %d\n", errno);
return ret;
}
//else printf("OK rdma_post_recv. \n");
// RDMA_get_recv_comp
/*
ret = rdma_get_recv_comp(id, &wc);
if (ret <= 0) {
printf("rdma_get_recv_comp %d\n", ret);
return ret;
}
*/
while (!ibv_poll_cq(id->recv_cq, 1, &wc))
;
//else printf("OK rdma_get_recv_comp %d\n", ret);
// ************ Server is sending ****************
// RDMA_post_send
ret = rdma_post_send(id, NULL, send_msg, BUF_SIZE, NULL,
IBV_SEND_INLINE);
if (ret) {
printf("rdma_post_send %d\n", errno);
return ret;
}
//else printf("OK rdma_post_send \n");
// RDMA_get_send_comp
/*
ret = rdma_get_send_comp(id, &wc);
if (ret <= 0) {
printf("rdma_get_send_comp %d\n", ret);
return ret;
}
*/
while (!ibv_poll_cq(id->send_cq, 1, &wc))
;
//else printf("OK rdma_get_send_comp %d\n", ret);
}// end loop
printf("Loop number = %i \n", i);
printf("RDMA server closes. \n \n");
rdma_disconnect(id);
rdma_dereg_mr(mr);
rdma_destroy_ep(id);
rdma_destroy_ep(listen_id);
}// end asServer
//******************************************************
//********************** CLIENT RDMA *******************
//******************************************************
int asClient(TestMessage &TM){
printf("\nRDMA client machine redigo-02 is starting. \n");
// Information from TCP socket
int BUF_SIZE=TM.data.message.tbuffer_size;
printf("BUF_SIZE = %i \n", BUF_SIZE);
int NTX=TM.data.message.loops;
printf("NTX = %i \n", NTX);
// Variables
static char *port = "7471";
static char *server = "10.10.10.1";
struct rdma_cm_id *listen_id, *id;
struct ibv_mr *mr;
char *send_msg=new char[BUF_SIZE];
char *recv_msg=new char[BUF_SIZE];
struct rdma_addrinfo hints, *res;
struct ibv_qp_init_attr attr;
struct ibv_wc wc;
int ret; // function return value
int i; // counter
time_t start,end;// clock values
double dif;// delta time
// RDMA_getaddrinfo
memset(&hints, 0, sizeof hints);
hints.ai_port_space = RDMA_PS_TCP;
ret = rdma_getaddrinfo(server, port, &hints, &res);
if (ret) {
printf("rdma_getaddrinfo %d\n", errno);
return ret;
}
else printf("OK rdma_getaddrinfo function. \n");
// RDMA_create_ep
memset(&attr, 0, sizeof attr);
attr.cap.max_send_wr = attr.cap.max_recv_wr = 128;
attr.cap.max_send_sge = attr.cap.max_recv_sge = 1;
attr.cap.max_inline_data = BUF_SIZE;
attr.qp_context = id;
attr.sq_sig_all = 1;
ret = rdma_create_ep(&id, res, NULL, &attr);
rdma_freeaddrinfo(res);
if (ret) {
printf("rdma_create_ep %d\n", errno);
return ret;
}
else printf("OK rdma_create_ep funtion. \n");
// RDMA_reg_msgs
mr = rdma_reg_msgs(id, recv_msg, BUF_SIZE);
if (!mr) {
printf("rdma_reg_msgs %d\n", errno);
return ret;
}
else printf("OK rdma_reg_msgs function. \n");
// Pre post receive: RDMA_post_recv
ret = rdma_post_recv(id, NULL, recv_msg, BUF_SIZE, mr);
if (ret) {
printf("rdma_post_recv %d\n", errno);
return ret;
}
else printf("OK PRE POST rdma_post_recv. \n");
// RDMA_connect on client side
ret = rdma_connect(id, NULL);
if (ret) {
printf("rdma_connect %d\n", errno);
return ret;
}
else printf("OK rdma_connect: client is connected. \n");
//----------------------------------------------------------
//--------- Clock starts -------------------------------------
//----------------------------------------------------------
time(&start);
//----------------------------------------------------------
//--------- CLIENT RDMA: data movement --------------------
//----------------------------------------------------------
// Loop
for(i = 0; i < NTX; i++){
// RDMA_post_recv
ret = rdma_post_recv(id, NULL, recv_msg, BUF_SIZE, mr);
if (ret) {
printf("rdma_post_recv %d\n", errno);
return ret;
}
//*************** Client is sending ***************************
// RDMA_post_send
ret = rdma_post_send(id, NULL, send_msg, BUF_SIZE, NULL,
IBV_SEND_INLINE);
if (ret) {
printf("rdma_post_send %d\n", errno);
return ret;
}
//else printf("OK rdma_post_send \n");
// RDMA_get_send_comp
/*
ret = rdma_get_send_comp(id, &wc);
if (ret <= 0) {
printf("rdma_get_send_comp %d\n", ret);
return ret;
}
*/
while (!ibv_poll_cq(id->send_cq, 1, &wc))
;
//else printf("OK rdma_get_send_comp %d\n", ret);
//********************** Client is receiving *******************
//else printf("OK rdma_post_recv \n");
// RDMA_get_recv_comp
/*
ret = rdma_get_recv_comp(id, &wc);
if (ret <= 0) {
printf("rdma_get_recv_comp %d\n", ret);
return ret;
}
*/
while (!ibv_poll_cq(id->recv_cq, 1, &wc))
;
//else printf("OK rdma_get_recv_comp \n");
}//end loop
//----------------------------------------------------------
//--------- Clock stops -------------------------------------
//----------------------------------------------------------
time(&end);
printf("Loop number = %i\n", i);
printf("RDMA client closes. \n \n");
rdma_disconnect(id);
rdma_dereg_mr(mr);
rdma_destroy_ep(id);
//------------------------------------------------------------------
//------- Output txt file with data --------------------------------
//------------------------------------------------------------------
ofstream OutFile("Test_Latency.txt");
// Open outfile test
if (! OutFile){
cout << "Error opening output txt file." << endl;
return -1;
}
// Calculation and other variables
dif = difftime(end,start); //delta time
long double NTXld = (long double) NTX;
long double BUF_SIZEld = (long double) BUF_SIZE;
long double dif_ld = (long double) dif;
long double speed = (NTXld*BUF_SIZEld)/(dif_ld);
long double latency = dif_ld/NTXld;
// Use setw(), setprecision(), scientific options if it is necessary
cout.setf(ios::left); // set option 'left justify output'
// Outfile format
OutFile <<
"#transfer|buffer_size(bytes)|total_time(s)|speed(bytes/s)|latency(s)| \n";
OutFile << setw(9) << NTX << "|";
OutFile << setw(18) << BUF_SIZE << "|";
OutFile << setw(13) << dif << "|";
OutFile << setw(14) << speed << "|";
OutFile << setw(10) << latency << "|";
OutFile << endl;
}// end asClient
//*******************************************************
//************* (fake) Main program *********************
//*******************************************************
// It does not work!
/*
int main(int narg, char **argv) {
if (narg > 1)
return asServer();
else
return asClient();
};
*/
REDIGO_Latency.xls
Description: MS-Excel spreadsheet
