On 06/22/05 09:43, Jeff Carr wrote: > On 06/21/2005 12:50 PM, Roland Dreier wrote: > > >>What happens if you try replacing the send_flags line with the one you >>have commented out? >> >>+ // send_wr.send_flags = IB_SEND_SIGNALED; > > > Thanks, you are correct. IB_SEND_SIGNALED gives me the behavior I was > expecting.
Here is an updated version and a simple perl script that tests it's performance. With 2K messages, these were the performance numbers between 2 systems (3.6ghz Xeon/w 133mhz/64bit pci). [EMAIL PROTECTED]:~# ./fast_test.pl 20 starting sends 0 messages/sec (0 Mb/sec) 131072 messages/sec (2047 Mb/sec) 131072 messages/sec (2047 Mb/sec) 131072 messages/sec (2047 Mb/sec) 174762 messages/sec (2730 Mb/sec) 163840 messages/sec (2559 Mb/sec) 196608 messages/sec (3071 Mb/sec) 183500 messages/sec (2867 Mb/sec) 174762 messages/sec (2730 Mb/sec) 196618 messages/sec (3072 Mb/sec) 187254 messages/sec (2925 Mb/sec) 180232 messages/sec (2816 Mb/sec) 196616 messages/sec (3072 Mb/sec) 189333 messages/sec (2958 Mb/sec) 183507 messages/sec (2867 Mb/sec) 196614 messages/sec (3072 Mb/sec) 190656 messages/sec (2978 Mb/sec) 202571 messages/sec (3165 Mb/sec) 138785 messages/sec (2168 Mb/sec) 131075 messages/sec (2048 Mb/sec) Jeff
config INFINIBAND_SPTS
tristate "A Simple Page Transfer Scheme over InfiniBand (SPiTS)"
depends on INFINIBAND
---help---
All this does is let you establish a simple connection
between two infiniband hosts so you can do 2 way
data transfers.
EXTRA_CFLAGS += -Idrivers/infiniband/include
obj-$(CONFIG_INFINIBAND_SPTS) += ib_spts.o
ib_spts-y := spts.o
obj-$(CONFIG_INFINIBAND_SPTS) += ib_cm_spts.o
ib_cm_spts-y := cm_spts.o
obj-$(CONFIG_INFINIBAND_SPTS) += ib_cm_spts_client.o
ib_cm_spts_client-y := client_start.o
obj-$(CONFIG_INFINIBAND_SPTS) += spts_fast.o
spts_fast-y := fast.o
all:
make -C $(BUILDDIR) SUBDIRS=$(PWD) modules
clean:
rm -rf .tmp_versions
rm -f *.o *.ko .*.o.cmd .*.o.d .*.ko.cmd *.mod.c
mytest:
/root/bin/make_and_copy_modules.pl
./restart_spts.pl
/*
* Copyright (c) 2005 Linux Machines Inc. All rights reserved.
*
* This software is available to you under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree.
*
*/
#include <linux/module.h>
#include <linux/moduleparam.h>
#include "spts.h"
MODULE_AUTHOR("Jeff Carr");
MODULE_DESCRIPTION("Trigger a client connection");
MODULE_LICENSE("GPL");
void ib_cm_spts_conn_client(int slid, int dlid);
static int slid = 0;
static int dlid = 0;
module_param(slid, int, 0444);
module_param(dlid, int, 0444);
MODULE_PARM_DESC(slid, "Source LID to use for connection.");
MODULE_PARM_DESC(dlid, "Destination LID to use for connection.");
static int __init client_start_init(void)
{
/* comment out for now for other tests... */
printk("client_start_init() slid = %d, dlid = %d\n", slid, dlid);
ib_cm_spts_conn_client(slid, dlid);
return 0;
}
static void __exit client_start_cleanup(void)
{
return;
}
module_init(client_start_init);
module_exit(client_start_cleanup);
/*
* A Simple Page Transfer Scheme "SPTS" listener
*
*
* Copyright (c) 2005 Linux Machines Inc. All rights reserved.
* Copyright (c) 2004 Intel Corporation. All rights reserved.
*
* This software is available to you under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree.
*
*
* This self contained kernel module will listen, establish,
* destroy and manage connections.
*
* I wrote this module so I could tuck away all the details
* of making and watching connections somewhere from SPiTS.
*
* I tried to make this simple and generic enough that it might
* be a good reference point for anyone else that might be
* in the same Infiniband boat.
*
* -- Jeff Carr <[EMAIL PROTECTED]>
*
*/
#include <asm/semaphore.h>
#include <linux/device.h>
#include <linux/err.h>
#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/pci.h>
#include <linux/time.h>
#include <linux/workqueue.h>
#include <asm/uaccess.h>
#include <linux/delay.h>
#include <ib_verbs.h>
#include <ib_cm.h>
#include "spts.h"
MODULE_AUTHOR("Jeff Carr");
MODULE_DESCRIPTION("A Simple CM listener intended for use with SPTS");
MODULE_LICENSE("GPL");
DECLARE_MUTEX(ib_cm_spts_lock);
static void ib_cm_spts_add_one (struct ib_device *device);
static void ib_cm_spts_remove_one (struct ib_device *device);
static struct ib_client ib_cm_spts_client = {
.name = "ib_cm_spts",
.add = ib_cm_spts_add_one,
.remove = ib_cm_spts_remove_one
};
struct cm_spts {
struct workqueue_struct *wq;
struct work_struct work;
atomic_t connects_left;
atomic_t disconnects_left;
struct semaphore sem;
wait_queue_head_t wait;
};
static struct cm_spts test;
struct ib_cm_id *listen_id;
struct ib_device *mydevice;
static int port_num = 1;
ib_cm_conn_init1 conn_init1 = NULL;
ib_cm_conn_init2 conn_init2 = NULL;
/*
* generic routine needed by the CM and used while a connection
* is established.
*/
int ib_cm_spts_modify_to_rtr(struct ib_cm_id *cm_id, struct ib_cm_event *event)
{
struct ib_qp_attr qp_attr;
int qp_attr_mask, ret;
struct spts_ib_connection *node = cm_id->context;
qp_attr.qp_state = IB_QPS_INIT;
ret = ib_cm_init_qp_attr(cm_id, &qp_attr, &qp_attr_mask);
if (ret) {
printk("ib_cm_spts: failed to init QP attr for INIT: %d\n", ret);
return ret;
}
ret = ib_modify_qp(node->qp, &qp_attr, qp_attr_mask);
if (ret) {
printk("ib_cm_spts: failed to modify QP to INIT: %d\n", ret);
return ret;
}
qp_attr.qp_state = IB_QPS_RTR;
ret = ib_cm_init_qp_attr(cm_id, &qp_attr, &qp_attr_mask);
if (ret) {
printk("ib_cm_spts: failed to init QP attr for RTR: %d\n", ret);
return ret;
}
qp_attr.rq_psn = node->qp->qp_num;
ret = ib_modify_qp(node->qp, &qp_attr, qp_attr_mask);
if (ret) {
printk("ib_cm_spts: failed to modify QP to RTR: %d\n", ret);
return ret;
}
return 0;
}
/*
* generic routine needed by the CM and used while a connection
* is established.
*/
int ib_cm_spts_modify_to_rts(struct ib_cm_id *cm_id, struct ib_cm_event *event)
{
struct ib_qp_attr qp_attr;
int qp_attr_mask, ret;
struct spts_ib_connection *node = cm_id->context;
qp_attr.qp_state = IB_QPS_RTS;
ret = ib_cm_init_qp_attr(cm_id, &qp_attr, &qp_attr_mask);
if (ret) {
printk("ib_cm_spts: failed to init QP attr for RTS: %d\n", ret);
return ret;
}
ret = ib_modify_qp(node->qp, &qp_attr, qp_attr_mask);
if (ret) {
printk("ib_cm_spts: failed to modify QP to RTS: %d\n", ret);
return ret;
}
return 0;
}
// if context == NULL, then we know this is a new cm_id and we should initialize it.
int ib_cm_init1( struct ib_cm_id *cm_id )
{
struct spts_ib_connection *anode = cm_id->context;
DEBUGK("conn_init1: START cm_id == 0x%08X\n", (int) cm_id);
DEBUGK("conn_init1: START cm_id->context == 0x%08X\n", (int) cm_id->context);
if (cm_id->context) {
DEBUGK("conn_init1: already is initialized\n");
return 0;
}
if (!conn_init1) {
printk("conn_init1: no one registered an init function yet!\n");
return -1;
}
anode = conn_init1(mydevice);
if (!anode) {
DEBUGK("conn_init1: AHHHHH! anode is still NULL\n");
return -1;
}
cm_id->context = anode;
anode->cm_id = cm_id;
DEBUGK("conn_init1: END cm_id == 0x%08X\n", (int) cm_id);
DEBUGK("conn_init1: END cm_id->context == 0x%08X\n", (int) cm_id->context);
return 0;
}
/*
* This is what runs on the client side of a connection after the server responds
*/
static void ib_cm_spts_rep_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
{
struct spts_ib_connection *anode = cm_id->context;
int ret;
ret = ib_cm_spts_modify_to_rtr(cm_id, event);
if (ret)
goto fail;
ret = ib_cm_spts_modify_to_rts(cm_id, event);
if (ret)
goto fail;
if (!conn_init2) {
printk("ib_cm_spts: no one registered an init2 function yet!\n");
goto fail;
}
ret = conn_init2(anode);
if (ret)
goto fail;
ret = ib_send_cm_rtu(cm_id, NULL, 0);
if (ret) {
printk("ib_cm_spts: failed to send CM RTU: %d\n", ret);
goto fail;
}
return;
fail:
printk("ib_cm_spts_rep_handler() failing connection reply\n");
ib_send_cm_rej(cm_id, IB_CM_REJ_UNSUPPORTED, NULL, 0, NULL, 0);
}
/*
* This is what runs on the listening side to establish a connection.
*/
static void ib_cm_spts_req_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
{
struct spts_ib_connection *anode = cm_id->context;
struct ib_cm_req_event_param *req;
struct ib_cm_rep_param rep;
int ret;
DEBUGK("ib_cm_spts_req_handler() START\n");
ret = ib_cm_spts_modify_to_rtr(cm_id, event);
if (ret)
goto fail;
DEBUGK("ib_cm_spts_req_handler() about to conn_init2()\n");
if (!conn_init2) {
printk("ib_cm_spts: no one registered an init2 function yet!\n");
goto fail;
}
ret = conn_init2(anode);
if (ret)
goto fail;
req = &event->param.req_rcvd;
memset(&rep, 0, sizeof rep);
rep.qp_num = anode->qp->qp_num;
rep.srq = (anode->qp->srq != NULL);
rep.starting_psn = anode->qp->qp_num;
rep.responder_resources = req->responder_resources;
rep.initiator_depth = req->initiator_depth;
rep.target_ack_delay = 20;
rep.flow_control = req->flow_control;
rep.rnr_retry_count = req->rnr_retry_count;
DEBUGK("ib_cm_spts_req_handler() about to ib_send_cm_rep()\n");
ret = ib_send_cm_rep(cm_id, &rep);
if (ret) {
printk("ib_cm_spts_req_handler() failed to send CM REP: %d\n", ret);
goto fail;
}
DEBUGK("ib_cm_spts_req_handler done\n");
return;
fail:
printk("ib_cm_spts_req_handler() failing connection request\n");
ib_send_cm_rej(cm_id, IB_CM_REJ_UNSUPPORTED, NULL, 0, NULL, 0);
// don't take this out
if (atomic_dec_and_test(&test.connects_left))
wake_up(&test.wait);
}
/*
* This routine is the generic routine registered with the Infiniband
* Communication Manager. This routine is called each time an event
* occurs and any connection.
*/
static int ib_cm_spts_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
{
// check to see there is already a connection for this cm_id
// If not, then this makes a new one.
if(ib_cm_init1(cm_id)) {
printk("ib_cm_init1() FAILED\n");
printk("event = 0x%08X\n", (int) event->event);
return -1;
}
switch (event->event) {
case IB_CM_REQ_RECEIVED:
printk("ib_cm_spts: IB_CM_REQ_RECEIVED\n");
ib_cm_spts_req_handler(cm_id, event);
break;
case IB_CM_REP_RECEIVED:
printk("ib_cm_spts: IB_CM_REP_RECEIVED\n");
ib_cm_spts_rep_handler(cm_id, event);
break;
case IB_CM_RTU_RECEIVED:
printk("ib_cm_spts: IB_CM_RTU_RECEIVED\n");
ib_cm_spts_modify_to_rts(cm_id, event);
break;
case IB_CM_DREQ_RECEIVED:
printk("ib_cm_spts: IB_CM_DREQ_RECEIVED\n");
ib_send_cm_drep(cm_id, NULL, 0);
break;
case IB_CM_DREP_RECEIVED:
printk("ib_cm_spts: UNHANDLED IB_CM_DREP_RECEIVED\n");
break;
case IB_CM_TIMEWAIT_EXIT:
printk("ib_cm_spts: UNHANDLED IB_CM_TIMEWAIT_EXIT\n");
break;
case IB_CM_REQ_ERROR:
printk("ib_cm_spts: UNHANDLED IB_CM_REQ_ERROR\n");
break;
default:
printk("ib_cm_spts: UNKNOWN event = 0x%08X\n", (int) event->event);
break;
}
return 0;
}
static void ib_cm_spts_query_handler(int status, struct ib_sa_path_rec *resp,
void *context)
{
struct spts_ib_connection *anode = context;
if (!status)
anode->path_rec = *resp;
anode->query_status = status;
wake_up(&test.wait);
}
static int ib_cm_spts_query_for_path(
struct spts_ib_connection *anode, int slid, int dlid)
{
int ret, retries = 0;
struct ib_sa_query *query;
anode->path_rec.dlid = cpu_to_be16(dlid);
anode->path_rec.slid = cpu_to_be16(slid);
anode->path_rec.numb_path = 1;
do {
anode->query_status = 1;
ret = ib_sa_path_rec_get(anode->device, port_num, &anode->path_rec,
IB_SA_PATH_REC_DLID | IB_SA_PATH_REC_SLID |
IB_SA_PATH_REC_NUMB_PATH, 5000,
GFP_KERNEL, ib_cm_spts_query_handler,
anode, &query);
if (ret < 0) {
printk("ib_cm_spts: ib_sa_path_rec_get failed: %d\n", ret);
goto out;
}
wait_event(test.wait, anode->query_status <= 0);
ret = anode->query_status;
retries++;
} while (ret == -ETIMEDOUT && retries <= 3);
out:
return ret;
}
/*
* This should be changed to return useful values
*
*/
void ib_cm_spts_conn_client(int slid, int dlid)
{
struct spts_ib_connection *anode = NULL;
struct ib_cm_req_param req;
struct ib_cm_id *cm_id;
int ret;
DEBUGK("ib_cm_spts_conn_client() START\n");
cm_id = ib_create_cm_id(ib_cm_spts_handler, NULL);
if (IS_ERR(cm_id)) {
ret = PTR_ERR(cm_id);
printk("ib_cm_spts: failed to create cm_id: %d\n", ret);
return;
}
if(ib_cm_init1(cm_id)) {
printk("ib_cm_init1() FAILED\n");
return;
}
anode = cm_id->context;
DEBUGK("ib_cm_spts_conn_client() about to query_for_path\n");
ret = ib_cm_spts_query_for_path(anode, slid, dlid);
if (ret) {
printk("ib_cm_spts: failed path record query: %d\n", ret);
return;
}
memset(&req, 0, sizeof req);
req.primary_path = &anode->path_rec;
req.service_id = LISTEN_NUM;
req.responder_resources = 1;
req.initiator_depth = 1;
req.remote_cm_response_timeout = 20;
req.local_cm_response_timeout = 20;
req.retry_count = 5;
req.max_cm_retries = 5;
req.qp_num = anode->qp->qp_num;
req.qp_type = anode->qp->qp_type;
req.srq = (anode->qp->srq != NULL);
req.starting_psn = anode->qp->qp_num;
ret = ib_send_cm_req(anode->cm_id, &req);
if (ret) {
printk("ib_cm_spts_conn_client() failure sending REQ: %d\n", ret);
return;
}
DEBUGK("ib_cm_spts_conn_client END\n");
}
EXPORT_SYMBOL(ib_cm_spts_conn_client);
static void ib_cm_spts_listen(void *data)
{
int ret;
DEBUGK("listen() START\n");
down(&ib_cm_spts_lock);
DEBUGK("listen() after down(&ib_cm_spts_lock)\n");
listen_id = ib_create_cm_id(ib_cm_spts_handler, &test);
if (IS_ERR(listen_id)) {
ret = PTR_ERR(listen_id);
printk("listen() WATCH: listen request failed: %d\n", ret);
return;
}
DEBUGK("listen() listen_id == 0x%08X\n", (int) listen_id);
DEBUGK("listen() listen_id->context == 0x%08X\n", (int) listen_id->context);
listen_id->context = NULL;
DEBUGK("listen() START listen_id == 0x%08X\n", (int) listen_id);
DEBUGK("listen() listen_id->context == 0x%08X\n", (int) listen_id->context);
DEBUGK("listen() about to ib_cm_listen()\n");
ret = ib_cm_listen(listen_id, LISTEN_NUM, 0);
DEBUGK("listen() finished ib_cm_listen() ret == %d\n", ret);
if (ret) {
printk("listen() WATCH: failure trying to listen: %d\n", ret);
}
DEBUGK("listen() END\n");
}
static void ib_cm_spts_add_one(struct ib_device *device)
{
printk("ib_cm_spts_add_one() START\n");
mydevice = device;
printk("ib_cm_spts_add_one() ib_set_client_data\n");
ib_set_client_data(device, &ib_cm_spts_client, &test);
up(&ib_cm_spts_lock);
printk("ib_cm_spts_add_one() END\n");
}
static void ib_cm_spts_remove_one(struct ib_device *device)
{
printk("ib_cm_spts_remove_one START\n");
// figure out what this really does
if (ib_get_client_data(device, &ib_cm_spts_client) != &test)
return;
printk("ib_cm_spts_remove_one END\n");
}
static void ib_cm_spts_start_listen(void)
{
DEBUGK("ib_cm_spts_start_listen() START\n");
test.wq = create_workqueue("ib_cm_spts");
if (!test.wq)
return;
DEBUGK("ib_cm_spts_start_listen() INIT_WORK(ib_cm_spts_listen)\n");
INIT_WORK(&test.work, ib_cm_spts_listen, &test);
init_waitqueue_head(&test.wait);
DEBUGK("ib_cm_spts_start_listen() about to queue work\n");
queue_work(test.wq, &test.work);
DEBUGK("ib_cm_spts_start_listen() FINISHED\n");
}
/*
* use this to register the initialization functions to
* establish new connections
*
* init1() should create the qp & cq's.
* init2() should do ib_post_recv()
*
*/
int ib_cm_reg_init_conn( ib_cm_conn_init1 init1, ib_cm_conn_init2 init2 )
{
conn_init1 = init1;
conn_init2 = init2;
return 0;
}
EXPORT_SYMBOL(ib_cm_reg_init_conn);
static int __init ib_cm_spts_init(void)
{
listen_id = NULL;
memset(&test, 0, sizeof test);
down(&ib_cm_spts_lock);
ib_cm_spts_start_listen();
DEBUGK("ib_cm_spts_init() starting ib_cm_spts_init()\n");
ib_register_client(&ib_cm_spts_client);
return 0;
}
static void __exit ib_cm_spts_cleanup(void)
{
wake_up(&test.wait);
flush_workqueue(test.wq);
destroy_workqueue(test.wq);
ib_destroy_cm_id(listen_id);
ib_unregister_client(&ib_cm_spts_client);
}
module_init(ib_cm_spts_init);
module_exit(ib_cm_spts_cleanup);
/*
* A fast test of spts.
*
* The same code for each node.
*
*/
/*
* Copyright (c) 2005 Linux Machines. All rights reserved.
*
* This software is available to you under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree.
*
* Jeff Carr <[EMAIL PROTECTED]>
*
*/
#include <asm/semaphore.h>
#include <linux/device.h>
#include <linux/err.h>
#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/pci.h>
#include <linux/time.h>
#include <linux/workqueue.h>
#include <linux/proc_fs.h>
#include <ib_cm.h>
#include "spts.h"
MODULE_AUTHOR("Jeff Carr");
MODULE_DESCRIPTION("SPTS proc tester");
MODULE_LICENSE("GPL");
struct proc_dir_entry *proc_fast_dir = NULL;
int mysendcounter = 0;
int totalrecv = 0;
int recvwatcher = 0;
int totalsent = 0;
int status = 0;
static int hostval = 1234;
module_param(hostval, int, 0444);
MODULE_PARM_DESC(hostval, "val for this host");
/*
* smart simple routine taken from /fs/proc/proc_misc.c
*/
int fast_proc_calc_metrics(char *page, char **start, off_t off,
int count, int *eof, int len)
{
if (len <= off+count) *eof = 1;
*start = page + off;
len -= off;
if (len>count) len = count;
if (len<0) len = 0;
return len;
}
/* cat /proc/fast/test to test IB send & receive */
int fast_proc_poll(char *page, char **start, off_t off,
int count, int *eof, void *data)
{
int num;
u32 len;
struct spts_msg *received_msgs;
struct spts_simple_msg *wrap;
DEBUGK("fast_proc_poll() START HERE\n");
num = spts_poll_one( &received_msgs);
if(num > 0) {
wrap = (struct spts_simple_msg *) received_msgs[0].mem;
len = wrap->size;
if( (len > 0) && (len < 0x20) ) {
printk("fast_proc_poll() num == %d, msg == %s\n", num, (char *) received_msgs[0].mem);
} else {
spts_dump_page( &received_msgs[0] );
}
} else if(num < 0) {
printk("fast_proc_poll() An error occurred in the communication\n");
} else {
printk("fast_proc_poll() RECV queue empty\n");
}
DEBUGK("fast_proc_poll() END HERE\n");
fast_proc_calc_metrics( page, start, off, count, eof, 0);
return 0;
}
int fast_poll_one(void)
{
int num;
int *dump;
struct spts_msg *received_msgs;
struct spts_simple_msg *wrap;
u32 len;
num = spts_poll_all( &received_msgs);
if(num > 0) {
recvwatcher = 0;
wrap = (struct spts_simple_msg *) received_msgs[0].mem;
len = wrap->size;
if( status > 0x8000 ) {
if( (len > 0) && (len < 0x20) ) {
// printk("fast_proc_test() num == %d, msg == %s\n", num, (char *) received_msgs[0].mem);
// printk("fast_proc_test() num == %d, msgnum == 0x%08X\n", num, (u32 *) received_msgs[0].mem);
printk("totalrecv=%d, totalsent=%d\n", totalrecv, totalsent);
dump = (uint *) received_msgs[0].mem;
printk("from host 0x%08X recv message: %d 0x%08X 0x%08X\n", dump[0], dump[1], dump[2], dump[3]);
}
status = 0;
}
} else if(num < 0) {
printk("fast_proc_test() An error occurred in the communication\n");
} else {
DEBUGK("fast_proc_test() RECV queue empty\n");
}
totalrecv += num;
status += num;
return num;
}
/* cat /proc/fast/test to test IB send & receive */
int fast_proc_test(char *page, char **start, off_t off,
int count, int *eof, void *data)
{
int j, ret;
char mybuf[255];
int runcount = 0;
uint *dump;
DEBUGK("fast_proc_test() START HERE\n");
memset( mybuf, 0x0, 0x20);
dump = (uint *) mybuf;
// for(i=0; i< 0x10000; ++i ) {
while(runcount < 0x10000) {
// while( fast_poll_one() > 0 ) {}
ret = fast_poll_one();
if( ret < 0x8 ) {
schedule();
}
if( recvwatcher > 0x200 ) {
// printk("recvwatcher == %d. not sending now\n", recvwatcher);
} else {
if( (totalsent - totalrecv) > 0x300 ) {
// hold off for a while before sending more
} else {
// might be risky to go over 0x80
for( j = 0; j < 0x20 ; ++j ) {
sprintf( mybuf, "hello %d", mysendcounter );
// mybuf[0] = mysendcounter;
dump[0] = 0x12345678;
dump[1] = mysendcounter;
dump[2] = hostval;
dump[3] = 0x0;
++mysendcounter;
// len = strlen(mybuf);
spts_send_all( mybuf, 0x10);
++totalsent;
++recvwatcher;
++runcount;
}
}
}
}
DEBUGK("fast_proc_test() END HERE totalrecv=%d, totalsent=%d\n", totalrecv, totalsent);
fast_proc_calc_metrics( page, start, off, count, eof, 0);
return 0;
}
/* cat /proc/fast/test to test IB send & receive */
int fast_proc_trigger1(char *page, char **start, off_t off,
int count, int *eof, void *data)
{
return fast_proc_calc_metrics( page, start, off, count, eof, 0);
}
static int __init fast_proc_init(void)
{
struct proc_dir_entry *proc_ent;
/* Create /proc/fast directory */
proc_ent = create_proc_entry("fast", S_IFDIR, &proc_root);
if (proc_ent) {
proc_fast_dir = proc_ent;
create_proc_read_entry("fast/poll", 0, NULL,
fast_proc_poll, NULL);
create_proc_read_entry("fast/test", 0, NULL,
fast_proc_test, NULL);
} else {
printk("Could not create \"/proc/fast/test\" entry\n");
}
return 0;
}
/*
* Need to check to make sure this is destroying everything correctly
* What else needs to be cleaned up & freed on exit?
*/
static void __exit fast_proc_cleanup(void)
{
if (proc_fast_dir) {
printk("cleaning up /proc/fast\n");
/* remove these in (quasi) reverse order just to be safe */
remove_proc_entry("fast/poll", NULL);
remove_proc_entry("fast/test", NULL);
/* remove the /proc/fast directory itself */
remove_proc_entry("fast", &proc_root);
}
}
module_init(fast_proc_init);
module_exit(fast_proc_cleanup);
fast_test.pl
Description: Perl program
restart_spts.pl
Description: Perl program
/*
*
* SPTS - A Simple Page Transfer Scheme (spits messages between smart hosts)
*
* Copyright (c) 2005 Linux Machines. All rights reserved.
*
* This software is available to you under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree.
*
*
* It does what it says: it lets you send messages between machines
* on an Infiniband network.
*
* It uses/requires Sean Hefty's Infiniband Connection Manager (ib_cm.ko)
* to establish connections.
*
* Jeff Carr <[EMAIL PROTECTED]>
*
*/
#include <asm/semaphore.h>
#include <linux/device.h>
#include <linux/err.h>
#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/pci.h>
#include <linux/time.h>
#include <linux/workqueue.h>
#include <ib_cm.h>
#include "spts.h"
// #include "spts_proc_test.h"
MODULE_AUTHOR("Jeff Carr");
MODULE_DESCRIPTION("SPTS - A Simple Page Tranfser Scheme");
MODULE_LICENSE("GPL");
struct spts_msg smsg[SPTS_MSG_COUNT]; // send buffers
struct spts_msg rmsg[SPTS_MSG_COUNT]; // recieve buffers
struct spts_ib_connection conn[SPTS_MAX_CONNECTIONS];
// struct ib_send_wr send_wr, *bad_send_wr;
// struct ib_send_wr send_wr, *send_wr_failure;
struct ib_sge sge;
struct ib_wc spts_wc[SPTS_MAX_QUEUE_POLL];
struct spts_msg *cur_msgs;
int cur_msg_count = 0;
/*
* A event handler routine that can be registered
* with infiniband to find out about certain events.
*/
void spts_event_handler(struct ib_event *event, void *data) {
printk("WATCH spts_event_handler()\n");
}
EXPORT_SYMBOL(spts_event_handler);
/*
* A completion handler routine that can be registered
* with infiniband to find out when transfers complete.
*/
void spts_comp_handler(struct ib_cq *cq, void *cq_context)
{
printk("WATCH spts_comp_handler()\n");
}
EXPORT_SYMBOL(spts_comp_handler);
/*
* Generate a single QP recieve entry
* <insert better description here>
*/
int spts_post_recv(struct ib_qp *qp, struct spts_msg *msg )
{
struct ib_recv_wr recv_wr, *recv_failure;
struct ib_sge sge;
int ret = 0;
recv_wr.next = NULL;
recv_wr.sg_list = &sge;
recv_wr.num_sge = 1;
// recv_wr.wr_id = &conn[0]; // always 0 for now. This might correlate to which connection is which
recv_wr.wr_id = 0;
// this is the actual size of the recv buffer in bytes
// sge.length = SPTS_MSG_SIZE / 4;
sge.length = SPTS_MSG_SIZE;
sge.lkey = msg->mr->lkey;
sge.addr = msg->addr;
ret = ib_post_recv(qp, &recv_wr, &recv_failure);
if (ret) {
printk("spts_post_recv() failed ib_post_recv == %d\n", ret);
}
// DEBUGK("single_post_recv() (aka ib_post_recv) finished\n");
return ret;
}
EXPORT_SYMBOL(spts_post_recv);
/*
* Sets up SPTS_IB_CQ_SIZE number of buffers to recieve
* messages into.
*/
int spts_post_recvs(struct spts_ib_connection *node)
{
int i, j = 0;
DEBUGK("spts_post_recvs() START\n");
// for (i = 0; i < (SPTS_IB_CQ_SIZE / 2); i++ ) {
for (i = 0; i < SPTS_MSG_COUNT; i++ ) {
if( spts_post_recv( node->qp, &rmsg[j] ))
return -1;
++j;
if(j == SPTS_MSG_COUNT)
j=0;
}
node->status = 1;
DEBUGK("spts_post_recvs() END node->status is set now\n");
return 0;
}
EXPORT_SYMBOL(spts_post_recvs);
/*
* Creates the messages for communication between 2 IB cards once
* a connection is established.
*/
int spts_create_messages(struct ib_device *device,
struct ib_pd *pd, struct spts_msg *msgs)
{
int i, ret;
for( i = 0; i < SPTS_MSG_COUNT; ++i ) {
msgs[i].mr = ib_get_dma_mr(pd, IB_ACCESS_LOCAL_WRITE);
if (IS_ERR(msgs[i].mr)) {
kfree(msgs[i].mem);
ret = PTR_ERR(msgs[i].mr);
printk("spts_create_messages() failed to get DMA MR: %d\n", ret);
return ret;
}
msgs[i].addr = dma_map_single(device->dma_device,
msgs[i].mem, SPTS_MSG_SIZE, DMA_TO_DEVICE);
pci_unmap_addr_set(msgs[i], mapping, msgs[i].addr);
}
DEBUGK("spts_create_messages() END\n");
return 0;
}
EXPORT_SYMBOL(spts_create_messages);
/*
* This raw primitive tells IB to transfer a buffer to a pre-existing
* and established connection to another IB card. (sometimes called RDMA)
*/
int spts_send_single_msg( int dest, struct spts_msg *msg)
{
struct ib_send_wr send_wr, *send_wr_bad;
struct ib_sge sge;
if(conn[dest].status == 0) {
printk("WARNING: this connection was never initialized\n");
return -1;
}
send_wr.next = NULL;
send_wr.sg_list = &sge;
send_wr.num_sge = 1;
send_wr.opcode = IB_WR_SEND;
// send_wr.send_flags = 0;
send_wr.send_flags = IB_SEND_SIGNALED;
// send_wr.send_flags = IB_SEND_SIGNALED | IB_SEND_SOLICITED;
// send_wr.send_flags = IB_SEND_SOLICITED;
send_wr.wr_id = 0;
// this is the actual size of the recv buffer in bytes
// sge.length = SPTS_MSG_SIZE / 4;
sge.length = SPTS_MSG_SIZE;
sge.addr = msg->addr;
sge.lkey = msg->mr->lkey;
return ib_post_send(conn[dest].qp, &send_wr, &send_wr_bad);
}
/*
* Useful for debugging, this just dumps the contents of a message
*/
void spts_dump_page( struct spts_msg *msg )
{
int i = 0;
int *testmem;
testmem = (int *) msg->mem;
DEBUGK("0x%08X 0x%08X 0x%08X 0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n",
testmem[i+0], testmem[i+1], testmem[i+2], testmem[i+3],
testmem[i+4], testmem[i+5], testmem[i+6], testmem[i+7]);
i+=8;
DEBUGK("0x%08X 0x%08X 0x%08X 0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n",
testmem[i+0], testmem[i+1], testmem[i+2], testmem[i+3],
testmem[i+4], testmem[i+5], testmem[i+6], testmem[i+7]);
i = (SPTS_MSG_SIZE / 4) - 8;
DEBUGK("0x%08X 0x%08X 0x%08X 0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n",
testmem[i+0], testmem[i+1], testmem[i+2], testmem[i+3],
testmem[i+4], testmem[i+5], testmem[i+6], testmem[i+7]);
i+=8;
DEBUGK("0x%08X 0x%08X 0x%08X 0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n",
testmem[i+0], testmem[i+1], testmem[i+2], testmem[i+3],
testmem[i+4], testmem[i+5], testmem[i+6], testmem[i+7]);
}
EXPORT_SYMBOL(spts_dump_page);
int spts_init_node(struct spts_ib_connection *node)
{
struct ib_qp_init_attr qp_attr;
int ret;
node->recv_cq = ib_create_cq(node->device, spts_comp_handler,
spts_event_handler, node, SPTS_IB_CQ_SIZE);
if (IS_ERR(node->recv_cq)) {
ret = PTR_ERR(node->recv_cq);
printk("spts_init_node() unable to create CQ: %d\n", ret);
goto out;
}
DEBUGK("spts_init_node recv_cq ib_create_cq()\n");
node->send_cq = ib_create_cq(node->device, spts_comp_handler,
spts_event_handler, node, SPTS_IB_CQ_SIZE);
if (IS_ERR(node->send_cq)) {
ret = PTR_ERR(node->send_cq);
printk("spts_init_node() unable to create CQ: %d\n", ret);
goto out;
}
DEBUGK("spts_init_node ib_create_send_cq()\n");
memset(&qp_attr, 0, sizeof qp_attr);
qp_attr.event_handler = spts_event_handler;
qp_attr.cap.max_send_wr = SPTS_IB_CQ_SIZE;
qp_attr.cap.max_recv_wr = SPTS_IB_CQ_SIZE;
qp_attr.cap.max_send_sge = 1;
qp_attr.cap.max_recv_sge = 1;
qp_attr.sq_sig_type = IB_SIGNAL_REQ_WR;
qp_attr.qp_type = IB_QPT_RC;
qp_attr.send_cq = node->send_cq;
qp_attr.recv_cq = node->recv_cq;
node->qp = ib_create_qp(node->pd, &qp_attr);
if (IS_ERR(node->qp)) {
ret = PTR_ERR(node->qp);
printk("pts_init_node() unable to create QP: %d\n", ret);
ib_destroy_cq(node->recv_cq);
ib_destroy_cq(node->send_cq);
goto out;
}
// ib_req_notify_cq(node->cq, 3);
DEBUGK("spts_init_node ib_create_qp()\n");
return 0;
out:
spts_destroy_node(node);
return ret;
}
EXPORT_SYMBOL(spts_init_node);
struct spts_ib_connection *spts_init_conn(struct ib_device *device)
{
int ret;
int i = 0;
DEBUGK("spts_init_conn() START i == %d\n", i);
for( i=0; i<SPTS_MAX_CONNECTIONS; ++i ) {
if( conn[i].status )
continue;
conn[i].device = device;
if( conn[0].pd ) {
DEBUGK("spts_init_conn() just use the already allocated PD\n");
conn[i].pd = conn[0].pd;
} else {
conn[i].pd = ib_alloc_pd(conn[i].device);
DEBUGK("spts_init_conn() ib_alloc_pd() ok\n");
if (IS_ERR(conn[i].pd)) {
ret = PTR_ERR(conn[i].pd);
printk("spts_init_conn() failed to alloc PD: %d\n", ret);
ib_dealloc_pd(conn[i].pd);
return NULL;
}
if( spts_create_messages(conn[i].device, conn[i].pd, rmsg) )
return NULL;
if( spts_create_messages(conn[i].device, conn[i].pd, smsg) )
return NULL;
}
ret = spts_init_node(&conn[i]);
if (ret) {
printk("spts_init_conn() unable to create test nodes: %d\n", ret);
return NULL;
}
DEBUGK("spts_init_conn() END\n");
return &conn[i];
}
return NULL;
}
EXPORT_SYMBOL(spts_init_conn);
/*
* Not sure if this is all that is needed to destroy a "node"
*/
void spts_destroy_node(struct spts_ib_connection *node)
{
DEBUGK("spts_destroy_node()\n");
if (!node) {
DEBUGK("BAD spts_destroy_node node == NULL\n");
return;
}
if (node->qp) {
ib_destroy_qp(node->qp);
}
if (node->send_cq) {
ib_destroy_cq(node->send_cq);
}
if (node->recv_cq) {
ib_destroy_cq(node->recv_cq);
}
}
EXPORT_SYMBOL(spts_destroy_node);
/*
* spts_send_single()
*
* This function sends a message over a particular QP.
* It returns len if the transfer completes correctly.
*
* dest the connection to send the buffer over
* char *buf the memory address of the buffer
* int len the length of the data to send
*
* TODO: break the transfer up if len > the max msg size
* TODO: pass in *conn instead of *qp
*
*/
int spts_send_single(int dest, char *buf, int len)
{
int ret, i;
struct spts_simple_msg *wrap;
// DEBUGK("spts_send_single() START\n");
if(conn[dest].status == 0) {
printk("WARNING: this connection was never initialized\n");
return -1;
}
i = conn[dest].cur_send_buf;
memcpy(smsg[i].mem, buf, len);
wrap = (struct spts_simple_msg *) smsg[i].mem;
wrap->size = len;
ret = spts_send_single_msg(dest, &smsg[i]);
DEBUGK("spts_send_single_msg() returned %d\n", ret);
// spts_dump_page( &conn[dest].smsg[i] );
conn[dest].cur_send_buf += 1;
if(conn[dest].cur_send_buf == SPTS_MSG_COUNT)
conn[dest].cur_send_buf=0;
// mthca_dump_qp( conn[dest].qp );
return len;
}
EXPORT_SYMBOL(spts_send_single);
/*
* spts_send_all()
*
* simply sends a message to every connection
*
*/
int spts_send_all(char *buf, int len)
{
int ret = 0;
int i;
DEBUGK("spts_send_all()\n");
for( i=0; i<SPTS_MAX_CONNECTIONS; ++i ) {
if( ! conn[i].status )
continue;
ret = spts_send_single(i, buf, len);
}
return ret;
}
EXPORT_SYMBOL(spts_send_all);
/*
* spts_poll()
*
* This function polls IB and send passes back the newly
* received buffers
*
* Returns number of received phys_buf's
* if return == SPTS_MAX_COUNT then there are more to receive.
*
* TODO: maybe change wc[8] to wc[SPTS_MSG_COUNT - cur_recv_buf] ???
*
*/
int spts_poll(int dest, struct spts_msg **msgs)
{
int send_cq, recv_cq;
int i, j;
if(conn[dest].status == 0) {
printk("WARNING: this connection was never initialized\n");
return -1;
}
send_cq = ib_poll_cq(conn[dest].send_cq, SPTS_MAX_QUEUE_POLL, spts_wc);
recv_cq = ib_poll_cq(conn[dest].recv_cq, SPTS_MAX_QUEUE_POLL, spts_wc);
i = conn[dest].cur_recv_buf;
*msgs = &rmsg[i];
DEBUGK("spts_poll() SEND CQ: %d, RECV CQ: %d\n", send_cq, recv_cq);
if( recv_cq >= SPTS_MSG_COUNT )
printk("WARNING: SPTS overflowing\n");
for(i=0; i < recv_cq; ++i) {
j = conn[dest].cur_recv_buf;
// spts_dump_page( &conn[dest].rmsg[j] );
conn[dest].cur_recv_buf += 1;
if(conn[dest].cur_recv_buf == SPTS_MSG_COUNT)
conn[dest].cur_recv_buf = 0;
}
// mthca_dump_qp( conn[dest].qp );
i = mthca_get_sq_free(conn[dest].qp);
j = mthca_get_rq_free(conn[dest].qp);
DEBUGK("spts_poll() SQ FREE: %d, RQ FREE: %d\n", i, j);
if( j < SPTS_MAX_QUEUE_POLL * 2 ) {
DEBUGK("posting more recv buffers\n");
spts_post_recvs( &conn[dest] );
}
if( recv_cq == 0 )
*msgs = NULL;
return recv_cq;
}
EXPORT_SYMBOL(spts_poll);
int spts_poll_one(struct spts_msg **msgs)
{
DEBUGK("spts_poll_one() START cur_msg_count == %d\n", cur_msg_count);
if( cur_msg_count > 0 ) {
*msgs = cur_msgs;
++cur_msgs;
--cur_msg_count;
// could free bufs here if msg_count == 0
return 1;
}
cur_msg_count = spts_poll_all( &cur_msgs);
*msgs = cur_msgs;
if(cur_msg_count > 0) {
++cur_msgs;
--cur_msg_count;
return 1;
} else if(cur_msg_count < 0) {
DEBUGK("An error occurred in the communication\n");
return -1;
} else {
DEBUGK("RECV queue empty\n");
}
return 0;
}
EXPORT_SYMBOL(spts_poll_one);
/*
* spts_poll_all()
*
* This searches through all the open connections and returns anything
* it finds.
*
*/
int spts_poll_all(struct spts_msg **msgs)
{
int recvcq, i;
// DEBUGK("spts_poll_all() START\n");
for( i=0; i<SPTS_MAX_CONNECTIONS; ++i ) {
if( ! conn[i].status )
continue;
recvcq = spts_poll( i, msgs);
if(recvcq>0) {
DEBUGK("spts_poll_all() recvcq == %d, i == %d\n", recvcq, i);
return recvcq;
} else if(recvcq < 0) {
DEBUGK("spts_poll_all() FAILURE: An error occurred in on connection %d\n", i);
DEBUGK("spts_poll_all() should kill this connection here %d\n", i);
return -1;
} else {
DEBUGK("spts_poll_all() RECV queue empty for i == %d\n", i);
}
}
return 0;
}
EXPORT_SYMBOL(spts_poll_all);
/*
* Setup message buffers and memset them to useful values
* so that the transfers can be checked to see if they are
* working.
*/
static int __init spts_init(void)
{
int i;
memset(&smsg, 0, sizeof smsg);
memset(&rmsg, 0, sizeof rmsg);
memset(&conn, 0, sizeof conn);
// memset(&send_wr, 0, sizeof send_wr);
conn[0].smsg = smsg;
conn[0].rmsg = rmsg;
conn[0].cur_recv_buf = 0;
conn[0].cur_send_buf = 0;
printk("spts_init() TEST: sizeof conn == %d\n", sizeof conn);
printk("spts_init() TEST: conn[0].status == %d\n", conn[0].status);
conn[0].status = 0;
printk("spts_init() TEST: conn[1].status == %d\n", conn[1].status);
conn[1].status = 0;
conn[0].id = 0;
conn[1].id = 1;
for( i = 0; i < SPTS_MSG_COUNT; ++i ) {
rmsg[i].mem = kmalloc( SPTS_MSG_SIZE, GFP_KERNEL);
if (!rmsg[i].mem) {
printk("spts_init: failed message allocation\n");
return -ENOMEM;
}
// printk("spts_init: about to memset i = %d\n", i);
memset(rmsg[i].mem, i, SPTS_MSG_SIZE);
}
for( i = 0; i < SPTS_MSG_COUNT; ++i ) {
smsg[i].mem = kmalloc( SPTS_MSG_SIZE, GFP_KERNEL);
if (!smsg[i].mem) {
printk("spts_init: failed message allocation\n");
return -ENOMEM;
}
// printk("spts_init: about to memset i = %d\n", i);
memset(smsg[i].mem, 0xA0 + i, SPTS_MSG_SIZE);
}
// spts_dump_page( &rmsg[0] );
ib_cm_reg_init_conn( &spts_init_conn, &spts_post_recvs);
printk("Finished spts_init() -- allocated recieve and send buffers\n");
return 0;
}
/*
* Need to check to make sure this is destroying everything correctly
* What else needs to be cleaned up & freed on exit?
*/
static void __exit spts_cleanup(void)
{
int i;
// spts_cleanup_proc();
for( i = 0; i < SPTS_MSG_COUNT; ++i ) {
if (rmsg[i].mem)
kfree(rmsg[i].mem);
if (smsg[i].mem)
kfree(smsg[i].mem);
}
for( i=0; i<SPTS_MAX_CONNECTIONS; ++i ) {
if( conn[i].status )
continue;
DEBUGK("spts_destroy_node() i == %d\n", i);
spts_destroy_node(&conn[i]);
}
printk("spts_cleanup() END\n");
}
module_init(spts_init);
module_exit(spts_cleanup);
/*
* Copyright (c) 2005 Linux Machines Inc. All rights reserved.
*
* This software is available to you under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree.
*
*/
#include <asm/semaphore.h>
#include <linux/device.h>
#include <linux/pci.h>
#include <linux/time.h>
#include <linux/workqueue.h>
#include <ib_cm.h>
// #define DEBUGK(fmt, args...) printk( fmt, ## args)
// #define DEBUGK(fmt, args...) printk( KERN_DEBUG fmt, ## args)
#define DEBUGK(fmt, args...)
// 0x2000 for MSG_COUNT & IB_CQ_SIZE seems to be the max...
// otherwise the server panics "unable to create QP"
#define SPTS_MSG_COUNT 0x1000 // how many page buffers to allocate
// make sure SPTS_MSG_COUNT > 2 * SPTS_MAX_QUEUE_POLL
#define SPTS_MSG_SIZE 0x800 // the size of each message to send (aka MTU)
// breaks over IB right now at 0x1000
#define SPTS_IB_CQ_SIZE 0x2000 // how big to make the CQ
// make sure SPTS_IB_CQ_SIZE > 4 * SPTS_MAX_QUEUE_POLL
#define SPTS_MAX_CONNECTIONS 0x10 // max simultanious (how many computers on the IB network)
#define SPTS_MAX_QUEUE_POLL 0x400 // max completion buffers to grab
#define LISTEN_NUM 0x1000 // the number to listen on
/*
* Simple Page Transfer Scheme "spits" message struct
*
* This is the raw primative struct that to create a pool
* of pages in memory that can be used for transfers
*
*/
struct spts_msg {
int size; // msg_size
int s; // source
int d; // destination
// struct ib_recv_wr recv_wr;
// struct ib_send_wr send_wr;
// struct ib_send_wr *send_wr_bad;
// struct ib_sge sge;
void *mem; // the message itself
// char mem[SPTS_MSG_SIZE];
u64 addr; // addr for IB HCA
struct ib_mr *mr; // the IB MR
DECLARE_PCI_UNMAP_ADDR(mapping)
};
/*
* This is a message wrapper struct for simple things
*
*/
struct spts_simple_msg {
u32 msg[(SPTS_MSG_SIZE >> 2)-2];
u32 msgnum;
u32 size;
};
/*
* Contains Infiniband specific info about the state
* of the connection between to nodes.
*
*/
struct spts_ib_connection {
int id;
struct ib_cm_id *cm_id;
struct ib_device *device;
struct ib_pd *pd;
struct ib_qp *qp;
struct ib_cq *send_cq;
struct ib_cq *recv_cq;
struct spts_msg *smsg; // send buffers
struct spts_msg *rmsg; // recieve buffers
int cur_recv_buf;
int cur_send_buf;
int status;
/* cm info */
struct ib_sa_path_rec path_rec;
int query_status;
};
typedef struct spts_ib_connection *(*ib_cm_conn_init1)(struct ib_device *device);
typedef int (*ib_cm_conn_init2)(struct spts_ib_connection *conn);
int ib_cm_reg_init_conn( ib_cm_conn_init1 init1, ib_cm_conn_init2 init2 );
void mthca_dump_qp(struct ib_qp *ibqp);
int mthca_get_sq_free(struct ib_qp *ibqp);
int mthca_get_rq_free(struct ib_qp *ibqp);
int spts_post_recv(struct ib_qp *qp, struct spts_msg *msg );
int spts_create_messages(struct ib_device *device, struct ib_pd *pd, struct spts_msg *msgs);
void spts_dump_page( struct spts_msg *msg );
void spts_destroy_node(struct spts_ib_connection *node);
void spts_event_handler(struct ib_event *event, void *data);
void spts_comp_handler(struct ib_cq *cq, void *cq_context);
// int spts_modify_to_rtr(struct ib_cm_id *cm_id, struct ib_cm_event *event);
// int spts_modify_to_rts(struct ib_cm_id *cm_id, struct ib_cm_event *event);
int spts_poll_all( struct spts_msg **msgs);
int spts_poll( int dest, struct spts_msg **msgs);
int spts_poll_one(struct spts_msg **msgs);
int spts_send_all(char *buf, int len);
int spts_send_single(int dest, char *buf, int len);
int spts_init_node(struct spts_ib_connection *node);
void spts_init_proc(void);
struct spts_ib_connection *spts_init_conn(struct ib_device *dev);
int spts_post_recvs(struct spts_ib_connection *node);
void ib_cm_spts_conn_client(int slid, int dlid);
_______________________________________________ openib-general mailing list [email protected] http://openib.org/mailman/listinfo/openib-general To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general
