Signed-off-by: Sean Hefty <[email protected]>
---
dapl/openib_ucm/SOURCES | 18 ++------
dapl/openib_ucm/cm.c | 48 ++++++++-------------
dapl/openib_ucm/dapl_ib_util.h | 2 +-
dapl/openib_ucm/device.c | 66 +++++++++++++----------------
dapl/openib_ucm/linux/openib_osd.h | 10 ++++
dapl/openib_ucm/makefile | 7 +++
dapl/openib_ucm/udapl.rc | 8 ++--
dapl/openib_ucm/udapl_ofa_ucm_exports.src | 14 ++++++
dapl/openib_ucm/windows/openib_osd.h | 11 +++++
9 files changed, 100 insertions(+), 84 deletions(-)
create mode 100644 dapl/openib_ucm/makefile
create mode 100644 dapl/openib_ucm/udapl_ofa_ucm_exports.src
diff --git a/dapl/openib_ucm/SOURCES b/dapl/openib_ucm/SOURCES
index dfe956f..7eecf48 100644
--- a/dapl/openib_ucm/SOURCES
+++ b/dapl/openib_ucm/SOURCES
@@ -16,22 +16,14 @@ DLLDEF=$(OBJ_PATH)\$O\udapl_ofa_ucm_exports.def
USE_MSVCRT = 1
-SOURCES = \
- udapl.rc \
- ..\dapl_common_src.c \
- ..\dapl_udapl_src.c \
- dapl_ib_cq.c \
- dapl_ib_extensions.c \
- dapl_ib_mem.c \
- dapl_ib_qp.c \
- dapl_ib_util.c \
- dapl_ib_cm.c
-
-INCLUDES = ..\include;..\common;windows;..\..\dat\include;\
+SOURCES = udapl.rc ..\dapl_common_src.c ..\dapl_udapl_src.c ..\openib_common.c
\
+ device.c cm.c
+
+INCLUDES = ..\include;..\openib_common\;..\common;windows;..\..\dat\include;\
..\..\dat\udat\windows;..\udapl\windows;\
..\..\..\..\inc;..\..\..\..\inc\user;..\..\..\libibverbs\include
-DAPL_OPTS = -DEXPORT_DAPL_SYMBOLS -DDAT_EXTENSIONS -DSOCK_CM -DOPENIB
-DCQ_WAIT_OBJECT
+DAPL_OPTS = -DEXPORT_DAPL_SYMBOLS -DDAT_EXTENSIONS -DOPENIB -DCQ_WAIT_OBJECT
USER_C_FLAGS = $(USER_C_FLAGS) $(DAPL_OPTS)
diff --git a/dapl/openib_ucm/cm.c b/dapl/openib_ucm/cm.c
index 41a7c9e..9d702a8 100644
--- a/dapl/openib_ucm/cm.c
+++ b/dapl/openib_ucm/cm.c
@@ -115,7 +115,6 @@ static int ucm_send(ib_hca_transport_t *tp, ib_cm_msg_t
*msg, DAT_PVOID p_data,
static void ucm_disconnect_final(dp_ib_cm_handle_t cm);
DAT_RETURN dapli_cm_disconnect(dp_ib_cm_handle_t cm);
DAT_RETURN dapli_cm_connect(DAPL_EP *ep, dp_ib_cm_handle_t cm);
-static void dapls_thread_signal(struct dapl_hca *hca);
#define UCM_SND_BURST 50
@@ -675,7 +674,7 @@ static void ucm_ud_free(DAPL_EP *ep)
/* wakeup work thread if necessary */
if (hca)
- dapls_thread_signal(hca);
+ dapls_thread_signal(&tp->signal);
}
/* mark for destroy, remove all references, schedule cleanup */
@@ -719,7 +718,7 @@ void dapls_ib_cm_free(dp_ib_cm_handle_t cm, DAPL_EP *ep)
dapl_os_unlock(&cm->lock);
/* wakeup work thread */
- dapls_thread_signal(cm->hca);
+ dapls_thread_signal(&cm->hca->ib_trans.signal);
}
/* ACTIVE/PASSIVE: queue up connection object on CM list */
@@ -731,7 +730,7 @@ static void ucm_queue_conn(dp_ib_cm_handle_t cm)
dapl_llist_add_tail(&cm->hca->ib_trans.list,
(DAPL_LLIST_ENTRY *)&cm->entry, cm);
dapl_os_unlock(&cm->hca->ib_trans.lock);
- dapls_thread_signal(cm->hca);
+ dapls_thread_signal(&cm->hca->ib_trans.signal);
}
/* PASSIVE: queue up listen object on listen list */
@@ -790,7 +789,7 @@ DAT_RETURN dapli_cm_disconnect(dp_ib_cm_handle_t cm)
cm->msg.op = htons(DCM_DREQ);
cm->retries = 1;
finalize = 0; /* wait for DREP, wakeup timer thread */
- dapls_thread_signal(cm->hca);
+ dapls_thread_signal(&cm->hca->ib_trans.signal);
break;
case DCM_DISC_PENDING:
/* DREQ timeout, resend until retries exhausted */
@@ -1272,6 +1271,10 @@ static int ucm_reply(dp_ib_cm_handle_t cm)
htonl(cm->msg.daddr.ib.qpn));
dapl_os_unlock(&cm->lock);
+#ifdef DAPL_COUNTERS
+ if (g_dapl_dbg_type & DAPL_DBG_TYPE_CM_LIST)
+
dapls_print_cm_list(dapl_llist_peek_head(&cm->hca->ia_list_head));
+#endif
#ifdef DAT_EXTENSIONS
if (cm->msg.saddr.ib.qp_type == IBV_QPT_UD) {
DAT_IB_EXTENSION_EVENT_DATA xevent;
@@ -1406,7 +1409,7 @@ dapli_accept_usr(DAPL_EP *ep, DAPL_CR *cr, DAT_COUNT
p_size, DAT_PVOID p_data)
goto bail;
dapl_dbg_log(DAPL_DBG_TYPE_CM, " PASSIVE: accepted!\n");
- dapls_thread_signal(cm->hca);
+ dapls_thread_signal(&cm->hca->ib_trans.signal);
return DAT_SUCCESS;
bail:
if (cm->msg.saddr.ib.qp_type != IBV_QPT_UD)
@@ -1921,31 +1924,24 @@ ib_cm_events_t dapls_ib_get_cm_event(IN
DAT_EVENT_NUMBER dat_event_num)
}
#if defined(_WIN32) || defined(_WIN64)
-static void dapls_thread_signal(struct dapl_hca *hca)
-{
-// CompSetCancel(&ufds);
-}
void cm_thread(void *arg)
{
struct dapl_hca *hca = arg;
dp_ib_cm_handle_t cm, next;
- COMP_SET ufds;
DWORD time_ms;
- CompSetInit(&ufds);
-
dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " cm_thread: ENTER hca %p\n", hca);
dapl_os_lock(&hca->ib_trans.lock);
for (hca->ib_trans.cm_state = IB_THREAD_RUN;
- hca->ib_trans.cm_state == IB_THREAD_RUN &&
- dapl_llist_is_empty(&hca->ib_trans.list);
+ hca->ib_trans.cm_state == IB_THREAD_RUN ||
+ !dapl_llist_is_empty(&hca->ib_trans.list);
dapl_os_lock(&hca->ib_trans.lock)) {
time_ms = INFINITE;
- CompSetZero(&ufds);
- CompSetAdd(&hca->ib_hca_handle->channel, &ufds);
- CompSetAdd(&hca->ib_trans.rch->comp_channel, &ufds);
+ CompSetZero(&hca->ib_trans.signal.set);
+ CompSetAdd(&hca->ib_hca_handle->channel,
&hca->ib_trans.signal.set);
+ CompSetAdd(&hca->ib_trans.rch->comp_channel,
&hca->ib_trans.signal.set);
next = dapl_llist_is_empty(&hca->ib_trans.list) ? NULL :
dapl_llist_peek_head(&hca->ib_trans.list);
@@ -1971,7 +1967,7 @@ void cm_thread(void *arg)
hca->ib_hca_handle->channel.Milliseconds = time_ms;
hca->ib_trans.rch->comp_channel.Milliseconds = time_ms;
- CompSetPoll(&ufds, time_ms);
+ CompSetPoll(&hca->ib_trans.signal.set, time_ms);
hca->ib_hca_handle->channel.Milliseconds = 0;
hca->ib_trans.rch->comp_channel.Milliseconds = 0;
@@ -1979,19 +1975,13 @@ void cm_thread(void *arg)
ucm_async_event(hca);
}
- CompSetCleanup(&ufds);
- hca->ib_trans.cm_state = IB_THREAD_EXIT;
dapl_os_unlock(&hca->ib_trans.lock);
+ hca->ib_trans.cm_state = IB_THREAD_EXIT;
dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " cm_thread(hca %p) exit\n", hca);
}
#else // _WIN32 || _WIN64
-static void dapls_thread_signal(struct dapl_hca *hca)
-{
- send(hca->ib_trans.scm[1], "w", sizeof "w", 0);
-}
-
void cm_thread(void *arg)
{
struct dapl_hca *hca = arg;
@@ -2011,7 +2001,7 @@ void cm_thread(void *arg)
while (1) {
time_ms = -1; /* reset to blocking */
dapl_fd_zero(set);
- dapl_fd_set(hca->ib_trans.scm[0], set, DAPL_FD_READ);
+ dapl_fd_set(hca->ib_trans.signal.scm[0], set, DAPL_FD_READ);
dapl_fd_set(hca->ib_hca_handle->async_fd, set, DAPL_FD_READ);
dapl_fd_set(hca->ib_trans.rch->fd, set, DAPL_FD_READ);
@@ -2056,9 +2046,9 @@ void cm_thread(void *arg)
DAPL_FD_READ) == DAPL_FD_READ) {
ucm_async_event(hca);
}
- while (dapl_poll(hca->ib_trans.scm[0],
+ while (dapl_poll(hca->ib_trans.signal.scm[0],
DAPL_FD_READ) == DAPL_FD_READ) {
- recv(hca->ib_trans.scm[0], rbuf, 2, 0);
+ recv(hca->ib_trans.signal.scm[0], rbuf, 2, 0);
}
dapl_os_lock(&hca->ib_trans.lock);
diff --git a/dapl/openib_ucm/dapl_ib_util.h b/dapl/openib_ucm/dapl_ib_util.h
index 53f00bb..eef6384 100644
--- a/dapl/openib_ucm/dapl_ib_util.h
+++ b/dapl/openib_ucm/dapl_ib_util.h
@@ -87,7 +87,7 @@ typedef struct _ib_hca_transport
uint8_t tclass;
uint8_t mtu;
DAT_NAMED_ATTR named_attr;
- DAPL_SOCKET scm[2];
+ struct dapl_thread_signal signal;
int cqe;
int qpe;
int retries;
diff --git a/dapl/openib_ucm/device.c b/dapl/openib_ucm/device.c
index 4f26e87..65f5f9b 100644
--- a/dapl/openib_ucm/device.c
+++ b/dapl/openib_ucm/device.c
@@ -36,7 +36,21 @@
static void ucm_service_destroy(IN DAPL_HCA *hca);
static int ucm_service_create(IN DAPL_HCA *hca);
-static int32_t create_cr_pipe(IN DAPL_HCA * hca_ptr)
+#if defined (_WIN32)
+
+static int32_t create_os_signal(IN DAPL_HCA * hca_ptr)
+{
+ return CompSetInit(&hca_ptr->ib_trans.signal.set);
+}
+
+static void destroy_os_signal(IN DAPL_HCA * hca_ptr)
+{
+ CompSetCleanup(&hca_ptr->ib_trans.signal.set);
+}
+
+#else // _WIN32
+
+static int32_t create_os_signal(IN DAPL_HCA * hca_ptr)
{
DAPL_SOCKET listen_socket;
struct sockaddr_in addr;
@@ -62,35 +76,36 @@ static int32_t create_cr_pipe(IN DAPL_HCA * hca_ptr)
if (ret)
goto err1;
- hca_ptr->ib_trans.scm[1] = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
- if (hca_ptr->ib_trans.scm[1] == DAPL_INVALID_SOCKET)
+ hca_ptr->ib_trans.signal.scm[1] = socket(AF_INET, SOCK_STREAM,
IPPROTO_TCP);
+ if (hca_ptr->ib_trans.signal.scm[1] == DAPL_INVALID_SOCKET)
goto err1;
- ret = connect(hca_ptr->ib_trans.scm[1],
+ ret = connect(hca_ptr->ib_trans.signal.scm[1],
(struct sockaddr *)&addr, sizeof(addr));
if (ret)
goto err2;
- hca_ptr->ib_trans.scm[0] = accept(listen_socket, NULL, NULL);
- if (hca_ptr->ib_trans.scm[0] == DAPL_INVALID_SOCKET)
+ hca_ptr->ib_trans.signal.scm[0] = accept(listen_socket, NULL, NULL);
+ if (hca_ptr->ib_trans.signal.scm[0] == DAPL_INVALID_SOCKET)
goto err2;
closesocket(listen_socket);
return 0;
err2:
- closesocket(hca_ptr->ib_trans.scm[1]);
+ closesocket(hca_ptr->ib_trans.signal.scm[1]);
err1:
closesocket(listen_socket);
return 1;
}
-static void destroy_cr_pipe(IN DAPL_HCA * hca_ptr)
+static void destroy_os_signal(IN DAPL_HCA * hca_ptr)
{
- closesocket(hca_ptr->ib_trans.scm[0]);
- closesocket(hca_ptr->ib_trans.scm[1]);
+ closesocket(hca_ptr->ib_trans.signal.scm[0]);
+ closesocket(hca_ptr->ib_trans.signal.scm[1]);
}
+#endif
/*
* dapls_ib_init, dapls_ib_release
@@ -117,28 +132,6 @@ int32_t dapls_ib_release(void)
return 0;
}
-#if defined(_WIN64) || defined(_WIN32)
-int dapls_config_comp_channel(struct ibv_comp_channel *channel)
-{
- return 0;
-}
-#else // _WIN64 || WIN32
-int dapls_config_comp_channel(struct ibv_comp_channel *channel)
-{
- int opts;
-
- opts = fcntl(channel->fd, F_GETFL); /* uCQ */
- if (opts < 0 || fcntl(channel->fd, F_SETFL, opts | O_NONBLOCK) < 0) {
- dapl_log(DAPL_DBG_TYPE_ERR,
- " dapls_create_comp_channel: fcntl on ib_cq->fd %d ERR
%d %s\n",
- channel->fd, opts, strerror(errno));
- return errno;
- }
-
- return 0;
-}
-#endif
-
/*
* dapls_ib_open_hca
*
@@ -260,8 +253,7 @@ found:
if (ucm_service_create(hca_ptr))
goto bail;
- /* initialize pipe, user level wakeup on select */
- if (create_cr_pipe(hca_ptr)) {
+ if (create_os_signal(hca_ptr)) {
dapl_log(DAPL_DBG_TYPE_ERR,
" open_hca: failed to init cr pipe - %s\n",
strerror(errno));
@@ -345,18 +337,18 @@ DAT_RETURN dapls_ib_close_hca(IN DAPL_HCA * hca_ptr)
if (hca_ptr->ib_trans.cm_state == IB_THREAD_RUN) {
hca_ptr->ib_trans.cm_state = IB_THREAD_CANCEL;
- send(hca_ptr->ib_trans.scm[1], "w", sizeof "w", 0);
+ dapls_thread_signal(&hca_ptr->ib_trans.signal);
while (hca_ptr->ib_trans.cm_state != IB_THREAD_EXIT) {
dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
" close_hca: waiting for cr_thread\n");
- send(hca_ptr->ib_trans.scm[1], "w", sizeof "w", 0);
+ dapls_thread_signal(&hca_ptr->ib_trans.signal);
dapl_os_sleep_usec(1000);
}
}
dapl_os_lock_destroy(&hca_ptr->ib_trans.lock);
dapl_os_lock_destroy(&hca_ptr->ib_trans.llock);
- destroy_cr_pipe(hca_ptr); /* no longer need pipe */
+ destroy_os_signal(hca_ptr);
ucm_service_destroy(hca_ptr);
if (hca_ptr->ib_hca_handle != IB_INVALID_HANDLE) {
diff --git a/dapl/openib_ucm/linux/openib_osd.h
b/dapl/openib_ucm/linux/openib_osd.h
index 191a55b..cf40fc9 100644
--- a/dapl/openib_ucm/linux/openib_osd.h
+++ b/dapl/openib_ucm/linux/openib_osd.h
@@ -18,4 +18,14 @@
#define closesocket close
+struct dapl_thread_signal
+{
+ DAPL_SOCKET scm[2];
+};
+
+static void dapls_thread_signal(struct dapl_thread_signal *signal)
+{
+ send(signal->scm[1], "w", sizeof "w", 0);
+}
+
#endif // OPENIB_OSD_H
diff --git a/dapl/openib_ucm/makefile b/dapl/openib_ucm/makefile
new file mode 100644
index 0000000..a0c0627
--- /dev/null
+++ b/dapl/openib_ucm/makefile
@@ -0,0 +1,7 @@
+#
+# DO NOT EDIT THIS FILE!!! Edit .\sources. if you want to add a new source
+# file to this component. This file merely indirects to the real make file
+# that is shared by all the driver components of the OpenIB Windows project.
+#
+
+!INCLUDE ..\..\..\..\inc\openib.def
diff --git a/dapl/openib_ucm/udapl.rc b/dapl/openib_ucm/udapl.rc
index 8550256..908376e 100644
--- a/dapl/openib_ucm/udapl.rc
+++ b/dapl/openib_ucm/udapl.rc
@@ -37,12 +37,12 @@
#if DBG
#define VER_FILEDESCRIPTION_STR "Direct Access Provider Library
v2.0 (OFA socket-cm) (Debug)"
-#define VER_INTERNALNAME_STR "dapl2-ofa-scmd.dll"
-#define VER_ORIGINALFILENAME_STR "dapl2-ofa-scmd.dll"
+#define VER_INTERNALNAME_STR "dapl2-ofa-ucmd.dll"
+#define VER_ORIGINALFILENAME_STR "dapl2-ofa-ucmd.dll"
#else
#define VER_FILEDESCRIPTION_STR "Direct Access Provider Library
v2.0 (OFA socket-cm)"
-#define VER_INTERNALNAME_STR "dapl2-ofa-scm.dll"
-#define VER_ORIGINALFILENAME_STR "dapl2-ofa-scm.dll"
+#define VER_INTERNALNAME_STR "dapl2-ofa-ucm.dll"
+#define VER_ORIGINALFILENAME_STR "dapl2-ofa-ucm.dll"
#endif
#include <common.ver>
diff --git a/dapl/openib_ucm/udapl_ofa_ucm_exports.src
b/dapl/openib_ucm/udapl_ofa_ucm_exports.src
new file mode 100644
index 0000000..d74b7bb
--- /dev/null
+++ b/dapl/openib_ucm/udapl_ofa_ucm_exports.src
@@ -0,0 +1,14 @@
+#if DBG
+LIBRARY dapl2-ofa-ucmd.dll
+#else
+LIBRARY dapl2-ofa-ucm.dll
+#endif
+
+
+EXPORTS
+dat_provider_init
+dat_provider_fini
+#ifdef DAT_EXTENSIONS
+dapl_extensions
+#endif
+
diff --git a/dapl/openib_ucm/windows/openib_osd.h
b/dapl/openib_ucm/windows/openib_osd.h
index 7eb3df3..73220dc 100644
--- a/dapl/openib_ucm/windows/openib_osd.h
+++ b/dapl/openib_ucm/windows/openib_osd.h
@@ -10,6 +10,7 @@
#include <ws2tcpip.h>
#include <io.h>
#include <fcntl.h>
+#include <comp_channel.h>
#define ntohll _byteswap_uint64
#define htonll _byteswap_uint64
@@ -32,4 +33,14 @@ static int writev(DAPL_SOCKET s, struct iovec *vector, int
count)
return ret ? ret : len;
}
+struct dapl_thread_signal
+{
+ COMP_SET set;
+};
+
+static void dapls_thread_signal(struct dapl_thread_signal *signal)
+{
+ CompSetCancel(&signal->set);
+}
+
#endif // OPENIB_OSD_H
--
1.5.2.5
_______________________________________________
ofw mailing list
[email protected]
http://lists.openfabrics.org/cgi-bin/mailman/listinfo/ofw
