The attached patch adds support for RDMAoE (RDMA over Ethernet) devices
to Openib BTL. The code changes are very minimal, actually we only
modified the RDMACM code to provide better support for IB and RDMAoE
devices. Please let me know if you have any comments.
Regards,Vasily.
diff -r 9aad663adc9f ompi/config/ompi_check_openib.m4
--- a/ompi/config/ompi_check_openib.m4 Sun Oct 25 08:29:01 2009 -0700
+++ b/ompi/config/ompi_check_openib.m4 Sun Nov 01 12:17:03 2009 +0200
@@ -13,7 +13,7 @@
# Copyright (c) 2006-2008 Cisco Systems, Inc. All rights reserved.
# Copyright (c) 2006-2007 Los Alamos National Security, LLC. All rights
# reserved.
-# Copyright (c) 2006-2008 Mellanox Technologies. All rights reserved.
+# Copyright (c) 2006-2009 Mellanox Technologies. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
@@ -204,6 +204,21 @@
[$1_have_ibcm=1
$1_LIBS="-libcm $$1_LIBS"])])
fi
+
+ # Check support for RDMAoE devices
+ $1_have_rdmaoe=0
+ AC_CHECK_DECLS([RDMA_TRANSPORT_RDMAOE],
+ [$1_have_rdmaoe=1], [],
+ [#include ])
+
+ AC_MSG_CHECKING([if RDMAoE support is enabled])
+ if test "1" = "$$1_have_rdmaoe"; then
+AC_DEFINE_UNQUOTED([OMPI_HAVE_RDMAOE], [$$1_have_rdmaoe],
[Enable RDMAoE support])
+AC_MSG_RESULT([yes])
+ else
+AC_MSG_RESULT([no])
+ fi
+
])
# Check to see if works. It is known to
diff -r 9aad663adc9f ompi/mca/btl/openib/btl_openib.c
--- a/ompi/mca/btl/openib/btl_openib.c Sun Oct 25 08:29:01 2009 -0700
+++ b/ompi/mca/btl/openib/btl_openib.c Sun Nov 01 12:17:03 2009 +0200
@@ -354,6 +354,13 @@
}
#endif
+#ifdef OMPI_HAVE_RDMAOE
+if(RDMA_TRANSPORT_RDMAOE == (openib_btl->ib_port_attr.transport) &&
+OPAL_PROC_ON_LOCAL_NODE(ompi_proc->proc_flags)) {
+continue;
+}
+#endif
+
if(NULL == (ib_proc = mca_btl_openib_proc_create(ompi_proc))) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
diff -r 9aad663adc9f ompi/mca/btl/openib/btl_openib_endpoint.c
--- a/ompi/mca/btl/openib/btl_openib_endpoint.c Sun Oct 25 08:29:01 2009 -0700
+++ b/ompi/mca/btl/openib/btl_openib_endpoint.c Sun Nov 01 12:17:03 2009 +0200
@@ -556,7 +556,6 @@
{
/* If the CPC uses the CTS protocol, then start it up */
if (endpoint->endpoint_local_cpc->cbm_uses_cts) {
-int transport_type_ib_p = 0;
/* Post our receives, which will make credit management happy
(i.e., rd_credits will be 0) */
if (OMPI_SUCCESS != mca_btl_openib_endpoint_post_recvs(endpoint)) {
@@ -572,16 +571,13 @@
receives this side's CTS). Also send the CTS if we already
received the peer's CTS (e.g., if this process was slow to
call cpc_complete(). */
-#if defined(HAVE_STRUCT_IBV_DEVICE_TRANSPORT_TYPE)
-transport_type_ib_p = (IBV_TRANSPORT_IB ==
endpoint->endpoint_btl->device->ib_dev->transport_type);
-#endif
+
OPAL_OUTPUT((-1, "cpc_complete to peer %s: is IB %d, initiatior %d,
cts received: %d",
endpoint->endpoint_proc->proc_ompi->proc_hostname,
transport_type_ib_p,
endpoint->endpoint_initiator,
endpoint->endpoint_cts_received));
-if (transport_type_ib_p ||
-endpoint->endpoint_initiator ||
+if (endpoint->endpoint_initiator ||
endpoint->endpoint_cts_received) {
mca_btl_openib_endpoint_send_cts(endpoint);
diff -r 9aad663adc9f ompi/mca/btl/openib/connect/base.h
--- a/ompi/mca/btl/openib/connect/base.hSun Oct 25 08:29:01 2009 -0700
+++ b/ompi/mca/btl/openib/connect/base.hSun Nov 01 12:17:03 2009 +0200
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2007-2008 Cisco Systems, Inc. All rights reserved.
- *
+ * Copyright (c) 2009 Mellanox Technologies. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
@@ -13,6 +13,17 @@
#include "connect/connect.h"
+#ifdef OMPI_HAVE_RDMAOE
+#define BTL_OPENIB_CONNECT_BASE_CHECK_IF_NOT_IB(btl) \
+(((IBV_TRANSPORT_IB != ((btl)->device->ib_dev->transport_type)) || \
+(RDMA_TRANSPORT_RDMAOE == ((btl)->ib_port_attr.transport))) ? \
+true : false)
+#else
+#define BTL_OPENIB_CONNECT_BASE_CHECK_TRANSPORT_TYPE(btl) \
+((IBV_TRANSPORT_IB != ((btl)->device->ib_dev->transport_type)) ? \
+true : false)
+#endif
+
BEGIN_C_DECLS
/*
diff -r 9aad663adc9f ompi/mca/btl/openib/connect/btl_openib_connect_ibcm.c
--- a/ompi/mca/btl/openib/connect/btl_openib_connect_ibcm.c Sun Oct 25
08:29:01 2009 -0700
+++ b/ompi/mca/btl/openib/connect/btl_openib_connect_ibcm.c Sun Nov 01
12:17:03 2009 +0200
@@ -1,6 +1,6 @@
/*
* Copyright
