Author: dkerr
Date: 2011-02-16 00:37:22 EST (Wed, 16 Feb 2011)
New Revision: 24395
URL: https://svn.open-mpi.org/trac/ompi/changeset/24395
Log:
on Solaris, when IBV_ACCESS_SO is available, use strong ordered memory region
for eager rdma connection
Text files modified:
trunk/ompi/mca/btl/openib/btl_openib_component.c | 13 ++++++++++---
trunk/ompi/mca/btl/openib/btl_openib_endpoint.c | 19 +++++++++++++++++--
trunk/ompi/mca/btl/openib/configure.m4 | 16 +++++++++++++++-
3 files changed, 42 insertions(+), 6 deletions(-)
Modified: trunk/ompi/mca/btl/openib/btl_openib_component.c
==============================================================================
--- trunk/ompi/mca/btl/openib/btl_openib_component.c (original)
+++ trunk/ompi/mca/btl/openib/btl_openib_component.c 2011-02-16 00:37:22 EST
(Wed, 16 Feb 2011)
@@ -15,7 +15,7 @@
* Copyright (c) 2006-2007 Los Alamos National Security, LLC. All rights
* reserved.
* Copyright (c) 2006-2007 Voltaire All rights reserved.
- * Copyright (c) 2009-2010 Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2009-2011 Oracle and/or its affiliates. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
@@ -527,9 +527,16 @@
{
mca_btl_openib_device_t *device = (mca_btl_openib_device_t*)reg_data;
mca_btl_openib_reg_t *openib_reg = (mca_btl_openib_reg_t*)reg;
+ enum ibv_access_flags access_flag = IBV_ACCESS_LOCAL_WRITE |
+ IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ;
- openib_reg->mr = ibv_reg_mr(device->ib_pd, base, size,
IBV_ACCESS_LOCAL_WRITE |
- IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ);
+#if defined(HAVE_IBV_ACCESS_SO)
+ if (reg->flags & MCA_MPOOL_FLAGS_SO_MEM) {
+ access_flag |= IBV_ACCESS_SO;
+ }
+#endif
+
+ openib_reg->mr = ibv_reg_mr(device->ib_pd, base, size, access_flag);
if (NULL == openib_reg->mr) {
return OMPI_ERR_OUT_OF_RESOURCE;
Modified: trunk/ompi/mca/btl/openib/btl_openib_endpoint.c
==============================================================================
--- trunk/ompi/mca/btl/openib/btl_openib_endpoint.c (original)
+++ trunk/ompi/mca/btl/openib/btl_openib_endpoint.c 2011-02-16 00:37:22 EST
(Wed, 16 Feb 2011)
@@ -16,7 +16,7 @@
* Copyright (c) 2006-2007 Voltaire All rights reserved.
* Copyright (c) 2006-2009 Mellanox Technologies, Inc. All rights reserved.
* Copyright (c) 2010 IBM Corporation. All rights reserved.
- * Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved
+ * Copyright (c) 2010-2011 Oracle and/or its affiliates. All rights reserved
*
* $COPYRIGHT$
*
@@ -911,6 +911,7 @@
char *buf;
mca_btl_openib_recv_frag_t *headers_buf;
int i;
+ uint32_t flag = MCA_MPOOL_FLAGS_CACHE_BYPASS;
/* Set local rdma pointer to 1 temporarily so other threads will not try
* to enter the function */
@@ -925,11 +926,25 @@
if(NULL == headers_buf)
goto unlock_rdma_local;
+#if defined(HAVE_IBV_ACCESS_SO)
+ /* Solaris implements the Relaxed Ordering feature defined in the
+ PCI Specification. With this in mind any memory region which
+ relies on a buffer being written in a specific order, for
+ example the eager rdma connections created in this routinue,
+ must set a strong order flag when registering the memory for
+ rdma operations.
+
+ The following flag will be interpreted and the appropriate
+ steps will be taken when the memory is registered in
+ openib_reg_mr(). */
+ flag |= MCA_MPOOL_FLAGS_SO_MEM;
+#endif
+
buf = (char *)
openib_btl->super.btl_mpool->mpool_alloc(openib_btl->super.btl_mpool,
openib_btl->eager_rdma_frag_size *
mca_btl_openib_component.eager_rdma_num,
mca_btl_openib_component.buffer_alignment,
- MCA_MPOOL_FLAGS_CACHE_BYPASS,
+ flag,
(mca_mpool_base_registration_t**)&endpoint->eager_rdma_local.reg);
if(!buf)
Modified: trunk/ompi/mca/btl/openib/configure.m4
==============================================================================
--- trunk/ompi/mca/btl/openib/configure.m4 (original)
+++ trunk/ompi/mca/btl/openib/configure.m4 2011-02-16 00:37:22 EST (Wed,
16 Feb 2011)
@@ -12,6 +12,7 @@
# All rights reserved.
# Copyright (c) 2007-2010 Cisco Systems, Inc. All rights reserved.
# Copyright (c) 2008 Mellanox Technologies. All rights reserved.
+# Copyright (c) 2011 Oracle and/or its affiliates. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
@@ -34,7 +35,7 @@
AC_DEFUN([MCA_ompi_btl_openib_CONFIG],[
AC_CONFIG_FILES([ompi/mca/btl/openib/Makefile])
- OPAL_VAR_SCOPE_PUSH([cpcs have_threads])
+ OPAL_VAR_SCOPE_PUSH([cpcs have_threads have_ibv_access_so])
cpcs="oob"
OMPI_CHECK_OPENIB([btl_openib],
@@ -78,6 +79,19 @@
AC_MSG_CHECKING([which openib btl cpcs will be built])
AC_MSG_RESULT([$cpcs])])
+ # check for Solaris specific memory access flag
+ AS_IF([test "$btl_openib_happy" = "yes"],
+ [if test "`echo $build_os | $GREP solaris`"; then
+ AC_TRY_COMPILE([#include <infiniband/verbs.h>],
+ [int flag = IBV_ACCESS_SO;],
+ [have_ibv_access_so="yes"
+ AC_DEFINE_UNQUOTED([HAVE_IBV_ACCESS_SO],
+ 1,[openib define HAVE_IBV_ACCESS_SO])],
+ [have_ibv_access_so="no"])
+ AC_MSG_CHECKING([for IBV_ACCESS_SO in Solaris])
+ AC_MSG_RESULT([$have_ibv_access_so])
