Oracle -- Is this really only specific to Solaris? More comments below about configure.m4.
On Feb 16, 2011, at 12:37 AM, dk...@osl.iu.edu wrote: > Author: dkerr > Date: 2011-02-16 00:37:22 EST (Wed, 16 Feb 2011) > New Revision: 24395 > URL: https://svn.open-mpi.org/trac/ompi/changeset/24395 > > Log: > on Solaris, when IBV_ACCESS_SO is available, use strong ordered memory region > for eager rdma connection > Text files modified: > trunk/ompi/mca/btl/openib/btl_openib_component.c | 13 ++++++++++--- > > trunk/ompi/mca/btl/openib/btl_openib_endpoint.c | 19 > +++++++++++++++++-- > trunk/ompi/mca/btl/openib/configure.m4 | 16 +++++++++++++++- > > 3 files changed, 42 insertions(+), 6 deletions(-) > > Modified: trunk/ompi/mca/btl/openib/btl_openib_component.c > ============================================================================== > --- trunk/ompi/mca/btl/openib/btl_openib_component.c (original) > +++ trunk/ompi/mca/btl/openib/btl_openib_component.c 2011-02-16 00:37:22 EST > (Wed, 16 Feb 2011) > @@ -15,7 +15,7 @@ > * Copyright (c) 2006-2007 Los Alamos National Security, LLC. All rights > * reserved. > * Copyright (c) 2006-2007 Voltaire All rights reserved. > - * Copyright (c) 2009-2010 Oracle and/or its affiliates. All rights > reserved. > + * Copyright (c) 2009-2011 Oracle and/or its affiliates. All rights > reserved. > * $COPYRIGHT$ > * > * Additional copyrights may follow > @@ -527,9 +527,16 @@ > { > mca_btl_openib_device_t *device = (mca_btl_openib_device_t*)reg_data; > mca_btl_openib_reg_t *openib_reg = (mca_btl_openib_reg_t*)reg; > + enum ibv_access_flags access_flag = IBV_ACCESS_LOCAL_WRITE | > + IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ; > > - openib_reg->mr = ibv_reg_mr(device->ib_pd, base, size, > IBV_ACCESS_LOCAL_WRITE | > - IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ); > +#if defined(HAVE_IBV_ACCESS_SO) > + if (reg->flags & MCA_MPOOL_FLAGS_SO_MEM) { > + access_flag |= IBV_ACCESS_SO; > + } > +#endif > + > + openib_reg->mr = ibv_reg_mr(device->ib_pd, base, size, access_flag); > > if (NULL == openib_reg->mr) { > return OMPI_ERR_OUT_OF_RESOURCE; > > Modified: trunk/ompi/mca/btl/openib/btl_openib_endpoint.c > ============================================================================== > --- trunk/ompi/mca/btl/openib/btl_openib_endpoint.c (original) > +++ trunk/ompi/mca/btl/openib/btl_openib_endpoint.c 2011-02-16 00:37:22 EST > (Wed, 16 Feb 2011) > @@ -16,7 +16,7 @@ > * Copyright (c) 2006-2007 Voltaire All rights reserved. > * Copyright (c) 2006-2009 Mellanox Technologies, Inc. All rights reserved. > * Copyright (c) 2010 IBM Corporation. All rights reserved. > - * Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved > + * Copyright (c) 2010-2011 Oracle and/or its affiliates. All rights reserved > * > * $COPYRIGHT$ > * > @@ -911,6 +911,7 @@ > char *buf; > mca_btl_openib_recv_frag_t *headers_buf; > int i; > + uint32_t flag = MCA_MPOOL_FLAGS_CACHE_BYPASS; > > /* Set local rdma pointer to 1 temporarily so other threads will not try > * to enter the function */ > @@ -925,11 +926,25 @@ > if(NULL == headers_buf) > goto unlock_rdma_local; > > +#if defined(HAVE_IBV_ACCESS_SO) > + /* Solaris implements the Relaxed Ordering feature defined in the > + PCI Specification. With this in mind any memory region which > + relies on a buffer being written in a specific order, for > + example the eager rdma connections created in this routinue, > + must set a strong order flag when registering the memory for > + rdma operations. > + > + The following flag will be interpreted and the appropriate > + steps will be taken when the memory is registered in > + openib_reg_mr(). */ > + flag |= MCA_MPOOL_FLAGS_SO_MEM; > +#endif > + > buf = (char *) > openib_btl->super.btl_mpool->mpool_alloc(openib_btl->super.btl_mpool, > openib_btl->eager_rdma_frag_size * > mca_btl_openib_component.eager_rdma_num, > mca_btl_openib_component.buffer_alignment, > - MCA_MPOOL_FLAGS_CACHE_BYPASS, > + flag, > (mca_mpool_base_registration_t**)&endpoint->eager_rdma_local.reg); > > if(!buf) > > Modified: trunk/ompi/mca/btl/openib/configure.m4 > ============================================================================== > --- trunk/ompi/mca/btl/openib/configure.m4 (original) > +++ trunk/ompi/mca/btl/openib/configure.m4 2011-02-16 00:37:22 EST (Wed, > 16 Feb 2011) > @@ -12,6 +12,7 @@ > # All rights reserved. > # Copyright (c) 2007-2010 Cisco Systems, Inc. All rights reserved. > # Copyright (c) 2008 Mellanox Technologies. All rights reserved. > +# Copyright (c) 2011 Oracle and/or its affiliates. All rights reserved. > # $COPYRIGHT$ > # > # Additional copyrights may follow > @@ -34,7 +35,7 @@ > AC_DEFUN([MCA_ompi_btl_openib_CONFIG],[ > AC_CONFIG_FILES([ompi/mca/btl/openib/Makefile]) > > - OPAL_VAR_SCOPE_PUSH([cpcs have_threads]) > + OPAL_VAR_SCOPE_PUSH([cpcs have_threads have_ibv_access_so]) > cpcs="oob" > > OMPI_CHECK_OPENIB([btl_openib], > @@ -78,6 +79,19 @@ > AC_MSG_CHECKING([which openib btl cpcs will be built]) > AC_MSG_RESULT([$cpcs])]) > > + # check for Solaris specific memory access flag > + AS_IF([test "$btl_openib_happy" = "yes"], > + [if test "`echo $build_os | $GREP solaris`"; then FWIW, the above could be a single AS_IF: AS_IF([test "$btl_openib_happy" = "yes" -a -n "echo $build_os | $GREP solaris" ], > + AC_TRY_COMPILE([#include <infiniband/verbs.h>], > + [int flag = IBV_ACCESS_SO;], > + [have_ibv_access_so="yes" > + AC_DEFINE_UNQUOTED([HAVE_IBV_ACCESS_SO], > + 1,[openib define HAVE_IBV_ACCESS_SO])], > + [have_ibv_access_so="no"]) > + AC_MSG_CHECKING([for IBV_ACCESS_SO in Solaris]) > + AC_MSG_RESULT([$have_ibv_access_so]) I *believe* you should be able to replace the entire above section with an AC_CHECK_DECLS. Check elsewhere in ompi_check_openib.m4 for uses of AC_CHECK_DECLS looking for the presence of various IBV_* constants. See: http://www.gnu.org/software/hello/manual/autoconf/Generic-Declarations.html > + fi]) > + > # Enable openib device failover. It is disabled by default. > AC_ARG_ENABLE([btl-openib-failover], > [AC_HELP_STRING([--enable-btl-openib-failover], > _______________________________________________ > svn-full mailing list > svn-f...@open-mpi.org > http://www.open-mpi.org/mailman/listinfo.cgi/svn-full -- Jeff Squyres jsquy...@cisco.com For corporate legal information go to: http://www.cisco.com/web/about/doing_business/legal/cri/
