Hello community,
here is the log from the commit of package libfabric for openSUSE:Factory
checked in at 2020-05-20 18:36:03
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Comparing /work/SRC/openSUSE:Factory/libfabric (Old)
and /work/SRC/openSUSE:Factory/.libfabric.new.2738 (New)
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "libfabric"
Wed May 20 18:36:03 2020 rev:21 rq:806814 version:1.10.1
Changes:
--------
--- /work/SRC/openSUSE:Factory/libfabric/fabtests.changes 2020-04-29
20:42:24.979647957 +0200
+++ /work/SRC/openSUSE:Factory/.libfabric.new.2738/fabtests.changes
2020-05-20 18:36:12.136072266 +0200
@@ -1,0 +2,6 @@
+Thu May 14 08:59:09 UTC 2020 - Nicolas Morey-Chaisemartin
<nmoreychaisemar...@suse.com>
+
+- Update to 1.10.1
+ - See NEWS.md for changelog
+
+-------------------------------------------------------------------
libfabric.changes: same change
Old:
----
libfabric-1.10.0.0.88a419cbd07a.tar.bz2
New:
----
libfabric-1.10.1.0.f62ffe4a587e.tar.bz2
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Other differences:
------------------
++++++ fabtests.spec ++++++
--- /var/tmp/diff_new_pack.q21o3y/_old 2020-05-20 18:36:14.164076516 +0200
+++ /var/tmp/diff_new_pack.q21o3y/_new 2020-05-20 18:36:14.168076524 +0200
@@ -16,10 +16,10 @@
#
-%define git_ver .0.88a419cbd07a
+%define git_ver .0.f62ffe4a587e
Name: fabtests
-Version: 1.10.0
+Version: 1.10.1
Release: 0
Summary: Test suite for libfabric API
License: BSD-2-Clause OR GPL-2.0-only
++++++ libfabric.spec ++++++
--- /var/tmp/diff_new_pack.q21o3y/_old 2020-05-20 18:36:14.192076575 +0200
+++ /var/tmp/diff_new_pack.q21o3y/_new 2020-05-20 18:36:14.192076575 +0200
@@ -17,10 +17,10 @@
#
-%define git_ver .0.88a419cbd07a
+%define git_ver .0.f62ffe4a587e
Name: libfabric
-Version: 1.10.0
+Version: 1.10.1
Release: 0
Summary: User-space RDMA Fabric Interfaces
License: GPL-2.0-only OR BSD-2-Clause
++++++ _service ++++++
--- /var/tmp/diff_new_pack.q21o3y/_old 2020-05-20 18:36:14.220076634 +0200
+++ /var/tmp/diff_new_pack.q21o3y/_new 2020-05-20 18:36:14.220076634 +0200
@@ -8,7 +8,7 @@
<param name="versionformat">@PARENT_TAG@.@TAG_OFFSET@.%h</param>
<param name="versionrewrite-pattern">v(.*)</param>
<param name="versionrewrite-replacement">\1</param>
- <param name="revision">88a419cbd07a9c186a3113f1aeab50cfde8a8efe</param>
+ <param name="revision">f62ffe4a587eddc3a579a52306f6a3c5e21dd503</param>
</service>
<service name="recompress" mode="disabled">
<param name="file">libfabric*.tar</param>
++++++ libfabric-1.10.0.0.88a419cbd07a.tar.bz2 ->
libfabric-1.10.1.0.f62ffe4a587e.tar.bz2 ++++++
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/libfabric-1.10.0.0.88a419cbd07a/AUTHORS
new/libfabric-1.10.1.0.f62ffe4a587e/AUTHORS
--- old/libfabric-1.10.0.0.88a419cbd07a/AUTHORS 2020-04-23 16:02:12.000000000
+0200
+++ new/libfabric-1.10.1.0.f62ffe4a587e/AUTHORS 2020-05-07 15:20:25.000000000
+0200
@@ -55,6 +55,7 @@
Holger Hoffstätte <hol...@applied-asynchrony.com>
Honggang Li <ho...@redhat.com>
Howard Pritchard <howa...@lanl.gov>
+Ian Ziemba <ian.zie...@hpe.com>
Ignacio Hernandez <ignacio.hernan...@intel.com>
Ira Weiny <ira.we...@intel.com>
Jaime Arteaga <jaime.a.arteaga.mol...@intel.com>
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/libfabric-1.10.0.0.88a419cbd07a/Makefile.am
new/libfabric-1.10.1.0.f62ffe4a587e/Makefile.am
--- old/libfabric-1.10.0.0.88a419cbd07a/Makefile.am 2020-04-23
16:02:12.000000000 +0200
+++ new/libfabric-1.10.1.0.f62ffe4a587e/Makefile.am 2020-05-07
15:20:25.000000000 +0200
@@ -175,7 +175,7 @@
src_libfabric_la_DEPENDENCIES = libfabric.map
if !EMBEDDED
-src_libfabric_la_LDFLAGS += -version-info 13:0:12
+src_libfabric_la_LDFLAGS += -version-info 14:1:13
endif
src_libfabric_la_LDFLAGS += -export-dynamic \
$(libfabric_version_script)
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/libfabric-1.10.0.0.88a419cbd07a/NEWS.md
new/libfabric-1.10.1.0.f62ffe4a587e/NEWS.md
--- old/libfabric-1.10.0.0.88a419cbd07a/NEWS.md 2020-04-23 16:02:12.000000000
+0200
+++ new/libfabric-1.10.1.0.f62ffe4a587e/NEWS.md 2020-05-07 15:20:25.000000000
+0200
@@ -6,6 +6,37 @@
version 1.0.
+v1.10.1, Fri May 8, 2020
+========================
+
+## Core
+
+- Fixed library version
+
+## EFA
+
+- Allow endpoint to choose shm usage
+- Fix handling of REQ packets
+- Fix logic writing a Tx completion entry
+- Use correct Tx operation flags for msg sends
+
+## Fabtests
+
+- Use pax tar format when creating source packages
+
+## RxD
+
+- Use correct peer address for atomic_inject calls
+
+## SHM
+
+- Fix BSD build failure
+
+## TCP
+
+- Add locking around signaling a wait fd
+
+
v1.10.0, Fri Apr 24, 2020
=========================
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/libfabric-1.10.0.0.88a419cbd07a/configure.ac
new/libfabric-1.10.1.0.f62ffe4a587e/configure.ac
--- old/libfabric-1.10.0.0.88a419cbd07a/configure.ac 2020-04-23
16:02:12.000000000 +0200
+++ new/libfabric-1.10.1.0.f62ffe4a587e/configure.ac 2020-05-07
15:20:25.000000000 +0200
@@ -6,12 +6,12 @@
dnl Process this file with autoconf to produce a configure script.
AC_PREREQ([2.60])
-AC_INIT([libfabric], [1.10.0], [of...@lists.openfabrics.org])
+AC_INIT([libfabric], [1.10.1], [of...@lists.openfabrics.org])
AC_CONFIG_SRCDIR([src/fabric.c])
AC_CONFIG_AUX_DIR(config)
AC_CONFIG_MACRO_DIR(config)
AC_CONFIG_HEADERS(config.h)
-AM_INIT_AUTOMAKE([1.11 dist-bzip2 foreign -Wall -Werror subdir-objects
parallel-tests tar-ustar])
+AM_INIT_AUTOMAKE([1.11 dist-bzip2 foreign -Wall -Werror subdir-objects
parallel-tests tar-pax])
m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])])
m4_include(config/fi_check_package.m4)
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/libfabric-1.10.0.0.88a419cbd07a/fabtests/configure.ac
new/libfabric-1.10.1.0.f62ffe4a587e/fabtests/configure.ac
--- old/libfabric-1.10.0.0.88a419cbd07a/fabtests/configure.ac 2020-04-23
16:02:12.000000000 +0200
+++ new/libfabric-1.10.1.0.f62ffe4a587e/fabtests/configure.ac 2020-05-07
15:20:25.000000000 +0200
@@ -5,11 +5,11 @@
dnl Process this file with autoconf to produce a configure script.
AC_PREREQ(2.57)
-AC_INIT([fabtests], [1.10.0], [of...@lists.openfabrics.org])
+AC_INIT([fabtests], [1.10.1], [of...@lists.openfabrics.org])
AC_CONFIG_AUX_DIR(config)
AC_CONFIG_MACRO_DIR(config)
AC_CONFIG_HEADERS(config.h)
-AM_INIT_AUTOMAKE([1.11 dist-bzip2 foreign -Wall -Werror subdir-objects])
+AM_INIT_AUTOMAKE([1.11 dist-bzip2 foreign -Wall -Werror subdir-objects
tar-pax])
m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])])
AC_CANONICAL_HOST
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/libfabric-1.10.0.0.88a419cbd07a/include/freebsd/osd.h
new/libfabric-1.10.1.0.f62ffe4a587e/include/freebsd/osd.h
--- old/libfabric-1.10.0.0.88a419cbd07a/include/freebsd/osd.h 2020-04-23
16:02:12.000000000 +0200
+++ new/libfabric-1.10.1.0.f62ffe4a587e/include/freebsd/osd.h 2020-05-07
15:20:25.000000000 +0200
@@ -75,6 +75,26 @@
return 0;
}
+static inline ssize_t ofi_process_vm_readv(pid_t pid,
+ const struct iovec *local_iov,
+ unsigned long liovcnt,
+ const struct iovec *remote_iov,
+ unsigned long riovcnt,
+ unsigned long flags)
+{
+ return -FI_ENOSYS;
+}
+
+static inline size_t ofi_process_vm_writev(pid_t pid,
+ const struct iovec *local_iov,
+ unsigned long liovcnt,
+ const struct iovec *remote_iov,
+ unsigned long riovcnt,
+ unsigned long flags)
+{
+ return -FI_ENOSYS;
+}
+
#endif /* _FREEBSD_OSD_H_ */
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/libfabric-1.10.0.0.88a419cbd07a/include/ofi_signal.h
new/libfabric-1.10.1.0.f62ffe4a587e/include/ofi_signal.h
--- old/libfabric-1.10.0.0.88a419cbd07a/include/ofi_signal.h 2020-04-23
16:02:12.000000000 +0200
+++ new/libfabric-1.10.1.0.f62ffe4a587e/include/ofi_signal.h 2020-05-07
15:20:25.000000000 +0200
@@ -45,6 +45,7 @@
#include <ofi_file.h>
#include <ofi_osd.h>
#include <rdma/fi_errno.h>
+#include <ofi_lock.h>
enum {
@@ -53,6 +54,7 @@
};
struct fd_signal {
+ fastlock_t lock;
int rcnt;
int wcnt;
int fd[2];
@@ -70,6 +72,10 @@
if (ret)
goto err;
+ ret = fastlock_init(&signal->lock);
+ if (ret)
+ goto err;
+
return 0;
err:
@@ -82,24 +88,30 @@
{
ofi_close_socket(signal->fd[0]);
ofi_close_socket(signal->fd[1]);
+
+ fastlock_destroy(&signal->lock);
}
static inline void fd_signal_set(struct fd_signal *signal)
{
char c = 0;
+ fastlock_acquire(&signal->lock);
if (signal->wcnt == signal->rcnt) {
if (ofi_write_socket(signal->fd[FI_WRITE_FD], &c, sizeof c) ==
sizeof c)
signal->wcnt++;
}
+ fastlock_release(&signal->lock);
}
static inline void fd_signal_reset(struct fd_signal *signal)
{
char c;
+ fastlock_acquire(&signal->lock);
if (signal->rcnt != signal->wcnt) {
if (ofi_read_socket(signal->fd[FI_READ_FD], &c, sizeof c) ==
sizeof c)
signal->rcnt++;
}
+ fastlock_release(&signal->lock);
}
static inline int fd_signal_poll(struct fd_signal *signal, int timeout)
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/libfabric-1.10.0.0.88a419cbd07a/include/rdma/fabric.h
new/libfabric-1.10.1.0.f62ffe4a587e/include/rdma/fabric.h
--- old/libfabric-1.10.0.0.88a419cbd07a/include/rdma/fabric.h 2020-04-23
16:02:12.000000000 +0200
+++ new/libfabric-1.10.1.0.f62ffe4a587e/include/rdma/fabric.h 2020-05-07
15:20:25.000000000 +0200
@@ -78,7 +78,7 @@
#define FI_MAJOR_VERSION 1
#define FI_MINOR_VERSION 10
-#define FI_REVISION_VERSION 0
+#define FI_REVISION_VERSION 1
enum {
FI_PATH_MAX = 256,
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/libfabric-1.10.0.0.88a419cbd07a/include/windows/config.h
new/libfabric-1.10.1.0.f62ffe4a587e/include/windows/config.h
--- old/libfabric-1.10.0.0.88a419cbd07a/include/windows/config.h
2020-04-23 16:02:12.000000000 +0200
+++ new/libfabric-1.10.1.0.f62ffe4a587e/include/windows/config.h
2020-05-07 15:20:25.000000000 +0200
@@ -165,7 +165,7 @@
#define PACKAGE_TARNAME PACKAGE
/* Define to the version of this package. */
-#define PACKAGE_VERSION "1.10.0"
+#define PACKAGE_VERSION "1.10.1"
/* Define to the full name and version of this package. */
#define PACKAGE_STRING PACKAGE_NAME " " PACKAGE_VERSION
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/libfabric-1.10.0.0.88a419cbd07a/prov/efa/src/efa.h
new/libfabric-1.10.1.0.f62ffe4a587e/prov/efa/src/efa.h
--- old/libfabric-1.10.0.0.88a419cbd07a/prov/efa/src/efa.h 2020-04-23
16:02:12.000000000 +0200
+++ new/libfabric-1.10.1.0.f62ffe4a587e/prov/efa/src/efa.h 2020-05-07
15:20:25.000000000 +0200
@@ -134,14 +134,24 @@
struct efa_ep_addr ep_addr;
};
+/*
+ * Common fields for the beginning of the efa_domain and rxr_domain structures.
+ * This structure must be kept in sync with rxr_domain and efa_domain. This
+ * will be removed when the rxr and efa domain structures are combined.
+ */
+struct efa_domain_base {
+ struct util_domain util_domain;
+ enum efa_domain_type type;
+};
+
struct efa_domain {
struct util_domain util_domain;
+ enum efa_domain_type type;
struct fid_domain *shm_domain;
struct efa_context *ctx;
struct ibv_pd *ibv_pd;
struct fi_info *info;
struct efa_fabric *fab;
- int rdm;
struct ofi_mr_cache cache;
struct efa_qp **qp_table;
size_t qp_table_sz_m1;
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/libfabric-1.10.0.0.88a419cbd07a/prov/efa/src/efa_av.c
new/libfabric-1.10.1.0.f62ffe4a587e/prov/efa/src/efa_av.c
--- old/libfabric-1.10.0.0.88a419cbd07a/prov/efa/src/efa_av.c 2020-04-23
16:02:12.000000000 +0200
+++ new/libfabric-1.10.1.0.f62ffe4a587e/prov/efa/src/efa_av.c 2020-05-07
15:20:25.000000000 +0200
@@ -349,9 +349,11 @@
dlist_foreach(&av->util_av.ep_list, ep_list_entry) {
util_ep = container_of(ep_list_entry, struct util_ep,
av_entry);
rxr_ep = container_of(util_ep, struct rxr_ep, util_ep);
- peer = rxr_ep_get_peer(rxr_ep, *fi_addr);
- peer->shm_fiaddr = shm_fiaddr;
- peer->is_local = 1;
+ if (rxr_ep->use_shm) {
+ peer = rxr_ep_get_peer(rxr_ep, *fi_addr);
+ peer->shm_fiaddr = shm_fiaddr;
+ peer->is_local = 1;
+ }
}
}
HASH_ADD(hh, av->av_map, ep_addr,
@@ -655,6 +657,7 @@
struct efa_domain *efa_domain;
struct util_domain *util_domain;
struct rxr_domain *rxr_domain;
+ struct efa_domain_base *efa_domain_base;
struct efa_av *av;
struct util_av_attr util_attr;
size_t universe_size;
@@ -683,16 +686,19 @@
av = calloc(1, sizeof(*av));
if (!av)
return -FI_ENOMEM;
- /*
- * This needs be revisited once fabric domain is set to efa for both
- * dgram and rdm.For rdm need both efa_domain and rxr_domain (for
shm_domain)
- */
+
util_domain = container_of(domain_fid, struct util_domain,
- domain_fid);
+ domain_fid);
+ efa_domain_base = container_of(util_domain, struct efa_domain_base,
+ util_domain.domain_fid);
attr->type = FI_AV_TABLE;
- if (strstr(util_domain->name, "rdm")) {
- rxr_domain = container_of(domain_fid, struct rxr_domain,
- util_domain.domain_fid);
+ /*
+ * An rxr_domain fid was passed to the user if this is an RDM
+ * endpoint, otherwise it is an efa_domain fid. This will be
+ * removed once the rxr and efa domain structures are combined.
+ */
+ if (efa_domain_base->type == EFA_DOMAIN_RDM) {
+ rxr_domain = (struct rxr_domain *)efa_domain_base;
efa_domain = container_of(rxr_domain->rdm_domain, struct
efa_domain,
util_domain.domain_fid);
av->ep_type = FI_EP_RDM;
@@ -732,9 +738,7 @@
av->shm_rdm_addr_map[i] = FI_ADDR_UNSPEC;
}
} else {
- // Currently the domain is set to efa for only dgram
- efa_domain = container_of(domain_fid, struct efa_domain,
- util_domain.domain_fid);
+ efa_domain = (struct efa_domain *)efa_domain_base;
av->ep_type = FI_EP_DGRAM;
}
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/libfabric-1.10.0.0.88a419cbd07a/prov/efa/src/efa_domain.c
new/libfabric-1.10.1.0.f62ffe4a587e/prov/efa/src/efa_domain.c
--- old/libfabric-1.10.0.0.88a419cbd07a/prov/efa/src/efa_domain.c
2020-04-23 16:02:12.000000000 +0200
+++ new/libfabric-1.10.1.0.f62ffe4a587e/prov/efa/src/efa_domain.c
2020-05-07 15:20:25.000000000 +0200
@@ -81,7 +81,7 @@
if (!ctx_list)
return -errno;
- if (domain->rdm)
+ if (domain->type == EFA_DOMAIN_RDM)
name_len = strlen(name) - strlen(efa_rdm_domain.suffix);
else
name_len = strlen(name) - strlen(efa_dgrm_domain.suffix);
@@ -163,7 +163,10 @@
goto err_close_domain;
}
- domain->rdm = EFA_EP_TYPE_IS_RDM(info);
+ if (EFA_EP_TYPE_IS_RDM(info))
+ domain->type = EFA_DOMAIN_RDM;
+ else
+ domain->type = EFA_DOMAIN_DGRAM;
ret = efa_open_device_by_name(domain, info->domain_attr->name);
if (ret)
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/libfabric-1.10.0.0.88a419cbd07a/prov/efa/src/efa_ep.c
new/libfabric-1.10.1.0.f62ffe4a587e/prov/efa/src/efa_ep.c
--- old/libfabric-1.10.0.0.88a419cbd07a/prov/efa/src/efa_ep.c 2020-04-23
16:02:12.000000000 +0200
+++ new/libfabric-1.10.1.0.f62ffe4a587e/prov/efa/src/efa_ep.c 2020-05-07
15:20:25.000000000 +0200
@@ -420,7 +420,7 @@
attr_ex.cap.max_inline_data = ep->domain->ctx->inline_buf_size;
- if (ep->domain->rdm) {
+ if (ep->domain->type == EFA_DOMAIN_RDM) {
attr_ex.qp_type = IBV_QPT_DRIVER;
attr_ex.comp_mask = IBV_QP_INIT_ATTR_PD |
IBV_QP_INIT_ATTR_SEND_OPS_FLAGS;
attr_ex.send_ops_flags = IBV_QP_EX_WITH_SEND;
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/libfabric-1.10.0.0.88a419cbd07a/prov/efa/src/rxr/rxr.h
new/libfabric-1.10.1.0.f62ffe4a587e/prov/efa/src/rxr/rxr.h
--- old/libfabric-1.10.0.0.88a419cbd07a/prov/efa/src/rxr/rxr.h 2020-04-23
16:02:12.000000000 +0200
+++ new/libfabric-1.10.1.0.f62ffe4a587e/prov/efa/src/rxr/rxr.h 2020-05-07
15:20:25.000000000 +0200
@@ -476,8 +476,14 @@
(*((enum rxr_x_entry_type *) \
((unsigned char *)((pkt_entry)->x_entry))))
+enum efa_domain_type {
+ EFA_DOMAIN_DGRAM = 0,
+ EFA_DOMAIN_RDM,
+};
+
struct rxr_domain {
struct util_domain util_domain;
+ enum efa_domain_type type;
struct fid_domain *rdm_domain;
size_t mtu_size;
size_t addrlen;
@@ -508,6 +514,7 @@
struct fid_cq *rdm_cq;
/* shm provider fid */
+ bool use_shm;
struct fid_ep *shm_ep;
struct fid_cq *shm_cq;
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/libfabric-1.10.0.0.88a419cbd07a/prov/efa/src/rxr/rxr_atomic.c
new/libfabric-1.10.1.0.f62ffe4a587e/prov/efa/src/rxr/rxr_atomic.c
--- old/libfabric-1.10.0.0.88a419cbd07a/prov/efa/src/rxr/rxr_atomic.c
2020-04-23 16:02:12.000000000 +0200
+++ new/libfabric-1.10.1.0.f62ffe4a587e/prov/efa/src/rxr/rxr_atomic.c
2020-05-07 15:20:25.000000000 +0200
@@ -170,7 +170,8 @@
rxr_ep = container_of(ep, struct rxr_ep, util_ep.ep_fid.fid);
peer = rxr_ep_get_peer(rxr_ep, dest_addr);
- if (rxr_env.enable_shm_transfer && peer->is_local) {
+ if (peer->is_local) {
+ assert(rxr_ep->use_shm);
if (!(shm_info->domain_attr->mr_mode & FI_MR_VIRT_ADDR))
remote_addr = 0;
return fi_inject_atomic(rxr_ep->shm_ep, buf, count,
peer->shm_fiaddr,
@@ -215,7 +216,8 @@
rxr_ep = container_of(ep, struct rxr_ep, util_ep.ep_fid.fid);
peer = rxr_ep_get_peer(rxr_ep, msg->addr);
- if (rxr_env.enable_shm_transfer && peer->is_local) {
+ if (peer->is_local) {
+ assert(rxr_ep->use_shm);
rxr_atomic_copy_shm_msg(&shm_msg, msg, rma_iov);
shm_msg.addr = peer->shm_fiaddr;
return fi_atomicmsg(rxr_ep->shm_ep, &shm_msg, flags);
@@ -287,7 +289,8 @@
rxr_ep = container_of(ep, struct rxr_ep, util_ep.ep_fid.fid);
peer = rxr_ep_get_peer(rxr_ep, msg->addr);
- if (rxr_env.enable_shm_transfer && peer->is_local) {
+ if (peer->is_local) {
+ assert(rxr_ep->use_shm);
rxr_atomic_copy_shm_msg(&shm_msg, msg, rma_iov);
shm_msg.addr = peer->shm_fiaddr;
return fi_fetch_atomicmsg(rxr_ep->shm_ep, &shm_msg,
@@ -368,7 +371,8 @@
rxr_ep = container_of(ep, struct rxr_ep, util_ep.ep_fid.fid);
peer = rxr_ep_get_peer(rxr_ep, msg->addr);
- if (rxr_env.enable_shm_transfer && peer->is_local) {
+ if (peer->is_local) {
+ assert(rxr_ep->use_shm);
rxr_atomic_copy_shm_msg(&shm_msg, msg, rma_iov);
shm_msg.addr = peer->shm_fiaddr;
return fi_compare_atomicmsg(rxr_ep->shm_ep, &shm_msg,
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/libfabric-1.10.0.0.88a419cbd07a/prov/efa/src/rxr/rxr_cq.c
new/libfabric-1.10.1.0.f62ffe4a587e/prov/efa/src/rxr/rxr_cq.c
--- old/libfabric-1.10.0.0.88a419cbd07a/prov/efa/src/rxr/rxr_cq.c
2020-04-23 16:02:12.000000000 +0200
+++ new/libfabric-1.10.1.0.f62ffe4a587e/prov/efa/src/rxr/rxr_cq.c
2020-05-07 15:20:25.000000000 +0200
@@ -724,14 +724,34 @@
}
}
+static inline
+bool rxr_cq_need_tx_completion(struct rxr_ep *ep,
+ struct rxr_tx_entry *tx_entry)
+
+{
+ if (tx_entry->fi_flags & RXR_NO_COMPLETION)
+ return false;
+
+ /*
+ * ep->util_ep.tx_msg_flags is either 0 or FI_COMPLETION, depend on
+ * whether app specfied FI_SELECTIVE_COMPLETION when binding CQ.
+ * (ep->util_ep.tx_msg_flags was set in ofi_ep_bind_cq())
+ *
+ * If tx_msg_flags is 0, we only write completion when app specify
+ * FI_COMPLETION in flags.
+ */
+ return ep->util_ep.tx_msg_flags == FI_COMPLETION ||
+ tx_entry->fi_flags & FI_COMPLETION;
+}
+
+
void rxr_cq_write_tx_completion(struct rxr_ep *ep,
struct rxr_tx_entry *tx_entry)
{
struct util_cq *tx_cq = ep->util_ep.tx_cq;
int ret;
- if (!(tx_entry->fi_flags & RXR_NO_COMPLETION) &&
- ofi_need_completion(rxr_tx_flags(ep), tx_entry->fi_flags)) {
+ if (rxr_cq_need_tx_completion(ep, tx_entry)) {
FI_DBG(&rxr_prov, FI_LOG_CQ,
"Writing send completion for tx_entry to peer: %" PRIu64
" tx_id: %" PRIu32 " msg_id: %" PRIu32 " tag: %lx len: %"
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/libfabric-1.10.0.0.88a419cbd07a/prov/efa/src/rxr/rxr_domain.c
new/libfabric-1.10.1.0.f62ffe4a587e/prov/efa/src/rxr/rxr_domain.c
--- old/libfabric-1.10.0.0.88a419cbd07a/prov/efa/src/rxr/rxr_domain.c
2020-04-23 16:02:12.000000000 +0200
+++ new/libfabric-1.10.1.0.f62ffe4a587e/prov/efa/src/rxr/rxr_domain.c
2020-05-07 15:20:25.000000000 +0200
@@ -190,6 +190,8 @@
if (!rxr_domain)
return -FI_ENOMEM;
+ rxr_domain->type = EFA_DOMAIN_RDM;
+
ret = rxr_get_lower_rdm_info(fabric->api_version, NULL, NULL, 0,
&rxr_util_prov, info, &rdm_info);
if (ret)
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/libfabric-1.10.0.0.88a419cbd07a/prov/efa/src/rxr/rxr_ep.c
new/libfabric-1.10.1.0.f62ffe4a587e/prov/efa/src/rxr/rxr_ep.c
--- old/libfabric-1.10.0.0.88a419cbd07a/prov/efa/src/rxr/rxr_ep.c
2020-04-23 16:02:12.000000000 +0200
+++ new/libfabric-1.10.1.0.f62ffe4a587e/prov/efa/src/rxr/rxr_ep.c
2020-05-07 15:20:25.000000000 +0200
@@ -632,7 +632,7 @@
tx_entry_entry);
rxr_release_tx_entry(rxr_ep, tx_entry);
}
- if (rxr_env.enable_shm_transfer) {
+ if (rxr_ep->use_shm) {
dlist_foreach_safe(&rxr_ep->rx_posted_buf_shm_list, entry, tmp)
{
pkt = container_of(entry, struct rxr_pkt_entry,
dbg_entry);
ofi_buf_free(pkt);
@@ -664,7 +664,7 @@
if (rxr_ep->tx_pkt_efa_pool)
ofi_bufpool_destroy(rxr_ep->tx_pkt_efa_pool);
- if (rxr_env.enable_shm_transfer) {
+ if (rxr_ep->use_shm) {
if (rxr_ep->rx_pkt_shm_pool)
ofi_bufpool_destroy(rxr_ep->rx_pkt_shm_pool);
@@ -693,7 +693,7 @@
}
/* Close shm provider's endpoint and cq */
- if (rxr_env.enable_shm_transfer) {
+ if (rxr_ep->use_shm) {
ret = fi_close(&rxr_ep->shm_ep->fid);
if (ret) {
FI_WARN(&rxr_prov, FI_LOG_EP_CTRL, "Unable to close shm
EP\n");
@@ -757,7 +757,7 @@
return -FI_ENOMEM;
/* Bind shm provider endpoint & shm av */
- if (rxr_env.enable_shm_transfer) {
+ if (rxr_ep->use_shm) {
ret = fi_ep_bind(rxr_ep->shm_ep, &av->shm_rdm_av->fid,
flags);
if (ret)
return ret;
@@ -876,7 +876,7 @@
* In this way, each peer is able to open and map to other
local peers'
* shared memory region.
*/
- if (rxr_env.enable_shm_transfer) {
+ if (ep->use_shm) {
ret = rxr_ep_efa_addr_to_str(ep->core_addr,
shm_ep_name);
if (ret < 0)
goto out;
@@ -1171,7 +1171,7 @@
goto err_free_rx_entry_pool;
/* create pkt pool for shm */
- if (rxr_env.enable_shm_transfer) {
+ if (ep->use_shm) {
ret = ofi_bufpool_create(&ep->tx_pkt_shm_pool,
entry_sz,
RXR_BUF_POOL_ALIGNMENT,
@@ -1293,7 +1293,7 @@
}
/* bulk post recv buf for shm provider */
flags = FI_MORE;
- while (rxr_env.enable_shm_transfer && ep->rx_bufs_shm_to_post) {
+ while (ep->use_shm && ep->rx_bufs_shm_to_post) {
if (ep->rx_bufs_shm_to_post == 1)
flags = 0;
ret = rxr_ep_post_buf(ep, flags, SHM_EP);
@@ -1315,8 +1315,7 @@
dlist_foreach_container_safe(pkts, struct rxr_pkt_entry,
pkt_entry, entry, tmp) {
- if (rxr_env.enable_shm_transfer &&
- rxr_ep_get_peer(ep, pkt_entry->addr)->is_local)
{
+ if (ep->use_shm && rxr_ep_get_peer(ep,
pkt_entry->addr)->is_local) {
dlist_remove(&pkt_entry->entry);
continue;
}
@@ -1435,7 +1434,7 @@
rxr_ep_poll_cq(ep, ep->rdm_cq, rxr_env.efa_cq_read_size, 0);
// Poll the SHM completion queue if enabled
- if (rxr_env.enable_shm_transfer)
+ if (ep->use_shm)
rxr_ep_poll_cq(ep, ep->shm_cq, rxr_env.shm_cq_read_size, 1);
ret = rxr_ep_bulk_post_recv(ep);
@@ -1578,6 +1577,28 @@
fastlock_release(&ep->util_ep.lock);
}
+static
+bool rxr_ep_use_shm(struct fi_info *info)
+{
+ /* App provided hints supercede environmental variables.
+ *
+ * Using the shm provider comes with some overheads, particularly in the
+ * progress engine when polling an empty completion queue, so avoid
+ * initializing the provider if the app provides a hint that it does not
+ * require node-local communication. We can still loopback over the EFA
+ * device in cases where the app violates the hint and continues
+ * communicating with node-local peers.
+ */
+ if (info
+ /* If the app requires explicitly remote communication */
+ && (info->caps & FI_REMOTE_COMM)
+ /* but not local communication */
+ && !(info->caps & FI_LOCAL_COMM))
+ return 0;
+
+ return rxr_env.enable_shm_transfer;
+}
+
int rxr_endpoint(struct fid_domain *domain, struct fi_info *info,
struct fid_ep **ep, void *context)
{
@@ -1618,8 +1639,10 @@
efa_domain = container_of(rxr_domain->rdm_domain, struct efa_domain,
util_domain.domain_fid);
- /* Open shm provider's endpoint */
- if (rxr_env.enable_shm_transfer) {
+
+ rxr_ep->use_shm = rxr_ep_use_shm(info);
+ if (rxr_ep->use_shm) {
+ /* Open shm provider's endpoint */
assert(!strcmp(shm_info->fabric_attr->name, "shm"));
ret = fi_endpoint(efa_domain->shm_domain, shm_info,
&rxr_ep->shm_ep, rxr_ep);
@@ -1690,7 +1713,7 @@
goto err_close_core_cq;
/* Bind ep with shm provider's cq */
- if (rxr_env.enable_shm_transfer) {
+ if (rxr_ep->use_shm) {
ret = fi_cq_open(efa_domain->shm_domain, &cq_attr,
&rxr_ep->shm_cq, rxr_ep);
if (ret)
@@ -1717,7 +1740,7 @@
return 0;
err_close_shm_cq:
- if (rxr_env.enable_shm_transfer && rxr_ep->shm_cq) {
+ if (rxr_ep->use_shm && rxr_ep->shm_cq) {
retv = fi_close(&rxr_ep->shm_cq->fid);
if (retv)
FI_WARN(&rxr_prov, FI_LOG_CQ, "Unable to close shm cq:
%s\n",
@@ -1729,7 +1752,7 @@
FI_WARN(&rxr_prov, FI_LOG_CQ, "Unable to close cq: %s\n",
fi_strerror(-retv));
err_close_shm_ep:
- if (rxr_env.enable_shm_transfer && rxr_ep->shm_ep) {
+ if (rxr_ep->use_shm && rxr_ep->shm_ep) {
retv = fi_close(&rxr_ep->shm_ep->fid);
if (retv)
FI_WARN(&rxr_prov, FI_LOG_EP_CTRL, "Unable to close shm
EP: %s\n",
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/libfabric-1.10.0.0.88a419cbd07a/prov/efa/src/rxr/rxr_init.c
new/libfabric-1.10.1.0.f62ffe4a587e/prov/efa/src/rxr/rxr_init.c
--- old/libfabric-1.10.0.0.88a419cbd07a/prov/efa/src/rxr/rxr_init.c
2020-04-23 16:02:12.000000000 +0200
+++ new/libfabric-1.10.1.0.f62ffe4a587e/prov/efa/src/rxr/rxr_init.c
2020-05-07 15:20:25.000000000 +0200
@@ -538,20 +538,6 @@
if (hints && hints->ep_attr && hints->ep_attr->type == FI_EP_DGRAM)
goto dgram_info;
- /*
- * Using the shm provider comes with some overheads, particularly in the
- * progress engine when polling an empty completion queue, so avoid
- * initializing the provider if the app provides a hint that it does not
- * require node-local communication. We can still loopback over the EFA
- * device in cases where the app violates the hint and continues
- * communicating with node-local peers.
- */
- if (hints
- /* If the app requires explicitly remote communication */
- && (hints->caps & FI_REMOTE_COMM)
- /* but not local communication */
- && !(hints->caps & FI_LOCAL_COMM))
- rxr_env.enable_shm_transfer = 0;
ret = rxr_get_lower_rdm_info(version, node, service, flags,
&rxr_util_prov, hints, &core_info);
@@ -578,6 +564,15 @@
goto free_info;
ofi_alter_info(util_info, hints, version);
+
+ /* If application asked for FI_REMOTE_COMM but not
FI_LOCAL_COMM, it
+ * does not want to use shm. In this case, we honor the request
by
+ * unsetting the FI_LOCAL_COMM flag in info. This way
rxr_endpoint()
+ * should disable shm transfer for the endpoint
+ */
+ if (hints && hints->caps & FI_REMOTE_COMM && !(hints->caps &
FI_LOCAL_COMM))
+ util_info->caps &= ~FI_LOCAL_COMM;
+
if (!*info)
*info = util_info;
else
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/libfabric-1.10.0.0.88a419cbd07a/prov/efa/src/rxr/rxr_msg.c
new/libfabric-1.10.1.0.f62ffe4a587e/prov/efa/src/rxr/rxr_msg.c
--- old/libfabric-1.10.0.0.88a419cbd07a/prov/efa/src/rxr/rxr_msg.c
2020-04-23 16:02:12.000000000 +0200
+++ new/libfabric-1.10.1.0.f62ffe4a587e/prov/efa/src/rxr/rxr_msg.c
2020-05-07 15:20:25.000000000 +0200
@@ -83,7 +83,9 @@
RXR_EAGER_MSGRTM_PKT +
tagged);
peer = rxr_ep_get_peer(rxr_ep, tx_entry->addr);
+
if (peer->is_local) {
+ assert(rxr_ep->use_shm);
/* intra instance message */
int rtm_type = (tx_entry->total_len <= max_rtm_data_size) ?
RXR_EAGER_MSGRTM_PKT
:
RXR_READ_MSGRTM_PKT;
@@ -194,6 +196,7 @@
void **desc, size_t count, fi_addr_t dest_addr,
void *context)
{
+ struct rxr_ep *rxr_ep;
struct fi_msg msg;
memset(&msg, 0, sizeof(msg));
@@ -203,7 +206,8 @@
msg.addr = dest_addr;
msg.context = context;
- return rxr_msg_sendmsg(ep, &msg, 0);
+ rxr_ep = container_of(ep, struct rxr_ep, util_ep.ep_fid.fid);
+ return rxr_msg_sendmsg(ep, &msg, rxr_tx_flags(rxr_ep));
}
static
@@ -224,6 +228,7 @@
{
struct fi_msg msg;
struct iovec iov;
+ struct rxr_ep *rxr_ep;
iov.iov_base = (void *)buf;
iov.iov_len = len;
@@ -236,16 +241,16 @@
msg.context = context;
msg.data = data;
- return rxr_msg_generic_send(ep, &msg, 0, ofi_op_msg, FI_REMOTE_CQ_DATA);
+ rxr_ep = container_of(ep, struct rxr_ep, util_ep.ep_fid.fid);
+ return rxr_msg_generic_send(ep, &msg, 0, ofi_op_msg,
+ rxr_tx_flags(rxr_ep) | FI_REMOTE_CQ_DATA);
}
static
ssize_t rxr_msg_inject(struct fid_ep *ep, const void *buf, size_t len,
fi_addr_t dest_addr)
{
-#if ENABLE_DEBUG
struct rxr_ep *rxr_ep;
-#endif
struct fi_msg msg;
struct iovec iov;
@@ -257,13 +262,11 @@
msg.iov_count = 1;
msg.addr = dest_addr;
-#if ENABLE_DEBUG
rxr_ep = container_of(ep, struct rxr_ep, util_ep.ep_fid.fid);
assert(len <= rxr_ep->core_inject_size - sizeof(struct
rxr_eager_msgrtm_hdr));
-#endif
return rxr_msg_generic_send(ep, &msg, 0, ofi_op_msg,
- RXR_NO_COMPLETION | FI_INJECT);
+ rxr_tx_flags(rxr_ep) | RXR_NO_COMPLETION |
FI_INJECT);
}
static
@@ -271,9 +274,7 @@
size_t len, uint64_t data,
fi_addr_t dest_addr)
{
-#if ENABLE_DEBUG
struct rxr_ep *rxr_ep;
-#endif
struct fi_msg msg;
struct iovec iov;
@@ -286,7 +287,6 @@
msg.addr = dest_addr;
msg.data = data;
-#if ENABLE_DEBUG
rxr_ep = container_of(ep, struct rxr_ep, util_ep.ep_fid.fid);
/*
* We advertise the largest possible inject size with no cq data or
@@ -294,10 +294,9 @@
* providers inject for this send.
*/
assert(len <= rxr_ep->core_inject_size - sizeof(struct
rxr_eager_msgrtm_hdr));
-#endif
-
return rxr_msg_generic_send(ep, &msg, 0, ofi_op_msg,
- RXR_NO_COMPLETION | FI_REMOTE_CQ_DATA |
FI_INJECT);
+ rxr_tx_flags(rxr_ep) | RXR_NO_COMPLETION |
+ FI_REMOTE_CQ_DATA | FI_INJECT);
}
/**
@@ -315,7 +314,6 @@
msg.addr = tmsg->addr;
msg.context = tmsg->context;
msg.data = tmsg->data;
-
return rxr_msg_generic_send(ep_fid, &msg, tmsg->tag, ofi_op_tagged,
flags);
}
@@ -324,6 +322,7 @@
void **desc, size_t count, fi_addr_t dest_addr,
uint64_t tag, void *context)
{
+ struct rxr_ep *rxr_ep;
struct fi_msg_tagged msg;
memset(&msg, 0, sizeof(msg));
@@ -334,7 +333,8 @@
msg.context = context;
msg.tag = tag;
- return rxr_msg_tsendmsg(ep_fid, &msg, 0);
+ rxr_ep = container_of(ep_fid, struct rxr_ep, util_ep.ep_fid.fid);
+ return rxr_msg_tsendmsg(ep_fid, &msg, rxr_tx_flags(rxr_ep));
}
static
@@ -347,7 +347,7 @@
msg_iov.iov_base = (void *)buf;
msg_iov.iov_len = len;
return rxr_msg_tsendv(ep_fid, &msg_iov, &desc, 1, dest_addr, tag,
- context);
+ context);
}
static
@@ -357,6 +357,7 @@
{
struct fi_msg msg;
struct iovec iov;
+ struct rxr_ep *rxr_ep;
iov.iov_base = (void *)buf;
iov.iov_len = len;
@@ -368,17 +369,16 @@
msg.context = context;
msg.data = data;
+ rxr_ep = container_of(ep_fid, struct rxr_ep, util_ep.ep_fid.fid);
return rxr_msg_generic_send(ep_fid, &msg, tag, ofi_op_tagged,
- FI_REMOTE_CQ_DATA);
+ rxr_tx_flags(rxr_ep) | FI_REMOTE_CQ_DATA);
}
static
ssize_t rxr_msg_tinject(struct fid_ep *ep_fid, const void *buf, size_t len,
fi_addr_t dest_addr, uint64_t tag)
{
-#if ENABLE_DEBUG
struct rxr_ep *rxr_ep;
-#endif
struct fi_msg msg;
struct iovec iov;
@@ -390,22 +390,18 @@
msg.iov_count = 1;
msg.addr = dest_addr;
-#if ENABLE_DEBUG
rxr_ep = container_of(ep_fid, struct rxr_ep, util_ep.ep_fid.fid);
assert(len <= rxr_ep->core_inject_size - sizeof(struct
rxr_eager_tagrtm_hdr));
-#endif
return rxr_msg_generic_send(ep_fid, &msg, tag, ofi_op_tagged,
- RXR_NO_COMPLETION | FI_INJECT);
+ rxr_tx_flags(rxr_ep) | RXR_NO_COMPLETION |
FI_INJECT);
}
static
ssize_t rxr_msg_tinjectdata(struct fid_ep *ep_fid, const void *buf, size_t len,
uint64_t data, fi_addr_t dest_addr, uint64_t tag)
{
-#if ENABLE_DEBUG
struct rxr_ep *rxr_ep;
-#endif
struct fi_msg msg;
struct iovec iov;
@@ -418,7 +414,6 @@
msg.addr = dest_addr;
msg.data = data;
-#if ENABLE_DEBUG
rxr_ep = container_of(ep_fid, struct rxr_ep, util_ep.ep_fid.fid);
/*
* We advertise the largest possible inject size with no cq data or
@@ -426,10 +421,10 @@
* providers inject for this send.
*/
assert(len <= rxr_ep->core_inject_size - sizeof(struct
rxr_eager_tagrtm_hdr));
-#endif
return rxr_msg_generic_send(ep_fid, &msg, tag, ofi_op_tagged,
- RXR_NO_COMPLETION | FI_REMOTE_CQ_DATA |
FI_INJECT);
+ rxr_tx_flags(rxr_ep) | RXR_NO_COMPLETION |
+ FI_REMOTE_CQ_DATA | FI_INJECT);
}
/**
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/libfabric-1.10.0.0.88a419cbd07a/prov/efa/src/rxr/rxr_pkt_cmd.c
new/libfabric-1.10.1.0.f62ffe4a587e/prov/efa/src/rxr/rxr_pkt_cmd.c
--- old/libfabric-1.10.0.0.88a419cbd07a/prov/efa/src/rxr/rxr_pkt_cmd.c
2020-04-23 16:02:12.000000000 +0200
+++ new/libfabric-1.10.1.0.f62ffe4a587e/prov/efa/src/rxr/rxr_pkt_cmd.c
2020-05-07 15:20:25.000000000 +0200
@@ -257,10 +257,12 @@
}
peer = rxr_ep_get_peer(rxr_ep, addr);
- if (peer->is_local)
+ if (peer->is_local) {
+ assert(rxr_ep->use_shm);
pkt_entry = rxr_pkt_entry_alloc(rxr_ep,
rxr_ep->tx_pkt_shm_pool);
- else
+ } else {
pkt_entry = rxr_pkt_entry_alloc(rxr_ep,
rxr_ep->tx_pkt_efa_pool);
+ }
if (!pkt_entry)
return -FI_EAGAIN;
@@ -492,6 +494,16 @@
assert(pkt_entry->pkt_size > 0);
base_hdr = rxr_get_base_hdr(pkt_entry->pkt);
+ if (base_hdr->type >= RXR_EXTRA_REQ_PKT_END) {
+ FI_WARN(&rxr_prov, FI_LOG_CQ,
+ "Peer %d is requesting feature %d, which this EP does
not support.\n",
+ (int)src_addr, base_hdr->type);
+
+ assert(0 && "invalid REQ packe type");
+ rxr_cq_handle_cq_error(ep, -FI_EIO);
+ return;
+ }
+
if (base_hdr->type >= RXR_REQ_PKT_BEGIN) {
rxr_pkt_proc_req_common_hdr(pkt_entry);
assert(pkt_entry->hdr_size > 0);
@@ -520,10 +532,12 @@
if (!(peer->flags & RXR_PEER_HANDSHAKE_SENT))
rxr_pkt_post_handshake(ep, peer, pkt_entry->addr);
- if (rxr_env.enable_shm_transfer && peer->is_local)
+ if (peer->is_local) {
+ assert(ep->use_shm);
ep->posted_bufs_shm--;
- else
+ } else {
ep->posted_bufs_efa--;
+ }
switch (base_hdr->type) {
case RXR_RETIRED_RTS_PKT:
@@ -583,19 +597,11 @@
rxr_pkt_handle_rtr_recv(ep, pkt_entry);
return;
default:
- if (base_hdr->type >= RXR_EXTRA_REQ_PKT_END) {
- FI_WARN(&rxr_prov, FI_LOG_CQ,
- "Peer %d is requesting feature %d, which this
EP does not support.\n"
- "The EP informed the peer about this issue via
a handshake packet.\n"
- "This packet is therefore ignored\n",
(int)pkt_entry->addr, base_hdr->type);
- } else {
- FI_WARN(&rxr_prov, FI_LOG_CQ,
- "invalid control pkt type %d\n",
- rxr_get_base_hdr(pkt_entry->pkt)->type);
- assert(0 && "invalid control pkt type");
- rxr_cq_handle_cq_error(ep, -FI_EIO);
- }
-
+ FI_WARN(&rxr_prov, FI_LOG_CQ,
+ "invalid control pkt type %d\n",
+ rxr_get_base_hdr(pkt_entry->pkt)->type);
+ assert(0 && "invalid control pkt type");
+ rxr_cq_handle_cq_error(ep, -FI_EIO);
return;
}
}
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/libfabric-1.10.0.0.88a419cbd07a/prov/efa/src/rxr/rxr_pkt_entry.c
new/libfabric-1.10.1.0.f62ffe4a587e/prov/efa/src/rxr/rxr_pkt_entry.c
--- old/libfabric-1.10.0.0.88a419cbd07a/prov/efa/src/rxr/rxr_pkt_entry.c
2020-04-23 16:02:12.000000000 +0200
+++ new/libfabric-1.10.1.0.f62ffe4a587e/prov/efa/src/rxr/rxr_pkt_entry.c
2020-05-07 15:20:25.000000000 +0200
@@ -284,7 +284,8 @@
rxr_pkt_print("Sent", ep, (struct rxr_base_hdr *)pkt_entry->pkt);
#endif
#endif
- if (rxr_env.enable_shm_transfer && peer->is_local) {
+ if (peer->is_local) {
+ assert(ep->use_shm);
ret = fi_sendmsg(ep->shm_ep, msg, flags);
} else {
ret = fi_sendmsg(ep->rdm_ep, msg, flags);
@@ -307,7 +308,7 @@
msg.desc = desc;
msg.iov_count = count;
peer = rxr_ep_get_peer(ep, addr);
- msg.addr = peer->is_local ? peer->shm_fiaddr : addr;
+ msg.addr = (peer->is_local) ? peer->shm_fiaddr : addr;
msg.context = pkt_entry;
msg.data = 0;
@@ -325,10 +326,12 @@
iov.iov_base = rxr_pkt_start(pkt_entry);
iov.iov_len = pkt_entry->pkt_size;
- if (rxr_ep_get_peer(ep, addr)->is_local)
+ if (rxr_ep_get_peer(ep, addr)->is_local) {
+ assert(ep->use_shm);
desc = NULL;
- else
+ } else {
desc = rxr_ep_mr_local(ep) ? fi_mr_desc(pkt_entry->mr) : NULL;
+ }
return rxr_pkt_entry_sendv(ep, pkt_entry, addr, &iov, &desc, 1, flags);
}
@@ -349,7 +352,7 @@
/* currently only EOR packet is injected using shm ep */
peer = rxr_ep_get_peer(ep, addr);
assert(peer);
- assert(rxr_env.enable_shm_transfer && peer->is_local);
+ assert(ep->use_shm && peer->is_local);
return fi_inject(ep->shm_ep, rxr_pkt_start(pkt_entry),
pkt_entry->pkt_size,
peer->shm_fiaddr);
}
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/libfabric-1.10.0.0.88a419cbd07a/prov/efa/src/rxr/rxr_pkt_type_req.c
new/libfabric-1.10.1.0.f62ffe4a587e/prov/efa/src/rxr/rxr_pkt_type_req.c
--- old/libfabric-1.10.0.0.88a419cbd07a/prov/efa/src/rxr/rxr_pkt_type_req.c
2020-04-23 16:02:12.000000000 +0200
+++ new/libfabric-1.10.1.0.f62ffe4a587e/prov/efa/src/rxr/rxr_pkt_type_req.c
2020-05-07 15:20:25.000000000 +0200
@@ -204,8 +204,10 @@
peer = rxr_ep_get_peer(ep, addr);
assert(peer);
- if (rxr_env.enable_shm_transfer && peer->is_local)
+ if (peer->is_local) {
+ assert(ep->use_shm);
return rxr_env.shm_max_medium_size;
+ }
int max_hdr_size = REQ_INF_LIST[pkt_type].base_hdr_size
+ sizeof(struct rxr_req_opt_raw_addr_hdr)
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/libfabric-1.10.0.0.88a419cbd07a/prov/efa/src/rxr/rxr_rma.c
new/libfabric-1.10.1.0.f62ffe4a587e/prov/efa/src/rxr/rxr_rma.c
--- old/libfabric-1.10.0.0.88a419cbd07a/prov/efa/src/rxr/rxr_rma.c
2020-04-23 16:02:12.000000000 +0200
+++ new/libfabric-1.10.1.0.f62ffe4a587e/prov/efa/src/rxr/rxr_rma.c
2020-05-07 15:20:25.000000000 +0200
@@ -312,7 +312,8 @@
assert(peer);
use_lower_ep_read = false;
- if (rxr_env.enable_shm_transfer && peer->is_local) {
+ if (peer->is_local) {
+ assert(rxr_ep->use_shm);
use_lower_ep_read = true;
} else if (efa_both_support_rdma_read(rxr_ep, peer)) {
/* efa_both_support_rdma_read also check
rxr_env.use_device_rdma,
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/libfabric-1.10.0.0.88a419cbd07a/prov/rxd/src/rxd_atomic.c
new/libfabric-1.10.1.0.f62ffe4a587e/prov/rxd/src/rxd_atomic.c
--- old/libfabric-1.10.0.0.88a419cbd07a/prov/rxd/src/rxd_atomic.c
2020-04-23 16:02:12.000000000 +0200
+++ new/libfabric-1.10.1.0.f62ffe4a587e/prov/rxd/src/rxd_atomic.c
2020-05-07 15:20:25.000000000 +0200
@@ -235,7 +235,7 @@
if (ofi_cirque_isfull(rxd_ep->util_ep.tx_cq->cirq))
goto out;
- rxd_addr = rxd_ep_av(rxd_ep)->fi_addr_table[addr];
+ rxd_addr = rxd_ep_av(rxd_ep)->fi_addr_table[dest_addr];
ret = rxd_send_rts_if_needed(rxd_ep, rxd_addr);
if (ret)
goto out;
