When mlx4 is not compiled directly as an independent shared object (e.g.
CONFIG_RTE_BUILD_SHARED_LIB not enabled for performance reasons), DPDK
applications inherit its dependencies on libibverbs and libmlx4 through
rte.app.mk.
This is an issue both when DPDK is delivered as a binary package (Linux
distributions) and for end users because rdma-core then propagates as a
mandatory dependency for everything.
Application writers relying on binary DPDK packages are not necessarily
aware of this fact and may end up delivering packages with broken
dependencies.
This patch therefore introduces an intermediate internal plug-in
hard-linked with rdma-core (to preserve symbol versioning) loaded by the
PMD through dlopen(), so that a missing rdma-core does not cause unresolved
symbols, allowing applications to start normally.
Signed-off-by: Adrien Mazarguil <adrien.mazarg...@6wind.com>
---
drivers/net/mlx4/Makefile | 40 +++++++++++++++++++++
drivers/net/mlx4/mlx4.c | 80 ++++++++++++++++++++++++++++++++++++++++++
mk/rte.app.mk | 2 +-
3 files changed, 121 insertions(+), 1 deletion(-)
diff --git a/drivers/net/mlx4/Makefile b/drivers/net/mlx4/Makefile
index 1d33c38ed..2ebe61e90 100644
--- a/drivers/net/mlx4/Makefile
+++ b/drivers/net/mlx4/Makefile
@@ -38,7 +38,11 @@ LIB = librte_pmd_mlx4.a
SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4.c
SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4_ethdev.c
SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4_flow.c
+ifneq ($(CONFIG_RTE_BUILD_SHARED_LIB),y)
+SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4_glue_lib.c
+else
SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4_glue.c
+endif
SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4_intr.c
SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4_mr.c
SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4_rxq.c
@@ -55,7 +59,12 @@ CFLAGS += -D_BSD_SOURCE
CFLAGS += -D_DEFAULT_SOURCE
CFLAGS += -D_XOPEN_SOURCE=600
CFLAGS += $(WERROR_FLAGS)
+ifneq ($(CONFIG_RTE_BUILD_SHARED_LIB),y)
+CFLAGS_mlx4_glue.o += -fPIC
+LDLIBS += -ldl
+else
LDLIBS += -libverbs -lmlx4
+endif
LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs
LDLIBS += -lrte_bus_pci
@@ -109,7 +118,38 @@ mlx4_autoconf.h: mlx4_autoconf.h.new
$(SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD):.c=.o): mlx4_autoconf.h
+# Generate dependency plug-in for rdma-core when the PMD cannot be linked
+# directly, so that applications do not inherit this dependency.
+
+ifneq ($(CONFIG_RTE_BUILD_SHARED_LIB),y)
+
+mlx4_glue_lib.c: mlx4_glue_lib.so
+ $Q printf '#include <stddef.h>\n' > $@
+ $Q printf '#include <stdint.h>\n\n' >> $@
+ $Q printf 'const uint8_t mlx4_glue_lib[][16] = {\n' >> $@
+ $Q od -vt x1 $< | \
+ sed -ne '/^[[:xdigit:]]\{1,\}/{' \
+ -e 's///;' \
+ -e '/^$$/d; ' \
+ -e 's/[[:space:]]*$$//;' \
+ -e 's/[[:space:]]\{1,\}/\\x/g;' \
+ -e 's/^/ "/;' \
+ -e 's/$$/",/;' \
+ -e 'p;' \
+ -e '}' >> $@
+ $Q printf '};\n\n' >> $@
+ $Q printf 'const size_t mlx4_glue_lib_size = %u;' $$(wc -c < $<) >> $@
+
+mlx4_glue_lib.so: mlx4_glue.o
+ $Q $(LD) $(LDFLAGS) $(EXTRA_LDFLAGS) \
+ -s -shared -o $@ $< -libverbs -lmlx4
+
+mlx4_glue.o: mlx4_autoconf.h
+
+endif
+
clean_mlx4: FORCE
$Q rm -f -- mlx4_autoconf.h mlx4_autoconf.h.new
+ $Q rm -f -- mlx4_glue.o mlx4_glue_lib.*
clean: clean_mlx4
diff --git a/drivers/net/mlx4/mlx4.c b/drivers/net/mlx4/mlx4.c
index 0fd9a999c..c173fbf56 100644
--- a/drivers/net/mlx4/mlx4.c
+++ b/drivers/net/mlx4/mlx4.c
@@ -37,6 +37,7 @@
*/
#include <assert.h>
+#include <dlfcn.h>
#include <errno.h>
#include <inttypes.h>
#include <stddef.h>
@@ -44,6 +45,7 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
+#include <unistd.h>
/* Verbs headers do not support -pedantic. */
#ifdef PEDANTIC
@@ -55,6 +57,7 @@
#endif
#include <rte_common.h>
+#include <rte_config.h>
#include <rte_dev.h>
#include <rte_errno.h>
#include <rte_ethdev_driver.h>
@@ -724,6 +727,78 @@ static struct rte_pci_driver mlx4_driver = {
RTE_PCI_DRV_INTR_RMV,
};
+#ifndef RTE_BUILD_SHARED_LIB
+
+extern const uint8_t mlx4_glue_lib[][16];
+extern const size_t mlx4_glue_lib_size;
+
+/**
+ * Initialization routine for run-time dependency on rdma-core.
+ */
+static int
+mlx4_glue_init(void)
+{
+ char file[] = "/tmp/" MLX4_DRIVER_NAME "_XXXXXX";
+ int fd = mkstemp(file);
+ size_t off = 0;
+ void *handle = NULL;
+ void **sym;
+ const char *dlmsg;
+
+ if (fd == -1) {
+ rte_errno = errno;
+ goto glue_error;
+ }
+ while (off != mlx4_glue_lib_size) {
+ ssize_t ret;
+
+ ret = write(fd, (const uint8_t *)mlx4_glue_lib + off,
+ mlx4_glue_lib_size - off);
+ if (ret == -1) {
+ if (errno != EINTR) {
+ rte_errno = errno;
+ goto glue_error;
+ }
+ ret = 0;
+ }
+ off += ret;
+ }
+ close(fd);
+ fd = -1;
+ handle = dlopen(file, RTLD_LAZY);
+ unlink(file);
+ if (!handle) {
+ rte_errno = EINVAL;
+ dlmsg = dlerror();
+ if (dlmsg)
+ ERROR("cannot load glue library: %s", dlmsg);
+ goto glue_error;
+ }
+ sym = dlsym(handle, "mlx4_glue");
+ if (!sym || !*sym) {
+ rte_errno = EINVAL;
+ dlmsg = dlerror();
+ if (dlmsg)
+ ERROR("cannot resolve glue symbol: %s", dlmsg);
+ goto glue_error;
+ }
+ mlx4_glue = *sym;
+ return 0;
+glue_error:
+ if (handle)
+ dlclose(handle);
+ if (fd != -1) {
+ close(fd);
+ unlink(file);
+ }
+ ERROR("cannot initialize PMD due to missing run-time"
+ " dependency on rdma-core libraries (libibverbs,"
+ " libmlx4)");
+ return -rte_errno;
+}
+
+#endif
+
/**
* Driver initialization routine.
*/
@@ -744,6 +819,11 @@ rte_mlx4_pmd_init(void)
* using this PMD, which is not supported in forked processes.
*/
setenv("RDMAV_HUGEPAGES_SAFE", "1", 1);
+#ifndef RTE_BUILD_SHARED_LIB
+ if (mlx4_glue_init())
+ return;
+ assert(mlx4_glue);
+#endif
mlx4_glue->fork_init();
rte_pci_register(&mlx4_driver);
}
diff --git a/mk/rte.app.mk b/mk/rte.app.mk
index 0169f3f5b..f9ac0e620 100644
--- a/mk/rte.app.mk
+++ b/mk/rte.app.mk
@@ -143,7 +143,7 @@ ifeq ($(CONFIG_RTE_LIBRTE_KNI),y)
_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_KNI) += -lrte_pmd_kni
endif
_LDLIBS-$(CONFIG_RTE_LIBRTE_LIO_PMD) += -lrte_pmd_lio
-_LDLIBS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += -lrte_pmd_mlx4 -libverbs -lmlx4
+_LDLIBS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += -lrte_pmd_mlx4 -ldl
_LDLIBS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += -lrte_pmd_mlx5 -libverbs -lmlx5
_LDLIBS-$(CONFIG_RTE_LIBRTE_MRVL_PMD) += -lrte_pmd_mrvl
-L$(LIBMUSDK_PATH)/lib -lmusdk
_LDLIBS-$(CONFIG_RTE_LIBRTE_NFP_PMD) += -lrte_pmd_nfp
--
2.11.0
