Hello community, here is the log from the commit of package libmlx4-rdmav2 for openSUSE:Factory checked in at 2014-08-11 10:07:10 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/libmlx4-rdmav2 (Old) and /work/SRC/openSUSE:Factory/.libmlx4-rdmav2.new (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "libmlx4-rdmav2" Changes: -------- New Changes file: --- /dev/null 2014-07-24 01:57:42.080040256 +0200 +++ /work/SRC/openSUSE:Factory/.libmlx4-rdmav2.new/libmlx4-rdmav2.changes 2014-08-11 10:07:12.000000000 +0200 @@ -0,0 +1,139 @@ +------------------------------------------------------------------- +Tue Jun 3 11:24:34 CEST 2014 - [email protected] + +- Update to OFED 3.12 final. + +------------------------------------------------------------------- +Sat Apr 5 20:17:31 CEST 2014 - [email protected] + +- Fix build. + +------------------------------------------------------------------- +Fri Apr 4 16:13:12 CEST 2014 - [email protected] + +- Update to latest version 1.0.5 from OFED 3.12 +- Add connectx_port_config script. + +------------------------------------------------------------------- +Mon Mar 31 18:21:29 CEST 2014 - [email protected] + +- Add patches from Mellanox (bnc#871161): + * libmlx4-Add-RoCE-IP-based-addressing-support-for-UD-QPs.patch + that adds RoCE IP based addressing support for UD QPs + * libmlx4-Add-ibv_query_port_ex-support.patch + to add the new extended support for query_port. + * libmlx4-Add-receive-flow-steering-support.patch + Add receive flow steering support. + +------------------------------------------------------------------- +Tue Mar 4 12:02:30 CET 2014 - [email protected] + +- Apply libmlx4-s390.patch last so that uncompilable code stays + guarded by ifndefs. Modify the other patches to make it work. + +------------------------------------------------------------------- +Tue Feb 11 13:07:54 CET 2014 - [email protected] + +- Add and adapt upstream patches that add XRC support (fate#315416 + bnc#858729): + libmlx4-Add_support_for_XRC-QPs.patch + libmlx4-Remove_compatibility_with_libibverbs-1.1.7.patch + libmlx4-resize_cq-Fix-possible-endless-loop-scanning-CQ.patch + libmlx4-Update_to_use_libiverbs_extension_support_verbs_regi.patch +- Run autoreconf + +------------------------------------------------------------------- +Fri Jan 31 12:20:03 CET 2014 - [email protected] + +- This is a plugin, no devel package needed. + +------------------------------------------------------------------- +Wed Jan 29 16:05:11 CET 2014 - [email protected] + +- Carry over PCI-IDs from SLE11. + +------------------------------------------------------------------- +Fri Jan 17 13:39:41 CET 2014 - [email protected] + +- Update to 1.0.5 from OFED 3.12 daily. +- Enable build for s390x. +- Add libmlx4-s390.patch from IBM for s390 support. +- Use spdx.org format for license specification. + +------------------------------------------------------------------- +Tue Sep 11 16:37:58 CEST 2012 - [email protected] + +- Update to 1.0.3 from OFED 1.5.4.1. +- Fix type punning (libmlx4-fix_type_punning.patch). + +------------------------------------------------------------------- +Tue Sep 11 02:55:37 UTC 2012 - [email protected] + +- Remove redundant sections from specfile, set SRPM group +- Parallel build with %_smp_mflags +- Cease shipping static libraries +- Do own the config directory + +------------------------------------------------------------------- +Tue May 8 18:31:04 CEST 2012 - [email protected] + +- BuildRequire automake, autoconf and libtool. + +------------------------------------------------------------------- +Thu Mar 4 05:06:37 CET 2010 - [email protected] + +- libmlx4-1.0-dev-id.patch: Added two device IDs (bnc#585269) + +------------------------------------------------------------------- +Thu Dec 11 14:40:10 CET 2008 - [email protected] + +- Updated to version from OFED 1.4 GA + - bugfixes and version update + +------------------------------------------------------------------- +Fri Nov 7 00:52:42 CET 2008 - [email protected] + +- adding ExcludeArch for s390 s390x + +------------------------------------------------------------------- +Thu Nov 6 16:40:34 CET 2008 - [email protected] + +- adding baselibs.conf + +------------------------------------------------------------------- +Tue Oct 28 06:34:38 CET 2008 - [email protected] + +- Updated to version from OFED 1.4 rc3 + +------------------------------------------------------------------- +Mon Sep 15 14:06:02 CEST 2008 - [email protected] + +- Changed package name to libmlx4-rdmav2 + +------------------------------------------------------------------- +Fri Feb 29 07:44:52 CET 2008 - [email protected] + +- Update to version 1.0 from OFED 1.3 GA release + +------------------------------------------------------------------- +Fri Feb 1 01:06:36 CET 2008 - [email protected] + +- Update to version 1.0 from OFED 1.3 rc2 +- Removed patches (none are applicable) + +------------------------------------------------------------------- +Tue Aug 21 16:41:05 CEST 2007 - [email protected] + +- Update to version from OFED 1.2.5 + +------------------------------------------------------------------- +Mon Aug 20 00:25:23 CEST 2007 - [email protected] + +- add rpmlintrc (shlib-policy-name-error) for the moment + to fix build + +------------------------------------------------------------------- +Tue Aug 7 14:05:50 CEST 2007 - [email protected] + +- Initial Package, Version 0.1 + New: ---- baselibs.conf connectx_port_config libmlx4-1.0.5.tar.gz libmlx4-Add-RoCE-IP-based-addressing-support-for-UD-QPs.patch libmlx4-Add-ibv_query_port_ex-support.patch libmlx4-Add-receive-flow-steering-support.patch libmlx4-Add_support_for_XRC-QPs.patch libmlx4-Remove_compatibility_with_libibverbs-1.1.7.patch libmlx4-Update_to_use_libiverbs_extension_support_verbs_regi.patch libmlx4-fix_type_punning.patch libmlx4-rdmav2.changes libmlx4-rdmav2.spec libmlx4-resize_cq-Fix-possible-endless-loop-scanning-CQ.patch libmlx4-s390.patch ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ libmlx4-rdmav2.spec ++++++ # # spec file for package libmlx4-rdmav2 (Version 1.0.5) # # Copyright (c) 2010-2014 SUSE LINUX Products GmbH, Nuernberg, Germany. # # All modifications and additions to the file contributed by third parties # remain the property of their copyright owners, unless otherwise agreed # upon. The license for this file, and modifications and additions to the # file, is the same license as for the pristine package itself (unless the # license for the pristine package is not an Open Source License, in which # case the license is the MIT License). An "Open Source License" is a # license that conforms to the Open Source Definition (Version 1.9) # published by the Open Source Initiative. # Please submit bugfixes or comments via http://bugs.opensuse.org/ # %define tarball_suf -1.20.g6771d22 Name: libmlx4-rdmav2 Summary: Mellanox InfiniBand HCA Userspace Driver Version: 1.0.5 Release: 0 Group: System/Libraries License: BSD-2-Clause or GPL-2.0 Url: http://www.openfabrics.org Source: libmlx4-%version.tar.gz Source1: baselibs.conf Source2: connectx_port_config Patch0: libmlx4-fix_type_punning.patch Patch1: libmlx4-s390.patch Patch2: libmlx4-resize_cq-Fix-possible-endless-loop-scanning-CQ.patch Patch3: libmlx4-Remove_compatibility_with_libibverbs-1.1.7.patch Patch4: libmlx4-Update_to_use_libiverbs_extension_support_verbs_regi.patch Patch5: libmlx4-Add_support_for_XRC-QPs.patch Patch6: libmlx4-Add-RoCE-IP-based-addressing-support-for-UD-QPs.patch Patch7: libmlx4-Add-ibv_query_port_ex-support.patch Patch8: libmlx4-Add-receive-flow-steering-support.patch BuildRoot: %{_tmppath}/%{name}-%{version}-build BuildRequires: libibverbs-devel BuildRequires: automake autoconf libtool Supplements: modalias(pci:v000015B3d00001000sv*sd*bc*sc*i*) Supplements: modalias(pci:v000015B3d00001001sv*sd*bc*sc*i*) Supplements: modalias(pci:v000015B3d00001002sv*sd*bc*sc*i*) Supplements: modalias(pci:v000015B3d00001003sv*sd*bc*sc*i*) Supplements: modalias(pci:v000015B3d00001004sv*sd*bc*sc*i*) Supplements: modalias(pci:v000015B3d00001005sv*sd*bc*sc*i*) Supplements: modalias(pci:v000015B3d00001006sv*sd*bc*sc*i*) Supplements: modalias(pci:v000015B3d00001007sv*sd*bc*sc*i*) Supplements: modalias(pci:v000015B3d00001008sv*sd*bc*sc*i*) Supplements: modalias(pci:v000015B3d00001009sv*sd*bc*sc*i*) Supplements: modalias(pci:v000015B3d0000100Asv*sd*bc*sc*i*) Supplements: modalias(pci:v000015B3d0000100Bsv*sd*bc*sc*i*) Supplements: modalias(pci:v000015B3d0000100Csv*sd*bc*sc*i*) Supplements: modalias(pci:v000015B3d0000100Dsv*sd*bc*sc*i*) Supplements: modalias(pci:v000015B3d0000100Esv*sd*bc*sc*i*) Supplements: modalias(pci:v000015B3d0000100Fsv*sd*bc*sc*i*) Supplements: modalias(pci:v000015B3d00006340sv*sd*bc*sc*i*) Supplements: modalias(pci:v000015B3d0000634Asv*sd*bc*sc*i*) Supplements: modalias(pci:v000015B3d00006354sv*sd*bc*sc*i*) Supplements: modalias(pci:v000015B3d00006368sv*sd*bc*sc*i*) Supplements: modalias(pci:v000015B3d00006372sv*sd*bc*sc*i*) Supplements: modalias(pci:v000015B3d00006732sv*sd*bc*sc*i*) Supplements: modalias(pci:v000015B3d0000673Csv*sd*bc*sc*i*) Supplements: modalias(pci:v000015B3d00006746sv*sd*bc*sc*i*) Supplements: modalias(pci:v000015B3d00006750sv*sd*bc*sc*i*) Supplements: modalias(pci:v000015B3d0000675Asv*sd*bc*sc*i*) Supplements: modalias(pci:v000015B3d00006764sv*sd*bc*sc*i*) Supplements: modalias(pci:v000015B3d0000676Esv*sd*bc*sc*i*) Supplements: modalias(pci:v000015B3d00006778sv*sd*bc*sc*i*) %description libmlx4 provides a device-specific userspace driver for Mellanox ConnectX HCAs for use with the libibverbs library. %prep %setup -q -n libmlx4-%version %patch0 %patch2 -p1 %patch3 -p1 %patch4 -p1 %patch5 -p1 %patch1 -p1 %patch6 -p1 %patch7 -p1 %patch8 -p1 %build autoreconf -fi %configure --disable-static make %{?_smp_mflags} V=1 %install make DESTDIR=%{buildroot} install rm -f %{buildroot}%{_libdir}/libmlx4.{la,so} mkdir -p %{buildroot}%{_bindir} install -m 755 %{S:2} %{buildroot}%{_bindir} %files %defattr(-,root,root,-) %doc AUTHORS COPYING README %{_libdir}/libmlx4-rdmav2.so %dir %_sysconfdir/libibverbs.d %config %{_sysconfdir}/libibverbs.d/mlx4.driver %{_bindir}/connectx_port_config %changelog ++++++ baselibs.conf ++++++ libmlx4-rdmav2 libmlx4-rdmav2-devel ++++++ connectx_port_config ++++++ #!/bin/bash # # Copyright (c) 2006 Mellanox Technologies. All rights reserved. # # This Software is licensed under one of the following licenses: # # 1) under the terms of the "Common Public License 1.0" a copy of which is # available from the Open Source Initiative, see # http://www.opensource.org/licenses/cpl.php. # # 2) under the terms of the "The BSD License" a copy of which is # available from the Open Source Initiative, see # http://www.opensource.org/licenses/bsd-license.php. # # 3) under the terms of the "GNU General Public License (GPL) Version 2" a # copy of which is available from the Open Source Initiative, see # http://www.opensource.org/licenses/gpl-license.php. # # Licensee has the right to choose one of the above licenses. # # Redistributions of source code must retain the above copyright # notice and one of the license notices. # # Redistributions in binary form must reproduce both the above copyright # notice, one of the license notices in the documentation # and/or other materials provided with the distribution. # # port_types=( "" "ib" "eth" "auto" ) usage() { echo "Usage:" echo "/sbin/connectx_port_config" echo "/sbin/connectx_port_config -s|--show" echo "/sbin/connectx_port_config -h|--help" echo "/sbin/connectx_port_config [-d|--device <PCI device id>] -c|--conf <port1,port2>" echo "Possible port configurations:" echo " eth,eth" echo " eth,ib" echo " eth,auto" echo " ib,ib" echo " ib,eth" echo " ib,auto" echo " auto,auto" echo " auto,eth" } is_integer() { printf "%s\n" $1 |grep -E "^[+-]?[0-9]+$" > /dev/null return $? } # If module $1 is loaded return - 0 else - 1 is_module() { local RC /sbin/lsmod | grep -w "$1" > /dev/null 2>&1 RC=$? return $RC } # Select PCI device interactively select_device() { printf "Please select device to modify [$dev]: " read dev; if is_integer $dev; then if [ $dev -lt 1 ]; then echo "Illegal index selected, quitting..."; echo exit 1; fi if [ -e ${Devices[$dev]}/mlx4_port1 ]; then echo "`echo ${Devices[$dev]} | cut -d '/' -f 6`"; echo else echo "Illegal value selected, quitting..."; echo exit 1 fi else echo "Numeric value expected, quitting..."; echo exit 1 fi } set_device() { ser=0; for d in ${Devices[@]}; do ser=$(($ser+1)) if [ "/sys/bus/pci/devices/${udevice}/" == "$d" ] || [ "/sys/bus/pci/devices/0000:${udevice}/" == "${d}" ] ; then dev=$ser return fi done } validate_device() { if [ "$udevice" = "0" ]; then if [ "$comm" != "" ]; then if [ ${#Devices[@]} -gt 1 ]; then echo "More then 1 ConnectX devices found, please specify required device"; echo usage exit 1 fi fi return fi for d in ${Devices[@]}; do ser=$(($ser+1)) if [ "/sys/bus/pci/devices/${udevice}/" == "$d" ] || [ "/sys/bus/pci/devices/0000:${udevice}/" == "${d}" ] ; then return fi done echo "Error: illegal device selected." exit 1; } validate_port_config() { for conf in "eth,eth" "eth,ib" "eth,auto" "auto,auto" "auto,eth" "ib,ib" "ib,auto" "ib,eth" "eth" "auto" "ib"; do if [ "$conf" == "$comm" ]; then return fi done echo "Error: $comm is not valid port configuration" usage exit 1; } prompt_options() { local port=$1 local options=$2 if [ "$port" == "2" ]; then if [ -e ${Devices[$dev]}/mlx4_port2 ]; then echo -n "" else return 0 fi fi printf "Select mode for port $port (${options}): " read mode if is_integer $mode; then if [ $mode -lt 0 ]; then echo "Illegal value selected, quitting..."; echo exit 1 fi if [ `echo ${options} | grep ${mode} -wc` == 0 ]; then echo "Illegal value selected, quitting..."; echo exit 1 fi return "${mode}" else echo "Illegal value selected, quitting..."; echo exit 1 fi } show_configuration() { local cnt=0 for pci_dev in /sys/bus/pci/devices/*/; do if [ -e $pci_dev/mlx4_port1 ]; then echo "--------------------------------" dev=`echo $pci_dev | cut -d '/' -f 6` echo "Port configuration for PCI device: $dev is:" cat ${pci_dev}/mlx4_port1 if [ -e $pci_dev/mlx4_port2 ]; then cat ${pci_dev}/mlx4_port2 fi echo "--------------------------------" cnt=$(($cnt+1)) fi done if [ $cnt == 0 ]; then echo "No ConnectX devices found or driver not loaded" fi } save_config() { local configuration_file=/etc/infiniband/connectx.conf local config_comment="# ConnectX Port Configuration for $udevice" local config_line=" -d $udevice -c $comm" local add_new_lines=1 if [ -e "${configuration_file}" ]; then # check if there is a configuration to the device grep "${udevice}" ${configuration_file} >> /dev/null if [ $? -eq 0 ]; then # replace the configuration lines sed -i "s/#.* ${udevice}/${config_comment}/" ${configuration_file} sed -i "s/.*connectx_port_config.*$udevice.*/\/sbin\/connectx_port_config${config_line}/" ${configuration_file} add_new_lines=0 fi fi # add the configuration lines to the end of the file if [ ${add_new_lines} -ne 0 ]; then echo "${config_comment}" >> ${configuration_file} echo "/sbin/connectx_port_config${config_line}" >> ${configuration_file} fi } set_port_config() { echo "|----------------------------|" echo "| Possible port modes: |" echo "| 1: Infiniband |" echo "| 2: Ethernet |" echo "| 3: AutoSense |" echo "|----------------------------|" local options="1,2,3" prompt_options 1 $options comm=${port_types[$?]} if [ -n "${comm}" ]; then prompt_options 2 $options local port2_conf=${port_types[$?]} if [ -n "${port2_conf}" ]; then comm="${comm},${port2_conf}" fi fi } udevice=0 comm="" ############################################################################### # Parsing command line # ############################################################################### while [ ! -z "$1" ]; do case "$1" in -s|--show) show_configuration if [ -n "$2" ]; then echo "Too many parameters" usage exit 1 fi exit 0 ;; -d|--device) if [ -z "$2" ]; then echo "Error: Illegal input" usage exit 1 fi udevice=$2 shift 2 ;; -c|--conf) if [ -z "$2" ]; then echo "Error: Illegal input" usage exit 1 fi comm=$2 shift 2; validate_port_config ;; -h|--help) usage exit 0 ;; *) echo "Wrong parameter error: $1" usage exit 1 ;; esac done ############################################################# # Load low level driver if not loaded # ############################################################# is_module mlx4_core RC=$? if [ $RC == 1 ]; then echo "Bus driver not loaded, loading..." /sbin/modprobe mlx4_core fi ############################################################# # List ConnectX devices # ############################################################# i=0 echo; echo "ConnectX PCI devices :" echo "|----------------------------|" for pci_dev in /sys/bus/pci/devices/*/; do if [ -e $pci_dev/mlx4_port1 ]; then (( i++ )) Devices[$i]=$pci_dev dev=`echo $pci_dev | cut -d '/' -f 6` echo "| $i $dev |" fi done echo "|----------------------------|"; echo validate_device if [ $i == 0 ]; then echo "No devices found, quiting..." exit 1; fi ############################################################# # Select device to modify # ############################################################# dev=1 if [ $i -ne 1 ]; then if [ "$comm" != "" ]; then if [ "$udevice" != "0" ]; then set_device fi else select_device fi fi udevice=`echo ${Devices[$dev]} | cut -d '/' -f 6` echo "Before port change:" cat ${Devices[$dev]}/mlx4_port1 if [ -e ${Devices[$dev]}/mlx4_port2 ]; then cat ${Devices[$dev]}/mlx4_port2 fi echo write_mask="0x222" mlx4_port_file="${Devices[$dev]}/mlx4_port1" if [ ! -w $mlx4_port_file ] || [ $((0x$(stat -c "%a" $mlx4_port_file) & $write_mask)) == "0" ]; then echo "Not allowed to change port configuration, quitting..." exit 1; fi mlx4_port_file="${Devices[$dev]}/mlx4_port2" if [ -e $mlx4_port_file ] && ([ ! -w $mlx4_port_file ] || [ $((0x$(stat -c "%a" $mlx4_port_file) & $write_mask)) == "0" ]); then echo "Not allowed to change port configuration, quitting..." exit 1; fi ############################################################ # Set port configuration for the selected device # ############################################################ if [ "$comm" == "" ]; then set_port_config fi if [ -e ${Devices[$dev]}/port_trigger ]; then echo "all" > ${Devices[$dev]}/port_trigger fi comm1=`echo ${comm} | cut -d ',' -f 1` comm2=`echo ${comm} | cut -d ',' -f 2` for uverbs_dir in ${Devices[$dev]}infiniband_verbs/uverbs* do if [[ -f $uverbs_dir/ref_cnt ]]; then if [[ `cat "$uverbs_dir/ref_cnt"` > 1 ]]; then echo "WARNING: Please close user level applications" \ "before running connectx_port_config. " echo "Operation aborted!" exit 1 fi fi done { echo "${comm1}" > ${Devices[$dev]}/mlx4_port1 } > /dev/null 2>&1 rc=$? if [ -e ${Devices[$dev]}/mlx4_port2 ]; then { echo "${comm2}" > ${Devices[$dev]}/mlx4_port2 } > /dev/null 2>&1 rc=$? fi if [ $rc -ne 0 ]; then echo "WARNING: Illegal port configuration attempted," echo " Please view dmesg for details."; echo exit 1 fi echo; echo "After port change:" cat ${Devices[$dev]}/mlx4_port1 if [ -e ${Devices[$dev]}/mlx4_port2 ]; then cat ${Devices[$dev]}/mlx4_port2 fi save_config exit 0 ++++++ libmlx4-Add-RoCE-IP-based-addressing-support-for-UD-QPs.patch ++++++ >From 0a56fa02857bd2dcce32f8a107b7c397f9357545 Mon Sep 17 00:00:00 2001 From: Matan Barak <[email protected]> Date: Tue, 11 Feb 2014 14:31:14 +0200 Subject: [PATCH 1/3] Add RoCE IP based addressing support for UD QPs In order to implement IP based addressing for UD QPs, we need a way to resolve the addresses internally. The L2 params are passed to the provider driver using an extension verbs - drv_ibv_create_ah_ex. libmlx4 gets the extra mac and vid params from libibverbs and sets mlx4_ah relevant attributes. Signed-off-by: Matan Barak <[email protected]> Signed-off-by: Or Gerlitz <[email protected]> --- src/mlx4.c | 5 ++- src/mlx4.h | 2 + src/verbs.c | 90 ++++++++++++++++++++++++++++++++++++++++++++++++++++-------- 3 files changed, 85 insertions(+), 12 deletions(-) Index: libmlx4-1.0.5/src/mlx4.c =================================================================== --- libmlx4-1.0.5.orig/src/mlx4.c 2014-03-31 18:15:17.755105851 +0200 +++ libmlx4-1.0.5/src/mlx4.c 2014-03-31 18:16:09.624909060 +0200 @@ -203,7 +203,8 @@ static int mlx4_init_context(struct verb ibv_ctx->ops = mlx4_ctx_ops; verbs_ctx->has_comp_mask = VERBS_CONTEXT_XRCD | VERBS_CONTEXT_SRQ | - VERBS_CONTEXT_QP; + VERBS_CONTEXT_QP | + VERBS_CONTEXT_CREATE_AH; verbs_set_ctx_op(verbs_ctx, close_xrcd, mlx4_close_xrcd); verbs_set_ctx_op(verbs_ctx, open_xrcd, mlx4_open_xrcd); verbs_set_ctx_op(verbs_ctx, create_srq_ex, mlx4_create_srq_ex); @@ -211,6 +212,8 @@ static int mlx4_init_context(struct verb verbs_set_ctx_op(verbs_ctx, create_qp_ex, mlx4_create_qp_ex); verbs_set_ctx_op(verbs_ctx, open_qp, mlx4_open_qp); + verbs_set_ctx_op(verbs_ctx, drv_ibv_create_ah_ex, mlx4_create_ah_ex); + return 0; } Index: libmlx4-1.0.5/src/mlx4.h =================================================================== --- libmlx4-1.0.5.orig/src/mlx4.h 2014-03-31 18:15:17.755105851 +0200 +++ libmlx4-1.0.5/src/mlx4.h 2014-03-31 18:15:17.769105528 +0200 @@ -437,6 +437,8 @@ struct mlx4_qp *mlx4_find_qp(struct mlx4 int mlx4_store_qp(struct mlx4_context *ctx, uint32_t qpn, struct mlx4_qp *qp); void mlx4_clear_qp(struct mlx4_context *ctx, uint32_t qpn); struct ibv_ah *mlx4_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr); +struct ibv_ah *mlx4_create_ah_ex(struct ibv_pd *pd, + struct ibv_ah_attr_ex *attr_ex); int mlx4_destroy_ah(struct ibv_ah *ah); int mlx4_alloc_av(struct mlx4_pd *pd, struct ibv_ah_attr *attr, struct mlx4_ah *ah); Index: libmlx4-1.0.5/src/verbs.c =================================================================== --- libmlx4-1.0.5.orig/src/verbs.c 2014-03-31 18:15:17.747106036 +0200 +++ libmlx4-1.0.5/src/verbs.c 2014-03-31 18:15:17.770105505 +0200 @@ -785,13 +785,11 @@ static int mlx4_resolve_grh_to_l2(struct return 0; } -struct ibv_ah *mlx4_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr) +static struct ibv_ah *mlx4_create_ah_common(struct ibv_pd *pd, + struct ibv_ah_attr *attr, + uint8_t link_layer) { struct mlx4_ah *ah; - struct ibv_port_attr port_attr; - - if (ibv_query_port(pd->context, attr->port_num, &port_attr)) - return NULL; ah = malloc(sizeof *ah); if (!ah) @@ -801,7 +799,7 @@ struct ibv_ah *mlx4_create_ah(struct ibv ah->av.port_pd = htonl(to_mpd(pd)->pdn | (attr->port_num << 24)); - if (port_attr.link_layer != IBV_LINK_LAYER_ETHERNET) { + if (link_layer != IBV_LINK_LAYER_ETHERNET) { ah->av.g_slid = attr->src_path_bits; ah->av.dlid = htons(attr->dlid); ah->av.sl_tclass_flowlabel = htonl(attr->sl << 28); @@ -822,13 +820,83 @@ struct ibv_ah *mlx4_create_ah(struct ibv memcpy(ah->av.dgid, attr->grh.dgid.raw, 16); } - if (port_attr.link_layer == IBV_LINK_LAYER_ETHERNET) - if (mlx4_resolve_grh_to_l2(pd, ah, attr)) { - free(ah); - return NULL; + return &ah->ibv_ah; +} + +struct ibv_ah *mlx4_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr) +{ + struct ibv_ah *ah; + struct ibv_port_attr port_attr; + + if (ibv_query_port(pd->context, attr->port_num, &port_attr)) + return NULL; + + ah = mlx4_create_ah_common(pd, attr, port_attr.link_layer); + if (NULL != ah && + (port_attr.link_layer != IBV_LINK_LAYER_ETHERNET || + !mlx4_resolve_grh_to_l2(pd, to_mah(ah), attr))) + return ah; + + if (ah) + free(ah); + return NULL; +} + +struct ibv_ah *mlx4_create_ah_ex(struct ibv_pd *pd, + struct ibv_ah_attr_ex *attr_ex) +{ + struct ibv_port_attr port_attr; + struct ibv_ah *ah; + struct mlx4_ah *mah; + + if (ibv_query_port(pd->context, attr_ex->port_num, &port_attr)) + return NULL; + + ah = mlx4_create_ah_common(pd, (struct ibv_ah_attr *)attr_ex, + port_attr.link_layer); + + if (NULL == ah) + return NULL; + + mah = to_mah(ah); + + /* If vlan was given, check that we could use it */ + if (attr_ex->comp_mask & IBV_AH_ATTR_EX_VID && + attr_ex->vid <= 0xfff && + (0 == attr_ex->ll_address.len || + !(attr_ex->comp_mask & IBV_AH_ATTR_EX_LL))) + goto err; + + /* ll_address.len == 0 means no ll address given */ + if (attr_ex->comp_mask & IBV_AH_ATTR_EX_LL && + 0 != attr_ex->ll_address.len) { + if (LL_ADDRESS_ETH != attr_ex->ll_address.type || + port_attr.link_layer != IBV_LINK_LAYER_ETHERNET) + /* mlx4 provider currently only support ethernet + * extensions */ + goto err; + + /* link layer is ethernet */ + if (6 != attr_ex->ll_address.len || + NULL == attr_ex->ll_address.address) + goto err; + + memcpy(mah->mac, attr_ex->ll_address.address, + attr_ex->ll_address.len); + + if (attr_ex->comp_mask & IBV_AH_ATTR_EX_VID && + attr_ex->vid <= 0xfff) { + mah->av.port_pd |= htonl(1 << 29); + mah->vlan = attr_ex->vid | + ((attr_ex->sl & 7) << 13); } + } - return &ah->ibv_ah; + return ah; + +err: + free(ah); + return NULL; } int mlx4_destroy_ah(struct ibv_ah *ah) ++++++ libmlx4-Add-ibv_query_port_ex-support.patch ++++++ >From feab8b51a037f9e68757c9e98f3271de87895856 Mon Sep 17 00:00:00 2001 From: Matan Barak <[email protected]> Date: Tue, 11 Feb 2014 14:31:15 +0200 Subject: [PATCH 2/3] Add ibv_query_port_ex support This patch adds the new extended support for query_port. The purpose of this is: 1. Request fields that aren't availible by today's ibv_query_port 2. Don't fetch fields that the user doesn't need. Hence, there is more chance to optimize. 3. Cache link layer's type in mlx4_context. Caching will allow us to avoid ibv_query_port calls and save time in ibv_create_ah. Signed-off-by: Matan Barak <[email protected]> Signed-off-by: Or Gerlitz <[email protected]> --- src/mlx4.c | 4 +++ src/mlx4.h | 9 +++++++ src/verbs.c | 73 +++++++++++++++++++++++++++++++++++++++++++++++++++++++----- 3 files changed, 80 insertions(+), 6 deletions(-) Index: libmlx4-1.0.5/src/mlx4.c =================================================================== --- libmlx4-1.0.5.orig/src/mlx4.c 2014-03-31 18:16:59.945747982 +0200 +++ libmlx4-1.0.5/src/mlx4.c 2014-03-31 18:18:06.374215205 +0200 @@ -157,6 +157,8 @@ static int mlx4_init_context(struct verb context->qp_table_shift = ffs(context->num_qps) - 1 - MLX4_QP_TABLE_BITS; context->qp_table_mask = (1 << context->qp_table_shift) - 1; + for (i = 0; i < MLX4_PORTS_NUM; ++i) + context->port_query_cache[i].valid = 0; pthread_mutex_init(&context->qp_table_mutex, NULL); for (i = 0; i < MLX4_QP_TABLE_SIZE; ++i) @@ -213,6 +215,8 @@ static int mlx4_init_context(struct verb verbs_set_ctx_op(verbs_ctx, open_qp, mlx4_open_qp); verbs_set_ctx_op(verbs_ctx, drv_ibv_create_ah_ex, mlx4_create_ah_ex); + verbs_set_ctx_op(verbs_ctx, drv_query_port_ex, + mlx4_query_port_ex); return 0; } Index: libmlx4-1.0.5/src/mlx4.h =================================================================== --- libmlx4-1.0.5.orig/src/mlx4.h 2014-03-31 18:16:59.946747959 +0200 +++ libmlx4-1.0.5/src/mlx4.h 2014-03-31 18:16:59.955747751 +0200 @@ -40,6 +40,8 @@ #include <infiniband/arch.h> #include <infiniband/verbs.h> +#define MLX4_PORTS_NUM 2 + #ifdef HAVE_VALGRIND_MEMCHECK_H # include <valgrind/memcheck.h> @@ -195,6 +197,11 @@ struct mlx4_context { pthread_mutex_t db_list_mutex; int cqe_size; struct mlx4_xsrq_table xsrq_table; + struct { + uint8_t valid; + uint8_t link_layer; + enum ibv_port_cap_flags caps; + } port_query_cache[MLX4_PORTS_NUM]; }; struct mlx4_buf { @@ -360,6 +367,8 @@ int mlx4_query_device(struct ibv_context struct ibv_device_attr *attr); int mlx4_query_port(struct ibv_context *context, uint8_t port, struct ibv_port_attr *attr); +int mlx4_query_port_ex(struct ibv_context *context, uint8_t port_num, + struct ibv_port_attr_ex *port_attr); struct ibv_pd *mlx4_alloc_pd(struct ibv_context *context); int mlx4_free_pd(struct ibv_pd *pd); Index: libmlx4-1.0.5/src/verbs.c =================================================================== --- libmlx4-1.0.5.orig/src/verbs.c 2014-03-31 18:16:59.946747959 +0200 +++ libmlx4-1.0.5/src/verbs.c 2014-03-31 18:16:59.956747728 +0200 @@ -70,8 +70,63 @@ int mlx4_query_port(struct ibv_context * struct ibv_port_attr *attr) { struct ibv_query_port cmd; + int err; - return ibv_cmd_query_port(context, port, attr, &cmd, sizeof cmd); + err = ibv_cmd_query_port(context, port, attr, &cmd, sizeof(cmd)); + if (!err && port <= MLX4_PORTS_NUM && port > 0) { + struct mlx4_context *mctx = to_mctx(context); + if (!mctx->port_query_cache[port - 1].valid) { + mctx->port_query_cache[port - 1].link_layer = + attr->link_layer; + mctx->port_query_cache[port - 1].caps = + attr->port_cap_flags; + mctx->port_query_cache[port - 1].valid = 1; + } + } + + return err; +} + +int mlx4_query_port_ex(struct ibv_context *context, uint8_t port_num, + struct ibv_port_attr_ex *port_attr) +{ + /* Check that only valid flags were given */ + if (!(port_attr->comp_mask & IBV_QUERY_PORT_EX_ATTR_MASK1) || + (port_attr->comp_mask & ~IBV_QUERY_PORT_EX_ATTR_MASKS) || + (port_attr->mask1 & ~IBV_QUERY_PORT_EX_MASK)) { + return EINVAL; + } + + /* Optimize the link type query */ + if (port_attr->comp_mask == IBV_QUERY_PORT_EX_ATTR_MASK1) { + if (!(port_attr->mask1 & ~(IBV_QUERY_PORT_EX_LINK_LAYER | + IBV_QUERY_PORT_EX_CAP_FLAGS))) { + struct mlx4_context *mctx = to_mctx(context); + if (port_num <= 0 || port_num > MLX4_PORTS_NUM) + return EINVAL; + if (mctx->port_query_cache[port_num - 1].valid) { + if (port_attr->mask1 & + IBV_QUERY_PORT_EX_LINK_LAYER) + port_attr->link_layer = + mctx-> + port_query_cache[port_num - 1]. + link_layer; + if (port_attr->mask1 & + IBV_QUERY_PORT_EX_CAP_FLAGS) + port_attr->port_cap_flags = + mctx-> + port_query_cache[port_num - 1]. + caps; + return 0; + } + } + if (port_attr->mask1 & IBV_QUERY_PORT_EX_STD_MASK) { + return mlx4_query_port(context, port_num, + &port_attr->port_attr); + } + } + + return EOPNOTSUPP; } struct ibv_pd *mlx4_alloc_pd(struct ibv_context *context) @@ -826,15 +881,18 @@ static struct ibv_ah *mlx4_create_ah_com struct ibv_ah *mlx4_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr) { struct ibv_ah *ah; - struct ibv_port_attr port_attr; + struct ibv_port_attr_ex port_attr; + + port_attr.comp_mask = IBV_QUERY_PORT_EX_ATTR_MASK1; + port_attr.mask1 = IBV_QUERY_PORT_EX_LINK_LAYER; - if (ibv_query_port(pd->context, attr->port_num, &port_attr)) + if (ibv_query_port_ex(pd->context, attr->port_num, &port_attr)) return NULL; ah = mlx4_create_ah_common(pd, attr, port_attr.link_layer); if (NULL != ah && (port_attr.link_layer != IBV_LINK_LAYER_ETHERNET || - !mlx4_resolve_grh_to_l2(pd, to_mah(ah), attr))) + !mlx4_resolve_grh_to_l2(pd, to_mah(ah), attr))) return ah; if (ah) @@ -845,11 +903,14 @@ struct ibv_ah *mlx4_create_ah(struct ibv struct ibv_ah *mlx4_create_ah_ex(struct ibv_pd *pd, struct ibv_ah_attr_ex *attr_ex) { - struct ibv_port_attr port_attr; + struct ibv_port_attr_ex port_attr; struct ibv_ah *ah; struct mlx4_ah *mah; - if (ibv_query_port(pd->context, attr_ex->port_num, &port_attr)) + port_attr.comp_mask = IBV_QUERY_PORT_EX_ATTR_MASK1; + port_attr.mask1 = IBV_QUERY_PORT_EX_LINK_LAYER; + + if (ibv_query_port_ex(pd->context, attr_ex->port_num, &port_attr)) return NULL; ah = mlx4_create_ah_common(pd, (struct ibv_ah_attr *)attr_ex, ++++++ libmlx4-Add-receive-flow-steering-support.patch ++++++ >From dcb68c418a7cb1b850f84e61f453ea7841fcbe10 Mon Sep 17 00:00:00 2001 From: Matan Barak <[email protected]> Date: Thu, 6 Feb 2014 14:21:36 +0200 Subject: [PATCH 3/3] Add receive flow steering support The implementation uses the default commands from libibverbs. Signed-off-by: Matan Barak <[email protected]> Signed-off-by: Or Gerlitz <[email protected]> --- src/mlx4.c | 2 ++ 1 file changed, 2 insertions(+) Index: libmlx4-1.0.5/src/mlx4.c =================================================================== --- libmlx4-1.0.5.orig/src/mlx4.c 2014-03-31 18:20:29.248918355 +0200 +++ libmlx4-1.0.5/src/mlx4.c 2014-03-31 18:20:56.709284681 +0200 @@ -217,6 +217,8 @@ static int mlx4_init_context(struct verb verbs_set_ctx_op(verbs_ctx, drv_ibv_create_ah_ex, mlx4_create_ah_ex); verbs_set_ctx_op(verbs_ctx, drv_query_port_ex, mlx4_query_port_ex); + verbs_set_ctx_op(verbs_ctx, drv_ibv_create_flow, ibv_cmd_create_flow); + verbs_set_ctx_op(verbs_ctx, drv_ibv_create_flow, ibv_cmd_create_flow); return 0; } ++++++ libmlx4-Add_support_for_XRC-QPs.patch ++++++ ++++ 1010 lines (skipped) ++++++ libmlx4-Remove_compatibility_with_libibverbs-1.1.7.patch ++++++ >From bcc5d064268073cb996bdbac1ae36728db3b2c96 Mon Sep 17 00:00:00 2001 From: Roland Dreier <[email protected]> Date: Wed, 1 Jan 2014 00:23:43 -0800 Subject: [PATCH 2/4] Remove compatibility with libibverbs < 1.1.7 Since the changes to add verbs extensions / XRC support will break compatibility with libibverbs < 1.1.8, take this chance to strip out all the backwards compatibility code that won't make any sense once we do that. Signed-off-by: Roland Dreier <[email protected]> --- Makefile.am | 19 ++++++------------- configure.ac | 20 ++++++++++---------- libmlx4.spec.in | 2 +- src/buf.c | 19 ------------------- src/mlx4.c | 22 ---------------------- src/mlx4.h | 4 ---- src/verbs.c | 28 +++++++--------------------- 7 files changed, 24 insertions(+), 90 deletions(-) Index: libmlx4-1.0.5/Makefile.am =================================================================== --- libmlx4-1.0.5.orig/Makefile.am 2011-03-08 18:56:59.000000000 +0100 +++ libmlx4-1.0.5/Makefile.am 2014-03-04 12:01:31.987367063 +0100 @@ -5,19 +5,12 @@ mlx4_version_script = @MLX4_VERSION_SCRI MLX4_SOURCES = src/buf.c src/cq.c src/dbrec.c src/mlx4.c src/qp.c \ src/srq.c src/verbs.c -if HAVE_IBV_DEVICE_LIBRARY_EXTENSION - lib_LTLIBRARIES = src/libmlx4.la - src_libmlx4_la_SOURCES = $(MLX4_SOURCES) - src_libmlx4_la_LDFLAGS = -avoid-version -release @IBV_DEVICE_LIBRARY_EXTENSION@ \ - $(mlx4_version_script) - mlx4confdir = $(sysconfdir)/libibverbs.d - mlx4conf_DATA = mlx4.driver -else - mlx4libdir = $(libdir)/infiniband - mlx4lib_LTLIBRARIES = src/mlx4.la - src_mlx4_la_SOURCES = $(MLX4_SOURCES) - src_mlx4_la_LDFLAGS = -avoid-version -module $(mlx4_version_script) -endif +lib_LTLIBRARIES = src/libmlx4.la +src_libmlx4_la_SOURCES = $(MLX4_SOURCES) +src_libmlx4_la_LDFLAGS = -avoid-version -release @IBV_DEVICE_LIBRARY_EXTENSION@ \ + $(mlx4_version_script) +mlx4confdir = $(sysconfdir)/libibverbs.d +mlx4conf_DATA = mlx4.driver EXTRA_DIST = src/doorbell.h src/mlx4.h src/mlx4-abi.h src/wqe.h \ src/mlx4.map libmlx4.spec.in mlx4.driver Index: libmlx4-1.0.5/configure.ac =================================================================== --- libmlx4-1.0.5.orig/configure.ac 2013-05-30 19:38:36.000000000 +0200 +++ libmlx4-1.0.5/configure.ac 2014-03-04 12:01:45.850045975 +0100 @@ -45,21 +45,20 @@ dnl Checks for typedefs, structures, and AC_C_CONST AC_CHECK_SIZEOF(long) +dnl Checks for library functions +AC_CHECK_FUNC(ibv_register_driver, [], + AC_MSG_ERROR([ibv_register_driver() not found. libmlx4 requires libibverbs >= 1.1.7.])) + AC_CACHE_CHECK([for IBV_QPT_RAW_PACKET], [ac_cv_ibv_qpt_raw_packet], - [AC_COMPILE_IFELSE([AC_LANG_PROGRAM([], + [AC_COMPILE_IFELSE([AC_LANG_PROGRAM([#include <infiniband/verbs.h>], [enum ibv_qp_type qpt = IBV_QPT_RAW_PACKET;])], [ac_cv_ibv_qpt_raw_packet=yes], [ac_cv_ibv_qpt_raw_packet=no])]) -if test $ac_cv_ibv_qpt_raw_packet = yes; then - AC_DEFINE([HAVE_IBV_QPT_RAW_PACKET], 1, [libibverbs has raw packet QP support]) +if test $ac_cv_ibv_qpt_raw_packet = no; then + AC_MSG_ERROR([IBV_QPT_RAW_PACKET not found. libmlx4 requires libibverbs >= 1.1.7.]) fi -dnl Checks for library functions -AC_CHECK_FUNC(ibv_read_sysfs_file, [], - AC_MSG_ERROR([ibv_read_sysfs_file() not found. libmlx4 requires libibverbs >= 1.0.3.])) -AC_CHECK_FUNCS(ibv_dontfork_range ibv_dofork_range ibv_register_driver) - dnl Now check if for libibverbs 1.0 vs 1.1 dummy=if$$ cat <<IBV_VERSION > $dummy.c @@ -68,8 +67,9 @@ IBV_DEVICE_LIBRARY_EXTENSION IBV_VERSION IBV_DEVICE_LIBRARY_EXTENSION=`$CC $CPPFLAGS -E $dummy.c 2> /dev/null | tail -1` rm -f $dummy.c -AM_CONDITIONAL(HAVE_IBV_DEVICE_LIBRARY_EXTENSION, - test $IBV_DEVICE_LIBRARY_EXTENSION != IBV_DEVICE_LIBRARY_EXTENSION) +if test $IBV_DEVICE_LIBRARY_EXTENSION = IBV_DEVICE_LIBRARY_EXTENSION; then + AC_MSG_ERROR([libibverbs does not define IBV_DEVICE_LIBRARY_EXTENSION.]) +fi AC_SUBST(IBV_DEVICE_LIBRARY_EXTENSION) AC_CACHE_CHECK(whether ld accepts --version-script, ac_cv_version_script, Index: libmlx4-1.0.5/libmlx4.spec.in =================================================================== --- libmlx4-1.0.5.orig/libmlx4.spec.in 2013-05-30 19:39:14.000000000 +0200 +++ libmlx4-1.0.5/libmlx4.spec.in 2014-03-04 12:01:45.850045975 +0100 @@ -9,7 +9,7 @@ Url: http://openfabrics.org/ Source: http://openfabrics.org/downloads/mlx4/libmlx4-1.0.5.tar.gz BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX) -BuildRequires: libibverbs-devel >= 1.1-0.1.rc2 +BuildRequires: libibverbs-devel >= 1.1.7 %description libmlx4 provides a device-specific userspace driver for Mellanox Index: libmlx4-1.0.5/src/buf.c =================================================================== --- libmlx4-1.0.5.orig/src/buf.c 2011-03-08 18:56:59.000000000 +0100 +++ libmlx4-1.0.5/src/buf.c 2014-03-04 12:01:45.778047643 +0100 @@ -40,25 +40,6 @@ #include "mlx4.h" -#if !(defined(HAVE_IBV_DONTFORK_RANGE) && defined(HAVE_IBV_DOFORK_RANGE)) - -/* - * If libibverbs isn't exporting these functions, then there's no - * point in doing it here, because the rest of libibverbs isn't going - * to be fork-safe anyway. - */ -static int ibv_dontfork_range(void *base, size_t size) -{ - return 0; -} - -static int ibv_dofork_range(void *base, size_t size) -{ - return 0; -} - -#endif /* HAVE_IBV_DONTFORK_RANGE && HAVE_IBV_DOFORK_RANGE */ - int mlx4_alloc_buf(struct mlx4_buf *buf, size_t size, int page_size) { int ret; Index: libmlx4-1.0.5/src/mlx4.c =================================================================== --- libmlx4-1.0.5.orig/src/mlx4.c 2013-04-03 20:21:44.000000000 +0200 +++ libmlx4-1.0.5/src/mlx4.c 2014-03-04 12:01:45.850045975 +0100 @@ -42,10 +42,6 @@ #include <pthread.h> #include <string.h> -#ifndef HAVE_IBV_REGISTER_DRIVER -#include <sysfs/libsysfs.h> -#endif - #include "mlx4.h" #include "mlx4-abi.h" @@ -268,25 +264,7 @@ found: return &dev->ibv_dev; } -#ifdef HAVE_IBV_REGISTER_DRIVER static __attribute__((constructor)) void mlx4_register_driver(void) { ibv_register_driver("mlx4", mlx4_driver_init); } -#else -/* - * Export the old libsysfs sysfs_class_device-based driver entry point - * if libibverbs does not export an ibv_register_driver() function. - */ -struct ibv_device *openib_driver_init(struct sysfs_class_device *sysdev) -{ - int abi_version = 0; - char value[8]; - - if (ibv_read_sysfs_file(sysdev->path, "abi_version", - value, sizeof value) > 0) - abi_ver = strtol(value, NULL, 10); - - return mlx4_driver_init(sysdev->path, abi_version); -} -#endif /* HAVE_IBV_REGISTER_DRIVER */ Index: libmlx4-1.0.5/src/mlx4.h =================================================================== --- libmlx4-1.0.5.orig/src/mlx4.h 2013-04-03 20:40:27.000000000 +0200 +++ libmlx4-1.0.5/src/mlx4.h 2014-03-04 12:01:45.850045975 +0100 @@ -83,10 +83,6 @@ #define PFX "mlx4: " -#ifndef HAVE_IBV_QPT_RAW_PACKET -#define IBV_QPT_RAW_PACKET 8 -#endif - enum { MLX4_STAT_RATE_OFFSET = 5 }; Index: libmlx4-1.0.5/src/verbs.c =================================================================== --- libmlx4-1.0.5.orig/src/verbs.c 2014-03-04 12:01:31.952367874 +0100 +++ libmlx4-1.0.5/src/verbs.c 2014-03-04 12:01:45.778047643 +0100 @@ -112,24 +112,16 @@ struct ibv_mr *mlx4_reg_mr(struct ibv_pd { struct ibv_mr *mr; struct ibv_reg_mr cmd; + struct ibv_reg_mr_resp resp; int ret; mr = malloc(sizeof *mr); if (!mr) return NULL; -#ifdef IBV_CMD_REG_MR_HAS_RESP_PARAMS - { - struct ibv_reg_mr_resp resp; - - ret = ibv_cmd_reg_mr(pd, addr, length, (uintptr_t) addr, - access, mr, &cmd, sizeof cmd, - &resp, sizeof resp); - } -#else - ret = ibv_cmd_reg_mr(pd, addr, length, (uintptr_t) addr, access, mr, - &cmd, sizeof cmd); -#endif + ret = ibv_cmd_reg_mr(pd, addr, length, (uintptr_t) addr, + access, mr, &cmd, sizeof cmd, + &resp, sizeof resp); if (ret) { free(mr); return NULL; @@ -227,6 +219,7 @@ int mlx4_resize_cq(struct ibv_cq *ibcq, { struct mlx4_cq *cq = to_mcq(ibcq); struct mlx4_resize_cq cmd; + struct ibv_resize_cq_resp resp; struct mlx4_buf buf; int old_cqe, outst_cqe, ret; @@ -256,15 +249,8 @@ int mlx4_resize_cq(struct ibv_cq *ibcq, old_cqe = ibcq->cqe; cmd.buf_addr = (uintptr_t) buf.buf; -#ifdef IBV_CMD_RESIZE_CQ_HAS_RESP_PARAMS - { - struct ibv_resize_cq_resp resp; - ret = ibv_cmd_resize_cq(ibcq, cqe - 1, &cmd.ibv_cmd, sizeof cmd, - &resp, sizeof resp); - } -#else - ret = ibv_cmd_resize_cq(ibcq, cqe - 1, &cmd.ibv_cmd, sizeof cmd); -#endif + ret = ibv_cmd_resize_cq(ibcq, cqe - 1, &cmd.ibv_cmd, sizeof cmd, + &resp, sizeof resp); if (ret) { mlx4_free_buf(&buf); goto out; ++++++ libmlx4-Update_to_use_libiverbs_extension_support_verbs_regi.patch ++++++ >From d00cfeb15ef4516539725b2f6c7d992e7201e13d Mon Sep 17 00:00:00 2001 From: Yishai Hadas <[email protected]> Date: Thu, 1 Aug 2013 18:05:54 +0300 Subject: [PATCH 3/4] Update to use libiverbs extension support / verbs_register_driver() Have libmlx4 register with libibverbs with verbs_register_driver() to indicate that libmlx4 supports verbs extensions, supplying its verbs_init_func() for further initialization. This will be used to add XRC support. Acked-by: Jason Gunthorpe <[email protected]> Signed-off-by: Yishai Hadas <[email protected]> Signed-off-by: Tzahi Oved <[email protected]> [ Update configure script and deb/rpm packaging to check for new enough libibverbs. - Roland ] Signed-off-by: Roland Dreier <[email protected]> --- configure.ac | 6 ++-- libmlx4.spec.in | 2 - src/mlx4.c | 74 ++++++++++++++++++++++++++++---------------------------- src/mlx4.h | 7 +++-- 4 files changed, 47 insertions(+), 42 deletions(-) Index: libmlx4-1.0.5/configure.ac =================================================================== --- libmlx4-1.0.5.orig/configure.ac 2014-03-04 12:29:28.091637017 +0100 +++ libmlx4-1.0.5/configure.ac 2014-03-04 12:29:28.109636602 +0100 @@ -46,8 +46,8 @@ AC_C_CONST AC_CHECK_SIZEOF(long) dnl Checks for library functions -AC_CHECK_FUNC(ibv_register_driver, [], - AC_MSG_ERROR([ibv_register_driver() not found. libmlx4 requires libibverbs >= 1.1.7.])) +AC_CHECK_FUNC(verbs_register_driver, [], + AC_MSG_ERROR([verbs_register_driver() not found. libmlx4 requires libibverbs >= 1.1.8.])) AC_CACHE_CHECK([for IBV_QPT_RAW_PACKET], [ac_cv_ibv_qpt_raw_packet], @@ -56,7 +56,7 @@ AC_CACHE_CHECK([for IBV_QPT_RAW_PACKET], [ac_cv_ibv_qpt_raw_packet=yes], [ac_cv_ibv_qpt_raw_packet=no])]) if test $ac_cv_ibv_qpt_raw_packet = no; then - AC_MSG_ERROR([IBV_QPT_RAW_PACKET not found. libmlx4 requires libibverbs >= 1.1.7.]) + AC_MSG_ERROR([IBV_QPT_RAW_PACKET not found. libmlx4 requires libibverbs >= 1.1.8.]) fi dnl Now check if for libibverbs 1.0 vs 1.1 Index: libmlx4-1.0.5/libmlx4.spec.in =================================================================== --- libmlx4-1.0.5.orig/libmlx4.spec.in 2014-03-04 12:29:28.091637017 +0100 +++ libmlx4-1.0.5/libmlx4.spec.in 2014-03-04 12:29:28.109636602 +0100 @@ -9,7 +9,7 @@ Url: http://openfabrics.org/ Source: http://openfabrics.org/downloads/mlx4/libmlx4-1.0.5.tar.gz BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX) -BuildRequires: libibverbs-devel >= 1.1.7 +BuildRequires: libibverbs-devel >= 1.1.8 %description libmlx4 provides a device-specific userspace driver for Mellanox Index: libmlx4-1.0.5/src/mlx4.c =================================================================== --- libmlx4-1.0.5.orig/src/mlx4.c 2014-03-04 12:29:28.092636994 +0100 +++ libmlx4-1.0.5/src/mlx4.c 2014-03-04 12:41:02.019614627 +0100 @@ -116,7 +116,8 @@ static struct ibv_context_ops mlx4_ctx_o .detach_mcast = ibv_cmd_detach_mcast }; -static struct ibv_context *mlx4_alloc_context(struct ibv_device *ibdev, int cmd_fd) +static int mlx4_init_context(struct verbs_device *v_device, + struct ibv_context *ibv_ctx, int cmd_fd) { struct mlx4_context *context; struct ibv_get_context cmd; @@ -124,26 +125,30 @@ static struct ibv_context *mlx4_alloc_co int i; struct mlx4_alloc_ucontext_resp_v3 resp_v3; __u16 bf_reg_size; - struct mlx4_device *dev = to_mdev(ibdev); + struct mlx4_device *dev = to_mdev(&v_device->device); + /* verbs_context should be used for new verbs + * struct verbs_context *verbs_ctx = verbs_get_ctx(ibv_ctx); + */ - context = calloc(1, sizeof *context); - if (!context) - return NULL; - context->ibv_ctx.cmd_fd = cmd_fd; + /* memory footprint of mlx4_context and verbs_context share + * struct ibv_context. + */ + context = to_mctx(ibv_ctx); + ibv_ctx->cmd_fd = cmd_fd; if (dev->abi_version <= MLX4_UVERBS_NO_DEV_CAPS_ABI_VERSION) { - if (ibv_cmd_get_context(&context->ibv_ctx, &cmd, sizeof cmd, + if (ibv_cmd_get_context(ibv_ctx, &cmd, sizeof cmd, &resp_v3.ibv_resp, sizeof resp_v3)) - goto err_free; + return errno; context->num_qps = resp_v3.qp_tab_size; bf_reg_size = resp_v3.bf_reg_size; context->cqe_size = sizeof (struct mlx4_cqe); } else { - if (ibv_cmd_get_context(&context->ibv_ctx, &cmd, sizeof cmd, + if (ibv_cmd_get_context(ibv_ctx, &cmd, sizeof cmd, &resp.ibv_resp, sizeof resp)) - goto err_free; + return errno; context->num_qps = resp.qp_tab_size; bf_reg_size = resp.bf_reg_size; @@ -165,15 +170,15 @@ static struct ibv_context *mlx4_alloc_co pthread_mutex_init(&context->db_list_mutex, NULL); - context->uar = mmap(NULL, to_mdev(ibdev)->page_size, PROT_WRITE, + context->uar = mmap(NULL, dev->page_size, PROT_WRITE, MAP_SHARED, cmd_fd, 0); if (context->uar == MAP_FAILED) - goto err_free; + return errno; if (bf_reg_size) { - context->bf_page = mmap(NULL, to_mdev(ibdev)->page_size, + context->bf_page = mmap(NULL, dev->page_size, PROT_WRITE, MAP_SHARED, cmd_fd, - to_mdev(ibdev)->page_size); + dev->page_size); if (context->bf_page == MAP_FAILED) { fprintf(stderr, PFX "Warning: BlueFlame available, " "but failed to mmap() BlueFlame page.\n"); @@ -190,32 +195,23 @@ static struct ibv_context *mlx4_alloc_co } pthread_spin_init(&context->uar_lock, PTHREAD_PROCESS_PRIVATE); + ibv_ctx->ops = mlx4_ctx_ops; + /* New verbs should be added by using verbs_set_ctx_op */ - context->ibv_ctx.ops = mlx4_ctx_ops; - - return &context->ibv_ctx; - -err_free: - free(context); - return NULL; + return 0; } -static void mlx4_free_context(struct ibv_context *ibctx) +static void mlx4_uninit_context(struct verbs_device *v_device, + struct ibv_context *ibv_ctx) { - struct mlx4_context *context = to_mctx(ibctx); + struct mlx4_context *context = to_mctx(ibv_ctx); - munmap(context->uar, to_mdev(ibctx->device)->page_size); + munmap(context->uar, to_mdev(&v_device->device)->page_size); if (context->bf_page) - munmap(context->bf_page, to_mdev(ibctx->device)->page_size); - free(context); + munmap(context->bf_page, to_mdev(&v_device->device)->page_size); } -static struct ibv_device_ops mlx4_dev_ops = { - .alloc_context = mlx4_alloc_context, - .free_context = mlx4_free_context -}; - -static struct ibv_device *mlx4_driver_init(const char *uverbs_sys_path, int abi_version) +static struct verbs_device *mlx4_driver_init(const char *uverbs_sys_path, int abi_version) { char value[8]; struct mlx4_device *dev; @@ -250,21 +246,27 @@ found: return NULL; } - dev = malloc(sizeof *dev); + dev = calloc(1, sizeof *dev); if (!dev) { fprintf(stderr, PFX "Fatal: couldn't allocate device for %s\n", uverbs_sys_path); return NULL; } - dev->ibv_dev.ops = mlx4_dev_ops; dev->page_size = sysconf(_SC_PAGESIZE); dev->abi_version = abi_version; - return &dev->ibv_dev; + dev->verbs_dev.sz = sizeof(*dev); + dev->verbs_dev.size_of_context = + sizeof(struct mlx4_context) - sizeof(struct ibv_context); + /* mlx4_init_context will initialize provider calls */ + dev->verbs_dev.init_context = mlx4_init_context; + dev->verbs_dev.uninit_context = mlx4_uninit_context; + + return &dev->verbs_dev; } static __attribute__((constructor)) void mlx4_register_driver(void) { - ibv_register_driver("mlx4", mlx4_driver_init); + verbs_register_driver("mlx4", mlx4_driver_init); } Index: libmlx4-1.0.5/src/mlx4.h =================================================================== --- libmlx4-1.0.5.orig/src/mlx4.h 2014-03-04 12:29:28.092636994 +0100 +++ libmlx4-1.0.5/src/mlx4.h 2014-03-04 12:39:48.446313644 +0100 @@ -127,7 +127,7 @@ enum { }; struct mlx4_device { - struct ibv_device ibv_dev; + struct verbs_device verbs_dev; int page_size; int abi_version; }; @@ -272,7 +272,10 @@ static inline unsigned long align(unsign static inline struct mlx4_device *to_mdev(struct ibv_device *ibdev) { - return to_mxxx(dev, device); + /* ibv_device is first field of verbs_device + * see try_driver() in libibverbs. + */ + return container_of(ibdev, struct mlx4_device, verbs_dev); } static inline struct mlx4_context *to_mctx(struct ibv_context *ibctx) ++++++ libmlx4-fix_type_punning.patch ++++++ --- src/qp.c | 3 ++- src/verbs.c | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) Index: src/qp.c =================================================================== --- src/qp.c.orig 2013-04-03 20:34:11.000000000 +0200 +++ src/qp.c 2014-01-21 20:15:55.762472770 +0100 @@ -403,8 +403,9 @@ out: ctx = to_mctx(ibqp->context); if (nreq == 1 && inl && size > 1 && size <= ctx->bf_buf_size / 16) { + union{ uint8_t *ui8p; uint32_t *ui32p; }pun = { ctrl->reserved }; ctrl->owner_opcode |= htonl((qp->sq.head & 0xffff) << 8); - *(uint32_t *) ctrl->reserved |= qp->doorbell_qpn; + *pun.ui32p |= qp->doorbell_qpn; /* * Make sure that descriptor is written to memory * before writing to BlueFlame page. Index: src/verbs.c =================================================================== --- src/verbs.c.orig 2013-04-03 20:21:48.000000000 +0200 +++ src/verbs.c 2014-01-21 20:17:45.631928219 +0100 @@ -628,8 +628,10 @@ int mlx4_destroy_qp(struct ibv_qp *ibqp) static int link_local_gid(const union ibv_gid *gid) { - uint32_t hi = *(uint32_t *)(gid->raw); - uint32_t lo = *(uint32_t *)(gid->raw + 4); + uint32_t hi, lo; + memcpy(&hi, gid->raw, sizeof(uint32_t)); + memcpy(&lo, gid->raw + 4, sizeof(uint32_t)); + if (hi == htonl(0xfe800000) && lo == 0) return 1; ++++++ libmlx4-resize_cq-Fix-possible-endless-loop-scanning-CQ.patch ++++++ >From 65585f680dc08d45d05f82320aab52c3d3221e82 Mon Sep 17 00:00:00 2001 From: Roland Dreier <[email protected]> Date: Fri, 15 Nov 2013 10:24:08 -0800 Subject: [PATCH 1/4] resize_cq: Fix possible endless loop scanning CQ When calling get_sw_cqe() we need pass the consumer_index and not the masked value. Failure to do so will cause incorrect result of get_sw_cqe() possibly leading to endless loop. Based on a patch by Eli Cohen <[email protected]> fixing the same bug in the kernel. Signed-off-by: Roland Dreier <[email protected]> --- src/cq.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/src/cq.c b/src/cq.c index 18447c4..bd9bf0f 100644 --- a/src/cq.c +++ b/src/cq.c @@ -439,7 +439,7 @@ int mlx4_get_outstanding_cqes(struct mlx4_cq *cq) { uint32_t i; - for (i = cq->cons_index; get_sw_cqe(cq, (i & cq->ibv_cq.cqe)); ++i) + for (i = cq->cons_index; get_sw_cqe(cq, i); ++i) ; return i - cq->cons_index; -- 1.7.8.2 ++++++ libmlx4-s390.patch ++++++ Cc: [email protected] Subject: DAPL support for Linux on System z s390: Enable libmlx4 for use with DAPL on Linux on System z The changes to libmlx4-1.0.5 userpace Mellanox device driver library required to provide support for the DAPL programming interface on the System z platform. The following System z-specific changes have been made: * code that used the mapped memory areas to access the PCI memory of the Mellanox networking device was replaced by the kernelspace device driver primitive calls. --- src/doorbell.h | 67 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ src/mlx4.c | 11 ++++++++- src/mlx4.h | 6 +++++ src/qp.c | 19 +++++++++++++++- 4 files changed, 101 insertions(+), 2 deletions(-) Index: libmlx4-1.0.5/src/doorbell.h =================================================================== --- libmlx4-1.0.5.orig/src/doorbell.h 2011-03-08 18:56:59.000000000 +0100 +++ libmlx4-1.0.5/src/doorbell.h 2014-03-04 12:43:02.656828680 +0100 @@ -33,6 +33,7 @@ #ifndef DOORBELL_H #define DOORBELL_H +#ifndef __s390x__ #if SIZEOF_LONG == 8 #if __BYTE_ORDER == __LITTLE_ENDIAN @@ -59,5 +60,71 @@ static inline void mlx4_write64(uint32_t } #endif +#else /* __s390x__ */ + +#include <stdio.h> +#include <unistd.h> +#include <errno.h> +#include <infiniband/driver.h> +#include <infiniband/arch.h> + +enum ib_uverbs_kwrite_mmio_location { + IB_UVERBS_KWRITE_MMIO_UAR, + IB_UVERBS_KWRITE_MMIO_BF_PAGE +}; + +struct mlx4_kwrite_mmio { + __u32 command; + __u16 in_words; + __u16 out_words; + __u16 offset; + __u16 length; + __u8 location; + __u8 reserved[3]; + __u8 value[0]; +}; + +static inline int mlx4_kwrite_mmio(int fd, + uint8_t location, + uint32_t offset, + uint32_t length, + void *value) +{ + struct mlx4_kwrite_mmio *cmd = NULL; + int cmd_size = 0; + int ret = 0; + + cmd_size = sizeof(*cmd) + length; + cmd = calloc(1, cmd_size); + if (!cmd) + return -ENOMEM; + + cmd->command = IB_USER_VERBS_CMD_KWRITE_MMIO; + cmd->in_words = cmd_size / 4; + cmd->out_words = 0; + cmd->offset = offset; + cmd->length = length; + cmd->location = location; + + memcpy(cmd->value, value, length); + + ret = write(fd, cmd, cmd_size); + free(cmd); + + return ret; +} + +static inline void mlx4_write64(uint32_t val[2], + struct mlx4_context *ctx, + int offset) +{ + mlx4_kwrite_mmio(ctx->ibv_ctx.cmd_fd, + IB_UVERBS_KWRITE_MMIO_UAR, /* UAR page */ + offset, + 2 * sizeof(val[0]), + val); +} + +#endif #endif /* DOORBELL_H */ Index: libmlx4-1.0.5/src/mlx4.c =================================================================== --- libmlx4-1.0.5.orig/src/mlx4.c 2014-03-04 12:43:02.647828888 +0100 +++ libmlx4-1.0.5/src/mlx4.c 2014-03-04 12:43:42.321912647 +0100 @@ -168,12 +168,14 @@ static int mlx4_init_context(struct verb mlx4_init_xsrq_table(&context->xsrq_table, context->num_qps); pthread_mutex_init(&context->db_list_mutex, NULL); +#ifndef __s390x__ context->uar = mmap(NULL, dev->page_size, PROT_WRITE, MAP_SHARED, cmd_fd, 0); if (context->uar == MAP_FAILED) return errno; - +#endif if (bf_reg_size) { +#ifndef __s390x__ context->bf_page = mmap(NULL, dev->page_size, PROT_WRITE, MAP_SHARED, cmd_fd, dev->page_size); @@ -183,12 +185,17 @@ static int mlx4_init_context(struct verb context->bf_page = NULL; context->bf_buf_size = 0; } else { +#endif context->bf_buf_size = bf_reg_size / 2; context->bf_offset = 0; pthread_spin_init(&context->bf_lock, PTHREAD_PROCESS_PRIVATE); +#ifndef __s390x__ } +#endif } else { +#ifndef __s390x__ context->bf_page = NULL; +#endif context->bf_buf_size = 0; } @@ -210,11 +217,13 @@ static int mlx4_init_context(struct verb static void mlx4_uninit_context(struct verbs_device *v_device, struct ibv_context *ibv_ctx) { +#ifndef __s390x__ struct mlx4_context *context = to_mctx(ibv_ctx); munmap(context->uar, to_mdev(&v_device->device)->page_size); if (context->bf_page) munmap(context->bf_page, to_mdev(&v_device->device)->page_size); +#endif } static struct verbs_device *mlx4_driver_init(const char *uverbs_sys_path, int abi_version) Index: libmlx4-1.0.5/src/mlx4.h =================================================================== --- libmlx4-1.0.5.orig/src/mlx4.h 2014-03-04 12:43:02.647828888 +0100 +++ libmlx4-1.0.5/src/mlx4.h 2014-03-04 12:43:02.657828657 +0100 @@ -74,6 +74,8 @@ #define wc_wmb() asm volatile("sfence" ::: "memory") #elif defined(__ia64__) #define wc_wmb() asm volatile("fwb" ::: "memory") +#elif defined(__s390x__) +#define wc_wmb asm volatile(::: "memory") #else #define wc_wmb() wmb() #endif @@ -168,10 +170,14 @@ struct mlx4_db_page; struct mlx4_context { struct ibv_context ibv_ctx; +#ifndef __s390x__ void *uar; +#endif pthread_spinlock_t uar_lock; +#ifndef __s390x__ void *bf_page; +#endif int bf_buf_size; int bf_offset; pthread_spinlock_t bf_lock; Index: libmlx4-1.0.5/src/qp.c =================================================================== --- libmlx4-1.0.5.orig/src/qp.c 2014-03-04 12:43:02.648828865 +0100 +++ libmlx4-1.0.5/src/qp.c 2014-03-04 12:43:02.657828657 +0100 @@ -173,6 +173,7 @@ static void set_data_seg(struct mlx4_wqe dseg->byte_count = htonl(sg->length); } +#ifndef __s390x__ /* * Avoid using memcpy() to copy to BlueFlame page, since memcpy() * implementations may use move-string-buffer assembler instructions, @@ -187,6 +188,7 @@ static void mlx4_bf_copy(unsigned long * } } +#endif int mlx4_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr, struct ibv_send_wr **bad_wr) { @@ -419,8 +421,16 @@ out: pthread_spin_lock(&ctx->bf_lock); +#ifndef __s390x__ mlx4_bf_copy(ctx->bf_page + ctx->bf_offset, (unsigned long *) ctrl, align(size * 16, 64)); +#else + mlx4_kwrite_mmio(ctx->ibv_ctx.cmd_fd, + IB_UVERBS_KWRITE_MMIO_BF_PAGE, /* BF page */ + ctx->bf_offset, + align(size * 16, 64), + ctrl); +#endif wc_wmb(); ctx->bf_offset ^= ctx->bf_buf_size; @@ -434,8 +444,15 @@ out: * doorbell record. */ wmb(); - +#ifndef __s390x__ *(uint32_t *) (ctx->uar + MLX4_SEND_DOORBELL) = qp->doorbell_qpn; +#else + mlx4_kwrite_mmio(ctx->ibv_ctx.cmd_fd, + IB_UVERBS_KWRITE_MMIO_UAR, /* UAR page */ + MLX4_SEND_DOORBELL, + sizeof(qp->doorbell_qpn), + &qp->doorbell_qpn); +#endif } if (nreq) -- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
