The branch main has been updated by jhb: URL: https://cgit.FreeBSD.org/src/commit/?id=8d57de34d51eeb70d758f2d463cf7cf2d829dd49
commit 8d57de34d51eeb70d758f2d463cf7cf2d829dd49 Author: John Baldwin <[email protected]> AuthorDate: 2026-06-24 19:30:55 +0000 Commit: John Baldwin <[email protected]> CommitDate: 2026-06-24 19:30:55 +0000 OFED: Various changes from Linux 4.11 This contains changes from the following Linux commits: 55efcfcd7776 RDMA/core: Fix incorrect structure packing for booleans aaaca121c7cf RDMA/core: add port state cache 9e2c3f1c7f3e RDMA/core: export ib_get_cached_port_state 93b1f29de71f RDMA/cma: resolve to first active ib port d43dbacfc063 IB/core: Change the type of an ib_dma_alloc_coherent() argument 6532c380bf40 IB/core: Remove ib_dma_*map_single_attrs() 0bbb3b7496ea IB/rxe, IB/rdmavt: Use dma_virt_ops instead of duplicating it f57e8ca50e23 IB/mad: Add port_num to error message 828f6fa65ce7 IB/umem: Release pid in error and ODP flow 19b752a19dce IB/cma: Allow port reuse for rdma_id 21d6454a392d RDMA/core: create struct ib_port_cache 25bf14d6f589 IB/core: Add implicit MR flag d9d0674c0f8a IB/umem: Indicate that process is being terminated d07d1d70ce1a IB/umem: Update on demand page (ODP) support a748d60df32e IB/mlx4: Take source GID by index from HW GID table 89052d784bc9 IB/cma: Add default RoCE TOS to CMA configfs 6df6b4a9ce43 IB/cma: Destination and source addr families must match On FreeBSD, the default RoCE TOS setting is available as per-port sysctls similar to the existing default RoCE mode sysctls. Tested by: Wafa Hamzah <[email protected]> (mlx5_ib) Tested by: John Baldwin <[email protected]> (iw_cxgbe) Sponsored by: Chelsio Communications --- sys/dev/mlx4/mlx4_ib/mlx4_ib_qp.c | 56 ++++---- sys/ofed/drivers/infiniband/core/core_priv.h | 3 + sys/ofed/drivers/infiniband/core/ib_cache.c | 145 ++++++++++--------- sys/ofed/drivers/infiniband/core/ib_cma.c | 161 ++++++++++++++++++++-- sys/ofed/drivers/infiniband/core/ib_mad.c | 4 +- sys/ofed/drivers/infiniband/core/ib_umem.c | 5 +- sys/ofed/drivers/infiniband/core/ib_umem_odp.c | 92 +++++++++++-- sys/ofed/drivers/infiniband/core/ib_umem_rbtree.c | 21 ++- sys/ofed/include/rdma/ib_cache.h | 13 ++ sys/ofed/include/rdma/ib_sa.h | 6 +- sys/ofed/include/rdma/ib_umem_odp.h | 21 ++- sys/ofed/include/rdma/ib_verbs.h | 148 +++----------------- 12 files changed, 410 insertions(+), 265 deletions(-) diff --git a/sys/dev/mlx4/mlx4_ib/mlx4_ib_qp.c b/sys/dev/mlx4/mlx4_ib/mlx4_ib_qp.c index b391946440b9..feea6c2a4edf 100644 --- a/sys/dev/mlx4/mlx4_ib/mlx4_ib_qp.c +++ b/sys/dev/mlx4/mlx4_ib/mlx4_ib_qp.c @@ -2423,11 +2423,31 @@ static u8 sl_to_vl(struct mlx4_ib_dev *dev, u8 sl, int port_num) return vl; } +static int fill_gid_by_hw_index(struct mlx4_ib_dev *ibdev, u8 port_num, + int index, union ib_gid *gid, + enum ib_gid_type *gid_type) +{ + struct mlx4_ib_iboe *iboe = &ibdev->iboe; + struct mlx4_port_gid_table *port_gid_table; + unsigned long flags; + + port_gid_table = &iboe->gids[port_num - 1]; + spin_lock_irqsave(&iboe->lock, flags); + memcpy(gid, &port_gid_table->gids[index].gid, sizeof(*gid)); + *gid_type = port_gid_table->gids[index].gid_type; + spin_unlock_irqrestore(&iboe->lock, flags); + if (!memcmp(gid, &zgid, sizeof(*gid))) + return -ENOENT; + + return 0; +} + #define MLX4_ROCEV2_QP1_SPORT 0xC000 static int build_mlx_header(struct mlx4_ib_sqp *sqp, const struct ib_ud_wr *wr, void *wqe, unsigned *mlx_seg_len) { struct ib_device *ib_dev = sqp->qp.ibqp.device; + struct mlx4_ib_dev *ibdev = to_mdev(ib_dev); struct mlx4_wqe_mlx_seg *mlx = wqe; struct mlx4_wqe_ctrl_seg *ctrl = wqe; struct mlx4_wqe_inline_seg *inl = wqe + sizeof *mlx; @@ -2453,8 +2473,7 @@ static int build_mlx_header(struct mlx4_ib_sqp *sqp, const struct ib_ud_wr *wr, is_eth = rdma_port_get_link_layer(sqp->qp.ibqp.device, sqp->qp.port) == IB_LINK_LAYER_ETHERNET; is_grh = mlx4_ib_ah_grh_present(ah); if (is_eth) { - struct ib_gid_attr gid_attr; - + enum ib_gid_type gid_type; if (mlx4_is_mfunc(to_mdev(ib_dev)->dev)) { /* When multi-function is enabled, the ib_core gid * indexes don't necessarily match the hw ones, so @@ -2465,18 +2484,11 @@ static int build_mlx_header(struct mlx4_ib_sqp *sqp, const struct ib_ud_wr *wr, if (err) return err; } else { - err = ib_get_cached_gid(ib_dev, - be32_to_cpu(ah->av.ib.port_pd) >> 24, - ah->av.ib.gid_index, &sgid, - &gid_attr); - if (!err) { - if (gid_attr.ndev) - if_rele(gid_attr.ndev); - if (!memcmp(&sgid, &zgid, sizeof(sgid))) - err = -ENOENT; - } + err = fill_gid_by_hw_index(ibdev, sqp->qp.port, + ah->av.ib.gid_index, + &sgid, &gid_type); if (!err) { - is_udp = gid_attr.gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP; + is_udp = gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP; if (is_udp) { if (ipv6_addr_v4mapped((struct in6_addr *)&sgid)) ip_version = 4; @@ -2954,21 +2966,17 @@ int mlx4_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, if (sqp->roce_v2_gsi) { struct mlx4_ib_ah *ah = to_mah(ud_wr(wr)->ah); - struct ib_gid_attr gid_attr; + enum ib_gid_type gid_type; union ib_gid gid; - if (!ib_get_cached_gid(ibqp->device, - be32_to_cpu(ah->av.ib.port_pd) >> 24, - ah->av.ib.gid_index, &gid, - &gid_attr)) { - if (gid_attr.ndev) - if_rele(gid_attr.ndev); - qp = (gid_attr.gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) ? - to_mqp(sqp->roce_v2_gsi) : qp; - } else { + if (!fill_gid_by_hw_index(mdev, sqp->qp.port, + ah->av.ib.gid_index, + &gid, &gid_type)) + qp = (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) ? + to_mqp(sqp->roce_v2_gsi) : qp; + else pr_err("Failed to get gid at index %d. RoCEv2 will not work properly\n", ah->av.ib.gid_index); - } } } diff --git a/sys/ofed/drivers/infiniband/core/core_priv.h b/sys/ofed/drivers/infiniband/core/core_priv.h index ccacb7c3f57f..e8c70de4fac0 100644 --- a/sys/ofed/drivers/infiniband/core/core_priv.h +++ b/sys/ofed/drivers/infiniband/core/core_priv.h @@ -69,6 +69,9 @@ int cma_get_default_gid_type(struct cma_device *cma_dev, int cma_set_default_gid_type(struct cma_device *cma_dev, unsigned int port, enum ib_gid_type default_gid_type); +int cma_get_default_roce_tos(struct cma_device *cma_dev, unsigned int port); +int cma_set_default_roce_tos(struct cma_device *a_dev, unsigned int port, + u8 default_roce_tos); struct ib_device *cma_get_ib_dev(struct cma_device *cma_dev); int ib_device_register_sysfs(struct ib_device *device, diff --git a/sys/ofed/drivers/infiniband/core/ib_cache.c b/sys/ofed/drivers/infiniband/core/ib_cache.c index 1f5ef4c5517e..d8960825427c 100644 --- a/sys/ofed/drivers/infiniband/core/ib_cache.c +++ b/sys/ofed/drivers/infiniband/core/ib_cache.c @@ -332,13 +332,12 @@ static void make_default_gid(if_t dev, union ib_gid *gid) int ib_cache_gid_add(struct ib_device *ib_dev, u8 port, union ib_gid *gid, struct ib_gid_attr *attr) { - struct ib_gid_table **ports_table = ib_dev->cache.gid_cache; struct ib_gid_table *table; int ix; int ret = 0; int empty; - table = ports_table[port - rdma_start_port(ib_dev)]; + table = ib_dev->cache.ports[port - rdma_start_port(ib_dev)].gid; if (!memcmp(gid, &zgid, sizeof(*gid))) return -EINVAL; @@ -370,11 +369,10 @@ out_unlock: int ib_cache_gid_del(struct ib_device *ib_dev, u8 port, union ib_gid *gid, struct ib_gid_attr *attr) { - struct ib_gid_table **ports_table = ib_dev->cache.gid_cache; struct ib_gid_table *table; int ix; - table = ports_table[port - rdma_start_port(ib_dev)]; + table = ib_dev->cache.ports[port - rdma_start_port(ib_dev)].gid; mutex_lock(&table->lock); write_lock_irq(&table->rwlock); @@ -400,12 +398,11 @@ out_unlock: int ib_cache_gid_del_all_netdev_gids(struct ib_device *ib_dev, u8 port, if_t ndev) { - struct ib_gid_table **ports_table = ib_dev->cache.gid_cache; struct ib_gid_table *table; int ix; bool deleted = false; - table = ports_table[port - rdma_start_port(ib_dev)]; + table = ib_dev->cache.ports[port - rdma_start_port(ib_dev)].gid; mutex_lock(&table->lock); write_lock_irq(&table->rwlock); @@ -429,10 +426,9 @@ int ib_cache_gid_del_all_netdev_gids(struct ib_device *ib_dev, u8 port, static int __ib_cache_gid_get(struct ib_device *ib_dev, u8 port, int index, union ib_gid *gid, struct ib_gid_attr *attr) { - struct ib_gid_table **ports_table = ib_dev->cache.gid_cache; struct ib_gid_table *table; - table = ports_table[port - rdma_start_port(ib_dev)]; + table = ib_dev->cache.ports[port - rdma_start_port(ib_dev)].gid; if (index < 0 || index >= table->sz) return -EINVAL; @@ -461,14 +457,13 @@ static int _ib_cache_gid_table_find(struct ib_device *ib_dev, unsigned long mask, u8 *port, u16 *index) { - struct ib_gid_table **ports_table = ib_dev->cache.gid_cache; struct ib_gid_table *table; u8 p; int local_index; unsigned long flags __writeonly; for (p = 0; p < ib_dev->phys_port_cnt; p++) { - table = ports_table[p]; + table = ib_dev->cache.ports[p].gid; read_lock_irqsave(&table->rwlock, flags); local_index = find_gid(table, gid, val, false, mask, NULL); if (local_index >= 0) { @@ -509,7 +504,6 @@ int ib_find_cached_gid_by_port(struct ib_device *ib_dev, u16 *index) { int local_index; - struct ib_gid_table **ports_table = ib_dev->cache.gid_cache; struct ib_gid_table *table; unsigned long mask = GID_ATTR_FIND_MASK_GID | GID_ATTR_FIND_MASK_GID_TYPE; @@ -519,7 +513,7 @@ int ib_find_cached_gid_by_port(struct ib_device *ib_dev, if (!rdma_is_port_valid(ib_dev, port)) return -ENOENT; - table = ports_table[port - rdma_start_port(ib_dev)]; + table = ib_dev->cache.ports[port - rdma_start_port(ib_dev)].gid; if (ndev) mask |= GID_ATTR_FIND_MASK_NETDEV; @@ -567,22 +561,18 @@ static int ib_cache_gid_find_by_filter(struct ib_device *ib_dev, void *context, u16 *index) { - struct ib_gid_table **ports_table = ib_dev->cache.gid_cache; struct ib_gid_table *table; unsigned int i; unsigned long flags __writeonly; bool found = false; - if (!ports_table) - return -EOPNOTSUPP; - if (!rdma_is_port_valid(ib_dev, port)) return -EINVAL; if (!rdma_protocol_roce(ib_dev, port)) return -EPROTONOSUPPORT; - table = ports_table[port - rdma_start_port(ib_dev)]; + table = ib_dev->cache.ports[port - rdma_start_port(ib_dev)].gid; read_lock_irqsave(&table->rwlock, flags); for (i = 0; i < table->sz; i++) { @@ -674,14 +664,13 @@ void ib_cache_gid_set_default_gid(struct ib_device *ib_dev, u8 port, unsigned long gid_type_mask, enum ib_cache_gid_default_mode mode) { - struct ib_gid_table **ports_table = ib_dev->cache.gid_cache; union ib_gid gid; struct ib_gid_attr gid_attr; struct ib_gid_attr zattr_type = zattr; struct ib_gid_table *table; unsigned int gid_type; - table = ports_table[port - rdma_start_port(ib_dev)]; + table = ib_dev->cache.ports[port - rdma_start_port(ib_dev)].gid; make_default_gid(ndev, &gid); memset(&gid_attr, 0, sizeof(gid_attr)); @@ -782,71 +771,64 @@ static int gid_table_reserve_default(struct ib_device *ib_dev, u8 port, static int _gid_table_setup_one(struct ib_device *ib_dev) { u8 port; - struct ib_gid_table **table; + struct ib_gid_table *table; int err = 0; - table = kcalloc(ib_dev->phys_port_cnt, sizeof(*table), GFP_KERNEL); - if (!table) - return -ENOMEM; - for (port = 0; port < ib_dev->phys_port_cnt; port++) { u8 rdma_port = port + rdma_start_port(ib_dev); - table[port] = + table = alloc_gid_table( ib_dev->port_immutable[rdma_port].gid_tbl_len); - if (!table[port]) { + if (!table) { err = -ENOMEM; goto rollback_table_setup; } err = gid_table_reserve_default(ib_dev, port + rdma_start_port(ib_dev), - table[port]); + table); if (err) goto rollback_table_setup; + ib_dev->cache.ports[port].gid = table; } - ib_dev->cache.gid_cache = table; return 0; rollback_table_setup: for (port = 0; port < ib_dev->phys_port_cnt; port++) { + table = ib_dev->cache.ports[port].gid; + cleanup_gid_table_port(ib_dev, port + rdma_start_port(ib_dev), - table[port]); - release_gid_table(table[port]); + table); + release_gid_table(table); } - kfree(table); return err; } static void gid_table_release_one(struct ib_device *ib_dev) { - struct ib_gid_table **table = ib_dev->cache.gid_cache; + struct ib_gid_table *table; u8 port; - if (!table) - return; - - for (port = 0; port < ib_dev->phys_port_cnt; port++) - release_gid_table(table[port]); - - kfree(table); - ib_dev->cache.gid_cache = NULL; + for (port = 0; port < ib_dev->phys_port_cnt; port++) { + table = ib_dev->cache.ports[port].gid; + release_gid_table(table); + ib_dev->cache.ports[port].gid = NULL; + } } static void gid_table_cleanup_one(struct ib_device *ib_dev) { - struct ib_gid_table **table = ib_dev->cache.gid_cache; + struct ib_gid_table *table; u8 port; - if (!table) - return; - - for (port = 0; port < ib_dev->phys_port_cnt; port++) + for (port = 0; port < ib_dev->phys_port_cnt; port++) { + table = ib_dev->cache.ports[port].gid; cleanup_gid_table_port(ib_dev, port + rdma_start_port(ib_dev), - table[port]); + table); + } } static int gid_table_setup_one(struct ib_device *ib_dev) @@ -876,12 +858,12 @@ int ib_get_cached_gid(struct ib_device *device, { int res; unsigned long flags __writeonly; - struct ib_gid_table **ports_table = device->cache.gid_cache; - struct ib_gid_table *table = ports_table[port_num - rdma_start_port(device)]; + struct ib_gid_table *table; if (!rdma_is_port_valid(device, port_num)) return -EINVAL; + table = device->cache.ports[port_num - rdma_start_port(device)].gid; read_lock_irqsave(&table->rwlock, flags); res = __ib_cache_gid_get(device, port_num, index, gid, gid_attr); read_unlock_irqrestore(&table->rwlock, flags); @@ -933,7 +915,7 @@ int ib_get_cached_pkey(struct ib_device *device, read_lock_irqsave(&device->cache.lock, flags); - cache = device->cache.pkey_cache[port_num - rdma_start_port(device)]; + cache = device->cache.ports[port_num - rdma_start_port(device)].pkey; if (index < 0 || index >= cache->table_len) ret = -EINVAL; @@ -962,7 +944,7 @@ int ib_find_cached_pkey(struct ib_device *device, read_lock_irqsave(&device->cache.lock, flags); - cache = device->cache.pkey_cache[port_num - rdma_start_port(device)]; + cache = device->cache.ports[port_num - rdma_start_port(device)].pkey; *index = -1; @@ -1002,7 +984,7 @@ int ib_find_exact_cached_pkey(struct ib_device *device, read_lock_irqsave(&device->cache.lock, flags); - cache = device->cache.pkey_cache[port_num - rdma_start_port(device)]; + cache = device->cache.ports[port_num - rdma_start_port(device)].pkey; *index = -1; @@ -1030,13 +1012,32 @@ int ib_get_cached_lmc(struct ib_device *device, return -EINVAL; read_lock_irqsave(&device->cache.lock, flags); - *lmc = device->cache.lmc_cache[port_num - rdma_start_port(device)]; + *lmc = device->cache.ports[port_num - rdma_start_port(device)].lmc; read_unlock_irqrestore(&device->cache.lock, flags); return ret; } EXPORT_SYMBOL(ib_get_cached_lmc); +int ib_get_cached_port_state(struct ib_device *device, + u8 port_num, + enum ib_port_state *port_state) +{ + unsigned long flags; + int ret = 0; + + if (!rdma_is_port_valid(device, port_num)) + return -EINVAL; + + read_lock_irqsave(&device->cache.lock, flags); + *port_state = device->cache.ports[port_num + - rdma_start_port(device)].port_state; + read_unlock_irqrestore(&device->cache.lock, flags); + + return ret; +} +EXPORT_SYMBOL(ib_get_cached_port_state); + static void ib_cache_update(struct ib_device *device, u8 port) { @@ -1049,14 +1050,13 @@ static void ib_cache_update(struct ib_device *device, int i; int ret; struct ib_gid_table *table; - struct ib_gid_table **ports_table = device->cache.gid_cache; bool use_roce_gid_table = rdma_cap_roce_gid_table(device, port); if (!rdma_is_port_valid(device, port)) return; - table = ports_table[port - rdma_start_port(device)]; + table = device->cache.ports[port - rdma_start_port(device)].gid; tprops = kmalloc(sizeof *tprops, GFP_KERNEL); if (!tprops) @@ -1108,9 +1108,10 @@ static void ib_cache_update(struct ib_device *device, write_lock_irq(&device->cache.lock); - old_pkey_cache = device->cache.pkey_cache[port - rdma_start_port(device)]; + old_pkey_cache = device->cache.ports[port - + rdma_start_port(device)].pkey; - device->cache.pkey_cache[port - rdma_start_port(device)] = pkey_cache; + device->cache.ports[port - rdma_start_port(device)].pkey = pkey_cache; if (!use_roce_gid_table) { write_lock(&table->rwlock); for (i = 0; i < gid_cache->table_len; i++) { @@ -1120,7 +1121,9 @@ static void ib_cache_update(struct ib_device *device, write_unlock(&table->rwlock); } - device->cache.lmc_cache[port - rdma_start_port(device)] = tprops->lmc; + device->cache.ports[port - rdma_start_port(device)].lmc = tprops->lmc; + device->cache.ports[port - rdma_start_port(device)].port_state = + tprops->state; write_unlock_irq(&device->cache.lock); @@ -1173,22 +1176,17 @@ int ib_cache_setup_one(struct ib_device *device) rwlock_init(&device->cache.lock); - device->cache.pkey_cache = - kzalloc(sizeof *device->cache.pkey_cache * + device->cache.ports = + kzalloc(sizeof(*device->cache.ports) * (rdma_end_port(device) - rdma_start_port(device) + 1), GFP_KERNEL); - device->cache.lmc_cache = kmalloc(sizeof *device->cache.lmc_cache * - (rdma_end_port(device) - - rdma_start_port(device) + 1), - GFP_KERNEL); - if (!device->cache.pkey_cache || - !device->cache.lmc_cache) { + if (!device->cache.ports) { err = -ENOMEM; - goto free; + goto out; } err = gid_table_setup_one(device); if (err) - goto free; + goto out; for (p = 0; p <= rdma_end_port(device) - rdma_start_port(device); ++p) ib_cache_update(device, p + rdma_start_port(device)); @@ -1203,9 +1201,7 @@ int ib_cache_setup_one(struct ib_device *device) err: gid_table_cleanup_one(device); -free: - kfree(device->cache.pkey_cache); - kfree(device->cache.lmc_cache); +out: return err; } @@ -1219,14 +1215,11 @@ void ib_cache_release_one(struct ib_device *device) * all the device's resources when the cache could no * longer be accessed. */ - if (device->cache.pkey_cache) - for (p = 0; - p <= rdma_end_port(device) - rdma_start_port(device); ++p) - kfree(device->cache.pkey_cache[p]); + for (p = 0; p <= rdma_end_port(device) - rdma_start_port(device); ++p) + kfree(device->cache.ports[p].pkey); gid_table_release_one(device); - kfree(device->cache.pkey_cache); - kfree(device->cache.lmc_cache); + kfree(device->cache.ports); } void ib_cache_cleanup_one(struct ib_device *device) diff --git a/sys/ofed/drivers/infiniband/core/ib_cma.c b/sys/ofed/drivers/infiniband/core/ib_cma.c index 07d598af3471..a3f73b5e93fa 100644 --- a/sys/ofed/drivers/infiniband/core/ib_cma.c +++ b/sys/ofed/drivers/infiniband/core/ib_cma.c @@ -223,6 +223,7 @@ struct cma_device { struct list_head id_list; struct sysctl_ctx_list sysctl_ctx; enum ib_gid_type *default_gid_type; + u8 *default_roce_tos; }; struct rdma_bind_list { @@ -320,6 +321,25 @@ int cma_set_default_gid_type(struct cma_device *cma_dev, return 0; } +int cma_get_default_roce_tos(struct cma_device *cma_dev, unsigned int port) +{ + if (!rdma_is_port_valid(cma_dev->device, port)) + return -EINVAL; + + return cma_dev->default_roce_tos[port - rdma_start_port(cma_dev->device)]; +} + +int cma_set_default_roce_tos(struct cma_device *cma_dev, unsigned int port, + u8 default_roce_tos) +{ + if (!rdma_is_port_valid(cma_dev->device, port)) + return -EINVAL; + + cma_dev->default_roce_tos[port - rdma_start_port(cma_dev->device)] = + default_roce_tos; + + return 0; +} struct ib_device *cma_get_ib_dev(struct cma_device *cma_dev) { return cma_dev->device; @@ -366,6 +386,7 @@ struct rdma_id_private { u32 options; u8 srq; u8 tos; + u8 tos_set:1; u8 timeout_set:1; u8 reuseaddr; u8 afonly; @@ -734,6 +755,7 @@ static int cma_resolve_ib_dev(struct rdma_id_private *id_priv) union ib_gid gid, sgid, *dgid; u16 pkey, index; u8 p; + enum ib_port_state port_state; int i; cma_dev = NULL; @@ -749,6 +771,8 @@ static int cma_resolve_ib_dev(struct rdma_id_private *id_priv) if (ib_find_cached_pkey(cur_dev->device, p, pkey, &index)) continue; + if (ib_get_cached_port_state(cur_dev->device, p, &port_state)) + continue; for (i = 0; !ib_get_cached_gid(cur_dev->device, p, i, &gid, NULL); i++) { @@ -760,7 +784,8 @@ static int cma_resolve_ib_dev(struct rdma_id_private *id_priv) } if (!cma_dev && (gid.global.subnet_prefix == - dgid->global.subnet_prefix)) { + dgid->global.subnet_prefix) && + port_state == IB_PORT_ACTIVE) { cma_dev = cur_dev; sgid = gid; id_priv->id.port_num = p; @@ -807,6 +832,7 @@ struct rdma_cm_id *rdma_create_id(struct vnet *net, id_priv->id.event_handler = event_handler; id_priv->id.ps = ps; id_priv->id.qp_type = qp_type; + id_priv->tos_set = false; id_priv->timeout_set = false; spin_lock_init(&id_priv->lock); mutex_init(&id_priv->qp_mutex); @@ -2560,6 +2586,7 @@ void rdma_set_service_type(struct rdma_cm_id *id, int tos) id_priv = container_of(id, struct rdma_id_private, id); id_priv->tos = (u8) tos; + id_priv->tos_set = true; } EXPORT_SYMBOL(rdma_set_service_type); @@ -2787,6 +2814,9 @@ static int cma_resolve_iboe_route(struct rdma_id_private *id_priv) struct cma_work *work; int ret; if_t ndev = NULL; + u8 default_roce_tos = id_priv->cma_dev->default_roce_tos[id_priv->id.port_num - + rdma_start_port(id_priv->cma_dev->device)]; + u8 tos = id_priv->tos_set ? id_priv->tos : default_roce_tos; work = kzalloc(sizeof *work, GFP_KERNEL); @@ -2846,8 +2876,8 @@ static int cma_resolve_iboe_route(struct rdma_id_private *id_priv) route->path_rec->reversible = 1; route->path_rec->pkey = cpu_to_be16(0xffff); route->path_rec->mtu_selector = IB_SA_EQ; - route->path_rec->sl = iboe_tos_to_sl(ndev, id_priv->tos); - route->path_rec->traffic_class = id_priv->tos; + route->path_rec->sl = iboe_tos_to_sl(ndev, tos); + route->path_rec->traffic_class = tos; route->path_rec->mtu = iboe_get_mtu(if_getmtu(ndev)); route->path_rec->rate_selector = IB_SA_EQ; route->path_rec->rate = iboe_get_rate(ndev); @@ -3108,20 +3138,26 @@ int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr, int ret; id_priv = container_of(id, struct rdma_id_private, id); + memcpy(cma_dst_addr(id_priv), dst_addr, rdma_addr_size(dst_addr)); if (id_priv->state == RDMA_CM_IDLE) { ret = cma_bind_addr(id, src_addr, dst_addr); - if (ret) + if (ret) { + memset(cma_dst_addr(id_priv), 0, rdma_addr_size(dst_addr)); return ret; + } } - if (cma_family(id_priv) != dst_addr->sa_family) + if (cma_family(id_priv) != dst_addr->sa_family) { + memset(cma_dst_addr(id_priv), 0, rdma_addr_size(dst_addr)); return -EINVAL; + } - if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_ADDR_QUERY)) + if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_ADDR_QUERY)) { + memset(cma_dst_addr(id_priv), 0, rdma_addr_size(dst_addr)); return -EINVAL; + } atomic_inc(&id_priv->refcount); - memcpy(cma_dst_addr(id_priv), dst_addr, rdma_addr_size(dst_addr)); if (cma_any_addr(vnet, dst_addr)) { ret = cma_resolve_loopback(id_priv); } else { @@ -3241,6 +3277,45 @@ err: return ret == -ENOSPC ? -EADDRNOTAVAIL : ret; } +static int cma_port_is_unique(struct rdma_bind_list *bind_list, + struct rdma_id_private *id_priv) +{ + struct rdma_id_private *cur_id; + struct vnet *vnet = id_priv->id.route.addr.dev_addr.net; + struct sockaddr *daddr = cma_dst_addr(id_priv); + struct sockaddr *saddr = cma_src_addr(id_priv); + __be16 dport = cma_port(daddr); + + hlist_for_each_entry(cur_id, &bind_list->owners, node) { + struct vnet *cur_vnet = cur_id->id.route.addr.dev_addr.net; + struct sockaddr *cur_daddr = cma_dst_addr(cur_id); + struct sockaddr *cur_saddr = cma_src_addr(cur_id); + __be16 cur_dport = cma_port(cur_daddr); + + if (id_priv == cur_id) + continue; + + /* different dest port -> unique */ + if (!cma_any_port(cur_daddr) && + (dport != cur_dport)) + continue; + + /* different src address -> unique */ + if (!cma_any_addr(vnet, saddr) && + !cma_any_addr(cur_vnet, cur_saddr) && + cma_addr_cmp(saddr, cur_saddr)) + continue; + + /* different dst address -> unique */ + if (!cma_any_addr(cur_vnet, cur_daddr) && + cma_addr_cmp(daddr, cur_daddr)) + continue; + + return -EADDRNOTAVAIL; + } + return 0; +} + static int cma_alloc_any_port(enum rdma_port_space ps, struct rdma_id_private *id_priv) { @@ -3253,9 +3328,19 @@ static int cma_alloc_any_port(enum rdma_port_space ps, remaining = (high - low) + 1; rover = prandom_u32() % remaining + low; retry: - if (last_used_port != rover && - !cma_ps_find(net, ps, (unsigned short)rover)) { - int ret = cma_alloc_port(ps, id_priv, rover); + if (last_used_port != rover) { + struct rdma_bind_list *bind_list; + int ret; + + bind_list = cma_ps_find(net, ps, (unsigned short)rover); + + if (!bind_list) { + ret = cma_alloc_port(ps, id_priv, rover); + } else { + ret = cma_port_is_unique(bind_list, id_priv); + if (!ret) + cma_bind_port(bind_list, id_priv); + } /* * Remember previously used port number in order to avoid * re-using same port immediately after it is closed. @@ -3495,6 +3580,7 @@ int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr) struct rdma_id_private *id_priv; struct vnet *vnet = id->route.addr.dev_addr.net; int ret; + struct sockaddr *daddr; if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6 && addr->sa_family != AF_IB) @@ -3534,6 +3620,9 @@ int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr) if (ret) goto err2; + daddr = cma_dst_addr(id_priv); + daddr->sa_family = addr->sa_family; + return 0; err2: if (id_priv->cma_dev) @@ -4531,6 +4620,25 @@ done: return (error); } +static int +sysctl_cma_default_roce_tos(SYSCTL_HANDLER_ARGS) +{ + struct cma_device *cma_dev = arg1; + const int port = arg2; + int error; + u8 tos; + + tos = cma_get_default_roce_tos(cma_dev, port); + + error = sysctl_handle_8(oidp, &tos, sizeof(tos), req); + if (error != 0 || req->newptr == NULL) + goto done; + + error = -cma_set_default_roce_tos(cma_dev, port, tos); +done: + return (error); +} + static void cma_add_one(struct ib_device *device) { struct cma_device *cma_dev; @@ -4547,10 +4655,15 @@ static void cma_add_one(struct ib_device *device) cma_dev->default_gid_type = kcalloc(device->phys_port_cnt, sizeof(*cma_dev->default_gid_type), GFP_KERNEL); - if (!cma_dev->default_gid_type) { - kfree(cma_dev); - return; - } + if (!cma_dev->default_gid_type) + goto free_cma_dev; + + cma_dev->default_roce_tos = kcalloc(device->phys_port_cnt, + sizeof(*cma_dev->default_roce_tos), + GFP_KERNEL); + if (!cma_dev->default_roce_tos) + goto free_gid_type; + for (i = rdma_start_port(device); i <= rdma_end_port(device); i++) { unsigned long supported_gids; unsigned int default_gid_type; @@ -4569,6 +4682,7 @@ static void cma_add_one(struct ib_device *device) } cma_dev->default_gid_type[i - rdma_start_port(device)] = default_gid_type; + cma_dev->default_roce_tos[i - rdma_start_port(device)] = 0; } init_completion(&cma_dev->comp); @@ -4592,7 +4706,25 @@ static void cma_add_one(struct ib_device *device) OID_AUTO, buf, CTLTYPE_STRING | CTLFLAG_RWTUN | CTLFLAG_MPSAFE, cma_dev, i, &sysctl_cma_default_roce_mode, "A", "Default RoCE mode. Valid values: IB/RoCE v1 and RoCE v2"); + + snprintf(buf, sizeof(buf), "default_roce_tos_port%d", i); + + (void) SYSCTL_ADD_PROC(&cma_dev->sysctl_ctx, + SYSCTL_CHILDREN(device->ports_parent->parent->oidp), + OID_AUTO, buf, CTLTYPE_U8 | CTLFLAG_RWTUN | CTLFLAG_MPSAFE, + cma_dev, i, &sysctl_cma_default_roce_tos, "CU", + "Default RoCE TOS. Valid values: 0-255"); } + + return; + +free_gid_type: + kfree(cma_dev->default_gid_type); + +free_cma_dev: + kfree(cma_dev); + + return; } static int cma_remove_id_dev(struct rdma_id_private *id_priv) @@ -4662,6 +4794,7 @@ static void cma_remove_one(struct ib_device *device, void *client_data) cma_process_remove(cma_dev); sysctl_ctx_free(&cma_dev->sysctl_ctx); + kfree(cma_dev->default_roce_tos); kfree(cma_dev->default_gid_type); kfree(cma_dev); } diff --git a/sys/ofed/drivers/infiniband/core/ib_mad.c b/sys/ofed/drivers/infiniband/core/ib_mad.c index 9eee0c818966..900d5943410f 100644 --- a/sys/ofed/drivers/infiniband/core/ib_mad.c +++ b/sys/ofed/drivers/infiniband/core/ib_mad.c @@ -325,7 +325,9 @@ struct ib_mad_agent *ib_register_mad_agent(struct ib_device *device, /* Validate device and port */ port_priv = ib_get_mad_port(device, port_num); if (!port_priv) { - dev_notice(&device->dev, "ib_register_mad_agent: Invalid port\n"); + dev_notice(&device->dev, + "ib_register_mad_agent: Invalid port %d\n", + port_num); ret = ERR_PTR(-ENODEV); goto error1; } diff --git a/sys/ofed/drivers/infiniband/core/ib_umem.c b/sys/ofed/drivers/infiniband/core/ib_umem.c index 1f99d2560224..8865401b7721 100644 --- a/sys/ofed/drivers/infiniband/core/ib_umem.c +++ b/sys/ofed/drivers/infiniband/core/ib_umem.c @@ -101,9 +101,6 @@ struct ib_umem *ib_umem_get(struct ib_ucontext *context, unsigned long addr, if (dmasync) dma_attrs |= DMA_ATTR_WRITE_BARRIER; - if (!size) - return ERR_PTR(-EINVAL); - /* * If the combination of the addr and size requested for this memory * region causes an integer overflow, return error. @@ -136,6 +133,7 @@ struct ib_umem *ib_umem_get(struct ib_ucontext *context, unsigned long addr, IB_ACCESS_REMOTE_ATOMIC | IB_ACCESS_MW_BIND)); if (access & IB_ACCESS_ON_DEMAND) { + put_pid(umem->pid); ret = ib_umem_odp_get(context, umem); if (ret) { kfree(umem); @@ -148,6 +146,7 @@ struct ib_umem *ib_umem_get(struct ib_ucontext *context, unsigned long addr, page_list = (struct page **) __get_free_page(GFP_KERNEL); if (!page_list) { + put_pid(umem->pid); kfree(umem); return ERR_PTR(-ENOMEM); } diff --git a/sys/ofed/drivers/infiniband/core/ib_umem_odp.c b/sys/ofed/drivers/infiniband/core/ib_umem_odp.c index 9a39e97bc4bc..ed5db3cf6a54 100644 --- a/sys/ofed/drivers/infiniband/core/ib_umem_odp.c +++ b/sys/ofed/drivers/infiniband/core/ib_umem_odp.c @@ -240,6 +240,71 @@ static const struct mmu_notifier_ops ib_umem_notifiers = { .invalidate_range_end = ib_umem_notifier_invalidate_range_end, }; +struct ib_umem *ib_alloc_odp_umem(struct ib_ucontext *context, + unsigned long addr, + size_t size) +{ + struct ib_umem *umem; + struct ib_umem_odp *odp_data; + int pages = size >> PAGE_SHIFT; + int ret; + + umem = kzalloc(sizeof(*umem), GFP_KERNEL); + if (!umem) + return ERR_PTR(-ENOMEM); + + umem->context = context; + umem->length = size; + umem->address = addr; + umem->page_size = PAGE_SIZE; + umem->writable = 1; + + odp_data = kzalloc(sizeof(*odp_data), GFP_KERNEL); + if (!odp_data) { + ret = -ENOMEM; + goto out_umem; + } + odp_data->umem = umem; + + mutex_init(&odp_data->umem_mutex); + init_completion(&odp_data->notifier_completion); + + odp_data->page_list = vzalloc(pages * sizeof(*odp_data->page_list)); + if (!odp_data->page_list) { + ret = -ENOMEM; + goto out_odp_data; + } + + odp_data->dma_list = vzalloc(pages * sizeof(*odp_data->dma_list)); + if (!odp_data->dma_list) { + ret = -ENOMEM; + goto out_page_list; + } + + down_write(&context->umem_rwsem); + context->odp_mrs_count++; + rbt_ib_umem_insert(&odp_data->interval_tree, &context->umem_tree); + if (likely(!atomic_read(&context->notifier_count))) + odp_data->mn_counters_active = true; + else + list_add(&odp_data->no_private_counters, + &context->no_private_counters); + up_write(&context->umem_rwsem); + + umem->odp_data = odp_data; + + return umem; + +out_page_list: + vfree(odp_data->page_list); *** 487 LINES SKIPPED ***
