RE: [RFC 03/11] Add support for RoCE HW init

[Amrani, Ram]

> > > + dev->max_sge = min_t(u32, RDMA_MAX_SGE_PER_SQ_WQE,
> > > +  RDMA_MAX_SGE_PER_RQ_WQE);
> >
> > Our kernel target mode consumers sort of rely on max_sge_rd, you need
> > to make sure to set it too.
> Good catch. Thanks!

Actually, as I come to code this, I just realized that max_sge_rd is already 
set in the  query verb itself:

+int qedr_query_device(struct ib_device *ibdev,
+ struct ib_device_attr *attr, struct ib_udata *udata)
+{



+   attr->max_sge = qattr->max_sge;
+   attr->max_sge_rd = qattr->max_sge;

RE: [RFC 03/11] Add support for RoCE HW init

[Amrani, Ram]

> > +   dev->max_sge = min_t(u32, RDMA_MAX_SGE_PER_SQ_WQE,
> > +RDMA_MAX_SGE_PER_RQ_WQE);
> 
> Our kernel target mode consumers sort of rely on max_sge_rd, you need to
> make sure to set it too.
Good catch. Thanks!

Re: [RFC 03/11] Add support for RoCE HW init

[Sagi Grimberg]

+   dev->max_sge = min_t(u32, RDMA_MAX_SGE_PER_SQ_WQE,
+RDMA_MAX_SGE_PER_RQ_WQE);


Our kernel target mode consumers sort of rely on max_sge_rd, you need
to make sure to set it too.

RE: [RFC 03/11] Add support for RoCE HW init

[Ram Amrani]

Thanks Mark.
This will be fixed in next version.

Ram


-Original Message-
From: Mark Bloch [mailto:ma...@mellanox.com] 
Sent: Monday, September 12, 2016 9:58 PM
To: Ram Amrani ; dledf...@redhat.com; David Miller 

Cc: Yuval Mintz ; Ariel Elior ; 
Michal Kalderon ; Rajesh Borundia 
; linux-r...@vger.kernel.org; netdev 

Subject: Re: [RFC 03/11] Add support for RoCE HW init



On 12/09/2016 19:07, Ram Amrani wrote:
> Allocate and setup RoCE resources, interrupts and completion queues.
> Adds device attributes.
> 
> Signed-off-by: Rajesh Borundia 
> Signed-off-by: Ram Amrani 
> ---
>  drivers/infiniband/hw/qedr/main.c  | 408 +++-
>  drivers/infiniband/hw/qedr/qedr.h  | 118 
>  drivers/infiniband/hw/qedr/qedr_hsi.h  |  56 ++
>  drivers/infiniband/hw/qedr/qedr_hsi_rdma.h |  96 +++
>  drivers/net/ethernet/qlogic/qed/Makefile   |   1 +
>  drivers/net/ethernet/qlogic/qed/qed.h  |  26 +-
>  drivers/net/ethernet/qlogic/qed/qed_cxt.c  |   6 +
>  drivers/net/ethernet/qlogic/qed/qed_cxt.h  |   6 +
>  drivers/net/ethernet/qlogic/qed/qed_dev.c  | 155 +
>  drivers/net/ethernet/qlogic/qed/qed_main.c |  44 +-
>  drivers/net/ethernet/qlogic/qed/qed_reg_addr.h |   7 +
>  drivers/net/ethernet/qlogic/qed/qed_roce.c | 887 
> +
>  drivers/net/ethernet/qlogic/qed/qed_roce.h | 117 
>  drivers/net/ethernet/qlogic/qed/qed_sp.h   |   1 +
>  drivers/net/ethernet/qlogic/qed/qed_spq.c  |   8 +
>  drivers/net/ethernet/qlogic/qede/qede_roce.c   |   2 +-
>  include/linux/qed/qed_if.h |   5 +-
>  include/linux/qed/qed_roce_if.h| 345 ++
>  18 files changed, 2281 insertions(+), 7 deletions(-)  create mode 
> 100644 drivers/infiniband/hw/qedr/qedr_hsi.h
>  create mode 100644 drivers/infiniband/hw/qedr/qedr_hsi_rdma.h
>  create mode 100644 drivers/net/ethernet/qlogic/qed/qed_roce.c
>  create mode 100644 drivers/net/ethernet/qlogic/qed/qed_roce.h
>  create mode 100644 include/linux/qed/qed_roce_if.h
> 
> diff --git a/drivers/infiniband/hw/qedr/main.c 
> b/drivers/infiniband/hw/qedr/main.c
> index 3fe58a3..0b5274a 100644
> --- a/drivers/infiniband/hw/qedr/main.c
> +++ b/drivers/infiniband/hw/qedr/main.c
> @@ -36,6 +36,8 @@
>  #include 
>  #include 
>  #include 
> +#include 
> +#include 
>  #include "qedr.h"
>  
>  MODULE_DESCRIPTION("QLogic 40G/100G ROCE Driver"); @@ -80,6 +82,139 
> @@ static int qedr_register_device(struct qedr_dev *dev)
>   return 0;
>  }
>  
> +/* This function allocates fast-path status block memory */ static 
> +int qedr_alloc_mem_sb(struct qedr_dev *dev,
> +  struct qed_sb_info *sb_info, u16 sb_id) {
> + struct status_block *sb_virt;
> + dma_addr_t sb_phys;
> + int rc;
> +
> + sb_virt = dma_alloc_coherent(&dev->pdev->dev,
> +  sizeof(*sb_virt), &sb_phys, GFP_KERNEL);
> + if (!sb_virt) {
> + pr_err("Status block allocation failed\n");
> + return -ENOMEM;
> + }
> +
> + rc = dev->ops->common->sb_init(dev->cdev, sb_info,
> +sb_virt, sb_phys, sb_id,
> +QED_SB_TYPE_CNQ);
> + if (rc) {
> + pr_err("Status block initialization failed\n");
> + dma_free_coherent(&dev->pdev->dev, sizeof(*sb_virt),
> +   sb_virt, sb_phys);
> + return rc;
> + }
> +
> + return 0;
> +}
> +
> +static void qedr_free_mem_sb(struct qedr_dev *dev,
> +  struct qed_sb_info *sb_info, int sb_id) {
> + if (sb_info->sb_virt) {
> + dev->ops->common->sb_release(dev->cdev, sb_info, sb_id);
> + dma_free_coherent(&dev->pdev->dev, sizeof(*sb_info->sb_virt),
> +   (void *)sb_info->sb_virt, sb_info->sb_phys);
> + }
> +}
> +
> +static void qedr_free_resources(struct qedr_dev *dev) {
> + int i;
> +
> + for (i = 0; i < dev->num_cnq; i++) {
> + qedr_free_mem_sb(dev, &dev->sb_array[i], dev->sb_start + i);
> + dev->ops->common->chain_free(dev->cdev, &dev->cnq_array[i].pbl);
> + }
> +
> + kfree(dev->cnq_array);
> + kfree(dev->sb_array);
> + kfree(dev->sgid_tbl);
> +}
> +
> +static int qedr_alloc_resources(struct qedr_dev *dev) {
> + struct qedr_cnq *cnq;
> + __le16 *cons_pi;
> + u16 n_entries;
> + int i, rc;
> +
> + dev->sgid_tbl

Re: [RFC 03/11] Add support for RoCE HW init

[Mark Bloch]

On 12/09/2016 19:07, Ram Amrani wrote:
> Allocate and setup RoCE resources, interrupts and completion queues.
> Adds device attributes.
> 
> Signed-off-by: Rajesh Borundia 
> Signed-off-by: Ram Amrani 
> ---
>  drivers/infiniband/hw/qedr/main.c  | 408 +++-
>  drivers/infiniband/hw/qedr/qedr.h  | 118 
>  drivers/infiniband/hw/qedr/qedr_hsi.h  |  56 ++
>  drivers/infiniband/hw/qedr/qedr_hsi_rdma.h |  96 +++
>  drivers/net/ethernet/qlogic/qed/Makefile   |   1 +
>  drivers/net/ethernet/qlogic/qed/qed.h  |  26 +-
>  drivers/net/ethernet/qlogic/qed/qed_cxt.c  |   6 +
>  drivers/net/ethernet/qlogic/qed/qed_cxt.h  |   6 +
>  drivers/net/ethernet/qlogic/qed/qed_dev.c  | 155 +
>  drivers/net/ethernet/qlogic/qed/qed_main.c |  44 +-
>  drivers/net/ethernet/qlogic/qed/qed_reg_addr.h |   7 +
>  drivers/net/ethernet/qlogic/qed/qed_roce.c | 887 
> +
>  drivers/net/ethernet/qlogic/qed/qed_roce.h | 117 
>  drivers/net/ethernet/qlogic/qed/qed_sp.h   |   1 +
>  drivers/net/ethernet/qlogic/qed/qed_spq.c  |   8 +
>  drivers/net/ethernet/qlogic/qede/qede_roce.c   |   2 +-
>  include/linux/qed/qed_if.h |   5 +-
>  include/linux/qed/qed_roce_if.h| 345 ++
>  18 files changed, 2281 insertions(+), 7 deletions(-)
>  create mode 100644 drivers/infiniband/hw/qedr/qedr_hsi.h
>  create mode 100644 drivers/infiniband/hw/qedr/qedr_hsi_rdma.h
>  create mode 100644 drivers/net/ethernet/qlogic/qed/qed_roce.c
>  create mode 100644 drivers/net/ethernet/qlogic/qed/qed_roce.h
>  create mode 100644 include/linux/qed/qed_roce_if.h
> 
> diff --git a/drivers/infiniband/hw/qedr/main.c 
> b/drivers/infiniband/hw/qedr/main.c
> index 3fe58a3..0b5274a 100644
> --- a/drivers/infiniband/hw/qedr/main.c
> +++ b/drivers/infiniband/hw/qedr/main.c
> @@ -36,6 +36,8 @@
>  #include 
>  #include 
>  #include 
> +#include 
> +#include 
>  #include "qedr.h"
>  
>  MODULE_DESCRIPTION("QLogic 40G/100G ROCE Driver");
> @@ -80,6 +82,139 @@ static int qedr_register_device(struct qedr_dev *dev)
>   return 0;
>  }
>  
> +/* This function allocates fast-path status block memory */
> +static int qedr_alloc_mem_sb(struct qedr_dev *dev,
> +  struct qed_sb_info *sb_info, u16 sb_id)
> +{
> + struct status_block *sb_virt;
> + dma_addr_t sb_phys;
> + int rc;
> +
> + sb_virt = dma_alloc_coherent(&dev->pdev->dev,
> +  sizeof(*sb_virt), &sb_phys, GFP_KERNEL);
> + if (!sb_virt) {
> + pr_err("Status block allocation failed\n");
> + return -ENOMEM;
> + }
> +
> + rc = dev->ops->common->sb_init(dev->cdev, sb_info,
> +sb_virt, sb_phys, sb_id,
> +QED_SB_TYPE_CNQ);
> + if (rc) {
> + pr_err("Status block initialization failed\n");
> + dma_free_coherent(&dev->pdev->dev, sizeof(*sb_virt),
> +   sb_virt, sb_phys);
> + return rc;
> + }
> +
> + return 0;
> +}
> +
> +static void qedr_free_mem_sb(struct qedr_dev *dev,
> +  struct qed_sb_info *sb_info, int sb_id)
> +{
> + if (sb_info->sb_virt) {
> + dev->ops->common->sb_release(dev->cdev, sb_info, sb_id);
> + dma_free_coherent(&dev->pdev->dev, sizeof(*sb_info->sb_virt),
> +   (void *)sb_info->sb_virt, sb_info->sb_phys);
> + }
> +}
> +
> +static void qedr_free_resources(struct qedr_dev *dev)
> +{
> + int i;
> +
> + for (i = 0; i < dev->num_cnq; i++) {
> + qedr_free_mem_sb(dev, &dev->sb_array[i], dev->sb_start + i);
> + dev->ops->common->chain_free(dev->cdev, &dev->cnq_array[i].pbl);
> + }
> +
> + kfree(dev->cnq_array);
> + kfree(dev->sb_array);
> + kfree(dev->sgid_tbl);
> +}
> +
> +static int qedr_alloc_resources(struct qedr_dev *dev)
> +{
> + struct qedr_cnq *cnq;
> + __le16 *cons_pi;
> + u16 n_entries;
> + int i, rc;
> +
> + dev->sgid_tbl = kzalloc(sizeof(union ib_gid) *
> + QEDR_MAX_SGID, GFP_KERNEL);
> + if (!dev->sgid_tbl)
> + return -ENOMEM;
> +
> + spin_lock_init(&dev->sgid_lock);
> +
> + /* Allocate Status blocks for CNQ */
> + dev->sb_array = kcalloc(dev->num_cnq, sizeof(*dev->sb_array),
> + GFP_KERNEL);
> + if (!dev->sb_array) {
> + rc = -ENOMEM;
> + goto err1;
> + }
> +
> + dev->cnq_array = kcalloc(dev->num_cnq,
> +  sizeof(*dev->cnq_array), GFP_KERNEL);
> + if (!dev->cnq_array) {
> + rc = -ENOMEM;
> + goto err2;
> + }
> +
> + dev->sb_start = dev->ops->rdma_get_start_sb(dev->cdev);
> +
> + /* Allocate CNQ PBLs */
> + n_entries = min_t(u32, QED_RDMA_MAX_CNQ_SIZE, QEDR_ROCE_MAX_CNQ_SIZE