Hi Som,
> -----Original Message-----
> From: [email protected] [mailto:linux-rdma-
> [email protected]] On Behalf Of Somnath Kotur
> Sent: Friday, February 20, 2015 3:33 AM
> To: [email protected]
> Cc: [email protected]; Devesh Sharma; Somnath Kotur
> Subject: [PATCH 15/30] RDMA/ocrdma: changes to support RoCE-v2 in UD path
>
> From: Devesh Sharma <[email protected]>
>
> To support UD protocol this patch adds following changes to existing UD
> implementation.
>
> 1. AH creation resolves gid-type for a given index.
> 2. Based on GID-type protocol header is built.
> 3. Work completion reports l3-type if f/w supports RoCE-v2
> and sets IB_WC_WITH_NETWORK_HDR_TYPE flag in wc->wc_flags.
>
> Signed-off-by: Somnath Kotur <[email protected]>
> Signed-off-by: Devesh Sharma <[email protected]>
> ---
> drivers/infiniband/hw/ocrdma/ocrdma.h | 1 +
> drivers/infiniband/hw/ocrdma/ocrdma_ah.c | 68
> ++++++++++++++++++++++-----
> drivers/infiniband/hw/ocrdma/ocrdma_sli.h | 5 ++-
> drivers/infiniband/hw/ocrdma/ocrdma_verbs.c | 23 +++++++--
> 4 files changed, 80 insertions(+), 17 deletions(-)
>
> diff --git a/drivers/infiniband/hw/ocrdma/ocrdma.h
> b/drivers/infiniband/hw/ocrdma/ocrdma.h
> index 97f971a..302fd0e 100644
> --- a/drivers/infiniband/hw/ocrdma/ocrdma.h
> +++ b/drivers/infiniband/hw/ocrdma/ocrdma.h
> @@ -341,6 +341,7 @@ struct ocrdma_ah {
> struct ocrdma_av *av;
> u16 sgid_index;
> u32 id;
> + u8 hdr_type;
> };
>
> struct ocrdma_qp_hwq_info {
> diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_ah.c
> b/drivers/infiniband/hw/ocrdma/ocrdma_ah.c
> index 7ecd230..70a885b 100644
> --- a/drivers/infiniband/hw/ocrdma/ocrdma_ah.c
> +++ b/drivers/infiniband/hw/ocrdma/ocrdma_ah.c
> @@ -39,6 +39,20 @@
>
> #define OCRDMA_VID_PCP_SHIFT 0xD
>
> +static u16 ocrdma_hdr_type_to_proto_num(u8 hdr_type) {
> + switch (hdr_type) {
> + case OCRDMA_L3_TYPE_IB_GRH:
> + return (u16)0x8915;
> + case OCRDMA_L3_TYPE_IPV4:
> + return (u16)0x0800;
> + case OCRDMA_L3_TYPE_IPV6:
> + return (u16)0x86dd;
> + default:
> + return 0;
> + }
> +}
> +
> static inline int set_av_attr(struct ocrdma_dev *dev, struct ocrdma_ah *ah,
> struct ib_ah_attr *attr, union ib_gid *sgid,
> int pdid, bool *isvlan, u16 vlan_tag) @@ -47,22 +61,32
> @@ static inline int set_av_attr(struct ocrdma_dev *dev, struct ocrdma_ah
> *ah,
> struct ocrdma_eth_vlan eth;
> struct ocrdma_grh grh;
> int eth_sz;
> + u16 proto_num = 0;
> + struct iphdr ipv4;
> + union {
> + struct sockaddr _sockaddr;
> + struct sockaddr_in _sockaddr_in;
> + struct sockaddr_in6 _sockaddr_in6;
> + } sgid_addr, dgid_addr;
>
> memset(ð, 0, sizeof(eth));
> memset(&grh, 0, sizeof(grh));
> + /* Protocol Number */
> + proto_num = ocrdma_hdr_type_to_proto_num(ah->hdr_type);
> +
>
> /* VLAN */
> if (!vlan_tag || (vlan_tag > 0xFFF))
> vlan_tag = dev->pvid;
> if (vlan_tag && (vlan_tag < 0x1000)) {
> eth.eth_type = cpu_to_be16(0x8100);
> - eth.roce_eth_type =
> cpu_to_be16(OCRDMA_ROCE_ETH_TYPE);
> + eth.roce_eth_type = cpu_to_be16(proto_num);
> vlan_tag |= (dev->sl & 0x07) << OCRDMA_VID_PCP_SHIFT;
> eth.vlan_tag = cpu_to_be16(vlan_tag);
> eth_sz = sizeof(struct ocrdma_eth_vlan);
> *isvlan = true;
> } else {
> - eth.eth_type = cpu_to_be16(OCRDMA_ROCE_ETH_TYPE);
> + eth.eth_type = cpu_to_be16(proto_num);
> eth_sz = sizeof(struct ocrdma_eth_basic);
> }
> /* MAC */
> @@ -71,18 +95,34 @@ static inline int set_av_attr(struct ocrdma_dev *dev,
> struct ocrdma_ah *ah,
> if (status)
> return status;
> ah->sgid_index = attr->grh.sgid_index;
> - memcpy(&grh.sgid[0], sgid->raw, sizeof(union ib_gid));
> - memcpy(&grh.dgid[0], attr->grh.dgid.raw, sizeof(attr->grh.dgid.raw));
> -
> - grh.tclass_flow = cpu_to_be32((6 << 28) |
> - (attr->grh.traffic_class << 24) |
> - attr->grh.flow_label);
> - /* 0x1b is next header value in GRH */
> - grh.pdid_hoplimit = cpu_to_be32((pdid << 16) |
> - (0x1b << 8) | attr->grh.hop_limit);
> /* Eth HDR */
> memcpy(&ah->av->eth_hdr, ð, eth_sz);
> - memcpy((u8 *)ah->av + eth_sz, &grh, sizeof(struct ocrdma_grh));
> + if (ah->hdr_type == RDMA_NETWORK_IPV4) {
> + *((__be16 *)&ipv4) = htons((4 << 12) | (5 << 8) |
> + attr->grh.traffic_class);
> + ipv4.id = cpu_to_be16(pdid);
> + ipv4.frag_off = htons(IP_DF);
> + ipv4.tot_len = htons(0);
> + ipv4.ttl = attr->grh.hop_limit;
> + ipv4.protocol = 0x11;
> + rdma_gid2ip(&sgid_addr._sockaddr, sgid);
> + ipv4.saddr = sgid_addr._sockaddr_in.sin_addr.s_addr;
> + rdma_gid2ip(&dgid_addr._sockaddr, &attr->grh.dgid);
> + ipv4.daddr = dgid_addr._sockaddr_in.sin_addr.s_addr;
> + memcpy((u8 *)ah->av + eth_sz, &ipv4, sizeof(struct iphdr));
> + } else {
> + memcpy(&grh.sgid[0], sgid->raw, sizeof(union ib_gid));
> + grh.tclass_flow = cpu_to_be32((6 << 28) |
> + (attr->grh.traffic_class << 24) |
> + attr->grh.flow_label);
> + memcpy(&grh.dgid[0], attr->grh.dgid.raw,
> + sizeof(attr->grh.dgid.raw));
> + /* 0x1b is next header value in GRH */
> + grh.pdid_hoplimit = cpu_to_be32((pdid << 16) |
> + (0x11 << 8) |
We should decide next-header = 0x1b/0x11 based on the proto_num
> + attr->grh.hop_limit);
> + memcpy((u8 *)ah->av + eth_sz, &grh, sizeof(struct
> ocrdma_grh));
> + }
> if (*isvlan)
> ah->av->valid |= OCRDMA_AV_VLAN_VALID;
> ah->av->valid = cpu_to_le32(ah->av->valid); @@ -106,6 +146,7 @@
> struct ib_ah *ocrdma_create_ah(struct ib_pd *ibpd, struct ib_ah_attr *attr)
>
> if (atomic_cmpxchg(&dev->update_sl, 1, 0))
> ocrdma_init_service_level(dev);
> +
> ah = kzalloc(sizeof(*ah), GFP_ATOMIC);
> if (!ah)
> return ERR_PTR(-ENOMEM);
> @@ -126,6 +167,9 @@ struct ib_ah *ocrdma_create_ah(struct ib_pd *ibpd,
> struct ib_ah_attr *attr)
> vlan_tag = vlan_dev_vlan_id(sgid_attr.ndev);
> rcu_read_unlock();
>
> + /* Get network header type for this GID */
> + ah->hdr_type = ib_gid_to_network_type(sgid_attr.gid_type, &sgid);
> +
> if (pd->uctx) {
> status = rdma_addr_find_dmac_by_grh(&sgid, &attr->grh.dgid,
> attr->dmac, &vlan_tag,
> diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_sli.h
> b/drivers/infiniband/hw/ocrdma/ocrdma_sli.h
> index 6b74eb9..4fb68ee 100644
> --- a/drivers/infiniband/hw/ocrdma/ocrdma_sli.h
> +++ b/drivers/infiniband/hw/ocrdma/ocrdma_sli.h
> @@ -1681,8 +1681,11 @@ enum {
>
> /* w1 */
> OCRDMA_CQE_UD_XFER_LEN_SHIFT = 16,
> + OCRDMA_CQE_UD_XFER_LEN_MASK = 0x1FFF,
> OCRDMA_CQE_PKEY_SHIFT = 0,
> OCRDMA_CQE_PKEY_MASK = 0xFFFF,
> + OCRDMA_CQE_UD_L3TYPE_SHIFT = 29,
> + OCRDMA_CQE_UD_L3TYPE_MASK = 0x07,
>
> /* w2 */
> OCRDMA_CQE_QPN_SHIFT = 0,
> @@ -1807,7 +1810,7 @@ struct ocrdma_ewqe_ud_hdr {
> u32 rsvd_dest_qpn;
> u32 qkey;
> u32 rsvd_ahid;
> - u32 rsvd;
> + u32 hdr_type;
> };
>
> /* extended wqe followed by hdr_wqe for Fast Memory register */ diff --git
> a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
> b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
> index 17c80d5..90a6351 100644
> --- a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
> +++ b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
> @@ -31,7 +31,6 @@
> #include <rdma/iw_cm.h>
> #include <rdma/ib_umem.h>
> #include <rdma/ib_addr.h>
> -#include <rdma/ib_cache.h>
>
> #include "ocrdma.h"
> #include "ocrdma_hw.h"
> @@ -1963,6 +1962,7 @@ static void ocrdma_build_ud_hdr(struct ocrdma_qp
> *qp,
> else
> ud_hdr->qkey = wr->wr.ud.remote_qkey;
> ud_hdr->rsvd_ahid = ah->id;
> + ud_hdr->hdr_type = ah->hdr_type;
> if (ah->av->valid & OCRDMA_AV_VLAN_VALID)
> hdr->cw |= (OCRDMA_FLAG_AH_VLAN_PR <<
> OCRDMA_WQE_FLAGS_SHIFT); } @@ -2698,9 +2698,11 @@ static bool
> ocrdma_poll_scqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe,
> return expand;
> }
>
> -static int ocrdma_update_ud_rcqe(struct ib_wc *ibwc, struct ocrdma_cqe
> *cqe)
> +static int ocrdma_update_ud_rcqe(struct ocrdma_dev *dev, struct ib_wc
> *ibwc,
> + struct ocrdma_cqe *cqe)
> {
> int status;
> + u16 hdr_type = 0;
>
> status = (le32_to_cpu(cqe->flags_status_srcqpn) &
> OCRDMA_CQE_UD_STATUS_MASK) >>
> OCRDMA_CQE_UD_STATUS_SHIFT; @@ -2710,7 +2712,17 @@ static int
> ocrdma_update_ud_rcqe(struct ib_wc *ibwc, struct ocrdma_cqe *cqe)
> OCRDMA_CQE_PKEY_MASK;
> ibwc->wc_flags = IB_WC_GRH;
> ibwc->byte_len = (le32_to_cpu(cqe->ud.rxlen_pkey) >>
> - OCRDMA_CQE_UD_XFER_LEN_SHIFT);
> + OCRDMA_CQE_UD_XFER_LEN_SHIFT) &
> + OCRDMA_CQE_UD_XFER_LEN_MASK;
> +
> + if (ocrdma_is_rocev2_supported(dev)) {
> + hdr_type = (le32_to_cpu(cqe->ud.rxlen_pkey) >>
> + OCRDMA_CQE_UD_L3TYPE_SHIFT) &
> + OCRDMA_CQE_UD_L3TYPE_MASK;
> + ibwc->wc_flags |= IB_WC_WITH_NETWORK_HDR_TYPE;
> + ibwc->network_hdr_type = hdr_type;
> + }
> +
> return status;
> }
>
> @@ -2773,12 +2785,15 @@ static bool ocrdma_poll_err_rcqe(struct
> ocrdma_qp *qp, struct ocrdma_cqe *cqe, static void
> ocrdma_poll_success_rcqe(struct ocrdma_qp *qp,
> struct ocrdma_cqe *cqe, struct ib_wc
> *ibwc) {
> + struct ocrdma_dev *dev;
> +
> + dev = get_ocrdma_dev(qp->ibqp.device);
> ibwc->opcode = IB_WC_RECV;
> ibwc->qp = &qp->ibqp;
> ibwc->status = IB_WC_SUCCESS;
>
> if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI)
> - ocrdma_update_ud_rcqe(ibwc, cqe);
> + ocrdma_update_ud_rcqe(dev, ibwc, cqe);
> else
> ibwc->byte_len = le32_to_cpu(cqe->rq.rxlen);
>
> --
> 1.7.1
>
> --
> To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the
> body
> of a message to [email protected] More majordomo info at
> http://vger.kernel.org/majordomo-info.html
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
