Tziporet, This IPoIB 4K MTU patch is built against today's 2.6.24 OFED-1.3-Git tree. This patch tested before Eli's patch successfully. This rebuilt patch is on top of Eli's patch. However this constant UD_POST_RCV_COUNT which is defined in Eli's patch as 16 does impact the behavior this patch. When I define this as 1, everything works OK, if I change the value to 8 or bigger, the patch won't work well.
We do see a couple of issues after Eli's patch checks in. So I suggest to check in the patch. Then we can work together to address these issues tomorrow. In Eli's patch I would suggest use kzalloc() to alloc 16 ib_sge and ib_recv_wr instead of defining this in ipoib_dev_priv since it might have some memory issue there. I am working on the patch now to see any better results. Vlad, There would be one line change for backporting regarding priv->stats vs. dev->stats. If you have any problem to create the backport patch, let me know. I will ask Nam to help. The attachment is for you to easily apply the patch, my email might have issues. Thanks Shirley Signed-off-by: Shirley Ma <[EMAIL PROTECTED]> --- diff -urpN ofed_1_3_a/drivers/infiniband/ulp/ipoib/ipoib.h ofed_1_3_b/drivers/infiniband/ulp/ipoib/ipoib.h --- ofed_1_3_a/drivers/infiniband/ulp/ipoib/ipoib.h 2008-02-04 15:45:44.000000000 -0800 +++ ofed_1_3_b/drivers/infiniband/ulp/ipoib/ipoib.h 2008-02-04 15:40:38.000000000 -0800 @@ -56,11 +56,11 @@ /* constants */ enum { - IPOIB_PACKET_SIZE = 2048, - IPOIB_BUF_SIZE = IPOIB_PACKET_SIZE + IB_GRH_BYTES, - IPOIB_ENCAP_LEN = 4, + IPOIB_UD_HEAD_SIZE = IB_GRH_BYTES + IPOIB_ENCAP_LEN, + IPOIB_UD_RX_SG = 2, /* for 4K MTU */ + IPOIB_CM_MTU = 0x10000 - 0x10, /* padding to align header to 16 */ IPOIB_CM_BUF_SIZE = IPOIB_CM_MTU + IPOIB_ENCAP_LEN, IPOIB_CM_HEAD_SIZE = IPOIB_CM_BUF_SIZE % PAGE_SIZE, @@ -141,9 +141,9 @@ struct ipoib_mcast { struct net_device *dev; }; -struct ipoib_rx_buf { +struct ipoib_sg_rx_buf { struct sk_buff *skb; - u64 mapping; + u64 mapping[IPOIB_UD_RX_SG]; }; struct ipoib_tx_buf { @@ -337,7 +337,7 @@ struct ipoib_dev_priv { struct net_device *dev; struct ib_recv_wr rx_wr_draft[UD_POST_RCV_COUNT]; - struct ib_sge sglist_draft[UD_POST_RCV_COUNT]; + struct ib_sge sglist_draft[UD_POST_RCV_COUNT][IPOIB_UD_RX_SG]; unsigned int rx_outst; struct napi_struct napi; @@ -378,7 +378,7 @@ struct ipoib_dev_priv { unsigned int admin_mtu; unsigned int mcast_mtu; - struct ipoib_rx_buf *rx_ring; + struct ipoib_sg_rx_buf *rx_ring; spinlock_t tx_lock; struct ipoib_tx_buf *tx_ring; @@ -412,6 +412,7 @@ struct ipoib_dev_priv { struct ipoib_ethtool_st etool; struct timer_list poll_timer; struct ib_ah *own_ah; + int max_ib_mtu; }; struct ipoib_ah { @@ -452,6 +453,19 @@ struct ipoib_neigh { struct list_head list; }; +#define IPOIB_UD_MTU(ib_mtu) (ib_mtu - IPOIB_ENCAP_LEN) +#define IPOIB_UD_BUF_SIZE(ib_mtu) (ib_mtu + IB_GRH_BYTES) +static inline int ipoib_ud_need_sg(int ib_mtu) +{ + return (IPOIB_UD_BUF_SIZE(ib_mtu) > PAGE_SIZE) ? 1 : 0; +} +static inline void ipoib_sg_dma_unmap_rx(struct ipoib_dev_priv *priv, + u64 mapping[IPOIB_UD_RX_SG]) +{ + ib_dma_unmap_single(priv->ca, mapping[0], IPOIB_UD_HEAD_SIZE, DMA_FROM_DEVICE); + ib_dma_unmap_single(priv->ca, mapping[1], PAGE_SIZE, DMA_FROM_DEVICE); +} + /* * We stash a pointer to our private neighbour information after our * hardware address in neigh->ha. The ALIGN() expression here makes diff -urpN ofed_1_3_a/drivers/infiniband/ulp/ipoib/ipoib_ib.c ofed_1_3_b/drivers/infiniband/ulp/ipoib/ipoib_ib.c --- ofed_1_3_a/drivers/infiniband/ulp/ipoib/ipoib_ib.c 2008-02-04 15:45:44.000000000 -0800 +++ ofed_1_3_b/drivers/infiniband/ulp/ipoib/ipoib_ib.c 2008-02-04 15:40:38.000000000 -0800 @@ -96,14 +96,82 @@ static void clean_pending_receives(struc for (i = 0; i < priv->rx_outst; ++i) { id = priv->rx_wr_draft[i].wr_id & ~IPOIB_OP_RECV; - ib_dma_unmap_single(priv->ca, priv->rx_ring[id].mapping, - IPOIB_BUF_SIZE, DMA_FROM_DEVICE); + if (ipoib_ud_need_sg(priv->max_ib_mtu)) + ipoib_sg_dma_unmap_rx(priv, + priv->rx_ring[i].mapping); + else + ib_dma_unmap_single(priv->ca, priv->rx_ring[id].mapping[0], + IPOIB_UD_BUF_SIZE(priv->max_ib_mtu), DMA_FROM_DEVICE); dev_kfree_skb_any(priv->rx_ring[id].skb); priv->rx_ring[id].skb = NULL; } priv->rx_outst = 0; } +static void ipoib_ud_skb_put_frags(struct sk_buff *skb, unsigned int length, + struct sk_buff *toskb) +{ + unsigned int size; + skb_frag_t *frag = &skb_shinfo(skb)->frags[0]; + + /* put header into skb */ + size = min(length, (unsigned)IPOIB_UD_HEAD_SIZE); + skb->tail += size; + skb->len += size; + length -= size; + + if (length == 0) { + /* don't need this page */ + skb_fill_page_desc(toskb, 0, frag->page, 0, PAGE_SIZE); + --skb_shinfo(skb)->nr_frags; + } else { + size = min(length, (unsigned) PAGE_SIZE); + frag->size = size; + skb->data_len += size; + skb->truesize += size; + skb->len += size; + length -= size; + } +} + +static struct sk_buff *ipoib_sg_alloc_rx_skb(struct net_device *dev, + int id, u64 mapping[IPOIB_UD_RX_SG]) +{ + struct ipoib_dev_priv *priv = netdev_priv(dev); + struct page *page; + struct sk_buff *skb; + + skb = dev_alloc_skb(IPOIB_UD_HEAD_SIZE); + + if (unlikely(!skb)) + return NULL; + + mapping[0] = ib_dma_map_single(priv->ca, skb->data, IPOIB_UD_HEAD_SIZE, + DMA_FROM_DEVICE); + if (unlikely(ib_dma_mapping_error(priv->ca, mapping[0]))) { + dev_kfree_skb_any(skb); + return NULL; + } + + page = alloc_page(GFP_ATOMIC); + if (!page) + goto partial_error; + + skb_fill_page_desc(skb, 0, page, 0, PAGE_SIZE); + mapping[1] = ib_dma_map_page(priv->ca, skb_shinfo(skb)->frags[0].page, + 0, PAGE_SIZE, DMA_FROM_DEVICE); + if (unlikely(ib_dma_mapping_error(priv->ca, mapping[1]))) + goto partial_error; + + priv->rx_ring[id].skb = skb; + return skb; + +partial_error: + ib_dma_unmap_single(priv->ca, mapping[0], IPOIB_UD_HEAD_SIZE, DMA_FROM_DEVICE); + dev_kfree_skb_any(skb); + return NULL; +} + static int ipoib_ib_post_receive(struct net_device *dev, int id) { struct ipoib_dev_priv *priv = netdev_priv(dev); @@ -111,8 +179,11 @@ static int ipoib_ib_post_receive(struct int ret = 0; int i = priv->rx_outst; - priv->sglist_draft[i].addr = priv->rx_ring[id].mapping; + priv->sglist_draft[i][0].addr = priv->rx_ring[id].mapping[0]; + priv->sglist_draft[i][1].addr = priv->rx_ring[id].mapping[1]; + priv->rx_wr_draft[i].wr_id = id | IPOIB_OP_RECV; + if (++priv->rx_outst == UD_POST_RCV_COUNT) { ret = ib_post_recv(priv->qp, priv->rx_wr_draft, &bad_wr); @@ -120,8 +191,13 @@ static int ipoib_ib_post_receive(struct ipoib_warn(priv, "receive failed for buf %d (%d)\n", id, ret); while (bad_wr) { id = bad_wr->wr_id & ~IPOIB_OP_RECV; - ib_dma_unmap_single(priv->ca, priv->rx_ring[id].mapping, - IPOIB_BUF_SIZE, DMA_FROM_DEVICE); + if (ipoib_ud_need_sg(priv->max_ib_mtu)) + ipoib_sg_dma_unmap_rx(priv, + priv->rx_ring[i].mapping); + else + ib_dma_unmap_single(priv->ca, priv->rx_ring[id].mapping[0], + IPOIB_UD_BUF_SIZE(priv->max_ib_mtu), + DMA_FROM_DEVICE); dev_kfree_skb_any(priv->rx_ring[id].skb); priv->rx_ring[id].skb = NULL; } @@ -138,7 +214,7 @@ static int ipoib_alloc_rx_skb(struct net struct sk_buff *skb; u64 addr; - skb = dev_alloc_skb(IPOIB_BUF_SIZE + 4); + skb = dev_alloc_skb(IPOIB_UD_BUF_SIZE(priv->max_ib_mtu) + 4); if (!skb) return -ENOMEM; @@ -149,7 +225,8 @@ static int ipoib_alloc_rx_skb(struct net */ skb_reserve(skb, 4); - addr = ib_dma_map_single(priv->ca, skb->data, IPOIB_BUF_SIZE, + addr = ib_dma_map_single(priv->ca, skb->data, + IPOIB_UD_BUF_SIZE(priv->max_ib_mtu), DMA_FROM_DEVICE); if (unlikely(ib_dma_mapping_error(priv->ca, addr))) { dev_kfree_skb_any(skb); @@ -157,7 +234,7 @@ static int ipoib_alloc_rx_skb(struct net } priv->rx_ring[id].skb = skb; - priv->rx_ring[id].mapping = addr; + priv->rx_ring[id].mapping[0] = addr; return 0; } @@ -165,10 +242,15 @@ static int ipoib_alloc_rx_skb(struct net static int ipoib_ib_post_receives(struct net_device *dev) { struct ipoib_dev_priv *priv = netdev_priv(dev); - int i; + int i, ret; for (i = 0; i < ipoib_recvq_size; ++i) { - if (ipoib_alloc_rx_skb(dev, i)) { + if (ipoib_ud_need_sg(priv->max_ib_mtu)) + ret = !(ipoib_sg_alloc_rx_skb(dev, i, + priv->rx_ring[i].mapping)); + else + ret = ipoib_alloc_rx_skb(dev, i); + if (ret) { ipoib_warn(priv, "failed to allocate receive buffer %d\n", i); return -ENOMEM; } @@ -186,7 +268,7 @@ static void ipoib_ib_handle_rx_wc(struct struct ipoib_dev_priv *priv = netdev_priv(dev); unsigned int wr_id = wc->wr_id & ~IPOIB_OP_RECV; struct sk_buff *skb; - u64 addr; + u64 mapping[IPOIB_UD_RX_SG]; ipoib_dbg_data(priv, "recv completion: id %d, status: %d\n", wr_id, wc->status); @@ -198,42 +280,74 @@ static void ipoib_ib_handle_rx_wc(struct } skb = priv->rx_ring[wr_id].skb; - addr = priv->rx_ring[wr_id].mapping; - if (unlikely(wc->status != IB_WC_SUCCESS)) { - if (wc->status != IB_WC_WR_FLUSH_ERR) - ipoib_warn(priv, "failed recv event " - "(status=%d, wrid=%d vend_err %x)\n", - wc->status, wr_id, wc->vendor_err); - ib_dma_unmap_single(priv->ca, addr, - IPOIB_BUF_SIZE, DMA_FROM_DEVICE); - dev_kfree_skb_any(skb); - priv->rx_ring[wr_id].skb = NULL; - return; - } + /* duplicate the code here, to omit fast path if need-sg condition check */ + if (ipoib_ud_need_sg(priv->max_ib_mtu)) { + struct sk_buff *newskb; + if (unlikely(wc->status != IB_WC_SUCCESS)) { + if (wc->status != IB_WC_WR_FLUSH_ERR) + ipoib_warn(priv, "failed recv event " + "(status=%d, wrid=%d vend_err %x)\n", + wc->status, wr_id, wc->vendor_err); + ipoib_sg_dma_unmap_rx(priv, priv->rx_ring[wr_id].mapping); + dev_kfree_skb_any(skb); + priv->rx_ring[wr_id].skb = NULL; + return; + } + /* + * Drop packets that this interface sent, ie multicast packets + * that the HCA has replicated. + */ + if (wc->slid == priv->local_lid && wc->src_qp == priv->qp->qp_num) + goto repost; + newskb = ipoib_sg_alloc_rx_skb(dev, wr_id, mapping); + if (unlikely(!newskb)) { + ++dev->stats.rx_dropped; + goto repost; + } + ipoib_dbg_data(priv, "received %d bytes, SLID 0x%04x\n", + wc->byte_len, wc->slid); + ipoib_sg_dma_unmap_rx(priv, priv->rx_ring[wr_id].mapping); + memcpy(priv->rx_ring[wr_id].mapping, mapping, + IPOIB_UD_RX_SG * sizeof *mapping); + ipoib_ud_skb_put_frags(skb, wc->byte_len, newskb); + } else { + u64 addr = priv->rx_ring[wr_id].mapping[0]; + if (unlikely(wc->status != IB_WC_SUCCESS)) { + if (wc->status != IB_WC_WR_FLUSH_ERR) + ipoib_warn(priv, "failed recv event " + "(status=%d, wrid=%d vend_err %x)\n", + wc->status, wr_id, wc->vendor_err); + ib_dma_unmap_single(priv->ca, addr, + IPOIB_UD_BUF_SIZE(priv->max_ib_mtu), DMA_FROM_DEVICE); + dev_kfree_skb_any(skb); + priv->rx_ring[wr_id].skb = NULL; + return; + } - /* - * Drop packets that this interface sent, ie multicast packets - * that the HCA has replicated. - */ - if (unlikely(wc->slid == priv->local_lid && wc->src_qp == priv->qp->qp_num)) - goto repost; + /* + * Drop packets that this interface sent, ie multicast packets + * that the HCA has replicated. + */ + if (unlikely(wc->slid == priv->local_lid && wc->src_qp == priv->qp->qp_num)) + goto repost; - /* - * If we can't allocate a new RX buffer, dump - * this packet and reuse the old buffer. - */ - if (unlikely(ipoib_alloc_rx_skb(dev, wr_id))) { - ++dev->stats.rx_dropped; - goto repost; - } + /* + * If we can't allocate a new RX buffer, dump + * this packet and reuse the old buffer. + */ + if (unlikely(ipoib_alloc_rx_skb(dev, wr_id))) { + ++dev->stats.rx_dropped; + goto repost; + } - ipoib_dbg_data(priv, "received %d bytes, SLID 0x%04x\n", - wc->byte_len, wc->slid); + ipoib_dbg_data(priv, "received %d bytes, SLID 0x%04x\n", + wc->byte_len, wc->slid); - ib_dma_unmap_single(priv->ca, addr, IPOIB_BUF_SIZE, DMA_FROM_DEVICE); + ib_dma_unmap_single(priv->ca, addr, IPOIB_UD_BUF_SIZE(priv->max_ib_mtu), DMA_FROM_DEVICE); - skb_put(skb, wc->byte_len); + skb_put(skb, wc->byte_len); + } skb_pull(skb, IB_GRH_BYTES); skb->protocol = ((struct ipoib_header *) skb->data)->proto; @@ -827,18 +941,21 @@ int ipoib_ib_dev_stop(struct net_device * all our pending work requests. */ for (i = 0; i < ipoib_recvq_size; ++i) { - struct ipoib_rx_buf *rx_req; + struct ipoib_sg_rx_buf *rx_req; rx_req = &priv->rx_ring[i]; - - if (rx_req->skb) { + if (!rx_req->skb) + continue; + if (ipoib_ud_need_sg(priv->max_ib_mtu)) + ipoib_sg_dma_unmap_rx(priv, + priv->rx_ring[i].mapping); + else ib_dma_unmap_single(priv->ca, - rx_req->mapping, - IPOIB_BUF_SIZE, + rx_req->mapping[0], + IPOIB_UD_BUF_SIZE(priv->max_ib_mtu), DMA_FROM_DEVICE); - dev_kfree_skb_any(rx_req->skb); - rx_req->skb = NULL; - } + dev_kfree_skb_any(rx_req->skb); + rx_req->skb = NULL; } goto timeout; diff -urpN ofed_1_3_a/drivers/infiniband/ulp/ipoib/ipoib_main.c ofed_1_3_b/drivers/infiniband/ulp/ipoib/ipoib_main.c --- ofed_1_3_a/drivers/infiniband/ulp/ipoib/ipoib_main.c 2008-02-04 15:45:44.000000000 -0800 +++ ofed_1_3_b/drivers/infiniband/ulp/ipoib/ipoib_main.c 2008-02-04 15:40:38.000000000 -0800 @@ -193,7 +193,7 @@ static int ipoib_change_mtu(struct net_d return 0; } - if (new_mtu > IPOIB_PACKET_SIZE - IPOIB_ENCAP_LEN) + if (new_mtu > IPOIB_UD_MTU(priv->max_ib_mtu)) return -EINVAL; priv->admin_mtu = new_mtu; @@ -1007,10 +1007,6 @@ static void ipoib_setup(struct net_devic dev->tx_queue_len = ipoib_sendq_size * 2; dev->features = NETIF_F_VLAN_CHALLENGED | NETIF_F_LLTX; - /* MTU will be reset when mcast join happens */ - dev->mtu = IPOIB_PACKET_SIZE - IPOIB_ENCAP_LEN; - priv->mcast_mtu = priv->admin_mtu = dev->mtu; - memcpy(dev->broadcast, ipv4_bcast_addr, INFINIBAND_ALEN); netif_carrier_off(dev); @@ -1156,6 +1152,7 @@ static struct net_device *ipoib_add_port struct ib_device *hca, u8 port) { struct ipoib_dev_priv *priv; + struct ib_port_attr attr; int result = -ENOMEM; priv = ipoib_intf_alloc(format); @@ -1166,6 +1163,18 @@ static struct net_device *ipoib_add_port priv->dev->features |= NETIF_F_HIGHDMA; + if (!ib_query_port(hca, port, &attr)) + priv->max_ib_mtu = ib_mtu_enum_to_int(attr.max_mtu); + else { + printk(KERN_WARNING "%s: ib_query_port %d failed\n", + hca->name, port); + goto device_init_failed; + } + + /* MTU will be reset when mcast join happens */ + priv->dev->mtu = IPOIB_UD_MTU(priv->max_ib_mtu); + priv->mcast_mtu = priv->admin_mtu = priv->dev->mtu; + result = ib_query_pkey(hca, port, 0, &priv->pkey); if (result) { printk(KERN_WARNING "%s: ib_query_pkey port %d failed (ret = %d)\n", diff -urpN ofed_1_3_a/drivers/infiniband/ulp/ipoib/ipoib_multicast.c ofed_1_3_b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c --- ofed_1_3_a/drivers/infiniband/ulp/ipoib/ipoib_multicast.c 2008-02-04 15:45:44.000000000 -0800 +++ ofed_1_3_b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c 2008-02-04 15:40:38.000000000 -0800 @@ -567,8 +567,7 @@ void ipoib_mcast_join_task(struct work_s return; } - priv->mcast_mtu = ib_mtu_enum_to_int(priv->broadcast->mcmember.mtu) - - IPOIB_ENCAP_LEN; + priv->mcast_mtu = IPOIB_UD_MTU(ib_mtu_enum_to_int(priv->broadcast->mcmember.mtu)); if (!ipoib_cm_admin_enabled(dev)) dev->mtu = min(priv->mcast_mtu, priv->admin_mtu); diff -urpN ofed_1_3_a/drivers/infiniband/ulp/ipoib/ipoib_verbs.c ofed_1_3_b/drivers/infiniband/ulp/ipoib/ipoib_verbs.c --- ofed_1_3_a/drivers/infiniband/ulp/ipoib/ipoib_verbs.c 2008-02-04 15:45:44.000000000 -0800 +++ ofed_1_3_b/drivers/infiniband/ulp/ipoib/ipoib_verbs.c 2008-02-04 15:40:38.000000000 -0800 @@ -151,7 +151,7 @@ int ipoib_transport_dev_init(struct net_ .max_send_wr = ipoib_sendq_size, .max_recv_wr = ipoib_recvq_size, .max_send_sge = dev->features & NETIF_F_SG ? MAX_SKB_FRAGS + 1 : 1, - .max_recv_sge = 1 + .max_recv_sge = IPOIB_UD_RX_SG }, .sq_sig_type = IB_SIGNAL_REQ_WR, .qp_type = IB_QPT_UD, @@ -225,18 +225,29 @@ int ipoib_transport_dev_init(struct net_ priv->tx_wr.opcode = IB_WR_SEND; priv->tx_wr.sg_list = priv->tx_sge; priv->tx_wr.send_flags = IB_SEND_SIGNALED; - + for (i = 0; i < UD_POST_RCV_COUNT; ++i) { - priv->sglist_draft[i].length = IPOIB_BUF_SIZE; - priv->sglist_draft[i].lkey = priv->mr->lkey; - - priv->rx_wr_draft[i].sg_list = &priv->sglist_draft[i]; - priv->rx_wr_draft[i].num_sge = 1; - if (i < UD_POST_RCV_COUNT - 1) - priv->rx_wr_draft[i].next = &priv->rx_wr_draft[i + 1]; + priv->sglist_draft[i][0].lkey = priv->mr->lkey; + priv->sglist_draft[i][1].lkey = priv->mr->lkey; + priv->rx_wr_draft[i].sg_list = &priv->sglist_draft[i][0]; } + if (i < UD_POST_RCV_COUNT - 1) + priv->rx_wr_draft[i].next = &priv->rx_wr_draft[i + 1]; priv->rx_wr_draft[i].next = NULL; + if (ipoib_ud_need_sg(priv->max_ib_mtu)) { + for (i = 0; i < UD_POST_RCV_COUNT; ++i) { + priv->sglist_draft[i][0].length = IPOIB_UD_HEAD_SIZE; + priv->sglist_draft[i][1].length = PAGE_SIZE; + priv->rx_wr_draft[i].num_sge = IPOIB_UD_RX_SG; + } + } else { + for (i = 0; i < UD_POST_RCV_COUNT; ++i) { + priv->sglist_draft[i][0].length = IPOIB_UD_BUF_SIZE(priv->max_ib_mtu); + priv->rx_wr_draft[i].num_sge = 1; + } + } + return 0; out_free_scq:
diff -urpN ofed_1_3_a/drivers/infiniband/ulp/ipoib/ipoib.h ofed_1_3_b/drivers/infiniband/ulp/ipoib/ipoib.h --- ofed_1_3_a/drivers/infiniband/ulp/ipoib/ipoib.h 2008-02-04 15:45:44.000000000 -0800 +++ ofed_1_3_b/drivers/infiniband/ulp/ipoib/ipoib.h 2008-02-04 15:40:38.000000000 -0800 @@ -56,11 +56,11 @@ /* constants */ enum { - IPOIB_PACKET_SIZE = 2048, - IPOIB_BUF_SIZE = IPOIB_PACKET_SIZE + IB_GRH_BYTES, - IPOIB_ENCAP_LEN = 4, + IPOIB_UD_HEAD_SIZE = IB_GRH_BYTES + IPOIB_ENCAP_LEN, + IPOIB_UD_RX_SG = 2, /* for 4K MTU */ + IPOIB_CM_MTU = 0x10000 - 0x10, /* padding to align header to 16 */ IPOIB_CM_BUF_SIZE = IPOIB_CM_MTU + IPOIB_ENCAP_LEN, IPOIB_CM_HEAD_SIZE = IPOIB_CM_BUF_SIZE % PAGE_SIZE, @@ -141,9 +141,9 @@ struct ipoib_mcast { struct net_device *dev; }; -struct ipoib_rx_buf { +struct ipoib_sg_rx_buf { struct sk_buff *skb; - u64 mapping; + u64 mapping[IPOIB_UD_RX_SG]; }; struct ipoib_tx_buf { @@ -337,7 +337,7 @@ struct ipoib_dev_priv { struct net_device *dev; struct ib_recv_wr rx_wr_draft[UD_POST_RCV_COUNT]; - struct ib_sge sglist_draft[UD_POST_RCV_COUNT]; + struct ib_sge sglist_draft[UD_POST_RCV_COUNT][IPOIB_UD_RX_SG]; unsigned int rx_outst; struct napi_struct napi; @@ -378,7 +378,7 @@ struct ipoib_dev_priv { unsigned int admin_mtu; unsigned int mcast_mtu; - struct ipoib_rx_buf *rx_ring; + struct ipoib_sg_rx_buf *rx_ring; spinlock_t tx_lock; struct ipoib_tx_buf *tx_ring; @@ -412,6 +412,7 @@ struct ipoib_dev_priv { struct ipoib_ethtool_st etool; struct timer_list poll_timer; struct ib_ah *own_ah; + int max_ib_mtu; }; struct ipoib_ah { @@ -452,6 +453,19 @@ struct ipoib_neigh { struct list_head list; }; +#define IPOIB_UD_MTU(ib_mtu) (ib_mtu - IPOIB_ENCAP_LEN) +#define IPOIB_UD_BUF_SIZE(ib_mtu) (ib_mtu + IB_GRH_BYTES) +static inline int ipoib_ud_need_sg(int ib_mtu) +{ + return (IPOIB_UD_BUF_SIZE(ib_mtu) > PAGE_SIZE) ? 1 : 0; +} +static inline void ipoib_sg_dma_unmap_rx(struct ipoib_dev_priv *priv, + u64 mapping[IPOIB_UD_RX_SG]) +{ + ib_dma_unmap_single(priv->ca, mapping[0], IPOIB_UD_HEAD_SIZE, DMA_FROM_DEVICE); + ib_dma_unmap_single(priv->ca, mapping[1], PAGE_SIZE, DMA_FROM_DEVICE); +} + /* * We stash a pointer to our private neighbour information after our * hardware address in neigh->ha. The ALIGN() expression here makes diff -urpN ofed_1_3_a/drivers/infiniband/ulp/ipoib/ipoib_ib.c ofed_1_3_b/drivers/infiniband/ulp/ipoib/ipoib_ib.c --- ofed_1_3_a/drivers/infiniband/ulp/ipoib/ipoib_ib.c 2008-02-04 15:45:44.000000000 -0800 +++ ofed_1_3_b/drivers/infiniband/ulp/ipoib/ipoib_ib.c 2008-02-04 15:40:38.000000000 -0800 @@ -96,14 +96,82 @@ static void clean_pending_receives(struc for (i = 0; i < priv->rx_outst; ++i) { id = priv->rx_wr_draft[i].wr_id & ~IPOIB_OP_RECV; - ib_dma_unmap_single(priv->ca, priv->rx_ring[id].mapping, - IPOIB_BUF_SIZE, DMA_FROM_DEVICE); + if (ipoib_ud_need_sg(priv->max_ib_mtu)) + ipoib_sg_dma_unmap_rx(priv, + priv->rx_ring[i].mapping); + else + ib_dma_unmap_single(priv->ca, priv->rx_ring[id].mapping[0], + IPOIB_UD_BUF_SIZE(priv->max_ib_mtu), DMA_FROM_DEVICE); dev_kfree_skb_any(priv->rx_ring[id].skb); priv->rx_ring[id].skb = NULL; } priv->rx_outst = 0; } +static void ipoib_ud_skb_put_frags(struct sk_buff *skb, unsigned int length, + struct sk_buff *toskb) +{ + unsigned int size; + skb_frag_t *frag = &skb_shinfo(skb)->frags[0]; + + /* put header into skb */ + size = min(length, (unsigned)IPOIB_UD_HEAD_SIZE); + skb->tail += size; + skb->len += size; + length -= size; + + if (length == 0) { + /* don't need this page */ + skb_fill_page_desc(toskb, 0, frag->page, 0, PAGE_SIZE); + --skb_shinfo(skb)->nr_frags; + } else { + size = min(length, (unsigned) PAGE_SIZE); + frag->size = size; + skb->data_len += size; + skb->truesize += size; + skb->len += size; + length -= size; + } +} + +static struct sk_buff *ipoib_sg_alloc_rx_skb(struct net_device *dev, + int id, u64 mapping[IPOIB_UD_RX_SG]) +{ + struct ipoib_dev_priv *priv = netdev_priv(dev); + struct page *page; + struct sk_buff *skb; + + skb = dev_alloc_skb(IPOIB_UD_HEAD_SIZE); + + if (unlikely(!skb)) + return NULL; + + mapping[0] = ib_dma_map_single(priv->ca, skb->data, IPOIB_UD_HEAD_SIZE, + DMA_FROM_DEVICE); + if (unlikely(ib_dma_mapping_error(priv->ca, mapping[0]))) { + dev_kfree_skb_any(skb); + return NULL; + } + + page = alloc_page(GFP_ATOMIC); + if (!page) + goto partial_error; + + skb_fill_page_desc(skb, 0, page, 0, PAGE_SIZE); + mapping[1] = ib_dma_map_page(priv->ca, skb_shinfo(skb)->frags[0].page, + 0, PAGE_SIZE, DMA_FROM_DEVICE); + if (unlikely(ib_dma_mapping_error(priv->ca, mapping[1]))) + goto partial_error; + + priv->rx_ring[id].skb = skb; + return skb; + +partial_error: + ib_dma_unmap_single(priv->ca, mapping[0], IPOIB_UD_HEAD_SIZE, DMA_FROM_DEVICE); + dev_kfree_skb_any(skb); + return NULL; +} + static int ipoib_ib_post_receive(struct net_device *dev, int id) { struct ipoib_dev_priv *priv = netdev_priv(dev); @@ -111,8 +179,11 @@ static int ipoib_ib_post_receive(struct int ret = 0; int i = priv->rx_outst; - priv->sglist_draft[i].addr = priv->rx_ring[id].mapping; + priv->sglist_draft[i][0].addr = priv->rx_ring[id].mapping[0]; + priv->sglist_draft[i][1].addr = priv->rx_ring[id].mapping[1]; + priv->rx_wr_draft[i].wr_id = id | IPOIB_OP_RECV; + if (++priv->rx_outst == UD_POST_RCV_COUNT) { ret = ib_post_recv(priv->qp, priv->rx_wr_draft, &bad_wr); @@ -120,8 +191,13 @@ static int ipoib_ib_post_receive(struct ipoib_warn(priv, "receive failed for buf %d (%d)\n", id, ret); while (bad_wr) { id = bad_wr->wr_id & ~IPOIB_OP_RECV; - ib_dma_unmap_single(priv->ca, priv->rx_ring[id].mapping, - IPOIB_BUF_SIZE, DMA_FROM_DEVICE); + if (ipoib_ud_need_sg(priv->max_ib_mtu)) + ipoib_sg_dma_unmap_rx(priv, + priv->rx_ring[i].mapping); + else + ib_dma_unmap_single(priv->ca, priv->rx_ring[id].mapping[0], + IPOIB_UD_BUF_SIZE(priv->max_ib_mtu), + DMA_FROM_DEVICE); dev_kfree_skb_any(priv->rx_ring[id].skb); priv->rx_ring[id].skb = NULL; } @@ -138,7 +214,7 @@ static int ipoib_alloc_rx_skb(struct net struct sk_buff *skb; u64 addr; - skb = dev_alloc_skb(IPOIB_BUF_SIZE + 4); + skb = dev_alloc_skb(IPOIB_UD_BUF_SIZE(priv->max_ib_mtu) + 4); if (!skb) return -ENOMEM; @@ -149,7 +225,8 @@ static int ipoib_alloc_rx_skb(struct net */ skb_reserve(skb, 4); - addr = ib_dma_map_single(priv->ca, skb->data, IPOIB_BUF_SIZE, + addr = ib_dma_map_single(priv->ca, skb->data, + IPOIB_UD_BUF_SIZE(priv->max_ib_mtu), DMA_FROM_DEVICE); if (unlikely(ib_dma_mapping_error(priv->ca, addr))) { dev_kfree_skb_any(skb); @@ -157,7 +234,7 @@ static int ipoib_alloc_rx_skb(struct net } priv->rx_ring[id].skb = skb; - priv->rx_ring[id].mapping = addr; + priv->rx_ring[id].mapping[0] = addr; return 0; } @@ -165,10 +242,15 @@ static int ipoib_alloc_rx_skb(struct net static int ipoib_ib_post_receives(struct net_device *dev) { struct ipoib_dev_priv *priv = netdev_priv(dev); - int i; + int i, ret; for (i = 0; i < ipoib_recvq_size; ++i) { - if (ipoib_alloc_rx_skb(dev, i)) { + if (ipoib_ud_need_sg(priv->max_ib_mtu)) + ret = !(ipoib_sg_alloc_rx_skb(dev, i, + priv->rx_ring[i].mapping)); + else + ret = ipoib_alloc_rx_skb(dev, i); + if (ret) { ipoib_warn(priv, "failed to allocate receive buffer %d\n", i); return -ENOMEM; } @@ -186,7 +268,7 @@ static void ipoib_ib_handle_rx_wc(struct struct ipoib_dev_priv *priv = netdev_priv(dev); unsigned int wr_id = wc->wr_id & ~IPOIB_OP_RECV; struct sk_buff *skb; - u64 addr; + u64 mapping[IPOIB_UD_RX_SG]; ipoib_dbg_data(priv, "recv completion: id %d, status: %d\n", wr_id, wc->status); @@ -198,42 +280,74 @@ static void ipoib_ib_handle_rx_wc(struct } skb = priv->rx_ring[wr_id].skb; - addr = priv->rx_ring[wr_id].mapping; - if (unlikely(wc->status != IB_WC_SUCCESS)) { - if (wc->status != IB_WC_WR_FLUSH_ERR) - ipoib_warn(priv, "failed recv event " - "(status=%d, wrid=%d vend_err %x)\n", - wc->status, wr_id, wc->vendor_err); - ib_dma_unmap_single(priv->ca, addr, - IPOIB_BUF_SIZE, DMA_FROM_DEVICE); - dev_kfree_skb_any(skb); - priv->rx_ring[wr_id].skb = NULL; - return; - } + /* duplicate the code here, to omit fast path if need-sg condition check */ + if (ipoib_ud_need_sg(priv->max_ib_mtu)) { + struct sk_buff *newskb; + if (unlikely(wc->status != IB_WC_SUCCESS)) { + if (wc->status != IB_WC_WR_FLUSH_ERR) + ipoib_warn(priv, "failed recv event " + "(status=%d, wrid=%d vend_err %x)\n", + wc->status, wr_id, wc->vendor_err); + ipoib_sg_dma_unmap_rx(priv, priv->rx_ring[wr_id].mapping); + dev_kfree_skb_any(skb); + priv->rx_ring[wr_id].skb = NULL; + return; + } + /* + * Drop packets that this interface sent, ie multicast packets + * that the HCA has replicated. + */ + if (wc->slid == priv->local_lid && wc->src_qp == priv->qp->qp_num) + goto repost; + newskb = ipoib_sg_alloc_rx_skb(dev, wr_id, mapping); + if (unlikely(!newskb)) { + ++dev->stats.rx_dropped; + goto repost; + } + ipoib_dbg_data(priv, "received %d bytes, SLID 0x%04x\n", + wc->byte_len, wc->slid); + ipoib_sg_dma_unmap_rx(priv, priv->rx_ring[wr_id].mapping); + memcpy(priv->rx_ring[wr_id].mapping, mapping, + IPOIB_UD_RX_SG * sizeof *mapping); + ipoib_ud_skb_put_frags(skb, wc->byte_len, newskb); + } else { + u64 addr = priv->rx_ring[wr_id].mapping[0]; + if (unlikely(wc->status != IB_WC_SUCCESS)) { + if (wc->status != IB_WC_WR_FLUSH_ERR) + ipoib_warn(priv, "failed recv event " + "(status=%d, wrid=%d vend_err %x)\n", + wc->status, wr_id, wc->vendor_err); + ib_dma_unmap_single(priv->ca, addr, + IPOIB_UD_BUF_SIZE(priv->max_ib_mtu), DMA_FROM_DEVICE); + dev_kfree_skb_any(skb); + priv->rx_ring[wr_id].skb = NULL; + return; + } - /* - * Drop packets that this interface sent, ie multicast packets - * that the HCA has replicated. - */ - if (unlikely(wc->slid == priv->local_lid && wc->src_qp == priv->qp->qp_num)) - goto repost; + /* + * Drop packets that this interface sent, ie multicast packets + * that the HCA has replicated. + */ + if (unlikely(wc->slid == priv->local_lid && wc->src_qp == priv->qp->qp_num)) + goto repost; - /* - * If we can't allocate a new RX buffer, dump - * this packet and reuse the old buffer. - */ - if (unlikely(ipoib_alloc_rx_skb(dev, wr_id))) { - ++dev->stats.rx_dropped; - goto repost; - } + /* + * If we can't allocate a new RX buffer, dump + * this packet and reuse the old buffer. + */ + if (unlikely(ipoib_alloc_rx_skb(dev, wr_id))) { + ++dev->stats.rx_dropped; + goto repost; + } - ipoib_dbg_data(priv, "received %d bytes, SLID 0x%04x\n", - wc->byte_len, wc->slid); + ipoib_dbg_data(priv, "received %d bytes, SLID 0x%04x\n", + wc->byte_len, wc->slid); - ib_dma_unmap_single(priv->ca, addr, IPOIB_BUF_SIZE, DMA_FROM_DEVICE); + ib_dma_unmap_single(priv->ca, addr, IPOIB_UD_BUF_SIZE(priv->max_ib_mtu), DMA_FROM_DEVICE); - skb_put(skb, wc->byte_len); + skb_put(skb, wc->byte_len); + } skb_pull(skb, IB_GRH_BYTES); skb->protocol = ((struct ipoib_header *) skb->data)->proto; @@ -827,18 +941,21 @@ int ipoib_ib_dev_stop(struct net_device * all our pending work requests. */ for (i = 0; i < ipoib_recvq_size; ++i) { - struct ipoib_rx_buf *rx_req; + struct ipoib_sg_rx_buf *rx_req; rx_req = &priv->rx_ring[i]; - - if (rx_req->skb) { + if (!rx_req->skb) + continue; + if (ipoib_ud_need_sg(priv->max_ib_mtu)) + ipoib_sg_dma_unmap_rx(priv, + priv->rx_ring[i].mapping); + else ib_dma_unmap_single(priv->ca, - rx_req->mapping, - IPOIB_BUF_SIZE, + rx_req->mapping[0], + IPOIB_UD_BUF_SIZE(priv->max_ib_mtu), DMA_FROM_DEVICE); - dev_kfree_skb_any(rx_req->skb); - rx_req->skb = NULL; - } + dev_kfree_skb_any(rx_req->skb); + rx_req->skb = NULL; } goto timeout; diff -urpN ofed_1_3_a/drivers/infiniband/ulp/ipoib/ipoib_main.c ofed_1_3_b/drivers/infiniband/ulp/ipoib/ipoib_main.c --- ofed_1_3_a/drivers/infiniband/ulp/ipoib/ipoib_main.c 2008-02-04 15:45:44.000000000 -0800 +++ ofed_1_3_b/drivers/infiniband/ulp/ipoib/ipoib_main.c 2008-02-04 15:40:38.000000000 -0800 @@ -193,7 +193,7 @@ static int ipoib_change_mtu(struct net_d return 0; } - if (new_mtu > IPOIB_PACKET_SIZE - IPOIB_ENCAP_LEN) + if (new_mtu > IPOIB_UD_MTU(priv->max_ib_mtu)) return -EINVAL; priv->admin_mtu = new_mtu; @@ -1007,10 +1007,6 @@ static void ipoib_setup(struct net_devic dev->tx_queue_len = ipoib_sendq_size * 2; dev->features = NETIF_F_VLAN_CHALLENGED | NETIF_F_LLTX; - /* MTU will be reset when mcast join happens */ - dev->mtu = IPOIB_PACKET_SIZE - IPOIB_ENCAP_LEN; - priv->mcast_mtu = priv->admin_mtu = dev->mtu; - memcpy(dev->broadcast, ipv4_bcast_addr, INFINIBAND_ALEN); netif_carrier_off(dev); @@ -1156,6 +1152,7 @@ static struct net_device *ipoib_add_port struct ib_device *hca, u8 port) { struct ipoib_dev_priv *priv; + struct ib_port_attr attr; int result = -ENOMEM; priv = ipoib_intf_alloc(format); @@ -1166,6 +1163,18 @@ static struct net_device *ipoib_add_port priv->dev->features |= NETIF_F_HIGHDMA; + if (!ib_query_port(hca, port, &attr)) + priv->max_ib_mtu = ib_mtu_enum_to_int(attr.max_mtu); + else { + printk(KERN_WARNING "%s: ib_query_port %d failed\n", + hca->name, port); + goto device_init_failed; + } + + /* MTU will be reset when mcast join happens */ + priv->dev->mtu = IPOIB_UD_MTU(priv->max_ib_mtu); + priv->mcast_mtu = priv->admin_mtu = priv->dev->mtu; + result = ib_query_pkey(hca, port, 0, &priv->pkey); if (result) { printk(KERN_WARNING "%s: ib_query_pkey port %d failed (ret = %d)\n", diff -urpN ofed_1_3_a/drivers/infiniband/ulp/ipoib/ipoib_multicast.c ofed_1_3_b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c --- ofed_1_3_a/drivers/infiniband/ulp/ipoib/ipoib_multicast.c 2008-02-04 15:45:44.000000000 -0800 +++ ofed_1_3_b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c 2008-02-04 15:40:38.000000000 -0800 @@ -567,8 +567,7 @@ void ipoib_mcast_join_task(struct work_s return; } - priv->mcast_mtu = ib_mtu_enum_to_int(priv->broadcast->mcmember.mtu) - - IPOIB_ENCAP_LEN; + priv->mcast_mtu = IPOIB_UD_MTU(ib_mtu_enum_to_int(priv->broadcast->mcmember.mtu)); if (!ipoib_cm_admin_enabled(dev)) dev->mtu = min(priv->mcast_mtu, priv->admin_mtu); diff -urpN ofed_1_3_a/drivers/infiniband/ulp/ipoib/ipoib_verbs.c ofed_1_3_b/drivers/infiniband/ulp/ipoib/ipoib_verbs.c --- ofed_1_3_a/drivers/infiniband/ulp/ipoib/ipoib_verbs.c 2008-02-04 15:45:44.000000000 -0800 +++ ofed_1_3_b/drivers/infiniband/ulp/ipoib/ipoib_verbs.c 2008-02-04 15:40:38.000000000 -0800 @@ -151,7 +151,7 @@ int ipoib_transport_dev_init(struct net_ .max_send_wr = ipoib_sendq_size, .max_recv_wr = ipoib_recvq_size, .max_send_sge = dev->features & NETIF_F_SG ? MAX_SKB_FRAGS + 1 : 1, - .max_recv_sge = 1 + .max_recv_sge = IPOIB_UD_RX_SG }, .sq_sig_type = IB_SIGNAL_REQ_WR, .qp_type = IB_QPT_UD, @@ -225,18 +225,29 @@ int ipoib_transport_dev_init(struct net_ priv->tx_wr.opcode = IB_WR_SEND; priv->tx_wr.sg_list = priv->tx_sge; priv->tx_wr.send_flags = IB_SEND_SIGNALED; - + for (i = 0; i < UD_POST_RCV_COUNT; ++i) { - priv->sglist_draft[i].length = IPOIB_BUF_SIZE; - priv->sglist_draft[i].lkey = priv->mr->lkey; - - priv->rx_wr_draft[i].sg_list = &priv->sglist_draft[i]; - priv->rx_wr_draft[i].num_sge = 1; - if (i < UD_POST_RCV_COUNT - 1) - priv->rx_wr_draft[i].next = &priv->rx_wr_draft[i + 1]; + priv->sglist_draft[i][0].lkey = priv->mr->lkey; + priv->sglist_draft[i][1].lkey = priv->mr->lkey; + priv->rx_wr_draft[i].sg_list = &priv->sglist_draft[i][0]; } + if (i < UD_POST_RCV_COUNT - 1) + priv->rx_wr_draft[i].next = &priv->rx_wr_draft[i + 1]; priv->rx_wr_draft[i].next = NULL; + if (ipoib_ud_need_sg(priv->max_ib_mtu)) { + for (i = 0; i < UD_POST_RCV_COUNT; ++i) { + priv->sglist_draft[i][0].length = IPOIB_UD_HEAD_SIZE; + priv->sglist_draft[i][1].length = PAGE_SIZE; + priv->rx_wr_draft[i].num_sge = IPOIB_UD_RX_SG; + } + } else { + for (i = 0; i < UD_POST_RCV_COUNT; ++i) { + priv->sglist_draft[i][0].length = IPOIB_UD_BUF_SIZE(priv->max_ib_mtu); + priv->rx_wr_draft[i].num_sge = 1; + } + } + return 0; out_free_scq:
_______________________________________________ ewg mailing list ewg@lists.openfabrics.org http://lists.openfabrics.org/cgi-bin/mailman/listinfo/ewg