[ewg] 4K MTU patch to kernel 2.6.26
Hi Shirley, We integrated the 4K MTU patch in OFED 1.3 but there was an agreement that you will send this path for kernel 2.6.26 Can you please do this in the coming days we will have enough time to review it and we will not miss the new merge window. Thanks Tziporet ___ ewg mailing list ewg@lists.openfabrics.org http://lists.openfabrics.org/cgi-bin/mailman/listinfo/ewg
[ewg] OpenSM release notes and QLogic HCAs
Hi Jeremy/Betsy, I saw mention of new QLogic HCAs support at OFED 1.3. Is that accurate ? If so, have they been tested with OpenSM ? The current table for QLogic HCAs in the OpenSM release notes is: QLogic/PathScale Device | Note |--- iPath | QHT6040 (PathScale InfiniPath HT-460) iPath | QHT6140 (PathScale InfiniPath HT-465) iPath | QLE6140 (PathScale InfiniPath PE-880) Is this up to date ? Is QoS relevant to any of the new HCAs and if so, has that been tested ? Thanks. -- Hal ___ ewg mailing list ewg@lists.openfabrics.org http://lists.openfabrics.org/cgi-bin/mailman/listinfo/ewg
[ewg] Re: 4K MTU patch to kernel 2.6.26
Hello Tziporet, Yes, that's I am working on. I am going on vacation next week, that's why I hesitated to submit this patch this week since I can't respond to the review comments on time. If submitting the patch on April.1 is too late, I can submit this patch by tomorrow. How do you think? thanks Shirley___ ewg mailing list ewg@lists.openfabrics.org http://lists.openfabrics.org/cgi-bin/mailman/listinfo/ewg
[ewg] [RFC][0/2] IPoIB UD 4K MTU support
Here is a patchset to enable IPoIB UD 4K MTU support for any IB fabric where the max IPoIB payload can be up to 4K. This patchset uses two S/G buffers when IPoIB payload + IB_GRH header size is greater than PAGE_SIZE. The first buffer size is IB_GRH_HEAD + IPOIB_ENCAP_LEN. The second buffer is the data. Please review it. Thanks Shirley ___ ewg mailing list ewg@lists.openfabrics.org http://lists.openfabrics.org/cgi-bin/mailman/listinfo/ewg
[ewg] [RFC][1/2] IPoIB UD 4K MTU support
This patch defines some parameters and creates a couple of APIs and for UD RX
S/G to be used later.
Signed-off-by: Shirley Ma [EMAIL PROTECTED]
---
drivers/infiniband/ulp/ipoib/ipoib.h | 48 ++
1 files changed, 48 insertions(+), 0 deletions(-)
diff --git a/drivers/infiniband/ulp/ipoib/ipoib.h
b/drivers/infiniband/ulp/ipoib/ipoib.h
index f9b7caa..73a8fe5 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib.h
+++ b/drivers/infiniband/ulp/ipoib/ipoib.h
@@ -61,6 +61,10 @@ enum {
IPOIB_ENCAP_LEN = 4,
+ IPOIB_UD_MAX_PAYLOAD = 4096,
+ IPOIB_UD_HEAD_SIZE= IB_GRH_BYTES + IPOIB_ENCAP_LEN,
+ IPOIB_UD_RX_SG= (IPOIB_UD_MAX_PAYLOAD + IB_GRH_BYTES) /
PAGE_SIZE,
+
IPOIB_CM_MTU = 0x1 - 0x10, /* padding to align header
to 16 */
IPOIB_CM_BUF_SIZE = IPOIB_CM_MTU + IPOIB_ENCAP_LEN,
IPOIB_CM_HEAD_SIZE= IPOIB_CM_BUF_SIZE % PAGE_SIZE,
@@ -141,6 +145,11 @@ struct ipoib_rx_buf {
u64 mapping;
};
+struct ipoib_ud_rx_buf {
+ struct sk_buff *skb;
+ u64 mapping[IPOIB_UD_RX_SG];
+};
+
struct ipoib_tx_buf {
struct sk_buff *skb;
u64 mapping[MAX_SKB_FRAGS + 1];
@@ -289,6 +298,7 @@ struct ipoib_dev_priv {
unsigned int admin_mtu;
unsigned int mcast_mtu;
+ unsigned int max_ib_mtu;
struct ipoib_rx_buf *rx_ring;
@@ -359,6 +369,44 @@ struct ipoib_neigh {
struct list_headlist;
};
+#define IPOIB_UD_MTU(ib_mtu) (ib_mtu - IPOIB_ENCAP_LEN)
+#define IPOIB_UD_BUF_SIZE(ib_mtu) (ib_mtu + IB_GRH_BYTES)
+
+static inline int ipoib_ud_need_sg(unsigned int ib_mtu)
+{
+ return (IPOIB_UD_BUF_SIZE(ib_mtu) PAGE_SIZE) ? 1 : 0;
+}
+
+static inline void ipoib_ud_dma_unmap_rx(struct ipoib_dev_priv *priv,
+u64 mapping[IPOIB_UD_RX_SG])
+{
+ if (ipoib_ud_need_sg(priv-max_ib_mtu)) {
+ ib_dma_unmap_single(priv-ca, mapping[0], IPOIB_UD_HEAD_SIZE,
DMA_FROM_DEVICE);
+ ib_dma_unmap_page(priv-ca, mapping[1], PAGE_SIZE,
DMA_FROM_DEVICE);
+ } else
+ ib_dma_unmap_single(priv-ca, mapping[0],
IPOIB_UD_BUF_SIZE(priv-max_ib_mtu), DMA_FROM_DEVICE);
+}
+
+static inline void ipoib_ud_skb_put_frags(struct ipoib_dev_priv *priv,
+ struct sk_buff *skb,
+ unsigned int length)
+{
+ if (ipoib_ud_need_sg(priv-max_ib_mtu)) {
+ skb_frag_t *frag = skb_shinfo(skb)-frags[0];
+ /*
+* There is only two buffers needed for max_payload = 4K,
+* first buf size is IPOIB_UD_HEAD_SIZE
+*/
+ skb-tail += IPOIB_UD_HEAD_SIZE;
+ frag-size = length - IPOIB_UD_HEAD_SIZE;
+ skb-data_len += frag-size;
+ skb-truesize += frag-size;
+ skb-len += length;
+ } else
+ skb_put(skb, length);
+
+}
+
/*
* We stash a pointer to our private neighbour information after our
* hardware address in neigh-ha. The ALIGN() expression here makes
___
ewg mailing list
ewg@lists.openfabrics.org
http://lists.openfabrics.org/cgi-bin/mailman/listinfo/ewg
[ewg] [RFC][2/2] IPoIB UD 4K MTU support
This patch enabled 4K MTU support for IPoIB UD.
I fixed unnecessary define in [RFC][1/2] patch
since there is only 2 buffers are needed. I will
integrate any comments later for this patchset and
resubmit it. I have touched test this patch for branch-2.6.25
git tree.
Signed-off-by: Shirley Ma [EMAIL PROTECTED]
---
drivers/infiniband/ulp/ipoib/ipoib.h | 13 +---
drivers/infiniband/ulp/ipoib/ipoib_ib.c| 86 +--
drivers/infiniband/ulp/ipoib/ipoib_main.c | 19 --
drivers/infiniband/ulp/ipoib/ipoib_multicast.c |3 +-
drivers/infiniband/ulp/ipoib/ipoib_verbs.c | 15 -
drivers/infiniband/ulp/ipoib/ipoib_vlan.c |1 +
6 files changed, 83 insertions(+), 54 deletions(-)
diff --git a/drivers/infiniband/ulp/ipoib/ipoib.h
b/drivers/infiniband/ulp/ipoib/ipoib.h
index 73a8fe5..fcbb618 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib.h
+++ b/drivers/infiniband/ulp/ipoib/ipoib.h
@@ -56,14 +56,11 @@
/* constants */
enum {
- IPOIB_PACKET_SIZE = 2048,
- IPOIB_BUF_SIZE= IPOIB_PACKET_SIZE + IB_GRH_BYTES,
-
IPOIB_ENCAP_LEN = 4,
IPOIB_UD_MAX_PAYLOAD = 4096,
IPOIB_UD_HEAD_SIZE= IB_GRH_BYTES + IPOIB_ENCAP_LEN,
- IPOIB_UD_RX_SG= (IPOIB_UD_MAX_PAYLOAD + IB_GRH_BYTES) /
PAGE_SIZE,
+ IPOIB_UD_RX_SG= 2, /* max buffer needed */
IPOIB_CM_MTU = 0x1 - 0x10, /* padding to align header
to 16 */
IPOIB_CM_BUF_SIZE = IPOIB_CM_MTU + IPOIB_ENCAP_LEN,
@@ -142,11 +139,6 @@ struct ipoib_mcast {
struct ipoib_rx_buf {
struct sk_buff *skb;
- u64 mapping;
-};
-
-struct ipoib_ud_rx_buf {
- struct sk_buff *skb;
u64 mapping[IPOIB_UD_RX_SG];
};
@@ -310,6 +302,9 @@ struct ipoib_dev_priv {
struct ib_send_wrtx_wr;
unsigned tx_outstanding;
+ struct ib_recv_wrrx_wr;
+ struct ib_sgerx_sge[IPOIB_UD_RX_SG];
+
struct ib_wc ibwc[IPOIB_NUM_WC];
struct list_head dead_ahs;
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_cm.c
b/drivers/infiniband/ulp/ipoib/ipoib_cm.c
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_fs.c
b/drivers/infiniband/ulp/ipoib/ipoib_fs.c
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_ib.c
b/drivers/infiniband/ulp/ipoib/ipoib_ib.c
index 9d3e778..072acc2 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_ib.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_ib.c
@@ -90,25 +90,18 @@ void ipoib_free_ah(struct kref *kref)
static int ipoib_ib_post_receive(struct net_device *dev, int id)
{
struct ipoib_dev_priv *priv = netdev_priv(dev);
- struct ib_sge list;
- struct ib_recv_wr param;
struct ib_recv_wr *bad_wr;
int ret;
- list.addr = priv-rx_ring[id].mapping;
- list.length = IPOIB_BUF_SIZE;
- list.lkey = priv-mr-lkey;
+ priv-rx_wr.wr_id = id | IPOIB_OP_RECV;
+ priv-rx_sge[0].addr = priv-rx_ring[id].mapping[0];
+ priv-rx_sge[1].addr = priv-rx_ring[id].mapping[1];
+
- param.next= NULL;
- param.wr_id = id | IPOIB_OP_RECV;
- param.sg_list = list;
- param.num_sge = 1;
-
- ret = ib_post_recv(priv-qp, param, bad_wr);
+ ret = ib_post_recv(priv-qp, priv-rx_wr, bad_wr);
if (unlikely(ret)) {
ipoib_warn(priv, receive failed for buf %d (%d)\n, id, ret);
- ib_dma_unmap_single(priv-ca, priv-rx_ring[id].mapping,
- IPOIB_BUF_SIZE, DMA_FROM_DEVICE);
+ ipoib_ud_dma_unmap_rx(priv, priv-rx_ring[id].mapping);
dev_kfree_skb_any(priv-rx_ring[id].skb);
priv-rx_ring[id].skb = NULL;
}
@@ -116,15 +109,22 @@ static int ipoib_ib_post_receive(struct net_device *dev,
int id)
return ret;
}
-static int ipoib_alloc_rx_skb(struct net_device *dev, int id)
+static struct sk_buff *ipoib_alloc_rx_skb(struct net_device *dev,
+ int id,
+ u64 mapping[IPOIB_UD_RX_SG])
{
struct ipoib_dev_priv *priv = netdev_priv(dev);
struct sk_buff *skb;
- u64 addr;
+ int buf_size;
+
+ if (ipoib_ud_need_sg(priv-max_ib_mtu))
+ buf_size = IPOIB_UD_HEAD_SIZE;
+ else
+ buf_size = IPOIB_UD_BUF_SIZE(priv-max_ib_mtu);
- skb = dev_alloc_skb(IPOIB_BUF_SIZE + 4);
- if (!skb)
- return -ENOMEM;
+ skb = dev_alloc_skb(buf_size + 4);
+ if (unlikely(!skb))
+ return NULL;
/*
* IB will leave a 40 byte gap for a GRH and IPoIB adds a 4 byte
@@ -133,17 +133,31 @@ static int ipoib_alloc_rx_skb(struct net_device *dev, int
id)
*/
skb_reserve(skb, 4);
- addr = ib_dma_map_single(priv-ca, skb-data, IPOIB_BUF_SIZE,
-
