Re: [PATCH bpf-next 09/11] i40e: implement AF_XDP zero-copy support for Rx
Den mån 4 juni 2018 kl 22:35 skrev Alexander Duyck :
>
> On Mon, Jun 4, 2018 at 5:05 AM, Björn Töpel wrote:
> > From: Björn Töpel
> >
> > This commit adds initial AF_XDP zero-copy support for i40e-based
> > NICs. First we add support for the new XDP_QUERY_XSK_UMEM and
> > XDP_SETUP_XSK_UMEM commands in ndo_bpf. This allows the AF_XDP socket
> > to pass a UMEM to the driver. The driver will then DMA map all the
> > frames in the UMEM for the driver. Next, the Rx code will allocate
> > frames from the UMEM fill queue, instead of the regular page
> > allocator.
> >
> > Externally, for the rest of the XDP code, the driver internal UMEM
> > allocator will appear as a MEM_TYPE_ZERO_COPY.
> >
> > The commit also introduces a completely new clean_rx_irq/allocator
> > functions for zero-copy, and means (functions pointers) to set
> > allocators and clean_rx functions.
> >
> > This first version does not support:
> > * passing frames to the stack via XDP_PASS (clone/copy to skb).
> > * doing XDP redirect to other than AF_XDP sockets
> > (convert_to_xdp_frame does not clone the frame yet).
> >
> > Signed-off-by: Björn Töpel
> > ---
> > drivers/net/ethernet/intel/i40e/Makefile| 3 +-
> > drivers/net/ethernet/intel/i40e/i40e.h | 23 ++
> > drivers/net/ethernet/intel/i40e/i40e_main.c | 35 +-
> > drivers/net/ethernet/intel/i40e/i40e_txrx.c | 163 ++---
> > drivers/net/ethernet/intel/i40e/i40e_txrx.h | 128 ++-
> > drivers/net/ethernet/intel/i40e/i40e_xsk.c | 537
> >
> > drivers/net/ethernet/intel/i40e/i40e_xsk.h | 17 +
> > include/net/xdp_sock.h | 19 +
> > net/xdp/xdp_umem.h | 10 -
> > 9 files changed, 789 insertions(+), 146 deletions(-)
> > create mode 100644 drivers/net/ethernet/intel/i40e/i40e_xsk.c
> > create mode 100644 drivers/net/ethernet/intel/i40e/i40e_xsk.h
> >
> > diff --git a/drivers/net/ethernet/intel/i40e/Makefile
> > b/drivers/net/ethernet/intel/i40e/Makefile
> > index 14397e7e9925..50590e8d1fd1 100644
> > --- a/drivers/net/ethernet/intel/i40e/Makefile
> > +++ b/drivers/net/ethernet/intel/i40e/Makefile
> > @@ -22,6 +22,7 @@ i40e-objs := i40e_main.o \
> > i40e_txrx.o \
> > i40e_ptp.o \
> > i40e_client.o \
> > - i40e_virtchnl_pf.o
> > + i40e_virtchnl_pf.o \
> > + i40e_xsk.o
> >
> > i40e-$(CONFIG_I40E_DCB) += i40e_dcb.o i40e_dcb_nl.o
> > diff --git a/drivers/net/ethernet/intel/i40e/i40e.h
> > b/drivers/net/ethernet/intel/i40e/i40e.h
> > index 7a80652e2500..20955e5dce02 100644
> > --- a/drivers/net/ethernet/intel/i40e/i40e.h
> > +++ b/drivers/net/ethernet/intel/i40e/i40e.h
> > @@ -786,6 +786,12 @@ struct i40e_vsi {
> >
> > /* VSI specific handlers */
> > irqreturn_t (*irq_handler)(int irq, void *data);
> > +
> > + /* AF_XDP zero-copy */
> > + struct xdp_umem **xsk_umems;
> > + u16 num_xsk_umems_used;
> > + u16 num_xsk_umems;
> > +
> > } cacheline_internodealigned_in_smp;
> >
> > struct i40e_netdev_priv {
> > @@ -1090,6 +1096,20 @@ static inline bool i40e_enabled_xdp_vsi(struct
> > i40e_vsi *vsi)
> > return !!vsi->xdp_prog;
> > }
> >
> > +static inline struct xdp_umem *i40e_xsk_umem(struct i40e_ring *ring)
> > +{
> > + bool xdp_on = i40e_enabled_xdp_vsi(ring->vsi);
> > + int qid = ring->queue_index;
> > +
> > + if (ring_is_xdp(ring))
> > + qid -= ring->vsi->alloc_queue_pairs;
> > +
> > + if (!ring->vsi->xsk_umems || !ring->vsi->xsk_umems[qid] || !xdp_on)
> > + return NULL;
> > +
> > + return ring->vsi->xsk_umems[qid];
> > +}
> > +
> > int i40e_create_queue_channel(struct i40e_vsi *vsi, struct i40e_channel
> > *ch);
> > int i40e_set_bw_limit(struct i40e_vsi *vsi, u16 seid, u64 max_tx_rate);
> > int i40e_add_del_cloud_filter(struct i40e_vsi *vsi,
> > @@ -1098,4 +1118,7 @@ int i40e_add_del_cloud_filter(struct i40e_vsi *vsi,
> > int i40e_add_del_cloud_filter_big_buf(struct i40e_vsi *vsi,
> > struct i40e_cloud_filter *filter,
> > bool add);
> > +int i40e_queue_pair_enable(struct i40e_vsi *vsi, int queue_pair);
> > +int i40e_queue_pair_disable(struct i40e_vsi *vsi, int queue_pair);
> > +
> > #endif /* _I40E_H_ */
> > diff --git a/drivers/net/ethernet/intel/i40e/i40e_main.c
> > b/drivers/net/ethernet/intel/i40e/i40e_main.c
> > index 369a116edaa1..8c602424d339 100644
> > --- a/drivers/net/ethernet/intel/i40e/i40e_main.c
> > +++ b/drivers/net/ethernet/intel/i40e/i40e_main.c
> > @@ -5,6 +5,7 @@
> > #include
> > #include
> > #include
> > +#include
> >
> > /* Local includes */
> > #include "i40e.h"
> > @@ -16,6 +17,7 @@
> > */
> > #define CREATE_TRACE_POINTS
> > #include "i40e_trace.h"
> > +#include "i40e_xsk.h"
> >
> > const char i40e_driver_name[] = "i40e";
> > static const char i40e_driver_string[] =
> > @@ -3071,6 +3073,9 @@ static int
Re: [PATCH bpf-next 09/11] i40e: implement AF_XDP zero-copy support for Rx
On Mon, Jun 4, 2018 at 5:05 AM, Björn Töpel wrote:
> From: Björn Töpel
>
> This commit adds initial AF_XDP zero-copy support for i40e-based
> NICs. First we add support for the new XDP_QUERY_XSK_UMEM and
> XDP_SETUP_XSK_UMEM commands in ndo_bpf. This allows the AF_XDP socket
> to pass a UMEM to the driver. The driver will then DMA map all the
> frames in the UMEM for the driver. Next, the Rx code will allocate
> frames from the UMEM fill queue, instead of the regular page
> allocator.
>
> Externally, for the rest of the XDP code, the driver internal UMEM
> allocator will appear as a MEM_TYPE_ZERO_COPY.
>
> The commit also introduces a completely new clean_rx_irq/allocator
> functions for zero-copy, and means (functions pointers) to set
> allocators and clean_rx functions.
>
> This first version does not support:
> * passing frames to the stack via XDP_PASS (clone/copy to skb).
> * doing XDP redirect to other than AF_XDP sockets
> (convert_to_xdp_frame does not clone the frame yet).
>
> Signed-off-by: Björn Töpel
> ---
> drivers/net/ethernet/intel/i40e/Makefile| 3 +-
> drivers/net/ethernet/intel/i40e/i40e.h | 23 ++
> drivers/net/ethernet/intel/i40e/i40e_main.c | 35 +-
> drivers/net/ethernet/intel/i40e/i40e_txrx.c | 163 ++---
> drivers/net/ethernet/intel/i40e/i40e_txrx.h | 128 ++-
> drivers/net/ethernet/intel/i40e/i40e_xsk.c | 537
>
> drivers/net/ethernet/intel/i40e/i40e_xsk.h | 17 +
> include/net/xdp_sock.h | 19 +
> net/xdp/xdp_umem.h | 10 -
> 9 files changed, 789 insertions(+), 146 deletions(-)
> create mode 100644 drivers/net/ethernet/intel/i40e/i40e_xsk.c
> create mode 100644 drivers/net/ethernet/intel/i40e/i40e_xsk.h
>
> diff --git a/drivers/net/ethernet/intel/i40e/Makefile
> b/drivers/net/ethernet/intel/i40e/Makefile
> index 14397e7e9925..50590e8d1fd1 100644
> --- a/drivers/net/ethernet/intel/i40e/Makefile
> +++ b/drivers/net/ethernet/intel/i40e/Makefile
> @@ -22,6 +22,7 @@ i40e-objs := i40e_main.o \
> i40e_txrx.o \
> i40e_ptp.o \
> i40e_client.o \
> - i40e_virtchnl_pf.o
> + i40e_virtchnl_pf.o \
> + i40e_xsk.o
>
> i40e-$(CONFIG_I40E_DCB) += i40e_dcb.o i40e_dcb_nl.o
> diff --git a/drivers/net/ethernet/intel/i40e/i40e.h
> b/drivers/net/ethernet/intel/i40e/i40e.h
> index 7a80652e2500..20955e5dce02 100644
> --- a/drivers/net/ethernet/intel/i40e/i40e.h
> +++ b/drivers/net/ethernet/intel/i40e/i40e.h
> @@ -786,6 +786,12 @@ struct i40e_vsi {
>
> /* VSI specific handlers */
> irqreturn_t (*irq_handler)(int irq, void *data);
> +
> + /* AF_XDP zero-copy */
> + struct xdp_umem **xsk_umems;
> + u16 num_xsk_umems_used;
> + u16 num_xsk_umems;
> +
> } cacheline_internodealigned_in_smp;
>
> struct i40e_netdev_priv {
> @@ -1090,6 +1096,20 @@ static inline bool i40e_enabled_xdp_vsi(struct
> i40e_vsi *vsi)
> return !!vsi->xdp_prog;
> }
>
> +static inline struct xdp_umem *i40e_xsk_umem(struct i40e_ring *ring)
> +{
> + bool xdp_on = i40e_enabled_xdp_vsi(ring->vsi);
> + int qid = ring->queue_index;
> +
> + if (ring_is_xdp(ring))
> + qid -= ring->vsi->alloc_queue_pairs;
> +
> + if (!ring->vsi->xsk_umems || !ring->vsi->xsk_umems[qid] || !xdp_on)
> + return NULL;
> +
> + return ring->vsi->xsk_umems[qid];
> +}
> +
> int i40e_create_queue_channel(struct i40e_vsi *vsi, struct i40e_channel *ch);
> int i40e_set_bw_limit(struct i40e_vsi *vsi, u16 seid, u64 max_tx_rate);
> int i40e_add_del_cloud_filter(struct i40e_vsi *vsi,
> @@ -1098,4 +1118,7 @@ int i40e_add_del_cloud_filter(struct i40e_vsi *vsi,
> int i40e_add_del_cloud_filter_big_buf(struct i40e_vsi *vsi,
> struct i40e_cloud_filter *filter,
> bool add);
> +int i40e_queue_pair_enable(struct i40e_vsi *vsi, int queue_pair);
> +int i40e_queue_pair_disable(struct i40e_vsi *vsi, int queue_pair);
> +
> #endif /* _I40E_H_ */
> diff --git a/drivers/net/ethernet/intel/i40e/i40e_main.c
> b/drivers/net/ethernet/intel/i40e/i40e_main.c
> index 369a116edaa1..8c602424d339 100644
> --- a/drivers/net/ethernet/intel/i40e/i40e_main.c
> +++ b/drivers/net/ethernet/intel/i40e/i40e_main.c
> @@ -5,6 +5,7 @@
> #include
> #include
> #include
> +#include
>
> /* Local includes */
> #include "i40e.h"
> @@ -16,6 +17,7 @@
> */
> #define CREATE_TRACE_POINTS
> #include "i40e_trace.h"
> +#include "i40e_xsk.h"
>
> const char i40e_driver_name[] = "i40e";
> static const char i40e_driver_string[] =
> @@ -3071,6 +3073,9 @@ static int i40e_configure_tx_ring(struct i40e_ring
> *ring)
> i40e_status err = 0;
> u32 qtx_ctl = 0;
>
> + if (ring_is_xdp(ring))
> + ring->xsk_umem = i40e_xsk_umem(ring);
> +
> /* some ATR related tx ring init */
> if (vsi->back->flags & I40E_FLAG_FD_ATR_ENABLED) {
>
[PATCH bpf-next 09/11] i40e: implement AF_XDP zero-copy support for Rx
From: Björn Töpel
This commit adds initial AF_XDP zero-copy support for i40e-based
NICs. First we add support for the new XDP_QUERY_XSK_UMEM and
XDP_SETUP_XSK_UMEM commands in ndo_bpf. This allows the AF_XDP socket
to pass a UMEM to the driver. The driver will then DMA map all the
frames in the UMEM for the driver. Next, the Rx code will allocate
frames from the UMEM fill queue, instead of the regular page
allocator.
Externally, for the rest of the XDP code, the driver internal UMEM
allocator will appear as a MEM_TYPE_ZERO_COPY.
The commit also introduces a completely new clean_rx_irq/allocator
functions for zero-copy, and means (functions pointers) to set
allocators and clean_rx functions.
This first version does not support:
* passing frames to the stack via XDP_PASS (clone/copy to skb).
* doing XDP redirect to other than AF_XDP sockets
(convert_to_xdp_frame does not clone the frame yet).
Signed-off-by: Björn Töpel
---
drivers/net/ethernet/intel/i40e/Makefile| 3 +-
drivers/net/ethernet/intel/i40e/i40e.h | 23 ++
drivers/net/ethernet/intel/i40e/i40e_main.c | 35 +-
drivers/net/ethernet/intel/i40e/i40e_txrx.c | 163 ++---
drivers/net/ethernet/intel/i40e/i40e_txrx.h | 128 ++-
drivers/net/ethernet/intel/i40e/i40e_xsk.c | 537
drivers/net/ethernet/intel/i40e/i40e_xsk.h | 17 +
include/net/xdp_sock.h | 19 +
net/xdp/xdp_umem.h | 10 -
9 files changed, 789 insertions(+), 146 deletions(-)
create mode 100644 drivers/net/ethernet/intel/i40e/i40e_xsk.c
create mode 100644 drivers/net/ethernet/intel/i40e/i40e_xsk.h
diff --git a/drivers/net/ethernet/intel/i40e/Makefile
b/drivers/net/ethernet/intel/i40e/Makefile
index 14397e7e9925..50590e8d1fd1 100644
--- a/drivers/net/ethernet/intel/i40e/Makefile
+++ b/drivers/net/ethernet/intel/i40e/Makefile
@@ -22,6 +22,7 @@ i40e-objs := i40e_main.o \
i40e_txrx.o \
i40e_ptp.o \
i40e_client.o \
- i40e_virtchnl_pf.o
+ i40e_virtchnl_pf.o \
+ i40e_xsk.o
i40e-$(CONFIG_I40E_DCB) += i40e_dcb.o i40e_dcb_nl.o
diff --git a/drivers/net/ethernet/intel/i40e/i40e.h
b/drivers/net/ethernet/intel/i40e/i40e.h
index 7a80652e2500..20955e5dce02 100644
--- a/drivers/net/ethernet/intel/i40e/i40e.h
+++ b/drivers/net/ethernet/intel/i40e/i40e.h
@@ -786,6 +786,12 @@ struct i40e_vsi {
/* VSI specific handlers */
irqreturn_t (*irq_handler)(int irq, void *data);
+
+ /* AF_XDP zero-copy */
+ struct xdp_umem **xsk_umems;
+ u16 num_xsk_umems_used;
+ u16 num_xsk_umems;
+
} cacheline_internodealigned_in_smp;
struct i40e_netdev_priv {
@@ -1090,6 +1096,20 @@ static inline bool i40e_enabled_xdp_vsi(struct i40e_vsi
*vsi)
return !!vsi->xdp_prog;
}
+static inline struct xdp_umem *i40e_xsk_umem(struct i40e_ring *ring)
+{
+ bool xdp_on = i40e_enabled_xdp_vsi(ring->vsi);
+ int qid = ring->queue_index;
+
+ if (ring_is_xdp(ring))
+ qid -= ring->vsi->alloc_queue_pairs;
+
+ if (!ring->vsi->xsk_umems || !ring->vsi->xsk_umems[qid] || !xdp_on)
+ return NULL;
+
+ return ring->vsi->xsk_umems[qid];
+}
+
int i40e_create_queue_channel(struct i40e_vsi *vsi, struct i40e_channel *ch);
int i40e_set_bw_limit(struct i40e_vsi *vsi, u16 seid, u64 max_tx_rate);
int i40e_add_del_cloud_filter(struct i40e_vsi *vsi,
@@ -1098,4 +1118,7 @@ int i40e_add_del_cloud_filter(struct i40e_vsi *vsi,
int i40e_add_del_cloud_filter_big_buf(struct i40e_vsi *vsi,
struct i40e_cloud_filter *filter,
bool add);
+int i40e_queue_pair_enable(struct i40e_vsi *vsi, int queue_pair);
+int i40e_queue_pair_disable(struct i40e_vsi *vsi, int queue_pair);
+
#endif /* _I40E_H_ */
diff --git a/drivers/net/ethernet/intel/i40e/i40e_main.c
b/drivers/net/ethernet/intel/i40e/i40e_main.c
index 369a116edaa1..8c602424d339 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_main.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_main.c
@@ -5,6 +5,7 @@
#include
#include
#include
+#include
/* Local includes */
#include "i40e.h"
@@ -16,6 +17,7 @@
*/
#define CREATE_TRACE_POINTS
#include "i40e_trace.h"
+#include "i40e_xsk.h"
const char i40e_driver_name[] = "i40e";
static const char i40e_driver_string[] =
@@ -3071,6 +3073,9 @@ static int i40e_configure_tx_ring(struct i40e_ring *ring)
i40e_status err = 0;
u32 qtx_ctl = 0;
+ if (ring_is_xdp(ring))
+ ring->xsk_umem = i40e_xsk_umem(ring);
+
/* some ATR related tx ring init */
if (vsi->back->flags & I40E_FLAG_FD_ATR_ENABLED) {
ring->atr_sample_rate = vsi->back->atr_sample_rate;
@@ -3180,13 +3185,30 @@ static int i40e_configure_rx_ring(struct i40e_ring
*ring)
struct i40e_hw *hw = >back->hw;
struct i40e_hmc_obj_rxq rx_ctx;
i40e_status err = 0;
+ int ret;
