Re: [PATCH bpf-next] xdp: sample code for redirecting vlan packets to specific cpus
On Wed, Nov 7, 2018 at 1:44 PM Daniel Borkmann wrote:
>
> On 10/29/2018 11:11 PM, John Fastabend wrote:
> > On 10/29/2018 02:19 PM, Shannon Nelson wrote:
> >> This is an example of using XDP to redirect the processing of
> >> particular vlan packets to specific CPUs. This is in response
> >> to comments received on a kernel patch put forth previously
> >> to do something similar using RPS.
> >> https://www.spinics.net/lists/netdev/msg528210.html
> >> [PATCH net-next] net: enable RPS on vlan devices
> >>
> >> This XDP application watches for inbound vlan-tagged packets
> >> and redirects those packets to be processed on a specific CPU
> >> as configured in a BPF map. The BPF map can be modified by
> >> this user program, which can also load and unload the kernel
> >> XDP code.
> >>
> >> One example use is for supporting VMs where we can't control the
> >> OS being used: we'd like to separate the VM CPU processing from
> >> the host's CPUs as a way to help mitigate L1TF related issues.
> >> When running the VM's traffic on a vlan we can stick the host's
> >> Rx processing on one set of CPUs separate from the VM's CPUs.
> >>
> >> This example currently uses a vlan key and cpu value in the
> >> BPF map, so only can do one CPU per vlan. This could easily
> >> be modified to use a bitpattern of CPUs rather than a CPU id
> >> to allow multiple CPUs per vlan.
> >
> > Great, so does this solve your use case then? At least on drivers
> > with XDP support?
> >
> >>
> >> Signed-off-by: Shannon Nelson
> >> ---
> >
> > Some really small and trivial nits below.
> >
> > Acked-by: John Fastabend
> >
> > [...]
> >
> >> +if (install) {
> >> +
> >
> > new line probably not needed.
> >
> >> +/* check to see if already installed */
> >> +errno = 0;
> >> +access(pin_prog_name, R_OK);
> >> +if (errno != ENOENT) {
> >> +fprintf(stderr, "ERR: %s is already installed\n",
> >> argv[0]);
> >> +return -1;
> >> +}
> >> +
> >> +/* load the XDP program and maps with the convenient library
> >> */
> >> +if (load_bpf_file(filename)) {
> >> +fprintf(stderr, "ERR: load_bpf_file(%s): \n%s",
> >> +filename, bpf_log_buf);
> >> +return -1;
> >> +}
> >> +if (!prog_fd[0]) {
> >> +fprintf(stderr, "ERR: load_bpf_file(%s): %d %s\n",
> >> +filename, errno, strerror(errno));
> >> +return -1;
> >> +}
> >> +
> >> +/* pin the XDP program and maps */
> >> +if (bpf_obj_pin(prog_fd[0], pin_prog_name) < 0) {
> >> +fprintf(stderr, "ERR: bpf_obj_pin(%s): %d %s\n",
> >> +pin_prog_name, errno, strerror(errno));
> >> +if (errno == 2)
> >> +fprintf(stderr, " (is the BPF fs mounted
> >> on /sys/fs/bpf?)\n");
> >> +return -1;
> >> +}
> >> +if (bpf_obj_pin(map_fd[0], pin_vlanmap_name) < 0) {
> >> +fprintf(stderr, "ERR: bpf_obj_pin(%s): %d %s\n",
> >> +pin_vlanmap_name, errno, strerror(errno));
> >> +return -1;
> >> +}
> >> +if (bpf_obj_pin(map_fd[2], pin_countermap_name) < 0) {
> >> +fprintf(stderr, "ERR: bpf_obj_pin(%s): %d %s\n",
> >> +pin_countermap_name, errno, strerror(errno));
> >> +return -1;
> >> +}
> >> +
> >> +/* prep the vlan map with "not used" values */
> >> +c64 = UNDEF_CPU;
> >> +for (v64 = 0; v64 < 4096; v64++) {
> >
> > maybe #define MAX_VLANS 4096 just to avoid constants.
> >
> >> +if (bpf_map_update_elem(map_fd[0], &v64, &c64, 0)) {
> >> +fprintf(stderr, "ERR: preping vlan map failed
> >> on v=%llu: %d %s\n",
> >> +v64, errno, strerror(errno));
> >> +return -1;
> >> +}
> >> +}
> >> +
> >> +/* prep the cpumap with queue sizes */
> >> +c64 = 128+64; /* see note in xdp_redirect_cpu_user.c */
> >> +for (v64 = 0; v64 < MAX_CPUS; v64++) {
> >> +if (bpf_map_update_elem(map_fd[1], &v64, &c64, 0)) {
> >> +if (errno == ENODEV) {
> >> +/* Save the last CPU number attempted
> >> + * into the counters map
> >> + */
> >> +c64 = CPU_COUNT;
> >> +ret = bpf_map_update_elem(map_fd[2],
> >> &c64, &v64, 0);
> >> +break;
> >> +
Re: [PATCH bpf-next] xdp: sample code for redirecting vlan packets to specific cpus
On 10/29/2018 11:11 PM, John Fastabend wrote:
> On 10/29/2018 02:19 PM, Shannon Nelson wrote:
>> This is an example of using XDP to redirect the processing of
>> particular vlan packets to specific CPUs. This is in response
>> to comments received on a kernel patch put forth previously
>> to do something similar using RPS.
>> https://www.spinics.net/lists/netdev/msg528210.html
>> [PATCH net-next] net: enable RPS on vlan devices
>>
>> This XDP application watches for inbound vlan-tagged packets
>> and redirects those packets to be processed on a specific CPU
>> as configured in a BPF map. The BPF map can be modified by
>> this user program, which can also load and unload the kernel
>> XDP code.
>>
>> One example use is for supporting VMs where we can't control the
>> OS being used: we'd like to separate the VM CPU processing from
>> the host's CPUs as a way to help mitigate L1TF related issues.
>> When running the VM's traffic on a vlan we can stick the host's
>> Rx processing on one set of CPUs separate from the VM's CPUs.
>>
>> This example currently uses a vlan key and cpu value in the
>> BPF map, so only can do one CPU per vlan. This could easily
>> be modified to use a bitpattern of CPUs rather than a CPU id
>> to allow multiple CPUs per vlan.
>
> Great, so does this solve your use case then? At least on drivers
> with XDP support?
>
>>
>> Signed-off-by: Shannon Nelson
>> ---
>
> Some really small and trivial nits below.
>
> Acked-by: John Fastabend
>
> [...]
>
>> +if (install) {
>> +
>
> new line probably not needed.
>
>> +/* check to see if already installed */
>> +errno = 0;
>> +access(pin_prog_name, R_OK);
>> +if (errno != ENOENT) {
>> +fprintf(stderr, "ERR: %s is already installed\n",
>> argv[0]);
>> +return -1;
>> +}
>> +
>> +/* load the XDP program and maps with the convenient library */
>> +if (load_bpf_file(filename)) {
>> +fprintf(stderr, "ERR: load_bpf_file(%s): \n%s",
>> +filename, bpf_log_buf);
>> +return -1;
>> +}
>> +if (!prog_fd[0]) {
>> +fprintf(stderr, "ERR: load_bpf_file(%s): %d %s\n",
>> +filename, errno, strerror(errno));
>> +return -1;
>> +}
>> +
>> +/* pin the XDP program and maps */
>> +if (bpf_obj_pin(prog_fd[0], pin_prog_name) < 0) {
>> +fprintf(stderr, "ERR: bpf_obj_pin(%s): %d %s\n",
>> +pin_prog_name, errno, strerror(errno));
>> +if (errno == 2)
>> +fprintf(stderr, " (is the BPF fs mounted on
>> /sys/fs/bpf?)\n");
>> +return -1;
>> +}
>> +if (bpf_obj_pin(map_fd[0], pin_vlanmap_name) < 0) {
>> +fprintf(stderr, "ERR: bpf_obj_pin(%s): %d %s\n",
>> +pin_vlanmap_name, errno, strerror(errno));
>> +return -1;
>> +}
>> +if (bpf_obj_pin(map_fd[2], pin_countermap_name) < 0) {
>> +fprintf(stderr, "ERR: bpf_obj_pin(%s): %d %s\n",
>> +pin_countermap_name, errno, strerror(errno));
>> +return -1;
>> +}
>> +
>> +/* prep the vlan map with "not used" values */
>> +c64 = UNDEF_CPU;
>> +for (v64 = 0; v64 < 4096; v64++) {
>
> maybe #define MAX_VLANS 4096 just to avoid constants.
>
>> +if (bpf_map_update_elem(map_fd[0], &v64, &c64, 0)) {
>> +fprintf(stderr, "ERR: preping vlan map failed
>> on v=%llu: %d %s\n",
>> +v64, errno, strerror(errno));
>> +return -1;
>> +}
>> +}
>> +
>> +/* prep the cpumap with queue sizes */
>> +c64 = 128+64; /* see note in xdp_redirect_cpu_user.c */
>> +for (v64 = 0; v64 < MAX_CPUS; v64++) {
>> +if (bpf_map_update_elem(map_fd[1], &v64, &c64, 0)) {
>> +if (errno == ENODEV) {
>> +/* Save the last CPU number attempted
>> + * into the counters map
>> + */
>> +c64 = CPU_COUNT;
>> +ret = bpf_map_update_elem(map_fd[2],
>> &c64, &v64, 0);
>> +break;
>> +}
>> +
>> +fprintf(stderr, "ERR: preping cpu map failed on
>> v=%llu: %d %s\n",
>> +v64, errno, strerror(errno));
>> +return -1;
>> +}
>> +}
>> +
>
Re: [PATCH bpf-next] xdp: sample code for redirecting vlan packets to specific cpus
On 10/29/2018 3:11 PM, John Fastabend wrote: On 10/29/2018 02:19 PM, Shannon Nelson wrote: This is an example of using XDP to redirect the processing of particular vlan packets to specific CPUs. This is in response to comments received on a kernel patch put forth previously to do something similar using RPS. https://www.spinics.net/lists/netdev/msg528210.html [PATCH net-next] net: enable RPS on vlan devices This XDP application watches for inbound vlan-tagged packets and redirects those packets to be processed on a specific CPU as configured in a BPF map. The BPF map can be modified by this user program, which can also load and unload the kernel XDP code. One example use is for supporting VMs where we can't control the OS being used: we'd like to separate the VM CPU processing from the host's CPUs as a way to help mitigate L1TF related issues. When running the VM's traffic on a vlan we can stick the host's Rx processing on one set of CPUs separate from the VM's CPUs. This example currently uses a vlan key and cpu value in the BPF map, so only can do one CPU per vlan. This could easily be modified to use a bitpattern of CPUs rather than a CPU id to allow multiple CPUs per vlan. Great, so does this solve your use case then? At least on drivers with XDP support? Well, more or less... the actual issue was a request for our UEK5 distribution, based on v4.14, which doesn't have support for the CPU redirect. Internal discussion continues. Signed-off-by: Shannon Nelson --- Some really small and trivial nits below. Acked-by: John Fastabend Thanks, sln
Re: [PATCH bpf-next] xdp: sample code for redirecting vlan packets to specific cpus
On 10/29/2018 02:19 PM, Shannon Nelson wrote:
> This is an example of using XDP to redirect the processing of
> particular vlan packets to specific CPUs. This is in response
> to comments received on a kernel patch put forth previously
> to do something similar using RPS.
> https://www.spinics.net/lists/netdev/msg528210.html
> [PATCH net-next] net: enable RPS on vlan devices
>
> This XDP application watches for inbound vlan-tagged packets
> and redirects those packets to be processed on a specific CPU
> as configured in a BPF map. The BPF map can be modified by
> this user program, which can also load and unload the kernel
> XDP code.
>
> One example use is for supporting VMs where we can't control the
> OS being used: we'd like to separate the VM CPU processing from
> the host's CPUs as a way to help mitigate L1TF related issues.
> When running the VM's traffic on a vlan we can stick the host's
> Rx processing on one set of CPUs separate from the VM's CPUs.
>
> This example currently uses a vlan key and cpu value in the
> BPF map, so only can do one CPU per vlan. This could easily
> be modified to use a bitpattern of CPUs rather than a CPU id
> to allow multiple CPUs per vlan.
Great, so does this solve your use case then? At least on drivers
with XDP support?
>
> Signed-off-by: Shannon Nelson
> ---
Some really small and trivial nits below.
Acked-by: John Fastabend
[...]
> + if (install) {
> +
new line probably not needed.
> + /* check to see if already installed */
> + errno = 0;
> + access(pin_prog_name, R_OK);
> + if (errno != ENOENT) {
> + fprintf(stderr, "ERR: %s is already installed\n",
> argv[0]);
> + return -1;
> + }
> +
> + /* load the XDP program and maps with the convenient library */
> + if (load_bpf_file(filename)) {
> + fprintf(stderr, "ERR: load_bpf_file(%s): \n%s",
> + filename, bpf_log_buf);
> + return -1;
> + }
> + if (!prog_fd[0]) {
> + fprintf(stderr, "ERR: load_bpf_file(%s): %d %s\n",
> + filename, errno, strerror(errno));
> + return -1;
> + }
> +
> + /* pin the XDP program and maps */
> + if (bpf_obj_pin(prog_fd[0], pin_prog_name) < 0) {
> + fprintf(stderr, "ERR: bpf_obj_pin(%s): %d %s\n",
> + pin_prog_name, errno, strerror(errno));
> + if (errno == 2)
> + fprintf(stderr, " (is the BPF fs mounted on
> /sys/fs/bpf?)\n");
> + return -1;
> + }
> + if (bpf_obj_pin(map_fd[0], pin_vlanmap_name) < 0) {
> + fprintf(stderr, "ERR: bpf_obj_pin(%s): %d %s\n",
> + pin_vlanmap_name, errno, strerror(errno));
> + return -1;
> + }
> + if (bpf_obj_pin(map_fd[2], pin_countermap_name) < 0) {
> + fprintf(stderr, "ERR: bpf_obj_pin(%s): %d %s\n",
> + pin_countermap_name, errno, strerror(errno));
> + return -1;
> + }
> +
> + /* prep the vlan map with "not used" values */
> + c64 = UNDEF_CPU;
> + for (v64 = 0; v64 < 4096; v64++) {
maybe #define MAX_VLANS 4096 just to avoid constants.
> + if (bpf_map_update_elem(map_fd[0], &v64, &c64, 0)) {
> + fprintf(stderr, "ERR: preping vlan map failed
> on v=%llu: %d %s\n",
> + v64, errno, strerror(errno));
> + return -1;
> + }
> + }
> +
> + /* prep the cpumap with queue sizes */
> + c64 = 128+64; /* see note in xdp_redirect_cpu_user.c */
> + for (v64 = 0; v64 < MAX_CPUS; v64++) {
> + if (bpf_map_update_elem(map_fd[1], &v64, &c64, 0)) {
> + if (errno == ENODEV) {
> + /* Save the last CPU number attempted
> + * into the counters map
> + */
> + c64 = CPU_COUNT;
> + ret = bpf_map_update_elem(map_fd[2],
> &c64, &v64, 0);
> + break;
> + }
> +
> + fprintf(stderr, "ERR: preping cpu map failed on
> v=%llu: %d %s\n",
> + v64, errno, strerror(errno));
> + return -1;
> + }
> + }
> +
> + /* wire the XDP program to the device */
> + if (bpf_set_link_xdp_fd(ifindex, prog_fd[0],
[PATCH bpf-next] xdp: sample code for redirecting vlan packets to specific cpus
This is an example of using XDP to redirect the processing of
particular vlan packets to specific CPUs. This is in response
to comments received on a kernel patch put forth previously
to do something similar using RPS.
https://www.spinics.net/lists/netdev/msg528210.html
[PATCH net-next] net: enable RPS on vlan devices
This XDP application watches for inbound vlan-tagged packets
and redirects those packets to be processed on a specific CPU
as configured in a BPF map. The BPF map can be modified by
this user program, which can also load and unload the kernel
XDP code.
One example use is for supporting VMs where we can't control the
OS being used: we'd like to separate the VM CPU processing from
the host's CPUs as a way to help mitigate L1TF related issues.
When running the VM's traffic on a vlan we can stick the host's
Rx processing on one set of CPUs separate from the VM's CPUs.
This example currently uses a vlan key and cpu value in the
BPF map, so only can do one CPU per vlan. This could easily
be modified to use a bitpattern of CPUs rather than a CPU id
to allow multiple CPUs per vlan.
Signed-off-by: Shannon Nelson
---
samples/bpf/Makefile | 3 +
samples/bpf/xdp_vlan_redirect_kern.c | 118 +++
samples/bpf/xdp_vlan_redirect_user.c | 393 +++
3 files changed, 514 insertions(+)
create mode 100644 samples/bpf/xdp_vlan_redirect_kern.c
create mode 100644 samples/bpf/xdp_vlan_redirect_user.c
diff --git a/samples/bpf/Makefile b/samples/bpf/Makefile
index 1303af1..a9746ce 100644
--- a/samples/bpf/Makefile
+++ b/samples/bpf/Makefile
@@ -44,6 +44,7 @@ hostprogs-y += load_sock_ops
hostprogs-y += xdp_redirect
hostprogs-y += xdp_redirect_map
hostprogs-y += xdp_redirect_cpu
+hostprogs-y += xdp_vlan_redirect
hostprogs-y += xdp_monitor
hostprogs-y += xdp_rxq_info
hostprogs-y += syscall_tp
@@ -99,6 +100,7 @@ per_socket_stats_example-objs := cookie_uid_helper_example.o
xdp_redirect-objs := bpf_load.o xdp_redirect_user.o
xdp_redirect_map-objs := bpf_load.o xdp_redirect_map_user.o
xdp_redirect_cpu-objs := bpf_load.o xdp_redirect_cpu_user.o
+xdp_vlan_redirect-objs := bpf_load.o xdp_vlan_redirect_user.o
xdp_monitor-objs := bpf_load.o xdp_monitor_user.o
xdp_rxq_info-objs := xdp_rxq_info_user.o
syscall_tp-objs := bpf_load.o syscall_tp_user.o
@@ -154,6 +156,7 @@ always += tcp_basertt_kern.o
always += xdp_redirect_kern.o
always += xdp_redirect_map_kern.o
always += xdp_redirect_cpu_kern.o
+always += xdp_vlan_redirect_kern.o
always += xdp_monitor_kern.o
always += xdp_rxq_info_kern.o
always += xdp2skb_meta_kern.o
diff --git a/samples/bpf/xdp_vlan_redirect_kern.c
b/samples/bpf/xdp_vlan_redirect_kern.c
new file mode 100644
index 000..3d561a0
--- /dev/null
+++ b/samples/bpf/xdp_vlan_redirect_kern.c
@@ -0,0 +1,118 @@
+// SPDX-License-Identifier: GPL-2.0
+/* XDP redirect vlans to CPUs - kernel code
+ *
+ * Copyright(c) 2018 Oracle Corp.
+ */
+
+#include
+#include
+
+#include
+#include "bpf_helpers.h"
+
+char _license[] SEC("license") = "GPL";
+
+#define MAX_CPUS 64 /* WARNING - sync with _user.c */
+#define UNDEF_CPU 0xff00 /* WARNING - sync with _user.c */
+
+/* The vlan index finds cpu(s) for processing a packet */
+struct bpf_map_def SEC("maps") vlan_redirect_map = {
+ .type = BPF_MAP_TYPE_ARRAY,
+ .key_size = sizeof(u32),/* vlan tag */
+ .value_size = sizeof(u64), /* cpu bitpattern */
+ .max_entries = 4096,
+};
+
+/* List of cpus that can participate in the vlan redirect */
+struct bpf_map_def SEC("maps") vlan_redirect_cpus_map = {
+ .type = BPF_MAP_TYPE_CPUMAP,
+ .key_size = sizeof(u32), /* cpu id */
+ .value_size = sizeof(u32), /* queue size */
+ .max_entries= MAX_CPUS,
+};
+
+/* Counters for debug */
+#define VRC_CALLS 0/* number of calls to this program */
+#define VRC_VLANS 1/* number of vlan packets seen */
+#define VRC_HITS 2/* number of redirects attempted */
+#define CPU_COUNT 3/* number of CPUs found */
+struct bpf_map_def SEC("maps") vlan_redirect_counters_map = {
+ .type = BPF_MAP_TYPE_ARRAY,
+ .key_size = sizeof(u32),/* vlan tag */
+ .value_size = sizeof(u64), /* cpu bitpattern */
+ .max_entries = 4,
+};
+
+SEC("xdp_vlan_redirect")
+int xdp_vlan_redirect(struct xdp_md *ctx)
+{
+ void *data_end = (void *)(long)ctx->data_end;
+ void *data = (void *)(long)ctx->data;
+ struct ethhdr *eth = data;
+ uint64_t h_proto, nh_off;
+ struct vlan_hdr *vhdr;
+ uint64_t *cpu_value;
+ uint32_t num_cpus;
+ uint32_t minlen;
+ uint64_t *vp;
+ uint64_t v, k;
+ uint64_t vlan;
+
+ /* count packets processed */
+ k = VRC_CALLS;
+ vp = bpf_map_lookup_elem(&vlan_redirect_counters_map, &k);
+ if (vp) {
+ v = (*vp) + 1;
+
