On 10/29/2018 11:11 PM, John Fastabend wrote: > On 10/29/2018 02:19 PM, Shannon Nelson wrote: >> This is an example of using XDP to redirect the processing of >> particular vlan packets to specific CPUs. This is in response >> to comments received on a kernel patch put forth previously >> to do something similar using RPS. >> https://www.spinics.net/lists/netdev/msg528210.html >> [PATCH net-next] net: enable RPS on vlan devices >> >> This XDP application watches for inbound vlan-tagged packets >> and redirects those packets to be processed on a specific CPU >> as configured in a BPF map. The BPF map can be modified by >> this user program, which can also load and unload the kernel >> XDP code. >> >> One example use is for supporting VMs where we can't control the >> OS being used: we'd like to separate the VM CPU processing from >> the host's CPUs as a way to help mitigate L1TF related issues. >> When running the VM's traffic on a vlan we can stick the host's >> Rx processing on one set of CPUs separate from the VM's CPUs. >> >> This example currently uses a vlan key and cpu value in the >> BPF map, so only can do one CPU per vlan. This could easily >> be modified to use a bitpattern of CPUs rather than a CPU id >> to allow multiple CPUs per vlan. > > Great, so does this solve your use case then? At least on drivers > with XDP support? > >> >> Signed-off-by: Shannon Nelson <shannon.nel...@oracle.com> >> --- > > Some really small and trivial nits below. > > Acked-by: John Fastabend <john.fastab...@gmail.com> > > [...] > >> + if (install) { >> + > > new line probably not needed. > >> + /* check to see if already installed */ >> + errno = 0; >> + access(pin_prog_name, R_OK); >> + if (errno != ENOENT) { >> + fprintf(stderr, "ERR: %s is already installed\n", >> argv[0]); >> + return -1; >> + } >> + >> + /* load the XDP program and maps with the convenient library */ >> + if (load_bpf_file(filename)) { >> + fprintf(stderr, "ERR: load_bpf_file(%s): \n%s", >> + filename, bpf_log_buf); >> + return -1; >> + } >> + if (!prog_fd[0]) { >> + fprintf(stderr, "ERR: load_bpf_file(%s): %d %s\n", >> + filename, errno, strerror(errno)); >> + return -1; >> + } >> + >> + /* pin the XDP program and maps */ >> + if (bpf_obj_pin(prog_fd[0], pin_prog_name) < 0) { >> + fprintf(stderr, "ERR: bpf_obj_pin(%s): %d %s\n", >> + pin_prog_name, errno, strerror(errno)); >> + if (errno == 2) >> + fprintf(stderr, " (is the BPF fs mounted on >> /sys/fs/bpf?)\n"); >> + return -1; >> + } >> + if (bpf_obj_pin(map_fd[0], pin_vlanmap_name) < 0) { >> + fprintf(stderr, "ERR: bpf_obj_pin(%s): %d %s\n", >> + pin_vlanmap_name, errno, strerror(errno)); >> + return -1; >> + } >> + if (bpf_obj_pin(map_fd[2], pin_countermap_name) < 0) { >> + fprintf(stderr, "ERR: bpf_obj_pin(%s): %d %s\n", >> + pin_countermap_name, errno, strerror(errno)); >> + return -1; >> + } >> + >> + /* prep the vlan map with "not used" values */ >> + c64 = UNDEF_CPU; >> + for (v64 = 0; v64 < 4096; v64++) { > > maybe #define MAX_VLANS 4096 just to avoid constants. > >> + if (bpf_map_update_elem(map_fd[0], &v64, &c64, 0)) { >> + fprintf(stderr, "ERR: preping vlan map failed >> on v=%llu: %d %s\n", >> + v64, errno, strerror(errno)); >> + return -1; >> + } >> + } >> + >> + /* prep the cpumap with queue sizes */ >> + c64 = 128+64; /* see note in xdp_redirect_cpu_user.c */ >> + for (v64 = 0; v64 < MAX_CPUS; v64++) { >> + if (bpf_map_update_elem(map_fd[1], &v64, &c64, 0)) { >> + if (errno == ENODEV) { >> + /* Save the last CPU number attempted >> + * into the counters map >> + */ >> + c64 = CPU_COUNT; >> + ret = bpf_map_update_elem(map_fd[2], >> &c64, &v64, 0); >> + break; >> + } >> + >> + fprintf(stderr, "ERR: preping cpu map failed on >> v=%llu: %d %s\n", >> + v64, errno, strerror(errno)); >> + return -1; >> + } >> + } >> + >> + /* wire the XDP program to the device */ >> + if (bpf_set_link_xdp_fd(ifindex, prog_fd[0], 0) < 0) { >> + fprintf(stderr, "ERR: bpf_set_link_xdp_fd(): %d %s\n", >> + errno, strerror(errno)); >> + return -1; >> + } >> + >> + return 0; >> + } >> + >> + if (remove) { >> +
Ditto >> + /* unlink the program from the device */ >> + if (bpf_set_link_xdp_fd(ifindex, -1, 0) < 0) >> + fprintf(stderr, "ERR: bpf_set_link_xdp_fd(): %d %s\n", >> + errno, strerror(errno)); >> + >> + /* unlink pinned files */ >> + if (unlink(pin_prog_name)) >> + fprintf(stderr, "ERR: unlink(%s): %d %s\n", >> + pin_prog_name, errno, strerror(errno)); >> + if (unlink(pin_vlanmap_name)) >> + fprintf(stderr, "ERR: unlink(%s): %d %s\n", >> + pin_vlanmap_name, errno, strerror(errno)); >> + if (unlink(pin_countermap_name)) >> + fprintf(stderr, "ERR: unlink(%s): %d %s\n", >> + pin_countermap_name, errno, strerror(errno)); >> + >> + return 0; >> + } >> + >> + if (vlan == 0) { >> + fprintf(stderr, "ERR: required option --vlan missing\n"); >> + goto error; >> + } >> + >> + if (cpu == MAX_CPUS && vlan > 0) { >> + fprintf(stderr, "ERR: required option --cpu missing\n"); >> + goto error; >> + } >> + >> + vfd = bpf_obj_get(pin_vlanmap_name); >> + if (vfd < 0) { >> + fprintf(stderr, "ERR: can't find pinned map %s: %d %s\n", >> + pin_vlanmap_name, errno, strerror(errno)); >> + if (errno == ENOENT) >> + fprintf(stderr, " (has %s been installed yet?)\n", >> argv[0]); >> + return -1; >> + } >> + >> + /* decode the requested action */ >> + if (vlan > 0) { >> + /* check cpu against the max value found */ >> + cfd = bpf_obj_get(pin_countermap_name); >> + if (cfd < 0) { >> + fprintf(stderr, "ERR: can't find pinned map %s: %d >> %s\n", >> + pin_countermap_name, errno, strerror(errno)); >> + return -1; >> + } >> + c64 = CPU_COUNT; >> + ret = bpf_map_lookup_elem(cfd, &c64, &v64); >> + if (cpu >= v64) { > > Need to check ret code? Yeah, I expect this would have another respin, thanks! >> + fprintf(stderr, "ERR: cpu %d greater than max %llu\n", >> cpu, v64); >> + return -1; >> + } >> + >> + /* Note that the value and key pointers really do need to be >> + * pointers to 64-bit values, else things get a bit muddled. >> + */ >> + v64 = vlan; >> + c64 = cpu; >> + ret = bpf_map_update_elem(vfd, &v64, &c64, 0); >> + if (ret) { >> + fprintf(stderr, "Adding vlan %d CPU %d failed: %d %s\n", >> + vlan, cpu, errno, strerror(errno)); >> + return -1; >> + } >> + >> + } else { >> + v64 = -vlan; >> + c64 = UNDEF_CPU; >> + >> + /* We can't actually delete from a TYPE_ARRAY map, so we >> + * simply set it to an undefined value. >> + */ >> + ret = bpf_map_update_elem(vfd, &v64, &c64, 0); >> + if (ret) { >> + fprintf(stderr, "Delete of vlan %llu failed: %d %s\n", >> + v64, errno, strerror(errno)); >> + return -1; >> + } >> + } >> + >> + return 0; >> + >> +error: >> + usage(argv); >> + return -1; >> +} >> >