This provider offloads classifier to software XDP.
It works only when a custom XDP object is loaded by afxdp netdev.
The BPF program needs to implement classifier with array-of-maps for
subtable hashmaps and arraymap for subtable masks. The flow api
provider detects classifier support in the custom XDP program when
loading it.
The requirements for the BPF program is as below:
- A struct named "meta_info" in ".ovs_meta" section
inform vswitchd of supported keys, actions, and the max number of
actions.
- A map named "subtbl_masks"
An array which implements a list. The entry contains struct
xdp_subtables_mask provided by lib/netdev-offload-xdp.h followed by
miniflow buf of any length for the mask.
- A map named "subtbl_masks_hd"
A one entry int array which contains the first entry index of the list
implemented by subtbl_masks.
- A map named "flow_table"
An array-of-maps. Each entry points to subtable hash-map instantiated
with the following "subtbl_template".
The entry of each index of subtbl_masks has miniflow mask of subtable
in the corresponding index of flow_table.
- A map named "subtbl_template"
A template for subtable, used for entries of "flow_table".
The key must be miniflow buf (without leading map).
- A map named "output_map"
A devmap. Each entry represents odp port.
- A map named "xsks_map"
A xskmap. Used for upcall.
For more details, refer to the reference BPF program in next commit.
In the future it may be possible to offload classifier to SmartNIC
through XDP, but currently it's not because map-in-map is not supported
by XDP hw-offload.
Signed-off-by: Toshiaki Makita <[email protected]>
---
acinclude.m4 | 25 +
configure.ac | 1 +
lib/automake.mk | 8 +
lib/bpf-util.c | 38 ++
lib/bpf-util.h | 22 +
lib/netdev-afxdp.c | 218 +++++-
lib/netdev-afxdp.h | 3 +
lib/netdev-linux-private.h | 2 +
lib/netdev-offload-provider.h | 8 +-
lib/netdev-offload-xdp.c | 1213 +++++++++++++++++++++++++++++++++
lib/netdev-offload-xdp.h | 49 ++
lib/netdev-offload.c | 3 +
12 files changed, 1588 insertions(+), 2 deletions(-)
create mode 100644 lib/bpf-util.c
create mode 100644 lib/bpf-util.h
create mode 100644 lib/netdev-offload-xdp.c
create mode 100644 lib/netdev-offload-xdp.h
diff --git a/acinclude.m4 b/acinclude.m4
index 4bac9dbdd..31ff0c013 100644
--- a/acinclude.m4
+++ b/acinclude.m4
@@ -329,6 +329,31 @@ AC_DEFUN([OVS_CHECK_LINUX_AF_XDP], [
AM_CONDITIONAL([HAVE_AF_XDP], test "$AF_XDP_ENABLE" = true)
])
+dnl OVS_CHECK_LINUX_XDP_OFFLOAD
+dnl
+dnl Check both llvm and libbpf support
+AC_DEFUN([OVS_CHECK_LINUX_XDP_OFFLOAD], [
+ AC_ARG_ENABLE([xdp_offload],
+ [AC_HELP_STRING([--enable-xdp-offload],
+ [Compile XDP offload])],
+ [], [enable_xdp_offload=no])
+ AC_MSG_CHECKING([whether XDP offload is enabled])
+ if test "$enable_xdp_offload" != yes; then
+ AC_MSG_RESULT([no])
+ XDP_OFFLOAD_ENABLE=false
+ else
+ if test "$enable_afxdp" == no; then
+ AC_MSG_ERROR([XDP offload depends on afxdp. Please add --enable-afxdp.])
+ fi
+ AC_MSG_RESULT([yes])
+ XDP_OFFLOAD_ENABLE=true
+
+ AC_DEFINE([HAVE_XDP_OFFLOAD], [1],
+ [Define to 1 if XDP offload compilation is available and
enabled.])
+ fi
+ AM_CONDITIONAL([HAVE_XDP_OFFLOAD], test "$XDP_OFFLOAD_ENABLE" = true)
+])
+
dnl OVS_CHECK_DPDK
dnl
dnl Configure DPDK source tree
diff --git a/configure.ac b/configure.ac
index 8d37af9db..530112c49 100644
--- a/configure.ac
+++ b/configure.ac
@@ -99,6 +99,7 @@ OVS_CHECK_DOT
OVS_CHECK_IF_DL
OVS_CHECK_STRTOK_R
OVS_CHECK_LINUX_AF_XDP
+OVS_CHECK_LINUX_XDP_OFFLOAD
AC_CHECK_DECLS([sys_siglist], [], [], [[#include <signal.h>]])
AC_CHECK_MEMBERS([struct stat.st_mtim.tv_nsec, struct stat.st_mtimensec],
[], [], [[#include <sys/stat.h>]])
diff --git a/lib/automake.mk b/lib/automake.mk
index 920c958e3..9486b32e7 100644
--- a/lib/automake.mk
+++ b/lib/automake.mk
@@ -451,6 +451,14 @@ lib_libopenvswitch_la_SOURCES += \
lib/netdev-afxdp.h
endif
+if HAVE_XDP_OFFLOAD
+lib_libopenvswitch_la_SOURCES += \
+ lib/bpf-util.c \
+ lib/bpf-util.h \
+ lib/netdev-offload-xdp.c \
+ lib/netdev-offload-xdp.h
+endif
+
if DPDK_NETDEV
lib_libopenvswitch_la_SOURCES += \
lib/dpdk.c \
diff --git a/lib/bpf-util.c b/lib/bpf-util.c
new file mode 100644
index 000000000..324cfbe1d
--- /dev/null
+++ b/lib/bpf-util.c
@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2020 NTT Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <config.h>
+
+#include "bpf-util.h"
+
+#include <bpf/libbpf.h>
+
+#include "ovs-thread.h"
+
+DEFINE_STATIC_PER_THREAD_DATA(struct { char s[128]; },
+ libbpf_strerror_buffer,
+ { "" });
+
+const char *
+ovs_libbpf_strerror(int err)
+{
+ enum { BUFSIZE = sizeof libbpf_strerror_buffer_get()->s };
+ char *buf = libbpf_strerror_buffer_get()->s;
+
+ libbpf_strerror(err, buf, BUFSIZE);
+
+ return buf;
+}
diff --git a/lib/bpf-util.h b/lib/bpf-util.h
new file mode 100644
index 000000000..6346935b3
--- /dev/null
+++ b/lib/bpf-util.h
@@ -0,0 +1,22 @@
+/*
+ * Copyright (c) 2020 NTT Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef BPF_UTIL_H
+#define BPF_UTIL_H 1
+
+const char *ovs_libbpf_strerror(int err);
+
+#endif /* bpf-util.h */
diff --git a/lib/netdev-afxdp.c b/lib/netdev-afxdp.c
index e1374ccbb..00204f299 100644
--- a/lib/netdev-afxdp.c
+++ b/lib/netdev-afxdp.c
@@ -22,6 +22,7 @@
#include "netdev-afxdp-pool.h"
#include <bpf/bpf.h>
+#include <bpf/btf.h>
#include <errno.h>
#include <inttypes.h>
#include <linux/rtnetlink.h>
@@ -37,10 +38,13 @@
#include <sys/types.h>
#include <unistd.h>
+#include "bpf-util.h"
#include "coverage.h"
#include "dp-packet.h"
#include "dpif-netdev.h"
#include "fatal-signal.h"
+#include "netdev-offload-provider.h"
+#include "netdev-offload-xdp.h"
#include "openvswitch/compiler.h"
#include "openvswitch/dynamic-string.h"
#include "openvswitch/list.h"
@@ -261,10 +265,204 @@ netdev_afxdp_sweep_unused_pools(void *aux OVS_UNUSED)
ovs_mutex_unlock(&unused_pools_mutex);
}
+bool
+has_xdp_flowtable(struct netdev *netdev)
+{
+ struct netdev_linux *dev = netdev_linux_cast(netdev);
+
+ return dev->has_xdp_flowtable;
+}
+
+struct bpf_object *
+get_xdp_object(struct netdev *netdev)
+{
+ struct netdev_linux *dev = netdev_linux_cast(netdev);
+
+ return dev->xdp_obj;
+}
+
+#ifdef HAVE_XDP_OFFLOAD
+static int
+xdp_preload(struct netdev *netdev, struct bpf_object *obj)
+{
+ static struct ovsthread_once output_map_once = OVSTHREAD_ONCE_INITIALIZER;
+ struct btf *btf;
+ struct bpf_map *flow_table, *subtbl_template, *subtbl_masks_hd, *xsks_map,
+ *output_map;
+ const struct bpf_map_def *flow_table_def, *subtbl_def;
+ int flow_table_fd, subtbl_meta_fd, xsks_map_fd;
+ static int output_map_fd = -1;
+ int n_rxq, err;
+
+ btf = bpf_object__btf(obj);
+ if (!btf) {
+ VLOG_DBG("BPF object for netdev \"%s\" does not contain BTF",
+ netdev_get_name(netdev));
+ return EOPNOTSUPP;
+ }
+ if (btf__find_by_name_kind(btf, ".ovs_meta", BTF_KIND_DATASEC) < 0) {
+ VLOG_DBG("\".ovs_meta\" datasec not found in BTF");
+ return EOPNOTSUPP;
+ }
+
+ flow_table = bpf_object__find_map_by_name(obj, "flow_table");
+ if (!flow_table) {
+ VLOG_DBG("%s: \"flow_table\" map does not exist",
+ netdev_get_name(netdev));
+ return EOPNOTSUPP;
+ }
+
+ subtbl_template = bpf_object__find_map_by_name(obj, "subtbl_template");
+ if (!subtbl_template) {
+ VLOG_DBG("%s: \"subtbl_template\" map does not exist",
+ netdev_get_name(netdev));
+ return EOPNOTSUPP;
+ }
+
+ output_map = bpf_object__find_map_by_name(obj, "output_map");
+ if (!output_map) {
+ VLOG_DBG("%s: \"output_map\" map does not exist",
+ netdev_get_name(netdev));
+ return EOPNOTSUPP;
+ }
+
+ if (ovsthread_once_start(&output_map_once)) {
+ output_map_fd = bpf_create_map_name(BPF_MAP_TYPE_DEVMAP, "output_map",
+ sizeof(int), sizeof(int),
+ XDP_MAX_PORTS, 0);
+ if (output_map_fd < 0) {
+ err = errno;
+ VLOG_WARN("%s: Map creation for output_map failed: %s",
+ netdev_get_name(netdev), ovs_strerror(errno));
+ ovsthread_once_done(&output_map_once);
+ return err;
+ }
+ ovsthread_once_done(&output_map_once);
+ }
+
+ flow_table_def = bpf_map__def(flow_table);
+ if (flow_table_def->type != BPF_MAP_TYPE_ARRAY_OF_MAPS) {
+ VLOG_WARN("%s: \"flow_table\" map type is not map-in-map array.",
+ netdev_get_name(netdev));
+ return EINVAL;
+ }
+
+ subtbl_def = bpf_map__def(subtbl_template);
+ if (subtbl_def->type != BPF_MAP_TYPE_HASH) {
+ VLOG_WARN("%s: \"subtbl_templates\" map type is not hash-table",
+ netdev_get_name(netdev));
+ return EINVAL;
+ }
+
+ subtbl_meta_fd = bpf_create_map(BPF_MAP_TYPE_HASH, subtbl_def->key_size,
+ subtbl_def->value_size,
+ subtbl_def->max_entries, 0);
+ if (subtbl_meta_fd < 0) {
+ err = errno;
+ VLOG_WARN("%s: Map creation for flow_table meta table failed: %s",
+ netdev_get_name(netdev), ovs_strerror(errno));
+ return err;
+ }
+ flow_table_fd = bpf_create_map_in_map(BPF_MAP_TYPE_ARRAY_OF_MAPS,
+ "flow_table", sizeof(uint32_t),
+ subtbl_meta_fd,
+ flow_table_def->max_entries,
+ 0);
+ if (flow_table_fd < 0) {
+ err = errno;
+ VLOG_WARN("%s: Map creation for flow_table failed: %s",
+ netdev_get_name(netdev), ovs_strerror(errno));
+ close(subtbl_meta_fd);
+ return err;
+ }
+ close(subtbl_meta_fd);
+
+ err = bpf_map__reuse_fd(flow_table, flow_table_fd);
+ if (err) {
+ VLOG_WARN("%s: Failed to reuse flow_table fd: %s",
+ netdev_get_name(netdev), ovs_libbpf_strerror(err));
+ close(flow_table_fd);
+ return EINVAL;
+ }
+ close(flow_table_fd);
