Since commit 3d010c8031e3 ("udp: do not accept non-tunnel GSO skbs
landing in a tunnel"), udp_unexpected_gso segments generic UDP L4 and
frag-list GSO before passing packets to an encapsulation callback,
because GRO socket lookup cannot prove that such an aggregate will not
later reach a tunnel. This protects callbacks which only understand UDP
tunnel GSO packets, but also prevents an encapsulator from deliberately
batching independent outer UDP records as a standard frag-list
aggregate.
Add an explicit per-socket opt-in for an encap_rcv callback which
accepts the exact SKB_GSO_UDP_L4 | SKB_GSO_FRAGLIST representation.
Publish acceptance only after the callbacks which consume that
representation.
GRO has already validated every datagram in this representation, while
the aggregate checksum state does not describe the complete skb. Treat
the opt-in as provisional, that is, bypass the pre-encapsulation
checksum check, but segment the aggregate and process every datagram
normally if the callback is absent or declines it. A declining callback
must leave the aggregate unmodified apart from layout-only pulls. This
keeps the combined representation away from the UDP socket and leaves
callbacks which do not opt in protected as before.
Preserve the sampled acceptance decision across concurrent callback
changes and account every consumed datagram in UDP statistics.
Consuming the aggregate whole avoids rebuilding every datagram only for
the encapsulator to discard its outer headers, and lets the callback
amortize per-batch work and look ahead across records. With this series
and the ovpn UDP GSO transmit series applied, five interleaved 60-second
iperf3 -Z single-flow AES-128-GCM runs in each direction over two
directly connected 100-Gbit/s mlx5 ports, after a 10-second warm-up,
compared direct consumption by the ovpn GRO callback with the same
kernel in which ovpn declines every aggregate:
Forward Reverse
UDP segmentation 20.561 Gbit/s 20.498 Gbit/s
Direct consumption 25.741 Gbit/s 25.150 Gbit/s
Direct consumption improved throughput by 25.2% forward and 22.7%
reverse, and reduced mean cycles per byte across the monitored data-path
CPUs by 10.6% and 8.2%, respectively.
Export skb_gro_receive_list, already shared by the core UDP and TCP
frag-list GRO paths, so modular encapsulators can construct the same
standard representation.
Signed-off-by: Ralf Lici <[email protected]>
---
include/linux/udp.h | 12 ++++++++++++
include/net/udp_tunnel.h | 1 +
net/core/gro.c | 1 +
net/ipv4/udp.c | 37 ++++++++++++++++++++++++++++++-------
net/ipv4/udp_tunnel_core.c | 10 ++++++++++
net/ipv6/udp.c | 37 ++++++++++++++++++++++++++++++-------
6 files changed, 84 insertions(+), 14 deletions(-)
diff --git a/include/linux/udp.h b/include/linux/udp.h
index 998906ec3b32..2a0c45ddd7b1 100644
--- a/include/linux/udp.h
+++ b/include/linux/udp.h
@@ -67,6 +67,11 @@ enum {
UDP_FLAGS_ACCEPT_FRAGLIST,
UDP_FLAGS_ACCEPT_L4,
UDP_FLAGS_ENCAP_ENABLED, /* This socket enabled encap */
+ /* encap_rcv accepts UDP L4 frag-list GSO. If it does not consume such
+ * an aggregate, it must leave packet contents unchanged. Layout-only
+ * pulls are permitted.
+ */
+ UDP_FLAGS_ENCAP_ACCEPT_FRAGLIST,
};
/* per NUMA structure for lockless producer usage. */
@@ -211,6 +216,13 @@ static inline bool udp_encap_needed(void)
return false;
}
+static inline bool udp_encap_accepts_fraglist(struct sock *sk,
+ struct sk_buff *skb)
+{
+ return skb_is_gso(skb) && udp_test_bit(ENCAP_ACCEPT_FRAGLIST, sk) &&
+ skb_shinfo(skb)->gso_type == (SKB_GSO_UDP_L4 | SKB_GSO_FRAGLIST);
+}
+
static inline bool udp_unexpected_gso(struct sock *sk, struct sk_buff *skb)
{
if (!skb_is_gso(skb))
diff --git a/include/net/udp_tunnel.h b/include/net/udp_tunnel.h
index 498b7b262fa9..a540e86d5f84 100644
--- a/include/net/udp_tunnel.h
+++ b/include/net/udp_tunnel.h
@@ -85,6 +85,7 @@ struct udp_tunnel_sock_cfg {
void *sk_user_data; /* user data used by encap_rcv call back */
/* Used for setting up udp_sock fields, see udp.h for details */
__u8 encap_type;
+ unsigned long udp_flags;
udp_tunnel_encap_rcv_t encap_rcv;
udp_tunnel_encap_err_lookup_t encap_err_lookup;
udp_tunnel_encap_err_rcv_t encap_err_rcv;
diff --git a/net/core/gro.c b/net/core/gro.c
index ff828a3dce3c..069df5d96075 100644
--- a/net/core/gro.c
+++ b/net/core/gro.c
@@ -266,6 +266,7 @@ int skb_gro_receive_list(struct sk_buff *p, struct sk_buff
*skb)
return 0;
}
+EXPORT_SYMBOL(skb_gro_receive_list);
static void gro_complete(struct gro_node *gro, struct sk_buff *skb)
{
diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c
index 303d0102519a..9b17ae823e56 100644
--- a/net/ipv4/udp.c
+++ b/net/ipv4/udp.c
@@ -2391,7 +2391,8 @@ static int __udp_queue_rcv_skb(struct sock *sk, struct
sk_buff *skb)
* Note that in the success and error cases, the skb is assumed to
* have either been requeued or freed.
*/
-static int udp_queue_rcv_one_skb(struct sock *sk, struct sk_buff *skb)
+static int udp_queue_rcv_one_skb(struct sock *sk, struct sk_buff *skb,
+ bool encap_fraglist)
{
enum skb_drop_reason drop_reason = SKB_DROP_REASON_NOT_SPECIFIED;
struct udp_sock *up = udp_sk(sk);
@@ -2424,14 +2425,25 @@ static int udp_queue_rcv_one_skb(struct sock *sk,
struct sk_buff *skb)
/* if we're overly short, let UDP handle it */
encap_rcv = READ_ONCE(up->encap_rcv);
if (encap_rcv) {
+ unsigned int segs = encap_fraglist ?
+ skb_shinfo(skb)->gso_segs : 1;
int ret;
- /* Verify checksum before giving to encap */
- if (udp_lib_checksum_complete(skb))
+ /* GRO validated every datagram before constructing an
+ * accepted frag-list. Its aggregate checksum state
+ * does not describe the complete skb, so validate only
+ * after a fallback to individual datagrams.
+ */
+ if (!encap_fraglist && udp_lib_checksum_complete(skb))
goto csum_error;
ret = encap_rcv(sk, skb);
- if (ret <= 0) {
+ if (!ret) {
+ __SNMP_ADD_STATS(net->mib.udp_statistics,
+ UDP_MIB_INDATAGRAMS, segs);
+ return 0;
+ }
+ if (ret < 0 && !encap_fraglist) {
__UDP_INC_STATS(net, UDP_MIB_INDATAGRAMS);
return -ret;
}
@@ -2440,6 +2452,13 @@ static int udp_queue_rcv_one_skb(struct sock *sk, struct
sk_buff *skb)
/* FALLTHROUGH -- it's a UDP Packet */
}
+ /* The encapsulator did not consume an aggregate it had opted into so
+ * we ask udp_queue_rcv_skb to retry after segmentation rather than
+ * exposing multiple datagrams to the socket as one.
+ */
+ if (encap_fraglist)
+ return -EAGAIN;
+
prefetch(&sk->sk_rmem_alloc);
if (rcu_access_pointer(sk->sk_filter) &&
udp_lib_checksum_complete(skb))
@@ -2466,11 +2485,15 @@ static int udp_queue_rcv_one_skb(struct sock *sk,
struct sk_buff *skb)
static int udp_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
{
+ bool encap_fraglist = udp_encap_accepts_fraglist(sk, skb);
struct sk_buff *next, *segs;
int ret;
- if (likely(!udp_unexpected_gso(sk, skb)))
- return udp_queue_rcv_one_skb(sk, skb);
+ if (likely(encap_fraglist || !udp_unexpected_gso(sk, skb))) {
+ ret = udp_queue_rcv_one_skb(sk, skb, encap_fraglist);
+ if (likely(ret != -EAGAIN))
+ return ret;
+ }
BUILD_BUG_ON(sizeof(struct udp_skb_cb) > SKB_GSO_CB_OFFSET);
__skb_push(skb, -skb_mac_offset(skb));
@@ -2479,7 +2502,7 @@ static int udp_queue_rcv_skb(struct sock *sk, struct
sk_buff *skb)
__skb_pull(skb, skb_transport_offset(skb));
udp_post_segment_fix_csum(skb);
- ret = udp_queue_rcv_one_skb(sk, skb);
+ ret = udp_queue_rcv_one_skb(sk, skb, false);
if (ret > 0)
ip_protocol_deliver_rcu(dev_net(skb->dev), skb, ret);
}
diff --git a/net/ipv4/udp_tunnel_core.c b/net/ipv4/udp_tunnel_core.c
index a128fe85620d..411073068075 100644
--- a/net/ipv4/udp_tunnel_core.c
+++ b/net/ipv4/udp_tunnel_core.c
@@ -71,6 +71,11 @@ static bool sk_saddr_any(struct sock *sk)
void setup_udp_tunnel_sock(struct net *net, struct sock *sk,
struct udp_tunnel_sock_cfg *cfg)
{
+ const unsigned long allowed = BIT(UDP_FLAGS_ENCAP_ACCEPT_FRAGLIST);
+ unsigned long flags = cfg->udp_flags & allowed;
+
+ WARN_ON_ONCE(cfg->udp_flags & ~allowed);
+
/* Disable multicast loopback */
inet_clear_bit(MC_LOOP, sk);
@@ -87,6 +92,11 @@ void setup_udp_tunnel_sock(struct net *net, struct sock *sk,
udp_sk(sk)->gro_receive = cfg->gro_receive;
udp_sk(sk)->gro_complete = cfg->gro_complete;
+ /* Publish frag-list acceptance only after the callbacks which consume
+ * that representation
+ */
+ set_mask_bits(&udp_sk(sk)->udp_flags, allowed, flags);
+
udp_tunnel_encap_enable(sk);
udp_tunnel_update_gro_rcv(sk, true);
diff --git a/net/ipv6/udp.c b/net/ipv6/udp.c
index 59f9872c3552..1b3d3adb3f44 100644
--- a/net/ipv6/udp.c
+++ b/net/ipv6/udp.c
@@ -810,7 +810,8 @@ static int __udpv6_queue_rcv_skb(struct sock *sk, struct
sk_buff *skb)
return 0;
}
-static int udpv6_queue_rcv_one_skb(struct sock *sk, struct sk_buff *skb)
+static int udpv6_queue_rcv_one_skb(struct sock *sk, struct sk_buff *skb,
+ bool encap_fraglist)
{
enum skb_drop_reason drop_reason = SKB_DROP_REASON_NOT_SPECIFIED;
struct udp_sock *up = udp_sk(sk);
@@ -840,14 +841,25 @@ static int udpv6_queue_rcv_one_skb(struct sock *sk,
struct sk_buff *skb)
/* if we're overly short, let UDP handle it */
encap_rcv = READ_ONCE(up->encap_rcv);
if (encap_rcv) {
+ unsigned int segs = encap_fraglist ?
+ skb_shinfo(skb)->gso_segs : 1;
int ret;
- /* Verify checksum before giving to encap */
- if (udp_lib_checksum_complete(skb))
+ /* GRO validated every datagram before constructing an
+ * accepted frag-list. Its aggregate checksum state
+ * does not describe the complete skb, so validate only
+ * after a fallback to individual datagrams.
+ */
+ if (!encap_fraglist && udp_lib_checksum_complete(skb))
goto csum_error;
ret = encap_rcv(sk, skb);
- if (ret <= 0) {
+ if (!ret) {
+ __SNMP_ADD_STATS(net->mib.udp_stats_in6,
+ UDP_MIB_INDATAGRAMS, segs);
+ return 0;
+ }
+ if (ret < 0 && !encap_fraglist) {
__UDP6_INC_STATS(net, UDP_MIB_INDATAGRAMS);
return -ret;
}
@@ -856,6 +868,13 @@ static int udpv6_queue_rcv_one_skb(struct sock *sk, struct
sk_buff *skb)
/* FALLTHROUGH -- it's a UDP Packet */
}
+ /* The encapsulator did not consume an aggregate it had opted into so
+ * we ask udpv6_queue_rcv_skb to retry after segmentation rather than
+ * exposing multiple datagrams to the socket as one.
+ */
+ if (encap_fraglist)
+ return -EAGAIN;
+
prefetch(&sk->sk_rmem_alloc);
if (rcu_access_pointer(sk->sk_filter) &&
udp_lib_checksum_complete(skb))
@@ -883,11 +902,15 @@ static int udpv6_queue_rcv_one_skb(struct sock *sk,
struct sk_buff *skb)
static int udpv6_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
{
+ bool encap_fraglist = udp_encap_accepts_fraglist(sk, skb);
struct sk_buff *next, *segs;
int ret;
- if (likely(!udp_unexpected_gso(sk, skb)))
- return udpv6_queue_rcv_one_skb(sk, skb);
+ if (likely(encap_fraglist || !udp_unexpected_gso(sk, skb))) {
+ ret = udpv6_queue_rcv_one_skb(sk, skb, encap_fraglist);
+ if (likely(ret != -EAGAIN))
+ return ret;
+ }
__skb_push(skb, -skb_mac_offset(skb));
segs = udp_rcv_segment(sk, skb, false);
@@ -895,7 +918,7 @@ static int udpv6_queue_rcv_skb(struct sock *sk, struct
sk_buff *skb)
__skb_pull(skb, skb_transport_offset(skb));
udp_post_segment_fix_csum(skb);
- ret = udpv6_queue_rcv_one_skb(sk, skb);
+ ret = udpv6_queue_rcv_one_skb(sk, skb, false);
if (ret > 0)
ip6_protocol_deliver_rcu(dev_net(skb->dev), skb, ret,
true);
--
2.56.0
_______________________________________________
Openvpn-devel mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/openvpn-devel
