UDP tunnel GRO callbacks normally remain installed until their socket is destroyed. This permits the tunnel callback registry to use one callback as a global static-call target when only one tunnel type is active.
A userspace-owned socket can instead outlive its association with an encapsulator. Allow a receive callback which produces only standard UDP frag-list GSO to opt out of the callback registry and be detached while the socket remains live. Such aggregates need no tunnel completion callback and remain valid after detach. Publish the detachable dispatch mode before the callback, then publish frag-list acceptance only after all callbacks which consume that representation. Load each callback once before invocation. Provide a helper which removes the socket from GRO lookup, clears its callbacks and frag-list acceptance, and waits for in-flight network readers before retiring detachable dispatch mode. Keep the helper in udp_offload.c, which is built into the UDP stack even when NET_UDP_TUNNEL is a module, so built-in callers do not gain a dependency on udp_tunnel.ko. Existing tunnel sockets retain the static-call path and semantics. Signed-off-by: Ralf Lici <[email protected]> --- include/linux/udp.h | 1 + include/net/udp_tunnel.h | 2 ++ net/ipv4/udp_offload.c | 59 +++++++++++++++++++++++++++++++++----- net/ipv4/udp_tunnel_core.c | 25 +++++++++++++--- 4 files changed, 76 insertions(+), 11 deletions(-) diff --git a/include/linux/udp.h b/include/linux/udp.h index 2a0c45ddd7b1..e81ed7248ea0 100644 --- a/include/linux/udp.h +++ b/include/linux/udp.h @@ -72,6 +72,7 @@ enum { * pulls are permitted. */ UDP_FLAGS_ENCAP_ACCEPT_FRAGLIST, + UDP_FLAGS_GRO_RCV_DETACHABLE, /* gro_receive may be detached */ }; /* per NUMA structure for lockless producer usage. */ diff --git a/include/net/udp_tunnel.h b/include/net/udp_tunnel.h index a540e86d5f84..3a137901c88b 100644 --- a/include/net/udp_tunnel.h +++ b/include/net/udp_tunnel.h @@ -211,10 +211,12 @@ static inline int udp_tunnel_handle_offloads(struct sk_buff *skb, bool udp_csum) #if IS_ENABLED(CONFIG_NET_UDP_TUNNEL) void udp_tunnel_update_gro_lookup(struct net *net, struct sock *sk, bool add); void udp_tunnel_update_gro_rcv(struct sock *sk, bool add); +void udp_tunnel_detach_gro(struct sock *sk); #else static inline void udp_tunnel_update_gro_lookup(struct net *net, struct sock *sk, bool add) {} static inline void udp_tunnel_update_gro_rcv(struct sock *sk, bool add) {} +static inline void udp_tunnel_detach_gro(struct sock *sk) {} #endif static inline void udp_tunnel_cleanup_gro(struct sock *sk) diff --git a/net/ipv4/udp_offload.c b/net/ipv4/udp_offload.c index cf07c3c6611a..9d6a5a52d879 100644 --- a/net/ipv4/udp_offload.c +++ b/net/ipv4/udp_offload.c @@ -82,6 +82,10 @@ void udp_tunnel_update_gro_rcv(struct sock *sk, bool add) struct udp_sock *up = udp_sk(sk); int i, old_gro_type_nr; + /* Dynamically detached callbacks cannot be static-call targets. */ + if (udp_test_bit(GRO_RCV_DETACHABLE, sk)) + return; + if (!UDP_MAX_TUNNEL_TYPES || !up->gro_receive) return; @@ -144,10 +148,45 @@ void udp_tunnel_update_gro_rcv(struct sock *sk, bool add) } EXPORT_SYMBOL_GPL(udp_tunnel_update_gro_rcv); +/** + * udp_tunnel_detach_gro - detach GRO callbacks from a live UDP socket + * @sk: UDP tunnel socket + * + * The caller must hold the socket lock and keep callback state alive until + * this function returns. + */ +void udp_tunnel_detach_gro(struct sock *sk) +{ + sock_owned_by_me(sk); + + if (WARN_ON_ONCE(!udp_test_bit(GRO_RCV_DETACHABLE, sk))) + return; + + udp_tunnel_update_gro_lookup(sock_net(sk), sk, false); + WRITE_ONCE(udp_sk(sk)->gro_receive, NULL); + WRITE_ONCE(udp_sk(sk)->gro_complete, NULL); + /* No new frag-list aggregates can be formed after gro_receive is + * cleared. Any aggregate already held by GRO may safely be segmented + * before encap_rcv. + */ + udp_clear_bit(ENCAP_ACCEPT_FRAGLIST, sk); + + /* Readers which captured a callback before it was cleared must keep + * bypassing the static-call path until they have completed. + */ + synchronize_net(); + udp_clear_bit(GRO_RCV_DETACHABLE, sk); +} +EXPORT_SYMBOL_GPL(udp_tunnel_detach_gro); + static struct sk_buff *udp_tunnel_gro_rcv(struct sock *sk, + udp_tunnel_gro_receive_t gro_receive, struct list_head *head, struct sk_buff *skb) { + if (unlikely(udp_test_bit(GRO_RCV_DETACHABLE, sk))) + return call_gro_receive_sk(gro_receive, sk, head, skb); + if (static_branch_likely(&udp_tunnel_static_call)) { if (unlikely(gro_recursion_inc_test(skb))) { NAPI_GRO_CB(skb)->flush |= 1; @@ -155,16 +194,17 @@ static struct sk_buff *udp_tunnel_gro_rcv(struct sock *sk, } return static_call(udp_tunnel_gro_rcv)(sk, head, skb); } - return call_gro_receive_sk(udp_sk(sk)->gro_receive, sk, head, skb); + return call_gro_receive_sk(gro_receive, sk, head, skb); } #else static struct sk_buff *udp_tunnel_gro_rcv(struct sock *sk, + udp_tunnel_gro_receive_t gro_receive, struct list_head *head, struct sk_buff *skb) { - return call_gro_receive_sk(udp_sk(sk)->gro_receive, sk, head, skb); + return call_gro_receive_sk(gro_receive, sk, head, skb); } #endif @@ -776,6 +816,7 @@ static struct sk_buff *udp_gro_receive_segment(struct list_head *head, struct sk_buff *udp_gro_receive(struct list_head *head, struct sk_buff *skb, struct udphdr *uh, struct sock *sk) { + udp_tunnel_gro_receive_t gro_receive = NULL; struct sk_buff *pp = NULL; struct sk_buff *p; struct udphdr *uh2; @@ -794,7 +835,9 @@ struct sk_buff *udp_gro_receive(struct list_head *head, struct sk_buff *skb, * cases. Such packets should be caught in udp_unexpected_gso later. */ NAPI_GRO_CB(skb)->is_flist = 0; - if (!sk || !udp_sk(sk)->gro_receive) { + if (sk) + gro_receive = READ_ONCE(udp_sk(sk)->gro_receive); + if (!gro_receive) { /* If the packet was locally encapsulated in a UDP tunnel that * wasn't detected above, do not GRO. */ @@ -841,7 +884,7 @@ struct sk_buff *udp_gro_receive(struct list_head *head, struct sk_buff *skb, skb_gro_pull(skb, sizeof(struct udphdr)); /* pull encapsulating udp header */ skb_gro_postpull_rcsum(skb, uh, sizeof(struct udphdr)); - pp = udp_tunnel_gro_rcv(sk, head, skb); + pp = udp_tunnel_gro_rcv(sk, gro_receive, head, skb); out: skb_gro_flush_final(skb, pp, flush); @@ -920,6 +963,7 @@ int udp_gro_complete(struct sk_buff *skb, int nhoff, { struct udphdr *uh = (struct udphdr *)(skb->data + nhoff); unsigned int newlen = skb->len - nhoff; + udp_tunnel_gro_complete_t gro_complete = NULL; struct sock *sk; int err; @@ -927,7 +971,9 @@ int udp_gro_complete(struct sk_buff *skb, int nhoff, sk = INDIRECT_CALL_INET(lookup, udp6_lib_lookup_skb, udp4_lib_lookup_skb, skb, uh->source, uh->dest); - if (sk && udp_sk(sk)->gro_complete) { + if (sk) + gro_complete = READ_ONCE(udp_sk(sk)->gro_complete); + if (gro_complete) { skb_shinfo(skb)->gso_type = uh->check ? SKB_GSO_UDP_TUNNEL_CSUM : SKB_GSO_UDP_TUNNEL; @@ -940,8 +986,7 @@ int udp_gro_complete(struct sk_buff *skb, int nhoff, * functions to make them set up the inner offsets. */ skb->encapsulation = 1; - err = udp_sk(sk)->gro_complete(sk, skb, - nhoff + sizeof(struct udphdr)); + err = gro_complete(sk, skb, nhoff + sizeof(struct udphdr)); } else { err = udp_gro_complete_segment(skb); } diff --git a/net/ipv4/udp_tunnel_core.c b/net/ipv4/udp_tunnel_core.c index 411073068075..f8fe5f673441 100644 --- a/net/ipv4/udp_tunnel_core.c +++ b/net/ipv4/udp_tunnel_core.c @@ -71,10 +71,22 @@ static bool sk_saddr_any(struct sock *sk) void setup_udp_tunnel_sock(struct net *net, struct sock *sk, struct udp_tunnel_sock_cfg *cfg) { - const unsigned long allowed = BIT(UDP_FLAGS_ENCAP_ACCEPT_FRAGLIST); + const unsigned long allowed = BIT(UDP_FLAGS_GRO_RCV_DETACHABLE) | + BIT(UDP_FLAGS_ENCAP_ACCEPT_FRAGLIST); unsigned long flags = cfg->udp_flags & allowed; WARN_ON_ONCE(cfg->udp_flags & ~allowed); + set_mask_bits(&udp_sk(sk)->udp_flags, BIT(UDP_FLAGS_GRO_RCV_DETACHABLE), + flags & BIT(UDP_FLAGS_GRO_RCV_DETACHABLE)); + if (flags & BIT(UDP_FLAGS_GRO_RCV_DETACHABLE)) { + WARN_ON_ONCE(cfg->gro_complete || + !(flags & BIT(UDP_FLAGS_ENCAP_ACCEPT_FRAGLIST))); + /* The socket may already be receiving packets so drain readers + * which could have observed the old dispatch mode before + * publishing the detachable callback. + */ + synchronize_net(); + } /* Disable multicast loopback */ inet_clear_bit(MC_LOOP, sk); @@ -89,13 +101,18 @@ void setup_udp_tunnel_sock(struct net *net, struct sock *sk, udp_sk(sk)->encap_err_rcv = cfg->encap_err_rcv; udp_sk(sk)->encap_err_lookup = cfg->encap_err_lookup; udp_sk(sk)->encap_destroy = cfg->encap_destroy; - udp_sk(sk)->gro_receive = cfg->gro_receive; - udp_sk(sk)->gro_complete = cfg->gro_complete; + WRITE_ONCE(udp_sk(sk)->gro_complete, cfg->gro_complete); + /* Publish receive after the completion callback which its aggregates + * may later require + */ + WRITE_ONCE(udp_sk(sk)->gro_receive, cfg->gro_receive); /* Publish frag-list acceptance only after the callbacks which consume * that representation */ - set_mask_bits(&udp_sk(sk)->udp_flags, allowed, flags); + set_mask_bits(&udp_sk(sk)->udp_flags, + BIT(UDP_FLAGS_ENCAP_ACCEPT_FRAGLIST), + flags & BIT(UDP_FLAGS_ENCAP_ACCEPT_FRAGLIST)); udp_tunnel_encap_enable(sk); -- 2.56.0 _______________________________________________ Openvpn-devel mailing list [email protected] https://lists.sourceforge.net/lists/listinfo/openvpn-devel
