The branch main has been updated by kib: URL: https://cgit.FreeBSD.org/src/commit/?id=240b7bfe56f25b20ee248e4fb4a3232c5f13390d
commit 240b7bfe56f25b20ee248e4fb4a3232c5f13390d Author: Konstantin Belousov <[email protected]> AuthorDate: 2024-02-14 12:52:56 +0000 Commit: Konstantin Belousov <[email protected]> CommitDate: 2024-07-12 04:27:58 +0000 ipsec_offload: offload inner checksums calculations for UDP/TCP/TSO and allow the interface driver to declare such support. Sponsored by: NVIDIA networking Differential revision: https://reviews.freebsd.org/D44221 --- sys/net/if_var.h | 3 ++ sys/netipsec/ipsec_offload.c | 5 ++- sys/netipsec/ipsec_offload.h | 7 +++- sys/netipsec/ipsec_output.c | 90 +++++++++++++++++++++++--------------------- 4 files changed, 60 insertions(+), 45 deletions(-) diff --git a/sys/net/if_var.h b/sys/net/if_var.h index 3e094dcb3cd5..579585b25dd2 100644 --- a/sys/net/if_var.h +++ b/sys/net/if_var.h @@ -148,6 +148,8 @@ enum IF_SA_CNT_WHICH { }; typedef int (*if_sa_cnt_fn_t)(if_t ifp, void *sa, uint32_t drv_spi, void *priv, struct seclifetime *lt); +typedef int (*if_ipsec_hwassist_fn_t)(if_t ifp, void *sav, + u_int drv_spi,void *priv); struct ifnet_hw_tsomax { u_int tsomaxbytes; /* TSO total burst length limit in bytes */ @@ -727,6 +729,7 @@ struct if_ipsec_accel_methods { if_sa_newkey_fn_t if_sa_newkey; if_sa_deinstall_fn_t if_sa_deinstall; if_sa_cnt_fn_t if_sa_cnt; + if_ipsec_hwassist_fn_t if_hwassist; }; void if_setipsec_accel_methods(if_t ifp, const struct if_ipsec_accel_methods *); diff --git a/sys/netipsec/ipsec_offload.c b/sys/netipsec/ipsec_offload.c index 851bacaf4ea1..7f63a5e0ccb6 100644 --- a/sys/netipsec/ipsec_offload.c +++ b/sys/netipsec/ipsec_offload.c @@ -799,12 +799,13 @@ ipsec_accel_output_tag(struct mbuf *m, u_int drv_spi) bool ipsec_accel_output(struct ifnet *ifp, struct mbuf *m, struct inpcb *inp, - struct secpolicy *sp, struct secasvar *sav, int af, int mtu) + struct secpolicy *sp, struct secasvar *sav, int af, int mtu, int *hwassist) { struct ifp_handle_sav *i; struct ip *ip; u_long ip_len, skip; + *hwassist = 0; if (ifp == NULL) return (false); @@ -845,6 +846,8 @@ ipsec_accel_output(struct ifnet *ifp, struct mbuf *m, struct inpcb *inp, if (sp != NULL) key_freesp(&sp); + *hwassist = ifp->if_ipsec_accel_m->if_hwassist(ifp, sav, + i->drv_spi, i->ifdata); return (true); } diff --git a/sys/netipsec/ipsec_offload.h b/sys/netipsec/ipsec_offload.h index 87e2a33288be..27b9c938832e 100644 --- a/sys/netipsec/ipsec_offload.h +++ b/sys/netipsec/ipsec_offload.h @@ -173,11 +173,14 @@ void ipsec_accel_spddel_impl(struct secpolicy *sp); int ipsec_accel_input(struct mbuf *m, int offset, int proto); bool ipsec_accel_output(struct ifnet *ifp, struct mbuf *m, struct inpcb *inp, struct secpolicy *sp, struct secasvar *sav, int af, - int mtu); + int mtu, int *hwassist); void ipsec_accel_forget_sav(struct secasvar *sav); #else #define ipsec_accel_input(a, b, c) (ENXIO) -#define ipsec_accel_output(a, b, c, d, e, f, g) (false) +#define ipsec_accel_output(a, b, c, d, e, f, g, h) ({ \ + *h = 0; \ + false; \ +}) #define ipsec_accel_forget_sav(a) #endif diff --git a/sys/netipsec/ipsec_output.c b/sys/netipsec/ipsec_output.c index 8f49bc8fce24..10f1728f72ac 100644 --- a/sys/netipsec/ipsec_output.c +++ b/sys/netipsec/ipsec_output.c @@ -195,7 +195,8 @@ ipsec4_perform_request(struct ifnet *ifp, struct mbuf *m, struct secpolicy *sp, union sockaddr_union *dst; struct secasvar *sav; struct ip *ip; - int error, i, off; + int error, hwassist, i, off; + bool accel; IPSEC_ASSERT(idx < sp->tcount, ("Wrong IPsec request index %d", idx)); @@ -212,7 +213,7 @@ ipsec4_perform_request(struct ifnet *ifp, struct mbuf *m, struct secpolicy *sp, if (sav == NULL) { if (error == EJUSTRETURN) { /* No IPsec required */ (void)ipsec_accel_output(ifp, m, inp, sp, NULL, - AF_INET, mtu); + AF_INET, mtu, &hwassist); key_freesp(&sp); return (error); } @@ -225,7 +226,28 @@ ipsec4_perform_request(struct ifnet *ifp, struct mbuf *m, struct secpolicy *sp, if ((error = ipsec_run_hhooks(&ctx, HHOOK_TYPE_IPSEC_OUT)) != 0) goto bad; - if (ipsec_accel_output(ifp, m, inp, sp, sav, AF_INET, mtu)) + hwassist = 0; + accel = ipsec_accel_output(ifp, m, inp, sp, sav, AF_INET, mtu, + &hwassist); + + /* + * Do delayed checksums now because we send before + * this is done in the normal processing path. + */ + if ((m->m_pkthdr.csum_flags & CSUM_DELAY_DATA & ~hwassist) != 0) { + in_delayed_cksum(m); + m->m_pkthdr.csum_flags &= ~CSUM_DELAY_DATA; + } +#if defined(SCTP) || defined(SCTP_SUPPORT) + if ((m->m_pkthdr.csum_flags & CSUM_SCTP & ~hwassist) != 0) { + struct ip *ip; + + ip = mtod(m, struct ip *); + sctp_delayed_cksum(m, (uint32_t)(ip->ip_hl << 2)); + m->m_pkthdr.csum_flags &= ~CSUM_SCTP; + } +#endif + if (accel) return (EJUSTRETURN); ip = mtod(m, struct ip *); @@ -401,25 +423,7 @@ ipsec4_common_output(struct ifnet *ifp, struct mbuf *m, struct inpcb *inp, * packets, and thus, even if they are forwarded, the replies will * return back to us. */ - if (!forwarding) { - /* - * Do delayed checksums now because we send before - * this is done in the normal processing path. - */ - if (m->m_pkthdr.csum_flags & CSUM_DELAY_DATA) { - in_delayed_cksum(m); - m->m_pkthdr.csum_flags &= ~CSUM_DELAY_DATA; - } -#if defined(SCTP) || defined(SCTP_SUPPORT) - if (m->m_pkthdr.csum_flags & CSUM_SCTP) { - struct ip *ip; - ip = mtod(m, struct ip *); - sctp_delayed_cksum(m, (uint32_t)(ip->ip_hl << 2)); - m->m_pkthdr.csum_flags &= ~CSUM_SCTP; - } -#endif - } /* NB: callee frees mbuf and releases reference to SP */ error = ipsec4_check_pmtu(ifp, m, sp, forwarding); if (error != 0) { @@ -596,7 +600,8 @@ ipsec6_perform_request(struct ifnet *ifp, struct mbuf *m, struct secpolicy *sp, union sockaddr_union *dst; struct secasvar *sav; struct ip6_hdr *ip6; - int error, i, off; + int error, hwassist, i, off; + bool accel; IPSEC_ASSERT(idx < sp->tcount, ("Wrong IPsec request index %d", idx)); @@ -604,7 +609,7 @@ ipsec6_perform_request(struct ifnet *ifp, struct mbuf *m, struct secpolicy *sp, if (sav == NULL) { if (error == EJUSTRETURN) { /* No IPsec required */ (void)ipsec_accel_output(ifp, m, inp, sp, NULL, - AF_INET6, mtu); + AF_INET6, mtu, &hwassist); key_freesp(&sp); return (error); } @@ -619,7 +624,26 @@ ipsec6_perform_request(struct ifnet *ifp, struct mbuf *m, struct secpolicy *sp, if ((error = ipsec_run_hhooks(&ctx, HHOOK_TYPE_IPSEC_OUT)) != 0) goto bad; - if (ipsec_accel_output(ifp, m, inp, sp, sav, AF_INET6, mtu)) + hwassist = 0; + accel = ipsec_accel_output(ifp, m, inp, sp, sav, AF_INET6, mtu, + &hwassist); + + /* + * Do delayed checksums now because we send before + * this is done in the normal processing path. + */ + if ((m->m_pkthdr.csum_flags & CSUM_DELAY_DATA_IPV6 & ~hwassist) != 0) { + in6_delayed_cksum(m, m->m_pkthdr.len - + sizeof(struct ip6_hdr), sizeof(struct ip6_hdr)); + m->m_pkthdr.csum_flags &= ~CSUM_DELAY_DATA_IPV6; + } +#if defined(SCTP) || defined(SCTP_SUPPORT) + if ((m->m_pkthdr.csum_flags & CSUM_SCTP_IPV6 & ~hwassist) != 0) { + sctp_delayed_cksum(m, sizeof(struct ip6_hdr)); + m->m_pkthdr.csum_flags &= ~CSUM_SCTP_IPV6; + } +#endif + if (accel) return (EJUSTRETURN); ip6 = mtod(m, struct ip6_hdr *); /* pfil can change mbuf */ @@ -778,24 +802,6 @@ ipsec6_common_output(struct ifnet *ifp, struct mbuf *m, struct inpcb *inp, return (0); /* No IPsec required. */ } - if (!forwarding) { - /* - * Do delayed checksums now because we send before - * this is done in the normal processing path. - */ - if (m->m_pkthdr.csum_flags & CSUM_DELAY_DATA_IPV6) { - in6_delayed_cksum(m, m->m_pkthdr.len - - sizeof(struct ip6_hdr), sizeof(struct ip6_hdr)); - m->m_pkthdr.csum_flags &= ~CSUM_DELAY_DATA_IPV6; - } -#if defined(SCTP) || defined(SCTP_SUPPORT) - if (m->m_pkthdr.csum_flags & CSUM_SCTP_IPV6) { - sctp_delayed_cksum(m, sizeof(struct ip6_hdr)); - m->m_pkthdr.csum_flags &= ~CSUM_SCTP_IPV6; - } -#endif - } - error = ipsec6_check_pmtu(ifp, m, sp, forwarding); if (error != 0) { if (error == EJUSTRETURN)
