The branch main has been updated by tuexen: URL: https://cgit.FreeBSD.org/src/commit/?id=bb50bb45f5f650873b202e644bfcd9f2992cdace
commit bb50bb45f5f650873b202e644bfcd9f2992cdace Author: Timo Völker <[email protected]> AuthorDate: 2026-08-27 03:09:55 +0000 Commit: Michael Tuexen <[email protected]> CommitDate: 2026-08-27 03:15:17 +0000 vtnet: move offload functions to virtio_net.h Move the functions vtnet_rxq_csum() and vtnet_txq_offload() and the subfunctions they call from if_vtnet.c to virtio_net.h. This allows us to call these functions from if_tuntap.c and if_ptnet.c. virtio_net.h already contained a copy of these functions, but a copy of an outdated version. The functions evolved in if_vtnet.c. In if_vtnet.c, the copy has never been used because it increments counters in their own functions. This patch removes the outdated copy from virtio_net.h and moves the new version of the functions from if_vtnet.c to virtio_net.h. if_tuntap.c, if_ptnet.c, and if_vtnet.c just call these functions, and if_vtnet.c increments its counters depending on the return value. Reviewed by: tuexen MFC after: 1 month MFC to: stable/15 Differential Revision: https://reviews.freebsd.org/D57299 --- sys/dev/netmap/if_ptnet.c | 22 ++- sys/dev/virtio/network/if_vtnet.c | 366 ++++++------------------------------ sys/dev/virtio/network/virtio_net.h | 341 +++++++++++++++++---------------- sys/net/if_tuntap.c | 25 ++- 4 files changed, 287 insertions(+), 467 deletions(-) diff --git a/sys/dev/netmap/if_ptnet.c b/sys/dev/netmap/if_ptnet.c index 9c06f7fec530..660b1930aa76 100644 --- a/sys/dev/netmap/if_ptnet.c +++ b/sys/dev/netmap/if_ptnet.c @@ -1464,8 +1464,17 @@ ptnet_drain_transmit_queue(struct ptnet_queue *pq, unsigned int budget, * two 8-bytes-wide writes. */ memset(nmbuf, 0, PTNET_HDR_SIZE); if (mhead->m_pkthdr.csum_flags & PTNET_ALL_OFFLOAD) { - mhead = virtio_net_tx_offload(ifp, mhead, false, - vh); + /* + * Translate the CSUM_* flags in the mbuf to the + * corresponding flags in the VirtIO header. + * + * ptnet does not negotiate ECN and orders the + * bytes in the VirtIO header as if the VirtIO + * modern mode is not used. So, pass false for + * both. + */ + virtio_net_tx_offload(ifp, &mhead, vh, false, + false); if (unlikely(!mhead)) { /* Packet dropped because errors * occurred while preparing the vnet @@ -1842,7 +1851,14 @@ host_sync: } } - if (unlikely(have_vnet_hdr && virtio_net_rx_csum(mhead, vh))) { + /* + * Translate the VirtIO header flags to the corresponding + * CSUM_* flags in the mbuf. + */ + if (unlikely(have_vnet_hdr && + ((vh->flags & (VIRTIO_NET_HDR_F_NEEDS_CSUM | + VIRTIO_NET_HDR_F_DATA_VALID)) != 0) && + (virtio_net_rx_csum(mhead, vh) != 0))) { m_freem(mhead); nm_prlim(1, "Csum offload error: dropping"); pq->stats.iqdrops ++; diff --git a/sys/dev/virtio/network/if_vtnet.c b/sys/dev/virtio/network/if_vtnet.c index e9683698decb..53f5bba146fc 100644 --- a/sys/dev/virtio/network/if_vtnet.c +++ b/sys/dev/virtio/network/if_vtnet.c @@ -145,14 +145,6 @@ static int vtnet_rxq_replace_lro_nomrg_buf(struct vtnet_rxq *, static int vtnet_rxq_replace_buf(struct vtnet_rxq *, struct mbuf *, int); static int vtnet_rxq_enqueue_buf(struct vtnet_rxq *, struct mbuf *); static int vtnet_rxq_new_buf(struct vtnet_rxq *); -#if defined(INET) || defined(INET6) -static void vtnet_rxq_csum_needs_csum(struct vtnet_rxq *, struct mbuf *, - bool, int, struct virtio_net_hdr *); -static void vtnet_rxq_csum_data_valid(struct vtnet_rxq *, struct mbuf *, - int); -static int vtnet_rxq_csum(struct vtnet_rxq *, struct mbuf *, - struct virtio_net_hdr *); -#endif static void vtnet_rxq_discard_merged_bufs(struct vtnet_rxq *, int); static void vtnet_rxq_discard_buf(struct vtnet_rxq *, struct mbuf *); static int vtnet_rxq_merged_eof(struct vtnet_rxq *, struct mbuf *, int); @@ -167,13 +159,6 @@ static int vtnet_txq_intr_threshold(struct vtnet_txq *); static int vtnet_txq_below_threshold(struct vtnet_txq *); static int vtnet_txq_notify(struct vtnet_txq *); static void vtnet_txq_free_mbufs(struct vtnet_txq *); -static int vtnet_txq_offload_ctx(struct vtnet_txq *, struct mbuf *, - int *, int *, int *); -static int vtnet_txq_offload_tso(struct vtnet_txq *, struct mbuf *, int, - int, struct virtio_net_hdr *); -static struct mbuf * - vtnet_txq_offload(struct vtnet_txq *, struct mbuf *, - struct virtio_net_hdr *); static int vtnet_txq_enqueue_buf(struct vtnet_txq *, struct mbuf **, struct vtnet_tx_header *); static int vtnet_txq_encap(struct vtnet_txq *, struct mbuf **, int); @@ -1964,124 +1949,6 @@ vtnet_rxq_new_buf(struct vtnet_rxq *rxq) return (error); } -#if defined(INET) || defined(INET6) -static void -vtnet_rxq_csum_needs_csum(struct vtnet_rxq *rxq, struct mbuf *m, bool isipv6, - int protocol, struct virtio_net_hdr *hdr) -{ - /* - * The packet is likely from another VM on the same host or from the - * host that itself performed checksum offloading so Tx/Rx is basically - * a memcpy and the checksum has little value so far. - */ - - KASSERT(protocol == IPPROTO_TCP || protocol == IPPROTO_UDP, - ("%s: unsupported IP protocol %d", __func__, protocol)); - - /* - * Just forward the order to compute the checksum by setting - * the corresponding mbuf flag (e.g., CSUM_TCP). - */ - switch (protocol) { - case IPPROTO_TCP: - m->m_pkthdr.csum_flags |= (isipv6 ? CSUM_TCP_IPV6 : CSUM_TCP); - break; - case IPPROTO_UDP: - m->m_pkthdr.csum_flags |= (isipv6 ? CSUM_UDP_IPV6 : CSUM_UDP); - break; - } - m->m_pkthdr.csum_data = hdr->csum_offset; -} - -static void -vtnet_rxq_csum_data_valid(struct vtnet_rxq *rxq, struct mbuf *m, int protocol) -{ - KASSERT(protocol == IPPROTO_TCP || protocol == IPPROTO_UDP, - ("%s: unsupported IP protocol %d", __func__, protocol)); - - m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR; - m->m_pkthdr.csum_data = 0xFFFF; -} - -static int -vtnet_rxq_csum(struct vtnet_rxq *rxq, struct mbuf *m, - struct virtio_net_hdr *hdr) -{ - const struct ether_header *eh; - struct vtnet_softc *sc; - int hoff, protocol; - uint16_t etype; - bool isipv6; - - KASSERT(hdr->flags & - (VIRTIO_NET_HDR_F_NEEDS_CSUM | VIRTIO_NET_HDR_F_DATA_VALID), - ("%s: missing checksum offloading flag %x", __func__, hdr->flags)); - - eh = mtod(m, const struct ether_header *); - etype = ntohs(eh->ether_type); - if (etype == ETHERTYPE_VLAN) { - /* TODO BMV: Handle QinQ. */ - const struct ether_vlan_header *evh = - mtod(m, const struct ether_vlan_header *); - etype = ntohs(evh->evl_proto); - hoff = sizeof(struct ether_vlan_header); - } else - hoff = sizeof(struct ether_header); - - sc = rxq->vtnrx_sc; - - /* Check whether ethernet type is IP or IPv6, and get protocol. */ - switch (etype) { -#if defined(INET) - case ETHERTYPE_IP: - if (__predict_false(m->m_len < hoff + sizeof(struct ip))) { - sc->vtnet_stats.rx_csum_inaccessible_ipproto++; - return (1); - } else { - struct ip *ip = (struct ip *)(m->m_data + hoff); - protocol = ip->ip_p; - } - isipv6 = false; - break; -#endif -#if defined(INET6) - case ETHERTYPE_IPV6: - if (__predict_false(m->m_len < hoff + sizeof(struct ip6_hdr)) - || ip6_lasthdr(m, hoff, IPPROTO_IPV6, &protocol) < 0) { - sc->vtnet_stats.rx_csum_inaccessible_ipproto++; - return (1); - } - isipv6 = true; - break; -#endif - default: - sc->vtnet_stats.rx_csum_bad_ethtype++; - return (1); - } - - /* Check whether protocol is TCP or UDP. */ - switch (protocol) { - case IPPROTO_TCP: - case IPPROTO_UDP: - break; - default: - /* - * FreeBSD does not support checksum offloading of this - * protocol here. - */ - sc->vtnet_stats.rx_csum_bad_ipproto++; - return (1); - } - - if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) - vtnet_rxq_csum_needs_csum(rxq, m, isipv6, protocol, hdr); - else /* VIRTIO_NET_HDR_F_DATA_VALID */ - vtnet_rxq_csum_data_valid(rxq, m, protocol); - - return (0); -} -#endif - static void vtnet_rxq_discard_merged_bufs(struct vtnet_rxq *rxq, int nbufs) { @@ -2220,10 +2087,29 @@ vtnet_rxq_input(struct vtnet_rxq *rxq, struct mbuf *m, if (hdr->flags & (VIRTIO_NET_HDR_F_NEEDS_CSUM | VIRTIO_NET_HDR_F_DATA_VALID)) { #if defined(INET) || defined(INET6) - if (vtnet_rxq_csum(rxq, m, hdr) == 0) + int ret; + + /* + * Translate the VirtIO header flags to the corresponding + * CSUM_* flags in the mbuf. + */ + ret = virtio_net_rx_csum(m, hdr); + if (ret == 0) rxq->vtnrx_stats.vrxs_csum++; - else + else { + switch (ret) { + case VIRTIO_NET_RX_CSUM_INACCESSIBLE_IPPROTO: + sc->vtnet_stats.rx_csum_inaccessible_ipproto++; + break; + case VIRTIO_NET_RX_CSUM_BAD_ETHTYPE: + sc->vtnet_stats.rx_csum_bad_ethtype++; + break; + case VIRTIO_NET_RX_CSUM_BAD_IPPROTO: + sc->vtnet_stats.rx_csum_bad_ipproto++; + break; + } rxq->vtnrx_stats.vrxs_csum_failed++; + } #else sc->vtnet_stats.rx_csum_bad_ethtype++; rxq->vtnrx_stats.vrxs_csum_failed++; @@ -2587,166 +2473,6 @@ vtnet_txq_free_mbufs(struct vtnet_txq *txq) ("%s: mbufs remaining in tx queue %p", __func__, txq)); } -/* - * BMV: This can go away once we finally have offsets in the mbuf header. - */ -static int -vtnet_txq_offload_ctx(struct vtnet_txq *txq, struct mbuf *m, int *etype, - int *proto, int *start) -{ - struct vtnet_softc *sc; - struct ether_vlan_header *evh; -#if defined(INET) || defined(INET6) - int offset; -#endif - - sc = txq->vtntx_sc; - - evh = mtod(m, struct ether_vlan_header *); - if (evh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { - /* BMV: We should handle nested VLAN tags too. */ - *etype = ntohs(evh->evl_proto); -#if defined(INET) || defined(INET6) - offset = sizeof(struct ether_vlan_header); -#endif - } else { - *etype = ntohs(evh->evl_encap_proto); -#if defined(INET) || defined(INET6) - offset = sizeof(struct ether_header); -#endif - } - - switch (*etype) { -#if defined(INET) - case ETHERTYPE_IP: { - struct ip *ip, iphdr; - if (__predict_false(m->m_len < offset + sizeof(struct ip))) { - m_copydata(m, offset, sizeof(struct ip), - (caddr_t) &iphdr); - ip = &iphdr; - } else - ip = (struct ip *)(m->m_data + offset); - *proto = ip->ip_p; - *start = offset + (ip->ip_hl << 2); - break; - } -#endif -#if defined(INET6) - case ETHERTYPE_IPV6: - *proto = -1; - *start = ip6_lasthdr(m, offset, IPPROTO_IPV6, proto); - /* Assert the network stack sent us a valid packet. */ - KASSERT(*start > offset, - ("%s: mbuf %p start %d offset %d proto %d", __func__, m, - *start, offset, *proto)); - break; -#endif - default: - sc->vtnet_stats.tx_csum_unknown_ethtype++; - return (EINVAL); - } - - return (0); -} - -static int -vtnet_txq_offload_tso(struct vtnet_txq *txq, struct mbuf *m, int eth_type, - int offset, struct virtio_net_hdr *hdr) -{ - static struct timeval lastecn; - static int curecn; - struct vtnet_softc *sc; - struct tcphdr *tcp, tcphdr; - - sc = txq->vtntx_sc; - - if (__predict_false(m->m_len < offset + sizeof(struct tcphdr))) { - m_copydata(m, offset, sizeof(struct tcphdr), (caddr_t) &tcphdr); - tcp = &tcphdr; - } else - tcp = (struct tcphdr *)(m->m_data + offset); - - hdr->hdr_len = vtnet_gtoh16(sc, offset + (tcp->th_off << 2)); - hdr->gso_size = vtnet_gtoh16(sc, m->m_pkthdr.tso_segsz); - hdr->gso_type = eth_type == ETHERTYPE_IP ? VIRTIO_NET_HDR_GSO_TCPV4 : - VIRTIO_NET_HDR_GSO_TCPV6; - - if (__predict_false(tcp_get_flags(tcp) & TH_CWR)) { - /* - * Drop if VIRTIO_NET_F_HOST_ECN was not negotiated. In - * FreeBSD, ECN support is not on a per-interface basis, - * but globally via the net.inet.tcp.ecn.enable sysctl - * knob. The default is off. - */ - if ((sc->vtnet_flags & VTNET_FLAG_TSO_ECN) == 0) { - if (ppsratecheck(&lastecn, &curecn, 1)) - if_printf(sc->vtnet_ifp, - "TSO with ECN not negotiated with host\n"); - return (ENOTSUP); - } - hdr->gso_type |= VIRTIO_NET_HDR_GSO_ECN; - } - - txq->vtntx_stats.vtxs_tso++; - - return (0); -} - -static struct mbuf * -vtnet_txq_offload(struct vtnet_txq *txq, struct mbuf *m, - struct virtio_net_hdr *hdr) -{ - struct vtnet_softc *sc; - int flags, etype, csum_start, proto, error; - - sc = txq->vtntx_sc; - flags = m->m_pkthdr.csum_flags; - - error = vtnet_txq_offload_ctx(txq, m, &etype, &proto, &csum_start); - if (error) - goto drop; - - if (flags & (VTNET_CSUM_OFFLOAD | VTNET_CSUM_OFFLOAD_IPV6)) { - /* Sanity check the parsed mbuf matches the offload flags. */ - if (__predict_false((flags & VTNET_CSUM_OFFLOAD && - etype != ETHERTYPE_IP) || (flags & VTNET_CSUM_OFFLOAD_IPV6 - && etype != ETHERTYPE_IPV6))) { - sc->vtnet_stats.tx_csum_proto_mismatch++; - goto drop; - } - - hdr->flags |= VIRTIO_NET_HDR_F_NEEDS_CSUM; - hdr->csum_start = vtnet_gtoh16(sc, csum_start); - hdr->csum_offset = vtnet_gtoh16(sc, m->m_pkthdr.csum_data); - txq->vtntx_stats.vtxs_csum++; - } - - if (flags & (CSUM_IP_TSO | CSUM_IP6_TSO)) { - /* - * Sanity check the parsed mbuf IP protocol is TCP, and - * VirtIO TSO reqires the checksum offloading above. - */ - if (__predict_false(proto != IPPROTO_TCP)) { - sc->vtnet_stats.tx_tso_not_tcp++; - goto drop; - } else if (__predict_false((hdr->flags & - VIRTIO_NET_HDR_F_NEEDS_CSUM) == 0)) { - sc->vtnet_stats.tx_tso_without_csum++; - goto drop; - } - - error = vtnet_txq_offload_tso(txq, m, etype, csum_start, hdr); - if (error) - goto drop; - } - - return (m); - -drop: - m_freem(m); - return (NULL); -} - static void vtnet_txq_enqueue_callback(void *arg, bus_dma_segment_t *segs, int nsegs, int error) @@ -2888,7 +2614,6 @@ static int vtnet_txq_encap(struct vtnet_txq *txq, struct mbuf **m_head, int flags) { struct vtnet_tx_header *txhdr; - struct virtio_net_hdr *hdr; struct mbuf *m; int error; @@ -2902,13 +2627,6 @@ vtnet_txq_encap(struct vtnet_txq *txq, struct mbuf **m_head, int flags) return (ENOMEM); } - /* - * Always use the non-mergeable header, regardless if mergable headers - * were negotiated, because for transmit num_buffers is always zero. - * The vtnet_hdr_size is used to enqueue the right header size segment. - */ - hdr = &txhdr->vth_uhdr.hdr; - if (m->m_flags & M_VLANTAG) { m = ether_vlanencap(m, m->m_pkthdr.ether_vtag); if ((*m_head = m) == NULL) { @@ -2919,11 +2637,51 @@ vtnet_txq_encap(struct vtnet_txq *txq, struct mbuf **m_head, int flags) } if (m->m_pkthdr.csum_flags & VTNET_CSUM_ALL_OFFLOAD) { - m = vtnet_txq_offload(txq, m, hdr); +#if defined(INET) || defined(INET6) + struct virtio_net_hdr *hdr; + int ret; + + /* + * Always use the non-mergeable header, regardless if mergable + * headers were negotiated, because for transmit num_buffers is + * always zero. The vtnet_hdr_size is used to enqueue the right + * header size segment. + */ + hdr = &txhdr->vth_uhdr.hdr; + + /* + * Translate the CSUM_* flags in the mbuf to the corresponding + * flags in the VirtIO header. + */ + ret = virtio_net_tx_offload(txq->vtntx_sc->vtnet_ifp, &m, hdr, + (txq->vtntx_sc->vtnet_flags & VTNET_FLAG_TSO_ECN), + vtnet_modern(txq->vtntx_sc)); + switch (ret) { + case VIRTIO_NET_TX_OFFLOAD_UNKNOWN_ETHTYPE: + txq->vtntx_sc->vtnet_stats.tx_csum_unknown_ethtype++; + break; + case VIRTIO_NET_TX_OFFLOAD_PROTO_MISMATCH: + txq->vtntx_sc->vtnet_stats.tx_csum_proto_mismatch++; + break; + case VIRTIO_NET_TX_OFFLOAD_TSO_NOT_TCP: + txq->vtntx_sc->vtnet_stats.tx_tso_not_tcp++; + break; + case VIRTIO_NET_TX_OFFLOAD_TSO_WITHOUT_CSUM: + txq->vtntx_sc->vtnet_stats.tx_tso_without_csum++; + break; + } if ((*m_head = m) == NULL) { error = ENOBUFS; goto fail; } + if (m->m_pkthdr.csum_flags & + (VTNET_CSUM_OFFLOAD | VTNET_CSUM_OFFLOAD_IPV6)) + txq->vtntx_stats.vtxs_csum++; + if (m->m_pkthdr.csum_flags & (CSUM_IP_TSO | CSUM_IP6_TSO)) + txq->vtntx_stats.vtxs_tso++; +#else + panic("INET/INET6 csum_flags set without INET/INET6 support"); +#endif /* defined(INET) || defined(INET6) */ } error = vtnet_txq_enqueue_buf(txq, m_head, txhdr); diff --git a/sys/dev/virtio/network/virtio_net.h b/sys/dev/virtio/network/virtio_net.h index f4256664fba8..36401109f301 100644 --- a/sys/dev/virtio/network/virtio_net.h +++ b/sys/dev/virtio/network/virtio_net.h @@ -31,6 +31,11 @@ #ifndef _VIRTIO_NET_H #define _VIRTIO_NET_H +#include "opt_inet.h" +#include "opt_inet6.h" + +#include <sys/endian.h> + /* The feature bitmap for virtio net */ #define VIRTIO_NET_F_CSUM (1ULL << 0) /* Host handles pkts w/ partial csum */ #define VIRTIO_NET_F_GUEST_CSUM (1ULL << 1) /* Guest handles pkts w/ partial csum*/ @@ -260,161 +265,149 @@ struct virtio_net_ctrl_mq { #define VIRTIO_NET_CTRL_GUEST_OFFLOADS 5 #define VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET 0 -/* - * Use the checksum offset in the VirtIO header to set the - * correct CSUM_* flags. - */ -static inline int -virtio_net_rx_csum_by_offset(struct mbuf *m, uint16_t eth_type, int ip_start, - struct virtio_net_hdr *hdr) -{ #if defined(INET) || defined(INET6) - int offset = hdr->csum_start + hdr->csum_offset; -#endif +static inline void +virtio_net_rx_csum_needs_csum(struct mbuf *m, bool isipv6, int protocol, + struct virtio_net_hdr *hdr) +{ + /* + * The packet is likely from another VM on the same host or from the + * host that itself performed checksum offloading so Tx/Rx is basically + * a memcpy and the checksum has little value so far. + */ - /* Only do a basic sanity check on the offset. */ - switch (eth_type) { -#if defined(INET) - case ETHERTYPE_IP: - if (__predict_false(offset < ip_start + sizeof(struct ip))) - return (1); - break; -#endif -#if defined(INET6) - case ETHERTYPE_IPV6: - if (__predict_false(offset < ip_start + sizeof(struct ip6_hdr))) - return (1); - break; -#endif - default: - /* Here we should increment the rx_csum_bad_ethtype counter. */ - return (1); - } + KASSERT(protocol == IPPROTO_TCP || protocol == IPPROTO_UDP, + ("%s: unsupported IP protocol %d", __func__, protocol)); /* - * Use the offset to determine the appropriate CSUM_* flags. This is - * a bit dirty, but we can get by with it since the checksum offsets - * happen to be different. We assume the host host does not do IPv4 - * header checksum offloading. + * Just forward the order to compute the checksum by setting + * the corresponding mbuf flag (e.g., CSUM_TCP). */ - switch (hdr->csum_offset) { - case offsetof(struct udphdr, uh_sum): - case offsetof(struct tcphdr, th_sum): - m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR; - m->m_pkthdr.csum_data = 0xFFFF; + switch (protocol) { + case IPPROTO_TCP: + m->m_pkthdr.csum_flags |= (isipv6 ? CSUM_TCP_IPV6 : CSUM_TCP); + break; + case IPPROTO_UDP: + m->m_pkthdr.csum_flags |= (isipv6 ? CSUM_UDP_IPV6 : CSUM_UDP); break; - default: - /* Here we should increment the rx_csum_bad_offset counter. */ - return (1); } + m->m_pkthdr.csum_data = hdr->csum_offset; +} - return (0); +static inline void +virtio_net_rx_csum_data_valid(struct mbuf *m, int protocol) +{ + KASSERT(protocol == IPPROTO_TCP || protocol == IPPROTO_UDP, + ("%s: unsupported IP protocol %d", __func__, protocol)); + + m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR; + m->m_pkthdr.csum_data = 0xFFFF; } +#define VIRTIO_NET_RX_CSUM_INACCESSIBLE_IPPROTO 1 +#define VIRTIO_NET_RX_CSUM_BAD_ETHTYPE 2 +#define VIRTIO_NET_RX_CSUM_BAD_IPPROTO 3 + +/* + * For a packet received over the VirtIO channel, it checks the given + * VirtIO header and sets the appropriate CSUM_* flags in the given mbuf. + * + * Unfortunately, the information provided is not directly useful to us. The + * VirtIO header gives the offset of the checksum, which is all Linux needs, but + * this is not how FreeBSD does things. We are forced to peek inside the packet + * a bit. + * + * It would be nice if VirtIO gave us the L4 protocol or if FreeBSD + * could accept the offsets and let the stack figure it out. + * + * @param m mbuf of the packet where CSUM_* flags might need to be set. + * @param hdr VirtIO header of the received packet that needs to be checked + * with its field values stored in the byte order this machine + * uses (i.e., readable without a byte swap). + * + * @return 0 on success, or one of the VIRTIO_NET_RX_CSUM_* error codes. + */ static inline int -virtio_net_rx_csum_by_parse(struct mbuf *m, uint16_t eth_type, int ip_start, - struct virtio_net_hdr *hdr) +virtio_net_rx_csum(struct mbuf *m, struct virtio_net_hdr *hdr) { - int offset, proto; + const struct ether_header *eh; + int hoff, protocol; + uint16_t etype; + bool isipv6; + + KASSERT(hdr->flags & + (VIRTIO_NET_HDR_F_NEEDS_CSUM | VIRTIO_NET_HDR_F_DATA_VALID), + ("%s: missing checksum offloading flag %x", __func__, hdr->flags)); + + eh = mtod(m, const struct ether_header *); + etype = ntohs(eh->ether_type); + if (etype == ETHERTYPE_VLAN) { + /* TODO BMV: Handle QinQ. */ + const struct ether_vlan_header *evh = + mtod(m, const struct ether_vlan_header *); + etype = ntohs(evh->evl_proto); + hoff = sizeof(struct ether_vlan_header); + } else + hoff = sizeof(struct ether_header); - switch (eth_type) { + /* Check whether ethernet type is IP or IPv6, and get protocol. */ + switch (etype) { #if defined(INET) - case ETHERTYPE_IP: { - struct ip *ip; - if (__predict_false(m->m_len < ip_start + sizeof(struct ip))) - return (1); - ip = (struct ip *)(m->m_data + ip_start); - proto = ip->ip_p; - offset = ip_start + (ip->ip_hl << 2); + case ETHERTYPE_IP: + if (__predict_false(m->m_len < hoff + sizeof(struct ip))) { + return (VIRTIO_NET_RX_CSUM_INACCESSIBLE_IPPROTO); + } else { + struct ip *ip = (struct ip *)(m->m_data + hoff); + protocol = ip->ip_p; + } + isipv6 = false; break; - } #endif #if defined(INET6) case ETHERTYPE_IPV6: - if (__predict_false(m->m_len < ip_start + - sizeof(struct ip6_hdr))) - return (1); - offset = ip6_lasthdr(m, ip_start, IPPROTO_IPV6, &proto); - if (__predict_false(offset < 0)) - return (1); + if (__predict_false(m->m_len < hoff + sizeof(struct ip6_hdr)) + || ip6_lasthdr(m, hoff, IPPROTO_IPV6, &protocol) < 0) { + return (VIRTIO_NET_RX_CSUM_INACCESSIBLE_IPPROTO); + } + isipv6 = true; break; #endif default: - /* Here we should increment the rx_csum_bad_ethtype counter. */ - return (1); + return (VIRTIO_NET_RX_CSUM_BAD_ETHTYPE); } - switch (proto) { + /* Check whether protocol is TCP or UDP. */ + switch (protocol) { case IPPROTO_TCP: - if (__predict_false(m->m_len < offset + sizeof(struct tcphdr))) - return (1); - m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR; - m->m_pkthdr.csum_data = 0xFFFF; - break; case IPPROTO_UDP: - if (__predict_false(m->m_len < offset + sizeof(struct udphdr))) - return (1); - m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR; - m->m_pkthdr.csum_data = 0xFFFF; break; default: /* - * For the remaining protocols, FreeBSD does not support - * checksum offloading, so the checksum will be recomputed. + * FreeBSD does not support checksum offloading of this + * protocol here. */ -#if 0 - if_printf(ifp, "cksum offload of unsupported " - "protocol eth_type=%#x proto=%d csum_start=%d " - "csum_offset=%d\n", __func__, eth_type, proto, - hdr->csum_start, hdr->csum_offset); -#endif - break; + return (VIRTIO_NET_RX_CSUM_BAD_IPPROTO); } + if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) + virtio_net_rx_csum_needs_csum(m, isipv6, protocol, hdr); + else /* VIRTIO_NET_HDR_F_DATA_VALID */ + virtio_net_rx_csum_data_valid(m, protocol); + return (0); } -/* - * Set the appropriate CSUM_* flags. Unfortunately, the information - * provided is not directly useful to us. The VirtIO header gives the - * offset of the checksum, which is all Linux needs, but this is not - * how FreeBSD does things. We are forced to peek inside the packet - * a bit. - * - * It would be nice if VirtIO gave us the L4 protocol or if FreeBSD - * could accept the offsets and let the stack figure it out. - */ -static inline int -virtio_net_rx_csum(struct mbuf *m, struct virtio_net_hdr *hdr) -{ - struct ether_header *eh; - struct ether_vlan_header *evh; - uint16_t eth_type; - int offset, error; +#define VIRTIO_NET_TX_OFFLOAD_UNKNOWN_ETHTYPE 1 +#define VIRTIO_NET_TX_OFFLOAD_PROTO_MISMATCH 2 +#define VIRTIO_NET_TX_OFFLOAD_TSO_NOT_TCP 3 +#define VIRTIO_NET_TX_OFFLOAD_TSO_WITHOUT_CSUM 4 +#define VIRTIO_NET_TX_OFFLOAD_TSO_ECN_UNEXPECTED 5 - if ((hdr->flags & (VIRTIO_NET_HDR_F_NEEDS_CSUM | - VIRTIO_NET_HDR_F_DATA_VALID)) == 0) { - return (0); - } - - eh = mtod(m, struct ether_header *); - eth_type = ntohs(eh->ether_type); - if (eth_type == ETHERTYPE_VLAN) { - /* BMV: We should handle nested VLAN tags too. */ - evh = mtod(m, struct ether_vlan_header *); - eth_type = ntohs(evh->evl_proto); - offset = sizeof(struct ether_vlan_header); - } else - offset = sizeof(struct ether_header); - - if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) - error = virtio_net_rx_csum_by_offset(m, eth_type, offset, hdr); - else - error = virtio_net_rx_csum_by_parse(m, eth_type, offset, hdr); - - return (error); -} +#define VIRTIO_NET_TX_MODERN_LE(modern, val) (modern ? htole16(val) : val) +/* + * BMV: This can go away once we finally have offsets in the mbuf header. + */ static inline int virtio_net_tx_offload_ctx(struct mbuf *m, int *etype, int *proto, int *start) { @@ -463,16 +456,15 @@ virtio_net_tx_offload_ctx(struct mbuf *m, int *etype, int *proto, int *start) break; #endif default: - /* Here we should increment the tx_csum_bad_ethtype counter. */ - return (EINVAL); + return (VIRTIO_NET_TX_OFFLOAD_UNKNOWN_ETHTYPE); } return (0); } static inline int -virtio_net_tx_offload_tso(if_t ifp, struct mbuf *m, int eth_type, - int offset, bool allow_ecn, struct virtio_net_hdr *hdr) +virtio_net_tx_offload_tso(struct ifnet *ifp, struct mbuf *m, int eth_type, + int offset, struct virtio_net_hdr *hdr, bool tso_ecn, bool modern) { static struct timeval lastecn; static int curecn; @@ -484,79 +476,114 @@ virtio_net_tx_offload_tso(if_t ifp, struct mbuf *m, int eth_type, } else tcp = (struct tcphdr *)(m->m_data + offset); - hdr->hdr_len = offset + (tcp->th_off << 2); - hdr->gso_size = m->m_pkthdr.tso_segsz; + /* + * Set VirtIO header fields with the correct byte order. + * In modern mode, this is little endian (LE). + * In legacy mode, this is the endianness of the guest, which means a + * FreeBSD guest can use its native endianness and a host must use the + * guests endianness. However, since a FreeBSD host with bhyve runs only + * on LE systems and supports only LE guests, no conversion is required. + */ + hdr->hdr_len = VIRTIO_NET_TX_MODERN_LE(modern, + offset + (tcp->th_off << 2)); + hdr->gso_size = VIRTIO_NET_TX_MODERN_LE(modern, m->m_pkthdr.tso_segsz); hdr->gso_type = eth_type == ETHERTYPE_IP ? VIRTIO_NET_HDR_GSO_TCPV4 : VIRTIO_NET_HDR_GSO_TCPV6; - if (tcp_get_flags(tcp) & TH_CWR) { + if (__predict_false(tcp_get_flags(tcp) & TH_CWR)) { /* - * Drop if VIRTIO_NET_F_HOST_ECN was not negotiated. In FreeBSD, - * ECN support is not on a per-interface basis, but globally via - * the net.inet.tcp.ecn.enable sysctl knob. The default is off. + * Drop if VIRTIO_NET_F_HOST_ECN was not negotiated. In + * FreeBSD, ECN support is not on a per-interface basis, + * but globally via the net.inet.tcp.ecn.enable sysctl + * knob. The default is off. */ - if (!allow_ecn) { + if (!tso_ecn) { if (ppsratecheck(&lastecn, &curecn, 1)) if_printf(ifp, "TSO with ECN not negotiated with host\n"); - return (ENOTSUP); + return (VIRTIO_NET_TX_OFFLOAD_TSO_ECN_UNEXPECTED); } hdr->gso_type |= VIRTIO_NET_HDR_GSO_ECN; } - /* Here we should increment tx_tso counter. */ - return (0); } -static inline struct mbuf * -virtio_net_tx_offload(if_t ifp, struct mbuf *m, bool allow_ecn, - struct virtio_net_hdr *hdr) +/* + * For a packet to be transmitted over the VirtIO channel, it checks the + * CSUM_* flags in the mbuf and sets the appropriate flags in the VirtIO header. + * In case of an error, it frees the mbuf and sets the pointer referenced by mp + * to NULL. + * + * @param ifp ifnet struct of outgoing interface. + * @param mp mbuf on which the CSUM_* flags needs to be checked. + * @param hdr VirtIO header to be filled for the outgoing packet. + * @param tso_ecn true if ECN has been negotiated between host and guest. + * @param modern true if VirtIO modern mode is used. + * + * @return 0 on success, or one of the VIRTIO_NET_TX_OFFLOAD_* error codes. + */ +static inline int +virtio_net_tx_offload(struct ifnet *ifp, struct mbuf **mp, + struct virtio_net_hdr *hdr, bool tso_ecn, bool modern) { int flags, etype, csum_start, proto, error; + struct mbuf *m; + m = *mp; flags = m->m_pkthdr.csum_flags; error = virtio_net_tx_offload_ctx(m, &etype, &proto, &csum_start); - if (error) + if (error != 0) goto drop; - if ((etype == ETHERTYPE_IP && (flags & (CSUM_TCP | CSUM_UDP))) || - (etype == ETHERTYPE_IPV6 && - (flags & (CSUM_TCP_IPV6 | CSUM_UDP_IPV6)))) { + if (flags & (CSUM_TCP | CSUM_UDP | CSUM_TCP_IPV6 | CSUM_UDP_IPV6)) { + /* Sanity check the parsed mbuf matches the offload flags. */ + if (__predict_false((flags & (CSUM_TCP | CSUM_UDP) && + etype != ETHERTYPE_IP) || + (flags & (CSUM_TCP_IPV6 | CSUM_UDP_IPV6) && + etype != ETHERTYPE_IPV6))) { + error = VIRTIO_NET_TX_OFFLOAD_PROTO_MISMATCH; + goto drop; + } + /* - * We could compare the IP protocol vs the CSUM_ flag too, - * but that really should not be necessary. + * Set VirtIO header fields with the correct byte order. + * See comment in virtio_net_tx_offload_tso() */ hdr->flags |= VIRTIO_NET_HDR_F_NEEDS_CSUM; - hdr->csum_start = csum_start; - hdr->csum_offset = m->m_pkthdr.csum_data; - /* Here we should increment the tx_csum counter. */ + hdr->csum_start = VIRTIO_NET_TX_MODERN_LE(modern, csum_start); + hdr->csum_offset = VIRTIO_NET_TX_MODERN_LE(modern, + m->m_pkthdr.csum_data); } - if (flags & CSUM_TSO) { + if (flags & (CSUM_IP_TSO | CSUM_IP6_TSO)) { + /* + * Sanity check the parsed mbuf IP protocol is TCP, and + * VirtIO TSO reqires the checksum offloading above. + */ if (__predict_false(proto != IPPROTO_TCP)) { - /* Likely failed to correctly parse the mbuf. - * Here we should increment the tx_tso_not_tcp - * counter. */ + error = VIRTIO_NET_TX_OFFLOAD_TSO_NOT_TCP; + goto drop; + } else if (__predict_false((hdr->flags & + VIRTIO_NET_HDR_F_NEEDS_CSUM) == 0)) { + error = VIRTIO_NET_TX_OFFLOAD_TSO_WITHOUT_CSUM; goto drop; } - KASSERT(hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM, - ("%s: mbuf %p TSO without checksum offload %#x", - __func__, m, flags)); - error = virtio_net_tx_offload_tso(ifp, m, etype, csum_start, - allow_ecn, hdr); - if (error) + hdr, tso_ecn, modern); + if (error != 0) goto drop; } - return (m); + return (error); drop: m_freem(m); - return (NULL); + *mp = NULL; + return (error); } +#endif /* defined(INET) || defined(INET6) */ #endif /* _VIRTIO_NET_H */ diff --git a/sys/net/if_tuntap.c b/sys/net/if_tuntap.c index a1ef2541ffba..b1ef08783c72 100644 --- a/sys/net/if_tuntap.c +++ b/sys/net/if_tuntap.c @@ -1881,10 +1881,21 @@ tunread(struct cdev *dev, struct uio *uio, int flag) struct virtio_net_hdr_mrg_rxbuf vhdr; bzero(&vhdr, sizeof(vhdr)); +#if defined(INET) || defined(INET6) if (m->m_pkthdr.csum_flags & TAP_ALL_OFFLOAD) { - m = virtio_net_tx_offload(ifp, m, false, &vhdr.hdr); - } + /* + * Translate the CSUM_* flags in the mbuf to the + * corresponding flags in the VirtIO header. + * + * Always indicate that ECN has not been negotiated + * and VirtIO modern mode is not used because bhyve + * does not do this. + */ + virtio_net_tx_offload(ifp, &m, &vhdr.hdr, false, + false); + } +#endif /* defined(INET) || defined(INET6) */ *** 23 LINES SKIPPED ***
