The branch main has been updated by kib: URL: https://cgit.FreeBSD.org/src/commit/?id=7c9febf9f11bec467ef79c6ff19d695ace7edb6b
commit 7c9febf9f11bec467ef79c6ff19d695ace7edb6b Author: Konstantin Belousov <[email protected]> AuthorDate: 2021-04-06 03:45:20 +0000 Commit: Konstantin Belousov <[email protected]> CommitDate: 2021-07-12 09:34:38 +0000 mlx5en: handle checksum and TSO offloading for VxLAN packets on TX Reviewed by: hselasky Sponsored by: Mellanox Technologies/NVidia Networking MFC after: 1 week --- sys/dev/mlx5/mlx5_en/mlx5_en_tx.c | 289 +++++++++++++++++++++++++++++++++++++- 1 file changed, 287 insertions(+), 2 deletions(-) diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c b/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c index 1ba47e44ae96..6bc0a91ca8ef 100644 --- a/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c +++ b/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c @@ -332,6 +332,221 @@ failure: return (0); } +/* + * Locate a pointer inside a mbuf chain. Returns NULL upon failure. + */ +static inline void * +mlx5e_parse_mbuf_chain(const struct mbuf **mb, int *poffset, int eth_hdr_len, + int min_len) +{ + if (unlikely(mb[0]->m_len == eth_hdr_len)) { + poffset[0] = eth_hdr_len; + if (unlikely((mb[0] = mb[0]->m_next) == NULL)) + return (NULL); + } + if (unlikely(mb[0]->m_len < eth_hdr_len - poffset[0] + min_len)) + return (NULL); + return (mb[0]->m_data + eth_hdr_len - poffset[0]); +} + +/* + * This function parse IPv4 and IPv6 packets looking for UDP, VXLAN + * and TCP headers. + * + * The return value indicates the number of bytes from the beginning + * of the packet until the first byte after the TCP header. If this + * function returns zero, the parsing failed. + */ +static int +mlx5e_get_vxlan_header_size(const struct mbuf *mb, struct mlx5e_tx_wqe *wqe, + uint8_t cs_mask, uint8_t opcode) +{ + const struct ether_vlan_header *eh; + struct ip *ip4; + struct ip6_hdr *ip6; + struct tcphdr *th; + struct udphdr *udp; + bool has_outer_vlan_tag; + uint16_t eth_type; + uint8_t ip_type; + int pkt_hdr_len; + int eth_hdr_len; + int tcp_hlen; + int ip_hlen; + int offset; + + pkt_hdr_len = mb->m_pkthdr.len; + has_outer_vlan_tag = (mb->m_flags & M_VLANTAG) != 0; + offset = 0; + + eh = mtod(mb, const struct ether_vlan_header *); + if (unlikely(mb->m_len < ETHER_HDR_LEN)) + return (0); + + if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { + if (unlikely(mb->m_len < ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN)) + return (0); + eth_type = eh->evl_proto; + eth_hdr_len = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; + } else { + eth_type = eh->evl_encap_proto; + eth_hdr_len = ETHER_HDR_LEN; + } + + switch (eth_type) { + case htons(ETHERTYPE_IP): + ip4 = mlx5e_parse_mbuf_chain(&mb, &offset, eth_hdr_len, + sizeof(*ip4)); + if (unlikely(ip4 == NULL)) + return (0); + ip_type = ip4->ip_p; + if (unlikely(ip_type != IPPROTO_UDP)) + return (0); + wqe->eth.swp_outer_l3_offset = eth_hdr_len / 2; + wqe->eth.cs_flags = MLX5_ETH_WQE_L3_CSUM | MLX5_ETH_WQE_L4_CSUM; + ip_hlen = ip4->ip_hl << 2; + eth_hdr_len += ip_hlen; + udp = mlx5e_parse_mbuf_chain(&mb, &offset, eth_hdr_len, + sizeof(*udp)); + if (unlikely(udp == NULL)) + return (0); + wqe->eth.swp_outer_l4_offset = eth_hdr_len / 2; + wqe->eth.swp_flags |= MLX5_ETH_WQE_SWP_OUTER_L4_TYPE; + eth_hdr_len += sizeof(*udp); + break; + case htons(ETHERTYPE_IPV6): + ip6 = mlx5e_parse_mbuf_chain(&mb, &offset, eth_hdr_len, + sizeof(*ip6)); + if (unlikely(ip6 == NULL)) + return (0); + ip_type = ip6->ip6_nxt; + if (unlikely(ip_type != IPPROTO_UDP)) + return (0); + wqe->eth.swp_outer_l3_offset = eth_hdr_len / 2; + wqe->eth.cs_flags = MLX5_ETH_WQE_L3_CSUM | MLX5_ETH_WQE_L4_CSUM; + eth_hdr_len += sizeof(*ip6); + udp = mlx5e_parse_mbuf_chain(&mb, &offset, eth_hdr_len, + sizeof(*udp)); + if (unlikely(udp == NULL)) + return (0); + wqe->eth.swp_outer_l4_offset = eth_hdr_len / 2; + wqe->eth.swp_flags |= MLX5_ETH_WQE_SWP_OUTER_L4_TYPE | + MLX5_ETH_WQE_SWP_OUTER_L3_TYPE; + eth_hdr_len += sizeof(*udp); + break; + default: + return (0); + } + + /* + * If the hardware is not computing inner IP checksum, then + * skip inlining the inner outer UDP and VXLAN header: + */ + if (unlikely((cs_mask & MLX5_ETH_WQE_L3_INNER_CSUM) == 0)) + goto done; + if (unlikely(mlx5e_parse_mbuf_chain(&mb, &offset, eth_hdr_len, + 8) == NULL)) + return (0); + eth_hdr_len += 8; + + /* Check for ethernet header again. */ + eh = mlx5e_parse_mbuf_chain(&mb, &offset, eth_hdr_len, ETHER_HDR_LEN); + if (unlikely(eh == NULL)) + return (0); + if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { + if (unlikely(mb->m_len < eth_hdr_len - offset + ETHER_HDR_LEN + + ETHER_VLAN_ENCAP_LEN)) + return (0); + eth_type = eh->evl_proto; + eth_hdr_len += ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; + } else { + eth_type = eh->evl_encap_proto; + eth_hdr_len += ETHER_HDR_LEN; + } + + /* Check for IP header again. */ + switch (eth_type) { + case htons(ETHERTYPE_IP): + ip4 = mlx5e_parse_mbuf_chain(&mb, &offset, eth_hdr_len, + sizeof(*ip4)); + if (unlikely(ip4 == NULL)) + return (0); + wqe->eth.swp_inner_l3_offset = eth_hdr_len / 2; + wqe->eth.cs_flags |= MLX5_ETH_WQE_L3_INNER_CSUM; + ip_type = ip4->ip_p; + ip_hlen = ip4->ip_hl << 2; + eth_hdr_len += ip_hlen; + break; + case htons(ETHERTYPE_IPV6): + ip6 = mlx5e_parse_mbuf_chain(&mb, &offset, eth_hdr_len, + sizeof(*ip6)); + if (unlikely(ip6 == NULL)) + return (0); + wqe->eth.swp_inner_l3_offset = eth_hdr_len / 2; + wqe->eth.cs_flags |= MLX5_ETH_WQE_L3_INNER_CSUM; + wqe->eth.swp_flags |= MLX5_ETH_WQE_SWP_INNER_L3_TYPE; + ip_type = ip6->ip6_nxt; + eth_hdr_len += sizeof(*ip6); + break; + default: + return (0); + } + + /* + * If the hardware is not computing inner UDP/TCP checksum, + * then skip inlining the inner UDP/TCP header: + */ + if (unlikely((cs_mask & MLX5_ETH_WQE_L4_INNER_CSUM) == 0)) + goto done; + + switch (ip_type) { + case IPPROTO_UDP: + udp = mlx5e_parse_mbuf_chain(&mb, &offset, eth_hdr_len, + sizeof(*udp)); + if (unlikely(udp == NULL)) + return (0); + wqe->eth.swp_inner_l4_offset = (eth_hdr_len / 2); + wqe->eth.cs_flags |= MLX5_ETH_WQE_L4_INNER_CSUM; + wqe->eth.swp_flags |= MLX5_ETH_WQE_SWP_INNER_L4_TYPE; + eth_hdr_len += sizeof(*udp); + break; + case IPPROTO_TCP: + th = mlx5e_parse_mbuf_chain(&mb, &offset, eth_hdr_len, + sizeof(*th)); + if (unlikely(th == NULL)) + return (0); + wqe->eth.swp_inner_l4_offset = eth_hdr_len / 2; + wqe->eth.cs_flags |= MLX5_ETH_WQE_L4_INNER_CSUM; + wqe->eth.swp_flags |= MLX5_ETH_WQE_SWP_INNER_L4_TYPE; + tcp_hlen = th->th_off << 2; + eth_hdr_len += tcp_hlen; + break; + default: + return (0); + } +done: + if (unlikely(pkt_hdr_len < eth_hdr_len)) + return (0); + + /* Account for software inserted VLAN tag, if any. */ + if (unlikely(has_outer_vlan_tag)) { + wqe->eth.swp_outer_l3_offset += ETHER_VLAN_ENCAP_LEN / 2; + wqe->eth.swp_outer_l4_offset += ETHER_VLAN_ENCAP_LEN / 2; + wqe->eth.swp_inner_l3_offset += ETHER_VLAN_ENCAP_LEN / 2; + wqe->eth.swp_inner_l4_offset += ETHER_VLAN_ENCAP_LEN / 2; + } + + /* + * When inner checksums are set, outer L4 checksum flag must + * be disabled. + */ + if (wqe->eth.cs_flags & (MLX5_ETH_WQE_L3_INNER_CSUM | + MLX5_ETH_WQE_L4_INNER_CSUM)) + wqe->eth.cs_flags &= ~MLX5_ETH_WQE_L4_CSUM; + + return (eth_hdr_len); +} + struct mlx5_wqe_dump_seg { struct mlx5_wqe_ctrl_seg ctrl; struct mlx5_wqe_data_seg data; @@ -574,8 +789,77 @@ top: num_pkts = DIV_ROUND_UP(payload_len, mss); sq->mbuf[pi].num_bytes = payload_len + (num_pkts * args.ihs); + sq->stats.tso_packets++; sq->stats.tso_bytes += payload_len; + } else if (mb->m_pkthdr.csum_flags & CSUM_ENCAP_VXLAN) { + /* check for inner TCP TSO first */ + if (mb->m_pkthdr.csum_flags & (CSUM_INNER_IP_TSO | + CSUM_INNER_IP6_TSO)) { + u32 payload_len; + u32 mss = mb->m_pkthdr.tso_segsz; + u32 num_pkts; + + wqe->eth.mss = cpu_to_be16(mss); + opcode = MLX5_OPCODE_LSO; + + if (likely(args.ihs == 0)) { + args.ihs = mlx5e_get_vxlan_header_size(mb, wqe, + MLX5_ETH_WQE_L3_INNER_CSUM | + MLX5_ETH_WQE_L4_INNER_CSUM | + MLX5_ETH_WQE_L4_CSUM | + MLX5_ETH_WQE_L3_CSUM, + opcode); + if (unlikely(args.ihs == 0)) { + err = EINVAL; + goto tx_drop; + } + } + + payload_len = mb->m_pkthdr.len - args.ihs; + if (payload_len == 0) + num_pkts = 1; + else + num_pkts = DIV_ROUND_UP(payload_len, mss); + sq->mbuf[pi].num_bytes = payload_len + + num_pkts * args.ihs; + + sq->stats.tso_packets++; + sq->stats.tso_bytes += payload_len; + } else { + opcode = MLX5_OPCODE_SEND; + + if (likely(args.ihs == 0)) { + uint8_t cs_mask; + + if (mb->m_pkthdr.csum_flags & + (CSUM_INNER_IP_TCP | CSUM_INNER_IP_UDP)) { + cs_mask = + MLX5_ETH_WQE_L3_INNER_CSUM | + MLX5_ETH_WQE_L4_INNER_CSUM | + MLX5_ETH_WQE_L4_CSUM | + MLX5_ETH_WQE_L3_CSUM; + } else if (mb->m_pkthdr.csum_flags & CSUM_INNER_IP) { + cs_mask = + MLX5_ETH_WQE_L3_INNER_CSUM | + MLX5_ETH_WQE_L4_CSUM | + MLX5_ETH_WQE_L3_CSUM; + } else { + cs_mask = + MLX5_ETH_WQE_L4_CSUM | + MLX5_ETH_WQE_L3_CSUM; + } + args.ihs = mlx5e_get_vxlan_header_size(mb, wqe, + cs_mask, opcode); + if (unlikely(args.ihs == 0)) { + err = EINVAL; + goto tx_drop; + } + } + + sq->mbuf[pi].num_bytes = max_t (unsigned int, + mb->m_pkthdr.len, ETHER_MIN_LEN - ETHER_CRC_LEN); + } } else { opcode = MLX5_OPCODE_SEND; @@ -622,7 +906,7 @@ top: /* Range checks */ if (unlikely(args.ihs > (sq->max_inline - ETHER_VLAN_ENCAP_LEN))) { - if (mb->m_pkthdr.csum_flags & CSUM_TSO) { + if (mb->m_pkthdr.csum_flags & (CSUM_TSO | CSUM_ENCAP_VXLAN)) { err = EINVAL; goto tx_drop; } @@ -646,7 +930,8 @@ top: } else { /* check if inline header size is too big */ if (unlikely(args.ihs > sq->max_inline)) { - if (unlikely(mb->m_pkthdr.csum_flags & CSUM_TSO)) { + if (unlikely(mb->m_pkthdr.csum_flags & (CSUM_TSO | + CSUM_ENCAP_VXLAN))) { err = EINVAL; goto tx_drop; } _______________________________________________ [email protected] mailing list https://lists.freebsd.org/mailman/listinfo/dev-commits-src-all To unsubscribe, send any mail to "[email protected]"
