Author: zbb Date: Thu Feb 25 14:29:57 2016 New Revision: 296039 URL: https://svnweb.freebsd.org/changeset/base/296039
Log: Introduce HW TSO support for VNIC This feature was added in Pass2.0. Significantly improves VNIC's TCP performance on Tx. Reviewed by: wma Obtained from: Semihalf Sponsored by: Cavium Differential Revision: https://reviews.freebsd.org/D5424 Modified: head/sys/dev/vnic/nic.h head/sys/dev/vnic/nicvf_main.c head/sys/dev/vnic/nicvf_queues.c head/sys/dev/vnic/nicvf_queues.h head/sys/dev/vnic/q_struct.h Modified: head/sys/dev/vnic/nic.h ============================================================================== --- head/sys/dev/vnic/nic.h Thu Feb 25 14:28:10 2016 (r296038) +++ head/sys/dev/vnic/nic.h Thu Feb 25 14:29:57 2016 (r296039) @@ -292,6 +292,7 @@ struct nicvf { uint8_t max_queues; struct resource *reg_base; boolean_t link_up; + boolean_t hw_tso; uint8_t duplex; uint32_t speed; uint8_t cpi_alg; Modified: head/sys/dev/vnic/nicvf_main.c ============================================================================== --- head/sys/dev/vnic/nicvf_main.c Thu Feb 25 14:28:10 2016 (r296038) +++ head/sys/dev/vnic/nicvf_main.c Thu Feb 25 14:29:57 2016 (r296039) @@ -66,6 +66,7 @@ __FBSDID("$FreeBSD$"); #include <net/if_vlan_var.h> #include <netinet/in.h> +#include <netinet/ip.h> #include <netinet/if_ether.h> #include <netinet/tcp_lro.h> @@ -194,6 +195,9 @@ nicvf_attach(device_t dev) nic->pnicvf = nic; NICVF_CORE_LOCK_INIT(nic); + /* Enable HW TSO on Pass2 */ + if (!pass1_silicon(dev)) + nic->hw_tso = TRUE; rid = VNIC_VF_REG_RID; nic->reg_base = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, @@ -356,6 +360,14 @@ nicvf_setup_ifnet(struct nicvf *nic) /* Set the default values */ if_setcapabilitiesbit(ifp, IFCAP_VLAN_MTU, 0); if_setcapabilitiesbit(ifp, IFCAP_LRO, 0); + if (nic->hw_tso) { + /* TSO */ + if_setcapabilitiesbit(ifp, IFCAP_TSO4, 0); + /* TSO parameters */ + ifp->if_hw_tsomax = NICVF_TSO_MAXSIZE; + ifp->if_hw_tsomaxsegcount = NICVF_TSO_NSEGS; + ifp->if_hw_tsomaxsegsize = MCLBYTES; + } /* IP/TCP/UDP HW checksums */ if_setcapabilitiesbit(ifp, IFCAP_HWCSUM, 0); if_setcapabilitiesbit(ifp, IFCAP_HWSTATS, 0); @@ -364,7 +376,8 @@ nicvf_setup_ifnet(struct nicvf *nic) */ if_clearhwassist(ifp); if_sethwassistbits(ifp, (CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_SCTP), 0); - + if (nic->hw_tso) + if_sethwassistbits(ifp, (CSUM_TSO), 0); if_setcapenable(ifp, if_getcapabilities(ifp)); return (0); @@ -513,6 +526,8 @@ nicvf_if_ioctl(struct ifnet *ifp, u_long ifp->if_capenable ^= IFCAP_TXCSUM; if (mask & IFCAP_RXCSUM) ifp->if_capenable ^= IFCAP_RXCSUM; + if ((mask & IFCAP_TSO4) && nic->hw_tso) + ifp->if_capenable ^= IFCAP_TSO4; if (mask & IFCAP_LRO) { /* * Lock the driver for a moment to avoid Modified: head/sys/dev/vnic/nicvf_queues.c ============================================================================== --- head/sys/dev/vnic/nicvf_queues.c Thu Feb 25 14:28:10 2016 (r296038) +++ head/sys/dev/vnic/nicvf_queues.c Thu Feb 25 14:29:57 2016 (r296039) @@ -1070,8 +1070,8 @@ nicvf_init_snd_queue(struct nicvf *nic, BUS_SPACE_MAXADDR, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filtfunc, filtfuncarg */ - NICVF_TXBUF_MAXSIZE, /* maxsize */ - NICVF_TXBUF_NSEGS, /* nsegments */ + NICVF_TSO_MAXSIZE, /* maxsize */ + NICVF_TSO_NSEGS, /* nsegments */ MCLBYTES, /* maxsegsize */ 0, /* flags */ NULL, NULL, /* lockfunc, lockfuncarg */ @@ -1727,14 +1727,18 @@ static __inline int nicvf_sq_add_hdr_subdesc(struct snd_queue *sq, int qentry, int subdesc_cnt, struct mbuf *mbuf, int len) { + struct nicvf *nic; struct sq_hdr_subdesc *hdr; struct ether_vlan_header *eh; #ifdef INET struct ip *ip; + struct tcphdr *th; #endif uint16_t etype; int ehdrlen, iphlen, poff; + nic = sq->nic; + hdr = (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, qentry); sq->snd_buff[qentry].mbuf = mbuf; @@ -1746,18 +1750,25 @@ nicvf_sq_add_hdr_subdesc(struct snd_queu hdr->subdesc_cnt = subdesc_cnt; hdr->tot_len = len; - if (mbuf->m_pkthdr.csum_flags != 0) { - hdr->csum_l3 = 1; /* Enable IP csum calculation */ - - eh = mtod(mbuf, struct ether_vlan_header *); - if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { - ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; - etype = ntohs(eh->evl_proto); - } else { - ehdrlen = ETHER_HDR_LEN; - etype = ntohs(eh->evl_encap_proto); - } + eh = mtod(mbuf, struct ether_vlan_header *); + if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { + ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; + etype = ntohs(eh->evl_proto); + } else { + ehdrlen = ETHER_HDR_LEN; + etype = ntohs(eh->evl_encap_proto); + } + switch (etype) { +#ifdef INET6 + case ETHERTYPE_IPV6: + /* ARM64TODO: Add support for IPv6 */ + hdr->csum_l3 = 0; + sq->snd_buff[qentry].mbuf = NULL; + return (ENXIO); +#endif +#ifdef INET + case ETHERTYPE_IP: if (mbuf->m_len < ehdrlen + sizeof(struct ip)) { mbuf = m_pullup(mbuf, ehdrlen + sizeof(struct ip)); sq->snd_buff[qentry].mbuf = mbuf; @@ -1765,21 +1776,13 @@ nicvf_sq_add_hdr_subdesc(struct snd_queu return (ENOBUFS); } - switch (etype) { -#ifdef INET6 - case ETHERTYPE_IPV6: - /* ARM64TODO: Add support for IPv6 */ - hdr->csum_l3 = 0; - sq->snd_buff[qentry].mbuf = NULL; - return (ENXIO); -#endif -#ifdef INET - case ETHERTYPE_IP: - ip = (struct ip *)(mbuf->m_data + ehdrlen); - ip->ip_sum = 0; - iphlen = ip->ip_hl << 2; - poff = ehdrlen + iphlen; + ip = (struct ip *)(mbuf->m_data + ehdrlen); + ip->ip_sum = 0; + iphlen = ip->ip_hl << 2; + poff = ehdrlen + iphlen; + if (mbuf->m_pkthdr.csum_flags != 0) { + hdr->csum_l3 = 1; /* Enable IP csum calculation */ switch (ip->ip_p) { case IPPROTO_TCP: if ((mbuf->m_pkthdr.csum_flags & CSUM_TCP) == 0) @@ -1820,17 +1823,28 @@ nicvf_sq_add_hdr_subdesc(struct snd_queu default: break; } - break; -#endif - default: - hdr->csum_l3 = 0; - return (0); + hdr->l3_offset = ehdrlen; + hdr->l4_offset = ehdrlen + iphlen; } - hdr->l3_offset = ehdrlen; - hdr->l4_offset = ehdrlen + iphlen; - } else + if ((mbuf->m_pkthdr.tso_segsz != 0) && nic->hw_tso) { + /* + * Extract ip again as m_data could have been modified. + */ + ip = (struct ip *)(mbuf->m_data + ehdrlen); + th = (struct tcphdr *)((caddr_t)ip + iphlen); + + hdr->tso = 1; + hdr->tso_start = ehdrlen + iphlen + (th->th_off * 4); + hdr->tso_max_paysize = mbuf->m_pkthdr.tso_segsz; + hdr->inner_l3_offset = ehdrlen - 2; + nic->drv_stats.tx_tso++; + } + break; +#endif + default: hdr->csum_l3 = 0; + } return (0); } @@ -1859,10 +1873,11 @@ int nicvf_tx_mbuf_locked(struct snd_queue *sq, struct mbuf *mbuf) { bus_dma_segment_t segs[256]; + struct nicvf *nic; struct snd_buff *snd_buff; size_t seg; int nsegs, qentry; - int subdesc_cnt = MIN_SQ_DESC_PER_PKT_XMIT - 1; + int subdesc_cnt; int err; NICVF_TX_LOCK_ASSERT(sq); @@ -1880,7 +1895,11 @@ nicvf_tx_mbuf_locked(struct snd_queue *s } /* Set how many subdescriptors is required */ - subdesc_cnt += nsegs; + nic = sq->nic; + if (mbuf->m_pkthdr.tso_segsz != 0 && nic->hw_tso) + subdesc_cnt = MIN_SQ_DESC_PER_PKT_XMIT; + else + subdesc_cnt = MIN_SQ_DESC_PER_PKT_XMIT + nsegs - 1; if (subdesc_cnt > sq->free_cnt) { /* ARM64TODO: Add mbuf defragmentation if we lack descriptors */ Modified: head/sys/dev/vnic/nicvf_queues.h ============================================================================== --- head/sys/dev/vnic/nicvf_queues.h Thu Feb 25 14:28:10 2016 (r296038) +++ head/sys/dev/vnic/nicvf_queues.h Thu Feb 25 14:29:57 2016 (r296039) @@ -131,10 +131,13 @@ #define NICVF_RCV_BUF_ALIGN_LEN(addr) \ (NICVF_ALIGNED_ADDR((addr), NICVF_RCV_BUF_ALIGN_BYTES) - (addr)) -#define NICVF_TXBUF_MAXSIZE 9212 /* Total max payload without TSO */ +#define NICVF_TXBUF_MAXSIZE NIC_HW_MAX_FRS /* Total max payload without TSO */ #define NICVF_TXBUF_NSEGS 256 /* Single command is at most 256 buffers (hdr + 255 subcmds) */ - +/* TSO-related definitions */ +#define NICVF_TSO_MAXSIZE IP_MAXPACKET +#define NICVF_TSO_NSEGS NICVF_TXBUF_NSEGS +#define NICVF_TSO_HEADER_SIZE 128 /* Queue enable/disable */ #define NICVF_SQ_EN (1UL << 19) Modified: head/sys/dev/vnic/q_struct.h ============================================================================== --- head/sys/dev/vnic/q_struct.h Thu Feb 25 14:28:10 2016 (r296038) +++ head/sys/dev/vnic/q_struct.h Thu Feb 25 14:29:57 2016 (r296039) @@ -565,25 +565,28 @@ struct sq_hdr_subdesc { uint64_t subdesc_cnt:8; uint64_t csum_l4:2; uint64_t csum_l3:1; - uint64_t rsvd0:5; + uint64_t csum_inner_l4:2; + uint64_t csum_inner_l3:1; + uint64_t rsvd0:2; uint64_t l4_offset:8; uint64_t l3_offset:8; uint64_t rsvd1:4; uint64_t tot_len:20; /* W0 */ - uint64_t tso_sdc_cont:8; - uint64_t tso_sdc_first:8; - uint64_t tso_l4_offset:8; - uint64_t tso_flags_last:12; - uint64_t tso_flags_first:12; - uint64_t rsvd2:2; + uint64_t rsvd2:24; + uint64_t inner_l4_offset:8; + uint64_t inner_l3_offset:8; + uint64_t tso_start:8; + uint64_t rsvd3:2; uint64_t tso_max_paysize:14; /* W1 */ #elif defined(__LITTLE_ENDIAN_BITFIELD) uint64_t tot_len:20; uint64_t rsvd1:4; uint64_t l3_offset:8; uint64_t l4_offset:8; - uint64_t rsvd0:5; + uint64_t rsvd0:2; + uint64_t csum_inner_l3:1; + uint64_t csum_inner_l4:2; uint64_t csum_l3:1; uint64_t csum_l4:2; uint64_t subdesc_cnt:8; @@ -594,12 +597,11 @@ struct sq_hdr_subdesc { uint64_t subdesc_type:4; /* W0 */ uint64_t tso_max_paysize:14; - uint64_t rsvd2:2; - uint64_t tso_flags_first:12; - uint64_t tso_flags_last:12; - uint64_t tso_l4_offset:8; - uint64_t tso_sdc_first:8; - uint64_t tso_sdc_cont:8; /* W1 */ + uint64_t rsvd3:2; + uint64_t tso_start:8; + uint64_t inner_l3_offset:8; + uint64_t inner_l4_offset:8; + uint64_t rsvd2:24; #endif }; _______________________________________________ [email protected] mailing list https://lists.freebsd.org/mailman/listinfo/svn-src-all To unsubscribe, send any mail to "[email protected]"
