From: Scott Mitchell <[email protected]> - Add rte_prefetch0() to prefetch next frame/mbuf while processing current packet, reducing cache miss latency - Use rte_pktmbuf_free_bulk() in TX path instead of individual rte_pktmbuf_free() calls for better batch efficiency - Add unlikely() hints for error paths (oversized packets, VLAN insertion failures, sendto errors) to optimize branch prediction - Remove unnecessary early nb_pkts == 0 when loop handles this and app may never call with 0 frames.
Signed-off-by: Scott Mitchell <[email protected]> --- drivers/net/af_packet/rte_eth_af_packet.c | 65 ++++++++++++----------- 1 file changed, 34 insertions(+), 31 deletions(-) diff --git a/drivers/net/af_packet/rte_eth_af_packet.c b/drivers/net/af_packet/rte_eth_af_packet.c index 6c276bb7fc..e357ae168b 100644 --- a/drivers/net/af_packet/rte_eth_af_packet.c +++ b/drivers/net/af_packet/rte_eth_af_packet.c @@ -161,9 +161,6 @@ eth_af_packet_rx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) uint32_t tp_status; unsigned int framecount, framenum; - if (unlikely(nb_pkts == 0)) - return 0; - /* * Reads the given number of packets from the AF_PACKET socket one by * one and copies the packet data into a newly allocated mbuf. @@ -177,6 +174,14 @@ eth_af_packet_rx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) if ((tp_status & TP_STATUS_USER) == 0) break; + unsigned int next_framenum = framenum + 1; + if (next_framenum >= framecount) + next_framenum = 0; + + /* prefetch the next frame for the next loop iteration */ + if (likely(i + 1 < nb_pkts)) + rte_prefetch0(pkt_q->rd[next_framenum].iov_base); + /* allocate the next mbuf */ mbuf = rte_pktmbuf_alloc(pkt_q->mb_pool); if (unlikely(mbuf == NULL)) { @@ -210,8 +215,7 @@ eth_af_packet_rx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) /* release incoming frame and advance ring buffer */ tpacket_write_status(&ppd->tp_status, TP_STATUS_KERNEL); - if (++framenum >= framecount) - framenum = 0; + framenum = next_framenum; mbuf->port = pkt_q->in_port; /* account for the receive frame */ @@ -261,9 +265,6 @@ eth_af_packet_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) uint32_t num_tx_bytes = 0; uint16_t i; - if (unlikely(nb_pkts == 0)) - return 0; - memset(&pfd, 0, sizeof(pfd)); pfd.fd = pkt_q->sockfd; pfd.events = POLLOUT; @@ -271,22 +272,25 @@ eth_af_packet_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) framecount = pkt_q->framecount; framenum = pkt_q->framenum; - ppd = (struct tpacket2_hdr *) pkt_q->rd[framenum].iov_base; for (i = 0; i < nb_pkts; i++) { - mbuf = *bufs++; - - /* drop oversized packets */ - if (mbuf->pkt_len > pkt_q->frame_data_size) { - rte_pktmbuf_free(mbuf); - continue; + unsigned int next_framenum = framenum + 1; + if (next_framenum >= framecount) + next_framenum = 0; + + /* prefetch the next source mbuf and destination TPACKET */ + if (likely(i + 1 < nb_pkts)) { + rte_prefetch0(bufs[i + 1]); + rte_prefetch0(pkt_q->rd[next_framenum].iov_base); } - /* insert vlan info if necessary */ - if (mbuf->ol_flags & RTE_MBUF_F_TX_VLAN) { - if (rte_vlan_insert(&mbuf)) { - rte_pktmbuf_free(mbuf); - continue; - } + mbuf = bufs[i]; + ppd = (struct tpacket2_hdr *)pkt_q->rd[framenum].iov_base; + + /* Drop oversized packets. Insert VLAN if necessary */ + if (unlikely(mbuf->pkt_len > pkt_q->frame_data_size || + ((mbuf->ol_flags & RTE_MBUF_F_TX_VLAN) != 0 && + rte_vlan_insert(&mbuf) != 0))) { + continue; } /* @@ -312,6 +316,9 @@ eth_af_packet_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) pbuf = (uint8_t *)ppd + ETH_AF_PACKET_FRAME_OVERHEAD; + ppd->tp_len = mbuf->pkt_len; + ppd->tp_snaplen = mbuf->pkt_len; + struct rte_mbuf *tmp_mbuf = mbuf; do { uint16_t data_len = rte_pktmbuf_data_len(tmp_mbuf); @@ -320,23 +327,19 @@ eth_af_packet_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) tmp_mbuf = tmp_mbuf->next; } while (tmp_mbuf); - ppd->tp_len = mbuf->pkt_len; - ppd->tp_snaplen = mbuf->pkt_len; - /* release incoming frame and advance ring buffer */ tpacket_write_status(&ppd->tp_status, TP_STATUS_SEND_REQUEST); - if (++framenum >= framecount) - framenum = 0; - ppd = (struct tpacket2_hdr *) pkt_q->rd[framenum].iov_base; - + framenum = next_framenum; num_tx++; num_tx_bytes += mbuf->pkt_len; - rte_pktmbuf_free(mbuf); } + rte_pktmbuf_free_bulk(&bufs[0], i); + /* kick-off transmits */ - if (sendto(pkt_q->sockfd, NULL, 0, MSG_DONTWAIT, NULL, 0) == -1 && - errno != ENOBUFS && errno != EAGAIN) { + if (unlikely(num_tx > 0 && + sendto(pkt_q->sockfd, NULL, 0, MSG_DONTWAIT, NULL, 0) == -1 && + errno != ENOBUFS && errno != EAGAIN)) { /* * In case of a ENOBUFS/EAGAIN error all of the enqueued * packets will be considered successful even though only some -- 2.39.5 (Apple Git-154)
