The AVX-512 TX completion path directly manipulated the mempool cache
internals (cache->objs, cache->len, cache->flushthresh) instead of using
the mempool API. This pattern is the same private bypass that existed in
the Intel common TX library before it was removed by commit 062d6fe5d00d
("net/intel: do not bypass mbuf lib for buffer fast-free") for the same
reason: it omits mbuf instrumentation (history marking) and reaches
directly into mempool cache internals, including the flushthresh field
that is now obsolete (kept only for API/ABI compatibility), making the
private fast path fragile against mempool cache layout changes.
Replace with a single rte_mbuf_raw_free_bulk() call, matching the Intel
common library. The RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE contract in
rte_ethdev.h requires the application to guarantee that per-queue all
mbufs come from the same mempool, have refcnt == 1, and are direct;
that documented guarantee, whose @see already points to
rte_mbuf_raw_free_bulk(), is exactly what makes this call correct. The
compiler inlines the bulk-free call to eliminate the overhead
difference.
Fixes: 0af0bdcdcf83 ("net/sxe2: add AVX512 Rx and Tx")
Cc: [email protected]
Signed-off-by: Kai Ji <[email protected]>
---
drivers/net/sxe2/sxe2_txrx_vec_avx512.c | 39 +++----------------------
1 file changed, 4 insertions(+), 35 deletions(-)
diff --git a/drivers/net/sxe2/sxe2_txrx_vec_avx512.c
b/drivers/net/sxe2/sxe2_txrx_vec_avx512.c
index a830c7a33b..1a4ebd93c3 100644
--- a/drivers/net/sxe2/sxe2_txrx_vec_avx512.c
+++ b/drivers/net/sxe2/sxe2_txrx_vec_avx512.c
@@ -12,15 +12,15 @@
#include "sxe2_txrx_vec_common.h"
#include "sxe2_vsi.h"
+static_assert(sizeof(struct sxe2_tx_buffer_vec) == sizeof(struct rte_mbuf *),
+ "sxe2_tx_buffer_vec must be pointer-sized for bulk free cast");
+
static __rte_always_inline int32_t sxe2_tx_bufs_free_vec_avx512(struct
sxe2_tx_queue *txq)
{
struct sxe2_tx_buffer_vec *buffer;
struct rte_mbuf *mbuf;
struct rte_mbuf *mbuf_free_arr[SXE2_TX_FREE_BUFFER_SIZE_MAX_VEC];
struct rte_mempool *mp;
- struct rte_mempool_cache *cache;
- void **cache_objs;
- uint32_t copied;
uint32_t i;
int32_t ret;
uint16_t rs_thresh;
@@ -41,41 +41,10 @@ static __rte_always_inline int32_t
sxe2_tx_bufs_free_vec_avx512(struct sxe2_tx_q
if ((txq->offloads & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE) &&
(rs_thresh & 31) == 0) {
mp = buffer[0].mbuf->pool;
- cache = rte_mempool_default_cache(mp, rte_lcore_id());
-
- if (cache == NULL || cache->len)
- goto normal;
-
- if (rs_thresh > RTE_MEMPOOL_CACHE_MAX_SIZE) {
- (void)rte_mempool_ops_enqueue_bulk(mp, (void *)buffer,
rs_thresh);
- goto done;
- }
- cache_objs = &cache->objs[cache->len];
-
- copied = 0;
- while (copied < rs_thresh) {
- const __m512i objs0 =
_mm512_loadu_si512(&buffer[copied]);
- const __m512i objs1 = _mm512_loadu_si512(&buffer[copied
+ 8]);
- const __m512i objs2 = _mm512_loadu_si512(&buffer[copied
+ 16]);
- const __m512i objs3 = _mm512_loadu_si512(&buffer[copied
+ 24]);
-
- _mm512_storeu_si512(&cache_objs[copied], objs0);
- _mm512_storeu_si512(&cache_objs[copied + 8], objs1);
- _mm512_storeu_si512(&cache_objs[copied + 16], objs2);
- _mm512_storeu_si512(&cache_objs[copied + 24], objs3);
- copied += 32;
- }
- cache->len += rs_thresh;
-
- if (cache->len >= cache->flushthresh) {
- (void)rte_mempool_ops_enqueue_bulk(mp,
- &cache->objs[cache->size], cache->len -
cache->size);
- cache->len = cache->size;
- }
+ rte_mbuf_raw_free_bulk(mp, (void *)buffer, rs_thresh);
goto done;
}
-normal:
mbuf = rte_pktmbuf_prefree_seg(buffer[0].mbuf);
if (likely(mbuf)) {
--
2.34.1
