rte_pktmbuf_free_bulk() currently stages freeable mbufs in a temporary array before returning them to their mempool. For flat packet arrays, this requires copying pointers even though the original array already contains contiguous freeable mbufs.
Track contiguous same-pool runs in the input array and pass them directly to rte_mbuf_raw_free_bulk(). Flush a run when encountering a NULL mbuf, an mbuf retained by reference counting, or a pool change. Preserve the existing array-based implementation as the fallback for chained packets. On an ARM64 Linux test environment, same-binary A/B measurements using rte_rdtsc showed lower median timer ticks per call for flat bulk frees: burst 32: 2.05 -> 1.50 burst 64: 5.16 -> 4.52 burst 128: 11.16 -> 6.90 burst 256: 26.46 -> 19.73 This corresponds to reductions of approximately 12% to 38% across the tested burst sizes. Add coverage for NULL entries, mixed mempools, and shared mbufs. Signed-off-by: Nam Tran <[email protected]> --- app/test/test_mbuf.c | 53 +++++++++++++++++++++++++++++++++ lib/mbuf/rte_mbuf.c | 71 ++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 121 insertions(+), 3 deletions(-) diff --git a/app/test/test_mbuf.c b/app/test/test_mbuf.c index db23259745..2aed37b225 100644 --- a/app/test/test_mbuf.c +++ b/app/test/test_mbuf.c @@ -833,6 +833,59 @@ test_pktmbuf_pool_bulk(void) goto err; } + printf("Test bulk free with NULL entries.\n"); + + ret = rte_pktmbuf_alloc_bulk(pool, mbufs, 4); + if (ret != 0) + goto err; + + m = mbufs[1]; + mbufs[1] = NULL; + rte_pktmbuf_free_bulk(mbufs, 4); + rte_pktmbuf_free(m); + + if (!rte_mempool_full(pool)) { + printf("mempool not full after NULL-entry bulk free\n"); + goto err; + } + + printf("Test bulk free with multiple pools.\n"); + + for (i = 0; i < 4; i++) { + mbufs[i] = rte_pktmbuf_alloc((i & 1) ? pool2 : pool); + if (mbufs[i] == NULL) + goto err; + } + + rte_pktmbuf_free_bulk(mbufs, 4); + + if (!(rte_mempool_full(pool) && rte_mempool_full(pool2))) { + printf("mempools not full after mixed-pool bulk free\n"); + goto err; + } + + printf("Test bulk free with shared mbuf.\n"); + + ret = rte_pktmbuf_alloc_bulk(pool, mbufs, 4); + if (ret != 0) + goto err; + + m = mbufs[1]; + rte_mbuf_refcnt_update(m, 1); + rte_pktmbuf_free_bulk(mbufs, 4); + + if (rte_mbuf_refcnt_read(m) != 1) { + printf("shared mbuf reference count incorrect\n"); + goto err; + } + + rte_pktmbuf_free(m); + + if (!rte_mempool_full(pool)) { + printf("mempool not full after shared mbuf free\n"); + goto err; + } + printf("Test bulk free of single long chain.\n"); /* Bulk allocate all mbufs in the pool, in one go. */ diff --git a/lib/mbuf/rte_mbuf.c b/lib/mbuf/rte_mbuf.c index 005bfaa573..796631a033 100644 --- a/lib/mbuf/rte_mbuf.c +++ b/lib/mbuf/rte_mbuf.c @@ -555,15 +555,15 @@ __rte_pktmbuf_free_seg_via_array(struct rte_mbuf *m, */ #define RTE_PKTMBUF_FREE_PENDING_SZ 64 -/* Free a bulk of packet mbufs back into their original mempools. */ -RTE_EXPORT_SYMBOL(rte_pktmbuf_free_bulk) -void rte_pktmbuf_free_bulk(struct rte_mbuf **mbufs, unsigned int count) +static void +__rte_pktmbuf_free_bulk_fallback(struct rte_mbuf **mbufs, unsigned int count) { struct rte_mbuf *m, *m_next, *pending[RTE_PKTMBUF_FREE_PENDING_SZ]; unsigned int idx, nb_pending = 0; for (idx = 0; idx < count; idx++) { m = mbufs[idx]; + if (unlikely(m == NULL)) continue; @@ -582,6 +582,71 @@ void rte_pktmbuf_free_bulk(struct rte_mbuf **mbufs, unsigned int count) rte_mbuf_raw_free_bulk(pending[0]->pool, pending, nb_pending); } +/* Free a bulk of packet mbufs back into their original mempools. */ +RTE_EXPORT_SYMBOL(rte_pktmbuf_free_bulk) +void rte_pktmbuf_free_bulk(struct rte_mbuf **mbufs, unsigned int count) +{ + struct rte_mempool *run_pool = NULL; + unsigned int run_start = 0; + unsigned int run_count = 0; + unsigned int idx; + + for (idx = 0; idx < count; idx++) { + struct rte_mbuf *m = mbufs[idx]; + + if (unlikely(m == NULL)) { + if (run_count != 0) { + rte_mbuf_raw_free_bulk(run_pool, + &mbufs[run_start], run_count); + run_count = 0; + } + continue; + } + + __rte_mbuf_sanity_check(m, 1); + + /* + * Preserve the generic path for chained packets. No mbuf in + * this suffix has been modified yet. + */ + if (unlikely(m->next != NULL)) { + if (run_count != 0) + rte_mbuf_raw_free_bulk(run_pool, + &mbufs[run_start], run_count); + + __rte_pktmbuf_free_bulk_fallback(&mbufs[idx], + count - idx); + return; + } + + m = rte_pktmbuf_prefree_seg(m); + if (unlikely(m == NULL)) { + if (run_count != 0) { + rte_mbuf_raw_free_bulk(run_pool, + &mbufs[run_start], run_count); + run_count = 0; + } + continue; + } + + if (run_count != 0 && m->pool != run_pool) { + rte_mbuf_raw_free_bulk(run_pool, + &mbufs[run_start], run_count); + run_count = 0; + } + + if (run_count == 0) { + run_pool = m->pool; + run_start = idx; + } + + run_count++; + } + + if (run_count != 0) + rte_mbuf_raw_free_bulk(run_pool, &mbufs[run_start], run_count); +} + /* Creates a shallow copy of mbuf */ RTE_EXPORT_SYMBOL(rte_pktmbuf_clone) struct rte_mbuf * -- 2.51.0
