The branch main has been updated by glebius: URL: https://cgit.FreeBSD.org/src/commit/?id=8a37f8c325b65abd889b31b394bebaca7ad42026
commit 8a37f8c325b65abd889b31b394bebaca7ad42026 Author: Gleb Smirnoff <[email protected]> AuthorDate: 2026-09-18 21:53:39 +0000 Commit: Gleb Smirnoff <[email protected]> CommitDate: 2026-09-18 21:53:39 +0000 buf_ring: use counter(9) for drops If a buffer ring experiences drops, then system is definitely starving in CPU resources. Having an extra cache miss on a racy shared variable increment doesn't help. Reviewed by: gallatin Differential Revision: https://reviews.freebsd.org/D59495 --- share/man/man9/buf_ring.9 | 11 ++++++++++- sys/dev/cxgb/cxgb_main.c | 2 +- sys/dev/cxgb/cxgb_sge.c | 2 +- sys/dev/mxge/if_mxge.c | 2 +- sys/kern/subr_bufring.c | 7 +++++++ sys/sys/buf_ring.h | 21 ++++++++++++++++++--- 6 files changed, 38 insertions(+), 7 deletions(-) diff --git a/share/man/man9/buf_ring.9 b/share/man/man9/buf_ring.9 index 29a5e2580800..cffbbccc9364 100644 --- a/share/man/man9/buf_ring.9 +++ b/share/man/man9/buf_ring.9 @@ -36,7 +36,8 @@ .Nm buf_ring_count , .Nm buf_ring_empty , .Nm buf_ring_full , -.Nm buf_ring_peek +.Nm buf_ring_peek , +.Nm buf_ring_drops .Nd multi-producer, {single, multi}-consumer lock-less ring buffer .Sh SYNOPSIS .In sys/param.h @@ -59,6 +60,8 @@ .Fn buf_ring_full "struct buf_ring *br" .Ft void * .Fn buf_ring_peek "struct buf_ring *br" +.Ft uint64_t +.Fn buf_ring_drops "struct buf_ring *br" .Sh DESCRIPTION The .Nm @@ -125,6 +128,12 @@ function returns a pointer to the last element in the buf_ring if the buf_ring is not empty, .Dv NULL otherwise. +.Pp +The +.Fn buf_ring_drops +functions returns how many times the +.Fn buf_ring_enqueue +was not able to queue an element due to no available slots. .Sh RETURN VALUES The .Fn buf_ring_enqueue diff --git a/sys/dev/cxgb/cxgb_main.c b/sys/dev/cxgb/cxgb_main.c index 81fd3cb45841..2137aea55795 100644 --- a/sys/dev/cxgb/cxgb_main.c +++ b/sys/dev/cxgb/cxgb_main.c @@ -2201,7 +2201,7 @@ cxgb_get_counter(if_t ifp, ift_counter c) drops = 0; if (sc->flags & FULL_INIT_DONE) { for (i = pi->first_qset; i < pi->first_qset + pi->nqsets; i++) - drops += sc->sge.qs[i].txq[TXQ_ETH].txq_mr->br_drops; + drops += buf_ring_drops(sc->sge.qs[i].txq[TXQ_ETH].txq_mr); } return (drops); diff --git a/sys/dev/cxgb/cxgb_sge.c b/sys/dev/cxgb/cxgb_sge.c index 9614b69cfc6a..05fbecde95d2 100644 --- a/sys/dev/cxgb/cxgb_sge.c +++ b/sys/dev/cxgb/cxgb_sge.c @@ -3512,7 +3512,7 @@ t3_add_configured_sysctls(adapter_t *sc) &qs->rspq, 0, t3_dump_rspq, "A", "dump of the response queue"); - SYSCTL_ADD_UQUAD(ctx, txqpoidlist, OID_AUTO, "dropped", + SYSCTL_ADD_COUNTER_U64(ctx, txqpoidlist, OID_AUTO, "dropped", CTLFLAG_RD, &qs->txq[TXQ_ETH].txq_mr->br_drops, "#tunneled packets dropped"); SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "sendqlen", diff --git a/sys/dev/mxge/if_mxge.c b/sys/dev/mxge/if_mxge.c index dd52d104ae5f..75bad049bbe9 100644 --- a/sys/dev/mxge/if_mxge.c +++ b/sys/dev/mxge/if_mxge.c @@ -3872,7 +3872,7 @@ mxge_get_counter(if_t ifp, ift_counter cnt) return (rv); case IFCOUNTER_OQDROPS: for (int s = 0; s < sc->num_slices; s++) - rv += sc->ss[s].tx.br->br_drops; + rv += buf_ring_drops(sc->ss[s].tx.br); return (rv); default: return (if_get_counter_default(ifp, cnt)); diff --git a/sys/kern/subr_bufring.c b/sys/kern/subr_bufring.c index 25691be69be2..ce077739efaa 100644 --- a/sys/kern/subr_bufring.c +++ b/sys/kern/subr_bufring.c @@ -30,6 +30,7 @@ #include <sys/systm.h> #include <sys/kernel.h> #include <sys/malloc.h> +#include <sys/counter.h> #include <sys/buf_ring.h> struct buf_ring * @@ -44,6 +45,11 @@ _buf_ring_alloc(int count, struct malloc_type *type, int flags, type, flags | M_ZERO); if (br == NULL) return (NULL); + br->br_drops = counter_u64_alloc(flags); + if (br->br_drops == NULL) { + free(br, type); + return (NULL); + } br->br_lock = lo; br->br_prod_size = br->br_cons_size = count; br->br_prod_mask = br->br_cons_mask = count-1; @@ -56,5 +62,6 @@ _buf_ring_alloc(int count, struct malloc_type *type, int flags, void buf_ring_free(struct buf_ring *br, struct malloc_type *type) { + counter_u64_free(br->br_drops); free(br, type); } diff --git a/sys/sys/buf_ring.h b/sys/sys/buf_ring.h index db934e083e1a..aa0dd2a077ce 100644 --- a/sys/sys/buf_ring.h +++ b/sys/sys/buf_ring.h @@ -38,10 +38,13 @@ #if defined(_KERNEL) #include <sys/lock.h> +#include <sys/counter.h> #define BR_LOCK_ASSERT(br) \ LOCK_CLASS((br)->br_lock)->lc_assert((br)->br_lock, LA_XLOCKED) #else #define BR_LOCK_ASSERT(br) do {} while (0) +typedef uint64_t counter_u64_t; +#define counter_u64_add(c,v) do { (c) += (v); } while (0) #endif /* @@ -58,7 +61,7 @@ struct buf_ring { uint32_t br_prod_tail; int br_prod_size; int br_prod_mask; - uint64_t br_drops; + counter_u64_t br_drops; uint32_t br_cons_head __aligned(CACHE_LINE_SIZE); uint32_t br_cons_tail; int br_cons_size; @@ -110,8 +113,8 @@ buf_ring_enqueue(struct buf_ring *br, void *buf) if ((int32_t)(cons_tail + br->br_prod_size - prod_next) < 1) { if (prod_head == atomic_load_32(&br->br_prod_head) && cons_tail == atomic_load_32(&br->br_cons_tail)) { - br->br_drops++; critical_exit(); + counter_u64_add(br->br_drops, 1); return (ENOBUFS); } continue; @@ -358,8 +361,14 @@ struct buf_ring *_buf_ring_alloc(int count, struct malloc_type *type, struct rwlock *: _buf_ring_alloc((c), (mt), (f), lk2lo(lk)), \ struct rmlock *: _buf_ring_alloc((c), (mt), (f), lk2lo(lk))) void buf_ring_free(struct buf_ring *br, struct malloc_type *type); -#else +static inline uint64_t +buf_ring_drops(struct buf_ring *br) +{ + return (counter_u64_fetch(br->br_drops)); +} + +#else /* !_KERNEL */ #include <stdlib.h> static inline struct buf_ring * @@ -385,5 +394,11 @@ buf_ring_free(struct buf_ring *br) free(br); } +static inline uint64_t +buf_ring_drops(struct buf_ring *br) +{ + return (br->br_drops); +} + #endif /* !_KERNEL */ #endif /* _SYS_BUF_RING_H_ */
