The branch main has been updated by jhibbits: URL: https://cgit.FreeBSD.org/src/commit/?id=aefd61a1de86521abc1551aa98ed94316fc83d86
commit aefd61a1de86521abc1551aa98ed94316fc83d86 Author: Justin Hibbits <[email protected]> AuthorDate: 2026-08-10 18:03:00 +0000 Commit: Justin Hibbits <[email protected]> CommitDate: 2026-08-24 03:11:27 +0000 dpaa: Add LRO and receive callback batching Reduce the code executed in the DQRR dequeue loop, and move the heavy-weight operations to post-dequeue loop. * Batch if_input() after DQRR dispatch loop completes. Only do the DQRR_CI_CINH write at the end of the loop, so only up to 16 entries will be processed. * Add software LRO per FQ. Each per-CPU RX FQ gets its own LRO tracking structure. Since LRO is configured at FQ initialization time, allocate the ifnet earlier in attach to prevent a panic. --- sys/dev/dpaa/dpaa_eth.c | 58 ++++++++++++++++++++++++++++++++++++++++++++- sys/dev/dpaa/dpaa_eth.h | 13 ++++++++++ sys/dev/dpaa/if_dtsec.c | 7 +++--- sys/dev/dpaa/if_memac.c | 10 +++++--- sys/dev/dpaa/qman.c | 7 ++++++ sys/dev/dpaa/qman.h | 20 ++++++++++++++++ sys/dev/dpaa/qman_portals.c | 38 ++++++++++++++++++++++------- sys/dev/dpaa/qman_var.h | 4 ++++ 8 files changed, 142 insertions(+), 15 deletions(-) diff --git a/sys/dev/dpaa/dpaa_eth.c b/sys/dev/dpaa/dpaa_eth.c index 23ebc78c1a6f..67e9da45b1d7 100644 --- a/sys/dev/dpaa/dpaa_eth.c +++ b/sys/dev/dpaa/dpaa_eth.c @@ -46,6 +46,7 @@ #include <net/if_arp.h> #include <netinet/ip.h> #include <netinet/ip6.h> +#include <netinet/tcp_lro.h> #include <dev/mii/mii.h> #include <dev/mii/miivar.h> @@ -507,7 +508,19 @@ dpaa_eth_fq_rx_callback(device_t portal, struct qman_fq *fq, m->m_len = frame->length; m_fixhdr(m); - if_input(sc->sc_ifnet, m); + /* + * Offer to LRO first. + */ + if (rxfq->lro_inited && + (if_getcapenable(sc->sc_ifnet) & IFCAP_LRO) != 0 && + (m->m_pkthdr.csum_flags & (CSUM_L4_CALC | CSUM_L4_VALID)) == + (CSUM_L4_CALC | CSUM_L4_VALID) && + tcp_lro_rx(&rxfq->lro, m, 0) == 0) + return (1); + + m->m_nextpkt = NULL; + *rxfq->rx_tailp = m; + rxfq->rx_tailp = &m->m_nextpkt; return (1); @@ -519,6 +532,32 @@ err: return (1); } +/* + * Post-poll flush hook invoked once per QMan portal poll on any + * RX FQ that dispatched at least one frame this cycle. Runs on + * the FQ's affine CPU, outside the DQRR dispatch loop. + */ +static void +dpaa_eth_fq_rx_flush(struct qman_fq *fq __unused, void *ctx) +{ + struct dpaa_eth_rx_fq *rxfq = ctx; + + if (rxfq->rx_head != NULL) { + struct mbuf *chain = rxfq->rx_head; + + rxfq->rx_head = NULL; + rxfq->rx_tailp = &rxfq->rx_head; + if_input(rxfq->sc->sc_ifnet, chain); + } + /* + * Flush LRO whenever initialised. If the user disabled + * IFCAP_LRO between the last callback and this flush, entries + * queued in that window still need to be drained. + */ + if (rxfq->lro_inited) + tcp_lro_flush_all(&rxfq->lro); +} + static int dpaa_eth_fq_tx_confirm_callback(device_t portal, struct qman_fq *fq, struct qman_fd *frame, void *app) @@ -592,6 +631,14 @@ dpaa_eth_fq_rx_free(struct dpaa_eth_softc *sc) for (i = 0; i < sc->sc_nrxfqs; i++) { if (sc->sc_rx_fqs[i].fq != NULL) qman_fq_free(sc->sc_rx_fqs[i].fq); + /* + * Any pending non-LRO mbufs on the batch chain + * are dropped here rather than delivered late. + */ + if (sc->sc_rx_fqs[i].rx_head != NULL) + m_freem(sc->sc_rx_fqs[i].rx_head); + if (sc->sc_rx_fqs[i].lro_inited) + tcp_lro_free(&sc->sc_rx_fqs[i].lro); } free(sc->sc_rx_fqs, M_DEVBUF); sc->sc_rx_fqs = NULL; @@ -664,6 +711,14 @@ dpaa_eth_fq_rx_init(struct dpaa_eth_softc *sc) sc->sc_rx_fqs[i].fqid = base_fqid + i; sc->sc_rx_fqs[i].cpu = i; sc->sc_rx_fqs[i].sc = sc; + sc->sc_rx_fqs[i].rx_head = NULL; + sc->sc_rx_fqs[i].rx_tailp = &sc->sc_rx_fqs[i].rx_head; + + /* Best-effort LRO per FQ. */ + if (tcp_lro_init(&sc->sc_rx_fqs[i].lro) == 0) { + sc->sc_rx_fqs[i].lro.ifp = sc->sc_ifnet; + sc->sc_rx_fqs[i].lro_inited = true; + } error = qman_fq_register_cb(fq, dpaa_eth_fq_rx_callback, &sc->sc_rx_fqs[i]); @@ -672,6 +727,7 @@ dpaa_eth_fq_rx_init(struct dpaa_eth_softc *sc) "could not register RX callback for FQ %d\n", i); goto err; } + (void)qman_fq_register_flush_cb(fq, dpaa_eth_fq_rx_flush); } /* diff --git a/sys/dev/dpaa/dpaa_eth.h b/sys/dev/dpaa/dpaa_eth.h index 4da52a84617e..43724ba391a5 100644 --- a/sys/dev/dpaa/dpaa_eth.h +++ b/sys/dev/dpaa/dpaa_eth.h @@ -27,6 +27,8 @@ #ifndef DPAA_ETH_H_ #define DPAA_ETH_H_ +#include <netinet/tcp_lro.h> + /* * TX csum-offload hwassist mask for dTSEC and mEMAC. */ #define DPAA_CSUM_TX_OFFLOAD \ (CSUM_IP | CSUM_DELAY_DATA | CSUM_DELAY_DATA_IPV6) @@ -54,6 +56,17 @@ struct dpaa_eth_rx_fq { * Sysctl readers get advisory (torn on 32-bit hosts) values. */ uint64_t frames_in; + + /* + * Batched-input state. The RX callback appends non-LRO mbufs + * onto rx_head via m_nextpkt; the per-FQ flush hook hands the + * whole chain to if_input() and then runs tcp_lro_flush_all(). + * Both fields are only touched from the affine CPU. + */ + struct mbuf *rx_head; + struct mbuf **rx_tailp; + struct lro_ctrl lro; + bool lro_inited; }; struct dpaa_eth_softc { diff --git a/sys/dev/dpaa/if_dtsec.c b/sys/dev/dpaa/if_dtsec.c index a0ea6e69dee3..7ef7de69da31 100644 --- a/sys/dev/dpaa/if_dtsec.c +++ b/sys/dev/dpaa/if_dtsec.c @@ -465,6 +465,9 @@ dtsec_attach(device_t dev) /* Init callouts */ callout_init(&sc->sc_base.sc_tick_callout, CALLOUT_MPSAFE); + /* Create network interface for upper layers */ + ifp = sc->sc_base.sc_ifnet = if_alloc(IFT_ETHER); + /* Create RX buffer pool */ error = dpaa_eth_pool_rx_init(&sc->sc_base); if (error != 0) @@ -507,8 +510,6 @@ dtsec_attach(device_t dev) return (ENXIO); } - /* Create network interface for upper layers */ - ifp = sc->sc_base.sc_ifnet = if_alloc(IFT_ETHER); if_setsoftc(ifp, sc); if_setflags(ifp, IFF_SIMPLEX | IFF_BROADCAST | IFF_MULTICAST); @@ -532,7 +533,7 @@ dtsec_attach(device_t dev) if_setcapabilities(ifp, IFCAP_JUMBO_MTU | IFCAP_VLAN_MTU | IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 | - IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6); + IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6 | IFCAP_LRO); if_setcapenable(ifp, if_getcapabilities(ifp)); if_sethwassist(ifp, DPAA_CSUM_TX_OFFLOAD); diff --git a/sys/dev/dpaa/if_memac.c b/sys/dev/dpaa/if_memac.c index 047cfea323a6..2e2ac116709a 100644 --- a/sys/dev/dpaa/if_memac.c +++ b/sys/dev/dpaa/if_memac.c @@ -354,6 +354,8 @@ memac_if_ioctl(if_t ifp, u_long command, caddr_t data) IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6); if_togglehwassist(ifp, DPAA_CSUM_TX_OFFLOAD); } + if ((changed & IFCAP_LRO) != 0) + if_togglecapenable(ifp, IFCAP_LRO); break; case SIOCGIFMEDIA: @@ -566,6 +568,9 @@ memac_attach(device_t dev) /* Init callouts */ callout_init(&sc->sc_base.sc_tick_callout, CALLOUT_MPSAFE); + /* Create network interface for upper layers */ + ifp = sc->sc_base.sc_ifnet = if_alloc(IFT_ETHER); + /* Create RX buffer pool */ error = dpaa_eth_pool_rx_init(&sc->sc_base); if (error != 0) @@ -596,8 +601,6 @@ memac_attach(device_t dev) dpaa_eth_fm_port_rx_init(&sc->sc_base); dpaa_eth_fm_port_tx_init(&sc->sc_base); - /* Create network interface for upper layers */ - ifp = sc->sc_base.sc_ifnet = if_alloc(IFT_ETHER); if_setsoftc(ifp, sc); if_setflags(ifp, IFF_SIMPLEX | IFF_BROADCAST | IFF_MULTICAST); @@ -618,7 +621,8 @@ memac_attach(device_t dev) if_setcapabilities(ifp, IFCAP_JUMBO_MTU | IFCAP_VLAN_MTU | IFCAP_VLAN_HWCSUM | IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 | - IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6); + IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6 | + IFCAP_LRO); if_setcapenable(ifp, if_getcapabilities(ifp)); if_sethwassist(ifp, DPAA_CSUM_TX_OFFLOAD); diff --git a/sys/dev/dpaa/qman.c b/sys/dev/dpaa/qman.c index 4c4c53beacfc..9da6e45d3b49 100644 --- a/sys/dev/dpaa/qman.c +++ b/sys/dev/dpaa/qman.c @@ -665,6 +665,13 @@ qman_fq_register_cb(struct qman_fq *fq, qman_cb_dqrr callback, return (0); } +int +qman_fq_register_flush_cb(struct qman_fq *fq, qman_cb_flush flush) +{ + fq->cb.flush = flush; + return (0); +} + int qman_fq_enqueue(struct qman_fq *fq, struct dpaa_fd *frame) { diff --git a/sys/dev/dpaa/qman.h b/sys/dev/dpaa/qman.h index 25ddc80eeedb..b009760d6257 100644 --- a/sys/dev/dpaa/qman.h +++ b/sys/dev/dpaa/qman.h @@ -164,11 +164,21 @@ typedef int (*qman_cb_dqrr)(device_t, struct qman_fq *, struct qman_fd *, void *); typedef void (*qman_cb_mr)(device_t, struct qman_fq *, struct qman_mr_entry *); +typedef void (*qman_cb_flush)(struct qman_fq *, void *); struct qman_cb { qman_cb_dqrr dqrr; qman_cb_mr ern; qman_cb_mr fqscn; + /* + * Optional post-poll hook. If set, qman_portal_loop_dqrr() + * calls it once per poll cycle on every FQ that had a frame + * dispatched, after all DQRR entries have been drained. + * Consumers use this for aggregation flushes (e.g., LRO + * flush, batched if_input) that must happen outside the + * per-frame dispatch path. + */ + qman_cb_flush flush; void *ctx; }; /** @@ -265,6 +275,16 @@ int qman_fq_free(struct qman_fq *fq); * @param app A pointer to the user's data. * @return E_OK on success; error code otherwise. */ +/* + * Register a post-poll flush callback on @fq. Called once per poll + * cycle on any FQ that had at least one dispatched frame during + * that cycle, after all DQRR entries were drained. Reuses the + * per-FQ ctx registered via qman_fq_register_cb(). May be called + * before or after qman_fq_register_cb() but only makes sense if the + * DQRR callback is also set. + */ +int qman_fq_register_flush_cb(struct qman_fq *fq, qman_cb_flush flush); + int qman_fq_register_cb(struct qman_fq *fq, qman_cb_dqrr callback, void *ctx); diff --git a/sys/dev/dpaa/qman_portals.c b/sys/dev/dpaa/qman_portals.c index be271c24bf06..7b5d795eb2a2 100644 --- a/sys/dev/dpaa/qman_portals.c +++ b/sys/dev/dpaa/qman_portals.c @@ -369,6 +369,7 @@ qman_portal_fq_enqueue(device_t dev, struct qman_fq *fq, struct dpaa_fd *frame) static int qman_portal_loop_dqrr(struct qman_portal_softc *sc) { + SLIST_HEAD(, qman_fq) dirty_fqs = SLIST_HEAD_INITIALIZER(dirty_fqs); struct qman_dqrr_entry *dqrr; struct qman_dqrr_entry *base; struct qman_fq *fq; @@ -376,24 +377,45 @@ qman_portal_loop_dqrr(struct qman_portal_softc *sc) DQRR_CI_CI_M; int pi = bus_read_4(sc->sc_base.sc_mres[1], QCSP_DQRR_PI_CINH) & DQRR_PI_PI_M; + int start_ci = ci; base = sc->sc_dqrr.ring; do { dqrr = &base[ci]; dpaa_flush_line(dqrr); dpaa_touch_line(dqrr); - if ((dqrr->stat & QMAN_DQRR_STAT_HAS_FRAME)) { - fq = qman_fq_from_index(dqrr->fqid); - if (fq != NULL && fq->cb.dqrr != NULL) { - fq->cb.dqrr(sc->sc_base.sc_dev, fq, - &dqrr->fd, fq->cb.ctx); - } - } else + if ((dqrr->stat & QMAN_DQRR_STAT_HAS_FRAME) == 0) break; + + fq = qman_fq_from_index(dqrr->fqid); + if (fq != NULL && fq->cb.dqrr != NULL) { + fq->cb.dqrr(sc->sc_base.sc_dev, fq, + &dqrr->fd, fq->cb.ctx); + /* + * Track the FQ for post-poll flush. Only need to track + * once. + */ + if (fq->cb.flush != NULL && !fq->dirty) { + fq->dirty = true; + SLIST_INSERT_HEAD(&dirty_fqs, fq, + dirty_next); + } + } ci = (ci + 1) & DQRR_CI_CI_M; - bus_write_4(sc->sc_base.sc_mres[1], QCSP_DQRR_CI_CINH, ci); } while (ci != pi); + /* Update CI after the loop. CI writes are expensive. */ + if (ci != start_ci) + bus_write_4(sc->sc_base.sc_mres[1], QCSP_DQRR_CI_CINH, ci); + + /* Drain the dirty list, invoking the per-FQ flush hook. */ + while (!SLIST_EMPTY(&dirty_fqs)) { + fq = SLIST_FIRST(&dirty_fqs); + SLIST_REMOVE_HEAD(&dirty_fqs, dirty_next); + fq->dirty = false; + fq->cb.flush(fq, fq->cb.ctx); + } + return (0); } diff --git a/sys/dev/dpaa/qman_var.h b/sys/dev/dpaa/qman_var.h index 2d64c10d534c..ac18e6acecb9 100644 --- a/sys/dev/dpaa/qman_var.h +++ b/sys/dev/dpaa/qman_var.h @@ -7,6 +7,8 @@ #ifndef QMAN_VAR_H #define QMAN_VAR_H +#include <sys/queue.h> + #include "dpaa_common.h" #include "portals.h" @@ -181,6 +183,8 @@ struct qman_fq { uint32_t fqid; /* base FQID of the range */ uint32_t fqid_count; /* length of the range (>=1) */ bool force_fqid; /* caller owns the FQID allocation */ + bool dirty; /* on a portal's dirty list this poll */ + SLIST_ENTRY(qman_fq) dirty_next;/* DQRR loop dirty list. */ struct qman_cb cb; };
