The branch main has been updated by kbowling: URL: https://cgit.FreeBSD.org/src/commit/?id=fdce3830d9a66aa30cb40ad4ea984f376ee00095
commit fdce3830d9a66aa30cb40ad4ea984f376ee00095 Author: Kevin Bowling <[email protected]> AuthorDate: 2026-07-29 09:17:31 +0000 Commit: Kevin Bowling <[email protected]> CommitDate: 2026-07-30 05:15:23 +0000 igbv: Retry transient VLAN admission failures A PF mailbox NACK does not distinguish the SR-IOV VLAN request rate limit from permanent VLVF exhaustion. Preserve desired VLAN membership and retry four additions per 500 ms timer tick, matching the PF sustained allowance. Bound the whole recovery batch to eight seconds from its first failure and consolidate restore diagnostics, so a full table cannot create a permanent mailbox poller or repeated per-VID log bursts. Sponsored by: BBOX.io --- sys/dev/e1000/if_em.c | 21 ++++++++--- sys/dev/e1000/if_em.h | 5 +++ sys/dev/e1000/if_igbv.c | 95 +++++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 113 insertions(+), 8 deletions(-) diff --git a/sys/dev/e1000/if_em.c b/sys/dev/e1000/if_em.c index d8bde14c6c20..2269f3014abb 100644 --- a/sys/dev/e1000/if_em.c +++ b/sys/dev/e1000/if_em.c @@ -4581,9 +4581,11 @@ em_if_vlan_register(if_ctx_t ctx, u16 vtag) ++sc->num_vlans; if (sc->vf_ifp && e1000_vfta_set_vf(&sc->hw, vtag, true) != E1000_SUCCESS) { + igbv_vlan_retry_add(sc, vtag); device_printf(sc->dev, "VF VLAN %u add request failed\n", vtag); - } + } else if (sc->vf_ifp) + igbv_vlan_retry_clear(sc, vtag); if (!sc->vf_ifp) { if (igb_iov_enabled(sc)) igb_iov_rebuild_vlan(sc); @@ -4602,6 +4604,8 @@ em_if_vlan_unregister(if_ctx_t ctx, u16 vtag) index = (vtag >> 5) & 0x7F; mask = 1U << (vtag & 0x1F); present = (sc->shadow_vfta[index] & mask) != 0; + if (sc->vf_ifp) + igbv_vlan_retry_clear(sc, vtag); if (sc->vf_ifp && e1000_vfta_set_vf(&sc->hw, vtag, false) != E1000_SUCCESS) { device_printf(sc->dev, @@ -4704,6 +4708,7 @@ em_setup_vlan_hw_support(if_ctx_t ctx) s32 error; u32 max_frame_size, reg; u16 vid; + int restore_failures; /* * Only PFs have control over VLAN HW filtering @@ -4714,6 +4719,7 @@ em_setup_vlan_hw_support(if_ctx_t ctx) max_frame_size = min(sc->shared->isc_max_frame_size + VLAN_TAG_SIZE, IGB_IOV_MAX_FRAME_SIZE); e1000_rlpml_set_vf(hw, max_frame_size); + restore_failures = 0; for (vid = 0; vid < 4096; vid++) { if ((sc->shadow_vfta[vid >> 5] & (1U << (vid & 0x1f))) == 0) @@ -4723,11 +4729,16 @@ em_setup_vlan_hw_support(if_ctx_t ctx) * replay if the PF mailbox is absent during reset. */ error = e1000_vfta_set_vf(hw, vid, true); - if (error != E1000_SUCCESS) - device_printf(sc->dev, - "VF VLAN %u restore request failed\n", - vid); + if (error != E1000_SUCCESS) { + igbv_vlan_retry_add(sc, vid); + restore_failures++; + } else + igbv_vlan_retry_clear(sc, vid); } + if (restore_failures != 0) + device_printf(sc->dev, + "VF VLAN restore failed for %d VIDs; retrying\n", + restore_failures); return; } diff --git a/sys/dev/e1000/if_em.h b/sys/dev/e1000/if_em.h index 9cdf013c994a..617ecaa8e294 100644 --- a/sys/dev/e1000/if_em.h +++ b/sys/dev/e1000/if_em.h @@ -584,6 +584,9 @@ struct e1000_softc { */ u32 shadow_vfta[EM_VFTA_SIZE]; u32 vf_vfta_stale[EM_VFTA_SIZE]; + u32 vf_vfta_retry[EM_VFTA_SIZE]; + sbintime_t vf_vlan_retry_deadline; + u16 vf_vlan_retry_cursor; /* Info about the interface */ enum em_link_state link_state; @@ -674,6 +677,8 @@ void igbv_initialize_transmit_unit(if_ctx_t); void igbv_reconcile_mac(struct e1000_softc *, if_t); bool igbv_reset(if_ctx_t); void igbv_update_uc_addr_list(struct e1000_softc *, if_t); +void igbv_vlan_retry_add(struct e1000_softc *, u16); +void igbv_vlan_retry_clear(struct e1000_softc *, u16); /******************************************************************************** * vendor_info_array diff --git a/sys/dev/e1000/if_igbv.c b/sys/dev/e1000/if_igbv.c index 378a0065242b..01c586fdb4b0 100644 --- a/sys/dev/e1000/if_igbv.c +++ b/sys/dev/e1000/if_igbv.c @@ -31,6 +31,8 @@ #include <sys/sbuf.h> #define IGBV_MAX_MAC_FILTERS 3 +#define IGBV_VLAN_RETRY_BATCH 4 +#define IGBV_VLAN_RETRY_WINDOW (8 * SBT_1S) struct igb_vf_uc_addr_list { struct e1000_softc *sc; @@ -38,6 +40,87 @@ struct igb_vf_uc_addr_list { }; static bool igbv_tx_pending(struct e1000_softc *); +static bool igbv_vlan_retry_pending(const struct e1000_softc *); +static void igbv_vlan_retry_tick(struct e1000_softc *); + +void +igbv_vlan_retry_add(struct e1000_softc *sc, u16 vid) +{ + bool pending; + + KASSERT(sc->vf_ifp, ("%s called for a PF", __func__)); + pending = igbv_vlan_retry_pending(sc); + sc->vf_vfta_retry[vid >> 5] |= 1U << (vid & 0x1f); + /* Bound the whole batch from its first failure, not each new VID. */ + if (!pending) + sc->vf_vlan_retry_deadline = + getsbinuptime() + IGBV_VLAN_RETRY_WINDOW; +} + +void +igbv_vlan_retry_clear(struct e1000_softc *sc, u16 vid) +{ + + KASSERT(sc->vf_ifp, ("%s called for a PF", __func__)); + sc->vf_vfta_retry[vid >> 5] &= ~(1U << (vid & 0x1f)); +} + +static bool +igbv_vlan_retry_pending(const struct e1000_softc *sc) +{ + int i; + + for (i = 0; i < EM_VFTA_SIZE; i++) + if (sc->vf_vfta_retry[i] != 0) + return (true); + return (false); +} + +static void +igbv_vlan_retry_tick(struct e1000_softc *sc) +{ + u32 bit; + u16 vid; + int attempts, i, remaining; + + if (!igbv_vlan_retry_pending(sc)) { + sc->vf_vlan_retry_deadline = 0; + return; + } + if (getsbinuptime() >= sc->vf_vlan_retry_deadline) { + remaining = 0; + for (i = 0; i < EM_VFTA_SIZE; i++) + remaining += bitcount32(sc->vf_vfta_retry[i]); + memset(sc->vf_vfta_retry, 0, sizeof(sc->vf_vfta_retry)); + sc->vf_vlan_retry_deadline = 0; + device_printf(sc->dev, + "VF VLAN restore retries exhausted for %d VIDs\n", + remaining); + return; + } + + /* + * The mailbox NACK does not distinguish a transient PF rate limit + * from permanent VLVF exhaustion. Retry at the PF's sustained + * allowance, but bound the entire recovery window so ENOSPC cannot + * create a permanent mailbox poller. + */ + for (attempts = 0, i = 0; + attempts < IGBV_VLAN_RETRY_BATCH && i < 4096; i++) { + vid = sc->vf_vlan_retry_cursor; + sc->vf_vlan_retry_cursor = (vid + 1) & 0xfff; + bit = 1U << (vid & 0x1f); + if ((sc->vf_vfta_retry[vid >> 5] & bit) == 0) + continue; + attempts++; + if ((sc->shadow_vfta[vid >> 5] & bit) == 0 || + e1000_vfta_set_vf(&sc->hw, vid, true) == + E1000_SUCCESS) + sc->vf_vfta_retry[vid >> 5] &= ~bit; + } + if (!igbv_vlan_retry_pending(sc)) + sc->vf_vlan_retry_deadline = 0; +} int igbv_if_attach_pre(if_ctx_t ctx) @@ -98,7 +181,7 @@ igbv_if_update_admin_status(if_ctx_t ctx) struct e1000_softc *sc; struct e1000_hw *hw; device_t dev; - bool link_check; + bool link_check, timer_tick; sc = iflib_get_softc(ctx); hw = &sc->hw; @@ -151,9 +234,12 @@ igbv_if_update_admin_status(if_ctx_t ctx) iflib_admin_intr_deferred(ctx); } /* em_if_init() establishes a new counter baseline after the reset. */ - if (!sc->vf_reset_pending && - atomic_readandclear_32(&sc->stats_pending) != 0) + timer_tick = !sc->vf_reset_pending && + atomic_readandclear_32(&sc->stats_pending) != 0; + if (timer_tick) { em_update_stats_counters(sc); + igbv_vlan_retry_tick(sc); + } } static bool @@ -197,6 +283,9 @@ igbv_reset(if_ctx_t ctx) return (false); } memset(sc->vf_vfta_stale, 0, sizeof(sc->vf_vfta_stale)); + memset(sc->vf_vfta_retry, 0, sizeof(sc->vf_vfta_retry)); + sc->vf_vlan_retry_deadline = 0; + sc->vf_vlan_retry_cursor = 0; if (e1000_init_hw(hw) < 0) { device_printf(sc->dev, "Hardware Initialization Failed\n"); return (false);
