Module Name: src
Committed By: msaitoh
Date: Tue Apr 14 07:17:06 UTC 2015
Modified Files:
src/sys/dev/pci/ixgbe: ixgbe.c ixgbe.h ixgbe_82599.c ixgbe_osdep.h
ixv.c
Log Message:
Sync ixg(4) up to FreeBSD r243716:
- A lot of bugfixes. Some of them are realted to multi queue and those
have not affected in NetBSD because we have not used it yet.
- Show 1000Base-SX correctly.
- Fix if_baudrate from 1G to 10G.
- Improve performance.
To generate a diff of this commit:
cvs rdiff -u -r1.25 -r1.26 src/sys/dev/pci/ixgbe/ixgbe.c
cvs rdiff -u -r1.5 -r1.6 src/sys/dev/pci/ixgbe/ixgbe.h \
src/sys/dev/pci/ixgbe/ixgbe_osdep.h
cvs rdiff -u -r1.7 -r1.8 src/sys/dev/pci/ixgbe/ixgbe_82599.c
cvs rdiff -u -r1.6 -r1.7 src/sys/dev/pci/ixgbe/ixv.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/dev/pci/ixgbe/ixgbe.c
diff -u src/sys/dev/pci/ixgbe/ixgbe.c:1.25 src/sys/dev/pci/ixgbe/ixgbe.c:1.26
--- src/sys/dev/pci/ixgbe/ixgbe.c:1.25 Thu Apr 2 09:26:55 2015
+++ src/sys/dev/pci/ixgbe/ixgbe.c Tue Apr 14 07:17:06 2015
@@ -58,8 +58,8 @@
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
-/*$FreeBSD: head/sys/dev/ixgbe/ixgbe.c 238149 2012-07-05 20:51:44Z jfv $*/
-/*$NetBSD: ixgbe.c,v 1.25 2015/04/02 09:26:55 msaitoh Exp $*/
+/*$FreeBSD: head/sys/dev/ixgbe/ixgbe.c 243716 2012-11-30 22:33:21Z jfv $*/
+/*$NetBSD: ixgbe.c,v 1.26 2015/04/14 07:17:06 msaitoh Exp $*/
#include "opt_inet.h"
#include "opt_inet6.h"
@@ -74,7 +74,7 @@ int ixgbe_display_debug_stat
/*********************************************************************
* Driver version
*********************************************************************/
-char ixgbe_driver_version[] = "2.4.8";
+char ixgbe_driver_version[] = "2.5.0 - 2";
/*********************************************************************
* PCI Device ID Table
@@ -134,13 +134,15 @@ static int ixgbe_detach(device_t, i
#if 0
static int ixgbe_shutdown(device_t);
#endif
-static void ixgbe_start(struct ifnet *);
-static void ixgbe_start_locked(struct tx_ring *, struct ifnet *);
#if __FreeBSD_version >= 800000
static int ixgbe_mq_start(struct ifnet *, struct mbuf *);
static int ixgbe_mq_start_locked(struct ifnet *,
struct tx_ring *, struct mbuf *);
static void ixgbe_qflush(struct ifnet *);
+static void ixgbe_deferred_mq_start(void *, int);
+#else
+static void ixgbe_start(struct ifnet *);
+static void ixgbe_start_locked(struct tx_ring *, struct ifnet *);
#endif
static int ixgbe_ioctl(struct ifnet *, u_long, void *);
static void ixgbe_ifstop(struct ifnet *, int);
@@ -216,6 +218,9 @@ static __inline void ixgbe_rx_discard(st
static __inline void ixgbe_rx_input(struct rx_ring *, struct ifnet *,
struct mbuf *, u32);
+static void ixgbe_enable_rx_drop(struct adapter *);
+static void ixgbe_disable_rx_drop(struct adapter *);
+
/* Support for pluggable optic modules */
static bool ixgbe_sfp_probe(struct adapter *);
static void ixgbe_setup_optics(struct adapter *);
@@ -296,15 +301,6 @@ static int ixgbe_smart_speed = ixgbe_sma
static int ixgbe_enable_msix = 1;
TUNABLE_INT("hw.ixgbe.enable_msix", &ixgbe_enable_msix);
-/*
- * Header split: this causes the hardware to DMA
- * the header into a separate mbuf from the payload,
- * it can be a performance win in some workloads, but
- * in others it actually hurts, its off by default.
- */
-static int ixgbe_header_split = FALSE;
-TUNABLE_INT("hw.ixgbe.hdr_split", &ixgbe_header_split);
-
#if defined(NETBSD_MSI_OR_MSIX)
/*
* Number of Queues, can be set to 0,
@@ -328,6 +324,20 @@ TUNABLE_INT("hw.ixgbe.txd", &ixgbe_txd);
static int ixgbe_rxd = PERFORM_RXD;
TUNABLE_INT("hw.ixgbe.rxd", &ixgbe_rxd);
+/*
+** HW RSC control:
+** this feature only works with
+** IPv4, and only on 82599 and later.
+** Also this will cause IP forwarding to
+** fail and that can't be controlled by
+** the stack as LRO can. For all these
+** reasons I've deemed it best to leave
+** this off and not bother with a tuneable
+** interface, this would need to be compiled
+** to enable.
+*/
+static bool ixgbe_rsc_enable = FALSE;
+
/* Keep running tab on them for sanity check */
static int ixgbe_total_ports;
@@ -697,10 +707,10 @@ static int
ixgbe_detach(device_t dev, int flags)
{
struct adapter *adapter = device_private(dev);
- struct tx_ring *txr = adapter->tx_rings;
struct rx_ring *rxr = adapter->rx_rings;
struct ixgbe_hw_stats *stats = &adapter->stats;
struct ix_queue *que = adapter->queues;
+ struct tx_ring *txr = adapter->tx_rings;
u32 ctrl_ext;
INIT_DEBUGOUT("ixgbe_detach: begin");
@@ -719,7 +729,10 @@ ixgbe_detach(device_t dev, int flags)
ixgbe_stop(adapter);
IXGBE_CORE_UNLOCK(adapter);
- for (int i = 0; i < adapter->num_queues; i++, que++) {
+ for (int i = 0; i < adapter->num_queues; i++, que++, txr++) {
+#if __FreeBSD_version >= 800000
+ softint_disestablish(txr->txq_si);
+#endif
softint_disestablish(que->que_si);
}
@@ -764,6 +777,8 @@ ixgbe_detach(device_t dev, int flags)
evcnt_detach(&adapter->tso_err);
evcnt_detach(&adapter->tso_tx);
evcnt_detach(&adapter->link_irq);
+
+ txr = adapter->tx_rings;
for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) {
evcnt_detach(&txr->no_desc_avail);
evcnt_detach(&txr->total_packets);
@@ -790,7 +805,6 @@ ixgbe_detach(device_t dev, int flags)
evcnt_detach(&rxr->rx_bytes);
evcnt_detach(&rxr->no_jmbuf);
evcnt_detach(&rxr->rx_discarded);
- evcnt_detach(&rxr->rx_split_packets);
evcnt_detach(&rxr->rx_irq);
}
evcnt_detach(&stats->ipcs);
@@ -883,6 +897,7 @@ ixgbe_shutdown(device_t dev)
#endif
+#if __FreeBSD_version < 800000
/*********************************************************************
* Transmit entry point
*
@@ -902,25 +917,20 @@ ixgbe_start_locked(struct tx_ring *txr,
IXGBE_TX_LOCK_ASSERT(txr);
- if ((ifp->if_flags & (IFF_RUNNING|IFF_OACTIVE)) !=
- IFF_RUNNING)
+ if ((ifp->if_flags & IFF_RUNNING) == 0)
return;
if (!adapter->link_active)
return;
while (!IFQ_IS_EMPTY(&ifp->if_snd)) {
- if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE) {
- txr->queue_status |= IXGBE_QUEUE_DEPLETED;
+ if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE)
break;
- }
IFQ_POLL(&ifp->if_snd, m_head);
if (m_head == NULL)
break;
if ((rc = ixgbe_xmit(txr, m_head)) == EAGAIN) {
- if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE)
- txr->queue_status |= IXGBE_QUEUE_DEPLETED;
break;
}
IFQ_DEQUEUE(&ifp->if_snd, m_head);
@@ -970,7 +980,7 @@ ixgbe_start(struct ifnet *ifp)
return;
}
-#if __FreeBSD_version >= 800000
+#else
/*
** Multiqueue Transmit driver
**
@@ -992,13 +1002,12 @@ ixgbe_mq_start(struct ifnet *ifp, struct
txr = &adapter->tx_rings[i];
que = &adapter->queues[i];
- if (((txr->queue_status & IXGBE_QUEUE_DEPLETED) == 0) &&
- IXGBE_TX_TRYLOCK(txr)) {
+ if (IXGBE_TX_TRYLOCK(txr)) {
err = ixgbe_mq_start_locked(ifp, txr, m);
IXGBE_TX_UNLOCK(txr);
} else {
err = drbr_enqueue(ifp, txr->br, m);
- softint_schedule(que->que_si);
+ softint_schedule(txr->txq_si);
}
return (err);
@@ -1012,7 +1021,6 @@ ixgbe_mq_start_locked(struct ifnet *ifp,
int enqueued, err = 0;
if (((ifp->if_flags & IFF_RUNNING) == 0) ||
- (txr->queue_status == IXGBE_QUEUE_DEPLETED) ||
adapter->link_active == 0) {
if (m != NULL)
err = drbr_enqueue(ifp, txr->br, m);
@@ -1037,23 +1045,18 @@ ixgbe_mq_start_locked(struct ifnet *ifp,
break;
}
enqueued++;
- drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags);
/* Send a copy of the frame to the BPF listener */
bpf_mtap(ifp, next);
if ((ifp->if_flags & IFF_RUNNING) == 0)
break;
if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD)
ixgbe_txeof(txr);
- if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD) {
- ifp->if_flags |= IFF_OACTIVE;
- break;
- }
next = drbr_dequeue(ifp, txr->br);
}
if (enqueued > 0) {
/* Set watchdog on */
- txr->queue_status |= IXGBE_QUEUE_WORKING;
+ txr->queue_status = IXGBE_QUEUE_WORKING;
getmicrotime(&txr->watchdog_time);
}
@@ -1064,6 +1067,22 @@ ixgbe_mq_start_locked(struct ifnet *ifp,
}
/*
+ * Called from a taskqueue to drain queued transmit packets.
+ */
+static void
+ixgbe_deferred_mq_start(void *arg, int pending)
+{
+ struct tx_ring *txr = arg;
+ struct adapter *adapter = txr->adapter;
+ struct ifnet *ifp = adapter->ifp;
+
+ IXGBE_TX_LOCK(txr);
+ if (!drbr_empty(ifp, txr->br))
+ ixgbe_mq_start_locked(ifp, txr, NULL);
+ IXGBE_TX_UNLOCK(txr);
+}
+
+/*
** Flush all ring buffers
*/
static void
@@ -1248,7 +1267,7 @@ ixgbe_init_locked(struct adapter *adapte
/*
** Determine the correct mbuf pool
- ** for doing jumbo/headersplit
+ ** for doing jumbo frames
*/
if (adapter->max_frame_size <= 2048)
adapter->rx_mbuf_sz = MCLBYTES;
@@ -1459,7 +1478,6 @@ ixgbe_init_locked(struct adapter *adapte
/* Now inform the stack we're ready */
ifp->if_flags |= IFF_RUNNING;
- ifp->if_flags &= ~IFF_OACTIVE;
return;
}
@@ -1571,7 +1589,6 @@ ixgbe_handle_que(void *context)
/* Reenable this interrupt */
ixgbe_enable_queue(adapter, que->msix);
-
return;
}
@@ -1835,7 +1852,7 @@ ixgbe_media_status(struct ifnet * ifp, s
ifmr->ifm_active |= IFM_100_TX | IFM_FDX;
break;
case IXGBE_LINK_SPEED_1GB_FULL:
- ifmr->ifm_active |= IFM_1000_T | IFM_FDX;
+ ifmr->ifm_active |= adapter->optics | IFM_FDX;
break;
case IXGBE_LINK_SPEED_10GB_FULL:
ifmr->ifm_active |= adapter->optics | IFM_FDX;
@@ -2157,13 +2174,11 @@ ixgbe_local_timer1(void *arg)
{
struct adapter *adapter = arg;
device_t dev = adapter->dev;
- struct ifnet *ifp = adapter->ifp;
struct ix_queue *que = adapter->queues;
struct tx_ring *txr = adapter->tx_rings;
- int hung, busy, paused;
+ int hung = 0, paused = 0;
KASSERT(mutex_owned(&adapter->core_mtx));
- hung = busy = paused = 0;
/* Check for pluggable optics */
if (adapter->sfp_probe)
@@ -2182,27 +2197,18 @@ ixgbe_local_timer1(void *arg)
/*
** Check the TX queues status
- ** - central locked handling of OACTIVE
** - watchdog only if all queues show hung
*/
for (int i = 0; i < adapter->num_queues; i++, que++, txr++) {
- if ((txr->queue_status & IXGBE_QUEUE_HUNG) &&
+ if ((txr->queue_status == IXGBE_QUEUE_HUNG) &&
(paused == 0))
++hung;
- if (txr->queue_status & IXGBE_QUEUE_DEPLETED)
- ++busy;
- if ((txr->queue_status & IXGBE_QUEUE_IDLE) == 0)
+ else if (txr->queue_status == IXGBE_QUEUE_WORKING)
softint_schedule(que->que_si);
}
/* Only truely watchdog if all queues show hung */
if (hung == adapter->num_queues)
goto watchdog;
- /* Only turn off the stack flow when ALL are depleted */
- if (busy == adapter->num_queues)
- ifp->if_flags |= IFF_OACTIVE;
- else if ((ifp->if_flags & IFF_OACTIVE) &&
- (busy < adapter->num_queues))
- ifp->if_flags &= ~IFF_OACTIVE;
out:
ixgbe_rearm_queues(adapter, adapter->que_mask);
@@ -2241,7 +2247,6 @@ static void
ixgbe_update_link_status(struct adapter *adapter)
{
struct ifnet *ifp = adapter->ifp;
- struct tx_ring *txr = adapter->tx_rings;
device_t dev = adapter->dev;
@@ -2262,9 +2267,6 @@ ixgbe_update_link_status(struct adapter
device_printf(dev,"Link is Down\n");
if_link_state_change(ifp, LINK_STATE_DOWN);
adapter->link_active = FALSE;
- for (int i = 0; i < adapter->num_queues;
- i++, txr++)
- txr->queue_status = IXGBE_QUEUE_IDLE;
}
}
@@ -2305,7 +2307,6 @@ ixgbe_stop(void *arg)
/* Let the stack know...*/
ifp->if_flags &= ~IFF_RUNNING;
- ifp->if_flags |= IFF_OACTIVE;
ixgbe_reset_hw(hw);
hw->adapter_stopped = FALSE;
@@ -2373,16 +2374,22 @@ ixgbe_setup_optics(struct adapter *adapt
int layer;
layer = ixgbe_get_supported_physical_layer(hw);
+
if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) {
adapter->optics = IFM_10G_T;
return;
- }
+ }
if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) {
adapter->optics = IFM_1000_T;
return;
}
+ if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX) {
+ adapter->optics = IFM_1000_SX;
+ return;
+ }
+
if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_LR |
IXGBE_PHYSICAL_LAYER_10GBASE_LRM)) {
adapter->optics = IFM_10G_LR;
@@ -2420,6 +2427,9 @@ ixgbe_allocate_legacy(struct adapter *ad
{
device_t dev = adapter->dev;
struct ix_queue *que = adapter->queues;
+#if __FreeBSD_version >= 800000
+ struct tx_ring *txr = adapter->tx_rings;
+#endif
char intrbuf[PCI_INTRSTR_LEN];
#if 0
int rid = 0;
@@ -2443,6 +2453,10 @@ ixgbe_allocate_legacy(struct adapter *ad
* Try allocating a fast interrupt and the associated deferred
* processing contexts.
*/
+#if __FreeBSD_version >= 800000
+ txr->txq_si = softint_establish(SOFTINT_NET, ixgbe_deferred_mq_start,
+ txr);
+#endif
que->que_si = softint_establish(SOFTINT_NET, ixgbe_handle_que, que);
/* Tasklets for Link, SFP and Multispeed Fiber */
@@ -2502,9 +2516,10 @@ ixgbe_allocate_msix(struct adapter *adap
#else
device_t dev = adapter->dev;
struct ix_queue *que = adapter->queues;
+ struct tx_ring *txr = adapter->tx_rings;
int error, rid, vector = 0;
- for (int i = 0; i < adapter->num_queues; i++, vector++, que++) {
+ for (int i = 0; i < adapter->num_queues; i++, vector++, que++, txr++) {
rid = vector + 1;
que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
RF_SHAREABLE | RF_ACTIVE);
@@ -2535,7 +2550,12 @@ ixgbe_allocate_msix(struct adapter *adap
if (adapter->num_queues > 1)
bus_bind_intr(dev, que->res, i);
- que->que_si = softint_establish(ixgbe_handle_que, que);
+#if __FreeBSD_version >= 800000
+ txr->txq_si = softint_establish(SOFTINT_NET,
+ ixgbe_deferred_mq_start, txr);
+#endif
+ que->que_si = softint_establish(SOFTINT_NET, ixgbe_handle_que,
+ que);
if (que->que_si == NULL) {
aprint_error_dev(dev,
"could not establish software interrupt\n");
@@ -2792,18 +2812,19 @@ ixgbe_setup_interface(device_t dev, stru
ifp = adapter->ifp = &ec->ec_if;
strlcpy(ifp->if_xname, device_xname(dev), IFNAMSIZ);
- ifp->if_baudrate = 1000000000;
+ ifp->if_baudrate = IF_Gbps(10);
ifp->if_init = ixgbe_init;
ifp->if_stop = ixgbe_ifstop;
ifp->if_softc = adapter;
ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
ifp->if_ioctl = ixgbe_ioctl;
- ifp->if_start = ixgbe_start;
#if __FreeBSD_version >= 800000
ifp->if_transmit = ixgbe_mq_start;
ifp->if_qflush = ixgbe_qflush;
+#else
+ ifp->if_start = ixgbe_start;
+ IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 2);
#endif
- ifp->if_snd.ifq_maxlen = adapter->num_tx_desc - 2;
if_attach(ifp);
ether_ifattach(ifp, adapter->hw.mac.addr);
@@ -2822,13 +2843,11 @@ ixgbe_setup_interface(device_t dev, stru
ec->ec_capabilities |= ETHERCAP_VLAN_HWCSUM;
ec->ec_capabilities |= ETHERCAP_JUMBO_MTU;
+ ifp->if_capabilities |= IFCAP_LRO;
ec->ec_capabilities |= ETHERCAP_VLAN_HWTAGGING
| ETHERCAP_VLAN_MTU;
ec->ec_capenable = ec->ec_capabilities;
- /* Don't enable LRO by default */
- ifp->if_capabilities |= IFCAP_LRO;
-
/*
** Don't turn this on by default, if vlans are
** created on another pseudo device (eg. lagg)
@@ -3902,10 +3921,6 @@ ixgbe_txeof(struct tx_ring *txr)
if (!processed && tvtohz(&elapsed) > IXGBE_WATCHDOG)
txr->queue_status = IXGBE_QUEUE_HUNG;
- /* With a minimum free clear the depleted state bit. */
- if (txr->tx_avail > IXGBE_TX_CLEANUP_THRESHOLD)
- txr->queue_status &= ~IXGBE_QUEUE_DEPLETED;
-
if (txr->tx_avail == adapter->num_tx_desc) {
txr->queue_status = IXGBE_QUEUE_IDLE;
return false;
@@ -3928,7 +3943,7 @@ ixgbe_refresh_mbufs(struct rx_ring *rxr,
{
struct adapter *adapter = rxr->adapter;
struct ixgbe_rx_buf *rxbuf;
- struct mbuf *mh, *mp;
+ struct mbuf *mp;
int i, j, error;
bool refreshed = false;
@@ -3939,36 +3954,7 @@ ixgbe_refresh_mbufs(struct rx_ring *rxr,
while (j != limit) {
rxbuf = &rxr->rx_buffers[i];
- if (rxr->hdr_split == FALSE)
- goto no_split;
-
- if (rxbuf->m_head == NULL) {
- mh = m_gethdr(M_DONTWAIT, MT_DATA);
- if (mh == NULL)
- goto update;
- } else
- mh = rxbuf->m_head;
-
- mh->m_pkthdr.len = mh->m_len = MHLEN;
- mh->m_len = MHLEN;
- mh->m_flags |= M_PKTHDR;
- /* Get the memory mapping */
- error = bus_dmamap_load_mbuf(rxr->htag->dt_dmat,
- rxbuf->hmap, mh, BUS_DMA_NOWAIT);
- if (error != 0) {
- printf("Refresh mbufs: hdr dmamap load"
- " failure - %d\n", error);
- m_free(mh);
- rxbuf->m_head = NULL;
- goto update;
- }
- rxbuf->m_head = mh;
- ixgbe_dmamap_sync(rxr->htag, rxbuf->hmap, BUS_DMASYNC_PREREAD);
- rxr->rx_base[i].read.hdr_addr =
- htole64(rxbuf->hmap->dm_segs[0].ds_addr);
-
-no_split:
- if (rxbuf->m_pack == NULL) {
+ if (rxbuf->buf == NULL) {
mp = ixgbe_getjcl(&adapter->jcl_head, M_DONTWAIT,
MT_DATA, M_PKTHDR, adapter->rx_mbuf_sz);
if (mp == NULL) {
@@ -3976,24 +3962,32 @@ no_split:
goto update;
}
} else
- mp = rxbuf->m_pack;
+ mp = rxbuf->buf;
mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
- /* Get the memory mapping */
- error = bus_dmamap_load_mbuf(rxr->ptag->dt_dmat,
- rxbuf->pmap, mp, BUS_DMA_NOWAIT);
- if (error != 0) {
- printf("Refresh mbufs: payload dmamap load"
- " failure - %d\n", error);
- m_free(mp);
- rxbuf->m_pack = NULL;
- goto update;
- }
- rxbuf->m_pack = mp;
- bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap,
- 0, mp->m_pkthdr.len, BUS_DMASYNC_PREREAD);
- rxr->rx_base[i].read.pkt_addr =
- htole64(rxbuf->pmap->dm_segs[0].ds_addr);
+ /* If we're dealing with an mbuf that was copied rather
+ * than replaced, there's no need to go through busdma.
+ */
+ if ((rxbuf->flags & IXGBE_RX_COPY) == 0) {
+ /* Get the memory mapping */
+ error = bus_dmamap_load_mbuf(rxr->tag->dt_dmat,
+ rxbuf->map, mp, BUS_DMA_NOWAIT);
+ if (error != 0) {
+ printf("Refresh mbufs: payload dmamap load"
+ " failure - %d\n", error);
+ m_free(mp);
+ rxbuf->buf = NULL;
+ goto update;
+ }
+ rxbuf->buf = mp;
+ bus_dmamap_sync(rxr->tag->dt_dmat, rxbuf->map,
+ 0, mp->m_pkthdr.len, BUS_DMASYNC_PREREAD);
+ rxbuf->addr = rxr->rx_base[i].read.pkt_addr =
+ htole64(rxbuf->map->dm_segs[0].ds_addr);
+ } else {
+ rxr->rx_base[i].read.pkt_addr = rxbuf->addr;
+ rxbuf->flags &= ~IXGBE_RX_COPY;
+ }
refreshed = true;
/* Next is precalculated */
@@ -4036,38 +4030,21 @@ ixgbe_allocate_receive_buffers(struct rx
if ((error = ixgbe_dma_tag_create(adapter->osdep.dmat, /* parent */
1, 0, /* alignment, bounds */
- MSIZE, /* maxsize */
- 1, /* nsegments */
- MSIZE, /* maxsegsize */
- 0, /* flags */
- &rxr->htag))) {
- aprint_error_dev(dev, "Unable to create RX DMA tag\n");
- goto fail;
- }
-
- if ((error = ixgbe_dma_tag_create(adapter->osdep.dmat, /* parent */
- 1, 0, /* alignment, bounds */
MJUM16BYTES, /* maxsize */
1, /* nsegments */
MJUM16BYTES, /* maxsegsize */
0, /* flags */
- &rxr->ptag))) {
+ &rxr->tag))) {
aprint_error_dev(dev, "Unable to create RX DMA tag\n");
goto fail;
}
for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) {
rxbuf = &rxr->rx_buffers[i];
- error = ixgbe_dmamap_create(rxr->htag,
- BUS_DMA_NOWAIT, &rxbuf->hmap);
- if (error) {
- aprint_error_dev(dev, "Unable to create RX head map\n");
- goto fail;
- }
- error = ixgbe_dmamap_create(rxr->ptag,
- BUS_DMA_NOWAIT, &rxbuf->pmap);
+ error = ixgbe_dmamap_create(rxr->tag,
+ BUS_DMA_NOWAIT, &rxbuf->map);
if (error) {
- aprint_error_dev(dev, "Unable to create RX pkt map\n");
+ aprint_error_dev(dev, "Unable to create RX dma map\n");
goto fail;
}
}
@@ -4097,6 +4074,10 @@ ixgbe_rsc_count(union ixgbe_adv_rx_desc
* for an RX ring, this is toggled by the LRO capability
* even though it is transparent to the stack.
*
+ * NOTE: since this HW feature only works with IPV4 and
+ * our testing has shown soft LRO to be as effective
+ * I have decided to disable this by default.
+ *
**********************************************************************/
static void
ixgbe_setup_hw_rsc(struct rx_ring *rxr)
@@ -4105,6 +4086,13 @@ ixgbe_setup_hw_rsc(struct rx_ring *rxr)
struct ixgbe_hw *hw = &adapter->hw;
u32 rscctrl, rdrxctl;
+ /* If turning LRO/RSC off we need to disable it */
+ if ((adapter->ifp->if_capenable & IFCAP_LRO) == 0) {
+ rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me));
+ rscctrl &= ~IXGBE_RSCCTL_RSCEN;
+ return;
+ }
+
rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
#ifdef DEV_NETMAP /* crcstrip is optional in netmap */
@@ -4154,23 +4142,15 @@ ixgbe_free_receive_ring(struct rx_ring *
adapter = rxr->adapter;
for (i = 0; i < adapter->num_rx_desc; i++) {
rxbuf = &rxr->rx_buffers[i];
- if (rxbuf->m_head != NULL) {
- ixgbe_dmamap_sync(rxr->htag, rxbuf->hmap,
- BUS_DMASYNC_POSTREAD);
- ixgbe_dmamap_unload(rxr->htag, rxbuf->hmap);
- rxbuf->m_head->m_flags |= M_PKTHDR;
- m_freem(rxbuf->m_head);
- }
- if (rxbuf->m_pack != NULL) {
- bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap,
- 0, rxbuf->m_pack->m_pkthdr.len,
+ if (rxbuf->buf != NULL) {
+ bus_dmamap_sync(rxr->tag->dt_dmat, rxbuf->map,
+ 0, rxbuf->buf->m_pkthdr.len,
BUS_DMASYNC_POSTREAD);
- ixgbe_dmamap_unload(rxr->ptag, rxbuf->pmap);
- rxbuf->m_pack->m_flags |= M_PKTHDR;
- m_freem(rxbuf->m_pack);
+ ixgbe_dmamap_unload(rxr->tag, rxbuf->map);
+ rxbuf->buf->m_flags |= M_PKTHDR;
+ m_freem(rxbuf->buf);
+ rxbuf->buf = NULL;
}
- rxbuf->m_head = NULL;
- rxbuf->m_pack = NULL;
}
}
@@ -4184,9 +4164,9 @@ static int
ixgbe_setup_receive_ring(struct rx_ring *rxr)
{
struct adapter *adapter;
- struct ifnet *ifp;
struct ixgbe_rx_buf *rxbuf;
#ifdef LRO
+ struct ifnet *ifp;
struct lro_ctrl *lro = &rxr->lro;
#endif /* LRO */
int rsize, error = 0;
@@ -4196,7 +4176,9 @@ ixgbe_setup_receive_ring(struct rx_ring
#endif /* DEV_NETMAP */
adapter = rxr->adapter;
+#ifdef LRO
ifp = adapter->ifp;
+#endif /* LRO */
/* Clear the ring contents */
IXGBE_RX_LOCK(rxr);
@@ -4216,18 +4198,14 @@ ixgbe_setup_receive_ring(struct rx_ring
/* Now reinitialize our supply of jumbo mbufs. The number
* or size of jumbo mbufs may have changed.
*/
- ixgbe_jcl_reinit(&adapter->jcl_head, rxr->ptag->dt_dmat,
+ ixgbe_jcl_reinit(&adapter->jcl_head, rxr->tag->dt_dmat,
2 * adapter->num_rx_desc, adapter->rx_mbuf_sz);
IXGBE_RX_LOCK(rxr);
- /* Configure header split? */
- if (ixgbe_header_split)
- rxr->hdr_split = TRUE;
-
/* Now replenish the mbufs */
for (int j = 0; j != adapter->num_rx_desc; ++j) {
- struct mbuf *mh, *mp;
+ struct mbuf *mp;
rxbuf = &rxr->rx_buffers[j];
#ifdef DEV_NETMAP
@@ -4244,60 +4222,30 @@ ixgbe_setup_receive_ring(struct rx_ring
void *addr;
addr = PNMB(slot + sj, &paddr);
- netmap_load_map(rxr->ptag, rxbuf->pmap, addr);
+ netmap_load_map(rxr->tag, rxbuf->map, addr);
/* Update descriptor */
rxr->rx_base[j].read.pkt_addr = htole64(paddr);
continue;
}
#endif /* DEV_NETMAP */
- /*
- ** Don't allocate mbufs if not
- ** doing header split, its wasteful
- */
- if (rxr->hdr_split == FALSE)
- goto skip_head;
-
- /* First the header */
- rxbuf->m_head = m_gethdr(M_DONTWAIT, MT_DATA);
- if (rxbuf->m_head == NULL) {
- error = ENOBUFS;
- goto fail;
- }
- m_adj(rxbuf->m_head, ETHER_ALIGN);
- mh = rxbuf->m_head;
- mh->m_len = mh->m_pkthdr.len = MHLEN;
- mh->m_flags |= M_PKTHDR;
- /* Get the memory mapping */
- error = bus_dmamap_load_mbuf(rxr->htag->dt_dmat,
- rxbuf->hmap, rxbuf->m_head, BUS_DMA_NOWAIT);
- if (error != 0) /* Nothing elegant to do here */
- goto fail;
- bus_dmamap_sync(rxr->htag->dt_dmat, rxbuf->hmap,
- 0, mh->m_pkthdr.len, BUS_DMASYNC_PREREAD);
- /* Update descriptor */
- rxr->rx_base[j].read.hdr_addr =
- htole64(rxbuf->hmap->dm_segs[0].ds_addr);
-
-skip_head:
- /* Now the payload cluster */
- rxbuf->m_pack = ixgbe_getjcl(&adapter->jcl_head, M_DONTWAIT,
+ rxbuf->buf = ixgbe_getjcl(&adapter->jcl_head, M_DONTWAIT,
MT_DATA, M_PKTHDR, adapter->rx_mbuf_sz);
- if (rxbuf->m_pack == NULL) {
+ if (rxbuf->buf == NULL) {
error = ENOBUFS;
goto fail;
}
- mp = rxbuf->m_pack;
+ mp = rxbuf->buf;
mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
/* Get the memory mapping */
- error = bus_dmamap_load_mbuf(rxr->ptag->dt_dmat,
- rxbuf->pmap, mp, BUS_DMA_NOWAIT);
+ error = bus_dmamap_load_mbuf(rxr->tag->dt_dmat,
+ rxbuf->map, mp, BUS_DMA_NOWAIT);
if (error != 0)
goto fail;
- bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap,
+ bus_dmamap_sync(rxr->tag->dt_dmat, rxbuf->map,
0, adapter->rx_mbuf_sz, BUS_DMASYNC_PREREAD);
/* Update descriptor */
rxr->rx_base[j].read.pkt_addr =
- htole64(rxbuf->pmap->dm_segs[0].ds_addr);
+ htole64(rxbuf->map->dm_segs[0].ds_addr);
}
@@ -4305,7 +4253,7 @@ skip_head:
rxr->next_to_check = 0;
rxr->next_to_refresh = 0;
rxr->lro_enabled = FALSE;
- rxr->rx_split_packets.ev_count = 0;
+ rxr->rx_copies.ev_count = 0;
rxr->rx_bytes.ev_count = 0;
rxr->discard = FALSE;
rxr->vtag_strip = FALSE;
@@ -4315,12 +4263,8 @@ skip_head:
/*
** Now set up the LRO interface:
- ** 82598 uses software LRO, the
- ** 82599 and X540 use a hardware assist.
*/
- if ((adapter->hw.mac.type != ixgbe_mac_82598EB) &&
- (ifp->if_capenable & IFCAP_RXCSUM) &&
- (ifp->if_capenable & IFCAP_LRO))
+ if (ixgbe_rsc_enable)
ixgbe_setup_hw_rsc(rxr);
#ifdef LRO
else if (ifp->if_capenable & IFCAP_LRO) {
@@ -4443,14 +4387,7 @@ ixgbe_initialize_receive_units(struct ad
srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
srrctl |= bufsz;
- if (rxr->hdr_split) {
- /* Use a standard mbuf for the header */
- srrctl |= ((IXGBE_RX_HDR <<
- IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT)
- & IXGBE_SRRCTL_BSIZEHDR_MASK);
- srrctl |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
- } else
- srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
+ srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
/* Setup the HW Rx Head and Tail Descriptor Pointers */
@@ -4561,30 +4498,18 @@ ixgbe_free_receive_buffers(struct rx_rin
if (rxr->rx_buffers != NULL) {
for (int i = 0; i < adapter->num_rx_desc; i++) {
rxbuf = &rxr->rx_buffers[i];
- if (rxbuf->m_head != NULL) {
- ixgbe_dmamap_sync(rxr->htag, rxbuf->hmap,
- BUS_DMASYNC_POSTREAD);
- ixgbe_dmamap_unload(rxr->htag, rxbuf->hmap);
- rxbuf->m_head->m_flags |= M_PKTHDR;
- m_freem(rxbuf->m_head);
- }
- if (rxbuf->m_pack != NULL) {
- bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap,
- 0, rxbuf->m_pack->m_pkthdr.len,
+ if (rxbuf->buf != NULL) {
+ bus_dmamap_sync(rxr->tag->dt_dmat, rxbuf->map,
+ 0, rxbuf->buf->m_pkthdr.len,
BUS_DMASYNC_POSTREAD);
- ixgbe_dmamap_unload(rxr->ptag, rxbuf->pmap);
- rxbuf->m_pack->m_flags |= M_PKTHDR;
- m_freem(rxbuf->m_pack);
+ ixgbe_dmamap_unload(rxr->tag, rxbuf->map);
+ rxbuf->buf->m_flags |= M_PKTHDR;
+ m_freem(rxbuf->buf);
}
- rxbuf->m_head = NULL;
- rxbuf->m_pack = NULL;
- if (rxbuf->hmap != NULL) {
- ixgbe_dmamap_destroy(rxr->htag, rxbuf->hmap);
- rxbuf->hmap = NULL;
- }
- if (rxbuf->pmap != NULL) {
- ixgbe_dmamap_destroy(rxr->ptag, rxbuf->pmap);
- rxbuf->pmap = NULL;
+ rxbuf->buf = NULL;
+ if (rxbuf->map != NULL) {
+ ixgbe_dmamap_destroy(rxr->tag, rxbuf->map);
+ rxbuf->map = NULL;
}
}
if (rxr->rx_buffers != NULL) {
@@ -4593,13 +4518,9 @@ ixgbe_free_receive_buffers(struct rx_rin
}
}
- if (rxr->htag != NULL) {
- ixgbe_dma_tag_destroy(rxr->htag);
- rxr->htag = NULL;
- }
- if (rxr->ptag != NULL) {
- ixgbe_dma_tag_destroy(rxr->ptag);
- rxr->ptag = NULL;
+ if (rxr->tag != NULL) {
+ ixgbe_dma_tag_destroy(rxr->tag);
+ rxr->tag = NULL;
}
return;
@@ -4671,16 +4592,11 @@ ixgbe_rx_discard(struct rx_ring *rxr, in
** the normal refresh path to get new buffers
** and mapping.
*/
- if (rbuf->m_head) {
- m_free(rbuf->m_head);
- rbuf->m_head = NULL;
+ if (rbuf->buf) {
+ m_free(rbuf->buf);
+ rbuf->buf = NULL;
}
- if (rbuf->m_pack) {
- m_free(rbuf->m_pack);
- rbuf->m_pack = NULL;
- }
-
return;
}
@@ -4733,9 +4649,9 @@ ixgbe_rxeof(struct ix_queue *que, int co
}
#endif /* DEV_NETMAP */
for (i = rxr->next_to_check; count != 0;) {
- struct mbuf *sendmp, *mh, *mp;
+ struct mbuf *sendmp, *mp;
u32 rsc, ptype;
- u16 hlen, plen, hdr;
+ u16 len;
u16 vtag = 0;
bool eop;
@@ -4757,19 +4673,13 @@ ixgbe_rxeof(struct ix_queue *que, int co
rsc = 0;
cur->wb.upper.status_error = 0;
rbuf = &rxr->rx_buffers[i];
- mh = rbuf->m_head;
- mp = rbuf->m_pack;
+ mp = rbuf->buf;
- plen = le16toh(cur->wb.upper.length);
+ len = le16toh(cur->wb.upper.length);
ptype = le32toh(cur->wb.lower.lo_dword.data) &
IXGBE_RXDADV_PKTTYPE_MASK;
- hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info);
eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0);
- /* Process vlan info */
- if ((rxr->vtag_strip) && (staterr & IXGBE_RXD_STAT_VP))
- vtag = le16toh(cur->wb.upper.vlan);
-
/* Make sure bad packets are discarded */
if (((staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) ||
(rxr->discard)) {
@@ -4817,126 +4727,72 @@ ixgbe_rxeof(struct ix_queue *que, int co
prefetch(nbuf);
}
/*
- ** The header mbuf is ONLY used when header
- ** split is enabled, otherwise we get normal
- ** behavior, ie, both header and payload
- ** are DMA'd into the payload buffer.
- **
** Rather than using the fmp/lmp global pointers
** we now keep the head of a packet chain in the
** buffer struct and pass this along from one
** descriptor to the next, until we get EOP.
*/
- if (rxr->hdr_split && (rbuf->fmp == NULL)) {
- /* This must be an initial descriptor */
- hlen = (hdr & IXGBE_RXDADV_HDRBUFLEN_MASK) >>
- IXGBE_RXDADV_HDRBUFLEN_SHIFT;
- if (hlen > IXGBE_RX_HDR)
- hlen = IXGBE_RX_HDR;
- mh->m_len = hlen;
- mh->m_flags |= M_PKTHDR;
- mh->m_next = NULL;
- mh->m_pkthdr.len = mh->m_len;
- /* Null buf pointer so it is refreshed */
- rbuf->m_head = NULL;
- /*
- ** Check the payload length, this
- ** could be zero if its a small
- ** packet.
- */
- if (plen > 0) {
- mp->m_len = plen;
- mp->m_next = NULL;
- mp->m_flags &= ~M_PKTHDR;
- mh->m_next = mp;
- mh->m_pkthdr.len += mp->m_len;
- /* Null buf pointer so it is refreshed */
- rbuf->m_pack = NULL;
- rxr->rx_split_packets.ev_count++;
- }
- /*
- ** Now create the forward
- ** chain so when complete
- ** we wont have to.
- */
- if (eop == 0) {
- /* stash the chain head */
- nbuf->fmp = mh;
- /* Make forward chain */
- if (plen)
- mp->m_next = nbuf->m_pack;
- else
- mh->m_next = nbuf->m_pack;
- } else {
- /* Singlet, prepare to send */
- sendmp = mh;
- /* If hardware handled vtag */
- if (vtag) {
- /* XXX Do something reasonable on
- * error.
- */
-#if 0
- printf("%s.%d: VLAN_INPUT_TAG\n",
- __func__, __LINE__);
- Debugger();
-#endif
- VLAN_INPUT_TAG(ifp, sendmp, vtag,
- printf("%s: could not apply VLAN "
- "tag", __func__));
- }
- }
+ mp->m_len = len;
+ /*
+ ** See if there is a stored head
+ ** that determines what we are
+ */
+ sendmp = rbuf->fmp;
+
+ if (sendmp != NULL) { /* secondary frag */
+ rbuf->buf = rbuf->fmp = NULL;
+ mp->m_flags &= ~M_PKTHDR;
+ sendmp->m_pkthdr.len += mp->m_len;
} else {
/*
- ** Either no header split, or a
- ** secondary piece of a fragmented
- ** split packet.
- */
- mp->m_len = plen;
- /*
- ** See if there is a stored head
- ** that determines what we are
- */
- sendmp = rbuf->fmp;
- rbuf->m_pack = rbuf->fmp = NULL;
-
- if (sendmp != NULL) { /* secondary frag */
- mp->m_flags &= ~M_PKTHDR;
- sendmp->m_pkthdr.len += mp->m_len;
- } else {
- /* first desc of a non-ps chain */
- sendmp = mp;
- sendmp->m_flags |= M_PKTHDR;
- sendmp->m_pkthdr.len = mp->m_len;
- if (staterr & IXGBE_RXD_STAT_VP) {
- /* XXX Do something reasonable on
- * error.
- */
-#if 0
- printf("%s.%d: VLAN_INPUT_TAG\n",
- __func__, __LINE__);
- Debugger();
-#endif
- VLAN_INPUT_TAG(ifp, sendmp, vtag,
- printf("%s: could not apply VLAN "
- "tag", __func__));
+ * Optimize. This might be a small packet,
+ * maybe just a TCP ACK. Do a fast copy that
+ * is cache aligned into a new mbuf, and
+ * leave the old mbuf+cluster for re-use.
+ */
+ if (eop && len <= IXGBE_RX_COPY_LEN) {
+ sendmp = m_gethdr(M_DONTWAIT, MT_DATA);
+ if (sendmp != NULL) {
+ sendmp->m_data +=
+ IXGBE_RX_COPY_ALIGN;
+ ixgbe_bcopy(mp->m_data,
+ sendmp->m_data, len);
+ sendmp->m_len = len;
+ rxr->rx_copies.ev_count++;
+ rbuf->flags |= IXGBE_RX_COPY;
}
- }
- /* Pass the head pointer on */
- if (eop == 0) {
- nbuf->fmp = sendmp;
- sendmp = NULL;
- mp->m_next = nbuf->m_pack;
}
+ if (sendmp == NULL) {
+ rbuf->buf = rbuf->fmp = NULL;
+ sendmp = mp;
+ }
+
+ /* first desc of a non-ps chain */
+ sendmp->m_flags |= M_PKTHDR;
+ sendmp->m_pkthdr.len = mp->m_len;
}
++processed;
- /* Sending this frame? */
- if (eop) {
+ /* Pass the head pointer on */
+ if (eop == 0) {
+ nbuf->fmp = sendmp;
+ sendmp = NULL;
+ mp->m_next = nbuf->buf;
+ } else { /* Sending this frame */
sendmp->m_pkthdr.rcvif = ifp;
ifp->if_ipackets++;
rxr->rx_packets.ev_count++;
/* capture data for AIM */
rxr->bytes += sendmp->m_pkthdr.len;
rxr->rx_bytes.ev_count += sendmp->m_pkthdr.len;
+ /* Process vlan info */
+ if ((rxr->vtag_strip) &&
+ (staterr & IXGBE_RXD_STAT_VP))
+ vtag = le16toh(cur->wb.upper.vlan);
+ if (vtag) {
+ VLAN_INPUT_TAG(ifp, sendmp, vtag,
+ printf("%s: could not apply VLAN "
+ "tag", __func__));
+ }
if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) {
ixgbe_rx_checksum(staterr, sendmp, ptype,
&adapter->stats);
@@ -5143,7 +4999,7 @@ ixgbe_setup_vlan_hw_support(struct adapt
/* On 82599 the VLAN enable is per/queue in RXDCTL */
if (hw->mac.type != ixgbe_mac_82598EB) {
ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
- ctrl |= IXGBE_RXDCTL_VME;
+ ctrl |= IXGBE_RXDCTL_VME;
IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
}
rxr->vtag_strip = TRUE;
@@ -5588,6 +5444,7 @@ ixgbe_update_stats_counters(struct adapt
ifp->if_ibytes = adapter->stats.gorc.ev_count;
ifp->if_obytes = adapter->stats.gotc.ev_count;
ifp->if_imcasts = adapter->stats.mprc.ev_count;
+ ifp->if_omcasts = adapter->stats.mptc.ev_count;
ifp->if_collisions = 0;
/* Rx Errors */
@@ -5909,12 +5766,12 @@ ixgbe_add_hw_stats(struct adapter *adapt
NULL, adapter->queues[i].evnamebuf, "Queue Packets Received");
evcnt_attach_dynamic(&rxr->rx_bytes, EVCNT_TYPE_MISC,
NULL, adapter->queues[i].evnamebuf, "Queue Bytes Received");
+ evcnt_attach_dynamic(&rxr->rx_copies, EVCNT_TYPE_MISC,
+ NULL, adapter->queues[i].evnamebuf, "Copied RX Frames");
evcnt_attach_dynamic(&rxr->no_jmbuf, EVCNT_TYPE_MISC,
NULL, adapter->queues[i].evnamebuf, "Rx no jumbo mbuf");
evcnt_attach_dynamic(&rxr->rx_discarded, EVCNT_TYPE_MISC,
NULL, adapter->queues[i].evnamebuf, "Rx discarded");
- evcnt_attach_dynamic(&rxr->rx_split_packets, EVCNT_TYPE_MISC,
- NULL, adapter->queues[i].evnamebuf, "Rx split packets");
evcnt_attach_dynamic(&rxr->rx_irq, EVCNT_TYPE_MISC,
NULL, adapter->queues[i].evnamebuf, "Rx interrupts");
#ifdef LRO
@@ -6083,10 +5940,14 @@ ixgbe_set_flowcntl(SYSCTLFN_ARGS)
case ixgbe_fc_tx_pause:
case ixgbe_fc_full:
adapter->hw.fc.requested_mode = adapter->fc;
+ if (adapter->num_queues > 1)
+ ixgbe_disable_rx_drop(adapter);
break;
case ixgbe_fc_none:
default:
adapter->hw.fc.requested_mode = ixgbe_fc_none;
+ if (adapter->num_queues > 1)
+ ixgbe_enable_rx_drop(adapter);
}
/* Don't autoneg if forcing a value */
adapter->hw.fc.disable_fc_autoneg = TRUE;
@@ -6209,3 +6070,34 @@ ixgbe_set_thermal_test(SYSCTLFN_ARGS)
return (0);
}
+
+/*
+** Enable the hardware to drop packets when the buffer is
+** full. This is useful when multiqueue,so that no single
+** queue being full stalls the entire RX engine. We only
+** enable this when Multiqueue AND when Flow Control is
+** disabled.
+*/
+static void
+ixgbe_enable_rx_drop(struct adapter *adapter)
+{
+ struct ixgbe_hw *hw = &adapter->hw;
+
+ for (int i = 0; i < adapter->num_queues; i++) {
+ u32 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
+ srrctl |= IXGBE_SRRCTL_DROP_EN;
+ IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
+ }
+}
+
+static void
+ixgbe_disable_rx_drop(struct adapter *adapter)
+{
+ struct ixgbe_hw *hw = &adapter->hw;
+
+ for (int i = 0; i < adapter->num_queues; i++) {
+ u32 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
+ srrctl &= ~IXGBE_SRRCTL_DROP_EN;
+ IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
+ }
+}
Index: src/sys/dev/pci/ixgbe/ixgbe.h
diff -u src/sys/dev/pci/ixgbe/ixgbe.h:1.5 src/sys/dev/pci/ixgbe/ixgbe.h:1.6
--- src/sys/dev/pci/ixgbe/ixgbe.h:1.5 Thu Apr 2 09:26:55 2015
+++ src/sys/dev/pci/ixgbe/ixgbe.h Tue Apr 14 07:17:06 2015
@@ -58,8 +58,8 @@
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
-/*$FreeBSD: head/sys/dev/ixgbe/ixgbe.h 234620 2012-04-23 22:05:09Z bz $*/
-/*$NetBSD: ixgbe.h,v 1.5 2015/04/02 09:26:55 msaitoh Exp $*/
+/*$FreeBSD: head/sys/dev/ixgbe/ixgbe.h 243716 2012-11-30 22:33:21Z jfv $*/
+/*$NetBSD: ixgbe.h,v 1.6 2015/04/14 07:17:06 msaitoh Exp $*/
#ifndef _IXGBE_H_
@@ -173,6 +173,19 @@
#define IXGBE_FC_HI 0x20000
#define IXGBE_FC_LO 0x10000
+/*
+ * Used for optimizing small rx mbufs. Effort is made to keep the copy
+ * small and aligned for the CPU L1 cache.
+ *
+ * MHLEN is typically 168 bytes, giving us 8-byte alignment. Getting
+ * 32 byte alignment needed for the fast bcopy results in 8 bytes being
+ * wasted. Getting 64 byte alignment, which _should_ be ideal for
+ * modern Intel CPUs, results in 40 bytes wasted and a significant drop
+ * in observed efficiency of the optimization, 97.9% -> 81.8%.
+ */
+#define IXGBE_RX_COPY_LEN 160
+#define IXGBE_RX_COPY_ALIGN (MHLEN - IXGBE_RX_COPY_LEN)
+
/* Keep older OS drivers building... */
#if !defined(SYSCTL_ADD_UQUAD)
#define SYSCTL_ADD_UQUAD SYSCTL_ADD_QUAD
@@ -204,10 +217,6 @@
#define IXGBE_VFTA_SIZE 128
#define IXGBE_BR_SIZE 4096
#define IXGBE_QUEUE_MIN_FREE 32
-#define IXGBE_QUEUE_IDLE 1
-#define IXGBE_QUEUE_WORKING 2
-#define IXGBE_QUEUE_HUNG 4
-#define IXGBE_QUEUE_DEPLETED 8
/* Offload bits in mbuf flag */
#define M_CSUM_OFFLOAD \
@@ -246,11 +255,12 @@ struct ixgbe_tx_buf {
};
struct ixgbe_rx_buf {
- struct mbuf *m_head;
- struct mbuf *m_pack;
+ struct mbuf *buf;
struct mbuf *fmp;
- bus_dmamap_t hmap;
- bus_dmamap_t pmap;
+ bus_dmamap_t map;
+ u_int flags;
+#define IXGBE_RX_COPY 0x01
+ uint64_t addr;
};
/*
@@ -291,7 +301,11 @@ struct tx_ring {
struct adapter *adapter;
kmutex_t tx_mtx;
u32 me;
- int queue_status;
+ enum {
+ IXGBE_QUEUE_IDLE,
+ IXGBE_QUEUE_WORKING,
+ IXGBE_QUEUE_HUNG,
+ } queue_status;
struct timeval watchdog_time;
union ixgbe_adv_tx_desc *tx_base;
struct ixgbe_dma_alloc txdma;
@@ -304,6 +318,7 @@ struct tx_ring {
char mtx_name[16];
#if __FreeBSD_version >= 800000
struct buf_ring *br;
+ struct task txq_task;
#endif
#ifdef IXGBE_FDIR
u16 atr_sample;
@@ -330,7 +345,6 @@ struct rx_ring {
struct lro_ctrl lro;
#endif /* LRO */
bool lro_enabled;
- bool hdr_split;
bool hw_rsc;
bool discard;
bool vtag_strip;
@@ -338,15 +352,14 @@ struct rx_ring {
u32 next_to_check;
char mtx_name[16];
struct ixgbe_rx_buf *rx_buffers;
- ixgbe_dma_tag_t *htag;
- ixgbe_dma_tag_t *ptag;
+ ixgbe_dma_tag_t *tag;
u32 bytes; /* Used for AIM calc */
u32 packets;
/* Soft stats */
struct evcnt rx_irq;
- struct evcnt rx_split_packets;
+ struct evcnt rx_copies;
struct evcnt rx_packets;
struct evcnt rx_bytes;
struct evcnt rx_discarded;
Index: src/sys/dev/pci/ixgbe/ixgbe_osdep.h
diff -u src/sys/dev/pci/ixgbe/ixgbe_osdep.h:1.5 src/sys/dev/pci/ixgbe/ixgbe_osdep.h:1.6
--- src/sys/dev/pci/ixgbe/ixgbe_osdep.h:1.5 Thu Apr 2 09:26:55 2015
+++ src/sys/dev/pci/ixgbe/ixgbe_osdep.h Tue Apr 14 07:17:06 2015
@@ -30,8 +30,8 @@
POSSIBILITY OF SUCH DAMAGE.
******************************************************************************/
-/*$FreeBSD: head/sys/dev/ixgbe/ixgbe_osdep.h 238149 2012-07-05 20:51:44Z jfv $*/
-/*$NetBSD: ixgbe_osdep.h,v 1.5 2015/04/02 09:26:55 msaitoh Exp $*/
+/*$FreeBSD: head/sys/dev/ixgbe/ixgbe_osdep.h 240466 2012-09-13 14:40:24Z eadler $*/
+/*$NetBSD: ixgbe_osdep.h,v 1.6 2015/04/14 07:17:06 msaitoh Exp $*/
#ifndef _IXGBE_OS_H_
#define _IXGBE_OS_H_
@@ -67,12 +67,17 @@
#define DEBUGOUT1(S,A) printf(S "\n",A)
#define DEBUGOUT2(S,A,B) printf(S "\n",A,B)
#define DEBUGOUT3(S,A,B,C) printf(S "\n",A,B,C)
+ #define DEBUGOUT4(S,A,B,C,D) printf(S "\n",A,B,C,D)
+ #define DEBUGOUT5(S,A,B,C,D,E) printf(S "\n",A,B,C,D,E)
+ #define DEBUGOUT6(S,A,B,C,D,E,F) printf(S "\n",A,B,C,D,E,F)
#define DEBUGOUT7(S,A,B,C,D,E,F,G) printf(S "\n",A,B,C,D,E,F,G)
#else
#define DEBUGOUT(S) do { } while (/*CONSTCOND*/false)
#define DEBUGOUT1(S,A) do { } while (/*CONSTCOND*/false)
#define DEBUGOUT2(S,A,B) do { } while (/*CONSTCOND*/false)
#define DEBUGOUT3(S,A,B,C) do { } while (/*CONSTCOND*/false)
+ #define DEBUGOUT4(S,A,B,C,D) do { } while (/*CONSTCOND*/false)
+ #define DEBUGOUT5(S,A,B,C,D,E) do { } while (/*CONSTCOND*/false)
#define DEBUGOUT6(S,A,B,C,D,E,F) \
do { } while (/*CONSTCOND*/false)
#define DEBUGOUT7(S,A,B,C,D,E,F,G) \
@@ -132,6 +137,25 @@ void prefetch(void *x)
#define prefetch(x)
#endif
+/*
+ * Optimized bcopy thanks to Luigi Rizzo's investigative work. Assumes
+ * non-overlapping regions and 32-byte padding on both src and dst.
+ */
+static __inline int
+ixgbe_bcopy(void *_src, void *_dst, int l)
+{
+ uint64_t *src = _src;
+ uint64_t *dst = _dst;
+
+ for (; l > 0; l -= 32) {
+ *dst++ = *src++;
+ *dst++ = *src++;
+ *dst++ = *src++;
+ *dst++ = *src++;
+ }
+ return (0);
+}
+
struct ixgbe_osdep
{
struct ethercom ec;
Index: src/sys/dev/pci/ixgbe/ixgbe_82599.c
diff -u src/sys/dev/pci/ixgbe/ixgbe_82599.c:1.7 src/sys/dev/pci/ixgbe/ixgbe_82599.c:1.8
--- src/sys/dev/pci/ixgbe/ixgbe_82599.c:1.7 Thu Apr 2 09:26:55 2015
+++ src/sys/dev/pci/ixgbe/ixgbe_82599.c Tue Apr 14 07:17:06 2015
@@ -30,8 +30,8 @@
POSSIBILITY OF SUCH DAMAGE.
******************************************************************************/
-/*$FreeBSD: head/sys/dev/ixgbe/ixgbe_82599.c 238149 2012-07-05 20:51:44Z jfv $*/
-/*$NetBSD: ixgbe_82599.c,v 1.7 2015/04/02 09:26:55 msaitoh Exp $*/
+/*$FreeBSD: head/sys/dev/ixgbe/ixgbe_82599.c 240155 2012-09-06 02:07:58Z kevlo $*/
+/*$NetBSD: ixgbe_82599.c,v 1.8 2015/04/14 07:17:06 msaitoh Exp $*/
#include "ixgbe_type.h"
#include "ixgbe_82599.h"
@@ -870,12 +870,13 @@ s32 ixgbe_setup_mac_link_82599(struct ix
link_mode == IXGBE_AUTOC_LMS_KX4_KX_KR_SGMII) {
/* Set KX4/KX/KR support according to speed requested */
autoc &= ~(IXGBE_AUTOC_KX4_KX_SUPP_MASK | IXGBE_AUTOC_KR_SUPP);
- if (speed & IXGBE_LINK_SPEED_10GB_FULL)
+ if (speed & IXGBE_LINK_SPEED_10GB_FULL) {
if (orig_autoc & IXGBE_AUTOC_KX4_SUPP)
autoc |= IXGBE_AUTOC_KX4_SUPP;
if ((orig_autoc & IXGBE_AUTOC_KR_SUPP) &&
(hw->phy.smart_speed_active == FALSE))
autoc |= IXGBE_AUTOC_KR_SUPP;
+ }
if (speed & IXGBE_LINK_SPEED_1GB_FULL)
autoc |= IXGBE_AUTOC_KX_SUPP;
} else if ((pma_pmd_1g == IXGBE_AUTOC_1G_SFI) &&
Index: src/sys/dev/pci/ixgbe/ixv.c
diff -u src/sys/dev/pci/ixgbe/ixv.c:1.6 src/sys/dev/pci/ixgbe/ixv.c:1.7
--- src/sys/dev/pci/ixgbe/ixv.c:1.6 Thu Apr 2 09:26:55 2015
+++ src/sys/dev/pci/ixgbe/ixv.c Tue Apr 14 07:17:06 2015
@@ -30,8 +30,8 @@
POSSIBILITY OF SUCH DAMAGE.
******************************************************************************/
-/*$FreeBSD: head/sys/dev/ixgbe/ixv.c 238149 2012-07-05 20:51:44Z jfv $*/
-/*$NetBSD: ixv.c,v 1.6 2015/04/02 09:26:55 msaitoh Exp $*/
+/*$FreeBSD: head/sys/dev/ixgbe/ixv.c 241917 2012-10-22 22:29:48Z eadler $*/
+/*$NetBSD: ixv.c,v 1.7 2015/04/14 07:17:06 msaitoh Exp $*/
#include "opt_inet.h"
#include "opt_inet6.h"
@@ -706,7 +706,9 @@ ixv_mq_start_locked(struct ifnet *ifp, s
break;
}
enqueued++;
- drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags);
+ ifp->if_obytes += next->m_pkthdr.len;
+ if (next->m_flags & M_MCAST)
+ ifp->if_omcasts++;
/* Send a copy of the frame to the BPF listener */
ETHER_BPF_MTAP(ifp, next);
if ((ifp->if_flags & IFF_RUNNING) == 0)
