Module Name: src Committed By: msaitoh Date: Wed Feb 1 10:47:13 UTC 2017
Modified Files: src/sys/dev/pci/ixgbe: ix_txrx.c ixgbe.c ixgbe.h ixgbe_netbsd.h ixv.c Log Message: TX multiqueue. If you want to disable it, enable IXGBE_LEGACY_TX in ixgbe_netbsd.h To generate a diff of this commit: cvs rdiff -u -r1.17 -r1.18 src/sys/dev/pci/ixgbe/ix_txrx.c cvs rdiff -u -r1.66 -r1.67 src/sys/dev/pci/ixgbe/ixgbe.c cvs rdiff -u -r1.21 -r1.22 src/sys/dev/pci/ixgbe/ixgbe.h cvs rdiff -u -r1.5 -r1.6 src/sys/dev/pci/ixgbe/ixgbe_netbsd.h cvs rdiff -u -r1.34 -r1.35 src/sys/dev/pci/ixgbe/ixv.c Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files.
Modified files: Index: src/sys/dev/pci/ixgbe/ix_txrx.c diff -u src/sys/dev/pci/ixgbe/ix_txrx.c:1.17 src/sys/dev/pci/ixgbe/ix_txrx.c:1.18 --- src/sys/dev/pci/ixgbe/ix_txrx.c:1.17 Mon Jan 30 05:02:43 2017 +++ src/sys/dev/pci/ixgbe/ix_txrx.c Wed Feb 1 10:47:13 2017 @@ -59,7 +59,7 @@ * POSSIBILITY OF SUCH DAMAGE. */ /*$FreeBSD: head/sys/dev/ixgbe/ix_txrx.c 301538 2016-06-07 04:51:50Z sephe $*/ -/*$NetBSD: ix_txrx.c,v 1.17 2017/01/30 05:02:43 msaitoh Exp $*/ +/*$NetBSD: ix_txrx.c,v 1.18 2017/02/01 10:47:13 msaitoh Exp $*/ #include "opt_inet.h" #include "opt_inet6.h" @@ -126,7 +126,6 @@ static __inline void ixgbe_rx_input(stru static void ixgbe_setup_hw_rsc(struct rx_ring *); -#ifdef IXGBE_LEGACY_TX /********************************************************************* * Transmit entry point * @@ -204,7 +203,7 @@ ixgbe_start(struct ifnet *ifp) return; } -#else /* ! IXGBE_LEGACY_TX */ +#ifndef IXGBE_LEGACY_TX /* ** Multiqueue Transmit Entry Point @@ -214,7 +213,6 @@ int ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m) { struct adapter *adapter = ifp->if_softc; - struct ix_queue *que; struct tx_ring *txr; int i, err = 0; #ifdef RSS @@ -228,6 +226,7 @@ ixgbe_mq_start(struct ifnet *ifp, struct * If everything is setup correctly, it should be the * same bucket that the current CPU we're on is. */ +#if 0 #if __FreeBSD_version < 1100054 if (m->m_flags & M_FLOWID) { #else @@ -244,26 +243,29 @@ ixgbe_mq_start(struct ifnet *ifp, struct "(%d)\n", bucket_id, adapter->num_queues); #endif } else -#endif +#endif /* RSS */ i = m->m_pkthdr.flowid % adapter->num_queues; } else - i = curcpu % adapter->num_queues; +#endif + i = cpu_index(curcpu()) % adapter->num_queues; /* Check for a hung queue and pick alternative */ if (((1 << i) & adapter->active_queues) == 0) - i = ffsl(adapter->active_queues); + i = ffs64(adapter->active_queues); txr = &adapter->tx_rings[i]; - que = &adapter->queues[i]; - err = drbr_enqueue(ifp, txr->br, m); - if (err) + err = pcq_put(txr->txr_interq, m); + if (err == false) { + m_freem(m); + txr->pcq_drops.ev_count++; return (err); + } if (IXGBE_TX_TRYLOCK(txr)) { ixgbe_mq_start_locked(ifp, txr); IXGBE_TX_UNLOCK(txr); } else - softint_schedule(txr->txq_si); + softint_schedule(txr->txr_si); return (0); } @@ -280,26 +282,12 @@ ixgbe_mq_start_locked(struct ifnet *ifp, return (ENETDOWN); /* Process the queue */ -#if __FreeBSD_version < 901504 - next = drbr_dequeue(ifp, txr->br); - while (next != NULL) { - if ((err = ixgbe_xmit(txr, &next)) != 0) { - if (next != NULL) - err = drbr_enqueue(ifp, txr->br, next); -#else - while ((next = drbr_peek(ifp, txr->br)) != NULL) { - if ((err = ixgbe_xmit(txr, &next)) != 0) { - if (next == NULL) { - drbr_advance(ifp, txr->br); - } else { - drbr_putback(ifp, txr->br, next); - } -#endif + while ((next = pcq_get(txr->txr_interq)) != NULL) { + if ((err = ixgbe_xmit(txr, next)) != 0) { + m_freem(next); + /* All errors are counted in ixgbe_xmit() */ break; } -#if __FreeBSD_version >= 901504 - drbr_advance(ifp, txr->br); -#endif enqueued++; #if 0 // this is VF-only #if __FreeBSD_version >= 1100036 @@ -311,14 +299,11 @@ ixgbe_mq_start_locked(struct ifnet *ifp, if (txr->tail < IXGBE_TDT(0) && next->m_flags & M_MCAST) if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1); #endif -#endif +#endif /* 0 */ /* Send a copy of the frame to the BPF listener */ bpf_mtap(ifp, next); if ((ifp->if_flags & IFF_RUNNING) == 0) break; -#if __FreeBSD_version < 901504 - next = drbr_dequeue(ifp, txr->br); -#endif } if (txr->tx_avail < IXGBE_TX_CLEANUP_THRESHOLD) @@ -331,36 +316,18 @@ ixgbe_mq_start_locked(struct ifnet *ifp, * Called from a taskqueue to drain queued transmit packets. */ void -ixgbe_deferred_mq_start(void *arg, int pending) +ixgbe_deferred_mq_start(void *arg) { struct tx_ring *txr = arg; struct adapter *adapter = txr->adapter; struct ifnet *ifp = adapter->ifp; IXGBE_TX_LOCK(txr); - if (!drbr_empty(ifp, txr->br)) + if (pcq_peek(txr->txr_interq) != NULL) ixgbe_mq_start_locked(ifp, txr); IXGBE_TX_UNLOCK(txr); } -/* - * Flush all ring buffers - */ -void -ixgbe_qflush(struct ifnet *ifp) -{ - struct adapter *adapter = ifp->if_softc; - struct tx_ring *txr = adapter->tx_rings; - struct mbuf *m; - - for (int i = 0; i < adapter->num_queues; i++, txr++) { - IXGBE_TX_LOCK(txr); - while ((m = buf_ring_dequeue_sc(txr->br)) != NULL) - m_freem(m); - IXGBE_TX_UNLOCK(txr); - } - if_qflush(ifp); -} #endif /* IXGBE_LEGACY_TX */ @@ -724,8 +691,13 @@ ixgbe_free_transmit_buffers(struct tx_ri } } #ifndef IXGBE_LEGACY_TX - if (txr->br != NULL) - buf_ring_free(txr->br, M_DEVBUF); + if (txr->txr_interq != NULL) { + struct mbuf *m; + + while ((m = pcq_get(txr->txr_interq)) != NULL) + m_freem(m); + pcq_destroy(txr->txr_interq); + } #endif if (txr->tx_buffers != NULL) { free(txr->tx_buffers, M_DEVBUF); @@ -2325,9 +2297,8 @@ ixgbe_allocate_queues(struct adapter *ad } #ifndef IXGBE_LEGACY_TX /* Allocate a buf ring */ - txr->br = buf_ring_alloc(IXGBE_BR_SIZE, M_DEVBUF, - M_WAITOK, &txr->tx_mtx); - if (txr->br == NULL) { + txr->txr_interq = pcq_create(IXGBE_BR_SIZE, KM_SLEEP); + if (txr->txr_interq == NULL) { aprint_error_dev(dev, "Critical Failure setting up buf ring\n"); error = ENOMEM; Index: src/sys/dev/pci/ixgbe/ixgbe.c diff -u src/sys/dev/pci/ixgbe/ixgbe.c:1.66 src/sys/dev/pci/ixgbe/ixgbe.c:1.67 --- src/sys/dev/pci/ixgbe/ixgbe.c:1.66 Wed Jan 25 13:08:31 2017 +++ src/sys/dev/pci/ixgbe/ixgbe.c Wed Feb 1 10:47:13 2017 @@ -59,7 +59,7 @@ * POSSIBILITY OF SUCH DAMAGE. */ /*$FreeBSD: head/sys/dev/ixgbe/if_ix.c 302384 2016-07-07 03:39:18Z sbruno $*/ -/*$NetBSD: ixgbe.c,v 1.66 2017/01/25 13:08:31 msaitoh Exp $*/ +/*$NetBSD: ixgbe.c,v 1.67 2017/02/01 10:47:13 msaitoh Exp $*/ #include "opt_inet.h" #include "opt_inet6.h" @@ -749,7 +749,7 @@ ixgbe_detach(device_t dev, int flags) for (int i = 0; i < adapter->num_queues; i++, que++, txr++) { #ifndef IXGBE_LEGACY_TX - softint_disestablish(txr->txq_si); + softint_disestablish(txr->txr_si); #endif softint_disestablish(que->que_si); } @@ -1030,6 +1030,35 @@ ixgbe_ioctl(struct ifnet * ifp, u_long c case SIOCSIFMTU: IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); break; +#ifdef __NetBSD__ + case SIOCINITIFADDR: + IOCTL_DEBUGOUT("ioctl: SIOCINITIFADDR"); + break; + case SIOCGIFFLAGS: + IOCTL_DEBUGOUT("ioctl: SIOCGIFFLAGS"); + break; + case SIOCGIFAFLAG_IN: + IOCTL_DEBUGOUT("ioctl: SIOCGIFAFLAG_IN"); + break; + case SIOCGIFADDR: + IOCTL_DEBUGOUT("ioctl: SIOCGIFADDR"); + break; + case SIOCGIFMTU: + IOCTL_DEBUGOUT("ioctl: SIOCGIFMTU (Get Interface MTU)"); + break; + case SIOCGIFCAP: + IOCTL_DEBUGOUT("ioctl: SIOCGIFCAP (Get IF cap)"); + break; + case SIOCGETHERCAP: + IOCTL_DEBUGOUT("ioctl: SIOCGETHERCAP (Get ethercap)"); + break; + case SIOCGLIFADDR: + IOCTL_DEBUGOUT("ioctl: SIOCGLIFADDR (Get Interface addr)"); + break; + case SIOCAIFADDR: + IOCTL_DEBUGOUT("ioctl: SIOCAIFADDR (add/chg IF alias)"); + break; +#endif default: IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)", (int)command); break; @@ -1540,7 +1569,7 @@ ixgbe_handle_que(void *context) IXGBE_TX_LOCK(txr); ixgbe_txeof(txr); #ifndef IXGBE_LEGACY_TX - if (!drbr_empty(ifp, txr->br)) + if (pcq_peek(txr->txr_interq) != NULL) ixgbe_mq_start_locked(ifp, txr); #else if (!IFQ_IS_EMPTY(&ifp->if_snd)) @@ -1601,7 +1630,7 @@ ixgbe_legacy_irq(void *arg) if (!IFQ_IS_EMPTY(&ifp->if_snd)) ixgbe_start_locked(txr, ifp); #else - if (!drbr_empty(ifp, txr->br)) + if (pcq_peek(txr->txr_interq) != NULL) ixgbe_mq_start_locked(ifp, txr); #endif IXGBE_TX_UNLOCK(txr); @@ -1626,7 +1655,7 @@ ixgbe_legacy_irq(void *arg) if (more) #ifndef IXGBE_LEGACY_TX - softint_schedule(txr->txq_si); + softint_schedule(txr->txr_si); #else softint_schedule(que->que_si); #endif @@ -1652,7 +1681,6 @@ ixgbe_msix_que(void *arg) bool more; u32 newitr = 0; - /* Protect against spurious interrupts */ if ((ifp->if_flags & IFF_RUNNING) == 0) return 0; @@ -1673,7 +1701,7 @@ ixgbe_msix_que(void *arg) if (!IFQ_IS_EMPTY(&adapter->ifp->if_snd)) ixgbe_start_locked(txr, ifp); #else - if (!drbr_empty(ifp, txr->br)) + if (pcq_peek(txr->txr_interq) != NULL) ixgbe_mq_start_locked(ifp, txr); #endif IXGBE_TX_UNLOCK(txr); @@ -2562,7 +2590,7 @@ alloc_retry: * processing contexts. */ #ifndef IXGBE_LEGACY_TX - txr->txq_si = softint_establish(SOFTINT_NET, ixgbe_deferred_mq_start, + txr->txr_si = softint_establish(SOFTINT_NET, ixgbe_deferred_mq_start, txr); #endif que->que_si = softint_establish(SOFTINT_NET, ixgbe_handle_que, que); @@ -2717,7 +2745,7 @@ ixgbe_allocate_msix(struct adapter *adap } aprint_normal("\n"); #ifndef IXGBE_LEGACY_TX - txr->txq_si = softint_establish(SOFTINT_NET, + txr->txr_si = softint_establish(SOFTINT_NET, ixgbe_deferred_mq_start, txr); #endif que->que_si = softint_establish(SOFTINT_NET, ixgbe_handle_que, @@ -2969,18 +2997,22 @@ ixgbe_setup_interface(device_t dev, stru #endif #ifndef IXGBE_LEGACY_TX ifp->if_transmit = ixgbe_mq_start; - ifp->if_qflush = ixgbe_qflush; #else - ifp->if_start = ixgbe_start; IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 2); #if 0 ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 2; #endif IFQ_SET_READY(&ifp->if_snd); #endif + ifp->if_start = ixgbe_start; if_initialize(ifp); ether_ifattach(ifp, adapter->hw.mac.addr); +#ifndef IXGBE_LEGACY_TX +#if 0 /* We use per TX queue softint */ + if_deferred_start_init(ifp, ixgbe_deferred_mq_start); +#endif +#endif if_register(ifp); ether_set_ifflags_cb(ec, ixgbe_ifflags_cb); @@ -4736,9 +4768,9 @@ ixgbe_add_hw_stats(struct adapter *adapt NULL, adapter->queues[i].evnamebuf, "Queue Packets Transmitted"); #ifndef IXGBE_LEGACY_TX - evcnt_attach_dynamic(&txr->br->br_drops, EVCNT_TYPE_MISC, + evcnt_attach_dynamic(&txr->pcq_drops, EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, - "Packets dropped in buf_ring"); + "Packets dropped in pcq"); #endif #ifdef LRO Index: src/sys/dev/pci/ixgbe/ixgbe.h diff -u src/sys/dev/pci/ixgbe/ixgbe.h:1.21 src/sys/dev/pci/ixgbe/ixgbe.h:1.22 --- src/sys/dev/pci/ixgbe/ixgbe.h:1.21 Mon Jan 30 05:02:43 2017 +++ src/sys/dev/pci/ixgbe/ixgbe.h Wed Feb 1 10:47:13 2017 @@ -59,7 +59,7 @@ * POSSIBILITY OF SUCH DAMAGE. */ /*$FreeBSD: head/sys/dev/ixgbe/ixgbe.h 303890 2016-08-09 19:32:06Z dumbbell $*/ -/*$NetBSD: ixgbe.h,v 1.21 2017/01/30 05:02:43 msaitoh Exp $*/ +/*$NetBSD: ixgbe.h,v 1.22 2017/02/01 10:47:13 msaitoh Exp $*/ #ifndef _IXGBE_H_ @@ -69,9 +69,7 @@ #include <sys/param.h> #include <sys/reboot.h> #include <sys/systm.h> -#if __FreeBSD_version >= 800000 -#include <sys/buf_ring.h> -#endif +#include <sys/pcq.h> #include <sys/mbuf.h> #include <sys/protosw.h> #include <sys/socket.h> @@ -107,6 +105,7 @@ #include <sys/workqueue.h> #include <sys/cpu.h> #include <sys/interrupt.h> +#include <sys/bitops.h> #ifdef PCI_IOV #include <sys/nv.h> @@ -390,8 +389,8 @@ struct tx_ring { ixgbe_dma_tag_t *txtag; char mtx_name[16]; #ifndef IXGBE_LEGACY_TX - struct buf_ring *br; - void *txq_si; + pcq_t *txr_interq; + void *txr_si; #endif #ifdef IXGBE_FDIR u16 atr_sample; @@ -404,6 +403,7 @@ struct tx_ring { struct evcnt no_tx_map_avail; struct evcnt no_desc_avail; struct evcnt total_packets; + struct evcnt pcq_drops; }; @@ -757,15 +757,13 @@ ixv_check_ether_addr(u8 *addr) /* Shared Prototypes */ -#ifdef IXGBE_LEGACY_TX void ixgbe_start(struct ifnet *); void ixgbe_start_locked(struct tx_ring *, struct ifnet *); -#else /* ! IXGBE_LEGACY_TX */ +#ifndef IXGBE_LEGACY_TX int ixgbe_mq_start(struct ifnet *, struct mbuf *); int ixgbe_mq_start_locked(struct ifnet *, struct tx_ring *); -void ixgbe_qflush(struct ifnet *); -void ixgbe_deferred_mq_start(void *, int); -#endif /* IXGBE_LEGACY_TX */ +void ixgbe_deferred_mq_start(void *); +#endif /* !IXGBE_LEGACY_TX */ int ixgbe_allocate_queues(struct adapter *); int ixgbe_allocate_transmit_buffers(struct tx_ring *); Index: src/sys/dev/pci/ixgbe/ixgbe_netbsd.h diff -u src/sys/dev/pci/ixgbe/ixgbe_netbsd.h:1.5 src/sys/dev/pci/ixgbe/ixgbe_netbsd.h:1.6 --- src/sys/dev/pci/ixgbe/ixgbe_netbsd.h:1.5 Thu Dec 1 06:56:28 2016 +++ src/sys/dev/pci/ixgbe/ixgbe_netbsd.h Wed Feb 1 10:47:13 2017 @@ -1,4 +1,4 @@ -/*$NetBSD: ixgbe_netbsd.h,v 1.5 2016/12/01 06:56:28 msaitoh Exp $*/ +/*$NetBSD: ixgbe_netbsd.h,v 1.6 2017/02/01 10:47:13 msaitoh Exp $*/ /* * Copyright (c) 2011 The NetBSD Foundation, Inc. * All rights reserved. @@ -31,7 +31,9 @@ #ifndef _IXGBE_NETBSD_H #define _IXGBE_NETBSD_H +#if 1 /* Enable this if you don't want to use TX multiqueue function */ #define IXGBE_LEGACY_TX 1 +#endif #define ETHERCAP_VLAN_HWFILTER 0 #define ETHERCAP_VLAN_HWCSUM 0 Index: src/sys/dev/pci/ixgbe/ixv.c diff -u src/sys/dev/pci/ixgbe/ixv.c:1.34 src/sys/dev/pci/ixgbe/ixv.c:1.35 --- src/sys/dev/pci/ixgbe/ixv.c:1.34 Mon Jan 30 06:11:56 2017 +++ src/sys/dev/pci/ixgbe/ixv.c Wed Feb 1 10:47:13 2017 @@ -31,7 +31,7 @@ ******************************************************************************/ /*$FreeBSD: head/sys/dev/ixgbe/if_ixv.c 302384 2016-07-07 03:39:18Z sbruno $*/ -/*$NetBSD: ixv.c,v 1.34 2017/01/30 06:11:56 msaitoh Exp $*/ +/*$NetBSD: ixv.c,v 1.35 2017/02/01 10:47:13 msaitoh Exp $*/ #include "opt_inet.h" #include "opt_inet6.h" @@ -527,7 +527,9 @@ ixv_detach(device_t dev, int flags) for (int i = 0; i < adapter->num_queues; i++, que++) { #ifndef IXGBE_LEGACY_TX - softint_disestablish(txr->txq_si); + struct tx_ring *txr = adapter->tx_rings; + + softint_disestablish(txr->txr_si); #endif softint_disestablish(que->que_si); } @@ -858,8 +860,8 @@ ixv_handle_que(void *context) more = ixgbe_rxeof(que); IXGBE_TX_LOCK(txr); ixgbe_txeof(txr); -#if __FreeBSD_version >= 800000 - if (!drbr_empty(ifp, txr->br)) +#ifndef IXGBE_LEGACY_TX + if (pcq_peek(txr->txr_interq) != NULL) ixgbe_mq_start_locked(ifp, txr); #else if (!IFQ_IS_EMPTY(&ifp->if_snd)) @@ -915,7 +917,7 @@ ixv_msix_que(void *arg) if (!IFQ_IS_EMPTY(&adapter->ifp->if_snd)) ixgbe_start_locked(txr, ifp); #else - if (!drbr_empty(adapter->ifp, txr->br)) + if (pcq_peek(txr->txr_interq) != NULL) ixgbe_mq_start_locked(ifp, txr); #endif IXGBE_TX_UNLOCK(txr); @@ -1392,7 +1394,7 @@ ixv_allocate_msix(struct adapter *adapte aprint_normal("\n"); #ifndef IXGBE_LEGACY_TX - txr->txq_si = softint_establish(SOFTINT_NET, + txr->txr_si = softint_establish(SOFTINT_NET, ixgbe_deferred_mq_start, txr); #endif que->que_si = softint_establish(SOFTINT_NET, ixv_handle_que, @@ -1603,16 +1605,19 @@ ixv_setup_interface(device_t dev, struct ifp->if_softc = adapter; ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; ifp->if_ioctl = ixv_ioctl; -#if __FreeBSD_version >= 800000 +#ifndef IXGBE_LEGACY_TX ifp->if_transmit = ixgbe_mq_start; - ifp->if_qflush = ixgbe_qflush; -#else - ifp->if_start = ixgbe_start; #endif + ifp->if_start = ixgbe_start; ifp->if_snd.ifq_maxlen = adapter->num_tx_desc - 2; if_initialize(ifp); ether_ifattach(ifp, adapter->hw.mac.addr); +#ifndef IXGBE_LEGACY_TX +#if 0 /* We use per TX queue softint */ + if_deferred_start_init(ifp, ixgbe_deferred_mq_start); +#endif +#endif if_register(ifp); ether_set_ifflags_cb(ec, ixv_ifflags_cb);