Module Name: src Committed By: martin Date: Thu Mar 30 11:36:26 UTC 2023
Modified Files: src/sys/dev/pci [netbsd-10]: if_vioif.c Log Message: Pull up following revision(s) (requested by yamaguchi in ticket #128): sys/dev/pci/if_vioif.c: revision 1.83-1.102,1.105,1.106 vioif(4): remove unnecessary lock release if_percpuq_enqueue() can call with rxq->rxq_lock held because of per-cpu. vioif(4): access to txq_active and rxq_active with lock held vioif(4): use device reset to stop interrupt completely vioif(4): rename {txq,rxq}_active to {txq,rxq}_running_handle vioif(4): stop interrupt before schedule handler vioif(4): adjust receive buffer to ETHER_ALIGN vioif(4): added event counters related to receive processing vioif(4): fix missing virtio_enqueue_abort for error handling vioif(4): drain receive buffer on stopping the device to remove branch in vioif_populate_rx_mbufs_locked() vioif(4): divide interrupt handler for receiving into dequeuing and preparing of buffers vioif(4): merge drain into clear of queue vioif(4): increase output error counter vioif(4): added a structure to manage variables for packet processings vioif(4): prepare slot before dequeuing vioif(4): added __predct_false to error check vioif(4): added new data structure for network queues and moved the same parameters in vioif_txqueue and vioif_rxqueue into the new structure vioif(4): added functions to manipulate network queues vioif(4): rename sc_hdr_segs to sc_segs vioif(4): reorganize functions This change is move of function and rename, and this is no functional change. vioif(4): divide IFF_OACTIVE into per-queue vioif(4): clear flags when configure is failed vioif(4): fix wrong memory allocation size To generate a diff of this commit: cvs rdiff -u -r1.82 -r1.82.4.1 src/sys/dev/pci/if_vioif.c Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files.
Modified files: Index: src/sys/dev/pci/if_vioif.c diff -u src/sys/dev/pci/if_vioif.c:1.82 src/sys/dev/pci/if_vioif.c:1.82.4.1 --- src/sys/dev/pci/if_vioif.c:1.82 Mon Sep 12 07:26:04 2022 +++ src/sys/dev/pci/if_vioif.c Thu Mar 30 11:36:26 2023 @@ -1,4 +1,4 @@ -/* $NetBSD: if_vioif.c,v 1.82 2022/09/12 07:26:04 knakahara Exp $ */ +/* $NetBSD: if_vioif.c,v 1.82.4.1 2023/03/30 11:36:26 martin Exp $ */ /* * Copyright (c) 2020 The NetBSD Foundation, Inc. @@ -27,7 +27,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: if_vioif.c,v 1.82 2022/09/12 07:26:04 knakahara Exp $"); +__KERNEL_RCSID(0, "$NetBSD: if_vioif.c,v 1.82.4.1 2023/03/30 11:36:26 martin Exp $"); #ifdef _KERNEL_OPT #include "opt_net_mpsafe.h" @@ -51,6 +51,7 @@ __KERNEL_RCSID(0, "$NetBSD: if_vioif.c,v #include <sys/module.h> #include <sys/pcq.h> #include <sys/workqueue.h> +#include <sys/xcall.h> #include <dev/pci/virtioreg.h> #include <dev/pci/virtiovar.h> @@ -204,12 +205,13 @@ struct virtio_net_ctrl_mq { /* * Locking notes: - * + a field in vioif_txqueue is protected by txq_lock (a spin mutex), and - * a field in vioif_rxqueue is protected by rxq_lock (a spin mutex). + * + a field in vioif_netueue is protected by netq_lock (a spin mutex) * - more than one lock cannot be held at onece + * + a field in vioif_tx_context and vioif_rx_context is also protected + * by netq_lock. * + ctrlq_inuse is protected by ctrlq_wait_lock. * - other fields in vioif_ctrlqueue are protected by ctrlq_inuse - * - txq_lock or rxq_lock cannot be held along with ctrlq_wait_lock + * - netq_lock cannot be held along with ctrlq_wait_lock * + fields in vioif_softc except queues are protected by * sc->sc_lock(an adaptive mutex) * - the lock is held before acquisition of other locks @@ -228,53 +230,52 @@ struct vioif_work { unsigned int added; }; -struct vioif_txqueue { - kmutex_t *txq_lock; /* lock for tx operations */ - - struct virtqueue *txq_vq; - bool txq_stopping; - bool txq_link_active; - pcq_t *txq_intrq; - - struct virtio_net_hdr *txq_hdrs; - bus_dmamap_t *txq_hdr_dmamaps; - - struct mbuf **txq_mbufs; - bus_dmamap_t *txq_dmamaps; - - void *txq_deferred_transmit; - void *txq_handle_si; - struct vioif_work txq_work; - bool txq_workqueue; - bool txq_active; - - char txq_evgroup[16]; - struct evcnt txq_defrag_failed; - struct evcnt txq_mbuf_load_failed; - struct evcnt txq_enqueue_reserve_failed; +struct vioif_net_map { + struct virtio_net_hdr *vnm_hdr; + bus_dmamap_t vnm_hdr_map; + struct mbuf *vnm_mbuf; + bus_dmamap_t vnm_mbuf_map; }; -struct vioif_rxqueue { - kmutex_t *rxq_lock; /* lock for rx operations */ - - struct virtqueue *rxq_vq; - bool rxq_stopping; +#define VIOIF_NETQ_RX 0 +#define VIOIF_NETQ_TX 1 +#define VIOIF_NETQ_IDX 2 +#define VIOIF_NETQ_DIR(n) ((n) % VIOIF_NETQ_IDX) +#define VIOIF_NETQ_PAIRIDX(n) ((n) / VIOIF_NETQ_IDX) +#define VIOIF_NETQ_RXQID(n) ((n) * VIOIF_NETQ_IDX + VIOIF_NETQ_RX) +#define VIOIF_NETQ_TXQID(n) ((n) * VIOIF_NETQ_IDX + VIOIF_NETQ_TX) + +struct vioif_netqueue { + kmutex_t netq_lock; + struct virtqueue *netq_vq; + bool netq_stopping; + bool netq_running_handle; + void *netq_maps_kva; + struct vioif_net_map *netq_maps; + + void *netq_softint; + struct vioif_work netq_work; + bool netq_workqueue; + + char netq_evgroup[32]; + struct evcnt netq_mbuf_load_failed; + struct evcnt netq_enqueue_failed; - struct virtio_net_hdr *rxq_hdrs; - bus_dmamap_t *rxq_hdr_dmamaps; - - struct mbuf **rxq_mbufs; - bus_dmamap_t *rxq_dmamaps; + void *netq_ctx; +}; - void *rxq_handle_si; - struct vioif_work rxq_work; - bool rxq_workqueue; - bool rxq_active; +struct vioif_tx_context { + bool txc_link_active; + bool txc_no_free_slots; + pcq_t *txc_intrq; + void *txc_deferred_transmit; - char rxq_evgroup[16]; - struct evcnt rxq_mbuf_add_failed; + struct evcnt txc_defrag_failed; }; +struct vioif_rx_context { + struct evcnt rxc_mbuf_enobufs; +}; struct vioif_ctrlqueue { struct virtqueue *ctrlq_vq; enum { @@ -321,17 +322,16 @@ struct vioif_softc { struct ethercom sc_ethercom; int sc_link_state; - struct vioif_txqueue *sc_txq; - struct vioif_rxqueue *sc_rxq; + struct vioif_netqueue *sc_netqs; bool sc_has_ctrl; struct vioif_ctrlqueue sc_ctrlq; - bus_dma_segment_t sc_hdr_segs[1]; + bus_dma_segment_t sc_segs[1]; void *sc_dmamem; void *sc_kmem; - void *sc_ctl_softint; + void *sc_cfg_softint; struct workqueue *sc_txrx_workqueue; bool sc_txrx_workqueue_sysctl; @@ -361,69 +361,87 @@ static int vioif_finalize_teardown(devic static int vioif_init(struct ifnet *); static void vioif_stop(struct ifnet *, int); static void vioif_start(struct ifnet *); -static void vioif_start_locked(struct ifnet *, struct vioif_txqueue *); static int vioif_transmit(struct ifnet *, struct mbuf *); -static void vioif_transmit_locked(struct ifnet *, struct vioif_txqueue *); static int vioif_ioctl(struct ifnet *, u_long, void *); static void vioif_watchdog(struct ifnet *); +static int vioif_ifflags(struct vioif_softc *); static int vioif_ifflags_cb(struct ethercom *); +/* tx & rx */ +static int vioif_netqueue_init(struct vioif_softc *, + struct virtio_softc *, size_t, u_int); +static void vioif_netqueue_teardown(struct vioif_softc *, + struct virtio_softc *, size_t); +static void vioif_net_intr_enable(struct vioif_softc *, + struct virtio_softc *); +static void vioif_net_intr_disable(struct vioif_softc *, + struct virtio_softc *); +static void vioif_net_sched_handle(struct vioif_softc *, + struct vioif_netqueue *); + /* rx */ -static int vioif_add_rx_mbuf(struct vioif_rxqueue *, int); -static void vioif_free_rx_mbuf(struct vioif_rxqueue *, int); static void vioif_populate_rx_mbufs_locked(struct vioif_softc *, - struct vioif_rxqueue *); -static void vioif_rx_queue_clear(struct vioif_rxqueue *); -static bool vioif_rx_deq_locked(struct vioif_softc *, struct virtio_softc *, - struct vioif_rxqueue *, u_int); + struct vioif_netqueue *); static int vioif_rx_intr(void *); static void vioif_rx_handle(void *); -static void vioif_rx_sched_handle(struct vioif_softc *, - struct vioif_rxqueue *); -static void vioif_rx_drain(struct vioif_rxqueue *); +static void vioif_rx_queue_clear(struct vioif_softc *, + struct virtio_softc *, struct vioif_netqueue *); /* tx */ +static void vioif_start_locked(struct ifnet *, struct vioif_netqueue *); +static void vioif_transmit_locked(struct ifnet *, struct vioif_netqueue *); +static void vioif_deferred_transmit(void *); static int vioif_tx_intr(void *); static void vioif_tx_handle(void *); -static void vioif_tx_sched_handle(struct vioif_softc *, - struct vioif_txqueue *); -static void vioif_tx_queue_clear(struct vioif_txqueue *); -static bool vioif_tx_deq_locked(struct vioif_softc *, struct virtio_softc *, - struct vioif_txqueue *, u_int); -static void vioif_tx_drain(struct vioif_txqueue *); -static void vioif_deferred_transmit(void *); - -/* workqueue */ -static struct workqueue* - vioif_workq_create(const char *, pri_t, int, int); -static void vioif_workq_destroy(struct workqueue *); -static void vioif_workq_work(struct work *, void *); -static void vioif_work_set(struct vioif_work *, void(*)(void *), void *); -static void vioif_work_add(struct workqueue *, struct vioif_work *); -static void vioif_work_wait(struct workqueue *, struct vioif_work *); +static void vioif_tx_queue_clear(struct vioif_softc *, struct virtio_softc *, + struct vioif_netqueue *); -/* other control */ -static int vioif_get_link_status(struct vioif_softc *); -static void vioif_update_link_status(struct vioif_softc *); +/* controls */ +static int vioif_ctrl_intr(void *); static int vioif_ctrl_rx(struct vioif_softc *, int, bool); static int vioif_set_promisc(struct vioif_softc *, bool); static int vioif_set_allmulti(struct vioif_softc *, bool); static int vioif_set_rx_filter(struct vioif_softc *); static int vioif_rx_filter(struct vioif_softc *); static int vioif_set_mac_addr(struct vioif_softc *); -static int vioif_ctrl_intr(void *); -static int vioif_config_change(struct virtio_softc *); -static void vioif_ctl_softint(void *); static int vioif_ctrl_mq_vq_pairs_set(struct vioif_softc *, int); -static void vioif_enable_interrupt_vqpairs(struct vioif_softc *); -static void vioif_disable_interrupt_vqpairs(struct vioif_softc *); + +/* config interrupt */ +static int vioif_config_change(struct virtio_softc *); +static void vioif_cfg_softint(void *); +static void vioif_update_link_status(struct vioif_softc *); + +/* others */ +static void vioif_alloc_queues(struct vioif_softc *); +static void vioif_free_queues(struct vioif_softc *); +static int vioif_alloc_mems(struct vioif_softc *); +static struct workqueue* + vioif_workq_create(const char *, pri_t, int, int); +static void vioif_workq_destroy(struct workqueue *); +static void vioif_work_set(struct vioif_work *, void(*)(void *), void *); +static void vioif_work_add(struct workqueue *, struct vioif_work *); +static void vioif_work_wait(struct workqueue *, struct vioif_work *); static int vioif_setup_sysctl(struct vioif_softc *); static void vioif_setup_stats(struct vioif_softc *); -static int vioif_ifflags(struct vioif_softc *); CFATTACH_DECL_NEW(vioif, sizeof(struct vioif_softc), vioif_match, vioif_attach, NULL, NULL); +static void +vioif_intr_barrier(void) +{ + + /* wait for finish all interrupt handler */ + xc_barrier(0); +} + +static void +vioif_notify(struct virtio_softc *vsc, struct virtqueue *vq) +{ + + virtio_enqueue_commit(vsc, vq, -1, true); +} + static int vioif_match(device_t parent, cfdata_t match, void *aux) { @@ -435,1190 +453,1630 @@ vioif_match(device_t parent, cfdata_t ma return 0; } -static int -vioif_dmamap_create(struct vioif_softc *sc, bus_dmamap_t *map, - bus_size_t size, int nsegs, const char *usage) +static void +vioif_attach(device_t parent, device_t self, void *aux) { - int r; - - r = bus_dmamap_create(virtio_dmat(sc->sc_virtio), size, - nsegs, size, 0, BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, map); + struct vioif_softc *sc = device_private(self); + struct virtio_softc *vsc = device_private(parent); + struct vioif_netqueue *txq0; + struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; + uint64_t features, req_features; + struct ifnet *ifp = &sc->sc_ethercom.ec_if; + u_int softint_flags; + int r, i, req_flags; + char xnamebuf[MAXCOMLEN]; + size_t netq_num; - if (r != 0) { - aprint_error_dev(sc->sc_dev, "%s dmamap creation failed, " - "error code %d\n", usage, r); + if (virtio_child(vsc) != NULL) { + aprint_normal(": child already attached for %s; " + "something wrong...\n", device_xname(parent)); + return; } - return r; -} + sc->sc_dev = self; + sc->sc_virtio = vsc; + sc->sc_link_state = LINK_STATE_UNKNOWN; -static void -vioif_dmamap_destroy(struct vioif_softc *sc, bus_dmamap_t *map) -{ + sc->sc_max_nvq_pairs = 1; + sc->sc_req_nvq_pairs = 1; + sc->sc_act_nvq_pairs = 1; + sc->sc_txrx_workqueue_sysctl = true; + sc->sc_tx_intr_process_limit = VIOIF_TX_INTR_PROCESS_LIMIT; + sc->sc_tx_process_limit = VIOIF_TX_PROCESS_LIMIT; + sc->sc_rx_intr_process_limit = VIOIF_RX_INTR_PROCESS_LIMIT; + sc->sc_rx_process_limit = VIOIF_RX_PROCESS_LIMIT; - if (*map) { - bus_dmamap_destroy(virtio_dmat(sc->sc_virtio), *map); - *map = NULL; - } -} + mutex_init(&sc->sc_lock, MUTEX_DEFAULT, IPL_NONE); -static int -vioif_dmamap_create_load(struct vioif_softc *sc, bus_dmamap_t *map, - void *buf, bus_size_t size, int nsegs, int rw, const char *usage) -{ - int r; + snprintf(xnamebuf, sizeof(xnamebuf), "%s_txrx", device_xname(self)); + sc->sc_txrx_workqueue = vioif_workq_create(xnamebuf, VIOIF_WORKQUEUE_PRI, + IPL_NET, WQ_PERCPU | WQ_MPSAFE); + if (sc->sc_txrx_workqueue == NULL) + goto err; - r = vioif_dmamap_create(sc, map, size, nsegs, usage); - if (r != 0) - return 1; + req_flags = 0; - r = bus_dmamap_load(virtio_dmat(sc->sc_virtio), *map, buf, - size, NULL, rw | BUS_DMA_NOWAIT); - if (r != 0) { - vioif_dmamap_destroy(sc, map); - aprint_error_dev(sc->sc_dev, "%s dmamap load failed. " - "error code %d\n", usage, r); - } +#ifdef VIOIF_MPSAFE + req_flags |= VIRTIO_F_INTR_MPSAFE; +#endif + req_flags |= VIRTIO_F_INTR_MSIX; - return r; -} + req_features = + VIRTIO_NET_F_MAC | VIRTIO_NET_F_STATUS | VIRTIO_NET_F_CTRL_VQ | + VIRTIO_NET_F_CTRL_RX | VIRTIO_F_NOTIFY_ON_EMPTY; + req_features |= VIRTIO_F_RING_EVENT_IDX; + req_features |= VIRTIO_NET_F_CTRL_MAC_ADDR; +#ifdef VIOIF_MULTIQ + req_features |= VIRTIO_NET_F_MQ; +#endif + virtio_child_attach_start(vsc, self, IPL_NET, NULL, + vioif_config_change, virtio_vq_intrhand, req_flags, + req_features, VIRTIO_NET_FLAG_BITS); -static void * -vioif_assign_mem(intptr_t *p, size_t size) -{ - intptr_t rv; + features = virtio_features(vsc); + if (features == 0) + goto err; - rv = *p; - *p += size; + if (features & VIRTIO_NET_F_MAC) { + for (i = 0; i < __arraycount(sc->sc_mac); i++) { + sc->sc_mac[i] = virtio_read_device_config_1(vsc, + VIRTIO_NET_CONFIG_MAC + i); + } + } else { + /* code stolen from sys/net/if_tap.c */ + struct timeval tv; + uint32_t ui; + getmicrouptime(&tv); + ui = (tv.tv_sec ^ tv.tv_usec) & 0xffffff; + memcpy(sc->sc_mac+3, (uint8_t *)&ui, 3); + for (i = 0; i < __arraycount(sc->sc_mac); i++) { + virtio_write_device_config_1(vsc, + VIRTIO_NET_CONFIG_MAC + i, sc->sc_mac[i]); + } + } - return (void *)rv; -} + /* 'Ethernet' with capital follows other ethernet driver attachment */ + aprint_normal_dev(self, "Ethernet address %s\n", + ether_sprintf(sc->sc_mac)); -static void -vioif_alloc_queues(struct vioif_softc *sc) -{ - int nvq_pairs = sc->sc_max_nvq_pairs; - int nvqs = nvq_pairs * 2; - int i; + if (features & (VIRTIO_NET_F_MRG_RXBUF | VIRTIO_F_VERSION_1)) { + sc->sc_hdr_size = sizeof(struct virtio_net_hdr); + } else { + sc->sc_hdr_size = offsetof(struct virtio_net_hdr, num_buffers); + } - KASSERT(nvq_pairs <= VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX); + if ((features & VIRTIO_NET_F_CTRL_VQ) && + (features & VIRTIO_NET_F_CTRL_RX)) { + sc->sc_has_ctrl = true; - sc->sc_rxq = kmem_zalloc(sizeof(sc->sc_rxq[0]) * nvq_pairs, - KM_SLEEP); - sc->sc_txq = kmem_zalloc(sizeof(sc->sc_txq[0]) * nvq_pairs, - KM_SLEEP); + cv_init(&ctrlq->ctrlq_wait, "ctrl_vq"); + mutex_init(&ctrlq->ctrlq_wait_lock, MUTEX_DEFAULT, IPL_NET); + ctrlq->ctrlq_inuse = FREE; + } else { + sc->sc_has_ctrl = false; + } - if (sc->sc_has_ctrl) - nvqs++; + if (sc->sc_has_ctrl && (features & VIRTIO_NET_F_MQ)) { + sc->sc_max_nvq_pairs = virtio_read_device_config_2(vsc, + VIRTIO_NET_CONFIG_MAX_VQ_PAIRS); - sc->sc_vqs = kmem_zalloc(sizeof(sc->sc_vqs[0]) * nvqs, KM_SLEEP); - nvqs = 0; - for (i = 0; i < nvq_pairs; i++) { - sc->sc_rxq[i].rxq_vq = &sc->sc_vqs[nvqs++]; - sc->sc_txq[i].txq_vq = &sc->sc_vqs[nvqs++]; - } + if (sc->sc_max_nvq_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX) + goto err; - if (sc->sc_has_ctrl) - sc->sc_ctrlq.ctrlq_vq = &sc->sc_vqs[nvqs++]; -} + /* Limit the number of queue pairs to use */ + sc->sc_req_nvq_pairs = MIN(sc->sc_max_nvq_pairs, ncpu); + } -static void -vioif_free_queues(struct vioif_softc *sc) -{ - int nvq_pairs = sc->sc_max_nvq_pairs; - int nvqs = nvq_pairs * 2; + vioif_alloc_queues(sc); + virtio_child_attach_set_vqs(vsc, sc->sc_vqs, sc->sc_req_nvq_pairs); - if (sc->sc_ctrlq.ctrlq_vq) - nvqs++; +#ifdef VIOIF_MPSAFE + softint_flags = SOFTINT_NET | SOFTINT_MPSAFE; +#else + softint_flags = SOFTINT_NET; +#endif - if (sc->sc_txq) { - kmem_free(sc->sc_txq, sizeof(sc->sc_txq[0]) * nvq_pairs); - sc->sc_txq = NULL; + /* + * Initialize network queues + */ + netq_num = sc->sc_max_nvq_pairs * 2; + for (i = 0; i < netq_num; i++) { + r = vioif_netqueue_init(sc, vsc, i, softint_flags); + if (r != 0) + goto err; } - if (sc->sc_rxq) { - kmem_free(sc->sc_rxq, sizeof(sc->sc_rxq[0]) * nvq_pairs); - sc->sc_rxq = NULL; + if (sc->sc_has_ctrl) { + int ctrlq_idx = sc->sc_max_nvq_pairs * 2; + /* + * Allocating a virtqueue for control channel + */ + sc->sc_ctrlq.ctrlq_vq = &sc->sc_vqs[ctrlq_idx]; + r = virtio_alloc_vq(vsc, ctrlq->ctrlq_vq, ctrlq_idx, + NBPG, 1, "control"); + if (r != 0) { + aprint_error_dev(self, "failed to allocate " + "a virtqueue for control channel, error code %d\n", + r); + + sc->sc_has_ctrl = false; + cv_destroy(&ctrlq->ctrlq_wait); + mutex_destroy(&ctrlq->ctrlq_wait_lock); + } else { + ctrlq->ctrlq_vq->vq_intrhand = vioif_ctrl_intr; + ctrlq->ctrlq_vq->vq_intrhand_arg = (void *) ctrlq; + } } - if (sc->sc_vqs) { - kmem_free(sc->sc_vqs, sizeof(sc->sc_vqs[0]) * nvqs); - sc->sc_vqs = NULL; + sc->sc_cfg_softint = softint_establish(softint_flags, + vioif_cfg_softint, sc); + if (sc->sc_cfg_softint == NULL) { + aprint_error_dev(self, "cannot establish ctl softint\n"); + goto err; } -} -/* allocate memory */ -/* - * dma memory is used for: - * rxq_hdrs[slot]: metadata array for received frames (READ) - * txq_hdrs[slot]: metadata array for frames to be sent (WRITE) - * ctrlq_cmd: command to be sent via ctrl vq (WRITE) - * ctrlq_status: return value for a command via ctrl vq (READ) - * ctrlq_rx: parameter for a VIRTIO_NET_CTRL_RX class command - * (WRITE) - * ctrlq_mac_tbl_uc: unicast MAC address filter for a VIRTIO_NET_CTRL_MAC - * class command (WRITE) - * ctrlq_mac_tbl_mc: multicast MAC address filter for a VIRTIO_NET_CTRL_MAC - * class command (WRITE) - * ctrlq_* structures are allocated only one each; they are protected by - * ctrlq_inuse variable and ctrlq_wait condvar. - */ -/* - * dynamically allocated memory is used for: - * rxq_hdr_dmamaps[slot]: bus_dmamap_t array for sc_rx_hdrs[slot] - * txq_hdr_dmamaps[slot]: bus_dmamap_t array for sc_tx_hdrs[slot] - * rxq_dmamaps[slot]: bus_dmamap_t array for received payload - * txq_dmamaps[slot]: bus_dmamap_t array for sent payload - * rxq_mbufs[slot]: mbuf pointer array for received frames - * txq_mbufs[slot]: mbuf pointer array for sent frames - */ -static int -vioif_alloc_mems(struct vioif_softc *sc) -{ - struct virtio_softc *vsc = sc->sc_virtio; - struct vioif_txqueue *txq; - struct vioif_rxqueue *rxq; - struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; - int allocsize, allocsize2, r, rsegs, i, qid; - void *vaddr; - intptr_t p; + if (vioif_alloc_mems(sc) < 0) + goto err; - allocsize = 0; - for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) { - rxq = &sc->sc_rxq[qid]; - txq = &sc->sc_txq[qid]; + if (virtio_child_attach_finish(vsc) != 0) + goto err; - allocsize += sizeof(struct virtio_net_hdr) * - (rxq->rxq_vq->vq_num + txq->txq_vq->vq_num); - } - if (sc->sc_has_ctrl) { - allocsize += sizeof(struct virtio_net_ctrl_cmd); - allocsize += sizeof(struct virtio_net_ctrl_status); - allocsize += sizeof(struct virtio_net_ctrl_rx); - allocsize += sizeof(struct virtio_net_ctrl_mac_tbl) - + ETHER_ADDR_LEN; - allocsize += sizeof(struct virtio_net_ctrl_mac_tbl) - + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES; - allocsize += sizeof(struct virtio_net_ctrl_mac_addr); - allocsize += sizeof(struct virtio_net_ctrl_mq); - } - r = bus_dmamem_alloc(virtio_dmat(vsc), allocsize, 0, 0, - &sc->sc_hdr_segs[0], 1, &rsegs, BUS_DMA_NOWAIT); - if (r != 0) { - aprint_error_dev(sc->sc_dev, - "DMA memory allocation failed, size %d, " - "error code %d\n", allocsize, r); - goto err_none; - } - r = bus_dmamem_map(virtio_dmat(vsc), - &sc->sc_hdr_segs[0], 1, allocsize, &vaddr, BUS_DMA_NOWAIT); - if (r != 0) { - aprint_error_dev(sc->sc_dev, - "DMA memory map failed, error code %d\n", r); - goto err_dmamem_alloc; + if (vioif_setup_sysctl(sc) != 0) { + aprint_error_dev(self, "unable to create sysctl node\n"); + /* continue */ } - memset(vaddr, 0, allocsize); - sc->sc_dmamem = vaddr; - p = (intptr_t) vaddr; + vioif_setup_stats(sc); + + strlcpy(ifp->if_xname, device_xname(self), IFNAMSIZ); + ifp->if_softc = sc; + ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; +#ifdef VIOIF_MPSAFE + ifp->if_extflags = IFEF_MPSAFE; +#endif + ifp->if_start = vioif_start; + if (sc->sc_req_nvq_pairs > 1) + ifp->if_transmit = vioif_transmit; + ifp->if_ioctl = vioif_ioctl; + ifp->if_init = vioif_init; + ifp->if_stop = vioif_stop; + ifp->if_capabilities = 0; + ifp->if_watchdog = vioif_watchdog; + txq0 = &sc->sc_netqs[VIOIF_NETQ_TXQID(0)]; + IFQ_SET_MAXLEN(&ifp->if_snd, MAX(txq0->netq_vq->vq_num, IFQ_MAXLEN)); + IFQ_SET_READY(&ifp->if_snd); + + sc->sc_ethercom.ec_capabilities |= ETHERCAP_VLAN_MTU; + + if_attach(ifp); + if_deferred_start_init(ifp, NULL); + ether_ifattach(ifp, sc->sc_mac); + ether_set_ifflags_cb(&sc->sc_ethercom, vioif_ifflags_cb); - for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) { - rxq = &sc->sc_rxq[qid]; - txq = &sc->sc_txq[qid]; - - rxq->rxq_hdrs = vioif_assign_mem(&p, - sizeof(struct virtio_net_hdr) * rxq->rxq_vq->vq_num); - txq->txq_hdrs = vioif_assign_mem(&p, - sizeof(struct virtio_net_hdr) * txq->txq_vq->vq_num); + return; + +err: + netq_num = sc->sc_max_nvq_pairs * 2; + for (i = 0; i < netq_num; i++) { + vioif_netqueue_teardown(sc, vsc, i); } + if (sc->sc_has_ctrl) { - ctrlq->ctrlq_cmd = vioif_assign_mem(&p, - sizeof(*ctrlq->ctrlq_cmd)); - ctrlq->ctrlq_status = vioif_assign_mem(&p, - sizeof(*ctrlq->ctrlq_status)); - ctrlq->ctrlq_rx = vioif_assign_mem(&p, - sizeof(*ctrlq->ctrlq_rx)); - ctrlq->ctrlq_mac_tbl_uc = vioif_assign_mem(&p, - sizeof(*ctrlq->ctrlq_mac_tbl_uc) - + ETHER_ADDR_LEN); - ctrlq->ctrlq_mac_tbl_mc = vioif_assign_mem(&p, - sizeof(*ctrlq->ctrlq_mac_tbl_mc) - + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES); - ctrlq->ctrlq_mac_addr = vioif_assign_mem(&p, - sizeof(*ctrlq->ctrlq_mac_addr)); - ctrlq->ctrlq_mq = vioif_assign_mem(&p, sizeof(*ctrlq->ctrlq_mq)); + cv_destroy(&ctrlq->ctrlq_wait); + mutex_destroy(&ctrlq->ctrlq_wait_lock); + virtio_free_vq(vsc, ctrlq->ctrlq_vq); + ctrlq->ctrlq_vq = NULL; } - allocsize2 = 0; - for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) { - int rxqsize, txqsize; - - rxq = &sc->sc_rxq[qid]; - txq = &sc->sc_txq[qid]; - rxqsize = rxq->rxq_vq->vq_num; - txqsize = txq->txq_vq->vq_num; - - allocsize2 += sizeof(rxq->rxq_dmamaps[0]) * rxqsize; - allocsize2 += sizeof(rxq->rxq_hdr_dmamaps[0]) * rxqsize; - allocsize2 += sizeof(rxq->rxq_mbufs[0]) * rxqsize; - - allocsize2 += sizeof(txq->txq_dmamaps[0]) * txqsize; - allocsize2 += sizeof(txq->txq_hdr_dmamaps[0]) * txqsize; - allocsize2 += sizeof(txq->txq_mbufs[0]) * txqsize; + vioif_free_queues(sc); + mutex_destroy(&sc->sc_lock); + virtio_child_attach_failed(vsc); + config_finalize_register(self, vioif_finalize_teardown); + + return; +} + +static int +vioif_finalize_teardown(device_t self) +{ + struct vioif_softc *sc = device_private(self); + + if (sc->sc_txrx_workqueue != NULL) { + vioif_workq_destroy(sc->sc_txrx_workqueue); + sc->sc_txrx_workqueue = NULL; } - vaddr = kmem_zalloc(allocsize2, KM_SLEEP); - sc->sc_kmem = vaddr; - p = (intptr_t) vaddr; - for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) { - int rxqsize, txqsize; - rxq = &sc->sc_rxq[qid]; - txq = &sc->sc_txq[qid]; - rxqsize = rxq->rxq_vq->vq_num; - txqsize = txq->txq_vq->vq_num; - - rxq->rxq_hdr_dmamaps = vioif_assign_mem(&p, - sizeof(rxq->rxq_hdr_dmamaps[0]) * rxqsize); - txq->txq_hdr_dmamaps = vioif_assign_mem(&p, - sizeof(txq->txq_hdr_dmamaps[0]) * txqsize); - rxq->rxq_dmamaps = vioif_assign_mem(&p, - sizeof(rxq->rxq_dmamaps[0]) * rxqsize); - txq->txq_dmamaps = vioif_assign_mem(&p, - sizeof(txq->txq_dmamaps[0]) * txqsize); - rxq->rxq_mbufs = vioif_assign_mem(&p, - sizeof(rxq->rxq_mbufs[0]) * rxqsize); - txq->txq_mbufs = vioif_assign_mem(&p, - sizeof(txq->txq_mbufs[0]) * txqsize); - } - - for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) { - rxq = &sc->sc_rxq[qid]; - txq = &sc->sc_txq[qid]; - - for (i = 0; i < rxq->rxq_vq->vq_num; i++) { - r = vioif_dmamap_create_load(sc, &rxq->rxq_hdr_dmamaps[i], - &rxq->rxq_hdrs[i], sc->sc_hdr_size, 1, - BUS_DMA_READ, "rx header"); - if (r != 0) - goto err_reqs; + return 0; +} - r = vioif_dmamap_create(sc, &rxq->rxq_dmamaps[i], - MCLBYTES, 1, "rx payload"); - if (r != 0) - goto err_reqs; - } +/* + * Interface functions for ifnet + */ +static int +vioif_init(struct ifnet *ifp) +{ + struct vioif_softc *sc = ifp->if_softc; + struct virtio_softc *vsc = sc->sc_virtio; + struct vioif_netqueue *netq; + struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; + int r, i; - for (i = 0; i < txq->txq_vq->vq_num; i++) { - r = vioif_dmamap_create_load(sc, &txq->txq_hdr_dmamaps[i], - &txq->txq_hdrs[i], sc->sc_hdr_size, 1, - BUS_DMA_READ, "tx header"); - if (r != 0) - goto err_reqs; + vioif_stop(ifp, 0); - r = vioif_dmamap_create(sc, &txq->txq_dmamaps[i], ETHER_MAX_LEN, - VIRTIO_NET_TX_MAXNSEGS, "tx payload"); - if (r != 0) - goto err_reqs; - } + r = virtio_reinit_start(vsc); + if (r != 0) { + log(LOG_ERR, "%s: reset failed\n", ifp->if_xname); + return EIO; } - if (sc->sc_has_ctrl) { - /* control vq class & command */ - r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_cmd_dmamap, - ctrlq->ctrlq_cmd, sizeof(*ctrlq->ctrlq_cmd), 1, - BUS_DMA_WRITE, "control command"); - if (r != 0) - goto err_reqs; + virtio_negotiate_features(vsc, virtio_features(vsc)); - r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_status_dmamap, - ctrlq->ctrlq_status, sizeof(*ctrlq->ctrlq_status), 1, - BUS_DMA_READ, "control status"); - if (r != 0) - goto err_reqs; + for (i = 0; i < sc->sc_req_nvq_pairs; i++) { + netq = &sc->sc_netqs[VIOIF_NETQ_RXQID(i)]; - /* control vq rx mode command parameter */ - r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_rx_dmamap, - ctrlq->ctrlq_rx, sizeof(*ctrlq->ctrlq_rx), 1, - BUS_DMA_WRITE, "rx mode control command"); - if (r != 0) - goto err_reqs; + mutex_enter(&netq->netq_lock); + vioif_populate_rx_mbufs_locked(sc, netq); + mutex_exit(&netq->netq_lock); + } - /* multiqueue set command */ - r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_mq_dmamap, - ctrlq->ctrlq_mq, sizeof(*ctrlq->ctrlq_mq), 1, - BUS_DMA_WRITE, "multiqueue set command"); - if (r != 0) - goto err_reqs; + virtio_reinit_end(vsc); - /* control vq MAC filter table for unicast */ - /* do not load now since its length is variable */ - r = vioif_dmamap_create(sc, &ctrlq->ctrlq_tbl_uc_dmamap, - sizeof(*ctrlq->ctrlq_mac_tbl_uc) - + ETHER_ADDR_LEN, 1, - "unicast MAC address filter command"); - if (r != 0) - goto err_reqs; + if (sc->sc_has_ctrl) + virtio_start_vq_intr(vsc, ctrlq->ctrlq_vq); - /* control vq MAC filter table for multicast */ - r = vioif_dmamap_create(sc, &ctrlq->ctrlq_tbl_mc_dmamap, - sizeof(*ctrlq->ctrlq_mac_tbl_mc) - + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES, 1, - "multicast MAC address filter command"); - if (r != 0) - goto err_reqs; + r = vioif_ctrl_mq_vq_pairs_set(sc, sc->sc_req_nvq_pairs); + if (r == 0) + sc->sc_act_nvq_pairs = sc->sc_req_nvq_pairs; + else + sc->sc_act_nvq_pairs = 1; - /* control vq MAC address set command */ - r = vioif_dmamap_create_load(sc, - &ctrlq->ctrlq_mac_addr_dmamap, - ctrlq->ctrlq_mac_addr, - sizeof(*ctrlq->ctrlq_mac_addr), 1, - BUS_DMA_WRITE, "mac addr set command"); - if (r != 0) - goto err_reqs; - } + SET(ifp->if_flags, IFF_RUNNING); - return 0; + vioif_net_intr_enable(sc, vsc); -err_reqs: - vioif_dmamap_destroy(sc, &ctrlq->ctrlq_tbl_mc_dmamap); - vioif_dmamap_destroy(sc, &ctrlq->ctrlq_tbl_uc_dmamap); - vioif_dmamap_destroy(sc, &ctrlq->ctrlq_rx_dmamap); - vioif_dmamap_destroy(sc, &ctrlq->ctrlq_status_dmamap); - vioif_dmamap_destroy(sc, &ctrlq->ctrlq_cmd_dmamap); - vioif_dmamap_destroy(sc, &ctrlq->ctrlq_mac_addr_dmamap); - for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) { - rxq = &sc->sc_rxq[qid]; - txq = &sc->sc_txq[qid]; - - for (i = 0; i < txq->txq_vq->vq_num; i++) { - vioif_dmamap_destroy(sc, &txq->txq_dmamaps[i]); - vioif_dmamap_destroy(sc, &txq->txq_hdr_dmamaps[i]); - } - for (i = 0; i < rxq->rxq_vq->vq_num; i++) { - vioif_dmamap_destroy(sc, &rxq->rxq_dmamaps[i]); - vioif_dmamap_destroy(sc, &rxq->rxq_hdr_dmamaps[i]); - } - } - if (sc->sc_kmem) { - kmem_free(sc->sc_kmem, allocsize2); - sc->sc_kmem = NULL; - } - bus_dmamem_unmap(virtio_dmat(vsc), sc->sc_dmamem, allocsize); -err_dmamem_alloc: - bus_dmamem_free(virtio_dmat(vsc), &sc->sc_hdr_segs[0], 1); -err_none: - return -1; + vioif_update_link_status(sc); + r = vioif_rx_filter(sc); + + return r; } static void -vioif_attach(device_t parent, device_t self, void *aux) +vioif_stop(struct ifnet *ifp, int disable) { - struct vioif_softc *sc = device_private(self); - struct virtio_softc *vsc = device_private(parent); + struct vioif_softc *sc = ifp->if_softc; + struct virtio_softc *vsc = sc->sc_virtio; + struct vioif_netqueue *netq; struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; - struct vioif_txqueue *txq; - struct vioif_rxqueue *rxq; - uint64_t features, req_features; - struct ifnet *ifp = &sc->sc_ethercom.ec_if; - u_int softint_flags; - int r, i, nvqs = 0, req_flags; - char xnamebuf[MAXCOMLEN]; + size_t i, act_qnum; - if (virtio_child(vsc) != NULL) { - aprint_normal(": child already attached for %s; " - "something wrong...\n", device_xname(parent)); - return; + act_qnum = sc->sc_act_nvq_pairs * 2; + + CLR(ifp->if_flags, IFF_RUNNING); + for (i = 0; i < act_qnum; i++) { + netq = &sc->sc_netqs[i]; + + mutex_enter(&netq->netq_lock); + netq->netq_stopping = true; + mutex_exit(&netq->netq_lock); } - sc->sc_dev = self; - sc->sc_virtio = vsc; - sc->sc_link_state = LINK_STATE_UNKNOWN; + /* disable interrupts */ + vioif_net_intr_disable(sc, vsc); + if (sc->sc_has_ctrl) + virtio_stop_vq_intr(vsc, ctrlq->ctrlq_vq); - sc->sc_max_nvq_pairs = 1; - sc->sc_req_nvq_pairs = 1; - sc->sc_act_nvq_pairs = 1; - sc->sc_txrx_workqueue_sysctl = true; - sc->sc_tx_intr_process_limit = VIOIF_TX_INTR_PROCESS_LIMIT; - sc->sc_tx_process_limit = VIOIF_TX_PROCESS_LIMIT; - sc->sc_rx_intr_process_limit = VIOIF_RX_INTR_PROCESS_LIMIT; - sc->sc_rx_process_limit = VIOIF_RX_PROCESS_LIMIT; + /* + * only way to stop interrupt, I/O and DMA is resetting... + * + * NOTE: Devices based on VirtIO draft specification can not + * stop interrupt completely even if virtio_stop_vq_intr() is called. + */ + virtio_reset(vsc); - mutex_init(&sc->sc_lock, MUTEX_DEFAULT, IPL_NONE); + vioif_intr_barrier(); - snprintf(xnamebuf, sizeof(xnamebuf), "%s_txrx", device_xname(self)); - sc->sc_txrx_workqueue = vioif_workq_create(xnamebuf, VIOIF_WORKQUEUE_PRI, - IPL_NET, WQ_PERCPU | WQ_MPSAFE); - if (sc->sc_txrx_workqueue == NULL) - goto err; + for (i = 0; i < act_qnum; i++) { + netq = &sc->sc_netqs[i]; + vioif_work_wait(sc->sc_txrx_workqueue, &netq->netq_work); + } - req_flags = 0; + for (i = 0; i < sc->sc_act_nvq_pairs; i++) { + netq = &sc->sc_netqs[VIOIF_NETQ_RXQID(i)]; + vioif_rx_queue_clear(sc, vsc, netq); + + netq = &sc->sc_netqs[VIOIF_NETQ_TXQID(i)]; + vioif_tx_queue_clear(sc, vsc, netq); + } + + /* all packet processing is stopped */ + for (i = 0; i < act_qnum; i++) { + netq = &sc->sc_netqs[i]; + + mutex_enter(&netq->netq_lock); + netq->netq_stopping = false; + mutex_exit(&netq->netq_lock); + } +} + +static void +vioif_start(struct ifnet *ifp) +{ + struct vioif_softc *sc = ifp->if_softc; + struct vioif_netqueue *txq0 = &sc->sc_netqs[VIOIF_NETQ_TXQID(0)]; #ifdef VIOIF_MPSAFE - req_flags |= VIRTIO_F_INTR_MPSAFE; + KASSERT(if_is_mpsafe(ifp)); #endif - req_flags |= VIRTIO_F_INTR_MSIX; - req_features = - VIRTIO_NET_F_MAC | VIRTIO_NET_F_STATUS | VIRTIO_NET_F_CTRL_VQ | - VIRTIO_NET_F_CTRL_RX | VIRTIO_F_NOTIFY_ON_EMPTY; - req_features |= VIRTIO_F_RING_EVENT_IDX; - req_features |= VIRTIO_NET_F_CTRL_MAC_ADDR; -#ifdef VIOIF_MULTIQ - req_features |= VIRTIO_NET_F_MQ; -#endif - virtio_child_attach_start(vsc, self, IPL_NET, NULL, - vioif_config_change, virtio_vq_intrhand, req_flags, - req_features, VIRTIO_NET_FLAG_BITS); + mutex_enter(&txq0->netq_lock); + vioif_start_locked(ifp, txq0); + mutex_exit(&txq0->netq_lock); +} - features = virtio_features(vsc); - if (features == 0) - goto err; +static inline int +vioif_select_txqueue(struct ifnet *ifp, struct mbuf *m) +{ + struct vioif_softc *sc = ifp->if_softc; + u_int cpuid = cpu_index(curcpu()); - if (features & VIRTIO_NET_F_MAC) { - for (i = 0; i < __arraycount(sc->sc_mac); i++) { - sc->sc_mac[i] = virtio_read_device_config_1(vsc, - VIRTIO_NET_CONFIG_MAC + i); + return VIOIF_NETQ_TXQID(cpuid % sc->sc_act_nvq_pairs); +} + +static int +vioif_transmit(struct ifnet *ifp, struct mbuf *m) +{ + struct vioif_softc *sc = ifp->if_softc; + struct vioif_netqueue *netq; + struct vioif_tx_context *txc; + int qid; + + qid = vioif_select_txqueue(ifp, m); + netq = &sc->sc_netqs[qid]; + txc = netq->netq_ctx; + + if (__predict_false(!pcq_put(txc->txc_intrq, m))) { + m_freem(m); + return ENOBUFS; + } + + net_stat_ref_t nsr = IF_STAT_GETREF(ifp); + if_statadd_ref(nsr, if_obytes, m->m_pkthdr.len); + if (m->m_flags & M_MCAST) + if_statinc_ref(nsr, if_omcasts); + IF_STAT_PUTREF(ifp); + + if (mutex_tryenter(&netq->netq_lock)) { + vioif_transmit_locked(ifp, netq); + mutex_exit(&netq->netq_lock); + } + + return 0; +} + +void +vioif_watchdog(struct ifnet *ifp) +{ + struct vioif_softc *sc = ifp->if_softc; + struct vioif_netqueue *netq; + int i; + + if (ISSET(ifp->if_flags, IFF_RUNNING)) { + if (ISSET(ifp->if_flags, IFF_DEBUG)) { + log(LOG_DEBUG, "%s: watchdog timed out\n", + ifp->if_xname); } - } else { - /* code stolen from sys/net/if_tap.c */ - struct timeval tv; - uint32_t ui; - getmicrouptime(&tv); - ui = (tv.tv_sec ^ tv.tv_usec) & 0xffffff; - memcpy(sc->sc_mac+3, (uint8_t *)&ui, 3); - for (i = 0; i < __arraycount(sc->sc_mac); i++) { - virtio_write_device_config_1(vsc, - VIRTIO_NET_CONFIG_MAC + i, sc->sc_mac[i]); + + for (i = 0; i < sc->sc_act_nvq_pairs; i++) { + netq = &sc->sc_netqs[VIOIF_NETQ_TXQID(i)]; + + mutex_enter(&netq->netq_lock); + if (!netq->netq_running_handle) { + netq->netq_running_handle = true; + vioif_net_sched_handle(sc, netq); + } + mutex_exit(&netq->netq_lock); + } + } +} + +static int +vioif_ioctl(struct ifnet *ifp, u_long cmd, void *data) +{ + int s, r; + + s = splnet(); + + r = ether_ioctl(ifp, cmd, data); + if (r == ENETRESET && (cmd == SIOCADDMULTI || cmd == SIOCDELMULTI)) { + if (ifp->if_flags & IFF_RUNNING) { + r = vioif_rx_filter(ifp->if_softc); + } else { + r = 0; + } + } + + splx(s); + + return r; +} + +static int +vioif_ifflags(struct vioif_softc *sc) +{ + struct ifnet *ifp = &sc->sc_ethercom.ec_if; + bool onoff; + int r; + + if (!sc->sc_has_ctrl) { + /* no ctrl vq; always promisc and allmulti */ + ifp->if_flags |= (IFF_PROMISC | IFF_ALLMULTI); + return 0; + } + + onoff = ifp->if_flags & IFF_ALLMULTI ? true : false; + r = vioif_set_allmulti(sc, onoff); + if (r != 0) { + log(LOG_WARNING, + "%s: couldn't %sable ALLMULTI\n", + ifp->if_xname, onoff ? "en" : "dis"); + if (onoff) { + CLR(ifp->if_flags, IFF_ALLMULTI); + } else { + SET(ifp->if_flags, IFF_ALLMULTI); + } + } + + onoff = ifp->if_flags & IFF_PROMISC ? true : false; + r = vioif_set_promisc(sc, onoff); + if (r != 0) { + log(LOG_WARNING, + "%s: couldn't %sable PROMISC\n", + ifp->if_xname, onoff ? "en" : "dis"); + if (onoff) { + CLR(ifp->if_flags, IFF_PROMISC); + } else { + SET(ifp->if_flags, IFF_PROMISC); + } + } + + return 0; +} + +static int +vioif_ifflags_cb(struct ethercom *ec) +{ + struct ifnet *ifp = &ec->ec_if; + struct vioif_softc *sc = ifp->if_softc; + + return vioif_ifflags(sc); +} + +static int +vioif_setup_sysctl(struct vioif_softc *sc) +{ + const char *devname; + struct sysctllog **log; + const struct sysctlnode *rnode, *rxnode, *txnode; + int error; + + log = &sc->sc_sysctllog; + devname = device_xname(sc->sc_dev); + + error = sysctl_createv(log, 0, NULL, &rnode, + 0, CTLTYPE_NODE, devname, + SYSCTL_DESCR("virtio-net information and settings"), + NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL); + if (error) + goto out; + + error = sysctl_createv(log, 0, &rnode, NULL, + CTLFLAG_READWRITE, CTLTYPE_BOOL, "txrx_workqueue", + SYSCTL_DESCR("Use workqueue for packet processing"), + NULL, 0, &sc->sc_txrx_workqueue_sysctl, 0, CTL_CREATE, CTL_EOL); + if (error) + goto out; + + error = sysctl_createv(log, 0, &rnode, &rxnode, + 0, CTLTYPE_NODE, "rx", + SYSCTL_DESCR("virtio-net information and settings for Rx"), + NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); + if (error) + goto out; + + error = sysctl_createv(log, 0, &rxnode, NULL, + CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", + SYSCTL_DESCR("max number of Rx packets to process for interrupt processing"), + NULL, 0, &sc->sc_rx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); + if (error) + goto out; + + error = sysctl_createv(log, 0, &rxnode, NULL, + CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", + SYSCTL_DESCR("max number of Rx packets to process for deferred processing"), + NULL, 0, &sc->sc_rx_process_limit, 0, CTL_CREATE, CTL_EOL); + if (error) + goto out; + + error = sysctl_createv(log, 0, &rnode, &txnode, + 0, CTLTYPE_NODE, "tx", + SYSCTL_DESCR("virtio-net information and settings for Tx"), + NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); + if (error) + goto out; + + error = sysctl_createv(log, 0, &txnode, NULL, + CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", + SYSCTL_DESCR("max number of Tx packets to process for interrupt processing"), + NULL, 0, &sc->sc_tx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); + if (error) + goto out; + + error = sysctl_createv(log, 0, &txnode, NULL, + CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", + SYSCTL_DESCR("max number of Tx packets to process for deferred processing"), + NULL, 0, &sc->sc_tx_process_limit, 0, CTL_CREATE, CTL_EOL); + +out: + if (error) + sysctl_teardown(log); + + return error; +} + +static void +vioif_setup_stats(struct vioif_softc *sc) +{ + struct vioif_netqueue *netq; + struct vioif_tx_context *txc; + struct vioif_rx_context *rxc; + size_t i, netq_num; + + netq_num = sc->sc_max_nvq_pairs * 2; + for (i = 0; i < netq_num; i++) { + netq = &sc->sc_netqs[i]; + evcnt_attach_dynamic(&netq->netq_mbuf_load_failed, EVCNT_TYPE_MISC, + NULL, netq->netq_evgroup, "failed to load mbuf to DMA"); + evcnt_attach_dynamic(&netq->netq_enqueue_failed, + EVCNT_TYPE_MISC, NULL, netq->netq_evgroup, + "virtqueue enqueue failed failed"); + + switch (VIOIF_NETQ_DIR(i)) { + case VIOIF_NETQ_RX: + rxc = netq->netq_ctx; + evcnt_attach_dynamic(&rxc->rxc_mbuf_enobufs, + EVCNT_TYPE_MISC, NULL, netq->netq_evgroup, + "no receive buffer"); + break; + case VIOIF_NETQ_TX: + txc = netq->netq_ctx; + evcnt_attach_dynamic(&txc->txc_defrag_failed, + EVCNT_TYPE_MISC, NULL, netq->netq_evgroup, + "m_defrag() failed"); + break; } } - /* 'Ethernet' with capital follows other ethernet driver attachment */ - aprint_normal_dev(self, "Ethernet address %s\n", - ether_sprintf(sc->sc_mac)); + evcnt_attach_dynamic(&sc->sc_ctrlq.ctrlq_cmd_load_failed, EVCNT_TYPE_MISC, + NULL, device_xname(sc->sc_dev), "control command dmamap load failed"); + evcnt_attach_dynamic(&sc->sc_ctrlq.ctrlq_cmd_failed, EVCNT_TYPE_MISC, + NULL, device_xname(sc->sc_dev), "control command failed"); +} + +/* + * allocate memory + */ +static int +vioif_dmamap_create(struct vioif_softc *sc, bus_dmamap_t *map, + bus_size_t size, int nsegs, const char *usage) +{ + int r; + + r = bus_dmamap_create(virtio_dmat(sc->sc_virtio), size, + nsegs, size, 0, BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, map); + + if (r != 0) { + aprint_error_dev(sc->sc_dev, "%s dmamap creation failed, " + "error code %d\n", usage, r); + } + + return r; +} + +static void +vioif_dmamap_destroy(struct vioif_softc *sc, bus_dmamap_t *map) +{ + + if (*map) { + bus_dmamap_destroy(virtio_dmat(sc->sc_virtio), *map); + *map = NULL; + } +} + +static int +vioif_dmamap_create_load(struct vioif_softc *sc, bus_dmamap_t *map, + void *buf, bus_size_t size, int nsegs, int rw, const char *usage) +{ + int r; + + r = vioif_dmamap_create(sc, map, size, nsegs, usage); + if (r != 0) + return 1; + + r = bus_dmamap_load(virtio_dmat(sc->sc_virtio), *map, buf, + size, NULL, rw | BUS_DMA_NOWAIT); + if (r != 0) { + vioif_dmamap_destroy(sc, map); + aprint_error_dev(sc->sc_dev, "%s dmamap load failed. " + "error code %d\n", usage, r); + } + + return r; +} + +static void * +vioif_assign_mem(intptr_t *p, size_t size) +{ + intptr_t rv; + + rv = *p; + *p += size; + + return (void *)rv; +} + +/* + * dma memory is used for: + * netq_maps_kva: metadata array for received frames (READ) and + * sent frames (WRITE) + * ctrlq_cmd: command to be sent via ctrl vq (WRITE) + * ctrlq_status: return value for a command via ctrl vq (READ) + * ctrlq_rx: parameter for a VIRTIO_NET_CTRL_RX class command + * (WRITE) + * ctrlq_mac_tbl_uc: unicast MAC address filter for a VIRTIO_NET_CTRL_MAC + * class command (WRITE) + * ctrlq_mac_tbl_mc: multicast MAC address filter for a VIRTIO_NET_CTRL_MAC + * class command (WRITE) + * ctrlq_* structures are allocated only one each; they are protected by + * ctrlq_inuse variable and ctrlq_wait condvar. + */ +static int +vioif_alloc_mems(struct vioif_softc *sc) +{ + struct virtio_softc *vsc = sc->sc_virtio; + struct vioif_netqueue *netq; + struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; + struct vioif_net_map *maps; + unsigned int vq_num; + int r, rsegs; + bus_size_t dmamemsize; + size_t qid, i, netq_num, kmemsize; + void *vaddr; + intptr_t p; + + netq_num = sc->sc_max_nvq_pairs * 2; + + /* allocate DMA memory */ + dmamemsize = 0; + + for (qid = 0; qid < netq_num; qid++) { + maps = sc->sc_netqs[qid].netq_maps; + vq_num = sc->sc_netqs[qid].netq_vq->vq_num; + dmamemsize += sizeof(*maps[0].vnm_hdr) * vq_num; + } + + if (sc->sc_has_ctrl) { + dmamemsize += sizeof(struct virtio_net_ctrl_cmd); + dmamemsize += sizeof(struct virtio_net_ctrl_status); + dmamemsize += sizeof(struct virtio_net_ctrl_rx); + dmamemsize += sizeof(struct virtio_net_ctrl_mac_tbl) + + ETHER_ADDR_LEN; + dmamemsize += sizeof(struct virtio_net_ctrl_mac_tbl) + + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES; + dmamemsize += sizeof(struct virtio_net_ctrl_mac_addr); + dmamemsize += sizeof(struct virtio_net_ctrl_mq); + } + + r = bus_dmamem_alloc(virtio_dmat(vsc), dmamemsize, 0, 0, + &sc->sc_segs[0], 1, &rsegs, BUS_DMA_NOWAIT); + if (r != 0) { + aprint_error_dev(sc->sc_dev, + "DMA memory allocation failed, size %zu, " + "error code %d\n", dmamemsize, r); + goto err_none; + } + r = bus_dmamem_map(virtio_dmat(vsc), &sc->sc_segs[0], 1, + dmamemsize, &vaddr, BUS_DMA_NOWAIT); + if (r != 0) { + aprint_error_dev(sc->sc_dev, + "DMA memory map failed, error code %d\n", r); + goto err_dmamem_alloc; + } + + /* assign DMA memory */ + memset(vaddr, 0, dmamemsize); + sc->sc_dmamem = vaddr; + p = (intptr_t) vaddr; + + for (qid = 0; qid < netq_num; qid++) { + netq = &sc->sc_netqs[qid]; + maps = netq->netq_maps; + vq_num = netq->netq_vq->vq_num; + + netq->netq_maps_kva = vioif_assign_mem(&p, + sizeof(*maps[0].vnm_hdr) * vq_num); + } + + if (sc->sc_has_ctrl) { + ctrlq->ctrlq_cmd = vioif_assign_mem(&p, + sizeof(*ctrlq->ctrlq_cmd)); + ctrlq->ctrlq_status = vioif_assign_mem(&p, + sizeof(*ctrlq->ctrlq_status)); + ctrlq->ctrlq_rx = vioif_assign_mem(&p, + sizeof(*ctrlq->ctrlq_rx)); + ctrlq->ctrlq_mac_tbl_uc = vioif_assign_mem(&p, + sizeof(*ctrlq->ctrlq_mac_tbl_uc) + + ETHER_ADDR_LEN); + ctrlq->ctrlq_mac_tbl_mc = vioif_assign_mem(&p, + sizeof(*ctrlq->ctrlq_mac_tbl_mc) + + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES); + ctrlq->ctrlq_mac_addr = vioif_assign_mem(&p, + sizeof(*ctrlq->ctrlq_mac_addr)); + ctrlq->ctrlq_mq = vioif_assign_mem(&p, sizeof(*ctrlq->ctrlq_mq)); + } + + /* allocate kmem */ + kmemsize = 0; - if (features & (VIRTIO_NET_F_MRG_RXBUF | VIRTIO_F_VERSION_1)) { - sc->sc_hdr_size = sizeof(struct virtio_net_hdr); - } else { - sc->sc_hdr_size = offsetof(struct virtio_net_hdr, num_buffers); + for (qid = 0; qid < netq_num; qid++) { + netq = &sc->sc_netqs[qid]; + vq_num = netq->netq_vq->vq_num; + + kmemsize += sizeof(netq->netq_maps[0]) * vq_num; } - if ((features & VIRTIO_NET_F_CTRL_VQ) && - (features & VIRTIO_NET_F_CTRL_RX)) { - sc->sc_has_ctrl = true; + vaddr = kmem_zalloc(kmemsize, KM_SLEEP); + sc->sc_kmem = vaddr; - cv_init(&ctrlq->ctrlq_wait, "ctrl_vq"); - mutex_init(&ctrlq->ctrlq_wait_lock, MUTEX_DEFAULT, IPL_NET); - ctrlq->ctrlq_inuse = FREE; - } else { - sc->sc_has_ctrl = false; - } + /* assign allocated kmem */ + p = (intptr_t) vaddr; - if (sc->sc_has_ctrl && (features & VIRTIO_NET_F_MQ)) { - sc->sc_max_nvq_pairs = virtio_read_device_config_2(vsc, - VIRTIO_NET_CONFIG_MAX_VQ_PAIRS); + for (qid = 0; qid < netq_num; qid++) { + netq = &sc->sc_netqs[qid]; + vq_num = netq->netq_vq->vq_num; + + netq->netq_maps = vioif_assign_mem(&p, + sizeof(netq->netq_maps[0]) * vq_num); + } + + /* prepare dmamaps */ + for (qid = 0; qid < netq_num; qid++) { + static const struct { + const char *msg_hdr; + const char *msg_payload; + int dma_flag; + bus_size_t dma_size; + int dma_nsegs; + } dmaparams[VIOIF_NETQ_IDX] = { + [VIOIF_NETQ_RX] = { + .msg_hdr = "rx header", + .msg_payload = "rx payload", + .dma_flag = BUS_DMA_READ, + .dma_size = MCLBYTES - ETHER_ALIGN, + .dma_nsegs = 1, + }, + [VIOIF_NETQ_TX] = { + .msg_hdr = "tx header", + .msg_payload = "tx payload", + .dma_flag = BUS_DMA_WRITE, + .dma_size = ETHER_MAX_LEN, + .dma_nsegs = VIRTIO_NET_TX_MAXNSEGS, + } + }; - if (sc->sc_max_nvq_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX) - goto err; + struct virtio_net_hdr *hdrs; + int dir; - /* Limit the number of queue pairs to use */ - sc->sc_req_nvq_pairs = MIN(sc->sc_max_nvq_pairs, ncpu); + dir = VIOIF_NETQ_DIR(qid); + netq = &sc->sc_netqs[qid]; + vq_num = netq->netq_vq->vq_num; + maps = netq->netq_maps; + hdrs = netq->netq_maps_kva; + + for (i = 0; i < vq_num; i++) { + maps[i].vnm_hdr = &hdrs[i]; + + r = vioif_dmamap_create_load(sc, &maps[i].vnm_hdr_map, + maps[i].vnm_hdr, sc->sc_hdr_size, 1, + dmaparams[dir].dma_flag, dmaparams[dir].msg_hdr); + if (r != 0) + goto err_reqs; + + r = vioif_dmamap_create(sc, &maps[i].vnm_mbuf_map, + dmaparams[dir].dma_size, dmaparams[dir].dma_nsegs, + dmaparams[dir].msg_payload); + if (r != 0) + goto err_reqs; + } } - vioif_alloc_queues(sc); - virtio_child_attach_set_vqs(vsc, sc->sc_vqs, sc->sc_req_nvq_pairs); + if (sc->sc_has_ctrl) { + /* control vq class & command */ + r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_cmd_dmamap, + ctrlq->ctrlq_cmd, sizeof(*ctrlq->ctrlq_cmd), 1, + BUS_DMA_WRITE, "control command"); + if (r != 0) + goto err_reqs; -#ifdef VIOIF_MPSAFE - softint_flags = SOFTINT_NET | SOFTINT_MPSAFE; -#else - softint_flags = SOFTINT_NET; -#endif + r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_status_dmamap, + ctrlq->ctrlq_status, sizeof(*ctrlq->ctrlq_status), 1, + BUS_DMA_READ, "control status"); + if (r != 0) + goto err_reqs; - /* - * Allocating virtqueues - */ - for (i = 0; i < sc->sc_max_nvq_pairs; i++) { - rxq = &sc->sc_rxq[i]; - txq = &sc->sc_txq[i]; - char qname[32]; - - rxq->rxq_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET); - - rxq->rxq_handle_si = softint_establish(softint_flags, - vioif_rx_handle, rxq); - if (rxq->rxq_handle_si == NULL) { - aprint_error_dev(self, "cannot establish rx softint\n"); - goto err; - } + /* control vq rx mode command parameter */ + r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_rx_dmamap, + ctrlq->ctrlq_rx, sizeof(*ctrlq->ctrlq_rx), 1, + BUS_DMA_WRITE, "rx mode control command"); + if (r != 0) + goto err_reqs; - snprintf(qname, sizeof(qname), "rx%d", i); - r = virtio_alloc_vq(vsc, rxq->rxq_vq, nvqs, - MCLBYTES + sc->sc_hdr_size, 2, qname); + /* multiqueue set command */ + r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_mq_dmamap, + ctrlq->ctrlq_mq, sizeof(*ctrlq->ctrlq_mq), 1, + BUS_DMA_WRITE, "multiqueue set command"); if (r != 0) - goto err; - nvqs++; - rxq->rxq_vq->vq_intrhand = vioif_rx_intr; - rxq->rxq_vq->vq_intrhand_arg = (void *)rxq; - rxq->rxq_stopping = true; - vioif_work_set(&rxq->rxq_work, vioif_rx_handle, rxq); - - txq->txq_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET); - - txq->txq_deferred_transmit = softint_establish(softint_flags, - vioif_deferred_transmit, txq); - if (txq->txq_deferred_transmit == NULL) { - aprint_error_dev(self, "cannot establish tx softint\n"); - goto err; - } - txq->txq_handle_si = softint_establish(softint_flags, - vioif_tx_handle, txq); - if (txq->txq_handle_si == NULL) { - aprint_error_dev(self, "cannot establish tx softint\n"); - goto err; - } + goto err_reqs; - snprintf(qname, sizeof(qname), "tx%d", i); - r = virtio_alloc_vq(vsc, txq->txq_vq, nvqs, - sc->sc_hdr_size + (ETHER_MAX_LEN - ETHER_HDR_LEN), - VIRTIO_NET_TX_MAXNSEGS + 1, qname); + /* control vq MAC filter table for unicast */ + /* do not load now since its length is variable */ + r = vioif_dmamap_create(sc, &ctrlq->ctrlq_tbl_uc_dmamap, + sizeof(*ctrlq->ctrlq_mac_tbl_uc) + + ETHER_ADDR_LEN, 1, + "unicast MAC address filter command"); if (r != 0) - goto err; - nvqs++; - txq->txq_vq->vq_intrhand = vioif_tx_intr; - txq->txq_vq->vq_intrhand_arg = (void *)txq; - txq->txq_link_active = VIOIF_IS_LINK_ACTIVE(sc); - txq->txq_stopping = false; - txq->txq_intrq = pcq_create(txq->txq_vq->vq_num, KM_SLEEP); - vioif_work_set(&txq->txq_work, vioif_tx_handle, txq); + goto err_reqs; + + /* control vq MAC filter table for multicast */ + r = vioif_dmamap_create(sc, &ctrlq->ctrlq_tbl_mc_dmamap, + sizeof(*ctrlq->ctrlq_mac_tbl_mc) + + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES, 1, + "multicast MAC address filter command"); + if (r != 0) + goto err_reqs; + + /* control vq MAC address set command */ + r = vioif_dmamap_create_load(sc, + &ctrlq->ctrlq_mac_addr_dmamap, + ctrlq->ctrlq_mac_addr, + sizeof(*ctrlq->ctrlq_mac_addr), 1, + BUS_DMA_WRITE, "mac addr set command"); + if (r != 0) + goto err_reqs; } - if (sc->sc_has_ctrl) { - /* - * Allocating a virtqueue for control channel - */ - r = virtio_alloc_vq(vsc, ctrlq->ctrlq_vq, nvqs, - NBPG, 1, "control"); - if (r != 0) { - aprint_error_dev(self, "failed to allocate " - "a virtqueue for control channel, error code %d\n", - r); + return 0; - sc->sc_has_ctrl = false; - cv_destroy(&ctrlq->ctrlq_wait); - mutex_destroy(&ctrlq->ctrlq_wait_lock); - } else { - nvqs++; - ctrlq->ctrlq_vq->vq_intrhand = vioif_ctrl_intr; - ctrlq->ctrlq_vq->vq_intrhand_arg = (void *) ctrlq; +err_reqs: + vioif_dmamap_destroy(sc, &ctrlq->ctrlq_tbl_mc_dmamap); + vioif_dmamap_destroy(sc, &ctrlq->ctrlq_tbl_uc_dmamap); + vioif_dmamap_destroy(sc, &ctrlq->ctrlq_rx_dmamap); + vioif_dmamap_destroy(sc, &ctrlq->ctrlq_status_dmamap); + vioif_dmamap_destroy(sc, &ctrlq->ctrlq_cmd_dmamap); + vioif_dmamap_destroy(sc, &ctrlq->ctrlq_mac_addr_dmamap); + for (qid = 0; qid < netq_num; qid++) { + vq_num = sc->sc_netqs[qid].netq_vq->vq_num; + maps = sc->sc_netqs[qid].netq_maps; + + for (i = 0; i < vq_num; i++) { + vioif_dmamap_destroy(sc, &maps[i].vnm_mbuf_map); + vioif_dmamap_destroy(sc, &maps[i].vnm_hdr_map); } } - - sc->sc_ctl_softint = softint_establish(softint_flags, - vioif_ctl_softint, sc); - if (sc->sc_ctl_softint == NULL) { - aprint_error_dev(self, "cannot establish ctl softint\n"); - goto err; + if (sc->sc_kmem) { + kmem_free(sc->sc_kmem, kmemsize); + sc->sc_kmem = NULL; } + bus_dmamem_unmap(virtio_dmat(vsc), sc->sc_dmamem, dmamemsize); +err_dmamem_alloc: + bus_dmamem_free(virtio_dmat(vsc), &sc->sc_segs[0], 1); +err_none: + return -1; +} - if (vioif_alloc_mems(sc) < 0) - goto err; - - if (virtio_child_attach_finish(vsc) != 0) - goto err; - - if (vioif_setup_sysctl(sc) != 0) { - aprint_error_dev(self, "unable to create sysctl node\n"); - /* continue */ - } +static void +vioif_alloc_queues(struct vioif_softc *sc) +{ + int nvq_pairs = sc->sc_max_nvq_pairs; + size_t nvqs, netq_num; - vioif_setup_stats(sc); + KASSERT(nvq_pairs <= VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX); - strlcpy(ifp->if_xname, device_xname(self), IFNAMSIZ); - ifp->if_softc = sc; - ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; -#ifdef VIOIF_MPSAFE - ifp->if_extflags = IFEF_MPSAFE; -#endif - ifp->if_start = vioif_start; - if (sc->sc_req_nvq_pairs > 1) - ifp->if_transmit = vioif_transmit; - ifp->if_ioctl = vioif_ioctl; - ifp->if_init = vioif_init; - ifp->if_stop = vioif_stop; - ifp->if_capabilities = 0; - ifp->if_watchdog = vioif_watchdog; - txq = &sc->sc_txq[0]; - IFQ_SET_MAXLEN(&ifp->if_snd, MAX(txq->txq_vq->vq_num, IFQ_MAXLEN)); - IFQ_SET_READY(&ifp->if_snd); + nvqs = netq_num = sc->sc_max_nvq_pairs * 2; + if (sc->sc_has_ctrl) + nvqs++; - sc->sc_ethercom.ec_capabilities |= ETHERCAP_VLAN_MTU; + sc->sc_vqs = kmem_zalloc(sizeof(sc->sc_vqs[0]) * nvqs, KM_SLEEP); + sc->sc_netqs = kmem_zalloc(sizeof(sc->sc_netqs[0]) * netq_num, + KM_SLEEP); +} - if_attach(ifp); - if_deferred_start_init(ifp, NULL); - ether_ifattach(ifp, sc->sc_mac); - ether_set_ifflags_cb(&sc->sc_ethercom, vioif_ifflags_cb); +static void +vioif_free_queues(struct vioif_softc *sc) +{ + size_t nvqs, netq_num; - return; + nvqs = netq_num = sc->sc_max_nvq_pairs * 2; + if (sc->sc_ctrlq.ctrlq_vq) + nvqs++; -err: - for (i = 0; i < sc->sc_max_nvq_pairs; i++) { - rxq = &sc->sc_rxq[i]; - txq = &sc->sc_txq[i]; + kmem_free(sc->sc_netqs, sizeof(sc->sc_netqs[0]) * netq_num); + kmem_free(sc->sc_vqs, sizeof(sc->sc_vqs[0]) * nvqs); + sc->sc_netqs = NULL; + sc->sc_vqs = NULL; +} - if (rxq->rxq_lock) { - mutex_obj_free(rxq->rxq_lock); - rxq->rxq_lock = NULL; - } +/* + * Network queues + */ +static int +vioif_netqueue_init(struct vioif_softc *sc, struct virtio_softc *vsc, + size_t qid, u_int softint_flags) +{ + static const struct { + const char *dirname; + int segsize; + int nsegs; + int (*intrhand)(void *); + void (*sihand)(void *); + } params[VIOIF_NETQ_IDX] = { + [VIOIF_NETQ_RX] = { + .dirname = "rx", + .segsize = MCLBYTES, + .nsegs = 2, + .intrhand = vioif_rx_intr, + .sihand = vioif_rx_handle, + }, + [VIOIF_NETQ_TX] = { + .dirname = "tx", + .segsize = ETHER_MAX_LEN - ETHER_HDR_LEN, + .nsegs = 2, + .intrhand = vioif_tx_intr, + .sihand = vioif_tx_handle, + } + }; + + struct virtqueue *vq; + struct vioif_netqueue *netq; + struct vioif_tx_context *txc; + struct vioif_rx_context *rxc; + char qname[32]; + int r, dir; + + txc = NULL; + rxc = NULL; + netq = &sc->sc_netqs[qid]; + vq = &sc->sc_vqs[qid]; + dir = VIOIF_NETQ_DIR(qid); + + netq->netq_vq = &sc->sc_vqs[qid]; + netq->netq_stopping = false; + netq->netq_running_handle = false; + + snprintf(qname, sizeof(qname), "%s%zu", + params[dir].dirname, VIOIF_NETQ_PAIRIDX(qid)); + snprintf(netq->netq_evgroup, sizeof(netq->netq_evgroup), + "%s-%s", device_xname(sc->sc_dev), qname); + + mutex_init(&netq->netq_lock, MUTEX_DEFAULT, IPL_NET); + r = virtio_alloc_vq(vsc, vq, qid, + params[dir].segsize + sc->sc_hdr_size, + params[dir].nsegs, qname); + if (r != 0) + goto err; + netq->netq_vq = vq; - if (rxq->rxq_handle_si) { - softint_disestablish(rxq->rxq_handle_si); - rxq->rxq_handle_si = NULL; - } + netq->netq_vq->vq_intrhand = params[dir].intrhand; + netq->netq_vq->vq_intrhand_arg = netq; + netq->netq_softint = softint_establish(softint_flags, + params[dir].sihand, netq); + if (netq->netq_softint == NULL) { + aprint_error_dev(sc->sc_dev, + "couldn't establish %s softint\n", + params[dir].dirname); + goto err; + } + vioif_work_set(&netq->netq_work, params[dir].sihand, netq); - if (txq->txq_lock) { - mutex_obj_free(txq->txq_lock); - txq->txq_lock = NULL; + switch (dir) { + case VIOIF_NETQ_RX: + rxc = kmem_zalloc(sizeof(*rxc), KM_SLEEP); + netq->netq_ctx = rxc; + /* nothing to do */ + break; + case VIOIF_NETQ_TX: + txc = kmem_zalloc(sizeof(*txc), KM_SLEEP); + netq->netq_ctx = (void *)txc; + txc->txc_deferred_transmit = softint_establish(softint_flags, + vioif_deferred_transmit, netq); + if (txc->txc_deferred_transmit == NULL) { + aprint_error_dev(sc->sc_dev, + "couldn't establish softint for " + "tx deferred transmit\n"); + goto err; } + txc->txc_link_active = VIOIF_IS_LINK_ACTIVE(sc); + txc->txc_no_free_slots = false; + txc->txc_intrq = pcq_create(vq->vq_num, KM_SLEEP); + break; + } - if (txq->txq_handle_si) { - softint_disestablish(txq->txq_handle_si); - txq->txq_handle_si = NULL; - } + return 0; - if (txq->txq_deferred_transmit) { - softint_disestablish(txq->txq_deferred_transmit); - txq->txq_deferred_transmit = NULL; - } +err: + netq->netq_ctx = NULL; - if (txq->txq_intrq) { - pcq_destroy(txq->txq_intrq); - txq->txq_intrq = NULL; - } + if (rxc != NULL) { + kmem_free(rxc, sizeof(*rxc)); } - if (sc->sc_has_ctrl) { - cv_destroy(&ctrlq->ctrlq_wait); - mutex_destroy(&ctrlq->ctrlq_wait_lock); + if (txc != NULL) { + if (txc->txc_deferred_transmit != NULL) + softint_disestablish(txc->txc_deferred_transmit); + if (txc->txc_intrq != NULL) + pcq_destroy(txc->txc_intrq); + kmem_free(txc, sizeof(txc)); } - while (nvqs > 0) - virtio_free_vq(vsc, &sc->sc_vqs[--nvqs]); + vioif_work_set(&netq->netq_work, NULL, NULL); + if (netq->netq_softint != NULL) { + softint_disestablish(netq->netq_softint); + netq->netq_softint = NULL; + } + netq->netq_vq->vq_intrhand = NULL; + netq->netq_vq->vq_intrhand_arg = NULL; - vioif_free_queues(sc); - mutex_destroy(&sc->sc_lock); - virtio_child_attach_failed(vsc); - config_finalize_register(self, vioif_finalize_teardown); + virtio_free_vq(vsc, vq); + mutex_destroy(&netq->netq_lock); + netq->netq_vq = NULL; - return; + return -1; } -static int -vioif_finalize_teardown(device_t self) +static void +vioif_netqueue_teardown(struct vioif_softc *sc, struct virtio_softc *vsc, + size_t qid) { - struct vioif_softc *sc = device_private(self); + struct vioif_netqueue *netq; + struct vioif_rx_context *rxc; + struct vioif_tx_context *txc; + int dir; - if (sc->sc_txrx_workqueue != NULL) { - vioif_workq_destroy(sc->sc_txrx_workqueue); - sc->sc_txrx_workqueue = NULL; + netq = &sc->sc_netqs[qid]; + + if (netq->netq_vq == NULL) + return; + + netq = &sc->sc_netqs[qid]; + dir = VIOIF_NETQ_DIR(qid); + switch (dir) { + case VIOIF_NETQ_RX: + rxc = netq->netq_ctx; + netq->netq_ctx = NULL; + kmem_free(rxc, sizeof(*rxc)); + break; + case VIOIF_NETQ_TX: + txc = netq->netq_ctx; + netq->netq_ctx = NULL; + softint_disestablish(txc->txc_deferred_transmit); + pcq_destroy(txc->txc_intrq); + kmem_free(txc, sizeof(*txc)); + break; } - return 0; + softint_disestablish(netq->netq_softint); + virtio_free_vq(vsc, netq->netq_vq); + mutex_destroy(&netq->netq_lock); + netq->netq_vq = NULL; } static void -vioif_enable_interrupt_vqpairs(struct vioif_softc *sc) +vioif_net_sched_handle(struct vioif_softc *sc, struct vioif_netqueue *netq) { - struct virtio_softc *vsc = sc->sc_virtio; - struct vioif_txqueue *txq; - struct vioif_rxqueue *rxq; - int i; - for (i = 0; i < sc->sc_act_nvq_pairs; i++) { - txq = &sc->sc_txq[i]; - rxq = &sc->sc_rxq[i]; + KASSERT(mutex_owned(&netq->netq_lock)); + KASSERT(!netq->netq_stopping); - virtio_start_vq_intr(vsc, txq->txq_vq); - virtio_start_vq_intr(vsc, rxq->rxq_vq); + if (netq->netq_workqueue) { + vioif_work_add(sc->sc_txrx_workqueue, &netq->netq_work); + } else { + softint_schedule(netq->netq_softint); } } -static void -vioif_disable_interrupt_vqpairs(struct vioif_softc *sc) +static int +vioif_net_load_mbuf(struct virtio_softc *vsc, struct vioif_net_map *map, + struct mbuf *m, int dma_flags) { - struct virtio_softc *vsc = sc->sc_virtio; - struct vioif_txqueue *txq; - struct vioif_rxqueue *rxq; - int i; + int r; - for (i = 0; i < sc->sc_act_nvq_pairs; i++) { - rxq = &sc->sc_rxq[i]; - txq = &sc->sc_txq[i]; + KASSERT(map->vnm_mbuf == NULL); - virtio_stop_vq_intr(vsc, rxq->rxq_vq); - virtio_stop_vq_intr(vsc, txq->txq_vq); + r = bus_dmamap_load_mbuf(virtio_dmat(vsc), + map->vnm_mbuf_map, m, dma_flags | BUS_DMA_NOWAIT); + if (r == 0) { + map->vnm_mbuf = m; } + + return r; +} + +static void +vioif_net_unload_mbuf(struct virtio_softc *vsc, struct vioif_net_map *map) +{ + + KASSERT(map->vnm_mbuf != NULL); + bus_dmamap_unload(virtio_dmat(vsc), map->vnm_mbuf_map); + map->vnm_mbuf = NULL; } -/* - * Interface functions for ifnet - */ static int -vioif_init(struct ifnet *ifp) +vioif_net_enqueue(struct virtio_softc *vsc, struct virtqueue *vq, + int slot, struct vioif_net_map *map, int dma_ops, bool is_write) { - struct vioif_softc *sc = ifp->if_softc; - struct virtio_softc *vsc = sc->sc_virtio; - struct vioif_rxqueue *rxq; - struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; - int r, i; + int r; - vioif_stop(ifp, 0); + KASSERT(map->vnm_mbuf != NULL); - r = virtio_reinit_start(vsc); + /* This should actually never fail */ + r = virtio_enqueue_reserve(vsc, vq, slot, + map->vnm_mbuf_map->dm_nsegs + 1); if (r != 0) { - log(LOG_ERR, "%s: reset failed\n", ifp->if_xname); - return EIO; + /* slot already freed by virtio_enqueue_reserve */ + return r; } - virtio_negotiate_features(vsc, virtio_features(vsc)); - - for (i = 0; i < sc->sc_req_nvq_pairs; i++) { - rxq = &sc->sc_rxq[i]; + bus_dmamap_sync(virtio_dmat(vsc), map->vnm_mbuf_map, + 0, map->vnm_mbuf_map->dm_mapsize, dma_ops); + bus_dmamap_sync(virtio_dmat(vsc), map->vnm_hdr_map, + 0, map->vnm_hdr_map->dm_mapsize, dma_ops); + + virtio_enqueue(vsc, vq, slot, map->vnm_hdr_map, is_write); + virtio_enqueue(vsc, vq, slot, map->vnm_mbuf_map, is_write); + virtio_enqueue_commit(vsc, vq, slot, false); - /* Have to set false before vioif_populate_rx_mbufs */ - mutex_enter(rxq->rxq_lock); - rxq->rxq_stopping = false; - vioif_populate_rx_mbufs_locked(sc, rxq); - mutex_exit(rxq->rxq_lock); + return 0; +} - } +static int +vioif_net_enqueue_tx(struct virtio_softc *vsc, struct virtqueue *vq, + int slot, struct vioif_net_map *map) +{ - virtio_reinit_end(vsc); + return vioif_net_enqueue(vsc, vq, slot, map, + BUS_DMASYNC_PREWRITE, true); +} - if (sc->sc_has_ctrl) - virtio_start_vq_intr(vsc, ctrlq->ctrlq_vq); +static int +vioif_net_enqueue_rx(struct virtio_softc *vsc, struct virtqueue *vq, + int slot, struct vioif_net_map *map) +{ - r = vioif_ctrl_mq_vq_pairs_set(sc, sc->sc_req_nvq_pairs); - if (r == 0) - sc->sc_act_nvq_pairs = sc->sc_req_nvq_pairs; - else - sc->sc_act_nvq_pairs = 1; + return vioif_net_enqueue(vsc, vq, slot, map, + BUS_DMASYNC_PREREAD, false); +} - for (i = 0; i < sc->sc_act_nvq_pairs; i++) - sc->sc_txq[i].txq_stopping = false; +static struct mbuf * +vioif_net_dequeue_commit(struct virtio_softc *vsc, struct virtqueue *vq, + int slot, struct vioif_net_map *map, int dma_flags) +{ + struct mbuf *m; - vioif_enable_interrupt_vqpairs(sc); + m = map->vnm_mbuf; + KASSERT(m != NULL); + map->vnm_mbuf = NULL; + + bus_dmamap_sync(virtio_dmat(vsc), map->vnm_hdr_map, + 0, map->vnm_hdr_map->dm_mapsize, dma_flags); + bus_dmamap_sync(virtio_dmat(vsc), map->vnm_mbuf_map, + 0, map->vnm_mbuf_map->dm_mapsize, dma_flags); - vioif_update_link_status(sc); - ifp->if_flags |= IFF_RUNNING; - ifp->if_flags &= ~IFF_OACTIVE; - r = vioif_rx_filter(sc); + bus_dmamap_unload(virtio_dmat(vsc), map->vnm_mbuf_map); + virtio_dequeue_commit(vsc, vq, slot); - return r; + return m; } static void -vioif_stop(struct ifnet *ifp, int disable) +vioif_net_intr_enable(struct vioif_softc *sc, struct virtio_softc *vsc) { - struct vioif_softc *sc = ifp->if_softc; - struct virtio_softc *vsc = sc->sc_virtio; - struct vioif_txqueue *txq; - struct vioif_rxqueue *rxq; - struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; - int i; - - /* disable interrupts */ - vioif_disable_interrupt_vqpairs(sc); - if (sc->sc_has_ctrl) - virtio_stop_vq_intr(vsc, ctrlq->ctrlq_vq); - - /* - * stop all packet processing: - * 1. stop interrupt handlers by rxq_stopping and txq_stopping - * 2. wait for stopping workqueue for packet processing - */ - for (i = 0; i < sc->sc_act_nvq_pairs; i++) { - txq = &sc->sc_txq[i]; - rxq = &sc->sc_rxq[i]; + struct vioif_netqueue *netq; + size_t i, act_qnum; + int enqueued; - mutex_enter(rxq->rxq_lock); - rxq->rxq_stopping = true; - mutex_exit(rxq->rxq_lock); - vioif_work_wait(sc->sc_txrx_workqueue, &rxq->rxq_work); + act_qnum = sc->sc_act_nvq_pairs * 2; + for (i = 0; i < act_qnum; i++) { + netq = &sc->sc_netqs[i]; - mutex_enter(txq->txq_lock); - txq->txq_stopping = true; - mutex_exit(txq->txq_lock); - vioif_work_wait(sc->sc_txrx_workqueue, &txq->txq_work); - } + KASSERT(!netq->netq_stopping); + KASSERT(!netq->netq_running_handle); - /* only way to stop I/O and DMA is resetting... */ - virtio_reset(vsc); + enqueued = virtio_start_vq_intr(vsc, netq->netq_vq); + if (enqueued != 0) { + virtio_stop_vq_intr(vsc, netq->netq_vq); - for (i = 0; i < sc->sc_act_nvq_pairs; i++) { - vioif_rx_queue_clear(&sc->sc_rxq[i]); - vioif_tx_queue_clear(&sc->sc_txq[i]); + mutex_enter(&netq->netq_lock); + netq->netq_running_handle = true; + vioif_net_sched_handle(sc, netq); + mutex_exit(&netq->netq_lock); + } } +} - ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE); - - for (i = 0; i < sc->sc_act_nvq_pairs; i++) { - txq = &sc->sc_txq[i]; - rxq = &sc->sc_rxq[i]; +static void +vioif_net_intr_disable(struct vioif_softc *sc, struct virtio_softc *vsc) +{ + struct vioif_netqueue *netq; + size_t i, act_qnum; - if (disable) - vioif_rx_drain(rxq); + act_qnum = sc->sc_act_nvq_pairs * 2; + for (i = 0; i < act_qnum; i++) { + netq = &sc->sc_netqs[i]; - vioif_tx_drain(txq); + virtio_stop_vq_intr(vsc, netq->netq_vq); } } +/* + * Receive implementation + */ +/* enqueue mbufs to receive slots */ static void -vioif_send_common_locked(struct ifnet *ifp, struct vioif_txqueue *txq, - bool is_transmit) +vioif_populate_rx_mbufs_locked(struct vioif_softc *sc, struct vioif_netqueue *netq) { - struct vioif_softc *sc = ifp->if_softc; - struct virtio_softc *vsc = sc->sc_virtio; - struct virtqueue *vq = txq->txq_vq; - struct virtio_net_hdr *hdr; + struct virtqueue *vq = netq->netq_vq; + struct virtio_softc *vsc = vq->vq_owner; + struct vioif_rx_context *rxc; + struct vioif_net_map *map; struct mbuf *m; - int queued = 0; - - KASSERT(mutex_owned(txq->txq_lock)); - - if ((ifp->if_flags & IFF_RUNNING) == 0) - return; - - if (!txq->txq_link_active || txq->txq_stopping) - return; - - if ((ifp->if_flags & IFF_OACTIVE) != 0 && !is_transmit) - return; + int i, r, ndone = 0; - for (;;) { - int slot, r; + KASSERT(mutex_owned(&netq->netq_lock)); - if (is_transmit) - m = pcq_get(txq->txq_intrq); - else - IFQ_DEQUEUE(&ifp->if_snd, m); + rxc = netq->netq_ctx; - if (m == NULL) + for (i = 0; i < vq->vq_num; i++) { + int slot; + r = virtio_enqueue_prep(vsc, vq, &slot); + if (r == EAGAIN) break; + if (__predict_false(r != 0)) + panic("enqueue_prep for rx buffers"); - r = virtio_enqueue_prep(vsc, vq, &slot); - if (r == EAGAIN) { - ifp->if_flags |= IFF_OACTIVE; + MGETHDR(m, M_DONTWAIT, MT_DATA); + if (m == NULL) { + virtio_enqueue_abort(vsc, vq, slot); + rxc->rxc_mbuf_enobufs.ev_count++; + break; + } + MCLGET(m, M_DONTWAIT); + if ((m->m_flags & M_EXT) == 0) { + virtio_enqueue_abort(vsc, vq, slot); m_freem(m); + rxc->rxc_mbuf_enobufs.ev_count++; break; } - if (r != 0) - panic("enqueue_prep for a tx buffer"); - - r = bus_dmamap_load_mbuf(virtio_dmat(vsc), - txq->txq_dmamaps[slot], m, BUS_DMA_WRITE | BUS_DMA_NOWAIT); - if (r != 0) { - /* maybe just too fragmented */ - struct mbuf *newm; - newm = m_defrag(m, M_NOWAIT); - if (newm == NULL) { - txq->txq_defrag_failed.ev_count++; - goto skip; - } + m->m_len = m->m_pkthdr.len = MCLBYTES; + m_adj(m, ETHER_ALIGN); - m = newm; - r = bus_dmamap_load_mbuf(virtio_dmat(vsc), - txq->txq_dmamaps[slot], m, - BUS_DMA_WRITE | BUS_DMA_NOWAIT); - if (r != 0) { - txq->txq_mbuf_load_failed.ev_count++; -skip: - m_freem(m); - virtio_enqueue_abort(vsc, vq, slot); - continue; - } + map = &netq->netq_maps[slot]; + r = vioif_net_load_mbuf(vsc, map, m, BUS_DMA_READ); + if (r != 0) { + virtio_enqueue_abort(vsc, vq, slot); + m_freem(m); + netq->netq_mbuf_load_failed.ev_count++; + break; } - /* This should actually never fail */ - r = virtio_enqueue_reserve(vsc, vq, slot, - txq->txq_dmamaps[slot]->dm_nsegs + 1); + r = vioif_net_enqueue_rx(vsc, vq, slot, map); if (r != 0) { - txq->txq_enqueue_reserve_failed.ev_count++; - bus_dmamap_unload(virtio_dmat(vsc), - txq->txq_dmamaps[slot]); - /* slot already freed by virtio_enqueue_reserve */ + vioif_net_unload_mbuf(vsc, map); + netq->netq_enqueue_failed.ev_count++; m_freem(m); - continue; + /* slot already freed by vioif_net_enqueue_rx */ + break; } - txq->txq_mbufs[slot] = m; - - hdr = &txq->txq_hdrs[slot]; - memset(hdr, 0, sc->sc_hdr_size); - bus_dmamap_sync(virtio_dmat(vsc), txq->txq_dmamaps[slot], - 0, txq->txq_dmamaps[slot]->dm_mapsize, - BUS_DMASYNC_PREWRITE); - bus_dmamap_sync(virtio_dmat(vsc), txq->txq_hdr_dmamaps[slot], - 0, txq->txq_hdr_dmamaps[slot]->dm_mapsize, - BUS_DMASYNC_PREWRITE); - virtio_enqueue(vsc, vq, slot, txq->txq_hdr_dmamaps[slot], true); - virtio_enqueue(vsc, vq, slot, txq->txq_dmamaps[slot], true); - virtio_enqueue_commit(vsc, vq, slot, false); - - queued++; - bpf_mtap(ifp, m, BPF_D_OUT); + ndone++; } - if (queued > 0) { - virtio_enqueue_commit(vsc, vq, -1, true); - ifp->if_timer = 5; - } + if (ndone > 0) + vioif_notify(vsc, vq); } -static void -vioif_start_locked(struct ifnet *ifp, struct vioif_txqueue *txq) +/* dequeue received packets */ +static bool +vioif_rx_deq_locked(struct vioif_softc *sc, struct virtio_softc *vsc, + struct vioif_netqueue *netq, u_int limit, size_t *ndeqp) { + struct virtqueue *vq = netq->netq_vq; + struct ifnet *ifp = &sc->sc_ethercom.ec_if; + struct vioif_net_map *map; + struct mbuf *m; + int slot, len; + bool more; + size_t ndeq; - /* - * ifp->if_obytes and ifp->if_omcasts are added in if_transmit()@if.c. - */ - vioif_send_common_locked(ifp, txq, false); + KASSERT(mutex_owned(&netq->netq_lock)); -} + more = false; + ndeq = 0; -static void -vioif_start(struct ifnet *ifp) -{ - struct vioif_softc *sc = ifp->if_softc; - struct vioif_txqueue *txq = &sc->sc_txq[0]; + if (virtio_vq_is_enqueued(vsc, vq) == false) + goto done; -#ifdef VIOIF_MPSAFE - KASSERT(if_is_mpsafe(ifp)); -#endif + for (;;ndeq++) { + if (ndeq >= limit) { + more = true; + break; + } - mutex_enter(txq->txq_lock); - vioif_start_locked(ifp, txq); - mutex_exit(txq->txq_lock); -} + if (virtio_dequeue(vsc, vq, &slot, &len) != 0) + break; -static inline int -vioif_select_txqueue(struct ifnet *ifp, struct mbuf *m) -{ - struct vioif_softc *sc = ifp->if_softc; - u_int cpuid = cpu_index(curcpu()); + map = &netq->netq_maps[slot]; + KASSERT(map->vnm_mbuf != NULL); + m = vioif_net_dequeue_commit(vsc, vq, slot, + map, BUS_DMASYNC_POSTREAD); + KASSERT(m != NULL); - return cpuid % sc->sc_act_nvq_pairs; -} + m->m_len = m->m_pkthdr.len = len - sc->sc_hdr_size; + m_set_rcvif(m, ifp); + if_percpuq_enqueue(ifp->if_percpuq, m); + } -static void -vioif_transmit_locked(struct ifnet *ifp, struct vioif_txqueue *txq) -{ +done: + if (ndeqp != NULL) + *ndeqp = ndeq; - vioif_send_common_locked(ifp, txq, true); + return more; } -static int -vioif_transmit(struct ifnet *ifp, struct mbuf *m) +static void +vioif_rx_queue_clear(struct vioif_softc *sc, struct virtio_softc *vsc, + struct vioif_netqueue *netq) { - struct vioif_softc *sc = ifp->if_softc; - struct vioif_txqueue *txq; - int qid; + struct vioif_net_map *map; + struct mbuf *m; + unsigned int i, vq_num; + bool more; - qid = vioif_select_txqueue(ifp, m); - txq = &sc->sc_txq[qid]; + mutex_enter(&netq->netq_lock); - if (__predict_false(!pcq_put(txq->txq_intrq, m))) { - m_freem(m); - return ENOBUFS; + vq_num = netq->netq_vq->vq_num; + for (;;) { + more = vioif_rx_deq_locked(sc, vsc, netq, vq_num, NULL); + if (more == false) + break; } - net_stat_ref_t nsr = IF_STAT_GETREF(ifp); - if_statadd_ref(nsr, if_obytes, m->m_pkthdr.len); - if (m->m_flags & M_MCAST) - if_statinc_ref(nsr, if_omcasts); - IF_STAT_PUTREF(ifp); + for (i = 0; i < vq_num; i++) { + map = &netq->netq_maps[i]; - if (mutex_tryenter(txq->txq_lock)) { - vioif_transmit_locked(ifp, txq); - mutex_exit(txq->txq_lock); - } + m = map->vnm_mbuf; + if (m == NULL) + continue; - return 0; + vioif_net_unload_mbuf(vsc, map); + m_freem(m); + } + mutex_exit(&netq->netq_lock); } static void -vioif_deferred_transmit(void *arg) +vioif_rx_handle_locked(void *xnetq, u_int limit) { - struct vioif_txqueue *txq = arg; - struct virtio_softc *vsc = txq->txq_vq->vq_owner; + struct vioif_netqueue *netq = xnetq; + struct virtqueue *vq = netq->netq_vq; + struct virtio_softc *vsc = vq->vq_owner; struct vioif_softc *sc = device_private(virtio_child(vsc)); - struct ifnet *ifp = &sc->sc_ethercom.ec_if; - - mutex_enter(txq->txq_lock); - vioif_send_common_locked(ifp, txq, true); - mutex_exit(txq->txq_lock); -} + bool more; + int enqueued; + size_t ndeq; -static int -vioif_ioctl(struct ifnet *ifp, u_long cmd, void *data) -{ - int s, r; + KASSERT(mutex_owned(&netq->netq_lock)); + KASSERT(!netq->netq_stopping); - s = splnet(); + more = vioif_rx_deq_locked(sc, vsc, netq, limit, &ndeq); + if (ndeq > 0) + vioif_populate_rx_mbufs_locked(sc, netq); - r = ether_ioctl(ifp, cmd, data); - if (r == ENETRESET && (cmd == SIOCADDMULTI || cmd == SIOCDELMULTI)) { - if (ifp->if_flags & IFF_RUNNING) { - r = vioif_rx_filter(ifp->if_softc); - } else { - r = 0; - } + if (more) { + vioif_net_sched_handle(sc, netq); + return; } - splx(s); + enqueued = virtio_start_vq_intr(vsc, netq->netq_vq); + if (enqueued != 0) { + virtio_stop_vq_intr(vsc, netq->netq_vq); + vioif_net_sched_handle(sc, netq); + return; + } - return r; + netq->netq_running_handle = false; } -void -vioif_watchdog(struct ifnet *ifp) +static int +vioif_rx_intr(void *arg) { - struct vioif_softc *sc = ifp->if_softc; - int i; + struct vioif_netqueue *netq = arg; + struct virtqueue *vq = netq->netq_vq; + struct virtio_softc *vsc = vq->vq_owner; + struct vioif_softc *sc = device_private(virtio_child(vsc)); + u_int limit; - if (ifp->if_flags & IFF_RUNNING) { - for (i = 0; i < sc->sc_act_nvq_pairs; i++) { - vioif_tx_queue_clear(&sc->sc_txq[i]); - } - } -} + mutex_enter(&netq->netq_lock); -/* - * Receive implementation - */ -/* allocate and initialize a mbuf for receive */ -static int -vioif_add_rx_mbuf(struct vioif_rxqueue *rxq, int i) -{ - struct virtio_softc *vsc = rxq->rxq_vq->vq_owner; - struct mbuf *m; - int r; + /* handler is already running in softint/workqueue */ + if (netq->netq_running_handle) + goto done; - MGETHDR(m, M_DONTWAIT, MT_DATA); - if (m == NULL) - return ENOBUFS; - MCLGET(m, M_DONTWAIT); - if ((m->m_flags & M_EXT) == 0) { - m_freem(m); - return ENOBUFS; - } - rxq->rxq_mbufs[i] = m; - m->m_len = m->m_pkthdr.len = m->m_ext.ext_size; - r = bus_dmamap_load_mbuf(virtio_dmat(vsc), - rxq->rxq_dmamaps[i], m, BUS_DMA_READ | BUS_DMA_NOWAIT); - if (r) { - m_freem(m); - rxq->rxq_mbufs[i] = NULL; - return r; - } + netq->netq_running_handle = true; - return 0; + limit = sc->sc_rx_intr_process_limit; + virtio_stop_vq_intr(vsc, vq); + vioif_rx_handle_locked(netq, limit); + +done: + mutex_exit(&netq->netq_lock); + return 1; } -/* free a mbuf for receive */ static void -vioif_free_rx_mbuf(struct vioif_rxqueue *rxq, int i) +vioif_rx_handle(void *xnetq) { - struct virtio_softc *vsc = rxq->rxq_vq->vq_owner; + struct vioif_netqueue *netq = xnetq; + struct virtqueue *vq = netq->netq_vq; + struct virtio_softc *vsc = vq->vq_owner; + struct vioif_softc *sc = device_private(virtio_child(vsc)); + u_int limit; + + mutex_enter(&netq->netq_lock); + + KASSERT(netq->netq_running_handle); + + if (netq->netq_stopping) { + netq->netq_running_handle = false; + goto done; + } + + limit = sc->sc_rx_process_limit; + vioif_rx_handle_locked(netq, limit); - bus_dmamap_unload(virtio_dmat(vsc), rxq->rxq_dmamaps[i]); - m_freem(rxq->rxq_mbufs[i]); - rxq->rxq_mbufs[i] = NULL; +done: + mutex_exit(&netq->netq_lock); } -/* add mbufs for all the empty receive slots */ +/* + * Transmition implementation + */ +/* enqueue mbufs to send */ static void -vioif_populate_rx_mbufs_locked(struct vioif_softc *sc, struct vioif_rxqueue *rxq) +vioif_send_common_locked(struct ifnet *ifp, struct vioif_netqueue *netq, + bool is_transmit) { - struct virtqueue *vq = rxq->rxq_vq; - struct virtio_softc *vsc = vq->vq_owner; - int i, r, ndone = 0; + struct vioif_softc *sc = ifp->if_softc; + struct virtio_softc *vsc = sc->sc_virtio; + struct virtqueue *vq = netq->netq_vq; + struct vioif_tx_context *txc; + struct vioif_net_map *map; + struct mbuf *m; + int queued = 0; + + KASSERT(mutex_owned(&netq->netq_lock)); - KASSERT(mutex_owned(rxq->rxq_lock)); + if (netq->netq_stopping || + !ISSET(ifp->if_flags, IFF_RUNNING)) + return; + + txc = netq->netq_ctx; - if (rxq->rxq_stopping) + if (!txc->txc_link_active || + txc->txc_no_free_slots) return; - for (i = 0; i < vq->vq_num; i++) { - int slot; + for (;;) { + int slot, r; r = virtio_enqueue_prep(vsc, vq, &slot); - if (r == EAGAIN) + if (r == EAGAIN) { + txc->txc_no_free_slots = true; break; - if (r != 0) - panic("enqueue_prep for rx buffers"); - if (rxq->rxq_mbufs[slot] == NULL) { - r = vioif_add_rx_mbuf(rxq, slot); + } + if (__predict_false(r != 0)) + panic("enqueue_prep for tx buffers"); + + if (is_transmit) + m = pcq_get(txc->txc_intrq); + else + IFQ_DEQUEUE(&ifp->if_snd, m); + + if (m == NULL) { + virtio_enqueue_abort(vsc, vq, slot); + break; + } + + map = &netq->netq_maps[slot]; + KASSERT(map->vnm_mbuf == NULL); + + r = vioif_net_load_mbuf(vsc, map, m, BUS_DMA_WRITE); + if (r != 0) { + /* maybe just too fragmented */ + struct mbuf *newm; + + newm = m_defrag(m, M_NOWAIT); + if (newm != NULL) { + m = newm; + r = vioif_net_load_mbuf(vsc, map, m, + BUS_DMA_WRITE); + } else { + txc->txc_defrag_failed.ev_count++; + r = -1; + } + if (r != 0) { - rxq->rxq_mbuf_add_failed.ev_count++; - break; + netq->netq_mbuf_load_failed.ev_count++; + m_freem(m); + if_statinc(ifp, if_oerrors); + virtio_enqueue_abort(vsc, vq, slot); + continue; } } - r = virtio_enqueue_reserve(vsc, vq, slot, - rxq->rxq_dmamaps[slot]->dm_nsegs + 1); + + memset(map->vnm_hdr, 0, sc->sc_hdr_size); + + r = vioif_net_enqueue_tx(vsc, vq, slot, map); if (r != 0) { - vioif_free_rx_mbuf(rxq, slot); - break; - } - bus_dmamap_sync(virtio_dmat(vsc), rxq->rxq_hdr_dmamaps[slot], - 0, sc->sc_hdr_size, BUS_DMASYNC_PREREAD); - bus_dmamap_sync(virtio_dmat(vsc), rxq->rxq_dmamaps[slot], - 0, MCLBYTES, BUS_DMASYNC_PREREAD); - virtio_enqueue(vsc, vq, slot, rxq->rxq_hdr_dmamaps[slot], - false); - virtio_enqueue(vsc, vq, slot, rxq->rxq_dmamaps[slot], false); - virtio_enqueue_commit(vsc, vq, slot, false); - ndone++; - } - if (ndone > 0) - virtio_enqueue_commit(vsc, vq, -1, true); -} + netq->netq_enqueue_failed.ev_count++; + vioif_net_unload_mbuf(vsc, map); + m_freem(m); + /* slot already freed by vioif_net_enqueue_tx */ -static void -vioif_rx_queue_clear(struct vioif_rxqueue *rxq) -{ - struct virtqueue *vq = rxq->rxq_vq; - struct virtio_softc *vsc = vq->vq_owner; - struct vioif_softc *sc = device_private(virtio_child(vsc)); - u_int limit = UINT_MAX; - bool more; + if_statinc(ifp, if_oerrors); + continue; + } - KASSERT(rxq->rxq_stopping); + queued++; + bpf_mtap(ifp, m, BPF_D_OUT); + } - mutex_enter(rxq->rxq_lock); - for (;;) { - more = vioif_rx_deq_locked(sc, vsc, rxq, limit); - if (more == false) - break; + if (queued > 0) { + vioif_notify(vsc, vq); + ifp->if_timer = 5; } - mutex_exit(rxq->rxq_lock); } -/* dequeue received packets */ +/* dequeue sent mbufs */ static bool -vioif_rx_deq_locked(struct vioif_softc *sc, struct virtio_softc *vsc, - struct vioif_rxqueue *rxq, u_int limit) +vioif_tx_deq_locked(struct vioif_softc *sc, struct virtio_softc *vsc, + struct vioif_netqueue *netq, u_int limit, size_t *ndeqp) { - struct virtqueue *vq = rxq->rxq_vq; + struct virtqueue *vq = netq->netq_vq; struct ifnet *ifp = &sc->sc_ethercom.ec_if; + struct vioif_net_map *map; struct mbuf *m; int slot, len; - bool more = false, dequeued = false; + bool more; + size_t ndeq; - KASSERT(mutex_owned(rxq->rxq_lock)); + KASSERT(mutex_owned(&netq->netq_lock)); + + more = false; + ndeq = 0; if (virtio_vq_is_enqueued(vsc, vq) == false) - return false; + goto done; - for (;;) { + for (;;ndeq++) { if (limit-- == 0) { more = true; break; @@ -1627,321 +2085,187 @@ vioif_rx_deq_locked(struct vioif_softc * if (virtio_dequeue(vsc, vq, &slot, &len) != 0) break; - dequeued = true; - - len -= sc->sc_hdr_size; - bus_dmamap_sync(virtio_dmat(vsc), rxq->rxq_hdr_dmamaps[slot], - 0, sc->sc_hdr_size, BUS_DMASYNC_POSTREAD); - bus_dmamap_sync(virtio_dmat(vsc), rxq->rxq_dmamaps[slot], - 0, MCLBYTES, BUS_DMASYNC_POSTREAD); - m = rxq->rxq_mbufs[slot]; + map = &netq->netq_maps[slot]; + KASSERT(map->vnm_mbuf != NULL); + m = vioif_net_dequeue_commit(vsc, vq, slot, + map, BUS_DMASYNC_POSTWRITE); KASSERT(m != NULL); - bus_dmamap_unload(virtio_dmat(vsc), rxq->rxq_dmamaps[slot]); - rxq->rxq_mbufs[slot] = NULL; - virtio_dequeue_commit(vsc, vq, slot); - m_set_rcvif(m, ifp); - m->m_len = m->m_pkthdr.len = len; - - mutex_exit(rxq->rxq_lock); - if_percpuq_enqueue(ifp->if_percpuq, m); - mutex_enter(rxq->rxq_lock); - if (rxq->rxq_stopping) - break; + if_statinc(ifp, if_opackets); + m_freem(m); } - if (dequeued) - vioif_populate_rx_mbufs_locked(sc, rxq); - +done: + if (ndeqp != NULL) + *ndeqp = ndeq; return more; } -/* rx interrupt; call _dequeue above and schedule a softint */ - static void -vioif_rx_handle_locked(void *xrxq, u_int limit) +vioif_tx_queue_clear(struct vioif_softc *sc, struct virtio_softc *vsc, + struct vioif_netqueue *netq) { - struct vioif_rxqueue *rxq = xrxq; - struct virtqueue *vq = rxq->rxq_vq; - struct virtio_softc *vsc = vq->vq_owner; - struct vioif_softc *sc = device_private(virtio_child(vsc)); + struct vioif_tx_context *txc; + struct vioif_net_map *map; + struct mbuf *m; + unsigned int i, vq_num; bool more; - KASSERT(!rxq->rxq_stopping); - - more = vioif_rx_deq_locked(sc, vsc, rxq, limit); - if (more) { - vioif_rx_sched_handle(sc, rxq); - return; - } - more = virtio_start_vq_intr(vsc, rxq->rxq_vq); - if (more) { - vioif_rx_sched_handle(sc, rxq); - return; - } - atomic_store_relaxed(&rxq->rxq_active, false); -} - -static int -vioif_rx_intr(void *arg) -{ - struct vioif_rxqueue *rxq = arg; - struct virtqueue *vq = rxq->rxq_vq; - struct virtio_softc *vsc = vq->vq_owner; - struct vioif_softc *sc = device_private(virtio_child(vsc)); - u_int limit; - - limit = sc->sc_rx_intr_process_limit; + mutex_enter(&netq->netq_lock); - if (atomic_load_relaxed(&rxq->rxq_active) == true) - return 1; + txc = netq->netq_ctx; + vq_num = netq->netq_vq->vq_num; - mutex_enter(rxq->rxq_lock); + for (;;) { + more = vioif_tx_deq_locked(sc, vsc, netq, vq_num, NULL); + if (more == false) + break; + } - if (!rxq->rxq_stopping) { - rxq->rxq_workqueue = sc->sc_txrx_workqueue_sysctl; + for (i = 0; i < vq_num; i++) { + map = &netq->netq_maps[i]; - virtio_stop_vq_intr(vsc, vq); - atomic_store_relaxed(&rxq->rxq_active, true); + m = map->vnm_mbuf; + if (m == NULL) + continue; - vioif_rx_handle_locked(rxq, limit); + vioif_net_unload_mbuf(vsc, map); + m_freem(m); } - mutex_exit(rxq->rxq_lock); - return 1; + txc->txc_no_free_slots = false; + + mutex_exit(&netq->netq_lock); } static void -vioif_rx_handle(void *xrxq) +vioif_start_locked(struct ifnet *ifp, struct vioif_netqueue *netq) { - struct vioif_rxqueue *rxq = xrxq; - struct virtqueue *vq = rxq->rxq_vq; - struct virtio_softc *vsc = vq->vq_owner; - struct vioif_softc *sc = device_private(virtio_child(vsc)); - u_int limit; - - limit = sc->sc_rx_process_limit; - - mutex_enter(rxq->rxq_lock); - if (!rxq->rxq_stopping) - vioif_rx_handle_locked(rxq, limit); + /* + * ifp->if_obytes and ifp->if_omcasts are added in if_transmit()@if.c. + */ + vioif_send_common_locked(ifp, netq, false); - mutex_exit(rxq->rxq_lock); } static void -vioif_rx_sched_handle(struct vioif_softc *sc, struct vioif_rxqueue *rxq) +vioif_transmit_locked(struct ifnet *ifp, struct vioif_netqueue *netq) { - KASSERT(mutex_owned(rxq->rxq_lock)); - - if (rxq->rxq_stopping) - return; - - if (rxq->rxq_workqueue) - vioif_work_add(sc->sc_txrx_workqueue, &rxq->rxq_work); - else - softint_schedule(rxq->rxq_handle_si); + vioif_send_common_locked(ifp, netq, true); } -/* free all the mbufs; called from if_stop(disable) */ static void -vioif_rx_drain(struct vioif_rxqueue *rxq) +vioif_deferred_transmit(void *arg) { - struct virtqueue *vq = rxq->rxq_vq; - int i; + struct vioif_netqueue *netq = arg; + struct virtio_softc *vsc = netq->netq_vq->vq_owner; + struct vioif_softc *sc = device_private(virtio_child(vsc)); + struct ifnet *ifp = &sc->sc_ethercom.ec_if; - for (i = 0; i < vq->vq_num; i++) { - if (rxq->rxq_mbufs[i] == NULL) - continue; - vioif_free_rx_mbuf(rxq, i); - } + mutex_enter(&netq->netq_lock); + vioif_send_common_locked(ifp, netq, true); + mutex_exit(&netq->netq_lock); } -/* - * Transmition implementation - */ -/* actual transmission is done in if_start */ -/* tx interrupt; dequeue and free mbufs */ -/* - * tx interrupt is actually disabled; this should be called upon - * tx vq full and watchdog - */ - static void -vioif_tx_handle_locked(struct vioif_txqueue *txq, u_int limit) +vioif_tx_handle_locked(struct vioif_netqueue *netq, u_int limit) { - struct virtqueue *vq = txq->txq_vq; + struct virtqueue *vq = netq->netq_vq; + struct vioif_tx_context *txc = netq->netq_ctx; struct virtio_softc *vsc = vq->vq_owner; struct vioif_softc *sc = device_private(virtio_child(vsc)); struct ifnet *ifp = &sc->sc_ethercom.ec_if; bool more; + int enqueued; + size_t ndeq; - KASSERT(!txq->txq_stopping); + KASSERT(mutex_owned(&netq->netq_lock)); + KASSERT(!netq->netq_stopping); + + more = vioif_tx_deq_locked(sc, vsc, netq, limit, &ndeq); + if (txc->txc_no_free_slots && ndeq > 0) { + txc->txc_no_free_slots = false; + softint_schedule(txc->txc_deferred_transmit); + } - more = vioif_tx_deq_locked(sc, vsc, txq, limit); if (more) { - vioif_tx_sched_handle(sc, txq); + vioif_net_sched_handle(sc, netq); return; } - if (virtio_features(vsc) & VIRTIO_F_RING_EVENT_IDX) - more = virtio_postpone_intr_smart(vsc, vq); - else - more = virtio_start_vq_intr(vsc, vq); - if (more) { - vioif_tx_sched_handle(sc, txq); + enqueued = (virtio_features(vsc) & VIRTIO_F_RING_EVENT_IDX) ? + virtio_postpone_intr_smart(vsc, vq): + virtio_start_vq_intr(vsc, vq); + if (enqueued != 0) { + virtio_stop_vq_intr(vsc, vq); + vioif_net_sched_handle(sc, netq); return; } - atomic_store_relaxed(&txq->txq_active, false); + netq->netq_running_handle = false; + /* for ALTQ */ - if (txq == &sc->sc_txq[0]) { + if (netq == &sc->sc_netqs[VIOIF_NETQ_TXQID(0)]) if_schedule_deferred_start(ifp); - ifp->if_flags &= ~IFF_OACTIVE; - } - softint_schedule(txq->txq_deferred_transmit); -} + softint_schedule(txc->txc_deferred_transmit); +} static int vioif_tx_intr(void *arg) { - struct vioif_txqueue *txq = arg; - struct virtqueue *vq = txq->txq_vq; + struct vioif_netqueue *netq = arg; + struct virtqueue *vq = netq->netq_vq; struct virtio_softc *vsc = vq->vq_owner; struct vioif_softc *sc = device_private(virtio_child(vsc)); u_int limit; - limit = sc->sc_tx_intr_process_limit; - - if (atomic_load_relaxed(&txq->txq_active) == true) - return 1; - - mutex_enter(txq->txq_lock); + mutex_enter(&netq->netq_lock); - if (!txq->txq_stopping) { - txq->txq_workqueue = sc->sc_txrx_workqueue_sysctl; + /* tx handler is already running in softint/workqueue */ + if (netq->netq_running_handle) + goto done; - virtio_stop_vq_intr(vsc, vq); - atomic_store_relaxed(&txq->txq_active, true); + if (netq->netq_stopping) + goto done; - vioif_tx_handle_locked(txq, limit); - } + netq->netq_running_handle = true; - mutex_exit(txq->txq_lock); + virtio_stop_vq_intr(vsc, vq); + netq->netq_workqueue = sc->sc_txrx_workqueue_sysctl; + limit = sc->sc_tx_intr_process_limit; + vioif_tx_handle_locked(netq, limit); +done: + mutex_exit(&netq->netq_lock); return 1; } static void -vioif_tx_handle(void *xtxq) +vioif_tx_handle(void *xnetq) { - struct vioif_txqueue *txq = xtxq; - struct virtqueue *vq = txq->txq_vq; + struct vioif_netqueue *netq = xnetq; + struct virtqueue *vq = netq->netq_vq; struct virtio_softc *vsc = vq->vq_owner; struct vioif_softc *sc = device_private(virtio_child(vsc)); u_int limit; - limit = sc->sc_tx_process_limit; - - mutex_enter(txq->txq_lock); - if (!txq->txq_stopping) - vioif_tx_handle_locked(txq, limit); - mutex_exit(txq->txq_lock); -} - -static void -vioif_tx_sched_handle(struct vioif_softc *sc, struct vioif_txqueue *txq) -{ - - KASSERT(mutex_owned(txq->txq_lock)); - - if (txq->txq_stopping) - return; - - if (txq->txq_workqueue) - vioif_work_add(sc->sc_txrx_workqueue, &txq->txq_work); - else - softint_schedule(txq->txq_handle_si); -} - -static void -vioif_tx_queue_clear(struct vioif_txqueue *txq) -{ - struct virtqueue *vq = txq->txq_vq; - struct virtio_softc *vsc = vq->vq_owner; - struct vioif_softc *sc = device_private(virtio_child(vsc)); - u_int limit = UINT_MAX; - bool more; - - mutex_enter(txq->txq_lock); - for (;;) { - more = vioif_tx_deq_locked(sc, vsc, txq, limit); - if (more == false) - break; - } - mutex_exit(txq->txq_lock); -} - -static bool -vioif_tx_deq_locked(struct vioif_softc *sc, struct virtio_softc *vsc, - struct vioif_txqueue *txq, u_int limit) -{ - struct virtqueue *vq = txq->txq_vq; - struct ifnet *ifp = &sc->sc_ethercom.ec_if; - struct mbuf *m; - int slot, len; - bool more = false; - - KASSERT(mutex_owned(txq->txq_lock)); - - if (virtio_vq_is_enqueued(vsc, vq) == false) - return false; - - for (;;) { - if (limit-- == 0) { - more = true; - break; - } + mutex_enter(&netq->netq_lock); - if (virtio_dequeue(vsc, vq, &slot, &len) != 0) - break; + KASSERT(netq->netq_running_handle); - bus_dmamap_sync(virtio_dmat(vsc), txq->txq_hdr_dmamaps[slot], - 0, sc->sc_hdr_size, BUS_DMASYNC_POSTWRITE); - bus_dmamap_sync(virtio_dmat(vsc), txq->txq_dmamaps[slot], - 0, txq->txq_dmamaps[slot]->dm_mapsize, - BUS_DMASYNC_POSTWRITE); - m = txq->txq_mbufs[slot]; - bus_dmamap_unload(virtio_dmat(vsc), txq->txq_dmamaps[slot]); - txq->txq_mbufs[slot] = NULL; - virtio_dequeue_commit(vsc, vq, slot); - if_statinc(ifp, if_opackets); - m_freem(m); + if (netq->netq_stopping) { + netq->netq_running_handle = false; + goto done; } - return more; -} - -/* free all the mbufs already put on vq; called from if_stop(disable) */ -static void -vioif_tx_drain(struct vioif_txqueue *txq) -{ - struct virtqueue *vq = txq->txq_vq; - struct virtio_softc *vsc = vq->vq_owner; - int i; - - KASSERT(txq->txq_stopping); + limit = sc->sc_tx_process_limit; + vioif_tx_handle_locked(netq, limit); - for (i = 0; i < vq->vq_num; i++) { - if (txq->txq_mbufs[i] == NULL) - continue; - bus_dmamap_unload(virtio_dmat(vsc), txq->txq_dmamaps[i]); - m_freem(txq->txq_mbufs[i]); - txq->txq_mbufs[i] = NULL; - } +done: + mutex_exit(&netq->netq_lock); } /* @@ -2083,6 +2407,31 @@ vioif_ctrl_send_command(struct vioif_sof return r; } +/* ctrl vq interrupt; wake up the command issuer */ +static int +vioif_ctrl_intr(void *arg) +{ + struct vioif_ctrlqueue *ctrlq = arg; + struct virtqueue *vq = ctrlq->ctrlq_vq; + struct virtio_softc *vsc = vq->vq_owner; + int r, slot; + + if (virtio_vq_is_enqueued(vsc, vq) == false) + return 0; + + r = virtio_dequeue(vsc, vq, &slot, NULL); + if (r == ENOENT) + return 0; + virtio_dequeue_commit(vsc, vq, slot); + + mutex_enter(&ctrlq->ctrlq_wait_lock); + ctrlq->ctrlq_inuse = DONE; + cv_signal(&ctrlq->ctrlq_wait); + mutex_exit(&ctrlq->ctrlq_wait_lock); + + return 1; +} + static int vioif_ctrl_rx(struct vioif_softc *sc, int cmd, bool onoff) { @@ -2119,46 +2468,30 @@ vioif_set_allmulti(struct vioif_softc *s return vioif_ctrl_rx(sc, VIRTIO_NET_CTRL_RX_ALLMULTI, onoff); } -/* issue VIRTIO_NET_CTRL_MAC_TABLE_SET command and wait for completion */ static int -vioif_set_rx_filter(struct vioif_softc *sc) +vioif_ctrl_mq_vq_pairs_set(struct vioif_softc *sc, int nvq_pairs) { - /* filter already set in ctrlq->ctrlq_mac_tbl */ - struct virtio_softc *vsc = sc->sc_virtio; - struct virtio_net_ctrl_mac_tbl *mac_tbl_uc, *mac_tbl_mc; - struct vioif_ctrl_cmdspec specs[2]; - int nspecs = __arraycount(specs); + struct virtio_net_ctrl_mq *mq = sc->sc_ctrlq.ctrlq_mq; + struct vioif_ctrl_cmdspec specs[1]; int r; - mac_tbl_uc = sc->sc_ctrlq.ctrlq_mac_tbl_uc; - mac_tbl_mc = sc->sc_ctrlq.ctrlq_mac_tbl_mc; - if (!sc->sc_has_ctrl) return ENOTSUP; - vioif_ctrl_acquire(sc); - - specs[0].dmamap = sc->sc_ctrlq.ctrlq_tbl_uc_dmamap; - specs[0].buf = mac_tbl_uc; - specs[0].bufsize = sizeof(*mac_tbl_uc) - + (ETHER_ADDR_LEN * virtio_rw32(vsc, mac_tbl_uc->nentries)); - - specs[1].dmamap = sc->sc_ctrlq.ctrlq_tbl_mc_dmamap; - specs[1].buf = mac_tbl_mc; - specs[1].bufsize = sizeof(*mac_tbl_mc) - + (ETHER_ADDR_LEN * virtio_rw32(vsc, mac_tbl_mc->nentries)); + if (nvq_pairs <= 1) + return EINVAL; - r = vioif_ctrl_load_cmdspec(sc, specs, nspecs); - if (r != 0) - goto out; + vioif_ctrl_acquire(sc); - r = vioif_ctrl_send_command(sc, - VIRTIO_NET_CTRL_MAC, VIRTIO_NET_CTRL_MAC_TABLE_SET, - specs, nspecs); + mq->virtqueue_pairs = virtio_rw16(sc->sc_virtio, nvq_pairs); + specs[0].dmamap = sc->sc_ctrlq.ctrlq_mq_dmamap; + specs[0].buf = mq; + specs[0].bufsize = sizeof(*mq); - vioif_ctrl_unload_cmdspec(sc, specs, nspecs); + r = vioif_ctrl_send_command(sc, + VIRTIO_NET_CTRL_MQ, VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET, + specs, __arraycount(specs)); -out: vioif_ctrl_release(sc); return r; @@ -2212,104 +2545,47 @@ vioif_set_mac_addr(struct vioif_softc *s } static int -vioif_ctrl_mq_vq_pairs_set(struct vioif_softc *sc, int nvq_pairs) +vioif_set_rx_filter(struct vioif_softc *sc) { - struct virtio_net_ctrl_mq *mq = sc->sc_ctrlq.ctrlq_mq; - struct vioif_ctrl_cmdspec specs[1]; + /* filter already set in ctrlq->ctrlq_mac_tbl */ + struct virtio_softc *vsc = sc->sc_virtio; + struct virtio_net_ctrl_mac_tbl *mac_tbl_uc, *mac_tbl_mc; + struct vioif_ctrl_cmdspec specs[2]; + int nspecs = __arraycount(specs); int r; + mac_tbl_uc = sc->sc_ctrlq.ctrlq_mac_tbl_uc; + mac_tbl_mc = sc->sc_ctrlq.ctrlq_mac_tbl_mc; + if (!sc->sc_has_ctrl) return ENOTSUP; - if (nvq_pairs <= 1) - return EINVAL; - vioif_ctrl_acquire(sc); - mq->virtqueue_pairs = virtio_rw16(sc->sc_virtio, nvq_pairs); - specs[0].dmamap = sc->sc_ctrlq.ctrlq_mq_dmamap; - specs[0].buf = mq; - specs[0].bufsize = sizeof(*mq); - - r = vioif_ctrl_send_command(sc, - VIRTIO_NET_CTRL_MQ, VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET, - specs, __arraycount(specs)); - - vioif_ctrl_release(sc); - - return r; -} - -/* ctrl vq interrupt; wake up the command issuer */ -static int -vioif_ctrl_intr(void *arg) -{ - struct vioif_ctrlqueue *ctrlq = arg; - struct virtqueue *vq = ctrlq->ctrlq_vq; - struct virtio_softc *vsc = vq->vq_owner; - int r, slot; - - if (virtio_vq_is_enqueued(vsc, vq) == false) - return 0; - - r = virtio_dequeue(vsc, vq, &slot, NULL); - if (r == ENOENT) - return 0; - virtio_dequeue_commit(vsc, vq, slot); - - mutex_enter(&ctrlq->ctrlq_wait_lock); - ctrlq->ctrlq_inuse = DONE; - cv_signal(&ctrlq->ctrlq_wait); - mutex_exit(&ctrlq->ctrlq_wait_lock); - - return 1; -} - -static int -vioif_ifflags(struct vioif_softc *sc) -{ - struct ifnet *ifp = &sc->sc_ethercom.ec_if; - bool onoff; - int r; + specs[0].dmamap = sc->sc_ctrlq.ctrlq_tbl_uc_dmamap; + specs[0].buf = mac_tbl_uc; + specs[0].bufsize = sizeof(*mac_tbl_uc) + + (ETHER_ADDR_LEN * virtio_rw32(vsc, mac_tbl_uc->nentries)); - if (!sc->sc_has_ctrl) { - /* no ctrl vq; always promisc and allmulti */ - ifp->if_flags |= (IFF_PROMISC | IFF_ALLMULTI); - return 0; - } + specs[1].dmamap = sc->sc_ctrlq.ctrlq_tbl_mc_dmamap; + specs[1].buf = mac_tbl_mc; + specs[1].bufsize = sizeof(*mac_tbl_mc) + + (ETHER_ADDR_LEN * virtio_rw32(vsc, mac_tbl_mc->nentries)); - onoff = ifp->if_flags & IFF_ALLMULTI ? true : false; - r = vioif_set_allmulti(sc, onoff); - if (r != 0) { - log(LOG_WARNING, - "%s: couldn't %sable ALLMULTI\n", - ifp->if_xname, onoff ? "en" : "dis"); - if (onoff == false) { - ifp->if_flags |= IFF_ALLMULTI; - } - } + r = vioif_ctrl_load_cmdspec(sc, specs, nspecs); + if (r != 0) + goto out; - onoff = ifp->if_flags & IFF_PROMISC ? true : false; - r = vioif_set_promisc(sc, onoff); - if (r != 0) { - log(LOG_WARNING, - "%s: couldn't %sable PROMISC\n", - ifp->if_xname, onoff ? "en" : "dis"); - if (onoff == false) { - ifp->if_flags |= IFF_PROMISC; - } - } + r = vioif_ctrl_send_command(sc, + VIRTIO_NET_CTRL_MAC, VIRTIO_NET_CTRL_MAC_TABLE_SET, + specs, nspecs); - return 0; -} + vioif_ctrl_unload_cmdspec(sc, specs, nspecs); -static int -vioif_ifflags_cb(struct ethercom *ec) -{ - struct ifnet *ifp = &ec->ec_if; - struct vioif_softc *sc = ifp->if_softc; +out: + vioif_ctrl_release(sc); - return vioif_ifflags(sc); + return r; } /* @@ -2392,6 +2668,28 @@ set_ifflags: return r; } +/* + * VM configuration changes + */ +static int +vioif_config_change(struct virtio_softc *vsc) +{ + struct vioif_softc *sc = device_private(virtio_child(vsc)); + + softint_schedule(sc->sc_cfg_softint); + return 0; +} + +static void +vioif_cfg_softint(void *arg) +{ + struct vioif_softc *sc = arg; + struct ifnet *ifp = &sc->sc_ethercom.ec_if; + + vioif_update_link_status(sc); + vioif_start(ifp); +} + static int vioif_get_link_status(struct vioif_softc *sc) { @@ -2410,12 +2708,12 @@ vioif_get_link_status(struct vioif_softc return LINK_STATE_DOWN; } -/* change link status */ static void vioif_update_link_status(struct vioif_softc *sc) { struct ifnet *ifp = &sc->sc_ethercom.ec_if; - struct vioif_txqueue *txq; + struct vioif_netqueue *netq; + struct vioif_tx_context *txc; bool active; int link, i; @@ -2430,11 +2728,12 @@ vioif_update_link_status(struct vioif_so active = VIOIF_IS_LINK_ACTIVE(sc); for (i = 0; i < sc->sc_act_nvq_pairs; i++) { - txq = &sc->sc_txq[i]; + netq = &sc->sc_netqs[VIOIF_NETQ_TXQID(i)]; - mutex_enter(txq->txq_lock); - txq->txq_link_active = active; - mutex_exit(txq->txq_lock); + mutex_enter(&netq->netq_lock); + txc = netq->netq_ctx; + txc->txc_link_active = active; + mutex_exit(&netq->netq_lock); } if_link_state_change(ifp, sc->sc_link_state); @@ -2443,23 +2742,15 @@ done: mutex_exit(&sc->sc_lock); } -static int -vioif_config_change(struct virtio_softc *vsc) -{ - struct vioif_softc *sc = device_private(virtio_child(vsc)); - - softint_schedule(sc->sc_ctl_softint); - return 0; -} - static void -vioif_ctl_softint(void *arg) +vioif_workq_work(struct work *wk, void *context) { - struct vioif_softc *sc = arg; - struct ifnet *ifp = &sc->sc_ethercom.ec_if; + struct vioif_work *work; - vioif_update_link_status(sc); - vioif_start(ifp); + work = container_of(wk, struct vioif_work, cookie); + + atomic_store_relaxed(&work->added, 0); + work->func(work->arg); } static struct workqueue * @@ -2485,17 +2776,6 @@ vioif_workq_destroy(struct workqueue *wq } static void -vioif_workq_work(struct work *wk, void *context) -{ - struct vioif_work *work; - - work = container_of(wk, struct vioif_work, cookie); - - atomic_store_relaxed(&work->added, 0); - work->func(work->arg); -} - -static void vioif_work_set(struct vioif_work *work, void (*func)(void *), void *arg) { @@ -2524,110 +2804,6 @@ vioif_work_wait(struct workqueue *wq, st workqueue_wait(wq, &work->cookie); } -static int -vioif_setup_sysctl(struct vioif_softc *sc) -{ - const char *devname; - struct sysctllog **log; - const struct sysctlnode *rnode, *rxnode, *txnode; - int error; - - log = &sc->sc_sysctllog; - devname = device_xname(sc->sc_dev); - - error = sysctl_createv(log, 0, NULL, &rnode, - 0, CTLTYPE_NODE, devname, - SYSCTL_DESCR("virtio-net information and settings"), - NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL); - if (error) - goto out; - - error = sysctl_createv(log, 0, &rnode, NULL, - CTLFLAG_READWRITE, CTLTYPE_BOOL, "txrx_workqueue", - SYSCTL_DESCR("Use workqueue for packet processing"), - NULL, 0, &sc->sc_txrx_workqueue_sysctl, 0, CTL_CREATE, CTL_EOL); - if (error) - goto out; - - error = sysctl_createv(log, 0, &rnode, &rxnode, - 0, CTLTYPE_NODE, "rx", - SYSCTL_DESCR("virtio-net information and settings for Rx"), - NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); - if (error) - goto out; - - error = sysctl_createv(log, 0, &rxnode, NULL, - CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", - SYSCTL_DESCR("max number of Rx packets to process for interrupt processing"), - NULL, 0, &sc->sc_rx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); - if (error) - goto out; - - error = sysctl_createv(log, 0, &rxnode, NULL, - CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", - SYSCTL_DESCR("max number of Rx packets to process for deferred processing"), - NULL, 0, &sc->sc_rx_process_limit, 0, CTL_CREATE, CTL_EOL); - if (error) - goto out; - - error = sysctl_createv(log, 0, &rnode, &txnode, - 0, CTLTYPE_NODE, "tx", - SYSCTL_DESCR("virtio-net information and settings for Tx"), - NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); - if (error) - goto out; - - error = sysctl_createv(log, 0, &txnode, NULL, - CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", - SYSCTL_DESCR("max number of Tx packets to process for interrupt processing"), - NULL, 0, &sc->sc_tx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); - if (error) - goto out; - - error = sysctl_createv(log, 0, &txnode, NULL, - CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", - SYSCTL_DESCR("max number of Tx packets to process for deferred processing"), - NULL, 0, &sc->sc_tx_process_limit, 0, CTL_CREATE, CTL_EOL); - -out: - if (error) - sysctl_teardown(log); - - return error; -} - -static void -vioif_setup_stats(struct vioif_softc *sc) -{ - struct vioif_rxqueue *rxq; - struct vioif_txqueue *txq; - int i; - - for (i = 0; i < sc->sc_max_nvq_pairs; i++) { - rxq = &sc->sc_rxq[i]; - txq = &sc->sc_txq[i]; - - snprintf(txq->txq_evgroup, sizeof(txq->txq_evgroup), "%s-TX%d", - device_xname(sc->sc_dev), i); - evcnt_attach_dynamic(&txq->txq_defrag_failed, EVCNT_TYPE_MISC, - NULL, txq->txq_evgroup, "tx m_defrag() failed"); - evcnt_attach_dynamic(&txq->txq_mbuf_load_failed, EVCNT_TYPE_MISC, - NULL, txq->txq_evgroup, "tx dmamap load failed"); - evcnt_attach_dynamic(&txq->txq_enqueue_reserve_failed, EVCNT_TYPE_MISC, - NULL, txq->txq_evgroup, "virtio_enqueue_reserve failed"); - - snprintf(rxq->rxq_evgroup, sizeof(rxq->rxq_evgroup), "%s-RX%d", - device_xname(sc->sc_dev), i); - evcnt_attach_dynamic(&rxq->rxq_mbuf_add_failed, EVCNT_TYPE_MISC, - NULL, rxq->rxq_evgroup, "rx mbuf allocation failed"); - } - - evcnt_attach_dynamic(&sc->sc_ctrlq.ctrlq_cmd_load_failed, EVCNT_TYPE_MISC, - NULL, device_xname(sc->sc_dev), "control command dmamap load failed"); - evcnt_attach_dynamic(&sc->sc_ctrlq.ctrlq_cmd_failed, EVCNT_TYPE_MISC, - NULL, device_xname(sc->sc_dev), "control command failed"); -} - MODULE(MODULE_CLASS_DRIVER, if_vioif, "virtio"); #ifdef _MODULE