Module Name: src Committed By: hikaru Date: Fri Nov 25 05:29:55 UTC 2016
Modified Files: src/sys/arch/x86/pci: if_vmx.c if_vmxreg.h Log Message: Sync code with FreeBSD to support RSS - Use MSI/MSI-X if it is available. - Support TSO. co-authored by k-nakahara To generate a diff of this commit: cvs rdiff -u -r1.7 -r1.8 src/sys/arch/x86/pci/if_vmx.c cvs rdiff -u -r1.1 -r1.2 src/sys/arch/x86/pci/if_vmxreg.h Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files.
Modified files: Index: src/sys/arch/x86/pci/if_vmx.c diff -u src/sys/arch/x86/pci/if_vmx.c:1.7 src/sys/arch/x86/pci/if_vmx.c:1.8 --- src/sys/arch/x86/pci/if_vmx.c:1.7 Fri Jun 10 13:27:13 2016 +++ src/sys/arch/x86/pci/if_vmx.c Fri Nov 25 05:29:54 2016 @@ -1,8 +1,9 @@ -/* $NetBSD: if_vmx.c,v 1.7 2016/06/10 13:27:13 ozaki-r Exp $ */ +/* $NetBSD: if_vmx.c,v 1.8 2016/11/25 05:29:54 hikaru Exp $ */ /* $OpenBSD: if_vmx.c,v 1.16 2014/01/22 06:04:17 brad Exp $ */ /* * Copyright (c) 2013 Tsubai Masanari + * Copyright (c) 2013 Bryan Venteicher <bry...@freebsd.org> * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -18,9 +19,12 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: if_vmx.c,v 1.7 2016/06/10 13:27:13 ozaki-r Exp $"); +__KERNEL_RCSID(0, "$NetBSD: if_vmx.c,v 1.8 2016/11/25 05:29:54 hikaru Exp $"); #include <sys/param.h> +#include <sys/cpu.h> +#include <sys/kernel.h> +#include <sys/kmem.h> #include <sys/bus.h> #include <sys/device.h> #include <sys/mbuf.h> @@ -35,6 +39,7 @@ __KERNEL_RCSID(0, "$NetBSD: if_vmx.c,v 1 #include <netinet/in_systm.h> /* for <netinet/ip.h> */ #include <netinet/in.h> /* for <netinet/ip.h> */ #include <netinet/ip.h> /* for struct ip */ +#include <netinet/ip6.h> /* for struct ip6_hdr */ #include <netinet/tcp.h> /* for struct tcphdr */ #include <netinet/udp.h> /* for struct udphdr */ @@ -44,137 +49,446 @@ __KERNEL_RCSID(0, "$NetBSD: if_vmx.c,v 1 #include <arch/x86/pci/if_vmxreg.h> -#define NRXQUEUE 1 -#define NTXQUEUE 1 +#define VMXNET3_DRIVER_VERSION 0x00010000 -#define NTXDESC 128 /* tx ring size */ -#define NTXSEGS 8 /* tx descriptors per packet */ -#define NRXDESC 128 -#define NTXCOMPDESC NTXDESC -#define NRXCOMPDESC (NRXDESC * 2) /* ring1 + ring2 */ +/* + * Max descriptors per Tx packet. We must limit the size of the + * any TSO packets based on the number of segments. + */ +#define VMXNET3_TX_MAXSEGS 32 +#define VMXNET3_TX_MAXSIZE (VMXNET3_TX_MAXSEGS * MCLBYTES) -#define VMXNET3_DRIVER_VERSION 0x00010000 +/* + * Maximum support Tx segments size. The length field in the + * Tx descriptor is 14 bits. + */ +#define VMXNET3_TX_MAXSEGSIZE (1 << 14) + +/* + * The maximum number of Rx segments we accept. + */ +#define VMXNET3_MAX_RX_SEGS 0 /* no segments */ + +/* + * Predetermined size of the multicast MACs filter table. If the + * number of multicast addresses exceeds this size, then the + * ALL_MULTI mode is use instead. + */ +#define VMXNET3_MULTICAST_MAX 32 + +/* + * Our Tx watchdog timeout. + */ +#define VMXNET3_WATCHDOG_TIMEOUT 5 + +/* + * IP protocols that we can perform Tx checksum offloading of. + */ +#define VMXNET3_CSUM_OFFLOAD \ + (M_CSUM_TCPv4 | M_CSUM_UDPv4) +#define VMXNET3_CSUM_OFFLOAD_IPV6 \ + (M_CSUM_TCPv6 | M_CSUM_UDPv6) + +#define VMXNET3_CSUM_ALL_OFFLOAD \ + (VMXNET3_CSUM_OFFLOAD | VMXNET3_CSUM_OFFLOAD_IPV6 | M_CSUM_TSOv4 | M_CSUM_TSOv6) + +#define VMXNET3_RXRINGS_PERQ 2 + +#define VMXNET3_CORE_LOCK(_sc) mutex_enter((_sc)->vmx_mtx) +#define VMXNET3_CORE_UNLOCK(_sc) mutex_exit((_sc)->vmx_mtx) +#define VMXNET3_CORE_LOCK_ASSERT(_sc) mutex_owned((_sc)->vmx_mtx) +#define VMXNET3_CORE_LOCK_ASSERT_NOTOWNED(_sc) \ + (!mutex_owned((_sc)->vmx_mtx)) + +#define VMXNET3_RXQ_LOCK(_rxq) mutex_enter((_rxq)->vxrxq_mtx) +#define VMXNET3_RXQ_UNLOCK(_rxq) mutex_exit((_rxq)->vxrxq_mtx) +#define VMXNET3_RXQ_LOCK_ASSERT(_rxq) \ + mutex_owned((_rxq)->vxrxq_mtx) +#define VMXNET3_RXQ_LOCK_ASSERT_NOTOWNED(_rxq) \ + (!mutex_owned((_rxq)->vxrxq_mtx)) + +#define VMXNET3_TXQ_LOCK(_txq) mutex_enter((_txq)->vxtxq_mtx) +#define VMXNET3_TXQ_UNLOCK(_txq) mutex_exit((_txq)->vxtxq_mtx) +#define VMXNET3_TXQ_LOCK_ASSERT(_txq) \ + mutex_owned((_txq)->vxtxq_mtx) +#define VMXNET3_TXQ_LOCK_ASSERT_NOTOWNED(_txq) \ + (!mutex_owned((_txq)->vxtxq_mtx)) + +struct vmxnet3_dma_alloc { + bus_addr_t dma_paddr; + void *dma_vaddr; + bus_dmamap_t dma_map; + bus_size_t dma_size; + bus_dma_segment_t dma_segs[1]; +}; + +struct vmxnet3_txbuf { + bus_dmamap_t vtxb_dmamap; + struct mbuf *vtxb_m; +}; struct vmxnet3_txring { - struct mbuf *m[NTXDESC]; - bus_dmamap_t dmap[NTXDESC]; - struct vmxnet3_txdesc *txd; - u_int head; - u_int next; - uint8_t gen; + struct vmxnet3_txbuf *vxtxr_txbuf; + struct vmxnet3_txdesc *vxtxr_txd; + u_int vxtxr_head; + u_int vxtxr_next; + u_int vxtxr_ndesc; + int vxtxr_gen; + struct vmxnet3_dma_alloc vxtxr_dma; +}; + +struct vmxnet3_rxbuf { + bus_dmamap_t vrxb_dmamap; + struct mbuf *vrxb_m; }; struct vmxnet3_rxring { - struct mbuf *m[NRXDESC]; - bus_dmamap_t dmap[NRXDESC]; - struct vmxnet3_rxdesc *rxd; - u_int fill; - uint8_t gen; - uint8_t rid; + struct vmxnet3_rxbuf *vxrxr_rxbuf; + struct vmxnet3_rxdesc *vxrxr_rxd; + u_int vxrxr_fill; + u_int vxrxr_ndesc; + int vxrxr_gen; + int vxrxr_rid; + struct vmxnet3_dma_alloc vxrxr_dma; + bus_dmamap_t vxrxr_spare_dmap; }; struct vmxnet3_comp_ring { union { struct vmxnet3_txcompdesc *txcd; struct vmxnet3_rxcompdesc *rxcd; - }; - u_int next; - uint8_t gen; + } vxcr_u; + u_int vxcr_next; + u_int vxcr_ndesc; + int vxcr_gen; + struct vmxnet3_dma_alloc vxcr_dma; +}; + +struct vmxnet3_txq_stats { + uint64_t vmtxs_opackets; /* if_opackets */ + uint64_t vmtxs_obytes; /* if_obytes */ + uint64_t vmtxs_omcasts; /* if_omcasts */ + uint64_t vmtxs_csum; + uint64_t vmtxs_tso; + uint64_t vmtxs_full; + uint64_t vmtxs_offload_failed; }; struct vmxnet3_txqueue { - struct vmxnet3_txring cmd_ring; - struct vmxnet3_comp_ring comp_ring; - struct vmxnet3_txq_shared *ts; + kmutex_t *vxtxq_mtx; + struct vmxnet3_softc *vxtxq_sc; + int vxtxq_id; + int vxtxq_intr_idx; + int vxtxq_watchdog; + struct vmxnet3_txring vxtxq_cmd_ring; + struct vmxnet3_comp_ring vxtxq_comp_ring; + struct vmxnet3_txq_stats vxtxq_stats; + struct vmxnet3_txq_shared *vxtxq_ts; + char vxtxq_name[16]; +}; + +struct vmxnet3_rxq_stats { + uint64_t vmrxs_ipackets; /* if_ipackets */ + uint64_t vmrxs_ibytes; /* if_ibytes */ + uint64_t vmrxs_iqdrops; /* if_iqdrops */ + uint64_t vmrxs_ierrors; /* if_ierrors */ }; struct vmxnet3_rxqueue { - struct vmxnet3_rxring cmd_ring[2]; - struct vmxnet3_comp_ring comp_ring; - struct vmxnet3_rxq_shared *rs; + kmutex_t *vxrxq_mtx; + struct vmxnet3_softc *vxrxq_sc; + int vxrxq_id; + int vxrxq_intr_idx; + struct mbuf *vxrxq_mhead; + struct mbuf *vxrxq_mtail; + struct vmxnet3_rxring vxrxq_cmd_ring[VMXNET3_RXRINGS_PERQ]; + struct vmxnet3_comp_ring vxrxq_comp_ring; + struct vmxnet3_rxq_stats vxrxq_stats; + struct vmxnet3_rxq_shared *vxrxq_rs; + char vxrxq_name[16]; +}; + +struct vmxnet3_statistics { + uint32_t vmst_defragged; + uint32_t vmst_defrag_failed; + uint32_t vmst_mgetcl_failed; + uint32_t vmst_mbuf_load_failed; }; struct vmxnet3_softc { - device_t sc_dev; - struct ethercom sc_ethercom; - struct ifmedia sc_media; - - bus_space_tag_t sc_iot0; - bus_space_tag_t sc_iot1; - bus_space_handle_t sc_ioh0; - bus_space_handle_t sc_ioh1; - bus_dma_tag_t sc_dmat; - - struct vmxnet3_txqueue sc_txq[NTXQUEUE]; - struct vmxnet3_rxqueue sc_rxq[NRXQUEUE]; - struct vmxnet3_driver_shared *sc_ds; - uint8_t *sc_mcast; + device_t vmx_dev; + struct ethercom vmx_ethercom; + struct ifmedia vmx_media; + struct vmxnet3_driver_shared *vmx_ds; + int vmx_flags; +#define VMXNET3_FLAG_NO_MSIX (1 << 0) +#define VMXNET3_FLAG_RSS (1 << 1) +#define VMXNET3_FLAG_ATTACHED (1 << 2) + + struct vmxnet3_txqueue *vmx_txq; + struct vmxnet3_rxqueue *vmx_rxq; + + struct pci_attach_args *vmx_pa; + + bus_space_tag_t vmx_iot0; + bus_space_tag_t vmx_iot1; + bus_space_handle_t vmx_ioh0; + bus_space_handle_t vmx_ioh1; + bus_size_t vmx_ios0; + bus_size_t vmx_ios1; + bus_dma_tag_t vmx_dmat; + + int vmx_link_active; + int vmx_ntxqueues; + int vmx_nrxqueues; + int vmx_ntxdescs; + int vmx_nrxdescs; + int vmx_max_rxsegs; + + struct vmxnet3_statistics vmx_stats; + + int vmx_intr_type; + int vmx_intr_mask_mode; + int vmx_event_intr_idx; + int vmx_nintrs; + pci_intr_handle_t *vmx_intrs; /* legacy use vmx_intrs[0] */ + void *vmx_ihs[VMXNET3_MAX_INTRS]; + + kmutex_t *vmx_mtx; + + uint8_t *vmx_mcast; + void *vmx_qs; + struct vmxnet3_rss_shared *vmx_rss; + callout_t vmx_tick; + struct vmxnet3_dma_alloc vmx_ds_dma; + struct vmxnet3_dma_alloc vmx_qs_dma; + struct vmxnet3_dma_alloc vmx_mcast_dma; + struct vmxnet3_dma_alloc vmx_rss_dma; + int vmx_max_ntxqueues; + int vmx_max_nrxqueues; + uint8_t vmx_lladdr[ETHER_ADDR_LEN]; }; #define VMXNET3_STAT #ifdef VMXNET3_STAT struct { - u_int ntxdesc; - u_int nrxdesc; u_int txhead; u_int txdone; u_int maxtxlen; u_int rxdone; u_int rxfill; u_int intr; -} vmxstat = { - .ntxdesc = NTXDESC, - .nrxdesc = NRXDESC -}; +} vmxstat; #endif +typedef enum { + VMXNET3_BARRIER_RD, + VMXNET3_BARRIER_WR, + VMXNET3_BARRIER_RDWR, +} vmxnet3_barrier_t; + #define JUMBO_LEN (MCLBYTES - ETHER_ALIGN) /* XXX */ #define DMAADDR(map) ((map)->dm_segs[0].ds_addr) -#define READ_BAR0(sc, reg) bus_space_read_4((sc)->sc_iot0, (sc)->sc_ioh0, reg) -#define READ_BAR1(sc, reg) bus_space_read_4((sc)->sc_iot1, (sc)->sc_ioh1, reg) -#define WRITE_BAR0(sc, reg, val) \ - bus_space_write_4((sc)->sc_iot0, (sc)->sc_ioh0, reg, val) -#define WRITE_BAR1(sc, reg, val) \ - bus_space_write_4((sc)->sc_iot1, (sc)->sc_ioh1, reg, val) -#define WRITE_CMD(sc, cmd) WRITE_BAR1(sc, VMXNET3_BAR1_CMD, cmd) #define vtophys(va) 0 /* XXX ok? */ int vmxnet3_match(device_t, cfdata_t, void *); void vmxnet3_attach(device_t, device_t, void *); -int vmxnet3_dma_init(struct vmxnet3_softc *); -int vmxnet3_alloc_txring(struct vmxnet3_softc *, int); -int vmxnet3_alloc_rxring(struct vmxnet3_softc *, int); -void vmxnet3_txinit(struct vmxnet3_softc *, struct vmxnet3_txqueue *); -void vmxnet3_rxinit(struct vmxnet3_softc *, struct vmxnet3_rxqueue *); +int vmxnet3_detach(device_t, int); + +int vmxnet3_alloc_pci_resources(struct vmxnet3_softc *); +void vmxnet3_free_pci_resources(struct vmxnet3_softc *); +int vmxnet3_check_version(struct vmxnet3_softc *); +void vmxnet3_check_multiqueue(struct vmxnet3_softc *); + +int vmxnet3_alloc_msix_interrupts(struct vmxnet3_softc *); +int vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *); +int vmxnet3_alloc_legacy_interrupts(struct vmxnet3_softc *); +int vmxnet3_alloc_interrupts(struct vmxnet3_softc *); +void vmxnet3_free_interrupts(struct vmxnet3_softc *); + +int vmxnet3_setup_msix_interrupts(struct vmxnet3_softc *); +int vmxnet3_setup_msi_interrupt(struct vmxnet3_softc *); +int vmxnet3_setup_legacy_interrupt(struct vmxnet3_softc *); +void vmxnet3_set_interrupt_idx(struct vmxnet3_softc *); +int vmxnet3_setup_interrupts(struct vmxnet3_softc *); + +int vmxnet3_init_rxq(struct vmxnet3_softc *, int); +int vmxnet3_init_txq(struct vmxnet3_softc *, int); +int vmxnet3_alloc_rxtx_queues(struct vmxnet3_softc *); +void vmxnet3_destroy_rxq(struct vmxnet3_rxqueue *); +void vmxnet3_destroy_txq(struct vmxnet3_txqueue *); +void vmxnet3_free_rxtx_queues(struct vmxnet3_softc *); + +int vmxnet3_alloc_shared_data(struct vmxnet3_softc *); +void vmxnet3_free_shared_data(struct vmxnet3_softc *); +int vmxnet3_alloc_txq_data(struct vmxnet3_softc *); +void vmxnet3_free_txq_data(struct vmxnet3_softc *); +int vmxnet3_alloc_rxq_data(struct vmxnet3_softc *); +void vmxnet3_free_rxq_data(struct vmxnet3_softc *); +int vmxnet3_alloc_queue_data(struct vmxnet3_softc *); +void vmxnet3_free_queue_data(struct vmxnet3_softc *); +int vmxnet3_alloc_mcast_table(struct vmxnet3_softc *); +void vmxnet3_free_mcast_table(struct vmxnet3_softc *); +void vmxnet3_init_shared_data(struct vmxnet3_softc *); +void vmxnet3_reinit_rss_shared_data(struct vmxnet3_softc *); +void vmxnet3_reinit_shared_data(struct vmxnet3_softc *); +int vmxnet3_alloc_data(struct vmxnet3_softc *); +void vmxnet3_free_data(struct vmxnet3_softc *); +int vmxnet3_setup_interface(struct vmxnet3_softc *); + +void vmxnet3_evintr(struct vmxnet3_softc *); +void vmxnet3_txq_eof(struct vmxnet3_txqueue *); +int vmxnet3_newbuf(struct vmxnet3_softc *, struct vmxnet3_rxring *); +void vmxnet3_rxq_eof_discard(struct vmxnet3_rxqueue *, + struct vmxnet3_rxring *, int); +void vmxnet3_rxq_discard_chain(struct vmxnet3_rxqueue *); +void vmxnet3_rx_csum(struct vmxnet3_rxcompdesc *, struct mbuf *); +void vmxnet3_rxq_input(struct vmxnet3_rxqueue *, + struct vmxnet3_rxcompdesc *, struct mbuf *); +void vmxnet3_rxq_eof(struct vmxnet3_rxqueue *); +int vmxnet3_legacy_intr(void *); +int vmxnet3_txq_intr(void *); +int vmxnet3_rxq_intr(void *); +int vmxnet3_event_intr(void *); + void vmxnet3_txstop(struct vmxnet3_softc *, struct vmxnet3_txqueue *); void vmxnet3_rxstop(struct vmxnet3_softc *, struct vmxnet3_rxqueue *); -void vmxnet3_link_state(struct vmxnet3_softc *); -void vmxnet3_enable_all_intrs(struct vmxnet3_softc *); -void vmxnet3_disable_all_intrs(struct vmxnet3_softc *); -int vmxnet3_intr(void *); -void vmxnet3_evintr(struct vmxnet3_softc *); -void vmxnet3_txintr(struct vmxnet3_softc *, struct vmxnet3_txqueue *); -void vmxnet3_rxintr(struct vmxnet3_softc *, struct vmxnet3_rxqueue *); -void vmxnet3_iff(struct vmxnet3_softc *); -int vmxnet3_ifflags_cb(struct ethercom *); +void vmxnet3_stop_locked(struct vmxnet3_softc *); +void vmxnet3_stop_rendezvous(struct vmxnet3_softc *); +void vmxnet3_stop(struct ifnet *, int); + +void vmxnet3_txinit(struct vmxnet3_softc *, struct vmxnet3_txqueue *); +int vmxnet3_rxinit(struct vmxnet3_softc *, struct vmxnet3_rxqueue *); +int vmxnet3_reinit_queues(struct vmxnet3_softc *); +int vmxnet3_enable_device(struct vmxnet3_softc *); +void vmxnet3_reinit_rxfilters(struct vmxnet3_softc *); +int vmxnet3_reinit(struct vmxnet3_softc *); + void vmxnet3_rx_csum(struct vmxnet3_rxcompdesc *, struct mbuf *); -int vmxnet3_getbuf(struct vmxnet3_softc *, struct vmxnet3_rxring *); -void vmxnet3_stop(struct ifnet *, int disable); -void vmxnet3_reset(struct ifnet *); +int vmxnet3_init_locked(struct vmxnet3_softc *); int vmxnet3_init(struct ifnet *); -int vmxnet3_ioctl(struct ifnet *, u_long, void *); -int vmxnet3_change_mtu(struct vmxnet3_softc *, int); + +int vmxnet3_txq_offload_ctx(struct vmxnet3_txqueue *, struct mbuf *, int *, int *); +int vmxnet3_txq_load_mbuf(struct vmxnet3_txqueue *, struct mbuf **, bus_dmamap_t); +void vmxnet3_txq_unload_mbuf(struct vmxnet3_txqueue *, bus_dmamap_t); +int vmxnet3_txq_encap(struct vmxnet3_txqueue *, struct mbuf **); +void vmxnet3_start_locked(struct ifnet *); void vmxnet3_start(struct ifnet *); -int vmxnet3_load_mbuf(struct vmxnet3_softc *, struct mbuf *); -void vmxnet3_watchdog(struct ifnet *); + +void vmxnet3_set_rxfilter(struct vmxnet3_softc *); +int vmxnet3_change_mtu(struct vmxnet3_softc *, int); +int vmxnet3_ioctl(struct ifnet *, u_long, void *); +int vmxnet3_ifflags_cb(struct ethercom *); + +int vmxnet3_watchdog(struct vmxnet3_txqueue *); +void vmxnet3_refresh_host_stats(struct vmxnet3_softc *); +void vmxnet3_tick(void *); +void vmxnet3_link_status(struct vmxnet3_softc *); void vmxnet3_media_status(struct ifnet *, struct ifmediareq *); int vmxnet3_media_change(struct ifnet *); -void *vmxnet3_dma_allocmem(struct vmxnet3_softc *, u_int, u_int, bus_addr_t *); +void vmxnet3_set_lladdr(struct vmxnet3_softc *); +void vmxnet3_get_lladdr(struct vmxnet3_softc *); + +void vmxnet3_enable_all_intrs(struct vmxnet3_softc *); +void vmxnet3_disable_all_intrs(struct vmxnet3_softc *); + +int vmxnet3_dma_malloc(struct vmxnet3_softc *, bus_size_t, bus_size_t, + struct vmxnet3_dma_alloc *); +void vmxnet3_dma_free(struct vmxnet3_softc *, struct vmxnet3_dma_alloc *); CFATTACH_DECL3_NEW(vmx, sizeof(struct vmxnet3_softc), - vmxnet3_match, vmxnet3_attach, NULL, NULL, NULL, NULL, 0); + vmxnet3_match, vmxnet3_attach, vmxnet3_detach, NULL, NULL, NULL, 0); + +static inline void +vmxnet3_write_bar0(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v) +{ + + bus_space_write_4(sc->vmx_iot0, sc->vmx_ioh0, r, v); +} + +static inline uint32_t +vmxnet3_read_bar1(struct vmxnet3_softc *sc, bus_size_t r) +{ + + return (bus_space_read_4(sc->vmx_iot1, sc->vmx_ioh1, r)); +} + +static inline void +vmxnet3_write_bar1(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v) +{ + + bus_space_write_4(sc->vmx_iot1, sc->vmx_ioh1, r, v); +} + +static inline void +vmxnet3_write_cmd(struct vmxnet3_softc *sc, uint32_t cmd) +{ + + vmxnet3_write_bar1(sc, VMXNET3_BAR1_CMD, cmd); +} + +static inline uint32_t +vmxnet3_read_cmd(struct vmxnet3_softc *sc, uint32_t cmd) +{ + + vmxnet3_write_cmd(sc, cmd); + return (vmxnet3_read_bar1(sc, VMXNET3_BAR1_CMD)); +} + +static inline void +vmxnet3_enable_intr(struct vmxnet3_softc *sc, int irq) +{ + vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 0); +} + +static inline void +vmxnet3_disable_intr(struct vmxnet3_softc *sc, int irq) +{ + vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 1); +} + +static inline void +vmxnet3_rxr_increment_fill(struct vmxnet3_rxring *rxr) +{ + + if (++rxr->vxrxr_fill == rxr->vxrxr_ndesc) { + rxr->vxrxr_fill = 0; + rxr->vxrxr_gen ^= 1; + } +} + +static inline int +vmxnet3_txring_avail(struct vmxnet3_txring *txr) +{ + int avail = txr->vxtxr_next - txr->vxtxr_head - 1; + return (avail < 0 ? txr->vxtxr_ndesc + avail : avail); +} + +/* + * Since this is a purely paravirtualized device, we do not have + * to worry about DMA coherency. But at times, we must make sure + * both the compiler and CPU do not reorder memory operations. + */ +static inline void +vmxnet3_barrier(struct vmxnet3_softc *sc, vmxnet3_barrier_t type) +{ + + switch (type) { + case VMXNET3_BARRIER_RD: + membar_consumer(); + break; + case VMXNET3_BARRIER_WR: + membar_producer(); + break; + case VMXNET3_BARRIER_RDWR: + membar_sync(); + break; + default: + panic("%s: bad barrier type %d", __func__, type); + } +} int vmxnet3_match(device_t parent, cfdata_t match, void *aux) @@ -193,617 +507,2297 @@ vmxnet3_attach(device_t parent, device_t { struct vmxnet3_softc *sc = device_private(self); struct pci_attach_args *pa = aux; - struct ifnet *ifp = &sc->sc_ethercom.ec_if; - pci_intr_handle_t ih; - const char *intrstr; - void *vih; - u_int memtype, ver, macl, mach; pcireg_t preg; - u_char enaddr[ETHER_ADDR_LEN]; - char intrbuf[PCI_INTRSTR_LEN]; + int error; - sc->sc_dev = self; + sc->vmx_dev = self; + sc->vmx_pa = pa; + if (pci_dma64_available(pa)) + sc->vmx_dmat = pa->pa_dmat64; + else + sc->vmx_dmat = pa->pa_dmat; pci_aprint_devinfo_fancy(pa, "Ethernet controller", "vmxnet3", 1); - memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, 0x10); - if (pci_mapreg_map(pa, 0x10, memtype, 0, &sc->sc_iot0, &sc->sc_ioh0, - NULL, NULL)) { - aprint_error_dev(sc->sc_dev, "failed to map BAR0\n"); - return; - } - memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, 0x14); - if (pci_mapreg_map(pa, 0x14, memtype, 0, &sc->sc_iot1, &sc->sc_ioh1, - NULL, NULL)) { - aprint_error_dev(sc->sc_dev, "failed to map BAR1\n"); - return; - } - - ver = READ_BAR1(sc, VMXNET3_BAR1_VRRS); - if ((ver & 0x1) == 0) { - aprint_error_dev(sc->sc_dev, - "unsupported hardware version 0x%x\n", ver); - return; - } - WRITE_BAR1(sc, VMXNET3_BAR1_VRRS, 1); - - ver = READ_BAR1(sc, VMXNET3_BAR1_UVRS); - if ((ver & 0x1) == 0) { - aprint_error_dev(sc->sc_dev, - "incompatiable UPT version 0x%x\n", ver); - return; - } - WRITE_BAR1(sc, VMXNET3_BAR1_UVRS, 1); - preg = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG); preg |= PCI_COMMAND_MASTER_ENABLE; pci_conf_write(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, preg); - if (pci_dma64_available(pa)) - sc->sc_dmat = pa->pa_dmat64; - else - sc->sc_dmat = pa->pa_dmat; - if (vmxnet3_dma_init(sc)) { - aprint_error_dev(sc->sc_dev, "failed to setup DMA\n"); + sc->vmx_mtx = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET); + callout_init(&sc->vmx_tick, CALLOUT_MPSAFE); + + sc->vmx_max_ntxqueues = ncpu; + sc->vmx_max_nrxqueues = ncpu; + sc->vmx_ntxdescs = 512; + sc->vmx_nrxdescs = 256; + sc->vmx_max_rxsegs = VMXNET3_MAX_RX_SEGS; + + error = vmxnet3_alloc_pci_resources(sc); + if (error) return; - } - if (pci_intr_map(pa, &ih)) { - aprint_error_dev(sc->sc_dev, "failed to map interrupt\n"); + error = vmxnet3_check_version(sc); + if (error) return; - } - intrstr = pci_intr_string(pa->pa_pc, ih, intrbuf, sizeof(intrbuf)); - vih = pci_intr_establish(pa->pa_pc, ih, IPL_NET, vmxnet3_intr, sc); - if (vih == NULL) { - aprint_error_dev(sc->sc_dev, - "unable to establish interrupt%s%s\n", - intrstr ? " at " : "", intrstr ? intrstr : ""); + + error = vmxnet3_alloc_rxtx_queues(sc); + if (error) return; - } - aprint_normal_dev(sc->sc_dev, "interrupting at %s\n", intrstr); - WRITE_CMD(sc, VMXNET3_CMD_GET_MACL); - macl = READ_BAR1(sc, VMXNET3_BAR1_CMD); - enaddr[0] = macl; - enaddr[1] = macl >> 8; - enaddr[2] = macl >> 16; - enaddr[3] = macl >> 24; - WRITE_CMD(sc, VMXNET3_CMD_GET_MACH); - mach = READ_BAR1(sc, VMXNET3_BAR1_CMD); - enaddr[4] = mach; - enaddr[5] = mach >> 8; - - WRITE_BAR1(sc, VMXNET3_BAR1_MACL, macl); - WRITE_BAR1(sc, VMXNET3_BAR1_MACH, mach); - aprint_normal_dev(sc->sc_dev, "Ethernet address %s\n", - ether_sprintf(enaddr)); + error = vmxnet3_alloc_interrupts(sc); + if (error) + return; - strlcpy(ifp->if_xname, device_xname(sc->sc_dev), IFNAMSIZ); - ifp->if_softc = sc; - ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST | IFF_SIMPLEX; - ifp->if_ioctl = vmxnet3_ioctl; - ifp->if_start = vmxnet3_start; - ifp->if_watchdog = vmxnet3_watchdog; - ifp->if_init = vmxnet3_init; - ifp->if_stop = vmxnet3_stop; - sc->sc_ethercom.ec_capabilities |= ETHERCAP_VLAN_MTU; - if (sc->sc_ds->upt_features & UPT1_F_CSUM) - sc->sc_ethercom.ec_if.if_capabilities |= - IFCAP_CSUM_IPv4_Rx | - IFCAP_CSUM_TCPv4_Tx | IFCAP_CSUM_TCPv4_Rx | - IFCAP_CSUM_UDPv4_Tx | IFCAP_CSUM_UDPv4_Rx; - if (sc->sc_ds->upt_features & UPT1_F_VLAN) - sc->sc_ethercom.ec_capabilities |= ETHERCAP_VLAN_HWTAGGING; + vmxnet3_check_multiqueue(sc); - IFQ_SET_MAXLEN(&ifp->if_snd, NTXDESC); - IFQ_SET_READY(&ifp->if_snd); + error = vmxnet3_alloc_data(sc); + if (error) + return; - ifmedia_init(&sc->sc_media, IFM_IMASK, vmxnet3_media_change, - vmxnet3_media_status); - ifmedia_add(&sc->sc_media, IFM_ETHER|IFM_AUTO, 0, NULL); - ifmedia_add(&sc->sc_media, IFM_ETHER|IFM_10G_T|IFM_FDX, 0, NULL); - ifmedia_add(&sc->sc_media, IFM_ETHER|IFM_10G_T, 0, NULL); - ifmedia_add(&sc->sc_media, IFM_ETHER|IFM_1000_T|IFM_FDX, 0, NULL); - ifmedia_add(&sc->sc_media, IFM_ETHER|IFM_1000_T, 0, NULL); - ifmedia_set(&sc->sc_media, IFM_ETHER|IFM_AUTO); + error = vmxnet3_setup_interface(sc); + if (error) + return; - if_attach(ifp); - ether_ifattach(ifp, enaddr); - ether_set_ifflags_cb(&sc->sc_ethercom, vmxnet3_ifflags_cb); - vmxnet3_link_state(sc); + error = vmxnet3_setup_interrupts(sc); + if (error) + return; + + sc->vmx_flags |= VMXNET3_FLAG_ATTACHED; } int -vmxnet3_dma_init(struct vmxnet3_softc *sc) +vmxnet3_detach(device_t self, int flags) { - struct vmxnet3_driver_shared *ds; - struct vmxnet3_txq_shared *ts; - struct vmxnet3_rxq_shared *rs; - bus_addr_t ds_pa, qs_pa, mcast_pa; - int i, queue, qs_len; - u_int major, minor, release_code, rev; - - qs_len = NTXQUEUE * sizeof *ts + NRXQUEUE * sizeof *rs; - ts = vmxnet3_dma_allocmem(sc, qs_len, VMXNET3_DMADESC_ALIGN, &qs_pa); - if (ts == NULL) - return -1; - for (queue = 0; queue < NTXQUEUE; queue++) - sc->sc_txq[queue].ts = ts++; - rs = (void *)ts; - for (queue = 0; queue < NRXQUEUE; queue++) - sc->sc_rxq[queue].rs = rs++; - - for (queue = 0; queue < NTXQUEUE; queue++) - if (vmxnet3_alloc_txring(sc, queue)) - return -1; - for (queue = 0; queue < NRXQUEUE; queue++) - if (vmxnet3_alloc_rxring(sc, queue)) - return -1; + struct vmxnet3_softc *sc; + struct ifnet *ifp; - sc->sc_mcast = vmxnet3_dma_allocmem(sc, 682 * ETHER_ADDR_LEN, 32, &mcast_pa); - if (sc->sc_mcast == NULL) - return -1; + sc = device_private(self); + ifp = &sc->vmx_ethercom.ec_if; - ds = vmxnet3_dma_allocmem(sc, sizeof *sc->sc_ds, 8, &ds_pa); - if (ds == NULL) - return -1; - sc->sc_ds = ds; - ds->magic = VMXNET3_REV1_MAGIC; - ds->version = VMXNET3_DRIVER_VERSION; + if (sc->vmx_flags & VMXNET3_FLAG_ATTACHED) { + VMXNET3_CORE_LOCK(sc); + vmxnet3_stop_locked(sc); + callout_halt(&sc->vmx_tick, sc->vmx_mtx); + VMXNET3_CORE_UNLOCK(sc); - /* - * XXX FreeBSD version uses following values: - * (Does the device behavior depend on them?) - * - * major = __FreeBSD_version / 100000; - * minor = (__FreeBSD_version / 1000) % 100; - * release_code = (__FreeBSD_version / 100) % 10; - * rev = __FreeBSD_version % 100; - */ - major = 0; - minor = 0; - release_code = 0; - rev = 0; -#ifdef __LP64__ - ds->guest = release_code << 30 | rev << 22 | major << 14 | minor << 6 - | VMXNET3_GOS_FREEBSD | VMXNET3_GOS_64BIT; -#else - ds->guest = release_code << 30 | rev << 22 | major << 14 | minor << 6 - | VMXNET3_GOS_FREEBSD | VMXNET3_GOS_32BIT; -#endif - ds->vmxnet3_revision = 1; - ds->upt_version = 1; - ds->upt_features = UPT1_F_CSUM | UPT1_F_VLAN; - ds->driver_data = vtophys(sc); - ds->driver_data_len = sizeof(struct vmxnet3_softc); - ds->queue_shared = qs_pa; - ds->queue_shared_len = qs_len; - ds->mtu = ETHERMTU; - ds->ntxqueue = NTXQUEUE; - ds->nrxqueue = NRXQUEUE; - ds->mcast_table = mcast_pa; - ds->automask = 1; - ds->nintr = VMXNET3_NINTR; - ds->evintr = 0; - ds->ictrl = VMXNET3_ICTRL_DISABLE_ALL; - for (i = 0; i < VMXNET3_NINTR; i++) - ds->modlevel[i] = UPT1_IMOD_ADAPTIVE; - WRITE_BAR1(sc, VMXNET3_BAR1_DSL, ds_pa); - WRITE_BAR1(sc, VMXNET3_BAR1_DSH, (uint64_t)ds_pa >> 32); - return 0; -} + ifmedia_delete_instance(&sc->vmx_media, IFM_INST_ANY); -int -vmxnet3_alloc_txring(struct vmxnet3_softc *sc, int queue) -{ - struct vmxnet3_txqueue *tq = &sc->sc_txq[queue]; - struct vmxnet3_txq_shared *ts; - struct vmxnet3_txring *ring = &tq->cmd_ring; - struct vmxnet3_comp_ring *comp_ring = &tq->comp_ring; - bus_addr_t pa, comp_pa; - int idx; + ether_ifdetach(ifp); + if_detach(ifp); + } - ring->txd = vmxnet3_dma_allocmem(sc, NTXDESC * sizeof ring->txd[0], 512, &pa); - if (ring->txd == NULL) - return -1; - comp_ring->txcd = vmxnet3_dma_allocmem(sc, - NTXCOMPDESC * sizeof comp_ring->txcd[0], 512, &comp_pa); - if (comp_ring->txcd == NULL) - return -1; + vmxnet3_free_interrupts(sc); - for (idx = 0; idx < NTXDESC; idx++) { - if (bus_dmamap_create(sc->sc_dmat, JUMBO_LEN, NTXSEGS, - JUMBO_LEN, 0, BUS_DMA_NOWAIT, &ring->dmap[idx])) - return -1; - } - - ts = tq->ts; - memset(ts, 0, sizeof *ts); - ts->npending = 0; - ts->intr_threshold = 1; - ts->cmd_ring = pa; - ts->cmd_ring_len = NTXDESC; - ts->comp_ring = comp_pa; - ts->comp_ring_len = NTXCOMPDESC; - ts->driver_data = vtophys(tq); - ts->driver_data_len = sizeof *tq; - ts->intr_idx = 0; - ts->stopped = 1; - ts->error = 0; - return 0; + vmxnet3_free_data(sc); + vmxnet3_free_pci_resources(sc); + vmxnet3_free_rxtx_queues(sc); + + if (sc->vmx_mtx) + mutex_obj_free(sc->vmx_mtx); + + return (0); } int -vmxnet3_alloc_rxring(struct vmxnet3_softc *sc, int queue) +vmxnet3_alloc_pci_resources(struct vmxnet3_softc *sc) { - struct vmxnet3_rxqueue *rq = &sc->sc_rxq[queue]; - struct vmxnet3_rxq_shared *rs; - struct vmxnet3_rxring *ring; - struct vmxnet3_comp_ring *comp_ring; - bus_addr_t pa[2], comp_pa; - int i, idx; - - for (i = 0; i < 2; i++) { - ring = &rq->cmd_ring[i]; - ring->rxd = vmxnet3_dma_allocmem(sc, NRXDESC * sizeof ring->rxd[0], - 512, &pa[i]); - if (ring->rxd == NULL) - return -1; - } - comp_ring = &rq->comp_ring; - comp_ring->rxcd = vmxnet3_dma_allocmem(sc, - NRXCOMPDESC * sizeof comp_ring->rxcd[0], 512, &comp_pa); - if (comp_ring->rxcd == NULL) - return -1; + struct pci_attach_args *pa = sc->vmx_pa; + pcireg_t memtype; - for (i = 0; i < 2; i++) { - ring = &rq->cmd_ring[i]; - ring->rid = i; - for (idx = 0; idx < NRXDESC; idx++) { - if (bus_dmamap_create(sc->sc_dmat, JUMBO_LEN, 1, - JUMBO_LEN, 0, BUS_DMA_NOWAIT, &ring->dmap[idx])) - return -1; - } + memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, PCI_BAR(0)); + if (pci_mapreg_map(pa, PCI_BAR(0), memtype, 0, &sc->vmx_iot0, &sc->vmx_ioh0, + NULL, &sc->vmx_ios0)) { + aprint_error_dev(sc->vmx_dev, "failed to map BAR0\n"); + return (ENXIO); + } + memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, PCI_BAR(1)); + if (pci_mapreg_map(pa, PCI_BAR(1), memtype, 0, &sc->vmx_iot1, &sc->vmx_ioh1, + NULL, &sc->vmx_ios1)) { + aprint_error_dev(sc->vmx_dev, "failed to map BAR1\n"); + return (ENXIO); + } + + if (!pci_get_capability(pa->pa_pc, pa->pa_tag, PCI_CAP_MSIX, NULL, NULL)) { + sc->vmx_flags |= VMXNET3_FLAG_NO_MSIX; + return (0); } - rs = rq->rs; - memset(rs, 0, sizeof *rs); - rs->cmd_ring[0] = pa[0]; - rs->cmd_ring[1] = pa[1]; - rs->cmd_ring_len[0] = NRXDESC; - rs->cmd_ring_len[1] = NRXDESC; - rs->comp_ring = comp_pa; - rs->comp_ring_len = NRXCOMPDESC; - rs->driver_data = vtophys(rq); - rs->driver_data_len = sizeof *rq; - rs->intr_idx = 0; - rs->stopped = 1; - rs->error = 0; - return 0; + return (0); } void -vmxnet3_txinit(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *tq) +vmxnet3_free_pci_resources(struct vmxnet3_softc *sc) { - struct vmxnet3_txring *ring = &tq->cmd_ring; - struct vmxnet3_comp_ring *comp_ring = &tq->comp_ring; - - ring->head = ring->next = 0; - ring->gen = 1; - comp_ring->next = 0; - comp_ring->gen = 1; - memset(ring->txd, 0, NTXDESC * sizeof ring->txd[0]); - memset(comp_ring->txcd, 0, NTXCOMPDESC * sizeof comp_ring->txcd[0]); -} -void -vmxnet3_rxinit(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rq) -{ - struct vmxnet3_rxring *ring; - struct vmxnet3_comp_ring *comp_ring; - int i, idx; + if (sc->vmx_ios0) { + bus_space_unmap(sc->vmx_iot0, sc->vmx_ioh0, sc->vmx_ios0); + sc->vmx_ios0 = 0; + } - for (i = 0; i < 2; i++) { - ring = &rq->cmd_ring[i]; - ring->fill = 0; - ring->gen = 1; - memset(ring->rxd, 0, NRXDESC * sizeof ring->rxd[0]); - for (idx = 0; idx < NRXDESC; idx++) { - if (vmxnet3_getbuf(sc, ring)) - break; - } + if (sc->vmx_ios1) { + bus_space_unmap(sc->vmx_iot1, sc->vmx_ioh1, sc->vmx_ios1); + sc->vmx_ios1 = 0; } - comp_ring = &rq->comp_ring; - comp_ring->next = 0; - comp_ring->gen = 1; - memset(comp_ring->rxcd, 0, NRXCOMPDESC * sizeof comp_ring->rxcd[0]); } -void -vmxnet3_txstop(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *tq) +int +vmxnet3_check_version(struct vmxnet3_softc *sc) { - struct vmxnet3_txring *ring = &tq->cmd_ring; - int idx; + u_int ver; - for (idx = 0; idx < NTXDESC; idx++) { - if (ring->m[idx]) { - bus_dmamap_unload(sc->sc_dmat, ring->dmap[idx]); - m_freem(ring->m[idx]); - ring->m[idx] = NULL; - } + ver = vmxnet3_read_bar1(sc, VMXNET3_BAR1_VRRS); + if ((ver & 0x1) == 0) { + aprint_error_dev(sc->vmx_dev, + "unsupported hardware version 0x%x\n", ver); + return (ENOTSUP); } -} - -void -vmxnet3_rxstop(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rq) -{ - struct vmxnet3_rxring *ring; - int i, idx; + vmxnet3_write_bar1(sc, VMXNET3_BAR1_VRRS, 1); - for (i = 0; i < 2; i++) { - ring = &rq->cmd_ring[i]; - for (idx = 0; idx < NRXDESC; idx++) { - if (ring->m[idx]) { - m_freem(ring->m[idx]); - ring->m[idx] = NULL; - } - } + ver = vmxnet3_read_bar1(sc, VMXNET3_BAR1_UVRS); + if ((ver & 0x1) == 0) { + aprint_error_dev(sc->vmx_dev, + "incompatiable UPT version 0x%x\n", ver); + return (ENOTSUP); } + vmxnet3_write_bar1(sc, VMXNET3_BAR1_UVRS, 1); + + return (0); } void -vmxnet3_link_state(struct vmxnet3_softc *sc) +vmxnet3_check_multiqueue(struct vmxnet3_softc *sc) { - struct ifnet *ifp = &sc->sc_ethercom.ec_if; - u_int x, link, speed; - WRITE_CMD(sc, VMXNET3_CMD_GET_LINK); - x = READ_BAR1(sc, VMXNET3_BAR1_CMD); - speed = x >> 16; - if (x & 1) { - ifp->if_baudrate = IF_Mbps(speed); - link = LINK_STATE_UP; - } else - link = LINK_STATE_DOWN; + if (sc->vmx_intr_type != VMXNET3_IT_MSIX) + goto out; - if_link_state_change(ifp, link); -} + /* Just use the maximum configured for now. */ + sc->vmx_nrxqueues = sc->vmx_max_nrxqueues; + sc->vmx_ntxqueues = sc->vmx_max_ntxqueues; -static inline void -vmxnet3_enable_intr(struct vmxnet3_softc *sc, int irq) -{ - WRITE_BAR0(sc, VMXNET3_BAR0_IMASK(irq), 0); + if (sc->vmx_nrxqueues > 1) + sc->vmx_flags |= VMXNET3_FLAG_RSS; + + return; + +out: + sc->vmx_ntxqueues = 1; + sc->vmx_nrxqueues = 1; } -static inline void -vmxnet3_disable_intr(struct vmxnet3_softc *sc, int irq) +int +vmxnet3_alloc_msix_interrupts(struct vmxnet3_softc *sc) { - WRITE_BAR0(sc, VMXNET3_BAR0_IMASK(irq), 1); + int required; + struct pci_attach_args *pa = sc->vmx_pa; + + if (sc->vmx_flags & VMXNET3_FLAG_NO_MSIX) + return (1); + + /* Allocate an additional vector for the events interrupt. */ + required = sc->vmx_max_nrxqueues + sc->vmx_max_ntxqueues + 1; + + if (pci_msix_count(pa->pa_pc, pa->pa_tag) < required) + return (1); + + if (pci_msix_alloc_exact(pa, &sc->vmx_intrs, required) == 0) { + sc->vmx_nintrs = required; + return (0); + } + + return (1); } -void -vmxnet3_enable_all_intrs(struct vmxnet3_softc *sc) +int +vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *sc) { - int i; + int nmsi, required; + struct pci_attach_args *pa = sc->vmx_pa; - sc->sc_ds->ictrl &= ~VMXNET3_ICTRL_DISABLE_ALL; - for (i = 0; i < VMXNET3_NINTR; i++) - vmxnet3_enable_intr(sc, i); + required = 1; + + nmsi = pci_msi_count(pa->pa_pc, pa->pa_tag); + if (nmsi < required) + return (1); + + if (pci_msi_alloc_exact(pa, &sc->vmx_intrs, required) == 0) { + sc->vmx_nintrs = required; + return (0); + } + + return (1); } -void -vmxnet3_disable_all_intrs(struct vmxnet3_softc *sc) +int +vmxnet3_alloc_legacy_interrupts(struct vmxnet3_softc *sc) { - int i; - sc->sc_ds->ictrl |= VMXNET3_ICTRL_DISABLE_ALL; - for (i = 0; i < VMXNET3_NINTR; i++) - vmxnet3_disable_intr(sc, i); + if (pci_intx_alloc(sc->vmx_pa, &sc->vmx_intrs) == 0) { + sc->vmx_nintrs = 1; + return (0); + } + + return (1); } int -vmxnet3_intr(void *arg) +vmxnet3_alloc_interrupts(struct vmxnet3_softc *sc) { - struct vmxnet3_softc *sc = arg; - struct ifnet *ifp = &sc->sc_ethercom.ec_if; + u_int config; + int error; + + config = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_INTRCFG); + + sc->vmx_intr_type = config & 0x03; + sc->vmx_intr_mask_mode = (config >> 2) & 0x03; + + switch (sc->vmx_intr_type) { + case VMXNET3_IT_AUTO: + sc->vmx_intr_type = VMXNET3_IT_MSIX; + /* FALLTHROUGH */ + case VMXNET3_IT_MSIX: + error = vmxnet3_alloc_msix_interrupts(sc); + if (error == 0) + break; + sc->vmx_intr_type = VMXNET3_IT_MSI; + /* FALLTHROUGH */ + case VMXNET3_IT_MSI: + error = vmxnet3_alloc_msi_interrupts(sc); + if (error == 0) + break; + sc->vmx_intr_type = VMXNET3_IT_LEGACY; + /* FALLTHROUGH */ + case VMXNET3_IT_LEGACY: + error = vmxnet3_alloc_legacy_interrupts(sc); + if (error == 0) + break; + /* FALLTHROUGH */ + default: + sc->vmx_intr_type = -1; + aprint_error_dev(sc->vmx_dev, "cannot allocate any interrupt resources\n"); + return (ENXIO); + } + + return (error); +} + +void +vmxnet3_free_interrupts(struct vmxnet3_softc *sc) +{ + pci_chipset_tag_t pc = sc->vmx_pa->pa_pc; + int i; + + for (i = 0; i < sc->vmx_nintrs; i++) { + pci_intr_disestablish(pc, sc->vmx_ihs[i]); + } + pci_intr_release(pc, sc->vmx_intrs, sc->vmx_nintrs); +} + +int +vmxnet3_setup_msix_interrupts(struct vmxnet3_softc *sc) +{ + pci_chipset_tag_t pc = sc->vmx_pa->pa_pc; + struct vmxnet3_txqueue *txq; + struct vmxnet3_rxqueue *rxq; + pci_intr_handle_t *intr; + void **ihs; + int intr_idx, i; + const char *intrstr; + char intrbuf[PCI_INTRSTR_LEN]; + char xnamebuf[32]; + + intr = sc->vmx_intrs; + intr_idx = 0; + ihs = sc->vmx_ihs; + + for (i = 0; i < sc->vmx_ntxqueues; i++, intr++, ihs++, intr_idx++) { + snprintf(xnamebuf, 32, "%s: tx %d", device_xname(sc->vmx_dev), i); + + txq = &sc->vmx_txq[i]; + + intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); + + pci_intr_setattr(pc, intr, PCI_INTR_MPSAFE, true); + *ihs = pci_intr_establish_xname(pc, *intr, IPL_NET, + vmxnet3_txq_intr, txq, xnamebuf); + if (*ihs == NULL) { + aprint_error_dev(sc->vmx_dev, + "unable to establish tx interrupt at %s\n", intrstr); + return (-1); + } + aprint_normal_dev(sc->vmx_dev, "tx interrupting at %s\n", intrstr); + + txq->vxtxq_intr_idx = intr_idx; + } + + for (i = 0; i < sc->vmx_nrxqueues; i++, intr++, ihs++, intr_idx++) { + snprintf(xnamebuf, 32, "%s: rx %d", device_xname(sc->vmx_dev), i); + + rxq = &sc->vmx_rxq[i]; + + intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); + + pci_intr_setattr(pc, intr, PCI_INTR_MPSAFE, true); + *ihs = pci_intr_establish_xname(pc, *intr, IPL_NET, + vmxnet3_rxq_intr, rxq, xnamebuf); + if (*ihs == NULL) { + aprint_error_dev(sc->vmx_dev, + "unable to establish rx interrupt at %s\n", intrstr); + return (-1); + } + aprint_normal_dev(sc->vmx_dev, "rx interrupting at %s\n", intrstr); + + rxq->vxrxq_intr_idx = intr_idx; + } + + intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); + + snprintf(xnamebuf, 32, "%s: link", device_xname(sc->vmx_dev)); + pci_intr_setattr(pc, intr, PCI_INTR_MPSAFE, true); + *ihs = pci_intr_establish_xname(pc, *intr, IPL_NET, + vmxnet3_event_intr, sc, xnamebuf); + if (*ihs == NULL) { + aprint_error_dev(sc->vmx_dev, + "unable to establish event interrupt at %s\n", intrstr); + return (-1); + } + aprint_normal_dev(sc->vmx_dev, "event interrupting at %s\n", intrstr); + + sc->vmx_event_intr_idx = intr_idx; + + return (0); +} + +int +vmxnet3_setup_msi_interrupt(struct vmxnet3_softc *sc) +{ + pci_chipset_tag_t pc = sc->vmx_pa->pa_pc; + pci_intr_handle_t *intr; + void **ihs; + int i; + const char *intrstr; + char intrbuf[PCI_INTRSTR_LEN]; + char xnamebuf[32]; + + intr = &sc->vmx_intrs[0]; + ihs = sc->vmx_ihs; + + intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); + + snprintf(xnamebuf, 32, "%s: msi", device_xname(sc->vmx_dev)); + pci_intr_setattr(pc, intr, PCI_INTR_MPSAFE, true); + *ihs = pci_intr_establish_xname(pc, *intr, IPL_NET, + vmxnet3_legacy_intr, sc, xnamebuf); + if (*ihs == NULL) { + aprint_error_dev(sc->vmx_dev, + "unable to establish interrupt at %s\n", intrstr); + return (-1); + } + aprint_normal_dev(sc->vmx_dev, "interrupting at %s\n", intrstr); + + for (i = 0; i < sc->vmx_ntxqueues; i++) + sc->vmx_txq[i].vxtxq_intr_idx = 0; + for (i = 0; i < sc->vmx_nrxqueues; i++) + sc->vmx_rxq[i].vxrxq_intr_idx = 0; + sc->vmx_event_intr_idx = 0; + + return (0); +} + +int +vmxnet3_setup_legacy_interrupt(struct vmxnet3_softc *sc) +{ + pci_chipset_tag_t pc = sc->vmx_pa->pa_pc; + pci_intr_handle_t *intr; + void **ihs; + int i; + const char *intrstr; + char intrbuf[PCI_INTRSTR_LEN]; + char xnamebuf[32]; + + intr = &sc->vmx_intrs[0]; + ihs = sc->vmx_ihs; + + intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); + + snprintf(xnamebuf, 32, "%s:legacy", device_xname(sc->vmx_dev)); + pci_intr_setattr(pc, intr, PCI_INTR_MPSAFE, true); + *ihs = pci_intr_establish_xname(pc, *intr, IPL_NET, + vmxnet3_legacy_intr, sc, xnamebuf); + if (*ihs == NULL) { + aprint_error_dev(sc->vmx_dev, + "unable to establish interrupt at %s\n", intrstr); + return (-1); + } + aprint_normal_dev(sc->vmx_dev, "interrupting at %s\n", intrstr); + + for (i = 0; i < sc->vmx_ntxqueues; i++) + sc->vmx_txq[i].vxtxq_intr_idx = 0; + for (i = 0; i < sc->vmx_nrxqueues; i++) + sc->vmx_rxq[i].vxrxq_intr_idx = 0; + sc->vmx_event_intr_idx = 0; + + return (0); +} + +void +vmxnet3_set_interrupt_idx(struct vmxnet3_softc *sc) +{ + struct vmxnet3_txqueue *txq; + struct vmxnet3_txq_shared *txs; + struct vmxnet3_rxqueue *rxq; + struct vmxnet3_rxq_shared *rxs; + int i; + + sc->vmx_ds->evintr = sc->vmx_event_intr_idx; + + for (i = 0; i < sc->vmx_ntxqueues; i++) { + txq = &sc->vmx_txq[i]; + txs = txq->vxtxq_ts; + txs->intr_idx = txq->vxtxq_intr_idx; + } + + for (i = 0; i < sc->vmx_nrxqueues; i++) { + rxq = &sc->vmx_rxq[i]; + rxs = rxq->vxrxq_rs; + rxs->intr_idx = rxq->vxrxq_intr_idx; + } +} + +int +vmxnet3_setup_interrupts(struct vmxnet3_softc *sc) +{ + int error; + + switch (sc->vmx_intr_type) { + case VMXNET3_IT_MSIX: + error = vmxnet3_setup_msix_interrupts(sc); + break; + case VMXNET3_IT_MSI: + error = vmxnet3_setup_msi_interrupt(sc); + break; + case VMXNET3_IT_LEGACY: + error = vmxnet3_setup_legacy_interrupt(sc); + break; + default: + panic("%s: invalid interrupt type %d", __func__, + sc->vmx_intr_type); + } + + if (error == 0) + vmxnet3_set_interrupt_idx(sc); + + return (error); +} + +int +vmxnet3_init_rxq(struct vmxnet3_softc *sc, int q) +{ + struct vmxnet3_rxqueue *rxq; + struct vmxnet3_rxring *rxr; + int i; + + rxq = &sc->vmx_rxq[q]; + + snprintf(rxq->vxrxq_name, sizeof(rxq->vxrxq_name), "%s-rx%d", + device_xname(sc->vmx_dev), q); + rxq->vxrxq_mtx = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET /* XXX */); + + rxq->vxrxq_sc = sc; + rxq->vxrxq_id = q; + + for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { + rxr = &rxq->vxrxq_cmd_ring[i]; + rxr->vxrxr_rid = i; + rxr->vxrxr_ndesc = sc->vmx_nrxdescs; + rxr->vxrxr_rxbuf = kmem_zalloc(rxr->vxrxr_ndesc * + sizeof(struct vmxnet3_rxbuf), KM_SLEEP); + if (rxr->vxrxr_rxbuf == NULL) + return (ENOMEM); + + rxq->vxrxq_comp_ring.vxcr_ndesc += sc->vmx_nrxdescs; + } + + return (0); +} + +int +vmxnet3_init_txq(struct vmxnet3_softc *sc, int q) +{ + struct vmxnet3_txqueue *txq; + struct vmxnet3_txring *txr; + + txq = &sc->vmx_txq[q]; + txr = &txq->vxtxq_cmd_ring; + + snprintf(txq->vxtxq_name, sizeof(txq->vxtxq_name), "%s-tx%d", + device_xname(sc->vmx_dev), q); + txq->vxtxq_mtx = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET /* XXX */); + + txq->vxtxq_sc = sc; + txq->vxtxq_id = q; + + txr->vxtxr_ndesc = sc->vmx_ntxdescs; + txr->vxtxr_txbuf = kmem_zalloc(txr->vxtxr_ndesc * + sizeof(struct vmxnet3_txbuf), KM_SLEEP); + if (txr->vxtxr_txbuf == NULL) + return (ENOMEM); + + txq->vxtxq_comp_ring.vxcr_ndesc = sc->vmx_ntxdescs; + + return (0); +} + +int +vmxnet3_alloc_rxtx_queues(struct vmxnet3_softc *sc) +{ + int i, error; + + KASSERT(!cpu_intr_p()); + KASSERT(!cpu_softintr_p()); + + /* + * Only attempt to create multiple queues if MSIX is available. + * This check prevents us from allocating queue structures that + * we will not use. + * + * FreeBSD: + * MSIX is disabled by default because its apparently broken for + * devices passed through by at least ESXi 5.1. + * The hw.pci.honor_msi_blacklist tunable must be set to zero for MSIX. + */ + if (sc->vmx_flags & VMXNET3_FLAG_NO_MSIX) { + sc->vmx_max_nrxqueues = 1; + sc->vmx_max_ntxqueues = 1; + } + + sc->vmx_rxq = kmem_zalloc( + sizeof(struct vmxnet3_rxqueue) * sc->vmx_max_nrxqueues, KM_SLEEP); + sc->vmx_txq = kmem_zalloc( + sizeof(struct vmxnet3_txqueue) * sc->vmx_max_ntxqueues, KM_SLEEP); + if (sc->vmx_rxq == NULL || sc->vmx_txq == NULL) + return (ENOMEM); + + for (i = 0; i < sc->vmx_max_nrxqueues; i++) { + error = vmxnet3_init_rxq(sc, i); + if (error) + return (error); + } + + for (i = 0; i < sc->vmx_max_ntxqueues; i++) { + error = vmxnet3_init_txq(sc, i); + if (error) + return (error); + } + + return (0); +} + +void +vmxnet3_destroy_rxq(struct vmxnet3_rxqueue *rxq) +{ + struct vmxnet3_rxring *rxr; + int i; + + rxq->vxrxq_sc = NULL; + rxq->vxrxq_id = -1; + + for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { + rxr = &rxq->vxrxq_cmd_ring[i]; + + if (rxr->vxrxr_rxbuf != NULL) { + kmem_free(rxr->vxrxr_rxbuf, + rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxbuf)); + rxr->vxrxr_rxbuf = NULL; + } + } + + if (rxq->vxrxq_mtx != NULL) + mutex_obj_free(rxq->vxrxq_mtx); +} + +void +vmxnet3_destroy_txq(struct vmxnet3_txqueue *txq) +{ + struct vmxnet3_txring *txr; + + txr = &txq->vxtxq_cmd_ring; + + txq->vxtxq_sc = NULL; + txq->vxtxq_id = -1; + + if (txr->vxtxr_txbuf != NULL) { + kmem_free(txr->vxtxr_txbuf, + txr->vxtxr_ndesc * sizeof(struct vmxnet3_txbuf)); + txr->vxtxr_txbuf = NULL; + } + + if (txq->vxtxq_mtx != NULL) + mutex_obj_free(txq->vxtxq_mtx); +} + +void +vmxnet3_free_rxtx_queues(struct vmxnet3_softc *sc) +{ + int i; + + if (sc->vmx_rxq != NULL) { + for (i = 0; i < sc->vmx_max_nrxqueues; i++) + vmxnet3_destroy_rxq(&sc->vmx_rxq[i]); + kmem_free(sc->vmx_rxq, + sizeof(struct vmxnet3_rxqueue) * sc->vmx_max_nrxqueues); + sc->vmx_rxq = NULL; + } + + if (sc->vmx_txq != NULL) { + for (i = 0; i < sc->vmx_max_ntxqueues; i++) + vmxnet3_destroy_txq(&sc->vmx_txq[i]); + kmem_free(sc->vmx_txq, + sizeof(struct vmxnet3_txqueue) * sc->vmx_max_ntxqueues); + sc->vmx_txq = NULL; + } +} + +int +vmxnet3_alloc_shared_data(struct vmxnet3_softc *sc) +{ + device_t dev; + uint8_t *kva; + size_t size; + int i, error; + + dev = sc->vmx_dev; + + size = sizeof(struct vmxnet3_driver_shared); + error = vmxnet3_dma_malloc(sc, size, 1, &sc->vmx_ds_dma); + if (error) { + device_printf(dev, "cannot alloc shared memory\n"); + return (error); + } + sc->vmx_ds = (struct vmxnet3_driver_shared *) sc->vmx_ds_dma.dma_vaddr; + + size = sc->vmx_ntxqueues * sizeof(struct vmxnet3_txq_shared) + + sc->vmx_nrxqueues * sizeof(struct vmxnet3_rxq_shared); + error = vmxnet3_dma_malloc(sc, size, 128, &sc->vmx_qs_dma); + if (error) { + device_printf(dev, "cannot alloc queue shared memory\n"); + return (error); + } + sc->vmx_qs = (void *) sc->vmx_qs_dma.dma_vaddr; + kva = sc->vmx_qs; + + for (i = 0; i < sc->vmx_ntxqueues; i++) { + sc->vmx_txq[i].vxtxq_ts = (struct vmxnet3_txq_shared *) kva; + kva += sizeof(struct vmxnet3_txq_shared); + } + for (i = 0; i < sc->vmx_nrxqueues; i++) { + sc->vmx_rxq[i].vxrxq_rs = (struct vmxnet3_rxq_shared *) kva; + kva += sizeof(struct vmxnet3_rxq_shared); + } + + if (sc->vmx_flags & VMXNET3_FLAG_RSS) { + size = sizeof(struct vmxnet3_rss_shared); + error = vmxnet3_dma_malloc(sc, size, 128, &sc->vmx_rss_dma); + if (error) { + device_printf(dev, "cannot alloc rss shared memory\n"); + return (error); + } + sc->vmx_rss = + (struct vmxnet3_rss_shared *) sc->vmx_rss_dma.dma_vaddr; + } + + return (0); +} + +void +vmxnet3_free_shared_data(struct vmxnet3_softc *sc) +{ + + if (sc->vmx_rss != NULL) { + vmxnet3_dma_free(sc, &sc->vmx_rss_dma); + sc->vmx_rss = NULL; + } + + if (sc->vmx_qs != NULL) { + vmxnet3_dma_free(sc, &sc->vmx_qs_dma); + sc->vmx_qs = NULL; + } + + if (sc->vmx_ds != NULL) { + vmxnet3_dma_free(sc, &sc->vmx_ds_dma); + sc->vmx_ds = NULL; + } +} + +int +vmxnet3_alloc_txq_data(struct vmxnet3_softc *sc) +{ + device_t dev; + struct vmxnet3_txqueue *txq; + struct vmxnet3_txring *txr; + struct vmxnet3_comp_ring *txc; + size_t descsz, compsz; + int i, q, error; + + dev = sc->vmx_dev; + + for (q = 0; q < sc->vmx_ntxqueues; q++) { + txq = &sc->vmx_txq[q]; + txr = &txq->vxtxq_cmd_ring; + txc = &txq->vxtxq_comp_ring; + + descsz = txr->vxtxr_ndesc * sizeof(struct vmxnet3_txdesc); + compsz = txr->vxtxr_ndesc * sizeof(struct vmxnet3_txcompdesc); + + error = vmxnet3_dma_malloc(sc, descsz, 512, &txr->vxtxr_dma); + if (error) { + device_printf(dev, "cannot alloc Tx descriptors for " + "queue %d error %d\n", q, error); + return (error); + } + txr->vxtxr_txd = + (struct vmxnet3_txdesc *) txr->vxtxr_dma.dma_vaddr; + + error = vmxnet3_dma_malloc(sc, compsz, 512, &txc->vxcr_dma); + if (error) { + device_printf(dev, "cannot alloc Tx comp descriptors " + "for queue %d error %d\n", q, error); + return (error); + } + txc->vxcr_u.txcd = + (struct vmxnet3_txcompdesc *) txc->vxcr_dma.dma_vaddr; + + for (i = 0; i < txr->vxtxr_ndesc; i++) { + error = bus_dmamap_create(sc->vmx_dmat, VMXNET3_TX_MAXSIZE, + VMXNET3_TX_MAXSEGS, VMXNET3_TX_MAXSEGSIZE, 0, BUS_DMA_NOWAIT, + &txr->vxtxr_txbuf[i].vtxb_dmamap); + if (error) { + device_printf(dev, "unable to create Tx buf " + "dmamap for queue %d idx %d\n", q, i); + return (error); + } + } + } + + return (0); +} + +void +vmxnet3_free_txq_data(struct vmxnet3_softc *sc) +{ + struct vmxnet3_txqueue *txq; + struct vmxnet3_txring *txr; + struct vmxnet3_comp_ring *txc; + struct vmxnet3_txbuf *txb; + int i, q; + + for (q = 0; q < sc->vmx_ntxqueues; q++) { + txq = &sc->vmx_txq[q]; + txr = &txq->vxtxq_cmd_ring; + txc = &txq->vxtxq_comp_ring; + + for (i = 0; i < txr->vxtxr_ndesc; i++) { + txb = &txr->vxtxr_txbuf[i]; + if (txb->vtxb_dmamap != NULL) { + bus_dmamap_destroy(sc->vmx_dmat, + txb->vtxb_dmamap); + txb->vtxb_dmamap = NULL; + } + } + + if (txc->vxcr_u.txcd != NULL) { + vmxnet3_dma_free(sc, &txc->vxcr_dma); + txc->vxcr_u.txcd = NULL; + } + + if (txr->vxtxr_txd != NULL) { + vmxnet3_dma_free(sc, &txr->vxtxr_dma); + txr->vxtxr_txd = NULL; + } + } +} + +int +vmxnet3_alloc_rxq_data(struct vmxnet3_softc *sc) +{ + device_t dev; + struct vmxnet3_rxqueue *rxq; + struct vmxnet3_rxring *rxr; + struct vmxnet3_comp_ring *rxc; + int descsz, compsz; + int i, j, q, error; + + dev = sc->vmx_dev; + + for (q = 0; q < sc->vmx_nrxqueues; q++) { + rxq = &sc->vmx_rxq[q]; + rxc = &rxq->vxrxq_comp_ring; + compsz = 0; + + for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { + rxr = &rxq->vxrxq_cmd_ring[i]; + + descsz = rxr->vxrxr_ndesc * + sizeof(struct vmxnet3_rxdesc); + compsz += rxr->vxrxr_ndesc * + sizeof(struct vmxnet3_rxcompdesc); + + error = vmxnet3_dma_malloc(sc, descsz, 512, + &rxr->vxrxr_dma); + if (error) { + device_printf(dev, "cannot allocate Rx " + "descriptors for queue %d/%d error %d\n", + i, q, error); + return (error); + } + rxr->vxrxr_rxd = + (struct vmxnet3_rxdesc *) rxr->vxrxr_dma.dma_vaddr; + } + + error = vmxnet3_dma_malloc(sc, compsz, 512, &rxc->vxcr_dma); + if (error) { + device_printf(dev, "cannot alloc Rx comp descriptors " + "for queue %d error %d\n", q, error); + return (error); + } + rxc->vxcr_u.rxcd = + (struct vmxnet3_rxcompdesc *) rxc->vxcr_dma.dma_vaddr; + + for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { + rxr = &rxq->vxrxq_cmd_ring[i]; + + error = bus_dmamap_create(sc->vmx_dmat, JUMBO_LEN, 1, + JUMBO_LEN, 0, BUS_DMA_NOWAIT, + &rxr->vxrxr_spare_dmap); + if (error) { + device_printf(dev, "unable to create spare " + "dmamap for queue %d/%d error %d\n", + q, i, error); + return (error); + } + + for (j = 0; j < rxr->vxrxr_ndesc; j++) { + error = bus_dmamap_create(sc->vmx_dmat, JUMBO_LEN, 1, + JUMBO_LEN, 0, BUS_DMA_NOWAIT, + &rxr->vxrxr_rxbuf[j].vrxb_dmamap); + if (error) { + device_printf(dev, "unable to create " + "dmamap for queue %d/%d slot %d " + "error %d\n", + q, i, j, error); + return (error); + } + } + } + } + + return (0); +} + +void +vmxnet3_free_rxq_data(struct vmxnet3_softc *sc) +{ + struct vmxnet3_rxqueue *rxq; + struct vmxnet3_rxring *rxr; + struct vmxnet3_comp_ring *rxc; + struct vmxnet3_rxbuf *rxb; + int i, j, q; + + for (q = 0; q < sc->vmx_nrxqueues; q++) { + rxq = &sc->vmx_rxq[q]; + rxc = &rxq->vxrxq_comp_ring; + + for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { + rxr = &rxq->vxrxq_cmd_ring[i]; + + if (rxr->vxrxr_spare_dmap != NULL) { + bus_dmamap_destroy(sc->vmx_dmat, + rxr->vxrxr_spare_dmap); + rxr->vxrxr_spare_dmap = NULL; + } + + for (j = 0; j < rxr->vxrxr_ndesc; j++) { + rxb = &rxr->vxrxr_rxbuf[j]; + if (rxb->vrxb_dmamap != NULL) { + bus_dmamap_destroy(sc->vmx_dmat, + rxb->vrxb_dmamap); + rxb->vrxb_dmamap = NULL; + } + } + } + + if (rxc->vxcr_u.rxcd != NULL) { + vmxnet3_dma_free(sc, &rxc->vxcr_dma); + rxc->vxcr_u.rxcd = NULL; + } + + for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { + rxr = &rxq->vxrxq_cmd_ring[i]; + + if (rxr->vxrxr_rxd != NULL) { + vmxnet3_dma_free(sc, &rxr->vxrxr_dma); + rxr->vxrxr_rxd = NULL; + } + } + } +} + +int +vmxnet3_alloc_queue_data(struct vmxnet3_softc *sc) +{ + int error; + + error = vmxnet3_alloc_txq_data(sc); + if (error) + return (error); + + error = vmxnet3_alloc_rxq_data(sc); + if (error) + return (error); + + return (0); +} + +void +vmxnet3_free_queue_data(struct vmxnet3_softc *sc) +{ + + if (sc->vmx_rxq != NULL) + vmxnet3_free_rxq_data(sc); + + if (sc->vmx_txq != NULL) + vmxnet3_free_txq_data(sc); +} + +int +vmxnet3_alloc_mcast_table(struct vmxnet3_softc *sc) +{ + int error; + + error = vmxnet3_dma_malloc(sc, VMXNET3_MULTICAST_MAX * ETHER_ADDR_LEN, + 32, &sc->vmx_mcast_dma); + if (error) + device_printf(sc->vmx_dev, "unable to alloc multicast table\n"); + else + sc->vmx_mcast = sc->vmx_mcast_dma.dma_vaddr; + + return (error); +} + +void +vmxnet3_free_mcast_table(struct vmxnet3_softc *sc) +{ + + if (sc->vmx_mcast != NULL) { + vmxnet3_dma_free(sc, &sc->vmx_mcast_dma); + sc->vmx_mcast = NULL; + } +} + +void +vmxnet3_init_shared_data(struct vmxnet3_softc *sc) +{ + struct vmxnet3_driver_shared *ds; + struct vmxnet3_txqueue *txq; + struct vmxnet3_txq_shared *txs; + struct vmxnet3_rxqueue *rxq; + struct vmxnet3_rxq_shared *rxs; + int i; + + ds = sc->vmx_ds; + + /* + * Initialize fields of the shared data that remains the same across + * reinits. Note the shared data is zero'd when allocated. + */ + + ds->magic = VMXNET3_REV1_MAGIC; + + /* DriverInfo */ + ds->version = VMXNET3_DRIVER_VERSION; + ds->guest = VMXNET3_GOS_FREEBSD | +#ifdef __LP64__ + VMXNET3_GOS_64BIT; +#else + VMXNET3_GOS_32BIT; +#endif + ds->vmxnet3_revision = 1; + ds->upt_version = 1; + + /* Misc. conf */ + ds->driver_data = vtophys(sc); + ds->driver_data_len = sizeof(struct vmxnet3_softc); + ds->queue_shared = sc->vmx_qs_dma.dma_paddr; + ds->queue_shared_len = sc->vmx_qs_dma.dma_size; + ds->nrxsg_max = sc->vmx_max_rxsegs; + + /* RSS conf */ + if (sc->vmx_flags & VMXNET3_FLAG_RSS) { + ds->rss.version = 1; + ds->rss.paddr = sc->vmx_rss_dma.dma_paddr; + ds->rss.len = sc->vmx_rss_dma.dma_size; + } + + /* Interrupt control. */ + ds->automask = sc->vmx_intr_mask_mode == VMXNET3_IMM_AUTO; + ds->nintr = sc->vmx_nintrs; + ds->evintr = sc->vmx_event_intr_idx; + ds->ictrl = VMXNET3_ICTRL_DISABLE_ALL; + + for (i = 0; i < sc->vmx_nintrs; i++) + ds->modlevel[i] = UPT1_IMOD_ADAPTIVE; + + /* Receive filter. */ + ds->mcast_table = sc->vmx_mcast_dma.dma_paddr; + ds->mcast_tablelen = sc->vmx_mcast_dma.dma_size; + + /* Tx queues */ + for (i = 0; i < sc->vmx_ntxqueues; i++) { + txq = &sc->vmx_txq[i]; + txs = txq->vxtxq_ts; + + txs->cmd_ring = txq->vxtxq_cmd_ring.vxtxr_dma.dma_paddr; + txs->cmd_ring_len = txq->vxtxq_cmd_ring.vxtxr_ndesc; + txs->comp_ring = txq->vxtxq_comp_ring.vxcr_dma.dma_paddr; + txs->comp_ring_len = txq->vxtxq_comp_ring.vxcr_ndesc; + txs->driver_data = vtophys(txq); + txs->driver_data_len = sizeof(struct vmxnet3_txqueue); + } + + /* Rx queues */ + for (i = 0; i < sc->vmx_nrxqueues; i++) { + rxq = &sc->vmx_rxq[i]; + rxs = rxq->vxrxq_rs; + + rxs->cmd_ring[0] = rxq->vxrxq_cmd_ring[0].vxrxr_dma.dma_paddr; + rxs->cmd_ring_len[0] = rxq->vxrxq_cmd_ring[0].vxrxr_ndesc; + rxs->cmd_ring[1] = rxq->vxrxq_cmd_ring[1].vxrxr_dma.dma_paddr; + rxs->cmd_ring_len[1] = rxq->vxrxq_cmd_ring[1].vxrxr_ndesc; + rxs->comp_ring = rxq->vxrxq_comp_ring.vxcr_dma.dma_paddr; + rxs->comp_ring_len = rxq->vxrxq_comp_ring.vxcr_ndesc; + rxs->driver_data = vtophys(rxq); + rxs->driver_data_len = sizeof(struct vmxnet3_rxqueue); + } +} + +void +vmxnet3_reinit_rss_shared_data(struct vmxnet3_softc *sc) +{ + /* + * Use the same key as the Linux driver until FreeBSD can do + * RSS (presumably Toeplitz) in software. + */ + static const uint8_t rss_key[UPT1_RSS_MAX_KEY_SIZE] = { + 0x3b, 0x56, 0xd1, 0x56, 0x13, 0x4a, 0xe7, 0xac, + 0xe8, 0x79, 0x09, 0x75, 0xe8, 0x65, 0x79, 0x28, + 0x35, 0x12, 0xb9, 0x56, 0x7c, 0x76, 0x4b, 0x70, + 0xd8, 0x56, 0xa3, 0x18, 0x9b, 0x0a, 0xee, 0xf3, + 0x96, 0xa6, 0x9f, 0x8f, 0x9e, 0x8c, 0x90, 0xc9, + }; + + struct vmxnet3_rss_shared *rss; + int i; + + rss = sc->vmx_rss; + + rss->hash_type = + UPT1_RSS_HASH_TYPE_IPV4 | UPT1_RSS_HASH_TYPE_TCP_IPV4 | + UPT1_RSS_HASH_TYPE_IPV6 | UPT1_RSS_HASH_TYPE_TCP_IPV6; + rss->hash_func = UPT1_RSS_HASH_FUNC_TOEPLITZ; + rss->hash_key_size = UPT1_RSS_MAX_KEY_SIZE; + rss->ind_table_size = UPT1_RSS_MAX_IND_TABLE_SIZE; + memcpy(rss->hash_key, rss_key, UPT1_RSS_MAX_KEY_SIZE); + + for (i = 0; i < UPT1_RSS_MAX_IND_TABLE_SIZE; i++) + rss->ind_table[i] = i % sc->vmx_nrxqueues; +} + +void +vmxnet3_reinit_shared_data(struct vmxnet3_softc *sc) +{ + struct ifnet *ifp; + struct vmxnet3_driver_shared *ds; + + ifp = &sc->vmx_ethercom.ec_if; + ds = sc->vmx_ds; + + ds->mtu = ifp->if_mtu; + ds->ntxqueue = sc->vmx_ntxqueues; + ds->nrxqueue = sc->vmx_nrxqueues; + + ds->upt_features = 0; + if (ifp->if_capenable & + (IFCAP_CSUM_IPv4_Rx | IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_UDPv4_Rx | + IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_UDPv6_Rx)) + ds->upt_features |= UPT1_F_CSUM; + if (sc->vmx_ethercom.ec_capenable & ETHERCAP_VLAN_HWTAGGING) + ds->upt_features |= UPT1_F_VLAN; + + if (sc->vmx_flags & VMXNET3_FLAG_RSS) { + ds->upt_features |= UPT1_F_RSS; + vmxnet3_reinit_rss_shared_data(sc); + } + + vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSL, sc->vmx_ds_dma.dma_paddr); + vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSH, + (uint64_t) sc->vmx_ds_dma.dma_paddr >> 32); +} + +int +vmxnet3_alloc_data(struct vmxnet3_softc *sc) +{ + int error; + + error = vmxnet3_alloc_shared_data(sc); + if (error) + return (error); + + error = vmxnet3_alloc_queue_data(sc); + if (error) + return (error); + + error = vmxnet3_alloc_mcast_table(sc); + if (error) + return (error); + + vmxnet3_init_shared_data(sc); + + return (0); +} + +void +vmxnet3_free_data(struct vmxnet3_softc *sc) +{ + + vmxnet3_free_mcast_table(sc); + vmxnet3_free_queue_data(sc); + vmxnet3_free_shared_data(sc); +} + +int +vmxnet3_setup_interface(struct vmxnet3_softc *sc) +{ + struct ifnet *ifp = &sc->vmx_ethercom.ec_if; + + vmxnet3_get_lladdr(sc); + aprint_normal_dev(sc->vmx_dev, "Ethernet address %s\n", + ether_sprintf(sc->vmx_lladdr)); + vmxnet3_set_lladdr(sc); + + strlcpy(ifp->if_xname, device_xname(sc->vmx_dev), IFNAMSIZ); + ifp->if_softc = sc; + ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST | IFF_SIMPLEX; + ifp->if_ioctl = vmxnet3_ioctl; + ifp->if_start = vmxnet3_start; + ifp->if_watchdog = NULL; + ifp->if_init = vmxnet3_init; + ifp->if_stop = vmxnet3_stop; + sc->vmx_ethercom.ec_if.if_capabilities |=IFCAP_CSUM_IPv4_Rx | + IFCAP_CSUM_TCPv4_Tx | IFCAP_CSUM_TCPv4_Rx | + IFCAP_CSUM_UDPv4_Tx | IFCAP_CSUM_UDPv4_Rx | + IFCAP_CSUM_TCPv6_Tx | IFCAP_CSUM_TCPv6_Rx | + IFCAP_CSUM_UDPv6_Tx | IFCAP_CSUM_UDPv6_Rx; + + ifp->if_capenable = ifp->if_capabilities; + + sc->vmx_ethercom.ec_if.if_capabilities |= IFCAP_TSOv4 | IFCAP_TSOv6; + + sc->vmx_ethercom.ec_capabilities |= ETHERCAP_VLAN_MTU | ETHERCAP_VLAN_HWTAGGING; + + IFQ_SET_MAXLEN(&ifp->if_snd, sc->vmx_ntxdescs); + IFQ_SET_READY(&ifp->if_snd); + + ifmedia_init(&sc->vmx_media, IFM_IMASK, vmxnet3_media_change, + vmxnet3_media_status); + ifmedia_add(&sc->vmx_media, IFM_ETHER|IFM_AUTO, 0, NULL); + ifmedia_add(&sc->vmx_media, IFM_ETHER|IFM_10G_T|IFM_FDX, 0, NULL); + ifmedia_add(&sc->vmx_media, IFM_ETHER|IFM_10G_T, 0, NULL); + ifmedia_add(&sc->vmx_media, IFM_ETHER|IFM_1000_T|IFM_FDX, 0, NULL); + ifmedia_add(&sc->vmx_media, IFM_ETHER|IFM_1000_T, 0, NULL); + ifmedia_set(&sc->vmx_media, IFM_ETHER|IFM_AUTO); + + if_attach(ifp); + ether_ifattach(ifp, sc->vmx_lladdr); + ether_set_ifflags_cb(&sc->vmx_ethercom, vmxnet3_ifflags_cb); + vmxnet3_link_status(sc); + + return (0); +} + +void +vmxnet3_evintr(struct vmxnet3_softc *sc) +{ + device_t dev; + struct ifnet *ifp; + struct vmxnet3_txq_shared *ts; + struct vmxnet3_rxq_shared *rs; + uint32_t event; + int reset; + + dev = sc->vmx_dev; + ifp = &sc->vmx_ethercom.ec_if; + reset = 0; + + VMXNET3_CORE_LOCK(sc); + + /* Clear events. */ + event = sc->vmx_ds->event; + vmxnet3_write_bar1(sc, VMXNET3_BAR1_EVENT, event); + + if (event & VMXNET3_EVENT_LINK) { + vmxnet3_link_status(sc); + if (sc->vmx_link_active != 0) + vmxnet3_start(&sc->vmx_ethercom.ec_if); + } + + if (event & (VMXNET3_EVENT_TQERROR | VMXNET3_EVENT_RQERROR)) { + reset = 1; + vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_STATUS); + ts = sc->vmx_txq[0].vxtxq_ts; + if (ts->stopped != 0) + device_printf(dev, "Tx queue error %#x\n", ts->error); + rs = sc->vmx_rxq[0].vxrxq_rs; + if (rs->stopped != 0) + device_printf(dev, "Rx queue error %#x\n", rs->error); + device_printf(dev, "Rx/Tx queue error event ... resetting\n"); + } + + if (event & VMXNET3_EVENT_DIC) + device_printf(dev, "device implementation change event\n"); + if (event & VMXNET3_EVENT_DEBUG) + device_printf(dev, "debug event\n"); + + if (reset != 0) { + ifp->if_flags &= ~IFF_RUNNING; + vmxnet3_init_locked(sc); + } + + VMXNET3_CORE_UNLOCK(sc); +} + +void +vmxnet3_txq_eof(struct vmxnet3_txqueue *txq) +{ + struct vmxnet3_softc *sc; + struct vmxnet3_txring *txr; + struct vmxnet3_comp_ring *txc; + struct vmxnet3_txcompdesc *txcd; + struct vmxnet3_txbuf *txb; + struct mbuf *m; + u_int sop; + + sc = txq->vxtxq_sc; + txr = &txq->vxtxq_cmd_ring; + txc = &txq->vxtxq_comp_ring; + + VMXNET3_TXQ_LOCK_ASSERT(txq); + + for (;;) { + txcd = &txc->vxcr_u.txcd[txc->vxcr_next]; + if (txcd->gen != txc->vxcr_gen) + break; + vmxnet3_barrier(sc, VMXNET3_BARRIER_RD); + + if (++txc->vxcr_next == txc->vxcr_ndesc) { + txc->vxcr_next = 0; + txc->vxcr_gen ^= 1; + } + + sop = txr->vxtxr_next; + txb = &txr->vxtxr_txbuf[sop]; + + if ((m = txb->vtxb_m) != NULL) { + bus_dmamap_sync(sc->vmx_dmat, txb->vtxb_dmamap, + 0, txb->vtxb_dmamap->dm_mapsize, + BUS_DMASYNC_POSTWRITE); + bus_dmamap_unload(sc->vmx_dmat, txb->vtxb_dmamap); + + txq->vxtxq_stats.vmtxs_opackets++; + txq->vxtxq_stats.vmtxs_obytes += m->m_pkthdr.len; + if (m->m_flags & M_MCAST) + txq->vxtxq_stats.vmtxs_omcasts++; + + m_freem(m); + txb->vtxb_m = NULL; + } + + txr->vxtxr_next = (txcd->eop_idx + 1) % txr->vxtxr_ndesc; + } + + if (txr->vxtxr_head == txr->vxtxr_next) + txq->vxtxq_watchdog = 0; +} + +int +vmxnet3_newbuf(struct vmxnet3_softc *sc, struct vmxnet3_rxring *rxr) +{ + struct mbuf *m; + struct vmxnet3_rxdesc *rxd; + struct vmxnet3_rxbuf *rxb; + bus_dma_tag_t tag; + bus_dmamap_t dmap; + int idx, btype, error; + + tag = sc->vmx_dmat; + dmap = rxr->vxrxr_spare_dmap; + idx = rxr->vxrxr_fill; + rxd = &rxr->vxrxr_rxd[idx]; + rxb = &rxr->vxrxr_rxbuf[idx]; + + /* Don't allocate buffers for ring 2 for now. */ + if (rxr->vxrxr_rid != 0) + return -1; + btype = VMXNET3_BTYPE_HEAD; + + MGETHDR(m, M_DONTWAIT, MT_DATA); + if (m == NULL) + return (ENOBUFS); + + MCLGET(m, M_DONTWAIT); + if ((m->m_flags & M_EXT) == 0) { + sc->vmx_stats.vmst_mgetcl_failed++; + m_freem(m); + return (ENOBUFS); + } + + m->m_pkthdr.len = m->m_len = JUMBO_LEN; + m_adj(m, ETHER_ALIGN); + + error = bus_dmamap_load_mbuf(sc->vmx_dmat, dmap, m, BUS_DMA_NOWAIT); + if (error) { + m_freem(m); + sc->vmx_stats.vmst_mbuf_load_failed++; + return (error); + } + + if (rxb->vrxb_m != NULL) { + bus_dmamap_sync(tag, rxb->vrxb_dmamap, + 0, rxb->vrxb_dmamap->dm_mapsize, + BUS_DMASYNC_POSTREAD); + bus_dmamap_unload(tag, rxb->vrxb_dmamap); + } + + rxr->vxrxr_spare_dmap = rxb->vrxb_dmamap; + rxb->vrxb_dmamap = dmap; + rxb->vrxb_m = m; + + rxd->addr = DMAADDR(dmap); + rxd->len = m->m_pkthdr.len; + rxd->btype = btype; + rxd->gen = rxr->vxrxr_gen; + + vmxnet3_rxr_increment_fill(rxr); + return (0); +} + + + +void +vmxnet3_rxq_eof_discard(struct vmxnet3_rxqueue *rxq, + struct vmxnet3_rxring *rxr, int idx) +{ + struct vmxnet3_rxdesc *rxd; + + rxd = &rxr->vxrxr_rxd[idx]; + rxd->gen = rxr->vxrxr_gen; + vmxnet3_rxr_increment_fill(rxr); +} + +void +vmxnet3_rxq_discard_chain(struct vmxnet3_rxqueue *rxq) +{ + struct vmxnet3_softc *sc; + struct vmxnet3_rxring *rxr; + struct vmxnet3_comp_ring *rxc; + struct vmxnet3_rxcompdesc *rxcd; + int idx, eof; + + sc = rxq->vxrxq_sc; + rxc = &rxq->vxrxq_comp_ring; + + do { + rxcd = &rxc->vxcr_u.rxcd[rxc->vxcr_next]; + if (rxcd->gen != rxc->vxcr_gen) + break; /* Not expected. */ + vmxnet3_barrier(sc, VMXNET3_BARRIER_RD); + + if (++rxc->vxcr_next == rxc->vxcr_ndesc) { + rxc->vxcr_next = 0; + rxc->vxcr_gen ^= 1; + } + + idx = rxcd->rxd_idx; + eof = rxcd->eop; + if (rxcd->qid < sc->vmx_nrxqueues) + rxr = &rxq->vxrxq_cmd_ring[0]; + else + rxr = &rxq->vxrxq_cmd_ring[1]; + vmxnet3_rxq_eof_discard(rxq, rxr, idx); + } while (!eof); +} + +void +vmxnet3_rx_csum(struct vmxnet3_rxcompdesc *rxcd, struct mbuf *m) +{ + if (rxcd->no_csum) + return; + + if (rxcd->ipv4) { + m->m_pkthdr.csum_flags |= M_CSUM_IPv4; + if (rxcd->ipcsum_ok == 0) + m->m_pkthdr.csum_flags |= M_CSUM_IPv4_BAD; + } + + if (rxcd->fragment) + return; + + if (rxcd->tcp) { + m->m_pkthdr.csum_flags |= + rxcd->ipv4 ? M_CSUM_TCPv4 : M_CSUM_TCPv6; + if ((rxcd->csum_ok) == 0) + m->m_pkthdr.csum_flags |= M_CSUM_TCP_UDP_BAD; + } + + if (rxcd->udp) { + m->m_pkthdr.csum_flags |= + rxcd->ipv4 ? M_CSUM_UDPv4 : M_CSUM_UDPv6 ; + if ((rxcd->csum_ok) == 0) + m->m_pkthdr.csum_flags |= M_CSUM_TCP_UDP_BAD; + } +} + +void +vmxnet3_rxq_input(struct vmxnet3_rxqueue *rxq, + struct vmxnet3_rxcompdesc *rxcd, struct mbuf *m) +{ + struct vmxnet3_softc *sc; + struct ifnet *ifp; + + sc = rxq->vxrxq_sc; + ifp = &sc->vmx_ethercom.ec_if; + + if (rxcd->error) { + rxq->vxrxq_stats.vmrxs_ierrors++; + m_freem(m); + return; + } + + if (!rxcd->no_csum) + vmxnet3_rx_csum(rxcd, m); + if (rxcd->vlan) { + VLAN_INPUT_TAG(ifp, m, rxcd->vtag, + rxq->vxrxq_stats.vmrxs_ierrors++; + m_freem(m); + return); + } + + rxq->vxrxq_stats.vmrxs_ipackets++; + rxq->vxrxq_stats.vmrxs_ibytes += m->m_pkthdr.len; + + if_percpuq_enqueue(ifp->if_percpuq, m); +} + +void +vmxnet3_rxq_eof(struct vmxnet3_rxqueue *rxq) +{ + struct vmxnet3_softc *sc; + struct ifnet *ifp; + struct vmxnet3_rxring *rxr; + struct vmxnet3_comp_ring *rxc; + struct vmxnet3_rxdesc *rxd; + struct vmxnet3_rxcompdesc *rxcd; + struct mbuf *m, *m_head, *m_tail; + int idx, length; + + sc = rxq->vxrxq_sc; + ifp = &sc->vmx_ethercom.ec_if; + rxc = &rxq->vxrxq_comp_ring; + + VMXNET3_RXQ_LOCK_ASSERT(rxq); if ((ifp->if_flags & IFF_RUNNING) == 0) - return 0; - if (READ_BAR1(sc, VMXNET3_BAR1_INTR) == 0) - return 0; - if (sc->sc_ds->event) + return; + + m_head = rxq->vxrxq_mhead; + rxq->vxrxq_mhead = NULL; + m_tail = rxq->vxrxq_mtail; + rxq->vxrxq_mtail = NULL; + KASSERT(m_head == NULL || m_tail != NULL); + + for (;;) { + rxcd = &rxc->vxcr_u.rxcd[rxc->vxcr_next]; + if (rxcd->gen != rxc->vxcr_gen) { + rxq->vxrxq_mhead = m_head; + rxq->vxrxq_mtail = m_tail; + break; + } + vmxnet3_barrier(sc, VMXNET3_BARRIER_RD); + + if (++rxc->vxcr_next == rxc->vxcr_ndesc) { + rxc->vxcr_next = 0; + rxc->vxcr_gen ^= 1; + } + + idx = rxcd->rxd_idx; + length = rxcd->len; + if (rxcd->qid < sc->vmx_nrxqueues) + rxr = &rxq->vxrxq_cmd_ring[0]; + else + rxr = &rxq->vxrxq_cmd_ring[1]; + rxd = &rxr->vxrxr_rxd[idx]; + + m = rxr->vxrxr_rxbuf[idx].vrxb_m; + KASSERT(m != NULL); + + /* + * The host may skip descriptors. We detect this when this + * descriptor does not match the previous fill index. Catch + * up with the host now. + */ + if (__predict_false(rxr->vxrxr_fill != idx)) { + while (rxr->vxrxr_fill != idx) { + rxr->vxrxr_rxd[rxr->vxrxr_fill].gen = + rxr->vxrxr_gen; + vmxnet3_rxr_increment_fill(rxr); + } + } + + if (rxcd->sop) { + /* start of frame w/o head buffer */ + KASSERT(rxd->btype == VMXNET3_BTYPE_HEAD); + /* start of frame not in ring 0 */ + KASSERT(rxr == &rxq->vxrxq_cmd_ring[0]); + /* duplicate start of frame? */ + KASSERT(m_head == NULL); + + if (length == 0) { + /* Just ignore this descriptor. */ + vmxnet3_rxq_eof_discard(rxq, rxr, idx); + goto nextp; + } + + if (vmxnet3_newbuf(sc, rxr) != 0) { + rxq->vxrxq_stats.vmrxs_iqdrops++; + vmxnet3_rxq_eof_discard(rxq, rxr, idx); + if (!rxcd->eop) + vmxnet3_rxq_discard_chain(rxq); + goto nextp; + } + + m_set_rcvif(m, ifp); + m->m_pkthdr.len = m->m_len = length; + m->m_pkthdr.csum_flags = 0; + m_head = m_tail = m; + + } else { + /* non start of frame w/o body buffer */ + KASSERT(rxd->btype == VMXNET3_BTYPE_BODY); + /* frame not started? */ + KASSERT(m_head != NULL); + + if (vmxnet3_newbuf(sc, rxr) != 0) { + rxq->vxrxq_stats.vmrxs_iqdrops++; + vmxnet3_rxq_eof_discard(rxq, rxr, idx); + if (!rxcd->eop) + vmxnet3_rxq_discard_chain(rxq); + m_freem(m_head); + m_head = m_tail = NULL; + goto nextp; + } + + m->m_len = length; + m_head->m_pkthdr.len += length; + m_tail->m_next = m; + m_tail = m; + } + + if (rxcd->eop) { + vmxnet3_rxq_input(rxq, rxcd, m_head); + m_head = m_tail = NULL; + + /* Must recheck after dropping the Rx lock. */ + if ((ifp->if_flags & IFF_RUNNING) == 0) + break; + } + +nextp: + if (__predict_false(rxq->vxrxq_rs->update_rxhead)) { + int qid = rxcd->qid; + bus_size_t r; + + idx = (idx + 1) % rxr->vxrxr_ndesc; + if (qid >= sc->vmx_nrxqueues) { + qid -= sc->vmx_nrxqueues; + r = VMXNET3_BAR0_RXH2(qid); + } else + r = VMXNET3_BAR0_RXH1(qid); + vmxnet3_write_bar0(sc, r, idx); + } + } +} + +int +vmxnet3_legacy_intr(void *xsc) +{ + struct vmxnet3_softc *sc; + struct vmxnet3_rxqueue *rxq; + struct vmxnet3_txqueue *txq; + + sc = xsc; + rxq = &sc->vmx_rxq[0]; + txq = &sc->vmx_txq[0]; + + if (sc->vmx_intr_type == VMXNET3_IT_LEGACY) { + if (vmxnet3_read_bar1(sc, VMXNET3_BAR1_INTR) == 0) + return (0); + } + if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) + vmxnet3_disable_all_intrs(sc); + + if (sc->vmx_ds->event != 0) vmxnet3_evintr(sc); - vmxnet3_rxintr(sc, &sc->sc_rxq[0]); - vmxnet3_txintr(sc, &sc->sc_txq[0]); -#ifdef VMXNET3_STAT - vmxstat.intr++; -#endif - vmxnet3_enable_intr(sc, 0); - return 1; + + VMXNET3_RXQ_LOCK(rxq); + vmxnet3_rxq_eof(rxq); + VMXNET3_RXQ_UNLOCK(rxq); + + VMXNET3_TXQ_LOCK(txq); + vmxnet3_txq_eof(txq); + vmxnet3_start_locked(&sc->vmx_ethercom.ec_if); + VMXNET3_TXQ_UNLOCK(txq); + + vmxnet3_enable_all_intrs(sc); + + return (1); +} + +int +vmxnet3_txq_intr(void *xtxq) +{ + struct vmxnet3_softc *sc; + struct vmxnet3_txqueue *txq; + + txq = xtxq; + sc = txq->vxtxq_sc; + + if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) + vmxnet3_disable_intr(sc, txq->vxtxq_intr_idx); + + VMXNET3_TXQ_LOCK(txq); + vmxnet3_txq_eof(txq); + vmxnet3_start_locked(&sc->vmx_ethercom.ec_if); + VMXNET3_TXQ_UNLOCK(txq); + + vmxnet3_enable_intr(sc, txq->vxtxq_intr_idx); + + return (1); +} + +int +vmxnet3_rxq_intr(void *xrxq) +{ + struct vmxnet3_softc *sc; + struct vmxnet3_rxqueue *rxq; + + rxq = xrxq; + sc = rxq->vxrxq_sc; + + if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) + vmxnet3_disable_intr(sc, rxq->vxrxq_intr_idx); + + VMXNET3_RXQ_LOCK(rxq); + vmxnet3_rxq_eof(rxq); + VMXNET3_RXQ_UNLOCK(rxq); + + vmxnet3_enable_intr(sc, rxq->vxrxq_intr_idx); + + return (1); +} + +int +vmxnet3_event_intr(void *xsc) +{ + struct vmxnet3_softc *sc; + + sc = xsc; + + if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) + vmxnet3_disable_intr(sc, sc->vmx_event_intr_idx); + + if (sc->vmx_ds->event != 0) + vmxnet3_evintr(sc); + + vmxnet3_enable_intr(sc, sc->vmx_event_intr_idx); + + return (1); } void -vmxnet3_evintr(struct vmxnet3_softc *sc) +vmxnet3_txstop(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *txq) { - struct ifnet *ifp = &sc->sc_ethercom.ec_if; - u_int event = sc->sc_ds->event; - struct vmxnet3_txq_shared *ts; - struct vmxnet3_rxq_shared *rs; + struct vmxnet3_txring *txr; + struct vmxnet3_txbuf *txb; + int i; - /* Clear events. */ - WRITE_BAR1(sc, VMXNET3_BAR1_EVENT, event); + txr = &txq->vxtxq_cmd_ring; + + for (i = 0; i < txr->vxtxr_ndesc; i++) { + txb = &txr->vxtxr_txbuf[i]; + + if (txb->vtxb_m == NULL) + continue; + + bus_dmamap_sync(sc->vmx_dmat, txb->vtxb_dmamap, + 0, txb->vtxb_dmamap->dm_mapsize, + BUS_DMASYNC_POSTWRITE); + bus_dmamap_unload(sc->vmx_dmat, txb->vtxb_dmamap); + m_freem(txb->vtxb_m); + txb->vtxb_m = NULL; + } +} + +void +vmxnet3_rxstop(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq) +{ + struct vmxnet3_rxring *rxr; + struct vmxnet3_rxbuf *rxb; + int i, j; + + if (rxq->vxrxq_mhead != NULL) { + m_freem(rxq->vxrxq_mhead); + rxq->vxrxq_mhead = NULL; + rxq->vxrxq_mtail = NULL; + } + + for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { + rxr = &rxq->vxrxq_cmd_ring[i]; + + for (j = 0; j < rxr->vxrxr_ndesc; j++) { + rxb = &rxr->vxrxr_rxbuf[j]; + + if (rxb->vrxb_m == NULL) + continue; + + bus_dmamap_sync(sc->vmx_dmat, rxb->vrxb_dmamap, + 0, rxb->vrxb_dmamap->dm_mapsize, + BUS_DMASYNC_POSTREAD); + bus_dmamap_unload(sc->vmx_dmat, rxb->vrxb_dmamap); + m_freem(rxb->vrxb_m); + rxb->vrxb_m = NULL; + } + } +} + +void +vmxnet3_stop_rendezvous(struct vmxnet3_softc *sc) +{ + struct vmxnet3_rxqueue *rxq; + struct vmxnet3_txqueue *txq; + int i; + + for (i = 0; i < sc->vmx_nrxqueues; i++) { + rxq = &sc->vmx_rxq[i]; + VMXNET3_RXQ_LOCK(rxq); + VMXNET3_RXQ_UNLOCK(rxq); + } + + for (i = 0; i < sc->vmx_ntxqueues; i++) { + txq = &sc->vmx_txq[i]; + VMXNET3_TXQ_LOCK(txq); + VMXNET3_TXQ_UNLOCK(txq); + } +} + +void +vmxnet3_stop_locked(struct vmxnet3_softc *sc) +{ + struct ifnet *ifp; + int q; + + ifp = &sc->vmx_ethercom.ec_if; + VMXNET3_CORE_LOCK_ASSERT(sc); + + ifp->if_flags &= ~IFF_RUNNING; + sc->vmx_link_active = 0; + callout_stop(&sc->vmx_tick); + + /* Disable interrupts. */ + vmxnet3_disable_all_intrs(sc); + vmxnet3_write_cmd(sc, VMXNET3_CMD_DISABLE); + + vmxnet3_stop_rendezvous(sc); + + for (q = 0; q < sc->vmx_ntxqueues; q++) + vmxnet3_txstop(sc, &sc->vmx_txq[q]); + for (q = 0; q < sc->vmx_nrxqueues; q++) + vmxnet3_rxstop(sc, &sc->vmx_rxq[q]); + + vmxnet3_write_cmd(sc, VMXNET3_CMD_RESET); +} + +void +vmxnet3_stop(struct ifnet *ifp, int disable) +{ + struct vmxnet3_softc *sc = ifp->if_softc; + + VMXNET3_CORE_LOCK(sc); + vmxnet3_stop_locked(sc); + VMXNET3_CORE_UNLOCK(sc); +} + +void +vmxnet3_txinit(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *txq) +{ + struct vmxnet3_txring *txr; + struct vmxnet3_comp_ring *txc; + + txr = &txq->vxtxq_cmd_ring; + txr->vxtxr_head = 0; + txr->vxtxr_next = 0; + txr->vxtxr_gen = VMXNET3_INIT_GEN; + memset(txr->vxtxr_txd, 0, + txr->vxtxr_ndesc * sizeof(struct vmxnet3_txdesc)); + + txc = &txq->vxtxq_comp_ring; + txc->vxcr_next = 0; + txc->vxcr_gen = VMXNET3_INIT_GEN; + memset(txc->vxcr_u.txcd, 0, + txc->vxcr_ndesc * sizeof(struct vmxnet3_txcompdesc)); +} + +int +vmxnet3_rxinit(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq) +{ + struct vmxnet3_rxring *rxr; + struct vmxnet3_comp_ring *rxc; + int i, populate, idx, error; + + /* LRO and jumbo frame is not supported yet */ + populate = 1; + + for (i = 0; i < populate; i++) { + rxr = &rxq->vxrxq_cmd_ring[i]; + rxr->vxrxr_fill = 0; + rxr->vxrxr_gen = VMXNET3_INIT_GEN; + memset(rxr->vxrxr_rxd, 0, + rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc)); + + for (idx = 0; idx < rxr->vxrxr_ndesc; idx++) { + error = vmxnet3_newbuf(sc, rxr); + if (error) + return (error); + } + } + + for (/**/; i < VMXNET3_RXRINGS_PERQ; i++) { + rxr = &rxq->vxrxq_cmd_ring[i]; + rxr->vxrxr_fill = 0; + rxr->vxrxr_gen = 0; + memset(rxr->vxrxr_rxd, 0, + rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc)); + } + + rxc = &rxq->vxrxq_comp_ring; + rxc->vxcr_next = 0; + rxc->vxcr_gen = VMXNET3_INIT_GEN; + memset(rxc->vxcr_u.rxcd, 0, + rxc->vxcr_ndesc * sizeof(struct vmxnet3_rxcompdesc)); + + return (0); +} + +int +vmxnet3_reinit_queues(struct vmxnet3_softc *sc) +{ + device_t dev; + int q, error; + dev = sc->vmx_dev; + + for (q = 0; q < sc->vmx_ntxqueues; q++) + vmxnet3_txinit(sc, &sc->vmx_txq[q]); + + for (q = 0; q < sc->vmx_nrxqueues; q++) { + error = vmxnet3_rxinit(sc, &sc->vmx_rxq[q]); + if (error) { + device_printf(dev, "cannot populate Rx queue %d\n", q); + return (error); + } + } + + return (0); +} + +int +vmxnet3_enable_device(struct vmxnet3_softc *sc) +{ + int q; + + if (vmxnet3_read_cmd(sc, VMXNET3_CMD_ENABLE) != 0) { + device_printf(sc->vmx_dev, "device enable command failed!\n"); + return (1); + } + + /* Reset the Rx queue heads. */ + for (q = 0; q < sc->vmx_nrxqueues; q++) { + vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH1(q), 0); + vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH2(q), 0); + } + + return (0); +} + +void +vmxnet3_reinit_rxfilters(struct vmxnet3_softc *sc) +{ + + vmxnet3_set_rxfilter(sc); + + memset(sc->vmx_ds->vlan_filter, 0, sizeof(sc->vmx_ds->vlan_filter)); + vmxnet3_write_cmd(sc, VMXNET3_CMD_VLAN_FILTER); +} + +int +vmxnet3_reinit(struct vmxnet3_softc *sc) +{ + + vmxnet3_set_lladdr(sc); + vmxnet3_reinit_shared_data(sc); + + if (vmxnet3_reinit_queues(sc) != 0) + return (ENXIO); + + if (vmxnet3_enable_device(sc) != 0) + return (ENXIO); + + vmxnet3_reinit_rxfilters(sc); + + return (0); +} + + + + + +int +vmxnet3_init_locked(struct vmxnet3_softc *sc) +{ + struct ifnet *ifp = &sc->vmx_ethercom.ec_if; + int error; + + if (ifp->if_flags & IFF_RUNNING) + return 0; + + vmxnet3_stop_locked(sc); + + error = vmxnet3_reinit(sc); + if (error) { + vmxnet3_stop_locked(sc); + return (error); + } + + ifp->if_flags |= IFF_RUNNING; + vmxnet3_link_status(sc); + + vmxnet3_enable_all_intrs(sc); + callout_reset(&sc->vmx_tick, hz, vmxnet3_tick, sc); + + return (0); +} + +int +vmxnet3_init(struct ifnet *ifp) +{ + struct vmxnet3_softc *sc = ifp->if_softc; + int error; + + VMXNET3_CORE_LOCK(sc); + error = vmxnet3_init_locked(sc); + VMXNET3_CORE_UNLOCK(sc); + + return (error); +} + +int +vmxnet3_txq_offload_ctx(struct vmxnet3_txqueue *txq, struct mbuf *m, + int *start, int *csum_start) +{ + struct ether_header *eh; + struct mbuf *mp; + int offset, csum_off, iphl, offp; + bool v4; + + eh = mtod(m, struct ether_header *); + switch (htons(eh->ether_type)) { + case ETHERTYPE_IP: + case ETHERTYPE_IPV6: + offset = ETHER_HDR_LEN; + break; + case ETHERTYPE_VLAN: + offset = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; + break; + default: + return (EINVAL); + } + + if ((m->m_pkthdr.csum_flags & + (M_CSUM_TSOv4|M_CSUM_UDPv4|M_CSUM_TCPv4)) != 0) { + iphl = M_CSUM_DATA_IPv4_IPHL(m->m_pkthdr.csum_data); + v4 = true; + } else { + iphl = M_CSUM_DATA_IPv6_HL(m->m_pkthdr.csum_data); + v4 = false; + } + *start = offset + iphl; + + if (m->m_pkthdr.csum_flags & + (M_CSUM_TCPv4 | M_CSUM_TCPv4 | M_CSUM_TSOv4 | M_CSUM_TSOv6)) { + csum_off = offsetof(struct tcphdr, th_sum); + } else { + csum_off = offsetof(struct udphdr, uh_sum); + } + + *csum_start = *start + csum_off; + mp = m_pulldown(m, 0, *csum_start + 2, &offp); + + if (m->m_pkthdr.csum_flags & (M_CSUM_TSOv4 | M_CSUM_TSOv6)) { + struct tcphdr *tcp; + + txq->vxtxq_stats.vmtxs_tso++; + tcp = (void *)(mtod(mp, char *) + offp + *start); + + if (v4) { + struct ip *ip; + + ip = (void *)(mtod(mp, char *) + offp + offset); + tcp->th_sum = in_cksum_phdr(ip->ip_src.s_addr, + ip->ip_dst.s_addr, htons(IPPROTO_TCP)); + } else { + struct ip6_hdr *ip6; + + ip6 = (void *)(mtod(mp, char *) + offp + offset); + tcp->th_sum = in6_cksum_phdr(&ip6->ip6_src, + &ip6->ip6_dst, 0, htonl(IPPROTO_TCP)); + } + + /* + * For TSO, the size of the protocol header is also + * included in the descriptor header size. + */ + *start += (tcp->th_off << 2); + } else + txq->vxtxq_stats.vmtxs_csum++; + + return (0); +} + +int +vmxnet3_txq_load_mbuf(struct vmxnet3_txqueue *txq, struct mbuf **m0, + bus_dmamap_t dmap) +{ + struct mbuf *m; + bus_dma_tag_t tag; + int error; - /* Link state change? */ - if (event & VMXNET3_EVENT_LINK) - vmxnet3_link_state(sc); + m = *m0; + tag = txq->vxtxq_sc->vmx_dmat; - /* Queue error? */ - if (event & (VMXNET3_EVENT_TQERROR | VMXNET3_EVENT_RQERROR)) { - WRITE_CMD(sc, VMXNET3_CMD_GET_STATUS); + error = bus_dmamap_load_mbuf(tag, dmap, m, BUS_DMA_NOWAIT); + if (error == 0 || error != EFBIG) + return (error); + + m = m_defrag(m, M_NOWAIT); + if (m != NULL) { + *m0 = m; + error = bus_dmamap_load_mbuf(tag, dmap, m, BUS_DMA_NOWAIT); + } else + error = ENOBUFS; - ts = sc->sc_txq[0].ts; - if (ts->stopped) - printf("%s: TX error 0x%x\n", ifp->if_xname, ts->error); - rs = sc->sc_rxq[0].rs; - if (rs->stopped) - printf("%s: RX error 0x%x\n", ifp->if_xname, rs->error); - vmxnet3_reset(ifp); - } + if (error) { + m_freem(*m0); + *m0 = NULL; + txq->vxtxq_sc->vmx_stats.vmst_defrag_failed++; + } else + txq->vxtxq_sc->vmx_stats.vmst_defragged++; - if (event & VMXNET3_EVENT_DIC) - printf("%s: device implementation change event\n", - ifp->if_xname); - if (event & VMXNET3_EVENT_DEBUG) - printf("%s: debug event\n", ifp->if_xname); + return (error); } void -vmxnet3_txintr(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *tq) +vmxnet3_txq_unload_mbuf(struct vmxnet3_txqueue *txq, bus_dmamap_t dmap) { - struct vmxnet3_txring *ring = &tq->cmd_ring; - struct vmxnet3_comp_ring *comp_ring = &tq->comp_ring; - struct vmxnet3_txcompdesc *txcd; - struct ifnet *ifp = &sc->sc_ethercom.ec_if; - u_int sop; - for (;;) { - txcd = &comp_ring->txcd[comp_ring->next]; + bus_dmamap_unload(txq->vxtxq_sc->vmx_dmat, dmap); +} - if (le32toh((txcd->txc_word3 >> VMXNET3_TXC_GEN_S) & - VMXNET3_TXC_GEN_M) != comp_ring->gen) - break; +int +vmxnet3_txq_encap(struct vmxnet3_txqueue *txq, struct mbuf **m0) +{ + struct vmxnet3_softc *sc; + struct vmxnet3_txring *txr; + struct vmxnet3_txdesc *txd, *sop; + struct mbuf *m; + bus_dmamap_t dmap; + bus_dma_segment_t *segs; + struct m_tag *mtag; + int i, gen, start, csum_start, nsegs, error; - comp_ring->next++; - if (comp_ring->next == NTXCOMPDESC) { - comp_ring->next = 0; - comp_ring->gen ^= 1; + sc = txq->vxtxq_sc; + start = 0; + txd = NULL; + txr = &txq->vxtxq_cmd_ring; + dmap = txr->vxtxr_txbuf[txr->vxtxr_head].vtxb_dmamap; + + error = vmxnet3_txq_load_mbuf(txq, m0, dmap); + if (error) + return (error); + + nsegs = dmap->dm_nsegs; + segs = dmap->dm_segs; + + m = *m0; + KASSERT(m->m_flags & M_PKTHDR); + KASSERT(nsegs <= VMXNET3_TX_MAXSEGS); + + if (vmxnet3_txring_avail(txr) < nsegs) { + txq->vxtxq_stats.vmtxs_full++; + vmxnet3_txq_unload_mbuf(txq, dmap); + return (ENOSPC); + } else if (m->m_pkthdr.csum_flags & VMXNET3_CSUM_ALL_OFFLOAD) { + error = vmxnet3_txq_offload_ctx(txq, m, &start, &csum_start); + if (error) { + txq->vxtxq_stats.vmtxs_offload_failed++; + vmxnet3_txq_unload_mbuf(txq, dmap); + m_freem(m); + *m0 = NULL; + return (error); } + } - sop = ring->next; - if (ring->m[sop] == NULL) - panic("vmxnet3_txintr"); - m_freem(ring->m[sop]); - ring->m[sop] = NULL; - bus_dmamap_unload(sc->sc_dmat, ring->dmap[sop]); - ring->next = (le32toh((txcd->txc_word0 >> - VMXNET3_TXC_EOPIDX_S) & VMXNET3_TXC_EOPIDX_M) + 1) - % NTXDESC; + txr->vxtxr_txbuf[txr->vxtxr_head].vtxb_m = m; + sop = &txr->vxtxr_txd[txr->vxtxr_head]; + gen = txr->vxtxr_gen ^ 1; /* Owned by cpu (yet) */ + + for (i = 0; i < nsegs; i++) { + txd = &txr->vxtxr_txd[txr->vxtxr_head]; + + txd->addr = segs[i].ds_addr; + txd->len = segs[i].ds_len; + txd->gen = gen; + txd->dtype = 0; + txd->offload_mode = VMXNET3_OM_NONE; + txd->offload_pos = 0; + txd->hlen = 0; + txd->eop = 0; + txd->compreq = 0; + txd->vtag_mode = 0; + txd->vtag = 0; + + if (++txr->vxtxr_head == txr->vxtxr_ndesc) { + txr->vxtxr_head = 0; + txr->vxtxr_gen ^= 1; + } + gen = txr->vxtxr_gen; + } + txd->eop = 1; + txd->compreq = 1; - ifp->if_flags &= ~IFF_OACTIVE; + if ((mtag = VLAN_OUTPUT_TAG(&sc->vmx_ethercom, m)) != NULL) { + sop->vtag_mode = 1; + sop->vtag = VLAN_TAG_VALUE(mtag); + } + + if (m->m_pkthdr.csum_flags & (M_CSUM_TSOv4 | M_CSUM_TSOv6)) { + sop->offload_mode = VMXNET3_OM_TSO; + sop->hlen = start; + sop->offload_pos = m->m_pkthdr.segsz; + } else if (m->m_pkthdr.csum_flags & (VMXNET3_CSUM_OFFLOAD | + VMXNET3_CSUM_OFFLOAD_IPV6)) { + sop->offload_mode = VMXNET3_OM_CSUM; + sop->hlen = start; + sop->offload_pos = csum_start; + } + + /* Finally, change the ownership. */ + vmxnet3_barrier(sc, VMXNET3_BARRIER_WR); + sop->gen ^= 1; + + txq->vxtxq_ts->npending += nsegs; + if (txq->vxtxq_ts->npending >= txq->vxtxq_ts->intr_threshold) { + txq->vxtxq_ts->npending = 0; + vmxnet3_write_bar0(sc, VMXNET3_BAR0_TXH(txq->vxtxq_id), + txr->vxtxr_head); } - if (ring->head == ring->next) - ifp->if_timer = 0; - vmxnet3_start(ifp); + + return (0); } void -vmxnet3_rxintr(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rq) +vmxnet3_start_locked(struct ifnet *ifp) { - struct vmxnet3_comp_ring *comp_ring = &rq->comp_ring; - struct vmxnet3_rxring *ring; - struct vmxnet3_rxdesc *rxd; - struct vmxnet3_rxcompdesc *rxcd; - struct ifnet *ifp = &sc->sc_ethercom.ec_if; - struct mbuf *m; - int idx, len; + struct vmxnet3_softc *sc; + struct vmxnet3_txqueue *txq; + struct vmxnet3_txring *txr; + struct mbuf *m_head; + int tx; + + sc = ifp->if_softc; + txq = &sc->vmx_txq[0]; + txr = &txq->vxtxq_cmd_ring; + tx = 0; + + VMXNET3_TXQ_LOCK_ASSERT(txq); + + if ((ifp->if_flags & IFF_RUNNING) == 0 || + sc->vmx_link_active == 0) + return; for (;;) { - rxcd = &comp_ring->rxcd[comp_ring->next]; - if (le32toh((rxcd->rxc_word3 >> VMXNET3_RXC_GEN_S) & - VMXNET3_RXC_GEN_M) != comp_ring->gen) + IFQ_POLL(&ifp->if_snd, m_head); + if (m_head == NULL) break; - comp_ring->next++; - if (comp_ring->next == NRXCOMPDESC) { - comp_ring->next = 0; - comp_ring->gen ^= 1; - } - - idx = le32toh((rxcd->rxc_word0 >> VMXNET3_RXC_IDX_S) & - VMXNET3_RXC_IDX_M); - if (le32toh((rxcd->rxc_word0 >> VMXNET3_RXC_QID_S) & - VMXNET3_RXC_QID_M) < NRXQUEUE) - ring = &rq->cmd_ring[0]; - else - ring = &rq->cmd_ring[1]; - rxd = &ring->rxd[idx]; - len = le32toh((rxcd->rxc_word2 >> VMXNET3_RXC_LEN_S) & - VMXNET3_RXC_LEN_M); - m = ring->m[idx]; - ring->m[idx] = NULL; - bus_dmamap_unload(sc->sc_dmat, ring->dmap[idx]); + if (vmxnet3_txring_avail(txr) < VMXNET3_TX_MAXSEGS) + break; - if (m == NULL) - panic("NULL mbuf"); + IFQ_DEQUEUE(&ifp->if_snd, m_head); + if (m_head == NULL) + break; - if (le32toh((rxd->rx_word2 >> VMXNET3_RX_BTYPE_S) & - VMXNET3_RX_BTYPE_M) != VMXNET3_BTYPE_HEAD) { - m_freem(m); - goto skip_buffer; - } - if (le32toh(rxcd->rxc_word2 & VMXNET3_RXC_ERROR)) { - ifp->if_ierrors++; - m_freem(m); - goto skip_buffer; - } - if (len < VMXNET3_MIN_MTU) { - printf("%s: short packet (%d)\n", ifp->if_xname, len); - m_freem(m); - goto skip_buffer; + if (vmxnet3_txq_encap(txq, &m_head) != 0) { + if (m_head != NULL) + m_freem(m_head); + break; } - ifp->if_ipackets++; - ifp->if_ibytes += len; + tx++; + bpf_mtap(ifp, m_head); + } - vmxnet3_rx_csum(rxcd, m); - m_set_rcvif(m, ifp); - m->m_pkthdr.len = m->m_len = len; - if (le32toh(rxcd->rxc_word2 & VMXNET3_RXC_VLAN)) { - VLAN_INPUT_TAG(ifp, m, - le32toh((rxcd->rxc_word2 >> - VMXNET3_RXC_VLANTAG_S) & VMXNET3_RXC_VLANTAG_M), - m_freem(m); goto skip_buffer); - } + if (tx > 0) + txq->vxtxq_watchdog = VMXNET3_WATCHDOG_TIMEOUT; +} - bpf_mtap(ifp, m); - if_percpuq_enqueue(ifp->if_percpuq, m); +void +vmxnet3_start(struct ifnet *ifp) +{ + struct vmxnet3_softc *sc; + struct vmxnet3_txqueue *txq; -skip_buffer: -#ifdef VMXNET3_STAT - vmxstat.rxdone = idx; -#endif - if (rq->rs->update_rxhead) { - u_int qid = le32toh((rxcd->rxc_word0 >> - VMXNET3_RXC_QID_S) & VMXNET3_RXC_QID_M); - - idx = (idx + 1) % NRXDESC; - if (qid < NRXQUEUE) { - WRITE_BAR0(sc, VMXNET3_BAR0_RXH1(qid), idx); - } else { - qid -= NRXQUEUE; - WRITE_BAR0(sc, VMXNET3_BAR0_RXH2(qid), idx); - } - } - } + sc = ifp->if_softc; + txq = &sc->vmx_txq[0]; - /* XXX Should we (try to) allocate buffers for ring 2 too? */ - ring = &rq->cmd_ring[0]; - for (;;) { - idx = ring->fill; - if (ring->m[idx]) - return; - if (vmxnet3_getbuf(sc, ring)) - return; - } + VMXNET3_TXQ_LOCK(txq); + vmxnet3_start_locked(ifp); + VMXNET3_TXQ_UNLOCK(txq); } void -vmxnet3_iff(struct vmxnet3_softc *sc) +vmxnet3_set_rxfilter(struct vmxnet3_softc *sc) { - struct ifnet *ifp = &sc->sc_ethercom.ec_if; - struct ethercom *ec = &sc->sc_ethercom; - struct vmxnet3_driver_shared *ds = sc->sc_ds; + struct ifnet *ifp = &sc->vmx_ethercom.ec_if; + struct ethercom *ec = &sc->vmx_ethercom; + struct vmxnet3_driver_shared *ds = sc->vmx_ds; struct ether_multi *enm; struct ether_multistep step; u_int mode; @@ -821,7 +2815,7 @@ vmxnet3_iff(struct vmxnet3_softc *sc) if (ISSET(ifp->if_flags, IFF_PROMISC) || ec->ec_multicnt > 682) goto allmulti; - p = sc->sc_mcast; + p = sc->vmx_mcast; ETHER_FIRST_MULTI(step, ec, enm); while (enm != NULL) { if (memcmp(enm->enm_addrlo, enm->enm_addrhi, ETHER_ADDR_LEN)) { @@ -844,7 +2838,7 @@ vmxnet3_iff(struct vmxnet3_softc *sc) if (ec->ec_multicnt > 0) { SET(mode, VMXNET3_RXMODE_MCAST); - ds->mcast_tablelen = p - sc->sc_mcast; + ds->mcast_tablelen = p - sc->vmx_mcast; } goto setit; @@ -856,414 +2850,312 @@ allmulti: SET(mode, VMXNET3_RXMODE_PROMISC); setit: - WRITE_CMD(sc, VMXNET3_CMD_SET_FILTER); + vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_FILTER); ds->rxmode = mode; - WRITE_CMD(sc, VMXNET3_CMD_SET_RXMODE); + vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_RXMODE); } int -vmxnet3_ifflags_cb(struct ethercom *ec) +vmxnet3_change_mtu(struct vmxnet3_softc *sc, int mtu) { + struct ifnet *ifp = &sc->vmx_ethercom.ec_if; + + if (mtu < VMXNET3_MIN_MTU || mtu > VMXNET3_MAX_MTU) + return EINVAL; - vmxnet3_iff((struct vmxnet3_softc *)ec->ec_if.if_softc); + if (ifp->if_flags & IFF_RUNNING) { + ifp->if_flags &= ~IFF_RUNNING; + vmxnet3_init_locked(sc); + } return 0; } - -void -vmxnet3_rx_csum(struct vmxnet3_rxcompdesc *rxcd, struct mbuf *m) +int +vmxnet3_ioctl(struct ifnet *ifp, u_long cmd, void *data) { - if (le32toh(rxcd->rxc_word0 & VMXNET3_RXC_NOCSUM)) - return; + struct vmxnet3_softc *sc = ifp->if_softc; + struct ifreq *ifr = (struct ifreq *)data; + int s, error = 0; - if (rxcd->rxc_word3 & VMXNET3_RXC_IPV4) { - m->m_pkthdr.csum_flags |= M_CSUM_IPv4; - if ((rxcd->rxc_word3 & VMXNET3_RXC_IPSUM_OK) == 0) - m->m_pkthdr.csum_flags |= M_CSUM_IPv4_BAD; + switch (cmd) { + case SIOCSIFMTU: + if (ifp->if_mtu != ifr->ifr_mtu) { + VMXNET3_CORE_LOCK(sc); + error = vmxnet3_change_mtu(sc, ifr->ifr_mtu); + VMXNET3_CORE_UNLOCK(sc); + } + break; + case SIOCSIFMEDIA: + case SIOCGIFMEDIA: + s = splnet(); + error = ifmedia_ioctl(ifp, ifr, &sc->vmx_media, cmd); + splx(s); + break; + case SIOCGIFDATA: + for (int i = 0; i < sc->vmx_nrxqueues; i++) { + ifp->if_ipackets = + sc->vmx_rxq[i].vxrxq_stats.vmrxs_ipackets; + ifp->if_iqdrops = + sc->vmx_rxq[i].vxrxq_stats.vmrxs_iqdrops; + ifp->if_ierrors = + sc->vmx_rxq[i].vxrxq_stats.vmrxs_ierrors; + } + for (int i = 0; i < sc->vmx_ntxqueues; i++) { + ifp->if_opackets = + sc->vmx_txq[i].vxtxq_stats.vmtxs_opackets; + ifp->if_obytes = + sc->vmx_txq[i].vxtxq_stats.vmtxs_obytes; + ifp->if_omcasts = + sc->vmx_txq[i].vxtxq_stats.vmtxs_omcasts; + } + /* FALLTHROUGH */ + default: + s = splnet(); + error = ether_ioctl(ifp, cmd, data); + splx(s); } - if (rxcd->rxc_word3 & VMXNET3_RXC_FRAGMENT) - return; - - if (rxcd->rxc_word3 & VMXNET3_RXC_TCP) { - m->m_pkthdr.csum_flags |= M_CSUM_TCPv4; - if ((rxcd->rxc_word3 & VMXNET3_RXC_CSUM_OK) == 0) - m->m_pkthdr.csum_flags |= M_CSUM_TCP_UDP_BAD; + if (error == ENETRESET) { + VMXNET3_CORE_LOCK(sc); + if (ifp->if_flags & IFF_RUNNING) + vmxnet3_set_rxfilter(sc); + VMXNET3_CORE_UNLOCK(sc); + error = 0; } - if (rxcd->rxc_word3 & VMXNET3_RXC_UDP) { - m->m_pkthdr.csum_flags |= M_CSUM_UDPv4; - if ((rxcd->rxc_word3 & VMXNET3_RXC_CSUM_OK) == 0) - m->m_pkthdr.csum_flags |= M_CSUM_TCP_UDP_BAD; - } + return error; } int -vmxnet3_getbuf(struct vmxnet3_softc *sc, struct vmxnet3_rxring *ring) +vmxnet3_ifflags_cb(struct ethercom *ec) { - int idx = ring->fill; - struct vmxnet3_rxdesc *rxd = &ring->rxd[idx]; - struct mbuf *m; - int btype; + struct vmxnet3_softc *sc; - if (ring->m[idx]) - panic("vmxnet3_getbuf: buffer has mbuf"); + sc = ec->ec_if.if_softc; -#if 1 - /* XXX Don't allocate buffers for ring 2 for now. */ - if (ring->rid != 0) - return -1; - btype = VMXNET3_BTYPE_HEAD; -#else - if (ring->rid == 0) - btype = VMXNET3_BTYPE_HEAD; - else - btype = VMXNET3_BTYPE_BODY; -#endif + VMXNET3_CORE_LOCK(sc); + vmxnet3_set_rxfilter(sc); + VMXNET3_CORE_UNLOCK(sc); - MGETHDR(m, M_DONTWAIT, MT_DATA); - if (m == NULL) - return -1; + return 0; +} - MCLGET(m, M_DONTWAIT); - if ((m->m_flags & M_EXT) == 0) { - m_freem(m); - return -1; - } +int +vmxnet3_watchdog(struct vmxnet3_txqueue *txq) +{ + struct vmxnet3_softc *sc; - m->m_pkthdr.len = m->m_len = JUMBO_LEN; - m_adj(m, ETHER_ALIGN); - ring->m[idx] = m; + sc = txq->vxtxq_sc; - if (bus_dmamap_load_mbuf(sc->sc_dmat, ring->dmap[idx], m, - BUS_DMA_NOWAIT)) - panic("load mbuf"); - rxd->rx_addr = htole64(DMAADDR(ring->dmap[idx])); - rxd->rx_word2 = htole32(((m->m_pkthdr.len & VMXNET3_RX_LEN_M) << - VMXNET3_RX_LEN_S) | ((btype & VMXNET3_RX_BTYPE_M) << - VMXNET3_RX_BTYPE_S) | ((ring->gen & VMXNET3_RX_GEN_M) << - VMXNET3_RX_GEN_S)); - idx++; - if (idx == NRXDESC) { - idx = 0; - ring->gen ^= 1; + VMXNET3_TXQ_LOCK(txq); + if (txq->vxtxq_watchdog == 0 || --txq->vxtxq_watchdog) { + VMXNET3_TXQ_UNLOCK(txq); + return (0); } - ring->fill = idx; -#ifdef VMXNET3_STAT - vmxstat.rxfill = ring->fill; -#endif - return 0; + VMXNET3_TXQ_UNLOCK(txq); + + device_printf(sc->vmx_dev, "watchdog timeout on queue %d\n", + txq->vxtxq_id); + return (1); } void -vmxnet3_stop(struct ifnet *ifp, int disable) +vmxnet3_refresh_host_stats(struct vmxnet3_softc *sc) { - struct vmxnet3_softc *sc = ifp->if_softc; - int queue; - if ((ifp->if_flags & IFF_RUNNING) == 0) - return; + vmxnet3_write_cmd(sc, VMXNET3_CMD_GET_STATS); +} - vmxnet3_disable_all_intrs(sc); - ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE); - ifp->if_timer = 0; +void +vmxnet3_tick(void *xsc) +{ + struct vmxnet3_softc *sc; + struct ifnet *ifp; + int i, timedout; - if (!disable) - return; + sc = xsc; + ifp = &sc->vmx_ethercom.ec_if; + timedout = 0; + + VMXNET3_CORE_LOCK(sc); - WRITE_CMD(sc, VMXNET3_CMD_DISABLE); + vmxnet3_refresh_host_stats(sc); - for (queue = 0; queue < NTXQUEUE; queue++) - vmxnet3_txstop(sc, &sc->sc_txq[queue]); - for (queue = 0; queue < NRXQUEUE; queue++) - vmxnet3_rxstop(sc, &sc->sc_rxq[queue]); + for (i = 0; i < sc->vmx_ntxqueues; i++) + timedout |= vmxnet3_watchdog(&sc->vmx_txq[i]); + + if (timedout != 0) { + ifp->if_flags &= ~IFF_RUNNING; + vmxnet3_init_locked(sc); + } else + callout_reset(&sc->vmx_tick, hz, vmxnet3_tick, sc); + + VMXNET3_CORE_UNLOCK(sc); } void -vmxnet3_reset(struct ifnet *ifp) +vmxnet3_link_status(struct vmxnet3_softc *sc) { - struct vmxnet3_softc *sc = ifp->if_softc; + struct ifnet *ifp = &sc->vmx_ethercom.ec_if; + u_int x, link, speed; + + x = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_LINK); + speed = x >> 16; + if (x & 1) { + sc->vmx_link_active = 1; + ifp->if_baudrate = IF_Mbps(speed); + link = LINK_STATE_UP; + } else { + sc->vmx_link_active = 0; + link = LINK_STATE_DOWN; + } - vmxnet3_stop(ifp, 1); - WRITE_CMD(sc, VMXNET3_CMD_RESET); - vmxnet3_init(ifp); + if_link_state_change(ifp, link); } -int -vmxnet3_init(struct ifnet *ifp) +void +vmxnet3_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) { struct vmxnet3_softc *sc = ifp->if_softc; - int queue; - - if (ifp->if_flags & IFF_RUNNING) - return 0; - ifp->if_flags |= IFF_RUNNING; - ifp->if_flags &= ~IFF_OACTIVE; + vmxnet3_link_status(sc); - for (queue = 0; queue < NTXQUEUE; queue++) - vmxnet3_txinit(sc, &sc->sc_txq[queue]); - for (queue = 0; queue < NRXQUEUE; queue++) - vmxnet3_rxinit(sc, &sc->sc_rxq[queue]); + ifmr->ifm_status = IFM_AVALID; + ifmr->ifm_active = IFM_ETHER; - WRITE_CMD(sc, VMXNET3_CMD_ENABLE); - if (READ_BAR1(sc, VMXNET3_BAR1_CMD)) { - printf("%s: failed to initialize\n", ifp->if_xname); - vmxnet3_stop(ifp, 1); - return EIO; + VMXNET3_CORE_LOCK(sc); + if (ifp->if_link_state != LINK_STATE_UP) { + VMXNET3_CORE_UNLOCK(sc); + return; } - for (queue = 0; queue < NRXQUEUE; queue++) { - WRITE_BAR0(sc, VMXNET3_BAR0_RXH1(queue), 0); - WRITE_BAR0(sc, VMXNET3_BAR0_RXH2(queue), 0); - } + ifmr->ifm_status |= IFM_ACTIVE; - vmxnet3_iff(sc); - vmxnet3_enable_all_intrs(sc); - vmxnet3_link_state(sc); - return 0; + if (ifp->if_baudrate >= IF_Gbps(10ULL)) + ifmr->ifm_active |= IFM_10G_T; + VMXNET3_CORE_UNLOCK(sc); } int -vmxnet3_change_mtu(struct vmxnet3_softc *sc, int mtu) +vmxnet3_media_change(struct ifnet *ifp) { - struct vmxnet3_driver_shared *ds = sc->sc_ds; - struct ifnet *ifp = &sc->sc_ethercom.ec_if; - int error; - - if (mtu < VMXNET3_MIN_MTU || mtu > VMXNET3_MAX_MTU) - return EINVAL; - vmxnet3_stop(ifp, 1); - ifp->if_mtu = ds->mtu = mtu; - error = vmxnet3_init(ifp); - return error; + return 0; } -int -vmxnet3_ioctl(struct ifnet *ifp, u_long cmd, void *data) +void +vmxnet3_set_lladdr(struct vmxnet3_softc *sc) { - struct vmxnet3_softc *sc = ifp->if_softc; - struct ifreq *ifr = (struct ifreq *)data; - int error = 0, s; + uint32_t ml, mh; - s = splnet(); + ml = sc->vmx_lladdr[0]; + ml |= sc->vmx_lladdr[1] << 8; + ml |= sc->vmx_lladdr[2] << 16; + ml |= sc->vmx_lladdr[3] << 24; + vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACL, ml); + + mh = sc->vmx_lladdr[4]; + mh |= sc->vmx_lladdr[5] << 8; + vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACH, mh); +} - switch (cmd) { - case SIOCSIFMTU: - error = vmxnet3_change_mtu(sc, ifr->ifr_mtu); - break; - case SIOCSIFMEDIA: - case SIOCGIFMEDIA: - error = ifmedia_ioctl(ifp, ifr, &sc->sc_media, cmd); - break; - default: - error = ether_ioctl(ifp, cmd, data); - } +void +vmxnet3_get_lladdr(struct vmxnet3_softc *sc) +{ + uint32_t ml, mh; - if (error == ENETRESET) { - if (ifp->if_flags & IFF_RUNNING) - vmxnet3_iff(sc); - error = 0; - } + ml = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACL); + mh = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACH); - splx(s); - return error; + sc->vmx_lladdr[0] = ml; + sc->vmx_lladdr[1] = ml >> 8; + sc->vmx_lladdr[2] = ml >> 16; + sc->vmx_lladdr[3] = ml >> 24; + sc->vmx_lladdr[4] = mh; + sc->vmx_lladdr[5] = mh >> 8; } void -vmxnet3_start(struct ifnet *ifp) +vmxnet3_enable_all_intrs(struct vmxnet3_softc *sc) { - struct vmxnet3_softc *sc = ifp->if_softc; - struct vmxnet3_txqueue *tq = &sc->sc_txq[0]; - struct vmxnet3_txring *ring = &tq->cmd_ring; - struct mbuf *m; - int n = 0; - - if ((ifp->if_flags & (IFF_RUNNING | IFF_OACTIVE)) != IFF_RUNNING) - return; - - for (;;) { - IFQ_POLL(&ifp->if_snd, m); - if (m == NULL) - break; - if ((ring->next - ring->head - 1) % NTXDESC < NTXSEGS) { - ifp->if_flags |= IFF_OACTIVE; - break; - } + int i; - IFQ_DEQUEUE(&ifp->if_snd, m); - if (vmxnet3_load_mbuf(sc, m) != 0) { - ifp->if_oerrors++; - continue; - } - bpf_mtap(ifp, m); + sc->vmx_ds->ictrl &= ~VMXNET3_ICTRL_DISABLE_ALL; + for (i = 0; i < sc->vmx_nintrs; i++) + vmxnet3_enable_intr(sc, i); +} - ifp->if_timer = 5; - ifp->if_opackets++; - n++; - } +void +vmxnet3_disable_all_intrs(struct vmxnet3_softc *sc) +{ + int i; - if (n > 0) - WRITE_BAR0(sc, VMXNET3_BAR0_TXH(0), ring->head); -#ifdef VMXNET3_STAT - vmxstat.txhead = ring->head; - vmxstat.txdone = ring->next; - vmxstat.maxtxlen = - max(vmxstat.maxtxlen, (ring->head - ring->next) % NTXDESC); -#endif + sc->vmx_ds->ictrl |= VMXNET3_ICTRL_DISABLE_ALL; + for (i = 0; i < sc->vmx_nintrs; i++) + vmxnet3_disable_intr(sc, i); } + int -vmxnet3_load_mbuf(struct vmxnet3_softc *sc, struct mbuf *m) +vmxnet3_dma_malloc(struct vmxnet3_softc *sc, bus_size_t size, bus_size_t align, + struct vmxnet3_dma_alloc *dma) { - struct vmxnet3_txqueue *tq = &sc->sc_txq[0]; - struct vmxnet3_txring *ring = &tq->cmd_ring; - struct vmxnet3_txdesc *txd = NULL, *sop; - struct mbuf *mp; - struct m_tag *mtag; - struct ip *ip; - bus_dmamap_t map = ring->dmap[ring->head]; - u_int hlen = ETHER_HDR_LEN, csum_off = 0; - int offp, gen, i; - -#if 0 - if (m->m_pkthdr.csum_flags & M_CSUM_IPv4) { - printf("%s: IP checksum offloading is not supported\n", - sc->sc_dev.dv_xname); - return -1; - } -#endif - if (m->m_pkthdr.csum_flags & (M_CSUM_UDPv4|M_CSUM_TCPv4)) { - if (m->m_pkthdr.csum_flags & M_CSUM_TCPv4) - csum_off = offsetof(struct tcphdr, th_sum); - else - csum_off = offsetof(struct udphdr, uh_sum); - - mp = m_pulldown(m, hlen, sizeof(*ip), &offp); - if (mp == NULL) - return (-1); + bus_dma_tag_t t = sc->vmx_dmat; + bus_dma_segment_t *segs = dma->dma_segs; + int n, error; - ip = (struct ip *)(mp->m_data + offp); - hlen += ip->ip_hl << 2; + memset(dma, 0, sizeof(*dma)); - mp = m_pulldown(m, 0, hlen + csum_off + 2, &offp); - if (mp == NULL) - return (-1); + error = bus_dmamem_alloc(t, size, align, 0, segs, 1, &n, BUS_DMA_NOWAIT); + if (error) { + aprint_error_dev(sc->vmx_dev, "bus_dmamem_alloc failed: %d\n", error); + goto fail1; } + KASSERT(n == 1); - switch (bus_dmamap_load_mbuf(sc->sc_dmat, map, m, BUS_DMA_NOWAIT)) { - case 0: - break; - case EFBIG: - mp = m_defrag(m, M_DONTWAIT); - if (mp != NULL) { - m = mp; - if (bus_dmamap_load_mbuf(sc->sc_dmat, map, m, - BUS_DMA_NOWAIT) == 0) - break; - } - /* FALLTHROUGH */ - default: - m_freem(m); - return -1; + error = bus_dmamem_map(t, segs, 1, size, &dma->dma_vaddr, BUS_DMA_NOWAIT); + if (error) { + aprint_error_dev(sc->vmx_dev, "bus_dmamem_map failed: %d\n", error); + goto fail2; } - ring->m[ring->head] = m; - sop = &ring->txd[ring->head]; - gen = ring->gen ^ 1; /* owned by cpu (yet) */ - for (i = 0; i < map->dm_nsegs; i++) { - txd = &ring->txd[ring->head]; - txd->tx_addr = htole64(map->dm_segs[i].ds_addr); - txd->tx_word2 = htole32(((map->dm_segs[i].ds_len & - VMXNET3_TX_LEN_M) << VMXNET3_TX_LEN_S) | - ((gen & VMXNET3_TX_GEN_M) << VMXNET3_TX_GEN_S)); - txd->tx_word3 = 0; - ring->head++; - if (ring->head == NTXDESC) { - ring->head = 0; - ring->gen ^= 1; - } - gen = ring->gen; + error = bus_dmamap_create(t, size, 1, size, 0, BUS_DMA_NOWAIT, &dma->dma_map); + if (error) { + aprint_error_dev(sc->vmx_dev, "bus_dmamap_create failed: %d\n", error); + goto fail3; } - if (txd != NULL) - txd->tx_word3 |= htole32(VMXNET3_TX_EOP | VMXNET3_TX_COMPREQ); - if ((mtag = VLAN_OUTPUT_TAG(&sc->sc_ethercom, m)) != NULL) { - sop->tx_word3 |= htole32(VMXNET3_TX_VTAG_MODE); - sop->tx_word3 |= htole32((VLAN_TAG_VALUE(mtag) & - VMXNET3_TX_VLANTAG_M) << VMXNET3_TX_VLANTAG_S); - } - if (m->m_pkthdr.csum_flags & (M_CSUM_UDPv4|M_CSUM_TCPv4)) { - sop->tx_word2 |= htole32(((hlen + csum_off) & - VMXNET3_TX_OP_M) << VMXNET3_TX_OP_S); - sop->tx_word3 |= htole32(((hlen & VMXNET3_TX_HLEN_M) << - VMXNET3_TX_HLEN_S) | (VMXNET3_OM_CSUM << VMXNET3_TX_OM_S)); + error = bus_dmamap_load(t, dma->dma_map, dma->dma_vaddr, size, NULL, + BUS_DMA_NOWAIT); + if (error) { + aprint_error_dev(sc->vmx_dev, "bus_dmamap_load failed: %d\n", error); + goto fail4; } - /* Change the ownership by flipping the "generation" bit */ - sop->tx_word2 ^= htole32(VMXNET3_TX_GEN_M << VMXNET3_TX_GEN_S); + memset(dma->dma_vaddr, 0, size); + dma->dma_paddr = DMAADDR(dma->dma_map); + dma->dma_size = size; return (0); +fail4: + bus_dmamap_destroy(t, dma->dma_map); +fail3: + bus_dmamem_unmap(t, dma->dma_vaddr, size); +fail2: + bus_dmamem_free(t, segs, 1); +fail1: + return (error); } void -vmxnet3_watchdog(struct ifnet *ifp) -{ - int s; - - printf("%s: device timeout\n", ifp->if_xname); - s = splnet(); - vmxnet3_stop(ifp, 1); - vmxnet3_init(ifp); - splx(s); -} - -void -vmxnet3_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) +vmxnet3_dma_free(struct vmxnet3_softc *sc, struct vmxnet3_dma_alloc *dma) { - struct vmxnet3_softc *sc = ifp->if_softc; - - vmxnet3_link_state(sc); - - ifmr->ifm_status = IFM_AVALID; - ifmr->ifm_active = IFM_ETHER; - - if (ifp->if_link_state != LINK_STATE_UP) - return; - - ifmr->ifm_status |= IFM_ACTIVE; - - if (ifp->if_baudrate >= IF_Gbps(10ULL)) - ifmr->ifm_active |= IFM_10G_T; -} + bus_dma_tag_t t = sc->vmx_dmat; -int -vmxnet3_media_change(struct ifnet *ifp) -{ - return 0; -} + bus_dmamap_unload(t, dma->dma_map); + bus_dmamap_destroy(t, dma->dma_map); + bus_dmamem_unmap(t, dma->dma_vaddr, dma->dma_size); + bus_dmamem_free(t, dma->dma_segs, 1); -void * -vmxnet3_dma_allocmem(struct vmxnet3_softc *sc, u_int size, u_int align, bus_addr_t *pa) -{ - bus_dma_tag_t t = sc->sc_dmat; - bus_dma_segment_t segs[1]; - bus_dmamap_t map; - void *va; - int n; - - if (bus_dmamem_alloc(t, size, align, 0, segs, 1, &n, BUS_DMA_NOWAIT)) - return NULL; - if (bus_dmamem_map(t, segs, 1, size, &va, BUS_DMA_NOWAIT)) - return NULL; - if (bus_dmamap_create(t, size, 1, size, 0, BUS_DMA_NOWAIT, &map)) - return NULL; - if (bus_dmamap_load(t, map, va, size, NULL, BUS_DMA_NOWAIT)) - return NULL; - memset(va, 0, size); - *pa = DMAADDR(map); - bus_dmamap_unload(t, map); - bus_dmamap_destroy(t, map); - return va; + memset(dma, 0, sizeof(*dma)); } Index: src/sys/arch/x86/pci/if_vmxreg.h diff -u src/sys/arch/x86/pci/if_vmxreg.h:1.1 src/sys/arch/x86/pci/if_vmxreg.h:1.2 --- src/sys/arch/x86/pci/if_vmxreg.h:1.1 Tue Jun 10 01:42:39 2014 +++ src/sys/arch/x86/pci/if_vmxreg.h Fri Nov 25 05:29:54 2016 @@ -1,4 +1,4 @@ -/* $NetBSD: if_vmxreg.h,v 1.1 2014/06/10 01:42:39 hikaru Exp $ */ +/* $NetBSD: if_vmxreg.h,v 1.2 2016/11/25 05:29:54 hikaru Exp $ */ /* $OpenBSD: if_vmxreg.h,v 1.3 2013/08/28 10:19:19 reyk Exp $ */ /* @@ -75,40 +75,35 @@ struct UPT1_RxStats { #define VMXNET3_CMD_RESET 0xcafe0002 /* reset device */ #define VMXNET3_CMD_SET_RXMODE 0xcafe0003 /* set interface flags */ #define VMXNET3_CMD_SET_FILTER 0xcafe0004 /* set address filter */ +#define VMXNET3_CMD_VLAN_FILTER 0xcafe0005 /* set VLAN filter */ #define VMXNET3_CMD_GET_STATUS 0xf00d0000 /* get queue errors */ +#define VMXNET3_CMD_GET_STATS 0xf00d0001 /* get queue statistics */ #define VMXNET3_CMD_GET_LINK 0xf00d0002 /* get link status */ #define VMXNET3_CMD_GET_MACL 0xf00d0003 #define VMXNET3_CMD_GET_MACH 0xf00d0004 +#define VMXNET3_CMD_GET_INTRCFG 0xf00d0008 /* get interrupt config */ #define VMXNET3_DMADESC_ALIGN 128 +#define VMXNET3_INIT_GEN 1 /* All descriptors are in little-endian format. */ struct vmxnet3_txdesc { - uint64_t tx_addr; + uint64_t addr; - uint32_t tx_word2; -#define VMXNET3_TX_LEN_M 0x00003fff -#define VMXNET3_TX_LEN_S 0 -#define VMXNET3_TX_GEN_M 0x00000001 /* generation */ -#define VMXNET3_TX_GEN_S 14 -#define VMXNET3_TX_RES0 0x00008000 -#define VMXNET3_TX_DTYPE_M 0x00000001 /* descriptor type */ -#define VMXNET3_TX_DTYPE_S 16 /* descriptor type */ -#define VMXNET3_TX_RES1 0x00000002 -#define VMXNET3_TX_OP_M 0x00003fff /* offloading position */ -#define VMXNET3_TX_OP_S 18 - - uint32_t tx_word3; -#define VMXNET3_TX_HLEN_M 0x000003ff /* header len */ -#define VMXNET3_TX_HLEN_S 0 -#define VMXNET3_TX_OM_M 0x00000003 /* offloading mode */ -#define VMXNET3_TX_OM_S 10 -#define VMXNET3_TX_EOP 0x00001000 /* end of packet */ -#define VMXNET3_TX_COMPREQ 0x00002000 /* completion request */ -#define VMXNET3_TX_RES2 0x00004000 -#define VMXNET3_TX_VTAG_MODE 0x00008000 /* VLAN tag insertion mode */ -#define VMXNET3_TX_VLANTAG_M 0x0000ffff -#define VMXNET3_TX_VLANTAG_S 16 + uint32_t len:14; + uint32_t gen:1; /* Generation */ + uint32_t pad1:1; + uint32_t dtype:1; /* Descriptor type */ + uint32_t pad2:1; + uint32_t offload_pos:14; /* Offloading position */ + + uint32_t hlen:10; /* Header len */ + uint32_t offload_mode:2; /* Offloading mode */ + uint32_t eop:1; /* End of packet */ + uint32_t compreq:1; /* Completion request */ + uint32_t pad3:1; + uint32_t vtag_mode:1; /* VLAN tag insertion mode */ + uint32_t vtag:16; /* VLAN tag */ } __packed; /* offloading modes */ @@ -117,39 +112,27 @@ struct vmxnet3_txdesc { #define VMXNET3_OM_TSO 3 struct vmxnet3_txcompdesc { - uint32_t txc_word0; -#define VMXNET3_TXC_EOPIDX_M 0x00000fff /* eop index in Tx ring */ -#define VMXNET3_TXC_EOPIDX_S 0 -#define VMXNET3_TXC_RES0_M 0x000fffff -#define VMXNET3_TXC_RES0_S 12 - - uint32_t txc_word1; - uint32_t txc_word2; - - uint32_t txc_word3; -#define VMXNET3_TXC_RES2_M 0x00ffffff -#define VMXNET3_TXC_TYPE_M 0x0000007f -#define VMXNET3_TXC_TYPE_S 24 -#define VMXNET3_TXC_GEN_M 0x00000001 -#define VMXNET3_TXC_GEN_S 31 + uint32_t eop_idx:12; /* EOP index in Tx ring */ + uint32_t pad1:20; + + uint32_t pad2:32; + uint32_t pad3:32; + + uint32_t rsvd:24; + uint32_t type:7; + uint32_t gen:1; } __packed; struct vmxnet3_rxdesc { - uint64_t rx_addr; + uint64_t addr; - uint32_t rx_word2; -#define VMXNET3_RX_LEN_M 0x00003fff -#define VMXNET3_RX_LEN_S 0 -#define VMXNET3_RX_BTYPE_M 0x00000001 /* buffer type */ -#define VMXNET3_RX_BTYPE_S 14 -#define VMXNET3_RX_DTYPE_M 0x00000001 /* descriptor type */ -#define VMXNET3_RX_DTYPE_S 15 -#define VMXNET3_RX_RES0_M 0x00007fff -#define VMXNET3_RX_RES0_S 16 -#define VMXNET3_RX_GEN_M 0x00000001 -#define VMXNET3_RX_GEN_S 31 + uint32_t len:14; + uint32_t btype:1; /* Buffer type */ + uint32_t dtype:1; /* Descriptor type */ + uint32_t rsvd:15; + uint32_t gen:1; - uint32_t rx_word3; + uint32_t pad1:32; } __packed; /* buffer types */ @@ -157,48 +140,41 @@ struct vmxnet3_rxdesc { #define VMXNET3_BTYPE_BODY 1 /* body only */ struct vmxnet3_rxcompdesc { - uint32_t rxc_word0; -#define VMXNET3_RXC_IDX_M 0x00000fff /* Rx descriptor index */ -#define VMXNET3_RXC_IDX_S 0 -#define VMXNET3_RXC_RES0_M 0x00000003 -#define VMXNET3_RXC_RES0_S 12 -#define VMXNET3_RXC_EOP 0x00004000 /* end of packet */ -#define VMXNET3_RXC_SOP 0x00008000 /* start of packet */ -#define VMXNET3_RXC_QID_M 0x000003ff -#define VMXNET3_RXC_QID_S 16 -#define VMXNET3_RXC_RSSTYPE_M 0x0000000f -#define VMXNET3_RXC_RSSTYPE_S 26 -#define VMXNET3_RXC_NOCSUM 0x40000000 /* no checksum calculated */ -#define VMXNET3_RXC_RES1 0x80000000 - - uint32_t rxc_word1; -#define VMXNET3_RXC_RSSHASH_M 0xffffffff /* RSS hash value */ -#define VMXNET3_RXC_RSSHASH_S 0 - - uint32_t rxc_word2; -#define VMXNET3_RXC_LEN_M 0x00003fff -#define VMXNET3_RXC_LEN_S 0 -#define VMXNET3_RXC_ERROR 0x00004000 -#define VMXNET3_RXC_VLAN 0x00008000 /* 802.1Q VLAN frame */ -#define VMXNET3_RXC_VLANTAG_M 0x0000ffff /* VLAN tag */ -#define VMXNET3_RXC_VLANTAG_S 16 - - uint32_t rxc_word3; -#define VMXNET3_RXC_CSUM_M 0x0000ffff /* TCP/UDP checksum */ -#define VMXNET3_RXC_CSUM_S 16 -#define VMXNET3_RXC_CSUM_OK 0x00010000 /* TCP/UDP checksum ok */ -#define VMXNET3_RXC_UDP 0x00020000 -#define VMXNET3_RXC_TCP 0x00040000 -#define VMXNET3_RXC_IPSUM_OK 0x00080000 /* IP checksum ok */ -#define VMXNET3_RXC_IPV6 0x00100000 -#define VMXNET3_RXC_IPV4 0x00200000 -#define VMXNET3_RXC_FRAGMENT 0x00400000 /* IP fragment */ -#define VMXNET3_RXC_FCS 0x00800000 /* frame CRC correct */ -#define VMXNET3_RXC_TYPE_M 0x7f000000 -#define VMXNET3_RXC_GEN_M 0x00000001 -#define VMXNET3_RXC_GEN_S 31 + uint32_t rxd_idx:12; /* Rx descriptor index */ + uint32_t pad1:2; + uint32_t eop:1; /* End of packet */ + uint32_t sop:1; /* Start of packet */ + uint32_t qid:10; + uint32_t rss_type:4; + uint32_t no_csum:1; /* No checksum calculated */ + uint32_t pad2:1; + + uint32_t rss_hash:32; /* RSS hash value */ + + uint32_t len:14; + uint32_t error:1; + uint32_t vlan:1; /* 802.1Q VLAN frame */ + uint32_t vtag:16; /* VLAN tag */ + + uint32_t csum:16; + uint32_t csum_ok:1; /* TCP/UDP checksum ok */ + uint32_t udp:1; + uint32_t tcp:1; + uint32_t ipcsum_ok:1; /* IP checksum OK */ + uint32_t ipv6:1; + uint32_t ipv4:1; + uint32_t fragment:1; /* IP fragment */ + uint32_t fcs:1; /* Frame CRC correct */ + uint32_t type:7; + uint32_t gen:1; } __packed; +#define VMXNET3_RCD_RSS_TYPE_NONE 0 +#define VMXNET3_RCD_RSS_TYPE_IPV4 1 +#define VMXNET3_RCD_RSS_TYPE_TCPIPV4 2 +#define VMXNET3_RCD_RSS_TYPE_IPV6 3 +#define VMXNET3_RCD_RSS_TYPE_TCPIPV6 4 + #define VMXNET3_REV1_MAGIC 0xbabefee1 #define VMXNET3_GOS_UNKNOWN 0x00 @@ -214,7 +190,10 @@ struct vmxnet3_rxcompdesc { #define VMXNET3_MAX_TX_QUEUES 8 #define VMXNET3_MAX_RX_QUEUES 16 #define VMXNET3_MAX_INTRS (VMXNET3_MAX_TX_QUEUES + VMXNET3_MAX_RX_QUEUES + 1) -#define VMXNET3_NINTR 1 + +#define VMXNET3_RX_INTR_INDEX 0 +#define VMXNET3_TX_INTR_INDEX 1 +#define VMXNET3_EV_INTR_INDEX 2 #define VMXNET3_ICTRL_DISABLE_ALL 0x01 @@ -233,6 +212,15 @@ struct vmxnet3_rxcompdesc { #define VMXNET3_MAX_MTU 9000 #define VMXNET3_MIN_MTU 60 +#define VMXNET3_IMM_AUTO 0x00 +#define VMXNET3_IMM_ACTIVE 0x01 +#define VMXNET3_IMM_LAZY 0x02 + +#define VMXNET3_IT_AUTO 0x00 +#define VMXNET3_IT_LEGACY 0x01 +#define VMXNET3_IT_MSI 0x02 +#define VMXNET3_IT_MSIX 0x03 + struct vmxnet3_driver_shared { uint32_t magic; uint32_t pad1; @@ -326,3 +314,25 @@ struct vmxnet3_rxq_shared { uint8_t pad4[88]; } __packed; + +#define UPT1_RSS_HASH_TYPE_NONE 0x00 +#define UPT1_RSS_HASH_TYPE_IPV4 0x01 +#define UPT1_RSS_HASH_TYPE_TCP_IPV4 0x02 +#define UPT1_RSS_HASH_TYPE_IPV6 0x04 +#define UPT1_RSS_HASH_TYPE_TCP_IPV6 0x08 + +#define UPT1_RSS_HASH_FUNC_NONE 0x00 +#define UPT1_RSS_HASH_FUNC_TOEPLITZ 0x01 + +#define UPT1_RSS_MAX_KEY_SIZE 40 +#define UPT1_RSS_MAX_IND_TABLE_SIZE 128 + +struct vmxnet3_rss_shared { + uint16_t hash_type; + uint16_t hash_func; + uint16_t hash_key_size; + uint16_t ind_table_size; + uint8_t hash_key[UPT1_RSS_MAX_KEY_SIZE]; + uint8_t ind_table[UPT1_RSS_MAX_IND_TABLE_SIZE]; +} __packed; +