The branch main has been updated by mw:
URL:
https://cgit.FreeBSD.org/src/commit/?id=3fc5d816f8831d6fc2816ac97bd78dc486cd080c
commit 3fc5d816f8831d6fc2816ac97bd78dc486cd080c
Merge: 0e7d31f63b9d 1f4f67f52424
Author: Marcin Wojtas <m...@freebsd.org>
AuthorDate: 2021-06-24 14:07:33 +0000
Commit: Marcin Wojtas <m...@freebsd.org>
CommitDate: 2021-06-24 14:15:18 +0000
Merge tag 'vendor/ena-com/2.4.0'
Update the driver in order not to break its compilation
and make use of the new ENA logging system
Migrate platform code to the new logging system provided by ena_com
layer.
Make ENA_INFO the new default log level.
Remove all explicit use of `device_printf`, all new logs requiring one
of the log macros to be used.
sys/contrib/ena-com/ena_com.c | 37 +--
sys/contrib/ena-com/ena_com.h | 6 +-
sys/contrib/ena-com/ena_defs/ena_admin_defs.h | 23 +-
sys/contrib/ena-com/ena_defs/ena_gen_info.h | 6 +-
sys/contrib/ena-com/ena_eth_com.c | 4 +-
sys/contrib/ena-com/ena_fbsd_log.h | 74 ++++++
sys/contrib/ena-com/ena_plat.h | 37 +--
sys/dev/ena/ena.c | 342 +++++++++++++-------------
sys/dev/ena/ena_datapath.c | 83 ++++---
sys/dev/ena/ena_netmap.c | 73 +++---
sys/dev/ena/ena_sysctl.c | 30 +--
11 files changed, 397 insertions(+), 318 deletions(-)
diff --cc sys/contrib/ena-com/ena_fbsd_log.h
index 000000000000,311e2f0f280a..311e2f0f280a
mode 000000,100644..100644
--- a/sys/contrib/ena-com/ena_fbsd_log.h
+++ b/sys/contrib/ena-com/ena_fbsd_log.h
diff --cc sys/dev/ena/ena.c
index fbe32a7a930e,000000000000..84d58c844332
mode 100644,000000..100644
--- a/sys/dev/ena/ena.c
+++ b/sys/dev/ena/ena.c
@@@ -1,3980 -1,0 +1,3984 @@@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2015-2021 Amazon.com, Inc. or its affiliates.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include "opt_rss.h"
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/endian.h>
+#include <sys/kernel.h>
+#include <sys/kthread.h>
+#include <sys/malloc.h>
+#include <sys/mbuf.h>
+#include <sys/module.h>
+#include <sys/rman.h>
+#include <sys/smp.h>
+#include <sys/socket.h>
+#include <sys/sockio.h>
+#include <sys/sysctl.h>
+#include <sys/taskqueue.h>
+#include <sys/time.h>
+#include <sys/eventhandler.h>
+
+#include <machine/bus.h>
+#include <machine/resource.h>
+#include <machine/in_cksum.h>
+
+#include <net/bpf.h>
+#include <net/ethernet.h>
+#include <net/if.h>
+#include <net/if_var.h>
+#include <net/if_arp.h>
+#include <net/if_dl.h>
+#include <net/if_media.h>
+#include <net/if_types.h>
+#include <net/if_vlan_var.h>
+#ifdef RSS
+#include <net/rss_config.h>
+#endif
+
+#include <netinet/in_systm.h>
+#include <netinet/in.h>
+#include <netinet/if_ether.h>
+#include <netinet/ip.h>
+#include <netinet/ip6.h>
+#include <netinet/tcp.h>
+#include <netinet/udp.h>
+
+#include <dev/pci/pcivar.h>
+#include <dev/pci/pcireg.h>
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+
+#include "ena_datapath.h"
+#include "ena.h"
+#include "ena_sysctl.h"
+
+#ifdef DEV_NETMAP
+#include "ena_netmap.h"
+#endif /* DEV_NETMAP */
+
+/*********************************************************
+ * Function prototypes
+ *********************************************************/
+static int ena_probe(device_t);
+static void ena_intr_msix_mgmnt(void *);
+static void ena_free_pci_resources(struct ena_adapter *);
+static int ena_change_mtu(if_t, int);
+static inline void ena_alloc_counters(counter_u64_t *, int);
+static inline void ena_free_counters(counter_u64_t *, int);
+static inline void ena_reset_counters(counter_u64_t *, int);
+static void ena_init_io_rings_common(struct ena_adapter *,
+ struct ena_ring *, uint16_t);
+static void ena_init_io_rings_basic(struct ena_adapter *);
+static void ena_init_io_rings_advanced(struct ena_adapter *);
+static void ena_init_io_rings(struct ena_adapter *);
+static void ena_free_io_ring_resources(struct ena_adapter *, unsigned int);
+static void ena_free_all_io_rings_resources(struct ena_adapter *);
+static int ena_setup_tx_dma_tag(struct ena_adapter *);
+static int ena_free_tx_dma_tag(struct ena_adapter *);
+static int ena_setup_rx_dma_tag(struct ena_adapter *);
+static int ena_free_rx_dma_tag(struct ena_adapter *);
+static void ena_release_all_tx_dmamap(struct ena_ring *);
+static int ena_setup_tx_resources(struct ena_adapter *, int);
+static void ena_free_tx_resources(struct ena_adapter *, int);
+static int ena_setup_all_tx_resources(struct ena_adapter *);
+static void ena_free_all_tx_resources(struct ena_adapter *);
+static int ena_setup_rx_resources(struct ena_adapter *, unsigned int);
+static void ena_free_rx_resources(struct ena_adapter *, unsigned int);
+static int ena_setup_all_rx_resources(struct ena_adapter *);
+static void ena_free_all_rx_resources(struct ena_adapter *);
+static inline int ena_alloc_rx_mbuf(struct ena_adapter *, struct ena_ring *,
+ struct ena_rx_buffer *);
+static void ena_free_rx_mbuf(struct ena_adapter *, struct ena_ring *,
+ struct ena_rx_buffer *);
+static void ena_free_rx_bufs(struct ena_adapter *, unsigned int);
+static void ena_refill_all_rx_bufs(struct ena_adapter *);
+static void ena_free_all_rx_bufs(struct ena_adapter *);
+static void ena_free_tx_bufs(struct ena_adapter *, unsigned int);
+static void ena_free_all_tx_bufs(struct ena_adapter *);
+static void ena_destroy_all_tx_queues(struct ena_adapter *);
+static void ena_destroy_all_rx_queues(struct ena_adapter *);
+static void ena_destroy_all_io_queues(struct ena_adapter *);
+static int ena_create_io_queues(struct ena_adapter *);
+static int ena_handle_msix(void *);
+static int ena_enable_msix(struct ena_adapter *);
+static void ena_setup_mgmnt_intr(struct ena_adapter *);
+static int ena_setup_io_intr(struct ena_adapter *);
+static int ena_request_mgmnt_irq(struct ena_adapter *);
+static int ena_request_io_irq(struct ena_adapter *);
+static void ena_free_mgmnt_irq(struct ena_adapter *);
+static void ena_free_io_irq(struct ena_adapter *);
+static void ena_free_irqs(struct ena_adapter*);
+static void ena_disable_msix(struct ena_adapter *);
+static void ena_unmask_all_io_irqs(struct ena_adapter *);
+static int ena_rss_configure(struct ena_adapter *);
+static int ena_up_complete(struct ena_adapter *);
+static uint64_t ena_get_counter(if_t, ift_counter);
+static int ena_media_change(if_t);
+static void ena_media_status(if_t, struct ifmediareq *);
+static void ena_init(void *);
+static int ena_ioctl(if_t, u_long, caddr_t);
+static int ena_get_dev_offloads(struct ena_com_dev_get_features_ctx *);
+static void ena_update_host_info(struct ena_admin_host_info *, if_t);
+static void ena_update_hwassist(struct ena_adapter *);
+static int ena_setup_ifnet(device_t, struct ena_adapter *,
+ struct ena_com_dev_get_features_ctx *);
- static int ena_enable_wc(struct resource *);
++static int ena_enable_wc(device_t, struct resource *);
+static int ena_set_queues_placement_policy(device_t, struct ena_com_dev *,
+ struct ena_admin_feature_llq_desc *, struct ena_llq_configurations *);
+static uint32_t ena_calc_max_io_queue_num(device_t, struct ena_com_dev
*,
+ struct ena_com_dev_get_features_ctx *);
+static int ena_calc_io_queue_size(struct ena_calc_queue_size_ctx *);
+static int ena_rss_init_default(struct ena_adapter *);
+static void ena_rss_init_default_deferred(void *);
+static void ena_config_host_info(struct ena_com_dev *, device_t);
+static int ena_attach(device_t);
+static int ena_detach(device_t);
+static int ena_device_init(struct ena_adapter *, device_t,
+ struct ena_com_dev_get_features_ctx *, int *);
+static int ena_enable_msix_and_set_admin_interrupts(struct ena_adapter *);
+static void ena_update_on_link_change(void *, struct ena_admin_aenq_entry *);
+static void unimplemented_aenq_handler(void *,
+ struct ena_admin_aenq_entry *);
+static int ena_copy_eni_metrics(struct ena_adapter *);
+static void ena_timer_service(void *);
+
+static char ena_version[] = DEVICE_NAME DRV_MODULE_NAME " v"
DRV_MODULE_VERSION;
+
+static ena_vendor_info_t ena_vendor_info_array[] = {
+ { PCI_VENDOR_ID_AMAZON, PCI_DEV_ID_ENA_PF, 0},
+ { PCI_VENDOR_ID_AMAZON, PCI_DEV_ID_ENA_PF_RSERV0, 0},
+ { PCI_VENDOR_ID_AMAZON, PCI_DEV_ID_ENA_VF, 0},
+ { PCI_VENDOR_ID_AMAZON, PCI_DEV_ID_ENA_VF_RSERV0, 0},
+ /* Last entry */
+ { 0, 0, 0 }
+};
+
+/*
+ * Contains pointers to event handlers, e.g. link state chage.
+ */
+static struct ena_aenq_handlers aenq_handlers;
+
+void
+ena_dmamap_callback(void *arg, bus_dma_segment_t *segs, int nseg, int error)
+{
+ if (error != 0)
+ return;
+ *(bus_addr_t *) arg = segs[0].ds_addr;
+}
+
+int
+ena_dma_alloc(device_t dmadev, bus_size_t size,
+ ena_mem_handle_t *dma, int mapflags, bus_size_t alignment)
+{
+ struct ena_adapter* adapter = device_get_softc(dmadev);
++ device_t pdev = adapter->pdev;
+ uint32_t maxsize;
+ uint64_t dma_space_addr;
+ int error;
+
+ maxsize = ((size - 1) / PAGE_SIZE + 1) * PAGE_SIZE;
+
+ dma_space_addr = ENA_DMA_BIT_MASK(adapter->dma_width);
+ if (unlikely(dma_space_addr == 0))
+ dma_space_addr = BUS_SPACE_MAXADDR;
+
+ error = bus_dma_tag_create(bus_get_dma_tag(dmadev), /* parent */
+ alignment, 0, /* alignment, bounds */
+ dma_space_addr, /* lowaddr of exclusion window */
+ BUS_SPACE_MAXADDR,/* highaddr of exclusion window */
+ NULL, NULL, /* filter, filterarg */
+ maxsize, /* maxsize */
+ 1, /* nsegments */
+ maxsize, /* maxsegsize */
+ BUS_DMA_ALLOCNOW, /* flags */
+ NULL, /* lockfunc */
+ NULL, /* lockarg */
+ &dma->tag);
+ if (unlikely(error != 0)) {
- ena_trace(NULL, ENA_ALERT, "bus_dma_tag_create failed: %d\n",
error);
++ ena_log(pdev, ERR, "bus_dma_tag_create failed: %d\n", error);
+ goto fail_tag;
+ }
+
+ error = bus_dmamem_alloc(dma->tag, (void**) &dma->vaddr,
+ BUS_DMA_COHERENT | BUS_DMA_ZERO, &dma->map);
+ if (unlikely(error != 0)) {
- ena_trace(NULL, ENA_ALERT, "bus_dmamem_alloc(%ju) failed: %d\n",
++ ena_log(pdev, ERR, "bus_dmamem_alloc(%ju) failed: %d\n",
+ (uintmax_t)size, error);
+ goto fail_map_create;
+ }
+
+ dma->paddr = 0;
+ error = bus_dmamap_load(dma->tag, dma->map, dma->vaddr,
+ size, ena_dmamap_callback, &dma->paddr, mapflags);
+ if (unlikely((error != 0) || (dma->paddr == 0))) {
- ena_trace(NULL, ENA_ALERT, ": bus_dmamap_load failed: %d\n",
error);
++ ena_log(pdev, ERR, "bus_dmamap_load failed: %d\n", error);
+ goto fail_map_load;
+ }
+
+ bus_dmamap_sync(dma->tag, dma->map,
+ BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
+
+ return (0);
+
+fail_map_load:
+ bus_dmamem_free(dma->tag, dma->vaddr, dma->map);
+fail_map_create:
+ bus_dma_tag_destroy(dma->tag);
+fail_tag:
+ dma->tag = NULL;
+ dma->vaddr = NULL;
+ dma->paddr = 0;
+
+ return (error);
+}
+
+/*
+ * This function should generate unique key for the whole driver.
+ * If the key was already genereated in the previous call (for example
+ * for another adapter), then it should be returned instead.
+ */
+void
+ena_rss_key_fill(void *key, size_t size)
+{
+ static bool key_generated;
+ static uint8_t default_key[ENA_HASH_KEY_SIZE];
+
+ KASSERT(size <= ENA_HASH_KEY_SIZE, ("Requested more bytes than ENA RSS
key can hold"));
+
+ if (!key_generated) {
+ arc4random_buf(default_key, ENA_HASH_KEY_SIZE);
+ key_generated = true;
+ }
+
+ memcpy(key, default_key, size);
+}
+
+static void
+ena_free_pci_resources(struct ena_adapter *adapter)
+{
+ device_t pdev = adapter->pdev;
+
+ if (adapter->memory != NULL) {
+ bus_release_resource(pdev, SYS_RES_MEMORY,
+ PCIR_BAR(ENA_MEM_BAR), adapter->memory);
+ }
+
+ if (adapter->registers != NULL) {
+ bus_release_resource(pdev, SYS_RES_MEMORY,
+ PCIR_BAR(ENA_REG_BAR), adapter->registers);
+ }
+
+ if (adapter->msix != NULL) {
+ bus_release_resource(pdev, SYS_RES_MEMORY,
+ adapter->msix_rid, adapter->msix);
+ }
+}
+
+static int
+ena_probe(device_t dev)
+{
+ ena_vendor_info_t *ent;
+ char adapter_name[60];
+ uint16_t pci_vendor_id = 0;
+ uint16_t pci_device_id = 0;
+
+ pci_vendor_id = pci_get_vendor(dev);
+ pci_device_id = pci_get_device(dev);
+
+ ent = ena_vendor_info_array;
+ while (ent->vendor_id != 0) {
+ if ((pci_vendor_id == ent->vendor_id) &&
+ (pci_device_id == ent->device_id)) {
- ena_trace(NULL, ENA_DBG, "vendor=%x device=%x\n",
++ ena_log_raw(DBG, "vendor=%x device=%x\n",
+ pci_vendor_id, pci_device_id);
+
+ sprintf(adapter_name, DEVICE_DESC);
+ device_set_desc_copy(dev, adapter_name);
+ return (BUS_PROBE_DEFAULT);
+ }
+
+ ent++;
+
+ }
+
+ return (ENXIO);
+}
+
+static int
+ena_change_mtu(if_t ifp, int new_mtu)
+{
+ struct ena_adapter *adapter = if_getsoftc(ifp);
++ device_t pdev = adapter->pdev;
+ int rc;
+
+ if ((new_mtu > adapter->max_mtu) || (new_mtu < ENA_MIN_MTU)) {
- device_printf(adapter->pdev, "Invalid MTU setting. "
++ ena_log(pdev, ERR, "Invalid MTU setting. "
+ "new_mtu: %d max mtu: %d min mtu: %d\n",
+ new_mtu, adapter->max_mtu, ENA_MIN_MTU);
+ return (EINVAL);
+ }
+
+ rc = ena_com_set_dev_mtu(adapter->ena_dev, new_mtu);
+ if (likely(rc == 0)) {
- ena_trace(NULL, ENA_DBG, "set MTU to %d\n", new_mtu);
++ ena_log(pdev, DBG, "set MTU to %d\n", new_mtu);
+ if_setmtu(ifp, new_mtu);
+ } else {
- device_printf(adapter->pdev, "Failed to set MTU to %d\n",
- new_mtu);
++ ena_log(pdev, ERR, "Failed to set MTU to %d\n", new_mtu);
+ }
+
+ return (rc);
+}
+
+static inline void
+ena_alloc_counters(counter_u64_t *begin, int size)
+{
+ counter_u64_t *end = (counter_u64_t *)((char *)begin + size);
+
+ for (; begin < end; ++begin)
+ *begin = counter_u64_alloc(M_WAITOK);
+}
+
+static inline void
+ena_free_counters(counter_u64_t *begin, int size)
+{
+ counter_u64_t *end = (counter_u64_t *)((char *)begin + size);
+
+ for (; begin < end; ++begin)
+ counter_u64_free(*begin);
+}
+
+static inline void
+ena_reset_counters(counter_u64_t *begin, int size)
+{
+ counter_u64_t *end = (counter_u64_t *)((char *)begin + size);
+
+ for (; begin < end; ++begin)
+ counter_u64_zero(*begin);
+}
+
+static void
+ena_init_io_rings_common(struct ena_adapter *adapter, struct ena_ring *ring,
+ uint16_t qid)
+{
+
+ ring->qid = qid;
+ ring->adapter = adapter;
+ ring->ena_dev = adapter->ena_dev;
+ ring->first_interrupt = false;
+ ring->no_interrupt_event_cnt = 0;
+}
+
+static void
+ena_init_io_rings_basic(struct ena_adapter *adapter)
+{
+ struct ena_com_dev *ena_dev;
+ struct ena_ring *txr, *rxr;
+ struct ena_que *que;
+ int i;
+
+ ena_dev = adapter->ena_dev;
+
+ for (i = 0; i < adapter->num_io_queues; i++) {
+ txr = &adapter->tx_ring[i];
+ rxr = &adapter->rx_ring[i];
+
+ /* TX/RX common ring state */
+ ena_init_io_rings_common(adapter, txr, i);
+ ena_init_io_rings_common(adapter, rxr, i);
+
+ /* TX specific ring state */
+ txr->tx_max_header_size = ena_dev->tx_max_header_size;
+ txr->tx_mem_queue_type = ena_dev->tx_mem_queue_type;
+
+ que = &adapter->que[i];
+ que->adapter = adapter;
+ que->id = i;
+ que->tx_ring = txr;
+ que->rx_ring = rxr;
+
+ txr->que = que;
+ rxr->que = que;
+
+ rxr->empty_rx_queue = 0;
+ rxr->rx_mbuf_sz = ena_mbuf_sz;
+ }
+}
+
+static void
+ena_init_io_rings_advanced(struct ena_adapter *adapter)
+{
+ struct ena_ring *txr, *rxr;
+ int i;
+
+ for (i = 0; i < adapter->num_io_queues; i++) {
+ txr = &adapter->tx_ring[i];
+ rxr = &adapter->rx_ring[i];
+
+ /* Allocate a buf ring */
+ txr->buf_ring_size = adapter->buf_ring_size;
+ txr->br = buf_ring_alloc(txr->buf_ring_size, M_DEVBUF,
+ M_WAITOK, &txr->ring_mtx);
+
+ /* Allocate Tx statistics. */
+ ena_alloc_counters((counter_u64_t *)&txr->tx_stats,
+ sizeof(txr->tx_stats));
+
+ /* Allocate Rx statistics. */
+ ena_alloc_counters((counter_u64_t *)&rxr->rx_stats,
+ sizeof(rxr->rx_stats));
+
+ /* Initialize locks */
+ snprintf(txr->mtx_name, nitems(txr->mtx_name), "%s:tx(%d)",
+ device_get_nameunit(adapter->pdev), i);
+ snprintf(rxr->mtx_name, nitems(rxr->mtx_name), "%s:rx(%d)",
+ device_get_nameunit(adapter->pdev), i);
+
+ mtx_init(&txr->ring_mtx, txr->mtx_name, NULL, MTX_DEF);
+ }
+}
+
+static void
+ena_init_io_rings(struct ena_adapter *adapter)
+{
+ /*
+ * IO rings initialization can be divided into the 2 steps:
+ * 1. Initialize variables and fields with initial values and copy
+ * them from adapter/ena_dev (basic)
+ * 2. Allocate mutex, counters and buf_ring (advanced)
+ */
+ ena_init_io_rings_basic(adapter);
+ ena_init_io_rings_advanced(adapter);
+}
+
+static void
+ena_free_io_ring_resources(struct ena_adapter *adapter, unsigned int qid)
+{
+ struct ena_ring *txr = &adapter->tx_ring[qid];
+ struct ena_ring *rxr = &adapter->rx_ring[qid];
+
+ ena_free_counters((counter_u64_t *)&txr->tx_stats,
+ sizeof(txr->tx_stats));
+ ena_free_counters((counter_u64_t *)&rxr->rx_stats,
+ sizeof(rxr->rx_stats));
+
+ ENA_RING_MTX_LOCK(txr);
+ drbr_free(txr->br, M_DEVBUF);
+ ENA_RING_MTX_UNLOCK(txr);
+
+ mtx_destroy(&txr->ring_mtx);
+}
+
+static void
+ena_free_all_io_rings_resources(struct ena_adapter *adapter)
+{
+ int i;
+
+ for (i = 0; i < adapter->num_io_queues; i++)
+ ena_free_io_ring_resources(adapter, i);
+
+}
+
+static int
+ena_setup_tx_dma_tag(struct ena_adapter *adapter)
+{
+ int ret;
+
+ /* Create DMA tag for Tx buffers */
+ ret = bus_dma_tag_create(bus_get_dma_tag(adapter->pdev),
+ 1, 0, /* alignment, bounds */
+ ENA_DMA_BIT_MASK(adapter->dma_width), /* lowaddr of excl window */
+ BUS_SPACE_MAXADDR, /* highaddr of excl window */
+ NULL, NULL, /* filter, filterarg */
+ ENA_TSO_MAXSIZE, /* maxsize */
+ adapter->max_tx_sgl_size - 1, /* nsegments */
+ ENA_TSO_MAXSIZE, /* maxsegsize */
+ 0, /* flags */
+ NULL, /* lockfunc */
+ NULL, /* lockfuncarg */
+ &adapter->tx_buf_tag);
+
+ return (ret);
+}
+
+static int
+ena_free_tx_dma_tag(struct ena_adapter *adapter)
+{
+ int ret;
+
+ ret = bus_dma_tag_destroy(adapter->tx_buf_tag);
+
+ if (likely(ret == 0))
+ adapter->tx_buf_tag = NULL;
+
+ return (ret);
+}
+
+static int
+ena_setup_rx_dma_tag(struct ena_adapter *adapter)
+{
+ int ret;
+
+ /* Create DMA tag for Rx buffers*/
+ ret = bus_dma_tag_create(bus_get_dma_tag(adapter->pdev), /* parent */
+ 1, 0, /* alignment, bounds */
+ ENA_DMA_BIT_MASK(adapter->dma_width), /* lowaddr of excl window */
+ BUS_SPACE_MAXADDR, /* highaddr of excl window */
+ NULL, NULL, /* filter, filterarg */
+ ena_mbuf_sz, /* maxsize */
+ adapter->max_rx_sgl_size, /* nsegments */
+ ena_mbuf_sz, /* maxsegsize */
+ 0, /* flags */
+ NULL, /* lockfunc */
+ NULL, /* lockarg */
+ &adapter->rx_buf_tag);
+
+ return (ret);
+}
+
+static int
+ena_free_rx_dma_tag(struct ena_adapter *adapter)
+{
+ int ret;
+
+ ret = bus_dma_tag_destroy(adapter->rx_buf_tag);
+
+ if (likely(ret == 0))
+ adapter->rx_buf_tag = NULL;
+
+ return (ret);
+}
+
+static void
+ena_release_all_tx_dmamap(struct ena_ring *tx_ring)
+{
+ struct ena_adapter *adapter = tx_ring->adapter;
+ struct ena_tx_buffer *tx_info;
+ bus_dma_tag_t tx_tag = adapter->tx_buf_tag;;
+ int i;
+#ifdef DEV_NETMAP
+ struct ena_netmap_tx_info *nm_info;
+ int j;
+#endif /* DEV_NETMAP */
+
+ for (i = 0; i < tx_ring->ring_size; ++i) {
+ tx_info = &tx_ring->tx_buffer_info[i];
+#ifdef DEV_NETMAP
+ if (adapter->ifp->if_capenable & IFCAP_NETMAP) {
+ nm_info = &tx_info->nm_info;
+ for (j = 0; j < ENA_PKT_MAX_BUFS; ++j) {
+ if (nm_info->map_seg[j] != NULL) {
+ bus_dmamap_destroy(tx_tag,
+ nm_info->map_seg[j]);
+ nm_info->map_seg[j] = NULL;
+ }
+ }
+ }
+#endif /* DEV_NETMAP */
+ if (tx_info->dmamap != NULL) {
+ bus_dmamap_destroy(tx_tag, tx_info->dmamap);
+ tx_info->dmamap = NULL;
+ }
+ }
+}
+
+/**
+ * ena_setup_tx_resources - allocate Tx resources (Descriptors)
+ * @adapter: network interface device structure
+ * @qid: queue index
+ *
+ * Returns 0 on success, otherwise on failure.
+ **/
+static int
+ena_setup_tx_resources(struct ena_adapter *adapter, int qid)
+{
++ device_t pdev = adapter->pdev;
+ struct ena_que *que = &adapter->que[qid];
+ struct ena_ring *tx_ring = que->tx_ring;
+ int size, i, err;
+#ifdef DEV_NETMAP
+ bus_dmamap_t *map;
+ int j;
+
+ ena_netmap_reset_tx_ring(adapter, qid);
+#endif /* DEV_NETMAP */
+
+ size = sizeof(struct ena_tx_buffer) * tx_ring->ring_size;
+
+ tx_ring->tx_buffer_info = malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO);
+ if (unlikely(tx_ring->tx_buffer_info == NULL))
+ return (ENOMEM);
+
+ size = sizeof(uint16_t) * tx_ring->ring_size;
+ tx_ring->free_tx_ids = malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO);
+ if (unlikely(tx_ring->free_tx_ids == NULL))
+ goto err_buf_info_free;
+
+ size = tx_ring->tx_max_header_size;
+ tx_ring->push_buf_intermediate_buf = malloc(size, M_DEVBUF,
+ M_NOWAIT | M_ZERO);
+ if (unlikely(tx_ring->push_buf_intermediate_buf == NULL))
+ goto err_tx_ids_free;
+
+ /* Req id stack for TX OOO completions */
+ for (i = 0; i < tx_ring->ring_size; i++)
+ tx_ring->free_tx_ids[i] = i;
+
+ /* Reset TX statistics. */
+ ena_reset_counters((counter_u64_t *)&tx_ring->tx_stats,
+ sizeof(tx_ring->tx_stats));
+
+ tx_ring->next_to_use = 0;
+ tx_ring->next_to_clean = 0;
+ tx_ring->acum_pkts = 0;
+
+ /* Make sure that drbr is empty */
+ ENA_RING_MTX_LOCK(tx_ring);
+ drbr_flush(adapter->ifp, tx_ring->br);
+ ENA_RING_MTX_UNLOCK(tx_ring);
+
+ /* ... and create the buffer DMA maps */
+ for (i = 0; i < tx_ring->ring_size; i++) {
+ err = bus_dmamap_create(adapter->tx_buf_tag, 0,
+ &tx_ring->tx_buffer_info[i].dmamap);
+ if (unlikely(err != 0)) {
- ena_trace(NULL, ENA_ALERT,
++ ena_log(pdev, ERR,
+ "Unable to create Tx DMA map for buffer %d\n",
+ i);
+ goto err_map_release;
+ }
+
+#ifdef DEV_NETMAP
+ if (adapter->ifp->if_capenable & IFCAP_NETMAP) {
+ map = tx_ring->tx_buffer_info[i].nm_info.map_seg;
+ for (j = 0; j < ENA_PKT_MAX_BUFS; j++) {
+ err = bus_dmamap_create(adapter->tx_buf_tag, 0,
+ &map[j]);
+ if (unlikely(err != 0)) {
- ena_trace(NULL, ENA_ALERT, "Unable to
create "
++ ena_log(pdev, ERR,
++ "Unable to create "
+ "Tx DMA for buffer %d %d\n", i, j);
+ goto err_map_release;
+ }
+ }
+ }
+#endif /* DEV_NETMAP */
+ }
+
+ /* Allocate taskqueues */
+ TASK_INIT(&tx_ring->enqueue_task, 0, ena_deferred_mq_start, tx_ring);
+ tx_ring->enqueue_tq = taskqueue_create_fast("ena_tx_enque", M_NOWAIT,
+ taskqueue_thread_enqueue, &tx_ring->enqueue_tq);
+ if (unlikely(tx_ring->enqueue_tq == NULL)) {
- ena_trace(NULL, ENA_ALERT,
++ ena_log(pdev, ERR,
+ "Unable to create taskqueue for enqueue task\n");
+ i = tx_ring->ring_size;
+ goto err_map_release;
+ }
+
+ tx_ring->running = true;
+
+ taskqueue_start_threads(&tx_ring->enqueue_tq, 1, PI_NET,
+ "%s txeq %d", device_get_nameunit(adapter->pdev), que->cpu);
+
+ return (0);
+
+err_map_release:
+ ena_release_all_tx_dmamap(tx_ring);
+err_tx_ids_free:
+ free(tx_ring->free_tx_ids, M_DEVBUF);
+ tx_ring->free_tx_ids = NULL;
+err_buf_info_free:
+ free(tx_ring->tx_buffer_info, M_DEVBUF);
+ tx_ring->tx_buffer_info = NULL;
+
+ return (ENOMEM);
+}
+
+/**
+ * ena_free_tx_resources - Free Tx Resources per Queue
+ * @adapter: network interface device structure
+ * @qid: queue index
+ *
+ * Free all transmit software resources
+ **/
+static void
+ena_free_tx_resources(struct ena_adapter *adapter, int qid)
+{
+ struct ena_ring *tx_ring = &adapter->tx_ring[qid];
+#ifdef DEV_NETMAP
+ struct ena_netmap_tx_info *nm_info;
+ int j;
+#endif /* DEV_NETMAP */
+
+ while (taskqueue_cancel(tx_ring->enqueue_tq, &tx_ring->enqueue_task,
+ NULL))
+ taskqueue_drain(tx_ring->enqueue_tq, &tx_ring->enqueue_task);
+
+ taskqueue_free(tx_ring->enqueue_tq);
+
+ ENA_RING_MTX_LOCK(tx_ring);
+ /* Flush buffer ring, */
+ drbr_flush(adapter->ifp, tx_ring->br);
+
+ /* Free buffer DMA maps, */
+ for (int i = 0; i < tx_ring->ring_size; i++) {
+ bus_dmamap_sync(adapter->tx_buf_tag,
+ tx_ring->tx_buffer_info[i].dmamap, BUS_DMASYNC_POSTWRITE);
+ bus_dmamap_unload(adapter->tx_buf_tag,
+ tx_ring->tx_buffer_info[i].dmamap);
+ bus_dmamap_destroy(adapter->tx_buf_tag,
+ tx_ring->tx_buffer_info[i].dmamap);
+
+#ifdef DEV_NETMAP
+ if (adapter->ifp->if_capenable & IFCAP_NETMAP) {
+ nm_info = &tx_ring->tx_buffer_info[i].nm_info;
+ for (j = 0; j < ENA_PKT_MAX_BUFS; j++) {
+ if (nm_info->socket_buf_idx[j] != 0) {
+ bus_dmamap_sync(adapter->tx_buf_tag,
+ nm_info->map_seg[j],
+ BUS_DMASYNC_POSTWRITE);
+ ena_netmap_unload(adapter,
+ nm_info->map_seg[j]);
+ }
+ bus_dmamap_destroy(adapter->tx_buf_tag,
+ nm_info->map_seg[j]);
+ nm_info->socket_buf_idx[j] = 0;
+ }
+ }
+#endif /* DEV_NETMAP */
+
+ m_freem(tx_ring->tx_buffer_info[i].mbuf);
+ tx_ring->tx_buffer_info[i].mbuf = NULL;
+ }
+ ENA_RING_MTX_UNLOCK(tx_ring);
+
+ /* And free allocated memory. */
+ free(tx_ring->tx_buffer_info, M_DEVBUF);
+ tx_ring->tx_buffer_info = NULL;
+
+ free(tx_ring->free_tx_ids, M_DEVBUF);
+ tx_ring->free_tx_ids = NULL;
+
+ free(tx_ring->push_buf_intermediate_buf, M_DEVBUF);
+ tx_ring->push_buf_intermediate_buf = NULL;
+}
+
+/**
+ * ena_setup_all_tx_resources - allocate all queues Tx resources
+ * @adapter: network interface device structure
+ *
+ * Returns 0 on success, otherwise on failure.
+ **/
+static int
+ena_setup_all_tx_resources(struct ena_adapter *adapter)
+{
+ int i, rc;
+
+ for (i = 0; i < adapter->num_io_queues; i++) {
+ rc = ena_setup_tx_resources(adapter, i);
+ if (rc != 0) {
- device_printf(adapter->pdev,
++ ena_log(adapter->pdev, ERR,
+ "Allocation for Tx Queue %u failed\n", i);
+ goto err_setup_tx;
+ }
+ }
+
+ return (0);
+
+err_setup_tx:
+ /* Rewind the index freeing the rings as we go */
+ while (i--)
+ ena_free_tx_resources(adapter, i);
+ return (rc);
+}
+
+/**
+ * ena_free_all_tx_resources - Free Tx Resources for All Queues
+ * @adapter: network interface device structure
+ *
+ * Free all transmit software resources
+ **/
+static void
+ena_free_all_tx_resources(struct ena_adapter *adapter)
+{
+ int i;
+
+ for (i = 0; i < adapter->num_io_queues; i++)
+ ena_free_tx_resources(adapter, i);
+}
+
+/**
+ * ena_setup_rx_resources - allocate Rx resources (Descriptors)
+ * @adapter: network interface device structure
+ * @qid: queue index
+ *
+ * Returns 0 on success, otherwise on failure.
+ **/
+static int
+ena_setup_rx_resources(struct ena_adapter *adapter, unsigned int qid)
+{
++ device_t pdev = adapter->pdev;
+ struct ena_que *que = &adapter->que[qid];
+ struct ena_ring *rx_ring = que->rx_ring;
+ int size, err, i;
+
+ size = sizeof(struct ena_rx_buffer) * rx_ring->ring_size;
+
+#ifdef DEV_NETMAP
+ ena_netmap_reset_rx_ring(adapter, qid);
+ rx_ring->initialized = false;
+#endif /* DEV_NETMAP */
+
+ /*
+ * Alloc extra element so in rx path
+ * we can always prefetch rx_info + 1
+ */
+ size += sizeof(struct ena_rx_buffer);
+
+ rx_ring->rx_buffer_info = malloc(size, M_DEVBUF, M_WAITOK | M_ZERO);
+
+ size = sizeof(uint16_t) * rx_ring->ring_size;
+ rx_ring->free_rx_ids = malloc(size, M_DEVBUF, M_WAITOK);
+
+ for (i = 0; i < rx_ring->ring_size; i++)
+ rx_ring->free_rx_ids[i] = i;
+
+ /* Reset RX statistics. */
+ ena_reset_counters((counter_u64_t *)&rx_ring->rx_stats,
+ sizeof(rx_ring->rx_stats));
+
+ rx_ring->next_to_clean = 0;
+ rx_ring->next_to_use = 0;
+
+ /* ... and create the buffer DMA maps */
+ for (i = 0; i < rx_ring->ring_size; i++) {
+ err = bus_dmamap_create(adapter->rx_buf_tag, 0,
+ &(rx_ring->rx_buffer_info[i].map));
+ if (err != 0) {
- ena_trace(NULL, ENA_ALERT,
++ ena_log(pdev, ERR,
+ "Unable to create Rx DMA map for buffer %d\n", i);
+ goto err_buf_info_unmap;
+ }
+ }
+
+ /* Create LRO for the ring */
+ if ((adapter->ifp->if_capenable & IFCAP_LRO) != 0) {
+ int err = tcp_lro_init(&rx_ring->lro);
+ if (err != 0) {
- device_printf(adapter->pdev,
- "LRO[%d] Initialization failed!\n", qid);
++ ena_log(pdev, ERR, "LRO[%d] Initialization failed!\n",
++ qid);
+ } else {
- ena_trace(NULL, ENA_INFO,
- "RX Soft LRO[%d] Initialized\n", qid);
++ ena_log(pdev, DBG, "RX Soft LRO[%d] Initialized\n",
++ qid);
+ rx_ring->lro.ifp = adapter->ifp;
+ }
+ }
+
+ return (0);
+
+err_buf_info_unmap:
+ while (i--) {
+ bus_dmamap_destroy(adapter->rx_buf_tag,
+ rx_ring->rx_buffer_info[i].map);
+ }
+
+ free(rx_ring->free_rx_ids, M_DEVBUF);
+ rx_ring->free_rx_ids = NULL;
+ free(rx_ring->rx_buffer_info, M_DEVBUF);
+ rx_ring->rx_buffer_info = NULL;
*** 6157 LINES SKIPPED ***
_______________________________________________
dev-commits-src-main@freebsd.org mailing list
https://lists.freebsd.org/mailman/listinfo/dev-commits-src-main
To unsubscribe, send any mail to "dev-commits-src-main-unsubscr...@freebsd.org"
