Author: br Date: Thu Apr 12 15:36:24 2018 New Revision: 332435 URL: https://svnweb.freebsd.org/changeset/base/332435
Log: Tune xDMA interface slightly: o Move descriptors allocation to DMA engine driver o Add generic xdma_request() routine o Add less-generic scatter-gather application based on xdma interface Typical operation flow in peripheral device driver is: 1. Get xDMA controller sc->xdma_tx = xdma_ofw_get(sc->dev, "tx"); 2. Allocate virtual channel sc->xchan_tx = xdma_channel_alloc(sc->xdma_tx, caps); 3. Setup transfer status callback xdma_setup_intr(sc->xchan_tx, my_tx_intr, sc, &sc->ih_tx); 4. Request a transfer(s) ret = xdma_request(sc->xchan_tx, &req); 5. Free the channel xdma_channel_free(sc->xdma_tx); 6. Free the controller xdma_put(sc->xdma_tx); Sponsored by: DARPA, AFRL Differential Revision: https://reviews.freebsd.org/D14971 Added: head/sys/dev/xdma/xdma_bank.c (contents, props changed) head/sys/dev/xdma/xdma_bio.c (contents, props changed) head/sys/dev/xdma/xdma_mbuf.c (contents, props changed) head/sys/dev/xdma/xdma_queue.c (contents, props changed) head/sys/dev/xdma/xdma_sg.c (contents, props changed) head/sys/dev/xdma/xdma_sglist.c (contents, props changed) Modified: head/sys/conf/files head/sys/dev/xdma/xdma.c head/sys/dev/xdma/xdma.h head/sys/dev/xdma/xdma_fdt_test.c head/sys/dev/xdma/xdma_if.m head/sys/mips/ingenic/jz4780_aic.c head/sys/mips/ingenic/jz4780_pdma.c head/sys/mips/ingenic/jz4780_pdma.h Modified: head/sys/conf/files ============================================================================== --- head/sys/conf/files Thu Apr 12 15:12:40 2018 (r332434) +++ head/sys/conf/files Thu Apr 12 15:36:24 2018 (r332435) @@ -3511,8 +3511,14 @@ wpi.fw optional wpifw \ no-obj no-implicit-rule \ clean "wpi.fw" dev/xdma/xdma.c optional xdma -dev/xdma/xdma_if.m optional xdma +dev/xdma/xdma_bank.c optional xdma +dev/xdma/xdma_bio.c optional xdma dev/xdma/xdma_fdt_test.c optional xdma xdma_test fdt +dev/xdma/xdma_if.m optional xdma +dev/xdma/xdma_mbuf.c optional xdma +dev/xdma/xdma_queue.c optional xdma +dev/xdma/xdma_sg.c optional xdma +dev/xdma/xdma_sglist.c optional xdma dev/xe/if_xe.c optional xe dev/xe/if_xe_pccard.c optional xe pccard dev/xen/balloon/balloon.c optional xenhvm Modified: head/sys/dev/xdma/xdma.c ============================================================================== --- head/sys/dev/xdma/xdma.c Thu Apr 12 15:12:40 2018 (r332434) +++ head/sys/dev/xdma/xdma.c Thu Apr 12 15:36:24 2018 (r332435) @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2016 Ruslan Bukin <b...@bsdpad.com> + * Copyright (c) 2016-2018 Ruslan Bukin <b...@bsdpad.com> * All rights reserved. * * This software was developed by SRI International and the University of @@ -39,7 +39,6 @@ __FBSDID("$FreeBSD$"); #include <sys/queue.h> #include <sys/kobj.h> #include <sys/malloc.h> -#include <sys/mutex.h> #include <sys/limits.h> #include <sys/lock.h> #include <sys/sysctl.h> @@ -58,40 +57,28 @@ __FBSDID("$FreeBSD$"); #include <xdma_if.h> -MALLOC_DEFINE(M_XDMA, "xdma", "xDMA framework"); - /* * Multiple xDMA controllers may work with single DMA device, * so we have global lock for physical channel management. */ -static struct mtx xdma_mtx; -#define XDMA_LOCK() mtx_lock(&xdma_mtx) -#define XDMA_UNLOCK() mtx_unlock(&xdma_mtx) -#define XDMA_ASSERT_LOCKED() mtx_assert(&xdma_mtx, MA_OWNED) +static struct sx xdma_sx; -/* - * Per channel locks. - */ -#define XCHAN_LOCK(xchan) mtx_lock(&(xchan)->mtx_lock) -#define XCHAN_UNLOCK(xchan) mtx_unlock(&(xchan)->mtx_lock) -#define XCHAN_ASSERT_LOCKED(xchan) mtx_assert(&(xchan)->mtx_lock, MA_OWNED) +#define XDMA_LOCK() sx_xlock(&xdma_sx) +#define XDMA_UNLOCK() sx_xunlock(&xdma_sx) +#define XDMA_ASSERT_LOCKED() sx_xassert(&xdma_sx, MA_OWNED) /* * Allocate virtual xDMA channel. */ xdma_channel_t * -xdma_channel_alloc(xdma_controller_t *xdma) +xdma_channel_alloc(xdma_controller_t *xdma, uint32_t caps) { xdma_channel_t *xchan; int ret; xchan = malloc(sizeof(xdma_channel_t), M_XDMA, M_WAITOK | M_ZERO); - if (xchan == NULL) { - device_printf(xdma->dev, - "%s: Can't allocate memory for channel.\n", __func__); - return (NULL); - } xchan->xdma = xdma; + xchan->caps = caps; XDMA_LOCK(); @@ -107,8 +94,18 @@ xdma_channel_alloc(xdma_controller_t *xdma) } TAILQ_INIT(&xchan->ie_handlers); - mtx_init(&xchan->mtx_lock, "xDMA", NULL, MTX_DEF); + sx_init(&xchan->sx_lock, "xDMA chan"); + sx_init(&xchan->sx_qin_lock, "xDMA qin"); + sx_init(&xchan->sx_qout_lock, "xDMA qout"); + sx_init(&xchan->sx_bank_lock, "xDMA bank"); + sx_init(&xchan->sx_proc_lock, "xDMA proc"); + + TAILQ_INIT(&xchan->bank); + TAILQ_INIT(&xchan->queue_in); + TAILQ_INIT(&xchan->queue_out); + TAILQ_INIT(&xchan->processing); + TAILQ_INSERT_TAIL(&xdma->channels, xchan, xchan_next); XDMA_UNLOCK(); @@ -123,6 +120,7 @@ xdma_channel_free(xdma_channel_t *xchan) int err; xdma = xchan->xdma; + KASSERT(xdma != NULL, ("xdma is NULL")); XDMA_LOCK(); @@ -135,13 +133,17 @@ xdma_channel_free(xdma_channel_t *xchan) return (-1); } + if (xchan->flags & XCHAN_TYPE_SG) + xdma_channel_free_sg(xchan); + xdma_teardown_all_intr(xchan); - /* Deallocate descriptors, if any. */ - xdma_desc_free(xchan); + sx_destroy(&xchan->sx_lock); + sx_destroy(&xchan->sx_qin_lock); + sx_destroy(&xchan->sx_qout_lock); + sx_destroy(&xchan->sx_bank_lock); + sx_destroy(&xchan->sx_proc_lock); - mtx_destroy(&xchan->mtx_lock); - TAILQ_REMOVE(&xdma->channels, xchan, xchan_next); free(xchan, M_XDMA); @@ -152,8 +154,9 @@ xdma_channel_free(xdma_channel_t *xchan) } int -xdma_setup_intr(xdma_channel_t *xchan, int (*cb)(void *), void *arg, - void **ihandler) +xdma_setup_intr(xdma_channel_t *xchan, + int (*cb)(void *, xdma_transfer_status_t *), + void *arg, void **ihandler) { struct xdma_intr_handler *ih; xdma_controller_t *xdma; @@ -172,22 +175,15 @@ xdma_setup_intr(xdma_channel_t *xchan, int (*cb)(void ih = malloc(sizeof(struct xdma_intr_handler), M_XDMA, M_WAITOK | M_ZERO); - if (ih == NULL) { - device_printf(xdma->dev, - "%s: Can't allocate memory for interrupt handler.\n", - __func__); - - return (-1); - } - ih->cb = cb; ih->cb_user = arg; + XCHAN_LOCK(xchan); TAILQ_INSERT_TAIL(&xchan->ie_handlers, ih, ih_next); + XCHAN_UNLOCK(xchan); - if (ihandler != NULL) { + if (ihandler != NULL) *ihandler = ih; - } return (0); } @@ -231,326 +227,67 @@ xdma_teardown_all_intr(xdma_channel_t *xchan) return (0); } -static void -xdma_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int err) -{ - xdma_channel_t *xchan; - int i; - - xchan = (xdma_channel_t *)arg; - KASSERT(xchan != NULL, ("xchan is NULL")); - - if (err) { - xchan->map_err = 1; - return; - } - - for (i = 0; i < nseg; i++) { - xchan->descs_phys[i].ds_addr = segs[i].ds_addr; - xchan->descs_phys[i].ds_len = segs[i].ds_len; - } -} - -static int -xdma_desc_alloc_bus_dma(xdma_channel_t *xchan, uint32_t desc_size, - uint32_t align) -{ - xdma_controller_t *xdma; - bus_size_t all_desc_sz; - xdma_config_t *conf; - int nsegments; - int err; - - xdma = xchan->xdma; - conf = &xchan->conf; - - nsegments = conf->block_num; - all_desc_sz = (nsegments * desc_size); - - err = bus_dma_tag_create( - bus_get_dma_tag(xdma->dev), - align, desc_size, /* alignment, boundary */ - BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ - BUS_SPACE_MAXADDR, /* highaddr */ - NULL, NULL, /* filter, filterarg */ - all_desc_sz, nsegments, /* maxsize, nsegments*/ - desc_size, 0, /* maxsegsize, flags */ - NULL, NULL, /* lockfunc, lockarg */ - &xchan->dma_tag); - if (err) { - device_printf(xdma->dev, - "%s: Can't create bus_dma tag.\n", __func__); - return (-1); - } - - err = bus_dmamem_alloc(xchan->dma_tag, (void **)&xchan->descs, - BUS_DMA_WAITOK | BUS_DMA_COHERENT, &xchan->dma_map); - if (err) { - device_printf(xdma->dev, - "%s: Can't allocate memory for descriptors.\n", __func__); - return (-1); - } - - xchan->descs_phys = malloc(nsegments * sizeof(xdma_descriptor_t), M_XDMA, - (M_WAITOK | M_ZERO)); - - xchan->map_err = 0; - err = bus_dmamap_load(xchan->dma_tag, xchan->dma_map, xchan->descs, - all_desc_sz, xdma_dmamap_cb, xchan, BUS_DMA_WAITOK); - if (err) { - device_printf(xdma->dev, - "%s: Can't load DMA map.\n", __func__); - return (-1); - } - - if (xchan->map_err != 0) { - device_printf(xdma->dev, - "%s: Can't load DMA map.\n", __func__); - return (-1); - } - - return (0); -} - -/* - * This function called by DMA controller driver. - */ int -xdma_desc_alloc(xdma_channel_t *xchan, uint32_t desc_size, uint32_t align) +xdma_request(xdma_channel_t *xchan, struct xdma_request *req) { xdma_controller_t *xdma; - xdma_config_t *conf; int ret; - XCHAN_ASSERT_LOCKED(xchan); - xdma = xchan->xdma; - if (xdma == NULL) { - device_printf(xdma->dev, - "%s: Channel was not allocated properly.\n", __func__); - return (-1); - } - if (xchan->flags & XCHAN_DESC_ALLOCATED) { - device_printf(xdma->dev, - "%s: Descriptors already allocated.\n", __func__); - return (-1); - } - - if ((xchan->flags & XCHAN_CONFIGURED) == 0) { - device_printf(xdma->dev, - "%s: Channel has no configuration.\n", __func__); - return (-1); - } - - conf = &xchan->conf; - - XCHAN_UNLOCK(xchan); - ret = xdma_desc_alloc_bus_dma(xchan, desc_size, align); - XCHAN_LOCK(xchan); - if (ret != 0) { - device_printf(xdma->dev, - "%s: Can't allocate memory for descriptors.\n", - __func__); - return (-1); - } - - xchan->flags |= XCHAN_DESC_ALLOCATED; - - /* We are going to write to descriptors. */ - bus_dmamap_sync(xchan->dma_tag, xchan->dma_map, BUS_DMASYNC_PREWRITE); - - return (0); -} - -int -xdma_desc_free(xdma_channel_t *xchan) -{ - - if ((xchan->flags & XCHAN_DESC_ALLOCATED) == 0) { - /* No descriptors allocated. */ - return (-1); - } - - bus_dmamap_unload(xchan->dma_tag, xchan->dma_map); - bus_dmamem_free(xchan->dma_tag, xchan->descs, xchan->dma_map); - bus_dma_tag_destroy(xchan->dma_tag); - free(xchan->descs_phys, M_XDMA); - - xchan->flags &= ~(XCHAN_DESC_ALLOCATED); - - return (0); -} - -int -xdma_prep_memcpy(xdma_channel_t *xchan, uintptr_t src_addr, - uintptr_t dst_addr, size_t len) -{ - xdma_controller_t *xdma; - xdma_config_t *conf; - int ret; - - xdma = xchan->xdma; KASSERT(xdma != NULL, ("xdma is NULL")); - conf = &xchan->conf; - conf->direction = XDMA_MEM_TO_MEM; - conf->src_addr = src_addr; - conf->dst_addr = dst_addr; - conf->block_len = len; - conf->block_num = 1; - - xchan->flags |= (XCHAN_CONFIGURED | XCHAN_TYPE_MEMCPY); - XCHAN_LOCK(xchan); - - /* Deallocate old descriptors, if any. */ - xdma_desc_free(xchan); - - ret = XDMA_CHANNEL_PREP_MEMCPY(xdma->dma_dev, xchan); + ret = XDMA_CHANNEL_REQUEST(xdma->dma_dev, xchan, req); if (ret != 0) { device_printf(xdma->dev, - "%s: Can't prepare memcpy transfer.\n", __func__); + "%s: Can't request a transfer.\n", __func__); XCHAN_UNLOCK(xchan); return (-1); } - - if (xchan->flags & XCHAN_DESC_ALLOCATED) { - /* Driver created xDMA descriptors. */ - bus_dmamap_sync(xchan->dma_tag, xchan->dma_map, - BUS_DMASYNC_POSTWRITE); - } - XCHAN_UNLOCK(xchan); return (0); } int -xdma_prep_cyclic(xdma_channel_t *xchan, enum xdma_direction dir, - uintptr_t src_addr, uintptr_t dst_addr, int block_len, - int block_num, int src_width, int dst_width) +xdma_control(xdma_channel_t *xchan, enum xdma_command cmd) { xdma_controller_t *xdma; - xdma_config_t *conf; int ret; xdma = xchan->xdma; KASSERT(xdma != NULL, ("xdma is NULL")); - conf = &xchan->conf; - conf->direction = dir; - conf->src_addr = src_addr; - conf->dst_addr = dst_addr; - conf->block_len = block_len; - conf->block_num = block_num; - conf->src_width = src_width; - conf->dst_width = dst_width; - - xchan->flags |= (XCHAN_CONFIGURED | XCHAN_TYPE_CYCLIC); - - XCHAN_LOCK(xchan); - - /* Deallocate old descriptors, if any. */ - xdma_desc_free(xchan); - - ret = XDMA_CHANNEL_PREP_CYCLIC(xdma->dma_dev, xchan); + ret = XDMA_CHANNEL_CONTROL(xdma->dma_dev, xchan, cmd); if (ret != 0) { device_printf(xdma->dev, - "%s: Can't prepare cyclic transfer.\n", __func__); - XCHAN_UNLOCK(xchan); - + "%s: Can't process command.\n", __func__); return (-1); } - if (xchan->flags & XCHAN_DESC_ALLOCATED) { - /* Driver has created xDMA descriptors. */ - bus_dmamap_sync(xchan->dma_tag, xchan->dma_map, - BUS_DMASYNC_POSTWRITE); - } - - XCHAN_UNLOCK(xchan); - return (0); } -int -xdma_begin(xdma_channel_t *xchan) +void +xdma_callback(xdma_channel_t *xchan, xdma_transfer_status_t *status) { + struct xdma_intr_handler *ih_tmp; + struct xdma_intr_handler *ih; xdma_controller_t *xdma; - int ret; xdma = xchan->xdma; + KASSERT(xdma != NULL, ("xdma is NULL")); - ret = XDMA_CHANNEL_CONTROL(xdma->dma_dev, xchan, XDMA_CMD_BEGIN); - if (ret != 0) { - device_printf(xdma->dev, - "%s: Can't begin the channel operation.\n", __func__); - return (-1); - } + TAILQ_FOREACH_SAFE(ih, &xchan->ie_handlers, ih_next, ih_tmp) + if (ih->cb != NULL) + ih->cb(ih->cb_user, status); - return (0); + if (xchan->flags & XCHAN_TYPE_SG) + xdma_queue_submit(xchan); } -int -xdma_terminate(xdma_channel_t *xchan) -{ - xdma_controller_t *xdma; - int ret; - - xdma = xchan->xdma; - - ret = XDMA_CHANNEL_CONTROL(xdma->dma_dev, xchan, XDMA_CMD_TERMINATE); - if (ret != 0) { - device_printf(xdma->dev, - "%s: Can't terminate the channel operation.\n", __func__); - return (-1); - } - - return (0); -} - -int -xdma_pause(xdma_channel_t *xchan) -{ - xdma_controller_t *xdma; - int ret; - - xdma = xchan->xdma; - - ret = XDMA_CHANNEL_CONTROL(xdma->dma_dev, xchan, XDMA_CMD_PAUSE); - if (ret != 0) { - device_printf(xdma->dev, - "%s: Can't pause the channel operation.\n", __func__); - return (-1); - } - - return (ret); -} - -int -xdma_callback(xdma_channel_t *xchan) -{ - struct xdma_intr_handler *ih_tmp; - struct xdma_intr_handler *ih; - - TAILQ_FOREACH_SAFE(ih, &xchan->ie_handlers, ih_next, ih_tmp) { - if (ih->cb != NULL) { - ih->cb(ih->cb_user); - } - } - - return (0); -} - -void -xdma_assert_locked(void) -{ - - XDMA_ASSERT_LOCKED(); -} - #ifdef FDT /* * Notify the DMA driver we have machine-dependent data in FDT. @@ -560,7 +297,8 @@ xdma_ofw_md_data(xdma_controller_t *xdma, pcell_t *cel { uint32_t ret; - ret = XDMA_OFW_MD_DATA(xdma->dma_dev, cells, ncells, (void **)&xdma->data); + ret = XDMA_OFW_MD_DATA(xdma->dma_dev, + cells, ncells, (void **)&xdma->data); return (ret); } @@ -581,10 +319,9 @@ xdma_ofw_get(device_t dev, const char *prop) int idx; node = ofw_bus_get_node(dev); - if (node <= 0) { + if (node <= 0) device_printf(dev, "%s called on not ofw based device.\n", __func__); - } error = ofw_bus_parse_xref_list_get_length(node, "dmas", "#dma-cells", &ndmas); @@ -622,12 +359,8 @@ xdma_ofw_get(device_t dev, const char *prop) return (NULL); } - xdma = malloc(sizeof(struct xdma_controller), M_XDMA, M_WAITOK | M_ZERO); - if (xdma == NULL) { - device_printf(dev, - "%s can't allocate memory for xdma.\n", __func__); - return (NULL); - } + xdma = malloc(sizeof(struct xdma_controller), + M_XDMA, M_WAITOK | M_ZERO); xdma->dev = dev; xdma->dma_dev = dma_dev; @@ -667,7 +400,7 @@ static void xdma_init(void) { - mtx_init(&xdma_mtx, "xDMA", NULL, MTX_DEF); + sx_init(&xdma_sx, "xDMA"); } SYSINIT(xdma, SI_SUB_DRIVERS, SI_ORDER_FIRST, xdma_init, NULL); Modified: head/sys/dev/xdma/xdma.h ============================================================================== --- head/sys/dev/xdma/xdma.h Thu Apr 12 15:12:40 2018 (r332434) +++ head/sys/dev/xdma/xdma.h Thu Apr 12 15:36:24 2018 (r332435) @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2016 Ruslan Bukin <b...@bsdpad.com> + * Copyright (c) 2016-2018 Ruslan Bukin <b...@bsdpad.com> * All rights reserved. * * This software was developed by SRI International and the University of @@ -30,9 +30,11 @@ * $FreeBSD$ */ -#ifndef _DEV_EXTRES_XDMA_H_ -#define _DEV_EXTRES_XDMA_H_ +#ifndef _DEV_XDMA_XDMA_H_ +#define _DEV_XDMA_XDMA_H_ +#include <sys/proc.h> + enum xdma_direction { XDMA_MEM_TO_MEM, XDMA_MEM_TO_DEV, @@ -42,17 +44,31 @@ enum xdma_direction { enum xdma_operation_type { XDMA_MEMCPY, - XDMA_SG, XDMA_CYCLIC, + XDMA_FIFO, + XDMA_SG, }; +enum xdma_request_type { + XR_TYPE_PHYS, + XR_TYPE_VIRT, + XR_TYPE_MBUF, + XR_TYPE_BIO, +}; + enum xdma_command { XDMA_CMD_BEGIN, XDMA_CMD_PAUSE, XDMA_CMD_TERMINATE, - XDMA_CMD_TERMINATE_ALL, }; +struct xdma_transfer_status { + uint32_t transferred; + int error; +}; + +typedef struct xdma_transfer_status xdma_transfer_status_t; + struct xdma_controller { device_t dev; /* DMA consumer device_t. */ device_t dma_dev; /* A real DMA device_t. */ @@ -64,85 +80,185 @@ struct xdma_controller { typedef struct xdma_controller xdma_controller_t; -struct xdma_channel_config { - enum xdma_direction direction; - uintptr_t src_addr; /* Physical address. */ - uintptr_t dst_addr; /* Physical address. */ - int block_len; /* In bytes. */ - int block_num; /* Count of blocks. */ - int src_width; /* In bytes. */ - int dst_width; /* In bytes. */ +struct xchan_buf { + bus_dmamap_t map; + uint32_t nsegs; + uint32_t nsegs_left; + void *cbuf; }; -typedef struct xdma_channel_config xdma_config_t; +struct xdma_request { + struct mbuf *m; + struct bio *bp; + enum xdma_operation_type operation; + enum xdma_request_type req_type; + enum xdma_direction direction; + bus_addr_t src_addr; + bus_addr_t dst_addr; + uint8_t src_width; + uint8_t dst_width; + bus_size_t block_num; + bus_size_t block_len; + xdma_transfer_status_t status; + void *user; + TAILQ_ENTRY(xdma_request) xr_next; + struct xchan_buf buf; +}; -struct xdma_descriptor { - bus_addr_t ds_addr; - bus_size_t ds_len; +struct xdma_sglist { + bus_addr_t src_addr; + bus_addr_t dst_addr; + size_t len; + uint8_t src_width; + uint8_t dst_width; + enum xdma_direction direction; + bool first; + bool last; }; -typedef struct xdma_descriptor xdma_descriptor_t; - struct xdma_channel { xdma_controller_t *xdma; - xdma_config_t conf; - uint8_t flags; -#define XCHAN_DESC_ALLOCATED (1 << 0) -#define XCHAN_CONFIGURED (1 << 1) -#define XCHAN_TYPE_CYCLIC (1 << 2) -#define XCHAN_TYPE_MEMCPY (1 << 3) + uint32_t flags; +#define XCHAN_BUFS_ALLOCATED (1 << 0) +#define XCHAN_SGLIST_ALLOCATED (1 << 1) +#define XCHAN_CONFIGURED (1 << 2) +#define XCHAN_TYPE_CYCLIC (1 << 3) +#define XCHAN_TYPE_MEMCPY (1 << 4) +#define XCHAN_TYPE_FIFO (1 << 5) +#define XCHAN_TYPE_SG (1 << 6) + uint32_t caps; +#define XCHAN_CAP_BUSDMA (1 << 0) +#define XCHAN_CAP_BUSDMA_NOSEG (1 << 1) + /* A real hardware driver channel. */ void *chan; /* Interrupt handlers. */ TAILQ_HEAD(, xdma_intr_handler) ie_handlers; + TAILQ_ENTRY(xdma_channel) xchan_next; - /* Descriptors. */ - bus_dma_tag_t dma_tag; - bus_dmamap_t dma_map; - void *descs; - xdma_descriptor_t *descs_phys; - uint8_t map_err; + struct sx sx_lock; + struct sx sx_qin_lock; + struct sx sx_qout_lock; + struct sx sx_bank_lock; + struct sx sx_proc_lock; - struct mtx mtx_lock; + /* Request queue. */ + bus_dma_tag_t dma_tag_bufs; + struct xdma_request *xr_mem; + uint32_t xr_num; - TAILQ_ENTRY(xdma_channel) xchan_next; + /* Bus dma tag options. */ + bus_size_t maxsegsize; + bus_size_t maxnsegs; + bus_size_t alignment; + bus_addr_t boundary; + bus_addr_t lowaddr; + bus_addr_t highaddr; + + struct xdma_sglist *sg; + + TAILQ_HEAD(, xdma_request) bank; + TAILQ_HEAD(, xdma_request) queue_in; + TAILQ_HEAD(, xdma_request) queue_out; + TAILQ_HEAD(, xdma_request) processing; }; typedef struct xdma_channel xdma_channel_t; -/* xDMA controller alloc/free */ +struct xdma_intr_handler { + int (*cb)(void *cb_user, xdma_transfer_status_t *status); + void *cb_user; + TAILQ_ENTRY(xdma_intr_handler) ih_next; +}; + +static MALLOC_DEFINE(M_XDMA, "xdma", "xDMA framework"); + +#define XCHAN_LOCK(xchan) sx_xlock(&(xchan)->sx_lock) +#define XCHAN_UNLOCK(xchan) sx_xunlock(&(xchan)->sx_lock) +#define XCHAN_ASSERT_LOCKED(xchan) \ + sx_assert(&(xchan)->sx_lock, SX_XLOCKED) + +#define QUEUE_IN_LOCK(xchan) sx_xlock(&(xchan)->sx_qin_lock) +#define QUEUE_IN_UNLOCK(xchan) sx_xunlock(&(xchan)->sx_qin_lock) +#define QUEUE_IN_ASSERT_LOCKED(xchan) \ + sx_assert(&(xchan)->sx_qin_lock, SX_XLOCKED) + +#define QUEUE_OUT_LOCK(xchan) sx_xlock(&(xchan)->sx_qout_lock) +#define QUEUE_OUT_UNLOCK(xchan) sx_xunlock(&(xchan)->sx_qout_lock) +#define QUEUE_OUT_ASSERT_LOCKED(xchan) \ + sx_assert(&(xchan)->sx_qout_lock, SX_XLOCKED) + +#define QUEUE_BANK_LOCK(xchan) sx_xlock(&(xchan)->sx_bank_lock) +#define QUEUE_BANK_UNLOCK(xchan) sx_xunlock(&(xchan)->sx_bank_lock) +#define QUEUE_BANK_ASSERT_LOCKED(xchan) \ + sx_assert(&(xchan)->sx_bank_lock, SX_XLOCKED) + +#define QUEUE_PROC_LOCK(xchan) sx_xlock(&(xchan)->sx_proc_lock) +#define QUEUE_PROC_UNLOCK(xchan) sx_xunlock(&(xchan)->sx_proc_lock) +#define QUEUE_PROC_ASSERT_LOCKED(xchan) \ + sx_assert(&(xchan)->sx_proc_lock, SX_XLOCKED) + +#define XDMA_SGLIST_MAXLEN 2048 +#define XDMA_MAX_SEG 128 + +/* xDMA controller ops */ xdma_controller_t *xdma_ofw_get(device_t dev, const char *prop); int xdma_put(xdma_controller_t *xdma); -xdma_channel_t * xdma_channel_alloc(xdma_controller_t *); +/* xDMA channel ops */ +xdma_channel_t * xdma_channel_alloc(xdma_controller_t *, uint32_t caps); int xdma_channel_free(xdma_channel_t *); +int xdma_request(xdma_channel_t *xchan, struct xdma_request *r); -int xdma_prep_cyclic(xdma_channel_t *, enum xdma_direction, - uintptr_t, uintptr_t, int, int, int, int); -int xdma_prep_memcpy(xdma_channel_t *, uintptr_t, uintptr_t, size_t len); -int xdma_desc_alloc(xdma_channel_t *, uint32_t, uint32_t); -int xdma_desc_free(xdma_channel_t *xchan); +/* SG interface */ +int xdma_prep_sg(xdma_channel_t *, uint32_t, + bus_size_t, bus_size_t, bus_size_t, bus_addr_t, bus_addr_t, bus_addr_t); +void xdma_channel_free_sg(xdma_channel_t *xchan); +int xdma_queue_submit_sg(xdma_channel_t *xchan); +void xchan_seg_done(xdma_channel_t *xchan, xdma_transfer_status_t *); +/* Queue operations */ +int xdma_dequeue_mbuf(xdma_channel_t *xchan, struct mbuf **m, + xdma_transfer_status_t *); +int xdma_enqueue_mbuf(xdma_channel_t *xchan, struct mbuf **m, uintptr_t addr, + uint8_t, uint8_t, enum xdma_direction dir); +int xdma_dequeue_bio(xdma_channel_t *xchan, struct bio **bp, + xdma_transfer_status_t *status); +int xdma_enqueue_bio(xdma_channel_t *xchan, struct bio **bp, bus_addr_t addr, + uint8_t, uint8_t, enum xdma_direction dir); +int xdma_dequeue(xdma_channel_t *xchan, void **user, + xdma_transfer_status_t *status); +int xdma_enqueue(xdma_channel_t *xchan, uintptr_t src, uintptr_t dst, + uint8_t, uint8_t, bus_size_t, enum xdma_direction dir, void *); +int xdma_queue_submit(xdma_channel_t *xchan); + +/* Mbuf operations */ +uint32_t xdma_mbuf_defrag(xdma_channel_t *xchan, struct xdma_request *xr); +uint32_t xdma_mbuf_chain_count(struct mbuf *m0); + /* Channel Control */ -int xdma_begin(xdma_channel_t *xchan); -int xdma_pause(xdma_channel_t *xchan); -int xdma_terminate(xdma_channel_t *xchan); +int xdma_control(xdma_channel_t *xchan, enum xdma_command cmd); /* Interrupt callback */ -int xdma_setup_intr(xdma_channel_t *xchan, int (*cb)(void *), void *arg, void **); +int xdma_setup_intr(xdma_channel_t *xchan, int (*cb)(void *, + xdma_transfer_status_t *), void *arg, void **); int xdma_teardown_intr(xdma_channel_t *xchan, struct xdma_intr_handler *ih); int xdma_teardown_all_intr(xdma_channel_t *xchan); -int xdma_callback(struct xdma_channel *xchan); -void xdma_assert_locked(void); +void xdma_callback(struct xdma_channel *xchan, xdma_transfer_status_t *status); -struct xdma_intr_handler { - int (*cb)(void *); - void *cb_user; - struct mtx ih_lock; - TAILQ_ENTRY(xdma_intr_handler) ih_next; -}; +/* Sglist */ +int xchan_sglist_alloc(xdma_channel_t *xchan); +void xchan_sglist_free(xdma_channel_t *xchan); +int xdma_sglist_add(struct xdma_sglist *sg, struct bus_dma_segment *seg, + uint32_t nsegs, struct xdma_request *xr); -#endif /* !_DEV_EXTRES_XDMA_H_ */ +/* Requests bank */ +void xchan_bank_init(xdma_channel_t *xchan); +int xchan_bank_free(xdma_channel_t *xchan); +struct xdma_request * xchan_bank_get(xdma_channel_t *xchan); +int xchan_bank_put(xdma_channel_t *xchan, struct xdma_request *xr); + +#endif /* !_DEV_XDMA_XDMA_H_ */ Added: head/sys/dev/xdma/xdma_bank.c ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ head/sys/dev/xdma/xdma_bank.c Thu Apr 12 15:36:24 2018 (r332435) @@ -0,0 +1,99 @@ +/*- + * Copyright (c) 2018 Ruslan Bukin <b...@bsdpad.com> + * All rights reserved. + * + * This software was developed by SRI International and the University of + * Cambridge Computer Laboratory under DARPA/AFRL contract FA8750-10-C-0237 + * ("CTSRD"), as part of the DARPA CRASH research programme. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include "opt_platform.h" +#include <sys/param.h> +#include <sys/conf.h> +#include <sys/bus.h> +#include <sys/kernel.h> +#include <sys/malloc.h> +#include <sys/sx.h> + +#include <machine/bus.h> + +#include <dev/xdma/xdma.h> + +void +xchan_bank_init(xdma_channel_t *xchan) +{ + struct xdma_request *xr; + xdma_controller_t *xdma; + int i; + + xdma = xchan->xdma; + KASSERT(xdma != NULL, ("xdma is NULL")); + + xchan->xr_mem = malloc(sizeof(struct xdma_request) * xchan->xr_num, + M_XDMA, M_WAITOK | M_ZERO); + + for (i = 0; i < xchan->xr_num; i++) { + xr = &xchan->xr_mem[i]; + TAILQ_INSERT_TAIL(&xchan->bank, xr, xr_next); + } +} + +int +xchan_bank_free(xdma_channel_t *xchan) +{ + + free(xchan->xr_mem, M_XDMA); + + return (0); +} + +struct xdma_request * +xchan_bank_get(xdma_channel_t *xchan) +{ + struct xdma_request *xr; + struct xdma_request *xr_tmp; + + QUEUE_BANK_LOCK(xchan); + TAILQ_FOREACH_SAFE(xr, &xchan->bank, xr_next, xr_tmp) { + TAILQ_REMOVE(&xchan->bank, xr, xr_next); + break; + } + QUEUE_BANK_UNLOCK(xchan); + + return (xr); +} + +int +xchan_bank_put(xdma_channel_t *xchan, struct xdma_request *xr) +{ + + QUEUE_BANK_LOCK(xchan); + TAILQ_INSERT_TAIL(&xchan->bank, xr, xr_next); + QUEUE_BANK_UNLOCK(xchan); + + return (0); +} Added: head/sys/dev/xdma/xdma_bio.c ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ head/sys/dev/xdma/xdma_bio.c Thu Apr 12 15:36:24 2018 (r332435) @@ -0,0 +1,105 @@ +/*- + * Copyright (c) 2017-2018 Ruslan Bukin <b...@bsdpad.com> + * All rights reserved. + * + * This software was developed by SRI International and the University of + * Cambridge Computer Laboratory under DARPA/AFRL contract FA8750-10-C-0237 + * ("CTSRD"), as part of the DARPA CRASH research programme. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include "opt_platform.h" +#include <sys/param.h> +#include <sys/conf.h> +#include <sys/bus.h> +#include <sys/kernel.h> +#include <sys/malloc.h> +#include <sys/sx.h> + +#include <machine/bus.h> + +#include <dev/xdma/xdma.h> + *** DIFF OUTPUT TRUNCATED AT 1000 LINES *** _______________________________________________ svn-src-head@freebsd.org mailing list https://lists.freebsd.org/mailman/listinfo/svn-src-head To unsubscribe, send any mail to "svn-src-head-unsubscr...@freebsd.org"
