Author: scottl
Date: Tue Jun 18 00:36:53 2013
New Revision: 251874
URL: http://svnweb.freebsd.org/changeset/base/251874

Log:
  Big MFC of the physbio changes necessary for unmapped I/O.  These changes
  have been in production at Netflix for several months with significant
  success.
  
  MFC r246713:
  
  Reform the busdma API so that new types may be added without modifying
  every architecture's busdma_machdep.c.  It is done by unifying the
  bus_dmamap_load_buffer() routines so that they may be called from MI
  code.  The MD busdma is then given a chance to do any final processing
  in the complete() callback.
  
  MFC r249538:
  Some compilers issue a warning when wider integer is casted to narrow
  pointer.  Supposedly shut down the warning by casting through
  uintptr_t.
  
  MFC r251479:
  Simplify the checking of flags for cam_periph_mapmem().  This gets rid of
  a lot of code redundancy and grossness at very minor expense.
  
  MFC r251837:
  MFC r251842:
  Add infrastructure for doing compatibility shims, as has been sorely
  needed for the last 10 years.  Far too much of the internal API is
  exposed, and every small adjustment causes applications to stop working.
  To kick this off, bump the API version to 0x17 as should have been done
  with r246713, but add shims to compensate.  Thanks to the shims, there
  should be no visible change in application behavior.
  
  Submitted by: kib, jeffr
  Approved by:  kib
  Obtained from:        Netflix

Added:
  stable/9/sys/cam/cam_compat.c
     - copied unchanged from r251837, head/sys/cam/cam_compat.c
  stable/9/sys/cam/cam_compat.h
     - copied unchanged from r251837, head/sys/cam/cam_compat.h
  stable/9/sys/kern/subr_bus_dma.c
     - copied, changed from r246713, head/sys/kern/subr_bus_dma.c
  stable/9/sys/sys/memdesc.h
     - copied unchanged from r246713, head/sys/sys/memdesc.h
Modified:
  stable/9/sys/arm/arm/busdma_machdep.c
  stable/9/sys/cam/cam_ccb.h
  stable/9/sys/cam/cam_periph.c
  stable/9/sys/cam/cam_xpt.c
  stable/9/sys/cam/ctl/ctl_frontend_cam_sim.c
  stable/9/sys/cam/ctl/scsi_ctl.c
  stable/9/sys/cam/scsi/scsi_pass.c
  stable/9/sys/cam/scsi/scsi_sg.c
  stable/9/sys/cam/scsi/scsi_target.c
  stable/9/sys/conf/files
  stable/9/sys/dev/aac/aac_cam.c
  stable/9/sys/dev/advansys/advansys.c
  stable/9/sys/dev/advansys/adwcam.c
  stable/9/sys/dev/aha/aha.c
  stable/9/sys/dev/ahb/ahb.c
  stable/9/sys/dev/ahci/ahci.c
  stable/9/sys/dev/aic/aic.c
  stable/9/sys/dev/aic7xxx/aic79xx_osm.c
  stable/9/sys/dev/aic7xxx/aic7xxx_osm.c
  stable/9/sys/dev/amr/amr_cam.c
  stable/9/sys/dev/arcmsr/arcmsr.c
  stable/9/sys/dev/ata/ata-dma.c
  stable/9/sys/dev/ata/atapi-cam.c
  stable/9/sys/dev/buslogic/bt.c
  stable/9/sys/dev/ciss/ciss.c
  stable/9/sys/dev/ciss/cissvar.h
  stable/9/sys/dev/dpt/dpt_scsi.c
  stable/9/sys/dev/firewire/sbp.c
  stable/9/sys/dev/hpt27xx/osm_bsd.c
  stable/9/sys/dev/hptiop/hptiop.c
  stable/9/sys/dev/hptmv/entry.c
  stable/9/sys/dev/hptrr/hptrr_osm_bsd.c
  stable/9/sys/dev/iir/iir.c
  stable/9/sys/dev/isci/isci_io_request.c
  stable/9/sys/dev/isp/isp_pci.c
  stable/9/sys/dev/isp/isp_sbus.c
  stable/9/sys/dev/mfi/mfi.c
  stable/9/sys/dev/mfi/mfi_cam.c
  stable/9/sys/dev/mly/mly.c
  stable/9/sys/dev/mly/mlyvar.h
  stable/9/sys/dev/mps/mps.c
  stable/9/sys/dev/mps/mps_sas.c
  stable/9/sys/dev/mps/mpsvar.h
  stable/9/sys/dev/mpt/mpt_cam.c
  stable/9/sys/dev/mvs/mvs.c
  stable/9/sys/dev/siis/siis.c
  stable/9/sys/dev/sym/sym_hipd.c
  stable/9/sys/dev/trm/trm.c
  stable/9/sys/dev/twa/tw_osl.h
  stable/9/sys/dev/twa/tw_osl_cam.c
  stable/9/sys/dev/twa/tw_osl_freebsd.c
  stable/9/sys/dev/tws/tws.h
  stable/9/sys/dev/tws/tws_cam.c
  stable/9/sys/dev/virtio/scsi/virtio_scsi.c
  stable/9/sys/dev/wds/wd7000.c
  stable/9/sys/ia64/ia64/busdma_machdep.c
  stable/9/sys/kern/subr_uio.c
  stable/9/sys/mips/mips/busdma_machdep.c
  stable/9/sys/modules/cam/Makefile
  stable/9/sys/powerpc/powerpc/busdma_machdep.c
  stable/9/sys/powerpc/ps3/ps3cdrom.c
  stable/9/sys/sparc64/include/bus_dma.h
  stable/9/sys/sparc64/sparc64/bus_machdep.c
  stable/9/sys/sparc64/sparc64/iommu.c
  stable/9/sys/sys/bus_dma.h
  stable/9/sys/sys/uio.h
  stable/9/sys/x86/x86/busdma_machdep.c
Directory Properties:
  stable/9/sys/   (props changed)
  stable/9/sys/conf/   (props changed)
  stable/9/sys/dev/   (props changed)
  stable/9/sys/dev/isp/   (props changed)
  stable/9/sys/modules/   (props changed)
  stable/9/sys/sys/   (props changed)

Modified: stable/9/sys/arm/arm/busdma_machdep.c
==============================================================================
--- stable/9/sys/arm/arm/busdma_machdep.c       Mon Jun 17 23:21:18 2013        
(r251873)
+++ stable/9/sys/arm/arm/busdma_machdep.c       Tue Jun 18 00:36:53 2013        
(r251874)
@@ -61,12 +61,12 @@ __FBSDID("$FreeBSD$");
 #include <sys/interrupt.h>
 #include <sys/lock.h>
 #include <sys/proc.h>
+#include <sys/memdesc.h>
 #include <sys/mutex.h>
-#include <sys/mbuf.h>
-#include <sys/uio.h>
 #include <sys/ktr.h>
 #include <sys/kernel.h>
 #include <sys/sysctl.h>
+#include <sys/uio.h>
 
 #include <vm/uma.h>
 #include <vm/vm.h>
@@ -125,10 +125,17 @@ struct bounce_page {
        vm_offset_t     vaddr_nocache;  /* kva of bounce buffer uncached */
        bus_addr_t      busaddr;        /* Physical address */
        vm_offset_t     datavaddr;      /* kva of client data */
+       bus_addr_t      dataaddr;       /* client physical address */
        bus_size_t      datacount;      /* client data count */
        STAILQ_ENTRY(bounce_page) links;
 };
 
+struct sync_list {
+       vm_offset_t     vaddr;          /* kva of bounce buffer */
+       bus_addr_t      busaddr;        /* Physical address */
+       bus_size_t      datacount;      /* client data count */
+};
+
 int busdma_swi_pending;
 
 struct bounce_zone {
@@ -158,24 +165,21 @@ static SYSCTL_NODE(_hw, OID_AUTO, busdma
 SYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bpages, 0,
           "Total bounce pages");
 
-#define DMAMAP_LINEAR          0x1
-#define DMAMAP_MBUF            0x2
-#define DMAMAP_UIO             0x4
-#define DMAMAP_CACHE_ALIGNED   0x10
-#define DMAMAP_TYPE_MASK       (DMAMAP_LINEAR|DMAMAP_MBUF|DMAMAP_UIO)
 #define DMAMAP_COHERENT                0x8
+#define DMAMAP_CACHE_ALIGNED   0x10
+
 struct bus_dmamap {
        struct bp_list  bpages;
        int             pagesneeded;
        int             pagesreserved;
         bus_dma_tag_t  dmat;
+       struct memdesc  mem;
        int             flags;
-       void            *buffer;
-       int             len;
        STAILQ_ENTRY(bus_dmamap) links;
        bus_dmamap_callback_t *callback;
        void                  *callback_arg;
-
+       int                    sync_count;
+       struct sync_list       *slist;
 };
 
 static STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist;
@@ -191,7 +195,8 @@ static int alloc_bounce_pages(bus_dma_ta
 static int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map,
                                int commit);
 static bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map,
-                                  vm_offset_t vaddr, bus_size_t size);
+                                 vm_offset_t vaddr, bus_addr_t addr,
+                                 bus_size_t size);
 static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage);
 
 /* Default tag, as most drivers provide no parent tag. */
@@ -564,13 +569,20 @@ bus_dma_tag_destroy(bus_dma_tag_t dmat)
 int
 bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp)
 {
+       struct sync_list *slist;
        bus_dmamap_t map;
        int error = 0;
 
+       slist = malloc(sizeof(*slist) * dmat->nsegments, M_DEVBUF, M_NOWAIT);
+       if (slist == NULL)
+               return (ENOMEM);
+
        map = uma_zalloc_arg(dmamap_zone, dmat, M_NOWAIT);
        *mapp = map;
-       if (map == NULL)
+       if (map == NULL) {
+               free(slist, M_DEVBUF);
                return (ENOMEM);
+       }
 
        /*
         * If the tag's segments haven't been allocated yet we need to do it
@@ -580,6 +592,7 @@ bus_dmamap_create(bus_dma_tag_t dmat, in
                dmat->segments = malloc(dmat->nsegments * 
                    sizeof(*dmat->segments), M_DEVBUF, M_NOWAIT);
                if (dmat->segments == NULL) {
+                       free(slist, M_DEVBUF);
                        uma_zfree(dmamap_zone, map);
                        *mapp = NULL;
                        return (ENOMEM);
@@ -599,6 +612,7 @@ bus_dmamap_create(bus_dma_tag_t dmat, in
 
                if (dmat->bounce_zone == NULL) {
                        if ((error = alloc_bounce_zone(dmat)) != 0) {
+                               free(slist, M_DEVBUF);
                                uma_zfree(dmamap_zone, map);
                                *mapp = NULL;
                                return (error);
@@ -633,6 +647,8 @@ bus_dmamap_create(bus_dma_tag_t dmat, in
                }
                bz->map_count++;
        }
+       map->sync_count = 0;
+       map->slist = slist;
        CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
            __func__, dmat, dmat->flags, error);
 
@@ -647,11 +663,12 @@ int
 bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map)
 {
 
-       if (STAILQ_FIRST(&map->bpages) != NULL) {
+       if (STAILQ_FIRST(&map->bpages) != NULL || map->sync_count != 0) {
                CTR3(KTR_BUSDMA, "%s: tag %p error %d",
                    __func__, dmat, EBUSY);
                return (EBUSY);
        }
+       free(map->slist, M_DEVBUF);
        uma_zfree(dmamap_zone, map);
        if (dmat->bounce_zone)
                dmat->bounce_zone->map_count--;
@@ -668,6 +685,7 @@ int
 bus_dmamem_alloc(bus_dma_tag_t dmat, void **vaddrp, int flags,
                  bus_dmamap_t *mapp)
 {
+       struct sync_list *slist;
        void * vaddr;
        struct busdma_bufzone *bufzone;
        busdma_bufalloc_t ba;
@@ -679,7 +697,6 @@ bus_dmamem_alloc(bus_dma_tag_t dmat, voi
                mflags = M_NOWAIT;
        else
                mflags = M_WAITOK;
-
        /*
         * If the tag's segments haven't been allocated yet we need to do it
         * now, because we can't sleep for resources at map load time.
@@ -688,10 +705,14 @@ bus_dmamem_alloc(bus_dma_tag_t dmat, voi
                dmat->segments = malloc(dmat->nsegments * 
                   sizeof(*dmat->segments), M_DEVBUF, mflags);
 
+       slist = malloc(sizeof(*slist) * dmat->nsegments, M_DEVBUF, M_NOWAIT);
+       if (slist == NULL)
+               return (ENOMEM);
        map = uma_zalloc_arg(dmamap_zone, dmat, mflags);
-       if (map == NULL)
+       if (map == NULL) {
+               free(slist, M_DEVBUF);
                return (ENOMEM);
-                       
+       }
        if (flags & BUS_DMA_COHERENT) {
                memattr = VM_MEMATTR_UNCACHEABLE;
                ba = coherent_allocator;
@@ -738,12 +759,14 @@ bus_dmamem_alloc(bus_dma_tag_t dmat, voi
                    mflags, 0, dmat->lowaddr, dmat->alignment, dmat->boundary,
                    memattr);
        }
-
        if (vaddr == NULL) {
+               free(slist, M_DEVBUF);
                uma_zfree(dmamap_zone, map);
                map = NULL;
+       } else {
+               map->slist = slist;
+               map->sync_count = 0;
        }
-
        *vaddrp = vaddr;
        *mapp = map;
 
@@ -762,10 +785,11 @@ bus_dmamem_free(bus_dma_tag_t dmat, void
 
        if (map->flags & DMAMAP_COHERENT)
                ba = coherent_allocator;
-        else
+       else
                ba = standard_allocator;
-        uma_zfree(dmamap_zone, map);
+       uma_zfree(dmamap_zone, map);
 
+       free(map->slist, M_DEVBUF);
        /* Be careful not to access map from here on. */
 
        bufzone = busdma_bufalloc_findzone(ba, dmat->maxsize);
@@ -777,7 +801,37 @@ bus_dmamem_free(bus_dma_tag_t dmat, void
                kmem_free(kernel_map, (vm_offset_t)vaddr, dmat->maxsize);
 }
 
-static int
+static void
+_bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf,
+    bus_size_t buflen, int flags)
+{
+       bus_addr_t curaddr;
+       bus_size_t sgsize;
+
+       if ((map->pagesneeded == 0)) {
+               CTR3(KTR_BUSDMA, "lowaddr= %d, boundary= %d, alignment= %d",
+                   dmat->lowaddr, dmat->boundary, dmat->alignment);
+               CTR2(KTR_BUSDMA, "map= %p, pagesneeded= %d",
+                   map, map->pagesneeded);
+               /*
+                * Count the number of bounce pages
+                * needed in order to complete this transfer
+                */
+               curaddr = buf;
+               while (buflen != 0) {
+                       sgsize = MIN(buflen, dmat->maxsegsz);
+                       if (run_filter(dmat, curaddr) != 0) {
+                               sgsize = MIN(sgsize, PAGE_SIZE);
+                               map->pagesneeded++;
+                       }
+                       curaddr += sgsize;
+                       buflen -= sgsize;
+               }
+               CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded);
+       }
+}
+
+static void
 _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap,
     void *buf, bus_size_t buflen, int flags)
 {
@@ -798,75 +852,183 @@ _bus_dmamap_count_pages(bus_dma_tag_t dm
                vendaddr = (vm_offset_t)buf + buflen;
 
                while (vaddr < vendaddr) {
-                       if (__predict_true(pmap == pmap_kernel()))
+                       if (__predict_true(pmap == kernel_pmap))
                                paddr = pmap_kextract(vaddr);
                        else
                                paddr = pmap_extract(pmap, vaddr);
-                       if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) &&
-                           run_filter(dmat, paddr) != 0)
+                       if (run_filter(dmat, paddr) != 0)
                                map->pagesneeded++;
                        vaddr += PAGE_SIZE;
                }
                CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded);
        }
+}
+
+static int
+_bus_dmamap_reserve_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int flags)
+{
 
        /* Reserve Necessary Bounce Pages */
-       if (map->pagesneeded != 0) {
-               mtx_lock(&bounce_lock);
-               if (flags & BUS_DMA_NOWAIT) {
-                       if (reserve_bounce_pages(dmat, map, 0) != 0) {
-                               mtx_unlock(&bounce_lock);
-                               return (ENOMEM);
-                       }
-               } else {
-                       if (reserve_bounce_pages(dmat, map, 1) != 0) {
-                               /* Queue us for resources */
-                               STAILQ_INSERT_TAIL(&bounce_map_waitinglist,
-                                   map, links);
-                               mtx_unlock(&bounce_lock);
-                               return (EINPROGRESS);
-                       }
+       mtx_lock(&bounce_lock);
+       if (flags & BUS_DMA_NOWAIT) {
+               if (reserve_bounce_pages(dmat, map, 0) != 0) {
+                       mtx_unlock(&bounce_lock);
+                       return (ENOMEM);
+               }
+       } else {
+               if (reserve_bounce_pages(dmat, map, 1) != 0) {
+                       /* Queue us for resources */
+                       STAILQ_INSERT_TAIL(&bounce_map_waitinglist, map, links);
+                       mtx_unlock(&bounce_lock);
+                       return (EINPROGRESS);
                }
-               mtx_unlock(&bounce_lock);
        }
+       mtx_unlock(&bounce_lock);
+
+       return (0);
+}
 
+/*
+ * Add a single contiguous physical range to the segment list.
+ */
+static int
+_bus_dmamap_addseg(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t curaddr,
+    bus_size_t sgsize, bus_dma_segment_t *segs, int *segp)
+{
+       bus_addr_t baddr, bmask;
+       int seg;
+
+       /*
+        * Make sure we don't cross any boundaries.
+        */
+       bmask = ~(dmat->boundary - 1);
+       if (dmat->boundary > 0) {
+               baddr = (curaddr + dmat->boundary) & bmask;
+               if (sgsize > (baddr - curaddr))
+                       sgsize = (baddr - curaddr);
+       }
+       if (dmat->ranges) {
+               struct arm32_dma_range *dr;
+
+               dr = _bus_dma_inrange(dmat->ranges, dmat->_nranges,
+                   curaddr);
+               if (dr == NULL)
+                       return (EINVAL);
+               /*
+                * In a valid DMA range.  Translate the physical
+                * memory address to an address in the DMA window.
+                */
+               curaddr = (curaddr - dr->dr_sysbase) + dr->dr_busbase;
+                                       
+       }
+
+       seg = *segp;
+       /*
+        * Insert chunk into a segment, coalescing with
+        * the previous segment if possible.
+        */
+       if (seg >= 0 &&
+           curaddr == segs[seg].ds_addr + segs[seg].ds_len &&
+           (segs[seg].ds_len + sgsize) <= dmat->maxsegsz &&
+           (dmat->boundary == 0 ||
+            (segs[seg].ds_addr & bmask) == (curaddr & bmask))) {
+               segs[seg].ds_len += sgsize;
+       } else {
+               if (++seg >= dmat->nsegments)
+                       return (EFBIG);
+               segs[seg].ds_addr = curaddr;
+               segs[seg].ds_len = sgsize;
+       }
+       *segp = seg;
        return (0);
 }
 
 /*
- * Utility function to load a linear buffer.  lastaddrp holds state
- * between invocations (for multiple-buffer loads).  segp contains
+ * Utility function to load a physical buffer.  segp contains
+ * the starting segment on entrace, and the ending segment on exit.
+ */
+int
+_bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf,
+    bus_size_t buflen, int flags, bus_dma_segment_t *segs, int *segp)
+{
+       bus_size_t sgsize;
+       bus_addr_t curaddr;
+       int error;
+
+       if (segs == NULL)
+               segs = dmat->segments;
+
+       if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) {
+               _bus_dmamap_count_phys(dmat, map, buf, buflen, flags);
+               if (map->pagesneeded != 0) {
+                       error = _bus_dmamap_reserve_pages(dmat, map, flags);
+                       if (error)
+                               return (error);
+               }
+       }
+
+       while (buflen > 0) {
+               curaddr = buf;
+               sgsize = MIN(buflen, dmat->maxsegsz);
+               if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) &&
+                   map->pagesneeded != 0 && run_filter(dmat, curaddr)) {
+                       sgsize = MIN(sgsize, PAGE_SIZE);
+                       curaddr = add_bounce_page(dmat, map, 0, curaddr,
+                           sgsize);
+               }
+               sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs,
+                   segp);
+               if (sgsize == 0)
+                       break;
+               buf += sgsize;
+               buflen -= sgsize;
+       }
+
+       /*
+        * Did we fit?
+        */
+       if (buflen != 0) {
+               _bus_dmamap_unload(dmat, map);
+               return (EFBIG); /* XXX better return value here? */
+       }
+       return (0);
+}
+/*
+ * Utility function to load a linear buffer.  segp contains
  * the starting segment on entrance, and the ending segment on exit.
- * first indicates if this is the first invocation of this function.
  */
-static __inline int
-bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dma_segment_t *segs,
-    bus_dmamap_t map, void *buf, bus_size_t buflen, struct pmap *pmap,
-    int flags, vm_offset_t *lastaddrp, int *segp)
+int
+_bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf,
+    bus_size_t buflen, struct pmap *pmap, int flags, bus_dma_segment_t *segs,
+    int *segp)
 {
        bus_size_t sgsize;
-       bus_addr_t curaddr, lastaddr, baddr, bmask;
+       bus_addr_t curaddr;
+       struct sync_list *sl;
        vm_offset_t vaddr = (vm_offset_t)buf;
-       int seg;
        int error = 0;
 
-       lastaddr = *lastaddrp;
-       bmask = ~(dmat->boundary - 1);
+       if (segs == NULL)
+               segs = dmat->segments;
+       if ((flags & BUS_DMA_LOAD_MBUF) != 0)
+               map->flags |= DMAMAP_CACHE_ALIGNED;
 
        if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) {
-               error = _bus_dmamap_count_pages(dmat, map, pmap, buf, buflen,
-                   flags);
-               if (error)
-                       return (error);
+               _bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, flags);
+               if (map->pagesneeded != 0) {
+                       error = _bus_dmamap_reserve_pages(dmat, map, flags);
+                       if (error)
+                               return (error);
+               }
        }
        CTR3(KTR_BUSDMA, "lowaddr= %d boundary= %d, "
            "alignment= %d", dmat->lowaddr, dmat->boundary, dmat->alignment);
 
-       for (seg = *segp; buflen > 0 ; ) {
+       while (buflen > 0) {
                /*
                 * Get the physical address for this segment.
                 */
-               if (__predict_true(pmap == pmap_kernel())) {
+               if (__predict_true(pmap == kernel_pmap)) {
                        curaddr = pmap_kextract(vaddr);
                } else {
                        curaddr = pmap_extract(pmap, vaddr);
@@ -882,260 +1044,63 @@ bus_dmamap_load_buffer(bus_dma_tag_t dma
                if (buflen < sgsize)
                        sgsize = buflen;
 
-               /*
-                * Make sure we don't cross any boundaries.
-                */
-               if (dmat->boundary > 0) {
-                       baddr = (curaddr + dmat->boundary) & bmask;
-                       if (sgsize > (baddr - curaddr))
-                               sgsize = (baddr - curaddr);
-               }
                if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) &&
-                   map->pagesneeded != 0 && run_filter(dmat, curaddr))
-                       curaddr = add_bounce_page(dmat, map, vaddr, sgsize);
-
-               if (dmat->ranges) {
-                       struct arm32_dma_range *dr;
-
-                       dr = _bus_dma_inrange(dmat->ranges, dmat->_nranges,
-                           curaddr);
-                       if (dr == NULL)
-                               return (EINVAL);
-                       /*
-                        * In a valid DMA range.  Translate the physical
-                        * memory address to an address in the DMA window.
-                        */
-                       curaddr = (curaddr - dr->dr_sysbase) + dr->dr_busbase;
-                                               
-               }
-
-               /*
-                * Insert chunk into a segment, coalescing with
-                * the previous segment if possible.
-                */
-               if (seg >= 0 && curaddr == lastaddr &&
-                   (segs[seg].ds_len + sgsize) <= dmat->maxsegsz &&
-                   (dmat->boundary == 0 ||
-                    (segs[seg].ds_addr & bmask) ==
-                    (curaddr & bmask))) {
-                       segs[seg].ds_len += sgsize;
-                       goto segdone;
+                   map->pagesneeded != 0 && run_filter(dmat, curaddr)) {
+                       curaddr = add_bounce_page(dmat, map, vaddr, curaddr,
+                           sgsize);
                } else {
-                       if (++seg >= dmat->nsegments)
-                               break;
-                       segs[seg].ds_addr = curaddr;
-                       segs[seg].ds_len = sgsize;
-               }
-               if (error)
+                       sl = &map->slist[map->sync_count - 1];
+                       if (map->sync_count == 0 ||
+                           vaddr != sl->vaddr + sl->datacount) {
+                               if (++map->sync_count > dmat->nsegments)
+                                       goto cleanup;
+                               sl++;
+                               sl->vaddr = vaddr;
+                               sl->datacount = sgsize;
+                               sl->busaddr = curaddr;
+                       } else
+                               sl->datacount += sgsize;
+               }
+               sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs,
+                   segp);
+               if (sgsize == 0)
                        break;
-segdone:
-               lastaddr = curaddr + sgsize;
                vaddr += sgsize;
                buflen -= sgsize;
        }
 
-       *segp = seg;
-       *lastaddrp = lastaddr;
-
+cleanup:
        /*
         * Did we fit?
         */
-       if (buflen != 0)
-               error = EFBIG; /* XXX better return value here? */
-       return (error);
+       if (buflen != 0) {
+               _bus_dmamap_unload(dmat, map);
+               return (EFBIG); /* XXX better return value here? */
+       }
+       return (0);
 }
 
-/*
- * Map the buffer buf into bus space using the dmamap map.
- */
-int
-bus_dmamap_load(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf,
-                bus_size_t buflen, bus_dmamap_callback_t *callback,
-                void *callback_arg, int flags)
+void
+__bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map,
+                   struct memdesc *mem, bus_dmamap_callback_t *callback,
+                   void *callback_arg)
 {
-       vm_offset_t     lastaddr = 0;
-       int             error, nsegs = -1;
 
        KASSERT(dmat != NULL, ("dmatag is NULL"));
        KASSERT(map != NULL, ("dmamap is NULL"));
+       map->mem = *mem;
        map->callback = callback;
        map->callback_arg = callback_arg;
-       map->flags &= ~DMAMAP_TYPE_MASK;
-       map->flags |= DMAMAP_LINEAR;
-       map->buffer = buf;
-       map->len = buflen;
-       error = bus_dmamap_load_buffer(dmat,
-           dmat->segments, map, buf, buflen, kernel_pmap,
-           flags, &lastaddr, &nsegs);
-       if (error == EINPROGRESS)
-               return (error);
-       if (error)
-               (*callback)(callback_arg, NULL, 0, error);
-       else
-               (*callback)(callback_arg, dmat->segments, nsegs + 1, error);
-       
-       CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d",
-           __func__, dmat, dmat->flags, nsegs + 1, error);
-
-       return (error);
 }
 
-/*
- * Like bus_dmamap_load(), but for mbufs.
- *
- * Note that the manpage states that BUS_DMA_NOWAIT is implied for mbufs.
- *
- * We know that the way the system allocates and uses mbufs implies that we can
- * treat them as DMAMAP_CACHE_ALIGNED in terms of handling partial cache line
- * flushes. Even though the flush may reference the data area within the mbuf
- * that isn't aligned to a cache line, we know the overall mbuf itself is
- * properly aligned, and we know that the CPU will not touch the header fields
- * before the data area while the DMA is in progress.
- */
-int
-bus_dmamap_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf *m0,
-                    bus_dmamap_callback2_t *callback, void *callback_arg,
-                    int flags)
-{
-       int nsegs = -1, error = 0;
-
-       M_ASSERTPKTHDR(m0);
-
-       flags |= BUS_DMA_NOWAIT;
-       map->flags &= ~DMAMAP_TYPE_MASK;
-       map->flags |= DMAMAP_MBUF | DMAMAP_CACHE_ALIGNED;
-       map->buffer = m0;
-       map->len = 0;
-       if (m0->m_pkthdr.len <= dmat->maxsize) {
-               vm_offset_t lastaddr = 0;
-               struct mbuf *m;
-
-               for (m = m0; m != NULL && error == 0; m = m->m_next) {
-                       if (m->m_len > 0) {
-                               error = bus_dmamap_load_buffer(dmat,
-                                   dmat->segments, map, m->m_data, m->m_len,
-                                   pmap_kernel(), flags, &lastaddr, &nsegs);
-                               map->len += m->m_len;
-                       }
-               }
-       } else {
-               error = EINVAL;
-       }
-
-       if (error) {
-               /*
-                * force "no valid mappings" on error in callback.
-                */
-               (*callback)(callback_arg, NULL, 0, 0, error);
-       } else {
-               (*callback)(callback_arg, dmat->segments, nsegs + 1,
-                   m0->m_pkthdr.len, error);
-       }
-       CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d",
-           __func__, dmat, dmat->flags, error, nsegs + 1);
-
-       return (error);
-}
-
-int
-bus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map,
-                       struct mbuf *m0, bus_dma_segment_t *segs, int *nsegs,
-                       int flags)
+bus_dma_segment_t *
+_bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map,
+                    bus_dma_segment_t *segs, int nsegs, int error)
 {
-       int error = 0;
-       M_ASSERTPKTHDR(m0);
-
-       flags |= BUS_DMA_NOWAIT;
-       *nsegs = -1;
-       map->flags &= ~DMAMAP_TYPE_MASK;
-       map->flags |= DMAMAP_MBUF | DMAMAP_CACHE_ALIGNED;
-       map->buffer = m0;                       
-       map->len = 0;
-       if (m0->m_pkthdr.len <= dmat->maxsize) {
-               vm_offset_t lastaddr = 0;
-               struct mbuf *m;
-
-               for (m = m0; m != NULL && error == 0; m = m->m_next) {
-                       if (m->m_len > 0) {
-                               error = bus_dmamap_load_buffer(dmat, segs, map,
-                                               m->m_data, m->m_len,
-                                               pmap_kernel(), flags, &lastaddr,
-                                               nsegs);
-                               map->len += m->m_len;
-                       }
-               }
-       } else {
-               error = EINVAL;
-       }
-
-       /* XXX FIXME: Having to increment nsegs is really annoying */
-       ++*nsegs;
-       CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d",
-           __func__, dmat, dmat->flags, error, *nsegs);
-       return (error);
-}
-
-/*
- * Like bus_dmamap_load(), but for uios.
- */
-int
-bus_dmamap_load_uio(bus_dma_tag_t dmat, bus_dmamap_t map, struct uio *uio,
-    bus_dmamap_callback2_t *callback, void *callback_arg,
-    int flags)
-{
-       vm_offset_t lastaddr = 0;
-       int nsegs, i, error;
-       bus_size_t resid;
-       struct iovec *iov;
-       struct pmap *pmap;
-
-       resid = uio->uio_resid;
-       iov = uio->uio_iov;
-       map->flags &= ~DMAMAP_TYPE_MASK;
-       map->flags |= DMAMAP_UIO;
-       map->buffer = uio;
-       map->len = 0;
-
-       if (uio->uio_segflg == UIO_USERSPACE) {
-               KASSERT(uio->uio_td != NULL,
-                   ("bus_dmamap_load_uio: USERSPACE but no proc"));
-               pmap = vmspace_pmap(uio->uio_td->td_proc->p_vmspace);
-       } else
-               pmap = kernel_pmap;
-
-       error = 0;
-       nsegs = -1;
-       for (i = 0; i < uio->uio_iovcnt && resid != 0 && !error; i++) {
-               /*
-                * Now at the first iovec to load.  Load each iovec
-                * until we have exhausted the residual count.
-                */
-               bus_size_t minlen =
-                   resid < iov[i].iov_len ? resid : iov[i].iov_len;
-               caddr_t addr = (caddr_t) iov[i].iov_base;
-
-               if (minlen > 0) {
-                       error = bus_dmamap_load_buffer(dmat, dmat->segments, 
-                           map, addr, minlen, pmap, flags, &lastaddr, &nsegs);
-
-                       map->len += minlen;
-                       resid -= minlen;
-               }
-       }
-
-       if (error) {
-               /*
-                * force "no valid mappings" on error in callback.
-                */
-               (*callback)(callback_arg, NULL, 0, 0, error);
-       } else {
-               (*callback)(callback_arg, dmat->segments, nsegs+1,
-                   uio->uio_resid, error);
-       }
 
-       CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d",
-           __func__, dmat, dmat->flags, error, nsegs + 1);
-       return (error);
+       if (segs == NULL)
+               segs = dmat->segments;
+       return (segs);
 }
 
 /*
@@ -1146,24 +1111,25 @@ _bus_dmamap_unload(bus_dma_tag_t dmat, b
 {
        struct bounce_page *bpage;
 
-       map->flags &= ~DMAMAP_TYPE_MASK;
        while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) {
                STAILQ_REMOVE_HEAD(&map->bpages, links);
                free_bounce_page(dmat, bpage);
        }
+       map->sync_count = 0;
        return;
 }
 
 static void
-bus_dmamap_sync_buf(void *buf, int len, bus_dmasync_op_t op, int bufaligned)
+bus_dmamap_sync_buf(vm_offset_t buf, int len, bus_dmasync_op_t op,
+    int bufaligned)
 {
        char _tmp_cl[arm_dcache_align], _tmp_clend[arm_dcache_align];
        register_t s;
        int partial;
 
        if ((op & BUS_DMASYNC_PREWRITE) && !(op & BUS_DMASYNC_PREREAD)) {
-               cpu_dcache_wb_range((vm_offset_t)buf, len);
-               cpu_l2cache_wb_range((vm_offset_t)buf, len);
+               cpu_dcache_wb_range(buf, len);
+               cpu_l2cache_wb_range(buf, len);
        }
 
        /*
@@ -1186,38 +1152,37 @@ bus_dmamap_sync_buf(void *buf, int len, 
 
        if (op & BUS_DMASYNC_PREREAD) {
                if (!(op & BUS_DMASYNC_PREWRITE) && !partial) {
-                       cpu_dcache_inv_range((vm_offset_t)buf, len);
-                       cpu_l2cache_inv_range((vm_offset_t)buf, len);
+                       cpu_dcache_inv_range(buf, len);
+                       cpu_l2cache_inv_range(buf, len);
                } else {
-                       cpu_dcache_wbinv_range((vm_offset_t)buf, len);
-                       cpu_l2cache_wbinv_range((vm_offset_t)buf, len);
+                       cpu_dcache_wbinv_range(buf, len);
+                       cpu_l2cache_wbinv_range(buf, len);
                }
        }
        if (op & BUS_DMASYNC_POSTREAD) {
                if (partial && !bufaligned) {
                        s = intr_disable();
-                       if ((vm_offset_t)buf & arm_dcache_align_mask)
-                               memcpy(_tmp_cl, (void *)((vm_offset_t)buf &
+                       if (buf & arm_dcache_align_mask)
+                               memcpy(_tmp_cl, (void *)(buf &
                                    ~arm_dcache_align_mask),
-                                   (vm_offset_t)buf & arm_dcache_align_mask);
-                       if (((vm_offset_t)buf + len) & arm_dcache_align_mask)
+                                   buf & arm_dcache_align_mask);
+                       if ((buf + len) & arm_dcache_align_mask)
                                memcpy(_tmp_clend,
-                                   (void *)((vm_offset_t)buf + len),
-                                   arm_dcache_align - (((vm_offset_t)(buf) +
-                                   len) & arm_dcache_align_mask));
+                                   (void *)(buf + len),
+                                   arm_dcache_align -
+                                   ((buf + len) & arm_dcache_align_mask));
                }
-               cpu_dcache_inv_range((vm_offset_t)buf, len);
-               cpu_l2cache_inv_range((vm_offset_t)buf, len);
+               cpu_dcache_inv_range(buf, len);
+               cpu_l2cache_inv_range(buf, len);
                if (partial && !bufaligned) {
-                       if ((vm_offset_t)buf & arm_dcache_align_mask)
-                               memcpy((void *)((vm_offset_t)buf &
+                       if (buf & arm_dcache_align_mask)
+                               memcpy((void *)(buf &
                                    ~arm_dcache_align_mask), _tmp_cl,
-                                   (vm_offset_t)buf & arm_dcache_align_mask);
-                       if (((vm_offset_t)buf + len) & arm_dcache_align_mask)
-                               memcpy((void *)((vm_offset_t)buf + len),
+                                   buf & arm_dcache_align_mask);
+                       if ((buf + len) & arm_dcache_align_mask)
+                               memcpy((void *)(buf + len),
                                    _tmp_clend, arm_dcache_align -
-                                   (((vm_offset_t)(buf) + len) &
-                                   arm_dcache_align_mask));
+                                   ((buf + len) & arm_dcache_align_mask));
                        intr_restore(s);
                }
        }
@@ -1230,10 +1195,18 @@ _bus_dmamap_sync_bp(bus_dma_tag_t dmat, 
 
        STAILQ_FOREACH(bpage, &map->bpages, links) {
                if (op & BUS_DMASYNC_PREWRITE) {
-                       bcopy((void *)bpage->datavaddr,
-                           (void *)(bpage->vaddr_nocache != 0 ?
-                                    bpage->vaddr_nocache : bpage->vaddr),
-                           bpage->datacount);
+                       if (bpage->datavaddr != 0)
+                               bcopy((void *)bpage->datavaddr,
+                                   (void *)(bpage->vaddr_nocache != 0 ?
+                                            bpage->vaddr_nocache :
+                                            bpage->vaddr),
+                                   bpage->datacount);
+                       else
+                               physcopyout(bpage->dataaddr,
+                                   (void *)(bpage->vaddr_nocache != 0 ?
+                                            bpage->vaddr_nocache :
+                                            bpage->vaddr),
+                                   bpage->datacount);
                        if (bpage->vaddr_nocache == 0) {
                                cpu_dcache_wb_range(bpage->vaddr,
                                    bpage->datacount);
@@ -1249,36 +1222,23 @@ _bus_dmamap_sync_bp(bus_dma_tag_t dmat, 
                                cpu_l2cache_inv_range(bpage->vaddr,
                                    bpage->datacount);
                        }
-                       bcopy((void *)(bpage->vaddr_nocache != 0 ?
-                           bpage->vaddr_nocache : bpage->vaddr),
-                           (void *)bpage->datavaddr, bpage->datacount);
+                       if (bpage->datavaddr != 0)
+                               bcopy((void *)(bpage->vaddr_nocache != 0 ?
+                                   bpage->vaddr_nocache : bpage->vaddr),
+                                   (void *)bpage->datavaddr, bpage->datacount);
+                       else
+                               physcopyin((void *)(bpage->vaddr_nocache != 0 ?
+                                   bpage->vaddr_nocache : bpage->vaddr),
+                                   bpage->dataaddr, bpage->datacount);
                        dmat->bounce_zone->total_bounced++;
                }
        }
 }
 
-static __inline int
-_bus_dma_buf_is_in_bp(bus_dmamap_t map, void *buf, int len)
-{
-       struct bounce_page *bpage;
-
-       STAILQ_FOREACH(bpage, &map->bpages, links) {
-               if ((vm_offset_t)buf >= bpage->datavaddr &&
-                   (vm_offset_t)buf + len <= bpage->datavaddr +
-                   bpage->datacount)
-                       return (1);
-       }
-       return (0);
-
-}
-
 void
 _bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
 {
-       struct mbuf *m;
-       struct uio *uio;
-       int resid;
-       struct iovec *iov;
+       struct sync_list *sl, *end;
        int bufaligned;
 
        if (op == BUS_DMASYNC_POSTWRITE)
@@ -1289,40 +1249,11 @@ _bus_dmamap_sync(bus_dma_tag_t dmat, bus
                _bus_dmamap_sync_bp(dmat, map, op);
        CTR3(KTR_BUSDMA, "%s: op %x flags %x", __func__, op, map->flags);
        bufaligned = (map->flags & DMAMAP_CACHE_ALIGNED);
-       switch(map->flags & DMAMAP_TYPE_MASK) {
-       case DMAMAP_LINEAR:
-               if (!(_bus_dma_buf_is_in_bp(map, map->buffer, map->len)))
-                       bus_dmamap_sync_buf(map->buffer, map->len, op, 
+       if (map->sync_count) {
+               end = &map->slist[map->sync_count];
+               for (sl = &map->slist[0]; sl != end; sl++)
+                       bus_dmamap_sync_buf(sl->vaddr, sl->datacount, op,
                            bufaligned);
-               break;
-       case DMAMAP_MBUF:
-               m = map->buffer;
-               while (m) {
-                       if (m->m_len > 0 &&
-                           !(_bus_dma_buf_is_in_bp(map, m->m_data, m->m_len)))
-                               bus_dmamap_sync_buf(m->m_data, m->m_len, op,
-                               bufaligned);
-                       m = m->m_next;
-               }
-               break;
-       case DMAMAP_UIO:
-               uio = map->buffer;
-               iov = uio->uio_iov;
-               resid = uio->uio_resid;
-               for (int i = 0; i < uio->uio_iovcnt && resid != 0; i++) {
-                       bus_size_t minlen = resid < iov[i].iov_len ? resid :
-                           iov[i].iov_len;
-                       if (minlen > 0) {
-                               if (!_bus_dma_buf_is_in_bp(map, iov[i].iov_base,
-                                   minlen))
-                                       bus_dmamap_sync_buf(iov[i].iov_base,
-                                           minlen, op, bufaligned);
-                               resid -= minlen;
-                       }
-               }
-               break;
-       default:
-               break;
        }
 
 drain:
@@ -1489,7 +1420,7 @@ reserve_bounce_pages(bus_dma_tag_t dmat,
 
 static bus_addr_t
 add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr,
-               bus_size_t size)
+               bus_addr_t addr, bus_size_t size)
 {
        struct bounce_zone *bz;
        struct bounce_page *bpage;
@@ -1522,6 +1453,7 @@ add_bounce_page(bus_dma_tag_t dmat, bus_
                bpage->busaddr |= vaddr & PAGE_MASK;
        }
        bpage->datavaddr = vaddr;
+       bpage->dataaddr = addr;
        bpage->datacount = size;
        STAILQ_INSERT_TAIL(&(map->bpages), bpage, links);
        return (bpage->busaddr);
@@ -1575,8 +1507,8 @@ busdma_swi(void)
                mtx_unlock(&bounce_lock);
                dmat = map->dmat;
                (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_LOCK);
-               bus_dmamap_load(map->dmat, map, map->buffer, map->len,
-                   map->callback, map->callback_arg, /*flags*/0);
+               bus_dmamap_load_mem(map->dmat, map, &map->mem,
+                   map->callback, map->callback_arg, BUS_DMA_WAITOK);
                (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_UNLOCK);
                mtx_lock(&bounce_lock);
        }

Modified: stable/9/sys/cam/cam_ccb.h
==============================================================================
--- stable/9/sys/cam/cam_ccb.h  Mon Jun 17 23:21:18 2013        (r251873)
+++ stable/9/sys/cam/cam_ccb.h  Tue Jun 18 00:36:53 2013        (r251874)
@@ -64,13 +64,19 @@ typedef enum {
                                              * Perform transport negotiation
                                              * with this command.
                                              */
-       CAM_SCATTER_VALID       = 0x00000010,/* Scatter/gather list is valid  */
+       CAM_DATA_ISPHYS         = 0x00000010,/* Data type with physical addrs */
        CAM_DIS_AUTOSENSE       = 0x00000020,/* Disable autosense feature     */
        CAM_DIR_BOTH            = 0x00000000,/* Data direction (00:IN/OUT)    */
        CAM_DIR_IN              = 0x00000040,/* Data direction (01:DATA IN)   */
        CAM_DIR_OUT             = 0x00000080,/* Data direction (10:DATA OUT)  */
        CAM_DIR_NONE            = 0x000000C0,/* Data direction (11:no data)   */
        CAM_DIR_MASK            = 0x000000C0,/* Data direction Mask           */
+       CAM_DATA_VADDR          = 0x00000000,/* Data type (000:Virtual)       */

*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
_______________________________________________
svn-src-all@freebsd.org mailing list
http://lists.freebsd.org/mailman/listinfo/svn-src-all
To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"

Reply via email to