Author: cperciva
Date: Wed Dec 8 19:42:21 2010
New Revision: 216304
URL: http://svn.freebsd.org/changeset/base/216304
Log:
MFi386 r1.81, r1.82, r1.84: Reorganize code to reduce cache pressure and
branch mispredictions.
No objections from: scottl
Modified:
head/sys/amd64/amd64/busdma_machdep.c
Modified: head/sys/amd64/amd64/busdma_machdep.c
==============================================================================
--- head/sys/amd64/amd64/busdma_machdep.c Wed Dec 8 19:33:44 2010
(r216303)
+++ head/sys/amd64/amd64/busdma_machdep.c Wed Dec 8 19:42:21 2010
(r216304)
@@ -51,6 +51,8 @@ __FBSDID("$FreeBSD$");
#include <machine/specialreg.h>
#define MAX_BPAGES 8192
+#define BUS_DMA_COULD_BOUNCE BUS_DMA_BUS3
+#define BUS_DMA_MIN_ALLOC_COMP BUS_DMA_BUS4
struct bounce_zone;
@@ -135,7 +137,9 @@ static int reserve_bounce_pages(bus_dma_
static bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map,
vm_offset_t vaddr, bus_size_t size);
static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage);
-static __inline int run_filter(bus_dma_tag_t dmat, bus_addr_t paddr);
+int run_filter(bus_dma_tag_t dmat, bus_addr_t paddr);
+int _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap,
+ void *buf, bus_size_t buflen, int flags);
/*
* Return true if a match is made.
@@ -145,7 +149,7 @@ static __inline int run_filter(bus_dma_t
* If paddr is within the bounds of the dma tag then call the filter callback
* to check for a match, if there is no filter callback then assume a match.
*/
-static __inline int
+int
run_filter(bus_dma_tag_t dmat, bus_addr_t paddr)
{
int retval;
@@ -200,8 +204,6 @@ dflt_lock(void *arg, bus_dma_lock_op_t o
panic("driver error: busdma dflt_lock called");
}
-#define BUS_DMA_COULD_BOUNCE BUS_DMA_BUS3
-#define BUS_DMA_MIN_ALLOC_COMP BUS_DMA_BUS4
/*
* Allocate a device specific dma_tag.
*/
@@ -266,6 +268,9 @@ bus_dma_tag_create(bus_dma_tag_t parent,
else if (parent->boundary != 0)
newtag->boundary = MIN(parent->boundary,
newtag->boundary);
+ if ((newtag->filter != NULL) ||
+ ((parent->flags & BUS_DMA_COULD_BOUNCE) != 0))
+ newtag->flags |= BUS_DMA_COULD_BOUNCE;
if (newtag->filter == NULL) {
/*
* Short circuit looking at our parent directly
@@ -561,36 +566,15 @@ bus_dmamem_free(bus_dma_tag_t dmat, void
CTR3(KTR_BUSDMA, "%s: tag %p flags 0x%x", __func__, dmat, dmat->flags);
}
-/*
- * Utility function to load a linear buffer. lastaddrp holds state
- * between invocations (for multiple-buffer loads). segp contains
- * the starting segment on entrace, and the ending segment on exit.
- * first indicates if this is the first invocation of this function.
- */
-static __inline int
-_bus_dmamap_load_buffer(bus_dma_tag_t dmat,
- bus_dmamap_t map,
- void *buf, bus_size_t buflen,
- pmap_t pmap,
- int flags,
- bus_addr_t *lastaddrp,
- bus_dma_segment_t *segs,
- int *segp,
- int first)
+int
+_bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap,
+ void *buf, bus_size_t buflen, int flags)
{
- bus_size_t sgsize;
- bus_addr_t curaddr, lastaddr, baddr, bmask;
vm_offset_t vaddr;
+ vm_offset_t vendaddr;
bus_addr_t paddr;
- int seg;
-
- if (map == NULL)
- map = &nobounce_dmamap;
-
- if ((map != &nobounce_dmamap && map->pagesneeded == 0)
- && ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0)) {
- vm_offset_t vendaddr;
+ if ((map != &nobounce_dmamap && map->pagesneeded == 0)) {
CTR4(KTR_BUSDMA, "lowaddr= %d Maxmem= %d, boundary= %d, "
"alignment= %d", dmat->lowaddr, ptoa((vm_paddr_t)Maxmem),
dmat->boundary, dmat->alignment);
@@ -611,7 +595,8 @@ _bus_dmamap_load_buffer(bus_dma_tag_t dm
paddr = pmap_extract(pmap, vaddr);
else
paddr = pmap_kextract(vaddr);
- if (run_filter(dmat, paddr) != 0) {
+ if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) &&
+ run_filter(dmat, paddr) != 0) {
sg_len = roundup2(sg_len, dmat->alignment);
map->pagesneeded++;
}
@@ -643,6 +628,40 @@ _bus_dmamap_load_buffer(bus_dma_tag_t dm
mtx_unlock(&bounce_lock);
}
+ return (0);
+}
+
+/*
+ * Utility function to load a linear buffer. lastaddrp holds state
+ * between invocations (for multiple-buffer loads). segp contains
+ * the starting segment on entrace, and the ending segment on exit.
+ * first indicates if this is the first invocation of this function.
+ */
+static __inline int
+_bus_dmamap_load_buffer(bus_dma_tag_t dmat,
+ bus_dmamap_t map,
+ void *buf, bus_size_t buflen,
+ pmap_t pmap,
+ int flags,
+ bus_addr_t *lastaddrp,
+ bus_dma_segment_t *segs,
+ int *segp,
+ int first)
+{
+ bus_size_t sgsize;
+ bus_addr_t curaddr, lastaddr, baddr, bmask;
+ vm_offset_t vaddr;
+ int seg, error;
+
+ if (map == NULL)
+ map = &nobounce_dmamap;
+
+ if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) {
+ error = _bus_dmamap_count_pages(dmat, map, pmap, buf, buflen,
flags);
+ if (error)
+ return (error);
+ }
+
vaddr = (vm_offset_t)buf;
lastaddr = *lastaddrp;
bmask = ~(dmat->boundary - 1);
@@ -663,7 +682,8 @@ _bus_dmamap_load_buffer(bus_dma_tag_t dm
*/
max_sgsize = MIN(buflen, dmat->maxsegsz);
sgsize = PAGE_SIZE - ((vm_offset_t)curaddr & PAGE_MASK);
- if (map->pagesneeded != 0 && run_filter(dmat, curaddr)) {
+ if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) &&
+ map->pagesneeded != 0 && run_filter(dmat, curaddr)) {
sgsize = roundup2(sgsize, dmat->alignment);
sgsize = MIN(sgsize, max_sgsize);
curaddr = add_bounce_page(dmat, map, vaddr, sgsize);
@@ -762,18 +782,17 @@ bus_dmamap_load(bus_dma_tag_t dmat, bus_
/*
* Like _bus_dmamap_load(), but for mbufs.
*/
-int
-bus_dmamap_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map,
- struct mbuf *m0,
- bus_dmamap_callback2_t *callback, void *callback_arg,
- int flags)
+static __inline int
+_bus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map,
+ struct mbuf *m0, bus_dma_segment_t *segs, int *nsegs,
+ int flags)
{
- int nsegs, error;
+ int error;
M_ASSERTPKTHDR(m0);
flags |= BUS_DMA_NOWAIT;
- nsegs = 0;
+ *nsegs = 0;
error = 0;
if (m0->m_pkthdr.len <= dmat->maxsize) {
int first = 1;
@@ -785,7 +804,7 @@ bus_dmamap_load_mbuf(bus_dma_tag_t dmat,
error = _bus_dmamap_load_buffer(dmat, map,
m->m_data, m->m_len,
NULL, flags, &lastaddr,
- dmat->segments, &nsegs, first);
+ segs, nsegs, first);
first = 0;
}
}
@@ -793,15 +812,33 @@ bus_dmamap_load_mbuf(bus_dma_tag_t dmat,
error = EINVAL;
}
+ /* XXX FIXME: Having to increment nsegs is really annoying */
+ ++*nsegs;
+ CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d",
+ __func__, dmat, dmat->flags, error, *nsegs);
+ return (error);
+}
+
+int
+bus_dmamap_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map,
+ struct mbuf *m0,
+ bus_dmamap_callback2_t *callback, void *callback_arg,
+ int flags)
+{
+ int nsegs, error;
+
+ error = _bus_dmamap_load_mbuf_sg(dmat, map, m0, dmat->segments, &nsegs,
+ flags);
+
if (error) {
/* force "no valid mappings" in callback */
(*callback)(callback_arg, dmat->segments, 0, 0, error);
} else {
(*callback)(callback_arg, dmat->segments,
- nsegs+1, m0->m_pkthdr.len, error);
+ nsegs, m0->m_pkthdr.len, error);
}
CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d",
- __func__, dmat, dmat->flags, error, nsegs + 1);
+ __func__, dmat, dmat->flags, error, nsegs);
return (error);
}
@@ -810,36 +847,7 @@ bus_dmamap_load_mbuf_sg(bus_dma_tag_t dm
struct mbuf *m0, bus_dma_segment_t *segs, int *nsegs,
int flags)
{
- int error;
-
- M_ASSERTPKTHDR(m0);
-
- flags |= BUS_DMA_NOWAIT;
- *nsegs = 0;
- error = 0;
- if (m0->m_pkthdr.len <= dmat->maxsize) {
- int first = 1;
- bus_addr_t lastaddr = 0;
- struct mbuf *m;
-
- for (m = m0; m != NULL && error == 0; m = m->m_next) {
- if (m->m_len > 0) {
- error = _bus_dmamap_load_buffer(dmat, map,
- m->m_data, m->m_len,
- NULL, flags, &lastaddr,
- segs, nsegs, first);
- first = 0;
- }
- }
- } else {
- error = EINVAL;
- }
-
- /* XXX FIXME: Having to increment nsegs is really annoying */
- ++*nsegs;
- CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d",
- __func__, dmat, dmat->flags, error, *nsegs);
- return (error);
+ return (_bus_dmamap_load_mbuf_sg(dmat, map, m0, segs, nsegs, flags));
}
/*
_______________________________________________
[email protected] mailing list
http://lists.freebsd.org/mailman/listinfo/svn-src-head
To unsubscribe, send any mail to "[email protected]"
