Author: jeff
Date: Thu Jul 23 19:13:41 2015
New Revision: 285819
URL: https://svnweb.freebsd.org/changeset/base/285819
Log:
Refactor unmapped buffer address handling.
- Use pointer assignment rather than a combination of pointers and
flags to switch buffers between unmapped and mapped. This eliminates
multiple flags and generally simplifies the logic.
- Eliminate b_saveaddr since it is only used with pager bufs which have
their b_data re-initialized on each allocation.
- Gather up some convenience routines in the buffer cache for
manipulating buf space and buf malloc space.
- Add an inline, buf_mapped(), to standardize checks around unmapped
buffers.
In collaboration with: mlaier
Reviewed by: kib
Tested by: pho (many small revisions ago)
Sponsored by: EMC / Isilon Storage Division
Modified:
head/sys/cam/cam_periph.c
head/sys/dev/nvme/nvme_ctrlr.c
head/sys/kern/vfs_bio.c
head/sys/kern/vfs_cluster.c
head/sys/sys/buf.h
head/sys/ufs/ffs/ffs_rawread.c
head/sys/ufs/ffs/ffs_vfsops.c
head/sys/ufs/ffs/ffs_vnops.c
head/sys/vm/swap_pager.c
head/sys/vm/vm_pager.c
head/sys/vm/vnode_pager.c
Modified: head/sys/cam/cam_periph.c
==============================================================================
--- head/sys/cam/cam_periph.c Thu Jul 23 18:11:52 2015 (r285818)
+++ head/sys/cam/cam_periph.c Thu Jul 23 19:13:41 2015 (r285819)
@@ -855,12 +855,12 @@ cam_periph_mapmem(union ccb *ccb, struct
*/
mapinfo->bp[i] = getpbuf(NULL);
- /* save the buffer's data address */
- mapinfo->bp[i]->b_saveaddr = mapinfo->bp[i]->b_data;
-
/* put our pointer in the data slot */
mapinfo->bp[i]->b_data = *data_ptrs[i];
+ /* save the user's data address */
+ mapinfo->bp[i]->b_caller1 = *data_ptrs[i];
+
/* set the transfer length, we know it's < MAXPHYS */
mapinfo->bp[i]->b_bufsize = lengths[i];
@@ -877,7 +877,7 @@ cam_periph_mapmem(union ccb *ccb, struct
*/
if (vmapbuf(mapinfo->bp[i], 1) < 0) {
for (j = 0; j < i; ++j) {
- *data_ptrs[j] = mapinfo->bp[j]->b_saveaddr;
+ *data_ptrs[j] = mapinfo->bp[j]->b_caller1;
vunmapbuf(mapinfo->bp[j]);
relpbuf(mapinfo->bp[j], NULL);
}
@@ -958,7 +958,7 @@ cam_periph_unmapmem(union ccb *ccb, stru
for (i = 0; i < numbufs; i++) {
/* Set the user's pointer back to the original value */
- *data_ptrs[i] = mapinfo->bp[i]->b_saveaddr;
+ *data_ptrs[i] = mapinfo->bp[i]->b_caller1;
/* unmap the buffer */
vunmapbuf(mapinfo->bp[i]);
Modified: head/sys/dev/nvme/nvme_ctrlr.c
==============================================================================
--- head/sys/dev/nvme/nvme_ctrlr.c Thu Jul 23 18:11:52 2015
(r285818)
+++ head/sys/dev/nvme/nvme_ctrlr.c Thu Jul 23 19:13:41 2015
(r285819)
@@ -840,7 +840,6 @@ nvme_ctrlr_passthrough_cmd(struct nvme_c
*/
PHOLD(curproc);
buf = getpbuf(NULL);
- buf->b_saveaddr = buf->b_data;
buf->b_data = pt->buf;
buf->b_bufsize = pt->len;
buf->b_iocmd = pt->is_read ? BIO_READ : BIO_WRITE;
Modified: head/sys/kern/vfs_bio.c
==============================================================================
--- head/sys/kern/vfs_bio.c Thu Jul 23 18:11:52 2015 (r285818)
+++ head/sys/kern/vfs_bio.c Thu Jul 23 19:13:41 2015 (r285819)
@@ -137,12 +137,11 @@ SYSCTL_PROC(_vfs, OID_AUTO, bufspace, CT
&bufspace, 0, sysctl_bufspace, "L", "Virtual memory used for buffers");
#else
SYSCTL_LONG(_vfs, OID_AUTO, bufspace, CTLFLAG_RD, &bufspace, 0,
- "Virtual memory used for buffers");
+ "Physical memory used for buffers");
#endif
-static long unmapped_bufspace;
-SYSCTL_LONG(_vfs, OID_AUTO, unmapped_bufspace, CTLFLAG_RD,
- &unmapped_bufspace, 0,
- "Amount of unmapped buffers, inclusive in the bufspace");
+static long bufkvaspace;
+SYSCTL_LONG(_vfs, OID_AUTO, bufkvaspace, CTLFLAG_RD, &bufkvaspace, 0,
+ "Kernel virtual memory used for buffers");
static long maxbufspace;
SYSCTL_LONG(_vfs, OID_AUTO, maxbufspace, CTLFLAG_RD, &maxbufspace, 0,
"Maximum allowed value of bufspace (including buf_daemon)");
@@ -454,15 +453,14 @@ bdirtyadd(void)
* sufficient buffer space. Buffer space becomes recoverable when
* bp's get placed back in the queues.
*/
-
static __inline void
bufspacewakeup(void)
{
int need_wakeup, on;
/*
- * If someone is waiting for BUF space, wake them up. Even
- * though we haven't freed the kva space yet, the waiting
+ * If someone is waiting for bufspace, wake them up. Even
+ * though we may not have freed the kva space yet, the waiting
* process will be able to now.
*/
rw_rlock(&nblock);
@@ -482,6 +480,50 @@ bufspacewakeup(void)
}
/*
+ * bufspaceadjust:
+ *
+ * Adjust the reported bufspace for a KVA managed buffer, possibly
+ * waking any waiters.
+ */
+static void
+bufspaceadjust(struct buf *bp, int bufsize)
+{
+ int diff;
+
+ KASSERT((bp->b_flags & B_MALLOC) == 0,
+ ("bufspaceadjust: malloc buf %p", bp));
+ diff = bufsize - bp->b_bufsize;
+ if (diff < 0) {
+ atomic_subtract_long(&bufspace, -diff);
+ bufspacewakeup();
+ } else
+ atomic_add_long(&bufspace, diff);
+ bp->b_bufsize = bufsize;
+}
+
+/*
+ * bufmallocadjust:
+ *
+ * Adjust the reported bufspace for a malloc managed buffer, possibly
+ * waking any waiters.
+ */
+static void
+bufmallocadjust(struct buf *bp, int bufsize)
+{
+ int diff;
+
+ KASSERT((bp->b_flags & B_MALLOC) != 0,
+ ("bufmallocadjust: non-malloc buf %p", bp));
+ diff = bufsize - bp->b_bufsize;
+ if (diff < 0) {
+ atomic_subtract_long(&bufmallocspace, -diff);
+ bufspacewakeup();
+ } else
+ atomic_add_long(&bufmallocspace, diff);
+ bp->b_bufsize = bufsize;
+}
+
+/*
* runningwakeup:
*
* Wake up processes that are waiting on asynchronous writes to fall
@@ -817,6 +859,8 @@ bufinit(void)
for (i = 0; i < BUFFER_QUEUES; i++)
TAILQ_INIT(&bufqueues[i]);
+ unmapped_buf = (caddr_t)kva_alloc(MAXPHYS);
+
/* finally, initialize each buffer header and stick on empty q */
for (i = 0; i < nbuf; i++) {
bp = &buf[i];
@@ -826,6 +870,7 @@ bufinit(void)
bp->b_wcred = NOCRED;
bp->b_qindex = QUEUE_EMPTY;
bp->b_xflags = 0;
+ bp->b_data = bp->b_kvabase = unmapped_buf;
LIST_INIT(&bp->b_dep);
BUF_LOCKINIT(bp);
TAILQ_INSERT_TAIL(&bufqueues[QUEUE_EMPTY], bp, b_freelist);
@@ -900,7 +945,6 @@ bufinit(void)
bogus_page = vm_page_alloc(NULL, 0, VM_ALLOC_NOOBJ |
VM_ALLOC_NORMAL | VM_ALLOC_WIRED);
- unmapped_buf = (caddr_t)kva_alloc(MAXPHYS);
}
#ifdef INVARIANTS
@@ -908,8 +952,6 @@ static inline void
vfs_buf_check_mapped(struct buf *bp)
{
- KASSERT((bp->b_flags & B_UNMAPPED) == 0,
- ("mapped buf %p %x", bp, bp->b_flags));
KASSERT(bp->b_kvabase != unmapped_buf,
("mapped buf: b_kvabase was not updated %p", bp));
KASSERT(bp->b_data != unmapped_buf,
@@ -920,10 +962,6 @@ static inline void
vfs_buf_check_unmapped(struct buf *bp)
{
- KASSERT((bp->b_flags & B_UNMAPPED) == B_UNMAPPED,
- ("unmapped buf %p %x", bp, bp->b_flags));
- KASSERT(bp->b_kvabase == unmapped_buf,
- ("unmapped buf: corrupted b_kvabase %p", bp));
KASSERT(bp->b_data == unmapped_buf,
("unmapped buf: corrupted b_data %p", bp));
}
@@ -952,37 +990,6 @@ bpmap_qenter(struct buf *bp)
}
/*
- * bfreekva() - free the kva allocation for a buffer.
- *
- * Since this call frees up buffer space, we call bufspacewakeup().
- */
-static void
-bfreekva(struct buf *bp)
-{
-
- if (bp->b_kvasize == 0)
- return;
-
- atomic_add_int(&buffreekvacnt, 1);
- atomic_subtract_long(&bufspace, bp->b_kvasize);
- if ((bp->b_flags & B_UNMAPPED) == 0) {
- BUF_CHECK_MAPPED(bp);
- vmem_free(buffer_arena, (vm_offset_t)bp->b_kvabase,
- bp->b_kvasize);
- } else {
- BUF_CHECK_UNMAPPED(bp);
- if ((bp->b_flags & B_KVAALLOC) != 0) {
- vmem_free(buffer_arena, (vm_offset_t)bp->b_kvaalloc,
- bp->b_kvasize);
- }
- atomic_subtract_long(&unmapped_bufspace, bp->b_kvasize);
- bp->b_flags &= ~(B_UNMAPPED | B_KVAALLOC);
- }
- bp->b_kvasize = 0;
- bufspacewakeup();
-}
-
-/*
* binsfree:
*
* Insert the buffer into the appropriate free list.
@@ -1104,6 +1111,75 @@ bremfreel(struct buf *bp)
}
/*
+ * bufkvafree:
+ *
+ * Free the kva allocation for a buffer.
+ *
+ */
+static void
+bufkvafree(struct buf *bp)
+{
+
+#ifdef INVARIANTS
+ if (bp->b_kvasize == 0) {
+ KASSERT(bp->b_kvabase == unmapped_buf &&
+ bp->b_data == unmapped_buf,
+ ("Leaked KVA space on %p", bp));
+ } else if (buf_mapped(bp))
+ BUF_CHECK_MAPPED(bp);
+ else
+ BUF_CHECK_UNMAPPED(bp);
+#endif
+ if (bp->b_kvasize == 0)
+ return;
+
+ vmem_free(buffer_arena, (vm_offset_t)bp->b_kvabase, bp->b_kvasize);
+ atomic_subtract_long(&bufkvaspace, bp->b_kvasize);
+ atomic_add_int(&buffreekvacnt, 1);
+ bp->b_data = bp->b_kvabase = unmapped_buf;
+ bp->b_kvasize = 0;
+}
+
+/*
+ * bufkvaalloc:
+ *
+ * Allocate the buffer KVA and set b_kvasize and b_kvabase.
+ */
+static int
+bufkvaalloc(struct buf *bp, int maxsize, int gbflags)
+{
+ vm_offset_t addr;
+ int error;
+
+ KASSERT((gbflags & GB_UNMAPPED) == 0 || (gbflags & GB_KVAALLOC) != 0,
+ ("Invalid gbflags 0x%x in %s", gbflags, __func__));
+
+ bufkvafree(bp);
+
+ addr = 0;
+ error = vmem_alloc(buffer_arena, maxsize, M_BESTFIT | M_NOWAIT, &addr);
+ if (error != 0) {
+ /*
+ * Buffer map is too fragmented. Request the caller
+ * to defragment the map.
+ */
+ atomic_add_int(&bufdefragcnt, 1);
+ return (error);
+ }
+ bp->b_kvabase = (caddr_t)addr;
+ bp->b_kvasize = maxsize;
+ atomic_add_long(&bufkvaspace, bp->b_kvasize);
+ if ((gbflags & GB_UNMAPPED) != 0) {
+ bp->b_data = unmapped_buf;
+ BUF_CHECK_UNMAPPED(bp);
+ } else {
+ bp->b_data = bp->b_kvabase;
+ BUF_CHECK_MAPPED(bp);
+ }
+ return (0);
+}
+
+/*
* Attempt to initiate asynchronous I/O on read-ahead blocks. We must
* clear BIO_ERROR and B_INVAL prior to initiating I/O . If B_CACHE is set,
* the buffer is valid and we do not have to do anything.
@@ -1715,7 +1791,8 @@ brelse(struct buf *bp)
}
VM_OBJECT_RUNLOCK(obj);
- if ((bp->b_flags & (B_INVAL | B_UNMAPPED)) ==
0) {
+ if ((bp->b_flags & B_INVAL) == 0 &&
+ buf_mapped(bp)) {
BUF_CHECK_MAPPED(bp);
pmap_qenter(
trunc_page((vm_offset_t)bp->b_data),
@@ -1872,7 +1949,7 @@ vfs_vmio_release(struct buf *bp)
vm_page_t m;
int i;
- if ((bp->b_flags & B_UNMAPPED) == 0) {
+ if (buf_mapped(bp)) {
BUF_CHECK_MAPPED(bp);
pmap_qremove(trunc_page((vm_offset_t)bp->b_data), bp->b_npages);
} else
@@ -1905,10 +1982,8 @@ vfs_vmio_release(struct buf *bp)
if (obj != NULL)
VM_OBJECT_WUNLOCK(obj);
- if (bp->b_bufsize) {
- bufspacewakeup();
- bp->b_bufsize = 0;
- }
+ if (bp->b_bufsize)
+ bufspaceadjust(bp, 0);
bp->b_npages = 0;
bp->b_flags &= ~B_VMIO;
if (bp->b_vp)
@@ -1977,7 +2052,7 @@ vfs_bio_awrite(struct buf *bp)
int gbflags;
bo = &vp->v_bufobj;
- gbflags = (bp->b_flags & B_UNMAPPED) != 0 ? GB_UNMAPPED : 0;
+ gbflags = (bp->b_data == unmapped_buf) ? GB_UNMAPPED : 0;
/*
* right now we support clustered writing only to regular files. If
* we find a clusterable block we could be in the middle of a cluster
@@ -2026,49 +2101,6 @@ vfs_bio_awrite(struct buf *bp)
return (nwritten);
}
-static void
-setbufkva(struct buf *bp, vm_offset_t addr, int maxsize, int gbflags)
-{
-
- KASSERT((bp->b_flags & (B_UNMAPPED | B_KVAALLOC)) == 0 &&
- bp->b_kvasize == 0, ("call bfreekva(%p)", bp));
- if ((gbflags & GB_UNMAPPED) == 0) {
- bp->b_kvabase = (caddr_t)addr;
- } else if ((gbflags & GB_KVAALLOC) != 0) {
- KASSERT((gbflags & GB_UNMAPPED) != 0,
- ("GB_KVAALLOC without GB_UNMAPPED"));
- bp->b_kvaalloc = (caddr_t)addr;
- bp->b_flags |= B_UNMAPPED | B_KVAALLOC;
- atomic_add_long(&unmapped_bufspace, bp->b_kvasize);
- }
- bp->b_kvasize = maxsize;
-}
-
-/*
- * Allocate the buffer KVA and set b_kvasize. Also set b_kvabase if
- * needed.
- */
-static int
-allocbufkva(struct buf *bp, int maxsize, int gbflags)
-{
- vm_offset_t addr;
-
- bfreekva(bp);
- addr = 0;
-
- if (vmem_alloc(buffer_arena, maxsize, M_BESTFIT | M_NOWAIT, &addr)) {
- /*
- * Buffer map is too fragmented. Request the caller
- * to defragment the map.
- */
- atomic_add_int(&bufdefragcnt, 1);
- return (1);
- }
- setbufkva(bp, addr, maxsize, gbflags);
- atomic_add_long(&bufspace, bp->b_kvasize);
- return (0);
-}
-
/*
* Ask the bufdaemon for help, or act as bufdaemon itself, when a
* locked vnode is supplied.
@@ -2192,7 +2224,7 @@ getnewbuf_reuse_bp(struct buf *bp, int q
if (bp->b_bufsize)
allocbuf(bp, 0);
- bp->b_flags &= B_UNMAPPED | B_KVAALLOC;
+ bp->b_flags = 0;
bp->b_ioflags = 0;
bp->b_xflags = 0;
KASSERT((bp->b_flags & B_INFREECNT) == 0,
@@ -2328,14 +2360,11 @@ restart:
}
/*
* If we are defragging then we need a buffer with
- * b_kvasize != 0. XXX this situation should no longer
- * occur, if defrag is non-zero the buffer's b_kvasize
- * should also be non-zero at this point. XXX
+ * b_kvasize != 0. This situation occurs when we
+ * have many unmapped bufs.
*/
- if (defrag && bp->b_kvasize == 0) {
- printf("Warning: defrag empty buffer %p\n", bp);
+ if (defrag && bp->b_kvasize == 0)
continue;
- }
/*
* Start freeing the bp. This is somewhat involved. nbp
@@ -2380,7 +2409,7 @@ restart:
*/
if (defrag) {
bp->b_flags |= B_INVAL;
- bfreekva(bp);
+ bufkvafree(bp);
brelse(bp);
defrag = 0;
goto restart;
@@ -2392,7 +2421,7 @@ restart:
*/
if (qindex == QUEUE_CLEAN && BUF_LOCKWAITERS(bp)) {
bp->b_flags |= B_INVAL;
- bfreekva(bp);
+ bufkvafree(bp);
brelse(bp);
goto restart;
}
@@ -2409,7 +2438,7 @@ restart:
flushingbufs = 1;
if (flushingbufs && bp->b_kvasize != 0) {
bp->b_flags |= B_INVAL;
- bfreekva(bp);
+ bufkvafree(bp);
brelse(bp);
goto restart;
}
@@ -2480,65 +2509,27 @@ restart:
} else if ((gbflags & (GB_UNMAPPED | GB_KVAALLOC)) == GB_UNMAPPED) {
mtx_assert(&bqclean, MA_NOTOWNED);
- bfreekva(bp);
- bp->b_flags |= B_UNMAPPED;
- bp->b_kvabase = bp->b_data = unmapped_buf;
- bp->b_kvasize = maxsize;
- atomic_add_long(&bufspace, bp->b_kvasize);
- atomic_add_long(&unmapped_bufspace, bp->b_kvasize);
+ bufkvafree(bp);
atomic_add_int(&bufreusecnt, 1);
} else {
mtx_assert(&bqclean, MA_NOTOWNED);
/*
* We finally have a valid bp. We aren't quite out of the
- * woods, we still have to reserve kva space. In order
- * to keep fragmentation sane we only allocate kva in
- * BKVASIZE chunks.
+ * woods, we still have to reserve kva space. In order to
+ * keep fragmentation sane we only allocate kva in BKVASIZE
+ * chunks.
*/
maxsize = (maxsize + BKVAMASK) & ~BKVAMASK;
- if (maxsize != bp->b_kvasize || (bp->b_flags & (B_UNMAPPED |
- B_KVAALLOC)) == B_UNMAPPED) {
- if (allocbufkva(bp, maxsize, gbflags)) {
- defrag = 1;
- bp->b_flags |= B_INVAL;
- brelse(bp);
- goto restart;
- }
- atomic_add_int(&bufreusecnt, 1);
- } else if ((bp->b_flags & B_KVAALLOC) != 0 &&
- (gbflags & (GB_UNMAPPED | GB_KVAALLOC)) == 0) {
- /*
- * If the reused buffer has KVA allocated,
- * reassign b_kvaalloc to b_kvabase.
- */
- bp->b_kvabase = bp->b_kvaalloc;
- bp->b_flags &= ~B_KVAALLOC;
- atomic_subtract_long(&unmapped_bufspace,
- bp->b_kvasize);
- atomic_add_int(&bufreusecnt, 1);
- } else if ((bp->b_flags & (B_UNMAPPED | B_KVAALLOC)) == 0 &&
- (gbflags & (GB_UNMAPPED | GB_KVAALLOC)) == (GB_UNMAPPED |
- GB_KVAALLOC)) {
- /*
- * The case of reused buffer already have KVA
- * mapped, but the request is for unmapped
- * buffer with KVA allocated.
- */
- bp->b_kvaalloc = bp->b_kvabase;
- bp->b_data = bp->b_kvabase = unmapped_buf;
- bp->b_flags |= B_UNMAPPED | B_KVAALLOC;
- atomic_add_long(&unmapped_bufspace,
- bp->b_kvasize);
- atomic_add_int(&bufreusecnt, 1);
- }
- if ((gbflags & GB_UNMAPPED) == 0) {
- bp->b_saveaddr = bp->b_kvabase;
- bp->b_data = bp->b_saveaddr;
- bp->b_flags &= ~B_UNMAPPED;
- BUF_CHECK_MAPPED(bp);
+ if (maxsize != bp->b_kvasize &&
+ bufkvaalloc(bp, maxsize, gbflags)) {
+ defrag = 1;
+ bp->b_flags |= B_INVAL;
+ brelse(bp);
+ goto restart;
}
+ atomic_add_int(&bufreusecnt, 1);
}
return (bp);
}
@@ -2968,9 +2959,9 @@ vfs_setdirty_locked_object(struct buf *b
}
/*
- * Allocate the KVA mapping for an existing buffer. It handles the
- * cases of both B_UNMAPPED buffer, and buffer with the preallocated
- * KVA which is not mapped (B_KVAALLOC).
+ * Allocate the KVA mapping for an existing buffer.
+ * If an unmapped buffer is provided but a mapped buffer is requested, take
+ * also care to properly setup mappings between pages and KVA.
*/
static void
bp_unmapped_get_kva(struct buf *bp, daddr_t blkno, int size, int gbflags)
@@ -2979,25 +2970,22 @@ bp_unmapped_get_kva(struct buf *bp, dadd
int bsize, maxsize, need_mapping, need_kva;
off_t offset;
- need_mapping = (bp->b_flags & B_UNMAPPED) != 0 &&
+ need_mapping = bp->b_data == unmapped_buf &&
(gbflags & GB_UNMAPPED) == 0;
- need_kva = (bp->b_flags & (B_KVAALLOC | B_UNMAPPED)) == B_UNMAPPED &&
+ need_kva = bp->b_kvabase == unmapped_buf &&
+ bp->b_data == unmapped_buf &&
(gbflags & GB_KVAALLOC) != 0;
if (!need_mapping && !need_kva)
return;
BUF_CHECK_UNMAPPED(bp);
- if (need_mapping && (bp->b_flags & B_KVAALLOC) != 0) {
+ if (need_mapping && bp->b_kvabase != unmapped_buf) {
/*
* Buffer is not mapped, but the KVA was already
* reserved at the time of the instantiation. Use the
* allocated space.
*/
- bp->b_flags &= ~B_KVAALLOC;
- KASSERT(bp->b_kvaalloc != 0, ("kvaalloc == 0"));
- bp->b_kvabase = bp->b_kvaalloc;
- atomic_subtract_long(&unmapped_bufspace, bp->b_kvasize);
goto has_addr;
}
@@ -3012,7 +3000,7 @@ bp_unmapped_get_kva(struct buf *bp, dadd
maxsize = imax(maxsize, bsize);
mapping_loop:
- if (allocbufkva(bp, maxsize, gbflags)) {
+ if (bufkvaalloc(bp, maxsize, gbflags)) {
/*
* Request defragmentation. getnewbuf() returns us the
* allocated space by the scratch buffer KVA.
@@ -3025,31 +3013,31 @@ mapping_loop:
* XXXKIB: defragmentation cannot
* succeed, not sure what else to do.
*/
- panic("GB_NOWAIT_BD and B_UNMAPPED %p", bp);
+ panic("GB_NOWAIT_BD and GB_UNMAPPED %p", bp);
}
atomic_add_int(&mappingrestarts, 1);
goto mapping_loop;
}
- KASSERT((scratch_bp->b_flags & B_KVAALLOC) != 0,
- ("scratch bp !B_KVAALLOC %p", scratch_bp));
- setbufkva(bp, (vm_offset_t)scratch_bp->b_kvaalloc,
- scratch_bp->b_kvasize, gbflags);
+ KASSERT(scratch_bp->b_kvabase != unmapped_buf,
+ ("scratch bp has no KVA %p", scratch_bp));
+ /* Grab pointers. */
+ bp->b_kvabase = scratch_bp->b_kvabase;
+ bp->b_kvasize = scratch_bp->b_kvasize;
+ bp->b_data = scratch_bp->b_data;
/* Get rid of the scratch buffer. */
scratch_bp->b_kvasize = 0;
scratch_bp->b_flags |= B_INVAL;
- scratch_bp->b_flags &= ~(B_UNMAPPED | B_KVAALLOC);
+ scratch_bp->b_data = scratch_bp->b_kvabase = unmapped_buf;
brelse(scratch_bp);
}
- if (!need_mapping)
- return;
-
has_addr:
- bp->b_saveaddr = bp->b_kvabase;
- bp->b_data = bp->b_saveaddr; /* b_offset is handled by bpmap_qenter */
- bp->b_flags &= ~B_UNMAPPED;
- BUF_CHECK_MAPPED(bp);
- bpmap_qenter(bp);
+ if (need_mapping) {
+ /* b_offset is handled by bpmap_qenter. */
+ bp->b_data = bp->b_kvabase;
+ BUF_CHECK_MAPPED(bp);
+ bpmap_qenter(bp);
+ }
}
/*
@@ -3265,7 +3253,7 @@ loop:
} else {
maxsize = size;
/* Do not allow non-VMIO notmapped buffers. */
- flags &= ~GB_UNMAPPED;
+ flags &= ~(GB_UNMAPPED | GB_KVAALLOC);
}
maxsize = imax(maxsize, bsize);
@@ -3358,7 +3346,6 @@ geteblk(int size, int flags)
return (bp);
}
-
/*
* This code constitutes the buffer memory from either anonymous system
* memory (in the case of non-VMIO operations) or from an associated
@@ -3382,7 +3369,7 @@ allocbuf(struct buf *bp, int size)
BUF_ASSERT_HELD(bp);
- if (bp->b_kvasize < size)
+ if (bp->b_kvasize != 0 && bp->b_kvasize < size)
panic("allocbuf: buffer too small");
if ((bp->b_flags & B_VMIO) == 0) {
@@ -3407,15 +3394,8 @@ allocbuf(struct buf *bp, int size)
bp->b_bcount = size;
} else {
free(bp->b_data, M_BIOBUF);
- if (bp->b_bufsize) {
- atomic_subtract_long(
- &bufmallocspace,
- bp->b_bufsize);
- bufspacewakeup();
- bp->b_bufsize = 0;
- }
- bp->b_saveaddr = bp->b_kvabase;
- bp->b_data = bp->b_saveaddr;
+ bufmallocadjust(bp, 0);
+ bp->b_data = bp->b_kvabase;
bp->b_bcount = 0;
bp->b_flags &= ~B_MALLOC;
}
@@ -3434,33 +3414,28 @@ allocbuf(struct buf *bp, int size)
* is probably extremely rare and not worth worrying
* over.
*/
- if ( (bufmallocspace < maxbufmallocspace) &&
+ if ((bufmallocspace < maxbufmallocspace) &&
(bp->b_bufsize == 0) &&
(mbsize <= PAGE_SIZE/2)) {
bp->b_data = malloc(mbsize, M_BIOBUF, M_WAITOK);
- bp->b_bufsize = mbsize;
bp->b_bcount = size;
bp->b_flags |= B_MALLOC;
- atomic_add_long(&bufmallocspace, mbsize);
+ bufmallocadjust(bp, mbsize);
return 1;
}
origbuf = NULL;
origbufsize = 0;
/*
- * If the buffer is growing on its other-than-first
allocation,
- * then we revert to the page-allocation scheme.
+ * If the buffer is growing on its other-than-first
+ * allocation then we revert to the page-allocation
+ * scheme.
*/
if (bp->b_flags & B_MALLOC) {
origbuf = bp->b_data;
origbufsize = bp->b_bufsize;
bp->b_data = bp->b_kvabase;
- if (bp->b_bufsize) {
- atomic_subtract_long(&bufmallocspace,
- bp->b_bufsize);
- bufspacewakeup();
- bp->b_bufsize = 0;
- }
+ bufmallocadjust(bp, 0);
bp->b_flags &= ~B_MALLOC;
newbsize = round_page(newbsize);
}
@@ -3498,7 +3473,7 @@ allocbuf(struct buf *bp, int size)
if (desiredpages < bp->b_npages) {
vm_page_t m;
- if ((bp->b_flags & B_UNMAPPED) == 0) {
+ if (buf_mapped(bp)) {
BUF_CHECK_MAPPED(bp);
pmap_qremove((vm_offset_t)trunc_page(
(vm_offset_t)bp->b_data) +
@@ -3611,18 +3586,18 @@ allocbuf(struct buf *bp, int size)
VM_OBJECT_WUNLOCK(obj);
/*
- * Step 3, fixup the KVM pmap.
+ * Step 3, fixup the KVA pmap.
*/
- if ((bp->b_flags & B_UNMAPPED) == 0)
+ if (buf_mapped(bp))
bpmap_qenter(bp);
else
BUF_CHECK_UNMAPPED(bp);
}
}
- if (newbsize < bp->b_bufsize)
- bufspacewakeup();
- bp->b_bufsize = newbsize; /* actual buffer allocation */
- bp->b_bcount = size; /* requested buffer size */
+ /* Record changes in allocation size. */
+ if (bp->b_bufsize != newbsize)
+ bufspaceadjust(bp, newbsize);
+ bp->b_bcount = size; /* requested buffer size. */
return 1;
}
@@ -3919,7 +3894,7 @@ bufdone_finish(struct buf *bp)
}
vm_object_pip_wakeupn(obj, 0);
VM_OBJECT_WUNLOCK(obj);
- if (bogus && (bp->b_flags & B_UNMAPPED) == 0) {
+ if (bogus && buf_mapped(bp)) {
BUF_CHECK_MAPPED(bp);
pmap_qenter(trunc_page((vm_offset_t)bp->b_data),
bp->b_pages, bp->b_npages);
@@ -3966,7 +3941,7 @@ vfs_unbusy_pages(struct buf *bp)
if (!m)
panic("vfs_unbusy_pages: page missing\n");
bp->b_pages[i] = m;
- if ((bp->b_flags & B_UNMAPPED) == 0) {
+ if (buf_mapped(bp)) {
BUF_CHECK_MAPPED(bp);
pmap_qenter(trunc_page((vm_offset_t)bp->b_data),
bp->b_pages, bp->b_npages);
@@ -4140,7 +4115,7 @@ vfs_busy_pages(struct buf *bp, int clear
foff = (foff + PAGE_SIZE) & ~(off_t)PAGE_MASK;
}
VM_OBJECT_WUNLOCK(obj);
- if (bogus && (bp->b_flags & B_UNMAPPED) == 0) {
+ if (bogus && buf_mapped(bp)) {
BUF_CHECK_MAPPED(bp);
pmap_qenter(trunc_page((vm_offset_t)bp->b_data),
bp->b_pages, bp->b_npages);
@@ -4260,7 +4235,7 @@ vfs_bio_bzero_buf(struct buf *bp, int ba
vm_page_t m;
int i, n;
- if ((bp->b_flags & B_UNMAPPED) == 0) {
+ if (buf_mapped(bp)) {
BUF_CHECK_MAPPED(bp);
bzero(bp->b_data + base, size);
} else {
@@ -4353,11 +4328,12 @@ vm_hold_free_pages(struct buf *bp, int n
* be valid, a race or a smaller-file mapped into a larger space may
* actually cause vmapbuf() to fail, so all callers of vmapbuf() MUST
* check the return value.
+ *
+ * This function only works with pager buffers.
*/
int
vmapbuf(struct buf *bp, int mapbuf)
{
- caddr_t kva;
vm_prot_t prot;
int pidx;
@@ -4371,24 +4347,20 @@ vmapbuf(struct buf *bp, int mapbuf)
btoc(MAXPHYS))) < 0)
return (-1);
bp->b_npages = pidx;
+ bp->b_offset = ((vm_offset_t)bp->b_data) & PAGE_MASK;
if (mapbuf || !unmapped_buf_allowed) {
- pmap_qenter((vm_offset_t)bp->b_saveaddr, bp->b_pages, pidx);
- kva = bp->b_saveaddr;
- bp->b_saveaddr = bp->b_data;
- bp->b_data = kva + (((vm_offset_t)bp->b_data) & PAGE_MASK);
- bp->b_flags &= ~B_UNMAPPED;
- } else {
- bp->b_flags |= B_UNMAPPED;
- bp->b_offset = ((vm_offset_t)bp->b_data) & PAGE_MASK;
- bp->b_saveaddr = bp->b_data;
+ pmap_qenter((vm_offset_t)bp->b_kvabase, bp->b_pages, pidx);
+ bp->b_data = bp->b_kvabase + bp->b_offset;
+ } else
bp->b_data = unmapped_buf;
- }
return(0);
}
/*
* Free the io map PTEs associated with this IO operation.
* We also invalidate the TLB entries and restore the original b_addr.
+ *
+ * This function only works with pager buffers.
*/
void
vunmapbuf(struct buf *bp)
@@ -4396,13 +4368,11 @@ vunmapbuf(struct buf *bp)
int npages;
npages = bp->b_npages;
- if (bp->b_flags & B_UNMAPPED)
- bp->b_flags &= ~B_UNMAPPED;
- else
+ if (buf_mapped(bp))
pmap_qremove(trunc_page((vm_offset_t)bp->b_data), npages);
vm_page_unhold_pages(bp->b_pages, npages);
-
- bp->b_data = bp->b_saveaddr;
+
+ bp->b_data = unmapped_buf;
}
void
@@ -4543,7 +4513,7 @@ void
bdata2bio(struct buf *bp, struct bio *bip)
{
- if ((bp->b_flags & B_UNMAPPED) != 0) {
+ if (!buf_mapped(bp)) {
KASSERT(unmapped_buf_allowed, ("unmapped"));
bip->bio_ma = bp->b_pages;
bip->bio_ma_n = bp->b_npages;
@@ -4586,6 +4556,8 @@ DB_SHOW_COMMAND(buffer, db_show_buffer)
bp->b_error, bp->b_bufsize, bp->b_bcount, bp->b_resid,
bp->b_bufobj, bp->b_data, (intmax_t)bp->b_blkno,
(intmax_t)bp->b_lblkno, bp->b_dep.lh_first);
+ db_printf("b_kvabase = %p, b_kvasize = %d\n",
+ bp->b_kvabase, bp->b_kvasize);
if (bp->b_npages) {
int i;
db_printf("b_npages = %d, pages(OBJ, IDX, PA): ", bp->b_npages);
Modified: head/sys/kern/vfs_cluster.c
==============================================================================
--- head/sys/kern/vfs_cluster.c Thu Jul 23 18:11:52 2015 (r285818)
+++ head/sys/kern/vfs_cluster.c Thu Jul 23 19:13:41 2015 (r285819)
@@ -354,7 +354,6 @@ cluster_rbuild(struct vnode *vp, u_quad_
*/
bp->b_flags = B_ASYNC | B_CLUSTER | B_VMIO;
if ((gbflags & GB_UNMAPPED) != 0) {
- bp->b_flags |= B_UNMAPPED;
bp->b_data = unmapped_buf;
} else {
bp->b_data = (char *)((vm_offset_t)bp->b_data |
@@ -517,9 +516,8 @@ clean_sbusy:
if (bp->b_bufsize > bp->b_kvasize)
panic("cluster_rbuild: b_bufsize(%ld) > b_kvasize(%d)\n",
bp->b_bufsize, bp->b_kvasize);
- bp->b_kvasize = bp->b_bufsize;
- if ((bp->b_flags & B_UNMAPPED) == 0) {
+ if (buf_mapped(bp)) {
pmap_qenter(trunc_page((vm_offset_t) bp->b_data),
(vm_page_t *)bp->b_pages, bp->b_npages);
}
@@ -545,7 +543,7 @@ cluster_callback(bp)
if (bp->b_ioflags & BIO_ERROR)
error = bp->b_error;
- if ((bp->b_flags & B_UNMAPPED) == 0) {
+ if (buf_mapped(bp)) {
pmap_qremove(trunc_page((vm_offset_t) bp->b_data),
bp->b_npages);
}
@@ -871,7 +869,6 @@ cluster_wbuild(struct vnode *vp, long si
bp->b_data = (char *)((vm_offset_t)bp->b_data |
((vm_offset_t)tbp->b_data & PAGE_MASK));
} else {
- bp->b_flags |= B_UNMAPPED;
bp->b_data = unmapped_buf;
}
bp->b_flags |= B_CLUSTER | (tbp->b_flags & (B_VMIO |
@@ -1004,7 +1001,7 @@ cluster_wbuild(struct vnode *vp, long si
tbp, b_cluster.cluster_entry);
}
finishcluster:
- if ((bp->b_flags & B_UNMAPPED) == 0) {
+ if (buf_mapped(bp)) {
pmap_qenter(trunc_page((vm_offset_t) bp->b_data),
(vm_page_t *)bp->b_pages, bp->b_npages);
}
@@ -1012,7 +1009,6 @@ cluster_wbuild(struct vnode *vp, long si
panic(
"cluster_wbuild: b_bufsize(%ld) > b_kvasize(%d)\n",
bp->b_bufsize, bp->b_kvasize);
- bp->b_kvasize = bp->b_bufsize;
totalwritten += bp->b_bufsize;
bp->b_dirtyoff = 0;
bp->b_dirtyend = bp->b_bufsize;
Modified: head/sys/sys/buf.h
==============================================================================
--- head/sys/sys/buf.h Thu Jul 23 18:11:52 2015 (r285818)
+++ head/sys/sys/buf.h Thu Jul 23 19:13:41 2015 (r285819)
@@ -112,17 +112,15 @@ struct buf {
b_xflags_t b_xflags; /* extra flags */
struct lock b_lock; /* Buffer lock */
long b_bufsize; /* Allocated buffer size. */
- long b_runningbufspace; /* when I/O is running, pipelining */
- caddr_t b_kvabase; /* base kva for buffer */
- caddr_t b_kvaalloc; /* allocated kva for B_KVAALLOC */
+ int b_runningbufspace; /* when I/O is running, pipelining */
int b_kvasize; /* size of kva for buffer */
- daddr_t b_lblkno; /* Logical block number. */
- struct vnode *b_vp; /* Device vnode. */
int b_dirtyoff; /* Offset in buffer of dirty region. */
int b_dirtyend; /* Offset of end of dirty region. */
+ caddr_t b_kvabase; /* base kva for buffer */
+ daddr_t b_lblkno; /* Logical block number. */
+ struct vnode *b_vp; /* Device vnode. */
struct ucred *b_rcred; /* Read credentials reference. */
struct ucred *b_wcred; /* Write credentials reference. */
- void *b_saveaddr; /* Original b_addr for physio. */
union {
TAILQ_ENTRY(buf) bu_freelist; /* (Q) */
struct {
@@ -206,8 +204,8 @@ struct buf {
#define B_PERSISTENT 0x00000100 /* Perm. ref'ed while EXT2FS
mounted. */
#define B_DONE 0x00000200 /* I/O completed. */
#define B_EINTR 0x00000400 /* I/O was interrupted */
-#define B_UNMAPPED 0x00000800 /* KVA is not mapped. */
-#define B_KVAALLOC 0x00001000 /* But allocated. */
+#define B_00000800 0x00000800 /* Available flag. */
+#define B_00001000 0x00001000 /* Available flag. */
#define B_INVAL 0x00002000 /* Does not contain valid info.
*/
#define B_BARRIER 0x00004000 /* Write this and all
preceeding first. */
#define B_NOCACHE 0x00008000 /* Do not cache block after
use. */
@@ -231,7 +229,7 @@ struct buf {
#define PRINT_BUF_FLAGS "\20\40remfree\37cluster\36vmio\35ram\34managed" \
"\33paging\32infreecnt\31nocopy\30b23\27relbuf\26dirty\25b20" \
"\24b19\23b18\22clusterok\21malloc\20nocache\17b14\16inval" \
- "\15kvaalloc\14unmapped\13eintr\12done\11persist\10delwri" \
+ "\15b12\14b11\13eintr\12done\11persist\10delwri" \
"\7validsuspwrt\6cache\5deferred\4direct\3async\2needcommit\1age"
/*
@@ -374,15 +372,11 @@ struct buf_queue_head {
};
/*
- * This structure describes a clustered I/O. It is stored in the b_saveaddr
- * field of the buffer on which I/O is done. At I/O completion, cluster
- * callback uses the structure to parcel I/O's to individual buffers, and
- * then free's this structure.
+ * This structure describes a clustered I/O.
*/
struct cluster_save {
long bs_bcount; /* Saved b_bcount. */
long bs_bufsize; /* Saved b_bufsize. */
- void *bs_saveaddr; /* Saved b_addr. */
int bs_nchildren; /* Number of associated buffers. */
struct buf **bs_children; /* List of associated buffers. */
};
@@ -478,7 +472,14 @@ extern int cluster_pbuf_freecnt; /* Numb
extern int vnode_pbuf_freecnt; /* Number of pbufs for vnode pager */
extern int vnode_async_pbuf_freecnt; /* Number of pbufs for vnode pager,
asynchronous reads */
-extern caddr_t unmapped_buf;
+extern caddr_t unmapped_buf; /* Data address for unmapped buffers. */
+
+static inline int
+buf_mapped(struct buf *bp)
+{
+
+ return (bp->b_data != unmapped_buf);
+}
void runningbufwakeup(struct buf *);
void waitrunningbufspace(void);
Modified: head/sys/ufs/ffs/ffs_rawread.c
==============================================================================
--- head/sys/ufs/ffs/ffs_rawread.c Thu Jul 23 18:11:52 2015
(r285818)
+++ head/sys/ufs/ffs/ffs_rawread.c Thu Jul 23 19:13:41 2015
(r285819)
@@ -62,8 +62,7 @@ static int ffs_rawread_readahead(struct
off_t offset,
size_t len,
struct thread *td,
- struct buf *bp,
- caddr_t sa);
+ struct buf *bp);
static int ffs_rawread_main(struct vnode *vp,
struct uio *uio);
@@ -190,8 +189,7 @@ ffs_rawread_readahead(struct vnode *vp,
off_t offset,
size_t len,
struct thread *td,
- struct buf *bp,
- caddr_t sa)
+ struct buf *bp)
{
int error;
*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
_______________________________________________
[email protected] mailing list
http://lists.freebsd.org/mailman/listinfo/svn-src-all
To unsubscribe, send any mail to "[email protected]"
