Author: jeff
Date: Wed Oct 14 02:10:07 2015
New Revision: 289279
URL: https://svnweb.freebsd.org/changeset/base/289279
Log:
Parallelize the buffer cache and rewrite getnewbuf(). This results in a
8x performance improvement in a micro benchmark on a 4 socket machine.
- Get buffer headers from a per-cpu uma cache that sits in from of the
free queue.
- Use a per-cpu quantum cache in vmem to eliminate contention for kva.
- Use multiple clean queues according to buffer cache size to eliminate
clean queue lock contention.
- Introduce a bufspace daemon that attempts to prevent getnewbuf() callers
from blocking or doing direct recycling.
- Close some bufspace allocation races that could lead to endless
recycling.
- Further the transition to a more modern style of small functions grouped
by prefix in order to improve growing complexity.
Sponsored by: EMC / Isilon
Reviewed by: kib
Tested by: pho
Modified:
head/sys/kern/vfs_bio.c
head/sys/vm/vm_init.c
Modified: head/sys/kern/vfs_bio.c
==============================================================================
--- head/sys/kern/vfs_bio.c Wed Oct 14 00:43:29 2015 (r289278)
+++ head/sys/kern/vfs_bio.c Wed Oct 14 02:10:07 2015 (r289279)
@@ -63,6 +63,7 @@ __FBSDID("$FreeBSD$");
#include <sys/proc.h>
#include <sys/resourcevar.h>
#include <sys/rwlock.h>
+#include <sys/smp.h>
#include <sys/sysctl.h>
#include <sys/sysproto.h>
#include <sys/vmem.h>
@@ -100,6 +101,7 @@ caddr_t unmapped_buf;
/* Used below and for softdep flushing threads in ufs/ffs/ffs_softdep.c */
struct proc *bufdaemonproc;
+struct proc *bufspacedaemonproc;
static int inmem(struct vnode *vp, daddr_t blkno);
static void vm_hold_free_pages(struct buf *bp, int newbsize);
@@ -116,11 +118,18 @@ static void vfs_vmio_extend(struct buf *
static int vfs_bio_clcheck(struct vnode *vp, int size,
daddr_t lblkno, daddr_t blkno);
static int buf_flush(struct vnode *vp, int);
+static int buf_recycle(bool);
+static int buf_scan(bool);
static int flushbufqueues(struct vnode *, int, int);
static void buf_daemon(void);
static void bremfreel(struct buf *bp);
static __inline void bd_wakeup(void);
static int sysctl_runningspace(SYSCTL_HANDLER_ARGS);
+static void bufkva_reclaim(vmem_t *, int);
+static void bufkva_free(struct buf *);
+static int buf_import(void *, void **, int, int);
+static void buf_release(void *, void **, int);
+
#if defined(COMPAT_FREEBSD4) || defined(COMPAT_FREEBSD5) || \
defined(COMPAT_FREEBSD6) || defined(COMPAT_FREEBSD7)
static int sysctl_bufspace(SYSCTL_HANDLER_ARGS);
@@ -145,23 +154,23 @@ static long bufkvaspace;
SYSCTL_LONG(_vfs, OID_AUTO, bufkvaspace, CTLFLAG_RD, &bufkvaspace, 0,
"Kernel virtual memory used for buffers");
static long maxbufspace;
-SYSCTL_LONG(_vfs, OID_AUTO, maxbufspace, CTLFLAG_RD, &maxbufspace, 0,
- "Maximum allowed value of bufspace (including buf_daemon)");
+SYSCTL_LONG(_vfs, OID_AUTO, maxbufspace, CTLFLAG_RW, &maxbufspace, 0,
+ "Maximum allowed value of bufspace (including metadata)");
static long bufmallocspace;
SYSCTL_LONG(_vfs, OID_AUTO, bufmallocspace, CTLFLAG_RD, &bufmallocspace, 0,
"Amount of malloced memory for buffers");
static long maxbufmallocspace;
-SYSCTL_LONG(_vfs, OID_AUTO, maxmallocbufspace, CTLFLAG_RW, &maxbufmallocspace,
0,
- "Maximum amount of malloced memory for buffers");
+SYSCTL_LONG(_vfs, OID_AUTO, maxmallocbufspace, CTLFLAG_RW, &maxbufmallocspace,
+ 0, "Maximum amount of malloced memory for buffers");
static long lobufspace;
-SYSCTL_LONG(_vfs, OID_AUTO, lobufspace, CTLFLAG_RD, &lobufspace, 0,
+SYSCTL_LONG(_vfs, OID_AUTO, lobufspace, CTLFLAG_RW, &lobufspace, 0,
"Minimum amount of buffers we want to have");
long hibufspace;
-SYSCTL_LONG(_vfs, OID_AUTO, hibufspace, CTLFLAG_RD, &hibufspace, 0,
- "Maximum allowed value of bufspace (excluding buf_daemon)");
-static int bufreusecnt;
-SYSCTL_INT(_vfs, OID_AUTO, bufreusecnt, CTLFLAG_RW, &bufreusecnt, 0,
- "Number of times we have reused a buffer");
+SYSCTL_LONG(_vfs, OID_AUTO, hibufspace, CTLFLAG_RW, &hibufspace, 0,
+ "Maximum allowed value of bufspace (excluding metadata)");
+long bufspacethresh;
+SYSCTL_LONG(_vfs, OID_AUTO, bufspacethresh, CTLFLAG_RW, &bufspacethresh,
+ 0, "Bufspace consumed before waking the daemon to free some");
static int buffreekvacnt;
SYSCTL_INT(_vfs, OID_AUTO, buffreekvacnt, CTLFLAG_RW, &buffreekvacnt, 0,
"Number of times we have freed the KVA space from some buffer");
@@ -205,10 +214,10 @@ SYSCTL_INT(_vfs, OID_AUTO, numfreebuffer
"Number of free buffers");
static int lofreebuffers;
SYSCTL_INT(_vfs, OID_AUTO, lofreebuffers, CTLFLAG_RW, &lofreebuffers, 0,
- "XXX Unused");
+ "Target number of free buffers");
static int hifreebuffers;
SYSCTL_INT(_vfs, OID_AUTO, hifreebuffers, CTLFLAG_RW, &hifreebuffers, 0,
- "XXX Complicatedly unused");
+ "Threshold for clean buffer recycling");
static int getnewbufcalls;
SYSCTL_INT(_vfs, OID_AUTO, getnewbufcalls, CTLFLAG_RW, &getnewbufcalls, 0,
"Number of calls to getnewbuf");
@@ -219,6 +228,9 @@ static int mappingrestarts;
SYSCTL_INT(_vfs, OID_AUTO, mappingrestarts, CTLFLAG_RW, &mappingrestarts, 0,
"Number of times getblk has had to restart a buffer mapping for "
"unmapped buffer");
+static int numbufallocfails;
+SYSCTL_INT(_vfs, OID_AUTO, numbufallocfails, CTLFLAG_RW, &numbufallocfails, 0,
+ "Number of times buffer allocations failed");
static int flushbufqtarget = 100;
SYSCTL_INT(_vfs, OID_AUTO, flushbufqtarget, CTLFLAG_RW, &flushbufqtarget, 0,
"Amount of work to do in flushbufqueues when helping bufdaemon");
@@ -233,16 +245,6 @@ SYSCTL_INT(_vfs, OID_AUTO, unmapped_buf_
"Permit the use of the unmapped i/o");
/*
- * Lock for the non-dirty bufqueues
- */
-static struct mtx_padalign bqclean;
-
-/*
- * Lock for the dirty queue.
- */
-static struct mtx_padalign bqdirty;
-
-/*
* This lock synchronizes access to bd_request.
*/
static struct mtx_padalign bdlock;
@@ -271,6 +273,11 @@ static struct mtx_padalign bdirtylock;
static int bd_request;
/*
+ * Request/wakeup point for the bufspace daemon.
+ */
+static int bufspace_request;
+
+/*
* Request for the buf daemon to write more buffers than is indicated by
* lodirtybuf. This may be necessary to push out excess dependencies or
* defragment the address space where a simple count of the number of dirty
@@ -298,7 +305,7 @@ static int runningbufreq;
* Synchronization (sleep/wakeup) variable for buffer requests.
* Can contain the VFS_BIO_NEED flags defined below; setting/clearing is done
* by and/or.
- * Used in numdirtywakeup(), bufspacewakeup(), bufcountadd(), bwillwrite(),
+ * Used in numdirtywakeup(), bufspace_wakeup(), bwillwrite(),
* getnewbuf(), and getblk().
*/
static volatile int needsbuffer;
@@ -311,14 +318,21 @@ static int bdirtywait;
/*
* Definitions for the buffer free lists.
*/
-#define BUFFER_QUEUES 4 /* number of free buffer queues */
-
#define QUEUE_NONE 0 /* on no queue */
-#define QUEUE_CLEAN 1 /* non-B_DELWRI buffers */
+#define QUEUE_EMPTY 1 /* empty buffer headers */
#define QUEUE_DIRTY 2 /* B_DELWRI buffers */
-#define QUEUE_EMPTY 3 /* empty buffer headers */
+#define QUEUE_CLEAN 3 /* non-B_DELWRI buffers */
#define QUEUE_SENTINEL 1024 /* not an queue index, but mark for sentinel */
+/* Maximum number of clean buffer queues. */
+#define CLEAN_QUEUES 16
+
+/* Configured number of clean queues. */
+static int clean_queues;
+
+/* Maximum number of buffer queues. */
+#define BUFFER_QUEUES (QUEUE_CLEAN + CLEAN_QUEUES)
+
/* Queues for free buffers with various properties */
static TAILQ_HEAD(bqueues, buf) bufqueues[BUFFER_QUEUES] = { { 0 } };
#ifdef INVARIANTS
@@ -326,15 +340,21 @@ static int bq_len[BUFFER_QUEUES];
#endif
/*
+ * Lock for each bufqueue
+ */
+static struct mtx_padalign bqlocks[BUFFER_QUEUES];
+
+/*
+ * per-cpu empty buffer cache.
+ */
+uma_zone_t buf_zone;
+
+/*
* Single global constant for BUF_WMESG, to avoid getting multiple references.
* buf_wmesg is referred from macros.
*/
const char *buf_wmesg = BUF_WMESG;
-#define VFS_BIO_NEED_ANY 0x01 /* any freeable buffer */
-#define VFS_BIO_NEED_FREE 0x04 /* wait for free bufs, hi hysteresis */
-#define VFS_BIO_NEED_BUFSPACE 0x08 /* wait for buf space, lo hysteresis */
-
static int
sysctl_runningspace(SYSCTL_HANDLER_ARGS)
{
@@ -382,6 +402,21 @@ sysctl_bufspace(SYSCTL_HANDLER_ARGS)
}
#endif
+static int
+bqcleanq(void)
+{
+ static int nextq;
+
+ return ((atomic_fetchadd_int(&nextq, 1) % clean_queues) + QUEUE_CLEAN);
+}
+
+static int
+bqisclean(int qindex)
+{
+
+ return (qindex >= QUEUE_CLEAN && qindex < QUEUE_CLEAN + CLEAN_QUEUES);
+}
+
/*
* bqlock:
*
@@ -391,9 +426,7 @@ static inline struct mtx *
bqlock(int qindex)
{
- if (qindex == QUEUE_DIRTY)
- return (struct mtx *)(&bqdirty);
- return (struct mtx *)(&bqclean);
+ return (struct mtx *)&bqlocks[qindex];
}
/*
@@ -447,62 +480,255 @@ bdirtyadd(void)
}
/*
- * bufspacewakeup:
+ * bufspace_wakeup:
*
* Called when buffer space is potentially available for recovery.
* getnewbuf() will block on this flag when it is unable to free
* sufficient buffer space. Buffer space becomes recoverable when
* bp's get placed back in the queues.
*/
-static __inline void
-bufspacewakeup(void)
+static void
+bufspace_wakeup(void)
{
- int need_wakeup, on;
/*
- * If someone is waiting for bufspace, wake them up. Even
- * though we may not have freed the kva space yet, the waiting
- * process will be able to now.
+ * If someone is waiting for bufspace, wake them up.
+ *
+ * Since needsbuffer is set prior to doing an additional queue
+ * scan it is safe to check for the flag prior to acquiring the
+ * lock. The thread that is preparing to scan again before
+ * blocking would discover the buf we released.
*/
+ if (needsbuffer) {
+ rw_rlock(&nblock);
+ if (atomic_cmpset_int(&needsbuffer, 1, 0) == 1)
+ wakeup(__DEVOLATILE(void *, &needsbuffer));
+ rw_runlock(&nblock);
+ }
+}
+
+/*
+ * bufspace_daemonwakeup:
+ *
+ * Wakeup the daemon responsible for freeing clean bufs.
+ */
+static void
+bufspace_daemonwakeup(void)
+{
rw_rlock(&nblock);
- for (;;) {
- need_wakeup = 0;
- on = needsbuffer;
- if ((on & VFS_BIO_NEED_BUFSPACE) == 0)
- break;
- need_wakeup = 1;
- if (atomic_cmpset_rel_int(&needsbuffer, on,
- on & ~VFS_BIO_NEED_BUFSPACE))
- break;
+ if (bufspace_request == 0) {
+ bufspace_request = 1;
+ wakeup(&bufspace_request);
}
- if (need_wakeup)
- wakeup(__DEVOLATILE(void *, &needsbuffer));
rw_runlock(&nblock);
}
/*
- * bufspaceadjust:
+ * bufspace_adjust:
*
* Adjust the reported bufspace for a KVA managed buffer, possibly
* waking any waiters.
*/
static void
-bufspaceadjust(struct buf *bp, int bufsize)
+bufspace_adjust(struct buf *bp, int bufsize)
{
+ long space;
int diff;
KASSERT((bp->b_flags & B_MALLOC) == 0,
- ("bufspaceadjust: malloc buf %p", bp));
+ ("bufspace_adjust: malloc buf %p", bp));
diff = bufsize - bp->b_bufsize;
if (diff < 0) {
atomic_subtract_long(&bufspace, -diff);
- bufspacewakeup();
- } else
- atomic_add_long(&bufspace, diff);
+ bufspace_wakeup();
+ } else {
+ space = atomic_fetchadd_long(&bufspace, diff);
+ /* Wake up the daemon on the transition. */
+ if (space < bufspacethresh && space + diff >= bufspacethresh)
+ bufspace_daemonwakeup();
+ }
bp->b_bufsize = bufsize;
}
/*
+ * bufspace_reserve:
+ *
+ * Reserve bufspace before calling allocbuf(). metadata has a
+ * different space limit than data.
+ */
+static int
+bufspace_reserve(int size, bool metadata)
+{
+ long limit;
+ long space;
+
+ if (metadata)
+ limit = maxbufspace;
+ else
+ limit = hibufspace;
+ do {
+ space = bufspace;
+ if (space + size > limit)
+ return (ENOSPC);
+ } while (atomic_cmpset_long(&bufspace, space, space + size) == 0);
+
+ /* Wake up the daemon on the transition. */
+ if (space < bufspacethresh && space + size >= bufspacethresh)
+ bufspace_daemonwakeup();
+
+ return (0);
+}
+
+/*
+ * bufspace_release:
+ *
+ * Release reserved bufspace after bufspace_adjust() has consumed it.
+ */
+static void
+bufspace_release(int size)
+{
+ atomic_subtract_long(&bufspace, size);
+ bufspace_wakeup();
+}
+
+/*
+ * bufspace_wait:
+ *
+ * Wait for bufspace, acting as the buf daemon if a locked vnode is
+ * supplied. needsbuffer must be set in a safe fashion prior to
+ * polling for space. The operation must be re-tried on return.
+ */
+static void
+bufspace_wait(struct vnode *vp, int gbflags, int slpflag, int slptimeo)
+{
+ struct thread *td;
+ int error, fl, norunbuf;
+
+ if ((gbflags & GB_NOWAIT_BD) != 0)
+ return;
+
+ td = curthread;
+ rw_wlock(&nblock);
+ while (needsbuffer != 0) {
+ if (vp != NULL && vp->v_type != VCHR &&
+ (td->td_pflags & TDP_BUFNEED) == 0) {
+ rw_wunlock(&nblock);
+ /*
+ * getblk() is called with a vnode locked, and
+ * some majority of the dirty buffers may as
+ * well belong to the vnode. Flushing the
+ * buffers there would make a progress that
+ * cannot be achieved by the buf_daemon, that
+ * cannot lock the vnode.
+ */
+ norunbuf = ~(TDP_BUFNEED | TDP_NORUNNINGBUF) |
+ (td->td_pflags & TDP_NORUNNINGBUF);
+
+ /*
+ * Play bufdaemon. The getnewbuf() function
+ * may be called while the thread owns lock
+ * for another dirty buffer for the same
+ * vnode, which makes it impossible to use
+ * VOP_FSYNC() there, due to the buffer lock
+ * recursion.
+ */
+ td->td_pflags |= TDP_BUFNEED | TDP_NORUNNINGBUF;
+ fl = buf_flush(vp, flushbufqtarget);
+ td->td_pflags &= norunbuf;
+ rw_wlock(&nblock);
+ if (fl != 0)
+ continue;
+ if (needsbuffer == 0)
+ break;
+ }
+ error = rw_sleep(__DEVOLATILE(void *, &needsbuffer), &nblock,
+ (PRIBIO + 4) | slpflag, "newbuf", slptimeo);
+ if (error != 0)
+ break;
+ }
+ rw_wunlock(&nblock);
+}
+
+
+/*
+ * bufspace_daemon:
+ *
+ * buffer space management daemon. Tries to maintain some marginal
+ * amount of free buffer space so that requesting processes neither
+ * block nor work to reclaim buffers.
+ */
+static void
+bufspace_daemon(void)
+{
+ for (;;) {
+ kproc_suspend_check(bufspacedaemonproc);
+
+ /*
+ * Free buffers from the clean queue until we meet our
+ * targets.
+ *
+ * Theory of operation: The buffer cache is most efficient
+ * when some free buffer headers and space are always
+ * available to getnewbuf(). This daemon attempts to prevent
+ * the excessive blocking and synchronization associated
+ * with shortfall. It goes through three phases according
+ * demand:
+ *
+ * 1) The daemon wakes up voluntarily once per-second
+ * during idle periods when the counters are below
+ * the wakeup thresholds (bufspacethresh, lofreebuffers).
+ *
+ * 2) The daemon wakes up as we cross the thresholds
+ * ahead of any potential blocking. This may bounce
+ * slightly according to the rate of consumption and
+ * release.
+ *
+ * 3) The daemon and consumers are starved for working
+ * clean buffers. This is the 'bufspace' sleep below
+ * which will inefficiently trade bufs with bqrelse
+ * until we return to condition 2.
+ */
+ while (bufspace > lobufspace ||
+ numfreebuffers < hifreebuffers) {
+ if (buf_recycle(false) != 0) {
+ atomic_set_int(&needsbuffer, 1);
+ if (buf_recycle(false) != 0) {
+ rw_wlock(&nblock);
+ if (needsbuffer)
+ rw_sleep(__DEVOLATILE(void *,
+ &needsbuffer), &nblock,
+ PRIBIO|PDROP, "bufspace",
+ hz/10);
+ else
+ rw_wunlock(&nblock);
+ }
+ }
+ maybe_yield();
+ }
+
+ /*
+ * Re-check our limits under the exclusive nblock.
+ */
+ rw_wlock(&nblock);
+ if (bufspace < bufspacethresh &&
+ numfreebuffers > lofreebuffers) {
+ bufspace_request = 0;
+ rw_sleep(&bufspace_request, &nblock, PRIBIO|PDROP,
+ "-", hz);
+ } else
+ rw_wunlock(&nblock);
+ }
+}
+
+static struct kproc_desc bufspace_kp = {
+ "bufspacedaemon",
+ bufspace_daemon,
+ &bufspacedaemonproc
+};
+SYSINIT(bufspacedaemon, SI_SUB_KTHREAD_BUF, SI_ORDER_FIRST, kproc_start,
+ &bufspace_kp);
+
+/*
* bufmallocadjust:
*
* Adjust the reported bufspace for a malloc managed buffer, possibly
@@ -516,10 +742,9 @@ bufmallocadjust(struct buf *bp, int bufs
KASSERT((bp->b_flags & B_MALLOC) != 0,
("bufmallocadjust: non-malloc buf %p", bp));
diff = bufsize - bp->b_bufsize;
- if (diff < 0) {
+ if (diff < 0)
atomic_subtract_long(&bufmallocspace, -diff);
- bufspacewakeup();
- } else
+ else
atomic_add_long(&bufmallocspace, diff);
bp->b_bufsize = bufsize;
}
@@ -571,67 +796,6 @@ runningbufwakeup(struct buf *bp)
}
/*
- * bufcountadd:
- *
- * Called when a buffer has been added to one of the free queues to
- * account for the buffer and to wakeup anyone waiting for free buffers.
- * This typically occurs when large amounts of metadata are being handled
- * by the buffer cache ( else buffer space runs out first, usually ).
- */
-static __inline void
-bufcountadd(struct buf *bp)
-{
- int mask, need_wakeup, old, on;
-
- KASSERT((bp->b_flags & B_INFREECNT) == 0,
- ("buf %p already counted as free", bp));
- bp->b_flags |= B_INFREECNT;
- old = atomic_fetchadd_int(&numfreebuffers, 1);
- KASSERT(old >= 0 && old < nbuf,
- ("numfreebuffers climbed to %d", old + 1));
- mask = VFS_BIO_NEED_ANY;
- if (numfreebuffers >= hifreebuffers)
- mask |= VFS_BIO_NEED_FREE;
- rw_rlock(&nblock);
- for (;;) {
- need_wakeup = 0;
- on = needsbuffer;
- if (on == 0)
- break;
- need_wakeup = 1;
- if (atomic_cmpset_rel_int(&needsbuffer, on, on & ~mask))
- break;
- }
- if (need_wakeup)
- wakeup(__DEVOLATILE(void *, &needsbuffer));
- rw_runlock(&nblock);
-}
-
-/*
- * bufcountsub:
- *
- * Decrement the numfreebuffers count as needed.
- */
-static void
-bufcountsub(struct buf *bp)
-{
- int old;
-
- /*
- * Fixup numfreebuffers count. If the buffer is invalid or not
- * delayed-write, the buffer was free and we must decrement
- * numfreebuffers.
- */
- if ((bp->b_flags & B_INVAL) || (bp->b_flags & B_DELWRI) == 0) {
- KASSERT((bp->b_flags & B_INFREECNT) != 0,
- ("buf %p not counted in numfreebuffers", bp));
- bp->b_flags &= ~B_INFREECNT;
- old = atomic_fetchadd_int(&numfreebuffers, -1);
- KASSERT(old > 0, ("numfreebuffers dropped to %d", old - 1));
- }
-}
-
-/*
* waitrunningbufspace()
*
* runningbufspace is a measure of the amount of I/O currently
@@ -847,8 +1011,10 @@ bufinit(void)
int i;
CTASSERT(MAXBCACHEBUF >= MAXBSIZE);
- mtx_init(&bqclean, "bufq clean lock", NULL, MTX_DEF);
- mtx_init(&bqdirty, "bufq dirty lock", NULL, MTX_DEF);
+ mtx_init(&bqlocks[QUEUE_DIRTY], "bufq dirty lock", NULL, MTX_DEF);
+ mtx_init(&bqlocks[QUEUE_EMPTY], "bufq empty lock", NULL, MTX_DEF);
+ for (i = QUEUE_CLEAN; i < QUEUE_CLEAN + CLEAN_QUEUES; i++)
+ mtx_init(&bqlocks[i], "bufq clean lock", NULL, MTX_DEF);
mtx_init(&rbreqlock, "runningbufspace lock", NULL, MTX_DEF);
rw_init(&nblock, "needsbuffer lock");
mtx_init(&bdlock, "buffer daemon lock", NULL, MTX_DEF);
@@ -864,7 +1030,7 @@ bufinit(void)
for (i = 0; i < nbuf; i++) {
bp = &buf[i];
bzero(bp, sizeof *bp);
- bp->b_flags = B_INVAL | B_INFREECNT;
+ bp->b_flags = B_INVAL;
bp->b_rcred = NOCRED;
bp->b_wcred = NOCRED;
bp->b_qindex = QUEUE_EMPTY;
@@ -881,18 +1047,19 @@ bufinit(void)
/*
* maxbufspace is the absolute maximum amount of buffer space we are
* allowed to reserve in KVM and in real terms. The absolute maximum
- * is nominally used by buf_daemon. hibufspace is the nominal maximum
- * used by most other processes. The differential is required to
- * ensure that buf_daemon is able to run when other processes might
- * be blocked waiting for buffer space.
+ * is nominally used by metadata. hibufspace is the nominal maximum
+ * used by most other requests. The differential is required to
+ * ensure that metadata deadlocks don't occur.
*
* maxbufspace is based on BKVASIZE. Allocating buffers larger then
* this may result in KVM fragmentation which is not handled optimally
- * by the system.
+ * by the system. XXX This is less true with vmem. We could use
+ * PAGE_SIZE.
*/
maxbufspace = (long)nbuf * BKVASIZE;
hibufspace = lmax(3 * maxbufspace / 4, maxbufspace - MAXBCACHEBUF * 10);
- lobufspace = hibufspace - MAXBCACHEBUF;
+ lobufspace = (hibufspace / 20) * 19; /* 95% */
+ bufspacethresh = lobufspace + (hibufspace - lobufspace) / 2;
/*
* Note: The 16 MiB upper limit for hirunningspace was chosen
@@ -906,44 +1073,61 @@ bufinit(void)
16 * 1024 * 1024), 1024 * 1024);
lorunningspace = roundup((hirunningspace * 2) / 3, MAXBCACHEBUF);
-/*
- * Limit the amount of malloc memory since it is wired permanently into
- * the kernel space. Even though this is accounted for in the buffer
- * allocation, we don't want the malloced region to grow uncontrolled.
- * The malloc scheme improves memory utilization significantly on average
- * (small) directories.
- */
+ /*
+ * Limit the amount of malloc memory since it is wired permanently into
+ * the kernel space. Even though this is accounted for in the buffer
+ * allocation, we don't want the malloced region to grow uncontrolled.
+ * The malloc scheme improves memory utilization significantly on
+ * average (small) directories.
+ */
maxbufmallocspace = hibufspace / 20;
-/*
- * Reduce the chance of a deadlock occuring by limiting the number
- * of delayed-write dirty buffers we allow to stack up.
- */
+ /*
+ * Reduce the chance of a deadlock occuring by limiting the number
+ * of delayed-write dirty buffers we allow to stack up.
+ */
hidirtybuffers = nbuf / 4 + 20;
dirtybufthresh = hidirtybuffers * 9 / 10;
numdirtybuffers = 0;
-/*
- * To support extreme low-memory systems, make sure hidirtybuffers cannot
- * eat up all available buffer space. This occurs when our minimum cannot
- * be met. We try to size hidirtybuffers to 3/4 our buffer space assuming
- * BKVASIZE'd buffers.
- */
+ /*
+ * To support extreme low-memory systems, make sure hidirtybuffers
+ * cannot eat up all available buffer space. This occurs when our
+ * minimum cannot be met. We try to size hidirtybuffers to 3/4 our
+ * buffer space assuming BKVASIZE'd buffers.
+ */
while ((long)hidirtybuffers * BKVASIZE > 3 * hibufspace / 4) {
hidirtybuffers >>= 1;
}
lodirtybuffers = hidirtybuffers / 2;
-/*
- * Try to keep the number of free buffers in the specified range,
- * and give special processes (e.g. like buf_daemon) access to an
- * emergency reserve.
- */
- lofreebuffers = nbuf / 18 + 5;
- hifreebuffers = 2 * lofreebuffers;
+ /*
+ * lofreebuffers should be sufficient to avoid stalling waiting on
+ * buf headers under heavy utilization. The bufs in per-cpu caches
+ * are counted as free but will be unavailable to threads executing
+ * on other cpus.
+ *
+ * hifreebuffers is the free target for the bufspace daemon. This
+ * should be set appropriately to limit work per-iteration.
+ */
+ lofreebuffers = MIN((nbuf / 25) + (20 * mp_ncpus), 128 * mp_ncpus);
+ hifreebuffers = (3 * lofreebuffers) / 2;
numfreebuffers = nbuf;
bogus_page = vm_page_alloc(NULL, 0, VM_ALLOC_NOOBJ |
VM_ALLOC_NORMAL | VM_ALLOC_WIRED);
+
+ /* Setup the kva and free list allocators. */
+ vmem_set_reclaim(buffer_arena, bufkva_reclaim);
+ buf_zone = uma_zcache_create("buf free cache", sizeof(struct buf),
+ NULL, NULL, NULL, NULL, buf_import, buf_release, NULL, 0);
+
+ /*
+ * Size the clean queue according to the amount of buffer space.
+ * One queue per-256mb up to the max. More queues gives better
+ * concurrency but less accurate LRU.
+ */
+ clean_queues = MIN(howmany(maxbufspace, 256*1024*1024), CLEAN_QUEUES);
+
}
#ifdef INVARIANTS
@@ -1129,10 +1313,25 @@ binsfree(struct buf *bp, int qindex)
{
struct mtx *olock, *nlock;
- BUF_ASSERT_XLOCKED(bp);
+ if (qindex != QUEUE_EMPTY) {
+ BUF_ASSERT_XLOCKED(bp);
+ }
+
+ /*
+ * Stick to the same clean queue for the lifetime of the buf to
+ * limit locking below. Otherwise pick ont sequentially.
+ */
+ if (qindex == QUEUE_CLEAN) {
+ if (bqisclean(bp->b_qindex))
+ qindex = bp->b_qindex;
+ else
+ qindex = bqcleanq();
+ }
+ /*
+ * Handle delayed bremfree() processing.
+ */
nlock = bqlock(qindex);
- /* Handle delayed bremfree() processing. */
if (bp->b_flags & B_REMFREE) {
olock = bqlock(bp->b_qindex);
mtx_lock(olock);
@@ -1156,15 +1355,263 @@ binsfree(struct buf *bp, int qindex)
bq_len[bp->b_qindex]++;
#endif
mtx_unlock(nlock);
+}
+
+/*
+ * buf_free:
+ *
+ * Free a buffer to the buf zone once it no longer has valid contents.
+ */
+static void
+buf_free(struct buf *bp)
+{
+
+ if (bp->b_flags & B_REMFREE)
+ bremfreef(bp);
+ if (bp->b_vflags & BV_BKGRDINPROG)
+ panic("losing buffer 1");
+ if (bp->b_rcred != NOCRED) {
+ crfree(bp->b_rcred);
+ bp->b_rcred = NOCRED;
+ }
+ if (bp->b_wcred != NOCRED) {
+ crfree(bp->b_wcred);
+ bp->b_wcred = NOCRED;
+ }
+ if (!LIST_EMPTY(&bp->b_dep))
+ buf_deallocate(bp);
+ bufkva_free(bp);
+ BUF_UNLOCK(bp);
+ uma_zfree(buf_zone, bp);
+ atomic_add_int(&numfreebuffers, 1);
+ bufspace_wakeup();
+}
+
+/*
+ * buf_import:
+ *
+ * Import bufs into the uma cache from the buf list. The system still
+ * expects a static array of bufs and much of the synchronization
+ * around bufs assumes type stable storage. As a result, UMA is used
+ * only as a per-cpu cache of bufs still maintained on a global list.
+ */
+static int
+buf_import(void *arg, void **store, int cnt, int flags)
+{
+ struct buf *bp;
+ int i;
+
+ mtx_lock(&bqlocks[QUEUE_EMPTY]);
+ for (i = 0; i < cnt; i++) {
+ bp = TAILQ_FIRST(&bufqueues[QUEUE_EMPTY]);
+ if (bp == NULL)
+ break;
+ bremfreel(bp);
+ store[i] = bp;
+ }
+ mtx_unlock(&bqlocks[QUEUE_EMPTY]);
+
+ return (i);
+}
+
+/*
+ * buf_release:
+ *
+ * Release bufs from the uma cache back to the buffer queues.
+ */
+static void
+buf_release(void *arg, void **store, int cnt)
+{
+ int i;
+
+ for (i = 0; i < cnt; i++)
+ binsfree(store[i], QUEUE_EMPTY);
+}
+
+/*
+ * buf_alloc:
+ *
+ * Allocate an empty buffer header.
+ */
+static struct buf *
+buf_alloc(void)
+{
+ struct buf *bp;
+
+ bp = uma_zalloc(buf_zone, M_NOWAIT);
+ if (bp == NULL) {
+ bufspace_daemonwakeup();
+ atomic_add_int(&numbufallocfails, 1);
+ return (NULL);
+ }
+
+ /*
+ * Wake-up the bufspace daemon on transition.
+ */
+ if (atomic_fetchadd_int(&numfreebuffers, -1) == lofreebuffers)
+ bufspace_daemonwakeup();
+
+ if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT, NULL) != 0)
+ panic("getnewbuf_empty: Locked buf %p on free queue.", bp);
+
+ KASSERT(bp->b_vp == NULL,
+ ("bp: %p still has vnode %p.", bp, bp->b_vp));
+ KASSERT((bp->b_flags & (B_DELWRI | B_NOREUSE)) == 0,
+ ("invalid buffer %p flags %#x", bp, bp->b_flags));
+ KASSERT((bp->b_xflags & (BX_VNCLEAN|BX_VNDIRTY)) == 0,
+ ("bp: %p still on a buffer list. xflags %X", bp, bp->b_xflags));
+ KASSERT(bp->b_npages == 0,
+ ("bp: %p still has %d vm pages\n", bp, bp->b_npages));
+ KASSERT(bp->b_kvasize == 0, ("bp: %p still has kva\n", bp));
+ KASSERT(bp->b_bufsize == 0, ("bp: %p still has bufspace\n", bp));
+
+ bp->b_flags = 0;
+ bp->b_ioflags = 0;
+ bp->b_xflags = 0;
+ bp->b_vflags = 0;
+ bp->b_vp = NULL;
+ bp->b_blkno = bp->b_lblkno = 0;
+ bp->b_offset = NOOFFSET;
+ bp->b_iodone = 0;
+ bp->b_error = 0;
+ bp->b_resid = 0;
+ bp->b_bcount = 0;
+ bp->b_npages = 0;
+ bp->b_dirtyoff = bp->b_dirtyend = 0;
+ bp->b_bufobj = NULL;
+ bp->b_pin_count = 0;
+ bp->b_data = bp->b_kvabase = unmapped_buf;
+ bp->b_fsprivate1 = NULL;
+ bp->b_fsprivate2 = NULL;
+ bp->b_fsprivate3 = NULL;
+ LIST_INIT(&bp->b_dep);
+
+ return (bp);
+}
+
+/*
+ * buf_qrecycle:
+ *
+ * Free a buffer from the given bufqueue. kva controls whether the
+ * freed buf must own some kva resources. This is used for
+ * defragmenting.
+ */
+static int
+buf_qrecycle(int qindex, bool kva)
+{
+ struct buf *bp, *nbp;
+
+ if (kva)
+ atomic_add_int(&bufdefragcnt, 1);
+ nbp = NULL;
+ mtx_lock(&bqlocks[qindex]);
+ nbp = TAILQ_FIRST(&bufqueues[qindex]);
+
+ /*
+ * Run scan, possibly freeing data and/or kva mappings on the fly
+ * depending.
+ */
+ while ((bp = nbp) != NULL) {
+ /*
+ * Calculate next bp (we can only use it if we do not
+ * release the bqlock).
+ */
+ nbp = TAILQ_NEXT(bp, b_freelist);
+
+ /*
+ * If we are defragging then we need a buffer with
+ * some kva to reclaim.
+ */
+ if (kva && bp->b_kvasize == 0)
+ continue;
+
+ if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT, NULL) != 0)
+ continue;
+
+ /*
+ * Skip buffers with background writes in progress.
+ */
+ if ((bp->b_vflags & BV_BKGRDINPROG) != 0) {
+ BUF_UNLOCK(bp);
+ continue;
+ }
+
+ KASSERT(bp->b_qindex == qindex,
+ ("getnewbuf: inconsistent queue %d bp %p", qindex, bp));
+ /*
+ * NOTE: nbp is now entirely invalid. We can only restart
+ * the scan from this point on.
+ */
+ bremfreel(bp);
+ mtx_unlock(&bqlocks[qindex]);
+
+ /*
+ * Requeue the background write buffer with error and
+ * restart the scan.
+ */
+ if ((bp->b_vflags & BV_BKGRDERR) != 0) {
+ bqrelse(bp);
+ mtx_lock(&bqlocks[qindex]);
+ nbp = TAILQ_FIRST(&bufqueues[qindex]);
+ continue;
+ }
+ bp->b_flags |= B_INVAL;
+ brelse(bp);
+ return (0);
+ }
+ mtx_unlock(&bqlocks[qindex]);
+
+ return (ENOBUFS);
+}
+
+/*
+ * buf_recycle:
+ *
+ * Iterate through all clean queues until we find a buf to recycle or
+ * exhaust the search.
+ */
+static int
+buf_recycle(bool kva)
+{
+ int qindex, first_qindex;
+
+ qindex = first_qindex = bqcleanq();
+ do {
+ if (buf_qrecycle(qindex, kva) == 0)
+ return (0);
+ if (++qindex == QUEUE_CLEAN + clean_queues)
+ qindex = QUEUE_CLEAN;
+ } while (qindex != first_qindex);
+
+ return (ENOBUFS);
+}
+
+/*
+ * buf_scan:
+ *
+ * Scan the clean queues looking for a buffer to recycle. needsbuffer
+ * is set on failure so that the caller may optionally bufspace_wait()
+ * in a race-free fashion.
+ */
+static int
+buf_scan(bool defrag)
+{
+ int error;
/*
- * Something we can maybe free or reuse.
- */
- if (bp->b_bufsize && !(bp->b_flags & B_DELWRI))
- bufspacewakeup();
-
- if ((bp->b_flags & B_INVAL) || !(bp->b_flags & B_DELWRI))
- bufcountadd(bp);
+ * To avoid heavy synchronization and wakeup races we set
+ * needsbuffer and re-poll before failing. This ensures that
+ * no frees can be missed between an unsuccessful poll and
+ * going to sleep in a synchronized fashion.
+ */
*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
_______________________________________________
svn-src-all@freebsd.org mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-all
To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"
