this diff lets the pool gc reclaim idle memory held by cpu caches.
it does this by recording when the global depot of items was first
added to. the gc checks that timestamp and if the depot hasnt been
emptied again in the last 4 seconds, it moves a list of items from
the depot back to the pools free lists. this in turn makes the items
available for freeing back to the system.
the obvious way to put items back on the free lists is to call
pool_put on them, but doing that double counts puts which can lead
to negative INUSE values in vmstat -m and systat pool output. to
avoid that the guts of pool_put have been pulled out into pool_do_put
(which mirrors pool_do_get in some ways). pool_do_put simply
manipulates the free lists, leaving pool_put responsible for work
it usually does around that like updating counters.
pulling pool_do_put out also means the gc can take the mutex once
to put several items on the lists in one go.
ok?
Index: subr_pool.c
===================================================================
RCS file: /cvs/src/sys/kern/subr_pool.c,v
retrieving revision 1.206
diff -u -p -d -r1.206 subr_pool.c
--- subr_pool.c 8 Feb 2017 05:28:30 -0000 1.206
+++ subr_pool.c 8 Feb 2017 05:42:47 -0000
@@ -132,6 +132,8 @@ struct pool_cache {
void *pool_cache_get(struct pool *);
void pool_cache_put(struct pool *, void *);
void pool_cache_destroy(struct pool *);
+struct pool_cache_item *
+ pool_cache_list_put(struct pool *, struct pool_cache_item *);
#endif
void pool_cache_info(struct pool *, struct kinfo_pool *);
@@ -151,6 +153,7 @@ void pool_p_free(struct pool *, struct
void pool_update_curpage(struct pool *);
void *pool_do_get(struct pool *, int, int *);
+void pool_do_put(struct pool *, void *);
int pool_chk_page(struct pool *, struct pool_page_header *, int);
int pool_chk(struct pool *);
void pool_get_done(void *, void *);
@@ -706,7 +709,6 @@ pool_do_get(struct pool *pp, int flags,
void
pool_put(struct pool *pp, void *v)
{
- struct pool_item *pi = v;
struct pool_page_header *ph, *freeph = NULL;
#ifdef DIAGNOSTIC
@@ -723,6 +725,37 @@ pool_put(struct pool *pp, void *v)
mtx_enter(&pp->pr_mtx);
+ pool_do_put(pp, v);
+
+ pp->pr_nout--;
+ pp->pr_nput++;
+
+ /* is it time to free a page? */
+ if (pp->pr_nidle > pp->pr_maxpages &&
+ (ph = TAILQ_FIRST(&pp->pr_emptypages)) != NULL &&
+ (ticks - ph->ph_tick) > (hz * pool_wait_free)) {
+ freeph = ph;
+ pool_p_remove(pp, freeph);
+ }
+
+ mtx_leave(&pp->pr_mtx);
+
+ if (freeph != NULL)
+ pool_p_free(pp, freeph);
+
+ if (!TAILQ_EMPTY(&pp->pr_requests)) {
+ mtx_enter(&pp->pr_requests_mtx);
+ pool_runqueue(pp, PR_NOWAIT);
+ mtx_leave(&pp->pr_requests_mtx);
+ }
+}
+
+void
+pool_do_put(struct pool *pp, void *v)
+{
+ struct pool_item *pi = v;
+ struct pool_page_header *ph;
+
splassert(pp->pr_ipl);
ph = pr_find_pagehead(pp, v);
@@ -766,27 +799,6 @@ pool_put(struct pool *pp, void *v)
TAILQ_INSERT_TAIL(&pp->pr_emptypages, ph, ph_entry);
pool_update_curpage(pp);
}
-
- pp->pr_nout--;
- pp->pr_nput++;
-
- /* is it time to free a page? */
- if (pp->pr_nidle > pp->pr_maxpages &&
- (ph = TAILQ_FIRST(&pp->pr_emptypages)) != NULL &&
- (ticks - ph->ph_tick) > (hz * pool_wait_free)) {
- freeph = ph;
- pool_p_remove(pp, freeph);
- }
- mtx_leave(&pp->pr_mtx);
-
- if (freeph != NULL)
- pool_p_free(pp, freeph);
-
- if (!TAILQ_EMPTY(&pp->pr_requests)) {
- mtx_enter(&pp->pr_requests_mtx);
- pool_runqueue(pp, PR_NOWAIT);
- mtx_leave(&pp->pr_requests_mtx);
- }
}
/*
@@ -1446,9 +1458,32 @@ pool_gc_pages(void *null)
{
struct pool *pp;
struct pool_page_header *ph, *freeph;
+ int diff = hz * pool_wait_gc;
rw_enter_read(&pool_lock);
SIMPLEQ_FOREACH(pp, &pool_head, pr_poollist) {
+#ifdef MULTIPROCESSOR
+ /* if there's percpu caches, try and gc a list of items */
+ if (pp->pr_cache != NULL &&
+ (ticks - pp->pr_cache_tick) > diff &&
+ !TAILQ_EMPTY(&pp->pr_cache_lists) &&
+ mtx_enter_try(&pp->pr_cache_mtx)) {
+ struct pool_cache_item *pl = NULL;
+
+ if ((ticks - pp->pr_cache_tick) > diff &&
+ (pl = TAILQ_FIRST(&pp->pr_cache_lists)) != NULL) {
+ TAILQ_REMOVE(&pp->pr_cache_lists, pl, ci_nextl);
+ pp->pr_cache_tick = ticks;
+ pp->pr_cache_nlist--;
+ }
+
+ mtx_leave(&pp->pr_cache_mtx);
+
+ if (pl != NULL)
+ pool_cache_list_put(pp, pl);
+ }
+#endif
+
if (pp->pr_nidle <= pp->pr_minpages || /* guess */
!mtx_enter_try(&pp->pr_mtx)) /* try */
continue;
@@ -1456,7 +1491,7 @@ pool_gc_pages(void *null)
/* is it time to free a page? */
if (pp->pr_nidle > pp->pr_minpages &&
(ph = TAILQ_FIRST(&pp->pr_emptypages)) != NULL &&
- (ticks - ph->ph_tick) > (hz * pool_wait_gc)) {
+ (ticks - ph->ph_tick) > diff) {
freeph = ph;
pool_p_remove(pp, freeph);
} else
@@ -1705,6 +1740,9 @@ pool_cache_list_free(struct pool *pp, st
struct pool_cache_item *ci)
{
pool_list_enter(pp);
+ if (TAILQ_EMPTY(&pp->pr_cache_lists))
+ pp->pr_cache_tick = ticks;
+
TAILQ_INSERT_TAIL(&pp->pr_cache_lists, ci, ci_nextl);
pp->pr_cache_nlist++;
@@ -1837,11 +1875,13 @@ pool_cache_list_put(struct pool *pp, str
rpl = TAILQ_NEXT(pl, ci_nextl);
+ mtx_enter(&pp->pr_mtx);
do {
next = pl->ci_next;
- pool_put(pp, pl);
+ pool_do_put(pp, pl);
pl = next;
} while (pl != NULL);
+ mtx_leave(&pp->pr_mtx);
return (rpl);
}
