Author: alc Date: Tue Nov 28 17:46:03 2017 New Revision: 326329 URL: https://svnweb.freebsd.org/changeset/base/326329
Log: When the swap pager allocates space on disk, it requests contiguous blocks in a single call to blist_alloc(). However, when it frees that space, it previously called blist_free() on each block, one at a time. With this change, the swap pager identifies ranges of contiguous blocks to be freed, and calls blist_free() once per range. In one extreme case, that is described in the review, the time to perform an munmap(2) was reduced by 55%. Submitted by: Doug Moore <do...@rice.edu> Reviewed by: kib MFC after: 1 week Differential Revision: https://reviews.freebsd.org/D12397 Modified: head/sys/vm/swap_pager.c Modified: head/sys/vm/swap_pager.c ============================================================================== --- head/sys/vm/swap_pager.c Tue Nov 28 17:33:10 2017 (r326328) +++ head/sys/vm/swap_pager.c Tue Nov 28 17:46:03 2017 (r326329) @@ -390,6 +390,7 @@ SYSCTL_INT(_vm, OID_AUTO, dmmax, CTLFLAG_RD, &nsw_clus static void swp_sizecheck(void); static void swp_pager_async_iodone(struct buf *bp); +static bool swp_pager_swblk_empty(struct swblk *sb, int start, int limit); static int swapongeom(struct vnode *); static int swaponvp(struct thread *, struct vnode *, u_long); static int swapoff_one(struct swdevt *sp, struct ucred *cred); @@ -397,7 +398,7 @@ static int swapoff_one(struct swdevt *sp, struct ucred /* * Swap bitmap functions */ -static void swp_pager_freeswapspace(daddr_t blk, int npages); +static void swp_pager_freeswapspace(daddr_t blk, daddr_t npages); static daddr_t swp_pager_getswapspace(int npages); /* @@ -768,10 +769,12 @@ swp_pager_strategy(struct buf *bp) * This routine may not sleep. */ static void -swp_pager_freeswapspace(daddr_t blk, int npages) +swp_pager_freeswapspace(daddr_t blk, daddr_t npages) { struct swdevt *sp; + if (npages == 0) + return; mtx_lock(&sw_dev_mtx); TAILQ_FOREACH(sp, &swtailq, sw_list) { if (blk >= sp->sw_first && blk < sp->sw_end) { @@ -1761,6 +1764,22 @@ next_obj: */ /* + * SWP_PAGER_SWBLK_EMPTY() - is a range of blocks free? + */ +static bool +swp_pager_swblk_empty(struct swblk *sb, int start, int limit) +{ + int i; + + MPASS(0 <= start && start <= limit && limit <= SWAP_META_PAGES); + for (i = start; i < limit; i++) { + if (sb->d[i] != SWAPBLK_NONE) + return (false); + } + return (true); +} + +/* * SWP_PAGER_META_BUILD() - add swap block to swap meta data for object * * We first convert the object to a swap object if it is a default @@ -1876,16 +1895,10 @@ allocated: /* * Free the swblk if we end up with the empty page run. */ - if (swapblk == SWAPBLK_NONE) { - for (i = 0; i < SWAP_META_PAGES; i++) { - if (sb->d[i] != SWAPBLK_NONE) - break; - } - if (i == SWAP_META_PAGES) { - SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, - rdpi); - uma_zfree(swblk_zone, sb); - } + if (swapblk == SWAPBLK_NONE && + swp_pager_swblk_empty(sb, 0, SWAP_META_PAGES)) { + SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, rdpi); + uma_zfree(swblk_zone, sb); } } @@ -1903,37 +1916,46 @@ static void swp_pager_meta_free(vm_object_t object, vm_pindex_t pindex, vm_pindex_t count) { struct swblk *sb; + daddr_t first_free, num_free; vm_pindex_t last; - int i; - bool empty; + int i, limit, start; VM_OBJECT_ASSERT_WLOCKED(object); if (object->type != OBJT_SWAP || count == 0) return; - last = pindex + count - 1; + first_free = SWAPBLK_NONE; + num_free = 0; + last = pindex + count; for (;;) { sb = SWAP_PCTRIE_LOOKUP_GE(&object->un_pager.swp.swp_blks, rounddown(pindex, SWAP_META_PAGES)); - if (sb == NULL || sb->p > last) + if (sb == NULL || sb->p >= last) break; - empty = true; - for (i = 0; i < SWAP_META_PAGES; i++) { + start = pindex > sb->p ? pindex - sb->p : 0; + limit = last - sb->p < SWAP_META_PAGES ? last - sb->p : + SWAP_META_PAGES; + for (i = start; i < limit; i++) { if (sb->d[i] == SWAPBLK_NONE) continue; - if (pindex <= sb->p + i && sb->p + i <= last) { - swp_pager_freeswapspace(sb->d[i], 1); - sb->d[i] = SWAPBLK_NONE; - } else - empty = false; + if (first_free + num_free == sb->d[i]) + num_free++; + else { + swp_pager_freeswapspace(first_free, num_free); + first_free = sb->d[i]; + num_free = 1; + } + sb->d[i] = SWAPBLK_NONE; } - pindex = sb->p + SWAP_META_PAGES; - if (empty) { + if (swp_pager_swblk_empty(sb, 0, start) && + swp_pager_swblk_empty(sb, limit, SWAP_META_PAGES)) { SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, sb->p); uma_zfree(swblk_zone, sb); } + pindex = sb->p + SWAP_META_PAGES; } + swp_pager_freeswapspace(first_free, num_free); } /* @@ -1946,6 +1968,7 @@ static void swp_pager_meta_free_all(vm_object_t object) { struct swblk *sb; + daddr_t first_free, num_free; vm_pindex_t pindex; int i; @@ -1953,16 +1976,26 @@ swp_pager_meta_free_all(vm_object_t object) if (object->type != OBJT_SWAP) return; + first_free = SWAPBLK_NONE; + num_free = 0; for (pindex = 0; (sb = SWAP_PCTRIE_LOOKUP_GE( &object->un_pager.swp.swp_blks, pindex)) != NULL;) { pindex = sb->p + SWAP_META_PAGES; for (i = 0; i < SWAP_META_PAGES; i++) { - if (sb->d[i] != SWAPBLK_NONE) - swp_pager_freeswapspace(sb->d[i], 1); + if (sb->d[i] == SWAPBLK_NONE) + continue; + if (first_free + num_free == sb->d[i]) + num_free++; + else { + swp_pager_freeswapspace(first_free, num_free); + first_free = sb->d[i]; + num_free = 1; + } } SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, sb->p); uma_zfree(swblk_zone, sb); } + swp_pager_freeswapspace(first_free, num_free); } /* @@ -1987,7 +2020,6 @@ swp_pager_meta_ctl(vm_object_t object, vm_pindex_t pin { struct swblk *sb; daddr_t r1; - int i; if ((flags & (SWM_FREE | SWM_POP)) != 0) VM_OBJECT_ASSERT_WLOCKED(object); @@ -2010,11 +2042,7 @@ swp_pager_meta_ctl(vm_object_t object, vm_pindex_t pin return (SWAPBLK_NONE); if ((flags & (SWM_FREE | SWM_POP)) != 0) { sb->d[pindex % SWAP_META_PAGES] = SWAPBLK_NONE; - for (i = 0; i < SWAP_META_PAGES; i++) { - if (sb->d[i] != SWAPBLK_NONE) - break; - } - if (i == SWAP_META_PAGES) { + if (swp_pager_swblk_empty(sb, 0, SWAP_META_PAGES)) { SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, rounddown(pindex, SWAP_META_PAGES)); uma_zfree(swblk_zone, sb); _______________________________________________ svn-src-head@freebsd.org mailing list https://lists.freebsd.org/mailman/listinfo/svn-src-head To unsubscribe, send any mail to "svn-src-head-unsubscr...@freebsd.org"
