Author: alc Date: Wed Aug 8 02:30:34 2018 New Revision: 337443 URL: https://svnweb.freebsd.org/changeset/base/337443
Log: Defer and aggregate swap_pager_meta_build frees. Before swp_pager_meta_build replaces an old swapblk with an new one, it frees the old one. To allow such freeing of blocks to be aggregated, have swp_pager_meta_build return the old swap block, and make the caller responsible for freeing it. Define a pair of short static functions, swp_pager_init_freerange and swp_pager_update_freerange, to do the initialization and updating of blk addresses and counters used in aggregating blocks to be freed. Submitted by: Doug Moore <do...@rice.edu> Reviewed by: kib, markj (an earlier version) Tested by: pho MFC after: 1 week Differential Revision: https://reviews.freebsd.org/D13707 Modified: head/sys/vm/swap_pager.c Modified: head/sys/vm/swap_pager.c ============================================================================== --- head/sys/vm/swap_pager.c Wed Aug 8 01:33:36 2018 (r337442) +++ head/sys/vm/swap_pager.c Wed Aug 8 02:30:34 2018 (r337443) @@ -403,11 +403,32 @@ static daddr_t swp_pager_getswapspace(int npages); /* * Metadata functions */ -static void swp_pager_meta_build(vm_object_t, vm_pindex_t, daddr_t); +static daddr_t swp_pager_meta_build(vm_object_t, vm_pindex_t, daddr_t); static void swp_pager_meta_free(vm_object_t, vm_pindex_t, vm_pindex_t); static void swp_pager_meta_free_all(vm_object_t); static daddr_t swp_pager_meta_ctl(vm_object_t, vm_pindex_t, int); +static void +swp_pager_init_freerange(daddr_t *start, daddr_t *num) +{ + + *start = SWAPBLK_NONE; + *num = 0; +} + +static void +swp_pager_update_freerange(daddr_t *start, daddr_t *num, daddr_t addr) +{ + + if (*start + *num == addr) { + (*num)++; + } else { + swp_pager_freeswapspace(*start, *num); + *start = addr; + *num = 1; + } +} + static void * swblk_trie_alloc(struct pctrie *ptree) { @@ -861,7 +882,9 @@ swap_pager_reserve(vm_object_t object, vm_pindex_t sta int n = 0; daddr_t blk = SWAPBLK_NONE; vm_pindex_t beg = start; /* save start index */ + daddr_t addr, n_free, s_free; + swp_pager_init_freerange(&s_free, &n_free); VM_OBJECT_WLOCK(object); while (size) { if (n == 0) { @@ -875,12 +898,15 @@ swap_pager_reserve(vm_object_t object, vm_pindex_t sta } } } - swp_pager_meta_build(object, start, blk); + addr = swp_pager_meta_build(object, start, blk); + if (addr != SWAPBLK_NONE) + swp_pager_update_freerange(&s_free, &n_free, addr); --size; ++start; ++blk; --n; } + swp_pager_freeswapspace(s_free, n_free); swp_pager_meta_free(object, start, n); VM_OBJECT_WUNLOCK(object); return (0); @@ -910,7 +936,7 @@ swap_pager_copy(vm_object_t srcobject, vm_object_t dst vm_pindex_t offset, int destroysource) { vm_pindex_t i; - daddr_t dstaddr, first_free, num_free, srcaddr; + daddr_t dstaddr, n_free, s_free, srcaddr; VM_OBJECT_ASSERT_WLOCKED(srcobject); VM_OBJECT_ASSERT_WLOCKED(dstobject); @@ -937,42 +963,38 @@ swap_pager_copy(vm_object_t srcobject, vm_object_t dst /* * Transfer source to destination. */ - first_free = SWAPBLK_NONE; - num_free = 0; + swp_pager_init_freerange(&s_free, &n_free); for (i = 0; i < dstobject->size; ++i) { srcaddr = swp_pager_meta_ctl(srcobject, i + offset, SWM_POP); if (srcaddr == SWAPBLK_NONE) continue; dstaddr = swp_pager_meta_ctl(dstobject, i, 0); - if (dstaddr == SWAPBLK_NONE) { + if (dstaddr != SWAPBLK_NONE) { /* - * Destination has no swapblk and is not resident, - * copy source. - * - * swp_pager_meta_build() can sleep. - */ - vm_object_pip_add(srcobject, 1); - VM_OBJECT_WUNLOCK(srcobject); - vm_object_pip_add(dstobject, 1); - swp_pager_meta_build(dstobject, i, srcaddr); - vm_object_pip_wakeup(dstobject); - VM_OBJECT_WLOCK(srcobject); - vm_object_pip_wakeup(srcobject); - } else { - /* * Destination has valid swapblk or it is represented - * by a resident page. We destroy the sourceblock. + * by a resident page. We destroy the source block. */ - if (first_free + num_free == srcaddr) - num_free++; - else { - swp_pager_freeswapspace(first_free, num_free); - first_free = srcaddr; - num_free = 1; - } + swp_pager_update_freerange(&s_free, &n_free, srcaddr); + continue; } + + /* + * Destination has no swapblk and is not resident, + * copy source. + * + * swp_pager_meta_build() can sleep. + */ + vm_object_pip_add(srcobject, 1); + VM_OBJECT_WUNLOCK(srcobject); + vm_object_pip_add(dstobject, 1); + dstaddr = swp_pager_meta_build(dstobject, i, srcaddr); + KASSERT(dstaddr == SWAPBLK_NONE, + ("Unexpected destination swapblk")); + vm_object_pip_wakeup(dstobject); + VM_OBJECT_WLOCK(srcobject); + vm_object_pip_wakeup(srcobject); } - swp_pager_freeswapspace(first_free, num_free); + swp_pager_freeswapspace(s_free, n_free); /* * Free left over swap blocks in source. @@ -1307,7 +1329,9 @@ swap_pager_putpages(vm_object_t object, vm_page_t *ma, { int i, n; boolean_t sync; + daddr_t addr, n_free, s_free; + swp_pager_init_freerange(&s_free, &n_free); if (count && ma[0]->object != object) { panic("swap_pager_putpages: object mismatch %p/%p", object, @@ -1322,8 +1346,11 @@ swap_pager_putpages(vm_object_t object, vm_page_t *ma, * check for bogus sysops * force sync if not pageout process */ - if (object->type != OBJT_SWAP) - swp_pager_meta_build(object, 0, SWAPBLK_NONE); + if (object->type != OBJT_SWAP) { + addr = swp_pager_meta_build(object, 0, SWAPBLK_NONE); + KASSERT(addr == SWAPBLK_NONE, + ("unexpected object swap block")); + } VM_OBJECT_WUNLOCK(object); n = 0; @@ -1391,11 +1418,11 @@ swap_pager_putpages(vm_object_t object, vm_page_t *ma, for (j = 0; j < n; ++j) { vm_page_t mreq = ma[i+j]; - swp_pager_meta_build( - mreq->object, - mreq->pindex, - blk + j - ); + addr = swp_pager_meta_build(mreq->object, mreq->pindex, + blk + j); + if (addr != SWAPBLK_NONE) + swp_pager_update_freerange(&s_free, &n_free, + addr); MPASS(mreq->dirty == VM_PAGE_BITS_ALL); mreq->oflags |= VPO_SWAPINPROG; bp->b_pages[j] = mreq; @@ -1453,6 +1480,7 @@ swap_pager_putpages(vm_object_t object, vm_page_t *ma, swp_pager_async_iodone(bp); } VM_OBJECT_WLOCK(object); + swp_pager_freeswapspace(s_free, n_free); } /* @@ -1783,14 +1811,15 @@ swp_pager_swblk_empty(struct swblk *sb, int start, int * * The specified swapblk is added to the object's swap metadata. If * the swapblk is not valid, it is freed instead. Any previously - * assigned swapblk is freed. + * assigned swapblk is returned. */ -static void +static daddr_t swp_pager_meta_build(vm_object_t object, vm_pindex_t pindex, daddr_t swapblk) { static volatile int swblk_zone_exhausted, swpctrie_zone_exhausted; struct swblk *sb, *sb1; vm_pindex_t modpi, rdpi; + daddr_t prev_swapblk; int error, i; VM_OBJECT_ASSERT_WLOCKED(object); @@ -1815,7 +1844,7 @@ swp_pager_meta_build(vm_object_t object, vm_pindex_t p sb = SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks, rdpi); if (sb == NULL) { if (swapblk == SWAPBLK_NONE) - return; + return (SWAPBLK_NONE); for (;;) { sb = uma_zalloc(swblk_zone, M_NOWAIT | (curproc == pageproc ? M_USE_RESERVE : 0)); @@ -1882,9 +1911,8 @@ allocated: MPASS(sb->p == rdpi); modpi = pindex % SWAP_META_PAGES; - /* Delete prior contents of metadata. */ - if (sb->d[modpi] != SWAPBLK_NONE) - swp_pager_freeswapspace(sb->d[modpi], 1); + /* Return prior contents of metadata. */ + prev_swapblk = sb->d[modpi]; /* Enter block into metadata. */ sb->d[modpi] = swapblk; @@ -1896,6 +1924,7 @@ allocated: SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, rdpi); uma_zfree(swblk_zone, sb); } + return (prev_swapblk); } /* @@ -1912,7 +1941,7 @@ static void swp_pager_meta_free(vm_object_t object, vm_pindex_t pindex, vm_pindex_t count) { struct swblk *sb; - daddr_t first_free, num_free; + daddr_t n_free, s_free; vm_pindex_t last; int i, limit, start; @@ -1920,8 +1949,7 @@ swp_pager_meta_free(vm_object_t object, vm_pindex_t pi if (object->type != OBJT_SWAP || count == 0) return; - first_free = SWAPBLK_NONE; - num_free = 0; + swp_pager_init_freerange(&s_free, &n_free); last = pindex + count; for (;;) { sb = SWAP_PCTRIE_LOOKUP_GE(&object->un_pager.swp.swp_blks, @@ -1934,13 +1962,7 @@ swp_pager_meta_free(vm_object_t object, vm_pindex_t pi for (i = start; i < limit; i++) { if (sb->d[i] == SWAPBLK_NONE) continue; - if (first_free + num_free == sb->d[i]) - num_free++; - else { - swp_pager_freeswapspace(first_free, num_free); - first_free = sb->d[i]; - num_free = 1; - } + swp_pager_update_freerange(&s_free, &n_free, sb->d[i]); sb->d[i] = SWAPBLK_NONE; } if (swp_pager_swblk_empty(sb, 0, start) && @@ -1951,7 +1973,7 @@ swp_pager_meta_free(vm_object_t object, vm_pindex_t pi } pindex = sb->p + SWAP_META_PAGES; } - swp_pager_freeswapspace(first_free, num_free); + swp_pager_freeswapspace(s_free, n_free); } /* @@ -1964,7 +1986,7 @@ static void swp_pager_meta_free_all(vm_object_t object) { struct swblk *sb; - daddr_t first_free, num_free; + daddr_t n_free, s_free; vm_pindex_t pindex; int i; @@ -1972,26 +1994,19 @@ swp_pager_meta_free_all(vm_object_t object) if (object->type != OBJT_SWAP) return; - first_free = SWAPBLK_NONE; - num_free = 0; + swp_pager_init_freerange(&s_free, &n_free); for (pindex = 0; (sb = SWAP_PCTRIE_LOOKUP_GE( &object->un_pager.swp.swp_blks, pindex)) != NULL;) { pindex = sb->p + SWAP_META_PAGES; for (i = 0; i < SWAP_META_PAGES; i++) { if (sb->d[i] == SWAPBLK_NONE) continue; - if (first_free + num_free == sb->d[i]) - num_free++; - else { - swp_pager_freeswapspace(first_free, num_free); - first_free = sb->d[i]; - num_free = 1; - } + swp_pager_update_freerange(&s_free, &n_free, sb->d[i]); } SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, sb->p); uma_zfree(swblk_zone, sb); } - swp_pager_freeswapspace(first_free, num_free); + swp_pager_freeswapspace(s_free, n_free); } /* _______________________________________________ svn-src-all@freebsd.org mailing list https://lists.freebsd.org/mailman/listinfo/svn-src-all To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"
