Author: markj Date: Mon Jan 30 18:51:43 2017 New Revision: 312994 URL: https://svnweb.freebsd.org/changeset/base/312994
Log: Avoid page lookups in the top-level object in vm_object_madvise(). We can iterate over consecutive resident pages in the top-level object using the object's page list rather than by performing lookups in the object radix tree. This extends one of the optimizations in r312208 to the case where a shadow chain is present. Suggested by: alc Reviewed by: alc, kib (previous version) MFC after: 2 weeks Differential Revision: https://reviews.freebsd.org/D9282 Modified: head/sys/vm/vm_object.c Modified: head/sys/vm/vm_object.c ============================================================================== --- head/sys/vm/vm_object.c Mon Jan 30 18:35:24 2017 (r312993) +++ head/sys/vm/vm_object.c Mon Jan 30 18:51:43 2017 (r312994) @@ -1075,6 +1075,33 @@ vm_object_sync(vm_object_t object, vm_oo } /* + * Determine whether the given advice can be applied to the object. Advice is + * not applied to unmanaged pages since they never belong to page queues, and + * since MADV_FREE is destructive, it can apply only to anonymous pages that + * have been mapped at most once. + */ +static bool +vm_object_advice_applies(vm_object_t object, int advice) +{ + + if ((object->flags & OBJ_UNMANAGED) != 0) + return (false); + if (advice != MADV_FREE) + return (true); + return ((object->type == OBJT_DEFAULT || object->type == OBJT_SWAP) && + (object->flags & OBJ_ONEMAPPING) != 0); +} + +static void +vm_object_madvise_freespace(vm_object_t object, int advice, vm_pindex_t pindex, + vm_size_t size) +{ + + if (advice == MADV_FREE && object->type == OBJT_SWAP) + swap_pager_freespace(object, pindex, size); +} + +/* * vm_object_madvise: * * Implements the madvise function at the object/page level. @@ -1101,100 +1128,102 @@ vm_object_madvise(vm_object_t object, vm { vm_pindex_t tpindex; vm_object_t backing_object, tobject; - vm_page_t m; + vm_page_t m, tm; if (object == NULL) return; - VM_OBJECT_WLOCK(object); - for (m = NULL; pindex < end; pindex++) { relookup: + VM_OBJECT_WLOCK(object); + if (!vm_object_advice_applies(object, advice)) { + VM_OBJECT_WUNLOCK(object); + return; + } + for (m = vm_page_find_least(object, pindex); pindex < end; pindex++) { tobject = object; - tpindex = pindex; -shadowlookup: - /* - * MADV_FREE only operates on OBJT_DEFAULT or OBJT_SWAP pages - * and those pages must be OBJ_ONEMAPPING. - */ - if (advice == MADV_FREE) { - if ((tobject->type != OBJT_DEFAULT && - tobject->type != OBJT_SWAP) || - (tobject->flags & OBJ_ONEMAPPING) == 0) { - goto unlock_tobject; - } - } else if ((tobject->flags & OBJ_UNMANAGED) != 0) - goto unlock_tobject; /* - * In the common case where the object has no backing object, we - * can avoid performing lookups at each pindex. In either case, - * when applying MADV_FREE we take care to release any swap - * space used to store non-resident pages. + * If the next page isn't resident in the top-level object, we + * need to search the shadow chain. When applying MADV_FREE, we + * take care to release any swap space used to store + * non-resident pages. */ - if (object->backing_object == NULL) { - m = (m != NULL) ? TAILQ_NEXT(m, listq) : - vm_page_find_least(object, pindex); - tpindex = (m != NULL && m->pindex < end) ? - m->pindex : end; - if (advice == MADV_FREE && object->type == OBJT_SWAP && - tpindex > pindex) - swap_pager_freespace(object, pindex, - tpindex - pindex); - if ((pindex = tpindex) == end) - break; - } else if ((m = vm_page_lookup(tobject, tpindex)) == NULL) { - if (advice == MADV_FREE && tobject->type == OBJT_SWAP) - swap_pager_freespace(tobject, tpindex, 1); + if (m == NULL || pindex < m->pindex) { /* - * Prepare to search the next object in the chain. + * Optimize a common case: if the top-level object has + * no backing object, we can skip over the non-resident + * range in constant time. */ - backing_object = tobject->backing_object; - if (backing_object == NULL) - goto unlock_tobject; - VM_OBJECT_WLOCK(backing_object); - tpindex += OFF_TO_IDX(tobject->backing_object_offset); - if (tobject != object) - VM_OBJECT_WUNLOCK(tobject); - tobject = backing_object; - goto shadowlookup; + if (object->backing_object == NULL) { + tpindex = (m != NULL && m->pindex < end) ? + m->pindex : end; + vm_object_madvise_freespace(object, advice, + pindex, tpindex - pindex); + if ((pindex = tpindex) == end) + break; + goto next_page; + } + + tpindex = pindex; + do { + vm_object_madvise_freespace(tobject, advice, + tpindex, 1); + /* + * Prepare to search the next object in the + * chain. + */ + backing_object = tobject->backing_object; + if (backing_object == NULL) + goto next_pindex; + VM_OBJECT_WLOCK(backing_object); + tpindex += + OFF_TO_IDX(tobject->backing_object_offset); + if (tobject != object) + VM_OBJECT_WUNLOCK(tobject); + tobject = backing_object; + if (!vm_object_advice_applies(tobject, advice)) + goto next_pindex; + } while ((tm = vm_page_lookup(tobject, tpindex)) == + NULL); + } else { +next_page: + tm = m; + m = TAILQ_NEXT(m, listq); } /* * If the page is not in a normal state, skip it. */ - if (m->valid != VM_PAGE_BITS_ALL) - goto unlock_tobject; - vm_page_lock(m); - if (m->hold_count != 0 || m->wire_count != 0) { - vm_page_unlock(m); - goto unlock_tobject; - } - KASSERT((m->flags & PG_FICTITIOUS) == 0, - ("vm_object_madvise: page %p is fictitious", m)); - KASSERT((m->oflags & VPO_UNMANAGED) == 0, - ("vm_object_madvise: page %p is not managed", m)); - if (vm_page_busied(m)) { + if (tm->valid != VM_PAGE_BITS_ALL) + goto next_pindex; + vm_page_lock(tm); + if (tm->hold_count != 0 || tm->wire_count != 0) { + vm_page_unlock(tm); + goto next_pindex; + } + KASSERT((tm->flags & PG_FICTITIOUS) == 0, + ("vm_object_madvise: page %p is fictitious", tm)); + KASSERT((tm->oflags & VPO_UNMANAGED) == 0, + ("vm_object_madvise: page %p is not managed", tm)); + if (vm_page_busied(tm)) { + if (object != tobject) + VM_OBJECT_WUNLOCK(tobject); + VM_OBJECT_WUNLOCK(object); if (advice == MADV_WILLNEED) { /* * Reference the page before unlocking and * sleeping so that the page daemon is less - * likely to reclaim it. + * likely to reclaim it. */ - vm_page_aflag_set(m, PGA_REFERENCED); + vm_page_aflag_set(tm, PGA_REFERENCED); } - if (object != tobject) - VM_OBJECT_WUNLOCK(object); - VM_OBJECT_WUNLOCK(tobject); - vm_page_busy_sleep(m, "madvpo", false); - m = NULL; - VM_OBJECT_WLOCK(object); + vm_page_busy_sleep(tm, "madvpo", false); goto relookup; } - vm_page_advise(m, advice); - vm_page_unlock(m); - if (advice == MADV_FREE && tobject->type == OBJT_SWAP) - swap_pager_freespace(tobject, tpindex, 1); -unlock_tobject: + vm_page_advise(tm, advice); + vm_page_unlock(tm); + vm_object_madvise_freespace(tobject, advice, tm->pindex, 1); +next_pindex: if (tobject != object) VM_OBJECT_WUNLOCK(tobject); } _______________________________________________ svn-src-all@freebsd.org mailing list https://lists.freebsd.org/mailman/listinfo/svn-src-all To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"
