Author: alc
Date: Wed Aug  9 04:23:04 2017
New Revision: 322296
URL: https://svnweb.freebsd.org/changeset/base/322296

Log:
  Introduce vm_page_grab_pages(), which is intended to replace loops calling
  vm_page_grab() on consecutive page indices.  Besides simplifying the code
  in the caller, vm_page_grab_pages() allows for batching optimizations.
  For example, the current implementation replaces calls to vm_page_lookup()
  on consecutive page indices by cheaper calls to vm_page_next().
  
  Reviewed by:  kib, markj
  Tested by:    pho (an earlier version)
  MFC after:    2 weeks
  Differential Revision:        https://reviews.freebsd.org/D11926

Modified:
  head/sys/kern/vfs_bio.c
  head/sys/sparc64/sparc64/pmap.c
  head/sys/vm/vm_glue.c
  head/sys/vm/vm_page.c
  head/sys/vm/vm_page.h

Modified: head/sys/kern/vfs_bio.c
==============================================================================
--- head/sys/kern/vfs_bio.c     Wed Aug  9 01:31:05 2017        (r322295)
+++ head/sys/kern/vfs_bio.c     Wed Aug  9 04:23:04 2017        (r322296)
@@ -2735,7 +2735,7 @@ vfs_vmio_extend(struct buf *bp, int desiredpages, int 
         */
        obj = bp->b_bufobj->bo_object;
        VM_OBJECT_WLOCK(obj);
-       while (bp->b_npages < desiredpages) {
+       if (bp->b_npages < desiredpages) {
                /*
                 * We must allocate system pages since blocking
                 * here could interfere with paging I/O, no
@@ -2746,14 +2746,12 @@ vfs_vmio_extend(struct buf *bp, int desiredpages, int 
                 * deadlocks once allocbuf() is called after
                 * pages are vfs_busy_pages().
                 */
-               m = vm_page_grab(obj, OFF_TO_IDX(bp->b_offset) + bp->b_npages,
-                   VM_ALLOC_NOBUSY | VM_ALLOC_SYSTEM |
-                   VM_ALLOC_WIRED | VM_ALLOC_IGN_SBUSY |
-                   VM_ALLOC_COUNT(desiredpages - bp->b_npages));
-               if (m->valid == 0)
-                       bp->b_flags &= ~B_CACHE;
-               bp->b_pages[bp->b_npages] = m;
-               ++bp->b_npages;
+               vm_page_grab_pages(obj,
+                   OFF_TO_IDX(bp->b_offset) + bp->b_npages,
+                   VM_ALLOC_SYSTEM | VM_ALLOC_IGN_SBUSY |
+                   VM_ALLOC_NOBUSY | VM_ALLOC_WIRED,
+                   &bp->b_pages[bp->b_npages], desiredpages - bp->b_npages);
+               bp->b_npages = desiredpages;
        }
 
        /*

Modified: head/sys/sparc64/sparc64/pmap.c
==============================================================================
--- head/sys/sparc64/sparc64/pmap.c     Wed Aug  9 01:31:05 2017        
(r322295)
+++ head/sys/sparc64/sparc64/pmap.c     Wed Aug  9 04:23:04 2017        
(r322296)
@@ -1226,7 +1226,6 @@ int
 pmap_pinit(pmap_t pm)
 {
        vm_page_t ma[TSB_PAGES];
-       vm_page_t m;
        int i;
 
        /*
@@ -1249,14 +1248,11 @@ pmap_pinit(pmap_t pm)
        CPU_ZERO(&pm->pm_active);
 
        VM_OBJECT_WLOCK(pm->pm_tsb_obj);
-       for (i = 0; i < TSB_PAGES; i++) {
-               m = vm_page_grab(pm->pm_tsb_obj, i, VM_ALLOC_NOBUSY |
-                   VM_ALLOC_WIRED | VM_ALLOC_ZERO);
-               m->valid = VM_PAGE_BITS_ALL;
-               m->md.pmap = pm;
-               ma[i] = m;
-       }
+       vm_page_grab_pages(pm->pm_tsb_obj, 0, VM_ALLOC_NORMAL |
+           VM_ALLOC_NOBUSY | VM_ALLOC_WIRED | VM_ALLOC_ZERO, ma, TSB_PAGES);
        VM_OBJECT_WUNLOCK(pm->pm_tsb_obj);
+       for (i = 0; i < TSB_PAGES; i++)
+               ma[i]->md.pmap = pm;
        pmap_qenter((vm_offset_t)pm->pm_tsb, ma, TSB_PAGES);
 
        bzero(&pm->pm_stats, sizeof(pm->pm_stats));

Modified: head/sys/vm/vm_glue.c
==============================================================================
--- head/sys/vm/vm_glue.c       Wed Aug  9 01:31:05 2017        (r322295)
+++ head/sys/vm/vm_glue.c       Wed Aug  9 04:23:04 2017        (r322296)
@@ -322,7 +322,7 @@ vm_thread_new(struct thread *td, int pages)
 {
        vm_object_t ksobj;
        vm_offset_t ks;
-       vm_page_t m, ma[KSTACK_MAX_PAGES];
+       vm_page_t ma[KSTACK_MAX_PAGES];
        struct kstack_cache_entry *ks_ce;
        int i;
 
@@ -391,15 +391,10 @@ vm_thread_new(struct thread *td, int pages)
         * page of stack.
         */
        VM_OBJECT_WLOCK(ksobj);
-       for (i = 0; i < pages; i++) {
-               /*
-                * Get a kernel stack page.
-                */
-               m = vm_page_grab(ksobj, i, VM_ALLOC_NOBUSY |
-                   VM_ALLOC_NORMAL | VM_ALLOC_WIRED);
-               ma[i] = m;
-               m->valid = VM_PAGE_BITS_ALL;
-       }
+       vm_page_grab_pages(ksobj, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY |
+           VM_ALLOC_WIRED, ma, pages);
+       for (i = 0; i < pages; i++)
+               ma[i]->valid = VM_PAGE_BITS_ALL;
        VM_OBJECT_WUNLOCK(ksobj);
        pmap_qenter(ks, ma, pages);
        return (1);
@@ -573,9 +568,8 @@ vm_thread_swapin(struct thread *td)
        pages = td->td_kstack_pages;
        ksobj = td->td_kstack_obj;
        VM_OBJECT_WLOCK(ksobj);
-       for (int i = 0; i < pages; i++)
-               ma[i] = vm_page_grab(ksobj, i, VM_ALLOC_NORMAL |
-                   VM_ALLOC_WIRED);
+       vm_page_grab_pages(ksobj, 0, VM_ALLOC_NORMAL | VM_ALLOC_WIRED, ma,
+           pages);
        for (int i = 0; i < pages;) {
                int j, a, count, rv;
 

Modified: head/sys/vm/vm_page.c
==============================================================================
--- head/sys/vm/vm_page.c       Wed Aug  9 01:31:05 2017        (r322295)
+++ head/sys/vm/vm_page.c       Wed Aug  9 04:23:04 2017        (r322296)
@@ -3155,6 +3155,100 @@ retrylookup:
 }
 
 /*
+ * Return the specified range of pages from the given object.  For each
+ * page offset within the range, if a page already exists within the object
+ * at that offset and it is busy, then wait for it to change state.  If,
+ * instead, the page doesn't exist, then allocate it.
+ *
+ * The caller must always specify an allocation class.
+ *
+ * allocation classes:
+ *     VM_ALLOC_NORMAL         normal process request
+ *     VM_ALLOC_SYSTEM         system *really* needs the pages
+ *
+ * The caller must always specify that the pages are to be busied and/or
+ * wired.
+ *
+ * optional allocation flags:
+ *     VM_ALLOC_IGN_SBUSY      do not sleep on soft busy pages
+ *     VM_ALLOC_NOBUSY         do not exclusive busy the page
+ *     VM_ALLOC_SBUSY          set page to sbusy state
+ *     VM_ALLOC_WIRED          wire the pages
+ *     VM_ALLOC_ZERO           zero and validate any invalid pages
+ *
+ * This routine may sleep.
+ */
+void
+vm_page_grab_pages(vm_object_t object, vm_pindex_t pindex, int allocflags,
+    vm_page_t *ma, int count)
+{
+       vm_page_t m;
+       int i;
+       bool sleep;
+
+       VM_OBJECT_ASSERT_WLOCKED(object);
+       KASSERT(((u_int)allocflags >> VM_ALLOC_COUNT_SHIFT) == 0,
+           ("vm_page_grap_pages: VM_ALLOC_COUNT() is not allowed"));
+       KASSERT((allocflags & VM_ALLOC_NOBUSY) == 0 ||
+           (allocflags & VM_ALLOC_WIRED) != 0,
+           ("vm_page_grab_pages: the pages must be busied or wired"));
+       KASSERT((allocflags & VM_ALLOC_SBUSY) == 0 ||
+           (allocflags & VM_ALLOC_IGN_SBUSY) != 0,
+           ("vm_page_grab_pages: VM_ALLOC_SBUSY/IGN_SBUSY mismatch"));
+       if (count == 0)
+               return;
+       i = 0;
+retrylookup:
+       m = vm_page_lookup(object, pindex + i);
+       for (; i < count; i++) {
+               if (m != NULL) {
+                       sleep = (allocflags & VM_ALLOC_IGN_SBUSY) != 0 ?
+                           vm_page_xbusied(m) : vm_page_busied(m);
+                       if (sleep) {
+                               /*
+                                * Reference the page before unlocking and
+                                * sleeping so that the page daemon is less
+                                * likely to reclaim it.
+                                */
+                               vm_page_aflag_set(m, PGA_REFERENCED);
+                               vm_page_lock(m);
+                               VM_OBJECT_WUNLOCK(object);
+                               vm_page_busy_sleep(m, "grbmaw", (allocflags &
+                                   VM_ALLOC_IGN_SBUSY) != 0);
+                               VM_OBJECT_WLOCK(object);
+                               goto retrylookup;
+                       }
+                       if ((allocflags & VM_ALLOC_WIRED) != 0) {
+                               vm_page_lock(m);
+                               vm_page_wire(m);
+                               vm_page_unlock(m);
+                       }
+                       if ((allocflags & (VM_ALLOC_NOBUSY |
+                           VM_ALLOC_SBUSY)) == 0)
+                               vm_page_xbusy(m);
+                       if ((allocflags & VM_ALLOC_SBUSY) != 0)
+                               vm_page_sbusy(m);
+               } else {
+                       m = vm_page_alloc(object, pindex + i, (allocflags &
+                           ~VM_ALLOC_IGN_SBUSY) | VM_ALLOC_COUNT(count - i));
+                       if (m == NULL) {
+                               VM_OBJECT_WUNLOCK(object);
+                               VM_WAIT;
+                               VM_OBJECT_WLOCK(object);
+                               goto retrylookup;
+                       }
+               }
+               if (m->valid == 0 && (allocflags & VM_ALLOC_ZERO) != 0) {
+                       if ((m->flags & PG_ZERO) == 0)
+                               pmap_zero_page(m);
+                       m->valid = VM_PAGE_BITS_ALL;
+               }
+               ma[i] = m;
+               m = vm_page_next(m);
+       }
+}
+
+/*
  * Mapping function for valid or dirty bits in a page.
  *
  * Inputs are required to range within a page.

Modified: head/sys/vm/vm_page.h
==============================================================================
--- head/sys/vm/vm_page.h       Wed Aug  9 01:31:05 2017        (r322295)
+++ head/sys/vm/vm_page.h       Wed Aug  9 04:23:04 2017        (r322296)
@@ -394,6 +394,9 @@ vm_page_t PHYS_TO_VM_PAGE(vm_paddr_t pa);
  * vm_page_alloc_freelist().  Some functions support only a subset
  * of the flags, and ignore others, see the flags legend.
  *
+ * The meaning of VM_ALLOC_ZERO differs slightly between the vm_page_alloc*()
+ * and the vm_page_grab*() functions.  See these functions for details.
+ *
  * Bits 0 - 1 define class.
  * Bits 2 - 15 dedicated for flags.
  * Legend:
@@ -401,6 +404,7 @@ vm_page_t PHYS_TO_VM_PAGE(vm_paddr_t pa);
  * (c) - vm_page_alloc_contig() supports the flag.
  * (f) - vm_page_alloc_freelist() supports the flag.
  * (g) - vm_page_grab() supports the flag.
+ * (p) - vm_page_grab_pages() supports the flag.
  * Bits above 15 define the count of additional pages that the caller
  * intends to allocate.
  */
@@ -408,13 +412,13 @@ vm_page_t PHYS_TO_VM_PAGE(vm_paddr_t pa);
 #define VM_ALLOC_INTERRUPT     1
 #define VM_ALLOC_SYSTEM                2
 #define        VM_ALLOC_CLASS_MASK     3
-#define        VM_ALLOC_WIRED          0x0020  /* (acfg) Allocate non pageable 
page */
-#define        VM_ALLOC_ZERO           0x0040  /* (acfg) Try to obtain a 
zeroed page */
+#define        VM_ALLOC_WIRED          0x0020  /* (acfgp) Allocate a wired 
page */
+#define        VM_ALLOC_ZERO           0x0040  /* (acfgp) Allocate a prezeroed 
page */
 #define        VM_ALLOC_NOOBJ          0x0100  /* (acg) No associated object */
-#define        VM_ALLOC_NOBUSY         0x0200  /* (acg) Do not busy the page */
-#define        VM_ALLOC_IGN_SBUSY      0x1000  /* (g) Ignore shared busy flag 
*/
+#define        VM_ALLOC_NOBUSY         0x0200  /* (acgp) Do not excl busy the 
page */
+#define        VM_ALLOC_IGN_SBUSY      0x1000  /* (gp) Ignore shared busy flag 
*/
 #define        VM_ALLOC_NODUMP         0x2000  /* (ag) don't include in dump */
-#define        VM_ALLOC_SBUSY          0x4000  /* (acg) Shared busy the page */
+#define        VM_ALLOC_SBUSY          0x4000  /* (acgp) Shared busy the page 
*/
 #define        VM_ALLOC_NOWAIT         0x8000  /* (g) Do not sleep, return 
NULL */
 #define        VM_ALLOC_COUNT_SHIFT    16
 #define        VM_ALLOC_COUNT(count)   ((count) << VM_ALLOC_COUNT_SHIFT)
@@ -466,6 +470,8 @@ vm_page_t vm_page_alloc_contig(vm_object_t object, vm_
     vm_paddr_t boundary, vm_memattr_t memattr);
 vm_page_t vm_page_alloc_freelist(int, int);
 vm_page_t vm_page_grab (vm_object_t, vm_pindex_t, int);
+void vm_page_grab_pages(vm_object_t object, vm_pindex_t pindex, int allocflags,
+    vm_page_t *ma, int count);
 int vm_page_try_to_free (vm_page_t);
 void vm_page_deactivate (vm_page_t);
 void vm_page_deactivate_noreuse(vm_page_t);
_______________________________________________
svn-src-all@freebsd.org mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-all
To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"

Reply via email to