Author: kib
Date: Fri Jan 13 13:45:34 2017
New Revision: 312073
URL: https://svnweb.freebsd.org/changeset/base/312073

Log:
  MFC r309710:
  Add a new populate() pager method and extend device pager ops vector
  with cdev_pg_populate() to provide device drivers access to it.
  
  MFC r310849:
  Fix two similar bugs in the populate vm_fault() code.

Modified:
  stable/11/sys/vm/device_pager.c
  stable/11/sys/vm/vm_fault.c
  stable/11/sys/vm/vm_object.h
  stable/11/sys/vm/vm_pager.h
Directory Properties:
  stable/11/   (props changed)

Modified: stable/11/sys/vm/device_pager.c
==============================================================================
--- stable/11/sys/vm/device_pager.c     Fri Jan 13 13:37:09 2017        
(r312072)
+++ stable/11/sys/vm/device_pager.c     Fri Jan 13 13:45:34 2017        
(r312073)
@@ -63,6 +63,8 @@ static int dev_pager_getpages(vm_object_
 static void dev_pager_putpages(vm_object_t, vm_page_t *, int, int, int *);
 static boolean_t dev_pager_haspage(vm_object_t, vm_pindex_t, int *, int *);
 static void dev_pager_free_page(vm_object_t object, vm_page_t m);
+static int dev_pager_populate(vm_object_t object, vm_pindex_t pidx,
+    int fault_type, vm_prot_t, vm_pindex_t *first, vm_pindex_t *last);
 
 /* list of device pager objects */
 static struct pagerlst dev_pager_object_list;
@@ -84,6 +86,7 @@ struct pagerops mgtdevicepagerops = {
        .pgo_getpages = dev_pager_getpages,
        .pgo_putpages = dev_pager_putpages,
        .pgo_haspage =  dev_pager_haspage,
+       .pgo_populate = dev_pager_populate,
 };
 
 static int old_dev_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot,
@@ -127,6 +130,8 @@ cdev_pager_allocate(void *handle, enum o
 
        if (tp != OBJT_DEVICE && tp != OBJT_MGTDEVICE)
                return (NULL);
+       KASSERT(tp == OBJT_MGTDEVICE || ops->cdev_pg_populate == NULL,
+           ("populate on unmanaged device pager"));
 
        /*
         * Offset should be page aligned.
@@ -179,6 +184,8 @@ cdev_pager_allocate(void *handle, enum o
                        object->handle = handle;
                        TAILQ_INSERT_TAIL(&dev_pager_object_list, object,
                            pager_object_list);
+                       if (ops->cdev_pg_populate != NULL)
+                               vm_object_set_flag(object, OBJ_POPULATE);
                }
        } else {
                if (pindex > object->size)
@@ -268,6 +275,8 @@ dev_pager_getpages(vm_object_t object, v
        /* Since our haspage reports zero after/before, the count is 1. */
        KASSERT(count == 1, ("%s: count %d", __func__, count));
        VM_OBJECT_ASSERT_WLOCKED(object);
+       if (object->un_pager.devp.ops->cdev_pg_fault == NULL)
+               return (VM_PAGER_FAIL);
        error = object->un_pager.devp.ops->cdev_pg_fault(object,
            IDX_TO_OFF(ma[0]->pindex), PROT_READ, &ma[0]);
 
@@ -293,6 +302,18 @@ dev_pager_getpages(vm_object_t object, v
 }
 
 static int
+dev_pager_populate(vm_object_t object, vm_pindex_t pidx, int fault_type,
+    vm_prot_t max_prot, vm_pindex_t *first, vm_pindex_t *last)
+{
+
+       VM_OBJECT_ASSERT_WLOCKED(object);
+       if (object->un_pager.devp.ops->cdev_pg_populate == NULL)
+               return (VM_PAGER_FAIL);
+       return (object->un_pager.devp.ops->cdev_pg_populate(object, pidx,
+           fault_type, max_prot, first, last));
+}
+
+static int
 old_dev_pager_fault(vm_object_t object, vm_ooffset_t offset, int prot,
     vm_page_t *mres)
 {

Modified: stable/11/sys/vm/vm_fault.c
==============================================================================
--- stable/11/sys/vm/vm_fault.c Fri Jan 13 13:37:09 2017        (r312072)
+++ stable/11/sys/vm/vm_fault.c Fri Jan 13 13:45:34 2017        (r312073)
@@ -289,6 +289,157 @@ vm_fault_soft_fast(struct faultstate *fs
        return (KERN_SUCCESS);
 }
 
+static void
+vm_fault_restore_map_lock(struct faultstate *fs)
+{
+
+       VM_OBJECT_ASSERT_WLOCKED(fs->first_object);
+       MPASS(fs->first_object->paging_in_progress > 0);
+
+       if (!vm_map_trylock_read(fs->map)) {
+               VM_OBJECT_WUNLOCK(fs->first_object);
+               vm_map_lock_read(fs->map);
+               VM_OBJECT_WLOCK(fs->first_object);
+       }
+       fs->lookup_still_valid = true;
+}
+
+static void
+vm_fault_populate_check_page(vm_page_t m)
+{
+
+       /*
+        * Check each page to ensure that the pager is obeying the
+        * interface: the page must be installed in the object, fully
+        * valid, and exclusively busied.
+        */
+       MPASS(m != NULL);
+       MPASS(m->valid == VM_PAGE_BITS_ALL);
+       MPASS(vm_page_xbusied(m));
+}
+
+static void
+vm_fault_populate_cleanup(vm_object_t object, vm_pindex_t first,
+    vm_pindex_t last)
+{
+       vm_page_t m;
+       vm_pindex_t pidx;
+
+       VM_OBJECT_ASSERT_WLOCKED(object);
+       MPASS(first <= last);
+       for (pidx = first, m = vm_page_lookup(object, pidx);
+           pidx <= last; pidx++, m = vm_page_next(m)) {
+               vm_fault_populate_check_page(m);
+               vm_page_lock(m);
+               vm_page_deactivate(m);
+               vm_page_unlock(m);
+               vm_page_xunbusy(m);
+       }
+}
+
+static int
+vm_fault_populate(struct faultstate *fs, vm_offset_t vaddr, vm_prot_t prot,
+    int fault_type, int fault_flags, boolean_t wired, vm_page_t *m_hold)
+{
+       vm_page_t m;
+       vm_pindex_t map_first, map_last, pager_first, pager_last, pidx;
+       int rv;
+
+       MPASS(fs->object == fs->first_object);
+       VM_OBJECT_ASSERT_WLOCKED(fs->first_object);
+       MPASS(fs->first_object->paging_in_progress > 0);
+       MPASS(fs->first_object->backing_object == NULL);
+       MPASS(fs->lookup_still_valid);
+
+       pager_first = OFF_TO_IDX(fs->entry->offset);
+       pager_last = OFF_TO_IDX(fs->entry->offset + fs->entry->end -
+           fs->entry->start) - 1;
+       unlock_map(fs);
+       unlock_vp(fs);
+
+       /*
+        * Call the pager (driver) populate() method.
+        *
+        * There is no guarantee that the method will be called again
+        * if the current fault is for read, and a future fault is
+        * for write.  Report the entry's maximum allowed protection
+        * to the driver.
+        */
+       rv = vm_pager_populate(fs->first_object, fs->first_pindex,
+           fault_type, fs->entry->max_protection, &pager_first, &pager_last);
+
+       VM_OBJECT_ASSERT_WLOCKED(fs->first_object);
+       if (rv == VM_PAGER_BAD) {
+               /*
+                * VM_PAGER_BAD is the backdoor for a pager to request
+                * normal fault handling.
+                */
+               vm_fault_restore_map_lock(fs);
+               if (fs->map->timestamp != fs->map_generation)
+                       return (KERN_RESOURCE_SHORTAGE); /* RetryFault */
+               return (KERN_NOT_RECEIVER);
+       }
+       if (rv != VM_PAGER_OK)
+               return (KERN_FAILURE); /* AKA SIGSEGV */
+
+       /* Ensure that the driver is obeying the interface. */
+       MPASS(pager_first <= pager_last);
+       MPASS(fs->first_pindex <= pager_last);
+       MPASS(fs->first_pindex >= pager_first);
+       MPASS(pager_last < fs->first_object->size);
+
+       vm_fault_restore_map_lock(fs);
+       if (fs->map->timestamp != fs->map_generation) {
+               vm_fault_populate_cleanup(fs->first_object, pager_first,
+                   pager_last);
+               return (KERN_RESOURCE_SHORTAGE); /* RetryFault */
+       }
+
+       /*
+        * The map is unchanged after our last unlock.  Process the fault.
+        *
+        * The range [pager_first, pager_last] that is given to the
+        * pager is only a hint.  The pager may populate any range
+        * within the object that includes the requested page index.
+        * In case the pager expanded the range, clip it to fit into
+        * the map entry.
+        */
+       map_first = MAX(OFF_TO_IDX(fs->entry->offset), pager_first);
+       if (map_first > pager_first)
+               vm_fault_populate_cleanup(fs->first_object, pager_first,
+                   map_first - 1);
+       map_last = MIN(OFF_TO_IDX(fs->entry->end - fs->entry->start +
+           fs->entry->offset), pager_last);
+       if (map_last < pager_last)
+               vm_fault_populate_cleanup(fs->first_object, map_last + 1,
+                   pager_last);
+
+       for (pidx = map_first, m = vm_page_lookup(fs->first_object, pidx);
+           pidx <= map_last; pidx++, m = vm_page_next(m)) {
+               vm_fault_populate_check_page(m);
+               vm_fault_dirty(fs->entry, m, prot, fault_type, fault_flags,
+                   true);
+               VM_OBJECT_WUNLOCK(fs->first_object);
+               pmap_enter(fs->map->pmap, fs->entry->start + IDX_TO_OFF(pidx) -
+                   fs->entry->offset, m, prot, fault_type | (wired ?
+                   PMAP_ENTER_WIRED : 0), 0);
+               VM_OBJECT_WLOCK(fs->first_object);
+               if (pidx == fs->first_pindex)
+                       vm_fault_fill_hold(m_hold, m);
+               vm_page_lock(m);
+               if ((fault_flags & VM_FAULT_WIRE) != 0) {
+                       KASSERT(wired, ("VM_FAULT_WIRE && !wired"));
+                       vm_page_wire(m);
+               } else {
+                       vm_page_activate(m);
+               }
+               vm_page_unlock(m);
+               vm_page_xunbusy(m);
+       }
+       curthread->td_ru.ru_majflt++;
+       return (KERN_SUCCESS);
+}
+
 /*
  *     vm_fault:
  *
@@ -554,6 +705,30 @@ RetryFault:;
                                return (KERN_PROTECTION_FAILURE);
                        }
 
+                       if (fs.object == fs.first_object &&
+                           (fs.first_object->flags & OBJ_POPULATE) != 0 &&
+                           fs.first_object->shadow_count == 0) {
+                               rv = vm_fault_populate(&fs, vaddr, prot,
+                                   fault_type, fault_flags, wired, m_hold);
+                               switch (rv) {
+                               case KERN_SUCCESS:
+                               case KERN_FAILURE:
+                                       unlock_and_deallocate(&fs);
+                                       return (rv);
+                               case KERN_RESOURCE_SHORTAGE:
+                                       unlock_and_deallocate(&fs);
+                                       goto RetryFault;
+                               case KERN_NOT_RECEIVER:
+                                       /*
+                                        * Pager's populate() method
+                                        * returned VM_PAGER_BAD.
+                                        */
+                                       break;
+                               default:
+                                       panic("inconsistent return codes");
+                               }
+                       }
+
                        /*
                         * Allocate a new page for this object/offset pair.
                         *

Modified: stable/11/sys/vm/vm_object.h
==============================================================================
--- stable/11/sys/vm/vm_object.h        Fri Jan 13 13:37:09 2017        
(r312072)
+++ stable/11/sys/vm/vm_object.h        Fri Jan 13 13:45:34 2017        
(r312073)
@@ -182,6 +182,7 @@ struct vm_object {
  */
 #define        OBJ_FICTITIOUS  0x0001          /* (c) contains fictitious 
pages */
 #define        OBJ_UNMANAGED   0x0002          /* (c) contains unmanaged pages 
*/
+#define        OBJ_POPULATE    0x0004          /* pager implements populate() 
*/
 #define OBJ_DEAD       0x0008          /* dead objects (during rundown) */
 #define        OBJ_NOSPLIT     0x0010          /* dont split this object */
 #define        OBJ_UMTXDEAD    0x0020          /* umtx pshared was terminated 
*/

Modified: stable/11/sys/vm/vm_pager.h
==============================================================================
--- stable/11/sys/vm/vm_pager.h Fri Jan 13 13:37:09 2017        (r312072)
+++ stable/11/sys/vm/vm_pager.h Fri Jan 13 13:45:34 2017        (r312073)
@@ -56,6 +56,8 @@ typedef int pgo_getpages_async_t(vm_obje
     pgo_getpages_iodone_t, void *);
 typedef void pgo_putpages_t(vm_object_t, vm_page_t *, int, int, int *);
 typedef boolean_t pgo_haspage_t(vm_object_t, vm_pindex_t, int *, int *);
+typedef int pgo_populate_t(vm_object_t, vm_pindex_t, int, vm_prot_t,
+    vm_pindex_t *, vm_pindex_t *);
 typedef void pgo_pageunswapped_t(vm_page_t);
 
 struct pagerops {
@@ -66,6 +68,7 @@ struct pagerops {
        pgo_getpages_async_t    *pgo_getpages_async;    /* Get page asyncly. */
        pgo_putpages_t          *pgo_putpages;          /* Put (write) page. */
        pgo_haspage_t           *pgo_haspage;           /* Query page. */
+       pgo_populate_t          *pgo_populate;          /* Bulk spec pagein. */
        pgo_pageunswapped_t     *pgo_pageunswapped;
 };
 
@@ -151,6 +154,19 @@ vm_pager_has_page(
        return (ret);
 } 
 
+static __inline int
+vm_pager_populate(vm_object_t object, vm_pindex_t pidx, int fault_type,
+    vm_prot_t max_prot, vm_pindex_t *first, vm_pindex_t *last)
+{
+
+       MPASS((object->flags & OBJ_POPULATE) != 0);
+       MPASS(pidx < object->size);
+       MPASS(object->paging_in_progress > 0);
+       return ((*pagertab[object->type]->pgo_populate)(object, pidx,
+           fault_type, max_prot, first, last));
+}
+
+
 /* 
  *      vm_pager_page_unswapped
  * 
@@ -176,6 +192,9 @@ vm_pager_page_unswapped(vm_page_t m)
 struct cdev_pager_ops {
        int (*cdev_pg_fault)(vm_object_t vm_obj, vm_ooffset_t offset,
            int prot, vm_page_t *mres);
+       int (*cdev_pg_populate)(vm_object_t vm_obj, vm_pindex_t pidx,
+           int fault_type, vm_prot_t max_prot, vm_pindex_t *first,
+           vm_pindex_t *last);
        int (*cdev_pg_ctor)(void *handle, vm_ooffset_t size, vm_prot_t prot,
            vm_ooffset_t foff, struct ucred *cred, u_short *color);
        void (*cdev_pg_dtor)(void *handle);
_______________________________________________
svn-src-all@freebsd.org mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-all
To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"

Reply via email to