Similar refactoring to what has been done for the upper part of the fault
handler, ok?
Index: uvm/uvm_fault.c
===================================================================
RCS file: /cvs/src/sys/uvm/uvm_fault.c,v
retrieving revision 1.115
diff -u -p -r1.115 uvm_fault.c
--- uvm/uvm_fault.c 16 Feb 2021 09:10:17 -0000 1.115
+++ uvm/uvm_fault.c 22 Feb 2021 09:11:53 -0000
@@ -1039,6 +1039,98 @@ uvm_fault_upper(struct uvm_faultinfo *uf
}
/*
+ * uvm_fault_lower_lookup: look up on-memory uobj pages.
+ *
+ * 1. get on-memory pages.
+ * 2. if failed, give up (get only center page later).
+ * 3. if succeeded, enter h/w mapping of neighbor pages.
+ */
+
+struct vm_page *
+uvm_fault_lower_lookup(
+ struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
+ struct vm_page **pages)
+{
+ struct uvm_object *uobj = ufi->entry->object.uvm_obj;
+ struct vm_page *uobjpage = NULL;
+ int lcv, gotpages;
+ vaddr_t currva;
+
+ counters_inc(uvmexp_counters, flt_lget);
+ gotpages = flt->npages;
+ (void) uobj->pgops->pgo_get(uobj,
+ ufi->entry->offset + (flt->startva - ufi->entry->start),
+ pages, &gotpages, flt->centeridx,
+ flt->access_type & MASK(ufi->entry), ufi->entry->advice,
+ PGO_LOCKED);
+
+ /*
+ * check for pages to map, if we got any
+ */
+ if (gotpages == 0) {
+ return NULL;
+ }
+
+ currva = flt->startva;
+ for (lcv = 0; lcv < flt->npages; lcv++, currva += PAGE_SIZE) {
+ if (pages[lcv] == NULL ||
+ pages[lcv] == PGO_DONTCARE)
+ continue;
+
+ KASSERT((pages[lcv]->pg_flags & PG_RELEASED) == 0);
+
+ /*
+ * if center page is resident and not
+ * PG_BUSY, then pgo_get made it PG_BUSY
+ * for us and gave us a handle to it.
+ * remember this page as "uobjpage."
+ * (for later use).
+ */
+ if (lcv == flt->centeridx) {
+ uobjpage = pages[lcv];
+ continue;
+ }
+
+ /*
+ * note: calling pgo_get with locked data
+ * structures returns us pages which are
+ * neither busy nor released, so we don't
+ * need to check for this. we can just
+ * directly enter the page (after moving it
+ * to the head of the active queue [useful?]).
+ */
+
+ uvm_lock_pageq();
+ uvm_pageactivate(pages[lcv]); /* reactivate */
+ uvm_unlock_pageq();
+ counters_inc(uvmexp_counters, flt_nomap);
+
+ /*
+ * Since this page isn't the page that's
+ * actually faulting, ignore pmap_enter()
+ * failures; it's not critical that we
+ * enter these right now.
+ */
+ (void) pmap_enter(ufi->orig_map->pmap, currva,
+ VM_PAGE_TO_PHYS(pages[lcv]) | flt->pa_flags,
+ flt->enter_prot & MASK(ufi->entry),
+ PMAP_CANFAIL |
+ (flt->wired ? PMAP_WIRED : 0));
+
+ /*
+ * NOTE: page can't be PG_WANTED because
+ * we've held the lock the whole time
+ * we've had the handle.
+ */
+ atomic_clearbits_int(&pages[lcv]->pg_flags, PG_BUSY);
+ UVM_PAGE_OWN(pages[lcv], NULL);
+ }
+ pmap_update(ufi->orig_map->pmap);
+
+ return uobjpage;
+}
+
+/*
* uvm_fault_lower: handle lower fault.
*
*/
@@ -1049,10 +1141,9 @@ uvm_fault_lower(struct uvm_faultinfo *uf
struct vm_amap *amap = ufi->entry->aref.ar_amap;
struct uvm_object *uobj = ufi->entry->object.uvm_obj;
boolean_t promote, locked;
- int result, lcv, gotpages;
+ int result;
struct vm_page *uobjpage, *pg = NULL;
struct vm_anon *anon = NULL;
- vaddr_t currva;
voff_t uoff;
/*
@@ -1081,82 +1172,11 @@ uvm_fault_lower(struct uvm_faultinfo *uf
* we ask (with pgo_get) the object for resident pages that we care
* about and attempt to map them in. we do not let pgo_get block
* (PGO_LOCKED).
- *
- * ("get" has the option of doing a pmap_enter for us)
*/
- if (uobj != NULL) {
- counters_inc(uvmexp_counters, flt_lget);
- gotpages = flt->npages;
- (void) uobj->pgops->pgo_get(uobj, ufi->entry->offset +
- (flt->startva - ufi->entry->start),
- pages, &gotpages, flt->centeridx,
- flt->access_type & MASK(ufi->entry),
- ufi->entry->advice, PGO_LOCKED);
-
- /* check for pages to map, if we got any */
+ if (uobj == NULL) {
uobjpage = NULL;
- if (gotpages) {
- currva = flt->startva;
- for (lcv = 0 ; lcv < flt->npages ;
- lcv++, currva += PAGE_SIZE) {
- if (pages[lcv] == NULL ||
- pages[lcv] == PGO_DONTCARE)
- continue;
-
- KASSERT((pages[lcv]->pg_flags & PG_RELEASED) ==
0);
-
- /*
- * if center page is resident and not
- * PG_BUSY, then pgo_get made it PG_BUSY
- * for us and gave us a handle to it.
- * remember this page as "uobjpage."
- * (for later use).
- */
- if (lcv == flt->centeridx) {
- uobjpage = pages[lcv];
- continue;
- }
-
- /*
- * note: calling pgo_get with locked data
- * structures returns us pages which are
- * neither busy nor released, so we don't
- * need to check for this. we can just
- * directly enter the page (after moving it
- * to the head of the active queue [useful?]).
- */
-
- uvm_lock_pageq();
- uvm_pageactivate(pages[lcv]); /* reactivate */
- uvm_unlock_pageq();
- counters_inc(uvmexp_counters, flt_nomap);
-
- /*
- * Since this page isn't the page that's
- * actually faulting, ignore pmap_enter()
- * failures; it's not critical that we
- * enter these right now.
- */
- (void) pmap_enter(ufi->orig_map->pmap, currva,
- VM_PAGE_TO_PHYS(pages[lcv]) | flt->pa_flags,
- flt->enter_prot & MASK(ufi->entry),
- PMAP_CANFAIL |
- (flt->wired ? PMAP_WIRED : 0));
-
- /*
- * NOTE: page can't be PG_WANTED because
- * we've held the lock the whole time
- * we've had the handle.
- */
- atomic_clearbits_int(&pages[lcv]->pg_flags,
- PG_BUSY);
- UVM_PAGE_OWN(pages[lcv], NULL);
- } /* for "lcv" loop */
- pmap_update(ufi->orig_map->pmap);
- } /* "gotpages" != 0 */
- /* note: object still _locked_ */
} else {
- uobjpage = NULL;
+ uobjpage = uvm_fault_lower_lookup(ufi, flt, pages);
}
/*
@@ -1195,6 +1215,8 @@ uvm_fault_lower(struct uvm_faultinfo *uf
/* update rusage counters */
curproc->p_ru.ru_minflt++;
} else {
+ int gotpages;
+
/* update rusage counters */
curproc->p_ru.ru_majflt++;
