Module Name: src
Committed By: riz
Date: Sun Nov 21 18:09:00 UTC 2010
Modified Files:
src/sys/uvm [netbsd-5]: uvm_bio.c uvm_fault.c uvm_map.c uvm_pager.c
Log Message:
Pull up following revision(s) (requested by rmind in ticket #1421):
sys/uvm/uvm_bio.c: revision 1.70
sys/uvm/uvm_map.c: revision 1.292
sys/uvm/uvm_pager.c: revision 1.98
sys/uvm/uvm_fault.c: revision 1.175
sys/uvm/uvm_bio.c: revision 1.69
ubc_fault: split-off code part handling a single page into ubc_fault_page().
Keep the lock around pmap_update() where required. While fixing this
in ubc_fault(), rework logic to "remember" the last object of page and
reduce locking overhead, since in common case pages belong to one and
the same UVM object (but not always, therefore add a comment).
Unlocks before pmap_update(), on removal of mappings, might cause TLB
coherency issues, since on architectures like x86 and mips64 invalidation
IPIs are deferred to pmap_update(). Hence, VA space might be globally
visible before IPIs are sent or while they are still in-flight.
OK a...@.
To generate a diff of this commit:
cvs rdiff -u -r1.65 -r1.65.10.1 src/sys/uvm/uvm_bio.c
cvs rdiff -u -r1.125.6.1 -r1.125.6.2 src/sys/uvm/uvm_fault.c
cvs rdiff -u -r1.263.4.3 -r1.263.4.4 src/sys/uvm/uvm_map.c
cvs rdiff -u -r1.92 -r1.92.14.1 src/sys/uvm/uvm_pager.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/uvm/uvm_bio.c
diff -u src/sys/uvm/uvm_bio.c:1.65 src/sys/uvm/uvm_bio.c:1.65.10.1
--- src/sys/uvm/uvm_bio.c:1.65 Mon May 5 17:11:17 2008
+++ src/sys/uvm/uvm_bio.c Sun Nov 21 18:09:00 2010
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_bio.c,v 1.65 2008/05/05 17:11:17 ad Exp $ */
+/* $NetBSD: uvm_bio.c,v 1.65.10.1 2010/11/21 18:09:00 riz Exp $ */
/*
* Copyright (c) 1998 Chuck Silvers.
@@ -34,7 +34,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_bio.c,v 1.65 2008/05/05 17:11:17 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_bio.c,v 1.65.10.1 2010/11/21 18:09:00 riz Exp $");
#include "opt_uvmhist.h"
#include "opt_ubc.h"
@@ -215,6 +215,82 @@
}
/*
+ * ubc_fault_page: helper of ubc_fault to handle a single page.
+ *
+ * => Caller has UVM object locked.
+ */
+
+static inline int
+ubc_fault_page(const struct uvm_faultinfo *ufi, const struct ubc_map *umap,
+ struct vm_page *pg, vm_prot_t prot, vm_prot_t access_type, vaddr_t va)
+{
+ struct uvm_object *uobj;
+ vm_prot_t mask;
+ int error;
+ bool rdonly;
+
+ uobj = pg->uobject;
+ KASSERT(mutex_owned(&uobj->vmobjlock));
+
+ if (pg->flags & PG_WANTED) {
+ wakeup(pg);
+ }
+ KASSERT((pg->flags & PG_FAKE) == 0);
+ if (pg->flags & PG_RELEASED) {
+ mutex_enter(&uvm_pageqlock);
+ uvm_pagefree(pg);
+ mutex_exit(&uvm_pageqlock);
+ return 0;
+ }
+ if (pg->loan_count != 0) {
+
+ /*
+ * Avoid unneeded loan break, if possible.
+ */
+
+ if ((access_type & VM_PROT_WRITE) == 0) {
+ prot &= ~VM_PROT_WRITE;
+ }
+ if (prot & VM_PROT_WRITE) {
+ struct vm_page *newpg;
+
+ newpg = uvm_loanbreak(pg);
+ if (newpg == NULL) {
+ uvm_page_unbusy(&pg, 1);
+ return ENOMEM;
+ }
+ pg = newpg;
+ }
+ }
+
+ /*
+ * Note that a page whose backing store is partially allocated
+ * is marked as PG_RDONLY.
+ */
+
+ KASSERT((pg->flags & PG_RDONLY) == 0 ||
+ (access_type & VM_PROT_WRITE) == 0 ||
+ pg->offset < umap->writeoff ||
+ pg->offset + PAGE_SIZE > umap->writeoff + umap->writelen);
+
+ rdonly = ((access_type & VM_PROT_WRITE) == 0 &&
+ (pg->flags & PG_RDONLY) != 0) ||
+ UVM_OBJ_NEEDS_WRITEFAULT(uobj);
+ mask = rdonly ? ~VM_PROT_WRITE : VM_PROT_ALL;
+
+ error = pmap_enter(ufi->orig_map->pmap, va, VM_PAGE_TO_PHYS(pg),
+ prot & mask, PMAP_CANFAIL | (access_type & mask));
+
+ mutex_enter(&uvm_pageqlock);
+ uvm_pageactivate(pg);
+ mutex_exit(&uvm_pageqlock);
+ pg->flags &= ~(PG_BUSY|PG_WANTED);
+ UVM_PAGE_OWN(pg, NULL);
+
+ return error;
+}
+
+/*
* ubc_fault: fault routine for ubc mapping
*/
@@ -225,10 +301,11 @@
struct uvm_object *uobj;
struct ubc_map *umap;
vaddr_t va, eva, ubc_offset, slot_offset;
+ struct vm_page *pgs[ubc_winsize >> PAGE_SHIFT];
int i, error, npages;
- struct vm_page *pgs[ubc_winsize >> PAGE_SHIFT], *pg;
vm_prot_t prot;
- UVMHIST_FUNC("ubc_fault"); UVMHIST_CALLED(ubchist);
+
+ UVMHIST_FUNC("ubc_fault"); UVMHIST_CALLED(ubchist);
/*
* no need to try with PGO_LOCKED...
@@ -299,106 +376,75 @@
0);
if (error == EAGAIN) {
- kpause("ubc_fault", false, hz, NULL);
+ kpause("ubc_fault", false, hz >> 2, NULL);
goto again;
}
if (error) {
return error;
}
+ /*
+ * For virtually-indexed, virtually-tagged caches we should avoid
+ * creating writable mappings when we do not absolutely need them,
+ * since the "compatible alias" trick does not work on such caches.
+ * Otherwise, we can always map the pages writable.
+ */
+
+#ifdef PMAP_CACHE_VIVT
+ prot = VM_PROT_READ | access_type;
+#else
+ prot = VM_PROT_READ | VM_PROT_WRITE;
+#endif
+
+ /*
+ * Note: in the common case, all returned pages would have the same
+ * UVM object. However, due to layered file-systems and e.g. tmpfs,
+ * returned pages may have different objects. We "remember" the
+ * last object in the loop to reduce locking overhead and to perform
+ * pmap_update() before object unlock.
+ */
+ uobj = NULL;
+
va = ufi->orig_rvaddr;
eva = ufi->orig_rvaddr + (npages << PAGE_SHIFT);
UVMHIST_LOG(ubchist, "va 0x%lx eva 0x%lx", va, eva, 0, 0);
for (i = 0; va < eva; i++, va += PAGE_SIZE) {
- bool rdonly;
- vm_prot_t mask;
+ struct vm_page *pg;
- /*
- * for virtually-indexed, virtually-tagged caches we should
- * avoid creating writable mappings when we don't absolutely
- * need them, since the "compatible alias" trick doesn't work
- * on such caches. otherwise, we can always map the pages
- * writable.
- */
-
-#ifdef PMAP_CACHE_VIVT
- prot = VM_PROT_READ | access_type;
-#else
- prot = VM_PROT_READ | VM_PROT_WRITE;
-#endif
UVMHIST_LOG(ubchist, "pgs[%d] = %p", i, pgs[i], 0, 0);
pg = pgs[i];
if (pg == NULL || pg == PGO_DONTCARE) {
continue;
}
-
- uobj = pg->uobject;
- mutex_enter(&uobj->vmobjlock);
- if (pg->flags & PG_WANTED) {
- wakeup(pg);
- }
- KASSERT((pg->flags & PG_FAKE) == 0);
- if (pg->flags & PG_RELEASED) {
- mutex_enter(&uvm_pageqlock);
- uvm_pagefree(pg);
- mutex_exit(&uvm_pageqlock);
- mutex_exit(&uobj->vmobjlock);
- continue;
+ if (__predict_false(pg->uobject != uobj)) {
+ /* Check for the first iteration and error cases. */
+ if (uobj != NULL) {
+ /* Must make VA visible before the unlock. */
+ pmap_update(ufi->orig_map->pmap);
+ mutex_exit(&uobj->vmobjlock);
+ }
+ uobj = pg->uobject;
+ mutex_enter(&uobj->vmobjlock);
}
- if (pg->loan_count != 0) {
-
+ error = ubc_fault_page(ufi, umap, pg, prot, access_type, va);
+ if (error) {
/*
- * avoid unneeded loan break if possible.
+ * Flush (there might be pages entered), drop the lock,
+ * "forget" the object and perform uvm_wait().
+ * Note: page will re-fault.
*/
-
- if ((access_type & VM_PROT_WRITE) == 0)
- prot &= ~VM_PROT_WRITE;
-
- if (prot & VM_PROT_WRITE) {
- struct vm_page *newpg;
-
- newpg = uvm_loanbreak(pg);
- if (newpg == NULL) {
- uvm_page_unbusy(&pg, 1);
- mutex_exit(&uobj->vmobjlock);
- uvm_wait("ubc_loanbrk");
- continue; /* will re-fault */
- }
- pg = newpg;
- }
+ pmap_update(ufi->orig_map->pmap);
+ mutex_exit(&uobj->vmobjlock);
+ uobj = NULL;
+ uvm_wait("ubc_fault");
}
-
- /*
- * note that a page whose backing store is partially allocated
- * is marked as PG_RDONLY.
- */
-
- rdonly = ((access_type & VM_PROT_WRITE) == 0 &&
- (pg->flags & PG_RDONLY) != 0) ||
- UVM_OBJ_NEEDS_WRITEFAULT(uobj);
- KASSERT((pg->flags & PG_RDONLY) == 0 ||
- (access_type & VM_PROT_WRITE) == 0 ||
- pg->offset < umap->writeoff ||
- pg->offset + PAGE_SIZE > umap->writeoff + umap->writelen);
- mask = rdonly ? ~VM_PROT_WRITE : VM_PROT_ALL;
- error = pmap_enter(ufi->orig_map->pmap, va, VM_PAGE_TO_PHYS(pg),
- prot & mask, PMAP_CANFAIL | (access_type & mask));
- mutex_enter(&uvm_pageqlock);
- uvm_pageactivate(pg);
- mutex_exit(&uvm_pageqlock);
- pg->flags &= ~(PG_BUSY|PG_WANTED);
- UVM_PAGE_OWN(pg, NULL);
+ }
+ if (__predict_true(uobj != NULL)) {
+ pmap_update(ufi->orig_map->pmap);
mutex_exit(&uobj->vmobjlock);
- if (error) {
- UVMHIST_LOG(ubchist, "pmap_enter fail %d",
- error, 0, 0, 0);
- uvm_wait("ubc_pmfail");
- /* will refault */
- }
}
- pmap_update(ufi->orig_map->pmap);
return 0;
}
Index: src/sys/uvm/uvm_fault.c
diff -u src/sys/uvm/uvm_fault.c:1.125.6.1 src/sys/uvm/uvm_fault.c:1.125.6.2
--- src/sys/uvm/uvm_fault.c:1.125.6.1 Mon Feb 2 18:31:37 2009
+++ src/sys/uvm/uvm_fault.c Sun Nov 21 18:09:00 2010
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_fault.c,v 1.125.6.1 2009/02/02 18:31:37 snj Exp $ */
+/* $NetBSD: uvm_fault.c,v 1.125.6.2 2010/11/21 18:09:00 riz Exp $ */
/*
*
@@ -39,7 +39,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_fault.c,v 1.125.6.1 2009/02/02 18:31:37 snj Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_fault.c,v 1.125.6.2 2010/11/21 18:09:00 riz Exp $");
#include "opt_uvmhist.h"
@@ -1441,10 +1441,11 @@
* done case 1! finish up by unlocking everything and returning success
*/
- if (anon != oanon)
+ if (anon != oanon) {
mutex_exit(&anon->an_lock);
- uvmfault_unlockall(&ufi, amap, uobj, oanon);
+ }
pmap_update(ufi.orig_map->pmap);
+ uvmfault_unlockall(&ufi, amap, uobj, oanon);
error = 0;
goto done;
@@ -1831,8 +1832,8 @@
pg->flags &= ~(PG_BUSY|PG_FAKE|PG_WANTED);
UVM_PAGE_OWN(pg, NULL);
- uvmfault_unlockall(&ufi, amap, uobj, anon);
pmap_update(ufi.orig_map->pmap);
+ uvmfault_unlockall(&ufi, amap, uobj, anon);
UVMHIST_LOG(maphist, "<- done (SUCCESS!)",0,0,0,0);
error = 0;
done:
Index: src/sys/uvm/uvm_map.c
diff -u src/sys/uvm/uvm_map.c:1.263.4.3 src/sys/uvm/uvm_map.c:1.263.4.4
--- src/sys/uvm/uvm_map.c:1.263.4.3 Sun Apr 19 15:43:14 2009
+++ src/sys/uvm/uvm_map.c Sun Nov 21 18:09:00 2010
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_map.c,v 1.263.4.3 2009/04/19 15:43:14 snj Exp $ */
+/* $NetBSD: uvm_map.c,v 1.263.4.4 2010/11/21 18:09:00 riz Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -71,7 +71,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_map.c,v 1.263.4.3 2009/04/19 15:43:14 snj Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_map.c,v 1.263.4.4 2010/11/21 18:09:00 riz Exp $");
#include "opt_ddb.h"
#include "opt_uvmhist.h"
@@ -2325,6 +2325,7 @@
* and we should free any such pages immediately.
* this is mostly used for kmem_map and mb_map.
*/
+ KASSERT(vm_map_pmap(map) == pmap_kernel());
if ((entry->flags & UVM_MAP_KMAPENT) == 0) {
uvm_km_pgremove_intrsafe(map, entry->start,
@@ -2429,8 +2430,15 @@
first_entry = entry;
entry = next;
}
+
+ /*
+ * Note: if map is dying, leave pmap_update() for pmap_destroy(),
+ * which will be called later.
+ */
if ((map->flags & VM_MAP_DYING) == 0) {
pmap_update(vm_map_pmap(map));
+ } else {
+ KASSERT(vm_map_pmap(map) != pmap_kernel());
}
uvm_map_check(map, "unmap_remove leave");
Index: src/sys/uvm/uvm_pager.c
diff -u src/sys/uvm/uvm_pager.c:1.92 src/sys/uvm/uvm_pager.c:1.92.14.1
--- src/sys/uvm/uvm_pager.c:1.92 Thu Apr 17 05:39:41 2008
+++ src/sys/uvm/uvm_pager.c Sun Nov 21 18:09:00 2010
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_pager.c,v 1.92 2008/04/17 05:39:41 simonb Exp $ */
+/* $NetBSD: uvm_pager.c,v 1.92.14.1 2010/11/21 18:09:00 riz Exp $ */
/*
*
@@ -39,7 +39,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_pager.c,v 1.92 2008/04/17 05:39:41 simonb Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_pager.c,v 1.92.14.1 2010/11/21 18:09:00 riz Exp $");
#include "opt_uvmhist.h"
#include "opt_readahead.h"
@@ -232,6 +232,8 @@
*/
pmap_kremove(kva, npages << PAGE_SHIFT);
+ pmap_update(pmap_kernel());
+
if (kva == emergva) {
mutex_enter(&pager_map_wanted_lock);
emerginuse = false;
@@ -251,7 +253,6 @@
vm_map_unlock(pager_map);
if (entries)
uvm_unmap_detach(entries, 0);
- pmap_update(pmap_kernel());
UVMHIST_LOG(maphist,"<- done",0,0,0,0);
}
