Module Name: src
Committed By: rmind
Date: Tue Jun 22 18:34:50 UTC 2010
Modified Files:
src/sys/uvm: uvm_bio.c uvm_fault.c uvm_map.c uvm_pager.c
Log Message:
Keep the lock around pmap_update() where required. While fixing this
in ubc_fault(), rework logic to "remember" the last object of page and
reduce locking overhead, since in common case pages belong to one and
the same UVM object (but not always, therefore add a comment).
Unlocks before pmap_update(), on removal of mappings, might cause TLB
coherency issues, since on architectures like x86 and mips64 invalidation
IPIs are deferred to pmap_update(). Hence, VA space might be globally
visible before IPIs are sent or while they are still in-flight.
OK a...@.
To generate a diff of this commit:
cvs rdiff -u -r1.69 -r1.70 src/sys/uvm/uvm_bio.c
cvs rdiff -u -r1.174 -r1.175 src/sys/uvm/uvm_fault.c
cvs rdiff -u -r1.291 -r1.292 src/sys/uvm/uvm_map.c
cvs rdiff -u -r1.97 -r1.98 src/sys/uvm/uvm_pager.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/uvm/uvm_bio.c
diff -u src/sys/uvm/uvm_bio.c:1.69 src/sys/uvm/uvm_bio.c:1.70
--- src/sys/uvm/uvm_bio.c:1.69 Sat May 29 23:17:53 2010
+++ src/sys/uvm/uvm_bio.c Tue Jun 22 18:34:50 2010
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_bio.c,v 1.69 2010/05/29 23:17:53 rmind Exp $ */
+/* $NetBSD: uvm_bio.c,v 1.70 2010/06/22 18:34:50 rmind Exp $ */
/*
* Copyright (c) 1998 Chuck Silvers.
@@ -34,7 +34,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_bio.c,v 1.69 2010/05/29 23:17:53 rmind Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_bio.c,v 1.70 2010/06/22 18:34:50 rmind Exp $");
#include "opt_uvmhist.h"
#include "opt_ubc.h"
@@ -217,9 +217,11 @@
/*
* ubc_fault_page: helper of ubc_fault to handle a single page.
+ *
+ * => Caller has UVM object locked.
*/
-static inline void
+static inline int
ubc_fault_page(const struct uvm_faultinfo *ufi, const struct ubc_map *umap,
struct vm_page *pg, vm_prot_t prot, vm_prot_t access_type, vaddr_t va)
{
@@ -229,7 +231,8 @@
bool rdonly;
uobj = pg->uobject;
- mutex_enter(&uobj->vmobjlock);
+ KASSERT(mutex_owned(&uobj->vmobjlock));
+
if (pg->flags & PG_WANTED) {
wakeup(pg);
}
@@ -238,8 +241,7 @@
mutex_enter(&uvm_pageqlock);
uvm_pagefree(pg);
mutex_exit(&uvm_pageqlock);
- mutex_exit(&uobj->vmobjlock);
- return;
+ return 0;
}
if (pg->loan_count != 0) {
@@ -256,12 +258,7 @@
newpg = uvm_loanbreak(pg);
if (newpg == NULL) {
uvm_page_unbusy(&pg, 1);
- mutex_exit(&uobj->vmobjlock);
- uvm_wait("ubc_loanbrk");
- /*
- * Note: will re-fault.
- */
- return;
+ return ENOMEM;
}
pg = newpg;
}
@@ -290,14 +287,8 @@
mutex_exit(&uvm_pageqlock);
pg->flags &= ~(PG_BUSY|PG_WANTED);
UVM_PAGE_OWN(pg, NULL);
- mutex_exit(&uobj->vmobjlock);
- if (error) {
- /*
- * Note: will re-fault.
- */
- uvm_wait("ubc_pmfail");
- }
+ return error;
}
/*
@@ -386,7 +377,7 @@
0);
if (error == EAGAIN) {
- kpause("ubc_fault", false, hz, NULL);
+ kpause("ubc_fault", false, hz >> 2, NULL);
goto again;
}
if (error) {
@@ -405,6 +396,16 @@
#else
prot = VM_PROT_READ | VM_PROT_WRITE;
#endif
+
+ /*
+ * Note: in the common case, all returned pages would have the same
+ * UVM object. However, due to layered file-systems and e.g. tmpfs,
+ * returned pages may have different objects. We "remember" the
+ * last object in the loop to reduce locking overhead and to perform
+ * pmap_update() before object unlock.
+ */
+ uobj = NULL;
+
va = ufi->orig_rvaddr;
eva = ufi->orig_rvaddr + (npages << PAGE_SHIFT);
@@ -418,9 +419,33 @@
if (pg == NULL || pg == PGO_DONTCARE) {
continue;
}
- ubc_fault_page(ufi, umap, pg, prot, access_type, va);
+ if (__predict_false(pg->uobject != uobj)) {
+ /* Check for the first iteration and error cases. */
+ if (uobj != NULL) {
+ /* Must make VA visible before the unlock. */
+ pmap_update(ufi->orig_map->pmap);
+ mutex_exit(&uobj->vmobjlock);
+ }
+ uobj = pg->uobject;
+ mutex_enter(&uobj->vmobjlock);
+ }
+ error = ubc_fault_page(ufi, umap, pg, prot, access_type, va);
+ if (error) {
+ /*
+ * Flush (there might be pages entered), drop the lock,
+ * "forget" the object and perform uvm_wait().
+ * Note: page will re-fault.
+ */
+ pmap_update(ufi->orig_map->pmap);
+ mutex_exit(&uobj->vmobjlock);
+ uobj = NULL;
+ uvm_wait("ubc_fault");
+ }
+ }
+ if (__predict_true(uobj != NULL)) {
+ pmap_update(ufi->orig_map->pmap);
+ mutex_exit(&uobj->vmobjlock);
}
- pmap_update(ufi->orig_map->pmap);
return 0;
}
Index: src/sys/uvm/uvm_fault.c
diff -u src/sys/uvm/uvm_fault.c:1.174 src/sys/uvm/uvm_fault.c:1.175
--- src/sys/uvm/uvm_fault.c:1.174 Fri May 28 23:41:14 2010
+++ src/sys/uvm/uvm_fault.c Tue Jun 22 18:34:50 2010
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_fault.c,v 1.174 2010/05/28 23:41:14 rmind Exp $ */
+/* $NetBSD: uvm_fault.c,v 1.175 2010/06/22 18:34:50 rmind Exp $ */
/*
*
@@ -39,7 +39,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_fault.c,v 1.174 2010/05/28 23:41:14 rmind Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_fault.c,v 1.175 2010/06/22 18:34:50 rmind Exp $");
#include "opt_uvmhist.h"
@@ -1496,10 +1496,11 @@
* done case 1! finish up by unlocking everything and returning success
*/
- if (anon != oanon)
+ if (anon != oanon) {
mutex_exit(&anon->an_lock);
- uvmfault_unlockall(ufi, amap, uobj, oanon);
+ }
pmap_update(ufi->orig_map->pmap);
+ uvmfault_unlockall(ufi, amap, uobj, oanon);
return 0;
}
@@ -2197,8 +2198,9 @@
uvm_fault_lower_done(ufi, flt, uobj, anon, pg);
- uvmfault_unlockall(ufi, amap, uobj, anon);
pmap_update(ufi->orig_map->pmap);
+ uvmfault_unlockall(ufi, amap, uobj, anon);
+
UVMHIST_LOG(maphist, "<- done (SUCCESS!)",0,0,0,0);
return 0;
}
Index: src/sys/uvm/uvm_map.c
diff -u src/sys/uvm/uvm_map.c:1.291 src/sys/uvm/uvm_map.c:1.292
--- src/sys/uvm/uvm_map.c:1.291 Fri May 14 05:32:06 2010
+++ src/sys/uvm/uvm_map.c Tue Jun 22 18:34:50 2010
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_map.c,v 1.291 2010/05/14 05:32:06 cegger Exp $ */
+/* $NetBSD: uvm_map.c,v 1.292 2010/06/22 18:34:50 rmind Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -71,7 +71,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_map.c,v 1.291 2010/05/14 05:32:06 cegger Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_map.c,v 1.292 2010/06/22 18:34:50 rmind Exp $");
#include "opt_ddb.h"
#include "opt_uvmhist.h"
@@ -2356,6 +2356,7 @@
* and we should free any such pages immediately.
* this is mostly used for kmem_map.
*/
+ KASSERT(vm_map_pmap(map) == pmap_kernel());
if ((entry->flags & UVM_MAP_KMAPENT) == 0) {
uvm_km_pgremove_intrsafe(map, entry->start,
@@ -2460,8 +2461,15 @@
first_entry = entry;
entry = next;
}
+
+ /*
+ * Note: if map is dying, leave pmap_update() for pmap_destroy(),
+ * which will be called later.
+ */
if ((map->flags & VM_MAP_DYING) == 0) {
pmap_update(vm_map_pmap(map));
+ } else {
+ KASSERT(vm_map_pmap(map) != pmap_kernel());
}
uvm_map_check(map, "unmap_remove leave");
Index: src/sys/uvm/uvm_pager.c
diff -u src/sys/uvm/uvm_pager.c:1.97 src/sys/uvm/uvm_pager.c:1.98
--- src/sys/uvm/uvm_pager.c:1.97 Sat Nov 7 07:27:50 2009
+++ src/sys/uvm/uvm_pager.c Tue Jun 22 18:34:50 2010
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_pager.c,v 1.97 2009/11/07 07:27:50 cegger Exp $ */
+/* $NetBSD: uvm_pager.c,v 1.98 2010/06/22 18:34:50 rmind Exp $ */
/*
*
@@ -39,7 +39,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_pager.c,v 1.97 2009/11/07 07:27:50 cegger Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_pager.c,v 1.98 2010/06/22 18:34:50 rmind Exp $");
#include "opt_uvmhist.h"
#include "opt_readahead.h"
@@ -230,6 +230,8 @@
*/
pmap_kremove(kva, npages << PAGE_SHIFT);
+ pmap_update(pmap_kernel());
+
if (kva == emergva) {
mutex_enter(&pager_map_wanted_lock);
emerginuse = false;
@@ -249,7 +251,6 @@
vm_map_unlock(pager_map);
if (entries)
uvm_unmap_detach(entries, 0);
- pmap_update(pmap_kernel());
UVMHIST_LOG(maphist,"<- done",0,0,0,0);
}
