Module Name: src
Committed By: ad
Date: Sun Dec 15 21:11:35 UTC 2019
Modified Files:
src/sys/arch/hppa/hppa: pmap.c
src/sys/arch/sparc64/include: pmap.h
src/sys/arch/sparc64/sparc64: pmap.c
src/sys/miscfs/genfs: genfs_io.c
src/sys/nfs: nfs_subs.c
src/sys/rump/librump/rumpkern: Makefile.rumpkern vm.c
src/sys/ufs/lfs: lfs_pages.c
src/sys/uvm: files.uvm uvm_aobj.c uvm_loan.c uvm_object.c uvm_object.h
uvm_page.c uvm_page.h
Added Files:
src/sys/uvm: uvm_page_array.c uvm_page_array.h
Log Message:
Merge from yamt-pagecache:
- do gang lookup of pages using radixtree.
- remove now unused uvm_object::uo_memq and vm_page::listq.queue.
To generate a diff of this commit:
cvs rdiff -u -r1.100 -r1.101 src/sys/arch/hppa/hppa/pmap.c
cvs rdiff -u -r1.62 -r1.63 src/sys/arch/sparc64/include/pmap.h
cvs rdiff -u -r1.310 -r1.311 src/sys/arch/sparc64/sparc64/pmap.c
cvs rdiff -u -r1.77 -r1.78 src/sys/miscfs/genfs/genfs_io.c
cvs rdiff -u -r1.235 -r1.236 src/sys/nfs/nfs_subs.c
cvs rdiff -u -r1.178 -r1.179 src/sys/rump/librump/rumpkern/Makefile.rumpkern
cvs rdiff -u -r1.175 -r1.176 src/sys/rump/librump/rumpkern/vm.c
cvs rdiff -u -r1.16 -r1.17 src/sys/ufs/lfs/lfs_pages.c
cvs rdiff -u -r1.30 -r1.31 src/sys/uvm/files.uvm
cvs rdiff -u -r1.131 -r1.132 src/sys/uvm/uvm_aobj.c
cvs rdiff -u -r1.90 -r1.91 src/sys/uvm/uvm_loan.c
cvs rdiff -u -r1.17 -r1.18 src/sys/uvm/uvm_object.c
cvs rdiff -u -r1.34 -r1.35 src/sys/uvm/uvm_object.h
cvs rdiff -u -r1.202 -r1.203 src/sys/uvm/uvm_page.c
cvs rdiff -u -r1.86 -r1.87 src/sys/uvm/uvm_page.h
cvs rdiff -u -r0 -r1.2 src/sys/uvm/uvm_page_array.c \
src/sys/uvm/uvm_page_array.h
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/hppa/hppa/pmap.c
diff -u src/sys/arch/hppa/hppa/pmap.c:1.100 src/sys/arch/hppa/hppa/pmap.c:1.101
--- src/sys/arch/hppa/hppa/pmap.c:1.100 Thu Dec 22 14:47:57 2016
+++ src/sys/arch/hppa/hppa/pmap.c Sun Dec 15 21:11:34 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.c,v 1.100 2016/12/22 14:47:57 cherry Exp $ */
+/* $NetBSD: pmap.c,v 1.101 2019/12/15 21:11:34 ad Exp $ */
/*-
* Copyright (c) 2001, 2002 The NetBSD Foundation, Inc.
@@ -65,7 +65,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.100 2016/12/22 14:47:57 cherry Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.101 2019/12/15 21:11:34 ad Exp $");
#include "opt_cputype.h"
@@ -76,6 +76,7 @@ __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.1
#include <sys/mutex.h>
#include <uvm/uvm.h>
+#include <uvm/uvm_page_array.h>
#include <machine/reg.h>
#include <machine/psl.h>
@@ -395,7 +396,7 @@ pmap_pde_release(pmap_t pmap, vaddr_t va
pmap_pde_set(pmap, va, 0);
pmap->pm_stats.resident_count--;
if (pmap->pm_ptphint == ptp)
- pmap->pm_ptphint = TAILQ_FIRST(&pmap->pm_obj.memq);
+ pmap->pm_ptphint = NULL;
ptp->wire_count = 0;
KASSERT((ptp->flags & PG_BUSY) == 0);
@@ -1101,7 +1102,9 @@ void
pmap_destroy(pmap_t pmap)
{
#ifdef DIAGNOSTIC
+ struct uvm_page_array a;
struct vm_page *pg;
+ off_t off;
#endif
int refs;
@@ -1115,12 +1118,18 @@ pmap_destroy(pmap_t pmap)
return;
#ifdef DIAGNOSTIC
- while ((pg = TAILQ_FIRST(&pmap->pm_obj.memq))) {
+ uvm_page_array_init(&a);
+ off = 0;
+ mutex_enter(pmap->pm_lock);
+ while ((pg = uvm_page_array_fill_and_peek(&a, &pmap->pm_obj, off, 0, 0))
+ != NULL) {
pt_entry_t *pde, *epde;
struct vm_page *spg;
struct pv_entry *pv, *npv;
paddr_t pa;
+ off = pg->offset + PAGE_SIZE;
+ uvm_page_array_advance(&a);
KASSERT(pg != pmap->pm_pdir_pg);
pa = VM_PAGE_TO_PHYS(pg);
@@ -1152,6 +1161,8 @@ pmap_destroy(pmap_t pmap)
}
DPRINTF(PDB_FOLLOW, ("\n"));
}
+ mutex_exit(pmap->pm_lock);
+ uvm_page_array_fini(&a);
#endif
pmap_sdir_set(pmap->pm_space, 0);
mutex_enter(pmap->pm_lock);
Index: src/sys/arch/sparc64/include/pmap.h
diff -u src/sys/arch/sparc64/include/pmap.h:1.62 src/sys/arch/sparc64/include/pmap.h:1.63
--- src/sys/arch/sparc64/include/pmap.h:1.62 Thu Jan 10 10:33:49 2019
+++ src/sys/arch/sparc64/include/pmap.h Sun Dec 15 21:11:34 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.h,v 1.62 2019/01/10 10:33:49 mrg Exp $ */
+/* $NetBSD: pmap.h,v 1.63 2019/12/15 21:11:34 ad Exp $ */
/*-
* Copyright (C) 1995, 1996 Wolfgang Solfrank.
@@ -126,10 +126,8 @@ extern struct page_size_map page_size_ma
#endif
struct pmap {
- struct uvm_object pm_obj;
- kmutex_t pm_obj_lock;
-#define pm_lock pm_obj.vmobjlock
-#define pm_refs pm_obj.uo_refs
+ unsigned int pm_refs;
+ TAILQ_HEAD(, vm_page) pm_ptps;
LIST_ENTRY(pmap) pm_list[PMAP_LIST_MAXNUMCPU]; /* per cpu ctx used list */
struct pmap_statistics pm_stats;
Index: src/sys/arch/sparc64/sparc64/pmap.c
diff -u src/sys/arch/sparc64/sparc64/pmap.c:1.310 src/sys/arch/sparc64/sparc64/pmap.c:1.311
--- src/sys/arch/sparc64/sparc64/pmap.c:1.310 Thu Jan 10 10:33:49 2019
+++ src/sys/arch/sparc64/sparc64/pmap.c Sun Dec 15 21:11:34 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.c,v 1.310 2019/01/10 10:33:49 mrg Exp $ */
+/* $NetBSD: pmap.c,v 1.311 2019/12/15 21:11:34 ad Exp $ */
/*
*
* Copyright (C) 1996-1999 Eduardo Horvath.
@@ -26,7 +26,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.310 2019/01/10 10:33:49 mrg Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.311 2019/12/15 21:11:34 ad Exp $");
#undef NO_VCACHE /* Don't forget the locked TLB in dostart */
#define HWREF
@@ -1473,10 +1473,8 @@ pmap_create(void)
memset(pm, 0, sizeof *pm);
DPRINTF(PDB_CREATE, ("pmap_create(): created %p\n", pm));
- mutex_init(&pm->pm_obj_lock, MUTEX_DEFAULT, IPL_NONE);
- uvm_obj_init(&pm->pm_obj, NULL, false, 1);
- uvm_obj_setlock(&pm->pm_obj, &pm->pm_obj_lock);
-
+ pm->pm_refs = 1;
+ TAILQ_INIT(&pm->pm_ptps);
if (pm != pmap_kernel()) {
while (!pmap_get_page(&pm->pm_physaddr)) {
uvm_wait("pmap_create");
@@ -1510,7 +1508,7 @@ pmap_destroy(struct pmap *pm)
#else
#define pmap_cpus_active 0
#endif
- struct vm_page *pg, *nextpg;
+ struct vm_page *pg;
if ((int)atomic_dec_uint_nv(&pm->pm_refs) > 0) {
return;
@@ -1538,22 +1536,18 @@ pmap_destroy(struct pmap *pm)
#endif
/* we could be a little smarter and leave pages zeroed */
- for (pg = TAILQ_FIRST(&pm->pm_obj.memq); pg != NULL; pg = nextpg) {
+ while ((pg = TAILQ_FIRST(&pm->pm_ptps)) != NULL) {
#ifdef DIAGNOSTIC
struct vm_page_md *md = VM_PAGE_TO_MD(pg);
#endif
- KASSERT((pg->flags & PG_MARKER) == 0);
- nextpg = TAILQ_NEXT(pg, listq.queue);
- TAILQ_REMOVE(&pm->pm_obj.memq, pg, listq.queue);
+ TAILQ_REMOVE(&pm->pm_ptps, pg, pageq.queue);
KASSERT(md->mdpg_pvh.pv_pmap == NULL);
dcache_flush_page_cpuset(VM_PAGE_TO_PHYS(pg), pmap_cpus_active);
uvm_pagefree(pg);
}
pmap_free_page((paddr_t)(u_long)pm->pm_segs, pmap_cpus_active);
- uvm_obj_destroy(&pm->pm_obj, false);
- mutex_destroy(&pm->pm_obj_lock);
pool_cache_put(&pmap_cache, pm);
}
@@ -1904,7 +1898,7 @@ pmap_enter(struct pmap *pm, vaddr_t va,
ptpg = PHYS_TO_VM_PAGE(ptp);
if (ptpg) {
ptpg->offset = (uint64_t)va & (0xfffffLL << 23);
- TAILQ_INSERT_TAIL(&pm->pm_obj.memq, ptpg, listq.queue);
+ TAILQ_INSERT_TAIL(&pm->pm_ptps, ptpg, pageq.queue);
} else {
KASSERT(pm == pmap_kernel());
}
@@ -1916,7 +1910,7 @@ pmap_enter(struct pmap *pm, vaddr_t va,
ptpg = PHYS_TO_VM_PAGE(ptp);
if (ptpg) {
ptpg->offset = (((uint64_t)va >> 43) & 0x3ffLL) << 13;
- TAILQ_INSERT_TAIL(&pm->pm_obj.memq, ptpg, listq.queue);
+ TAILQ_INSERT_TAIL(&pm->pm_ptps, ptpg, pageq.queue);
} else {
KASSERT(pm == pmap_kernel());
}
Index: src/sys/miscfs/genfs/genfs_io.c
diff -u src/sys/miscfs/genfs/genfs_io.c:1.77 src/sys/miscfs/genfs/genfs_io.c:1.78
--- src/sys/miscfs/genfs/genfs_io.c:1.77 Fri Dec 13 20:10:21 2019
+++ src/sys/miscfs/genfs/genfs_io.c Sun Dec 15 21:11:34 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: genfs_io.c,v 1.77 2019/12/13 20:10:21 ad Exp $ */
+/* $NetBSD: genfs_io.c,v 1.78 2019/12/15 21:11:34 ad Exp $ */
/*
* Copyright (c) 1982, 1986, 1989, 1993
@@ -31,7 +31,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: genfs_io.c,v 1.77 2019/12/13 20:10:21 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: genfs_io.c,v 1.78 2019/12/15 21:11:34 ad Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -50,6 +50,7 @@ __KERNEL_RCSID(0, "$NetBSD: genfs_io.c,v
#include <uvm/uvm.h>
#include <uvm/uvm_pager.h>
+#include <uvm/uvm_page_array.h>
static int genfs_do_directio(struct vmspace *, vaddr_t, size_t, struct vnode *,
off_t, enum uio_rw);
@@ -785,13 +786,6 @@ loopdone:
* (e.g. vm_map) before calling flush.
* => if neither PGO_CLEANIT nor PGO_SYNCIO is set, we will not block
* => if PGO_ALLPAGES is set, then all pages in the object will be processed.
- * => NOTE: we rely on the fact that the object's memq is a TAILQ and
- * that new pages are inserted on the tail end of the list. thus,
- * we can make a complete pass through the object in one go by starting
- * at the head and working towards the tail (new pages are put in
- * front of us).
- * => NOTE: we are allowed to lock the page queues, so the caller
- * must not be holding the page queue lock.
*
* note on "cleaning" object and PG_BUSY pages:
* this routine is holding the lock on the object. the only time
@@ -806,19 +800,6 @@ loopdone:
* object we need to wait for the other PG_BUSY pages to clear
* off (i.e. we need to do an iosync). also note that once a
* page is PG_BUSY it must stay in its object until it is un-busyed.
- *
- * note on page traversal:
- * we can traverse the pages in an object either by going down the
- * linked list in "uobj->memq", or we can go over the address range
- * by page doing hash table lookups for each address. depending
- * on how many pages are in the object it may be cheaper to do one
- * or the other. we set "by_list" to true if we are using memq.
- * if the cost of a hash lookup was equal to the cost of the list
- * traversal we could compare the number of pages in the start->stop
- * range to the total number of pages in the object. however, it
- * seems that a hash table lookup is more expensive than the linked
- * list traversal, so we multiply the number of pages in the
- * range by an estimate of the relatively higher cost of the hash lookup.
*/
int
@@ -841,7 +822,7 @@ genfs_do_putpages(struct vnode *vp, off_
{
struct uvm_object * const uobj = &vp->v_uobj;
kmutex_t * const slock = uobj->vmobjlock;
- off_t off;
+ off_t nextoff;
int i, error, npages, nback;
int freeflag;
/*
@@ -850,8 +831,9 @@ genfs_do_putpages(struct vnode *vp, off_
*/
struct vm_page *pgs[MAXPHYS / MIN_PAGE_SIZE];
#define MAXPAGES (MAXPHYS / PAGE_SIZE)
- struct vm_page *pg, *nextpg, *tpg, curmp, endmp;
- bool wasclean, by_list, needs_clean, yld;
+ struct vm_page *pg, *tpg;
+ struct uvm_page_array a;
+ bool wasclean, needs_clean;
bool async = (origflags & PGO_SYNCIO) == 0;
bool pagedaemon = curlwp == uvm.pagedaemon_lwp;
struct lwp * const l = curlwp ? curlwp : &lwp0;
@@ -944,12 +926,10 @@ retry:
error = 0;
wasclean = (vp->v_numoutput == 0);
- off = startoff;
+ nextoff = startoff;
if (endoff == 0 || flags & PGO_ALLPAGES) {
endoff = trunc_page(LLONG_MAX);
}
- by_list = (uobj->uo_npages <=
- ((endoff - startoff) >> PAGE_SHIFT) * UVM_PAGE_TREE_PENALTY);
/*
* if this vnode is known not to have dirty pages,
@@ -966,10 +946,7 @@ retry:
}
/*
- * start the loop. when scanning by list, hold the last page
- * in the list before we start. pages allocated after we start
- * will be added to the end of the list, so we can stop at the
- * current last page.
+ * start the loop to scan pages.
*/
cleanall = (flags & PGO_CLEANIT) != 0 && wasclean &&
@@ -977,51 +954,48 @@ retry:
(vp->v_iflag & VI_ONWORKLST) != 0;
dirtygen = gp->g_dirtygen;
freeflag = pagedaemon ? PG_PAGEOUT : PG_RELEASED;
- if (by_list) {
- curmp.flags = PG_MARKER;
- endmp.flags = PG_MARKER;
- pg = TAILQ_FIRST(&uobj->memq);
- TAILQ_INSERT_TAIL(&uobj->memq, &endmp, listq.queue);
- } else {
- pg = uvm_pagelookup(uobj, off);
- }
- nextpg = NULL;
- while (by_list || off < endoff) {
-
+ uvm_page_array_init(&a);
+ for (;;) {
/*
* if the current page is not interesting, move on to the next.
*/
- KASSERT(pg == NULL || (pg->flags & PG_MARKER) != 0 ||
- pg->uobject == uobj);
- KASSERT(pg == NULL ||
- (pg->flags & (PG_RELEASED|PG_PAGEOUT)) == 0 ||
- (pg->flags & (PG_BUSY|PG_MARKER)) != 0);
- if (by_list) {
- if (pg == &endmp) {
- break;
- }
- if (pg->flags & PG_MARKER) {
- pg = TAILQ_NEXT(pg, listq.queue);
- continue;
- }
- if (pg->offset < startoff || pg->offset >= endoff ||
- pg->flags & (PG_RELEASED|PG_PAGEOUT)) {
- if (pg->flags & (PG_RELEASED|PG_PAGEOUT)) {
- wasclean = false;
- }
- pg = TAILQ_NEXT(pg, listq.queue);
- continue;
- }
- off = pg->offset;
- } else if (pg == NULL || pg->flags & (PG_RELEASED|PG_PAGEOUT)) {
- if (pg != NULL) {
- wasclean = false;
- }
- off += PAGE_SIZE;
- if (off < endoff) {
- pg = uvm_pagelookup(uobj, off);
- }
+ pg = uvm_page_array_fill_and_peek(&a, uobj, nextoff, 0, 0);
+ if (pg == NULL) {
+ break;
+ }
+
+ KASSERT(pg->uobject == uobj);
+ KASSERT((pg->flags & (PG_RELEASED|PG_PAGEOUT)) == 0 ||
+ (pg->flags & (PG_BUSY)) != 0);
+ KASSERT(pg->offset >= startoff);
+ KASSERT(pg->offset >= nextoff);
+
+ if (pg->offset >= endoff) {
+ break;
+ }
+
+ if (pg->flags & (PG_RELEASED|PG_PAGEOUT)) {
+ wasclean = false;
+ nextoff = pg->offset + PAGE_SIZE;
+ continue;
+ }
+
+ /*
+ * a preempt point.
+ */
+
+ if ((l->l_cpu->ci_schedstate.spc_flags & SPCF_SHOULDYIELD)
+ != 0) {
+ nextoff = pg->offset; /* visit this page again */
+ mutex_exit(slock);
+ preempt();
+ /*
+ * as we dropped the object lock, our cached pages can
+ * be stale.
+ */
+ uvm_page_array_clear(&a);
+ mutex_enter(slock);
continue;
}
@@ -1030,9 +1004,7 @@ retry:
* wait for it to become unbusy.
*/
- yld = (l->l_cpu->ci_schedstate.spc_flags &
- SPCF_SHOULDYIELD) && !pagedaemon;
- if (pg->flags & PG_BUSY || yld) {
+ if (pg->flags & PG_BUSY) {
UVMHIST_LOG(ubchist, "busy %#jx", (uintptr_t)pg,
0, 0, 0);
if (flags & PGO_BUSYFAIL && pg->flags & PG_BUSY) {
@@ -1050,33 +1022,21 @@ retry:
*/
break;
}
- if (by_list) {
- TAILQ_INSERT_BEFORE(pg, &curmp, listq.queue);
- UVMHIST_LOG(ubchist, "curmp next %#jx",
- (uintptr_t)TAILQ_NEXT(&curmp, listq.queue),
- 0, 0, 0);
- }
- if (yld) {
- mutex_exit(slock);
- preempt();
- mutex_enter(slock);
- } else {
- pg->flags |= PG_WANTED;
- UVM_UNLOCK_AND_WAIT(pg, slock, 0, "genput", 0);
- mutex_enter(slock);
- }
- if (by_list) {
- UVMHIST_LOG(ubchist, "after next %#jx",
- (uintptr_t)TAILQ_NEXT(&curmp, listq.queue),
- 0, 0, 0);
- pg = TAILQ_NEXT(&curmp, listq.queue);
- TAILQ_REMOVE(&uobj->memq, &curmp, listq.queue);
- } else {
- pg = uvm_pagelookup(uobj, off);
- }
+ nextoff = pg->offset; /* visit this page again */
+ pg->flags |= PG_WANTED;
+ UVM_UNLOCK_AND_WAIT(pg, slock, 0, "genput", 0);
+ /*
+ * as we dropped the object lock, our cached pages can
+ * be stale.
+ */
+ uvm_page_array_clear(&a);
+ mutex_enter(slock);
continue;
}
+ nextoff = pg->offset + PAGE_SIZE;
+ uvm_page_array_advance(&a);
+
/*
* if we're freeing, remove all mappings of the page now.
* if we're cleaning, check if the page is needs to be cleaned.
@@ -1136,7 +1096,7 @@ retry:
* it fits in the "pgs" pages array.
*/
- off_t fslo, fshi, genlo, lo;
+ off_t fslo, fshi, genlo, lo, off = pg->offset;
GOP_PUTRANGE(vp, off, &fslo, &fshi);
KASSERT(fslo == trunc_page(fslo));
KASSERT(fslo <= off);
@@ -1200,8 +1160,6 @@ retry:
for (i = 0; i < npages; i++) {
tpg = pgs[i];
KASSERT(tpg->uobject == uobj);
- if (by_list && tpg == TAILQ_NEXT(pg, listq.queue))
- pg = tpg;
if (tpg->offset < startoff || tpg->offset >= endoff)
continue;
if (flags & PGO_DEACTIVATE && tpg->wire_count == 0) {
@@ -1222,7 +1180,11 @@ retry:
* and needs_clean is false.
*/
- nextpg = TAILQ_NEXT(tpg, listq.queue);
+ KASSERT(npages == 1);
+ KASSERT(!needs_clean);
+ KASSERT(pg == tpg);
+ KASSERT(nextoff ==
+ tpg->offset + PAGE_SIZE);
uvm_pagefree(tpg);
if (pagedaemon)
uvmexp.pdfreed++;
@@ -1231,52 +1193,29 @@ retry:
}
if (needs_clean) {
modified = true;
+ KASSERT(nextoff == pg->offset + PAGE_SIZE);
+ KASSERT(nback < npages);
+ nextoff = pg->offset + ((npages - nback) << PAGE_SHIFT);
+ KASSERT(pgs[nback] == pg);
+ KASSERT(nextoff == pgs[npages - 1]->offset + PAGE_SIZE);
/*
- * start the i/o. if we're traversing by list,
- * keep our place in the list with a marker page.
+ * start the i/o.
*/
-
- if (by_list) {
- TAILQ_INSERT_AFTER(&uobj->memq, pg, &curmp,
- listq.queue);
- }
mutex_exit(slock);
error = GOP_WRITE(vp, pgs, npages, flags);
+ /*
+ * as we dropped the object lock, our cached pages can
+ * be stale.
+ */
+ uvm_page_array_clear(&a);
mutex_enter(slock);
- if (by_list) {
- pg = TAILQ_NEXT(&curmp, listq.queue);
- TAILQ_REMOVE(&uobj->memq, &curmp, listq.queue);
- }
if (error) {
break;
}
- if (by_list) {
- continue;
- }
- }
-
- /*
- * find the next page and continue if there was no error.
- */
-
- if (by_list) {
- if (nextpg) {
- pg = nextpg;
- nextpg = NULL;
- } else {
- pg = TAILQ_NEXT(pg, listq.queue);
- }
- } else {
- off += (npages - nback) << PAGE_SHIFT;
- if (off < endoff) {
- pg = uvm_pagelookup(uobj, off);
- }
}
}
- if (by_list) {
- TAILQ_REMOVE(&uobj->memq, &endmp, listq.queue);
- }
+ uvm_page_array_fini(&a);
if (modified && (vp->v_iflag & VI_WRMAPDIRTY) != 0 &&
(vp->v_type != VBLK ||
@@ -1377,8 +1316,17 @@ genfs_gop_write(struct vnode *vp, struct
return error;
}
+/*
+ * genfs_gop_write_rwmap:
+ *
+ * a variant of genfs_gop_write. it's used by UDF for its directory buffers.
+ * this maps pages with PROT_WRITE so that VOP_STRATEGY can modifies
+ * the contents before writing it out to the underlying storage.
+ */
+
int
-genfs_gop_write_rwmap(struct vnode *vp, struct vm_page **pgs, int npages, int flags)
+genfs_gop_write_rwmap(struct vnode *vp, struct vm_page **pgs, int npages,
+ int flags)
{
off_t off;
vaddr_t kva;
@@ -1441,6 +1389,10 @@ genfs_do_io(struct vnode *vp, off_t off,
KASSERT(bytes != 0);
if (iowrite) {
+ /*
+ * why += 2?
+ * 1 for biodone, 1 for uvm_aio_aiodone.
+ */
mutex_enter(vp->v_interlock);
vp->v_numoutput += 2;
mutex_exit(vp->v_interlock);
Index: src/sys/nfs/nfs_subs.c
diff -u src/sys/nfs/nfs_subs.c:1.235 src/sys/nfs/nfs_subs.c:1.236
--- src/sys/nfs/nfs_subs.c:1.235 Sat Dec 22 14:28:57 2018
+++ src/sys/nfs/nfs_subs.c Sun Dec 15 21:11:34 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: nfs_subs.c,v 1.235 2018/12/22 14:28:57 maxv Exp $ */
+/* $NetBSD: nfs_subs.c,v 1.236 2019/12/15 21:11:34 ad Exp $ */
/*
* Copyright (c) 1989, 1993
@@ -70,7 +70,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: nfs_subs.c,v 1.235 2018/12/22 14:28:57 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: nfs_subs.c,v 1.236 2019/12/15 21:11:34 ad Exp $");
#ifdef _KERNEL_OPT
#include "opt_nfs.h"
@@ -101,6 +101,7 @@ __KERNEL_RCSID(0, "$NetBSD: nfs_subs.c,v
#include <sys/cprng.h>
#include <uvm/uvm.h>
+#include <uvm/uvm_page_array.h>
#include <nfs/rpcv2.h>
#include <nfs/nfsproto.h>
@@ -1751,6 +1752,8 @@ nfs_clearcommit_selector(void *cl, struc
struct nfs_clearcommit_ctx *c = cl;
struct nfsnode *np;
struct vm_page *pg;
+ struct uvm_page_array a;
+ voff_t off;
KASSERT(mutex_owned(vp->v_interlock));
@@ -1761,9 +1764,15 @@ nfs_clearcommit_selector(void *cl, struc
np->n_pushedhi = 0;
np->n_commitflags &=
~(NFS_COMMIT_PUSH_VALID | NFS_COMMIT_PUSHED_VALID);
- TAILQ_FOREACH(pg, &vp->v_uobj.memq, listq.queue) {
+ uvm_page_array_init(&a);
+ off = 0;
+ while ((pg = uvm_page_array_fill_and_peek(&a, &vp->v_uobj, off,
+ 0, 0)) != NULL) {
pg->flags &= ~PG_NEEDCOMMIT;
+ uvm_page_array_advance(&a);
+ off = pg->offset + PAGE_SIZE;
}
+ uvm_page_array_fini(&a);
return false;
}
Index: src/sys/rump/librump/rumpkern/Makefile.rumpkern
diff -u src/sys/rump/librump/rumpkern/Makefile.rumpkern:1.178 src/sys/rump/librump/rumpkern/Makefile.rumpkern:1.179
--- src/sys/rump/librump/rumpkern/Makefile.rumpkern:1.178 Thu Dec 12 22:55:20 2019
+++ src/sys/rump/librump/rumpkern/Makefile.rumpkern Sun Dec 15 21:11:35 2019
@@ -1,4 +1,4 @@
-# $NetBSD: Makefile.rumpkern,v 1.178 2019/12/12 22:55:20 pgoyette Exp $
+# $NetBSD: Makefile.rumpkern,v 1.179 2019/12/15 21:11:35 ad Exp $
#
IOCONFDIR:= ${.PARSEDIR}
@@ -138,6 +138,7 @@ SRCS+= init_sysctl_base.c \
# sys/uvm
SRCS+= uvm_aobj.c uvm_readahead.c uvm_object.c uvm_swapstub.c
+SRCS+= uvm_page_array.c
# 4.4BSD secmodel. selection is hardcoded for now
SRCS+= secmodel.c
Index: src/sys/rump/librump/rumpkern/vm.c
diff -u src/sys/rump/librump/rumpkern/vm.c:1.175 src/sys/rump/librump/rumpkern/vm.c:1.176
--- src/sys/rump/librump/rumpkern/vm.c:1.175 Sat Dec 14 17:28:58 2019
+++ src/sys/rump/librump/rumpkern/vm.c Sun Dec 15 21:11:35 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: vm.c,v 1.175 2019/12/14 17:28:58 ad Exp $ */
+/* $NetBSD: vm.c,v 1.176 2019/12/15 21:11:35 ad Exp $ */
/*
* Copyright (c) 2007-2011 Antti Kantee. All Rights Reserved.
@@ -41,7 +41,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: vm.c,v 1.175 2019/12/14 17:28:58 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: vm.c,v 1.176 2019/12/15 21:11:35 ad Exp $");
#include <sys/param.h>
#include <sys/atomic.h>
@@ -176,7 +176,6 @@ uvm_pagealloc_strat(struct uvm_object *u
uvm_pagezero(pg);
}
- TAILQ_INSERT_TAIL(&uobj->memq, pg, listq.queue);
if (radix_tree_insert_node(&uobj->uo_pages, off >> PAGE_SHIFT,
pg) != 0) {
pool_cache_put(&pagecache, pg);
@@ -216,8 +215,6 @@ uvm_pagefree(struct vm_page *pg)
if (pg->flags & PG_WANTED)
wakeup(pg);
- TAILQ_REMOVE(&uobj->memq, pg, listq.queue);
-
uobj->uo_npages--;
pg2 = radix_tree_remove_node(&uobj->uo_pages, pg->offset >> PAGE_SHIFT);
KASSERT(pg == pg2);
Index: src/sys/ufs/lfs/lfs_pages.c
diff -u src/sys/ufs/lfs/lfs_pages.c:1.16 src/sys/ufs/lfs/lfs_pages.c:1.17
--- src/sys/ufs/lfs/lfs_pages.c:1.16 Fri Dec 13 20:10:22 2019
+++ src/sys/ufs/lfs/lfs_pages.c Sun Dec 15 21:11:35 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: lfs_pages.c,v 1.16 2019/12/13 20:10:22 ad Exp $ */
+/* $NetBSD: lfs_pages.c,v 1.17 2019/12/15 21:11:35 ad Exp $ */
/*-
* Copyright (c) 1999, 2000, 2001, 2002, 2003, 2019 The NetBSD Foundation, Inc.
@@ -60,7 +60,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: lfs_pages.c,v 1.16 2019/12/13 20:10:22 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: lfs_pages.c,v 1.17 2019/12/15 21:11:35 ad Exp $");
#ifdef _KERNEL_OPT
#include "opt_compat_netbsd.h"
@@ -242,8 +242,6 @@ check_dirty(struct lfs *fs, struct vnode
off_t startoffset, off_t endoffset, off_t blkeof,
int flags, int checkfirst, struct vm_page **pgp)
{
- int by_list;
- struct vm_page *curpg = NULL; /* XXX: gcc */
struct vm_page *pgs[MAXBSIZE / MIN_PAGE_SIZE], *pg;
off_t soff = 0; /* XXX: gcc */
voff_t off;
@@ -258,39 +256,11 @@ check_dirty(struct lfs *fs, struct vnode
KASSERT(mutex_owned(vp->v_interlock));
ASSERT_MAYBE_SEGLOCK(fs);
top:
- by_list = (vp->v_uobj.uo_npages <=
- ((endoffset - startoffset) >> PAGE_SHIFT) *
- UVM_PAGE_TREE_PENALTY);
any_dirty = 0;
- if (by_list) {
- curpg = TAILQ_FIRST(&vp->v_uobj.memq);
- } else {
- soff = startoffset;
- }
- while (by_list || soff < MIN(blkeof, endoffset)) {
- if (by_list) {
- /*
- * Find the first page in a block. Skip
- * blocks outside our area of interest or beyond
- * the end of file.
- */
- KASSERT(curpg == NULL
- || (curpg->flags & PG_MARKER) == 0);
- if (pages_per_block > 1) {
- while (curpg &&
- ((curpg->offset & lfs_sb_getbmask(fs)) ||
- curpg->offset >= vp->v_size ||
- curpg->offset >= endoffset)) {
- curpg = TAILQ_NEXT(curpg, listq.queue);
- KASSERT(curpg == NULL ||
- (curpg->flags & PG_MARKER) == 0);
- }
- }
- if (curpg == NULL)
- break;
- soff = curpg->offset;
- }
+ soff = startoffset;
+ KASSERT((soff & (lfs_sb_getbsize(fs) - 1)) == 0);
+ while (soff < MIN(blkeof, endoffset)) {
/*
* Mark all pages in extended range busy; find out if any
@@ -299,15 +269,11 @@ check_dirty(struct lfs *fs, struct vnode
nonexistent = dirty = 0;
for (i = 0; i == 0 || i < pages_per_block; i++) {
KASSERT(mutex_owned(vp->v_interlock));
- if (by_list && pages_per_block <= 1) {
- pgs[i] = pg = curpg;
- } else {
- off = soff + (i << PAGE_SHIFT);
- pgs[i] = pg = uvm_pagelookup(&vp->v_uobj, off);
- if (pg == NULL) {
- ++nonexistent;
- continue;
- }
+ off = soff + (i << PAGE_SHIFT);
+ pgs[i] = pg = uvm_pagelookup(&vp->v_uobj, off);
+ if (pg == NULL) {
+ ++nonexistent;
+ continue;
}
KASSERT(pg != NULL);
@@ -344,12 +310,8 @@ check_dirty(struct lfs *fs, struct vnode
(pg->flags & PG_CLEAN) == 0);
dirty += tdirty;
}
- if (pages_per_block > 0 && nonexistent >= pages_per_block) {
- if (by_list) {
- curpg = TAILQ_NEXT(curpg, listq.queue);
- } else {
- soff += lfs_sb_getbsize(fs);
- }
+ if (nonexistent >= pages_per_block) {
+ soff += MAX(PAGE_SIZE, lfs_sb_getbsize(fs));
continue;
}
@@ -390,11 +352,7 @@ check_dirty(struct lfs *fs, struct vnode
if (checkfirst && any_dirty)
break;
- if (by_list) {
- curpg = TAILQ_NEXT(curpg, listq.queue);
- } else {
- soff += MAX(PAGE_SIZE, lfs_sb_getbsize(fs));
- }
+ soff += MAX(PAGE_SIZE, lfs_sb_getbsize(fs));
}
KASSERT(mutex_owned(vp->v_interlock));
@@ -492,8 +450,7 @@ retry:
* If there are no pages, don't do anything.
*/
if (vp->v_uobj.uo_npages == 0) {
- if (TAILQ_EMPTY(&vp->v_uobj.memq) &&
- (vp->v_iflag & VI_ONWORKLST) &&
+ if ((vp->v_iflag & VI_ONWORKLST) &&
LIST_FIRST(&vp->v_dirtyblkhd) == NULL) {
vp->v_iflag &= ~VI_WRMAPDIRTY;
vn_syncer_remove_from_worklist(vp);
Index: src/sys/uvm/files.uvm
diff -u src/sys/uvm/files.uvm:1.30 src/sys/uvm/files.uvm:1.31
--- src/sys/uvm/files.uvm:1.30 Wed Nov 20 19:37:54 2019
+++ src/sys/uvm/files.uvm Sun Dec 15 21:11:35 2019
@@ -1,4 +1,4 @@
-# $NetBSD: files.uvm,v 1.30 2019/11/20 19:37:54 pgoyette Exp $
+# $NetBSD: files.uvm,v 1.31 2019/12/15 21:11:35 ad Exp $
#
# UVM options
@@ -37,6 +37,7 @@ file uvm/uvm_mmap.c uvm
file uvm/uvm_mremap.c uvm
file uvm/uvm_object.c uvm
file uvm/uvm_page.c uvm
+file uvm/uvm_page_array.c uvm
file uvm/uvm_pager.c uvm
file uvm/uvm_pdaemon.c uvm
file uvm/uvm_pdpolicy_clock.c !pdpolicy_clockpro
Index: src/sys/uvm/uvm_aobj.c
diff -u src/sys/uvm/uvm_aobj.c:1.131 src/sys/uvm/uvm_aobj.c:1.132
--- src/sys/uvm/uvm_aobj.c:1.131 Fri Dec 13 20:10:22 2019
+++ src/sys/uvm/uvm_aobj.c Sun Dec 15 21:11:35 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_aobj.c,v 1.131 2019/12/13 20:10:22 ad Exp $ */
+/* $NetBSD: uvm_aobj.c,v 1.132 2019/12/15 21:11:35 ad Exp $ */
/*
* Copyright (c) 1998 Chuck Silvers, Charles D. Cranor and
@@ -38,7 +38,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_aobj.c,v 1.131 2019/12/13 20:10:22 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_aobj.c,v 1.132 2019/12/15 21:11:35 ad Exp $");
#ifdef _KERNEL_OPT
#include "opt_uvmhist.h"
@@ -52,6 +52,7 @@ __KERNEL_RCSID(0, "$NetBSD: uvm_aobj.c,v
#include <sys/atomic.h>
#include <uvm/uvm.h>
+#include <uvm/uvm_page_array.h>
/*
* An anonymous UVM object (aobj) manages anonymous-memory. In addition to
@@ -137,7 +138,7 @@ static struct pool uao_swhash_elt_pool _
*/
struct uvm_aobj {
- struct uvm_object u_obj; /* has: lock, pgops, memq, #pages, #refs */
+ struct uvm_object u_obj; /* has: lock, pgops, #pages, #refs */
pgoff_t u_pages; /* number of pages in entire object */
int u_flags; /* the flags (see uvm_aobj.h) */
int *u_swslots; /* array of offset->swapslot mappings */
@@ -575,6 +576,7 @@ void
uao_detach(struct uvm_object *uobj)
{
struct uvm_aobj *aobj = (struct uvm_aobj *)uobj;
+ struct uvm_page_array a;
struct vm_page *pg;
UVMHIST_FUNC("uao_detach"); UVMHIST_CALLED(maphist);
@@ -612,19 +614,24 @@ uao_detach(struct uvm_object *uobj)
* involved in is complete), release any swap resources and free
* the page itself.
*/
+ uvm_page_array_init(&a);
mutex_enter(uobj->vmobjlock);
- while ((pg = TAILQ_FIRST(&uobj->memq)) != NULL) {
+ while ((pg = uvm_page_array_fill_and_peek(&a, uobj, 0, 0, 0))
+ != NULL) {
+ uvm_page_array_advance(&a);
pmap_page_protect(pg, VM_PROT_NONE);
if (pg->flags & PG_BUSY) {
pg->flags |= PG_WANTED;
UVM_UNLOCK_AND_WAIT(pg, uobj->vmobjlock, false,
"uao_det", 0);
+ uvm_page_array_clear(&a);
mutex_enter(uobj->vmobjlock);
continue;
}
uao_dropswap(&aobj->u_obj, pg->offset >> PAGE_SHIFT);
uvm_pagefree(pg);
}
+ uvm_page_array_fini(&a);
/*
* Finally, free the anonymous UVM object itself.
@@ -646,45 +653,25 @@ uao_detach(struct uvm_object *uobj)
* or block.
* => if PGO_ALLPAGE is set, then all pages in the object are valid targets
* for flushing.
- * => NOTE: we rely on the fact that the object's memq is a TAILQ and
- * that new pages are inserted on the tail end of the list. thus,
- * we can make a complete pass through the object in one go by starting
- * at the head and working towards the tail (new pages are put in
- * front of us).
* => we return 0 unless we encountered some sort of I/O error
* XXXJRT currently never happens, as we never directly initiate
* XXXJRT I/O
- *
- * note on page traversal:
- * we can traverse the pages in an object either by going down the
- * linked list in "uobj->memq", or we can go over the address range
- * by page doing hash table lookups for each address. depending
- * on how many pages are in the object it may be cheaper to do one
- * or the other. we set "by_list" to true if we are using memq.
- * if the cost of a hash lookup was equal to the cost of the list
- * traversal we could compare the number of pages in the start->stop
- * range to the total number of pages in the object. however, it
- * seems that a hash table lookup is more expensive than the linked
- * list traversal, so we multiply the number of pages in the
- * start->stop range by a penalty which we define below.
*/
static int
uao_put(struct uvm_object *uobj, voff_t start, voff_t stop, int flags)
{
struct uvm_aobj *aobj = (struct uvm_aobj *)uobj;
- struct vm_page *pg, *nextpg, curmp, endmp;
- bool by_list;
+ struct uvm_page_array a;
+ struct vm_page *pg;
voff_t curoff;
UVMHIST_FUNC("uao_put"); UVMHIST_CALLED(maphist);
KASSERT(mutex_owned(uobj->vmobjlock));
- curoff = 0;
if (flags & PGO_ALLPAGES) {
start = 0;
stop = aobj->u_pages << PAGE_SHIFT;
- by_list = true; /* always go by the list */
} else {
start = trunc_page(start);
if (stop == 0) {
@@ -699,12 +686,10 @@ uao_put(struct uvm_object *uobj, voff_t
(uintmax_t)(aobj->u_pages << PAGE_SHIFT));
stop = aobj->u_pages << PAGE_SHIFT;
}
- by_list = (uobj->uo_npages <=
- ((stop - start) >> PAGE_SHIFT) * UVM_PAGE_TREE_PENALTY);
}
UVMHIST_LOG(maphist,
- " flush start=0x%jx, stop=0x%jx, by_list=%jd, flags=0x%jx",
- start, stop, by_list, flags);
+ " flush start=0x%jx, stop=0x%jx, flags=0x%jx",
+ start, stop, flags, 0);
/*
* Don't need to do any work here if we're not freeing
@@ -716,47 +701,13 @@ uao_put(struct uvm_object *uobj, voff_t
return 0;
}
- /*
- * Initialize the marker pages. See the comment in
- * genfs_putpages() also.
- */
-
- curmp.flags = PG_MARKER;
- endmp.flags = PG_MARKER;
-
- /*
- * now do it. note: we must update nextpg in the body of loop or we
- * will get stuck. we need to use nextpg if we'll traverse the list
- * because we may free "pg" before doing the next loop.
- */
-
- if (by_list) {
- TAILQ_INSERT_TAIL(&uobj->memq, &endmp, listq.queue);
- nextpg = TAILQ_FIRST(&uobj->memq);
- } else {
- curoff = start;
- nextpg = NULL; /* Quell compiler warning */
- }
-
/* locked: uobj */
- for (;;) {
- if (by_list) {
- pg = nextpg;
- if (pg == &endmp)
- break;
- nextpg = TAILQ_NEXT(pg, listq.queue);
- if (pg->flags & PG_MARKER)
- continue;
- if (pg->offset < start || pg->offset >= stop)
- continue;
- } else {
- if (curoff < stop) {
- pg = uvm_pagelookup(uobj, curoff);
- curoff += PAGE_SIZE;
- } else
- break;
- if (pg == NULL)
- continue;
+ uvm_page_array_init(&a);
+ curoff = start;
+ while ((pg = uvm_page_array_fill_and_peek(&a, uobj, curoff, 0, 0)) !=
+ NULL) {
+ if (pg->offset >= stop) {
+ break;
}
/*
@@ -764,21 +715,15 @@ uao_put(struct uvm_object *uobj, voff_t
*/
if (pg->flags & PG_BUSY) {
- if (by_list) {
- TAILQ_INSERT_BEFORE(pg, &curmp, listq.queue);
- }
pg->flags |= PG_WANTED;
UVM_UNLOCK_AND_WAIT(pg, uobj->vmobjlock, 0,
"uao_put", 0);
+ uvm_page_array_clear(&a);
mutex_enter(uobj->vmobjlock);
- if (by_list) {
- nextpg = TAILQ_NEXT(&curmp, listq.queue);
- TAILQ_REMOVE(&uobj->memq, &curmp,
- listq.queue);
- } else
- curoff -= PAGE_SIZE;
continue;
}
+ uvm_page_array_advance(&a);
+ curoff = pg->offset + PAGE_SIZE;
switch (flags & (PGO_CLEANIT|PGO_FREE|PGO_DEACTIVATE)) {
@@ -825,10 +770,8 @@ uao_put(struct uvm_object *uobj, voff_t
panic("%s: impossible", __func__);
}
}
- if (by_list) {
- TAILQ_REMOVE(&uobj->memq, &endmp, listq.queue);
- }
mutex_exit(uobj->vmobjlock);
+ uvm_page_array_fini(&a);
return 0;
}
Index: src/sys/uvm/uvm_loan.c
diff -u src/sys/uvm/uvm_loan.c:1.90 src/sys/uvm/uvm_loan.c:1.91
--- src/sys/uvm/uvm_loan.c:1.90 Sat Dec 14 15:08:45 2019
+++ src/sys/uvm/uvm_loan.c Sun Dec 15 21:11:35 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_loan.c,v 1.90 2019/12/14 15:08:45 ad Exp $ */
+/* $NetBSD: uvm_loan.c,v 1.91 2019/12/15 21:11:35 ad Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -32,7 +32,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_loan.c,v 1.90 2019/12/14 15:08:45 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_loan.c,v 1.91 2019/12/15 21:11:35 ad Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -844,8 +844,8 @@ again:
* first, get ahold of our single zero page.
*/
- if (__predict_false((pg =
- TAILQ_FIRST(&uvm_loanzero_object.memq)) == NULL)) {
+ pg = uvm_pagelookup(&uvm_loanzero_object, 0);
+ if (__predict_false(pg == NULL)) {
while ((pg = uvm_pagealloc(&uvm_loanzero_object, 0, NULL,
UVM_PGA_ZERO)) == NULL) {
mutex_exit(uvm_loanzero_object.vmobjlock);
@@ -1060,13 +1060,12 @@ ulz_put(struct uvm_object *uobj, voff_t
* just reactivate or dequeue it.
*/
- pg = TAILQ_FIRST(&uobj->memq);
+ pg = uvm_pagelookup(uobj, 0);
KASSERT(pg != NULL);
- KASSERT(TAILQ_NEXT(pg, listq.queue) == NULL);
- if (pg->uanon)
+ if (pg->uanon) {
uvm_pageactivate(pg);
- else {
+ } else {
uvm_pagedequeue(pg);
}
Index: src/sys/uvm/uvm_object.c
diff -u src/sys/uvm/uvm_object.c:1.17 src/sys/uvm/uvm_object.c:1.18
--- src/sys/uvm/uvm_object.c:1.17 Sat Dec 14 17:28:58 2019
+++ src/sys/uvm/uvm_object.c Sun Dec 15 21:11:35 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_object.c,v 1.17 2019/12/14 17:28:58 ad Exp $ */
+/* $NetBSD: uvm_object.c,v 1.18 2019/12/15 21:11:35 ad Exp $ */
/*
* Copyright (c) 2006, 2010, 2019 The NetBSD Foundation, Inc.
@@ -37,7 +37,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_object.c,v 1.17 2019/12/14 17:28:58 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_object.c,v 1.18 2019/12/15 21:11:35 ad Exp $");
#ifdef _KERNEL_OPT
#include "opt_ddb.h"
@@ -49,6 +49,7 @@ __KERNEL_RCSID(0, "$NetBSD: uvm_object.c
#include <uvm/uvm.h>
#include <uvm/uvm_ddb.h>
+#include <uvm/uvm_page_array.h>
/* Page count to fetch per single step. */
#define FETCH_PAGECOUNT 16
@@ -72,7 +73,6 @@ uvm_obj_init(struct uvm_object *uo, cons
uo->vmobjlock = NULL;
}
uo->pgops = ops;
- TAILQ_INIT(&uo->memq);
LIST_INIT(&uo->uo_ubc);
uo->uo_npages = 0;
uo->uo_refs = refs;
@@ -237,8 +237,10 @@ void
uvm_object_printit(struct uvm_object *uobj, bool full,
void (*pr)(const char *, ...))
{
+ struct uvm_page_array a;
struct vm_page *pg;
int cnt = 0;
+ voff_t off;
(*pr)("OBJECT %p: locked=%d, pgops=%p, npages=%d, ",
uobj, mutex_owned(uobj->vmobjlock), uobj->pgops, uobj->uo_npages);
@@ -251,16 +253,22 @@ uvm_object_printit(struct uvm_object *uo
return;
}
(*pr)(" PAGES <pg,offset>:\n ");
- TAILQ_FOREACH(pg, &uobj->memq, listq.queue) {
+ uvm_page_array_init(&a);
+ off = 0;
+ while ((pg = uvm_page_array_fill_and_peek(&a, uobj, off, 0, 0))
+ != NULL) {
cnt++;
(*pr)("<%p,0x%llx> ", pg, (long long)pg->offset);
if ((cnt % 3) == 0) {
(*pr)("\n ");
}
+ off = pg->offset + PAGE_SIZE;
+ uvm_page_array_advance(&a);
}
if ((cnt % 3) != 0) {
(*pr)("\n");
}
+ uvm_page_array_fini(&a);
}
#endif /* DDB || DEBUGPRINT */
Index: src/sys/uvm/uvm_object.h
diff -u src/sys/uvm/uvm_object.h:1.34 src/sys/uvm/uvm_object.h:1.35
--- src/sys/uvm/uvm_object.h:1.34 Sat Dec 14 17:28:58 2019
+++ src/sys/uvm/uvm_object.h Sun Dec 15 21:11:35 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_object.h,v 1.34 2019/12/14 17:28:58 ad Exp $ */
+/* $NetBSD: uvm_object.h,v 1.35 2019/12/15 21:11:35 ad Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -56,7 +56,6 @@
struct uvm_object {
kmutex_t * vmobjlock; /* lock on object */
const struct uvm_pagerops *pgops; /* pager ops */
- struct pglist memq; /* pages in this object */
int uo_npages; /* # of pages in uo_pages */
unsigned uo_refs; /* reference count */
struct radix_tree uo_pages; /* tree of pages */
Index: src/sys/uvm/uvm_page.c
diff -u src/sys/uvm/uvm_page.c:1.202 src/sys/uvm/uvm_page.c:1.203
--- src/sys/uvm/uvm_page.c:1.202 Sat Dec 14 17:28:58 2019
+++ src/sys/uvm/uvm_page.c Sun Dec 15 21:11:35 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_page.c,v 1.202 2019/12/14 17:28:58 ad Exp $ */
+/* $NetBSD: uvm_page.c,v 1.203 2019/12/15 21:11:35 ad Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -66,7 +66,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_page.c,v 1.202 2019/12/14 17:28:58 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_page.c,v 1.203 2019/12/15 21:11:35 ad Exp $");
#include "opt_ddb.h"
#include "opt_uvm.h"
@@ -167,15 +167,12 @@ static void uvm_pageremove(struct uvm_ob
*/
static inline void
-uvm_pageinsert_list(struct uvm_object *uobj, struct vm_page *pg,
- struct vm_page *where)
+uvm_pageinsert_object(struct uvm_object *uobj, struct vm_page *pg)
{
KASSERT(uobj == pg->uobject);
KASSERT(mutex_owned(uobj->vmobjlock));
KASSERT((pg->flags & PG_TABLED) == 0);
- KASSERT(where == NULL || (where->flags & PG_TABLED));
- KASSERT(where == NULL || (where->uobject == uobj));
if (UVM_OBJ_IS_VNODE(uobj)) {
if (uobj->uo_npages == 0) {
@@ -191,11 +188,6 @@ uvm_pageinsert_list(struct uvm_object *u
} else if (UVM_OBJ_IS_AOBJ(uobj)) {
atomic_inc_uint(&uvmexp.anonpages);
}
-
- if (where)
- TAILQ_INSERT_AFTER(&uobj->memq, where, pg, listq.queue);
- else
- TAILQ_INSERT_TAIL(&uobj->memq, pg, listq.queue);
pg->flags |= PG_TABLED;
uobj->uo_npages++;
}
@@ -225,7 +217,7 @@ uvm_pageinsert(struct uvm_object *uobj,
KASSERT(error == ENOMEM);
return error;
}
- uvm_pageinsert_list(uobj, pg, NULL);
+ uvm_pageinsert_object(uobj, pg);
return error;
}
@@ -236,7 +228,7 @@ uvm_pageinsert(struct uvm_object *uobj,
*/
static inline void
-uvm_pageremove_list(struct uvm_object *uobj, struct vm_page *pg)
+uvm_pageremove_object(struct uvm_object *uobj, struct vm_page *pg)
{
KASSERT(uobj == pg->uobject);
@@ -260,7 +252,6 @@ uvm_pageremove_list(struct uvm_object *u
/* object should be locked */
uobj->uo_npages--;
- TAILQ_REMOVE(&uobj->memq, pg, listq.queue);
pg->flags &= ~PG_TABLED;
pg->uobject = NULL;
}
@@ -280,7 +271,7 @@ uvm_pageremove(struct uvm_object *uobj,
KDASSERT(uobj != NULL);
KASSERT(uobj == pg->uobject);
- uvm_pageremove_list(uobj, pg);
+ uvm_pageremove_object(uobj, pg);
uvm_pageremove_tree(uobj, pg);
}
@@ -1105,8 +1096,8 @@ uvm_pagereplace(struct vm_page *oldpg, s
uvm_pageremove_tree(uobj, oldpg);
uvm_pageinsert_tree(uobj, newpg);
- uvm_pageinsert_list(uobj, newpg, oldpg);
- uvm_pageremove_list(uobj, oldpg);
+ uvm_pageinsert_object(uobj, newpg);
+ uvm_pageremove_object(uobj, oldpg);
}
/*
@@ -1542,7 +1533,7 @@ uvm_pagelookup(struct uvm_object *obj, v
{
struct vm_page *pg;
- KASSERT(mutex_owned(obj->vmobjlock));
+ /* No - used from DDB. KASSERT(mutex_owned(obj->vmobjlock)); */
pg = radix_tree_lookup_node(&obj->uo_pages, off >> PAGE_SHIFT);
@@ -1841,11 +1832,7 @@ uvm_page_printit(struct vm_page *pg, boo
uobj = pg->uobject;
if (uobj) {
(*pr)(" checking object list\n");
- TAILQ_FOREACH(tpg, &uobj->memq, listq.queue) {
- if (tpg == pg) {
- break;
- }
- }
+ tpg = uvm_pagelookup(uobj, pg->offset);
if (tpg)
(*pr)(" page found on object list\n");
else
Index: src/sys/uvm/uvm_page.h
diff -u src/sys/uvm/uvm_page.h:1.86 src/sys/uvm/uvm_page.h:1.87
--- src/sys/uvm/uvm_page.h:1.86 Sat Dec 14 17:31:53 2019
+++ src/sys/uvm/uvm_page.h Sun Dec 15 21:11:35 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_page.h,v 1.86 2019/12/14 17:31:53 ad Exp $ */
+/* $NetBSD: uvm_page.h,v 1.87 2019/12/15 21:11:35 ad Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -125,7 +125,6 @@
* => PG_FREE is set in flags
* o owned by a uvm_object
* => pageq.queue is entry on wired page queue, if any
- * => listq.queue is entry on list of pages in object
* => uanon is NULL or the vm_anon to which it has been O->A loaned
* => uobject is owner
* o owned by a vm_anon
@@ -156,7 +155,6 @@ struct vm_page {
} pageq;
union {
- TAILQ_ENTRY(vm_page) queue; /* o: pages in same object */
LIST_ENTRY(vm_page) list; /* f: CPU free page queue */
} listq;
Added files:
Index: src/sys/uvm/uvm_page_array.c
diff -u /dev/null src/sys/uvm/uvm_page_array.c:1.2
--- /dev/null Sun Dec 15 21:11:35 2019
+++ src/sys/uvm/uvm_page_array.c Sun Dec 15 21:11:35 2019
@@ -0,0 +1,217 @@
+/* $NetBSD: uvm_page_array.c,v 1.2 2019/12/15 21:11:35 ad Exp $ */
+
+/*-
+ * Copyright (c)2011 YAMAMOTO Takashi,
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__KERNEL_RCSID(0, "$NetBSD: uvm_page_array.c,v 1.2 2019/12/15 21:11:35 ad Exp $");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+
+#include <uvm/uvm_extern.h>
+#include <uvm/uvm_object.h>
+#include <uvm/uvm_page.h>
+#include <uvm/uvm_page_array.h>
+
+/*
+ * uvm_page_array_init: initialize the array.
+ */
+
+void
+uvm_page_array_init(struct uvm_page_array *ar)
+{
+
+ ar->ar_idx = ar->ar_npages = 0;
+}
+
+/*
+ * uvm_page_array_fini: clean up the array.
+ */
+
+void
+uvm_page_array_fini(struct uvm_page_array *ar)
+{
+
+ /*
+ * currently nothing to do.
+ */
+#if defined(DIAGNOSTIC)
+ /*
+ * poison to trigger assertion in uvm_page_array_peek to
+ * detect usage errors.
+ */
+ ar->ar_npages = 1;
+ ar->ar_idx = 1000;
+#endif /* defined(DIAGNOSTIC) */
+}
+
+/*
+ * uvm_page_array_clear: forget the cached pages and initialize the array.
+ */
+
+void
+uvm_page_array_clear(struct uvm_page_array *ar)
+{
+
+ KASSERT(ar->ar_idx <= ar->ar_npages);
+ uvm_page_array_init(ar);
+}
+
+/*
+ * uvm_page_array_peek: return the next cached page.
+ */
+
+struct vm_page *
+uvm_page_array_peek(struct uvm_page_array *ar)
+{
+
+ KASSERT(ar->ar_idx <= ar->ar_npages);
+ if (ar->ar_idx == ar->ar_npages) {
+ return NULL;
+ }
+ return ar->ar_pages[ar->ar_idx];
+}
+
+/*
+ * uvm_page_array_advance: advance the array to the next cached page
+ */
+
+void
+uvm_page_array_advance(struct uvm_page_array *ar)
+{
+
+ KASSERT(ar->ar_idx <= ar->ar_npages);
+ ar->ar_idx++;
+ KASSERT(ar->ar_idx <= ar->ar_npages);
+}
+
+/*
+ * uvm_page_array_fill: lookup pages and keep them cached.
+ *
+ * return 0 on success. in that case, cache the result in the array
+ * so that they will be picked by later uvm_page_array_peek.
+ *
+ * nwant is a number of pages to fetch. a caller should consider it a hint.
+ * nwant == 0 means a caller have no specific idea.
+ *
+ * return ENOENT if no pages are found.
+ *
+ * called with object lock held.
+ */
+
+int
+uvm_page_array_fill(struct uvm_page_array *ar, struct uvm_object *uobj,
+ voff_t off, unsigned int nwant, unsigned int flags)
+{
+ unsigned int npages;
+#if defined(DEBUG)
+ unsigned int i;
+#endif /* defined(DEBUG) */
+ unsigned int maxpages = __arraycount(ar->ar_pages);
+ const bool dense = (flags & UVM_PAGE_ARRAY_FILL_DENSE) != 0;
+ const bool backward = (flags & UVM_PAGE_ARRAY_FILL_BACKWARD) != 0;
+
+ if (nwant != 0 && nwant < maxpages) {
+ maxpages = nwant;
+ }
+#if 0 /* called from DDB for "show obj/f" without lock */
+ KASSERT(mutex_owned(uobj->vmobjlock));
+#endif
+ KASSERT(uvm_page_array_peek(ar) == NULL);
+#if 0 /* not merged from yamt-pagecache yet */
+ if ((flags & UVM_PAGE_ARRAY_FILL_DIRTY) != 0) {
+ unsigned int tagmask = UVM_PAGE_DIRTY_TAG;
+
+ if ((flags & UVM_PAGE_ARRAY_FILL_WRITEBACK) != 0) {
+ tagmask |= UVM_PAGE_WRITEBACK_TAG;
+ }
+ npages =
+ (backward ? radix_tree_gang_lookup_tagged_node_reverse :
+ radix_tree_gang_lookup_tagged_node)(
+ &uobj->uo_pages, off >> PAGE_SHIFT, (void **)ar->ar_pages,
+ maxpages, dense, tagmask);
+ } else
+#endif
+ {
+ npages =
+ (backward ? radix_tree_gang_lookup_node_reverse :
+ radix_tree_gang_lookup_node)(
+ &uobj->uo_pages, off >> PAGE_SHIFT, (void **)ar->ar_pages,
+ maxpages, dense);
+ }
+ if (npages == 0) {
+ uvm_page_array_clear(ar);
+ return ENOENT;
+ }
+ KASSERT(npages <= maxpages);
+ ar->ar_npages = npages;
+ ar->ar_idx = 0;
+#if defined(DEBUG)
+ for (i = 0; i < ar->ar_npages; i++) {
+ struct vm_page * const pg = ar->ar_pages[i];
+
+ KDASSERT(pg != NULL);
+ KDASSERT(pg->uobject == uobj);
+ if (backward) {
+ KDASSERT(pg->offset <= off);
+ KDASSERT(i == 0 ||
+ pg->offset < ar->ar_pages[i - 1]->offset);
+ } else {
+ KDASSERT(pg->offset >= off);
+ KDASSERT(i == 0 ||
+ pg->offset > ar->ar_pages[i - 1]->offset);
+ }
+ }
+#endif /* defined(DEBUG) */
+ return 0;
+}
+
+/*
+ * uvm_page_array_fill_and_peek:
+ * same as uvm_page_array_peek except that, if the array is empty, try to fill
+ * it first.
+ */
+
+struct vm_page *
+uvm_page_array_fill_and_peek(struct uvm_page_array *a, struct uvm_object *uobj,
+ voff_t off, unsigned int nwant, unsigned int flags)
+{
+ struct vm_page *pg;
+ int error;
+
+ pg = uvm_page_array_peek(a);
+ if (pg != NULL) {
+ return pg;
+ }
+ error = uvm_page_array_fill(a, uobj, off, nwant, flags);
+ if (error != 0) {
+ return NULL;
+ }
+ pg = uvm_page_array_peek(a);
+ KASSERT(pg != NULL);
+ return pg;
+}
Index: src/sys/uvm/uvm_page_array.h
diff -u /dev/null src/sys/uvm/uvm_page_array.h:1.2
--- /dev/null Sun Dec 15 21:11:35 2019
+++ src/sys/uvm/uvm_page_array.h Sun Dec 15 21:11:35 2019
@@ -0,0 +1,79 @@
+/* $NetBSD: uvm_page_array.h,v 1.2 2019/12/15 21:11:35 ad Exp $ */
+
+/*-
+ * Copyright (c)2011 YAMAMOTO Takashi,
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#if !defined(_UVM_UVM_PAGE_ARRAY_H_)
+#define _UVM_UVM_PAGE_ARRAY_H_
+
+/*
+ * uvm_page_array: an array of pages.
+ *
+ * these structure and functions simply manipulate struct vm_page pointers.
+ * it's caller's responsibity to acquire and keep the object lock so that
+ * the result is valid.
+ *
+ * typical usage:
+ *
+ * struct uvm_page_array a;
+ *
+ * uvm_page_array_init(&a);
+ * while ((pg = uvm_page_array_fill_and_peek(&a, uobj, off, ....))
+ * != NULL) {
+ * off = pg->offset + PAGE_SIZE;
+ * do_something(pg);
+ * uvm_page_array_advance(&a);
+ * }
+ * uvm_page_array_fini(&a);
+ */
+
+struct vm_page;
+
+struct uvm_page_array {
+ struct vm_page *ar_pages[16]; /* XXX tune */
+ unsigned int ar_npages; /* valid elements in ar_pages */
+ unsigned int ar_idx; /* index in ar_pages */
+};
+
+void uvm_page_array_init(struct uvm_page_array *);
+void uvm_page_array_fini(struct uvm_page_array *);
+void uvm_page_array_clear(struct uvm_page_array *);
+struct vm_page *uvm_page_array_peek(struct uvm_page_array *);
+void uvm_page_array_advance(struct uvm_page_array *);
+int uvm_page_array_fill(struct uvm_page_array *, struct uvm_object *,
+ voff_t, unsigned int, unsigned int);
+struct vm_page *uvm_page_array_fill_and_peek(struct uvm_page_array *,
+ struct uvm_object *, voff_t, unsigned int, unsigned int);
+
+/*
+ * flags for uvm_page_array_fill and uvm_page_array_fill_and_peek
+ */
+#define UVM_PAGE_ARRAY_FILL_DIRTY 1 /* dirty pages */
+#define UVM_PAGE_ARRAY_FILL_WRITEBACK 2 /* dirty or written-back */
+#define UVM_PAGE_ARRAY_FILL_DENSE 4 /* stop on a hole */
+#define UVM_PAGE_ARRAY_FILL_BACKWARD 8 /* descend order */
+
+#endif /* defined(_UVM_UVM_ARRAY_H_) */
