Module Name: src
Committed By: matt
Date: Wed May 25 23:58:51 UTC 2011
Modified Files:
src/sys/arch/mips/mips [matt-nb5-mips64]: bus_dma.c
bus_space_alignstride_chipdep.c cache.c pmap.c vm_machdep.c
src/sys/kern [matt-nb5-mips64]: kern_malloc_debug.c kern_verifiedexec.c
sys_pipe.c uipc_socket.c
src/sys/miscfs/genfs [matt-nb5-mips64]: genfs_io.c
src/sys/sys [matt-nb5-mips64]: socketvar.h
src/sys/uvm [matt-nb5-mips64]: uvm_aobj.c uvm_extern.h uvm_fault.c
uvm_km.c uvm_map.c uvm_page.h uvm_pager.c uvm_pglist.c uvm_vnode.c
Log Message:
Make uvm_map recognize UVM_FLAG_COLORMATCH which tells uvm_map that the
'align' argument specifies the starting color of the KVA range to be returned.
When calling uvm_km_alloc with UVM_KMF_VAONLY, also specify the starting
color of the kva range returned (UMV_KMF_COLORMATCH) and pass those to
uvm_map.
In uvm_pglistalloc, make sure the pages being returned have sequentially
advancing colors (so they can be mapped in a contiguous address range).
Add a few missing UVM_FLAG_COLORMATCH flags to uvm_pagealloc calls.
Make the socket and pipe loan color-safe.
Make the mips pmap enforce strict page color (color(VA) == color(PA)).
To generate a diff of this commit:
cvs rdiff -u -r1.22.16.18 -r1.22.16.19 src/sys/arch/mips/mips/bus_dma.c
cvs rdiff -u -r1.10.18.12 -r1.10.18.13 \
src/sys/arch/mips/mips/bus_space_alignstride_chipdep.c
cvs rdiff -u -r1.33.96.4 -r1.33.96.5 src/sys/arch/mips/mips/cache.c
cvs rdiff -u -r1.179.16.28 -r1.179.16.29 src/sys/arch/mips/mips/pmap.c
cvs rdiff -u -r1.121.6.1.2.19 -r1.121.6.1.2.20 \
src/sys/arch/mips/mips/vm_machdep.c
cvs rdiff -u -r1.20 -r1.20.12.1 src/sys/kern/kern_malloc_debug.c
cvs rdiff -u -r1.111.4.1 -r1.111.4.1.4.1 src/sys/kern/kern_verifiedexec.c
cvs rdiff -u -r1.103.4.5 -r1.103.4.5.4.1 src/sys/kern/sys_pipe.c
cvs rdiff -u -r1.177.4.2.2.1.2.1 -r1.177.4.2.2.1.2.2 \
src/sys/kern/uipc_socket.c
cvs rdiff -u -r1.13.4.2.4.2 -r1.13.4.2.4.3 src/sys/miscfs/genfs/genfs_io.c
cvs rdiff -u -r1.116.4.2 -r1.116.4.2.4.1 src/sys/sys/socketvar.h
cvs rdiff -u -r1.104 -r1.104.10.1 src/sys/uvm/uvm_aobj.c
cvs rdiff -u -r1.148.4.2.4.1 -r1.148.4.2.4.2 src/sys/uvm/uvm_extern.h
cvs rdiff -u -r1.125.6.1.4.1 -r1.125.6.1.4.2 src/sys/uvm/uvm_fault.c
cvs rdiff -u -r1.101.4.2.4.3 -r1.101.4.2.4.4 src/sys/uvm/uvm_km.c
cvs rdiff -u -r1.263.4.3.4.3 -r1.263.4.3.4.4 src/sys/uvm/uvm_map.c
cvs rdiff -u -r1.55.14.2 -r1.55.14.3 src/sys/uvm/uvm_page.h
cvs rdiff -u -r1.92 -r1.92.18.1 src/sys/uvm/uvm_pager.c
cvs rdiff -u -r1.42.16.6 -r1.42.16.7 src/sys/uvm/uvm_pglist.c
cvs rdiff -u -r1.90 -r1.90.28.1 src/sys/uvm/uvm_vnode.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/mips/mips/bus_dma.c
diff -u src/sys/arch/mips/mips/bus_dma.c:1.22.16.18 src/sys/arch/mips/mips/bus_dma.c:1.22.16.19
--- src/sys/arch/mips/mips/bus_dma.c:1.22.16.18 Fri Apr 29 08:26:23 2011
+++ src/sys/arch/mips/mips/bus_dma.c Wed May 25 23:58:47 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: bus_dma.c,v 1.22.16.18 2011/04/29 08:26:23 matt Exp $ */
+/* bus_dma.c,v 1.22.16.18 2011/04/29 08:26:23 matt Exp */
/*-
* Copyright (c) 1997, 1998, 2001 The NetBSD Foundation, Inc.
@@ -32,7 +32,7 @@
#include <sys/cdefs.h> /* RCS ID & Copyright macro defns */
-__KERNEL_RCSID(0, "$NetBSD: bus_dma.c,v 1.22.16.18 2011/04/29 08:26:23 matt Exp $");
+__KERNEL_RCSID(0, "bus_dma.c,v 1.22.16.18 2011/04/29 08:26:23 matt Exp");
#include <sys/param.h>
#include <sys/systm.h>
@@ -934,6 +934,7 @@
" address 0x%"PRIxPADDR"\n", curaddr);
panic("_bus_dmamem_alloc");
}
+ KASSERT((atop(curaddr - lastaddr) & uvmexp.colormask) == 1);
#endif
if (curaddr == (lastaddr + PAGE_SIZE))
segs[curseg].ds_len += PAGE_SIZE;
@@ -1020,7 +1021,9 @@
size = round_page(size);
- va = uvm_km_alloc(kernel_map, size, 0, UVM_KMF_VAONLY | kmflags);
+ va = uvm_km_alloc(kernel_map, size,
+ atop(segs[0].ds_addr) & uvmexp.colormask,
+ UVM_KMF_COLORMATCH | UVM_KMF_VAONLY | kmflags);
if (va == 0)
return (ENOMEM);
Index: src/sys/arch/mips/mips/bus_space_alignstride_chipdep.c
diff -u src/sys/arch/mips/mips/bus_space_alignstride_chipdep.c:1.10.18.12 src/sys/arch/mips/mips/bus_space_alignstride_chipdep.c:1.10.18.13
--- src/sys/arch/mips/mips/bus_space_alignstride_chipdep.c:1.10.18.12 Thu Jan 14 05:02:38 2010
+++ src/sys/arch/mips/mips/bus_space_alignstride_chipdep.c Wed May 25 23:58:47 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: bus_space_alignstride_chipdep.c,v 1.10.18.12 2010/01/14 05:02:38 cliff Exp $ */
+/* bus_space_alignstride_chipdep.c,v 1.10.18.12 2010/01/14 05:02:38 cliff Exp */
/*-
* Copyright (c) 1998, 2000, 2001 The NetBSD Foundation, Inc.
@@ -81,7 +81,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: bus_space_alignstride_chipdep.c,v 1.10.18.12 2010/01/14 05:02:38 cliff Exp $");
+__KERNEL_RCSID(0, "bus_space_alignstride_chipdep.c,v 1.10.18.12 2010/01/14 05:02:38 cliff Exp");
#ifdef CHIP_EXTENT
#include <sys/extent.h>
@@ -759,8 +759,9 @@
int s;
size = round_page((addr % PAGE_SIZE) + size);
- va = uvm_km_alloc(kernel_map, size, PAGE_SIZE,
- UVM_KMF_VAONLY | UVM_KMF_NOWAIT);
+ va = uvm_km_alloc(kernel_map, size,
+ atop(addr) & uvmexp.colormask,
+ UVM_KMF_COLORMATCH | UVM_KMF_VAONLY | UVM_KMF_NOWAIT);
if (va == 0)
return ENOMEM;
Index: src/sys/arch/mips/mips/cache.c
diff -u src/sys/arch/mips/mips/cache.c:1.33.96.4 src/sys/arch/mips/mips/cache.c:1.33.96.5
--- src/sys/arch/mips/mips/cache.c:1.33.96.4 Fri Apr 29 08:26:23 2011
+++ src/sys/arch/mips/mips/cache.c Wed May 25 23:58:47 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: cache.c,v 1.33.96.4 2011/04/29 08:26:23 matt Exp $ */
+/* cache.c,v 1.33.96.4 2011/04/29 08:26:23 matt Exp */
/*
* Copyright 2001, 2002 Wasabi Systems, Inc.
@@ -68,7 +68,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cache.c,v 1.33.96.4 2011/04/29 08:26:23 matt Exp $");
+__KERNEL_RCSID(0, "cache.c,v 1.33.96.4 2011/04/29 08:26:23 matt Exp");
#include "opt_cputype.h"
#include "opt_mips_cache.h"
@@ -931,7 +931,7 @@
mci->mci_pdcache_size =
mci->mci_pdcache_way_size * mci->mci_pdcache_ways;
mci->mci_pdcache_way_mask = mci->mci_pdcache_way_size - 1;
- uvmexp.ncolors = atop(mci->mci_pdcache_size) / mci->mci_pdcache_ways;
+ uvmexp.ncolors = atop(mci->mci_pdcache_way_size);
break;
}
@@ -959,6 +959,8 @@
mci->mci_picache_size =
mci->mci_picache_way_size * mci->mci_picache_ways;
mci->mci_picache_way_mask = mci->mci_picache_way_size - 1;
+ if (uvmexp.ncolors < atop(mci->mci_picache_way_size))
+ uvmexp.ncolors = atop(mci->mci_picache_way_size);
break;
}
Index: src/sys/arch/mips/mips/pmap.c
diff -u src/sys/arch/mips/mips/pmap.c:1.179.16.28 src/sys/arch/mips/mips/pmap.c:1.179.16.29
--- src/sys/arch/mips/mips/pmap.c:1.179.16.28 Fri Apr 29 08:26:29 2011
+++ src/sys/arch/mips/mips/pmap.c Wed May 25 23:58:48 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.c,v 1.179.16.28 2011/04/29 08:26:29 matt Exp $ */
+/* pmap.c,v 1.179.16.28 2011/04/29 08:26:29 matt Exp */
/*-
* Copyright (c) 1998, 2001 The NetBSD Foundation, Inc.
@@ -67,7 +67,7 @@
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.179.16.28 2011/04/29 08:26:29 matt Exp $");
+__KERNEL_RCSID(0, "pmap.c,v 1.179.16.28 2011/04/29 08:26:29 matt Exp");
/*
* Manages physical address maps.
@@ -739,15 +739,17 @@
*/
if (mips_avail_end > MIPS_KSEG1_START - MIPS_KSEG0_START) {
curcpu()->ci_pmap_dstbase = uvm_km_alloc(kernel_map,
- uvmexp.ncolors * PAGE_SIZE, 0, UVM_KMF_VAONLY);
+ uvmexp.ncolors * PAGE_SIZE, 0,
+ UVM_KMF_COLORMATCH | UVM_KMF_VAONLY);
KASSERT(curcpu()->ci_pmap_dstbase);
curcpu()->ci_pmap_srcbase = uvm_km_alloc(kernel_map,
- uvmexp.ncolors * PAGE_SIZE, 0, UVM_KMF_VAONLY);
+ uvmexp.ncolors * PAGE_SIZE, 0,
+ UVM_KMF_COLORMATCH | UVM_KMF_VAONLY);
KASSERT(curcpu()->ci_pmap_srcbase);
}
#endif
-#ifdef MIPS3
+#if defined(MIPS3) && 0
if (MIPS_HAS_R4K_MMU) {
/*
* XXX
@@ -1361,9 +1363,12 @@
#ifdef DEBUG
if (pmapdebug & (PDB_FOLLOW|PDB_ENTER))
printf("pmap_enter(%p, %#"PRIxVADDR", %#"PRIxPADDR", %x, %x)\n",
- pmap, va, pa, prot, wired);
+ pmap, va, pa, prot, flags);
#endif
const bool good_color = PMAP_PAGE_COLOROK_P(pa, va);
+ KASSERTMSG(good_color,
+ ("%s(%p, %#"PRIxVADDR", %#"PRIxPADDR", %x, %x): color mismatch\n",
+ __func__, pmap, va, pa, prot, flags));
if (pmap == pmap_kernel()) {
PMAP_COUNT(kernel_mappings);
if (!good_color)
@@ -1622,8 +1627,12 @@
printf("pmap_kenter_pa(%#"PRIxVADDR", %#"PRIxPADDR", %x)\n", va, pa, prot);
#endif
PMAP_COUNT(kenter_pa);
- if (!PMAP_PAGE_COLOROK_P(pa, va) && managed)
+ KASSERTMSG(!managed || PMAP_PAGE_COLOROK_P(pa, va),
+ ("%s(%#"PRIxVADDR", %#"PRIxPADDR", %x): color mismatch\n",
+ __func__, va, pa, prot));
+ if (!PMAP_PAGE_COLOROK_P(pa, va) && managed) {
PMAP_COUNT(kenter_pa_bad);
+ }
if (!managed)
PMAP_COUNT(kenter_pa_unmanaged);
Index: src/sys/arch/mips/mips/vm_machdep.c
diff -u src/sys/arch/mips/mips/vm_machdep.c:1.121.6.1.2.19 src/sys/arch/mips/mips/vm_machdep.c:1.121.6.1.2.20
--- src/sys/arch/mips/mips/vm_machdep.c:1.121.6.1.2.19 Fri Apr 29 08:26:31 2011
+++ src/sys/arch/mips/mips/vm_machdep.c Wed May 25 23:58:48 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: vm_machdep.c,v 1.121.6.1.2.19 2011/04/29 08:26:31 matt Exp $ */
+/* vm_machdep.c,v 1.121.6.1.2.19 2011/04/29 08:26:31 matt Exp */
/*
* Copyright (c) 1988 University of Utah.
@@ -39,7 +39,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: vm_machdep.c,v 1.121.6.1.2.19 2011/04/29 08:26:31 matt Exp $");
+__KERNEL_RCSID(0, "vm_machdep.c,v 1.121.6.1.2.19 2011/04/29 08:26:31 matt Exp");
#include "opt_ddb.h"
#include "opt_coredump.h"
@@ -366,12 +366,6 @@
}
#endif
-static struct evcnt evcnt_vmapbuf =
- EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "vmapbuf", "calls");
-static struct evcnt evcnt_vmapbuf_adjustments =
- EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &evcnt_vmapbuf,
- "vmapbuf", "adjustments");
-
/*
* Map a user I/O request into kernel virtual address space.
*/
@@ -383,21 +377,15 @@
vaddr_t kva; /* Kernel VA (new to) */
paddr_t pa; /* physical address */
vsize_t off;
- vsize_t coloroff;
if ((bp->b_flags & B_PHYS) == 0)
panic("vmapbuf");
- evcnt_vmapbuf.ev_count++;
uva = mips_trunc_page(bp->b_saveaddr = bp->b_data);
- coloroff = uva & ptoa(uvmexp.colormask);
- if (coloroff)
- evcnt_vmapbuf_adjustments.ev_count++;
off = (vaddr_t)bp->b_data - uva;
len = mips_round_page(off + len);
- kva = uvm_km_alloc(phys_map, len + coloroff, ptoa(uvmexp.ncolors),
- UVM_KMF_VAONLY | UVM_KMF_WAITVA);
- kva += coloroff;
+ kva = uvm_km_alloc(phys_map, len, atop(uva) & uvmexp.colormask,
+ UVM_FLAG_COLORMATCH | UVM_KMF_VAONLY | UVM_KMF_WAITVA);
bp->b_data = (void *)(kva + off);
upmap = vm_map_pmap(&bp->b_proc->p_vmspace->vm_map);
do {
@@ -420,18 +408,16 @@
{
vaddr_t kva;
vsize_t off;
- vsize_t coloroff;
if ((bp->b_flags & B_PHYS) == 0)
panic("vunmapbuf");
kva = mips_trunc_page(bp->b_data);
- coloroff = kva & ptoa(uvmexp.colormask);
off = (vaddr_t)bp->b_data - kva;
len = mips_round_page(off + len);
pmap_remove(vm_map_pmap(phys_map), kva, kva + len);
pmap_update(pmap_kernel());
- uvm_km_free(phys_map, kva - coloroff, len + coloroff, UVM_KMF_VAONLY);
+ uvm_km_free(phys_map, kva, len, UVM_KMF_VAONLY);
bp->b_data = bp->b_saveaddr;
bp->b_saveaddr = NULL;
}
Index: src/sys/kern/kern_malloc_debug.c
diff -u src/sys/kern/kern_malloc_debug.c:1.20 src/sys/kern/kern_malloc_debug.c:1.20.12.1
--- src/sys/kern/kern_malloc_debug.c:1.20 Thu Aug 7 01:40:21 2008
+++ src/sys/kern/kern_malloc_debug.c Wed May 25 23:58:48 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: kern_malloc_debug.c,v 1.20 2008/08/07 01:40:21 matt Exp $ */
+/* kern_malloc_debug.c,v 1.20 2008/08/07 01:40:21 matt Exp */
/*
* Copyright (c) 1999, 2000 Artur Grabowski <[email protected]>
@@ -56,7 +56,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_malloc_debug.c,v 1.20 2008/08/07 01:40:21 matt Exp $");
+__KERNEL_RCSID(0, "kern_malloc_debug.c,v 1.20 2008/08/07 01:40:21 matt Exp");
#include <sys/param.h>
#include <sys/proc.h>
@@ -240,7 +240,7 @@
offset = va - vm_map_min(kernel_map);
for (;;) {
- pg = uvm_pagealloc(NULL, offset, NULL, 0);
+ pg = uvm_pagealloc(NULL, offset, NULL, UVM_FLAG_COLORMATCH);
if (pg) {
pg->flags &= ~PG_BUSY; /* new page */
UVM_PAGE_OWN(pg, NULL);
Index: src/sys/kern/kern_verifiedexec.c
diff -u src/sys/kern/kern_verifiedexec.c:1.111.4.1 src/sys/kern/kern_verifiedexec.c:1.111.4.1.4.1
--- src/sys/kern/kern_verifiedexec.c:1.111.4.1 Thu Dec 18 00:56:27 2008
+++ src/sys/kern/kern_verifiedexec.c Wed May 25 23:58:48 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: kern_verifiedexec.c,v 1.111.4.1 2008/12/18 00:56:27 snj Exp $ */
+/* kern_verifiedexec.c,v 1.111.4.1 2008/12/18 00:56:27 snj Exp */
/*-
* Copyright (c) 2005, 2006 Elad Efrat <[email protected]>
@@ -29,7 +29,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_verifiedexec.c,v 1.111.4.1 2008/12/18 00:56:27 snj Exp $");
+__KERNEL_RCSID(0, "kern_verifiedexec.c,v 1.111.4.1 2008/12/18 00:56:27 snj Exp");
#include "opt_veriexec.h"
@@ -765,7 +765,8 @@
ctx = kmem_alloc(vfe->ops->context_size, KM_SLEEP);
fp = kmem_alloc(vfe->ops->hash_len, KM_SLEEP);
- kva = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_VAONLY | UVM_KMF_WAITVA);
+ kva = uvm_km_alloc(kernel_map, PAGE_SIZE, VM_PAGE_TO_COLOR(pg),
+ UVM_KMF_COLORMATCH | UVM_KMF_VAONLY | UVM_KMF_WAITVA);
pmap_kenter_pa(kva, VM_PAGE_TO_PHYS(pg), VM_PROT_READ);
pmap_update(pmap_kernel());
Index: src/sys/kern/sys_pipe.c
diff -u src/sys/kern/sys_pipe.c:1.103.4.5 src/sys/kern/sys_pipe.c:1.103.4.5.4.1
--- src/sys/kern/sys_pipe.c:1.103.4.5 Sat Apr 4 23:36:27 2009
+++ src/sys/kern/sys_pipe.c Wed May 25 23:58:48 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: sys_pipe.c,v 1.103.4.5 2009/04/04 23:36:27 snj Exp $ */
+/* sys_pipe.c,v 1.103.4.5 2009/04/04 23:36:27 snj Exp */
/*-
* Copyright (c) 2003, 2007, 2008, 2009 The NetBSD Foundation, Inc.
@@ -68,7 +68,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: sys_pipe.c,v 1.103.4.5 2009/04/04 23:36:27 snj Exp $");
+__KERNEL_RCSID(0, "sys_pipe.c,v 1.103.4.5 2009/04/04 23:36:27 snj Exp");
#include <sys/param.h>
#include <sys/systm.h>
@@ -508,6 +508,7 @@
#ifndef PIPE_NODIRECT
if ((rpipe->pipe_state & PIPE_DIRECTR) != 0) {
+ struct pipemapping * const rmap = &rpipe->pipe_map;
/*
* Direct copy, bypassing a kernel buffer.
*/
@@ -515,20 +516,20 @@
KASSERT(rpipe->pipe_state & PIPE_DIRECTW);
- size = rpipe->pipe_map.cnt;
+ size = rmap->cnt;
if (size > uio->uio_resid)
size = uio->uio_resid;
- va = (char *)rpipe->pipe_map.kva + rpipe->pipe_map.pos;
+ va = (char *)rmap->kva + rmap->koff + rmap->pos;
mutex_exit(lock);
error = uiomove(va, size, uio);
mutex_enter(lock);
if (error)
break;
nread += size;
- rpipe->pipe_map.pos += size;
- rpipe->pipe_map.cnt -= size;
- if (rpipe->pipe_map.cnt == 0) {
+ rmap->pos += size;
+ rmap->cnt -= size;
+ if (rmap->cnt == 0) {
rpipe->pipe_state &= ~PIPE_DIRECTR;
cv_broadcast(&rpipe->pipe_wcv);
}
@@ -621,20 +622,19 @@
static int
pipe_loan_alloc(struct pipe *wpipe, int npages)
{
- vsize_t len;
+ struct pipemapping * const wmap = &wpipe->pipe_map;
+ const vsize_t len = ptoa(npages);
- len = (vsize_t)npages << PAGE_SHIFT;
atomic_add_int(&amountpipekva, len);
- wpipe->pipe_map.kva = uvm_km_alloc(kernel_map, len, 0,
- UVM_KMF_VAONLY | UVM_KMF_WAITVA);
- if (wpipe->pipe_map.kva == 0) {
+ wmap->kva = uvm_km_alloc(kernel_map, len, 0,
+ UVM_KMF_COLORMATCH | UVM_KMF_VAONLY | UVM_KMF_WAITVA);
+ if (wmap->kva == 0) {
atomic_add_int(&amountpipekva, -len);
return (ENOMEM);
}
- wpipe->pipe_map.npages = npages;
- wpipe->pipe_map.pgs = kmem_alloc(npages * sizeof(struct vm_page *),
- KM_SLEEP);
+ wmap->npages = npages;
+ wmap->pgs = kmem_alloc(npages * sizeof(struct vm_page *), KM_SLEEP);
return (0);
}
@@ -644,15 +644,19 @@
static void
pipe_loan_free(struct pipe *wpipe)
{
- vsize_t len;
+ struct pipemapping * const wmap = &wpipe->pipe_map;
+ const vsize_t len = ptoa(wmap->npages);
- len = (vsize_t)wpipe->pipe_map.npages << PAGE_SHIFT;
- uvm_km_free(kernel_map, wpipe->pipe_map.kva, len, UVM_KMF_VAONLY);
- wpipe->pipe_map.kva = 0;
+ uvm_km_free(kernel_map, wmap->kva, len, UVM_KMF_VAONLY);
+ wmap->kva = 0;
atomic_add_int(&amountpipekva, -len);
- kmem_free(wpipe->pipe_map.pgs,
- wpipe->pipe_map.npages * sizeof(struct vm_page *));
- wpipe->pipe_map.pgs = NULL;
+ kmem_free(wmap->pgs, wmap->npages * sizeof(struct vm_page *));
+ wmap->pgs = NULL;
+#if 0
+ wmap->npages = 0;
+ wmap->pos = 0;
+ wmap->cnt = 0;
+#endif
}
/*
@@ -668,15 +672,17 @@
static int
pipe_direct_write(struct file *fp, struct pipe *wpipe, struct uio *uio)
{
+ struct pipemapping * const wmap = &wpipe->pipe_map;
+ kmutex_t * const lock = wpipe->pipe_lock;
int error, npages, j;
struct vm_page **pgs;
vaddr_t bbase, kva, base, bend;
vsize_t blen, bcnt;
voff_t bpos;
- kmutex_t *lock = wpipe->pipe_lock;
+ u_int starting_color;
KASSERT(mutex_owned(wpipe->pipe_lock));
- KASSERT(wpipe->pipe_map.cnt == 0);
+ KASSERT(wmap->cnt == 0);
mutex_exit(lock);
@@ -697,18 +703,19 @@
} else {
bcnt = uio->uio_iov->iov_len;
}
- npages = blen >> PAGE_SHIFT;
+ npages = atop(blen);
+ starting_color = atop(base) & uvmexp.colormask;
/*
* Free the old kva if we need more pages than we have
* allocated.
*/
- if (wpipe->pipe_map.kva != 0 && npages > wpipe->pipe_map.npages)
+ if (wmap->kva != 0 && starting_color + npages > wmap->npages)
pipe_loan_free(wpipe);
/* Allocate new kva. */
- if (wpipe->pipe_map.kva == 0) {
- error = pipe_loan_alloc(wpipe, npages);
+ if (wmap->kva == 0) {
+ error = pipe_loan_alloc(wpipe, starting_color + npages);
if (error) {
mutex_enter(lock);
return (error);
@@ -716,7 +723,7 @@
}
/* Loan the write buffer memory from writer process */
- pgs = wpipe->pipe_map.pgs;
+ pgs = wmap->pgs + starting_color;
error = uvm_loan(&uio->uio_vmspace->vm_map, base, blen,
pgs, UVM_LOAN_TOPAGE);
if (error) {
@@ -726,15 +733,15 @@
}
/* Enter the loaned pages to kva */
- kva = wpipe->pipe_map.kva;
+ kva = wmap->kva + ptoa(starting_color);
for (j = 0; j < npages; j++, kva += PAGE_SIZE) {
pmap_kenter_pa(kva, VM_PAGE_TO_PHYS(pgs[j]), VM_PROT_READ);
}
pmap_update(pmap_kernel());
/* Now we can put the pipe in direct write mode */
- wpipe->pipe_map.pos = bpos;
- wpipe->pipe_map.cnt = bcnt;
+ wmap->pos = bpos + ptoa(starting_color);
+ wmap->cnt = bcnt;
/*
* But before we can let someone do a direct read, we
@@ -772,7 +779,7 @@
mutex_exit(lock);
if (pgs != NULL) {
- pmap_kremove(wpipe->pipe_map.kva, blen);
+ pmap_kremove(wmap->kva, blen);
pmap_update(pmap_kernel());
uvm_unloan(pgs, npages, UVM_LOAN_TOPAGE);
}
@@ -789,13 +796,13 @@
* will deal with the error condition, returning short
* write, error, or restarting the write(2) as appropriate.
*/
- if (wpipe->pipe_map.cnt == bcnt) {
- wpipe->pipe_map.cnt = 0;
+ if (wmap->cnt == bcnt) {
+ wmap->cnt = 0;
cv_broadcast(&wpipe->pipe_wcv);
return (error);
}
- bcnt -= wpipe->pipe_map.cnt;
+ bcnt -= wpipe->cnt;
}
uio->uio_resid -= bcnt;
@@ -807,7 +814,7 @@
uio->uio_iovcnt--;
}
- wpipe->pipe_map.cnt = 0;
+ wmap->cnt = 0;
return (error);
}
#endif /* !PIPE_NODIRECT */
@@ -1240,7 +1247,6 @@
if (pipe->pipe_map.kva != 0) {
pipe_loan_free(pipe);
pipe->pipe_map.cnt = 0;
- pipe->pipe_map.kva = 0;
pipe->pipe_map.pos = 0;
pipe->pipe_map.npages = 0;
}
Index: src/sys/kern/uipc_socket.c
diff -u src/sys/kern/uipc_socket.c:1.177.4.2.2.1.2.1 src/sys/kern/uipc_socket.c:1.177.4.2.2.1.2.2
--- src/sys/kern/uipc_socket.c:1.177.4.2.2.1.2.1 Wed Apr 21 00:28:18 2010
+++ src/sys/kern/uipc_socket.c Wed May 25 23:58:48 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: uipc_socket.c,v 1.177.4.2.2.1.2.1 2010/04/21 00:28:18 matt Exp $ */
+/* uipc_socket.c,v 1.177.4.2.2.1.2.1 2010/04/21 00:28:18 matt Exp */
/*-
* Copyright (c) 2002, 2007, 2008, 2009 The NetBSD Foundation, Inc.
@@ -63,7 +63,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uipc_socket.c,v 1.177.4.2.2.1.2.1 2010/04/21 00:28:18 matt Exp $");
+__KERNEL_RCSID(0, "uipc_socket.c,v 1.177.4.2.2.1.2.1 2010/04/21 00:28:18 matt Exp");
#include "opt_sock_counters.h"
#include "opt_sosend_loan.h"
@@ -199,7 +199,7 @@
*/
vaddr_t
-sokvaalloc(vsize_t len, struct socket *so)
+sokvaalloc(vaddr_t sva, vsize_t len, struct socket *so)
{
vaddr_t lva;
@@ -214,7 +214,8 @@
* allocate kva.
*/
- lva = uvm_km_alloc(kernel_map, len, 0, UVM_KMF_VAONLY | UVM_KMF_WAITVA);
+ lva = uvm_km_alloc(kernel_map, len, atop(sva) & uvmexp.colormask,
+ UVM_KMF_COLORMATCH | UVM_KMF_VAONLY | UVM_KMF_WAITVA);
if (lva == 0) {
sokvaunreserve(len);
return (0);
@@ -362,7 +363,7 @@
KASSERT(npgs <= M_EXT_MAXPAGES);
- lva = sokvaalloc(len, so);
+ lva = sokvaalloc(sva, len, so);
if (lva == 0)
return 0;
Index: src/sys/miscfs/genfs/genfs_io.c
diff -u src/sys/miscfs/genfs/genfs_io.c:1.13.4.2.4.2 src/sys/miscfs/genfs/genfs_io.c:1.13.4.2.4.3
--- src/sys/miscfs/genfs/genfs_io.c:1.13.4.2.4.2 Fri May 20 08:11:28 2011
+++ src/sys/miscfs/genfs/genfs_io.c Wed May 25 23:58:48 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: genfs_io.c,v 1.13.4.2.4.2 2011/05/20 08:11:28 matt Exp $ */
+/* genfs_io.c,v 1.13.4.2.4.2 2011/05/20 08:11:28 matt Exp */
/*
* Copyright (c) 1982, 1986, 1989, 1993
@@ -31,7 +31,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: genfs_io.c,v 1.13.4.2.4.2 2011/05/20 08:11:28 matt Exp $");
+__KERNEL_RCSID(0, "genfs_io.c,v 1.13.4.2.4.2 2011/05/20 08:11:28 matt Exp");
#include <sys/param.h>
#include <sys/systm.h>
@@ -1746,9 +1746,11 @@
map = &vs->vm_map;
upm = vm_map_pmap(map);
kpm = vm_map_pmap(kernel_map);
- kva = uvm_km_alloc(kernel_map, klen, 0,
- UVM_KMF_VAONLY | UVM_KMF_WAITVA);
puva = trunc_page(uva);
+
+ kva = uvm_km_alloc(kernel_map, klen, atop(puva) & uvmexp.colormask,
+ UVM_KMF_COLORMATCH | UVM_KMF_VAONLY | UVM_KMF_WAITVA);
+
for (poff = 0; poff < klen; poff += PAGE_SIZE) {
rv = pmap_extract(upm, puva + poff, &pa);
KASSERT(rv);
Index: src/sys/sys/socketvar.h
diff -u src/sys/sys/socketvar.h:1.116.4.2 src/sys/sys/socketvar.h:1.116.4.2.4.1
--- src/sys/sys/socketvar.h:1.116.4.2 Sat Apr 4 23:36:28 2009
+++ src/sys/sys/socketvar.h Wed May 25 23:58:49 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: socketvar.h,v 1.116.4.2 2009/04/04 23:36:28 snj Exp $ */
+/* socketvar.h,v 1.116.4.2 2009/04/04 23:36:28 snj Exp */
/*-
* Copyright (c) 2008, 2009 The NetBSD Foundation, Inc.
@@ -505,7 +505,7 @@
#endif /* SOCKBUF_DEBUG */
/* sosend loan */
-vaddr_t sokvaalloc(vsize_t, struct socket *);
+vaddr_t sokvaalloc(vaddr_t, vsize_t, struct socket *);
void sokvafree(vaddr_t, vsize_t);
void soloanfree(struct mbuf *, void *, size_t, void *);
Index: src/sys/uvm/uvm_aobj.c
diff -u src/sys/uvm/uvm_aobj.c:1.104 src/sys/uvm/uvm_aobj.c:1.104.10.1
--- src/sys/uvm/uvm_aobj.c:1.104 Sat Oct 18 03:46:22 2008
+++ src/sys/uvm/uvm_aobj.c Wed May 25 23:58:49 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_aobj.c,v 1.104 2008/10/18 03:46:22 rmind Exp $ */
+/* uvm_aobj.c,v 1.104 2008/10/18 03:46:22 rmind Exp */
/*
* Copyright (c) 1998 Chuck Silvers, Charles D. Cranor and
@@ -43,7 +43,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_aobj.c,v 1.104 2008/10/18 03:46:22 rmind Exp $");
+__KERNEL_RCSID(0, "uvm_aobj.c,v 1.104 2008/10/18 03:46:22 rmind Exp");
#include "opt_uvmhist.h"
@@ -976,7 +976,7 @@
if (ptmp == NULL && uao_find_swslot(&aobj->u_obj,
current_offset >> PAGE_SHIFT) == 0) {
ptmp = uvm_pagealloc(uobj, current_offset,
- NULL, UVM_PGA_ZERO);
+ NULL, UVM_FLAG_COLORMATCH|UVM_PGA_ZERO);
if (ptmp) {
/* new page */
ptmp->flags &= ~(PG_FAKE);
Index: src/sys/uvm/uvm_extern.h
diff -u src/sys/uvm/uvm_extern.h:1.148.4.2.4.1 src/sys/uvm/uvm_extern.h:1.148.4.2.4.2
--- src/sys/uvm/uvm_extern.h:1.148.4.2.4.1 Tue Jan 26 21:26:28 2010
+++ src/sys/uvm/uvm_extern.h Wed May 25 23:58:49 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_extern.h,v 1.148.4.2.4.1 2010/01/26 21:26:28 matt Exp $ */
+/* uvm_extern.h,v 1.148.4.2.4.1 2010/01/26 21:26:28 matt Exp */
/*
*
@@ -173,6 +173,7 @@
#define UVM_KMF_TRYLOCK UVM_FLAG_TRYLOCK /* try locking only */
#define UVM_KMF_NOWAIT UVM_FLAG_NOWAIT /* not allowed to sleep */
#define UVM_KMF_WAITVA UVM_FLAG_WAITVA /* sleep for va */
+#define UVM_KMF_COLORMATCH UVM_FLAG_COLORMATCH /* start at color in align */
/*
* the following defines the strategies for uvm_pagealloc_strat()
Index: src/sys/uvm/uvm_fault.c
diff -u src/sys/uvm/uvm_fault.c:1.125.6.1.4.1 src/sys/uvm/uvm_fault.c:1.125.6.1.4.2
--- src/sys/uvm/uvm_fault.c:1.125.6.1.4.1 Tue Jan 26 21:26:28 2010
+++ src/sys/uvm/uvm_fault.c Wed May 25 23:58:49 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_fault.c,v 1.125.6.1.4.1 2010/01/26 21:26:28 matt Exp $ */
+/* uvm_fault.c,v 1.125.6.1.4.1 2010/01/26 21:26:28 matt Exp */
/*
*
@@ -39,7 +39,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_fault.c,v 1.125.6.1.4.1 2010/01/26 21:26:28 matt Exp $");
+__KERNEL_RCSID(0, "uvm_fault.c,v 1.125.6.1.4.1 2010/01/26 21:26:28 matt Exp");
#include "opt_uvmhist.h"
@@ -361,7 +361,7 @@
/*
* no page, we must try and bring it in.
*/
-
+ KASSERT(ufi != NULL);
pg = uvm_pagealloc(NULL, ufi->orig_rvaddr,
NULL, UVM_FLAG_COLORMATCH);
if (pg == NULL) { /* out of RAM. */
Index: src/sys/uvm/uvm_km.c
diff -u src/sys/uvm/uvm_km.c:1.101.4.2.4.3 src/sys/uvm/uvm_km.c:1.101.4.2.4.4
--- src/sys/uvm/uvm_km.c:1.101.4.2.4.3 Sat Feb 6 05:28:30 2010
+++ src/sys/uvm/uvm_km.c Wed May 25 23:58:50 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_km.c,v 1.101.4.2.4.3 2010/02/06 05:28:30 matt Exp $ */
+/* uvm_km.c,v 1.101.4.2.4.3 2010/02/06 05:28:30 matt Exp */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -128,7 +128,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_km.c,v 1.101.4.2.4.3 2010/02/06 05:28:30 matt Exp $");
+__KERNEL_RCSID(0, "uvm_km.c,v 1.101.4.2.4.3 2010/02/06 05:28:30 matt Exp");
#include "opt_uvmhist.h"
@@ -541,6 +541,8 @@
KASSERT((flags & UVM_KMF_TYPEMASK) == UVM_KMF_WIRED ||
(flags & UVM_KMF_TYPEMASK) == UVM_KMF_PAGEABLE ||
(flags & UVM_KMF_TYPEMASK) == UVM_KMF_VAONLY);
+ KASSERT((flags & UVM_KMF_VAONLY) != 0 || (flags & UVM_KMF_COLORMATCH) == 0);
+ KASSERT((flags & UVM_KMF_COLORMATCH) == 0 || (flags & UVM_KMF_VAONLY) != 0);
/*
* setup for call
@@ -559,7 +561,7 @@
if (__predict_false(uvm_map(map, &kva, size, obj, UVM_UNKNOWN_OFFSET,
align, UVM_MAPFLAG(UVM_PROT_ALL, UVM_PROT_ALL, UVM_INH_NONE,
UVM_ADV_RANDOM,
- (flags & (UVM_KMF_TRYLOCK | UVM_KMF_NOWAIT | UVM_KMF_WAITVA))
+ (flags & (UVM_KMF_TRYLOCK | UVM_KMF_NOWAIT | UVM_KMF_WAITVA | UVM_KMF_COLORMATCH))
| UVM_FLAG_QUANTUM)) != 0)) {
UVMHIST_LOG(maphist, "<- done (no VM)",0,0,0,0);
return(0);
Index: src/sys/uvm/uvm_map.c
diff -u src/sys/uvm/uvm_map.c:1.263.4.3.4.3 src/sys/uvm/uvm_map.c:1.263.4.3.4.4
--- src/sys/uvm/uvm_map.c:1.263.4.3.4.3 Thu Aug 19 07:30:31 2010
+++ src/sys/uvm/uvm_map.c Wed May 25 23:58:50 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_map.c,v 1.263.4.3.4.3 2010/08/19 07:30:31 matt Exp $ */
+/* uvm_map.c,v 1.263.4.3.4.3 2010/08/19 07:30:31 matt Exp */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -71,7 +71,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_map.c,v 1.263.4.3.4.3 2010/08/19 07:30:31 matt Exp $");
+__KERNEL_RCSID(0, "uvm_map.c,v 1.263.4.3.4.3 2010/08/19 07:30:31 matt Exp");
#include "opt_ddb.h"
#include "opt_uvmhist.h"
@@ -304,7 +304,7 @@
static void uvm_map_entry_unwire(struct vm_map *, struct vm_map_entry *);
static void uvm_map_reference_amap(struct vm_map_entry *, int);
static int uvm_map_space_avail(vaddr_t *, vsize_t, voff_t, vsize_t, int,
- struct vm_map_entry *);
+ int, struct vm_map_entry *);
static void uvm_map_unreference_amap(struct vm_map_entry *, int);
int _uvm_map_sanity(struct vm_map *);
@@ -1852,7 +1852,7 @@
*/
static int
uvm_map_space_avail(vaddr_t *start, vsize_t length, voff_t uoffset,
- vsize_t align, int topdown, struct vm_map_entry *entry)
+ vsize_t align, int flags, int topdown, struct vm_map_entry *entry)
{
vaddr_t end;
@@ -1865,7 +1865,27 @@
if (uoffset != UVM_UNKNOWN_OFFSET)
PMAP_PREFER(uoffset, start, length, topdown);
#endif
- if (align != 0) {
+ if ((flags & UVM_FLAG_COLORMATCH) != 0) {
+ KASSERT(align < uvmexp.ncolors);
+ if (uvmexp.ncolors > 1) {
+ const u_int colormask = uvmexp.colormask;
+ const u_int colorsize = colormask + 1;
+ vaddr_t hint = atop(*start);
+ const u_int color = hint & colormask;
+ if (color != align) {
+ hint -= color; /* adjust to color boundary */
+ KASSERT((hint & colormask) == 0);
+ if (topdown) {
+ if (align > color)
+ hint -= colorsize;
+ } else {
+ if (align < color)
+ hint += colorsize;
+ }
+ *start = ptoa(hint + align); /* adjust to color */
+ }
+ }
+ } else if (align != 0) {
if ((*start & (align - 1)) != 0) {
if (topdown)
*start &= ~(align - 1);
@@ -1921,7 +1941,8 @@
UVMHIST_LOG(maphist, "(map=0x%x, hint=0x%x, len=%d, flags=0x%x)",
map, hint, length, flags);
- KASSERT((align & (align - 1)) == 0);
+ KASSERT((flags & UVM_FLAG_COLORMATCH) != 0 || (align & (align - 1)) == 0);
+ KASSERT((flags & UVM_FLAG_COLORMATCH) == 0 || align < uvmexp.ncolors);
KASSERT((flags & UVM_FLAG_FIXED) == 0 || align == 0);
uvm_map_check(map, "map_findspace entry");
@@ -1999,7 +2020,7 @@
* See if given hint fits in this gap.
*/
switch (uvm_map_space_avail(&hint, length,
- uoffset, align, topdown, entry)) {
+ uoffset, align, flags, topdown, entry)) {
case 1:
goto found;
case -1:
@@ -2030,7 +2051,7 @@
/* Check slot before any entry */
hint = topdown ? entry->next->start - length : entry->end;
- switch (uvm_map_space_avail(&hint, length, uoffset, align,
+ switch (uvm_map_space_avail(&hint, length, uoffset, align, flags,
topdown, entry)) {
case 1:
goto found;
@@ -2099,7 +2120,7 @@
hint = tmp->end;
}
switch (uvm_map_space_avail(&hint, length, uoffset, align,
- topdown, tmp)) {
+ flags, topdown, tmp)) {
case 1:
entry = tmp;
goto found;
@@ -2121,7 +2142,7 @@
hint = prev->end;
}
switch (uvm_map_space_avail(&hint, length, uoffset, align,
- topdown, prev)) {
+ flags, topdown, prev)) {
case 1:
entry = prev;
goto found;
@@ -2162,7 +2183,7 @@
hint = tmp->end;
}
switch (uvm_map_space_avail(&hint, length, uoffset, align,
- topdown, tmp)) {
+ flags, topdown, tmp)) {
case 1:
entry = tmp;
goto found;
@@ -2188,7 +2209,7 @@
/* See if it fits. */
switch (uvm_map_space_avail(&hint, length, uoffset, align,
- topdown, entry)) {
+ flags, topdown, entry)) {
case 1:
goto found;
case -1:
@@ -4639,7 +4660,7 @@
}
error = uvm_map_prepare(map, 0, PAGE_SIZE, NULL, UVM_UNKNOWN_OFFSET,
- 0, mapflags, &args);
+ VM_PAGE_TO_COLOR(pg), mapflags | UVM_FLAG_COLORMATCH, &args);
if (error) {
uvm_pagefree(pg);
return NULL;
Index: src/sys/uvm/uvm_page.h
diff -u src/sys/uvm/uvm_page.h:1.55.14.2 src/sys/uvm/uvm_page.h:1.55.14.3
--- src/sys/uvm/uvm_page.h:1.55.14.2 Fri Apr 29 08:16:42 2011
+++ src/sys/uvm/uvm_page.h Wed May 25 23:58:50 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_page.h,v 1.55.14.2 2011/04/29 08:16:42 matt Exp $ */
+/* uvm_page.h,v 1.55.14.2 2011/04/29 08:16:42 matt Exp */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -307,6 +307,7 @@
(((unsigned long)obj+(unsigned long)atop(off)) & uvm.page_hashmask)
#define VM_PAGE_TO_PHYS(entry) ((entry)->phys_addr)
+#define VM_PAGE_TO_COLOR(entry) (atop((entry)->phys_addr) & uvmexp.colormask)
#ifdef __HAVE_VM_PAGE_MD
#define VM_PAGE_TO_MD(pg) (&(pg)->mdpage)
Index: src/sys/uvm/uvm_pager.c
diff -u src/sys/uvm/uvm_pager.c:1.92 src/sys/uvm/uvm_pager.c:1.92.18.1
--- src/sys/uvm/uvm_pager.c:1.92 Thu Apr 17 05:39:41 2008
+++ src/sys/uvm/uvm_pager.c Wed May 25 23:58:50 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_pager.c,v 1.92 2008/04/17 05:39:41 simonb Exp $ */
+/* uvm_pager.c,v 1.92 2008/04/17 05:39:41 simonb Exp */
/*
*
@@ -39,7 +39,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_pager.c,v 1.92 2008/04/17 05:39:41 simonb Exp $");
+__KERNEL_RCSID(0, "uvm_pager.c,v 1.92 2008/04/17 05:39:41 simonb Exp");
#include "opt_uvmhist.h"
#include "opt_readahead.h"
@@ -112,7 +112,8 @@
false, NULL);
mutex_init(&pager_map_wanted_lock, MUTEX_DEFAULT, IPL_NONE);
pager_map_wanted = false;
- emergva = uvm_km_alloc(kernel_map, round_page(MAXPHYS), 0,
+ emergva = uvm_km_alloc(kernel_map,
+ round_page(MAXPHYS) + ptoa(uvmexp.ncolors), 0,
UVM_KMF_VAONLY);
#if defined(DEBUG)
if (emergva == 0)
@@ -150,10 +151,12 @@
vaddr_t cva;
struct vm_page *pp;
vm_prot_t prot;
- const bool pdaemon = curlwp == uvm.pagedaemon_lwp;
+ const bool pdaemon = (curlwp == uvm.pagedaemon_lwp);
+ const u_int first_color = VM_PAGE_TO_COLOR(*pps);
UVMHIST_FUNC("uvm_pagermapin"); UVMHIST_CALLED(maphist);
- UVMHIST_LOG(maphist,"(pps=0x%x, npages=%d)", pps, npages,0,0);
+ UVMHIST_LOG(maphist,"(pps=0x%x, npages=%d, first_color=%u)",
+ pps, npages, first_color, 0);
/*
* compute protection. outgoing I/O only needs read
@@ -165,11 +168,12 @@
prot |= VM_PROT_WRITE;
ReStart:
- size = npages << PAGE_SHIFT;
+ size = ptoa(npages);
kva = 0; /* let system choose VA */
- if (uvm_map(pager_map, &kva, size, NULL, UVM_UNKNOWN_OFFSET, 0,
- UVM_FLAG_NOMERGE | (pdaemon ? UVM_FLAG_NOWAIT : 0)) != 0) {
+ if (uvm_map(pager_map, &kva, size, NULL, UVM_UNKNOWN_OFFSET,
+ first_color, UVM_FLAG_COLORMATCH | UVM_FLAG_NOMERGE
+ | (pdaemon ? UVM_FLAG_NOWAIT : 0)) != 0) {
if (pdaemon) {
mutex_enter(&pager_map_wanted_lock);
if (emerginuse) {
@@ -180,7 +184,7 @@
}
emerginuse = true;
mutex_exit(&pager_map_wanted_lock);
- kva = emergva;
+ kva = emergva + ptoa(first_color);
/* The shift implicitly truncates to PAGE_SIZE */
KASSERT(npages <= (MAXPHYS >> PAGE_SHIFT));
goto enter;
@@ -199,9 +203,10 @@
enter:
/* got it */
- for (cva = kva ; size != 0 ; size -= PAGE_SIZE, cva += PAGE_SIZE) {
+ for (cva = kva; npages != 0; npages--, cva += PAGE_SIZE) {
pp = *pps++;
KASSERT(pp);
+ KASSERT(((VM_PAGE_TO_PHYS(pp) ^ cva) & uvmexp.colormask) == 0);
KASSERT(pp->flags & PG_BUSY);
pmap_kenter_pa(cva, VM_PAGE_TO_PHYS(pp), prot);
}
@@ -221,7 +226,7 @@
void
uvm_pagermapout(vaddr_t kva, int npages)
{
- vsize_t size = npages << PAGE_SHIFT;
+ vsize_t size = ptoa(npages);
struct vm_map_entry *entries;
UVMHIST_FUNC("uvm_pagermapout"); UVMHIST_CALLED(maphist);
@@ -231,8 +236,9 @@
* duplicate uvm_unmap, but add in pager_map_wanted handling.
*/
- pmap_kremove(kva, npages << PAGE_SHIFT);
- if (kva == emergva) {
+ pmap_kremove(kva, size);
+
+ if ((kva & ~ptoa(uvmexp.colormask)) == emergva) {
mutex_enter(&pager_map_wanted_lock);
emerginuse = false;
wakeup(&emergva);
Index: src/sys/uvm/uvm_pglist.c
diff -u src/sys/uvm/uvm_pglist.c:1.42.16.6 src/sys/uvm/uvm_pglist.c:1.42.16.7
--- src/sys/uvm/uvm_pglist.c:1.42.16.6 Tue Jun 1 19:04:02 2010
+++ src/sys/uvm/uvm_pglist.c Wed May 25 23:58:50 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_pglist.c,v 1.42.16.6 2010/06/01 19:04:02 matt Exp $ */
+/* uvm_pglist.c,v 1.42.16.6 2010/06/01 19:04:02 matt Exp */
/*-
* Copyright (c) 1997 The NetBSD Foundation, Inc.
@@ -35,7 +35,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_pglist.c,v 1.42.16.6 2010/06/01 19:04:02 matt Exp $");
+__KERNEL_RCSID(0, "uvm_pglist.c,v 1.42.16.6 2010/06/01 19:04:02 matt Exp");
#include <sys/param.h>
#include <sys/systm.h>
@@ -353,9 +353,10 @@
uvm_pglistalloc_s_ps(struct vm_physseg *ps, int num, paddr_t low, paddr_t high,
struct pglist *rlist)
{
- int todo, limit, try;
+ int todo, limit, try, color;
struct vm_page *pg;
- bool second_pass;
+ bool second_pass, colorless;
+ const int colormask = uvmexp.colormask;
#ifdef DEBUG
int cidx = 0; /* XXX: GCC */
#endif
@@ -366,6 +367,20 @@
KASSERT(mutex_owned(&uvm_fpageqlock));
+ /*
+ * If the pageq (rlist) is empty, then no pages have been allocated
+ * and we can start with any color. If it wasn't empty, then the
+ * starting color is the last page's next color.
+ */
+ colorless = TAILQ_EMPTY(rlist);
+ color = (ps->avail_start + ps->start_hint) & colormask;
+#ifdef DIAGNOSTIC
+ if (!colorless) {
+ pg = TAILQ_LAST(rlist, pglist);
+ KASSERT(color == ((VM_PAGE_TO_COLOR(pg) + 1) & colormask));
+ }
+#endif
+
low = atop(low);
high = atop(high);
todo = num;
@@ -379,7 +394,7 @@
if (ps->start_hint == 0 || second_pass)
break;
second_pass = true;
- try = max(low, ps->avail_start);
+ try = max(low, ps->avail_start) - 1;
limit = min(high, ps->avail_start + ps->start_hint);
pg = &ps->pgs[try - ps->start];
continue;
@@ -390,13 +405,42 @@
if (cidx != (try - ps->start))
panic("pgalloc simple: botch2");
#endif
- if (VM_PAGE_IS_FREE(pg) == 0)
+ /*
+ * If this page isn't free, then we need to skip a colors worth
+ * of pages to get a matching color. Note that colormask is 1
+ * less than what we need since the loop will also increment
+ * try and pgs.
+ */
+ if (VM_PAGE_IS_FREE(pg) == 0) {
+ try += colormask;
+ pg += colormask;
continue;
+ }
+
+ /*
+ * If this page doesn't have the right color, figure out how
+ * many pages we need to skip until we get to the right color.
+ * Note that skip is 1 less that what we need since the loop
+ * will also increment try and pgs.
+ */
+ if (!colorless && (try & colormask) != color) {
+ const int skip = (color - (try + 1)) & colormask;
+ try += skip;
+ pg += skip;
+ continue;
+ }
uvm_pglist_add(pg, rlist);
if (--todo == 0) {
break;
}
+
+ /*
+ * Advance the color (use try instead of color in case we
+ * haven't set an initial color).
+ */
+ color = (try + 1) & colormask;
+ colorless = false;
}
/*
Index: src/sys/uvm/uvm_vnode.c
diff -u src/sys/uvm/uvm_vnode.c:1.90 src/sys/uvm/uvm_vnode.c:1.90.28.1
--- src/sys/uvm/uvm_vnode.c:1.90 Wed Jan 2 11:49:21 2008
+++ src/sys/uvm/uvm_vnode.c Wed May 25 23:58:50 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_vnode.c,v 1.90 2008/01/02 11:49:21 ad Exp $ */
+/* uvm_vnode.c,v 1.90 2008/01/02 11:49:21 ad Exp */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -50,7 +50,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_vnode.c,v 1.90 2008/01/02 11:49:21 ad Exp $");
+__KERNEL_RCSID(0, "uvm_vnode.c,v 1.90 2008/01/02 11:49:21 ad Exp");
#include "fs_nfs.h"
#include "opt_uvmhist.h"
@@ -257,7 +257,8 @@
UVMHIST_LOG(ubchist, "noalloc", 0,0,0,0);
return 0;
}
- pg = uvm_pagealloc(uobj, offset, NULL, 0);
+ pg = uvm_pagealloc(uobj, offset, NULL,
+ UVM_FLAG_COLORMATCH);
if (pg == NULL) {
if (flags & UFP_NOWAIT) {
UVMHIST_LOG(ubchist, "nowait",0,0,0,0);
