Module Name: src
Committed By: matt
Date: Sat Aug 4 07:20:32 UTC 2012
Modified Files:
src/sys/arch/mips/include [matt-nb5-mips64]: cpu.h mips_param.h pmap.h
pte.h vmparam.h
src/sys/arch/mips/mips [matt-nb5-mips64]: cpu_subr.c genassym.cf
mipsX_subr.S mips_machdep.c pmap.c pmap_segtab.c trap.c
vm_machdep.c
Log Message:
Make MIPS use a multi-level page table for the kernel address space.
(just like the user address does). XXX fix mips1
To generate a diff of this commit:
cvs rdiff -u -r1.90.16.45 -r1.90.16.46 src/sys/arch/mips/include/cpu.h
cvs rdiff -u -r1.23.78.12 -r1.23.78.13 src/sys/arch/mips/include/mips_param.h
cvs rdiff -u -r1.54.26.26 -r1.54.26.27 src/sys/arch/mips/include/pmap.h
cvs rdiff -u -r1.19.18.4 -r1.19.18.5 src/sys/arch/mips/include/pte.h
cvs rdiff -u -r1.41.28.27 -r1.41.28.28 src/sys/arch/mips/include/vmparam.h
cvs rdiff -u -r1.1.2.25 -r1.1.2.26 src/sys/arch/mips/mips/cpu_subr.c
cvs rdiff -u -r1.44.12.33 -r1.44.12.34 src/sys/arch/mips/mips/genassym.cf
cvs rdiff -u -r1.26.36.1.2.57 -r1.26.36.1.2.58 \
src/sys/arch/mips/mips/mipsX_subr.S
cvs rdiff -u -r1.205.4.1.2.1.2.67 -r1.205.4.1.2.1.2.68 \
src/sys/arch/mips/mips/mips_machdep.c
cvs rdiff -u -r1.179.16.45 -r1.179.16.46 src/sys/arch/mips/mips/pmap.c
cvs rdiff -u -r1.1.2.17 -r1.1.2.18 src/sys/arch/mips/mips/pmap_segtab.c
cvs rdiff -u -r1.217.12.44 -r1.217.12.45 src/sys/arch/mips/mips/trap.c
cvs rdiff -u -r1.121.6.1.2.29 -r1.121.6.1.2.30 \
src/sys/arch/mips/mips/vm_machdep.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/mips/include/cpu.h
diff -u src/sys/arch/mips/include/cpu.h:1.90.16.45 src/sys/arch/mips/include/cpu.h:1.90.16.46
--- src/sys/arch/mips/include/cpu.h:1.90.16.45 Mon Jul 9 17:23:37 2012
+++ src/sys/arch/mips/include/cpu.h Sat Aug 4 07:20:31 2012
@@ -120,9 +120,9 @@ struct cpu_info {
int ci_tlb_slot; /* reserved tlb entry for cpu_info */
u_int ci_pmap_asid_cur; /* current ASID */
struct pmap_tlb_info *ci_tlb_info; /* tlb information for this cpu */
- union pmap_segtab *ci_pmap_seg0tab;
+ union pmap_segtab *ci_pmap_seg0tab[2];
#ifdef _LP64
- union pmap_segtab *ci_pmap_segtab;
+ union pmap_segtab *ci_pmap_segtab[2];
#else
vaddr_t ci_pmap_srcbase; /* starting VA of ephemeral src space */
vaddr_t ci_pmap_dstbase; /* starting VA of ephemeral dst space */
Index: src/sys/arch/mips/include/mips_param.h
diff -u src/sys/arch/mips/include/mips_param.h:1.23.78.12 src/sys/arch/mips/include/mips_param.h:1.23.78.13
--- src/sys/arch/mips/include/mips_param.h:1.23.78.12 Mon Feb 27 16:57:58 2012
+++ src/sys/arch/mips/include/mips_param.h Sat Aug 4 07:20:31 2012
@@ -87,18 +87,24 @@
#define PGSHIFT PAGE_SHIFT /* LOG2(NBPG) */
#define NBPG (1 << PGSHIFT) /* bytes/page */
#define PGOFSET (NBPG-1) /* byte offset into page */
-#define NPTEPG (NBPG/4)
+#define PTPSHIFT (2)
+#define PTPLENGTH (PGSHIFT-PTPSHIFT)
+#define NPTEPG (1 << PTPLENGTH)
#define NBSEG (NBPG*NPTEPG) /* bytes/segment */
#define SEGOFSET (NBSEG-1) /* byte offset into segment */
-#define SEGSHIFT (PGSHIFT+(PGSHIFT-2)) /* LOG2(NBSEG) */
+#define SEGSHIFT (PGSHIFT+PTPLENGTH) /* LOG2(NBSEG) */
#ifdef _LP64
-#define NSEGPG (NBPG/8)
+#define SEGLENGTH (PGSHIFT-3)
#define NBXSEG ((uint64_t)NSEGPG*NBSEG) /* bytes/xsegment */
#define XSEGOFSET (NBXSEG-1) /* byte offset into xsegment */
-#define XSEGSHIFT (SEGSHIFT+(PGSHIFT-3)) /* LOG2(NBXSEG) */
+#define XSEGSHIFT (SEGSHIFT+SEGLENGTH) /* LOG2(NBXSEG) */
+#define XSEGLENGTH (PGSHIFT-3)
+#else
+#define SEGLENGTH (31-SEGSHIFT)
#endif
+#define NSEGPG (1 << SEGLENGTH)
/*
* Minimum and maximum sizes of the kernel malloc arena in PAGE_SIZE-sized
Index: src/sys/arch/mips/include/pmap.h
diff -u src/sys/arch/mips/include/pmap.h:1.54.26.26 src/sys/arch/mips/include/pmap.h:1.54.26.27
--- src/sys/arch/mips/include/pmap.h:1.54.26.26 Mon Jul 9 17:23:37 2012
+++ src/sys/arch/mips/include/pmap.h Sat Aug 4 07:20:31 2012
@@ -109,17 +109,11 @@
#define mips_trunc_seg(x) ((vaddr_t)(x) & ~SEGOFSET)
#define mips_round_seg(x) (((vaddr_t)(x) + SEGOFSET) & ~SEGOFSET)
-#ifdef _LP64
-#define PMAP_SEGTABSIZE NSEGPG
-#else
-#define PMAP_SEGTABSIZE (1 << (31 - SEGSHIFT))
-#endif
-
union pt_entry;
typedef union pmap_segtab {
- union pmap_segtab * seg_seg[PMAP_SEGTABSIZE];
- union pt_entry * seg_tab[PMAP_SEGTABSIZE];
+ union pmap_segtab * seg_seg[NSEGPG];
+ union pt_entry * seg_tab[NPTEPG];
} pmap_segtab_t;
#else
/*
Index: src/sys/arch/mips/include/pte.h
diff -u src/sys/arch/mips/include/pte.h:1.19.18.4 src/sys/arch/mips/include/pte.h:1.19.18.5
--- src/sys/arch/mips/include/pte.h:1.19.18.4 Thu Mar 11 08:13:18 2010
+++ src/sys/arch/mips/include/pte.h Sat Aug 4 07:20:31 2012
@@ -1,4 +1,4 @@
-/* $NetBSD: pte.h,v 1.19.18.4 2010/03/11 08:13:18 matt Exp $ */
+/* pte.h,v 1.19.18.4 2010/03/11 08:13:18 matt Exp */
/*-
* Copyright (c) 1997 The NetBSD Foundation, Inc.
@@ -257,16 +257,4 @@ mips_paddr_to_tlbpfn(paddr_t pa)
#endif /* ! _LOCORE */
-#if defined(_KERNEL) && !defined(_LOCORE)
-/*
- * Kernel virtual address to page table entry and visa versa.
- */
-#define kvtopte(va) \
- (Sysmap + (((vaddr_t)(va) - VM_MIN_KERNEL_ADDRESS) >> PGSHIFT))
-#define ptetokv(pte) \
- ((((pt_entry_t *)(pte) - Sysmap) << PGSHIFT) + VM_MIN_KERNEL_ADDRESS)
-
-extern pt_entry_t *Sysmap; /* kernel pte table */
-extern u_int Sysmapsize; /* number of pte's in Sysmap */
-#endif /* defined(_KERNEL) && !defined(_LOCORE) */
#endif /* __MIPS_PTE_H__ */
Index: src/sys/arch/mips/include/vmparam.h
diff -u src/sys/arch/mips/include/vmparam.h:1.41.28.27 src/sys/arch/mips/include/vmparam.h:1.41.28.28
--- src/sys/arch/mips/include/vmparam.h:1.41.28.27 Thu Feb 16 10:52:13 2012
+++ src/sys/arch/mips/include/vmparam.h Sat Aug 4 07:20:31 2012
@@ -180,7 +180,7 @@
#ifdef ENABLE_MIPS_TX3900
#define VM_MAX_KERNEL_ADDRESS ((vaddr_t)-0x01000000) /* 0xFFFFFFFFFF000000 */
#else
-#define VM_MAX_KERNEL_ADDRESS ((vaddr_t)-0x00008000) /* 0xFFFFFFFFFFF08000 */
+#define VM_MAX_KERNEL_ADDRESS ((vaddr_t)-0x00008000) /* 0xFFFFFFFFFFFF8000 */
#endif
#endif
#define VM_MAXUSER32_ADDRESS ((vaddr_t)(1UL << 31))/* 0x0000000080000000 */
Index: src/sys/arch/mips/mips/cpu_subr.c
diff -u src/sys/arch/mips/mips/cpu_subr.c:1.1.2.25 src/sys/arch/mips/mips/cpu_subr.c:1.1.2.26
--- src/sys/arch/mips/mips/cpu_subr.c:1.1.2.25 Mon Feb 27 17:04:39 2012
+++ src/sys/arch/mips/mips/cpu_subr.c Sat Aug 4 07:20:31 2012
@@ -80,9 +80,13 @@ struct cpu_info cpu_info_store
.ci_fpcurlwp = &lwp0,
#endif
.ci_tlb_info = &pmap_tlb0_info,
- .ci_pmap_seg0tab = (void *)(MIPS_KSEG2_START + 0x1eadbeef),
+ .ci_pmap_seg0tab = {
+ [0] = (void *)(MIPS_KSEG2_START + 0x1eadbeef),
+ },
#ifdef _LP64
- .ci_pmap_segtab = (void *)(MIPS_KSEG2_START + 0x1eadbeef),
+ .ci_pmap_segtab = {
+ [0] = (void *)(MIPS_KSEG2_START + 0x1eadbeef),
+ },
#endif
.ci_cpl = IPL_HIGH,
.ci_tlb_slot = -1,
@@ -161,6 +165,11 @@ cpu_info_alloc(struct pmap_tlb_info *ti,
ci->ci_divisor_recip = cpu_info_store.ci_divisor_recip;
ci->ci_cpuwatch_count = cpu_info_store.ci_cpuwatch_count;
+#ifdef _LP64
+ ci->ci_pmap_segtab[1] = cpu_info_store.ci_pmap_segtab[1];
+#endif
+ ci->ci_pmap_seg0tab[1] = cpu_info_store.ci_pmap_seg0tab[1];
+
/*
* Attach its TLB info (which must be direct-mapped)
*/
Index: src/sys/arch/mips/mips/genassym.cf
diff -u src/sys/arch/mips/mips/genassym.cf:1.44.12.33 src/sys/arch/mips/mips/genassym.cf:1.44.12.34
--- src/sys/arch/mips/mips/genassym.cf:1.44.12.33 Mon Jul 9 17:13:06 2012
+++ src/sys/arch/mips/mips/genassym.cf Sat Aug 4 07:20:31 2012
@@ -305,9 +305,9 @@ define CPU_INFO_EV_KERN_TLBMISSES offset
define CPU_INFO_EV_USER_TLBINVALID offsetof(struct cpu_info, ci_ev_traps[1][T_TLB_LD_MISS].ev_count)
define CPU_INFO_EV_KERN_TLBINVALID offsetof(struct cpu_info, ci_ev_traps[0][T_TLB_LD_MISS].ev_count)
define CPU_INFO_EV_TLBLOCKED offsetof(struct cpu_info, ci_ev_tlblocked.ev_count)
-define CPU_INFO_PMAP_SEG0TAB offsetof(struct cpu_info, ci_pmap_seg0tab)
+define CPU_INFO_PMAP_SEG0TAB offsetof(struct cpu_info, ci_pmap_seg0tab[0])
ifdef _LP64
-define CPU_INFO_PMAP_SEGTAB offsetof(struct cpu_info, ci_pmap_segtab)
+define CPU_INFO_PMAP_SEGTAB offsetof(struct cpu_info, ci_pmap_segtab[0])
endif
define CPU_INFO_DIVISOR_DELAY offsetof(struct cpu_info, ci_divisor_delay)
define CPU_INFO_MTX_COUNT offsetof(struct cpu_info, ci_mtx_count)
Index: src/sys/arch/mips/mips/mipsX_subr.S
diff -u src/sys/arch/mips/mips/mipsX_subr.S:1.26.36.1.2.57 src/sys/arch/mips/mips/mipsX_subr.S:1.26.36.1.2.58
--- src/sys/arch/mips/mips/mipsX_subr.S:1.26.36.1.2.57 Mon Jul 9 17:11:32 2012
+++ src/sys/arch/mips/mips/mipsX_subr.S Sat Aug 4 07:20:31 2012
@@ -1,4 +1,4 @@
-/* $NetBSD: mipsX_subr.S,v 1.26.36.1.2.57 2012/07/09 17:11:32 matt Exp $ */
+/* $NetBSD: mipsX_subr.S,v 1.26.36.1.2.58 2012/08/04 07:20:31 matt Exp $ */
/*
* Copyright 2002 Wasabi Systems, Inc.
@@ -355,38 +355,48 @@ VECTOR(MIPSX(tlb_miss), unknown)
bnez k0, MIPSX(tlblocked) #03: a lie
# lui in delay slot
#endif
- lui k1, %hi(CPUVAR(PMAP_SEG0TAB)) #00: k1=hi of seg0tab
- _MFC0 k0, MIPS_COP_0_BAD_VADDR #01: k0=bad address
- bltz k0, MIPSX(kernelfault) #02: k0<0 -> kernel fault
- PTR_SRL k0, 1*(PGSHIFT-PTR_SCALESHIFT)+(PGSHIFT-2)#03: k0=seg offset (almost)
+ _MFC0 k0, MIPS_COP_0_BAD_VADDR #00: k0=bad address
+ bgez k0, 1f #01: k0<0 -> kernel fault
+ lui k1, %hi(CPUVAR(PMAP_SEG0TAB)) #02: k1=hi of seg0tab
+ PTR_ADDU k1, 1 << PTR_SCALESHIFT #03: point to kernel slot
+1:
+#if (MIPS32R2 + MIPS64R2 + MIPS64R2_RMIXL) > 0
+ _EXT k0, k0, SEGSHIFT, 31-SEGSHIFT #04: k0=VA[30:22]
+#else
+ PTR_SRL k0, (SEGSHIFT-PTR_SCALESHIFT) #04: k0=seg offset (almost)
+#endif
PTR_L k1, %lo(CPUVAR(PMAP_SEG0TAB))(k1)#04: k1=seg0tab
MIPSX(tlb_miss_common):
#ifdef _LP64
beqz k1, MIPSX(nopagetable) #05: is there a pagetable?
nop #06
#endif
- andi k0, (NBPG-(1<<PTR_SCALESHIFT)) #07: k0=seg offset (mask 0x3)
+#if (MIPS32R2 + MIPS64R2 + MIPS64R2_RMIXL) > 0
+ _INS k1, k0, PTR_SCALESHIFT, SEGLENGTH #07: k0=seg table index
+#else
+ andi k0, (NSEGPG-1)<<PTR_SCALESHIFT #07: k0=seg offset (mask 0x3)
PTR_ADDU k1, k0 #08: k1=seg entry address
+#endif
PTR_L k1, 0(k1) #09: k1=seg entry
_MFC0 k0, MIPS_COP_0_BAD_VADDR #0a: k0=bad address (again)
#if (MIPS32R2 + MIPS64R2 + MIPS64R2_RMIXL) > 0
beqz k1, MIPSX(nopagetable) #0b: ==0 -- no page table
#if (PGSHIFT & 1) == 0
- _EXT k0, k0, PGSHIFT+1, PGSHIFT-3 #0c: k0=VA[13:21]
- _INS k1, k0, 3, PGSHIFT-3 #0d: k0=page table index
+ _EXT k0, k0, PGSHIFT+1, PTPLENGTH-1 #0c: k0=VA[21:13]
+ _INS k1, k0, PTPSHIFT+1, PTPLENGTH-1 #0d: k0=page table index
#0d: k1=pte address
#else
- _EXT k0, k0, PGSHIFT, PGSHIFT-2 #0c: k0=VA[13:21]
- _INS k1, k0, 2, PGSHIFT-2 #0d: k0=page table index
+ _EXT k0, k0, PGSHIFT, PTPLENGTH #0c: k0=VA[21:12]
+ _INS k1, k0, PTPSHIFT, PTPLENGTH #0d: k0=page table index
#0d: k1=pte address
#endif
#else
beqz k1, MIPSX(nopagetable) #0b: ==0 -- no page table
- PTR_SRL k0, (PGSHIFT-2) #0c: k0=VPN (aka va>>10)
+ PTR_SRL k0, PTPLENGTH #0c: k0=VPN (aka va>>10)
#if (PGSHIFT & 1) == 0
- andi k0, (NBPG-8) #0d: k0=page table offset
+ andi k0, (NPTEPG/2-1) << (PTPSHIFT+1)#0d: k0=page table offset
#else
- andi k0, (NBPG-4) #0d: k0=page table offset
+ andi k0, (NPTEPG-1) << PTPSHIFT #0d: k0=page table offset
#endif
PTR_ADDU k1, k0 #0e: k1=pte address
#endif
@@ -471,33 +481,55 @@ VECTOR(MIPSX(xtlb_miss), unknown)
nop
#endif
dmfc0 k0, MIPS_COP_0_BAD_VADDR #00: k0=bad address
-#ifdef _LP64
#ifdef MIPS3
nop #01: nop
#endif
- bltz k0, MIPSX(kernelfault) #02: k0<0 -> kernel fault
- PTR_SRL k1,k0,2*(PGSHIFT-PTR_SCALESHIFT)+(PGSHIFT-2)+PGSHIFT #03: clear valid bits
+#ifdef _LP64
+ PTR_SLL k1, k0, 2 #02: clear top 2 bits
+ PTR_SRL k1, (XSEGSHIFT+XSEGLENGTH+2) #03: clear valid bits
bnez k1, MIPSX(nopagetable) #04: not legal address
- PTR_SRL k0, 2*(PGSHIFT-PTR_SCALESHIFT)+(PGSHIFT-2) #05: k0=seg offset (almost)
- lui k1, %hi(CPUVAR(PMAP_SEGTAB)) #06: k1=hi of segtab
- andi k0, NBPG-(1<<PTR_SCALESHIFT) #07: k0=seg offset (mask 0x3)
- PTR_L k1, %lo(CPUVAR(PMAP_SEGTAB))(k1)#08: k1=segment tab
- PTR_ADDU k1, k0 #09: k1=seg entry address
- PTR_L k1, 0(k1) #0a: k1=seg entry
-#else
- lui k1, %hi(CPUVAR(PMAP_SEG0TAB)) #01: k1=hi of seg0tab
- bltz k0, MIPSX(kernelfault) #02: k0<0 -> kernel fault
- dsrl k0, 31 #03: clear low 31 bits
- bnez k0, MIPSX(nopagetable) #04: not legal address
- nop
- PTR_L k1, %lo(CPUVAR(PMAP_SEG0TAB))(k1)#05: k1=segment tab base
+ lui k1, %hi(CPUVAR(PMAP_SEGTAB)) #06: k1=hi of segtab
+ bgez k0, 1f #07: k0<0 -> kernel access
+#if (MIPS32R2 + MIPS64R2 + MIPS64R2_RMIXL) > 0
+ _EXT k0, k0, XSEGSHIFT, XSEGLENGTH #08: k0=VA[30:22]
+#else
+ PTR_SRL k0, (XSEGSHIFT-PTR_SCALESHIFT) #08: k0=seg offset (almost)
+#endif
+ PTR_ADDU k1, 1 << PTR_SCALESHIFT #09: point to kernel one
+1: PTR_L k1, %lo(CPUVAR(PMAP_SEGTAB))(k1)#0a: k1=segment tab
+
+#if (MIPS32R2 + MIPS64R2 + MIPS64R2_RMIXL) > 0
+ _INS k1, k0, PTR_SCALESHIFT, XSEGLENGTH #0b: k0=seg table index
+#else
+ andi k0, (NSEGPG-1)<<PTR_SCALESHIFT #0b: k0=seg offset (mask 0x3)
+ PTR_ADDU k1, k0 #0c: k1=seg entry address
+#endif
+ PTR_L k1, 0(k1) #0d: k1=seg entry
+#else
+ bgez k0, 1f #02: k0<0 -> kernel fault
+ lui k1, %hi(CPUVAR(PMAP_SEG0TAB)) #03: k1=hi of seg0tab
+ PTR_ADDU k1, 1 << PTR_SCALESHIFT #04: point to kernel entry
+1: PTR_SRA k0, 31 #05: clear low 31 bits
+ beqz k0, 2f #06: legal address
+ PTR_ADDU k0, 1 #08: kernel=was -1, now 0
+ bnez k0, MIPSX(nopagetable) #09: not legal address
+ nop #0a: delay slot
+2: PTR_L k1, %lo(CPUVAR(PMAP_SEG0TAB))(k1)#0b: k1=segment tab base
#endif /* _LP64 */
- dmfc0 k0, MIPS_COP_0_BAD_VADDR #0b: k0=bad address (again)
+ dmfc0 k0, MIPS_COP_0_BAD_VADDR #0e: k0=bad address (again)
#ifdef MIPS3
- nop #0c
+ nop #0f
+#endif
+ b MIPSX(tlb_miss_common) #10
+#if (MIPS32R2 + MIPS64R2 + MIPS64R2_RMIXL) > 0
+#ifdef _LP64
+ _EXT k0, k0, SEGSHIFT, SEGLENGTH #11: k0=VA[30:22]
+#else
+ _EXT k0, k0, SEGSHIFT, 31-SEGSHIFT #11: k0=VA[30:22]
+#endif
+#else
+ PTR_SRL k0, SEGSHIFT-PTR_SCALESHIFT #11: k0=seg offset (almost)
#endif
- b MIPSX(tlb_miss_common) #0d
- PTR_SRL k0, 1*(PGSHIFT-PTR_SCALESHIFT)+(PGSHIFT-2) #0e: k0=seg offset (almost)
.set at
_VECTOR_END(MIPSX(xtlb_miss))
#else
@@ -1606,6 +1638,11 @@ END(MIPSX(cache_exception))
*/
LEAF_NOPROFILE(MIPSX(kern_tlb_miss))
.set noat
+#if defined(MULTIPROCESSOR) && (MIPS64_RMIXL + MIPS64R2_RMIXL) > 0
+#define TLB_MISS_EXCEPTION_EXIT _C_LABEL(MIPSX(tlbunlock_kern_gen_exception))
+#else
+#define TLB_MISS_EXCEPTION_EXIT _C_LABEL(MIPSX(kern_gen_exception))
+#endif
_MFC0 k0, MIPS_COP_0_BAD_VADDR # get the fault address
#if VM_MIN_KERNEL_ADDRESS == MIPS_KSEG2_START
li k1, VM_MIN_KERNEL_ADDRESS # compute index
@@ -1614,9 +1651,15 @@ LEAF_NOPROFILE(MIPSX(kern_tlb_miss))
dsll32 k1, k1, 0
#endif
PTR_SUBU k0, k1
- INT_L k1, _C_LABEL(Sysmapsize) # index within range?
+#ifdef _LP64
+ li k1, (NSEGPG * NSEGPG * NPTEPG)
+#elif defined(newsmips)
+ li k1, 0xfff00000
+#else
+ li k1, -MIPS_KSEG2_START >> PGSHIFT
+#endif
PTR_SRL k0, PGSHIFT
- sltu k1, k0, k1
+ sltu k1, k0, k1 # index within range? (unsigned)
#ifdef newsmips
/* news5000 has ROM work area at 0xfff00000. */
bnez k1, 1f
@@ -1624,23 +1667,43 @@ LEAF_NOPROFILE(MIPSX(kern_tlb_miss))
j checkromwork
nop # - delay slot -
1:
-#elif defined(MULTIPROCESSOR) && (MIPS64_RMIXL + MIPS64R2_RMIXL) > 0
- beqz k1, _C_LABEL(MIPSX(tlbunlock_kern_gen_exception)) # full trap processing
- nop
#else
+ beqz k1, TLB_MISS_EXCEPTION_EXIT # full trap processing
+ nop
+#endif
+ _MFC0 k0, MIPS_COP_0_BAD_VADDR # get the fault address (again)
+ PTR_LA k1, _C_LABEL(pmap_kern_segtab)
+ # 64: [39:31][30:22][21:12] -> [11:3][11:3][11:2]
+ # 32: [31:22][21:12] -> [11:2][11:2]
+#ifdef _LP64
/*
- * If we are beyond the bounds of Sysmap, let trap panic for us.
+ * Get the top level page table pointer.
*/
- beqz k1, _C_LABEL(MIPSX(kern_gen_exception)) # full trap processing
+ PTR_SRL k0, (3 * PGSHIFT - 2*PTR_SCALESHIFT - 2)
+ and k0, (NSEGPG-1) << PTR_SCALESHIFT
+ PTR_ADD k1, k0
+ PTR_L k1, 0(k1)
+ beqz k1, TLB_MISS_EXCEPTION_EXIT # full trap processing
nop # - delay slot -
+ _MFC0 k0, MIPS_COP_0_BAD_VADDR # get the fault address (again)
#endif
- PTR_L k1, _C_LABEL(Sysmap)
+ PTR_SRL k0, (2 * PGSHIFT - PTR_SCALESHIFT - 2)
+ and k0, (NSEGPG-1) << PTR_SCALESHIFT
+ PTR_ADD k1, k0
+ PTR_L k1, 0(k1)
+ beqz k1, TLB_MISS_EXCEPTION_EXIT # full trap processing
+ nop # - delay slot -
+ #
+ # Now get the index of the PTE in the bottom page.
+ #
+ _MFC0 k0, MIPS_COP_0_BAD_VADDR # get the fault address
+ PTR_SRL k0, PGSHIFT - 2
#if (PGSHIFT & 1) == 0
- PTR_SRL k0, 1
- PTR_SLL k0, 3 # compute offset from index
+ and k0, (NPTEPG/2-1) << 3
#else
- PTR_SLL k0, 2 # compute offset from index
+ and k0, (NPTEPG-1) << 2
#endif
+
PTR_ADDU k1, k0
INT_L k0, 0(k1) # get PTE entry
#if (MIPS32R2 + MIPS64R2 + MIPS64R2_RMIXL) > 0
@@ -1735,14 +1798,48 @@ LEAF_NOPROFILE(MIPSX(kern_tlb_invalid_ex
bgez k0, TLB_INVALID_EXCEPTION_EXIT # full trap processing
nop # - delay slot -
PTR_SUBU k0, k1
- INT_L k1, _C_LABEL(Sysmapsize) # index within range?
+#ifdef _LP64
+ li k1, (NSEGPG * NSEGPG * NPTEPG)
+#elif defined(newsmips)
+ li k1, 0xfff00000
+#else
+ li k1, -MIPS_KSEG2_START >> PGSHIFT
+#endif
PTR_SRL k0, PGSHIFT
sltu k1, k0, k1
beqz k1, TLB_INVALID_EXCEPTION_EXIT # No. Failing beyond...
nop # - delay slot -
- PTR_L k1, _C_LABEL(Sysmap)
- PTR_SLL k0, 2 # compute offset from index
- PTR_ADDU k1, k0
+
+ _MFC0 k0, MIPS_COP_0_BAD_VADDR # get the fault address (again)
+ PTR_LA k1, _C_LABEL(pmap_kern_segtab)
+ # 64: [39:31][30:22][21:12] -> [11:3][11:3][11:2]
+ # 32: [31:22][21:12] -> [11:2][11:2]
+#ifdef _LP64
+ /*
+ * Get the top level page table pointer.
+ */
+ PTR_SRL k0, (3 * PGSHIFT - 2*PTR_SCALESHIFT - 2)
+ and k0, (NSEGPG-1) << PTR_SCALESHIFT
+ PTR_ADD k1, k0
+ PTR_L k1, 0(k1)
+ beqz k1, TLB_MISS_EXCEPTION_EXIT # full trap processing
+ nop # - delay slot -
+ _MFC0 k0, MIPS_COP_0_BAD_VADDR # get the fault address (again)
+#endif
+ PTR_SRL k0, (2 * PGSHIFT - PTR_SCALESHIFT - 2)
+ and k0, (NSEGPG-1) << PTR_SCALESHIFT
+ PTR_ADD k1, k0
+ PTR_L k1, 0(k1)
+ beqz k1, TLB_MISS_EXCEPTION_EXIT # full trap processing
+ nop # - delay slot -
+ #
+ # Now get the index of the PTE in the bottom page.
+ #
+ _MFC0 k0, MIPS_COP_0_BAD_VADDR # get the fault address
+ PTR_SRL k0, PGSHIFT - 2
+ and k0, (NPTEPG-1) << 2
+ PTR_ADDU k1, k0 # k1 now points to pte
+
tlbp # Probe the invalid entry
COP0_SYNC
Index: src/sys/arch/mips/mips/mips_machdep.c
diff -u src/sys/arch/mips/mips/mips_machdep.c:1.205.4.1.2.1.2.67 src/sys/arch/mips/mips/mips_machdep.c:1.205.4.1.2.1.2.68
--- src/sys/arch/mips/mips/mips_machdep.c:1.205.4.1.2.1.2.67 Thu Feb 16 18:58:57 2012
+++ src/sys/arch/mips/mips/mips_machdep.c Sat Aug 4 07:20:31 2012
@@ -1869,8 +1869,10 @@ cpu_dump(void)
cpuhdrp->pg_frame = MIPS1_PG_FRAME;
cpuhdrp->pg_v = MIPS1_PG_V;
}
+#ifdef OBSOLETE
cpuhdrp->sysmappa = MIPS_KSEG0_TO_PHYS(Sysmap);
cpuhdrp->sysmapsize = Sysmapsize;
+#endif
cpuhdrp->nmemsegs = mem_cluster_cnt;
/*
Index: src/sys/arch/mips/mips/pmap.c
diff -u src/sys/arch/mips/mips/pmap.c:1.179.16.45 src/sys/arch/mips/mips/pmap.c:1.179.16.46
--- src/sys/arch/mips/mips/pmap.c:1.179.16.45 Mon Jul 9 17:23:37 2012
+++ src/sys/arch/mips/mips/pmap.c Sat Aug 4 07:20:31 2012
@@ -156,6 +156,15 @@ CTASSERT((uint32_t)MIPS_MAX_MEM_ADDR ==
CTASSERT((uint32_t)MIPS_RESERVED_ADDR == 0xbfc80000);
CTASSERT(MIPS_KSEG0_P(MIPS_PHYS_TO_KSEG0(0)));
CTASSERT(MIPS_KSEG1_P(MIPS_PHYS_TO_KSEG1(0)));
+#ifdef ENABLE_MIPS_KSEGX
+CTASSERT(VM_KSEGX_ADDRESS % NBSEG == 0);
+CTASSERT(VM_KSEGX_SIZE % NBSEG == 0);
+#endif
+#ifdef _LP64
+CTASSERT(VM_MIN_KERNEL_ADDRESS % NBXSEG == 0);
+#else
+CTASSERT(VM_MIN_KERNEL_ADDRESS % NBSEG == 0);
+#endif
#define PMAP_COUNT(name) (pmap_evcnt_##name.ev_count++ + 0)
#define PMAP_COUNTER(name, desc) \
@@ -261,10 +270,29 @@ CTASSERT(PMAP_ASID_RESERVED == 0);
kmutex_t pmap_pvlist_mutex __aligned(COHERENCY_UNIT);
#endif
+/*
+ * We need the pmap_segtab's to be aligned on MIPS*R2 so we can use the
+ * EXT/INS instructions on their addresses.
+ */
+#if (MIPS32R2 + MIPS64R2 + MIPS64R2_RMIXL) > 0
+#define SEGALIGN __aligned(sizeof(void *)*NSEGPG) __section(".data1")
+#else
+#define SEGALIGN
+#endif
+#ifdef _LP64
+pmap_segtab_t pmap_kstart_segtab SEGALIGN; /* first mid-level segtab for kernel */
+#endif
+pmap_segtab_t pmap_kern_segtab SEGALIGN = { /* top level segtab for kernel */
+#ifdef _LP64
+ .seg_seg[(VM_MIN_KERNEL_ADDRESS & XSEGOFSET) >> SEGSHIFT] = &pmap_kstart_segtab,
+#endif
+};
+#undef SEGALIGN
+
struct pmap_kernel kernel_pmap_store = {
.kernel_pmap = {
.pm_count = 1,
- .pm_segtab = (void *)(MIPS_KSEG2_START + 0x1eadbeef),
+ .pm_segtab = &pmap_kern_segtab,
#ifdef MULTIPROCESSOR
.pm_active = 1,
.pm_onproc = 1,
@@ -276,9 +304,6 @@ paddr_t mips_avail_start; /* PA of first
paddr_t mips_avail_end; /* PA of last available physical page */
vaddr_t mips_virtual_end; /* VA of last avail page (end of kernel AS) */
-pt_entry_t *Sysmap; /* kernel pte table */
-unsigned int Sysmapsize; /* number of pte's in Sysmap */
-
static void pmap_pvlist_lock_init(void);
/*
@@ -290,12 +315,12 @@ struct pool pmap_pv_pool;
#ifndef PMAP_PV_LOWAT
#define PMAP_PV_LOWAT 16
#endif
-int pmap_pv_lowat = PMAP_PV_LOWAT;
+int pmap_pv_lowat = PMAP_PV_LOWAT;
-bool pmap_initialized = false;
+bool pmap_initialized = false;
#define PMAP_PAGE_COLOROK_P(a, b) \
((((int)(a) ^ (int)(b)) & pmap_page_colormask) == 0)
-u_int pmap_page_colormask;
+u_int pmap_page_colormask;
#define PAGE_IS_MANAGED(pa) \
(pmap_initialized == true && vm_physseg_find(atop(pa), NULL) != -1)
@@ -307,7 +332,6 @@ u_int pmap_page_colormask;
/* Forward function declarations */
void pmap_remove_pv(pmap_t, vaddr_t, struct vm_page *, bool);
void pmap_enter_pv(pmap_t, vaddr_t, struct vm_page *, u_int *);
-pt_entry_t *pmap_pte(pmap_t, vaddr_t);
/*
* PV table management functions.
@@ -322,6 +346,24 @@ struct pool_allocator pmap_pv_page_alloc
#define pmap_pv_alloc() pool_get(&pmap_pv_pool, PR_NOWAIT)
#define pmap_pv_free(pv) pool_put(&pmap_pv_pool, (pv))
+#ifdef PARANOIADIAG
+static inline void
+pmap_asid_check(pmap_t pmap, const char *func)
+{
+ if (!PMAP_IS_ACTIVE(pmap){
+ return;
+
+ __asm volatile("mfc0 %0,$10; nop" : "=r"(asid));
+ uint32_t asid = (MIPS_HAS_R4K_MMU)
+ ? (asid & 0xff)
+ : (asid & 0xfc0) >> 6;
+ if (asid != pai->pai_asid)
+ panic("%s: inconsistency for active TLB update: %u <-> %u",
+ func, asid, pai->pai_asid);
+}
+#endif
+
+
/*
* Misc. functions.
*/
@@ -381,7 +423,7 @@ pmap_map_ephemeral_page(struct vm_page *
va = (prot & VM_PROT_WRITE ? ci->ci_pmap_dstbase : ci->ci_pmap_srcbase)
+ mips_cache_indexof(MIPS_CACHE_VIRTUAL_ALIAS ? pv->pv_va : pa);
- *old_pt_entry_p = *kvtopte(va);
+ *old_pt_entry_p = *pmap_pte_lookup(pmap_kernel(), va);
pmap_kenter_pa(va, pa, prot);
#endif
}
@@ -434,7 +476,7 @@ pmap_unmap_ephemeral_page(struct vm_page
if (va >= VM_MIN_KERNEL_ADDRESS) {
pmap_kremove(va, PAGE_SIZE);
if (mips_pg_v(old_pt_entry.pt_entry)) {
- *kvtopte(va) = old_pt_entry;
+ *pmap_pte_lookup(pmap_kernel(), va) = old_pt_entry;
pmap_tlb_update_addr(pmap_kernel(), va,
old_pt_entry.pt_entry, false);
}
@@ -482,6 +524,8 @@ void
pmap_bootstrap(void)
{
vsize_t bufsz;
+ size_t sysmap_size;
+ pt_entry_t *sysmap;
if (MIPS_CACHE_VIRTUAL_ALIAS && uvmexp.ncolors)
pmap_page_colormask = (uvmexp.ncolors - 1) << PAGE_SHIFT;
@@ -509,28 +553,28 @@ pmap_bootstrap(void)
bufsz = buf_memcalc();
buf_setvalimit(bufsz);
- Sysmapsize = (VM_PHYS_SIZE + (ubc_nwins << ubc_winshift) +
+ sysmap_size = (VM_PHYS_SIZE + (ubc_nwins << ubc_winshift) +
bufsz + 16 * NCARGS + pager_map_size) / NBPG +
(maxproc * UPAGES) + nkmempages;
#ifdef SYSVSHM
- Sysmapsize += shminfo.shmall;
+ sysmap_size += shminfo.shmall;
#endif
#ifdef KSEG2IOBUFSIZE
- Sysmapsize += (KSEG2IOBUFSIZE >> PGSHIFT);
+ sysmap_size += (KSEG2IOBUFSIZE >> PGSHIFT);
#endif
#ifdef _LP64
/*
* If we are using tmpfs, then we might want to use a great deal of
* our memory with it. Make sure we have enough VM to do that.
*/
- Sysmapsize += physmem;
+ sysmap_size += physmem;
#else
/* XXX: else runs out of space on 256MB sbmips!! */
- Sysmapsize += 20000;
+ sysmap_size += 20000;
#endif
/* Rounup to a even number of pte page tables */
- Sysmapsize = (Sysmapsize + NPTEPG - 1) & -NPTEPG;
+ sysmap_size = (sysmap_size + NPTEPG - 1) & -NPTEPG;
/*
* Initialize `FYI' variables. Note we're relying on
@@ -540,7 +584,7 @@ pmap_bootstrap(void)
*/
mips_avail_start = ptoa(VM_PHYSMEM_PTR(0)->start);
mips_avail_end = ptoa(VM_PHYSMEM_PTR(vm_nphysseg - 1)->end);
- mips_virtual_end = VM_MIN_KERNEL_ADDRESS + (vaddr_t)Sysmapsize * NBPG;
+ mips_virtual_end = VM_MIN_KERNEL_ADDRESS + (vaddr_t)sysmap_size * NBPG;
#ifndef _LP64
#ifdef ENABLE_MIPS_KSEGX
@@ -556,11 +600,11 @@ pmap_bootstrap(void)
if (mips_virtual_end > VM_MAX_KERNEL_ADDRESS
|| mips_virtual_end < VM_MIN_KERNEL_ADDRESS) {
- printf("%s: chaning last kernel VA from %#"PRIxVADDR
+ printf("%s: changing last kernel VA from %#"PRIxVADDR
" to %#"PRIxVADDR"\n", __func__,
mips_virtual_end, VM_MAX_KERNEL_ADDRESS);
mips_virtual_end = VM_MAX_KERNEL_ADDRESS;
- Sysmapsize =
+ sysmap_size =
(VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS) / NBPG;
}
#endif
@@ -570,8 +614,66 @@ pmap_bootstrap(void)
* Now actually allocate the kernel PTE array (must be done
* after virtual_end is initialized).
*/
- Sysmap = (pt_entry_t *)
- uvm_pageboot_alloc(sizeof(pt_entry_t) * Sysmapsize);
+ sysmap = (pt_entry_t *)
+ uvm_pageboot_alloc(sizeof(pt_entry_t) * sysmap_size);
+
+ vaddr_t va = VM_MIN_KERNEL_ADDRESS;
+#ifdef _LP64
+ /*
+ * Do we need more than one XSEG's worth virtual address space?
+ * If so, we have to allocate the additional pmap_segtab_t's for them
+ * and insert them into the kernel's top level segtab.
+ */
+ const size_t xsegs = (sysmap_size + NBXSEG - 1) / NBXSEG;
+ if (xsegs > 1) {
+ pmap_segtab_t *stp = (pmap_segtab_t *)
+ uvm_pageboot_alloc(sizeof(pmap_segtab_t) * (xsegs - 1));
+ for (size_t i = 1; i <= xsegs; i++, stp++) {
+ pmap_kern_segtab.seg_seg[i] = stp;
+ }
+ }
+ pmap_segtab_t ** const xstp = pmap_kern_segtab.seg_seg;
+ curcpu()->ci_pmap_segtab[1] = &pmap_kern_segtab;
+#else
+ const size_t xsegs = 1;
+ pmap_segtab_t * const stp = &pmap_kern_segtab;
+ curcpu()->ci_pmap_seg0tab[1] = &pmap_kern_segtab;
+#endif
+
+ for (size_t k = 0, i = 0; k < xsegs; k++) {
+#ifdef _LP64
+ pmap_segtab_t * const stp =
+ xstp[(va >> XSEGSHIFT) & (NSEGPG - 1)];
+#endif
+ bool done = false;
+
+ for (size_t j = (va >> SEGSHIFT) & (NSEGPG - 1);
+ !done && i < sysmap_size;
+ i += NPTEPG, j++, va += NBSEG) {
+#ifdef ENABLE_MIPS_KSEGX
+ /*
+ * Skip over the KSEGX region since they
+ * don't need page table pages.
+ */
+ if (va == VM_KSEGX_ADDRESS) {
+ va += VM_KSEGX_SIZE;
+ j += VM_KSEGX_SIZE / NBSEG;
+ }
+#endif
+ /*
+ * Now set the page table pointer...
+ */
+ stp->seg_tab[j] = &sysmap[i];
+#ifdef _LP64
+ /*
+ * If we are at end of this XSEG, terminate the loop
+ * so we advance to the next one.
+ */
+ done = (j + 1 == NSEGPG);
+#endif
+ }
+ }
+ KASSERT(pmap_pte_lookup(pmap_kernel(), VM_MIN_KERNEL_ADDRESS) == sysmap);
/*
* Initialize the pools.
@@ -592,11 +694,9 @@ pmap_bootstrap(void)
* they will produce a global bit to store in the tlb.
*/
if (MIPS_HAS_R4K_MMU) {
- u_int i;
- pt_entry_t *spte;
-
- for (i = 0, spte = Sysmap; i < Sysmapsize; i++, spte++)
- spte->pt_entry = MIPS3_PG_G;
+ for (; sysmap_size-- > 0; sysmap++) {
+ sysmap->pt_entry = MIPS3_PG_G;
+ }
}
#endif /* MIPS3_PLUS */
}
@@ -922,10 +1022,10 @@ pmap_deactivate(struct lwp *l)
kpreempt_disable();
KASSERT(l == curlwp || l->l_cpu == curlwp->l_cpu);
#ifdef _LP64
- curcpu()->ci_pmap_segtab = (void *)(MIPS_KSEG2_START + 0x1eadbeef);
- curcpu()->ci_pmap_seg0tab = NULL;
+ curcpu()->ci_pmap_segtab[0] = (void *)(MIPS_KSEG2_START + 0x1eadbeef);
+ curcpu()->ci_pmap_seg0tab[0] = NULL;
#else
- curcpu()->ci_pmap_seg0tab = (void *)(MIPS_KSEG2_START + 0x1eadbeef);
+ curcpu()->ci_pmap_seg0tab[0] = (void *)(MIPS_KSEG2_START + 0x1eadbeef);
#endif
pmap_tlb_asid_deactivate(l->l_proc->p_vmspace->vm_map.pmap);
kpreempt_enable();
@@ -974,7 +1074,6 @@ pmap_pte_remove(pmap_t pmap, vaddr_t sva
KASSERT(kpreempt_disabled());
for (; sva < eva; sva += NBPG, pte++) {
- struct vm_page *pg;
uint32_t pt_entry = pte->pt_entry;
if (!mips_pg_v(pt_entry))
continue;
@@ -982,11 +1081,20 @@ pmap_pte_remove(pmap_t pmap, vaddr_t sva
if (mips_pg_wired(pt_entry))
pmap->pm_stats.wired_count--;
pmap->pm_stats.resident_count--;
+
+ struct vm_page *pg;
pg = PHYS_TO_VM_PAGE(mips_tlbpfn_to_paddr(pt_entry));
if (pg) {
pmap_remove_pv(pmap, sva, pg,
pt_entry & mips_pg_m_bit());
}
+#if 0
+ if (MIPS_HAS_R4K_MMU)
+ /* See above about G bit */
+ pte->pt_entry = MIPS3_PG_NV | MIPS3_PG_G;
+ else
+ pte->pt_entry = MIPS1_PG_NV;
+#endif
pte->pt_entry = mips_pg_nv_bit();
/*
* Flush the TLB for the given address.
@@ -999,8 +1107,6 @@ pmap_pte_remove(pmap_t pmap, vaddr_t sva
void
pmap_remove(pmap_t pmap, vaddr_t sva, vaddr_t eva)
{
- struct vm_page *pg;
-
#ifdef DEBUG
if (pmapdebug & (PDB_FOLLOW|PDB_REMOVE|PDB_PROTECT))
printf("pmap_remove(%p, %#"PRIxVADDR", %#"PRIxVADDR")\n", pmap, sva, eva);
@@ -1014,49 +1120,14 @@ pmap_remove(pmap_t pmap, vaddr_t sva, va
if (sva < VM_MIN_KERNEL_ADDRESS || eva >= mips_virtual_end)
panic("pmap_remove: kva not in range");
#endif
- pt_entry_t *pte = kvtopte(sva);
- for (; sva < eva; sva += NBPG, pte++) {
- uint32_t pt_entry = pte->pt_entry;
- if (!mips_pg_v(pt_entry))
- continue;
- PMAP_COUNT(remove_kernel_pages);
- if (mips_pg_wired(pt_entry))
- pmap->pm_stats.wired_count--;
- pmap->pm_stats.resident_count--;
- pg = PHYS_TO_VM_PAGE(mips_tlbpfn_to_paddr(pt_entry));
- if (pg)
- pmap_remove_pv(pmap, sva, pg, false);
- if (MIPS_HAS_R4K_MMU)
- /* See above about G bit */
- pte->pt_entry = MIPS3_PG_NV | MIPS3_PG_G;
- else
- pte->pt_entry = MIPS1_PG_NV;
-
- /*
- * Flush the TLB for the given address.
- */
- pmap_tlb_invalidate_addr(pmap, sva);
- }
- kpreempt_enable();
- return;
- }
-
- PMAP_COUNT(remove_user_calls);
+ } else {
+ PMAP_COUNT(remove_user_calls);
#ifdef PARANOIADIAG
- if (eva > VM_MAXUSER_ADDRESS)
- panic("pmap_remove: uva not in range");
- if (PMAP_IS_ACTIVE(pmap)) {
- struct pmap_asid_info * const pai = PMAP_PAI(pmap, curcpu());
- uint32_t asid;
-
- __asm volatile("mfc0 %0,$10; nop" : "=r"(asid));
- asid = (MIPS_HAS_R4K_MMU) ? (asid & 0xff) : (asid & 0xfc0) >> 6;
- if (asid != pai->pai_asid) {
- panic("inconsistency for active TLB flush: %d <-> %d",
- asid, pai->pai_asid);
- }
- }
+ if (eva > VM_MAXUSER_ADDRESS)
+ panic("pmap_remove: uva not in range");
+ pmap_asid_check(asid, __func__);
#endif
+ }
#ifdef PMAP_FAULTINFO
curpcb->pcb_faultinfo.pfi_faultaddr = 0;
curpcb->pcb_faultinfo.pfi_repeats = 0;
@@ -1188,10 +1259,6 @@ pmap_pte_protect(pmap_t pmap, vaddr_t sv
void
pmap_protect(pmap_t pmap, vaddr_t sva, vaddr_t eva, vm_prot_t prot)
{
- const uint32_t pg_mask = ~(mips_pg_m_bit() | mips_pg_ro_bit());
- pt_entry_t *pte;
- u_int p;
-
PMAP_COUNT(protect);
#ifdef DEBUG
if (pmapdebug & (PDB_FOLLOW|PDB_PROTECT))
@@ -1203,7 +1270,7 @@ pmap_protect(pmap_t pmap, vaddr_t sva, v
return;
}
- p = (prot & VM_PROT_WRITE) ? mips_pg_rw_bit() : mips_pg_ro_bit();
+ uintptr_t p = (prot & VM_PROT_WRITE) ? mips_pg_rw_bit() : mips_pg_ro_bit();
kpreempt_disable();
if (pmap == pmap_kernel()) {
@@ -1219,36 +1286,13 @@ pmap_protect(pmap_t pmap, vaddr_t sva, v
if (sva < VM_MIN_KERNEL_ADDRESS || eva >= mips_virtual_end)
panic("pmap_protect: kva not in range");
#endif
- pte = kvtopte(sva);
- for (; sva < eva; sva += NBPG, pte++) {
- uint32_t pt_entry = pte->pt_entry;
- if (!mips_pg_v(pt_entry))
- continue;
- if (MIPS_HAS_R4K_MMU && (pt_entry & mips_pg_m_bit()))
- mips_dcache_wb_range(sva, PAGE_SIZE);
- pt_entry &= (pt_entry & pg_mask) | p;
- pte->pt_entry = pt_entry;
- pmap_tlb_update_addr(pmap, sva, pt_entry, true);
- }
- kpreempt_enable();
- return;
- }
-
+ } else {
#ifdef PARANOIADIAG
- if (eva > VM_MAXUSER_ADDRESS)
- panic("pmap_protect: uva not in range");
- if (PMAP_IS_ACTIVE(pmap)) {
- struct pmap_asid_info * const pai = PMAP_PAI(pmap, curcpu());
- uint32_t asid;
-
- __asm volatile("mfc0 %0,$10; nop" : "=r"(asid));
- asid = (MIPS_HAS_R4K_MMU) ? (asid & 0xff) : (asid & 0xfc0) >> 6;
- if (asid != pai->pai_asid) {
- panic("inconsistency for active TLB update: %d <-> %d",
- asid, pai->pai_asid);
- }
- }
+ if (eva > VM_MAXUSER_ADDRESS)
+ panic("pmap_protect: uva not in range");
+ pmap_asid_check(asid, __func__);
#endif
+ }
/*
* Change protection on every valid mapping within this segment.
@@ -1289,12 +1333,11 @@ pmap_procwr(struct proc *p, vaddr_t va,
unsigned entry;
kpreempt_disable();
- if (pmap == pmap_kernel()) {
- pte = kvtopte(va);
- } else {
- pte = pmap_pte_lookup(pmap, va);
- }
- entry = pte->pt_entry;
+ pte = pmap_pte_lookup(pmap, va);
+ if (pte)
+ entry = pte->pt_entry;
+ else
+ entry = 0;
kpreempt_enable();
if (!mips_pg_v(entry))
return;
@@ -1359,16 +1402,9 @@ pmap_page_cache(struct vm_page *pg, bool
KASSERT(pmap != NULL);
KASSERT(!mm_md_direct_mapped_virt(va, NULL, NULL));
- if (pmap == pmap_kernel()) {
- /*
- * Change entries in kernel pmap.
- */
- pte = kvtopte(va);
- } else {
- pte = pmap_pte_lookup(pmap, va);
- if (pte == NULL)
- continue;
- }
+ pte = pmap_pte_lookup(pmap, va);
+ if (pte == NULL)
+ continue;
pt_entry = pte->pt_entry;
if (pt_entry & MIPS3_PG_V) {
pt_entry = (pt_entry & ~MIPS3_PG_CACHEMODE) | newmode;
@@ -1401,6 +1437,7 @@ pmap_enter(pmap_t pmap, vaddr_t va, padd
bool cached = true;
#endif
bool wired = (flags & PMAP_WIRED) != 0;
+ const bool kernel_pmap_p = (pmap == pmap_kernel());
#ifdef DEBUG
if (pmapdebug & (PDB_FOLLOW|PDB_ENTER))
@@ -1411,7 +1448,7 @@ pmap_enter(pmap_t pmap, vaddr_t va, padd
KASSERTMSG(good_color,
("%s(%p, %#"PRIxVADDR", %#"PRIxPADDR", %x, %x): color mismatch\n",
__func__, pmap, va, pa, prot, flags));
- if (pmap == pmap_kernel()) {
+ if (kernel_pmap_p) {
PMAP_COUNT(kernel_mappings);
if (!good_color)
PMAP_COUNT(kernel_mappings_bad);
@@ -1529,42 +1566,13 @@ pmap_enter(pmap_t pmap, vaddr_t va, padd
#endif
#endif
- kpreempt_disable();
- if (pmap == pmap_kernel()) {
- if (pg)
- pmap_enter_pv(pmap, va, pg, &npte);
-
- /* enter entries into kernel pmap */
- pte = kvtopte(va);
-
- if (MIPS_HAS_R4K_MMU)
- npte |= mips3_paddr_to_tlbpfn(pa) | MIPS3_PG_G;
- else
- npte |= mips1_paddr_to_tlbpfn(pa) |
- MIPS1_PG_V | MIPS1_PG_G;
-
- if (wired) {
- pmap->pm_stats.wired_count++;
- npte |= mips_pg_wired_bit();
- }
- const bool resident_p = mips_pg_v(pte->pt_entry);
- if (resident_p) {
- if (mips_tlbpfn_to_paddr(pte->pt_entry) != pa) {
- pmap_remove(pmap, va, va + NBPG);
- PMAP_COUNT(kernel_mappings_changed);
- }
- } else {
- pmap->pm_stats.resident_count++;
- }
- pte->pt_entry = npte;
+ /*
+ * Kernel entries need the global bit.
+ */
+ if (kernel_pmap_p)
+ npte |= (MIPS_HAS_R4K_MMU ? MIPS3_PG_G : MIPS1_PG_G);
- /*
- * Update the same virtual address entry.
- */
- pmap_tlb_update_addr(pmap, va, npte, resident_p);
- kpreempt_enable();
- return 0;
- }
+ kpreempt_disable();
pte = pmap_pte_reserve(pmap, va, flags);
if (__predict_false(pte == NULL)) {
@@ -1599,16 +1607,7 @@ pmap_enter(pmap_t pmap, vaddr_t va, padd
#endif
#ifdef PARANOIADIAG
- if (PMAP_IS_ACTIVE(pmap)) {
- uint32_t asid;
-
- __asm volatile("mfc0 %0,$10; nop" : "=r"(asid));
- asid = (MIPS_HAS_R4K_MMU) ? (asid & 0xff) : (asid & 0xfc0) >> 6;
- if (asid != pai->pai_asid) {
- panic("inconsistency for active TLB update: %u <-> %u",
- asid, pai->pai_asid);
- }
- }
+ pmap_asid_check(asid, __func__);
#endif
if (mips_pg_v(pte->pt_entry) &&
@@ -1620,7 +1619,10 @@ pmap_enter(pmap_t pmap, vaddr_t va, padd
#ifdef PMAP_FAULTINFO
curpcb->pcb_faultinfo = tmp_fi;
#endif
- PMAP_COUNT(user_mappings_changed);
+ if (kernel_pmap_p)
+ PMAP_COUNT(kernel_mappings_changed);
+ else
+ PMAP_COUNT(user_mappings_changed);
}
KASSERT(mips_pg_v(npte));
@@ -1692,7 +1694,8 @@ pmap_kenter_pa(vaddr_t va, paddr_t pa, v
| MIPS1_PG_WIRED | MIPS1_PG_V | MIPS1_PG_G;
}
kpreempt_disable();
- pte = kvtopte(va);
+ pte = pmap_pte_lookup(pmap_kernel(), va);
+ KASSERT(pte != NULL);
KASSERT(!mips_pg_v(pte->pt_entry));
pte->pt_entry = npte;
pmap_tlb_update_addr(pmap_kernel(), va, npte, false);
@@ -1711,7 +1714,7 @@ pmap_kremove(vaddr_t va, vsize_t len)
(MIPS_HAS_R4K_MMU ? MIPS3_PG_NV | MIPS3_PG_G : MIPS1_PG_NV);
kpreempt_disable();
- pt_entry_t *pte = kvtopte(va);
+ pt_entry_t *pte = pmap_pte_lookup(pmap_kernel(), va);
for (vaddr_t eva = va + len; va < eva; va += PAGE_SIZE, pte++) {
uint32_t pt_entry = pte->pt_entry;
if (!mips_pg_v(pt_entry)) {
@@ -1790,21 +1793,12 @@ pmap_unwire(pmap_t pmap, vaddr_t va)
* Don't need to flush the TLB since PG_WIRED is only in software.
*/
kpreempt_disable();
- if (pmap == pmap_kernel()) {
- /* change entries in kernel pmap */
-#ifdef PARANOIADIAG
- if (va < VM_MIN_KERNEL_ADDRESS || va >= virtual_end)
- panic("pmap_unwire");
-#endif
- pte = kvtopte(va);
- } else {
- pte = pmap_pte_lookup(pmap, va);
+ pte = pmap_pte_lookup(pmap, va);
#ifdef DIAGNOSTIC
- if (pte == NULL)
- panic("pmap_unwire: pmap %p va %#"PRIxVADDR" invalid STE",
- pmap, va);
+ if (pte == NULL)
+ panic("pmap_unwire: pmap %p va %#"PRIxVADDR" invalid STE",
+ pmap, va);
#endif
- }
#ifdef DIAGNOSTIC
if (mips_pg_v(pte->pt_entry) == 0)
@@ -1850,18 +1844,15 @@ pmap_extract(pmap_t pmap, vaddr_t va, pa
#endif
if (va >= mips_virtual_end)
panic("pmap_extract: illegal kernel mapped address %#"PRIxVADDR"", va);
- pte = kvtopte(va);
- kpreempt_disable();
- } else {
- kpreempt_disable();
- if (!(pte = pmap_pte_lookup(pmap, va))) {
+ }
+ kpreempt_disable();
+ if (!(pte = pmap_pte_lookup(pmap, va))) {
#ifdef DEBUG
- if (pmapdebug & PDB_FOLLOW)
- printf("not in segmap\n");
+ if (pmapdebug & PDB_FOLLOW)
+ printf("not in segtab\n");
#endif
- kpreempt_enable();
- return false;
- }
+ kpreempt_enable();
+ return false;
}
if (!mips_pg_v(pte->pt_entry)) {
#ifdef DEBUG
@@ -2051,12 +2042,8 @@ pmap_clear_modify(struct vm_page *pg)
pt_entry_t *pte;
uint32_t pt_entry;
pv_next = pv->pv_next;
- if (pmap == pmap_kernel()) {
- pte = kvtopte(va);
- } else {
- pte = pmap_pte_lookup(pmap, va);
- KASSERT(pte);
- }
+ pte = pmap_pte_lookup(pmap, va);
+ KASSERT(pte);
pt_entry = pte->pt_entry & ~mips_pg_m_bit();
if (pte->pt_entry == pt_entry) {
continue;
@@ -2243,14 +2230,11 @@ again:
pt_entry_t *pte;
uint32_t pt_entry;
- if (pmap == pmap_kernel()) {
- pt_entry = kvtopte(va)->pt_entry;
+ pte = pmap_pte_lookup(pmap, va);
+ if (pte) {
+ pt_entry = pte->pt_entry;
} else {
- pte = pmap_pte_lookup(pmap, va);
- if (pte) {
- pt_entry = pte->pt_entry;
- } else
- pt_entry = 0;
+ pt_entry = 0;
}
if (!mips_pg_v(pt_entry) ||
mips_tlbpfn_to_paddr(pt_entry) !=
@@ -2541,18 +2525,6 @@ pmap_pv_page_free(struct pool *pp, void
uvm_km_pagefree(pg);
}
-pt_entry_t *
-pmap_pte(pmap_t pmap, vaddr_t va)
-{
- pt_entry_t *pte;
-
- if (pmap == pmap_kernel())
- pte = kvtopte(va);
- else
- pte = pmap_pte_lookup(pmap, va);
- return pte;
-}
-
#ifdef MIPS3_PLUS /* XXX mmu XXX */
/*
* Find first virtual address >= *vap that doesn't cause
Index: src/sys/arch/mips/mips/pmap_segtab.c
diff -u src/sys/arch/mips/mips/pmap_segtab.c:1.1.2.17 src/sys/arch/mips/mips/pmap_segtab.c:1.1.2.18
--- src/sys/arch/mips/mips/pmap_segtab.c:1.1.2.17 Mon Jul 9 17:23:37 2012
+++ src/sys/arch/mips/mips/pmap_segtab.c Sat Aug 4 07:20:31 2012
@@ -181,7 +181,7 @@ pmap_segmap(struct pmap *pmap, vaddr_t v
return NULL;
#endif
- return stp->seg_tab[(va >> SEGSHIFT) & (PMAP_SEGTABSIZE - 1)];
+ return stp->seg_tab[(va >> SEGSHIFT) & (NSEGPG - 1)];
}
pt_entry_t *
@@ -214,8 +214,8 @@ pmap_segtab_release(pmap_t pmap, pmap_se
{
pmap_segtab_t *stp = *stp_p;
- KASSERT(((va / vinc) & (PMAP_SEGTABSIZE - 1)) == 0);
- for (size_t i = 0; i < PMAP_SEGTABSIZE; i++, va += vinc) {
+ KASSERT(((va / vinc) & (NSEGPG - 1)) == 0);
+ for (size_t i = 0; i < NSEGPG; i++, va += vinc) {
#ifdef _LP64
if (vinc > NBSEG) {
if (stp->seg_seg[i] != NULL) {
@@ -341,7 +341,7 @@ pmap_segtab_alloc(void)
}
#ifdef PARANOIADIAG
- for (i = 0; i < PMAP_SEGTABSIZE; i++) {
+ for (i = 0; i < NSEGPG; i++) {
if (stp->seg_tab[i] != 0)
panic("pmap_create: pm_segtab.seg_tab[%zu] != 0");
}
@@ -388,14 +388,19 @@ pmap_segtab_activate(struct pmap *pm, st
if (l == curlwp) {
KASSERT(pm == l->l_proc->p_vmspace->vm_map.pmap);
#ifdef _LP64
- l->l_cpu->ci_pmap_segtab = pm->pm_segtab;
if (pm != pmap_kernel()) {
- l->l_cpu->ci_pmap_seg0tab = pm->pm_segtab->seg_seg[0];
+ l->l_cpu->ci_pmap_segtab[0] = pm->pm_segtab;
+ l->l_cpu->ci_pmap_seg0tab[0] = pm->pm_segtab->seg_seg[0];
} else {
- l->l_cpu->ci_pmap_seg0tab = NULL;
+ l->l_cpu->ci_pmap_segtab[0] = NULL;
+ l->l_cpu->ci_pmap_seg0tab[0] = NULL;
}
#else
- l->l_cpu->ci_pmap_seg0tab = pm->pm_segtab;
+ if (pm != pmap_kernel()) {
+ l->l_cpu->ci_pmap_seg0tab[0] = pm->pm_segtab;
+ } else {
+ l->l_cpu->ci_pmap_seg0tab[0] = NULL;
+ }
#endif
}
}
@@ -493,7 +498,7 @@ pmap_pte_reserve(pmap_t pmap, vaddr_t va
#endif
pte = (pt_entry_t *)mips_pmap_map_poolpage(pa);
pt_entry_t ** const pte_p =
- &stp->seg_tab[(va >> SEGSHIFT) & (PMAP_SEGTABSIZE - 1)];
+ &stp->seg_tab[(va >> SEGSHIFT) & (NSEGPG - 1)];
#ifdef MULTIPROCESSOR
pt_entry_t *opte = atomic_cas_ptr(pte_p, NULL, pte);
/*
@@ -514,7 +519,7 @@ pmap_pte_reserve(pmap_t pmap, vaddr_t va
#else
*pte_p = pte;
#endif
- KASSERT(pte == stp->seg_tab[(va >> SEGSHIFT) & (PMAP_SEGTABSIZE - 1)]);
+ KASSERT(pte == stp->seg_tab[(va >> SEGSHIFT) & (NSEGPG - 1)]);
pte += (va >> PGSHIFT) & (NPTEPG - 1);
#ifdef PARANOIADIAG
Index: src/sys/arch/mips/mips/trap.c
diff -u src/sys/arch/mips/mips/trap.c:1.217.12.44 src/sys/arch/mips/mips/trap.c:1.217.12.45
--- src/sys/arch/mips/mips/trap.c:1.217.12.44 Mon Jul 9 17:14:47 2012
+++ src/sys/arch/mips/mips/trap.c Sat Aug 4 07:20:31 2012
@@ -261,7 +261,7 @@ trap(uint32_t status, uint32_t cause, va
kpreempt_disable();
- pte = kvtopte(vaddr);
+ pte = pmap_pte_lookup(pmap_kernel(), vaddr);
pt_entry = pte->pt_entry;
if (!mips_pg_v(pt_entry)) {
panic("ktlbmod: invalid pte");
Index: src/sys/arch/mips/mips/vm_machdep.c
diff -u src/sys/arch/mips/mips/vm_machdep.c:1.121.6.1.2.29 src/sys/arch/mips/mips/vm_machdep.c:1.121.6.1.2.30
--- src/sys/arch/mips/mips/vm_machdep.c:1.121.6.1.2.29 Thu Feb 16 18:58:57 2012
+++ src/sys/arch/mips/mips/vm_machdep.c Sat Aug 4 07:20:31 2012
@@ -165,7 +165,7 @@ cpu_lwp_fork(struct lwp *l1, struct lwp
l2->l_md.md_flags = l1->l_md.md_flags & MDP_FPUSED;
if (!mm_md_direct_mapped_virt(ua2, NULL, NULL)) {
- pt_entry_t * const pte = kvtopte(ua2);
+ pt_entry_t * const pte = pmap_pte_lookup(pmap_kernel(), ua2);
const uint32_t x = (MIPS_HAS_R4K_MMU) ?
(MIPS3_PG_G | MIPS3_PG_RO | MIPS3_PG_WIRED) : MIPS1_PG_G;
@@ -331,8 +331,6 @@ cpu_uarea_remap(struct lwp *l)
void
cpu_swapin(struct lwp *l)
{
- pt_entry_t *pte;
- int i, x;
vaddr_t kva = (vaddr_t) lwp_getpcb(l);
if (mm_md_direct_mapped_virt(kva, NULL, NULL))
@@ -343,11 +341,11 @@ cpu_swapin(struct lwp *l)
* part of the proc struct so cpu_switchto() can quickly map
* in the user struct and kernel stack.
*/
- x = (MIPS_HAS_R4K_MMU) ?
+ uint32_t x = (MIPS_HAS_R4K_MMU) ?
(MIPS3_PG_G | MIPS3_PG_RO | MIPS3_PG_WIRED) :
MIPS1_PG_G;
- pte = kvtopte(kva);
- for (i = 0; i < UPAGES; i++)
+ pt_entry_t * const pte = pmap_pte_lookup(pmap_kernel(), kva);
+ for (size_t i = 0; i < UPAGES; i++)
l->l_md.md_upte[i] = pte[i].pt_entry &~ x;
}
@@ -506,17 +504,16 @@ kvtophys(vaddr_t kva)
return phys;
if (VM_MIN_KERNEL_ADDRESS <= kva && kva < VM_MAX_KERNEL_ADDRESS) {
- pt_entry_t *pte = kvtopte(kva);
- if ((size_t) (pte - Sysmap) >= Sysmapsize) {
- printf("oops: Sysmap overrun, max %d index %zd\n",
- Sysmapsize, pte - Sysmap);
+ pt_entry_t *pte = pmap_pte_lookup(pmap_kernel(), kva);
+ if (pte != NULL) {
+ if (!mips_pg_v(pte->pt_entry)) {
+ printf("%s: pte not valid for %#"PRIxVADDR"\n",
+ __func__, kva);
+ }
+ phys = mips_tlbpfn_to_paddr(pte->pt_entry)
+ | (kva & PGOFSET);
+ return phys;
}
- if (!mips_pg_v(pte->pt_entry)) {
- printf("kvtophys: pte not valid for %#"PRIxVADDR"\n",
- kva);
- }
- phys = mips_tlbpfn_to_paddr(pte->pt_entry) | (kva & PGOFSET);
- return phys;
}
panic("%s: Virtual address %#"PRIxVADDR": cannot map to physical\n",
