ChangeSet 1.2181.2.51, 2005/03/26 21:37:09-08:00, [EMAIL PROTECTED]
[SPARC64]: Handle non-8K PAGE_SIZE better in TLB miss handlers.
The existing code mostly worked, but only for 32-bit
processes.
This caught a serious bug too, pgd_index() was masking
with plain PTRS_PER_PGD instead of (PTRS_PER_PGD - 1).
That has been with us for several years. It is a wonder
that things work at all some times :-)
Signed-off-by: David S. Miller <[EMAIL PROTECTED]>
arch/sparc64/kernel/dtlb_backend.S | 158 ++++++++++++++++++++++++++-------
arch/sparc64/kernel/dtlb_base.S | 2
arch/sparc64/kernel/entry.S | 173 ++++++++++++++++++++++++-------------
arch/sparc64/mm/init.c | 54 +----------
arch/sparc64/mm/tlb.c | 8 -
include/asm-sparc64/cpudata.h | 2
include/asm-sparc64/pgalloc.h | 64 -------------
include/asm-sparc64/pgtable.h | 38 +-------
8 files changed, 253 insertions(+), 246 deletions(-)
diff -Nru a/arch/sparc64/kernel/dtlb_backend.S
b/arch/sparc64/kernel/dtlb_backend.S
--- a/arch/sparc64/kernel/dtlb_backend.S 2005-04-03 21:13:56 -07:00
+++ b/arch/sparc64/kernel/dtlb_backend.S 2005-04-03 21:13:56 -07:00
@@ -10,57 +10,140 @@
#include <asm/mmu_context.h>
#if PAGE_SHIFT == 13
-#define FILL_VALID_SZ_BITS1(r1) \
- sllx %g2, 62, r1
-#define FILL_VALID_SZ_BITS2(r1)
-#define FILL_VALID_SZ_BITS_NOP nop
+#define SZ_BITS _PAGE_SZ8K
#elif PAGE_SHIFT == 16
-#define FILL_VALID_SZ_BITS1(r1) \
- or %g0, 5, r1
-#define FILL_VALID_SZ_BITS2(r1) \
- sllx r1, 61, r1
-#define FILL_VALID_SZ_BITS_NOP
-#else
-#error unsupported PAGE_SIZE
-#endif /* PAGE_SHIFT */
+#define SZ_BITS _PAGE_SZ64K
+#elif PAGE_SHIFT == 19
+#define SZ_BITS _PAGE_SZ512K
+#elif PAGE_SHIFT == 22
+#define SZ_BITS _PAGE_SZ4M
+#endif
+
+#define VALID_SZ_BITS (_PAGE_VALID | SZ_BITS)
#define VPTE_BITS (_PAGE_CP | _PAGE_CV | _PAGE_P )
#define VPTE_SHIFT (PAGE_SHIFT - 3)
-#define TLB_PMD_SHIFT (PAGE_SHIFT - 3 + 3)
-#define TLB_PGD_SHIFT (PMD_BITS + PAGE_SHIFT - 3 + 3)
-#define TLB_PMD_MASK (((1 << PMD_BITS) - 1) << 1)
-#define TLB_PGD_MASK (((1 << (VA_BITS - PAGE_SHIFT - (PAGE_SHIFT -
3) - PMD_BITS)) - 1) << 2)
/* Ways we can get here:
*
* 1) Nucleus loads and stores to/from PA-->VA direct mappings at tl>1.
* 2) Nucleus loads and stores to/from user/kernel window save areas.
* 3) VPTE misses from dtlb_base and itlb_base.
+ *
+ * We need to extract out the PMD and PGDIR indexes from the
+ * linear virtual page table access address. The PTE index
+ * is at the bottom, but we are not concerned with it. Bits
+ * 0 to 2 are clear since each PTE is 8 bytes in size. Each
+ * PMD and PGDIR entry are 4 bytes in size. Thus, this
+ * address looks something like:
+ *
+ * |---------------------------------------------------------------|
+ * | ... | PGDIR index | PMD index | PTE index | |
+ * |---------------------------------------------------------------|
+ * 63 F E D C B A 3 2 0 <- bit nr
+ *
+ * The variable bits above are defined as:
+ * A --> 3 + (PAGE_SHIFT - log2(8))
+ * --> 3 + (PAGE_SHIFT - 3) - 1
+ * (ie. this is "bit 3" + PAGE_SIZE - size of PTE entry in bits - 1)
+ * B --> A + 1
+ * C --> B + (PAGE_SHIFT - log2(4))
+ * --> B + (PAGE_SHIFT - 2) - 1
+ * (ie. this is "bit B" + PAGE_SIZE - size of PMD entry in bits - 1)
+ * D --> C + 1
+ * E --> D + (PAGE_SHIFT - log2(4))
+ * --> D + (PAGE_SHIFT - 2) - 1
+ * (ie. this is "bit D" + PAGE_SIZE - size of PGDIR entry in bits - 1)
+ * F --> E + 1
+ *
+ * (Note how "B" always evalutes to PAGE_SHIFT, all the other constants
+ * cancel out.)
+ *
+ * For 8K PAGE_SIZE (thus, PAGE_SHIFT of 13) the bit numbers are:
+ * A --> 12
+ * B --> 13
+ * C --> 23
+ * D --> 24
+ * E --> 34
+ * F --> 35
+ *
+ * For 64K PAGE_SIZE (thus, PAGE_SHIFT of 16) the bit numbers are:
+ * A --> 15
+ * B --> 16
+ * C --> 29
+ * D --> 30
+ * E --> 43
+ * F --> 44
+ *
+ * Because bits both above and below each PGDIR and PMD index need to
+ * be masked out, and the index can be as long as 14 bits (when using a
+ * 64K PAGE_SIZE, and thus a PAGE_SHIFT of 16), we need 3 instructions
+ * to extract each index out.
+ *
+ * Shifts do not pair very well on UltraSPARC-I, II, IIi, and IIe, so
+ * we try to avoid using them for the entire operation. We could setup
+ * a mask anywhere from bit 31 down to bit 10 using the sethi instruction.
+ *
+ * We need a mask covering bits B --> C and one covering D --> E.
+ * For 8K PAGE_SIZE these masks are 0x00ffe000 and 0x7ff000000.
+ * For 64K PAGE_SIZE these masks are 0x3fff0000 and 0xfffc0000000.
+ * The second in each set cannot be loaded with a single sethi
+ * instruction, because the upper bits are past bit 32. We would
+ * need to use a sethi + a shift.
+ *
+ * For the time being, we use 2 shifts and a simple "and" mask.
+ * We shift left to clear the bits above the index, we shift down
+ * to clear the bits below the index (sans the log2(4 or 8) bits)
+ * and a mask to clear the log2(4 or 8) bits. We need therefore
+ * define 4 shift counts, all of which are relative to PAGE_SHIFT.
+ *
+ * Although unsupportable for other reasons, this does mean that
+ * 512K and 4MB page sizes would be generaally supported by the
+ * kernel. (ELF binaries would break with > 64K PAGE_SIZE since
+ * the sections are only aligned that strongly).
+ *
+ * The operations performed for extraction are thus:
+ *
+ * ((X << FOO_SHIFT_LEFT) >> FOO_SHIFT_RIGHT) & ~0x3
+ *
*/
+#define A (3 + (PAGE_SHIFT - 3) - 1)
+#define B (A + 1)
+#define C (B + (PAGE_SHIFT - 2) - 1)
+#define D (C + 1)
+#define E (D + (PAGE_SHIFT - 2) - 1)
+#define F (E + 1)
+
+#define PMD_SHIFT_LEFT (64 - D)
+#define PMD_SHIFT_RIGHT (64 - (D - B) - 2)
+#define PGDIR_SHIFT_LEFT (64 - F)
+#define PGDIR_SHIFT_RIGHT (64 - (F - D) - 2)
+#define LOW_MASK_BITS 0x3
+
/* TLB1 ** ICACHE line 1: tl1 DTLB and quick VPTE miss */
ldxa [%g1 + %g1] ASI_DMMU, %g4 ! Get TAG_ACCESS
add %g3, %g3, %g5 ! Compute VPTE base
cmp %g4, %g5 ! VPTE miss?
bgeu,pt %xcc, 1f ! Continue here
- andcc %g4, TAG_CONTEXT_BITS, %g5 ! From Nucleus? (for
tl0 miss)
- ba,pt %xcc, from_tl1_trap ! Fall to tl0 miss
- rdpr %tl, %g5 ! For tl0 miss TL==3
test
+ andcc %g4, TAG_CONTEXT_BITS, %g5 ! tl0 miss Nucleus test
+ ba,a,pt %xcc, from_tl1_trap ! Fall to tl0 miss
1: sllx %g6, VPTE_SHIFT, %g4 ! Position TAG_ACCESS
+ or %g4, %g5, %g4 ! Prepare TAG_ACCESS
/* TLB1 ** ICACHE line 2: Quick VPTE miss */
- or %g4, %g5, %g4 ! Prepare TAG_ACCESS
mov TSB_REG, %g1 ! Grab TSB reg
ldxa [%g1] ASI_DMMU, %g5 ! Doing PGD caching?
- srlx %g6, (TLB_PMD_SHIFT - 1), %g1 ! Position PMD offset
+ sllx %g6, PMD_SHIFT_LEFT, %g1 ! Position PMD offset
be,pn %xcc, sparc64_vpte_nucleus ! Is it from Nucleus?
- and %g1, TLB_PMD_MASK, %g1 ! Mask PMD offset bits
+ srlx %g1, PMD_SHIFT_RIGHT, %g1 ! Mask PMD offset bits
brnz,pt %g5, sparc64_vpte_continue ! Yep, go like smoke
- add %g1, %g1, %g1 ! Position PMD offset
some more
+ andn %g1, LOW_MASK_BITS, %g1 ! Final PMD mask
+ sllx %g6, PGDIR_SHIFT_LEFT, %g5 ! Position PGD offset
/* TLB1 ** ICACHE line 3: Quick VPTE miss */
- srlx %g6, (TLB_PGD_SHIFT - 2), %g5 ! Position PGD offset
- and %g5, TLB_PGD_MASK, %g5 ! Mask PGD offset
+ srlx %g5, PGDIR_SHIFT_RIGHT, %g5 ! Mask PGD offset bits
+ andn %g5, LOW_MASK_BITS, %g5 ! Final PGD mask
lduwa [%g7 + %g5] ASI_PHYS_USE_EC, %g5! Load PGD
brz,pn %g5, vpte_noent ! Valid?
sparc64_kpte_continue:
@@ -71,23 +154,28 @@
brz,pn %g5, vpte_noent ! Valid?
/* TLB1 ** ICACHE line 4: Quick VPTE miss */
- FILL_VALID_SZ_BITS1(%g1) ! Put _PAGE_VALID into
%g1
- FILL_VALID_SZ_BITS2(%g1) ! Put _PAGE_VALID into
%g1
+ mov (VALID_SZ_BITS >> 61), %g1 ! upper vpte into %g1
+ sllx %g1, 61, %g1 ! finish calc
or %g5, VPTE_BITS, %g5 ! Prepare VPTE data
or %g5, %g1, %g5 ! ...
mov TLB_SFSR, %g1 ! Restore %g1 value
stxa %g5, [%g0] ASI_DTLB_DATA_IN ! Load VPTE into TLB
stxa %g4, [%g1 + %g1] ASI_DMMU ! Restore previous
TAG_ACCESS
retry ! Load PTE once again
- FILL_VALID_SZ_BITS_NOP
+#undef SZ_BITS
+#undef VALID_SZ_BITS
#undef VPTE_SHIFT
-#undef TLB_PMD_SHIFT
-#undef TLB_PGD_SHIFT
#undef VPTE_BITS
-#undef TLB_PMD_MASK
-#undef TLB_PGD_MASK
-#undef FILL_VALID_SZ_BITS1
-#undef FILL_VALID_SZ_BITS2
-#undef FILL_VALID_SZ_BITS_NOP
+#undef A
+#undef B
+#undef C
+#undef D
+#undef E
+#undef F
+#undef PMD_SHIFT_LEFT
+#undef PMD_SHIFT_RIGHT
+#undef PGDIR_SHIFT_LEFT
+#undef PGDIR_SHIFT_RIGHT
+#undef LOW_MASK_BITS
diff -Nru a/arch/sparc64/kernel/dtlb_base.S b/arch/sparc64/kernel/dtlb_base.S
--- a/arch/sparc64/kernel/dtlb_base.S 2005-04-03 21:13:56 -07:00
+++ b/arch/sparc64/kernel/dtlb_base.S 2005-04-03 21:13:56 -07:00
@@ -68,8 +68,8 @@
/* DTLB ** ICACHE line 1: Quick user TLB misses */
ldxa [%g1 + %g1] ASI_DMMU, %g4 ! Get TAG_ACCESS
andcc %g4, TAG_CONTEXT_BITS, %g0 ! From Nucleus?
- mov 1, %g5 ! For TL==3 test
from_tl1_trap:
+ rdpr %tl, %g5 ! For TL==3 test
CREATE_VPTE_OFFSET1(%g4, %g6) ! Create VPTE offset
be,pn %xcc, 3f ! Yep, special
processing
CREATE_VPTE_OFFSET2(%g4, %g6) ! Create VPTE offset
diff -Nru a/arch/sparc64/kernel/entry.S b/arch/sparc64/kernel/entry.S
--- a/arch/sparc64/kernel/entry.S 2005-04-03 21:13:56 -07:00
+++ b/arch/sparc64/kernel/entry.S 2005-04-03 21:13:56 -07:00
@@ -38,97 +38,150 @@
* range (note that this is only possible for instruction miss, data misses to
* obp range do not use vpte). If so, go back directly to the faulting address.
* This is because we want to read the tpc, otherwise we have no way of knowing
- * the 8k aligned faulting address if we are using >8k kernel pagesize. This
also
- * ensures no vpte range addresses are dropped into tlb while obp is executing
- * (see inherit_locked_prom_mappings() rant).
+ * the 8k aligned faulting address if we are using >8k kernel pagesize. This
+ * also ensures no vpte range addresses are dropped into tlb while obp is
+ * executing (see inherit_locked_prom_mappings() rant).
*/
sparc64_vpte_nucleus:
+ /* Load 0xf0000000, which is LOW_OBP_ADDRESS. */
mov 0xf, %g5
- sllx %g5, 28, %g5 ! Load 0xf0000000
- cmp %g4, %g5 ! Is addr >=
LOW_OBP_ADDRESS?
+ sllx %g5, 28, %g5
+
+ /* Is addr >= LOW_OBP_ADDRESS? */
+ cmp %g4, %g5
blu,pn %xcc, sparc64_vpte_patchme1
mov 0x1, %g5
- sllx %g5, 32, %g5 ! Load 0x100000000
- cmp %g4, %g5 ! Is addr <
HI_OBP_ADDRESS?
+
+ /* Load 0x100000000, which is HI_OBP_ADDRESS. */
+ sllx %g5, 32, %g5
+
+ /* Is addr < HI_OBP_ADDRESS? */
+ cmp %g4, %g5
blu,pn %xcc, obp_iaddr_patch
nop
+
+ /* These two instructions are patched by paginig_init(). */
sparc64_vpte_patchme1:
- sethi %hi(0), %g5 ! This has to be patched
+ sethi %hi(0), %g5
sparc64_vpte_patchme2:
- or %g5, %lo(0), %g5 ! This is patched too
- ba,pt %xcc, sparc64_kpte_continue ! Part of dtlb_backend
- add %g1, %g1, %g1 ! Finish PMD offset
adjustment
+ or %g5, %lo(0), %g5
+
+ /* With kernel PGD in %g5, branch back into dtlb_backend. */
+ ba,pt %xcc, sparc64_kpte_continue
+ andn %g1, 0x3, %g1 /* Finish PMD offset adjustment. */
vpte_noent:
- mov TLB_SFSR, %g1 ! Restore %g1 value
- stxa %g4, [%g1 + %g1] ASI_DMMU ! Restore previous
TAG_ACCESS
- done ! Slick trick
+ /* Restore previous TAG_ACCESS, %g5 is zero, and we will
+ * skip over the trap instruction so that the top level
+ * TLB miss handler will thing this %g5 value is just an
+ * invalid PTE, thus branching to full fault processing.
+ */
+ mov TLB_SFSR, %g1
+ stxa %g4, [%g1 + %g1] ASI_DMMU
+ done
.globl obp_iaddr_patch
- .globl obp_daddr_patch
-
obp_iaddr_patch:
- sethi %hi(0), %g5 ! This and following is
patched
- or %g5, %lo(0), %g5 ! g5 now holds obp pmd
base physaddr
- wrpr %g0, 1, %tl ! Behave as if we are
at TL0
- rdpr %tpc, %g4 ! Find original
faulting iaddr
- srlx %g4, 13, %g4 ! Throw out context bits
- sllx %g4, 13, %g4 ! g4 has vpn + ctx0 now
- mov TLB_SFSR, %g1 ! Restore %g1 value
- stxa %g4, [%g1 + %g1] ASI_IMMU ! Restore previous
TAG_ACCESS
- srlx %g4, 23, %g6 ! Find pmd number
- and %g6, 0x7ff, %g6 ! Find pmd number
- sllx %g6, 2, %g6 ! Find pmd offset
- lduwa [%g5 + %g6] ASI_PHYS_USE_EC, %g5! Load pmd, ie
pagetable physaddr
- brz,pn %g5, longpath ! Kill the PROM ? :-)
- sllx %g5, 11, %g5 ! Shift into place
- srlx %g4, 13, %g6 ! find pte number in
pagetable
- and %g6, 0x3ff, %g6 ! find pte number in
pagetable
- sllx %g6, 3, %g6 ! find pte offset in
pagetable
- ldxa [%g5 + %g6] ASI_PHYS_USE_EC, %g5! Load pte
- brgez,pn %g5, longpath ! Kill the PROM ? :-)
+ /* These two instructions patched by inherit_prom_mappings(). */
+ sethi %hi(0), %g5
+ or %g5, %lo(0), %g5
+
+ /* Behave as if we are at TL0. */
+ wrpr %g0, 1, %tl
+ rdpr %tpc, %g4 /* Find original faulting iaddr */
+ srlx %g4, 13, %g4 /* Throw out context bits */
+ sllx %g4, 13, %g4 /* g4 has vpn + ctx0 now */
+
+ /* Restore previous TAG_ACCESS. */
+ mov TLB_SFSR, %g1
+ stxa %g4, [%g1 + %g1] ASI_IMMU
+
+ /* Get PMD offset. */
+ srlx %g4, 23, %g6
+ and %g6, 0x7ff, %g6
+ sllx %g6, 2, %g6
+
+ /* Load PMD, is it valid? */
+ lduwa [%g5 + %g6] ASI_PHYS_USE_EC, %g5
+ brz,pn %g5, longpath
+ sllx %g5, 11, %g5
+
+ /* Get PTE offset. */
+ srlx %g4, 13, %g6
+ and %g6, 0x3ff, %g6
+ sllx %g6, 3, %g6
+
+ /* Load PTE. */
+ ldxa [%g5 + %g6] ASI_PHYS_USE_EC, %g5
+ brgez,pn %g5, longpath
nop
- stxa %g5, [%g0] ASI_ITLB_DATA_IN ! put into tlb
- retry ! go back to original
fault
+ /* TLB load and return from trap. */
+ stxa %g5, [%g0] ASI_ITLB_DATA_IN
+ retry
+
+ .globl obp_daddr_patch
obp_daddr_patch:
- sethi %hi(0), %g5 ! This and following is
patched
- or %g5, %lo(0), %g5 ! g5 now holds obp pmd
base physaddr
- srlx %g4, 23, %g6 ! Find pmd number
- and %g6, 0x7ff, %g6 ! Find pmd number
- sllx %g6, 2, %g6 ! Find pmd offset
- lduwa [%g5 + %g6] ASI_PHYS_USE_EC, %g5! Load pmd, ie
pagetable physaddr
+ /* These two instructions patched by inherit_prom_mappings(). */
+ sethi %hi(0), %g5
+ or %g5, %lo(0), %g5
+
+ /* Get PMD offset. */
+ srlx %g4, 23, %g6
+ and %g6, 0x7ff, %g6
+ sllx %g6, 2, %g6
+
+ /* Load PMD, is it valid? */
+ lduwa [%g5 + %g6] ASI_PHYS_USE_EC, %g5
brz,pn %g5, longpath
- sllx %g5, 11, %g5 ! Shift into place
- srlx %g4, 13, %g6 ! find pte number in
pagetable
- and %g6, 0x3ff, %g6 ! find pte number in
pagetable
- sllx %g6, 3, %g6 ! find pte offset in
pagetable
- ldxa [%g5 + %g6] ASI_PHYS_USE_EC, %g5! Load pte
+ sllx %g5, 11, %g5
+
+ /* Get PTE offset. */
+ srlx %g4, 13, %g6
+ and %g6, 0x3ff, %g6
+ sllx %g6, 3, %g6
+
+ /* Load PTE. */
+ ldxa [%g5 + %g6] ASI_PHYS_USE_EC, %g5
brgez,pn %g5, longpath
nop
- stxa %g5, [%g0] ASI_DTLB_DATA_IN ! put into tlb
+
+ /* TLB load and return from trap. */
+ stxa %g5, [%g0] ASI_DTLB_DATA_IN
retry
/*
- * On a first level data miss, check whether this is to the OBP range (note
that
- * such accesses can be made by prom, as well as by kernel using
prom_getproperty
- * on "address"), and if so, do not use vpte access ... rather, use information
- * saved during inherit_prom_mappings() using 8k pagesize.
+ * On a first level data miss, check whether this is to the OBP range (note
+ * that such accesses can be made by prom, as well as by kernel using
+ * prom_getproperty on "address"), and if so, do not use vpte access ...
+ * rather, use information saved during inherit_prom_mappings() using 8k
+ * pagesize.
*/
kvmap:
+ /* Load 0xf0000000, which is LOW_OBP_ADDRESS. */
mov 0xf, %g5
- sllx %g5, 28, %g5 ! Load 0xf0000000
- cmp %g4, %g5 ! Is addr >=
LOW_OBP_ADDRESS?
+ sllx %g5, 28, %g5
+
+ /* Is addr >= LOW_OBP_ADDRESS? */
+ cmp %g4, %g5
blu,pn %xcc, vmalloc_addr
mov 0x1, %g5
- sllx %g5, 32, %g5 ! Load 0x100000000
- cmp %g4, %g5 ! Is addr <
HI_OBP_ADDRESS?
+
+ /* Load 0x100000000, which is HI_OBP_ADDRESS. */
+ sllx %g5, 32, %g5
+
+ /* Is addr < HI_OBP_ADDRESS? */
+ cmp %g4, %g5
blu,pn %xcc, obp_daddr_patch
nop
-vmalloc_addr: ! vmalloc addr accessed
- ldxa [%g3 + %g6] ASI_N, %g5 ! Yep, load k-vpte
- brgez,pn %g5, longpath ! Valid, load into TLB
+
+vmalloc_addr:
+ /* If we get here, a vmalloc addr accessed, load kernel VPTE. */
+ ldxa [%g3 + %g6] ASI_N, %g5
+ brgez,pn %g5, longpath
nop
+
+ /* PTE is valid, load into TLB and return from trap. */
stxa %g5, [%g0] ASI_DTLB_DATA_IN ! Reload TLB
retry
diff -Nru a/arch/sparc64/mm/init.c b/arch/sparc64/mm/init.c
--- a/arch/sparc64/mm/init.c 2005-04-03 21:13:56 -07:00
+++ b/arch/sparc64/mm/init.c 2005-04-03 21:13:56 -07:00
@@ -85,40 +85,14 @@
preempt_disable();
if (pgtable_cache_size > PGT_CACHE_HIGH) {
do {
-#ifdef CONFIG_SMP
if (pgd_quicklist)
free_pgd_slow(get_pgd_fast());
-#endif
if (pte_quicklist[0])
free_pte_slow(pte_alloc_one_fast(NULL, 0));
if (pte_quicklist[1])
free_pte_slow(pte_alloc_one_fast(NULL, 1 <<
(PAGE_SHIFT + 10)));
} while (pgtable_cache_size > PGT_CACHE_LOW);
}
-#ifndef CONFIG_SMP
- if (pgd_cache_size > PGT_CACHE_HIGH / 4) {
- struct page *page, *page2;
- for (page2 = NULL, page = (struct page *)pgd_quicklist; page;)
{
- if ((unsigned long)page->lru.prev == 3) {
- if (page2)
- page2->lru.next = page->lru.next;
- else
- pgd_quicklist = (void *)
page->lru.next;
- pgd_cache_size -= 2;
- __free_page(page);
- if (page2)
- page = (struct page *)page2->lru.next;
- else
- page = (struct page *)pgd_quicklist;
- if (pgd_cache_size <= PGT_CACHE_LOW / 4)
- break;
- continue;
- }
- page2 = page;
- page = (struct page *)page->lru.next;
- }
- }
-#endif
preempt_enable();
}
@@ -219,6 +193,7 @@
put_cpu();
}
+
if (get_thread_fault_code())
__update_mmu_cache(vma->vm_mm->context & TAG_CONTEXT_BITS,
address, pte, get_thread_fault_code());
@@ -281,9 +256,6 @@
printk("%ld pages of RAM\n", num_physpages);
printk("%d free pages\n", nr_free_pages());
printk("%d pages in page table cache\n",pgtable_cache_size);
-#ifndef CONFIG_SMP
- printk("%d entries in page dir cache\n",pgd_cache_size);
-#endif
}
void mmu_info(struct seq_file *m)
@@ -392,10 +364,10 @@
n = n / sizeof(*trans);
/*
- * The obp translations are saved based on 8k pagesize, since obp can
use
- * a mixture of pagesizes. Misses to the 0xf0000000 - 0x100000000, ie
obp
- * range, are handled in entry.S and do not use the vpte scheme (see
rant
- * in inherit_locked_prom_mappings()).
+ * The obp translations are saved based on 8k pagesize, since obp can
+ * use a mixture of pagesizes. Misses to the 0xf0000000 - 0x100000000,
+ * ie obp range, are handled in entry.S and do not use the vpte scheme
+ * (see rant in inherit_locked_prom_mappings()).
*/
#define OBP_PMD_SIZE 2048
prompmd = __alloc_bootmem(OBP_PMD_SIZE, OBP_PMD_SIZE, bootmap_base);
@@ -1703,22 +1675,6 @@
datapages = PAGE_ALIGN(datapages) >> PAGE_SHIFT;
initpages = (((unsigned long) __init_end) - ((unsigned long)
__init_begin));
initpages = PAGE_ALIGN(initpages) >> PAGE_SHIFT;
-
-#ifndef CONFIG_SMP
- {
- /* Put empty_pg_dir on pgd_quicklist */
- extern pgd_t empty_pg_dir[1024];
- unsigned long addr = (unsigned long)empty_pg_dir;
- unsigned long alias_base = kern_base + PAGE_OFFSET -
- (long)(KERNBASE);
-
- memset(empty_pg_dir, 0, sizeof(empty_pg_dir));
- addr += alias_base;
- free_pgd_fast((pgd_t *)addr);
- num_physpages++;
- totalram_pages++;
- }
-#endif
printk("Memory: %uk available (%ldk kernel code, %ldk data, %ldk init)
[%016lx,%016lx]\n",
nr_free_pages() << (PAGE_SHIFT-10),
diff -Nru a/arch/sparc64/mm/tlb.c b/arch/sparc64/mm/tlb.c
--- a/arch/sparc64/mm/tlb.c 2005-04-03 21:13:56 -07:00
+++ b/arch/sparc64/mm/tlb.c 2005-04-03 21:13:56 -07:00
@@ -73,6 +73,7 @@
}
no_cache_flush:
+
if (mp->tlb_frozen)
return;
@@ -149,11 +150,4 @@
}
if (nr)
flush_tlb_pending();
-}
-
-unsigned long __ptrs_per_pmd(void)
-{
- if (test_thread_flag(TIF_32BIT))
- return (1UL << (32 - (PAGE_SHIFT-3) - PAGE_SHIFT));
- return REAL_PTRS_PER_PMD;
}
diff -Nru a/include/asm-sparc64/cpudata.h b/include/asm-sparc64/cpudata.h
--- a/include/asm-sparc64/cpudata.h 2005-04-03 21:13:56 -07:00
+++ b/include/asm-sparc64/cpudata.h 2005-04-03 21:13:56 -07:00
@@ -19,7 +19,7 @@
/* Dcache line 2 */
unsigned int pgcache_size;
- unsigned int pgdcache_size;
+ unsigned int __pad1;
unsigned long *pte_cache[2];
unsigned long *pgd_cache;
} cpuinfo_sparc;
diff -Nru a/include/asm-sparc64/pgalloc.h b/include/asm-sparc64/pgalloc.h
--- a/include/asm-sparc64/pgalloc.h 2005-04-03 21:13:56 -07:00
+++ b/include/asm-sparc64/pgalloc.h 2005-04-03 21:13:56 -07:00
@@ -22,74 +22,12 @@
unsigned long *pgd_cache;
unsigned long *pte_cache[2];
unsigned int pgcache_size;
- unsigned int pgdcache_size;
} pgt_quicklists;
#endif
#define pgd_quicklist (pgt_quicklists.pgd_cache)
#define pmd_quicklist ((unsigned long *)0)
#define pte_quicklist (pgt_quicklists.pte_cache)
#define pgtable_cache_size (pgt_quicklists.pgcache_size)
-#define pgd_cache_size (pgt_quicklists.pgdcache_size)
-
-#ifndef CONFIG_SMP
-
-static __inline__ void free_pgd_fast(pgd_t *pgd)
-{
- struct page *page = virt_to_page(pgd);
-
- preempt_disable();
- if (!page->lru.prev) {
- page->lru.next = (void *) pgd_quicklist;
- pgd_quicklist = (unsigned long *)page;
- }
- page->lru.prev = (void *)
- (((unsigned long)page->lru.prev) |
- (((unsigned long)pgd & (PAGE_SIZE / 2)) ? 2 : 1));
- pgd_cache_size++;
- preempt_enable();
-}
-
-static __inline__ pgd_t *get_pgd_fast(void)
-{
- struct page *ret;
-
- preempt_disable();
- if ((ret = (struct page *)pgd_quicklist) != NULL) {
- unsigned long mask = (unsigned long)ret->lru.prev;
- unsigned long off = 0;
-
- if (mask & 1)
- mask &= ~1;
- else {
- off = PAGE_SIZE / 2;
- mask &= ~2;
- }
- ret->lru.prev = (void *) mask;
- if (!mask)
- pgd_quicklist = (unsigned long *)ret->lru.next;
- ret = (struct page *)(__page_address(ret) + off);
- pgd_cache_size--;
- preempt_enable();
- } else {
- struct page *page;
-
- preempt_enable();
- page = alloc_page(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO);
- if (page) {
- ret = (struct page *)page_address(page);
- page->lru.prev = (void *) 2UL;
-
- preempt_disable();
- page->lru.next = (void *) pgd_quicklist;
- pgd_quicklist = (unsigned long *)page;
- pgd_cache_size++;
- preempt_enable();
- }
- }
- return (pgd_t *)ret;
-}
-
-#else /* CONFIG_SMP */
static __inline__ void free_pgd_fast(pgd_t *pgd)
{
@@ -123,8 +61,6 @@
{
free_page((unsigned long)pgd);
}
-
-#endif /* CONFIG_SMP */
#ifdef DCACHE_ALIASING_POSSIBLE
#define VPTE_COLOR(address) (((address) >> (PAGE_SHIFT + 10)) & 1UL)
diff -Nru a/include/asm-sparc64/pgtable.h b/include/asm-sparc64/pgtable.h
--- a/include/asm-sparc64/pgtable.h 2005-04-03 21:13:56 -07:00
+++ b/include/asm-sparc64/pgtable.h 2005-04-03 21:13:56 -07:00
@@ -59,44 +59,24 @@
#define PMD_SHIFT (PAGE_SHIFT + (PAGE_SHIFT-3))
#define PMD_SIZE (1UL << PMD_SHIFT)
#define PMD_MASK (~(PMD_SIZE-1))
-#define PMD_BITS 11
+#define PMD_BITS (PAGE_SHIFT - 2)
/* PGDIR_SHIFT determines what a third-level page table entry can map */
#define PGDIR_SHIFT (PAGE_SHIFT + (PAGE_SHIFT-3) + PMD_BITS)
#define PGDIR_SIZE (1UL << PGDIR_SHIFT)
#define PGDIR_MASK (~(PGDIR_SIZE-1))
+#define PGDIR_BITS (PAGE_SHIFT - 2)
#ifndef __ASSEMBLY__
#include <linux/sched.h>
/* Entries per page directory level. */
-#define PTRS_PER_PTE (1UL << (PAGE_SHIFT-3))
-
-/* We the first one in this file, what we export to the kernel
- * is different so we can optimize correctly for 32-bit tasks.
- */
-#define REAL_PTRS_PER_PMD (1UL << PMD_BITS)
-
-/* This is gross, but unless we do this gcc retests the
- * thread flag every interation in pmd traversal loops.
- */
-extern unsigned long __ptrs_per_pmd(void) __attribute_const__;
-#define PTRS_PER_PMD __ptrs_per_pmd()
-
-/*
- * We cannot use the top address range because VPTE table lives there. This
- * formula finds the total legal virtual space in the processor, subtracts the
- * vpte size, then aligns it to the number of bytes mapped by one pgde, and
- * thus calculates the number of pgdes needed.
- */
-#define PTRS_PER_PGD (((1UL << VA_BITS) - VPTE_SIZE + (1UL << (PAGE_SHIFT + \
- (PAGE_SHIFT-3) + PMD_BITS)) - 1) / (1UL << (PAGE_SHIFT + \
- (PAGE_SHIFT-3) + PMD_BITS)))
+#define PTRS_PER_PTE (1UL << (PAGE_SHIFT-3))
+#define PTRS_PER_PMD (1UL << PMD_BITS)
+#define PTRS_PER_PGD (1UL << PGDIR_BITS)
/* Kernel has a separate 44bit address space. */
-#define USER_PTRS_PER_PGD ((const int)(test_thread_flag(TIF_32BIT)) ? \
- (1) : (PTRS_PER_PGD))
#define FIRST_USER_PGD_NR 0
#define pte_ERROR(e) __builtin_trap()
@@ -235,8 +215,8 @@
/* PFNs are real physical page numbers. However, mem_map only begins to record
* per-page information starting at pfn_base. This is to handle systems where
- * the first physical page in the machine is at some huge physical address,
such
- * as 4GB. This is common on a partitioned E10000, for example.
+ * the first physical page in the machine is at some huge physical address,
+ * such as 4GB. This is common on a partitioned E10000, for example.
*/
#define pfn_pte(pfn, prot) \
@@ -307,7 +287,7 @@
#define pte_mkdirty(pte) (__pte(pte_val(pte) | _PAGE_MODIFIED | _PAGE_W))
/* to find an entry in a page-table-directory. */
-#define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD))
+#define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD -
1))
#define pgd_offset(mm, address) ((mm)->pgd + pgd_index(address))
/* to find an entry in a kernel page-table-directory */
@@ -321,7 +301,7 @@
/* Find an entry in the second-level page table.. */
#define pmd_offset(pudp, address) \
((pmd_t *) pud_page(*(pudp)) + \
- (((address) >> PMD_SHIFT) & (REAL_PTRS_PER_PMD-1)))
+ (((address) >> PMD_SHIFT) & (PTRS_PER_PMD-1)))
/* Find an entry in the third-level page table.. */
#define pte_index(dir, address) \
-
To unsubscribe from this list: send the line "unsubscribe bk-commits-head" in
the body of a message to [EMAIL PROTECTED]
More majordomo info at http://vger.kernel.org/majordomo-info.html
