Author: nwhitehorn
Date: Thu Sep 16 03:46:17 2010
New Revision: 212722
URL: http://svn.freebsd.org/changeset/base/212722
Log:
Split the SLB mirror cache into two kinds of object, one for kernel maps
which are similar to the previous ones, and one for user maps, which
are arrays of pointers into the SLB tree. This changes makes user SLB
updates atomic, closing a window for memory corruption. While here,
rearrange the allocation functions to make context switches faster.
Modified:
head/sys/powerpc/aim/mmu_oea64.c
head/sys/powerpc/aim/slb.c
head/sys/powerpc/aim/trap.c
head/sys/powerpc/aim/trap_subr64.S
head/sys/powerpc/include/pcpu.h
head/sys/powerpc/include/pmap.h
head/sys/powerpc/include/sr.h
Modified: head/sys/powerpc/aim/mmu_oea64.c
==============================================================================
--- head/sys/powerpc/aim/mmu_oea64.c Thu Sep 16 02:59:25 2010
(r212721)
+++ head/sys/powerpc/aim/mmu_oea64.c Thu Sep 16 03:46:17 2010
(r212722)
@@ -838,7 +838,7 @@ moea64_bootstrap_slb_prefault(vm_offset_
if (large)
entry.slbv |= SLBV_L;
- slb_insert(kernel_pmap, cache, &entry);
+ slb_insert_kernel(entry.slbe, entry.slbv);
}
#endif
@@ -2099,6 +2099,7 @@ moea64_pinit(mmu_t mmu, pmap_t pmap)
pmap->pm_slb_tree_root = slb_alloc_tree();
pmap->pm_slb = slb_alloc_user_cache();
+ pmap->pm_slb_len = 0;
}
#else
void
Modified: head/sys/powerpc/aim/slb.c
==============================================================================
--- head/sys/powerpc/aim/slb.c Thu Sep 16 02:59:25 2010 (r212721)
+++ head/sys/powerpc/aim/slb.c Thu Sep 16 03:46:17 2010 (r212722)
@@ -263,13 +263,14 @@ va_to_vsid(pmap_t pm, vm_offset_t va)
entry = user_va_to_slb_entry(pm, va);
if (entry == NULL)
- return (allocate_vsid(pm, (uintptr_t)va >> ADDR_SR_SHFT, 0));
+ return (allocate_user_vsid(pm,
+ (uintptr_t)va >> ADDR_SR_SHFT, 0));
return ((entry->slbv & SLBV_VSID_MASK) >> SLBV_VSID_SHIFT);
}
uint64_t
-allocate_vsid(pmap_t pm, uint64_t esid, int large)
+allocate_user_vsid(pmap_t pm, uint64_t esid, int large)
{
uint64_t vsid, slbv;
struct slbtnode *ua, *next, *inter;
@@ -327,7 +328,7 @@ allocate_vsid(pmap_t pm, uint64_t esid,
* SLB mapping, so pre-spill this entry.
*/
eieio();
- slb_insert(pm, pm->pm_slb, slb);
+ slb_insert_user(pm, slb);
return (vsid);
}
@@ -410,57 +411,68 @@ slb_alloc_tree(void)
(slbe & SLBE_ESID_MASK) > 16*SEGMENT_LENGTH) || \
(slbe & SLBE_ESID_MASK) > VM_MAX_KERNEL_ADDRESS)
void
-slb_insert(pmap_t pm, struct slb *slbcache, struct slb *slb_entry)
+slb_insert_kernel(uint64_t slbe, uint64_t slbv)
{
- uint64_t slbe, slbv;
- int i, j, to_spill;
+ struct slb *slbcache;
+ int i, j;
/* We don't want to be preempted while modifying the kernel map */
critical_enter();
- to_spill = -1;
- slbv = slb_entry->slbv;
- slbe = slb_entry->slbe;
+ slbcache = PCPU_GET(slb);
- /* Hunt for a likely candidate */
+ /* Check for an unused slot, abusing the USER_SR slot as a full flag */
+ if (slbcache[USER_SR].slbe == 0) {
+ for (i = 0; i < USER_SR; i++) {
+ if (!(slbcache[i].slbe & SLBE_VALID))
+ goto fillkernslb;
+ }
+
+ if (i == USER_SR)
+ slbcache[USER_SR].slbe = 1;
+ }
for (i = mftb() % 64, j = 0; j < 64; j++, i = (i+1) % 64) {
- if (pm == kernel_pmap && i == USER_SR)
- continue;
+ if (i == USER_SR)
+ continue;
- if (!(slbcache[i].slbe & SLBE_VALID)) {
- to_spill = i;
+ if (SLB_SPILLABLE(slbcache[i].slbe))
break;
- }
-
- if (to_spill < 0 && (pm != kernel_pmap ||
- SLB_SPILLABLE(slbcache[i].slbe)))
- to_spill = i;
- }
-
- if (to_spill < 0)
- panic("SLB spill on ESID %#lx, but no available candidates!\n",
- (slbe & SLBE_ESID_MASK) >> SLBE_ESID_SHIFT);
-
- if (slbcache[to_spill].slbe & SLBE_VALID) {
- /* Invalidate this first to avoid races */
- slbcache[to_spill].slbe = 0;
- mb();
}
- slbcache[to_spill].slbv = slbv;
- slbcache[to_spill].slbe = slbe | (uint64_t)to_spill;
+
+ KASSERT(j < 64, ("All kernel SLB slots locked!"));
+
+fillkernslb:
+ slbcache[i].slbv = slbv;
+ slbcache[i].slbe = slbe | (uint64_t)i;
/* If it is for this CPU, put it in the SLB right away */
- if (pm == kernel_pmap && pmap_bootstrapped) {
+ if (pmap_bootstrapped) {
/* slbie not required */
__asm __volatile ("slbmte %0, %1" ::
- "r"(slbcache[to_spill].slbv),
- "r"(slbcache[to_spill].slbe));
+ "r"(slbcache[i].slbv), "r"(slbcache[i].slbe));
}
critical_exit();
}
+void
+slb_insert_user(pmap_t pm, struct slb *slb)
+{
+ int i;
+
+ PMAP_LOCK_ASSERT(pm, MA_OWNED);
+
+ if (pm->pm_slb_len < 64) {
+ i = pm->pm_slb_len;
+ pm->pm_slb_len++;
+ } else {
+ i = mftb() % 64;
+ }
+
+ /* Note that this replacement is atomic with respect to trap_subr */
+ pm->pm_slb[i] = slb;
+}
static void
slb_zone_init(void *dummy)
@@ -468,18 +480,18 @@ slb_zone_init(void *dummy)
slbt_zone = uma_zcreate("SLB tree node", sizeof(struct slbtnode),
NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM);
- slb_cache_zone = uma_zcreate("SLB cache", 64*sizeof(struct slb),
+ slb_cache_zone = uma_zcreate("SLB cache", 64*sizeof(struct slb *),
NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM);
}
-struct slb *
+struct slb **
slb_alloc_user_cache(void)
{
return (uma_zalloc(slb_cache_zone, M_ZERO));
}
void
-slb_free_user_cache(struct slb *slb)
+slb_free_user_cache(struct slb **slb)
{
uma_zfree(slb_cache_zone, slb);
}
Modified: head/sys/powerpc/aim/trap.c
==============================================================================
--- head/sys/powerpc/aim/trap.c Thu Sep 16 02:59:25 2010 (r212721)
+++ head/sys/powerpc/aim/trap.c Thu Sep 16 03:46:17 2010 (r212722)
@@ -445,17 +445,15 @@ syscall(struct trapframe *frame)
static int
handle_slb_spill(pmap_t pm, vm_offset_t addr)
{
- struct slb kern_entry, *user_entry;
+ struct slb *user_entry;
uint64_t esid;
int i;
esid = (uintptr_t)addr >> ADDR_SR_SHFT;
if (pm == kernel_pmap) {
- kern_entry.slbv = kernel_va_to_slbv(addr);
- kern_entry.slbe = (esid << SLBE_ESID_SHIFT) | SLBE_VALID;
-
- slb_insert(pm, PCPU_GET(slb), &kern_entry);
+ slb_insert_kernel((esid << SLBE_ESID_SHIFT) | SLBE_VALID,
+ kernel_va_to_slbv(addr));
return (0);
}
@@ -464,18 +462,18 @@ handle_slb_spill(pmap_t pm, vm_offset_t
if (user_entry == NULL) {
/* allocate_vsid auto-spills it */
- (void)allocate_vsid(pm, esid, 0);
+ (void)allocate_user_vsid(pm, esid, 0);
} else {
/*
* Check that another CPU has not already mapped this.
* XXX: Per-thread SLB caches would be better.
*/
- for (i = 0; i < 64; i++)
- if (pm->pm_slb[i].slbe == (user_entry->slbe | i))
+ for (i = 0; i < pm->pm_slb_len; i++)
+ if (pm->pm_slb[i] == user_entry)
break;
- if (i == 64)
- slb_insert(pm, pm->pm_slb, user_entry);
+ if (i == pm->pm_slb_len)
+ slb_insert_user(pm, user_entry);
}
PMAP_UNLOCK(pm);
Modified: head/sys/powerpc/aim/trap_subr64.S
==============================================================================
--- head/sys/powerpc/aim/trap_subr64.S Thu Sep 16 02:59:25 2010
(r212721)
+++ head/sys/powerpc/aim/trap_subr64.S Thu Sep 16 03:46:17 2010
(r212722)
@@ -49,45 +49,59 @@
* Requires that r28-r31 be scratch, with r28 initialized to the SLB cache
*/
-restoresrs:
+/*
+ * User SRs are loaded through a pointer to the current pmap.
+ */
+restore_usersrs:
+ GET_CPUINFO(%r28);
+ ld %r28,PC_USERSLB(%r28);
li %r29, 0 /* Set the counter to zero */
slbia
slbmfee %r31,%r29
clrrdi %r31,%r31,28
slbie %r31
-instslb:
- ld %r31, 8(%r28); /* Load SLBE */
-
- cmpli 0, %r31, 0; /* If SLBE is not valid, get the next */
- beq nslb;
-
- ld %r30, 0(%r28) /* Load SLBV */
+instuserslb:
+ ld %r31, 0(%r28); /* Load SLB entry pointer */
+ cmpli 0, %r31, 0; /* If NULL, stop */
+ beqlr;
+
+ ld %r30, 0(%r31) /* Load SLBV */
+ ld %r31, 8(%r31) /* Load SLBE */
+ or %r31, %r31, %r29 /* Set SLBE slot */
slbmte %r30, %r31; /* Install SLB entry */
-nslb:
- addi %r28, %r28, 16; /* Advance */
+ addi %r28, %r28, 8; /* Advance pointer */
addi %r29, %r29, 1;
cmpli 0, %r29, 64; /* Repeat if we are not at the end */
- blt instslb;
-
+ blt instuserslb;
blr;
/*
- * User SRs are loaded through a pointer to the current pmap.
+ * Kernel SRs are loaded directly from the PCPU fields
*/
-#define RESTORE_USER_SRS() \
- GET_CPUINFO(%r28); \
- ld %r28,PC_USERSLB(%r28); \
- bl restoresrs;
+restore_kernsrs:
+ GET_CPUINFO(%r28);
+ addi %r28,%r28,PC_KERNSLB;
+ li %r29, 0 /* Set the counter to zero */
-/*
- * Kernel SRs are loaded directly from kernel_pmap_
- */
-#define RESTORE_KERN_SRS() \
- GET_CPUINFO(%r28); \
- addi %r28,%r28,PC_KERNSLB; \
- bl restoresrs;
+ slbia
+ slbmfee %r31,%r29
+ clrrdi %r31,%r31,28
+ slbie %r31
+instkernslb:
+ ld %r31, 8(%r28); /* Load SLBE */
+
+ cmpli 0, %r31, 0; /* If SLBE is not valid, stop */
+ beqlr;
+ ld %r30, 0(%r28) /* Load SLBV */
+ slbmte %r30, %r31; /* Install SLB entry */
+
+ addi %r28, %r28, 16; /* Advance pointer */
+ addi %r29, %r29, 1;
+ cmpli 0, %r29, USER_SR; /* Repeat if we are not at the end */
+ blt instkernslb;
+ blr;
/*
* FRAME_SETUP assumes:
@@ -237,7 +251,7 @@ nslb:
std %r30,(savearea+CPUSAVE_R30)(%r3); \
std %r31,(savearea+CPUSAVE_R31)(%r3); \
mflr %r27; /* preserve LR */ \
- RESTORE_USER_SRS(); /* uses r28-r31 */ \
+ bl restore_usersrs; /* uses r28-r31 */ \
mtlr %r27; \
ld %r31,(savearea+CPUSAVE_R31)(%r3); \
ld %r30,(savearea+CPUSAVE_R30)(%r3); \
@@ -432,7 +446,7 @@ realtrap:
ld %r1,PC_CURPCB(%r1)
mr %r27,%r28 /* Save LR, r29 */
mtsprg2 %r29
- RESTORE_KERN_SRS() /* enable kernel mapping */
+ bl restore_kernsrs /* enable kernel mapping */
mfsprg2 %r29
mr %r28,%r27
ba s_trap
@@ -482,7 +496,7 @@ u_trap:
ld %r1,PC_CURPCB(%r1)
mr %r27,%r28 /* Save LR, r29 */
mtsprg2 %r29
- RESTORE_KERN_SRS() /* enable kernel mapping */
+ bl restore_kernsrs /* enable kernel mapping */
mfsprg2 %r29
mr %r28,%r27
Modified: head/sys/powerpc/include/pcpu.h
==============================================================================
--- head/sys/powerpc/include/pcpu.h Thu Sep 16 02:59:25 2010
(r212721)
+++ head/sys/powerpc/include/pcpu.h Thu Sep 16 03:46:17 2010
(r212722)
@@ -55,7 +55,7 @@ struct pmap;
#define PCPU_MD_AIM64_FIELDS \
struct slb pc_slb[64]; \
- struct slb *pc_userslb;
+ struct slb **pc_userslb;
#ifdef __powerpc64__
#define PCPU_MD_AIM_FIELDS PCPU_MD_AIM64_FIELDS
Modified: head/sys/powerpc/include/pmap.h
==============================================================================
--- head/sys/powerpc/include/pmap.h Thu Sep 16 02:59:25 2010
(r212721)
+++ head/sys/powerpc/include/pmap.h Thu Sep 16 03:46:17 2010
(r212722)
@@ -93,7 +93,8 @@ struct pmap {
#ifdef __powerpc64__
struct slbtnode *pm_slb_tree_root;
- struct slb *pm_slb;
+ struct slb **pm_slb;
+ int pm_slb_len;
#else
register_t pm_sr[16];
#endif
@@ -142,14 +143,15 @@ uint64_t va_to_vsid(pmap_t pm, vm_offset
uint64_t kernel_va_to_slbv(vm_offset_t va);
struct slb *user_va_to_slb_entry(pmap_t pm, vm_offset_t va);
-uint64_t allocate_vsid(pmap_t pm, uint64_t esid, int large);
+uint64_t allocate_user_vsid(pmap_t pm, uint64_t esid, int large);
void free_vsid(pmap_t pm, uint64_t esid, int large);
-void slb_insert(pmap_t pm, struct slb *dst, struct slb *);
+void slb_insert_user(pmap_t pm, struct slb *slb);
+void slb_insert_kernel(uint64_t slbe, uint64_t slbv);
struct slbtnode *slb_alloc_tree(void);
void slb_free_tree(pmap_t pm);
-struct slb *slb_alloc_user_cache(void);
-void slb_free_user_cache(struct slb *);
+struct slb **slb_alloc_user_cache(void);
+void slb_free_user_cache(struct slb **);
#else
Modified: head/sys/powerpc/include/sr.h
==============================================================================
--- head/sys/powerpc/include/sr.h Thu Sep 16 02:59:25 2010
(r212721)
+++ head/sys/powerpc/include/sr.h Thu Sep 16 03:46:17 2010
(r212722)
@@ -42,7 +42,11 @@
#define SR_VSID_MASK 0x00ffffff /* Virtual Segment ID mask */
/* Kernel segment register usage */
+#ifdef __powerpc64__
+#define USER_SR 63
+#else
#define USER_SR 12
+#endif
#define KERNEL_SR 13
#define KERNEL2_SR 14
#define KERNEL3_SR 15
_______________________________________________
[email protected] mailing list
http://lists.freebsd.org/mailman/listinfo/svn-src-all
To unsubscribe, send any mail to "[email protected]"
