Module Name: src
Committed By: rmind
Date: Thu Mar 17 04:46:29 UTC 2011
Modified Files:
src/sys/arch/amd64/amd64 [rmind-uvmplock]: cpufunc.S genassym.cf
vector.S
src/sys/arch/i386/i386 [rmind-uvmplock]: genassym.cf i386func.S
vector.S
src/sys/arch/x86/include [rmind-uvmplock]: pmap.h
src/sys/arch/x86/x86 [rmind-uvmplock]: cpu.c pmap.c pmap_tlb.c
Log Message:
- Fix tlbflushg() to behave like tlbflush(), if page global extension (PGE)
is not (yet) enabled. This fixes the issue of stale TLB entry, experienced
early on boot, when PGE is not yet set on primary CPU.
- Rewrite i386/amd64 TLB interrupt handlers in C (only stubs are in assembly),
which simplifies and unifies (under x86) code, plus fixes few bugs.
- cpu_attach: remove assignment to cpus_running, as primary CPU might not be
attached first, which causes reset (and thus missed secondary CPUs).
To generate a diff of this commit:
cvs rdiff -u -r1.16.4.1 -r1.16.4.2 src/sys/arch/amd64/amd64/cpufunc.S
cvs rdiff -u -r1.43.4.3 -r1.43.4.4 src/sys/arch/amd64/amd64/genassym.cf
cvs rdiff -u -r1.33.2.3 -r1.33.2.4 src/sys/arch/amd64/amd64/vector.S
cvs rdiff -u -r1.85.2.3 -r1.85.2.4 src/sys/arch/i386/i386/genassym.cf
cvs rdiff -u -r1.15 -r1.15.20.1 src/sys/arch/i386/i386/i386func.S
cvs rdiff -u -r1.53.2.3 -r1.53.2.4 src/sys/arch/i386/i386/vector.S
cvs rdiff -u -r1.29.2.8 -r1.29.2.9 src/sys/arch/x86/include/pmap.h
cvs rdiff -u -r1.69.2.3 -r1.69.2.4 src/sys/arch/x86/x86/cpu.c
cvs rdiff -u -r1.105.2.12 -r1.105.2.13 src/sys/arch/x86/x86/pmap.c
cvs rdiff -u -r1.1.2.4 -r1.1.2.5 src/sys/arch/x86/x86/pmap_tlb.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/amd64/amd64/cpufunc.S
diff -u src/sys/arch/amd64/amd64/cpufunc.S:1.16.4.1 src/sys/arch/amd64/amd64/cpufunc.S:1.16.4.2
--- src/sys/arch/amd64/amd64/cpufunc.S:1.16.4.1 Sat Mar 5 20:49:14 2011
+++ src/sys/arch/amd64/amd64/cpufunc.S Thu Mar 17 04:46:29 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: cpufunc.S,v 1.16.4.1 2011/03/05 20:49:14 rmind Exp $ */
+/* $NetBSD: cpufunc.S,v 1.16.4.2 2011/03/17 04:46:29 rmind Exp $ */
/*-
* Copyright (c) 1998, 2007, 2008 The NetBSD Foundation, Inc.
@@ -136,10 +136,14 @@
* modify the PSE, PGE, or PAE flag."
*
* (the alternatives not quoted above are not an option here.)
+ *
+ * If PGE is not in use, we reload CR3.
*/
#ifndef XEN
ENTRY(tlbflushg)
movq %cr4, %rax
+ testq $CR4_PGE, %rax
+ jz 1f
movq %rax, %rdx
andq $~CR4_PGE, %rdx
movq %rdx, %cr4
@@ -147,6 +151,7 @@
ret
ENTRY(tlbflush)
+1:
movq %cr3, %rax
movq %rax, %cr3
ret
Index: src/sys/arch/amd64/amd64/genassym.cf
diff -u src/sys/arch/amd64/amd64/genassym.cf:1.43.4.3 src/sys/arch/amd64/amd64/genassym.cf:1.43.4.4
--- src/sys/arch/amd64/amd64/genassym.cf:1.43.4.3 Sat Mar 5 20:49:15 2011
+++ src/sys/arch/amd64/amd64/genassym.cf Thu Mar 17 04:46:29 2011
@@ -1,4 +1,4 @@
-# $NetBSD: genassym.cf,v 1.43.4.3 2011/03/05 20:49:15 rmind Exp $
+# $NetBSD: genassym.cf,v 1.43.4.4 2011/03/17 04:46:29 rmind Exp $
#
# Copyright (c) 1998, 2006, 2007, 2008 The NetBSD Foundation, Inc.
@@ -136,8 +136,6 @@
define KERNTEXTOFF_LO KERNTEXTOFF_LO
define KERNTEXTOFF KERNTEXTOFF
-define PG_G PG_G
-
define NBPG NBPG
define L4_SLOT_KERNBASE L4_SLOT_KERNBASE
@@ -230,8 +228,6 @@
define CPU_INFO_WANT_PMAPLOAD offsetof(struct cpu_info, ci_want_pmapload)
define CPU_INFO_TLBSTATE offsetof(struct cpu_info, ci_tlbstate)
define TLBSTATE_VALID TLBSTATE_VALID
-define TLBSTATE_LAZY TLBSTATE_LAZY
-define TLBSTATE_STALE TLBSTATE_STALE
define CPU_INFO_TLB_EVCNT offsetof(struct cpu_info, ci_tlb_evcnt)
define CPU_INFO_CURLWP offsetof(struct cpu_info, ci_curlwp)
define CPU_INFO_CURLDT offsetof(struct cpu_info, ci_curldt)
@@ -352,14 +348,6 @@
define RW_READER RW_READER
define RW_WRITER RW_WRITER
-define TM_PENDING offsetof(struct pmap_tlb_mailbox, tm_pending)
-define TP_COUNT offsetof(struct pmap_tlb_packet, tp_count)
-define TP_VA offsetof(struct pmap_tlb_packet, tp_va)
-define TP_USERMASK offsetof(struct pmap_tlb_packet, tp_usermask)
-define TP_PTE offsetof(struct pmap_tlb_packet, tp_pte)
-
-define PM_CPUS offsetof(struct pmap, pm_cpus)
-
define EV_COUNT offsetof(struct evcnt, ev_count)
define OPTERON_MSR_PASSCODE OPTERON_MSR_PASSCODE
Index: src/sys/arch/amd64/amd64/vector.S
diff -u src/sys/arch/amd64/amd64/vector.S:1.33.2.3 src/sys/arch/amd64/amd64/vector.S:1.33.2.4
--- src/sys/arch/amd64/amd64/vector.S:1.33.2.3 Sat Mar 5 20:49:15 2011
+++ src/sys/arch/amd64/amd64/vector.S Thu Mar 17 04:46:29 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: vector.S,v 1.33.2.3 2011/03/05 20:49:15 rmind Exp $ */
+/* $NetBSD: vector.S,v 1.33.2.4 2011/03/17 04:46:29 rmind Exp $ */
/*-
* Copyright (c) 1998, 2007, 2008 The NetBSD Foundation, Inc.
@@ -479,97 +479,12 @@
* TLB shootdown handler.
*/
IDTVEC(intr_lapic_tlb)
- /* Save state and ack the interrupt. */
- testq $SEL_UPL,8(%rsp)
- jz 0f
- swapgs
-0:
- pushq %rax
- pushq %rbx
- pushq %rdi
- pushq %rsi
- pushq %rdx
- pushq %rcx
- pushq %r8
- pushq %r9
+ pushq $0
+ pushq $T_ASTFLT
+ INTRENTRY
movl $0, _C_LABEL(local_apic)+LAPIC_EOI
-
- /* Find out what we need to invalidate. */
- leaq _C_LABEL(pmap_tlb_packet)(%rip), %rbx
- movswq TP_COUNT(%rbx), %rcx
- cmpq $-1, %rcx
- je 5f
- leaq TP_VA(%rbx), %rdx
-1:
- /* Invalidate a single page or a range of pages. */
- movq (%rdx), %rax
- invlpg (%rax)
- addq $8, %rdx
- decq %rcx
- jg 1b
-2:
- /*
- * Check the current TLB state. If we do not want further
- * invalidations for this pmap, then take the CPU out of
- * the pmap's bitmask.
- */
- movl CPUVAR(CPUMASK), %eax
- cmpl $TLBSTATE_LAZY, CPUVAR(TLBSTATE)
- jne 3f
- testl %eax, TP_USERMASK(%rbx)
- jz 3f
- movl CPUVAR(PMAP), %edx
- movl %eax, %ecx
- notl %ecx
- lock
- andl %ecx, PM_CPUS(%edx)
- movl $TLBSTATE_STALE, CPUVAR(TLBSTATE)
-3:
- /* Ack the request, restore state & return. */
- lock
- xorl %eax, _C_LABEL(pmap_tlb_mailbox)+TM_PENDING
- popq %r9
- popq %r8
- popq %rcx
- popq %rdx
- popq %rsi
- popq %rdi
- popq %rbx
- popq %rax
- testq $SEL_UPL, 8(%rsp)
- jz 4f
- swapgs
-4:
- iretq
-5:
- /*
- * Note that caller-save registers might be modified (all saved in the
- * beginning). Only %rbx value must be preserved for the 2f context.
- */
-
- /* Get the emap generation number. */
- callq _C_LABEL(uvm_emap_gen_return)
- movq %rax, %rdi
-
- /* Which entries we are invalidating? */
- testw $PG_G, TP_PTE(%rbx)
- jnz 6f
-
- /* a) Invalidating user TLB entries only. */
- movq %cr3, %rax
- movq %rax, %cr3
- jmp 7f
-6:
- /* b) Invalidating user and kernel TLB entries. */
- movq %cr4, %rax
- movq %rax, %rdx
- andq $~CR4_PGE, %rdx
- movq %rdx, %cr4
- movq %rax, %cr4
-7:
- /* Perform emap update, pass the generation number. */
- callq _C_LABEL(uvm_emap_update)
- jmp 2b
+ callq _C_LABEL(pmap_tlb_intr)
+ INTRFASTEXIT
#endif /* !XEN */
Index: src/sys/arch/i386/i386/genassym.cf
diff -u src/sys/arch/i386/i386/genassym.cf:1.85.2.3 src/sys/arch/i386/i386/genassym.cf:1.85.2.4
--- src/sys/arch/i386/i386/genassym.cf:1.85.2.3 Sat Mar 5 20:50:39 2011
+++ src/sys/arch/i386/i386/genassym.cf Thu Mar 17 04:46:29 2011
@@ -1,4 +1,4 @@
-# $NetBSD: genassym.cf,v 1.85.2.3 2011/03/05 20:50:39 rmind Exp $
+# $NetBSD: genassym.cf,v 1.85.2.4 2011/03/17 04:46:29 rmind Exp $
#
# Copyright (c) 1998, 2006, 2007, 2008 The NetBSD Foundation, Inc.
@@ -160,7 +160,6 @@
define PG_V PG_V
define PG_KW PG_KW
define PG_KR PG_KR
-define PG_G PG_G
define PGEX_U PGEX_U
define L2_SLOT_KERNBASE pl2_pi(KERNBASE)
@@ -288,8 +287,6 @@
define CPU_INFO_WANT_PMAPLOAD offsetof(struct cpu_info, ci_want_pmapload)
define CPU_INFO_TLBSTATE offsetof(struct cpu_info, ci_tlbstate)
define TLBSTATE_VALID TLBSTATE_VALID
-define TLBSTATE_LAZY TLBSTATE_LAZY
-define TLBSTATE_STALE TLBSTATE_STALE
define CPU_INFO_TLB_EVCNT offsetof(struct cpu_info, ci_tlb_evcnt)
define CPU_INFO_CURLWP offsetof(struct cpu_info, ci_curlwp)
define CPU_INFO_FPCURLWP offsetof(struct cpu_info, ci_fpcurlwp)
@@ -424,14 +421,6 @@
define RW_READER RW_READER
define RW_WRITER RW_WRITER
-define TM_PENDING offsetof(struct pmap_tlb_mailbox, tm_pending)
-define TP_COUNT offsetof(struct pmap_tlb_packet, tp_count)
-define TP_VA offsetof(struct pmap_tlb_packet, tp_va)
-define TP_USERMASK offsetof(struct pmap_tlb_packet, tp_usermask)
-define TP_PTE offsetof(struct pmap_tlb_packet, tp_pte)
-
-define PM_CPUS offsetof(struct pmap, pm_cpus)
-
define EV_COUNT offsetof(struct evcnt, ev_count)
define OPTERON_MSR_PASSCODE OPTERON_MSR_PASSCODE
Index: src/sys/arch/i386/i386/i386func.S
diff -u src/sys/arch/i386/i386/i386func.S:1.15 src/sys/arch/i386/i386/i386func.S:1.15.20.1
--- src/sys/arch/i386/i386/i386func.S:1.15 Sun May 25 15:56:12 2008
+++ src/sys/arch/i386/i386/i386func.S Thu Mar 17 04:46:29 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: i386func.S,v 1.15 2008/05/25 15:56:12 chs Exp $ */
+/* $NetBSD: i386func.S,v 1.15.20.1 2011/03/17 04:46:29 rmind Exp $ */
/*-
* Copyright (c) 1998, 2007, 2008 The NetBSD Foundation, Inc.
@@ -36,7 +36,7 @@
*/
#include <machine/asm.h>
-__KERNEL_RCSID(0, "$NetBSD: i386func.S,v 1.15 2008/05/25 15:56:12 chs Exp $");
+__KERNEL_RCSID(0, "$NetBSD: i386func.S,v 1.15.20.1 2011/03/17 04:46:29 rmind Exp $");
#include <machine/specialreg.h>
#include <machine/segments.h>
@@ -99,13 +99,12 @@
*
* (the alternatives not quoted above are not an option here.)
*
- * If PGE is not in use, we reload CR3 for the benefit of
- * pre-P6-family processors.
+ * If PGE is not in use, we reload CR3.
*/
ENTRY(tlbflushg)
- testl $CPUID_PGE, _C_LABEL(cpu_feature)
- jz 1f
movl %cr4, %eax
+ testl $CR4_PGE, %eax
+ jz 1f
movl %eax, %edx
andl $~CR4_PGE, %edx
movl %edx, %cr4
Index: src/sys/arch/i386/i386/vector.S
diff -u src/sys/arch/i386/i386/vector.S:1.53.2.3 src/sys/arch/i386/i386/vector.S:1.53.2.4
--- src/sys/arch/i386/i386/vector.S:1.53.2.3 Sat Mar 5 20:50:41 2011
+++ src/sys/arch/i386/i386/vector.S Thu Mar 17 04:46:29 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: vector.S,v 1.53.2.3 2011/03/05 20:50:41 rmind Exp $ */
+/* $NetBSD: vector.S,v 1.53.2.4 2011/03/17 04:46:29 rmind Exp $ */
/*
* Copyright 2002 (c) Wasabi Systems, Inc.
@@ -65,7 +65,7 @@
*/
#include <machine/asm.h>
-__KERNEL_RCSID(0, "$NetBSD: vector.S,v 1.53.2.3 2011/03/05 20:50:41 rmind Exp $");
+__KERNEL_RCSID(0, "$NetBSD: vector.S,v 1.53.2.4 2011/03/17 04:46:29 rmind Exp $");
#include "opt_ddb.h"
#include "opt_multiprocessor.h"
@@ -187,91 +187,12 @@
* TLB shootdown handler.
*/
IDTVEC(intr_lapic_tlb)
- /* Save state and ack the interrupt. */
- pushl %eax
- pushl %ebx
- pushl %ecx
- pushl %edx
- pushl %ds
- pushl %fs
- movl $GSEL(GDATA_SEL, SEL_KPL), %eax
- movl $GSEL(GCPU_SEL, SEL_KPL), %edx
- mov %ax, %ds
- mov %dx, %fs
+ pushl $0
+ pushl $T_ASTFLT
+ INTRENTRY
movl $0, _C_LABEL(local_apic)+LAPIC_EOI
-
- /* Find out what we need to invalidate. */
- leal _C_LABEL(pmap_tlb_packet), %ebx
- movswl TP_COUNT(%ebx), %ecx
- cmpl $-1, %ecx
- je 4f
- leal TP_VA(%ebx), %edx
-1:
- /* Invalidate a single page or a range of pages. */
- movl (%edx), %eax
- invlpg (%eax)
- addl $4, %edx
- decl %ecx
- jg 1b
-2:
- /*
- * Check the current TLB state. If we do not want further
- * invalidations for this pmap, then take the CPU out of
- * the pmap's bitmask.
- */
- movl CPUVAR(CPUMASK), %eax
- cmpl $TLBSTATE_LAZY, CPUVAR(TLBSTATE)
- jne 3f
- testl %eax, TP_USERMASK(%ebx)
- jz 3f
- movl CPUVAR(PMAP), %edx
- movl %eax, %ecx
- notl %ecx
- lock
- andl %ecx, PM_CPUS(%edx)
- movl $TLBSTATE_STALE, CPUVAR(TLBSTATE)
-3:
- /* Ack the request, restore state & return. */
- lock
- xorl %eax, _C_LABEL(pmap_tlb_mailbox)+TM_PENDING
- popl %fs
- popl %ds
- popl %edx
- popl %ecx
- popl %ebx
- popl %eax
- iret
-4:
- /*
- * Note that caller-save registers might be modified (all saved in the
- * beginning). Only %ebx value must be preserved for the 2b context.
- */
-
- /* Get the emap generation number. */
- call _C_LABEL(uvm_emap_gen_return)
- movl %eax, %ecx
-
- /* Which entries we are invalidating? */
- testw $PG_G, TP_PTE(%ebx)
- jnz 5f
-
- /* a) Invalidating user TLB entries only. */
- movl %cr3, %eax
- movl %eax, %cr3
- jmp 6f
-5:
- /* b) Invalidating user and kernel TLB entries. */
- movl %cr4, %eax
- movl %eax, %edx
- andl $~CR4_PGE, %edx
- movl %edx, %cr4
- movl %eax, %cr4
-6:
- /* Perform emap update, pass the generation number. */
- pushl %ecx
- call _C_LABEL(uvm_emap_update)
- addl $4, %esp
- jmp 2b
+ call _C_LABEL(pmap_tlb_intr)
+ INTRFASTEXIT
IDTVEC_END(intr_lapic_tlb)
#if defined(DDB)
Index: src/sys/arch/x86/include/pmap.h
diff -u src/sys/arch/x86/include/pmap.h:1.29.2.8 src/sys/arch/x86/include/pmap.h:1.29.2.9
--- src/sys/arch/x86/include/pmap.h:1.29.2.8 Tue Mar 8 23:27:50 2011
+++ src/sys/arch/x86/include/pmap.h Thu Mar 17 04:46:28 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.h,v 1.29.2.8 2011/03/08 23:27:50 rmind Exp $ */
+/* $NetBSD: pmap.h,v 1.29.2.9 2011/03/17 04:46:28 rmind Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -266,6 +266,7 @@
void pmap_tlb_init(void);
void pmap_tlb_shootdown(pmap_t, vaddr_t, pt_entry_t, tlbwhy_t);
void pmap_tlb_shootnow(void);
+void pmap_tlb_intr(void);
#define __HAVE_PMAP_EMAP
@@ -459,33 +460,6 @@
*/
#define POOL_VTOPHYS(va) vtophys((vaddr_t) (va))
-/*
- * TLB shootdown structures.
- */
-
-struct pmap_tlb_packet {
-#ifdef _LP64
- uintptr_t tp_va[14]; /* whole struct: 128 bytes */
-#else
- uintptr_t tp_va[13]; /* whole struct: 64 bytes */
-#endif
- uint16_t tp_count;
- uint16_t tp_pte;
- uint32_t tp_cpumask;
- uint32_t tp_usermask;
-};
-
-/* No more than N seperate invlpg. */
-#define TP_MAXVA 6
-
-struct pmap_tlb_mailbox {
- volatile uint32_t tm_pending;
- volatile uint32_t tm_gen;
- uint32_t tm_usergen;
- uint32_t tm_globalgen;
- char tm_pad[64 - sizeof(uintptr_t) * 4];
-};
-
#endif /* _KERNEL */
#endif /* _X86_PMAP_H_ */
Index: src/sys/arch/x86/x86/cpu.c
diff -u src/sys/arch/x86/x86/cpu.c:1.69.2.3 src/sys/arch/x86/x86/cpu.c:1.69.2.4
--- src/sys/arch/x86/x86/cpu.c:1.69.2.3 Sat Mar 5 20:52:29 2011
+++ src/sys/arch/x86/x86/cpu.c Thu Mar 17 04:46:28 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.c,v 1.69.2.3 2011/03/05 20:52:29 rmind Exp $ */
+/* $NetBSD: cpu.c,v 1.69.2.4 2011/03/17 04:46:28 rmind Exp $ */
/*-
* Copyright (c) 2000, 2006, 2007, 2008 The NetBSD Foundation, Inc.
@@ -62,7 +62,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.69.2.3 2011/03/05 20:52:29 rmind Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.69.2.4 2011/03/17 04:46:28 rmind Exp $");
#include "opt_ddb.h"
#include "opt_mpbios.h" /* for MPDEBUG */
@@ -354,7 +354,6 @@
cpu_init_tss(ci);
} else {
KASSERT(ci->ci_data.cpu_idlelwp != NULL);
- cpus_running = (1 << cpu_index(ci));
}
ci->ci_cpumask = (1 << cpu_index(ci));
Index: src/sys/arch/x86/x86/pmap.c
diff -u src/sys/arch/x86/x86/pmap.c:1.105.2.12 src/sys/arch/x86/x86/pmap.c:1.105.2.13
--- src/sys/arch/x86/x86/pmap.c:1.105.2.12 Tue Mar 8 23:26:35 2011
+++ src/sys/arch/x86/x86/pmap.c Thu Mar 17 04:46:29 2011
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.c,v 1.105.2.12 2011/03/08 23:26:35 rmind Exp $ */
+/* $NetBSD: pmap.c,v 1.105.2.13 2011/03/17 04:46:29 rmind Exp $ */
/*-
* Copyright (c) 2008, 2010 The NetBSD Foundation, Inc.
@@ -171,7 +171,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.105.2.12 2011/03/08 23:26:35 rmind Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.105.2.13 2011/03/17 04:46:29 rmind Exp $");
#include "opt_user_ldt.h"
#include "opt_lockdebug.h"
@@ -4301,9 +4301,7 @@
* Initiate any pending TLB shootdowns. Wait for them to
* complete before returning control to the caller.
*/
- if (((struct pmap_tlb_packet *)curcpu()->ci_pmap_data)->tp_count) {
- pmap_tlb_shootnow();
- }
+ pmap_tlb_shootnow();
KPREEMPT_ENABLE(l);
/*
Index: src/sys/arch/x86/x86/pmap_tlb.c
diff -u src/sys/arch/x86/x86/pmap_tlb.c:1.1.2.4 src/sys/arch/x86/x86/pmap_tlb.c:1.1.2.5
--- src/sys/arch/x86/x86/pmap_tlb.c:1.1.2.4 Tue Mar 8 23:41:09 2011
+++ src/sys/arch/x86/x86/pmap_tlb.c Thu Mar 17 04:46:29 2011
@@ -1,11 +1,11 @@
-/* $NetBSD: pmap_tlb.c,v 1.1.2.4 2011/03/08 23:41:09 rmind Exp $ */
+/* $NetBSD: pmap_tlb.c,v 1.1.2.5 2011/03/17 04:46:29 rmind Exp $ */
/*-
- * Copyright (c) 2008, 2010 The NetBSD Foundation, Inc.
+ * Copyright (c) 2008-2011 The NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
- * by Andrew Doran.
+ * by Andrew Doran and Mindaugas Rasiukevicius.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -35,12 +35,12 @@
* TLB shootdowns are hard interrupts that operate outside the SPL framework:
* They do not need to be blocked, provided that the pmap module gets the
* order of events correct. The calls are made by poking the LAPIC directly.
- * The stub to handle the interrupts is short and does one of the following:
- * invalidate a set of pages, all user TLB entries or the entire TLB.
+ * The interrupt handler is short and does one of the following: invalidate
+ * a set of pages, all user TLB entries or the entire TLB.
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pmap_tlb.c,v 1.1.2.4 2011/03/08 23:41:09 rmind Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap_tlb.c,v 1.1.2.5 2011/03/17 04:46:29 rmind Exp $");
#include <sys/param.h>
#include <sys/kernel.h>
@@ -57,11 +57,38 @@
#include <x86/i82489var.h>
/*
+ * TLB shootdown structures.
+ */
+
+typedef struct {
+#ifdef _LP64
+ uintptr_t tp_va[14]; /* whole struct: 128 bytes */
+#else
+ uintptr_t tp_va[13]; /* whole struct: 64 bytes */
+#endif
+ uint16_t tp_count;
+ uint16_t tp_pte;
+ uint32_t tp_cpumask;
+ uint32_t tp_usermask;
+} pmap_tlb_packet_t;
+
+/* No more than N seperate invlpg. */
+#define TP_MAXVA 6
+
+typedef struct {
+ volatile uint32_t tm_pending;
+ volatile uint32_t tm_gen;
+ uint32_t tm_usergen;
+ uint32_t tm_globalgen;
+ char tm_pad[64 - sizeof(uintptr_t) * 4];
+} pmap_tlb_mailbox_t;
+
+/*
* TLB shootdown state.
*/
static struct evcnt pmap_tlb_evcnt __cacheline_aligned;
-struct pmap_tlb_packet pmap_tlb_packet __cacheline_aligned;
-struct pmap_tlb_mailbox pmap_tlb_mailbox __cacheline_aligned;
+static pmap_tlb_packet_t pmap_tlb_packet __cacheline_aligned;
+static pmap_tlb_mailbox_t pmap_tlb_mailbox __cacheline_aligned;
/*
* TLB shootdown statistics.
@@ -93,15 +120,16 @@
void
pmap_tlb_init(void)
{
- int i = 0;
- memset(&pmap_tlb_packet, 0, sizeof(struct pmap_tlb_packet));
- memset(&pmap_tlb_mailbox, 0, sizeof(struct pmap_tlb_mailbox));
+ memset(&pmap_tlb_packet, 0, sizeof(pmap_tlb_packet_t));
+ memset(&pmap_tlb_mailbox, 0, sizeof(pmap_tlb_mailbox_t));
evcnt_attach_dynamic(&pmap_tlb_evcnt, EVCNT_TYPE_INTR,
NULL, "TLB", "shootdown");
#ifdef TLBSTATS
+ int i;
+
for (i = 0; i < TLBSHOOT__MAX; i++) {
evcnt_attach_dynamic(&tlbstat_local[i], EVCNT_TYPE_MISC,
NULL, "tlbshoot local", tlbstat_name[i]);
@@ -148,13 +176,37 @@
#endif
}
+static inline void
+pmap_tlb_invalidate(pmap_tlb_packet_t *tp)
+{
+ int i;
+
+ /* Find out what we need to invalidate. */
+ if (tp->tp_count == (uint16_t)-1) {
+ u_int egen = uvm_emap_gen_return();
+ if (tp->tp_pte & PG_G) {
+ /* Invalidating user and kernel TLB entries. */
+ tlbflushg();
+ } else {
+ /* Invalidating user TLB entries only. */
+ tlbflush();
+ }
+ uvm_emap_update(egen);
+ } else {
+ /* Invalidating a single page or a range of pages. */
+ for (i = tp->tp_count - 1; i >= 0; i--) {
+ pmap_update_pg(tp->tp_va[i]);
+ }
+ }
+}
+
/*
* pmap_tlb_shootdown: invalidate a page on all CPUs using pmap 'pm'
*/
void
pmap_tlb_shootdown(struct pmap *pm, vaddr_t va, pt_entry_t pte, tlbwhy_t why)
{
- struct pmap_tlb_packet *tp;
+ pmap_tlb_packet_t *tp;
int s;
KASSERT((pte & PG_G) == 0 || pm == pmap_kernel());
@@ -175,7 +227,7 @@
* Add the shootdown operation to our pending set.
*/
s = splvm();
- tp = (struct pmap_tlb_packet *)curcpu()->ci_pmap_data;
+ tp = (pmap_tlb_packet_t *)curcpu()->ci_pmap_data;
/* Whole address flush will be needed if PG_G is set. */
CTASSERT(PG_G == (uint16_t)PG_G);
@@ -215,18 +267,24 @@
void
pmap_tlb_shootnow(void)
{
- struct pmap_tlb_packet *tp;
- struct pmap_tlb_mailbox *tm;
+ pmap_tlb_packet_t *tp;
+ pmap_tlb_mailbox_t *tm;
struct cpu_info *ci;
uint32_t remote;
uintptr_t gen;
- int s, i, count;
+ int s;
KASSERT(kpreempt_disabled());
- s = splvm();
ci = curcpu();
- tp = (struct pmap_tlb_packet *)ci->ci_pmap_data;
+ tp = (pmap_tlb_packet_t *)ci->ci_pmap_data;
+
+ /* Pre-check first. */
+ if (tp->tp_count == 0) {
+ return;
+ }
+
+ s = splvm();
if (tp->tp_count == 0) {
splx(s);
return;
@@ -239,7 +297,7 @@
if (remote != 0) {
CPU_INFO_ITERATOR cii;
struct cpu_info *lci;
- int err;
+ int count, err;
/*
* Gain ownership of the shootdown mailbox. We must stay
* at IPL_VM once we own it or could deadlock against an
@@ -298,19 +356,7 @@
* perform local shootdowns and do not forget to update emap gen.
*/
if ((tp->tp_cpumask & ci->ci_cpumask) != 0) {
- if (tp->tp_count == (uint16_t)-1) {
- u_int egen = uvm_emap_gen_return();
- if ((tp->tp_pte & PG_G) != 0) {
- tlbflushg();
- } else {
- tlbflush();
- }
- uvm_emap_update(egen);
- } else {
- for (i = tp->tp_count - 1; i >= 0; i--) {
- pmap_update_pg(tp->tp_va[i]);
- }
- }
+ pmap_tlb_invalidate(tp);
}
/*
@@ -332,9 +378,43 @@
* processed by remote CPUs.
*/
if (remote != 0 && tm->tm_pending != 0) {
- count = SPINLOCK_BACKOFF_MIN;
+ int count = SPINLOCK_BACKOFF_MIN;
while (tm->tm_pending != 0 && tm->tm_gen == gen) {
SPINLOCK_BACKOFF(count);
}
}
}
+
+/*
+ * pmap_tlb_ipi: pmap shootdown interrupt handler to invalidate TLB entries.
+ *
+ * => Called from IPI only.
+ */
+void
+pmap_tlb_intr(void)
+{
+ pmap_tlb_packet_t *tp = &pmap_tlb_packet;
+ pmap_tlb_mailbox_t *tm;
+ struct cpu_info *ci;
+ uint32_t cm;
+
+ pmap_tlb_invalidate(tp);
+
+ /*
+ * Check the current TLB state. If we do not want further
+ * invalidations for this pmap, then take the CPU out of
+ * the pmap's bitmask.
+ */
+ ci = curcpu();
+ cm = ci->ci_cpumask;
+ if (ci->ci_tlbstate == TLBSTATE_LAZY && (tp->tp_usermask & cm) != 0) {
+ struct pmap *pm = ci->ci_pmap;
+
+ atomic_and_32(&pm->pm_cpus, ~cm);
+ ci->ci_tlbstate = TLBSTATE_STALE;
+ }
+
+ /* Ack the request. */
+ tm = &pmap_tlb_mailbox;
+ atomic_and_32(&tm->tm_pending, ~cm);
+}
