Author: royger
Date: Tue Mar 11 10:03:29 2014
New Revision: 263001
URL: http://svnweb.freebsd.org/changeset/base/263001
Log:
amd64/i386: switch IPI handlers to C code.
Move asm IPIs handlers to C code, so both Xen and native IPI handlers
share the same code.
Reviewed by: jhb
Approved by: gibbs
Sponsored by: Citrix Systems R&D
amd64/amd64/apic_vector.S:
i386/i386/apic_vector.s:
- Remove asm coded IPI handlers and instead call the newly introduced
C variants.
amd64/amd64/mp_machdep.c:
i386/i386/mp_machdep.c:
- Add C coded clones to the asm IPI handlers (moved from
x86/xen/hvm.c).
i386/include/smp.h:
amd64/include/smp.h:
- Add prototypes for the C IPI handlers.
x86/xen/hvm.c:
- Move the C IPI handlers to mp_machdep and call those in the Xen IPI
handlers.
i386/xen/mp_machdep.c:
- Add dummy IPI handlers to the i386 Xen PV port (this port doesn't
support SMP).
Modified:
head/sys/amd64/amd64/apic_vector.S
head/sys/amd64/amd64/mp_machdep.c
head/sys/amd64/include/smp.h
head/sys/i386/i386/apic_vector.s
head/sys/i386/i386/mp_machdep.c
head/sys/i386/include/smp.h
head/sys/i386/xen/mp_machdep.c
head/sys/x86/xen/hvm.c
Modified: head/sys/amd64/amd64/apic_vector.S
==============================================================================
--- head/sys/amd64/amd64/apic_vector.S Tue Mar 11 07:33:33 2014
(r263000)
+++ head/sys/amd64/amd64/apic_vector.S Tue Mar 11 10:03:29 2014
(r263001)
@@ -159,101 +159,26 @@ IDTVEC(xen_intr_upcall)
#define NAKE_INTR_CS 24
SUPERALIGN_TEXT
-global_invltlb:
- movq %cr4,%rax
- andq $~0x80,%rax /* PGE */
- movq %rax,%cr4
- orq $0x80,%rax
- movq %rax,%cr4
-invltlb_ret_clear_pm_save:
- movq smp_tlb_pmap,%rdx
- testq %rdx,%rdx
- jz invltlb_ret_rdx
- testb $SEL_RPL_MASK,NAKE_INTR_CS(%rsp)
- jz 1f
- swapgs
-1:
- movl PCPU(CPUID),%eax
- jz 2f
- swapgs
-2:
- LK btcl %eax,PM_SAVE(%rdx)
- SUPERALIGN_TEXT
-invltlb_ret_rdx:
- popq %rdx
-invltlb_ret_rax:
+invltlb_ret:
movq lapic, %rax
movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
- LK incl smp_tlb_wait
- popq %rax
+ POP_FRAME
jmp doreti_iret
SUPERALIGN_TEXT
IDTVEC(invltlb_pcid)
-#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
PUSH_FRAME
- movl PCPU(CPUID), %eax
-#ifdef COUNT_XINVLTLB_HITS
- incl xhits_gbl(,%rax,4)
-#endif
-#ifdef COUNT_IPIS
- movq ipi_invltlb_counts(,%rax,8),%rax
- incq (%rax)
-#endif
- POP_FRAME
-#endif
-
- pushq %rax
- pushq %rdx
- movq %cr3,%rax
+ call invltlb_pcid_handler
+ jmp invltlb_ret
- movq $smp_tlb_invpcid,%rdx
- cmpl $0,(%rdx)
- je global_invltlb
- cmpl $-1,(%rdx)
- je global_invltlb
-
- /*
- * Only invalidate TLB for entries with current PCID.
- */
- cmpl $0,invpcid_works
- je 1f
- /* Use invpcid if available. */
- movl $1,%eax /* INVPCID_CTX */
- /* invpcid (%rdx),%rax */
- .byte 0x66,0x0f,0x38,0x82,0x02
- jmp invltlb_ret_clear_pm_save
-1:
- /* Otherwise reload %cr3 twice. */
- movq pcid_cr3,%rdx
- cmpq %rax,%rdx
- je 2f
- movq %rdx,%cr3 /* Invalidate, bit 63 is zero. */
- btsq $63,%rax
-2:
- movq %rax,%cr3
- jmp invltlb_ret_clear_pm_save
SUPERALIGN_TEXT
IDTVEC(invltlb)
-#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
PUSH_FRAME
- movl PCPU(CPUID), %eax
-#ifdef COUNT_XINVLTLB_HITS
- incl xhits_gbl(,%rax,4)
-#endif
-#ifdef COUNT_IPIS
- movq ipi_invltlb_counts(,%rax,8),%rax
- incq (%rax)
-#endif
- POP_FRAME
-#endif
- pushq %rax
- movq %cr3, %rax /* invalidate the TLB */
- movq %rax, %cr3
- jmp invltlb_ret_rax
+ call invltlb_handler
+ jmp invltlb_ret
/*
* Single page TLB shootdown
@@ -261,86 +186,17 @@ IDTVEC(invltlb)
.text
SUPERALIGN_TEXT
IDTVEC(invlpg_pcid)
-#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
PUSH_FRAME
- movl PCPU(CPUID), %eax
-#ifdef COUNT_XINVLTLB_HITS
- incl xhits_pg(,%rax,4)
-#endif
-#ifdef COUNT_IPIS
- movq ipi_invlpg_counts(,%rax,8),%rax
- incq (%rax)
-#endif
- POP_FRAME
-#endif
- pushq %rax
- pushq %rdx
- movq $smp_tlb_invpcid,%rdx
- cmpl $0,invpcid_works
- jne 2f
-
- /* kernel pmap - use invlpg to invalidate global mapping */
- cmpl $0,(%rdx)
- je 3f
- cmpl $-1,(%rdx)
- je global_invltlb
-
- /*
- * PCID supported, but INVPCID is not.
- * Temporarily switch to the target address space and do INVLPG.
- */
- pushq %rcx
- movq %cr3,%rcx
- movq pcid_cr3,%rax
- cmp %rcx,%rax
- je 1f
- btsq $63,%rax
- movq %rax,%cr3
-1: movq 8(%rdx),%rax
- invlpg (%rax)
- btsq $63,%rcx
- movq %rcx,%cr3
- popq %rcx
- jmp invltlb_ret_rdx
-
- /*
- * Invalidate the TLB entry using INVPCID_ADDR.
- */
-2:
- xorl %eax,%eax
-/* invpcid (%rdx),%rax */
- .byte 0x66,0x0f,0x38,0x82,0x02
- jmp invltlb_ret_rdx
-
- /*
- * PCID is not supported or kernel pmap.
- * Invalidate single page using INVLPG.
- */
-3:
- movq 8(%rdx),%rax
- invlpg (%rax)
- jmp invltlb_ret_rdx
+ call invlpg_pcid_handler
+ jmp invltlb_ret
SUPERALIGN_TEXT
IDTVEC(invlpg)
-#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
PUSH_FRAME
- movl PCPU(CPUID), %eax
-#ifdef COUNT_XINVLTLB_HITS
- incl xhits_pg(,%rax,4)
-#endif
-#ifdef COUNT_IPIS
- movq ipi_invlpg_counts(,%rax,8),%rax
- incq (%rax)
-#endif
- POP_FRAME
-#endif
- pushq %rax
- movq smp_tlb_invpcid+8,%rax
- invlpg (%rax) /* invalidate single page */
- jmp invltlb_ret_rax
+ call invlpg_handler
+ jmp invltlb_ret
/*
* Page range TLB shootdown.
@@ -348,81 +204,10 @@ IDTVEC(invlpg)
.text
SUPERALIGN_TEXT
IDTVEC(invlrng)
-#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
PUSH_FRAME
- movl PCPU(CPUID), %eax
-#ifdef COUNT_XINVLTLB_HITS
- incl xhits_rng(,%rax,4)
-#endif
-#ifdef COUNT_IPIS
- movq ipi_invlrng_counts(,%rax,8),%rax
- incq (%rax)
-#endif
- POP_FRAME
-#endif
- pushq %rax
- pushq %rdx
- movq $smp_tlb_invpcid,%rdx
- cmpl $0,pmap_pcid_enabled
- je invlrng_single_page
-
- /* kernel pmap - use invlpg to invalidate global mapping */
- cmpl $0,(%rdx)
- je invlrng_single_page
- cmpl $-1,(%rdx)
- je global_invltlb
- cmpl $0,invpcid_works
- jne invlrng_invpcid
-
- pushq %rcx
- movq %cr3,%rcx
- movq pcid_cr3,%rax
- cmpq %rcx,%rax
- je 1f
- btsq $63,%rax
- movq %rax,%cr3
-1:
- movq 8(%rdx),%rdx
- movq smp_tlb_addr2,%rax
-2:
- invlpg (%rdx)
- addq $PAGE_SIZE,%rdx
- cmpq %rax,%rdx
- jb 2b
- btsq $63,%rcx
- movq %rcx,%cr3
- popq %rcx
- jmp invltlb_ret_rdx
-
-invlrng_invpcid:
- pushq %rcx
- subq $16,%rsp
- movq (%rdx),%rcx
- movq %rcx,(%rsp)
- movq 8(%rdx),%rax
- movq %rax,8(%rsp)
- movq smp_tlb_addr2,%rcx
- subq %rax,%rcx
- shrq $PAGE_SHIFT,%rcx
-1:
-// invpcid (%rdx),%rax
- .byte 0x66,0x0f,0x38,0x82,0x02
- addq $PAGE_SIZE,8(%rsp)
- dec %rcx
- jne 1b
- addq $16,%rsp
- popq %rcx
- jmp invltlb_ret_rdx
-
-invlrng_single_page:
- movq 8(%rdx),%rdx
- movq smp_tlb_addr2,%rax
-1: invlpg (%rdx) /* invalidate single page */
- addq $PAGE_SIZE,%rdx
- cmpq %rax,%rdx
- jb 1b
- jmp invltlb_ret_rdx
+ call invlrng_handler
+ jmp invltlb_ret
/*
* Invalidate cache.
@@ -430,17 +215,10 @@ invlrng_single_page:
.text
SUPERALIGN_TEXT
IDTVEC(invlcache)
-#ifdef COUNT_IPIS
PUSH_FRAME
- movl PCPU(CPUID), %eax
- movq ipi_invlcache_counts(,%rax,8),%rax
- incq (%rax)
- POP_FRAME
-#endif
- pushq %rax
- wbinvd
- jmp invltlb_ret_rax
+ call invlcache_handler
+ jmp invltlb_ret
/*
* Handler for IPIs sent via the per-cpu IPI bitmap.
Modified: head/sys/amd64/amd64/mp_machdep.c
==============================================================================
--- head/sys/amd64/amd64/mp_machdep.c Tue Mar 11 07:33:33 2014
(r263000)
+++ head/sys/amd64/amd64/mp_machdep.c Tue Mar 11 10:03:29 2014
(r263001)
@@ -109,6 +109,7 @@ struct invpcid_descr smp_tlb_invpcid;
volatile int smp_tlb_wait;
uint64_t pcid_cr3;
pmap_t smp_tlb_pmap;
+extern int invpcid_works;
#ifdef COUNT_IPIS
/* Interrupt counts. */
@@ -1496,6 +1497,175 @@ cpususpend_handler(void)
}
/*
+ * Handlers for TLB related IPIs
+ */
+void
+invltlb_handler(void)
+{
+#ifdef COUNT_XINVLTLB_HITS
+ xhits_gbl[PCPU_GET(cpuid)]++;
+#endif /* COUNT_XINVLTLB_HITS */
+#ifdef COUNT_IPIS
+ (*ipi_invltlb_counts[PCPU_GET(cpuid)])++;
+#endif /* COUNT_IPIS */
+
+ invltlb();
+ atomic_add_int(&smp_tlb_wait, 1);
+}
+
+void
+invltlb_pcid_handler(void)
+{
+ uint64_t cr3;
+#ifdef COUNT_XINVLTLB_HITS
+ xhits_gbl[PCPU_GET(cpuid)]++;
+#endif /* COUNT_XINVLTLB_HITS */
+#ifdef COUNT_IPIS
+ (*ipi_invltlb_counts[PCPU_GET(cpuid)])++;
+#endif /* COUNT_IPIS */
+
+ cr3 = rcr3();
+ if (smp_tlb_invpcid.pcid != (uint64_t)-1 &&
+ smp_tlb_invpcid.pcid != 0) {
+
+ if (invpcid_works) {
+ invpcid(&smp_tlb_invpcid, INVPCID_CTX);
+ } else {
+ /* Otherwise reload %cr3 twice. */
+ if (cr3 != pcid_cr3) {
+ load_cr3(pcid_cr3);
+ cr3 |= CR3_PCID_SAVE;
+ }
+ load_cr3(cr3);
+ }
+ } else {
+ invltlb_globpcid();
+ }
+ if (smp_tlb_pmap != NULL)
+ CPU_CLR_ATOMIC(PCPU_GET(cpuid), &smp_tlb_pmap->pm_save);
+
+ atomic_add_int(&smp_tlb_wait, 1);
+}
+
+void
+invlpg_handler(void)
+{
+#ifdef COUNT_XINVLTLB_HITS
+ xhits_pg[PCPU_GET(cpuid)]++;
+#endif /* COUNT_XINVLTLB_HITS */
+#ifdef COUNT_IPIS
+ (*ipi_invlpg_counts[PCPU_GET(cpuid)])++;
+#endif /* COUNT_IPIS */
+
+ invlpg(smp_tlb_invpcid.addr);
+ atomic_add_int(&smp_tlb_wait, 1);
+}
+
+void
+invlpg_pcid_handler(void)
+{
+#ifdef COUNT_XINVLTLB_HITS
+ xhits_pg[PCPU_GET(cpuid)]++;
+#endif /* COUNT_XINVLTLB_HITS */
+#ifdef COUNT_IPIS
+ (*ipi_invlpg_counts[PCPU_GET(cpuid)])++;
+#endif /* COUNT_IPIS */
+
+ if (invpcid_works) {
+ invpcid(&smp_tlb_invpcid, INVPCID_ADDR);
+ } else if (smp_tlb_invpcid.pcid == 0) {
+ invlpg(smp_tlb_invpcid.addr);
+ } else if (smp_tlb_invpcid.pcid == (uint64_t)-1) {
+ invltlb_globpcid();
+ } else {
+ uint64_t cr3;
+
+ /*
+ * PCID supported, but INVPCID is not.
+ * Temporarily switch to the target address
+ * space and do INVLPG.
+ */
+ cr3 = rcr3();
+ if (cr3 != pcid_cr3)
+ load_cr3(pcid_cr3 | CR3_PCID_SAVE);
+ invlpg(smp_tlb_invpcid.addr);
+ load_cr3(cr3 | CR3_PCID_SAVE);
+ }
+
+ atomic_add_int(&smp_tlb_wait, 1);
+}
+
+static inline void
+invlpg_range(vm_offset_t start, vm_offset_t end)
+{
+
+ do {
+ invlpg(start);
+ start += PAGE_SIZE;
+ } while (start < end);
+}
+
+void
+invlrng_handler(void)
+{
+ vm_offset_t addr;
+#ifdef COUNT_XINVLTLB_HITS
+ xhits_rng[PCPU_GET(cpuid)]++;
+#endif /* COUNT_XINVLTLB_HITS */
+#ifdef COUNT_IPIS
+ (*ipi_invlrng_counts[PCPU_GET(cpuid)])++;
+#endif /* COUNT_IPIS */
+
+ addr = smp_tlb_invpcid.addr;
+ if (pmap_pcid_enabled) {
+ if (invpcid_works) {
+ struct invpcid_descr d;
+
+ d = smp_tlb_invpcid;
+ do {
+ invpcid(&d, INVPCID_ADDR);
+ d.addr += PAGE_SIZE;
+ } while (d.addr < smp_tlb_addr2);
+ } else if (smp_tlb_invpcid.pcid == 0) {
+ /*
+ * kernel pmap - use invlpg to invalidate
+ * global mapping.
+ */
+ invlpg_range(addr, smp_tlb_addr2);
+ } else if (smp_tlb_invpcid.pcid == (uint64_t)-1) {
+ invltlb_globpcid();
+ if (smp_tlb_pmap != NULL) {
+ CPU_CLR_ATOMIC(PCPU_GET(cpuid),
+ &smp_tlb_pmap->pm_save);
+ }
+ } else {
+ uint64_t cr3;
+
+ cr3 = rcr3();
+ if (cr3 != pcid_cr3)
+ load_cr3(pcid_cr3 | CR3_PCID_SAVE);
+ invlpg_range(addr, smp_tlb_addr2);
+ load_cr3(cr3 | CR3_PCID_SAVE);
+ }
+ } else {
+ invlpg_range(addr, smp_tlb_addr2);
+ }
+
+ atomic_add_int(&smp_tlb_wait, 1);
+}
+
+void
+invlcache_handler(void)
+{
+#ifdef COUNT_IPIS
+ (*ipi_invlcache_counts[PCPU_GET(cpuid)])++;
+#endif /* COUNT_IPIS */
+
+ wbinvd();
+ atomic_add_int(&smp_tlb_wait, 1);
+}
+
+/*
* This is called once the rest of the system is up and running and we're
* ready to let the AP's out of the pen.
*/
Modified: head/sys/amd64/include/smp.h
==============================================================================
--- head/sys/amd64/include/smp.h Tue Mar 11 07:33:33 2014
(r263000)
+++ head/sys/amd64/include/smp.h Tue Mar 11 10:03:29 2014
(r263001)
@@ -62,6 +62,12 @@ struct pmap;
void cpu_add(u_int apic_id, char boot_cpu);
void cpustop_handler(void);
void cpususpend_handler(void);
+void invltlb_handler(void);
+void invltlb_pcid_handler(void);
+void invlpg_handler(void);
+void invlpg_pcid_handler(void);
+void invlrng_handler(void);
+void invlcache_handler(void);
void init_secondary(void);
void ipi_startup(int apic_id, int vector);
void ipi_all_but_self(u_int ipi);
Modified: head/sys/i386/i386/apic_vector.s
==============================================================================
--- head/sys/i386/i386/apic_vector.s Tue Mar 11 07:33:33 2014
(r263000)
+++ head/sys/i386/i386/apic_vector.s Tue Mar 11 10:03:29 2014
(r263001)
@@ -163,39 +163,21 @@ IDTVEC(xen_intr_upcall)
*/
.text
SUPERALIGN_TEXT
-IDTVEC(invltlb)
- pushl %eax
- pushl %ds
- movl $KDSEL, %eax /* Kernel data selector */
- movl %eax, %ds
-
-#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
- pushl %fs
- movl $KPSEL, %eax /* Private space selector */
- movl %eax, %fs
- movl PCPU(CPUID), %eax
- popl %fs
-#ifdef COUNT_XINVLTLB_HITS
- incl xhits_gbl(,%eax,4)
-#endif
-#ifdef COUNT_IPIS
- movl ipi_invltlb_counts(,%eax,4),%eax
- incl (%eax)
-#endif
-#endif
-
- movl %cr3, %eax /* invalidate the TLB */
- movl %eax, %cr3
-
+invltlb_ret:
movl lapic, %eax
movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */
+ POP_FRAME
+ iret
+
+ SUPERALIGN_TEXT
+IDTVEC(invltlb)
+ PUSH_FRAME
+ SET_KERNEL_SREGS
+ cld
- lock
- incl smp_tlb_wait
+ call invltlb_handler
- popl %ds
- popl %eax
- iret
+ jmp invltlb_ret
/*
* Single page TLB shootdown
@@ -203,38 +185,13 @@ IDTVEC(invltlb)
.text
SUPERALIGN_TEXT
IDTVEC(invlpg)
- pushl %eax
- pushl %ds
- movl $KDSEL, %eax /* Kernel data selector */
- movl %eax, %ds
-
-#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
- pushl %fs
- movl $KPSEL, %eax /* Private space selector */
- movl %eax, %fs
- movl PCPU(CPUID), %eax
- popl %fs
-#ifdef COUNT_XINVLTLB_HITS
- incl xhits_pg(,%eax,4)
-#endif
-#ifdef COUNT_IPIS
- movl ipi_invlpg_counts(,%eax,4),%eax
- incl (%eax)
-#endif
-#endif
-
- movl smp_tlb_addr1, %eax
- invlpg (%eax) /* invalidate single page */
-
- movl lapic, %eax
- movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */
+ PUSH_FRAME
+ SET_KERNEL_SREGS
+ cld
- lock
- incl smp_tlb_wait
+ call invlpg_handler
- popl %ds
- popl %eax
- iret
+ jmp invltlb_ret
/*
* Page range TLB shootdown.
@@ -242,44 +199,13 @@ IDTVEC(invlpg)
.text
SUPERALIGN_TEXT
IDTVEC(invlrng)
- pushl %eax
- pushl %edx
- pushl %ds
- movl $KDSEL, %eax /* Kernel data selector */
- movl %eax, %ds
-
-#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
- pushl %fs
- movl $KPSEL, %eax /* Private space selector */
- movl %eax, %fs
- movl PCPU(CPUID), %eax
- popl %fs
-#ifdef COUNT_XINVLTLB_HITS
- incl xhits_rng(,%eax,4)
-#endif
-#ifdef COUNT_IPIS
- movl ipi_invlrng_counts(,%eax,4),%eax
- incl (%eax)
-#endif
-#endif
+ PUSH_FRAME
+ SET_KERNEL_SREGS
+ cld
- movl smp_tlb_addr1, %edx
- movl smp_tlb_addr2, %eax
-1: invlpg (%edx) /* invalidate single page */
- addl $PAGE_SIZE, %edx
- cmpl %eax, %edx
- jb 1b
+ call invlrng_handler
- movl lapic, %eax
- movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */
-
- lock
- incl smp_tlb_wait
-
- popl %ds
- popl %edx
- popl %eax
- iret
+ jmp invltlb_ret
/*
* Invalidate cache.
@@ -287,32 +213,13 @@ IDTVEC(invlrng)
.text
SUPERALIGN_TEXT
IDTVEC(invlcache)
- pushl %eax
- pushl %ds
- movl $KDSEL, %eax /* Kernel data selector */
- movl %eax, %ds
-
-#ifdef COUNT_IPIS
- pushl %fs
- movl $KPSEL, %eax /* Private space selector */
- movl %eax, %fs
- movl PCPU(CPUID), %eax
- popl %fs
- movl ipi_invlcache_counts(,%eax,4),%eax
- incl (%eax)
-#endif
-
- wbinvd
-
- movl lapic, %eax
- movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */
+ PUSH_FRAME
+ SET_KERNEL_SREGS
+ cld
- lock
- incl smp_tlb_wait
+ call invlcache_handler
- popl %ds
- popl %eax
- iret
+ jmp invltlb_ret
/*
* Handler for IPIs sent via the per-cpu IPI bitmap.
Modified: head/sys/i386/i386/mp_machdep.c
==============================================================================
--- head/sys/i386/i386/mp_machdep.c Tue Mar 11 07:33:33 2014
(r263000)
+++ head/sys/i386/i386/mp_machdep.c Tue Mar 11 10:03:29 2014
(r263001)
@@ -1551,6 +1551,72 @@ cpususpend_handler(void)
CPU_CLR_ATOMIC(cpu, &suspended_cpus);
CPU_CLR_ATOMIC(cpu, &started_cpus);
}
+
+/*
+ * Handlers for TLB related IPIs
+ */
+void
+invltlb_handler(void)
+{
+ uint64_t cr3;
+#ifdef COUNT_XINVLTLB_HITS
+ xhits_gbl[PCPU_GET(cpuid)]++;
+#endif /* COUNT_XINVLTLB_HITS */
+#ifdef COUNT_IPIS
+ (*ipi_invltlb_counts[PCPU_GET(cpuid)])++;
+#endif /* COUNT_IPIS */
+
+ cr3 = rcr3();
+ load_cr3(cr3);
+ atomic_add_int(&smp_tlb_wait, 1);
+}
+
+void
+invlpg_handler(void)
+{
+#ifdef COUNT_XINVLTLB_HITS
+ xhits_pg[PCPU_GET(cpuid)]++;
+#endif /* COUNT_XINVLTLB_HITS */
+#ifdef COUNT_IPIS
+ (*ipi_invlpg_counts[PCPU_GET(cpuid)])++;
+#endif /* COUNT_IPIS */
+
+ invlpg(smp_tlb_addr1);
+
+ atomic_add_int(&smp_tlb_wait, 1);
+}
+
+void
+invlrng_handler(void)
+{
+ vm_offset_t addr;
+#ifdef COUNT_XINVLTLB_HITS
+ xhits_rng[PCPU_GET(cpuid)]++;
+#endif /* COUNT_XINVLTLB_HITS */
+#ifdef COUNT_IPIS
+ (*ipi_invlrng_counts[PCPU_GET(cpuid)])++;
+#endif /* COUNT_IPIS */
+
+ addr = smp_tlb_addr1;
+ do {
+ invlpg(addr);
+ addr += PAGE_SIZE;
+ } while (addr < smp_tlb_addr2);
+
+ atomic_add_int(&smp_tlb_wait, 1);
+}
+
+void
+invlcache_handler(void)
+{
+#ifdef COUNT_IPIS
+ (*ipi_invlcache_counts[PCPU_GET(cpuid)])++;
+#endif /* COUNT_IPIS */
+
+ wbinvd();
+ atomic_add_int(&smp_tlb_wait, 1);
+}
+
/*
* This is called once the rest of the system is up and running and we're
* ready to let the AP's out of the pen.
Modified: head/sys/i386/include/smp.h
==============================================================================
--- head/sys/i386/include/smp.h Tue Mar 11 07:33:33 2014 (r263000)
+++ head/sys/i386/include/smp.h Tue Mar 11 10:03:29 2014 (r263001)
@@ -63,6 +63,10 @@ void cpustop_handler(void);
#ifndef XEN
void cpususpend_handler(void);
#endif
+void invltlb_handler(void);
+void invlpg_handler(void);
+void invlrng_handler(void);
+void invlcache_handler(void);
void init_secondary(void);
void ipi_startup(int apic_id, int vector);
void ipi_all_but_self(u_int ipi);
Modified: head/sys/i386/xen/mp_machdep.c
==============================================================================
--- head/sys/i386/xen/mp_machdep.c Tue Mar 11 07:33:33 2014
(r263000)
+++ head/sys/i386/xen/mp_machdep.c Tue Mar 11 10:03:29 2014
(r263001)
@@ -1276,6 +1276,31 @@ cpustop_handler(void)
}
/*
+ * Handlers for TLB related IPIs
+ *
+ * On i386 Xen PV this are no-ops since this port doesn't support SMP.
+ */
+void
+invltlb_handler(void)
+{
+}
+
+void
+invlpg_handler(void)
+{
+}
+
+void
+invlrng_handler(void)
+{
+}
+
+void
+invlcache_handler(void)
+{
+}
+
+/*
* This is called once the rest of the system is up and running and we're
* ready to let the AP's out of the pen.
*/
Modified: head/sys/x86/xen/hvm.c
==============================================================================
--- head/sys/x86/xen/hvm.c Tue Mar 11 07:33:33 2014 (r263000)
+++ head/sys/x86/xen/hvm.c Tue Mar 11 10:03:29 2014 (r263001)
@@ -77,22 +77,12 @@ static void xen_hvm_cpu_resume(void);
static void xen_hvm_cpu_init(void);
/*---------------------------- Extern Declarations
---------------------------*/
-/* Variables used by mp_machdep to perform the MMU related IPIs */
-extern volatile int smp_tlb_wait;
-extern vm_offset_t smp_tlb_addr2;
-#ifdef __i386__
-extern vm_offset_t smp_tlb_addr1;
-#else
-extern struct invpcid_descr smp_tlb_invpcid;
-extern uint64_t pcid_cr3;
-extern int invpcid_works;
-extern int pmap_pcid_enabled;
-extern pmap_t smp_tlb_pmap;
-#endif
-
#ifdef __i386__
extern void pmap_lazyfix_action(void);
#endif
+#ifdef __amd64__
+extern int pmap_pcid_enabled;
+#endif
/* Variables used by mp_machdep to perform the bitmap IPI */
extern volatile u_int cpu_ipi_pending[MAXCPU];
@@ -179,10 +169,7 @@ static int
xen_smp_rendezvous_action(void *arg)
{
#ifdef COUNT_IPIS
- int cpu;
-
- cpu = PCPU_GET(cpuid);
- (*ipi_rendezvous_counts[cpu])++;
+ (*ipi_rendezvous_counts[PCPU_GET(cpuid)])++;
#endif /* COUNT_IPIS */
smp_rendezvous_action();
@@ -192,20 +179,8 @@ xen_smp_rendezvous_action(void *arg)
static int
xen_invltlb(void *arg)
{
-#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
- int cpu;
-
- cpu = PCPU_GET(cpuid);
-#ifdef COUNT_XINVLTLB_HITS
- xhits_gbl[cpu]++;
-#endif /* COUNT_XINVLTLB_HITS */
-#ifdef COUNT_IPIS
- (*ipi_invltlb_counts[cpu])++;
-#endif /* COUNT_IPIS */
-#endif /* COUNT_XINVLTLB_HITS || COUNT_IPIS */
- invltlb();
- atomic_add_int(&smp_tlb_wait, 1);
+ invltlb_handler();
return (FILTER_HANDLED);
}
@@ -213,40 +188,8 @@ xen_invltlb(void *arg)
static int
xen_invltlb_pcid(void *arg)
{
- uint64_t cr3;
-#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
- int cpu;
-
- cpu = PCPU_GET(cpuid);
-#ifdef COUNT_XINVLTLB_HITS
- xhits_gbl[cpu]++;
-#endif /* COUNT_XINVLTLB_HITS */
-#ifdef COUNT_IPIS
- (*ipi_invltlb_counts[cpu])++;
-#endif /* COUNT_IPIS */
-#endif /* COUNT_XINVLTLB_HITS || COUNT_IPIS */
- cr3 = rcr3();
- if (smp_tlb_invpcid.pcid != (uint64_t)-1 &&
- smp_tlb_invpcid.pcid != 0) {
-
- if (invpcid_works) {
- invpcid(&smp_tlb_invpcid, INVPCID_CTX);
- } else {
- /* Otherwise reload %cr3 twice. */
- if (cr3 != pcid_cr3) {
- load_cr3(pcid_cr3);
- cr3 |= CR3_PCID_SAVE;
- }
- load_cr3(cr3);
- }
- } else {
- invltlb_globpcid();
- }
- if (smp_tlb_pmap != NULL)
- CPU_CLR_ATOMIC(PCPU_GET(cpuid), &smp_tlb_pmap->pm_save);
-
- atomic_add_int(&smp_tlb_wait, 1);
+ invltlb_pcid_handler();
return (FILTER_HANDLED);
}
#endif
@@ -254,24 +197,8 @@ xen_invltlb_pcid(void *arg)
static int
xen_invlpg(void *arg)
{
-#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
- int cpu;
-
- cpu = PCPU_GET(cpuid);
-#ifdef COUNT_XINVLTLB_HITS
- xhits_pg[cpu]++;
-#endif /* COUNT_XINVLTLB_HITS */
-#ifdef COUNT_IPIS
- (*ipi_invlpg_counts[cpu])++;
-#endif /* COUNT_IPIS */
-#endif /* COUNT_XINVLTLB_HITS || COUNT_IPIS */
-#ifdef __i386__
- invlpg(smp_tlb_addr1);
-#else
- invlpg(smp_tlb_invpcid.addr);
-#endif
- atomic_add_int(&smp_tlb_wait, 1);
+ invlpg_handler();
return (FILTER_HANDLED);
}
@@ -279,125 +206,25 @@ xen_invlpg(void *arg)
static int
xen_invlpg_pcid(void *arg)
{
-#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
- int cpu;
-
- cpu = PCPU_GET(cpuid);
-#ifdef COUNT_XINVLTLB_HITS
- xhits_pg[cpu]++;
-#endif /* COUNT_XINVLTLB_HITS */
-#ifdef COUNT_IPIS
- (*ipi_invlpg_counts[cpu])++;
-#endif /* COUNT_IPIS */
-#endif /* COUNT_XINVLTLB_HITS || COUNT_IPIS */
-
- if (invpcid_works) {
- invpcid(&smp_tlb_invpcid, INVPCID_ADDR);
- } else if (smp_tlb_invpcid.pcid == 0) {
- invlpg(smp_tlb_invpcid.addr);
- } else if (smp_tlb_invpcid.pcid == (uint64_t)-1) {
- invltlb_globpcid();
- } else {
- uint64_t cr3;
-
- /*
- * PCID supported, but INVPCID is not.
- * Temporarily switch to the target address
- * space and do INVLPG.
- */
- cr3 = rcr3();
- if (cr3 != pcid_cr3)
- load_cr3(pcid_cr3 | CR3_PCID_SAVE);
- invlpg(smp_tlb_invpcid.addr);
- load_cr3(cr3 | CR3_PCID_SAVE);
- }
- atomic_add_int(&smp_tlb_wait, 1);
+ invlpg_pcid_handler();
return (FILTER_HANDLED);
}
#endif
-static inline void
-invlpg_range(vm_offset_t start, vm_offset_t end)
-{
- do {
- invlpg(start);
- start += PAGE_SIZE;
- } while (start < end);
-}
-
static int
xen_invlrng(void *arg)
{
- vm_offset_t addr;
-#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
_______________________________________________
[email protected] mailing list
http://lists.freebsd.org/mailman/listinfo/svn-src-head
To unsubscribe, send any mail to "[email protected]"
