Author: mjg
Date: Sun May 12 06:36:54 2019
New Revision: 347502
URL: https://svnweb.freebsd.org/changeset/base/347502
Log:
x86: store pending bitmapped IPIs in per-cpu areas
This gets rid of the global cpu_ipi_pending array.
While replace cmpset with fcmpset in the delivery code and opportunistically
check if given IPI is already pending.
Sponsored by: The FreeBSD Foundation
Modified:
head/sys/amd64/amd64/mp_machdep.c
head/sys/amd64/include/pcpu.h
head/sys/i386/i386/mp_machdep.c
head/sys/i386/include/pcpu.h
head/sys/x86/include/x86_smp.h
head/sys/x86/x86/mp_x86.c
Modified: head/sys/amd64/amd64/mp_machdep.c
==============================================================================
--- head/sys/amd64/amd64/mp_machdep.c Sun May 12 06:34:58 2019
(r347501)
+++ head/sys/amd64/amd64/mp_machdep.c Sun May 12 06:36:54 2019
(r347502)
@@ -193,7 +193,6 @@ cpu_mp_start(void)
/* Initialize the logical ID to APIC ID table. */
for (i = 0; i < MAXCPU; i++) {
cpu_apic_ids[i] = -1;
- cpu_ipi_pending[i] = 0;
}
/* Install an inter-CPU IPI for TLB invalidation */
Modified: head/sys/amd64/include/pcpu.h
==============================================================================
--- head/sys/amd64/include/pcpu.h Sun May 12 06:34:58 2019
(r347501)
+++ head/sys/amd64/include/pcpu.h Sun May 12 06:36:54 2019
(r347502)
@@ -84,7 +84,8 @@ _Static_assert(sizeof(struct monitorbuf) == 128, "2x c
uint32_t pc_pcid_gen; \
uint32_t pc_smp_tlb_done; /* TLB op acknowledgement */ \
uint32_t pc_ibpb_set; \
- char __pad[3288] /* pad to UMA_PCPU_ALLOC_SIZE */
+ u_int pc_ipi_bitmap; \
+ char __pad[3284] /* pad to UMA_PCPU_ALLOC_SIZE */
#define PC_DBREG_CMD_NONE 0
#define PC_DBREG_CMD_LOAD 1
Modified: head/sys/i386/i386/mp_machdep.c
==============================================================================
--- head/sys/i386/i386/mp_machdep.c Sun May 12 06:34:58 2019
(r347501)
+++ head/sys/i386/i386/mp_machdep.c Sun May 12 06:36:54 2019
(r347502)
@@ -153,7 +153,6 @@ cpu_mp_start(void)
/* Initialize the logical ID to APIC ID table. */
for (i = 0; i < MAXCPU; i++) {
cpu_apic_ids[i] = -1;
- cpu_ipi_pending[i] = 0;
}
/* Install an inter-CPU IPI for TLB invalidation */
Modified: head/sys/i386/include/pcpu.h
==============================================================================
--- head/sys/i386/include/pcpu.h Sun May 12 06:34:58 2019
(r347501)
+++ head/sys/i386/include/pcpu.h Sun May 12 06:36:54 2019
(r347502)
@@ -87,7 +87,8 @@ _Static_assert(sizeof(struct monitorbuf) == 128, "2x c
caddr_t pc_pmap_eh_ptep;
\
uint32_t pc_smp_tlb_done; /* TLB op acknowledgement */ \
uint32_t pc_ibpb_set; \
- char __pad[3610]
+ u_int pc_ipi_bitmap; \
+ char __pad[3606]
#ifdef _KERNEL
Modified: head/sys/x86/include/x86_smp.h
==============================================================================
--- head/sys/x86/include/x86_smp.h Sun May 12 06:34:58 2019
(r347501)
+++ head/sys/x86/include/x86_smp.h Sun May 12 06:36:54 2019
(r347502)
@@ -34,7 +34,6 @@ extern char *bootSTK;
extern void *bootstacks[];
extern unsigned int boot_address;
extern unsigned int bootMP_size;
-extern volatile u_int cpu_ipi_pending[];
extern volatile int aps_ready;
extern struct mtx ap_boot_mtx;
extern int cpu_logical;
Modified: head/sys/x86/x86/mp_x86.c
==============================================================================
--- head/sys/x86/x86/mp_x86.c Sun May 12 06:34:58 2019 (r347501)
+++ head/sys/x86/x86/mp_x86.c Sun May 12 06:36:54 2019 (r347502)
@@ -137,9 +137,6 @@ _Static_assert(MAXCPU <= MAX_APIC_ID,
_Static_assert(xAPIC_MAX_APIC_ID <= MAX_APIC_ID,
"xAPIC_MAX_APIC_ID cannot be larger that MAX_APIC_ID");
-/* Holds pending bitmap based IPIs per CPU */
-volatile u_int cpu_ipi_pending[MAXCPU];
-
static void release_aps(void *dummy);
static void cpustop_handler_post(u_int cpu);
@@ -1224,19 +1221,24 @@ ipi_startup(int apic_id, int vector)
void
ipi_send_cpu(int cpu, u_int ipi)
{
- u_int bitmap, old_pending, new_pending;
+ u_int bitmap, old, new;
+ u_int *cpu_bitmap;
KASSERT(cpu_apic_ids[cpu] != -1, ("IPI to non-existent CPU %d", cpu));
if (IPI_IS_BITMAPED(ipi)) {
bitmap = 1 << ipi;
ipi = IPI_BITMAP_VECTOR;
- do {
- old_pending = cpu_ipi_pending[cpu];
- new_pending = old_pending | bitmap;
- } while (!atomic_cmpset_int(&cpu_ipi_pending[cpu],
- old_pending, new_pending));
- if (old_pending)
+ cpu_bitmap = &cpuid_to_pcpu[cpu]->pc_ipi_bitmap;
+ old = *cpu_bitmap;
+ for (;;) {
+ if ((old & bitmap) == bitmap)
+ break;
+ new = old | bitmap;
+ if (atomic_fcmpset_int(cpu_bitmap, &old, new))
+ break;
+ }
+ if (old)
return;
}
lapic_ipi_vectored(ipi, cpu_apic_ids[cpu]);
@@ -1255,7 +1257,7 @@ ipi_bitmap_handler(struct trapframe frame)
td->td_intr_nesting_level++;
oldframe = td->td_intr_frame;
td->td_intr_frame = &frame;
- ipi_bitmap = atomic_readandclear_int(&cpu_ipi_pending[cpu]);
+ ipi_bitmap =
atomic_readandclear_int(&cpuid_to_pcpu[cpu]->pc_ipi_bitmap);
if (ipi_bitmap & (1 << IPI_PREEMPT)) {
#ifdef COUNT_IPIS
(*ipi_preempt_counts[cpu])++;
_______________________________________________
svn-src-all@freebsd.org mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-all
To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"
