Author: mav
Date: Fri Aug 7 00:40:28 2020
New Revision: 364000
URL: https://svnweb.freebsd.org/changeset/base/364000
Log:
MFC r363527: Allow swi_sched() to be called from NMI context.
For purposes of handling hardware error reported via NMIs I need a way to
escape NMI context, being too restrictive to do something significant.
To do it this change introduces new swi_sched() flag SWI_FROMNMI, making
it careful about used KPIs. On platforms allowing IPI sending from NMI
context (x86 for now) it immediately wakes clk_intr_event via new IPI_SWI,
otherwise it works just like SWI_DELAY.
Modified:
stable/12/share/man/man9/swi.9
stable/12/sys/amd64/amd64/apic_vector.S
stable/12/sys/amd64/amd64/mp_machdep.c
stable/12/sys/amd64/include/smp.h
stable/12/sys/i386/i386/apic_vector.s
stable/12/sys/i386/i386/mp_machdep.c
stable/12/sys/kern/kern_intr.c
stable/12/sys/sys/interrupt.h
stable/12/sys/x86/include/apicvar.h
stable/12/sys/x86/include/x86_smp.h
stable/12/sys/x86/x86/mp_x86.c
stable/12/sys/x86/xen/xen_apic.c
Directory Properties:
stable/12/ (props changed)
Modified: stable/12/share/man/man9/swi.9
==============================================================================
--- stable/12/share/man/man9/swi.9 Fri Aug 7 00:33:28 2020
(r363999)
+++ stable/12/share/man/man9/swi.9 Fri Aug 7 00:40:28 2020
(r364000)
@@ -24,7 +24,7 @@
.\"
.\" $FreeBSD$
.\"
-.Dd April 19, 2012
+.Dd July 25, 2020
.Dt SWI 9
.Os
.Sh NAME
@@ -133,7 +133,7 @@ The
.Fa flags
argument specifies how and when the handler should be run and is a mask of one
or more of the following flags:
-.Bl -tag -width SWI_DELAY
+.Bl -tag -width SWI_FROMNMI
.It Dv SWI_DELAY
Specifies that the kernel should mark the specified handler as needing to run,
but the kernel should not schedule the software interrupt thread to run.
@@ -147,6 +147,13 @@ functionality performed by
.Fn setdelayed
in earlier versions of
.Fx .
+.It Dv SWI_FROMNMI
+Specifies that
+.Fn swi_sched
+is called from NMI context and should be careful about used KPIs.
+On platforms allowing IPI sending from NMI context it immediately wakes
+.Va clk_intr_event
+via the IPI, otherwise it works just like SWI_DELAY.
.El
.Pp
The
Modified: stable/12/sys/amd64/amd64/apic_vector.S
==============================================================================
--- stable/12/sys/amd64/amd64/apic_vector.S Fri Aug 7 00:33:28 2020
(r363999)
+++ stable/12/sys/amd64/amd64/apic_vector.S Fri Aug 7 00:40:28 2020
(r364000)
@@ -256,6 +256,16 @@ invltlb_ret:
jmp doreti
/*
+ * Executed by a CPU when it receives an IPI_SWI.
+ */
+ INTR_HANDLER ipi_swi
+ call as_lapic_eoi
+ FAKE_MCOUNT(TF_RIP(%rsp))
+ call ipi_swi_handler
+ MEXITCOUNT
+ jmp doreti
+
+/*
* Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU.
*
* - Calls the generic rendezvous action function.
Modified: stable/12/sys/amd64/amd64/mp_machdep.c
==============================================================================
--- stable/12/sys/amd64/amd64/mp_machdep.c Fri Aug 7 00:33:28 2020
(r363999)
+++ stable/12/sys/amd64/amd64/mp_machdep.c Fri Aug 7 00:40:28 2020
(r364000)
@@ -251,6 +251,10 @@ cpu_mp_start(void)
setidt(IPI_SUSPEND, pti ? IDTVEC(cpususpend_pti) : IDTVEC(cpususpend),
SDT_SYSIGT, SEL_KPL, 0);
+ /* Install an IPI for calling delayed SWI */
+ setidt(IPI_SWI, pti ? IDTVEC(ipi_swi_pti) : IDTVEC(ipi_swi),
+ SDT_SYSIGT, SEL_KPL, 0);
+
/* Set boot_cpu_id if needed. */
if (boot_cpu_id == -1) {
boot_cpu_id = PCPU_GET(apic_id);
Modified: stable/12/sys/amd64/include/smp.h
==============================================================================
--- stable/12/sys/amd64/include/smp.h Fri Aug 7 00:33:28 2020
(r363999)
+++ stable/12/sys/amd64/include/smp.h Fri Aug 7 00:40:28 2020
(r364000)
@@ -46,6 +46,7 @@ inthand_t
IDTVEC(invlrng_pcid),
IDTVEC(invlcache_pti),
IDTVEC(ipi_intr_bitmap_handler_pti),
+ IDTVEC(ipi_swi_pti),
IDTVEC(cpustop_pti),
IDTVEC(cpususpend_pti),
IDTVEC(rendezvous_pti);
Modified: stable/12/sys/i386/i386/apic_vector.s
==============================================================================
--- stable/12/sys/i386/i386/apic_vector.s Fri Aug 7 00:33:28 2020
(r363999)
+++ stable/12/sys/i386/i386/apic_vector.s Fri Aug 7 00:40:28 2020
(r364000)
@@ -309,6 +309,23 @@ IDTVEC(cpususpend)
jmp doreti
/*
+ * Executed by a CPU when it receives an IPI_SWI.
+ */
+ .text
+ SUPERALIGN_TEXT
+IDTVEC(ipi_swi)
+ PUSH_FRAME
+ SET_KERNEL_SREGS
+ cld
+ KENTER
+ call as_lapic_eoi
+ FAKE_MCOUNT(TF_EIP(%esp))
+ movl $ipi_swi_handler, %eax
+ call *%eax
+ MEXITCOUNT
+ jmp doreti
+
+/*
* Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU.
*
* - Calls the generic rendezvous action function.
Modified: stable/12/sys/i386/i386/mp_machdep.c
==============================================================================
--- stable/12/sys/i386/i386/mp_machdep.c Fri Aug 7 00:33:28 2020
(r363999)
+++ stable/12/sys/i386/i386/mp_machdep.c Fri Aug 7 00:40:28 2020
(r364000)
@@ -188,6 +188,10 @@ cpu_mp_start(void)
setidt(IPI_SUSPEND, IDTVEC(cpususpend),
SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
+ /* Install an IPI for calling delayed SWI */
+ setidt(IPI_SWI, IDTVEC(ipi_swi),
+ SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
+
/* Set boot_cpu_id if needed. */
if (boot_cpu_id == -1) {
boot_cpu_id = PCPU_GET(apic_id);
Modified: stable/12/sys/kern/kern_intr.c
==============================================================================
--- stable/12/sys/kern/kern_intr.c Fri Aug 7 00:33:28 2020
(r363999)
+++ stable/12/sys/kern/kern_intr.c Fri Aug 7 00:40:28 2020
(r364000)
@@ -59,6 +59,7 @@ __FBSDID("$FreeBSD$");
#include <machine/atomic.h>
#include <machine/cpu.h>
#include <machine/md_var.h>
+#include <machine/smp.h>
#include <machine/stdarg.h>
#ifdef DDB
#include <ddb/ddb.h>
@@ -994,7 +995,7 @@ swi_add(struct intr_event **eventp, const char *name,
void *arg, int pri, enum intr_type flags, void **cookiep)
{
struct intr_event *ie;
- int error;
+ int error = 0;
if (flags & INTR_ENTROPY)
return (EINVAL);
@@ -1012,8 +1013,10 @@ swi_add(struct intr_event **eventp, const char *name,
if (eventp != NULL)
*eventp = ie;
}
- error = intr_event_add_handler(ie, name, NULL, handler, arg,
- PI_SWI(pri), flags, cookiep);
+ if (handler != NULL) {
+ error = intr_event_add_handler(ie, name, NULL, handler, arg,
+ PI_SWI(pri), flags, cookiep);
+ }
return (error);
}
@@ -1031,9 +1034,11 @@ swi_sched(void *cookie, int flags)
CTR3(KTR_INTR, "swi_sched: %s %s need=%d", ie->ie_name, ih->ih_name,
ih->ih_need);
- entropy.event = (uintptr_t)ih;
- entropy.td = curthread;
- random_harvest_queue(&entropy, sizeof(entropy), RANDOM_SWI);
+ if ((flags & SWI_FROMNMI) == 0) {
+ entropy.event = (uintptr_t)ih;
+ entropy.td = curthread;
+ random_harvest_queue(&entropy, sizeof(entropy), RANDOM_SWI);
+ }
/*
* Set ih_need for this handler so that if the ithread is already
@@ -1042,7 +1047,16 @@ swi_sched(void *cookie, int flags)
*/
ih->ih_need = 1;
- if (!(flags & SWI_DELAY)) {
+ if (flags & SWI_DELAY)
+ return;
+
+ if (flags & SWI_FROMNMI) {
+#if defined(SMP) && (defined(__i386__) || defined(__amd64__))
+ KASSERT(ie == clk_intr_event,
+ ("SWI_FROMNMI used not with clk_intr_event"));
+ ipi_self_from_nmi(IPI_SWI);
+#endif
+ } else {
VM_CNT_INC(v_soft);
error = intr_event_schedule_thread(ie);
KASSERT(error == 0, ("stray software interrupt"));
@@ -1303,6 +1317,8 @@ intr_event_handle(struct intr_event *ie, struct trapfr
CK_SLIST_FOREACH(ih, &ie->ie_handlers, ih_next) {
if ((ih->ih_flags & IH_SUSP) != 0)
+ continue;
+ if ((ie->ie_flags & IE_SOFT) != 0 && ih->ih_need == 0)
continue;
if (ih->ih_filter == NULL) {
thread = true;
Modified: stable/12/sys/sys/interrupt.h
==============================================================================
--- stable/12/sys/sys/interrupt.h Fri Aug 7 00:33:28 2020
(r363999)
+++ stable/12/sys/sys/interrupt.h Fri Aug 7 00:40:28 2020
(r364000)
@@ -133,7 +133,8 @@ struct intr_event {
#define IE_ENTROPY 0x000002 /* Interrupt is an entropy
source. */
#define IE_ADDING_THREAD 0x000004 /* Currently building an
ithread. */
-/* Flags to pass to sched_swi. */
+/* Flags to pass to swi_sched. */
+#define SWI_FROMNMI 0x1
#define SWI_DELAY 0x2
/*
Modified: stable/12/sys/x86/include/apicvar.h
==============================================================================
--- stable/12/sys/x86/include/apicvar.h Fri Aug 7 00:33:28 2020
(r363999)
+++ stable/12/sys/x86/include/apicvar.h Fri Aug 7 00:40:28 2020
(r364000)
@@ -129,7 +129,8 @@
#define IPI_STOP (APIC_IPI_INTS + 6) /* Stop CPU until
restarted. */
#define IPI_SUSPEND (APIC_IPI_INTS + 7) /* Suspend CPU until
restarted. */
-#define IPI_DYN_FIRST (APIC_IPI_INTS + 8)
+#define IPI_SWI (APIC_IPI_INTS + 8) /* Run clk_intr_event.
*/
+#define IPI_DYN_FIRST (APIC_IPI_INTS + 9)
#define IPI_DYN_LAST (253) /* IPIs allocated at
runtime */
/*
Modified: stable/12/sys/x86/include/x86_smp.h
==============================================================================
--- stable/12/sys/x86/include/x86_smp.h Fri Aug 7 00:33:28 2020
(r363999)
+++ stable/12/sys/x86/include/x86_smp.h Fri Aug 7 00:40:28 2020
(r364000)
@@ -76,6 +76,7 @@ inthand_t
IDTVEC(invlrng), /* TLB shootdowns - page range */
IDTVEC(invlcache), /* Write back and invalidate cache */
IDTVEC(ipi_intr_bitmap_handler), /* Bitmap based IPIs */
+ IDTVEC(ipi_swi), /* Runs delayed SWI */
IDTVEC(cpustop), /* CPU stops & waits to be restarted */
IDTVEC(cpususpend), /* CPU suspends & waits to be resumed */
IDTVEC(rendezvous); /* handle CPU rendezvous */
@@ -100,6 +101,7 @@ void ipi_all_but_self(u_int ipi);
void ipi_bitmap_handler(struct trapframe frame);
void ipi_cpu(int cpu, u_int ipi);
int ipi_nmi_handler(void);
+void ipi_swi_handler(struct trapframe frame);
void ipi_selected(cpuset_t cpus, u_int ipi);
void ipi_self_from_nmi(u_int vector);
void set_interrupt_apic_ids(void);
Modified: stable/12/sys/x86/x86/mp_x86.c
==============================================================================
--- stable/12/sys/x86/x86/mp_x86.c Fri Aug 7 00:33:28 2020
(r363999)
+++ stable/12/sys/x86/x86/mp_x86.c Fri Aug 7 00:40:28 2020
(r364000)
@@ -44,6 +44,7 @@ __FBSDID("$FreeBSD$");
#ifdef GPROF
#include <sys/gmon.h>
#endif
+#include <sys/interrupt.h>
#include <sys/kdb.h>
#include <sys/kernel.h>
#include <sys/ktr.h>
@@ -1565,6 +1566,16 @@ invlcache_handler(void)
generation = smp_tlb_generation;
wbinvd();
PCPU_SET(smp_tlb_done, generation);
+}
+
+/*
+ * Handle an IPI_SWI by waking delayed SWI thread.
+ */
+void
+ipi_swi_handler(struct trapframe frame)
+{
+
+ intr_event_handle(clk_intr_event, &frame);
}
/*
Modified: stable/12/sys/x86/xen/xen_apic.c
==============================================================================
--- stable/12/sys/x86/xen/xen_apic.c Fri Aug 7 00:33:28 2020
(r363999)
+++ stable/12/sys/x86/xen/xen_apic.c Fri Aug 7 00:40:28 2020
(r364000)
@@ -72,6 +72,7 @@ static driver_filter_t xen_invlcache;
static driver_filter_t xen_ipi_bitmap_handler;
static driver_filter_t xen_cpustop_handler;
static driver_filter_t xen_cpususpend_handler;
+static driver_filter_t xen_ipi_swi_handler;
#endif
/*---------------------------------- Macros
----------------------------------*/
@@ -95,6 +96,7 @@ static struct xen_ipi_handler xen_ipis[] =
[IPI_TO_IDX(IPI_BITMAP_VECTOR)] = { xen_ipi_bitmap_handler, "b" },
[IPI_TO_IDX(IPI_STOP)] = { xen_cpustop_handler, "st" },
[IPI_TO_IDX(IPI_SUSPEND)] = { xen_cpususpend_handler, "sp" },
+ [IPI_TO_IDX(IPI_SWI)] = { xen_ipi_swi_handler, "sw" },
};
#endif
@@ -557,6 +559,15 @@ xen_cpususpend_handler(void *arg)
{
cpususpend_handler();
+ return (FILTER_HANDLED);
+}
+
+static int
+xen_ipi_swi_handler(void *arg)
+{
+ struct trapframe *frame = arg;
+
+ ipi_swi_handler(*frame);
return (FILTER_HANDLED);
}
_______________________________________________
[email protected] mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-all
To unsubscribe, send any mail to "[email protected]"
