Module Name: src Committed By: bouyer Date: Sat Apr 11 18:26:08 UTC 2020
Modified Files: src/sys/arch/amd64/amd64 [bouyer-xenpvh]: lock_stubs.S spl.S src/sys/arch/i386/i386 [bouyer-xenpvh]: spl.S src/sys/arch/x86/conf [bouyer-xenpvh]: files.x86 src/sys/arch/x86/include [bouyer-xenpvh]: intr.h src/sys/arch/x86/x86 [bouyer-xenpvh]: intr.c x86_machdep.c src/sys/arch/xen/conf [bouyer-xenpvh]: files.xen src/sys/arch/xen/include [bouyer-xenpvh]: intrdefs.h src/sys/arch/xen/x86 [bouyer-xenpvh]: xen_ipi.c xenfunc.c src/sys/arch/xen/xen [bouyer-xenpvh]: clock.c Added Files: src/sys/arch/x86/x86 [bouyer-xenpvh]: x86_softintr.c Log Message: Move softint and preemtion-related functions out of x86/x86/intr.c to its own file, x86/x86/x86_softintr.c Add x86/x86/x86_softintr.c for native and XenPV Make sure XenPV also check ci_ioending, which is used for softints. Switch XenPV to fast softints and allow kernel preemption. kpreempt_disable() before calling pmap_changeprot_local() run xen_wallclock_time() and xen_global_systime_ns() at splshed() to avoid being interrupted. XXX amd64 lock stubs are racy for XPENDING To generate a diff of this commit: cvs rdiff -u -r1.35 -r1.35.6.1 src/sys/arch/amd64/amd64/lock_stubs.S cvs rdiff -u -r1.43.4.4 -r1.43.4.5 src/sys/arch/amd64/amd64/spl.S cvs rdiff -u -r1.50.4.3 -r1.50.4.4 src/sys/arch/i386/i386/spl.S cvs rdiff -u -r1.107 -r1.107.10.1 src/sys/arch/x86/conf/files.x86 cvs rdiff -u -r1.61.6.1 -r1.61.6.2 src/sys/arch/x86/include/intr.h cvs rdiff -u -r1.150 -r1.150.6.1 src/sys/arch/x86/x86/intr.c cvs rdiff -u -r1.137.2.2 -r1.137.2.3 src/sys/arch/x86/x86/x86_machdep.c cvs rdiff -u -r0 -r1.1.2.1 src/sys/arch/x86/x86/x86_softintr.c cvs rdiff -u -r1.180 -r1.180.2.1 src/sys/arch/xen/conf/files.xen cvs rdiff -u -r1.15 -r1.15.2.1 src/sys/arch/xen/include/intrdefs.h cvs rdiff -u -r1.35 -r1.35.6.1 src/sys/arch/xen/x86/xen_ipi.c cvs rdiff -u -r1.26 -r1.26.8.1 src/sys/arch/xen/x86/xenfunc.c cvs rdiff -u -r1.80 -r1.80.6.1 src/sys/arch/xen/xen/clock.c Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files.
Modified files: Index: src/sys/arch/amd64/amd64/lock_stubs.S diff -u src/sys/arch/amd64/amd64/lock_stubs.S:1.35 src/sys/arch/amd64/amd64/lock_stubs.S:1.35.6.1 --- src/sys/arch/amd64/amd64/lock_stubs.S:1.35 Sun Dec 8 20:00:56 2019 +++ src/sys/arch/amd64/amd64/lock_stubs.S Sat Apr 11 18:26:06 2020 @@ -1,4 +1,4 @@ -/* $NetBSD: lock_stubs.S,v 1.35 2019/12/08 20:00:56 ad Exp $ */ +/* $NetBSD: lock_stubs.S,v 1.35.6.1 2020/04/11 18:26:06 bouyer Exp $ */ /* * Copyright (c) 2006, 2007, 2008, 2009 The NetBSD Foundation, Inc. @@ -126,15 +126,12 @@ ENTRY(mutex_spin_exit) jnz 1f cmpl CPU_INFO_ILEVEL(%r8), %edi jae 1f -#if !defined(XENPV) movl CPU_INFO_IUNMASK(%r8,%rdi,4), %esi CLI(ax) testl CPU_INFO_IPENDING(%r8), %esi jnz _C_LABEL(Xspllower) -#endif #if defined(XEN) movl CPU_INFO_XUNMASK(%r8,%rdi,4), %esi - CLI(ax) testl CPU_INFO_XPENDING(%r8), %esi jnz _C_LABEL(Xspllower) #endif @@ -155,14 +152,12 @@ ENTRY(mutex_spin_exit) cmpl %edx,%ecx /* new level is lower? */ jae 2f 1: -#if !defined(XENPV) movl CPU_INFO_IPENDING(%rsi),%eax testl %eax,CPU_INFO_IUNMASK(%rsi,%rcx,4)/* deferred interrupts? */ jnz 3f movl %eax,%ebx cmpxchg8b CPU_INFO_ISTATE(%rsi) /* swap in new ilevel */ jnz 4f -#endif #if defined(XEN) movl CPU_INFO_XPENDING(%rsi),%eax testl %eax,CPU_INFO_XUNMASK(%rsi,%rcx,4)/* deferred interrupts? */ Index: src/sys/arch/amd64/amd64/spl.S diff -u src/sys/arch/amd64/amd64/spl.S:1.43.4.4 src/sys/arch/amd64/amd64/spl.S:1.43.4.5 --- src/sys/arch/amd64/amd64/spl.S:1.43.4.4 Sat Apr 11 10:11:30 2020 +++ src/sys/arch/amd64/amd64/spl.S Sat Apr 11 18:26:06 2020 @@ -1,4 +1,4 @@ -/* $NetBSD: spl.S,v 1.43.4.4 2020/04/11 10:11:30 bouyer Exp $ */ +/* $NetBSD: spl.S,v 1.43.4.5 2020/04/11 18:26:06 bouyer Exp $ */ /* * Copyright (c) 2003 Wasabi Systems, Inc. @@ -91,7 +91,6 @@ ENTRY(splraise) ret END(splraise) -#ifndef XENPV /* * Xsoftintr() * @@ -148,11 +147,11 @@ IDTVEC(softintr) movq PCB_RSP0(%rdx),%rsp /* dispatch */ - sti + STI(di) movq %r15,%rdi /* interrupted LWP */ movl IS_MAXLEVEL(%rax),%esi /* ipl to run at */ call _C_LABEL(softint_dispatch)/* run handlers */ - cli + CLI(di) /* restore old context */ movq L_PCB(%r15),%rcx @@ -174,7 +173,7 @@ IDTVEC_END(softintr) */ ENTRY(softintr_ret) incl CPUVAR(MTX_COUNT) /* re-adjust after mi_switch */ - cli + CLI(ax) /* %rax not used by Xspllower/Xdoreti */ jmp *%r13 /* back to Xspllower/Xdoreti */ END(softintr_ret) @@ -196,11 +195,11 @@ END(softint_trigger) */ IDTVEC(recurse_preempt) movl $IPL_PREEMPT,CPUVAR(ILEVEL) - sti + STI(di) xorq %rdi,%rdi KMSAN_INIT_ARG(8) call _C_LABEL(kpreempt) - cli + CLI(di) jmp *%r13 /* back to Xspllower */ IDTVEC_END(recurse_preempt) @@ -211,20 +210,19 @@ IDTVEC_END(recurse_preempt) */ IDTVEC(resume_preempt) movl $IPL_PREEMPT,CPUVAR(ILEVEL) - sti + STI(ax) testq $SEL_RPL,TF_CS(%rsp) jnz 1f movq TF_RIP(%rsp),%rdi KMSAN_INIT_ARG(8) call _C_LABEL(kpreempt) /* from kernel */ - cli + CLI(ax) jmp *%r13 /* back to Xdoreti */ 1: call _C_LABEL(preempt) /* from user */ - cli + CLI(ax) jmp *%r13 /* back to Xdoreti */ IDTVEC_END(resume_preempt) -#endif /* XEN */ /* * void spllower(int s); @@ -336,7 +334,6 @@ IDTVEC(spllower) movl %edi,%ebx leaq 1f(%rip),%r13 /* address to resume loop at */ 1: -#if !defined(XENPV) movl %ebx,%eax /* get cpl */ movl CPUVAR(IUNMASK)(,%rax,4),%eax CLI(si) @@ -346,7 +343,6 @@ IDTVEC(spllower) btrl %eax,CPUVAR(IPENDING) movq CPUVAR(ISOURCES)(,%rax,8),%rax jmp *IS_RECURSE(%rax) -#endif 2: #if defined(XEN) movl %ebx,%eax /* get cpl */ @@ -382,7 +378,6 @@ IDTVEC(doreti) decl CPUVAR(IDEPTH) leaq 1f(%rip),%r13 1: -#if !defined(XENPV) movl %ebx,%eax movl CPUVAR(IUNMASK)(,%rax,4),%eax CLI(si) @@ -392,7 +387,6 @@ IDTVEC(doreti) btrl %eax,CPUVAR(IPENDING) movq CPUVAR(ISOURCES)(,%rax,8),%rax jmp *IS_RESUME(%rax) -#endif 2: #if defined(XEN) movl %ebx,%eax Index: src/sys/arch/i386/i386/spl.S diff -u src/sys/arch/i386/i386/spl.S:1.50.4.3 src/sys/arch/i386/i386/spl.S:1.50.4.4 --- src/sys/arch/i386/i386/spl.S:1.50.4.3 Sat Apr 11 12:01:42 2020 +++ src/sys/arch/i386/i386/spl.S Sat Apr 11 18:26:07 2020 @@ -1,4 +1,4 @@ -/* $NetBSD: spl.S,v 1.50.4.3 2020/04/11 12:01:42 bouyer Exp $ */ +/* $NetBSD: spl.S,v 1.50.4.4 2020/04/11 18:26:07 bouyer Exp $ */ /* * Copyright (c) 1998, 2007, 2008 The NetBSD Foundation, Inc. @@ -30,7 +30,7 @@ */ #include <machine/asm.h> -__KERNEL_RCSID(0, "$NetBSD: spl.S,v 1.50.4.3 2020/04/11 12:01:42 bouyer Exp $"); +__KERNEL_RCSID(0, "$NetBSD: spl.S,v 1.50.4.4 2020/04/11 18:26:07 bouyer Exp $"); #include "opt_ddb.h" #include "opt_spldebug.h" @@ -213,7 +213,6 @@ IDTVEC(spllower) jz .Lspllower_panic #endif /* XENPV */ #endif /* defined(DEBUG) */ -#if !defined(XENPV) movl %ebx,%eax /* get cpl */ movl CPUVAR(IUNMASK)(,%eax,4),%eax andl CPUVAR(IPENDING),%eax /* any non-masked bits left? */ @@ -222,7 +221,6 @@ IDTVEC(spllower) btrl %eax,CPUVAR(IPENDING) movl CPUVAR(ISOURCES)(,%eax,4),%eax jmp *IS_RECURSE(%eax) -#endif 2: #if defined(XEN) movl %ebx,%eax /* get cpl */ @@ -361,7 +359,6 @@ END(doreti_checkast) #endif IDTVEC_END(doreti) -#ifndef XENPV /* * Xsoftintr() * @@ -379,6 +376,7 @@ IDTVEC(softintr) pushl %esi pushl %edi movl $IPL_HIGH,CPUVAR(ILEVEL) + STI(%esi) movl CPUVAR(CURLWP),%esi movl IS_LWP(%eax),%edi /* switch to handler LWP */ movl %edi,CPUVAR(CURLWP) @@ -387,12 +385,11 @@ IDTVEC(softintr) movl %esp,PCB_ESP(%ecx) movl %ebp,PCB_EBP(%ecx) movl PCB_ESP0(%edx),%esp /* onto new stack */ - sti pushl IS_MAXLEVEL(%eax) /* ipl to run at */ pushl %esi call _C_LABEL(softint_dispatch)/* run handlers */ addl $8,%esp - cli + CLI(%ecx) movl L_PCB(%esi),%ecx movl PCB_ESP(%ecx),%esp xchgl %esi,CPUVAR(CURLWP) /* must be globally visible */ @@ -412,7 +409,7 @@ IDTVEC_END(softintr) */ ENTRY(softintr_ret) incl CPUVAR(MTX_COUNT) /* re-adjust after mi_switch */ - cli + CLI(%eax) jmp *%esi /* back to splx/doreti */ END(softintr_ret) @@ -434,11 +431,11 @@ END(softint_trigger) */ IDTVEC(recurse_preempt) movl $IPL_PREEMPT,CPUVAR(ILEVEL) - sti + STI(%eax) pushl $0 call _C_LABEL(kpreempt) addl $4,%esp - cli + CLI(%eax) jmp *%esi IDTVEC_END(recurse_preempt) @@ -449,18 +446,17 @@ IDTVEC_END(recurse_preempt) */ IDTVEC(resume_preempt) movl $IPL_PREEMPT,CPUVAR(ILEVEL) - sti + STI(%eax) testb $CHK_UPL,TF_CS(%esp) jnz 1f movl TF_EIP(%esp),%eax pushl %eax call _C_LABEL(kpreempt) /* from kernel */ addl $4,%esp - cli + CLI(%eax) jmp *%esi 1: call _C_LABEL(preempt) /* from user */ - cli + CLI(%eax) jmp *%esi IDTVEC_END(resume_preempt) -#endif /* !XENPV */ Index: src/sys/arch/x86/conf/files.x86 diff -u src/sys/arch/x86/conf/files.x86:1.107 src/sys/arch/x86/conf/files.x86:1.107.10.1 --- src/sys/arch/x86/conf/files.x86:1.107 Fri Feb 15 08:54:01 2019 +++ src/sys/arch/x86/conf/files.x86 Sat Apr 11 18:26:07 2020 @@ -1,4 +1,4 @@ -# $NetBSD: files.x86,v 1.107 2019/02/15 08:54:01 nonaka Exp $ +# $NetBSD: files.x86,v 1.107.10.1 2020/04/11 18:26:07 bouyer Exp $ # options for MP configuration through the MP spec defflag opt_mpbios.h MPBIOS MPDEBUG MPBIOS_SCANPCI @@ -93,6 +93,7 @@ file arch/x86/x86/genfb_machdep.c machde file arch/x86/x86/identcpu.c machdep file arch/x86/x86/i8259.c machdep file arch/x86/x86/intr.c machdep +file arch/x86/x86/x86_softintr.c machdep file arch/x86/x86/kgdb_machdep.c kgdb file arch/x86/x86/nmi.c machdep file arch/x86/x86/idt.c machdep Index: src/sys/arch/x86/include/intr.h diff -u src/sys/arch/x86/include/intr.h:1.61.6.1 src/sys/arch/x86/include/intr.h:1.61.6.2 --- src/sys/arch/x86/include/intr.h:1.61.6.1 Fri Apr 10 14:42:00 2020 +++ src/sys/arch/x86/include/intr.h Sat Apr 11 18:26:07 2020 @@ -1,4 +1,4 @@ -/* $NetBSD: intr.h,v 1.61.6.1 2020/04/10 14:42:00 bouyer Exp $ */ +/* $NetBSD: intr.h,v 1.61.6.2 2020/04/11 18:26:07 bouyer Exp $ */ /*- * Copyright (c) 1998, 2001, 2006, 2007, 2008, 2019 The NetBSD Foundation, Inc. @@ -32,12 +32,10 @@ #ifndef _X86_INTR_H_ #define _X86_INTR_H_ -#if !defined(XENPV) #define __HAVE_FAST_SOFTINTS #if !defined(NO_PREEMPTION) #define __HAVE_PREEMPTION #endif /* !defined(NO_PREEMPTION) */ -#endif /* !defined(XENPV) */ #ifdef _KERNEL #include <sys/types.h> @@ -239,6 +237,9 @@ const char *intr_create_intrid(int, stru struct intrsource *intr_allocate_io_intrsource(const char *); void intr_free_io_intrsource(const char *); +void x86_init_preempt(struct cpu_info *); +void x86_intr_calculatemasks(struct cpu_info *); + int x86_send_ipi(struct cpu_info *, int); void x86_broadcast_ipi(int); void x86_ipi_handler(void); Index: src/sys/arch/x86/x86/intr.c diff -u src/sys/arch/x86/x86/intr.c:1.150 src/sys/arch/x86/x86/intr.c:1.150.6.1 --- src/sys/arch/x86/x86/intr.c:1.150 Mon Dec 30 23:32:30 2019 +++ src/sys/arch/x86/x86/intr.c Sat Apr 11 18:26:07 2020 @@ -1,4 +1,4 @@ -/* $NetBSD: intr.c,v 1.150 2019/12/30 23:32:30 thorpej Exp $ */ +/* $NetBSD: intr.c,v 1.150.6.1 2020/04/11 18:26:07 bouyer Exp $ */ /* * Copyright (c) 2007, 2008, 2009, 2019 The NetBSD Foundation, Inc. @@ -133,7 +133,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: intr.c,v 1.150 2019/12/30 23:32:30 thorpej Exp $"); +__KERNEL_RCSID(0, "$NetBSD: intr.c,v 1.150.6.1 2020/04/11 18:26:07 bouyer Exp $"); #include "opt_intrdebug.h" #include "opt_multiprocessor.h" @@ -207,16 +207,6 @@ extern void Xrecurse_hyperv_hypercall(vo #define DPRINTF(msg) #endif -struct pic softintr_pic = { - .pic_name = "softintr_fakepic", - .pic_type = PIC_SOFT, - .pic_vecbase = 0, - .pic_apicid = 0, - .pic_lock = __SIMPLELOCK_UNLOCKED, -}; - -static void intr_calculatemasks(struct cpu_info *); - static SIMPLEQ_HEAD(, intrsource) io_interrupt_sources = SIMPLEQ_HEAD_INITIALIZER(io_interrupt_sources); @@ -290,73 +280,6 @@ x86_nmi(void) } /* - * Recalculate the interrupt masks from scratch. - * During early boot, anything goes and we are always called on the BP. - * When the system is up and running: - * - * => called with ci == curcpu() - * => cpu_lock held by the initiator - * => interrupts disabled on-chip (PSL_I) - * - * Do not call printf(), kmem_free() or other "heavyweight" routines - * from here. This routine must be quick and must not block. - */ -static void -intr_calculatemasks(struct cpu_info *ci) -{ - int irq, level, unusedirqs, intrlevel[MAX_INTR_SOURCES]; - struct intrhand *q; - - /* First, figure out which levels each IRQ uses. */ - unusedirqs = 0xffffffff; - for (irq = 0; irq < MAX_INTR_SOURCES; irq++) { - int levels = 0; - - if (ci->ci_isources[irq] == NULL) { - intrlevel[irq] = 0; - continue; - } - for (q = ci->ci_isources[irq]->is_handlers; q; q = q->ih_next) - levels |= 1U << q->ih_level; - intrlevel[irq] = levels; - if (levels) - unusedirqs &= ~(1U << irq); - } - - /* Then figure out which IRQs use each level. */ - for (level = 0; level < NIPL; level++) { - int irqs = 0; - for (irq = 0; irq < MAX_INTR_SOURCES; irq++) - if (intrlevel[irq] & (1U << level)) - irqs |= 1U << irq; - ci->ci_imask[level] = irqs | unusedirqs; - } - - for (level = 0; level<(NIPL-1); level++) - ci->ci_imask[level+1] |= ci->ci_imask[level]; - - for (irq = 0; irq < MAX_INTR_SOURCES; irq++) { - int maxlevel = IPL_NONE; - int minlevel = IPL_HIGH; - - if (ci->ci_isources[irq] == NULL) - continue; - for (q = ci->ci_isources[irq]->is_handlers; q; - q = q->ih_next) { - if (q->ih_level < minlevel) - minlevel = q->ih_level; - if (q->ih_level > maxlevel) - maxlevel = q->ih_level; - } - ci->ci_isources[irq]->is_maxlevel = maxlevel; - ci->ci_isources[irq]->is_minlevel = minlevel; - } - - for (level = 0; level < NIPL; level++) - ci->ci_iunmask[level] = ~ci->ci_imask[level]; -} - -/* * Create an interrupt id such as "ioapic0 pin 9". This interrupt id is used * by MI code and intrctl(8). */ @@ -801,7 +724,7 @@ intr_establish_xcall(void *arg1, void *a /* Link in the handler and re-calculate masks. */ *(ih->ih_prevp) = ih; - intr_calculatemasks(ci); + x86_intr_calculatemasks(ci); /* Hook in new IDT vector and SPL state. */ if (source->is_resume == NULL || source->is_idtvec != idt_vec) { @@ -1186,7 +1109,7 @@ intr_disestablish_xcall(void *arg1, void *p = q->ih_next; - intr_calculatemasks(ci); + x86_intr_calculatemasks(ci); /* * If there is no any handler, 1) do delroute because it has no * any source and 2) dont' hwunmask to prevent spurious interrupt. @@ -1327,15 +1250,10 @@ intr_string(intr_handle_t ih, char *buf, /* * Fake interrupt handler structures for the benefit of symmetry with - * other interrupt sources, and the benefit of intr_calculatemasks() + * other interrupt sources, and the benefit of x86_intr_calculatemasks() */ -struct intrhand fake_softclock_intrhand; -struct intrhand fake_softnet_intrhand; -struct intrhand fake_softserial_intrhand; -struct intrhand fake_softbio_intrhand; struct intrhand fake_timer_intrhand; struct intrhand fake_ipi_intrhand; -struct intrhand fake_preempt_intrhand; #if NHYPERV > 0 struct intrhand fake_hyperv_intrhand; #endif @@ -1369,7 +1287,7 @@ redzone_const_or_zero(int x) void cpu_intr_init(struct cpu_info *ci) { -#if (NLAPIC > 0) || defined(MULTIPROCESSOR) || defined(__HAVE_PREEMPTION) || \ +#if (NLAPIC > 0) || defined(MULTIPROCESSOR) || \ (NHYPERV > 0) struct intrsource *isp; #endif @@ -1423,16 +1341,10 @@ cpu_intr_init(struct cpu_info *ci) #endif #if defined(__HAVE_PREEMPTION) - isp = kmem_zalloc(sizeof(*isp), KM_SLEEP); - isp->is_recurse = Xrecurse_preempt; - isp->is_resume = Xresume_preempt; - fake_preempt_intrhand.ih_level = IPL_PREEMPT; - isp->is_handlers = &fake_preempt_intrhand; - isp->is_pic = &softintr_pic; - ci->ci_isources[SIR_PREEMPT] = isp; + x86_init_preempt(ci); #endif - intr_calculatemasks(ci); + x86_intr_calculatemasks(ci); #if defined(INTRSTACKSIZE) vaddr_t istack; @@ -1521,55 +1433,6 @@ intr_printconfig(void) #endif -#if defined(__HAVE_FAST_SOFTINTS) -void -softint_init_md(lwp_t *l, u_int level, uintptr_t *machdep) -{ - struct intrsource *isp; - struct cpu_info *ci; - u_int sir; - - ci = l->l_cpu; - - isp = kmem_zalloc(sizeof(*isp), KM_SLEEP); - isp->is_recurse = Xsoftintr; - isp->is_resume = Xsoftintr; - isp->is_pic = &softintr_pic; - - switch (level) { - case SOFTINT_BIO: - sir = SIR_BIO; - fake_softbio_intrhand.ih_level = IPL_SOFTBIO; - isp->is_handlers = &fake_softbio_intrhand; - break; - case SOFTINT_NET: - sir = SIR_NET; - fake_softnet_intrhand.ih_level = IPL_SOFTNET; - isp->is_handlers = &fake_softnet_intrhand; - break; - case SOFTINT_SERIAL: - sir = SIR_SERIAL; - fake_softserial_intrhand.ih_level = IPL_SOFTSERIAL; - isp->is_handlers = &fake_softserial_intrhand; - break; - case SOFTINT_CLOCK: - sir = SIR_CLOCK; - fake_softclock_intrhand.ih_level = IPL_SOFTCLOCK; - isp->is_handlers = &fake_softclock_intrhand; - break; - default: - panic("softint_init_md"); - } - - KASSERT(ci->ci_isources[sir] == NULL); - - *machdep = (1 << sir); - ci->ci_isources[sir] = isp; - ci->ci_isources[sir]->is_lwp = l; - - intr_calculatemasks(ci); -} -#endif /* __HAVE_FAST_SOFTINTS */ /* * Save current affinitied cpu's interrupt count. */ @@ -1628,7 +1491,7 @@ intr_redistribute_xc_t(void *arg1, void /* Hook it in and re-calculate masks. */ ci->ci_isources[slot] = isp; - intr_calculatemasks(curcpu()); + x86_intr_calculatemasks(curcpu()); /* Re-enable interrupts locally. */ x86_write_psl(psl); @@ -1682,7 +1545,7 @@ intr_redistribute_xc_s2(void *arg1, void /* Patch out the source and re-calculate masks. */ ci->ci_isources[slot] = NULL; - intr_calculatemasks(ci); + x86_intr_calculatemasks(ci); /* Re-enable interrupts locally. */ x86_write_psl(psl); @@ -1872,7 +1735,7 @@ intr_activate_xcall(void *arg1, void *ar psl = x86_read_psl(); x86_disable_intr(); - intr_calculatemasks(ci); + x86_intr_calculatemasks(ci); if (source->is_type == IST_LEVEL) { stubp = &source->is_pic->pic_level_stubs[slot]; @@ -1917,7 +1780,7 @@ intr_deactivate_xcall(void *arg1, void * ci->ci_nintrhand--; } - intr_calculatemasks(ci); + x86_intr_calculatemasks(ci); /* * Skip unsetgate(), because the same itd[] entry is overwritten in Index: src/sys/arch/x86/x86/x86_machdep.c diff -u src/sys/arch/x86/x86/x86_machdep.c:1.137.2.2 src/sys/arch/x86/x86/x86_machdep.c:1.137.2.3 --- src/sys/arch/x86/x86/x86_machdep.c:1.137.2.2 Sat Apr 11 08:06:16 2020 +++ src/sys/arch/x86/x86/x86_machdep.c Sat Apr 11 18:26:07 2020 @@ -1,4 +1,4 @@ -/* $NetBSD: x86_machdep.c,v 1.137.2.2 2020/04/11 08:06:16 bouyer Exp $ */ +/* $NetBSD: x86_machdep.c,v 1.137.2.3 2020/04/11 18:26:07 bouyer Exp $ */ /*- * Copyright (c) 2002, 2006, 2007 YAMAMOTO Takashi, @@ -31,7 +31,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: x86_machdep.c,v 1.137.2.2 2020/04/11 08:06:16 bouyer Exp $"); +__KERNEL_RCSID(0, "$NetBSD: x86_machdep.c,v 1.137.2.3 2020/04/11 18:26:07 bouyer Exp $"); #include "opt_modular.h" #include "opt_physmem.h" @@ -307,7 +307,11 @@ cpu_need_resched(struct cpu_info *ci, st #ifdef __HAVE_PREEMPTION if ((flags & RESCHED_KPREEMPT) != 0) { if ((flags & RESCHED_REMOTE) != 0) { +#ifdef XENPV + xen_send_ipi(ci, XEN_IPI_KPREEMPT); +#else x86_send_ipi(ci, X86_IPI_KPREEMPT); +#endif } else { softint_trigger(1 << SIR_PREEMPT); } Index: src/sys/arch/xen/conf/files.xen diff -u src/sys/arch/xen/conf/files.xen:1.180 src/sys/arch/xen/conf/files.xen:1.180.2.1 --- src/sys/arch/xen/conf/files.xen:1.180 Fri Apr 3 22:45:30 2020 +++ src/sys/arch/xen/conf/files.xen Sat Apr 11 18:26:07 2020 @@ -1,4 +1,4 @@ -# $NetBSD: files.xen,v 1.180 2020/04/03 22:45:30 ad Exp $ +# $NetBSD: files.xen,v 1.180.2.1 2020/04/11 18:26:07 bouyer Exp $ # NetBSD: files.x86,v 1.10 2003/10/08 17:30:00 bouyer Exp # NetBSD: files.i386,v 1.254 2004/03/25 23:32:10 jmc Exp @@ -148,6 +148,7 @@ file arch/xen/x86/pintr.c machdep & dom file arch/xen/x86/xen_ipi.c multiprocessor & xenpv file arch/x86/x86/idt.c machdep file arch/x86/x86/intr.c machdep & xenpvhvm +file arch/x86/x86/x86_softintr.c machdep file arch/x86/x86/ipi.c xenpvhvm file arch/x86/x86/pmap.c machdep file arch/x86/x86/x86_tlb.c machdep Index: src/sys/arch/xen/include/intrdefs.h diff -u src/sys/arch/xen/include/intrdefs.h:1.15 src/sys/arch/xen/include/intrdefs.h:1.15.2.1 --- src/sys/arch/xen/include/intrdefs.h:1.15 Fri Apr 3 22:20:36 2020 +++ src/sys/arch/xen/include/intrdefs.h Sat Apr 11 18:26:07 2020 @@ -1,4 +1,4 @@ -/* $NetBSD: intrdefs.h,v 1.15 2020/04/03 22:20:36 ad Exp $ */ +/* $NetBSD: intrdefs.h,v 1.15.2.1 2020/04/11 18:26:07 bouyer Exp $ */ /* This file co-exists, and is included via machine/intrdefs.h */ @@ -14,9 +14,10 @@ #define XEN_IPI_HVCB 0x00000010 #define XEN_IPI_GENERIC 0x00000020 #define XEN_IPI_AST 0x00000040 +#define XEN_IPI_KPREEMPT 0x00000080 /* Note: IPI_KICK does not have a handler. */ -#define XEN_NIPIS 7 +#define XEN_NIPIS 8 /* The number of 'irqs' that XEN understands */ #define NUM_XEN_IRQS 256 @@ -24,6 +25,6 @@ #define XEN_IPI_NAMES { "halt IPI", "FPU synch IPI", \ "DDB IPI", "xcall IPI", \ "HVCB IPI", "generic IPI", \ - "AST IPI" } + "AST IPI", "kpreempt IPI" } #endif /* _XEN_INTRDEFS_H_ */ Index: src/sys/arch/xen/x86/xen_ipi.c diff -u src/sys/arch/xen/x86/xen_ipi.c:1.35 src/sys/arch/xen/x86/xen_ipi.c:1.35.6.1 --- src/sys/arch/xen/x86/xen_ipi.c:1.35 Sun Dec 1 15:34:46 2019 +++ src/sys/arch/xen/x86/xen_ipi.c Sat Apr 11 18:26:07 2020 @@ -1,4 +1,4 @@ -/* $NetBSD: xen_ipi.c,v 1.35 2019/12/01 15:34:46 ad Exp $ */ +/* $NetBSD: xen_ipi.c,v 1.35.6.1 2020/04/11 18:26:07 bouyer Exp $ */ /*- * Copyright (c) 2011, 2019 The NetBSD Foundation, Inc. @@ -33,10 +33,10 @@ /* * Based on: x86/ipi.c - * __KERNEL_RCSID(0, "$NetBSD: xen_ipi.c,v 1.35 2019/12/01 15:34:46 ad Exp $"); + * __KERNEL_RCSID(0, "$NetBSD: xen_ipi.c,v 1.35.6.1 2020/04/11 18:26:07 bouyer Exp $"); */ -__KERNEL_RCSID(0, "$NetBSD: xen_ipi.c,v 1.35 2019/12/01 15:34:46 ad Exp $"); +__KERNEL_RCSID(0, "$NetBSD: xen_ipi.c,v 1.35.6.1 2020/04/11 18:26:07 bouyer Exp $"); #include "opt_ddb.h" @@ -72,6 +72,7 @@ static void xen_ipi_xcall(struct cpu_inf static void xen_ipi_hvcb(struct cpu_info *, struct intrframe *); static void xen_ipi_generic(struct cpu_info *, struct intrframe *); static void xen_ipi_ast(struct cpu_info *, struct intrframe *); +static void xen_ipi_kpreempt(struct cpu_info *ci, struct intrframe *); static void (*ipifunc[XEN_NIPIS])(struct cpu_info *, struct intrframe *) = { /* In order of priority (see: xen/include/intrdefs.h */ @@ -85,7 +86,8 @@ static void (*ipifunc[XEN_NIPIS])(struct xen_ipi_xcall, xen_ipi_hvcb, xen_ipi_generic, - xen_ipi_ast + xen_ipi_ast, + xen_ipi_kpreempt }; static int @@ -342,3 +344,9 @@ xen_ipi_hvcb(struct cpu_info *ci, struct hypervisor_force_callback(); } + +static void +xen_ipi_kpreempt(struct cpu_info *ci, struct intrframe * intrf) +{ + softint_trigger(1 << SIR_PREEMPT); +} Index: src/sys/arch/xen/x86/xenfunc.c diff -u src/sys/arch/xen/x86/xenfunc.c:1.26 src/sys/arch/xen/x86/xenfunc.c:1.26.8.1 --- src/sys/arch/xen/x86/xenfunc.c:1.26 Sat May 4 11:15:49 2019 +++ src/sys/arch/xen/x86/xenfunc.c Sat Apr 11 18:26:07 2020 @@ -1,4 +1,4 @@ -/* $NetBSD: xenfunc.c,v 1.26 2019/05/04 11:15:49 kre Exp $ */ +/* $NetBSD: xenfunc.c,v 1.26.8.1 2020/04/11 18:26:07 bouyer Exp $ */ /* * Copyright (c) 2004 Christian Limpach. @@ -26,7 +26,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: xenfunc.c,v 1.26 2019/05/04 11:15:49 kre Exp $"); +__KERNEL_RCSID(0, "$NetBSD: xenfunc.c,v 1.26.8.1 2020/04/11 18:26:07 bouyer Exp $"); #include <sys/param.h> @@ -61,6 +61,7 @@ lidt(struct region_descriptor *rd) * will be available at the boot stage when this is called. */ static char xen_idt_page[PAGE_SIZE] __attribute__((__aligned__ (PAGE_SIZE))); + kpreempt_disable(); memset(xen_idt_page, 0, PAGE_SIZE); struct trap_info *xen_idt = (void * )xen_idt_page; @@ -96,6 +97,7 @@ lidt(struct region_descriptor *rd) /* reset */ pmap_changeprot_local((vaddr_t) xen_idt, VM_PROT_READ|VM_PROT_WRITE); #endif /* __x86_64 */ + kpreempt_enable(); } void @@ -256,6 +258,12 @@ rcr2(void) return curcpu()->ci_vcpu->arch.cr2; } +void +lcr2(register_t v) +{ + curcpu()->ci_vcpu->arch.cr2 = v; +} + #ifdef __x86_64__ void setusergs(int gssel) Index: src/sys/arch/xen/xen/clock.c diff -u src/sys/arch/xen/xen/clock.c:1.80 src/sys/arch/xen/xen/clock.c:1.80.6.1 --- src/sys/arch/xen/xen/clock.c:1.80 Wed Oct 16 18:29:49 2019 +++ src/sys/arch/xen/xen/clock.c Sat Apr 11 18:26:07 2020 @@ -1,4 +1,4 @@ -/* $NetBSD: clock.c,v 1.80 2019/10/16 18:29:49 christos Exp $ */ +/* $NetBSD: clock.c,v 1.80.6.1 2020/04/11 18:26:07 bouyer Exp $ */ /*- * Copyright (c) 2017, 2018 The NetBSD Foundation, Inc. @@ -36,7 +36,7 @@ #endif #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: clock.c,v 1.80 2019/10/16 18:29:49 christos Exp $"); +__KERNEL_RCSID(0, "$NetBSD: clock.c,v 1.80.6.1 2020/04/11 18:26:07 bouyer Exp $"); #include <sys/param.h> #include <sys/types.h> @@ -501,6 +501,7 @@ xen_wallclock_time(struct timespec *tsp) struct xen_wallclock_ticket ticket; uint64_t systime_ns; + int s = splsched(); /* make sure we won't be interrupted */ /* Read the last wall clock sample from the hypervisor. */ do { xen_wallclock_enter(&ticket); @@ -510,6 +511,7 @@ xen_wallclock_time(struct timespec *tsp) /* Get the global system time. */ systime_ns = xen_global_systime_ns(); + splx(s); /* Add the system time to the wall clock time. */ systime_ns += tsp->tv_nsec; @@ -530,7 +532,6 @@ xen_global_systime_ns(void) { struct cpu_info *ci; uint64_t local, global, result; - int bound; /* * Find the local timecount on this CPU, and make sure it does @@ -540,7 +541,7 @@ xen_global_systime_ns(void) * * XXX Can we avoid retrying if the CAS fails? */ - bound = curlwp_bind(); + int s = splsched(); /* make sure we won't be interrupted */ ci = curcpu(); do { local = xen_vcputime_systime_ns(); @@ -555,7 +556,7 @@ xen_global_systime_ns(void) } while (atomic_cas_64(&xen_global_systime_ns_stamp, global, result) != global); KASSERT(ci == curcpu()); - curlwp_bindx(bound); + splx(s); return result; } Added files: Index: src/sys/arch/x86/x86/x86_softintr.c diff -u /dev/null src/sys/arch/x86/x86/x86_softintr.c:1.1.2.1 --- /dev/null Sat Apr 11 18:26:08 2020 +++ src/sys/arch/x86/x86/x86_softintr.c Sat Apr 11 18:26:07 2020 @@ -0,0 +1,289 @@ +/* $NetBSD: x86_softintr.c,v 1.1.2.1 2020/04/11 18:26:07 bouyer Exp $ */ + +/* + * Copyright (c) 2007, 2008, 2009, 2019 The NetBSD Foundation, Inc. + * All rights reserved. + * + * This code is derived from software contributed to The NetBSD Foundation + * by Andrew Doran, and by Jason R. Thorpe. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * Copyright 2002 (c) Wasabi Systems, Inc. + * All rights reserved. + * + * Written by Frank van der Linden for Wasabi Systems, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed for the NetBSD Project by + * Wasabi Systems, Inc. + * 4. The name of Wasabi Systems, Inc. may not be used to endorse + * or promote products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/*- + * Copyright (c) 1991 The Regents of the University of California. + * All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * William Jolitz. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)isa.c 7.2 (Berkeley) 5/13/91 + */ + +/*- + * Copyright (c) 1993, 1994 Charles Hannum. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)isa.c 7.2 (Berkeley) 5/13/91 + */ + +#include <sys/cdefs.h> +__KERNEL_RCSID(0, "$NetBSD: x86_softintr.c,v 1.1.2.1 2020/04/11 18:26:07 bouyer Exp $"); + +#include <sys/kmem.h> +#include <sys/proc.h> +#include <sys/intr.h> + +struct pic softintr_pic = { + .pic_name = "softintr_fakepic", + .pic_type = PIC_SOFT, + .pic_vecbase = 0, + .pic_apicid = 0, + .pic_lock = __SIMPLELOCK_UNLOCKED, +}; + +/* + * Recalculate the interrupt masks from scratch. + * During early boot, anything goes and we are always called on the BP. + * When the system is up and running: + * + * => called with ci == curcpu() + * => cpu_lock held by the initiator + * => interrupts disabled on-chip (PSL_I) + * + * Do not call printf(), kmem_free() or other "heavyweight" routines + * from here. This routine must be quick and must not block. + */ +void +x86_intr_calculatemasks(struct cpu_info *ci) +{ + int irq, level, unusedirqs, intrlevel[MAX_INTR_SOURCES]; + struct intrhand *q; + + /* First, figure out which levels each IRQ uses. */ + unusedirqs = 0xffffffff; + for (irq = 0; irq < MAX_INTR_SOURCES; irq++) { + int levels = 0; + + if (ci->ci_isources[irq] == NULL) { + intrlevel[irq] = 0; + continue; + } + for (q = ci->ci_isources[irq]->is_handlers; q; q = q->ih_next) + levels |= 1U << q->ih_level; + intrlevel[irq] = levels; + if (levels) + unusedirqs &= ~(1U << irq); + } + + /* Then figure out which IRQs use each level. */ + for (level = 0; level < NIPL; level++) { + int irqs = 0; + for (irq = 0; irq < MAX_INTR_SOURCES; irq++) + if (intrlevel[irq] & (1U << level)) + irqs |= 1U << irq; + ci->ci_imask[level] = irqs | unusedirqs; + } + + for (level = 0; level<(NIPL-1); level++) + ci->ci_imask[level+1] |= ci->ci_imask[level]; + + for (irq = 0; irq < MAX_INTR_SOURCES; irq++) { + int maxlevel = IPL_NONE; + int minlevel = IPL_HIGH; + + if (ci->ci_isources[irq] == NULL) + continue; + for (q = ci->ci_isources[irq]->is_handlers; q; + q = q->ih_next) { + if (q->ih_level < minlevel) + minlevel = q->ih_level; + if (q->ih_level > maxlevel) + maxlevel = q->ih_level; + } + ci->ci_isources[irq]->is_maxlevel = maxlevel; + ci->ci_isources[irq]->is_minlevel = minlevel; + } + + for (level = 0; level < NIPL; level++) + ci->ci_iunmask[level] = ~ci->ci_imask[level]; +} + + + +#if defined(__HAVE_PREEMPTION) +struct intrhand fake_preempt_intrhand; + +void +x86_init_preempt(struct cpu_info *ci) +{ + struct intrsource *isp; + isp = kmem_zalloc(sizeof(*isp), KM_SLEEP); + isp->is_recurse = Xrecurse_preempt; + isp->is_resume = Xresume_preempt; + fake_preempt_intrhand.ih_level = IPL_PREEMPT; + isp->is_handlers = &fake_preempt_intrhand; + isp->is_pic = &softintr_pic; + ci->ci_isources[SIR_PREEMPT] = isp; +} +#endif + +#if defined(__HAVE_FAST_SOFTINTS) +struct intrhand fake_softclock_intrhand; +struct intrhand fake_softnet_intrhand; +struct intrhand fake_softserial_intrhand; +struct intrhand fake_softbio_intrhand; + +void +softint_init_md(lwp_t *l, u_int level, uintptr_t *machdep) +{ + struct intrsource *isp; + struct cpu_info *ci; + u_int sir; + + ci = l->l_cpu; + + isp = kmem_zalloc(sizeof(*isp), KM_SLEEP); + isp->is_recurse = Xsoftintr; + isp->is_resume = Xsoftintr; + isp->is_pic = &softintr_pic; + + switch (level) { + case SOFTINT_BIO: + sir = SIR_BIO; + fake_softbio_intrhand.ih_level = IPL_SOFTBIO; + isp->is_handlers = &fake_softbio_intrhand; + break; + case SOFTINT_NET: + sir = SIR_NET; + fake_softnet_intrhand.ih_level = IPL_SOFTNET; + isp->is_handlers = &fake_softnet_intrhand; + break; + case SOFTINT_SERIAL: + sir = SIR_SERIAL; + fake_softserial_intrhand.ih_level = IPL_SOFTSERIAL; + isp->is_handlers = &fake_softserial_intrhand; + break; + case SOFTINT_CLOCK: + sir = SIR_CLOCK; + fake_softclock_intrhand.ih_level = IPL_SOFTCLOCK; + isp->is_handlers = &fake_softclock_intrhand; + break; + default: + panic("softint_init_md"); + } + + KASSERT(ci->ci_isources[sir] == NULL); + + *machdep = (1 << sir); + ci->ci_isources[sir] = isp; + ci->ci_isources[sir]->is_lwp = l; + + x86_intr_calculatemasks(ci); +} +#endif /* __HAVE_FAST_SOFTINTS */