Module Name: src
Committed By: jmcneill
Date: Sat Sep 18 12:25:07 UTC 2021
Modified Files:
src/sys/arch/aarch64/aarch64: genassym.cf vectors.S
src/sys/arch/aarch64/include: cpu.h
src/sys/arch/arm/cortex: gic_splfuncs.c
src/sys/arch/evbarm/conf: std.generic64
Log Message:
gic_splx: performance optimizations
Avoid any kind of register access (DAIF, PMR, etc), barriers, and atomic
operations in the common case where no interrupt fires between spl being
raised and lowered.
This introduces a per-CPU return address (ci_splx_restart) used by the
vector handler to restart a sequence in splx that compares the new ipl
with the per-CPU hardware priority state stored in ci_hwpl.
To generate a diff of this commit:
cvs rdiff -u -r1.33 -r1.34 src/sys/arch/aarch64/aarch64/genassym.cf
cvs rdiff -u -r1.23 -r1.24 src/sys/arch/aarch64/aarch64/vectors.S
cvs rdiff -u -r1.38 -r1.39 src/sys/arch/aarch64/include/cpu.h
cvs rdiff -u -r1.1 -r1.2 src/sys/arch/arm/cortex/gic_splfuncs.c
cvs rdiff -u -r1.14 -r1.15 src/sys/arch/evbarm/conf/std.generic64
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/aarch64/aarch64/genassym.cf
diff -u src/sys/arch/aarch64/aarch64/genassym.cf:1.33 src/sys/arch/aarch64/aarch64/genassym.cf:1.34
--- src/sys/arch/aarch64/aarch64/genassym.cf:1.33 Fri Dec 11 18:03:33 2020
+++ src/sys/arch/aarch64/aarch64/genassym.cf Sat Sep 18 12:25:06 2021
@@ -1,4 +1,4 @@
-# $NetBSD: genassym.cf,v 1.33 2020/12/11 18:03:33 skrll Exp $
+# $NetBSD: genassym.cf,v 1.34 2021/09/18 12:25:06 jmcneill Exp $
#-
# Copyright (c) 2014 The NetBSD Foundation, Inc.
# All rights reserved.
@@ -296,6 +296,7 @@ define CI_MTX_COUNT offsetof(struct cpu
define CI_SOFTINTS offsetof(struct cpu_info, ci_softints)
define CI_IDLELWP offsetof(struct cpu_info, ci_data.cpu_idlelwp)
define CI_CC_NINTR offsetof(struct cpu_info, ci_data.cpu_nintr)
+define CI_SPLX_RESTART offsetof(struct cpu_info, ci_splx_restart)
define V_RESCHED_KPREEMPT ilog2(RESCHED_KPREEMPT)
Index: src/sys/arch/aarch64/aarch64/vectors.S
diff -u src/sys/arch/aarch64/aarch64/vectors.S:1.23 src/sys/arch/aarch64/aarch64/vectors.S:1.24
--- src/sys/arch/aarch64/aarch64/vectors.S:1.23 Mon Aug 30 23:20:00 2021
+++ src/sys/arch/aarch64/aarch64/vectors.S Sat Sep 18 12:25:06 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: vectors.S,v 1.23 2021/08/30 23:20:00 jmcneill Exp $ */
+/* $NetBSD: vectors.S,v 1.24 2021/09/18 12:25:06 jmcneill Exp $ */
#include <aarch64/asm.h>
#include <aarch64/locore.h>
@@ -9,8 +9,9 @@
#include "opt_cpuoptions.h"
#include "opt_ddb.h"
#include "opt_dtrace.h"
+#include "opt_gic.h"
-RCSID("$NetBSD: vectors.S,v 1.23 2021/08/30 23:20:00 jmcneill Exp $")
+RCSID("$NetBSD: vectors.S,v 1.24 2021/09/18 12:25:06 jmcneill Exp $")
ARMV8_DEFINE_OPTIONS
@@ -220,7 +221,23 @@ ENTRY_NP(el1_trap_exit)
unwind_x3_x30
-#if TF_PC + 8 == TF_SPSR
+#ifdef GIC_SPLFUNCS
+ mrs x0, tpidr_el1 /* get curlwp */
+ ldr x0, [x0, #L_CPU] /* get curcpu */
+
+ /*
+ * If ci_intr_depth == 0 and ci_splx_restart != 0, return
+ * to splx restart. Otherwise return to exception pc.
+ */
+ ldr w1, [x0, #CI_INTR_DEPTH]
+ cbnz w1, 1f
+ ldr x0, [x0, #CI_SPLX_RESTART]
+ cbnz x0, 2f
+1:
+ ldr x0, [sp, #TF_PC]
+2:
+ ldr x1, [sp, #TF_SPSR]
+#elif TF_PC + 8 == TF_SPSR
ldp x0, x1, [sp, #TF_PC]
#else
ldr x0, [sp, #TF_PC]
Index: src/sys/arch/aarch64/include/cpu.h
diff -u src/sys/arch/aarch64/include/cpu.h:1.38 src/sys/arch/aarch64/include/cpu.h:1.39
--- src/sys/arch/aarch64/include/cpu.h:1.38 Sat Aug 14 17:51:18 2021
+++ src/sys/arch/aarch64/include/cpu.h Sat Sep 18 12:25:06 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.h,v 1.38 2021/08/14 17:51:18 ryo Exp $ */
+/* $NetBSD: cpu.h,v 1.39 2021/09/18 12:25:06 jmcneill Exp $ */
/*-
* Copyright (c) 2014, 2020 The NetBSD Foundation, Inc.
@@ -108,6 +108,8 @@ struct cpu_info {
volatile uint32_t ci_blocked_pics;
volatile uint32_t ci_pending_pics;
volatile uint32_t ci_pending_ipls;
+ void *ci_splx_restart;
+ int ci_splx_savedipl;
int ci_kfpu_spl;
Index: src/sys/arch/arm/cortex/gic_splfuncs.c
diff -u src/sys/arch/arm/cortex/gic_splfuncs.c:1.1 src/sys/arch/arm/cortex/gic_splfuncs.c:1.2
--- src/sys/arch/arm/cortex/gic_splfuncs.c:1.1 Tue Aug 10 15:33:09 2021
+++ src/sys/arch/arm/cortex/gic_splfuncs.c Sat Sep 18 12:25:07 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: gic_splfuncs.c,v 1.1 2021/08/10 15:33:09 jmcneill Exp $ */
+/* $NetBSD: gic_splfuncs.c,v 1.2 2021/09/18 12:25:07 jmcneill Exp $ */
/*-
* Copyright (c) 2021 Jared McNeill <jmcne...@invisible.ca>
@@ -27,7 +27,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: gic_splfuncs.c,v 1.1 2021/08/10 15:33:09 jmcneill Exp $");
+__KERNEL_RCSID(0, "$NetBSD: gic_splfuncs.c,v 1.2 2021/09/18 12:25:07 jmcneill Exp $");
#include <sys/param.h>
#include <sys/atomic.h>
@@ -75,26 +75,52 @@ gic_spllower(int newipl)
static void
gic_splx(int newipl)
{
- struct cpu_info * const ci = curcpu();
+ struct cpu_info *ci = curcpu();
+ register_t psw;
if (newipl >= ci->ci_cpl) {
return;
}
- if (ci->ci_hwpl <= newipl) {
- ci->ci_cpl = newipl;
+ /*
+ * Try to avoid touching any hardware registers (DAIF, PMR) as an
+ * optimization for the common case of splraise followed by splx
+ * with no interrupts in between.
+ *
+ * If an interrupt fires in this critical section, the vector
+ * handler is responsible for returning to the address pointed
+ * to by ci_splx_restart to restart the sequence.
+ */
+ if (__predict_true(ci->ci_intr_depth == 0)) {
+ ci->ci_splx_savedipl = newipl;
+ ci->ci_splx_restart = &&restart;
+ __insn_barrier();
+checkhwpl:
if (ci->ci_hwpl <= newipl) {
- return;
+ ci->ci_cpl = newipl;
+ __insn_barrier();
+ ci->ci_splx_restart = NULL;
+ goto dosoft;
+ } else {
+ ci->ci_splx_restart = NULL;
+ goto dohard;
}
+restart:
+ ci = curcpu();
+ newipl = ci->ci_splx_savedipl;
+ goto checkhwpl;
}
- register_t psw = DISABLE_INTERRUPT_SAVE();
+dohard:
+ psw = DISABLE_INTERRUPT_SAVE();
ci->ci_intr_depth++;
pic_do_pending_ints(psw, newipl, NULL);
ci->ci_intr_depth--;
if ((psw & I32_bit) == 0) {
ENABLE_INTERRUPT();
}
+
+dosoft:
cpu_dosoftints();
}
Index: src/sys/arch/evbarm/conf/std.generic64
diff -u src/sys/arch/evbarm/conf/std.generic64:1.14 src/sys/arch/evbarm/conf/std.generic64:1.15
--- src/sys/arch/evbarm/conf/std.generic64:1.14 Sun Aug 8 12:31:42 2021
+++ src/sys/arch/evbarm/conf/std.generic64 Sat Sep 18 12:25:07 2021
@@ -1,4 +1,4 @@
-# $NetBSD: std.generic64,v 1.14 2021/08/08 12:31:42 jmcneill Exp $
+# $NetBSD: std.generic64,v 1.15 2021/09/18 12:25:07 jmcneill Exp $
#
# generic NetBSD/evbarm64 with FDT support
@@ -16,6 +16,7 @@ options DRAM_BLOCKS=256
options EVBARM_BOARDTYPE="FDT"
options FDT # Flattened Device Tree support
options FPU_VFP
+options GIC_SPLFUNCS
options MODULAR
options MODULAR_DEFAULT_AUTOLOAD
options PCI_NETBSD_CONFIGURE
