Module Name: src Committed By: ryo Date: Mon Aug 3 05:56:50 UTC 2020
Modified Files: src/sys/arch/aarch64/aarch64: copyinout.S cpuswitch.S fault.c fusu.S genassym.cf src/sys/arch/aarch64/include: frame.h machdep.h Log Message: Fix a problem in which a fault occured in an interrupt handler during copyin/copyout was erroneously detected as being occured by copyin. - keep idepth in faultbuf and compare it to avoid unnecessary fault recovery - make cpu_set_onfault() nestable to use bus_space_{peek,poke}() in hardware interrupt handlers during copyin & copyout. To generate a diff of this commit: cvs rdiff -u -r1.11 -r1.12 src/sys/arch/aarch64/aarch64/copyinout.S cvs rdiff -u -r1.22 -r1.23 src/sys/arch/aarch64/aarch64/cpuswitch.S cvs rdiff -u -r1.15 -r1.16 src/sys/arch/aarch64/aarch64/fault.c cvs rdiff -u -r1.7 -r1.8 src/sys/arch/aarch64/aarch64/fusu.S cvs rdiff -u -r1.26 -r1.27 src/sys/arch/aarch64/aarch64/genassym.cf cvs rdiff -u -r1.3 -r1.4 src/sys/arch/aarch64/include/frame.h cvs rdiff -u -r1.14 -r1.15 src/sys/arch/aarch64/include/machdep.h Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files.
Modified files: Index: src/sys/arch/aarch64/aarch64/copyinout.S diff -u src/sys/arch/aarch64/aarch64/copyinout.S:1.11 src/sys/arch/aarch64/aarch64/copyinout.S:1.12 --- src/sys/arch/aarch64/aarch64/copyinout.S:1.11 Sun Aug 2 06:58:16 2020 +++ src/sys/arch/aarch64/aarch64/copyinout.S Mon Aug 3 05:56:49 2020 @@ -1,4 +1,4 @@ -/* $NetBSD: copyinout.S,v 1.11 2020/08/02 06:58:16 maxv Exp $ */ +/* $NetBSD: copyinout.S,v 1.12 2020/08/03 05:56:49 ryo Exp $ */ /*- * Copyright (c) 2014 The NetBSD Foundation, Inc. @@ -33,7 +33,7 @@ #include <aarch64/asm.h> #include "assym.h" -RCSID("$NetBSD: copyinout.S,v 1.11 2020/08/02 06:58:16 maxv Exp $"); +RCSID("$NetBSD: copyinout.S,v 1.12 2020/08/03 05:56:49 ryo Exp $"); #ifdef ARMV81_PAN #define PAN_ENABLE \ @@ -63,7 +63,7 @@ RCSID("$NetBSD: copyinout.S,v 1.11 2020/ mov x19, x0 /* x19 = arg0 */ mov x20, x1 /* x20 = arg1 */ - /* if (cpu_set_onfault(fb) != 0) return -1 */ + /* if (cpu_set_onfault(fb) != 0) return error */ sub sp, sp, #FB_T_SIZE /* allocate struct faultbuf */ mov x0, sp /* x0 = faultbuf */ stp x2, x3, [sp, #-16]! /* save x2, x3 */ @@ -79,10 +79,8 @@ RCSID("$NetBSD: copyinout.S,v 1.11 2020/ .endm .macro exit_cpu_onfault - /* curlwp->l_md.md_onfault = NULL */ - mrs x0, tpidr_el1 /* curcpu */ - ldr x0, [x0, #CI_CURLWP] /* x0 = curlwp */ - str xzr, [x0, #L_MD_ONFAULT] /* lwp->l_md_onfault = NULL */ + bl cpu_unset_onfault + mov x8, xzr 9: PAN_ENABLE /* enable PAN */ add sp, sp, #FB_T_SIZE /* pop stack */ Index: src/sys/arch/aarch64/aarch64/cpuswitch.S diff -u src/sys/arch/aarch64/aarch64/cpuswitch.S:1.22 src/sys/arch/aarch64/aarch64/cpuswitch.S:1.23 --- src/sys/arch/aarch64/aarch64/cpuswitch.S:1.22 Thu Jul 23 13:12:54 2020 +++ src/sys/arch/aarch64/aarch64/cpuswitch.S Mon Aug 3 05:56:50 2020 @@ -1,4 +1,4 @@ -/* $NetBSD: cpuswitch.S,v 1.22 2020/07/23 13:12:54 skrll Exp $ */ +/* $NetBSD: cpuswitch.S,v 1.23 2020/08/03 05:56:50 ryo Exp $ */ /*- * Copyright (c) 2014 The NetBSD Foundation, Inc. @@ -38,7 +38,7 @@ #include "opt_ddb.h" #include "opt_kasan.h" -RCSID("$NetBSD: cpuswitch.S,v 1.22 2020/07/23 13:12:54 skrll Exp $") +RCSID("$NetBSD: cpuswitch.S,v 1.23 2020/08/03 05:56:50 ryo Exp $") ARMV8_DEFINE_OPTIONS @@ -447,22 +447,40 @@ END(cpu_Debugger) */ ENTRY_NP(cpu_set_onfault) mrs x3, tpidr_el1 - ldr x2, [x3, #CI_CURLWP] /* curlwp = curcpu()->ci_curlwp */ + ldr x2, [x3, #CI_CURLWP] /* x2 = curcpu()->ci_curlwp */ + ldr x1, [x2, #L_MD_ONFAULT] + str x1, [x0, #FB_OLD] /* fb->fb_old = curlwp->l_md.md_onfault */ + ldr w1, [x3, #CI_INTR_DEPTH] + str w1, [x0, #FB_IDEPTH] /* fb->fb_idepth = curcpu()->ci_intr_depth */ str x0, [x2, #L_MD_ONFAULT] /* l_md.md_onfault = fb */ - stp x19, x20, [x0, #(FB_X19 * 8)] - stp x21, x22, [x0, #(FB_X21 * 8)] - stp x23, x24, [x0, #(FB_X23 * 8)] - stp x25, x26, [x0, #(FB_X25 * 8)] - stp x27, x28, [x0, #(FB_X27 * 8)] - stp x29, x30, [x0, #(FB_X29 * 8)] + stp x19, x20, [x0, #FB_REG_X19] + stp x21, x22, [x0, #FB_REG_X21] + stp x23, x24, [x0, #FB_REG_X23] + stp x25, x26, [x0, #FB_REG_X25] + stp x27, x28, [x0, #FB_REG_X27] + stp x29, x30, [x0, #FB_REG_X29] mov x1, sp - str x1, [x0, #(FB_SP * 8)] + str x1, [x0, #FB_REG_SP] mov x0, #0 ret END(cpu_set_onfault) /* + * void cpu_unset_onfault(void) + */ +ENTRY_NP(cpu_unset_onfault) + mrs x3, tpidr_el1 + ldr x2, [x3, #CI_CURLWP] /* curlwp = curcpu()->ci_curlwp */ + ldr x0, [x2, #L_MD_ONFAULT] /* x0 = curlwp->l_md.md_onfault */ + cbz x0, 1f + ldr x0, [x0, #FB_OLD] /* restore old faultbuf */ + str x0, [x2, #L_MD_ONFAULT] +1: + ret +END(cpu_unset_onfault) + +/* * setjmp(9) * int setjmp(label_t *label); * void longjmp(label_t *label); Index: src/sys/arch/aarch64/aarch64/fault.c diff -u src/sys/arch/aarch64/aarch64/fault.c:1.15 src/sys/arch/aarch64/aarch64/fault.c:1.16 --- src/sys/arch/aarch64/aarch64/fault.c:1.15 Sun Aug 2 06:58:16 2020 +++ src/sys/arch/aarch64/aarch64/fault.c Mon Aug 3 05:56:50 2020 @@ -1,4 +1,4 @@ -/* $NetBSD: fault.c,v 1.15 2020/08/02 06:58:16 maxv Exp $ */ +/* $NetBSD: fault.c,v 1.16 2020/08/03 05:56:50 ryo Exp $ */ /* * Copyright (c) 2017 Ryo Shimizu <r...@nerv.org> @@ -27,7 +27,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: fault.c,v 1.15 2020/08/02 06:58:16 maxv Exp $"); +__KERNEL_RCSID(0, "$NetBSD: fault.c,v 1.16 2020/08/03 05:56:50 ryo Exp $"); #include "opt_compat_netbsd32.h" #include "opt_ddb.h" @@ -220,11 +220,18 @@ data_abort_handler(struct trapframe *tf, return; } - do_fault: /* faultbail path? */ - fb = cpu_disable_onfault(); - if (fb != NULL) { + fb = l->l_md.md_onfault; + if (fb != NULL && fb->fb_idepth == curcpu()->ci_intr_depth) { + cpu_unset_onfault(); +#ifdef DEBUG_DUMP_ON_FAULTBAIL + printf("fault in failtbail[%p]: " + "fb_sp=%016lx, fb_lr=%016lx, fb_idepth=%u, fb_old=%p\n", + fb, fb->fb_reg[FB_SP], fb->fb_reg[FB_LR], + fb->fb_idepth, fb->fb_old); + dump_trapframe(tf, printf); +#endif cpu_jump_onfault(tf, fb, EFAULT); return; } Index: src/sys/arch/aarch64/aarch64/fusu.S diff -u src/sys/arch/aarch64/aarch64/fusu.S:1.7 src/sys/arch/aarch64/aarch64/fusu.S:1.8 --- src/sys/arch/aarch64/aarch64/fusu.S:1.7 Sun Aug 2 06:58:16 2020 +++ src/sys/arch/aarch64/aarch64/fusu.S Mon Aug 3 05:56:50 2020 @@ -1,4 +1,4 @@ -/* $NetBSD: fusu.S,v 1.7 2020/08/02 06:58:16 maxv Exp $ */ +/* $NetBSD: fusu.S,v 1.8 2020/08/03 05:56:50 ryo Exp $ */ /*- * Copyright (c) 2014, 2019 The NetBSD Foundation, Inc. @@ -32,7 +32,7 @@ #include <aarch64/asm.h> #include "assym.h" -RCSID("$NetBSD: fusu.S,v 1.7 2020/08/02 06:58:16 maxv Exp $"); +RCSID("$NetBSD: fusu.S,v 1.8 2020/08/03 05:56:50 ryo Exp $"); #ifdef ARMV81_PAN #define PAN_ENABLE \ @@ -72,10 +72,8 @@ RCSID("$NetBSD: fusu.S,v 1.7 2020/08/02 .endm .macro exit_cpu_onfault - /* curlwp->l_md.md_onfault = NULL */ - mrs x1, tpidr_el1 /* curcpu */ - ldr x1, [x1, #CI_CURLWP] /* x1 = curlwp */ - str xzr, [x1, #L_MD_ONFAULT] /* lwp->l_md_onfault = NULL */ + bl cpu_unset_onfault + mov x0, xzr 9: PAN_ENABLE /* enable PAN */ add sp, sp, #FB_T_SIZE /* pop stack */ Index: src/sys/arch/aarch64/aarch64/genassym.cf diff -u src/sys/arch/aarch64/aarch64/genassym.cf:1.26 src/sys/arch/aarch64/aarch64/genassym.cf:1.27 --- src/sys/arch/aarch64/aarch64/genassym.cf:1.26 Thu May 28 04:51:44 2020 +++ src/sys/arch/aarch64/aarch64/genassym.cf Mon Aug 3 05:56:50 2020 @@ -1,4 +1,4 @@ -# $NetBSD: genassym.cf,v 1.26 2020/05/28 04:51:44 ryo Exp $ +# $NetBSD: genassym.cf,v 1.27 2020/08/03 05:56:50 ryo Exp $ #- # Copyright (c) 2014 The NetBSD Foundation, Inc. # All rights reserved. @@ -159,20 +159,21 @@ define L_MD_DB_USER offsetof(struct lwp define L_MD_GA_USER offsetof(struct lwp, l_md.md_ga_user) define LW_SYSTEM LW_SYSTEM -define FB_X19 FB_X19 -define FB_X20 FB_X20 -define FB_X21 FB_X21 -define FB_X22 FB_X22 -define FB_X23 FB_X23 -define FB_X24 FB_X24 -define FB_X25 FB_X25 -define FB_X26 FB_X26 -define FB_X27 FB_X27 -define FB_X28 FB_X28 -define FB_X29 FB_X29 -define FB_LR FB_LR -define FB_SP FB_SP -define FB_MAX FB_MAX +define FB_REG_X19 offsetof(struct faultbuf, fb_reg[FB_X19]) +define FB_REG_X20 offsetof(struct faultbuf, fb_reg[FB_X20]) +define FB_REG_X21 offsetof(struct faultbuf, fb_reg[FB_X21]) +define FB_REG_X22 offsetof(struct faultbuf, fb_reg[FB_X22]) +define FB_REG_X23 offsetof(struct faultbuf, fb_reg[FB_X23]) +define FB_REG_X24 offsetof(struct faultbuf, fb_reg[FB_X24]) +define FB_REG_X25 offsetof(struct faultbuf, fb_reg[FB_X25]) +define FB_REG_X26 offsetof(struct faultbuf, fb_reg[FB_X26]) +define FB_REG_X27 offsetof(struct faultbuf, fb_reg[FB_X27]) +define FB_REG_X28 offsetof(struct faultbuf, fb_reg[FB_X28]) +define FB_REG_X29 offsetof(struct faultbuf, fb_reg[FB_X29]) +define FB_REG_LR offsetof(struct faultbuf, fb_reg[FB_LR]) +define FB_REG_SP offsetof(struct faultbuf, fb_reg[FB_SP]) +define FB_OLD offsetof(struct faultbuf, fb_old) +define FB_IDEPTH offsetof(struct faultbuf, fb_idepth) define FB_T_SIZE roundup(sizeof(struct faultbuf), 16) define LBL_X19 LBL_X19 Index: src/sys/arch/aarch64/include/frame.h diff -u src/sys/arch/aarch64/include/frame.h:1.3 src/sys/arch/aarch64/include/frame.h:1.4 --- src/sys/arch/aarch64/include/frame.h:1.3 Tue Dec 3 22:03:14 2019 +++ src/sys/arch/aarch64/include/frame.h Mon Aug 3 05:56:50 2020 @@ -1,4 +1,4 @@ -/* $NetBSD: frame.h,v 1.3 2019/12/03 22:03:14 jmcneill Exp $ */ +/* $NetBSD: frame.h,v 1.4 2020/08/03 05:56:50 ryo Exp $ */ /*- * Copyright (c) 2014 The NetBSD Foundation, Inc. @@ -70,6 +70,8 @@ __CTASSERT((sizeof(struct trapframe) & 1 #define FB_MAX 13 struct faultbuf { register_t fb_reg[FB_MAX]; + struct faultbuf *fb_old; + u_int fb_idepth; }; #define lwp_trapframe(l) ((l)->l_md.md_utf) Index: src/sys/arch/aarch64/include/machdep.h diff -u src/sys/arch/aarch64/include/machdep.h:1.14 src/sys/arch/aarch64/include/machdep.h:1.15 --- src/sys/arch/aarch64/include/machdep.h:1.14 Wed Jul 8 03:45:13 2020 +++ src/sys/arch/aarch64/include/machdep.h Mon Aug 3 05:56:50 2020 @@ -1,4 +1,4 @@ -/* $NetBSD: machdep.h,v 1.14 2020/07/08 03:45:13 ryo Exp $ */ +/* $NetBSD: machdep.h,v 1.15 2020/08/03 05:56:50 ryo Exp $ */ /* * Copyright (c) 2017 Ryo Shimizu <r...@nerv.org> @@ -113,16 +113,11 @@ void interrupt(struct trapframe *); /* cpu_onfault */ int cpu_set_onfault(struct faultbuf *) __returns_twice; +void cpu_unset_onfault(void); void cpu_jump_onfault(struct trapframe *, const struct faultbuf *, int); #if defined(_KERNEL) static inline void -cpu_unset_onfault(void) -{ - curlwp->l_md.md_onfault = NULL; -} - -static inline void cpu_enable_onfault(struct faultbuf *fb) { curlwp->l_md.md_onfault = fb;