Module Name: src
Committed By: rmind
Date: Sat Mar 28 22:56:20 UTC 2009
Modified Files:
src/sys/arch/amd64/amd64: copy.S trap.c
Log Message:
Change amd64 fault handler to check instruction pointer for copyin/out et al
functions, like in i386. Avoids setting pcb_onfault, saves few instructions.
To generate a diff of this commit:
cvs rdiff -u -r1.13 -r1.14 src/sys/arch/amd64/amd64/copy.S
cvs rdiff -u -r1.54 -r1.55 src/sys/arch/amd64/amd64/trap.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/amd64/amd64/copy.S
diff -u src/sys/arch/amd64/amd64/copy.S:1.13 src/sys/arch/amd64/amd64/copy.S:1.14
--- src/sys/arch/amd64/amd64/copy.S:1.13 Mon Feb 23 20:27:59 2009
+++ src/sys/arch/amd64/amd64/copy.S Sat Mar 28 22:56:19 2009
@@ -1,4 +1,4 @@
-/* $NetBSD: copy.S,v 1.13 2009/02/23 20:27:59 rmind Exp $ */
+/* $NetBSD: copy.S,v 1.14 2009/03/28 22:56:19 rmind Exp $ */
/*
* Copyright (c) 2001 Wasabi Systems, Inc.
@@ -62,6 +62,10 @@
call _C_LABEL(do_pmap_load) ; \
jmp 98b
+/*
+ * The following primitives are to copy regions of memory.
+ * Label must be before all copy functions.
+ */
.text
x86_copyfunc_start: .globl x86_copyfunc_start
@@ -103,6 +107,9 @@
ret
/*
+ * int kcopy(const void *from, void *to, size_t len);
+ * Copy len bytes, abort on fault.
+ *
* Copy routines from and to userland, plus a few more. See the
* section 9 manpages for info. Some cases can be optimized more.
*
@@ -119,14 +126,9 @@
*/
ENTRY(kcopy)
- GET_CURPCB(%rax)
- pushq PCB_ONFAULT(%rax)
- leaq _C_LABEL(kcopy_fault)(%rip),%r11
- movq %r11, PCB_ONFAULT(%rax)
-
xchgq %rdi,%rsi
movq %rdx,%rcx
-
+.Lkcopy_start:
movq %rdi,%rax
subq %rsi,%rax
cmpq %rcx,%rax # overlapping?
@@ -141,8 +143,6 @@
rep
movsb
- GET_CURPCB(%rdx)
- popq PCB_ONFAULT(%rdx)
xorq %rax,%rax
ret
@@ -166,29 +166,23 @@
rep
movsq
cld
-
- GET_CURPCB(%rdx)
- popq PCB_ONFAULT(%rdx)
+.Lkcopy_end:
xorq %rax,%rax
ret
ENTRY(copyout)
DEFERRED_SWITCH_CHECK
- pushq $0 # Restored to PCB_ONFAULT(%rdx)
xchgq %rdi,%rsi # kernel address to %rsi, user to %rdi
movq %rdx,%rax # save transfer length (bytes)
-
+
addq %rdi,%rdx # end address to %rdx
jc _C_LABEL(copy_efault) # jump if wraps
movq $VM_MAXUSER_ADDRESS,%r8
cmpq %r8,%rdx
ja _C_LABEL(copy_efault) # jump if end in kernel space
- GET_CURPCB(%rdx)
- leaq _C_LABEL(copy_fault)(%rip),%r11
- movq %r11,PCB_ONFAULT(%rdx) # prime fault handler
-
+.Lcopyout_start:
movq %rax,%rcx # length
shrq $3,%rcx # count of 8-byte words
rep
@@ -197,18 +191,13 @@
andb $7,%cl # remaining number of bytes
rep
movsb # copy remaining bytes
-
- popq PCB_ONFAULT(%rdx)
+.Lcopyout_end:
xorl %eax,%eax
ret
DEFERRED_SWITCH_CALL
ENTRY(copyin)
DEFERRED_SWITCH_CHECK
- GET_CURPCB(%rax)
- pushq $0
- leaq _C_LABEL(copy_fault)(%rip),%r11
- movq %r11,PCB_ONFAULT(%rax)
xchgq %rdi,%rsi
movq %rdx,%rax
@@ -219,6 +208,7 @@
cmpq %r8,%rdx
ja _C_LABEL(copy_efault) # j if end in kernel space
+.Lcopyin_start:
3: /* bcopy(%rsi, %rdi, %rax); */
movq %rax,%rcx
shrq $3,%rcx
@@ -228,9 +218,7 @@
andb $7,%cl
rep
movsb
-
- GET_CURPCB(%rdx)
- popq PCB_ONFAULT(%rdx)
+.Lcopyin_end:
xorl %eax,%eax
ret
DEFERRED_SWITCH_CALL
@@ -245,13 +233,9 @@
*/
NENTRY(kcopy_fault)
- GET_CURPCB(%rdx)
- popq PCB_ONFAULT(%rdx)
ret
NENTRY(copy_fault)
- GET_CURPCB(%rdx)
- popq PCB_ONFAULT(%rdx)
ret
ENTRY(copyoutstr)
@@ -260,9 +244,6 @@
movq %rdx,%r8
movq %rcx,%r9
-5: GET_CURPCB(%rax)
- leaq _C_LABEL(copystr_fault)(%rip),%r11
- movq %r11,PCB_ONFAULT(%rax)
/*
* Get min(%rdx, VM_MAXUSER_ADDRESS-%rdi).
*/
@@ -273,7 +254,7 @@
jae 1f
movq %rax,%rdx
movq %rax,%r8
-
+.Lcopyoutstr_start:
1: incq %rdx
1: decq %rdx
@@ -282,7 +263,7 @@
stosb
testb %al,%al
jnz 1b
-
+.Lcopyoutstr_end:
/* Success -- 0 byte reached. */
decq %rdx
xorq %rax,%rax
@@ -302,10 +283,6 @@
movq %rdx,%r8
movq %rcx,%r9
- GET_CURPCB(%rcx)
- leaq _C_LABEL(copystr_fault)(%rip),%r11
- movq %r11,PCB_ONFAULT(%rcx)
-
/*
* Get min(%rdx, VM_MAXUSER_ADDRESS-%rsi).
*/
@@ -316,7 +293,7 @@
jae 1f
movq %rax,%rdx
movq %rax,%r8
-
+.Lcopyinstr_start:
1: incq %rdx
1: decq %rdx
@@ -325,6 +302,7 @@
stosb
testb %al,%al
jnz 1b
+.Lcopyinstr_end:
/* Success -- 0 byte reached. */
decq %rdx
@@ -345,8 +323,6 @@
ENTRY(copystr_fault)
copystr_return:
/* Set *lencopied and return %eax. */
- GET_CURPCB(%rcx)
- movq $0,PCB_ONFAULT(%rcx)
testq %r9,%r9
jz 8f
subq %rdx,%r8
@@ -526,15 +502,14 @@
movq $VM_MAXUSER_ADDRESS-8, %r8
cmpq %r8, %rdi
ja 1f
- /* Set the fault handler */
- GET_CURPCB(%r8)
- leaq _C_LABEL(ucas_fault)(%rip), %r9
- movq %r9, PCB_ONFAULT(%r8)
+ /* Label for fault handler */
+.Lucas64_start:
/* Perform the CAS */
lock
cmpxchgq %rdx, (%rdi)
/* Clear the fault handler */
movq %rax, PCB_ONFAULT(%r8)
+.Lucas64_end:
/*
* Note: %rax is "old" value.
* Set the return values.
@@ -558,13 +533,12 @@
movq $VM_MAXUSER_ADDRESS-4, %r8
cmpq %r8, %rdi
ja 1f
- /* Set the fault handler */
- GET_CURPCB(%r8)
- leaq _C_LABEL(ucas_fault)(%rip), %r9
- movq %r9, PCB_ONFAULT(%r8)
+ /* Label for fault handler */
+.Lucas32_start:
/* Perform the CAS */
lock
cmpxchgl %edx, (%rdi)
+.Lucas32_end:
/*
* Note: %eax is "old" value.
* Set the return values.
@@ -582,7 +556,6 @@
* Unset the handler and return the failure.
*/
NENTRY(ucas_fault)
- movq $0, PCB_ONFAULT(%r8)
movq $EFAULT, %rax
ret
@@ -592,4 +565,45 @@
STRONG_ALIAS(ucas_ptr, ucas_64)
STRONG_ALIAS(ucas_int, ucas_32)
+/*
+ * Label must be after all copy functions.
+ */
x86_copyfunc_end: .globl x86_copyfunc_end
+
+/*
+ * Fault table of copy functions for trap().
+ */
+ .section ".rodata"
+ .globl _C_LABEL(onfault_table)
+_C_LABEL(onfault_table):
+ .quad .Lcopyin_start
+ .quad .Lcopyin_end
+ .quad _C_LABEL(copy_fault)
+
+ .quad .Lcopyout_start
+ .quad .Lcopyout_end
+ .quad _C_LABEL(copy_fault)
+
+ .quad .Lkcopy_start
+ .quad .Lkcopy_end
+ .quad _C_LABEL(kcopy_fault)
+
+ .quad .Lcopyoutstr_start
+ .quad .Lcopyoutstr_end
+ .quad _C_LABEL(copystr_fault)
+
+ .quad .Lcopyinstr_start
+ .quad .Lcopyinstr_end
+ .quad _C_LABEL(copystr_fault)
+
+ .quad .Lucas64_start
+ .quad .Lucas64_end
+ .quad _C_LABEL(ucas_fault)
+
+ .quad .Lucas32_start
+ .quad .Lucas32_end
+ .quad _C_LABEL(ucas_fault)
+
+ .quad 0 /* terminate */
+
+ .text
Index: src/sys/arch/amd64/amd64/trap.c
diff -u src/sys/arch/amd64/amd64/trap.c:1.54 src/sys/arch/amd64/amd64/trap.c:1.55
--- src/sys/arch/amd64/amd64/trap.c:1.54 Tue Feb 24 06:03:54 2009
+++ src/sys/arch/amd64/amd64/trap.c Sat Mar 28 22:56:19 2009
@@ -1,4 +1,4 @@
-/* $NetBSD: trap.c,v 1.54 2009/02/24 06:03:54 yamt Exp $ */
+/* $NetBSD: trap.c,v 1.55 2009/03/28 22:56:19 rmind Exp $ */
/*-
* Copyright (c) 1998, 2000 The NetBSD Foundation, Inc.
@@ -68,7 +68,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.54 2009/02/24 06:03:54 yamt Exp $");
+__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.55 2009/03/28 22:56:19 rmind Exp $");
#include "opt_ddb.h"
#include "opt_kgdb.h"
@@ -151,6 +151,31 @@
static void frame_dump(struct trapframe *);
#endif
+static void *
+onfault_handler(const struct pcb *pcb, const struct trapframe *tf)
+{
+ struct onfault_table {
+ uintptr_t start;
+ uintptr_t end;
+ void *handler;
+ };
+ extern const struct onfault_table onfault_table[];
+ const struct onfault_table *p;
+ uintptr_t pc;
+
+ if (pcb->pcb_onfault != NULL) {
+ return pcb->pcb_onfault;
+ }
+
+ pc = tf->tf_rip;
+ for (p = onfault_table; p->start; p++) {
+ if (p->start <= pc && pc < p->end) {
+ return p->handler;
+ }
+ }
+ return NULL;
+}
+
/*
* trap(frame):
* Exception, fault, and trap interface to BSD kernel. This
@@ -250,11 +275,12 @@
if (p == NULL)
goto we_re_toast;
/* Check for copyin/copyout fault. */
- if (pcb->pcb_onfault != 0) {
+ onfault = onfault_handler(pcb, frame);
+ if (onfault != NULL) {
copyefault:
error = EFAULT;
copyfault:
- frame->tf_rip = (uint64_t)pcb->pcb_onfault;
+ frame->tf_rip = (uintptr_t)onfault;
frame->tf_rax = error;
return;
}
@@ -412,7 +438,8 @@
* fusuintrfailure is used by [fs]uswintr() to prevent
* page faulting from inside the profiling interrupt.
*/
- if (pcb->pcb_onfault == fusuintrfailure) {
+ onfault = pcb->pcb_onfault;
+ if (onfault == fusuintrfailure) {
goto copyefault;
}
if (cpu_intr_p() || (l->l_pflag & LP_INTR) != 0) {
@@ -489,6 +516,7 @@
*/
kpreempt_disable();
if (curcpu()->ci_want_pmapload) {
+ onfault = onfault_handler(pcb, frame);
if (onfault != kcopy_fault) {
pmap_load();
}
@@ -537,7 +565,8 @@
ksi.ksi_code = SEGV_MAPERR;
if (type == T_PAGEFLT) {
- if (pcb->pcb_onfault != 0)
+ onfault = onfault_handler(pcb, frame);
+ if (onfault != NULL)
goto copyfault;
printf("uvm_fault(%p, 0x%lx, %d) -> %x\n",
map, va, ftype, error);
