Author: mjg Date: Sat Oct 13 21:18:31 2018 New Revision: 339349 URL: https://svnweb.freebsd.org/changeset/base/339349
Log: amd64: partially depessimize cpu_fetch_syscall_args and cpu_set_syscall_retval Vast majority of syscalls take 6 or less arguments. Move handling of other cases to a fallback function. Similarly, special casing for _syscall and __syscall magic syscalls is moved away. Return is almost always 0. The change replaces 3 branches with 1 in the common case. Also the 'frame' variable convinces clang not to reload it on each access. Reviewed by: kib Approved by: re (gjb) Sponsored by: The FreeBSD Foundation Differential Revision: https://reviews.freebsd.org/D17542 Modified: head/sys/amd64/amd64/trap.c head/sys/amd64/amd64/vm_machdep.c head/sys/amd64/include/proc.h Modified: head/sys/amd64/amd64/trap.c ============================================================================== --- head/sys/amd64/amd64/trap.c Sat Oct 13 21:17:28 2018 (r339348) +++ head/sys/amd64/amd64/trap.c Sat Oct 13 21:18:31 2018 (r339349) @@ -970,21 +970,19 @@ dblfault_handler(struct trapframe *frame) panic("double fault"); } -int -cpu_fetch_syscall_args(struct thread *td) +static int __noinline +cpu_fetch_syscall_args_fallback(struct thread *td, struct syscall_args *sa) { struct proc *p; struct trapframe *frame; register_t *argp; - struct syscall_args *sa; caddr_t params; int reg, regcnt, error; p = td->td_proc; frame = td->td_frame; - sa = &td->td_sa; reg = 0; - regcnt = 6; + regcnt = NARGREGS; sa->code = frame->tf_rax; @@ -1002,24 +1000,58 @@ cpu_fetch_syscall_args(struct thread *td) sa->callp = &p->p_sysent->sv_table[sa->code]; sa->narg = sa->callp->sy_narg; - KASSERT(sa->narg <= sizeof(sa->args) / sizeof(sa->args[0]), - ("Too many syscall arguments!")); - error = 0; + KASSERT(sa->narg <= nitems(sa->args), ("Too many syscall arguments!")); argp = &frame->tf_rdi; argp += reg; - memcpy(sa->args, argp, sizeof(sa->args[0]) * 6); + memcpy(sa->args, argp, sizeof(sa->args[0]) * NARGREGS); if (sa->narg > regcnt) { params = (caddr_t)frame->tf_rsp + sizeof(register_t); error = copyin(params, &sa->args[regcnt], (sa->narg - regcnt) * sizeof(sa->args[0])); + if (__predict_false(error != 0)) + return (error); } - if (error == 0) { - td->td_retval[0] = 0; - td->td_retval[1] = frame->tf_rdx; - } + td->td_retval[0] = 0; + td->td_retval[1] = frame->tf_rdx; - return (error); + return (0); +} + +int +cpu_fetch_syscall_args(struct thread *td) +{ + struct proc *p; + struct trapframe *frame; + struct syscall_args *sa; + + p = td->td_proc; + frame = td->td_frame; + sa = &td->td_sa; + + sa->code = frame->tf_rax; + + if (__predict_false(sa->code == SYS_syscall || + sa->code == SYS___syscall || + sa->code >= p->p_sysent->sv_size)) + return (cpu_fetch_syscall_args_fallback(td, sa)); + + sa->callp = &p->p_sysent->sv_table[sa->code]; + sa->narg = sa->callp->sy_narg; + KASSERT(sa->narg <= nitems(sa->args), ("Too many syscall arguments!")); + + if (p->p_sysent->sv_mask) + sa->code &= p->p_sysent->sv_mask; + + if (__predict_false(sa->narg > NARGREGS)) + return (cpu_fetch_syscall_args_fallback(td, sa)); + + memcpy(sa->args, &frame->tf_rdi, sizeof(sa->args[0]) * NARGREGS); + + td->td_retval[0] = 0; + td->td_retval[1] = frame->tf_rdx; + + return (0); } #include "../../kern/subr_syscall.c" Modified: head/sys/amd64/amd64/vm_machdep.c ============================================================================== --- head/sys/amd64/amd64/vm_machdep.c Sat Oct 13 21:17:28 2018 (r339348) +++ head/sys/amd64/amd64/vm_machdep.c Sat Oct 13 21:18:31 2018 (r339349) @@ -372,14 +372,17 @@ cpu_thread_free(struct thread *td) void cpu_set_syscall_retval(struct thread *td, int error) { + struct trapframe *frame; - switch (error) { - case 0: - td->td_frame->tf_rax = td->td_retval[0]; - td->td_frame->tf_rdx = td->td_retval[1]; - td->td_frame->tf_rflags &= ~PSL_C; - break; + frame = td->td_frame; + if (__predict_true(error == 0)) { + frame->tf_rax = td->td_retval[0]; + frame->tf_rdx = td->td_retval[1]; + frame->tf_rflags &= ~PSL_C; + return; + } + switch (error) { case ERESTART: /* * Reconstruct pc, we know that 'syscall' is 2 bytes, @@ -393,8 +396,8 @@ cpu_set_syscall_retval(struct thread *td, int error) * Require full context restore to get the arguments * in the registers reloaded at return to usermode. */ - td->td_frame->tf_rip -= td->td_frame->tf_err; - td->td_frame->tf_r10 = td->td_frame->tf_rcx; + frame->tf_rip -= frame->tf_err; + frame->tf_r10 = frame->tf_rcx; set_pcb_flags(td->td_pcb, PCB_FULL_IRET); break; @@ -402,8 +405,8 @@ cpu_set_syscall_retval(struct thread *td, int error) break; default: - td->td_frame->tf_rax = SV_ABI_ERRNO(td->td_proc, error); - td->td_frame->tf_rflags |= PSL_C; + frame->tf_rax = SV_ABI_ERRNO(td->td_proc, error); + frame->tf_rflags |= PSL_C; break; } } Modified: head/sys/amd64/include/proc.h ============================================================================== --- head/sys/amd64/include/proc.h Sat Oct 13 21:17:28 2018 (r339348) +++ head/sys/amd64/include/proc.h Sat Oct 13 21:18:31 2018 (r339349) @@ -101,6 +101,9 @@ int amd64_set_ldt_data(struct thread *td, int start, i extern struct mtx dt_lock; extern int max_ldt_segment; + +#define NARGREGS 6 + #endif /* _KERNEL */ #endif /* !_MACHINE_PROC_H_ */ _______________________________________________ svn-src-head@freebsd.org mailing list https://lists.freebsd.org/mailman/listinfo/svn-src-head To unsubscribe, send any mail to "svn-src-head-unsubscr...@freebsd.org"
