Author: kib Date: Mon Dec 7 12:20:26 2015 New Revision: 291937 URL: https://svnweb.freebsd.org/changeset/base/291937
Log: Add support for usermode (vdso-like) gettimeofday(2) and clock_gettime(2) on ARMv7 and ARMv8 systems which have architectural generic timer hardware. It is similar how the RDTSC timer is used in userspace on x86. Fix a permission problem where generic timer access from EL0 (or userspace on v7) was not properly initialized on APs. For ARMv7, mark the stack non-executable. The shared page is added for all arms (including ARMv8 64bit), and the signal trampoline code is moved to the page. Reviewed by: andrew Discussed with: emaste, mmel Sponsored by: The FreeBSD Foundation Differential revision: https://reviews.freebsd.org/D4209 Added: head/lib/libc/aarch64/sys/__vdso_gettc.c - copied, changed from r291936, head/lib/libc/amd64/sys/__vdso_gettc.c head/lib/libc/arm/sys/__vdso_gettc.c - copied, changed from r291936, head/lib/libc/amd64/sys/__vdso_gettc.c Modified: head/lib/libc/aarch64/sys/Makefile.inc head/lib/libc/arm/sys/Makefile.inc head/sys/arm/arm/elf_machdep.c head/sys/arm/arm/generic_timer.c head/sys/arm/arm/machdep.c head/sys/arm/include/md_var.h head/sys/arm/include/vdso.h head/sys/arm/include/vmparam.h head/sys/arm64/arm64/elf_machdep.c head/sys/arm64/arm64/machdep.c head/sys/arm64/include/md_var.h head/sys/arm64/include/vdso.h head/sys/arm64/include/vmparam.h head/sys/conf/files.arm head/sys/conf/files.arm64 head/sys/kern/imgact_elf.c Modified: head/lib/libc/aarch64/sys/Makefile.inc ============================================================================== --- head/lib/libc/aarch64/sys/Makefile.inc Mon Dec 7 12:09:04 2015 (r291936) +++ head/lib/libc/aarch64/sys/Makefile.inc Mon Dec 7 12:20:26 2015 (r291937) @@ -1,6 +1,6 @@ # $FreeBSD$ -SRCS+= trivial-vdso_tc.c +SRCS+= __vdso_gettc.c #MDASM= ptrace.S MDASM= brk.S \ Copied and modified: head/lib/libc/aarch64/sys/__vdso_gettc.c (from r291936, head/lib/libc/amd64/sys/__vdso_gettc.c) ============================================================================== --- head/lib/libc/amd64/sys/__vdso_gettc.c Mon Dec 7 12:09:04 2015 (r291936, copy source) +++ head/lib/libc/aarch64/sys/__vdso_gettc.c Mon Dec 7 12:20:26 2015 (r291937) @@ -1,5 +1,8 @@ /*- - * Copyright (c) 2012 Konstantin Belousov <k...@freebsd.org> + * Copyright (c) 2015 The FreeBSD Foundation + * + * This software was developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -33,32 +36,33 @@ __FBSDID("$FreeBSD$"); #include <machine/cpufunc.h> #include "libc_private.h" -static u_int -__vdso_gettc_low(const struct vdso_timehands *th) +static inline uint64_t +cp15_cntvct_get(void) { - u_int rv; + uint64_t reg; - __asm __volatile("lfence; rdtsc; shrd %%cl, %%edx, %0" - : "=a" (rv) : "c" (th->th_x86_shift) : "edx"); - return (rv); + __asm __volatile("mrs %0, cntvct_el0" : "=r" (reg)); + return (reg); } -static u_int -__vdso_rdtsc32(void) +static inline uint64_t +cp15_cntpct_get(void) { - u_int rv; + uint64_t reg; - __asm __volatile("lfence;rdtsc" : "=a" (rv) : : "edx"); - return (rv); + __asm __volatile("mrs %0, cntpct_el0" : "=r" (reg)); + return (reg); } #pragma weak __vdso_gettc u_int __vdso_gettc(const struct vdso_timehands *th) { + uint64_t val; - return (th->th_x86_shift > 0 ? __vdso_gettc_low(th) : - __vdso_rdtsc32()); + __asm __volatile("isb" : : : "memory"); + val = th->th_physical == 0 ? cp15_cntvct_get() : cp15_cntpct_get(); + return (val); } #pragma weak __vdso_gettimekeep Modified: head/lib/libc/arm/sys/Makefile.inc ============================================================================== --- head/lib/libc/arm/sys/Makefile.inc Mon Dec 7 12:09:04 2015 (r291936) +++ head/lib/libc/arm/sys/Makefile.inc Mon Dec 7 12:20:26 2015 (r291937) @@ -1,6 +1,6 @@ # $FreeBSD$ -SRCS+= trivial-vdso_tc.c +SRCS+= __vdso_gettc.c MDASM= Ovfork.S brk.S cerror.S pipe.S ptrace.S sbrk.S shmat.S sigreturn.S syscall.S Copied and modified: head/lib/libc/arm/sys/__vdso_gettc.c (from r291936, head/lib/libc/amd64/sys/__vdso_gettc.c) ============================================================================== --- head/lib/libc/amd64/sys/__vdso_gettc.c Mon Dec 7 12:09:04 2015 (r291936, copy source) +++ head/lib/libc/arm/sys/__vdso_gettc.c Mon Dec 7 12:20:26 2015 (r291937) @@ -1,5 +1,8 @@ /*- - * Copyright (c) 2012 Konstantin Belousov <k...@freebsd.org> + * Copyright (c) 2015 The FreeBSD Foundation + * + * This software was developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -33,32 +36,38 @@ __FBSDID("$FreeBSD$"); #include <machine/cpufunc.h> #include "libc_private.h" -static u_int -__vdso_gettc_low(const struct vdso_timehands *th) +static inline uint64_t +cp15_cntvct_get(void) { - u_int rv; + uint64_t reg; - __asm __volatile("lfence; rdtsc; shrd %%cl, %%edx, %0" - : "=a" (rv) : "c" (th->th_x86_shift) : "edx"); - return (rv); + __asm __volatile("mrrc\tp15, 1, %Q0, %R0, c14" : "=r" (reg)); + return (reg); } -static u_int -__vdso_rdtsc32(void) +static inline uint64_t +cp15_cntpct_get(void) { - u_int rv; + uint64_t reg; - __asm __volatile("lfence;rdtsc" : "=a" (rv) : : "edx"); - return (rv); + __asm __volatile("mrrc\tp15, 0, %Q0, %R0, c14" : "=r" (reg)); + return (reg); } #pragma weak __vdso_gettc u_int __vdso_gettc(const struct vdso_timehands *th) { + uint64_t val; - return (th->th_x86_shift > 0 ? __vdso_gettc_low(th) : - __vdso_rdtsc32()); + /* + * Userspace gettimeofday() is only enabled on ARMv7 CPUs, but + * libc is compiled for ARMv6. Due to clang issues, .arch + * armv7-a directive does not work. + */ + __asm __volatile(".word\t0xf57ff06f" : : : "memory"); /* isb */ + val = th->th_physical == 0 ? cp15_cntvct_get() : cp15_cntpct_get(); + return (val); } #pragma weak __vdso_gettimekeep Modified: head/sys/arm/arm/elf_machdep.c ============================================================================== --- head/sys/arm/arm/elf_machdep.c Mon Dec 7 12:09:04 2015 (r291936) +++ head/sys/arm/arm/elf_machdep.c Mon Dec 7 12:20:26 2015 (r291937) @@ -43,6 +43,7 @@ __FBSDID("$FreeBSD$"); #include <vm/pmap.h> #include <vm/vm_param.h> +#include <machine/acle-compat.h> #include <machine/elf.h> #include <machine/md_var.h> @@ -73,13 +74,20 @@ struct sysentvec elf32_freebsd_sysvec = .sv_setregs = exec_setregs, .sv_fixlimit = NULL, .sv_maxssiz = NULL, - .sv_flags = SV_ABI_FREEBSD | SV_ILP32, + .sv_flags = +#if __ARM_ARCH >= 6 + SV_SHP | SV_TIMEKEEP | +#endif + SV_ABI_FREEBSD | SV_ILP32, .sv_set_syscall_retval = cpu_set_syscall_retval, .sv_fetch_syscall_args = cpu_fetch_syscall_args, .sv_syscallnames = syscallnames, + .sv_shared_page_base = SHAREDPAGE, + .sv_shared_page_len = PAGE_SIZE, .sv_schedtail = NULL, .sv_thread_detach = NULL, }; +INIT_SYSENTVEC(elf32_sysvec, &elf32_freebsd_sysvec); static Elf32_Brandinfo freebsd_brand_info = { .brand = ELFOSABI_FREEBSD, Modified: head/sys/arm/arm/generic_timer.c ============================================================================== --- head/sys/arm/arm/generic_timer.c Mon Dec 7 12:09:04 2015 (r291936) +++ head/sys/arm/arm/generic_timer.c Mon Dec 7 12:20:26 2015 (r291937) @@ -49,10 +49,13 @@ __FBSDID("$FreeBSD$"); #include <sys/rman.h> #include <sys/timeet.h> #include <sys/timetc.h> +#include <sys/smp.h> +#include <sys/vdso.h> #include <sys/watchdog.h> #include <machine/bus.h> #include <machine/cpu.h> #include <machine/intr.h> +#include <machine/md_var.h> #ifdef FDT #include <dev/fdt/fdt_common.h> @@ -121,6 +124,9 @@ static struct timecounter arm_tmr_timeco #define set_el1(x, val) WRITE_SPECIALREG(x ##_el1, val) #endif +static uint32_t arm_tmr_fill_vdso_timehands(struct vdso_timehands *vdso_th, + struct timecounter *tc); + static int get_freq(void) { @@ -181,17 +187,32 @@ get_ctrl(bool physical) } static void -disable_user_access(void) +setup_user_access(void *arg __unused) { uint32_t cntkctl; cntkctl = get_el1(cntkctl); cntkctl &= ~(GT_CNTKCTL_PL0PTEN | GT_CNTKCTL_PL0VTEN | - GT_CNTKCTL_EVNTEN | GT_CNTKCTL_PL0VCTEN | GT_CNTKCTL_PL0PCTEN); + GT_CNTKCTL_EVNTEN); + if (arm_tmr_sc->physical) { + cntkctl |= GT_CNTKCTL_PL0PCTEN; + cntkctl &= ~GT_CNTKCTL_PL0VCTEN; + } else { + cntkctl |= GT_CNTKCTL_PL0VCTEN; + cntkctl &= ~GT_CNTKCTL_PL0PCTEN; + } set_el1(cntkctl, cntkctl); isb(); } +static void +tmr_setup_user_access(void *arg __unused) +{ + + smp_rendezvous(NULL, setup_user_access, NULL, NULL); +} +SYSINIT(tmr_ua, SI_SUB_SMP, SI_ORDER_SECOND, tmr_setup_user_access, NULL); + static unsigned arm_tmr_get_timecount(struct timecounter *tc) { @@ -381,7 +402,7 @@ arm_tmr_attach(device_t dev) } } - disable_user_access(); + arm_cpu_fill_vdso_timehands = arm_tmr_fill_vdso_timehands; arm_tmr_timecount.tc_frequency = sc->clkfreq; tc_init(&arm_tmr_timecount); @@ -485,3 +506,13 @@ DELAY(int usec) first = last; } } + +static uint32_t +arm_tmr_fill_vdso_timehands(struct vdso_timehands *vdso_th, + struct timecounter *tc) +{ + + vdso_th->th_physical = arm_tmr_sc->physical; + bzero(vdso_th->th_res, sizeof(vdso_th->th_res)); + return (tc == &arm_tmr_timecount); +} Modified: head/sys/arm/arm/machdep.c ============================================================================== --- head/sys/arm/arm/machdep.c Mon Dec 7 12:09:04 2015 (r291936) +++ head/sys/arm/arm/machdep.c Mon Dec 7 12:20:26 2015 (r291937) @@ -82,6 +82,7 @@ __FBSDID("$FreeBSD$"); #include <sys/sysent.h> #include <sys/sysproto.h> #include <sys/uio.h> +#include <sys/vdso.h> #include <vm/vm.h> #include <vm/pmap.h> @@ -271,6 +272,7 @@ sendsig(catcher, ksi, mask) struct trapframe *tf; struct sigframe *fp, frame; struct sigacts *psp; + struct sysentvec *sysent; int onstack; int sig; int code; @@ -337,7 +339,12 @@ sendsig(catcher, ksi, mask) tf->tf_r5 = (register_t)&fp->sf_uc; tf->tf_pc = (register_t)catcher; tf->tf_usr_sp = (register_t)fp; - tf->tf_usr_lr = (register_t)(PS_STRINGS - *(p->p_sysent->sv_szsigcode)); + sysent = p->p_sysent; + if (sysent->sv_sigcode_base != 0) + tf->tf_usr_lr = (register_t)sysent->sv_sigcode_base; + else + tf->tf_usr_lr = (register_t)(sysent->sv_psstrings - + *(sysent->sv_szsigcode)); /* Set the mode to enter in the signal handler */ #if __ARM_ARCH >= 7 if ((register_t)catcher & 1) @@ -1910,3 +1917,14 @@ initarm(struct arm_boot_params *abp) #endif /* !ARM_NEW_PMAP */ #endif /* FDT */ + +uint32_t (*arm_cpu_fill_vdso_timehands)(struct vdso_timehands *, + struct timecounter *); + +uint32_t +cpu_fill_vdso_timehands(struct vdso_timehands *vdso_th, struct timecounter *tc) +{ + + return (arm_cpu_fill_vdso_timehands != NULL ? + arm_cpu_fill_vdso_timehands(vdso_th, tc) : 0); +} Modified: head/sys/arm/include/md_var.h ============================================================================== --- head/sys/arm/include/md_var.h Mon Dec 7 12:09:04 2015 (r291936) +++ head/sys/arm/include/md_var.h Mon Dec 7 12:20:26 2015 (r291937) @@ -45,6 +45,11 @@ extern int (*_arm_bzero)(void *, int, in extern int _min_memcpy_size; extern int _min_bzero_size; +struct vdso_timehands; +struct timecounter; +extern uint32_t (*arm_cpu_fill_vdso_timehands)(struct vdso_timehands *, + struct timecounter *); + #define DST_IS_USER 0x1 #define SRC_IS_USER 0x2 #define IS_PHYSICAL 0x4 Modified: head/sys/arm/include/vdso.h ============================================================================== --- head/sys/arm/include/vdso.h Mon Dec 7 12:09:04 2015 (r291936) +++ head/sys/arm/include/vdso.h Mon Dec 7 12:20:26 2015 (r291937) @@ -29,6 +29,7 @@ #define _ARM_VDSO_H #define VDSO_TIMEHANDS_MD \ - uint32_t th_res[8]; + uint32_t th_physical; \ + uint32_t th_res[7]; #endif Modified: head/sys/arm/include/vmparam.h ============================================================================== --- head/sys/arm/include/vmparam.h Mon Dec 7 12:09:04 2015 (r291936) +++ head/sys/arm/include/vmparam.h Mon Dec 7 12:20:26 2015 (r291937) @@ -124,7 +124,8 @@ #endif #define VM_MAX_ADDRESS VM_MAXUSER_ADDRESS -#define USRSTACK VM_MAXUSER_ADDRESS +#define SHAREDPAGE (VM_MAXUSER_ADDRESS - PAGE_SIZE) +#define USRSTACK SHAREDPAGE /* initial pagein size of beginning of executable file */ #ifndef VM_INITIAL_PAGEIN Modified: head/sys/arm64/arm64/elf_machdep.c ============================================================================== --- head/sys/arm64/arm64/elf_machdep.c Mon Dec 7 12:09:04 2015 (r291936) +++ head/sys/arm64/arm64/elf_machdep.c Mon Dec 7 12:20:26 2015 (r291937) @@ -80,12 +80,15 @@ static struct sysentvec elf64_freebsd_sy .sv_setregs = exec_setregs, .sv_fixlimit = NULL, .sv_maxssiz = NULL, - .sv_flags = SV_ABI_FREEBSD | SV_LP64, + .sv_flags = SV_SHP | SV_TIMEKEEP | SV_ABI_FREEBSD | SV_LP64, .sv_set_syscall_retval = cpu_set_syscall_retval, .sv_fetch_syscall_args = cpu_fetch_syscall_args, .sv_syscallnames = syscallnames, + .sv_shared_page_base = SHAREDPAGE, + .sv_shared_page_len = PAGE_SIZE, .sv_schedtail = NULL, }; +INIT_SYSENTVEC(elf64_sysvec, &elf64_freebsd_sysvec); static Elf64_Brandinfo freebsd_brand_info = { .brand = ELFOSABI_FREEBSD, Modified: head/sys/arm64/arm64/machdep.c ============================================================================== --- head/sys/arm64/arm64/machdep.c Mon Dec 7 12:09:04 2015 (r291936) +++ head/sys/arm64/arm64/machdep.c Mon Dec 7 12:20:26 2015 (r291937) @@ -56,6 +56,7 @@ __FBSDID("$FreeBSD$"); #include <sys/sysent.h> #include <sys/sysproto.h> #include <sys/ucontext.h> +#include <sys/vdso.h> #include <vm/vm.h> #include <vm/vm_kern.h> @@ -72,6 +73,7 @@ __FBSDID("$FreeBSD$"); #include <machine/devmap.h> #include <machine/machdep.h> #include <machine/metadata.h> +#include <machine/md_var.h> #include <machine/pcb.h> #include <machine/reg.h> #include <machine/vmparam.h> @@ -505,6 +507,7 @@ sendsig(sig_t catcher, ksiginfo_t *ksi, struct trapframe *tf; struct sigframe *fp, frame; struct sigacts *psp; + struct sysentvec *sysent; int code, onstack, sig; td = curthread; @@ -563,7 +566,12 @@ sendsig(sig_t catcher, ksiginfo_t *ksi, tf->tf_elr = (register_t)catcher; tf->tf_sp = (register_t)fp; - tf->tf_lr = (register_t)(PS_STRINGS - *(p->p_sysent->sv_szsigcode)); + sysent = p->p_sysent; + if (sysent->sv_sigcode_base != 0) + tf->tf_lr = (register_t)sysent->sv_sigcode_base; + else + tf->tf_lr = (register_t)(sysent->sv_psstrings - + *(sysent->sv_szsigcode)); CTR3(KTR_SIG, "sendsig: return td=%p pc=%#x sp=%#x", td, tf->tf_elr, tf->tf_sp); @@ -875,6 +883,17 @@ initarm(struct arm64_bootparams *abp) early_boot = 0; } +uint32_t (*arm_cpu_fill_vdso_timehands)(struct vdso_timehands *, + struct timecounter *); + +uint32_t +cpu_fill_vdso_timehands(struct vdso_timehands *vdso_th, struct timecounter *tc) +{ + + return (arm_cpu_fill_vdso_timehands != NULL ? + arm_cpu_fill_vdso_timehands(vdso_th, tc) : 0); +} + #ifdef DDB #include <ddb/ddb.h> Modified: head/sys/arm64/include/md_var.h ============================================================================== --- head/sys/arm64/include/md_var.h Mon Dec 7 12:09:04 2015 (r291936) +++ head/sys/arm64/include/md_var.h Mon Dec 7 12:20:26 2015 (r291937) @@ -46,4 +46,9 @@ void dump_add_page(vm_paddr_t); void dump_drop_page(vm_paddr_t); int minidumpsys(struct dumperinfo *); +struct vdso_timehands; +struct timecounter; +extern uint32_t (*arm_cpu_fill_vdso_timehands)(struct vdso_timehands *, + struct timecounter *); + #endif /* !_MACHINE_MD_VAR_H_ */ Modified: head/sys/arm64/include/vdso.h ============================================================================== --- head/sys/arm64/include/vdso.h Mon Dec 7 12:09:04 2015 (r291936) +++ head/sys/arm64/include/vdso.h Mon Dec 7 12:20:26 2015 (r291937) @@ -29,6 +29,7 @@ #define _MACHINE_VDSO_H_ #define VDSO_TIMEHANDS_MD \ - uint32_t th_res[8]; + uint32_t th_physical; \ + uint32_t th_res[7]; #endif /* !_MACHINE_VDSO_H_ */ Modified: head/sys/arm64/include/vmparam.h ============================================================================== --- head/sys/arm64/include/vmparam.h Mon Dec 7 12:09:04 2015 (r291936) +++ head/sys/arm64/include/vmparam.h Mon Dec 7 12:20:26 2015 (r291937) @@ -194,7 +194,8 @@ extern vm_paddr_t dmap_phys_base; #define VM_MAXUSER_ADDRESS (VM_MAX_USER_ADDRESS) #define KERNBASE (VM_MIN_KERNEL_ADDRESS) -#define USRSTACK (VM_MAX_USER_ADDRESS) +#define SHAREDPAGE (VM_MAXUSER_ADDRESS - PAGE_SIZE) +#define USRSTACK SHAREDPAGE /* * How many physical pages per kmem arena virtual page. Modified: head/sys/conf/files.arm ============================================================================== --- head/sys/conf/files.arm Mon Dec 7 12:09:04 2015 (r291936) +++ head/sys/conf/files.arm Mon Dec 7 12:20:26 2015 (r291937) @@ -110,7 +110,6 @@ font.h optional sc \ no-obj no-implicit-rule before-depend \ clean "font.h ${SC_DFLT_FONT}-8x14 ${SC_DFLT_FONT}-8x16 ${SC_DFLT_FONT}-8x8" kern/subr_busdma_bufalloc.c standard -kern/subr_dummy_vdso_tc.c standard kern/subr_sfbuf.c standard libkern/arm/aeabi_unwind.c standard libkern/arm/divsi3.S standard Modified: head/sys/conf/files.arm64 ============================================================================== --- head/sys/conf/files.arm64 Mon Dec 7 12:09:04 2015 (r291936) +++ head/sys/conf/files.arm64 Mon Dec 7 12:20:26 2015 (r291937) @@ -79,7 +79,6 @@ dev/vnic/thunder_mdio_fdt.c optional vni dev/vnic/thunder_mdio.c optional vnic dev/vnic/lmac_if.m optional vnic kern/kern_clocksource.c standard -kern/subr_dummy_vdso_tc.c standard libkern/bcmp.c standard libkern/ffs.c standard libkern/ffsl.c standard Modified: head/sys/kern/imgact_elf.c ============================================================================== --- head/sys/kern/imgact_elf.c Mon Dec 7 12:09:04 2015 (r291936) +++ head/sys/kern/imgact_elf.c Mon Dec 7 12:20:26 2015 (r291937) @@ -80,6 +80,9 @@ __FBSDID("$FreeBSD$"); #include <machine/elf.h> #include <machine/md_var.h> +#ifdef __arm__ +#include <machine/acle-compat.h> +#endif #define ELF_NOTE_ROUNDSIZE 4 #define OLD_EI_BRAND 8 @@ -116,7 +119,8 @@ SYSCTL_INT(_debug, OID_AUTO, __elfN(lega &elf_legacy_coredump, 0, ""); int __elfN(nxstack) = -#if defined(__amd64__) || defined(__powerpc64__) /* both 64 and 32 bit */ +#if defined(__amd64__) || defined(__powerpc64__) /* both 64 and 32 bit */ || \ + (defined(__arm__) && __ARM_ARCH >= 7) || defined(__aarch64__) 1; #else 0; _______________________________________________ svn-src-head@freebsd.org mailing list https://lists.freebsd.org/mailman/listinfo/svn-src-head To unsubscribe, send any mail to "svn-src-head-unsubscr...@freebsd.org"
