Author: kib Date: Tue Aug 7 18:29:10 2018 New Revision: 337431 URL: https://svnweb.freebsd.org/changeset/base/337431
Log: Futex support functions in linux.ko and linux32.ko on amd64 should be aware of SMAP. Reported and tested by: Johannes Lundberg <[email protected]>, wulf Sponsored by: The FreeBSD Foundation Modified: head/sys/amd64/linux/linux_machdep.c head/sys/amd64/linux/linux_support.s head/sys/amd64/linux32/linux32_machdep.c head/sys/amd64/linux32/linux32_support.s Modified: head/sys/amd64/linux/linux_machdep.c ============================================================================== --- head/sys/amd64/linux/linux_machdep.c Tue Aug 7 18:26:46 2018 (r337430) +++ head/sys/amd64/linux/linux_machdep.c Tue Aug 7 18:29:10 2018 (r337431) @@ -78,6 +78,9 @@ __FBSDID("$FreeBSD$"); #include <vm/vm_kern.h> #include <vm/vm_map.h> +#include <x86/ifunc.h> +#include <x86/sysarch.h> + #include <amd64/linux/linux.h> #include <amd64/linux/linux_proto.h> #include <compat/linux/linux_emul.h> @@ -88,8 +91,6 @@ __FBSDID("$FreeBSD$"); #include <compat/linux/linux_signal.h> #include <compat/linux/linux_util.h> -#include <x86/include/sysarch.h> - int linux_execve(struct thread *td, struct linux_execve_args *args) { @@ -275,4 +276,49 @@ linux_set_cloned_tls(struct thread *td, void *desc) td->td_frame->tf_fs = _ufssel; return (0); +} + +int futex_xchgl_nosmap(int oparg, uint32_t *uaddr, int *oldval); +int futex_xchgl_smap(int oparg, uint32_t *uaddr, int *oldval); +DEFINE_IFUNC(, int, futex_xchgl, (int, uint32_t *, int *), static) +{ + + return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? + futex_xchgl_smap : futex_xchgl_nosmap); +} + +int futex_addl_nosmap(int oparg, uint32_t *uaddr, int *oldval); +int futex_addl_smap(int oparg, uint32_t *uaddr, int *oldval); +DEFINE_IFUNC(, int, futex_addl, (int, uint32_t *, int *), static) +{ + + return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? + futex_addl_smap : futex_addl_nosmap); +} + +int futex_orl_nosmap(int oparg, uint32_t *uaddr, int *oldval); +int futex_orl_smap(int oparg, uint32_t *uaddr, int *oldval); +DEFINE_IFUNC(, int, futex_orl, (int, uint32_t *, int *), static) +{ + + return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? + futex_orl_smap : futex_orl_nosmap); +} + +int futex_andl_nosmap(int oparg, uint32_t *uaddr, int *oldval); +int futex_andl_smap(int oparg, uint32_t *uaddr, int *oldval); +DEFINE_IFUNC(, int, futex_andl, (int, uint32_t *, int *), static) +{ + + return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? + futex_andl_smap : futex_andl_nosmap); +} + +int futex_xorl_nosmap(int oparg, uint32_t *uaddr, int *oldval); +int futex_xorl_smap(int oparg, uint32_t *uaddr, int *oldval); +DEFINE_IFUNC(, int, futex_xorl, (int, uint32_t *, int *), static) +{ + + return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? + futex_xorl_smap : futex_xorl_nosmap); } Modified: head/sys/amd64/linux/linux_support.s ============================================================================== --- head/sys/amd64/linux/linux_support.s Tue Aug 7 18:26:46 2018 (r337430) +++ head/sys/amd64/linux/linux_support.s Tue Aug 7 18:29:10 2018 (r337431) @@ -38,7 +38,7 @@ futex_fault: movl $-EFAULT,%eax ret -ENTRY(futex_xchgl) +ENTRY(futex_xchgl_nosmap) movq PCPU(CURPCB),%r8 movq $futex_fault,PCB_ONFAULT(%r8) movq $VM_MAXUSER_ADDRESS-4,%rax @@ -49,14 +49,29 @@ ENTRY(futex_xchgl) xorl %eax,%eax movq %rax,PCB_ONFAULT(%r8) ret -END(futex_xchgl) +END(futex_xchgl_nosmap) -ENTRY(futex_addl) +ENTRY(futex_xchgl_smap) movq PCPU(CURPCB),%r8 movq $futex_fault,PCB_ONFAULT(%r8) movq $VM_MAXUSER_ADDRESS-4,%rax cmpq %rax,%rsi ja futex_fault + stac + xchgl %edi,(%rsi) + clac + movl %edi,(%rdx) + xorl %eax,%eax + movq %rax,PCB_ONFAULT(%r8) + ret +END(futex_xchgl_smap) + +ENTRY(futex_addl_nosmap) + movq PCPU(CURPCB),%r8 + movq $futex_fault,PCB_ONFAULT(%r8) + movq $VM_MAXUSER_ADDRESS-4,%rax + cmpq %rax,%rsi + ja futex_fault #ifdef SMP lock #endif @@ -65,14 +80,32 @@ ENTRY(futex_addl) xorl %eax,%eax movq %rax,PCB_ONFAULT(%r8) ret -END(futex_addl) +END(futex_addl_nosmap) -ENTRY(futex_orl) +ENTRY(futex_addl_smap) movq PCPU(CURPCB),%r8 movq $futex_fault,PCB_ONFAULT(%r8) movq $VM_MAXUSER_ADDRESS-4,%rax cmpq %rax,%rsi ja futex_fault + stac +#ifdef SMP + lock +#endif + xaddl %edi,(%rsi) + clac + movl %edi,(%rdx) + xorl %eax,%eax + movq %rax,PCB_ONFAULT(%r8) + ret +END(futex_addl_smap) + +ENTRY(futex_orl_nosmap) + movq PCPU(CURPCB),%r8 + movq $futex_fault,PCB_ONFAULT(%r8) + movq $VM_MAXUSER_ADDRESS-4,%rax + cmpq %rax,%rsi + ja futex_fault movl (%rsi),%eax 1: movl %eax,%ecx orl %edi,%ecx @@ -85,9 +118,9 @@ ENTRY(futex_orl) xorl %eax,%eax movq %rax,PCB_ONFAULT(%r8) ret -END(futex_orl) +END(futex_orl_nosmap) -ENTRY(futex_andl) +ENTRY(futex_orl_smap) movq PCPU(CURPCB),%r8 movq $futex_fault,PCB_ONFAULT(%r8) movq $VM_MAXUSER_ADDRESS-4,%rax @@ -95,6 +128,28 @@ ENTRY(futex_andl) ja futex_fault movl (%rsi),%eax 1: movl %eax,%ecx + orl %edi,%ecx + stac +#ifdef SMP + lock +#endif + cmpxchgl %ecx,(%rsi) + clac + jnz 1b + movl %eax,(%rdx) + xorl %eax,%eax + movq %rax,PCB_ONFAULT(%r8) + ret +END(futex_orl_smap) + +ENTRY(futex_andl_nosmap) + movq PCPU(CURPCB),%r8 + movq $futex_fault,PCB_ONFAULT(%r8) + movq $VM_MAXUSER_ADDRESS-4,%rax + cmpq %rax,%rsi + ja futex_fault + movl (%rsi),%eax +1: movl %eax,%ecx andl %edi,%ecx #ifdef SMP lock @@ -105,9 +160,9 @@ ENTRY(futex_andl) xorl %eax,%eax movq %rax,PCB_ONFAULT(%r8) ret -END(futex_andl) +END(futex_andl_nosmap) -ENTRY(futex_xorl) +ENTRY(futex_andl_smap) movq PCPU(CURPCB),%r8 movq $futex_fault,PCB_ONFAULT(%r8) movq $VM_MAXUSER_ADDRESS-4,%rax @@ -115,6 +170,28 @@ ENTRY(futex_xorl) ja futex_fault movl (%rsi),%eax 1: movl %eax,%ecx + andl %edi,%ecx + stac +#ifdef SMP + lock +#endif + cmpxchgl %ecx,(%rsi) + clac + jnz 1b + movl %eax,(%rdx) + xorl %eax,%eax + movq %rax,PCB_ONFAULT(%r8) + ret +END(futex_andl_smap) + +ENTRY(futex_xorl_nosmap) + movq PCPU(CURPCB),%r8 + movq $futex_fault,PCB_ONFAULT(%r8) + movq $VM_MAXUSER_ADDRESS-4,%rax + cmpq %rax,%rsi + ja futex_fault + movl (%rsi),%eax +1: movl %eax,%ecx xorl %edi,%ecx #ifdef SMP lock @@ -125,4 +202,26 @@ ENTRY(futex_xorl) xorl %eax,%eax movq %rax,PCB_ONFAULT(%r8) ret -END(futex_xorl) +END(futex_xorl_nosmap) + +ENTRY(futex_xorl_smap) + movq PCPU(CURPCB),%r8 + movq $futex_fault,PCB_ONFAULT(%r8) + movq $VM_MAXUSER_ADDRESS-4,%rax + cmpq %rax,%rsi + ja futex_fault + movl (%rsi),%eax +1: movl %eax,%ecx + xorl %edi,%ecx + stac +#ifdef SMP + lock +#endif + cmpxchgl %ecx,(%rsi) + clac + jnz 1b + movl %eax,(%rdx) + xorl %eax,%eax + movq %rax,PCB_ONFAULT(%r8) + ret +END(futex_xorl_smap) Modified: head/sys/amd64/linux32/linux32_machdep.c ============================================================================== --- head/sys/amd64/linux32/linux32_machdep.c Tue Aug 7 18:26:46 2018 (r337430) +++ head/sys/amd64/linux32/linux32_machdep.c Tue Aug 7 18:29:10 2018 (r337431) @@ -58,10 +58,12 @@ __FBSDID("$FreeBSD$"); #include <sys/wait.h> #include <machine/frame.h> +#include <machine/md_var.h> #include <machine/pcb.h> #include <machine/psl.h> #include <machine/segments.h> #include <machine/specialreg.h> +#include <x86/ifunc.h> #include <vm/pmap.h> #include <vm/vm.h> @@ -821,4 +823,49 @@ linux_set_thread_area(struct thread *td, update_gdt_gsbase(td, info.base_addr); return (0); +} + +int futex_xchgl_nosmap(int oparg, uint32_t *uaddr, int *oldval); +int futex_xchgl_smap(int oparg, uint32_t *uaddr, int *oldval); +DEFINE_IFUNC(, int, futex_xchgl, (int, uint32_t *, int *), static) +{ + + return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? + futex_xchgl_smap : futex_xchgl_nosmap); +} + +int futex_addl_nosmap(int oparg, uint32_t *uaddr, int *oldval); +int futex_addl_smap(int oparg, uint32_t *uaddr, int *oldval); +DEFINE_IFUNC(, int, futex_addl, (int, uint32_t *, int *), static) +{ + + return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? + futex_addl_smap : futex_addl_nosmap); +} + +int futex_orl_nosmap(int oparg, uint32_t *uaddr, int *oldval); +int futex_orl_smap(int oparg, uint32_t *uaddr, int *oldval); +DEFINE_IFUNC(, int, futex_orl, (int, uint32_t *, int *), static) +{ + + return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? + futex_orl_smap : futex_orl_nosmap); +} + +int futex_andl_nosmap(int oparg, uint32_t *uaddr, int *oldval); +int futex_andl_smap(int oparg, uint32_t *uaddr, int *oldval); +DEFINE_IFUNC(, int, futex_andl, (int, uint32_t *, int *), static) +{ + + return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? + futex_andl_smap : futex_andl_nosmap); +} + +int futex_xorl_nosmap(int oparg, uint32_t *uaddr, int *oldval); +int futex_xorl_smap(int oparg, uint32_t *uaddr, int *oldval); +DEFINE_IFUNC(, int, futex_xorl, (int, uint32_t *, int *), static) +{ + + return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? + futex_xorl_smap : futex_xorl_nosmap); } Modified: head/sys/amd64/linux32/linux32_support.s ============================================================================== --- head/sys/amd64/linux32/linux32_support.s Tue Aug 7 18:26:46 2018 (r337430) +++ head/sys/amd64/linux32/linux32_support.s Tue Aug 7 18:29:10 2018 (r337431) @@ -38,7 +38,7 @@ futex_fault: movl $-EFAULT,%eax ret -ENTRY(futex_xchgl) +ENTRY(futex_xchgl_nosmap) movq PCPU(CURPCB),%r8 movq $futex_fault,PCB_ONFAULT(%r8) movq $VM_MAXUSER_ADDRESS-4,%rax @@ -49,14 +49,29 @@ ENTRY(futex_xchgl) xorl %eax,%eax movq %rax,PCB_ONFAULT(%r8) ret -END(futex_xchgl) +END(futex_xchgl_nosmap) -ENTRY(futex_addl) +ENTRY(futex_xchgl_smap) movq PCPU(CURPCB),%r8 movq $futex_fault,PCB_ONFAULT(%r8) movq $VM_MAXUSER_ADDRESS-4,%rax cmpq %rax,%rsi ja futex_fault + stac + xchgl %edi,(%rsi) + clac + movl %edi,(%rdx) + xorl %eax,%eax + movq %rax,PCB_ONFAULT(%r8) + ret +END(futex_xchgl_smap) + +ENTRY(futex_addl_nosmap) + movq PCPU(CURPCB),%r8 + movq $futex_fault,PCB_ONFAULT(%r8) + movq $VM_MAXUSER_ADDRESS-4,%rax + cmpq %rax,%rsi + ja futex_fault #ifdef SMP lock #endif @@ -65,14 +80,32 @@ ENTRY(futex_addl) xorl %eax,%eax movq %rax,PCB_ONFAULT(%r8) ret -END(futex_addl) +END(futex_addl_nosmap) -ENTRY(futex_orl) +ENTRY(futex_addl_smap) movq PCPU(CURPCB),%r8 movq $futex_fault,PCB_ONFAULT(%r8) movq $VM_MAXUSER_ADDRESS-4,%rax cmpq %rax,%rsi ja futex_fault + stac +#ifdef SMP + lock +#endif + xaddl %edi,(%rsi) + clac + movl %edi,(%rdx) + xorl %eax,%eax + movq %rax,PCB_ONFAULT(%r8) + ret +END(futex_addl_smap) + +ENTRY(futex_orl_nosmap) + movq PCPU(CURPCB),%r8 + movq $futex_fault,PCB_ONFAULT(%r8) + movq $VM_MAXUSER_ADDRESS-4,%rax + cmpq %rax,%rsi + ja futex_fault movl (%rsi),%eax 1: movl %eax,%ecx orl %edi,%ecx @@ -85,9 +118,9 @@ ENTRY(futex_orl) xorl %eax,%eax movq %rax,PCB_ONFAULT(%r8) ret -END(futex_orl) +END(futex_orl_nosmap) -ENTRY(futex_andl) +ENTRY(futex_orl_smap) movq PCPU(CURPCB),%r8 movq $futex_fault,PCB_ONFAULT(%r8) movq $VM_MAXUSER_ADDRESS-4,%rax @@ -95,6 +128,28 @@ ENTRY(futex_andl) ja futex_fault movl (%rsi),%eax 1: movl %eax,%ecx + orl %edi,%ecx + stac +#ifdef SMP + lock +#endif + cmpxchgl %ecx,(%rsi) + clac + jnz 1b + movl %eax,(%rdx) + xorl %eax,%eax + movq %rax,PCB_ONFAULT(%r8) + ret +END(futex_orl_smap) + +ENTRY(futex_andl_nosmap) + movq PCPU(CURPCB),%r8 + movq $futex_fault,PCB_ONFAULT(%r8) + movq $VM_MAXUSER_ADDRESS-4,%rax + cmpq %rax,%rsi + ja futex_fault + movl (%rsi),%eax +1: movl %eax,%ecx andl %edi,%ecx #ifdef SMP lock @@ -105,9 +160,9 @@ ENTRY(futex_andl) xorl %eax,%eax movq %rax,PCB_ONFAULT(%r8) ret -END(futex_andl) +END(futex_andl_nosmap) -ENTRY(futex_xorl) +ENTRY(futex_andl_smap) movq PCPU(CURPCB),%r8 movq $futex_fault,PCB_ONFAULT(%r8) movq $VM_MAXUSER_ADDRESS-4,%rax @@ -115,6 +170,28 @@ ENTRY(futex_xorl) ja futex_fault movl (%rsi),%eax 1: movl %eax,%ecx + andl %edi,%ecx + stac +#ifdef SMP + lock +#endif + cmpxchgl %ecx,(%rsi) + clac + jnz 1b + movl %eax,(%rdx) + xorl %eax,%eax + movq %rax,PCB_ONFAULT(%r8) + ret +END(futex_andl_smap) + +ENTRY(futex_xorl_nosmap) + movq PCPU(CURPCB),%r8 + movq $futex_fault,PCB_ONFAULT(%r8) + movq $VM_MAXUSER_ADDRESS-4,%rax + cmpq %rax,%rsi + ja futex_fault + movl (%rsi),%eax +1: movl %eax,%ecx xorl %edi,%ecx #ifdef SMP lock @@ -125,4 +202,26 @@ ENTRY(futex_xorl) xorl %eax,%eax movq %rax,PCB_ONFAULT(%r8) ret -END(futex_xorl) +END(futex_xorl_nosmap) + +ENTRY(futex_xorl_smap) + movq PCPU(CURPCB),%r8 + movq $futex_fault,PCB_ONFAULT(%r8) + movq $VM_MAXUSER_ADDRESS-4,%rax + cmpq %rax,%rsi + ja futex_fault + movl (%rsi),%eax +1: movl %eax,%ecx + xorl %edi,%ecx + stac +#ifdef SMP + lock +#endif + cmpxchgl %ecx,(%rsi) + clac + jnz 1b + movl %eax,(%rdx) + xorl %eax,%eax + movq %rax,PCB_ONFAULT(%r8) + ret +END(futex_xorl_smap) _______________________________________________ [email protected] mailing list https://lists.freebsd.org/mailman/listinfo/svn-src-head To unsubscribe, send any mail to "[email protected]"
