Author: mjg Date: Fri Sep 21 15:00:46 2018 New Revision: 338858 URL: https://svnweb.freebsd.org/changeset/base/338858
Log: amd64: even up copyin/copyout with memcpy + other cleanup - _fault handlers for both primitives are identical, provide just one - change the copying scheme to match memcpy (in particular jump avoidance for the most common case of multiply of 8) - stop re-reading pcb address on exit, just store it locally (in r9) Reviewed by: kib Approved by: re (gjb) Differential Revision: https://reviews.freebsd.org/D17265 Modified: head/sys/amd64/amd64/support.S Modified: head/sys/amd64/amd64/support.S ============================================================================== --- head/sys/amd64/amd64/support.S Fri Sep 21 13:44:05 2018 (r338857) +++ head/sys/amd64/amd64/support.S Fri Sep 21 15:00:46 2018 (r338858) @@ -309,9 +309,9 @@ END(fillw) */ .macro COPYOUT smap erms PUSH_FRAME_POINTER - movq PCPU(CURPCB),%rax + movq PCPU(CURPCB),%r9 /* Trap entry clears PSL.AC */ - movq $copyout_fault,PCB_ONFAULT(%rax) + movq $copy_fault,PCB_ONFAULT(%r9) testq %rdx,%rdx /* anything to do? */ jz 2f @@ -327,7 +327,7 @@ END(fillw) */ movq %rsi,%rax addq %rdx,%rax - jc copyout_fault + jc copy_fault /* * XXX STOP USING VM_MAXUSER_ADDRESS. * It is an end address, not a max, so every time it is used correctly it @@ -336,7 +336,7 @@ END(fillw) */ movq $VM_MAXUSER_ADDRESS,%rcx cmpq %rcx,%rax - ja copyout_fault + ja copy_fault xchgq %rdi,%rsi /* bcopy(%rsi, %rdi, %rdx) */ @@ -344,21 +344,27 @@ END(fillw) SMAP_DISABLE \smap .if \erms == 0 + cmpq $15,%rcx + jbe 1f shrq $3,%rcx rep movsq movb %dl,%cl andb $7,%cl - je 1f + jne 1f + SMAP_ENABLE \smap + xorl %eax,%eax + movq %rax,PCB_ONFAULT(%r9) + POP_FRAME_POINTER + ret .endif +1: rep movsb -1: SMAP_ENABLE \smap 2: xorl %eax,%eax - movq PCPU(CURPCB),%rdx - movq %rax,PCB_ONFAULT(%rdx) + movq %rax,PCB_ONFAULT(%r9) POP_FRAME_POINTER ret .endmacro @@ -379,22 +385,14 @@ ENTRY(copyout_smap_erms) COPYOUT smap=1 erms=1 END(copyout_smap_erms) - ALIGN_TEXT -copyout_fault: - movq PCPU(CURPCB),%rdx - movq $0,PCB_ONFAULT(%rdx) - movq $EFAULT,%rax - POP_FRAME_POINTER - ret - /* * copyin(from_user, to_kernel, len) * %rdi, %rsi, %rdx */ .macro COPYIN smap erms PUSH_FRAME_POINTER - movq PCPU(CURPCB),%rax - movq $copyin_fault,PCB_ONFAULT(%rax) + movq PCPU(CURPCB),%r9 + movq $copy_fault,PCB_ONFAULT(%r9) testq %rdx,%rdx /* anything to do? */ jz 2f @@ -403,10 +401,10 @@ copyout_fault: */ movq %rdi,%rax addq %rdx,%rax - jc copyin_fault + jc copy_fault movq $VM_MAXUSER_ADDRESS,%rcx cmpq %rcx,%rax - ja copyin_fault + ja copy_fault xchgq %rdi,%rsi movq %rdx,%rcx @@ -414,22 +412,28 @@ copyout_fault: SMAP_DISABLE \smap .if \erms == 0 + cmpq $15,%rcx + jbe 1f shrq $3,%rcx /* copy longword-wise */ rep movsq movb %al,%cl andb $7,%cl /* copy remaining bytes */ - je 1f + jne 1f + SMAP_ENABLE \smap + xorl %eax,%eax + movq %rax,PCB_ONFAULT(%r9) + POP_FRAME_POINTER + ret .endif +1: rep movsb -1: SMAP_ENABLE \smap 2: xorl %eax,%eax - movq PCPU(CURPCB),%rdx - movq %rax,PCB_ONFAULT(%rdx) + movq %rax,PCB_ONFAULT(%r9) POP_FRAME_POINTER ret .endmacro @@ -451,10 +455,10 @@ ENTRY(copyin_smap_erms) END(copyin_smap_erms) ALIGN_TEXT -copyin_fault: +copy_fault: movq PCPU(CURPCB),%rdx movq $0,PCB_ONFAULT(%rdx) - movq $EFAULT,%rax + movl $EFAULT,%eax POP_FRAME_POINTER ret _______________________________________________ [email protected] mailing list https://lists.freebsd.org/mailman/listinfo/svn-src-head To unsubscribe, send any mail to "[email protected]"
