pkarashchenko commented on code in PR #8042: URL: https://github.com/apache/nuttx/pull/8042#discussion_r1064061210
########## libs/libc/machine/risc-v/gnu/asm.h: ########## @@ -0,0 +1,52 @@ +/**************************************************************************** + * libs/libc/machine/risc-v/gnu/asm.h + * + * Copyright (c) 2017 SiFive Inc. All rights reserved. + * + * This copyrighted material is made available to anyone wishing to use, + * modify, copy, or redistribute it subject to the terms and conditions + * of the FreeBSD License. This program is distributed in the hope that + * it will be useful, but WITHOUT ANY WARRANTY expressed or implied, + * including the implied warranties of MERCHANTABILITY or FITNESS FOR + * A PARTICULAR PURPOSE. A copy of this license is available at + * http://www.opensource.org/licenses. + ****************************************************************************/ + +#ifndef _ASM_H +#define _ASM_H Review Comment: ```suggestion #ifndef __LIBS_LIBC_MACHINE_RISCV_GNU_ASM_H #define __LIBS_LIBC_MACHINE_RISCV_GNU_ASM_H ``` ########## libs/libc/machine/risc-v/gnu/asm.h: ########## @@ -0,0 +1,52 @@ +/**************************************************************************** + * libs/libc/machine/risc-v/gnu/asm.h + * + * Copyright (c) 2017 SiFive Inc. All rights reserved. + * + * This copyrighted material is made available to anyone wishing to use, + * modify, copy, or redistribute it subject to the terms and conditions + * of the FreeBSD License. This program is distributed in the hope that + * it will be useful, but WITHOUT ANY WARRANTY expressed or implied, + * including the implied warranties of MERCHANTABILITY or FITNESS FOR + * A PARTICULAR PURPOSE. A copy of this license is available at + * http://www.opensource.org/licenses. + ****************************************************************************/ + +#ifndef _ASM_H +#define _ASM_H + +/**************************************************************************** + * Included Files + ****************************************************************************/ + +#include <nuttx/config.h> + +/**************************************************************************** + * Pre-processor Definitions + ****************************************************************************/ + +#ifdef CONFIG_ARCH_RV64 +# define SZREG 8 +# define REG_S sd +# define REG_L ld +#elif defined(CONFIG_ARCH_RV32) +# define SZREG 4 +# define REG_S sw +# define REG_L lw +#endif + +#ifdef CONFIG_ARCH_QPFPU +# define SZFREG 16 +# define FREG_S fsq +# define FREG_L flq +#elif defined(CONFIG_ARCH_DPFPU) +# define SZFREG 8 +# define FREG_S fsd +# define FREG_L fld +#elif defined(CONFIG_ARCH_FPU) +# define SZFREG 4 +# define FREG_S fsw +# define FREG_L flw +#endif + +#endif /* asm.h */ Review Comment: ```suggestion #endif /* __LIBS_LIBC_MACHINE_RISCV_GNU_ASM_H */ ``` ########## libs/libc/machine/risc-v/gnu/arch_strcmp.S: ########## @@ -0,0 +1,184 @@ +/**************************************************************************** + * libs/libc/machine/risc-v/gnu/arch_strcmp.S + * + * Copyright (c) 2017 SiFive Inc. All rights reserved. + * + * This copyrighted material is made available to anyone wishing to use, + * modify, copy, or redistribute it subject to the terms and conditions + * of the FreeBSD License. This program is distributed in the hope that + * it will be useful, but WITHOUT ANY WARRANTY expressed or implied, + * including the implied warranties of MERCHANTABILITY or FITNESS FOR + * A PARTICULAR PURPOSE. A copy of this license is available at + * http://www.opensource.org/licenses. + * + ****************************************************************************/ + +#include "asm.h" + +.text +.globl strcmp +.type strcmp, @function +strcmp: + or a4, a0, a1 + li t2, -1 + and a4, a4, SZREG-1 + bnez a4, .Lmisaligned + +#if SZREG == 4 + li a5, 0x7f7f7f7f +#else + ld a5, mask +#endif + + .macro check_one_word i n + REG_L a2, \i*SZREG(a0) + REG_L a3, \i*SZREG(a1) + + and t0, a2, a5 + or t1, a2, a5 + add t0, t0, a5 + or t0, t0, t1 + + bne t0, t2, .Lnull\i + .if \i+1-\n + bne a2, a3, .Lmismatch + .else + add a0, a0, \n*SZREG + add a1, a1, \n*SZREG + beq a2, a3, .Lloop + # fall through to .Lmismatch + .endif + .endm + + .macro foundnull i n + .ifne \i + .Lnull\i: + add a0, a0, \i*SZREG + add a1, a1, \i*SZREG + .ifeq \i-1 + .Lnull0: + .endif + bne a2, a3, .Lmisaligned + li a0, 0 + ret + .endif + .endm + +.Lloop: + # examine full words at a time, favoring strings of a couple dozen chars +#if __riscv_xlen == 32 + check_one_word 0 5 + check_one_word 1 5 + check_one_word 2 5 + check_one_word 3 5 + check_one_word 4 5 +#else + check_one_word 0 3 + check_one_word 1 3 + check_one_word 2 3 +#endif + # backwards branch to .Lloop contained above + +.Lmismatch: + # words don't match, but a2 has no null byte. + +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + +#if __riscv_xlen == 64 + sll a4, a2, 48 + sll a5, a3, 48 + bne a4, a5, .Lmismatch_upper + sll a4, a2, 32 + sll a5, a3, 32 + bne a4, a5, .Lmismatch_upper +#endif + sll a4, a2, 16 + sll a5, a3, 16 + bne a4, a5, .Lmismatch_upper + + srl a4, a2, 8*SZREG-16 + srl a5, a3, 8*SZREG-16 + sub a0, a4, a5 + and a1, a0, 0xff + bnez a1, 1f + ret + +.Lmismatch_upper: + srl a4, a4, 8*SZREG-16 + srl a5, a5, 8*SZREG-16 + sub a0, a4, a5 + and a1, a0, 0xff + bnez a1, 1f + ret + +1:and a4, a4, 0xff + and a5, a5, 0xff + sub a0, a4, a5 + ret + +#else + +#if __riscv_xlen == 64 + srl a4, a2, 48 + srl a5, a3, 48 + bne a4, a5, .Lmismatch_lower + srl a4, a2, 32 + srl a5, a3, 32 + bne a4, a5, .Lmismatch_lower +#endif + srl a4, a2, 16 + srl a5, a3, 16 + bne a4, a5, .Lmismatch_lower + + srl a4, a2, 8 + srl a5, a3, 8 + bne a4, a5, 1f + and a4, a2, 0xff + and a5, a3, 0xff +1:sub a0, a4, a5 + ret + +.Lmismatch_lower: + srl a2, a4, 8 + srl a3, a5, 8 + bne a2, a3, 1f + and a2, a4, 0xff + and a3, a5, 0xff +1:sub a0, a2, a3 + ret + +#endif + +.Lmisaligned: + # misaligned + lbu a2, 0(a0) + lbu a3, 0(a1) + add a0, a0, 1 + add a1, a1, 1 + bne a2, a3, 1f + bnez a2, .Lmisaligned + +1: + sub a0, a2, a3 + ret + + # cases in which a null byte was detected +#if __riscv_xlen == 32 Review Comment: let's use `CONFIG_ARCH_RV64` and `CONFIG_ARCH_RV32` instead of `__riscv_xlen`? ########## libs/libc/machine/risc-v/gnu/arch_strcmp.S: ########## @@ -0,0 +1,184 @@ +/**************************************************************************** + * libs/libc/machine/risc-v/gnu/arch_strcmp.S + * + * Copyright (c) 2017 SiFive Inc. All rights reserved. + * + * This copyrighted material is made available to anyone wishing to use, + * modify, copy, or redistribute it subject to the terms and conditions + * of the FreeBSD License. This program is distributed in the hope that + * it will be useful, but WITHOUT ANY WARRANTY expressed or implied, + * including the implied warranties of MERCHANTABILITY or FITNESS FOR + * A PARTICULAR PURPOSE. A copy of this license is available at + * http://www.opensource.org/licenses. + * + ****************************************************************************/ + +#include "asm.h" + +.text +.globl strcmp +.type strcmp, @function +strcmp: + or a4, a0, a1 + li t2, -1 + and a4, a4, SZREG-1 + bnez a4, .Lmisaligned + +#if SZREG == 4 + li a5, 0x7f7f7f7f +#else + ld a5, mask +#endif + + .macro check_one_word i n + REG_L a2, \i*SZREG(a0) + REG_L a3, \i*SZREG(a1) + + and t0, a2, a5 + or t1, a2, a5 + add t0, t0, a5 + or t0, t0, t1 + + bne t0, t2, .Lnull\i + .if \i+1-\n + bne a2, a3, .Lmismatch + .else + add a0, a0, \n*SZREG + add a1, a1, \n*SZREG + beq a2, a3, .Lloop + # fall through to .Lmismatch + .endif + .endm + + .macro foundnull i n + .ifne \i + .Lnull\i: + add a0, a0, \i*SZREG + add a1, a1, \i*SZREG + .ifeq \i-1 + .Lnull0: + .endif + bne a2, a3, .Lmisaligned + li a0, 0 + ret + .endif + .endm + +.Lloop: + # examine full words at a time, favoring strings of a couple dozen chars +#if __riscv_xlen == 32 + check_one_word 0 5 + check_one_word 1 5 + check_one_word 2 5 + check_one_word 3 5 + check_one_word 4 5 +#else + check_one_word 0 3 + check_one_word 1 3 + check_one_word 2 3 +#endif + # backwards branch to .Lloop contained above + +.Lmismatch: + # words don't match, but a2 has no null byte. + +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + +#if __riscv_xlen == 64 + sll a4, a2, 48 + sll a5, a3, 48 + bne a4, a5, .Lmismatch_upper + sll a4, a2, 32 + sll a5, a3, 32 + bne a4, a5, .Lmismatch_upper +#endif + sll a4, a2, 16 + sll a5, a3, 16 + bne a4, a5, .Lmismatch_upper + + srl a4, a2, 8*SZREG-16 + srl a5, a3, 8*SZREG-16 + sub a0, a4, a5 + and a1, a0, 0xff + bnez a1, 1f + ret + +.Lmismatch_upper: + srl a4, a4, 8*SZREG-16 + srl a5, a5, 8*SZREG-16 + sub a0, a4, a5 + and a1, a0, 0xff + bnez a1, 1f + ret + +1:and a4, a4, 0xff + and a5, a5, 0xff + sub a0, a4, a5 + ret + +#else + +#if __riscv_xlen == 64 + srl a4, a2, 48 + srl a5, a3, 48 + bne a4, a5, .Lmismatch_lower + srl a4, a2, 32 + srl a5, a3, 32 + bne a4, a5, .Lmismatch_lower +#endif + srl a4, a2, 16 + srl a5, a3, 16 + bne a4, a5, .Lmismatch_lower + + srl a4, a2, 8 + srl a5, a3, 8 Review Comment: ```suggestion srl a4, a2, 8 srl a5, a3, 8 ``` ########## libs/libc/machine/risc-v/gnu/arch_strcmp.S: ########## @@ -0,0 +1,184 @@ +/**************************************************************************** + * libs/libc/machine/risc-v/gnu/arch_strcmp.S + * + * Copyright (c) 2017 SiFive Inc. All rights reserved. + * + * This copyrighted material is made available to anyone wishing to use, + * modify, copy, or redistribute it subject to the terms and conditions + * of the FreeBSD License. This program is distributed in the hope that + * it will be useful, but WITHOUT ANY WARRANTY expressed or implied, + * including the implied warranties of MERCHANTABILITY or FITNESS FOR + * A PARTICULAR PURPOSE. A copy of this license is available at + * http://www.opensource.org/licenses. + * + ****************************************************************************/ + +#include "asm.h" + +.text +.globl strcmp +.type strcmp, @function +strcmp: + or a4, a0, a1 + li t2, -1 + and a4, a4, SZREG-1 + bnez a4, .Lmisaligned + +#if SZREG == 4 + li a5, 0x7f7f7f7f +#else + ld a5, mask +#endif + + .macro check_one_word i n + REG_L a2, \i*SZREG(a0) + REG_L a3, \i*SZREG(a1) + + and t0, a2, a5 + or t1, a2, a5 + add t0, t0, a5 + or t0, t0, t1 + + bne t0, t2, .Lnull\i + .if \i+1-\n + bne a2, a3, .Lmismatch + .else + add a0, a0, \n*SZREG + add a1, a1, \n*SZREG + beq a2, a3, .Lloop + # fall through to .Lmismatch + .endif + .endm + + .macro foundnull i n + .ifne \i + .Lnull\i: + add a0, a0, \i*SZREG + add a1, a1, \i*SZREG + .ifeq \i-1 + .Lnull0: + .endif + bne a2, a3, .Lmisaligned + li a0, 0 + ret + .endif + .endm + +.Lloop: + # examine full words at a time, favoring strings of a couple dozen chars +#if __riscv_xlen == 32 + check_one_word 0 5 + check_one_word 1 5 + check_one_word 2 5 + check_one_word 3 5 + check_one_word 4 5 +#else + check_one_word 0 3 + check_one_word 1 3 + check_one_word 2 3 +#endif + # backwards branch to .Lloop contained above + +.Lmismatch: + # words don't match, but a2 has no null byte. + +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ Review Comment: ```suggestion #if BYTE_ORDER == LITTLE_ENDIAN ``` ########## libs/libc/machine/arm64/gnu/arch_strnlen.S: ########## @@ -0,0 +1,188 @@ +/**************************************************************************** + * libs/libc/machine/arm64/gnu/arch_strnlen.S + * + * Copyright (c) 2013, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the Linaro nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ****************************************************************************/ + +/* Assumptions: + * + * ARMv8-a, AArch64 + */ + +/* Arguments and results. */ +#define srcin x0 +#define len x0 +#define limit x1 + +/* Locals and temporaries. */ +#define src x2 +#define data1 x3 +#define data2 x4 +#define data2a x5 +#define has_nul1 x6 +#define has_nul2 x7 +#define tmp1 x8 +#define tmp2 x9 +#define tmp3 x10 +#define tmp4 x11 +#define zeroones x12 +#define pos x13 +#define limit_wd x14 + + .macro def_fn f p2align=0 + .text + .p2align \p2align + .global \f + .type \f, %function +\f: + .endm + +#define REP8_01 0x0101010101010101 +#define REP8_7f 0x7f7f7f7f7f7f7f7f +#define REP8_80 0x8080808080808080 + + .text + .p2align 6 +.Lstart: + /* Pre-pad to ensure critical loop begins an icache line. */ + .rep 7 + nop + .endr + /* Put this code here to avoid wasting more space with pre-padding. */ +.Lhit_limit: + mov len, limit + ret + +def_fn strnlen + cbz limit, .Lhit_limit + mov zeroones, #REP8_01 + bic src, srcin, #15 + ands tmp1, srcin, #15 + b.ne .Lmisaligned + /* Calculate the number of full and partial words -1. */ + sub limit_wd, limit, #1 /* Limit != 0, so no underflow. */ + lsr limit_wd, limit_wd, #4 /* Convert to Qwords. */ + + /* NUL detection works on the principle that (X - 1) & (~X) & 0x80 + (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and + can be done in parallel across the entire word. */ + /* The inner loop deals with two Dwords at a time. This has a + slightly higher start-up cost, but we should win quite quickly, + especially on cores with a high number of issue slots per + cycle, as we get much better parallelism out of the operations. */ + + /* Start of critial section -- keep to one 64Byte cache line. */ +.Lloop: + ldp data1, data2, [src], #16 +.Lrealigned: + sub tmp1, data1, zeroones + orr tmp2, data1, #REP8_7f + sub tmp3, data2, zeroones + orr tmp4, data2, #REP8_7f + bic has_nul1, tmp1, tmp2 + bic has_nul2, tmp3, tmp4 + subs limit_wd, limit_wd, #1 + orr tmp1, has_nul1, has_nul2 + ccmp tmp1, #0, #0, pl /* NZCV = 0000 */ + b.eq .Lloop + /* End of critical section -- keep to one 64Byte cache line. */ + + orr tmp1, has_nul1, has_nul2 + cbz tmp1, .Lhit_limit /* No null in final Qword. */ + + /* We know there's a null in the final Qword. The easiest thing + to do now is work out the length of the string and return + MIN (len, limit). */ + + sub len, src, srcin + cbz has_nul1, .Lnul_in_data2 +#ifdef __AARCH64EB__ + mov data2, data1 +#endif + sub len, len, #8 + mov has_nul2, has_nul1 +.Lnul_in_data2: +#ifdef __AARCH64EB__ Review Comment: `#if BYTE_ORDER == BIG_ENDIAN` ? ########## libs/libc/machine/arm64/gnu/arch_strcpy.S: ########## @@ -0,0 +1,338 @@ +/**************************************************************************** + * libs/libc/machine/arm64/gnu/arch_strcpy.S + * + * Copyright (c) 2013, 2014, 2015 ARM Ltd. + * All rights Reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the company nor the names of its contributors + * may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ****************************************************************************/ + +/* Assumptions: + * + * ARMv8-a, AArch64, unaligned accesses, min page size 4k. + */ + +/* To build as stpcpy, define BUILD_STPCPY before compiling this file. + + To test the page crossing code path more thoroughly, compile with + -DSTRCPY_TEST_PAGE_CROSS - this will force all copies through the slower + entry path. This option is not intended for production use. */ + +/* Arguments and results. */ +#define dstin x0 +#define srcin x1 + +/* Locals and temporaries. */ +#define src x2 +#define dst x3 +#define data1 x4 +#define data1w w4 +#define data2 x5 +#define data2w w5 +#define has_nul1 x6 +#define has_nul2 x7 +#define tmp1 x8 +#define tmp2 x9 +#define tmp3 x10 +#define tmp4 x11 +#define zeroones x12 +#define data1a x13 +#define data2a x14 +#define pos x15 +#define len x16 +#define to_align x17 + +#ifdef BUILD_STPCPY +#define STRCPY stpcpy +#else +#define STRCPY strcpy +#endif + + .macro def_fn f p2align=0 + .text + .p2align \p2align + .global \f + .type \f, %function +\f: + .endm + + /* NUL detection works on the principle that (X - 1) & (~X) & 0x80 + (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and + can be done in parallel across the entire word. */ + +#define REP8_01 0x0101010101010101 +#define REP8_7f 0x7f7f7f7f7f7f7f7f +#define REP8_80 0x8080808080808080 + + /* AArch64 systems have a minimum page size of 4k. We can do a quick + page size check for crossing this boundary on entry and if we + do not, then we can short-circuit much of the entry code. We + expect early page-crossing strings to be rare (probability of + 16/MIN_PAGE_SIZE ~= 0.4%), so the branch should be quite + predictable, even with random strings. + + We don't bother checking for larger page sizes, the cost of setting + up the correct page size is just not worth the extra gain from + a small reduction in the cases taking the slow path. Note that + we only care about whether the first fetch, which may be + misaligned, crosses a page boundary - after that we move to aligned + fetches for the remainder of the string. */ + +#ifdef STRCPY_TEST_PAGE_CROSS + /* Make everything that isn't Qword aligned look like a page cross. */ +#define MIN_PAGE_P2 4 +#else +#define MIN_PAGE_P2 12 +#endif + +#define MIN_PAGE_SIZE (1 << MIN_PAGE_P2) + +def_fn STRCPY p2align=6 + /* For moderately short strings, the fastest way to do the copy is to + calculate the length of the string in the same way as strlen, then + essentially do a memcpy of the result. This avoids the need for + multiple byte copies and further means that by the time we + reach the bulk copy loop we know we can always use DWord + accesses. We expect strcpy to rarely be called repeatedly + with the same source string, so branch prediction is likely to + always be difficult - we mitigate against this by preferring + conditional select operations over branches whenever this is + feasible. */ + and tmp2, srcin, #(MIN_PAGE_SIZE - 1) + mov zeroones, #REP8_01 + and to_align, srcin, #15 + cmp tmp2, #(MIN_PAGE_SIZE - 16) + neg tmp1, to_align + /* The first fetch will straddle a (possible) page boundary iff + srcin + 15 causes bit[MIN_PAGE_P2] to change value. A 16-byte + aligned string will never fail the page align check, so will + always take the fast path. */ + b.gt .Lpage_cross + +.Lpage_cross_ok: + ldp data1, data2, [srcin] +#ifdef __AARCH64EB__ Review Comment: `#if BYTE_ORDER == BIG_ENDIAN` ? ########## libs/libc/machine/arm64/gnu/arch_setjmp.S: ########## @@ -0,0 +1,77 @@ +/**************************************************************************** + * libs/libc/machine/arm64/gnu/arch_setjmp.S + * + * Copyright (c) 2011, 2012 ARM Ltd + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the company may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ****************************************************************************/ + +#define GPR_LAYOUT \ + REG_PAIR (x19, x20, 0); \ + REG_PAIR (x21, x22, 16); \ + REG_PAIR (x23, x24, 32); \ + REG_PAIR (x25, x26, 48); \ + REG_PAIR (x27, x28, 64); \ + REG_PAIR (x29, x30, 80); \ + REG_ONE (x16, 96) Review Comment: ```suggestion REG_ONE (x16, 96) ``` ########## libs/libc/machine/arm/armv8-m/gnu/arch_memcpy.S: ########## @@ -0,0 +1,345 @@ +/**************************************************************************** + * libs/libc/machine/arm/armv8-m/gnu/arch_memcpy.S + * + * Copyright (c) 2013 ARM Ltd + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the company may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ****************************************************************************/ + +#ifndef __ARM_FEATURE_MVE +/* This memcpy routine is optimised for Cortex-M3/M4 cores with/without + unaligned access. + + If compiled with GCC, this file should be enclosed within following + pre-processing check: + if defined (__ARM_ARCH_7M__) || defined (__ARM_ARCH_7EM__) + + Prototype: void *memcpy (void *dst, const void *src, size_t count); + + The job will be done in 5 steps. + Step 1: Align src/dest pointers, copy mis-aligned if fail to align both + Step 2: Repeatedly copy big block size of __OPT_BIG_BLOCK_SIZE + Step 3: Repeatedly copy big block size of __OPT_MID_BLOCK_SIZE + Step 4: Copy word by word + Step 5: Copy byte-to-byte + + Tunable options: + __OPT_BIG_BLOCK_SIZE: Size of big block in words. Default to 64. + __OPT_MID_BLOCK_SIZE: Size of big block in words. Default to 16. + */ +#ifndef __OPT_BIG_BLOCK_SIZE +#define __OPT_BIG_BLOCK_SIZE (4 * 16) +#endif + +#ifndef __OPT_MID_BLOCK_SIZE +#define __OPT_MID_BLOCK_SIZE (4 * 4) +#endif + +#if __OPT_BIG_BLOCK_SIZE == 16 +#define BEGIN_UNROLL_BIG_BLOCK \ + .irp offset, 0,4,8,12 +#elif __OPT_BIG_BLOCK_SIZE == 32 +#define BEGIN_UNROLL_BIG_BLOCK \ + .irp offset, 0,4,8,12,16,20,24,28 +#elif __OPT_BIG_BLOCK_SIZE == 64 +#define BEGIN_UNROLL_BIG_BLOCK \ + .irp offset, 0,4,8,12,16,20,24,28,32,36,40,44,48,52,56,60 +#else +#error "Illegal __OPT_BIG_BLOCK_SIZE" +#endif + +#if __OPT_MID_BLOCK_SIZE == 8 +#define BEGIN_UNROLL_MID_BLOCK \ + .irp offset, 0,4 +#elif __OPT_MID_BLOCK_SIZE == 16 +#define BEGIN_UNROLL_MID_BLOCK \ + .irp offset, 0,4,8,12 +#else +#error "Illegal __OPT_MID_BLOCK_SIZE" +#endif + +#define END_UNROLL .endr +#endif + + .syntax unified + .text + .align 2 + .global memcpy + .thumb + .thumb_func + .type memcpy, %function +memcpy: + @ r0: dst + @ r1: src + @ r2: len +#ifdef __ARM_FEATURE_MVE + mov r3, lr + wlstp.8 lr, r2, 2f + mov r2, r0 +1: + vldrb.8 q0, [r1], #16 + vstrb.8 q0, [r2], #16 + letp lr, 1b +2: + bx r3 +#else +#ifdef __ARM_FEATURE_UNALIGNED + /* In case of UNALIGNED access supported, ip is not used in + function body. */ + mov ip, r0 +#else + push {r0} +#endif + orr r3, r1, r0 + ands r3, r3, #3 + bne .Lmisaligned_copy + +.Lbig_block: + subs r2, __OPT_BIG_BLOCK_SIZE + blo .Lmid_block + + /* Kernel loop for big block copy */ + .align 2 +.Lbig_block_loop: + BEGIN_UNROLL_BIG_BLOCK +#ifdef __ARM_ARCH_7EM__ + ldr r3, [r1], #4 + str r3, [r0], #4 + END_UNROLL +#else /* __ARM_ARCH_7M__ */ + ldr r3, [r1, \offset] + str r3, [r0, \offset] + END_UNROLL + adds r0, __OPT_BIG_BLOCK_SIZE + adds r1, __OPT_BIG_BLOCK_SIZE +#endif + subs r2, __OPT_BIG_BLOCK_SIZE + bhs .Lbig_block_loop + +.Lmid_block: + adds r2, __OPT_BIG_BLOCK_SIZE - __OPT_MID_BLOCK_SIZE + blo .Lcopy_word_by_word + + /* Kernel loop for mid-block copy */ + .align 2 +.Lmid_block_loop: + BEGIN_UNROLL_MID_BLOCK +#ifdef __ARM_ARCH_7EM__ + ldr r3, [r1], #4 + str r3, [r0], #4 + END_UNROLL +#else /* __ARM_ARCH_7M__ */ + ldr r3, [r1, \offset] + str r3, [r0, \offset] + END_UNROLL + adds r0, __OPT_MID_BLOCK_SIZE + adds r1, __OPT_MID_BLOCK_SIZE +#endif + subs r2, __OPT_MID_BLOCK_SIZE + bhs .Lmid_block_loop + +.Lcopy_word_by_word: + adds r2, __OPT_MID_BLOCK_SIZE - 4 + blo .Lcopy_less_than_4 + + /* Kernel loop for small block copy */ + .align 2 +.Lcopy_word_by_word_loop: + ldr r3, [r1], #4 + str r3, [r0], #4 + subs r2, #4 + bhs .Lcopy_word_by_word_loop + +.Lcopy_less_than_4: + adds r2, #4 + beq .Ldone + + lsls r2, r2, #31 + itt ne + ldrbne r3, [r1], #1 + strbne r3, [r0], #1 + + bcc .Ldone +#ifdef __ARM_FEATURE_UNALIGNED + ldrh r3, [r1] + strh r3, [r0] +#else + ldrb r3, [r1] + strb r3, [r0] + ldrb r3, [r1, #1] + strb r3, [r0, #1] +#endif /* __ARM_FEATURE_UNALIGNED */ + +.Ldone: +#ifdef __ARM_FEATURE_UNALIGNED + mov r0, ip +#else + pop {r0} +#endif + bx lr + + .align 2 +.Lmisaligned_copy: +#ifdef __ARM_FEATURE_UNALIGNED + /* Define label DST_ALIGNED to BIG_BLOCK. It will go to aligned copy + once destination is adjusted to aligned. */ +#define Ldst_aligned Lbig_block + + /* Copy word by word using LDR when alignment can be done in hardware, + i.e., SCTLR.A is set, supporting unaligned access in LDR and STR. */ + + cmp r2, #8 + blo .Lbyte_copy + + /* if src is aligned, just go to the big block loop. */ + lsls r3, r1, #30 + beq .Ldst_aligned +#else + /* if len < 12, misalignment adjustment has more overhead than + just byte-to-byte copy. Also, len must >=8 to guarantee code + afterward work correctly. */ + cmp r2, #12 + blo .Lbyte_copy +#endif /* __ARM_FEATURE_UNALIGNED */ + + /* Align dst only, not trying to align src. That is the because + handling of aligned src and misaligned dst need more overhead than + otherwise. By doing this the worst case is when initial src is aligned, + additional up to 4 byte additional copy will executed, which is + acceptable. */ + + ands r3, r0, #3 + beq .Ldst_aligned + + rsb r3, #4 + subs r2, r3 + + lsls r3, r3, #31 + itt ne + ldrbne r3, [r1], #1 + strbne r3, [r0], #1 + + bcc .Ldst_aligned + +#ifdef __ARM_FEATURE_UNALIGNED + ldrh r3, [r1], #2 + strh r3, [r0], #2 + b .Ldst_aligned +#else + ldrb r3, [r1], #1 + strb r3, [r0], #1 + ldrb r3, [r1], #1 + strb r3, [r0], #1 + /* Now that dst is aligned */ +.Ldst_aligned: + /* if r1 is aligned now, it means r0/r1 has the same misalignment, + and they are both aligned now. Go aligned copy. */ + ands r3, r1, #3 + beq .Lbig_block + + /* dst is aligned, but src isn't. Misaligned copy. */ + + push {r4, r5} + subs r2, #4 + + /* Backward r1 by misaligned bytes, to make r1 aligned. + Since we need to restore r1 to unaligned address after the loop, + we need keep the offset bytes to ip and sub it from r1 afterward. */ + subs r1, r3 + rsb ip, r3, #4 + + /* Pre-load on word */ + ldr r4, [r1], #4 + + cmp r3, #2 + beq .Lmisaligned_copy_2_2 + cmp r3, #3 + beq .Lmisaligned_copy_3_1 + + .macro mis_src_copy shift +1: +#ifdef __ARM_BIG_ENDIAN Review Comment: `#if BYTE_ORDER == BIG_ENDIAN` ? ########## libs/libc/machine/arm/armv8-m/gnu/arch_memcpy.S: ########## @@ -0,0 +1,345 @@ +/**************************************************************************** + * libs/libc/machine/arm/armv8-m/gnu/arch_memcpy.S + * + * Copyright (c) 2013 ARM Ltd + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the company may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ****************************************************************************/ + +#ifndef __ARM_FEATURE_MVE +/* This memcpy routine is optimised for Cortex-M3/M4 cores with/without + unaligned access. + + If compiled with GCC, this file should be enclosed within following + pre-processing check: + if defined (__ARM_ARCH_7M__) || defined (__ARM_ARCH_7EM__) + + Prototype: void *memcpy (void *dst, const void *src, size_t count); + + The job will be done in 5 steps. + Step 1: Align src/dest pointers, copy mis-aligned if fail to align both + Step 2: Repeatedly copy big block size of __OPT_BIG_BLOCK_SIZE + Step 3: Repeatedly copy big block size of __OPT_MID_BLOCK_SIZE + Step 4: Copy word by word + Step 5: Copy byte-to-byte + + Tunable options: + __OPT_BIG_BLOCK_SIZE: Size of big block in words. Default to 64. + __OPT_MID_BLOCK_SIZE: Size of big block in words. Default to 16. + */ +#ifndef __OPT_BIG_BLOCK_SIZE +#define __OPT_BIG_BLOCK_SIZE (4 * 16) +#endif + +#ifndef __OPT_MID_BLOCK_SIZE +#define __OPT_MID_BLOCK_SIZE (4 * 4) +#endif + +#if __OPT_BIG_BLOCK_SIZE == 16 +#define BEGIN_UNROLL_BIG_BLOCK \ + .irp offset, 0,4,8,12 +#elif __OPT_BIG_BLOCK_SIZE == 32 +#define BEGIN_UNROLL_BIG_BLOCK \ + .irp offset, 0,4,8,12,16,20,24,28 +#elif __OPT_BIG_BLOCK_SIZE == 64 +#define BEGIN_UNROLL_BIG_BLOCK \ + .irp offset, 0,4,8,12,16,20,24,28,32,36,40,44,48,52,56,60 +#else +#error "Illegal __OPT_BIG_BLOCK_SIZE" +#endif + +#if __OPT_MID_BLOCK_SIZE == 8 +#define BEGIN_UNROLL_MID_BLOCK \ + .irp offset, 0,4 +#elif __OPT_MID_BLOCK_SIZE == 16 +#define BEGIN_UNROLL_MID_BLOCK \ + .irp offset, 0,4,8,12 +#else +#error "Illegal __OPT_MID_BLOCK_SIZE" +#endif + +#define END_UNROLL .endr +#endif + + .syntax unified + .text + .align 2 + .global memcpy + .thumb + .thumb_func + .type memcpy, %function +memcpy: + @ r0: dst + @ r1: src + @ r2: len +#ifdef __ARM_FEATURE_MVE + mov r3, lr + wlstp.8 lr, r2, 2f + mov r2, r0 +1: + vldrb.8 q0, [r1], #16 + vstrb.8 q0, [r2], #16 + letp lr, 1b +2: + bx r3 +#else +#ifdef __ARM_FEATURE_UNALIGNED + /* In case of UNALIGNED access supported, ip is not used in + function body. */ + mov ip, r0 +#else + push {r0} +#endif + orr r3, r1, r0 + ands r3, r3, #3 + bne .Lmisaligned_copy + +.Lbig_block: + subs r2, __OPT_BIG_BLOCK_SIZE + blo .Lmid_block + + /* Kernel loop for big block copy */ + .align 2 +.Lbig_block_loop: + BEGIN_UNROLL_BIG_BLOCK +#ifdef __ARM_ARCH_7EM__ Review Comment: is this relevant for `armv8-m`? ########## libs/libc/machine/arm/armv8-m/gnu/acle-compat.h: ########## @@ -0,0 +1,186 @@ +/**************************************************************************** + * libs/libc/machine/arm/armv8-m/gnu/acle-compat.h + * + * Copyright (c) 2014 ARM Ltd + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the company may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + ****************************************************************************/ + +#ifndef __ARM_ARCH + +/* ACLE standardises a set of pre-defines that describe the ARM architecture. + * These were mostly implemented in GCC around GCC-4.8; older versions + * have no, or only partial support. To provide a level of backwards + * compatibility we try to work out what the definitions should be, given + * the older pre-defines that GCC did produce. This isn't complete, but + * it should be enough for use by routines that depend on this header. + */ + +/* No need to handle ARMv8, GCC had ACLE support before that. */ + +# ifdef __ARM_ARCH_7__ Review Comment: I'm not sure if this file is relevant for `armv8-m` ########## libs/libc/machine/arm/armv8-m/gnu/acle-compat.h: ########## @@ -0,0 +1,186 @@ +/**************************************************************************** + * libs/libc/machine/arm/armv8-m/gnu/acle-compat.h + * + * Copyright (c) 2014 ARM Ltd + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the company may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + ****************************************************************************/ + +#ifndef __ARM_ARCH + +/* ACLE standardises a set of pre-defines that describe the ARM architecture. + * These were mostly implemented in GCC around GCC-4.8; older versions + * have no, or only partial support. To provide a level of backwards + * compatibility we try to work out what the definitions should be, given + * the older pre-defines that GCC did produce. This isn't complete, but + * it should be enough for use by routines that depend on this header. + */ + +/* No need to handle ARMv8, GCC had ACLE support before that. */ + +# ifdef __ARM_ARCH_7__ +/* The common subset of ARMv7 in all profiles. */ +# define __ARM_ARCH 7 +# define __ARM_ARCH_ISA_THUMB 2 +# define __ARM_FEATURE_CLZ +# define __ARM_FEATURE_LDREX 7 +# define __ARM_FEATURE_UNALIGNED +# endif + +# if defined (__ARM_ARCH_7A__) || defined (__ARM_ARCH_7R__) +# define __ARM_ARCH 7 +# define __ARM_ARCH_ISA_THUMB 2 +# define __ARM_ARCH_ISA_ARM +# define __ARM_FEATURE_CLZ +# define __ARM_FEATURE_SIMD32 +# define __ARM_FEATURE_DSP +# define __ARM_FEATURE_QBIT +# define __ARM_FEATURE_SAT +# define __ARM_FEATURE_LDREX 15 +# define __ARM_FEATURE_UNALIGNED +# ifdef __ARM_ARCH_7A__ +# define __ARM_ARCH_PROFILE 'A' +# else +# define __ARM_ARCH_PROFILE 'R' +# endif +# endif + +# ifdef __ARM_ARCH_7EM__ +# define __ARM_ARCH 7 +# define __ARM_ARCH_ISA_THUMB 2 +# define __ARM_FEATURE_CLZ +# define __ARM_FEATURE_SIMD32 +# define __ARM_FEATURE_DSP +# define __ARM_FEATURE_QBIT +# define __ARM_FEATURE_SAT +# define __ARM_FEATURE_LDREX 7 +# define __ARM_FEATURE_UNALIGNED +# define __ARM_ARCH_PROFILE 'M' +# endif + +# ifdef __ARM_ARCH_7M__ +# define __ARM_ARCH 7 +# define __ARM_ARCH_ISA_THUMB 2 +# define __ARM_FEATURE_CLZ +# define __ARM_FEATURE_QBIT +# define __ARM_FEATURE_SAT +# define __ARM_FEATURE_LDREX 7 +# define __ARM_FEATURE_UNALIGNED +# define __ARM_ARCH_PROFILE 'M' +# endif + +# ifdef __ARM_ARCH_6T2__ +# define __ARM_ARCH 6 +# define __ARM_ARCH_ISA_THUMB 2 +# define __ARM_ARCH_ISA_ARM +# define __ARM_FEATURE_CLZ +# define __ARM_FEATURE_SIMD32 +# define __ARM_FEATURE_DSP +# define __ARM_FEATURE_QBIT +# define __ARM_FEATURE_SAT +# define __ARM_FEATURE_LDREX 4 +# define __ARM_FEATURE_UNALIGNED +# endif + +# ifdef __ARM_ARCH_6M__ +# define __ARM_ARCH 6 +# define __ARM_ARCH_ISA_THUMB 1 +# define __ARM_ARCH_PROFILE 'M' +# endif + +# if defined (__ARM_ARCH_6__) || defined (__ARM_ARCH_6J__) \ + || defined (__ARM_ARCH_6K__) || defined (__ARM_ARCH_6Z__) \ + || defined (__ARM_ARCH_6ZK__) +# define __ARM_ARCH 6 +# define __ARM_ARCH_ISA_THUMB 1 +# define __ARM_ARCH_ISA_ARM +# define __ARM_FEATURE_CLZ +# define __ARM_FEATURE_SIMD32 +# define __ARM_FEATURE_DSP +# define __ARM_FEATURE_QBIT +# define __ARM_FEATURE_SAT +# define __ARM_FEATURE_UNALIGNED +# ifndef __thumb__ +# if defined (__ARM_ARCH_6K__) || defined (__ARM_ARCH_6ZK__) +# define __ARM_FEATURE_LDREX 15 +# else +# define __ARM_FEATURE_LDREX 4 +# endif +# endif +# endif + +# if defined (__ARM_ARCH_5TE__) || defined (__ARM_ARCH_5E__) +# define __ARM_ARCH 5 +# define __ARM_ARCH_ISA_ARM +# ifdef __ARM_ARCH_5TE__ +# define __ARM_ARCH_ISA_THUMB 1 +# endif +# define __ARM_FEATURE_CLZ +# define __ARM_FEATURE_DSP +# endif + +# if defined (__ARM_ARCH_5T__) || defined (__ARM_ARCH_5__) +# define __ARM_ARCH 5 +# define __ARM_ARCH_ISA_ARM +# ifdef __ARM_ARCH_5TE__ +# define __ARM_ARCH_ISA_THUMB 1 +# endif +# define __ARM_FEATURE_CLZ +# endif + +# ifdef __ARM_ARCH_4T__ +# define __ARM_ARCH 4 +# define __ARM_ARCH_ISA_ARM +# define __ARM_ARCH_ISA_THUMB 1 +# endif + +# ifdef __ARM_ARCH_4__ +# define __ARM_ARCH 4 +# define __ARM_ARCH_ISA_ARM +# endif + +# if defined (__ARM_ARCH_3__) || defined (__ARM_ARCH_3M__) +# define __ARM_ARCH 3 +# define __ARM_ARCH_ISA_ARM +# endif + +# ifdef __ARM_ARCH_2__ +# define __ARM_ARCH 2 +# define __ARM_ARCH_ISA_ARM +# endif + +# ifdef __ARMEB__ +# define __ARM_BIG_ENDIAN +# endif + +/* If we still don't know what the target architecture is, then we're + * probably not using GCC. + */ + +# ifndef __ARM_ARCH +# error Unable to determine architecture version. +# endif + +#endif /* __ARM_ARCH */ Review Comment: ```suggestion #endif /* __LIBS_LIBC_MACHINE_ARM_ARMV8_M_GNU_ACLE_COMPAT_H */ ``` ########## libs/libc/machine/risc-v/gnu/asm.h: ########## @@ -0,0 +1,52 @@ +/**************************************************************************** + * libs/libc/machine/risc-v/gnu/asm.h + * + * Copyright (c) 2017 SiFive Inc. All rights reserved. + * + * This copyrighted material is made available to anyone wishing to use, + * modify, copy, or redistribute it subject to the terms and conditions + * of the FreeBSD License. This program is distributed in the hope that + * it will be useful, but WITHOUT ANY WARRANTY expressed or implied, + * including the implied warranties of MERCHANTABILITY or FITNESS FOR + * A PARTICULAR PURPOSE. A copy of this license is available at + * http://www.opensource.org/licenses. + ****************************************************************************/ + +#ifndef _ASM_H +#define _ASM_H + +/**************************************************************************** + * Included Files + ****************************************************************************/ + +#include <nuttx/config.h> + +/**************************************************************************** + * Pre-processor Definitions + ****************************************************************************/ + +#ifdef CONFIG_ARCH_RV64 +# define SZREG 8 +# define REG_S sd +# define REG_L ld +#elif defined(CONFIG_ARCH_RV32) +# define SZREG 4 +# define REG_S sw +# define REG_L lw +#endif + +#ifdef CONFIG_ARCH_QPFPU Review Comment: ```suggestion #if defined(CONFIG_ARCH_QPFPU) ``` ########## libs/libc/machine/arm64/gnu/arch_strncmp.S: ########## @@ -0,0 +1,294 @@ +/**************************************************************************** + * libs/libc/machine/arm64/gnu/arch_strncmp.S + * + * Copyright (c) 2013, 2018, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the Linaro nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ****************************************************************************/ + +/* Assumptions: + * + * ARMv8-a, AArch64 + */ + + .macro def_fn f p2align=0 + .text + .p2align \p2align + .global \f + .type \f, %function +\f: + .endm + +#define REP8_01 0x0101010101010101 +#define REP8_7f 0x7f7f7f7f7f7f7f7f +#define REP8_80 0x8080808080808080 + +/* Parameters and result. */ +#define src1 x0 +#define src2 x1 +#define limit x2 +#define result x0 + +/* Internal variables. */ +#define data1 x3 +#define data1w w3 +#define data2 x4 +#define data2w w4 +#define has_nul x5 +#define diff x6 +#define syndrome x7 +#define tmp1 x8 +#define tmp2 x9 +#define tmp3 x10 +#define zeroones x11 +#define pos x12 +#define limit_wd x13 +#define mask x14 +#define endloop x15 +#define count mask + + .text + .p2align 6 + .rep 7 + nop /* Pad so that the loop below fits a cache line. */ + .endr +def_fn strncmp + cbz limit, .Lret0 + eor tmp1, src1, src2 + mov zeroones, #REP8_01 + tst tmp1, #7 + and count, src1, #7 + b.ne .Lmisaligned8 + cbnz count, .Lmutual_align + /* Calculate the number of full and partial words -1. */ + sub limit_wd, limit, #1 /* limit != 0, so no underflow. */ + lsr limit_wd, limit_wd, #3 /* Convert to Dwords. */ + + /* NUL detection works on the principle that (X - 1) & (~X) & 0x80 + (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and + can be done in parallel across the entire word. */ + /* Start of performance-critical section -- one 64B cache line. */ +.Lloop_aligned: + ldr data1, [src1], #8 + ldr data2, [src2], #8 +.Lstart_realigned: + subs limit_wd, limit_wd, #1 + sub tmp1, data1, zeroones + orr tmp2, data1, #REP8_7f + eor diff, data1, data2 /* Non-zero if differences found. */ + csinv endloop, diff, xzr, pl /* Last Dword or differences. */ + bics has_nul, tmp1, tmp2 /* Non-zero if NUL terminator. */ + ccmp endloop, #0, #0, eq + b.eq .Lloop_aligned + /* End of performance-critical section -- one 64B cache line. */ + + /* Not reached the limit, must have found the end or a diff. */ + tbz limit_wd, #63, .Lnot_limit + + /* Limit % 8 == 0 => all bytes significant. */ + ands limit, limit, #7 + b.eq .Lnot_limit + + lsl limit, limit, #3 /* Bits -> bytes. */ + mov mask, #~0 +#ifdef __AARCH64EB__ + lsr mask, mask, limit +#else + lsl mask, mask, limit +#endif + bic data1, data1, mask + bic data2, data2, mask + + /* Make sure that the NUL byte is marked in the syndrome. */ + orr has_nul, has_nul, mask + +.Lnot_limit: + orr syndrome, diff, has_nul + +#ifndef __AARCH64EB__ + rev syndrome, syndrome + rev data1, data1 + /* The MS-non-zero bit of the syndrome marks either the first bit + that is different, or the top bit of the first zero byte. + Shifting left now will bring the critical information into the + top bits. */ + clz pos, syndrome + rev data2, data2 + lsl data1, data1, pos + lsl data2, data2, pos + /* But we need to zero-extend (char is unsigned) the value and then + perform a signed 32-bit subtraction. */ + lsr data1, data1, #56 + sub result, data1, data2, lsr #56 + ret +#else + /* For big-endian we cannot use the trick with the syndrome value + as carry-propagation can corrupt the upper bits if the trailing + bytes in the string contain 0x01. */ + /* However, if there is no NUL byte in the dword, we can generate + the result directly. We can't just subtract the bytes as the + MSB might be significant. */ + cbnz has_nul, 1f + cmp data1, data2 + cset result, ne + cneg result, result, lo + ret +1: + /* Re-compute the NUL-byte detection, using a byte-reversed value. */ + rev tmp3, data1 + sub tmp1, tmp3, zeroones + orr tmp2, tmp3, #REP8_7f + bic has_nul, tmp1, tmp2 + rev has_nul, has_nul + orr syndrome, diff, has_nul + clz pos, syndrome + /* The MS-non-zero bit of the syndrome marks either the first bit + that is different, or the top bit of the first zero byte. + Shifting left now will bring the critical information into the + top bits. */ + lsl data1, data1, pos + lsl data2, data2, pos + /* But we need to zero-extend (char is unsigned) the value and then + perform a signed 32-bit subtraction. */ + lsr data1, data1, #56 + sub result, data1, data2, lsr #56 + ret +#endif + +.Lmutual_align: + /* Sources are mutually aligned, but are not currently at an + alignment boundary. Round down the addresses and then mask off + the bytes that precede the start point. + We also need to adjust the limit calculations, but without + overflowing if the limit is near ULONG_MAX. */ + bic src1, src1, #7 + bic src2, src2, #7 + ldr data1, [src1], #8 + neg tmp3, count, lsl #3 /* 64 - bits(bytes beyond align). */ + ldr data2, [src2], #8 + mov tmp2, #~0 + sub limit_wd, limit, #1 /* limit != 0, so no underflow. */ +#ifdef __AARCH64EB__ Review Comment: `#if BYTE_ORDER == BIG_ENDIAN` ? ########## libs/libc/machine/risc-v/gnu/arch_memset.S: ########## @@ -0,0 +1,102 @@ +/**************************************************************************** + * libs/libc/machine/risc-v/gnu/arch_memset.S + * + * Copyright (c) 2017 SiFive Inc. All rights reserved. + * + * This copyrighted material is made available to anyone wishing to use, + * modify, copy, or redistribute it subject to the terms and conditions + * of the FreeBSD License. This program is distributed in the hope that + * it will be useful, but WITHOUT ANY WARRANTY expressed or implied, + * including the implied warranties of MERCHANTABILITY or FITNESS FOR + * A PARTICULAR PURPOSE. A copy of this license is available at + * http://www.opensource.org/licenses. + * + ****************************************************************************/ + +.text +.global memset +.type memset, @function +memset: + li t1, 15 + move a4, a0 + bleu a2, t1, .Ltiny + and a5, a4, 15 + bnez a5, .Lmisaligned + +.Laligned: + bnez a1, .Lwordify + +.Lwordified: + and a3, a2, ~15 + and a2, a2, 15 + add a3, a3, a4 + +#if __riscv_xlen == 64 +1:sd a1, 0(a4) Review Comment: Please add newline after the label ########## libs/libc/machine/risc-v/gnu/asm.h: ########## @@ -0,0 +1,52 @@ +/**************************************************************************** + * libs/libc/machine/risc-v/gnu/asm.h + * + * Copyright (c) 2017 SiFive Inc. All rights reserved. + * + * This copyrighted material is made available to anyone wishing to use, + * modify, copy, or redistribute it subject to the terms and conditions + * of the FreeBSD License. This program is distributed in the hope that + * it will be useful, but WITHOUT ANY WARRANTY expressed or implied, + * including the implied warranties of MERCHANTABILITY or FITNESS FOR + * A PARTICULAR PURPOSE. A copy of this license is available at + * http://www.opensource.org/licenses. + ****************************************************************************/ + +#ifndef _ASM_H +#define _ASM_H + +/**************************************************************************** + * Included Files + ****************************************************************************/ + +#include <nuttx/config.h> + +/**************************************************************************** + * Pre-processor Definitions + ****************************************************************************/ + +#ifdef CONFIG_ARCH_RV64 +# define SZREG 8 +# define REG_S sd +# define REG_L ld +#elif defined(CONFIG_ARCH_RV32) Review Comment: ```suggestion #else ``` ########## libs/libc/machine/risc-v/gnu/arch_strcmp.S: ########## @@ -0,0 +1,184 @@ +/**************************************************************************** + * libs/libc/machine/risc-v/gnu/arch_strcmp.S + * + * Copyright (c) 2017 SiFive Inc. All rights reserved. + * + * This copyrighted material is made available to anyone wishing to use, + * modify, copy, or redistribute it subject to the terms and conditions + * of the FreeBSD License. This program is distributed in the hope that + * it will be useful, but WITHOUT ANY WARRANTY expressed or implied, + * including the implied warranties of MERCHANTABILITY or FITNESS FOR + * A PARTICULAR PURPOSE. A copy of this license is available at + * http://www.opensource.org/licenses. + * + ****************************************************************************/ + +#include "asm.h" + +.text +.globl strcmp +.type strcmp, @function +strcmp: + or a4, a0, a1 + li t2, -1 + and a4, a4, SZREG-1 + bnez a4, .Lmisaligned + +#if SZREG == 4 + li a5, 0x7f7f7f7f +#else + ld a5, mask +#endif + + .macro check_one_word i n + REG_L a2, \i*SZREG(a0) + REG_L a3, \i*SZREG(a1) + + and t0, a2, a5 + or t1, a2, a5 + add t0, t0, a5 + or t0, t0, t1 + + bne t0, t2, .Lnull\i + .if \i+1-\n + bne a2, a3, .Lmismatch + .else + add a0, a0, \n*SZREG + add a1, a1, \n*SZREG + beq a2, a3, .Lloop + # fall through to .Lmismatch + .endif + .endm + + .macro foundnull i n + .ifne \i + .Lnull\i: + add a0, a0, \i*SZREG + add a1, a1, \i*SZREG + .ifeq \i-1 + .Lnull0: + .endif + bne a2, a3, .Lmisaligned + li a0, 0 + ret + .endif + .endm + +.Lloop: + # examine full words at a time, favoring strings of a couple dozen chars +#if __riscv_xlen == 32 + check_one_word 0 5 + check_one_word 1 5 + check_one_word 2 5 + check_one_word 3 5 + check_one_word 4 5 +#else + check_one_word 0 3 + check_one_word 1 3 + check_one_word 2 3 +#endif + # backwards branch to .Lloop contained above + +.Lmismatch: + # words don't match, but a2 has no null byte. + +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + +#if __riscv_xlen == 64 + sll a4, a2, 48 + sll a5, a3, 48 + bne a4, a5, .Lmismatch_upper + sll a4, a2, 32 + sll a5, a3, 32 + bne a4, a5, .Lmismatch_upper +#endif + sll a4, a2, 16 + sll a5, a3, 16 + bne a4, a5, .Lmismatch_upper + + srl a4, a2, 8*SZREG-16 + srl a5, a3, 8*SZREG-16 + sub a0, a4, a5 + and a1, a0, 0xff + bnez a1, 1f + ret + +.Lmismatch_upper: + srl a4, a4, 8*SZREG-16 + srl a5, a5, 8*SZREG-16 + sub a0, a4, a5 + and a1, a0, 0xff + bnez a1, 1f + ret + +1:and a4, a4, 0xff + and a5, a5, 0xff + sub a0, a4, a5 + ret + +#else + +#if __riscv_xlen == 64 + srl a4, a2, 48 + srl a5, a3, 48 + bne a4, a5, .Lmismatch_lower + srl a4, a2, 32 + srl a5, a3, 32 + bne a4, a5, .Lmismatch_lower +#endif + srl a4, a2, 16 + srl a5, a3, 16 + bne a4, a5, .Lmismatch_lower + + srl a4, a2, 8 + srl a5, a3, 8 + bne a4, a5, 1f + and a4, a2, 0xff + and a5, a3, 0xff +1:sub a0, a4, a5 + ret + +.Lmismatch_lower: + srl a2, a4, 8 + srl a3, a5, 8 Review Comment: ```suggestion srl a2, a4, 8 srl a3, a5, 8 ``` ########## libs/libc/machine/arm64/gnu/arch_strlen.S: ########## @@ -0,0 +1,242 @@ +/**************************************************************************** + * libs/libc/machine/arm64/gnu/arch_strlen.S + * + * Copyright (c) 2013-2015, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the Linaro nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ****************************************************************************/ + +/* Assumptions: + * + * ARMv8-a, AArch64, unaligned accesses, min page size 4k. + */ + +/* To test the page crossing code path more thoroughly, compile with + -DTEST_PAGE_CROSS - this will force all calls through the slower + entry path. This option is not intended for production use. */ + +/* Arguments and results. */ +#define srcin x0 +#define len x0 + +/* Locals and temporaries. */ +#define src x1 +#define data1 x2 +#define data2 x3 +#define has_nul1 x4 +#define has_nul2 x5 +#define tmp1 x4 +#define tmp2 x5 +#define tmp3 x6 +#define tmp4 x7 +#define zeroones x8 + +#define L(l) .L ## l + + .macro def_fn f p2align=0 + .text + .p2align \p2align + .global \f + .type \f, %function +\f: + .endm + + /* NUL detection works on the principle that (X - 1) & (~X) & 0x80 + (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and + can be done in parallel across the entire word. A faster check + (X - 1) & 0x80 is zero for non-NUL ASCII characters, but gives + false hits for characters 129..255. */ + +#define REP8_01 0x0101010101010101 +#define REP8_7f 0x7f7f7f7f7f7f7f7f +#define REP8_80 0x8080808080808080 + +#ifdef TEST_PAGE_CROSS +# define MIN_PAGE_SIZE 15 +#else +# define MIN_PAGE_SIZE 4096 +#endif + + /* Since strings are short on average, we check the first 16 bytes + of the string for a NUL character. In order to do an unaligned ldp + safely we have to do a page cross check first. If there is a NUL + byte we calculate the length from the 2 8-byte words using + conditional select to reduce branch mispredictions (it is unlikely + strlen will be repeatedly called on strings with the same length). + + If the string is longer than 16 bytes, we align src so don't need + further page cross checks, and process 32 bytes per iteration + using the fast NUL check. If we encounter non-ASCII characters, + fallback to a second loop using the full NUL check. + + If the page cross check fails, we read 16 bytes from an aligned + address, remove any characters before the string, and continue + in the main loop using aligned loads. Since strings crossing a + page in the first 16 bytes are rare (probability of + 16/MIN_PAGE_SIZE ~= 0.4%), this case does not need to be optimized. + + AArch64 systems have a minimum page size of 4k. We don't bother + checking for larger page sizes - the cost of setting up the correct + page size is just not worth the extra gain from a small reduction in + the cases taking the slow path. Note that we only care about + whether the first fetch, which may be misaligned, crosses a page + boundary. */ + +def_fn strlen p2align=6 + and tmp1, srcin, MIN_PAGE_SIZE - 1 + mov zeroones, REP8_01 + cmp tmp1, MIN_PAGE_SIZE - 16 + b.gt L(page_cross) + ldp data1, data2, [srcin] +#ifdef __AARCH64EB__ + /* For big-endian, carry propagation (if the final byte in the + string is 0x01) means we cannot use has_nul1/2 directly. + Since we expect strings to be small and early-exit, + byte-swap the data now so has_null1/2 will be correct. */ + rev data1, data1 + rev data2, data2 +#endif + sub tmp1, data1, zeroones + orr tmp2, data1, REP8_7f + sub tmp3, data2, zeroones + orr tmp4, data2, REP8_7f + bics has_nul1, tmp1, tmp2 + bic has_nul2, tmp3, tmp4 + ccmp has_nul2, 0, 0, eq + beq L(main_loop_entry) + + /* Enter with C = has_nul1 == 0. */ + csel has_nul1, has_nul1, has_nul2, cc + mov len, 8 + rev has_nul1, has_nul1 + clz tmp1, has_nul1 + csel len, xzr, len, cc + add len, len, tmp1, lsr 3 + ret + + /* The inner loop processes 32 bytes per iteration and uses the fast + NUL check. If we encounter non-ASCII characters, use a second + loop with the accurate NUL check. */ + .p2align 4 +L(main_loop_entry): + bic src, srcin, 15 + sub src, src, 16 +L(main_loop): + ldp data1, data2, [src, 32]! +.Lpage_cross_entry: + sub tmp1, data1, zeroones + sub tmp3, data2, zeroones + orr tmp2, tmp1, tmp3 + tst tmp2, zeroones, lsl 7 + bne 1f + ldp data1, data2, [src, 16] + sub tmp1, data1, zeroones + sub tmp3, data2, zeroones + orr tmp2, tmp1, tmp3 + tst tmp2, zeroones, lsl 7 + beq L(main_loop) + add src, src, 16 +1: + /* The fast check failed, so do the slower, accurate NUL check. */ + orr tmp2, data1, REP8_7f + orr tmp4, data2, REP8_7f + bics has_nul1, tmp1, tmp2 + bic has_nul2, tmp3, tmp4 + ccmp has_nul2, 0, 0, eq + beq L(nonascii_loop) + + /* Enter with C = has_nul1 == 0. */ +L(tail): +#ifdef __AARCH64EB__ Review Comment: `#if BYTE_ORDER == BIG_ENDIAN` ? ########## libs/libc/machine/risc-v/gnu/arch_memset.S: ########## @@ -0,0 +1,102 @@ +/**************************************************************************** + * libs/libc/machine/risc-v/gnu/arch_memset.S + * + * Copyright (c) 2017 SiFive Inc. All rights reserved. + * + * This copyrighted material is made available to anyone wishing to use, + * modify, copy, or redistribute it subject to the terms and conditions + * of the FreeBSD License. This program is distributed in the hope that + * it will be useful, but WITHOUT ANY WARRANTY expressed or implied, + * including the implied warranties of MERCHANTABILITY or FITNESS FOR + * A PARTICULAR PURPOSE. A copy of this license is available at + * http://www.opensource.org/licenses. + * + ****************************************************************************/ + +.text +.global memset +.type memset, @function +memset: + li t1, 15 + move a4, a0 + bleu a2, t1, .Ltiny + and a5, a4, 15 + bnez a5, .Lmisaligned + +.Laligned: + bnez a1, .Lwordify + +.Lwordified: + and a3, a2, ~15 + and a2, a2, 15 + add a3, a3, a4 + +#if __riscv_xlen == 64 Review Comment: `CONFIG_ARCH_RV32`/`CONFIG_ARCH_RV64` instead of `__riscv_xlen` ########## libs/libc/machine/risc-v/gnu/arch_strcmp.S: ########## @@ -0,0 +1,184 @@ +/**************************************************************************** + * libs/libc/machine/risc-v/gnu/arch_strcmp.S + * + * Copyright (c) 2017 SiFive Inc. All rights reserved. + * + * This copyrighted material is made available to anyone wishing to use, + * modify, copy, or redistribute it subject to the terms and conditions + * of the FreeBSD License. This program is distributed in the hope that + * it will be useful, but WITHOUT ANY WARRANTY expressed or implied, + * including the implied warranties of MERCHANTABILITY or FITNESS FOR + * A PARTICULAR PURPOSE. A copy of this license is available at + * http://www.opensource.org/licenses. + * + ****************************************************************************/ + +#include "asm.h" + +.text +.globl strcmp +.type strcmp, @function +strcmp: + or a4, a0, a1 + li t2, -1 + and a4, a4, SZREG-1 + bnez a4, .Lmisaligned + +#if SZREG == 4 + li a5, 0x7f7f7f7f +#else + ld a5, mask +#endif + + .macro check_one_word i n + REG_L a2, \i*SZREG(a0) + REG_L a3, \i*SZREG(a1) + + and t0, a2, a5 + or t1, a2, a5 + add t0, t0, a5 + or t0, t0, t1 + + bne t0, t2, .Lnull\i + .if \i+1-\n + bne a2, a3, .Lmismatch + .else + add a0, a0, \n*SZREG + add a1, a1, \n*SZREG + beq a2, a3, .Lloop + # fall through to .Lmismatch + .endif + .endm + + .macro foundnull i n + .ifne \i + .Lnull\i: + add a0, a0, \i*SZREG + add a1, a1, \i*SZREG + .ifeq \i-1 + .Lnull0: + .endif + bne a2, a3, .Lmisaligned + li a0, 0 + ret + .endif + .endm + +.Lloop: + # examine full words at a time, favoring strings of a couple dozen chars +#if __riscv_xlen == 32 + check_one_word 0 5 + check_one_word 1 5 + check_one_word 2 5 + check_one_word 3 5 + check_one_word 4 5 +#else + check_one_word 0 3 + check_one_word 1 3 + check_one_word 2 3 +#endif + # backwards branch to .Lloop contained above + +.Lmismatch: + # words don't match, but a2 has no null byte. + +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + +#if __riscv_xlen == 64 + sll a4, a2, 48 + sll a5, a3, 48 + bne a4, a5, .Lmismatch_upper + sll a4, a2, 32 + sll a5, a3, 32 + bne a4, a5, .Lmismatch_upper +#endif + sll a4, a2, 16 + sll a5, a3, 16 + bne a4, a5, .Lmismatch_upper + + srl a4, a2, 8*SZREG-16 + srl a5, a3, 8*SZREG-16 + sub a0, a4, a5 + and a1, a0, 0xff + bnez a1, 1f + ret + +.Lmismatch_upper: + srl a4, a4, 8*SZREG-16 + srl a5, a5, 8*SZREG-16 + sub a0, a4, a5 + and a1, a0, 0xff + bnez a1, 1f + ret + +1:and a4, a4, 0xff + and a5, a5, 0xff + sub a0, a4, a5 + ret + +#else + +#if __riscv_xlen == 64 + srl a4, a2, 48 + srl a5, a3, 48 + bne a4, a5, .Lmismatch_lower + srl a4, a2, 32 + srl a5, a3, 32 + bne a4, a5, .Lmismatch_lower +#endif + srl a4, a2, 16 + srl a5, a3, 16 + bne a4, a5, .Lmismatch_lower + + srl a4, a2, 8 + srl a5, a3, 8 + bne a4, a5, 1f + and a4, a2, 0xff + and a5, a3, 0xff +1:sub a0, a4, a5 + ret Review Comment: ```suggestion 1: sub a0, a4, a5 ret ``` ########## libs/libc/machine/arm/armv7-r/gnu/arch_strcmp.S: ########## @@ -0,0 +1,303 @@ +/**************************************************************************** + * libs/libc/machine/arm/armv7-r/gnu/arch_strcmp.S + * + * Copyright (c) 2011 The Android Open Source Project + * Copyright (c) 2008 ARM Ltd + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the company may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ****************************************************************************/ + +#ifdef __ARMEB__ +#define SHFT2LSB lsl +#define SHFT2LSBEQ lsleq +#define SHFT2MSB lsr +#define SHFT2MSBEQ lsreq +#define MSB 0x000000ff +#define LSB 0xff000000 +#else +#define SHFT2LSB lsr +#define SHFT2LSBEQ lsreq +#define SHFT2MSB lsl +#define SHFT2MSBEQ lsleq +#define MSB 0xff000000 +#define LSB 0x000000ff +#endif +#define magic1(REG) REG +#define magic2(REG) REG, lsl #7 + + .arm + .syntax unified + .global strcmp + .type strcmp, %function +strcmp: + pld [r0, #0] + pld [r1, #0] + eor r2, r0, r1 + tst r2, #3 + /* Strings not at same byte offset from a word boundary. */ + bne .Lstrcmp_unaligned + ands r2, r0, #3 + bic r0, r0, #3 + bic r1, r1, #3 + ldr ip, [r0], #4 + it eq + ldreq r3, [r1], #4 + beq 1f + /* Although s1 and s2 have identical initial alignment, they are + * not currently word aligned. Rather than comparing bytes, + * make sure that any bytes fetched from before the addressed + * bytes are forced to 0xff. Then they will always compare + * equal. + */ + eor r2, r2, #3 + lsl r2, r2, #3 + mvn r3, #MSB + SHFT2LSB r2, r3, r2 + ldr r3, [r1], #4 + orr ip, ip, r2 + orr r3, r3, r2 +1: + /* Load the 'magic' constant 0x01010101. */ + str r4, [sp, #-4]! + mov r4, #1 + orr r4, r4, r4, lsl #8 + orr r4, r4, r4, lsl #16 + .p2align 2 +4: + pld [r0, #8] + pld [r1, #8] + sub r2, ip, magic1(r4) + cmp ip, r3 + itttt eq + /* check for any zero bytes in first word */ + biceq r2, r2, ip + tsteq r2, magic2(r4) + ldreq ip, [r0], #4 + ldreq r3, [r1], #4 + beq 4b +2: + /* There's a zero or a different byte in the word */ + SHFT2MSB r0, ip, #24 + SHFT2LSB ip, ip, #8 + cmp r0, #1 + it cs + cmpcs r0, r3, SHFT2MSB #24 + it eq + SHFT2LSBEQ r3, r3, #8 + beq 2b + /* On a big-endian machine, r0 contains the desired byte in bits + * 0-7; on a little-endian machine they are in bits 24-31. In + * both cases the other bits in r0 are all zero. For r3 the + * interesting byte is at the other end of the word, but the + * other bits are not necessarily zero. We need a signed result + * representing the differnece in the unsigned bytes, so for the + * little-endian case we can't just shift the interesting bits up. + */ +#ifdef __ARMEB__ + sub r0, r0, r3, lsr #24 +#else + and r3, r3, #255 + /* No RSB instruction in Thumb2 */ +#ifdef __thumb2__ + lsr r0, r0, #24 + sub r0, r0, r3 +#else + rsb r0, r3, r0, lsr #24 +#endif +#endif + ldr r4, [sp], #4 + bx lr +.Lstrcmp_unaligned: + wp1 .req r0 + wp2 .req r1 + b1 .req r2 + w1 .req r4 + w2 .req r5 + t1 .req ip + @ r3 is scratch + /* First of all, compare bytes until wp1(sp1) is word-aligned. */ +1: + tst wp1, #3 + beq 2f + ldrb r2, [wp1], #1 + ldrb r3, [wp2], #1 + cmp r2, #1 + it cs + cmpcs r2, r3 + beq 1b + sub r0, r2, r3 + bx lr +2: + str r5, [sp, #-4]! + str r4, [sp, #-4]! + mov b1, #1 + orr b1, b1, b1, lsl #8 + orr b1, b1, b1, lsl #16 + and t1, wp2, #3 + bic wp2, wp2, #3 + ldr w1, [wp1], #4 + ldr w2, [wp2], #4 + cmp t1, #2 + beq 2f + bhi 3f + /* Critical inner Loop: Block with 3 bytes initial overlap */ + .p2align 2 +1: + bic t1, w1, #MSB + cmp t1, w2, SHFT2LSB #8 + sub r3, w1, b1 + bic r3, r3, w1 + bne 4f + ands r3, r3, b1, lsl #7 + it eq + ldreq w2, [wp2], #4 + bne 5f + eor t1, t1, w1 + cmp t1, w2, SHFT2MSB #24 + bne 6f + ldr w1, [wp1], #4 + b 1b +4: + SHFT2LSB w2, w2, #8 + b 8f +5: +#ifdef __ARMEB__ + /* The syndrome value may contain false ones if the string ends + * with the bytes 0x01 0x00 + */ + tst w1, #0xff000000 + itt ne + tstne w1, #0x00ff0000 + tstne w1, #0x0000ff00 + beq 7f +#else + bics r3, r3, #0xff000000 + bne 7f +#endif + ldrb w2, [wp2] + SHFT2LSB t1, w1, #24 +#ifdef __ARMEB__ + lsl w2, w2, #24 +#endif + b 8f +6: + SHFT2LSB t1, w1, #24 + and w2, w2, #LSB + b 8f + /* Critical inner Loop: Block with 2 bytes initial overlap */ + .p2align 2 +2: + SHFT2MSB t1, w1, #16 + sub r3, w1, b1 + SHFT2LSB t1, t1, #16 + bic r3, r3, w1 + cmp t1, w2, SHFT2LSB #16 + bne 4f + ands r3, r3, b1, lsl #7 + it eq + ldreq w2, [wp2], #4 + bne 5f + eor t1, t1, w1 + cmp t1, w2, SHFT2MSB #16 + bne 6f + ldr w1, [wp1], #4 + b 2b +5: +#ifdef __ARMEB__ + /* The syndrome value may contain false ones if the string ends + * with the bytes 0x01 0x00 + */ + tst w1, #0xff000000 + it ne + tstne w1, #0x00ff0000 + beq 7f +#else + lsls r3, r3, #16 + bne 7f +#endif + ldrh w2, [wp2] + SHFT2LSB t1, w1, #16 +#ifdef __ARMEB__ + lsl w2, w2, #16 +#endif + b 8f +6: + SHFT2MSB w2, w2, #16 + SHFT2LSB t1, w1, #16 +4: + SHFT2LSB w2, w2, #16 + b 8f + /* Critical inner Loop: Block with 1 byte initial overlap */ + .p2align 2 +3: + and t1, w1, #LSB + cmp t1, w2, SHFT2LSB #24 + sub r3, w1, b1 + bic r3, r3, w1 + bne 4f + ands r3, r3, b1, lsl #7 + it eq + ldreq w2, [wp2], #4 + bne 5f + eor t1, t1, w1 + cmp t1, w2, SHFT2MSB #8 + bne 6f + ldr w1, [wp1], #4 + b 3b +4: + SHFT2LSB w2, w2, #24 + b 8f +5: + /* The syndrome value may contain false ones if the string ends + * with the bytes 0x01 0x00 + */ + tst w1, #LSB + beq 7f + ldr w2, [wp2], #4 +6: + SHFT2LSB t1, w1, #8 + bic w2, w2, #MSB + b 8f +7: + mov r0, #0 + ldr r4, [sp], #4 + ldr r5, [sp], #4 + bx lr +8: + and r2, t1, #LSB + and r0, w2, #LSB + cmp r0, #1 + it cs + cmpcs r0, r2 + itt eq + SHFT2LSBEQ t1, t1, #8 + SHFT2LSBEQ w2, w2, #8 + beq 8b + sub r0, r2, r0 + ldr r4, [sp], #4 + ldr r5, [sp], #4 + bx lr + .size strcmp, . - strcmp Review Comment: ```suggestion .size strcmp, . - strcmp ``` ########## libs/libc/machine/risc-v/gnu/asm.h: ########## @@ -0,0 +1,52 @@ +/**************************************************************************** + * libs/libc/machine/risc-v/gnu/asm.h + * + * Copyright (c) 2017 SiFive Inc. All rights reserved. + * + * This copyrighted material is made available to anyone wishing to use, + * modify, copy, or redistribute it subject to the terms and conditions + * of the FreeBSD License. This program is distributed in the hope that + * it will be useful, but WITHOUT ANY WARRANTY expressed or implied, + * including the implied warranties of MERCHANTABILITY or FITNESS FOR + * A PARTICULAR PURPOSE. A copy of this license is available at + * http://www.opensource.org/licenses. + ****************************************************************************/ + +#ifndef _ASM_H +#define _ASM_H + +/**************************************************************************** + * Included Files + ****************************************************************************/ + +#include <nuttx/config.h> + +/**************************************************************************** + * Pre-processor Definitions + ****************************************************************************/ + +#ifdef CONFIG_ARCH_RV64 +# define SZREG 8 +# define REG_S sd +# define REG_L ld +#elif defined(CONFIG_ARCH_RV32) +# define SZREG 4 +# define REG_S sw +# define REG_L lw +#endif + +#ifdef CONFIG_ARCH_QPFPU +# define SZFREG 16 +# define FREG_S fsq +# define FREG_L flq +#elif defined(CONFIG_ARCH_DPFPU) +# define SZFREG 8 +# define FREG_S fsd +# define FREG_L fld +#elif defined(CONFIG_ARCH_FPU) Review Comment: ```suggestion #else ``` ########## libs/libc/machine/arm64/gnu/arch_strcmp.S: ########## @@ -0,0 +1,205 @@ +/**************************************************************************** + * libs/libc/machine/arm64/gnu/arch_strcmp.S + * + * Copyright (c) 2012-2018, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the Linaro nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ****************************************************************************/ + +/* Assumptions: + * + * ARMv8-a, AArch64 + */ + + .macro def_fn f p2align=0 + .text + .p2align \p2align + .global \f + .type \f, %function +\f: + .endm + +#define L(label) .L ## label + +#define REP8_01 0x0101010101010101 +#define REP8_7f 0x7f7f7f7f7f7f7f7f +#define REP8_80 0x8080808080808080 + +/* Parameters and result. */ +#define src1 x0 +#define src2 x1 +#define result x0 + +/* Internal variables. */ +#define data1 x2 +#define data1w w2 +#define data2 x3 +#define data2w w3 +#define has_nul x4 +#define diff x5 +#define syndrome x6 +#define tmp1 x7 +#define tmp2 x8 +#define tmp3 x9 +#define zeroones x10 +#define pos x11 + + /* Start of performance-critical section -- one 64B cache line. */ +def_fn strcmp p2align=6 + eor tmp1, src1, src2 + mov zeroones, #REP8_01 + tst tmp1, #7 + b.ne L(misaligned8) + ands tmp1, src1, #7 + b.ne L(mutual_align) + /* NUL detection works on the principle that (X - 1) & (~X) & 0x80 + (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and + can be done in parallel across the entire word. */ +L(loop_aligned): + ldr data1, [src1], #8 + ldr data2, [src2], #8 +L(start_realigned): + sub tmp1, data1, zeroones + orr tmp2, data1, #REP8_7f + eor diff, data1, data2 /* Non-zero if differences found. */ + bic has_nul, tmp1, tmp2 /* Non-zero if NUL terminator. */ + orr syndrome, diff, has_nul + cbz syndrome, L(loop_aligned) + /* End of performance-critical section -- one 64B cache line. */ + +L(end): +#ifndef __AARCH64EB__ Review Comment: `#if BYTE_ORDER == BIG_ENDIAN` ? ########## libs/libc/machine/arm64/gnu/arch_memcmp.S: ########## @@ -0,0 +1,196 @@ +/**************************************************************************** + * libs/libc/machine/arm64/gnu/arch_memcmp.S + * + * Copyright (c) 2018 Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the Linaro nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Copyright (c) 2017 ARM Ltd + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the company may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ****************************************************************************/ + +/* Assumptions: + * + * ARMv8-a, AArch64, unaligned accesses. + */ + +#define L(l) .L ## l + +/* Parameters and result. */ +#define src1 x0 +#define src2 x1 +#define limit x2 +#define result w0 + +/* Internal variables. */ +#define data1 x3 +#define data1w w3 +#define data1h x4 +#define data2 x5 +#define data2w w5 +#define data2h x6 +#define tmp1 x7 +#define tmp2 x8 + + .macro def_fn f p2align=0 + .text + .p2align \p2align + .global \f + .type \f, %function +\f: + .endm + +def_fn memcmp p2align=6 + subs limit, limit, 8 + b.lo L(less8) + + ldr data1, [src1], 8 + ldr data2, [src2], 8 + cmp data1, data2 + b.ne L(return) + + subs limit, limit, 8 + b.gt L(more16) + + ldr data1, [src1, limit] + ldr data2, [src2, limit] + b L(return) + +L(more16): + ldr data1, [src1], 8 + ldr data2, [src2], 8 + cmp data1, data2 + bne L(return) + + /* Jump directly to comparing the last 16 bytes for 32 byte (or less) + strings. */ + subs limit, limit, 16 + b.ls L(last_bytes) + + /* We overlap loads between 0-32 bytes at either side of SRC1 when we + try to align, so limit it only to strings larger than 128 bytes. */ + cmp limit, 96 + b.ls L(loop16) + + /* Align src1 and adjust src2 with bytes not yet done. */ + and tmp1, src1, 15 + add limit, limit, tmp1 + sub src1, src1, tmp1 + sub src2, src2, tmp1 + + /* Loop performing 16 bytes per iteration using aligned src1. + Limit is pre-decremented by 16 and must be larger than zero. + Exit if <= 16 bytes left to do or if the data is not equal. */ + .p2align 4 +L(loop16): + ldp data1, data1h, [src1], 16 + ldp data2, data2h, [src2], 16 + subs limit, limit, 16 + ccmp data1, data2, 0, hi + ccmp data1h, data2h, 0, eq + b.eq L(loop16) + + cmp data1, data2 + bne L(return) + mov data1, data1h + mov data2, data2h + cmp data1, data2 + bne L(return) + + /* Compare last 1-16 bytes using unaligned access. */ +L(last_bytes): + add src1, src1, limit + add src2, src2, limit + ldp data1, data1h, [src1] + ldp data2, data2h, [src2] + cmp data1, data2 + bne L(return) + mov data1, data1h + mov data2, data2h + cmp data1, data2 + + /* Compare data bytes and set return value to 0, -1 or 1. */ +L(return): +#ifndef __AARCH64EB__ Review Comment: `#if BYTE_ORDER == BIG_ENDIAN` ? ########## libs/libc/machine/arm/armv8-m/gnu/arch_strcmp.S: ########## @@ -0,0 +1,411 @@ +/**************************************************************************** + * libs/libc/machine/arm/armv8-m/gnu/arch_strcmp.S + * + * Copyright (c) 2012-2014 ARM Ltd + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the company may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ****************************************************************************/ + +#ifdef __ARM_BIG_ENDIAN Review Comment: `#if BYTE_ORDER == BIG_ENDIAN` ? ########## libs/libc/machine/arm/armv8-m/gnu/arch_strlen.S: ########## @@ -0,0 +1,184 @@ +/**************************************************************************** + * libs/libc/machine/arm/armv8-m/gnu/arch_strlen.S + * + * Copyright (c) 2010-2011,2013 Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of Linaro Limited nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Assumes: + * ARMv6T2 or ARMv7E-M, AArch32 + * + * Copyright (c) 2015 ARM Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the Linaro nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ****************************************************************************/ + +#include "acle-compat.h" + + .macro def_fn f p2align=0 + .text + .p2align \p2align + .global \f + .type \f, %function +\f: + .endm + +#ifdef __ARMEB__ +#define S2LO lsl +#define S2HI lsr +#else +#define S2LO lsr +#define S2HI lsl +#endif + + /* This code requires Thumb. */ +#if __ARM_ARCH_PROFILE == 'M' + .arch armv7e-m +#else + .arch armv6t2 +#endif Review Comment: Is this still true for `armv8-m`? ########## libs/libc/machine/arm/armv8-m/gnu/arch_memset.S: ########## @@ -0,0 +1,120 @@ +/**************************************************************************** + * libs/libc/machine/arm/armv8-m/gnu/arch_memset.S + * + * Copyright (c) 2015 ARM Ltd + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the company may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ****************************************************************************/ + + .thumb + .syntax unified + .global memset + .type memset, %function +memset: +#ifdef __ARM_FEATURE_MVE + vdup.8 q0, r1 + mov r3, lr + mov r1, r0 + wlstp.8 lr, r2, 2f +1: + vstrb.8 q0, [r1], #16 + letp lr, 1b +2: + bx r3 +#else + push {r4, r5, r6} + lsls r4, r0, #30 + beq 10f + subs r4, r2, #1 + cmp r2, #0 + beq 9f + uxtb r5, r1 + mov r3, r0 + b 2f +1: + subs r2, r4, #1 + cbz r4, 9f + mov r4, r2 +2: + strb r5, [r3], #1 + lsls r2, r3, #30 + bne 1b +3: + cmp r4, #3 + bls 7f + uxtb r5, r1 + orr r5, r5, r5, lsl #8 + cmp r4, #15 + orr r5, r5, r5, lsl #16 + bls 5f + mov r6, r4 + add r2, r3, #16 +4: + subs r6, r6, #16 + cmp r6, #15 + str r5, [r2, #-16] + str r5, [r2, #-12] + str r5, [r2, #-8] + str r5, [r2, #-4] + add r2, r2, #16 + bhi 4b + sub r2, r4, #16 + bic r2, r2, #15 + and r4, r4, #15 + adds r2, r2, #16 + cmp r4, #3 + add r3, r3, r2 + bls 7f +5: + mov r6, r3 + mov r2, r4 +6: + subs r2, r2, #4 + cmp r2, #3 + str r5, [r6], #4 + bhi 6b + subs r2, r4, #4 + bic r2, r2, #3 + adds r2, r2, #4 + add r3, r3, r2 + and r4, r4, #3 +7: + cbz r4, 9f + uxtb r1, r1 + add r4, r4, r3 +8: + strb r1, [r3], #1 + cmp r3, r4 + bne 8b +9: + pop {r4, r5, r6} + bx lr +10: + mov r4, r2 + mov r3, r0 + b 3b +#endif + .size memset, . - memset Review Comment: ```suggestion .size memset, . - memset ``` ########## libs/libc/machine/arm/armv8-m/gnu/arch_memmove.S: ########## @@ -0,0 +1,66 @@ +/**************************************************************************** + * libs/libc/machine/arm/armv8-m/gnu/arch_memmove.S + * + * Copyright (c) 2015 ARM Ltd + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the company may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ****************************************************************************/ + + .thumb + .syntax unified + .global memmove + .type memmove, %function +memmove: + cmp r0, r1 + push {r4} + bls 3f + adds r3, r1, r2 + cmp r0, r3 + bcs 3f + adds r1, r0, r2 + cbz r2, 2f + subs r2, r3, r2 +1: + ldrb r4, [r3, #-1]! + cmp r2, r3 + strb r4, [r1, #-1]! + bne 1b +2: + pop {r4} + bx lr +3: + cmp r2, #0 + beq 2b + add r2, r2, r1 + subs r3, r0, #1 +4: + ldrb r4, [r1], #1 + cmp r2, r1 + strb r4, [r3, #1]! + bne 4b + pop {r4} + bx lr + .size memmove, . - memmove Review Comment: ```suggestion .size memmove, . - memmove ``` ########## libs/libc/machine/arm/armv8-m/gnu/arch_strlen.S: ########## @@ -0,0 +1,184 @@ +/**************************************************************************** + * libs/libc/machine/arm/armv8-m/gnu/arch_strlen.S + * + * Copyright (c) 2010-2011,2013 Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of Linaro Limited nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Assumes: + * ARMv6T2 or ARMv7E-M, AArch32 + * + * Copyright (c) 2015 ARM Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the Linaro nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ****************************************************************************/ + +#include "acle-compat.h" + + .macro def_fn f p2align=0 + .text + .p2align \p2align + .global \f + .type \f, %function +\f: + .endm + +#ifdef __ARMEB__ Review Comment: `#if BYTE_ORDER == BIG_ENDIAN` ? ########## libs/libc/machine/arm/armv8-m/gnu/arch_memcpy.S: ########## @@ -0,0 +1,345 @@ +/**************************************************************************** + * libs/libc/machine/arm/armv8-m/gnu/arch_memcpy.S + * + * Copyright (c) 2013 ARM Ltd + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the company may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ****************************************************************************/ + +#ifndef __ARM_FEATURE_MVE +/* This memcpy routine is optimised for Cortex-M3/M4 cores with/without + unaligned access. + + If compiled with GCC, this file should be enclosed within following + pre-processing check: + if defined (__ARM_ARCH_7M__) || defined (__ARM_ARCH_7EM__) + + Prototype: void *memcpy (void *dst, const void *src, size_t count); + + The job will be done in 5 steps. + Step 1: Align src/dest pointers, copy mis-aligned if fail to align both + Step 2: Repeatedly copy big block size of __OPT_BIG_BLOCK_SIZE + Step 3: Repeatedly copy big block size of __OPT_MID_BLOCK_SIZE + Step 4: Copy word by word + Step 5: Copy byte-to-byte + + Tunable options: + __OPT_BIG_BLOCK_SIZE: Size of big block in words. Default to 64. + __OPT_MID_BLOCK_SIZE: Size of big block in words. Default to 16. + */ +#ifndef __OPT_BIG_BLOCK_SIZE +#define __OPT_BIG_BLOCK_SIZE (4 * 16) +#endif + +#ifndef __OPT_MID_BLOCK_SIZE +#define __OPT_MID_BLOCK_SIZE (4 * 4) +#endif + +#if __OPT_BIG_BLOCK_SIZE == 16 +#define BEGIN_UNROLL_BIG_BLOCK \ + .irp offset, 0,4,8,12 +#elif __OPT_BIG_BLOCK_SIZE == 32 +#define BEGIN_UNROLL_BIG_BLOCK \ + .irp offset, 0,4,8,12,16,20,24,28 +#elif __OPT_BIG_BLOCK_SIZE == 64 +#define BEGIN_UNROLL_BIG_BLOCK \ + .irp offset, 0,4,8,12,16,20,24,28,32,36,40,44,48,52,56,60 +#else +#error "Illegal __OPT_BIG_BLOCK_SIZE" +#endif + +#if __OPT_MID_BLOCK_SIZE == 8 +#define BEGIN_UNROLL_MID_BLOCK \ + .irp offset, 0,4 +#elif __OPT_MID_BLOCK_SIZE == 16 +#define BEGIN_UNROLL_MID_BLOCK \ + .irp offset, 0,4,8,12 +#else +#error "Illegal __OPT_MID_BLOCK_SIZE" +#endif + +#define END_UNROLL .endr +#endif + + .syntax unified + .text + .align 2 + .global memcpy + .thumb + .thumb_func + .type memcpy, %function +memcpy: + @ r0: dst + @ r1: src + @ r2: len +#ifdef __ARM_FEATURE_MVE + mov r3, lr + wlstp.8 lr, r2, 2f + mov r2, r0 +1: + vldrb.8 q0, [r1], #16 + vstrb.8 q0, [r2], #16 + letp lr, 1b +2: + bx r3 +#else +#ifdef __ARM_FEATURE_UNALIGNED + /* In case of UNALIGNED access supported, ip is not used in + function body. */ + mov ip, r0 +#else + push {r0} +#endif + orr r3, r1, r0 + ands r3, r3, #3 + bne .Lmisaligned_copy + +.Lbig_block: + subs r2, __OPT_BIG_BLOCK_SIZE + blo .Lmid_block + + /* Kernel loop for big block copy */ + .align 2 +.Lbig_block_loop: + BEGIN_UNROLL_BIG_BLOCK +#ifdef __ARM_ARCH_7EM__ + ldr r3, [r1], #4 + str r3, [r0], #4 + END_UNROLL +#else /* __ARM_ARCH_7M__ */ + ldr r3, [r1, \offset] + str r3, [r0, \offset] + END_UNROLL + adds r0, __OPT_BIG_BLOCK_SIZE + adds r1, __OPT_BIG_BLOCK_SIZE +#endif + subs r2, __OPT_BIG_BLOCK_SIZE + bhs .Lbig_block_loop + +.Lmid_block: + adds r2, __OPT_BIG_BLOCK_SIZE - __OPT_MID_BLOCK_SIZE + blo .Lcopy_word_by_word + + /* Kernel loop for mid-block copy */ + .align 2 +.Lmid_block_loop: + BEGIN_UNROLL_MID_BLOCK +#ifdef __ARM_ARCH_7EM__ + ldr r3, [r1], #4 + str r3, [r0], #4 + END_UNROLL +#else /* __ARM_ARCH_7M__ */ + ldr r3, [r1, \offset] + str r3, [r0, \offset] + END_UNROLL + adds r0, __OPT_MID_BLOCK_SIZE + adds r1, __OPT_MID_BLOCK_SIZE +#endif + subs r2, __OPT_MID_BLOCK_SIZE + bhs .Lmid_block_loop + +.Lcopy_word_by_word: + adds r2, __OPT_MID_BLOCK_SIZE - 4 + blo .Lcopy_less_than_4 + + /* Kernel loop for small block copy */ + .align 2 +.Lcopy_word_by_word_loop: + ldr r3, [r1], #4 + str r3, [r0], #4 + subs r2, #4 + bhs .Lcopy_word_by_word_loop + +.Lcopy_less_than_4: + adds r2, #4 + beq .Ldone + + lsls r2, r2, #31 + itt ne + ldrbne r3, [r1], #1 + strbne r3, [r0], #1 + + bcc .Ldone +#ifdef __ARM_FEATURE_UNALIGNED + ldrh r3, [r1] + strh r3, [r0] +#else + ldrb r3, [r1] + strb r3, [r0] + ldrb r3, [r1, #1] + strb r3, [r0, #1] +#endif /* __ARM_FEATURE_UNALIGNED */ + +.Ldone: +#ifdef __ARM_FEATURE_UNALIGNED + mov r0, ip +#else + pop {r0} +#endif + bx lr + + .align 2 +.Lmisaligned_copy: +#ifdef __ARM_FEATURE_UNALIGNED + /* Define label DST_ALIGNED to BIG_BLOCK. It will go to aligned copy + once destination is adjusted to aligned. */ +#define Ldst_aligned Lbig_block + + /* Copy word by word using LDR when alignment can be done in hardware, + i.e., SCTLR.A is set, supporting unaligned access in LDR and STR. */ + + cmp r2, #8 + blo .Lbyte_copy + + /* if src is aligned, just go to the big block loop. */ + lsls r3, r1, #30 + beq .Ldst_aligned +#else + /* if len < 12, misalignment adjustment has more overhead than + just byte-to-byte copy. Also, len must >=8 to guarantee code + afterward work correctly. */ + cmp r2, #12 + blo .Lbyte_copy +#endif /* __ARM_FEATURE_UNALIGNED */ + + /* Align dst only, not trying to align src. That is the because + handling of aligned src and misaligned dst need more overhead than + otherwise. By doing this the worst case is when initial src is aligned, + additional up to 4 byte additional copy will executed, which is + acceptable. */ + + ands r3, r0, #3 + beq .Ldst_aligned + + rsb r3, #4 + subs r2, r3 + + lsls r3, r3, #31 + itt ne + ldrbne r3, [r1], #1 + strbne r3, [r0], #1 + + bcc .Ldst_aligned + +#ifdef __ARM_FEATURE_UNALIGNED + ldrh r3, [r1], #2 + strh r3, [r0], #2 + b .Ldst_aligned +#else + ldrb r3, [r1], #1 + strb r3, [r0], #1 + ldrb r3, [r1], #1 + strb r3, [r0], #1 + /* Now that dst is aligned */ +.Ldst_aligned: + /* if r1 is aligned now, it means r0/r1 has the same misalignment, + and they are both aligned now. Go aligned copy. */ + ands r3, r1, #3 + beq .Lbig_block + + /* dst is aligned, but src isn't. Misaligned copy. */ + + push {r4, r5} + subs r2, #4 + + /* Backward r1 by misaligned bytes, to make r1 aligned. + Since we need to restore r1 to unaligned address after the loop, + we need keep the offset bytes to ip and sub it from r1 afterward. */ + subs r1, r3 + rsb ip, r3, #4 + + /* Pre-load on word */ + ldr r4, [r1], #4 + + cmp r3, #2 + beq .Lmisaligned_copy_2_2 + cmp r3, #3 + beq .Lmisaligned_copy_3_1 + + .macro mis_src_copy shift +1: +#ifdef __ARM_BIG_ENDIAN + lsls r4, r4, \shift +#else + lsrs r4, r4, \shift +#endif + ldr r3, [r1], #4 +#ifdef __ARM_BIG_ENDIAN + lsrs r5, r3, 32-\shift +#else + lsls r5, r3, 32-\shift +#endif + orr r4, r4, r5 + str r4, [r0], #4 + mov r4, r3 + subs r2, #4 + bhs 1b + .endm + +.Lmisaligned_copy_1_3: + mis_src_copy shift=8 + b .Lsrc_misaligned_tail + +.Lmisaligned_copy_3_1: + mis_src_copy shift=24 + b .Lsrc_misaligned_tail + +.Lmisaligned_copy_2_2: + /* For 2_2 misalignment, ldr is still faster than 2 x ldrh. */ + mis_src_copy shift=16 + +.Lsrc_misaligned_tail: + adds r2, #4 + subs r1, ip + pop {r4, r5} + +#endif /* __ARM_FEATURE_UNALIGNED */ + +.Lbyte_copy: + subs r2, #4 + blo .Lcopy_less_than_4 + +.Lbyte_copy_loop: + subs r2, #1 + ldrb r3, [r1], #1 + strb r3, [r0], #1 + bhs .Lbyte_copy_loop + + ldrb r3, [r1] + strb r3, [r0] + ldrb r3, [r1, #1] + strb r3, [r0, #1] + ldrb r3, [r1, #2] + strb r3, [r0, #2] + +#ifdef __ARM_FEATURE_UNALIGNED + mov r0, ip +#else + pop {r0} +#endif + bx lr +#endif + .size memcpy, .-memcpy Review Comment: ```suggestion .size memcpy, .-memcpy ``` ########## libs/libc/machine/arm/armv8-m/gnu/acle-compat.h: ########## @@ -0,0 +1,186 @@ +/**************************************************************************** + * libs/libc/machine/arm/armv8-m/gnu/acle-compat.h + * + * Copyright (c) 2014 ARM Ltd + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the company may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + ****************************************************************************/ + +#ifndef __ARM_ARCH Review Comment: ```suggestion #ifndef __LIBS_LIBC_MACHINE_ARM_ARMV8_M_GNU_ACLE_COMPAT_H #define __LIBS_LIBC_MACHINE_ARM_ARMV8_M_GNU_ACLE_COMPAT_H ``` ########## libs/libc/machine/arm/armv8-m/gnu/arch_memcpy.S: ########## @@ -0,0 +1,345 @@ +/**************************************************************************** + * libs/libc/machine/arm/armv8-m/gnu/arch_memcpy.S + * + * Copyright (c) 2013 ARM Ltd + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the company may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ****************************************************************************/ + +#ifndef __ARM_FEATURE_MVE +/* This memcpy routine is optimised for Cortex-M3/M4 cores with/without + unaligned access. + + If compiled with GCC, this file should be enclosed within following + pre-processing check: + if defined (__ARM_ARCH_7M__) || defined (__ARM_ARCH_7EM__) + + Prototype: void *memcpy (void *dst, const void *src, size_t count); + + The job will be done in 5 steps. + Step 1: Align src/dest pointers, copy mis-aligned if fail to align both + Step 2: Repeatedly copy big block size of __OPT_BIG_BLOCK_SIZE + Step 3: Repeatedly copy big block size of __OPT_MID_BLOCK_SIZE + Step 4: Copy word by word + Step 5: Copy byte-to-byte + + Tunable options: + __OPT_BIG_BLOCK_SIZE: Size of big block in words. Default to 64. + __OPT_MID_BLOCK_SIZE: Size of big block in words. Default to 16. + */ +#ifndef __OPT_BIG_BLOCK_SIZE +#define __OPT_BIG_BLOCK_SIZE (4 * 16) +#endif + +#ifndef __OPT_MID_BLOCK_SIZE +#define __OPT_MID_BLOCK_SIZE (4 * 4) +#endif + +#if __OPT_BIG_BLOCK_SIZE == 16 +#define BEGIN_UNROLL_BIG_BLOCK \ + .irp offset, 0,4,8,12 +#elif __OPT_BIG_BLOCK_SIZE == 32 +#define BEGIN_UNROLL_BIG_BLOCK \ + .irp offset, 0,4,8,12,16,20,24,28 +#elif __OPT_BIG_BLOCK_SIZE == 64 +#define BEGIN_UNROLL_BIG_BLOCK \ + .irp offset, 0,4,8,12,16,20,24,28,32,36,40,44,48,52,56,60 +#else +#error "Illegal __OPT_BIG_BLOCK_SIZE" +#endif + +#if __OPT_MID_BLOCK_SIZE == 8 +#define BEGIN_UNROLL_MID_BLOCK \ + .irp offset, 0,4 +#elif __OPT_MID_BLOCK_SIZE == 16 +#define BEGIN_UNROLL_MID_BLOCK \ + .irp offset, 0,4,8,12 +#else +#error "Illegal __OPT_MID_BLOCK_SIZE" +#endif + +#define END_UNROLL .endr +#endif + + .syntax unified + .text + .align 2 + .global memcpy + .thumb + .thumb_func + .type memcpy, %function +memcpy: + @ r0: dst + @ r1: src + @ r2: len +#ifdef __ARM_FEATURE_MVE + mov r3, lr + wlstp.8 lr, r2, 2f + mov r2, r0 +1: + vldrb.8 q0, [r1], #16 + vstrb.8 q0, [r2], #16 + letp lr, 1b +2: + bx r3 +#else +#ifdef __ARM_FEATURE_UNALIGNED + /* In case of UNALIGNED access supported, ip is not used in + function body. */ + mov ip, r0 +#else + push {r0} +#endif + orr r3, r1, r0 + ands r3, r3, #3 + bne .Lmisaligned_copy + +.Lbig_block: + subs r2, __OPT_BIG_BLOCK_SIZE + blo .Lmid_block + + /* Kernel loop for big block copy */ + .align 2 +.Lbig_block_loop: + BEGIN_UNROLL_BIG_BLOCK +#ifdef __ARM_ARCH_7EM__ + ldr r3, [r1], #4 + str r3, [r0], #4 + END_UNROLL +#else /* __ARM_ARCH_7M__ */ + ldr r3, [r1, \offset] + str r3, [r0, \offset] + END_UNROLL + adds r0, __OPT_BIG_BLOCK_SIZE + adds r1, __OPT_BIG_BLOCK_SIZE +#endif + subs r2, __OPT_BIG_BLOCK_SIZE + bhs .Lbig_block_loop + +.Lmid_block: + adds r2, __OPT_BIG_BLOCK_SIZE - __OPT_MID_BLOCK_SIZE + blo .Lcopy_word_by_word + + /* Kernel loop for mid-block copy */ + .align 2 +.Lmid_block_loop: + BEGIN_UNROLL_MID_BLOCK +#ifdef __ARM_ARCH_7EM__ + ldr r3, [r1], #4 + str r3, [r0], #4 + END_UNROLL +#else /* __ARM_ARCH_7M__ */ + ldr r3, [r1, \offset] + str r3, [r0, \offset] + END_UNROLL + adds r0, __OPT_MID_BLOCK_SIZE + adds r1, __OPT_MID_BLOCK_SIZE +#endif + subs r2, __OPT_MID_BLOCK_SIZE + bhs .Lmid_block_loop + +.Lcopy_word_by_word: + adds r2, __OPT_MID_BLOCK_SIZE - 4 + blo .Lcopy_less_than_4 + + /* Kernel loop for small block copy */ + .align 2 +.Lcopy_word_by_word_loop: + ldr r3, [r1], #4 + str r3, [r0], #4 + subs r2, #4 + bhs .Lcopy_word_by_word_loop + +.Lcopy_less_than_4: + adds r2, #4 + beq .Ldone + + lsls r2, r2, #31 + itt ne + ldrbne r3, [r1], #1 + strbne r3, [r0], #1 + + bcc .Ldone +#ifdef __ARM_FEATURE_UNALIGNED Review Comment: does `armv8-m` support unaligned access? ########## libs/libc/machine/arm/armv7-r/gnu/arch_strlen.S: ########## @@ -0,0 +1,184 @@ +/**************************************************************************** + * libs/libc/machine/arm/armv7-r/gnu/arch_strlen.S + * + * Copyright (c) 2010-2011,2013 Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of Linaro Limited nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Assumes: + * ARMv6T2 or ARMv7E-M, AArch32 + * + * Copyright (c) 2015 ARM Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the Linaro nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ****************************************************************************/ + +#include "acle-compat.h" + + .macro def_fn f p2align=0 + .text + .p2align \p2align + .global \f + .type \f, %function +\f: + .endm + +#ifdef __ARMEB__ +#define S2LO lsl +#define S2HI lsr +#else +#define S2LO lsr +#define S2HI lsl +#endif + + /* This code requires Thumb. */ +#if __ARM_ARCH_PROFILE == 'M' + .arch armv7e-m +#else + .arch armv6t2 +#endif Review Comment: `armv7-r`? ########## libs/libc/machine/arm/armv8-m/gnu/arch_memchr.S: ########## @@ -0,0 +1,391 @@ +/**************************************************************************** + * libs/libc/machine/arm/armv8-m/gnu/arch_memchr.S + * + * Copyright (c) 2010-2011, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of Linaro Limited nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Written by Dave Gilbert <[email protected]> + * + * This memchr routine is optimised on a Cortex-A9 and should work on + * all ARMv7 processors. It has a fast path for short sizes, and has + * an optimised path for large data sets; the worst case is finding the + * match early in a large data set. + * + * Copyright (c) 2015 ARM Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the Linaro nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ****************************************************************************/ + +@ 2011-02-07 [email protected] +@ Extracted from local git a5b438d861 +@ 2011-07-14 [email protected] +@ Import endianness fix from local git ea786f1b +@ 2011-10-11 [email protected] +@ Import from cortex-strings bzr rev 63 +@ Flip to ldrd (as suggested by Greta Yorsh) +@ Make conditional on CPU type +@ tidy + +@ This code requires armv6t2 or later. Uses Thumb2. + + .syntax unified + +#include "acle-compat.h" + +@ NOTE: This ifdef MUST match the one in memchr-stub.c +#if defined (__ARM_NEON__) || defined (__ARM_NEON) +#if __ARM_ARCH >= 8 && __ARM_ARCH_PROFILE == 'R' + .arch armv8-r +#else + .arch armv7-a +#endif + .fpu neon + + +/* Arguments */ +#define srcin r0 +#define chrin r1 +#define cntin r2 + +/* Retval */ +#define result r0 /* Live range does not overlap with srcin */ + +/* Working registers */ +#define src r1 /* Live range does not overlap with chrin */ +#define tmp r3 +#define synd r0 /* No overlap with srcin or result */ +#define soff r12 + +/* Working NEON registers */ +#define vrepchr q0 +#define vdata0 q1 +#define vdata0_0 d2 /* Lower half of vdata0 */ +#define vdata0_1 d3 /* Upper half of vdata0 */ +#define vdata1 q2 +#define vdata1_0 d4 /* Lower half of vhas_chr0 */ +#define vdata1_1 d5 /* Upper half of vhas_chr0 */ +#define vrepmask q3 +#define vrepmask0 d6 +#define vrepmask1 d7 +#define vend q4 +#define vend0 d8 +#define vend1 d9 + +/* + * Core algorithm: + * + * For each 32-byte chunk we calculate a 32-bit syndrome value, with one bit per + * byte. Each bit is set if the relevant byte matched the requested character + * and cleared otherwise. Since the bits in the syndrome reflect exactly the + * order in which things occur in the original string, counting trailing zeros + * allows to identify exactly which byte has matched. + */ + + .text + .thumb_func + .align 4 + .p2align 4,,15 + .global memchr + .type memchr,%function + +memchr: + .cfi_sections .debug_frame + .cfi_startproc + /* Use a simple loop if there are less than 8 bytes to search. */ + cmp cntin, #7 + bhi .Llargestr + and chrin, chrin, #0xff + +.Lsmallstr: + subs cntin, cntin, #1 + blo .Lnotfound /* Return not found if reached end. */ + ldrb tmp, [srcin], #1 + cmp tmp, chrin + bne .Lsmallstr /* Loop again if not found. */ + /* Otherwise fixup address and return. */ + sub result, result, #1 + bx lr + + +.Llargestr: + vdup.8 vrepchr, chrin /* Duplicate char across all lanes. */ + /* + * Magic constant 0x8040201008040201 allows us to identify which lane + * matches the requested byte. + */ + movw tmp, #0x0201 + movt tmp, #0x0804 + lsl soff, tmp, #4 + vmov vrepmask0, tmp, soff + vmov vrepmask1, tmp, soff + /* Work with aligned 32-byte chunks */ + bic src, srcin, #31 + ands soff, srcin, #31 + beq .Lloopintro /* Go straight to main loop if it's aligned. */ + + /* + * Input string is not 32-byte aligned. We calculate the syndrome + * value for the aligned 32 bytes block containing the first bytes + * and mask the irrelevant part. + */ + vld1.8 {vdata0, vdata1}, [src:256]! + sub tmp, soff, #32 + adds cntin, cntin, tmp + vceq.i8 vdata0, vdata0, vrepchr + vceq.i8 vdata1, vdata1, vrepchr + vand vdata0, vdata0, vrepmask + vand vdata1, vdata1, vrepmask + vpadd.i8 vdata0_0, vdata0_0, vdata0_1 + vpadd.i8 vdata1_0, vdata1_0, vdata1_1 + vpadd.i8 vdata0_0, vdata0_0, vdata1_0 + vpadd.i8 vdata0_0, vdata0_0, vdata0_0 + vmov synd, vdata0_0[0] + + /* Clear the soff lower bits */ + lsr synd, synd, soff + lsl synd, synd, soff + /* The first block can also be the last */ + bls .Lmasklast + /* Have we found something already? */ + cbnz synd, .Ltail + + +.Lloopintro: + vpush {vend} + /* 264/265 correspond to d8/d9 for q4 */ + .cfi_adjust_cfa_offset 16 + .cfi_rel_offset 264, 0 + .cfi_rel_offset 265, 8 + .p2align 3,,7 +.Lloop: + vld1.8 {vdata0, vdata1}, [src:256]! + subs cntin, cntin, #32 + vceq.i8 vdata0, vdata0, vrepchr + vceq.i8 vdata1, vdata1, vrepchr + /* If we're out of data we finish regardless of the result. */ + bls .Lend + /* Use a fast check for the termination condition. */ + vorr vend, vdata0, vdata1 + vorr vend0, vend0, vend1 + vmov synd, tmp, vend0 + orrs synd, synd, tmp + /* We're not out of data, loop if we haven't found the character. */ + beq .Lloop + +.Lend: + vpop {vend} + .cfi_adjust_cfa_offset -16 + .cfi_restore 264 + .cfi_restore 265 + + /* Termination condition found, let's calculate the syndrome value. */ + vand vdata0, vdata0, vrepmask + vand vdata1, vdata1, vrepmask + vpadd.i8 vdata0_0, vdata0_0, vdata0_1 + vpadd.i8 vdata1_0, vdata1_0, vdata1_1 + vpadd.i8 vdata0_0, vdata0_0, vdata1_0 + vpadd.i8 vdata0_0, vdata0_0, vdata0_0 + vmov synd, vdata0_0[0] + cbz synd, .Lnotfound + bhi .Ltail + + +.Lmasklast: + /* Clear the (-cntin) upper bits to avoid out-of-bounds matches. */ + neg cntin, cntin + lsl synd, synd, cntin + lsrs synd, synd, cntin + it eq + moveq src, #0 /* If no match, set src to 0 so the retval is 0. */ + + +.Ltail: + /* Count the trailing zeros using bit reversing */ + rbit synd, synd + /* Compensate the last post-increment */ + sub src, src, #32 + /* Count the leading zeros */ + clz synd, synd + /* Compute the potential result and return */ + add result, src, synd + bx lr + + +.Lnotfound: + /* Set result to NULL if not found and return */ + mov result, #0 + bx lr + + .cfi_endproc + .size memchr, . - memchr + +#elif __ARM_ARCH_ISA_THUMB >= 2 && defined (__ARM_FEATURE_DSP) + +#if __ARM_ARCH_PROFILE == 'M' + .arch armv7e-m +#else + .arch armv6t2 +#endif + +@ this lets us check a flag in a 00/ff byte easily in either endianness +#ifdef __ARMEB__ +#define CHARTSTMASK(c) 1<<(31-(c*8)) +#else +#define CHARTSTMASK(c) 1<<(c*8) +#endif + .text + .thumb + +@ --------------------------------------------------------------------------- + .thumb_func + .align 2 + .p2align 4,,15 + .global memchr + .type memchr,%function +memchr: + @ r0 = start of memory to scan + @ r1 = character to look for + @ r2 = length + @ returns r0 = pointer to character or NULL if not found + and r1,r1,#0xff @ Don't trust the caller to pass a char + + cmp r2,#16 @ If short don't bother with anything clever + blt 20f + + tst r0, #7 @ If it's already aligned skip the next bit + beq 10f + + @ Work up to an aligned point +5: + ldrb r3, [r0],#1 + subs r2, r2, #1 + cmp r3, r1 + beq 50f @ If it matches exit found + tst r0, #7 + cbz r2, 40f @ If we run off the end, exit not found + bne 5b @ If not aligned yet then do next byte + +10: + @ We are aligned, we know we have at least 8 bytes to work with + push {r4,r5,r6,r7} + orr r1, r1, r1, lsl #8 @ expand the match word across all bytes + orr r1, r1, r1, lsl #16 + bic r4, r2, #7 @ Number of double words to work with * 8 + mvns r7, #0 @ all F's + movs r3, #0 + +15: + ldrd r5,r6,[r0],#8 + subs r4, r4, #8 + eor r5,r5, r1 @ r5,r6 have 00's where bytes match the target + eor r6,r6, r1 + uadd8 r5, r5, r7 @ Par add 0xff - sets GE bits for bytes!=0 + sel r5, r3, r7 @ bytes are 00 for none-00 bytes, + @ or ff for 00 bytes - NOTE INVERSION + uadd8 r6, r6, r7 @ Par add 0xff - sets GE bits for bytes!=0 + sel r6, r5, r7 @ chained....bytes are 00 for none-00 bytes + @ or ff for 00 bytes - NOTE INVERSION + cbnz r6, 60f + bne 15b @ (Flags from the subs above) + + pop {r4,r5,r6,r7} + and r1,r1,#0xff @ r1 back to a single character + and r2,r2,#7 @ Leave the count remaining as the number + @ after the double words have been done + +20: + cbz r2, 40f @ 0 length or hit the end already then not found + +21: @ Post aligned section, or just a short call + ldrb r3,[r0],#1 + subs r2,r2,#1 + eor r3,r3,r1 @ r3 = 0 if match - doesn't break flags from sub + cbz r3, 50f + bne 21b @ on r2 flags + +40: + movs r0,#0 @ not found + bx lr + +50: + subs r0,r0,#1 @ found + bx lr + +60: @ We're here because the fast path found a hit + @ now we have to track down exactly which word it was + @ r0 points to the start of the double word after the one tested + @ r5 has the 00/ff pattern for the first word, r6 has the chained value + cmp r5, #0 + itte eq + moveq r5, r6 @ the end is in the 2nd word + subeq r0,r0,#3 @ Points to 2nd byte of 2nd word + subne r0,r0,#7 @ or 2nd byte of 1st word + + @ r0 currently points to the 2nd byte of the word containing the hit + tst r5, # CHARTSTMASK(0) @ 1st character + bne 61f + adds r0,r0,#1 + tst r5, # CHARTSTMASK(1) @ 2nd character + ittt eq + addeq r0,r0,#1 + tsteq r5, # (3<<15) @ 2nd & 3rd character + @ If not the 3rd must be the last one + addeq r0,r0,#1 + +61: + pop {r4,r5,r6,r7} + subs r0,r0,#1 + bx lr +#else + /* Defined in memchr-stub.c. */ +#endif Review Comment: ```suggestion #endif ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
