From: Dhruv Chawla <[email protected]>
Saturating addition and subtraction in SI and DI modes are already computed
in general registers to avoid the transfers to and from the FP registers
that [SU]QADD and [SU]QSUB require. The QI and HI modes were left out, so
vqaddb_u8(a, b) on values held in W registers becomes for example:
dup v30.8b, w0
dup v31.8b, w1
uqadd b31, b31, b30
umov w0, v31.b[0]
where three of the four instructions only move the data into and back out
of the vector register file.
The trick used for the SI and DI modes does not carry over directly (as the
comment above the existing patterns notes): the flags set by a 32-bit ADDS
say nothing about overflow of an 8 or 16-bit value, which cannot overflow
a W register at all. Shifting both operands up to the top of the register
first fixes that. The carry and overflow flags then describe the narrow
operation exactly (so the same sequence applies), and the shift doubles as
the extension of the operand. The bits above the mode are also discarded
and don't have to be masked or sign-extended separately:
lsl w2, w1, 24
lsl w0, w0, 24
adds w0, w0, w2
csinv w0, w0, wzr, cc
lsr w0, w0, 24
Both alternatives are kept, so operands that are already in vector
registers still use the single instruction with no transfers.
Bootstrapped and regtested on aarch64-linux-gnu.
Signed-off-by: Dhruv Chawla <[email protected]>
gcc/ChangeLog:
* config/aarch64/aarch64-protos.h (aarch64_split_saturating_arith):
New declaration.
* config/aarch64/aarch64.cc (aarch64_split_saturating_arith): New
function.
* config/aarch64/aarch64.md (add<mode>3_compare0): Rename to ...
(@add<mode>3_compare0): ... this.
(sub<mode>3_compare1): Rename to...
(@sub<mode>3_compare1): ... this.
(sub<mode>3_compare1_imm): Rename to ...
(@sub<mode>3_compare1_imm): ... this.
* config/aarch64/aarch64-simd.md
(<su_optab>s<addsub><mode>3<vczle><vczbe>): Restrict the vector
pattern to VDQ_I, and use aarch64_split_saturating_arith in the
GPI one.
(aarch64_<su_optab>s<addsub><mode>3<vczle><vczbe>): Use
aarch64_split_saturating_arith.
(<su_optab>s<addsub><mode>3): New expanders for SHORT.
(aarch64_<su_optab>s<addsub><mode>3): New patterns for SHORT.
* config/aarch64/iterators.md (VSDQ_I_QI_HI): Delete unused
iterator.
gcc/testsuite/ChangeLog:
* gcc.target/aarch64/neon/vqadd.c (test_vqaddb_s8, test_vqaddb_u8,
test_vqaddh_s16, test_vqaddh_u16): Update test with new
instruction sequence.
* gcc.target/aarch64/neon/vqsub.c (test_vqsubb_s8, test_vqsubb_u8,
test_vqsubh_s16, test_vqsubh_u16): Likewise.
* gcc.target/aarch64/saturating_arithmetic_1.c: Likewise.
* gcc.target/aarch64/saturating_arithmetic_2.c: Likewise. Drop
xfail on the now-passing usub test.
* gcc.target/aarch64/saturating-arithmetic-signed.c (sadd8, ssub8,
sadd16, ssub16): New functions.
* gcc.target/aarch64/scalar_intrinsics.c (test_vqaddh_s16,
test_vqaddb_s8, test_vqsubh_s16, test_vqsubb_s8): Modify to take
vector arguments to continue generating NEON instructions.
---
gcc/config/aarch64/aarch64-protos.h | 2 +
gcc/config/aarch64/aarch64-simd.md | 220 +++++++-----------
gcc/config/aarch64/aarch64.cc | 145 ++++++++++++
gcc/config/aarch64/aarch64.md | 6 +-
gcc/config/aarch64/iterators.md | 4 -
gcc/testsuite/gcc.target/aarch64/neon/vqadd.c | 40 ++--
gcc/testsuite/gcc.target/aarch64/neon/vqsub.c | 40 ++--
.../aarch64/saturating-arithmetic-signed.c | 108 ++++++++-
.../aarch64/saturating_arithmetic_1.c | 30 ++-
.../aarch64/saturating_arithmetic_2.c | 31 +--
.../gcc.target/aarch64/scalar_intrinsics.c | 16 +-
11 files changed, 435 insertions(+), 207 deletions(-)
diff --git a/gcc/config/aarch64/aarch64-protos.h
b/gcc/config/aarch64/aarch64-protos.h
index e8ae3d427949..29a7aa2445ca 100644
--- a/gcc/config/aarch64/aarch64-protos.h
+++ b/gcc/config/aarch64/aarch64-protos.h
@@ -1027,6 +1027,8 @@ bool aarch64_sve_prefetch_operand_p (rtx, machine_mode);
bool aarch64_sve_struct_memory_operand_p (rtx);
bool aarch64_sme_ldr_vnum_offset_p (rtx, rtx);
rtx aarch64_simd_vect_par_cnst_half (machine_mode, int, bool);
+void aarch64_split_saturating_arith (rtx_code, machine_mode, rtx, rtx,
+ rtx, rtx, rtx);
rtx aarch64_gen_stepped_int_parallel (unsigned int, int, int);
bool aarch64_stepped_int_parallel_p (rtx, int);
bool aarch64_strided_registers_p (rtx *, unsigned int, unsigned int);
diff --git a/gcc/config/aarch64/aarch64-simd.md
b/gcc/config/aarch64/aarch64-simd.md
index 98c2da034c55..3b8c091ab670 100644
--- a/gcc/config/aarch64/aarch64-simd.md
+++ b/gcc/config/aarch64/aarch64-simd.md
@@ -5993,10 +5993,10 @@
;; <su>q<addsub>
(define_insn "<su_optab>s<addsub><mode>3<vczle><vczbe>"
- [(set (match_operand:VSDQ_I_QI_HI 0 "register_operand" "=w")
- (BINQOPS:VSDQ_I_QI_HI
- (match_operand:VSDQ_I_QI_HI 1 "register_operand" "w")
- (match_operand:VSDQ_I_QI_HI 2 "register_operand" "w")))]
+ [(set (match_operand:VDQ_I 0 "register_operand" "=w")
+ (BINQOPS:VDQ_I
+ (match_operand:VDQ_I 1 "register_operand" "w")
+ (match_operand:VDQ_I 2 "register_operand" "w")))]
"TARGET_SIMD"
"<su_optab>q<addsub>\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
[(set_attr "type" "neon_q<addsub><q>")]
@@ -6044,84 +6044,12 @@
[ r , r , JIr , &r ; * , * , 8 ] #
}
"&& reload_completed && GP_REGNUM_P (REGNO (operands[0]))"
- [(set (match_dup 0)
- (if_then_else:GPI
- (match_dup 4)
- (match_dup 5)
- (match_dup 6)))]
+ [(const_int 0)]
{
- if (REG_P (operands[2]))
- {
- rtx shift_constant = gen_int_mode (GET_MODE_BITSIZE (<MODE>mode) - 1,
- <MODE>mode);
- auto limit = HOST_WIDE_INT_1U << (GET_MODE_BITSIZE (<MODE>mode) - 1);
- rtx limit_constant = gen_int_mode (limit, <MODE>mode);
- emit_insn (gen_ashr<mode>3 (operands[3], operands[2], shift_constant));
- emit_insn (gen_xor<mode>3 (operands[3], operands[3], limit_constant));
-
- switch (<CODE>)
- {
- case SS_MINUS:
- emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1],
- operands[2]));
- break;
- case SS_PLUS:
- emit_insn (gen_add<mode>3_compare0 (operands[0], operands[1],
- operands[2]));
- break;
- default:
- gcc_unreachable ();
- }
-
- rtx ccin = gen_rtx_REG (E_CC_Vmode, CC_REGNUM);
- switch (<CODE>)
- {
- case SS_PLUS:
- operands[4] = gen_rtx_NE (<MODE>mode, ccin, const0_rtx);
- operands[5] = gen_rtx_NOT (<MODE>mode, operands[3]);
- operands[6] = operands[0];
- break;
- case SS_MINUS:
- operands[4] = gen_rtx_EQ (<MODE>mode, ccin, const0_rtx);
- operands[5] = operands[0];
- operands[6] = operands[3];
- break;
- default:
- gcc_unreachable ();
- }
- }
- else
- {
- auto imm = INTVAL (operands[2]);
- rtx neg_imm = gen_int_mode (-imm, <MODE>mode);
- wide_int limit;
-
- switch (<CODE>)
- {
- case SS_MINUS:
- emit_insn (gen_sub<mode>3_compare1_imm (operands[0], operands[1],
- operands[2], neg_imm));
- limit = imm >= 0 ? wi::min_value (<MODE>mode, SIGNED)
- : wi::max_value (<MODE>mode, SIGNED);
- break;
- case SS_PLUS:
- emit_insn (gen_sub<mode>3_compare1_imm (operands[0], operands[1],
- neg_imm, operands[2]));
- limit = imm >= 0 ? wi::max_value (<MODE>mode, SIGNED)
- : wi::min_value (<MODE>mode, SIGNED);
- break;
- default:
- gcc_unreachable ();
- }
-
- rtx sat_limit = immed_wide_int_const (limit, <MODE>mode);
- emit_insn (gen_rtx_SET (operands[3], sat_limit));
-
- rtx ccin = gen_rtx_REG (E_CC_Vmode, CC_REGNUM);
- operands[4] = gen_rtx_EQ (<MODE>mode, ccin, const0_rtx);
- operands[5] = operands[0];
- operands[6] = operands[3];
- }
+ aarch64_split_saturating_arith (<CODE>, <MODE>mode, operands[0],
+ operands[1], operands[2], operands[3],
+ NULL_RTX);
+ DONE;
}
)
@@ -6141,62 +6069,86 @@
[ r , r , JIr ; * , * , 8 ] #
}
"&& reload_completed && GP_REGNUM_P (REGNO (operands[0]))"
- [(set (match_dup 0)
- (if_then_else:GPI
- (match_dup 3)
- (match_dup 0)
- (match_dup 4)))]
+ [(const_int 0)]
{
+ aarch64_split_saturating_arith (<CODE>, <MODE>mode, operands[0],
+ operands[1], operands[2], NULL_RTX,
+ NULL_RTX);
+ DONE;
+ }
+)
- if (REG_P (operands[2]))
- {
- switch (<CODE>)
- {
- case US_MINUS:
- emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1],
- operands[2]));
- break;
- case US_PLUS:
- emit_insn (gen_add<mode>3_compare0 (operands[0], operands[1],
- operands[2]));
- break;
- default:
- gcc_unreachable ();
- }
- }
- else
- {
- auto imm = UINTVAL (operands[2]);
- rtx neg_imm = gen_int_mode (-imm, <MODE>mode);
- switch (<CODE>)
- {
- case US_MINUS:
- emit_insn (gen_sub<mode>3_compare1_imm (operands[0], operands[1],
- operands[2], neg_imm));
- break;
- case US_PLUS:
- emit_insn (gen_sub<mode>3_compare1_imm (operands[0], operands[1],
- neg_imm, operands[2]));
- break;
- default:
- gcc_unreachable ();
- }
- }
+;; Saturating arithmetic on 8 and 16-bit GPR operands. As for SI and DI modes
+;; above we would rather not pay for transfers to and from the FP registers,
+;; and compute the result only in GPRs. This is done through a series of left
+;; and right shifts (see aarch64_split_saturating_arith in aarch64.cc for a
+;; complete explanation). For example, saturating unsigned addition becomes:
+;;
+;; lsl w1, w1, 24
+;; lsl w0, w0, 24
+;; adds w0, w0, w1
+;; csinv w0, w0, wzr, cc
+;; lsr w0, w0, 24
- rtx ccin = gen_rtx_REG (CCmode, CC_REGNUM);
- switch (<CODE>)
- {
- case US_PLUS:
- operands[3] = gen_rtx_LTU (<MODE>mode, ccin, const0_rtx);
- operands[4] = gen_int_mode (-1, <MODE>mode);
- break;
- case US_MINUS:
- operands[3] = gen_rtx_GEU (<MODE>mode, ccin, const0_rtx);
- operands[4] = const0_rtx;
- break;
- default:
- gcc_unreachable ();
- }
+(define_expand "<su_optab>s<addsub><mode>3"
+ [(parallel
+ [(set (match_operand:SHORT 0 "register_operand")
+ (UBINQOPS:SHORT (match_operand:SHORT 1 "register_operand")
+ (match_operand:SHORT 2 "register_operand")))
+ (clobber (scratch:SI))
+ (clobber (reg:CC CC_REGNUM))])]
+)
+
+(define_insn_and_split "aarch64_<su_optab>s<addsub><mode>3"
+ [(set (match_operand:SHORT 0 "register_operand")
+ (UBINQOPS:SHORT (match_operand:SHORT 1 "register_operand")
+ (match_operand:SHORT 2 "register_operand")))
+ (clobber (match_scratch:SI 3))
+ (clobber (reg:CC CC_REGNUM))]
+ ""
+ {@ [ cons: =0, 1 , 2 , =3 ; attrs: type , arch , length ]
+ [ w , w , w , X ; neon_q<addsub><q> , simd , 4 ]
<su_optab>q<addsub>\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>
+ [ r , r , r , &r ; * , * , 20 ] #
+ }
+ "&& reload_completed && GP_REGNUM_P (REGNO (operands[0]))"
+ [(const_int 0)]
+ {
+ aarch64_split_saturating_arith (<CODE>, <MODE>mode, operands[0],
+ operands[1], operands[2], NULL_RTX,
+ operands[3]);
+ DONE;
+ }
+)
+
+(define_expand "<su_optab>s<addsub><mode>3"
+ [(parallel
+ [(set (match_operand:SHORT 0 "register_operand")
+ (SBINQOPS:SHORT (match_operand:SHORT 1 "register_operand")
+ (match_operand:SHORT 2 "register_operand")))
+ (clobber (scratch:SI))
+ (clobber (scratch:SI))
+ (clobber (reg:CC CC_REGNUM))])]
+)
+
+(define_insn_and_split "aarch64_<su_optab>s<addsub><mode>3"
+ [(set (match_operand:SHORT 0 "register_operand")
+ (SBINQOPS:SHORT (match_operand:SHORT 1 "register_operand")
+ (match_operand:SHORT 2 "register_operand")))
+ (clobber (match_scratch:SI 3))
+ (clobber (match_scratch:SI 4))
+ (clobber (reg:CC CC_REGNUM))]
+ ""
+ {@ [ cons: =0, 1 , 2 , =3, =4 ; attrs: type , arch , length ]
+ [ w , w , w , X , X ; neon_q<addsub><q> , simd , 4 ]
<su_optab>q<addsub>\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>
+ [ r , r , r , &r, &r ; * , * , 28 ] #
+ }
+ "&& reload_completed && GP_REGNUM_P (REGNO (operands[0]))"
+ [(const_int 0)]
+ {
+ aarch64_split_saturating_arith (<CODE>, <MODE>mode, operands[0],
+ operands[1], operands[2], operands[4],
+ operands[3]);
+ DONE;
}
)
diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc
index 6693cec98bdb..aafd4601b2b0 100644
--- a/gcc/config/aarch64/aarch64.cc
+++ b/gcc/config/aarch64/aarch64.cc
@@ -25552,6 +25552,151 @@ aarch64_simd_check_vect_par_cnst_half (rtx op,
machine_mode mode,
return true;
}
+/* Split a saturating addition or subtraction into a sequence that computes
+ the result in general registers, rather than transferring the operands to
+ and from the FP registers to use [SU]QADD or [SU]QSUB.
+
+ CODE is the saturating operation and MODE the mode of its operands. DST is
+ the destination and OP1 and OP2 are the operands. OP2 may be a constant
only
+ when MODE is SImode or DImode. LIMIT is a scratch register used to hold the
+ saturating limit (which only signed operations need) and TMP a second
+ scratch (which only QI and HI modes need). Either may be NULL_RTX when the
+ operation does not require it.
+
+ For SI and DI modes the flags set by ADDS or SUBS describe the operation
+ exactly, so the saturated result can be selected against the limit directly.
+ QI and HI mode values cannot overflow a W register, so shifting those up to
+ the top of the register first makes the flags describe the narrow
+ operation and at the same time discards the bits above the mode. This saves
+ a separate extension. */
+
+void
+aarch64_split_saturating_arith (rtx_code code, machine_mode mode, rtx dst,
+ rtx op1, rtx op2, rtx limit, rtx tmp)
+{
+ bool signed_p = (code == SS_PLUS || code == SS_MINUS);
+ bool minus_p = (code == SS_MINUS || code == US_MINUS);
+
+ /* QI/HI modes have to be computed in SI mode. */
+ int shift = 0;
+ if (mode == QImode || mode == HImode)
+ {
+ /* The amount to left-shift by to reach the top of the register. */
+ shift = GET_MODE_BITSIZE ((machine_mode) SImode).to_constant ()
+ - GET_MODE_BITSIZE (mode).to_constant ();
+ mode = SImode;
+ }
+
+ int bits = GET_MODE_BITSIZE (mode).to_constant ();
+
+ if (shift > 0)
+ {
+ gcc_assert (REG_P (op2) && tmp);
+ /* Refer to QI or HI mode operands using SI mode (i.e. W) registers. The
+ scratches are already SImode. */
+ dst = gen_rtx_REG (mode, REGNO (dst));
+ op1 = gen_rtx_REG (mode, REGNO (op1));
+ op2 = gen_rtx_REG (mode, REGNO (op2));
+
+ /* Shift the values to the top of the registers. */
+ rtx amount = GEN_INT (shift);
+ emit_insn (gen_rtx_SET (tmp, gen_rtx_ASHIFT (mode, op2, amount)));
+ emit_insn (gen_rtx_SET (dst, gen_rtx_ASHIFT (mode, op1, amount)));
+ op1 = dst;
+ op2 = tmp;
+ }
+
+ /* Work out the value to use when the operation saturates and emit the
+ flag-setting add or subtract. The default saturating value is 0 for
+ minus operations and all-ones for plus. */
+ rtx saturated = minus_p ? const0_rtx : gen_int_mode (-1, mode);
+ if (REG_P (op2))
+ {
+ if (signed_p)
+ {
+ /* Signed operations use LIMIT as a scratch register. */
+ gcc_assert (limit != NULL_RTX);
+
+ /* The saturating limit depends on the sign of the second operand.
+ For SI mode, it is INT_MIN when it is negative and INT_MAX
+ otherwise:
+
+ op2 >= 0: op2 >> 31 = 0x00000000 ^ 0x80000000 = 0x80000000
+ op2 < 0: op2 >> 31 = 0xFFFFFFFF ^ 0x80000000 = 0x7FFFFFFF
+
+ ... and likewise for the other modes. */
+ rtx sign_bit = GEN_INT (bits - 1);
+ emit_insn (
+ gen_rtx_SET (limit, gen_rtx_ASHIFTRT (mode, op2, sign_bit)));
+
+ rtx min = immed_wide_int_const (wi::min_value (mode, SIGNED), mode);
+ emit_insn (gen_rtx_SET (limit, gen_rtx_XOR (mode, limit, min)));
+
+ /* CSINV supplies the complement, so SS_PLUS wants ~LIMIT. */
+ saturated = minus_p ? limit : gen_rtx_NOT (mode, limit);
+ }
+
+ insn_code icode = minus_p ? code_for_sub3_compare1 (mode)
+ : code_for_add3_compare0 (mode);
+ emit_insn (GEN_FCN (icode) (dst, op1, op2));
+ }
+ else
+ {
+ /* A constant second operand. Only reachable for SI and DI modes due to
+ the SHORT patterns requiring a register_operand. */
+ gcc_assert (shift == 0);
+ auto imm = INTVAL (op2);
+ rtx neg_imm = gen_int_mode (-imm, mode);
+
+ /* Operand 2 sets the flags and operand 3 is added, and the pattern
+ requires them to be negatives of each other. Which way round they go
+ selects SUBS or ADDS. */
+ rtx cmp_imm = minus_p ? op2 : neg_imm;
+ rtx add_imm = minus_p ? neg_imm : op2;
+
+ insn_code icode = code_for_sub3_compare1_imm (mode);
+ emit_insn (GEN_FCN (icode) (dst, op1, cmp_imm, add_imm));
+
+ if (signed_p)
+ {
+ bool want_max = (imm >= 0) ^ minus_p;
+ wide_int lim = want_max ? wi::max_value (mode, SIGNED)
+ : wi::min_value (mode, SIGNED);
+ emit_insn (gen_rtx_SET (limit, immed_wide_int_const (lim, mode)));
+ saturated = limit;
+ }
+ }
+
+ /* Select between the computed result and the saturating limit. CSINV only
+ matches with the complement in the true arm, so that case tests for
+ overflow rather than for its absence. */
+ rtx cc = gen_rtx_REG (signed_p ? E_CC_Vmode : CCmode, CC_REGNUM);
+ if (GET_CODE (saturated) == NOT)
+ {
+ gcc_assert (code == SS_PLUS);
+ emit_insn (gen_rtx_SET (
+ dst, gen_rtx_IF_THEN_ELSE (mode, gen_rtx_NE (mode, cc, const0_rtx),
+ saturated, dst)));
+ }
+ else
+ {
+ rtx no_ovf = (signed_p ? gen_rtx_EQ (mode, cc, const0_rtx)
+ : minus_p ? gen_rtx_GEU (mode, cc, const0_rtx)
+ : gen_rtx_LTU (mode, cc, const0_rtx));
+ emit_insn (
+ gen_rtx_SET (dst, gen_rtx_IF_THEN_ELSE (mode, no_ovf, dst, saturated)));
+ }
+
+ /* Bring a narrow result back down. */
+ if (shift > 0)
+ {
+ rtx amount = GEN_INT (shift);
+ rtx back = (signed_p ? gen_rtx_ASHIFTRT (mode, dst, amount)
+ : gen_rtx_LSHIFTRT (mode, dst, amount));
+ emit_insn (gen_rtx_SET (dst, back));
+ }
+}
+
/* Return a PARALLEL containing NELTS elements, with element I equal
to BASE + I * STEP. */
diff --git a/gcc/config/aarch64/aarch64.md b/gcc/config/aarch64/aarch64.md
index 7c8915ec3890..fa31947b2cd4 100644
--- a/gcc/config/aarch64/aarch64.md
+++ b/gcc/config/aarch64/aarch64.md
@@ -3023,7 +3023,7 @@
DONE;
})
-(define_insn "add<mode>3_compare0"
+(define_insn "@add<mode>3_compare0"
[(set (reg:CC_NZ CC_REGNUM)
(compare:CC_NZ
(plus:GPI (match_operand:GPI 1 "register_operand")
@@ -3824,7 +3824,7 @@
[(set_attr "type" "alus_sreg")]
)
-(define_insn "sub<mode>3_compare1_imm"
+(define_insn "@sub<mode>3_compare1_imm"
[(set (reg:CC CC_REGNUM)
(compare:CC
(match_operand:GPI 1 "register_operand")
@@ -3841,7 +3841,7 @@
[(set_attr "type" "alus_imm")]
)
-(define_insn "sub<mode>3_compare1"
+(define_insn "@sub<mode>3_compare1"
[(set (reg:CC CC_REGNUM)
(compare:CC
(match_operand:GPI 1 "aarch64_reg_or_zero" "rkZ")
diff --git a/gcc/config/aarch64/iterators.md b/gcc/config/aarch64/iterators.md
index 568786e37149..65f8eee3bc13 100644
--- a/gcc/config/aarch64/iterators.md
+++ b/gcc/config/aarch64/iterators.md
@@ -102,10 +102,6 @@
;; integer modes; 64-bit scalar integer mode.
(define_mode_iterator VSDQ_I_DI [V8QI V16QI V4HI V8HI V2SI V4SI V2DI DI])
-;; Advanced SIMD and scalar, 64 & 128-bit container; 8 and 16-bit scalar
-;; integer modes.
-(define_mode_iterator VSDQ_I_QI_HI [VDQ_I HI QI])
-
;; Double vector modes.
(define_mode_iterator VD [V8QI V4HI V4HF V2SI V2SF V4BF])
diff --git a/gcc/testsuite/gcc.target/aarch64/neon/vqadd.c
b/gcc/testsuite/gcc.target/aarch64/neon/vqadd.c
index 322f54c5a42f..e78797a01d6e 100644
--- a/gcc/testsuite/gcc.target/aarch64/neon/vqadd.c
+++ b/gcc/testsuite/gcc.target/aarch64/neon/vqadd.c
@@ -117,40 +117,48 @@ TEST_UNIFORM_BINARY (vqaddq_s64, int64x2_t)
/*
** test_vqaddb_u8:
-** dup v([0-9]+)\.8b, w[0-9]+
-** dup v([0-9]+)\.8b, w[0-9]+
-** uqadd b([0-9]+), (b\2, b\1|b\1, b\2)
-** umov w0, v\3\.b\[0\]
+** lsl w([0-9]+), w1, 24
+** lsl w0, w0, 24
+** adds w0, w0, w\1
+** csinv w0, w0, wzr, cc
+** lsr w0, w0, 24
** ret
*/
TEST_UNIFORM_BINARY (vqaddb_u8, uint8_t)
/*
** test_vqaddb_s8:
-** dup v([0-9]+)\.8b, w[0-9]+
-** dup v([0-9]+)\.8b, w[0-9]+
-** sqadd b([0-9]+), (b\2, b\1|b\1, b\2)
-** umov w0, v\3\.b\[0\]
+** lsl w([0-9]+), w0, 24
+** lsl w0, w1, 24
+** adds w0, w0, w\1
+** asr w([0-9]+), w\1, 31
+** eor w\2, w\2, -2147483648
+** csinv w0, w0, w\2, vc
+** asr w0, w0, 24
** ret
*/
TEST_UNIFORM_BINARY (vqaddb_s8, int8_t)
/*
** test_vqaddh_u16:
-** dup v([0-9]+)\.4h, w[0-9]+
-** dup v([0-9]+)\.4h, w[0-9]+
-** uqadd h([0-9]+), (h\2, h\1|h\1, h\2)
-** umov w0, v\3\.h\[0\]
+** lsl w([0-9]+), w1, 16
+** lsl w0, w0, 16
+** adds w0, w0, w\1
+** csinv w0, w0, wzr, cc
+** lsr w0, w0, 16
** ret
*/
TEST_UNIFORM_BINARY (vqaddh_u16, uint16_t)
/*
** test_vqaddh_s16:
-** dup v([0-9]+)\.4h, w[0-9]+
-** dup v([0-9]+)\.4h, w[0-9]+
-** sqadd h([0-9]+), (h\2, h\1|h\1, h\2)
-** umov w0, v\3\.h\[0\]
+** lsl w([0-9]+), w0, 16
+** lsl w0, w1, 16
+** adds w0, w0, w\1
+** asr w([0-9]+), w\1, 31
+** eor w\2, w\2, -2147483648
+** csinv w0, w0, w\2, vc
+** asr w0, w0, 16
** ret
*/
TEST_UNIFORM_BINARY (vqaddh_s16, int16_t)
diff --git a/gcc/testsuite/gcc.target/aarch64/neon/vqsub.c
b/gcc/testsuite/gcc.target/aarch64/neon/vqsub.c
index 52ac2f102ebc..4d6705ba9e53 100644
--- a/gcc/testsuite/gcc.target/aarch64/neon/vqsub.c
+++ b/gcc/testsuite/gcc.target/aarch64/neon/vqsub.c
@@ -117,40 +117,48 @@ TEST_UNIFORM_BINARY (vqsubq_s64, int64x2_t)
/*
** test_vqsubb_u8:
-** dup v([0-9]+)\.8b, w0
-** dup v([0-9]+)\.8b, w1
-** uqsub b([0-9]+), b\1, b\2
-** umov w0, v\3\.b\[0\]
+** lsl w([0-9]+), w1, 24
+** lsl w0, w0, 24
+** subs w0, w0, w\1
+** csel w0, w0, wzr, cs
+** lsr w0, w0, 24
** ret
*/
TEST_UNIFORM_BINARY (vqsubb_u8, uint8_t)
/*
** test_vqsubb_s8:
-** dup v([0-9]+)\.8b, w0
-** dup v([0-9]+)\.8b, w1
-** sqsub b([0-9]+), b\1, b\2
-** umov w0, v\3\.b\[0\]
+** lsl w([0-9]+), w1, 24
+** lsl w0, w0, 24
+** subs w0, w0, w\1
+** asr w([0-9]+), w\1, 31
+** eor w\2, w\2, -2147483648
+** csel w0, w0, w\2, vc
+** asr w0, w0, 24
** ret
*/
TEST_UNIFORM_BINARY (vqsubb_s8, int8_t)
/*
** test_vqsubh_u16:
-** dup v([0-9]+)\.4h, w0
-** dup v([0-9]+)\.4h, w1
-** uqsub h([0-9]+), h\1, h\2
-** umov w0, v\3\.h\[0\]
+** lsl w([0-9]+), w1, 16
+** lsl w0, w0, 16
+** subs w0, w0, w\1
+** csel w0, w0, wzr, cs
+** lsr w0, w0, 16
** ret
*/
TEST_UNIFORM_BINARY (vqsubh_u16, uint16_t)
/*
** test_vqsubh_s16:
-** dup v([0-9]+)\.4h, w0
-** dup v([0-9]+)\.4h, w1
-** sqsub h([0-9]+), h\1, h\2
-** umov w0, v\3\.h\[0\]
+** lsl w([0-9]+), w1, 16
+** lsl w0, w0, 16
+** subs w0, w0, w\1
+** asr w([0-9]+), w\1, 31
+** eor w\2, w\2, -2147483648
+** csel w0, w0, w\2, vc
+** asr w0, w0, 16
** ret
*/
TEST_UNIFORM_BINARY (vqsubh_s16, int16_t)
diff --git a/gcc/testsuite/gcc.target/aarch64/saturating-arithmetic-signed.c
b/gcc/testsuite/gcc.target/aarch64/saturating-arithmetic-signed.c
index de652bf1d9e6..6112fca9b3ac 100644
--- a/gcc/testsuite/gcc.target/aarch64/saturating-arithmetic-signed.c
+++ b/gcc/testsuite/gcc.target/aarch64/saturating-arithmetic-signed.c
@@ -190,6 +190,82 @@ ssub64_imm2 (int64_t __a)
return !overflow ? result : __a < 0 ? LONG_MIN : LONG_MAX;
}
+/*
+** sadd8:
+** lsl w([0-9]+), w0, 24
+** lsl w([0-9]+), w1, 24
+** asr w([0-9]+), w\1, 31
+** eor w\3, w\3, -2147483648
+** adds w\2, (?:w\2, w\1|w\1, w\2)
+** csinv w0, w\2, w\3, vc
+** asr w0, w0, 24
+** ret
+*/
+int8_t __attribute__((noipa))
+sadd8 (int8_t __a, int8_t __b)
+{
+ int8_t sum;
+ bool overflow = __builtin_add_overflow (__a, __b, &sum);
+ return !overflow ? sum : __a < 0 ? SCHAR_MIN : SCHAR_MAX;
+}
+
+/*
+** ssub8:
+** lsl w([0-9]+), w1, 24
+** lsl w([0-9]+), w0, 24
+** asr w([0-9]+), w\1, 31
+** eor w\3, w\3, -2147483648
+** subs w\2, w\2, w\1
+** csel w0, w\2, w\3, vc
+** asr w0, w0, 24
+** ret
+*/
+int8_t __attribute__((noipa))
+ssub8 (int8_t __a, int8_t __b)
+{
+ int8_t result;
+ bool overflow = __builtin_sub_overflow (__a, __b, &result);
+ return !overflow ? result : __a < 0 ? SCHAR_MIN : SCHAR_MAX;
+}
+
+/*
+** sadd16:
+** lsl w([0-9]+), w0, 16
+** lsl w([0-9]+), w1, 16
+** asr w([0-9]+), w\1, 31
+** eor w\3, w\3, -2147483648
+** adds w\2, (?:w\2, w\1|w\1, w\2)
+** csinv w0, w\2, w\3, vc
+** asr w0, w0, 16
+** ret
+*/
+int16_t __attribute__((noipa))
+sadd16 (int16_t __a, int16_t __b)
+{
+ int16_t sum;
+ bool overflow = __builtin_add_overflow (__a, __b, &sum);
+ return !overflow ? sum : __a < 0 ? SHRT_MIN : SHRT_MAX;
+}
+
+/*
+** ssub16:
+** lsl w([0-9]+), w1, 16
+** lsl w([0-9]+), w0, 16
+** asr w([0-9]+), w\1, 31
+** eor w\3, w\3, -2147483648
+** subs w\2, w\2, w\1
+** csel w0, w\2, w\3, vc
+** asr w0, w0, 16
+** ret
+*/
+int16_t __attribute__((noipa))
+ssub16 (int16_t __a, int16_t __b)
+{
+ int16_t result;
+ bool overflow = __builtin_sub_overflow (__a, __b, &result);
+ return !overflow ? result : __a < 0 ? SHRT_MIN : SHRT_MAX;
+}
+
int
main (void)
{
@@ -266,5 +342,35 @@ main (void)
if (ssub64 (a_64, -b_64) != (a_64 + b_64))
__builtin_abort ();
+ /* 8- and 16-bit, which are computed in general registers after being
+ shifted up to the top of the word. */
+ int8_t sa = 4, sb = 70;
+ if (sadd8 (sa, sb) != (int8_t) (sa + sb))
+ __builtin_abort ();
+ if (sadd8 (sa, SCHAR_MAX) != SCHAR_MAX)
+ __builtin_abort ();
+ if (sadd8 (-sa, SCHAR_MIN) != SCHAR_MIN)
+ __builtin_abort ();
+ if (ssub8 (sa, sb) != (int8_t) (sa - sb))
+ __builtin_abort ();
+ if (ssub8 (SCHAR_MIN, sb) != SCHAR_MIN)
+ __builtin_abort ();
+ if (ssub8 (SCHAR_MAX, -sb) != SCHAR_MAX)
+ __builtin_abort ();
+
+ int16_t ha = 4, hb = 70;
+ if (sadd16 (ha, hb) != (int16_t) (ha + hb))
+ __builtin_abort ();
+ if (sadd16 (ha, SHRT_MAX) != SHRT_MAX)
+ __builtin_abort ();
+ if (sadd16 (-ha, SHRT_MIN) != SHRT_MIN)
+ __builtin_abort ();
+ if (ssub16 (ha, hb) != (int16_t) (ha - hb))
+ __builtin_abort ();
+ if (ssub16 (SHRT_MIN, hb) != SHRT_MIN)
+ __builtin_abort ();
+ if (ssub16 (SHRT_MAX, -hb) != SHRT_MAX)
+ __builtin_abort ();
+
return 0;
-}
\ No newline at end of file
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/saturating_arithmetic_1.c
b/gcc/testsuite/gcc.target/aarch64/saturating_arithmetic_1.c
index c73a024edec6..d66bad3d3e5a 100644
--- a/gcc/testsuite/gcc.target/aarch64/saturating_arithmetic_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/saturating_arithmetic_1.c
@@ -2,28 +2,34 @@
/* { dg-options "-O2 --save-temps -fno-schedule-insns2" } */
/* { dg-final { check-function-bodies "**" "" "" } } */
+
/*
** uadd:
-** dup v([0-9]+).8b, w[01]
-** dup v([0-9]+).8b, w[01]
-** uqadd b([0-9]+), (?:b\2, b\1|b\1, b\2)
-** umov w0, v\3.b\[0\]
+** lsl w([0-9]+), w1, 24
+** lsl w([0-9]+), w0, 24
+** adds w\2, (?:w\2, w\1|w\1, w\2)
+** csinv w\2, w\2, wzr, cc
+** lsr w0, w\2, 24
** ret
*/
+
/*
** uadd2:
-** dup v([0-9]+).8b, w[01]
-** dup v([0-9]+).8b, w[01]
-** uqadd b([0-9]+), (?:b\2, b\1|b\1, b\2)
-** umov w0, v\3.b\[0\]
+** lsl w([0-9]+), w1, 24
+** lsl w([0-9]+), w0, 24
+** adds w\2, (?:w\2, w\1|w\1, w\2)
+** csinv w\2, w\2, wzr, cc
+** lsr w0, w\2, 24
** ret
*/
+
/*
** usub:
-** dup v([0-9]+).8b, w[01]
-** dup v([0-9]+).8b, w[01]
-** uqsub b([0-9]+), b\1, b\2
-** umov w0, v\3.b\[0\]
+** lsl w([0-9]+), w1, 24
+** lsl w([0-9]+), w0, 24
+** subs w\2, w\2, w\1
+** csel w\2, w\2, wzr, cs
+** lsr w0, w\2, 24
** ret
*/
diff --git a/gcc/testsuite/gcc.target/aarch64/saturating_arithmetic_2.c
b/gcc/testsuite/gcc.target/aarch64/saturating_arithmetic_2.c
index dd0fefac6393..174aaf97c381 100644
--- a/gcc/testsuite/gcc.target/aarch64/saturating_arithmetic_2.c
+++ b/gcc/testsuite/gcc.target/aarch64/saturating_arithmetic_2.c
@@ -4,26 +4,31 @@
/*
** uadd:
-** dup v([0-9]+).4h, w[01]
-** dup v([0-9]+).4h, w[01]
-** uqadd h([0-9]+), (?:h\2, h\1|h\1, h\2)
-** umov w0, v\3.h\[0\]
+** lsl w([0-9]+), w1, 16
+** lsl w([0-9]+), w0, 16
+** adds w\2, (?:w\2, w\1|w\1, w\2)
+** csinv w\2, w\2, wzr, cc
+** lsr w0, w\2, 16
** ret
*/
+
/*
** uadd2:
-** dup v([0-9]+).4h, w[01]
-** dup v([0-9]+).4h, w[01]
-** uqadd h([0-9]+), (?:h\2, h\1|h\1, h\2)
-** umov w0, v\3.h\[0\]
+** lsl w([0-9]+), w1, 16
+** lsl w([0-9]+), w0, 16
+** adds w\2, (?:w\2, w\1|w\1, w\2)
+** csinv w\2, w\2, wzr, cc
+** lsr w0, w\2, 16
** ret
*/
+
/*
-** usub: { xfail *-*-* }
-** dup v([0-9]+).4h, w1
-** dup v([0-9]+).4h, w0
-** uqsub h([0-9]+), h\1, h\2
-** umov w0, v\3.h\[0\]
+** usub:
+** lsl w([0-9]+), w1, 16
+** lsl w([0-9]+), w0, 16
+** subs w\2, w\2, w\1
+** csel w\2, w\2, wzr, cs
+** lsr w0, w\2, 16
** ret
*/
diff --git a/gcc/testsuite/gcc.target/aarch64/scalar_intrinsics.c
b/gcc/testsuite/gcc.target/aarch64/scalar_intrinsics.c
index 094aaff7b1c1..0122e7d12c76 100644
--- a/gcc/testsuite/gcc.target/aarch64/scalar_intrinsics.c
+++ b/gcc/testsuite/gcc.target/aarch64/scalar_intrinsics.c
@@ -366,17 +366,17 @@ test_vqadds_s32 (int32_t a, int32_t b)
/* { dg-final { scan-assembler-times "\\tsqadd\\th\[0-9\]+, h\[0-9\]+" 1 } } */
int16_t
-test_vqaddh_s16 (int16_t a, int16_t b)
+test_vqaddh_s16 (int16x8_t a, int16x8_t b)
{
- return vqaddh_s16 (a, b);
+ return vqaddh_s16 (a[0], b[0]);
}
/* { dg-final { scan-assembler-times "\\tsqadd\\tb\[0-9\]+, b\[0-9\]+" 1 } } */
int8_t
-test_vqaddb_s8 (int8_t a, int8_t b)
+test_vqaddb_s8 (int8x16_t a, int8x16_t b)
{
- return vqaddb_s8 (a, b);
+ return vqaddb_s8 (a[0], b[0]);
}
/* { dg-final { scan-assembler-times "\\tsqdmlal\\ts\[0-9\]+, h\[0-9\]+,
h\[0-9\]+" 1 } } */
@@ -809,17 +809,17 @@ test_vqsubs_s32 (int32_t a, int32_t b)
/* { dg-final { scan-assembler-times "\\tsqsub\\th\[0-9\]+" 1 } } */
int16_t
-test_vqsubh_s16 (int16_t a, int16_t b)
+test_vqsubh_s16 (int16x8_t a, int16x8_t b)
{
- return vqsubh_s16 (a, b);
+ return vqsubh_s16 (a[0], b[0]);
}
/* { dg-final { scan-assembler-times "\\tsqsub\\tb\[0-9\]+" 1 } } */
int8_t
-test_vqsubb_s8 (int8_t a, int8_t b)
+test_vqsubb_s8 (int8x16_t a, int8x16_t b)
{
- return vqsubb_s8 (a, b);
+ return vqsubb_s8 (a[0], b[0]);
}
/* { dg-final { scan-assembler-times "\\tsshl\\td\[0-9\]+" 1 } } */
--
2.43.0
