> -----Original Message-----
> From: Kyrylo Tkachov <[email protected]>
> Sent: 14 September 2026 10:56
> To: Tamar Christina <[email protected]>
> Cc: [email protected]; nd <[email protected]>; Richard Earnshaw
> <[email protected]>; [email protected]; Alex Coplan
> <[email protected]>; [email protected]; Wilco Dijkstra
> <[email protected]>; Alice Carlotti <[email protected]>
> Subject: Re: [patch 3/13]AArch64: implement new control flow IFNs for
> Adv.SIMD
>
>
>
> > On 14 Sep 2026, at 11:25, Tamar Christina <[email protected]>
> wrote:
> >
> > This patch implements the control flow IFNs for Adv.SIMD. It gives the
> > AArch64 expansions for counting active lanes, finding the
> > first active lane, creating inclusive and exclusive active-lane masks, and
> > extracting the first active value from a vector.
> >
> > The implementation uses ADDHN to compress byte, halfword and word
> predicates
> > down to a scalar bitmap. Plain vector branches use the cheaper
> zero/nonzero
> > compressed predicate path. When the vectorizer needs the active lane
> number,
> > the exit block masks the compressed bitmap and uses scalar bit operations
> to
> > find the lane. For live register values the final extraction can use a
> > vector
> > table lookup instead of replaying scalar loads.
> >
> > For example, consider:
> >
> > unsigned char a[1008], b[1008];
> >
> > unsigned char f1 (void)
> > {
> > unsigned char res = 0;
> > for (unsigned int i = 0; i < 1008; ++i)
> > {
> > unsigned char x = a[i] & b[i];
> > res = x;
> > if (x > 5)
> > break;
> > }
> > return res;
> > }
> >
> > Today we we generate at -O3 -march=armv8-a:
> >
> > cmhi v30.16b, v31.16b, v29.16b
> > umaxp v30.4s, v30.4s, v30.4s
> > fmov x0, d30
> > cbz x0, .L2
> > ldrb w0, [x2, w1, uxtw]
> > cmp w0, 5
> > bhi .L3
> > ... repeated scalar lane checks ...
> >
> > and with this branch:
> >
> > cmhi v31.16b, v30.16b, v29.16b
> > addhn v31.8b, v31.8h, v31.8h
> > umov x0, v31.d[0]
> > cbz x0, .L8
> > rbit x0, x0
> > clz x0, x0
> > ...
> > tbl v30.16b, {v30.16b}, v31.16b
> > fmov w0, s30
> >
> > There is one complication in how we perform the Adv. SIMD optimization
> which
> > replaces an Adv. SIMD compare with an SVE one.
> >
> > This now has to be delayed until after expand when we can tell if the
> compare
> > is single use. Otherwise we'd end up with an Adv. SIMD and SVE compare
> for the
> > same as CSE can't remove the duplicate then.
> >
> > Bootstrapped Regtested on aarch64-none-linux-gnu,
> > arm-none-linux-gnueabihf, x86_64-pc-linux-gnu
> > -m32, -m64 and no issues.
> >
> > -O3, profiled, profiled -O3 and -O3 -LTO bootstrap tested and no issues.
> >
> > Also SPECCPU 2017 and SPECCPU 2026 tested.
> >
> > OK for master?
> >
> > Thanks,
> > Tamar
> >
> > gcc/ChangeLog:
> >
> > * config/aarch64/aarch64-protos.h (aarch64_gen_shareable_ptrue):
> > Declare.
> > * config/aarch64/aarch64-simd.md (cbranch<mode>4): Use compressed
> > predicate branches for Adv.SIMD masks.
> > (@aarch64_addhn_scalar<mode>): New expander.
> > (@aarch64_addhn_mask_scalar<mode>): Likewise.
> > (count_active_<mode>): Likewise.
> > (index_first_active_<mode>): Likewise.
> > (extract_first_<mode>): Likewise.
> > (take_<take_active>_active_<mode>): Likewise.
> > * config/aarch64/aarch64.cc (aarch64_gen_shareable_ptrue): New
> > function.
> > * config/aarch64/aarch64.h (advsimd_ptrue_insn): New.
> > * config/aarch64/iterators.md (VADDHN_MASK_WIDE,
> VADDHN_MASK_NARROW,
> > vaddhn_mask_wide, vaddhn_lane_mask, TAKE_ACTIVE_CMP, take_active):
> New.
> >
> > gcc/testsuite/ChangeLog:
> >
> > * gcc.target/aarch64/noeffect11.c: Update expected code generation.
> > * gcc.target/aarch64/noeffect2.c: Likewise.
> > * gcc.target/aarch64/noeffect3.c: Likewise.
> > * gcc.target/aarch64/noeffect4.c: Likewise.
> > * gcc.target/aarch64/noeffect5.c: Likewise.
> > * gcc.target/aarch64/noeffect6.c: Likewise.
> > * gcc.target/aarch64/noeffect7.c: Likewise.
> > * gcc.target/aarch64/noeffect8.c: Likewise.
> > * gcc.target/aarch64/vect-early-break-cbranch.c: Likewise.
> > * gcc.target/aarch64/vect-early-break-cbranch_2.c: Likewise.
> > * gcc.target/aarch64/vect-early-break-cbranch_3.c: Likewise.
> > * gcc.target/aarch64/vect-early-break-cbranch_4.c: Likewise.
> > * gcc.target/aarch64/vect-early-break-cbranch_5.c: Likewise.
> > * lib/target-supports.exp: Add Adv.SIMD LUT support checks.
> > * gcc.target/aarch64/noeffect12.c: New test.
> > * gcc.target/aarch64/noeffect13-run.c: New test.
> > * gcc.target/aarch64/noeffect13.c: New test.
> > * gcc.target/aarch64/vect-early-break-cbranch_7.c: New test.
> >
> > ---
> > diff --git a/gcc/config/aarch64/aarch64-protos.h
> b/gcc/config/aarch64/aarch64-protos.h
> > index
> e8ae3d427949896f5a34d21bd55ce770cdcdcd76..4d882ce90ae2ea50b9b3
> 3a380af7a6f1d0bc27a5 100644
> > --- a/gcc/config/aarch64/aarch64-protos.h
> > +++ b/gcc/config/aarch64/aarch64-protos.h
> > @@ -1015,6 +1015,7 @@ rtx aarch64_return_addr_rtx (void);
> > rtx aarch64_return_addr (int, rtx);
> > rtx aarch64_simd_gen_const_vector_dup (machine_mode,
> HOST_WIDE_INT);
> > rtx aarch64_gen_shareable_zero (machine_mode);
> > +rtx aarch64_gen_shareable_ptrue (machine_mode, unsigned int);
> > bool aarch64_split_simd_shift_p (rtx_insn *);
> > bool aarch64_simd_mem_operand_p (rtx);
> > bool aarch64_sve_ld1r_operand_p (rtx);
> > diff --git a/gcc/config/aarch64/aarch64-simd.md
> b/gcc/config/aarch64/aarch64-simd.md
> > index
> f9b2fd6b2db1c88e50aac3b6f6f77ab9ba626654..5c45db988b3a83948a01e
> bf1071c88e9e8814cb7 100644
> > --- a/gcc/config/aarch64/aarch64-simd.md
> > +++ b/gcc/config/aarch64/aarch64-simd.md
> > @@ -4475,8 +4475,8 @@ (define_expand "cbranch<mode>4"
> > [(set (pc)
> > (if_then_else
> > (match_operator 0 "aarch64_equality_operator"
> > - [(match_operand:VDQ_I 1 "register_operand")
> > - (match_operand:VDQ_I 2 "aarch64_simd_reg_or_zero")])
> > + [(match_operand:VD_BHSI 1 "register_operand")
> > + (match_operand:VD_BHSI 2 "aarch64_simd_reg_or_zero")])
> > (label_ref (match_operand 3 ""))
> > (pc)))]
> > "TARGET_SIMD"
> > @@ -4492,16 +4492,46 @@ (define_expand "cbranch<mode>4"
> > emit_insn (gen_xor<mode>3 (tmp, operands[1], operands[2]));
> > }
> >
> > - /* For 64-bit vectors we need no reductions. */
> > - if (known_eq (128, GET_MODE_BITSIZE (<MODE>mode)))
> > + rtx val = gen_reg_rtx (DImode);
> > + emit_move_insn (val, gen_lowpart (DImode, tmp));
> > +
> > + rtx cc_reg = aarch64_gen_compare_reg (code, val, const0_rtx);
> > + rtx cmp_rtx = gen_rtx_fmt_ee (code, DImode, cc_reg, const0_rtx);
> > + emit_jump_insn (gen_aarch64_bcond (cmp_rtx, cc_reg, operands[3]));
> > + DONE;
> > +})
> > +
> > +(define_expand "cbranch<mode>4"
> > + [(set (pc)
> > + (if_then_else
> > + (match_operator 0 "aarch64_equality_operator"
> > + [(match_operand:VQ_I 1 "register_operand")
> > + (match_operand:VQ_I 2 "aarch64_simd_reg_or_zero")])
> > + (label_ref (match_operand 3 ""))
> > + (pc)))]
> > + "TARGET_SIMD"
> > +{
> > + auto code = GET_CODE (operands[0]);
> > + rtx tmp = operands[1];
> > +
> > + /* If comparing against a non-zero vector we have to do a comparison
> > first
> > + so we can have a != 0 comparison with the result. */
> > + if (operands[2] != CONST0_RTX (<MODE>mode))
> > {
> > - /* Always reduce using a V4SI. */
> > - rtx reduc = gen_lowpart (V4SImode, tmp);
> > - rtx res = gen_reg_rtx (V4SImode);
> > - emit_insn (gen_aarch64_umaxpv4si (res, reduc, reduc));
> > - emit_move_insn (tmp, gen_lowpart (<MODE>mode, res));
> > + tmp = gen_reg_rtx (<MODE>mode);
> > + emit_insn (gen_xor<mode>3 (tmp, operands[1], operands[2]));
> > }
> >
> > + /* This expander can be used for arbitrary vector zero tests, for example
> > + the XOR result produced by __builtin_memcmp_eq. ADDHN is only valid
> for
> > + boolean masks whose true lanes are all ones. Use UMAXP here; the
> > + vec_cbranch expanders below still use ADDHN after producing a compare
> > + mask. */
> > + rtx reduc = gen_lowpart (V4SImode, tmp);
> > + rtx res = gen_reg_rtx (V4SImode);
> > + emit_insn (gen_aarch64_umaxpv4si (res, reduc, reduc));
> > + tmp = gen_lowpart (<MODE>mode, res);
> > +
> > rtx val = gen_reg_rtx (DImode);
> > emit_move_insn (val, gen_lowpart (DImode, tmp));
> >
> > @@ -4511,6 +4541,298 @@ (define_expand "cbranch<mode>4"
> > DONE;
> > })
> >
> > +;; Compress a 128-bit Adv.SIMD boolean vector into a 64-bit scalar value.
> > +;; ADDHN folds each adjacent pair of lanes into one narrower lane. This is
> > +;; enough for zero/nonzero tests: a nonzero bit in any original lane
> > produces
> > +;; a nonzero bit in the scalar result.
> > +;;
> > +;; Keep this separate from aarch64_addhn_mask_scalar. Vector branches
> such as
> > +;; those in gcc.dg/vect/vect-uncounted_1.c only need a cbz/cbnz-style test
> and
> > +;; should not pay for the lane mask that index/count operations need later.
> > +(define_expand "@aarch64_addhn_scalar<mode>"
> > + [(match_operand:DI 0 "register_operand")
> > + (match_operand:VQ_I 1 "register_operand")]
> > + "TARGET_SIMD"
> > +{
> > + rtx wide = gen_lowpart (<VADDHN_MASK_WIDE>mode, operands[1]);
> > + rtx narrow = gen_reg_rtx (<VADDHN_MASK_NARROW>mode);
> > + emit_insn (gen_aarch64_addhn<vaddhn_mask_wide> (narrow, wide,
> wide));
> > +
> > + emit_move_insn (operands[0], gen_lowpart (DImode, narrow));
> > + DONE;
> > +})
> > +
> > +;; Compress a 128-bit Adv.SIMD boolean vector into a 64-bit scalar bitmap.
> > +;; The raw ADDHN result is sparse. After masking with
> <vaddhn_lane_mask>,
> > +;; each original lane has exactly one bit in the scalar result, so ctz and
> > +;; popcount can recover a lane index or count.
> > +;; This is the exit-block form used by INDEX_FIRST_ACTIVE and
> COUNT_ACTIVE.
> > +(define_expand "@aarch64_addhn_mask_scalar<mode>"
> > + [(match_operand:DI 0 "register_operand")
> > + (match_operand:VQ_I 1 "register_operand")]
> > + "TARGET_SIMD"
> > +{
> > + rtx val = gen_reg_rtx (DImode);
> > + emit_insn (gen_aarch64_addhn_scalar<mode> (val, operands[1]));
> > + emit_move_insn (operands[0],
> > + gen_rtx_AND (DImode, val,
> > + gen_int_mode (<vaddhn_lane_mask>, DImode)));
> > + DONE;
> > +})
> > +
> > +(define_expand "count_active_<mode>"
> > + [(match_operand:DI 0 "register_operand")
> > + (match_operand:VQ_I 1 "register_operand")]
> > + "TARGET_SIMD"
> > +{
> > + /* For example: addhn v0.2s, v1.2d, v1.2d; and; cnt. */
> > + rtx mask = gen_reg_rtx (DImode);
> > + emit_insn (gen_aarch64_addhn_mask_scalar<mode> (mask,
> operands[1]));
> > + emit_insn (gen_popcountdi2 (operands[0], mask));
> > + DONE;
> > +})
> > +
> > +(define_expand "index_first_active_<mode>"
> > + [(match_operand:DI 0 "register_operand")
> > + (match_operand:VQ_I 1 "register_operand")]
> > + "TARGET_SIMD"
> > +{
> > + /* For example: addhn v0.2s, v1.2d, v1.2d; and; rbit; clz; lsr. */
> > + rtx mask = gen_reg_rtx (DImode);
> > + rtx bitno = gen_reg_rtx (DImode);
> > + emit_insn (gen_aarch64_addhn_mask_scalar<mode> (mask,
> operands[1]));
> > + emit_insn (gen_ctzdi2 (bitno, mask));
> > + emit_move_insn (operands[0],
> > + gen_rtx_LSHIFTRT (DImode, bitno,
> > + GEN_INT (exact_log2
> > + (GET_MODE_UNIT_BITSIZE
> > + (<MODE>mode) / 2))));
> > + DONE;
> > +})
> > +
> > +(define_expand "extract_first_<mode>"
> > + [(match_operand:<VEL> 0 "register_operand")
> > + (match_operand:<V_INT_EQUIV> 1 "register_operand")
> > + (match_operand:VQ_NO2E 2 "register_operand")]
> > + "TARGET_SIMD"
> > +{
> > + /* For example: addhn; ctz; dup; add; tbl. */
> > + rtx index = gen_reg_rtx (DImode);
> > + emit_insn (gen_index_first_active_<v_int_equiv> (index, operands[1]));
> > +
> > + if (TARGET_LUT && <MODE>mode == V16QImode)
> > + {
> > + rtx index_vec = gen_reg_rtx (V8QImode);
> > + emit_move_insn (gen_lowpart (DImode, index_vec), index);
> > +
> > + rtx selected = gen_reg_rtx (V16QImode);
> > + emit_insn (gen_aarch64_lutv16qiv8qi
> > + (selected, operands[2], index_vec, const0_rtx, GEN_INT (4)));
> > +
> > + emit_move_insn (operands[0], gen_lowpart (<VEL>mode, selected));
> > + DONE;
> > + }
> > +
> > + if (TARGET_LUT && <MODE>mode == V8HImode)
> > + {
> > + rtx index_vec = gen_reg_rtx (V8QImode);
> > + emit_move_insn (gen_lowpart (DImode, index_vec), index);
> > +
> > + rtx table = gen_reg_rtx (V2x8HImode);
> > + rtx first = simplify_gen_subreg (V8HImode, table, V2x8HImode, 0);
> > + emit_move_insn (first, operands[2]);
> > +
> > + rtx zero = force_reg (V8HImode, CONST0_RTX (V8HImode));
> > + rtx second = simplify_gen_subreg (V8HImode, table, V2x8HImode, 16);
> > + emit_move_insn (second, zero);
> > +
> > + rtx selected = gen_reg_rtx (V8HImode);
> > + emit_insn (gen_aarch64_lutv2x8hiv8qi
> > + (selected, table, index_vec, const0_rtx, GEN_INT (4)));
> > +
> > + emit_move_insn (operands[0], gen_lowpart (<VEL>mode, selected));
> > + DONE;
> > + }
> > +
>
> Drive-by comment:
> Can you add somewhere in the commit message or in the comments how the
> LUT instructions are used here?
Sure, there were lots of patterns so wasn't quite sure which to highlight.
LUT here is used when we have to extract an arbitrary element from a vector.
The problem is all our vector rotate instructions like EXT, etc require a
constant
Immediate. So extracting lane `i` from a[i] is hard. (note this isn't used for
memory,
so this is about extracting a vector element, which is what the comment is
trying to say.)
And so to do this for adv. Simd we need to use TBL, but then the complicated
part is
computing the byte index for the TBL lookup. This normally needs a literal
which carries
the "mask" and then you multiply that with the calculated index.
e.g. without lut you get
.L3:
ldr q31, [x1, x3]
ldr q30, [x2, x1]
add x1, x1, 16
and v30.16b, v30.16b, v31.16b
cmhi v31.16b, v30.16b, v29.16b
addhn v31.8b, v31.8h, v31.8h
umov x0, v31.d[0]
cbz x0, .L8
and x0, x0, 1229782938247303441
adrp x1, .LANCHOR1
rbit x0, x0
clz x0, x0
fmov d31, x0
ldr q29, [x1, #:lo12:.LANCHOR1]
ushr d31, d31, 2
dup v31.16b, v31.b[0]
add v31.16b, v31.16b, v29.16b
tbl v30.16b, {v30.16b}, v31.16b
fmov w0, s30
ret
for
__attribute__ ((noipa, noinline))
unsigned char f1 (void)
{
unsigned char res = 0;
for (unsigned int i = 0; i < N; ++i)
{
unsigned char x = a[i] & b[i];
res = x;
if (x > 5)
break;
}
return res;
}
With LUT
.L3:
ldr q31, [x1, x3]
ldr q30, [x2, x1]
add x1, x1, 16
and v30.16b, v30.16b, v31.16b
cmhi v31.16b, v30.16b, v29.16b
addhn v31.8b, v31.8h, v31.8h
umov x0, v31.d[0]
cbz x0, .L8
and x0, x0, 1229782938247303441
rbit x0, x0
clz x0, x0
fmov d31, x0
ushr d31, d31, 2
luti4 v30.16b, {v30.16b}, v31[0]
fmov w0, s30
ret
which simplifies the mask creation for lookup.
But where to comment....
Thanks,
Tamar
>
> Thanks,
> Kyrill
>
> > + /* The live-load fast path in tree-vect-loop.cc avoids this sequence for
> > + memory values by reloading the selected scalar element. This
> > expander is
> > + for values that are already in vector registers, such as the register
> > + live-out cases in gcc.target/aarch64/noeffect11.c.
> > +
> > + INDEX_FIRST_ACTIVE gives the active element number. Turn it into a
> byte
> > + offset, duplicate the offset across a byte vector and add {0, 1, ...
> > 15}.
> > + TBL then moves the selected element's bytes down to lane 0. This
> > avoids
> > + transferring both 64-bit halves to general registers before choosing
> > which
> > + half contains the first active element. */
> > + unsigned int elt_bytes = GET_MODE_UNIT_SIZE (<MODE>mode);
> > + if (elt_bytes > 1)
> > + {
> > + rtx byte_index = gen_reg_rtx (DImode);
> > + emit_move_insn (byte_index,
> > + gen_rtx_ASHIFT (DImode, index,
> > + GEN_INT (exact_log2 (elt_bytes))));
> > + index = byte_index;
> > + }
> > +
> > + rtx byte_index_vec = gen_reg_rtx (V16QImode);
> > + emit_insn (gen_aarch64_simd_dupv16qi (byte_index_vec,
> > + gen_lowpart (QImode, index)));
> > +
> > + rtvec v = rtvec_alloc (16);
> > + for (unsigned int i = 0; i < 16; ++i)
> > + RTVEC_ELT (v, i) = GEN_INT (i);
> > + rtx byte_numbers = force_reg (V16QImode,
> > + gen_rtx_CONST_VECTOR (V16QImode, v));
> > +
> > + rtx selector = gen_reg_rtx (V16QImode);
> > + emit_insn (gen_addv16qi3 (selector, byte_index_vec, byte_numbers));
> > +
> > + rtx selected = gen_reg_rtx (V16QImode);
> > + emit_insn (gen_aarch64_qtbl1v16qi (selected,
> > + gen_lowpart (V16QImode, operands[2]),
> > + selector));
> > +
> > + emit_move_insn (operands[0], gen_lowpart (<VEL>mode, selected));
> > + DONE;
> > +})
> > +
> > +(define_expand "extract_first_<mode>"
> > + [(match_operand:<VEL> 0 "register_operand")
> > + (match_operand:<V_INT_EQUIV> 1 "register_operand")
> > + (match_operand:VQ_2E 2 "register_operand")]
> > + "TARGET_SIMD"
> > +{
> > + /* For example: addhn; ctz; umov xlo; umov xhi; csel. */
> > + rtx index = gen_reg_rtx (DImode);
> > + emit_insn (gen_index_first_active_<v_int_equiv> (index, operands[1]));
> > +
> > + /* V2DI/V2DF only need to choose between the two 64-bit halves. This
> keeps
> > + the sequence in the scalar integer domain after INDEX_FIRST_ACTIVE and
> > + avoids building a vector table control. */
> > + rtx lo = gen_reg_rtx (DImode);
> > + emit_insn (gen_aarch64_get_lanev2di (lo, gen_lowpart (V2DImode,
> operands[2]),
> > + aarch64_endian_lane_rtx (V2DImode,
> > + 0)));
> > +
> > + rtx hi = gen_reg_rtx (DImode);
> > + emit_insn (gen_aarch64_get_lanev2di (hi, gen_lowpart (V2DImode,
> operands[2]),
> > + aarch64_endian_lane_rtx (V2DImode,
> > + 1)));
> > +
> > + rtx cc_reg = aarch64_gen_compare_reg (NE, index, const0_rtx);
> > + rtx cmp = gen_rtx_NE (VOIDmode, cc_reg, const0_rtx);
> > + rtx data = gen_reg_rtx (DImode);
> > + emit_insn (gen_rtx_SET (data, gen_rtx_IF_THEN_ELSE (DImode, cmp, hi,
> lo)));
> > +
> > + emit_move_insn (operands[0], gen_lowpart (<VEL>mode, data));
> > + DONE;
> > +})
> > +
> > +(define_expand "take_<take_active>_active_<mode>"
> > + [(set (match_operand:VQ_I 0 "register_operand")
> > + (neg:VQ_I
> > + (TAKE_ACTIVE_CMP:VQ_I
> > + (match_operand:VQ_I 1 "register_operand")
> > + (match_dup 1))))]
> > + "TARGET_SIMD"
> > +{
> > + /* For example: addhn; ctz; dup; cmhs. */
> > + rtx index = gen_reg_rtx (DImode);
> > + emit_insn (gen_index_first_active_<mode> (index, operands[1]));
> > +
> > + rtx elt_index = gen_lowpart (<VEL>mode, index);
> > + rtx index_vec = gen_reg_rtx (<MODE>mode);
> > + emit_insn (gen_aarch64_simd_dup<mode> (index_vec, elt_index));
> > +
> > + rtvec v = rtvec_alloc (<nunits>);
> > + for (unsigned int i = 0; i < <nunits>; ++i)
> > + RTVEC_ELT (v, i) = GEN_INT (i);
> > + rtx lane_numbers = force_reg (<MODE>mode,
> > + gen_rtx_CONST_VECTOR (<MODE>mode, v));
> > +
> > + emit_insn (gen_vec_cmp<mode><v_int_equiv> (operands[0],
> > + gen_rtx_<CMP> (VOIDmode,
> > + lane_numbers,
> > + index_vec),
> > + lane_numbers, index_vec));
> > + DONE;
> > +})
> > +
> > +;; If a vector compare is used only to feed a vector branch, use an SVE
> > +;; predicate compare and PTEST instead of materializing and reducing an
> > +;; Adv.SIMD mask. Keep this as a combine-time split rather than doing it
> > in
> > +;; the vec_cbranch expander, since the compare mask might also be needed
> later
> > +;; for live-value extraction. In that case the ADDHN result remains live
> > after
> > +;; the branch and combine cannot consume the reduction into this pattern.
> > +;;
> > +;; This must be a define_split rather than a define_insn_and_split. The
> > split
> > +;; needs the original compare producer to be the previous insn; if that
> > stops
> > +;; being true, FAILing is correct during combine, but would be an ICE if a
> > +;; recognized "#" insn survived until final.
> > +(define_split
> > + [(set (pc)
> > + (if_then_else
> > + (match_operator 0 "aarch64_equality_operator"
> > + [(subreg:DI
> > + (truncate:<VADDHN_MASK_NARROW>
> > + (ashiftrt:<VADDHN_MASK_WIDE>
> > + (plus:<VADDHN_MASK_WIDE>
> > + (subreg:<VADDHN_MASK_WIDE>
> > + (match_operand:VQ_I 1 "register_operand") 0)
> > + (subreg:<VADDHN_MASK_WIDE>
> > + (match_dup 1) 0))
> > + (match_operand:<VADDHN_MASK_WIDE> 3
> > + "aarch64_simd_shift_imm_vec_exact_top")))
> > + 0)
> > + (const_int 0)])
> > + (label_ref (match_operand 2 ""))
> > + (pc)))]
> > + "TARGET_SIMD
> > + && TARGET_SVE
> > + && can_create_pseudo_p ()
> > + && known_eq (128, GET_MODE_BITSIZE (<MODE>mode))"
> > + [(const_int 0)]
> > +{
> > + if (find_regno_note (curr_insn, REG_DEAD, REGNO (operands[1])) ==
> NULL_RTX)
> > + FAIL;
> > +
> > + rtx_insn *prev = prev_nonnote_nondebug_insn (curr_insn);
> > + rtx set = prev ? single_set (prev) : NULL_RTX;
> > + if (!set || !rtx_equal_p (SET_DEST (set), operands[1]))
> > + FAIL;
> > +
> > + rtx src = SET_SRC (set);
> > + if (GET_CODE (src) != NEG || !COMPARISON_P (XEXP (src, 0)))
> > + FAIL;
> > +
> > + rtx cmp = XEXP (src, 0);
> > + rtx_code cmp_code = GET_CODE (cmp);
> > + machine_mode full_mode = aarch64_full_sve_mode (<VEL>mode).require
> ();
> > +
> > + rtx in1 = force_lowpart_subreg (full_mode, XEXP (cmp, 0),
> <MODE>mode);
> > + rtx in2;
> > + if (CONST0_RTX (<MODE>mode) == XEXP (cmp, 1))
> > + in2 = CONST0_RTX (full_mode);
> > + else
> > + in2 = force_lowpart_subreg (full_mode, XEXP (cmp, 1), <MODE>mode);
> > +
> > + machine_mode pred_mode = aarch64_sve_pred_mode (full_mode);
> > + unsigned int vl = exact_div (GET_MODE_BITSIZE (<MODE>mode),
> 8).to_constant ();
> > + rtx ptrue = aarch64_gen_shareable_ptrue (VNx16BImode, vl);
> > + rtx cast_ptrue = gen_lowpart (pred_mode, ptrue);
> > + rtx hint = gen_int_mode (SVE_MAYBE_NOT_PTRUE, SImode);
> > + rtx tmp = gen_reg_rtx (pred_mode);
> > + emit_insn (gen_aarch64_pred_cmp_ptest (cmp_code, full_mode, tmp,
> ptrue,
> > + in1, in2, cast_ptrue, hint,
> > + cast_ptrue, hint));
> > +
> > + rtx cc_reg = gen_rtx_REG (CC_NZCmode, CC_REGNUM);
> > + rtx cmp_reg = gen_rtx_fmt_ee (GET_CODE (operands[0]), VOIDmode,
> > + cc_reg, const0_rtx);
> > + emit_jump_insn (gen_aarch64_bcond (cmp_reg, cc_reg, operands[2]));
> > + DONE;
> > +})
> > +
> > ;; Define vec_cbranch_any and vec_cbranch_all
> > ;; Vector comparison and branch for Adv. SIMD Integer types using SVE
> > ;; instructions.
> > @@ -4526,62 +4848,29 @@ (define_expand "<optab><mode>"
> > CBRANCH_CMP))]
> > "TARGET_SIMD"
> > {
> > - auto code = GET_CODE (operands[0]);
> > - if (TARGET_SVE)
> > - {
> > - machine_mode full_mode = aarch64_full_sve_mode
> (<VEL>mode).require ();
> > -
> > - rtx in1 = force_lowpart_subreg (full_mode, operands[1],
> <MODE>mode);
> > - rtx in2;
> > - if (CONST0_RTX (<MODE>mode) == operands[2])
> > - in2 = CONST0_RTX (full_mode);
> > - else
> > - in2 = force_lowpart_subreg (full_mode, operands[2], <MODE>mode);
> > -
> > - unsigned lanes
> > - = exact_div (GET_MODE_BITSIZE (<MODE>mode), 8).to_constant ();
> > - machine_mode pred_mode = aarch64_sve_pred_mode (full_mode);
> > - rtx ptrue = aarch64_ptrue_reg (VNx16BImode, lanes);
> > - rtx hint = gen_int_mode (SVE_MAYBE_NOT_PTRUE, SImode);
> > -
> > - rtx tmp = gen_reg_rtx (pred_mode);
> > - rtx cast_ptrue = gen_lowpart (pred_mode, ptrue);
> > -
> > - if (FLOAT_MODE_P (full_mode))
> > - {
> > - aarch64_expand_sve_vec_cmp<sve_cmp_suff> (tmp, code, in1, in2);
> > - emit_insn (gen_and3 (pred_mode, tmp, tmp, cast_ptrue));
> > - emit_insn (gen_aarch64_ptest (pred_mode, ptrue, cast_ptrue, hint,
> > - tmp));
> > - }
> > - else
> > - emit_insn (gen_aarch64_pred_cmp_ptest (code, full_mode, tmp, ptrue,
> in1,
> > - in2, cast_ptrue, hint,
> > - cast_ptrue, hint));
> > -
> > - rtx cc_reg = gen_rtx_REG (CC_NZCmode, CC_REGNUM);
> > - rtx cmp_reg = gen_rtx_<cbranch_op> (VOIDmode, cc_reg, const0_rtx);
> > - emit_jump_insn (gen_aarch64_bcond (cmp_reg, cc_reg, operands[3]));
> > - DONE;
> > - }
> > -
> > rtx tmp = gen_reg_rtx (<V_INT_EQUIV>mode);
> > emit_insn (gen_vec_cmp<mode><v_int_equiv> (tmp, operands[0],
> operands[1],
> > operands[2]));
> >
> > - /* For 128-bit vectors we need a reduction to 64-bit first. */
> > - if (known_eq (128, GET_MODE_BITSIZE (<MODE>mode)))
> > + rtx val = gen_reg_rtx (DImode);
> > + rtx addhn_scalar
> > + = maybe_gen_aarch64_addhn_scalar (<V_INT_EQUIV>mode, val, tmp);
> > + if (known_eq (128, GET_MODE_BITSIZE (<MODE>mode)) &&
> addhn_scalar)
> > + emit_insn (addhn_scalar);
> > + else
> > {
> > - /* Always reduce using a V4SI. */
> > - rtx reduc = gen_lowpart (V4SImode, tmp);
> > - rtx res = gen_reg_rtx (V4SImode);
> > - emit_insn (gen_aarch64_umaxpv4si (res, reduc, reduc));
> > - emit_move_insn (tmp, gen_lowpart (<V_INT_EQUIV>mode, res));
> > + /* For 128-bit vectors we need a reduction to 64-bit first. */
> > + if (known_eq (128, GET_MODE_BITSIZE (<MODE>mode)))
> > + {
> > + /* Always reduce using a V4SI. */
> > + rtx reduc = gen_lowpart (V4SImode, tmp);
> > + rtx res = gen_reg_rtx (V4SImode);
> > + emit_insn (gen_aarch64_umaxpv4si (res, reduc, reduc));
> > + emit_move_insn (tmp, gen_lowpart (<V_INT_EQUIV>mode, res));
> > + }
> > + emit_move_insn (val, gen_lowpart (DImode, tmp));
> > }
> >
> > - rtx val = gen_reg_rtx (DImode);
> > - emit_move_insn (val, gen_lowpart (DImode, tmp));
> > -
> > rtx cc_reg = aarch64_gen_compare_reg (<cbranch_op>, val, const0_rtx);
> > rtx cmp_rtx = gen_rtx_fmt_ee (<cbranch_op>, DImode, cc_reg, const0_rtx);
> > emit_jump_insn (gen_aarch64_bcond (cmp_rtx, cc_reg, operands[3]));
> > diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc
> > index
> 8e62ab6a72e0043fffd16d3d187f18da7fd41d4e..77e29815f224a9e3bfc2ab
> 3c5662c5d0aedf753e 100644
> > --- a/gcc/config/aarch64/aarch64.cc
> > +++ b/gcc/config/aarch64/aarch64.cc
> > @@ -25316,6 +25316,28 @@ aarch64_gen_shareable_zero
> (machine_mode mode)
> > return lowpart_subreg (mode, reg, GET_MODE (reg));
> > }
> >
> > +/* Return a shareable predicate register whose first VL bytes are active,
> > + viewed in mode MODE.
> > + This is for fixed-width Adv.SIMD sequences that want to use SVE flags
> > for
> > + a vector branch. Hoisting the predicate constant avoids recreating it
> > in
> > + every loop iteration. */
> > +
> > +rtx
> > +aarch64_gen_shareable_ptrue (machine_mode mode, unsigned int vl)
> > +{
> > + gcc_assert (vl == 8 || vl == 16);
> > +
> > + rtx_vector_builder builder (VNx16BImode, vl, 2);
> > + for (unsigned int i = 0; i < vl; ++i)
> > + builder.quick_push (CONST1_RTX (BImode));
> > + for (unsigned int i = 0; i < vl; ++i)
> > + builder.quick_push (CONST0_RTX (BImode));
> > +
> > + rtx reg = aarch64_get_shareable_reg (&cfun->machine-
> >advsimd_ptrue_insn,
> > + builder.build ());
> > + return lowpart_subreg (mode, reg, GET_MODE (reg));
> > +}
> > +
> > /* INSN is some form of extension or shift that can be split into a
> > permutation involving a shared zero. Return true if we should
> > perform such a split.
> > @@ -28913,7 +28935,7 @@ aarch64_vectorize_vec_perm_const
> (machine_mode vmode, machine_mode op_mode,
> > {
> > d.op1 = op1;
> > if (d.op1 && !register_operand (d.op1, op_mode))
> > - d.op1 = force_reg (op_mode, d.op1);
> > + d.op1 = force_reg (op_mode, d.op1);
> > }
> > d.testing_p = !target;
> >
> > @@ -28926,6 +28948,7 @@ aarch64_vectorize_vec_perm_const
> (machine_mode vmode, machine_mode op_mode,
> >
> > return ret;
> > }
> > +
> > /* Generate a byte permute mask for a register of mode MODE,
> > which has NUNITS units. */
> >
> > diff --git a/gcc/config/aarch64/aarch64.h b/gcc/config/aarch64/aarch64.h
> > index
> fdedb04391127c5a1b821ed5d4b399c1c28d9f31..0673c8a64f7097e2d2ec4
> a155edeb12dfe72e957 100644
> > --- a/gcc/config/aarch64/aarch64.h
> > +++ b/gcc/config/aarch64/aarch64.h
> > @@ -1249,6 +1249,10 @@ typedef struct GTY (()) machine_function
> > can be reused by other code. */
> > rtx_insn *advsimd_zero_insn;
> >
> > + /* Like advsimd_zero_insn, but for a predicate with a fixed number of
> > + leading true bytes. */
> > + rtx_insn *advsimd_ptrue_insn;
> > +
> > /* During SEH output, this is non-null. */
> > struct seh_frame_state * GTY ((skip (""))) seh;
> >
> > diff --git a/gcc/config/aarch64/iterators.md
> b/gcc/config/aarch64/iterators.md
> > index
> b0571460ec7ded9252bc2a216b6cd8867e6a595b..af417b51ccf31bf56d4e4
> f368979971c05e0b482 100644
> > --- a/gcc/config/aarch64/iterators.md
> > +++ b/gcc/config/aarch64/iterators.md
> > @@ -2181,6 +2181,25 @@ (define_mode_attr Vdblw [(V8QI "v4hi")
> (V16QI "v8hi")
> > (V4HI "v2si") (V8HI "v4si")
> > (V2SI "di") (V4SI "v2di")])
> >
> > +;; Source mode for using ADDHN to compress a boolean mask to a scalar.
> > +(define_mode_attr VADDHN_MASK_WIDE [(V16QI "V8HI") (V8HI "V4SI")
> > + (V4SI "V2DI") (V2DI "V2DI")])
> > +
> > +;; Result mode for using ADDHN to compress a boolean mask to a scalar.
> > +(define_mode_attr VADDHN_MASK_NARROW [(V16QI "V8QI") (V8HI
> "V4HI")
> > + (V4SI "V2SI") (V2DI "V2SI")])
> > +
> > +;; Source mode suffix for using ADDHN to compress a boolean mask to a
> scalar.
> > +(define_mode_attr vaddhn_mask_wide [(V16QI "v8hi") (V8HI "v4si")
> > + (V4SI "v2di") (V2DI "v2di")])
> > +
> > +;; Scalar mask for the useful bits produced by using ADDHN to compress a
> > +;; boolean mask to a scalar.
> > +(define_mode_attr vaddhn_lane_mask [(V16QI "0x1111111111111111")
> > + (V8HI "0x0101010101010101")
> > + (V4SI "0x0001000100010001")
> > + (V2DI "0x0000000100000001")])
> > +
> > (define_mode_attr VQUADW [(V8QI "V4SI") (V16QI "V8SI")
> > (V4HI "V2DI") (V8HI "V4DI")])
> >
> > @@ -3237,6 +3256,9 @@ (define_code_iterator SAT_MINUS [ss_minus
> us_minus])
> > ;; Comparison operators for <F>CM.
> > (define_code_iterator COMPARISONS [lt le eq ge gt])
> >
> > +;; Comparison operators for TAKE_{EXCL,INCL}_ACTIVE.
> > +(define_code_iterator TAKE_ACTIVE_CMP [lt le])
> > +
> > ;; Unsigned comparison operators.
> > (define_code_iterator UCOMPARISONS [ltu leu geu gtu])
> >
> > @@ -3374,6 +3396,9 @@ (define_code_attr cmp_op [(lt "lt")
> > (geu "hs")
> > (gtu "hi")])
> >
> > +;; The TAKE_ACTIVE optab suffix associated with an rtl comparison code.
> > +(define_code_attr take_active [(lt "excl") (le "incl")])
> > +
> > (define_code_attr inv_cmp_op [(lt "ge")
> > (le "gt")
> > (eq "ne")
> > diff --git a/gcc/testsuite/gcc.target/aarch64/noeffect11.c
> b/gcc/testsuite/gcc.target/aarch64/noeffect11.c
> > index
> 82c2f00c7cb16df22fa051b8d5d421863321e464..875e3ca184d925691eaf9
> 9d3d45b8cfab8b6f3e6 100644
> > --- a/gcc/testsuite/gcc.target/aarch64/noeffect11.c
> > +++ b/gcc/testsuite/gcc.target/aarch64/noeffect11.c
> > @@ -8,7 +8,10 @@
> > ** ...
> > ** cmhi v[0-9]+\.2d, v[0-9]+\.2d, v[0-9]+\.2d
> > ** ...
> > -** umaxp v[0-9]+\.4s, v[0-9]+\.4s, v[0-9]+\.4s
> > +** addhn v[0-9]+\.2s, v[0-9]+\.2d, v[0-9]+\.2d
> > +** addhn v[0-9]+\.2s, v[0-9]+\.2d, v[0-9]+\.2d
> > +** fmov x[0-9]+, d[0-9]+
> > +** cbz x[0-9]+, \.L[0-9]+
> > ** ...
> > ** ldr x[0-9]+, \[x[0-9]+, x[0-9]+\]
> > ** ldr x[0-9]+, \[x[0-9]+, x[0-9]+\]
> > @@ -34,7 +37,10 @@ int f1 (const unsigned long *restrict a, const unsigned
> long *b, int n)
> > ** ...
> > ** cmhi v[0-9]+\.2d, v[0-9]+\.2d, v[0-9]+\.2d
> > ** ...
> > -** umaxp v[0-9]+\.4s, v[0-9]+\.4s, v[0-9]+\.4s
> > +** addhn v[0-9]+\.2s, v[0-9]+\.2d, v[0-9]+\.2d
> > +** addhn v[0-9]+\.2s, v[0-9]+\.2d, v[0-9]+\.2d
> > +** fmov x[0-9]+, d[0-9]+
> > +** cbz x[0-9]+, \.L[0-9]+
> > ** ...
> > ** ldr x[0-9]+, \[x[0-9]+, x[0-9]+\]
> > ** ldr x[0-9]+, \[x[0-9]+, x[0-9]+\]
> > diff --git a/gcc/testsuite/gcc.target/aarch64/noeffect12.c
> b/gcc/testsuite/gcc.target/aarch64/noeffect12.c
> > new file mode 100644
> > index
> 0000000000000000000000000000000000000000..a42d67b56c47ed0749
> 7a40ea4758f07aea2bae4a
> > --- /dev/null
> > +++ b/gcc/testsuite/gcc.target/aarch64/noeffect12.c
> > @@ -0,0 +1,64 @@
> > +/* { dg-do compile } */
> > +/* { dg-options "-O3 -march=armv8-a" } */
> > +/* { dg-final { check-function-bodies "**" "" "" } } */
> > +
> > +#define N 1008
> > +
> > +unsigned char a[N] = {0};
> > +unsigned char b[N] = {0};
> > +unsigned int c[N] = {0};
> > +unsigned int d[N] = {0};
> > +
> > +/*
> > +** f1:
> > +** ...
> > +** cmhi v[0-9]+\.16b, v[0-9]+\.16b, v[0-9]+\.16b
> > +** addhn v[0-9]+\.8b, v[0-9]+\.8h, v[0-9]+\.8h
> > +** umov x[0-9]+, v[0-9]+\.d\[0\]
> > +** cbz x[0-9]+, \.L[0-9]+
> > +** ...
> > +** tbl v[0-9]+\.16b, \{v[0-9]+\.16b\}, v[0-9]+\.16b
> > +** fmov w0, s[0-9]+
> > +** ret
> > +** ...
> > +*/
> > +__attribute__ ((noipa, noinline))
> > +unsigned char f1 (void)
> > +{
> > + unsigned char res = 0;
> > + for (unsigned int i = 0; i < N; ++i)
> > + {
> > + unsigned char x = a[i] & b[i];
> > + res = x;
> > + if (x > 5)
> > + break;
> > + }
> > + return res;
> > +}
> > +
> > +/*
> > +** f2:
> > +** ...
> > +** cmhi v[0-9]+\.4s, v[0-9]+\.4s, v[0-9]+\.4s
> > +** addhn v[0-9]+\.2s, v[0-9]+\.2d, v[0-9]+\.2d
> > +** umov x[0-9]+, v[0-9]+\.d\[0\]
> > +** cbz x[0-9]+, \.L[0-9]+
> > +** ...
> > +** tbl v[0-9]+\.16b, \{v[0-9]+\.16b\}, v[0-9]+\.16b
> > +** fmov w0, s[0-9]+
> > +** ret
> > +** ...
> > +*/
> > +__attribute__ ((noipa, noinline))
> > +unsigned int f2 (void)
> > +{
> > + unsigned int res = 0;
> > + for (unsigned int i = 0; i < 1000; ++i)
> > + {
> > + unsigned int x = c[i] & d[i];
> > + res = x;
> > + if (x > 5)
> > + break;
> > + }
> > + return res;
> > +}
> > diff --git a/gcc/testsuite/gcc.target/aarch64/noeffect13-run.c
> b/gcc/testsuite/gcc.target/aarch64/noeffect13-run.c
> > new file mode 100644
> > index
> 0000000000000000000000000000000000000000..9e9eb715c83f6bda79
> 81f36d1682908a8380c114
> > --- /dev/null
> > +++ b/gcc/testsuite/gcc.target/aarch64/noeffect13-run.c
> > @@ -0,0 +1,62 @@
> > +/* { dg-do run { target aarch64_lut_hw } } */
> > +/* { dg-options "-O3 -march=armv9-a+lut -mautovec-preference=asimd-
> only" } */
> > +
> > +#define N 1008
> > +
> > +unsigned char a[N];
> > +unsigned char b[N];
> > +unsigned short c[N];
> > +unsigned short d[N];
> > +
> > +__attribute__ ((noipa, noinline))
> > +unsigned char f1 (void)
> > +{
> > + unsigned char res = 0;
> > + for (unsigned int i = 0; i < N; ++i)
> > + {
> > + unsigned char x = a[i] & b[i];
> > + res = x;
> > + if (x > 5)
> > + break;
> > + }
> > + return res;
> > +}
> > +
> > +__attribute__ ((noipa, noinline))
> > +unsigned short f2 (void)
> > +{
> > + unsigned short res = 0;
> > + for (unsigned int i = 0; i < N; ++i)
> > + {
> > + unsigned short x = c[i] & d[i];
> > + res = x;
> > + if (x > 5)
> > + break;
> > + }
> > + return res;
> > +}
> > +
> > +int
> > +main (void)
> > +{
> > + for (unsigned int i = 0; i < N; ++i)
> > + {
> > + a[i] = 1;
> > + b[i] = 1;
> > + c[i] = 1;
> > + d[i] = 1;
> > + }
> > +
> > + if (f1 () != 1 || f2 () != 1)
> > + __builtin_abort ();
> > +
> > + a[13] = 7;
> > + b[13] = 7;
> > + c[11] = 9;
> > + d[11] = 9;
> > +
> > + if (f1 () != 7 || f2 () != 9)
> > + __builtin_abort ();
> > +
> > + return 0;
> > +}
> > diff --git a/gcc/testsuite/gcc.target/aarch64/noeffect13.c
> b/gcc/testsuite/gcc.target/aarch64/noeffect13.c
> > new file mode 100644
> > index
> 0000000000000000000000000000000000000000..502126d6c005775b0f
> 08c8807b3dda14c6372d34
> > --- /dev/null
> > +++ b/gcc/testsuite/gcc.target/aarch64/noeffect13.c
> > @@ -0,0 +1,66 @@
> > +/* { dg-do compile { target aarch64_asm_lut_ok } } */
> > +/* { dg-options "-O3 -march=armv9-a+lut -mautovec-preference=asimd-
> only" } */
> > +/* { dg-final { check-function-bodies "**" "" "" } } */
> > +
> > +#define N 1008
> > +
> > +unsigned char a[N] = {0};
> > +unsigned char b[N] = {0};
> > +unsigned short c[N] = {0};
> > +unsigned short d[N] = {0};
> > +
> > +/*
> > +** f1:
> > +** ...
> > +** cmhi v[0-9]+\.16b, v[0-9]+\.16b, v[0-9]+\.16b
> > +** ...
> > +** addhn v[0-9]+\.8b, v[0-9]+\.8h, v[0-9]+\.8h
> > +** fmov x[0-9]+, d[0-9]+
> > +** ...
> > +** luti4 v[0-9]+\.16b, \{v[0-9]+\.16b\}, v[0-9]+\[0\]
> > +** fmov w0, s[0-9]+
> > +** ret
> > +** ...
> > +*/
> > +__attribute__ ((noipa, noinline))
> > +unsigned char f1 (void)
> > +{
> > + unsigned char res = 0;
> > + for (unsigned int i = 0; i < N; ++i)
> > + {
> > + unsigned char x = a[i] & b[i];
> > + res = x;
> > + if (x > 5)
> > + break;
> > + }
> > + return res;
> > +}
> > +
> > +/*
> > +** f2:
> > +** ...
> > +** cmhi v[0-9]+\.8h, v[0-9]+\.8h, v[0-9]+\.8h
> > +** ...
> > +** addhn v[0-9]+\.4h, v[0-9]+\.4s, v[0-9]+\.4s
> > +** fmov x[0-9]+, d[0-9]+
> > +** ...
> > +** movi v[0-9]+\.4s, 0
> > +** ...
> > +** luti4 v[0-9]+\.8h, \{v[0-9]+\.8h, v[0-9]+\.8h\}, v[0-9]+\[0\]
> > +** fmov w0, s[0-9]+
> > +** ret
> > +** ...
> > +*/
> > +__attribute__ ((noipa, noinline))
> > +unsigned short f2 (void)
> > +{
> > + unsigned short res = 0;
> > + for (unsigned int i = 0; i < N; ++i)
> > + {
> > + unsigned short x = c[i] & d[i];
> > + res = x;
> > + if (x > 5)
> > + break;
> > + }
> > + return res;
> > +}
> > diff --git a/gcc/testsuite/gcc.target/aarch64/noeffect2.c
> b/gcc/testsuite/gcc.target/aarch64/noeffect2.c
> > index
> 08c531fb18c817b018776a88db4f62eb03b001cb..bcd8848038527d3af1a4
> 966ef55580e262b5d871 100644
> > --- a/gcc/testsuite/gcc.target/aarch64/noeffect2.c
> > +++ b/gcc/testsuite/gcc.target/aarch64/noeffect2.c
> > @@ -13,7 +13,7 @@ int b[N] = {0};
> > ** ldr q[0-9]+, \[x[0-9]+, x[0-9]+\]
> > ** add x[0-9]+, x[0-9]+, 16
> > ** cmgt v[0-9]+\.4s, v[0-9]+\.4s, v[0-9]+\.4s
> > -** umaxp v[0-9]+\.4s, v[0-9]+\.4s, v[0-9]+\.4s
> > +** addhn v[0-9]+\.2s, v[0-9]+\.2d, v[0-9]+\.2d
> > ** fmov x[0-9]+, d[0-9]+
> > ** cbz x[0-9]+, \.L[0-9]+
> > ** mov w0, 1
> > diff --git a/gcc/testsuite/gcc.target/aarch64/noeffect3.c
> b/gcc/testsuite/gcc.target/aarch64/noeffect3.c
> > index
> 886ad7bda2f3ebd5048cc618bcf4fe473cd2f820..f4ece387fc355b5a7d8c23b
> 416400ec81da47051 100644
> > --- a/gcc/testsuite/gcc.target/aarch64/noeffect3.c
> > +++ b/gcc/testsuite/gcc.target/aarch64/noeffect3.c
> > @@ -10,11 +10,14 @@ int b[N] = {0};
> > ** foo:
> > ** ...
> > ** cmgt v[0-9]+\.4s, v[0-9]+\.4s, v[0-9]+\.4s
> > -** umaxp v[0-9]+\.4s, v[0-9]+\.4s, v[0-9]+\.4s
> > -** ...
> > -** ldr w[0-9]+, \[x[0-9]+, w[0-9]+, sxtw 2\]
> > -** ldr w[0-9]+, \[x[0-9]+, w[0-9]+, sxtw 2\]
> > -** cmp w[0-9]+, w[0-9]+
> > +** addhn v[0-9]+\.2s, v[0-9]+\.2d, v[0-9]+\.2d
> > +** umov x[0-9]+, v[0-9]+\.d\[0\]
> > +** cbz x[0-9]+, \.L[0-9]+
> > +** and x[0-9]+, x[0-9]+, 281479271743489
> > +** rbit x[0-9]+, x[0-9]+
> > +** clz x[0-9]+, x[0-9]+
> > +** lsr x[0-9]+, x[0-9]+, 4
> > +** add w0, w0, w[0-9]+
> > ** ...
> > ** ret
> > */
> > diff --git a/gcc/testsuite/gcc.target/aarch64/noeffect4.c
> b/gcc/testsuite/gcc.target/aarch64/noeffect4.c
> > index
> 276843c9bbd82d9fd409418734cc9a39cddf0aeb..6a477d165f7fa006ef6cf3
> 4ae3e84abba9a63fe0 100644
> > --- a/gcc/testsuite/gcc.target/aarch64/noeffect4.c
> > +++ b/gcc/testsuite/gcc.target/aarch64/noeffect4.c
> > @@ -10,12 +10,18 @@ short b[N] = {0};
> > ** foo:
> > ** ...
> > ** cmgt v[0-9]+\.8h, v[0-9]+\.8h, v[0-9]+\.8h
> > -** umaxp v[0-9]+\.4s, v[0-9]+\.4s, v[0-9]+\.4s
> > -** ...
> > -** ldrsh w[0-9]+, \[[^\n]+\]
> > -** ldrsh w[0-9]+, \[[^\n]+\]
> > -** cmp w[0-9]+, w[0-9]+
> > +** addhn v[0-9]+\.4h, v[0-9]+\.4s, v[0-9]+\.4s
> > +** umov x[0-9]+, v[0-9]+\.d\[0\]
> > +** cbz x[0-9]+, \.L[0-9]+
> > +** and x[0-9]+, x[0-9]+, 72340172838076673
> > +** rbit x[0-9]+, x[0-9]+
> > +** clz x[0-9]+, x[0-9]+
> > +** ubfx x[0-9]+, x[0-9]+, 3, 32
> > +** add w[0-9]+, w[0-9]+, w[0-9]+
> > +** and w[0-9]+, w[0-9]+, 65535
> > +** ret
> > ** ...
> > +** mov w0, 0
> > ** ret
> > */
> > __attribute__ ((noipa, noinline))
> > @@ -28,3 +34,5 @@ int foo (void)
> > }
> > return 0;
> > }
> > +
> > +/* { dg-final { scan-assembler-not {\tldrsh\t} } } */
> > diff --git a/gcc/testsuite/gcc.target/aarch64/noeffect5.c
> b/gcc/testsuite/gcc.target/aarch64/noeffect5.c
> > index
> c15e52ebdfc4d07b0e6caf12f1c8e0c6d41d377a..e463509a299eb84323397
> cae68aeda84fa4c543a 100644
> > --- a/gcc/testsuite/gcc.target/aarch64/noeffect5.c
> > +++ b/gcc/testsuite/gcc.target/aarch64/noeffect5.c
> > @@ -10,11 +10,14 @@ short b[N] = {0};
> > ** foo:
> > ** ...
> > ** cmgt v[0-9]+\.8h, v[0-9]+\.8h, v[0-9]+\.8h
> > -** umaxp v[0-9]+\.4s, v[0-9]+\.4s, v[0-9]+\.4s
> > -** ...
> > -** ldrsh w[0-9]+, \[[^\n]+\]
> > -** ldrsh w[0-9]+, \[[^\n]+\]
> > -** cmp w[0-9]+, w[0-9]+
> > +** addhn v[0-9]+\.4h, v[0-9]+\.4s, v[0-9]+\.4s
> > +** umov x[0-9]+, v[0-9]+\.d\[0\]
> > +** cbz x[0-9]+, \.L[0-9]+
> > +** and x[0-9]+, x[0-9]+, 72340172838076673
> > +** rbit x[0-9]+, x[0-9]+
> > +** clz x[0-9]+, x[0-9]+
> > +** ubfx x[0-9]+, x[0-9]+, 3, 32
> > +** add w0, w0, w[0-9]+
> > ** ...
> > ** ret
> > */
> > diff --git a/gcc/testsuite/gcc.target/aarch64/noeffect6.c
> b/gcc/testsuite/gcc.target/aarch64/noeffect6.c
> > index
> 9da4f496a02e96f053b9c001748a91df7b31d12d..7b4c742da5b8af2e1117
> 98a8187c34789aa05bdb 100644
> > --- a/gcc/testsuite/gcc.target/aarch64/noeffect6.c
> > +++ b/gcc/testsuite/gcc.target/aarch64/noeffect6.c
> > @@ -10,11 +10,14 @@ short b[N] = {0};
> > ** foo:
> > ** ...
> > ** cmgt v[0-9]+\.8h, v[0-9]+\.8h, v[0-9]+\.8h
> > -** umaxp v[0-9]+\.4s, v[0-9]+\.4s, v[0-9]+\.4s
> > -** ...
> > -** ldrsh w[0-9]+, \[[^\n]+\]
> > -** ldrsh w[0-9]+, \[[^\n]+\]
> > -** cmp w[0-9]+, w[0-9]+
> > +** addhn v[0-9]+\.4h, v[0-9]+\.4s, v[0-9]+\.4s
> > +** umov x[0-9]+, v[0-9]+\.d\[0\]
> > +** cbz x[0-9]+, \.L[0-9]+
> > +** and x[0-9]+, x[0-9]+, 72340172838076673
> > +** rbit x[0-9]+, x[0-9]+
> > +** clz x[0-9]+, x[0-9]+
> > +** lsr x[0-9]+, x[0-9]+, 3
> > +** ldrh w0, \[x[0-9]+, x[0-9]+, lsl 1\]
> > ** ...
> > ** ret
> > */
> > diff --git a/gcc/testsuite/gcc.target/aarch64/noeffect7.c
> b/gcc/testsuite/gcc.target/aarch64/noeffect7.c
> > index
> 16fc921117aa2a94ac7dd8fe795fe046f34cdb07..fb20481351495a63d94f7ff
> e8c06f0a886222f1b 100644
> > --- a/gcc/testsuite/gcc.target/aarch64/noeffect7.c
> > +++ b/gcc/testsuite/gcc.target/aarch64/noeffect7.c
> > @@ -1,26 +1,13 @@
> > /* { dg-do compile } */
> > /* { dg-options "-O3 -march=armv8-a" } */
> > -/* { dg-final { check-function-bodies "**" "" "" } } */
> > +/* { dg-final { scan-assembler {\tcmp\tw[0-9]+, w[0-9]+} } } */
> > +/* { dg-final { scan-assembler {\tmov\tw0, 1} } } */
> > +/* { dg-final { scan-assembler {\tmov\tw0, 0} } } */
> >
> > #define N 1000
> > int a[N] = {0};
> > int b[N] = {0};
> >
> > -/*
> > -** foo:
> > -** ...
> > -** add x[0-9]+, x[0-9]+, 4000
> > -** add x[0-9]+, x[0-9]+, 2000
> > -** b \.L[0-9]+
> > -** ldr w[0-9]+, \[x[0-9]+[^\n]*\]
> > -** ...
> > -** cmp w[0-9]+, w[0-9]+
> > -** ...
> > -** mov w0, 1
> > -** ret
> > -** mov w0, 0
> > -** ret
> > -*/
> > __attribute__ ((noipa, noinline))
> > int foo (void)
> > {
> > diff --git a/gcc/testsuite/gcc.target/aarch64/noeffect8.c
> b/gcc/testsuite/gcc.target/aarch64/noeffect8.c
> > index
> ada79391a8acab2073517b4172570947ec16a8a6..d3e03a618fa970b5480e
> ed619e00c670ff3aaab4 100644
> > --- a/gcc/testsuite/gcc.target/aarch64/noeffect8.c
> > +++ b/gcc/testsuite/gcc.target/aarch64/noeffect8.c
> > @@ -6,7 +6,9 @@
> > ** add:
> > ** ...
> > ** cmeq v[0-9]+\.4s, v[0-9]+\.4s, #0
> > -** umaxp v[0-9]+\.4s, v[0-9]+\.4s, v[0-9]+\.4s
> > +** addhn v[0-9]+\.2s, v[0-9]+\.2d, v[0-9]+\.2d
> > +** fmov x[0-9]+, d[0-9]+
> > +** cbnz x[0-9]+, \.L[0-9]+
> > ** ...
> > ** ldr q[0-9]+, \[x[0-9]+[^\n]*\]
> > ** ...
> > diff --git a/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch.c
> b/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch.c
> > index
> ca4ef498485f68dbb8a178750db3f552cf7a66d0..b29f02a4418cd3a6d87bd
> 617d9d3fb06d5e3f0c8 100644
> > --- a/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch.c
> > +++ b/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch.c
> > @@ -13,7 +13,7 @@ int b[N] = {0};
> > ** f1:
> > ** ...
> > ** cmgt v[0-9]+.4s, v[0-9]+.4s, #0
> > -** umaxp v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
> > +** addhn v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
> > ** fmov x[0-9]+, d[0-9]+
> > ** cbn?z x[0-9]+, \.L[0-9]+
> > ** ...
> > @@ -32,7 +32,7 @@ void f1 ()
> > ** f2:
> > ** ...
> > ** cmge v[0-9]+.4s, v[0-9]+.4s, #0
> > -** umaxp v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
> > +** addhn v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
> > ** fmov x[0-9]+, d[0-9]+
> > ** cbn?z x[0-9]+, \.L[0-9]+
> > ** ...
> > @@ -50,7 +50,8 @@ void f2 ()
> > /*
> > ** f3:
> > ** ...
> > -** umaxp v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
> > +** cmeq v[0-9]+.4s, v[0-9]+.4s, #0
> > +** addhn v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
> > ** fmov x[0-9]+, d[0-9]+
> > ** cbn?z x[0-9]+, \.L[0-9]+
> > ** ...
> > @@ -68,7 +69,8 @@ void f3 ()
> > /*
> > ** f4:
> > ** ...
> > -** umaxp v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
> > +** cmtst v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
> > +** addhn v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
> > ** fmov x[0-9]+, d[0-9]+
> > ** cbn?z x[0-9]+, \.L[0-9]+
> > ** ...
> > @@ -87,7 +89,7 @@ void f4 ()
> > ** f5:
> > ** ...
> > ** cmlt v[0-9]+.4s, v[0-9]+.4s, #0
> > -** umaxp v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
> > +** addhn v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
> > ** fmov x[0-9]+, d[0-9]+
> > ** cbn?z x[0-9]+, \.L[0-9]+
> > ** ...
> > @@ -106,7 +108,7 @@ void f5 ()
> > ** f6:
> > ** ...
> > ** cmle v[0-9]+.4s, v[0-9]+.4s, #0
> > -** umaxp v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
> > +** addhn v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
> > ** fmov x[0-9]+, d[0-9]+
> > ** cbn?z x[0-9]+, \.L[0-9]+
> > ** ...
> > diff --git a/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_2.c
> b/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_2.c
> > index
> d5cb1946873f5e720c6e12e0dd4991582764f18f..e40843f11b046dc8b8c9
> 9428a93da5ab6234d16e 100644
> > --- a/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_2.c
> > +++ b/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_2.c
> > @@ -10,8 +10,10 @@ int b[N] = {0};
> > /*
> > ** f1:
> > ** ...
> > -** cmpgt p[0-9]+.s, p[0-9]+/z, z[0-9]+.s, #0
> > -** b(\.?eq|\.none) \.L[0-9]+
> > +** cmgt v[0-9]+.4s, v[0-9]+.4s, #0
> > +** addhn v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
> > +** fmov x[0-9]+, d[0-9]+
> > +** cbz x[0-9]+, \.L[0-9]+
> > ** ...
> > */
> > void f1 ()
> > @@ -26,8 +28,10 @@ void f1 ()
> > /*
> > ** f2:
> > ** ...
> > -** cmpge p[0-9]+.s, p[0-9]+/z, z[0-9]+.s, #0
> > -** b(\.?eq|\.none) \.L[0-9]+
> > +** cmge v[0-9]+.4s, v[0-9]+.4s, #0
> > +** addhn v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
> > +** fmov x[0-9]+, d[0-9]+
> > +** cbz x[0-9]+, \.L[0-9]+
> > ** ...
> > */
> > void f2 ()
> > @@ -42,8 +46,10 @@ void f2 ()
> > /*
> > ** f3:
> > ** ...
> > -** cmpeq p[0-9]+.s, p[0-9]+/z, z[0-9]+.s, #0
> > -** b(\.?eq|\.none) \.L[0-9]+
> > +** cmeq v[0-9]+.4s, v[0-9]+.4s, #0
> > +** addhn v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
> > +** fmov x[0-9]+, d[0-9]+
> > +** cbz x[0-9]+, \.L[0-9]+
> > ** ...
> > */
> > void f3 ()
> > @@ -58,8 +64,10 @@ void f3 ()
> > /*
> > ** f4:
> > ** ...
> > -** cmpne p[0-9]+.s, p[0-9]+/z, z[0-9]+.s, #0
> > -** b(\.?eq|\.none) \.L[0-9]+
> > +** cmtst v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
> > +** addhn v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
> > +** fmov x[0-9]+, d[0-9]+
> > +** cbz x[0-9]+, \.L[0-9]+
> > ** ...
> > */
> > void f4 ()
> > @@ -74,8 +82,10 @@ void f4 ()
> > /*
> > ** f5:
> > ** ...
> > -** cmplt p[0-9]+.s, p7/z, z[0-9]+.s, #0
> > -** b(\.?eq|\.none) .L[0-9]+
> > +** cmlt v[0-9]+.4s, v[0-9]+.4s, #0
> > +** addhn v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
> > +** fmov x[0-9]+, d[0-9]+
> > +** cbz x[0-9]+, \.L[0-9]+
> > ** ...
> > */
> > void f5 ()
> > @@ -90,8 +100,10 @@ void f5 ()
> > /*
> > ** f6:
> > ** ...
> > -** cmple p[0-9]+.s, p[0-9]+/z, z[0-9]+.s, #0
> > -** b(\.?eq|\.none) \.L[0-9]+
> > +** cmle v[0-9]+.4s, v[0-9]+.4s, #0
> > +** addhn v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
> > +** fmov x[0-9]+, d[0-9]+
> > +** cbz x[0-9]+, \.L[0-9]+
> > ** ...
> > */
> > void f6 ()
> > diff --git a/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_3.c
> b/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_3.c
> > index
> b252b80ef291efd4f6fc98224a44ef7fadc68d6c..9e4d3f951ad111df0f76032
> 445ccce8673bca74e 100644
> > --- a/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_3.c
> > +++ b/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_3.c
> > @@ -11,9 +11,10 @@ float b[N] = {0};
> > /*
> > ** f1:
> > ** ...
> > -** fcmgt p[0-9]+.s, p[0-9]+/z, z[0-9]+.s, #0.0
> > -** ptest p[0-9]+, p[0-9]+\.b
> > -** b(\.?eq|\.none) \.L[0-9]+
> > +** fcmgt v[0-9]+.4s, v[0-9]+.4s, 0
> > +** addhn v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
> > +** fmov x[0-9]+, d[0-9]+
> > +** cbz x[0-9]+, \.L[0-9]+
> > ** ...
> > */
> > void f1 ()
> > @@ -28,9 +29,10 @@ void f1 ()
> > /*
> > ** f2:
> > ** ...
> > -** fcmge p[0-9]+.s, p[0-9]+/z, z[0-9]+.s, #0.0
> > -** ptest p[0-9]+, p[0-9]+\.b
> > -** b(\.?eq|\.none) \.L[0-9]+
> > +** fcmge v[0-9]+.4s, v[0-9]+.4s, 0
> > +** addhn v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
> > +** fmov x[0-9]+, d[0-9]+
> > +** cbz x[0-9]+, \.L[0-9]+
> > ** ...
> > */
> > void f2 ()
> > @@ -45,9 +47,10 @@ void f2 ()
> > /*
> > ** f3:
> > ** ...
> > -** fcmeq p[0-9]+.s, p[0-9]+/z, z[0-9]+.s, #0.0
> > -** ptest p[0-9]+, p[0-9]+\.b
> > -** b(\.?eq|\.none) \.L[0-9]+
> > +** fcmeq v[0-9]+.4s, v[0-9]+.4s, 0
> > +** addhn v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
> > +** fmov x[0-9]+, d[0-9]+
> > +** cbz x[0-9]+, \.L[0-9]+
> > ** ...
> > */
> > void f3 ()
> > @@ -62,9 +65,11 @@ void f3 ()
> > /*
> > ** f4:
> > ** ...
> > -** fcmne p[0-9]+.s, p[0-9]+/z, z[0-9]+.s, #0.0
> > -** ptest p[0-9]+, p[0-9]+\.b
> > -** b(\.?eq|\.none) \.L[0-9]+
> > +** fcmeq v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
> > +** not v[0-9]+.16b, v[0-9]+.16b
> > +** addhn v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
> > +** fmov x[0-9]+, d[0-9]+
> > +** cbz x[0-9]+, \.L[0-9]+
> > ** ...
> > */
> > void f4 ()
> > @@ -79,9 +84,10 @@ void f4 ()
> > /*
> > ** f5:
> > ** ...
> > -** fcmlt p[0-9]+.s, p7/z, z[0-9]+.s, #0.0
> > -** ptest p[0-9]+, p[0-9]+\.b
> > -** b(\.?eq|\.none) .L[0-9]+
> > +** fcmlt v[0-9]+.4s, v[0-9]+.4s, 0
> > +** addhn v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
> > +** fmov x[0-9]+, d[0-9]+
> > +** cbz x[0-9]+, \.L[0-9]+
> > ** ...
> > */
> > void f5 ()
> > @@ -96,9 +102,10 @@ void f5 ()
> > /*
> > ** f6:
> > ** ...
> > -** fcmle p[0-9]+.s, p[0-9]+/z, z[0-9]+.s, #0.0
> > -** ptest p[0-9]+, p[0-9]+\.b
> > -** b(\.?eq|\.none) \.L[0-9]+
> > +** fcmle v[0-9]+.4s, v[0-9]+.4s, 0
> > +** addhn v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
> > +** fmov x[0-9]+, d[0-9]+
> > +** cbz x[0-9]+, \.L[0-9]+
> > ** ...
> > */
> > void f6 ()
> > diff --git a/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_4.c
> b/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_4.c
> > index
> a49e7963cfa1afdce4bb1bf33224a1d79eb7b874..026c774e4de7f6381728f
> 164fbfd89f8643f7c3d 100644
> > --- a/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_4.c
> > +++ b/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_4.c
> > @@ -13,9 +13,9 @@ unsigned int b[N] = {0};
> > ** f1:
> > ** ...
> > ** cmhi v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
> > -** umaxp v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
> > +** addhn v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
> > ** fmov x[0-9]+, d[0-9]+
> > -** cbn?z x[0-9]+, \.L[0-9]+
> > +** cbz x[0-9]+, \.L[0-9]+
> > ** ...
> > */
> > void f1 ()
> > @@ -32,9 +32,9 @@ void f1 ()
> > ** f2:
> > ** ...
> > ** cmtst v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
> > -** umaxp v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
> > +** addhn v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
> > ** fmov x[0-9]+, d[0-9]+
> > -** cbn?z x[0-9]+, \.L[0-9]+
> > +** cbz x[0-9]+, \.L[0-9]+
> > ** ...
> > */
> > void f2 ()
> > @@ -50,9 +50,10 @@ void f2 ()
> > /*
> > ** f3:
> > ** ...
> > -** umaxp v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
> > +** cmeq v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
> > +** addhn v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
> > ** fmov x[0-9]+, d[0-9]+
> > -** cbn?z x[0-9]+, \.L[0-9]+
> > +** cbz x[0-9]+, \.L[0-9]+
> > ** ...
> > */
> > void f3 ()
> > @@ -68,9 +69,11 @@ void f3 ()
> > /*
> > ** f4:
> > ** ...
> > -** umaxp v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
> > +** cmeq v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
> > +** not v[0-9]+.16b, v[0-9]+.16b
> > +** addhn v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
> > ** fmov x[0-9]+, d[0-9]+
> > -** cbn?z x[0-9]+, \.L[0-9]+
> > +** cbz x[0-9]+, \.L[0-9]+
> > ** ...
> > */
> > void f4 ()
> > @@ -87,9 +90,9 @@ void f4 ()
> > ** f5:
> > ** ...
> > ** cmhs v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
> > -** umaxp v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
> > +** addhn v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
> > ** fmov x[0-9]+, d[0-9]+
> > -** cbn?z x[0-9]+, \.L[0-9]+
> > +** cbz x[0-9]+, \.L[0-9]+
> > ** ...
> > */
> > void f5 ()
> > @@ -106,9 +109,9 @@ void f5 ()
> > ** f6:
> > ** ...
> > ** cmhs v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
> > -** umaxp v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
> > +** addhn v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
> > ** fmov x[0-9]+, d[0-9]+
> > -** cbn?z x[0-9]+, \.L[0-9]+
> > +** cbz x[0-9]+, \.L[0-9]+
> > ** ...
> > */
> > void f6 ()
> > diff --git a/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_5.c
> b/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_5.c
> > index
> c28969dde1ff24872a5c3d9c236808f215cfdaa2..9679a9342f9b74170e21ec
> 41e1959525942c0f14 100644
> > --- a/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_5.c
> > +++ b/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_5.c
> > @@ -10,8 +10,12 @@ unsigned int b[N] = {0};
> > /*
> > ** f1:
> > ** ...
> > -** cmphi p[0-9]+.s, p[0-9]+/z, z[0-9]+.s, #2
> > -** b(\.?eq|\.none) \.L[0-9]+
> > +** movi v[0-9]+.4s, 0x2
> > +** ...
> > +** cmhi v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
> > +** addhn v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
> > +** fmov x[0-9]+, d[0-9]+
> > +** cbz x[0-9]+, \.L[0-9]+
> > ** ...
> > */
> > void f1 ()
> > @@ -26,8 +30,12 @@ void f1 ()
> > /*
> > ** f2:
> > ** ...
> > -** cmphi p[0-9]+.s, p[0-9]+/z, z[0-9]+.s, #1
> > -** b(\.?eq|\.none) \.L[0-9]+
> > +** movi v[0-9]+.4s, 0x1
> > +** ...
> > +** cmhi v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
> > +** addhn v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
> > +** fmov x[0-9]+, d[0-9]+
> > +** cbz x[0-9]+, \.L[0-9]+
> > ** ...
> > */
> > void f2 ()
> > @@ -42,8 +50,12 @@ void f2 ()
> > /*
> > ** f3:
> > ** ...
> > -** cmpeq p[0-9]+.s, p[0-9]+/z, z[0-9]+.s, #1
> > -** b(\.?eq|\.none) \.L[0-9]+
> > +** movi v[0-9]+.4s, 0x1
> > +** ...
> > +** cmeq v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
> > +** addhn v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
> > +** fmov x[0-9]+, d[0-9]+
> > +** cbz x[0-9]+, \.L[0-9]+
> > ** ...
> > */
> > void f3 ()
> > @@ -58,8 +70,13 @@ void f3 ()
> > /*
> > ** f4:
> > ** ...
> > -** cmpne p[0-9]+.s, p[0-9]+/z, z[0-9]+.s, #1
> > -** b(\.?eq|\.none) \.L[0-9]+
> > +** movi v[0-9]+.4s, 0x1
> > +** ...
> > +** cmeq v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
> > +** not v[0-9]+.16b, v[0-9]+.16b
> > +** addhn v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
> > +** fmov x[0-9]+, d[0-9]+
> > +** cbz x[0-9]+, \.L[0-9]+
> > ** ...
> > */
> > void f4 ()
> > @@ -74,8 +91,12 @@ void f4 ()
> > /*
> > ** f5:
> > ** ...
> > -** cmpls p[0-9]+.s, p7/z, z[0-9]+.s, #1
> > -** b(\.?eq|\.none) .L[0-9]+
> > +** movi v[0-9]+.4s, 0x1
> > +** ...
> > +** cmhs v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
> > +** addhn v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
> > +** fmov x[0-9]+, d[0-9]+
> > +** cbz x[0-9]+, \.L[0-9]+
> > ** ...
> > */
> > void f5 ()
> > @@ -90,8 +111,12 @@ void f5 ()
> > /*
> > ** f6:
> > ** ...
> > -** cmpls p[0-9]+.s, p[0-9]+/z, z[0-9]+.s, #1
> > -** b(\.?eq|\.none) \.L[0-9]+
> > +** movi v[0-9]+.4s, 0x1
> > +** ...
> > +** cmhs v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
> > +** addhn v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
> > +** fmov x[0-9]+, d[0-9]+
> > +** cbz x[0-9]+, \.L[0-9]+
> > ** ...
> > */
> > void f6 ()
> > diff --git a/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_7.c
> b/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_7.c
> > new file mode 100644
> > index
> 0000000000000000000000000000000000000000..e698ee5aa537c35a1c8
> 0a0bb0fbdf53790aac220
> > --- /dev/null
> > +++ b/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_7.c
> > @@ -0,0 +1,34 @@
> > +/* { dg-do compile } */
> > +/* { dg-options "-O3 -mcpu=neoverse-v2 -fno-schedule-insns -fno-reorder-
> blocks -fno-schedule-insns2 --param aarch64-autovec-preference=asimd-only
> -fdump-tree-vect-details" } */
> > +/* { dg-final { check-function-bodies "**" "" "" { target lp64 } } } */
> > +
> > +#pragma GCC target "+nosve"
> > +
> > +void *a[640] = { 0 };
> > +
> > +void sink (void **);
> > +
> > +/*
> > +** f1:
> > +** ...
> > +** ldp q[0-9]+, q[0-9]+, \[[x0-9]+.*\]
> > +** cmeq v[0-9]+.2d, v[0-9]+.2d, v[0-9]+.2d
> > +** cmeq v[0-9]+.2d, v[0-9]+.2d, v[0-9]+.2d
> > +** addhn v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
> > +** fmov x[0-9]+, d[0-9]+
> > +** cbz x[0-9]+, \.L[0-9]+
> > +** ...
> > +*/
> > +void
> > +f1 (void *x, unsigned long n)
> > +{
> > + void **end = a + n;
> > + for (void **p = a; p != end; ++p)
> > + if (*p == x)
> > + {
> > + sink (p);
> > + return;
> > + }
> > +}
> > +
> > +/* { dg-final { scan-tree-dump "loop vectorized using 16 byte vectors and
> unroll factor 4" "vect" } } */
> > diff --git a/gcc/testsuite/lib/target-supports.exp
> > b/gcc/testsuite/lib/target-
> supports.exp
> > index
> 94a3dba222fd3d257263a0c3bd0daf83b8fe7490..73bf24a1073090686476
> c3cdea1b8cbf061cd2a0 100644
> > --- a/gcc/testsuite/lib/target-supports.exp
> > +++ b/gcc/testsuite/lib/target-supports.exp
> > @@ -6800,6 +6800,24 @@ proc check_effective_target_aarch64_sve_hw {
> } {
> > } [add_options_for_aarch64_sve ""]]
> > }
> >
> > +# Return true if this is an AArch64 target that can run Adv.SIMD LUT code.
> > +
> > +proc check_effective_target_aarch64_lut_hw { } {
> > + if { ![istarget aarch64*-*-*] } {
> > + return 0
> > + }
> > + return [check_runtime aarch64_lut_hw_available {
> > + #pragma GCC target "+lut"
> > + int
> > + main (void)
> > + {
> > + asm volatile ("movi v0.16b, #0\n\t"
> > + "luti4 v0.16b, {v0.16b}, v0[0]");
> > + return 0;
> > + }
> > + }]
> > +}
> > +
> > # Return true if this is an AArch64 target that can run SVE2 code.
> >
> > proc check_effective_target_aarch64_sve2_hw { } {
> >
> >
> > --
> > <rb20863.patch>
