[patch 3/13]AArch64: implement new control flow IFNs for Adv.SIMD

[Tamar Christina]

This patch implements the control flow IFNs for Adv.SIMD.  It gives the
AArch64 expansions for counting active lanes, finding the
first active lane, creating inclusive and exclusive active-lane masks, and
extracting the first active value from a vector.

The implementation uses ADDHN to compress byte, halfword and word predicates
down to a scalar bitmap.  Plain vector branches use the cheaper zero/nonzero
compressed predicate path.  When the vectorizer needs the active lane number,
the exit block masks the compressed bitmap and uses scalar bit operations to
find the lane.  For live register values the final extraction can use a vector
table lookup instead of replaying scalar loads.

For example, consider:

  unsigned char a[1008], b[1008];

  unsigned char f1 (void)
  {
    unsigned char res = 0;
    for (unsigned int i = 0; i < 1008; ++i)
      {
        unsigned char x = a[i] & b[i];
        res = x;
        if (x > 5)
          break;
      }
    return res;
  }

Today we we generate at -O3 -march=armv8-a:

  cmhi    v30.16b, v31.16b, v29.16b
  umaxp   v30.4s, v30.4s, v30.4s
  fmov    x0, d30
  cbz     x0, .L2
  ldrb    w0, [x2, w1, uxtw]
  cmp     w0, 5
  bhi     .L3
  ... repeated scalar lane checks ...

and with this branch:

  cmhi    v31.16b, v30.16b, v29.16b
  addhn   v31.8b, v31.8h, v31.8h
  umov    x0, v31.d[0]
  cbz     x0, .L8
  rbit    x0, x0
  clz     x0, x0
  ...
  tbl     v30.16b, {v30.16b}, v31.16b
  fmov    w0, s30

There is one complication in how we perform the Adv. SIMD optimization which
replaces an Adv. SIMD compare with an SVE one.

This now has to be delayed until after expand when we can tell if the compare
is single use.  Otherwise we'd end up with an Adv. SIMD and SVE compare for the
same as CSE can't remove the duplicate then.

Bootstrapped Regtested on aarch64-none-linux-gnu,
arm-none-linux-gnueabihf, x86_64-pc-linux-gnu
-m32, -m64 and no issues.

-O3, profiled, profiled -O3 and -O3 -LTO bootstrap tested and no issues.

 Also SPECCPU 2017 and SPECCPU 2026 tested.

OK for master?

Thanks,
Tamar

gcc/ChangeLog:

        * config/aarch64/aarch64-protos.h (aarch64_gen_shareable_ptrue):
        Declare.
        * config/aarch64/aarch64-simd.md (cbranch<mode>4): Use compressed
        predicate branches for Adv.SIMD masks.
        (@aarch64_addhn_scalar<mode>): New expander.
        (@aarch64_addhn_mask_scalar<mode>): Likewise.
        (count_active_<mode>): Likewise.
        (index_first_active_<mode>): Likewise.
        (extract_first_<mode>): Likewise.
        (take_<take_active>_active_<mode>): Likewise.
        * config/aarch64/aarch64.cc (aarch64_gen_shareable_ptrue): New
        function.
        * config/aarch64/aarch64.h (advsimd_ptrue_insn): New.
        * config/aarch64/iterators.md (VADDHN_MASK_WIDE, VADDHN_MASK_NARROW,
        vaddhn_mask_wide, vaddhn_lane_mask, TAKE_ACTIVE_CMP, take_active): New.

gcc/testsuite/ChangeLog:

        * gcc.target/aarch64/noeffect11.c: Update expected code generation.
        * gcc.target/aarch64/noeffect2.c: Likewise.
        * gcc.target/aarch64/noeffect3.c: Likewise.
        * gcc.target/aarch64/noeffect4.c: Likewise.
        * gcc.target/aarch64/noeffect5.c: Likewise.
        * gcc.target/aarch64/noeffect6.c: Likewise.
        * gcc.target/aarch64/noeffect7.c: Likewise.
        * gcc.target/aarch64/noeffect8.c: Likewise.
        * gcc.target/aarch64/vect-early-break-cbranch.c: Likewise.
        * gcc.target/aarch64/vect-early-break-cbranch_2.c: Likewise.
        * gcc.target/aarch64/vect-early-break-cbranch_3.c: Likewise.
        * gcc.target/aarch64/vect-early-break-cbranch_4.c: Likewise.
        * gcc.target/aarch64/vect-early-break-cbranch_5.c: Likewise.
        * lib/target-supports.exp: Add Adv.SIMD LUT support checks.
        * gcc.target/aarch64/noeffect12.c: New test.
        * gcc.target/aarch64/noeffect13-run.c: New test.
        * gcc.target/aarch64/noeffect13.c: New test.
        * gcc.target/aarch64/vect-early-break-cbranch_7.c: New test.

---
diff --git a/gcc/config/aarch64/aarch64-protos.h 
b/gcc/config/aarch64/aarch64-protos.h
index 
e8ae3d427949896f5a34d21bd55ce770cdcdcd76..4d882ce90ae2ea50b9b33a380af7a6f1d0bc27a5
 100644
--- a/gcc/config/aarch64/aarch64-protos.h
+++ b/gcc/config/aarch64/aarch64-protos.h
@@ -1015,6 +1015,7 @@ rtx aarch64_return_addr_rtx (void);
 rtx aarch64_return_addr (int, rtx);
 rtx aarch64_simd_gen_const_vector_dup (machine_mode, HOST_WIDE_INT);
 rtx aarch64_gen_shareable_zero (machine_mode);
+rtx aarch64_gen_shareable_ptrue (machine_mode, unsigned int);
 bool aarch64_split_simd_shift_p (rtx_insn *);
 bool aarch64_simd_mem_operand_p (rtx);
 bool aarch64_sve_ld1r_operand_p (rtx);
diff --git a/gcc/config/aarch64/aarch64-simd.md 
b/gcc/config/aarch64/aarch64-simd.md
index 
f9b2fd6b2db1c88e50aac3b6f6f77ab9ba626654..5c45db988b3a83948a01ebf1071c88e9e8814cb7
 100644
--- a/gcc/config/aarch64/aarch64-simd.md
+++ b/gcc/config/aarch64/aarch64-simd.md
@@ -4475,8 +4475,8 @@ (define_expand "cbranch<mode>4"
   [(set (pc)
         (if_then_else
           (match_operator 0 "aarch64_equality_operator"
-            [(match_operand:VDQ_I 1 "register_operand")
-             (match_operand:VDQ_I 2 "aarch64_simd_reg_or_zero")])
+            [(match_operand:VD_BHSI 1 "register_operand")
+             (match_operand:VD_BHSI 2 "aarch64_simd_reg_or_zero")])
           (label_ref (match_operand 3 ""))
           (pc)))]
   "TARGET_SIMD"
@@ -4492,16 +4492,46 @@ (define_expand "cbranch<mode>4"
       emit_insn (gen_xor<mode>3 (tmp, operands[1], operands[2]));
     }
 
-  /* For 64-bit vectors we need no reductions.  */
-  if (known_eq (128, GET_MODE_BITSIZE (<MODE>mode)))
+  rtx val = gen_reg_rtx (DImode);
+  emit_move_insn (val, gen_lowpart (DImode, tmp));
+
+  rtx cc_reg = aarch64_gen_compare_reg (code, val, const0_rtx);
+  rtx cmp_rtx = gen_rtx_fmt_ee (code, DImode, cc_reg, const0_rtx);
+  emit_jump_insn (gen_aarch64_bcond (cmp_rtx, cc_reg, operands[3]));
+  DONE;
+})
+
+(define_expand "cbranch<mode>4"
+  [(set (pc)
+        (if_then_else
+          (match_operator 0 "aarch64_equality_operator"
+            [(match_operand:VQ_I 1 "register_operand")
+             (match_operand:VQ_I 2 "aarch64_simd_reg_or_zero")])
+          (label_ref (match_operand 3 ""))
+          (pc)))]
+  "TARGET_SIMD"
+{
+  auto code = GET_CODE (operands[0]);
+  rtx tmp = operands[1];
+
+  /* If comparing against a non-zero vector we have to do a comparison first
+     so we can have a != 0 comparison with the result.  */
+  if (operands[2] != CONST0_RTX (<MODE>mode))
     {
-      /* Always reduce using a V4SI.  */
-      rtx reduc = gen_lowpart (V4SImode, tmp);
-      rtx res = gen_reg_rtx (V4SImode);
-      emit_insn (gen_aarch64_umaxpv4si (res, reduc, reduc));
-      emit_move_insn (tmp, gen_lowpart (<MODE>mode, res));
+      tmp = gen_reg_rtx (<MODE>mode);
+      emit_insn (gen_xor<mode>3 (tmp, operands[1], operands[2]));
     }
 
+  /* This expander can be used for arbitrary vector zero tests, for example
+     the XOR result produced by __builtin_memcmp_eq.  ADDHN is only valid for
+     boolean masks whose true lanes are all ones.  Use UMAXP here; the
+     vec_cbranch expanders below still use ADDHN after producing a compare
+     mask.  */
+  rtx reduc = gen_lowpart (V4SImode, tmp);
+  rtx res = gen_reg_rtx (V4SImode);
+  emit_insn (gen_aarch64_umaxpv4si (res, reduc, reduc));
+  tmp = gen_lowpart (<MODE>mode, res);
+
   rtx val = gen_reg_rtx (DImode);
   emit_move_insn (val, gen_lowpart (DImode, tmp));
 
@@ -4511,6 +4541,298 @@ (define_expand "cbranch<mode>4"
   DONE;
 })
 
+;; Compress a 128-bit Adv.SIMD boolean vector into a 64-bit scalar value.
+;; ADDHN folds each adjacent pair of lanes into one narrower lane.  This is
+;; enough for zero/nonzero tests: a nonzero bit in any original lane produces
+;; a nonzero bit in the scalar result.
+;;
+;; Keep this separate from aarch64_addhn_mask_scalar.  Vector branches such as
+;; those in gcc.dg/vect/vect-uncounted_1.c only need a cbz/cbnz-style test and
+;; should not pay for the lane mask that index/count operations need later.
+(define_expand "@aarch64_addhn_scalar<mode>"
+  [(match_operand:DI 0 "register_operand")
+   (match_operand:VQ_I 1 "register_operand")]
+  "TARGET_SIMD"
+{
+  rtx wide = gen_lowpart (<VADDHN_MASK_WIDE>mode, operands[1]);
+  rtx narrow = gen_reg_rtx (<VADDHN_MASK_NARROW>mode);
+  emit_insn (gen_aarch64_addhn<vaddhn_mask_wide> (narrow, wide, wide));
+
+  emit_move_insn (operands[0], gen_lowpart (DImode, narrow));
+  DONE;
+})
+
+;; Compress a 128-bit Adv.SIMD boolean vector into a 64-bit scalar bitmap.
+;; The raw ADDHN result is sparse.  After masking with <vaddhn_lane_mask>,
+;; each original lane has exactly one bit in the scalar result, so ctz and
+;; popcount can recover a lane index or count.
+;; This is the exit-block form used by INDEX_FIRST_ACTIVE and COUNT_ACTIVE.
+(define_expand "@aarch64_addhn_mask_scalar<mode>"
+  [(match_operand:DI 0 "register_operand")
+   (match_operand:VQ_I 1 "register_operand")]
+  "TARGET_SIMD"
+{
+  rtx val = gen_reg_rtx (DImode);
+  emit_insn (gen_aarch64_addhn_scalar<mode> (val, operands[1]));
+  emit_move_insn (operands[0],
+                 gen_rtx_AND (DImode, val,
+                              gen_int_mode (<vaddhn_lane_mask>, DImode)));
+  DONE;
+})
+
+(define_expand "count_active_<mode>"
+  [(match_operand:DI 0 "register_operand")
+   (match_operand:VQ_I 1 "register_operand")]
+  "TARGET_SIMD"
+{
+  /* For example: addhn v0.2s, v1.2d, v1.2d; and; cnt.  */
+  rtx mask = gen_reg_rtx (DImode);
+  emit_insn (gen_aarch64_addhn_mask_scalar<mode> (mask, operands[1]));
+  emit_insn (gen_popcountdi2 (operands[0], mask));
+  DONE;
+})
+
+(define_expand "index_first_active_<mode>"
+  [(match_operand:DI 0 "register_operand")
+   (match_operand:VQ_I 1 "register_operand")]
+  "TARGET_SIMD"
+{
+  /* For example: addhn v0.2s, v1.2d, v1.2d; and; rbit; clz; lsr.  */
+  rtx mask = gen_reg_rtx (DImode);
+  rtx bitno = gen_reg_rtx (DImode);
+  emit_insn (gen_aarch64_addhn_mask_scalar<mode> (mask, operands[1]));
+  emit_insn (gen_ctzdi2 (bitno, mask));
+  emit_move_insn (operands[0],
+                 gen_rtx_LSHIFTRT (DImode, bitno,
+                                   GEN_INT (exact_log2
+                                            (GET_MODE_UNIT_BITSIZE
+                                               (<MODE>mode) / 2))));
+  DONE;
+})
+
+(define_expand "extract_first_<mode>"
+  [(match_operand:<VEL> 0 "register_operand")
+   (match_operand:<V_INT_EQUIV> 1 "register_operand")
+   (match_operand:VQ_NO2E 2 "register_operand")]
+  "TARGET_SIMD"
+{
+  /* For example: addhn; ctz; dup; add; tbl.  */
+  rtx index = gen_reg_rtx (DImode);
+  emit_insn (gen_index_first_active_<v_int_equiv> (index, operands[1]));
+
+  if (TARGET_LUT && <MODE>mode == V16QImode)
+    {
+      rtx index_vec = gen_reg_rtx (V8QImode);
+      emit_move_insn (gen_lowpart (DImode, index_vec), index);
+
+      rtx selected = gen_reg_rtx (V16QImode);
+      emit_insn (gen_aarch64_lutv16qiv8qi
+                (selected, operands[2], index_vec, const0_rtx, GEN_INT (4)));
+
+      emit_move_insn (operands[0], gen_lowpart (<VEL>mode, selected));
+      DONE;
+    }
+
+  if (TARGET_LUT && <MODE>mode == V8HImode)
+    {
+      rtx index_vec = gen_reg_rtx (V8QImode);
+      emit_move_insn (gen_lowpart (DImode, index_vec), index);
+
+      rtx table = gen_reg_rtx (V2x8HImode);
+      rtx first = simplify_gen_subreg (V8HImode, table, V2x8HImode, 0);
+      emit_move_insn (first, operands[2]);
+
+      rtx zero = force_reg (V8HImode, CONST0_RTX (V8HImode));
+      rtx second = simplify_gen_subreg (V8HImode, table, V2x8HImode, 16);
+      emit_move_insn (second, zero);
+
+      rtx selected = gen_reg_rtx (V8HImode);
+      emit_insn (gen_aarch64_lutv2x8hiv8qi
+                (selected, table, index_vec, const0_rtx, GEN_INT (4)));
+
+      emit_move_insn (operands[0], gen_lowpart (<VEL>mode, selected));
+      DONE;
+    }
+
+  /* The live-load fast path in tree-vect-loop.cc avoids this sequence for
+     memory values by reloading the selected scalar element.  This expander is
+     for values that are already in vector registers, such as the register
+     live-out cases in gcc.target/aarch64/noeffect11.c.
+
+     INDEX_FIRST_ACTIVE gives the active element number.  Turn it into a byte
+     offset, duplicate the offset across a byte vector and add {0, 1, ... 15}.
+     TBL then moves the selected element's bytes down to lane 0.  This avoids
+     transferring both 64-bit halves to general registers before choosing which
+     half contains the first active element.  */
+  unsigned int elt_bytes = GET_MODE_UNIT_SIZE (<MODE>mode);
+  if (elt_bytes > 1)
+    {
+      rtx byte_index = gen_reg_rtx (DImode);
+      emit_move_insn (byte_index,
+                     gen_rtx_ASHIFT (DImode, index,
+                                     GEN_INT (exact_log2 (elt_bytes))));
+      index = byte_index;
+    }
+
+  rtx byte_index_vec = gen_reg_rtx (V16QImode);
+  emit_insn (gen_aarch64_simd_dupv16qi (byte_index_vec,
+                                       gen_lowpart (QImode, index)));
+
+  rtvec v = rtvec_alloc (16);
+  for (unsigned int i = 0; i < 16; ++i)
+    RTVEC_ELT (v, i) = GEN_INT (i);
+  rtx byte_numbers = force_reg (V16QImode,
+                               gen_rtx_CONST_VECTOR (V16QImode, v));
+
+  rtx selector = gen_reg_rtx (V16QImode);
+  emit_insn (gen_addv16qi3 (selector, byte_index_vec, byte_numbers));
+
+  rtx selected = gen_reg_rtx (V16QImode);
+  emit_insn (gen_aarch64_qtbl1v16qi (selected,
+                                    gen_lowpart (V16QImode, operands[2]),
+                                    selector));
+
+  emit_move_insn (operands[0], gen_lowpart (<VEL>mode, selected));
+  DONE;
+})
+
+(define_expand "extract_first_<mode>"
+  [(match_operand:<VEL> 0 "register_operand")
+   (match_operand:<V_INT_EQUIV> 1 "register_operand")
+   (match_operand:VQ_2E 2 "register_operand")]
+  "TARGET_SIMD"
+{
+  /* For example: addhn; ctz; umov xlo; umov xhi; csel.  */
+  rtx index = gen_reg_rtx (DImode);
+  emit_insn (gen_index_first_active_<v_int_equiv> (index, operands[1]));
+
+  /* V2DI/V2DF only need to choose between the two 64-bit halves.  This keeps
+     the sequence in the scalar integer domain after INDEX_FIRST_ACTIVE and
+     avoids building a vector table control.  */
+  rtx lo = gen_reg_rtx (DImode);
+  emit_insn (gen_aarch64_get_lanev2di (lo, gen_lowpart (V2DImode, operands[2]),
+                                      aarch64_endian_lane_rtx (V2DImode,
+                                                               0)));
+
+  rtx hi = gen_reg_rtx (DImode);
+  emit_insn (gen_aarch64_get_lanev2di (hi, gen_lowpart (V2DImode, operands[2]),
+                                      aarch64_endian_lane_rtx (V2DImode,
+                                                               1)));
+
+  rtx cc_reg = aarch64_gen_compare_reg (NE, index, const0_rtx);
+  rtx cmp = gen_rtx_NE (VOIDmode, cc_reg, const0_rtx);
+  rtx data = gen_reg_rtx (DImode);
+  emit_insn (gen_rtx_SET (data, gen_rtx_IF_THEN_ELSE (DImode, cmp, hi, lo)));
+
+  emit_move_insn (operands[0], gen_lowpart (<VEL>mode, data));
+  DONE;
+})
+
+(define_expand "take_<take_active>_active_<mode>"
+  [(set (match_operand:VQ_I 0 "register_operand")
+       (neg:VQ_I
+         (TAKE_ACTIVE_CMP:VQ_I
+           (match_operand:VQ_I 1 "register_operand")
+           (match_dup 1))))]
+  "TARGET_SIMD"
+{
+  /* For example: addhn; ctz; dup; cmhs.  */
+  rtx index = gen_reg_rtx (DImode);
+  emit_insn (gen_index_first_active_<mode> (index, operands[1]));
+
+  rtx elt_index = gen_lowpart (<VEL>mode, index);
+  rtx index_vec = gen_reg_rtx (<MODE>mode);
+  emit_insn (gen_aarch64_simd_dup<mode> (index_vec, elt_index));
+
+  rtvec v = rtvec_alloc (<nunits>);
+  for (unsigned int i = 0; i < <nunits>; ++i)
+    RTVEC_ELT (v, i) = GEN_INT (i);
+  rtx lane_numbers = force_reg (<MODE>mode,
+                               gen_rtx_CONST_VECTOR (<MODE>mode, v));
+
+  emit_insn (gen_vec_cmp<mode><v_int_equiv> (operands[0],
+                                            gen_rtx_<CMP> (VOIDmode,
+                                                           lane_numbers,
+                                                           index_vec),
+                                            lane_numbers, index_vec));
+  DONE;
+})
+
+;; If a vector compare is used only to feed a vector branch, use an SVE
+;; predicate compare and PTEST instead of materializing and reducing an
+;; Adv.SIMD mask.  Keep this as a combine-time split rather than doing it in
+;; the vec_cbranch expander, since the compare mask might also be needed later
+;; for live-value extraction.  In that case the ADDHN result remains live after
+;; the branch and combine cannot consume the reduction into this pattern.
+;;
+;; This must be a define_split rather than a define_insn_and_split.  The split
+;; needs the original compare producer to be the previous insn; if that stops
+;; being true, FAILing is correct during combine, but would be an ICE if a
+;; recognized "#" insn survived until final.
+(define_split
+  [(set (pc)
+       (if_then_else
+         (match_operator 0 "aarch64_equality_operator"
+           [(subreg:DI
+              (truncate:<VADDHN_MASK_NARROW>
+                (ashiftrt:<VADDHN_MASK_WIDE>
+                  (plus:<VADDHN_MASK_WIDE>
+                    (subreg:<VADDHN_MASK_WIDE>
+                      (match_operand:VQ_I 1 "register_operand") 0)
+                    (subreg:<VADDHN_MASK_WIDE>
+                      (match_dup 1) 0))
+                  (match_operand:<VADDHN_MASK_WIDE> 3
+                    "aarch64_simd_shift_imm_vec_exact_top")))
+              0)
+            (const_int 0)])
+         (label_ref (match_operand 2 ""))
+         (pc)))]
+  "TARGET_SIMD
+   && TARGET_SVE
+   && can_create_pseudo_p ()
+   && known_eq (128, GET_MODE_BITSIZE (<MODE>mode))"
+  [(const_int 0)]
+{
+  if (find_regno_note (curr_insn, REG_DEAD, REGNO (operands[1])) == NULL_RTX)
+    FAIL;
+
+  rtx_insn *prev = prev_nonnote_nondebug_insn (curr_insn);
+  rtx set = prev ? single_set (prev) : NULL_RTX;
+  if (!set || !rtx_equal_p (SET_DEST (set), operands[1]))
+    FAIL;
+
+  rtx src = SET_SRC (set);
+  if (GET_CODE (src) != NEG || !COMPARISON_P (XEXP (src, 0)))
+    FAIL;
+
+  rtx cmp = XEXP (src, 0);
+  rtx_code cmp_code = GET_CODE (cmp);
+  machine_mode full_mode = aarch64_full_sve_mode (<VEL>mode).require ();
+
+  rtx in1 = force_lowpart_subreg (full_mode, XEXP (cmp, 0), <MODE>mode);
+  rtx in2;
+  if (CONST0_RTX (<MODE>mode) == XEXP (cmp, 1))
+    in2 = CONST0_RTX (full_mode);
+  else
+    in2 = force_lowpart_subreg (full_mode, XEXP (cmp, 1), <MODE>mode);
+
+  machine_mode pred_mode = aarch64_sve_pred_mode (full_mode);
+  unsigned int vl = exact_div (GET_MODE_BITSIZE (<MODE>mode), 8).to_constant 
();
+  rtx ptrue = aarch64_gen_shareable_ptrue (VNx16BImode, vl);
+  rtx cast_ptrue = gen_lowpart (pred_mode, ptrue);
+  rtx hint = gen_int_mode (SVE_MAYBE_NOT_PTRUE, SImode);
+  rtx tmp = gen_reg_rtx (pred_mode);
+  emit_insn (gen_aarch64_pred_cmp_ptest (cmp_code, full_mode, tmp, ptrue,
+                                        in1, in2, cast_ptrue, hint,
+                                        cast_ptrue, hint));
+
+  rtx cc_reg = gen_rtx_REG (CC_NZCmode, CC_REGNUM);
+  rtx cmp_reg = gen_rtx_fmt_ee (GET_CODE (operands[0]), VOIDmode,
+                               cc_reg, const0_rtx);
+  emit_jump_insn (gen_aarch64_bcond (cmp_reg, cc_reg, operands[2]));
+  DONE;
+})
+
 ;; Define vec_cbranch_any and vec_cbranch_all
 ;; Vector comparison and branch for Adv. SIMD Integer types using SVE
 ;; instructions.
@@ -4526,62 +4848,29 @@ (define_expand "<optab><mode>"
         CBRANCH_CMP))]
   "TARGET_SIMD"
 {
-  auto code = GET_CODE (operands[0]);
-  if (TARGET_SVE)
-    {
-      machine_mode full_mode = aarch64_full_sve_mode (<VEL>mode).require ();
-
-      rtx in1 = force_lowpart_subreg (full_mode, operands[1], <MODE>mode);
-      rtx in2;
-      if (CONST0_RTX (<MODE>mode) == operands[2])
-       in2 = CONST0_RTX (full_mode);
-      else
-       in2 = force_lowpart_subreg (full_mode, operands[2], <MODE>mode);
-
-      unsigned lanes
-       = exact_div (GET_MODE_BITSIZE (<MODE>mode), 8).to_constant ();
-      machine_mode pred_mode = aarch64_sve_pred_mode (full_mode);
-      rtx ptrue = aarch64_ptrue_reg (VNx16BImode, lanes);
-      rtx hint = gen_int_mode (SVE_MAYBE_NOT_PTRUE, SImode);
-
-      rtx tmp = gen_reg_rtx (pred_mode);
-      rtx cast_ptrue = gen_lowpart (pred_mode, ptrue);
-
-      if (FLOAT_MODE_P (full_mode))
-       {
-         aarch64_expand_sve_vec_cmp<sve_cmp_suff> (tmp, code, in1, in2);
-         emit_insn (gen_and3 (pred_mode, tmp, tmp, cast_ptrue));
-         emit_insn (gen_aarch64_ptest (pred_mode, ptrue, cast_ptrue, hint,
-                                       tmp));
-       }
-      else
-       emit_insn (gen_aarch64_pred_cmp_ptest (code, full_mode, tmp, ptrue, in1,
-                                              in2, cast_ptrue, hint,
-                                              cast_ptrue, hint));
-
-      rtx cc_reg = gen_rtx_REG (CC_NZCmode, CC_REGNUM);
-      rtx cmp_reg = gen_rtx_<cbranch_op> (VOIDmode, cc_reg, const0_rtx);
-      emit_jump_insn (gen_aarch64_bcond (cmp_reg, cc_reg, operands[3]));
-      DONE;
-    }
-
   rtx tmp = gen_reg_rtx (<V_INT_EQUIV>mode);
   emit_insn (gen_vec_cmp<mode><v_int_equiv> (tmp, operands[0], operands[1],
                                             operands[2]));
 
-  /* For 128-bit vectors we need a reduction to 64-bit first.  */
-  if (known_eq (128, GET_MODE_BITSIZE (<MODE>mode)))
+  rtx val = gen_reg_rtx (DImode);
+  rtx addhn_scalar
+    = maybe_gen_aarch64_addhn_scalar (<V_INT_EQUIV>mode, val, tmp);
+  if (known_eq (128, GET_MODE_BITSIZE (<MODE>mode)) && addhn_scalar)
+    emit_insn (addhn_scalar);
+  else
     {
-      /* Always reduce using a V4SI.  */
-      rtx reduc = gen_lowpart (V4SImode, tmp);
-      rtx res = gen_reg_rtx (V4SImode);
-      emit_insn (gen_aarch64_umaxpv4si (res, reduc, reduc));
-      emit_move_insn (tmp, gen_lowpart (<V_INT_EQUIV>mode, res));
+      /* For 128-bit vectors we need a reduction to 64-bit first.  */
+      if (known_eq (128, GET_MODE_BITSIZE (<MODE>mode)))
+       {
+         /* Always reduce using a V4SI.  */
+         rtx reduc = gen_lowpart (V4SImode, tmp);
+         rtx res = gen_reg_rtx (V4SImode);
+         emit_insn (gen_aarch64_umaxpv4si (res, reduc, reduc));
+         emit_move_insn (tmp, gen_lowpart (<V_INT_EQUIV>mode, res));
+       }
+      emit_move_insn (val, gen_lowpart (DImode, tmp));
     }
 
-  rtx val = gen_reg_rtx (DImode);
-  emit_move_insn (val, gen_lowpart (DImode, tmp));
-
   rtx cc_reg = aarch64_gen_compare_reg (<cbranch_op>, val, const0_rtx);
   rtx cmp_rtx = gen_rtx_fmt_ee (<cbranch_op>, DImode, cc_reg, const0_rtx);
   emit_jump_insn (gen_aarch64_bcond (cmp_rtx, cc_reg, operands[3]));
diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc
index 
8e62ab6a72e0043fffd16d3d187f18da7fd41d4e..77e29815f224a9e3bfc2ab3c5662c5d0aedf753e
 100644
--- a/gcc/config/aarch64/aarch64.cc
+++ b/gcc/config/aarch64/aarch64.cc
@@ -25316,6 +25316,28 @@ aarch64_gen_shareable_zero (machine_mode mode)
   return lowpart_subreg (mode, reg, GET_MODE (reg));
 }
 
+/* Return a shareable predicate register whose first VL bytes are active,
+   viewed in mode MODE.
+   This is for fixed-width Adv.SIMD sequences that want to use SVE flags for
+   a vector branch.  Hoisting the predicate constant avoids recreating it in
+   every loop iteration.  */
+
+rtx
+aarch64_gen_shareable_ptrue (machine_mode mode, unsigned int vl)
+{
+  gcc_assert (vl == 8 || vl == 16);
+
+  rtx_vector_builder builder (VNx16BImode, vl, 2);
+  for (unsigned int i = 0; i < vl; ++i)
+    builder.quick_push (CONST1_RTX (BImode));
+  for (unsigned int i = 0; i < vl; ++i)
+    builder.quick_push (CONST0_RTX (BImode));
+
+  rtx reg = aarch64_get_shareable_reg (&cfun->machine->advsimd_ptrue_insn,
+                                      builder.build ());
+  return lowpart_subreg (mode, reg, GET_MODE (reg));
+}
+
 /* INSN is some form of extension or shift that can be split into a
    permutation involving a shared zero.  Return true if we should
    perform such a split.
@@ -28913,7 +28935,7 @@ aarch64_vectorize_vec_perm_const (machine_mode vmode, 
machine_mode op_mode,
     {
       d.op1 = op1;
       if (d.op1 && !register_operand (d.op1, op_mode))
-       d.op1 = force_reg (op_mode, d.op1);
+       d.op1 = force_reg (op_mode, d.op1);
     }
   d.testing_p = !target;
 
@@ -28926,6 +28948,7 @@ aarch64_vectorize_vec_perm_const (machine_mode vmode, 
machine_mode op_mode,
 
   return ret;
 }
+
 /* Generate a byte permute mask for a register of mode MODE,
    which has NUNITS units.  */
 
diff --git a/gcc/config/aarch64/aarch64.h b/gcc/config/aarch64/aarch64.h
index 
fdedb04391127c5a1b821ed5d4b399c1c28d9f31..0673c8a64f7097e2d2ec4a155edeb12dfe72e957
 100644
--- a/gcc/config/aarch64/aarch64.h
+++ b/gcc/config/aarch64/aarch64.h
@@ -1249,6 +1249,10 @@ typedef struct GTY (()) machine_function
      can be reused by other code.  */
   rtx_insn *advsimd_zero_insn;
 
+  /* Like advsimd_zero_insn, but for a predicate with a fixed number of
+     leading true bytes.  */
+  rtx_insn *advsimd_ptrue_insn;
+
   /* During SEH output, this is non-null.  */
   struct seh_frame_state * GTY ((skip (""))) seh;
 
diff --git a/gcc/config/aarch64/iterators.md b/gcc/config/aarch64/iterators.md
index 
b0571460ec7ded9252bc2a216b6cd8867e6a595b..af417b51ccf31bf56d4e4f368979971c05e0b482
 100644
--- a/gcc/config/aarch64/iterators.md
+++ b/gcc/config/aarch64/iterators.md
@@ -2181,6 +2181,25 @@ (define_mode_attr Vdblw [(V8QI "v4hi") (V16QI "v8hi")
                         (V4HI "v2si") (V8HI "v4si")
                         (V2SI "di")   (V4SI "v2di")])
 
+;; Source mode for using ADDHN to compress a boolean mask to a scalar.
+(define_mode_attr VADDHN_MASK_WIDE [(V16QI "V8HI") (V8HI "V4SI")
+                                   (V4SI "V2DI") (V2DI "V2DI")])
+
+;; Result mode for using ADDHN to compress a boolean mask to a scalar.
+(define_mode_attr VADDHN_MASK_NARROW [(V16QI "V8QI") (V8HI "V4HI")
+                                     (V4SI "V2SI") (V2DI "V2SI")])
+
+;; Source mode suffix for using ADDHN to compress a boolean mask to a scalar.
+(define_mode_attr vaddhn_mask_wide [(V16QI "v8hi") (V8HI "v4si")
+                                   (V4SI "v2di") (V2DI "v2di")])
+
+;; Scalar mask for the useful bits produced by using ADDHN to compress a
+;; boolean mask to a scalar.
+(define_mode_attr vaddhn_lane_mask [(V16QI "0x1111111111111111")
+                                   (V8HI "0x0101010101010101")
+                                   (V4SI "0x0001000100010001")
+                                   (V2DI "0x0000000100000001")])
+
 (define_mode_attr VQUADW [(V8QI "V4SI") (V16QI "V8SI")
                   (V4HI "V2DI") (V8HI "V4DI")])
 
@@ -3237,6 +3256,9 @@ (define_code_iterator SAT_MINUS [ss_minus us_minus])
 ;; Comparison operators for <F>CM.
 (define_code_iterator COMPARISONS [lt le eq ge gt])
 
+;; Comparison operators for TAKE_{EXCL,INCL}_ACTIVE.
+(define_code_iterator TAKE_ACTIVE_CMP [lt le])
+
 ;; Unsigned comparison operators.
 (define_code_iterator UCOMPARISONS [ltu leu geu gtu])
 
@@ -3374,6 +3396,9 @@ (define_code_attr cmp_op [(lt "lt")
                          (geu "hs")
                          (gtu "hi")])
 
+;; The TAKE_ACTIVE optab suffix associated with an rtl comparison code.
+(define_code_attr take_active [(lt "excl") (le "incl")])
+
 (define_code_attr inv_cmp_op [(lt "ge")
                          (le "gt")
                          (eq "ne")
diff --git a/gcc/testsuite/gcc.target/aarch64/noeffect11.c 
b/gcc/testsuite/gcc.target/aarch64/noeffect11.c
index 
82c2f00c7cb16df22fa051b8d5d421863321e464..875e3ca184d925691eaf99d3d45b8cfab8b6f3e6
 100644
--- a/gcc/testsuite/gcc.target/aarch64/noeffect11.c
+++ b/gcc/testsuite/gcc.target/aarch64/noeffect11.c
@@ -8,7 +8,10 @@
 **     ...
 **     cmhi    v[0-9]+\.2d, v[0-9]+\.2d, v[0-9]+\.2d
 **     ...
-**     umaxp   v[0-9]+\.4s, v[0-9]+\.4s, v[0-9]+\.4s
+**     addhn   v[0-9]+\.2s, v[0-9]+\.2d, v[0-9]+\.2d
+**     addhn   v[0-9]+\.2s, v[0-9]+\.2d, v[0-9]+\.2d
+**     fmov    x[0-9]+, d[0-9]+
+**     cbz     x[0-9]+, \.L[0-9]+
 **     ...
 **     ldr     x[0-9]+, \[x[0-9]+, x[0-9]+\]
 **     ldr     x[0-9]+, \[x[0-9]+, x[0-9]+\]
@@ -34,7 +37,10 @@ int f1 (const unsigned long *restrict a, const unsigned long 
*b, int n)
 **     ...
 **     cmhi    v[0-9]+\.2d, v[0-9]+\.2d, v[0-9]+\.2d
 **     ...
-**     umaxp   v[0-9]+\.4s, v[0-9]+\.4s, v[0-9]+\.4s
+**     addhn   v[0-9]+\.2s, v[0-9]+\.2d, v[0-9]+\.2d
+**     addhn   v[0-9]+\.2s, v[0-9]+\.2d, v[0-9]+\.2d
+**     fmov    x[0-9]+, d[0-9]+
+**     cbz     x[0-9]+, \.L[0-9]+
 **     ...
 **     ldr     x[0-9]+, \[x[0-9]+, x[0-9]+\]
 **     ldr     x[0-9]+, \[x[0-9]+, x[0-9]+\]
diff --git a/gcc/testsuite/gcc.target/aarch64/noeffect12.c 
b/gcc/testsuite/gcc.target/aarch64/noeffect12.c
new file mode 100644
index 
0000000000000000000000000000000000000000..a42d67b56c47ed07497a40ea4758f07aea2bae4a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/noeffect12.c
@@ -0,0 +1,64 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -march=armv8-a" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+
+#define N 1008
+
+unsigned char a[N] = {0};
+unsigned char b[N] = {0};
+unsigned int c[N] = {0};
+unsigned int d[N] = {0};
+
+/*
+** f1:
+**     ...
+**     cmhi    v[0-9]+\.16b, v[0-9]+\.16b, v[0-9]+\.16b
+**     addhn   v[0-9]+\.8b, v[0-9]+\.8h, v[0-9]+\.8h
+**     umov    x[0-9]+, v[0-9]+\.d\[0\]
+**     cbz     x[0-9]+, \.L[0-9]+
+**     ...
+**     tbl     v[0-9]+\.16b, \{v[0-9]+\.16b\}, v[0-9]+\.16b
+**     fmov    w0, s[0-9]+
+**     ret
+**     ...
+*/
+__attribute__ ((noipa, noinline))
+unsigned char f1 (void)
+{
+  unsigned char res = 0;
+  for (unsigned int i = 0; i < N; ++i)
+    {
+      unsigned char x = a[i] & b[i];
+      res = x;
+      if (x > 5)
+       break;
+    }
+  return res;
+}
+
+/*
+** f2:
+**     ...
+**     cmhi    v[0-9]+\.4s, v[0-9]+\.4s, v[0-9]+\.4s
+**     addhn   v[0-9]+\.2s, v[0-9]+\.2d, v[0-9]+\.2d
+**     umov    x[0-9]+, v[0-9]+\.d\[0\]
+**     cbz     x[0-9]+, \.L[0-9]+
+**     ...
+**     tbl     v[0-9]+\.16b, \{v[0-9]+\.16b\}, v[0-9]+\.16b
+**     fmov    w0, s[0-9]+
+**     ret
+**     ...
+*/
+__attribute__ ((noipa, noinline))
+unsigned int f2 (void)
+{
+  unsigned int res = 0;
+  for (unsigned int i = 0; i < 1000; ++i)
+    {
+      unsigned int x = c[i] & d[i];
+      res = x;
+      if (x > 5)
+       break;
+    }
+  return res;
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/noeffect13-run.c 
b/gcc/testsuite/gcc.target/aarch64/noeffect13-run.c
new file mode 100644
index 
0000000000000000000000000000000000000000..9e9eb715c83f6bda7981f36d1682908a8380c114
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/noeffect13-run.c
@@ -0,0 +1,62 @@
+/* { dg-do run { target aarch64_lut_hw } } */
+/* { dg-options "-O3 -march=armv9-a+lut -mautovec-preference=asimd-only" } */
+
+#define N 1008
+
+unsigned char a[N];
+unsigned char b[N];
+unsigned short c[N];
+unsigned short d[N];
+
+__attribute__ ((noipa, noinline))
+unsigned char f1 (void)
+{
+  unsigned char res = 0;
+  for (unsigned int i = 0; i < N; ++i)
+    {
+      unsigned char x = a[i] & b[i];
+      res = x;
+      if (x > 5)
+       break;
+    }
+  return res;
+}
+
+__attribute__ ((noipa, noinline))
+unsigned short f2 (void)
+{
+  unsigned short res = 0;
+  for (unsigned int i = 0; i < N; ++i)
+    {
+      unsigned short x = c[i] & d[i];
+      res = x;
+      if (x > 5)
+       break;
+    }
+  return res;
+}
+
+int
+main (void)
+{
+  for (unsigned int i = 0; i < N; ++i)
+    {
+      a[i] = 1;
+      b[i] = 1;
+      c[i] = 1;
+      d[i] = 1;
+    }
+
+  if (f1 () != 1 || f2 () != 1)
+    __builtin_abort ();
+
+  a[13] = 7;
+  b[13] = 7;
+  c[11] = 9;
+  d[11] = 9;
+
+  if (f1 () != 7 || f2 () != 9)
+    __builtin_abort ();
+
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/noeffect13.c 
b/gcc/testsuite/gcc.target/aarch64/noeffect13.c
new file mode 100644
index 
0000000000000000000000000000000000000000..502126d6c005775b0f08c8807b3dda14c6372d34
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/noeffect13.c
@@ -0,0 +1,66 @@
+/* { dg-do compile { target aarch64_asm_lut_ok } } */
+/* { dg-options "-O3 -march=armv9-a+lut -mautovec-preference=asimd-only" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+
+#define N 1008
+
+unsigned char a[N] = {0};
+unsigned char b[N] = {0};
+unsigned short c[N] = {0};
+unsigned short d[N] = {0};
+
+/*
+** f1:
+**     ...
+**     cmhi    v[0-9]+\.16b, v[0-9]+\.16b, v[0-9]+\.16b
+**     ...
+**     addhn   v[0-9]+\.8b, v[0-9]+\.8h, v[0-9]+\.8h
+**     fmov    x[0-9]+, d[0-9]+
+**     ...
+**     luti4   v[0-9]+\.16b, \{v[0-9]+\.16b\}, v[0-9]+\[0\]
+**     fmov    w0, s[0-9]+
+**     ret
+**     ...
+*/
+__attribute__ ((noipa, noinline))
+unsigned char f1 (void)
+{
+  unsigned char res = 0;
+  for (unsigned int i = 0; i < N; ++i)
+    {
+      unsigned char x = a[i] & b[i];
+      res = x;
+      if (x > 5)
+       break;
+    }
+  return res;
+}
+
+/*
+** f2:
+**     ...
+**     cmhi    v[0-9]+\.8h, v[0-9]+\.8h, v[0-9]+\.8h
+**     ...
+**     addhn   v[0-9]+\.4h, v[0-9]+\.4s, v[0-9]+\.4s
+**     fmov    x[0-9]+, d[0-9]+
+**     ...
+**     movi    v[0-9]+\.4s, 0
+**     ...
+**     luti4   v[0-9]+\.8h, \{v[0-9]+\.8h, v[0-9]+\.8h\}, v[0-9]+\[0\]
+**     fmov    w0, s[0-9]+
+**     ret
+**     ...
+*/
+__attribute__ ((noipa, noinline))
+unsigned short f2 (void)
+{
+  unsigned short res = 0;
+  for (unsigned int i = 0; i < N; ++i)
+    {
+      unsigned short x = c[i] & d[i];
+      res = x;
+      if (x > 5)
+       break;
+    }
+  return res;
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/noeffect2.c 
b/gcc/testsuite/gcc.target/aarch64/noeffect2.c
index 
08c531fb18c817b018776a88db4f62eb03b001cb..bcd8848038527d3af1a4966ef55580e262b5d871
 100644
--- a/gcc/testsuite/gcc.target/aarch64/noeffect2.c
+++ b/gcc/testsuite/gcc.target/aarch64/noeffect2.c
@@ -13,7 +13,7 @@ int b[N] = {0};
 **     ldr     q[0-9]+, \[x[0-9]+, x[0-9]+\]
 **     add     x[0-9]+, x[0-9]+, 16
 **     cmgt    v[0-9]+\.4s, v[0-9]+\.4s, v[0-9]+\.4s
-**     umaxp   v[0-9]+\.4s, v[0-9]+\.4s, v[0-9]+\.4s
+**     addhn   v[0-9]+\.2s, v[0-9]+\.2d, v[0-9]+\.2d
 **     fmov    x[0-9]+, d[0-9]+
 **     cbz     x[0-9]+, \.L[0-9]+
 **     mov     w0, 1
diff --git a/gcc/testsuite/gcc.target/aarch64/noeffect3.c 
b/gcc/testsuite/gcc.target/aarch64/noeffect3.c
index 
886ad7bda2f3ebd5048cc618bcf4fe473cd2f820..f4ece387fc355b5a7d8c23b416400ec81da47051
 100644
--- a/gcc/testsuite/gcc.target/aarch64/noeffect3.c
+++ b/gcc/testsuite/gcc.target/aarch64/noeffect3.c
@@ -10,11 +10,14 @@ int b[N] = {0};
 ** foo:
 **     ...
 **     cmgt    v[0-9]+\.4s, v[0-9]+\.4s, v[0-9]+\.4s
-**     umaxp   v[0-9]+\.4s, v[0-9]+\.4s, v[0-9]+\.4s
-**     ...
-**     ldr     w[0-9]+, \[x[0-9]+, w[0-9]+, sxtw 2\]
-**     ldr     w[0-9]+, \[x[0-9]+, w[0-9]+, sxtw 2\]
-**     cmp     w[0-9]+, w[0-9]+
+**     addhn   v[0-9]+\.2s, v[0-9]+\.2d, v[0-9]+\.2d
+**     umov    x[0-9]+, v[0-9]+\.d\[0\]
+**     cbz     x[0-9]+, \.L[0-9]+
+**     and     x[0-9]+, x[0-9]+, 281479271743489
+**     rbit    x[0-9]+, x[0-9]+
+**     clz     x[0-9]+, x[0-9]+
+**     lsr     x[0-9]+, x[0-9]+, 4
+**     add     w0, w0, w[0-9]+
 **     ...
 **     ret
 */
diff --git a/gcc/testsuite/gcc.target/aarch64/noeffect4.c 
b/gcc/testsuite/gcc.target/aarch64/noeffect4.c
index 
276843c9bbd82d9fd409418734cc9a39cddf0aeb..6a477d165f7fa006ef6cf34ae3e84abba9a63fe0
 100644
--- a/gcc/testsuite/gcc.target/aarch64/noeffect4.c
+++ b/gcc/testsuite/gcc.target/aarch64/noeffect4.c
@@ -10,12 +10,18 @@ short b[N] = {0};
 ** foo:
 **     ...
 **     cmgt    v[0-9]+\.8h, v[0-9]+\.8h, v[0-9]+\.8h
-**     umaxp   v[0-9]+\.4s, v[0-9]+\.4s, v[0-9]+\.4s
-**     ...
-**     ldrsh   w[0-9]+, \[[^\n]+\]
-**     ldrsh   w[0-9]+, \[[^\n]+\]
-**     cmp     w[0-9]+, w[0-9]+
+**     addhn   v[0-9]+\.4h, v[0-9]+\.4s, v[0-9]+\.4s
+**     umov    x[0-9]+, v[0-9]+\.d\[0\]
+**     cbz     x[0-9]+, \.L[0-9]+
+**     and     x[0-9]+, x[0-9]+, 72340172838076673
+**     rbit    x[0-9]+, x[0-9]+
+**     clz     x[0-9]+, x[0-9]+
+**     ubfx    x[0-9]+, x[0-9]+, 3, 32
+**     add     w[0-9]+, w[0-9]+, w[0-9]+
+**     and     w[0-9]+, w[0-9]+, 65535
+**     ret
 **     ...
+**     mov     w0, 0
 **     ret
 */
 __attribute__ ((noipa, noinline))
@@ -28,3 +34,5 @@ int foo (void)
     }
   return 0;
 }
+
+/* { dg-final { scan-assembler-not {\tldrsh\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/noeffect5.c 
b/gcc/testsuite/gcc.target/aarch64/noeffect5.c
index 
c15e52ebdfc4d07b0e6caf12f1c8e0c6d41d377a..e463509a299eb84323397cae68aeda84fa4c543a
 100644
--- a/gcc/testsuite/gcc.target/aarch64/noeffect5.c
+++ b/gcc/testsuite/gcc.target/aarch64/noeffect5.c
@@ -10,11 +10,14 @@ short b[N] = {0};
 ** foo:
 **     ...
 **     cmgt    v[0-9]+\.8h, v[0-9]+\.8h, v[0-9]+\.8h
-**     umaxp   v[0-9]+\.4s, v[0-9]+\.4s, v[0-9]+\.4s
-**     ...
-**     ldrsh   w[0-9]+, \[[^\n]+\]
-**     ldrsh   w[0-9]+, \[[^\n]+\]
-**     cmp     w[0-9]+, w[0-9]+
+**     addhn   v[0-9]+\.4h, v[0-9]+\.4s, v[0-9]+\.4s
+**     umov    x[0-9]+, v[0-9]+\.d\[0\]
+**     cbz     x[0-9]+, \.L[0-9]+
+**     and     x[0-9]+, x[0-9]+, 72340172838076673
+**     rbit    x[0-9]+, x[0-9]+
+**     clz     x[0-9]+, x[0-9]+
+**     ubfx    x[0-9]+, x[0-9]+, 3, 32
+**     add     w0, w0, w[0-9]+
 **     ...
 **     ret
 */
diff --git a/gcc/testsuite/gcc.target/aarch64/noeffect6.c 
b/gcc/testsuite/gcc.target/aarch64/noeffect6.c
index 
9da4f496a02e96f053b9c001748a91df7b31d12d..7b4c742da5b8af2e111798a8187c34789aa05bdb
 100644
--- a/gcc/testsuite/gcc.target/aarch64/noeffect6.c
+++ b/gcc/testsuite/gcc.target/aarch64/noeffect6.c
@@ -10,11 +10,14 @@ short b[N] = {0};
 ** foo:
 **     ...
 **     cmgt    v[0-9]+\.8h, v[0-9]+\.8h, v[0-9]+\.8h
-**     umaxp   v[0-9]+\.4s, v[0-9]+\.4s, v[0-9]+\.4s
-**     ...
-**     ldrsh   w[0-9]+, \[[^\n]+\]
-**     ldrsh   w[0-9]+, \[[^\n]+\]
-**     cmp     w[0-9]+, w[0-9]+
+**     addhn   v[0-9]+\.4h, v[0-9]+\.4s, v[0-9]+\.4s
+**     umov    x[0-9]+, v[0-9]+\.d\[0\]
+**     cbz     x[0-9]+, \.L[0-9]+
+**     and     x[0-9]+, x[0-9]+, 72340172838076673
+**     rbit    x[0-9]+, x[0-9]+
+**     clz     x[0-9]+, x[0-9]+
+**     lsr     x[0-9]+, x[0-9]+, 3
+**     ldrh    w0, \[x[0-9]+, x[0-9]+, lsl 1\]
 **     ...
 **     ret
 */
diff --git a/gcc/testsuite/gcc.target/aarch64/noeffect7.c 
b/gcc/testsuite/gcc.target/aarch64/noeffect7.c
index 
16fc921117aa2a94ac7dd8fe795fe046f34cdb07..fb20481351495a63d94f7ffe8c06f0a886222f1b
 100644
--- a/gcc/testsuite/gcc.target/aarch64/noeffect7.c
+++ b/gcc/testsuite/gcc.target/aarch64/noeffect7.c
@@ -1,26 +1,13 @@
 /* { dg-do compile } */
 /* { dg-options "-O3 -march=armv8-a" } */
-/* { dg-final { check-function-bodies "**" "" "" } } */
+/* { dg-final { scan-assembler {\tcmp\tw[0-9]+, w[0-9]+} } } */
+/* { dg-final { scan-assembler {\tmov\tw0, 1} } } */
+/* { dg-final { scan-assembler {\tmov\tw0, 0} } } */
 
 #define N 1000
 int a[N] = {0};
 int b[N] = {0};
 
-/*
-** foo:
-**     ...
-**     add     x[0-9]+, x[0-9]+, 4000
-**     add     x[0-9]+, x[0-9]+, 2000
-**     b       \.L[0-9]+
-**     ldr     w[0-9]+, \[x[0-9]+[^\n]*\]
-**     ...
-**     cmp     w[0-9]+, w[0-9]+
-**     ...
-**     mov     w0, 1
-**     ret
-**     mov     w0, 0
-**     ret
-*/
 __attribute__ ((noipa, noinline))
 int foo (void)
 {
diff --git a/gcc/testsuite/gcc.target/aarch64/noeffect8.c 
b/gcc/testsuite/gcc.target/aarch64/noeffect8.c
index 
ada79391a8acab2073517b4172570947ec16a8a6..d3e03a618fa970b5480eed619e00c670ff3aaab4
 100644
--- a/gcc/testsuite/gcc.target/aarch64/noeffect8.c
+++ b/gcc/testsuite/gcc.target/aarch64/noeffect8.c
@@ -6,7 +6,9 @@
 ** add:
 **     ...
 **     cmeq    v[0-9]+\.4s, v[0-9]+\.4s, #0
-**     umaxp   v[0-9]+\.4s, v[0-9]+\.4s, v[0-9]+\.4s
+**     addhn   v[0-9]+\.2s, v[0-9]+\.2d, v[0-9]+\.2d
+**     fmov    x[0-9]+, d[0-9]+
+**     cbnz    x[0-9]+, \.L[0-9]+
 **     ...
 **     ldr     q[0-9]+, \[x[0-9]+[^\n]*\]
 **     ...
diff --git a/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch.c 
b/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch.c
index 
ca4ef498485f68dbb8a178750db3f552cf7a66d0..b29f02a4418cd3a6d87bd617d9d3fb06d5e3f0c8
 100644
--- a/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch.c
+++ b/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch.c
@@ -13,7 +13,7 @@ int b[N] = {0};
 ** f1:
 **     ...
 **     cmgt    v[0-9]+.4s, v[0-9]+.4s, #0
-**     umaxp   v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
+**     addhn   v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
 **     fmov    x[0-9]+, d[0-9]+
 **     cbn?z   x[0-9]+, \.L[0-9]+
 **     ...
@@ -32,7 +32,7 @@ void f1 ()
 ** f2:
 **     ...
 **     cmge    v[0-9]+.4s, v[0-9]+.4s, #0
-**     umaxp   v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
+**     addhn   v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
 **     fmov    x[0-9]+, d[0-9]+
 **     cbn?z   x[0-9]+, \.L[0-9]+
 **     ...
@@ -50,7 +50,8 @@ void f2 ()
 /*
 ** f3:
 **     ...
-**     umaxp   v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
+**     cmeq    v[0-9]+.4s, v[0-9]+.4s, #0
+**     addhn   v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
 **     fmov    x[0-9]+, d[0-9]+
 **     cbn?z   x[0-9]+, \.L[0-9]+
 **     ...
@@ -68,7 +69,8 @@ void f3 ()
 /*
 ** f4:
 **     ...
-**     umaxp   v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
+**     cmtst   v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
+**     addhn   v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
 **     fmov    x[0-9]+, d[0-9]+
 **     cbn?z   x[0-9]+, \.L[0-9]+
 **     ...
@@ -87,7 +89,7 @@ void f4 ()
 ** f5:
 **     ...
 **     cmlt    v[0-9]+.4s, v[0-9]+.4s, #0
-**     umaxp   v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
+**     addhn   v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
 **     fmov    x[0-9]+, d[0-9]+
 **     cbn?z   x[0-9]+, \.L[0-9]+
 **     ...
@@ -106,7 +108,7 @@ void f5 ()
 ** f6:
 **     ...
 **     cmle    v[0-9]+.4s, v[0-9]+.4s, #0
-**     umaxp   v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
+**     addhn   v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
 **     fmov    x[0-9]+, d[0-9]+
 **     cbn?z   x[0-9]+, \.L[0-9]+
 **     ...
diff --git a/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_2.c 
b/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_2.c
index 
d5cb1946873f5e720c6e12e0dd4991582764f18f..e40843f11b046dc8b8c99428a93da5ab6234d16e
 100644
--- a/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_2.c
+++ b/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_2.c
@@ -10,8 +10,10 @@ int b[N] = {0};
 /*
 ** f1:
 **     ...
-**     cmpgt   p[0-9]+.s, p[0-9]+/z, z[0-9]+.s, #0
-**     b(\.?eq|\.none) \.L[0-9]+
+**     cmgt    v[0-9]+.4s, v[0-9]+.4s, #0
+**     addhn   v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
+**     fmov    x[0-9]+, d[0-9]+
+**     cbz     x[0-9]+, \.L[0-9]+
 **     ...
 */
 void f1 ()
@@ -26,8 +28,10 @@ void f1 ()
 /*
 ** f2:
 **     ...
-**     cmpge   p[0-9]+.s, p[0-9]+/z, z[0-9]+.s, #0
-**     b(\.?eq|\.none) \.L[0-9]+
+**     cmge    v[0-9]+.4s, v[0-9]+.4s, #0
+**     addhn   v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
+**     fmov    x[0-9]+, d[0-9]+
+**     cbz     x[0-9]+, \.L[0-9]+
 **     ...
 */
 void f2 ()
@@ -42,8 +46,10 @@ void f2 ()
 /*
 ** f3:
 **     ...
-**     cmpeq   p[0-9]+.s, p[0-9]+/z, z[0-9]+.s, #0
-**     b(\.?eq|\.none) \.L[0-9]+
+**     cmeq    v[0-9]+.4s, v[0-9]+.4s, #0
+**     addhn   v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
+**     fmov    x[0-9]+, d[0-9]+
+**     cbz     x[0-9]+, \.L[0-9]+
 **     ...
 */
 void f3 ()
@@ -58,8 +64,10 @@ void f3 ()
 /*
 ** f4:
 **     ...
-**     cmpne   p[0-9]+.s, p[0-9]+/z, z[0-9]+.s, #0
-**     b(\.?eq|\.none) \.L[0-9]+
+**     cmtst   v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
+**     addhn   v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
+**     fmov    x[0-9]+, d[0-9]+
+**     cbz     x[0-9]+, \.L[0-9]+
 **     ...
 */
 void f4 ()
@@ -74,8 +82,10 @@ void f4 ()
 /*
 ** f5:
 **     ...
-**     cmplt   p[0-9]+.s, p7/z, z[0-9]+.s, #0
-**     b(\.?eq|\.none) .L[0-9]+
+**     cmlt    v[0-9]+.4s, v[0-9]+.4s, #0
+**     addhn   v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
+**     fmov    x[0-9]+, d[0-9]+
+**     cbz     x[0-9]+, \.L[0-9]+
 **     ...
 */
 void f5 ()
@@ -90,8 +100,10 @@ void f5 ()
 /*
 ** f6:
 **     ...
-**     cmple   p[0-9]+.s, p[0-9]+/z, z[0-9]+.s, #0
-**     b(\.?eq|\.none) \.L[0-9]+
+**     cmle    v[0-9]+.4s, v[0-9]+.4s, #0
+**     addhn   v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
+**     fmov    x[0-9]+, d[0-9]+
+**     cbz     x[0-9]+, \.L[0-9]+
 **     ...
 */
 void f6 ()
diff --git a/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_3.c 
b/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_3.c
index 
b252b80ef291efd4f6fc98224a44ef7fadc68d6c..9e4d3f951ad111df0f76032445ccce8673bca74e
 100644
--- a/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_3.c
+++ b/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_3.c
@@ -11,9 +11,10 @@ float b[N] = {0};
 /*
 ** f1:
 **     ...
-**     fcmgt   p[0-9]+.s, p[0-9]+/z, z[0-9]+.s, #0.0
-**     ptest   p[0-9]+, p[0-9]+\.b
-**     b(\.?eq|\.none) \.L[0-9]+
+**     fcmgt   v[0-9]+.4s, v[0-9]+.4s, 0
+**     addhn   v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
+**     fmov    x[0-9]+, d[0-9]+
+**     cbz     x[0-9]+, \.L[0-9]+
 **     ...
 */
 void f1 ()
@@ -28,9 +29,10 @@ void f1 ()
 /*
 ** f2:
 **     ...
-**     fcmge   p[0-9]+.s, p[0-9]+/z, z[0-9]+.s, #0.0
-**     ptest   p[0-9]+, p[0-9]+\.b
-**     b(\.?eq|\.none) \.L[0-9]+
+**     fcmge   v[0-9]+.4s, v[0-9]+.4s, 0
+**     addhn   v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
+**     fmov    x[0-9]+, d[0-9]+
+**     cbz     x[0-9]+, \.L[0-9]+
 **     ...
 */
 void f2 ()
@@ -45,9 +47,10 @@ void f2 ()
 /*
 ** f3:
 **     ...
-**     fcmeq   p[0-9]+.s, p[0-9]+/z, z[0-9]+.s, #0.0
-**     ptest   p[0-9]+, p[0-9]+\.b
-**     b(\.?eq|\.none) \.L[0-9]+
+**     fcmeq   v[0-9]+.4s, v[0-9]+.4s, 0
+**     addhn   v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
+**     fmov    x[0-9]+, d[0-9]+
+**     cbz     x[0-9]+, \.L[0-9]+
 **     ...
 */
 void f3 ()
@@ -62,9 +65,11 @@ void f3 ()
 /*
 ** f4:
 **     ...
-**     fcmne   p[0-9]+.s, p[0-9]+/z, z[0-9]+.s, #0.0
-**     ptest   p[0-9]+, p[0-9]+\.b
-**     b(\.?eq|\.none) \.L[0-9]+
+**     fcmeq   v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
+**     not     v[0-9]+.16b, v[0-9]+.16b
+**     addhn   v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
+**     fmov    x[0-9]+, d[0-9]+
+**     cbz     x[0-9]+, \.L[0-9]+
 **     ...
 */
 void f4 ()
@@ -79,9 +84,10 @@ void f4 ()
 /*
 ** f5:
 **     ...
-**     fcmlt   p[0-9]+.s, p7/z, z[0-9]+.s, #0.0
-**     ptest   p[0-9]+, p[0-9]+\.b
-**     b(\.?eq|\.none) .L[0-9]+
+**     fcmlt   v[0-9]+.4s, v[0-9]+.4s, 0
+**     addhn   v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
+**     fmov    x[0-9]+, d[0-9]+
+**     cbz     x[0-9]+, \.L[0-9]+
 **     ...
 */
 void f5 ()
@@ -96,9 +102,10 @@ void f5 ()
 /*
 ** f6:
 **     ...
-**     fcmle   p[0-9]+.s, p[0-9]+/z, z[0-9]+.s, #0.0
-**     ptest   p[0-9]+, p[0-9]+\.b
-**     b(\.?eq|\.none) \.L[0-9]+
+**     fcmle   v[0-9]+.4s, v[0-9]+.4s, 0
+**     addhn   v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
+**     fmov    x[0-9]+, d[0-9]+
+**     cbz     x[0-9]+, \.L[0-9]+
 **     ...
 */
 void f6 ()
diff --git a/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_4.c 
b/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_4.c
index 
a49e7963cfa1afdce4bb1bf33224a1d79eb7b874..026c774e4de7f6381728f164fbfd89f8643f7c3d
 100644
--- a/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_4.c
+++ b/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_4.c
@@ -13,9 +13,9 @@ unsigned int b[N] = {0};
 ** f1:
 **     ...
 **     cmhi    v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
-**     umaxp   v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
+**     addhn   v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
 **     fmov    x[0-9]+, d[0-9]+
-**     cbn?z   x[0-9]+, \.L[0-9]+
+**     cbz     x[0-9]+, \.L[0-9]+
 **     ...
 */
 void f1 ()
@@ -32,9 +32,9 @@ void f1 ()
 ** f2:
 **     ...
 **     cmtst   v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
-**     umaxp   v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
+**     addhn   v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
 **     fmov    x[0-9]+, d[0-9]+
-**     cbn?z   x[0-9]+, \.L[0-9]+
+**     cbz     x[0-9]+, \.L[0-9]+
 **     ...
 */
 void f2 ()
@@ -50,9 +50,10 @@ void f2 ()
 /*
 ** f3:
 **     ...
-**     umaxp   v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
+**     cmeq    v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
+**     addhn   v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
 **     fmov    x[0-9]+, d[0-9]+
-**     cbn?z   x[0-9]+, \.L[0-9]+
+**     cbz     x[0-9]+, \.L[0-9]+
 **     ...
 */
 void f3 ()
@@ -68,9 +69,11 @@ void f3 ()
 /*
 ** f4:
 **     ...
-**     umaxp   v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
+**     cmeq    v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
+**     not     v[0-9]+.16b, v[0-9]+.16b
+**     addhn   v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
 **     fmov    x[0-9]+, d[0-9]+
-**     cbn?z   x[0-9]+, \.L[0-9]+
+**     cbz     x[0-9]+, \.L[0-9]+
 **     ...
 */
 void f4 ()
@@ -87,9 +90,9 @@ void f4 ()
 ** f5:
 **     ...
 **     cmhs    v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
-**     umaxp   v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
+**     addhn   v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
 **     fmov    x[0-9]+, d[0-9]+
-**     cbn?z   x[0-9]+, \.L[0-9]+
+**     cbz     x[0-9]+, \.L[0-9]+
 **     ...
 */
 void f5 ()
@@ -106,9 +109,9 @@ void f5 ()
 ** f6:
 **     ...
 **     cmhs    v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
-**     umaxp   v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
+**     addhn   v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
 **     fmov    x[0-9]+, d[0-9]+
-**     cbn?z   x[0-9]+, \.L[0-9]+
+**     cbz     x[0-9]+, \.L[0-9]+
 **     ...
 */
 void f6 ()
diff --git a/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_5.c 
b/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_5.c
index 
c28969dde1ff24872a5c3d9c236808f215cfdaa2..9679a9342f9b74170e21ec41e1959525942c0f14
 100644
--- a/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_5.c
+++ b/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_5.c
@@ -10,8 +10,12 @@ unsigned int b[N] = {0};
 /*
 ** f1:
 **     ...
-**     cmphi   p[0-9]+.s, p[0-9]+/z, z[0-9]+.s, #2
-**     b(\.?eq|\.none) \.L[0-9]+
+**     movi    v[0-9]+.4s, 0x2
+**     ...
+**     cmhi    v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
+**     addhn   v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
+**     fmov    x[0-9]+, d[0-9]+
+**     cbz     x[0-9]+, \.L[0-9]+
 **     ...
 */
 void f1 ()
@@ -26,8 +30,12 @@ void f1 ()
 /*
 ** f2:
 **     ...
-**     cmphi   p[0-9]+.s, p[0-9]+/z, z[0-9]+.s, #1
-**     b(\.?eq|\.none) \.L[0-9]+
+**     movi    v[0-9]+.4s, 0x1
+**     ...
+**     cmhi    v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
+**     addhn   v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
+**     fmov    x[0-9]+, d[0-9]+
+**     cbz     x[0-9]+, \.L[0-9]+
 **     ...
 */
 void f2 ()
@@ -42,8 +50,12 @@ void f2 ()
 /*
 ** f3:
 **     ...
-**     cmpeq   p[0-9]+.s, p[0-9]+/z, z[0-9]+.s, #1
-**     b(\.?eq|\.none) \.L[0-9]+
+**     movi    v[0-9]+.4s, 0x1
+**     ...
+**     cmeq    v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
+**     addhn   v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
+**     fmov    x[0-9]+, d[0-9]+
+**     cbz     x[0-9]+, \.L[0-9]+
 **     ...
 */
 void f3 ()
@@ -58,8 +70,13 @@ void f3 ()
 /*
 ** f4:
 **     ...
-**     cmpne   p[0-9]+.s, p[0-9]+/z, z[0-9]+.s, #1
-**     b(\.?eq|\.none) \.L[0-9]+
+**     movi    v[0-9]+.4s, 0x1
+**     ...
+**     cmeq    v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
+**     not     v[0-9]+.16b, v[0-9]+.16b
+**     addhn   v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
+**     fmov    x[0-9]+, d[0-9]+
+**     cbz     x[0-9]+, \.L[0-9]+
 **     ...
 */
 void f4 ()
@@ -74,8 +91,12 @@ void f4 ()
 /*
 ** f5:
 **     ...
-**     cmpls   p[0-9]+.s, p7/z, z[0-9]+.s, #1
-**     b(\.?eq|\.none) .L[0-9]+
+**     movi    v[0-9]+.4s, 0x1
+**     ...
+**     cmhs    v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
+**     addhn   v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
+**     fmov    x[0-9]+, d[0-9]+
+**     cbz     x[0-9]+, \.L[0-9]+
 **     ...
 */
 void f5 ()
@@ -90,8 +111,12 @@ void f5 ()
 /*
 ** f6:
 **     ...
-**     cmpls   p[0-9]+.s, p[0-9]+/z, z[0-9]+.s, #1
-**     b(\.?eq|\.none) \.L[0-9]+
+**     movi    v[0-9]+.4s, 0x1
+**     ...
+**     cmhs    v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
+**     addhn   v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
+**     fmov    x[0-9]+, d[0-9]+
+**     cbz     x[0-9]+, \.L[0-9]+
 **     ...
 */
 void f6 ()
diff --git a/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_7.c 
b/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_7.c
new file mode 100644
index 
0000000000000000000000000000000000000000..e698ee5aa537c35a1c80a0bb0fbdf53790aac220
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_7.c
@@ -0,0 +1,34 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -mcpu=neoverse-v2 -fno-schedule-insns -fno-reorder-blocks 
-fno-schedule-insns2 --param aarch64-autovec-preference=asimd-only 
-fdump-tree-vect-details" } */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } } } */
+
+#pragma GCC target "+nosve"
+
+void *a[640] = { 0 };
+
+void sink (void **);
+
+/*
+** f1:
+**     ...
+**     ldp     q[0-9]+, q[0-9]+, \[[x0-9]+.*\]
+**     cmeq    v[0-9]+.2d, v[0-9]+.2d, v[0-9]+.2d
+**     cmeq    v[0-9]+.2d, v[0-9]+.2d, v[0-9]+.2d
+**     addhn   v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
+**     fmov    x[0-9]+, d[0-9]+
+**     cbz     x[0-9]+, \.L[0-9]+
+**     ...
+*/
+void
+f1 (void *x, unsigned long n)
+{
+  void **end = a + n;
+  for (void **p = a; p != end; ++p)
+    if (*p == x)
+      {
+       sink (p);
+       return;
+      }
+}
+
+/* { dg-final { scan-tree-dump "loop vectorized using 16 byte vectors and 
unroll factor 4" "vect" } } */
diff --git a/gcc/testsuite/lib/target-supports.exp 
b/gcc/testsuite/lib/target-supports.exp
index 
94a3dba222fd3d257263a0c3bd0daf83b8fe7490..73bf24a1073090686476c3cdea1b8cbf061cd2a0
 100644
--- a/gcc/testsuite/lib/target-supports.exp
+++ b/gcc/testsuite/lib/target-supports.exp
@@ -6800,6 +6800,24 @@ proc check_effective_target_aarch64_sve_hw { } {
     } [add_options_for_aarch64_sve ""]]
 }
 
+# Return true if this is an AArch64 target that can run Adv.SIMD LUT code.
+
+proc check_effective_target_aarch64_lut_hw { } {
+    if { ![istarget aarch64*-*-*] } {
+       return 0
+    }
+    return [check_runtime aarch64_lut_hw_available {
+       #pragma GCC target "+lut"
+       int
+       main (void)
+       {
+         asm volatile ("movi v0.16b, #0\n\t"
+                       "luti4 v0.16b, {v0.16b}, v0[0]");
+         return 0;
+       }
+    }]
+}
+
 # Return true if this is an AArch64 target that can run SVE2 code.
 
 proc check_effective_target_aarch64_sve2_hw { } {


-- 

diff --git a/gcc/config/aarch64/aarch64-protos.h b/gcc/config/aarch64/aarch64-protos.h
index e8ae3d427949896f5a34d21bd55ce770cdcdcd76..4d882ce90ae2ea50b9b33a380af7a6f1d0bc27a5 100644
--- a/gcc/config/aarch64/aarch64-protos.h
+++ b/gcc/config/aarch64/aarch64-protos.h
@@ -1015,6 +1015,7 @@ rtx aarch64_return_addr_rtx (void);
 rtx aarch64_return_addr (int, rtx);
 rtx aarch64_simd_gen_const_vector_dup (machine_mode, HOST_WIDE_INT);
 rtx aarch64_gen_shareable_zero (machine_mode);
+rtx aarch64_gen_shareable_ptrue (machine_mode, unsigned int);
 bool aarch64_split_simd_shift_p (rtx_insn *);
 bool aarch64_simd_mem_operand_p (rtx);
 bool aarch64_sve_ld1r_operand_p (rtx);
diff --git a/gcc/config/aarch64/aarch64-simd.md b/gcc/config/aarch64/aarch64-simd.md
index f9b2fd6b2db1c88e50aac3b6f6f77ab9ba626654..5c45db988b3a83948a01ebf1071c88e9e8814cb7 100644
--- a/gcc/config/aarch64/aarch64-simd.md
+++ b/gcc/config/aarch64/aarch64-simd.md
@@ -4475,8 +4475,8 @@ (define_expand "cbranch<mode>4"
   [(set (pc)
         (if_then_else
           (match_operator 0 "aarch64_equality_operator"
-            [(match_operand:VDQ_I 1 "register_operand")
-             (match_operand:VDQ_I 2 "aarch64_simd_reg_or_zero")])
+            [(match_operand:VD_BHSI 1 "register_operand")
+             (match_operand:VD_BHSI 2 "aarch64_simd_reg_or_zero")])
           (label_ref (match_operand 3 ""))
           (pc)))]
   "TARGET_SIMD"
@@ -4492,16 +4492,46 @@ (define_expand "cbranch<mode>4"
       emit_insn (gen_xor<mode>3 (tmp, operands[1], operands[2]));
     }
 
-  /* For 64-bit vectors we need no reductions.  */
-  if (known_eq (128, GET_MODE_BITSIZE (<MODE>mode)))
+  rtx val = gen_reg_rtx (DImode);
+  emit_move_insn (val, gen_lowpart (DImode, tmp));
+
+  rtx cc_reg = aarch64_gen_compare_reg (code, val, const0_rtx);
+  rtx cmp_rtx = gen_rtx_fmt_ee (code, DImode, cc_reg, const0_rtx);
+  emit_jump_insn (gen_aarch64_bcond (cmp_rtx, cc_reg, operands[3]));
+  DONE;
+})
+
+(define_expand "cbranch<mode>4"
+  [(set (pc)
+        (if_then_else
+          (match_operator 0 "aarch64_equality_operator"
+            [(match_operand:VQ_I 1 "register_operand")
+             (match_operand:VQ_I 2 "aarch64_simd_reg_or_zero")])
+          (label_ref (match_operand 3 ""))
+          (pc)))]
+  "TARGET_SIMD"
+{
+  auto code = GET_CODE (operands[0]);
+  rtx tmp = operands[1];
+
+  /* If comparing against a non-zero vector we have to do a comparison first
+     so we can have a != 0 comparison with the result.  */
+  if (operands[2] != CONST0_RTX (<MODE>mode))
     {
-      /* Always reduce using a V4SI.  */
-      rtx reduc = gen_lowpart (V4SImode, tmp);
-      rtx res = gen_reg_rtx (V4SImode);
-      emit_insn (gen_aarch64_umaxpv4si (res, reduc, reduc));
-      emit_move_insn (tmp, gen_lowpart (<MODE>mode, res));
+      tmp = gen_reg_rtx (<MODE>mode);
+      emit_insn (gen_xor<mode>3 (tmp, operands[1], operands[2]));
     }
 
+  /* This expander can be used for arbitrary vector zero tests, for example
+     the XOR result produced by __builtin_memcmp_eq.  ADDHN is only valid for
+     boolean masks whose true lanes are all ones.  Use UMAXP here; the
+     vec_cbranch expanders below still use ADDHN after producing a compare
+     mask.  */
+  rtx reduc = gen_lowpart (V4SImode, tmp);
+  rtx res = gen_reg_rtx (V4SImode);
+  emit_insn (gen_aarch64_umaxpv4si (res, reduc, reduc));
+  tmp = gen_lowpart (<MODE>mode, res);
+
   rtx val = gen_reg_rtx (DImode);
   emit_move_insn (val, gen_lowpart (DImode, tmp));
 
@@ -4511,6 +4541,298 @@ (define_expand "cbranch<mode>4"
   DONE;
 })
 
+;; Compress a 128-bit Adv.SIMD boolean vector into a 64-bit scalar value.
+;; ADDHN folds each adjacent pair of lanes into one narrower lane.  This is
+;; enough for zero/nonzero tests: a nonzero bit in any original lane produces
+;; a nonzero bit in the scalar result.
+;;
+;; Keep this separate from aarch64_addhn_mask_scalar.  Vector branches such as
+;; those in gcc.dg/vect/vect-uncounted_1.c only need a cbz/cbnz-style test and
+;; should not pay for the lane mask that index/count operations need later.
+(define_expand "@aarch64_addhn_scalar<mode>"
+  [(match_operand:DI 0 "register_operand")
+   (match_operand:VQ_I 1 "register_operand")]
+  "TARGET_SIMD"
+{
+  rtx wide = gen_lowpart (<VADDHN_MASK_WIDE>mode, operands[1]);
+  rtx narrow = gen_reg_rtx (<VADDHN_MASK_NARROW>mode);
+  emit_insn (gen_aarch64_addhn<vaddhn_mask_wide> (narrow, wide, wide));
+
+  emit_move_insn (operands[0], gen_lowpart (DImode, narrow));
+  DONE;
+})
+
+;; Compress a 128-bit Adv.SIMD boolean vector into a 64-bit scalar bitmap.
+;; The raw ADDHN result is sparse.  After masking with <vaddhn_lane_mask>,
+;; each original lane has exactly one bit in the scalar result, so ctz and
+;; popcount can recover a lane index or count.
+;; This is the exit-block form used by INDEX_FIRST_ACTIVE and COUNT_ACTIVE.
+(define_expand "@aarch64_addhn_mask_scalar<mode>"
+  [(match_operand:DI 0 "register_operand")
+   (match_operand:VQ_I 1 "register_operand")]
+  "TARGET_SIMD"
+{
+  rtx val = gen_reg_rtx (DImode);
+  emit_insn (gen_aarch64_addhn_scalar<mode> (val, operands[1]));
+  emit_move_insn (operands[0],
+		  gen_rtx_AND (DImode, val,
+			       gen_int_mode (<vaddhn_lane_mask>, DImode)));
+  DONE;
+})
+
+(define_expand "count_active_<mode>"
+  [(match_operand:DI 0 "register_operand")
+   (match_operand:VQ_I 1 "register_operand")]
+  "TARGET_SIMD"
+{
+  /* For example: addhn v0.2s, v1.2d, v1.2d; and; cnt.  */
+  rtx mask = gen_reg_rtx (DImode);
+  emit_insn (gen_aarch64_addhn_mask_scalar<mode> (mask, operands[1]));
+  emit_insn (gen_popcountdi2 (operands[0], mask));
+  DONE;
+})
+
+(define_expand "index_first_active_<mode>"
+  [(match_operand:DI 0 "register_operand")
+   (match_operand:VQ_I 1 "register_operand")]
+  "TARGET_SIMD"
+{
+  /* For example: addhn v0.2s, v1.2d, v1.2d; and; rbit; clz; lsr.  */
+  rtx mask = gen_reg_rtx (DImode);
+  rtx bitno = gen_reg_rtx (DImode);
+  emit_insn (gen_aarch64_addhn_mask_scalar<mode> (mask, operands[1]));
+  emit_insn (gen_ctzdi2 (bitno, mask));
+  emit_move_insn (operands[0],
+		  gen_rtx_LSHIFTRT (DImode, bitno,
+				    GEN_INT (exact_log2
+					     (GET_MODE_UNIT_BITSIZE
+						(<MODE>mode) / 2))));
+  DONE;
+})
+
+(define_expand "extract_first_<mode>"
+  [(match_operand:<VEL> 0 "register_operand")
+   (match_operand:<V_INT_EQUIV> 1 "register_operand")
+   (match_operand:VQ_NO2E 2 "register_operand")]
+  "TARGET_SIMD"
+{
+  /* For example: addhn; ctz; dup; add; tbl.  */
+  rtx index = gen_reg_rtx (DImode);
+  emit_insn (gen_index_first_active_<v_int_equiv> (index, operands[1]));
+
+  if (TARGET_LUT && <MODE>mode == V16QImode)
+    {
+      rtx index_vec = gen_reg_rtx (V8QImode);
+      emit_move_insn (gen_lowpart (DImode, index_vec), index);
+
+      rtx selected = gen_reg_rtx (V16QImode);
+      emit_insn (gen_aarch64_lutv16qiv8qi
+		 (selected, operands[2], index_vec, const0_rtx, GEN_INT (4)));
+
+      emit_move_insn (operands[0], gen_lowpart (<VEL>mode, selected));
+      DONE;
+    }
+
+  if (TARGET_LUT && <MODE>mode == V8HImode)
+    {
+      rtx index_vec = gen_reg_rtx (V8QImode);
+      emit_move_insn (gen_lowpart (DImode, index_vec), index);
+
+      rtx table = gen_reg_rtx (V2x8HImode);
+      rtx first = simplify_gen_subreg (V8HImode, table, V2x8HImode, 0);
+      emit_move_insn (first, operands[2]);
+
+      rtx zero = force_reg (V8HImode, CONST0_RTX (V8HImode));
+      rtx second = simplify_gen_subreg (V8HImode, table, V2x8HImode, 16);
+      emit_move_insn (second, zero);
+
+      rtx selected = gen_reg_rtx (V8HImode);
+      emit_insn (gen_aarch64_lutv2x8hiv8qi
+		 (selected, table, index_vec, const0_rtx, GEN_INT (4)));
+
+      emit_move_insn (operands[0], gen_lowpart (<VEL>mode, selected));
+      DONE;
+    }
+
+  /* The live-load fast path in tree-vect-loop.cc avoids this sequence for
+     memory values by reloading the selected scalar element.  This expander is
+     for values that are already in vector registers, such as the register
+     live-out cases in gcc.target/aarch64/noeffect11.c.
+
+     INDEX_FIRST_ACTIVE gives the active element number.  Turn it into a byte
+     offset, duplicate the offset across a byte vector and add {0, 1, ... 15}.
+     TBL then moves the selected element's bytes down to lane 0.  This avoids
+     transferring both 64-bit halves to general registers before choosing which
+     half contains the first active element.  */
+  unsigned int elt_bytes = GET_MODE_UNIT_SIZE (<MODE>mode);
+  if (elt_bytes > 1)
+    {
+      rtx byte_index = gen_reg_rtx (DImode);
+      emit_move_insn (byte_index,
+		      gen_rtx_ASHIFT (DImode, index,
+				      GEN_INT (exact_log2 (elt_bytes))));
+      index = byte_index;
+    }
+
+  rtx byte_index_vec = gen_reg_rtx (V16QImode);
+  emit_insn (gen_aarch64_simd_dupv16qi (byte_index_vec,
+					gen_lowpart (QImode, index)));
+
+  rtvec v = rtvec_alloc (16);
+  for (unsigned int i = 0; i < 16; ++i)
+    RTVEC_ELT (v, i) = GEN_INT (i);
+  rtx byte_numbers = force_reg (V16QImode,
+				gen_rtx_CONST_VECTOR (V16QImode, v));
+
+  rtx selector = gen_reg_rtx (V16QImode);
+  emit_insn (gen_addv16qi3 (selector, byte_index_vec, byte_numbers));
+
+  rtx selected = gen_reg_rtx (V16QImode);
+  emit_insn (gen_aarch64_qtbl1v16qi (selected,
+				     gen_lowpart (V16QImode, operands[2]),
+				     selector));
+
+  emit_move_insn (operands[0], gen_lowpart (<VEL>mode, selected));
+  DONE;
+})
+
+(define_expand "extract_first_<mode>"
+  [(match_operand:<VEL> 0 "register_operand")
+   (match_operand:<V_INT_EQUIV> 1 "register_operand")
+   (match_operand:VQ_2E 2 "register_operand")]
+  "TARGET_SIMD"
+{
+  /* For example: addhn; ctz; umov xlo; umov xhi; csel.  */
+  rtx index = gen_reg_rtx (DImode);
+  emit_insn (gen_index_first_active_<v_int_equiv> (index, operands[1]));
+
+  /* V2DI/V2DF only need to choose between the two 64-bit halves.  This keeps
+     the sequence in the scalar integer domain after INDEX_FIRST_ACTIVE and
+     avoids building a vector table control.  */
+  rtx lo = gen_reg_rtx (DImode);
+  emit_insn (gen_aarch64_get_lanev2di (lo, gen_lowpart (V2DImode, operands[2]),
+				       aarch64_endian_lane_rtx (V2DImode,
+								0)));
+
+  rtx hi = gen_reg_rtx (DImode);
+  emit_insn (gen_aarch64_get_lanev2di (hi, gen_lowpart (V2DImode, operands[2]),
+				       aarch64_endian_lane_rtx (V2DImode,
+								1)));
+
+  rtx cc_reg = aarch64_gen_compare_reg (NE, index, const0_rtx);
+  rtx cmp = gen_rtx_NE (VOIDmode, cc_reg, const0_rtx);
+  rtx data = gen_reg_rtx (DImode);
+  emit_insn (gen_rtx_SET (data, gen_rtx_IF_THEN_ELSE (DImode, cmp, hi, lo)));
+
+  emit_move_insn (operands[0], gen_lowpart (<VEL>mode, data));
+  DONE;
+})
+
+(define_expand "take_<take_active>_active_<mode>"
+  [(set (match_operand:VQ_I 0 "register_operand")
+	(neg:VQ_I
+	  (TAKE_ACTIVE_CMP:VQ_I
+	    (match_operand:VQ_I 1 "register_operand")
+	    (match_dup 1))))]
+  "TARGET_SIMD"
+{
+  /* For example: addhn; ctz; dup; cmhs.  */
+  rtx index = gen_reg_rtx (DImode);
+  emit_insn (gen_index_first_active_<mode> (index, operands[1]));
+
+  rtx elt_index = gen_lowpart (<VEL>mode, index);
+  rtx index_vec = gen_reg_rtx (<MODE>mode);
+  emit_insn (gen_aarch64_simd_dup<mode> (index_vec, elt_index));
+
+  rtvec v = rtvec_alloc (<nunits>);
+  for (unsigned int i = 0; i < <nunits>; ++i)
+    RTVEC_ELT (v, i) = GEN_INT (i);
+  rtx lane_numbers = force_reg (<MODE>mode,
+				gen_rtx_CONST_VECTOR (<MODE>mode, v));
+
+  emit_insn (gen_vec_cmp<mode><v_int_equiv> (operands[0],
+					     gen_rtx_<CMP> (VOIDmode,
+							    lane_numbers,
+							    index_vec),
+					     lane_numbers, index_vec));
+  DONE;
+})
+
+;; If a vector compare is used only to feed a vector branch, use an SVE
+;; predicate compare and PTEST instead of materializing and reducing an
+;; Adv.SIMD mask.  Keep this as a combine-time split rather than doing it in
+;; the vec_cbranch expander, since the compare mask might also be needed later
+;; for live-value extraction.  In that case the ADDHN result remains live after
+;; the branch and combine cannot consume the reduction into this pattern.
+;;
+;; This must be a define_split rather than a define_insn_and_split.  The split
+;; needs the original compare producer to be the previous insn; if that stops
+;; being true, FAILing is correct during combine, but would be an ICE if a
+;; recognized "#" insn survived until final.
+(define_split
+  [(set (pc)
+	(if_then_else
+	  (match_operator 0 "aarch64_equality_operator"
+	    [(subreg:DI
+	       (truncate:<VADDHN_MASK_NARROW>
+		 (ashiftrt:<VADDHN_MASK_WIDE>
+		   (plus:<VADDHN_MASK_WIDE>
+		     (subreg:<VADDHN_MASK_WIDE>
+		       (match_operand:VQ_I 1 "register_operand") 0)
+		     (subreg:<VADDHN_MASK_WIDE>
+		       (match_dup 1) 0))
+		   (match_operand:<VADDHN_MASK_WIDE> 3
+		     "aarch64_simd_shift_imm_vec_exact_top")))
+	       0)
+	     (const_int 0)])
+	  (label_ref (match_operand 2 ""))
+	  (pc)))]
+  "TARGET_SIMD
+   && TARGET_SVE
+   && can_create_pseudo_p ()
+   && known_eq (128, GET_MODE_BITSIZE (<MODE>mode))"
+  [(const_int 0)]
+{
+  if (find_regno_note (curr_insn, REG_DEAD, REGNO (operands[1])) == NULL_RTX)
+    FAIL;
+
+  rtx_insn *prev = prev_nonnote_nondebug_insn (curr_insn);
+  rtx set = prev ? single_set (prev) : NULL_RTX;
+  if (!set || !rtx_equal_p (SET_DEST (set), operands[1]))
+    FAIL;
+
+  rtx src = SET_SRC (set);
+  if (GET_CODE (src) != NEG || !COMPARISON_P (XEXP (src, 0)))
+    FAIL;
+
+  rtx cmp = XEXP (src, 0);
+  rtx_code cmp_code = GET_CODE (cmp);
+  machine_mode full_mode = aarch64_full_sve_mode (<VEL>mode).require ();
+
+  rtx in1 = force_lowpart_subreg (full_mode, XEXP (cmp, 0), <MODE>mode);
+  rtx in2;
+  if (CONST0_RTX (<MODE>mode) == XEXP (cmp, 1))
+    in2 = CONST0_RTX (full_mode);
+  else
+    in2 = force_lowpart_subreg (full_mode, XEXP (cmp, 1), <MODE>mode);
+
+  machine_mode pred_mode = aarch64_sve_pred_mode (full_mode);
+  unsigned int vl = exact_div (GET_MODE_BITSIZE (<MODE>mode), 8).to_constant ();
+  rtx ptrue = aarch64_gen_shareable_ptrue (VNx16BImode, vl);
+  rtx cast_ptrue = gen_lowpart (pred_mode, ptrue);
+  rtx hint = gen_int_mode (SVE_MAYBE_NOT_PTRUE, SImode);
+  rtx tmp = gen_reg_rtx (pred_mode);
+  emit_insn (gen_aarch64_pred_cmp_ptest (cmp_code, full_mode, tmp, ptrue,
+					 in1, in2, cast_ptrue, hint,
+					 cast_ptrue, hint));
+
+  rtx cc_reg = gen_rtx_REG (CC_NZCmode, CC_REGNUM);
+  rtx cmp_reg = gen_rtx_fmt_ee (GET_CODE (operands[0]), VOIDmode,
+				cc_reg, const0_rtx);
+  emit_jump_insn (gen_aarch64_bcond (cmp_reg, cc_reg, operands[2]));
+  DONE;
+})
+
 ;; Define vec_cbranch_any and vec_cbranch_all
 ;; Vector comparison and branch for Adv. SIMD Integer types using SVE
 ;; instructions.
@@ -4526,62 +4848,29 @@ (define_expand "<optab><mode>"
 	 CBRANCH_CMP))]
   "TARGET_SIMD"
 {
-  auto code = GET_CODE (operands[0]);
-  if (TARGET_SVE)
-    {
-      machine_mode full_mode = aarch64_full_sve_mode (<VEL>mode).require ();
-
-      rtx in1 = force_lowpart_subreg (full_mode, operands[1], <MODE>mode);
-      rtx in2;
-      if (CONST0_RTX (<MODE>mode) == operands[2])
-	in2 = CONST0_RTX (full_mode);
-      else
-	in2 = force_lowpart_subreg (full_mode, operands[2], <MODE>mode);
-
-      unsigned lanes
-	= exact_div (GET_MODE_BITSIZE (<MODE>mode), 8).to_constant ();
-      machine_mode pred_mode = aarch64_sve_pred_mode (full_mode);
-      rtx ptrue = aarch64_ptrue_reg (VNx16BImode, lanes);
-      rtx hint = gen_int_mode (SVE_MAYBE_NOT_PTRUE, SImode);
-
-      rtx tmp = gen_reg_rtx (pred_mode);
-      rtx cast_ptrue = gen_lowpart (pred_mode, ptrue);
-
-      if (FLOAT_MODE_P (full_mode))
-	{
-	  aarch64_expand_sve_vec_cmp<sve_cmp_suff> (tmp, code, in1, in2);
-	  emit_insn (gen_and3 (pred_mode, tmp, tmp, cast_ptrue));
-	  emit_insn (gen_aarch64_ptest (pred_mode, ptrue, cast_ptrue, hint,
-					tmp));
-	}
-      else
-	emit_insn (gen_aarch64_pred_cmp_ptest (code, full_mode, tmp, ptrue, in1,
-					       in2, cast_ptrue, hint,
-					       cast_ptrue, hint));
-
-      rtx cc_reg = gen_rtx_REG (CC_NZCmode, CC_REGNUM);
-      rtx cmp_reg = gen_rtx_<cbranch_op> (VOIDmode, cc_reg, const0_rtx);
-      emit_jump_insn (gen_aarch64_bcond (cmp_reg, cc_reg, operands[3]));
-      DONE;
-    }
-
   rtx tmp = gen_reg_rtx (<V_INT_EQUIV>mode);
   emit_insn (gen_vec_cmp<mode><v_int_equiv> (tmp, operands[0], operands[1],
 					     operands[2]));
 
-  /* For 128-bit vectors we need a reduction to 64-bit first.  */
-  if (known_eq (128, GET_MODE_BITSIZE (<MODE>mode)))
+  rtx val = gen_reg_rtx (DImode);
+  rtx addhn_scalar
+    = maybe_gen_aarch64_addhn_scalar (<V_INT_EQUIV>mode, val, tmp);
+  if (known_eq (128, GET_MODE_BITSIZE (<MODE>mode)) && addhn_scalar)
+    emit_insn (addhn_scalar);
+  else
     {
-      /* Always reduce using a V4SI.  */
-      rtx reduc = gen_lowpart (V4SImode, tmp);
-      rtx res = gen_reg_rtx (V4SImode);
-      emit_insn (gen_aarch64_umaxpv4si (res, reduc, reduc));
-      emit_move_insn (tmp, gen_lowpart (<V_INT_EQUIV>mode, res));
+      /* For 128-bit vectors we need a reduction to 64-bit first.  */
+      if (known_eq (128, GET_MODE_BITSIZE (<MODE>mode)))
+	{
+	  /* Always reduce using a V4SI.  */
+	  rtx reduc = gen_lowpart (V4SImode, tmp);
+	  rtx res = gen_reg_rtx (V4SImode);
+	  emit_insn (gen_aarch64_umaxpv4si (res, reduc, reduc));
+	  emit_move_insn (tmp, gen_lowpart (<V_INT_EQUIV>mode, res));
+	}
+      emit_move_insn (val, gen_lowpart (DImode, tmp));
     }
 
-  rtx val = gen_reg_rtx (DImode);
-  emit_move_insn (val, gen_lowpart (DImode, tmp));
-
   rtx cc_reg = aarch64_gen_compare_reg (<cbranch_op>, val, const0_rtx);
   rtx cmp_rtx = gen_rtx_fmt_ee (<cbranch_op>, DImode, cc_reg, const0_rtx);
   emit_jump_insn (gen_aarch64_bcond (cmp_rtx, cc_reg, operands[3]));
diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc
index 8e62ab6a72e0043fffd16d3d187f18da7fd41d4e..77e29815f224a9e3bfc2ab3c5662c5d0aedf753e 100644
--- a/gcc/config/aarch64/aarch64.cc
+++ b/gcc/config/aarch64/aarch64.cc
@@ -25316,6 +25316,28 @@ aarch64_gen_shareable_zero (machine_mode mode)
   return lowpart_subreg (mode, reg, GET_MODE (reg));
 }
 
+/* Return a shareable predicate register whose first VL bytes are active,
+   viewed in mode MODE.
+   This is for fixed-width Adv.SIMD sequences that want to use SVE flags for
+   a vector branch.  Hoisting the predicate constant avoids recreating it in
+   every loop iteration.  */
+
+rtx
+aarch64_gen_shareable_ptrue (machine_mode mode, unsigned int vl)
+{
+  gcc_assert (vl == 8 || vl == 16);
+
+  rtx_vector_builder builder (VNx16BImode, vl, 2);
+  for (unsigned int i = 0; i < vl; ++i)
+    builder.quick_push (CONST1_RTX (BImode));
+  for (unsigned int i = 0; i < vl; ++i)
+    builder.quick_push (CONST0_RTX (BImode));
+
+  rtx reg = aarch64_get_shareable_reg (&cfun->machine->advsimd_ptrue_insn,
+				       builder.build ());
+  return lowpart_subreg (mode, reg, GET_MODE (reg));
+}
+
 /* INSN is some form of extension or shift that can be split into a
    permutation involving a shared zero.  Return true if we should
    perform such a split.
@@ -28913,7 +28935,7 @@ aarch64_vectorize_vec_perm_const (machine_mode vmode, machine_mode op_mode,
     {
       d.op1 = op1;
       if (d.op1 && !register_operand (d.op1, op_mode))
-       d.op1 = force_reg (op_mode, d.op1);
+	d.op1 = force_reg (op_mode, d.op1);
     }
   d.testing_p = !target;
 
@@ -28926,6 +28948,7 @@ aarch64_vectorize_vec_perm_const (machine_mode vmode, machine_mode op_mode,
 
   return ret;
 }
+
 /* Generate a byte permute mask for a register of mode MODE,
    which has NUNITS units.  */
 
diff --git a/gcc/config/aarch64/aarch64.h b/gcc/config/aarch64/aarch64.h
index fdedb04391127c5a1b821ed5d4b399c1c28d9f31..0673c8a64f7097e2d2ec4a155edeb12dfe72e957 100644
--- a/gcc/config/aarch64/aarch64.h
+++ b/gcc/config/aarch64/aarch64.h
@@ -1249,6 +1249,10 @@ typedef struct GTY (()) machine_function
      can be reused by other code.  */
   rtx_insn *advsimd_zero_insn;
 
+  /* Like advsimd_zero_insn, but for a predicate with a fixed number of
+     leading true bytes.  */
+  rtx_insn *advsimd_ptrue_insn;
+
   /* During SEH output, this is non-null.  */
   struct seh_frame_state * GTY ((skip (""))) seh;
 
diff --git a/gcc/config/aarch64/iterators.md b/gcc/config/aarch64/iterators.md
index b0571460ec7ded9252bc2a216b6cd8867e6a595b..af417b51ccf31bf56d4e4f368979971c05e0b482 100644
--- a/gcc/config/aarch64/iterators.md
+++ b/gcc/config/aarch64/iterators.md
@@ -2181,6 +2181,25 @@ (define_mode_attr Vdblw [(V8QI "v4hi") (V16QI "v8hi")
 			 (V4HI "v2si") (V8HI "v4si")
 			 (V2SI "di")   (V4SI "v2di")])
 
+;; Source mode for using ADDHN to compress a boolean mask to a scalar.
+(define_mode_attr VADDHN_MASK_WIDE [(V16QI "V8HI") (V8HI "V4SI")
+				    (V4SI "V2DI") (V2DI "V2DI")])
+
+;; Result mode for using ADDHN to compress a boolean mask to a scalar.
+(define_mode_attr VADDHN_MASK_NARROW [(V16QI "V8QI") (V8HI "V4HI")
+				      (V4SI "V2SI") (V2DI "V2SI")])
+
+;; Source mode suffix for using ADDHN to compress a boolean mask to a scalar.
+(define_mode_attr vaddhn_mask_wide [(V16QI "v8hi") (V8HI "v4si")
+				    (V4SI "v2di") (V2DI "v2di")])
+
+;; Scalar mask for the useful bits produced by using ADDHN to compress a
+;; boolean mask to a scalar.
+(define_mode_attr vaddhn_lane_mask [(V16QI "0x1111111111111111")
+				    (V8HI "0x0101010101010101")
+				    (V4SI "0x0001000100010001")
+				    (V2DI "0x0000000100000001")])
+
 (define_mode_attr VQUADW [(V8QI "V4SI") (V16QI "V8SI")
                   (V4HI "V2DI") (V8HI "V4DI")])
 
@@ -3237,6 +3256,9 @@ (define_code_iterator SAT_MINUS [ss_minus us_minus])
 ;; Comparison operators for <F>CM.
 (define_code_iterator COMPARISONS [lt le eq ge gt])
 
+;; Comparison operators for TAKE_{EXCL,INCL}_ACTIVE.
+(define_code_iterator TAKE_ACTIVE_CMP [lt le])
+
 ;; Unsigned comparison operators.
 (define_code_iterator UCOMPARISONS [ltu leu geu gtu])
 
@@ -3374,6 +3396,9 @@ (define_code_attr cmp_op [(lt "lt")
 			  (geu "hs")
 			  (gtu "hi")])
 
+;; The TAKE_ACTIVE optab suffix associated with an rtl comparison code.
+(define_code_attr take_active [(lt "excl") (le "incl")])
+
 (define_code_attr inv_cmp_op [(lt "ge")
 			  (le "gt")
 			  (eq "ne")
diff --git a/gcc/testsuite/gcc.target/aarch64/noeffect11.c b/gcc/testsuite/gcc.target/aarch64/noeffect11.c
index 82c2f00c7cb16df22fa051b8d5d421863321e464..875e3ca184d925691eaf99d3d45b8cfab8b6f3e6 100644
--- a/gcc/testsuite/gcc.target/aarch64/noeffect11.c
+++ b/gcc/testsuite/gcc.target/aarch64/noeffect11.c
@@ -8,7 +8,10 @@
 **	...
 **	cmhi	v[0-9]+\.2d, v[0-9]+\.2d, v[0-9]+\.2d
 **	...
-**	umaxp	v[0-9]+\.4s, v[0-9]+\.4s, v[0-9]+\.4s
+**	addhn	v[0-9]+\.2s, v[0-9]+\.2d, v[0-9]+\.2d
+**	addhn	v[0-9]+\.2s, v[0-9]+\.2d, v[0-9]+\.2d
+**	fmov	x[0-9]+, d[0-9]+
+**	cbz	x[0-9]+, \.L[0-9]+
 **	...
 **	ldr	x[0-9]+, \[x[0-9]+, x[0-9]+\]
 **	ldr	x[0-9]+, \[x[0-9]+, x[0-9]+\]
@@ -34,7 +37,10 @@ int f1 (const unsigned long *restrict a, const unsigned long *b, int n)
 **	...
 **	cmhi	v[0-9]+\.2d, v[0-9]+\.2d, v[0-9]+\.2d
 **	...
-**	umaxp	v[0-9]+\.4s, v[0-9]+\.4s, v[0-9]+\.4s
+**	addhn	v[0-9]+\.2s, v[0-9]+\.2d, v[0-9]+\.2d
+**	addhn	v[0-9]+\.2s, v[0-9]+\.2d, v[0-9]+\.2d
+**	fmov	x[0-9]+, d[0-9]+
+**	cbz	x[0-9]+, \.L[0-9]+
 **	...
 **	ldr	x[0-9]+, \[x[0-9]+, x[0-9]+\]
 **	ldr	x[0-9]+, \[x[0-9]+, x[0-9]+\]
diff --git a/gcc/testsuite/gcc.target/aarch64/noeffect12.c b/gcc/testsuite/gcc.target/aarch64/noeffect12.c
new file mode 100644
index 0000000000000000000000000000000000000000..a42d67b56c47ed07497a40ea4758f07aea2bae4a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/noeffect12.c
@@ -0,0 +1,64 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -march=armv8-a" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+
+#define N 1008
+
+unsigned char a[N] = {0};
+unsigned char b[N] = {0};
+unsigned int c[N] = {0};
+unsigned int d[N] = {0};
+
+/*
+** f1:
+**	...
+**	cmhi	v[0-9]+\.16b, v[0-9]+\.16b, v[0-9]+\.16b
+**	addhn	v[0-9]+\.8b, v[0-9]+\.8h, v[0-9]+\.8h
+**	umov	x[0-9]+, v[0-9]+\.d\[0\]
+**	cbz	x[0-9]+, \.L[0-9]+
+**	...
+**	tbl	v[0-9]+\.16b, \{v[0-9]+\.16b\}, v[0-9]+\.16b
+**	fmov	w0, s[0-9]+
+**	ret
+**	...
+*/
+__attribute__ ((noipa, noinline))
+unsigned char f1 (void)
+{
+  unsigned char res = 0;
+  for (unsigned int i = 0; i < N; ++i)
+    {
+      unsigned char x = a[i] & b[i];
+      res = x;
+      if (x > 5)
+	break;
+    }
+  return res;
+}
+
+/*
+** f2:
+**	...
+**	cmhi	v[0-9]+\.4s, v[0-9]+\.4s, v[0-9]+\.4s
+**	addhn	v[0-9]+\.2s, v[0-9]+\.2d, v[0-9]+\.2d
+**	umov	x[0-9]+, v[0-9]+\.d\[0\]
+**	cbz	x[0-9]+, \.L[0-9]+
+**	...
+**	tbl	v[0-9]+\.16b, \{v[0-9]+\.16b\}, v[0-9]+\.16b
+**	fmov	w0, s[0-9]+
+**	ret
+**	...
+*/
+__attribute__ ((noipa, noinline))
+unsigned int f2 (void)
+{
+  unsigned int res = 0;
+  for (unsigned int i = 0; i < 1000; ++i)
+    {
+      unsigned int x = c[i] & d[i];
+      res = x;
+      if (x > 5)
+	break;
+    }
+  return res;
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/noeffect13-run.c b/gcc/testsuite/gcc.target/aarch64/noeffect13-run.c
new file mode 100644
index 0000000000000000000000000000000000000000..9e9eb715c83f6bda7981f36d1682908a8380c114
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/noeffect13-run.c
@@ -0,0 +1,62 @@
+/* { dg-do run { target aarch64_lut_hw } } */
+/* { dg-options "-O3 -march=armv9-a+lut -mautovec-preference=asimd-only" } */
+
+#define N 1008
+
+unsigned char a[N];
+unsigned char b[N];
+unsigned short c[N];
+unsigned short d[N];
+
+__attribute__ ((noipa, noinline))
+unsigned char f1 (void)
+{
+  unsigned char res = 0;
+  for (unsigned int i = 0; i < N; ++i)
+    {
+      unsigned char x = a[i] & b[i];
+      res = x;
+      if (x > 5)
+	break;
+    }
+  return res;
+}
+
+__attribute__ ((noipa, noinline))
+unsigned short f2 (void)
+{
+  unsigned short res = 0;
+  for (unsigned int i = 0; i < N; ++i)
+    {
+      unsigned short x = c[i] & d[i];
+      res = x;
+      if (x > 5)
+	break;
+    }
+  return res;
+}
+
+int
+main (void)
+{
+  for (unsigned int i = 0; i < N; ++i)
+    {
+      a[i] = 1;
+      b[i] = 1;
+      c[i] = 1;
+      d[i] = 1;
+    }
+
+  if (f1 () != 1 || f2 () != 1)
+    __builtin_abort ();
+
+  a[13] = 7;
+  b[13] = 7;
+  c[11] = 9;
+  d[11] = 9;
+
+  if (f1 () != 7 || f2 () != 9)
+    __builtin_abort ();
+
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/noeffect13.c b/gcc/testsuite/gcc.target/aarch64/noeffect13.c
new file mode 100644
index 0000000000000000000000000000000000000000..502126d6c005775b0f08c8807b3dda14c6372d34
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/noeffect13.c
@@ -0,0 +1,66 @@
+/* { dg-do compile { target aarch64_asm_lut_ok } } */
+/* { dg-options "-O3 -march=armv9-a+lut -mautovec-preference=asimd-only" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+
+#define N 1008
+
+unsigned char a[N] = {0};
+unsigned char b[N] = {0};
+unsigned short c[N] = {0};
+unsigned short d[N] = {0};
+
+/*
+** f1:
+**	...
+**	cmhi	v[0-9]+\.16b, v[0-9]+\.16b, v[0-9]+\.16b
+**	...
+**	addhn	v[0-9]+\.8b, v[0-9]+\.8h, v[0-9]+\.8h
+**	fmov	x[0-9]+, d[0-9]+
+**	...
+**	luti4	v[0-9]+\.16b, \{v[0-9]+\.16b\}, v[0-9]+\[0\]
+**	fmov	w0, s[0-9]+
+**	ret
+**	...
+*/
+__attribute__ ((noipa, noinline))
+unsigned char f1 (void)
+{
+  unsigned char res = 0;
+  for (unsigned int i = 0; i < N; ++i)
+    {
+      unsigned char x = a[i] & b[i];
+      res = x;
+      if (x > 5)
+	break;
+    }
+  return res;
+}
+
+/*
+** f2:
+**	...
+**	cmhi	v[0-9]+\.8h, v[0-9]+\.8h, v[0-9]+\.8h
+**	...
+**	addhn	v[0-9]+\.4h, v[0-9]+\.4s, v[0-9]+\.4s
+**	fmov	x[0-9]+, d[0-9]+
+**	...
+**	movi	v[0-9]+\.4s, 0
+**	...
+**	luti4	v[0-9]+\.8h, \{v[0-9]+\.8h, v[0-9]+\.8h\}, v[0-9]+\[0\]
+**	fmov	w0, s[0-9]+
+**	ret
+**	...
+*/
+__attribute__ ((noipa, noinline))
+unsigned short f2 (void)
+{
+  unsigned short res = 0;
+  for (unsigned int i = 0; i < N; ++i)
+    {
+      unsigned short x = c[i] & d[i];
+      res = x;
+      if (x > 5)
+	break;
+    }
+  return res;
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/noeffect2.c b/gcc/testsuite/gcc.target/aarch64/noeffect2.c
index 08c531fb18c817b018776a88db4f62eb03b001cb..bcd8848038527d3af1a4966ef55580e262b5d871 100644
--- a/gcc/testsuite/gcc.target/aarch64/noeffect2.c
+++ b/gcc/testsuite/gcc.target/aarch64/noeffect2.c
@@ -13,7 +13,7 @@ int b[N] = {0};
 **	ldr	q[0-9]+, \[x[0-9]+, x[0-9]+\]
 **	add	x[0-9]+, x[0-9]+, 16
 **	cmgt	v[0-9]+\.4s, v[0-9]+\.4s, v[0-9]+\.4s
-**	umaxp	v[0-9]+\.4s, v[0-9]+\.4s, v[0-9]+\.4s
+**	addhn	v[0-9]+\.2s, v[0-9]+\.2d, v[0-9]+\.2d
 **	fmov	x[0-9]+, d[0-9]+
 **	cbz	x[0-9]+, \.L[0-9]+
 **	mov	w0, 1
diff --git a/gcc/testsuite/gcc.target/aarch64/noeffect3.c b/gcc/testsuite/gcc.target/aarch64/noeffect3.c
index 886ad7bda2f3ebd5048cc618bcf4fe473cd2f820..f4ece387fc355b5a7d8c23b416400ec81da47051 100644
--- a/gcc/testsuite/gcc.target/aarch64/noeffect3.c
+++ b/gcc/testsuite/gcc.target/aarch64/noeffect3.c
@@ -10,11 +10,14 @@ int b[N] = {0};
 ** foo:
 **	...
 **	cmgt	v[0-9]+\.4s, v[0-9]+\.4s, v[0-9]+\.4s
-**	umaxp	v[0-9]+\.4s, v[0-9]+\.4s, v[0-9]+\.4s
-**	...
-**	ldr	w[0-9]+, \[x[0-9]+, w[0-9]+, sxtw 2\]
-**	ldr	w[0-9]+, \[x[0-9]+, w[0-9]+, sxtw 2\]
-**	cmp	w[0-9]+, w[0-9]+
+**	addhn	v[0-9]+\.2s, v[0-9]+\.2d, v[0-9]+\.2d
+**	umov	x[0-9]+, v[0-9]+\.d\[0\]
+**	cbz	x[0-9]+, \.L[0-9]+
+**	and	x[0-9]+, x[0-9]+, 281479271743489
+**	rbit	x[0-9]+, x[0-9]+
+**	clz	x[0-9]+, x[0-9]+
+**	lsr	x[0-9]+, x[0-9]+, 4
+**	add	w0, w0, w[0-9]+
 **	...
 **	ret
 */
diff --git a/gcc/testsuite/gcc.target/aarch64/noeffect4.c b/gcc/testsuite/gcc.target/aarch64/noeffect4.c
index 276843c9bbd82d9fd409418734cc9a39cddf0aeb..6a477d165f7fa006ef6cf34ae3e84abba9a63fe0 100644
--- a/gcc/testsuite/gcc.target/aarch64/noeffect4.c
+++ b/gcc/testsuite/gcc.target/aarch64/noeffect4.c
@@ -10,12 +10,18 @@ short b[N] = {0};
 ** foo:
 **	...
 **	cmgt	v[0-9]+\.8h, v[0-9]+\.8h, v[0-9]+\.8h
-**	umaxp	v[0-9]+\.4s, v[0-9]+\.4s, v[0-9]+\.4s
-**	...
-**	ldrsh	w[0-9]+, \[[^\n]+\]
-**	ldrsh	w[0-9]+, \[[^\n]+\]
-**	cmp	w[0-9]+, w[0-9]+
+**	addhn	v[0-9]+\.4h, v[0-9]+\.4s, v[0-9]+\.4s
+**	umov	x[0-9]+, v[0-9]+\.d\[0\]
+**	cbz	x[0-9]+, \.L[0-9]+
+**	and	x[0-9]+, x[0-9]+, 72340172838076673
+**	rbit	x[0-9]+, x[0-9]+
+**	clz	x[0-9]+, x[0-9]+
+**	ubfx	x[0-9]+, x[0-9]+, 3, 32
+**	add	w[0-9]+, w[0-9]+, w[0-9]+
+**	and	w[0-9]+, w[0-9]+, 65535
+**	ret
 **	...
+**	mov	w0, 0
 **	ret
 */
 __attribute__ ((noipa, noinline))
@@ -28,3 +34,5 @@ int foo (void)
     }
   return 0;
 }
+
+/* { dg-final { scan-assembler-not {\tldrsh\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/noeffect5.c b/gcc/testsuite/gcc.target/aarch64/noeffect5.c
index c15e52ebdfc4d07b0e6caf12f1c8e0c6d41d377a..e463509a299eb84323397cae68aeda84fa4c543a 100644
--- a/gcc/testsuite/gcc.target/aarch64/noeffect5.c
+++ b/gcc/testsuite/gcc.target/aarch64/noeffect5.c
@@ -10,11 +10,14 @@ short b[N] = {0};
 ** foo:
 **	...
 **	cmgt	v[0-9]+\.8h, v[0-9]+\.8h, v[0-9]+\.8h
-**	umaxp	v[0-9]+\.4s, v[0-9]+\.4s, v[0-9]+\.4s
-**	...
-**	ldrsh	w[0-9]+, \[[^\n]+\]
-**	ldrsh	w[0-9]+, \[[^\n]+\]
-**	cmp	w[0-9]+, w[0-9]+
+**	addhn	v[0-9]+\.4h, v[0-9]+\.4s, v[0-9]+\.4s
+**	umov	x[0-9]+, v[0-9]+\.d\[0\]
+**	cbz	x[0-9]+, \.L[0-9]+
+**	and	x[0-9]+, x[0-9]+, 72340172838076673
+**	rbit	x[0-9]+, x[0-9]+
+**	clz	x[0-9]+, x[0-9]+
+**	ubfx	x[0-9]+, x[0-9]+, 3, 32
+**	add	w0, w0, w[0-9]+
 **	...
 **	ret
 */
diff --git a/gcc/testsuite/gcc.target/aarch64/noeffect6.c b/gcc/testsuite/gcc.target/aarch64/noeffect6.c
index 9da4f496a02e96f053b9c001748a91df7b31d12d..7b4c742da5b8af2e111798a8187c34789aa05bdb 100644
--- a/gcc/testsuite/gcc.target/aarch64/noeffect6.c
+++ b/gcc/testsuite/gcc.target/aarch64/noeffect6.c
@@ -10,11 +10,14 @@ short b[N] = {0};
 ** foo:
 **	...
 **	cmgt	v[0-9]+\.8h, v[0-9]+\.8h, v[0-9]+\.8h
-**	umaxp	v[0-9]+\.4s, v[0-9]+\.4s, v[0-9]+\.4s
-**	...
-**	ldrsh	w[0-9]+, \[[^\n]+\]
-**	ldrsh	w[0-9]+, \[[^\n]+\]
-**	cmp	w[0-9]+, w[0-9]+
+**	addhn	v[0-9]+\.4h, v[0-9]+\.4s, v[0-9]+\.4s
+**	umov	x[0-9]+, v[0-9]+\.d\[0\]
+**	cbz	x[0-9]+, \.L[0-9]+
+**	and	x[0-9]+, x[0-9]+, 72340172838076673
+**	rbit	x[0-9]+, x[0-9]+
+**	clz	x[0-9]+, x[0-9]+
+**	lsr	x[0-9]+, x[0-9]+, 3
+**	ldrh	w0, \[x[0-9]+, x[0-9]+, lsl 1\]
 **	...
 **	ret
 */
diff --git a/gcc/testsuite/gcc.target/aarch64/noeffect7.c b/gcc/testsuite/gcc.target/aarch64/noeffect7.c
index 16fc921117aa2a94ac7dd8fe795fe046f34cdb07..fb20481351495a63d94f7ffe8c06f0a886222f1b 100644
--- a/gcc/testsuite/gcc.target/aarch64/noeffect7.c
+++ b/gcc/testsuite/gcc.target/aarch64/noeffect7.c
@@ -1,26 +1,13 @@
 /* { dg-do compile } */
 /* { dg-options "-O3 -march=armv8-a" } */
-/* { dg-final { check-function-bodies "**" "" "" } } */
+/* { dg-final { scan-assembler {\tcmp\tw[0-9]+, w[0-9]+} } } */
+/* { dg-final { scan-assembler {\tmov\tw0, 1} } } */
+/* { dg-final { scan-assembler {\tmov\tw0, 0} } } */
 
 #define N 1000
 int a[N] = {0};
 int b[N] = {0};
 
-/*
-** foo:
-**	...
-**	add	x[0-9]+, x[0-9]+, 4000
-**	add	x[0-9]+, x[0-9]+, 2000
-**	b	\.L[0-9]+
-**	ldr	w[0-9]+, \[x[0-9]+[^\n]*\]
-**	...
-**	cmp	w[0-9]+, w[0-9]+
-**	...
-**	mov	w0, 1
-**	ret
-**	mov	w0, 0
-**	ret
-*/
 __attribute__ ((noipa, noinline))
 int foo (void)
 {
diff --git a/gcc/testsuite/gcc.target/aarch64/noeffect8.c b/gcc/testsuite/gcc.target/aarch64/noeffect8.c
index ada79391a8acab2073517b4172570947ec16a8a6..d3e03a618fa970b5480eed619e00c670ff3aaab4 100644
--- a/gcc/testsuite/gcc.target/aarch64/noeffect8.c
+++ b/gcc/testsuite/gcc.target/aarch64/noeffect8.c
@@ -6,7 +6,9 @@
 ** add:
 **	...
 **	cmeq	v[0-9]+\.4s, v[0-9]+\.4s, #0
-**	umaxp	v[0-9]+\.4s, v[0-9]+\.4s, v[0-9]+\.4s
+**	addhn	v[0-9]+\.2s, v[0-9]+\.2d, v[0-9]+\.2d
+**	fmov	x[0-9]+, d[0-9]+
+**	cbnz	x[0-9]+, \.L[0-9]+
 **	...
 **	ldr	q[0-9]+, \[x[0-9]+[^\n]*\]
 **	...
diff --git a/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch.c b/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch.c
index ca4ef498485f68dbb8a178750db3f552cf7a66d0..b29f02a4418cd3a6d87bd617d9d3fb06d5e3f0c8 100644
--- a/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch.c
+++ b/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch.c
@@ -13,7 +13,7 @@ int b[N] = {0};
 ** f1:
 **	...
 **	cmgt	v[0-9]+.4s, v[0-9]+.4s, #0
-**	umaxp	v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
+**	addhn	v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
 **	fmov	x[0-9]+, d[0-9]+
 **	cbn?z	x[0-9]+, \.L[0-9]+
 **	...
@@ -32,7 +32,7 @@ void f1 ()
 ** f2:
 **	...
 **	cmge	v[0-9]+.4s, v[0-9]+.4s, #0
-**	umaxp	v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
+**	addhn	v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
 **	fmov	x[0-9]+, d[0-9]+
 **	cbn?z	x[0-9]+, \.L[0-9]+
 **	...
@@ -50,7 +50,8 @@ void f2 ()
 /*
 ** f3:
 **	...
-**	umaxp	v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
+**	cmeq	v[0-9]+.4s, v[0-9]+.4s, #0
+**	addhn	v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
 **	fmov	x[0-9]+, d[0-9]+
 **	cbn?z	x[0-9]+, \.L[0-9]+
 **	...
@@ -68,7 +69,8 @@ void f3 ()
 /*
 ** f4:
 **	...
-**	umaxp	v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
+**	cmtst	v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
+**	addhn	v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
 **	fmov	x[0-9]+, d[0-9]+
 **	cbn?z	x[0-9]+, \.L[0-9]+
 **	...
@@ -87,7 +89,7 @@ void f4 ()
 ** f5:
 **	...
 **	cmlt	v[0-9]+.4s, v[0-9]+.4s, #0
-**	umaxp	v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
+**	addhn	v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
 **	fmov	x[0-9]+, d[0-9]+
 **	cbn?z	x[0-9]+, \.L[0-9]+
 **	...
@@ -106,7 +108,7 @@ void f5 ()
 ** f6:
 **	...
 **	cmle	v[0-9]+.4s, v[0-9]+.4s, #0
-**	umaxp	v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
+**	addhn	v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
 **	fmov	x[0-9]+, d[0-9]+
 **	cbn?z	x[0-9]+, \.L[0-9]+
 **	...
diff --git a/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_2.c b/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_2.c
index d5cb1946873f5e720c6e12e0dd4991582764f18f..e40843f11b046dc8b8c99428a93da5ab6234d16e 100644
--- a/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_2.c
+++ b/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_2.c
@@ -10,8 +10,10 @@ int b[N] = {0};
 /*
 ** f1:
 **	...
-**	cmpgt	p[0-9]+.s, p[0-9]+/z, z[0-9]+.s, #0
-**	b(\.?eq|\.none)	\.L[0-9]+
+**	cmgt	v[0-9]+.4s, v[0-9]+.4s, #0
+**	addhn	v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
+**	fmov	x[0-9]+, d[0-9]+
+**	cbz	x[0-9]+, \.L[0-9]+
 **	...
 */
 void f1 ()
@@ -26,8 +28,10 @@ void f1 ()
 /*
 ** f2:
 **	...
-**	cmpge	p[0-9]+.s, p[0-9]+/z, z[0-9]+.s, #0
-**	b(\.?eq|\.none)	\.L[0-9]+
+**	cmge	v[0-9]+.4s, v[0-9]+.4s, #0
+**	addhn	v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
+**	fmov	x[0-9]+, d[0-9]+
+**	cbz	x[0-9]+, \.L[0-9]+
 **	...
 */
 void f2 ()
@@ -42,8 +46,10 @@ void f2 ()
 /*
 ** f3:
 **	...
-**	cmpeq	p[0-9]+.s, p[0-9]+/z, z[0-9]+.s, #0
-**	b(\.?eq|\.none)	\.L[0-9]+
+**	cmeq	v[0-9]+.4s, v[0-9]+.4s, #0
+**	addhn	v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
+**	fmov	x[0-9]+, d[0-9]+
+**	cbz	x[0-9]+, \.L[0-9]+
 **	...
 */
 void f3 ()
@@ -58,8 +64,10 @@ void f3 ()
 /*
 ** f4:
 **	...
-**	cmpne	p[0-9]+.s, p[0-9]+/z, z[0-9]+.s, #0
-**	b(\.?eq|\.none)	\.L[0-9]+
+**	cmtst	v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
+**	addhn	v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
+**	fmov	x[0-9]+, d[0-9]+
+**	cbz	x[0-9]+, \.L[0-9]+
 **	...
 */
 void f4 ()
@@ -74,8 +82,10 @@ void f4 ()
 /*
 ** f5:
 **	...
-**	cmplt	p[0-9]+.s, p7/z, z[0-9]+.s, #0
-**	b(\.?eq|\.none)	.L[0-9]+
+**	cmlt	v[0-9]+.4s, v[0-9]+.4s, #0
+**	addhn	v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
+**	fmov	x[0-9]+, d[0-9]+
+**	cbz	x[0-9]+, \.L[0-9]+
 **	...
 */
 void f5 ()
@@ -90,8 +100,10 @@ void f5 ()
 /*
 ** f6:
 **	...
-**	cmple	p[0-9]+.s, p[0-9]+/z, z[0-9]+.s, #0
-**	b(\.?eq|\.none)	\.L[0-9]+
+**	cmle	v[0-9]+.4s, v[0-9]+.4s, #0
+**	addhn	v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
+**	fmov	x[0-9]+, d[0-9]+
+**	cbz	x[0-9]+, \.L[0-9]+
 **	...
 */
 void f6 ()
diff --git a/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_3.c b/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_3.c
index b252b80ef291efd4f6fc98224a44ef7fadc68d6c..9e4d3f951ad111df0f76032445ccce8673bca74e 100644
--- a/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_3.c
+++ b/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_3.c
@@ -11,9 +11,10 @@ float b[N] = {0};
 /*
 ** f1:
 **	...
-**	fcmgt	p[0-9]+.s, p[0-9]+/z, z[0-9]+.s, #0.0
-**	ptest	p[0-9]+, p[0-9]+\.b
-**	b(\.?eq|\.none)	\.L[0-9]+
+**	fcmgt	v[0-9]+.4s, v[0-9]+.4s, 0
+**	addhn	v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
+**	fmov	x[0-9]+, d[0-9]+
+**	cbz	x[0-9]+, \.L[0-9]+
 **	...
 */
 void f1 ()
@@ -28,9 +29,10 @@ void f1 ()
 /*
 ** f2:
 **	...
-**	fcmge	p[0-9]+.s, p[0-9]+/z, z[0-9]+.s, #0.0
-**	ptest	p[0-9]+, p[0-9]+\.b
-**	b(\.?eq|\.none)	\.L[0-9]+
+**	fcmge	v[0-9]+.4s, v[0-9]+.4s, 0
+**	addhn	v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
+**	fmov	x[0-9]+, d[0-9]+
+**	cbz	x[0-9]+, \.L[0-9]+
 **	...
 */
 void f2 ()
@@ -45,9 +47,10 @@ void f2 ()
 /*
 ** f3:
 **	...
-**	fcmeq	p[0-9]+.s, p[0-9]+/z, z[0-9]+.s, #0.0
-**	ptest	p[0-9]+, p[0-9]+\.b
-**	b(\.?eq|\.none)	\.L[0-9]+
+**	fcmeq	v[0-9]+.4s, v[0-9]+.4s, 0
+**	addhn	v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
+**	fmov	x[0-9]+, d[0-9]+
+**	cbz	x[0-9]+, \.L[0-9]+
 **	...
 */
 void f3 ()
@@ -62,9 +65,11 @@ void f3 ()
 /*
 ** f4:
 **	...
-**	fcmne	p[0-9]+.s, p[0-9]+/z, z[0-9]+.s, #0.0
-**	ptest	p[0-9]+, p[0-9]+\.b
-**	b(\.?eq|\.none)	\.L[0-9]+
+**	fcmeq	v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
+**	not	v[0-9]+.16b, v[0-9]+.16b
+**	addhn	v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
+**	fmov	x[0-9]+, d[0-9]+
+**	cbz	x[0-9]+, \.L[0-9]+
 **	...
 */
 void f4 ()
@@ -79,9 +84,10 @@ void f4 ()
 /*
 ** f5:
 **	...
-**	fcmlt	p[0-9]+.s, p7/z, z[0-9]+.s, #0.0
-**	ptest	p[0-9]+, p[0-9]+\.b
-**	b(\.?eq|\.none)	.L[0-9]+
+**	fcmlt	v[0-9]+.4s, v[0-9]+.4s, 0
+**	addhn	v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
+**	fmov	x[0-9]+, d[0-9]+
+**	cbz	x[0-9]+, \.L[0-9]+
 **	...
 */
 void f5 ()
@@ -96,9 +102,10 @@ void f5 ()
 /*
 ** f6:
 **	...
-**	fcmle	p[0-9]+.s, p[0-9]+/z, z[0-9]+.s, #0.0
-**	ptest	p[0-9]+, p[0-9]+\.b
-**	b(\.?eq|\.none)	\.L[0-9]+
+**	fcmle	v[0-9]+.4s, v[0-9]+.4s, 0
+**	addhn	v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
+**	fmov	x[0-9]+, d[0-9]+
+**	cbz	x[0-9]+, \.L[0-9]+
 **	...
 */
 void f6 ()
diff --git a/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_4.c b/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_4.c
index a49e7963cfa1afdce4bb1bf33224a1d79eb7b874..026c774e4de7f6381728f164fbfd89f8643f7c3d 100644
--- a/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_4.c
+++ b/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_4.c
@@ -13,9 +13,9 @@ unsigned int b[N] = {0};
 ** f1:
 **	...
 **	cmhi	v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
-**	umaxp	v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
+**	addhn	v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
 **	fmov	x[0-9]+, d[0-9]+
-**	cbn?z	x[0-9]+, \.L[0-9]+
+**	cbz	x[0-9]+, \.L[0-9]+
 **	...
 */
 void f1 ()
@@ -32,9 +32,9 @@ void f1 ()
 ** f2:
 **	...
 **	cmtst	v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
-**	umaxp	v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
+**	addhn	v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
 **	fmov	x[0-9]+, d[0-9]+
-**	cbn?z	x[0-9]+, \.L[0-9]+
+**	cbz	x[0-9]+, \.L[0-9]+
 **	...
 */
 void f2 ()
@@ -50,9 +50,10 @@ void f2 ()
 /*
 ** f3:
 **	...
-**	umaxp	v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
+**	cmeq	v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
+**	addhn	v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
 **	fmov	x[0-9]+, d[0-9]+
-**	cbn?z	x[0-9]+, \.L[0-9]+
+**	cbz	x[0-9]+, \.L[0-9]+
 **	...
 */
 void f3 ()
@@ -68,9 +69,11 @@ void f3 ()
 /*
 ** f4:
 **	...
-**	umaxp	v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
+**	cmeq	v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
+**	not	v[0-9]+.16b, v[0-9]+.16b
+**	addhn	v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
 **	fmov	x[0-9]+, d[0-9]+
-**	cbn?z	x[0-9]+, \.L[0-9]+
+**	cbz	x[0-9]+, \.L[0-9]+
 **	...
 */
 void f4 ()
@@ -87,9 +90,9 @@ void f4 ()
 ** f5:
 **	...
 **	cmhs	v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
-**	umaxp	v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
+**	addhn	v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
 **	fmov	x[0-9]+, d[0-9]+
-**	cbn?z	x[0-9]+, \.L[0-9]+
+**	cbz	x[0-9]+, \.L[0-9]+
 **	...
 */
 void f5 ()
@@ -106,9 +109,9 @@ void f5 ()
 ** f6:
 **	...
 **	cmhs	v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
-**	umaxp	v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
+**	addhn	v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
 **	fmov	x[0-9]+, d[0-9]+
-**	cbn?z	x[0-9]+, \.L[0-9]+
+**	cbz	x[0-9]+, \.L[0-9]+
 **	...
 */
 void f6 ()
diff --git a/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_5.c b/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_5.c
index c28969dde1ff24872a5c3d9c236808f215cfdaa2..9679a9342f9b74170e21ec41e1959525942c0f14 100644
--- a/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_5.c
+++ b/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_5.c
@@ -10,8 +10,12 @@ unsigned int b[N] = {0};
 /*
 ** f1:
 **	...
-**	cmphi	p[0-9]+.s, p[0-9]+/z, z[0-9]+.s, #2
-**	b(\.?eq|\.none)	\.L[0-9]+
+**	movi	v[0-9]+.4s, 0x2
+**	...
+**	cmhi	v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
+**	addhn	v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
+**	fmov	x[0-9]+, d[0-9]+
+**	cbz	x[0-9]+, \.L[0-9]+
 **	...
 */
 void f1 ()
@@ -26,8 +30,12 @@ void f1 ()
 /*
 ** f2:
 **	...
-**	cmphi	p[0-9]+.s, p[0-9]+/z, z[0-9]+.s, #1
-**	b(\.?eq|\.none)	\.L[0-9]+
+**	movi	v[0-9]+.4s, 0x1
+**	...
+**	cmhi	v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
+**	addhn	v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
+**	fmov	x[0-9]+, d[0-9]+
+**	cbz	x[0-9]+, \.L[0-9]+
 **	...
 */
 void f2 ()
@@ -42,8 +50,12 @@ void f2 ()
 /*
 ** f3:
 **	...
-**	cmpeq	p[0-9]+.s, p[0-9]+/z, z[0-9]+.s, #1
-**	b(\.?eq|\.none)	\.L[0-9]+
+**	movi	v[0-9]+.4s, 0x1
+**	...
+**	cmeq	v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
+**	addhn	v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
+**	fmov	x[0-9]+, d[0-9]+
+**	cbz	x[0-9]+, \.L[0-9]+
 **	...
 */
 void f3 ()
@@ -58,8 +70,13 @@ void f3 ()
 /*
 ** f4:
 **	...
-**	cmpne	p[0-9]+.s, p[0-9]+/z, z[0-9]+.s, #1
-**	b(\.?eq|\.none)	\.L[0-9]+
+**	movi	v[0-9]+.4s, 0x1
+**	...
+**	cmeq	v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
+**	not	v[0-9]+.16b, v[0-9]+.16b
+**	addhn	v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
+**	fmov	x[0-9]+, d[0-9]+
+**	cbz	x[0-9]+, \.L[0-9]+
 **	...
 */
 void f4 ()
@@ -74,8 +91,12 @@ void f4 ()
 /*
 ** f5:
 **	...
-**	cmpls	p[0-9]+.s, p7/z, z[0-9]+.s, #1
-**	b(\.?eq|\.none)	.L[0-9]+
+**	movi	v[0-9]+.4s, 0x1
+**	...
+**	cmhs	v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
+**	addhn	v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
+**	fmov	x[0-9]+, d[0-9]+
+**	cbz	x[0-9]+, \.L[0-9]+
 **	...
 */
 void f5 ()
@@ -90,8 +111,12 @@ void f5 ()
 /*
 ** f6:
 **	...
-**	cmpls	p[0-9]+.s, p[0-9]+/z, z[0-9]+.s, #1
-**	b(\.?eq|\.none)	\.L[0-9]+
+**	movi	v[0-9]+.4s, 0x1
+**	...
+**	cmhs	v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
+**	addhn	v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
+**	fmov	x[0-9]+, d[0-9]+
+**	cbz	x[0-9]+, \.L[0-9]+
 **	...
 */
 void f6 ()
diff --git a/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_7.c b/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_7.c
new file mode 100644
index 0000000000000000000000000000000000000000..e698ee5aa537c35a1c80a0bb0fbdf53790aac220
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/vect-early-break-cbranch_7.c
@@ -0,0 +1,34 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -mcpu=neoverse-v2 -fno-schedule-insns -fno-reorder-blocks -fno-schedule-insns2 --param aarch64-autovec-preference=asimd-only -fdump-tree-vect-details" } */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } } } */
+
+#pragma GCC target "+nosve"
+
+void *a[640] = { 0 };
+
+void sink (void **);
+
+/*
+** f1:
+**	...
+**	ldp	q[0-9]+, q[0-9]+, \[[x0-9]+.*\]
+**	cmeq	v[0-9]+.2d, v[0-9]+.2d, v[0-9]+.2d
+**	cmeq	v[0-9]+.2d, v[0-9]+.2d, v[0-9]+.2d
+**	addhn	v[0-9]+.2s, v[0-9]+.2d, v[0-9]+.2d
+**	fmov	x[0-9]+, d[0-9]+
+**	cbz	x[0-9]+, \.L[0-9]+
+**	...
+*/
+void
+f1 (void *x, unsigned long n)
+{
+  void **end = a + n;
+  for (void **p = a; p != end; ++p)
+    if (*p == x)
+      {
+	sink (p);
+	return;
+      }
+}
+
+/* { dg-final { scan-tree-dump "loop vectorized using 16 byte vectors and unroll factor 4" "vect" } } */
diff --git a/gcc/testsuite/lib/target-supports.exp b/gcc/testsuite/lib/target-supports.exp
index 94a3dba222fd3d257263a0c3bd0daf83b8fe7490..73bf24a1073090686476c3cdea1b8cbf061cd2a0 100644
--- a/gcc/testsuite/lib/target-supports.exp
+++ b/gcc/testsuite/lib/target-supports.exp
@@ -6800,6 +6800,24 @@ proc check_effective_target_aarch64_sve_hw { } {
     } [add_options_for_aarch64_sve ""]]
 }
 
+# Return true if this is an AArch64 target that can run Adv.SIMD LUT code.
+
+proc check_effective_target_aarch64_lut_hw { } {
+    if { ![istarget aarch64*-*-*] } {
+	return 0
+    }
+    return [check_runtime aarch64_lut_hw_available {
+	#pragma GCC target "+lut"
+	int
+	main (void)
+	{
+	  asm volatile ("movi v0.16b, #0\n\t"
+			"luti4 v0.16b, {v0.16b}, v0[0]");
+	  return 0;
+	}
+    }]
+}
+
 # Return true if this is an AArch64 target that can run SVE2 code.
 
 proc check_effective_target_aarch64_sve2_hw { } {