This patch implements the AArch64 SVE optabs for early break/control flow
vectorization. The Splice change is needed to avoid an early-ra weirdness
in that for some reason it really doesn't like using the same register for
all operands.
The SVE implementation maps the active-predicate operations onto BRK*, CNTP,
PFIRST and splice/extract-first sequences. The count_active_ expander emits a
CNTP under ptrue. The cond_count_active_ expander emits CNTP under the
caller-supplied governing predicate and now has only the two predicate inputs
that it actually consumes.
For example, a conditional count-active operation is represented as:
cond_count_active (pg, pred)
and lowers directly to:
cntp x0, pg, pred.d
rather than carrying an unused third predicate operand in the expander
pattern.
The patch also generalizes the SVE splice/extract patterns to SVE_ALL so the
same machinery works for partial vectors, derives the 128-bit container mode
from the element mode, and updates the existing no-epilogue SVE codegen tests.
Bootstrapped Regtested on aarch64-none-linux-gnu,
arm-none-linux-gnueabihf, x86_64-pc-linux-gnu
-m32, -m64 and no issues.
-O3, profiled, profiled -O3 and -O3 -LTO bootstrap tested and no issues.
Also SPECCPU 2017 and SPECCPU 2026 tested.
OK for master?
gcc/ChangeLog:
* config/aarch64/aarch64-sve.md
(*vec_extract<mode><Vel>_0): Use SVE_ALL and derive the 128-bit
container mode from the element mode.
(@aarch64_sve_splice<mode>): Use SVE_ALL and Vctype.
(extract_first_<mode>): New expander.
(take_<brk_name>_active_<mode>): Likewise.
(cond_take_<brk_name>_active_<mode>): Likewise.
(count_active_<mode>): Likewise.
(cond_count_active_<mode>): Likewise.
* config/aarch64/aarch64-sve2.md
(index_first_active_<mode>): New expander.
(cond_index_first_active_<mode>): Likewise.
* config/aarch64/iterators.md (brk_name): New int attribute.
gcc/testsuite/ChangeLog:
* gcc.target/aarch64/sve/noeffect3.c: Update expected output.
* gcc.target/aarch64/sve/noeffect4.c: Likewise.
* gcc.target/aarch64/sve/noeffect5.c: Likewise.
* gcc.target/aarch64/sve/noeffect6.c: Likewise.
* gcc.target/aarch64/sve/noeffect7.c: Likewise.
* gcc.target/aarch64/sve2/vect-early-break-firstp_1.c: New test.
---
diff --git a/gcc/config/aarch64/aarch64-sve.md
b/gcc/config/aarch64/aarch64-sve.md
index
7a5db18586f1bdf9df1626477d81d42368f9e0a4..f5c0066ad11126797176b2092635045442d77783
100644
--- a/gcc/config/aarch64/aarch64-sve.md
+++ b/gcc/config/aarch64/aarch64-sve.md
@@ -3220,11 +3220,12 @@ (define_expand "vec_extract<mode><v64>"
(define_insn_and_split "*vec_extract<mode><Vel>_0"
[(set (match_operand:<VEL> 0 "aarch64_simd_nonimmediate_operand" "=r, w,
Utv")
(vec_select:<VEL>
- (match_operand:SVE_FULL 1 "register_operand" "w, 0, w")
+ (match_operand:SVE_ALL 1 "register_operand" "w, 0, w")
(parallel [(const_int 0)])))]
"TARGET_SVE"
{
- operands[1] = gen_rtx_REG (<V128>mode, REGNO (operands[1]));
+ machine_mode vq_mode = aarch64_v128_mode (<VEL>mode).require ();
+ operands[1] = gen_rtx_REG (vq_mode, REGNO (operands[1]));
switch (which_alternative)
{
case 0:
@@ -10449,20 +10450,40 @@ (define_insn "@aarch64_sve_rev<mode>"
;; Like EXT, but start at the first active element.
(define_insn "@aarch64_sve_splice<mode>"
- [(set (match_operand:SVE_FULL 0 "register_operand")
- (unspec:SVE_FULL
+ [(set (match_operand:SVE_ALL 0 "register_operand")
+ (unspec:SVE_ALL
[(match_operand:<VPRED> 1 "register_operand")
- (match_operand:SVE_FULL 2 "register_operand")
- (match_operand:SVE_FULL 3 "register_operand")]
+ (match_operand:SVE_ALL 2 "register_operand")
+ (match_operand:SVE_ALL 3 "register_operand")]
UNSPEC_SVE_SPLICE))]
"TARGET_SVE"
{@ [ cons: =0 , 1 , 2 , 3 ; attrs: movprfx ]
- [ w , Upl , 0 , w ; * ] splice\t%0.<Vetype>, %1,
%0.<Vetype>, %3.<Vetype>
- [ ?&w , Upl , w , w ; yes ] movprfx\t%0,
%2\;splice\t%0.<Vetype>, %1, %0.<Vetype>, %3.<Vetype>
+ [ w , Upl , 0 , w ; * ] splice\t%0.<Vctype>, %1,
%0.<Vctype>, %3.<Vctype>
+ [ w , Upl , 0 , 0 ; * ] splice\t%0.<Vctype>, %1,
%0.<Vctype>, %0.<Vctype>
+ [ ?&w , Upl , w , w ; yes ] movprfx\t%0,
%2\;splice\t%0.<Vctype>, %1, %0.<Vctype>, %3.<Vctype>
}
[(set_attr "sve_type" "sve_int_extract")]
)
+;; Vectorizer EXTRACT_FIRST implementation.
+;; For example: splice z0.d, p1, z0.d, z2.d; mov x0, z0.d[0].
+(define_expand "extract_first_<mode>"
+ [(set (match_operand:<VEL> 0 "register_operand")
+ (unspec:<VEL>
+ [(match_operand:<VPRED> 1 "register_operand")
+ (match_operand:SVE_ALL 2 "register_operand")]
+ UNSPEC_SVE_SPLICE))]
+ "TARGET_SVE"
+{
+ rtx dest = gen_reg_rtx (<MODE>mode);
+ emit_insn (gen_aarch64_sve_splice (<MODE>mode, dest, operands[1],
+ operands[2], operands[2]));
+ rtx sel = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (1, const0_rtx));
+ emit_insn (gen_rtx_SET (operands[0],
+ gen_rtx_VEC_SELECT (<VEL>mode, dest, sel)));
+ DONE;
+})
+
;; Permutes that take half the elements from one vector and half the
;; elements from the other.
(define_insn "@aarch64_sve_<perm_insn><mode>"
@@ -11703,6 +11724,45 @@ (define_insn "@aarch64_brk<brk_op>"
[(set_attr "sve_type" "sve_pred_cnt_ctrl")]
)
+;; Vectorizer implementations for take_*_active and cond_take_*_active.
+;; For example: brka p0.b, ptrue/z, p1.b.
+(define_expand "take_<brk_name>_active_<mode>"
+ [(set (match_operand:PRED_ALL 0 "register_operand")
+ (unspec:PRED_ALL
+ [(match_operand:PRED_ALL 1 "register_operand")]
+ SVE_BRK_UNARY))]
+ "TARGET_SVE"
+{
+ rtx dest = gen_reg_rtx (VNx16BImode);
+ rtx mask = aarch64_ptrue_reg (VNx16BImode);
+ rtx pred = force_lowpart_subreg (VNx16BImode, operands[1], <MODE>mode);
+ emit_insn (gen_aarch64_brk<brk_op> (dest, mask, pred,
+ CONST0_RTX (VNx16BImode)));
+ emit_move_insn (operands[0],
+ force_lowpart_subreg (<MODE>mode, dest, VNx16BImode));
+ DONE;
+})
+
+;; For example: brkb p0.b, p1/m, p2.b.
+(define_expand "cond_take_<brk_name>_active_<mode>"
+ [(set (match_operand:PRED_ALL 0 "register_operand")
+ (unspec:PRED_ALL
+ [(match_operand:PRED_ALL 1 "register_operand")
+ (match_operand:PRED_ALL 2 "register_operand")
+ (match_operand:PRED_ALL 3 "aarch64_simd_reg_or_zero")]
+ SVE_BRK_UNARY))]
+ "TARGET_SVE"
+{
+ rtx dest = gen_reg_rtx (VNx16BImode);
+ rtx tmp1 = force_lowpart_subreg (VNx16BImode, operands[1], <MODE>mode);
+ rtx tmp2 = force_lowpart_subreg (VNx16BImode, operands[2], <MODE>mode);
+ rtx tmp3 = force_lowpart_subreg (VNx16BImode, operands[3], <MODE>mode);
+ emit_insn (gen_aarch64_brk<brk_op> (dest, tmp1, tmp2, tmp3));
+ emit_move_insn (operands[0],
+ force_lowpart_subreg (<MODE>mode, dest, VNx16BImode));
+ DONE;
+})
+
;; Same, but also producing a flags result.
(define_insn "*aarch64_brk<brk_op>_cc"
[(set (reg:CC_NZC CC_REGNUM)
@@ -12401,6 +12461,39 @@ (define_insn "@aarch64_pred_cntp<mode>"
[(set_attr "sve_type" "sve_pred_cnt_scalar")]
)
+
+;; Vectorizer implementation of CNTP including special support for partial
+;; vectors which need to not double count.
+;; For example: cntp x0, ptrue, p1.d.
+(define_expand "count_active_<mode>"
+ [(set (match_operand:DI 0 "register_operand" "=r")
+ (zero_extend:DI
+ (unspec:SI [(match_operand:PRED_ALL 1 "register_operand")]
+ UNSPEC_CNTP)))]
+ "TARGET_SVE"
+{
+ rtx ptrue = aarch64_ptrue_reg (<MODE>mode);
+ rtx gp_flag = gen_int_mode (SVE_KNOWN_PTRUE, SImode);
+ emit_insn (gen_aarch64_pred_cntp (<MODE>mode, operands[0], ptrue,
+ gp_flag, operands[1]));
+ DONE;
+})
+
+;; For example: cntp x0, p1, p2.d.
+(define_expand "cond_count_active_<mode>"
+ [(set (match_operand:DI 0 "register_operand" "=r")
+ (zero_extend:DI
+ (unspec:SI [(match_operand:PRED_ALL 1 "register_operand" "Upl")
+ (match_operand:PRED_ALL 2 "register_operand" "Upa")]
+ UNSPEC_CNTP)))]
+ "TARGET_SVE"
+{
+ rtx gp_flag = gen_int_mode (SVE_MAYBE_NOT_PTRUE, SImode);
+ emit_insn (gen_aarch64_pred_cntp (<MODE>mode, operands[0], operands[1],
+ gp_flag, operands[2]));
+ DONE;
+})
+
;; -------------------------------------------------------------------------
;; ---- [INT] Increment by the number of elements in a predicate (scalar)
;; -------------------------------------------------------------------------
diff --git a/gcc/config/aarch64/aarch64-sve2.md
b/gcc/config/aarch64/aarch64-sve2.md
index
3a1285a6a008d934a79eb91ac31c5598e4604898..4c1ded3ce6c93cf94093bfcb815c038458d4d4d0
100644
--- a/gcc/config/aarch64/aarch64-sve2.md
+++ b/gcc/config/aarch64/aarch64-sve2.md
@@ -744,6 +744,32 @@ (define_insn "@aarch64_pred_firstp<mode>"
[(set_attr "sve_type" "sve_pred_cnt_scalar")]
)
+;; For example: firstp x0, ptrue, p1.d.
+(define_expand "index_first_active_<mode>"
+ [(set (match_operand:DI 0 "register_operand")
+ (unspec:DI [(match_operand:PRED_ALL 1 "register_operand")]
+ UNSPEC_FIRSTP))]
+ "TARGET_SVE2p2_OR_SME2p2"
+{
+ rtx ptrue = aarch64_ptrue_reg (<MODE>mode);
+ emit_insn (gen_aarch64_pred_firstp (<MODE>mode, operands[0], ptrue,
+ operands[1]));
+ DONE;
+})
+
+;; For example: firstp x0, p1, p2.d.
+(define_expand "cond_index_first_active_<mode>"
+ [(set (match_operand:DI 0 "register_operand")
+ (unspec:DI [(match_operand:PRED_ALL 1 "register_operand")
+ (match_operand:PRED_ALL 2 "register_operand")]
+ UNSPEC_FIRSTP))]
+ "TARGET_SVE2p2_OR_SME2p2"
+{
+ emit_insn (gen_aarch64_pred_firstp (<MODE>mode, operands[0], operands[1],
+ operands[2]));
+ DONE;
+})
+
;; Count the number of set bits in a predicate. Operand 3 is true if
;; operand 1 is known to be all-true.
(define_insn "@aarch64_pred_lastp<mode>"
diff --git a/gcc/config/aarch64/iterators.md b/gcc/config/aarch64/iterators.md
index
af417b51ccf31bf56d4e4f368979971c05e0b482..7ed204a0b3669a887b453ac2dff36550986c53ef
100644
--- a/gcc/config/aarch64/iterators.md
+++ b/gcc/config/aarch64/iterators.md
@@ -4997,6 +4997,8 @@ (define_int_attr brk_op [(UNSPEC_BRKA "a") (UNSPEC_BRKB
"b")
(UNSPEC_BRKN "n")
(UNSPEC_BRKPA "pa") (UNSPEC_BRKPB "pb")])
+(define_int_attr brk_name [(UNSPEC_BRKA "incl") (UNSPEC_BRKB "excl")])
+
(define_int_attr sve_pred_op [(UNSPEC_PFIRST "pfirst") (UNSPEC_PNEXT "pnext")])
(define_int_attr sve_int_op [(UNSPEC_ADCLB "adclb")
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/noeffect3.c
b/gcc/testsuite/gcc.target/aarch64/sve/noeffect3.c
index
c5f81bb0d303a070b482ec40d2921f6e6236ce67..031af2f4887d3602d384e74754778d032c8ccba5
100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/noeffect3.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/noeffect3.c
@@ -14,10 +14,10 @@ int b[N] = {0};
** ld1w z[0-9]+\.s, p[0-9]+/z, \[x[0-9]+, x[0-9]+, lsl 2\]
** ld1w z[0-9]+\.s, p[0-9]+/z, \[x[0-9]+, x[0-9]+, lsl 2\]
** cmpgt p[0-9]+\.s, p[0-9]+/z, z[0-9]+\.s, z[0-9]+\.s
+** b\.none \.L[0-9]+
** ...
-** ldr w[0-9]+, \[x[0-9]+, x[0-9]+, lsl 2\]
-** ldr w[0-9]+, \[x[0-9]+, x[0-9]+, lsl 2\]
-** cmp w[0-9]+, w[0-9]+
+** brkb p[0-9]+\.b, p[0-9]+/z, p[0-9]+\.b
+** incp x[0-9]+, p[0-9]+\.s
** ...
** ret
** ...
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/noeffect4.c
b/gcc/testsuite/gcc.target/aarch64/sve/noeffect4.c
index
735b54523b4d29de56cc2d894c3a69d57173e4ca..230d9d7761baef1bb91be7b92086bb0475921dac
100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/noeffect4.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/noeffect4.c
@@ -12,11 +12,10 @@ short b[N] = {0};
** ld1h z[0-9]+\.s, p[0-9]+/z, \[x[0-9]+, x[0-9]+, lsl 1\]
** ld1h z[0-9]+\.s, p[0-9]+/z, \[x[0-9]+, x[0-9]+, lsl 1\]
** cmpgt p[0-9]+\.h, p[0-9]+/z, z[0-9]+\.h, z[0-9]+\.h
-** ...
-** ldrsh w[0-9]+, \[x[0-9]+, x[0-9]+\]
-** ldrsh w[0-9]+, \[x[0-9]+, x[0-9]+\]
-** ...
-** cmp w[0-9]+, w[0-9]+
+** ptest p[0-9]+, p[0-9]+\.b
+** b\.none \.L[0-9]+
+** splice z[0-9]+\.s, p[0-9]+, z[0-9]+\.s, z[0-9]+\.s
+** umov w0, v[0-9]+\.h\[0\]
** ...
** ret
** ...
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/noeffect5.c
b/gcc/testsuite/gcc.target/aarch64/sve/noeffect5.c
index
bfaec5805785217ae3d2672c3c6669e59d8a5c1a..8397c92760d64a43a930d3d258602f36b91b128b
100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/noeffect5.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/noeffect5.c
@@ -12,11 +12,10 @@ short b[N] = {0};
** ld1h z[0-9]+\.h, p[0-9]+/z, \[x[0-9]+, x[0-9]+, lsl 1\]
** ld1h z[0-9]+\.h, p[0-9]+/z, \[x[0-9]+, x[0-9]+, lsl 1\]
** cmpgt p[0-9]+\.h, p[0-9]+/z, z[0-9]+\.h, z[0-9]+\.h
+** b\.none \.L[0-9]+
** ...
-** ldrsh w[0-9]+, \[x[0-9]+, x[0-9]+\]
-** ldrsh w[0-9]+, \[x[0-9]+, x[0-9]+\]
-** ...
-** cmp w[0-9]+, w[0-9]+
+** brkb p[0-9]+\.b, p[0-9]+/z, p[0-9]+\.b
+** incp x[0-9]+, p[0-9]+\.h
** ...
** ret
** ...
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/noeffect6.c
b/gcc/testsuite/gcc.target/aarch64/sve/noeffect6.c
index
14438830453aad97b325ac1e5aad849a095343d9..6675f97c41986659a8b03a6c3015b3c31a2046a9
100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/noeffect6.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/noeffect6.c
@@ -1,26 +1,14 @@
/* { dg-do compile } */
/* { dg-options "-O3 -march=armv8-a+sve
--param=aarch64-autovec-preference=sve-only -msve-vector-bits=scalable" } */
-/* { dg-final { check-function-bodies "**" "" "" } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.h, p[0-9]+/z,
\[x[0-9]+, x[0-9]+(, lsl 1)?\]} 2 } } */
+/* { dg-final { scan-assembler {\tcmpgt\tp[0-9]+\.h, p[0-9]+/z, z[0-9]+\.h,
z[0-9]+\.h} } } */
+/* { dg-final { scan-assembler {\tb\.none\t\.L[0-9]+} } } */
+/* { dg-final { scan-assembler {\tsplice\tz[0-9]+\.h, p[0-9]+, z[0-9]+\.h,
z[0-9]+\.h} } } */
#define N 1000
short a[N] = {0};
short b[N] = {0};
-/*
-** foo:
-** ...
-** ld1h z[0-9]+\.h, p[0-9]+/z, \[x[0-9]+, x[0-9]+, lsl 1\]
-** ld1h z[0-9]+\.h, p[0-9]+/z, \[x[0-9]+, x[0-9]+, lsl 1\]
-** cmpgt p[0-9]+\.h, p[0-9]+/z, z[0-9]+\.h, z[0-9]+\.h
-** ...
-** ldrsh w[0-9]+, \[x[0-9]+, x[0-9]+\]
-** ldrsh w[0-9]+, \[x[0-9]+, x[0-9]+\]
-** ...
-** cmp w[0-9]+, w[0-9]+
-** ...
-** ret
-** ...
-*/
__attribute__ ((noipa, noinline))
short foo (void)
{
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/noeffect7.c
b/gcc/testsuite/gcc.target/aarch64/sve/noeffect7.c
index
9dab90e72b25b552daad81fdc27dc69dc329cbb0..22f686f70faf418230b0671634770ab8c0356d43
100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/noeffect7.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/noeffect7.c
@@ -12,7 +12,7 @@ int b[N] = {0};
** ld2w \{z[0-9]+\.s - z[0-9]+\.s\}, p[0-9]+/z, \[x[0-9]+, x[0-9]+, lsl
2\]
** ...
** cmpgt p[0-9]+\.s, p[0-9]+/z, z[0-9]+\.s, z[0-9]+\.s
-** ptest p[0-9]+, p[0-9]+\.b
+** b\.any \.L[0-9]+
** ...
** ldr w[0-9]+, \[x[0-9]+, x[0-9]+, lsl 2\]
** ldr w[0-9]+, \[x[0-9]+, x[0-9]+, lsl 2\]
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/vect-early-break-firstp_1.c
b/gcc/testsuite/gcc.target/aarch64/sve2/vect-early-break-firstp_1.c
new file mode 100644
index
0000000000000000000000000000000000000000..a67352ff1a94b7c3ac40cb53bdd9f34b2805c086
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/vect-early-break-firstp_1.c
@@ -0,0 +1,30 @@
+/* { dg-do compile } */
+/* { dg-options "-Ofast -march=armv8.3-a+sve2p2 -mautovec-preference=sve-only"
} */
+
+enum codetype { DISTS };
+
+int max_count;
+short count[5];
+
+int __attribute__ ((noipa))
+f (enum codetype type, short lens, unsigned codes, int table, unsigned *bits,
+ short work)
+{
+ unsigned int min;
+
+ if (max_count == 0)
+ {
+ *bits = 1;
+ return 0;
+ }
+
+ min = 1;
+ for (; min < max_count; min++)
+ if (count[min])
+ break;
+
+ *bits = min;
+ return 0;
+}
+
+/* { dg-final { scan-assembler {\tfirstp\tx[0-9]+, p[0-9]+, p[0-9]+\.s} } } */
--
diff --git a/gcc/config/aarch64/aarch64-sve.md b/gcc/config/aarch64/aarch64-sve.md
index 7a5db18586f1bdf9df1626477d81d42368f9e0a4..f5c0066ad11126797176b2092635045442d77783 100644
--- a/gcc/config/aarch64/aarch64-sve.md
+++ b/gcc/config/aarch64/aarch64-sve.md
@@ -3220,11 +3220,12 @@ (define_expand "vec_extract<mode><v64>"
(define_insn_and_split "*vec_extract<mode><Vel>_0"
[(set (match_operand:<VEL> 0 "aarch64_simd_nonimmediate_operand" "=r, w, Utv")
(vec_select:<VEL>
- (match_operand:SVE_FULL 1 "register_operand" "w, 0, w")
+ (match_operand:SVE_ALL 1 "register_operand" "w, 0, w")
(parallel [(const_int 0)])))]
"TARGET_SVE"
{
- operands[1] = gen_rtx_REG (<V128>mode, REGNO (operands[1]));
+ machine_mode vq_mode = aarch64_v128_mode (<VEL>mode).require ();
+ operands[1] = gen_rtx_REG (vq_mode, REGNO (operands[1]));
switch (which_alternative)
{
case 0:
@@ -10449,20 +10450,40 @@ (define_insn "@aarch64_sve_rev<mode>"
;; Like EXT, but start at the first active element.
(define_insn "@aarch64_sve_splice<mode>"
- [(set (match_operand:SVE_FULL 0 "register_operand")
- (unspec:SVE_FULL
+ [(set (match_operand:SVE_ALL 0 "register_operand")
+ (unspec:SVE_ALL
[(match_operand:<VPRED> 1 "register_operand")
- (match_operand:SVE_FULL 2 "register_operand")
- (match_operand:SVE_FULL 3 "register_operand")]
+ (match_operand:SVE_ALL 2 "register_operand")
+ (match_operand:SVE_ALL 3 "register_operand")]
UNSPEC_SVE_SPLICE))]
"TARGET_SVE"
{@ [ cons: =0 , 1 , 2 , 3 ; attrs: movprfx ]
- [ w , Upl , 0 , w ; * ] splice\t%0.<Vetype>, %1, %0.<Vetype>, %3.<Vetype>
- [ ?&w , Upl , w , w ; yes ] movprfx\t%0, %2\;splice\t%0.<Vetype>, %1, %0.<Vetype>, %3.<Vetype>
+ [ w , Upl , 0 , w ; * ] splice\t%0.<Vctype>, %1, %0.<Vctype>, %3.<Vctype>
+ [ w , Upl , 0 , 0 ; * ] splice\t%0.<Vctype>, %1, %0.<Vctype>, %0.<Vctype>
+ [ ?&w , Upl , w , w ; yes ] movprfx\t%0, %2\;splice\t%0.<Vctype>, %1, %0.<Vctype>, %3.<Vctype>
}
[(set_attr "sve_type" "sve_int_extract")]
)
+;; Vectorizer EXTRACT_FIRST implementation.
+;; For example: splice z0.d, p1, z0.d, z2.d; mov x0, z0.d[0].
+(define_expand "extract_first_<mode>"
+ [(set (match_operand:<VEL> 0 "register_operand")
+ (unspec:<VEL>
+ [(match_operand:<VPRED> 1 "register_operand")
+ (match_operand:SVE_ALL 2 "register_operand")]
+ UNSPEC_SVE_SPLICE))]
+ "TARGET_SVE"
+{
+ rtx dest = gen_reg_rtx (<MODE>mode);
+ emit_insn (gen_aarch64_sve_splice (<MODE>mode, dest, operands[1],
+ operands[2], operands[2]));
+ rtx sel = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (1, const0_rtx));
+ emit_insn (gen_rtx_SET (operands[0],
+ gen_rtx_VEC_SELECT (<VEL>mode, dest, sel)));
+ DONE;
+})
+
;; Permutes that take half the elements from one vector and half the
;; elements from the other.
(define_insn "@aarch64_sve_<perm_insn><mode>"
@@ -11703,6 +11724,45 @@ (define_insn "@aarch64_brk<brk_op>"
[(set_attr "sve_type" "sve_pred_cnt_ctrl")]
)
+;; Vectorizer implementations for take_*_active and cond_take_*_active.
+;; For example: brka p0.b, ptrue/z, p1.b.
+(define_expand "take_<brk_name>_active_<mode>"
+ [(set (match_operand:PRED_ALL 0 "register_operand")
+ (unspec:PRED_ALL
+ [(match_operand:PRED_ALL 1 "register_operand")]
+ SVE_BRK_UNARY))]
+ "TARGET_SVE"
+{
+ rtx dest = gen_reg_rtx (VNx16BImode);
+ rtx mask = aarch64_ptrue_reg (VNx16BImode);
+ rtx pred = force_lowpart_subreg (VNx16BImode, operands[1], <MODE>mode);
+ emit_insn (gen_aarch64_brk<brk_op> (dest, mask, pred,
+ CONST0_RTX (VNx16BImode)));
+ emit_move_insn (operands[0],
+ force_lowpart_subreg (<MODE>mode, dest, VNx16BImode));
+ DONE;
+})
+
+;; For example: brkb p0.b, p1/m, p2.b.
+(define_expand "cond_take_<brk_name>_active_<mode>"
+ [(set (match_operand:PRED_ALL 0 "register_operand")
+ (unspec:PRED_ALL
+ [(match_operand:PRED_ALL 1 "register_operand")
+ (match_operand:PRED_ALL 2 "register_operand")
+ (match_operand:PRED_ALL 3 "aarch64_simd_reg_or_zero")]
+ SVE_BRK_UNARY))]
+ "TARGET_SVE"
+{
+ rtx dest = gen_reg_rtx (VNx16BImode);
+ rtx tmp1 = force_lowpart_subreg (VNx16BImode, operands[1], <MODE>mode);
+ rtx tmp2 = force_lowpart_subreg (VNx16BImode, operands[2], <MODE>mode);
+ rtx tmp3 = force_lowpart_subreg (VNx16BImode, operands[3], <MODE>mode);
+ emit_insn (gen_aarch64_brk<brk_op> (dest, tmp1, tmp2, tmp3));
+ emit_move_insn (operands[0],
+ force_lowpart_subreg (<MODE>mode, dest, VNx16BImode));
+ DONE;
+})
+
;; Same, but also producing a flags result.
(define_insn "*aarch64_brk<brk_op>_cc"
[(set (reg:CC_NZC CC_REGNUM)
@@ -12401,6 +12461,39 @@ (define_insn "@aarch64_pred_cntp<mode>"
[(set_attr "sve_type" "sve_pred_cnt_scalar")]
)
+
+;; Vectorizer implementation of CNTP including special support for partial
+;; vectors which need to not double count.
+;; For example: cntp x0, ptrue, p1.d.
+(define_expand "count_active_<mode>"
+ [(set (match_operand:DI 0 "register_operand" "=r")
+ (zero_extend:DI
+ (unspec:SI [(match_operand:PRED_ALL 1 "register_operand")]
+ UNSPEC_CNTP)))]
+ "TARGET_SVE"
+{
+ rtx ptrue = aarch64_ptrue_reg (<MODE>mode);
+ rtx gp_flag = gen_int_mode (SVE_KNOWN_PTRUE, SImode);
+ emit_insn (gen_aarch64_pred_cntp (<MODE>mode, operands[0], ptrue,
+ gp_flag, operands[1]));
+ DONE;
+})
+
+;; For example: cntp x0, p1, p2.d.
+(define_expand "cond_count_active_<mode>"
+ [(set (match_operand:DI 0 "register_operand" "=r")
+ (zero_extend:DI
+ (unspec:SI [(match_operand:PRED_ALL 1 "register_operand" "Upl")
+ (match_operand:PRED_ALL 2 "register_operand" "Upa")]
+ UNSPEC_CNTP)))]
+ "TARGET_SVE"
+{
+ rtx gp_flag = gen_int_mode (SVE_MAYBE_NOT_PTRUE, SImode);
+ emit_insn (gen_aarch64_pred_cntp (<MODE>mode, operands[0], operands[1],
+ gp_flag, operands[2]));
+ DONE;
+})
+
;; -------------------------------------------------------------------------
;; ---- [INT] Increment by the number of elements in a predicate (scalar)
;; -------------------------------------------------------------------------
diff --git a/gcc/config/aarch64/aarch64-sve2.md b/gcc/config/aarch64/aarch64-sve2.md
index 3a1285a6a008d934a79eb91ac31c5598e4604898..4c1ded3ce6c93cf94093bfcb815c038458d4d4d0 100644
--- a/gcc/config/aarch64/aarch64-sve2.md
+++ b/gcc/config/aarch64/aarch64-sve2.md
@@ -744,6 +744,32 @@ (define_insn "@aarch64_pred_firstp<mode>"
[(set_attr "sve_type" "sve_pred_cnt_scalar")]
)
+;; For example: firstp x0, ptrue, p1.d.
+(define_expand "index_first_active_<mode>"
+ [(set (match_operand:DI 0 "register_operand")
+ (unspec:DI [(match_operand:PRED_ALL 1 "register_operand")]
+ UNSPEC_FIRSTP))]
+ "TARGET_SVE2p2_OR_SME2p2"
+{
+ rtx ptrue = aarch64_ptrue_reg (<MODE>mode);
+ emit_insn (gen_aarch64_pred_firstp (<MODE>mode, operands[0], ptrue,
+ operands[1]));
+ DONE;
+})
+
+;; For example: firstp x0, p1, p2.d.
+(define_expand "cond_index_first_active_<mode>"
+ [(set (match_operand:DI 0 "register_operand")
+ (unspec:DI [(match_operand:PRED_ALL 1 "register_operand")
+ (match_operand:PRED_ALL 2 "register_operand")]
+ UNSPEC_FIRSTP))]
+ "TARGET_SVE2p2_OR_SME2p2"
+{
+ emit_insn (gen_aarch64_pred_firstp (<MODE>mode, operands[0], operands[1],
+ operands[2]));
+ DONE;
+})
+
;; Count the number of set bits in a predicate. Operand 3 is true if
;; operand 1 is known to be all-true.
(define_insn "@aarch64_pred_lastp<mode>"
diff --git a/gcc/config/aarch64/iterators.md b/gcc/config/aarch64/iterators.md
index af417b51ccf31bf56d4e4f368979971c05e0b482..7ed204a0b3669a887b453ac2dff36550986c53ef 100644
--- a/gcc/config/aarch64/iterators.md
+++ b/gcc/config/aarch64/iterators.md
@@ -4997,6 +4997,8 @@ (define_int_attr brk_op [(UNSPEC_BRKA "a") (UNSPEC_BRKB "b")
(UNSPEC_BRKN "n")
(UNSPEC_BRKPA "pa") (UNSPEC_BRKPB "pb")])
+(define_int_attr brk_name [(UNSPEC_BRKA "incl") (UNSPEC_BRKB "excl")])
+
(define_int_attr sve_pred_op [(UNSPEC_PFIRST "pfirst") (UNSPEC_PNEXT "pnext")])
(define_int_attr sve_int_op [(UNSPEC_ADCLB "adclb")
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/noeffect3.c b/gcc/testsuite/gcc.target/aarch64/sve/noeffect3.c
index c5f81bb0d303a070b482ec40d2921f6e6236ce67..031af2f4887d3602d384e74754778d032c8ccba5 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/noeffect3.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/noeffect3.c
@@ -14,10 +14,10 @@ int b[N] = {0};
** ld1w z[0-9]+\.s, p[0-9]+/z, \[x[0-9]+, x[0-9]+, lsl 2\]
** ld1w z[0-9]+\.s, p[0-9]+/z, \[x[0-9]+, x[0-9]+, lsl 2\]
** cmpgt p[0-9]+\.s, p[0-9]+/z, z[0-9]+\.s, z[0-9]+\.s
+** b\.none \.L[0-9]+
** ...
-** ldr w[0-9]+, \[x[0-9]+, x[0-9]+, lsl 2\]
-** ldr w[0-9]+, \[x[0-9]+, x[0-9]+, lsl 2\]
-** cmp w[0-9]+, w[0-9]+
+** brkb p[0-9]+\.b, p[0-9]+/z, p[0-9]+\.b
+** incp x[0-9]+, p[0-9]+\.s
** ...
** ret
** ...
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/noeffect4.c b/gcc/testsuite/gcc.target/aarch64/sve/noeffect4.c
index 735b54523b4d29de56cc2d894c3a69d57173e4ca..230d9d7761baef1bb91be7b92086bb0475921dac 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/noeffect4.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/noeffect4.c
@@ -12,11 +12,10 @@ short b[N] = {0};
** ld1h z[0-9]+\.s, p[0-9]+/z, \[x[0-9]+, x[0-9]+, lsl 1\]
** ld1h z[0-9]+\.s, p[0-9]+/z, \[x[0-9]+, x[0-9]+, lsl 1\]
** cmpgt p[0-9]+\.h, p[0-9]+/z, z[0-9]+\.h, z[0-9]+\.h
-** ...
-** ldrsh w[0-9]+, \[x[0-9]+, x[0-9]+\]
-** ldrsh w[0-9]+, \[x[0-9]+, x[0-9]+\]
-** ...
-** cmp w[0-9]+, w[0-9]+
+** ptest p[0-9]+, p[0-9]+\.b
+** b\.none \.L[0-9]+
+** splice z[0-9]+\.s, p[0-9]+, z[0-9]+\.s, z[0-9]+\.s
+** umov w0, v[0-9]+\.h\[0\]
** ...
** ret
** ...
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/noeffect5.c b/gcc/testsuite/gcc.target/aarch64/sve/noeffect5.c
index bfaec5805785217ae3d2672c3c6669e59d8a5c1a..8397c92760d64a43a930d3d258602f36b91b128b 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/noeffect5.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/noeffect5.c
@@ -12,11 +12,10 @@ short b[N] = {0};
** ld1h z[0-9]+\.h, p[0-9]+/z, \[x[0-9]+, x[0-9]+, lsl 1\]
** ld1h z[0-9]+\.h, p[0-9]+/z, \[x[0-9]+, x[0-9]+, lsl 1\]
** cmpgt p[0-9]+\.h, p[0-9]+/z, z[0-9]+\.h, z[0-9]+\.h
+** b\.none \.L[0-9]+
** ...
-** ldrsh w[0-9]+, \[x[0-9]+, x[0-9]+\]
-** ldrsh w[0-9]+, \[x[0-9]+, x[0-9]+\]
-** ...
-** cmp w[0-9]+, w[0-9]+
+** brkb p[0-9]+\.b, p[0-9]+/z, p[0-9]+\.b
+** incp x[0-9]+, p[0-9]+\.h
** ...
** ret
** ...
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/noeffect6.c b/gcc/testsuite/gcc.target/aarch64/sve/noeffect6.c
index 14438830453aad97b325ac1e5aad849a095343d9..6675f97c41986659a8b03a6c3015b3c31a2046a9 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/noeffect6.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/noeffect6.c
@@ -1,26 +1,14 @@
/* { dg-do compile } */
/* { dg-options "-O3 -march=armv8-a+sve --param=aarch64-autovec-preference=sve-only -msve-vector-bits=scalable" } */
-/* { dg-final { check-function-bodies "**" "" "" } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.h, p[0-9]+/z, \[x[0-9]+, x[0-9]+(, lsl 1)?\]} 2 } } */
+/* { dg-final { scan-assembler {\tcmpgt\tp[0-9]+\.h, p[0-9]+/z, z[0-9]+\.h, z[0-9]+\.h} } } */
+/* { dg-final { scan-assembler {\tb\.none\t\.L[0-9]+} } } */
+/* { dg-final { scan-assembler {\tsplice\tz[0-9]+\.h, p[0-9]+, z[0-9]+\.h, z[0-9]+\.h} } } */
#define N 1000
short a[N] = {0};
short b[N] = {0};
-/*
-** foo:
-** ...
-** ld1h z[0-9]+\.h, p[0-9]+/z, \[x[0-9]+, x[0-9]+, lsl 1\]
-** ld1h z[0-9]+\.h, p[0-9]+/z, \[x[0-9]+, x[0-9]+, lsl 1\]
-** cmpgt p[0-9]+\.h, p[0-9]+/z, z[0-9]+\.h, z[0-9]+\.h
-** ...
-** ldrsh w[0-9]+, \[x[0-9]+, x[0-9]+\]
-** ldrsh w[0-9]+, \[x[0-9]+, x[0-9]+\]
-** ...
-** cmp w[0-9]+, w[0-9]+
-** ...
-** ret
-** ...
-*/
__attribute__ ((noipa, noinline))
short foo (void)
{
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/noeffect7.c b/gcc/testsuite/gcc.target/aarch64/sve/noeffect7.c
index 9dab90e72b25b552daad81fdc27dc69dc329cbb0..22f686f70faf418230b0671634770ab8c0356d43 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/noeffect7.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/noeffect7.c
@@ -12,7 +12,7 @@ int b[N] = {0};
** ld2w \{z[0-9]+\.s - z[0-9]+\.s\}, p[0-9]+/z, \[x[0-9]+, x[0-9]+, lsl 2\]
** ...
** cmpgt p[0-9]+\.s, p[0-9]+/z, z[0-9]+\.s, z[0-9]+\.s
-** ptest p[0-9]+, p[0-9]+\.b
+** b\.any \.L[0-9]+
** ...
** ldr w[0-9]+, \[x[0-9]+, x[0-9]+, lsl 2\]
** ldr w[0-9]+, \[x[0-9]+, x[0-9]+, lsl 2\]
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/vect-early-break-firstp_1.c b/gcc/testsuite/gcc.target/aarch64/sve2/vect-early-break-firstp_1.c
new file mode 100644
index 0000000000000000000000000000000000000000..a67352ff1a94b7c3ac40cb53bdd9f34b2805c086
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/vect-early-break-firstp_1.c
@@ -0,0 +1,30 @@
+/* { dg-do compile } */
+/* { dg-options "-Ofast -march=armv8.3-a+sve2p2 -mautovec-preference=sve-only" } */
+
+enum codetype { DISTS };
+
+int max_count;
+short count[5];
+
+int __attribute__ ((noipa))
+f (enum codetype type, short lens, unsigned codes, int table, unsigned *bits,
+ short work)
+{
+ unsigned int min;
+
+ if (max_count == 0)
+ {
+ *bits = 1;
+ return 0;
+ }
+
+ min = 1;
+ for (; min < max_count; min++)
+ if (count[min])
+ break;
+
+ *bits = min;
+ return 0;
+}
+
+/* { dg-final { scan-assembler {\tfirstp\tx[0-9]+, p[0-9]+, p[0-9]+\.s} } } */
