This patch implements the SVE target expanders for conditional vector
comparisons.
The integer and floating-point cond_vec_cmp expanders take the governing
predicate supplied by the middle end and fold it into the compare during
expand. This makes the RTL for IFN_COND_VEC_CMP match the shape produced by
vec_cbranch_* for the same predicate-producing compare.
That is useful because CSE can then see equivalent comparisons directly and
reuse them. It also means the backend relies less on later, more complicated
combine patterns to rediscover that a separate predicate operation can be
folded into the compare.
For example, this source:
void f (int *a, int *b, int *c, int *d)
{
for (int i = 0; i < 1024; ++i)
if (c[i] != 0 && a[i] < b[i])
d[i] = a[i];
}
uses one predicate-producing compare as the governing predicate for the next
compare:
cmpne p0.s, p7/z, z0.s, #0
cmplt p0.s, p0/z, z1.s, z2.s
The important part is the expand-time RTL shape: the second compare consumes
the governing predicate directly, matching the form used by SVE vector
compare-and-branch expansion.
The tests for the earlier gimple isel lowering live here because they require
these SVE cond_vec_cmp expanders in order to pass.
Bootstrapped Regtested on aarch64-none-linux-gnu,
arm-none-linux-gnueabihf, x86_64-pc-linux-gnu
-m32, -m64 and no issues.
-O3, profiled, profiled -O3 and -O3 -LTO bootstrap tested and no issues.
Also SPECCPU 2017 and SPECCPU 2026 tested.
OK for master?
gcc/ChangeLog:
* config/aarch64/aarch64-sve.md
(cond_vec_cmp<mode><vpred>): New SVE integer and floating-point
expanders.
(@aarch64_pred_cmp<cmp_op><mode>): Fix comment typo.
gcc/testsuite/ChangeLog:
* gcc.dg/vect/vect-cond-cmp-1.c: New test.
* gcc.target/aarch64/sve/cond_vec_cmp_1.c: Likewise.
* gcc.target/aarch64/sve/cond_vec_cmp_2.c: Likewise.
* gcc.target/aarch64/sve/cond_vec_cmp_2_run.c: Likewise.
---
diff --git a/gcc/config/aarch64/aarch64-sve.md
b/gcc/config/aarch64/aarch64-sve.md
index
f5c0066ad11126797176b2092635045442d77783..f7c4814e41cc996a66d045df70e4f4e8405e7228
100644
--- a/gcc/config/aarch64/aarch64-sve.md
+++ b/gcc/config/aarch64/aarch64-sve.md
@@ -8748,13 +8748,53 @@ (define_expand "vec_cmpu<mode><vpred>"
}
)
+;; Predicated integer comparisons where the predicate value is provided by the
+;; middle-end.
+;; For example: cmhi p0.d, p1/z, z2.d, z3.d.
+(define_expand "cond_vec_cmp<mode><vpred>"
+ [(set (match_operand:<VPRED> 0 "register_operand")
+ (unspec:<VPRED>
+ [(match_operand:<VPRED> 2 "register_operand")
+ (match_dup 5)
+ (match_operator:<VPRED> 1 "comparison_operator"
+ [(match_operand:SVE_I 3 "register_operand")
+ (match_operand:SVE_I 4 "nonmemory_operand")])]
+ UNSPEC_PRED_Z))
+ (clobber (reg:CC_NZC CC_REGNUM))]
+ "TARGET_SVE"
+ {
+ aarch64_expand_sve_vec_cmp_int (operands[0], GET_CODE (operands[1]),
+ operands[3], operands[4], operands[2]);
+ DONE;
+ }
+)
+
+;; Predicated floating-point comparisons where the predicate value is provided
+;; by the middle-end.
+;; For example: fcmgt p0.d, p1/z, z2.d, z3.d.
+(define_expand "cond_vec_cmp<mode><vpred>"
+ [(set (match_operand:<VPRED> 0 "register_operand")
+ (unspec:<VPRED>
+ [(match_operand:<VPRED> 2 "register_operand")
+ (match_operator:<VPRED> 1 "aarch64_comparison_operator"
+ [(match_operand:SVE_F 3 "register_operand")
+ (match_operand:SVE_F 4 "nonmemory_operand")])]
+ UNSPEC_PRED_Z))]
+ "TARGET_SVE"
+ {
+ aarch64_expand_sve_vec_cmp_float (operands[0], GET_CODE (operands[1]),
+ operands[3], operands[4], operands[2]);
+ DONE;
+ }
+)
+
;; Predicated integer comparisons.
;;
;; For unpacked vectors, only the lowpart element in each input container
;; has a defined value, and only the predicate bits associated with
;; those elements are defined. For example, when comparing two VNx2SIs:
;;
-;; - The VNx2SIs can be seem as VNx2DIs in which the low halves of each
+;; - The VNx2SIs can be seen as VNx2DIs in which the low halves of each
;; DI container store an SI element. The upper bits of each DI container
;; are undefined.
;;
diff --git a/gcc/testsuite/gcc.dg/vect/vect-cond-cmp-1.c
b/gcc/testsuite/gcc.dg/vect/vect-cond-cmp-1.c
new file mode 100644
index
0000000000000000000000000000000000000000..6677673d06703a2506b23adac90b66e5ef3057a0
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-cond-cmp-1.c
@@ -0,0 +1,17 @@
+/* { dg-do compile } */
+/* { dg-require-effective-target vect_int } */
+/* { dg-add-options aarch64_sve } */
+/* { dg-additional-options "-fdump-tree-isel" } */
+
+#define N 1024
+
+void __attribute__ ((noipa))
+f (int *__restrict a, int *__restrict b, int *__restrict c,
+ int *__restrict d)
+{
+ for (int i = 0; i < N; ++i)
+ if (c[i] != 0 && a[i] < b[i])
+ d[i] = a[i];
+}
+
+/* { dg-final { scan-tree-dump-times "\\.COND_VEC_CMP" 2 "isel" { target
aarch64*-*-* } } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/cond_vec_cmp_1.c
b/gcc/testsuite/gcc.target/aarch64/sve/cond_vec_cmp_1.c
new file mode 100644
index
0000000000000000000000000000000000000000..9f2355e36713c78b212413de442d56ebd7c62741
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/cond_vec_cmp_1.c
@@ -0,0 +1,15 @@
+/* { dg-do compile { target aarch64_asm_sve_ok } } */
+/* { dg-options "-O2 -ftree-vectorize -march=armv8-a+sve -fdump-tree-isel" } */
+
+#define N 1024
+
+void __attribute__ ((noipa))
+f (int *__restrict a, int *__restrict b, int *__restrict c,
+ int *__restrict d)
+{
+ for (int i = 0; i < N; ++i)
+ if (c[i] != 0 && a[i] < b[i])
+ d[i] = a[i];
+}
+
+/* { dg-final { scan-tree-dump-times "\\.COND_VEC_CMP" 2 "isel" } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/cond_vec_cmp_2.c
b/gcc/testsuite/gcc.target/aarch64/sve/cond_vec_cmp_2.c
new file mode 100644
index
0000000000000000000000000000000000000000..d67e8ffaa1c2961858ee25fea4b80155715ad009
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/cond_vec_cmp_2.c
@@ -0,0 +1,26 @@
+/* { dg-do compile { target aarch64_asm_sve_ok } } */
+/* { dg-options "-O2 -ftree-vectorize -march=armv8-a+sve
-msve-vector-bits=scalable --param=aarch64-autovec-preference=sve-only" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+
+#define N 1024
+
+/*
+** f:
+** ...
+** cmpne p[0-9]+\.s, p[0-9]+/z, z[0-9]+\.s, #0
+** ...
+** cmplt p[0-9]+\.s, p[0-9]+/z, z[0-9]+\.s, z[0-9]+\.s
+** ...
+** st1w z[0-9]+\.s, p[0-9]+, \[x[0-9]+, x[0-9]+, lsl 2\]
+** ...
+** whilelo p[0-9]+\.s, w[0-9]+, w[0-9]+
+** ...
+*/
+void __attribute__ ((noipa))
+f (int *__restrict a, int *__restrict b, int *__restrict c,
+ int *__restrict d)
+{
+ for (int i = 0; i < N; ++i)
+ if (c[i] != 0 && a[i] < b[i])
+ d[i] = a[i];
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/cond_vec_cmp_2_run.c
b/gcc/testsuite/gcc.target/aarch64/sve/cond_vec_cmp_2_run.c
new file mode 100644
index
0000000000000000000000000000000000000000..a23a11ef7210f1412ce477bd3df32e93263e6393
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/cond_vec_cmp_2_run.c
@@ -0,0 +1,38 @@
+/* { dg-do run { target aarch64_sve_hw } } */
+/* { dg-options "-O2 -ftree-vectorize -march=armv8-a+sve
-msve-vector-bits=scalable --param=aarch64-autovec-preference=sve-only" } */
+
+#define N 1024
+
+void __attribute__ ((noipa))
+f (int *__restrict a, int *__restrict b, int *__restrict c,
+ int *__restrict d)
+{
+ for (int i = 0; i < N; ++i)
+ if (c[i] != 0 && a[i] < b[i])
+ d[i] = a[i];
+}
+
+int
+main (void)
+{
+ int a[N], b[N], c[N], d[N];
+
+ for (int i = 0; i < N; ++i)
+ {
+ a[i] = i % 31;
+ b[i] = 15;
+ c[i] = (i & 3) != 0;
+ d[i] = -1;
+ }
+
+ f (a, b, c, d);
+
+ for (int i = 0; i < N; ++i)
+ {
+ int expected = (c[i] != 0 && a[i] < b[i]) ? a[i] : -1;
+ if (d[i] != expected)
+ __builtin_abort ();
+ }
+
+ return 0;
+}
--
diff --git a/gcc/config/aarch64/aarch64-sve.md b/gcc/config/aarch64/aarch64-sve.md
index f5c0066ad11126797176b2092635045442d77783..f7c4814e41cc996a66d045df70e4f4e8405e7228 100644
--- a/gcc/config/aarch64/aarch64-sve.md
+++ b/gcc/config/aarch64/aarch64-sve.md
@@ -8748,13 +8748,53 @@ (define_expand "vec_cmpu<mode><vpred>"
}
)
+;; Predicated integer comparisons where the predicate value is provided by the
+;; middle-end.
+;; For example: cmhi p0.d, p1/z, z2.d, z3.d.
+(define_expand "cond_vec_cmp<mode><vpred>"
+ [(set (match_operand:<VPRED> 0 "register_operand")
+ (unspec:<VPRED>
+ [(match_operand:<VPRED> 2 "register_operand")
+ (match_dup 5)
+ (match_operator:<VPRED> 1 "comparison_operator"
+ [(match_operand:SVE_I 3 "register_operand")
+ (match_operand:SVE_I 4 "nonmemory_operand")])]
+ UNSPEC_PRED_Z))
+ (clobber (reg:CC_NZC CC_REGNUM))]
+ "TARGET_SVE"
+ {
+ aarch64_expand_sve_vec_cmp_int (operands[0], GET_CODE (operands[1]),
+ operands[3], operands[4], operands[2]);
+ DONE;
+ }
+)
+
+;; Predicated floating-point comparisons where the predicate value is provided
+;; by the middle-end.
+;; For example: fcmgt p0.d, p1/z, z2.d, z3.d.
+(define_expand "cond_vec_cmp<mode><vpred>"
+ [(set (match_operand:<VPRED> 0 "register_operand")
+ (unspec:<VPRED>
+ [(match_operand:<VPRED> 2 "register_operand")
+ (match_operator:<VPRED> 1 "aarch64_comparison_operator"
+ [(match_operand:SVE_F 3 "register_operand")
+ (match_operand:SVE_F 4 "nonmemory_operand")])]
+ UNSPEC_PRED_Z))]
+ "TARGET_SVE"
+ {
+ aarch64_expand_sve_vec_cmp_float (operands[0], GET_CODE (operands[1]),
+ operands[3], operands[4], operands[2]);
+ DONE;
+ }
+)
+
;; Predicated integer comparisons.
;;
;; For unpacked vectors, only the lowpart element in each input container
;; has a defined value, and only the predicate bits associated with
;; those elements are defined. For example, when comparing two VNx2SIs:
;;
-;; - The VNx2SIs can be seem as VNx2DIs in which the low halves of each
+;; - The VNx2SIs can be seen as VNx2DIs in which the low halves of each
;; DI container store an SI element. The upper bits of each DI container
;; are undefined.
;;
diff --git a/gcc/testsuite/gcc.dg/vect/vect-cond-cmp-1.c b/gcc/testsuite/gcc.dg/vect/vect-cond-cmp-1.c
new file mode 100644
index 0000000000000000000000000000000000000000..6677673d06703a2506b23adac90b66e5ef3057a0
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-cond-cmp-1.c
@@ -0,0 +1,17 @@
+/* { dg-do compile } */
+/* { dg-require-effective-target vect_int } */
+/* { dg-add-options aarch64_sve } */
+/* { dg-additional-options "-fdump-tree-isel" } */
+
+#define N 1024
+
+void __attribute__ ((noipa))
+f (int *__restrict a, int *__restrict b, int *__restrict c,
+ int *__restrict d)
+{
+ for (int i = 0; i < N; ++i)
+ if (c[i] != 0 && a[i] < b[i])
+ d[i] = a[i];
+}
+
+/* { dg-final { scan-tree-dump-times "\\.COND_VEC_CMP" 2 "isel" { target aarch64*-*-* } } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/cond_vec_cmp_1.c b/gcc/testsuite/gcc.target/aarch64/sve/cond_vec_cmp_1.c
new file mode 100644
index 0000000000000000000000000000000000000000..9f2355e36713c78b212413de442d56ebd7c62741
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/cond_vec_cmp_1.c
@@ -0,0 +1,15 @@
+/* { dg-do compile { target aarch64_asm_sve_ok } } */
+/* { dg-options "-O2 -ftree-vectorize -march=armv8-a+sve -fdump-tree-isel" } */
+
+#define N 1024
+
+void __attribute__ ((noipa))
+f (int *__restrict a, int *__restrict b, int *__restrict c,
+ int *__restrict d)
+{
+ for (int i = 0; i < N; ++i)
+ if (c[i] != 0 && a[i] < b[i])
+ d[i] = a[i];
+}
+
+/* { dg-final { scan-tree-dump-times "\\.COND_VEC_CMP" 2 "isel" } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/cond_vec_cmp_2.c b/gcc/testsuite/gcc.target/aarch64/sve/cond_vec_cmp_2.c
new file mode 100644
index 0000000000000000000000000000000000000000..d67e8ffaa1c2961858ee25fea4b80155715ad009
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/cond_vec_cmp_2.c
@@ -0,0 +1,26 @@
+/* { dg-do compile { target aarch64_asm_sve_ok } } */
+/* { dg-options "-O2 -ftree-vectorize -march=armv8-a+sve -msve-vector-bits=scalable --param=aarch64-autovec-preference=sve-only" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+
+#define N 1024
+
+/*
+** f:
+** ...
+** cmpne p[0-9]+\.s, p[0-9]+/z, z[0-9]+\.s, #0
+** ...
+** cmplt p[0-9]+\.s, p[0-9]+/z, z[0-9]+\.s, z[0-9]+\.s
+** ...
+** st1w z[0-9]+\.s, p[0-9]+, \[x[0-9]+, x[0-9]+, lsl 2\]
+** ...
+** whilelo p[0-9]+\.s, w[0-9]+, w[0-9]+
+** ...
+*/
+void __attribute__ ((noipa))
+f (int *__restrict a, int *__restrict b, int *__restrict c,
+ int *__restrict d)
+{
+ for (int i = 0; i < N; ++i)
+ if (c[i] != 0 && a[i] < b[i])
+ d[i] = a[i];
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/cond_vec_cmp_2_run.c b/gcc/testsuite/gcc.target/aarch64/sve/cond_vec_cmp_2_run.c
new file mode 100644
index 0000000000000000000000000000000000000000..a23a11ef7210f1412ce477bd3df32e93263e6393
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/cond_vec_cmp_2_run.c
@@ -0,0 +1,38 @@
+/* { dg-do run { target aarch64_sve_hw } } */
+/* { dg-options "-O2 -ftree-vectorize -march=armv8-a+sve -msve-vector-bits=scalable --param=aarch64-autovec-preference=sve-only" } */
+
+#define N 1024
+
+void __attribute__ ((noipa))
+f (int *__restrict a, int *__restrict b, int *__restrict c,
+ int *__restrict d)
+{
+ for (int i = 0; i < N; ++i)
+ if (c[i] != 0 && a[i] < b[i])
+ d[i] = a[i];
+}
+
+int
+main (void)
+{
+ int a[N], b[N], c[N], d[N];
+
+ for (int i = 0; i < N; ++i)
+ {
+ a[i] = i % 31;
+ b[i] = 15;
+ c[i] = (i & 3) != 0;
+ d[i] = -1;
+ }
+
+ f (a, b, c, d);
+
+ for (int i = 0; i < N; ++i)
+ {
+ int expected = (c[i] != 0 && a[i] < b[i]) ? a[i] : -1;
+ if (d[i] != expected)
+ __builtin_abort ();
+ }
+
+ return 0;
+}
