Existing vector comparisons and VEC_COND_EXPRs do not fit the scalar
comparison and select forms used by SLP pattern recognition. To analyze them
we build patterns with scalar element and boolean result types, while keeping
the original vectors as SLP inputs. The goal here is to recognize when the
original branchless vector code was attempting to do predication.
e.g.
#include <arm_neon.h>
void select_add (uint8x8_t *restrict out, const uint8x8_t *a, unsigned n)
{
for (unsigned i = 0; i < n; ++i)
{
uint8x8_t x = a[i];
out[i] = vbsl_u8 (vcgt_u8 (x, vdup_n_u8 (0)),
x + vdup_n_u8 (1), x);
}
}
vect_recog_vect_cond_pattern works by finding a operation that produces -1 and 0
lanes, in this case compares and then finds sees if it finds a SEL consuming it.
Unfortunately we do need to find the producing operation because in general a
select is bitwise, and so without knowing the value of the lanes we can't
conclude
it's a predicated operation. For the example above we now generate:
condition = x != {0, ...};
result = .COND_ADD (condition, x, {1, ...}, x);
SLP chooses the vector data and predicate types for these nodes.
With -O3 -march=armv8.5-a+sve2 -msve-vector-bits=256 we generated before:
ldr d30, [x1, x3]
cmeq v29.8b, v30.8b, #0
add v28.8b, v30.8b, v31.8b
bit v28.8b, v30.8b, v29.8b
str d28, [x0, x3]
After these patches we get:
ld1b z30.b, p7/z, [x1, x3]
cmpne p6.b, p7/z, z30.b, #0
add z30.b, p6/m, z30.b, z31.b
st1b z30.b, p7, [x0, x3]
The comparison now predicates the SVE add, and the loop advances by 32 bytes
rather than eight. The whilelo predicate handles the tail. This comparison
includes patch 8's native source-vector support.
Bootstrapped Regtested on aarch64-none-linux-gnu,
arm-none-linux-gnueabihf, x86_64-pc-linux-gnu
-m32, -m64 and no issues.
Any feedback?
Thanks,
Tamar
gcc/ChangeLog:
PR tree-optimization/119187
* tree-vect-patterns.cc (vect_recog_vect_cond_pattern): New.
(vect_vect_recog_func_ptrs): Register the pattern.
gcc/testsuite/ChangeLog:
PR tree-optimization/119187
* gcc.dg/vect/vect-revectorize-operations.c: New test.
* gcc.target/aarch64/vect-revectorize-cond-1.c: New test.
* gcc.target/aarch64/vect-revectorize-cond-2.c: New test.
* gcc.target/aarch64/vect-revectorize-cond.h: New test support.
---
--
diff --git a/gcc/testsuite/gcc.dg/vect/vect-revectorize-operations.c
b/gcc/testsuite/gcc.dg/vect/vect-revectorize-operations.c
new file mode 100644
index
0000000000000000000000000000000000000000..03e5169361f3a2096d9a75b932a74794d1da27b7
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-revectorize-operations.c
@@ -0,0 +1,34 @@
+/* { dg-do run } */
+/* { dg-require-effective-target vect_int } */
+/* { dg-additional-options "-O3 -fno-vect-cost-model" } */
+
+#include "tree-vect.h"
+
+typedef unsigned V __attribute__ ((vector_size (32)));
+__attribute__ ((noipa)) void
+f (V *out, const V *a, const V *b)
+{
+ V x = a[0], y = b[0];
+ out[0] = (x + (V) { 1, 3, 5, 7, 9, 11, 13, 15 }) ^ y;
+ out[1] = x < y;
+ out[2] = __builtin_shuffle (x, y, (V) { 7, 8, 2, 13, 4, 11, 6, 9 });
+}
+
+int
+main (void)
+{
+ check_vect ();
+ V a = { 0, 9, 18, 27, 36, 45, 54, 63 };
+ V b = { 61, 58, 55, 52, 49, 46, 43, 40 };
+ V expected[] = {
+ { 60, 54, 32, 22, 28, 22, 104, 102 },
+ { -1, -1, -1, -1, -1, -1, 0, 0 },
+ { 63, 61, 18, 46, 36, 52, 54, 58 }
+ };
+ V out[3];
+ f (out, &a, &b);
+ if (__builtin_memcmp (out, expected, sizeof (out)))
+ __builtin_abort ();
+
+ return 0;
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/vect-revectorize-cond-1.c
b/gcc/testsuite/gcc.target/aarch64/vect-revectorize-cond-1.c
new file mode 100644
index
0000000000000000000000000000000000000000..dc356ad745af3baedde2680efbad19fe3f67f534
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/vect-revectorize-cond-1.c
@@ -0,0 +1,7 @@
+/* { dg-do run } */
+/* { dg-require-effective-target vect_int } */
+/* { dg-options "-O3 -march=armv8-a -fdump-tree-vect-details -save-temps" } */
+
+#include "vect-revectorize-cond.h"
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 5 "vect" { target
aarch64_little_endian } } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/vect-revectorize-cond-2.c
b/gcc/testsuite/gcc.target/aarch64/vect-revectorize-cond-2.c
new file mode 100644
index
0000000000000000000000000000000000000000..cf7763c004a2054a9cc2daa99c23f1579cfe0a01
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/vect-revectorize-cond-2.c
@@ -0,0 +1,9 @@
+/* { dg-do run { target aarch64_sve_hw } } */
+/* { dg-require-effective-target vect_int } */
+/* { dg-options "-O3 -march=armv8-a+sve -fdump-tree-vect-details
-fdump-tree-optimized -save-temps" } */
+
+#include "vect-revectorize-cond.h"
+
+/* { dg-final { scan-tree-dump-times "loop vectorized using variable length
vectors" 5 "vect" { target aarch64_little_endian } } } */
+/* { dg-final { scan-tree-dump {\.COND_ADD \(} "optimized" { target
aarch64_little_endian } } } */
+/* { dg-final { scan-tree-dump {\.COND_SUB \(} "optimized" { target
aarch64_little_endian } } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/vect-revectorize-cond.h
b/gcc/testsuite/gcc.target/aarch64/vect-revectorize-cond.h
new file mode 100644
index
0000000000000000000000000000000000000000..df4edb4c3aa8b32a1c0559cf5359b54963028352
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/vect-revectorize-cond.h
@@ -0,0 +1,112 @@
+#include "../../gcc.dg/vect/tree-vect.h"
+
+#include <arm_neon.h>
+
+__attribute__ ((noipa)) void
+compare (uint8_t *a, uint8_t *b, uint8_t *c, int n)
+{
+ for (int i = 0; i < n; i += 8, a += 8, b += 8, c += 8)
+ vst1_u8 (c, vcgt_u8 (vld1_u8 (a), vld1_u8 (b)));
+}
+
+__attribute__ ((noipa)) void
+select_value (uint8_t *a, uint8_t *b, uint8_t *c, int n)
+{
+ for (int i = 0; i < n; i += 8, a += 8, b += 8, c += 8)
+ {
+ uint8x8_t av = vld1_u8 (a), bv = vld1_u8 (b);
+ vst1_u8 (c, vbsl_u8 (vcgt_u8 (bv, vdup_n_u8 (0)), bv, av));
+ }
+}
+
+__attribute__ ((noipa)) void
+conditional_add (uint8_t *a, uint8_t *b, uint8_t *c, int n)
+{
+ for (int i = 0; i < n; i += 8, a += 8, b += 8, c += 8)
+ {
+ uint8x8_t bv = vld1_u8 (b);
+ vst1_u8 (c, vbsl_u8 (vcgt_u8 (bv, vdup_n_u8 (0)),
+ bv + vdup_n_u8 (1), bv));
+ }
+}
+
+/* Exercise an operation in the false arm, and nonuniform constants. */
+__attribute__ ((noipa)) void
+conditional_sub (uint8_t *a, uint8_t *b, uint8_t *c, int n)
+{
+ for (int i = 0; i < n; i += 8, a += 8, b += 8, c += 8)
+ {
+ uint8x8_t av = vld1_u8 (a), bv = vld1_u8 (b);
+ uint8x8_t k = { 1, 3, 5, 7, 9, 11, 13, 15 };
+ vst1_u8 (c, vbsl_u8 (vcgt_u8 (bv, k), bv, bv - av));
+ }
+}
+
+/* The selected values and the comparison need not have the same sign. */
+__attribute__ ((noipa)) void
+signed_select (uint8_t *a, uint8_t *b, uint8_t *c, int n)
+{
+ for (int i = 0; i < n; i += 8, a += 8, b += 8, c += 8)
+ {
+ uint8x8_t av = vld1_u8 (a), bv = vld1_u8 (b);
+ uint8x8_t mask
+ = vcgt_s8 (vreinterpret_s8_u8 (av), vreinterpret_s8_u8 (bv));
+ vst1_u8 (c, vbsl_u8 (mask, av, bv));
+ }
+}
+
+__attribute__ ((noipa, optimize ("no-tree-vectorize"))) void
+reference (uint8_t *a, uint8_t *b, uint8_t *c, int n, unsigned kind)
+{
+ for (int i = 0; i < n; i += 8, a += 8, b += 8, c += 8)
+ {
+ uint8_t tmp[8];
+ for (unsigned j = 0; j < 8; ++j)
+ switch (kind)
+ {
+ case 0: tmp[j] = a[j] > b[j] ? 255 : 0; break;
+ case 1: tmp[j] = b[j] > 0 ? b[j] : a[j]; break;
+ case 2: tmp[j] = b[j] > 0 ? b[j] + 1 : b[j]; break;
+ case 3:
+ {
+ /* K uses C vector indices; the loads use ACLE lane order. */
+ unsigned lane
+ = (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ ? 7 - j : j);
+ tmp[j] = b[j] > 2 * lane + 1 ? b[j] : b[j] - a[j];
+ break;
+ }
+
+ case 4:
+ tmp[j] = (int8_t) a[j] > (int8_t) b[j] ? a[j] : b[j]; break;
+ }
+
+ for (unsigned j = 0; j < 8; ++j)
+ c[j] = tmp[j];
+ }
+}
+
+__attribute__ ((optimize ("no-tree-vectorize"))) int
+main (void)
+{
+ check_vect ();
+ void (*tests[]) (uint8_t *, uint8_t *, uint8_t *, int)
+ = { compare, select_value, conditional_add, conditional_sub,
+ signed_select };
+ static const unsigned offsets[]
+ = { 640, 32, 33, 40, 48, 31, 24, 320, 321, 328, 336, 319, 312 };
+ uint8_t actual[1024], expected[1024];
+ for (unsigned kind = 0; kind < 5; ++kind)
+ for (unsigned k = 0; k < sizeof (offsets) / sizeof (offsets[0]); ++k)
+ for (int n = -1; n <= 193; ++n)
+ {
+ for (unsigned j = 0; j < sizeof (actual); ++j)
+ actual[j] = expected[j] = 17 * j + 13 * n;
+ reference (expected + 32, expected + 320, expected + offsets[k],
+ n, kind);
+ tests[kind] (actual + 32, actual + 320, actual + offsets[k], n);
+ if (__builtin_memcmp (actual, expected, sizeof (actual)))
+ __builtin_abort ();
+ }
+
+ return 0;
+}
diff --git a/gcc/tree-vect-patterns.cc b/gcc/tree-vect-patterns.cc
index
cbe834e2aafb8a770ad2bc3ed9be614481302309..1e9067e4e4e7a9d136f54f3ec7a91db6c770fd36
100644
--- a/gcc/tree-vect-patterns.cc
+++ b/gcc/tree-vect-patterns.cc
@@ -6475,6 +6475,124 @@ vect_recog_bool_pattern (vec_info *vinfo,
return NULL;
}
+/* Recognize M = A < B; R = VEC_COND_EXPR <M, X, Y>, where M is a vector
+ mask, and build scalar comparison and COND_EXPR pattern statements.
+ This lets the existing scalar condition vectorizer handle the select;
+ SLP repeats the patterns for each source lane and chooses a new width.
+ If X or Y is a single-use operation supported by a conditional internal
+ function, build that function instead of a separate operation and select.
*/
+
+static gimple *
+vect_recog_vect_cond_pattern (vec_info *vinfo,
+ stmt_vec_info stmt_vinfo, tree *type_out)
+{
+ gimple *last_stmt = STMT_VINFO_STMT (stmt_vinfo);
+ enum tree_code rhs_code;
+ tree var, lhs, vectype;
+ gimple *pattern_stmt;
+
+ if (!is_gimple_assign (last_stmt))
+ return NULL;
+
+ var = gimple_assign_rhs1 (last_stmt);
+ lhs = gimple_assign_lhs (last_stmt);
+ rhs_code = gimple_assign_rhs_code (last_stmt);
+
+ if (rhs_code != VEC_COND_EXPR
+ || !VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (var))
+ || !TYPE_VECTOR_SUBPARTS (TREE_TYPE (lhs)).is_constant ())
+ return NULL;
+
+ tree op0 = gimple_assign_rhs2 (last_stmt);
+ tree op1 = gimple_assign_rhs3 (last_stmt);
+
+ /* Find the comparison and make it part of the pattern we'll make. */
+ auto cmp_info = vinfo->lookup_def (var);
+ if (!cmp_info)
+ return NULL;
+
+ gimple *cmp_stmt = STMT_VINFO_STMT (cmp_info);
+ if (!is_gimple_assign (cmp_stmt)
+ || TREE_CODE_CLASS (gimple_assign_rhs_code (cmp_stmt)) != tcc_comparison)
+ return NULL;
+
+ vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (lhs));
+ if (vectype == NULL_TREE)
+ return NULL;
+
+ tree type = TREE_TYPE (TREE_TYPE (lhs));
+ tree new_vectype = truth_type_for (vectype);
+
+ tree_code cmp_code = gimple_assign_rhs_code (cmp_stmt);
+ gassign *operation = nullptr;
+ internal_fn cond_fn = IFN_LAST;
+ for (unsigned arm = 0; arm < 2; ++arm)
+ {
+ tree selected = arm == 0 ? op0 : op1;
+ if (TREE_CODE (selected) != SSA_NAME || !has_single_use (selected))
+ continue;
+
+ auto def_info = vinfo->lookup_def (selected);
+ auto def = def_info ? dyn_cast <gassign *> (def_info->stmt) : nullptr;
+ if (!def || gimple_could_trap_p (def))
+ continue;
+
+ internal_fn fn
+ = get_conditional_internal_fn (gimple_assign_rhs_code (def));
+ if (fn == IFN_LAST
+ || !direct_internal_fn_supported_p (fn, vectype,
+ OPTIMIZE_FOR_SPEED))
+ continue;
+
+ if (arm == 1)
+ {
+ tree cmp_type = TREE_TYPE (gimple_assign_rhs1 (cmp_stmt));
+ tree_code inverted
+ = invert_tree_comparison (cmp_code, HONOR_NANS (cmp_type));
+ if (inverted == ERROR_MARK)
+ continue;
+
+ cmp_code = inverted;
+ std::swap (op0, op1);
+ }
+
+ operation = def;
+ cond_fn = fn;
+ break;
+ }
+
+ /* The pattern condition is scalar even though the original comparison
+ operates on vectors. Its SLP node retains the predicate vector type. */
+ tree lhs_cmp = vect_recog_temp_ssa_var (boolean_type_node, NULL);
+ gimple *copy_stmt
+ = gimple_build_assign (lhs_cmp, cmp_code,
+ gimple_assign_rhs1 (cmp_stmt),
+ gimple_assign_rhs2 (cmp_stmt));
+ append_pattern_def_seq (vinfo, stmt_vinfo, copy_stmt, new_vectype, type);
+
+ tree lhs_ivar = vect_recog_temp_ssa_var (type, NULL);
+ if (operation)
+ {
+ auto_vec<tree, 5> args;
+ unsigned nops = gimple_num_ops (operation);
+ args.quick_grow (nops);
+ args[0] = lhs_cmp;
+ std::copy (gimple_ops (operation) + 1, gimple_ops (operation) + nops,
+ args.address () + 1);
+ args.quick_push (op1);
+ gcall *call = gimple_build_call_internal_vec (cond_fn, args);
+ gimple_call_set_lhs (call, lhs_ivar);
+ pattern_stmt = call;
+ }
+ else
+ pattern_stmt
+ = gimple_build_assign (lhs_ivar, COND_EXPR, lhs_cmp, op0, op1);
+
+ *type_out = vectype;
+ vect_pattern_detected ("vect_recog_vect_cond_pattern", last_stmt);
+
+ return pattern_stmt;
+}
/* Function vect_recog_mask_conversion_pattern
@@ -7701,6 +7819,7 @@ static vect_recog_func vect_vect_recog_func_ptrs[] = {
{ vect_recog_sat_trunc_pattern, "sat_trunc" },
{ vect_recog_gcond_pattern, "gcond" },
{ vect_recog_bool_pattern, "bool" },
+ { vect_recog_vect_cond_pattern, "vect_cond" },
/* This must come before mask conversion, and includes the parts
of mask conversion that are needed for gather and scatter
internal functions. */
