To keep a target vector builtin in an SLP tree we must check whether the
target can implement it with the vector types chosen for its operands. We use
the MASK_CALL folding protocol to make that query during analysis, then
resolve the call again with the real operands during transformation.
e.g.
#include <arm_neon.h>
void aes_rounds (uint8x16_t *restrict out, const uint8x16_t *a,
const uint8x16_t *b, unsigned n)
{
for (unsigned i = 0; i < n; ++i)
out[i] = vaesmcq_u8 (vaeseq_u8 (a[i], b[i]));
}
vectorizable_vect_call gets vector argument types from the SLP children.
Scalar arguments keep their values and must be invariant and equal across
combined calls, so immediate constraints can still be checked. Calls with a
virtual memory use or a possible internal exception are rejected.
vect_fold_vector_call builds a temporary MASK_CALL and runs the ordinary
folder. Unknown vector operands use temporary SSA names, since constant zeros
could give a special-case fold that does not work for arbitrary data. We first
probe with an all-true mask, then with an unknown mask to check partial-vector
support. A failed partial probe disables partial vectors for the loop. The
temporary statements and SSA names are then discarded.
With -O3 -fno-vect-cost-model -march=armv8-a+crypto, this example is
accepted at its original width. Its SLP tree contains:
store, 16 source byte lanes
vaesmcq_u8, 16 source byte lanes
vaeseq_u8, 16 source byte lanes
load a, 16 source byte lanes
load b, 16 source byte lanes
Transformation resolves the calls using the actual SLP operands and selected
mask, inserts the folded sequences and records their results as vector
definitions. The optimized GIMPLE contains:
aes = __builtin_aarch64_vaeseq_u8 (a_piece, b_piece);
result = __builtin_aarch64_vaesmcq_u8 (aes);
No private MASK_CALL remains. AES still produces one aese and one aesmc per
source iteration at VF 1; the change is that SLP can analyze and rebuild the
calls. The next patch adds arithmetic implementations that can change width.
Bootstrapped Regtested on aarch64-none-linux-gnu,
arm-none-linux-gnueabihf, x86_64-pc-linux-gnu
-m32, -m64 and no issues.
Any feedback?
Thanks,
Tamar
gcc/ChangeLog:
PR tree-optimization/119187
* tree-vect-stmts.cc (vect_fold_vector_call, vectorizable_vect_call):
New.
(vectorizable_call): Dispatch target vector builtin calls.
* tree-vect-slp.cc (vect_slp_tree_uniform_p): Make externally visible.
(vect_slp_analyze_node_operations, vect_schedule_slp_node): Handle calls
with scalar arguments.
* tree-vectorizer.h (vect_slp_tree_uniform_p): Declare.
gcc/testsuite/ChangeLog:
PR tree-optimization/119187
* gcc.target/aarch64/vect-revectorize-intrinsics-fold-aes-2.c: New test.
* gcc.target/aarch64/vect-revectorize-intrinsics-fold-14.c: New test.
---
--
diff --git
a/gcc/testsuite/gcc.target/aarch64/vect-revectorize-intrinsics-fold-14.c
b/gcc/testsuite/gcc.target/aarch64/vect-revectorize-intrinsics-fold-14.c
new file mode 100644
index
0000000000000000000000000000000000000000..93d0533e1aae0e035227e60c24a816dfe627ff34
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/vect-revectorize-intrinsics-fold-14.c
@@ -0,0 +1,67 @@
+/* { dg-do run } */
+/* { dg-require-effective-target vect_int } */
+/* { dg-options "-O3 -fno-vect-cost-model -fchecking=2 -fdump-tree-optimized"
} */
+
+#include "../../gcc.dg/vect/tree-vect.h"
+
+#include <arm_neon.h>
+
+/* Unsupported scalar immediates must leave the original calls intact. */
+__attribute__ ((noipa)) void
+same_shift (uint8x8_t *restrict out, const uint8x8_t *in, unsigned n)
+{
+ for (unsigned i = 0; i < n; ++i)
+ out[i] = vrshr_n_u8 (in[i], 3);
+}
+
+/* Different scalar immediates prevent combining otherwise compatible calls.
*/
+__attribute__ ((noipa)) void
+different_shifts (uint8x8_t *restrict out, const uint8x8_t *in)
+{
+ out[0] = vrshr_n_u8 (in[0], 3);
+ out[1] = vrshr_n_u8 (in[1], 5);
+}
+
+__attribute__ ((optimize ("no-tree-vectorize"))) int
+main (void)
+{
+ check_vect ();
+ uint8x8_t in[33], out[33], expected3[33], expected5[33];
+ const uint8x8_t guard = { 123, 123, 123, 123, 123, 123, 123, 123 };
+
+ /* Form the rounding reference in a wider scalar type. */
+ for (unsigned k = 0; k < 33 * 8; ++k)
+ {
+ unsigned i = k / 8, j = k % 8;
+ unsigned x = (17 * i + 13 * j) & 255;
+ in[i][j] = x;
+ expected3[i][j] = (x + 4) / 8;
+ expected5[i][j] = (x + 16) / 32;
+ }
+
+ for (unsigned n = 0; n <= 33; ++n)
+ {
+ __builtin_memset (out, 123, sizeof (out));
+ same_shift (out, in, n);
+ if (__builtin_memcmp (out, expected3, n * sizeof (out[0])))
+ __builtin_abort ();
+
+ for (unsigned i = n; i < 33; ++i)
+ if (__builtin_memcmp (&out[i], &guard, sizeof (guard)))
+ __builtin_abort ();
+ }
+
+ for (unsigned i = 0; i < 32; ++i)
+ {
+ different_shifts (out, in + i);
+ if (__builtin_memcmp (&out[0], &expected3[i], sizeof (out[0]))
+ || __builtin_memcmp (&out[1], &expected5[i + 1], sizeof (out[1])))
+ __builtin_abort ();
+ }
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump {__builtin_aarch64_urshr_nv8qi_uus \([^,]+,
3\)} "optimized" } } */
+/* { dg-final { scan-tree-dump {__builtin_aarch64_urshr_nv8qi_uus \([^,]+,
5\)} "optimized" } } */
+/* { dg-final { scan-tree-dump-not "\\.MASK_CALL" "optimized" } } */
diff --git
a/gcc/testsuite/gcc.target/aarch64/vect-revectorize-intrinsics-fold-aes-2.c
b/gcc/testsuite/gcc.target/aarch64/vect-revectorize-intrinsics-fold-aes-2.c
new file mode 100644
index
0000000000000000000000000000000000000000..89c920ccc47819e461ed3de33506c9f8c36ad707
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/vect-revectorize-intrinsics-fold-aes-2.c
@@ -0,0 +1,19 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -march=armv8-a+aes -fno-vect-cost-model
-fdump-tree-vect-details -fdump-tree-optimized" } */
+
+#include <arm_neon.h>
+
+/* Both calls remain builtins until the vectorizer folds them. */
+void
+aes_rounds (uint8x16_t *restrict out, const uint8x16_t *a,
+ const uint8x16_t *b, unsigned n)
+{
+ for (unsigned i = 0; i < n; ++i)
+ out[i] = vaesmcq_u8 (vaeseq_u8 (a[i], b[i]));
+}
+
+/* { dg-final { scan-tree-dump "vectorizable_vect_call" "vect" } } */
+/* { dg-final { scan-tree-dump "LOOP VECTORIZED" "vect" } } */
+/* { dg-final { scan-tree-dump-not "\\.MASK_CALL|VIEW_CONVERT_EXPR"
"optimized" } } */
+/* { dg-final { scan-assembler {aese\tv[0-9]+\.16b, v[0-9]+\.16b} } } */
+/* { dg-final { scan-assembler {aesmc\tv[0-9]+\.16b, v[0-9]+\.16b} } } */
diff --git a/gcc/tree-vect-slp.cc b/gcc/tree-vect-slp.cc
index
c6e94bd422b4d4b811c20cac4407c4aaa09fc318..37ea25ba8384dd0c721d0a1491e7362c44e230ba
100644
--- a/gcc/tree-vect-slp.cc
+++ b/gcc/tree-vect-slp.cc
@@ -397,7 +397,7 @@ vect_slp_tree_scalar_ops_p (slp_tree node)
/* Return true when all operands in the external or constant NODE have
the same value. Vector operands still have their implicit lane order. */
-static bool
+bool
vect_slp_tree_uniform_p (slp_tree node)
{
gcc_assert (SLP_TREE_DEF_TYPE (node) == vect_constant_def
@@ -9745,11 +9745,12 @@ vect_slp_analyze_node_operations (vec_info *vinfo,
slp_tree node,
|| SLP_TREE_DEF_TYPE (child) == vect_constant_def))
continue;
- /* For shifts with a scalar argument we don't need
+ /* For shifts and calls with a scalar argument we don't need
to cost or code-generate anything.
??? Represent this more explicitly. */
- gcc_assert (SLP_TREE_TYPE (node) == shift_vec_info_type
- && j == 1);
+ gcc_assert ((SLP_TREE_TYPE (node) == shift_vec_info_type
+ && j == 1)
+ || SLP_TREE_TYPE (node) == call_vec_info_type);
continue;
}
@@ -12817,7 +12818,7 @@ vect_schedule_slp_node (vec_info *vinfo,
return true;
}
- /* ??? vectorizable_shift can end up using a scalar operand which is
+ /* ??? Shifts and calls can end up using a scalar operand which is
currently denoted as !SLP_TREE_VECTYPE. No need to vectorize the
node in this case. */
if (!SLP_TREE_VECTYPE (node))
diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
index
bfb884203aa99f5c25487c853b22f7240c96b304..457c2d6b46cf02e0ee52ff1984822dd3343fe2f5
100644
--- a/gcc/tree-vect-stmts.cc
+++ b/gcc/tree-vect-stmts.cc
@@ -3403,6 +3403,157 @@ simple_integer_narrowing (tree vectype_out, tree
vectype_in,
return true;
}
+/* Try to fold STMT's target builtin with result LHS, operands ARGS and
+ execution mask MASK. Return the folded sequence in SEQ if requested,
+ otherwise discard it after checking whether folding succeeded. For
+ a probe, LHS is the result type, and types in ARGS or MASK stand for
+ unknown values of those types. */
+
+static bool
+vect_fold_vector_call (gcall *stmt, tree lhs, const vec<tree> &args,
+ tree mask, gimple_seq *seq = nullptr)
+{
+ auto_vec<tree> operands (args.length () + 2);
+ operands.quick_push (gimple_call_fn (stmt));
+ operands.splice (args);
+ operands.quick_push (mask);
+ auto_vec<tree> temporaries;
+ if (!seq)
+ {
+ lhs = make_ssa_name (lhs);
+ for (tree &op : operands)
+ if (TYPE_P (op))
+ {
+ op = make_ssa_name (op, gimple_build_nop ());
+ temporaries.safe_push (op);
+ }
+ }
+
+ gcall *call = gimple_build_call_internal_vec (IFN_MASK_CALL, operands);
+ gimple_call_set_lhs (call, lhs);
+ gimple_set_location (call, gimple_location (stmt));
+ gimple_seq stmts = NULL;
+ gimple_seq_add_stmt (&stmts, call);
+ gimple_stmt_iterator si = gsi_start (stmts);
+ fold_stmt (&si);
+ bool supported = !gimple_call_internal_p (gsi_stmt (si), IFN_MASK_CALL);
+ if (seq)
+ *seq = stmts;
+ else
+ gimple_seq_discard (stmts);
+
+ for (tree tmp : temporaries)
+ release_ssa_name (tmp);
+ return supported;
+}
+
+/* Vectorize a target builtin operating on vector operands. Folding
+ decides whether the operation can use the requested types; it can keep the
+ operation as a builtin, lower it to GIMPLE, or reject the request. */
+
+static bool
+vectorizable_vect_call (vec_info *vinfo, stmt_vec_info stmt_info, gcall *stmt,
+ gimple_stmt_iterator *gsi, slp_tree slp_node,
+ stmt_vector_for_cost *cost_vec)
+{
+ if (gimple_vuse (stmt) || stmt_can_throw_internal (cfun, stmt))
+ return false;
+
+ tree vectype = SLP_TREE_VECTYPE (slp_node);
+ unsigned nargs = gimple_call_num_args (stmt);
+ if (!nargs || vect_emulated_vector_p (vectype))
+ return false;
+
+ loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
+ unsigned nvectors = vect_get_num_copies (vinfo, slp_node);
+ tree mask_type = truth_type_for (vectype);
+ auto_vec<tree> args (nargs);
+ if (cost_vec)
+ {
+ for (unsigned i = 0; i < nargs; ++i)
+ {
+ slp_tree op;
+ vect_def_type dt;
+ tree argtype;
+ if (!vect_is_simple_use (vinfo, slp_node, i, &op, &dt, &argtype))
+ return false;
+
+ tree arg = gimple_call_arg (stmt, i);
+ if (!VECTOR_TYPE_P (TREE_TYPE (arg)))
+ {
+ /* Scalar arguments, such as immediate shift amounts, must be
+ invariant and equal across the calls being combined. */
+ if ((dt != vect_constant_def && dt != vect_external_def)
+ || !vect_slp_tree_uniform_p (op))
+ return false;
+
+ arg = vect_get_slp_scalar_def (op, 0);
+ }
+ else
+ {
+ if (!argtype)
+ argtype
+ = get_vectype_for_scalar_type (vinfo, TREE_TYPE (arg), op);
+
+ if (!argtype || !vect_maybe_update_slp_op_vectype (op, argtype))
+ return false;
+
+ arg = argtype;
+ }
+
+ args.quick_push (arg);
+ }
+
+ if (!vect_fold_vector_call (stmt, vectype, args,
+ build_all_ones_cst (mask_type)))
+ return false;
+
+ if (loop_vinfo && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
+ {
+ if (vect_fold_vector_call (stmt, vectype, args, mask_type))
+ vect_record_loop_mask (loop_vinfo, &LOOP_VINFO_MASKS (loop_vinfo),
+ nvectors, vectype, NULL_TREE);
+ else
+ LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
+ }
+
+ SLP_TREE_TYPE (slp_node) = call_vec_info_type;
+ DUMP_VECT_SCOPE ("vectorizable_vect_call");
+ vect_model_simple_cost (vinfo, 1, slp_node, cost_vec);
+ return true;
+ }
+
+ for (unsigned copy = 0; copy < nvectors; ++copy)
+ {
+ args.truncate (0);
+ for (unsigned i = 0; i < nargs; ++i)
+ {
+ slp_tree op = SLP_TREE_CHILDREN (slp_node)[i];
+ args.quick_push (VECTOR_TYPE_P (TREE_TYPE (gimple_call_arg (stmt, i)))
+ ? vect_get_slp_vect_def (op, copy)
+ : vect_get_slp_scalar_def (op, 0));
+ }
+
+ tree mask = build_all_ones_cst (mask_type);
+ if (loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
+ mask
+ = vect_get_loop_mask (loop_vinfo, gsi,
+ &LOOP_VINFO_MASKS (loop_vinfo), nvectors,
+ vectype, copy);
+
+ gimple_seq seq = NULL;
+ tree result = make_ssa_name (vectype);
+ bool folded = vect_fold_vector_call (stmt, result, args, mask, &seq);
+ gcc_assert (folded);
+ for (auto si = gsi_start (seq); !gsi_end_p (si); gsi_next (&si))
+ vect_finish_stmt_generation_1 (vinfo, stmt_info, gsi_stmt (si));
+ gsi_insert_seq_before (gsi, seq, GSI_SAME_STMT);
+ slp_node->push_vec_def (result);
+ }
+
+ return true;
+}
+
/* Function vectorizable_call.
Check if STMT_INFO performs a function call that can be vectorized.
@@ -3459,6 +3610,11 @@ vectorizable_call (vec_info *vinfo,
|| TREE_CODE (gimple_call_lhs (stmt)) != SSA_NAME)
return false;
+ if (gimple_call_builtin_p (stmt, BUILT_IN_MD)
+ && VECTOR_TYPE_P (TREE_TYPE (gimple_call_lhs (stmt))))
+ return vectorizable_vect_call (vinfo, stmt_info, stmt, gsi, slp_node,
+ cost_vec);
+
gcc_checking_assert (!stmt_can_throw_internal (cfun, stmt));
vectype_out = SLP_TREE_VECTYPE (slp_node);
diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
index
b97140b65bf407dbbbaafeacc0609a8c9bb5e58e..a4adad03a648d9e65e22d99a465e33604a781361
100644
--- a/gcc/tree-vectorizer.h
+++ b/gcc/tree-vectorizer.h
@@ -2792,6 +2792,7 @@ extern bool vect_make_slp_decision (loop_vec_info);
extern void vect_optimize_slp (vec_info *);
extern void vect_gather_slp_loads (vec_info *);
extern tree vect_get_slp_scalar_def (slp_tree, unsigned);
+extern bool vect_slp_tree_uniform_p (slp_tree);
extern void vect_get_slp_defs (slp_tree, vec<tree> *);
extern void vect_get_slp_defs (vec_info *, slp_tree, vec<vec<tree> > *,
unsigned n = -1U);
