Vector PHIs must stay inside the SLP graph when we split a vector recurrence.
We also need to reconstruct the original vector type when its value is used
outside the graph.
e.g.
typedef unsigned V8 __attribute__((vector_size(32)));
void recurrence (V8 *out, V8 x, unsigned n)
{
for (unsigned i = 0; i < n; ++i)
x = x * 2 + (V8) {1,3,5,7,9,11,13,15};
*out = x;
}
We mark these PHIs as internal definitions and vectorize the cycle at loop
VF 1: each transformed loop iteration processes one original loop iteration.
The SLP node still has eight element lanes for the V8 statement. On Adv. SIMD
we split those lanes into two four-element vectors, with a PHI for each piece:
Before splitting:
x = PHI <initial_x, next_x>;
next_x = x * {2, ...} + {1,3,5,7,9,11,13,15};
After splitting:
x_lo = PHI <initial_lo, next_lo>;
x_hi = PHI <initial_hi, next_hi>;
next_lo = x_lo * {2,2,2,2} + {1,3,5,7};
next_hi = x_hi * {2,2,2,2} + {9,11,13,15};
The scheduler creates the PHIs with their initial values and fills in each
backedge after scheduling its definition. Scalar PHI-to-PHI cycles use the
same path. Induction-variable advancement excludes these internal PHIs so
epilogue setup does not advance them as scalar induction variables.
With -O3 -march=armv8-a we generated before:
sshl v28.4s, v30.4s, v26.4s
sshl v29.4s, v31.4s, v26.4s
add v28.4s, v27.4s, v28.4s
add v29.4s, v25.4s, v29.4s
mov v30.16b, v28.16b
mov v31.16b, v29.16b
After these patches we get:
sshl v30.4s, v30.4s, v28.4s
sshl v31.4s, v31.4s, v28.4s
add v30.4s, v29.4s, v30.4s
add v31.4s, v27.4s, v31.4s
The recurrence is carried directly in the arithmetic result registers,
removing the copies between iterations.
For a whole-vector live-out we wait until all pieces are available, then build
{result_lo, result_hi}, or extract the source-sized range from a wider result.
Bootstrapped Regtested on aarch64-none-linux-gnu,
arm-none-linux-gnueabihf, x86_64-pc-linux-gnu
-m32, -m64 and no issues.
Any feedback?
Thanks,
Tamar
gcc/ChangeLog:
PR tree-optimization/119187
* tree-vect-loop-manip.cc (iv_phi_p): Exclude internal PHI cycles.
(vect_can_advance_ivs_p): Diagnose non-induction PHIs.
(vect_update_ivs_after_vectorizer): Likewise.
* tree-vect-loop.cc (vect_analyze_scalar_cycles_1): Recognize vector
PHIs and scalar PHI cycles.
(vect_compute_single_scalar_iteration_cost): Cost emulated vector PHIs.
(vectorizable_phi): Handle internal loop PHIs at VF equal to one.
(vectorizable_live_operation_1): Reconstruct whole vectors using the
existing loop-exit PHI path.
(vectorizable_live_operation): Track one live lane per source vector and
assemble split live-outs after all their pieces are available.
* tree-vect-slp.cc (vect_get_slp_live_lc_phi_def,
vect_analyze_slp_live_lanes): New.
(vect_analyze_slp): Build complete vector live-out roots and group
lowered lanes by bit position in one pass per exit edge, preserving
source destination order.
* tree-vect-stmts.cc (vect_analyze_stmt): Allow internal loop PHIs.
(vect_transform_stmt): Likewise.
* tree-vectorizer.h (vectorizable_phi): Handle internal loop PHIs at VF
equal to one.
gcc/testsuite/ChangeLog:
PR tree-optimization/119187
* gcc.dg/vect/vect-revectorize-phi-cycle.c: New test.
* gcc.dg/vect/vect-revectorize-bb-live.c: New test.
* gcc.dg/vect/vect-revectorize-loop-live.c: Likewise.
* gcc.dg/vect/vect-revectorize-wide-recurrence.c: New test.
* gcc.target/aarch64/vect-revectorize-sra-unsupported-vector-1.c: New
test.
---
--
diff --git a/gcc/testsuite/gcc.dg/vect/vect-revectorize-bb-live.c
b/gcc/testsuite/gcc.dg/vect/vect-revectorize-bb-live.c
new file mode 100644
index
0000000000000000000000000000000000000000..2615b2eace1991817bfbfa55c8a4d424345785e9
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-revectorize-bb-live.c
@@ -0,0 +1,34 @@
+/* { dg-do run } */
+/* { dg-require-effective-target vect_int } */
+/* { dg-additional-options "-O3 -fno-vect-cost-model -Wno-psabi" } */
+
+#include "tree-vect.h"
+
+typedef unsigned V __attribute__ ((vector_size (32)));
+
+/* The store seeds SLP, while the return keeps the whole source vector live.
*/
+__attribute__ ((noipa)) V
+store_and_return (V *out, const V *a, const V *b)
+{
+ V value = (*a + *b) ^ (V) { 1, 3, 5, 7, 9, 11, 13, 15 };
+ *out = value;
+ return value;
+}
+
+int
+main (void)
+{
+ check_vect ();
+ V a, b, stored;
+ for (unsigned i = 0; i < 8; ++i)
+ {
+ a[i] = 13 * i;
+ b[i] = 100 - 7 * i;
+ }
+ V returned = store_and_return (&stored, &a, &b);
+ for (unsigned i = 0; i < 8; ++i)
+ if (stored[i] != ((100 + 6 * i) ^ (2 * i + 1))
+ || returned[i] != stored[i])
+ __builtin_abort ();
+ return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/vect/vect-revectorize-loop-live.c
b/gcc/testsuite/gcc.dg/vect/vect-revectorize-loop-live.c
new file mode 100644
index
0000000000000000000000000000000000000000..27ec5bfc6611a042ac00f8ecbd878b7abbb9f213
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-revectorize-loop-live.c
@@ -0,0 +1,85 @@
+/* { dg-do run } */
+/* { dg-require-effective-target vect_int } */
+/* { dg-additional-options "-O3 -fno-vect-cost-model -Wno-psabi" } */
+
+#include "tree-vect.h"
+
+typedef unsigned V __attribute__ ((vector_size (64)));
+
+__attribute__ ((noipa)) V
+live (const V *a, V bias, unsigned n)
+{
+ V x = bias;
+ for (unsigned i = 0; i < n; ++i)
+ x = (x + a[i]) ^ bias;
+ return x;
+}
+
+/* Early exits must retain every lane of the last completed iteration. */
+__attribute__ ((noipa)) V
+live_break (const V *a, const unsigned *stop, V bias, unsigned n)
+{
+ V x = bias;
+ for (unsigned i = 0; i < n; ++i)
+ {
+ if (stop[i])
+ break;
+
+ x = (x + a[i]) ^ bias;
+ }
+
+ return x;
+}
+
+static const V a[] = {
+ { 0, 11, 22, 33, 44, 55, 66, 77, 88, 99, 110, 121, 132, 143, 154, 165 },
+ { 7, 18, 29, 40, 51, 62, 73, 84, 95, 106, 117, 128, 139, 150, 161, 172 },
+ { 14, 25, 36, 47, 58, 69, 80, 91, 102, 113, 124, 135, 146, 157, 168, 179 },
+ { 21, 32, 43, 54, 65, 76, 87, 98, 109, 120, 131, 142, 153, 164, 175, 186 },
+ { 28, 39, 50, 61, 72, 83, 94, 105, 116, 127, 138, 149, 160, 171, 182, 193 },
+ { 35, 46, 57, 68, 79, 90, 101, 112, 123, 134, 145, 156, 167, 178, 189, 200 },
+ { 42, 53, 64, 75, 86, 97, 108, 119, 130, 141, 152, 163, 174, 185, 196, 207 },
+ { 49, 60, 71, 82, 93, 104, 115, 126, 137, 148, 159, 170, 181, 192, 203, 214
+ },
+ { 56, 67, 78, 89, 100, 111, 122, 133, 144, 155, 166, 177, 188, 199, 210, 221
+ }
+};
+static const V expected[] = {
+ { 3, 20, 37, 54, 71, 88, 105, 122, 139, 156, 173, 190, 207, 224, 241, 258 },
+ { 0, 11, 30, 97, 52, 215, 194, 189, 104, 99, 438, 393, 412, 399, 378, 165 },
+ { 4, 9, 30, 191, 32, 333, 354, 363, 76, 81, 646, 695, 744, 709, 746, 83 },
+ { 17, 54, 103, 216, 29, 458, 475, 444, 57, 94, 943, 896, 949, 898, 867, 4 },
+ { 37, 66, 183, 312, 25, 590, 603, 612, 45, 74, 1183, 1200, 1153, 1222, 1251,
+ 444 },
+ { 66, 125, 204, 323, 38, 761, 720, 695, 42, 85, 1412, 1531, 1518, 1425,
+ 1384, 895 },
+ { 102, 191, 288, 433, 50, 779, 860, 861, 46, 71, 1720, 1577, 1626, 1699,
+ 1748, 1349 },
+ { 147, 224, 325, 458, 207, 820, 929, 942, 59, 72, 2045, 1650, 1991, 1980,
+ 1897, 1814 },
+ { 199, 264, 425, 554, 363, 964, 1149, 1110, 79, 64, 2097, 1954, 2227, 2204,
+ 2245, 1774 },
+ { 252, 351, 466, 693, 392, 1131, 1182, 1185, 84, 71, 2170, 2285, 2464, 2435,
+ 2406, 1737 }
+};
+
+int
+main (void)
+{
+ check_vect ();
+ for (unsigned n = 0; n <= 9; ++n)
+ for (unsigned end = 0; end <= n; ++end)
+ {
+ unsigned stop[9] = { 0 };
+ if (end < 9)
+ stop[end] = 1;
+
+ V result = live (a, expected[0], n);
+ V early = live_break (a, stop, expected[0], n);
+ if (__builtin_memcmp (&result, &expected[n], sizeof (V))
+ || __builtin_memcmp (&early, &expected[end], sizeof (V)))
+ __builtin_abort ();
+ }
+
+ return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/vect/vect-revectorize-phi-cycle.c
b/gcc/testsuite/gcc.dg/vect/vect-revectorize-phi-cycle.c
new file mode 100644
index
0000000000000000000000000000000000000000..dc16092d4c9b4e3984cab420c4060398e1d0a672
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-revectorize-phi-cycle.c
@@ -0,0 +1,43 @@
+/* { dg-do run } */
+/* { dg-require-effective-target vect_int } */
+/* { dg-additional-options "-O3 -fno-vect-cost-model" } */
+
+#include "tree-vect.h"
+
+__attribute__ ((noipa)) void
+f (unsigned *out, unsigned n)
+{
+ unsigned a = 1, b = 2, c = 3;
+ for (unsigned i = 0; i < n; ++i)
+ {
+ unsigned t = a;
+ a = b;
+ b = c;
+ c = t;
+ }
+
+ out[0] = a;
+ out[1] = b;
+ out[2] = c;
+}
+
+static const unsigned expected[][3] = {
+ { 1, 2, 3 },
+ { 2, 3, 1 },
+ { 3, 1, 2 }
+};
+
+int
+main (void)
+{
+ check_vect ();
+ for (unsigned n = 0; n < 19; ++n)
+ {
+ unsigned out[3];
+ f (out, n);
+ if (__builtin_memcmp (out, expected[n % 3], sizeof (out)))
+ __builtin_abort ();
+ }
+
+ return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/vect/vect-revectorize-wide-recurrence.c
b/gcc/testsuite/gcc.dg/vect/vect-revectorize-wide-recurrence.c
new file mode 100644
index
0000000000000000000000000000000000000000..ce578201cb61ea34adb8d28c7032a1337faec56b
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-revectorize-wide-recurrence.c
@@ -0,0 +1,54 @@
+/* { dg-do run } */
+/* { dg-require-effective-target vect_int } */
+/* { dg-additional-options "-O3 -fno-vect-cost-model" } */
+
+#include "tree-vect.h"
+
+typedef unsigned V __attribute__ ((vector_size (32)));
+__attribute__ ((noipa)) void
+f (V *out, unsigned n)
+{
+ V x = (V) { 1, 2, 3, 4, 5, 6, 7, 8 };
+ V y = (V) { 19, 20, 21, 22, 23, 24, 25, 26 };
+ for (unsigned i = 0; i < n; ++i)
+ {
+ y = x;
+ x = x * 2 + (V) { 1, 3, 5, 7, 9, 11, 13, 15 };
+ }
+
+ out[0] = x;
+ out[1] = y;
+}
+
+static const unsigned expected[][16] = {
+ { 1, 2, 3, 4, 5, 6, 7, 8, 19, 20, 21, 22, 23, 24, 25, 26 },
+ { 3, 7, 11, 15, 19, 23, 27, 31, 1, 2, 3, 4, 5, 6, 7, 8 },
+ { 7, 17, 27, 37, 47, 57, 67, 77, 3, 7, 11, 15, 19, 23, 27, 31 },
+ { 15, 37, 59, 81, 103, 125, 147, 169, 7, 17, 27, 37, 47, 57, 67, 77 },
+ { 31, 77, 123, 169, 215, 261, 307, 353, 15, 37, 59, 81, 103, 125, 147, 169 },
+ { 63, 157, 251, 345, 439, 533, 627, 721, 31, 77, 123, 169, 215, 261, 307,
+ 353 },
+ { 127, 317, 507, 697, 887, 1077, 1267, 1457, 63, 157, 251, 345, 439, 533,
+ 627, 721 },
+ { 255, 637, 1019, 1401, 1783, 2165, 2547, 2929, 127, 317, 507, 697, 887,
+ 1077, 1267, 1457 },
+ { 511, 1277, 2043, 2809, 3575, 4341, 5107, 5873, 255, 637, 1019, 1401, 1783,
+ 2165, 2547, 2929 },
+ { 1023, 2557, 4091, 5625, 7159, 8693, 10227, 11761, 511, 1277, 2043, 2809,
+ 3575, 4341, 5107, 5873 }
+};
+
+int
+main (void)
+{
+ check_vect ();
+ for (unsigned n = 0; n < 10; ++n)
+ {
+ V out[2];
+ f (out, n);
+ if (__builtin_memcmp (out, expected[n], sizeof (out)))
+ __builtin_abort ();
+ }
+
+ return 0;
+}
diff --git
a/gcc/testsuite/gcc.target/aarch64/vect-revectorize-sra-unsupported-vector-1.c
b/gcc/testsuite/gcc.target/aarch64/vect-revectorize-sra-unsupported-vector-1.c
new file mode 100644
index
0000000000000000000000000000000000000000..8369eaf68b622982fc1540e3f60fd8284b59d37f
--- /dev/null
+++
b/gcc/testsuite/gcc.target/aarch64/vect-revectorize-sra-unsupported-vector-1.c
@@ -0,0 +1,22 @@
+/* Check lowering of a loop-carried 256-bit vector on AdvSIMD. */
+/* { dg-do compile } */
+/* { dg-options "-O3 -march=armv8-a" } */
+
+typedef float __attribute__ ((vector_size (32))) TYPE;
+
+void
+foo (TYPE *a, TYPE *b, unsigned n)
+{
+ TYPE x = *a;
+ TYPE y = *b;
+ for (int i = 0; i < n; i++)
+ {
+ y = x;
+ x = x * 2.0;
+ }
+
+ *a = x;
+ *b = y;
+}
+
+/* { dg-final { scan-assembler-times {\tmov\tv[0-9]+\.16b, v[0-9]+\.16b} 2 } }
*/
diff --git a/gcc/tree-vect-loop-manip.cc b/gcc/tree-vect-loop-manip.cc
index
51c8b42deff08be6951534941fd6afcb0076b52f..5764744c233cb59411716c59cfeaed2977ad9ffc
100644
--- a/gcc/tree-vect-loop-manip.cc
+++ b/gcc/tree-vect-loop-manip.cc
@@ -2284,7 +2284,10 @@ iv_phi_p (stmt_vec_info stmt_info)
return false;
if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def
- || STMT_VINFO_DEF_TYPE (stmt_info) == vect_double_reduction_def)
+ || STMT_VINFO_DEF_TYPE (stmt_info) == vect_double_reduction_def
+ /* Internal PHIs include whole vectors and scalar PHI cycles. They
+ have no induction step to advance after vectorization. */
+ || STMT_VINFO_DEF_TYPE (stmt_info) == vect_internal_def)
return false;
return true;
@@ -2407,12 +2410,13 @@ vect_can_advance_ivs_p (loop_vec_info loop_vinfo)
/* Skip virtual phi's. The data dependences that are associated with
virtual defs/uses (i.e., memory accesses) are analyzed elsewhere.
- Skip reduction phis. */
+ Skip reduction and internal cycle phis. */
if (!iv_phi_p (phi_info))
{
if (dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location,
- "reduc or virtual phi. skip.\n");
+ "non-induction phi. skip.\n");
+
continue;
}
@@ -2545,12 +2549,13 @@ vect_update_ivs_after_vectorizer (loop_vec_info
loop_vinfo,
"vect_update_ivs_after_vectorizer: phi: %G",
(gimple *) phi);
- /* Skip reduction and virtual phis. */
+ /* Skip reduction, internal cycle and virtual phis. */
if (!iv_phi_p (phi_info))
{
if (dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location,
- "reduc or virtual phi. skip.\n");
+ "non-induction phi. skip.\n");
+
continue;
}
diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc
index
fb28596006a16b558ac1779345b6f7457cdf8438..a6e6d3ab4a4813c0a70946973d3a3ee938c1489c
100644
--- a/gcc/tree-vect-loop.cc
+++ b/gcc/tree-vect-loop.cc
@@ -485,6 +485,46 @@ vect_analyze_scalar_cycles_1 (loop_vec_info loop_vinfo,
class loop *loop)
}
else if (vect_phi_first_order_recurrence_p (loop_vinfo, loop, phi))
STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_first_order_recurrence;
+ else if (VECTOR_TYPE_P (TREE_TYPE (def)))
+ {
+ tree type = TREE_TYPE (def);
+ unsigned HOST_WIDE_INT nelts;
+ if (TYPE_VECTOR_SUBPARTS (type).is_constant (&nelts)
+ && vect_emulated_vector_p (type)
+ && get_vectype_for_scalar_type (loop_vinfo, type,
+ nelts))
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "Detected splittable vector PHI.\n");
+
+ STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_internal_def;
+ }
+ else if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "Unknown def-use cycle pattern.\n");
+ }
+ else if (gimple_bb (phi) == loop->header)
+ {
+ edge latch_e = loop_latch_edge (loop);
+ tree latch_arg = PHI_ARG_DEF_FROM_EDGE (phi, latch_e);
+ gimple *def_stmt
+ = (TREE_CODE (latch_arg) == SSA_NAME
+ ? SSA_NAME_DEF_STMT (latch_arg) : NULL);
+ if (def_stmt
+ && gimple_code (def_stmt) == GIMPLE_PHI
+ && gimple_bb (def_stmt) == loop->header)
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "Detected splittable scalar PHI cycle.\n");
+
+ STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_internal_def;
+ }
+ else if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "Unknown def-use cycle pattern.\n");
+ }
else
if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -1429,6 +1469,23 @@ vect_compute_single_scalar_iteration_cost (loop_vec_info
loop_vinfo)
gimple_stmt_iterator si;
basic_block bb = bbs[i];
+ /* Match the PHI costs charged by vectorizable_phi. An emulated
+ input PHI represents one PHI for each piece after vector lowering. */
+ for (auto gsi = gsi_start_phis (bb); !gsi_end_p (gsi); gsi_next (&gsi))
+ {
+ gphi *phi = gsi.phi ();
+ stmt_vec_info stmt_info = loop_vinfo->lookup_stmt (phi);
+ tree type = TREE_TYPE (gimple_phi_result (phi));
+ if (vect_can_revectorize_type_p (type)
+ && vect_emulated_vector_p (type)
+ && gimple_phi_num_args (phi) > 1
+ && (STMT_VINFO_RELEVANT_P (stmt_info)
+ || STMT_VINFO_LIVE_P (stmt_info)))
+ record_stmt_cost (&LOOP_VINFO_SCALAR_ITERATION_COST (loop_vinfo),
+ vect_scalar_stmt_count (phi),
+ scalar_stmt, stmt_info, 0, vect_body);
+ }
+
for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
{
gimple *stmt = gsi_stmt (si);
@@ -8720,7 +8777,7 @@ vect_transform_lc_phi (loop_vec_info loop_vinfo,
/* Vectorizes PHIs. */
bool
-vectorizable_phi (bb_vec_info vinfo,
+vectorizable_phi (vec_info *vinfo,
stmt_vec_info stmt_info,
slp_tree slp_node, stmt_vector_for_cost *cost_vec)
{
@@ -8730,6 +8787,14 @@ vectorizable_phi (bb_vec_info vinfo,
if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
return false;
+ /* At VF 1, a PHI cycle such as A <- B <- C <- A still advances once
+ per iteration when its values are packed into vector lanes. A larger
+ VF would need to advance the cycle by VF steps, which this generic
+ PHI transformation does not implement. */
+ if (loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo))
+ if (!known_eq (LOOP_VINFO_VECT_FACTOR (loop_vinfo), 1U))
+ return false;
+
tree vectype = SLP_TREE_VECTYPE (slp_node);
if (cost_vec) /* transformation not required. */
@@ -10265,8 +10330,13 @@ vectorizable_live_operation_1 (loop_vec_info
loop_vinfo, basic_block exit_bb,
gimple_seq stmts = NULL;
tree new_tree;
+ if (VECTOR_TYPE_P (lhs_type))
+ new_tree
+ = gimple_build (&stmts, BIT_FIELD_REF, lhs_type, vec_lhs_phi,
+ TYPE_SIZE (lhs_type), bitstart);
+
/* If bitstart is 0 then we can use a BIT_FIELD_REF */
- if (integer_zerop (bitstart))
+ else if (integer_zerop (bitstart))
{
tree scalar_res = gimple_build (&stmts, BIT_FIELD_REF, TREE_TYPE
(vectype),
vec_lhs_phi, bitsize, bitstart);
@@ -10457,6 +10527,27 @@ vectorizable_live_operation (vec_info *vinfo,
stmt_vec_info stmt_info,
return false;
}
+ /* Use the lhs of the original scalar statement. */
+ gimple *stmt = vect_orig_stmt (stmt_info)->stmt;
+ if (dump_enabled_p () && vec_stmt_p)
+ dump_printf_loc (MSG_NOTE, vect_location, "extracting lane for live "
+ "stmt %G", stmt);
+
+ lhs = gimple_get_lhs (stmt);
+ lhs_type = TREE_TYPE (lhs);
+
+ /* Each source vector occupies consecutive lanes. Reconstruct it once,
+ including when it is not the first vector packed into the node. */
+ if (VECTOR_TYPE_P (lhs_type)
+ && slp_index % TYPE_VECTOR_SUBPARTS (lhs_type).to_constant () != 0)
+ return true;
+
+ /* A whole-vector live-out needs the final value of every source lane.
+ The scalar extraction path does not handle early exits for this case. */
+ if (VECTOR_TYPE_P (lhs_type) && loop_vinfo
+ && LOOP_VINFO_EARLY_BREAKS (loop_vinfo))
+ return false;
+
if (!vec_stmt_p)
{
/* No transformation required. */
@@ -10509,15 +10600,6 @@ vectorizable_live_operation (vec_info *vinfo,
stmt_vec_info stmt_info,
return true;
}
- /* Use the lhs of the original scalar statement. */
- gimple *stmt = vect_orig_stmt (stmt_info)->stmt;
- if (dump_enabled_p ())
- dump_printf_loc (MSG_NOTE, vect_location, "extracting lane for live "
- "stmt %G", stmt);
-
- lhs = gimple_get_lhs (stmt);
- lhs_type = TREE_TYPE (lhs);
-
bitsize = vector_element_bits_tree (vectype);
/* Get the vectorized lhs of STMT and the lane to use (counted in bits). */
@@ -10532,6 +10614,53 @@ vectorizable_live_operation (vec_info *vinfo,
stmt_vec_info stmt_info,
/* In case we need to early break vectorize also get the first stmt. */
tree vec_lhs0 = SLP_TREE_VEC_DEFS (slp_node)[0];
+ /* Assemble split source vectors in the ordinary IL. The existing
+ live-out path then needs only one value, and cleanup can forward its
+ pieces through the exit PHI. */
+ if (VECTOR_TYPE_P (lhs_type)
+ && known_lt (nunits, TYPE_VECTOR_SUBPARTS (lhs_type)))
+ {
+ gimple_seq stmts = NULL;
+ gimple *last_def = nullptr;
+ unsigned nparts
+ = exact_div (TYPE_VECTOR_SUBPARTS (lhs_type), nunits).to_constant ();
+ vec<constructor_elt, va_gc> *v = NULL;
+ vec_alloc (v, nparts);
+ auto defs = SLP_TREE_VEC_DEFS (slp_node);
+ for (unsigned i = vec_entry; i < vec_entry + nparts; ++i)
+ {
+ CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, defs[i]);
+ if (TREE_CODE (defs[i]) != SSA_NAME
+ || SSA_NAME_IS_DEFAULT_DEF (defs[i]))
+ continue;
+
+ /* Permutation optimization can reorder the pieces. Place the
+ constructor after the last definition, rather than the last
+ lane. */
+ gimple *def = SSA_NAME_DEF_STMT (defs[i]);
+ if (!last_def || vect_stmt_dominates_stmt_p (last_def, def))
+ last_def = def;
+ }
+
+ vec_lhs
+ = force_gimple_operand (build_constructor (lhs_type, v), &stmts, true,
+ NULL_TREE);
+ if (!last_def)
+ vinfo->insert_seq_on_entry (stmt_info, stmts);
+ else if (is_a <gphi *> (last_def))
+ {
+ auto si = gsi_after_labels (gimple_bb (last_def));
+ gsi_insert_seq_before (&si, stmts, GSI_SAME_STMT);
+ }
+ else
+ {
+ auto si = gsi_for_stmt (last_def);
+ gsi_insert_seq_after (&si, stmts, GSI_SAME_STMT);
+ }
+
+ vec_index = 0;
+ }
+
/* Get entry to use. */
tree bitstart = bitsize_int (vec_index);
bitstart = int_const_binop (MULT_EXPR, bitsize, bitstart);
@@ -10603,19 +10732,33 @@ vectorizable_live_operation (vec_info *vinfo,
stmt_vec_info stmt_info,
}
else
{
- /* For basic-block vectorization simply insert the lane-extraction. */
- tree bftype = TREE_TYPE (vectype);
- if (VECTOR_BOOLEAN_TYPE_P (vectype))
- bftype = build_nonstandard_integer_type (tree_to_uhwi (bitsize), 1);
- tree new_tree = build3 (BIT_FIELD_REF, bftype,
- vec_lhs, bitsize, bitstart);
gimple_seq stmts = NULL;
- new_tree = force_gimple_operand (fold_convert (lhs_type, new_tree),
- &stmts, true, NULL_TREE);
+ tree new_tree;
+ if (VECTOR_TYPE_P (lhs_type))
+ new_tree
+ = gimple_build (&stmts, BIT_FIELD_REF, lhs_type, vec_lhs,
+ TYPE_SIZE (lhs_type), bitstart);
+ else
+ {
+ /* Extract the live lane for basic-block vectorization. */
+ tree bftype = TREE_TYPE (vectype);
+ if (VECTOR_BOOLEAN_TYPE_P (vectype))
+ bftype = build_nonstandard_integer_type (tree_to_uhwi (bitsize), 1);
+
+ new_tree
+ = build3 (BIT_FIELD_REF, bftype, vec_lhs, bitsize, bitstart);
+ new_tree
+ = force_gimple_operand (fold_convert (lhs_type, new_tree), &stmts,
+ true, NULL_TREE);
+ }
+
if (TREE_CODE (new_tree) == SSA_NAME
&& SSA_NAME_OCCURS_IN_ABNORMAL_PHI (lhs))
SSA_NAME_OCCURS_IN_ABNORMAL_PHI (new_tree) = 1;
- gimple *vec_stmt = SSA_NAME_DEF_STMT (vec_lhs);
+
+ gimple *vec_stmt
+ = TREE_CODE (vec_lhs) == SSA_NAME
+ ? SSA_NAME_DEF_STMT (vec_lhs) : nullptr;
if (TREE_CODE (vec_lhs) != SSA_NAME
|| SSA_NAME_IS_DEFAULT_DEF (vec_lhs)
|| (vinfo->bbs[0] != gimple_bb (vec_stmt)
diff --git a/gcc/tree-vect-slp.cc b/gcc/tree-vect-slp.cc
index
4990852cbe6795accb0b6d273bff489938b38447..7723bc1b681aee6fe80bc8aeb57a11c2a12ee081
100644
--- a/gcc/tree-vect-slp.cc
+++ b/gcc/tree-vect-slp.cc
@@ -52,6 +52,7 @@ along with GCC; see the file COPYING3. If not see
#include "alloc-pool.h"
#include "sreal.h"
#include "predict.h"
+#include "ordered-hash-map.h"
#define REDUC_GROUP_FIRST_ELEMENT(S) \
(gcc_checking_assert (!(S)->dr_aux.dr), (S)->first_element)
@@ -6049,6 +6050,112 @@ vect_lower_load_permutations (loop_vec_info loop_vinfo,
/* Check if there are stmts in the loop can be vectorized using SLP. Build SLP
trees of packed scalar stmts if SLP is possible. */
+/* If LC_PHI is a loop-closed PHI for exit edge E and its incoming value is
+ an only-live scalar statement that SLP can vectorize, return that statement.
+ Otherwise return null. */
+
+static stmt_vec_info
+vect_get_slp_live_lc_phi_def (loop_vec_info loop_vinfo, gphi *lc_phi, edge e)
+{
+ tree def = gimple_phi_arg_def_from_edge (lc_phi, e);
+ stmt_vec_info stmt_info;
+ if (TREE_CODE (def) == SSA_NAME
+ && !virtual_operand_p (def)
+ && (stmt_info = loop_vinfo->lookup_def (def))
+ && ((stmt_info = vect_stmt_to_vectorize (stmt_info)), true)
+ && STMT_VINFO_RELEVANT (stmt_info) == vect_used_only_live
+ && STMT_VINFO_LIVE_P (stmt_info)
+ && !VECTORIZABLE_CYCLE_DEF (STMT_VINFO_DEF_TYPE (stmt_info))
+ && STMT_VINFO_REDUC_IDX (stmt_info) == -1)
+ return stmt_info;
+
+ return NULL;
+}
+
+/* Group scalar live-outs that rebuild a source vector in E's destination.
+ Vector lowering stores each LC PHI into a BIT_FIELD_REF of the original
+ vector. Collect these stores once, indexed by destination and lane. */
+
+static void
+vect_analyze_slp_live_lanes (loop_vec_info loop_vinfo,
+ scalar_stmts_to_slp_tree_map_t *bst_map, edge e,
+ unsigned max_tree_size, unsigned *limit,
+ bool force_single_lane,
+ hash_set<stmt_vec_info> &grouped_live_stmts)
+{
+ ordered_hash_map<tree, vec<stmt_vec_info>> live_vectors;
+ for (auto gsi = gsi_after_labels (e->dest); !gsi_end_p (gsi); gsi_next
(&gsi))
+ {
+ auto assign = dyn_cast <gassign *> (gsi_stmt (gsi));
+ if (!assign)
+ continue;
+
+ tree lhs = gimple_assign_lhs (assign);
+ if (TREE_CODE (lhs) != BIT_FIELD_REF)
+ continue;
+
+ tree obj = TREE_OPERAND (lhs, 0);
+ tree type = TREE_TYPE (obj);
+ if (!vect_can_revectorize_type_p (type))
+ continue;
+
+ bool existed;
+ auto &stmts = live_vectors.get_or_insert (obj, &existed);
+ if (!existed)
+ stmts.safe_grow_cleared (TYPE_VECTOR_SUBPARTS (type).to_constant ());
+
+ /* An empty entry marks an object that cannot be grouped. */
+ if (stmts.is_empty ())
+ continue;
+
+ unsigned elt_bits = vector_element_bits (type);
+ unsigned HOST_WIDE_INT lane;
+ if (!known_eq (bit_field_size (lhs), elt_bits)
+ || !constant_multiple_p (bit_field_offset (lhs), elt_bits, &lane)
+ || lane >= stmts.length ())
+ {
+ stmts.release ();
+ continue;
+ }
+
+ tree rhs = gimple_assign_rhs1 (assign);
+ auto phi
+ = TREE_CODE (rhs) == SSA_NAME
+ ? dyn_cast <gphi *> (SSA_NAME_DEF_STMT (rhs)) : nullptr;
+ stmt_vec_info def
+ = phi && gimple_bb (phi) == e->dest
+ ? vect_get_slp_live_lc_phi_def (loop_vinfo, phi, e) : NULL;
+ if (!def || stmts[lane])
+ stmts.release ();
+ else
+ stmts[lane] = def;
+ }
+
+ for (auto entry : live_vectors)
+ {
+ auto stmts = entry.second;
+ bool complete = !stmts.is_empty ();
+ for (auto stmt : stmts)
+ if (!stmt || grouped_live_stmts.contains (stmt))
+ complete = false;
+
+ if (complete)
+ {
+ auto lane_stmts = stmts.copy ();
+ vec<stmt_vec_info> roots = vNULL;
+ vec<tree> remain = vNULL;
+ if (vect_build_slp_instance (loop_vinfo, slp_inst_kind_reduc_group,
+ lane_stmts, roots, remain,
+ max_tree_size, limit, bst_map,
+ force_single_lane))
+ for (auto stmt : stmts)
+ grouped_live_stmts.add (stmt);
+ }
+
+ stmts.release ();
+ }
+}
+
opt_result
vect_analyze_slp (vec_info *vinfo, unsigned max_tree_size,
bool force_single_lane)
@@ -6154,28 +6261,38 @@ vect_analyze_slp (vec_info *vinfo, unsigned
max_tree_size,
return opt_result::failure_at (vect_location, "SLP build failed.\n");
}
+ hash_set<stmt_vec_info> grouped_live_stmts;
+
+ for (edge e : get_loop_exit_edges (LOOP_VINFO_LOOP (loop_vinfo)))
+ vect_analyze_slp_live_lanes (loop_vinfo, bst_map, e, max_tree_size,
+ &limit, force_single_lane,
+ grouped_live_stmts);
+
/* Make sure to vectorize only-live stmts, usually inductions. */
for (edge e : get_loop_exit_edges (LOOP_VINFO_LOOP (loop_vinfo)))
for (auto gsi = gsi_start_phis (e->dest); !gsi_end_p (gsi);
gsi_next (&gsi))
{
gphi *lc_phi = *gsi;
- tree def = gimple_phi_arg_def_from_edge (lc_phi, e);
- stmt_vec_info stmt_info;
- if (TREE_CODE (def) == SSA_NAME
- && !virtual_operand_p (def)
- && (stmt_info = loop_vinfo->lookup_def (def))
- && ((stmt_info = vect_stmt_to_vectorize (stmt_info)), true)
- && STMT_VINFO_RELEVANT (stmt_info) == vect_used_only_live
- && STMT_VINFO_LIVE_P (stmt_info)
- && !VECTORIZABLE_CYCLE_DEF (STMT_VINFO_DEF_TYPE (stmt_info))
- && STMT_VINFO_REDUC_IDX (stmt_info) == -1)
+ stmt_vec_info stmt_info
+ = vect_get_slp_live_lc_phi_def (loop_vinfo, lc_phi, e);
+ if (stmt_info && !grouped_live_stmts.contains (stmt_info))
{
vec<stmt_vec_info> stmts;
vec<stmt_vec_info> roots = vNULL;
vec<tree> remain = vNULL;
- stmts.create (1);
- stmts.quick_push (vect_stmt_to_vectorize (stmt_info));
+ unsigned HOST_WIDE_INT nelts = 1;
+ tree lhs = gimple_get_lhs (stmt_info->stmt);
+ if (lhs && vect_can_revectorize_type_p (TREE_TYPE (lhs)))
+ {
+ if (!TYPE_VECTOR_SUBPARTS (TREE_TYPE (lhs))
+ .is_constant (&nelts))
+ continue;
+ }
+
+ stmts.create (nelts);
+ for (unsigned HOST_WIDE_INT j = 0; j < nelts; ++j)
+ stmts.quick_push (vect_stmt_to_vectorize (stmt_info));
if (! vect_build_slp_instance (vinfo,
slp_inst_kind_reduc_group,
stmts, roots, remain,
diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
index
4a459e53ccdb68b0f327309febc35133e680bbd8..6f0201e9efd7c5909adf6cfcd0ab031beecd6497
100644
--- a/gcc/tree-vect-stmts.cc
+++ b/gcc/tree-vect-stmts.cc
@@ -13198,8 +13198,7 @@ vect_analyze_stmt (vec_info *vinfo,
|| vectorizable_shift (vinfo, stmt_info, NULL, node, cost_vec)
|| vectorizable_condition (vinfo, stmt_info, NULL, node, cost_vec)
|| vectorizable_comparison (vinfo, stmt_info, NULL, node, cost_vec)
- || (bb_vinfo
- && vectorizable_phi (bb_vinfo, stmt_info, node, cost_vec))
+ || vectorizable_phi (vinfo, stmt_info, node, cost_vec)
|| (is_a <loop_vec_info> (vinfo)
&& (vectorizable_lane_reducing (as_a <loop_vec_info> (vinfo),
stmt_info, node, cost_vec)
@@ -13342,8 +13341,7 @@ vect_transform_stmt (vec_info *vinfo,
break;
case phi_info_type:
- done = vectorizable_phi (as_a <bb_vec_info> (vinfo),
- stmt_info, slp_node, NULL);
+ done = vectorizable_phi (vinfo, stmt_info, slp_node, NULL);
gcc_assert (done);
break;
diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
index
7be54e3343daa1ca55b9bcf79bc363902b374445..b97140b65bf407dbbbaafeacc0609a8c9bb5e58e
100644
--- a/gcc/tree-vectorizer.h
+++ b/gcc/tree-vectorizer.h
@@ -2756,7 +2756,7 @@ extern bool vect_transform_cycle_phi (loop_vec_info,
stmt_vec_info,
slp_tree, slp_instance);
extern bool vectorizable_lc_phi (loop_vec_info, stmt_vec_info, slp_tree);
extern bool vect_transform_lc_phi (loop_vec_info, stmt_vec_info, slp_tree);
-extern bool vectorizable_phi (bb_vec_info, stmt_vec_info, slp_tree,
+extern bool vectorizable_phi (vec_info *, stmt_vec_info, slp_tree,
stmt_vector_for_cost *);
extern bool vectorizable_recurr (loop_vec_info, stmt_vec_info,
slp_tree, stmt_vector_for_cost *);
