Vector constants and invariants need to supply consecutive element lanes when
we build their SLP operands. Repeating the vector statement must not blindly
broadcast its first element, since the other elements can have different values.
e.g.
typedef unsigned V8 __attribute__((vector_size(32)));
void invariant (V8 *restrict out, const V8 *a, V8 bias, unsigned n)
{
for (unsigned i = 0; i < n; ++i)
out[i] = (a[i] + bias) ^ (V8) {1,3,5,7,9,11,13,15};
}
To do this vect_create_constant_vectors tracks the source element position. It
takes elements from VECTOR_CST_ELT for constants and BIT_FIELD_REF for
invariants. Where the source already contains a complete native vector piece,
we can use that piece directly.
For this example the Adv. SIMD operands and results are built as follows:
bias_lo = BIT_FIELD_REF <bias, 128, 0>;
bias_hi = BIT_FIELD_REF <bias, 128, 128>;
sum_lo = load_lo + bias_lo;
sum_hi = load_hi + bias_hi;
result_lo = sum_lo ^ {1, 3, 5, 7};
result_hi = sum_hi ^ {9, 11, 13, 15};
SLP layout selection must preserve this element order. We also check whether
the source group can be split or repeated at the requested vector width,
including whether a scalable repeating pattern can be constructed. Costing
counts the native pieces needed to build the operand.
Shifts need the same distinction between a vector operand and a scalar
immediate:
out[i] = (a[i] << s[i]) ^ (a[i] >> 2);
Here s[i] supplies a count per element, while 2 is shared by every element.
vectorizable_shift uses the source element type for its support checks and
keeps those two operand forms separate.
This adds operand construction for the analysis introduced in patch 1. With
the later memory support, the example uses two four-element operations per V8
on Adv. SIMD, as ordinary vector lowering does.
Bootstrapped Regtested on aarch64-none-linux-gnu,
arm-none-linux-gnueabihf, x86_64-pc-linux-gnu
-m32, -m64 and no issues.
Any feedback?
Thanks,
Tamar
gcc/ChangeLog:
PR tree-optimization/119187
* tree-vectorizer.cc (vect_native_vector_type): New.
* tree-vectorizer.h (vect_native_vector_type): Declare.
* tree-vect-slp.cc (vect_slp_tree_scalar_ops_p,
vect_slp_tree_vector_ops_p): New.
(vect_slp_tree_uniform_p): Clarify whole-vector operand semantics.
(vect_build_slp_tree_2): Distinguish vector operands from scalar splats.
(vect_slp_gather_extern_scalar_stmts): Ignore null operands.
(vect_bb_slp_mark_stmts_vectorized): Likewise.
(vect_optimize_slp_pass::create_partitions): Preserve vector input lane
order and assign element-based vector types.
(vect_optimize_slp_pass::is_compatible_layout): Check vector type.
(vect_optimize_slp_pass::get_result_with_layout): Only reorder scalar
operands directly.
(vect_get_num_copies_for_invariant): Check complete source layouts.
(vect_prologue_cost_for_slp): Cost vector operand construction and reuse
of native pieces.
(vect_bb_slp_scalar_cost): Ignore null operands.
(vect_create_constant_vectors): Extract or construct source lanes.
* tree-vect-stmts.cc (vectorizable_shift): Use vector element types and
recognize vector shift amounts.
* tree-vect-slp.cc (vect_slp_analyze_node_operations): Build supported
vector operand pieces for external nodes.
gcc/testsuite/ChangeLog:
PR tree-optimization/119187
* gcc.dg/vect/vect-revectorize-invariant.c: New test.
* gcc.dg/vect/vect-revectorize-shift.c: New test.
---
--
diff --git a/gcc/testsuite/gcc.dg/vect/vect-revectorize-invariant.c
b/gcc/testsuite/gcc.dg/vect/vect-revectorize-invariant.c
new file mode 100644
index
0000000000000000000000000000000000000000..cfce46cddf0e014175c5896047b8ea5b53e0febc
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-revectorize-invariant.c
@@ -0,0 +1,100 @@
+/* { dg-do run } */
+/* { dg-require-effective-target vect_int } */
+/* { dg-additional-options "-O3 -fno-vect-cost-model -Wno-psabi" } */
+
+#include "tree-vect.h"
+
+typedef unsigned V __attribute__ ((vector_size (32)));
+
+/* Exercise both default definitions and definitions outside the loop. */
+__attribute__ ((noipa)) void
+parameter (V *out, const V *a, V bias, unsigned n)
+{
+ for (unsigned i = 0; i < n; ++i)
+ out[i] = (a[i] + bias) ^ (V) { 1, 3, 5, 7, 9, 11, 13, 15 };
+}
+
+__attribute__ ((noipa)) void
+loaded (V *out, const V *a, const V *b, unsigned n, int which)
+{
+ V bias;
+ if (which)
+ bias = b[0] + (V) { 1, 3, 5, 7, 9, 11, 13, 15 };
+ else
+ bias = b[1] ^ (V) { 2, 4, 6, 8, 10, 12, 14, 16 };
+
+ for (unsigned i = 0; i < n; ++i)
+ out[i] = a[i] + bias;
+}
+
+static const V a[] = {
+ { 0, 13, 26, 39, 52, 65, 78, 91 },
+ { 101, 114, 127, 140, 153, 166, 179, 192 },
+ { 202, 215, 228, 241, 254, 267, 280, 293 },
+ { 303, 316, 329, 342, 355, 368, 381, 394 },
+ { 404, 417, 430, 443, 456, 469, 482, 495 },
+ { 505, 518, 531, 544, 557, 570, 583, 596 },
+ { 606, 619, 632, 645, 658, 671, 684, 697 },
+ { 707, 720, 733, 746, 759, 772, 785, 798 },
+ { 808, 821, 834, 847, 860, 873, 886, 899 }
+};
+static const V b[] = {
+ { 5, 12, 19, 26, 33, 40, 47, 54 },
+ { 2, 21, 40, 59, 78, 97, 116, 135 }
+};
+static const V expected_parameter[] = {
+ { 4, 26, 40, 70, 92, 98, 112, 158 },
+ { 107, 125, 151, 161, 179, 197, 239, 249 },
+ { 206, 224, 242, 268, 278, 312, 330, 340 },
+ { 309, 331, 345, 375, 397, 403, 417, 463 },
+ { 408, 430, 452, 466, 480, 502, 540, 554 },
+ { 511, 529, 547, 573, 583, 617, 635, 645 },
+ { 610, 628, 654, 664, 698, 716, 726, 736 },
+ { 713, 735, 757, 771, 785, 807, 845, 859 },
+ { 812, 834, 848, 878, 884, 922, 936, 950 }
+};
+static const V expected_xor[] = {
+ { 0, 30, 72, 90, 120, 174, 200, 242 },
+ { 101, 131, 173, 191, 221, 275, 301, 343 },
+ { 202, 232, 274, 292, 322, 376, 402, 444 },
+ { 303, 333, 375, 393, 423, 477, 503, 545 },
+ { 404, 434, 476, 494, 524, 578, 604, 646 },
+ { 505, 535, 577, 595, 625, 679, 705, 747 },
+ { 606, 636, 678, 696, 726, 780, 806, 848 },
+ { 707, 737, 779, 797, 827, 881, 907, 949 },
+ { 808, 838, 880, 898, 928, 982, 1008, 1050 }
+};
+static const V expected_add[] = {
+ { 6, 28, 50, 72, 94, 116, 138, 160 },
+ { 107, 129, 151, 173, 195, 217, 239, 261 },
+ { 208, 230, 252, 274, 296, 318, 340, 362 },
+ { 309, 331, 353, 375, 397, 419, 441, 463 },
+ { 410, 432, 454, 476, 498, 520, 542, 564 },
+ { 511, 533, 555, 577, 599, 621, 643, 665 },
+ { 612, 634, 656, 678, 700, 722, 744, 766 },
+ { 713, 735, 757, 779, 801, 823, 845, 867 },
+ { 814, 836, 858, 880, 902, 924, 946, 968 }
+};
+
+int
+main (void)
+{
+ check_vect ();
+ V out[9];
+ for (unsigned n = 0; n <= 9; ++n)
+ {
+ parameter (out, a, b[0], n);
+ if (__builtin_memcmp (out, expected_parameter, n * sizeof (V)))
+ __builtin_abort ();
+
+ loaded (out, a, b, n, 0);
+ if (__builtin_memcmp (out, expected_xor, n * sizeof (V)))
+ __builtin_abort ();
+
+ loaded (out, a, b, n, 1);
+ if (__builtin_memcmp (out, expected_add, n * sizeof (V)))
+ __builtin_abort ();
+ }
+
+ return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/vect/vect-revectorize-shift.c
b/gcc/testsuite/gcc.dg/vect/vect-revectorize-shift.c
new file mode 100644
index
0000000000000000000000000000000000000000..65f9b29baa539a130e87713cf76289856b3efd82
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-revectorize-shift.c
@@ -0,0 +1,64 @@
+/* { dg-do run } */
+/* { dg-require-effective-target vect_int } */
+/* { dg-additional-options "-O3 -fno-vect-cost-model" } */
+
+#include "tree-vect.h"
+
+typedef unsigned V __attribute__ ((vector_size (32)));
+
+__attribute__ ((noipa)) void
+shift_vectors (V *restrict out, const V *restrict in,
+ const V *restrict amounts, unsigned n)
+{
+ for (unsigned i = 0; i < n; ++i)
+ out[i] = (in[i] << amounts[i]) ^ (in[i] >> 2);
+}
+
+static const V in[] = {
+ { 1, 8, 15, 22, 29, 36, 43, 50 },
+ { 14, 21, 28, 35, 42, 49, 56, 63 },
+ { 27, 34, 41, 48, 55, 62, 69, 76 },
+ { 40, 47, 54, 61, 68, 75, 82, 89 },
+ { 53, 60, 67, 74, 81, 88, 95, 102 },
+ { 66, 73, 80, 87, 94, 101, 108, 115 },
+ { 79, 86, 93, 100, 107, 114, 121, 128 },
+ { 92, 99, 106, 113, 120, 127, 134, 141 },
+ { 105, 112, 119, 126, 133, 140, 147, 154 }
+};
+static const V amounts[] = {
+ { 0, 1, 2, 3, 4, 5, 6, 7 },
+ { 1, 2, 3, 4, 5, 6, 7, 0 },
+ { 2, 3, 4, 5, 6, 7, 0, 1 },
+ { 3, 4, 5, 6, 7, 0, 1, 2 },
+ { 4, 5, 6, 7, 0, 1, 2, 3 },
+ { 5, 6, 7, 0, 1, 2, 3, 4 },
+ { 6, 7, 0, 1, 2, 3, 4, 5 },
+ { 7, 0, 1, 2, 3, 4, 5, 6 },
+ { 0, 1, 2, 3, 4, 5, 6, 7 }
+};
+static const V expected[] = {
+ { 1, 18, 63, 181, 471, 1161, 2762, 6412 },
+ { 31, 81, 231, 568, 1354, 3148, 7182, 48 },
+ { 106, 280, 666, 1548, 3533, 7951, 84, 139 },
+ { 330, 763, 1741, 3919, 8721, 89, 176, 370 },
+ { 861, 1935, 4304, 9490, 69, 166, 363, 809 },
+ { 2128, 4690, 10260, 66, 171, 397, 891, 1836 },
+ { 5075, 11029, 74, 209, 438, 908, 1934, 4128 },
+ { 11799, 123, 206, 472, 990, 2031, 4321, 9059 },
+ { 115, 252, 449, 1007, 2161, 4515, 9444, 19750 }
+};
+
+int
+main (void)
+{
+ check_vect ();
+ V out[9];
+ for (unsigned n = 0; n <= 9; ++n)
+ {
+ shift_vectors (out, in, amounts, n);
+ if (__builtin_memcmp (out, expected, n * sizeof (V)))
+ __builtin_abort ();
+ }
+
+ return 0;
+}
diff --git a/gcc/tree-vect-slp.cc b/gcc/tree-vect-slp.cc
index
e67a53f68f41f9a6605a0ceba6e81495c03c0c38..b76400b12f2bcc236682136f4b36efcee9cd580a
100644
--- a/gcc/tree-vect-slp.cc
+++ b/gcc/tree-vect-slp.cc
@@ -371,8 +371,30 @@ vect_contains_pattern_stmt_p (vec<stmt_vec_info> stmts)
return false;
}
-/* Return true when all lanes in the external or constant NODE have
- the same value. */
+/* Return true if NODE contains whole-vector operands, repeated once per
+ source element. */
+
+static bool
+vect_slp_tree_vector_ops_p (slp_tree node)
+{
+ return !SLP_TREE_SCALAR_OPS (node).is_empty ()
+ && VECTOR_TYPE_P (TREE_TYPE (SLP_TREE_SCALAR_OPS (node)[0]));
+}
+
+/* Return true if NODE builds vectors from scalar operands whose order can
+ be changed directly. Existing vector definitions and whole-vector
+ operands require a vector permutation instead. */
+
+static bool
+vect_slp_tree_scalar_ops_p (slp_tree node)
+{
+ return !SLP_TREE_SCALAR_OPS (node).is_empty ()
+ && SLP_TREE_VEC_DEFS (node).is_empty ()
+ && !vect_slp_tree_vector_ops_p (node);
+}
+
+/* Return true when all operands in the external or constant NODE have
+ the same value. Vector operands still have their implicit lane order. */
static bool
vect_slp_tree_uniform_p (slp_tree node)
@@ -3339,6 +3361,8 @@ fail:
;
else if (SLP_TREE_DEF_TYPE (child) == vect_internal_def)
all_uniform_p = false;
+ else if (!vect_slp_tree_scalar_ops_p (child))
+ all_uniform_p = false;
else if (!vect_slp_tree_uniform_p (child))
{
all_uniform_p = false;
@@ -3698,6 +3722,9 @@ vect_slp_gather_extern_scalar_stmts (vec_info *vinfo,
slp_tree node,
else
for (tree def : SLP_TREE_SCALAR_OPS (node))
{
+ if (!def)
+ continue;
+
stmt_vec_info def_stmt = vinfo->lookup_def (def);
if (def_stmt)
estmts.add (def_stmt);
@@ -3746,6 +3773,9 @@ vect_bb_slp_mark_stmts_vectorized (bb_vec_info vinfo)
for (unsigned i = 0; i < child_ops.num_slp_children; ++i)
{
tree op = child_ops.get_op_for_slp_child (stmt, i);
+ if (!op)
+ continue;
+
stmt_vec_info def = vinfo->lookup_def (op);
if (def
&& !scalar_stmts_in_externs.contains (def)
@@ -7006,9 +7036,19 @@ vect_optimize_slp_pass::create_partitions ()
permuting them. */
unsigned int node_i = m_partitioned_nodes[rpo_i];
auto &vertex = m_vertices[node_i];
+ if (!SLP_TREE_VECTYPE (vertex.node)
+ && vect_slp_tree_vector_ops_p (vertex.node))
+ {
+ /* Cost permutations in the input element type, which can differ
+ from the consumer's result type (for example a comparison). */
+ tree type = TREE_TYPE (SLP_TREE_SCALAR_OPS (vertex.node)[0]);
+ SLP_TREE_VECTYPE (vertex.node)
+ = get_vectype_for_scalar_type (m_vinfo, type, vertex.node);
+ }
+
if ((SLP_TREE_DEF_TYPE (vertex.node) == vect_external_def
- && !SLP_TREE_VEC_DEFS (vertex.node).exists ())
- || SLP_TREE_DEF_TYPE (vertex.node) == vect_constant_def)
+ || SLP_TREE_DEF_TYPE (vertex.node) == vect_constant_def)
+ && vect_slp_tree_scalar_ops_p (vertex.node))
vertex.partition = -1;
else
{
@@ -7101,6 +7141,9 @@ vect_optimize_slp_pass::is_compatible_layout (slp_tree
node,
if (layout_i == 0)
return true;
+ if (!SLP_TREE_VECTYPE (node))
+ return false;
+
if (SLP_TREE_LANES (node) != m_perms[layout_i].length ())
return false;
@@ -7980,10 +8023,9 @@ vect_optimize_slp_pass::get_result_with_layout (slp_tree
node,
if (result)
return result;
- if (SLP_TREE_DEF_TYPE (node) == vect_constant_def
- || (SLP_TREE_DEF_TYPE (node) == vect_external_def
- /* We can't permute vector defs in place. */
- && SLP_TREE_VEC_DEFS (node).is_empty ()))
+ if ((SLP_TREE_DEF_TYPE (node) == vect_constant_def
+ || SLP_TREE_DEF_TYPE (node) == vect_external_def)
+ && vect_slp_tree_scalar_ops_p (node))
{
/* If the vector is uniform or unchanged, there's nothing to do. */
if (to_layout_i == 0 || vect_slp_tree_uniform_p (node))
@@ -9113,6 +9155,28 @@ vect_get_num_copies_for_invariant (vec_info *vinfo,
slp_tree node,
vf *= SLP_TREE_LANES (node);
tree vectype = SLP_TREE_VECTYPE (node);
+ if (vect_slp_tree_vector_ops_p (node))
+ {
+ tree type = TREE_TYPE (SLP_TREE_SCALAR_OPS (node)[0]);
+ unsigned lanes = SLP_TREE_LANES (node);
+
+ /* Each source vector must fit wholly into the input group, and the
+ output must split or repeat that group without a partial vector. */
+ if (!TYPE_VECTOR_SUBPARTS (type).is_constant ()
+ || !multiple_p (lanes, TYPE_VECTOR_SUBPARTS (type))
+ || (!multiple_p (lanes, TYPE_VECTOR_SUBPARTS (vectype))
+ && !multiple_p (TYPE_VECTOR_SUBPARTS (vectype), lanes))
+ || !types_compatible_p (TREE_TYPE (type), TREE_TYPE (vectype)))
+ return false;
+
+ /* Scalable invariant vectors need a supported way to repeat the
+ fixed-length input group. Constants use vector constant encoding. */
+ if (SLP_TREE_DEF_TYPE (node) == vect_external_def
+ && !TYPE_VECTOR_SUBPARTS (vectype).is_constant ()
+ && !can_duplicate_and_interleave_p (vinfo, lanes, TREE_TYPE (type)))
+ return false;
+ }
+
uint64_t rem;
if (can_div_away_from_zero_p (vf, TYPE_VECTOR_SUBPARTS (vectype),
nvectors)
@@ -9165,13 +9229,29 @@ vect_prologue_cost_for_slp (slp_tree node, unsigned
nvectors,
Ideally we'd do something more global. */
for (unsigned int start : starts)
{
+ vect_scalar_ops_slice slice = { ops, start, nelt_limit };
vect_cost_for_stmt kind;
if (SLP_TREE_DEF_TYPE (node) == vect_constant_def)
kind = vector_load;
- else if (vect_scalar_ops_slice { ops, start, nelt_limit }.all_same_p ())
+ else if (VECTOR_TYPE_P (TREE_TYPE (slice.op (0))))
+ {
+ /* Reusing a native vector, or one native piece of a wider vector,
+ needs no instruction. Combining pieces needs a permutation. */
+ tree type = TREE_TYPE (slice.op (0));
+ tree piece_type = vect_native_vector_type (type);
+ if (slice.all_same_p ()
+ && (TYPE_MODE (type) == TYPE_MODE (vectype)
+ || (piece_type
+ && TYPE_MODE (piece_type) == TYPE_MODE (vectype))))
+ continue;
+
+ kind = vec_perm;
+ }
+ else if (slice.all_same_p ())
kind = scalar_to_vec;
else
kind = vec_construct;
+
record_stmt_cost (cost_vec, 1, kind, nullptr, node,
vectype, 0, vect_prologue);
}
@@ -9244,15 +9324,6 @@ vect_slp_analyze_node_operations (vec_info *vinfo,
slp_tree node,
if (res)
res = vect_slp_analyze_node_operations_1 (vinfo, node, node_instance,
cost_vec);
- /* If analysis failed we have to pop all recursive visited nodes
- plus ourselves. */
- if (!res)
- {
- while (visited_vec.length () >= visited_rec_start)
- visited_set.remove (visited_vec.pop ());
- cost_vec->truncate (cost_vec_rec_start);
- }
-
/* When the node can be vectorized cost invariant nodes it references.
This is not done in DFS order to allow the referring node
vectorizable_* calls to nail down the invariant nodes vector type
@@ -9299,10 +9370,23 @@ vect_slp_analyze_node_operations (vec_info *vinfo,
slp_tree node,
unsigned nvectors, excess_elts;
if (!vect_get_num_copies_for_invariant (vinfo, child, &nvectors,
&excess_elts))
- return false;
+ {
+ res = false;
+ break;
+ }
+
vect_prologue_cost_for_slp (child, nvectors, cost_vec);
}
+ /* If analysis failed we have to pop all recursive visited nodes
+ plus ourselves. */
+ if (!res)
+ {
+ while (visited_vec.length () >= visited_rec_start)
+ visited_set.remove (visited_vec.pop ());
+ cost_vec->truncate (cost_vec_rec_start);
+ }
+
/* If this node or any of its children can't be vectorized, try pruning
the tree here rather than felling the whole thing. */
if (!res && vect_slp_convert_to_external (vinfo, node, node_instance))
@@ -9897,6 +9981,9 @@ vect_bb_slp_scalar_cost (bb_vec_info vinfo,
for (unsigned i = 0; i < child_ops.num_slp_children; ++i)
{
tree op = child_ops.get_op_for_slp_child (stmt, i);
+ if (!op)
+ continue;
+
stmt_vec_info def = vinfo->lookup_def (op);
if (def && !visited.add (def))
worklist.safe_push (def);
@@ -11163,6 +11250,25 @@ vect_create_constant_vectors (vec_info *vinfo,
slp_tree op_node)
{
/* Create 'vect_ = {op0,op1,...,opn}'. */
tree orig_op = op;
+ unsigned source_lane = 0;
+ if (VECTOR_TYPE_P (TREE_TYPE (op)))
+ {
+ unsigned nelts = TYPE_VECTOR_SUBPARTS (TREE_TYPE
(op)).to_constant ();
+ tree elt_type = TREE_TYPE (TREE_TYPE (op));
+ source_lane = i % nelts;
+
+ /* Repeated entries denote successive elements of OP, rather
+ than a splat of the whole vector. */
+ if (TREE_CODE (op) == VECTOR_CST)
+ op = VECTOR_CST_ELT (op, source_lane);
+ else
+ op
+ = gimple_build (&ctor_seq, BIT_FIELD_REF, elt_type, op,
+ vector_element_bits_tree (TREE_TYPE (op)),
+ bitsize_int (source_lane
+ * vector_element_bits (TREE_TYPE
(op))));
+ }
+
if (number_of_places_left_in_vector == nunits)
uniform_elt = op;
else if (uniform_elt && operand_equal_p (uniform_elt, op))
@@ -11239,7 +11345,19 @@ vect_create_constant_vectors (vec_info *vinfo,
slp_tree op_node)
if (number_of_places_left_in_vector == 0)
{
auto type_nunits = TYPE_VECTOR_SUBPARTS (vector_type);
- if (uniform_elt)
+
+ /* For example, reuse V8SI elements 4..7 as one V4SI slice.
+ This also folds to ORIG_OP when the vector types match. */
+ if (VECTOR_TYPE_P (TREE_TYPE (orig_op))
+ && multiple_p (TYPE_VECTOR_SUBPARTS (TREE_TYPE (orig_op)),
+ type_nunits))
+ vec_cst
+ = gimple_build (&ctor_seq, BIT_FIELD_REF, vector_type,
+ orig_op, TYPE_SIZE (vector_type),
+ bitsize_int (source_lane
+ * vector_element_bits
+ (TREE_TYPE (orig_op))));
+ else if (uniform_elt)
vec_cst = gimple_build_vector_from_val (&ctor_seq, vector_type,
elts[0]);
else if (constant_p
@@ -11254,6 +11372,7 @@ vect_create_constant_vectors (vec_info *vinfo, slp_tree
op_node)
permute_results);
vec_cst = permute_results[number_of_vectors - j - 1];
}
+
if (!gimple_seq_empty_p (ctor_seq))
{
if (insert_after)
diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
index
ddbacb597c1dd8bb516baa59c6a46808cd74a3a2..5b5e0c7d2227cc805d15667f138925c778bd51dd
100644
--- a/gcc/tree-vect-stmts.cc
+++ b/gcc/tree-vect-stmts.cc
@@ -6297,7 +6297,11 @@ vectorizable_shift (vec_info *vinfo,
scalar_dest = gimple_assign_lhs (stmt);
vectype_out = SLP_TREE_VECTYPE (slp_node);
- if (!type_has_mode_precision_p (TREE_TYPE (scalar_dest)))
+ tree scalar_type = TREE_TYPE (scalar_dest);
+ if (VECTOR_TYPE_P (scalar_type))
+ scalar_type = TREE_TYPE (scalar_type);
+
+ if (!type_has_mode_precision_p (scalar_type))
{
if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -6347,10 +6351,11 @@ vectorizable_shift (vec_info *vinfo,
/* Determine whether the shift amount is a vector, or scalar. If the
shift/rotate amount is a vector, use the vector/vector shift optabs. */
- if ((dt[1] == vect_internal_def
- || dt[1] == vect_induction_def
- || dt[1] == vect_nested_cycle)
- && SLP_TREE_LANES (slp_node) == 1)
+ if (VECTOR_TYPE_P (TREE_TYPE (op1))
+ || ((dt[1] == vect_internal_def
+ || dt[1] == vect_induction_def
+ || dt[1] == vect_nested_cycle)
+ && SLP_TREE_LANES (slp_node) == 1))
scalar_shift_arg = false;
else if (dt[1] == vect_constant_def
|| dt[1] == vect_external_def
diff --git a/gcc/tree-vectorizer.cc b/gcc/tree-vectorizer.cc
index
496fb5830d2748bcc37785f542d86add5f1b9d18..a0b1e3e6ab613aad612f4ab7ac65b753ad194811
100644
--- a/gcc/tree-vectorizer.cc
+++ b/gcc/tree-vectorizer.cc
@@ -1234,6 +1234,24 @@ try_vectorize_loop (hash_table<simduid_to_vf>
*&simduid_to_vf_htab,
}
+/* Return the largest native vector piece of the fixed-length source TYPE.
+ This describes the original representation, independently of the vector
+ mode currently being considered for vectorization. */
+
+tree
+vect_native_vector_type (tree type)
+{
+ unsigned HOST_WIDE_INT nelts = TYPE_VECTOR_SUBPARTS (type).to_constant ();
+ for (; nelts > 1; nelts /= 2)
+ {
+ tree piece = build_vector_type (TREE_TYPE (type), nelts);
+ if (VECTOR_MODE_P (TYPE_MODE (piece)))
+ return piece;
+ }
+
+ return NULL_TREE;
+}
+
/* Loop autovectorization. */
namespace {
diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
index
e1d3fd47af5e7f8c0a968267d700f6a0270321a6..eeeee6e1fc63a2e6d8808f765add38b7d66b8e69
100644
--- a/gcc/tree-vectorizer.h
+++ b/gcc/tree-vectorizer.h
@@ -2819,6 +2819,7 @@ void vect_pattern_recog (vec_info *);
/* In tree-vectorizer.cc. */
extern bool vect_can_revectorize_type_p (tree);
+extern tree vect_native_vector_type (tree);
unsigned vectorize_loops (void);
void vect_free_loop_info_assumptions (class loop *);
gimple *vect_loop_vectorized_call (class loop *, gcond **cond = NULL);
