In current loop vectorizer, negative-step contiguous DRs are vectorized
with VMAT_CONTIGUOUS_REVERSE, which normally requires generating reverse
permutations for the vector loads and stores. However, when most of the
DRs are negative-step and contiguous, the number of reverse permutations
could be reduced.
This patch adds an invert_rev_state flag to DR nodes in the SLP graph to
record whether their reverse state is inverted. In the SLP optimization
pass, we partition the SLP graph into weakly connected components (WCCs)
and analyze each WCC independently. If negative-step contiguous DRs are
a majority, we set invert_rev_state on all DR nodes in the WCC.
Then vectorizable_load and vectorizable_store combine this flag with the
VMAT to decide whether a reverse permutation is needed. If a DR with
invert_rev_state set has a non-contiguous VMAT, we reject vectorization
conservatively. This case is expected to be rare since we have done many
eligibility checks before setting that flag in the optimization.
This optimization can elide all reverse permutations when all DRs are
negative-step and contiguous, or reduce their number for mixed forward
and backward DR cases.
This patch is bootstrapped and regression-tested on x86_64-linux-gnu,
and aarch64-linux-gnu with bootstrap-O3.
gcc/ChangeLog:
PR tree-optimization/61338
* tree-vect-slp.cc (_slp_tree::_slp_tree): Initialize flag
invert_rev_state to false.
(vect_stmt_lane_independent_p): New function.
(vect_print_slp_tree): Dump invert_rev_state.
(vect_optimize_slp_pass::start_choosing_layouts): Use
vect_stmt_lane_independent_p.
(vect_optimize_slp_pass::reduce_rev_perms_for_wcc): New helper
function for processing a single WCC.
(vect_optimize_slp_pass::reduce_reverse_permutations): New
function to reduce reverse permutations.
(vect_optimize_slp_pass::run): Call reduce_reverse_permutations.
* tree-vect-stmts.cc (vectorizable_store): Honor
invert_rev_state when adding a reverse permutation.
(vectorizable_load): Likewise.
* tree-vectorizer.h (struct _slp_tree): Add invert_rev_state.
gcc/testsuite/ChangeLog:
PR tree-optimization/61338
* gcc.dg/vect/vect-rev-perm-1.c: New test.
* gcc.dg/vect/vect-rev-perm-2.c: New test.
* gcc.dg/vect/vect-rev-perm-3.c: New test.
* gcc.dg/vect/vect-rev-perm-4.c: New test.
* gcc.dg/vect/vect-rev-perm-5.c: New test.
* gcc.dg/vect/vect-rev-perm-6.c: New test.
* gcc.dg/vect/vect-rev-perm-7.c: New test.
---
gcc/testsuite/gcc.dg/vect/vect-rev-perm-1.c | 34 ++++
gcc/testsuite/gcc.dg/vect/vect-rev-perm-2.c | 39 ++++
gcc/testsuite/gcc.dg/vect/vect-rev-perm-3.c | 41 ++++
gcc/testsuite/gcc.dg/vect/vect-rev-perm-4.c | 38 ++++
gcc/testsuite/gcc.dg/vect/vect-rev-perm-5.c | 42 +++++
gcc/testsuite/gcc.dg/vect/vect-rev-perm-6.c | 38 ++++
gcc/testsuite/gcc.dg/vect/vect-rev-perm-7.c | 45 +++++
gcc/tree-vect-slp.cc | 196 ++++++++++++++++++--
gcc/tree-vect-stmts.cc | 38 +++-
gcc/tree-vectorizer.h | 4 +
10 files changed, 500 insertions(+), 15 deletions(-)
create mode 100644 gcc/testsuite/gcc.dg/vect/vect-rev-perm-1.c
create mode 100644 gcc/testsuite/gcc.dg/vect/vect-rev-perm-2.c
create mode 100644 gcc/testsuite/gcc.dg/vect/vect-rev-perm-3.c
create mode 100644 gcc/testsuite/gcc.dg/vect/vect-rev-perm-4.c
create mode 100644 gcc/testsuite/gcc.dg/vect/vect-rev-perm-5.c
create mode 100644 gcc/testsuite/gcc.dg/vect/vect-rev-perm-6.c
create mode 100644 gcc/testsuite/gcc.dg/vect/vect-rev-perm-7.c
diff --git a/gcc/testsuite/gcc.dg/vect/vect-rev-perm-1.c
b/gcc/testsuite/gcc.dg/vect/vect-rev-perm-1.c
new file mode 100644
index 000000000000..f3a042f60e88
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-rev-perm-1.c
@@ -0,0 +1,34 @@
+/* { dg-require-effective-target vect_int } */
+/* { dg-require-effective-target vect_perm } */
+/* { dg-options "-O2 -ftree-vectorize -fdump-tree-vect-details" } */
+
+#include "tree-vect.h"
+
+__attribute__ ((noipa))
+void
+foo (int *a)
+{
+ for (int i = 99; i >= 0; i--)
+ a[i] *= 2;
+}
+
+int
+main ()
+{
+ check_vect ();
+
+ int a[100];
+
+#pragma GCC novector
+ for (int i = 0; i < 100; i++)
+ a[i] = i + 1;
+
+ foo (a);
+
+#pragma GCC novector
+ for (int i = 0; i < 100; i++)
+ if (a[i] != 2 * (i + 1))
+ __builtin_abort ();
+}
+
+/* { dg-final { scan-tree-dump "Inverting reverse state for 2 DR nodes" "vect"
} } */
diff --git a/gcc/testsuite/gcc.dg/vect/vect-rev-perm-2.c
b/gcc/testsuite/gcc.dg/vect/vect-rev-perm-2.c
new file mode 100644
index 000000000000..d27a5c64a709
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-rev-perm-2.c
@@ -0,0 +1,39 @@
+/* { dg-require-effective-target vect_int } */
+/* { dg-require-effective-target vect_perm } */
+/* { dg-options "-O2 -ftree-vectorize -fdump-tree-vect-details" } */
+
+#include "tree-vect.h"
+
+__attribute__ ((noipa))
+void
+foo (int *restrict a, int *restrict b, int *restrict c)
+{
+ for (int i = 99; i >= 0; i--)
+ {
+ b[i] = a[i] - 1;
+ c[i] = a[i] + 2;
+ }
+}
+
+int
+main ()
+{
+ check_vect ();
+
+ int a[100];
+ int b[100];
+ int c[100];
+
+#pragma GCC novector
+ for (int i = 0; i < 100; i++)
+ a[i] = i;
+
+ foo (a, b, c);
+
+#pragma GCC novector
+ for (int i = 0; i < 100; i++)
+ if (b[i] != i - 1 || c[i] != i + 2)
+ __builtin_abort ();
+}
+
+/* { dg-final { scan-tree-dump "Inverting reverse state for 3 DR nodes" "vect"
} } */
diff --git a/gcc/testsuite/gcc.dg/vect/vect-rev-perm-3.c
b/gcc/testsuite/gcc.dg/vect/vect-rev-perm-3.c
new file mode 100644
index 000000000000..ebffd259bf63
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-rev-perm-3.c
@@ -0,0 +1,41 @@
+/* { dg-require-effective-target vect_int } */
+/* { dg-require-effective-target vect_perm } */
+/* { dg-options "-O2 -ftree-vectorize -fdump-tree-vect-details" } */
+
+#include "tree-vect.h"
+
+__attribute__ ((noipa))
+void
+foo (int *restrict a, int *restrict b, int *restrict c)
+{
+ for (int i = 99; i >= 0; i--)
+ {
+ c[99 - i] = a[i] + b[i];
+ }
+}
+
+int
+main ()
+{
+ check_vect ();
+
+ int a[100];
+ int b[100];
+ int c[100];
+
+#pragma GCC novector
+ for (int i = 0; i < 100; i++)
+ {
+ a[i] = i;
+ b[i] = i;
+ }
+
+ foo (a, b, c);
+
+#pragma GCC novector
+ for (int i = 0; i < 100; i++)
+ if (c[i] != 2 * (99 - i))
+ __builtin_abort ();
+}
+
+/* { dg-final { scan-tree-dump "Inverting reverse state for 3 DR nodes" "vect"
} } */
diff --git a/gcc/testsuite/gcc.dg/vect/vect-rev-perm-4.c
b/gcc/testsuite/gcc.dg/vect/vect-rev-perm-4.c
new file mode 100644
index 000000000000..cc97e635fdda
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-rev-perm-4.c
@@ -0,0 +1,38 @@
+/* { dg-require-effective-target vect_int } */
+/* { dg-require-effective-target vect_perm } */
+/* { dg-options "-O2 -ftree-vectorize -fdump-tree-vect-details" } */
+
+#include "tree-vect.h"
+
+__attribute__ ((noipa))
+void
+foo (int *restrict a, int *restrict b)
+{
+ for (int i = 99; i >= 0; i--)
+ a[i] += b[99 - i];
+}
+
+int
+main ()
+{
+ check_vect ();
+
+ int a[100];
+ int b[100];
+
+#pragma GCC novector
+ for (int i = 0; i < 100; i++)
+ {
+ a[i] = i;
+ b[i] = i;
+ }
+
+ foo (a, b);
+
+#pragma GCC novector
+ for (int i = 0; i < 100; i++)
+ if (a[i] != 99)
+ __builtin_abort ();
+}
+
+/* { dg-final { scan-tree-dump "Inverting reverse state for 3 DR nodes" "vect"
} } */
diff --git a/gcc/testsuite/gcc.dg/vect/vect-rev-perm-5.c
b/gcc/testsuite/gcc.dg/vect/vect-rev-perm-5.c
new file mode 100644
index 000000000000..fea98c26b292
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-rev-perm-5.c
@@ -0,0 +1,42 @@
+/* { dg-require-effective-target vect_int } */
+/* { dg-require-effective-target vect_perm } */
+/* { dg-options "-O2 -ftree-vectorize -fdump-tree-vect-details" } */
+
+#include "tree-vect.h"
+
+__attribute__ ((noipa))
+int
+foo (int *a)
+{
+ int x = 100;
+ for (int i = 99; i >= 0; i--)
+ {
+ x = a[i];
+ a[i] = x * 2;
+ }
+ return x - 1;
+}
+
+int
+main ()
+{
+ check_vect ();
+
+ int a[100];
+
+#pragma GCC novector
+ for (int i = 0; i < 100; i++)
+ a[i] = i;
+
+ int x = foo (a);
+
+#pragma GCC novector
+ for (int i = 0; i < 100; i++)
+ if (a[i] != 2 * i)
+ __builtin_abort ();
+
+ if (x != -1)
+ __builtin_abort ();
+}
+
+/* { dg-final { scan-tree-dump-not "Inverting reverse state for" "vect" } } */
diff --git a/gcc/testsuite/gcc.dg/vect/vect-rev-perm-6.c
b/gcc/testsuite/gcc.dg/vect/vect-rev-perm-6.c
new file mode 100644
index 000000000000..3a686bb9190e
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-rev-perm-6.c
@@ -0,0 +1,38 @@
+/* { dg-require-effective-target vect_int } */
+/* { dg-require-effective-target vect_perm } */
+/* { dg-options "-O2 -ftree-vectorize -fdump-tree-vect-details" } */
+
+#include "tree-vect.h"
+
+__attribute__ ((noipa))
+void
+foo (int *restrict a, signed char *restrict b)
+{
+ for (int i = 99; i >= 0; i--)
+ a[i] += b[i];
+}
+
+int
+main ()
+{
+ check_vect ();
+
+ int a[100];
+ signed char b[100];
+
+#pragma GCC novector
+ for (int i = 0; i < 100; i++)
+ {
+ a[i] = i + 1;
+ b[i] = i - 1;
+ }
+
+ foo (a, b);
+
+#pragma GCC novector
+ for (int i = 0; i < 100; i++)
+ if (a[i] != 2 * i)
+ __builtin_abort ();
+}
+
+/* { dg-final { scan-tree-dump-not "Inverting reverse state for" "vect" } } */
diff --git a/gcc/testsuite/gcc.dg/vect/vect-rev-perm-7.c
b/gcc/testsuite/gcc.dg/vect/vect-rev-perm-7.c
new file mode 100644
index 000000000000..d47be56e95e7
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-rev-perm-7.c
@@ -0,0 +1,45 @@
+/* { dg-require-effective-target vect_int } */
+/* { dg-require-effective-target vect_perm } */
+/* { dg-require-effective-target aarch64_sve_hw { target aarch64*-*-* } } */
+/* { dg-options "-O2 -ftree-vectorize -fdump-tree-vect-details" } */
+/* { dg-additional-options "-march=armv8-a+sve" { target aarch64-*-* } } */
+
+#include "tree-vect.h"
+
+__attribute__ ((noipa))
+void
+foo (int *restrict a, int *restrict b, int *restrict c, int *restrict d)
+{
+ for (int i = 99; i >= 0; i--)
+ if (a[i] + b[i] + c[i] > 0)
+ d[99 - i] = a[i];
+}
+
+int
+main ()
+{
+ check_vect ();
+
+ int a[100];
+ int b[100];
+ int c[100];
+ int d[100];
+
+#pragma GCC novector
+ for (int i = 0; i < 100; i++)
+ {
+ a[i] = (i % 2 == 0) ? 1 : -1;
+ b[i] = 0;
+ c[i] = 0;
+ d[i] = 0;
+ }
+
+ foo (a, b, c, d);
+
+#pragma GCC novector
+ for (int i = 0; i < 100; i++)
+ if (d[i] != ((i % 2 == 0) ? 0 : 1))
+ __builtin_abort ();
+}
+
+/* { dg-final { scan-tree-dump-not "Inverting reverse state for" "vect" } } */
diff --git a/gcc/tree-vect-slp.cc b/gcc/tree-vect-slp.cc
index 12b1abe3d786..8120530ba5cf 100644
--- a/gcc/tree-vect-slp.cc
+++ b/gcc/tree-vect-slp.cc
@@ -125,6 +125,7 @@ _slp_tree::_slp_tree ()
SLP_TREE_GS_BASE (this) = NULL_TREE;
this->ldst_lanes = false;
this->avoid_stlf_fail = false;
+ this->invert_rev_state = false;
SLP_TREE_VECTYPE (this) = NULL_TREE;
SLP_TREE_REPRESENTATIVE (this) = NULL;
this->cycle_info.id = -1;
@@ -395,6 +396,30 @@ vect_slp_tree_uniform_p (slp_tree node)
return true;
}
+/* Return true if the vectorized operation of statement STMT is independent
+ of lane order. */
+
+static bool
+vect_stmt_lane_independent_p (gimple *stmt)
+{
+ if (!is_gimple_call (stmt))
+ return true;
+
+ switch (gimple_call_combined_fn (stmt))
+ {
+ case CFN_COMPLEX_ADD_ROT90:
+ case CFN_COMPLEX_ADD_ROT270:
+ case CFN_COMPLEX_MUL:
+ case CFN_COMPLEX_MUL_CONJ:
+ case CFN_VEC_ADDSUB:
+ case CFN_VEC_FMADDSUB:
+ case CFN_VEC_FMSUBADD:
+ return false;
+ default:
+ return true;
+ }
+}
+
/* Find the place of the data-ref in STMT_INFO in the interleaving chain
that starts from FIRST_STMT_INFO. Return -1 if the data-ref is not a part
of the chain. */
@@ -3486,6 +3511,8 @@ vect_print_slp_tree (dump_flags_t dump_kind,
dump_location_t loc,
dump_printf (metadata, " %T", SLP_TREE_VECTYPE (node));
dump_printf (metadata, "%s",
node->avoid_stlf_fail ? " (avoid-stlf-fail)" : "");
+ dump_printf (metadata, "%s",
+ node->invert_rev_state ? " (invert-rev-state)" : "");
if (node->cycle_info.id != -1 || node->cycle_info.reduc_idx != -1)
dump_printf (metadata, " cycle %d, link %d", node->cycle_info.id,
node->cycle_info.reduc_idx);
@@ -6685,6 +6712,13 @@ private:
/* Masked load lanes discovery. */
void decide_masked_load_lanes ();
+ /* Reduce reverse permutations within a weakly connected component. */
+ void reduce_rev_perms_for_wcc (loop_vec_info, const auto_vec<int> &,
+ unsigned, unsigned);
+
+ /* Reduce reverse permutations when possible. */
+ void reduce_reverse_permutations ();
+
void dump ();
vec_info *m_vinfo;
@@ -7486,19 +7520,8 @@ vect_optimize_slp_pass::start_choosing_layouts ()
not independent on lanes. Note this is an explicit
negative list since that's much shorter than the respective
positive one but it's critical to keep maintaining it. */
- if (is_gimple_call (STMT_VINFO_STMT (rep)))
- switch (gimple_call_combined_fn (STMT_VINFO_STMT (rep)))
- {
- case CFN_COMPLEX_ADD_ROT90:
- case CFN_COMPLEX_ADD_ROT270:
- case CFN_COMPLEX_MUL:
- case CFN_COMPLEX_MUL_CONJ:
- case CFN_VEC_ADDSUB:
- case CFN_VEC_FMADDSUB:
- case CFN_VEC_FMSUBADD:
- partition.layout = 0;
- default:;
- }
+ if (!vect_stmt_lane_independent_p (STMT_VINFO_STMT (rep)))
+ partition.layout = 0;
}
auto process_edge = [&](graph_edge *ud, unsigned int other_node_i)
@@ -8406,6 +8429,152 @@ vect_optimize_slp_pass::decide_masked_load_lanes ()
}
}
+/* Analyze a WCC described by WCC_GROUPING [WCC_BEGIN, WCC_END). Traverse
+ all nodes in the WCC, collect DR nodes and count the DRs that potentially
+ need reverse permutations. Abort early if the WCC contains a node that
+ cannot safely take part in the reverse state inversion.
+
+ If the DRs that may need reverse permutations form a majority, invert
+ the reverse states of all collected DR nodes to reduce the number of
+ generated reverse permutations. */
+
+void
+vect_optimize_slp_pass::reduce_rev_perms_for_wcc (loop_vec_info loop_vinfo,
+ const auto_vec<int> &
+ wcc_grouping,
+ unsigned wcc_begin,
+ unsigned wcc_end)
+{
+ auto_vec<slp_tree> dr_nodes;
+ poly_uint64 uniform_nunits = 0;
+ unsigned int num_negative_step_drs = 0;
+
+ for (unsigned int i = wcc_begin; i < wcc_end; i++)
+ {
+ slp_tree node = m_vertices[wcc_grouping[i]].node;
+ gcc_assert (node);
+
+ /* Abort if a node has an existing permutation. */
+ if (SLP_TREE_PERMUTE_P (node)
+ || SLP_TREE_LOAD_PERMUTATION (node).exists ())
+ return;
+
+ /* Require all nodes in the WCC to have the same nunits. */
+ tree vectype = SLP_TREE_VECTYPE (node);
+ if (vectype != NULL_TREE)
+ {
+ poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
+ if (known_eq (uniform_nunits, 0U))
+ uniform_nunits = nunits;
+ else if (maybe_ne (nunits, uniform_nunits))
+ return;
+ }
+
+ vect_def_type def_type = SLP_TREE_DEF_TYPE (node);
+ if (def_type == vect_internal_def)
+ {
+ /* An SLP node with vect_internal_def can still contain scalar
+ statements with other def types, such as vect_induction_def,
+ so each statement needs to be checked as well. This also rejects
+ statements with live-out results. */
+ unsigned int si;
+ stmt_vec_info stmt_info;
+ FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), si, stmt_info)
+ if (stmt_info
+ && (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
+ || STMT_VINFO_LIVE_P (stmt_info)))
+ return;
+
+ /* Abort if the vector operation depends on lane order. */
+ stmt_vec_info rep = SLP_TREE_REPRESENTATIVE (node);
+ if (!rep
+ || !vect_stmt_lane_independent_p (rep->stmt))
+ return;
+
+ /* Analyze data references and count reverse DRs. */
+ if (STMT_VINFO_DATA_REF (rep))
+ {
+ gcc_assert (vectype);
+ /* Only handle contiguous DRs for which a reverse permutation is
+ supported. Otherwise, the DR may have a non-contiguous VMAT,
+ which cannot be vectorized with inverted reverse state. */
+ if (!perm_mask_for_reverse (vectype)
+ || STMT_VINFO_STRIDED_P (rep)
+ || STMT_VINFO_GATHER_SCATTER_P (rep)
+ || STMT_VINFO_GROUPED_ACCESS (rep))
+ return;
+
+ /* Do not handle masked DRs as inverting their reverse states
+ also requires updating their masks. */
+ gcall *call = dyn_cast <gcall *> (rep->stmt);
+ if (call != NULL
+ && gimple_call_internal_p (call)
+ && internal_fn_mask_index
+ (gimple_call_internal_fn (call)) >= 0)
+ return;
+
+ int cmp = compare_step_with_zero (loop_vinfo, rep);
+ if (cmp != 0)
+ {
+ dr_nodes.safe_push (node);
+ if (cmp < 0)
+ num_negative_step_drs++;
+ }
+ else
+ return;
+ }
+ }
+ else if (def_type == vect_constant_def
+ || def_type == vect_external_def)
+ {
+ /* Only allow uniform vectors for constant or external def. */
+ if (!vect_slp_tree_uniform_p (node))
+ return;
+ }
+ else
+ return;
+ }
+
+ /* Majority vote: if a majority of the DRs may require reverse permutations,
+ invert the reverse states for all collected DRs. */
+ if (num_negative_step_drs > dr_nodes.length () / 2)
+ {
+ unsigned int i;
+ slp_tree node;
+ FOR_EACH_VEC_ELT (dr_nodes, i, node)
+ node->invert_rev_state = true;
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "Inverting reverse state for %u DR nodes\n",
+ dr_nodes.length ());
+ }
+}
+
+/* Analyze and reduce potential reverse permutations. */
+
+void
+vect_optimize_slp_pass::reduce_reverse_permutations ()
+{
+ loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (m_vinfo);
+ if (!loop_vinfo)
+ return;
+
+ /* Partition the graph into weakly connected components (WCCs) and process
+ them one by one. */
+ auto_vec<int> wcc_grouping;
+ int num_wccs = graphds_wcc (m_slpg, &wcc_grouping);
+
+ unsigned wcc_begin, wcc_end = 0;
+ for (int comp = 0; comp < num_wccs; comp++)
+ {
+ wcc_begin = wcc_end;
+ while (wcc_end < wcc_grouping.length ()
+ && m_slpg->vertices[wcc_grouping[wcc_end]].component == comp)
+ wcc_end++;
+ reduce_rev_perms_for_wcc (loop_vinfo, wcc_grouping, wcc_begin, wcc_end);
+ }
+}
+
/* Perform legitimizing attempts. This is intended to improve the
situation when layout 0 is not valid which is a situation the cost
based propagation does not handle well.
@@ -8487,6 +8656,7 @@ vect_optimize_slp_pass::run ()
remove_redundant_permutations ();
free_graph (m_slpg);
build_graph ();
+ reduce_reverse_permutations ();
decide_masked_load_lanes ();
free_graph (m_slpg);
}
diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
index eb78fd2eba39..509652060b95 100644
--- a/gcc/tree-vect-stmts.cc
+++ b/gcc/tree-vect-stmts.cc
@@ -8238,6 +8238,25 @@ vectorizable_store (vec_info *vinfo,
const int misalignment = ls.misalignment;
const poly_int64 poffset = ls.poffset;
+ bool need_reverse_p = false;
+ if (memory_access_type == VMAT_CONTIGUOUS)
+ need_reverse_p = slp_node->invert_rev_state;
+ else if (memory_access_type == VMAT_CONTIGUOUS_REVERSE)
+ need_reverse_p = !slp_node->invert_rev_state;
+ else if (slp_node->invert_rev_state
+ && memory_access_type != VMAT_CONTIGUOUS_DOWN)
+ {
+ /* If a DR with inverted reverse state has a non-contiguous VMAT, reject
+ vectorization conservatively. This is expected to be rare.
+ VMAT_CONTIGUOUS_DOWN is only used for storing invariants, so it does
+ not need a reverse. */
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "Reverse permutation state is inverted but VMAT "
+ "for this store is not contiguous.\n");
+ return false;
+ }
+
if (slp_node->ldst_lanes
&& memory_access_type != VMAT_LOAD_STORE_LANES)
{
@@ -9378,7 +9397,7 @@ vectorizable_store (vec_info *vinfo,
if (!costing_p)
vec_oprnd = vec_oprnds[i];
- if (memory_access_type == VMAT_CONTIGUOUS_REVERSE)
+ if (need_reverse_p)
{
if (costing_p)
inside_cost += record_stmt_cost (cost_vec, 1, vec_perm,
@@ -9907,6 +9926,21 @@ vectorizable_load (vec_info *vinfo,
int maskload_elsval = 0;
bool need_zeroing = false;
+ bool need_reverse_p = false;
+ if (memory_access_type == VMAT_CONTIGUOUS)
+ need_reverse_p = slp_node->invert_rev_state;
+ else if (memory_access_type == VMAT_CONTIGUOUS_REVERSE)
+ need_reverse_p = !slp_node->invert_rev_state;
+ else if (slp_node->invert_rev_state)
+ {
+ /* If a DR with inverted reverse state has a non-contiguous VMAT, reject
+ vectorization conservatively. This is expected to be rare. */
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "Reverse permutation state is inverted but VMAT "
+ "for this load is not contiguous.\n");
+ return false;
+ }
/* We might need to explicitly zero inactive elements if there are
padding bits in the type that might leak otherwise.
@@ -11924,7 +11958,7 @@ vectorizable_load (vec_info *vinfo,
}
}
- if (memory_access_type == VMAT_CONTIGUOUS_REVERSE)
+ if (need_reverse_p)
{
if (costing_p)
inside_cost = record_stmt_cost (cost_vec, 1, vec_perm,
diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
index abd9edc27431..d8ff9e8003f4 100644
--- a/gcc/tree-vectorizer.h
+++ b/gcc/tree-vectorizer.h
@@ -397,6 +397,10 @@ struct _slp_tree {
/* For BB vect, flag to indicate this load node should be vectorized
as to avoid STLF fails because of related stores. */
bool avoid_stlf_fail;
+ /* For loop vect, flag to indicate whether the reverse state on a DR node
+ is inverted. When it's set, a VMAT_CONTIGUOUS_REVERSE access should
+ omit its reverse permutation, while a VMAT_CONTIGUOUS should add one. */
+ bool invert_rev_state;
/* The vertex index of this node when a full graph is built. */
int vertex;
--
2.43.0
