This enables loop versioning of the shape:
│
▼
non-alignment versioning
┌──checks ────────────┐
┌──────▼──────┐ ▼
│ Scalar Loop │ ┌──────alignment checks─────┐
└──────┬──────┘ ▼ ┌──────────▼────────────┐
│ ┌───hssr safety checks───┐ │ Mutually aligned loop │
│ ┌──────▼──────┐ ┌──────▼────┐ └───────────┬───────────┘
│ │ Scalar Loop │ │ HSSR Loop │ │
│ └──────┬──────┘ └──────┬────┘ │
│ │ │ │
└─────────┴──┬─────────────────────┴──────────────────┘
▼
For supporting aligned loop vectorization where pointers are mutually
aligned and the (slower) HSSR vectorization when the pointers aren't
mutually aligned.
There is room for improvement in this at a later date.
gcc/ChangeLog:
* config/aarch64/aarch64.cc (better_main_loop_than_p): Update the
use of param_vect_hssr_usage.
* params.opt: Add new value for versioning.
* tree-vect-loop-manip.cc (vect_loop_versioning): Dont version
for alignment when doing HSSR versioning.
(vect_loop_hssr_alias_versioning): New function.
* tree-vect-loop.cc (_loop_vec_info::_loop_vec_info): Add
use_hssr_unaligned_version_p.
(vect_determine_hssr_and_versioning): Add logic for opting into
HSSR versioning when already versioning for alignment.
(vect_analyze_loop_1): Add force_hssr option.
(vect_analyze_loop): Add logic for enabling HSSR versioning.
(vect_transform_loop): Add simduid_to_vf_htab and
num_vectorized_loops arguments and logic for doing HSSR
versioning.
* tree-vectorizer.cc (class simduid_to_vf): Move to
tree-vectorizer.h.
(vect_transform_loops): Add simduid_to_vf_htab and
num_vectorized_loops arguments to vect_transform_loop call.
(try_vectorize_loop_1): Make non-static and add
hssr_alignment_loop flag.
(try_vectorize_loop): Update call to try_vectorize_loop_1.
* tree-vectorizer.h (LOOP_VINFO_USE_HSSR_UNALIGNED_VERSION_P):
New macro.
(class _loop_vec_info ): Add use_ffr_unaligned_version_p flag.
(vect_analyze_loop): Add new argument.
(class simduid_to_vf): Move from tree-vectorizer.cc
(vect_transform_loop): Add new arguments.
(try_vectorize_loop_1): Add new arguments.
gcc/testsuite/ChangeLog:
* gcc.target/aarch64/sve/ffr_1.c:
Update usage of param_vect_hssr_usage.
* gcc.target/aarch64/sve/ffr_11.c: Likewise.
* gcc.target/aarch64/sve/ffr_12.c: Likewise.
* gcc.target/aarch64/sve/ffr_13.c: Likewise.
* gcc.target/aarch64/sve/ffr_14.c: Likewise.
* gcc.target/aarch64/sve/ffr_2.c: Likewise.
* gcc.target/aarch64/sve/ffr_3.c: Likewise.
* gcc.target/aarch64/sve/ffr_4.c: Likewise.
* gcc.target/aarch64/sve/ffr_5.c: Likewise.
* gcc.target/aarch64/sve/ffr_6.c: Likewise.
* gcc.target/aarch64/sve/ffr_6_run.c: Likewise.
* gcc.target/aarch64/sve/ffr_7.c: Likewise.
* gcc.target/aarch64/sve/ffr_8.c: Likewise.
* gcc.target/aarch64/sve/ffr_9.c: Likewise.
---
gcc/config/aarch64/aarch64.cc | 2 +-
gcc/params.opt | 4 +-
gcc/testsuite/gcc.target/aarch64/sve/ffr_1.c | 2 +-
gcc/testsuite/gcc.target/aarch64/sve/ffr_11.c | 2 +-
gcc/testsuite/gcc.target/aarch64/sve/ffr_12.c | 2 +-
gcc/testsuite/gcc.target/aarch64/sve/ffr_13.c | 2 +-
gcc/testsuite/gcc.target/aarch64/sve/ffr_14.c | 2 +-
gcc/testsuite/gcc.target/aarch64/sve/ffr_2.c | 2 +-
gcc/testsuite/gcc.target/aarch64/sve/ffr_3.c | 2 +-
gcc/testsuite/gcc.target/aarch64/sve/ffr_4.c | 2 +-
gcc/testsuite/gcc.target/aarch64/sve/ffr_5.c | 2 +-
gcc/testsuite/gcc.target/aarch64/sve/ffr_6.c | 2 +-
.../gcc.target/aarch64/sve/ffr_6_run.c | 2 +-
gcc/testsuite/gcc.target/aarch64/sve/ffr_7.c | 2 +-
gcc/testsuite/gcc.target/aarch64/sve/ffr_8.c | 2 +-
gcc/testsuite/gcc.target/aarch64/sve/ffr_9.c | 2 +-
gcc/tree-vect-loop-manip.cc | 193 +++++++++++++++++-
gcc/tree-vect-loop.cc | 63 +++++-
gcc/tree-vectorizer.cc | 47 ++---
gcc/tree-vectorizer.h | 41 +++-
20 files changed, 317 insertions(+), 61 deletions(-)
diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc
index 7ea3bc0f061..1eb2930cdb1 100644
--- a/gcc/config/aarch64/aarch64.cc
+++ b/gcc/config/aarch64/aarch64.cc
@@ -19619,7 +19619,7 @@ better_main_loop_than_p (const vector_costs
*uncast_other) const
if ((LOOP_VINFO_USING_HSSR_P (this_loop_vinfo)
!= LOOP_VINFO_USING_HSSR_P (other_loop_vinfo))
- && param_vect_hssr_usage == 2)
+ && param_vect_hssr_usage == 3)
{
if (dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location,
diff --git a/gcc/params.opt b/gcc/params.opt
index 881efa2d02b..3e8f83e3fb2 100644
--- a/gcc/params.opt
+++ b/gcc/params.opt
@@ -1292,8 +1292,8 @@ Common Joined UInteger
Var(param_vect_partial_vector_usage) Init(2) IntegerRange
Controls how loop vectorizer uses partial vectors. 0 means never, 1 means
only for loops whose need to iterate can be removed, 2 means for all loops.
The default value is 2.
-param=vect-hssr-usage=
-Common Joined UInteger Var(param_vect_hssr_usage) Init(0) IntegerRange(0, 2)
Param Optimization NoOffload
-Controls how loop vectorizer uses hardware safe speculative loads. 0 means
never, 1 means only when profitable, 2 means whenever possible. The default
value is 2.
+Common Joined UInteger Var(param_vect_hssr_usage) Init(0) IntegerRange(0, 3)
Param Optimization NoOffload
+Controls how loop vectorizer uses hardware safe speculative loads. 0 means
never, 1 means only when profitable, 2 means whenever profitable possibly as a
versioned option, 3 means whenever possible. The default value is 0.
-param=vect-inner-loop-cost-factor=
Common Joined UInteger Var(param_vect_inner_loop_cost_factor) Init(50)
IntegerRange(1, 10000) Param Optimization NoOffload
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/ffr_1.c
b/gcc/testsuite/gcc.target/aarch64/sve/ffr_1.c
index 3027c6d69f0..58e08db2010 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/ffr_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/ffr_1.c
@@ -1,5 +1,5 @@
/* { dg-do compile } */
-/* { dg-options "-O3 -fdump-tree-vect-details --save-temps
--param=vect-hssr-usage=2" } */
+/* { dg-options "-O3 -fdump-tree-vect-details --save-temps
--param=vect-hssr-usage=3" } */
#include <stdint.h>
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/ffr_11.c
b/gcc/testsuite/gcc.target/aarch64/sve/ffr_11.c
index dedea70dd3b..842705446b1 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/ffr_11.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/ffr_11.c
@@ -1,5 +1,5 @@
/* { dg-do compile } */
-/* { dg-options "-O2 --param=vect-hssr-usage=2" } */
+/* { dg-options "-O2 --param=vect-hssr-usage=3" } */
// Check this doesnt ICE
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/ffr_12.c
b/gcc/testsuite/gcc.target/aarch64/sve/ffr_12.c
index db70c7ee87b..02f92da5093 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/ffr_12.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/ffr_12.c
@@ -1,5 +1,5 @@
/* { dg-do compile } */
-/* { dg-options "-mcpu=neoverse-v3 -O3 --param=vect-hssr-usage=2" } */
+/* { dg-options "-mcpu=neoverse-v3 -O3 --param=vect-hssr-usage=3" } */
// Check this doesnt ICE
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/ffr_13.c
b/gcc/testsuite/gcc.target/aarch64/sve/ffr_13.c
index a6b49c3c9df..31912596f66 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/ffr_13.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/ffr_13.c
@@ -1,5 +1,5 @@
/* { dg-do compile } */
-/* { dg-options "-mcpu=neoverse-v3 -O2 -mautovec-preference=sve-only
-mmax-vectorization --param=vect-hssr-usage=2" } */
+/* { dg-options "-mcpu=neoverse-v3 -O2 -mautovec-preference=sve-only
-mmax-vectorization --param=vect-hssr-usage=3" } */
// Check this doesn't ICE
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/ffr_14.c
b/gcc/testsuite/gcc.target/aarch64/sve/ffr_14.c
index bba9893c3a0..1101dd8d220 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/ffr_14.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/ffr_14.c
@@ -1,5 +1,5 @@
/* { dg-do compile } */
-/* { dg-options "-mcpu=neoverse-v3 -O2 -mautovec-preference=sve-only
-mmax-vectorization --param=vect-hssr-usage=2" } */
+/* { dg-options "-mcpu=neoverse-v3 -O2 -mautovec-preference=sve-only
-mmax-vectorization --param=vect-hssr-usage=3" } */
// Check this doesn't ICE
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/ffr_2.c
b/gcc/testsuite/gcc.target/aarch64/sve/ffr_2.c
index 45198266cee..9f6d72e6ca9 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/ffr_2.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/ffr_2.c
@@ -1,5 +1,5 @@
/* { dg-do compile } */
-/* { dg-options "-O3 -fdump-tree-vect-details --save-temps
--param=vect-hssr-usage=2" } */
+/* { dg-options "-O3 -fdump-tree-vect-details --save-temps
--param=vect-hssr-usage=3" } */
#include <stdint.h>
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/ffr_3.c
b/gcc/testsuite/gcc.target/aarch64/sve/ffr_3.c
index b52869c99a3..3fac6c2a817 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/ffr_3.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/ffr_3.c
@@ -1,5 +1,5 @@
/* { dg-do compile } */
-/* { dg-options "-O3 -fdump-tree-vect-details --save-temps
--param=vect-hssr-usage=2" } */
+/* { dg-options "-O3 -fdump-tree-vect-details --save-temps
--param=vect-hssr-usage=3" } */
/* { dg-final { check-function-bodies "**" "" "" } } */
#include <stdint.h>
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/ffr_4.c
b/gcc/testsuite/gcc.target/aarch64/sve/ffr_4.c
index 9def8e348ec..937d0b6c55e 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/ffr_4.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/ffr_4.c
@@ -1,5 +1,5 @@
/* { dg-do compile } */
-/* { dg-options "-O3 -fdump-tree-vect-details --save-temps
--param=vect-hssr-usage=2" } */
+/* { dg-options "-O3 -fdump-tree-vect-details --save-temps
--param=vect-hssr-usage=3" } */
#include <stdint.h>
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/ffr_5.c
b/gcc/testsuite/gcc.target/aarch64/sve/ffr_5.c
index e41c453188a..dc9b7cace08 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/ffr_5.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/ffr_5.c
@@ -1,5 +1,5 @@
/* { dg-do compile } */
-/* { dg-options "-O3 -fdump-tree-vect-details --save-temps
--param=vect-hssr-usage=2" } */
+/* { dg-options "-O3 -fdump-tree-vect-details --save-temps
--param=vect-hssr-usage=3" } */
#include <stdint.h>
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/ffr_6.c
b/gcc/testsuite/gcc.target/aarch64/sve/ffr_6.c
index 4f225746be3..08a3c866bc7 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/ffr_6.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/ffr_6.c
@@ -1,5 +1,5 @@
/* { dg-do compile } */
-/* { dg-options "-O3 --param=vect-hssr-usage=2" } */
+/* { dg-options "-O3 --param=vect-hssr-usage=3" } */
#include <stdint.h>
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/ffr_6_run.c
b/gcc/testsuite/gcc.target/aarch64/sve/ffr_6_run.c
index 452c7307cff..243c0af34ec 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/ffr_6_run.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/ffr_6_run.c
@@ -1,5 +1,5 @@
/* { dg-do run { target aarch64_sve_hw } } */
-/* { dg-options "-O3 --param=vect-hssr-usage=2" } */
+/* { dg-options "-O3 --param=vect-hssr-usage=3" } */
#include "ffr_6.c"
#include <stdint.h>
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/ffr_7.c
b/gcc/testsuite/gcc.target/aarch64/sve/ffr_7.c
index 1f508eb370e..6c5881b1c4e 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/ffr_7.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/ffr_7.c
@@ -1,6 +1,6 @@
/* { dg-do compile } */
-/* { dg-options "-O3 -fdump-tree-vect-details --save-temps
-mtune=generic-armv9-a --param=vect-hssr-usage=2" } */
+/* { dg-options "-O3 -fdump-tree-vect-details --save-temps
-mtune=generic-armv9-a --param=vect-hssr-usage=3" } */
// Options to try get the cost model to select VNx4SI
#include <stdint.h>
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/ffr_8.c
b/gcc/testsuite/gcc.target/aarch64/sve/ffr_8.c
index bdecf2e1875..ff75167f880 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/ffr_8.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/ffr_8.c
@@ -1,5 +1,5 @@
/* { dg-do compile } */
-/* { dg-options "-O3 -fdump-tree-vect-details --save-temps -mtune=generic
--param=vect-hssr-usage=2" } */
+/* { dg-options "-O3 -fdump-tree-vect-details --save-temps -mtune=generic
--param=vect-hssr-usage=3" } */
// Options to get the cost model to select VNx8HI
#include <stdint.h>
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/ffr_9.c
b/gcc/testsuite/gcc.target/aarch64/sve/ffr_9.c
index 5b4a38355af..8cc822b1ba4 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/ffr_9.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/ffr_9.c
@@ -1,5 +1,5 @@
/* { dg-do compile } */
-/* { dg-options "-O2 -fdump-tree-vect-details --save-temps
--param=vect-hssr-usage=2" } */
+/* { dg-options "-O2 -fdump-tree-vect-details --save-temps
--param=vect-hssr-usage=3" } */
/* Check we use HSSR at O2. */
diff --git a/gcc/tree-vect-loop-manip.cc b/gcc/tree-vect-loop-manip.cc
index 28a5978d9a3..7ac7f2141ba 100644
--- a/gcc/tree-vect-loop-manip.cc
+++ b/gcc/tree-vect-loop-manip.cc
@@ -4559,7 +4559,8 @@ vect_loop_versioning (loop_vec_info loop_vinfo,
profile_probability prob = profile_probability::likely ();
gimple_seq gimplify_stmt_list = NULL;
tree scalar_loop_iters = LOOP_VINFO_NITERSM1 (loop_vinfo);
- bool version_align = LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT (loop_vinfo);
+ bool version_align = LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT (loop_vinfo)
+ && !LOOP_VINFO_USE_HSSR_UNALIGNED_VERSION_P (loop_vinfo);
bool version_spec_read = LOOP_REQUIRES_VERSIONING_FOR_SPEC_READ (loop_vinfo);
bool version_alias = LOOP_REQUIRES_VERSIONING_FOR_ALIAS (loop_vinfo);
bool version_niter = LOOP_REQUIRES_VERSIONING_FOR_NITERS (loop_vinfo);
@@ -4940,6 +4941,196 @@ vect_loop_versioning (loop_vec_info loop_vinfo,
return nloop;
}
+/* Function vect_loop_hssr_alias_versioning.
+
+ Similar to vect_loop_versioning, however only handles alias checks to go
+ to an HSSR loop. */
+
+class loop *
+vect_loop_hssr_alias_versioning (loop_vec_info loop_vinfo)
+{
+ class loop *loop = LOOP_VINFO_LOOP (loop_vinfo), *nloop;
+
+ basic_block condition_bb;
+ gphi_iterator gsi;
+ gimple_stmt_iterator cond_exp_gsi;
+ basic_block merge_bb;
+ basic_block new_exit_bb;
+ edge new_exit_e, e;
+ gphi *orig_phi, *new_phi;
+ tree cond_expr = NULL_TREE;
+ gimple_seq cond_expr_stmt_list = NULL;
+ tree arg;
+ profile_probability prob = profile_probability::likely ();
+ gimple_seq gimplify_stmt_list = NULL;
+
+ vect_create_cond_for_align_checks (loop_vinfo, &cond_expr,
+ &cond_expr_stmt_list);
+
+ cond_expr = force_gimple_operand_1 (unshare_expr (cond_expr),
+ &gimplify_stmt_list,
+ is_gimple_condexpr_for_cond, NULL_TREE);
+
+ gimple_seq_add_seq (&cond_expr_stmt_list, gimplify_stmt_list);
+
+ /* Compute the outermost loop cond_expr and cond_expr_stmt_list are
+ invariant in. */
+ class loop *outermost = outermost_invariant_loop_for_expr (loop, cond_expr);
+ for (gimple_stmt_iterator gsi = gsi_start (cond_expr_stmt_list);
+ !gsi_end_p (gsi); gsi_next (&gsi))
+ {
+ gimple *stmt = gsi_stmt (gsi);
+ update_stmt (stmt);
+ ssa_op_iter iter;
+ use_operand_p use_p;
+ basic_block def_bb;
+ FOR_EACH_SSA_USE_OPERAND (use_p, stmt, iter, SSA_OP_USE)
+ if ((def_bb = gimple_bb (SSA_NAME_DEF_STMT (USE_FROM_PTR (use_p))))
+ && flow_bb_inside_loop_p (outermost, def_bb))
+ outermost = superloop_at_depth (loop, bb_loop_depth (def_bb) + 1);
+ }
+
+ /* Search for the outermost loop we can version. Avoid versioning of
+ non-perfect nests but allow if-conversion versioned loops inside. */
+ class loop *loop_to_version = loop;
+ if (flow_loop_nested_p (outermost, loop))
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "trying to apply versioning to outer loop %d\n",
+ outermost->num);
+ if (outermost->num == 0)
+ outermost = superloop_at_depth (loop, 1);
+ /* And avoid applying versioning on non-perfect nests. */
+ while (loop_to_version != outermost
+ && (e = single_exit (loop_outer (loop_to_version)))
+ && !(e->flags & EDGE_COMPLEX)
+ && (!loop_outer (loop_to_version)->inner->next
+ || vect_loop_vectorized_call (loop_to_version))
+ && (!loop_outer (loop_to_version)->inner->next
+ || !loop_outer (loop_to_version)->inner->next->next)
+ && can_duplicate_loop_p (loop_outer (loop_to_version)))
+ loop_to_version = loop_outer (loop_to_version);
+ }
+
+ /* Apply versioning. */
+ if (loop_to_version != loop && dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "applying loop versioning to outer loop %d\n",
+ loop_to_version->num);
+
+ unsigned orig_pe_idx = loop_preheader_edge (loop)->dest_idx;
+
+ initialize_original_copy_tables ();
+ nloop = loop_version (loop_to_version, cond_expr, &condition_bb, prob,
+ (prob).invert (), prob,
+ (prob).invert (), true);
+
+ /* If the PHI nodes in the loop header were reallocated, we need to fix up
+ our internally stashed copies of those. */
+ if (loop_to_version == loop)
+ for (auto gsi = gsi_start_phis (loop->header); !gsi_end_p (gsi);
+ gsi_next (&gsi))
+ loop_vinfo->resync_stmt_addr (gsi.phi ());
+
+ /* We will later insert second conditional so overall outcome of
+ both is prob * prob2. */
+ edge true_e, false_e;
+ extract_true_false_edges_from_block (condition_bb, &true_e, &false_e);
+ true_e->probability = prob;
+ false_e->probability = prob.invert ();
+ gcc_assert (nloop);
+ nloop = get_loop_copy (loop);
+
+ /* Assign hierarchical discriminators to distinguish loop versions.
+ Only assign to the scalar version here; the vectorized version will
+ get discriminators later during transformation/peeling.
+ Use dynamic copy_id allocation instead of hardcoded constants. */
+ gimple *nloop_last = last_nondebug_stmt (nloop->header);
+ location_t nloop_loc
+ = nloop_last ? gimple_location (nloop_last) : UNKNOWN_LOCATION;
+ if (nloop_loc != UNKNOWN_LOCATION)
+ {
+ unsigned int nloop_copyid = allocate_copyid_base (nloop_loc, 1);
+ assign_discriminators_to_loop (nloop, 0, nloop_copyid);
+ }
+ /* For cycle vectorization with SLP we rely on the PHI arguments
+ appearing in the same order as the SLP node operands which for the
+ loop PHI nodes means the preheader edge dest index needs to remain
+ the same for the analyzed loop which also becomes the vectorized one.
+ Make it so in case the state after versioning differs by redirecting
+ the first edge into the header to the same destination which moves
+ it last. */
+ if (loop_preheader_edge (loop)->dest_idx != orig_pe_idx)
+ {
+ edge e = EDGE_PRED (loop->header, 0);
+ ssa_redirect_edge (e, e->dest);
+ flush_pending_stmts (e);
+ }
+ gcc_assert (loop_preheader_edge (loop)->dest_idx == orig_pe_idx);
+
+ free_original_copy_tables ();
+
+ if (cond_expr_stmt_list)
+ {
+ cond_exp_gsi = gsi_last_bb (condition_bb);
+ gsi_insert_seq_before (&cond_exp_gsi, cond_expr_stmt_list,
GSI_SAME_STMT);
+ }
+
+ /* Loop versioning violates an assumption we try to maintain during
+ vectorization - that the loop exit block has a single predecessor.
+ After versioning, the exit block of both loop versions is the same
+ basic block (i.e. it has two predecessors). Just in order to simplify
+ following transformations in the vectorizer, we fix this situation
+ here by adding a new (empty) block on the exit-edge of the loop,
+ with the proper loop-exit phis to maintain loop-closed-form.
+ If loop versioning wasn't done from loop, but scalar_loop instead,
+ merge_bb will have already just a single successor. */
+
+ /* When the loop has multiple exits then we can only version itself.
+ This is denoted by loop_to_version == loop. In this case we can
+ do the versioning by selecting the exit edge the vectorizer is
+ currently using. */
+ edge exit_edge;
+ if (loop_to_version == loop)
+ exit_edge = LOOP_VINFO_MAIN_EXIT (loop_vinfo);
+ else
+ exit_edge = single_exit (loop_to_version);
+
+ gcc_assert (exit_edge);
+ merge_bb = exit_edge->dest;
+ if (EDGE_COUNT (merge_bb->preds) >= 2)
+ {
+ gcc_assert (EDGE_COUNT (merge_bb->preds) >= 2);
+ new_exit_bb = split_edge (exit_edge);
+ new_exit_e = exit_edge;
+ e = EDGE_SUCC (new_exit_bb, 0);
+
+ for (gsi = gsi_start_phis (merge_bb); !gsi_end_p (gsi); gsi_next (&gsi))
+ {
+ tree new_res;
+ orig_phi = gsi.phi ();
+ new_res = copy_ssa_name (PHI_RESULT (orig_phi));
+ new_phi = create_phi_node (new_res, new_exit_bb);
+ arg = PHI_ARG_DEF_FROM_EDGE (orig_phi, e);
+ add_phi_arg (new_phi, arg, new_exit_e,
+ gimple_phi_arg_location_from_edge (orig_phi, e));
+ adjust_phi_and_debug_stmts (orig_phi, e, PHI_RESULT (new_phi));
+ }
+ }
+
+ update_ssa (TODO_update_ssa_no_phi);
+
+ if (LOCATION_LOCUS (vect_location.get_location_t ()) != UNKNOWN_LOCATION
+ && dump_enabled_p ())
+ dump_printf_loc (MSG_OPTIMIZED_LOCATIONS | MSG_PRIORITY_USER_FACING,
+ vect_location,
+ "loop versioned for hssr vectorization to enhance "
+ "alignment\n");
+
+ return nloop;
+}
+
/* Checks if it's possible to create the necessary controls for an HSSR loop.
For loops with HSSR we need to generate all the required masks from the
diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc
index aa55c846719..20fbaea1656 100644
--- a/gcc/tree-vect-loop.cc
+++ b/gcc/tree-vect-loop.cc
@@ -759,6 +759,7 @@ _loop_vec_info::_loop_vec_info (class loop *loop_in,
vec_info_shared *shared)
must_use_hssr_p (false),
can_use_hssr_p (true),
using_hssr_p (false),
+ use_hssr_unaligned_version_p (false),
using_partial_vectors_p (false),
using_decrementing_iv_p (false),
using_select_vl_p (false),
@@ -2360,11 +2361,27 @@ vect_determine_hssr_and_versioning (loop_vec_info
loop_vinfo)
goto exit;
}
- /* We can and will use HSSR. */
+ /* We can use HSSR. */
if (dump_enabled_p ())
- dump_printf_loc (MSG_NOTE, vect_location, "HSSR: Will use hssr\n");
+ dump_printf_loc (MSG_NOTE, vect_location, "HSSR: Can use HSSR\n");
- /* We no longer need a bunch of versioning. */
+ /* If we are going to use versioning, then record that we can use HSSR,
+ but dont use it for this loop, as we can consider it for the versioned
+ alternative. */
+ if (LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT (loop_vinfo))
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "FFR: Not using FFR because versioning, will "
+ "consider it for versioned unaligned loop.\n");
+ goto exit;
+ }
+
+ /* We can use HSSR. */
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location, "HSSR: Using HSSR\n");
+
+ /* Can and will use HSSR. */
LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P (loop_vinfo) = true;
LOOP_VINFO_USING_HSSR_P (loop_vinfo) = true;
LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo) = vNULL;
@@ -3033,6 +3050,7 @@ again:
LOOP_VINFO_USING_HSSR_P (loop_vinfo) = false;
LOOP_VINFO_MUST_USE_HSSR_P (loop_vinfo) = false;
LOOP_VINFO_CAN_USE_HSSR_P (loop_vinfo) = true;
+ LOOP_VINFO_USE_HSSR_UNALIGNED_VERSION_P (loop_vinfo) = false;
LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo) = false;
LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo) = false;
LOOP_VINFO_USING_DECREMENTING_IV_P (loop_vinfo) = false;
@@ -3109,7 +3127,7 @@ vect_analyze_loop_1 (class loop *loop, vec_info_shared
*shared,
const vector_modes &vector_modes, unsigned &mode_i,
int masked_p,
machine_mode &autodetected_vector_mode,
- bool &fatal)
+ bool &fatal, bool force_hssr)
{
loop_vec_info loop_vinfo
= vect_create_loop_vinfo (loop, shared, loop_form_info, orig_loop_vinfo);
@@ -3118,6 +3136,7 @@ vect_analyze_loop_1 (class loop *loop, vec_info_shared
*shared,
loop_vinfo->vector_mode = vector_mode;
unsigned int suggested_unroll_factor = 1;
bool single_lane_slp_done_for_suggested_uf = false;
+ LOOP_VINFO_MUST_USE_HSSR_P (loop_vinfo) = force_hssr;
/* Run the main analysis. */
opt_result res = vect_analyze_loop_2 (loop_vinfo, masked_p, fatal,
@@ -3227,7 +3246,7 @@ vect_analyze_loop_1 (class loop *loop, vec_info_shared
*shared,
loop_vec_info struct. */
opt_loop_vec_info
vect_analyze_loop (class loop *loop, gimple *loop_vectorized_call,
- vec_info_shared *shared)
+ vec_info_shared *shared, bool force_hssr)
{
DUMP_VECT_SCOPE ("analyze_loop_nest");
@@ -3289,6 +3308,8 @@ vect_analyze_loop (class loop *loop, gimple
*loop_vectorized_call,
for (unsigned i = 0; i < vector_modes.length (); ++i)
cached_vf_per_mode.safe_push (0);
+ bool can_hssr_any_mode = false;
+
/* First determine the main loop vectorization mode, either the first
one that works, starting with auto-detecting the vector mode and then
following the targets order of preference, or the one with the
@@ -3303,12 +3324,19 @@ vect_analyze_loop (class loop *loop, gimple
*loop_vectorized_call,
opt_loop_vec_info loop_vinfo
= vect_analyze_loop_1 (loop, shared, &loop_form_info,
NULL, vector_modes, mode_i, -1,
- autodetected_vector_mode, fatal);
+ autodetected_vector_mode, fatal, force_hssr);
if (fatal)
break;
if (loop_vinfo)
{
+ /* If this loop could have used FFR, that means it's possible and
+ is profitable over scalar so it should be used for the
+ alignment versioning alternative. */
+ if (LOOP_VINFO_CAN_USE_HSSR_P (loop_vinfo)
+ && LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT (loop_vinfo))
+ can_hssr_any_mode = true;
+
/* Analysis has been successful so update the VF value. The
VF should always be a multiple of unroll_factor and we want to
capture the original VF here. */
@@ -3358,6 +3386,14 @@ vect_analyze_loop (class loop *loop, gimple
*loop_vectorized_call,
if (!first_loop_vinfo)
return opt_loop_vec_info::propagate_failure (res);
+ /* It it was profitable to use FFR over scalar for any of the modes, but we
+ decided not to and to version for alignment, then we should use this for
+ the misaligned pointer fallback. */
+ if (can_hssr_any_mode
+ && !LOOP_VINFO_USING_HSSR_P (first_loop_vinfo)
+ && LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT (first_loop_vinfo))
+ LOOP_VINFO_USE_HSSR_UNALIGNED_VERSION_P (first_loop_vinfo) = true;
+
if (dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location,
"***** Choosing vector mode %s\n",
@@ -3456,7 +3492,7 @@ vect_analyze_loop (class loop *loop, gimple
*loop_vectorized_call,
= vect_analyze_loop_1 (loop, shared, &loop_form_info,
orig_loop_vinfo,
vector_modes, mode_i, masked_p,
- autodetected_vector_mode, fatal);
+ autodetected_vector_mode, fatal, false);
if (fatal)
break;
@@ -11563,7 +11599,9 @@ vect_update_ivs_after_vectorizer_for_early_breaks
(loop_vec_info loop_vinfo)
Returns scalar epilogue loop if any. */
class loop *
-vect_transform_loop (loop_vec_info loop_vinfo, gimple *loop_vectorized_call)
+vect_transform_loop (loop_vec_info loop_vinfo, gimple *loop_vectorized_call,
+ hash_table<simduid_to_vf> *&simduid_to_vf_htab,
+ unsigned *num_vectorized_loops)
{
class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
class loop *epilogue = NULL;
@@ -11622,6 +11660,15 @@ vect_transform_loop (loop_vec_info loop_vinfo, gimple
*loop_vectorized_call)
check_profitability = false;
}
+ /* Version for the alignment check and vectorize the alternative with FFR.
*/
+ if (LOOP_VINFO_USE_HSSR_UNALIGNED_VERSION_P (loop_vinfo)
+ && LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo).length () > 0)
+ {
+ class loop *sloop = vect_loop_hssr_alias_versioning (loop_vinfo);
+ try_vectorize_loop_1 (simduid_to_vf_htab, num_vectorized_loops, sloop,
+ NULL, NULL, true, cfun);
+ }
+
/* Make sure there exists a single-predecessor exit bb also on the
scalar loop copy. Do this after versioning but before peeling
so CFG structure is fine for both scalar and if-converted loop
diff --git a/gcc/tree-vectorizer.cc b/gcc/tree-vectorizer.cc
index 398edbd7ecc..831d40cad9f 100644
--- a/gcc/tree-vectorizer.cc
+++ b/gcc/tree-vectorizer.cc
@@ -196,31 +196,6 @@ dump_stmt_cost (FILE *f, int count, enum
vect_cost_for_stmt kind,
fprintf (f, "in %s\n", ws);
}
-/* For mapping simduid to vectorization factor. */
-
-class simduid_to_vf : public free_ptr_hash<simduid_to_vf>
-{
-public:
- unsigned int simduid;
- poly_uint64 vf;
-
- /* hash_table support. */
- static inline hashval_t hash (const simduid_to_vf *);
- static inline int equal (const simduid_to_vf *, const simduid_to_vf *);
-};
-
-inline hashval_t
-simduid_to_vf::hash (const simduid_to_vf *p)
-{
- return p->simduid;
-}
-
-inline int
-simduid_to_vf::equal (const simduid_to_vf *p1, const simduid_to_vf *p2)
-{
- return p1->simduid == p2->simduid;
-}
-
/* This hash maps the OMP simd array to the corresponding simduid used
to index into it. Like thus,
@@ -1009,8 +984,9 @@ set_uid_loop_bbs (loop_vec_info loop_vinfo, gimple
*loop_vectorized_call,
/* Generate vectorized code for LOOP and its epilogues. */
-static unsigned
+unsigned
vect_transform_loops (hash_table<simduid_to_vf> *&simduid_to_vf_htab,
+ unsigned *num_vectorized_loops,
loop_p loop, gimple *loop_vectorized_call,
function *fun)
{
@@ -1039,8 +1015,9 @@ vect_transform_loops (hash_table<simduid_to_vf>
*&simduid_to_vf_htab,
? "epilogue " : "");
}
- loop_p new_loop = vect_transform_loop (loop_vinfo,
- loop_vectorized_call);
+ loop_p new_loop
+ = vect_transform_loop (loop_vinfo, loop_vectorized_call,
simduid_to_vf_htab,
+ num_vectorized_loops);
/* Now that the loop has been vectorized, allow it to be unrolled
etc. */
loop->force_vectorize = false;
@@ -1072,18 +1049,20 @@ vect_transform_loops (hash_table<simduid_to_vf>
*&simduid_to_vf_htab,
/* Epilogue of vectorized loop must be vectorized too. */
if (new_loop)
- todo |= vect_transform_loops (simduid_to_vf_htab, new_loop, NULL, fun);
+ todo |= vect_transform_loops (simduid_to_vf_htab, num_vectorized_loops,
+ new_loop, NULL, fun);
return todo;
}
/* Try to vectorize LOOP. */
-static unsigned
+unsigned
try_vectorize_loop_1 (hash_table<simduid_to_vf> *&simduid_to_vf_htab,
unsigned *num_vectorized_loops, loop_p loop,
gimple *loop_vectorized_call,
gimple *loop_dist_alias_call,
+ bool hssr_alignment_loop,
function *fun)
{
unsigned ret = 0;
@@ -1100,7 +1079,8 @@ try_vectorize_loop_1 (hash_table<simduid_to_vf>
*&simduid_to_vf_htab,
/* Try to analyze the loop, retaining an opt_problem if dump_enabled_p. */
opt_loop_vec_info loop_vinfo = vect_analyze_loop (loop, loop_vectorized_call,
- &shared);
+ &shared,
+ hssr_alignment_loop);
loop->aux = loop_vinfo;
if (!loop_vinfo)
@@ -1188,7 +1168,7 @@ try_vectorize_loop_1 (hash_table<simduid_to_vf>
*&simduid_to_vf_htab,
(*num_vectorized_loops)++;
/* Transform LOOP and its epilogues. */
- ret |= vect_transform_loops (simduid_to_vf_htab, loop,
+ ret |= vect_transform_loops (simduid_to_vf_htab, num_vectorized_loops, loop,
loop_vectorized_call, fun);
if (loop_vectorized_call)
@@ -1220,7 +1200,8 @@ try_vectorize_loop (hash_table<simduid_to_vf>
*&simduid_to_vf_htab,
return try_vectorize_loop_1 (simduid_to_vf_htab, num_vectorized_loops, loop,
vect_loop_vectorized_call (loop),
- vect_loop_dist_alias_call (loop, fun), fun);
+ vect_loop_dist_alias_call (loop, fun),
+ false, fun);
}
diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
index b8297d95e5d..040ccfd77d1 100644
--- a/gcc/tree-vectorizer.h
+++ b/gcc/tree-vectorizer.h
@@ -1189,6 +1189,8 @@ public:
bool can_use_hssr_p;
bool using_hssr_p;
+ bool use_hssr_unaligned_version_p;
+
/* True if we've decided to use partially-populated vectors, so that
the vector loop can handle fewer than VF scalars. */
bool using_partial_vectors_p;
@@ -1353,6 +1355,8 @@ public:
#define LOOP_VINFO_CAN_USE_HSSR_P(L) (L)->can_use_hssr_p
#define LOOP_VINFO_MUST_USE_HSSR_P(L) (L)->must_use_hssr_p
#define LOOP_VINFO_USING_HSSR_P(L) (L)->using_hssr_p
+#define LOOP_VINFO_USE_HSSR_UNALIGNED_VERSION_P(L) \
+ (L)->use_hssr_unaligned_version_p
#define LOOP_VINFO_USING_DECREMENTING_IV_P(L) (L)->using_decrementing_iv_p
#define LOOP_VINFO_USING_SELECT_VL_P(L) (L)->using_select_vl_p
#define LOOP_VINFO_ALLOW_MUTUAL_ALIGNMENT(L) (L)->allow_mutual_alignment
@@ -2559,6 +2563,7 @@ class loop *vect_loop_versioning (loop_vec_info, gimple
*);
extern bool vect_can_add_hssr_controls (loop_vec_info);
extern void vect_add_hssr_fixup_controls (loop_vec_info);
extern void vect_add_hssr_read (vec_info *, slp_instance);
+class loop *vect_loop_hssr_alias_versioning (loop_vec_info);
extern class loop *vect_do_peeling (loop_vec_info, tree, tree,
tree *, tree *, tree *, int, bool, bool,
tree *);
@@ -2744,7 +2749,7 @@ extern bool check_reduction_path (dump_user_location_t,
loop_p, gphi *, tree,
extern bool needs_fold_left_reduction_p (tree, code_helper);
/* Drive for loop analysis stage. */
extern opt_loop_vec_info vect_analyze_loop (class loop *, gimple *,
- vec_info_shared *);
+ vec_info_shared *, bool);
extern tree vect_build_loop_niters (loop_vec_info, bool * = NULL);
extern void vect_gen_vector_loop_niters (loop_vec_info, tree, tree *,
tree *, bool);
@@ -2769,8 +2774,40 @@ extern gimple_seq vect_gen_len (tree, tree, tree, tree);
extern vect_reduc_info info_for_reduction (loop_vec_info, slp_tree);
extern bool reduction_fn_for_scalar_code (code_helper, internal_fn *);
extern unsigned vect_min_prec_for_max_niters (loop_vec_info, unsigned int);
+/* For mapping simduid to vectorization factor. */
+
+class simduid_to_vf : public free_ptr_hash<simduid_to_vf>
+{
+public:
+ unsigned int simduid;
+ poly_uint64 vf;
+
+ /* hash_table support. */
+ static inline hashval_t hash (const simduid_to_vf *);
+ static inline int equal (const simduid_to_vf *, const simduid_to_vf *);
+};
+
+inline hashval_t
+simduid_to_vf::hash (const simduid_to_vf *p)
+{
+ return p->simduid;
+}
+
+inline int
+simduid_to_vf::equal (const simduid_to_vf *p1, const simduid_to_vf *p2)
+{
+ return p1->simduid == p2->simduid;
+}
+
/* Drive for loop transformation stage. */
-extern class loop *vect_transform_loop (loop_vec_info, gimple *);
+extern class loop *vect_transform_loop (loop_vec_info, gimple *,
+ hash_table<simduid_to_vf> *&,
+ unsigned *);
+
+extern unsigned
+try_vectorize_loop_1 (hash_table<simduid_to_vf> *&, unsigned *, loop_p,
+ gimple *, gimple *, bool, function *);
+
struct vect_loop_form_info
{
tree number_of_iterations;
--
2.43.0
