This patch adds a vec_basic_block that records the scalar phis and
scalar statements that we need to vectorise. This is a slight
simplification in its own right, since it avoids unnecesary statement
lookups and shaves >50 LOC. But the main reason for doing it is
to allow the final patch in the series to treat pattern statements
less specially.
Putting phis (which are logically parallel) and normal statements
(which are logically serial) into a single list might seem dangerous,
but I think in practice it should be fine. Very little vectoriser
code needs to handle the parallel nature of phis specially, and code
that does can still do so. Having a single list simplifies code that
wants to look at every scalar phi or stmt in isolation.
2018-07-30 Richard Sandiford <richard.sandif...@arm.com>
gcc/
* tree-vectorizer.h (vec_basic_block): New structure.
(vec_info::blocks, _stmt_vec_info::block, _stmt_vec_info::prev)
(_stmt_vec_info::next): New member variables.
(FOR_EACH_VEC_BB_STMT, FOR_EACH_VEC_BB_STMT_REVERSE): New macros.
(vec_basic_block::vec_basic_block): New function.
* tree-vectorizer.c (vec_basic_block::add_to_end): Likewise.
(vec_basic_block::add_before): Likewise.
(vec_basic_block::remove): Likewise.
(vec_info::~vec_info): Free the vec_basic_blocks.
(vec_info::remove_stmt): Remove the statement from the containing
vec_basic_block.
* tree-vect-patterns.c (vect_determine_precisions)
(vect_pattern_recog): Iterate over vec_basic_blocks.
* tree-vect-loop.c (vect_determine_vectorization_factor)
(vect_compute_single_scalar_iteration_cost, vect_update_vf_for_slp)
(vect_analyze_loop_operations, vect_transform_loop): Likewise.
(_loop_vec_info::_loop_vec_info): Construct vec_basic_blocks.
* tree-vect-slp.c (_bb_vec_info::_bb_vec_info): Likewise.
(vect_detect_hybrid_slp): Iterate over vec_basic_blocks.
* tree-vect-stmts.c (vect_mark_stmts_to_be_vectorized): Likewise.
(vect_finish_replace_stmt, vectorizable_condition): Remove the original
statement from the containing block.
(hoist_defs_of_uses): Likewise the statement that we're hoisting.
Index: gcc/tree-vectorizer.h
===================================================================
*** gcc/tree-vectorizer.h 2018-07-30 12:43:34.512651826 +0100
--- gcc/tree-vectorizer.h 2018-07-30 12:43:34.508651861 +0100
*************** #define SLP_TREE_LOAD_PERMUTATION(S)
*** 171,177 ****
--- 171,200 ----
#define SLP_TREE_TWO_OPERATORS(S) (S)->two_operators
#define SLP_TREE_DEF_TYPE(S) (S)->def_type
+ /* Information about the phis and statements in a block that we're trying
+ to vectorize, in their original order. */
+ class vec_basic_block
+ {
+ public:
+ vec_basic_block (basic_block);
+
+ void add_to_end (stmt_vec_info);
+ void add_before (stmt_vec_info, stmt_vec_info);
+ void remove (stmt_vec_info);
+
+ basic_block bb () const { return m_bb; }
+ stmt_vec_info first () const { return m_first; }
+ stmt_vec_info last () const { return m_last; }
+
+ private:
+ /* The block itself. */
+ basic_block m_bb;
+ /* The first and last statements in the block, forming a double-linked list.
+ The list includes both phis and true statements. */
+ stmt_vec_info m_first;
+ stmt_vec_info m_last;
+ };
/* Describes two objects whose addresses must be unequal for the vectorized
loop to be valid. */
*************** struct vec_info {
*** 249,254 ****
--- 272,280 ----
/* Cost data used by the target cost model. */
void *target_cost_data;
+ /* The basic blocks in the vectorization region. */
+ auto_vec<vec_basic_block *, 5> blocks;
+
private:
stmt_vec_info new_stmt_vec_info (gimple *stmt);
void set_vinfo_for_stmt (gimple *, stmt_vec_info);
*************** struct dr_vec_info {
*** 776,781 ****
--- 802,812 ----
typedef struct data_reference *dr_p;
struct _stmt_vec_info {
+ /* The block to which the statement belongs, or null if none. */
+ vec_basic_block *block;
+
+ /* Link chains for the previous and next statements in BLOCK. */
+ stmt_vec_info prev, next;
enum stmt_vec_info_type type;
*************** #define VECT_SCALAR_BOOLEAN_TYPE_P(TYPE)
*** 1072,1077 ****
--- 1103,1129 ----
&& TYPE_PRECISION (TYPE) == 1 \
&& TYPE_UNSIGNED (TYPE)))
+ /* Make STMT_INFO iterate over each statement in vec_basic_block VEC_BB
+ in forward order. */
+
+ #define FOR_EACH_VEC_BB_STMT(VEC_BB, STMT_INFO) \
+ for (stmt_vec_info STMT_INFO = (VEC_BB)->first (); STMT_INFO; \
+ STMT_INFO = STMT_INFO->next)
+
+ /* Make STMT_INFO iterate over each statement in vec_basic_block VEC_BB
+ in backward order. */
+
+ #define FOR_EACH_VEC_BB_STMT_REVERSE(VEC_BB, STMT_INFO) \
+ for (stmt_vec_info STMT_INFO = (VEC_BB)->last (); STMT_INFO; \
+ STMT_INFO = STMT_INFO->prev)
+
+ /* Construct a vec_basic_block for BB. */
+
+ inline vec_basic_block::vec_basic_block (basic_block bb)
+ : m_bb (bb), m_first (NULL), m_last (NULL)
+ {
+ }
+
static inline bool
nested_in_vect_loop_p (struct loop *loop, stmt_vec_info stmt_info)
{
Index: gcc/tree-vectorizer.c
===================================================================
*** gcc/tree-vectorizer.c 2018-07-30 12:43:34.512651826 +0100
--- gcc/tree-vectorizer.c 2018-07-30 12:43:34.508651861 +0100
*************** note_simd_array_uses (hash_table<simd_ar
*** 444,449 ****
--- 444,504 ----
delete simd_array_to_simduid_htab;
}
+ /* Add STMT_INFO to the end of the block. */
+
+ void
+ vec_basic_block::add_to_end (stmt_vec_info stmt_info)
+ {
+ gcc_checking_assert (!stmt_info->block
+ && !stmt_info->prev
+ && !stmt_info->next);
+ if (m_last)
+ m_last->next = stmt_info;
+ else
+ m_first = stmt_info;
+ stmt_info->block = this;
+ stmt_info->prev = m_last;
+ m_last = stmt_info;
+ }
+
+ /* Add STMT_INFO to the block, inserting it before NEXT_STMT_INFO. */
+
+ void
+ vec_basic_block::add_before (stmt_vec_info stmt_info,
+ stmt_vec_info next_stmt_info)
+ {
+ gcc_checking_assert (!stmt_info->block
+ && !stmt_info->prev
+ && !stmt_info->next
+ && next_stmt_info->block == this);
+ if (next_stmt_info->prev)
+ next_stmt_info->prev->next = stmt_info;
+ else
+ m_first = stmt_info;
+ stmt_info->block = this;
+ stmt_info->prev = next_stmt_info->prev;
+ stmt_info->next = next_stmt_info;
+ next_stmt_info->prev = stmt_info;
+ }
+
+ /* Remove STMT_INFO from the block. */
+
+ void
+ vec_basic_block::remove (stmt_vec_info stmt_info)
+ {
+ gcc_checking_assert (stmt_info->block == this);
+ if (stmt_info->prev)
+ stmt_info->prev->next = stmt_info->next;
+ else
+ m_first = stmt_info->next;
+ if (stmt_info->next)
+ stmt_info->next->prev = stmt_info->prev;
+ else
+ m_last = stmt_info->prev;
+ stmt_info->block = NULL;
+ stmt_info->prev = stmt_info->next = NULL;
+ }
+
/* Initialize the vec_info with kind KIND_IN and target cost data
TARGET_COST_DATA_IN. */
*************** vec_info::vec_info (vec_info::vec_kind k
*** 459,466 ****
--- 514,525 ----
vec_info::~vec_info ()
{
slp_instance instance;
+ vec_basic_block *vec_bb;
unsigned int i;
+ FOR_EACH_VEC_ELT (blocks, i, vec_bb)
+ delete vec_bb;
+
FOR_EACH_VEC_ELT (slp_instances, i, instance)
vect_free_slp_instance (instance, true);
*************** vec_info::remove_stmt (stmt_vec_info stm
*** 596,601 ****
--- 655,661 ----
unlink_stmt_vdef (stmt_info->stmt);
gsi_remove (&si, true);
release_defs (stmt_info->stmt);
+ stmt_info->block->remove (stmt_info);
free_stmt_vec_info (stmt_info);
}
Index: gcc/tree-vect-patterns.c
===================================================================
*** gcc/tree-vect-patterns.c 2018-07-30 12:43:34.512651826 +0100
--- gcc/tree-vect-patterns.c 2018-07-30 12:43:34.504651897 +0100
*************** vect_determine_precisions (vec_info *vin
*** 4631,4669 ****
{
DUMP_VECT_SCOPE ("vect_determine_precisions");
! if (loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo))
! {
! struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
! basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
! unsigned int nbbs = loop->num_nodes;
!
! for (unsigned int i = 0; i < nbbs; i++)
! {
! basic_block bb = bbs[nbbs - i - 1];
! for (gimple_stmt_iterator si = gsi_last_bb (bb);
! !gsi_end_p (si); gsi_prev (&si))
! vect_determine_stmt_precisions
! (vinfo->lookup_stmt (gsi_stmt (si)));
! }
! }
! else
! {
! bb_vec_info bb_vinfo = as_a <bb_vec_info> (vinfo);
! gimple_stmt_iterator si = bb_vinfo->region_end;
! gimple *stmt;
! do
! {
! if (!gsi_stmt (si))
! si = gsi_last_bb (bb_vinfo->bb);
! else
! gsi_prev (&si);
! stmt = gsi_stmt (si);
! stmt_vec_info stmt_info = vinfo->lookup_stmt (stmt);
! if (stmt_info && STMT_VINFO_VECTORIZABLE (stmt_info))
! vect_determine_stmt_precisions (stmt_info);
! }
! while (stmt != gsi_stmt (bb_vinfo->region_begin));
! }
}
typedef gimple *(*vect_recog_func_ptr) (stmt_vec_info, tree *);
--- 4631,4641 ----
{
DUMP_VECT_SCOPE ("vect_determine_precisions");
! unsigned int i;
! vec_basic_block *vec_bb;
! FOR_EACH_VEC_ELT_REVERSE (vinfo->blocks, i, vec_bb)
! FOR_EACH_VEC_BB_STMT_REVERSE (vec_bb, stmt_info)
! vect_determine_stmt_precisions (stmt_info);
}
typedef gimple *(*vect_recog_func_ptr) (stmt_vec_info, tree *);
*************** vect_pattern_recog_1 (vect_recog_func *r
*** 4923,4973 ****
void
vect_pattern_recog (vec_info *vinfo)
{
- struct loop *loop;
- basic_block *bbs;
- unsigned int nbbs;
- gimple_stmt_iterator si;
- unsigned int i, j;
-
vect_determine_precisions (vinfo);
DUMP_VECT_SCOPE ("vect_pattern_recog");
! if (loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo))
! {
! loop = LOOP_VINFO_LOOP (loop_vinfo);
! bbs = LOOP_VINFO_BBS (loop_vinfo);
! nbbs = loop->num_nodes;
!
! /* Scan through the loop stmts, applying the pattern recognition
! functions starting at each stmt visited: */
! for (i = 0; i < nbbs; i++)
! {
! basic_block bb = bbs[i];
! for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
! {
! stmt_vec_info stmt_info = vinfo->lookup_stmt (gsi_stmt (si));
! /* Scan over all generic vect_recog_xxx_pattern functions. */
! for (j = 0; j < NUM_PATTERNS; j++)
! vect_pattern_recog_1 (&vect_vect_recog_func_ptrs[j],
! stmt_info);
! }
! }
! }
! else
! {
! bb_vec_info bb_vinfo = as_a <bb_vec_info> (vinfo);
! for (si = bb_vinfo->region_begin;
! gsi_stmt (si) != gsi_stmt (bb_vinfo->region_end); gsi_next (&si))
! {
! gimple *stmt = gsi_stmt (si);
! stmt_vec_info stmt_info = bb_vinfo->lookup_stmt (stmt);
! if (stmt_info && !STMT_VINFO_VECTORIZABLE (stmt_info))
! continue;
!
! /* Scan over all generic vect_recog_xxx_pattern functions. */
! for (j = 0; j < NUM_PATTERNS; j++)
! vect_pattern_recog_1 (&vect_vect_recog_func_ptrs[j], stmt_info);
! }
! }
}
--- 4895,4910 ----
void
vect_pattern_recog (vec_info *vinfo)
{
vect_determine_precisions (vinfo);
DUMP_VECT_SCOPE ("vect_pattern_recog");
! unsigned int i;
! vec_basic_block *vec_bb;
! FOR_EACH_VEC_ELT (vinfo->blocks, i, vec_bb)
! FOR_EACH_VEC_BB_STMT (vec_bb, stmt_info)
! if (STMT_VINFO_VECTORIZABLE (stmt_info))
! /* Scan over all generic vect_recog_xxx_pattern functions. */
! for (unsigned int j = 0; j < NUM_PATTERNS; j++)
! vect_pattern_recog_1 (&vect_vect_recog_func_ptrs[j], stmt_info);
}
Index: gcc/tree-vect-loop.c
===================================================================
*** gcc/tree-vect-loop.c 2018-07-30 12:43:34.512651826 +0100
--- gcc/tree-vect-loop.c 2018-07-30 12:43:34.500651932 +0100
*************** vect_determine_vf_for_stmt (stmt_vec_inf
*** 286,321 ****
static bool
vect_determine_vectorization_factor (loop_vec_info loop_vinfo)
{
- struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
- basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
- unsigned nbbs = loop->num_nodes;
poly_uint64 vectorization_factor = 1;
tree scalar_type = NULL_TREE;
- gphi *phi;
tree vectype;
stmt_vec_info stmt_info;
unsigned i;
auto_vec<stmt_vec_info> mask_producers;
DUMP_VECT_SCOPE ("vect_determine_vectorization_factor");
! for (i = 0; i < nbbs; i++)
! {
! basic_block bb = bbs[i];
!
! for (gphi_iterator si = gsi_start_phis (bb); !gsi_end_p (si);
! gsi_next (&si))
{
- phi = si.phi ();
- stmt_info = loop_vinfo->lookup_stmt (phi);
if (dump_enabled_p ())
{
dump_printf_loc (MSG_NOTE, vect_location, "==> examining phi: ");
dump_gimple_stmt (MSG_NOTE, TDF_SLIM, phi, 0);
}
- gcc_assert (stmt_info);
-
if (STMT_VINFO_RELEVANT_P (stmt_info)
|| STMT_VINFO_LIVE_P (stmt_info))
{
--- 286,311 ----
static bool
vect_determine_vectorization_factor (loop_vec_info loop_vinfo)
{
poly_uint64 vectorization_factor = 1;
tree scalar_type = NULL_TREE;
tree vectype;
stmt_vec_info stmt_info;
unsigned i;
auto_vec<stmt_vec_info> mask_producers;
+ vec_basic_block *vec_bb;
DUMP_VECT_SCOPE ("vect_determine_vectorization_factor");
! FOR_EACH_VEC_ELT (loop_vinfo->blocks, i, vec_bb)
! FOR_EACH_VEC_BB_STMT (vec_bb, stmt_info)
! if (gphi *phi = dyn_cast <gphi *> (stmt_info->stmt))
{
if (dump_enabled_p ())
{
dump_printf_loc (MSG_NOTE, vect_location, "==> examining phi: ");
dump_gimple_stmt (MSG_NOTE, TDF_SLIM, phi, 0);
}
if (STMT_VINFO_RELEVANT_P (stmt_info)
|| STMT_VINFO_LIVE_P (stmt_info))
{
*************** vect_determine_vectorization_factor (loo
*** 363,378 ****
vect_update_max_nunits (&vectorization_factor, vectype);
}
}
!
! for (gimple_stmt_iterator si = gsi_start_bb (bb); !gsi_end_p (si);
! gsi_next (&si))
! {
! stmt_info = loop_vinfo->lookup_stmt (gsi_stmt (si));
! if (!vect_determine_vf_for_stmt (stmt_info, &vectorization_factor,
! &mask_producers))
! return false;
! }
! }
/* TODO: Analyze cost. Decide if worth while to vectorize. */
if (dump_enabled_p ())
--- 353,361 ----
vect_update_max_nunits (&vectorization_factor, vectype);
}
}
! else if (!vect_determine_vf_for_stmt (stmt_info, &vectorization_factor,
! &mask_producers))
! return false;
/* TODO: Analyze cost. Decide if worth while to vectorize. */
if (dump_enabled_p ())
*************** _loop_vec_info::_loop_vec_info (struct l
*** 846,866 ****
for (unsigned int i = 0; i < nbbs; i++)
{
basic_block bb = bbs[i];
gimple_stmt_iterator si;
for (si = gsi_start_phis (bb); !gsi_end_p (si); gsi_next (&si))
{
gimple *phi = gsi_stmt (si);
gimple_set_uid (phi, 0);
! add_stmt (phi);
}
for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
{
gimple *stmt = gsi_stmt (si);
gimple_set_uid (stmt, 0);
! add_stmt (stmt);
}
}
}
--- 829,851 ----
for (unsigned int i = 0; i < nbbs; i++)
{
basic_block bb = bbs[i];
+ vec_basic_block *vec_bb = new vec_basic_block (bb);
gimple_stmt_iterator si;
for (si = gsi_start_phis (bb); !gsi_end_p (si); gsi_next (&si))
{
gimple *phi = gsi_stmt (si);
gimple_set_uid (phi, 0);
! vec_bb->add_to_end (add_stmt (phi));
}
for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
{
gimple *stmt = gsi_stmt (si);
gimple_set_uid (stmt, 0);
! vec_bb->add_to_end (add_stmt (stmt));
}
+ blocks.safe_push (vec_bb);
}
}
*************** vect_verify_full_masking (loop_vec_info
*** 1066,1074 ****
vect_compute_single_scalar_iteration_cost (loop_vec_info loop_vinfo)
{
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
! basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
! int nbbs = loop->num_nodes, factor;
! int innerloop_iters, i;
/* Gather costs for statements in the scalar loop. */
--- 1051,1058 ----
vect_compute_single_scalar_iteration_cost (loop_vec_info loop_vinfo)
{
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
! int factor, innerloop_iters;
! unsigned int i;
/* Gather costs for statements in the scalar loop. */
*************** vect_compute_single_scalar_iteration_cos
*** 1077,1099 ****
if (loop->inner)
innerloop_iters = 50; /* FIXME */
! for (i = 0; i < nbbs; i++)
{
! gimple_stmt_iterator si;
! basic_block bb = bbs[i];
!
! if (bb->loop_father == loop->inner)
factor = innerloop_iters;
else
factor = 1;
! for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
! {
! gimple *stmt = gsi_stmt (si);
! stmt_vec_info stmt_info = loop_vinfo->lookup_stmt (stmt);
!
! if (!is_gimple_assign (stmt) && !is_gimple_call (stmt))
! continue;
/* Skip stmts that are not vectorized inside the loop. */
if (stmt_info
--- 1061,1079 ----
if (loop->inner)
innerloop_iters = 50; /* FIXME */
! vec_basic_block *vec_bb;
! FOR_EACH_VEC_ELT (loop_vinfo->blocks, i, vec_bb)
{
! if (vec_bb->bb ()->loop_father == loop->inner)
factor = innerloop_iters;
else
factor = 1;
! FOR_EACH_VEC_BB_STMT (vec_bb, stmt_info)
! {
! if (!is_gimple_assign (stmt_info->stmt)
! && !is_gimple_call (stmt_info->stmt))
! continue;
/* Skip stmts that are not vectorized inside the loop. */
if (stmt_info
*************** vect_analyze_loop_form (struct loop *loo
*** 1397,1407 ****
static void
vect_update_vf_for_slp (loop_vec_info loop_vinfo)
{
- struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
- basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
- int nbbs = loop->num_nodes;
poly_uint64 vectorization_factor;
! int i;
DUMP_VECT_SCOPE ("vect_update_vf_for_slp");
--- 1377,1384 ----
static void
vect_update_vf_for_slp (loop_vec_info loop_vinfo)
{
poly_uint64 vectorization_factor;
! unsigned int i;
DUMP_VECT_SCOPE ("vect_update_vf_for_slp");
*************** vect_update_vf_for_slp (loop_vec_info lo
*** 1414,1434 ****
perform pure SLP on loop - cross iteration parallelism is not
exploited. */
bool only_slp_in_loop = true;
! for (i = 0; i < nbbs; i++)
! {
! basic_block bb = bbs[i];
! for (gimple_stmt_iterator si = gsi_start_bb (bb); !gsi_end_p (si);
! gsi_next (&si))
! {
! stmt_vec_info stmt_info = loop_vinfo->lookup_stmt (gsi_stmt (si));
! stmt_info = vect_stmt_to_vectorize (stmt_info);
! if ((STMT_VINFO_RELEVANT_P (stmt_info)
! || VECTORIZABLE_CYCLE_DEF (STMT_VINFO_DEF_TYPE (stmt_info)))
! && !PURE_SLP_STMT (stmt_info))
! /* STMT needs both SLP and loop-based vectorization. */
! only_slp_in_loop = false;
! }
! }
if (only_slp_in_loop)
{
--- 1391,1407 ----
perform pure SLP on loop - cross iteration parallelism is not
exploited. */
bool only_slp_in_loop = true;
! vec_basic_block *vec_bb;
! FOR_EACH_VEC_ELT (loop_vinfo->blocks, i, vec_bb)
! FOR_EACH_VEC_BB_STMT (vec_bb, stmt_info)
! {
! stmt_vec_info final_info = vect_stmt_to_vectorize (stmt_info);
! if ((STMT_VINFO_RELEVANT_P (final_info)
! || VECTORIZABLE_CYCLE_DEF (STMT_VINFO_DEF_TYPE (final_info)))
! && !PURE_SLP_STMT (final_info))
! /* STMT needs both SLP and loop-based vectorization. */
! only_slp_in_loop = false;
! }
if (only_slp_in_loop)
{
*************** vect_active_double_reduction_p (stmt_vec
*** 1491,1501 ****
static bool
vect_analyze_loop_operations (loop_vec_info loop_vinfo)
{
! struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
! basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
! int nbbs = loop->num_nodes;
! int i;
! stmt_vec_info stmt_info;
bool need_to_vectorize = false;
bool ok;
--- 1464,1470 ----
static bool
vect_analyze_loop_operations (loop_vec_info loop_vinfo)
{
! unsigned int i;
bool need_to_vectorize = false;
bool ok;
*************** vect_analyze_loop_operations (loop_vec_i
*** 1504,1520 ****
stmt_vector_for_cost cost_vec;
cost_vec.create (2);
! for (i = 0; i < nbbs; i++)
! {
! basic_block bb = bbs[i];
!
! for (gphi_iterator si = gsi_start_phis (bb); !gsi_end_p (si);
! gsi_next (&si))
{
- gphi *phi = si.phi ();
ok = true;
-
- stmt_info = loop_vinfo->lookup_stmt (phi);
if (dump_enabled_p ())
{
dump_printf_loc (MSG_NOTE, vect_location, "examining phi: ");
--- 1473,1484 ----
stmt_vector_for_cost cost_vec;
cost_vec.create (2);
! vec_basic_block *vec_bb;
! FOR_EACH_VEC_ELT (loop_vinfo->blocks, i, vec_bb)
! FOR_EACH_VEC_BB_STMT (vec_bb, stmt_info)
! if (gphi *phi = dyn_cast <gphi *> (stmt_info->stmt))
{
ok = true;
if (dump_enabled_p ())
{
dump_printf_loc (MSG_NOTE, vect_location, "examining phi: ");
*************** vect_analyze_loop_operations (loop_vec_i
*** 1525,1531 ****
/* Inner-loop loop-closed exit phi in outer-loop vectorization
(i.e., a phi in the tail of the outer-loop). */
! if (! is_loop_header_bb_p (bb))
{
/* FORNOW: we currently don't support the case that these phis
are not used in the outerloop (unless it is double reduction,
--- 1489,1495 ----
/* Inner-loop loop-closed exit phi in outer-loop vectorization
(i.e., a phi in the tail of the outer-loop). */
! if (! is_loop_header_bb_p (vec_bb->bb ()))
{
/* FORNOW: we currently don't support the case that these phis
are not used in the outerloop (unless it is double reduction,
*************** vect_analyze_loop_operations (loop_vec_i
*** 1564,1571 ****
continue;
}
- gcc_assert (stmt_info);
-
if ((STMT_VINFO_RELEVANT (stmt_info) == vect_used_in_scope
|| STMT_VINFO_LIVE_P (stmt_info))
&& STMT_VINFO_DEF_TYPE (stmt_info) != vect_induction_def)
--- 1528,1533 ----
*************** vect_analyze_loop_operations (loop_vec_i
*** 1610,1627 ****
return false;
}
}
!
! for (gimple_stmt_iterator si = gsi_start_bb (bb); !gsi_end_p (si);
! gsi_next (&si))
! {
! gimple *stmt = gsi_stmt (si);
! if (!gimple_clobber_p (stmt)
! && !vect_analyze_stmt (loop_vinfo->lookup_stmt (stmt),
! &need_to_vectorize,
! NULL, NULL, &cost_vec))
! return false;
! }
! } /* bbs */
add_stmt_costs (loop_vinfo->target_cost_data, &cost_vec);
cost_vec.release ();
--- 1572,1581 ----
return false;
}
}
! else if (!gimple_clobber_p (stmt_info->stmt)
! && !vect_analyze_stmt (stmt_info, &need_to_vectorize,
! NULL, NULL, &cost_vec))
! return false;
add_stmt_costs (loop_vinfo->target_cost_data, &cost_vec);
cost_vec.release ();
*************** vect_analyze_loop_2 (loop_vec_info loop_
*** 2207,2238 ****
vect_free_slp_instance (instance, false);
LOOP_VINFO_SLP_INSTANCES (loop_vinfo).release ();
/* Reset SLP type to loop_vect on all stmts. */
! for (i = 0; i < LOOP_VINFO_LOOP (loop_vinfo)->num_nodes; ++i)
! {
! basic_block bb = LOOP_VINFO_BBS (loop_vinfo)[i];
! for (gimple_stmt_iterator si = gsi_start_phis (bb);
! !gsi_end_p (si); gsi_next (&si))
! {
! stmt_vec_info stmt_info = loop_vinfo->lookup_stmt (gsi_stmt (si));
! STMT_SLP_TYPE (stmt_info) = loop_vect;
! }
! for (gimple_stmt_iterator si = gsi_start_bb (bb);
! !gsi_end_p (si); gsi_next (&si))
! {
! stmt_vec_info stmt_info = loop_vinfo->lookup_stmt (gsi_stmt (si));
! STMT_SLP_TYPE (stmt_info) = loop_vect;
! if (STMT_VINFO_IN_PATTERN_P (stmt_info))
! {
! gimple *pattern_def_seq = STMT_VINFO_PATTERN_DEF_SEQ (stmt_info);
! stmt_info = STMT_VINFO_RELATED_STMT (stmt_info);
! STMT_SLP_TYPE (stmt_info) = loop_vect;
! for (gimple_stmt_iterator pi = gsi_start (pattern_def_seq);
! !gsi_end_p (pi); gsi_next (&pi))
! STMT_SLP_TYPE (loop_vinfo->lookup_stmt (gsi_stmt (pi)))
! = loop_vect;
! }
! }
! }
/* Free optimized alias test DDRS. */
LOOP_VINFO_LOWER_BOUNDS (loop_vinfo).truncate (0);
LOOP_VINFO_COMP_ALIAS_DDRS (loop_vinfo).release ();
--- 2161,2182 ----
vect_free_slp_instance (instance, false);
LOOP_VINFO_SLP_INSTANCES (loop_vinfo).release ();
/* Reset SLP type to loop_vect on all stmts. */
! vec_basic_block *vec_bb;
! FOR_EACH_VEC_ELT (loop_vinfo->blocks, i, vec_bb)
! FOR_EACH_VEC_BB_STMT (vec_bb, stmt_info)
! {
! STMT_SLP_TYPE (stmt_info) = loop_vect;
! if (STMT_VINFO_IN_PATTERN_P (stmt_info))
! {
! gimple *pattern_def_seq = STMT_VINFO_PATTERN_DEF_SEQ (stmt_info);
! STMT_SLP_TYPE (STMT_VINFO_RELATED_STMT (stmt_info)) = loop_vect;
! for (gimple_stmt_iterator pi = gsi_start (pattern_def_seq);
! !gsi_end_p (pi); gsi_next (&pi))
! STMT_SLP_TYPE (loop_vinfo->lookup_stmt (gsi_stmt (pi)))
! = loop_vect;
! }
! }
!
/* Free optimized alias test DDRS. */
LOOP_VINFO_LOWER_BOUNDS (loop_vinfo).truncate (0);
LOOP_VINFO_COMP_ALIAS_DDRS (loop_vinfo).release ();
*************** vect_transform_loop (loop_vec_info loop_
*** 8237,8251 ****
{
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
struct loop *epilogue = NULL;
! basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
! int nbbs = loop->num_nodes;
! int i;
tree niters_vector = NULL_TREE;
tree step_vector = NULL_TREE;
tree niters_vector_mult_vf = NULL_TREE;
poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
unsigned int lowest_vf = constant_lower_bound (vf);
- gimple *stmt;
bool check_profitability = false;
unsigned int th;
--- 8181,8192 ----
{
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
struct loop *epilogue = NULL;
! unsigned int i;
tree niters_vector = NULL_TREE;
tree step_vector = NULL_TREE;
tree niters_vector_mult_vf = NULL_TREE;
poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
unsigned int lowest_vf = constant_lower_bound (vf);
bool check_profitability = false;
unsigned int th;
*************** vect_transform_loop (loop_vec_info loop_
*** 8363,8452 ****
support more involved loop forms, the order by which the BBs are
traversed need to be reconsidered. */
! for (i = 0; i < nbbs; i++)
{
! basic_block bb = bbs[i];
! stmt_vec_info stmt_info;
!
! for (gphi_iterator si = gsi_start_phis (bb); !gsi_end_p (si);
! gsi_next (&si))
! {
! gphi *phi = si.phi ();
! if (dump_enabled_p ())
{
! dump_printf_loc (MSG_NOTE, vect_location,
! "------>vectorizing phi: ");
! dump_gimple_stmt (MSG_NOTE, TDF_SLIM, phi, 0);
! }
! stmt_info = loop_vinfo->lookup_stmt (phi);
! if (!stmt_info)
! continue;
! if (MAY_HAVE_DEBUG_BIND_STMTS && !STMT_VINFO_LIVE_P (stmt_info))
! vect_loop_kill_debug_uses (loop, stmt_info);
! if (!STMT_VINFO_RELEVANT_P (stmt_info)
! && !STMT_VINFO_LIVE_P (stmt_info))
! continue;
!
! if (STMT_VINFO_VECTYPE (stmt_info)
! && (maybe_ne
! (TYPE_VECTOR_SUBPARTS (STMT_VINFO_VECTYPE (stmt_info)), vf))
! && dump_enabled_p ())
! dump_printf_loc (MSG_NOTE, vect_location, "multiple-types.\n");
!
! if ((STMT_VINFO_DEF_TYPE (stmt_info) == vect_induction_def
! || STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def
! || STMT_VINFO_DEF_TYPE (stmt_info) == vect_nested_cycle)
! && ! PURE_SLP_STMT (stmt_info))
! {
! if (dump_enabled_p ())
! dump_printf_loc (MSG_NOTE, vect_location, "transform phi.\n");
! vect_transform_stmt (stmt_info, NULL, NULL, NULL);
}
- }
-
- for (gimple_stmt_iterator si = gsi_start_bb (bb);
- !gsi_end_p (si);)
- {
- stmt = gsi_stmt (si);
/* During vectorization remove existing clobber stmts. */
! if (gimple_clobber_p (stmt))
! {
! unlink_stmt_vdef (stmt);
! gsi_remove (&si, true);
! release_defs (stmt);
! }
else
{
- stmt_info = loop_vinfo->lookup_stmt (stmt);
-
- /* vector stmts created in the outer-loop during vectorization of
- stmts in an inner-loop may not have a stmt_info, and do not
- need to be vectorized. */
stmt_vec_info seen_store = NULL;
! if (stmt_info)
{
! if (STMT_VINFO_IN_PATTERN_P (stmt_info))
{
- gimple *def_seq = STMT_VINFO_PATTERN_DEF_SEQ (stmt_info);
- for (gimple_stmt_iterator subsi = gsi_start (def_seq);
- !gsi_end_p (subsi); gsi_next (&subsi))
- {
- stmt_vec_info pat_stmt_info
- = loop_vinfo->lookup_stmt (gsi_stmt (subsi));
- vect_transform_loop_stmt (loop_vinfo, pat_stmt_info,
- &si, &seen_store);
- }
stmt_vec_info pat_stmt_info
! = STMT_VINFO_RELATED_STMT (stmt_info);
! vect_transform_loop_stmt (loop_vinfo, pat_stmt_info, &si,
! &seen_store);
}
! vect_transform_loop_stmt (loop_vinfo, stmt_info, &si,
&seen_store);
}
! gsi_next (&si);
if (seen_store)
{
if (STMT_VINFO_GROUPED_ACCESS (seen_store))
--- 8304,8376 ----
support more involved loop forms, the order by which the BBs are
traversed need to be reconsidered. */
! vec_basic_block *vec_bb;
! FOR_EACH_VEC_ELT (loop_vinfo->blocks, i, vec_bb)
{
! stmt_vec_info next_stmt_info;
! for (stmt_vec_info stmt_info = vec_bb->first (); stmt_info;
! stmt_info = next_stmt_info)
! {
! next_stmt_info = stmt_info->next;
! if (gphi *phi = dyn_cast <gphi *> (stmt_info->stmt))
{
! if (dump_enabled_p ())
! {
! dump_printf_loc (MSG_NOTE, vect_location,
! "------>vectorizing phi: ");
! dump_gimple_stmt (MSG_NOTE, TDF_SLIM, phi, 0);
! }
! if (MAY_HAVE_DEBUG_BIND_STMTS && !STMT_VINFO_LIVE_P (stmt_info))
! vect_loop_kill_debug_uses (loop, stmt_info);
! if (!STMT_VINFO_RELEVANT_P (stmt_info)
! && !STMT_VINFO_LIVE_P (stmt_info))
! continue;
!
! if (STMT_VINFO_VECTYPE (stmt_info)
! && (maybe_ne
! (TYPE_VECTOR_SUBPARTS (STMT_VINFO_VECTYPE (stmt_info)),
! vf))
! && dump_enabled_p ())
! dump_printf_loc (MSG_NOTE, vect_location, "multiple-types.\n");
!
! if ((STMT_VINFO_DEF_TYPE (stmt_info) == vect_induction_def
! || STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def
! || STMT_VINFO_DEF_TYPE (stmt_info) == vect_nested_cycle)
! && ! PURE_SLP_STMT (stmt_info))
! {
! if (dump_enabled_p ())
! dump_printf_loc (MSG_NOTE, vect_location,
! "transform phi.\n");
! vect_transform_stmt (stmt_info, NULL, NULL, NULL);
! }
}
/* During vectorization remove existing clobber stmts. */
! else if (gimple_clobber_p (stmt_info->stmt))
! loop_vinfo->remove_stmt (stmt_info);
else
{
stmt_vec_info seen_store = NULL;
! gimple_stmt_iterator si = gsi_for_stmt (stmt_info->stmt);
! if (STMT_VINFO_IN_PATTERN_P (stmt_info))
{
! gimple *def_seq = STMT_VINFO_PATTERN_DEF_SEQ (stmt_info);
! for (gimple_stmt_iterator subsi = gsi_start (def_seq);
! !gsi_end_p (subsi); gsi_next (&subsi))
{
stmt_vec_info pat_stmt_info
! = loop_vinfo->lookup_stmt (gsi_stmt (subsi));
! vect_transform_loop_stmt (loop_vinfo, pat_stmt_info,
! &si, &seen_store);
}
! stmt_vec_info pat_stmt_info
! = STMT_VINFO_RELATED_STMT (stmt_info);
! vect_transform_loop_stmt (loop_vinfo, pat_stmt_info, &si,
&seen_store);
}
! vect_transform_loop_stmt (loop_vinfo, stmt_info, &si,
! &seen_store);
if (seen_store)
{
if (STMT_VINFO_GROUPED_ACCESS (seen_store))
*************** vect_transform_loop (loop_vec_info loop_
*** 8464,8470 ****
/* Stub out scalar statements that must not survive vectorization.
Doing this here helps with grouped statements, or statements that
are involved in patterns. */
! for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
!gsi_end_p (gsi); gsi_next (&gsi))
{
gcall *call = dyn_cast <gcall *> (gsi_stmt (gsi));
--- 8388,8394 ----
/* Stub out scalar statements that must not survive vectorization.
Doing this here helps with grouped statements, or statements that
are involved in patterns. */
! for (gimple_stmt_iterator gsi = gsi_start_bb (vec_bb->bb ());
!gsi_end_p (gsi); gsi_next (&gsi))
{
gcall *call = dyn_cast <gcall *> (gsi_stmt (gsi));
Index: gcc/tree-vect-slp.c
===================================================================
*** gcc/tree-vect-slp.c 2018-07-30 12:43:34.512651826 +0100
--- gcc/tree-vect-slp.c 2018-07-30 12:43:34.504651897 +0100
*************** vect_detect_hybrid_slp (loop_vec_info lo
*** 2408,2436 ****
/* First walk all pattern stmt in the loop and mark defs of uses as
hybrid because immediate uses in them are not recorded. */
! for (i = 0; i < LOOP_VINFO_LOOP (loop_vinfo)->num_nodes; ++i)
! {
! basic_block bb = LOOP_VINFO_BBS (loop_vinfo)[i];
! for (gimple_stmt_iterator gsi = gsi_start_bb (bb); !gsi_end_p (gsi);
! gsi_next (&gsi))
{
! gimple *stmt = gsi_stmt (gsi);
! stmt_vec_info stmt_info = loop_vinfo->lookup_stmt (stmt);
! if (STMT_VINFO_IN_PATTERN_P (stmt_info))
! {
! walk_stmt_info wi;
! memset (&wi, 0, sizeof (wi));
! wi.info = loop_vinfo;
! gimple_stmt_iterator gsi2
! = gsi_for_stmt (STMT_VINFO_RELATED_STMT (stmt_info)->stmt);
! walk_gimple_stmt (&gsi2, vect_detect_hybrid_slp_2,
! vect_detect_hybrid_slp_1, &wi);
! walk_gimple_seq (STMT_VINFO_PATTERN_DEF_SEQ (stmt_info),
! vect_detect_hybrid_slp_2,
! vect_detect_hybrid_slp_1, &wi);
! }
}
- }
/* Then walk the SLP instance trees marking stmts with uses in
non-SLP stmts as hybrid, also propagating hybrid down the
--- 2408,2429 ----
/* First walk all pattern stmt in the loop and mark defs of uses as
hybrid because immediate uses in them are not recorded. */
! vec_basic_block *vec_bb;
! FOR_EACH_VEC_ELT (loop_vinfo->blocks, i, vec_bb)
! FOR_EACH_VEC_BB_STMT (vec_bb, stmt_info)
! if (STMT_VINFO_IN_PATTERN_P (stmt_info))
{
! walk_stmt_info wi;
! memset (&wi, 0, sizeof (wi));
! wi.info = loop_vinfo;
! gimple_stmt_iterator gsi2
! = gsi_for_stmt (STMT_VINFO_RELATED_STMT (stmt_info)->stmt);
! walk_gimple_stmt (&gsi2, vect_detect_hybrid_slp_2,
! vect_detect_hybrid_slp_1, &wi);
! walk_gimple_seq (STMT_VINFO_PATTERN_DEF_SEQ (stmt_info),
! vect_detect_hybrid_slp_2,
! vect_detect_hybrid_slp_1, &wi);
}
/* Then walk the SLP instance trees marking stmts with uses in
non-SLP stmts as hybrid, also propagating hybrid down the
*************** _bb_vec_info::_bb_vec_info (gimple_stmt_
*** 2457,2469 ****
{
gimple_stmt_iterator gsi;
for (gsi = region_begin; gsi_stmt (gsi) != gsi_stmt (region_end);
gsi_next (&gsi))
{
gimple *stmt = gsi_stmt (gsi);
gimple_set_uid (stmt, 0);
! add_stmt (stmt);
}
bb->aux = this;
}
--- 2450,2464 ----
{
gimple_stmt_iterator gsi;
+ vec_basic_block *vec_bb = new vec_basic_block (bb);
for (gsi = region_begin; gsi_stmt (gsi) != gsi_stmt (region_end);
gsi_next (&gsi))
{
gimple *stmt = gsi_stmt (gsi);
gimple_set_uid (stmt, 0);
! vec_bb->add_to_end (add_stmt (stmt));
}
+ blocks.quick_push (vec_bb);
bb->aux = this;
}
Index: gcc/tree-vect-stmts.c
===================================================================
*** gcc/tree-vect-stmts.c 2018-07-30 12:43:34.512651826 +0100
--- gcc/tree-vect-stmts.c 2018-07-30 12:43:34.504651897 +0100
*************** process_use (stmt_vec_info stmt_vinfo, t
*** 612,623 ****
bool
vect_mark_stmts_to_be_vectorized (loop_vec_info loop_vinfo)
{
- struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
- basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
- unsigned int nbbs = loop->num_nodes;
- gimple_stmt_iterator si;
unsigned int i;
- basic_block bb;
bool live_p;
enum vect_relevant relevant;
--- 612,618 ----
*************** vect_mark_stmts_to_be_vectorized (loop_v
*** 626,659 ****
auto_vec<stmt_vec_info, 64> worklist;
/* 1. Init worklist. */
! for (i = 0; i < nbbs; i++)
! {
! bb = bbs[i];
! for (si = gsi_start_phis (bb); !gsi_end_p (si); gsi_next (&si))
! {
! stmt_vec_info phi_info = loop_vinfo->lookup_stmt (gsi_stmt (si));
! if (dump_enabled_p ())
! {
! dump_printf_loc (MSG_NOTE, vect_location, "init: phi relevant? ");
! dump_gimple_stmt (MSG_NOTE, TDF_SLIM, phi_info->stmt, 0);
! }
!
! if (vect_stmt_relevant_p (phi_info, loop_vinfo, &relevant, &live_p))
! vect_mark_relevant (&worklist, phi_info, relevant, live_p);
! }
! for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
! {
! stmt_vec_info stmt_info = loop_vinfo->lookup_stmt (gsi_stmt (si));
! if (dump_enabled_p ())
! {
! dump_printf_loc (MSG_NOTE, vect_location, "init: stmt relevant?
");
! dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt_info->stmt, 0);
! }
!
! if (vect_stmt_relevant_p (stmt_info, loop_vinfo, &relevant, &live_p))
! vect_mark_relevant (&worklist, stmt_info, relevant, live_p);
! }
! }
/* 2. Process_worklist */
while (worklist.length () > 0)
--- 621,631 ----
auto_vec<stmt_vec_info, 64> worklist;
/* 1. Init worklist. */
! vec_basic_block *vec_bb;
! FOR_EACH_VEC_ELT (loop_vinfo->blocks, i, vec_bb)
! FOR_EACH_VEC_BB_STMT (vec_bb, stmt_info)
! if (vect_stmt_relevant_p (stmt_info, loop_vinfo, &relevant, &live_p))
! vect_mark_relevant (&worklist, stmt_info, relevant, live_p);
/* 2. Process_worklist */
while (worklist.length () > 0)
*************** vect_finish_replace_stmt (stmt_vec_info
*** 1753,1758 ****
--- 1725,1731 ----
gimple_stmt_iterator gsi = gsi_for_stmt (stmt_info->stmt);
gsi_replace (&gsi, vec_stmt, false);
+ stmt_info->block->remove (stmt_info);
return vect_finish_stmt_generation_1 (stmt_info, vec_stmt);
}
*************** hoist_defs_of_uses (stmt_vec_info stmt_i
*** 7352,7357 ****
--- 7325,7331 ----
{
gimple_stmt_iterator gsi = gsi_for_stmt (def_stmt_info->stmt);
gsi_remove (&gsi, false);
+ def_stmt_info->block->remove (def_stmt_info);
gsi_insert_on_edge_immediate (loop_preheader_edge (loop),
def_stmt_info->stmt);
}
*************** vectorizable_condition (stmt_vec_info st
*** 9066,9071 ****
--- 9040,9046 ----
gimple_stmt_iterator old_gsi
= gsi_for_stmt (stmt_info->stmt);
gsi_remove (&old_gsi, true);
+ stmt_info->block->remove (stmt_info);
new_stmt_info
= vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
}
