After if-conversion, a predicated pack index may wrap in its effective
arithmetic type while still matching the mask ? index + 1 : index shape.
Require a compile-time no-wrap proof before classifying the PHI as
vect_predicated_index_def so compress-style vectorization is not enabled
when the index can wrap.
gcc/ChangeLog:
* tree-vect-loop.cc (vect_analyze_predicated_index_phi,
vect_predicated_index_plus_one, vect_predicated_index_compress_legal_p):
Require no-wrap legality before classifying predicated-index PHIs.
gcc/testsuite/ChangeLog:
* gcc.dg/vect/vect-predicated-index-3.c: New test.
* gcc.dg/vect/vect-predicated-index-4.c: New test.
---
.../gcc.dg/vect/vect-predicated-index-3.c | 20 +++
.../gcc.dg/vect/vect-predicated-index-4.c | 20 +++
gcc/tree-vect-loop.cc | 115 +++++++++++++++++-
3 files changed, 151 insertions(+), 4 deletions(-)
create mode 100644 gcc/testsuite/gcc.dg/vect/vect-predicated-index-3.c
create mode 100644 gcc/testsuite/gcc.dg/vect/vect-predicated-index-4.c
diff --git a/gcc/testsuite/gcc.dg/vect/vect-predicated-index-3.c
b/gcc/testsuite/gcc.dg/vect/vect-predicated-index-3.c
new file mode 100644
index 00000000000..bb66084adfb
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-predicated-index-3.c
@@ -0,0 +1,20 @@
+/* { dg-do compile } */
+/* { dg-require-effective-target vect_int } */
+/* { dg-additional-options "-mavx512bw" { target { i?86-*-* x86_64-*-* } } } */
+
+/* Bounded unsigned char pack index: trip count provably small enough that
+ the counter cannot wrap. */
+
+void
+foo (unsigned char *restrict q, const unsigned char *restrict a)
+{
+ unsigned char j = 0;
+
+ for (int i = 0; i < 100; ++i)
+ if (a[i])
+ q[j++] = a[i];
+}
+
+/* { dg-final { scan-tree-dump "Detected predicated-index PHI." "vect" } } */
+/* { dg-final { scan-tree-dump "compress legality ok" "vect" } } */
+/* { dg-final { scan-tree-dump "unsupported: predicated-index PHI." "vect" } }
*/
diff --git a/gcc/testsuite/gcc.dg/vect/vect-predicated-index-4.c
b/gcc/testsuite/gcc.dg/vect/vect-predicated-index-4.c
new file mode 100644
index 00000000000..506aa987225
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-predicated-index-4.c
@@ -0,0 +1,20 @@
+/* { dg-do compile } */
+/* { dg-require-effective-target vect_int } */
+/* { dg-additional-options "-mavx512bw" { target { i?86-*-* x86_64-*-* } } } */
+
+/* Unsigned char pack index with constant trip count large enough that the
+ counter would wrap; compress legality must fail and the PHI must not be
+ classified. */
+
+void
+foo (unsigned char *restrict q, const unsigned char *restrict a)
+{
+ unsigned char j = 0;
+
+ for (int i = 0; i < 300; ++i)
+ if (a[i])
+ q[j++] = a[i];
+}
+
+/* { dg-final { scan-tree-dump-not "Detected predicated-index PHI." "vect" } }
*/
+/* { dg-final { scan-tree-dump "compress legality unproven \\(wrap
possible\\)" "vect" } } */
diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc
index 8c2392530e7..32b66b76af2 100644
--- a/gcc/tree-vect-loop.cc
+++ b/gcc/tree-vect-loop.cc
@@ -200,6 +200,82 @@ vect_pred_phi_strip_conversion (tree *cur)
return true;
}
+/* Return the narrower of two integral types for predicated-index legality.
+ When precision matches, prefer unsigned (defined wrap semantics). */
+
+static tree
+vect_pred_idx_narrower_int_type (tree t1, tree t2)
+{
+ if (TYPE_PRECISION (t1) < TYPE_PRECISION (t2))
+ return t1;
+ if (TYPE_PRECISION (t2) < TYPE_PRECISION (t1))
+ return t2;
+ /* Same width: prefer unsigned, where wrap is defined. */
+ if (TYPE_UNSIGNED (t1) || TYPE_UNSIGNED (t2))
+ return TYPE_UNSIGNED (t1) ? t1 : t2;
+ return t1;
+}
+
+/* Return true if a predicated pack index with effective type EFF_TYPE and
+ constant start value START cannot wrap during LOOP execution, so
+ compress-style vectorization may be considered later. Follow the same
+ signed/unsigned split as is_nonwrapping_integer_induction. At most one
+ +1 occurs per iteration. */
+
+static bool
+vect_predicated_index_compress_legal_p (class loop *loop, tree pred_phi,
+ tree eff_type, tree start)
+{
+ tree phi_type = TREE_TYPE (pred_phi);
+
+ /* Signed counters: overflow is undefined, so treat as safe. */
+ if (TYPE_OVERFLOW_UNDEFINED (phi_type))
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "predicated-index PHI: compress legality ok "
+ "(signed).\n");
+ return true;
+ }
+
+ widest_int nit;
+
+ /* Need a known upper bound on how many times the loop body can run. */
+ if (!max_stmt_executions (loop, &nit))
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "predicated-index PHI: compress legality unproven "
+ "(unknown trip count).\n");
+ return false;
+ }
+
+ /* Starting value must be a compile-time constant. */
+ if (TREE_CODE (start) != INTEGER_CST)
+ return false;
+
+ /* Highest index = start + number of possible increments. */
+ wi::overflow_type overflow = wi::OVF_NONE;
+ widest_int max_index = wi::add (wi::to_widest (start), nit,
+ TYPE_SIGN (eff_type), &overflow);
+ /* Reject if that highest index cannot fit in the counter's type. */
+ if (overflow
+ || wi::min_precision (max_index, TYPE_SIGN (eff_type))
+ > TYPE_PRECISION (eff_type))
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "predicated-index PHI: compress legality unproven "
+ "(wrap possible).\n");
+ return false;
+ }
+
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "predicated-index PHI: compress legality ok.\n");
+ return true;
+}
+
/* Function vect_predicated_index_plus_one.
This function checks if one of the arms of the COND_EXPR,
@@ -217,8 +293,11 @@ vect_pred_phi_strip_conversion (tree *cur)
n_2 = mask ? n_1 : n; */
static bool
vect_predicated_index_plus_one (tree &pred_phi,
- gassign *pred_phi_back_edge_asgn_stmt)
+ gassign *pred_phi_back_edge_asgn_stmt,
+ tree *eff_type)
{
+ *eff_type = TREE_TYPE (pred_phi);
+
/* Check that one of the arms of COND_EXPR is pred-phi (n) and the other is
an increment value (n_1 = n + 1). */
tree true_val = gimple_assign_rhs2 (pred_phi_back_edge_asgn_stmt);
@@ -233,6 +312,10 @@ vect_predicated_index_plus_one (tree &pred_phi,
if (TREE_CODE (pred_phi_inc_val) != SSA_NAME)
return false;
+ /* Fold the incremented arm's type into the effective type so far. */
+ *eff_type = vect_pred_idx_narrower_int_type (*eff_type,
+ TREE_TYPE (pred_phi_inc_val));
+
/* Check if n1 = n + 1?. We may need to strip the cast instructions to reach
the PLUS_EXPR, traversing until CUR goes out of current basic block. */
basic_block bb = gimple_bb (pred_phi_back_edge_asgn_stmt);
@@ -264,6 +347,16 @@ vect_predicated_index_plus_one (tree &pred_phi,
if (!plus_base)
return false;
+ /* Record types of the +1 result and of any SSA operands. */
+ tree lhs_type = TREE_TYPE (gimple_assign_lhs (cur_asgn));
+ *eff_type = vect_pred_idx_narrower_int_type (*eff_type, lhs_type);
+ if (TREE_CODE (rhs1) == SSA_NAME)
+ *eff_type = vect_pred_idx_narrower_int_type (*eff_type,
+ TREE_TYPE (rhs1));
+ if (TREE_CODE (rhs2) == SSA_NAME)
+ *eff_type = vect_pred_idx_narrower_int_type (*eff_type,
+ TREE_TYPE (rhs2));
+
/* Check if plus_base is pred_phi itself. We may still need to strip
some casts to reach the actual base, traversing until plus_base
goes out of the current basic block. */
@@ -272,6 +365,9 @@ vect_predicated_index_plus_one (tree &pred_phi,
gimple *base_def = SSA_NAME_DEF_STMT (plus_base);
if (gimple_bb (base_def) != bb)
break;
+ tree plus_base_type = TREE_TYPE (plus_base);
+ *eff_type = vect_pred_idx_narrower_int_type (*eff_type,
+ plus_base_type);
if (!vect_pred_phi_strip_conversion (&plus_base))
break;
}
@@ -279,8 +375,13 @@ vect_predicated_index_plus_one (tree &pred_phi,
if (plus_base == pred_phi)
{
if (dump_enabled_p ())
- dump_printf_loc (MSG_NOTE, vect_location,
- "predicated-index PHI: update is +1.\n");
+ {
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "predicated-index PHI: update is +1.\n");
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "predicated-index PHI: effective type "
+ "for legality: %T.\n", *eff_type);
+ }
return true;
}
/* We don't have +1 pattern. */
@@ -523,7 +624,9 @@ vect_analyze_predicated_index_phi (class loop *loop, gphi
*phi,
/* Check that one of the arms of COND_EXPR is pred-phi (n) and the other is
an increment value (n_1 = n + 1). */
- if (!vect_predicated_index_plus_one (pred_phi, pred_phi_back_edge_asgn_stmt))
+ tree eff_type;
+ if (!vect_predicated_index_plus_one (pred_phi, pred_phi_back_edge_asgn_stmt,
+ &eff_type))
{
if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -531,6 +634,10 @@ vect_analyze_predicated_index_phi (class loop *loop, gphi
*phi,
return false;
}
+ if (!vect_predicated_index_compress_legal_p (loop, pred_phi, eff_type,
+ pre_init))
+ return false;
+
/* The conditional-increment cycle is recognized. Record the mask governing
the increment on the PHI's stmt_info. */
tree mask = gimple_assign_rhs1 (pred_phi_back_edge_asgn_stmt);
--
2.53.0
