This patch first refactors the masked vector-compare recognition so it can be
re-used outside the old optabs.cc local code and uses it during out-of-SSA.
The new helper identifies plain masked comparisons, length-masked
VCOND_MASK_LEN comparisons, and the conditional compare IFNs added
by the following patches.
The out-of-SSA path can then duplicate the comparison at the point where a
conditional branch needs it. That matters for vector predicates because the
branch expander often needs to see the original comparison operands, not only
an SSA name that has already been lowered to a predicate value.
For example:
void f (int *__restrict a, int *__restrict b, int *__restrict c,
int *__restrict d)
{
for (int i = 0; i < 1024; ++i)
if (c[i] != 0)
return c[i];
}
The compares can requires as both data and mask producing. For vec_cbranch_* to
be used in tree-outof-ssa the compare needs to be single use but in the case of
the vectorized form of the above it may not be since the results is used in two
places.
To fix this we duplicate vector compares by use when the compare is on a
back-edge.
This allows vec_cbranch_* to keep working with the control flow patches.
There is a bit of code movement as gimple_is_masked_p is refactored to a
helper that can be used by ISEL as well.
Bootstrapped Regtested on aarch64-none-linux-gnu,
arm-none-linux-gnueabihf, x86_64-pc-linux-gnu
-m32, -m64 and no issues.
-O3, profiled, profiled -O3 and -O3 -LTO bootstrap tested and no issues.
Also SPECCPU 2017 and SPECCPU 2026 tested.
OK for master?
Thanks,
Tamar
gcc/ChangeLog:
* gimple.cc (gimple_get_ssa_value): Accept masked compare statements.
(gimple_is_masked_p): New helper.
* gimple.h (gimple_is_masked_p): Declare.
* optabs.cc (emit_cmp_and_jump_insns): Use gimple_is_masked_p.
* tree-outof-ssa.cc (duplicate_vector_condition_def): New helper.
(insert_backedge_copies): Duplicate masked vector conditions.
---
diff --git a/gcc/gimple.cc b/gcc/gimple.cc
index
3756a60a8e6e21e8533567295136f59fc10bc76c..4c6e712b34ca34e26884e29ee816cf248f27bb36
100644
--- a/gcc/gimple.cc
+++ b/gcc/gimple.cc
@@ -50,6 +50,8 @@ along with GCC; see the file COPYING3. If not see
#include "ipa-modref-tree.h"
#include "ipa-modref.h"
#include "dbgcnt.h"
+#include "tree-ssa-live.h"
+#include "tree-outof-ssa.h"
/* All the tuples have their operand vector (if present) at the very bottom of
the structure. Therefore, the offset required to find the operands vector
is
@@ -2474,6 +2476,116 @@ gimple_could_trap_p (const gimple *s)
return gimple_could_trap_p_1 (s, true, true);
}
+/* Return the gimple def of ssa_name NAME using a method that's safe to call
+ regarless of whether we are still in SSA or not. */
+static inline gimple *
+gimple_get_ssa_value (tree name)
+{
+ if (gimple_in_ssa_p (cfun))
+ return SSA_NAME_DEF_STMT (name);
+ return get_gimple_for_ssa_name (name);
+}
+
+/* Return true if the operation in s is a masked operation. If so return the
+ operation being masked in OP and if available the MASK, LEN and BIAS for
+ the operation. */
+
+bool
+gimple_is_masked_p (gimple *s, gimple **op, tree *mask, tree *len, tree *bias)
+{
+ tree masked_op = NULL_TREE;
+ tree len_op = NULL_TREE;
+ tree len_bias = NULL_TREE;
+ gimple *def_stmt = s;
+
+ /* First determine if the operation should be masked or unmasked. */
+ if (is_gimple_assign (def_stmt)
+ && gimple_assign_rhs_code (def_stmt) == BIT_AND_EXPR)
+ {
+ /* See if one side if a comparison, if so use the other side as
+ the mask. */
+ gimple *mask_def = NULL;
+ tree rhs1 = gimple_assign_rhs1 (def_stmt);
+ tree rhs2 = gimple_assign_rhs2 (def_stmt);
+ if (TREE_CODE (rhs1) == SSA_NAME
+ && (mask_def = gimple_get_ssa_value (rhs1))
+ && is_gimple_assign (mask_def)
+ && TREE_CODE_CLASS (gimple_assign_rhs_code (mask_def))
+ == tcc_comparison)
+ masked_op = rhs2;
+ else if (TREE_CODE (rhs2) == SSA_NAME
+ && (mask_def = gimple_get_ssa_value (rhs2))
+ && is_gimple_assign (mask_def)
+ && TREE_CODE_CLASS (gimple_assign_rhs_code (mask_def))
+ == tcc_comparison)
+ masked_op = rhs1;
+
+ if (masked_op)
+ def_stmt = mask_def;
+ }
+ /* Else check to see if we're a LEN target. */
+ else if (is_gimple_call (def_stmt)
+ && gimple_call_internal_p (def_stmt)
+ && gimple_call_internal_fn (def_stmt) == IFN_VCOND_MASK_LEN)
+ {
+ /* Example to consume:
+
+ a = _59 != vect__4.17_75;
+ vcmp = .VCOND_MASK_LEN (a, { -1, ... }, { 0, ... }, _90, 0);
+ if (vcmp != { 0, ... })
+
+ and transform into
+
+ if (cond_len_vec_cbranch_any ({-1, ...}, a, _90, 0)). */
+ gcall *call = dyn_cast <gcall *> (def_stmt);
+ tree true_branch = gimple_call_arg (call, 1);
+ tree false_branch = gimple_call_arg (call, 2);
+ if (integer_minus_onep (true_branch)
+ && integer_zerop (false_branch))
+ {
+ len_op = gimple_call_arg (call, 3);
+ len_bias = gimple_call_arg (call, 4);
+ tree arg0 = gimple_call_arg (call, 0);
+
+ if (TREE_CODE (arg0) == SSA_NAME)
+ def_stmt = gimple_get_ssa_value (arg0);
+ }
+ }
+ else if (is_gimple_call (def_stmt)
+ && gimple_call_internal_p (def_stmt))
+ {
+ switch (gimple_call_internal_fn (def_stmt))
+ {
+ case IFN_COND_VEC_CMP:
+ {
+ masked_op = gimple_call_arg (def_stmt, 1);
+ break;
+ }
+ case IFN_COND_LEN_VEC_CMP:
+ {
+ masked_op = gimple_call_arg (def_stmt, 1);
+ len_op = gimple_call_arg (def_stmt, 4);
+ len_bias = gimple_call_arg (def_stmt, 5);
+ break;
+ }
+ default:
+ return false;
+ }
+ }
+ else
+ return false;
+
+ if (!masked_op && !len_op)
+ return false;
+
+ *op = def_stmt;
+ *mask = masked_op;
+ *len = len_op;
+ *bias = len_bias;
+
+ return true;
+}
+
/* Return true if RHS of a GIMPLE_ASSIGN S can trap. */
bool
diff --git a/gcc/gimple.h b/gcc/gimple.h
index
82045c44bbeb5184752d2bf7e984991d0ce79c1f..26ea22152a3a6e73a15a7a4c65b3aa6b09a34469
100644
--- a/gcc/gimple.h
+++ b/gcc/gimple.h
@@ -1653,6 +1653,7 @@ bool gimple_has_side_effects (const gimple *);
bool gimple_could_trap_p_1 (const gimple *, bool, bool);
bool gimple_could_trap_p (const gimple *);
bool gimple_assign_rhs_could_trap_p (gimple *);
+bool gimple_is_masked_p (gimple *, gimple **, tree *, tree *, tree *);
extern void dump_gimple_statistics (void);
unsigned get_gimple_rhs_num_ops (enum tree_code);
gcall *gimple_call_copy_skip_args (gcall *, bitmap);
diff --git a/gcc/optabs.cc b/gcc/optabs.cc
index
012c0e9736ec681ec298f9081e3ccb8cdc6fdf55..466fa581c47120eef2e366e0d8fc789249b5bd45
100644
--- a/gcc/optabs.cc
+++ b/gcc/optabs.cc
@@ -5024,57 +5024,10 @@ emit_cmp_and_jump_insns (rtx x, rtx y, enum rtx_code
comparison, rtx size,
tree masked_op = NULL_TREE;
tree len_op = NULL_TREE;
tree len_bias = NULL_TREE;
- /* First determine if the operation should be masked or unmasked. */
- if (is_gimple_assign (def_stmt)
- && gimple_assign_rhs_code (def_stmt) == BIT_AND_EXPR)
- {
- /* See if one side if a comparison, if so use the other side as
- the mask. */
- gimple *mask_def = NULL;
- tree rhs1 = gimple_assign_rhs1 (def_stmt);
- tree rhs2 = gimple_assign_rhs2 (def_stmt);
- if (TREE_CODE (rhs1) == SSA_NAME
- && (mask_def = get_gimple_for_ssa_name (rhs1))
- && is_gimple_assign (mask_def)
- && TREE_CODE_CLASS (gimple_assign_rhs_code (mask_def)))
- masked_op = rhs2;
- else if (TREE_CODE (rhs2) == SSA_NAME
- && (mask_def = get_gimple_for_ssa_name (rhs2))
- && is_gimple_assign (mask_def)
- && TREE_CODE_CLASS (gimple_assign_rhs_code (mask_def)))
- masked_op = rhs1;
-
- if (masked_op)
- def_stmt = mask_def;
- }
- /* Else check to see if we're a LEN target. */
- else if (is_gimple_call (def_stmt)
- && gimple_call_internal_p (def_stmt)
- && gimple_call_internal_fn (def_stmt) == IFN_VCOND_MASK_LEN)
- {
- /* Example to consume:
-
- a = _59 != vect__4.17_75;
- vcmp = .VCOND_MASK_LEN (a, { -1, ... }, { 0, ... }, _90, 0);
- if (vcmp != { 0, ... })
- and transform into
-
- if (cond_len_vec_cbranch_any ({-1, ...}, a, _90, 0)). */
- gcall *call = dyn_cast <gcall *> (def_stmt);
- tree true_branch = gimple_call_arg (call, 1);
- tree false_branch = gimple_call_arg (call, 2);
- if (integer_minus_onep (true_branch)
- && integer_zerop (false_branch))
- {
- len_op = gimple_call_arg (call, 3);
- len_bias = gimple_call_arg (call, 4);
- tree arg0 = gimple_call_arg (call, 0);
-
- if (TREE_CODE (arg0) == SSA_NAME)
- def_stmt = get_gimple_for_ssa_name (arg0);
- }
- }
+ /* First determine if the operation should be masked or unmasked. */
+ gimple_is_masked_p (def_stmt, &def_stmt, &masked_op, &len_op,
+ &len_bias);
enum insn_code icode;
if (def_stmt
diff --git a/gcc/tree-outof-ssa.cc b/gcc/tree-outof-ssa.cc
index
409e951b2127dd2b72211a170032c1ed2d3b906c..fa7264ba832888abf18d1dc73497e856c67b6793
100644
--- a/gcc/tree-outof-ssa.cc
+++ b/gcc/tree-outof-ssa.cc
@@ -49,6 +49,7 @@ along with GCC; see the file COPYING3. If not see
#include "dojump.h"
#include "internal-fn.h"
#include "gimple-fold.h"
+#include "gimplify.h"
/* FIXME: A lot of code here deals with expanding to RTL. All that code
should be in cfgexpand.cc. */
@@ -1272,6 +1273,76 @@ trivially_conflicts_p (basic_block bb, tree result, tree
arg)
return false;
}
+/* Duplicate the vector comparison that feeds LHS, returning the new lhs or
+ null if LHS isn't a supported conditional operand. Add any new statements
+ to STMTS. */
+
+static tree
+duplicate_vector_condition_def (tree lhs, gimple_seq *stmts)
+{
+ gimple *def_stmt = SSA_NAME_DEF_STMT (lhs);
+ gimple *masked_stmt = def_stmt;
+ tree masked_op = NULL_TREE;
+ tree len_op = NULL_TREE;
+ tree len_bias = NULL_TREE;
+
+ gimple_is_masked_p (def_stmt, &masked_stmt, &masked_op, &len_op,
+ &len_bias);
+
+ if (is_gimple_assign (masked_stmt)
+ && TREE_CODE_CLASS (gimple_assign_rhs_code (masked_stmt))
+ == tcc_comparison)
+ {
+ tree new_lhs
+ = gimple_build (stmts, gimple_assign_rhs_code (masked_stmt),
+ TREE_TYPE (gimple_get_lhs (masked_stmt)),
+ gimple_assign_rhs1 (masked_stmt),
+ gimple_assign_rhs2 (masked_stmt));
+ tree type = TREE_TYPE (new_lhs);
+
+ if (masked_op)
+ return gimple_build (stmts, BIT_AND_EXPR, type, masked_op, new_lhs);
+
+ if (!len_op)
+ return new_lhs;
+
+ tree cmp_lhs = new_lhs;
+ new_lhs = make_ssa_name (type);
+ gcall *call = gimple_build_call_internal (IFN_VCOND_MASK_LEN, 5,
+ cmp_lhs,
+ build_minus_one_cst (type),
+ build_zero_cst (type),
+ len_op, len_bias);
+ gimple_call_set_lhs (call, new_lhs);
+ gimple_set_location (call, gimple_location (masked_stmt));
+ gimple_seq_add_stmt (stmts, call);
+ return new_lhs;
+ }
+
+ gcall *call = dyn_cast <gcall *> (masked_stmt);
+ if (!call || !gimple_call_internal_p (call))
+ return NULL_TREE;
+
+ internal_fn ifn = gimple_call_internal_fn (call);
+ if (ifn != IFN_COND_VEC_CMP && ifn != IFN_COND_LEN_VEC_CMP)
+ return NULL_TREE;
+
+ auto_vec<tree> args;
+ unsigned int nargs = gimple_call_num_args (call);
+ args.create (nargs);
+ args.quick_grow (nargs);
+ for (unsigned int i = 0; i < nargs; ++i)
+ args[i] = unshare_expr (gimple_call_arg (call, i));
+
+ gcall *copy = gimple_build_call_internal_vec (ifn, args);
+ tree new_lhs = duplicate_ssa_name (lhs, copy);
+ gimple_call_set_lhs (copy, new_lhs);
+ gimple_set_location (copy, gimple_location (call));
+ update_stmt (copy);
+ gimple_seq_add_stmt (stmts, copy);
+ return new_lhs;
+}
+
/* Search every PHI node for arguments associated with backedges which
we can trivially determine will need a copy (the argument is either
@@ -1448,6 +1519,52 @@ insert_backedge_copies (void)
}
}
+ /* Find any block with a jump inside it and identify if it's a vector
+ conditional. */
+ gcond *cond = NULL;
+ if (!single_succ_p (bb))
+ {
+ gimple *last_stmt = gsi_stmt (gsi_last_nondebug_bb (bb));
+ cond = dyn_cast<gcond *> (last_stmt);
+ }
+ tree lhs = cond ? gimple_cond_lhs (cond) : NULL_TREE;
+ if (lhs
+ && TREE_CODE (lhs) == SSA_NAME
+ && VECTOR_TYPE_P (TREE_TYPE (lhs))
+ && !has_single_use (lhs))
+ {
+ gimple *def_stmt = SSA_NAME_DEF_STMT (lhs);
+ gimple_seq stmts = NULL;
+ tree new_lhs = duplicate_vector_condition_def (lhs, &stmts);
+ if (new_lhs)
+ {
+ gimple_stmt_iterator def_gsi = gsi_for_stmt (def_stmt);
+ gsi_insert_seq_after (&def_gsi, stmts, GSI_SAME_STMT);
+
+ imm_use_iterator imm_iter;
+ gimple *use_stmt;
+ use_operand_p use_p;
+ FOR_EACH_IMM_USE_STMT (use_stmt, imm_iter, lhs)
+ {
+ if (is_a <gcond *> (use_stmt))
+ continue;
+
+ FOR_EACH_IMM_USE_ON_STMT (use_p, imm_iter)
+ SET_USE (use_p, new_lhs);
+
+ update_stmt (use_stmt);
+ }
+
+ if (dump_file && (dump_flags & TDF_DETAILS))
+ {
+ fprintf (dump_file, "Duplicating vector compare ");
+ print_gimple_stmt (dump_file, def_stmt, 3, TDF_DETAILS);
+ fprintf (dump_file, " to ");
+ print_gimple_stmt (dump_file, stmts, 3, TDF_DETAILS);
+ fprintf (dump_file, "\n");
+ }
+ }
+ }
/* Unmark this block again. */
bb->aux = NULL;
}
--
diff --git a/gcc/gimple.cc b/gcc/gimple.cc
index 3756a60a8e6e21e8533567295136f59fc10bc76c..4c6e712b34ca34e26884e29ee816cf248f27bb36 100644
--- a/gcc/gimple.cc
+++ b/gcc/gimple.cc
@@ -50,6 +50,8 @@ along with GCC; see the file COPYING3. If not see
#include "ipa-modref-tree.h"
#include "ipa-modref.h"
#include "dbgcnt.h"
+#include "tree-ssa-live.h"
+#include "tree-outof-ssa.h"
/* All the tuples have their operand vector (if present) at the very bottom of
the structure. Therefore, the offset required to find the operands vector is
@@ -2474,6 +2476,116 @@ gimple_could_trap_p (const gimple *s)
return gimple_could_trap_p_1 (s, true, true);
}
+/* Return the gimple def of ssa_name NAME using a method that's safe to call
+ regarless of whether we are still in SSA or not. */
+static inline gimple *
+gimple_get_ssa_value (tree name)
+{
+ if (gimple_in_ssa_p (cfun))
+ return SSA_NAME_DEF_STMT (name);
+ return get_gimple_for_ssa_name (name);
+}
+
+/* Return true if the operation in s is a masked operation. If so return the
+ operation being masked in OP and if available the MASK, LEN and BIAS for
+ the operation. */
+
+bool
+gimple_is_masked_p (gimple *s, gimple **op, tree *mask, tree *len, tree *bias)
+{
+ tree masked_op = NULL_TREE;
+ tree len_op = NULL_TREE;
+ tree len_bias = NULL_TREE;
+ gimple *def_stmt = s;
+
+ /* First determine if the operation should be masked or unmasked. */
+ if (is_gimple_assign (def_stmt)
+ && gimple_assign_rhs_code (def_stmt) == BIT_AND_EXPR)
+ {
+ /* See if one side if a comparison, if so use the other side as
+ the mask. */
+ gimple *mask_def = NULL;
+ tree rhs1 = gimple_assign_rhs1 (def_stmt);
+ tree rhs2 = gimple_assign_rhs2 (def_stmt);
+ if (TREE_CODE (rhs1) == SSA_NAME
+ && (mask_def = gimple_get_ssa_value (rhs1))
+ && is_gimple_assign (mask_def)
+ && TREE_CODE_CLASS (gimple_assign_rhs_code (mask_def))
+ == tcc_comparison)
+ masked_op = rhs2;
+ else if (TREE_CODE (rhs2) == SSA_NAME
+ && (mask_def = gimple_get_ssa_value (rhs2))
+ && is_gimple_assign (mask_def)
+ && TREE_CODE_CLASS (gimple_assign_rhs_code (mask_def))
+ == tcc_comparison)
+ masked_op = rhs1;
+
+ if (masked_op)
+ def_stmt = mask_def;
+ }
+ /* Else check to see if we're a LEN target. */
+ else if (is_gimple_call (def_stmt)
+ && gimple_call_internal_p (def_stmt)
+ && gimple_call_internal_fn (def_stmt) == IFN_VCOND_MASK_LEN)
+ {
+ /* Example to consume:
+
+ a = _59 != vect__4.17_75;
+ vcmp = .VCOND_MASK_LEN (a, { -1, ... }, { 0, ... }, _90, 0);
+ if (vcmp != { 0, ... })
+
+ and transform into
+
+ if (cond_len_vec_cbranch_any ({-1, ...}, a, _90, 0)). */
+ gcall *call = dyn_cast <gcall *> (def_stmt);
+ tree true_branch = gimple_call_arg (call, 1);
+ tree false_branch = gimple_call_arg (call, 2);
+ if (integer_minus_onep (true_branch)
+ && integer_zerop (false_branch))
+ {
+ len_op = gimple_call_arg (call, 3);
+ len_bias = gimple_call_arg (call, 4);
+ tree arg0 = gimple_call_arg (call, 0);
+
+ if (TREE_CODE (arg0) == SSA_NAME)
+ def_stmt = gimple_get_ssa_value (arg0);
+ }
+ }
+ else if (is_gimple_call (def_stmt)
+ && gimple_call_internal_p (def_stmt))
+ {
+ switch (gimple_call_internal_fn (def_stmt))
+ {
+ case IFN_COND_VEC_CMP:
+ {
+ masked_op = gimple_call_arg (def_stmt, 1);
+ break;
+ }
+ case IFN_COND_LEN_VEC_CMP:
+ {
+ masked_op = gimple_call_arg (def_stmt, 1);
+ len_op = gimple_call_arg (def_stmt, 4);
+ len_bias = gimple_call_arg (def_stmt, 5);
+ break;
+ }
+ default:
+ return false;
+ }
+ }
+ else
+ return false;
+
+ if (!masked_op && !len_op)
+ return false;
+
+ *op = def_stmt;
+ *mask = masked_op;
+ *len = len_op;
+ *bias = len_bias;
+
+ return true;
+}
+
/* Return true if RHS of a GIMPLE_ASSIGN S can trap. */
bool
diff --git a/gcc/gimple.h b/gcc/gimple.h
index 82045c44bbeb5184752d2bf7e984991d0ce79c1f..26ea22152a3a6e73a15a7a4c65b3aa6b09a34469 100644
--- a/gcc/gimple.h
+++ b/gcc/gimple.h
@@ -1653,6 +1653,7 @@ bool gimple_has_side_effects (const gimple *);
bool gimple_could_trap_p_1 (const gimple *, bool, bool);
bool gimple_could_trap_p (const gimple *);
bool gimple_assign_rhs_could_trap_p (gimple *);
+bool gimple_is_masked_p (gimple *, gimple **, tree *, tree *, tree *);
extern void dump_gimple_statistics (void);
unsigned get_gimple_rhs_num_ops (enum tree_code);
gcall *gimple_call_copy_skip_args (gcall *, bitmap);
diff --git a/gcc/optabs.cc b/gcc/optabs.cc
index 012c0e9736ec681ec298f9081e3ccb8cdc6fdf55..466fa581c47120eef2e366e0d8fc789249b5bd45 100644
--- a/gcc/optabs.cc
+++ b/gcc/optabs.cc
@@ -5024,57 +5024,10 @@ emit_cmp_and_jump_insns (rtx x, rtx y, enum rtx_code comparison, rtx size,
tree masked_op = NULL_TREE;
tree len_op = NULL_TREE;
tree len_bias = NULL_TREE;
- /* First determine if the operation should be masked or unmasked. */
- if (is_gimple_assign (def_stmt)
- && gimple_assign_rhs_code (def_stmt) == BIT_AND_EXPR)
- {
- /* See if one side if a comparison, if so use the other side as
- the mask. */
- gimple *mask_def = NULL;
- tree rhs1 = gimple_assign_rhs1 (def_stmt);
- tree rhs2 = gimple_assign_rhs2 (def_stmt);
- if (TREE_CODE (rhs1) == SSA_NAME
- && (mask_def = get_gimple_for_ssa_name (rhs1))
- && is_gimple_assign (mask_def)
- && TREE_CODE_CLASS (gimple_assign_rhs_code (mask_def)))
- masked_op = rhs2;
- else if (TREE_CODE (rhs2) == SSA_NAME
- && (mask_def = get_gimple_for_ssa_name (rhs2))
- && is_gimple_assign (mask_def)
- && TREE_CODE_CLASS (gimple_assign_rhs_code (mask_def)))
- masked_op = rhs1;
-
- if (masked_op)
- def_stmt = mask_def;
- }
- /* Else check to see if we're a LEN target. */
- else if (is_gimple_call (def_stmt)
- && gimple_call_internal_p (def_stmt)
- && gimple_call_internal_fn (def_stmt) == IFN_VCOND_MASK_LEN)
- {
- /* Example to consume:
-
- a = _59 != vect__4.17_75;
- vcmp = .VCOND_MASK_LEN (a, { -1, ... }, { 0, ... }, _90, 0);
- if (vcmp != { 0, ... })
- and transform into
-
- if (cond_len_vec_cbranch_any ({-1, ...}, a, _90, 0)). */
- gcall *call = dyn_cast <gcall *> (def_stmt);
- tree true_branch = gimple_call_arg (call, 1);
- tree false_branch = gimple_call_arg (call, 2);
- if (integer_minus_onep (true_branch)
- && integer_zerop (false_branch))
- {
- len_op = gimple_call_arg (call, 3);
- len_bias = gimple_call_arg (call, 4);
- tree arg0 = gimple_call_arg (call, 0);
-
- if (TREE_CODE (arg0) == SSA_NAME)
- def_stmt = get_gimple_for_ssa_name (arg0);
- }
- }
+ /* First determine if the operation should be masked or unmasked. */
+ gimple_is_masked_p (def_stmt, &def_stmt, &masked_op, &len_op,
+ &len_bias);
enum insn_code icode;
if (def_stmt
diff --git a/gcc/tree-outof-ssa.cc b/gcc/tree-outof-ssa.cc
index 409e951b2127dd2b72211a170032c1ed2d3b906c..fa7264ba832888abf18d1dc73497e856c67b6793 100644
--- a/gcc/tree-outof-ssa.cc
+++ b/gcc/tree-outof-ssa.cc
@@ -49,6 +49,7 @@ along with GCC; see the file COPYING3. If not see
#include "dojump.h"
#include "internal-fn.h"
#include "gimple-fold.h"
+#include "gimplify.h"
/* FIXME: A lot of code here deals with expanding to RTL. All that code
should be in cfgexpand.cc. */
@@ -1272,6 +1273,76 @@ trivially_conflicts_p (basic_block bb, tree result, tree arg)
return false;
}
+/* Duplicate the vector comparison that feeds LHS, returning the new lhs or
+ null if LHS isn't a supported conditional operand. Add any new statements
+ to STMTS. */
+
+static tree
+duplicate_vector_condition_def (tree lhs, gimple_seq *stmts)
+{
+ gimple *def_stmt = SSA_NAME_DEF_STMT (lhs);
+ gimple *masked_stmt = def_stmt;
+ tree masked_op = NULL_TREE;
+ tree len_op = NULL_TREE;
+ tree len_bias = NULL_TREE;
+
+ gimple_is_masked_p (def_stmt, &masked_stmt, &masked_op, &len_op,
+ &len_bias);
+
+ if (is_gimple_assign (masked_stmt)
+ && TREE_CODE_CLASS (gimple_assign_rhs_code (masked_stmt))
+ == tcc_comparison)
+ {
+ tree new_lhs
+ = gimple_build (stmts, gimple_assign_rhs_code (masked_stmt),
+ TREE_TYPE (gimple_get_lhs (masked_stmt)),
+ gimple_assign_rhs1 (masked_stmt),
+ gimple_assign_rhs2 (masked_stmt));
+ tree type = TREE_TYPE (new_lhs);
+
+ if (masked_op)
+ return gimple_build (stmts, BIT_AND_EXPR, type, masked_op, new_lhs);
+
+ if (!len_op)
+ return new_lhs;
+
+ tree cmp_lhs = new_lhs;
+ new_lhs = make_ssa_name (type);
+ gcall *call = gimple_build_call_internal (IFN_VCOND_MASK_LEN, 5,
+ cmp_lhs,
+ build_minus_one_cst (type),
+ build_zero_cst (type),
+ len_op, len_bias);
+ gimple_call_set_lhs (call, new_lhs);
+ gimple_set_location (call, gimple_location (masked_stmt));
+ gimple_seq_add_stmt (stmts, call);
+ return new_lhs;
+ }
+
+ gcall *call = dyn_cast <gcall *> (masked_stmt);
+ if (!call || !gimple_call_internal_p (call))
+ return NULL_TREE;
+
+ internal_fn ifn = gimple_call_internal_fn (call);
+ if (ifn != IFN_COND_VEC_CMP && ifn != IFN_COND_LEN_VEC_CMP)
+ return NULL_TREE;
+
+ auto_vec<tree> args;
+ unsigned int nargs = gimple_call_num_args (call);
+ args.create (nargs);
+ args.quick_grow (nargs);
+ for (unsigned int i = 0; i < nargs; ++i)
+ args[i] = unshare_expr (gimple_call_arg (call, i));
+
+ gcall *copy = gimple_build_call_internal_vec (ifn, args);
+ tree new_lhs = duplicate_ssa_name (lhs, copy);
+ gimple_call_set_lhs (copy, new_lhs);
+ gimple_set_location (copy, gimple_location (call));
+ update_stmt (copy);
+ gimple_seq_add_stmt (stmts, copy);
+ return new_lhs;
+}
+
/* Search every PHI node for arguments associated with backedges which
we can trivially determine will need a copy (the argument is either
@@ -1448,6 +1519,52 @@ insert_backedge_copies (void)
}
}
+ /* Find any block with a jump inside it and identify if it's a vector
+ conditional. */
+ gcond *cond = NULL;
+ if (!single_succ_p (bb))
+ {
+ gimple *last_stmt = gsi_stmt (gsi_last_nondebug_bb (bb));
+ cond = dyn_cast<gcond *> (last_stmt);
+ }
+ tree lhs = cond ? gimple_cond_lhs (cond) : NULL_TREE;
+ if (lhs
+ && TREE_CODE (lhs) == SSA_NAME
+ && VECTOR_TYPE_P (TREE_TYPE (lhs))
+ && !has_single_use (lhs))
+ {
+ gimple *def_stmt = SSA_NAME_DEF_STMT (lhs);
+ gimple_seq stmts = NULL;
+ tree new_lhs = duplicate_vector_condition_def (lhs, &stmts);
+ if (new_lhs)
+ {
+ gimple_stmt_iterator def_gsi = gsi_for_stmt (def_stmt);
+ gsi_insert_seq_after (&def_gsi, stmts, GSI_SAME_STMT);
+
+ imm_use_iterator imm_iter;
+ gimple *use_stmt;
+ use_operand_p use_p;
+ FOR_EACH_IMM_USE_STMT (use_stmt, imm_iter, lhs)
+ {
+ if (is_a <gcond *> (use_stmt))
+ continue;
+
+ FOR_EACH_IMM_USE_ON_STMT (use_p, imm_iter)
+ SET_USE (use_p, new_lhs);
+
+ update_stmt (use_stmt);
+ }
+
+ if (dump_file && (dump_flags & TDF_DETAILS))
+ {
+ fprintf (dump_file, "Duplicating vector compare ");
+ print_gimple_stmt (dump_file, def_stmt, 3, TDF_DETAILS);
+ fprintf (dump_file, " to ");
+ print_gimple_stmt (dump_file, stmts, 3, TDF_DETAILS);
+ fprintf (dump_file, "\n");
+ }
+ }
+ }
/* Unmark this block again. */
bb->aux = NULL;
}
