Define IFN_PREDICATED_INDEX_UPDATE and IFN_MASK_COMPRESS_SCATTER_STORE,
add scalar RTL expanders, and implement phase-2 pattern recognition for
the predicated-index PHI latch update and paired IFN_MASK_STORE.
During phase-1 detection, record the mask-store base and element scale in
loop_vec_info so compress-scatter recog can build
IFN_MASK_COMPRESS_SCATTER_STORE without re-walking the address chain.
---
gcc/internal-fn.cc | 58 +++++++++
gcc/internal-fn.def | 4 +
gcc/tree-vect-loop.cc | 7 +-
gcc/tree-vect-patterns.cc | 3 +
gcc/tree-vect-vpcompress.cc | 229 ++++++++++++++++++++++++++++++------
gcc/tree-vect-vpcompress.h | 6 +
gcc/tree-vectorizer.h | 7 ++
7 files changed, 279 insertions(+), 35 deletions(-)
diff --git a/gcc/internal-fn.cc b/gcc/internal-fn.cc
index 0138c6f7ef0..bf0f430ec7d 100644
--- a/gcc/internal-fn.cc
+++ b/gcc/internal-fn.cc
@@ -4777,6 +4777,56 @@ set_edom_supported_p (void)
}
#include "internal-fn.def"
+/* Expand IFN_PREDICATED_INDEX_UPDATE to mask ? idx + 1 : idx. */
+
+static void
+expand_PREDICATED_INDEX_UPDATE (internal_fn, gcall *stmt)
+{
+ tree lhs = gimple_call_lhs (stmt);
+ gcc_assert (lhs != NULL_TREE);
+ tree idx = gimple_call_arg (stmt, 0);
+ tree mask = gimple_call_arg (stmt, 1);
+ tree type = TREE_TYPE (lhs);
+ tree one = build_int_cst (type, 1);
+ tree inc = fold_build2 (PLUS_EXPR, type, idx, one);
+ tree cond_expr;
+ if (VECTOR_TYPE_P (TREE_TYPE (mask)))
+ gcc_unreachable ();
+ if (TREE_CODE (TREE_TYPE (mask)) == BOOLEAN_TYPE)
+ cond_expr = fold_build3 (COND_EXPR, type, mask, inc, idx);
+ else
+ {
+ tree cmp = fold_build2 (NE_EXPR, boolean_type_node, mask,
+ build_int_cst (TREE_TYPE (mask), 0));
+ cond_expr = fold_build3 (COND_EXPR, type, cmp, inc, idx);
+ }
+ rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
+ rtx tmp = expand_expr (cond_expr, target, TYPE_MODE (type), EXPAND_NORMAL);
+ if (!rtx_equal_p (tmp, target))
+ emit_move_insn (target, tmp);
+}
+
+/* Lower IFN_MASK_COMPRESS_SCATTER_STORE to IFN_MASK_STORE for now. */
+
+static void
+expand_MASK_COMPRESS_SCATTER_STORE (internal_fn, gcall *stmt)
+{
+ tree base = gimple_call_arg (stmt, 0);
+ tree align = gimple_call_arg (stmt, 1);
+ tree pack_idx = gimple_call_arg (stmt, 2);
+ tree scale = gimple_call_arg (stmt, 3);
+ tree value = gimple_call_arg (stmt, 4);
+ tree mask = gimple_call_arg (stmt, 5);
+ tree byte_off = fold_build2 (MULT_EXPR, sizetype, pack_idx,
+ fold_convert (sizetype, scale));
+ tree ptr = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (base), base,
+ byte_off);
+ gcall *call
+ = gimple_build_call_internal (IFN_MASK_STORE, 4, ptr, align, mask, value);
+ gimple_set_location (call, gimple_location (stmt));
+ expand_internal_call (IFN_MASK_STORE, call);
+}
+
/* Routines to expand each internal function, indexed by function number.
Each routine has the prototype:
@@ -5104,6 +5154,7 @@ internal_store_fn_p (internal_fn fn)
case IFN_MASK_LEN_SCATTER_STORE:
case IFN_LEN_STORE:
case IFN_MASK_LEN_STORE:
+ case IFN_MASK_COMPRESS_SCATTER_STORE:
return true;
default:
@@ -5124,6 +5175,7 @@ internal_gather_scatter_fn_p (internal_fn fn)
case IFN_SCATTER_STORE:
case IFN_MASK_SCATTER_STORE:
case IFN_MASK_LEN_SCATTER_STORE:
+ case IFN_MASK_COMPRESS_SCATTER_STORE:
return true;
default:
@@ -5303,6 +5355,7 @@ internal_fn_mask_index (internal_fn fn)
case IFN_MASK_SCATTER_STORE:
case IFN_MASK_LEN_GATHER_LOAD:
case IFN_MASK_LEN_SCATTER_STORE:
+ case IFN_MASK_COMPRESS_SCATTER_STORE:
return 5;
case IFN_VCOND_MASK:
@@ -5329,6 +5382,8 @@ internal_fn_stored_value_index (internal_fn fn)
case IFN_MASK_STORE:
case IFN_MASK_STORE_LANES:
return 3;
+
+ case IFN_MASK_COMPRESS_SCATTER_STORE:
case IFN_SCATTER_STORE:
case IFN_MASK_SCATTER_STORE:
case IFN_MASK_LEN_SCATTER_STORE:
@@ -5361,6 +5416,7 @@ internal_fn_alias_ptr_index (internal_fn fn)
case IFN_SCATTER_STORE:
case IFN_MASK_SCATTER_STORE:
case IFN_MASK_LEN_SCATTER_STORE:
+ case IFN_MASK_COMPRESS_SCATTER_STORE:
return 1;
default:
@@ -5385,6 +5441,7 @@ internal_fn_offset_index (internal_fn fn)
case IFN_SCATTER_STORE:
case IFN_MASK_SCATTER_STORE:
case IFN_MASK_LEN_SCATTER_STORE:
+ case IFN_MASK_COMPRESS_SCATTER_STORE:
return 2;
default:
@@ -5409,6 +5466,7 @@ internal_fn_scale_index (internal_fn fn)
case IFN_SCATTER_STORE:
case IFN_MASK_SCATTER_STORE:
case IFN_MASK_LEN_SCATTER_STORE:
+ case IFN_MASK_COMPRESS_SCATTER_STORE:
return 3;
default:
diff --git a/gcc/internal-fn.def b/gcc/internal-fn.def
index af9f92950c7..7bb2ff37edd 100644
--- a/gcc/internal-fn.def
+++ b/gcc/internal-fn.def
@@ -639,6 +639,10 @@ DEF_INTERNAL_FN (ASSUME, ECF_CONST | ECF_LEAF | ECF_NOTHROW
/* For if-conversion of inbranch SIMD clones. */
DEF_INTERNAL_FN (MASK_CALL, ECF_NOVOPS, NULL)
+/* For Predicated index PHI / compress-scatter . */
+DEF_INTERNAL_FN (PREDICATED_INDEX_UPDATE, 0, NULL)
+DEF_INTERNAL_FN (MASK_COMPRESS_SCATTER_STORE, 0, NULL)
+
/* _BitInt support. */
DEF_INTERNAL_FN (MULBITINT, ECF_LEAF | ECF_NOTHROW, ". O . R . R . ")
DEF_INTERNAL_FN (DIVMODBITINT, ECF_LEAF, ". O . O . R . R . ")
diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc
index 003f9565a2a..b93a6648acf 100644
--- a/gcc/tree-vect-loop.cc
+++ b/gcc/tree-vect-loop.cc
@@ -788,7 +788,12 @@ _loop_vec_info::_loop_vec_info (class loop *loop_in,
vec_info_shared *shared)
early_break_needs_epilogue (false),
early_break_niters_var (NULL),
predicated_index_phi (NULL),
- predicated_index_mask_store (NULL)
+ predicated_index_mask_store (NULL),
+ predicated_index_store_base (NULL_TREE),
+ predicated_index_store_scale (NULL_TREE),
+ predicated_index_mask_compress_scatter_store (NULL),
+ predicated_index_update_ifn_p (false),
+ predicated_index_compress_scatter_ifn_p (false)
{
/* CHECKME: We want to visit all BBs before their successors (except for
latch blocks, for which this assertion wouldn't hold). In the simple
diff --git a/gcc/tree-vect-patterns.cc b/gcc/tree-vect-patterns.cc
index ab7892b6c7b..8aab8a6de1c 100644
--- a/gcc/tree-vect-patterns.cc
+++ b/gcc/tree-vect-patterns.cc
@@ -41,6 +41,7 @@ along with GCC; see the file COPYING3. If not see
#include "gimplify-me.h"
#include "cfgloop.h"
#include "tree-vectorizer.h"
+#include "tree-vect-vpcompress.h"
#include "dumpfile.h"
#include "builtins.h"
#include "internal-fn.h"
@@ -7453,6 +7454,7 @@ static vect_recog_func vect_vect_recog_func_ptrs[] = {
possible beforehand. */
{ vect_recog_average_pattern, "average" },
{ vect_recog_cond_expr_convert_pattern, "cond_expr_convert" },
+ { vect_recog_predicated_index_update_pattern, "predicated_index_update" },
{ vect_recog_mulhs_pattern, "mult_high" },
{ vect_recog_cast_forwprop_pattern, "cast_forwprop" },
{ vect_recog_widen_mult_pattern, "widen_mult" },
@@ -7477,6 +7479,7 @@ static vect_recog_func vect_vect_recog_func_ptrs[] = {
of mask conversion that are needed for gather and scatter
internal functions. */
{ vect_recog_gather_scatter_pattern, "gather_scatter" },
+ { vect_recog_compress_scatter_pattern, "compress_scatter" },
{ vect_recog_cond_store_pattern, "cond_store" },
{ vect_recog_mask_conversion_pattern, "mask_conversion" },
{ vect_recog_widen_plus_pattern, "widen_plus" },
diff --git a/gcc/tree-vect-vpcompress.cc b/gcc/tree-vect-vpcompress.cc
index c1a87405b94..0060ad5d780 100644
--- a/gcc/tree-vect-vpcompress.cc
+++ b/gcc/tree-vect-vpcompress.cc
@@ -164,27 +164,31 @@ predicated_index_plus_one (tree &pred_phi,
return false;
}
-/* Function pred_phi_in_mask_store_addr_computation.
+/* Function pred_phi_mask_store_base_and_scale.
- Starting from the store pointer in MASK_STORE_CALL, this function tries to
- check if the base + offset computation is done and offset finaly reaches
- PRED_PHI.
+ Starting from the store pointer in MASK_STORE_CALL, check that the address
+ has the form base + pred_phi * element_size and return BASE_OUT and
+ SCALE_OUT on success.
# n = PHI <n_1, ..>
...
a_1 = (long unsigned int) n;
a_2 = a_1 * 4;
a_3 = base + a_2;
- addr = (int *) a_3;
+ addr = (int *) a_3;
.MASK_STORE (addr, mask, ..)
n_1 = n + 1;
...
- n_2 = mask ? n_1 : n; */
+ n_2 = mask ? n_1 : n;
+
+ On success BASE_OUT is base and SCALE_OUT is 4. */
+
static bool
-pred_phi_in_mask_store_addr_computation (class loop *loop,
- gcall *mask_store_call, tree pred_phi)
+pred_phi_mask_store_base_and_scale (class loop *loop,
+ gcall *mask_store_call, tree pred_phi,
+ tree *base_out, tree *scale_out)
{
- /* Get the pointer operand of MASK_SLP_CALL. 'addr' in the example. */
+ /* Get the pointer operand of MASK_STORE_CALL. 'addr' in the example. */
tree ptr = gimple_call_arg (mask_store_call, 0);
if (TREE_CODE (ptr) != SSA_NAME)
return false;
@@ -222,21 +226,21 @@ pred_phi_in_mask_store_addr_computation (class loop *loop,
if (!add_stmt)
return false;
- /* Check if add_stmt is base + offset format. Note that it can be either
- base + offset or offset + base. We take the loop invariant pointer as
- the base. Example - a_3 = base + a_2; */
+ /* Identify the loop-invariant base and the varying offset.
+ a_3 = base + a_2; */
tree add_op0 = gimple_assign_rhs1 (add_stmt);
tree add_op1 = gimple_assign_rhs2 (add_stmt);
- tree offset;
+ tree base = add_op0;
+ tree offset = add_op1;
if (expr_invariant_in_loop_p (loop, add_op1))
- offset = add_op0;
- else if (expr_invariant_in_loop_p (loop, add_op0))
- offset = add_op1;
- else
+ {
+ base = add_op1;
+ offset = add_op0;
+ }
+ else if (!expr_invariant_in_loop_p (loop, add_op0))
return false;
- /* Now offset can be a multiplication with the element size as in
- a_2 = a_1 * 4; */
+ /* Offset should be index * element size, as in a_2 = a_1 * 4. */
if (TREE_CODE (offset) != SSA_NAME)
return false;
gimple *def = SSA_NAME_DEF_STMT (offset);
@@ -245,7 +249,7 @@ pred_phi_in_mask_store_addr_computation (class loop *loop,
return false;
tree mul_op0 = gimple_assign_rhs1 (mul);
tree mul_op1 = gimple_assign_rhs2 (mul);
- tree index_op = mul_op0;
+ tree index_op = mul_op0;
tree size_op = mul_op1;
if (TREE_CODE (mul_op0) == INTEGER_CST)
{
@@ -254,10 +258,11 @@ pred_phi_in_mask_store_addr_computation (class loop *loop,
}
else if (TREE_CODE (mul_op1) != INTEGER_CST)
return false;
- /* The index_op should be an SSA_NAM. */
+ /* The index operand should be an SSA_NAME. */
if (TREE_CODE (index_op) != SSA_NAME)
- return false;
- /* The size_op should be the element size pointed by ptr. */
+ return false;
+
+ /* SIZE_OP must match the element size of the store pointer type. */
tree eltype = TREE_TYPE (TREE_TYPE (ptr));
tree elem_size_unit = TYPE_SIZE_UNIT (eltype);
if (!elem_size_unit || TREE_CODE (elem_size_unit) != INTEGER_CST)
@@ -265,12 +270,16 @@ pred_phi_in_mask_store_addr_computation (class loop *loop,
if (!operand_equal_p (size_op, elem_size_unit, OEP_ONLY_CONST))
return false;
- /* No the index_op can be a casted one. So strip off that cast as in
- a_1 = (long unsigned int) n; */
+ /* Index may be cast, as in a_1 = (long unsigned int) n. */
pred_phi_strip_conversion (&index_op);
+ if (index_op != pred_phi)
+ return false;
- /* Finally index_op should be PRED_PHI. */
- return index_op == pred_phi;
+ if (base_out)
+ *base_out = base;
+ if (scale_out)
+ *scale_out = elem_size_unit;
+ return true;
}
/* Function vect_find_predicated_index_mask_store.
@@ -295,7 +304,8 @@ pred_phi_in_mask_store_addr_computation (class loop *loop,
n_2 = mask ? n_1 : n; */
static gcall *
vect_find_predicated_index_mask_store (class loop *loop, gphi *phi,
- tree pred_phi, tree pred_phi_mask)
+ tree pred_phi, tree pred_phi_mask,
+ tree *base_out, tree *scale_out)
{
/* Search only the basic block that holds PHI; TODO: to scan other blocks
when generalizing. */
@@ -317,9 +327,8 @@ vect_find_predicated_index_mask_store (class loop *loop,
gphi *phi,
continue;
/* Check if PRED_PHI is used to compute mask_store_call address. */
- /* TODO: Can use any match operator (as in instcombine)?? */
- if (!pred_phi_in_mask_store_addr_computation(loop, mask_store_call,
- pred_phi))
+ if (!pred_phi_mask_store_base_and_scale (loop, mask_store_call, pred_phi,
+ base_out, scale_out))
continue;
if (dump_enabled_p ())
@@ -349,8 +358,8 @@ vect_find_predicated_index_mask_store (class loop *loop,
gphi *phi,
...
n_2 = mask ? n_1 : n; */
bool
-vect_vpcompress_analyze_predicated_index_phi (loop_vec_info loop_vinfo,
- class loop *loop, gphi *phi)
+vect_vpcompress_analyze_predicated_index_phi (loop_vec_info loop_vinfo, class
loop *loop,
+ gphi *phi)
{
/* Require a loop-header PHI. */
if (gimple_bb (phi) != loop->header)
@@ -397,8 +406,11 @@ vect_vpcompress_analyze_predicated_index_phi
(loop_vec_info loop_vinfo,
* computed based on pred-phi and it should use the same mask as that of
* COND_EXPR. */
tree mask = gimple_assign_rhs1 (pred_phi_back_edge_asgn_stmt);
+ tree store_base;
+ tree store_scale;
gcall *mask_store =
- vect_find_predicated_index_mask_store (loop, phi, pred_phi, mask);
+ vect_find_predicated_index_mask_store (loop, phi, pred_phi, mask,
+ &store_base, &store_scale);
if (dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location,
@@ -415,7 +427,156 @@ vect_vpcompress_analyze_predicated_index_phi
(loop_vec_info loop_vinfo,
loop_vinfo->predicated_index_phi = phi;
loop_vinfo->predicated_index_mask_store = mask_store;
+ loop_vinfo->predicated_index_store_base = store_base;
+ loop_vinfo->predicated_index_store_scale = store_scale;
/* All requirements satisfiled. */
return true;
}
+
+/* Function vect_recog_predicated_index_update_pattern.
+
+ After phase-1 detection, rewrite the predicated-index PHI latch update
+ from a COND_EXPR into IFN_PREDICATED_INDEX_UPDATE. The statement must be
+ the latch definition of the predicated-index PHI and use the same mask as
+ the paired IFN_MASK_STORE.
+
+ # n = PHI <n_1, 0>
+ ...
+ .MASK_STORE (addr, mask, ..)
+ ...
+ n_2 = mask ? n_1 : n;
+
+ is rewritten to:
+
+ n_2 = IFN_PREDICATED_INDEX_UPDATE (n, mask); */
+
+gimple *
+vect_recog_predicated_index_update_pattern (vec_info *vinfo,
+ stmt_vec_info stmt_info,
+ tree *type_out)
+{
+ loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
+ /* Require a loop that already matched the predicated-index PHI pattern. */
+ if (!loop_vinfo
+ || !loop_vinfo->predicated_index_phi
+ || !loop_vinfo->predicated_index_mask_store)
+ return NULL;
+
+ gphi *phi = loop_vinfo->predicated_index_phi;
+ class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
+ tree pred_phi = PHI_RESULT (phi);
+ /* STMT_INFO must be the latch-edge definition of the predicated-index PHI,
+ e.g. n_2 = mask ? n_1 : n. */
+ tree latch_val = PHI_ARG_DEF_FROM_EDGE (phi, loop_latch_edge (loop));
+ if (TREE_CODE (latch_val) != SSA_NAME
+ || SSA_NAME_DEF_STMT (latch_val) != stmt_info->stmt)
+ return NULL;
+
+ gassign *asg = dyn_cast <gassign *> (stmt_info->stmt);
+ if (!asg || gimple_assign_rhs_code (asg) != COND_EXPR)
+ return NULL;
+
+ /* The COND_EXPR condition must be the same mask used by the mask store. */
+ tree mask = gimple_assign_rhs1 (asg);
+ if (TREE_CODE (mask) != SSA_NAME)
+ return NULL;
+
+ gcall *mask_store = loop_vinfo->predicated_index_mask_store;
+ if (gimple_call_arg (mask_store, 2) != mask)
+ return NULL;
+
+ /* Build IFN_PREDICATED_INDEX_UPDATE (pred_phi, mask). */
+ gcall *call
+ = gimple_build_call_internal (IFN_PREDICATED_INDEX_UPDATE, 2, pred_phi,
+ mask);
+ gimple_call_set_lhs (call, gimple_assign_lhs (asg));
+ gimple_set_location (call, gimple_location (asg));
+ gimple_call_set_nothrow (call, true);
+
+ /* pred_phi stays scalar even in the vector loop (idx + popcount (mask) per
+ vector iteration). No vector SSA is produced here, so leave type_out
+ unset for the generic pattern machinery. */
+ *type_out = NULL_TREE;
+ loop_vinfo->predicated_index_update_ifn_p = true;
+
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location, "%s: detected: %G",
+ "predicated_index_update", stmt_info->stmt);
+
+ return call;
+}
+
+/* Function vect_recog_compress_scatter_pattern.
+
+ After phase-1 detection, replace the paired IFN_MASK_STORE with
+ IFN_MASK_COMPRESS_SCATTER_STORE. Base and scale were recorded during
+ phase-1 address analysis.
+
+ # n = PHI <n_1, ..>
+ ...
+ a_1 = (long unsigned int) n;
+ a_2 = a_1 * 4;
+ a_3 = base + a_2;
+ addr = (int *) a_3;
+ .MASK_STORE (addr, mask, ..);
+
+ is rewritten to:
+
+ IFN_MASK_COMPRESS_SCATTER_STORE (base, align, n, 4, .., mask); */
+
+gimple *
+vect_recog_compress_scatter_pattern (vec_info *vinfo,
+ stmt_vec_info stmt_info,
+ tree *type_out)
+{
+ loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
+ /* Require a loop that already matched the predicated-index PHI pattern. */
+ if (!loop_vinfo
+ || !loop_vinfo->predicated_index_phi
+ || !loop_vinfo->predicated_index_mask_store)
+ return NULL;
+
+ /* STMT_INFO must be the IFN_MASK_STORE recorded during phase-1. */
+ gcall *orig = dyn_cast <gcall *> (stmt_info->stmt);
+ if (!orig || orig != loop_vinfo->predicated_index_mask_store)
+ return NULL;
+ if (!gimple_call_internal_p (orig, IFN_MASK_STORE))
+ return NULL;
+
+ tree pred_phi = PHI_RESULT (loop_vinfo->predicated_index_phi);
+ tree mask = gimple_call_arg (orig, 2);
+ tree value = gimple_call_arg (orig, 3);
+ tree pack_idx = pred_phi;
+ tree base = loop_vinfo->predicated_index_store_base;
+ tree scale = loop_vinfo->predicated_index_store_scale;
+ if (!base || !scale)
+ return NULL;
+
+ tree align = gimple_call_arg (orig, 1);
+
+ /* Build IFN_MASK_COMPRESS_SCATTER_STORE (base, align, pack_idx, scale,
+ value, mask). */
+ gcall *call
+ = gimple_build_call_internal (IFN_MASK_COMPRESS_SCATTER_STORE, 6,
+ base, align, pack_idx, scale, value, mask);
+ gimple_set_location (call, gimple_location (orig));
+ gimple_call_set_nothrow (call, true);
+
+ tree scalar_type = TREE_TYPE (value);
+ tree vectype = get_vectype_for_scalar_type (vinfo, scalar_type);
+ if (vectype == NULL_TREE)
+ return NULL;
+
+ stmt_vec_info pattern_stmt_info = loop_vinfo->add_stmt (call);
+ loop_vinfo->move_dr (pattern_stmt_info, stmt_info);
+
+ loop_vinfo->predicated_index_mask_compress_scatter_store = call;
+ loop_vinfo->predicated_index_compress_scatter_ifn_p = true;
+
+ *type_out = vectype;
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location, "%s: detected: %G",
+ "compress_scatter", stmt_info->stmt);
+ return call;
+}
diff --git a/gcc/tree-vect-vpcompress.h b/gcc/tree-vect-vpcompress.h
index b8d0093831f..0df8bf95ce1 100644
--- a/gcc/tree-vect-vpcompress.h
+++ b/gcc/tree-vect-vpcompress.h
@@ -24,4 +24,10 @@ along with GCC; see the file COPYING3. If not see
extern bool vect_vpcompress_analyze_predicated_index_phi (loop_vec_info,
class loop *, gphi *);
+extern gimple *vect_recog_predicated_index_update_pattern (vec_info *,
+ stmt_vec_info,
+ tree *);
+extern gimple *vect_recog_compress_scatter_pattern (vec_info *,
+ stmt_vec_info,
+ tree *);
#endif /* GCC_TREE_VECT_VPCOMPRESS_H */
diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
index 82c79e101d7..90d9e723ed0 100644
--- a/gcc/tree-vectorizer.h
+++ b/gcc/tree-vectorizer.h
@@ -1278,6 +1278,13 @@ public:
/* Predicated-index PHI and matching IFN_MASK_STORE. */
gphi *predicated_index_phi;
gcall *predicated_index_mask_store;
+ tree predicated_index_store_base;
+ tree predicated_index_store_scale;
+ /* Non-NULL after compress-scatter pattern installs
IFN_MASK_COMPRESS_SCATTER_STORE. */
+ gcall *predicated_index_mask_compress_scatter_store;
+ /* Set when pattern recog rewrites the idiom to internal functions. */
+ bool predicated_index_update_ifn_p;
+ bool predicated_index_compress_scatter_ifn_p;
} *loop_vec_info;
/* Access Functions. */
--
2.34.1
