On Tue, 30 Jun 2026, Raghesh Aloor wrote:
> Refactor the vpcompress phase-1 helpers out of tree-vect-loop.cc into
> tree-vect-vpcompress.{cc,h} and wire the new object into the backend
> build. No intended functional change: pred_phi_strip_conversion,
> predicated_index_plus_one, mask-store address matching, and PHI
> classification are moved as-is, with vect_analyze_predicated_index_phi
> renamed to vect_vpcompress_analyze_predicated_index_phi.
Please don't add new files, this is just inconsistent with the rest.
> ---
> gcc/Makefile.in | 1 +
> gcc/tree-vect-loop.cc | 387 +--------------------------------
> gcc/tree-vect-vpcompress.cc | 421 ++++++++++++++++++++++++++++++++++++
> gcc/tree-vect-vpcompress.h | 27 +++
> 4 files changed, 451 insertions(+), 385 deletions(-)
> create mode 100644 gcc/tree-vect-vpcompress.cc
> create mode 100644 gcc/tree-vect-vpcompress.h
>
> diff --git a/gcc/Makefile.in b/gcc/Makefile.in
> index 536e2faaaf8..21c01e8f12f 100644
> --- a/gcc/Makefile.in
> +++ b/gcc/Makefile.in
> @@ -1839,6 +1839,7 @@ OBJS = \
> tree-vect-data-refs.o \
> tree-vect-stmts.o \
> tree-vect-loop.o \
> + tree-vect-vpcompress.o \
> tree-vect-loop-manip.o \
> tree-vect-slp.o \
> tree-vect-slp-patterns.o \
> diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc
> index 139a574d675..003f9565a2a 100644
> --- a/gcc/tree-vect-loop.cc
> +++ b/gcc/tree-vect-loop.cc
> @@ -48,6 +48,7 @@ along with GCC; see the file COPYING3. If not see
> #include "cfgloop.h"
> #include "tree-scalar-evolution.h"
> #include "tree-vectorizer.h"
> +#include "tree-vect-vpcompress.h"
> #include "gimple-fold.h"
> #include "cgraph.h"
> #include "tree-cfg.h"
> @@ -164,136 +165,6 @@ static void vect_estimate_min_profitable_iters
> (loop_vec_info, int *, int *,
> static stmt_vec_info vect_is_simple_reduction (loop_vec_info, stmt_vec_info,
> gphi **);
>
> -
> -/* Function pred_phi_strip_conversion.
> -
> - This function check if CUR is an assigment statement and if it is an SSA
> - whose defining statement is NOP/CONVERT/VIEW_CONVERT/NON_LVALUE to SSA.
> - Stores that rhs SSA into CUR
> -
> - t_1 = (unsigned int) n;
> -
> - Stores n into CUR. */
> -static bool
> -pred_phi_strip_conversion (tree *cur)
> -{
> - if (TREE_CODE (*cur) != SSA_NAME)
> - return false;
> -
> - gimple *def = SSA_NAME_DEF_STMT (*cur);
> - gassign *assign = dyn_cast <gassign *> (def);
> - if (!assign)
> - return false;
> -
> - tree_code rhs_code = gimple_assign_rhs_code (assign);
> - if (rhs_code != NOP_EXPR
> - && rhs_code != CONVERT_EXPR
> - && rhs_code != VIEW_CONVERT_EXPR
> - && rhs_code != NON_LVALUE_EXPR)
> - return false;
> -
> - tree rhs1 = gimple_assign_rhs1 (assign);
> - if (TREE_CODE (rhs1) != SSA_NAME)
> - return false;
> -
> - *cur = rhs1;
> - return true;
> -}
> -
> -/* Function predicated_index_plus_one.
> -
> - This function checks if one of the arms of the COND_EXPR,
> - PRED_PHI_BACK_EDGE_ASGN_STMT is PRED_PHI itself and the other is
> - PRED_PHI + 1. Note that the increment can be passed through cast
> - instructions as in the below example. So we need to traverse through them.
> -
> - # n = PHI <n_1, ..>
> - ...
> - addr = base + n;
> - .MASK_STORE (addr, mask, ..)
> - t_1 = (unsigned int) n;
> - t_2 = t1_1 + 1;
> - n_1 = int (t_2);
> - n_2 = mask ? n_1 : n; */
> -static bool
> -predicated_index_plus_one (tree &pred_phi,
> - gassign *pred_phi_back_edge_asgn_stmt)
> -{
> - /* Check that one of the arms of COND_EXPR is pred-phi (n) and the other is
> - an increment value (n_1 = n + 1). */
> - tree true_val = gimple_assign_rhs2 (pred_phi_back_edge_asgn_stmt);
> - tree false_val = gimple_assign_rhs3 (pred_phi_back_edge_asgn_stmt);
> - tree pred_phi_inc_val = NULL_TREE;
> - if (true_val == pred_phi)
> - pred_phi_inc_val = false_val;
> - else if (false_val == pred_phi)
> - pred_phi_inc_val = true_val;
> - else
> - return false;
> - if (TREE_CODE (pred_phi_inc_val) != SSA_NAME)
> - return false;
> -
> - /* Check if n1 = n + 1?. We may need to strip the cast instructions to
> reach
> - the PLUS_EXPR. TODO: Change depth logic - traverse until CUR go out of
> - the current basic block. */
> - gimple *cur = SSA_NAME_DEF_STMT (pred_phi_inc_val);
> - for (unsigned depth = 0; depth < 8; depth++)
> - {
> - /* CUR should be an assignment statement. */
> - gassign *cur_asgn = dyn_cast <gassign *> (cur);
> - if (!cur_asgn)
> - return false;
> - /* Stop traversing further if we reach a PLUS_EXPR. */
> - if (gimple_assign_rhs_code (cur_asgn) == PLUS_EXPR)
> - {
> - /* Check for +1 increment. It can be either t_1 + 1 or 1 + t_1. */
> - tree rhs1 = gimple_assign_rhs1 (cur_asgn);
> - tree rhs2 = gimple_assign_rhs2 (cur_asgn);
> - tree plus_base = NULL_TREE;
> - if (TREE_CODE (rhs1) == SSA_NAME && TREE_CODE (rhs2) == INTEGER_CST
> - && integer_onep (rhs2))
> - plus_base = rhs1;
> - else if (TREE_CODE (rhs2) == SSA_NAME &&
> - TREE_CODE (rhs1) == INTEGER_CST && integer_onep (rhs1))
> - plus_base = rhs2;
> - /* If plus_base is not set we dont have a +1 pattern. */
> - if (!plus_base)
> - return false;
> -
> - /* Check if plus_base is pred_phi itself. We may still need to strip
> - some casts to reacy the actual base. TODO: Change depth logic -
> - traverse untill plus_base go out of current basic block. */
> - for (unsigned depth_1 = 0; depth_1 < 8; depth_1++)
> - {
> - if (!pred_phi_strip_conversion (&plus_base))
> - break;
> - }
> - /* The stripped base should be pred_phi. */
> - if (plus_base == pred_phi)
> - {
> - if (dump_enabled_p ())
> - dump_printf_loc (MSG_NOTE, vect_location,
> - "predicated_index_plus_one match: found\n");
> - return true;
> - }
> - /* We don't have +1 pattern. */
> - return false;
> - }
> - /* If we reach an non-PLUS_EXPR check if we can continue walk through
> - casts */
> - tree cur_ssa = gimple_assign_lhs (cur_asgn);
> - if (pred_phi_strip_conversion(&cur_ssa))
> - {
> - cur = SSA_NAME_DEF_STMT (cur_ssa);
> - continue;
> - }
> - /* We cannot reach a PLUS_EXPR. */
> - return false;
> - }
> -
> - return false;
> -}
> -
> /* Function vect_is_simple_iv_evolution.
>
> FORNOW: A simple evolution of an induction variables in the loop is
> @@ -462,261 +333,7 @@ vect_phi_first_order_recurrence_p (loop_vec_info
> loop_vinfo, class loop *loop,
> return true;
> }
>
> -/* Function pred_phi_in_mask_store_addr_computation.
>
> - Starting from the store pointer in MASK_STORE_CALL, this function tries to
> - check if the base + offset computation is done and offset finaly reaches
> - PRED_PHI.
> -
> - # n = PHI <n_1, ..>
> - ...
> - a_1 = (long unsigned int) n;
> - a_2 = a_1 * 4;
> - a_3 = base + a_2;
> - addr = (int *) a_3;
> - .MASK_STORE (addr, mask, ..)
> - n_1 = n + 1;
> - ...
> - n_2 = mask ? n_1 : n; */
> -static bool
> -pred_phi_in_mask_store_addr_computation (class loop *loop,
> - gcall *mask_store_call, tree pred_phi)
> -{
> - /* Get the pointer operand of MASK_SLP_CALL. 'addr' in the example. */
> - tree ptr = gimple_call_arg (mask_store_call, 0);
> - if (TREE_CODE (ptr) != SSA_NAME)
> - return false;
> -
> - /* Strip off the cast and see if we can reach to reach a PLUS_EXPR.
> - a_3 = base + a_2;
> - addr = (int *) a_3; */
> - gassign *add_stmt = NULL;
> - tree cur = ptr;
> - /* TODO: traverse until CUR go out of the current bb. */
> - /* TODO: Can we club it with pred_phi_strip_conversion */
> - for (unsigned depth = 0; depth < 8; depth++)
> - {
> - gimple *def = SSA_NAME_DEF_STMT (cur);
> - gassign *a = dyn_cast <gassign *> (def);
> - if (!a)
> - return false;
> -
> - tree_code code = gimple_assign_rhs_code (a);
> - if (code == PLUS_EXPR || code == POINTER_PLUS_EXPR)
> - {
> - add_stmt = a;
> - break;
> - }
> - if (code != NOP_EXPR && code != CONVERT_EXPR
> - && code != VIEW_CONVERT_EXPR && code != NON_LVALUE_EXPR)
> - return false;
> -
> - tree rhs1 = gimple_assign_rhs1 (a);
> - if (TREE_CODE (rhs1) != SSA_NAME)
> - return false;
> - cur = rhs1;
> - }
> -
> - if (!add_stmt)
> - return false;
> -
> - /* Check if add_stmt is base + offset format. Note that it can be either
> - base + offset or offset + base. We take the loop invariant pointer as
> - the base. Example - a_3 = base + a_2; */
> - tree add_op0 = gimple_assign_rhs1 (add_stmt);
> - tree add_op1 = gimple_assign_rhs2 (add_stmt);
> - tree offset;
> - if (expr_invariant_in_loop_p (loop, add_op1))
> - offset = add_op0;
> - else if (expr_invariant_in_loop_p (loop, add_op0))
> - offset = add_op1;
> - else
> - return false;
> -
> - /* Now offset can be a multiplication with the element size as in
> - a_2 = a_1 * 4; */
> - if (TREE_CODE (offset) != SSA_NAME)
> - return false;
> - gimple *def = SSA_NAME_DEF_STMT (offset);
> - gassign *mul = dyn_cast <gassign *> (def);
> - if (!mul || gimple_assign_rhs_code (mul) != MULT_EXPR)
> - return false;
> - tree mul_op0 = gimple_assign_rhs1 (mul);
> - tree mul_op1 = gimple_assign_rhs2 (mul);
> - tree index_op = mul_op0;
> - tree size_op = mul_op1;
> - if (TREE_CODE (mul_op0) == INTEGER_CST)
> - {
> - index_op = mul_op1;
> - size_op = mul_op0;
> - }
> - else if (TREE_CODE (mul_op1) != INTEGER_CST)
> - return false;
> - /* The index_op should be an SSA_NAM. */
> - if (TREE_CODE (index_op) != SSA_NAME)
> - return false;
> - /* The size_op should be the element size pointed by ptr. */
> - tree eltype = TREE_TYPE (TREE_TYPE (ptr));
> - tree elem_size_unit = TYPE_SIZE_UNIT (eltype);
> - if (!elem_size_unit || TREE_CODE (elem_size_unit) != INTEGER_CST)
> - return false;
> - if (!operand_equal_p (size_op, elem_size_unit, OEP_ONLY_CONST))
> - return false;
> -
> - /* No the index_op can be a casted one. So strip off that cast as in
> - a_1 = (long unsigned int) n; */
> - pred_phi_strip_conversion (&index_op);
> -
> - /* Finally index_op should be PRED_PHI. */
> - return index_op == pred_phi;
> -}
> -
> -/* Function vect_find_predicated_index_mask_store.
> -
> - This function establishes there relationship between the PRED_PHI detected
> - so far with an IFN_MASK_STORE. The store address for IFN_MASK_STORE should
> - be computed using PRED_PHI as index and PRED_PHI_MASK should be used as
> its
> - mask. Note that the address computation is done as (base + n * size) and
> - can pass through multiiple cast instructions as in the below example.
> - So we need to traverse through them.
> -
> -
> - # n = PHI <n_1, ..>
> - ...
> - a_1 = (long unsigned int) n;
> - a_2 = a_1 * 4;
> - a_3 = base + a_2;
> - addr = (int *) a_3;
> - .MASK_STORE (addr, mask, ..)
> - n_1 = n + 1;
> - ...
> - n_2 = mask ? n_1 : n; */
> -static gcall *
> -vect_find_predicated_index_mask_store (class loop *loop, gphi *phi,
> - tree pred_phi, tree pred_phi_mask)
> -{
> - /* Search only the basic block that holds PHI; TODO: to scan other blocks
> - when generalizing. */
> - basic_block pred_bb = gimple_bb (phi);
> - /* Iterate over GIMPLE statements to find a matching IFN_MASK_STORE. */
> - for (gimple_stmt_iterator gsi = gsi_after_labels (pred_bb); !gsi_end_p
> (gsi);
> - gsi_next (&gsi))
> - {
> - gimple *st = gsi_stmt (gsi);
> -
> - /* If not an IFN_MASK_STORE call continue. */
> - gcall *mask_store_call = dyn_cast <gcall *> (st);
> - if (!mask_store_call || !gimple_call_internal_p (mask_store_call)
> - || gimple_call_internal_fn (mask_store_call) != IFN_MASK_STORE)
> - continue;
> -
> - /* Check if the mask of call is same as PRED_PHI_MASK. */
> - if (gimple_call_arg (mask_store_call, 2) != pred_phi_mask)
> - continue;
> -
> - /* Check if PRED_PHI is used to compute mask_store_call address. */
> - /* TODO: Can use any match operator (as in instcombine)?? */
> - if (!pred_phi_in_mask_store_addr_computation(loop, mask_store_call,
> - pred_phi))
> - continue;
> -
> - if (dump_enabled_p ())
> - dump_printf_loc (MSG_NOTE, vect_location,
> - "vect_find_predicated_index_mask_store: matched "
> - "IFN_MASK_STORE %G\n",
> - (gimple *) mask_store_call);
> -
> - return mask_store_call;
> - }
> -
> - if (TREE_CODE (pred_phi_mask) != SSA_NAME)
> - return NULL;
> - return NULL;
> -}
> -
> -/* Part of vpcompress pattern detection. Checks if PHI matches the
> predicated-
> - index phi shape, which expects its incoming value from the backedge to be
> - incremented under the same mask used by an IFN_MASK_STORE, whose address
> is
> - indexed by the PHI itself. An outline of the pattern is given below.
> -
> - # n = PHI <n_1, ..>
> - ...
> - addr = base + n;
> - .MASK_STORE (addr, mask, ..)
> - n_1 = n + 1;
> - ...
> - n_2 = mask ? n_1 : n; */
> -static bool
> -vect_analyze_predicated_index_phi (loop_vec_info loop_vinfo, class loop
> *loop,
> - gphi *phi)
> -{
> - /* Require a loop-header PHI. */
> - if (gimple_bb (phi) != loop->header)
> - return false;
> -
> - tree pred_phi = PHI_RESULT (phi);
> - /* PHI should be an integer. */
> - if (TREE_CODE (pred_phi) != SSA_NAME
> - || TREE_CODE (TREE_TYPE (pred_phi)) != INTEGER_TYPE)
> - return false;
> -
> - /* Restricting to zero initializatin from preheader for now. TODO: Can be
> - generalized to arbitrary integer starts. */
> - tree pre_init = PHI_ARG_DEF_FROM_EDGE (phi, loop_preheader_edge (loop));
> - if (TREE_CODE (pre_init) != INTEGER_CST || !integer_zerop (pre_init))
> - return false;
> -
> - /* The incoming value for pred-phi must be in the if-converted form.
> - n_2 = mask ? n_1 : n; TODO: This may be generalized later for more
> - patterns. */
> - /* Get the incoming value (n_2) of pred-phi through backedge. */
> - tree pred_phi_back_edge_val =
> - PHI_ARG_DEF_FROM_EDGE (phi, loop_latch_edge (loop));
> - if (TREE_CODE (pred_phi_back_edge_val) != SSA_NAME)
> - return false;
> - /* Check if pred_phi_back_edge_val is a COND_EXPR. */
> - gimple *pred_phi_back_edge_stmt
> - = SSA_NAME_DEF_STMT (pred_phi_back_edge_val);
> - if (!pred_phi_back_edge_stmt)
> - return false;
> - gassign *pred_phi_back_edge_asgn_stmt
> - = dyn_cast <gassign *> (pred_phi_back_edge_stmt);
> - if (!pred_phi_back_edge_asgn_stmt
> - || gimple_assign_rhs_code (pred_phi_back_edge_asgn_stmt) != COND_EXPR)
> - return false;
> -
> - /* Check that one of the arms of COND_EXPR is pred-phi (n) and the other is
> - an increment value (n_1 = n + 1). */
> - if (!predicated_index_plus_one (pred_phi, pred_phi_back_edge_asgn_stmt))
> - return false;
> -
> - /* Check if there is a IFN_MASK_STORE, which is connected to the same
> - * phi that we detected (pred-phi) so far. The store address should be
> - * computed based on pred-phi and it should use the same mask as that of
> - * COND_EXPR. */
> - tree mask = gimple_assign_rhs1 (pred_phi_back_edge_asgn_stmt);
> - gcall *mask_store =
> - vect_find_predicated_index_mask_store (loop, phi, pred_phi, mask);
> -
> - if (dump_enabled_p ())
> - dump_printf_loc (MSG_NOTE, vect_location,
> - "predicated-index PHI: matched predicated phi %G\n",
> - (gimple *) phi);
> -
> - if (!mask_store)
> - return false;
> -
> - /* Keep at most one predicated-index PHI per loop_vinfo. */
> - if (loop_vinfo->predicated_index_phi != NULL
> - && loop_vinfo->predicated_index_phi != phi)
> - return false;
> -
> - loop_vinfo->predicated_index_phi = phi;
> - loop_vinfo->predicated_index_mask_store = mask_store;
> -
> - /* All requirements satisfiled. */
> - return true;
> -}
> /* Function vect_analyze_scalar_cycles_1.
>
> Examine the cross iteration def-use cycles of scalar variables
> @@ -864,7 +481,7 @@ vect_analyze_scalar_cycles_1 (loop_vec_info loop_vinfo,
> class loop *loop)
> else if (vect_phi_first_order_recurrence_p (loop_vinfo, loop, phi))
> STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_first_order_recurrence;
> else if (loop == LOOP_VINFO_LOOP (loop_vinfo)
> - && vect_analyze_predicated_index_phi (loop_vinfo, loop, phi))
> + && vect_vpcompress_analyze_predicated_index_phi (loop_vinfo,
> loop, phi))
> {
> STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_predicated_index_def;
> if (dump_enabled_p ())
> diff --git a/gcc/tree-vect-vpcompress.cc b/gcc/tree-vect-vpcompress.cc
> new file mode 100644
> index 00000000000..c1a87405b94
> --- /dev/null
> +++ b/gcc/tree-vect-vpcompress.cc
> @@ -0,0 +1,421 @@
> +/* Pattern detection for vpcompress.
> + Copyright (C) 2016-2026 Free Software Foundation, Inc.
> +
> +This file is part of GCC.
> +
> +GCC is free software; you can redistribute it and/or modify it under
> +the terms of the GNU General Public License as published by the Free
> +Software Foundation; either version 3, or (at your option) any later
> +version.
> +
> +GCC is distributed in the hope that it will be useful, but WITHOUT ANY
> +WARRANTY; without even the implied warranty of MERCHANTABILITY or
> +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
> +for more details.
> +
> +You should have received a copy of the GNU General Public License
> +along with GCC; see the file COPYING3. If not see
> +<http://www.gnu.org/licenses/>. */
> +
> +#include "config.h"
> +#include "system.h"
> +#include "coretypes.h"
> +#include "backend.h"
> +#include "tree.h"
> +#include "gimple.h"
> +#include "tree-pass.h"
> +#include "ssa.h"
> +#include "fold-const.h"
> +#include "gimple-iterator.h"
> +#include "cfgloop.h"
> +#include "tree-ssa-loop.h"
> +#include "tree-ssa-loop-ivopts.h"
> +#include "diagnostic-core.h"
> +#include "tree-vectorizer.h"
> +#include "internal-fn.h"
> +#include "tree-vect-vpcompress.h"
> +
> +/* Function pred_phi_strip_conversion.
> +
> + This function check if CUR is an assigment statement and if it is an SSA
> + whose defining statement is NOP/CONVERT/VIEW_CONVERT/NON_LVALUE to SSA.
> + Stores that rhs SSA into CUR
> +
> + t_1 = (unsigned int) n;
> +
> + Stores n into CUR. */
> +static bool
> +pred_phi_strip_conversion (tree *cur)
> +{
> + if (TREE_CODE (*cur) != SSA_NAME)
> + return false;
> +
> + gimple *def = SSA_NAME_DEF_STMT (*cur);
> + gassign *assign = dyn_cast <gassign *> (def);
> + if (!assign)
> + return false;
> +
> + tree_code rhs_code = gimple_assign_rhs_code (assign);
> + if (rhs_code != NOP_EXPR
> + && rhs_code != CONVERT_EXPR
> + && rhs_code != VIEW_CONVERT_EXPR
> + && rhs_code != NON_LVALUE_EXPR)
> + return false;
> +
> + tree rhs1 = gimple_assign_rhs1 (assign);
> + if (TREE_CODE (rhs1) != SSA_NAME)
> + return false;
> +
> + *cur = rhs1;
> + return true;
> +}
> +
> +/* Function predicated_index_plus_one.
> +
> + This function checks if one of the arms of the COND_EXPR,
> + PRED_PHI_BACK_EDGE_ASGN_STMT is PRED_PHI itself and the other is
> + PRED_PHI + 1. Note that the increment can be passed through cast
> + instructions as in the below example. So we need to traverse through them.
> +
> + # n = PHI <n_1, ..>
> + ...
> + addr = base + n;
> + .MASK_STORE (addr, mask, ..)
> + t_1 = (unsigned int) n;
> + t_2 = t1_1 + 1;
> + n_1 = int (t_2);
> + n_2 = mask ? n_1 : n; */
> +static bool
> +predicated_index_plus_one (tree &pred_phi,
> + gassign *pred_phi_back_edge_asgn_stmt)
> +{
> + /* Check that one of the arms of COND_EXPR is pred-phi (n) and the other is
> + an increment value (n_1 = n + 1). */
> + tree true_val = gimple_assign_rhs2 (pred_phi_back_edge_asgn_stmt);
> + tree false_val = gimple_assign_rhs3 (pred_phi_back_edge_asgn_stmt);
> + tree pred_phi_inc_val = NULL_TREE;
> + if (true_val == pred_phi)
> + pred_phi_inc_val = false_val;
> + else if (false_val == pred_phi)
> + pred_phi_inc_val = true_val;
> + else
> + return false;
> + if (TREE_CODE (pred_phi_inc_val) != SSA_NAME)
> + return false;
> +
> + /* Check if n1 = n + 1?. We may need to strip the cast instructions to
> reach
> + the PLUS_EXPR. TODO: Change depth logic - traverse until CUR go out of
> + the current basic block. */
> + gimple *cur = SSA_NAME_DEF_STMT (pred_phi_inc_val);
> + for (unsigned depth = 0; depth < 8; depth++)
> + {
> + /* CUR should be an assignment statement. */
> + gassign *cur_asgn = dyn_cast <gassign *> (cur);
> + if (!cur_asgn)
> + return false;
> + /* Stop traversing further if we reach a PLUS_EXPR. */
> + if (gimple_assign_rhs_code (cur_asgn) == PLUS_EXPR)
> + {
> + /* Check for +1 increment. It can be either t_1 + 1 or 1 + t_1. */
> + tree rhs1 = gimple_assign_rhs1 (cur_asgn);
> + tree rhs2 = gimple_assign_rhs2 (cur_asgn);
> + tree plus_base = NULL_TREE;
> + if (TREE_CODE (rhs1) == SSA_NAME && TREE_CODE (rhs2) == INTEGER_CST
> + && integer_onep (rhs2))
> + plus_base = rhs1;
> + else if (TREE_CODE (rhs2) == SSA_NAME &&
> + TREE_CODE (rhs1) == INTEGER_CST && integer_onep (rhs1))
> + plus_base = rhs2;
> + /* If plus_base is not set we dont have a +1 pattern. */
> + if (!plus_base)
> + return false;
> +
> + /* Check if plus_base is pred_phi itself. We may still need to strip
> + some casts to reacy the actual base. TODO: Change depth logic -
> + traverse untill plus_base go out of current basic block. */
> + for (unsigned depth_1 = 0; depth_1 < 8; depth_1++)
> + {
> + if (!pred_phi_strip_conversion (&plus_base))
> + break;
> + }
> + /* The stripped base should be pred_phi. */
> + if (plus_base == pred_phi)
> + {
> + if (dump_enabled_p ())
> + dump_printf_loc (MSG_NOTE, vect_location,
> + "predicated_index_plus_one match: found\n");
> + return true;
> + }
> + /* We don't have +1 pattern. */
> + return false;
> + }
> + /* If we reach an non-PLUS_EXPR check if we can continue walk through
> + casts */
> + tree cur_ssa = gimple_assign_lhs (cur_asgn);
> + if (pred_phi_strip_conversion(&cur_ssa))
> + {
> + cur = SSA_NAME_DEF_STMT (cur_ssa);
> + continue;
> + }
> + /* We cannot reach a PLUS_EXPR. */
> + return false;
> + }
> +
> + return false;
> +}
> +
> +/* Function pred_phi_in_mask_store_addr_computation.
> +
> + Starting from the store pointer in MASK_STORE_CALL, this function tries to
> + check if the base + offset computation is done and offset finaly reaches
> + PRED_PHI.
> +
> + # n = PHI <n_1, ..>
> + ...
> + a_1 = (long unsigned int) n;
> + a_2 = a_1 * 4;
> + a_3 = base + a_2;
> + addr = (int *) a_3;
> + .MASK_STORE (addr, mask, ..)
> + n_1 = n + 1;
> + ...
> + n_2 = mask ? n_1 : n; */
> +static bool
> +pred_phi_in_mask_store_addr_computation (class loop *loop,
> + gcall *mask_store_call, tree pred_phi)
> +{
> + /* Get the pointer operand of MASK_SLP_CALL. 'addr' in the example. */
> + tree ptr = gimple_call_arg (mask_store_call, 0);
> + if (TREE_CODE (ptr) != SSA_NAME)
> + return false;
> +
> + /* Strip off the cast and see if we can reach to reach a PLUS_EXPR.
> + a_3 = base + a_2;
> + addr = (int *) a_3; */
> + gassign *add_stmt = NULL;
> + tree cur = ptr;
> + /* TODO: traverse until CUR go out of the current bb. */
> + /* TODO: Can we club it with pred_phi_strip_conversion */
> + for (unsigned depth = 0; depth < 8; depth++)
> + {
> + gimple *def = SSA_NAME_DEF_STMT (cur);
> + gassign *a = dyn_cast <gassign *> (def);
> + if (!a)
> + return false;
> +
> + tree_code code = gimple_assign_rhs_code (a);
> + if (code == PLUS_EXPR || code == POINTER_PLUS_EXPR)
> + {
> + add_stmt = a;
> + break;
> + }
> + if (code != NOP_EXPR && code != CONVERT_EXPR
> + && code != VIEW_CONVERT_EXPR && code != NON_LVALUE_EXPR)
> + return false;
> +
> + tree rhs1 = gimple_assign_rhs1 (a);
> + if (TREE_CODE (rhs1) != SSA_NAME)
> + return false;
> + cur = rhs1;
> + }
> +
> + if (!add_stmt)
> + return false;
> +
> + /* Check if add_stmt is base + offset format. Note that it can be either
> + base + offset or offset + base. We take the loop invariant pointer as
> + the base. Example - a_3 = base + a_2; */
> + tree add_op0 = gimple_assign_rhs1 (add_stmt);
> + tree add_op1 = gimple_assign_rhs2 (add_stmt);
> + tree offset;
> + if (expr_invariant_in_loop_p (loop, add_op1))
> + offset = add_op0;
> + else if (expr_invariant_in_loop_p (loop, add_op0))
> + offset = add_op1;
> + else
> + return false;
> +
> + /* Now offset can be a multiplication with the element size as in
> + a_2 = a_1 * 4; */
> + if (TREE_CODE (offset) != SSA_NAME)
> + return false;
> + gimple *def = SSA_NAME_DEF_STMT (offset);
> + gassign *mul = dyn_cast <gassign *> (def);
> + if (!mul || gimple_assign_rhs_code (mul) != MULT_EXPR)
> + return false;
> + tree mul_op0 = gimple_assign_rhs1 (mul);
> + tree mul_op1 = gimple_assign_rhs2 (mul);
> + tree index_op = mul_op0;
> + tree size_op = mul_op1;
> + if (TREE_CODE (mul_op0) == INTEGER_CST)
> + {
> + index_op = mul_op1;
> + size_op = mul_op0;
> + }
> + else if (TREE_CODE (mul_op1) != INTEGER_CST)
> + return false;
> + /* The index_op should be an SSA_NAM. */
> + if (TREE_CODE (index_op) != SSA_NAME)
> + return false;
> + /* The size_op should be the element size pointed by ptr. */
> + tree eltype = TREE_TYPE (TREE_TYPE (ptr));
> + tree elem_size_unit = TYPE_SIZE_UNIT (eltype);
> + if (!elem_size_unit || TREE_CODE (elem_size_unit) != INTEGER_CST)
> + return false;
> + if (!operand_equal_p (size_op, elem_size_unit, OEP_ONLY_CONST))
> + return false;
> +
> + /* No the index_op can be a casted one. So strip off that cast as in
> + a_1 = (long unsigned int) n; */
> + pred_phi_strip_conversion (&index_op);
> +
> + /* Finally index_op should be PRED_PHI. */
> + return index_op == pred_phi;
> +}
> +
> +/* Function vect_find_predicated_index_mask_store.
> +
> + This function establishes there relationship between the PRED_PHI detected
> + so far with an IFN_MASK_STORE. The store address for IFN_MASK_STORE should
> + be computed using PRED_PHI as index and PRED_PHI_MASK should be used as
> its
> + mask. Note that the address computation is done as (base + n * size) and
> + can pass through multiiple cast instructions as in the below example.
> + So we need to traverse through them.
> +
> +
> + # n = PHI <n_1, ..>
> + ...
> + a_1 = (long unsigned int) n;
> + a_2 = a_1 * 4;
> + a_3 = base + a_2;
> + addr = (int *) a_3;
> + .MASK_STORE (addr, mask, ..)
> + n_1 = n + 1;
> + ...
> + n_2 = mask ? n_1 : n; */
> +static gcall *
> +vect_find_predicated_index_mask_store (class loop *loop, gphi *phi,
> + tree pred_phi, tree pred_phi_mask)
> +{
> + /* Search only the basic block that holds PHI; TODO: to scan other blocks
> + when generalizing. */
> + basic_block pred_bb = gimple_bb (phi);
> + /* Iterate over GIMPLE statements to find a matching IFN_MASK_STORE. */
> + for (gimple_stmt_iterator gsi = gsi_after_labels (pred_bb); !gsi_end_p
> (gsi);
> + gsi_next (&gsi))
> + {
> + gimple *st = gsi_stmt (gsi);
> +
> + /* If not an IFN_MASK_STORE call continue. */
> + gcall *mask_store_call = dyn_cast <gcall *> (st);
> + if (!mask_store_call || !gimple_call_internal_p (mask_store_call)
> + || gimple_call_internal_fn (mask_store_call) != IFN_MASK_STORE)
> + continue;
> +
> + /* Check if the mask of call is same as PRED_PHI_MASK. */
> + if (gimple_call_arg (mask_store_call, 2) != pred_phi_mask)
> + continue;
> +
> + /* Check if PRED_PHI is used to compute mask_store_call address. */
> + /* TODO: Can use any match operator (as in instcombine)?? */
> + if (!pred_phi_in_mask_store_addr_computation(loop, mask_store_call,
> + pred_phi))
> + continue;
> +
> + if (dump_enabled_p ())
> + dump_printf_loc (MSG_NOTE, vect_location,
> + "vect_find_predicated_index_mask_store: matched "
> + "IFN_MASK_STORE %G\n",
> + (gimple *) mask_store_call);
> +
> + return mask_store_call;
> + }
> +
> + if (TREE_CODE (pred_phi_mask) != SSA_NAME)
> + return NULL;
> + return NULL;
> +}
> +
> +/* Part of vpcompress pattern detection. Checks if PHI matches the
> predicated-
> + index phi shape, which expects its incoming value from the backedge to be
> + incremented under the same mask used by an IFN_MASK_STORE, whose address
> is
> + indexed by the PHI itself. An outline of the pattern is given below.
> +
> + # n = PHI <n_1, ..>
> + ...
> + addr = base + n;
> + .MASK_STORE (addr, mask, ..)
> + n_1 = n + 1;
> + ...
> + n_2 = mask ? n_1 : n; */
> +bool
> +vect_vpcompress_analyze_predicated_index_phi (loop_vec_info loop_vinfo,
> + class loop *loop, gphi *phi)
> +{
> + /* Require a loop-header PHI. */
> + if (gimple_bb (phi) != loop->header)
> + return false;
> +
> + tree pred_phi = PHI_RESULT (phi);
> + /* PHI should be an integer. */
> + if (TREE_CODE (pred_phi) != SSA_NAME
> + || TREE_CODE (TREE_TYPE (pred_phi)) != INTEGER_TYPE)
> + return false;
> +
> + /* Restricting to zero initializatin from preheader for now. TODO: Can be
> + generalized to arbitrary integer starts. */
> + tree pre_init = PHI_ARG_DEF_FROM_EDGE (phi, loop_preheader_edge (loop));
> + if (TREE_CODE (pre_init) != INTEGER_CST || !integer_zerop (pre_init))
> + return false;
> +
> + /* The incoming value for pred-phi must be in the if-converted form.
> + n_2 = mask ? n_1 : n; TODO: This may be generalized later for more
> + patterns. */
> + /* Get the incoming value (n_2) of pred-phi through backedge. */
> + tree pred_phi_back_edge_val =
> + PHI_ARG_DEF_FROM_EDGE (phi, loop_latch_edge (loop));
> + if (TREE_CODE (pred_phi_back_edge_val) != SSA_NAME)
> + return false;
> + /* Check if pred_phi_back_edge_val is a COND_EXPR. */
> + gimple *pred_phi_back_edge_stmt
> + = SSA_NAME_DEF_STMT (pred_phi_back_edge_val);
> + if (!pred_phi_back_edge_stmt)
> + return false;
> + gassign *pred_phi_back_edge_asgn_stmt
> + = dyn_cast <gassign *> (pred_phi_back_edge_stmt);
> + if (!pred_phi_back_edge_asgn_stmt
> + || gimple_assign_rhs_code (pred_phi_back_edge_asgn_stmt) != COND_EXPR)
> + return false;
> +
> + /* Check that one of the arms of COND_EXPR is pred-phi (n) and the other is
> + an increment value (n_1 = n + 1). */
> + if (!predicated_index_plus_one (pred_phi, pred_phi_back_edge_asgn_stmt))
> + return false;
> +
> + /* Check if there is a IFN_MASK_STORE, which is connected to the same
> + * phi that we detected (pred-phi) so far. The store address should be
> + * computed based on pred-phi and it should use the same mask as that of
> + * COND_EXPR. */
> + tree mask = gimple_assign_rhs1 (pred_phi_back_edge_asgn_stmt);
> + gcall *mask_store =
> + vect_find_predicated_index_mask_store (loop, phi, pred_phi, mask);
> +
> + if (dump_enabled_p ())
> + dump_printf_loc (MSG_NOTE, vect_location,
> + "predicated-index PHI: matched predicated phi %G\n",
> + (gimple *) phi);
> +
> + if (!mask_store)
> + return false;
> +
> + /* Keep at most one predicated-index PHI per loop_vinfo. */
> + if (loop_vinfo->predicated_index_phi != NULL
> + && loop_vinfo->predicated_index_phi != phi)
> + return false;
> +
> + loop_vinfo->predicated_index_phi = phi;
> + loop_vinfo->predicated_index_mask_store = mask_store;
> +
> + /* All requirements satisfiled. */
> + return true;
> +}
> diff --git a/gcc/tree-vect-vpcompress.h b/gcc/tree-vect-vpcompress.h
> new file mode 100644
> index 00000000000..b8d0093831f
> --- /dev/null
> +++ b/gcc/tree-vect-vpcompress.h
> @@ -0,0 +1,27 @@
> +/* Pattern detection for vpcompress.
> + Copyright (C) 2016-2026 Free Software Foundation, Inc.
> +
> +
> +This file is part of GCC.
> +
> +GCC is free software; you can redistribute it and/or modify it under
> +the terms of the GNU General Public License as published by the Free
> +Software Foundation; either version 3, or (at your option) any later
> +version.
> +
> +GCC is distributed in the hope that it will be useful, but WITHOUT ANY
> +WARRANTY; without even the implied warranty of MERCHANTABILITY or
> +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
> +for more details.
> +
> +You should have received a copy of the GNU General Public License
> +along with GCC; see the file COPYING3. If not see
> +<http://www.gnu.org/licenses/>. */
> +
> +#ifndef GCC_TREE_VECT_VPCOMPRESS_H
> +#define GCC_TREE_VECT_VPCOMPRESS_H
> +
> +extern bool vect_vpcompress_analyze_predicated_index_phi (loop_vec_info,
> + class loop *, gphi *);
> +
> +#endif /* GCC_TREE_VECT_VPCOMPRESS_H */
>
--
Richard Biener <[email protected]>
SUSE Software Solutions Germany GmbH,
Frankenstrasse 146, 90461 Nuernberg, Germany;
GF: Jochen Jaser, Andrew McDonald; (HRB 36809, AG Nuernberg)
