std::max_element and std::min_element carry the running min/max's index as
an iterator, so the index reduction PHI is pointer-typed, which the
minmax+index epilogue cannot reduce: it masks losing lanes with
TYPE_MAX_VALUE, defined for numerical types only. The index and value
vectors can also differ in lane count, since a pointer is wider than an int
or float value on a 64-bit target.
Reduce the index over an unsigned surrogate of the value's precision
instead. vect_get_vector_types_for_stmt types the index PHI and its update
with the surrogate, so the vectorization factor is not capped by the
pointer's wider elements, and vectorizable_condition synthesizes the
winning arm as an index vector induction. The epilogue rebuilds the result
as base + idx * stride, where the base is the index PHI's initial value and
the stride the byte step of the affine pointer IV the update selects.
The two can start a whole number of elements apart, one for the
pre-incremented while (++first != last) shape, and that distance becomes
the surrogate's start offset.
The surrogate counts iterations in its own type, and the last index it
produces, index_start + niters - 1, has to fit there. The bound is decided
at loop level, before pattern recognition, since recording an assumption
clears what scev and niter concluded without it. Where the trip count type
can exceed the bound the loop is versioned on it; for a surrogate as wide
as that type nothing is emitted.
The pointer shape is refused where the epilogue could not rebuild the
iterator from base and stride, and where the surrogate could not count
the whole iteration space.
Bootstrapped/regtested on AArch64 and x86-64.
PR tree-optimization/88259
gcc/ChangeLog:
* tree-vect-loop.cc (vect_analyze_minmax_index_niters): New
function.
(vect_analyze_loop_2): Call it.
(vect_capture_pointer_iter_base_stride): New function.
(vect_minmax_surrogate_vectype): New function.
(vect_minmax_lanes_ok_p): Gate a pointer index on the surrogate
rather than the pointer vectype.
(vect_model_reduction_cost): Account for materializing the result
pointer.
(vect_create_epilog_for_reduction): Materialize it.
(vectorizable_reduction): Capture base, stride and start offset for
a pointer index reduction and add the niters versioning assumption.
Refuse the reduction when that capture fails, the loop is an
epilogue, or the loop is peeled for alignment.
(vect_transform_cycle_phi): Use a zero initial index for it.
* tree-vect-stmts.cc (vect_coupled_ptr_argmax_index_vectype): New
function, exported.
(vectorizable_condition): Vectorize the index update over the
surrogate, synthesizing the winning arm as an index vector
induction.
(vect_get_vector_types_for_stmt): Use it to type the pointer index
stmts with the surrogate.
* tree-vectorizer.h (VECT_REDUC_INFO_COUPLED_MINMAX_BASE): New
macro.
(VECT_REDUC_INFO_COUPLED_MINMAX_STRIDE): New macro.
(VECT_REDUC_INFO_COUPLED_IS_POINTER): New macro.
(VECT_REDUC_INFO_COUPLED_MINMAX_INDEX_START): New macro.
(vect_minmax_surrogate_vectype): Declare.
(vect_coupled_ptr_argmax_index_vectype): Declare.
gcc/testsuite/ChangeLog:
* g++.dg/vect/vect-minmax-index.cc: New test.
* gcc.dg/vect/vect-minmax-index-7.c: New test.
* gcc.dg/vect/vect-minmax-index-8.c: New test.
Co-authored-by: Philipp Tomsich <[email protected]>
Signed-off-by: Konstantinos Eleftheriou <[email protected]>
---
.../g++.dg/vect/vect-minmax-index.cc | 68 ++++
.../gcc.dg/vect/vect-minmax-index-7.c | 191 +++++++++
.../gcc.dg/vect/vect-minmax-index-8.c | 84 ++++
gcc/tree-vect-loop.cc | 377 +++++++++++++++++-
gcc/tree-vect-stmts.cc | 131 +++++-
gcc/tree-vectorizer.h | 21 +
6 files changed, 859 insertions(+), 13 deletions(-)
create mode 100644 gcc/testsuite/g++.dg/vect/vect-minmax-index.cc
create mode 100644 gcc/testsuite/gcc.dg/vect/vect-minmax-index-7.c
create mode 100644 gcc/testsuite/gcc.dg/vect/vect-minmax-index-8.c
diff --git a/gcc/testsuite/g++.dg/vect/vect-minmax-index.cc
b/gcc/testsuite/g++.dg/vect/vect-minmax-index.cc
new file mode 100644
index 000000000000..250ca7e455f9
--- /dev/null
+++ b/gcc/testsuite/g++.dg/vect/vect-minmax-index.cc
@@ -0,0 +1,68 @@
+/* { dg-do run } */
+/* { dg-require-effective-target vect_int } */
+/* { dg-additional-options "-O3 -fvect-cost-model=dynamic" } */
+
+/* The library shape the pointer-iterator support exists for: the index of the
+ reduction is the iterator itself, so it is reduced over an unsigned
+ surrogate and rebuilt as base + idx * stride. */
+
+#include <algorithm>
+#include "../../gcc.dg/vect/tree-vect.h"
+
+__attribute__((noipa)) long
+argmax (const int *v, int n)
+{
+ return std::max_element (v, v + n) - v;
+}
+
+__attribute__((noipa)) long
+argmin (const int *v, int n)
+{
+ return std::min_element (v, v + n) - v;
+}
+
+int g[1024];
+int v[1024];
+
+/* A file-scope array, where base and iterator are address constants. The
+ iterator starts one element in, so a peeled loop would be refused. The
+ explicit cost model above keeps the peel from being forced. */
+__attribute__((noipa)) long
+argmax_static ()
+{
+ return std::max_element (g, g + 1024) - g;
+}
+
+int
+main ()
+{
+ check_vect ();
+
+#pragma GCC novector
+ for (int i = 0; i < 1024; i++)
+ {
+ v[i] = (i * 37 + 13) % 101;
+ g[i] = v[i];
+ }
+ /* Odd positions: an even one agrees with the scalar loop whatever the
+ epilogue does. */
+ v[603] = 500;
+ g[701] = 500;
+ v[97] = -7;
+
+ if (argmax (v, 1024) != 603)
+ __builtin_abort ();
+ if (argmin (v, 1024) != 97)
+ __builtin_abort ();
+ if (argmax_static () != 701)
+ __builtin_abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump "detected minmax\\+index compound reduction"
"vect" } } */
+/* One per kernel. A coupled pointer index refuses epilogue
+ vectorization, so no epilogue can add to the count. The gate is
+ vect_max_reduc: the epilogue needs a min/max reduction on the value vector
+ and a min reduction on the index vector. */
+/* { dg-final { scan-tree-dump-times "LOOP VECTORIZED" 3 "vect" { target
vect_max_reduc } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/vect-minmax-index-7.c
b/gcc/testsuite/gcc.dg/vect/vect-minmax-index-7.c
new file mode 100644
index 000000000000..e74b3650b8c2
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-minmax-index-7.c
@@ -0,0 +1,191 @@
+/* { dg-do run } */
+/* { dg-additional-options "-O3 -ffast-math -fdump-tree-optimized" } */
+
+/* The index of a min/max+index reduction may be an iterator rather than an
+ integer, the shape std::max_element and std::min_element produce. */
+
+#include "tree-vect.h"
+
+#define N 128
+
+/* Pre-incremented iterator: the first compared position is vec + 1, so the
+ surrogate index is offset by one from the base. Without -ffast-math the
+ float value never reaches a MAX_EXPR and the offset goes untested. */
+long __attribute__((noipa))
+max_elem (const float *vec, unsigned long n)
+{
+ const float *result = vec, *first = vec, *last = vec + n;
+ while (++first != last)
+ if (*result < *first)
+ result = first;
+ return result - vec;
+}
+
+/* Both iterators start together, so the surrogate index starts at zero. */
+long __attribute__((noipa))
+min_elem (const int *vec, unsigned long n)
+{
+ const int *result = vec, *first = vec, *last = vec + n;
+ for (; first != last; ++first)
+ if (*first < *result)
+ result = first;
+ return result - vec;
+}
+
+/* A backward scan gives the iterator a negative stride: the epilogue has no
+ base and positive step to rebuild it from, and the coupling is refused. */
+long __attribute__((noipa))
+last_max (const int *vec, unsigned long n)
+{
+ const int *result = vec, *p = vec + n;
+ while (p != vec)
+ {
+ --p;
+ if (*p > *result)
+ result = p;
+ }
+ return result - vec;
+}
+
+/* An iterator starting before the result pointer gives a negative start
+ offset, which surrogate index 0 cannot map onto; the coupling is refused.
+ This kernel and the one above run scalar and still have to be right, this
+ one including a negative answer. */
+long __attribute__((noipa))
+before_base (const int *base, unsigned long n)
+{
+ const int *result = base, *first = result - 3, *last = first + n;
+ int max = *result;
+ for (; first != last; ++first)
+ if (*first > max)
+ {
+ max = *first;
+ result = first;
+ }
+ return result - base;
+}
+
+/* A 64-bit value halves the lane count; its surrogate index is 64-bit too.
+ Vectorizing it needs a 64-bit integer reduction, which Advanced SIMD lacks;
+ there this kernel checks the scalar path. */
+long __attribute__((noipa))
+min_elem_long (const long *vec, unsigned long n)
+{
+ const long *result = vec, *first = vec, *last = vec + n;
+ for (; first != last; ++first)
+ if (*first < *result)
+ result = first;
+ return result - vec;
+}
+
+float a[N];
+int b[N];
+long c[N];
+
+/* A file-scope base is an address constant, so it is related to the iterator
+ by their common address rather than by a chain of definitions. */
+long __attribute__((noipa))
+max_static (unsigned long n)
+{
+ const int *result = b, *first = b, *last = b + n;
+ for (; first != last; ++first)
+ if (*first > *result)
+ result = first;
+ return result - b;
+}
+
+int
+main (void)
+{
+ check_vect ();
+
+#pragma GCC novector
+ for (int i = 0; i < N; i++)
+ {
+ a[i] = i % 7;
+ b[i] = 100 - (i % 11);
+ }
+
+ /* A unique maximum, away from both ends. */
+ a[70] = 1000.0f;
+ if (max_elem (a, N) != 70)
+ abort ();
+
+ /* Ties keep the first occurrence, matching the strict comparison. */
+ a[70] = 6;
+ if (max_elem (a, N) != 6)
+ abort ();
+
+ b[90] = -5;
+ if (min_elem (b, N) != 90)
+ abort ();
+
+ /* The maximum at the very first position is never compared by the
+ pre-incremented form, so it must come from the initial value. */
+#pragma GCC novector
+ for (int i = 0; i < N; i++)
+ a[i] = -i;
+ if (max_elem (a, N) != 0)
+ abort ();
+
+ /* The refused backward scan still has to be correct. */
+#pragma GCC novector
+ for (int i = 0; i < N; i++)
+ b[i] = i % 13;
+ b[42] = 500;
+ if (last_max (b, N) != 42)
+ abort ();
+
+ /* The refused negative start offset, with the winner after the base. */
+#pragma GCC novector
+ for (int i = 0; i < N; i++)
+ b[i] = i % 17;
+ b[7] = 500;
+ if (before_base (b + 3, 32) != 4)
+ abort ();
+
+ /* And with the winner before it, so the answer is negative. */
+#pragma GCC novector
+ for (int i = 0; i < N; i++)
+ b[i] = i % 17;
+ b[1] = 700;
+ if (before_base (b + 3, 32) != -2)
+ abort ();
+
+ /* The file-scope base. */
+#pragma GCC novector
+ for (int i = 0; i < N; i++)
+ b[i] = i % 19;
+ b[85] = 400;
+ if (max_static (N) != 85)
+ abort ();
+
+ /* Ties keep the first occurrence here too: 18 is reached at 18 first. */
+ b[85] = 18;
+ if (max_static (N) != 18)
+ abort ();
+
+ /* The 64-bit value form. */
+#pragma GCC novector
+ for (int i = 0; i < N; i++)
+ c[i] = 1000 - (i % 23);
+ c[77] = -5;
+ if (min_elem_long (c, N) != 77)
+ abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump "detected minmax\\+index compound reduction"
"vect" } } */
+/* The setup loops carry novector; only a kernel can match this. */
+/* { dg-final { scan-tree-dump "LOOP VECTORIZED" "vect" { target { vect_int &&
vect_max_reduc } } } } */
+/* { dg-final { scan-tree-dump "minmax\\+index: cannot rebuild the pointer
index" "vect" } } */
+/* The loop is versioned on its trip count against the unsigned int
+ surrogate's range: 2^32 where the iterator starts at the base, one less
+ for the pre-incremented shape. That count is a sizetype difference, and a
+ 32-bit sizetype cannot overflow the surrogate, hence the lp64 gate. */
+/* { dg-final { scan-tree-dump "<= 4294967296" "optimized" { target { {
vect_int && vect_max_reduc } && lp64 } } } } */
+/* { dg-final { scan-tree-dump "<= 4294967295" "optimized" { target { {
vect_int && vect_max_reduc } && lp64 } } } } */
+
+/* No shape here may be refused for a lane-count mismatch. */
+/* { dg-final { scan-tree-dump-not "no value/index vector types with equal
lane counts" "vect" } } */
diff --git a/gcc/testsuite/gcc.dg/vect/vect-minmax-index-8.c
b/gcc/testsuite/gcc.dg/vect/vect-minmax-index-8.c
new file mode 100644
index 000000000000..b2581e8fd143
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-minmax-index-8.c
@@ -0,0 +1,84 @@
+/* { dg-do run } */
+/* { dg-additional-options "-O3 -ffast-math" } */
+/* { dg-additional-options "-mstrict-align" { target aarch64*-*-* } } */
+
+/* A prologue peel moves the vector loop's first iteration past the base that
+ surrogate index 0 denotes, so a coupled pointer index is refused for a loop
+ peeled for alignment. An integer index rides a real induction variable and
+ is unaffected. -mstrict-align forces the peel on aarch64. */
+
+#include "tree-vect.h"
+
+#define N 1024
+
+/* Pre-incremented iterator: the surrogate starts at one. */
+long __attribute__((noipa))
+max_f (const float *vec, unsigned long n)
+{
+ const float *r = vec, *f = vec, *l = vec + n;
+ while (++f != l)
+ if (*r < *f)
+ r = f;
+ return r - vec;
+}
+
+/* Both iterators start together: the surrogate starts at zero. */
+long __attribute__((noipa))
+min_i (const int *vec, unsigned long n)
+{
+ const int *r = vec, *f = vec, *l = vec + n;
+ for (; f != l; ++f)
+ if (*f < *r)
+ r = f;
+ return r - vec;
+}
+
+int __attribute__((noipa))
+argmax_i (const int *a, int n)
+{
+ int m = a[0], k = 0;
+ for (int i = 1; i < n; i++)
+ if (a[i] > m)
+ {
+ m = a[i];
+ k = i;
+ }
+ return k;
+}
+
+float fa[N];
+int ia[N], ja[N];
+
+int
+main (void)
+{
+ check_vect ();
+
+ /* Sweep the base misalignment, so the peel count varies with it. */
+ for (int off = 0; off < 8; off++)
+ for (int n = 8; n <= 200; n += 7)
+ {
+ int spike = n / 2;
+
+ for (int i = 0; i < N; i++)
+ {
+ fa[i] = (float) (i % 37);
+ ia[i] = 100 - i % 41;
+ ja[i] = i % 29;
+ }
+ fa[off + spike] = 1000.0f;
+ ia[off + spike] = -1000;
+ ja[off + spike] = 500;
+
+ if (max_f (fa + off, n) != spike)
+ abort ();
+ if (min_i (ia + off, n) != spike)
+ abort ();
+ if (argmax_i (ja + off, n) != spike)
+ abort ();
+ }
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump "minmax\\+index: pointer index not supported
with peeling for alignment" "vect" { target aarch64*-*-* } } } */
diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc
index faee78623fd4..1e0d0d24619c 100644
--- a/gcc/tree-vect-loop.cc
+++ b/gcc/tree-vect-loop.cc
@@ -163,6 +163,7 @@ static void vect_estimate_min_profitable_iters
(loop_vec_info, int *, int *,
unsigned *);
static stmt_vec_info vect_is_simple_reduction (loop_vec_info, stmt_vec_info,
gphi **);
+static opt_result vect_analyze_minmax_index_niters (loop_vec_info);
/* Function vect_is_simple_iv_evolution.
@@ -2253,6 +2254,12 @@ vect_analyze_loop_2 (loop_vec_info loop_vinfo, int
masked_p, bool &fatal,
Cross-iteration cycles caused by virtual phis are analyzed separately. */
vect_analyze_scalar_cycles (loop_vinfo);
+ /* A coupled pointer index needs its trip count bounded, which is decided
+ from the cycles just classified. */
+ ok = vect_analyze_minmax_index_niters (loop_vinfo);
+ if (!ok)
+ return ok;
+
vect_pattern_recog (loop_vinfo);
/* Analyze the access patterns of the data-refs in the loop (consecutive,
@@ -3713,20 +3720,261 @@ check_reduction_path (dump_user_location_t loc, loop_p
loop, gphi *phi,
}
-/* Return true when the coupled index (scalar type IDX_SCALAR) and the min/max
- value (scalar type VAL_SCALAR) map to vector types with the same lane count,
+/* For a MINMAX_INDEX_REDUCTION whose index reduction PHI is pointer-typed
+ (the std::max_element/std::min_element iterator shape), capture what the
+ epilogue needs to form RESULT = BASE + IDX * STRIDE. PHI_INFO's latch arm
+ must select an affine pointer IV with a positive constant byte stride. On
+ success *OUT_BASE is the PHI's preheader value, *OUT_STRIDE the byte
+ advance per iteration and *OUT_INDEX_START the IV's non-negative start
+ offset in elements. */
+
+static bool
+vect_capture_pointer_iter_base_stride (loop_vec_info loop_vinfo,
+ stmt_vec_info phi_info,
+ tree *out_base, tree *out_stride,
+ int *out_index_start)
+{
+ gphi *phi = dyn_cast <gphi *> (phi_info->stmt);
+ if (!phi)
+ return false;
+
+ tree result_ptr = PHI_RESULT (phi);
+ tree ptr_type = TREE_TYPE (result_ptr);
+ if (!POINTER_TYPE_P (ptr_type))
+ return false;
+
+ /* The epilogue scales the surrogate index by the element size, so the
+ pointee needs a constant nonzero one. void * has none, its arithmetic
+ being a GCC extension, and neither has an incomplete or variable-size
+ type. */
+ tree pointee_type = TREE_TYPE (ptr_type);
+ if (VOID_TYPE_P (pointee_type)
+ || !COMPLETE_TYPE_P (pointee_type)
+ || !TYPE_SIZE_UNIT (pointee_type)
+ || TREE_CODE (TYPE_SIZE_UNIT (pointee_type)) != INTEGER_CST
+ || integer_zerop (TYPE_SIZE_UNIT (pointee_type)))
+ return false;
+
+ class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
+ if (gimple_bb (phi) != loop->header)
+ return false;
+
+ edge preheader = loop_preheader_edge (loop);
+ edge latch = loop_latch_edge (loop);
+ tree result_init = PHI_ARG_DEF_FROM_EDGE (phi, preheader);
+ tree result_next = PHI_ARG_DEF_FROM_EDGE (phi, latch);
+ if (TREE_CODE (result_next) != SSA_NAME)
+ return false;
+
+ /* The latch-side arg must be the result of a COND_EXPR taking an IV in the
+ winning arm and keeping the PHI in the losing one. */
+ gimple *cond_def = SSA_NAME_DEF_STMT (result_next);
+ if (!is_gimple_assign (cond_def)
+ || gimple_assign_rhs_code (cond_def) != COND_EXPR)
+ return false;
+
+ tree winning_arm = gimple_assign_rhs2 (cond_def);
+ tree losing_arm = gimple_assign_rhs3 (cond_def);
+ if (losing_arm != result_ptr || TREE_CODE (winning_arm) != SSA_NAME)
+ return false;
+
+ /* Walk SCEV on the winning arm; require an affine chrec in this loop
+ with a positive constant byte stride. */
+ tree chrec = analyze_scalar_evolution (loop, winning_arm);
+ if (chrec_contains_undetermined (chrec))
+ return false;
+ chrec = instantiate_scev (preheader, loop, chrec);
+ if (TREE_CODE (chrec) != POLYNOMIAL_CHREC
+ || CHREC_VARIABLE (chrec) != (unsigned) loop->num)
+ return false;
+
+ tree iv_base = CHREC_LEFT (chrec);
+ tree iv_step = CHREC_RIGHT (chrec);
+ if (TREE_CODE (iv_step) != INTEGER_CST
+ || tree_int_cst_sign_bit (iv_step)
+ || integer_zerop (iv_step))
+ return false;
+
+ /* IV_BASE is RESULT_INIT + K elements, K being 1 for the pre-incremented
+ "while (++first != last)" shape and 0 when both start together. Walk
+ the POINTER_PLUS chain to find it, since ptr_difference_const cannot
+ relate two distinct SSA names. A POINTER_PLUS offset has unsigned
+ sizetype but denotes a signed displacement, so read it as one: read
+ unsigned, "result - 3" is a huge positive offset rather than a
+ negative K. */
+ offset_int off = 0;
+ offset_int step = offset_int::from (wi::to_wide (iv_step), SIGNED);
+ tree curr_base = iv_base;
+ while (curr_base != result_init && TREE_CODE (curr_base) == SSA_NAME)
+ {
+ gimple *def = SSA_NAME_DEF_STMT (curr_base);
+ if (!is_gimple_assign (def)
+ || gimple_assign_rhs_code (def) != POINTER_PLUS_EXPR
+ || TREE_CODE (gimple_assign_rhs2 (def)) != INTEGER_CST)
+ break;
+ off += offset_int::from (wi::to_wide (gimple_assign_rhs2 (def)),
+ SIGNED);
+ curr_base = gimple_assign_rhs1 (def);
+ }
+
+ /* A file-scope array base leaves the walk at an address constant rather
+ than at RESULT_INIT; ptr_difference_const relates the two, the SSA walk
+ cannot, as there is no def chain between them. */
+ tree base_core = curr_base;
+ tree init_core = result_init;
+ STRIP_NOPS (base_core);
+ STRIP_NOPS (init_core);
+ if (!operand_equal_p (base_core, init_core, 0))
+ {
+ poly_int64 diff;
+ HOST_WIDE_INT const_diff;
+ if (!ptr_difference_const (base_core, init_core, &diff)
+ || !diff.is_constant (&const_diff))
+ return false;
+ off += const_diff;
+ }
+
+ /* iv_base = result_init + k * step. A negative K would put the iterator
+ before BASE, breaking the mapping of surrogate index 0 to it. The
+ upper bound is what *OUT_INDEX_START can hold. */
+ offset_int k;
+ if (!wi::multiple_of_p (off, step, SIGNED, &k)
+ || wi::neg_p (k)
+ || wi::gts_p (k, INT_MAX))
+ return false;
+
+ *out_base = result_init;
+ *out_stride = fold_convert (sizetype, iv_step);
+ *out_index_start = (int) k.to_shwi ();
+ return true;
+}
+
+
+/* Bound the trip count of a loop whose coupled index is a pointer. The
+ surrogate it reduces over counts iterations in its own type, so the last
+ index it produces, INDEX_START + NITERS - 1, has to fit there. Reaching
+ the sentinel the epilogue masks losing lanes with is not a problem: the
+ losing lanes hold that same value, and the reduction still answers with the
+ winner.
+
+ Version the loop when the trip count type can exceed the bound, and refuse
+ it when there is no trip count to bound the surrogate with. Installing an
+ assumption changes what scev and niter conclude. This therefore runs before
+ the analyses that consult them and before the single-lane retry point, as
+ the loop-form path does for the assumptions it records. */
+
+static opt_result
+vect_analyze_minmax_index_niters (loop_vec_info loop_vinfo)
+{
+ /* Nothing carries the surrogate into an epilogue: the coupling is refused
+ for one and needs no bound. */
+ if (LOOP_VINFO_EPILOGUE_P (loop_vinfo))
+ return opt_result::success ();
+
+ class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
+ for (gphi_iterator gsi = gsi_start_phis (loop->header);
+ !gsi_end_p (gsi); gsi_next (&gsi))
+ {
+ gphi *phi = gsi.phi ();
+ if (virtual_operand_p (PHI_RESULT (phi)))
+ continue;
+ stmt_vec_info phi_info = loop_vinfo->lookup_stmt (phi);
+ if (!phi_info)
+ continue;
+ tree surrogate_vt
+ = vect_coupled_ptr_argmax_index_vectype (loop_vinfo, phi_info);
+ if (!surrogate_vt)
+ continue;
+
+ tree base, stride;
+ int index_start;
+ if (!vect_capture_pointer_iter_base_stride (loop_vinfo, phi_info, &base,
+ &stride, &index_start))
+ continue;
+
+ widest_int max_niters
+ = (wi::to_widest (TYPE_MAX_VALUE (TREE_TYPE (surrogate_vt))) + 1
+ - index_start);
+ tree niters = LOOP_VINFO_NITERS (loop_vinfo);
+ tree niters_type = niters ? TREE_TYPE (niters) : NULL_TREE;
+ /* An uncounted loop carries chrec_dont_know, typed void, and there is
+ nothing to bound the surrogate with. */
+ if (!niters_type || !INTEGRAL_TYPE_P (niters_type))
+ return opt_result::failure_at (phi,
+ "not vectorized: no trip count to"
+ " bound the pointer index"
+ " surrogate.\n");
+
+ /* The trip count type cannot exceed the bound. */
+ if (!wi::gtu_p (wi::to_widest (TYPE_MAX_VALUE (niters_type)),
+ max_niters))
+ continue;
+
+ /* A trip count that always fits needs no guard. Fold against NITERS:
+ LOOP_C_FINITE set by an earlier vector mode outlives the loop_vec_info
+ whose assumption justified it, and a bound derived under it could
+ wrongly say the count fits. */
+ tree cond = fold_build2 (LE_EXPR, boolean_type_node, niters,
+ wide_int_to_tree (niters_type, max_niters));
+ if (integer_onep (cond))
+ continue;
+
+ tree prev = LOOP_VINFO_NITERS_ASSUMPTIONS (loop_vinfo);
+ LOOP_VINFO_NITERS_ASSUMPTIONS (loop_vinfo)
+ = (prev && !integer_onep (prev)
+ ? fold_build2 (TRUTH_AND_EXPR, boolean_type_node, prev, cond)
+ : cond);
+ /* Clear what scev and niter concluded without the assumption and mark
+ the loop finite, which niters versioning requires, as
+ vect_analyze_loop_1 does for a loop-form assumption. */
+ scev_reset_htab ();
+ free_numbers_of_iterations_estimates (loop);
+ loop_constraint_set (loop, LOOP_C_FINITE);
+ }
+
+ return opt_result::success ();
+}
+
+/* Return the unsigned vector type a coupled pointer index is reduced over for
+ a min/max value of type VAL_SCALAR, or NULL_TREE when there is none. The
+ surrogate takes the value's precision, giving the two vectors the same
+ number of lanes. Precisions below 32 bits are refused, since the surrogate
+ would overflow within an ordinary trip count; on a target whose int is
+ 16 bits that rules out an int value. */
+
+tree
+vect_minmax_surrogate_vectype (vec_info *vinfo, tree val_scalar)
+{
+ unsigned int prec = TYPE_PRECISION (val_scalar);
+ if (prec < 32)
+ return NULL_TREE;
+ tree val_vt = get_vectype_for_scalar_type (vinfo, val_scalar);
+ if (!val_vt)
+ return NULL_TREE;
+ return get_same_sized_vectype (build_nonstandard_integer_type (prec, true),
+ val_vt);
+}
+
+/* Return true when the min/max value (scalar type VAL_SCALAR) and the coupled
+ index (scalar type IDX_SCALAR) map to vector types with the same lane count,
so the MINMAX_INDEX epilogue can apply the value-compare mask to the index
vector lane-by-lane. A double value (2 lanes/128b) cannot couple with an
- int32 index (4 lanes/128b). */
+ int32 index (4 lanes/128b). A pointer index (the std::max_element iterator)
+ is reduced via an unsigned surrogate sized to the value, so it is gated
+ on that rather than the raw pointer vectype. */
static bool
vect_minmax_lanes_ok_p (vec_info *vinfo, tree val_scalar, tree idx_scalar)
{
- /* The index is reduced as an integer. */
- if (!INTEGRAL_TYPE_P (idx_scalar))
+ /* The epilogue masks losing lanes with TYPE_MAX_VALUE and reduces with
+ REDUC_MIN, neither of which an index that is not an integer or an
+ iterator supports. */
+ if (!INTEGRAL_TYPE_P (idx_scalar) && !POINTER_TYPE_P (idx_scalar))
return false;
tree val_vt = get_vectype_for_scalar_type (vinfo, val_scalar);
- tree idx_vt = get_vectype_for_scalar_type (vinfo, idx_scalar);
+ tree idx_vt = (POINTER_TYPE_P (idx_scalar)
+ ? vect_minmax_surrogate_vectype (vinfo, val_scalar)
+ : get_vectype_for_scalar_type (vinfo, idx_scalar));
return (val_vt
&& idx_vt
&& known_eq (TYPE_VECTOR_SUBPARTS (val_vt),
@@ -5121,6 +5369,15 @@ vect_model_reduction_cost (loop_vec_info loop_vinfo,
vector_stmt, node, 0,
vect_epilogue);
+ /* A pointer index is rebuilt as base + idx * stride, which
+ adds a convert, a multiply and a pointer add. NODE is the
+ reduction statement here, so it carries a cycle id only once
+ the cycle is known. */
+ vect_reduc_info ri = info_for_reduction (loop_vinfo, node);
+ if (ri && VECT_REDUC_INFO_COUPLED_IS_POINTER (ri))
+ epilogue_cost += record_stmt_cost (cost_vec, 3,
+ scalar_stmt, node, 0,
+ vect_epilogue);
}
else
{
@@ -5861,6 +6118,11 @@ vect_create_epilog_for_reduction (loop_vec_info
loop_vinfo,
tree scalar_dest = gimple_get_lhs (vect_orig_stmt (stmt_info)->stmt);
tree scalar_type = TREE_TYPE (scalar_dest);
+ /* Pointer-iterator argmax: the index is reduced over an unsigned surrogate,
+ so the epilogue's index arithmetic uses the surrogate scalar type. */
+ if (VECT_REDUC_INFO_TYPE (reduc_info) == MINMAX_INDEX_REDUCTION
+ && VECT_REDUC_INFO_COUPLED_IS_POINTER (reduc_info))
+ scalar_type = TREE_TYPE (vectype);
scalar_results.truncate (0);
scalar_results.reserve_exact (group_size);
new_scalar_dest = vect_create_destination_var (scalar_dest, NULL);
@@ -6262,6 +6524,25 @@ vect_create_epilog_for_reduction (loop_vec_info
loop_vinfo,
new_temp = scalar_idx;
+ if (VECT_REDUC_INFO_COUPLED_IS_POINTER (reduc_info))
+ {
+ /* SCALAR_IDX counts elements from BASE, so the iterator is
+ BASE + IDX * STRIDE. The conversion to sizetype narrows a
+ 64-bit surrogate on a 32-bit target. An index into a 32-bit
+ address space cannot exceed what sizetype holds. */
+ tree base = VECT_REDUC_INFO_COUPLED_MINMAX_BASE (reduc_info);
+ tree stride = VECT_REDUC_INFO_COUPLED_MINMAX_STRIDE (reduc_info);
+ tree ptr_type = TREE_TYPE (base);
+ gimple_seq stmts = NULL;
+ tree offset
+ = gimple_build (&stmts, MULT_EXPR, sizetype,
+ gimple_convert (&stmts, sizetype, scalar_idx),
+ stride);
+ new_temp = gimple_build (&stmts, POINTER_PLUS_EXPR, ptr_type,
+ base, offset);
+ gsi_insert_seq_before (&exit_gsi, stmts, GSI_SAME_STMT);
+ }
+
scalar_results.safe_push (new_temp);
}
else if (reduc_fn != IFN_LAST && (!slp_reduc || group_size == 1))
@@ -7840,6 +8121,76 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
VECT_REDUC_INFO_COUPLED_MINMAX_PHI (reduc_info) = minmax_phi_info;
}
+ /* Pointer-typed index (std::max_element iterator): capture what the
+ epilogue needs to rebuild the iterator. vect_analyze_minmax_index_niters
+ has already bounded the trip count, so the surrogate cannot count past
+ its own type. */
+ gphi *idx_phi = NULL;
+ if (reduction_type == MINMAX_INDEX_REDUCTION
+ && !VECT_REDUC_INFO_COUPLED_IS_POINTER (reduc_info)
+ && phi_info)
+ idx_phi = dyn_cast <gphi *> (phi_info->stmt);
+ if (idx_phi && POINTER_TYPE_P (TREE_TYPE (gimple_phi_result (idx_phi))))
+ {
+ /* Surrogate index 0 denotes BASE, and a prologue peel moves the vector
+ loop's first iteration past BASE, so the rebuilt iterator would come
+ out short by the peel count. Peeling is chosen in
+ vect_enhance_data_refs_alignment, before this analysis runs. */
+ if (LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo) != 0)
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "minmax+index: pointer index not supported with"
+ " peeling for alignment.\n");
+ return false;
+ }
+
+ /* An epilogue loop starts the surrogate over: its index would be a
+ position in the epilogue rather than in the whole iteration space.
+ The surrogate is created during vectorization, so there is no scalar
+ PHI for vect_update_ivs_after_vectorizer to restart it from. */
+ if (LOOP_VINFO_EPILOGUE_P (loop_vinfo))
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "minmax+index: pointer index not supported in a"
+ " vectorized epilogue.\n");
+ return false;
+ }
+
+ stmt_vec_info mmphi = VECT_REDUC_INFO_COUPLED_MINMAX_PHI (reduc_info);
+ tree surrogate_vt
+ = (mmphi
+ ? vect_minmax_surrogate_vectype (loop_vinfo,
+ TREE_TYPE (gimple_phi_result
+ (mmphi->stmt)))
+ : NULL_TREE);
+ tree ibase = NULL_TREE;
+ tree istride = NULL_TREE;
+ int index_start = 0;
+ if (surrogate_vt
+ && vect_capture_pointer_iter_base_stride (loop_vinfo, phi_info,
+ &ibase, &istride,
+ &index_start))
+ {
+ VECT_REDUC_INFO_COUPLED_MINMAX_BASE (reduc_info) = ibase;
+ VECT_REDUC_INFO_COUPLED_MINMAX_STRIDE (reduc_info) = istride;
+ VECT_REDUC_INFO_COUPLED_IS_POINTER (reduc_info) = true;
+ VECT_REDUC_INFO_COUPLED_MINMAX_INDEX_START (reduc_info) = index_start;
+ }
+ else
+ {
+ /* Without a base, a positive constant stride and a start offset it
+ can represent, the epilogue cannot rebuild the iterator, and the
+ index stays pointer typed for a path that reduces integers. */
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "minmax+index: cannot rebuild the pointer"
+ " index in the epilogue.\n");
+ return false;
+ }
+ }
+
/* The minmax+index epilogue only handles single-lane SLP groups. */
if (reduction_type == MINMAX_INDEX_REDUCTION
&& SLP_TREE_LANES (slp_node) != 1)
@@ -8950,6 +9301,16 @@ vect_transform_cycle_phi (loop_vec_info loop_vinfo,
/* Leave the scalar phi in place. */
return true;
+ /* Coupled pointer-iterator argmax: the index PHI is pointer-typed in the
+ IR but is reduced over the unsigned surrogate, so its accumulator
+ starts at zero rather than at the PHI's pointer value. */
+ bool coupled_ptr_idx
+ = (reduc_info
+ && VECT_REDUC_INFO_TYPE (reduc_info) == MINMAX_INDEX_REDUCTION
+ && VECT_REDUC_INFO_COUPLED_IS_POINTER (reduc_info)
+ && is_a <gphi *> (stmt_info->stmt)
+ && POINTER_TYPE_P (TREE_TYPE (gimple_phi_result
+ (as_a <gphi *> (stmt_info->stmt)))));
if (reduc_info && reduc_info->is_reduc_chain && dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location,
"vectorizing a reduction chain\n");
@@ -9013,7 +9374,9 @@ vect_transform_cycle_phi (loop_vec_info loop_vinfo,
for (unsigned int i = 0; i < num_phis; ++i)
{
gphi *this_phi = as_a<gphi *> (stmts[i]->stmt);
- initial_values.quick_push (vect_phi_initial_value (this_phi));
+ initial_values.quick_push
+ (coupled_ptr_idx ? build_zero_cst (TREE_TYPE (vectype_out))
+ : vect_phi_initial_value (this_phi));
}
tree neutral_op = VECT_REDUC_INFO_NEUTRAL_OP (reduc_info);
if (vec_num == 1
diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
index eb78fd2eba39..f16029a2b4da 100644
--- a/gcc/tree-vect-stmts.cc
+++ b/gcc/tree-vect-stmts.cc
@@ -12122,10 +12122,23 @@ vectorizable_condition (vec_info *vinfo,
&else_clause, &else_slp_node, &dts[3], &vectype2))
return false;
- if (vectype1 && !useless_type_conversion_p (vectype, vectype1))
- return false;
-
- if (vectype2 && !useless_type_conversion_p (vectype, vectype2))
+ /* Coupled pointer-iterator argmax: the index-update select
+ "result = cond ? ptr_iv : result" feeds a MINMAX_INDEX reduction whose
+ index is reduced over an unsigned surrogate. Vectorize the select over
+ that surrogate: the winning pointer-IV arm is replaced at transform time
+ by a synthesized unsigned iteration index, which is why the operand type
+ check below is skipped. VECTYPE is already the surrogate, via
+ vect_get_vector_types_for_stmt. */
+ bool coupled_ptr_argmax = false;
+ if (tree surr = vect_coupled_ptr_argmax_index_vectype (vinfo, stmt_info))
+ {
+ gcc_checking_assert (types_compatible_p (vectype, surr));
+ coupled_ptr_argmax = true;
+ }
+
+ if (!coupled_ptr_argmax
+ && ((vectype1 && !useless_type_conversion_p (vectype, vectype1))
+ || (vectype2 && !useless_type_conversion_p (vectype, vectype2))))
return false;
if (maybe_ne (TYPE_VECTOR_SUBPARTS (vectype),
@@ -12181,7 +12194,10 @@ vectorizable_condition (vec_info *vinfo,
&LOOP_VINFO_MASKS (loop_vinfo),
vec_num, vectype, NULL);
}
- /* Extra inactive lanes should be safe for vect_nested_cycle. */
+ /* Extra inactive lanes should be safe for vect_nested_cycle. A
+ coupled min/max+index reduction depends on this too: its epilogue
+ reduces the accumulators the vector loop ends with, which masking
+ would leave holding inactive lanes. */
else if (!nested_cycle_p)
{
if (dump_enabled_p ())
@@ -12252,10 +12268,55 @@ vectorizable_condition (vec_info *vinfo,
if (reduction_type == EXTRACT_LAST_REDUCTION)
vec_else_clause = else_clause;
+ /* Coupled pointer-iterator argmax: synthesize the winning arm as an
+ unsigned iteration-index vector induction {0,1,...} stepping by the vector
+ width, replacing the pointer-IV then-clause. The MINMAX_INDEX epilogue
+ reduces this surrogate index and materializes the result pointer. */
+ tree ptr_argmax_index = NULL_TREE;
+ if (coupled_ptr_argmax && !vec_oprnds3.is_empty ())
+ {
+ class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
+ /* The surrogate replaces the then-arm. The else-arm therefore has to
+ be the index PHI this select feeds on the latch. */
+ gphi *idx_phi = (TREE_CODE (else_clause) == SSA_NAME
+ ? dyn_cast <gphi *> (SSA_NAME_DEF_STMT (else_clause))
+ : NULL);
+ gcc_checking_assert (idx_phi
+ && (PHI_ARG_DEF_FROM_EDGE (idx_phi,
+ loop_latch_edge (loop))
+ == gimple_assign_lhs (stmt)));
+ /* Build the index vector with the accumulator arm's exact vector
+ type so the VEC_COND operands agree in lane count and
+ signedness. */
+ tree index_vectype = TREE_TYPE (vec_oprnds3[0]);
+ poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (index_vectype);
+ /* The iterator's first compared position may be offset from the base
+ (K=1 for the pre-incremented std::max_element shape): start the
+ index vector at K so surrogate index 0 keeps mapping to the base
+ pointer. */
+ int index_start = 0;
+ vect_reduc_info ri = info_for_reduction (loop_vinfo, slp_node);
+ if (ri && VECT_REDUC_INFO_COUPLED_IS_POINTER (ri))
+ index_start = VECT_REDUC_INFO_COUPLED_MINMAX_INDEX_START (ri);
+ tree series = build_index_vector (index_vectype, index_start, 1);
+ tree vec_step
+ = build_vector_from_val (index_vectype,
+ build_int_cst (TREE_TYPE (index_vectype),
+ nunits));
+ gimple_stmt_iterator incr_gsi;
+ bool insert_after;
+ vect_iv_increment_position (LOOP_VINFO_MAIN_EXIT (loop_vinfo),
+ &incr_gsi, &insert_after);
+ tree indx_before, indx_after;
+ create_iv (series, PLUS_EXPR, vec_step, NULL_TREE, loop, &incr_gsi,
+ insert_after, &indx_before, &indx_after);
+ ptr_argmax_index = indx_before;
+ }
+
/* Arguments are ready. Create the new vector stmt. */
FOR_EACH_VEC_ELT (vec_oprnds0, i, vec_cond_lhs)
{
- vec_then_clause = vec_oprnds2[i];
+ vec_then_clause = coupled_ptr_argmax ? ptr_argmax_index : vec_oprnds2[i];
if (reduction_type != EXTRACT_LAST_REDUCTION)
vec_else_clause = vec_oprnds3[i];
@@ -14502,6 +14563,51 @@ vect_gen_while_not (gimple_seq *seq, tree mask_type,
tree start_index,
return gimple_build (seq, BIT_NOT_EXPR, mask_type, tmp);
}
+/* If STMT_INFO is the pointer-typed index PHI or index-update COND_EXPR of a
+ coupled min/max+index (std::max_element) reduction, return the unsigned
+ surrogate vector type it should be vectorized over, sized to the coupled
+ value's vector so the two agree in lane count and a wider pointer does not
+ cap the vectorization factor. Return NULL_TREE otherwise. */
+
+tree
+vect_coupled_ptr_argmax_index_vectype (vec_info *vinfo, stmt_vec_info
stmt_info)
+{
+ gimple *stmt = stmt_info->stmt;
+ stmt_vec_info sel_info = NULL;
+ loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
+ /* Nothing carries the surrogate into an epilogue; see the refusal in
+ vectorizable_reduction. */
+ if (loop_vinfo && LOOP_VINFO_EPILOGUE_P (loop_vinfo))
+ return NULL_TREE;
+ gphi *phi = dyn_cast <gphi *> (stmt);
+ if (stmt_info->coupled_minmax_peer != NULL
+ && is_gimple_assign (stmt)
+ && gimple_assign_rhs_code (stmt) == COND_EXPR
+ && POINTER_TYPE_P (TREE_TYPE (gimple_assign_lhs (stmt))))
+ sel_info = stmt_info;
+ else if (loop_vinfo && phi
+ && POINTER_TYPE_P (TREE_TYPE (gimple_phi_result (phi)))
+ && gimple_bb (phi) == LOOP_VINFO_LOOP (loop_vinfo)->header)
+ {
+ class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
+ tree latch = PHI_ARG_DEF_FROM_EDGE (phi, loop_latch_edge (loop));
+ if (TREE_CODE (latch) == SSA_NAME)
+ {
+ stmt_vec_info ldef = loop_vinfo->lookup_def (latch);
+ if (ldef && ldef->coupled_minmax_peer != NULL)
+ sel_info = ldef;
+ }
+ }
+ if (!sel_info)
+ return NULL_TREE;
+ stmt_vec_info val_phi = sel_info->coupled_minmax_peer;
+ if (!val_phi || !is_a <gphi *> (val_phi->stmt))
+ return NULL_TREE;
+ return vect_minmax_surrogate_vectype (vinfo,
+ TREE_TYPE (gimple_phi_result
+ (val_phi->stmt)));
+}
+
/* Try to compute the vector types required to vectorize STMT_INFO,
returning true on success and false if vectorization isn't possible.
If GROUP_SIZE is nonzero and we're performing BB vectorization,
@@ -14521,6 +14627,19 @@ vect_get_vector_types_for_stmt (vec_info *vinfo,
stmt_vec_info stmt_info,
{
gimple *stmt = stmt_info->stmt;
+ /* Coupled pointer-iterator argmax index: vectorize over the unsigned
+ surrogate, which has the value's lane count. The VF is then not capped
+ by the pointer's wider elements. */
+ if (tree surr = vect_coupled_ptr_argmax_index_vectype (vinfo, stmt_info))
+ {
+ *stmt_vectype_out = surr;
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "coupled ptr-argmax index surrogate vectype: %T\n",
+ surr);
+ return opt_result::success ();
+ }
+
/* For BB vectorization, we should always have a group size once we've
constructed the SLP tree; the only valid uses of zero GROUP_SIZEs
are tentative requests during things like early data reference
diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
index 5abf8231cf16..37ab248ed88b 100644
--- a/gcc/tree-vectorizer.h
+++ b/gcc/tree-vectorizer.h
@@ -916,6 +916,20 @@ public:
paired min/max vector. NULL if not part of a coupled pair. */
stmt_vec_info coupled_minmax_phi;
+ /* For MINMAX_INDEX_REDUCTION over a pointer-typed index PHI (the
+ std::max_element iterator shape): the index PHI's initial value, and the
+ constant byte stride of the affine IV the coupled COND_EXPR selects. The
+ index is tracked as an integer surrogate during vectorization and the
+ pointer materialized in the epilogue as RESULT = BASE + IDX * STRIDE. */
+ tree coupled_minmax_base;
+ tree coupled_minmax_stride;
+ bool coupled_is_pointer;
+ /* Surrogate index of the iterator's first compared position relative to
+ BASE, the reduction PHI's initial value. 0 when the reduction PHI and
+ the iterator IV start together, 1 for the std::max_element
+ "while (++first != last)" shape where the iterator is pre-incremented. */
+ int coupled_minmax_index_start;
+
/* If the vector code is performing N scalar reductions in parallel,
this variable gives the initial scalar values of those N reductions. */
auto_vec<tree> reduc_initial_values;
@@ -941,6 +955,11 @@ typedef class vect_reduc_info_s *vect_reduc_info;
#define VECT_REDUC_INFO_VECTYPE(I) ((I)->reduc_vectype)
#define VECT_REDUC_INFO_VECTYPE_FOR_MASK(I) ((I)->reduc_vectype_for_mask)
#define VECT_REDUC_INFO_COUPLED_MINMAX_PHI(I) ((I)->coupled_minmax_phi)
+#define VECT_REDUC_INFO_COUPLED_MINMAX_BASE(I) ((I)->coupled_minmax_base)
+#define VECT_REDUC_INFO_COUPLED_MINMAX_STRIDE(I) ((I)->coupled_minmax_stride)
+#define VECT_REDUC_INFO_COUPLED_IS_POINTER(I) ((I)->coupled_is_pointer)
+#define VECT_REDUC_INFO_COUPLED_MINMAX_INDEX_START(I) \
+ ((I)->coupled_minmax_index_start)
#define VECT_REDUC_INFO_FORCE_SINGLE_CYCLE(I) ((I)->force_single_cycle)
#define VECT_REDUC_INFO_RESULT_POS(I) ((I)->reduc_result_pos)
#define VECT_REDUC_INFO_NEUTRAL_OP(I) ((I)->neutral_op)
@@ -2686,6 +2705,8 @@ extern tree vect_create_addr_base_for_vector_ref
(vec_info *,
extern tree neutral_op_for_reduction (tree, code_helper, tree, bool = true);
extern widest_int vect_iv_limit_for_partial_vectors (loop_vec_info loop_vinfo);
bool vect_rgroup_iv_might_wrap_p (loop_vec_info, rgroup_controls *);
+extern tree vect_minmax_surrogate_vectype (vec_info *, tree);
+extern tree vect_coupled_ptr_argmax_index_vectype (vec_info *, stmt_vec_info);
/* Used in gimple-loop-interchange.c and tree-parloops.cc. */
extern bool check_reduction_path (dump_user_location_t, loop_p, gphi *, tree,
enum tree_code);
--
2.55.0
