The following implements BB reduction epilog handling for two-lane
vectors with lane extracts. This allows targets to omit defining
reduc_*_scal optabs for two lane vector modes and enables trivial
handling of in-order reductions with two lanes. The former is
one issue we run into with PR126028 on x86_64.
This causes some no-op vectorization since we now accept vector
costs equal to scalar costs.
For gcc.target/i386/pr54400.c this shows that after SLP vectorizing
a two lane reduction we are no longer able to match up the x86 haddpd
instruction I have sent a partial x86 backend fix.
For g++.target/i386/pr114187.C it shows the usual
argument/return costing difficulties but also a too broad testcase
and inadverted coverage of -m32 - I have adjusted the testcase.
Bootstrapped and tested on x86_64-unknown-linux-gnu. I'll push
this once the x86 prerequesite has landed.
Richard.
PR tree-optimization/126028
* tree-vect-slp.cc (vect_slp_check_for_roots): Move
fold-left reduction check ...
(vectorizable_bb_reduc_epilogue): ... here and allow
two reduction lanes to be unaffected. Handle the two
vector lane without target support for the reduction.
(vectorize_slp_instance_root_stmt): Implement manual two-lane
reduction.
* gcc.dg/vect/bb-slp-reduc-1.c: New testcase for a two-lane
in-order reduction.
* c-c++-common/vector-subscript-4.c: Use -fno-vectorize.
* g++.target/i386/pr114187.C: Narrow pattern to better
only catch the reported issue.
---
.../c-c++-common/vector-subscript-4.c | 2 +-
gcc/testsuite/g++.target/i386/pr114187.C | 3 +-
gcc/testsuite/gcc.dg/vect/bb-slp-reduc-1.c | 12 +++++
gcc/tree-vect-slp.cc | 47 ++++++++++++++-----
4 files changed, 49 insertions(+), 15 deletions(-)
create mode 100644 gcc/testsuite/gcc.dg/vect/bb-slp-reduc-1.c
diff --git a/gcc/testsuite/c-c++-common/vector-subscript-4.c
b/gcc/testsuite/c-c++-common/vector-subscript-4.c
index 3138dc619d3..acd8bd19595 100644
--- a/gcc/testsuite/c-c++-common/vector-subscript-4.c
+++ b/gcc/testsuite/c-c++-common/vector-subscript-4.c
@@ -1,5 +1,5 @@
/* { dg-do compile } */
-/* { dg-options "-O2 -fdump-tree-nrv -w -Wno-psabi" } */
+/* { dg-options "-O2 -fno-tree-vectorize -fdump-tree-nrv -w -Wno-psabi" } */
#define foobar(n) \
typedef int v##n##si __attribute__ ((vector_size (4 * n))); \
diff --git a/gcc/testsuite/g++.target/i386/pr114187.C
b/gcc/testsuite/g++.target/i386/pr114187.C
index 69912a94cef..437e54cadcb 100644
--- a/gcc/testsuite/g++.target/i386/pr114187.C
+++ b/gcc/testsuite/g++.target/i386/pr114187.C
@@ -9,5 +9,6 @@ double sumxy_p(P2d p) {
return p.x + p.y;
}
-/* { dg-final { scan-assembler-not "movq" } } */
+/* No move between GPR and XMM. */
+/* { dg-final { scan-assembler-not "movq\[ \t%\]*r" } } */
/* { dg-final { scan-assembler-not "xchg" } } */
diff --git a/gcc/testsuite/gcc.dg/vect/bb-slp-reduc-1.c
b/gcc/testsuite/gcc.dg/vect/bb-slp-reduc-1.c
new file mode 100644
index 00000000000..18357be2df2
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/bb-slp-reduc-1.c
@@ -0,0 +1,12 @@
+/* { dg-do compile } */
+/* { dg-require-effective-target vect_double } */
+
+double foo (double *x, double *y, double *z)
+{
+ return (x[0] * y[0] - z[0]) + (x[1] * y[1] - z[1]);
+}
+
+/* Even though without -ffast-math a reduction with double requires in-order
+ vectorization which we do not fully implement for BB vectorization we
+ should be able to handle the two-lane vector case just fine. */
+/* { dg-final { scan-tree-dump "optimized: basic block part vectorized" "slp2"
{ target vect_hw_misalign } } } */
diff --git a/gcc/tree-vect-slp.cc b/gcc/tree-vect-slp.cc
index 850cb1efacc..d1317fdde6e 100644
--- a/gcc/tree-vect-slp.cc
+++ b/gcc/tree-vect-slp.cc
@@ -9209,11 +9209,19 @@ vectorizable_bb_reduc_epilogue (slp_instance instance,
internal_fn reduc_fn;
tree vectype = SLP_TREE_VECTYPE (SLP_INSTANCE_TREE (instance));
if (!vectype
- || !reduction_fn_for_scalar_code (reduc_code, &reduc_fn)
- || reduc_fn == IFN_LAST
- || !direct_internal_fn_supported_p (reduc_fn, vectype, OPTIMIZE_FOR_BOTH)
|| !useless_type_conversion_p (TREE_TYPE (gimple_assign_lhs (stmt)),
- TREE_TYPE (vectype)))
+ TREE_TYPE (vectype))
+ || (maybe_ne (TYPE_VECTOR_SUBPARTS (vectype), 2u)
+ && (!reduction_fn_for_scalar_code (reduc_code, &reduc_fn)
+ || reduc_fn == IFN_LAST
+ || !direct_internal_fn_supported_p (reduc_fn, vectype,
+ OPTIMIZE_FOR_BOTH)))
+ /* Two-element reductions do not need special-handling for fold-left,
+ other cases are not yet implemented. remain_defs also have to
+ be included here. */
+ || (needs_fold_left_reduction_p (TREE_TYPE (vectype), reduc_code)
+ && (!instance->remain_defs.is_empty ()
+ || maybe_ne (TYPE_VECTOR_SUBPARTS (vectype), 2u))))
{
if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -9996,10 +10004,6 @@ vect_slp_check_for_roots (bb_vec_info bb_vinfo)
}
else if (!VECTOR_TYPE_P (TREE_TYPE (rhs))
&& (associative_tree_code (code) || code == MINUS_EXPR)
- /* ??? This pessimizes a two-element reduction. PR54400.
- ??? In-order reduction could be handled if we only
- traverse one operand chain in vect_slp_linearize_chain. */
- && !needs_fold_left_reduction_p (TREE_TYPE (rhs), code)
/* Ops with constants at the tail can be stripped here. */
&& TREE_CODE (rhs) == SSA_NAME
&& TREE_CODE (gimple_assign_rhs2 (assign)) == SSA_NAME
@@ -12215,13 +12219,30 @@ vectorize_slp_instance_root_stmt (vec_info *vinfo,
slp_tree node, slp_instance i
vec_def, def);
}
vec_defs.release ();
- /* ??? Support other schemes than direct internal fn. */
+ /* ??? Support other schemes than direct internal fn or two
+ element vectors. */
+ tree scalar_def;
internal_fn reduc_fn;
if (!reduction_fn_for_scalar_code (reduc_code, &reduc_fn)
- || reduc_fn == IFN_LAST)
- gcc_unreachable ();
- tree scalar_def = gimple_build (&epilogue, as_combined_fn (reduc_fn),
- TREE_TYPE (compute_vectype), vec_def);
+ || reduc_fn == IFN_LAST
+ || !direct_internal_fn_supported_p (reduc_fn, compute_vectype,
+ OPTIMIZE_FOR_BOTH))
+ {
+ gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (compute_vectype), 2u));
+ tree tem0 = gimple_build (&epilogue, BIT_FIELD_REF,
+ TREE_TYPE (compute_vectype), vec_def,
+ TYPE_SIZE (TREE_TYPE (compute_vectype)),
+ bitsize_zero_node);
+ tree tem1 = gimple_build (&epilogue, BIT_FIELD_REF,
+ TREE_TYPE (compute_vectype), vec_def,
+ TYPE_SIZE (TREE_TYPE (compute_vectype)),
+ TYPE_SIZE (TREE_TYPE (compute_vectype)));
+ scalar_def = gimple_build (&epilogue, reduc_code,
+ TREE_TYPE (compute_vectype), tem0, tem1);
+ }
+ else
+ scalar_def = gimple_build (&epilogue, as_combined_fn (reduc_fn),
+ TREE_TYPE (compute_vectype), vec_def);
if (!SLP_INSTANCE_REMAIN_DEFS (instance).is_empty ())
{
tree rem_def = NULL_TREE;
--
2.51.0
