When the code involves load in narrower type followed by arithmetic operation
in wider type, GCC is generating code that uses vextract instructions, while it
could instead make use of extended loads with ZMM registers to achieve better
performance. This patch adds support to identify load + convert pattern in SLP
tree and narrow the load vector type to narrower vector type if the target
supports the fused load-and-extend for the case. The function
vect_optimize_slp_widening_loads runs in tree-vect-loop.cc before SLP operation
analysis.
The implementation does not affect costing as no vectorization cost is computed
using the old (wide) load type. When SLP operation costs are first recorded,
they already see the narrowed load type. The implementation also may have no
unintended side effects as the narrowing is applied only if load is referred
only once in the SLP graph. i.e. by the convert following it and the SLP tree
structure and GIMPLE IR is not modified, only an SLP analysis field is updated.
The tests vect-epilogues-2b.c and vect-unpack-2.c are updated because the
patch fuses load + widening-convert into direct extended-load instructions,
which changes epilogue vectorization in vect-epilogues-2b.c (now 32-byte
instead of smaller cascaded epilogues) and replaces unpack-based vpmovsxbw
sequences with fused vpmovsxbd in vect-unpack-2.c.
gcc/ChangeLog:
* tree-vect-loop.cc: Add call to vect_optimize_slp_widening_loads.
* tree-vect-slp.cc (vect_optimize_widening_loads_1): Walk through each
SLP node to identify a load followed by convert to update the vector type so
that it can be fused into single extended load operation if the target supports
it.
(vect_optimize_slp_widening_loads): Call vect_optimize_widening_loads_1
for each SLP node.
* tree-vectorizer.h (vect_optimize_slp_widening_loads): Add
vect_optimize_slp_widening_loads.
gcc/testsuite/ChangeLog:
* gcc.target/i386/vect-epilogues-2b.c: Updated epilogue vector factor
test check.
* gcc.target/i386/vect-unpack-2.c: Updated test check for the assembly
instruction generated.
* gcc.dg/vect/widening-load-u8.c: New test.
---
Hi GCC Team,
The patch enables extended load optimization for x86_64 target when the code
involves load in narrower type followed by arithmetic operation in wider type.
I have bootstrapped and tested this patch on x86_64-linux-gnu with no
regressions.
Please let me know if any modifications are needed.
Thanks,
Sridevi
gcc/testsuite/gcc.dg/vect/widening-load-u8.c | 19 +++
.../gcc.target/i386/vect-epilogues-2b.c | 4 +-
gcc/testsuite/gcc.target/i386/vect-unpack-2.c | 2 +-
gcc/tree-vect-loop.cc | 7 +
gcc/tree-vect-slp.cc | 143 ++++++++++++++++++
gcc/tree-vectorizer.h | 1 +
6 files changed, 172 insertions(+), 4 deletions(-)
create mode 100644 gcc/testsuite/gcc.dg/vect/widening-load-u8.c
diff --git a/gcc/testsuite/gcc.dg/vect/widening-load-u8.c
b/gcc/testsuite/gcc.dg/vect/widening-load-u8.c
new file mode 100644
index 00000000000..59ce0bb6d43
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/widening-load-u8.c
@@ -0,0 +1,19 @@
+/* { dg-do compile { target avx512bw } } */
+/* { dg-additional-options "-mavx512f -mavx512bw --param
vect-epilogues-nomask=0 -save-temps" } */
+/* { dg-require-effective-target avx512bw } */
+
+#include <stdint.h>
+
+/* uint8 loads widened for int mul/add/shift/store.
+ Exercises SLP narrowing of widening loads for load+extend fusion. */
+void
+widening_load_u8 (uint8_t * restrict dst, uint8_t *restrict src,
+ uint8_t *restrict srcp, int cA, int cC, int n)
+{
+ for (int x = 0; x < n; x++)
+ dst[x] = (cA * src[x] + cC * srcp[x] + 32) >> 6;
+}
+
+/* { dg-final { scan-tree-dump "widening load optimization: narrowing load
vectype" "vect" } } */
+/* { dg-final { scan-assembler "vpmovzxbw\[ \\t\]+\\\(\%" } } */
+/* { dg-final { scan-assembler-not "vpmovzxbw\[ \\t\]+%ymm" } } */
diff --git a/gcc/testsuite/gcc.target/i386/vect-epilogues-2b.c
b/gcc/testsuite/gcc.target/i386/vect-epilogues-2b.c
index 4a58dc67bde..1f0c47e8189 100644
--- a/gcc/testsuite/gcc.target/i386/vect-epilogues-2b.c
+++ b/gcc/testsuite/gcc.target/i386/vect-epilogues-2b.c
@@ -10,6 +10,4 @@ int test (signed char *data, int n)
}
/* { dg-final { scan-tree-dump "loop vectorized using 64 byte vectors" "vect"
} } */
-/* { dg-final { scan-tree-dump-not "loop vectorized using 32 byte vectors"
"vect" } } */
-/* { dg-final { scan-tree-dump "loop vectorized using 16 byte vectors" "vect"
} } */
-/* { dg-final { scan-tree-dump "loop vectorized using 8 byte vectors" "vect" {
target { ! ia32 } } } } */
+/* { dg-final { scan-tree-dump "epilogue loop vectorized using 32 byte
vectors" "vect" } } */
diff --git a/gcc/testsuite/gcc.target/i386/vect-unpack-2.c
b/gcc/testsuite/gcc.target/i386/vect-unpack-2.c
index 51c518e6111..1846e41b78d 100644
--- a/gcc/testsuite/gcc.target/i386/vect-unpack-2.c
+++ b/gcc/testsuite/gcc.target/i386/vect-unpack-2.c
@@ -28,4 +28,4 @@ avx512bw_test ()
abort ();
}
-/* { dg-final { scan-assembler-times "vpmovsxbw\[ \\t\]+\[^\n\]*%zmm" 2 } } */
+/* { dg-final { scan-assembler-times "vpmovsxbd\[ \\t\]+\[^\n\]*%zmm" 4 } } */
diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc
index a1e60a31ecb..c20624955d8 100644
--- a/gcc/tree-vect-loop.cc
+++ b/gcc/tree-vect-loop.cc
@@ -2361,6 +2361,13 @@ start_over:
if (!ok)
return ok;
+ /* Go through the SLP tree nodes to identify load + convert pattern.
+ For such patterns, narrow the load vector type so that the load and
convert
+ instructions are fused into a single extended load instruction with
+ ZMM register (e.g.vpmovzxbw) if the target supports direct load+extend
+ fusion. */
+ vect_optimize_slp_widening_loads (loop_vinfo);
+
/* Analyze operations in the SLP instances. We can't simply
remove unsupported SLP instances as this makes the above
SLP kind detection invalid and might also affect the VF. */
diff --git a/gcc/tree-vect-slp.cc b/gcc/tree-vect-slp.cc
index 0e74ac00075..7481876f0b2 100644
--- a/gcc/tree-vect-slp.cc
+++ b/gcc/tree-vect-slp.cc
@@ -8571,6 +8571,149 @@ vect_gather_slp_loads (vec_info *vinfo)
}
}
+/* Walk through the SLP tree rooted at NODE finding widening integer
conversions
+ whose input operand is a contiguous load with a single consumer. For such
+ patterns, narrow the load's vector type so that the load element count
+ matches the conversion output element count. This enables the target to
+ fuse the load and extension into a single extended load instruction
+ instead of loading a full-width vector and splitting it
+ with VEC_UNPACK_LO/HI.
+ Example:
+ INPUT:
+ LOAD_EXPR <vec_type: i8x64>
+ CONVERT_EXPR <vec_type: i16x32>
+
+ Example OUTPUT:
+ LOAD_EXPR <vec_type: i8x32>
+ CONVERT_EXPR <vec_type: i16x32>
+ With the load narrowed to i8x32, the vector type of the convert matches the
+ vector type of the load.Thus the load and convert are fused into a single
+ extended load instruction with ZMM register. */
+
+static void
+vect_optimize_widening_loads_1 (vec_info *vinfo, slp_tree node,
+ hash_set<slp_tree> &visited)
+{
+ if (!node || visited.add (node))
+ return;
+ /* Call the function recursively for all the children of the node. */
+ for (slp_tree child : SLP_TREE_CHILDREN (node))
+ vect_optimize_widening_loads_1 (vinfo, child, visited);
+ if (SLP_TREE_CHILDREN (node).length () < 1)
+ return;
+
+ /* Check if the node is a convert expression containing a load as child
+ node. */
+ stmt_vec_info stmt_info = SLP_TREE_REPRESENTATIVE (node);
+ if (!stmt_info || !is_gimple_assign (stmt_info->stmt))
+ return;
+ tree_code assign_code = gimple_assign_rhs_code (stmt_info->stmt);
+ if (!CONVERT_EXPR_CODE_P (assign_code))
+ return;
+ slp_tree child_node = SLP_TREE_CHILDREN (node)[0];
+ if (!child_node)
+ return;
+
+ /* The input must be a load with a single consumer so we can safely
+ change its vector type without affecting other nodes. */
+ if (SLP_TREE_DEF_TYPE (child_node) != vect_internal_def)
+ return;
+ stmt_vec_info child_node_info = SLP_TREE_REPRESENTATIVE (child_node);
+ if (!child_node_info || !STMT_VINFO_DATA_REF (child_node_info)
+ || !DR_IS_READ (STMT_VINFO_DATA_REF (child_node_info)))
+ return;
+ if (SLP_TREE_REF_COUNT (child_node) != 1)
+ return;
+
+ /* Only narrow a plain contiguous load. Narrowing grouped, strided or
+ permuted loads breaks the permute and forces a partial-vector
+ load the target may not support. */
+ if (STMT_VINFO_GROUPED_ACCESS (child_node_info)
+ || STMT_VINFO_STRIDED_P (child_node_info)
+ || !SLP_TREE_LOAD_PERMUTATION (child_node).is_empty ())
+ return;
+
+ loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
+ if (loop_vinfo)
+ {
+ /* If the loop uses partial vectors, a narrowed sub-width load
+ cannot be expressed as a masked/partial-vector load on
+ the target. */
+ if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
+ && param_vect_partial_vector_usage != 0)
+ return;
+ }
+
+ /* Check if the vector types of the node and the child node are set and the
+ number of units in the vector type of the node is multiple and not same as
+ that of the child node. */
+ tree vectype_out = SLP_TREE_VECTYPE (node);
+ tree vectype_in = SLP_TREE_VECTYPE (child_node);
+ if (!vectype_out || !vectype_in)
+ return;
+ poly_uint64 nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
+ poly_uint64 nunits_in = TYPE_VECTOR_SUBPARTS (vectype_in);
+
+ if (!multiple_p (nunits_in, nunits_out) || known_eq (nunits_in, nunits_out))
+ return;
+
+ tree lhs_type = TREE_TYPE (gimple_assign_lhs (stmt_info->stmt));
+ tree rhs_type = TREE_TYPE (gimple_assign_rhs1 (stmt_info->stmt));
+ if (!INTEGRAL_TYPE_P (lhs_type) || !INTEGRAL_TYPE_P (rhs_type))
+ return;
+
+ /* The vector type of the child node can be updated to narrow_vectype if
+ the number of units in the narrow_vectype is equal to the number of
+ units in the vector type of the node and that it is supported by the
+ target. */
+ tree narrow_vectype = get_related_vectype_for_scalar_type (
+ TYPE_MODE (vectype_in), rhs_type, nunits_out);
+ if (narrow_vectype == NULL_TREE
+ || maybe_ne (TYPE_VECTOR_SUBPARTS (narrow_vectype), nunits_out))
+ return;
+ if (!supportable_convert_operation (assign_code, vectype_out,
narrow_vectype))
+ return;
+
+ if (loop_vinfo)
+ {
+ /* Narrowing the load increases vec_num. Outer-loop vectorization
+ rejects inner-loop loads with vec_num > 1 (see vectorizable_load
+ function in tree-vect-stmts.cc). */
+ class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
+ if (nested_in_vect_loop_p (loop, child_node_info))
+ {
+ poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
+ vf *= SLP_TREE_LANES (child_node);
+ if (vect_get_num_vectors (vf, narrow_vectype) > 1)
+ return;
+ }
+ }
+
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "widening load optimization: narrowing load vectype "
+ "from %T to %T for direct load+convert fusion\n",
+ vectype_in, narrow_vectype);
+ SLP_TREE_VECTYPE (child_node) = narrow_vectype;
+}
+
+/* Optimize widening conversion + load patterns across all SLP instances. */
+
+void vect_optimize_slp_widening_loads (vec_info *vinfo)
+{
+ if (vinfo->slp_instances.is_empty ())
+ return;
+
+ unsigned int i;
+ slp_instance instance;
+ FOR_EACH_VEC_ELT (vinfo->slp_instances, i, instance)
+ {
+ hash_set<slp_tree> visited;
+ vect_optimize_widening_loads_1 (vinfo, SLP_INSTANCE_TREE (instance),
+ visited);
+ }
+}
+
/* For NODE update VF based on the number of lanes and the vector types
used. */
diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
index d21f132a9e5..e2abf2f96b9 100644
--- a/gcc/tree-vectorizer.h
+++ b/gcc/tree-vectorizer.h
@@ -2765,6 +2765,7 @@ extern bool vect_make_slp_decision (loop_vec_info);
extern bool vect_detect_hybrid_slp (loop_vec_info);
extern void vect_optimize_slp (vec_info *);
extern void vect_gather_slp_loads (vec_info *);
+extern void vect_optimize_slp_widening_loads (vec_info *);
extern tree vect_get_slp_scalar_def (slp_tree, unsigned);
extern void vect_get_slp_defs (slp_tree, vec<tree> *);
extern void vect_get_slp_defs (vec_info *, slp_tree, vec<vec<tree> > *,
--
2.34.1
