Recognize a contiguous load whose only use is an integer widening
conversion and replace the pair with .VEC_LOAD_EXTEND when the target
supports it. The pattern runs after the other widening patterns so
that widen_mult, dot_prod and sad still see the conversion.
VEC_LOAD_EXTEND has no masked or length-controlled form, so the pattern
is not applied to loops that may use partial vectors. The vectorization
factor stays that of the narrower memory type.
The tests vect-epilogues-2b.c, vect-epilogues-3.c and vect-unpack-2.c are
updated because the patch fuses a load and a widening conversion into a
direct extending load. In vect-epilogues-2b.c the epilogue is now a single
32-byte loop instead of cascaded 16-byte and 8-byte epilogues.
vect-epilogues-3.c no longer produces a 16-byte vectorized loop.
vect-unpack-2.c replaces the unpack-based vpmovsxbw sequence with fused
vpmovsxbd.
gcc/ChangeLog:
* tree-vect-loop.cc (vect_analyze_loop_2): Record when an epilogue
cannot use partial vectors before pattern recognition.
* tree-vect-patterns.cc (vect_name_used_in_pattern_p): New function.
(vect_load_used_only_by_p): New function.
(vect_loop_may_use_partial_vectors_p): New function.
(vect_recog_load_extend_pattern): Recognize a widening conversion of
a contiguous load and replace it with .VEC_LOAD_EXTEND.
(vect_pattern_recog): Run extending-load recognition after the other
patterns.
* tree-vect-slp.cc (vect_build_slp_tree_3): Treat VEC_LOAD_EXTEND as
a load.
(vect_build_slp_tree_2): Do not require a mask on VEC_LOAD_EXTEND.
(vect_update_slp_vf_for_node): Keep the vectorization factor required
by the narrower memory type.
* tree-vect-stmts.cc (vectorizable_load): Expand IFN_VEC_LOAD_EXTEND
as a contiguous extending load.
gcc/testsuite/ChangeLog:
* gcc.dg/vect/widening-load-u8.c: New test.
* gcc.target/i386/vect-load-extend-1.c: New test.
* gcc.target/i386/vect-epilogues-2b.c: Expect a 32-byte epilogue.
* gcc.target/i386/vect-epilogues-3.c: Drop the 16-byte vector scan.
* gcc.target/i386/vect-unpack-2.c: Expect vpmovsxbd.
---
gcc/testsuite/gcc.dg/vect/widening-load-u8.c | 20 ++
.../gcc.target/i386/vect-epilogues-2b.c | 4 +-
.../gcc.target/i386/vect-epilogues-3.c | 1 -
.../gcc.target/i386/vect-load-extend-1.c | 24 +++
gcc/testsuite/gcc.target/i386/vect-unpack-2.c | 2 +-
gcc/tree-vect-loop.cc | 9 +
gcc/tree-vect-patterns.cc | 190 ++++++++++++++++++
gcc/tree-vect-slp.cc | 22 +-
gcc/tree-vect-stmts.cc | 59 +++++-
9 files changed, 323 insertions(+), 8 deletions(-)
create mode 100644 gcc/testsuite/gcc.dg/vect/widening-load-u8.c
create mode 100644 gcc/testsuite/gcc.target/i386/vect-load-extend-1.c
diff --git a/gcc/testsuite/gcc.dg/vect/widening-load-u8.c
b/gcc/testsuite/gcc.dg/vect/widening-load-u8.c
new file mode 100644
index 00000000000..5e85ab1659e
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/widening-load-u8.c
@@ -0,0 +1,20 @@
+/* { dg-do compile { target avx512bw } } */
+/* { dg-additional-options "-mavx512f -mavx512bw --param
vect-epilogues-nomask=0 -save-temps -fdump-tree-optimized" } */
+/* { dg-require-effective-target avx512bw } */
+
+#include <stdint.h>
+
+/* uint8 loads widened for int mul/add/shift/store.
+ Exercises load+extend fusion of widening loads. */
+void
+widening_load_u8 (uint8_t * restrict dst, uint8_t *restrict src,
+ uint8_t *restrict srcp, int cA, int cC, int n)
+{
+ for (int x = 0; x < n; x++)
+ dst[x] = (cA * src[x] + cC * srcp[x] + 32) >> 6;
+}
+
+/* { dg-final { scan-tree-dump "load_extend pattern recognized" "vect" } } */
+/* { dg-final { scan-tree-dump "= \\.VEC_LOAD_EXTEND \\(" "optimized" } } */
+/* { dg-final { scan-assembler "vpmovzxbw\[ \\t\]+\\\(\%" } } */
+/* { dg-final { scan-assembler-not "vpmovzxbw\[ \\t\]+%ymm" } } */
diff --git a/gcc/testsuite/gcc.target/i386/vect-epilogues-2b.c
b/gcc/testsuite/gcc.target/i386/vect-epilogues-2b.c
index 4a58dc67bde..1f0c47e8189 100644
--- a/gcc/testsuite/gcc.target/i386/vect-epilogues-2b.c
+++ b/gcc/testsuite/gcc.target/i386/vect-epilogues-2b.c
@@ -10,6 +10,4 @@ int test (signed char *data, int n)
}
/* { dg-final { scan-tree-dump "loop vectorized using 64 byte vectors" "vect"
} } */
-/* { dg-final { scan-tree-dump-not "loop vectorized using 32 byte vectors"
"vect" } } */
-/* { dg-final { scan-tree-dump "loop vectorized using 16 byte vectors" "vect"
} } */
-/* { dg-final { scan-tree-dump "loop vectorized using 8 byte vectors" "vect" {
target { ! ia32 } } } } */
+/* { dg-final { scan-tree-dump "epilogue loop vectorized using 32 byte
vectors" "vect" } } */
diff --git a/gcc/testsuite/gcc.target/i386/vect-epilogues-3.c
b/gcc/testsuite/gcc.target/i386/vect-epilogues-3.c
index e88ab30c770..6ef2bac1505 100644
--- a/gcc/testsuite/gcc.target/i386/vect-epilogues-3.c
+++ b/gcc/testsuite/gcc.target/i386/vect-epilogues-3.c
@@ -11,5 +11,4 @@ int test (signed char *data, int n)
/* { dg-final { scan-tree-dump "loop vectorized using 64 byte vectors" "vect"
} } */
/* { dg-final { scan-tree-dump "loop vectorized using 32 byte vectors" "vect"
} } */
-/* { dg-final { scan-tree-dump "loop vectorized using 16 byte vectors" "vect"
} } */
/* { dg-final { scan-tree-dump "loop vectorized using 8 byte vectors" "vect" {
target { ! ia32 } } } } */
diff --git a/gcc/testsuite/gcc.target/i386/vect-load-extend-1.c
b/gcc/testsuite/gcc.target/i386/vect-load-extend-1.c
new file mode 100644
index 00000000000..128b15afaf7
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/vect-load-extend-1.c
@@ -0,0 +1,24 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -mavx512bw -mprefer-vector-width=512 --param
vect-epilogues-nomask=0 -fdump-tree-optimized" } */
+
+/* A widening conversion of a loaded value should become an extending load
+ from memory, keeping the vectorization factor of the narrow load. */
+
+void
+u16_u32 (unsigned int *__restrict d, unsigned short *__restrict s, int n)
+{
+ for (int i = 0; i < n; i++)
+ d[i] = s[i];
+}
+
+void
+s8_s32 (int *__restrict d, signed char *__restrict s, int n)
+{
+ for (int i = 0; i < n; i++)
+ d[i] = s[i];
+}
+
+/* { dg-final { scan-tree-dump-times "= \\.VEC_LOAD_EXTEND \\(" 6 "optimized"
} } */
+/* { dg-final { scan-assembler-times "vpmovzxwd\[ \\t\]+\[^\n\]*\\), %zmm" 2 }
} */
+/* { dg-final { scan-assembler-times "vpmovsxbd\[ \\t\]+\[^\n\]*\\), %zmm" 4 }
} */
+/* { dg-final { scan-assembler-not "vpmov\[sz\]x\[bwd\]+\[ \\t\]+%\[xyz\]mm" }
} */
diff --git a/gcc/testsuite/gcc.target/i386/vect-unpack-2.c
b/gcc/testsuite/gcc.target/i386/vect-unpack-2.c
index c578aa16d4c..c4a5dedc620 100644
--- a/gcc/testsuite/gcc.target/i386/vect-unpack-2.c
+++ b/gcc/testsuite/gcc.target/i386/vect-unpack-2.c
@@ -28,4 +28,4 @@ avx512bw_test ()
abort ();
}
-/* { dg-final { scan-assembler-times "vpmovsxbw\[ \\t\]+\[^\n\]*%zmm" 2 } } */
+/* { dg-final { scan-assembler-times "vpmovsxbd\[ \\t\]+\[^\n\]*%zmm" 4 } } */
diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc
index 78bfd607c5d..c7c114f6751 100644
--- a/gcc/tree-vect-loop.cc
+++ b/gcc/tree-vect-loop.cc
@@ -2261,6 +2261,15 @@ vect_analyze_loop_2 (loop_vec_info loop_vinfo, int
masked_p, bool &fatal,
if (LOOP_VINFO_NITERS_UNCOUNTED_P (loop_vinfo))
LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
+ /* Record up front when an epilogue cannot use partial vectors so that
+ pattern recognition can take it into account. This is not done for
+ the main loop since epilogue analysis consults its
+ LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P. */
+ if (LOOP_VINFO_EPILOGUE_P (loop_vinfo)
+ && (masked_p == 0
+ || (masked_p == -1 && param_vect_partial_vector_usage == 0)))
+ LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
+
/* The first group of checks is independent of the vector size. */
fatal = true;
diff --git a/gcc/tree-vect-patterns.cc b/gcc/tree-vect-patterns.cc
index cbe834e2aaf..7352e039057 100644
--- a/gcc/tree-vect-patterns.cc
+++ b/gcc/tree-vect-patterns.cc
@@ -6687,6 +6687,183 @@ vect_recog_mask_conversion_pattern (vec_info *vinfo,
return pattern_stmt;
}
+/* Return true if NAME is used by a pattern statement other than SKIP that
+ replaced STMT_INFO, including nested replacements in its definition
+ sequence. */
+
+static bool
+vect_name_used_in_pattern_p (vec_info *vinfo, stmt_vec_info stmt_info,
+ tree name, gimple *skip)
+{
+ //STMT_VINFO_IN_PATTERN_P is set when statement was rewritten by pattern
+ // recognition.If it is clear, there is no replacement to search.
+ if (!STMT_VINFO_IN_PATTERN_P (stmt_info))
+ return false;
+
+ auto uses_name_p = [name, skip] (gimple *stmt)
+ {
+ if (stmt == skip)
+ return false;
+ for (unsigned i = 0; i < gimple_num_ops (stmt); ++i)
+ if (gimple_op (stmt, i) == name)
+ return true;
+ return false;
+ };
+
+ //For a statement that is in a pattern, fetch GIMPLE of the replacement.
+ if (uses_name_p (STMT_VINFO_RELATED_STMT (stmt_info)->stmt))
+ return true;
+
+ //A pattern can also have setup statements stored on the original
+ // statement in STMT_VINFO_PATTERN_DEF_SEQ.
+ // Search through all the setup statements to see if any of them use name.
+ for (gimple_stmt_iterator gsi
+ = gsi_start (STMT_VINFO_PATTERN_DEF_SEQ (stmt_info));
+ !gsi_end_p (gsi); gsi_next (&gsi))
+ if (uses_name_p (gsi_stmt (gsi))
+ || vect_name_used_in_pattern_p (vinfo,
+ vinfo->lookup_stmt (gsi_stmt (gsi)),
+ name, skip))
+ return true;
+ return false;
+}
+
+/* Return true if STMT is the only user of NAME, the result of a load. */
+
+static bool
+vect_load_used_only_by_p (vec_info *vinfo, tree name, gimple *stmt)
+{
+ if (!has_single_use (name))
+ return false;
+
+ for (unsigned i = 0; i < vinfo->nbbs; ++i)
+ for (auto si = gsi_start_bb (vinfo->bbs[i]); !gsi_end_p (si);
+ gsi_next (&si))
+ if (stmt_vec_info info = vinfo->lookup_stmt (gsi_stmt (si)))
+ if (vect_name_used_in_pattern_p (vinfo, info, name, stmt))
+ return false;
+
+ return true;
+}
+
+/* Return true if LOOP_VINFO might be vectorized using partial vectors.
+ IFN_VEC_LOAD_EXTEND has no masked or length-controlled form, so using
+ it there would force a loop to use full vectors only. */
+
+static bool
+vect_loop_may_use_partial_vectors_p (loop_vec_info loop_vinfo)
+{
+ if (!LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
+ return false;
+
+ /* Epilogues that cannot use partial vectors have been marked already. */
+ if (LOOP_VINFO_EPILOGUE_P (loop_vinfo))
+ return true;
+
+ /* The main loop follows --param vect-partial-vector-usage. With 1 it
+ operates on full vectors unless the number of iterations is known to
+ be smaller than the vectorization factor. */
+ if (param_vect_partial_vector_usage == 0
+ || (param_vect_partial_vector_usage == 1
+ && !LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)))
+ return false;
+
+ return true;
+}
+
+/* Recognize an integer widening conversion fed by a plain contiguous load
+ and replace the pair with an extending-load internal function.
+
+ narrow = *ptr;
+ wide = (WIDE_TYPE) narrow;
+
+ becomes
+
+ wide = .VEC_LOAD_EXTEND (ptr, alignment, (NARROW_TYPE) 0); */
+
+static gimple *
+vect_recog_load_extend_pattern (vec_info *vinfo, stmt_vec_info stmt_info,
+ tree *type_out)
+{
+ loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
+ if (!loop_vinfo
+ || LOOP_VINFO_EARLY_BREAKS (loop_vinfo)
+ || vect_loop_may_use_partial_vectors_p (loop_vinfo))
+ return NULL;
+
+ gassign *convert = dyn_cast <gassign *> (stmt_info->stmt);
+ if (!convert
+ || !CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (convert)))
+ return NULL;
+
+ tree lhs = gimple_assign_lhs (convert);
+ tree rhs = gimple_assign_rhs1 (convert);
+ if (TREE_CODE (rhs) != SSA_NAME)
+ return NULL;
+
+ tree lhs_type = TREE_TYPE (lhs);
+ tree rhs_type = TREE_TYPE (rhs);
+ if (!INTEGRAL_TYPE_P (lhs_type) || !INTEGRAL_TYPE_P (rhs_type))
+ return NULL;
+
+ if (TYPE_PRECISION (lhs_type) <= TYPE_PRECISION (rhs_type))
+ return NULL;
+
+ stmt_vec_info load_info = vinfo->lookup_def (rhs);
+ if (!load_info || !STMT_VINFO_DATA_REF (load_info))
+ return NULL;
+
+ gassign *load = dyn_cast <gassign *> (load_info->stmt);
+ if (!load || !vect_load_used_only_by_p (vinfo, rhs, convert))
+ return NULL;
+
+ /* Access grouping is not known yet, so require a unit step to rule out
+ strided access. */
+ data_reference *dr = STMT_VINFO_DATA_REF (load_info);
+ if (!tree_int_cst_equal (DR_STEP (dr), TYPE_SIZE_UNIT (rhs_type)))
+ return NULL;
+
+ tree result_vectype = get_vectype_for_scalar_type (vinfo, lhs_type);
+ if (!result_vectype)
+ return NULL;
+
+ poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (result_vectype);
+ tree memory_vectype
+ = get_related_vectype_for_scalar_type (TYPE_MODE (result_vectype),
+ rhs_type, nunits);
+ if (!memory_vectype
+ || !direct_internal_fn_supported_p
+ (IFN_VEC_LOAD_EXTEND,
+ tree_pair (result_vectype, memory_vectype),
+ OPTIMIZE_FOR_BOTH))
+ return NULL;
+
+ //Build the new pattern statement.
+ //addr= address of the data reference.
+ //alignment= alignment of the data reference.
+ //type_operand= 0.
+ tree addr = build_fold_addr_expr (DR_REF (dr));
+ tree alias_type = reference_alias_ptr_type (DR_REF (dr));
+ tree alignment = build_int_cst (alias_type, TYPE_ALIGN (rhs_type));
+ tree type_operand = build_zero_cst (rhs_type);
+ gcall *pattern_stmt
+ = gimple_build_call_internal (IFN_VEC_LOAD_EXTEND, 3,
+ addr, alignment, type_operand);
+ //Set the left-hand side of the new pattern statement.
+ tree pattern_lhs = vect_recog_temp_ssa_var (lhs_type, NULL);
+ gimple_call_set_lhs (pattern_stmt, pattern_lhs);
+ gimple_set_location (pattern_stmt, gimple_location (convert));
+
+ //Add the new pattern statement to the vectorization info.
+ stmt_vec_info pattern_info = vinfo->add_stmt (pattern_stmt);
+ //Copy the data reference to the new pattern statement.
+ vinfo->move_dr (pattern_info, load_info);
+
+ *type_out = result_vectype;
+ vect_pattern_detected ("vect_recog_load_extend_pattern", convert);
+ return pattern_stmt;
+}
+
/* STMT_INFO is a load or store. If the load or store is conditional, return
the boolean condition under which it occurs, otherwise return null. */
@@ -8034,6 +8211,19 @@ vect_pattern_recog (vec_info *vinfo)
}
}
+ /* Extending loads must be recognized after every other pattern, since
+ replacing the conversion would hide it from widening patterns such as
+ widen_mult, dot_prod and sad that consume it. */
+ const vect_recog_func load_extend_recog
+ = { vect_recog_load_extend_pattern, "load_extend" };
+ for (unsigned i = 0; i < nbbs; ++i)
+ for (auto si = gsi_start_bb (bbs[i]); !gsi_end_p (si); gsi_next (&si))
+ {
+ stmt_vec_info stmt_info = vinfo->lookup_stmt (gsi_stmt (si));
+ if (stmt_info && STMT_VINFO_VECTORIZABLE (stmt_info))
+ vect_pattern_recog_1 (vinfo, load_extend_recog, stmt_info);
+ }
+
/* After this no more add_stmt calls are allowed. */
vinfo->stmt_vec_info_ro = true;
}
diff --git a/gcc/tree-vect-slp.cc b/gcc/tree-vect-slp.cc
index ae73837a76d..92ecb175a90 100644
--- a/gcc/tree-vect-slp.cc
+++ b/gcc/tree-vect-slp.cc
@@ -1226,7 +1226,8 @@ vect_build_slp_tree_3 (vec_info *vinfo,
vec<stmt_vec_info> stmts,
else
rhs_code = CALL_EXPR;
- if (cfn == CFN_GATHER_LOAD
+ if (cfn == CFN_VEC_LOAD_EXTEND
+ || cfn == CFN_GATHER_LOAD
|| cfn == CFN_SCATTER_STORE)
ldst_p = true;
else if (cfn == CFN_MASK_LOAD
@@ -2276,7 +2277,8 @@ vect_build_slp_tree_2 (vec_info *vinfo, slp_tree node,
load_permutation.quick_push (load_place);
}
- if (gcall *stmt = dyn_cast <gcall *> (stmt_info->stmt))
+ gcall *stmt = dyn_cast <gcall *> (stmt_info->stmt);
+ if (stmt && !gimple_call_internal_p (stmt, IFN_VEC_LOAD_EXTEND))
{
gcc_assert (gimple_call_internal_p (stmt, IFN_MASK_LOAD));
bool has_gaps = false;
@@ -8758,6 +8760,22 @@ vect_update_slp_vf_for_node (slp_tree node, poly_uint64
&vf,
SLP_TREE_LANES (node));
vf = force_common_multiple (vf, node_vf);
+ /* An extending load must not lower the vectorization factor below what
+ the load of the narrower memory type would have required. */
+ stmt_vec_info rep = SLP_TREE_REPRESENTATIVE (node);
+ if (gcall *call = rep ? dyn_cast <gcall *> (rep->stmt) : NULL)
+ if (gimple_call_internal_p (call, IFN_VEC_LOAD_EXTEND))
+ {
+ tree memory_type = TREE_TYPE (gimple_call_arg (call, 2));
+ if (tree memory_vectype
+ = get_related_vectype_for_scalar_type (TYPE_MODE (vectype),
+ memory_type))
+ vf = force_common_multiple
+ (vf, calculate_unrolling_factor
+ (TYPE_VECTOR_SUBPARTS (memory_vectype),
+ SLP_TREE_LANES (node)));
+ }
+
/* For permute nodes that are fed from externs or constants we have to
consider their number of lanes as well. Likewise for store-lanes. */
if (SLP_TREE_PERMUTE_P (node) || node->ldst_lanes)
diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
index cd2300cd30d..4dc5929cdd0 100644
--- a/gcc/tree-vect-stmts.cc
+++ b/gcc/tree-vect-stmts.cc
@@ -9838,6 +9838,7 @@ vectorizable_load (vec_info *vinfo,
tree mask_vectype = NULL_TREE;
tree els = NULL_TREE; tree els_vectype = NULL_TREE;
+ bool load_extend_p = false;
int mask_index = -1;
int els_index = -1;
@@ -9869,6 +9870,7 @@ vectorizable_load (vec_info *vinfo,
internal_fn ifn = gimple_call_internal_fn (call);
if (!internal_load_fn_p (ifn))
return false;
+ load_extend_p = ifn == IFN_VEC_LOAD_EXTEND;
scalar_dest = gimple_call_lhs (call);
if (!scalar_dest)
@@ -9969,6 +9971,14 @@ vectorizable_load (vec_info *vinfo,
const poly_int64 poffset = ls.poffset;
const vec<int> &elsvals = ls.elsvals;
+ if (load_extend_p)
+ {
+ if (memory_access_type != VMAT_CONTIGUOUS)
+ return false;
+ if (cost_vec)
+ vinfo->any_known_not_updated_vssa = true;
+ }
+
int maskload_elsval = 0;
bool need_zeroing = false;
@@ -10037,7 +10047,11 @@ vectorizable_load (vec_info *vinfo,
if (costing_p) /* transformation not required. */
{
- if (loop_vinfo
+ /* IFN_VEC_LOAD_EXTEND has no masked or length-controlled form;
+ vect_recog_load_extend_pattern is not applied to loops that may
+ use partial vectors. */
+ if (!load_extend_p
+ && loop_vinfo
&& LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
check_load_store_for_partial_vectors (loop_vinfo, vectype, slp_node,
VLS_LOAD, group_size, &ls,
@@ -10826,6 +10840,49 @@ vectorizable_load (vec_info *vinfo,
tree vec_mask = NULL_TREE;
tree vec_els = NULL_TREE;
+ if (load_extend_p && !costing_p)
+ {
+ gcc_assert (!LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo));
+
+ tree memory_type
+ = TREE_TYPE (gimple_call_arg (as_a <gcall *> (stmt_info->stmt), 2));
+ tree load_extend_memory_vectype
+ = get_related_vectype_for_scalar_type (TYPE_MODE (vectype),
+ memory_type, nunits);
+ dr_increment = vect_get_data_ptr_step (vinfo, dr_info,
+ memory_access_type);
+ dr_bump = vect_get_data_ptr_bump (vinfo, dr_info,
+ load_extend_memory_vectype,
+ memory_access_type);
+ dataref_ptr
+ = vect_create_data_ref_ptr (vinfo, stmt_info,
+ load_extend_memory_vectype, at_loop,
+ offset, &dummy, gsi, NULL, false,
+ dr_increment);
+
+ unsigned int align
+ = vect_known_alignment_in_bytes (dr_info, load_extend_memory_vectype);
+ tree alignment = build_int_cst (ref_type, align * BITS_PER_UNIT);
+ tree type_operand = build_zero_cst (load_extend_memory_vectype);
+
+ for (i = 0; i < vec_num; ++i)
+ {
+ if (i > 0)
+ dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, gsi,
+ stmt_info, dr_bump);
+
+ gcall *call
+ = gimple_build_call_internal (IFN_VEC_LOAD_EXTEND, 3,
+ dataref_ptr, alignment, type_operand);
+ tree lhs = make_ssa_name (vectype);
+ gimple_call_set_lhs (call, lhs);
+ gimple_call_set_nothrow (call, true);
+ vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
+ slp_node->push_vec_def (call);
+ }
+ return true;
+ }
+
if (memory_access_type == VMAT_LOAD_STORE_LANES)
{
const internal_fn lanes_ifn = ls.lanes_ifn;
--
2.34.1
