This patch adds a new internal function IFN_VEC_LOAD_EXTEND with its
optab, and the x86 sse.md patterns that implement it.
Nothing generates the internal function yet, so this patch does not
change code generation.
gcc/ChangeLog:
* config/i386/sse.md (vec_widen_<s>load<load_ext_src_lower><mode>): New
expander.
(vec_widen_<s>load<load_ext_qi_src_lower><mode>): New expander.
* internal-fn.cc (load_extend_direct): New direct optab.
(expand_load_extend_optab_fn): New expander.
(direct_load_extend_optab_supported_p): New supported optab.
(internal_load_fn_p): New internal load function.
* internal-fn.def (VEC_LOAD_EXTEND): New internal load function.
* optabs.def (OPTAB_CD): New optab.
---
gcc/config/i386/sse.md | 30 ++++++++++++++++++++++++++++++
gcc/internal-fn.cc | 23 +++++++++++++++++++++++
gcc/internal-fn.def | 3 +++
gcc/optabs.def | 2 ++
4 files changed, 58 insertions(+)
diff --git a/gcc/config/i386/sse.md b/gcc/config/i386/sse.md
index 3a445c9746d..8c40e523d2f 100644
--- a/gcc/config/i386/sse.md
+++ b/gcc/config/i386/sse.md
@@ -24978,6 +24978,36 @@
(set_attr "prefix" "orig,orig,vex")
(set_attr "mode" "TI")])
+;; Same-lane vector extending loads. SImode vectors can be loaded from
+;; both QImode and HImode elements, so the 2x and 4x widenings need
+;; separate expanders.
+(define_mode_iterator VI24_LOAD_EXT
+ [V8HI (V16HI "TARGET_AVX2") (V32HI "TARGET_AVX512BW")
+ V4SI (V8SI "TARGET_AVX2") (V16SI "TARGET_AVX512F")])
+(define_mode_attr load_ext_src
+ [(V8HI "V8QI") (V16HI "V16QI") (V32HI "V32QI")
+ (V4SI "V4HI") (V8SI "V8HI") (V16SI "V16HI")])
+(define_mode_attr load_ext_src_lower
+ [(V8HI "v8qi") (V16HI "v16qi") (V32HI "v32qi")
+ (V4SI "v4hi") (V8SI "v8hi") (V16SI "v16hi")])
+
+(define_expand "vec_widen_<s>load<load_ext_src_lower><mode>"
+ [(set (match_operand:VI24_LOAD_EXT 0 "register_operand")
+ (any_extend:VI24_LOAD_EXT
+ (match_operand:<load_ext_src> 1 "memory_operand")))]
+ "TARGET_SSE4_1")
+
+(define_mode_attr load_ext_qi_src
+ [(V4SI "V4QI") (V8SI "V8QI") (V16SI "V16QI")])
+(define_mode_attr load_ext_qi_src_lower
+ [(V4SI "v4qi") (V8SI "v8qi") (V16SI "v16qi")])
+
+(define_expand "vec_widen_<s>load<load_ext_qi_src_lower><mode>"
+ [(set (match_operand:VI4_AVX512F 0 "register_operand")
+ (any_extend:VI4_AVX512F
+ (match_operand:<load_ext_qi_src> 1 "memory_operand")))]
+ "TARGET_SSE4_1")
+
(define_insn "avx2_<code>v16qiv16hi2<mask_name>"
[(set (match_operand:V16HI 0 "register_operand" "=Yw")
(any_extend:V16HI
diff --git a/gcc/internal-fn.cc b/gcc/internal-fn.cc
index d277a07eaf7..3f117d1bbe2 100644
--- a/gcc/internal-fn.cc
+++ b/gcc/internal-fn.cc
@@ -158,6 +158,7 @@ init_internal_fns ()
direct_internal_fn. */
#define not_direct { -2, -2, false, 0, 0 }
#define mask_load_direct { -1, 2, false, 4, 4 }
+#define load_extend_direct { -1, 2, false, 3, 3 }
#define load_lanes_direct { -1, -1, false, 1, 1 }
#define mask_load_lanes_direct { -1, -1, false, 4, 4 }
#define mask_len_load_lanes_direct { -1, -1, false, 6, 6 }
@@ -3223,6 +3224,26 @@ expand_partial_load_optab_fn (internal_fn ifn, gcall
*stmt, convert_optab optab)
assign_call_lhs (lhs, target, &ops[0]);
}
+/* Expand VEC_LOAD_EXTEND call STMT using OPTAB. */
+
+static void
+expand_load_extend_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
+{
+ class expand_operand ops[2];
+ tree lhs = gimple_call_lhs (stmt);
+ tree result_type = TREE_TYPE (lhs);
+ tree memory_type = TREE_TYPE (gimple_call_arg (stmt, 2));
+ tree rhs = expand_call_mem_ref (memory_type, stmt, 0);
+ rtx mem = expand_expr (rhs, NULL_RTX, VOIDmode, EXPAND_MEMORY);
+ rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
+ create_call_lhs_operand (&ops[0], target, TYPE_MODE (result_type));
+ create_fixed_operand (&ops[1], mem);
+ expand_insn (convert_optab_handler (optab, TYPE_MODE (result_type),
+ TYPE_MODE (memory_type)),
+ 2, ops);
+ assign_call_lhs (lhs, target, &ops[0]);
+}
+
#define expand_mask_load_optab_fn expand_partial_load_optab_fn
#define expand_mask_load_lanes_optab_fn expand_mask_load_optab_fn
#define expand_mask_len_load_lanes_optab_fn expand_mask_load_optab_fn
@@ -4485,6 +4506,7 @@ multi_vector_optab_supported_p (convert_optab optab,
tree_pair types,
#define direct_cond_len_ternary_optab_supported_p direct_optab_supported_p
#define direct_crc_optab_supported_p convert_optab_supported_p
#define direct_mask_load_optab_supported_p convert_optab_supported_p
+#define direct_load_extend_optab_supported_p convert_optab_supported_p
#define direct_load_lanes_optab_supported_p multi_vector_optab_supported_p
#define direct_mask_load_lanes_optab_supported_p multi_vector_optab_supported_p
#define direct_mask_len_load_lanes_optab_supported_p \
@@ -5215,6 +5237,7 @@ internal_load_fn_p (internal_fn fn)
switch (fn)
{
case IFN_MASK_LOAD:
+ case IFN_VEC_LOAD_EXTEND:
case IFN_LOAD_LANES:
case IFN_MASK_LOAD_LANES:
case IFN_MASK_LEN_LOAD_LANES:
diff --git a/gcc/internal-fn.def b/gcc/internal-fn.def
index 7626836ba64..68108865a35 100644
--- a/gcc/internal-fn.def
+++ b/gcc/internal-fn.def
@@ -91,6 +91,7 @@ along with GCC; see the file COPYING3. If not see
- fold_left: for scalar = FN (scalar, vector), keyed off the vector mode
- check_ptrs: used for check_{raw,war}_ptrs
+ - load_extend: an extending load, keyed off the result and memory modes
- cond_len_unary: a conditional unary optab, such as cond_len_neg<mode>
- cond_len_binary: a conditional binary optab, such as cond_len_add<mode>
@@ -232,6 +233,8 @@ along with GCC; see the file COPYING3. If not see
DEF_INTERNAL_OPTAB_FN (CRC, ECF_CONST | ECF_NOTHROW, crc, crc)
DEF_INTERNAL_OPTAB_FN (CRC_REV, ECF_CONST | ECF_NOTHROW, crc_rev, crc)
DEF_INTERNAL_OPTAB_FN (MASK_LOAD, ECF_PURE, maskload, mask_load)
+DEF_INTERNAL_SIGNED_OPTAB_FN (VEC_LOAD_EXTEND, ECF_PURE, second,
+ vec_widen_sload, vec_widen_uload, load_extend)
DEF_INTERNAL_OPTAB_FN (LOAD_LANES, ECF_CONST, vec_load_lanes, load_lanes)
DEF_INTERNAL_OPTAB_FN (MASK_LOAD_LANES, ECF_PURE,
vec_mask_load_lanes, mask_load_lanes)
diff --git a/gcc/optabs.def b/gcc/optabs.def
index d6806a6156a..0bf127e1e89 100644
--- a/gcc/optabs.def
+++ b/gcc/optabs.def
@@ -95,6 +95,8 @@ OPTAB_CD(vec_mask_load_lanes_optab, "vec_mask_load_lanes$a$b")
OPTAB_CD(vec_mask_store_lanes_optab, "vec_mask_store_lanes$a$b")
OPTAB_CD(vec_mask_len_load_lanes_optab, "vec_mask_len_load_lanes$a$b")
OPTAB_CD(vec_mask_len_store_lanes_optab, "vec_mask_len_store_lanes$a$b")
+OPTAB_CD(vec_widen_sload_optab, "vec_widen_sload$b$a")
+OPTAB_CD(vec_widen_uload_optab, "vec_widen_uload$b$a")
OPTAB_CD(vcond_mask_optab, "vcond_mask_$a$b")
OPTAB_CD(vec_cmp_optab, "vec_cmp$a$b")
OPTAB_CD(vec_cmpu_optab, "vec_cmpu$a$b")
--
2.34.1
