From 86dad266052886a93f6e8d8148e521d2475aea8a Mon Sep 17 00:00:00 2001 From: Joel Hutton Date: Fri, 27 Aug 2021 10:21:28 +0100 Subject: [PATCH 2/3] [vect-patterns] Refactor widen_plus as internal_fn This patch replaces the existing tree_code widen_plus and widen_minus patterns with internal_fn versions. gcc/ChangeLog: * internal-fn.c (DEF_INTERNAL_OPTAB_MULTI_FN): Macro to define hi/lo widening dunctions. (lookup_multi_ifn_optab): Function to match hi/lo internal functions to optabs. (lookup_multi_internal_fn): Function to return the hi/lo internal functions. (first_commutative_argument): Add widening ifns. * internal-fn.def (DEF_INTERNAL_OPTAB_MULTI_FN): Macro to define hi/lo widening dunctions. (VEC_WIDEN_PLUS): Define widening plus function. (VEC_WIDEN_MINUS):Define widening minus function. * internal-fn.h (lookup_multi_ifn_optab): Declaration. (lookup_multi_internal_fn) Declaration: * optabs.c (commutative_optab_p): Add widening cases. * optabs.def (OPTAB_CD): Widening optabs. * tree-core.h (ECF_WIDEN): Internal function widening flag. (ECF_MULTI): Internal function hi/lo split flag. * tree-vect-patterns.c (vect_recog_widen_op_pattern): Support internal_fns with a hi/lo split. (vect_recog_widen_plus_pattern): Change to internal_fn. (vect_recog_widen_minus_pattern): Change to internal_fn. * tree-vect-stmts.c (vectorizable_conversion): Add cases for widening functions. (supportable_widening_operation): Add cases for widening functions. gcc/testsuite/ChangeLog: * gcc.target/aarch64/vect-widen-add.c: Test widen add patterns are substituted in gimple. * gcc.target/aarch64/vect-widen-sub.c: Test widen sub patterns are substituted in gimple. --- gcc/internal-fn.c | 94 +++++++++++++++++++ gcc/internal-fn.def | 19 ++++ gcc/internal-fn.h | 6 ++ gcc/optabs.c | 26 ++++- gcc/optabs.def | 2 + .../gcc.target/aarch64/vect-widen-add.c | 4 +- .../gcc.target/aarch64/vect-widen-sub.c | 4 +- gcc/tree-core.h | 6 ++ gcc/tree-vect-patterns.c | 35 +++++-- gcc/tree-vect-stmts.c | 79 +++++++++++----- 10 files changed, 238 insertions(+), 37 deletions(-) diff --git a/gcc/internal-fn.c b/gcc/internal-fn.c index e8fd16b9c21..4ab80d24b89 100644 --- a/gcc/internal-fn.c +++ b/gcc/internal-fn.c @@ -52,6 +52,7 @@ along with GCC; see the file COPYING3. If not see #include "explow.h" #include "rtl-iter.h" #include "gimple-range.h" +#include /* For lang_hooks.types.type_for_mode. */ #include "langhooks.h" @@ -70,6 +71,26 @@ const int internal_fn_flags_array[] = { 0 }; +const enum internal_fn internal_fn_hilo_keys_array[] = { +#undef DEF_INTERNAL_OPTAB_MULTI_FN +#define DEF_INTERNAL_OPTAB_MULTI_FN(NAME, FLAGS, OPTAB, SOPTAB, UOPTAB, TYPE) \ + IFN_##NAME##_LO, \ + IFN_##NAME##_HI, +#include "internal-fn.def" + IFN_LAST +#undef DEF_INTERNAL_OPTAB_MULTI_FN +}; + +const optab internal_fn_hilo_values_array[] = { +#undef DEF_INTERNAL_OPTAB_MULTI_FN +#define DEF_INTERNAL_OPTAB_MULTI_FN(NAME, FLAGS, OPTAB, SOPTAB, UOPTAB, TYPE) \ + SOPTAB##_lo_optab, UOPTAB##_lo_optab, \ + SOPTAB##_hi_optab, UOPTAB##_hi_optab, +#include "internal-fn.def" + unknown_optab, unknown_optab +#undef DEF_INTERNAL_OPTAB_MULTI_FN +}; + /* Return the internal function called NAME, or IFN_LAST if there's no such function. */ @@ -90,6 +111,49 @@ lookup_internal_fn (const char *name) return entry ? *entry : IFN_LAST; } +/* Return the optab belonging to the given internal function NAME for the given + SIGN or unknown_optab. */ + +optab +lookup_multi_ifn_optab (enum internal_fn fn, unsigned sign) +{ + typedef std::pair ifn_pair; + typedef std::map fn_to_optab_map_type; + static fn_to_optab_map_type *fn_to_optab_map; + + if (!fn_to_optab_map) + { + unsigned num + = sizeof (internal_fn_hilo_keys_array) / sizeof (enum internal_fn); + fn_to_optab_map = new fn_to_optab_map_type (); + for (unsigned int i = 0; i < num - 1; ++i) + { + enum internal_fn fn = internal_fn_hilo_keys_array[i]; + optab v1 = internal_fn_hilo_values_array[2*i]; + optab v2 = internal_fn_hilo_values_array[2*i + 1]; + ifn_pair key1 (fn, 0); + fn_to_optab_map->insert ({key1, v1}); + ifn_pair key2 (fn, 1); + fn_to_optab_map->insert ({key2, v2}); + } + } + + ifn_pair new_pair (fn, sign ? 1 : 0); + auto entry = fn_to_optab_map->find (new_pair); + return entry != fn_to_optab_map->end () ? entry->second : unknown_optab; +} + +extern void +lookup_multi_internal_fn (enum internal_fn ifn, enum internal_fn *lo, + enum internal_fn *hi) +{ + int ecf_flags = internal_fn_flags (ifn); + gcc_assert (ecf_flags & ECF_MULTI); + + *lo = internal_fn (ifn + 1); + *hi = internal_fn (ifn + 2); +} + /* Fnspec of each internal function, indexed by function number. */ const_tree internal_fn_fnspec_array[IFN_LAST + 1]; @@ -3849,6 +3913,9 @@ first_commutative_argument (internal_fn fn) case IFN_COND_FMS: case IFN_COND_FNMA: case IFN_COND_FNMS: + case IFN_VEC_WIDEN_PLUS: + case IFN_VEC_WIDEN_PLUS_LO: + case IFN_VEC_WIDEN_PLUS_HI: return 1; default: @@ -3868,6 +3935,32 @@ set_edom_supported_p (void) #endif } +#undef DEF_INTERNAL_OPTAB_MULTI_FN +#define DEF_INTERNAL_OPTAB_MULTI_FN(CODE, FLAGS, OPTAB, SOPTAB, UOPTAB, TYPE) \ + static void \ + expand_##CODE (internal_fn, gcall *) \ + { \ + gcc_unreachable (); \ + } \ + static void \ + expand_##CODE##_LO (internal_fn fn, gcall *stmt) \ + { \ + tree ty = TREE_TYPE (gimple_get_lhs (stmt)); \ + if (!TYPE_UNSIGNED (ty)) \ + expand_##TYPE##_optab_fn (fn, stmt, SOPTAB##_lo##_optab); \ + else \ + expand_##TYPE##_optab_fn (fn, stmt, UOPTAB##_lo##_optab); \ + } \ + static void \ + expand_##CODE##_HI (internal_fn fn, gcall *stmt) \ + { \ + tree ty = TREE_TYPE (gimple_get_lhs (stmt)); \ + if (!TYPE_UNSIGNED (ty)) \ + expand_##TYPE##_optab_fn (fn, stmt, SOPTAB##_hi##_optab); \ + else \ + expand_##TYPE##_optab_fn (fn, stmt, UOPTAB##_hi##_optab); \ + } + #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \ static void \ expand_##CODE (internal_fn fn, gcall *stmt) \ @@ -3884,6 +3977,7 @@ set_edom_supported_p (void) expand_##TYPE##_optab_fn (fn, stmt, which_optab); \ } #include "internal-fn.def" +#undef DEF_INTERNAL_OPTAB_MULTI_FN /* Routines to expand each internal function, indexed by function number. Each routine has the prototype: diff --git a/gcc/internal-fn.def b/gcc/internal-fn.def index bb13c6cce1b..d8aef4c7012 100644 --- a/gcc/internal-fn.def +++ b/gcc/internal-fn.def @@ -82,6 +82,12 @@ along with GCC; see the file COPYING3. If not see says that the function extends the C-level BUILT_IN_{,L,LL,IMAX} group of functions to any integral mode (including vector modes). + DEF_INTERNAL_OPTAB_MULTI_FN is like DEF_INTERNAL_OPTAB_FN except it + provides convenience wrappers for defining conversions that require a + hi/lo split, like widening and narrowing operations. Each definition + for will require an optab named and two other optabs that + you specify for signed and unsigned. + Each entry must have a corresponding expander of the form: void expand_NAME (gimple_call stmt) @@ -120,6 +126,13 @@ along with GCC; see the file COPYING3. If not see DEF_INTERNAL_OPTAB_FN (NAME, FLAGS, OPTAB, TYPE) #endif +#ifndef DEF_INTERNAL_OPTAB_MULTI_FN +#define DEF_INTERNAL_OPTAB_MULTI_FN(NAME, FLAGS, OPTAB, SOPTAB, UOPTAB, TYPE) \ + DEF_INTERNAL_OPTAB_FN (NAME, FLAGS | ECF_MULTI, OPTAB, TYPE) \ + DEF_INTERNAL_OPTAB_FN (NAME ## _LO, FLAGS, unknown, TYPE) \ + DEF_INTERNAL_OPTAB_FN (NAME ## _HI, FLAGS, unknown, TYPE) +#endif + DEF_INTERNAL_OPTAB_FN (MASK_LOAD, ECF_PURE, maskload, mask_load) DEF_INTERNAL_OPTAB_FN (LOAD_LANES, ECF_CONST, vec_load_lanes, load_lanes) DEF_INTERNAL_OPTAB_FN (MASK_LOAD_LANES, ECF_PURE, @@ -286,6 +299,12 @@ DEF_INTERNAL_OPTAB_FN (COMPLEX_ADD_ROT270, ECF_CONST, cadd270, binary) DEF_INTERNAL_OPTAB_FN (COMPLEX_MUL, ECF_CONST, cmul, binary) DEF_INTERNAL_OPTAB_FN (COMPLEX_MUL_CONJ, ECF_CONST, cmul_conj, binary) DEF_INTERNAL_OPTAB_FN (VEC_ADDSUB, ECF_CONST, vec_addsub, binary) +DEF_INTERNAL_OPTAB_MULTI_FN (VEC_WIDEN_PLUS, ECF_CONST | ECF_WIDEN | ECF_NOTHROW, + vec_widen_add, vec_widen_saddl, vec_widen_uaddl, + binary) +DEF_INTERNAL_OPTAB_MULTI_FN (VEC_WIDEN_MINUS, ECF_CONST | ECF_WIDEN | ECF_NOTHROW, + vec_widen_sub, vec_widen_ssubl, vec_widen_usubl, + binary) DEF_INTERNAL_OPTAB_FN (VEC_FMADDSUB, ECF_CONST, vec_fmaddsub, ternary) DEF_INTERNAL_OPTAB_FN (VEC_FMSUBADD, ECF_CONST, vec_fmsubadd, ternary) diff --git a/gcc/internal-fn.h b/gcc/internal-fn.h index 06e540ffd6a..406fb52bced 100644 --- a/gcc/internal-fn.h +++ b/gcc/internal-fn.h @@ -20,6 +20,9 @@ along with GCC; see the file COPYING3. If not see #ifndef GCC_INTERNAL_FN_H #define GCC_INTERNAL_FN_H +#include "insn-codes.h" +#include "insn-opinit.h" + /* INTEGER_CST values for IFN_UNIQUE function arg-0. UNSPEC: Undifferentiated UNIQUE. @@ -113,6 +116,9 @@ internal_fn_name (enum internal_fn fn) } extern internal_fn lookup_internal_fn (const char *); +extern optab lookup_multi_ifn_optab (enum internal_fn, unsigned); +extern void lookup_multi_internal_fn (enum internal_fn, enum internal_fn *, + enum internal_fn *); /* Return the ECF_* flags for function FN. */ diff --git a/gcc/optabs.c b/gcc/optabs.c index 019bbb62882..065fa4510c3 100644 --- a/gcc/optabs.c +++ b/gcc/optabs.c @@ -1303,7 +1303,31 @@ commutative_optab_p (optab binoptab) || binoptab == smul_widen_optab || binoptab == umul_widen_optab || binoptab == smul_highpart_optab - || binoptab == umul_highpart_optab); + || binoptab == umul_highpart_optab + || binoptab == and_optab + || binoptab == ior_optab + || binoptab == xor_optab + || binoptab == smin_optab + || binoptab == smax_optab + || binoptab == umin_optab + || binoptab == umax_optab + || binoptab == add_optab + || binoptab == vec_widen_add_optab + || binoptab == vec_widen_sub_optab + || binoptab == vec_widen_smult_hi_optab + || binoptab == vec_widen_smult_lo_optab + || binoptab == vec_widen_saddl_hi_optab + || binoptab == vec_widen_saddl_lo_optab + || binoptab == vec_widen_ssubl_hi_optab + || binoptab == vec_widen_ssubl_lo_optab + || binoptab == vec_widen_umult_hi_optab + || binoptab == vec_widen_umult_lo_optab + || binoptab == vec_widen_umult_odd_optab + || binoptab == vec_widen_uaddl_hi_optab + || binoptab == vec_widen_uaddl_lo_optab + || binoptab == vec_widen_usubl_hi_optab + || binoptab == vec_widen_usubl_lo_optab + ); } /* X is to be used in mode MODE as operand OPN to BINOPTAB. If we're diff --git a/gcc/optabs.def b/gcc/optabs.def index b889ad2e5a0..433ef3d5edc 100644 --- a/gcc/optabs.def +++ b/gcc/optabs.def @@ -78,6 +78,8 @@ OPTAB_CD(smsub_widen_optab, "msub$b$a4") OPTAB_CD(umsub_widen_optab, "umsub$b$a4") OPTAB_CD(ssmsub_widen_optab, "ssmsub$b$a4") OPTAB_CD(usmsub_widen_optab, "usmsub$a$b4") +OPTAB_CD(vec_widen_add_optab, "add$a$b3") +OPTAB_CD(vec_widen_sub_optab, "sub$a$b3") OPTAB_CD(vec_load_lanes_optab, "vec_load_lanes$a$b") OPTAB_CD(vec_store_lanes_optab, "vec_store_lanes$a$b") OPTAB_CD(vec_mask_load_lanes_optab, "vec_mask_load_lanes$a$b") diff --git a/gcc/testsuite/gcc.target/aarch64/vect-widen-add.c b/gcc/testsuite/gcc.target/aarch64/vect-widen-add.c index 220bd9352a4..7037673d32b 100644 --- a/gcc/testsuite/gcc.target/aarch64/vect-widen-add.c +++ b/gcc/testsuite/gcc.target/aarch64/vect-widen-add.c @@ -1,5 +1,5 @@ /* { dg-do run } */ -/* { dg-options "-O3 -save-temps" } */ +/* { dg-options "-O3 -save-temps -fdump-tree-vect-all" } */ #include #include @@ -86,6 +86,8 @@ main() return 0; } +/* { dg-final { scan-tree-dump "add new stmt.*VEC_WIDEN_PLUS_LO" "vect" } } */ +/* { dg-final { scan-tree-dump "add new stmt.*VEC_WIDEN_PLUS_HI" "vect" } } */ /* { dg-final { scan-assembler-times {\tuaddl\t} 1} } */ /* { dg-final { scan-assembler-times {\tuaddl2\t} 1} } */ /* { dg-final { scan-assembler-times {\tsaddl\t} 1} } */ diff --git a/gcc/testsuite/gcc.target/aarch64/vect-widen-sub.c b/gcc/testsuite/gcc.target/aarch64/vect-widen-sub.c index a2bed63affb..83bc1edb610 100644 --- a/gcc/testsuite/gcc.target/aarch64/vect-widen-sub.c +++ b/gcc/testsuite/gcc.target/aarch64/vect-widen-sub.c @@ -1,5 +1,5 @@ /* { dg-do run } */ -/* { dg-options "-O3 -save-temps" } */ +/* { dg-options "-O3 -save-temps -fdump-tree-vect-all" } */ #include #include @@ -86,6 +86,8 @@ main() return 0; } +/* { dg-final { scan-tree-dump "add new stmt.*VEC_WIDEN_MINUS_LO" "vect" } } */ +/* { dg-final { scan-tree-dump "add new stmt.*VEC_WIDEN_MINUS_HI" "vect" } } */ /* { dg-final { scan-assembler-times {\tusubl\t} 1} } */ /* { dg-final { scan-assembler-times {\tusubl2\t} 1} } */ /* { dg-final { scan-assembler-times {\tssubl\t} 1} } */ diff --git a/gcc/tree-core.h b/gcc/tree-core.h index 8ab119dc9a2..f8dd7925e91 100644 --- a/gcc/tree-core.h +++ b/gcc/tree-core.h @@ -96,6 +96,12 @@ struct die_struct; /* Nonzero if this is a cold function. */ #define ECF_COLD (1 << 15) +/* Nonzero if this is a widening function. */ +#define ECF_WIDEN (1 << 16) + +/* Nonzero if this is a function that decomposes into a lo/hi operation. */ +#define ECF_MULTI (1 << 17) + /* Call argument flags. */ /* Nonzero if the argument is not used by the function. */ diff --git a/gcc/tree-vect-patterns.c b/gcc/tree-vect-patterns.c index 4a8ea67e62f..7e586be7f6d 100644 --- a/gcc/tree-vect-patterns.c +++ b/gcc/tree-vect-patterns.c @@ -1246,14 +1246,15 @@ static gimple * vect_recog_widen_op_pattern (vec_info *vinfo, stmt_vec_info last_stmt_info, tree *type_out, tree_code orig_code, code_helper wide_code_or_ifn, - bool shift_p, const char *name) + bool shift_p, const char *name, + enum optab_subtype *subtype = NULL) { gimple *last_stmt = last_stmt_info->stmt; vect_unpromoted_value unprom[2]; tree half_type; if (!vect_widened_op_tree (vinfo, last_stmt_info, orig_code, orig_code, - shift_p, 2, unprom, &half_type)) + shift_p, 2, unprom, &half_type, subtype)) return NULL; /* Pattern detected. */ @@ -1329,6 +1330,19 @@ vect_recog_widen_op_pattern (vec_info *vinfo, type, pattern_stmt, vecctype); } +static gimple * +vect_recog_widen_op_pattern (vec_info *vinfo, + stmt_vec_info last_stmt_info, tree *type_out, + tree_code orig_code, internal_fn wide_ifn, + bool shift_p, const char *name, + enum optab_subtype *subtype = NULL) +{ + combined_fn ifn = as_combined_fn (wide_ifn); + return vect_recog_widen_op_pattern (vinfo, last_stmt_info, type_out, + orig_code, ifn, shift_p, name, + subtype); +} + /* Try to detect multiplication on widened inputs, converting MULT_EXPR to WIDEN_MULT_EXPR. See vect_recog_widen_op_pattern for details. */ @@ -1342,26 +1356,30 @@ vect_recog_widen_mult_pattern (vec_info *vinfo, stmt_vec_info last_stmt_info, } /* Try to detect addition on widened inputs, converting PLUS_EXPR - to WIDEN_PLUS_EXPR. See vect_recog_widen_op_pattern for details. */ + to IFN_VEC_WIDEN_PLUS. See vect_recog_widen_op_pattern for details. */ static gimple * vect_recog_widen_plus_pattern (vec_info *vinfo, stmt_vec_info last_stmt_info, tree *type_out) { + enum optab_subtype subtype; return vect_recog_widen_op_pattern (vinfo, last_stmt_info, type_out, - PLUS_EXPR, WIDEN_PLUS_EXPR, false, - "vect_recog_widen_plus_pattern"); + PLUS_EXPR, IFN_VEC_WIDEN_PLUS, + false, "vect_recog_widen_plus_pattern", + &subtype); } /* Try to detect subtraction on widened inputs, converting MINUS_EXPR - to WIDEN_MINUS_EXPR. See vect_recog_widen_op_pattern for details. */ + to IFN_VEC_WIDEN_MINUS. See vect_recog_widen_op_pattern for details. */ static gimple * vect_recog_widen_minus_pattern (vec_info *vinfo, stmt_vec_info last_stmt_info, tree *type_out) { + enum optab_subtype subtype; return vect_recog_widen_op_pattern (vinfo, last_stmt_info, type_out, - MINUS_EXPR, WIDEN_MINUS_EXPR, false, - "vect_recog_widen_minus_pattern"); + MINUS_EXPR, IFN_VEC_WIDEN_MINUS, + false, "vect_recog_widen_minus_pattern", + &subtype); } /* Function vect_recog_popcount_pattern @@ -5523,6 +5541,7 @@ static vect_recog_func vect_vect_recog_func_ptrs[] = { { vect_recog_mask_conversion_pattern, "mask_conversion" }, { vect_recog_widen_plus_pattern, "widen_plus" }, { vect_recog_widen_minus_pattern, "widen_minus" }, + /* These must come after the double widening ones. */ }; const unsigned int NUM_PATTERNS = ARRAY_SIZE (vect_vect_recog_func_ptrs); diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c index d5e1619fabc..3f25a5ee09a 100644 --- a/gcc/tree-vect-stmts.c +++ b/gcc/tree-vect-stmts.c @@ -4751,22 +4751,30 @@ vectorizable_conversion (vec_info *vinfo, if (!(is_gimple_assign (stmt) || is_gimple_call (stmt))) return false; + bool widen_arith = false; + if (is_gimple_assign (stmt)) - { - code_or_ifn = gimple_assign_rhs_code (stmt); - } + { + code_or_ifn = gimple_assign_rhs_code (stmt); + op_type = TREE_CODE_LENGTH ((tree_code) code_or_ifn); + widen_arith = (code_or_ifn == WIDEN_PLUS_EXPR + || code_or_ifn == WIDEN_MINUS_EXPR + || code_or_ifn == WIDEN_MULT_EXPR + || code_or_ifn == WIDEN_LSHIFT_EXPR); + } else - code_or_ifn = gimple_call_combined_fn (stmt); + { + code_or_ifn = gimple_call_combined_fn (stmt); + op_type = gimple_call_num_args (stmt); + widen_arith = gimple_call_flags (stmt) & ECF_WIDEN; + } - if (TREE_CODE (gimple_get_lhs (stmt)) != SSA_NAME) + if (!widen_arith + && !CONVERT_EXPR_CODE_P (code_or_ifn) + && code_or_ifn != FIX_TRUNC_EXPR + && code_or_ifn != FLOAT_EXPR) return false; - bool widen_arith = (code_or_ifn == WIDEN_PLUS_EXPR - || code_or_ifn == WIDEN_MINUS_EXPR - || code_or_ifn == WIDEN_MULT_EXPR - || code_or_ifn == WIDEN_LSHIFT_EXPR); - op_type = TREE_CODE_LENGTH ((tree_code) code_or_ifn); - /* Check types of lhs and rhs. */ scalar_dest = gimple_get_lhs (stmt); lhs_type = TREE_TYPE (scalar_dest); @@ -4784,8 +4792,7 @@ vectorizable_conversion (vec_info *vinfo, } rhs_type = TREE_TYPE (op0); - if ((code_or_ifn.is_tree_code () && code_or_ifn != FIX_TRUNC_EXPR - && code_or_ifn != FLOAT_EXPR) + if ((code_or_ifn != FIX_TRUNC_EXPR && code_or_ifn != FLOAT_EXPR) && !((INTEGRAL_TYPE_P (lhs_type) && INTEGRAL_TYPE_P (rhs_type)) || (SCALAR_FLOAT_TYPE_P (lhs_type) @@ -4810,7 +4817,8 @@ vectorizable_conversion (vec_info *vinfo, gcc_assert (code_or_ifn == WIDEN_MULT_EXPR || code_or_ifn == WIDEN_LSHIFT_EXPR || code_or_ifn == WIDEN_PLUS_EXPR - || code_or_ifn == WIDEN_MINUS_EXPR); + || code_or_ifn == WIDEN_MINUS_EXPR + || widen_arith); if (is_gimple_assign (stmt)) op1 = gimple_assign_rhs2 (stmt); @@ -11877,16 +11885,38 @@ supportable_widening_operation (vec_info *vinfo, gcc_unreachable (); } - switch (code_or_ifn.as_fn_code ()) + if ( code_or_ifn.is_tree_code ()) + { + *code_or_ifn1 = c1; + *code_or_ifn2 = c2; + if (BYTES_BIG_ENDIAN && c1 != VEC_WIDEN_MULT_EVEN_EXPR) + std::swap (c1, c2); + } + else { - case CFN_LAST: - break; - default: - gcc_unreachable (); - } + internal_fn ifn = as_internal_fn ((combined_fn) code_or_ifn); + int ecf_flags = internal_fn_flags (ifn); + gcc_assert (ecf_flags & ECF_MULTI); - if (BYTES_BIG_ENDIAN && c1 != VEC_WIDEN_MULT_EVEN_EXPR) - std::swap (c1, c2); + switch (code_or_ifn.as_fn_code ()) + { + case CFN_LAST: + break; + case CFN_VEC_WIDEN_PLUS: + break; + case CFN_VEC_WIDEN_MINUS: + break; + default: + gcc_unreachable (); + } + + internal_fn lo, hi; + lookup_multi_internal_fn (ifn, &lo, &hi); + *code_or_ifn1 = as_combined_fn (lo); + *code_or_ifn2 = as_combined_fn (hi); + optab1 = lookup_multi_ifn_optab (lo, !TYPE_UNSIGNED (vectype)); + optab2 = lookup_multi_ifn_optab (hi, !TYPE_UNSIGNED (vectype)); + } if (code_or_ifn == FIX_TRUNC_EXPR) { @@ -11905,7 +11935,7 @@ supportable_widening_operation (vec_info *vinfo, optab1 = vec_unpacks_sbool_lo_optab; optab2 = vec_unpacks_sbool_hi_optab; } - else + else if (code_or_ifn.is_tree_code ()) { optab1 = optab_for_tree_code (c1, vectype, optab_default); optab2 = optab_for_tree_code (c2, vectype, optab_default); @@ -11919,9 +11949,6 @@ supportable_widening_operation (vec_info *vinfo, || (icode2 = optab_handler (optab2, vec_mode)) == CODE_FOR_nothing) return false; - *code_or_ifn1 = c1; - *code_or_ifn2 = c2; - if (insn_data[icode1].operand[0].mode == TYPE_MODE (wide_vectype) && insn_data[icode2].operand[0].mode == TYPE_MODE (wide_vectype)) { -- 2.17.1