diff --git a/gcc/config/aarch64/aarch64-sve2.md b/gcc/config/aarch64/aarch64-sve2.md index 2334e5a7b7dc524bbd1f4d0a48ba5cd991970118..51783604ad8f83eb1d070c133009ed41a2a0252d 100644 --- a/gcc/config/aarch64/aarch64-sve2.md +++ b/gcc/config/aarch64/aarch64-sve2.md @@ -63,3 +63,89 @@ movprfx\t%0, %2\;h\t%0., %1/m, %0., %3." [(set_attr "movprfx" "*,yes")] ) + +;; Multiply long top / bottom +(define_insn "*mull" + [(set (match_operand: 0 "register_operand" "=w") + (unspec: [(match_operand:SVE_BHSI 1 "register_operand" "w") + (match_operand:SVE_BHSI 2 "register_operand" "w")] + MULLBT))] + "TARGET_SVE2" + "mull\t%0., %1., %2." +) + +(define_expand "mull" + [(set (match_operand: 0 "register_operand") + (unspec: [(match_operand:SVE_BHSI 1 "register_operand") + (match_operand:SVE_BHSI 2 "register_operand")] + MULLBT))] + "TARGET_SVE2" + "" +) + +;; (Rounding) Right shift narrow bottom +(define_insn "*shrnb" + [(set (match_operand:SVE_BHSI 0 "register_operand" "=w") + (unspec:SVE_BHSI [(match_operand: 1 "register_operand" "w") + (match_operand 2 "aarch64_simd_shift_imm_offset_" "i")] + SHRNB))] + "TARGET_SVE2" + "shrnb\t%0., %1., #%2" +) + +(define_expand "shrnb" + [(set (match_operand:SVE_BHSI 0 "register_operand") + (unspec:SVE_BHSI [(match_operand: 1 "register_operand") + (match_operand 2 "aarch64_simd_shift_imm_offset_")] + SHRNB))] + "TARGET_SVE2" + "" +) + +;; (Rounding) Right shift narrow top +(define_insn "*shrnt" + [(set (match_operand:SVE_BHSI 0 "register_operand" "=w") + (unspec:SVE_BHSI [(match_operand:SVE_BHSI 1 "register_operand" "%0") + (match_operand: 2 "register_operand" "w") + (match_operand 3 "aarch64_simd_shift_imm_offset_" "i")] + SHRNT))] + "TARGET_SVE2" + "shrnt\t%0., %2., #%3" +) + +(define_expand "shrnt" + [(set (match_operand:SVE_BHSI 0 "register_operand") + (unspec:SVE_BHSI [(match_operand: 1 "register_operand") + (match_operand: 2 "register_operand") + (match_operand 3 "aarch64_simd_shift_imm_offset_")] + SHRNT))] + "TARGET_SVE2" + "" +) + +;; Unpredicated integer multiply-high-with-(round-and-)scale +(define_expand "mulhs3" + [(set (match_operand:SVE_BHSI 0 "register_operand") + (unspec:SVE_BHSI + [(match_dup 3) + (unspec:SVE_BHSI [(match_operand:SVE_BHSI 1 "register_operand") + (match_operand:SVE_BHSI 2 "register_operand")] + MULHRS)] + UNSPEC_PRED_X))] + "TARGET_SVE2" + { + operands[3] = force_reg (mode, CONSTM1_RTX (mode)); + + rtx prod_b = gen_reg_rtx (mode); + rtx prod_t = gen_reg_rtx (mode); + emit_insn (gen_mullb (prod_b, operands[1], operands[2])); + emit_insn (gen_mullt (prod_t, operands[1], operands[2])); + + rtx shift = GEN_INT (GET_MODE_UNIT_BITSIZE (mode) - 1); + emit_insn (gen_shrnb (operands[0], prod_b, shift)); + emit_insn (gen_shrnt (operands[0], operands[0], prod_t, shift)); + + DONE; + } +) + diff --git a/gcc/config/aarch64/iterators.md b/gcc/config/aarch64/iterators.md index e8ba4f3d987b0a79d290db6f75b9638b867a5f0e..258b0a8cca6c1c4877fad1820da8ad3386bd8f98 100644 --- a/gcc/config/aarch64/iterators.md +++ b/gcc/config/aarch64/iterators.md @@ -375,6 +375,10 @@ UNSPEC_RSUBHN2 ; Used in aarch64-simd.md. UNSPEC_SQDMULH ; Used in aarch64-simd.md. UNSPEC_SQRDMULH ; Used in aarch64-simd.md. + UNSPEC_SMULLB ; Used in aarch64-sve2.md. + UNSPEC_SMULLT ; Used in aarch64-sve2.md. + UNSPEC_UMULLB ; Used in aarch64-sve2.md. + UNSPEC_UMULLT ; Used in aarch64-sve2.md. UNSPEC_PMUL ; Used in aarch64-simd.md. UNSPEC_FMULX ; Used in aarch64-simd.md. UNSPEC_USQADD ; Used in aarch64-simd.md. @@ -397,6 +401,10 @@ UNSPEC_UQSHRN ; Used in aarch64-simd.md. UNSPEC_SQRSHRN ; Used in aarch64-simd.md. UNSPEC_UQRSHRN ; Used in aarch64-simd.md. + UNSPEC_SHRNB ; Used in aarch64-sve2.md. + UNSPEC_SHRNT ; Used in aarch64-sve2.md. + UNSPEC_RSHRNB ; Used in aarch64-sve2.md. + UNSPEC_RSHRNT ; Used in aarch64-sve2.md. UNSPEC_SSHL ; Used in aarch64-simd.md. UNSPEC_USHL ; Used in aarch64-simd.md. UNSPEC_SRSHL ; Used in aarch64-simd.md. @@ -520,6 +528,10 @@ UNSPEC_FCMLA90 ; Used in aarch64-simd.md. UNSPEC_FCMLA180 ; Used in aarch64-simd.md. UNSPEC_FCMLA270 ; Used in aarch64-simd.md. + UNSPEC_SMULHS ; Used in aarch64-sve2.md. + UNSPEC_SMULHRS ; Used in aarch64-sve2.md. + UNSPEC_UMULHS ; Used in aarch64-sve2.md. + UNSPEC_UMULHRS ; Used in aarch64-sve2.md. ]) ;; ------------------------------------------------------------------ @@ -1585,6 +1597,13 @@ (define_int_iterator RHADD [UNSPEC_SRHADD UNSPEC_URHADD]) +(define_int_iterator MULLBT [UNSPEC_SMULLB UNSPEC_UMULLB + UNSPEC_SMULLT UNSPEC_UMULLT]) + +(define_int_iterator SHRNB [UNSPEC_SHRNB UNSPEC_RSHRNB]) + +(define_int_iterator SHRNT [UNSPEC_SHRNT UNSPEC_RSHRNT]) + (define_int_iterator DOTPROD [UNSPEC_SDOT UNSPEC_UDOT]) (define_int_iterator ADDSUBHN [UNSPEC_ADDHN UNSPEC_RADDHN @@ -1604,6 +1623,9 @@ (define_int_iterator VQDMULH [UNSPEC_SQDMULH UNSPEC_SQRDMULH]) +(define_int_iterator MULHRS [UNSPEC_SMULHS UNSPEC_UMULHS + UNSPEC_SMULHRS UNSPEC_UMULHRS]) + (define_int_iterator USSUQADD [UNSPEC_SUQADD UNSPEC_USQADD]) (define_int_iterator SUQMOVN [UNSPEC_SQXTN UNSPEC_UQXTN]) @@ -1866,7 +1888,11 @@ (UNSPEC_COND_FCVTZS "s") (UNSPEC_COND_FCVTZU "u") (UNSPEC_COND_SCVTF "s") - (UNSPEC_COND_UCVTF "u")]) + (UNSPEC_COND_UCVTF "u") + (UNSPEC_SMULLB "s") (UNSPEC_UMULLB "u") + (UNSPEC_SMULLT "s") (UNSPEC_UMULLT "u") + (UNSPEC_SMULHS "s") (UNSPEC_UMULHS "u") + (UNSPEC_SMULHRS "s") (UNSPEC_UMULHRS "u")]) (define_int_attr sur [(UNSPEC_SHADD "s") (UNSPEC_UHADD "u") (UNSPEC_SRHADD "sr") (UNSPEC_URHADD "ur") @@ -1904,6 +1930,10 @@ (UNSPEC_SQRSHRN "r") (UNSPEC_UQRSHRN "r") (UNSPEC_SQSHL "") (UNSPEC_UQSHL "") (UNSPEC_SQRSHL "r")(UNSPEC_UQRSHL "r") + (UNSPEC_SHRNB "") (UNSPEC_SHRNT "") + (UNSPEC_RSHRNB "r") (UNSPEC_RSHRNT "r") + (UNSPEC_SMULHS "") (UNSPEC_UMULHS "") + (UNSPEC_SMULHRS "r") (UNSPEC_UMULHRS "r") ]) (define_int_attr lr [(UNSPEC_SSLI "l") (UNSPEC_USLI "l") @@ -1916,6 +1946,9 @@ (UNSPEC_SHADD "") (UNSPEC_UHADD "u") (UNSPEC_SRHADD "") (UNSPEC_URHADD "u")]) +(define_int_attr bt [(UNSPEC_SMULLB "b") (UNSPEC_UMULLB "b") + (UNSPEC_SMULLT "t") (UNSPEC_UMULLT "t")]) + (define_int_attr addsub [(UNSPEC_SHADD "add") (UNSPEC_UHADD "add") (UNSPEC_SRHADD "add") diff --git a/gcc/internal-fn.def b/gcc/internal-fn.def index 9461693bcd12dd54eedd0d983758947bfc016b73..67eb30dee81b0a9b22459f3c1cb8e031b7ac32c9 100644 --- a/gcc/internal-fn.def +++ b/gcc/internal-fn.def @@ -149,6 +149,11 @@ DEF_INTERNAL_SIGNED_OPTAB_FN (AVG_FLOOR, ECF_CONST | ECF_NOTHROW, first, DEF_INTERNAL_SIGNED_OPTAB_FN (AVG_CEIL, ECF_CONST | ECF_NOTHROW, first, savg_ceil, uavg_ceil, binary) +DEF_INTERNAL_SIGNED_OPTAB_FN (MULHS, ECF_CONST | ECF_NOTHROW, first, + smulhs, umulhs, binary) +DEF_INTERNAL_SIGNED_OPTAB_FN (MULHRS, ECF_CONST | ECF_NOTHROW, first, + smulhrs, umulhrs, binary) + DEF_INTERNAL_OPTAB_FN (COND_ADD, ECF_CONST, cond_add, cond_binary) DEF_INTERNAL_OPTAB_FN (COND_SUB, ECF_CONST, cond_sub, cond_binary) DEF_INTERNAL_OPTAB_FN (COND_MUL, ECF_CONST, cond_smul, cond_binary) diff --git a/gcc/optabs.def b/gcc/optabs.def index 5283e6753f26eb0da08a393949128d09643e50dd..adcd9477b4c4b2e73cad7c72ee51336e98e81954 100644 --- a/gcc/optabs.def +++ b/gcc/optabs.def @@ -342,6 +342,10 @@ OPTAB_D (udot_prod_optab, "udot_prod$I$a") OPTAB_D (usum_widen_optab, "widen_usum$I$a3") OPTAB_D (usad_optab, "usad$I$a") OPTAB_D (ssad_optab, "ssad$I$a") +OPTAB_D (smulhs_optab, "smulhs$a3") +OPTAB_D (smulhrs_optab, "smulhrs$a3") +OPTAB_D (umulhs_optab, "umulhs$a3") +OPTAB_D (umulhrs_optab, "umulhrs$a3") OPTAB_D (vec_pack_sfix_trunc_optab, "vec_pack_sfix_trunc_$a") OPTAB_D (vec_pack_ssat_optab, "vec_pack_ssat_$a") OPTAB_D (vec_pack_trunc_optab, "vec_pack_trunc_$a") diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/mulhrs_1.c b/gcc/testsuite/gcc.target/aarch64/sve2/mulhrs_1.c new file mode 100644 index 0000000000000000000000000000000000000000..98cdc73d5256d712c985c541c1a585d081e5f5ea --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve2/mulhrs_1.c @@ -0,0 +1,123 @@ +/* { dg-do compile } */ +/* { dg-options "-O2 -ftree-vectorize -fdump-tree-vect-details --save-temps" } */ + +#include + +#define MULTHI(TYPE, BIGGER, RND) \ +TYPE __attribute__ ((noinline, noclone)) \ +mulhs_##TYPE##_##RND (TYPE *restrict x, \ + TYPE *restrict y, TYPE *restrict z, int n) \ +{ \ + for (int i = 0; i < n; i++) \ + { \ + z[i] = ((((BIGGER)x[i] * (BIGGER)y[i]) >> \ + (sizeof(BIGGER)/2-(RND+1)) + RND) >> 1; \ + } \ +} + +MULTHI (int8_t, int16_t, 0) +MULTHI (int16_t, int32_t, 0) +MULTHI (int32_t, int64_t, 0) + +MULTHI (uint8_t, uint16_t, 0) +MULTHI (uint16_t, uint32_t, 0) +MULTHI (uint32_t, uint64_t, 0) + +MULTHI (int8_t, int16_t, 1) +MULTHI (int16_t, int32_t, 1) +MULTHI (int32_t, int64_t, 1) + +MULTHI (uint8_t, uint16_t, 1) +MULTHI (uint16_t, uint32_t, 1) +MULTHI (uint32_t, uint64_t, 1) + +/* { dg-final { scan-tree-dump-times "vectorized 1 loops in function" 12 "vect" } } */ + +/* { dg-final { scan-assembler-times { + \tsmullb\tz[0-9]+\.h, z[0-9]+\.b, z[0-9]+\.b\n + \tsmullt\tz[0-9]+\.h, z[0-9]+\.b, z[0-9]+\.b\n + \tshrnb\tz[0-9]+\.b, z[0-9]+\.h, #7\n + \tshrnt\tz[0-9]+\.b, z[0-9]+\.h, #7\n + } + 1 } } */ +/* { dg-final { scan-assembler-times { + \tsmullb\tz[0-9]+\.s, z[0-9]+\.h, z[0-9]+\.h\n + \tsmullt\tz[0-9]+\.s, z[0-9]+\.h, z[0-9]+\.h\n + \tshrnb\tz[0-9]+\.h, z[0-9]+\.s, #15\n + \tshrnt\tz[0-9]+\.h, z[0-9]+\.s, #15\n + } + 1 } } */ +/* { dg-final { scan-assembler-times { + \tsmullb\tz[0-9]+\.d, z[0-9]+\.s, z[0-9]+\.s\n + \tsmullt\tz[0-9]+\.d, z[0-9]+\.s, z[0-9]+\.s\n + \tshrnb\tz[0-9]+\.s, z[0-9]+\.d, #31\n + \tshrnt\tz[0-9]+\.s, z[0-9]+\.d, #31\n + } + 1 } } */ + +/* { dg-final { scan-assembler-times { + \tumullb\tz[0-9]+\.h, z[0-9]+\.b, z[0-9]+\.b\n + \tumullt\tz[0-9]+\.h, z[0-9]+\.b, z[0-9]+\.b\n + \tshrnb\tz[0-9]+\.b, z[0-9]+\.h, #7\n + \tshrnt\tz[0-9]+\.b, z[0-9]+\.h, #7\n + } + 1 } } */ +/* { dg-final { scan-assembler-times { + \tumullb\tz[0-9]+\.s, z[0-9]+\.h, z[0-9]+\.h\n + \tumullt\tz[0-9]+\.s, z[0-9]+\.h, z[0-9]+\.h\n + \tshrnb\tz[0-9]+\.h, z[0-9]+\.s, #15\n + \tshrnt\tz[0-9]+\.h, z[0-9]+\.s, #15\n + } + 1 } } */ +/* { dg-final { scan-assembler-times { + \tumullb\tz[0-9]+\.d, z[0-9]+\.s, z[0-9]+\.s\n + \tumullt\tz[0-9]+\.d, z[0-9]+\.s, z[0-9]+\.s\n + \tshrnb\tz[0-9]+\.s, z[0-9]+\.d, #31\n + \tshrnt\tz[0-9]+\.s, z[0-9]+\.d, #31\n + } + 1 } } */ + +/* { dg-final { scan-assembler-times { + \tsmullb\tz[0-9]+\.h, z[0-9]+\.b, z[0-9]+\.b\n + \tsmullt\tz[0-9]+\.h, z[0-9]+\.b, z[0-9]+\.b\n + \trshrnb\tz[0-9]+\.b, z[0-9]+\.h, #7\n + \trshrnt\tz[0-9]+\.b, z[0-9]+\.h, #7\n + } + 1 } } */ +/* { dg-final { scan-assembler-times { + \tsmullb\tz[0-9]+\.s, z[0-9]+\.h, z[0-9]+\.h\n + \tsmullt\tz[0-9]+\.s, z[0-9]+\.h, z[0-9]+\.h\n + \trshrnb\tz[0-9]+\.h, z[0-9]+\.s, #15\n + \trshrnt\tz[0-9]+\.h, z[0-9]+\.s, #15\n + } + 1 } } */ +/* { dg-final { scan-assembler-times { + \tsmullb\tz[0-9]+\.d, z[0-9]+\.s, z[0-9]+\.s\n + \tsmullt\tz[0-9]+\.d, z[0-9]+\.s, z[0-9]+\.s\n + \trshrnb\tz[0-9]+\.s, z[0-9]+\.d, #31\n + \trshrnt\tz[0-9]+\.s, z[0-9]+\.d, #31\n + } + 1 } } */ + +/* { dg-final { scan-assembler-times { + \tumullb\tz[0-9]+\.h, z[0-9]+\.b, z[0-9]+\.b\n + \tumullt\tz[0-9]+\.h, z[0-9]+\.b, z[0-9]+\.b\n + \trshrnb\tz[0-9]+\.b, z[0-9]+\.h, #7\n + \trshrnt\tz[0-9]+\.b, z[0-9]+\.h, #7\n + } + 1 } } */ +/* { dg-final { scan-assembler-times { + \tumullb\tz[0-9]+\.s, z[0-9]+\.h, z[0-9]+\.h\n + \tumullt\tz[0-9]+\.s, z[0-9]+\.h, z[0-9]+\.h\n + \trshrnb\tz[0-9]+\.h, z[0-9]+\.s, #15\n + \trshrnt\tz[0-9]+\.h, z[0-9]+\.s, #15\n + } + 1 } } */ +/* { dg-final { scan-assembler-times { + \tumullb\tz[0-9]+\.d, z[0-9]+\.s, z[0-9]+\.s\n + \tumullt\tz[0-9]+\.d, z[0-9]+\.s, z[0-9]+\.s\n + \trshrnb\tz[0-9]+\.s, z[0-9]+\.d, #31\n + \trshrnt\tz[0-9]+\.s, z[0-9]+\.d, #31\n + } + 1 } } */ + diff --git a/gcc/tree-vect-patterns.c b/gcc/tree-vect-patterns.c index ccb2e1edecda09db786c0d98ccd25f5be107c7e4..96e86f716da0b819640cca2ff5d01c3ce8b342e3 100644 --- a/gcc/tree-vect-patterns.c +++ b/gcc/tree-vect-patterns.c @@ -1723,6 +1723,169 @@ vect_recog_over_widening_pattern (stmt_vec_info last_stmt_info, tree *type_out) return pattern_stmt; } +/* Recognize the following patterns: + + ATYPE a; // narrower than TYPE + BTYPE b; // narrower than TYPE + + // multiply high with scaling + 1) TYPE res = ((TYPE) a * (TYPE) b) >> c; + // or also with rounding + 2) TYPE res = (((TYPE) a * (TYPE) b) >> d + 1) >> 1; + + where only the bottom half of res is used. */ + +static gimple * +vect_recog_multhi_pattern (stmt_vec_info last_stmt_info, tree *type_out) +{ + vec_info *vinfo; + + /* Check for a right shift. */ + gassign *last_stmt = dyn_cast (last_stmt_info->stmt); + if (!last_stmt + || gimple_assign_rhs_code (last_stmt) != RSHIFT_EXPR) + return NULL; + vinfo = last_stmt_info->vinfo; + + /* Check that the shift result is wider than the users of the + result need (i.e. that narrowing would be a natural choice). */ + tree lhs = gimple_assign_lhs (last_stmt); + tree lhs_type = TREE_TYPE (lhs); + unsigned int target_precision = + vect_element_precision (last_stmt_info->min_output_precision); + if (!INTEGRAL_TYPE_P (lhs_type) + || target_precision >= TYPE_PRECISION (lhs_type)) + return NULL; + + /* Look through any change in sign on the outer shift input. */ + vect_unpromoted_value unprom_input; + tree rhs_rshift_inp = vect_look_through_possible_promotion (vinfo, + gimple_assign_rhs1 (last_stmt), &unprom_input); + if (!rhs_rshift_inp + || TYPE_PRECISION (TREE_TYPE (rhs_rshift_inp)) + != TYPE_PRECISION (lhs_type)) + return NULL; + + /* Get the definition of the shift input. */ + stmt_vec_info input_stmt_info = + vect_get_internal_def (vinfo, rhs_rshift_inp); + if (!input_stmt_info) + return NULL; + gassign *input_stmt = dyn_cast (input_stmt_info->stmt); + if (!input_stmt) + return NULL; + + stmt_vec_info mulhs_stmt_info; + internal_fn ifn; + unsigned int expect_offset; + + /* Check for presence of rounding term. */ + if (gimple_assign_rhs_code (input_stmt) == PLUS_EXPR) + { + /* Check that the outer shift was by 1. */ + if (!integer_onep (gimple_assign_rhs2 (last_stmt))) + return NULL; + + /* Check that one of the operands is 1. */ + tree oprnds[2] = { gimple_assign_rhs1 (input_stmt), + gimple_assign_rhs2 (input_stmt) }; + bool isone[2] = { integer_onep (oprnds[0]), + integer_onep (oprnds[1]) }; + if (!(isone[0] ^ isone[1])) + return NULL; + tree mulhs = oprnds[(int)(isone[0])]; + + mulhs_stmt_info = vect_get_internal_def (vinfo, mulhs); + if (!mulhs_stmt_info) + return NULL; + + expect_offset = target_precision + 2; + ifn = IFN_MULHRS; + } + else + { + mulhs_stmt_info = last_stmt_info; + + expect_offset = target_precision + 1; + ifn = IFN_MULHS; + } + + /* Get the definition of the multiply-high-scale part. */ + gassign *mulhs_stmt = dyn_cast (mulhs_stmt_info->stmt); + if (!mulhs_stmt + || gimple_assign_rhs_code (mulhs_stmt) != RSHIFT_EXPR) + return NULL; + + /* Get the scaling factor. */ + tree rhs_scale = gimple_assign_rhs2 (mulhs_stmt); + tree rhs_scale_type = TREE_TYPE (rhs_scale); + if (TREE_CODE (rhs_scale) != INTEGER_CST + || TREE_CODE (rhs_scale_type) != INTEGER_TYPE + || !type_has_mode_precision_p (rhs_scale_type)) + return NULL; + + /* Get the definition of the scaling input term. */ + tree rhs_mulh = gimple_assign_rhs1 (mulhs_stmt); + tree rhs_mulh_type = TREE_TYPE (rhs_mulh); + if (!INTEGRAL_TYPE_P (rhs_mulh_type)) + return NULL; + stmt_vec_info mulh_stmt_info = vect_get_internal_def (vinfo, rhs_mulh); + if (!mulh_stmt_info) + return NULL; + + /* Check that the shift represents the correct scaling. */ + if (wi::ne_p(wi::to_widest(rhs_scale) + expect_offset, + TYPE_PRECISION (rhs_mulh_type))) + return NULL; + + /* Check whether the scaling input term can be seen as two widened + inputs multiplied together. */ + vect_unpromoted_value unprom_mult[2]; + tree new_type; + unsigned int nops = vect_widened_op_tree (mulh_stmt_info, MULT_EXPR, + WIDEN_MULT_EXPR, false, 2, + unprom_mult, &new_type); + if (nops != 2) + return NULL; + + vect_pattern_detected ("vect_recog_multhi_pattern", last_stmt); + + /* Adjust output precision. */ + if (TYPE_PRECISION (new_type) < target_precision) + new_type = build_nonstandard_integer_type (target_precision, + TYPE_UNSIGNED (new_type)); + + /* Check for target support. */ + tree new_vectype = get_vectype_for_scalar_type (new_type); + if (!new_vectype + || !direct_internal_fn_supported_p (ifn, new_vectype, + OPTIMIZE_FOR_SPEED)) + return NULL; + + /* The IR requires a valid vector type for the cast result, even though + it's likely to be discarded. */ + *type_out = get_vectype_for_scalar_type (lhs_type); + if (!*type_out) + return NULL; + + /* Generate the IFN_MULHRS call. */ + tree new_var = vect_recog_temp_ssa_var (new_type, NULL); + tree new_ops[2]; + vect_convert_inputs (last_stmt_info, 2, new_ops, new_type, + unprom_mult, new_vectype); + gcall *mulhrs_stmt = gimple_build_call_internal (ifn, 2, + new_ops[0], new_ops[1]); + gimple_call_set_lhs (mulhrs_stmt, new_var); + gimple_set_location (mulhrs_stmt, gimple_location (last_stmt)); + + if (dump_enabled_p ()) + dump_printf_loc (MSG_NOTE, vect_location, + "created pattern stmt: %G", mulhrs_stmt); + + return vect_convert_output (last_stmt_info, lhs_type, + mulhrs_stmt, new_vectype); +} + /* Recognize the patterns: ATYPE a; // narrower than TYPE @@ -4713,6 +4876,7 @@ static vect_recog_func vect_vect_recog_func_ptrs[] = { /* Must come after over_widening, which narrows the shift as much as possible beforehand. */ { vect_recog_average_pattern, "average" }, + { vect_recog_multhi_pattern, "mult_high" }, { vect_recog_cast_forwprop_pattern, "cast_forwprop" }, { vect_recog_widen_mult_pattern, "widen_mult" }, { vect_recog_dot_prod_pattern, "dot_prod" },