PR target/122751
Recognize vector IFN_FMA and IFN_COND_FMA forms whose multiplicands
are extended from half-width integer element types. Lower them in
GIMPLE isel to vector WIDEN_MULT_PLUS_EXPR, which exposes a widening
multiply-add to target expansion.
For masked forms, choose which multiplicand gets the explicit zeroing
operation based on whether the mask is likely to fold away or lower
cheaply. This keeps selects on masked-load-derived values where
possible.
Add AArch64 SVE2 optab support so the new GIMPLE form can expand to
SMLALB and UMLALB.
Bootstrapped and regression tested on aarch64-linux-gnu.
gcc/ChangeLog:
* config/aarch64/aarch64-sve2.md (madd<Vhalf_el><mode>4): New
expand.
(umadd<Vhalf_el><mode>4): Likewise.
(@aarch64_sve_add_smlalb_partial<mode>): New insn.
(@aarch64_sve_add_umlalb_partial<mode>): Likewise.
* config/aarch64/iterators.md (VHALF_EL): New mode attribute.
(Vhalf_el): Likewise.
* doc/md.texi (maddmn4): Document vector forms.
* gimple-isel.cc (INCLUDE_ALGORITHM): Define.
(gimple_widen_madd_fma): Declare.
(gimple_widen_madd_cond_fma): Likewise.
(gimple_widen_madd_accumulator): Likewise.
(build_widen_madd_operand): New function.
(widen_madd_masked_operand_cost): Likewise.
(widen_madd_profitable_masked_operand): Likewise.
(gimple_try_build_widen_madd): Likewise.
(gimple_recog_widen_madd): Likewise.
(pass_gimple_isel::execute): Call gimple_recog_widen_madd.
* match.pd (widen_madd_extend): New match pattern.
(widen_madd_fma): Likewise.
(widen_madd_cond_fma): Likewise.
(widen_madd_accumulator): Likewise.
* optabs.cc (expand_widen_pattern_expr): Use convert_optab_handler
for vector WIDEN_MULT_PLUS_EXPR.
* tree-cfg.cc (verify_gimple_assign_ternary): Accept vector
WIDEN_MULT_PLUS_EXPR operands.
* tree.def (WIDEN_MULT_PLUS_EXPR): Document vector forms.
gcc/testsuite/ChangeLog:
* gcc.target/aarch64/sve2/cond-fma-test.c: New test.
* gcc.target/aarch64/sve2/fma-test.c: New test.
---
gcc/config/aarch64/aarch64-sve2.md | 68 +++
gcc/config/aarch64/iterators.md | 8 +
gcc/doc/md.texi | 6 +-
gcc/gimple-isel.cc | 256 +++++++++++
gcc/match.pd | 32 ++
gcc/optabs.cc | 5 +
.../gcc.target/aarch64/sve2/cond-fma-test.c | 419 ++++++++++++++++++
.../gcc.target/aarch64/sve2/fma-test.c | 278 ++++++++++++
gcc/tree-cfg.cc | 28 ++
gcc/tree.def | 12 +-
10 files changed, 1107 insertions(+), 5 deletions(-)
create mode 100644 gcc/testsuite/gcc.target/aarch64/sve2/cond-fma-test.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/sve2/fma-test.c
diff --git a/gcc/config/aarch64/aarch64-sve2.md
b/gcc/config/aarch64/aarch64-sve2.md
index c33df283cc2..e7ad88fc979 100644
--- a/gcc/config/aarch64/aarch64-sve2.md
+++ b/gcc/config/aarch64/aarch64-sve2.md
@@ -2715,6 +2715,74 @@
;; - UMLSLT
;; -------------------------------------------------------------------------
+(define_expand "madd<Vhalf_el><mode>4"
+ [(match_operand:SVE_FULL_HSDI 0 "register_operand")
+ (match_operand:<VHALF_EL> 1 "register_operand")
+ (match_operand:<VHALF_EL> 2 "register_operand")
+ (match_operand:SVE_FULL_HSDI 3 "register_operand")]
+ "TARGET_SVE2"
+ {
+ emit_insn (gen_aarch64_sve_add_smlalb_partial<mode>
+ (operands[0], operands[3], operands[1], operands[2]));
+ DONE;
+ }
+)
+
+(define_expand "umadd<Vhalf_el><mode>4"
+ [(match_operand:SVE_FULL_HSDI 0 "register_operand")
+ (match_operand:<VHALF_EL> 1 "register_operand")
+ (match_operand:<VHALF_EL> 2 "register_operand")
+ (match_operand:SVE_FULL_HSDI 3 "register_operand")]
+ "TARGET_SVE2"
+ {
+ emit_insn (gen_aarch64_sve_add_umlalb_partial<mode>
+ (operands[0], operands[3], operands[1], operands[2]));
+ DONE;
+ }
+)
+
+;; Signed MLA bottom-half operation with same-lane-count source operands.
+(define_insn "@aarch64_sve_add_smlalb_partial<mode>"
+ [(set (match_operand:SVE_FULL_HSDI 0 "register_operand" "=w,?&w")
+ (plus:SVE_FULL_HSDI
+ (unspec:SVE_FULL_HSDI
+ [(match_operand:<VHALF_EL> 2 "register_operand" "w,w")
+ (match_operand:<VHALF_EL> 3 "register_operand" "w,w")]
+ UNSPEC_SMULLB)
+ (match_operand:SVE_FULL_HSDI 1 "register_operand" "0,w")))]
+ "TARGET_SVE2"
+ {
+ if (which_alternative == 0)
+ return "smlalb\t%0.<Vetype>, %2.<Ventype>, "
+ "%3.<Ventype>";
+ return "movprfx\t%0, %1\;smlalb\t%0.<Vetype>, "
+ "%2.<Ventype>, %3.<Ventype>";
+ }
+ [(set_attr "movprfx" "*,yes")
+ (set_attr "sve_type" "sve_int_mul")]
+)
+
+;; Unsigned MLA bottom-half operation with same-lane-count source operands.
+(define_insn "@aarch64_sve_add_umlalb_partial<mode>"
+ [(set (match_operand:SVE_FULL_HSDI 0 "register_operand" "=w,?&w")
+ (plus:SVE_FULL_HSDI
+ (unspec:SVE_FULL_HSDI
+ [(match_operand:<VHALF_EL> 2 "register_operand" "w,w")
+ (match_operand:<VHALF_EL> 3 "register_operand" "w,w")]
+ UNSPEC_UMULLB)
+ (match_operand:SVE_FULL_HSDI 1 "register_operand" "0,w")))]
+ "TARGET_SVE2"
+ {
+ if (which_alternative == 0)
+ return "umlalb\t%0.<Vetype>, %2.<Ventype>, "
+ "%3.<Ventype>";
+ return "movprfx\t%0, %1\;umlalb\t%0.<Vetype>, "
+ "%2.<Ventype>, %3.<Ventype>";
+ }
+ [(set_attr "movprfx" "*,yes")
+ (set_attr "sve_type" "sve_int_mul")]
+)
+
;; Non-saturating MLA operations.
(define_insn "@aarch64_sve_add_<sve_int_op><mode>"
[(set (match_operand:SVE_FULL_HSDI 0 "register_operand")
diff --git a/gcc/config/aarch64/iterators.md b/gcc/config/aarch64/iterators.md
index 568786e3714..36c0e6dedf5 100644
--- a/gcc/config/aarch64/iterators.md
+++ b/gcc/config/aarch64/iterators.md
@@ -2231,6 +2231,14 @@
(VNx2HI "vnx2qi") (VNx4SI "vnx4hi")
(VNx2SI "vnx2hi") (VNx2DI "vnx2si")])
+;; Half element, same lane count vector modes.
+(define_mode_attr VHALF_EL [(VNx8HI "VNx8QI")
+ (VNx4SI "VNx4HI")
+ (VNx2DI "VNx2SI")])
+(define_mode_attr Vhalf_el [(VNx8HI "vnx8qi")
+ (VNx4SI "vnx4hi")
+ (VNx2DI "vnx2si")])
+
;; Narrowed quad-modes for VQN (Used for XTN2).
(define_mode_attr VNARROWQ2 [(V8HI "V16QI") (V4SI "V8HI")
(V2DI "V4SI")])
diff --git a/gcc/doc/md.texi b/gcc/doc/md.texi
index 648c12f6e7b..4f6d13bfc04 100644
--- a/gcc/doc/md.texi
+++ b/gcc/doc/md.texi
@@ -5204,8 +5204,10 @@ in RTL using an @code{umul_highpart} RTX expression.
Multiply operands 1 and 2, sign-extend them to mode @var{n}, add
operand 3, and store the result in operand 0. Operands 1 and 2
have mode @var{m} and operands 0 and 3 have mode @var{n}.
-Both modes must be integer or fixed-point modes and @var{n} must be twice
-the size of @var{m}.
+For scalar modes, both modes must be integer or fixed-point modes and
+@var{n} must be twice the size of @var{m}. For vector modes, both modes
+must be integer modes, they must have the same number of elements, and
+each element of @var{n} must be twice the size of an element of @var{m}.
In other words, @code{madd@var{m}@var{n}4} is like
@code{mul@var{m}@var{n}3} except that it also adds operand 3.
diff --git a/gcc/gimple-isel.cc b/gcc/gimple-isel.cc
index 1d346ee124f..d99063b42b8 100644
--- a/gcc/gimple-isel.cc
+++ b/gcc/gimple-isel.cc
@@ -17,6 +17,7 @@ You should have received a copy of the GNU General Public
License
along with GCC; see the file COPYING3. If not see
<http://www.gnu.org/licenses/>. */
+#define INCLUDE_ALGORITHM
#include "config.h"
#include "system.h"
#include "coretypes.h"
@@ -41,6 +42,7 @@ along with GCC; see the file COPYING3. If not see
#include "internal-fn.h"
#include "fold-const.h"
#include "gimple-range.h"
+#include "gimple-match.h"
/* Expand all ARRAY_REF(VIEW_CONVERT_EXPR) gimple assignments into calls to
@@ -377,6 +379,10 @@ extern bool gimple_nop_atomic_bit_test_and_p (tree, tree *,
tree (*) (tree));
extern bool gimple_nop_convert (tree, tree*, tree (*) (tree));
+extern bool gimple_widen_madd_fma (tree, tree*, tree (*) (tree));
+extern bool gimple_widen_madd_cond_fma (tree, tree*, tree (*) (tree));
+extern bool gimple_widen_madd_accumulator (tree, tree*, tree (*) (tree));
+
namespace {
const pass_data pass_data_gimple_isel =
@@ -1350,6 +1356,254 @@ gimple_isel_builtin_call (gcall *call,
gimple_stmt_iterator *gsi)
}
}
+static tree
+build_widen_madd_operand (gimple_stmt_iterator *gsi,
+ tree op,
+ tree narrow_type)
+{
+ gcc_assert (TREE_CODE (op) == SSA_NAME);
+ if (types_compatible_p (TREE_TYPE (op), narrow_type))
+ return op;
+
+ tree lhs = make_ssa_name (narrow_type);
+ gassign *stmt = gimple_build_assign (lhs, NOP_EXPR, op);
+ gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
+
+ return lhs;
+}
+
+/* Return a relative cost for using OP as the operand that gets the
+ explicit VEC_COND_EXPR mask in a masked widening multiply-add. Lower
+ values are preferred. */
+
+static unsigned
+widen_madd_masked_operand_cost (tree op, unsigned depth = 0)
+{
+ if (TREE_CODE (op) != SSA_NAME)
+ return 4;
+
+ gimple *stmt = SSA_NAME_DEF_STMT (op);
+
+ /* Values loaded by IFN_MASK_LOAD already have zeros in inactive lanes.
+ A following VEC_COND_EXPR is therefore redundant, and can be removed
+ by later folding. */
+ if (gimple_call_internal_p (stmt, IFN_MASK_LOAD))
+ return 0;
+
+ /* PHIs and default definitions are often IVs or loop-carried values.
+ Masking them usually creates a real select, so prefer an assign-based
+ operand if one is available. */
+ if (!is_gimple_assign (stmt))
+ return 4;
+
+ enum tree_code code = gimple_assign_rhs_code (stmt);
+
+ /* Avoid walking too far through arbitrary scalar evolution or address
+ calculation chains. The profitable cases are local to the multiply. */
+ if (depth >= 4)
+ return 3;
+
+ /* A conversion of a masked load is almost as good as the masked load
+ itself: the select is still on a value that is already zero for
+ inactive lanes. */
+ if (CONVERT_EXPR_CODE_P (code) || code == VIEW_CONVERT_EXPR)
+ {
+ tree rhs = gimple_assign_rhs1 (stmt);
+ if (TREE_CODE (rhs) == SSA_NAME)
+ {
+ unsigned rhs_cost = widen_madd_masked_operand_cost (rhs, depth + 1);
+ if (rhs_cost <= 1)
+ return 1;
+ if (rhs_cost == 2)
+ return 2;
+ }
+ return 3;
+ }
+
+ unsigned best_rhs_cost = 4;
+ for (unsigned i = 1; i < gimple_num_ops (stmt); ++i)
+ {
+ tree rhs = gimple_op (stmt, i);
+ if (rhs && TREE_CODE (rhs) == SSA_NAME)
+ best_rhs_cost
+ = std::min (best_rhs_cost,
+ widen_madd_masked_operand_cost (rhs, depth + 1));
+ }
+
+ /* Arithmetic fed by a masked load is still a good masking candidate.
+ The select may fold into a predicated arithmetic operation, as with
+ a loaded byte adjusted by a constant. */
+ if (best_rhs_cost <= 2)
+ return 2;
+
+ /* Plain assignments are a better fallback than PHIs/default definitions,
+ since they at least have a dedicated SSA result for the masked value. */
+ return 3;
+}
+
+/* Choose which multiplicand of a masked widening multiply-add should receive
+ the explicit VEC_COND_EXPR. Multiplication is commutative, so use the
+ operand whose mask is cheapest or most likely to disappear. */
+
+static void
+widen_madd_profitable_masked_operand (tree& op1, tree& op2)
+{
+ gcc_assert (TREE_CODE (op1) == SSA_NAME && TREE_CODE (op2) == SSA_NAME);
+
+ unsigned cost1 = widen_madd_masked_operand_cost (op1);
+ unsigned cost2 = widen_madd_masked_operand_cost (op2);
+
+ if (cost2 < cost1)
+ std::swap (op1, op2);
+}
+
+/* Try to replace the statement at GSI with a WIDEN_MULT_PLUS_EXPR using the
+ matched MULTIPLICAND0, MULTIPLICAND1 and ACCUMULATOR values. The helper
+ first derives the narrow operand type from the accumulator's wide type and
+ checks that the target supports the resulting operation. For masked forms,
+ it chooses the multiplicand that is cheapest to mask, inserts the explicit
+ zeroing operation, and gives folding a chance to remove or lower it before
+ building the final widening multiply-add. Return true if the statement was
+ replaced, otherwise leave it unchanged. */
+
+static bool
+gimple_try_build_widen_madd (gimple_stmt_iterator *gsi,
+ tree lhs,
+ tree multiplicand0,
+ tree multiplicand1,
+ tree accumulator,
+ tree mask = NULL_TREE)
+{
+ if (!gimple_call_internal_p (gsi_stmt (*gsi), IFN_FMA))
+ gimple_widen_madd_accumulator (accumulator, &accumulator, NULL);
+
+ tree wide_type = TREE_TYPE (accumulator);
+
+ if (mask && !expand_vec_cond_expr_p (wide_type, TREE_TYPE (mask)))
+ return false;
+
+ tree wide_element_type = TREE_TYPE (wide_type);
+ bool unsigned_p = TYPE_UNSIGNED (wide_element_type);
+ unsigned wide_precision = TYPE_PRECISION (wide_element_type);
+ unsigned narrow_precision = wide_precision / 2;
+ tree narrow_element_type
+ = build_nonstandard_integer_type (narrow_precision, unsigned_p);
+ tree narrow_type
+ = build_vector_type (narrow_element_type,
+ TYPE_VECTOR_SUBPARTS (wide_type));
+
+ if (!directly_supported_p (WIDEN_MULT_PLUS_EXPR,
+ wide_type,
+ narrow_type))
+ return false;
+
+ if (mask)
+ widen_madd_profitable_masked_operand (multiplicand0, multiplicand1);
+
+ multiplicand0 = build_widen_madd_operand (gsi, multiplicand0, narrow_type);
+ multiplicand1 = build_widen_madd_operand (gsi, multiplicand1, narrow_type);
+
+ if (mask)
+ {
+ tree cond_lhs = make_ssa_name (narrow_type);
+ gassign* cond_stmt
+ = gimple_build_assign (cond_lhs, VEC_COND_EXPR, mask, multiplicand0,
+ build_zero_cst (narrow_type));
+ gsi_insert_before (gsi, cond_stmt, GSI_SAME_STMT);
+ gimple_stmt_iterator cond_gsi = gsi_for_stmt (cond_stmt);
+ if (fold_stmt (&cond_gsi, follow_all_ssa_edges))
+ update_stmt (gsi_stmt (cond_gsi));
+
+ if (gimple_assign_rhs_code (gsi_stmt (cond_gsi)) == VEC_COND_EXPR)
+ {
+ gimple* g = gimple_expand_vec_cond_expr (&cond_gsi);
+ gimple_set_lhs (g, cond_lhs);
+ gsi_replace (&cond_gsi, g, false);
+ }
+ multiplicand0 = cond_lhs;
+ }
+
+ gassign* new_stmt;
+ if (TREE_TYPE (lhs) != wide_type)
+ {
+ tree madd_lhs = make_ssa_name (wide_type);
+ new_stmt = gimple_build_assign (madd_lhs,
+ WIDEN_MULT_PLUS_EXPR,
+ multiplicand0,
+ multiplicand1,
+ accumulator);
+ gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
+ new_stmt = gimple_build_assign (lhs,
+ NOP_EXPR,
+ madd_lhs);
+ }
+ else
+ {
+ new_stmt = gimple_build_assign (lhs,
+ WIDEN_MULT_PLUS_EXPR,
+ multiplicand0,
+ multiplicand1,
+ accumulator);
+ }
+
+ gsi_replace (gsi, new_stmt, false);
+
+ return true;
+}
+
+/* Recognize vector IFN_FMA and conditional IFN_FMA forms that can be
+ represented as GIMPLE widening multiply-adds. The matchers provide the
+ two narrow multiplicands, the wide accumulator and, for conditional FMA,
+ the mask.
+ Once replacement succeeds, run local DCE from the original FMA operands so
+ now-unused conversion and FMA feeding statements are removed. */
+
+static bool
+gimple_recog_widen_madd (gimple_stmt_iterator *gsi)
+{
+ gcall *call = dyn_cast <gcall*> (gsi_stmt (*gsi));
+ if (!call)
+ return false;
+
+ tree lhs = gimple_call_lhs (call);
+ if (!lhs)
+ return false;
+
+ bitmap dce_worklist = BITMAP_ALLOC (NULL);
+ tree res_ops[5];
+ if (gimple_widen_madd_cond_fma (lhs, res_ops, NULL))
+ {
+ bitmap_set_bit (dce_worklist,
+ SSA_NAME_VERSION (gimple_call_arg (call, 1)));
+ bitmap_set_bit (dce_worklist,
+ SSA_NAME_VERSION (gimple_call_arg (call, 2)));
+ if (gimple_try_build_widen_madd (gsi, lhs, res_ops[0], res_ops[1],
+ res_ops[2], res_ops[3]))
+ {
+ simple_dce_from_worklist (dce_worklist);
+ BITMAP_FREE (dce_worklist);
+ return true;
+ }
+ }
+
+ if (gimple_widen_madd_fma (lhs, res_ops, NULL))
+ {
+ bitmap_set_bit (dce_worklist,
+ SSA_NAME_VERSION (gimple_call_arg (call, 0)));
+ bitmap_set_bit (dce_worklist,
+ SSA_NAME_VERSION (gimple_call_arg (call, 1)));
+ if (gimple_try_build_widen_madd (gsi, lhs, res_ops[0], res_ops[1],
+ res_ops[2]))
+ {
+ simple_dce_from_worklist (dce_worklist);
+ BITMAP_FREE (dce_worklist);
+ return true;
+ }
+ }
+
+ return false;
+}
+
/* Iterate all gimple statements and perform pre RTL expansion
GIMPLE massaging to improve instruction selection. */
@@ -1369,6 +1623,8 @@ pass_gimple_isel::execute (struct function *fun)
/* Give the target first try at replacing the instruction. */
cfg_changed |= targetm.instruction_selection (fun, &gsi);
+ cfg_changed |= gimple_recog_widen_madd (&gsi);
+
/* Pre-expand VEC_COND_EXPRs to .VCOND* internal function
calls mapping to supported optabs. */
gimple *g = gimple_expand_vec_cond_expr (&gsi);
diff --git a/gcc/match.pd b/gcc/match.pd
index 648bda9bf11..57d4f1c14b1 100644
--- a/gcc/match.pd
+++ b/gcc/match.pd
@@ -13724,3 +13724,35 @@ and,
&& wi::le_p (wi::to_wide (@1), bound, TYPE_SIGN (type))))
@3)))))
#endif
+
+#if GIMPLE
+(match (widen_madd_extend @0)
+ (convert @0)
+ (if (VECTOR_TYPE_P (type)
+ && VECTOR_TYPE_P (TREE_TYPE (@0))
+ && INTEGRAL_TYPE_P (TREE_TYPE (type))
+ && INTEGRAL_TYPE_P (TREE_TYPE (TREE_TYPE (@0)))
+ && known_eq (TYPE_VECTOR_SUBPARTS (type),
+ TYPE_VECTOR_SUBPARTS (TREE_TYPE (@0)))
+ && (TYPE_PRECISION (TREE_TYPE (type))
+ >= 2 * TYPE_PRECISION (TREE_TYPE (TREE_TYPE (@0)))))))
+
+(match (widen_madd_fma @0 @1 @2)
+ (IFN_FMA
+ (widen_madd_extend @0)
+ (widen_madd_extend @1)
+ @2))
+
+(match (widen_madd_cond_fma @0 @1 @2 @3)
+ (IFN_COND_FMA
+ @3
+ (widen_madd_extend @0)
+ (widen_madd_extend @1)
+ @2
+ @2))
+
+(match (widen_madd_accumulator @0)
+ (view_convert @0)
+ (if (TYPE_PRECISION (TREE_TYPE (type))
+ == TYPE_PRECISION (TREE_TYPE (TREE_TYPE (@0))))))
+#endif
diff --git a/gcc/optabs.cc b/gcc/optabs.cc
index 628500f8c87..ae5728b219d 100644
--- a/gcc/optabs.cc
+++ b/gcc/optabs.cc
@@ -319,6 +319,11 @@ expand_widen_pattern_expr (const_sepops ops, rtx op0, rtx
op1, rtx wide_op,
widen_pattern_optab
= optab_for_tree_code (ops->code, TREE_TYPE (oprnd0), optab_default);
if (ops->code == WIDEN_MULT_PLUS_EXPR
+ && VECTOR_MODE_P (TYPE_MODE (TREE_TYPE (ops->op2))))
+ icode = convert_optab_handler (widen_pattern_optab,
+ TYPE_MODE (TREE_TYPE (ops->op2)),
+ tmode0);
+ else if (ops->code == WIDEN_MULT_PLUS_EXPR
|| ops->code == WIDEN_MULT_MINUS_EXPR
|| ops->code == DOT_PROD_EXPR)
icode = find_widening_optab_handler (widen_pattern_optab,
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/cond-fma-test.c
b/gcc/testsuite/gcc.target/aarch64/sve2/cond-fma-test.c
new file mode 100644
index 00000000000..993b65e8b4b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/cond-fma-test.c
@@ -0,0 +1,419 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=armv8-a+sve2 -mautovec-preference=sve-only" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+
+#include <stdint.h>
+
+/*
+** cond_fma_unsigned_iv_plus_constant:
+** ...
+** umlalb z[0-9]+\.s, z[0-9]+\.h, z[0-9]+\.h
+** ...
+** ret
+*/
+uint32_t
+cond_fma_unsigned_iv_plus_constant (uint8_t *buf, int len)
+{
+ if (len < 0 || len >= 256)
+ return 0;
+
+ uint32_t x = 0;
+ for (uint16_t i = 0, y = 0; i < len; i++, y = i * 10)
+ x += (uint32_t) (y + 2) * (uint32_t) buf[i];
+ return x;
+}
+
+
+/*
+** cond_fma_signed_char_digit_offset:
+** ...
+** smlalb z[0-9]+\.s, z[0-9]+\.h, z[0-9]+\.h
+** ...
+** ret
+*/
+int
+cond_fma_signed_char_digit_offset (char *buf, int len)
+{
+ if (len < 0 || len >= 256)
+ return 0;
+
+ int x = 0;
+ for (int16_t i = 0, y = 0; i < len; i++, y = i * 10)
+ x += y * (buf[i] - '0');
+ return x;
+}
+
+/*
+** cond_fma_unsigned_short_load:
+** ...
+** umlalb z[0-9]+\.s, z[0-9]+\.h, z[0-9]+\.h
+** ...
+** ret
+*/
+unsigned
+cond_fma_unsigned_short_load (uint16_t *buf, uint8_t len)
+{
+ unsigned x = 0;
+ for (uint16_t i = 0, y = 0; i < len; i++, y = i * 10)
+ x += buf[i] * y;
+
+ return x;
+}
+
+/*
+** cond_fma_float_to_int_load:
+** ...
+** fcvtzs z[0-9]+\.s, p[0-9]+/m, z[0-9]+\.s
+** ...
+** smlalb z[0-9]+\.d, z[0-9]+\.s, z[0-9]+\.s
+** ...
+** ret
+*/
+long
+cond_fma_float_to_int_load (float *buf, uint16_t len)
+{
+ long x = 0;
+ for (int i = 0, y = 0; i < len; i++, y = i * 10)
+ x += (long) y * (long) (int) buf[i];
+ return x;
+}
+
+/*
+** cond_fma_double_to_int_load:
+** ...
+** fcvtzs z[0-9]+\.s, p[0-9]+/m, z[0-9]+\.d
+** ...
+** smlalb z[0-9]+\.d, z[0-9]+\.s, z[0-9]+\.s
+** ...
+** ret
+*/
+long
+cond_fma_double_to_int_load (double *buf, uint16_t len)
+{
+ long x = 0;
+ for (int i = 0, y = 0; i < len; i++, y = i * 10)
+ x += (long) y * (long) (int) buf[i];
+ return x;
+}
+
+/*
+** cond_fma_two_signed_char_loads:
+** ...
+** smlalb z[0-9]+\.s, z[0-9]+\.h, z[0-9]+\.h
+** ...
+** ret
+*/
+int
+cond_fma_two_signed_char_loads (char *__restrict buf1,
+ char *__restrict buf2, uint8_t len)
+{
+ int x = 0;
+ for (int i = 0; i < len; ++i)
+ x += (buf1[i] - 2) * (buf2[i]);
+ return x;
+}
+
+/*
+** cond_fma_signed_iv_square:
+** ...
+** smlalb z[0-9]+\.s, z[0-9]+\.h, z[0-9]+\.h
+** ...
+** ret
+*/
+int
+cond_fma_signed_iv_square (int len)
+{
+ if (len <= 0 || len >= 255)
+ return 0;
+
+ int res = 0;
+ for (short i = 0, y = 0; i < len; i++, y = i * 10)
+ res += y * y;
+ return res;
+}
+
+/*
+** cond_fma_unsigned_iv_square:
+** ...
+** umlalb z[0-9]+\.s, z[0-9]+\.h, z[0-9]+\.h
+** ...
+** ret
+*/
+unsigned
+cond_fma_unsigned_iv_square (unsigned len)
+{
+ if (len == 0 || len >= 255)
+ return 0;
+
+ unsigned res = 0;
+ for (unsigned short i = 0, y = 0; i < len; i++, y = i * 10)
+ res += y * y;
+ return res;
+}
+
+/*
+** cond_fma_s8_to_s16_reduction:
+** ...
+** smlalb z[0-9]+\.h, z[0-9]+\.b, z[0-9]+\.b
+** ...
+** ret
+*/
+short
+cond_fma_s8_to_s16_reduction (int8_t *a, int8_t *b, int len)
+{
+ if (len < 0 || len >= 128)
+ return 0;
+
+ short x = 0;
+ for (int8_t i = 0; i < len; ++i)
+ x += (short) a[i] * (short) b[i];
+
+ return x;
+}
+
+/*
+** cond_fma_u8_to_u16_reduction:
+** ...
+** umlalb z[0-9]+\.h, z[0-9]+\.b, z[0-9]+\.b
+** ...
+** ret
+*/
+unsigned short
+cond_fma_u8_to_u16_reduction (uint8_t *a, uint8_t *b, int len)
+{
+ if (len < 0 || len >= 128)
+ return 0;
+
+ unsigned short x = 0;
+ for (uint8_t i = 0; i < len; ++i)
+ x += (unsigned short) a[i] * (unsigned short) b[i];
+
+ return x;
+}
+
+/*
+** cond_fma_s16_to_s32_reduction:
+** ...
+** smlalb z[0-9]+\.s, z[0-9]+\.h, z[0-9]+\.h
+** ...
+** ret
+*/
+int
+cond_fma_s16_to_s32_reduction (int16_t *a, int16_t *b, int len)
+{
+ if (len < 0 || len >= 256)
+ return 0;
+
+ int x = 0;
+ for (int16_t i = 0; i < len; ++i)
+ x += (int) a[i] * (int) b[i];
+
+ return x;
+}
+
+/*
+** cond_fma_u16_to_u32_reduction:
+** ...
+** umlalb z[0-9]+\.s, z[0-9]+\.h, z[0-9]+\.h
+** ...
+** ret
+*/
+unsigned int
+cond_fma_u16_to_u32_reduction (uint16_t *a, uint16_t *b, int len)
+{
+ if (len < 0 || len >= 256)
+ return 0;
+
+ unsigned int x = 0;
+ for (uint16_t i = 0; i < len; ++i)
+ x += (unsigned int) a[i] * (unsigned int) b[i];
+
+ return x;
+}
+
+/*
+** cond_fma_s32_to_s64_reduction:
+** ...
+** smlalb z[0-9]+\.d, z[0-9]+\.s, z[0-9]+\.s
+** ...
+** ret
+*/
+long int
+cond_fma_s32_to_s64_reduction (int32_t *a, int32_t *b, int len)
+{
+ if (len < 0 || len >= 65536)
+ return 0;
+
+ long int x = 0;
+ for (int32_t i = 0; i < len; ++i)
+ x += (long int) a[i] * (long int) b[i];
+
+ return x;
+}
+
+/*
+** cond_fma_u32_to_u64_reduction:
+** ...
+** umlalb z[0-9]+\.d, z[0-9]+\.s, z[0-9]+\.s
+** ...
+** ret
+*/
+unsigned long int
+cond_fma_u32_to_u64_reduction (uint32_t *a, uint32_t *b, int len)
+{
+ if (len < 0 || len >= 65536)
+ return 0;
+
+ unsigned long int x = 0;
+ for (uint32_t i = 0; i < len; ++i)
+ x += (unsigned long int) a[i] * (unsigned long int) b[i];
+
+ return x;
+}
+
+/*
+** cond_fma_signed_reinterpret_to_smlalb:
+** ...
+** smlalb z[0-9]+\.s, z[0-9]+\.h, z[0-9]+\.h
+** ...
+** ret
+*/
+int
+cond_fma_signed_reinterpret_to_smlalb (uint16_t *buf, int len)
+{
+ if (len < 0 || len >= 256)
+ return 0;
+
+ int x = 0;
+ for (int16_t i = 0, y = 0; i < len; ++i, y = i * 10)
+ {
+ int16_t v = (int16_t) buf[i];
+ x += y * (int) v;
+ }
+ return x;
+}
+
+/*
+** cond_fma_unsigned_reinterpret_to_umlalb:
+** ...
+** umlalb z[0-9]+\.s, z[0-9]+\.h, z[0-9]+\.h
+** ...
+** ret
+*/
+unsigned int
+cond_fma_unsigned_reinterpret_to_umlalb (int16_t *buf, int len)
+{
+ if (len < 0 || len >= 256)
+ return 0;
+
+ unsigned int x = 0;
+ for (uint16_t i = 0, y = 0; i < len; ++i, y = i * 10)
+ {
+ uint16_t v = (uint16_t) buf[i];
+ x += (unsigned int) y * (unsigned int) v;
+ }
+ return x;
+}
+
+/*
+** cond_fma_signed_product_to_unsigned_accumulator:
+** ...
+** umlalb z[0-9]+\.s, z[0-9]+\.h, z[0-9]+\.h
+** ...
+** ret
+*/
+unsigned int
+cond_fma_signed_product_to_unsigned_accumulator (int16_t *a, int16_t *b,
+ int len)
+{
+ if (len < 0 || len >= 256)
+ return 0;
+
+ unsigned int x = 0;
+ for (int16_t i = 0; i < len; ++i)
+ {
+ int product = (int) a[i] * (int) b[i];
+ x += (unsigned int) product;
+ }
+ return x;
+}
+
+/*
+** cond_fma_unsigned_product_to_signed_accumulator:
+** ...
+** smlalb z[0-9]+\.s, z[0-9]+\.h, z[0-9]+\.h
+** ...
+** ret
+*/
+int
+cond_fma_unsigned_product_to_signed_accumulator (uint16_t *a, uint16_t *b,
+ int len)
+{
+ if (len < 0 || len >= 256)
+ return 0;
+
+ int x = 0;
+ for (int16_t i = 0; i < len; ++i)
+ {
+ unsigned int product = (unsigned int) a[i] * (unsigned int) b[i];
+ x += (int) product;
+ }
+ return x;
+}
+
+/*
+** cond_fma_signed_iv_square_s64:
+** ...
+** smlalb z[0-9]+\.d, z[0-9]+\.s, z[0-9]+\.s
+** ...
+** ret
+*/
+long
+cond_fma_signed_iv_square_s64 (int len)
+{
+ if (len <= 0 || len >= 255)
+ return 0;
+
+ long res = 0;
+ for (int i = 0, y = 0; i < len; i++, y = i * 10)
+ res += (long) y * (long) y;
+ return res;
+}
+
+/*
+** cond_fma_signed_long_iv_square_s64:
+** ...
+** smlalb z[0-9]+\.d, z[0-9]+\.s, z[0-9]+\.s
+** ...
+** ret
+*/
+long
+cond_fma_signed_long_iv_square_s64 (int len)
+{
+ if (len <= 0 || len >= 255)
+ return 0;
+
+ long res = 0;
+ for (int i = 0, y = 0; i < len; i++, y = i * 10)
+ res += (long) y * (long) y;
+ return res;
+}
+
+/*
+** cond_fma_unsigned_long_iv_square_u64:
+** ...
+** umlalb z[0-9]+\.d, z[0-9]+\.s, z[0-9]+\.s
+** ...
+** ret
+*/
+unsigned long
+cond_fma_unsigned_long_iv_square_u64 (unsigned len)
+{
+ if (len <= 0 || len >= 255)
+ return 0;
+
+ unsigned long res = 0;
+ for (unsigned i = 0, y = 0; i < len; i++, y = i * 10)
+ res += (unsigned long) y * (unsigned long) y;
+ return res;
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/fma-test.c
b/gcc/testsuite/gcc.target/aarch64/sve2/fma-test.c
new file mode 100644
index 00000000000..bed04ed28e1
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/fma-test.c
@@ -0,0 +1,278 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=armv8-a+sve2 -mautovec-preference=sve-only" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+
+#include <stdint.h>
+
+/*
+** plain_fma_signed_char_digit_offset:
+** ...
+** whilelo p[0-9]+\.s, wzr, w[0-9]+
+** index z[0-9]+\.s, #0, #10
+** ...
+** ld1w z[0-9]+\.s, p[0-9]+/z, \[x[0-9]+, x[0-9]+, lsl 2\]
+** ld1b z[0-9]+\.s, p[0-9]+/z, \[x[0-9]+, x[0-9]+\]
+** sub z[0-9]+\.h, z[0-9]+\.h, #48
+** smlalb z[0-9]+\.s, z[0-9]+\.h, z[0-9]+\.h
+** st1w z[0-9]+\.s, p[0-9]+, \[x[0-9]+, x[0-9]+, lsl 2\]
+** ...
+** ret
+*/
+void
+plain_fma_signed_char_digit_offset (char *__restrict buf, int8_t len, int *x)
+{
+ for (uint16_t i = 0, y = 0; i < len; i++, y = i * 10)
+ x[i] += (int) y * (buf[i] - '0');
+}
+
+/*
+** test:
+** ...
+** whilelo p[0-9]+\.s, wzr, w[0-9]+
+** ...
+** ld1w z[0-9]+\.s, p[0-9]+/z, \[x[0-9]+, x[0-9]+, lsl 2\]
+** ld1h z[0-9]+\.s, p[0-9]+/z, \[x[0-9]+, x[0-9]+, lsl 1\]
+** ld1h z[0-9]+\.s, p[0-9]+/z, \[x[0-9]+, x[0-9]+, lsl 1\]
+** smlalb z[0-9]+\.s, z[0-9]+\.h, z[0-9]+\.h
+** st1w z[0-9]+\.s, p[0-9]+, \[x[0-9]+, x[0-9]+, lsl 2\]
+** ...
+** ret
+*/
+void
+test (short *__restrict a, short *__restrict b, int *__restrict c, uint8_t len)
+{
+ for (uint64_t i = 0; i < len; ++i)
+ c[i] += (int) a[i] * (int) b[i];
+}
+
+/*
+** plain_fma_s32_to_s64_memory_accumulator:
+** ...
+** whilelo p[0-9]+\.d, wzr, w[0-9]+
+** index z[0-9]+\.d, #0, #10
+** ...
+** ld1d z[0-9]+\.d, p[0-9]+/z, \[x[0-9]+, x[0-9]+, lsl 3\]
+** ld1w z[0-9]+\.d, p[0-9]+/z, \[x[0-9]+, x[0-9]+, lsl 2\]
+** smlalb z[0-9]+\.d, z[0-9]+\.s, z[0-9]+\.s
+** st1d z[0-9]+\.d, p[0-9]+, \[x[0-9]+, x[0-9]+, lsl 3\]
+** ...
+** ret
+*/
+void
+plain_fma_s32_to_s64_memory_accumulator (int *__restrict buf, uint16_t len,
+ long *x)
+{
+ for (int i = 0, y = 0; i < len; i++, y = i * 10)
+ x[i] += (long) y * (long) buf[i];
+}
+
+/*
+** plain_fma_float_to_int_load:
+** ...
+** whilelo p[0-9]+\.d, wzr, w[0-9]+
+** index z[0-9]+\.d, #0, #10
+** ...
+** ld1d z[0-9]+\.d, p[0-9]+/z, \[x[0-9]+, x[0-9]+, lsl 3\]
+** ld1w z[0-9]+\.d, p[0-9]+/z, \[x[0-9]+, x[0-9]+, lsl 2\]
+** fcvtzs z[0-9]+\.s, p[0-9]+/m, z[0-9]+\.s
+** smlalb z[0-9]+\.d, z[0-9]+\.s, z[0-9]+\.s
+** st1d z[0-9]+\.d, p[0-9]+, \[x[0-9]+, x[0-9]+, lsl 3\]
+** ...
+** ret
+*/
+void
+plain_fma_float_to_int_load (float *__restrict buf, uint16_t len, long *x)
+{
+ for (int i = 0, y = 0; i < len; i++, y = i * 10)
+ x[i] += (long) y * (long) (int) buf[i];
+}
+
+/*
+** plain_fma_sub_loaded_source:
+** ...
+** whilelo p[0-9]+\.s, wzr, w[0-9]+
+** index z[0-9]+\.s, #0, #10
+** ...
+** ld1h z[0-9]+\.s, p[0-9]+/z, \[x[0-9]+, x[0-9]+, lsl 1\]
+** ld1w z[0-9]+\.s, p[0-9]+/z, \[x[0-9]+, x[0-9]+, lsl 2\]
+** sub z[0-9]+\.h, z[0-9]+\.h, #7
+** smlalb z[0-9]+\.s, z[0-9]+\.h, z[0-9]+\.h
+** st1w z[0-9]+\.s, p[0-9]+, \[x[0-9]+, x[0-9]+, lsl 2\]
+** ...
+** ret
+*/
+void
+plain_fma_sub_loaded_source (int16_t *__restrict buf, int8_t len, int *x)
+{
+ for (int16_t i = 0, y = 0; i < len; i++, y = i * 10)
+ {
+ int16_t v = (int16_t) (buf[i] - 7);
+ x[i] += (int) y * (int) v;
+ }
+}
+
+/*
+** plain_fma_mul_loaded_source:
+** ...
+** whilelo p[0-9]+\.s, wzr, w[0-9]+
+** index z[0-9]+\.s, #0, #10
+** ...
+** ld1h z[0-9]+\.s, p[0-9]+/z, \[x[0-9]+, x[0-9]+, lsl 1\]
+** ld1w z[0-9]+\.s, p[0-9]+/z, \[x[0-9]+, x[0-9]+, lsl 2\]
+** adr z[0-9]+\.s, \[z[0-9]+\.s, z[0-9]+\.s, lsl 1\]
+** smlalb z[0-9]+\.s, z[0-9]+\.h, z[0-9]+\.h
+** st1w z[0-9]+\.s, p[0-9]+, \[x[0-9]+, x[0-9]+, lsl 2\]
+** ...
+** ret
+*/
+void
+plain_fma_mul_loaded_source (int16_t *__restrict buf, int8_t len, int *x)
+{
+ for (int16_t i = 0, y = 0; i < len; i++, y = i * 10)
+ {
+ int16_t v = (int16_t) (buf[i] * 3);
+ x[i] += (int) y * (int) v;
+ }
+}
+
+/*
+** plain_fma_add_loaded_source:
+** ...
+** whilelo p[0-9]+\.s, wzr, w[0-9]+
+** index z[0-9]+\.s, #0, #10
+** ...
+** ld1h z[0-9]+\.s, p[0-9]+/z, \[x[0-9]+, x[0-9]+, lsl 1\]
+** ld1w z[0-9]+\.s, p[0-9]+/z, \[x[0-9]+, x[0-9]+, lsl 2\]
+** add z[0-9]+\.h, z[0-9]+\.h, #7
+** smlalb z[0-9]+\.s, z[0-9]+\.h, z[0-9]+\.h
+** st1w z[0-9]+\.s, p[0-9]+, \[x[0-9]+, x[0-9]+, lsl 2\]
+** ...
+** ret
+*/
+void
+plain_fma_add_loaded_source (int16_t *__restrict buf, int8_t len, int *x)
+{
+ for (int16_t i = 0, y = 0; i < len; i++, y = i * 10)
+ {
+ int16_t v = (int16_t) (buf[i] + 7);
+ x[i] += (int) y * (int) v;
+ }
+}
+
+/*
+** plain_fma_unsigned_add_loaded_source:
+** ...
+** whilelo p[0-9]+\.s, wzr, w[0-9]+
+** index z[0-9]+\.s, #0, #10
+** ...
+** ld1h z[0-9]+\.s, p[0-9]+/z, \[x[0-9]+, x[0-9]+, lsl 1\]
+** ld1w z[0-9]+\.s, p[0-9]+/z, \[x[0-9]+, x[0-9]+, lsl 2\]
+** add z[0-9]+\.h, z[0-9]+\.h, #7
+** umlalb z[0-9]+\.s, z[0-9]+\.h, z[0-9]+\.h
+** st1w z[0-9]+\.s, p[0-9]+, \[x[0-9]+, x[0-9]+, lsl 2\]
+** ...
+** ret
+*/
+void
+plain_fma_unsigned_add_loaded_source (uint16_t *__restrict buf, int8_t len,
+ unsigned int *x)
+{
+ for (uint16_t i = 0, y = 0; i < len; i++, y = i * 10)
+ {
+ uint16_t v = (uint16_t) (buf[i] + 7);
+ x[i] += (unsigned int) y * (unsigned int) v;
+ }
+}
+
+/*
+** plain_fma_accumulator_signedness_convert:
+** ...
+** whilelo p[0-9]+\.s, wzr, w[0-9]+
+** index z[0-9]+\.s, #0, #10
+** ...
+** ld1h z[0-9]+\.s, p[0-9]+/z, \[x[0-9]+, x[0-9]+, lsl 1\]
+** ld1w z[0-9]+\.s, p[0-9]+/z, \[x[0-9]+, x[0-9]+, lsl 2\]
+** smlalb z[0-9]+\.s, z[0-9]+\.h, z[0-9]+\.h
+** st1w z[0-9]+\.s, p[0-9]+, \[x[0-9]+, x[0-9]+, lsl 2\]
+** ...
+** ret
+*/
+void
+plain_fma_accumulator_signedness_convert (int16_t *__restrict buf, int8_t len,
+ unsigned int *x)
+{
+ for (int16_t i = 0, y = 0; i < len; i++, y = i * 10)
+ x[i] = (unsigned int) ((int) x[i] + (int) y * (int) buf[i]);
+}
+
+/*
+** plain_fma_unsigned_short_load:
+** ...
+** whilelo p[0-9]+\.s, wzr, w[0-9]+
+** index z[0-9]+\.s, #0, #10
+** ...
+** ld1w z[0-9]+\.s, p[0-9]+/z, \[x[0-9]+, x[0-9]+, lsl 2\]
+** ld1h z[0-9]+\.s, p[0-9]+/z, \[x[0-9]+, x[0-9]+, lsl 1\]
+** umlalb z[0-9]+\.s, z[0-9]+\.h, z[0-9]+\.h
+** st1w z[0-9]+\.s, p[0-9]+, \[x[0-9]+, x[0-9]+, lsl 2\]
+** ...
+** ret
+*/
+void
+plain_fma_unsigned_short_load (uint16_t *__restrict buf, uint16_t len,
+ unsigned int *x)
+{
+ for (uint16_t i = 0, y = 0; i < len; i++, y = i * 10)
+ x[i] += (unsigned int) y * (unsigned int) buf[i];
+}
+
+/*
+** plain_fma_named_old_accumulator_convert:
+** ...
+** whilelo p[0-9]+\.s, wzr, w[0-9]+
+** index z[0-9]+\.s, #0, #10
+** ...
+** ld1h z[0-9]+\.s, p[0-9]+/z, \[x[0-9]+, x[0-9]+, lsl 1\]
+** ld1w z[0-9]+\.s, p[0-9]+/z, \[x[0-9]+, x[0-9]+, lsl 2\]
+** smlalb z[0-9]+\.s, z[0-9]+\.h, z[0-9]+\.h
+** st1w z[0-9]+\.s, p[0-9]+, \[x[0-9]+, x[0-9]+, lsl 2\]
+** ...
+** ret
+*/
+void
+plain_fma_named_old_accumulator_convert (int16_t *__restrict buf, int8_t len,
+ unsigned int *x)
+{
+ for (int16_t i = 0, y = 0; i < len; i++, y = i * 10)
+ {
+ int old = (int) x[i];
+ x[i] = (unsigned int) (old + (int) y * (int) buf[i]);
+ }
+}
+
+/*
+** plain_fma_preserve_old_accumulator:
+** ...
+** whilelo p[0-9]+\.s, wzr, w[0-9]+
+** index z[0-9]+\.s, #0, #10
+** ...
+** ld1w z[0-9]+\.s, p[0-9]+/z, \[x[0-9]+, x[0-9]+, lsl 2\]
+** ld1h z[0-9]+\.s, p[0-9]+/z, \[x[0-9]+, x[0-9]+, lsl 1\]
+** movprfx z[0-9]+, z[0-9]+
+** smlalb z[0-9]+\.s, z[0-9]+\.h, z[0-9]+\.h
+** st1w z[0-9]+\.s, p[0-9]+, \[x[0-9]+, x[0-9]+, lsl 2\]
+** st1w z[0-9]+\.s, p[0-9]+, \[x[0-9]+, x[0-9]+, lsl 2\]
+** ...
+** ret
+*/
+void
+plain_fma_preserve_old_accumulator (int16_t *__restrict buf, int8_t len,
+ int *__restrict x, int *__restrict old_x)
+{
+ for (int16_t i = 0, y = 0; i < len; i++, y = i * 10)
+ {
+ int old = x[i];
+ x[i] = old + (int) y * (int) buf[i];
+ old_x[i] = old;
+ }
+}
diff --git a/gcc/tree-cfg.cc b/gcc/tree-cfg.cc
index ba1d4a125ad..bf630a715d9 100644
--- a/gcc/tree-cfg.cc
+++ b/gcc/tree-cfg.cc
@@ -4295,6 +4295,34 @@ verify_gimple_assign_ternary (gassign *stmt)
switch (rhs_code)
{
case WIDEN_MULT_PLUS_EXPR:
+ if (VECTOR_TYPE_P (rhs1_type))
+ {
+ if (!VECTOR_TYPE_P (lhs_type)
+ || !VECTOR_TYPE_P (rhs2_type)
+ || !VECTOR_TYPE_P (rhs3_type)
+ || !INTEGRAL_TYPE_P (TREE_TYPE (lhs_type))
+ || !INTEGRAL_TYPE_P (TREE_TYPE (rhs1_type))
+ || !INTEGRAL_TYPE_P (TREE_TYPE (rhs2_type))
+ || !INTEGRAL_TYPE_P (TREE_TYPE (rhs3_type))
+ || !useless_type_conversion_p (rhs1_type, rhs2_type)
+ || !useless_type_conversion_p (lhs_type, rhs3_type)
+ || maybe_ne (TYPE_VECTOR_SUBPARTS (lhs_type),
+ TYPE_VECTOR_SUBPARTS (rhs1_type))
+ || maybe_ne (TYPE_VECTOR_SUBPARTS (rhs1_type),
+ TYPE_VECTOR_SUBPARTS (rhs2_type))
+ || maybe_ne (TYPE_VECTOR_SUBPARTS (rhs2_type),
+ TYPE_VECTOR_SUBPARTS (rhs3_type)))
+ {
+ error ("type mismatch in %qs", code_name);
+ debug_generic_expr (lhs_type);
+ debug_generic_expr (rhs1_type);
+ debug_generic_expr (rhs2_type);
+ debug_generic_expr (rhs3_type);
+ return true;
+ }
+ break;
+ }
+ /* Fallthrough. */
case WIDEN_MULT_MINUS_EXPR:
if ((!INTEGRAL_TYPE_P (rhs1_type)
&& !FIXED_POINT_TYPE_P (rhs1_type))
diff --git a/gcc/tree.def b/gcc/tree.def
index 2e767dbe045..57ea0d1b007 100644
--- a/gcc/tree.def
+++ b/gcc/tree.def
@@ -1518,9 +1518,15 @@ DEFTREECODE (SAD_EXPR, "sad_expr", tcc_expression, 3)
DEFTREECODE (WIDEN_MULT_EXPR, "widen_mult_expr", tcc_binary, 2)
/* Widening multiply-accumulate.
- The first two arguments are of type t1.
- The third argument and the result are of type t2, such as t2 is at least
- twice the size of t1. t1 and t2 must be integral or fixed-point types.
+
+ For scalar forms, the first two arguments have integral or fixed-point
+ type t1, while the third argument and result have type t2, with t2 at
+ least twice as wide as t1.
+
+ Vector forms apply the same operation element-wise. The first two
+ arguments have type vector<N x t1> and the third argument and result
+ have type vector<N x t2>, where t1 and t2 are integral element types
+ and t2 is at least twice as wide as t1.
The expression is equivalent to a WIDEN_MULT_EXPR operation
of the first two operands followed by an add or subtract of the third
operand. */
--
2.43.0
