On Mon, 22 Jun 2026, Abhishek Kaushik wrote:
> Add optabs and internal functions for SVE2 SMLALB and UMLALB, and
> teach AArch64 instruction selection to use them for suitable IFN_FMA,
> IFN_COND_FMA and IFN_COND_ADD operations.
>
> The instruction selection code recognizes multiply-add forms where both
> multiplicands are known to fit in half-width signed or unsigned elements.
> For predicated forms, it also rebuilds the selected multiplicand so that
> inactive lanes are zero when needed, allowing the final MLALB operation to
> preserve the original merge value.
>
> Fixed-length SVE vectors are rejected for now because the new expanders use
> scalable SVE source modes.
>
> Bootstrapped and regtested on aarch64-linux-gnu.
optabs need documenting in md.texi, internal functions need documenting
in ifn.texi
> PR/target 122751
>
> gcc/ChangeLog:
>
> * config/aarch64/aarch64-sve2.md (smlalb<mode>4): New expander.
> (umlalb<mode>4): Likewise.
> (@aarch64_sve_add_smlalb_partial<mode>): New insn.
> (@aarch64_sve_add_umlalb_partial<mode>): Likewise.
> * config/aarch64/aarch64.cc: Include internal-fn.h.
> (aarch64_mlalb_vector_type_fits_half_p): New function.
> (aarch64_mlalb_extend_p): Likewise.
> (aarch64_mlalb_operand_fits_half_p): Likewise.
> (aarch64_mlalb_zeroing_mask_load_p): Likewise.
> (aarch64_mlalb_same_mode_accumulator_conversion_p): Likewise.
> (aarch64_strip_mlalb_accumulator_conversions): Likewise.
> (aarch64_cond_ifn_for_mlalb_arith): Likewise.
> (aarch64_build_mlalb_zeroing_op): Likewise.
> (aarch64_mlalb_source_element_type): Likewise.
> (aarch64_build_mlalb_source_op): Likewise.
> (aarch64_mlalb_mult_node_p): Likewise.
> (aarch64_mlalb_candidate): New struct.
> (aarch64_check_mlalb_candidate): New function.
> (aarch64_replace_with_mlalb): Likewise.
> (aarch64_try_cond_mlalb): Likewise.
> (aarch64_try_fma_mlalb): Likewise.
> (aarch64_instruction_selection): Try MLALB selection.
> * config/aarch64/iterators.md (VHALF_EL): New mode attribute.
> * internal-fn.def (SMLALB): New internal function.
> (UMLALB): Likewise.
> * optabs.def (smlalb_optab): New optab.
> (umlalb_optab): Likewise.
>
> gcc/testsuite/ChangeLog:
>
> * gcc.target/aarch64/sve2/mlalb-cond-add-selection.c: New test.
> * gcc.target/aarch64/sve2/mlalb-cond-fma-selection.c: New test.
> * gcc.target/aarch64/sve2/mlalb-fma-fixed-vector-length.c: New test.
> * gcc.target/aarch64/sve2/mlalb-fma-selection.c: New test.
> ---
> gcc/config/aarch64/aarch64-sve2.md | 69 ++
> gcc/config/aarch64/aarch64.cc | 634 ++++++++++++++++++
> gcc/config/aarch64/iterators.md | 5 +
> gcc/internal-fn.def | 2 +
> gcc/optabs.def | 2 +
> .../aarch64/sve2/mlalb-cond-add-selection.c | 276 ++++++++
> .../aarch64/sve2/mlalb-cond-fma-selection.c | 126 ++++
> .../sve2/mlalb-fma-fixed-vector-length.c | 16 +
> .../aarch64/sve2/mlalb-fma-selection.c | 134 ++++
> 9 files changed, 1264 insertions(+)
> create mode 100644
> gcc/testsuite/gcc.target/aarch64/sve2/mlalb-cond-add-selection.c
> create mode 100644
> gcc/testsuite/gcc.target/aarch64/sve2/mlalb-cond-fma-selection.c
> create mode 100644
> gcc/testsuite/gcc.target/aarch64/sve2/mlalb-fma-fixed-vector-length.c
> create mode 100644
> gcc/testsuite/gcc.target/aarch64/sve2/mlalb-fma-selection.c
>
> diff --git a/gcc/config/aarch64/aarch64-sve2.md
> b/gcc/config/aarch64/aarch64-sve2.md
> index 1d428619c07..6e1c302a946 100644
> --- a/gcc/config/aarch64/aarch64-sve2.md
> +++ b/gcc/config/aarch64/aarch64-sve2.md
> @@ -2696,6 +2696,75 @@
> ;; - UMLSLT
> ;; -------------------------------------------------------------------------
>
> +;; Signed MLA bottom-half operation.
> +(define_expand "smlalb<mode>4"
> + [(match_operand:SVE_FULL_HSDI 0 "register_operand")
> + (match_operand:SVE_FULL_HSDI 1 "register_operand")
> + (match_operand:<VHALF_EL> 2 "register_operand")
> + (match_operand:<VHALF_EL> 3 "register_operand")]
> + "TARGET_SVE2"
> + {
> + emit_insn (gen_aarch64_sve_add_smlalb_partial<mode>
> + (operands[0], operands[1], operands[2], operands[3]));
> + DONE;
> + }
> +)
> +
> +;; Unsigned MLA bottom-half operation.
> +(define_expand "umlalb<mode>4"
> + [(match_operand:SVE_FULL_HSDI 0 "register_operand")
> + (match_operand:SVE_FULL_HSDI 1 "register_operand")
> + (match_operand:<VHALF_EL> 2 "register_operand")
> + (match_operand:<VHALF_EL> 3 "register_operand")]
> + "TARGET_SVE2"
> + {
> + emit_insn (gen_aarch64_sve_add_umlalb_partial<mode>
> + (operands[0], operands[1], operands[2], operands[3]));
> + DONE;
> + }
> +)
> +
> +(define_insn "@aarch64_sve_add_smlalb_partial<mode>"
> + [(set (match_operand:SVE_FULL_HSDI 0 "register_operand" "=w,?&w")
> + (plus:SVE_FULL_HSDI
> + (unspec:SVE_FULL_HSDI
> + [(match_operand:<VHALF_EL> 2 "register_operand" "w,w")
> + (match_operand:<VHALF_EL> 3 "register_operand" "w,w")]
> + UNSPEC_SMULLB)
> + (match_operand:SVE_FULL_HSDI 1 "register_operand" "0,w")))]
> + "TARGET_SVE2"
> + {
> + if (which_alternative == 0)
> + return "smlalb\t%0.<Vetype>, %2.<Ventype>, "
> + "%3.<Ventype>";
> + return "movprfx\t%0, %1\;smlalb\t%0.<Vetype>, "
> + "%2.<Ventype>, %3.<Ventype>";
> + }
> + [(set_attr "movprfx" "*,yes")
> + (set_attr "sve_type" "sve_int_mul")]
> +)
> +
> +;; Unsigned MLA bottom-half operation with same-lane-count source operands.
> +(define_insn "@aarch64_sve_add_umlalb_partial<mode>"
> + [(set (match_operand:SVE_FULL_HSDI 0 "register_operand" "=w,?&w")
> + (plus:SVE_FULL_HSDI
> + (unspec:SVE_FULL_HSDI
> + [(match_operand:<VHALF_EL> 2 "register_operand" "w,w")
> + (match_operand:<VHALF_EL> 3 "register_operand" "w,w")]
> + UNSPEC_UMULLB)
> + (match_operand:SVE_FULL_HSDI 1 "register_operand" "0,w")))]
> + "TARGET_SVE2"
> + {
> + if (which_alternative == 0)
> + return "umlalb\t%0.<Vetype>, %2.<Ventype>, "
> + "%3.<Ventype>";
> + return "movprfx\t%0, %1\;umlalb\t%0.<Vetype>, "
> + "%2.<Ventype>, %3.<Ventype>";
> + }
> + [(set_attr "movprfx" "*,yes")
> + (set_attr "sve_type" "sve_int_mul")]
> +)
> +
> ;; Non-saturating MLA operations.
> (define_insn "@aarch64_sve_add_<sve_int_op><mode>"
> [(set (match_operand:SVE_FULL_HSDI 0 "register_operand")
> diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc
> index c9403ed94e7..f094e026d12 100644
> --- a/gcc/config/aarch64/aarch64.cc
> +++ b/gcc/config/aarch64/aarch64.cc
> @@ -100,6 +100,7 @@
> #include "ipa-fnsummary.h"
> #include "hash-map.h"
> #include "ifcvt.h"
> +#include "internal-fn.h"
> #include "aarch64-sched-dispatch.h"
> #include "aarch64-json-tunings-printer.h"
> #include "aarch64-json-tunings-parser.h"
> @@ -2229,6 +2230,629 @@ aarch64_preferred_else_value (unsigned, tree,
> unsigned int nops, tree *ops)
> return nops == 3 ? ops[2] : ops[0];
> }
>
> +/* Return true if TYPE is a vector integer type whose elements are known
> + to fit in a SOURCE_PRECISION-bit integer of the selected signedness. */
> +
> +static bool
> +aarch64_mlalb_vector_type_fits_half_p (tree type,
> + unsigned int source_precision,
> + bool unsigned_p)
> +{
> + if (!VECTOR_TYPE_P (type))
> + return false;
> +
> + tree element_type = TREE_TYPE (type);
> + if (!INTEGRAL_TYPE_P (element_type))
> + return false;
> +
> + unsigned int precision = TYPE_PRECISION (element_type);
> + if (unsigned_p)
> + return TYPE_UNSIGNED (element_type) && precision <= source_precision;
> +
> + return (precision < source_precision
> + || (precision == source_precision && !TYPE_UNSIGNED (element_type)));
> +}
> +
> +/* Return true if ASSIGN widens or preserves vector integer element width.
> */
> +
> +static bool
> +aarch64_mlalb_extend_p (gassign *assign)
> +{
> + tree_code code = gimple_assign_rhs_code (assign);
> + if (!CONVERT_EXPR_CODE_P (code))
> + return false;
> +
> + tree type = TREE_TYPE (gimple_assign_lhs (assign));
> + tree inner_type = TREE_TYPE (gimple_assign_rhs1 (assign));
> + if (!VECTOR_TYPE_P (type) || !VECTOR_TYPE_P (inner_type))
> + return false;
> +
> + tree element_type = TREE_TYPE (type);
> + tree inner_element_type = TREE_TYPE (inner_type);
> + if (!INTEGRAL_TYPE_P (element_type)
> + || !INTEGRAL_TYPE_P (inner_element_type))
> + return false;
> +
> + unsigned int precision = TYPE_PRECISION (element_type);
> + unsigned int inner_precision = TYPE_PRECISION (inner_element_type);
> +
> + return precision >= inner_precision;
> +}
> +
> +/* Return true if OP is known locally to fit in a SOURCE_PRECISION-bit
> integer
> + element of the selected signedness. When nonnull, set FITTING_OP to the
> + operand that provides the half-width value. */
> +
> +static bool
> +aarch64_mlalb_operand_fits_half_p (tree op, unsigned int source_precision,
> + bool unsigned_p,
> + tree *fitting_op = nullptr,
> + unsigned int depth = 0)
> +{
> + if (!op || depth > 20)
> + return false;
> +
> + tree type = TREE_TYPE (op);
> + if (aarch64_mlalb_vector_type_fits_half_p (type, source_precision,
> + unsigned_p))
> + {
> + if (fitting_op)
> + *fitting_op = op;
> + return true;
> + }
> +
> + if (TREE_CODE (op) != SSA_NAME)
> + return false;
> +
> + gassign *assign = dyn_cast<gassign *> (SSA_NAME_DEF_STMT (op));
> + if (!assign || !aarch64_mlalb_extend_p (assign))
> + return false;
> +
> + return aarch64_mlalb_operand_fits_half_p (gimple_assign_rhs1 (assign),
> + source_precision, unsigned_p,
> + fitting_op, depth + 1);
> +}
> +
> +/* Return true if CALL is a zeroing MASK_LOAD under MASK. */
> +
> +static bool
> +aarch64_mlalb_zeroing_mask_load_p (gcall *call, tree mask)
> +{
> + if (!gimple_call_internal_p (call, IFN_MASK_LOAD))
> + return false;
> +
> + tree call_mask
> + = gimple_call_arg (call, internal_fn_mask_index (IFN_MASK_LOAD));
> + if (!operand_equal_p (call_mask, mask, 0))
> + return false;
> +
> + tree else_value
> + = gimple_call_arg (call, internal_fn_else_index (IFN_MASK_LOAD));
> + return initializer_zerop (else_value);
> +}
> +
> +/* Return true if converting INNER_TYPE to TYPE leaves the MLALB accumulator
> + in the same vector mode. */
> +
> +static bool
> +aarch64_mlalb_same_mode_accumulator_conversion_p (tree type, tree inner_type)
> +{
> + if (!VECTOR_TYPE_P (type) || !VECTOR_TYPE_P (inner_type))
> + return false;
> +
> + tree element_type = TREE_TYPE (type);
> + tree inner_element_type = TREE_TYPE (inner_type);
> + if (!INTEGRAL_TYPE_P (element_type)
> + || !INTEGRAL_TYPE_P (inner_element_type))
> + return false;
> +
> + return (known_eq (TYPE_VECTOR_SUBPARTS (type),
> + TYPE_VECTOR_SUBPARTS (inner_type))
> + && (TYPE_PRECISION (element_type)
> + == TYPE_PRECISION (inner_element_type)));
> +}
> +
> +/* Strip conversions from OP that do not change accumulator vector mode. */
> +
> +static tree
> +aarch64_strip_mlalb_accumulator_conversions (tree op, unsigned int depth = 0)
> +{
> + if (!op || depth > 20)
> + return op;
> +
> + tree inner = NULL_TREE;
> + if (TREE_CODE (op) == SSA_NAME)
> + {
> + gassign *assign = dyn_cast<gassign *> (SSA_NAME_DEF_STMT (op));
> + if (!assign)
> + return op;
> +
> + tree_code code = gimple_assign_rhs_code (assign);
> + if (code == VIEW_CONVERT_EXPR)
> + {
> + inner = gimple_assign_rhs1 (assign);
> + if (TREE_CODE (inner) == VIEW_CONVERT_EXPR)
> + inner = TREE_OPERAND (inner, 0);
> + }
> + else if (CONVERT_EXPR_CODE_P (code))
> + inner = gimple_assign_rhs1 (assign);
> + }
> +
> + if (inner
> + && aarch64_mlalb_same_mode_accumulator_conversion_p (TREE_TYPE (op),
> + TREE_TYPE (inner)))
> + return aarch64_strip_mlalb_accumulator_conversions (inner, depth + 1);
> +
> + return op;
> +}
> +
> +/* Return the conditional form of CODE, or IFN_LAST if CODE isn't supported
> + by this transformation. */
> +
> +static internal_fn
> +aarch64_cond_ifn_for_mlalb_arith (tree_code code)
> +{
> + switch (code)
> + {
> + case PLUS_EXPR:
> + return IFN_COND_ADD;
> + case MINUS_EXPR:
> + return IFN_COND_SUB;
> + default:
> + return IFN_LAST;
> + }
> +}
> +
> +/* Rebuild OP so that the part of the expression depending on a zeroing
> + MASK_LOAD under MASK is also zero when MASK is false. Set FOUND_P if OP
> + already has, or could be rebuilt to have, that property. If FORCE_ZEROING
> + is true, insert a final conditional add with zero so that OP itself is
> zero
> + on inactive lanes. */
> +
> +static void
> +aarch64_build_mlalb_zeroing_op (tree op, tree mask, gimple_stmt_iterator
> *gsi,
> + tree &new_op, bool &found_p,
> + bool force_zeroing = false,
> + unsigned int depth = 0)
> +{
> + new_op = op;
> + found_p = false;
> +
> + if (!op || depth > 20)
> + return;
> +
> + auto force_zero = [&] ()
> + {
> + tree type = TREE_TYPE (op);
> + if (!direct_internal_fn_supported_p (IFN_COND_ADD, type,
> + OPTIMIZE_FOR_SPEED))
> + return;
> + tree zero = build_zero_cst (type);
> + gcall *cond_stmt = gimple_build_call_internal (IFN_COND_ADD, 4,
> + mask, op, zero, zero);
> + gimple_call_set_nothrow (cond_stmt, true);
> + new_op = make_ssa_name (type);
> + gimple_call_set_lhs (cond_stmt, new_op);
> + gsi_insert_before (gsi, cond_stmt, GSI_SAME_STMT);
> + found_p = true;
> + };
> +
> + if (TREE_CODE (op) != SSA_NAME)
> + {
> + if (force_zeroing)
> + force_zero ();
> + return;
> + }
> +
> + gimple *def_stmt = SSA_NAME_DEF_STMT (op);
> + if (gcall *call = dyn_cast<gcall *> (def_stmt))
> + {
> + if (aarch64_mlalb_zeroing_mask_load_p (call, mask))
> + found_p = true;
> + else if (force_zeroing)
> + force_zero ();
> + return;
> + }
> +
> + gassign *assign = dyn_cast<gassign *> (def_stmt);
> + if (!assign)
> + {
> + if (force_zeroing)
> + force_zero ();
> + return;
> + }
> +
> + tree_code code = gimple_assign_rhs_code (assign);
> + if (code == FIX_TRUNC_EXPR
> + || code == VIEW_CONVERT_EXPR
> + || CONVERT_EXPR_CODE_P (code))
> + {
> + tree inner;
> + bool inner_found;
> + tree rhs = gimple_assign_rhs1 (assign);
> + if (code == VIEW_CONVERT_EXPR)
> + rhs = TREE_OPERAND (rhs, 0);
> +
> + aarch64_build_mlalb_zeroing_op (rhs, mask, gsi, inner, inner_found,
> + force_zeroing,
> + depth + 1);
> +
> + if (inner_found)
> + {
> + tree type = TREE_TYPE (op);
> + gimple *stmt;
> + if (code == VIEW_CONVERT_EXPR)
> + stmt = gimple_build_assign (make_ssa_name (type),
> + fold_build1 (VIEW_CONVERT_EXPR, type,
> + inner));
> + else
> + stmt = gimple_build_assign (make_ssa_name (type), code, inner);
> + gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
> + new_op = gimple_get_lhs (stmt);
> + found_p = true;
> + return;
> + }
> +
> + if (force_zeroing)
> + force_zero ();
> + return;
> + }
> +
> + internal_fn ifn = aarch64_cond_ifn_for_mlalb_arith (code);
> + if (ifn != IFN_LAST
> + && direct_internal_fn_supported_p (ifn, TREE_TYPE (op),
> + OPTIMIZE_FOR_SPEED))
> + {
> + tree lhs;
> + tree rhs;
> + bool lhs_found;
> + bool rhs_found;
> + aarch64_build_mlalb_zeroing_op (gimple_assign_rhs1 (assign), mask,
> + gsi, lhs, lhs_found,
> + false, depth + 1);
> + aarch64_build_mlalb_zeroing_op (gimple_assign_rhs2 (assign), mask,
> + gsi, rhs, rhs_found,
> + false, depth + 1);
> +
> + if (lhs_found || rhs_found)
> + {
> + tree type = TREE_TYPE (op);
> + if (!types_compatible_p (TREE_TYPE (lhs), type))
> + {
> + gimple *convert = gimple_build_assign (make_ssa_name (type),
> + NOP_EXPR, lhs);
> + gsi_insert_before (gsi, convert, GSI_SAME_STMT);
> + lhs = gimple_get_lhs (convert);
> + }
> +
> + if (!types_compatible_p (TREE_TYPE (rhs), type))
> + {
> + gimple *convert = gimple_build_assign (make_ssa_name (type),
> + NOP_EXPR, rhs);
> + gsi_insert_before (gsi, convert, GSI_SAME_STMT);
> + rhs = gimple_get_lhs (convert);
> + }
> +
> + /* Reuse an operand that is already zero on inactive lanes as
> + the merge value. This avoids reconstructing zeroing
> + separately. */
> + tree merge = lhs_found ? lhs : rhs;
> + gcall *cond_stmt = gimple_build_call_internal (ifn, 4, mask, lhs,
> + rhs, merge);
> + gimple_call_set_nothrow (cond_stmt, true);
> + new_op = make_ssa_name (type);
> + gimple_call_set_lhs (cond_stmt, new_op);
> + gsi_insert_before (gsi, cond_stmt, GSI_SAME_STMT);
> + found_p = true;
> + return;
> + }
> + }
> +
> + if (force_zeroing)
> + force_zero ();
> +}
> +
> +/* Return the half-width element type for an MLALB result element
> + precision, or NULL_TREE if the precision is not supported. */
> +
> +static tree
> +aarch64_mlalb_source_element_type (unsigned int result_precision,
> + bool unsigned_p)
> +{
> + switch (result_precision)
> + {
> + case 16:
> + return unsigned_p ? unsigned_char_type_node : signed_char_type_node;
> + case 32:
> + return unsigned_p ? short_unsigned_type_node : short_integer_type_node;
> + case 64:
> + return unsigned_p ? unsigned_type_node : integer_type_node;
> + default:
> + return NULL_TREE;
> + }
> +}
> +
> +/* Convert OP to SOURCE_TYPE before GSI. */
> +
> +static tree
> +aarch64_build_mlalb_source_op (tree op, tree source_type,
> + gimple_stmt_iterator *gsi)
> +{
> + if (types_compatible_p (TREE_TYPE (op), source_type))
> + return op;
> +
> + tree lhs = make_ssa_name (source_type);
> + tree op_type = TREE_TYPE (op);
> + gimple *stmt;
> + if (VECTOR_TYPE_P (op_type)
> + && known_eq (TYPE_VECTOR_SUBPARTS (op_type),
> + TYPE_VECTOR_SUBPARTS (source_type))
> + && (TYPE_PRECISION (TREE_TYPE (op_type))
> + == TYPE_PRECISION (TREE_TYPE (source_type))))
> + stmt = gimple_build_assign (lhs, fold_build1 (VIEW_CONVERT_EXPR,
> + source_type, op));
> + else
> + stmt = gimple_build_assign (lhs, NOP_EXPR, op);
> + gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
> + return lhs;
> +}
> +
> +static bool
> +aarch64_mlalb_mult_node_p (tree op, tree result_type,
> + tree &multiplicand0, tree &multiplicand1)
> +{
> + unsigned int result_precision = TYPE_PRECISION (TREE_TYPE (result_type));
> +
> + while (TREE_CODE (op) == SSA_NAME)
> + {
> + gimple *def_stmt = SSA_NAME_DEF_STMT (op);
> + gassign *assign = dyn_cast<gassign *> (def_stmt);
> + if (!assign)
> + return false;
> +
> + tree_code code = gimple_assign_rhs_code (assign);
> + if (code == VIEW_CONVERT_EXPR)
> + op = TREE_OPERAND (gimple_assign_rhs1 (assign), 0);
> + else if (aarch64_mlalb_extend_p (assign))
> + op = gimple_assign_rhs1 (assign);
> + else if (code == MULT_EXPR)
> + {
> + tree mult_type = TREE_TYPE (gimple_assign_lhs (assign));
> + if (!VECTOR_TYPE_P (mult_type))
> + return false;
> +
> + tree mult_element_type = TREE_TYPE (mult_type);
> + unsigned int mult_precision = TYPE_PRECISION (mult_element_type);
> + /* If the original product is narrower than the accumulator, MLALB
> + computes more high product bits than the original expression kept.
> + This is only equivalent when overflow of the original product is
> + undefined; otherwise the original expression has defined wrapping
> + semantics that MLALB would not preserve. */
> + if (mult_precision < result_precision
> + && !TYPE_OVERFLOW_UNDEFINED (mult_element_type))
> + return false;
> +
> + multiplicand0 = gimple_assign_rhs1 (assign);
> + multiplicand1 = gimple_assign_rhs2 (assign);
> + return true;
> + }
> + else
> + return false;
> + }
> +
> + return false;
> +}
> +
> +/* Information that is common to all MLALB candidate forms once the
> + multiplicands and accumulator have been identified. */
> +
> +struct aarch64_mlalb_candidate
> +{
> + tree lhs;
> + tree result_type;
> + unsigned int source_precision;
> + bool unsigned_p;
> + internal_fn ifn;
> +};
> +
> +/* Check whether CALL can be replaced by an MLALB-like internal function
> using
> + MULTIPLICAND0 and MULTIPLICAND1. Fill in CANDIDATE on success. */
> +
> +static bool
> +aarch64_check_mlalb_candidate (gcall *call, tree &multiplicand0,
> + tree &multiplicand1,
> + aarch64_mlalb_candidate &candidate)
> +{
> + candidate.lhs = gimple_call_lhs (call);
> +
> + /* MLALB is an SVE2 instruction, so do not match unless the target can
> + expand the replacement. */
> + if (!TARGET_SVE2)
> + return false;
> +
> + /* The result must be an SVE integer vector whose elements can be split
> into
> + bottom halves for MLALB. */
> + if (!candidate.lhs
> + || !VECTOR_TYPE_P (TREE_TYPE (candidate.lhs))
> + || !aarch64_sve_mode_p (TYPE_MODE (TREE_TYPE (candidate.lhs))))
> + return false;
> +
> + candidate.result_type = TREE_TYPE (candidate.lhs);
> + /* The current MLALB optabs only have scalable SVE expanders. Fixed-length
> + SVE vector types can have a mode that maps to one of those expanders,
> but
> + building the half-width source type below would give fixed-length source
> + operands, which the scalable expander cannot accept. */
> + if (TYPE_VECTOR_SUBPARTS (candidate.result_type).is_constant ())
> + return false;
> +
> + tree element_type = TREE_TYPE (candidate.result_type);
> + if (!INTEGRAL_TYPE_P (element_type))
> + return false;
> +
> + unsigned int element_precision = TYPE_PRECISION (element_type);
> +
> + if (!aarch64_mlalb_source_element_type (element_precision, false))
> + return false;
> +
> + unsigned int source_precision = element_precision / 2;
> +
> + tree signed_multiplicand0;
> + tree signed_multiplicand1;
> + tree unsigned_multiplicand0;
> + tree unsigned_multiplicand1;
> +
> + if (aarch64_mlalb_operand_fits_half_p (multiplicand0, source_precision,
> + false, &signed_multiplicand0)
> + && aarch64_mlalb_operand_fits_half_p (multiplicand1, source_precision,
> + false, &signed_multiplicand1))
> + {
> + candidate.unsigned_p = false;
> + multiplicand0 = signed_multiplicand0;
> + multiplicand1 = signed_multiplicand1;
> + }
> + else if (aarch64_mlalb_operand_fits_half_p (multiplicand0,
> source_precision,
> + true, &unsigned_multiplicand0)
> + && aarch64_mlalb_operand_fits_half_p (multiplicand1,
> + source_precision, true,
> + &unsigned_multiplicand1))
> + {
> + candidate.unsigned_p = true;
> + multiplicand0 = unsigned_multiplicand0;
> + multiplicand1 = unsigned_multiplicand1;
> + }
> + else
> + return false;
> +
> + candidate.source_precision = source_precision;
> + candidate.ifn = candidate.unsigned_p ? IFN_UMLALB : IFN_SMLALB;
> + return direct_internal_fn_supported_p (candidate.ifn,
> candidate.result_type,
> + OPTIMIZE_FOR_SPEED);
> +}
> +
> +/* Replace the current statement with the MLALB operation described by
> + CANDIDATE. MULTIPLICAND0 and MULTIPLICAND1 are already known to fit in
> the
> + selected half-width source type; ACCUMULATOR is the value to preserve and
> + accumulate into. */
> +
> +static void
> +aarch64_replace_with_mlalb (gimple_stmt_iterator *gsi,
> + const aarch64_mlalb_candidate &candidate,
> + tree accumulator, tree multiplicand0,
> + tree multiplicand1)
> +{
> + tree result_type = candidate.result_type;
> + bool unsigned_p = candidate.unsigned_p;
> +
> + /* The IFN expects the same-lane-count source mode used by the
> + bottom-half instruction. For example, a VNx4SI result uses
> + VNx4HI sources, not the fully packed VNx8HI mode. */
> + tree result_element_type = TREE_TYPE (result_type);
> + tree source_element_type
> + = aarch64_mlalb_source_element_type (TYPE_PRECISION
> (result_element_type),
> + unsigned_p);
> + tree source_type = build_vector_type (source_element_type,
> + TYPE_VECTOR_SUBPARTS (result_type));
> + multiplicand0 = aarch64_build_mlalb_source_op (multiplicand0, source_type,
> + gsi);
> + multiplicand1 = aarch64_build_mlalb_source_op (multiplicand1, source_type,
> + gsi);
> +
> + /* The accumulator must stay in the wide result mode. Only strip casts
> that
> + keep that mode unchanged, such as signedness-only vector conversions.
> */
> + accumulator = aarch64_strip_mlalb_accumulator_conversions (accumulator);
> +
> + gcall *mlalb = gimple_build_call_internal (candidate.ifn, 3, accumulator,
> + multiplicand0,
> + multiplicand1);
> + gimple_call_set_lhs (mlalb, candidate.lhs);
> + gimple_call_set_nothrow (mlalb, true);
> + gsi_replace (gsi, mlalb, true);
> +}
> +
> +static bool
> +aarch64_try_cond_mlalb (gimple *stmt, gimple_stmt_iterator *gsi)
> +{
> + gcall *call = as_a <gcall *> (stmt);
> + tree mask, multiplicand0, multiplicand1, merge;
> +
> + if (gimple_call_internal_fn (call) == IFN_COND_ADD)
> + {
> + mask = gimple_call_arg (call, 0);
> + tree addend0 = gimple_call_arg (call, 1);
> + tree addend1 = gimple_call_arg (call, 2);
> + merge = gimple_call_arg (call, 3);
> +
> + /* The conditional add must be a merging operation whose inactive lanes
> + keep the accumulator. */
> + if (!operand_equal_p (merge, addend1))
> + return false;
> +
> + /* addend0 must be a multiplication node where each multiplicand fits
> in
> + half as many bits as the result. */
> + tree result_type = TREE_TYPE (gimple_call_lhs (call));
> + if (!aarch64_mlalb_mult_node_p (addend0, result_type, multiplicand0,
> + multiplicand1))
> + return false;
> + }
> + else if (gimple_call_internal_fn (call) == IFN_COND_FMA)
> + {
> + mask = gimple_call_arg (call, 0);
> + multiplicand0 = gimple_call_arg (call, 1);
> + multiplicand1 = gimple_call_arg (call, 2);
> + tree addend = gimple_call_arg (call, 3);
> + merge = gimple_call_arg (call, 4);
> +
> + /* The fused multiply-add must be a merging operation whose inactive
> + lanes keep the accumulator. */
> + if (!operand_equal_p (addend, merge, 0))
> + return false;
> + }
> + else
> + gcc_unreachable ();
> +
> + aarch64_mlalb_candidate candidate;
> + if (!aarch64_check_mlalb_candidate (call, multiplicand0, multiplicand1,
> + candidate))
> + return false;
> +
> + tree new_multiplicand0;
> + tree new_multiplicand1 = multiplicand1;
> + bool zeroing0;
> + bool zeroing1;
> + aarch64_build_mlalb_zeroing_op (multiplicand0, mask, gsi,
> + new_multiplicand0, zeroing0);
> + if (!zeroing0)
> + {
> + aarch64_build_mlalb_zeroing_op (multiplicand1, mask, gsi,
> + new_multiplicand1, zeroing1,
> + true);
> + if (!zeroing1)
> + return false;
> + }
> +
> + aarch64_replace_with_mlalb (gsi, candidate, merge, new_multiplicand0,
> + new_multiplicand1);
> + return true;
> +}
> +
> +static bool
> +aarch64_try_fma_mlalb (gimple *stmt, gimple_stmt_iterator *gsi)
> +{
> + gcall *call = as_a <gcall *> (stmt);
> + tree multiplicand0 = gimple_call_arg (call, 0);
> + tree multiplicand1 = gimple_call_arg (call, 1);
> + tree addend = gimple_call_arg (call, 2);
> +
> + aarch64_mlalb_candidate candidate;
> + if (!aarch64_check_mlalb_candidate (call, multiplicand0, multiplicand1,
> + candidate))
> + return false;
> +
> + aarch64_replace_with_mlalb (gsi, candidate, addend, multiplicand0,
> + multiplicand1);
> + return true;
> +}
> +
> /* Implement TARGET_INSTRUCTION_SELECTION. The target hook is used to
> change generic sequences to a form AArch64 has an easier time expanding
> instructions for. It's not supposed to be used for generic rewriting that
> @@ -2238,6 +2862,16 @@ static bool
> aarch64_instruction_selection (function * /* fun */, gimple_stmt_iterator
> *gsi)
> {
> auto stmt = gsi_stmt (*gsi);
> +
> + if ((gimple_call_internal_p (stmt, IFN_COND_FMA)
> + || gimple_call_internal_p (stmt, IFN_COND_ADD))
> + && aarch64_try_cond_mlalb (stmt, gsi))
> + return true;
> +
> + if (gimple_call_internal_p (stmt, IFN_FMA)
> + && aarch64_try_fma_mlalb (stmt, gsi))
> + return true;
> +
> gassign *assign = dyn_cast<gassign *> (stmt);
>
> if (!assign)
> diff --git a/gcc/config/aarch64/iterators.md b/gcc/config/aarch64/iterators.md
> index 2d1522a348a..c1c92c4a9ce 100644
> --- a/gcc/config/aarch64/iterators.md
> +++ b/gcc/config/aarch64/iterators.md
> @@ -2139,6 +2139,11 @@
> (VNx2DI "VNx4SI") (VNx2DF "VNx4SF")
> (VNx8SI "VNx8HI") (VNx16SI "VNx16QI")
> (VNx8DI "VNx8HI")])
> +
> +;; Half-element modes with the same number of units.
> +(define_mode_attr VHALF_EL [(VNx8HI "VNx8QI")
> + (VNx4SI "VNx4HI")
> + (VNx2DI "VNx2SI")])
> (define_mode_attr Vnarrow [(VNx8HI "vnx16qi")
> (VNx4SI "vnx8hi") (VNx4SF "vnx8hf")
> (VNx2DI "vnx4si") (VNx2DF "vnx4sf")
> diff --git a/gcc/internal-fn.def b/gcc/internal-fn.def
> index af9f92950c7..d79c140f86d 100644
> --- a/gcc/internal-fn.def
> +++ b/gcc/internal-fn.def
> @@ -471,6 +471,8 @@ DEF_INTERNAL_WIDENING_OPTAB_FN (VEC_WIDEN_ABD,
> first,
> vec_widen_sabd, vec_widen_uabd,
> binary)
> +DEF_INTERNAL_OPTAB_FN (SMLALB, ECF_CONST | ECF_NOTHROW, smlalb, ternary)
> +DEF_INTERNAL_OPTAB_FN (UMLALB, ECF_CONST | ECF_NOTHROW, umlalb, ternary)
> DEF_INTERNAL_OPTAB_FN (VEC_FMADDSUB, ECF_CONST, vec_fmaddsub, ternary)
> DEF_INTERNAL_OPTAB_FN (VEC_FMSUBADD, ECF_CONST, vec_fmsubadd, ternary)
>
> diff --git a/gcc/optabs.def b/gcc/optabs.def
> index 7ccea18543f..a0225fe4b68 100644
> --- a/gcc/optabs.def
> +++ b/gcc/optabs.def
> @@ -476,6 +476,8 @@ OPTAB_D (vec_widen_smult_even_optab,
> "vec_widen_smult_even_$a")
> OPTAB_D (vec_widen_smult_hi_optab, "vec_widen_smult_hi_$a")
> OPTAB_D (vec_widen_smult_lo_optab, "vec_widen_smult_lo_$a")
> OPTAB_D (vec_widen_smult_odd_optab, "vec_widen_smult_odd_$a")
> +OPTAB_D (smlalb_optab, "smlalb$a4")
> +OPTAB_D (umlalb_optab, "umlalb$a4")
> OPTAB_D (vec_widen_ssub_optab, "vec_widen_ssub_$a")
> OPTAB_D (vec_widen_ssub_hi_optab, "vec_widen_ssub_hi_$a")
> OPTAB_D (vec_widen_ssub_lo_optab, "vec_widen_ssub_lo_$a")
> diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/mlalb-cond-add-selection.c
> b/gcc/testsuite/gcc.target/aarch64/sve2/mlalb-cond-add-selection.c
> new file mode 100644
> index 00000000000..1209ca2e563
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/aarch64/sve2/mlalb-cond-add-selection.c
> @@ -0,0 +1,276 @@
> +/* { dg-do compile } */
> +/* { dg-options "-O2 -fdump-tree-vect -fdump-tree-isel" } */
> +
> +#include <stdint.h>
> +
> +/* Test COND_ADD selection of SMLALB for signed 8-bit multiplicands
> accumulated
> + into a 16-bit reduction. */
> +short
> +cond_add_s8_to_s16_reduction (int8_t *a, int8_t *b, int len)
> +{
> + if (len < 0 || len >= 128)
> + return 0;
> +
> + short x = 0;
> + for (int8_t i = 0; i < len; ++i)
> + x += (short) a[i] * (short) b[i];
> +
> + return x;
> +}
> +
> +/* Test COND_ADD selection of UMLALB for unsigned 8-bit multiplicands
> + accumulated into a 16-bit reduction. */
> +unsigned short
> +cond_add_u8_to_u16_reduction (uint8_t *a, uint8_t *b, int len)
> +{
> + if (len < 0 || len >= 128)
> + return 0;
> +
> + unsigned short x = 0;
> + for (uint8_t i = 0; i < len; ++i)
> + x += (unsigned short) a[i] * (unsigned short) b[i];
> +
> + return x;
> +}
> +
> +/* Test COND_ADD selection of SMLALB for signed 16-bit multiplicands
> + accumulated into a 32-bit reduction. */
> +int
> +cond_add_s16_to_s32_reduction (int16_t *a, int16_t *b, int len)
> +{
> + if (len < 0 || len >= 256)
> + return 0;
> +
> + int x = 0;
> + for (int16_t i = 0; i < len; ++i)
> + x += (int) a[i] * (int) b[i];
> +
> + return x;
> +}
> +
> +/* Test COND_ADD selection of UMLALB for unsigned 16-bit multiplicands
> + accumulated into a 32-bit reduction. */
> +unsigned int
> +cond_add_u16_to_u32_reduction (uint16_t *a, uint16_t *b, int len)
> +{
> + if (len < 0 || len >= 256)
> + return 0;
> +
> + unsigned int x = 0;
> + for (uint16_t i = 0; i < len; ++i)
> + x += (unsigned int) a[i] * (unsigned int) b[i];
> +
> + return x;
> +}
> +
> +/* Test COND_ADD selection of SMLALB for signed 32-bit multiplicands
> + accumulated into a 64-bit reduction. */
> +long int
> +cond_add_s32_to_s64_reduction (int32_t *a, int32_t *b, int len)
> +{
> + if (len < 0 || len >= 65536)
> + return 0;
> +
> + long int x = 0;
> + for (int32_t i = 0; i < len; ++i)
> + x += (long int) a[i] * (long int) b[i];
> +
> + return x;
> +}
> +
> +/* Test COND_ADD selection of UMLALB for unsigned 32-bit multiplicands
> + accumulated into a 64-bit reduction. */
> +unsigned long int
> +cond_add_u32_to_u64_reduction (uint32_t *a, uint32_t *b, int len)
> +{
> + if (len < 0 || len >= 65536)
> + return 0;
> +
> + unsigned long int x = 0;
> + for (uint32_t i = 0; i < len; ++i)
> + x += (unsigned long int) a[i] * (unsigned long int) b[i];
> +
> + return x;
> +}
> +
> +/* Test that a 32-bit signed product widened to a 64-bit accumulator
> + selects SMLALB, since signed overflow is undefined. */
> +long
> +cond_add_s32_product_widened_to_s64 (int32_t *a, int32_t *b, int len)
> +{
> + if (len < 0 || len >= 65536)
> + return 0;
> +
> + long x = 0;
> + for (int32_t i = 0; i < len; ++i)
> + {
> + int32_t product = a[i] * b[i];
> + x += (long) product;
> + }
> + return x;
> +}
> +
> +/* Test that a 32-bit unsigned product widened to a 64-bit accumulator is not
> + selected, since UMLALB would compute the full 32x32 product. */
> +unsigned long
> +cond_add_reject_u32_product_widened_to_u64 (uint32_t *a, uint32_t *b, int
> len)
> +{
> + if (len < 0 || len >= 65536)
> + return 0;
> +
> + unsigned long x = 0;
> + for (uint32_t i = 0; i < len; ++i)
> + {
> + uint32_t product = a[i] * b[i];
> + x += (unsigned long) product;
> + }
> + return x;
> +}
> +
> +/* Test a signed source reinterpretation that is still valid for SMLALB. */
> +int
> +cond_add_signed_reinterpret_to_smlalb (uint16_t *buf, int len)
> +{
> + if (len < 0 || len >= 256)
> + return 0;
> +
> + int x = 0;
> + for (int16_t i = 0, y = 0; i < len; ++i, y = i * 10)
> + {
> + int16_t v = (int16_t) buf[i];
> + x += y * (int) v;
> + }
> + return x;
> +}
> +
> +/* Test an unsigned source reinterpretation that selects UMLALB rather than
> + SMLALB. */
> +unsigned int
> +cond_add_unsigned_reinterpret_to_umlalb (int16_t *buf, int len)
> +{
> + if (len < 0 || len >= 256)
> + return 0;
> +
> + unsigned int x = 0;
> + for (uint16_t i = 0, y = 0; i < len; ++i, y = i * 10)
> + {
> + uint16_t v = (uint16_t) buf[i];
> + x += (unsigned int) y * (unsigned int) v;
> + }
> + return x;
> +}
> +
> +/* Test that aarch64_mlalb_mult_node_p can walk through a VIEW_CONVERT_EXPR
> + from a signed product to an unsigned accumulator type. */
> +unsigned int
> +cond_add_signed_product_to_unsigned_accumulator (int16_t *a, int16_t *b,
> + int len)
> +{
> + if (len < 0 || len >= 256)
> + return 0;
> +
> + unsigned int x = 0;
> + for (int16_t i = 0; i < len; ++i)
> + {
> + int product = (int) a[i] * (int) b[i];
> + x += (unsigned int) product;
> + }
> + return x;
> +}
> +
> +/* Test that an unsigned product reinterpreted to signed selects UMLALB
> rather
> + than SMLALB. */
> +int
> +cond_add_unsigned_product_to_signed_accumulator (uint16_t *a, uint16_t *b,
> + int len)
> +{
> + if (len < 0 || len >= 256)
> + return 0;
> +
> + int x = 0;
> + for (int16_t i = 0; i < len; ++i)
> + {
> + unsigned int product = (unsigned int) a[i] * (unsigned int) b[i];
> + x += (int) product;
> + }
> + return x;
> +}
> +
> +/* Test that truncating a product before accumulation is not selected.
> + MLALB computes the full product, rather than the low half of a product.
> */
> +int
> +cond_add_reject_truncated_product_low_half (int16_t *a, int16_t *b, int len)
> +{
> + if (len < 0 || len >= 256)
> + return 0;
> +
> + int x = 0;
> + for (int16_t i = 0; i < len; ++i)
> + {
> + int product = (int) a[i] * (int) b[i];
> + x += (int16_t) product;
> + }
> + return x;
> +}
> +
> +/* Test COND_ADD selection of SMLALB when a signed IV square is widened
> + to a 64-bit accumulator. */
> +long
> +cond_add_signed_iv_square_s64 (int len)
> +{
> + if (len <= 0 || len >= 255)
> + return 0;
> +
> + long res = 0;
> + for (int i = 0, y = 0; i < len; i++, y = i * 10)
> + res += y * y;
> + return res;
> +}
> +
> +/* Test COND_ADD selection of SMLALB when a signed IV is widened to long
> + before multiplication. */
> +long
> +cond_add_signed_long_iv_square_s64 (int len)
> +{
> + if (len <= 0 || len >= 255)
> + return 0;
> +
> + long res = 0;
> + for (int i = 0, y = 0; i < len; i++, y = i * 10)
> + res += (long) y * (long) y;
> + return res;
> +}
> +
> +/* Test COND_ADD selection of UMLALB when an unsigned IV is widened to
> + unsigned long before multiplication. */
> +unsigned long
> +cond_add_unsigned_long_iv_square_u64 (unsigned len)
> +{
> + if (len <= 0 || len >= 255)
> + return 0;
> +
> + unsigned long res = 0;
> + for (unsigned i = 0, y = 0; i < len; i++, y = i * 10)
> + res += (unsigned long) y * (unsigned long) y;
> + return res;
> +}
> +
> +/* Test that an unsigned IV square widened to a 64-bit accumulator is not
> + selected when the product itself is formed in 32 bits. */
> +unsigned long
> +cond_add_reject_unsigned_iv_square_u64 (unsigned len)
> +{
> + if (len <= 0 || len >= 255)
> + return 0;
> +
> + unsigned long res = 0;
> + for (unsigned i = 0, y = 0; i < len; i++, y = i * 10)
> + res += y * y;
> + return res;
> +}
> +
> +/* { dg-final { scan-tree-dump-times {\.COND_ADD} 17 "vect" } } */
> +/* { dg-final { scan-tree-dump-times {\.SMLALB} 8 "isel" } } */
> +/* { dg-final { scan-tree-dump-times {\.UMLALB} 6 "isel" } } */
> +/* { dg-final { scan-assembler-times {\tsmlalb\t} 8 } } */
> +/* { dg-final { scan-assembler-times {\tumlalb\t} 6 } } */
> diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/mlalb-cond-fma-selection.c
> b/gcc/testsuite/gcc.target/aarch64/sve2/mlalb-cond-fma-selection.c
> new file mode 100644
> index 00000000000..a09d74a4450
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/aarch64/sve2/mlalb-cond-fma-selection.c
> @@ -0,0 +1,126 @@
> +/* { dg-do compile } */
> +/* { dg-options "-O2 -fdump-tree-widening_mul -fdump-tree-isel" } */
> +
> +#include <stdint.h>
> +
> +/* Test COND_FMA selection of SMLALB with a signed character load adjusted by
> + a constant offset. */
> +int
> +cond_fma_signed_char_digit_offset (char *buf, int len)
> +{
> + if (len < 0 || len >= 256)
> + return 0;
> +
> + int x = 0;
> + for (int16_t i = 0, y = 0; i < len; i++, y = i * 10)
> + x += y * (buf[i] - '0');
> + return x;
> +}
> +
> +/* Test COND_FMA selection of UMLALB when one unsigned multiplicand is built
> + from an IV plus a constant. */
> +uint32_t
> +cond_fma_unsigned_iv_plus_constant (uint8_t *buf, int len)
> +{
> + if (len < 0 || len >= 256)
> + return 0;
> +
> + uint32_t x = 0;
> + for (uint16_t i = 0, y = 0; i < len; i++, y = i * 10)
> + x += (uint32_t) (y + 2) * (uint32_t) buf[i];
> + return x;
> +}
> +
> +/* Test COND_FMA selection of SMLALB when one signed multiplicand is built
> + from an IV minus a constant. */
> +int32_t
> +cond_fma_signed_iv_minus_constant (int8_t *buf, int len)
> +{
> + if (len < 0 || len >= 256)
> + return 0;
> +
> + int32_t x = 0;
> + for (int16_t i = 0, y = 0; i < len; i++, y = i * 10)
> + x += (int32_t) (y - 2) * (int32_t) buf[i];
> + return x;
> +}
> +
> +/* Test COND_FMA selection of UMLALB for an unsigned short IV multiplied by
> an
> + unsigned short load. */
> +unsigned
> +cond_fma_unsigned_short_load (uint16_t *buf, uint8_t len)
> +{
> + unsigned x = 0;
> + for (uint16_t i = 0, y = 0; i < len; i++, y = i * 10)
> + x += buf[i] * y;
> +
> + return x;
> +}
> +
> +/* Test COND_FMA selection of SMLALB when the loaded multiplicand is produced
> + by float-to-int conversion. */
> +long
> +cond_fma_float_to_int_load (float *buf, uint16_t len)
> +{
> + long x = 0;
> + for (int i = 0, y = 0; i < len; i++, y = i * 10)
> + x += (long) y * (long) (int) buf[i];
> + return x;
> +}
> +
> +/* Test COND_FMA selection of SMLALB when the loaded multiplicand is produced
> + by double-to-int conversion. */
> +long
> +cond_fma_double_to_int_load (double *buf, uint16_t len)
> +{
> + long x = 0;
> + for (int i = 0, y = 0; i < len; i++, y = i * 10)
> + x += (long) y * (long) (int) buf[i];
> + return x;
> +}
> +
> +/* Test COND_FMA selection of SMLALB when one signed character load is
> + adjusted by a constant before multiplication. */
> +int
> +cond_fma_two_signed_char_loads (char *__restrict buf1,
> + char *__restrict buf2, uint8_t len)
> +{
> + int x = 0;
> + for (int i = 0; i < len; ++i)
> + x += (buf1[i] - 2) * (buf2[i]);
> + return x;
> +}
> +
> +/* Test COND_FMA selection of SMLALB when both multiplicands are the same
> + signed IV. */
> +int
> +cond_fma_signed_iv_square (int len)
> +{
> + if (len <= 0 || len >= 255)
> + return 0;
> +
> + int res = 0;
> + for (short i = 0, y = 0; i < len; i++, y = i * 10)
> + res += y * y;
> + return res;
> +}
> +
> +/* Test COND_FMA selection of UMLALB when both multiplicands are the same
> + unsigned IV. */
> +unsigned
> +cond_fma_unsigned_iv_square (unsigned len)
> +{
> + if (len == 0 || len >= 255)
> + return 0;
> +
> + unsigned res = 0;
> + for (unsigned short i = 0, y = 0; i < len; i++, y = i * 10)
> + res += y * y;
> + return res;
> +}
> +
> +/* { dg-final { scan-tree-dump-times {\.COND_FMA} 9 "widening_mul" } } */
> +/* { dg-final { scan-tree-dump-times {\.SMLALB} 6 "isel" } } */
> +/* { dg-final { scan-tree-dump-times {\.UMLALB} 3 "isel" } } */
> +/* { dg-final { scan-assembler-times {\tsmlalb\t} 6 } } */
> +/* { dg-final { scan-assembler-times {\tumlalb\t} 3 } } */
> diff --git
> a/gcc/testsuite/gcc.target/aarch64/sve2/mlalb-fma-fixed-vector-length.c
> b/gcc/testsuite/gcc.target/aarch64/sve2/mlalb-fma-fixed-vector-length.c
> new file mode 100644
> index 00000000000..8da14824ac2
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/aarch64/sve2/mlalb-fma-fixed-vector-length.c
> @@ -0,0 +1,16 @@
> +/* { dg-do compile } */
> +/* { dg-options "-O2 -msve-vector-bits=128 -fdump-tree-isel" } */
> +
> +typedef unsigned short uint16_t;
> +
> +/* Test that MLALB selection rejects fixed-length SVE result types until the
> + optab supports the matching fixed-length half-width source operands. */
> +void __attribute__ ((noinline, noclone))
> +bar_test (int *__restrict buf, uint16_t len, long *x)
> +{
> + for (int i = 0, y = 0; i < len; i++, y = i * 10)
> + x[i] += (long) y * (long) buf[i];
> +}
> +
> +/* { dg-final { scan-tree-dump-not {\.SMLALB} "isel" } } */
> +/* { dg-final { scan-assembler-not {\tsmlalb\t} } } */
> diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/mlalb-fma-selection.c
> b/gcc/testsuite/gcc.target/aarch64/sve2/mlalb-fma-selection.c
> new file mode 100644
> index 00000000000..d21968be204
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/aarch64/sve2/mlalb-fma-selection.c
> @@ -0,0 +1,134 @@
> +/* { dg-do compile } */
> +/* { dg-options "-O2 -fdump-tree-isel" } */
> +
> +#include <stdint.h>
> +
> +/* Test plain FMA selection of SMLALB with a signed character load adjusted
> by
> + a constant offset. */
> +void
> +plain_fma_signed_char_digit_offset (char *__restrict buf, int8_t len, int *x)
> +{
> + for (int16_t i = 0, y = 0; i < len; i++, y = i * 10)
> + x[i] += (int) y * (buf[i] - '0');
> +}
> +
> +/* Test plain FMA selection of SMLALB for signed 32-bit multiplicands
> + accumulated into a 64-bit memory operand. */
> +void
> +plain_fma_s32_to_s64_memory_accumulator (int *__restrict buf, uint16_t len,
> + long *x)
> +{
> + for (int i = 0, y = 0; i < len; i++, y = i * 10)
> + x[i] += (long) y * (long) buf[i];
> +}
> +
> +/* Test plain FMA selection of SMLALB when a float-to-int conversion
> + feeds one multiplicand. */
> +void
> +plain_fma_float_to_int_load (float *__restrict buf, uint16_t len, long *x)
> +{
> + for (int i = 0, y = 0; i < len; i++, y = i * 10)
> + x[i] += (long) y * (long) (int) buf[i];
> +}
> +
> +/* Test plain FMA selection of SMLALB when one multiplicand is built
> + by subtracting a constant from a loaded value. */
> +void
> +plain_fma_sub_loaded_source (int16_t *__restrict buf, int8_t len, int *x)
> +{
> + for (int16_t i = 0, y = 0; i < len; i++, y = i * 10)
> + {
> + int16_t v = (int16_t) (buf[i] - 7);
> + x[i] += (int) y * (int) v;
> + }
> +}
> +
> +/* Test plain FMA selection of SMLALB when one multiplicand is built
> + by multiplying a loaded value by a constant. */
> +void
> +plain_fma_mul_loaded_source (int16_t *__restrict buf, int8_t len, int *x)
> +{
> + for (int16_t i = 0, y = 0; i < len; i++, y = i * 10)
> + {
> + int16_t v = (int16_t) (buf[i] * 3);
> + x[i] += (int) y * (int) v;
> + }
> +}
> +
> +/* Test plain FMA selection of SMLALB when one multiplicand is built
> + by adding a constant to a loaded value. */
> +void
> +plain_fma_add_loaded_source (int16_t *__restrict buf, int8_t len, int *x)
> +{
> + for (int16_t i = 0, y = 0; i < len; i++, y = i * 10)
> + {
> + int16_t v = (int16_t) (buf[i] + 7);
> + x[i] += (int) y * (int) v;
> + }
> +}
> +
> +/* Test plain FMA selection of UMLALB when one unsigned multiplicand is
> + built by adding a constant to a loaded value. */
> +void
> +plain_fma_unsigned_add_loaded_source (uint16_t *__restrict buf, int8_t len,
> + unsigned int *x)
> +{
> + for (uint16_t i = 0, y = 0; i < len; i++, y = i * 10)
> + {
> + uint16_t v = (uint16_t) (buf[i] + 7);
> + x[i] += (unsigned int) y * (unsigned int) v;
> + }
> +}
> +
> +/* Test plain FMA selection of SMLALB after stripping a signedness conversion
> + around the accumulator. */
> +void
> +plain_fma_accumulator_signedness_convert (int16_t *__restrict buf, int8_t
> len,
> + unsigned int *x)
> +{
> + for (int16_t i = 0, y = 0; i < len; i++, y = i * 10)
> + x[i] = (unsigned int) ((int) x[i] + (int) y * (int) buf[i]);
> +}
> +
> +/* Test plain FMA selection of UMLALB for an unsigned short IV multiplied by
> an
> + unsigned short load. */
> +void
> +plain_fma_unsigned_short_load (uint16_t *__restrict buf, uint16_t len,
> + unsigned int *x)
> +{
> + for (uint16_t i = 0, y = 0; i < len; i++, y = i * 10)
> + x[i] += (unsigned int) y * (unsigned int) buf[i];
> +}
> +
> +/* Test the same accumulator signedness-conversion path with an explicitly
> + named old value kept live in the loop body. */
> +void
> +plain_fma_named_old_accumulator_convert (int16_t *__restrict buf, int8_t len,
> + unsigned int *x)
> +{
> + for (int16_t i = 0, y = 0; i < len; i++, y = i * 10)
> + {
> + int old = (int) x[i];
> + x[i] = (unsigned int) (old + (int) y * (int) buf[i]);
> + }
> +}
> +
> +/* Test that plain FMA selection still allows the old accumulator value to be
> + used after the selected MLALB operation. */
> +void
> +plain_fma_preserve_old_accumulator (int16_t *__restrict buf, int8_t len,
> + int *__restrict x, int *__restrict old_x)
> +{
> + for (int16_t i = 0, y = 0; i < len; i++, y = i * 10)
> + {
> + int old = x[i];
> + x[i] = old + (int) y * (int) buf[i];
> + old_x[i] = old;
> + }
> +}
> +
> +/* { dg-final { scan-tree-dump-times {\.SMLALB} 9 "isel" } } */
> +/* { dg-final { scan-tree-dump-times {\.UMLALB} 2 "isel" } } */
> +/* { dg-final { scan-assembler-times {\tsmlalb\t} 9 } } */
> +/* { dg-final { scan-assembler-times {\tumlalb\t} 2 } } */
> +/* { dg-final { scan-assembler {\tmovprfx\t} } } */
>
--
Richard Biener <[email protected]>
SUSE Software Solutions Germany GmbH,
Frankenstrasse 146, 90461 Nuernberg, Germany;
GF: Jochen Jaser, Andrew McDonald, Werner Knoblich; (HRB 36809, AG Nuernberg)
