Add optabs and internal functions for SVE2 SMLALB and UMLALB, and
teach AArch64 instruction selection to use them for suitable IFN_FMA,
IFN_COND_FMA and IFN_COND_ADD operations.
The instruction selection code recognizes multiply-add forms where both
multiplicands are known to fit in half-width signed or unsigned elements.
For predicated forms, it also rebuilds the selected multiplicand so that
inactive lanes are zero when needed, allowing the final MLALB operation to
preserve the original merge value.
Fixed-length SVE vectors are rejected for now because the new expanders use
scalable SVE source modes.
Bootstrapped and regtested on aarch64-linux-gnu.
PR/target 122751
gcc/ChangeLog:
* config/aarch64/aarch64-sve2.md (smlalb<mode>4): New expander.
(umlalb<mode>4): Likewise.
(@aarch64_sve_add_smlalb_partial<mode>): New insn.
(@aarch64_sve_add_umlalb_partial<mode>): Likewise.
* config/aarch64/aarch64.cc: Include internal-fn.h.
(aarch64_mlalb_vector_type_fits_half_p): New function.
(aarch64_mlalb_extend_p): Likewise.
(aarch64_mlalb_operand_fits_half_p): Likewise.
(aarch64_mlalb_zeroing_mask_load_p): Likewise.
(aarch64_mlalb_same_mode_accumulator_conversion_p): Likewise.
(aarch64_strip_mlalb_accumulator_conversions): Likewise.
(aarch64_cond_ifn_for_mlalb_arith): Likewise.
(aarch64_build_mlalb_zeroing_op): Likewise.
(aarch64_mlalb_source_element_type): Likewise.
(aarch64_build_mlalb_source_op): Likewise.
(aarch64_mlalb_mult_node_p): Likewise.
(aarch64_mlalb_candidate): New struct.
(aarch64_check_mlalb_candidate): New function.
(aarch64_replace_with_mlalb): Likewise.
(aarch64_try_cond_mlalb): Likewise.
(aarch64_try_fma_mlalb): Likewise.
(aarch64_instruction_selection): Try MLALB selection.
* config/aarch64/iterators.md (VHALF_EL): New mode attribute.
* internal-fn.def (SMLALB): New internal function.
(UMLALB): Likewise.
* optabs.def (smlalb_optab): New optab.
(umlalb_optab): Likewise.
gcc/testsuite/ChangeLog:
* gcc.target/aarch64/sve2/mlalb-cond-add-selection.c: New test.
* gcc.target/aarch64/sve2/mlalb-cond-fma-selection.c: New test.
* gcc.target/aarch64/sve2/mlalb-fma-fixed-vector-length.c: New test.
* gcc.target/aarch64/sve2/mlalb-fma-selection.c: New test.
---
gcc/config/aarch64/aarch64-sve2.md | 69 ++
gcc/config/aarch64/aarch64.cc | 634 ++++++++++++++++++
gcc/config/aarch64/iterators.md | 5 +
gcc/internal-fn.def | 2 +
gcc/optabs.def | 2 +
.../aarch64/sve2/mlalb-cond-add-selection.c | 276 ++++++++
.../aarch64/sve2/mlalb-cond-fma-selection.c | 126 ++++
.../sve2/mlalb-fma-fixed-vector-length.c | 16 +
.../aarch64/sve2/mlalb-fma-selection.c | 134 ++++
9 files changed, 1264 insertions(+)
create mode 100644
gcc/testsuite/gcc.target/aarch64/sve2/mlalb-cond-add-selection.c
create mode 100644
gcc/testsuite/gcc.target/aarch64/sve2/mlalb-cond-fma-selection.c
create mode 100644
gcc/testsuite/gcc.target/aarch64/sve2/mlalb-fma-fixed-vector-length.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/sve2/mlalb-fma-selection.c
diff --git a/gcc/config/aarch64/aarch64-sve2.md
b/gcc/config/aarch64/aarch64-sve2.md
index 1d428619c07..6e1c302a946 100644
--- a/gcc/config/aarch64/aarch64-sve2.md
+++ b/gcc/config/aarch64/aarch64-sve2.md
@@ -2696,6 +2696,75 @@
;; - UMLSLT
;; -------------------------------------------------------------------------
+;; Signed MLA bottom-half operation.
+(define_expand "smlalb<mode>4"
+ [(match_operand:SVE_FULL_HSDI 0 "register_operand")
+ (match_operand:SVE_FULL_HSDI 1 "register_operand")
+ (match_operand:<VHALF_EL> 2 "register_operand")
+ (match_operand:<VHALF_EL> 3 "register_operand")]
+ "TARGET_SVE2"
+ {
+ emit_insn (gen_aarch64_sve_add_smlalb_partial<mode>
+ (operands[0], operands[1], operands[2], operands[3]));
+ DONE;
+ }
+)
+
+;; Unsigned MLA bottom-half operation.
+(define_expand "umlalb<mode>4"
+ [(match_operand:SVE_FULL_HSDI 0 "register_operand")
+ (match_operand:SVE_FULL_HSDI 1 "register_operand")
+ (match_operand:<VHALF_EL> 2 "register_operand")
+ (match_operand:<VHALF_EL> 3 "register_operand")]
+ "TARGET_SVE2"
+ {
+ emit_insn (gen_aarch64_sve_add_umlalb_partial<mode>
+ (operands[0], operands[1], operands[2], operands[3]));
+ DONE;
+ }
+)
+
+(define_insn "@aarch64_sve_add_smlalb_partial<mode>"
+ [(set (match_operand:SVE_FULL_HSDI 0 "register_operand" "=w,?&w")
+ (plus:SVE_FULL_HSDI
+ (unspec:SVE_FULL_HSDI
+ [(match_operand:<VHALF_EL> 2 "register_operand" "w,w")
+ (match_operand:<VHALF_EL> 3 "register_operand" "w,w")]
+ UNSPEC_SMULLB)
+ (match_operand:SVE_FULL_HSDI 1 "register_operand" "0,w")))]
+ "TARGET_SVE2"
+ {
+ if (which_alternative == 0)
+ return "smlalb\t%0.<Vetype>, %2.<Ventype>, "
+ "%3.<Ventype>";
+ return "movprfx\t%0, %1\;smlalb\t%0.<Vetype>, "
+ "%2.<Ventype>, %3.<Ventype>";
+ }
+ [(set_attr "movprfx" "*,yes")
+ (set_attr "sve_type" "sve_int_mul")]
+)
+
+;; Unsigned MLA bottom-half operation with same-lane-count source operands.
+(define_insn "@aarch64_sve_add_umlalb_partial<mode>"
+ [(set (match_operand:SVE_FULL_HSDI 0 "register_operand" "=w,?&w")
+ (plus:SVE_FULL_HSDI
+ (unspec:SVE_FULL_HSDI
+ [(match_operand:<VHALF_EL> 2 "register_operand" "w,w")
+ (match_operand:<VHALF_EL> 3 "register_operand" "w,w")]
+ UNSPEC_UMULLB)
+ (match_operand:SVE_FULL_HSDI 1 "register_operand" "0,w")))]
+ "TARGET_SVE2"
+ {
+ if (which_alternative == 0)
+ return "umlalb\t%0.<Vetype>, %2.<Ventype>, "
+ "%3.<Ventype>";
+ return "movprfx\t%0, %1\;umlalb\t%0.<Vetype>, "
+ "%2.<Ventype>, %3.<Ventype>";
+ }
+ [(set_attr "movprfx" "*,yes")
+ (set_attr "sve_type" "sve_int_mul")]
+)
+
;; Non-saturating MLA operations.
(define_insn "@aarch64_sve_add_<sve_int_op><mode>"
[(set (match_operand:SVE_FULL_HSDI 0 "register_operand")
diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc
index c9403ed94e7..f094e026d12 100644
--- a/gcc/config/aarch64/aarch64.cc
+++ b/gcc/config/aarch64/aarch64.cc
@@ -100,6 +100,7 @@
#include "ipa-fnsummary.h"
#include "hash-map.h"
#include "ifcvt.h"
+#include "internal-fn.h"
#include "aarch64-sched-dispatch.h"
#include "aarch64-json-tunings-printer.h"
#include "aarch64-json-tunings-parser.h"
@@ -2229,6 +2230,629 @@ aarch64_preferred_else_value (unsigned, tree, unsigned
int nops, tree *ops)
return nops == 3 ? ops[2] : ops[0];
}
+/* Return true if TYPE is a vector integer type whose elements are known
+ to fit in a SOURCE_PRECISION-bit integer of the selected signedness. */
+
+static bool
+aarch64_mlalb_vector_type_fits_half_p (tree type,
+ unsigned int source_precision,
+ bool unsigned_p)
+{
+ if (!VECTOR_TYPE_P (type))
+ return false;
+
+ tree element_type = TREE_TYPE (type);
+ if (!INTEGRAL_TYPE_P (element_type))
+ return false;
+
+ unsigned int precision = TYPE_PRECISION (element_type);
+ if (unsigned_p)
+ return TYPE_UNSIGNED (element_type) && precision <= source_precision;
+
+ return (precision < source_precision
+ || (precision == source_precision && !TYPE_UNSIGNED (element_type)));
+}
+
+/* Return true if ASSIGN widens or preserves vector integer element width. */
+
+static bool
+aarch64_mlalb_extend_p (gassign *assign)
+{
+ tree_code code = gimple_assign_rhs_code (assign);
+ if (!CONVERT_EXPR_CODE_P (code))
+ return false;
+
+ tree type = TREE_TYPE (gimple_assign_lhs (assign));
+ tree inner_type = TREE_TYPE (gimple_assign_rhs1 (assign));
+ if (!VECTOR_TYPE_P (type) || !VECTOR_TYPE_P (inner_type))
+ return false;
+
+ tree element_type = TREE_TYPE (type);
+ tree inner_element_type = TREE_TYPE (inner_type);
+ if (!INTEGRAL_TYPE_P (element_type)
+ || !INTEGRAL_TYPE_P (inner_element_type))
+ return false;
+
+ unsigned int precision = TYPE_PRECISION (element_type);
+ unsigned int inner_precision = TYPE_PRECISION (inner_element_type);
+
+ return precision >= inner_precision;
+}
+
+/* Return true if OP is known locally to fit in a SOURCE_PRECISION-bit integer
+ element of the selected signedness. When nonnull, set FITTING_OP to the
+ operand that provides the half-width value. */
+
+static bool
+aarch64_mlalb_operand_fits_half_p (tree op, unsigned int source_precision,
+ bool unsigned_p,
+ tree *fitting_op = nullptr,
+ unsigned int depth = 0)
+{
+ if (!op || depth > 20)
+ return false;
+
+ tree type = TREE_TYPE (op);
+ if (aarch64_mlalb_vector_type_fits_half_p (type, source_precision,
+ unsigned_p))
+ {
+ if (fitting_op)
+ *fitting_op = op;
+ return true;
+ }
+
+ if (TREE_CODE (op) != SSA_NAME)
+ return false;
+
+ gassign *assign = dyn_cast<gassign *> (SSA_NAME_DEF_STMT (op));
+ if (!assign || !aarch64_mlalb_extend_p (assign))
+ return false;
+
+ return aarch64_mlalb_operand_fits_half_p (gimple_assign_rhs1 (assign),
+ source_precision, unsigned_p,
+ fitting_op, depth + 1);
+}
+
+/* Return true if CALL is a zeroing MASK_LOAD under MASK. */
+
+static bool
+aarch64_mlalb_zeroing_mask_load_p (gcall *call, tree mask)
+{
+ if (!gimple_call_internal_p (call, IFN_MASK_LOAD))
+ return false;
+
+ tree call_mask
+ = gimple_call_arg (call, internal_fn_mask_index (IFN_MASK_LOAD));
+ if (!operand_equal_p (call_mask, mask, 0))
+ return false;
+
+ tree else_value
+ = gimple_call_arg (call, internal_fn_else_index (IFN_MASK_LOAD));
+ return initializer_zerop (else_value);
+}
+
+/* Return true if converting INNER_TYPE to TYPE leaves the MLALB accumulator
+ in the same vector mode. */
+
+static bool
+aarch64_mlalb_same_mode_accumulator_conversion_p (tree type, tree inner_type)
+{
+ if (!VECTOR_TYPE_P (type) || !VECTOR_TYPE_P (inner_type))
+ return false;
+
+ tree element_type = TREE_TYPE (type);
+ tree inner_element_type = TREE_TYPE (inner_type);
+ if (!INTEGRAL_TYPE_P (element_type)
+ || !INTEGRAL_TYPE_P (inner_element_type))
+ return false;
+
+ return (known_eq (TYPE_VECTOR_SUBPARTS (type),
+ TYPE_VECTOR_SUBPARTS (inner_type))
+ && (TYPE_PRECISION (element_type)
+ == TYPE_PRECISION (inner_element_type)));
+}
+
+/* Strip conversions from OP that do not change accumulator vector mode. */
+
+static tree
+aarch64_strip_mlalb_accumulator_conversions (tree op, unsigned int depth = 0)
+{
+ if (!op || depth > 20)
+ return op;
+
+ tree inner = NULL_TREE;
+ if (TREE_CODE (op) == SSA_NAME)
+ {
+ gassign *assign = dyn_cast<gassign *> (SSA_NAME_DEF_STMT (op));
+ if (!assign)
+ return op;
+
+ tree_code code = gimple_assign_rhs_code (assign);
+ if (code == VIEW_CONVERT_EXPR)
+ {
+ inner = gimple_assign_rhs1 (assign);
+ if (TREE_CODE (inner) == VIEW_CONVERT_EXPR)
+ inner = TREE_OPERAND (inner, 0);
+ }
+ else if (CONVERT_EXPR_CODE_P (code))
+ inner = gimple_assign_rhs1 (assign);
+ }
+
+ if (inner
+ && aarch64_mlalb_same_mode_accumulator_conversion_p (TREE_TYPE (op),
+ TREE_TYPE (inner)))
+ return aarch64_strip_mlalb_accumulator_conversions (inner, depth + 1);
+
+ return op;
+}
+
+/* Return the conditional form of CODE, or IFN_LAST if CODE isn't supported
+ by this transformation. */
+
+static internal_fn
+aarch64_cond_ifn_for_mlalb_arith (tree_code code)
+{
+ switch (code)
+ {
+ case PLUS_EXPR:
+ return IFN_COND_ADD;
+ case MINUS_EXPR:
+ return IFN_COND_SUB;
+ default:
+ return IFN_LAST;
+ }
+}
+
+/* Rebuild OP so that the part of the expression depending on a zeroing
+ MASK_LOAD under MASK is also zero when MASK is false. Set FOUND_P if OP
+ already has, or could be rebuilt to have, that property. If FORCE_ZEROING
+ is true, insert a final conditional add with zero so that OP itself is zero
+ on inactive lanes. */
+
+static void
+aarch64_build_mlalb_zeroing_op (tree op, tree mask, gimple_stmt_iterator *gsi,
+ tree &new_op, bool &found_p,
+ bool force_zeroing = false,
+ unsigned int depth = 0)
+{
+ new_op = op;
+ found_p = false;
+
+ if (!op || depth > 20)
+ return;
+
+ auto force_zero = [&] ()
+ {
+ tree type = TREE_TYPE (op);
+ if (!direct_internal_fn_supported_p (IFN_COND_ADD, type,
+ OPTIMIZE_FOR_SPEED))
+ return;
+ tree zero = build_zero_cst (type);
+ gcall *cond_stmt = gimple_build_call_internal (IFN_COND_ADD, 4,
+ mask, op, zero, zero);
+ gimple_call_set_nothrow (cond_stmt, true);
+ new_op = make_ssa_name (type);
+ gimple_call_set_lhs (cond_stmt, new_op);
+ gsi_insert_before (gsi, cond_stmt, GSI_SAME_STMT);
+ found_p = true;
+ };
+
+ if (TREE_CODE (op) != SSA_NAME)
+ {
+ if (force_zeroing)
+ force_zero ();
+ return;
+ }
+
+ gimple *def_stmt = SSA_NAME_DEF_STMT (op);
+ if (gcall *call = dyn_cast<gcall *> (def_stmt))
+ {
+ if (aarch64_mlalb_zeroing_mask_load_p (call, mask))
+ found_p = true;
+ else if (force_zeroing)
+ force_zero ();
+ return;
+ }
+
+ gassign *assign = dyn_cast<gassign *> (def_stmt);
+ if (!assign)
+ {
+ if (force_zeroing)
+ force_zero ();
+ return;
+ }
+
+ tree_code code = gimple_assign_rhs_code (assign);
+ if (code == FIX_TRUNC_EXPR
+ || code == VIEW_CONVERT_EXPR
+ || CONVERT_EXPR_CODE_P (code))
+ {
+ tree inner;
+ bool inner_found;
+ tree rhs = gimple_assign_rhs1 (assign);
+ if (code == VIEW_CONVERT_EXPR)
+ rhs = TREE_OPERAND (rhs, 0);
+
+ aarch64_build_mlalb_zeroing_op (rhs, mask, gsi, inner, inner_found,
+ force_zeroing,
+ depth + 1);
+
+ if (inner_found)
+ {
+ tree type = TREE_TYPE (op);
+ gimple *stmt;
+ if (code == VIEW_CONVERT_EXPR)
+ stmt = gimple_build_assign (make_ssa_name (type),
+ fold_build1 (VIEW_CONVERT_EXPR, type,
+ inner));
+ else
+ stmt = gimple_build_assign (make_ssa_name (type), code, inner);
+ gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
+ new_op = gimple_get_lhs (stmt);
+ found_p = true;
+ return;
+ }
+
+ if (force_zeroing)
+ force_zero ();
+ return;
+ }
+
+ internal_fn ifn = aarch64_cond_ifn_for_mlalb_arith (code);
+ if (ifn != IFN_LAST
+ && direct_internal_fn_supported_p (ifn, TREE_TYPE (op),
+ OPTIMIZE_FOR_SPEED))
+ {
+ tree lhs;
+ tree rhs;
+ bool lhs_found;
+ bool rhs_found;
+ aarch64_build_mlalb_zeroing_op (gimple_assign_rhs1 (assign), mask,
+ gsi, lhs, lhs_found,
+ false, depth + 1);
+ aarch64_build_mlalb_zeroing_op (gimple_assign_rhs2 (assign), mask,
+ gsi, rhs, rhs_found,
+ false, depth + 1);
+
+ if (lhs_found || rhs_found)
+ {
+ tree type = TREE_TYPE (op);
+ if (!types_compatible_p (TREE_TYPE (lhs), type))
+ {
+ gimple *convert = gimple_build_assign (make_ssa_name (type),
+ NOP_EXPR, lhs);
+ gsi_insert_before (gsi, convert, GSI_SAME_STMT);
+ lhs = gimple_get_lhs (convert);
+ }
+
+ if (!types_compatible_p (TREE_TYPE (rhs), type))
+ {
+ gimple *convert = gimple_build_assign (make_ssa_name (type),
+ NOP_EXPR, rhs);
+ gsi_insert_before (gsi, convert, GSI_SAME_STMT);
+ rhs = gimple_get_lhs (convert);
+ }
+
+ /* Reuse an operand that is already zero on inactive lanes as
+ the merge value. This avoids reconstructing zeroing
+ separately. */
+ tree merge = lhs_found ? lhs : rhs;
+ gcall *cond_stmt = gimple_build_call_internal (ifn, 4, mask, lhs,
+ rhs, merge);
+ gimple_call_set_nothrow (cond_stmt, true);
+ new_op = make_ssa_name (type);
+ gimple_call_set_lhs (cond_stmt, new_op);
+ gsi_insert_before (gsi, cond_stmt, GSI_SAME_STMT);
+ found_p = true;
+ return;
+ }
+ }
+
+ if (force_zeroing)
+ force_zero ();
+}
+
+/* Return the half-width element type for an MLALB result element
+ precision, or NULL_TREE if the precision is not supported. */
+
+static tree
+aarch64_mlalb_source_element_type (unsigned int result_precision,
+ bool unsigned_p)
+{
+ switch (result_precision)
+ {
+ case 16:
+ return unsigned_p ? unsigned_char_type_node : signed_char_type_node;
+ case 32:
+ return unsigned_p ? short_unsigned_type_node : short_integer_type_node;
+ case 64:
+ return unsigned_p ? unsigned_type_node : integer_type_node;
+ default:
+ return NULL_TREE;
+ }
+}
+
+/* Convert OP to SOURCE_TYPE before GSI. */
+
+static tree
+aarch64_build_mlalb_source_op (tree op, tree source_type,
+ gimple_stmt_iterator *gsi)
+{
+ if (types_compatible_p (TREE_TYPE (op), source_type))
+ return op;
+
+ tree lhs = make_ssa_name (source_type);
+ tree op_type = TREE_TYPE (op);
+ gimple *stmt;
+ if (VECTOR_TYPE_P (op_type)
+ && known_eq (TYPE_VECTOR_SUBPARTS (op_type),
+ TYPE_VECTOR_SUBPARTS (source_type))
+ && (TYPE_PRECISION (TREE_TYPE (op_type))
+ == TYPE_PRECISION (TREE_TYPE (source_type))))
+ stmt = gimple_build_assign (lhs, fold_build1 (VIEW_CONVERT_EXPR,
+ source_type, op));
+ else
+ stmt = gimple_build_assign (lhs, NOP_EXPR, op);
+ gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
+ return lhs;
+}
+
+static bool
+aarch64_mlalb_mult_node_p (tree op, tree result_type,
+ tree &multiplicand0, tree &multiplicand1)
+{
+ unsigned int result_precision = TYPE_PRECISION (TREE_TYPE (result_type));
+
+ while (TREE_CODE (op) == SSA_NAME)
+ {
+ gimple *def_stmt = SSA_NAME_DEF_STMT (op);
+ gassign *assign = dyn_cast<gassign *> (def_stmt);
+ if (!assign)
+ return false;
+
+ tree_code code = gimple_assign_rhs_code (assign);
+ if (code == VIEW_CONVERT_EXPR)
+ op = TREE_OPERAND (gimple_assign_rhs1 (assign), 0);
+ else if (aarch64_mlalb_extend_p (assign))
+ op = gimple_assign_rhs1 (assign);
+ else if (code == MULT_EXPR)
+ {
+ tree mult_type = TREE_TYPE (gimple_assign_lhs (assign));
+ if (!VECTOR_TYPE_P (mult_type))
+ return false;
+
+ tree mult_element_type = TREE_TYPE (mult_type);
+ unsigned int mult_precision = TYPE_PRECISION (mult_element_type);
+ /* If the original product is narrower than the accumulator, MLALB
+ computes more high product bits than the original expression kept.
+ This is only equivalent when overflow of the original product is
+ undefined; otherwise the original expression has defined wrapping
+ semantics that MLALB would not preserve. */
+ if (mult_precision < result_precision
+ && !TYPE_OVERFLOW_UNDEFINED (mult_element_type))
+ return false;
+
+ multiplicand0 = gimple_assign_rhs1 (assign);
+ multiplicand1 = gimple_assign_rhs2 (assign);
+ return true;
+ }
+ else
+ return false;
+ }
+
+ return false;
+}
+
+/* Information that is common to all MLALB candidate forms once the
+ multiplicands and accumulator have been identified. */
+
+struct aarch64_mlalb_candidate
+{
+ tree lhs;
+ tree result_type;
+ unsigned int source_precision;
+ bool unsigned_p;
+ internal_fn ifn;
+};
+
+/* Check whether CALL can be replaced by an MLALB-like internal function using
+ MULTIPLICAND0 and MULTIPLICAND1. Fill in CANDIDATE on success. */
+
+static bool
+aarch64_check_mlalb_candidate (gcall *call, tree &multiplicand0,
+ tree &multiplicand1,
+ aarch64_mlalb_candidate &candidate)
+{
+ candidate.lhs = gimple_call_lhs (call);
+
+ /* MLALB is an SVE2 instruction, so do not match unless the target can
+ expand the replacement. */
+ if (!TARGET_SVE2)
+ return false;
+
+ /* The result must be an SVE integer vector whose elements can be split into
+ bottom halves for MLALB. */
+ if (!candidate.lhs
+ || !VECTOR_TYPE_P (TREE_TYPE (candidate.lhs))
+ || !aarch64_sve_mode_p (TYPE_MODE (TREE_TYPE (candidate.lhs))))
+ return false;
+
+ candidate.result_type = TREE_TYPE (candidate.lhs);
+ /* The current MLALB optabs only have scalable SVE expanders. Fixed-length
+ SVE vector types can have a mode that maps to one of those expanders, but
+ building the half-width source type below would give fixed-length source
+ operands, which the scalable expander cannot accept. */
+ if (TYPE_VECTOR_SUBPARTS (candidate.result_type).is_constant ())
+ return false;
+
+ tree element_type = TREE_TYPE (candidate.result_type);
+ if (!INTEGRAL_TYPE_P (element_type))
+ return false;
+
+ unsigned int element_precision = TYPE_PRECISION (element_type);
+
+ if (!aarch64_mlalb_source_element_type (element_precision, false))
+ return false;
+
+ unsigned int source_precision = element_precision / 2;
+
+ tree signed_multiplicand0;
+ tree signed_multiplicand1;
+ tree unsigned_multiplicand0;
+ tree unsigned_multiplicand1;
+
+ if (aarch64_mlalb_operand_fits_half_p (multiplicand0, source_precision,
+ false, &signed_multiplicand0)
+ && aarch64_mlalb_operand_fits_half_p (multiplicand1, source_precision,
+ false, &signed_multiplicand1))
+ {
+ candidate.unsigned_p = false;
+ multiplicand0 = signed_multiplicand0;
+ multiplicand1 = signed_multiplicand1;
+ }
+ else if (aarch64_mlalb_operand_fits_half_p (multiplicand0, source_precision,
+ true, &unsigned_multiplicand0)
+ && aarch64_mlalb_operand_fits_half_p (multiplicand1,
+ source_precision, true,
+ &unsigned_multiplicand1))
+ {
+ candidate.unsigned_p = true;
+ multiplicand0 = unsigned_multiplicand0;
+ multiplicand1 = unsigned_multiplicand1;
+ }
+ else
+ return false;
+
+ candidate.source_precision = source_precision;
+ candidate.ifn = candidate.unsigned_p ? IFN_UMLALB : IFN_SMLALB;
+ return direct_internal_fn_supported_p (candidate.ifn, candidate.result_type,
+ OPTIMIZE_FOR_SPEED);
+}
+
+/* Replace the current statement with the MLALB operation described by
+ CANDIDATE. MULTIPLICAND0 and MULTIPLICAND1 are already known to fit in the
+ selected half-width source type; ACCUMULATOR is the value to preserve and
+ accumulate into. */
+
+static void
+aarch64_replace_with_mlalb (gimple_stmt_iterator *gsi,
+ const aarch64_mlalb_candidate &candidate,
+ tree accumulator, tree multiplicand0,
+ tree multiplicand1)
+{
+ tree result_type = candidate.result_type;
+ bool unsigned_p = candidate.unsigned_p;
+
+ /* The IFN expects the same-lane-count source mode used by the
+ bottom-half instruction. For example, a VNx4SI result uses
+ VNx4HI sources, not the fully packed VNx8HI mode. */
+ tree result_element_type = TREE_TYPE (result_type);
+ tree source_element_type
+ = aarch64_mlalb_source_element_type (TYPE_PRECISION (result_element_type),
+ unsigned_p);
+ tree source_type = build_vector_type (source_element_type,
+ TYPE_VECTOR_SUBPARTS (result_type));
+ multiplicand0 = aarch64_build_mlalb_source_op (multiplicand0, source_type,
+ gsi);
+ multiplicand1 = aarch64_build_mlalb_source_op (multiplicand1, source_type,
+ gsi);
+
+ /* The accumulator must stay in the wide result mode. Only strip casts that
+ keep that mode unchanged, such as signedness-only vector conversions. */
+ accumulator = aarch64_strip_mlalb_accumulator_conversions (accumulator);
+
+ gcall *mlalb = gimple_build_call_internal (candidate.ifn, 3, accumulator,
+ multiplicand0,
+ multiplicand1);
+ gimple_call_set_lhs (mlalb, candidate.lhs);
+ gimple_call_set_nothrow (mlalb, true);
+ gsi_replace (gsi, mlalb, true);
+}
+
+static bool
+aarch64_try_cond_mlalb (gimple *stmt, gimple_stmt_iterator *gsi)
+{
+ gcall *call = as_a <gcall *> (stmt);
+ tree mask, multiplicand0, multiplicand1, merge;
+
+ if (gimple_call_internal_fn (call) == IFN_COND_ADD)
+ {
+ mask = gimple_call_arg (call, 0);
+ tree addend0 = gimple_call_arg (call, 1);
+ tree addend1 = gimple_call_arg (call, 2);
+ merge = gimple_call_arg (call, 3);
+
+ /* The conditional add must be a merging operation whose inactive lanes
+ keep the accumulator. */
+ if (!operand_equal_p (merge, addend1))
+ return false;
+
+ /* addend0 must be a multiplication node where each multiplicand fits in
+ half as many bits as the result. */
+ tree result_type = TREE_TYPE (gimple_call_lhs (call));
+ if (!aarch64_mlalb_mult_node_p (addend0, result_type, multiplicand0,
+ multiplicand1))
+ return false;
+ }
+ else if (gimple_call_internal_fn (call) == IFN_COND_FMA)
+ {
+ mask = gimple_call_arg (call, 0);
+ multiplicand0 = gimple_call_arg (call, 1);
+ multiplicand1 = gimple_call_arg (call, 2);
+ tree addend = gimple_call_arg (call, 3);
+ merge = gimple_call_arg (call, 4);
+
+ /* The fused multiply-add must be a merging operation whose inactive
+ lanes keep the accumulator. */
+ if (!operand_equal_p (addend, merge, 0))
+ return false;
+ }
+ else
+ gcc_unreachable ();
+
+ aarch64_mlalb_candidate candidate;
+ if (!aarch64_check_mlalb_candidate (call, multiplicand0, multiplicand1,
+ candidate))
+ return false;
+
+ tree new_multiplicand0;
+ tree new_multiplicand1 = multiplicand1;
+ bool zeroing0;
+ bool zeroing1;
+ aarch64_build_mlalb_zeroing_op (multiplicand0, mask, gsi,
+ new_multiplicand0, zeroing0);
+ if (!zeroing0)
+ {
+ aarch64_build_mlalb_zeroing_op (multiplicand1, mask, gsi,
+ new_multiplicand1, zeroing1,
+ true);
+ if (!zeroing1)
+ return false;
+ }
+
+ aarch64_replace_with_mlalb (gsi, candidate, merge, new_multiplicand0,
+ new_multiplicand1);
+ return true;
+}
+
+static bool
+aarch64_try_fma_mlalb (gimple *stmt, gimple_stmt_iterator *gsi)
+{
+ gcall *call = as_a <gcall *> (stmt);
+ tree multiplicand0 = gimple_call_arg (call, 0);
+ tree multiplicand1 = gimple_call_arg (call, 1);
+ tree addend = gimple_call_arg (call, 2);
+
+ aarch64_mlalb_candidate candidate;
+ if (!aarch64_check_mlalb_candidate (call, multiplicand0, multiplicand1,
+ candidate))
+ return false;
+
+ aarch64_replace_with_mlalb (gsi, candidate, addend, multiplicand0,
+ multiplicand1);
+ return true;
+}
+
/* Implement TARGET_INSTRUCTION_SELECTION. The target hook is used to
change generic sequences to a form AArch64 has an easier time expanding
instructions for. It's not supposed to be used for generic rewriting that
@@ -2238,6 +2862,16 @@ static bool
aarch64_instruction_selection (function * /* fun */, gimple_stmt_iterator *gsi)
{
auto stmt = gsi_stmt (*gsi);
+
+ if ((gimple_call_internal_p (stmt, IFN_COND_FMA)
+ || gimple_call_internal_p (stmt, IFN_COND_ADD))
+ && aarch64_try_cond_mlalb (stmt, gsi))
+ return true;
+
+ if (gimple_call_internal_p (stmt, IFN_FMA)
+ && aarch64_try_fma_mlalb (stmt, gsi))
+ return true;
+
gassign *assign = dyn_cast<gassign *> (stmt);
if (!assign)
diff --git a/gcc/config/aarch64/iterators.md b/gcc/config/aarch64/iterators.md
index 2d1522a348a..c1c92c4a9ce 100644
--- a/gcc/config/aarch64/iterators.md
+++ b/gcc/config/aarch64/iterators.md
@@ -2139,6 +2139,11 @@
(VNx2DI "VNx4SI") (VNx2DF "VNx4SF")
(VNx8SI "VNx8HI") (VNx16SI "VNx16QI")
(VNx8DI "VNx8HI")])
+
+;; Half-element modes with the same number of units.
+(define_mode_attr VHALF_EL [(VNx8HI "VNx8QI")
+ (VNx4SI "VNx4HI")
+ (VNx2DI "VNx2SI")])
(define_mode_attr Vnarrow [(VNx8HI "vnx16qi")
(VNx4SI "vnx8hi") (VNx4SF "vnx8hf")
(VNx2DI "vnx4si") (VNx2DF "vnx4sf")
diff --git a/gcc/internal-fn.def b/gcc/internal-fn.def
index af9f92950c7..d79c140f86d 100644
--- a/gcc/internal-fn.def
+++ b/gcc/internal-fn.def
@@ -471,6 +471,8 @@ DEF_INTERNAL_WIDENING_OPTAB_FN (VEC_WIDEN_ABD,
first,
vec_widen_sabd, vec_widen_uabd,
binary)
+DEF_INTERNAL_OPTAB_FN (SMLALB, ECF_CONST | ECF_NOTHROW, smlalb, ternary)
+DEF_INTERNAL_OPTAB_FN (UMLALB, ECF_CONST | ECF_NOTHROW, umlalb, ternary)
DEF_INTERNAL_OPTAB_FN (VEC_FMADDSUB, ECF_CONST, vec_fmaddsub, ternary)
DEF_INTERNAL_OPTAB_FN (VEC_FMSUBADD, ECF_CONST, vec_fmsubadd, ternary)
diff --git a/gcc/optabs.def b/gcc/optabs.def
index 7ccea18543f..a0225fe4b68 100644
--- a/gcc/optabs.def
+++ b/gcc/optabs.def
@@ -476,6 +476,8 @@ OPTAB_D (vec_widen_smult_even_optab,
"vec_widen_smult_even_$a")
OPTAB_D (vec_widen_smult_hi_optab, "vec_widen_smult_hi_$a")
OPTAB_D (vec_widen_smult_lo_optab, "vec_widen_smult_lo_$a")
OPTAB_D (vec_widen_smult_odd_optab, "vec_widen_smult_odd_$a")
+OPTAB_D (smlalb_optab, "smlalb$a4")
+OPTAB_D (umlalb_optab, "umlalb$a4")
OPTAB_D (vec_widen_ssub_optab, "vec_widen_ssub_$a")
OPTAB_D (vec_widen_ssub_hi_optab, "vec_widen_ssub_hi_$a")
OPTAB_D (vec_widen_ssub_lo_optab, "vec_widen_ssub_lo_$a")
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/mlalb-cond-add-selection.c
b/gcc/testsuite/gcc.target/aarch64/sve2/mlalb-cond-add-selection.c
new file mode 100644
index 00000000000..1209ca2e563
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/mlalb-cond-add-selection.c
@@ -0,0 +1,276 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fdump-tree-vect -fdump-tree-isel" } */
+
+#include <stdint.h>
+
+/* Test COND_ADD selection of SMLALB for signed 8-bit multiplicands accumulated
+ into a 16-bit reduction. */
+short
+cond_add_s8_to_s16_reduction (int8_t *a, int8_t *b, int len)
+{
+ if (len < 0 || len >= 128)
+ return 0;
+
+ short x = 0;
+ for (int8_t i = 0; i < len; ++i)
+ x += (short) a[i] * (short) b[i];
+
+ return x;
+}
+
+/* Test COND_ADD selection of UMLALB for unsigned 8-bit multiplicands
+ accumulated into a 16-bit reduction. */
+unsigned short
+cond_add_u8_to_u16_reduction (uint8_t *a, uint8_t *b, int len)
+{
+ if (len < 0 || len >= 128)
+ return 0;
+
+ unsigned short x = 0;
+ for (uint8_t i = 0; i < len; ++i)
+ x += (unsigned short) a[i] * (unsigned short) b[i];
+
+ return x;
+}
+
+/* Test COND_ADD selection of SMLALB for signed 16-bit multiplicands
+ accumulated into a 32-bit reduction. */
+int
+cond_add_s16_to_s32_reduction (int16_t *a, int16_t *b, int len)
+{
+ if (len < 0 || len >= 256)
+ return 0;
+
+ int x = 0;
+ for (int16_t i = 0; i < len; ++i)
+ x += (int) a[i] * (int) b[i];
+
+ return x;
+}
+
+/* Test COND_ADD selection of UMLALB for unsigned 16-bit multiplicands
+ accumulated into a 32-bit reduction. */
+unsigned int
+cond_add_u16_to_u32_reduction (uint16_t *a, uint16_t *b, int len)
+{
+ if (len < 0 || len >= 256)
+ return 0;
+
+ unsigned int x = 0;
+ for (uint16_t i = 0; i < len; ++i)
+ x += (unsigned int) a[i] * (unsigned int) b[i];
+
+ return x;
+}
+
+/* Test COND_ADD selection of SMLALB for signed 32-bit multiplicands
+ accumulated into a 64-bit reduction. */
+long int
+cond_add_s32_to_s64_reduction (int32_t *a, int32_t *b, int len)
+{
+ if (len < 0 || len >= 65536)
+ return 0;
+
+ long int x = 0;
+ for (int32_t i = 0; i < len; ++i)
+ x += (long int) a[i] * (long int) b[i];
+
+ return x;
+}
+
+/* Test COND_ADD selection of UMLALB for unsigned 32-bit multiplicands
+ accumulated into a 64-bit reduction. */
+unsigned long int
+cond_add_u32_to_u64_reduction (uint32_t *a, uint32_t *b, int len)
+{
+ if (len < 0 || len >= 65536)
+ return 0;
+
+ unsigned long int x = 0;
+ for (uint32_t i = 0; i < len; ++i)
+ x += (unsigned long int) a[i] * (unsigned long int) b[i];
+
+ return x;
+}
+
+/* Test that a 32-bit signed product widened to a 64-bit accumulator
+ selects SMLALB, since signed overflow is undefined. */
+long
+cond_add_s32_product_widened_to_s64 (int32_t *a, int32_t *b, int len)
+{
+ if (len < 0 || len >= 65536)
+ return 0;
+
+ long x = 0;
+ for (int32_t i = 0; i < len; ++i)
+ {
+ int32_t product = a[i] * b[i];
+ x += (long) product;
+ }
+ return x;
+}
+
+/* Test that a 32-bit unsigned product widened to a 64-bit accumulator is not
+ selected, since UMLALB would compute the full 32x32 product. */
+unsigned long
+cond_add_reject_u32_product_widened_to_u64 (uint32_t *a, uint32_t *b, int len)
+{
+ if (len < 0 || len >= 65536)
+ return 0;
+
+ unsigned long x = 0;
+ for (uint32_t i = 0; i < len; ++i)
+ {
+ uint32_t product = a[i] * b[i];
+ x += (unsigned long) product;
+ }
+ return x;
+}
+
+/* Test a signed source reinterpretation that is still valid for SMLALB. */
+int
+cond_add_signed_reinterpret_to_smlalb (uint16_t *buf, int len)
+{
+ if (len < 0 || len >= 256)
+ return 0;
+
+ int x = 0;
+ for (int16_t i = 0, y = 0; i < len; ++i, y = i * 10)
+ {
+ int16_t v = (int16_t) buf[i];
+ x += y * (int) v;
+ }
+ return x;
+}
+
+/* Test an unsigned source reinterpretation that selects UMLALB rather than
+ SMLALB. */
+unsigned int
+cond_add_unsigned_reinterpret_to_umlalb (int16_t *buf, int len)
+{
+ if (len < 0 || len >= 256)
+ return 0;
+
+ unsigned int x = 0;
+ for (uint16_t i = 0, y = 0; i < len; ++i, y = i * 10)
+ {
+ uint16_t v = (uint16_t) buf[i];
+ x += (unsigned int) y * (unsigned int) v;
+ }
+ return x;
+}
+
+/* Test that aarch64_mlalb_mult_node_p can walk through a VIEW_CONVERT_EXPR
+ from a signed product to an unsigned accumulator type. */
+unsigned int
+cond_add_signed_product_to_unsigned_accumulator (int16_t *a, int16_t *b,
+ int len)
+{
+ if (len < 0 || len >= 256)
+ return 0;
+
+ unsigned int x = 0;
+ for (int16_t i = 0; i < len; ++i)
+ {
+ int product = (int) a[i] * (int) b[i];
+ x += (unsigned int) product;
+ }
+ return x;
+}
+
+/* Test that an unsigned product reinterpreted to signed selects UMLALB rather
+ than SMLALB. */
+int
+cond_add_unsigned_product_to_signed_accumulator (uint16_t *a, uint16_t *b,
+ int len)
+{
+ if (len < 0 || len >= 256)
+ return 0;
+
+ int x = 0;
+ for (int16_t i = 0; i < len; ++i)
+ {
+ unsigned int product = (unsigned int) a[i] * (unsigned int) b[i];
+ x += (int) product;
+ }
+ return x;
+}
+
+/* Test that truncating a product before accumulation is not selected.
+ MLALB computes the full product, rather than the low half of a product. */
+int
+cond_add_reject_truncated_product_low_half (int16_t *a, int16_t *b, int len)
+{
+ if (len < 0 || len >= 256)
+ return 0;
+
+ int x = 0;
+ for (int16_t i = 0; i < len; ++i)
+ {
+ int product = (int) a[i] * (int) b[i];
+ x += (int16_t) product;
+ }
+ return x;
+}
+
+/* Test COND_ADD selection of SMLALB when a signed IV square is widened
+ to a 64-bit accumulator. */
+long
+cond_add_signed_iv_square_s64 (int len)
+{
+ if (len <= 0 || len >= 255)
+ return 0;
+
+ long res = 0;
+ for (int i = 0, y = 0; i < len; i++, y = i * 10)
+ res += y * y;
+ return res;
+}
+
+/* Test COND_ADD selection of SMLALB when a signed IV is widened to long
+ before multiplication. */
+long
+cond_add_signed_long_iv_square_s64 (int len)
+{
+ if (len <= 0 || len >= 255)
+ return 0;
+
+ long res = 0;
+ for (int i = 0, y = 0; i < len; i++, y = i * 10)
+ res += (long) y * (long) y;
+ return res;
+}
+
+/* Test COND_ADD selection of UMLALB when an unsigned IV is widened to
+ unsigned long before multiplication. */
+unsigned long
+cond_add_unsigned_long_iv_square_u64 (unsigned len)
+{
+ if (len <= 0 || len >= 255)
+ return 0;
+
+ unsigned long res = 0;
+ for (unsigned i = 0, y = 0; i < len; i++, y = i * 10)
+ res += (unsigned long) y * (unsigned long) y;
+ return res;
+}
+
+/* Test that an unsigned IV square widened to a 64-bit accumulator is not
+ selected when the product itself is formed in 32 bits. */
+unsigned long
+cond_add_reject_unsigned_iv_square_u64 (unsigned len)
+{
+ if (len <= 0 || len >= 255)
+ return 0;
+
+ unsigned long res = 0;
+ for (unsigned i = 0, y = 0; i < len; i++, y = i * 10)
+ res += y * y;
+ return res;
+}
+
+/* { dg-final { scan-tree-dump-times {\.COND_ADD} 17 "vect" } } */
+/* { dg-final { scan-tree-dump-times {\.SMLALB} 8 "isel" } } */
+/* { dg-final { scan-tree-dump-times {\.UMLALB} 6 "isel" } } */
+/* { dg-final { scan-assembler-times {\tsmlalb\t} 8 } } */
+/* { dg-final { scan-assembler-times {\tumlalb\t} 6 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/mlalb-cond-fma-selection.c
b/gcc/testsuite/gcc.target/aarch64/sve2/mlalb-cond-fma-selection.c
new file mode 100644
index 00000000000..a09d74a4450
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/mlalb-cond-fma-selection.c
@@ -0,0 +1,126 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fdump-tree-widening_mul -fdump-tree-isel" } */
+
+#include <stdint.h>
+
+/* Test COND_FMA selection of SMLALB with a signed character load adjusted by
+ a constant offset. */
+int
+cond_fma_signed_char_digit_offset (char *buf, int len)
+{
+ if (len < 0 || len >= 256)
+ return 0;
+
+ int x = 0;
+ for (int16_t i = 0, y = 0; i < len; i++, y = i * 10)
+ x += y * (buf[i] - '0');
+ return x;
+}
+
+/* Test COND_FMA selection of UMLALB when one unsigned multiplicand is built
+ from an IV plus a constant. */
+uint32_t
+cond_fma_unsigned_iv_plus_constant (uint8_t *buf, int len)
+{
+ if (len < 0 || len >= 256)
+ return 0;
+
+ uint32_t x = 0;
+ for (uint16_t i = 0, y = 0; i < len; i++, y = i * 10)
+ x += (uint32_t) (y + 2) * (uint32_t) buf[i];
+ return x;
+}
+
+/* Test COND_FMA selection of SMLALB when one signed multiplicand is built
+ from an IV minus a constant. */
+int32_t
+cond_fma_signed_iv_minus_constant (int8_t *buf, int len)
+{
+ if (len < 0 || len >= 256)
+ return 0;
+
+ int32_t x = 0;
+ for (int16_t i = 0, y = 0; i < len; i++, y = i * 10)
+ x += (int32_t) (y - 2) * (int32_t) buf[i];
+ return x;
+}
+
+/* Test COND_FMA selection of UMLALB for an unsigned short IV multiplied by an
+ unsigned short load. */
+unsigned
+cond_fma_unsigned_short_load (uint16_t *buf, uint8_t len)
+{
+ unsigned x = 0;
+ for (uint16_t i = 0, y = 0; i < len; i++, y = i * 10)
+ x += buf[i] * y;
+
+ return x;
+}
+
+/* Test COND_FMA selection of SMLALB when the loaded multiplicand is produced
+ by float-to-int conversion. */
+long
+cond_fma_float_to_int_load (float *buf, uint16_t len)
+{
+ long x = 0;
+ for (int i = 0, y = 0; i < len; i++, y = i * 10)
+ x += (long) y * (long) (int) buf[i];
+ return x;
+}
+
+/* Test COND_FMA selection of SMLALB when the loaded multiplicand is produced
+ by double-to-int conversion. */
+long
+cond_fma_double_to_int_load (double *buf, uint16_t len)
+{
+ long x = 0;
+ for (int i = 0, y = 0; i < len; i++, y = i * 10)
+ x += (long) y * (long) (int) buf[i];
+ return x;
+}
+
+/* Test COND_FMA selection of SMLALB when one signed character load is
+ adjusted by a constant before multiplication. */
+int
+cond_fma_two_signed_char_loads (char *__restrict buf1,
+ char *__restrict buf2, uint8_t len)
+{
+ int x = 0;
+ for (int i = 0; i < len; ++i)
+ x += (buf1[i] - 2) * (buf2[i]);
+ return x;
+}
+
+/* Test COND_FMA selection of SMLALB when both multiplicands are the same
+ signed IV. */
+int
+cond_fma_signed_iv_square (int len)
+{
+ if (len <= 0 || len >= 255)
+ return 0;
+
+ int res = 0;
+ for (short i = 0, y = 0; i < len; i++, y = i * 10)
+ res += y * y;
+ return res;
+}
+
+/* Test COND_FMA selection of UMLALB when both multiplicands are the same
+ unsigned IV. */
+unsigned
+cond_fma_unsigned_iv_square (unsigned len)
+{
+ if (len == 0 || len >= 255)
+ return 0;
+
+ unsigned res = 0;
+ for (unsigned short i = 0, y = 0; i < len; i++, y = i * 10)
+ res += y * y;
+ return res;
+}
+
+/* { dg-final { scan-tree-dump-times {\.COND_FMA} 9 "widening_mul" } } */
+/* { dg-final { scan-tree-dump-times {\.SMLALB} 6 "isel" } } */
+/* { dg-final { scan-tree-dump-times {\.UMLALB} 3 "isel" } } */
+/* { dg-final { scan-assembler-times {\tsmlalb\t} 6 } } */
+/* { dg-final { scan-assembler-times {\tumlalb\t} 3 } } */
diff --git
a/gcc/testsuite/gcc.target/aarch64/sve2/mlalb-fma-fixed-vector-length.c
b/gcc/testsuite/gcc.target/aarch64/sve2/mlalb-fma-fixed-vector-length.c
new file mode 100644
index 00000000000..8da14824ac2
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/mlalb-fma-fixed-vector-length.c
@@ -0,0 +1,16 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -msve-vector-bits=128 -fdump-tree-isel" } */
+
+typedef unsigned short uint16_t;
+
+/* Test that MLALB selection rejects fixed-length SVE result types until the
+ optab supports the matching fixed-length half-width source operands. */
+void __attribute__ ((noinline, noclone))
+bar_test (int *__restrict buf, uint16_t len, long *x)
+{
+ for (int i = 0, y = 0; i < len; i++, y = i * 10)
+ x[i] += (long) y * (long) buf[i];
+}
+
+/* { dg-final { scan-tree-dump-not {\.SMLALB} "isel" } } */
+/* { dg-final { scan-assembler-not {\tsmlalb\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/mlalb-fma-selection.c
b/gcc/testsuite/gcc.target/aarch64/sve2/mlalb-fma-selection.c
new file mode 100644
index 00000000000..d21968be204
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/mlalb-fma-selection.c
@@ -0,0 +1,134 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fdump-tree-isel" } */
+
+#include <stdint.h>
+
+/* Test plain FMA selection of SMLALB with a signed character load adjusted by
+ a constant offset. */
+void
+plain_fma_signed_char_digit_offset (char *__restrict buf, int8_t len, int *x)
+{
+ for (int16_t i = 0, y = 0; i < len; i++, y = i * 10)
+ x[i] += (int) y * (buf[i] - '0');
+}
+
+/* Test plain FMA selection of SMLALB for signed 32-bit multiplicands
+ accumulated into a 64-bit memory operand. */
+void
+plain_fma_s32_to_s64_memory_accumulator (int *__restrict buf, uint16_t len,
+ long *x)
+{
+ for (int i = 0, y = 0; i < len; i++, y = i * 10)
+ x[i] += (long) y * (long) buf[i];
+}
+
+/* Test plain FMA selection of SMLALB when a float-to-int conversion
+ feeds one multiplicand. */
+void
+plain_fma_float_to_int_load (float *__restrict buf, uint16_t len, long *x)
+{
+ for (int i = 0, y = 0; i < len; i++, y = i * 10)
+ x[i] += (long) y * (long) (int) buf[i];
+}
+
+/* Test plain FMA selection of SMLALB when one multiplicand is built
+ by subtracting a constant from a loaded value. */
+void
+plain_fma_sub_loaded_source (int16_t *__restrict buf, int8_t len, int *x)
+{
+ for (int16_t i = 0, y = 0; i < len; i++, y = i * 10)
+ {
+ int16_t v = (int16_t) (buf[i] - 7);
+ x[i] += (int) y * (int) v;
+ }
+}
+
+/* Test plain FMA selection of SMLALB when one multiplicand is built
+ by multiplying a loaded value by a constant. */
+void
+plain_fma_mul_loaded_source (int16_t *__restrict buf, int8_t len, int *x)
+{
+ for (int16_t i = 0, y = 0; i < len; i++, y = i * 10)
+ {
+ int16_t v = (int16_t) (buf[i] * 3);
+ x[i] += (int) y * (int) v;
+ }
+}
+
+/* Test plain FMA selection of SMLALB when one multiplicand is built
+ by adding a constant to a loaded value. */
+void
+plain_fma_add_loaded_source (int16_t *__restrict buf, int8_t len, int *x)
+{
+ for (int16_t i = 0, y = 0; i < len; i++, y = i * 10)
+ {
+ int16_t v = (int16_t) (buf[i] + 7);
+ x[i] += (int) y * (int) v;
+ }
+}
+
+/* Test plain FMA selection of UMLALB when one unsigned multiplicand is
+ built by adding a constant to a loaded value. */
+void
+plain_fma_unsigned_add_loaded_source (uint16_t *__restrict buf, int8_t len,
+ unsigned int *x)
+{
+ for (uint16_t i = 0, y = 0; i < len; i++, y = i * 10)
+ {
+ uint16_t v = (uint16_t) (buf[i] + 7);
+ x[i] += (unsigned int) y * (unsigned int) v;
+ }
+}
+
+/* Test plain FMA selection of SMLALB after stripping a signedness conversion
+ around the accumulator. */
+void
+plain_fma_accumulator_signedness_convert (int16_t *__restrict buf, int8_t len,
+ unsigned int *x)
+{
+ for (int16_t i = 0, y = 0; i < len; i++, y = i * 10)
+ x[i] = (unsigned int) ((int) x[i] + (int) y * (int) buf[i]);
+}
+
+/* Test plain FMA selection of UMLALB for an unsigned short IV multiplied by an
+ unsigned short load. */
+void
+plain_fma_unsigned_short_load (uint16_t *__restrict buf, uint16_t len,
+ unsigned int *x)
+{
+ for (uint16_t i = 0, y = 0; i < len; i++, y = i * 10)
+ x[i] += (unsigned int) y * (unsigned int) buf[i];
+}
+
+/* Test the same accumulator signedness-conversion path with an explicitly
+ named old value kept live in the loop body. */
+void
+plain_fma_named_old_accumulator_convert (int16_t *__restrict buf, int8_t len,
+ unsigned int *x)
+{
+ for (int16_t i = 0, y = 0; i < len; i++, y = i * 10)
+ {
+ int old = (int) x[i];
+ x[i] = (unsigned int) (old + (int) y * (int) buf[i]);
+ }
+}
+
+/* Test that plain FMA selection still allows the old accumulator value to be
+ used after the selected MLALB operation. */
+void
+plain_fma_preserve_old_accumulator (int16_t *__restrict buf, int8_t len,
+ int *__restrict x, int *__restrict old_x)
+{
+ for (int16_t i = 0, y = 0; i < len; i++, y = i * 10)
+ {
+ int old = x[i];
+ x[i] = old + (int) y * (int) buf[i];
+ old_x[i] = old;
+ }
+}
+
+/* { dg-final { scan-tree-dump-times {\.SMLALB} 9 "isel" } } */
+/* { dg-final { scan-tree-dump-times {\.UMLALB} 2 "isel" } } */
+/* { dg-final { scan-assembler-times {\tsmlalb\t} 9 } } */
+/* { dg-final { scan-assembler-times {\tumlalb\t} 2 } } */
+/* { dg-final { scan-assembler {\tmovprfx\t} } } */
--
2.43.0
