https://gcc.gnu.org/g:c2e7d7158971b16d62b4b024ca40836f2381d0db

commit r17-6222-gc2e7d7158971b16d62b4b024ca40836f2381d0db
Author: Dhruv Chawla <[email protected]>
Date:   Thu Nov 5 15:43:50 2026 +0530

    aarch64: Add intrinsic framework support for NEON shift intrinsics [NFC]
    
    Add the basic support that subsequent patches need.
    
    New intrinsic shape checkers:
     - shift_right: [1, element width]
     - shift_right_narrow: [1, result width / 2]
     - shift_left_long: [0, element width]
    
    Add `d<elt>` syntax support to parse_element_type for the widening
    intrinsics.
    
    New classes:
     - gimple_insn_based_function: base class for intrinsics that map to a 
single
       instruction. Subclasses:
       - gimple_exact_insn: Like before.
       - gimple_code_for_mode_function: Choosing instruction only by mode.
       - gimple_rtx_code_function: Choosing instruction by rtx code iterator
         attribute.
       - gimple_unspec_based_function: Choosing instruction by int code iterator
         attribute.
     - gimple_shift_imm: shift by immediate with an optional accumulator.
     - gimple_shift_reg: shift by register, fold only for constant arguments.
    
    These are inspired from the SVE counterparts. Both gimple_rtx_code_function 
and
    gimple_unspec_based_function derive from gimple_iterator_function as they 
only
    differ in the usage of rtx_code or int, respectively.
    
    Bootstrapped and regtested on aarch64-linux-gnu.
    
    Signed-off-by: Dhruv Chawla <[email protected]>
    
    gcc/ChangeLog:
    
            * config/aarch64/aarch64-acle-builtins.h (TYPES_b_signed,
            TYPES_h_signed): New macros.
            (b_signed, h_signed): New type arrays.
            (TYPES_b_integer, TYPES_h_integer, TYPES_d_integer): Compose from 
the
            corresponding signed list, as TYPES_s_integer already does.
            * config/aarch64/aarch64-neon-builtins-base.cc (build_shift_right): 
New
            function.
            (gimple_insn_based_function): New class.
            (gimple_exact_insn): Derive from it.
            (gimple_code_for_mode_function, gimple_iterator_function,
            gimple_shift_imm, gimple_shift_reg): New classes.
            (gimple_rtx_code_function, gimple_unspec_based_function): New 
typedefs.
            * config/aarch64/aarch64-neon-builtins-shapes.cc (shift_right,
            shift_right_narrow, shift_left_long): New checkers.
            * config/aarch64/aarch64-sve-builtins-shapes.cc 
(parse_element_type):
            Handle `d<elt>`.
            * config/aarch64/aarch64-simd.md (aarch64_<sra_op>rsra_n<mode>): 
Rename
            to ...
            (@aarch64_<sra_op>rsra_n<mode>): ... this.
            (aarch64_<sur>shl<mode>): Rename to ...
            (@aarch64_<sur>shl<mode>): ... this.
            (aarch64_<sur>q<r>shl<mode>): Rename to ...
            (@aarch64_<sur>q<r>shl<mode>): ... this.
            (aarch64_<sur>shll_n<mode>): Rename to ...
            (@aarch64_<sur>shll_n<mode>): ... this.
            (aarch64_<sur>shll2_n<mode>): Rename to ...
            (@aarch64_<sur>shll2_n<mode>): ... this.
            (aarch64_<sra_op>rshr_n<mode>): Rename to ...
            (@aarch64_<sra_op>rshr_n<mode>): ... this.
            (aarch64_<sur>s<lr>i_n<mode>): Rename to ...
            (@aarch64_<sur>s<lr>i_n<mode>): ... this.
            (aarch64_<sur>qshl<u>_n<mode>): Rename to ...
            (@aarch64_<sur>qshl<u>_n<mode>): ... this.
            (aarch64_<shrn_op>shrn_n<mode>): Rename to ...
            (@aarch64_<shrn_op>shrn_n<mode>): ... this.
            (aarch64_<shrn_op>rshrn_n<mode>): Rename to ...
            (@aarch64_<shrn_op>rshrn_n<mode>): ... this.
            (aarch64_sqshrun_n<mode>): Rename to ...
            (@aarch64_sqshrun_n<mode>): ... this.
            (aarch64_sqrshrun_n<mode>): Rename to ...
            (@aarch64_sqrshrun_n<mode>): ... this.
            (aarch64_<shrn_op><sra_op>shrn2_n<mode>): Rename to ...
            (@aarch64_<shrn_op><sra_op>shrn2_n<mode>): ... this.
            (aarch64_<shrn_op>rshrn2_n<mode>): Rename to ...
            (@aarch64_<shrn_op>rshrn2_n<mode>): ... this.
            (aarch64_sqshrun2_n<mode>): Rename to ...
            (@aarch64_sqshrun2_n<mode>): ... this.
            (aarch64_sqrshrun2_n<mode>): Rename to ...
            (@aarch64_sqrshrun2_n<mode>): ... this.
    
    gcc/testsuite/ChangeLog:
    
            * gcc.target/aarch64/neon/arm_neon_test.h (TEST_UNARY_N,
            TEST_UNIFORM_UNARY_N, TEST_BINARY_N, TEST_UNIFORM_BINARY_N): New
            macros.

Diff:
---
 gcc/config/aarch64/aarch64-acle-builtins.h         |  16 +-
 gcc/config/aarch64/aarch64-neon-builtins-base.cc   | 234 ++++++++++++++++++++-
 gcc/config/aarch64/aarch64-neon-builtins-shapes.cc |  32 +++
 gcc/config/aarch64/aarch64-simd.md                 |  40 ++--
 gcc/config/aarch64/aarch64-sve-builtins-shapes.cc  |   8 +
 .../gcc.target/aarch64/neon/arm_neon_test.h        |  14 ++
 6 files changed, 311 insertions(+), 33 deletions(-)

diff --git a/gcc/config/aarch64/aarch64-acle-builtins.h 
b/gcc/config/aarch64/aarch64-acle-builtins.h
index 91c922fdae6f..dbe0d4bf953d 100644
--- a/gcc/config/aarch64/aarch64-acle-builtins.h
+++ b/gcc/config/aarch64/aarch64-acle-builtins.h
@@ -1264,10 +1264,14 @@ function_expander::result_mode () const
 #define TYPES_b_unsigned(S, D, T) \
   S (u8)
 
+/* _s8.  */
+#define TYPES_b_signed(S, D, T) \
+  S (s8)
+
 /* _s8
    _u8.  */
 #define TYPES_b_integer(S, D, T) \
-  S (s8), TYPES_b_unsigned (S, D, T)
+  TYPES_b_signed (S, D, T), TYPES_b_unsigned (S, D, T)
 
 /* _mf8
    _s8
@@ -1324,10 +1328,14 @@ function_expander::result_mode () const
 #define TYPES_h_float(S, D, T) \
   S (f16)
 
+/* _s16.  */
+#define TYPES_h_signed(S, D, T) \
+  S (s16)
+
 /* _s16
    _u16.  */
 #define TYPES_h_integer(S, D, T) \
-  S (s16), S (u16)
+  TYPES_h_signed (S, D, T), S (u16)
 
 /* _bf16
    _f16
@@ -1464,7 +1472,7 @@ function_expander::result_mode () const
 /* _s64
    _u64.  */
 #define TYPES_d_integer(S, D, T) \
-  S (s64), TYPES_d_unsigned (S, D, T)
+  TYPES_d_signed (S, D, T), TYPES_d_unsigned (S, D, T)
 
 /* _f64
    _s64
@@ -2127,6 +2135,8 @@ DEF_SVE_TYPES_ARRAY (all_signed);
 DEF_SVE_TYPES_ARRAY (all_float_and_signed);
 DEF_SVE_TYPES_ARRAY (all_unsigned);
 DEF_SVE_TYPES_ARRAY (all_integer);
+DEF_SVE_TYPES_ARRAY (b_signed);
+DEF_SVE_TYPES_ARRAY (h_signed);
 DEF_SVE_TYPES_ARRAY (all_arith);
 DEF_SVE_TYPES_ARRAY (all_arith_no_fp16);
 DEF_SVE_TYPES_ARRAY (all_data);
diff --git a/gcc/config/aarch64/aarch64-neon-builtins-base.cc 
b/gcc/config/aarch64/aarch64-neon-builtins-base.cc
index 9b77445334b5..339df23504eb 100644
--- a/gcc/config/aarch64/aarch64-neon-builtins-base.cc
+++ b/gcc/config/aarch64/aarch64-neon-builtins-base.cc
@@ -125,6 +125,25 @@ build_vec_dup (tree type, tree elem)
           : fold_build1 (VEC_DUPLICATE_EXPR, type, elem);
 }
 
+/* Build `VAL >> AMOUNT` for the integer vector type TYPE, conforming to ACLE
+   semantics.  AMOUNT is an integer constant.  */
+tree
+build_shift_right (tree type, tree val, tree amount)
+{
+  auto bits = element_precision (type);
+  if (wi::ltu_p (wi::to_wide (amount), bits))
+    return fold_build2 (RSHIFT_EXPR, type, val, amount);
+
+  /* A logical (unsigned) shift by the element width shifts every bit out.  */
+  if (TYPE_UNSIGNED (type))
+    return build_zero_cst (type);
+
+  /* An arithmetic (signed) shift by the element width leaves every bit equal 
to
+     the sign bit which a shift by one less already gives.  */
+  return fold_build2 (RSHIFT_EXPR, type, val,
+                     build_int_cst (TREE_TYPE (amount), bits - 1));
+}
+
 /* Build a `TUPLE.val[INDEX]` expression.  */
 tree
 build_tuple_get (tree tuple, tree index)
@@ -239,15 +258,124 @@ public:
   }
 };
 
-template <insn_code insn> struct gimple_exact_insn : gimple_function_base
+/* Base class for intrinsics that map to a single instruction.  Derived classes
+   name the instruction by overriding `code_for`.
+
+   M_MODE_ARG is the index of the argument whose mode selects the instruction,
+   or -1 if the intrinsic goes through use_exact_insn.  If the argument has a
+   one-element vector type, such as int64x1_t, there is no instruction for it,
+   so `code_for` is passed the element mode instead and the arguments and the
+   result are unwrapped to match.  This is what arm_neon.h did for these types
+   before they moved to the pragma-based framework.  */
+class gimple_insn_based_function : public gimple_function_base
 {
+  int m_mode_arg;
+
+protected:
+  /* Return the instruction to use for mode MODE.  */
+  virtual insn_code code_for (function_expander &e, machine_mode mode) const
+    = 0;
+
+public:
+  constexpr gimple_insn_based_function (int mode_arg) : m_mode_arg (mode_arg) 
{}
+
   gimple *fold (gimple_folder &) const override { return nullptr; }
+
   rtx expand (function_expander &e) const override
   {
-    return e.use_exact_insn (insn);
+    gcc_assert (m_mode_arg >= -1);
+    if (m_mode_arg == -1)
+      return e.use_exact_insn (code_for (e, VOIDmode));
+
+    auto type = TREE_TYPE (CALL_EXPR_ARG (e.call_expr, m_mode_arg));
+
+    /* Handle vector types.  */
+    if (!one_element_vector_type_p (type))
+      return e.use_exact_insn (code_for (e, TYPE_MODE (type)));
+
+    /* Handle scalar types.  This is similar to the handling in gimple_ifn for
+       V1DI and V1DF types.  */
+    auto icode = code_for (e, TYPE_MODE (TREE_TYPE (type)));
+    auto scalar_mode = GET_MODE_INNER (e.result_mode ());
+
+    for (unsigned int i = 0; i < e.args.length (); i++)
+      {
+       auto arg_type = TREE_TYPE (CALL_EXPR_ARG (e.call_expr, i));
+       if (one_element_vector_type_p (arg_type))
+         e.args[i] = force_lowpart_subreg (scalar_mode, e.args[i],
+                                           TYPE_MODE (arg_type));
+      }
+
+    auto res = e.use_exact_insn (icode);
+    return force_lowpart_subreg (e.result_mode (), res, scalar_mode);
+  }
+};
+
+/* For intrinsics that always map to instruction INSN.  */
+template <insn_code INSN>
+struct gimple_exact_insn : public gimple_insn_based_function
+{
+  constexpr gimple_exact_insn () : gimple_insn_based_function (-1) {}
+
+  insn_code code_for (function_expander &, machine_mode) const override
+  {
+    return INSN;
   }
 };
 
+/* For intrinsics whose instruction is selected by a mode through the
+   code_for_* helpers that insn-opinit.h generates for `@`-prefixed
+   pattterns.  */
+class gimple_code_for_mode_function : public gimple_insn_based_function
+{
+  insn_code (*m_code_for) (machine_mode);
+
+  insn_code code_for (function_expander &, machine_mode mode) const override
+  {
+    return this->m_code_for (mode);
+  }
+
+public:
+  constexpr gimple_code_for_mode_function (insn_code (*code_for) 
(machine_mode),
+                                          int mode_arg = 0)
+    : gimple_insn_based_function (mode_arg), m_code_for (code_for)
+  {}
+};
+
+/* Like gimple_code_for_mode_function, but for a pattern whose name also
+   contains an attribute of an iterator that distinguishes signed forms from
+   unsigned ones, such as `<sra_op>` or `<sur>`.  SIGNED_VALUE and
+   UNSIGNED_VALUE give the iterator value to use for signed and unsigned type
+   suffixes respectively.
+
+   T is the type that insn-opinit.h uses for the iterator: `rtx_code` for a
+   code iterator and `int` for an int iterator.  */
+template <typename T>
+class gimple_iterator_function : public gimple_insn_based_function
+{
+  insn_code (*m_code_for) (T, machine_mode);
+  T m_signed_value;
+  T m_unsigned_value;
+
+  insn_code code_for (function_expander &e, machine_mode mode) const override
+  {
+    return m_code_for (e.type_suffix (0).unsigned_p ? m_unsigned_value
+                                                   : m_signed_value,
+                      mode);
+  }
+
+public:
+  constexpr gimple_iterator_function (insn_code (*code_for) (T, machine_mode),
+                                     T signed_value, T unsigned_value,
+                                     int mode_arg = 0)
+    : gimple_insn_based_function (mode_arg), m_code_for (code_for),
+      m_signed_value (signed_value), m_unsigned_value (unsigned_value)
+  {}
+};
+
+using gimple_rtx_code_function = gimple_iterator_function<rtx_code>;
+using gimple_unspec_based_function = gimple_iterator_function<int>;
+
 /* For intrinsics that map to a single GIMPLE expression with no argument
    preparation necessary.  */
 class gimple_expr : public gimple_function_base
@@ -478,14 +606,6 @@ struct gimple_dup : public gimple_function_base
   }
 };
 
-// Vector creation
-NEON_FUNCTION (vcreate,                gimple_create,)
-NEON_FUNCTION (vcombine,       gimple_combine,)
-NEON_FUNCTION (vdup_n,         gimple_dup,)
-NEON_FUNCTION (vdupq_n,                gimple_dup,)
-NEON_FUNCTION (vmov_n,         gimple_dup,)
-NEON_FUNCTION (vmovq_n,                gimple_dup,)
-
 struct gimple_dup_lane : public gimple_function_base
 {
   gimple *fold (gimple_folder &f) const override
@@ -1317,9 +1437,102 @@ struct gimple_reduce_minmax : public 
gimple_function_base
   }
 };
 
+/* Shift by an immediate with an optional accumulator.  */
+class gimple_shift_imm : public gimple_function_base
+{
+  /* LSHIFT_EXPR or RSHIFT_EXPR.  */
+  tree_code m_code;
+
+  /* PLUS_EXPR for vsra_n and ERROR_MARK for the non-accumulating forms.  */
+  tree_code m_acc_code;
+
+  /* The value to shift is operand 0, unless an accumulator comes first.  */
+  int first_shift_arg () const { return m_acc_code != ERROR_MARK; }
+
+public:
+  constexpr gimple_shift_imm (tree_code code, tree_code acc_code = ERROR_MARK)
+    : m_code (code), m_acc_code (acc_code)
+  {}
+
+  gimple *fold (gimple_folder &f) const override
+  {
+    auto type = TREE_TYPE (f.lhs);
+    int first = first_shift_arg ();
+    auto val = gimple_call_arg (f.call, first);
+    auto amount = gimple_call_arg (f.call, first + 1);
+
+    auto shifted = m_code == LSHIFT_EXPR
+                    ? fold_build2 (LSHIFT_EXPR, type, val, amount)
+                    : build_shift_right (type, val, amount);
+
+    /* Non-accumulating intrinsic.  */
+    if (m_acc_code == ERROR_MARK)
+      return gimple_build_assign (f.lhs, shifted);
+
+    /* Accumulating intrinsic.  */
+    auto acc = gimple_call_arg (f.call, 0);
+    shifted = f.force_val (shifted);
+
+    /* Accumulate in an unsigned type so that signed overflow wraps as the
+       ACLE requires.  */
+    auto utype = type;
+    if (!TYPE_UNSIGNED (type))
+      {
+       utype = unsigned_type_for (type);
+       acc = f.force_val (build_cast (utype, acc));
+       shifted = f.force_val (build_cast (utype, shifted));
+      }
+
+    return gimple_build_assign (
+      f.lhs, build_cast (type, f.force_val (fold_build2 (m_acc_code, utype, 
acc,
+                                                        shifted))));
+  }
+};
+
+/* Shift by a vector amount.  The amount is signed and a negative
+   value shifts right, so this only folds to a GIMPLE shift when the amount is
+   a constant whose magnitude is in range.  */
+class gimple_shift_reg : public gimple_unspec_based_function
+{
+public:
+  using gimple_unspec_based_function::gimple_unspec_based_function;
+
+  gimple *fold (gimple_folder &f) const override
+  {
+    auto val = gimple_call_arg (f.call, 0);
+    auto amount = gimple_call_arg (f.call, 1);
+
+    /* Fold when the shift is a known constant value.  */
+    auto cst = uniform_integer_cst_p (amount);
+    if (!cst)
+      return nullptr;
+
+    auto wcst = wi::to_wide (cst);
+    if (wi::geu_p (wi::abs (wcst), element_precision (val)))
+      return nullptr;
+
+    if (!wi::neg_p (wcst))
+      return gimple_build_assign (f.lhs, LSHIFT_EXPR, val, amount);
+
+    /* A negative amount shifts right by its magnitude.  */
+    auto rhs
+      = build_uniform_cst (TREE_TYPE (amount),
+                          wide_int_to_tree (TREE_TYPE (cst), wi::abs (wcst)));
+    return gimple_build_assign (f.lhs, RSHIFT_EXPR, val, rhs);
+  }
+};
+
 // Reinterpret
 NEON_FUNCTION (vreinterpret, vreinterpretq, gimple_reinterpret,)
 
+// Vector creation
+NEON_FUNCTION (vcreate,                gimple_create,)
+NEON_FUNCTION (vcombine,       gimple_combine,)
+NEON_FUNCTION (vdup_n,         gimple_dup,)
+NEON_FUNCTION (vdupq_n,                gimple_dup,)
+NEON_FUNCTION (vmov_n,         gimple_dup,)
+NEON_FUNCTION (vmovq_n,                gimple_dup,)
+
 // Lane get/set
 NEON_FUNCTION (vget_lane, vgetq_lane,   gimple_get_lane,)
 NEON_FUNCTION (vget_low,                gimple_get_half<false>,)
@@ -1410,6 +1623,7 @@ NEON_FUNCTION (vrndn, vrndnq, vrndns, gimple_ifn, 
(IFN_ROUNDEVEN))
 NEON_FUNCTION (vrndp, vrndpq,         gimple_ifn, (IFN_CEIL))
 NEON_FUNCTION (vrndx, vrndxq,         gimple_ifn, (IFN_RINT))
 
+
 // Bitwise operations
 NEON_FUNCTION (vand,  vandq,  gimple_expr,    (BIT_AND_EXPR))
 NEON_FUNCTION (vbic,  vbicq,  gimple_not_rhs, (BIT_AND_EXPR))
diff --git a/gcc/config/aarch64/aarch64-neon-builtins-shapes.cc 
b/gcc/config/aarch64/aarch64-neon-builtins-shapes.cc
index c50fb4d4f8c5..7f3f81c9f075 100644
--- a/gcc/config/aarch64/aarch64-neon-builtins-shapes.cc
+++ b/gcc/config/aarch64/aarch64-neon-builtins-shapes.cc
@@ -82,6 +82,38 @@ shift (function_checker &c)
   return c.require_immediate_range (PARAM_INDEX, 0, bits - 1);
 }
 
+/* Require that the parameter at PARAM_INDEX is a valid right shift amount (1 
to
+   the element width).  */
+template <unsigned int PARAM_INDEX>
+bool
+shift_right (function_checker &c)
+{
+  auto bits = c.type_suffix (0).element_bits;
+  return c.require_immediate_range (PARAM_INDEX, 1, bits);
+}
+
+/* Require that the parameter at PARAM_INDEX is a valid right shift amount for
+   a narrowing shift (whose result elements are half the width of the ones
+   named by the type suffix).  */
+template <unsigned int PARAM_INDEX>
+bool
+shift_right_narrow (function_checker &c)
+{
+  auto bits = c.type_suffix (0).element_bits / 2;
+  return c.require_immediate_range (PARAM_INDEX, 1, bits);
+}
+
+/* Require that the parameter at PARAM_INDEX is a valid left shift amount for
+   a widening shift.  Unlike the other left shifts, these may name the element
+   width itself, which selects the SHLL form.  */
+template <unsigned int PARAM_INDEX>
+bool
+shift_left_long (function_checker &c)
+{
+  auto bits = c.type_suffix (0).element_bits;
+  return c.require_immediate_range (PARAM_INDEX, 0, bits);
+}
+
 /* A checker that always returns true.  */
 bool
 trivial (function_checker &)
diff --git a/gcc/config/aarch64/aarch64-simd.md 
b/gcc/config/aarch64/aarch64-simd.md
index b76e01d81db9..08ea269c6939 100644
--- a/gcc/config/aarch64/aarch64-simd.md
+++ b/gcc/config/aarch64/aarch64-simd.md
@@ -1661,7 +1661,7 @@
   }
 )
 
-(define_expand "aarch64_<sra_op>rsra_n<mode>"
+(define_expand "@aarch64_<sra_op>rsra_n<mode>"
   [(match_operand:VSDQ_I_DI 0 "register_operand")
    (match_operand:VSDQ_I_DI 1 "register_operand")
    (SHIFTRT:VSDQ_I_DI
@@ -7302,7 +7302,7 @@
 
 ;; vshl
 
-(define_insn "aarch64_<sur>shl<mode><vczle><vczbe>"
+(define_insn "@aarch64_<sur>shl<mode><vczle><vczbe>"
   [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
         (unspec:VSDQ_I_DI
          [(match_operand:VSDQ_I_DI 1 "register_operand" "w")
@@ -7316,7 +7316,7 @@
 
 ;; vqshl
 
-(define_insn "aarch64_<sur>q<r>shl<mode><vczle><vczbe>"
+(define_insn "@aarch64_<sur>q<r>shl<mode><vczle><vczbe>"
   [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
         (unspec:VSDQ_I
          [(match_operand:VSDQ_I 1 "register_operand" "w")
@@ -7343,7 +7343,7 @@
   [(set_attr "type" "neon_shift_imm_long")]
 )
 
-(define_expand "aarch64_<sur>shll_n<mode>"
+(define_expand "@aarch64_<sur>shll_n<mode>"
   [(set (match_operand:<VWIDE> 0 "register_operand")
        (unspec:<VWIDE> [(match_operand:VD_BHSI 1 "register_operand")
                         (match_operand:SI 2
@@ -7375,7 +7375,7 @@
   [(set_attr "type" "neon_shift_imm_long")]
 )
 
-(define_expand "aarch64_<sur>shll2_n<mode>"
+(define_expand "@aarch64_<sur>shll2_n<mode>"
   [(set (match_operand:<VWIDE> 0 "register_operand")
        (unspec:<VWIDE> [(match_operand:VQW 1 "register_operand")
                         (match_operand:SI 2
@@ -7408,7 +7408,7 @@
   [(set_attr "type" "neon_sat_shift_imm<q>")]
 )
 
-(define_expand "aarch64_<sra_op>rshr_n<mode>"
+(define_expand "@aarch64_<sra_op>rshr_n<mode>"
   [(match_operand:VSDQ_I_DI 0 "register_operand")
    (SHIFTRT:VSDQ_I_DI
      (match_operand:VSDQ_I_DI 1 "register_operand")
@@ -7450,7 +7450,7 @@
 
 ;; vs<lr>i_n
 
-(define_insn "aarch64_<sur>s<lr>i_n<mode>"
+(define_insn "@aarch64_<sur>s<lr>i_n<mode>"
   [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
        (unspec:VSDQ_I_DI [(match_operand:VSDQ_I_DI 1 "register_operand" "0")
                       (match_operand:VSDQ_I_DI 2 "register_operand" "w")
@@ -7464,7 +7464,7 @@
 
 ;; vqshl(u)
 
-(define_insn "aarch64_<sur>qshl<u>_n<mode>"
+(define_insn "@aarch64_<sur>qshl<u>_n<mode>"
   [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
        (unspec:VSDQ_I [(match_operand:VSDQ_I 1 "register_operand" "w")
                       (match_operand:SI 2
@@ -7478,7 +7478,7 @@
 
 ;; vq(r)shr(u)n_n
 
-(define_insn "aarch64_<shrn_op>shrn_n<mode>"
+(define_insn "@aarch64_<shrn_op>shrn_n<mode>"
   [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
        (SAT_TRUNC:<VNARROWQ>
          (<TRUNC_SHIFT>:SD_HSDI
@@ -7500,7 +7500,7 @@
   [(set_attr "type" "neon_shift_imm_narrow_q")]
 )
 
-(define_expand "aarch64_<shrn_op>shrn_n<mode>"
+(define_expand "@aarch64_<shrn_op>shrn_n<mode>"
   [(set (match_operand:<VNARROWQ> 0 "register_operand")
        (ALL_TRUNC:<VNARROWQ>
          (<TRUNC_SHIFT>:VQN
@@ -7543,7 +7543,7 @@
   [(set_attr "type" "neon_shift_imm_narrow_q")]
 )
 
-(define_expand "aarch64_<shrn_op>rshrn_n<mode>"
+(define_expand "@aarch64_<shrn_op>rshrn_n<mode>"
   [(set (match_operand:<VNARROWQ> 0 "register_operand")
        (SAT_TRUNC:<VNARROWQ>
          (<TRUNC_SHIFT>:<V2XWIDE>
@@ -7563,7 +7563,7 @@
   }
 )
 
-(define_expand "aarch64_<shrn_op>rshrn_n<mode>"
+(define_expand "@aarch64_<shrn_op>rshrn_n<mode>"
   [(set (match_operand:<VNARROWQ> 0 "register_operand")
        (ALL_TRUNC:<VNARROWQ>
          (<TRUNC_SHIFT>:<V2XWIDE>
@@ -7633,7 +7633,7 @@
   [(set_attr "type" "neon_shift_imm_narrow_q")]
 )
 
-(define_expand "aarch64_sqshrun_n<mode>"
+(define_expand "@aarch64_sqshrun_n<mode>"
   [(match_operand:<VNARROWQ> 0 "register_operand")
    (match_operand:SD_HSDI 1 "register_operand")
    (match_operand:SI 2 "aarch64_simd_shift_imm_offset_<vn_mode>")]
@@ -7647,7 +7647,7 @@
   }
 )
 
-(define_expand "aarch64_sqshrun_n<mode>"
+(define_expand "@aarch64_sqshrun_n<mode>"
   [(set (match_operand:<VNARROWQ> 0 "register_operand")
        (us_truncate:<VNARROWQ>
          (smax:VQN
@@ -7698,7 +7698,7 @@
   [(set_attr "type" "neon_shift_imm_narrow_q")]
 )
 
-(define_expand "aarch64_sqrshrun_n<mode>"
+(define_expand "@aarch64_sqrshrun_n<mode>"
   [(match_operand:<VNARROWQ> 0 "register_operand")
    (match_operand:SD_HSDI 1 "register_operand")
    (match_operand:SI 2 "aarch64_simd_shift_imm_offset_<vn_mode>")]
@@ -7714,7 +7714,7 @@
   }
 )
 
-(define_expand "aarch64_sqrshrun_n<mode>"
+(define_expand "@aarch64_sqrshrun_n<mode>"
   [(set (match_operand:<VNARROWQ> 0 "register_operand")
        (us_truncate:<VNARROWQ>
          (smax:<V2XWIDE>
@@ -7764,7 +7764,7 @@
   [(set_attr "type" "neon_shift_imm_narrow_q")]
 )
 
-(define_expand "aarch64_<shrn_op><sra_op>shrn2_n<mode>"
+(define_expand "@aarch64_<shrn_op><sra_op>shrn2_n<mode>"
   [(match_operand:<VNARROWQ2> 0 "register_operand")
    (match_operand:<VNARROWQ> 1 "register_operand")
    (ALL_TRUNC:<VNARROWQ>
@@ -7851,7 +7851,7 @@
   [(set_attr "type" "neon_shift_imm_narrow_q")]
 )
 
-(define_expand "aarch64_<shrn_op>rshrn2_n<mode>"
+(define_expand "@aarch64_<shrn_op>rshrn2_n<mode>"
   [(match_operand:<VNARROWQ2> 0 "register_operand")
    (match_operand:<VNARROWQ> 1 "register_operand")
    (ALL_TRUNC:<VNARROWQ> (match_operand:VQN 2 "register_operand"))
@@ -7920,7 +7920,7 @@
   [(set_attr "type" "neon_shift_imm_narrow_q")]
 )
 
-(define_expand "aarch64_sqshrun2_n<mode>"
+(define_expand "@aarch64_sqshrun2_n<mode>"
   [(match_operand:<VNARROWQ2> 0 "register_operand")
    (match_operand:<VNARROWQ> 1 "register_operand")
    (match_operand:VQN 2 "register_operand")
@@ -7980,7 +7980,7 @@
   [(set_attr "type" "neon_shift_imm_narrow_q")]
 )
 
-(define_expand "aarch64_sqrshrun2_n<mode>"
+(define_expand "@aarch64_sqrshrun2_n<mode>"
   [(match_operand:<VNARROWQ2> 0 "register_operand")
    (match_operand:<VNARROWQ> 1 "register_operand")
    (match_operand:VQN 2 "register_operand")
diff --git a/gcc/config/aarch64/aarch64-sve-builtins-shapes.cc 
b/gcc/config/aarch64/aarch64-sve-builtins-shapes.cc
index 0af0d55366c4..341ec97fe072 100644
--- a/gcc/config/aarch64/aarch64-sve-builtins-shapes.cc
+++ b/gcc/config/aarch64/aarch64-sve-builtins-shapes.cc
@@ -112,6 +112,7 @@ apply_predication (function_instance &instance, tree 
return_type,
    f[01]   - a floating-point type with the same width as type suffix 0 or 1
    B       - bfloat16_t
    c       - a predicate-as-counter
+   d<elt>  - a double-width version of <elt>
    h<elt>  - a half-sized version of <elt>
    M       - mfloat8_t
    p       - a predicate (represented as TYPE_SUFFIX_b)
@@ -168,6 +169,13 @@ parse_element_type (const function_instance &instance, 
const char *&format)
                               type_suffixes[suffix].element_bits / 4);
     }
 
+  if (ch == 'd')
+    {
+      type_suffix_index suffix = parse_element_type (instance, format);
+      return find_type_suffix (type_suffixes[suffix].tclass,
+                              type_suffixes[suffix].element_bits * 2);
+    }
+
   if (ch == 'h')
     {
       type_suffix_index suffix = parse_element_type (instance, format);
diff --git a/gcc/testsuite/gcc.target/aarch64/neon/arm_neon_test.h 
b/gcc/testsuite/gcc.target/aarch64/neon/arm_neon_test.h
index 5a9d02decd62..81000be2b62b 100644
--- a/gcc/testsuite/gcc.target/aarch64/neon/arm_neon_test.h
+++ b/gcc/testsuite/gcc.target/aarch64/neon/arm_neon_test.h
@@ -9,6 +9,11 @@
 
 #define TEST_UNIFORM_UNARY(NAME, TYPE) TEST_UNARY (NAME, TYPE, TYPE)
 
+#define TEST_UNARY_N(NAME, RET_TYPE, ARG_1_TYPE, N)                            
\
+  RET_TYPE test_##NAME##_##N (ARG_1_TYPE a) { return NAME (a, N); }
+
+#define TEST_UNIFORM_UNARY_N(NAME, TYPE, N) TEST_UNARY_N (NAME, TYPE, TYPE, N)
+
 #define TEST_BINARY(NAME, RET_TYPE, ARG_1_TYPE, ARG_2_TYPE)                    
\
   RET_TYPE test_##NAME (ARG_1_TYPE a, ARG_2_TYPE b) { return NAME (a, b); }
 
@@ -20,6 +25,15 @@
 
 #define TEST_UNIFORM_BINARY(NAME, TYPE) TEST_BINARY (NAME, TYPE, TYPE, TYPE)
 
+#define TEST_BINARY_N(NAME, RET_TYPE, ARG_1_TYPE, ARG_2_TYPE, N)               
\
+  RET_TYPE test_##NAME##_##N (ARG_1_TYPE a, ARG_2_TYPE b)                      
\
+  {                                                                            
\
+    return NAME (a, b, N);                                                     
\
+  }
+
+#define TEST_UNIFORM_BINARY_N(NAME, TYPE, N)                                   
\
+  TEST_BINARY_N (NAME, TYPE, TYPE, TYPE, N)
+
 #define TEST_TERNARY(NAME, RET_TYPE, ARG_1_TYPE, ARG_2_TYPE, ARG_3_TYPE)       
\
   RET_TYPE test_##NAME (ARG_1_TYPE a, ARG_2_TYPE b, ARG_3_TYPE c)              
\
   {                                                                            
\

Reply via email to