https://gcc.gnu.org/g:ad9bdc5afb624ec8b3398adf3bf0fd7238f4e3ec
commit r16-6386-gad9bdc5afb624ec8b3398adf3bf0fd7238f4e3ec Author: Claudio Bantaloukas <[email protected]> Date: Wed Dec 24 11:41:27 2025 +0000 aarch64: Add the ability to have three types in an sve/sme intrinsic name The majority of sve/sme intrinsics have names which are defined by one type (like svuint8_t svextq[_u8]) or two types (like svsub_za32[_f32]_vg1x2). Some intrinsics now have three types (like svtmopa_lane_za32[_s8_u8]). This change extends the number of type_suffix_indexes from two to three to cover this case. gcc/ * config/aarch64/aarch64-sve-builtins-base.cc: (svmul_impl::fold): Replace use of type_suffix_pair with type_suffix_triple. * config/aarch64/aarch64-sve-builtins-shapes.cc: (parse_element_type): Handle third type suffix. (parse_type): Handle c2 in function signature. Add the u signature with the ability to pass a tuple with twice as many vectors as the base type. Calculate number of vectors against the type with the maximum number of bits rather than "the other one". (load_contiguous_base::resolve): Add argument to resolve_to call. (compare_scalar_def::resolve): Likewise. (ternary_mfloat8_def::resolve): Likewise. (ternary_mfloat8_lane_def::resolve): Likewise. (ternary_mfloat8_opt_n_def::resolve): Likewise. * config/aarch64/aarch64-sve-builtins.cc: (TYPES_all_pred, TYPES_all_count, TYPES_all_pred_count, TYPES_all_float, TYPES_all_signed, TYPES_all_float_and_signed, TYPES_all_unsigned, TYPES_all_integer, TYPES_all_arith, TYPES_all_data, TYPES_b, TYPES_c, TYPES_b_unsigned, TYPES_b_integer, TYPES_b_data, TYPES_bh_integer, TYPES_bs_unsigned, TYPES_bhs_signed, TYPES_bhs_unsigned, TYPES_bhs_integer, TYPES_bh_data, TYPES_bhs_data, TYPES_bhs_widen, TYPES_h_bfloat, TYPES_h_float, TYPES_h_integer, TYPES_h_data, TYPES_hs_signed, TYPES_hs_integer, TYPES_hs_float, TYPES_hs_data, TYPES_hd_unsigned, TYPES_hsd_signed, TYPES_hsd_integer, TYPES_hsd_data, TYPES_h_float_mf8, TYPES_s_float, TYPES_s_float_mf8, TYPES_s_float_hsd_integer, TYPES_s_float_sd_integer, TYPES_s_signed, TYPES_s_unsigned, TYPES_s_integer, TYPES_s_data, TYPES_sd_signed, TYPES_sd_unsigned, TYPES_sd_integer, TYPES_sd_data, TYPES_all_float_and_sd_integer, TYPES_d_float, TYPES_d_unsigned, TYPES_d_integer, TYPES_d_data, TYPES_cvt, TYPES_cvt_bfloat, TYPES_cvt_h_s_float, TYPES_cvt_f32_f16, TYPES_cvt_long, TYPES_cvt_narrow_s, TYPES_cvt_narrow, TYPES_cvt_s_s, TYPES_cvt_mf8, TYPES_cvtn_mf8, TYPES_cvtnx_mf8, TYPES_inc_dec_n, TYPES_qcvt_x2, TYPES_qcvt_x4, TYPES_qrshr_x2,TYPES_qrshru_x2, TYPES_qrshr_x4, TYPES_qrshru_x4, TYPES_reinterpret, TYPES_reinterpret_b,TYPES_while, TYPES_while_x, TYPES_while_x_c, TYPES_s_narrow_fsu,TYPES_all_za, TYPES_d_za, TYPES_za_bhsd_data,TYPES_za_all_data, TYPES_za_h_mf8, TYPES_za_hs_mf8, TYPES_za_h_bfloat, TYPES_za_h_float, TYPES_za_s_b_signed, TYPES_za_s_b_unsigned, TYPES_za_s_b_integer, TYPES_za_s_h_integer,TYPES_za_s_h_data, TYPES_za_s_unsigned, TYPES_za_s_integer, TYPES_za_s_mf8, TYPES_za_s_float, TYPES_za_s_data, TYPES_za_d_h_integer, TYPES_za_d_float, TYPES_za_d_integer, TYPES_mop_base, TYPES_mop_base_signed, TYPES_mop_base_unsigned, TYPES_mop_i16i64, TYPES_mop_i16i64_signed, TYPES_mop_i16i64_unsigned, ΤYPES_za): Extend defines to three arguments. (DEF_VECTOR_TYPE, DEF_DOUBLE_TYPE): Likewise. (DEF_TRIPLE_TYPE): Add new define. (DEF_SVE_TYPES_ARRAY): Redefine all types_ arrays into arrays of type_suffix_triple. (types_none): Likewise. (function_instance::hash): Add third type to hash calculation. (function_builder::get_name): Add third type to function name. (function_builder::add_overloaded_functions): Handle third type. (function_resolver::lookup_form): Likewise. (function_resolver::resolve_to): Likewise. (function_resolver::resolve_unary): Likewise. * config/aarch64/aarch64-sve-builtins.h: (type_suffix_triple): replace type_suffix_pair. (function_group_info::types): Likewise. (function_instance::ctor): Likewise. (function_instance::type_suffix_ids): Likewise. (function_resolver::lookup_form): Add third type argument. (function_resolver::resolve_to): Likewise. (function_instance::operator==): Add third type to equality calculation. Diff: --- gcc/config/aarch64/aarch64-sve-builtins-base.cc | 8 +- gcc/config/aarch64/aarch64-sve-builtins-shapes.cc | 41 ++- gcc/config/aarch64/aarch64-sve-builtins.cc | 336 +++++++++++----------- gcc/config/aarch64/aarch64-sve-builtins.h | 19 +- 4 files changed, 217 insertions(+), 187 deletions(-) diff --git a/gcc/config/aarch64/aarch64-sve-builtins-base.cc b/gcc/config/aarch64/aarch64-sve-builtins-base.cc index ca6e16578e8b..c3638624af06 100644 --- a/gcc/config/aarch64/aarch64-sve-builtins-base.cc +++ b/gcc/config/aarch64/aarch64-sve-builtins-base.cc @@ -2341,9 +2341,11 @@ public: tree negated_op = op1; if (integer_minus_onep (op1)) negated_op = op2; - type_suffix_pair signed_tsp = - {find_type_suffix (TYPE_signed, f.type_suffix (0).element_bits), - f.type_suffix_ids[1]}; + type_suffix_triple signed_tsp = { + find_type_suffix (TYPE_signed, f.type_suffix (0).element_bits), + f.type_suffix_ids[1], + NUM_TYPE_SUFFIXES + }; function_instance instance ("svneg", functions::svneg, shapes::unary, MODE_none, signed_tsp, GROUP_none, f.pred, FPM_unused); diff --git a/gcc/config/aarch64/aarch64-sve-builtins-shapes.cc b/gcc/config/aarch64/aarch64-sve-builtins-shapes.cc index 648f94395186..683e9603583b 100644 --- a/gcc/config/aarch64/aarch64-sve-builtins-shapes.cc +++ b/gcc/config/aarch64/aarch64-sve-builtins-shapes.cc @@ -178,7 +178,7 @@ parse_element_type (const function_instance &instance, const char *&format) type_suffixes[suffix].element_bits / 2); } - if (ch == '0' || ch == '1') + if (ch == '0' || ch == '1' || ch == '2') return instance.type_suffix_ids[ch - '0']; gcc_unreachable (); @@ -194,11 +194,13 @@ parse_element_type (const function_instance &instance, const char *&format) b - base vector type (from a _<m0>base suffix) c0 - the result of a conversion, based on type and group suffixes c1 - the source of a conversion, based on type and group suffixes + c2 - the source of a conversion, based on type and group suffixes d - displacement vector type (from a _<m1>index or _<m1>offset suffix) e<name> - an enum with the given name s<elt> - a scalar type with the given element suffix t<elt> - a vector or tuple type with given element suffix [*1] T<elt> - a vector or tuple type with given element suffix [*2] + u<elt> - a vector or tuple type with given element suffix [*3] v<elt> - a vector with the given element suffix D<elt> - a 64 bit neon vector Q<elt> - a 128 bit neon vector @@ -208,6 +210,7 @@ parse_element_type (const function_instance &instance, const char *&format) [*1] the vectors_per_tuple function indicates whether the type should be a tuple, and if so, how many vectors it should contain. [*2] same as for [*1], but the tuple contains half as many vectors. + [*3] same as for [*1], but the tuple contains twice as many vectors. */ static tree parse_type (const function_instance &instance, const char *&format) @@ -235,16 +238,19 @@ parse_type (const function_instance &instance, const char *&format) if (ch == 'c') { int ch = *format++; - gcc_assert (ch == '0' || ch == '1'); - unsigned int id = (ch == '0' ? 0 : 1); + gcc_assert (ch == '0' || ch == '1' || ch == '2'); + unsigned int id = ch - '0'; auto vector_type = instance.type_suffix (id).vector_type; unsigned int num_vectors = instance.group_suffix ().vectors_per_tuple; if (num_vectors != 1) { - unsigned int bits = instance.type_suffix (id).element_bits; - unsigned int other_bits = instance.type_suffix (1 - id).element_bits; - if (other_bits > bits) - num_vectors /= other_bits / bits; + unsigned int type_bits = instance.type_suffix (id).element_bits; + unsigned int max_bits = std::max ( std:: max ( + instance.type_suffix (0).element_bits, + instance.type_suffix (1).element_bits), + instance.type_suffix (2).element_bits); + if (max_bits > type_bits) + num_vectors /= max_bits / type_bits; } return acle_vector_types[num_vectors - 1][vector_type]; } @@ -288,6 +294,14 @@ parse_type (const function_instance &instance, const char *&format) return acle_vector_types[num_vectors - 1][vector_type]; } + if (ch == 'u') + { + type_suffix_index suffix = parse_element_type (instance, format); + vector_type_index vector_type = type_suffixes[suffix].vector_type; + unsigned int num_vectors = instance.vectors_per_tuple () * 2; + return acle_vector_types[num_vectors - 1][vector_type]; + } + if (ch == 'v') { type_suffix_index suffix = parse_element_type (instance, format); @@ -850,7 +864,7 @@ struct load_contiguous_base : public overloaded_base<0> return error_mark_node; return r.resolve_to (r.mode_suffix_id, type, NUM_TYPE_SUFFIXES, - r.group_suffix_id); + NUM_TYPE_SUFFIXES, r.group_suffix_id); } }; @@ -2318,7 +2332,7 @@ struct compare_scalar_def : public overloaded_base<1> return error_mark_node; return r.resolve_to (r.mode_suffix_id, r.type_suffix_ids[0], type, - r.group_suffix_id); + NUM_TYPE_SUFFIXES, r.group_suffix_id); } }; SHAPE (compare_scalar) @@ -4167,7 +4181,8 @@ struct ternary_mfloat8_def || !r.require_scalar_type (3, "uint64_t")) return error_mark_node; - return r.resolve_to (r.mode_suffix_id, type, TYPE_SUFFIX_mf8, GROUP_none); + return r.resolve_to (r.mode_suffix_id, type, TYPE_SUFFIX_mf8, + NUM_TYPE_SUFFIXES, GROUP_none); } }; SHAPE (ternary_mfloat8) @@ -4219,7 +4234,8 @@ struct ternary_mfloat8_lane_def || !r.require_scalar_type (4, "uint64_t")) return error_mark_node; - return r.resolve_to (r.mode_suffix_id, type, TYPE_SUFFIX_mf8, GROUP_none); + return r.resolve_to (r.mode_suffix_id, type, TYPE_SUFFIX_mf8, + NUM_TYPE_SUFFIXES, GROUP_none); } }; SHAPE (ternary_mfloat8_lane) @@ -4290,7 +4306,8 @@ struct ternary_mfloat8_opt_n_def else if (!r.require_vector_type (2, VECTOR_TYPE_svmfloat8_t)) return error_mark_node; - return r.resolve_to (mode, type, TYPE_SUFFIX_mf8, GROUP_none); + return r.resolve_to (mode, type, TYPE_SUFFIX_mf8, NUM_TYPE_SUFFIXES, + GROUP_none); } }; SHAPE (ternary_mfloat8_opt_n) diff --git a/gcc/config/aarch64/aarch64-sve-builtins.cc b/gcc/config/aarch64/aarch64-sve-builtins.cc index bf025adde890..79a1fca02917 100644 --- a/gcc/config/aarch64/aarch64-sve-builtins.cc +++ b/gcc/config/aarch64/aarch64-sve-builtins.cc @@ -192,255 +192,255 @@ CONSTEXPR const group_suffix_info group_suffixes[] = { class ("b", "f", etc.) and a numerical bit count. */ /* _b8 _b16 _b32 _b64. */ -#define TYPES_all_pred(S, D) \ +#define TYPES_all_pred(S, D, T) \ S (b8), S (b16), S (b32), S (b64) /* _c8 _c16 _c32 _c64. */ -#define TYPES_all_count(S, D) \ +#define TYPES_all_count(S, D, T) \ S (c8), S (c16), S (c32), S (c64) /* _b8 _b16 _b32 _b64 _c8 _c16 _c32 _c64. */ -#define TYPES_all_pred_count(S, D) \ - TYPES_all_pred (S, D), \ - TYPES_all_count (S, D) +#define TYPES_all_pred_count(S, D, T) \ + TYPES_all_pred (S, D, T), \ + TYPES_all_count (S, D, T) /* _f16 _f32 _f64. */ -#define TYPES_all_float(S, D) \ +#define TYPES_all_float(S, D, T) \ S (f16), S (f32), S (f64) /* _s8 _s16 _s32 _s64. */ -#define TYPES_all_signed(S, D) \ +#define TYPES_all_signed(S, D, T) \ S (s8), S (s16), S (s32), S (s64) /* _f16 _f32 _f64 _s8 _s16 _s32 _s64. */ -#define TYPES_all_float_and_signed(S, D) \ - TYPES_all_float (S, D), TYPES_all_signed (S, D) +#define TYPES_all_float_and_signed(S, D, T) \ + TYPES_all_float (S, D, T), TYPES_all_signed (S, D, T) /* _u8 _u16 _u32 _u64. */ -#define TYPES_all_unsigned(S, D) \ +#define TYPES_all_unsigned(S, D, T) \ S (u8), S (u16), S (u32), S (u64) /* _s8 _s16 _s32 _s64 _u8 _u16 _u32 _u64. */ -#define TYPES_all_integer(S, D) \ - TYPES_all_signed (S, D), TYPES_all_unsigned (S, D) +#define TYPES_all_integer(S, D, T) \ + TYPES_all_signed (S, D, T), TYPES_all_unsigned (S, D, T) /* _f16 _f32 _f64 _s8 _s16 _s32 _s64 _u8 _u16 _u32 _u64. */ -#define TYPES_all_arith(S, D) \ - TYPES_all_float (S, D), TYPES_all_integer (S, D) +#define TYPES_all_arith(S, D, T) \ + TYPES_all_float (S, D, T), TYPES_all_integer (S, D, T) -#define TYPES_all_data(S, D) \ - TYPES_b_data (S, D), \ - TYPES_h_data (S, D), \ - TYPES_s_data (S, D), \ - TYPES_d_data (S, D) +#define TYPES_all_data(S, D, T) \ + TYPES_b_data (S, D, T), \ + TYPES_h_data (S, D, T), \ + TYPES_s_data (S, D, T), \ + TYPES_d_data (S, D, T) /* _b only. */ -#define TYPES_b(S, D) \ +#define TYPES_b(S, D, T) \ S (b) /* _c only. */ -#define TYPES_c(S, D) \ +#define TYPES_c(S, D, T) \ S (c) /* _u8. */ -#define TYPES_b_unsigned(S, D) \ +#define TYPES_b_unsigned(S, D, T) \ S (u8) /* _s8 _u8. */ -#define TYPES_b_integer(S, D) \ - S (s8), TYPES_b_unsigned (S, D) +#define TYPES_b_integer(S, D, T) \ + S (s8), TYPES_b_unsigned (S, D, T) /* _mf8 _s8 _u8. */ -#define TYPES_b_data(S, D) \ - S (mf8), TYPES_b_integer (S, D) +#define TYPES_b_data(S, D, T) \ + S (mf8), TYPES_b_integer (S, D, T) /* _s8 _s16 _u8 _u16. */ -#define TYPES_bh_integer(S, D) \ +#define TYPES_bh_integer(S, D, T) \ S (s8), S (s16), S (u8), S (u16) /* _u8 _u32. */ -#define TYPES_bs_unsigned(S, D) \ +#define TYPES_bs_unsigned(S, D, T) \ S (u8), S (u32) /* _s8 _s16 _s32. */ -#define TYPES_bhs_signed(S, D) \ +#define TYPES_bhs_signed(S, D, T) \ S (s8), S (s16), S (s32) /* _u8 _u16 _u32. */ -#define TYPES_bhs_unsigned(S, D) \ +#define TYPES_bhs_unsigned(S, D, T) \ S (u8), S (u16), S (u32) /* _s8 _s16 _s32 _u8 _u16 _u32. */ -#define TYPES_bhs_integer(S, D) \ - TYPES_bhs_signed (S, D), TYPES_bhs_unsigned (S, D) +#define TYPES_bhs_integer(S, D, T) \ + TYPES_bhs_signed (S, D, T), TYPES_bhs_unsigned (S, D, T) -#define TYPES_bh_data(S, D) \ - TYPES_b_data (S, D), \ - TYPES_h_data (S, D) +#define TYPES_bh_data(S, D, T) \ + TYPES_b_data (S, D, T), \ + TYPES_h_data (S, D, T) -#define TYPES_bhs_data(S, D) \ - TYPES_b_data (S, D), \ - TYPES_h_data (S, D), \ - TYPES_s_data (S, D) +#define TYPES_bhs_data(S, D, T) \ + TYPES_b_data (S, D, T), \ + TYPES_h_data (S, D, T), \ + TYPES_s_data (S, D, T) /* _s16_s8 _s32_s16 _s64_s32 _u16_u8 _u32_u16 _u64_u32. */ -#define TYPES_bhs_widen(S, D) \ +#define TYPES_bhs_widen(S, D, T) \ D (s16, s8), D (s32, s16), D (s64, s32), \ D (u16, u8), D (u32, u16), D (u64, u32) /* _bf16. */ -#define TYPES_h_bfloat(S, D) \ +#define TYPES_h_bfloat(S, D, T) \ S (bf16) /* _f16. */ -#define TYPES_h_float(S, D) \ +#define TYPES_h_float(S, D, T) \ S (f16) /* _s16 _u16. */ -#define TYPES_h_integer(S, D) \ +#define TYPES_h_integer(S, D, T) \ S (s16), S (u16) /* _bf16 _f16 _s16 _u16. */ -#define TYPES_h_data(S, D) \ - S (bf16), S (f16), TYPES_h_integer (S, D) +#define TYPES_h_data(S, D, T) \ + S (bf16), S (f16), TYPES_h_integer (S, D, T) /* _s16 _s32. */ -#define TYPES_hs_signed(S, D) \ +#define TYPES_hs_signed(S, D, T) \ S (s16), S (s32) /* _s16 _s32 _u16 _u32. */ -#define TYPES_hs_integer(S, D) \ - TYPES_hs_signed (S, D), S (u16), S (u32) +#define TYPES_hs_integer(S, D, T) \ + TYPES_hs_signed (S, D, T), S (u16), S (u32) /* _f16 _f32. */ -#define TYPES_hs_float(S, D) \ +#define TYPES_hs_float(S, D, T) \ S (f16), S (f32) -#define TYPES_hs_data(S, D) \ - TYPES_h_data (S, D), \ - TYPES_s_data (S, D) +#define TYPES_hs_data(S, D, T) \ + TYPES_h_data (S, D, T), \ + TYPES_s_data (S, D, T) /* _u16 _u64. */ -#define TYPES_hd_unsigned(S, D) \ +#define TYPES_hd_unsigned(S, D, T) \ S (u16), S (u64) /* _s16 _s32 _s64. */ -#define TYPES_hsd_signed(S, D) \ +#define TYPES_hsd_signed(S, D, T) \ S (s16), S (s32), S (s64) /* _s16 _s32 _s64 _u16 _u32 _u64. */ -#define TYPES_hsd_integer(S, D) \ - TYPES_hsd_signed (S, D), S (u16), S (u32), S (u64) +#define TYPES_hsd_integer(S, D, T) \ + TYPES_hsd_signed (S, D, T), S (u16), S (u32), S (u64) -#define TYPES_hsd_data(S, D) \ - TYPES_h_data (S, D), \ - TYPES_s_data (S, D), \ - TYPES_d_data (S, D) +#define TYPES_hsd_data(S, D, T) \ + TYPES_h_data (S, D, T), \ + TYPES_s_data (S, D, T), \ + TYPES_d_data (S, D, T) /* _f16_mf8. */ -#define TYPES_h_float_mf8(S, D) \ +#define TYPES_h_float_mf8(S, D, T) \ D (f16, mf8) /* _f32. */ -#define TYPES_s_float(S, D) \ +#define TYPES_s_float(S, D, T) \ S (f32) /* _f32_mf8. */ -#define TYPES_s_float_mf8(S, D) \ +#define TYPES_s_float_mf8(S, D, T) \ D (f32, mf8) /* _f32 _s16 _s32 _s64 _u16 _u32 _u64. */ -#define TYPES_s_float_hsd_integer(S, D) \ - TYPES_s_float (S, D), TYPES_hsd_integer (S, D) +#define TYPES_s_float_hsd_integer(S, D, T) \ + TYPES_s_float (S, D, T), TYPES_hsd_integer (S, D, T) /* _f32 _s32 _s64 _u32 _u64. */ -#define TYPES_s_float_sd_integer(S, D) \ - TYPES_s_float (S, D), TYPES_sd_integer (S, D) +#define TYPES_s_float_sd_integer(S, D, T) \ + TYPES_s_float (S, D, T), TYPES_sd_integer (S, D, T) /* _s32. */ -#define TYPES_s_signed(S, D) \ +#define TYPES_s_signed(S, D, T) \ S (s32) /* _u32. */ -#define TYPES_s_unsigned(S, D) \ +#define TYPES_s_unsigned(S, D, T) \ S (u32) /* _s32 _u32. */ -#define TYPES_s_integer(S, D) \ - TYPES_s_signed (S, D), TYPES_s_unsigned (S, D) +#define TYPES_s_integer(S, D, T) \ + TYPES_s_signed (S, D, T), TYPES_s_unsigned (S, D, T) /* _f32 _s32 _u32. */ -#define TYPES_s_data(S, D) \ - TYPES_s_float (S, D), TYPES_s_integer (S, D) +#define TYPES_s_data(S, D, T) \ + TYPES_s_float (S, D, T), TYPES_s_integer (S, D, T) /* _s32 _s64. */ -#define TYPES_sd_signed(S, D) \ +#define TYPES_sd_signed(S, D, T) \ S (s32), S (s64) /* _u32 _u64. */ -#define TYPES_sd_unsigned(S, D) \ +#define TYPES_sd_unsigned(S, D, T) \ S (u32), S (u64) /* _s32 _s64 _u32 _u64. */ -#define TYPES_sd_integer(S, D) \ - TYPES_sd_signed (S, D), TYPES_sd_unsigned (S, D) +#define TYPES_sd_integer(S, D, T) \ + TYPES_sd_signed (S, D, T), TYPES_sd_unsigned (S, D, T) -#define TYPES_sd_data(S, D) \ - TYPES_s_data (S, D), \ - TYPES_d_data (S, D) +#define TYPES_sd_data(S, D, T) \ + TYPES_s_data (S, D, T), \ + TYPES_d_data (S, D, T) /* _f16 _f32 _f64 _s32 _s64 _u32 _u64. */ -#define TYPES_all_float_and_sd_integer(S, D) \ - TYPES_all_float (S, D), TYPES_sd_integer (S, D) +#define TYPES_all_float_and_sd_integer(S, D, T) \ + TYPES_all_float (S, D, T), TYPES_sd_integer (S, D, T) /* _f64. */ -#define TYPES_d_float(S, D) \ +#define TYPES_d_float(S, D, T) \ S (f64) /* _u64. */ -#define TYPES_d_unsigned(S, D) \ +#define TYPES_d_unsigned(S, D, T) \ S (u64) /* _s64 _u64. */ -#define TYPES_d_integer(S, D) \ - S (s64), TYPES_d_unsigned (S, D) +#define TYPES_d_integer(S, D, T) \ + S (s64), TYPES_d_unsigned (S, D, T) /* _f64 _s64 _u64. */ -#define TYPES_d_data(S, D) \ - TYPES_d_float (S, D), TYPES_d_integer (S, D) +#define TYPES_d_data(S, D, T) \ + TYPES_d_float (S, D, T), TYPES_d_integer (S, D, T) /* All the type combinations allowed by svcvt. */ -#define TYPES_cvt(S, D) \ +#define TYPES_cvt(S, D, T) \ D (f16, f32), D (f16, f64), \ D (f16, s16), D (f16, s32), D (f16, s64), \ D (f16, u16), D (f16, u32), D (f16, u64), \ @@ -462,35 +462,35 @@ CONSTEXPR const group_suffix_info group_suffixes[] = { D (u64, f16), D (u64, f32), D (u64, f64) /* _bf16_f32. */ -#define TYPES_cvt_bfloat(S, D) \ +#define TYPES_cvt_bfloat(S, D, T) \ D (bf16, f32) /* { _bf16 _f16 } x _f32. */ -#define TYPES_cvt_h_s_float(S, D) \ +#define TYPES_cvt_h_s_float(S, D, T) \ D (bf16, f32), D (f16, f32) /* _f32_f16. */ -#define TYPES_cvt_f32_f16(S, D) \ +#define TYPES_cvt_f32_f16(S, D, T) \ D (f32, f16) /* _f32_f16 _f64_f32. */ -#define TYPES_cvt_long(S, D) \ +#define TYPES_cvt_long(S, D, T) \ D (f32, f16), D (f64, f32) /* _f16_f32. */ -#define TYPES_cvt_narrow_s(S, D) \ +#define TYPES_cvt_narrow_s(S, D, T) \ D (f32, f64) /* _f16_f32 _f32_f64. */ -#define TYPES_cvt_narrow(S, D) \ - D (f16, f32), TYPES_cvt_narrow_s (S, D) +#define TYPES_cvt_narrow(S, D, T) \ + D (f16, f32), TYPES_cvt_narrow_s (S, D, T) /* { _s32 _u32 } x _f32 _f32 x { _s32 _u32 }. */ -#define TYPES_cvt_s_s(S, D) \ +#define TYPES_cvt_s_s(S, D, T) \ D (s32, f32), \ D (u32, f32), \ D (f32, s32), \ @@ -498,23 +498,23 @@ CONSTEXPR const group_suffix_info group_suffixes[] = { /* _f16_mf8 _bf16_mf8. */ -#define TYPES_cvt_mf8(S, D) \ +#define TYPES_cvt_mf8(S, D, T) \ D (f16, mf8), D (bf16, mf8) /* _mf8_f16 _mf8_bf16. */ -#define TYPES_cvtn_mf8(S, D) \ +#define TYPES_cvtn_mf8(S, D, T) \ D (mf8, f16), D (mf8, bf16) /* _mf8_f32. */ -#define TYPES_cvtnx_mf8(S, D) \ +#define TYPES_cvtnx_mf8(S, D, T) \ D (mf8, f32) /* { _s32 _s64 } x { _b8 _b16 _b32 _b64 } { _u32 _u64 }. */ #define TYPES_inc_dec_n1(D, A) \ D (A, b8), D (A, b16), D (A, b32), D (A, b64) -#define TYPES_inc_dec_n(S, D) \ +#define TYPES_inc_dec_n(S, D, T) \ TYPES_inc_dec_n1 (D, s32), \ TYPES_inc_dec_n1 (D, s64), \ TYPES_inc_dec_n1 (D, u32), \ @@ -523,7 +523,7 @@ CONSTEXPR const group_suffix_info group_suffixes[] = { /* { _s16 _u16 } x _s32 { _u16 } x _u32. */ -#define TYPES_qcvt_x2(S, D) \ +#define TYPES_qcvt_x2(S, D, T) \ D (s16, s32), \ D (u16, u32), \ D (u16, s32) @@ -535,7 +535,7 @@ CONSTEXPR const group_suffix_info group_suffixes[] = { { _s16 _u16 } x _s64 { _u16 } x _u64. */ -#define TYPES_qcvt_x4(S, D) \ +#define TYPES_qcvt_x4(S, D, T) \ D (s8, s32), \ D (u8, u32), \ D (u8, s32), \ @@ -545,19 +545,19 @@ CONSTEXPR const group_suffix_info group_suffixes[] = { /* _s16_s32 _u16_u32. */ -#define TYPES_qrshr_x2(S, D) \ +#define TYPES_qrshr_x2(S, D, T) \ D (s16, s32), \ D (u16, u32) /* _u16_s32. */ -#define TYPES_qrshru_x2(S, D) \ +#define TYPES_qrshru_x2(S, D, T) \ D (u16, s32) /* _s8_s32 _s16_s64 _u8_u32 _u16_u64. */ -#define TYPES_qrshr_x4(S, D) \ +#define TYPES_qrshr_x4(S, D, T) \ D (s8, s32), \ D (s16, s64), \ D (u8, u32), \ @@ -565,7 +565,7 @@ CONSTEXPR const group_suffix_info group_suffixes[] = { /* _u8_s32 _u16_s64. */ -#define TYPES_qrshru_x4(S, D) \ +#define TYPES_qrshru_x4(S, D, T) \ D (u8, s32), \ D (u16, s64) @@ -579,7 +579,7 @@ CONSTEXPR const group_suffix_info group_suffixes[] = { D (A, f16), D (A, f32), D (A, f64), \ D (A, s8), D (A, s16), D (A, s32), D (A, s64), \ D (A, u8), D (A, u16), D (A, u32), D (A, u64) -#define TYPES_reinterpret(S, D) \ +#define TYPES_reinterpret(S, D, T) \ TYPES_reinterpret1 (D, mf8), \ TYPES_reinterpret1 (D, bf16), \ TYPES_reinterpret1 (D, f16), \ @@ -596,7 +596,7 @@ CONSTEXPR const group_suffix_info group_suffixes[] = { /* _b_c _c_b. */ -#define TYPES_reinterpret_b(S, D) \ +#define TYPES_reinterpret_b(S, D, T) \ D (b, c), \ D (c, b) @@ -604,7 +604,7 @@ CONSTEXPR const group_suffix_info group_suffixes[] = { { _u32 _u64 } */ #define TYPES_while1(D, bn) \ D (bn, s32), D (bn, s64), D (bn, u32), D (bn, u64) -#define TYPES_while(S, D) \ +#define TYPES_while(S, D, T) \ TYPES_while1 (D, b8), \ TYPES_while1 (D, b16), \ TYPES_while1 (D, b32), \ @@ -612,7 +612,7 @@ CONSTEXPR const group_suffix_info group_suffixes[] = { /* { _b8 _b16 _b32 _b64 } x { _s64 } { _u64 } */ -#define TYPES_while_x(S, D) \ +#define TYPES_while_x(S, D, T) \ D (b8, s64), D (b8, u64), \ D (b16, s64), D (b16, u64), \ D (b32, s64), D (b32, u64), \ @@ -620,7 +620,7 @@ CONSTEXPR const group_suffix_info group_suffixes[] = { /* { _c8 _c16 _c32 _c64 } x { _s64 } { _u64 } */ -#define TYPES_while_x_c(S, D) \ +#define TYPES_while_x_c(S, D, T) \ D (c8, s64), D (c8, u64), \ D (c16, s64), D (c16, u64), \ D (c32, s64), D (c32, u64), \ @@ -629,15 +629,15 @@ CONSTEXPR const group_suffix_info group_suffixes[] = { /* _f32_f16 _s32_s16 _u32_u16. */ -#define TYPES_s_narrow_fsu(S, D) \ +#define TYPES_s_narrow_fsu(S, D, T) \ D (f32, f16), D (s32, s16), D (u32, u16) /* _za8 _za16 _za32 _za64 _za128. */ -#define TYPES_all_za(S, D) \ +#define TYPES_all_za(S, D, T) \ S (za8), S (za16), S (za32), S (za64), S (za128) /* _za64. */ -#define TYPES_d_za(S, D) \ +#define TYPES_d_za(S, D, T) \ S (za64) /* { _za8 } x { _mf8 _s8 _u8 } @@ -647,7 +647,7 @@ CONSTEXPR const group_suffix_info group_suffixes[] = { { _za32 } x { _f32 _s32 _u32 } { _za64 } x { _f64 _s64 _u64 }. */ -#define TYPES_za_bhsd_data(S, D) \ +#define TYPES_za_bhsd_data(S, D, T) \ D (za8, mf8), D (za8, s8), D (za8, u8), \ D (za16, bf16), D (za16, f16), D (za16, s16), D (za16, u16), \ D (za32, f32), D (za32, s32), D (za32, u32), \ @@ -660,128 +660,130 @@ CONSTEXPR const group_suffix_info group_suffixes[] = { { _s8 _s16 _s32 _s64 } { _u8 _u16 _u32 _u64 }. */ -#define TYPES_za_all_data(S, D) \ - TYPES_za_bhsd_data (S, D), \ +#define TYPES_za_all_data(S, D, T) \ + TYPES_za_bhsd_data (S, D, T), \ TYPES_reinterpret1 (D, za128) /* _za16_mf8. */ -#define TYPES_za_h_mf8(S, D) \ +#define TYPES_za_h_mf8(S, D, T) \ D (za16, mf8) -/* _za32_mf8. */ -#define TYPES_za_s_mf8(S, D) \ - D (za32, mf8) - /* { _za_16 _za_32 } x _mf8. */ -#define TYPES_za_hs_mf8(S, D) \ +#define TYPES_za_hs_mf8(S, D, T) \ D (za16, mf8), D (za32, mf8) /* _za16_bf16. */ -#define TYPES_za_h_bfloat(S, D) \ +#define TYPES_za_h_bfloat(S, D, T) \ D (za16, bf16) /* _za16_f16. */ -#define TYPES_za_h_float(S, D) \ +#define TYPES_za_h_float(S, D, T) \ D (za16, f16) /* _za32_s8. */ -#define TYPES_za_s_b_signed(S, D) \ +#define TYPES_za_s_b_signed(S, D, T) \ D (za32, s8) /* _za32_u8. */ -#define TYPES_za_s_b_unsigned(S, D) \ +#define TYPES_za_s_b_unsigned(S, D, T) \ D (za32, u8) /* _za32 x { _s8 _u8 }. */ -#define TYPES_za_s_b_integer(S, D) \ +#define TYPES_za_s_b_integer(S, D, T) \ D (za32, s8), D (za32, u8) /* _za32 x { _s16 _u16 }. */ -#define TYPES_za_s_h_integer(S, D) \ +#define TYPES_za_s_h_integer(S, D, T) \ D (za32, s16), D (za32, u16) /* _za32 x { _bf16 _f16 _s16 _u16 }. */ -#define TYPES_za_s_h_data(S, D) \ +#define TYPES_za_s_h_data(S, D, T) \ D (za32, bf16), D (za32, f16), D (za32, s16), D (za32, u16) /* _za32_u32. */ -#define TYPES_za_s_unsigned(S, D) \ +#define TYPES_za_s_unsigned(S, D, T) \ D (za32, u32) /* _za32 x { _s32 _u32 }. */ -#define TYPES_za_s_integer(S, D) \ +#define TYPES_za_s_integer(S, D, T) \ D (za32, s32), D (za32, u32) /* _za32_mf8. */ -#define TYPES_za_s_mf8(S, D) \ +#define TYPES_za_s_mf8(S, D, T) \ D (za32, mf8) /* _za32_f32. */ -#define TYPES_za_s_float(S, D) \ +#define TYPES_za_s_float(S, D, T) \ D (za32, f32) /* _za32 x { _f32 _s32 _u32 }. */ -#define TYPES_za_s_data(S, D) \ +#define TYPES_za_s_data(S, D, T) \ D (za32, f32), D (za32, s32), D (za32, u32) /* _za64 x { _s16 _u16 }. */ -#define TYPES_za_d_h_integer(S, D) \ +#define TYPES_za_d_h_integer(S, D, T) \ D (za64, s16), D (za64, u16) /* _za64_f64. */ -#define TYPES_za_d_float(S, D) \ +#define TYPES_za_d_float(S, D, T) \ D (za64, f64) /* _za64 x { _s64 _u64 }. */ -#define TYPES_za_d_integer(S, D) \ +#define TYPES_za_d_integer(S, D, T) \ D (za64, s64), D (za64, u64) /* _za32 x { _s8 _u8 _bf16 _f16 _f32 }. */ -#define TYPES_mop_base(S, D) \ +#define TYPES_mop_base(S, D, T) \ D (za32, s8), D (za32, u8), D (za32, bf16), D (za32, f16), D (za32, f32) /* _za32_s8. */ -#define TYPES_mop_base_signed(S, D) \ +#define TYPES_mop_base_signed(S, D, T) \ D (za32, s8) /* _za32_u8. */ -#define TYPES_mop_base_unsigned(S, D) \ +#define TYPES_mop_base_unsigned(S, D, T) \ D (za32, u8) /* _za64 x { _s16 _u16 }. */ -#define TYPES_mop_i16i64(S, D) \ +#define TYPES_mop_i16i64(S, D, T) \ D (za64, s16), D (za64, u16) /* _za64_s16. */ -#define TYPES_mop_i16i64_signed(S, D) \ +#define TYPES_mop_i16i64_signed(S, D, T) \ D (za64, s16) /* _za64_u16. */ -#define TYPES_mop_i16i64_unsigned(S, D) \ +#define TYPES_mop_i16i64_unsigned(S, D, T) \ D (za64, u16) /* _za. */ -#define TYPES_za(S, D) \ +#define TYPES_za(S, D, T) \ S (za) -/* Describe a pair of type suffixes in which only the first is used. */ -#define DEF_VECTOR_TYPE(X) { TYPE_SUFFIX_ ## X, NUM_TYPE_SUFFIXES } +/* Describe a tuple of type suffixes in which only the first is used. */ +#define DEF_VECTOR_TYPE(X) \ + { TYPE_SUFFIX_ ## X, NUM_TYPE_SUFFIXES, NUM_TYPE_SUFFIXES } + +/* Describe a tuple of type suffixes in which only the first two are used. */ +#define DEF_DOUBLE_TYPE(X, Y) \ + { TYPE_SUFFIX_ ## X, TYPE_SUFFIX_ ## Y, NUM_TYPE_SUFFIXES } -/* Describe a pair of type suffixes in which both are used. */ -#define DEF_DOUBLE_TYPE(X, Y) { TYPE_SUFFIX_ ## X, TYPE_SUFFIX_ ## Y } +/* Describe a tuple of type suffixes in which three elements are used. */ +#define DEF_TRIPLE_TYPE(X, Y, Z) \ + { TYPE_SUFFIX_ ## X, TYPE_SUFFIX_ ## Y, TYPE_SUFFIX_ ## Z } /* Create an array that can be used in aarch64-sve-builtins.def to select the type suffixes in TYPES_<NAME>. */ #define DEF_SVE_TYPES_ARRAY(NAME) \ - static const type_suffix_pair types_##NAME[] = { \ - TYPES_##NAME (DEF_VECTOR_TYPE, DEF_DOUBLE_TYPE), \ - { NUM_TYPE_SUFFIXES, NUM_TYPE_SUFFIXES } \ + static const type_suffix_triple types_##NAME[] = { \ + TYPES_##NAME (DEF_VECTOR_TYPE, DEF_DOUBLE_TYPE, DEF_TRIPLE_TYPE), \ + { NUM_TYPE_SUFFIXES, NUM_TYPE_SUFFIXES, NUM_TYPE_SUFFIXES } \ } /* For functions that don't take any type suffixes. */ -static const type_suffix_pair types_none[] = { - { NUM_TYPE_SUFFIXES, NUM_TYPE_SUFFIXES }, - { NUM_TYPE_SUFFIXES, NUM_TYPE_SUFFIXES } +static const type_suffix_triple types_none[] = { + { NUM_TYPE_SUFFIXES, NUM_TYPE_SUFFIXES, NUM_TYPE_SUFFIXES }, + { NUM_TYPE_SUFFIXES, NUM_TYPE_SUFFIXES, NUM_TYPE_SUFFIXES } }; /* Create an array for each TYPES_<combination> macro above. */ @@ -1226,6 +1228,7 @@ function_instance::hash () const h.add_int (mode_suffix_id); h.add_int (type_suffix_ids[0]); h.add_int (type_suffix_ids[1]); + h.add_int (type_suffix_ids[2]); h.add_int (group_suffix_id); h.add_int (pred); h.add_int (fpm_mode); @@ -1397,7 +1400,7 @@ function_builder::get_name (const function_instance &instance, } else append_name (instance.mode_suffix ().string); - for (unsigned int i = 0; i < 2; ++i) + for (unsigned int i = 0; i < 3; ++i) if (!overloaded_p || instance.shape->explicit_type_suffix_p (i)) append_name (instance.type_suffix (i).string); if (!overloaded_p || instance.shape->explicit_group_suffix_p ()) @@ -1642,8 +1645,9 @@ function_builder::add_overloaded_functions (const function_group_info &group, { bool explicit_type0 = (*group.shape)->explicit_type_suffix_p (0); bool explicit_type1 = (*group.shape)->explicit_type_suffix_p (1); + bool explicit_type2 = (*group.shape)->explicit_type_suffix_p (2); bool explicit_group = (*group.shape)->explicit_group_suffix_p (); - auto add_function = [&](const type_suffix_pair &types, + auto add_function = [&](const type_suffix_triple &types, group_suffix_index group_suffix_id, unsigned int pi) { @@ -1671,9 +1675,10 @@ function_builder::add_overloaded_functions (const function_group_info &group, { /* Stub out the types that are determined by overload resolution. */ - type_suffix_pair types = { + type_suffix_triple types = { explicit_type0 ? group.types[ti][0] : NUM_TYPE_SUFFIXES, - explicit_type1 ? group.types[ti][1] : NUM_TYPE_SUFFIXES + explicit_type1 ? group.types[ti][1] : NUM_TYPE_SUFFIXES, + explicit_type2 ? group.types[ti][2] : NUM_TYPE_SUFFIXES }; add_function (types, group_suffix_id, pi); } @@ -1824,9 +1829,10 @@ tree function_resolver::lookup_form (mode_suffix_index mode, type_suffix_index type0, type_suffix_index type1, + type_suffix_index type2, group_suffix_index group) { - type_suffix_pair types = { type0, type1 }; + type_suffix_triple types = { type0, type1, type2 }; function_instance instance (base_name, base, shape, mode, types, group, pred, fpm_mode); registered_function *rfn @@ -1846,13 +1852,14 @@ function_resolver::lookup_form (mode_suffix_index mode, sve_type type) { type_suffix_index type0 = type_suffix_ids[0]; type_suffix_index type1 = type_suffix_ids[1]; + type_suffix_index type2 = type_suffix_ids[2]; (type0 == NUM_TYPE_SUFFIXES ? type0 : type1) = type.type; group_suffix_index group = group_suffix_id; if (group == GROUP_none && type.num_vectors != vectors_per_tuple ()) group = num_vectors_to_group (type.num_vectors); - return lookup_form (mode, type0, type1, group); + return lookup_form (mode, type0, type1, type2, group); } /* Resolve the function to one with the mode suffix given by MODE, the @@ -1863,9 +1870,10 @@ tree function_resolver::resolve_to (mode_suffix_index mode, type_suffix_index type0, type_suffix_index type1, + type_suffix_index type2, group_suffix_index group) { - tree res = lookup_form (mode, type0, type1, group); + tree res = lookup_form (mode, type0, type1, type2, group); if (!res) { if (type1 == NUM_TYPE_SUFFIXES) @@ -3227,7 +3235,7 @@ function_resolver::resolve_unary (type_class_index merge_tclass, explicit. */ if (type_suffix_ids[0] != NUM_TYPE_SUFFIXES) return resolve_to (mode_suffix_id, type_suffix_ids[0], type, - group_suffix_id); + NUM_TYPE_SUFFIXES, group_suffix_id); return resolve_to (mode_suffix_id, type); } diff --git a/gcc/config/aarch64/aarch64-sve-builtins.h b/gcc/config/aarch64/aarch64-sve-builtins.h index 6098d8ff229f..2d0fb0501da1 100644 --- a/gcc/config/aarch64/aarch64-sve-builtins.h +++ b/gcc/config/aarch64/aarch64-sve-builtins.h @@ -38,7 +38,7 @@ - mode_suffix_index represents the mode suffix - type_suffix_index represents individual type suffixes, while - type_suffix_pair represents a pair of them + type_suffix_triple represents a tuple of them - prediction_index extends the predication suffix with an additional alternative: PRED_implicit for implicitly-predicated operations @@ -227,8 +227,8 @@ enum group_suffix_index NUM_GROUP_SUFFIXES }; -/* Combines two type suffixes. */ -typedef enum type_suffix_index type_suffix_pair[2]; +/* Combines three type suffixes. */ +typedef enum type_suffix_index type_suffix_triple[3]; class function_base; class function_shape; @@ -367,12 +367,12 @@ struct function_group_info /* A list of the available type suffixes, group suffixes, and predication types. The function supports every combination of the three. - The list of type suffixes is terminated by two NUM_TYPE_SUFFIXES. + The list of type suffixes is terminated by three NUM_TYPE_SUFFIXES. It is lexicographically ordered based on the index value. The list of group suffixes is terminated by NUM_GROUP_SUFFIXES and the list of predication types is terminated by NUM_PREDS. */ - const type_suffix_pair *types; + const type_suffix_triple *types; const group_suffix_index *groups; const predication_index *preds; @@ -390,7 +390,7 @@ class GTY((user)) function_instance public: function_instance (const char *, const function_base *, const function_shape *, mode_suffix_index, - const type_suffix_pair &, group_suffix_index, + const type_suffix_triple &, group_suffix_index, predication_index, fpm_mode_index); bool operator== (const function_instance &) const; @@ -434,7 +434,7 @@ public: const function_base *base; const function_shape *shape; mode_suffix_index mode_suffix_id; - type_suffix_pair type_suffix_ids; + type_suffix_triple type_suffix_ids; group_suffix_index group_suffix_id; predication_index pred; fpm_mode_index fpm_mode; @@ -525,11 +525,13 @@ public: tree report_no_such_form (sve_type); tree lookup_form (mode_suffix_index, + type_suffix_index = NUM_TYPE_SUFFIXES, type_suffix_index = NUM_TYPE_SUFFIXES, type_suffix_index = NUM_TYPE_SUFFIXES, group_suffix_index = GROUP_none); tree lookup_form (mode_suffix_index, sve_type); tree resolve_to (mode_suffix_index, + type_suffix_index = NUM_TYPE_SUFFIXES, type_suffix_index = NUM_TYPE_SUFFIXES, type_suffix_index = NUM_TYPE_SUFFIXES, group_suffix_index = GROUP_none); @@ -907,7 +909,7 @@ inline function_instance:: function_instance (const char *base_name_in, const function_base *base_in, const function_shape *shape_in, mode_suffix_index mode_suffix_id_in, - const type_suffix_pair &type_suffix_ids_in, + const type_suffix_triple &type_suffix_ids_in, group_suffix_index group_suffix_id_in, predication_index pred_in, fpm_mode_index fpm_mode_in) : base_name (base_name_in), base (base_in), shape (shape_in), @@ -925,6 +927,7 @@ function_instance::operator== (const function_instance &other) const && mode_suffix_id == other.mode_suffix_id && type_suffix_ids[0] == other.type_suffix_ids[0] && type_suffix_ids[1] == other.type_suffix_ids[1] + && type_suffix_ids[2] == other.type_suffix_ids[2] && group_suffix_id == other.group_suffix_id && pred == other.pred && fpm_mode == other.fpm_mode);
