Port the v{ld,st}{2,3,4} families of intrinsics to the pragma-based
framework, and add corresponding assembly tests for each variant.
Lowering the functions to `IFN_LOAD_LANES`/`IFN_STORE_LANES` required
adding extending the expanders for `vec_load_lanes<mode><vstruct_elt>`
and `vec_store_lanes<mode><vstruct_elt>` to cover tuples of D-sized
vectors. This made the loop in `vect-early-break_39.c` fail to
vectorize.
gcc/ChangeLog:
* config/aarch64/aarch64-builtins.cc (TYPES_STORE1,
TYPES_STORESTRUCT, TYPES_STORE1_U, TYPES_STORESTRUCT_U,
TYPES_STORE1_P, TYPES_STORESTRUCT_P): Delete macros.
(aarch64_expand_pragma_builtin): Delete cases handling
`UNSPEC_LD{2,3,4}` and `UNSPEC_ST{2,3,4}`.
* config/aarch64/aarch64-neon-builtins-base.cc
(build_tuple_get_array): New function.
(build_tuple_get): Call `build_tuple_get_array`.
(struct gimple_load_store_interleaved): New struct.
* config/aarch64/aarch64-neon-builtins-base.def (vld2, vld2q,
vld3, vld3q, vld4, vld4q, vst2, vst2q, vst3, vst3q, vst4,
vst4q): Declare new intrinsics.
* config/aarch64/aarch64-simd-builtins.def: Delete builtin
function declarations.
* config/aarch64/aarch64-simd-pragma-builtins.def: Delete
intrinsic function declarations.
* config/aarch64/aarch64-simd.md
(vec_load_lanes<mode><vstruct_elt>,
@aarch64_ld<nregs><vstruct_elt>, aarch64_simd_ld2<vstruct_elt>,
aarch64_simd_ld3<vstruct_elt>, aarch64_simd_ld4<vstruct_elt>,
aarch64_ld2<vstruct_elt>_dreg, aarch64_ld3<vstruct_elt>_dreg,
aarch64_ld4<vstruct_elt>_dreg): Replace with...
(vec_load_lanes<mode><vstruct_elt>, @aarch64_load_interleaved<mode>):
...these.
(vec_store_lanes<mode><vstruct_elt>,
@aarch64_st<nregs><vstruct_elt>, aarch64_simd_st2<vstruct_elt>,
aarch64_simd_st3<vstruct_elt>, aarch64_simd_st4<vstruct_elt>,
aarch64_st2<vstruct_elt>_dreg, aarch64_st3<vstruct_elt>_dreg,
aarch64_st4<vstruct_elt>_dreg): Replace with...
(vec_store_lanes<mode><vstruct_elt>, @aarch64_store_interleaved<mode>):
...these.
* config/aarch64/aarch64.md
(UNSPEC_LD2, UNSPEC_LD2_DREG, UNSPEC_LD3, UNSPEC_LD3_DREG,
UNSPEC_LD4, UNSPEC_LD4_DREG): Replace with ...
(UNSPEC_LOAD_INTERLEAVED): ...this.
(UNSPEC_ST2, UNSPEC_ST2_DREG, UNSPEC_ST3, UNSPEC_ST3_DREG,
UNSPEC_ST4, UNSPEC_ST4_DREG): Replace with ...
(UNSPEC_STORE_INTERLEAVED): ...this.
* config/aarch64/iterators.md (VSTRUCT_DNX): New iterator.
gcc/testsuite/ChangeLog:
* gcc.dg/vect/vect-early-break_39.c: Mark as `xfail`.
* gcc.target/aarch64/pr99873_1.c: Update expected assembly (GCC
can now vectorize the loop tail as well as the loop body).
* gcc.target/aarch64/vld2-1.c: Update expected number of fwprop
iterations.
* gcc.target/aarch64/neon/vld2.c: New test.
* gcc.target/aarch64/neon/vld3.c: New test.
* gcc.target/aarch64/neon/vld4.c: New test.
* gcc.target/aarch64/neon/vst2.c: New test.
* gcc.target/aarch64/neon/vst3.c: New test.
* gcc.target/aarch64/neon/vst4.c: New test.
---
gcc/config/aarch64/aarch64-builtins.cc | 37 +-
.../aarch64/aarch64-neon-builtins-base.cc | 95 +++-
.../aarch64/aarch64-neon-builtins-base.def | 16 +
gcc/config/aarch64/aarch64-simd-builtins.def | 22 -
.../aarch64/aarch64-simd-pragma-builtins.def | 12 -
gcc/config/aarch64/aarch64-simd.md | 411 ++++--------------
gcc/config/aarch64/aarch64.md | 11 +-
gcc/config/aarch64/iterators.md | 4 +
.../gcc.dg/vect/vect-early-break_39.c | 7 +-
gcc/testsuite/gcc.target/aarch64/neon/vld2.c | 228 ++++++++++
gcc/testsuite/gcc.target/aarch64/neon/vld3.c | 228 ++++++++++
gcc/testsuite/gcc.target/aarch64/neon/vld4.c | 228 ++++++++++
gcc/testsuite/gcc.target/aarch64/neon/vst2.c | 228 ++++++++++
gcc/testsuite/gcc.target/aarch64/neon/vst3.c | 228 ++++++++++
gcc/testsuite/gcc.target/aarch64/neon/vst4.c | 228 ++++++++++
gcc/testsuite/gcc.target/aarch64/pr99873_1.c | 9 +-
gcc/testsuite/gcc.target/aarch64/vld2-1.c | 2 +-
17 files changed, 1564 insertions(+), 430 deletions(-)
create mode 100644 gcc/testsuite/gcc.target/aarch64/neon/vld2.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/neon/vld3.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/neon/vld4.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/neon/vst2.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/neon/vst3.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/neon/vst4.c
diff --git a/gcc/config/aarch64/aarch64-builtins.cc
b/gcc/config/aarch64/aarch64-builtins.cc
index 75d6cea7b13..cbb68d7b84f 100644
--- a/gcc/config/aarch64/aarch64-builtins.cc
+++ b/gcc/config/aarch64/aarch64-builtins.cc
@@ -377,27 +377,6 @@
aarch64_types_loadstruct_lane_p_qualifiers[SIMD_MAX_BUILTIN_ARGS]
qualifier_poly, qualifier_struct_load_store_lane_index };
#define TYPES_LOADSTRUCT_LANE_P (aarch64_types_loadstruct_lane_p_qualifiers)
-/* The first argument (return type) of a store should be void type,
- which we represent with qualifier_void. Their first operand will be
- a DImode pointer to the location to store to, so we must use
- qualifier_map_mode | qualifier_pointer to build a pointer to the
- element type of the vector. */
-static enum aarch64_type_qualifiers
-aarch64_types_store1_qualifiers[SIMD_MAX_BUILTIN_ARGS]
- = { qualifier_void, qualifier_pointer_map_mode, qualifier_none };
-#define TYPES_STORE1 (aarch64_types_store1_qualifiers)
-#define TYPES_STORESTRUCT (aarch64_types_store1_qualifiers)
-static enum aarch64_type_qualifiers
-aarch64_types_store1_u_qualifiers[SIMD_MAX_BUILTIN_ARGS]
- = { qualifier_void, qualifier_pointer_map_mode, qualifier_unsigned };
-#define TYPES_STORE1_U (aarch64_types_store1_u_qualifiers)
-#define TYPES_STORESTRUCT_U (aarch64_types_store1_u_qualifiers)
-static enum aarch64_type_qualifiers
-aarch64_types_store1_p_qualifiers[SIMD_MAX_BUILTIN_ARGS]
- = { qualifier_void, qualifier_pointer_map_mode, qualifier_poly };
-#define TYPES_STORE1_P (aarch64_types_store1_p_qualifiers)
-#define TYPES_STORESTRUCT_P (aarch64_types_store1_p_qualifiers)
-
static enum aarch64_type_qualifiers
aarch64_types_storestruct_lane_qualifiers[SIMD_MAX_BUILTIN_ARGS]
= { qualifier_void, qualifier_pointer_map_mode,
@@ -4241,12 +4220,6 @@ aarch64_expand_pragma_builtin (tree exp, rtx target,
gcc_unreachable ();
break;
- case UNSPEC_LD2:
- case UNSPEC_LD3:
- case UNSPEC_LD4:
- icode = code_for_aarch64_ld (ops[0].mode, ops[0].mode);
- break;
-
case UNSPEC_LD2_DUP:
aarch64_dereference_pointer (&ops[1], GET_MODE_INNER (ops[0].mode), 2);
icode = code_for_aarch64_simd_ld2r (ops[0].mode);
@@ -4293,12 +4266,6 @@ aarch64_expand_pragma_builtin (tree exp, rtx target,
icode = code_for_aarch64_get_lane (ops[1].mode);
break;
- case UNSPEC_ST2:
- case UNSPEC_ST3:
- case UNSPEC_ST4:
- icode = code_for_aarch64_st (ops[1].mode, ops[1].mode);
- break;
-
case UNSPEC_ST2_LANE:
case UNSPEC_ST3_LANE:
case UNSPEC_ST4_LANE:
@@ -4752,7 +4719,9 @@ aarch64_fold_to_val (gcall *stmt, gimple_stmt_iterator
*gsi,
tree
aarch64_dereference (tree addr, tree type)
{
- tree elt_type = (VECTOR_TYPE_P (type) ? TREE_TYPE (type) : type);
+ tree elt_type = VECTOR_TYPE_P (type) || TREE_CODE (type) == ARRAY_TYPE
+ ? TREE_TYPE (type)
+ : type;
tree elt_ptr_type = build_pointer_type_for_mode (elt_type, VOIDmode, true);
tree zero = build_zero_cst (elt_ptr_type);
/* Use element type alignment. */
diff --git a/gcc/config/aarch64/aarch64-neon-builtins-base.cc
b/gcc/config/aarch64/aarch64-neon-builtins-base.cc
index 19786d6de9a..a70530d3c78 100644
--- a/gcc/config/aarch64/aarch64-neon-builtins-base.cc
+++ b/gcc/config/aarch64/aarch64-neon-builtins-base.cc
@@ -144,21 +144,25 @@ build_shift_right (tree type, tree val, tree amount)
build_int_cst (TREE_TYPE (amount), bits - 1));
}
-/* Build a `TUPLE.val[INDEX]` expression. */
+/* Build `TUPLE.val`. */
tree
-build_tuple_get (tree tuple, tree index)
+build_tuple_get_array (tree tuple)
{
auto tuple_type = TREE_TYPE (tuple);
auto field = tuple_type_field (tuple_type);
auto array_type = TREE_TYPE (field);
- auto vec_type = TREE_TYPE (array_type);
-
- auto field_ref = fold_build3 (COMPONENT_REF, array_type, unshare_expr
(tuple),
- field, NULL_TREE);
- auto array_ref
- = build4 (ARRAY_REF, vec_type, field_ref, index, NULL_TREE, NULL_TREE);
+ return fold_build3 (COMPONENT_REF, array_type, unshare_expr (tuple), field,
+ NULL_TREE);
+}
- return array_ref;
+/* Build a `TUPLE.val[INDEX]` expression. */
+tree
+build_tuple_get (tree tuple, tree index)
+{
+ auto field_ref = build_tuple_get_array (tuple);
+ auto array_type = TREE_TYPE (field_ref);
+ auto vec_type = TREE_TYPE (array_type);
+ return build4 (ARRAY_REF, vec_type, field_ref, index, NULL_TREE, NULL_TREE);
}
/* Build a `TUPLE.val[INDEX][LANE]` expression. */
@@ -1603,6 +1607,75 @@ public:
}
};
+template <bool load_p>
+struct gimple_load_store_interleaved : public gimple_function_base
+{
+ unsigned int call_properties (const function_instance &) const override
+ {
+ return load_p ? CP_READ_MEMORY : CP_WRITE_MEMORY;
+ }
+
+ rtx expand (function_expander &e) const override
+ {
+ auto tuple_mode = load_p ? e.result_mode () : e.args[1]->mode;
+ insn_code icode;
+
+ auto &ptr_arg = e.args[0];
+ ptr_arg = gen_rtx_MEM (tuple_mode, e.convert_to_pmode (ptr_arg));
+
+ switch (tuple_mode)
+ {
+ case V2x1DImode: case V2x1DFmode:
+ case V3x1DImode: case V3x1DFmode:
+ case V4x1DImode: case V4x1DFmode:
+ icode = load_p ? code_for_aarch64_simd_load_multiple (tuple_mode)
+ : code_for_aarch64_simd_store_multiple (tuple_mode);
+ break;
+ default:
+ icode = load_p ? code_for_aarch64_load_interleaved (tuple_mode)
+ : code_for_aarch64_store_interleaved (tuple_mode);
+ break;
+ }
+
+ gcc_assert (icode != CODE_FOR_nothing);
+ return e.use_exact_insn (icode);
+ }
+
+ gimple *fold (gimple_folder &f) const override
+ {
+ auto ptr = gimple_call_arg (f.call, 0);
+ auto tuple_type = load_p ? TREE_TYPE (f.lhs) : TREE_TYPE (gimple_call_arg
(f.call, 1));
+ auto array_type = TREE_TYPE (tuple_type_field (tuple_type));
+ auto vec_type = TREE_TYPE (array_type);
+
+ if (BYTES_BIG_ENDIAN)
+ return nullptr;
+
+ if (one_element_vector_type_p (vec_type))
+ return nullptr;
+
+ auto deref = aarch64_dereference (ptr, array_type);
+ if (load_p)
+ {
+ /* lhs = VIEW_CONVERT (.LOAD_LANES (*ptr)) */
+ auto array = build_var (array_type);
+ auto call = gimple_build_call_internal (IFN_LOAD_LANES, 1, deref);
+ gimple_seq_add_stmt_without_update (&f.stmts,call);
+ gimple_call_set_lhs (call, array);
+ return gimple_build_assign (f.lhs, build_cast (tuple_type, array));
+ }
+ else
+ {
+ /* *ptr = .STORE_LANES (tuple.val) */
+ auto tuple = gimple_call_arg (f.call, 1);
+ auto array = build_tuple_get_array (tuple);
+ auto call = gimple_build_call_internal (IFN_STORE_LANES, 1, array);
+ gimple_call_set_lhs (call, deref);
+ return call;
+ }
+ }
+};
+
// Reinterpret
NEON_FUNCTION (vreinterpret, vreinterpretq, gimple_reinterpret,)
@@ -1824,6 +1897,10 @@ NEON_FUNCTION (vluti4q_lane, vluti4q_laneq,
gimple_luti<4>,)
NEON_FUNCTION (vld1, vld1q, vldrq, gimple_load_store_multiple<true>,)
NEON_FUNCTION (vst1, vst1q, vstrq, gimple_load_store_multiple<false>,)
+// Load/Store interleaved tuples
+NEON_FUNCTION (vld2, vld2q, vld3, vld3q, vld4, vld4q,
gimple_load_store_interleaved<true>,)
+NEON_FUNCTION (vst2, vst2q, vst3, vst3q, vst4, vst4q,
gimple_load_store_interleaved<false>,)
+
// AES
NEON_FUNCTION (vaeseq, gimple_exact_insn<CODE_FOR_aarch64_crypto_aesev16qi>,)
NEON_FUNCTION (vaesdq, gimple_exact_insn<CODE_FOR_aarch64_crypto_aesdv16qi>,)
diff --git a/gcc/config/aarch64/aarch64-neon-builtins-base.def
b/gcc/config/aarch64/aarch64-neon-builtins-base.def
index 73e86b978d6..db757866906 100644
--- a/gcc/config/aarch64/aarch64-neon-builtins-base.def
+++ b/gcc/config/aarch64/aarch64-neon-builtins-base.def
@@ -610,6 +610,14 @@ DEF_NEON_FUNCTION_GS (vluti4q_laneq, neon_luti2_h, x2,
("Q0,Q0x2,Qu8,ss32", rang
DEF_NEON_FUNCTION_GS (vld1, all_neon, x1234, ("D0xt,al"))
DEF_NEON_FUNCTION_GS (vld1q, all_neon, x1234, ("Q0xt,al"))
DEF_NEON_FUNCTION (vldrq, p128, ("s0,al"))
+
+// Load interleaved tuples
+DEF_NEON_FUNCTION (vld2, all_neon, ("D0x2,al"))
+DEF_NEON_FUNCTION (vld2q, all_neon, ("Q0x2,al"))
+DEF_NEON_FUNCTION (vld3, all_neon, ("D0x3,al"))
+DEF_NEON_FUNCTION (vld3q, all_neon, ("Q0x3,al"))
+DEF_NEON_FUNCTION (vld4, all_neon, ("D0x4,al"))
+DEF_NEON_FUNCTION (vld4q, all_neon, ("Q0x4,al"))
#undef REQUIRED_EXTENSIONS
// Stores
@@ -618,6 +626,14 @@ DEF_NEON_FUNCTION (vldrq, p128, ("s0,al"))
DEF_NEON_FUNCTION_GS (vst1, all_neon, x1234, ("_,as,D0xt"))
DEF_NEON_FUNCTION_GS (vst1q, all_neon, x1234, ("_,as,Q0xt"))
DEF_NEON_FUNCTION (vstrq, p128, ("_,as,s0"))
+
+// Store interleaved tuples
+DEF_NEON_FUNCTION (vst2, all_neon, ("_,as,D0x2"))
+DEF_NEON_FUNCTION (vst2q, all_neon, ("_,as,Q0x2"))
+DEF_NEON_FUNCTION (vst3, all_neon, ("_,as,D0x3"))
+DEF_NEON_FUNCTION (vst3q, all_neon, ("_,as,Q0x3"))
+DEF_NEON_FUNCTION (vst4, all_neon, ("_,as,D0x4"))
+DEF_NEON_FUNCTION (vst4q, all_neon, ("_,as,Q0x4"))
#undef REQUIRED_EXTENSIONS
// AES
diff --git a/gcc/config/aarch64/aarch64-simd-builtins.def
b/gcc/config/aarch64/aarch64-simd-builtins.def
index 9c772321bbd..c75d3d48a90 100644
--- a/gcc/config/aarch64/aarch64-simd-builtins.def
+++ b/gcc/config/aarch64/aarch64-simd-builtins.def
@@ -67,28 +67,6 @@
BUILTIN_VSDQ_I (BINOP_SSU, suqadd, 0, DEFAULT)
BUILTIN_VSDQ_I (BINOP_UUS, usqadd, 0, DEFAULT)
- /* Implemented by aarch64_ld<nregs><vstruct_elt>. */
- BUILTIN_VALLDIF (LOADSTRUCT, ld2, 0, LOAD)
- BUILTIN_VSDQ_I_DI (LOADSTRUCT_U, ld2, 0, LOAD)
- BUILTIN_VALLP (LOADSTRUCT_P, ld2, 0, LOAD)
- BUILTIN_VALLDIF (LOADSTRUCT, ld3, 0, LOAD)
- BUILTIN_VSDQ_I_DI (LOADSTRUCT_U, ld3, 0, LOAD)
- BUILTIN_VALLP (LOADSTRUCT_P, ld3, 0, LOAD)
- BUILTIN_VALLDIF (LOADSTRUCT, ld4, 0, LOAD)
- BUILTIN_VSDQ_I_DI (LOADSTRUCT_U, ld4, 0, LOAD)
- BUILTIN_VALLP (LOADSTRUCT_P, ld4, 0, LOAD)
-
- /* Implemented by aarch64_st<nregs><vstruct_elt>. */
- BUILTIN_VALLDIF (STORESTRUCT, st2, 0, STORE)
- BUILTIN_VSDQ_I_DI (STORESTRUCT_U, st2, 0, STORE)
- BUILTIN_VALLP (STORESTRUCT_P, st2, 0, STORE)
- BUILTIN_VALLDIF (STORESTRUCT, st3, 0, STORE)
- BUILTIN_VSDQ_I_DI (STORESTRUCT_U, st3, 0, STORE)
- BUILTIN_VALLP (STORESTRUCT_P, st3, 0, STORE)
- BUILTIN_VALLDIF (STORESTRUCT, st4, 0, STORE)
- BUILTIN_VSDQ_I_DI (STORESTRUCT_U, st4, 0, STORE)
- BUILTIN_VALLP (STORESTRUCT_P, st4, 0, STORE)
-
/* Implemented by aarch64_ld<nregs>r<vstruct_elt>. */
BUILTIN_VALLDIF (LOADSTRUCT, ld2r, 0, LOAD)
BUILTIN_VSDQ_I_DI (LOADSTRUCT_U, ld2r, 0, LOAD)
diff --git a/gcc/config/aarch64/aarch64-simd-pragma-builtins.def
b/gcc/config/aarch64/aarch64-simd-pragma-builtins.def
index e5b807613bd..abd2f80862c 100644
--- a/gcc/config/aarch64/aarch64-simd-pragma-builtins.def
+++ b/gcc/config/aarch64/aarch64-simd-pragma-builtins.def
@@ -165,8 +165,6 @@ ENTRY_LOAD_LANE (vld1q_lane_mf8, mf8q,
mf8_scalar_const_ptr, mf8q,
// ld<n>
#define REQUIRED_EXTENSIONS nonstreaming_only (AARCH64_FL_SIMD)
-ENTRY_LOAD (vld2_mf8, mf8x2, mf8_scalar_const_ptr, UNSPEC_LD2)
-ENTRY_LOAD (vld2q_mf8, mf8qx2, mf8_scalar_const_ptr, UNSPEC_LD2)
ENTRY_LOAD (vld2_dup_mf8, mf8x2, mf8_scalar_const_ptr, UNSPEC_LD2_DUP)
ENTRY_LOAD (vld2q_dup_mf8, mf8qx2, mf8_scalar_const_ptr, UNSPEC_LD2_DUP)
ENTRY_LOAD_LANE (vld2_lane_mf8, mf8x2, mf8_scalar_const_ptr, mf8x2,
@@ -174,8 +172,6 @@ ENTRY_LOAD_LANE (vld2_lane_mf8, mf8x2,
mf8_scalar_const_ptr, mf8x2,
ENTRY_LOAD_LANE (vld2q_lane_mf8, mf8qx2, mf8_scalar_const_ptr, mf8qx2,
UNSPEC_LD2_LANE)
-ENTRY_LOAD (vld3_mf8, mf8x3, mf8_scalar_const_ptr, UNSPEC_LD3)
-ENTRY_LOAD (vld3q_mf8, mf8qx3, mf8_scalar_const_ptr, UNSPEC_LD3)
ENTRY_LOAD (vld3_dup_mf8, mf8x3, mf8_scalar_const_ptr, UNSPEC_LD3_DUP)
ENTRY_LOAD (vld3q_dup_mf8, mf8qx3, mf8_scalar_const_ptr, UNSPEC_LD3_DUP)
ENTRY_LOAD_LANE (vld3_lane_mf8, mf8x3, mf8_scalar_const_ptr, mf8x3,
@@ -183,8 +179,6 @@ ENTRY_LOAD_LANE (vld3_lane_mf8, mf8x3,
mf8_scalar_const_ptr, mf8x3,
ENTRY_LOAD_LANE (vld3q_lane_mf8, mf8qx3, mf8_scalar_const_ptr, mf8qx3,
UNSPEC_LD3_LANE)
-ENTRY_LOAD (vld4_mf8, mf8x4, mf8_scalar_const_ptr, UNSPEC_LD4)
-ENTRY_LOAD (vld4q_mf8, mf8qx4, mf8_scalar_const_ptr, UNSPEC_LD4)
ENTRY_LOAD (vld4_dup_mf8, mf8x4, mf8_scalar_const_ptr, UNSPEC_LD4_DUP)
ENTRY_LOAD (vld4q_dup_mf8, mf8qx4, mf8_scalar_const_ptr, UNSPEC_LD4_DUP)
ENTRY_LOAD_LANE (vld4_lane_mf8, mf8x4, mf8_scalar_const_ptr, mf8x4,
@@ -211,18 +205,12 @@ ENTRY_STORE_LANE (vst1q_lane_mf8, mf8_scalar_ptr, mf8q,
UNSPEC_ST1_LANE)
// st<n>
#define REQUIRED_EXTENSIONS nonstreaming_only (AARCH64_FL_SIMD)
-ENTRY_STORE (vst2_mf8, mf8_scalar_ptr, mf8x2, UNSPEC_ST2)
-ENTRY_STORE (vst2q_mf8, mf8_scalar_ptr, mf8qx2, UNSPEC_ST2)
ENTRY_STORE_LANE (vst2_lane_mf8, mf8_scalar_ptr, mf8x2, UNSPEC_ST2_LANE)
ENTRY_STORE_LANE (vst2q_lane_mf8, mf8_scalar_ptr, mf8qx2, UNSPEC_ST2_LANE)
-ENTRY_STORE (vst3_mf8, mf8_scalar_ptr, mf8x3, UNSPEC_ST3)
-ENTRY_STORE (vst3q_mf8, mf8_scalar_ptr, mf8qx3, UNSPEC_ST3)
ENTRY_STORE_LANE (vst3_lane_mf8, mf8_scalar_ptr, mf8x3, UNSPEC_ST3_LANE)
ENTRY_STORE_LANE (vst3q_lane_mf8, mf8_scalar_ptr, mf8qx3, UNSPEC_ST3_LANE)
-ENTRY_STORE (vst4_mf8, mf8_scalar_ptr, mf8x4, UNSPEC_ST4)
-ENTRY_STORE (vst4q_mf8, mf8_scalar_ptr, mf8qx4, UNSPEC_ST4)
ENTRY_STORE_LANE (vst4_lane_mf8, mf8_scalar_ptr, mf8x4, UNSPEC_ST4_LANE)
ENTRY_STORE_LANE (vst4q_lane_mf8, mf8_scalar_ptr, mf8qx4, UNSPEC_ST4_LANE)
#undef REQUIRED_EXTENSIONS
diff --git a/gcc/config/aarch64/aarch64-simd.md
b/gcc/config/aarch64/aarch64-simd.md
index 84157be320f..8e80f5d3fa0 100644
--- a/gcc/config/aarch64/aarch64-simd.md
+++ b/gcc/config/aarch64/aarch64-simd.md
@@ -8407,16 +8407,6 @@ (define_insn "*sqrt<mode>2<vczle><vczbe>"
;; Patterns for vector struct loads and stores.
-(define_insn "aarch64_simd_ld2<vstruct_elt>"
- [(set (match_operand:VSTRUCT_2Q 0 "register_operand" "=w")
- (unspec:VSTRUCT_2Q [
- (match_operand:VSTRUCT_2Q 1 "aarch64_simd_struct_operand" "Utv")]
- UNSPEC_LD2))]
- "TARGET_SIMD"
- "ld2\\t{%S0.<Vtype> - %T0.<Vtype>}, %1"
- [(set_attr "type" "neon_load2_2reg<q>")]
-)
-
(define_insn "@aarch64_simd_ld2r<vstruct_elt>"
[(set (match_operand:VSTRUCT_2QD 0 "register_operand" "=w")
(unspec:VSTRUCT_2QD [
@@ -8443,36 +8433,6 @@ (define_insn
"@aarch64_vec_load_lanes<mode>_lane<vstruct_elt>"
[(set_attr "type" "neon_load2_one_lane")]
)
-(define_expand "vec_load_lanes<mode><vstruct_elt>"
- [(set (match_operand:VSTRUCT_2Q 0 "register_operand")
- (unspec:VSTRUCT_2Q [
- (match_operand:VSTRUCT_2Q 1 "aarch64_simd_struct_operand")]
- UNSPEC_LD2))]
- "TARGET_SIMD"
-{
- if (BYTES_BIG_ENDIAN)
- {
- rtx tmp = gen_reg_rtx (<MODE>mode);
- rtx mask = aarch64_reverse_mask (<VSTRUCT_ELT>mode,
- GET_MODE_NUNITS (<MODE>mode).to_constant () / <nregs>);
- emit_insn (gen_aarch64_simd_ld2<vstruct_elt> (tmp, operands[1]));
- emit_insn (gen_aarch64_rev_reglist<mode> (operands[0], tmp, mask));
- }
- else
- emit_insn (gen_aarch64_simd_ld2<vstruct_elt> (operands[0], operands[1]));
- DONE;
-})
-
-(define_insn "aarch64_simd_st2<vstruct_elt>"
- [(set (match_operand:VSTRUCT_2Q 0 "aarch64_simd_struct_operand" "=Utv")
- (unspec:VSTRUCT_2Q [
- (match_operand:VSTRUCT_2Q 1 "register_operand" "w")]
- UNSPEC_ST2))]
- "TARGET_SIMD"
- "st2\\t{%S1.<Vtype> - %T1.<Vtype>}, %0"
- [(set_attr "type" "neon_store2_2reg<q>")]
-)
-
;; RTL uses GCC vector extension indices, so flip only for assembly.
(define_insn "aarch64_vec_store_lanes<mode>_lane<vstruct_elt>"
[(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
@@ -8488,35 +8448,6 @@ (define_insn
"aarch64_vec_store_lanes<mode>_lane<vstruct_elt>"
[(set_attr "type" "neon_store2_one_lane<q>")]
)
-(define_expand "vec_store_lanes<mode><vstruct_elt>"
- [(set (match_operand:VSTRUCT_2Q 0 "aarch64_simd_struct_operand")
- (unspec:VSTRUCT_2Q [(match_operand:VSTRUCT_2Q 1 "register_operand")]
- UNSPEC_ST2))]
- "TARGET_SIMD"
-{
- if (BYTES_BIG_ENDIAN)
- {
- rtx tmp = gen_reg_rtx (<MODE>mode);
- rtx mask = aarch64_reverse_mask (<VSTRUCT_ELT>mode,
- GET_MODE_NUNITS (<MODE>mode).to_constant () / <nregs>);
- emit_insn (gen_aarch64_rev_reglist<mode> (tmp, operands[1], mask));
- emit_insn (gen_aarch64_simd_st2<vstruct_elt> (operands[0], tmp));
- }
- else
- emit_insn (gen_aarch64_simd_st2<vstruct_elt> (operands[0], operands[1]));
- DONE;
-})
-
-(define_insn "aarch64_simd_ld3<vstruct_elt>"
- [(set (match_operand:VSTRUCT_3Q 0 "register_operand" "=w")
- (unspec:VSTRUCT_3Q [
- (match_operand:VSTRUCT_3Q 1 "aarch64_simd_struct_operand" "Utv")]
- UNSPEC_LD3))]
- "TARGET_SIMD"
- "ld3\\t{%S0.<Vtype> - %U0.<Vtype>}, %1"
- [(set_attr "type" "neon_load3_3reg<q>")]
-)
-
(define_insn "@aarch64_simd_ld3r<vstruct_elt>"
[(set (match_operand:VSTRUCT_3QD 0 "register_operand" "=w")
(unspec:VSTRUCT_3QD [
@@ -8543,35 +8474,6 @@ (define_insn
"aarch64_vec_load_lanes<mode>_lane<vstruct_elt>"
[(set_attr "type" "neon_load3_one_lane")]
)
-(define_expand "vec_load_lanes<mode><vstruct_elt>"
- [(set (match_operand:VSTRUCT_3Q 0 "register_operand")
- (unspec:VSTRUCT_3Q [
- (match_operand:VSTRUCT_3Q 1 "aarch64_simd_struct_operand")]
- UNSPEC_LD3))]
- "TARGET_SIMD"
-{
- if (BYTES_BIG_ENDIAN)
- {
- rtx tmp = gen_reg_rtx (<MODE>mode);
- rtx mask = aarch64_reverse_mask (<VSTRUCT_ELT>mode,
- GET_MODE_NUNITS (<MODE>mode).to_constant () / <nregs>);
- emit_insn (gen_aarch64_simd_ld3<vstruct_elt> (tmp, operands[1]));
- emit_insn (gen_aarch64_rev_reglist<mode> (operands[0], tmp, mask));
- }
- else
- emit_insn (gen_aarch64_simd_ld3<vstruct_elt> (operands[0], operands[1]));
- DONE;
-})
-
-(define_insn "aarch64_simd_st3<vstruct_elt>"
- [(set (match_operand:VSTRUCT_3Q 0 "aarch64_simd_struct_operand" "=Utv")
- (unspec:VSTRUCT_3Q [(match_operand:VSTRUCT_3Q 1 "register_operand" "w")]
- UNSPEC_ST3))]
- "TARGET_SIMD"
- "st3\\t{%S1.<Vtype> - %U1.<Vtype>}, %0"
- [(set_attr "type" "neon_store3_3reg<q>")]
-)
-
;; RTL uses GCC vector extension indices, so flip only for assembly.
(define_insn "aarch64_vec_store_lanes<mode>_lane<vstruct_elt>"
[(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
@@ -8587,36 +8489,6 @@ (define_insn
"aarch64_vec_store_lanes<mode>_lane<vstruct_elt>"
[(set_attr "type" "neon_store3_one_lane<q>")]
)
-(define_expand "vec_store_lanes<mode><vstruct_elt>"
- [(set (match_operand:VSTRUCT_3Q 0 "aarch64_simd_struct_operand")
- (unspec:VSTRUCT_3Q [
- (match_operand:VSTRUCT_3Q 1 "register_operand")]
- UNSPEC_ST3))]
- "TARGET_SIMD"
-{
- if (BYTES_BIG_ENDIAN)
- {
- rtx tmp = gen_reg_rtx (<MODE>mode);
- rtx mask = aarch64_reverse_mask (<VSTRUCT_ELT>mode,
- GET_MODE_NUNITS (<MODE>mode).to_constant () / <nregs>);
- emit_insn (gen_aarch64_rev_reglist<mode> (tmp, operands[1], mask));
- emit_insn (gen_aarch64_simd_st3<vstruct_elt> (operands[0], tmp));
- }
- else
- emit_insn (gen_aarch64_simd_st3<vstruct_elt> (operands[0], operands[1]));
- DONE;
-})
-
-(define_insn "aarch64_simd_ld4<vstruct_elt>"
- [(set (match_operand:VSTRUCT_4Q 0 "register_operand" "=w")
- (unspec:VSTRUCT_4Q [
- (match_operand:VSTRUCT_4Q 1 "aarch64_simd_struct_operand" "Utv")]
- UNSPEC_LD4))]
- "TARGET_SIMD"
- "ld4\\t{%S0.<Vtype> - %V0.<Vtype>}, %1"
- [(set_attr "type" "neon_load4_4reg<q>")]
-)
-
(define_insn "@aarch64_simd_ld4r<vstruct_elt>"
[(set (match_operand:VSTRUCT_4QD 0 "register_operand" "=w")
(unspec:VSTRUCT_4QD [
@@ -8643,36 +8515,6 @@ (define_insn
"aarch64_vec_load_lanes<mode>_lane<vstruct_elt>"
[(set_attr "type" "neon_load4_one_lane")]
)
-(define_expand "vec_load_lanes<mode><vstruct_elt>"
- [(set (match_operand:VSTRUCT_4Q 0 "register_operand")
- (unspec:VSTRUCT_4Q [
- (match_operand:VSTRUCT_4Q 1 "aarch64_simd_struct_operand")]
- UNSPEC_LD4))]
- "TARGET_SIMD"
-{
- if (BYTES_BIG_ENDIAN)
- {
- rtx tmp = gen_reg_rtx (<MODE>mode);
- rtx mask = aarch64_reverse_mask (<VSTRUCT_ELT>mode,
- GET_MODE_NUNITS (<MODE>mode).to_constant () / <nregs>);
- emit_insn (gen_aarch64_simd_ld4<vstruct_elt> (tmp, operands[1]));
- emit_insn (gen_aarch64_rev_reglist<mode> (operands[0], tmp, mask));
- }
- else
- emit_insn (gen_aarch64_simd_ld4<vstruct_elt> (operands[0], operands[1]));
- DONE;
-})
-
-(define_insn "aarch64_simd_st4<vstruct_elt>"
- [(set (match_operand:VSTRUCT_4Q 0 "aarch64_simd_struct_operand" "=Utv")
- (unspec:VSTRUCT_4Q [
- (match_operand:VSTRUCT_4Q 1 "register_operand" "w")]
- UNSPEC_ST4))]
- "TARGET_SIMD"
- "st4\\t{%S1.<Vtype> - %V1.<Vtype>}, %0"
- [(set_attr "type" "neon_store4_4reg<q>")]
-)
-
;; RTL uses GCC vector extension indices, so flip only for assembly.
(define_insn "aarch64_vec_store_lanes<mode>_lane<vstruct_elt>"
[(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
@@ -8688,25 +8530,6 @@ (define_insn
"aarch64_vec_store_lanes<mode>_lane<vstruct_elt>"
[(set_attr "type" "neon_store4_one_lane<q>")]
)
-(define_expand "vec_store_lanes<mode><vstruct_elt>"
- [(set (match_operand:VSTRUCT_4Q 0 "aarch64_simd_struct_operand")
- (unspec:VSTRUCT_4Q [(match_operand:VSTRUCT_4Q 1 "register_operand")]
- UNSPEC_ST4))]
- "TARGET_SIMD"
-{
- if (BYTES_BIG_ENDIAN)
- {
- rtx tmp = gen_reg_rtx (<MODE>mode);
- rtx mask = aarch64_reverse_mask (<VSTRUCT_ELT>mode,
- GET_MODE_NUNITS (<MODE>mode).to_constant () / <nregs>);
- emit_insn (gen_aarch64_rev_reglist<mode> (tmp, operands[1], mask));
- emit_insn (gen_aarch64_simd_st4<vstruct_elt> (operands[0], tmp));
- }
- else
- emit_insn (gen_aarch64_simd_st4<vstruct_elt> (operands[0], operands[1]));
- DONE;
-})
-
;; Patterns for rcpc3 vector lane loads and stores.
(define_insn "aarch64_vec_stl1_lanes<mode>_lane<Vel>"
@@ -9145,86 +8968,6 @@ (define_expand "aarch64_ld<nregs>r<vstruct_elt>"
DONE;
})
-(define_insn "aarch64_ld2<vstruct_elt>_dreg"
- [(set (match_operand:VSTRUCT_2DNX 0 "register_operand" "=w")
- (unspec:VSTRUCT_2DNX [
- (match_operand:VSTRUCT_2DNX 1 "aarch64_simd_struct_operand" "Utv")]
- UNSPEC_LD2_DREG))]
- "TARGET_SIMD"
- "ld2\\t{%S0.<Vtype> - %T0.<Vtype>}, %1"
- [(set_attr "type" "neon_load2_2reg<q>")]
-)
-
-(define_insn "aarch64_ld2<vstruct_elt>_dreg"
- [(set (match_operand:VSTRUCT_2DX 0 "register_operand" "=w")
- (unspec:VSTRUCT_2DX [
- (match_operand:VSTRUCT_2DX 1 "aarch64_simd_struct_operand" "Utv")]
- UNSPEC_LD2_DREG))]
- "TARGET_SIMD"
- "ld1\\t{%S0.1d - %T0.1d}, %1"
- [(set_attr "type" "neon_load1_2reg<q>")]
-)
-
-(define_insn "aarch64_ld3<vstruct_elt>_dreg"
- [(set (match_operand:VSTRUCT_3DNX 0 "register_operand" "=w")
- (unspec:VSTRUCT_3DNX [
- (match_operand:VSTRUCT_3DNX 1 "aarch64_simd_struct_operand" "Utv")]
- UNSPEC_LD3_DREG))]
- "TARGET_SIMD"
- "ld3\\t{%S0.<Vtype> - %U0.<Vtype>}, %1"
- [(set_attr "type" "neon_load3_3reg<q>")]
-)
-
-(define_insn "aarch64_ld3<vstruct_elt>_dreg"
- [(set (match_operand:VSTRUCT_3DX 0 "register_operand" "=w")
- (unspec:VSTRUCT_3DX [
- (match_operand:VSTRUCT_3DX 1 "aarch64_simd_struct_operand" "Utv")]
- UNSPEC_LD3_DREG))]
- "TARGET_SIMD"
- "ld1\\t{%S0.1d - %U0.1d}, %1"
- [(set_attr "type" "neon_load1_3reg<q>")]
-)
-
-(define_insn "aarch64_ld4<vstruct_elt>_dreg"
- [(set (match_operand:VSTRUCT_4DNX 0 "register_operand" "=w")
- (unspec:VSTRUCT_4DNX [
- (match_operand:VSTRUCT_4DNX 1 "aarch64_simd_struct_operand" "Utv")]
- UNSPEC_LD4_DREG))]
- "TARGET_SIMD"
- "ld4\\t{%S0.<Vtype> - %V0.<Vtype>}, %1"
- [(set_attr "type" "neon_load4_4reg<q>")]
-)
-
-(define_insn "aarch64_ld4<vstruct_elt>_dreg"
- [(set (match_operand:VSTRUCT_4DX 0 "register_operand" "=w")
- (unspec:VSTRUCT_4DX [
- (match_operand:VSTRUCT_4DX 1 "aarch64_simd_struct_operand" "Utv")]
- UNSPEC_LD4_DREG))]
- "TARGET_SIMD"
- "ld1\\t{%S0.1d - %V0.1d}, %1"
- [(set_attr "type" "neon_load1_4reg<q>")]
-)
-
-(define_expand "@aarch64_ld<nregs><vstruct_elt>"
- [(match_operand:VSTRUCT_D 0 "register_operand")
- (match_operand:DI 1 "register_operand")]
- "TARGET_SIMD"
-{
- rtx mem = gen_rtx_MEM (<MODE>mode, operands[1]);
- emit_insn (gen_aarch64_ld<nregs><vstruct_elt>_dreg (operands[0], mem));
- DONE;
-})
-
-(define_expand "@aarch64_ld<nregs><vstruct_elt>"
- [(match_operand:VSTRUCT_Q 0 "register_operand")
- (match_operand:DI 1 "register_operand")]
- "TARGET_SIMD"
-{
- rtx mem = gen_rtx_MEM (<MODE>mode, operands[1]);
- emit_insn (gen_aarch64_simd_ld<nregs><vstruct_elt> (operands[0], mem));
- DONE;
-})
-
(define_expand "@aarch64_ld<nregs>_lane<vstruct_elt>"
[(match_operand:VSTRUCT_QD 0 "register_operand")
(match_operand:DI 1 "register_operand")
@@ -9437,86 +9180,6 @@ (define_insn
"@aarch64_rev<REVERSE:rev_op><mode><vczle><vczbe>"
[(set_attr "type" "neon_rev<q>")]
)
-(define_insn "aarch64_st2<vstruct_elt>_dreg"
- [(set (match_operand:VSTRUCT_2DNX 0 "aarch64_simd_struct_operand" "=Utv")
- (unspec:VSTRUCT_2DNX [
- (match_operand:VSTRUCT_2DNX 1 "register_operand" "w")]
- UNSPEC_ST2))]
- "TARGET_SIMD"
- "st2\\t{%S1.<Vtype> - %T1.<Vtype>}, %0"
- [(set_attr "type" "neon_store2_2reg")]
-)
-
-(define_insn "aarch64_st2<vstruct_elt>_dreg"
- [(set (match_operand:VSTRUCT_2DX 0 "aarch64_simd_struct_operand" "=Utv")
- (unspec:VSTRUCT_2DX [
- (match_operand:VSTRUCT_2DX 1 "register_operand" "w")]
- UNSPEC_ST2))]
- "TARGET_SIMD"
- "st1\\t{%S1.1d - %T1.1d}, %0"
- [(set_attr "type" "neon_store1_2reg")]
-)
-
-(define_insn "aarch64_st3<vstruct_elt>_dreg"
- [(set (match_operand:VSTRUCT_3DNX 0 "aarch64_simd_struct_operand" "=Utv")
- (unspec:VSTRUCT_3DNX [
- (match_operand:VSTRUCT_3DNX 1 "register_operand" "w")]
- UNSPEC_ST3))]
- "TARGET_SIMD"
- "st3\\t{%S1.<Vtype> - %U1.<Vtype>}, %0"
- [(set_attr "type" "neon_store3_3reg")]
-)
-
-(define_insn "aarch64_st3<vstruct_elt>_dreg"
- [(set (match_operand:VSTRUCT_3DX 0 "aarch64_simd_struct_operand" "=Utv")
- (unspec:VSTRUCT_3DX [
- (match_operand:VSTRUCT_3DX 1 "register_operand" "w")]
- UNSPEC_ST3))]
- "TARGET_SIMD"
- "st1\\t{%S1.1d - %U1.1d}, %0"
- [(set_attr "type" "neon_store1_3reg")]
-)
-
-(define_insn "aarch64_st4<vstruct_elt>_dreg"
- [(set (match_operand:VSTRUCT_4DNX 0 "aarch64_simd_struct_operand" "=Utv")
- (unspec:VSTRUCT_4DNX [
- (match_operand:VSTRUCT_4DNX 1 "register_operand" "w")]
- UNSPEC_ST4))]
- "TARGET_SIMD"
- "st4\\t{%S1.<Vtype> - %V1.<Vtype>}, %0"
- [(set_attr "type" "neon_store4_4reg")]
-)
-
-(define_insn "aarch64_st4<vstruct_elt>_dreg"
- [(set (match_operand:VSTRUCT_4DX 0 "aarch64_simd_struct_operand" "=Utv")
- (unspec:VSTRUCT_4DX [
- (match_operand:VSTRUCT_4DX 1 "register_operand" "w")]
- UNSPEC_ST4))]
- "TARGET_SIMD"
- "st1\\t{%S1.1d - %V1.1d}, %0"
- [(set_attr "type" "neon_store1_4reg")]
-)
-
-(define_expand "@aarch64_st<nregs><vstruct_elt>"
- [(match_operand:DI 0 "register_operand")
- (match_operand:VSTRUCT_D 1 "register_operand")]
- "TARGET_SIMD"
-{
- rtx mem = gen_rtx_MEM (<MODE>mode, operands[0]);
- emit_insn (gen_aarch64_st<nregs><vstruct_elt>_dreg (mem, operands[1]));
- DONE;
-})
-
-(define_expand "@aarch64_st<nregs><vstruct_elt>"
- [(match_operand:DI 0 "register_operand")
- (match_operand:VSTRUCT_Q 1 "register_operand")]
- "TARGET_SIMD"
-{
- rtx mem = gen_rtx_MEM (<MODE>mode, operands[0]);
- emit_insn (gen_aarch64_simd_st<nregs><vstruct_elt> (mem, operands[1]));
- DONE;
-})
-
(define_expand "@aarch64_st<nregs>_lane<vstruct_elt>"
[(match_operand:DI 0 "register_operand")
(match_operand:VSTRUCT_QD 1 "register_operand")
@@ -11114,3 +10777,77 @@ (define_insn "@aarch64_<insn><mode>"
[ w , 0 , w , w ] <insn>\t%0.<Vtype>, %2.16b, %3.16b
}
)
+
+(define_expand "vec_load_lanes<mode><vstruct_elt>"
+ [(set (match_operand:VSTRUCT_INTERLEAVED 0 "register_operand")
+ (unspec:VSTRUCT_INTERLEAVED [
+ (match_operand:VSTRUCT_INTERLEAVED 1
"aarch64_simd_struct_operand")]
+ UNSPEC_LOAD_INTERLEAVED))]
+ "TARGET_SIMD"
+{
+ if (BYTES_BIG_ENDIAN)
+ {
+ rtx tmp = gen_reg_rtx (<MODE>mode);
+ rtx mask = aarch64_reverse_mask (<VSTRUCT_ELT>mode,
+ GET_MODE_NUNITS (<MODE>mode).to_constant () / <nregs>);
+ emit_insn (gen_aarch64_load_interleaved<mode> (tmp, operands[1]));
+ emit_insn (gen_aarch64_rev_reglist<mode> (operands[0], tmp, mask));
+ }
+ else
+ emit_insn (gen_aarch64_load_interleaved<mode> (operands[0], operands[1]));
+ DONE;
+})
+(define_insn "@aarch64_load_interleaved<mode>"
+ [(set (match_operand:VSTRUCT_QDNX 0 "register_operand" "=w")
+ (unspec:VSTRUCT_QDNX [
+ (match_operand:VSTRUCT_QDNX 1 "aarch64_simd_struct_operand" "Utv")]
+ UNSPEC_LOAD_INTERLEAVED))]
+ "TARGET_SIMD"
+ {
+ switch (<nregs>)
+ {
+ case 2: return "ld2\t{%S0.<Vtype> - %T0.<Vtype>}, %1";
+ case 3: return "ld3\t{%S0.<Vtype> - %U0.<Vtype>}, %1";
+ case 4: return "ld4\t{%S0.<Vtype> - %V0.<Vtype>}, %1";
+ default: gcc_unreachable();
+ }
+ }
+ [(set_attr "type" "neon_load<nregs>_<nregs>reg<q>")]
+)
+
+(define_expand "vec_store_lanes<mode><vstruct_elt>"
+ [(set (match_operand:VSTRUCT_INTERLEAVED 0 "aarch64_simd_struct_operand")
+ (unspec:VSTRUCT_INTERLEAVED [
+ (match_operand:VSTRUCT_INTERLEAVED 1 "register_operand")]
+ UNSPEC_STORE_INTERLEAVED))]
+ "TARGET_SIMD"
+{
+ if (BYTES_BIG_ENDIAN)
+ {
+ rtx tmp = gen_reg_rtx (<MODE>mode);
+ rtx mask = aarch64_reverse_mask (<VSTRUCT_ELT>mode,
+ GET_MODE_NUNITS (<MODE>mode).to_constant () / <nregs>);
+ emit_insn (gen_aarch64_rev_reglist<mode> (tmp, operands[1], mask));
+ emit_insn (gen_aarch64_store_interleaved<mode> (operands[0], tmp));
+ }
+ else
+ emit_insn (gen_aarch64_store_interleaved<mode> (operands[0], operands[1]));
+ DONE;
+})
+(define_insn "@aarch64_store_interleaved<mode>"
+ [(set (match_operand:VSTRUCT_QDNX 0 "aarch64_simd_struct_operand" "=Utv")
+ (unspec:VSTRUCT_QDNX [
+ (match_operand:VSTRUCT_QDNX 1 "register_operand" "w")]
+ UNSPEC_STORE_INTERLEAVED))]
+ "TARGET_SIMD"
+ {
+ switch (<nregs>)
+ {
+ case 2: return "st2\t{%S1.<Vtype> - %T1.<Vtype>}, %0";
+ case 3: return "st3\t{%S1.<Vtype> - %U1.<Vtype>}, %0";
+ case 4: return "st4\t{%S1.<Vtype> - %V1.<Vtype>}, %0";
+ default: gcc_unreachable();
+ }
+ }
+ [(set_attr "type" "neon_store<nregs>_<nregs>reg<q>")]
+)
diff --git a/gcc/config/aarch64/aarch64.md b/gcc/config/aarch64/aarch64.md
index 2b3efe6a022..8541ef088cc 100644
--- a/gcc/config/aarch64/aarch64.md
+++ b/gcc/config/aarch64/aarch64.md
@@ -267,15 +267,10 @@ (define_c_enum "unspec" [
UNSPEC_LDP_FST
UNSPEC_LDP_SND
UNSPEC_LOAD_MULTIPLE
+ UNSPEC_LOAD_INTERLEAVED
UNSPEC_LD1_DUP
- UNSPEC_LD2
- UNSPEC_LD2_DREG
UNSPEC_LD2_DUP
- UNSPEC_LD3
- UNSPEC_LD3_DREG
UNSPEC_LD3_DUP
- UNSPEC_LD4
- UNSPEC_LD4_DREG
UNSPEC_LD4_DUP
UNSPEC_LD2_LANE
UNSPEC_LD3_LANE
@@ -305,10 +300,8 @@ (define_c_enum "unspec" [
UNSPEC_SISD_USHL
UNSPEC_SSHL_2S
UNSPEC_STORE_MULTIPLE
+ UNSPEC_STORE_INTERLEAVED
UNSPEC_ST1_LANE
- UNSPEC_ST2
- UNSPEC_ST3
- UNSPEC_ST4
UNSPEC_ST2_LANE
UNSPEC_ST3_LANE
UNSPEC_ST4_LANE
diff --git a/gcc/config/aarch64/iterators.md b/gcc/config/aarch64/iterators.md
index d995299c68e..69687125a92 100644
--- a/gcc/config/aarch64/iterators.md
+++ b/gcc/config/aarch64/iterators.md
@@ -456,6 +456,8 @@ (define_mode_iterator VSTRUCT_3DNX [V3x8QI V3x4HI V3x2SI
V3x4HF
(define_mode_iterator VSTRUCT_4DNX [V4x8QI V4x4HI V4x2SI V4x4HF
V4x2SF V4x4BF])
+(define_mode_iterator VSTRUCT_DNX [VSTRUCT_2DNX VSTRUCT_3DNX VSTRUCT_4DNX])
+
;; Advanced SIMD 64-bit structure modes with 64-bit elements.
(define_mode_iterator VSTRUCT_DX [V2x1DI V2x1DF V3x1DI V3x1DF V4x1DI V4x1DF])
@@ -496,6 +498,8 @@ (define_mode_iterator VSTRUCT_4QD [VSTRUCT_4D VSTRUCT_4Q])
;; Advanced SIMD vector structure modes.
(define_mode_iterator VSTRUCT_QD [VSTRUCT_D VSTRUCT_Q])
+(define_mode_iterator VSTRUCT_QDNX [VSTRUCT_DNX VSTRUCT_Q])
+(define_mode_iterator VSTRUCT_INTERLEAVED [(VSTRUCT_DNX "!BYTES_BIG_ENDIAN")
VSTRUCT_Q])
;; Double scalar modes
(define_mode_iterator DX [DI DF DD])
diff --git a/gcc/testsuite/gcc.dg/vect/vect-early-break_39.c
b/gcc/testsuite/gcc.dg/vect/vect-early-break_39.c
index bc862ad20e6..6cb2c982a46 100644
--- a/gcc/testsuite/gcc.dg/vect/vect-early-break_39.c
+++ b/gcc/testsuite/gcc.dg/vect/vect-early-break_39.c
@@ -8,7 +8,7 @@
#endif
unsigned vect_a[N];
unsigned vect_b[N];
-
+
unsigned test4(unsigned x, unsigned n)
{
unsigned ret = 0;
@@ -18,11 +18,10 @@ unsigned test4(unsigned x, unsigned n)
if (vect_a[i]*2 > x)
break;
vect_a[i] = x;
-
+
}
return ret;
}
-/* AArch64 will scalarize the load and is able to vectorize it. */
-/* { dg-final { scan-tree-dump-times "vectorized 1 loops in function" 1 "vect"
{ target aarch64*-*-* } } } */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops in function" 1 "vect"
{ xfail aarch64*-*-* } } } */
/* { dg-final { scan-tree-dump-times "vectorized 1 loops in function" 0 "vect"
{ target { ! aarch64*-*-* } } } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/neon/vld2.c
b/gcc/testsuite/gcc.target/aarch64/neon/vld2.c
new file mode 100644
index 00000000000..63a39bfa172
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/neon/vld2.c
@@ -0,0 +1,228 @@
+/* { dg-do compile } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include "arm_neon_test.h"
+
+/*
+** test_vld2_mf8:
+** ld2 {v0\.8b - v1\.8b}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld2_mf8, mfloat8x8x2_t, const mfloat8_t *)
+
+/*
+** test_vld2_s8:
+** ld2 {v0\.8b - v1\.8b}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld2_s8, int8x8x2_t, const int8_t *)
+
+/*
+** test_vld2_u8:
+** ld2 {v0\.8b - v1\.8b}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld2_u8, uint8x8x2_t, const uint8_t *)
+
+/*
+** test_vld2_s16:
+** ld2 {v0\.4h - v1\.4h}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld2_s16, int16x4x2_t, const int16_t *)
+
+/*
+** test_vld2_u16:
+** ld2 {v0\.4h - v1\.4h}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld2_u16, uint16x4x2_t, const uint16_t *)
+
+/*
+** test_vld2_s32:
+** ld2 {v0\.2s - v1\.2s}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld2_s32, int32x2x2_t, const int32_t *)
+
+/*
+** test_vld2_u32:
+** ld2 {v0\.2s - v1\.2s}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld2_u32, uint32x2x2_t, const uint32_t *)
+
+/*
+** test_vld2_s64:
+** ld1 {v0\.1d - v1\.1d}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld2_s64, int64x1x2_t, const int64_t *)
+
+/*
+** test_vld2_u64:
+** ld1 {v0\.1d - v1\.1d}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld2_u64, uint64x1x2_t, const uint64_t *)
+
+/*
+** test_vld2_p8:
+** ld2 {v0\.8b - v1\.8b}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld2_p8, poly8x8x2_t, const poly8_t *)
+
+/*
+** test_vld2_p16:
+** ld2 {v0\.4h - v1\.4h}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld2_p16, poly16x4x2_t, const poly16_t *)
+
+/*
+** test_vld2_p64:
+** ld1 {v0\.1d - v1\.1d}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld2_p64, poly64x1x2_t, const poly64_t *)
+
+/*
+** test_vld2_bf16:
+** ld2 {v0\.4h - v1\.4h}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld2_bf16, bfloat16x4x2_t, const bfloat16_t *)
+
+/*
+** test_vld2_f16:
+** ld2 {v0\.4h - v1\.4h}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld2_f16, float16x4x2_t, const float16_t *)
+
+/*
+** test_vld2_f32:
+** ld2 {v0\.2s - v1\.2s}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld2_f32, float32x2x2_t, const float32_t *)
+
+/*
+** test_vld2_f64:
+** ld1 {v0\.1d - v1\.1d}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld2_f64, float64x1x2_t, const float64_t *)
+
+/*
+** test_vld2q_mf8:
+** ld2 {v0\.16b - v1\.16b}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld2q_mf8, mfloat8x16x2_t, const mfloat8_t *)
+
+/*
+** test_vld2q_s8:
+** ld2 {v0\.16b - v1\.16b}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld2q_s8, int8x16x2_t, const int8_t *)
+
+/*
+** test_vld2q_u8:
+** ld2 {v0\.16b - v1\.16b}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld2q_u8, uint8x16x2_t, const uint8_t *)
+
+/*
+** test_vld2q_s16:
+** ld2 {v0\.8h - v1\.8h}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld2q_s16, int16x8x2_t, const int16_t *)
+
+/*
+** test_vld2q_u16:
+** ld2 {v0\.8h - v1\.8h}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld2q_u16, uint16x8x2_t, const uint16_t *)
+
+/*
+** test_vld2q_s32:
+** ld2 {v0\.4s - v1\.4s}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld2q_s32, int32x4x2_t, const int32_t *)
+
+/*
+** test_vld2q_u32:
+** ld2 {v0\.4s - v1\.4s}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld2q_u32, uint32x4x2_t, const uint32_t *)
+
+/*
+** test_vld2q_s64:
+** ld2 {v0\.2d - v1\.2d}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld2q_s64, int64x2x2_t, const int64_t *)
+
+/*
+** test_vld2q_u64:
+** ld2 {v0\.2d - v1\.2d}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld2q_u64, uint64x2x2_t, const uint64_t *)
+
+/*
+** test_vld2q_p8:
+** ld2 {v0\.16b - v1\.16b}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld2q_p8, poly8x16x2_t, const poly8_t *)
+
+/*
+** test_vld2q_p16:
+** ld2 {v0\.8h - v1\.8h}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld2q_p16, poly16x8x2_t, const poly16_t *)
+
+/*
+** test_vld2q_p64:
+** ld2 {v0\.2d - v1\.2d}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld2q_p64, poly64x2x2_t, const poly64_t *)
+
+/*
+** test_vld2q_bf16:
+** ld2 {v0\.8h - v1\.8h}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld2q_bf16, bfloat16x8x2_t, const bfloat16_t *)
+
+/*
+** test_vld2q_f16:
+** ld2 {v0\.8h - v1\.8h}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld2q_f16, float16x8x2_t, const float16_t *)
+
+/*
+** test_vld2q_f32:
+** ld2 {v0\.4s - v1\.4s}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld2q_f32, float32x4x2_t, const float32_t *)
+
+/*
+** test_vld2q_f64:
+** ld2 {v0\.2d - v1\.2d}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld2q_f64, float64x2x2_t, const float64_t *)
diff --git a/gcc/testsuite/gcc.target/aarch64/neon/vld3.c
b/gcc/testsuite/gcc.target/aarch64/neon/vld3.c
new file mode 100644
index 00000000000..9dda6ec5471
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/neon/vld3.c
@@ -0,0 +1,228 @@
+/* { dg-do compile } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include "arm_neon_test.h"
+
+/*
+** test_vld3_mf8:
+** ld3 {v0\.8b - v2\.8b}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld3_mf8, mfloat8x8x3_t, const mfloat8_t *)
+
+/*
+** test_vld3_s8:
+** ld3 {v0\.8b - v2\.8b}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld3_s8, int8x8x3_t, const int8_t *)
+
+/*
+** test_vld3_u8:
+** ld3 {v0\.8b - v2\.8b}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld3_u8, uint8x8x3_t, const uint8_t *)
+
+/*
+** test_vld3_s16:
+** ld3 {v0\.4h - v2\.4h}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld3_s16, int16x4x3_t, const int16_t *)
+
+/*
+** test_vld3_u16:
+** ld3 {v0\.4h - v2\.4h}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld3_u16, uint16x4x3_t, const uint16_t *)
+
+/*
+** test_vld3_s32:
+** ld3 {v0\.2s - v2\.2s}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld3_s32, int32x2x3_t, const int32_t *)
+
+/*
+** test_vld3_u32:
+** ld3 {v0\.2s - v2\.2s}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld3_u32, uint32x2x3_t, const uint32_t *)
+
+/*
+** test_vld3_s64:
+** ld1 {v0\.1d - v2\.1d}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld3_s64, int64x1x3_t, const int64_t *)
+
+/*
+** test_vld3_u64:
+** ld1 {v0\.1d - v2\.1d}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld3_u64, uint64x1x3_t, const uint64_t *)
+
+/*
+** test_vld3_p8:
+** ld3 {v0\.8b - v2\.8b}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld3_p8, poly8x8x3_t, const poly8_t *)
+
+/*
+** test_vld3_p16:
+** ld3 {v0\.4h - v2\.4h}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld3_p16, poly16x4x3_t, const poly16_t *)
+
+/*
+** test_vld3_p64:
+** ld1 {v0\.1d - v2\.1d}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld3_p64, poly64x1x3_t, const poly64_t *)
+
+/*
+** test_vld3_bf16:
+** ld3 {v0\.4h - v2\.4h}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld3_bf16, bfloat16x4x3_t, const bfloat16_t *)
+
+/*
+** test_vld3_f16:
+** ld3 {v0\.4h - v2\.4h}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld3_f16, float16x4x3_t, const float16_t *)
+
+/*
+** test_vld3_f32:
+** ld3 {v0\.2s - v2\.2s}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld3_f32, float32x2x3_t, const float32_t *)
+
+/*
+** test_vld3_f64:
+** ld1 {v0\.1d - v2\.1d}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld3_f64, float64x1x3_t, const float64_t *)
+
+/*
+** test_vld3q_mf8:
+** ld3 {v0\.16b - v2\.16b}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld3q_mf8, mfloat8x16x3_t, const mfloat8_t *)
+
+/*
+** test_vld3q_s8:
+** ld3 {v0\.16b - v2\.16b}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld3q_s8, int8x16x3_t, const int8_t *)
+
+/*
+** test_vld3q_u8:
+** ld3 {v0\.16b - v2\.16b}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld3q_u8, uint8x16x3_t, const uint8_t *)
+
+/*
+** test_vld3q_s16:
+** ld3 {v0\.8h - v2\.8h}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld3q_s16, int16x8x3_t, const int16_t *)
+
+/*
+** test_vld3q_u16:
+** ld3 {v0\.8h - v2\.8h}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld3q_u16, uint16x8x3_t, const uint16_t *)
+
+/*
+** test_vld3q_s32:
+** ld3 {v0\.4s - v2\.4s}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld3q_s32, int32x4x3_t, const int32_t *)
+
+/*
+** test_vld3q_u32:
+** ld3 {v0\.4s - v2\.4s}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld3q_u32, uint32x4x3_t, const uint32_t *)
+
+/*
+** test_vld3q_s64:
+** ld3 {v0\.2d - v2\.2d}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld3q_s64, int64x2x3_t, const int64_t *)
+
+/*
+** test_vld3q_u64:
+** ld3 {v0\.2d - v2\.2d}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld3q_u64, uint64x2x3_t, const uint64_t *)
+
+/*
+** test_vld3q_p8:
+** ld3 {v0\.16b - v2\.16b}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld3q_p8, poly8x16x3_t, const poly8_t *)
+
+/*
+** test_vld3q_p16:
+** ld3 {v0\.8h - v2\.8h}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld3q_p16, poly16x8x3_t, const poly16_t *)
+
+/*
+** test_vld3q_p64:
+** ld3 {v0\.2d - v2\.2d}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld3q_p64, poly64x2x3_t, const poly64_t *)
+
+/*
+** test_vld3q_bf16:
+** ld3 {v0\.8h - v2\.8h}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld3q_bf16, bfloat16x8x3_t, const bfloat16_t *)
+
+/*
+** test_vld3q_f16:
+** ld3 {v0\.8h - v2\.8h}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld3q_f16, float16x8x3_t, const float16_t *)
+
+/*
+** test_vld3q_f32:
+** ld3 {v0\.4s - v2\.4s}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld3q_f32, float32x4x3_t, const float32_t *)
+
+/*
+** test_vld3q_f64:
+** ld3 {v0\.2d - v2\.2d}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld3q_f64, float64x2x3_t, const float64_t *)
diff --git a/gcc/testsuite/gcc.target/aarch64/neon/vld4.c
b/gcc/testsuite/gcc.target/aarch64/neon/vld4.c
new file mode 100644
index 00000000000..b8ff70f3474
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/neon/vld4.c
@@ -0,0 +1,228 @@
+/* { dg-do compile } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include "arm_neon_test.h"
+
+/*
+** test_vld4_mf8:
+** ld4 {v0\.8b - v3\.8b}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld4_mf8, mfloat8x8x4_t, const mfloat8_t *)
+
+/*
+** test_vld4_s8:
+** ld4 {v0\.8b - v3\.8b}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld4_s8, int8x8x4_t, const int8_t *)
+
+/*
+** test_vld4_u8:
+** ld4 {v0\.8b - v3\.8b}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld4_u8, uint8x8x4_t, const uint8_t *)
+
+/*
+** test_vld4_s16:
+** ld4 {v0\.4h - v3\.4h}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld4_s16, int16x4x4_t, const int16_t *)
+
+/*
+** test_vld4_u16:
+** ld4 {v0\.4h - v3\.4h}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld4_u16, uint16x4x4_t, const uint16_t *)
+
+/*
+** test_vld4_s32:
+** ld4 {v0\.2s - v3\.2s}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld4_s32, int32x2x4_t, const int32_t *)
+
+/*
+** test_vld4_u32:
+** ld4 {v0\.2s - v3\.2s}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld4_u32, uint32x2x4_t, const uint32_t *)
+
+/*
+** test_vld4_s64:
+** ld1 {v0\.1d - v3\.1d}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld4_s64, int64x1x4_t, const int64_t *)
+
+/*
+** test_vld4_u64:
+** ld1 {v0\.1d - v3\.1d}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld4_u64, uint64x1x4_t, const uint64_t *)
+
+/*
+** test_vld4_p8:
+** ld4 {v0\.8b - v3\.8b}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld4_p8, poly8x8x4_t, const poly8_t *)
+
+/*
+** test_vld4_p16:
+** ld4 {v0\.4h - v3\.4h}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld4_p16, poly16x4x4_t, const poly16_t *)
+
+/*
+** test_vld4_p64:
+** ld1 {v0\.1d - v3\.1d}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld4_p64, poly64x1x4_t, const poly64_t *)
+
+/*
+** test_vld4_bf16:
+** ld4 {v0\.4h - v3\.4h}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld4_bf16, bfloat16x4x4_t, const bfloat16_t *)
+
+/*
+** test_vld4_f16:
+** ld4 {v0\.4h - v3\.4h}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld4_f16, float16x4x4_t, const float16_t *)
+
+/*
+** test_vld4_f32:
+** ld4 {v0\.2s - v3\.2s}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld4_f32, float32x2x4_t, const float32_t *)
+
+/*
+** test_vld4_f64:
+** ld1 {v0\.1d - v3\.1d}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld4_f64, float64x1x4_t, const float64_t *)
+
+/*
+** test_vld4q_mf8:
+** ld4 {v0\.16b - v3\.16b}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld4q_mf8, mfloat8x16x4_t, const mfloat8_t *)
+
+/*
+** test_vld4q_s8:
+** ld4 {v0\.16b - v3\.16b}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld4q_s8, int8x16x4_t, const int8_t *)
+
+/*
+** test_vld4q_u8:
+** ld4 {v0\.16b - v3\.16b}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld4q_u8, uint8x16x4_t, const uint8_t *)
+
+/*
+** test_vld4q_s16:
+** ld4 {v0\.8h - v3\.8h}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld4q_s16, int16x8x4_t, const int16_t *)
+
+/*
+** test_vld4q_u16:
+** ld4 {v0\.8h - v3\.8h}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld4q_u16, uint16x8x4_t, const uint16_t *)
+
+/*
+** test_vld4q_s32:
+** ld4 {v0\.4s - v3\.4s}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld4q_s32, int32x4x4_t, const int32_t *)
+
+/*
+** test_vld4q_u32:
+** ld4 {v0\.4s - v3\.4s}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld4q_u32, uint32x4x4_t, const uint32_t *)
+
+/*
+** test_vld4q_s64:
+** ld4 {v0\.2d - v3\.2d}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld4q_s64, int64x2x4_t, const int64_t *)
+
+/*
+** test_vld4q_u64:
+** ld4 {v0\.2d - v3\.2d}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld4q_u64, uint64x2x4_t, const uint64_t *)
+
+/*
+** test_vld4q_p8:
+** ld4 {v0\.16b - v3\.16b}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld4q_p8, poly8x16x4_t, const poly8_t *)
+
+/*
+** test_vld4q_p16:
+** ld4 {v0\.8h - v3\.8h}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld4q_p16, poly16x8x4_t, const poly16_t *)
+
+/*
+** test_vld4q_p64:
+** ld4 {v0\.2d - v3\.2d}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld4q_p64, poly64x2x4_t, const poly64_t *)
+
+/*
+** test_vld4q_bf16:
+** ld4 {v0\.8h - v3\.8h}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld4q_bf16, bfloat16x8x4_t, const bfloat16_t *)
+
+/*
+** test_vld4q_f16:
+** ld4 {v0\.8h - v3\.8h}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld4q_f16, float16x8x4_t, const float16_t *)
+
+/*
+** test_vld4q_f32:
+** ld4 {v0\.4s - v3\.4s}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld4q_f32, float32x4x4_t, const float32_t *)
+
+/*
+** test_vld4q_f64:
+** ld4 {v0\.2d - v3\.2d}, \[x0\]
+** ret
+*/
+TEST_UNARY (vld4q_f64, float64x2x4_t, const float64_t *)
diff --git a/gcc/testsuite/gcc.target/aarch64/neon/vst2.c
b/gcc/testsuite/gcc.target/aarch64/neon/vst2.c
new file mode 100644
index 00000000000..1fa33f49d41
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/neon/vst2.c
@@ -0,0 +1,228 @@
+/* { dg-do compile } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include "arm_neon_test.h"
+
+/*
+** test_vst2_mf8:
+** st2 {v0\.8b - v1\.8b}, \[x0\]
+** ret
+*/
+TEST_STORE (vst2_mf8, mfloat8_t *, mfloat8x8x2_t)
+
+/*
+** test_vst2_s8:
+** st2 {v0\.8b - v1\.8b}, \[x0\]
+** ret
+*/
+TEST_STORE (vst2_s8, int8_t *, int8x8x2_t)
+
+/*
+** test_vst2_u8:
+** st2 {v0\.8b - v1\.8b}, \[x0\]
+** ret
+*/
+TEST_STORE (vst2_u8, uint8_t *, uint8x8x2_t)
+
+/*
+** test_vst2_s16:
+** st2 {v0\.4h - v1\.4h}, \[x0\]
+** ret
+*/
+TEST_STORE (vst2_s16, int16_t *, int16x4x2_t)
+
+/*
+** test_vst2_u16:
+** st2 {v0\.4h - v1\.4h}, \[x0\]
+** ret
+*/
+TEST_STORE (vst2_u16, uint16_t *, uint16x4x2_t)
+
+/*
+** test_vst2_s32:
+** st2 {v0\.2s - v1\.2s}, \[x0\]
+** ret
+*/
+TEST_STORE (vst2_s32, int32_t *, int32x2x2_t)
+
+/*
+** test_vst2_u32:
+** st2 {v0\.2s - v1\.2s}, \[x0\]
+** ret
+*/
+TEST_STORE (vst2_u32, uint32_t *, uint32x2x2_t)
+
+/*
+** test_vst2_s64:
+** st1 {v0\.1d - v1\.1d}, \[x0\]
+** ret
+*/
+TEST_STORE (vst2_s64, int64_t *, int64x1x2_t)
+
+/*
+** test_vst2_u64:
+** st1 {v0\.1d - v1\.1d}, \[x0\]
+** ret
+*/
+TEST_STORE (vst2_u64, uint64_t *, uint64x1x2_t)
+
+/*
+** test_vst2_p8:
+** st2 {v0\.8b - v1\.8b}, \[x0\]
+** ret
+*/
+TEST_STORE (vst2_p8, poly8_t *, poly8x8x2_t)
+
+/*
+** test_vst2_p16:
+** st2 {v0\.4h - v1\.4h}, \[x0\]
+** ret
+*/
+TEST_STORE (vst2_p16, poly16_t *, poly16x4x2_t)
+
+/*
+** test_vst2_p64:
+** st1 {v0\.1d - v1\.1d}, \[x0\]
+** ret
+*/
+TEST_STORE (vst2_p64, poly64_t *, poly64x1x2_t)
+
+/*
+** test_vst2_bf16:
+** st2 {v0\.4h - v1\.4h}, \[x0\]
+** ret
+*/
+TEST_STORE (vst2_bf16, bfloat16_t *, bfloat16x4x2_t)
+
+/*
+** test_vst2_f16:
+** st2 {v0\.4h - v1\.4h}, \[x0\]
+** ret
+*/
+TEST_STORE (vst2_f16, float16_t *, float16x4x2_t)
+
+/*
+** test_vst2_f32:
+** st2 {v0\.2s - v1\.2s}, \[x0\]
+** ret
+*/
+TEST_STORE (vst2_f32, float32_t *, float32x2x2_t)
+
+/*
+** test_vst2_f64:
+** st1 {v0\.1d - v1\.1d}, \[x0\]
+** ret
+*/
+TEST_STORE (vst2_f64, float64_t *, float64x1x2_t)
+
+/*
+** test_vst2q_mf8:
+** st2 {v0\.16b - v1\.16b}, \[x0\]
+** ret
+*/
+TEST_STORE (vst2q_mf8, mfloat8_t *, mfloat8x16x2_t)
+
+/*
+** test_vst2q_s8:
+** st2 {v0\.16b - v1\.16b}, \[x0\]
+** ret
+*/
+TEST_STORE (vst2q_s8, int8_t *, int8x16x2_t)
+
+/*
+** test_vst2q_u8:
+** st2 {v0\.16b - v1\.16b}, \[x0\]
+** ret
+*/
+TEST_STORE (vst2q_u8, uint8_t *, uint8x16x2_t)
+
+/*
+** test_vst2q_s16:
+** st2 {v0\.8h - v1\.8h}, \[x0\]
+** ret
+*/
+TEST_STORE (vst2q_s16, int16_t *, int16x8x2_t)
+
+/*
+** test_vst2q_u16:
+** st2 {v0\.8h - v1\.8h}, \[x0\]
+** ret
+*/
+TEST_STORE (vst2q_u16, uint16_t *, uint16x8x2_t)
+
+/*
+** test_vst2q_s32:
+** st2 {v0\.4s - v1\.4s}, \[x0\]
+** ret
+*/
+TEST_STORE (vst2q_s32, int32_t *, int32x4x2_t)
+
+/*
+** test_vst2q_u32:
+** st2 {v0\.4s - v1\.4s}, \[x0\]
+** ret
+*/
+TEST_STORE (vst2q_u32, uint32_t *, uint32x4x2_t)
+
+/*
+** test_vst2q_s64:
+** st2 {v0\.2d - v1\.2d}, \[x0\]
+** ret
+*/
+TEST_STORE (vst2q_s64, int64_t *, int64x2x2_t)
+
+/*
+** test_vst2q_u64:
+** st2 {v0\.2d - v1\.2d}, \[x0\]
+** ret
+*/
+TEST_STORE (vst2q_u64, uint64_t *, uint64x2x2_t)
+
+/*
+** test_vst2q_p8:
+** st2 {v0\.16b - v1\.16b}, \[x0\]
+** ret
+*/
+TEST_STORE (vst2q_p8, poly8_t *, poly8x16x2_t)
+
+/*
+** test_vst2q_p16:
+** st2 {v0\.8h - v1\.8h}, \[x0\]
+** ret
+*/
+TEST_STORE (vst2q_p16, poly16_t *, poly16x8x2_t)
+
+/*
+** test_vst2q_p64:
+** st2 {v0\.2d - v1\.2d}, \[x0\]
+** ret
+*/
+TEST_STORE (vst2q_p64, poly64_t *, poly64x2x2_t)
+
+/*
+** test_vst2q_bf16:
+** st2 {v0\.8h - v1\.8h}, \[x0\]
+** ret
+*/
+TEST_STORE (vst2q_bf16, bfloat16_t *, bfloat16x8x2_t)
+
+/*
+** test_vst2q_f16:
+** st2 {v0\.8h - v1\.8h}, \[x0\]
+** ret
+*/
+TEST_STORE (vst2q_f16, float16_t *, float16x8x2_t)
+
+/*
+** test_vst2q_f32:
+** st2 {v0\.4s - v1\.4s}, \[x0\]
+** ret
+*/
+TEST_STORE (vst2q_f32, float32_t *, float32x4x2_t)
+
+/*
+** test_vst2q_f64:
+** st2 {v0\.2d - v1\.2d}, \[x0\]
+** ret
+*/
+TEST_STORE (vst2q_f64, float64_t *, float64x2x2_t)
diff --git a/gcc/testsuite/gcc.target/aarch64/neon/vst3.c
b/gcc/testsuite/gcc.target/aarch64/neon/vst3.c
new file mode 100644
index 00000000000..1084efda274
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/neon/vst3.c
@@ -0,0 +1,228 @@
+/* { dg-do compile } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include "arm_neon_test.h"
+
+/*
+** test_vst3_mf8:
+** st3 {v0\.8b - v2\.8b}, \[x0\]
+** ret
+*/
+TEST_STORE (vst3_mf8, mfloat8_t *, mfloat8x8x3_t)
+
+/*
+** test_vst3_s8:
+** st3 {v0\.8b - v2\.8b}, \[x0\]
+** ret
+*/
+TEST_STORE (vst3_s8, int8_t *, int8x8x3_t)
+
+/*
+** test_vst3_u8:
+** st3 {v0\.8b - v2\.8b}, \[x0\]
+** ret
+*/
+TEST_STORE (vst3_u8, uint8_t *, uint8x8x3_t)
+
+/*
+** test_vst3_s16:
+** st3 {v0\.4h - v2\.4h}, \[x0\]
+** ret
+*/
+TEST_STORE (vst3_s16, int16_t *, int16x4x3_t)
+
+/*
+** test_vst3_u16:
+** st3 {v0\.4h - v2\.4h}, \[x0\]
+** ret
+*/
+TEST_STORE (vst3_u16, uint16_t *, uint16x4x3_t)
+
+/*
+** test_vst3_s32:
+** st3 {v0\.2s - v2\.2s}, \[x0\]
+** ret
+*/
+TEST_STORE (vst3_s32, int32_t *, int32x2x3_t)
+
+/*
+** test_vst3_u32:
+** st3 {v0\.2s - v2\.2s}, \[x0\]
+** ret
+*/
+TEST_STORE (vst3_u32, uint32_t *, uint32x2x3_t)
+
+/*
+** test_vst3_s64:
+** st1 {v0\.1d - v2\.1d}, \[x0\]
+** ret
+*/
+TEST_STORE (vst3_s64, int64_t *, int64x1x3_t)
+
+/*
+** test_vst3_u64:
+** st1 {v0\.1d - v2\.1d}, \[x0\]
+** ret
+*/
+TEST_STORE (vst3_u64, uint64_t *, uint64x1x3_t)
+
+/*
+** test_vst3_p8:
+** st3 {v0\.8b - v2\.8b}, \[x0\]
+** ret
+*/
+TEST_STORE (vst3_p8, poly8_t *, poly8x8x3_t)
+
+/*
+** test_vst3_p16:
+** st3 {v0\.4h - v2\.4h}, \[x0\]
+** ret
+*/
+TEST_STORE (vst3_p16, poly16_t *, poly16x4x3_t)
+
+/*
+** test_vst3_p64:
+** st1 {v0\.1d - v2\.1d}, \[x0\]
+** ret
+*/
+TEST_STORE (vst3_p64, poly64_t *, poly64x1x3_t)
+
+/*
+** test_vst3_bf16:
+** st3 {v0\.4h - v2\.4h}, \[x0\]
+** ret
+*/
+TEST_STORE (vst3_bf16, bfloat16_t *, bfloat16x4x3_t)
+
+/*
+** test_vst3_f16:
+** st3 {v0\.4h - v2\.4h}, \[x0\]
+** ret
+*/
+TEST_STORE (vst3_f16, float16_t *, float16x4x3_t)
+
+/*
+** test_vst3_f32:
+** st3 {v0\.2s - v2\.2s}, \[x0\]
+** ret
+*/
+TEST_STORE (vst3_f32, float32_t *, float32x2x3_t)
+
+/*
+** test_vst3_f64:
+** st1 {v0\.1d - v2\.1d}, \[x0\]
+** ret
+*/
+TEST_STORE (vst3_f64, float64_t *, float64x1x3_t)
+
+/*
+** test_vst3q_mf8:
+** st3 {v0\.16b - v2\.16b}, \[x0\]
+** ret
+*/
+TEST_STORE (vst3q_mf8, mfloat8_t *, mfloat8x16x3_t)
+
+/*
+** test_vst3q_s8:
+** st3 {v0\.16b - v2\.16b}, \[x0\]
+** ret
+*/
+TEST_STORE (vst3q_s8, int8_t *, int8x16x3_t)
+
+/*
+** test_vst3q_u8:
+** st3 {v0\.16b - v2\.16b}, \[x0\]
+** ret
+*/
+TEST_STORE (vst3q_u8, uint8_t *, uint8x16x3_t)
+
+/*
+** test_vst3q_s16:
+** st3 {v0\.8h - v2\.8h}, \[x0\]
+** ret
+*/
+TEST_STORE (vst3q_s16, int16_t *, int16x8x3_t)
+
+/*
+** test_vst3q_u16:
+** st3 {v0\.8h - v2\.8h}, \[x0\]
+** ret
+*/
+TEST_STORE (vst3q_u16, uint16_t *, uint16x8x3_t)
+
+/*
+** test_vst3q_s32:
+** st3 {v0\.4s - v2\.4s}, \[x0\]
+** ret
+*/
+TEST_STORE (vst3q_s32, int32_t *, int32x4x3_t)
+
+/*
+** test_vst3q_u32:
+** st3 {v0\.4s - v2\.4s}, \[x0\]
+** ret
+*/
+TEST_STORE (vst3q_u32, uint32_t *, uint32x4x3_t)
+
+/*
+** test_vst3q_s64:
+** st3 {v0\.2d - v2\.2d}, \[x0\]
+** ret
+*/
+TEST_STORE (vst3q_s64, int64_t *, int64x2x3_t)
+
+/*
+** test_vst3q_u64:
+** st3 {v0\.2d - v2\.2d}, \[x0\]
+** ret
+*/
+TEST_STORE (vst3q_u64, uint64_t *, uint64x2x3_t)
+
+/*
+** test_vst3q_p8:
+** st3 {v0\.16b - v2\.16b}, \[x0\]
+** ret
+*/
+TEST_STORE (vst3q_p8, poly8_t *, poly8x16x3_t)
+
+/*
+** test_vst3q_p16:
+** st3 {v0\.8h - v2\.8h}, \[x0\]
+** ret
+*/
+TEST_STORE (vst3q_p16, poly16_t *, poly16x8x3_t)
+
+/*
+** test_vst3q_p64:
+** st3 {v0\.2d - v2\.2d}, \[x0\]
+** ret
+*/
+TEST_STORE (vst3q_p64, poly64_t *, poly64x2x3_t)
+
+/*
+** test_vst3q_bf16:
+** st3 {v0\.8h - v2\.8h}, \[x0\]
+** ret
+*/
+TEST_STORE (vst3q_bf16, bfloat16_t *, bfloat16x8x3_t)
+
+/*
+** test_vst3q_f16:
+** st3 {v0\.8h - v2\.8h}, \[x0\]
+** ret
+*/
+TEST_STORE (vst3q_f16, float16_t *, float16x8x3_t)
+
+/*
+** test_vst3q_f32:
+** st3 {v0\.4s - v2\.4s}, \[x0\]
+** ret
+*/
+TEST_STORE (vst3q_f32, float32_t *, float32x4x3_t)
+
+/*
+** test_vst3q_f64:
+** st3 {v0\.2d - v2\.2d}, \[x0\]
+** ret
+*/
+TEST_STORE (vst3q_f64, float64_t *, float64x2x3_t)
diff --git a/gcc/testsuite/gcc.target/aarch64/neon/vst4.c
b/gcc/testsuite/gcc.target/aarch64/neon/vst4.c
new file mode 100644
index 00000000000..3239555284c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/neon/vst4.c
@@ -0,0 +1,228 @@
+/* { dg-do compile } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include "arm_neon_test.h"
+
+/*
+** test_vst4_mf8:
+** st4 {v0\.8b - v3\.8b}, \[x0\]
+** ret
+*/
+TEST_STORE (vst4_mf8, mfloat8_t *, mfloat8x8x4_t)
+
+/*
+** test_vst4_s8:
+** st4 {v0\.8b - v3\.8b}, \[x0\]
+** ret
+*/
+TEST_STORE (vst4_s8, int8_t *, int8x8x4_t)
+
+/*
+** test_vst4_u8:
+** st4 {v0\.8b - v3\.8b}, \[x0\]
+** ret
+*/
+TEST_STORE (vst4_u8, uint8_t *, uint8x8x4_t)
+
+/*
+** test_vst4_s16:
+** st4 {v0\.4h - v3\.4h}, \[x0\]
+** ret
+*/
+TEST_STORE (vst4_s16, int16_t *, int16x4x4_t)
+
+/*
+** test_vst4_u16:
+** st4 {v0\.4h - v3\.4h}, \[x0\]
+** ret
+*/
+TEST_STORE (vst4_u16, uint16_t *, uint16x4x4_t)
+
+/*
+** test_vst4_s32:
+** st4 {v0\.2s - v3\.2s}, \[x0\]
+** ret
+*/
+TEST_STORE (vst4_s32, int32_t *, int32x2x4_t)
+
+/*
+** test_vst4_u32:
+** st4 {v0\.2s - v3\.2s}, \[x0\]
+** ret
+*/
+TEST_STORE (vst4_u32, uint32_t *, uint32x2x4_t)
+
+/*
+** test_vst4_s64:
+** st1 {v0\.1d - v3\.1d}, \[x0\]
+** ret
+*/
+TEST_STORE (vst4_s64, int64_t *, int64x1x4_t)
+
+/*
+** test_vst4_u64:
+** st1 {v0\.1d - v3\.1d}, \[x0\]
+** ret
+*/
+TEST_STORE (vst4_u64, uint64_t *, uint64x1x4_t)
+
+/*
+** test_vst4_p8:
+** st4 {v0\.8b - v3\.8b}, \[x0\]
+** ret
+*/
+TEST_STORE (vst4_p8, poly8_t *, poly8x8x4_t)
+
+/*
+** test_vst4_p16:
+** st4 {v0\.4h - v3\.4h}, \[x0\]
+** ret
+*/
+TEST_STORE (vst4_p16, poly16_t *, poly16x4x4_t)
+
+/*
+** test_vst4_p64:
+** st1 {v0\.1d - v3\.1d}, \[x0\]
+** ret
+*/
+TEST_STORE (vst4_p64, poly64_t *, poly64x1x4_t)
+
+/*
+** test_vst4_bf16:
+** st4 {v0\.4h - v3\.4h}, \[x0\]
+** ret
+*/
+TEST_STORE (vst4_bf16, bfloat16_t *, bfloat16x4x4_t)
+
+/*
+** test_vst4_f16:
+** st4 {v0\.4h - v3\.4h}, \[x0\]
+** ret
+*/
+TEST_STORE (vst4_f16, float16_t *, float16x4x4_t)
+
+/*
+** test_vst4_f32:
+** st4 {v0\.2s - v3\.2s}, \[x0\]
+** ret
+*/
+TEST_STORE (vst4_f32, float32_t *, float32x2x4_t)
+
+/*
+** test_vst4_f64:
+** st1 {v0\.1d - v3\.1d}, \[x0\]
+** ret
+*/
+TEST_STORE (vst4_f64, float64_t *, float64x1x4_t)
+
+/*
+** test_vst4q_mf8:
+** st4 {v0\.16b - v3\.16b}, \[x0\]
+** ret
+*/
+TEST_STORE (vst4q_mf8, mfloat8_t *, mfloat8x16x4_t)
+
+/*
+** test_vst4q_s8:
+** st4 {v0\.16b - v3\.16b}, \[x0\]
+** ret
+*/
+TEST_STORE (vst4q_s8, int8_t *, int8x16x4_t)
+
+/*
+** test_vst4q_u8:
+** st4 {v0\.16b - v3\.16b}, \[x0\]
+** ret
+*/
+TEST_STORE (vst4q_u8, uint8_t *, uint8x16x4_t)
+
+/*
+** test_vst4q_s16:
+** st4 {v0\.8h - v3\.8h}, \[x0\]
+** ret
+*/
+TEST_STORE (vst4q_s16, int16_t *, int16x8x4_t)
+
+/*
+** test_vst4q_u16:
+** st4 {v0\.8h - v3\.8h}, \[x0\]
+** ret
+*/
+TEST_STORE (vst4q_u16, uint16_t *, uint16x8x4_t)
+
+/*
+** test_vst4q_s32:
+** st4 {v0\.4s - v3\.4s}, \[x0\]
+** ret
+*/
+TEST_STORE (vst4q_s32, int32_t *, int32x4x4_t)
+
+/*
+** test_vst4q_u32:
+** st4 {v0\.4s - v3\.4s}, \[x0\]
+** ret
+*/
+TEST_STORE (vst4q_u32, uint32_t *, uint32x4x4_t)
+
+/*
+** test_vst4q_s64:
+** st4 {v0\.2d - v3\.2d}, \[x0\]
+** ret
+*/
+TEST_STORE (vst4q_s64, int64_t *, int64x2x4_t)
+
+/*
+** test_vst4q_u64:
+** st4 {v0\.2d - v3\.2d}, \[x0\]
+** ret
+*/
+TEST_STORE (vst4q_u64, uint64_t *, uint64x2x4_t)
+
+/*
+** test_vst4q_p8:
+** st4 {v0\.16b - v3\.16b}, \[x0\]
+** ret
+*/
+TEST_STORE (vst4q_p8, poly8_t *, poly8x16x4_t)
+
+/*
+** test_vst4q_p16:
+** st4 {v0\.8h - v3\.8h}, \[x0\]
+** ret
+*/
+TEST_STORE (vst4q_p16, poly16_t *, poly16x8x4_t)
+
+/*
+** test_vst4q_p64:
+** st4 {v0\.2d - v3\.2d}, \[x0\]
+** ret
+*/
+TEST_STORE (vst4q_p64, poly64_t *, poly64x2x4_t)
+
+/*
+** test_vst4q_bf16:
+** st4 {v0\.8h - v3\.8h}, \[x0\]
+** ret
+*/
+TEST_STORE (vst4q_bf16, bfloat16_t *, bfloat16x8x4_t)
+
+/*
+** test_vst4q_f16:
+** st4 {v0\.8h - v3\.8h}, \[x0\]
+** ret
+*/
+TEST_STORE (vst4q_f16, float16_t *, float16x8x4_t)
+
+/*
+** test_vst4q_f32:
+** st4 {v0\.4s - v3\.4s}, \[x0\]
+** ret
+*/
+TEST_STORE (vst4q_f32, float32_t *, float32x4x4_t)
+
+/*
+** test_vst4q_f64:
+** st4 {v0\.2d - v3\.2d}, \[x0\]
+** ret
+*/
+TEST_STORE (vst4q_f64, float64_t *, float64x2x4_t)
diff --git a/gcc/testsuite/gcc.target/aarch64/pr99873_1.c
b/gcc/testsuite/gcc.target/aarch64/pr99873_1.c
index bc4d81e3ae5..b79ca06db6b 100644
--- a/gcc/testsuite/gcc.target/aarch64/pr99873_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/pr99873_1.c
@@ -13,5 +13,10 @@ f (int *restrict x, int *restrict y, int *restrict z, int n)
}
}
-/* { dg-final { scan-assembler-times {\tld3\t} 2 } } */
-/* { dg-final { scan-assembler-times {\tst3\t} 1 } } */
+// SIMD loop body:
+/* { dg-final { scan-assembler-times { ld3 \{v[0-9]+\.4s - v[0-9]+\.4s\}}
2 } } */
+/* { dg-final { scan-assembler-times { st3 \{v[0-9]+\.4s - v[0-9]+\.4s\}}
1 } } */
+
+// SIMD loop tail:
+/* { dg-final { scan-assembler-times { ld3 \{v[0-9]+\.2s - v[0-9]+\.2s\}}
2 } } */
+/* { dg-final { scan-assembler-times { st3 \{v[0-9]+\.2s - v[0-9]+\.2s\}}
1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/vld2-1.c
b/gcc/testsuite/gcc.target/aarch64/vld2-1.c
index 0a5b97bfb75..5f0f0626b7a 100644
--- a/gcc/testsuite/gcc.target/aarch64/vld2-1.c
+++ b/gcc/testsuite/gcc.target/aarch64/vld2-1.c
@@ -43,4 +43,4 @@ void func3(float32x2x2_t *p, const float *p1, const float *p2)
}
/* 2 copy props for each function */
-/* { dg-final { scan-tree-dump-times "after previous" 6 "forwprop1" } } */
+/* { dg-final { scan-tree-dump-times "after previous" 3 "forwprop1" } } */
--
2.56.0
