From: Richard Earnshaw <[email protected]>
Generalize the SVE vector extraction patterns to handle partial vectors
as well as full vectors. This allows us to handle extractions that
are now exposed by optimizing vec_select to dup operations.
gcc/ChangeLog:
* config/aarch64/aarch64-sve.md (@aarch64_vec_duplicate_vq<mode>_be):
Iterate over SVE_ALL.
(vec_extract<mode><Vel>): Likewise.
(vec_extract<mode><v64>): Likewise.
(@aarch64_sve_dupq_lane<mode>): Likewise.
(@aarch64_sve_get_neonq_<mode>): Likewise.
(*vec_extract<mode><Vel>_v128): Likewise. Adjust calculations for
partial vectors.
(*vec_extract<mode><Vel>_dup): Likewise.
(*vec_extract<mode><Vel>_ext): Likewise.
* config/aarch64/iterators.md (elem_bits): Add partial vector modes.
(VEL_INT): Likewise.
(V128): Likewise.
(V_INT_EQUIV): Likewise.
(v_int_equiv): Likewise.
gcc/testsuite/ChangeLog:
* gcc.target/aarch64/sve/vec_lane_1.c: New test.
---
gcc/config/aarch64/aarch64-sve.md | 34 +++---
gcc/config/aarch64/iterators.md | 58 +++++++++-
.../gcc.target/aarch64/sve/vec_lane_1.c | 103 ++++++++++++++++++
3 files changed, 176 insertions(+), 19 deletions(-)
create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/vec_lane_1.c
diff --git a/gcc/config/aarch64/aarch64-sve.md
b/gcc/config/aarch64/aarch64-sve.md
index d0b3612f340a9..30fcc9d1a00d5 100644
--- a/gcc/config/aarch64/aarch64-sve.md
+++ b/gcc/config/aarch64/aarch64-sve.md
@@ -2880,8 +2880,8 @@
;; lsb into the register lsb. We therefore have to describe this in rtl
;; terms as a reverse of the V128 vector followed by a duplicate.
(define_insn "@aarch64_vec_duplicate_vq<mode>_be"
- [(set (match_operand:SVE_FULL 0 "register_operand" "=w")
- (vec_duplicate:SVE_FULL
+ [(set (match_operand:SVE_ALL 0 "register_operand" "=w")
+ (vec_duplicate:SVE_ALL
(vec_select:<V128>
(match_operand:<V128> 1 "register_operand" "w")
(match_operand 2 "descending_int_parallel"))))]
@@ -3142,7 +3142,7 @@
(define_expand "vec_extract<mode><Vel>"
[(set (match_operand:<VEL> 0 "register_operand")
(vec_select:<VEL>
- (match_operand:SVE_FULL 1 "register_operand")
+ (match_operand:SVE_ALL 1 "register_operand")
(parallel [(match_operand:SI 2 "nonmemory_operand")])))]
"TARGET_SVE"
{
@@ -3229,7 +3229,7 @@
(define_insn_and_split "*vec_extract<mode><Vel>_0"
[(set (match_operand:<VEL> 0 "aarch64_simd_nonimmediate_operand" "=r, w,
Utv")
(vec_select:<VEL>
- (match_operand:SVE_FULL 1 "register_operand" "w, 0, w")
+ (match_operand:SVE_ALL 1 "register_operand" "w, 0, w")
(parallel [(const_int 0)])))]
"TARGET_SVE"
{
@@ -3263,12 +3263,14 @@
(define_insn "*vec_extract<mode><Vel>_v128"
[(set (match_operand:<VEL> 0 "aarch64_simd_nonimmediate_operand" "=r, w,
Utv")
(vec_select:<VEL>
- (match_operand:SVE_FULL 1 "register_operand" "w, w, w")
+ (match_operand:SVE_ALL 1 "register_operand" "w, w, w")
(parallel [(match_operand:SI 2 "const_int_operand")])))]
"TARGET_SVE
- && IN_RANGE (INTVAL (operands[2]) * GET_MODE_SIZE (<VEL>mode), 1, 15)"
+ && IN_RANGE (INTVAL (operands[2]) * <container_bits> / 8, 1, 15)"
{
operands[1] = gen_rtx_REG (<V128>mode, REGNO (operands[1]));
+ operands[2] = GEN_INT (INTVAL (operands[2])
+ * <container_bits> / <elem_bits>);
switch (which_alternative)
{
case 0:
@@ -3289,12 +3291,14 @@
(define_insn "*vec_extract<mode><Vel>_dup"
[(set (match_operand:<VEL> 0 "register_operand" "=w")
(vec_select:<VEL>
- (match_operand:SVE_FULL 1 "register_operand" "w")
+ (match_operand:SVE_ALL 1 "register_operand" "w")
(parallel [(match_operand:SI 2 "const_int_operand")])))]
"TARGET_SVE
- && IN_RANGE (INTVAL (operands[2]) * GET_MODE_SIZE (<VEL>mode), 16, 63)"
+ && IN_RANGE (INTVAL (operands[2]) * <container_bits> / 8, 16, 63)"
{
operands[0] = gen_rtx_REG (<MODE>mode, REGNO (operands[0]));
+ operands[2] = GEN_INT (INTVAL (operands[2])
+ * <container_bits> / <elem_bits>);
return "dup\t%0.<Vetype>, %1.<Vetype>[%2]";
}
[(set_attr "sve_type" "sve_int_general")]
@@ -3305,12 +3309,12 @@
(define_insn "*vec_extract<mode><Vel>_ext"
[(set (match_operand:<VEL> 0 "register_operand" "=w, ?&w")
(vec_select:<VEL>
- (match_operand:SVE_FULL 1 "register_operand" "0, w")
+ (match_operand:SVE_ALL 1 "register_operand" "0, w")
(parallel [(match_operand:SI 2 "const_int_operand")])))]
- "TARGET_SVE && INTVAL (operands[2]) * GET_MODE_SIZE (<VEL>mode) >= 64"
+ "TARGET_SVE && INTVAL (operands[2]) * <container_bits> / 8 >= 64"
{
operands[0] = gen_rtx_REG (<MODE>mode, REGNO (operands[0]));
- operands[2] = GEN_INT (INTVAL (operands[2]) * GET_MODE_SIZE (<VEL>mode));
+ operands[2] = GEN_INT (INTVAL (operands[2]) * <container_bits> / 8);
return (which_alternative == 0
? "ext\t%0.b, %0.b, %0.b, #%2"
: "movprfx\t%0, %1\;ext\t%0.b, %0.b, %1.b, #%2");
@@ -10415,10 +10419,10 @@
;; for big-endian targets. In this fused pattern the two reverses cancel
;; each other out.
(define_insn "@aarch64_sve_dupq_lane<mode>"
- [(set (match_operand:SVE_FULL 0 "register_operand" "=w")
- (vec_duplicate:SVE_FULL
+ [(set (match_operand:SVE_ALL 0 "register_operand" "=w")
+ (vec_duplicate:SVE_ALL
(vec_select:<V128>
- (match_operand:SVE_FULL 1 "register_operand" "w")
+ (match_operand:SVE_ALL 1 "register_operand" "w")
(match_operand 2 "ascending_int_parallel"))))]
"TARGET_SVE
&& (INTVAL (XVECEXP (operands[2], 0, 0))
@@ -12898,7 +12902,7 @@
(define_insn_and_split "@aarch64_sve_get_neonq_<mode>"
[(set (match_operand:<V128> 0 "register_operand" "=w")
(vec_select:<V128>
- (match_operand:SVE_FULL 1 "register_operand" "w")
+ (match_operand:SVE_ALL 1 "register_operand" "w")
(match_operand 2 "descending_int_parallel")))]
"TARGET_SVE
&& BYTES_BIG_ENDIAN
diff --git a/gcc/config/aarch64/iterators.md b/gcc/config/aarch64/iterators.md
index f69bf88a47d47..600521a2991ab 100644
--- a/gcc/config/aarch64/iterators.md
+++ b/gcc/config/aarch64/iterators.md
@@ -1659,7 +1659,13 @@
(VNx4SF "32") (VNx8SF "32") (VNx16SF "32")
(VNx2DI "64") (VNx4DI "64") (VNx8DI "64")
(VNx2DF "64") (VNx4DF "64") (VNx8DF "64")
- (VNx1TI "128")])
+ (VNx1TI "128")
+ (VNx8QI "8") (VNx4QI "8") (VNx2QI "8")
+ (VNx4HI "16") (VNx2HI "16")
+ (VNx4HF "16") (VNx2HF "16")
+ (VNx4BF "16") (VNx2BF "16")
+ (VNx2SI "32") (VNx2SF "32")
+])
;; The number of bits in a vector container.
(define_mode_attr container_bits [(VNx16QI "8")
@@ -2031,19 +2037,42 @@
(define_mode_attr VEL_INT [(VNx16QI "QI")
(VNx8HI "HI") (VNx8HF "HI") (VNx8BF "HI")
(VNx4SI "SI") (VNx4SF "SI")
- (VNx2DI "DI") (VNx2DF "DI")])
+ (VNx2DI "DI") (VNx2DF "DI")
+ (VNx8QI "QI")
+ (VNx4HI "HI") (VNx4HF "HI") (VNx4BF "HI")
+ (VNx2SI "SI") (VNx2SF "SI")
+ (VNx1DI "DI")
+ (VNx4QI "QI")
+ (VNx2HI "HI")
+ (VNx2HF "HI") (VNx2BF "HI")
+ (VNx2QI "QI")
+ ])
;; Gives the mode of the 128-bit lowpart of an SVE vector.
(define_mode_attr V128 [(VNx16QI "V16QI")
(VNx8HI "V8HI") (VNx8HF "V8HF") (VNx8BF "V8BF")
(VNx4SI "V4SI") (VNx4SF "V4SF")
- (VNx2DI "V2DI") (VNx2DF "V2DF")])
+ (VNx2DI "V2DI") (VNx2DF "V2DF")
+ (VNx8QI "V16QI")
+ (VNx4HI "V8HI") (VNx4HF "V8HF") (VNx4BF "V8BF")
+ (VNx2SI "V4SI") (VNx2SF "V4SF")
+ (VNx1DI "V2DI")
+ (VNx4QI "V16QI") (VNx2QI "V16QI")
+ (VNx2HI "V8HI") (VNx2HF "V8HF") (VNx2BF "V8BF")
+])
;; ...and again in lower case.
(define_mode_attr v128 [(VNx16QI "v16qi")
(VNx8HI "v8hi") (VNx8HF "v8hf") (VNx8BF "v8bf")
(VNx4SI "v4si") (VNx4SF "v4sf")
- (VNx2DI "v2di") (VNx2DF "v2df")])
+ (VNx2DI "v2di") (VNx2DF "v2df")
+ (VNx8QI "v16qi")
+ (VNx4HI "v8hi") (VNx4HF "v8hf") (VNx4BF "v8bf")
+ (VNx2SI "v4si") (VNx2SF "v4sf")
+ (VNx1DI "v2di")
+ (VNx4QI "v16qi") (VNx2QI "v16qi")
+ (VNx2HI "v8hi") (VNx2HF "v8hf") (VNx2BF "v8bf")
+])
;; Gives the mode of the 64-bit lowpart of an SVE vector.
(define_mode_attr V64 [(VNx16QI "V8QI")
@@ -2542,6 +2571,15 @@
(VNx4SI "VNx4SI") (VNx4SF "VNx4SI")
(VNx2DI "VNx2DI") (VNx2DF "VNx2DI")
(VNx8SF "VNx8SI") (VNx16SF "VNx16SI")
+ (VNx8QI "VNx8QI")
+ (VNx4HI "VNx4HI") (VNx4HF "VNx4HI")
+ (VNx4BF "VNx4HI")
+ (VNx2SI "VNx2SI") (VNx2SF "VNx2SI")
+ (VNx1DI "VNx1DI")
+ (VNx4QI "VNx4QI")
+ (VNx2HI "VNx2HI") (VNx2HF "VNx2HI")
+ (VNx2BF "VNx2HI")
+ (VNx2QI "VNx2QI")
])
;; Lower case mode with floating-point values replaced by like-sized integers.
@@ -2554,12 +2592,24 @@
(V2SF "v2si") (V4SF "v4si")
(DF "di") (V2DF "v2di")
(SF "si")
+ (HF "hi") (BF "hi")
(VNx16QI "vnx16qi")
(VNx8HI "vnx8hi") (VNx8HF "vnx8hi")
(VNx8BF "vnx8hi")
+ (VNx16BF "vnx16hi")
+ (VNx32BF "vnx32hi")
(VNx4SI "vnx4si") (VNx4SF "vnx4si")
(VNx2DI "vnx2di") (VNx2DF "vnx2di")
(VNx8SF "vnx8si") (VNx16SF "vnx16si")
+ (VNx8QI "vnx8qi")
+ (VNx4HI "vnx4hi") (VNx4HF "vnx4hi")
+ (VNx4BF "vnx4hi")
+ (VNx2SI "vnx2si") (VNx2SF "vnx2si")
+ (VNx1DI "vnx1di")
+ (VNx4QI "vnx4qi")
+ (VNx2HI "vnx2hi") (VNx2HF "vnx2hi")
+ (VNx2BF "vnx2hi")
+ (VNx2QI "vnx2qi")
])
;; Mode with floating-point values replaced by 128-bit vector integers.
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/vec_lane_1.c
b/gcc/testsuite/gcc.target/aarch64/sve/vec_lane_1.c
new file mode 100644
index 0000000000000..c90bd1d725783
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/vec_lane_1.c
@@ -0,0 +1,103 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=armv8.2-a+sve -fdump-rtl-expand -Wno-attributes" }
*/
+/* { dg-final { check-function-bodies "**" "" } } */
+
+typedef unsigned char partial_qi __attribute__((mode(VNx8QI)));
+typedef float partial_sf __attribute__((mode(VNx2SF)));
+
+__attribute__((noipa))
+unsigned char
+extract_0_qi (partial_qi *v)
+{
+ partial_qi x = *v;
+ asm ("": "+w"(x): : "memory");
+ return (x)[0];
+}
+/*
+** extract_0_qi:
+** ptrue (p[0-9]+)\.b, all
+** ld1b z([0-9]+)\.h, \1/z, \[x0\]
+** umov w0, v\2.b\[0\]
+** ret
+*/
+
+__attribute__((noipa))
+unsigned char
+extract_1_qi (partial_qi *v)
+{
+ partial_qi x = *v;
+ asm ("": "+w"(x): : "memory");
+ return (x)[1];
+}
+/*
+** extract_1_qi:
+** ptrue (p[0-9]+)\.b, all
+** ld1b z([0-9]+)\.h, \1/z, \[x0\]
+** umov w0, v\2.b\[2\]
+** ret
+*/
+
+__attribute__((noipa))
+unsigned char
+extract_3_qi (partial_qi *v)
+{
+ partial_qi x = *v;
+ asm ("": : "w"(x) : "memory");
+ return (x)[3];
+}
+/*
+** extract_3_qi:
+** ptrue (p[0-9]+)\.b, all
+** ld1b z([0-9]+)\.h, \1/z, \[x0\]
+** umov w0, v\2.b\[6\]
+** ret
+*/
+
+__attribute__((noipa))
+unsigned char
+extract_7_qi (partial_qi *v)
+{
+ partial_qi x = *v;
+ asm ("": : "w"(x) : "memory");
+ return (x)[7];
+}
+/*
+** extract_7_qi:
+** ptrue (p[0-9]+)\.b, all
+** ld1b z([0-9]+)\.h, \1/z, \[x0\]
+** umov w0, v\2.b\[14\]
+** ret
+*/
+
+__attribute__((noipa))
+unsigned char
+extract_8_qi (partial_qi *v)
+{
+ partial_qi x = *v;
+ asm ("": : "w"(x) : "memory");
+ return (x)[8];
+}
+/*
+** extract_8_qi: { xfail *-*-* }
+** ptrue (p[0-9]+)\.b, all
+** ld1b (z[0-9]+)\.h, \1/z, \[x0\]
+** dup z([0-9]+)\.b, \2.b\[16\]
+** umov w0, v\3\.b\[0\]
+** ret
+*/
+
+__attribute__((noipa))
+float
+extract_1_sf (partial_sf *v)
+{
+ partial_sf x = *v;
+ asm ("": : "w"(x) : "memory");
+ return (x)[1];
+}
+/*
+** extract_1_sf:
+** ptrue (p[0-9]+)\.b, all
+** ld1w z([0-9]+)\.d, \1/z, \[x0\]
+** dup s0, v\2.s\[2\]
+** ret
+*/
--
2.56.0
