Use vbitrevb for scalar bitreverse when AVX512BMM and AVX512VL are
available, while retaining GFNI as the fallback without AVX512VL.
Bootstrapped and regression tested on x86_64-pc-linux-gnu.
The vector bitreverse optab has also been added; vectorizer support will be
addressed separately.
gcc/
* config/i386/i386-expand.cc (ix86_expand_bitreverse): Use vbitrevb only
when AVX512BMM and AVX512VL are available; retain GFNI otherwise.
* config/i386/i386.md (bitreverse<mode>2): Require AVX512VL for BMM.
gcc/testsuite/
* gcc.target/i386/avx512bmm-bitreverse-1.c: Test BMM scalar lowering.
* gcc.target/i386/gfni-bitreverse-scalar.c: Test the GFNI-only path.
Co-authored-by: Venkataramanan Kumar <[email protected]>
Signed-off-by: Sarvesh Chandra <[email protected]>
---
> > It bypasses avx256_optimal/prefer-vector-width=256 and unconditionally
> > generates ZMM even with -mavx512bmm + -mprefer-vector-width=256.
Changed now for simplicity. However, even BMM with ZMMs is more optimal code
than the GFNI counterpart.
> > It appears that even with the vector optab added, the loop is still not
> > being vectorized.
Yes, we must do so. Venkat was planning to take this up.
gcc/config/i386/i386-expand.cc | 32 ++++++++++++--------
gcc/config/i386/i386-protos.h | 2 +-
gcc/config/i386/i386.md | 4 +--
gcc/config/i386/sse.md | 10 ++++++
.../gcc.target/i386/avx512bmm-bitreverse-1.c | 28 ++++++++++++++++
.../gcc.target/i386/gfni-bitreverse-scalar.c | 30 +++++++++++++++++
6 files changed, 90 insertions(+), 16 deletions(-)
create mode 100644 gcc/testsuite/gcc.target/i386/avx512bmm-bitreverse-1.c
create mode 100644 gcc/testsuite/gcc.target/i386/gfni-bitreverse-scalar.c
diff --git a/gcc/config/i386/i386-expand.cc b/gcc/config/i386/i386-expand.cc
index 7cc210f8e68..7418cb4f1d2 100644
--- a/gcc/config/i386/i386-expand.cc
+++ b/gcc/config/i386/i386-expand.cc
@@ -28151,10 +28151,10 @@ ix86_expand_vector_bf2sf_with_vec_perm (rtx dest, rtx
src)
emit_move_insn (dest, lowpart_subreg (GET_MODE (dest), target, vperm_mode));
}
-/* Implement bitreverse<mode>2 using gf2p8affineqb. */
+/* Implement bitreverse<mode>2 using vbitrevb or gf2p8affineqb. */
void
-ix86_expand_gfni_bitreverse (rtx dest, rtx src)
+ix86_expand_bitreverse (rtx dest, rtx src)
{
machine_mode mode = GET_MODE (dest);
rtx temp;
@@ -28201,18 +28201,24 @@ ix86_expand_gfni_bitreverse (rtx dest, rtx src)
}
src = temp;
temp = gen_reg_rtx (V16QImode);
- rtx src2 = gen_rtx_CONST_VECTOR (V16QImode,
- gen_rtvec (16, GEN_INT (1), GEN_INT (2),
- GEN_INT (4), GEN_INT (8),
- GEN_INT (16), GEN_INT (32),
- GEN_INT (64), GEN_INT (-128),
- GEN_INT (1), GEN_INT (2),
- GEN_INT (4), GEN_INT (8),
- GEN_INT (16), GEN_INT (32),
- GEN_INT (64), GEN_INT (-128)));
- src2 = validize_mem (force_const_mem (V16QImode, src2));
src = lowpart_subreg (V16QImode, src, GET_MODE (src));
- emit_insn (gen_vgf2p8affineqb_v16qi (temp, src, src2, const0_rtx));
+ if (TARGET_AVX512BMM && TARGET_AVX512VL)
+ emit_insn (gen_avx512bmm_vbitrevb_v16qi (temp, src));
+ else if (TARGET_GFNI)
+ {
+ rtx src2 = gen_rtx_CONST_VECTOR (V16QImode,
+ gen_rtvec (16, GEN_INT (1), GEN_INT (2),
+ GEN_INT (4), GEN_INT (8),
+ GEN_INT (16), GEN_INT (32),
+ GEN_INT (64), GEN_INT (-128),
+ GEN_INT (1), GEN_INT (2),
+ GEN_INT (4), GEN_INT (8),
+ GEN_INT (16), GEN_INT (32),
+ GEN_INT (64),
+ GEN_INT (-128)));
+ src2 = validize_mem (force_const_mem (V16QImode, src2));
+ emit_insn (gen_vgf2p8affineqb_v16qi (temp, src, src2, const0_rtx));
+ }
if (mode == QImode)
{
rtx temp1 = gen_reg_rtx (SImode);
diff --git a/gcc/config/i386/i386-protos.h b/gcc/config/i386/i386-protos.h
index 106a79dd41e..99814eb5bc1 100644
--- a/gcc/config/i386/i386-protos.h
+++ b/gcc/config/i386/i386-protos.h
@@ -263,7 +263,7 @@ extern rtx ix86_expand_ternlog (machine_mode mode, rtx op0,
rtx op1, rtx op2,
int idx, rtx target);
extern void ix86_expand_vector_sf2bf_with_vec_perm (rtx, rtx);
extern void ix86_expand_vector_bf2sf_with_vec_perm (rtx, rtx);
-extern void ix86_expand_gfni_bitreverse (rtx, rtx);
+extern void ix86_expand_bitreverse (rtx, rtx);
#ifdef TREE_CODE
diff --git a/gcc/config/i386/i386.md b/gcc/config/i386/i386.md
index 063e14cb2c1..59d02551f2d 100644
--- a/gcc/config/i386/i386.md
+++ b/gcc/config/i386/i386.md
@@ -22967,9 +22967,9 @@
(define_expand "bitreverse<mode>2"
[(set (match_operand:SWIDWI 0 "register_operand")
(bitreverse:SWIDWI (match_operand:SWIDWI 1 "nonimmediate_operand")))]
- "TARGET_GFNI && TARGET_SSE2"
+ "(TARGET_GFNI || (TARGET_AVX512BMM && TARGET_AVX512VL)) && TARGET_SSE2"
{
- ix86_expand_gfni_bitreverse (operands[0], operands[1]);
+ ix86_expand_bitreverse (operands[0], operands[1]);
DONE;
})
diff --git a/gcc/config/i386/sse.md b/gcc/config/i386/sse.md
index de092f4b9ae..6cb2a8ba36c 100644
--- a/gcc/config/i386/sse.md
+++ b/gcc/config/i386/sse.md
@@ -33668,3 +33668,13 @@
"vbitrevb\t{%1, %0|%0, %1}"
[(set_attr "prefix" "evex")
(set_attr "mode" "<sseinsnmode>")])
+
+(define_expand "bitreverse<mode>2"
+ [(set (match_operand:VI1_AVX512VL 0 "register_operand")
+ (bitreverse:VI1_AVX512VL
+ (match_operand:VI1_AVX512VL 1 "nonimmediate_operand")))]
+ "TARGET_AVX512BMM"
+{
+ emit_insn (gen_avx512bmm_vbitrevb_<mode> (operands[0], operands[1]));
+ DONE;
+})
diff --git a/gcc/testsuite/gcc.target/i386/avx512bmm-bitreverse-1.c
b/gcc/testsuite/gcc.target/i386/avx512bmm-bitreverse-1.c
new file mode 100644
index 00000000000..a2dd451e533
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/avx512bmm-bitreverse-1.c
@@ -0,0 +1,28 @@
+/* { dg-do compile } */
+/* { dg-options "-mavx512bmm -mavx512vl -mno-gfni -O2" } */
+/* { dg-final { scan-assembler-times "vbitrevb" 4 } } */
+/* { dg-final { scan-assembler-not "gf2p8affineqb" } } */
+
+unsigned char
+br8 (unsigned char x)
+{
+ return __builtin_bitreverse8 (x);
+}
+
+unsigned short
+br16 (unsigned short x)
+{
+ return __builtin_bitreverse16 (x);
+}
+
+unsigned int
+br32 (unsigned int x)
+{
+ return __builtin_bitreverse32 (x);
+}
+
+unsigned long long
+br64 (unsigned long long x)
+{
+ return __builtin_bitreverse64 (x);
+}
diff --git a/gcc/testsuite/gcc.target/i386/gfni-bitreverse-scalar.c
b/gcc/testsuite/gcc.target/i386/gfni-bitreverse-scalar.c
new file mode 100644
index 00000000000..94c032f27b7
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/gfni-bitreverse-scalar.c
@@ -0,0 +1,30 @@
+/* { dg-do compile } */
+/* { dg-options "-mgfni -mno-avx512bmm -O2" } */
+/* { dg-final { scan-assembler-times "gf2p8affineqb" 4 } } */
+/* { dg-final { scan-assembler-not "vbitrevb" } } */
+/* { dg-final { scan-assembler-not "vgf2p8affineqb" } } */
+
+/* GFNI-only scalar bitreverse must retain the existing legacy GFNI path. */
+unsigned char
+br8 (unsigned char x)
+{
+ return __builtin_bitreverse8 (x);
+}
+
+unsigned short
+br16 (unsigned short x)
+{
+ return __builtin_bitreverse16 (x);
+}
+
+unsigned int
+br32 (unsigned int x)
+{
+ return __builtin_bitreverse32 (x);
+}
+
+unsigned long long
+br64 (unsigned long long x)
+{
+ return __builtin_bitreverse64 (x);
+}
--
2.34.1
