Add vbitrevb lowering for scalar and generic vector bitreverse operations
when AVX512BMM is enabled. Prefer BMM over GFNI whenever BMM is available:
use narrow xmm/ymm forms with AVX512VL, and widen scalar temporaries to
zmm without AVX512VL. Use GFNI only when AVX512BMM is unavailable.
Bootstrapped and regression tested on x86_64-pc-linux-gnu.
gcc/
* config/i386/i386-expand.cc (ix86_expand_bitreverse): Prefer BMM over
GFNI; use zmm vbitrevb when AVX512VL is unavailable.
* config/i386/i386-protos.h (ix86_expand_bitreverse): Update the
expander declaration.
* config/i386/i386.md (bitreverse<mode>2): Enable AVX512BMM and call
the shared expander.
* config/i386/sse.md (bitreverse<mode>2): Reuse
avx512bmm_vbitrevb_<mode>.
gcc/testsuite/
* gcc.target/i386/avx512bmm-bitrev-novl-scalar.c: Test BMM zmm
fallback and runtime correctness.
* gcc.target/i386/avx512bmm-bitreverse-2.c: Test generic vectorizer
lowering to vbitrevb.
* gcc.target/i386/avx512bmm-gfni-bitreverse.c: Test BMM precedence
over GFNI.
* gcc.target/i386/gfni-bitreverse-scalar.c: Test the GFNI-only path.
Co-authored-by: Venkataramanan Kumar <[email protected]>
Signed-off-by: Sarvesh Chandra <[email protected]>
---
gcc/config/i386/i386-expand.cc | 44 +++++++++++++------
gcc/config/i386/i386-protos.h | 2 +-
gcc/config/i386/i386.md | 4 +-
gcc/config/i386/sse.md | 10 +++++
.../i386/avx512bmm-bitrev-novl-scalar.c | 18 ++++++++
.../gcc.target/i386/avx512bmm-bitreverse-2.c | 13 ++++++
.../i386/avx512bmm-gfni-bitreverse.c | 29 ++++++++++++
.../gcc.target/i386/gfni-bitreverse-scalar.c | 30 +++++++++++++
8 files changed, 134 insertions(+), 16 deletions(-)
create mode 100644 gcc/testsuite/gcc.target/i386/avx512bmm-bitrev-novl-scalar.c
create mode 100644 gcc/testsuite/gcc.target/i386/avx512bmm-bitreverse-2.c
create mode 100644 gcc/testsuite/gcc.target/i386/avx512bmm-gfni-bitreverse.c
create mode 100644 gcc/testsuite/gcc.target/i386/gfni-bitreverse-scalar.c
diff --git a/gcc/config/i386/i386-expand.cc b/gcc/config/i386/i386-expand.cc
index 7cc210f8e68..bec00478308 100644
--- a/gcc/config/i386/i386-expand.cc
+++ b/gcc/config/i386/i386-expand.cc
@@ -28151,10 +28151,10 @@ ix86_expand_vector_bf2sf_with_vec_perm (rtx dest, rtx
src)
emit_move_insn (dest, lowpart_subreg (GET_MODE (dest), target, vperm_mode));
}
-/* Implement bitreverse<mode>2 using gf2p8affineqb. */
+/* Implement bitreverse<mode>2 using vbitrevb or gf2p8affineqb. */
void
-ix86_expand_gfni_bitreverse (rtx dest, rtx src)
+ix86_expand_bitreverse (rtx dest, rtx src)
{
machine_mode mode = GET_MODE (dest);
rtx temp;
@@ -28201,18 +28201,36 @@ ix86_expand_gfni_bitreverse (rtx dest, rtx src)
}
src = temp;
temp = gen_reg_rtx (V16QImode);
- rtx src2 = gen_rtx_CONST_VECTOR (V16QImode,
- gen_rtvec (16, GEN_INT (1), GEN_INT (2),
- GEN_INT (4), GEN_INT (8),
- GEN_INT (16), GEN_INT (32),
- GEN_INT (64), GEN_INT (-128),
- GEN_INT (1), GEN_INT (2),
- GEN_INT (4), GEN_INT (8),
- GEN_INT (16), GEN_INT (32),
- GEN_INT (64), GEN_INT (-128)));
- src2 = validize_mem (force_const_mem (V16QImode, src2));
src = lowpart_subreg (V16QImode, src, GET_MODE (src));
- emit_insn (gen_vgf2p8affineqb_v16qi (temp, src, src2, const0_rtx));
+ if (TARGET_AVX512BMM)
+ {
+ if (TARGET_AVX512VL)
+ emit_insn (gen_avx512bmm_vbitrevb_v16qi (temp, src));
+ else
+ {
+ /* Without AVX512VL, use the zmm form. The upper bytes are
+ unspecified, but vbitrevb processes each byte independently. */
+ rtx srcz = lowpart_subreg (V64QImode, force_reg (V16QImode, src),
+ V16QImode);
+ rtx tmpz = gen_reg_rtx (V64QImode);
+ emit_insn (gen_avx512bmm_vbitrevb_v64qi (tmpz, srcz));
+ emit_move_insn (temp, lowpart_subreg (V16QImode, tmpz, V64QImode));
+ }
+ }
+ else if (TARGET_GFNI)
+ {
+ rtx src2 = gen_rtx_CONST_VECTOR (V16QImode,
+ gen_rtvec (16, GEN_INT (1), GEN_INT (2),
+ GEN_INT (4), GEN_INT (8),
+ GEN_INT (16), GEN_INT (32),
+ GEN_INT (64), GEN_INT (-128),
+ GEN_INT (1), GEN_INT (2),
+ GEN_INT (4), GEN_INT (8),
+ GEN_INT (16), GEN_INT (32),
+ GEN_INT (64), GEN_INT
(-128)));
+ src2 = validize_mem (force_const_mem (V16QImode, src2));
+ emit_insn (gen_vgf2p8affineqb_v16qi (temp, src, src2, const0_rtx));
+ }
if (mode == QImode)
{
rtx temp1 = gen_reg_rtx (SImode);
diff --git a/gcc/config/i386/i386-protos.h b/gcc/config/i386/i386-protos.h
index 106a79dd41e..99814eb5bc1 100644
--- a/gcc/config/i386/i386-protos.h
+++ b/gcc/config/i386/i386-protos.h
@@ -263,7 +263,7 @@ extern rtx ix86_expand_ternlog (machine_mode mode, rtx op0,
rtx op1, rtx op2,
int idx, rtx target);
extern void ix86_expand_vector_sf2bf_with_vec_perm (rtx, rtx);
extern void ix86_expand_vector_bf2sf_with_vec_perm (rtx, rtx);
-extern void ix86_expand_gfni_bitreverse (rtx, rtx);
+extern void ix86_expand_bitreverse (rtx, rtx);
#ifdef TREE_CODE
diff --git a/gcc/config/i386/i386.md b/gcc/config/i386/i386.md
index 063e14cb2c1..618813fe69f 100644
--- a/gcc/config/i386/i386.md
+++ b/gcc/config/i386/i386.md
@@ -22967,9 +22967,9 @@
(define_expand "bitreverse<mode>2"
[(set (match_operand:SWIDWI 0 "register_operand")
(bitreverse:SWIDWI (match_operand:SWIDWI 1 "nonimmediate_operand")))]
- "TARGET_GFNI && TARGET_SSE2"
+ "(TARGET_GFNI || TARGET_AVX512BMM) && TARGET_SSE2"
{
- ix86_expand_gfni_bitreverse (operands[0], operands[1]);
+ ix86_expand_bitreverse (operands[0], operands[1]);
DONE;
})
diff --git a/gcc/config/i386/sse.md b/gcc/config/i386/sse.md
index de092f4b9ae..6cb2a8ba36c 100644
--- a/gcc/config/i386/sse.md
+++ b/gcc/config/i386/sse.md
@@ -33668,3 +33668,13 @@
"vbitrevb\t{%1, %0|%0, %1}"
[(set_attr "prefix" "evex")
(set_attr "mode" "<sseinsnmode>")])
+
+(define_expand "bitreverse<mode>2"
+ [(set (match_operand:VI1_AVX512VL 0 "register_operand")
+ (bitreverse:VI1_AVX512VL
+ (match_operand:VI1_AVX512VL 1 "nonimmediate_operand")))]
+ "TARGET_AVX512BMM"
+{
+ emit_insn (gen_avx512bmm_vbitrevb_<mode> (operands[0], operands[1]));
+ DONE;
+})
diff --git a/gcc/testsuite/gcc.target/i386/avx512bmm-bitrev-novl-scalar.c
b/gcc/testsuite/gcc.target/i386/avx512bmm-bitrev-novl-scalar.c
new file mode 100644
index 00000000000..dfd9412a501
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/avx512bmm-bitrev-novl-scalar.c
@@ -0,0 +1,18 @@
+/* { dg-do run { target avx512bmm } } */
+/* { dg-options "-mavx512bmm -mno-avx512vl -mno-gfni -O2" } */
+/* { dg-final { scan-assembler-times "vbitrevb.*%zmm" 4 } } */
+/* { dg-final { scan-assembler-not "gf2p8affineqb" } } */
+/* { dg-final { scan-assembler-not "vgf2p8affineqb" } } */
+
+/* BMM without VL must use the zmm form for scalar bitreverse. */
+
+#define main do_test
+#include "../../gcc.dg/builtin-bitreverse-1.c"
+#undef main
+
+int
+main ()
+{
+ do_test ();
+ return 0;
+}
diff --git a/gcc/testsuite/gcc.target/i386/avx512bmm-bitreverse-2.c
b/gcc/testsuite/gcc.target/i386/avx512bmm-bitreverse-2.c
new file mode 100644
index 00000000000..9c82671107d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/avx512bmm-bitreverse-2.c
@@ -0,0 +1,13 @@
+/* { dg-do compile } */
+/* { dg-options "-mavx512bmm -mavx512vl -O3" } */
+/* { dg-final { scan-assembler "vbitrevb" } } */
+/* The vectorizer should use vbitrevb for a vector bitreverse rather than
+ scalarizing it. */
+
+void
+rev (unsigned char *restrict d, const unsigned char *restrict s, int n)
+{
+ for (int i = 0; i < n; i++)
+ d[i] = __builtin_bitreverse8 (s[i]);
+}
+
diff --git a/gcc/testsuite/gcc.target/i386/avx512bmm-gfni-bitreverse.c
b/gcc/testsuite/gcc.target/i386/avx512bmm-gfni-bitreverse.c
new file mode 100644
index 00000000000..e7610a5cc80
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/avx512bmm-gfni-bitreverse.c
@@ -0,0 +1,29 @@
+/* { dg-do compile } */
+/* { dg-options "-mavx512bmm -mno-avx512vl -mgfni -O2" } */
+/* { dg-final { scan-assembler-times "vbitrevb.*%zmm" 4 } } */
+/* { dg-final { scan-assembler-not "gf2p8affineqb" } } */
+
+/* BMM is preferred over GFNI; without VL it uses the zmm form. */
+unsigned char
+br8 (unsigned char x)
+{
+ return __builtin_bitreverse8 (x);
+}
+
+unsigned short
+br16 (unsigned short x)
+{
+ return __builtin_bitreverse16 (x);
+}
+
+unsigned int
+br32 (unsigned int x)
+{
+ return __builtin_bitreverse32 (x);
+}
+
+unsigned long long
+br64 (unsigned long long x)
+{
+ return __builtin_bitreverse64 (x);
+}
diff --git a/gcc/testsuite/gcc.target/i386/gfni-bitreverse-scalar.c
b/gcc/testsuite/gcc.target/i386/gfni-bitreverse-scalar.c
new file mode 100644
index 00000000000..94c032f27b7
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/gfni-bitreverse-scalar.c
@@ -0,0 +1,30 @@
+/* { dg-do compile } */
+/* { dg-options "-mgfni -mno-avx512bmm -O2" } */
+/* { dg-final { scan-assembler-times "gf2p8affineqb" 4 } } */
+/* { dg-final { scan-assembler-not "vbitrevb" } } */
+/* { dg-final { scan-assembler-not "vgf2p8affineqb" } } */
+
+/* GFNI-only scalar bitreverse must retain the existing legacy GFNI path. */
+unsigned char
+br8 (unsigned char x)
+{
+ return __builtin_bitreverse8 (x);
+}
+
+unsigned short
+br16 (unsigned short x)
+{
+ return __builtin_bitreverse16 (x);
+}
+
+unsigned int
+br32 (unsigned int x)
+{
+ return __builtin_bitreverse32 (x);
+}
+
+unsigned long long
+br64 (unsigned long long x)
+{
+ return __builtin_bitreverse64 (x);
+}
--
2.34.1
