Add support to vectorize __builtin_bitreverse* calls by mapping
BUILT_IN_BITREVERSE* to internal function IFN_BITREVERSE.
It also extends the AVX512BMM vector bitreverse expander to 16-, 32- and
64-bit elements: vbitrevb reverses the bits within each byte and a
constant byte permutation (vpshufb) then reverses the byte order within
each element.
gcc/ChangeLog:
* builtins.cc (associated_internal_fn): Map
BUILT_IN_BITREVERSE{8,16,32,64} to IFN_BITREVERSE.
* config/i386/i386-expand.cc (ix86_expand_vector_bitreverse):
New function.
* config/i386/i386-protos.h (ix86_expand_vector_bitreverse):
Declare.
* config/i386/sse.md (bitreverse<mode>2): New expander for
VI248_AVX512VL.
gcc/testsuite/ChangeLog:
* gcc.target/i386/avx512bmm-bitreverse-vect-1.c: New test.
* gcc.target/i386/avx512bmm-bitreverse-vect-2.c: New test.
---
Hi Maintainers,
This patch adds support to vectorize __builtin_bitreverse* calls when AVX512BMM
is available.
Ok for staging ??
regards,
venkat.
gcc/builtins.cc | 6 +++
gcc/config/i386/i386-expand.cc | 42 +++++++++++++++++++
gcc/config/i386/i386-protos.h | 1 +
gcc/config/i386/sse.md | 10 +++++
.../i386/avx512bmm-bitreverse-vect-1.c | 34 +++++++++++++++
.../i386/avx512bmm-bitreverse-vect-2.c | 8 ++++
6 files changed, 101 insertions(+)
create mode 100644 gcc/testsuite/gcc.target/i386/avx512bmm-bitreverse-vect-1.c
create mode 100644 gcc/testsuite/gcc.target/i386/avx512bmm-bitreverse-vect-2.c
diff --git a/gcc/builtins.cc b/gcc/builtins.cc
index 128a12a898f..f03f272ce61 100644
--- a/gcc/builtins.cc
+++ b/gcc/builtins.cc
@@ -2223,6 +2223,12 @@ associated_internal_fn (built_in_function fn, tree
return_type)
CASE_INT_FN (BUILT_IN_##NAME): return IFN_##NAME;
#include "internal-fn.def"
+ case BUILT_IN_BITREVERSE8:
+ case BUILT_IN_BITREVERSE16:
+ case BUILT_IN_BITREVERSE32:
+ case BUILT_IN_BITREVERSE64:
+ return IFN_BITREVERSE;
+
CASE_FLT_FN (BUILT_IN_POW10):
return IFN_EXP10;
diff --git a/gcc/config/i386/i386-expand.cc b/gcc/config/i386/i386-expand.cc
index 4980a04fdaf..852e5287d97 100644
--- a/gcc/config/i386/i386-expand.cc
+++ b/gcc/config/i386/i386-expand.cc
@@ -30688,6 +30688,48 @@ ix86_expand_bitreverse (rtx dest, rtx src)
emit_insn (gen_rtx_SET (dest, gen_rtx_BSWAP (mode, target)));
}
+/* Implement vector bitreverse<mode>2 for elements wider than a byte:
+ reverse the bits in each byte with vbitrevb, then reverse the order
+ of the bytes within each element. */
+
+void
+ix86_expand_vector_bitreverse (rtx dest, rtx src)
+{
+ machine_mode mode = GET_MODE (dest);
+ machine_mode qimode = mode_for_vector (QImode,
+ GET_MODE_SIZE (mode)).require ();
+ rtx temp = gen_reg_rtx (qimode);
+ src = lowpart_subreg (qimode, force_reg (mode, src), mode);
+ switch (qimode)
+ {
+ case E_V16QImode:
+ emit_insn (gen_avx512bmm_vbitrevb_v16qi (temp, src));
+ break;
+ case E_V32QImode:
+ emit_insn (gen_avx512bmm_vbitrevb_v32qi (temp, src));
+ break;
+ case E_V64QImode:
+ emit_insn (gen_avx512bmm_vbitrevb_v64qi (temp, src));
+ break;
+ default:
+ gcc_unreachable ();
+ }
+
+ int nelt = GET_MODE_NUNITS (qimode);
+ int esize = GET_MODE_UNIT_SIZE (mode);
+ vec_perm_builder sel (nelt, nelt, 1);
+ sel.quick_grow (nelt);
+ for (int i = 0; i != nelt; i++)
+ sel[i] = (i / esize + 1) * esize - i % esize - 1;
+ vec_perm_indices indices (sel, 1, nelt);
+
+ rtx target = gen_reg_rtx (qimode);
+ bool ok = targetm.vectorize.vec_perm_const (qimode, qimode, target,
+ temp, temp, indices);
+ gcc_assert (ok);
+ emit_move_insn (dest, lowpart_subreg (mode, target, qimode));
+}
+
/* Expand LCP stall or long immediate peephole for INSN. Use the
previous scratch register if possible. If USE_XOR is true,
generate "*movsi_xor". */
diff --git a/gcc/config/i386/i386-protos.h b/gcc/config/i386/i386-protos.h
index 8ac329a716a..2b2e46b4e0a 100644
--- a/gcc/config/i386/i386-protos.h
+++ b/gcc/config/i386/i386-protos.h
@@ -268,6 +268,7 @@ extern rtx ix86_expand_ternlog (machine_mode mode, rtx op0,
rtx op1, rtx op2,
extern void ix86_expand_vector_sf2bf_with_vec_perm (rtx, rtx);
extern void ix86_expand_vector_bf2sf_with_vec_perm (rtx, rtx);
extern void ix86_expand_bitreverse (rtx, rtx);
+extern void ix86_expand_vector_bitreverse (rtx, rtx);
#ifdef TREE_CODE
diff --git a/gcc/config/i386/sse.md b/gcc/config/i386/sse.md
index 3a445c9746d..2ecd282f2b9 100644
--- a/gcc/config/i386/sse.md
+++ b/gcc/config/i386/sse.md
@@ -34229,6 +34229,16 @@
DONE;
})
+(define_expand "bitreverse<mode>2"
+ [(set (match_operand:VI248_AVX512VL 0 "register_operand")
+ (bitreverse:VI248_AVX512VL
+ (match_operand:VI248_AVX512VL 1 "nonimmediate_operand")))]
+ "TARGET_AVX512BMM"
+{
+ ix86_expand_vector_bitreverse (operands[0], operands[1]);
+ DONE;
+})
+
;; AVX10V2AUX instructions
;; FP32 to FP8 single-source converts (VCVTPS2BF8, VCVTPS2BF8S,
;; VCVTPS2HF8, VCVTPS2HF8S, VCVTROPS2HF8, VCVTROPS2HF8S)
diff --git a/gcc/testsuite/gcc.target/i386/avx512bmm-bitreverse-vect-1.c
b/gcc/testsuite/gcc.target/i386/avx512bmm-bitreverse-vect-1.c
new file mode 100644
index 00000000000..13588bde13a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/avx512bmm-bitreverse-vect-1.c
@@ -0,0 +1,34 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -mavx512bmm -mavx512vl -mprefer-vector-width=512
-fdump-tree-vect-details" } */
+/* { dg-final { scan-tree-dump-times "optimized: loop vectorized" 4 "vect" } }
*/
+/* { dg-final { scan-assembler-times "vbitrevb\[^\n\r\]*%zmm" 4 } } */
+/* { dg-final { scan-assembler-times "vpshufb\[^\n\r\]*%zmm" 3 } } */
+
+void
+br8 (unsigned char *__restrict a, unsigned char *__restrict b, int n)
+{
+ for (int i = 0; i < n; i++)
+ a[i] = __builtin_bitreverse8 (b[i]);
+}
+
+void
+br16 (unsigned short *__restrict a, unsigned short *__restrict b, int n)
+{
+ for (int i = 0; i < n; i++)
+ a[i] = __builtin_bitreverse16 (b[i]);
+}
+
+void
+br32 (unsigned int *__restrict a, unsigned int *__restrict b, int n)
+{
+ for (int i = 0; i < n; i++)
+ a[i] = __builtin_bitreverse32 (b[i]);
+}
+
+void
+br64 (unsigned long long *__restrict a, unsigned long long *__restrict b,
+ int n)
+{
+ for (int i = 0; i < n; i++)
+ a[i] = __builtin_bitreverse64 (b[i]);
+}
diff --git a/gcc/testsuite/gcc.target/i386/avx512bmm-bitreverse-vect-2.c
b/gcc/testsuite/gcc.target/i386/avx512bmm-bitreverse-vect-2.c
new file mode 100644
index 00000000000..946f61f5c98
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/avx512bmm-bitreverse-vect-2.c
@@ -0,0 +1,8 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -mavx512bmm -mavx512vl -mprefer-vector-width=256
-fdump-tree-vect-details" } */
+/* { dg-final { scan-tree-dump-times "optimized: loop vectorized" 4 "vect" } }
*/
+/* { dg-final { scan-assembler-not "%zmm" } } */
+/* { dg-final { scan-assembler-times "vbitrevb\[^\n\r\]*%ymm" 4 } } */
+/* { dg-final { scan-assembler-times "vpshufb\[^\n\r\]*%ymm" 3 } } */
+
+#include "avx512bmm-bitreverse-vect-1.c"
--
2.34.1
