From: Loeka Rogge <[email protected]>
This removes the guards that disables the clmulr pattern added by
patch r17-4206-ge91f79891fc657: Use clmulr for reversed crc32 on rv32
from being used on rv64.
This changes the calculation of the reflected quotient (on the host) to
fit inside 64 bits. gf2n_poly_long_div_quotient calculates
x^128 / (x^64 + lower order terms) = x^64 + lower order terms.
Meaning the result is missing an implicit 1 in front of the msb.
After reversing, this missing 1 should be after the lsb. This is
re-added with a shift and an or. This causes the msb to fall outside
the result stored in t0.
The value in t0 is only used by the emitted clmul or vclmul
instructions. Clmul only emits the bottom 64 bits, representing x^63 and
lower order terms. This is unaffected by the missing bit that
represents x^64, because it would always fall outside the result.
Regtested on rv32gc, rv32gc_zbc, rv64gc, and rv64gc_zbc.
gcc/ChangeLog:
* config/riscv/bitmanip.md: Remove 64 bit guard.
* config/riscv/riscv.cc (riscv_optab_supported_p): Likewise.
64 bit guard.
(expand_reversed_crc_using_clmul): Adapt quotient calculation
for 65 bits
gcc/testsuite/ChangeLog:
* gcc.target/riscv/crc-builtin-zbc64.c: test whether clmulr is
applied for crc64
Signed-off-by: Loeka Rogge <[email protected]>
---
Changes since v1:
- Clarified the comment in crc_rev pattern
- expand_reversed_crc_using_clmul's assert now uses BITS_PER_WORD
- the change in expr.cc is removed, has been addressed in a previous commit
addressing the same problem for i386
- Added a run test
gcc/config/riscv/bitmanip.md | 5 +-
gcc/config/riscv/riscv.cc | 36 +++++++++----
.../gcc.target/riscv/crc-builtin-zbc64.c | 23 ++++++++-
.../gcc.target/riscv/crc-rev-crc64-rv64-zbc.c | 51 +++++++++++++++++++
4 files changed, 103 insertions(+), 12 deletions(-)
create mode 100644 gcc/testsuite/gcc.target/riscv/crc-rev-crc64-rv64-zbc.c
diff --git a/gcc/config/riscv/bitmanip.md b/gcc/config/riscv/bitmanip.md
index b35743d7f36..bb69a6974d0 100644
--- a/gcc/config/riscv/bitmanip.md
+++ b/gcc/config/riscv/bitmanip.md
@@ -1375,10 +1375,11 @@
(E.g. CRC64's quotient may need 65 bits,
we can't keep it in 64 bit variable.)
then use clmul instruction to implement the CRC.
- We can also use clmulr for CRC-32 on RV32, requiring ZBC. */
+ If we have the ZBC extension (ie, clmulr) 1 bit can be left
+ implicit, allowing CRC64 using a 64 bit variable. */
if (((TARGET_ZBKC || TARGET_ZBC || TARGET_ZVBC)
&& <ANYI:MODE>mode < word_mode)
- || (!TARGET_64BIT && TARGET_ZBC && <ANYI:MODE>mode == word_mode))
+ || (TARGET_ZBC && <ANYI:MODE>mode == word_mode))
expand_reversed_crc_using_clmul (<ANYI:MODE>mode, <ANYI1:MODE>mode,
operands);
else
diff --git a/gcc/config/riscv/riscv.cc b/gcc/config/riscv/riscv.cc
index 47b20b8cdb4..ec3a0342196 100644
--- a/gcc/config/riscv/riscv.cc
+++ b/gcc/config/riscv/riscv.cc
@@ -13534,7 +13534,7 @@ riscv_optab_supported_p (int op, machine_mode,
machine_mode result_mode,
if (op == crc_rev_optab && opt_type != OPTIMIZE_FOR_SPEED)
return (((TARGET_ZBKC || TARGET_ZBC || TARGET_ZVBC)
&& result_mode < word_mode)
- || (!TARGET_64BIT && TARGET_ZBC && result_mode == word_mode));
+ || (TARGET_ZBC && result_mode == word_mode));
return true;
}
@@ -15978,15 +15978,31 @@ expand_reversed_crc_using_clmul (scalar_mode
crc_mode, scalar_mode data_mode,
gcc_assert (!CONST_INT_P (operands[0]));
gcc_assert (CONST_INT_P (operands[3]));
unsigned short crc_size = GET_MODE_BITSIZE (crc_mode);
- gcc_assert (crc_size <= 32);
+ gcc_assert (crc_size <= BITS_PER_WORD);
unsigned short data_size = GET_MODE_BITSIZE (data_mode);
rtx polynomial = operands[3];
- /* Calculate the quotient. */
- unsigned HOST_WIDE_INT
- q = gf2n_poly_long_div_quotient (UINTVAL (polynomial), crc_size);
- /* Reflect the calculated quotient. */
- q = reflect_hwi (q, crc_size + 1);
+ unsigned HOST_WIDE_INT q;
+ if (crc_size == 64)
+ {
+ /* Calculate the quotient.
+ x^64 is left implicit at the msb to fit in 64 bits. */
+ q = gf2n_poly_long_div_quotient (UINTVAL (polynomial), crc_size);
+ /* Reflect the calculated quotient.
+ Implicit bit is now missing at the lsb. */
+ q = reflect_hwi (q, crc_size);
+ /* Add the implicit bit back at the lsb.
+ This is okay because the extra bit falls outside the 64 lsbs output
+ by clmul. */
+ q = (q << 1) | 1;
+ }
+ else
+ {
+ /* Calculate the quotient. */
+ q = gf2n_poly_long_div_quotient (UINTVAL (polynomial), crc_size);
+ /* Reflect the calculated quotient. */
+ q = reflect_hwi (q, crc_size + 1);
+ }
rtx t0 = gen_reg_rtx (word_mode);
riscv_emit_move (t0, gen_int_mode (q, word_mode));
@@ -16025,8 +16041,10 @@ expand_reversed_crc_using_clmul (scalar_mode crc_mode,
scalar_mode data_mode,
if (use_clmulr)
{
- gcc_assert (!TARGET_64BIT);
- emit_insn (gen_riscv_clmulr_si (a0, a0, t1));
+ if (TARGET_64BIT)
+ emit_insn (gen_riscv_clmulr_di (a0, a0, t1));
+ else
+ emit_insn (gen_riscv_clmulr_si (a0, a0, t1));
}
else
{
diff --git a/gcc/testsuite/gcc.target/riscv/crc-builtin-zbc64.c
b/gcc/testsuite/gcc.target/riscv/crc-builtin-zbc64.c
index 1f46300b140..92740588941 100644
--- a/gcc/testsuite/gcc.target/riscv/crc-builtin-zbc64.c
+++ b/gcc/testsuite/gcc.target/riscv/crc-builtin-zbc64.c
@@ -63,5 +63,26 @@ int32_t rev_crc32_data32 (int32_t x)
{
return __builtin_rev_crc32_data32 (x, 0x123546ff, 0x4002123);
}
-/* { dg-final { scan-assembler-times "clmul\t" 18 } } */
+
+int64_t rev_crc64_data8 (int64_t x)
+{
+ return __builtin_rev_crc64_data8 (x, 0x32, 0x4002123);
+}
+
+int64_t rev_crc64_data16 (int64_t x)
+{
+ return __builtin_rev_crc64_data16 (x, 0x3232, 0x4002123);
+}
+
+int64_t rev_crc64_data32 (int64_t x)
+{
+ return __builtin_rev_crc64_data32 (x, 0x123546ff, 0x4002123);
+}
+
+int64_t rev_crc64_data64 (int64_t x)
+{
+ return __builtin_rev_crc64_data64 (x, 0x123546ff123546ff, 0x4002123);
+}
+/* { dg-final { scan-assembler-times "clmul\t" 22 } } */
/* { dg-final { scan-assembler-times "clmulh" 6 } } */
+/* { dg-final { scan-assembler-times "clmulr" 4 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/crc-rev-crc64-rv64-zbc.c
b/gcc/testsuite/gcc.target/riscv/crc-rev-crc64-rv64-zbc.c
new file mode 100644
index 00000000000..0643ef3ac1c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/crc-rev-crc64-rv64-zbc.c
@@ -0,0 +1,51 @@
+/* { dg-do run { target { rv64 && riscv_zbc_ok } } } */
+/* { dg-options "-march=rv64gc_zbc -O2 -save-temps" } */
+/* { dg-skip-if "" { *-*-* } { "-O0" "-O1" "-Og" "-Os" "-Oz" } } */
+
+#include <stdint-gcc.h>
+
+#define CRC64_POLY 0x42f0e1eba9ea3693ull
+
+__attribute__ ((target ("arch=rv64gc"), noinline))
+int64_t
+rev_crc64_ref (int64_t crc, int64_t data)
+{
+ return __builtin_rev_crc64_data64 (crc, data, CRC64_POLY);
+}
+
+__attribute__ ((noinline))
+int64_t
+rev_crc64_fast (int64_t crc, int64_t data)
+{
+ return __builtin_rev_crc64_data64 (crc, data, CRC64_POLY);
+}
+
+int
+main (void)
+{
+ int64_t crc = 0x0D800D800D800D80LL;
+
+ for (int64_t i = 0; i < 0xff; i++)
+ {
+ int64_t ref = rev_crc64_ref (crc, i);
+ int64_t fast = rev_crc64_fast (crc, i);
+ if (ref != fast)
+ __builtin_abort ();
+ crc = ref;
+ }
+
+ for (int64_t i = 0; i < 0xff; i++)
+ {
+ int64_t j = (i << 40) | i;
+ int64_t ref = rev_crc64_ref (crc, j);
+ int64_t fast = rev_crc64_fast (crc, j);
+ if (ref != fast)
+ __builtin_abort ();
+ crc = ref;
+ }
+
+ return 0;
+}
+
+/* rev_crc64_fast only: Zbc clmulr (rev_crc64_ref is rv64gc / table). */
+/* { dg-final { scan-assembler-times "clmulr" 1 } } */
--
2.55.0
