Michiel Derhaeg <[email protected]> writes: > From: Loeka Rogge <[email protected]> > > This removes the guards that disables the clmulr pattern added by > patch r17-4206-ge91f79891fc657: Use clmulr for reversed crc32 on rv32 > from being used on rv64. > > This changes the calculation of the reflected quotient (on the host) to > fit inside 64 bits. gf2n_poly_long_div_quotient calculates > x^128 / (x^64 + lower order terms) = x^64 + lower order terms. > Meaning the result is missing an implicit 1 in front of the msb. > > After reversing, this missing 1 should be after the lsb. This is > re-added with a shift and an or. This causes the msb to fall outside > the result stored in t0. > > The value in t0 is only used by the emitted clmul or vclmul > instructions. Clmul only emits the bottom 64 bits, representing x^63 and > lower order terms. This is unaffected by the missing bit that > represents x^64, because it would always fall outside the result. > > Regtested on rv32gc, rv32gc_zbc, rv64gc, and rv64gc_zbc. > > gcc/ChangeLog: > > * config/riscv/bitmanip.md: Remove 64 bit guard. > * config/riscv/riscv.cc (riscv_optab_supported_p): Likewise. > 64 bit guard. > (expand_reversed_crc_using_clmul): Adapt quotient calculation > for 65 bits > > gcc/testsuite/ChangeLog: > > * gcc.target/riscv/crc-builtin-zbc64.c: test whether clmulr is > applied for crc64 I noticed too late that we forgot to add a changelog for the new test.
> > Signed-off-by: Loeka Rogge <[email protected]> > --- > Changes since v1: > - Clarified the comment in crc_rev pattern > - expand_reversed_crc_using_clmul's assert now uses BITS_PER_WORD > - the change in expr.cc is removed, has been addressed in a previous commit > addressing the same problem for i386 > - Added a run test > > gcc/config/riscv/bitmanip.md | 5 +- > gcc/config/riscv/riscv.cc | 36 +++++++++---- > .../gcc.target/riscv/crc-builtin-zbc64.c | 23 ++++++++- > .../gcc.target/riscv/crc-rev-crc64-rv64-zbc.c | 51 +++++++++++++++++++ > 4 files changed, 103 insertions(+), 12 deletions(-) > create mode 100644 gcc/testsuite/gcc.target/riscv/crc-rev-crc64-rv64-zbc.c > > diff --git a/gcc/config/riscv/bitmanip.md b/gcc/config/riscv/bitmanip.md > index b35743d7f36..bb69a6974d0 100644 > --- a/gcc/config/riscv/bitmanip.md > +++ b/gcc/config/riscv/bitmanip.md > @@ -1375,10 +1375,11 @@ > (E.g. CRC64's quotient may need 65 bits, > we can't keep it in 64 bit variable.) > then use clmul instruction to implement the CRC. > - We can also use clmulr for CRC-32 on RV32, requiring ZBC. */ > + If we have the ZBC extension (ie, clmulr) 1 bit can be left > + implicit, allowing CRC64 using a 64 bit variable. */ > if (((TARGET_ZBKC || TARGET_ZBC || TARGET_ZVBC) > && <ANYI:MODE>mode < word_mode) > - || (!TARGET_64BIT && TARGET_ZBC && <ANYI:MODE>mode == word_mode)) > + || (TARGET_ZBC && <ANYI:MODE>mode == word_mode)) > expand_reversed_crc_using_clmul (<ANYI:MODE>mode, <ANYI1:MODE>mode, > operands); > else > diff --git a/gcc/config/riscv/riscv.cc b/gcc/config/riscv/riscv.cc > index 47b20b8cdb4..ec3a0342196 100644 > --- a/gcc/config/riscv/riscv.cc > +++ b/gcc/config/riscv/riscv.cc > @@ -13534,7 +13534,7 @@ riscv_optab_supported_p (int op, machine_mode, > machine_mode result_mode, > if (op == crc_rev_optab && opt_type != OPTIMIZE_FOR_SPEED) > return (((TARGET_ZBKC || TARGET_ZBC || TARGET_ZVBC) > && result_mode < word_mode) > - || (!TARGET_64BIT && TARGET_ZBC && result_mode == word_mode)); > + || (TARGET_ZBC && result_mode == word_mode)); > > return true; > } > @@ -15978,15 +15978,31 @@ expand_reversed_crc_using_clmul (scalar_mode > crc_mode, scalar_mode data_mode, > gcc_assert (!CONST_INT_P (operands[0])); > gcc_assert (CONST_INT_P (operands[3])); > unsigned short crc_size = GET_MODE_BITSIZE (crc_mode); > - gcc_assert (crc_size <= 32); > + gcc_assert (crc_size <= BITS_PER_WORD); > unsigned short data_size = GET_MODE_BITSIZE (data_mode); > rtx polynomial = operands[3]; > > - /* Calculate the quotient. */ > - unsigned HOST_WIDE_INT > - q = gf2n_poly_long_div_quotient (UINTVAL (polynomial), crc_size); > - /* Reflect the calculated quotient. */ > - q = reflect_hwi (q, crc_size + 1); > + unsigned HOST_WIDE_INT q; > + if (crc_size == 64) > + { > + /* Calculate the quotient. > + x^64 is left implicit at the msb to fit in 64 bits. */ > + q = gf2n_poly_long_div_quotient (UINTVAL (polynomial), crc_size); > + /* Reflect the calculated quotient. > + Implicit bit is now missing at the lsb. */ > + q = reflect_hwi (q, crc_size); > + /* Add the implicit bit back at the lsb. > + This is okay because the extra bit falls outside the 64 lsbs output > + by clmul. */ > + q = (q << 1) | 1; > + } > + else > + { > + /* Calculate the quotient. */ > + q = gf2n_poly_long_div_quotient (UINTVAL (polynomial), crc_size); > + /* Reflect the calculated quotient. */ > + q = reflect_hwi (q, crc_size + 1); > + } > rtx t0 = gen_reg_rtx (word_mode); > riscv_emit_move (t0, gen_int_mode (q, word_mode)); > > @@ -16025,8 +16041,10 @@ expand_reversed_crc_using_clmul (scalar_mode > crc_mode, scalar_mode data_mode, > > if (use_clmulr) > { > - gcc_assert (!TARGET_64BIT); > - emit_insn (gen_riscv_clmulr_si (a0, a0, t1)); > + if (TARGET_64BIT) > + emit_insn (gen_riscv_clmulr_di (a0, a0, t1)); > + else > + emit_insn (gen_riscv_clmulr_si (a0, a0, t1)); > } > else > { > diff --git a/gcc/testsuite/gcc.target/riscv/crc-builtin-zbc64.c > b/gcc/testsuite/gcc.target/riscv/crc-builtin-zbc64.c > index 1f46300b140..92740588941 100644 > --- a/gcc/testsuite/gcc.target/riscv/crc-builtin-zbc64.c > +++ b/gcc/testsuite/gcc.target/riscv/crc-builtin-zbc64.c > @@ -63,5 +63,26 @@ int32_t rev_crc32_data32 (int32_t x) > { > return __builtin_rev_crc32_data32 (x, 0x123546ff, 0x4002123); > } > -/* { dg-final { scan-assembler-times "clmul\t" 18 } } */ > + > +int64_t rev_crc64_data8 (int64_t x) > +{ > + return __builtin_rev_crc64_data8 (x, 0x32, 0x4002123); > +} > + > +int64_t rev_crc64_data16 (int64_t x) > +{ > + return __builtin_rev_crc64_data16 (x, 0x3232, 0x4002123); > +} > + > +int64_t rev_crc64_data32 (int64_t x) > +{ > + return __builtin_rev_crc64_data32 (x, 0x123546ff, 0x4002123); > +} > + > +int64_t rev_crc64_data64 (int64_t x) > +{ > + return __builtin_rev_crc64_data64 (x, 0x123546ff123546ff, 0x4002123); > +} > +/* { dg-final { scan-assembler-times "clmul\t" 22 } } */ > /* { dg-final { scan-assembler-times "clmulh" 6 } } */ > +/* { dg-final { scan-assembler-times "clmulr" 4 } } */ > diff --git a/gcc/testsuite/gcc.target/riscv/crc-rev-crc64-rv64-zbc.c > b/gcc/testsuite/gcc.target/riscv/crc-rev-crc64-rv64-zbc.c > new file mode 100644 > index 00000000000..0643ef3ac1c > --- /dev/null > +++ b/gcc/testsuite/gcc.target/riscv/crc-rev-crc64-rv64-zbc.c > @@ -0,0 +1,51 @@ > +/* { dg-do run { target { rv64 && riscv_zbc_ok } } } */ > +/* { dg-options "-march=rv64gc_zbc -O2 -save-temps" } */ > +/* { dg-skip-if "" { *-*-* } { "-O0" "-O1" "-Og" "-Os" "-Oz" } } */ > + > +#include <stdint-gcc.h> > + > +#define CRC64_POLY 0x42f0e1eba9ea3693ull > + > +__attribute__ ((target ("arch=rv64gc"), noinline)) > +int64_t > +rev_crc64_ref (int64_t crc, int64_t data) > +{ > + return __builtin_rev_crc64_data64 (crc, data, CRC64_POLY); > +} > + > +__attribute__ ((noinline)) > +int64_t > +rev_crc64_fast (int64_t crc, int64_t data) > +{ > + return __builtin_rev_crc64_data64 (crc, data, CRC64_POLY); > +} > + > +int > +main (void) > +{ > + int64_t crc = 0x0D800D800D800D80LL; > + > + for (int64_t i = 0; i < 0xff; i++) > + { > + int64_t ref = rev_crc64_ref (crc, i); > + int64_t fast = rev_crc64_fast (crc, i); > + if (ref != fast) > + __builtin_abort (); > + crc = ref; > + } > + > + for (int64_t i = 0; i < 0xff; i++) > + { > + int64_t j = (i << 40) | i; > + int64_t ref = rev_crc64_ref (crc, j); > + int64_t fast = rev_crc64_fast (crc, j); > + if (ref != fast) > + __builtin_abort (); > + crc = ref; > + } > + > + return 0; > +} > + > +/* rev_crc64_fast only: Zbc clmulr (rev_crc64_ref is rv64gc / table). */ > +/* { dg-final { scan-assembler-times "clmulr" 1 } } */ > -- > 2.55.0
