https://gcc.gnu.org/g:63ce140b32df8a12b4a707634b6b652576adfeb2
commit r17-5475-g63ce140b32df8a12b4a707634b6b652576adfeb2 Author: Shreesh Adiga <[email protected]> Date: Wed Oct 21 15:50:56 2026 +0530 i386: Extend pclmul expander for efficient CRC64 computation This patch extends pclmul expanders for the i386 backend to generate optimized code for CRC64 computations. This is an extension to the 16-bit and 32-bit data CRC computation added earlier. The algorithm is slightly different which is described in the paper "Fast CRC Computation for Generic Polynomials Using PCLMULQDQ Instruction" described in the section "Barrett Reduction Algorithm for a degree-64 polynomial modulus (polynomials defined over GF(2))" Page 17 and 18. Bit-reverse CRC also is based on the above, except that the bit 65 after performing bit-reflect of polynomial maybe 0 and thus the xor operation needs to be conditionally performed only when LSB=1. The quotient computation which needs 65 bits now returns the lower 64 bits and the 65th bit is assumed to be 1 since the polynomials are expected to have the leading bit to be 1. Thus the assert is modified to allow the 64 bit cases as well. Added tests for covering 8-bit, 16-bit, 32-bit and 64-bit for both forward and reverse CRC64, along with reverse CRC64 testing for polynomials with LSB=0 and LSB=1 as well that gets passed into the CRC expander. gcc/ChangeLog: * config/i386/i386-expand.cc (ix86_expand_crc_using_pclmul): Modified to allow 64-bit CRC. (ix86_expand_reversed_crc_using_pclmul): Likewise. * config/i386/i386.md (crc<SWI248x:mode><SWI24:mode>4): Renamed the second iterator to allow 64-bit CRC. (crc<SWI1248x:mode><SWI248x:mode>4): Likewise. (crc_rev<SWI1248x:mode><SWI24:mode>4): Likewise. (crc_rev<SWI1248x:mode><SWI248x:mode>4): Likewise. * expr.cc (gf2n_poly_long_div_quotient): Relaxed assert to allow computing lower 64-bit of quotient for 64-bit CRC. gcc/testsuite/ChangeLog: * gcc.dg/torture/crc-34.c: New test. * gcc.dg/torture/crc-35.c: New test. * gcc.dg/torture/crc-36.c: New test. * gcc.dg/torture/crc-37.c: New test. * gcc.dg/torture/crc-38.c: New test. * gcc.dg/torture/crc-39.c: New test. * gcc.dg/torture/crc-40.c: New test. * gcc.dg/torture/crc-41.c: New test. * gcc.target/i386/crc-34-pclmul.c: New test. * gcc.target/i386/crc-35-pclmul.c: New test. * gcc.target/i386/crc-36-pclmul.c: New test. * gcc.target/i386/crc-37-pclmul.c: New test. * gcc.target/i386/crc-38-pclmul.c: New test. * gcc.target/i386/crc-39-pclmul.c: New test. * gcc.target/i386/crc-40-pclmul.c: New test. * gcc.target/i386/crc-41-pclmul.c: New test. Signed-off-by: Shreesh Adiga <[email protected]> Diff: --- gcc/config/i386/i386-expand.cc | 123 ++++++++++++++++++-------- gcc/config/i386/i386.md | 38 ++++---- gcc/expr.cc | 6 +- gcc/testsuite/gcc.dg/torture/crc-34.c | 56 ++++++++++++ gcc/testsuite/gcc.dg/torture/crc-35.c | 56 ++++++++++++ gcc/testsuite/gcc.dg/torture/crc-36.c | 65 ++++++++++++++ gcc/testsuite/gcc.dg/torture/crc-37.c | 65 ++++++++++++++ gcc/testsuite/gcc.dg/torture/crc-38.c | 89 +++++++++++++++++++ gcc/testsuite/gcc.dg/torture/crc-39.c | 89 +++++++++++++++++++ gcc/testsuite/gcc.dg/torture/crc-40.c | 98 ++++++++++++++++++++ gcc/testsuite/gcc.dg/torture/crc-41.c | 98 ++++++++++++++++++++ gcc/testsuite/gcc.target/i386/crc-34-pclmul.c | 9 ++ gcc/testsuite/gcc.target/i386/crc-35-pclmul.c | 8 ++ gcc/testsuite/gcc.target/i386/crc-36-pclmul.c | 8 ++ gcc/testsuite/gcc.target/i386/crc-37-pclmul.c | 8 ++ gcc/testsuite/gcc.target/i386/crc-38-pclmul.c | 9 ++ gcc/testsuite/gcc.target/i386/crc-39-pclmul.c | 8 ++ gcc/testsuite/gcc.target/i386/crc-40-pclmul.c | 8 ++ gcc/testsuite/gcc.target/i386/crc-41-pclmul.c | 8 ++ 19 files changed, 794 insertions(+), 55 deletions(-) diff --git a/gcc/config/i386/i386-expand.cc b/gcc/config/i386/i386-expand.cc index 5d3e038adc6e..8a6f0280ac6a 100644 --- a/gcc/config/i386/i386-expand.cc +++ b/gcc/config/i386/i386-expand.cc @@ -30823,15 +30823,17 @@ ix86_expand_crc_using_pclmul (scalar_mode crc_mode, unsigned HOST_WIDE_INT crc_size = GET_MODE_BITSIZE (crc_mode); unsigned HOST_WIDE_INT data_size = GET_MODE_BITSIZE (data_mode); - gcc_assert (crc_size <= 32); + gcc_assert (crc_size <= 64); gcc_assert (data_size <= crc_size); + machine_mode op_mode = crc_size == 64 ? DImode : SImode; + /* Calculate the quotient. */ unsigned HOST_WIDE_INT q = gf2n_poly_long_div_quotient (UINTVAL (polynomial), crc_size); if (crc_size > data_size) - crc = expand_shift (RSHIFT_EXPR, SImode, crc, crc_size - data_size, + crc = expand_shift (RSHIFT_EXPR, op_mode, crc, crc_size - data_size, NULL_RTX, 1); /* Keep the quotient and polynomial in V2DImode. */ @@ -30843,29 +30845,53 @@ ix86_expand_crc_using_pclmul (scalar_mode crc_mode, emit_move_insn (constants_reg, constants_v2di); /* crc ^ data and keep in V2DImode. */ - rtx cd_xor = expand_binop (SImode, xor_optab, crc, data, NULL_RTX, 1, - OPTAB_WIDEN); - rtx res = gen_reg_rtx (V4SImode); - emit_insn (gen_vec_setv4si_0 (res, CONST0_RTX (V4SImode), cd_xor)); - res = gen_lowpart (V2DImode, res); + rtx cd_xor = expand_binop (op_mode, xor_optab, crc, data, + NULL_RTX, 1, OPTAB_WIDEN); + rtx crc_temp; + if (op_mode == DImode) + { + crc_temp = gen_reg_rtx (V2DImode); + emit_insn (gen_vec_concatv2di (crc_temp, cd_xor, cd_xor)); + } + else + { + crc_temp = gen_reg_rtx (V4SImode); + emit_insn (gen_vec_setv4si_0 (crc_temp, CONST0_RTX (V4SImode), cd_xor)); + crc_temp = gen_lowpart (V2DImode, crc_temp); + } + /* Perform carry-less multiplication with quotient. */ - emit_insn (gen_pclmulqdq (res, res, constants_reg, const0_rtx)); + rtx res = gen_reg_rtx (V2DImode); + emit_insn (gen_pclmulqdq (res, crc_temp, constants_reg, const0_rtx)); - res = expand_shift (RSHIFT_EXPR, V2DImode, res, crc_size, NULL_RTX, 1); + if (op_mode == DImode) + { + /* Perform res ^ crc_temp to handle 65th bit of quotient. */ + res = expand_binop (V2DImode, xor_optab, res, crc_temp, + NULL_RTX, 1, OPTAB_DIRECT); + } + else + res = expand_shift (RSHIFT_EXPR, V2DImode, res, crc_size, NULL_RTX, 1); /* Perform carry-less multiplication with polynomial and get low part. */ emit_insn (gen_pclmulqdq (res, res, constants_reg, - gen_int_mode (0x10, SImode))); - rtx crc_part = gen_reg_rtx (SImode); - res = gen_lowpart (V4SImode, res); - emit_insn (gen_vec_extractv4sisi (crc_part, res, const0_rtx)); + GEN_INT (op_mode == DImode ? 0x11 : 0x10))); + + rtx crc_part = gen_reg_rtx (op_mode); + if (op_mode == DImode) + emit_insn (gen_vec_extractv2didi (crc_part, res, const0_rtx)); + else + { + res = gen_lowpart (V4SImode, res); + emit_insn (gen_vec_extractv4sisi (crc_part, res, const0_rtx)); + } if (crc_size > data_size) { - rtx shift = expand_shift (LSHIFT_EXPR, SImode, operands[1], data_size, + rtx shift = expand_shift (LSHIFT_EXPR, op_mode, operands[1], data_size, NULL_RTX, 1); - crc_part = expand_binop (SImode, xor_optab, crc_part, shift, NULL_RTX, 1, - OPTAB_DIRECT); + crc_part = expand_binop (op_mode, xor_optab, crc_part, shift, + NULL_RTX, 1, OPTAB_DIRECT); } emit_move_insn (operands[0], gen_lowpart (crc_mode, crc_part)); } @@ -30889,14 +30915,16 @@ ix86_expand_reversed_crc_using_pclmul (scalar_mode crc_mode, unsigned HOST_WIDE_INT crc_size = GET_MODE_BITSIZE (crc_mode); unsigned HOST_WIDE_INT data_size = GET_MODE_BITSIZE (data_mode); - gcc_assert (crc_size <= 32); + gcc_assert (crc_size <= 64); gcc_assert (data_size <= crc_size); + machine_mode op_mode = crc_size == 64 ? DImode : SImode; + /* Calculate the quotient. */ unsigned HOST_WIDE_INT q = gf2n_poly_long_div_quotient (UINTVAL (polynomial), crc_size); /* Reflect the calculated quotient. */ - q = reflect_hwi (q, crc_size + 1); + q = (reflect_hwi (q, crc_size) << 1) + 1; unsigned HOST_WIDE_INT reflected_op3 = reflect_hwi (UINTVAL (polynomial), crc_size); @@ -30909,30 +30937,55 @@ ix86_expand_reversed_crc_using_pclmul (scalar_mode crc_mode, emit_move_insn (constants_reg, constants_v2di); /* crc ^ data and keep in V2DImode. */ - rtx cd_xor = expand_binop (SImode, xor_optab, crc, data, NULL_RTX, 1, - OPTAB_WIDEN); + rtx cd_xor = expand_binop (op_mode, xor_optab, crc, data, + NULL_RTX, 1, OPTAB_WIDEN); + rtx crc_temp; + if (op_mode == DImode) + { + crc_temp = gen_reg_rtx (V2DImode); + emit_insn (gen_vec_concatv2di (crc_temp, cd_xor, const0_rtx)); + } + else + { + crc_temp = gen_reg_rtx (V4SImode); + emit_insn (gen_vec_setv4si_0 (crc_temp, CONST0_RTX (V4SImode), cd_xor)); + crc_temp = gen_lowpart (V2DImode, crc_temp); + } /* Perform carry-less multiplication with quotient. */ - rtx res = gen_reg_rtx (V4SImode); - emit_insn (gen_vec_setv4si_0 (res, CONST0_RTX (V4SImode), cd_xor)); - res = gen_lowpart (V2DImode, res); - emit_insn (gen_pclmulqdq (res, res, constants_reg, const0_rtx)); - - res = expand_shift (LSHIFT_EXPR, V2DImode, res, 64 - data_size, NULL_RTX, 0); + emit_insn (gen_pclmulqdq (crc_temp, crc_temp, constants_reg, const0_rtx)); + if (data_size < 64) + crc_temp = expand_shift (LSHIFT_EXPR, V2DImode, crc_temp, 64 - data_size, + NULL_RTX, 0); + + rtx res = gen_reg_rtx (V2DImode); + /* Perform carry-less multiplication with polynomial. */ + emit_insn (gen_pclmulqdq (res, crc_temp, constants_reg, GEN_INT (0x10))); + if (op_mode == DImode && (UINTVAL (polynomial) & 1)) + { + /* Perform xor of res[127:64] with crc_temp[63:0]. + This is needed when the 65th bit of reflected polynomial is 1. */ + rtx tmp = gen_reg_rtx (V2DImode); + emit_insn (gen_vec_interleave_lowv2di (tmp, crc_temp, crc_temp)); + res = expand_binop (V2DImode, xor_optab, res, tmp, + NULL_RTX, 1, OPTAB_DIRECT); + } - /* Perform carry-less multiplication with polynomial and get high part. */ - emit_insn (gen_pclmulqdq (res, res, constants_reg, - gen_int_mode (0x10, SImode))); - rtx res_high = gen_reg_rtx (SImode); - res = gen_lowpart (V4SImode, res); - emit_insn (gen_vec_extractv4sisi (res_high, res, const2_rtx)); + rtx res_high = gen_reg_rtx (op_mode); + if (op_mode == DImode) + emit_insn (gen_vec_extractv2didi (res_high, res, const1_rtx)); + else + { + res = gen_lowpart (V4SImode, res); + emit_insn (gen_vec_extractv4sisi (res_high, res, const2_rtx)); + } if (crc_size > data_size) { - rtx shift = expand_shift (RSHIFT_EXPR, SImode, crc, data_size, + rtx shift = expand_shift (RSHIFT_EXPR, op_mode, crc, data_size, NULL_RTX, 1); - res_high = expand_binop (SImode, xor_optab, res_high, shift, NULL_RTX, 1, - OPTAB_DIRECT); + res_high = expand_binop (op_mode, xor_optab, res_high, shift, + NULL_RTX, 1, OPTAB_DIRECT); } emit_move_insn (operands[0], gen_lowpart (crc_mode, res_high)); } diff --git a/gcc/config/i386/i386.md b/gcc/config/i386/i386.md index cce99217f44d..6203b7cfbdea 100644 --- a/gcc/config/i386/i386.md +++ b/gcc/config/i386/i386.md @@ -30714,37 +30714,39 @@ (set_attr "mode" "DI")]) ;; Bit-forward CRC -(define_expand "crc<SWI248x:mode><SWI24:mode>4" - [(match_operand:SWI24 0 "register_operand") ;; return value (CRC) - (match_operand:SWI24 1 "register_operand") ;; initial CRC - (match_operand:SWI248x 2 "nonimmediate_operand") ;; data - (match_operand:SWI24 3 "const_int_operand")] ;; polynomial - "TARGET_PCLMUL && <SWI24:MODE>mode >= <SWI248x:MODE>mode" -{ - ix86_expand_crc_using_pclmul (<SWI24:MODE>mode, <SWI248x:MODE>mode, +(define_expand "crc<SWI1248x:mode><SWI248x:mode>4" + [(match_operand:SWI248x 0 "register_operand") ;; return value (CRC) + (match_operand:SWI248x 1 "register_operand") ;; initial CRC + (match_operand:SWI1248x 2 "nonimmediate_operand") ;; data + (match_operand:SWI248x 3 "const_int_operand")] ;; polynomial + "TARGET_PCLMUL && <SWI248x:MODE>mode >= <SWI1248x:MODE>mode + && <SWI1248x:MODE>mode != QImode" +{ + /* Generate pclmul expansion for 16-bit, 32-bit and 64-bit data. */ + ix86_expand_crc_using_pclmul (<SWI248x:MODE>mode, <SWI1248x:MODE>mode, operands); DONE; }) ;; Reversed CRC -(define_expand "crc_rev<SWI1248x:mode><SWI24:mode>4" - [(match_operand:SWI24 0 "register_operand") ;; return value (CRC) - (match_operand:SWI24 1 "register_operand") ;; initial CRC +(define_expand "crc_rev<SWI1248x:mode><SWI248x:mode>4" + [(match_operand:SWI248x 0 "register_operand") ;; return value (CRC) + (match_operand:SWI248x 1 "register_operand") ;; initial CRC (match_operand:SWI1248x 2 "nonimmediate_operand") ;; data - (match_operand:SWI24 3 "const_int_operand")] ;; polynomial - "(TARGET_PCLMUL && <SWI24:MODE>mode >= <SWI1248x:MODE>mode + (match_operand:SWI248x 3 "const_int_operand")] ;; polynomial + "(TARGET_PCLMUL && <SWI248x:MODE>mode >= <SWI1248x:MODE>mode && <SWI1248x:MODE>mode != QImode) - || (TARGET_CRC32 && <SWI24:MODE>mode == SImode)" + || (TARGET_CRC32 && <SWI248x:MODE>mode == SImode)" { /* If it is iSCSI polynomial (0x1EDC6F41), generate crc32 instruction. */ if (TARGET_CRC32 && INTVAL (operands[3]) == 0x1EDC6F41 - && <SWI24:MODE>mode == SImode) + && <SWI248x:MODE>mode == SImode) emit_insn (gen_sse4_2_crc32<SWI1248x:mode> (operands[0], operands[1], operands[2])); - /* Generate pclmul expansion for 16-bit and 32-bit data. */ - else if (TARGET_PCLMUL && <SWI24:MODE>mode >= <SWI1248x:MODE>mode + /* Generate pclmul expansion for 16-bit, 32-bit and 64-bit data. */ + else if (TARGET_PCLMUL && <SWI248x:MODE>mode >= <SWI1248x:MODE>mode && <SWI1248x:MODE>mode != QImode) - ix86_expand_reversed_crc_using_pclmul (<SWI24:MODE>mode, + ix86_expand_reversed_crc_using_pclmul (<SWI248x:MODE>mode, <SWI1248x:MODE>mode, operands); else expand_reversed_crc_table_based (operands[0], operands[1], operands[2], diff --git a/gcc/expr.cc b/gcc/expr.cc index 96e1bf6d19ae..c654387ab016 100644 --- a/gcc/expr.cc +++ b/gcc/expr.cc @@ -14587,8 +14587,10 @@ unsigned HOST_WIDE_INT gf2n_poly_long_div_quotient (unsigned HOST_WIDE_INT polynomial, unsigned short n) { - /* The result has degree N, so needs N + 1 bits. */ - gcc_assert (n < 64); + /* The result has degree N, so needs N + 1 bits. + For 64-bit CRC, compute the lower 64 bits of the division. + The 65th bit is 1 since the leading bit of polynomial is 1. */ + gcc_assert (n <= 64); /* Perform a division step for the x^2N coefficient. At this point the quotient and remainder have N implicit trailing zeros. */ diff --git a/gcc/testsuite/gcc.dg/torture/crc-34.c b/gcc/testsuite/gcc.dg/torture/crc-34.c new file mode 100644 index 000000000000..d25fcc516202 --- /dev/null +++ b/gcc/testsuite/gcc.dg/torture/crc-34.c @@ -0,0 +1,56 @@ +/* { dg-do run { target lp64 } } */ +/* { dg-options "-fdump-tree-crc-details -w" } */ +/* { dg-skip-if "" { *-*-* } { "-O0" "-O1" "-g" "-Os" "-flto" } } */ + +#include <stdint.h> +#include <stdlib.h> + +// CRC64_ECMA_182 +#define CRC64_POLY 0x42F0E1EBA9EA3693 + +// Test bit-forward CRC64 8-bit data +__attribute__ ((noinline,optimize(0))) +uint64_t _crc64_O0 (uint8_t data, uint64_t crc) { + int i; + for (i = 0; i < 8; i++) { + if (((crc & 0x8000000000000000) >> 56 ^ (data & 0x80))) + crc = (crc << 1) ^ CRC64_POLY; + else + crc = (crc << 1); + data <<= 1; + } + return crc; +} + +uint64_t _crc64 (uint8_t data, uint64_t crc) { + int i; + for (i = 0; i < 8; i++) { + if (((crc & 0x8000000000000000) >> 56 ^ (data & 0x80))) + crc = (crc << 1) ^ CRC64_POLY; + else + crc = (crc << 1); + data <<= 1; + } + return crc; +} + +int main () +{ + uint64_t crc = 0; + + for (uint16_t i = 0; i <= 0xff; i++) + { + uint8_t data = (uint8_t)i; + uint64_t res1 = _crc64_O0 (data, crc); + uint64_t res2 = _crc64 (data, crc); + crc = res2; + if (res1 != res2) + abort (); + } +} + +/* { dg-final { scan-tree-dump "function maybe contains CRC calculation." "crc" } } */ +/* { dg-final { scan-tree-dump "Loop iteration number is 7" "crc" } } */ +/* { dg-final { scan-tree-dump "Bit forward" "crc" } } */ +/* { dg-final { scan-tree-dump "calculates CRC!" "crc" } } */ +/* { dg-final { scan-tree-dump-times "Couldn't generate faster CRC code." 0 "crc" } } */ diff --git a/gcc/testsuite/gcc.dg/torture/crc-35.c b/gcc/testsuite/gcc.dg/torture/crc-35.c new file mode 100644 index 000000000000..40e7d4473955 --- /dev/null +++ b/gcc/testsuite/gcc.dg/torture/crc-35.c @@ -0,0 +1,56 @@ +/* { dg-do run { target lp64 } } */ +/* { dg-options "-fdump-tree-crc-details -w" } */ +/* { dg-skip-if "" { *-*-* } { "-O0" "-O1" "-g" "-Os" "-flto" } } */ + +#include <stdint.h> +#include <stdlib.h> + +// CRC64_ECMA_182 +#define CRC64_POLY 0x42F0E1EBA9EA3693 + +// Test bit-forward CRC64 16-bit data +__attribute__ ((noinline,optimize(0))) +uint64_t _crc64_O0 (uint16_t data, uint64_t crc) { + int i; + for (i = 0; i < 16; i++) { + if (((crc & 0x8000000000000000) >> 48 ^ (data & 0x8000))) + crc = (crc << 1) ^ CRC64_POLY; + else + crc = (crc << 1); + data <<= 1; + } + return crc; +} + +uint64_t _crc64 (uint16_t data, uint64_t crc) { + int i; + for (i = 0; i < 16; i++) { + if (((crc & 0x8000000000000000) >> 48 ^ (data & 0x8000))) + crc = (crc << 1) ^ CRC64_POLY; + else + crc = (crc << 1); + data <<= 1; + } + return crc; +} + +int main () +{ + uint64_t crc = 0; + + for (uint16_t i = 0; i <= 0xff; i++) + { + uint16_t data = (i << 8) | (~i); // cover 2-byte range + uint64_t res1 = _crc64_O0 (data, crc); + uint64_t res2 = _crc64 (data, crc); + crc = res2; + if (res1 != res2) + abort (); + } +} + +/* { dg-final { scan-tree-dump "function maybe contains CRC calculation." "crc" } } */ +/* { dg-final { scan-tree-dump "Loop iteration number is 15" "crc" } } */ +/* { dg-final { scan-tree-dump "Bit forward" "crc" } } */ +/* { dg-final { scan-tree-dump "calculates CRC!" "crc" } } */ +/* { dg-final { scan-tree-dump-times "Couldn't generate faster CRC code." 0 "crc" } } */ diff --git a/gcc/testsuite/gcc.dg/torture/crc-36.c b/gcc/testsuite/gcc.dg/torture/crc-36.c new file mode 100644 index 000000000000..b9b0c3fc7bdb --- /dev/null +++ b/gcc/testsuite/gcc.dg/torture/crc-36.c @@ -0,0 +1,65 @@ +/* { dg-do run { target lp64 } } */ +/* { dg-options "-fdump-tree-crc-details -w" } */ +/* { dg-skip-if "" { *-*-* } { "-O0" "-O1" "-g" "-Os" "-flto" } } */ + +#include <stdint.h> +#include <stdlib.h> + +// CRC64_ECMA_182 +#define CRC64_POLY 0x42F0E1EBA9EA3693 + +uint32_t expand8to32 (uint32_t x) { + x ^= x >> 16; + x *= 0x7feb352dU; + x ^= x >> 15; + x *= 0x846ca68bU; + x ^= x >> 16; + return x; +} + +// Test bit-forward CRC64 32-bit data +__attribute__ ((noinline,optimize(0))) +uint64_t _crc64_O0 (uint32_t data, uint64_t crc) { + int i; + for (i = 0; i < 32; i++) { + if (((crc & 0x8000000000000000) >> 32 ^ (data & 0x80000000))) + crc = (crc << 1) ^ CRC64_POLY; + else + crc = (crc << 1); + data <<= 1; + } + return crc; +} + +uint64_t _crc64 (uint32_t data, uint64_t crc) { + int i; + for (i = 0; i < 32; i++) { + if (((crc & 0x8000000000000000) >> 32 ^ (data & 0x80000000))) + crc = (crc << 1) ^ CRC64_POLY; + else + crc = (crc << 1); + data <<= 1; + } + return crc; +} + +int main () +{ + uint64_t crc = 0; + + for (uint32_t i = 0; i <= 0xff; i++) + { + uint32_t data = expand8to32 (i); + uint64_t res1 = _crc64_O0 (data, crc); + uint64_t res2 = _crc64 (data, crc); + crc = res2; + if (res1 != res2) + abort (); + } +} + +/* { dg-final { scan-tree-dump "function maybe contains CRC calculation." "crc" } } */ +/* { dg-final { scan-tree-dump "Loop iteration number is 31" "crc" } } */ +/* { dg-final { scan-tree-dump "Bit forward" "crc" } } */ +/* { dg-final { scan-tree-dump "calculates CRC!" "crc" } } */ +/* { dg-final { scan-tree-dump-times "Couldn't generate faster CRC code." 0 "crc" } } */ diff --git a/gcc/testsuite/gcc.dg/torture/crc-37.c b/gcc/testsuite/gcc.dg/torture/crc-37.c new file mode 100644 index 000000000000..2445730d4498 --- /dev/null +++ b/gcc/testsuite/gcc.dg/torture/crc-37.c @@ -0,0 +1,65 @@ +/* { dg-do run { target lp64 } } */ +/* { dg-options "-fdump-tree-crc-details -w" } */ +/* { dg-skip-if "" { *-*-* } { "-O0" "-O1" "-g" "-Os" "-flto" } } */ + +#include <stdint.h> +#include <stdlib.h> + +// CRC64_ECMA_182 +#define CRC64_POLY 0x42F0E1EBA9EA3693 + +uint64_t expand8to64 (uint64_t x) { + x ^= x >> 30; + x *= UINT64_C(0xbf58476d1ce4e5b9); + x ^= x >> 27; + x *= UINT64_C(0x94d049bb133111eb); + x ^= x >> 31; + return x; +} + +// Test bit-forward CRC64 64-bit data +__attribute__ ((noinline,optimize(0))) +uint64_t _crc64_O0 (uint64_t data, uint64_t crc) { + int i; + for (i = 0; i < 64; i++) { + if (((crc & 0x8000000000000000) ^ (data & 0x8000000000000000))) + crc = (crc << 1) ^ CRC64_POLY; + else + crc = (crc << 1); + data <<= 1; + } + return crc; +} + +uint64_t _crc64 (uint64_t data, uint64_t crc) { + int i; + for (i = 0; i < 64; i++) { + if (((crc & 0x8000000000000000) ^ (data & 0x8000000000000000))) + crc = (crc << 1) ^ CRC64_POLY; + else + crc = (crc << 1); + data <<= 1; + } + return crc; +} + +int main () +{ + uint64_t crc = 0; + + for (uint64_t i = 0; i <= 0xff; i++) + { + uint64_t data = expand8to64 (i); + uint64_t res1 = _crc64_O0 (data, crc); + uint64_t res2 = _crc64 (data, crc); + crc = res2; + if (res1 != res2) + abort (); + } +} + +/* { dg-final { scan-tree-dump "function maybe contains CRC calculation." "crc" } } */ +/* { dg-final { scan-tree-dump "Loop iteration number is 63" "crc" } } */ +/* { dg-final { scan-tree-dump "Bit forward" "crc" } } */ +/* { dg-final { scan-tree-dump "calculates CRC!" "crc" } } */ +/* { dg-final { scan-tree-dump-times "Couldn't generate faster CRC code." 0 "crc" } } */ diff --git a/gcc/testsuite/gcc.dg/torture/crc-38.c b/gcc/testsuite/gcc.dg/torture/crc-38.c new file mode 100644 index 000000000000..a37aa492a17f --- /dev/null +++ b/gcc/testsuite/gcc.dg/torture/crc-38.c @@ -0,0 +1,89 @@ +/* { dg-do run { target lp64 } } */ +/* { dg-options "-fdump-tree-crc-details -w" } */ +/* { dg-skip-if "" { *-*-* } { "-O0" "-O1" "-g" "-Os" "-flto" } } */ + +#include <stdint.h> +#include <stdlib.h> + +#define CRC64_POLY1 0x9A6C9329AC4BC9B5 +#define CRC64_POLY2 0x1A6C9329AC4BC9B4 + +// Test bit-reverse CRC64 8-bit data with Poly having LSB=1 +__attribute__ ((noinline,optimize(0))) +uint64_t _crc64_poly1_O0 (uint8_t data, uint64_t crc) { + int i; + for (i = 0; i < 8; i++) { + if (((crc & 0x1) ^ (data & 0x1))) + crc = (crc >> 1) ^ CRC64_POLY1; + else + crc = (crc >> 1); + data >>= 1; + } + return crc; +} + +uint64_t _crc64_poly1 (uint8_t data, uint64_t crc) { + int i; + for (i = 0; i < 8; i++) { + if (((crc & 0x1) ^ (data & 0x1))) + crc = (crc >> 1) ^ CRC64_POLY1; + else + crc = (crc >> 1); + data >>= 1; + } + return crc; +} + +// Test bit-reverse CRC64 8-bit data with Poly having LSB=0 +__attribute__ ((noinline,optimize(0))) +uint64_t _crc64_poly2_O0 (uint8_t data, uint64_t crc) { + int i; + for (i = 0; i < 8; i++) { + if (((crc & 0x1) ^ (data & 0x1))) + crc = (crc >> 1) ^ CRC64_POLY2; + else + crc = (crc >> 1); + data >>= 1; + } + return crc; +} + +uint64_t _crc64_poly2 (uint8_t data, uint64_t crc) { + int i; + for (i = 0; i < 8; i++) { + if (((crc & 0x1) ^ (data & 0x1))) + crc = (crc >> 1) ^ CRC64_POLY2; + else + crc = (crc >> 1); + data >>= 1; + } + return crc; +} + +int main () +{ + uint64_t crc1 = 0; + uint64_t crc2 = 0; + + for (uint16_t i = 0; i <= 0xff; i++) + { + uint8_t data = (uint8_t)i; + uint64_t res1 = _crc64_poly1_O0 (data, crc1); + uint64_t res2 = _crc64_poly1 (data, crc1); + crc1 = res2; + if (res1 != res2) + abort (); + + uint64_t res3 = _crc64_poly2_O0 (data, crc2); + uint64_t res4 = _crc64_poly2 (data, crc2); + crc2 = res4; + if (res3 != res4) + abort (); + } +} + +/* { dg-final { scan-tree-dump "function maybe contains CRC calculation." "crc" } } */ +/* { dg-final { scan-tree-dump "Loop iteration number is 7" "crc" } } */ +/* { dg-final { scan-tree-dump "Bit reverse" "crc" } } */ +/* { dg-final { scan-tree-dump "calculates CRC!" "crc" } } */ +/* { dg-final { scan-tree-dump-times "Couldn't generate faster CRC code." 0 "crc" } } */ diff --git a/gcc/testsuite/gcc.dg/torture/crc-39.c b/gcc/testsuite/gcc.dg/torture/crc-39.c new file mode 100644 index 000000000000..2bc5894009fa --- /dev/null +++ b/gcc/testsuite/gcc.dg/torture/crc-39.c @@ -0,0 +1,89 @@ +/* { dg-do run { target lp64 } } */ +/* { dg-options "-fdump-tree-crc-details -w" } */ +/* { dg-skip-if "" { *-*-* } { "-O0" "-O1" "-g" "-Os" "-flto" } } */ + +#include <stdint.h> +#include <stdlib.h> + +#define CRC64_POLY1 0x9A6C9329AC4BC9B5 +#define CRC64_POLY2 0x1A6C9329AC4BC9B4 + +// Test bit-reverse CRC64 16-bit data with Poly having LSB=1 +__attribute__ ((noinline,optimize(0))) +uint64_t _crc64_poly1_O0 (uint16_t data, uint64_t crc) { + int i; + for (i = 0; i < 16; i++) { + if (((crc & 0x1) ^ (data & 0x1))) + crc = (crc >> 1) ^ CRC64_POLY1; + else + crc = (crc >> 1); + data >>= 1; + } + return crc; +} + +uint64_t _crc64_poly1 (uint16_t data, uint64_t crc) { + int i; + for (i = 0; i < 16; i++) { + if (((crc & 0x1) ^ (data & 0x1))) + crc = (crc >> 1) ^ CRC64_POLY1; + else + crc = (crc >> 1); + data >>= 1; + } + return crc; +} + +// Test bit-reverse CRC64 16-bit data with Poly having LSB=0 +__attribute__ ((noinline,optimize(0))) +uint64_t _crc64_poly2_O0 (uint16_t data, uint64_t crc) { + int i; + for (i = 0; i < 16; i++) { + if (((crc & 0x1) ^ (data & 0x1))) + crc = (crc >> 1) ^ CRC64_POLY2; + else + crc = (crc >> 1); + data >>= 1; + } + return crc; +} + +uint64_t _crc64_poly2 (uint16_t data, uint64_t crc) { + int i; + for (i = 0; i < 16; i++) { + if (((crc & 0x1) ^ (data & 0x1))) + crc = (crc >> 1) ^ CRC64_POLY2; + else + crc = (crc >> 1); + data >>= 1; + } + return crc; +} + +int main () +{ + uint64_t crc1 = 0; + uint64_t crc2 = 0; + + for (uint16_t i = 0; i <= 0xff; i++) + { + uint16_t data = (i << 8) | (~i); + uint64_t res1 = _crc64_poly1_O0 (data, crc1); + uint64_t res2 = _crc64_poly1 (data, crc1); + crc1 = res2; + if (res1 != res2) + abort (); + + uint64_t res3 = _crc64_poly2_O0 (data, crc2); + uint64_t res4 = _crc64_poly2 (data, crc2); + crc2 = res4; + if (res3 != res4) + abort (); + } +} + +/* { dg-final { scan-tree-dump "function maybe contains CRC calculation." "crc" } } */ +/* { dg-final { scan-tree-dump "Loop iteration number is 15" "crc" } } */ +/* { dg-final { scan-tree-dump "Bit reverse" "crc" } } */ +/* { dg-final { scan-tree-dump "calculates CRC!" "crc" } } */ +/* { dg-final { scan-tree-dump-times "Couldn't generate faster CRC code." 0 "crc" } } */ diff --git a/gcc/testsuite/gcc.dg/torture/crc-40.c b/gcc/testsuite/gcc.dg/torture/crc-40.c new file mode 100644 index 000000000000..47556c91ab91 --- /dev/null +++ b/gcc/testsuite/gcc.dg/torture/crc-40.c @@ -0,0 +1,98 @@ +/* { dg-do run { target lp64 } } */ +/* { dg-options "-fdump-tree-crc-details -w" } */ +/* { dg-skip-if "" { *-*-* } { "-O0" "-O1" "-g" "-Os" "-flto" } } */ + +#include <stdint.h> +#include <stdlib.h> + +#define CRC64_POLY1 0x9A6C9329AC4BC9B5 +#define CRC64_POLY2 0x1A6C9329AC4BC9B4 + +uint32_t expand8to32 (uint32_t x) { + x ^= x >> 16; + x *= 0x7feb352dU; + x ^= x >> 15; + x *= 0x846ca68bU; + x ^= x >> 16; + return x; +} + +// Test bit-reverse CRC64 32-bit data with Poly having LSB=1 +__attribute__ ((noinline,optimize(0))) +uint64_t _crc64_poly1_O0 (uint32_t data, uint64_t crc) { + int i; + for (i = 0; i < 32; i++) { + if (((crc & 0x1) ^ (data & 0x1))) + crc = (crc >> 1) ^ CRC64_POLY1; + else + crc = (crc >> 1); + data >>= 1; + } + return crc; +} + +uint64_t _crc64_poly1 (uint32_t data, uint64_t crc) { + int i; + for (i = 0; i < 32; i++) { + if (((crc & 0x1) ^ (data & 0x1))) + crc = (crc >> 1) ^ CRC64_POLY1; + else + crc = (crc >> 1); + data >>= 1; + } + return crc; +} + +// Test bit-reverse CRC64 32-bit data with Poly having LSB=0 +__attribute__ ((noinline,optimize(0))) +uint64_t _crc64_poly2_O0 (uint32_t data, uint64_t crc) { + int i; + for (i = 0; i < 32; i++) { + if (((crc & 0x1) ^ (data & 0x1))) + crc = (crc >> 1) ^ CRC64_POLY2; + else + crc = (crc >> 1); + data >>= 1; + } + return crc; +} + +uint64_t _crc64_poly2 (uint32_t data, uint64_t crc) { + int i; + for (i = 0; i < 32; i++) { + if (((crc & 0x1) ^ (data & 0x1))) + crc = (crc >> 1) ^ CRC64_POLY2; + else + crc = (crc >> 1); + data >>= 1; + } + return crc; +} + +int main () +{ + uint64_t crc1 = 0; + uint64_t crc2 = 0; + + for (uint32_t i = 0; i <= 0xff; i++) + { + uint32_t data = expand8to32 (i); + uint64_t res1 = _crc64_poly1_O0 (data, crc1); + uint64_t res2 = _crc64_poly1 (data, crc1); + crc1 = res2; + if (res1 != res2) + abort (); + + uint64_t res3 = _crc64_poly2_O0 (data, crc2); + uint64_t res4 = _crc64_poly2 (data, crc2); + crc2 = res4; + if (res3 != res4) + abort (); + } +} + +/* { dg-final { scan-tree-dump "function maybe contains CRC calculation." "crc" } } */ +/* { dg-final { scan-tree-dump "Loop iteration number is 31" "crc" } } */ +/* { dg-final { scan-tree-dump "Bit reverse" "crc" } } */ +/* { dg-final { scan-tree-dump "calculates CRC!" "crc" } } */ +/* { dg-final { scan-tree-dump-times "Couldn't generate faster CRC code." 0 "crc" } } */ diff --git a/gcc/testsuite/gcc.dg/torture/crc-41.c b/gcc/testsuite/gcc.dg/torture/crc-41.c new file mode 100644 index 000000000000..6aaa71c41e57 --- /dev/null +++ b/gcc/testsuite/gcc.dg/torture/crc-41.c @@ -0,0 +1,98 @@ +/* { dg-do run { target lp64 } } */ +/* { dg-options "-fdump-tree-crc-details -w" } */ +/* { dg-skip-if "" { *-*-* } { "-O0" "-O1" "-g" "-Os" "-flto" } } */ + +#include <stdint.h> +#include <stdlib.h> + +#define CRC64_POLY1 0x9A6C9329AC4BC9B5 +#define CRC64_POLY2 0x1A6C9329AC4BC9B4 + +uint64_t expand8to64 (uint64_t x) { + x ^= x >> 30; + x *= UINT64_C(0xbf58476d1ce4e5b9); + x ^= x >> 27; + x *= UINT64_C(0x94d049bb133111eb); + x ^= x >> 31; + return x; +} + +// Test bit-reverse CRC64 64-bit data with Poly having LSB=1 +__attribute__ ((noinline,optimize(0))) +uint64_t _crc64_poly1_O0 (uint64_t data, uint64_t crc) { + int i; + for (i = 0; i < 64; i++) { + if (((crc & 0x1) ^ (data & 0x1))) + crc = (crc >> 1) ^ CRC64_POLY1; + else + crc = (crc >> 1); + data >>= 1; + } + return crc; +} + +uint64_t _crc64_poly1 (uint64_t data, uint64_t crc) { + int i; + for (i = 0; i < 64; i++) { + if (((crc & 0x1) ^ (data & 0x1))) + crc = (crc >> 1) ^ CRC64_POLY1; + else + crc = (crc >> 1); + data >>= 1; + } + return crc; +} + +// Test bit-reverse CRC64 64-bit data with Poly having LSB=0 +__attribute__ ((noinline,optimize(0))) +uint64_t _crc64_poly2_O0 (uint64_t data, uint64_t crc) { + int i; + for (i = 0; i < 64; i++) { + if (((crc & 0x1) ^ (data & 0x1))) + crc = (crc >> 1) ^ CRC64_POLY2; + else + crc = (crc >> 1); + data >>= 1; + } + return crc; +} + +uint64_t _crc64_poly2 (uint64_t data, uint64_t crc) { + int i; + for (i = 0; i < 64; i++) { + if (((crc & 0x1) ^ (data & 0x1))) + crc = (crc >> 1) ^ CRC64_POLY2; + else + crc = (crc >> 1); + data >>= 1; + } + return crc; +} + +int main () +{ + uint64_t crc1 = 0; + uint64_t crc2 = 0; + + for (uint64_t i = 0; i <= 0xff; i++) + { + uint64_t data = expand8to64 (i); + uint64_t res1 = _crc64_poly1_O0 (data, crc1); + uint64_t res2 = _crc64_poly1 (data, crc1); + crc1 = res2; + if (res1 != res2) + abort (); + + uint64_t res3 = _crc64_poly2_O0 (data, crc2); + uint64_t res4 = _crc64_poly2 (data, crc2); + crc2 = res4; + if (res3 != res4) + abort (); + } +} + +/* { dg-final { scan-tree-dump "function maybe contains CRC calculation." "crc" } } */ +/* { dg-final { scan-tree-dump "Loop iteration number is 63" "crc" } } */ +/* { dg-final { scan-tree-dump "Bit reverse" "crc" } } */ +/* { dg-final { scan-tree-dump "calculates CRC!" "crc" } } */ +/* { dg-final { scan-tree-dump-times "Couldn't generate faster CRC code." 0 "crc" } } */ diff --git a/gcc/testsuite/gcc.target/i386/crc-34-pclmul.c b/gcc/testsuite/gcc.target/i386/crc-34-pclmul.c new file mode 100644 index 000000000000..51f747ed11c4 --- /dev/null +++ b/gcc/testsuite/gcc.target/i386/crc-34-pclmul.c @@ -0,0 +1,9 @@ +/* { dg-do run { target { pclmul_runtime } } } */ +/* { dg-options "-mpclmul -O2 -fdump-rtl-dfinish -fdump-tree-crc" } */ + +#include "../../gcc.dg/torture/crc-34.c" + +/* { dg-final { scan-tree-dump "calculates CRC!" "crc"} } */ +/* { dg-final { scan-tree-dump-times "Couldn't generate faster CRC code." 0 "crc"} } */ +/* Check 8-bit data CRC doesn't expand with pclmul */ +/* { dg-final { scan-rtl-dump-not "pclmulqdq" "dfinish"} } */ diff --git a/gcc/testsuite/gcc.target/i386/crc-35-pclmul.c b/gcc/testsuite/gcc.target/i386/crc-35-pclmul.c new file mode 100644 index 000000000000..054928ccae42 --- /dev/null +++ b/gcc/testsuite/gcc.target/i386/crc-35-pclmul.c @@ -0,0 +1,8 @@ +/* { dg-do run { target { pclmul_runtime } } } */ +/* { dg-options "-mpclmul -O2 -fdump-rtl-dfinish -fdump-tree-crc" } */ + +#include "../../gcc.dg/torture/crc-35.c" + +/* { dg-final { scan-tree-dump "calculates CRC!" "crc"} } */ +/* { dg-final { scan-tree-dump-times "Couldn't generate faster CRC code." 0 "crc"} } */ +/* { dg-final { scan-rtl-dump "pclmulqdq" "dfinish"} } */ diff --git a/gcc/testsuite/gcc.target/i386/crc-36-pclmul.c b/gcc/testsuite/gcc.target/i386/crc-36-pclmul.c new file mode 100644 index 000000000000..8db16b4c09a2 --- /dev/null +++ b/gcc/testsuite/gcc.target/i386/crc-36-pclmul.c @@ -0,0 +1,8 @@ +/* { dg-do run { target { pclmul_runtime } } } */ +/* { dg-options "-mpclmul -O2 -fdump-rtl-dfinish -fdump-tree-crc" } */ + +#include "../../gcc.dg/torture/crc-36.c" + +/* { dg-final { scan-tree-dump "calculates CRC!" "crc"} } */ +/* { dg-final { scan-tree-dump-times "Couldn't generate faster CRC code." 0 "crc"} } */ +/* { dg-final { scan-rtl-dump "pclmulqdq" "dfinish"} } */ diff --git a/gcc/testsuite/gcc.target/i386/crc-37-pclmul.c b/gcc/testsuite/gcc.target/i386/crc-37-pclmul.c new file mode 100644 index 000000000000..8d8567732dfb --- /dev/null +++ b/gcc/testsuite/gcc.target/i386/crc-37-pclmul.c @@ -0,0 +1,8 @@ +/* { dg-do run { target { pclmul_runtime } } } */ +/* { dg-options "-mpclmul -O2 -fdump-rtl-dfinish -fdump-tree-crc" } */ + +#include "../../gcc.dg/torture/crc-37.c" + +/* { dg-final { scan-tree-dump "calculates CRC!" "crc"} } */ +/* { dg-final { scan-tree-dump-times "Couldn't generate faster CRC code." 0 "crc"} } */ +/* { dg-final { scan-rtl-dump "pclmulqdq" "dfinish"} } */ diff --git a/gcc/testsuite/gcc.target/i386/crc-38-pclmul.c b/gcc/testsuite/gcc.target/i386/crc-38-pclmul.c new file mode 100644 index 000000000000..55767ac1c71d --- /dev/null +++ b/gcc/testsuite/gcc.target/i386/crc-38-pclmul.c @@ -0,0 +1,9 @@ +/* { dg-do run { target { pclmul_runtime } } } */ +/* { dg-options "-mpclmul -O2 -fdump-rtl-dfinish -fdump-tree-crc" } */ + +#include "../../gcc.dg/torture/crc-38.c" + +/* { dg-final { scan-tree-dump "calculates CRC!" "crc"} } */ +/* { dg-final { scan-tree-dump-times "Couldn't generate faster CRC code." 0 "crc"} } */ +/* Check 8-bit data CRC doesn't expand with pclmul */ +/* { dg-final { scan-rtl-dump-not "pclmulqdq" "dfinish"} } */ diff --git a/gcc/testsuite/gcc.target/i386/crc-39-pclmul.c b/gcc/testsuite/gcc.target/i386/crc-39-pclmul.c new file mode 100644 index 000000000000..b26b93139aa9 --- /dev/null +++ b/gcc/testsuite/gcc.target/i386/crc-39-pclmul.c @@ -0,0 +1,8 @@ +/* { dg-do run { target { pclmul_runtime } } } */ +/* { dg-options "-mpclmul -O2 -fdump-rtl-dfinish -fdump-tree-crc" } */ + +#include "../../gcc.dg/torture/crc-39.c" + +/* { dg-final { scan-tree-dump "calculates CRC!" "crc"} } */ +/* { dg-final { scan-tree-dump-times "Couldn't generate faster CRC code." 0 "crc"} } */ +/* { dg-final { scan-rtl-dump "pclmulqdq" "dfinish"} } */ diff --git a/gcc/testsuite/gcc.target/i386/crc-40-pclmul.c b/gcc/testsuite/gcc.target/i386/crc-40-pclmul.c new file mode 100644 index 000000000000..3f48f84a1cf1 --- /dev/null +++ b/gcc/testsuite/gcc.target/i386/crc-40-pclmul.c @@ -0,0 +1,8 @@ +/* { dg-do run { target { pclmul_runtime } } } */ +/* { dg-options "-mpclmul -O2 -fdump-rtl-dfinish -fdump-tree-crc" } */ + +#include "../../gcc.dg/torture/crc-40.c" + +/* { dg-final { scan-tree-dump "calculates CRC!" "crc"} } */ +/* { dg-final { scan-tree-dump-times "Couldn't generate faster CRC code." 0 "crc"} } */ +/* { dg-final { scan-rtl-dump "pclmulqdq" "dfinish"} } */ diff --git a/gcc/testsuite/gcc.target/i386/crc-41-pclmul.c b/gcc/testsuite/gcc.target/i386/crc-41-pclmul.c new file mode 100644 index 000000000000..734b81bb4bb9 --- /dev/null +++ b/gcc/testsuite/gcc.target/i386/crc-41-pclmul.c @@ -0,0 +1,8 @@ +/* { dg-do run { target { pclmul_runtime } } } */ +/* { dg-options "-mpclmul -O2 -fdump-rtl-dfinish -fdump-tree-crc" } */ + +#include "../../gcc.dg/torture/crc-41.c" + +/* { dg-final { scan-tree-dump "calculates CRC!" "crc"} } */ +/* { dg-final { scan-tree-dump-times "Couldn't generate faster CRC code." 0 "crc"} } */ +/* { dg-final { scan-rtl-dump "pclmulqdq" "dfinish"} } */
