https://gcc.gnu.org/g:3c80dc4603e1ece9ff75b801e75943d8c9a19b05
commit r17-5584-g3c80dc4603e1ece9ff75b801e75943d8c9a19b05 Author: Shreesh Adiga <[email protected]> Date: Mon Oct 26 20:32:17 2026 +0530 i386: Use crc32q for reversed CRC64 with the CRC-32C polynomial A 64-bit CRC with polynomial 0x1EDC6F4100000000 is x^32 times the CRC-32C polynomial (0x1EDC6F41). The result of the reversed 64-bit update is therefore CRC-32C of crc ^ data in the low 32 bits, with the upper 32 bits zero. This is exactly what the crc32q instruction computes with a zero accumulator and crc ^ data as source, so __builtin_rev_crc64_data64 (crc, data, 0x1EDC6F4100000000) can be expanded to: crc32q (0, crc ^ data) instead of a sequence of pclmulqdq and movq instructions. This matches code that keeps a CRC-32C in a 64-bit variable and uses the reflected constant 0x82F63B78. The special case only applies to 64-bit targets with TARGET_CRC32, and is checked before the pclmul expansion. Also stop enabling the crc32 instruction for SImode CRC with DImode data, which was never valid for the iSCSI polynomial path. gcc/ChangeLog: * config/i386/i386.md (crc_rev<SWI1248x:mode><SWI248x:mode>4): Enable for 64-bit TARGET_CRC32 with DImode CRC and data. Expand to crc32q for the 0x1EDC6F4100000000 polynomial. Do not use the crc32 instruction for SImode CRC with DImode data. gcc/testsuite/ChangeLog: * gcc.target/i386/crc-crc32-data64-negative.c: New test. * gcc.target/i386/crc-crc32-data64.c: New test. Signed-off-by: Shreesh Adiga <[email protected]> Diff: --- gcc/config/i386/i386.md | 57 +++++++++++++--------- .../gcc.target/i386/crc-crc32-data64-negative.c | 54 ++++++++++++++++++++ gcc/testsuite/gcc.target/i386/crc-crc32-data64.c | 54 ++++++++++++++++++++ 3 files changed, 142 insertions(+), 23 deletions(-) diff --git a/gcc/config/i386/i386.md b/gcc/config/i386/i386.md index 6203b7cfbdea..ce47379c1746 100644 --- a/gcc/config/i386/i386.md +++ b/gcc/config/i386/i386.md @@ -30719,13 +30719,13 @@ (match_operand:SWI248x 1 "register_operand") ;; initial CRC (match_operand:SWI1248x 2 "nonimmediate_operand") ;; data (match_operand:SWI248x 3 "const_int_operand")] ;; polynomial - "TARGET_PCLMUL && <SWI248x:MODE>mode >= <SWI1248x:MODE>mode - && <SWI1248x:MODE>mode != QImode" + "<SWI1248x:MODE_SIZE> <= <SWI248x:MODE_SIZE> + && TARGET_PCLMUL && <SWI1248x:MODE>mode != QImode" { - /* Generate pclmul expansion for 16-bit, 32-bit and 64-bit data. */ - ix86_expand_crc_using_pclmul (<SWI248x:MODE>mode, <SWI1248x:MODE>mode, - operands); - DONE; + /* Generate pclmul expansion for 16-bit, 32-bit and 64-bit data. */ + ix86_expand_crc_using_pclmul (<SWI248x:MODE>mode, <SWI1248x:MODE>mode, + operands); + DONE; }) ;; Reversed CRC @@ -30734,23 +30734,34 @@ (match_operand:SWI248x 1 "register_operand") ;; initial CRC (match_operand:SWI1248x 2 "nonimmediate_operand") ;; data (match_operand:SWI248x 3 "const_int_operand")] ;; polynomial - "(TARGET_PCLMUL && <SWI248x:MODE>mode >= <SWI1248x:MODE>mode - && <SWI1248x:MODE>mode != QImode) - || (TARGET_CRC32 && <SWI248x:MODE>mode == SImode)" - { - /* If it is iSCSI polynomial (0x1EDC6F41), generate crc32 instruction. */ - if (TARGET_CRC32 && INTVAL (operands[3]) == 0x1EDC6F41 - && <SWI248x:MODE>mode == SImode) - emit_insn (gen_sse4_2_crc32<SWI1248x:mode> (operands[0], operands[1], - operands[2])); - /* Generate pclmul expansion for 16-bit, 32-bit and 64-bit data. */ - else if (TARGET_PCLMUL && <SWI248x:MODE>mode >= <SWI1248x:MODE>mode - && <SWI1248x:MODE>mode != QImode) - ix86_expand_reversed_crc_using_pclmul (<SWI248x:MODE>mode, - <SWI1248x:MODE>mode, operands); - else - expand_reversed_crc_table_based (operands[0], operands[1], operands[2], - operands[3], <SWI1248x:MODE>mode); + "<SWI1248x:MODE_SIZE> <= <SWI248x:MODE_SIZE> + && ((TARGET_PCLMUL && <SWI1248x:MODE>mode != QImode) + || (TARGET_CRC32 + && <SWI248x:MODE_SIZE> >= GET_MODE_SIZE (SImode) + && <SWI248x:MODE_SIZE> <= UNITS_PER_WORD))" +{ + if (TARGET_CRC32 && <SWI248x:MODE>mode == SImode + && INTVAL (operands[3]) == 0x1EDC6F41) + emit_insn (gen_sse4_2_crc32<SWI1248x:mode> (operands[0], operands[1], + operands[2])); + /* 0x1EDC6F4100000000 is x^32 times the iSCSI polynomial, so the result + is CRC-32C of crc ^ data in the low 32 bits, with upper bits zero. */ + else if (TARGET_64BIT && TARGET_CRC32 && <SWI248x:MODE>mode == DImode + && <SWI1248x:MODE>mode == DImode + && INTVAL (operands[3]) == HOST_WIDE_INT_C (0x1EDC6F4100000000)) + { + rtx cd_xor = expand_binop (DImode, xor_optab, operands[1], operands[2], + NULL_RTX, 1, OPTAB_DIRECT); + rtx zero = force_reg (SImode, const0_rtx); + emit_insn (gen_sse4_2_crc32di (operands[0], zero, cd_xor)); + } + /* Generate pclmul expansion for 16-bit, 32-bit and 64-bit data. */ + else if (TARGET_PCLMUL && <SWI1248x:MODE>mode != QImode) + ix86_expand_reversed_crc_using_pclmul (<SWI248x:MODE>mode, + <SWI1248x:MODE>mode, operands); + else + expand_reversed_crc_table_based (operands[0], operands[1], operands[2], + operands[3], <SWI1248x:MODE>mode); DONE; }) diff --git a/gcc/testsuite/gcc.target/i386/crc-crc32-data64-negative.c b/gcc/testsuite/gcc.target/i386/crc-crc32-data64-negative.c new file mode 100644 index 000000000000..07200dc89917 --- /dev/null +++ b/gcc/testsuite/gcc.target/i386/crc-crc32-data64-negative.c @@ -0,0 +1,54 @@ +/* { dg-do run { target { { sse4_runtime && pclmul_runtime } && { ! ia32 } } } } */ +/* { dg-options "-msse4.2 -mpclmul -O2 -save-temps -fdump-rtl-dfinish -fdump-tree-crc" } */ +/* { dg-skip-if "" { *-*-* } { "-flto"} } */ + +#include <stdint.h> +#include <stdlib.h> + +#define POLY 0x1EDC6F4100000001ULL + +uint64_t expand8to64 (uint64_t x) { + x ^= x >> 30; + x *= UINT64_C(0xbf58476d1ce4e5b9); + x ^= x >> 27; + x *= UINT64_C(0x94d049bb133111eb); + x ^= x >> 31; + return x; +} + +__attribute__ ((noinline,optimize(0))) +uint64_t _crc32_u64_O0 (uint64_t crc, uint64_t data) { + const uint64_t POLY_REV = __builtin_bitreverse64(POLY); + int i; + crc = crc ^ data; + + for (i = 0; i < 64; i++) { + if (crc & 1) + crc = (crc >> 1) ^ POLY_REV; + else + crc = (crc >> 1); + } + + return crc; +} + +uint64_t _crc32_u64 (uint64_t crc, uint64_t data) { + return __builtin_rev_crc64_data64(crc, data, POLY); +} + +int main () +{ + uint64_t crc = 0x0D800D80; + for (uint32_t i = 0; i <= 0xff; i++) + { + uint64_t data = expand8to64(i); + uint64_t res1 = _crc32_u64_O0 (crc, data); + uint64_t res2 = _crc32_u64 (crc, data); + if (res1 != res2) + abort (); + crc = res1; + } +} + +/* { dg-final { scan-assembler-not "crc32q"} } */ +/* { dg-final { scan-assembler "pclmul"} } */ diff --git a/gcc/testsuite/gcc.target/i386/crc-crc32-data64.c b/gcc/testsuite/gcc.target/i386/crc-crc32-data64.c new file mode 100644 index 000000000000..02f6fcc9fbfe --- /dev/null +++ b/gcc/testsuite/gcc.target/i386/crc-crc32-data64.c @@ -0,0 +1,54 @@ +/* { dg-do run { target { { sse4_runtime && pclmul_runtime } && { ! ia32 } } } } */ +/* { dg-options "-msse4.2 -mpclmul -O2 -save-temps -fdump-rtl-dfinish -fdump-tree-crc" } */ +/* { dg-skip-if "" { *-*-* } { "-flto"} } */ + +#include <stdint.h> +#include <stdlib.h> + +#define POLY 0x1EDC6F4100000000ULL + +uint64_t expand8to64 (uint64_t x) { + x ^= x >> 30; + x *= UINT64_C(0xbf58476d1ce4e5b9); + x ^= x >> 27; + x *= UINT64_C(0x94d049bb133111eb); + x ^= x >> 31; + return x; +} + +__attribute__ ((noinline,optimize(0))) +uint64_t _crc32_u64_O0 (uint64_t crc, uint64_t data) { + const uint64_t POLY_REV = __builtin_bitreverse64(POLY); + int i; + crc = crc ^ data; + + for (i = 0; i < 64; i++) { + if (crc & 1) + crc = (crc >> 1) ^ POLY_REV; + else + crc = (crc >> 1); + } + + return crc; +} + +uint64_t _crc32_u64 (uint64_t crc, uint64_t data) { + return __builtin_rev_crc64_data64(crc, data, POLY); +} + +int main () +{ + uint64_t crc = 0x0D800D80; + for (uint32_t i = 0; i <= 0xff; i++) + { + uint64_t data = expand8to64(i); + uint64_t res1 = _crc32_u64_O0 (crc, data); + uint64_t res2 = _crc32_u64 (crc, data); + if (res1 != res2) + abort (); + crc = res1; + } +} + +/* { dg-final { scan-assembler "crc32q"} } */ +/* { dg-final { scan-assembler-not "pclmul"} } */
