https://gcc.gnu.org/g:373c7598a996ff568c831a653bc25292691125b1
commit r17-5729-g373c7598a996ff568c831a653bc25292691125b1 Author: Uros Bizjak <[email protected]> Date: Wed Oct 28 10:10:34 2026 +0100 i386: Vectorize _Float16 <-> float conversions with F16C and AVX512F [PR122382] The vectorizer converts between float and _Float16 vectors of different lane counts through the vec_pack_trunc and vec_unpacks_{lo,hi} optabs. These were only provided for TARGET_AVX512FP16, so for x86-64-v3 (F16C) and for AVX512F-only targets the conversion loops stayed scalar, even though vcvtps2ph and vcvtph2ps can do the work. Enable the pack and unpack expanders for all the float and _Float16 vector modes they already handle, using new mode iterators: - V16SF/V32HF with AVX512F, - V8SF/V4SF and V16HF/V8HF with F16C or AVX512VL. The 256- and 512-bit modes reuse the same-lane trunc<mode><ssePHmodelower>2 and extend<ssePHmodelower><mode>2 expanders, which already fall back to vcvtps2ph/vcvtph2ps without AVX512FP16. The 128-bit modes needed two changes: - Add f16c_truncv4sfv8hf2, which emits the V8HF result directly. It uses the AVX512FP16 insn when available and vcvtps2ph otherwise. vec_pack_trunc_v4sf uses it instead of the AVX512FP16-only insn, and trunc<mode>v4hf2 is rewritten on top of it. - Unpack V8HF through extendv4hfv4sf2 instead of calling the AVX512FP16-only avx512fp16_float_extend_phv4sf2 directly. Targets with AVX512FP16 continue to use the direct vcvtps2phx and vcvtph2psx forms (the 128- and 256-bit ones also need AVX512VL). The compile tests check that the loops vectorize and that no scalar vpinsrw/vpextrw remain, for F16C, for AVX512VL without F16C, and for AVX512F alone. The run tests compare the vectorized conversions, including lane order and epilogues, against scalar reference loops. Assisted-by: Claude Sonnet 5.5 PR target/122382 gcc/ChangeLog: * config/i386/sse.md (VHF_F16C, VF1_F16C): New mode iterators. (vec_unpacks_lo_<mode>, vec_unpacks_hi_<mode>): Use VHF_F16C. Remove the TARGET_AVX512FP16 condition. Call gen_extendv4hfv4sf2 for V8HFmode. (f16c_truncv4sfv8hf2): New expander. (trunc<mode>v4hf2): Use it for V4SFmode. (vec_pack_trunc_<mode>): Use VF1_F16C. Remove the TARGET_AVX512FP16 condition. Call gen_f16c_truncv4sfv8hf2 for V4SFmode. gcc/testsuite/ChangeLog: * gcc.target/i386/pr122382.c: New test. * gcc.target/i386/pr122382-1.c: Likewise. * gcc.target/i386/pr122382-1a.c: Likewise. * gcc.target/i386/pr122382-2.c: Likewise. * gcc.target/i386/pr122382-2a.c: Likewise. * gcc.target/i386/pr122382-3.c: Likewise. * gcc.target/i386/pr122382-4.c: Likewise. * gcc.target/i386/pr122382-5.c: Likewise. * gcc.target/i386/pr122382-6.c: Likewise. * gcc.target/i386/pr122382.h: New file. Diff: --- gcc/config/i386/sse.md | 72 +++++++++++++----- gcc/testsuite/gcc.target/i386/pr122382-1.c | 25 ++++++ gcc/testsuite/gcc.target/i386/pr122382-1a.c | 25 ++++++ gcc/testsuite/gcc.target/i386/pr122382-2.c | 25 ++++++ gcc/testsuite/gcc.target/i386/pr122382-2a.c | 25 ++++++ gcc/testsuite/gcc.target/i386/pr122382-3.c | 25 ++++++ gcc/testsuite/gcc.target/i386/pr122382-4.c | 13 ++++ gcc/testsuite/gcc.target/i386/pr122382-5.c | 17 +++++ gcc/testsuite/gcc.target/i386/pr122382-6.c | 13 ++++ gcc/testsuite/gcc.target/i386/pr122382.c | 25 ++++++ gcc/testsuite/gcc.target/i386/pr122382.h | 113 ++++++++++++++++++++++++++++ 11 files changed, 358 insertions(+), 20 deletions(-) diff --git a/gcc/config/i386/sse.md b/gcc/config/i386/sse.md index 1456c92e4981..ed9a1e3de9e0 100644 --- a/gcc/config/i386/sse.md +++ b/gcc/config/i386/sse.md @@ -8011,12 +8011,18 @@ (define_mode_attr vunpck_extract_mode [(V32HF "v32hf") (V16HF "v16hf") (V8HF "v16hf")]) +;; HFmode vectors that can be unpacked to float by vcvtph2ps. +(define_mode_iterator VHF_F16C + [(V32HF "TARGET_AVX512F") + (V16HF "TARGET_F16C || TARGET_AVX512VL") + (V8HF "TARGET_F16C || TARGET_AVX512VL")]) + (define_expand "vec_unpacks_lo_<mode>" [(match_operand:<ssePSmode> 0 "register_operand") - (match_operand:VHF_AVX512VL 1 "register_operand")] - "TARGET_AVX512FP16" + (match_operand:VHF_F16C 1 "register_operand")] + "" { - rtx tem = operands[1]; + rtx tem; rtx (*gen) (rtx, rtx); if (<MODE>mode != V8HFmode) { @@ -8026,7 +8032,11 @@ gen = gen_extend<ssehalfvecmodelower><ssePSmodelower>2; } else - gen = gen_avx512fp16_float_extend_phv4sf2; + { + tem = force_reg (V8HFmode, operands[1]); + tem = lowpart_subreg (V4HFmode, tem, V8HFmode); + gen = gen_extendv4hfv4sf2; + } emit_insn (gen (operands[0], tem)); DONE; @@ -8034,10 +8044,10 @@ (define_expand "vec_unpacks_hi_<mode>" [(match_operand:<ssePSmode> 0 "register_operand") - (match_operand:VHF_AVX512VL 1 "register_operand")] - "TARGET_AVX512FP16" + (match_operand:VHF_F16C 1 "register_operand")] + "" { - rtx tem = operands[1]; + rtx tem; rtx (*gen) (rtx, rtx); if (<MODE>mode != V8HFmode) { @@ -8055,8 +8065,9 @@ rtx selector = gen_rtx_PARALLEL (VOIDmode, tmp); emit_move_insn (tem, - gen_rtx_VEC_SELECT (V8HFmode, operands[1], selector)); - gen = gen_avx512fp16_float_extend_phv4sf2; + gen_rtx_VEC_SELECT (V8HFmode, operands[1], selector)); + tem = lowpart_subreg (V4HFmode, tem, V8HFmode); + gen = gen_extendv4hfv4sf2; } emit_insn (gen (operands[0], tem)); @@ -8673,21 +8684,37 @@ (set_attr "prefix" "evex") (set_attr "mode" "<sseinsnmode>")]) -(define_expand "trunc<mode>v4hf2" - [(set (match_operand:V4HF 0 "register_operand") - (float_truncate:V4HF (match_operand:VF4_128_8_256 1 "vector_operand")))] +;; V4SF -> V8HF, upper half zero. Shared by truncv4sfv4hf2 and +;; vec_pack_trunc_v4sf so neither needs to go through V4HFmode. +(define_expand "f16c_truncv4sfv8hf2" + [(match_operand:V8HF 0 "register_operand") + (match_operand:V4SF 1 "vector_operand")] "TARGET_F16C || TARGET_AVX512VL" { - rtx dest = gen_reg_rtx (V8HFmode); if (TARGET_AVX512FP16 && TARGET_AVX512VL) - emit_insn (gen_avx512fp16_trunc<mode>v4hf2 (dest, operands[1])); + emit_insn (gen_avx512fp16_truncv4sfv4hf2 (operands[0], operands[1])); else { rtx tem = gen_reg_rtx (V8HImode); emit_insn (gen_vcvtps2ph (tem, force_reg (V4SFmode, operands[1]), GEN_INT (4))); - emit_move_insn (dest, lowpart_subreg (V8HFmode, tem, V8HImode)); + emit_move_insn (operands[0], lowpart_subreg (V8HFmode, tem, V8HImode)); } + DONE; +}) + +(define_expand "trunc<mode>v4hf2" + [(set (match_operand:V4HF 0 "register_operand") + (float_truncate:V4HF (match_operand:VF4_128_8_256 1 "vector_operand")))] + "TARGET_F16C || TARGET_AVX512VL" +{ + rtx dest = gen_reg_rtx (V8HFmode); + + if (<MODE>mode == V4SFmode) + emit_insn (gen_f16c_truncv4sfv8hf2 (dest, operands[1])); + else + emit_insn (gen_avx512fp16_trunc<mode>v4hf2 (dest, operands[1])); + emit_move_insn (operands[0], lowpart_subreg (V4HFmode, dest, V8HFmode)); DONE; }) @@ -11584,18 +11611,24 @@ operands[4] = gen_reg_rtx (<sf2dfmode>mode); }) +;; HFmode vectors that can be packed from float by vcvtps2ph. +(define_mode_iterator VF1_F16C + [(V16SF "TARGET_AVX512F") + (V8SF "TARGET_F16C || TARGET_AVX512VL") + (V4SF "TARGET_F16C || TARGET_AVX512VL")]) + (define_expand "vec_pack_trunc_<mode>" [(match_operand:<ssepackPHmode> 0 "register_operand") - (match_operand:VF1_AVX512VL 1 "register_operand") - (match_operand:VF1_AVX512VL 2 "register_operand")] - "TARGET_AVX512FP16" + (match_operand:VF1_F16C 1 "register_operand") + (match_operand:VF1_F16C 2 "register_operand")] + "" { rtx r1 = gen_reg_rtx (<vpckfloat_temp_mode>mode); rtx r2 = gen_reg_rtx (<vpckfloat_temp_mode>mode); rtx (*gen) (rtx, rtx); if (<MODE>mode == V4SFmode) - gen = gen_avx512fp16_truncv4sfv4hf2; + gen = gen_f16c_truncv4sfv8hf2; else gen = gen_trunc<mode><vpckfloat_op_mode>2; emit_insn (gen (r1, operands[1])); @@ -11606,7 +11639,6 @@ emit_insn (gen_avx_vec_concat<vpckfloat_concat_mode> (operands[0], r1, r2)); DONE; - }) (define_expand "vec_pack_trunc_v2df" diff --git a/gcc/testsuite/gcc.target/i386/pr122382-1.c b/gcc/testsuite/gcc.target/i386/pr122382-1.c new file mode 100644 index 000000000000..861b4daf6105 --- /dev/null +++ b/gcc/testsuite/gcc.target/i386/pr122382-1.c @@ -0,0 +1,25 @@ +/* PR target/122382 */ +/* { dg-do compile } */ +/* { dg-options "-O2 -mf16c -mno-avx512f -fdump-tree-vect-details" } */ + +/* { dg-final { scan-tree-dump-times "vectorized 1 loops in function" 2 "vect" } } */ + +void +foo (_Float16 *__restrict h, const float *__restrict f) +{ + for (int n = 0; n < 8; ++n) + h[n] = f[n]; +} + +/* { dg-final { scan-assembler "vcvtps2ph" } } */ +/* { dg-final { scan-assembler-not "vpinsrw" } } */ + +void +bar (float *__restrict f, const _Float16 *__restrict h) +{ + for (int n = 0; n < 8; ++n) + f[n] = h[n]; +} + +/* { dg-final { scan-assembler "vcvtph2ps" } } */ +/* { dg-final { scan-assembler-not "vpextrw" } } */ diff --git a/gcc/testsuite/gcc.target/i386/pr122382-1a.c b/gcc/testsuite/gcc.target/i386/pr122382-1a.c new file mode 100644 index 000000000000..8f7e1d475634 --- /dev/null +++ b/gcc/testsuite/gcc.target/i386/pr122382-1a.c @@ -0,0 +1,25 @@ +/* PR target/122382 */ +/* { dg-do compile } */ +/* { dg-options "-O2 -mavx512vl -mno-f16c -mno-avx512fp16 -fdump-tree-vect-details" } */ + +/* { dg-final { scan-tree-dump-times "vectorized 1 loops in function" 2 "vect" } } */ + +void +foo (_Float16 *__restrict h, const float *__restrict f) +{ + for (int n = 0; n < 8; ++n) + h[n] = f[n]; +} + +/* { dg-final { scan-assembler "vcvtps2ph" } } */ +/* { dg-final { scan-assembler-not "vpinsrw" } } */ + +void +bar (float *__restrict f, const _Float16 *__restrict h) +{ + for (int n = 0; n < 8; ++n) + f[n] = h[n]; +} + +/* { dg-final { scan-assembler "vcvtph2ps" } } */ +/* { dg-final { scan-assembler-not "vpextrw" } } */ diff --git a/gcc/testsuite/gcc.target/i386/pr122382-2.c b/gcc/testsuite/gcc.target/i386/pr122382-2.c new file mode 100644 index 000000000000..0e4cef3350f6 --- /dev/null +++ b/gcc/testsuite/gcc.target/i386/pr122382-2.c @@ -0,0 +1,25 @@ +/* PR target/122382 */ +/* { dg-do compile } */ +/* { dg-options "-O2 -mf16c -mno-avx512f -fdump-tree-vect-details" } */ + +/* { dg-final { scan-tree-dump-times "vectorized 1 loops in function" 2 "vect" } } */ + +void +foo (_Float16 *__restrict h, const float *__restrict f) +{ + for (int n = 0; n < 16; ++n) + h[n] = f[n]; +} + +/* { dg-final { scan-assembler "vcvtps2ph" } } */ +/* { dg-final { scan-assembler-not "vpinsrw" } } */ + +void +bar (float *__restrict f, const _Float16 *__restrict h) +{ + for (int n = 0; n < 16; ++n) + f[n] = h[n]; +} + +/* { dg-final { scan-assembler "vcvtph2ps" } } */ +/* { dg-final { scan-assembler-not "vpextrw" } } */ diff --git a/gcc/testsuite/gcc.target/i386/pr122382-2a.c b/gcc/testsuite/gcc.target/i386/pr122382-2a.c new file mode 100644 index 000000000000..e9026bb24e02 --- /dev/null +++ b/gcc/testsuite/gcc.target/i386/pr122382-2a.c @@ -0,0 +1,25 @@ +/* PR target/122382 */ +/* { dg-do compile } */ +/* { dg-options "-O2 -mavx512vl -mno-f16c -mno-avx512fp16 -fdump-tree-vect-details" } */ + +/* { dg-final { scan-tree-dump-times "vectorized 1 loops in function" 2 "vect" } } */ + +void +foo (_Float16 *__restrict h, const float *__restrict f) +{ + for (int n = 0; n < 16; ++n) + h[n] = f[n]; +} + +/* { dg-final { scan-assembler "vcvtps2ph" } } */ +/* { dg-final { scan-assembler-not "vpinsrw" } } */ + +void +bar (float *__restrict f, const _Float16 *__restrict h) +{ + for (int n = 0; n < 16; ++n) + f[n] = h[n]; +} + +/* { dg-final { scan-assembler "vcvtph2ps" } } */ +/* { dg-final { scan-assembler-not "vpextrw" } } */ diff --git a/gcc/testsuite/gcc.target/i386/pr122382-3.c b/gcc/testsuite/gcc.target/i386/pr122382-3.c new file mode 100644 index 000000000000..f8239fdf4838 --- /dev/null +++ b/gcc/testsuite/gcc.target/i386/pr122382-3.c @@ -0,0 +1,25 @@ +/* PR target/122382 */ +/* { dg-do compile } */ +/* { dg-options "-O2 -mavx512f -mno-f16c -mno-avx512vl -mno-avx512fp16 -mprefer-vector-width=512 -fdump-tree-vect-details" } */ + +/* { dg-final { scan-tree-dump-times "vectorized 1 loops in function" 2 "vect" } } */ + +void +foo (_Float16 *__restrict h, const float *__restrict f) +{ + for (int n = 0; n < 32; ++n) + h[n] = f[n]; +} + +/* { dg-final { scan-assembler "vcvtps2ph" } } */ +/* { dg-final { scan-assembler-not "vpinsrw" } } */ + +void +bar (float *__restrict f, const _Float16 *__restrict h) +{ + for (int n = 0; n < 32; ++n) + f[n] = h[n]; +} + +/* { dg-final { scan-assembler "vcvtph2ps" } } */ +/* { dg-final { scan-assembler-not "vpextrw" } } */ diff --git a/gcc/testsuite/gcc.target/i386/pr122382-4.c b/gcc/testsuite/gcc.target/i386/pr122382-4.c new file mode 100644 index 000000000000..fe91c530102e --- /dev/null +++ b/gcc/testsuite/gcc.target/i386/pr122382-4.c @@ -0,0 +1,13 @@ +/* PR target/122382 */ +/* { dg-do run } */ +/* { dg-options "-O3 -mf16c -mno-avx512f" } */ +/* { dg-require-effective-target f16c } */ + +#include "f16c-check.h" +#include "pr122382.h" + +static void +f16c_test (void) +{ + run_tests (); +} diff --git a/gcc/testsuite/gcc.target/i386/pr122382-5.c b/gcc/testsuite/gcc.target/i386/pr122382-5.c new file mode 100644 index 000000000000..1f181f599fa1 --- /dev/null +++ b/gcc/testsuite/gcc.target/i386/pr122382-5.c @@ -0,0 +1,17 @@ +/* PR target/122382 */ +/* { dg-do run } */ +/* { dg-options "-O3 -mavx512f -mno-f16c -mno-avx512vl -mno-avx512fp16 -mprefer-vector-width=512" } */ +/* { dg-require-effective-target avx512f } */ + +/* Only loops of at least 32 elements are loop-vectorized here, through the + V16SF <-> V32HF pack/unpack expanders. Shorter loops are left to SLP or + stay scalar, calling __truncsfhf2 and __extendhfsf2. */ + +#include "avx512f-check.h" +#include "pr122382.h" + +void +avx512f_test (void) +{ + run_tests (); +} diff --git a/gcc/testsuite/gcc.target/i386/pr122382-6.c b/gcc/testsuite/gcc.target/i386/pr122382-6.c new file mode 100644 index 000000000000..c3aeb20759fb --- /dev/null +++ b/gcc/testsuite/gcc.target/i386/pr122382-6.c @@ -0,0 +1,13 @@ +/* PR target/122382 */ +/* { dg-do run } */ +/* { dg-options "-O3 -mavx512vl -mno-f16c -mno-avx512fp16 -mprefer-vector-width=256" } */ +/* { dg-require-effective-target avx512vl } */ + +#include "avx512vl-check.h" +#include "pr122382.h" + +static void +avx512vl_test (void) +{ + run_tests (); +} diff --git a/gcc/testsuite/gcc.target/i386/pr122382.c b/gcc/testsuite/gcc.target/i386/pr122382.c new file mode 100644 index 000000000000..db19eaea076e --- /dev/null +++ b/gcc/testsuite/gcc.target/i386/pr122382.c @@ -0,0 +1,25 @@ +/* PR target/122382 */ +/* { dg-do compile } */ +/* { dg-options "-O2 -mf16c -mno-avx512f -fdump-tree-vect-details" } */ + +/* { dg-final { scan-tree-dump-times "vectorized 1 loops in function" 2 "vect" } } */ + +void +foo (_Float16 *__restrict h, const float *__restrict f) +{ + for (int n = 0; n < 128; ++n) + h[n] = f[n]; +} + +/* { dg-final { scan-assembler "vcvtps2ph" } } */ +/* { dg-final { scan-assembler-not "vpinsrw" } } */ + +void +bar (float *__restrict f, const _Float16 *__restrict h) +{ + for (int n = 0; n < 128; ++n) + f[n] = h[n]; +} + +/* { dg-final { scan-assembler "vcvtph2ps" } } */ +/* { dg-final { scan-assembler-not "vpextrw" } } */ diff --git a/gcc/testsuite/gcc.target/i386/pr122382.h b/gcc/testsuite/gcc.target/i386/pr122382.h new file mode 100644 index 000000000000..9ee1fcd3c3ea --- /dev/null +++ b/gcc/testsuite/gcc.target/i386/pr122382.h @@ -0,0 +1,113 @@ +/* Common body for the PR target/122382 run tests. Check that vectorized + float <-> _Float16 conversions agree with scalar reference loops, + including lane order, and that nothing is written past the end. */ + +#include <stdlib.h> +#include <string.h> + +#define MAXN 128 + +#define REF_ATTR __attribute__ ((noipa, optimize ("no-tree-vectorize"))) + +#define DEF(N) \ +__attribute__ ((noipa)) static void \ +pack##N (_Float16 *__restrict h, const float *__restrict f) \ +{ \ + for (int n = 0; n < N; ++n) \ + h[n] = f[n]; \ +} \ +__attribute__ ((noipa)) static void \ +unpack##N (float *__restrict f, const _Float16 *__restrict h) \ +{ \ + for (int n = 0; n < N; ++n) \ + f[n] = h[n]; \ +} + +DEF (8) +DEF (16) +DEF (32) +DEF (128) + +__attribute__ ((noipa)) static void +packv (_Float16 *__restrict h, const float *__restrict f, int cnt) +{ + for (int n = 0; n < cnt; ++n) + h[n] = f[n]; +} + +__attribute__ ((noipa)) static void +unpackv (float *__restrict f, const _Float16 *__restrict h, int cnt) +{ + for (int n = 0; n < cnt; ++n) + f[n] = h[n]; +} + +REF_ATTR static void +ref_pack (_Float16 *h, const float *f, int cnt) +{ + for (int n = 0; n < cnt; ++n) + h[n] = f[n]; +} + +REF_ATTR static void +ref_unpack (float *f, const _Float16 *h, int cnt) +{ + for (int n = 0; n < cnt; ++n) + f[n] = h[n]; +} + +static float fin[MAXN], fout[MAXN], fref[MAXN]; +static _Float16 hin[MAXN], hout[MAXN], href[MAXN]; + +#define CHECK_PACK(CALL, CNT) \ + do { \ + memset (hout, 0x55, sizeof hout); \ + memset (href, 0x55, sizeof href); \ + CALL; \ + ref_pack (href, fin, CNT); \ + if (memcmp (hout, href, sizeof hout) != 0) \ + abort (); \ + } while (0) + +#define CHECK_UNPACK(CALL, CNT) \ + do { \ + memset (fout, 0x55, sizeof fout); \ + memset (fref, 0x55, sizeof fref); \ + CALL; \ + ref_unpack (fref, hin, CNT); \ + if (memcmp (fout, fref, sizeof fout) != 0) \ + abort (); \ + } while (0) + +static void +run_tests (void) +{ + /* Even lanes are exactly representable as _Float16, odd lanes need + rounding. Values differ from lane to lane so that any lane + permutation is detected. */ + for (int i = 0; i < MAXN; i++) + { + int v = (i * 7) % 251 - 100; + fin[i] = (i & 1) ? v * 0.1f : v * 0.25f; + } + ref_pack (hin, fin, MAXN); + + CHECK_PACK (pack8 (hout, fin), 8); + CHECK_PACK (pack16 (hout, fin), 16); + CHECK_PACK (pack32 (hout, fin), 32); + CHECK_PACK (pack128 (hout, fin), 128); + + CHECK_UNPACK (unpack8 (fout, hin), 8); + CHECK_UNPACK (unpack16 (fout, hin), 16); + CHECK_UNPACK (unpack32 (fout, hin), 32); + CHECK_UNPACK (unpack128 (fout, hin), 128); + + /* Counts that are not a multiple of the vector length exercise the + epilogue, which can use narrower vectors. */ + static const int cnts[] = { 0, 1, 3, 5, 9, 17, 37, 100, 127 }; + for (unsigned int i = 0; i < sizeof cnts / sizeof cnts[0]; i++) + { + CHECK_PACK (packv (hout, fin, cnts[i]), cnts[i]); + CHECK_UNPACK (unpackv (fout, hin, cnts[i]), cnts[i]); + } +}
