https://gcc.gnu.org/g:c2cdc43bc60b321c07ea6d6e85d3da94cab0177c
commit r17-1764-gc2cdc43bc60b321c07ea6d6e85d3da94cab0177c Author: Hu, Lin1 <[email protected]> Date: Tue Jun 2 00:13:52 2026 -0700 i386: implement cond_vec_cbranch_any/all for narrow boolean vectors gcc/ChangeLog: * config/i386/sse.md (V_ANY_CBRANCH): New mode iterator. (cond_vec_cbranch_any<mode>): New define_expand. (cond_vec_cbranch_all<mode>): Ditto. gcc/testsuite/ChangeLog: * gcc.target/i386/vect-cbranch-cond-any-128.c: New test. * gcc.target/i386/vect-cbranch-cond-any-256.c: Ditto. * gcc.target/i386/vect-cbranch-cond-any-run.c: Ditto. * gcc.target/i386/vect-cbranch-cond-all-128.c: Ditto. * gcc.target/i386/vect-cbranch-cond-all-256.c: Ditto. * gcc.target/i386/vect-cbranch-cond-all-run.c: Ditto. Diff: --- gcc/config/i386/sse.md | 72 ++++++++++++++++++ .../gcc.target/i386/vect-cbranch-cond-all-128.c | 87 ++++++++++++++++++++++ .../gcc.target/i386/vect-cbranch-cond-all-256.c | 47 ++++++++++++ .../gcc.target/i386/vect-cbranch-cond-all-run.c | 72 ++++++++++++++++++ .../gcc.target/i386/vect-cbranch-cond-any-128.c | 86 +++++++++++++++++++++ .../gcc.target/i386/vect-cbranch-cond-any-256.c | 46 ++++++++++++ .../gcc.target/i386/vect-cbranch-cond-any-run.c | 67 +++++++++++++++++ 7 files changed, 477 insertions(+) diff --git a/gcc/config/i386/sse.md b/gcc/config/i386/sse.md index 6ca3e34b2c7d..531d35175fc2 100644 --- a/gcc/config/i386/sse.md +++ b/gcc/config/i386/sse.md @@ -29765,6 +29765,78 @@ DONE; }) +;; vec_cbranch modes with fewer than 8 elements (integer + floating-point). +;; Limited to AVX512VL+DQ so that kortestb is available for the QImode mask. +(define_mode_iterator V_ANY_CBRANCH + [(V4SI "TARGET_AVX512VL && TARGET_AVX512DQ") + (V2DI "TARGET_AVX512VL && TARGET_AVX512DQ") + (V4DI "TARGET_AVX512VL && TARGET_AVX512DQ") + (V4SF "TARGET_AVX512VL && TARGET_AVX512DQ") + (V4DF "TARGET_AVX512VL && TARGET_AVX512DQ") + (V2DF "TARGET_AVX512VL && TARGET_AVX512DQ")]) + +;; Masked variant of vec_cbranch_any. Eliminates the redundant kmov + and +;; + test sequence the vectorizer would otherwise emit when an early-break +;; is guarded by a loop mask. +(define_expand "cond_vec_cbranch_any<mode>" + [(set (pc) + (if_then_else + (match_operator 0 "" + [(match_operand:<avx512fmaskmode> 1 "register_operand") + (match_operand:V_ANY_CBRANCH 2 "register_operand") + (match_operand:V_ANY_CBRANCH 3 "nonimmediate_operand")]) + (label_ref (match_operand 4 "")) + (pc)))] + "" +{ + machine_mode mask_mode = <avx512fmaskmode>mode; + rtx cmp_mask = gen_reg_rtx (mask_mode); + rtx and_mask = gen_reg_rtx (mask_mode); + + emit_insn (gen_vec_cmp<mode><avx512fmaskmodelower> (cmp_mask, + operands[0], + operands[2], + operands[3])); + and_mask = expand_simple_binop (mask_mode, AND, cmp_mask, operands[1], + and_mask, 1, OPTAB_DIRECT); + + rtx cc_reg = gen_rtx_REG (CCZmode, FLAGS_REG); + emit_insn (gen_rtx_SET (cc_reg, + gen_rtx_COMPARE (CCZmode, and_mask, const0_rtx))); + ix86_expand_branch (NE, cc_reg, const0_rtx, operands[4]); + DONE; +}) + +;; Reached when the middle-end reverses an NE comparison to EQ and swaps the +;; branch labels. +(define_expand "cond_vec_cbranch_all<mode>" + [(set (pc) + (if_then_else + (match_operator 0 "" + [(match_operand:<avx512fmaskmode> 1 "register_operand") + (match_operand:V_ANY_CBRANCH 2 "register_operand") + (match_operand:V_ANY_CBRANCH 3 "nonimmediate_operand")]) + (label_ref (match_operand 4 "")) + (pc)))] + "" +{ + machine_mode mask_mode = <avx512fmaskmode>mode; + rtx cmp_mask = gen_reg_rtx (mask_mode); + rtx and_mask = gen_reg_rtx (mask_mode); + + emit_insn (gen_vec_cmp<mode><avx512fmaskmodelower> (cmp_mask, + operands[0], + operands[2], + operands[3])); + and_mask = expand_simple_binop (mask_mode, AND, cmp_mask, operands[1], + and_mask, 1, OPTAB_DIRECT); + + rtx cc_reg = gen_rtx_REG (CCZmode, FLAGS_REG); + emit_insn (gen_rtx_SET (cc_reg, + gen_rtx_COMPARE (CCZmode, and_mask, const0_rtx))); + ix86_expand_branch (EQ, cc_reg, const0_rtx, operands[4]); + DONE; +}) (define_insn_and_split "avx_<castmode><avxsizesuffix>_<castmode>" [(set (match_operand:AVX256MODE2P 0 "nonimmediate_operand" "=x,m") diff --git a/gcc/testsuite/gcc.target/i386/vect-cbranch-cond-all-128.c b/gcc/testsuite/gcc.target/i386/vect-cbranch-cond-all-128.c new file mode 100644 index 000000000000..f12178d4a90d --- /dev/null +++ b/gcc/testsuite/gcc.target/i386/vect-cbranch-cond-all-128.c @@ -0,0 +1,87 @@ +/* Test cond_vec_cbranch_all<mode> for the 128-bit V_ANY_CBRANCH modes + (V4SI, V2DI, V4SF, V2DF). */ +/* { dg-do compile } */ +/* { dg-options "-Ofast -mavx512f -mavx512vl -mavx512dq -mprefer-vector-width=128 -fno-schedule-insns -fno-reorder-blocks -fno-schedule-insns2" } */ +/* { dg-final { check-function-bodies "**" "" "" } } */ + +#define N 100000 +int a_int[N], b_int[N], c_int[N]; +long long a_long[N], b_long[N], c_long[N]; +float a_float[N], b_float[N], c_float[N]; +double a_double[N], b_double[N], c_double[N]; + +/* +** f_v4si: +** ... +** vpcmpd \$4, %xmm[0-9]+, %xmm[0-9]+, %k[0-9]+ +** vpcmpd \$4, %xmm[0-9]+, %xmm[0-9]+, %k[0-9]+\{%k[0-9]+\} +** kortestb %k[0-9]+, %k[0-9]+ +** je \.L[0-9]+ +** ... +*/ +void f_v4si (int d, int e) +{ + for (int i = 0; i < N; i++) + { + b_int[i] += a_int[i]; + if (c_int[i] == e) continue; + if (a_int[i] != d) break; + } +} + +/* +** f_v2di: +** ... +** vpcmpq \$4, %xmm[0-9]+, %xmm[0-9]+, %k[0-9]+ +** vpcmpq \$4, %xmm[0-9]+, %xmm[0-9]+, %k[0-9]+\{%k[0-9]+\} +** kortestb %k[0-9]+, %k[0-9]+ +** je \.L[0-9]+ +** ... +*/ +void f_v2di (long long d, long long e) +{ + for (int i = 0; i < N; i++) + { + b_long[i] += a_long[i]; + if (c_long[i] == e) continue; + if (a_long[i] != d) break; + } +} + +/* +** f_v4sf: +** ... +** vcmpps \$4, %xmm[0-9]+, %xmm[0-9]+, %k[0-9]+ +** vcmpps \$4, %xmm[0-9]+, %xmm[0-9]+, %k[0-9]+\{%k[0-9]+\} +** kortestb %k[0-9]+, %k[0-9]+ +** je \.L[0-9]+ +** ... +*/ +void f_v4sf (float d, float e) +{ + for (int i = 0; i < N; i++) + { + b_float[i] += a_float[i]; + if (c_float[i] == e) continue; + if (a_float[i] != d) break; + } +} + +/* +** f_v2df: +** ... +** vcmppd \$4, %xmm[0-9]+, %xmm[0-9]+, %k[0-9]+ +** vcmppd \$4, %xmm[0-9]+, %xmm[0-9]+, %k[0-9]+\{%k[0-9]+\} +** kortestb %k[0-9]+, %k[0-9]+ +** je \.L[0-9]+ +** ... +*/ +void f_v2df (double d, double e) +{ + for (int i = 0; i < N; i++) + { + b_double[i] += a_double[i]; + if (c_double[i] == e) continue; + if (a_double[i] != d) break; + } +} diff --git a/gcc/testsuite/gcc.target/i386/vect-cbranch-cond-all-256.c b/gcc/testsuite/gcc.target/i386/vect-cbranch-cond-all-256.c new file mode 100644 index 000000000000..8abad799280c --- /dev/null +++ b/gcc/testsuite/gcc.target/i386/vect-cbranch-cond-all-256.c @@ -0,0 +1,47 @@ +/* Test cond_vec_cbranch_all<mode> for the 256-bit V_ANY_CBRANCH modes + (V4DI, V4DF). */ +/* { dg-do compile } */ +/* { dg-options "-Ofast -mavx512f -mavx512vl -mavx512dq -mprefer-vector-width=256 -fno-schedule-insns -fno-reorder-blocks -fno-schedule-insns2" } */ +/* { dg-final { check-function-bodies "**" "" "" } } */ + +#define N 100000 +long long a_long[N], b_long[N], c_long[N]; +double a_double[N], b_double[N], c_double[N]; + +/* +** f_v4di: +** ... +** vpcmpq \$4, %ymm[0-9]+, %ymm[0-9]+, %k[0-9]+ +** vpcmpq \$4, %ymm[0-9]+, %ymm[0-9]+, %k[0-9]+\{%k[0-9]+\} +** kortestb %k[0-9]+, %k[0-9]+ +** je \.L[0-9]+ +** ... +*/ +void f_v4di (long long d, long long e) +{ + for (int i = 0; i < N; i++) + { + b_long[i] += a_long[i]; + if (c_long[i] == e) continue; + if (a_long[i] != d) break; + } +} + +/* +** f_v4df: +** ... +** vcmppd \$4, %ymm[0-9]+, %ymm[0-9]+, %k[0-9]+ +** vcmppd \$4, %ymm[0-9]+, %ymm[0-9]+, %k[0-9]+\{%k[0-9]+\} +** kortestb %k[0-9]+, %k[0-9]+ +** je \.L[0-9]+ +** ... +*/ +void f_v4df (double d, double e) +{ + for (int i = 0; i < N; i++) + { + b_double[i] += a_double[i]; + if (c_double[i] == e) continue; + if (a_double[i] != d) break; + } +} diff --git a/gcc/testsuite/gcc.target/i386/vect-cbranch-cond-all-run.c b/gcc/testsuite/gcc.target/i386/vect-cbranch-cond-all-run.c new file mode 100644 index 000000000000..7c459e995daf --- /dev/null +++ b/gcc/testsuite/gcc.target/i386/vect-cbranch-cond-all-run.c @@ -0,0 +1,72 @@ +/* { dg-do run { target { avx512vl && avx512dq } } } */ +/* { dg-options "-Ofast -mavx512f -mavx512vl -mavx512dq -mprefer-vector-width=128" } */ + +#define AVX512VL +#define AVX512DQ +#define DO_TEST test_cbranch_all +static void test_cbranch_all (void); +#include "avx512-check.h" + +#define N 100 +long long a[N]; +long long b[N]; +long long c[N]; + +__attribute__((noipa)) +void f (long long d, long long e) +{ + for (int i = 0; i < N; i++) + { + b[i] += a[i]; + if (c[i] == e) continue; + if (a[i] != d) break; + } +} + +static void reset (void) +{ + for (int i = 0; i < N; i++) { a[i] = 0; b[i] = 0; c[i] = 0; } +} + +#define CHECK(_i, _v) do { if (b[_i] != (_v)) __builtin_abort (); } while (0) +#define CHECK_RANGE_EQ(_lo, _hi, _v) \ + do { \ + for (int i = _lo; i < _hi; i++) \ + if (b[i] != (_v)) __builtin_abort (); \ + } while (0) + +static void +test_cbranch_all (void) +{ + /* No break — all a[i] == d. Whole loop runs; b[i] = a[i]. */ + reset (); + for (int i = 0; i < N; i++) { a[i] = 7; c[i] = 0; } + f (7 /*d*/, 99 /*e*/); + CHECK_RANGE_EQ (0, N, 7); + + /* Break at index 50: a[i]==d for i<50, a[50]!=d AND c[50]!=e. */ + reset (); + for (int i = 0; i < N; i++) { a[i] = 7; c[i] = 0; } + a[50] = 99; + f (7, 5); + CHECK_RANGE_EQ (0, 50, 7); CHECK (50, 99); + CHECK_RANGE_EQ (51, N, 0); + + /* "continue" gates the break: c[50]==e at the same i where a[i]!=d + should *not* break, loop keeps going. */ + reset (); + for (int i = 0; i < N; i++) { a[i] = 7; c[i] = 0; } + a[50] = 99; c[50] = 5; /* c==e -> continue, skip break check */ + f (7, 5); + CHECK (50, 99); + for (int i = 0; i < N; i++) + if (i != 50 && b[i] != 7) __builtin_abort (); + + /* Break at index 0. */ + reset (); + a[0] = 99; c[0] = 0; /* a!=d AND c!=e at i=0 */ + for (int i = 1; i < N; i++) { a[i] = 7; c[i] = 0; } + f (7, 5); + CHECK (0, 99); + CHECK_RANGE_EQ (1, N, 0); +} diff --git a/gcc/testsuite/gcc.target/i386/vect-cbranch-cond-any-128.c b/gcc/testsuite/gcc.target/i386/vect-cbranch-cond-any-128.c new file mode 100644 index 000000000000..b8e321306a04 --- /dev/null +++ b/gcc/testsuite/gcc.target/i386/vect-cbranch-cond-any-128.c @@ -0,0 +1,86 @@ +/* Test cond_vec_cbranch_any<mode> for the 128-bit V_ANY_CBRANCH modes + (V4SI, V2DI, V4SF, V2DF). */ +/* { dg-do compile } */ +/* { dg-options "-Ofast -mavx512f -mavx512vl -mavx512dq -mprefer-vector-width=128 -fno-schedule-insns -fno-reorder-blocks -fno-schedule-insns2" } */ +/* { dg-final { check-function-bodies "**" "" "" } } */ + +int a_int[5], b_int[5], c_int[5]; +long long a_long[3], b_long[3], c_long[3]; +float a_float[5], b_float[5], c_float[5]; +double a_double[3], b_double[3], c_double[3]; + +/* +** f_v4si: +** ... +** vpcmpd \$4, %xmm[0-9]+, %xmm[0-9]+, %k[0-9]+ +** vpcmpd \$4, %xmm[0-9]+, %xmm[0-9]+, %k[0-9]+\{%k[0-9]+\} +** kortestb %k[0-9]+, %k[0-9]+ +** jne \.L[0-9]+ +** ... +*/ +void f_v4si (int d, int e) +{ + for (int i = 0; i < 5; i++) + { + b_int[i] += a_int[i]; + if (c_int[i] == e) continue; + if (a_int[i] != d) break; + } +} + +/* +** f_v2di: +** ... +** vpcmpq \$4, %xmm[0-9]+, %xmm[0-9]+, %k[0-9]+ +** vpcmpq \$4, %xmm[0-9]+, %xmm[0-9]+, %k[0-9]+\{%k[0-9]+\} +** kortestb %k[0-9]+, %k[0-9]+ +** jne \.L[0-9]+ +** ... +*/ +void f_v2di (long long d, long long e) +{ + for (int i = 0; i < 3; i++) + { + b_long[i] += a_long[i]; + if (c_long[i] == e) continue; + if (a_long[i] != d) break; + } +} + +/* +** f_v4sf: +** ... +** vcmpps \$4, %xmm[0-9]+, %xmm[0-9]+, %k[0-9]+ +** vcmpps \$4, %xmm[0-9]+, %xmm[0-9]+, %k[0-9]+\{%k[0-9]+\} +** kortestb %k[0-9]+, %k[0-9]+ +** jne \.L[0-9]+ +** ... +*/ +void f_v4sf (float d, float e) +{ + for (int i = 0; i < 5; i++) + { + b_float[i] += a_float[i]; + if (c_float[i] == e) continue; + if (a_float[i] != d) break; + } +} + +/* +** f_v2df: +** ... +** vcmppd \$4, %xmm[0-9]+, %xmm[0-9]+, %k[0-9]+ +** vcmppd \$4, %xmm[0-9]+, %xmm[0-9]+, %k[0-9]+\{%k[0-9]+\} +** kortestb %k[0-9]+, %k[0-9]+ +** jne \.L[0-9]+ +** ... +*/ +void f_v2df (double d, double e) +{ + for (int i = 0; i < 3; i++) + { + b_double[i] += a_double[i]; + if (c_double[i] == e) continue; + if (a_double[i] != d) break; + } +} diff --git a/gcc/testsuite/gcc.target/i386/vect-cbranch-cond-any-256.c b/gcc/testsuite/gcc.target/i386/vect-cbranch-cond-any-256.c new file mode 100644 index 000000000000..7e88834f7c50 --- /dev/null +++ b/gcc/testsuite/gcc.target/i386/vect-cbranch-cond-any-256.c @@ -0,0 +1,46 @@ +/* Test cond_vec_cbranch_any<mode> for the 256-bit V_ANY_CBRANCH modes + (V4DI, V4DF). */ +/* { dg-do compile } */ +/* { dg-options "-Ofast -mavx512f -mavx512vl -mavx512dq -mprefer-vector-width=256 -fno-schedule-insns -fno-reorder-blocks -fno-schedule-insns2" } */ +/* { dg-final { check-function-bodies "**" "" "" } } */ + +long long a_long[5], b_long[5], c_long[5]; +double a_double[5], b_double[5], c_double[5]; + +/* +** f_v4di: +** ... +** vpcmpq \$4, %ymm[0-9]+, %ymm[0-9]+, %k[0-9]+ +** vpcmpq \$4, %ymm[0-9]+, %ymm[0-9]+, %k[0-9]+\{%k[0-9]+\} +** kortestb %k[0-9]+, %k[0-9]+ +** jne \.L[0-9]+ +** ... +*/ +void f_v4di (long long d, long long e) +{ + for (int i = 0; i < 5; i++) + { + b_long[i] += a_long[i]; + if (c_long[i] == e) continue; + if (a_long[i] != d) break; + } +} + +/* +** f_v4df: +** ... +** vcmppd \$4, %ymm[0-9]+, %ymm[0-9]+, %k[0-9]+ +** vcmppd \$4, %ymm[0-9]+, %ymm[0-9]+, %k[0-9]+\{%k[0-9]+\} +** kortestb %k[0-9]+, %k[0-9]+ +** jne \.L[0-9]+ +** ... +*/ +void f_v4df (double d, double e) +{ + for (int i = 0; i < 5; i++) + { + b_double[i] += a_double[i]; + if (c_double[i] == e) continue; + if (a_double[i] != d) break; + } +} diff --git a/gcc/testsuite/gcc.target/i386/vect-cbranch-cond-any-run.c b/gcc/testsuite/gcc.target/i386/vect-cbranch-cond-any-run.c new file mode 100644 index 000000000000..323b6d1e545d --- /dev/null +++ b/gcc/testsuite/gcc.target/i386/vect-cbranch-cond-any-run.c @@ -0,0 +1,67 @@ +/* { dg-do run { target { avx512vl && avx512dq } } } */ +/* { dg-options "-Ofast -mavx512f -mavx512vl -mavx512dq -mprefer-vector-width=128" } */ + +#define AVX512VL +#define AVX512DQ +#define DO_TEST test_cbranch_any +static void test_cbranch_any (void); +#include "avx512-check.h" + +#define N 3 /* V2DI nelem (2) + 1 */ +long long a[N]; +long long b[N]; +long long c[N]; + +__attribute__((noipa)) +void f (long long d, long long e) +{ + for (int i = 0; i < N; i++) + { + b[i] += a[i]; + if (c[i] == e) continue; + if (a[i] != d) break; + } +} + +static void reset (void) +{ + for (int i = 0; i < N; i++) { a[i] = 0; b[i] = 0; c[i] = 0; } +} + +#define CHECK(_i, _v) do { if (b[_i] != (_v)) __builtin_abort (); } while (0) + +static void +test_cbranch_any (void) +{ + /* a[0] != d AND c[0] != e → break immediately, only b[0] updated. */ + reset (); + a[0] = 5; c[0] = 5; + a[1] = 1; c[1] = 7; + a[2] = 1; c[2] = 7; + f (1 /*d*/, 7 /*e*/); + CHECK (0, 5); CHECK (1, 0); CHECK (2, 0); + + /* All a[i]==d → continue cond never breaks. All updated. */ + reset (); + a[0] = 1; c[0] = 7; + a[1] = 1; c[1] = 7; + a[2] = 1; c[2] = 7; + f (1, 7); + CHECK (0, 1); CHECK (1, 1); CHECK (2, 1); + + /* All c[i]==e → outer cond hits "continue" each iter, no break. */ + reset (); + a[0] = 5; c[0] = 7; /* c==e (e=7) — continue */ + a[1] = 6; c[1] = 7; + a[2] = 8; c[2] = 7; + f (1, 7); + CHECK (0, 5); CHECK (1, 6); CHECK (2, 8); + + /* break at index 1. */ + reset (); + a[0] = 1; c[0] = 0; /* a==d, no break */ + a[1] = 9; c[1] = 0; /* a!=d AND c!=e → break here */ + a[2] = 0; c[2] = 0; + f (1, 7); + CHECK (0, 1); CHECK (1, 9); CHECK (2, 0); +}
