https://gcc.gnu.org/g:8d7b042f7c9092e168d2c745bdc3403061e41466
commit r17-1152-g8d7b042f7c9092e168d2c745bdc3403061e41466 Author: Tianle Shi <[email protected]> Date: Tue Jun 2 01:05:02 2026 +0000 i386: Add avx512_avoid_vec_perm tune for c86-4g-m7 This implements the AVX512 avoid-vec-perm cost model for c86-4g-m7 following the existing AVX256 support r15-4234-g9eaecce3d8c1d9. Extend the vectorizer cost model to treat 512-bit cross-lane vec_perm as unprofitable when this tune is enabled, since c86-4g-m7 implements AVX512 as two 256-bit halves and cross-lane reorder is slower than staying in AVX256. Factor out the cross-lane permutation classification into a helper so the AVX256 and AVX512 cost handling can share the same logic without duplicating it. gcc/ChangeLog: * config/i386/i386.cc (ix86_vector_costs): Add m_num_avx512_vec_perm. (ix86_vector_costs::ix86_vector_costs): Initialize it. (ix86_count_cross_lane_perm_p): New helper to classify cross-lane vec_perm for AVX256 and AVX512 handling. (ix86_vector_costs::add_stmt_cost): Use the helper for vec_perm accounting, record 256-bit and 512-bit counts, and print detected width as avx%u. (ix86_vector_costs::finish_cost): Reject AVX512 costs when TARGET_AVX512_AVOID_VEC_PERM is set and 512-bit cross-lane vec_perm is present. * config/i386/i386.h (TARGET_AVX512_AVOID_VEC_PERM): New macro. * config/i386/x86-tune.def (X86_TUNE_AVX512_AVOID_VEC_PERM): New tune for c86-4g-m7. gcc/testsuite/ChangeLog: * gcc.target/i386/avx512_avoid_vec_perm-1.c: New test. * gcc.target/i386/avx512_avoid_vec_perm-2.c: New test. * gcc.target/i386/avx512_avoid_vec_perm-3.c: New test. Signed-off-by: Tianle Shi <[email protected]> Diff: --- gcc/config/i386/i386.cc | 105 ++++++++++++--------- gcc/config/i386/i386.h | 2 + gcc/config/i386/x86-tune.def | 4 + .../gcc.target/i386/avx512_avoid_vec_perm-1.c | 27 ++++++ .../gcc.target/i386/avx512_avoid_vec_perm-2.c | 25 +++++ .../gcc.target/i386/avx512_avoid_vec_perm-3.c | 32 +++++++ 6 files changed, 150 insertions(+), 45 deletions(-) diff --git a/gcc/config/i386/i386.cc b/gcc/config/i386/i386.cc index 22980411180e..a1959f8170a3 100644 --- a/gcc/config/i386/i386.cc +++ b/gcc/config/i386/i386.cc @@ -26183,6 +26183,8 @@ private: unsigned m_num_sse_needed[3]; /* Number of 256-bit vector permutation. */ unsigned m_num_avx256_vec_perm[3]; + /* Number of 512-bit vector permutation. */ + unsigned m_num_avx512_vec_perm[3]; /* Number of reductions for FMA/DOT_PROD_EXPR/SAD_EXPR */ unsigned m_num_reduc[X86_REDUC_LAST]; /* Don't do unroll if m_prefer_unroll is false, default is true. */ @@ -26194,6 +26196,7 @@ ix86_vector_costs::ix86_vector_costs (vec_info* vinfo, bool costing_for_scalar) m_num_gpr_needed (), m_num_sse_needed (), m_num_avx256_vec_perm (), + m_num_avx512_vec_perm (), m_num_reduc (), m_prefer_unroll (true) {} @@ -26206,6 +26209,47 @@ ix86_vectorize_create_costs (vec_info *vinfo, bool costing_for_scalar) return new ix86_vector_costs (vinfo, costing_for_scalar); } +/* Return true if a vec_perm should be counted as a cross-lane vector + permutation for a vector with NUNITS elements. */ +static bool +ix86_count_cross_lane_perm_p (vec_info *vinfo, slp_tree node, unsigned nunits) +{ + /* TODO: For loop vectorization with no SLP load-permutation + information, conservatively treat these perms as cross-lane. + Repeated-index cases such as {0, 0, 0, 0} are emitted as + separate vec_perm_exprs for each index, so we cannot reliably + separate false positives from real cross-lane shuffles yet. */ + if (!node + || !SLP_TREE_LOAD_PERMUTATION (node).exists () + || !is_a<bb_vec_info> (vinfo)) + return true; + + unsigned half = nunits / 2; + bool allsame = true; + unsigned first = SLP_TREE_LOAD_PERMUTATION (node)[0]; + bool cross_lane_p = false; + + for (unsigned i = 0; i != SLP_TREE_LANES (node); i++) + { + unsigned tmp = SLP_TREE_LOAD_PERMUTATION (node)[i]; + /* allsame is just a broadcast. */ + if (tmp != first) + allsame = false; + + /* The load permutation can cover multiple vectors, so compare + source and destination lanes modulo NUNITS. */ + tmp = tmp & (nunits - 1); + unsigned index = i & (nunits - 1); + if ((index < half && tmp >= half) || (index >= half && tmp < half)) + cross_lane_p = true; + + if (!allsame && cross_lane_p) + return true; + } + + return false; +} + unsigned ix86_vector_costs::add_stmt_cost (int count, vect_cost_for_stmt kind, stmt_vec_info stmt_info, slp_tree node, @@ -26771,58 +26815,25 @@ ix86_vector_costs::add_stmt_cost (int count, vect_cost_for_stmt kind, if (stmt_cost == -1) stmt_cost = ix86_default_vector_cost (kind, mode); - if (kind == vec_perm && vectype - && GET_MODE_SIZE (TYPE_MODE (vectype)) == 32 - /* BIT_FIELD_REF <vect_**, 64, 0> 0 times vec_perm costs 0 in body. */ - && count != 0) + /* BIT_FIELD_REF <vect_**, 64, 0> with count 0 costs 0 in body. */ + if (kind == vec_perm && vectype && count != 0) { - bool real_perm = true; + unsigned vec_size = GET_MODE_SIZE (TYPE_MODE (vectype)); unsigned nunits = TYPE_VECTOR_SUBPARTS (vectype); + unsigned *num_vec_perm = NULL; - if (node - && SLP_TREE_LOAD_PERMUTATION (node).exists () - /* Loop vectorization will have 4 times vec_perm - with index as {0, 0, 0, 0}. - But it actually generates - vec_perm_expr <vect, vect, 0, 0, 0, 0> - vec_perm_expr <vect, vect, 1, 1, 1, 1> - vec_perm_expr <vect, vect, 2, 2, 2, 2> - Need to be handled separately. */ - && is_a <bb_vec_info> (m_vinfo)) - { - unsigned half = nunits / 2; - unsigned i = 0; - bool allsame = true; - unsigned first = SLP_TREE_LOAD_PERMUTATION (node)[0]; - bool cross_lane_p = false; - for (i = 0 ; i != SLP_TREE_LANES (node); i++) - { - unsigned tmp = SLP_TREE_LOAD_PERMUTATION (node)[i]; - /* allsame is just a broadcast. */ - if (tmp != first) - allsame = false; - - /* 4 times vec_perm with number of lanes multiple of nunits. */ - tmp = tmp & (nunits - 1); - unsigned index = i & (nunits - 1); - if ((index < half && tmp >= half) - || (index >= half && tmp < half)) - cross_lane_p = true; - - if (!allsame && cross_lane_p) - break; - } - - if (i == SLP_TREE_LANES (node)) - real_perm = false; - } + if (vec_size == 32) + num_vec_perm = m_num_avx256_vec_perm; + else if (vec_size == 64) + num_vec_perm = m_num_avx512_vec_perm; - if (real_perm) + if (num_vec_perm && ix86_count_cross_lane_perm_p (m_vinfo, node, nunits)) { - m_num_avx256_vec_perm[where] += count; + num_vec_perm[where] += count; if (dump_file && (dump_flags & TDF_DETAILS)) { - fprintf (dump_file, "Detected avx256 cross-lane permutation: "); + fprintf (dump_file, + "Detected avx%u cross-lane permutation: ", vec_size * 8); if (stmt_info) print_gimple_expr (dump_file, stmt_info->stmt, 0, TDF_SLIM); fprintf (dump_file, " \n"); @@ -26945,6 +26956,10 @@ ix86_vector_costs::finish_cost (const vector_costs *scalar_costs) && TARGET_AVX256_AVOID_VEC_PERM) m_costs[i] = INT_MAX; + for (int i = 0; i != 3; i++) + if (m_num_avx512_vec_perm[i] && TARGET_AVX512_AVOID_VEC_PERM) + m_costs[i] = INT_MAX; + /* When X86_TUNE_AVX512_TWO_EPILOGUES is enabled arrange for both a AVX2 and a SSE epilogue for AVX512 vectorized loops. */ if (loop_vinfo diff --git a/gcc/config/i386/i386.h b/gcc/config/i386/i386.h index c4befa06ba6b..375ba82f5ff7 100644 --- a/gcc/config/i386/i386.h +++ b/gcc/config/i386/i386.h @@ -480,6 +480,8 @@ extern unsigned char ix86_tune_features[X86_TUNE_LAST]; ix86_tune_features[X86_TUNE_AVX256_AVOID_VEC_PERM] #define TARGET_AVX512_SPLIT_REGS \ ix86_tune_features[X86_TUNE_AVX512_SPLIT_REGS] +#define TARGET_AVX512_AVOID_VEC_PERM \ + ix86_tune_features[X86_TUNE_AVX512_AVOID_VEC_PERM] #define TARGET_GENERAL_REGS_SSE_SPILL \ ix86_tune_features[X86_TUNE_GENERAL_REGS_SSE_SPILL] #define TARGET_AVOID_MEM_OPND_FOR_CMOVE \ diff --git a/gcc/config/i386/x86-tune.def b/gcc/config/i386/x86-tune.def index 5b208dc7372a..2ef962b2ee0c 100644 --- a/gcc/config/i386/x86-tune.def +++ b/gcc/config/i386/x86-tune.def @@ -645,6 +645,10 @@ DEF_TUNE (X86_TUNE_AVX256_AVOID_VEC_PERM, DEF_TUNE (X86_TUNE_AVX512_SPLIT_REGS, "avx512_split_regs", m_ZNVER4 | m_C86_4G_M7) +/* X86_TUNE_AVX512_AVOID_VEC_PERM: Avoid using 512-bit cross-lane + vector permutation instructions in the auto-vectorizer. */ +DEF_TUNE (X86_TUNE_AVX512_AVOID_VEC_PERM, "avx512_avoid_vec_perm", m_C86_4G_M7) + /* It's better to align MOVE_MAX with prefer_vector_width to reduce risk of STLF stalls(small store followed by big load.) */ /* X86_TUNE_AVX256_MOVE_BY_PIECES: Optimize move_by_pieces with 256-bit diff --git a/gcc/testsuite/gcc.target/i386/avx512_avoid_vec_perm-1.c b/gcc/testsuite/gcc.target/i386/avx512_avoid_vec_perm-1.c new file mode 100644 index 000000000000..bc8226e0a222 --- /dev/null +++ b/gcc/testsuite/gcc.target/i386/avx512_avoid_vec_perm-1.c @@ -0,0 +1,27 @@ +/* { dg-do compile } */ +/* { dg-options "-O2 -march=c86-4g-m7 -fdump-tree-vect-details" } */ +/* { dg-final { scan-tree-dump "loop vectorized using 32 byte vectors" "vect" } } */ + +int a[512], b[512]; + +void foo(void) { + int i; + for (i = 0; i < 32; ++i) { + b[i * 16 + 0] = a[i * 16 + 0]; + b[i * 16 + 1] = a[i * 16 + 0]; + b[i * 16 + 2] = a[i * 16 + 3]; + b[i * 16 + 3] = a[i * 16 + 5]; + b[i * 16 + 4] = a[i * 16 + 4]; + b[i * 16 + 5] = a[i * 16 + 6]; + b[i * 16 + 6] = a[i * 16 + 4]; + b[i * 16 + 7] = a[i * 16 + 9]; + b[i * 16 + 8] = a[i * 16 + 8]; + b[i * 16 + 9] = a[i * 16 + 8]; + b[i * 16 + 10] = a[i * 16 + 11]; + b[i * 16 + 11] = a[i * 16 + 13]; + b[i * 16 + 12] = a[i * 16 + 12]; + b[i * 16 + 13] = a[i * 16 + 14]; + b[i * 16 + 14] = a[i * 16 + 12]; + b[i * 16 + 15] = a[i * 16 + 14]; + } +} diff --git a/gcc/testsuite/gcc.target/i386/avx512_avoid_vec_perm-2.c b/gcc/testsuite/gcc.target/i386/avx512_avoid_vec_perm-2.c new file mode 100644 index 000000000000..fd2dd4c1e7fb --- /dev/null +++ b/gcc/testsuite/gcc.target/i386/avx512_avoid_vec_perm-2.c @@ -0,0 +1,25 @@ +/* { dg-do compile } */ +/* { dg-options "-march=c86-4g-m7 -O2 -fdump-tree-slp-details" } */ +/* { dg-final { scan-tree-dump-times {(?n)Detected avx512 cross-lane permutation} 1 "slp2" } } */ + +void foo(double *a, double *__restrict b, int c, int n) { + a[0] = b[100] * b[4]; + a[1] = b[100] * b[5]; + a[2] = b[100] * b[6]; + a[3] = b[100] * b[7]; + a[4] = b[100] * b[0]; + a[5] = b[100] * b[1]; + a[6] = b[100] * b[2]; + a[7] = b[100] * b[3]; +} + +void foo1(double *a, double *__restrict b, int c, int n) { + a[0] = b[100] * b[0]; + a[1] = b[100] * b[1]; + a[2] = b[100] * b[3]; + a[3] = b[100] * b[2]; + a[4] = b[100] * b[4]; + a[5] = b[100] * b[5]; + a[6] = b[100] * b[7]; + a[7] = b[100] * b[6]; +} diff --git a/gcc/testsuite/gcc.target/i386/avx512_avoid_vec_perm-3.c b/gcc/testsuite/gcc.target/i386/avx512_avoid_vec_perm-3.c new file mode 100644 index 000000000000..5988865b6b82 --- /dev/null +++ b/gcc/testsuite/gcc.target/i386/avx512_avoid_vec_perm-3.c @@ -0,0 +1,32 @@ +/* { dg-do compile } */ +/* { dg-options "-O2 -march=c86-4g-m7 -fdump-tree-vect-details" } */ +/* { dg-final { scan-tree-dump "loop vectorized using 64 byte vectors" "vect" { xfail *-*-* } } } */ + +/* This case only has false cross-lane permutations, which should ideally be + excluded from the loop vectorizer cost downgrade. The current loop-vect + handling does not distinguish false cross-lane permutations from real ones, + so the expected full-width vectorization is marked as xfail. */ + +int a[512], b[512]; + +void foo(void) { + int i; + for (i = 0; i < 32; ++i) { + b[i * 16 + 0] = a[i * 16 + 0]; + b[i * 16 + 1] = a[i * 16 + 0]; + b[i * 16 + 2] = a[i * 16 + 3]; + b[i * 16 + 3] = a[i * 16 + 3]; + b[i * 16 + 4] = a[i * 16 + 4]; + b[i * 16 + 5] = a[i * 16 + 6]; + b[i * 16 + 6] = a[i * 16 + 4]; + b[i * 16 + 7] = a[i * 16 + 6]; + b[i * 16 + 8] = a[i * 16 + 8]; + b[i * 16 + 9] = a[i * 16 + 8]; + b[i * 16 + 10] = a[i * 16 + 11]; + b[i * 16 + 11] = a[i * 16 + 11]; + b[i * 16 + 12] = a[i * 16 + 12]; + b[i * 16 + 13] = a[i * 16 + 14]; + b[i * 16 + 14] = a[i * 16 + 12]; + b[i * 16 + 15] = a[i * 16 + 14]; + } +}
