Hi,
This implements the AVX512 avoid-vec-perm cost model for c86-4g-m7
following the existing AVX256 support r15-4234-g9eaecce3d8c1d9.
Extend the vectorizer cost model to treat 512-bit cross-lane vec_perm
as unprofitable when this tune is enabled, since c86-4g-m7 implements
AVX512 as two 256-bit halves and cross-lane reorder is slower than
staying in AVX256.
Factor out the cross-lane permutation classification into a helper so
the AVX256 and AVX512 cost handling can share the same logic without
duplicating it.
Bootstrapped and regress-tested on one c86-4g-m7 machine, is it ok
for trunk?
BR,
Kewen
-----
gcc/ChangeLog:
* config/i386/i386.cc (ix86_vector_costs): Add
m_num_avx512_vec_perm.
(ix86_vector_costs::ix86_vector_costs): Initialize it.
(ix86_count_cross_lane_perm_p): New helper to classify
cross-lane vec_perm for AVX256 and AVX512 handling.
(ix86_vector_costs::add_stmt_cost): Use the helper for
vec_perm accounting, record 256-bit and 512-bit counts,
and print detected width as avx%u.
(ix86_vector_costs::finish_cost): Reject AVX512 costs
when TARGET_AVX512_AVOID_VEC_PERM is set and 512-bit
cross-lane vec_perm is present.
* config/i386/i386.h (TARGET_AVX512_AVOID_VEC_PERM):
New macro.
* config/i386/x86-tune.def (X86_TUNE_AVX512_AVOID_VEC_PERM):
New tune for c86-4g-m7.
gcc/testsuite/ChangeLog:
* gcc.target/i386/avx512_avoid_vec_perm-1.c: New test.
* gcc.target/i386/avx512_avoid_vec_perm-2.c: New test.
* gcc.target/i386/avx512_avoid_vec_perm-3.c: New test.
Authored-by: shitianle <[email protected]>
Signed-off-by: shitianle <[email protected]>
---
gcc/config/i386/i386.cc | 105 ++++++++++--------
gcc/config/i386/i386.h | 2 +
gcc/config/i386/x86-tune.def | 4 +
.../gcc.target/i386/avx512_avoid_vec_perm-1.c | 27 +++++
.../gcc.target/i386/avx512_avoid_vec_perm-2.c | 25 +++++
.../gcc.target/i386/avx512_avoid_vec_perm-3.c | 32 ++++++
6 files changed, 150 insertions(+), 45 deletions(-)
create mode 100644 gcc/testsuite/gcc.target/i386/avx512_avoid_vec_perm-1.c
create mode 100644 gcc/testsuite/gcc.target/i386/avx512_avoid_vec_perm-2.c
create mode 100644 gcc/testsuite/gcc.target/i386/avx512_avoid_vec_perm-3.c
diff --git a/gcc/config/i386/i386.cc b/gcc/config/i386/i386.cc
index 9148ac1e057..71781210fe2 100644
--- a/gcc/config/i386/i386.cc
+++ b/gcc/config/i386/i386.cc
@@ -26183,6 +26183,8 @@ private:
unsigned m_num_sse_needed[3];
/* Number of 256-bit vector permutation. */
unsigned m_num_avx256_vec_perm[3];
+ /* Number of 512-bit vector permutation. */
+ unsigned m_num_avx512_vec_perm[3];
/* Number of reductions for FMA/DOT_PROD_EXPR/SAD_EXPR */
unsigned m_num_reduc[X86_REDUC_LAST];
/* Don't do unroll if m_prefer_unroll is false, default is true. */
@@ -26194,6 +26196,7 @@ ix86_vector_costs::ix86_vector_costs (vec_info* vinfo,
bool costing_for_scalar)
m_num_gpr_needed (),
m_num_sse_needed (),
m_num_avx256_vec_perm (),
+ m_num_avx512_vec_perm (),
m_num_reduc (),
m_prefer_unroll (true)
{}
@@ -26206,6 +26209,47 @@ ix86_vectorize_create_costs (vec_info *vinfo, bool
costing_for_scalar)
return new ix86_vector_costs (vinfo, costing_for_scalar);
}
+/* Return true if a vec_perm should be counted as a cross-lane vector
+ permutation for a vector with NUNITS elements. */
+static bool
+ix86_count_cross_lane_perm_p (vec_info *vinfo, slp_tree node, unsigned nunits)
+{
+ /* TODO: For loop vectorization with no SLP load-permutation
+ information, conservatively treat these perms as cross-lane.
+ Repeated-index cases such as {0, 0, 0, 0} are emitted as
+ separate vec_perm_exprs for each index, so we cannot reliably
+ separate false positives from real cross-lane shuffles yet. */
+ if (!node
+ || !SLP_TREE_LOAD_PERMUTATION (node).exists ()
+ || !is_a<bb_vec_info> (vinfo))
+ return true;
+
+ unsigned half = nunits / 2;
+ bool allsame = true;
+ unsigned first = SLP_TREE_LOAD_PERMUTATION (node)[0];
+ bool cross_lane_p = false;
+
+ for (unsigned i = 0; i != SLP_TREE_LANES (node); i++)
+ {
+ unsigned tmp = SLP_TREE_LOAD_PERMUTATION (node)[i];
+ /* allsame is just a broadcast. */
+ if (tmp != first)
+ allsame = false;
+
+ /* The load permutation can cover multiple vectors, so compare
+ source and destination lanes modulo NUNITS. */
+ tmp = tmp & (nunits - 1);
+ unsigned index = i & (nunits - 1);
+ if ((index < half && tmp >= half) || (index >= half && tmp < half))
+ cross_lane_p = true;
+
+ if (!allsame && cross_lane_p)
+ return true;
+ }
+
+ return false;
+}
+
unsigned
ix86_vector_costs::add_stmt_cost (int count, vect_cost_for_stmt kind,
stmt_vec_info stmt_info, slp_tree node,
@@ -26771,58 +26815,25 @@ ix86_vector_costs::add_stmt_cost (int count,
vect_cost_for_stmt kind,
if (stmt_cost == -1)
stmt_cost = ix86_default_vector_cost (kind, mode);
- if (kind == vec_perm && vectype
- && GET_MODE_SIZE (TYPE_MODE (vectype)) == 32
- /* BIT_FIELD_REF <vect_**, 64, 0> 0 times vec_perm costs 0 in body. */
- && count != 0)
+ /* BIT_FIELD_REF <vect_**, 64, 0> with count 0 costs 0 in body. */
+ if (kind == vec_perm && vectype && count != 0)
{
- bool real_perm = true;
+ unsigned vec_size = GET_MODE_SIZE (TYPE_MODE (vectype));
unsigned nunits = TYPE_VECTOR_SUBPARTS (vectype);
+ unsigned *num_vec_perm = NULL;
- if (node
- && SLP_TREE_LOAD_PERMUTATION (node).exists ()
- /* Loop vectorization will have 4 times vec_perm
- with index as {0, 0, 0, 0}.
- But it actually generates
- vec_perm_expr <vect, vect, 0, 0, 0, 0>
- vec_perm_expr <vect, vect, 1, 1, 1, 1>
- vec_perm_expr <vect, vect, 2, 2, 2, 2>
- Need to be handled separately. */
- && is_a <bb_vec_info> (m_vinfo))
- {
- unsigned half = nunits / 2;
- unsigned i = 0;
- bool allsame = true;
- unsigned first = SLP_TREE_LOAD_PERMUTATION (node)[0];
- bool cross_lane_p = false;
- for (i = 0 ; i != SLP_TREE_LANES (node); i++)
- {
- unsigned tmp = SLP_TREE_LOAD_PERMUTATION (node)[i];
- /* allsame is just a broadcast. */
- if (tmp != first)
- allsame = false;
-
- /* 4 times vec_perm with number of lanes multiple of nunits. */
- tmp = tmp & (nunits - 1);
- unsigned index = i & (nunits - 1);
- if ((index < half && tmp >= half)
- || (index >= half && tmp < half))
- cross_lane_p = true;
-
- if (!allsame && cross_lane_p)
- break;
- }
-
- if (i == SLP_TREE_LANES (node))
- real_perm = false;
- }
+ if (vec_size == 32)
+ num_vec_perm = m_num_avx256_vec_perm;
+ else if (vec_size == 64)
+ num_vec_perm = m_num_avx512_vec_perm;
- if (real_perm)
+ if (num_vec_perm && ix86_count_cross_lane_perm_p (m_vinfo, node, nunits))
{
- m_num_avx256_vec_perm[where] += count;
+ num_vec_perm[where] += count;
if (dump_file && (dump_flags & TDF_DETAILS))
{
- fprintf (dump_file, "Detected avx256 cross-lane permutation: ");
+ fprintf (dump_file,
+ "Detected avx%u cross-lane permutation: ", vec_size * 8);
if (stmt_info)
print_gimple_expr (dump_file, stmt_info->stmt, 0, TDF_SLIM);
fprintf (dump_file, " \n");
@@ -26945,6 +26956,10 @@ ix86_vector_costs::finish_cost (const vector_costs
*scalar_costs)
&& TARGET_AVX256_AVOID_VEC_PERM)
m_costs[i] = INT_MAX;
+ for (int i = 0; i != 3; i++)
+ if (m_num_avx512_vec_perm[i] && TARGET_AVX512_AVOID_VEC_PERM)
+ m_costs[i] = INT_MAX;
+
/* When X86_TUNE_AVX512_TWO_EPILOGUES is enabled arrange for both
a AVX2 and a SSE epilogue for AVX512 vectorized loops. */
if (loop_vinfo
diff --git a/gcc/config/i386/i386.h b/gcc/config/i386/i386.h
index ac3f2213e78..a5d20c263ff 100644
--- a/gcc/config/i386/i386.h
+++ b/gcc/config/i386/i386.h
@@ -480,6 +480,8 @@ extern unsigned char ix86_tune_features[X86_TUNE_LAST];
ix86_tune_features[X86_TUNE_AVX256_AVOID_VEC_PERM]
#define TARGET_AVX512_SPLIT_REGS \
ix86_tune_features[X86_TUNE_AVX512_SPLIT_REGS]
+#define TARGET_AVX512_AVOID_VEC_PERM \
+ ix86_tune_features[X86_TUNE_AVX512_AVOID_VEC_PERM]
#define TARGET_GENERAL_REGS_SSE_SPILL \
ix86_tune_features[X86_TUNE_GENERAL_REGS_SSE_SPILL]
#define TARGET_AVOID_MEM_OPND_FOR_CMOVE \
diff --git a/gcc/config/i386/x86-tune.def b/gcc/config/i386/x86-tune.def
index 202b38ad50b..81388cc690f 100644
--- a/gcc/config/i386/x86-tune.def
+++ b/gcc/config/i386/x86-tune.def
@@ -638,6 +638,10 @@ DEF_TUNE (X86_TUNE_AVX256_AVOID_VEC_PERM,
DEF_TUNE (X86_TUNE_AVX512_SPLIT_REGS, "avx512_split_regs", m_ZNVER4
| m_C86_4G_M7)
+/* X86_TUNE_AVX512_AVOID_VEC_PERM: Avoid using 512-bit cross-lane
+ vector permutation instructions in the auto-vectorizer. */
+DEF_TUNE (X86_TUNE_AVX512_AVOID_VEC_PERM, "avx512_avoid_vec_perm", m_C86_4G_M7)
+
/* It's better to align MOVE_MAX with prefer_vector_width to reduce
risk of STLF stalls(small store followed by big load.) */
/* X86_TUNE_AVX256_MOVE_BY_PIECES: Optimize move_by_pieces with 256-bit
diff --git a/gcc/testsuite/gcc.target/i386/avx512_avoid_vec_perm-1.c
b/gcc/testsuite/gcc.target/i386/avx512_avoid_vec_perm-1.c
new file mode 100644
index 00000000000..bc8226e0a22
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/avx512_avoid_vec_perm-1.c
@@ -0,0 +1,27 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=c86-4g-m7 -fdump-tree-vect-details" } */
+/* { dg-final { scan-tree-dump "loop vectorized using 32 byte vectors" "vect"
} } */
+
+int a[512], b[512];
+
+void foo(void) {
+ int i;
+ for (i = 0; i < 32; ++i) {
+ b[i * 16 + 0] = a[i * 16 + 0];
+ b[i * 16 + 1] = a[i * 16 + 0];
+ b[i * 16 + 2] = a[i * 16 + 3];
+ b[i * 16 + 3] = a[i * 16 + 5];
+ b[i * 16 + 4] = a[i * 16 + 4];
+ b[i * 16 + 5] = a[i * 16 + 6];
+ b[i * 16 + 6] = a[i * 16 + 4];
+ b[i * 16 + 7] = a[i * 16 + 9];
+ b[i * 16 + 8] = a[i * 16 + 8];
+ b[i * 16 + 9] = a[i * 16 + 8];
+ b[i * 16 + 10] = a[i * 16 + 11];
+ b[i * 16 + 11] = a[i * 16 + 13];
+ b[i * 16 + 12] = a[i * 16 + 12];
+ b[i * 16 + 13] = a[i * 16 + 14];
+ b[i * 16 + 14] = a[i * 16 + 12];
+ b[i * 16 + 15] = a[i * 16 + 14];
+ }
+}
diff --git a/gcc/testsuite/gcc.target/i386/avx512_avoid_vec_perm-2.c
b/gcc/testsuite/gcc.target/i386/avx512_avoid_vec_perm-2.c
new file mode 100644
index 00000000000..fd2dd4c1e7f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/avx512_avoid_vec_perm-2.c
@@ -0,0 +1,25 @@
+/* { dg-do compile } */
+/* { dg-options "-march=c86-4g-m7 -O2 -fdump-tree-slp-details" } */
+/* { dg-final { scan-tree-dump-times {(?n)Detected avx512 cross-lane
permutation} 1 "slp2" } } */
+
+void foo(double *a, double *__restrict b, int c, int n) {
+ a[0] = b[100] * b[4];
+ a[1] = b[100] * b[5];
+ a[2] = b[100] * b[6];
+ a[3] = b[100] * b[7];
+ a[4] = b[100] * b[0];
+ a[5] = b[100] * b[1];
+ a[6] = b[100] * b[2];
+ a[7] = b[100] * b[3];
+}
+
+void foo1(double *a, double *__restrict b, int c, int n) {
+ a[0] = b[100] * b[0];
+ a[1] = b[100] * b[1];
+ a[2] = b[100] * b[3];
+ a[3] = b[100] * b[2];
+ a[4] = b[100] * b[4];
+ a[5] = b[100] * b[5];
+ a[6] = b[100] * b[7];
+ a[7] = b[100] * b[6];
+}
diff --git a/gcc/testsuite/gcc.target/i386/avx512_avoid_vec_perm-3.c
b/gcc/testsuite/gcc.target/i386/avx512_avoid_vec_perm-3.c
new file mode 100644
index 00000000000..5988865b6b8
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/avx512_avoid_vec_perm-3.c
@@ -0,0 +1,32 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=c86-4g-m7 -fdump-tree-vect-details" } */
+/* { dg-final { scan-tree-dump "loop vectorized using 64 byte vectors" "vect"
{ xfail *-*-* } } } */
+
+/* This case only has false cross-lane permutations, which should ideally be
+ excluded from the loop vectorizer cost downgrade. The current loop-vect
+ handling does not distinguish false cross-lane permutations from real ones,
+ so the expected full-width vectorization is marked as xfail. */
+
+int a[512], b[512];
+
+void foo(void) {
+ int i;
+ for (i = 0; i < 32; ++i) {
+ b[i * 16 + 0] = a[i * 16 + 0];
+ b[i * 16 + 1] = a[i * 16 + 0];
+ b[i * 16 + 2] = a[i * 16 + 3];
+ b[i * 16 + 3] = a[i * 16 + 3];
+ b[i * 16 + 4] = a[i * 16 + 4];
+ b[i * 16 + 5] = a[i * 16 + 6];
+ b[i * 16 + 6] = a[i * 16 + 4];
+ b[i * 16 + 7] = a[i * 16 + 6];
+ b[i * 16 + 8] = a[i * 16 + 8];
+ b[i * 16 + 9] = a[i * 16 + 8];
+ b[i * 16 + 10] = a[i * 16 + 11];
+ b[i * 16 + 11] = a[i * 16 + 11];
+ b[i * 16 + 12] = a[i * 16 + 12];
+ b[i * 16 + 13] = a[i * 16 + 14];
+ b[i * 16 + 14] = a[i * 16 + 12];
+ b[i * 16 + 15] = a[i * 16 + 14];
+ }
+}
--
2.53.0
