https://gcc.gnu.org/g:f75c34a0728f11297cc7e4c9aedcb01493894cef

commit r17-6055-gf75c34a0728f11297cc7e4c9aedcb01493894cef
Author: Tamar Christina <[email protected]>
Date:   Mon Nov 9 10:35:23 2026 +0000

    AArch64: model throughput dependencies of unpacks
    
    When costing a conversion through several integer element sizes, we 
currently
    count all the unpack instructions together.  This loses the dependency 
between
    the conversion stages.
    
    For example, a QI -> SI conversion goes through
    
      QI -> HI -> SI
    
    With a full byte vector, the first step can need two unpacks and the second
    four.  The four unpacks in the second step need the results of the first 
two, so
    simply counting six operations doesn't describe how they can issue.
    
    This patch adds a small approximation to the AArch64 throughput model.  When
    costing successive integer unpack stages, it rounds the preceding stage's
    instruction count up to a whole number of issue cycles, using the model's
    general operation issue rate.
    
    On a core that issues four general operations per cycle, the two-unpack 
stage
    above therefore gets two additional issue slots.  Together with the six 
actual
    instructions, that gives eight slots, or two cycles in the throughput 
estimate.
    
    The preceding unpack count is kept separately for each issue model and is 
reset
    when costing another kind of operation.  This approximates the stage 
dependency
    using the operation counts that the throughput model already tracks.
    
    The patch also counts operations on SVE boolean vectors against predicate
    throughput.  These were previously included in the general operation count, 
even
    though they execute on predicate registers.
    
    The tests use
    
    void
    widen_add (unsigned long *restrict dst,
               const unsigned char *restrict src,
               unsigned bias, unsigned n)
    {
      for (unsigned i = 0; i < n; ++i)
        dst[i] = (unsigned) src[i] + bias;
    }
    
    Here the byte input is widened to unsigned before the addition, giving the
    two-stage conversion above.  At -O3 -mcpu=neoverse-v3 the tests check that 
the
    chosen mode avoids integer unpacks, and that forcing VNx16QI accounts for 
the
    two extra issue slots.  The later SI -> DI conversion is a separate stage 
and
    needs no extra slots.
    
    gcc/ChangeLog:
    
            * config/aarch64/aarch64.cc
            (aarch64_vec_op_count::last_unpack_count): New member.
            (aarch64_vector_costs::count_ops): Track successive integer unpack
            stages for each issue model.  Round the preceding stage's count up
            to a whole number of issue cycles using general_ops_per_cycle.
            Reset the unpack count for other operations.  Count operations on
            SVE boolean vectors as predicate operations.
    
    gcc/testsuite/ChangeLog:
    
            * gcc.target/aarch64/sve/cost_model_21.c: New test.
            * gcc.target/aarch64/sve/cost_model_22.c: New test.

Diff:
---
 gcc/config/aarch64/aarch64.cc                      | 39 +++++++++++++++++++++-
 .../gcc.target/aarch64/sve/cost_model_21.c         | 15 +++++++++
 .../gcc.target/aarch64/sve/cost_model_22.c         | 11 ++++++
 3 files changed, 64 insertions(+), 1 deletion(-)

diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc
index afe817408475..730a1fee8359 100644
--- a/gcc/config/aarch64/aarch64.cc
+++ b/gcc/config/aarch64/aarch64.cc
@@ -17598,6 +17598,9 @@ public:
      in aarch64_base_vec_issue_info for details.  */
   unsigned int general_ops = 0;
 
+  /* The instruction count of the preceding integer unpack stage.  */
+  unsigned int last_unpack_count = 0;
+
   /* The number of load and store operations, under the same scheme
      as above.  */
   unsigned int loads = 0;
@@ -18708,6 +18711,32 @@ aarch64_vector_costs::count_ops (unsigned int count, 
vect_cost_for_stmt kind,
   const aarch64_simd_vec_issue_info *simd_issue = ops->simd_issue_info ();
   const aarch64_sve_vec_issue_info *sve_issue = ops->sve_issue_info ();
 
+  /* Multi-step integer conversions create a dependency chain.  For example,
+     a QI -> SI conversion goes through QI -> HI -> SI, and the vectorizer
+     might cost these stages as 2 unpacks followed by 4 unpacks.  Counting
+     just the 6 operations makes them look as though they could all issue
+     in one cycle on a wide enough core.  However, the second stage needs
+     the results of the first, so they cannot all issue together.
+
+     Our issue model only counts operations and does not track these
+     dependencies.  Approximate their effect by rounding the previous
+     stage up to a whole number of issue cycles.  For example, on a core
+     that can issue 4 general operations per cycle, add 2 slots for the
+     first stage above.  Together with the 6 actual unpacks, this gives
+     an estimate of 2 cycles.  Use each model's general issue rate here,
+     and keep the original unpack count for costing the next stage.  */
+  if (kind == vec_promote_demote && node && stmt_info
+      && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (stmt_info->stmt))
+      && TREE_CODE (TREE_TYPE (SLP_TREE_VECTYPE (node))) == INTEGER_TYPE)
+    {
+      unsigned int width = base_issue->general_ops_per_cycle;
+      ops->general_ops += (CEIL (ops->last_unpack_count, width) * width
+                         - ops->last_unpack_count);
+      ops->last_unpack_count = count;
+    }
+  else
+    ops->last_unpack_count = 0;
+
   /* Calculate the minimum cycles per iteration imposed by a reduction
      operation.  */
   if (stmt_info
@@ -18806,7 +18835,15 @@ aarch64_vector_costs::count_ops (unsigned int count, 
vect_cost_for_stmt kind,
     case scalar_to_vec:
     case vector_stmt:
     case scalar_stmt:
-      ops->general_ops += count;
+      /* Count predicate conversions on the predicate issue units.  */
+      if (sve_issue
+         && kind == vec_promote_demote
+         && node
+         && SLP_TREE_VECTYPE (node)
+         && VECTOR_BOOLEAN_TYPE_P (SLP_TREE_VECTYPE (node)))
+       ops->pred_ops += count;
+      else
+       ops->general_ops += count;
       break;
 
     case scalar_load:
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/cost_model_21.c 
b/gcc/testsuite/gcc.target/aarch64/sve/cost_model_21.c
new file mode 100644
index 000000000000..1a0f6b5c75a4
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/cost_model_21.c
@@ -0,0 +1,15 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -mcpu=neoverse-v3 -mautovec-preference=sve-only 
-msve-vector-bits=scalable -fdump-tree-vect-details" } */
+
+/* Widening through several element sizes creates dependent unpack stages.
+   Prefer widening loads to a chain of unpack instructions.  */
+void
+widen_add (unsigned long *restrict dst, const unsigned char *restrict src,
+          unsigned bias, unsigned n)
+{
+  for (unsigned i = 0; i < n; ++i)
+    dst[i] = (unsigned) src[i] + bias;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops in function" "vect" } } */
+/* { dg-final { scan-assembler-not {\tuunpk(lo|hi)\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/cost_model_22.c 
b/gcc/testsuite/gcc.target/aarch64/sve/cost_model_22.c
new file mode 100644
index 000000000000..3e6d5ca5b69f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/cost_model_22.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -mcpu=neoverse-v3 -mautovec-preference=sve-only 
-msve-vector-bits=scalable --param=aarch64-autovec-mode=VNx16QI 
-fdump-tree-vect-details" } */
+
+#include "cost_model_21.c"
+
+/* The QI-to-SI conversion has two stages with counts 2 and 4.  On this
+   four-wide model, round the first stage up by two issue slots.  The separate
+   SI-to-DI conversion has only one stage and needs no padding.  Count the
+   two extra slots with the 34 general operations.  */
+/* { dg-final { scan-tree-dump "general operations = 36\\n" "vect" } } */
+/* { dg-final { scan-tree-dump "vectorized 1 loops in function" "vect" } } */

Reply via email to