Introduce TARGET_IRA_LOOP_REGION_FLATTEN_COST so that a target can keep
a low-pressure loop as a separate IRA region.  The hook is called before
such a loop is flattened, when more pseudos live across calls in the
loop than the callee-saved registers of their class, and the pseudos
live across the loop without being referenced can cover the excess.
It gets the frequency of these calls and of the loop entry and exit
edges, both summed over the pseudos live into or out of the loop, and
the number of these edges.  The loop is kept if it returns a positive
value.

For the check, IRA now records the pressure of the pseudos live across
calls for each loop, and marks the subloops first so that a flattened
subloop adds its calls to its parent.

The default is NULL, then IRA computes nothing.

gcc/ChangeLog:

        * target.def (ira_loop_region_flatten_cost): New hook.
        * doc/tm.texi.in (TARGET_IRA_LOOP_REGION_FLATTEN_COST): New hook.
        * doc/tm.texi: Regenerate.
        * ira-int.h (struct ira_loop_tree_node): Add
        call_crossing_pressure.
        (ira_flatten_cost_p): Declare.
        * ira-lives.cc (process_bb_node_lives): Compute
        call_crossing_pressure.
        * ira-build.cc (ira_flatten_cost_p): New variable.
        (init_loop_tree_node): Clear call_crossing_pressure.
        (create_loop_tree_nodes): Likewise.
        (region_call_freq): New function.
        (region_ref_p): New function.
        (flatten_cost_keep_p): New function.
        (mark_low_pressure_loop_for_removal): New function.
        (mark_loops_for_removal): Use it to mark the subloops first, and
        do not report a kept loop as low pressure when the loop limit
        removes it.
        (ira_build): Set ira_flatten_cost_p.
---
 gcc/doc/tm.texi    |  26 ++++++
 gcc/doc/tm.texi.in |   2 +
 gcc/ira-build.cc   | 203 +++++++++++++++++++++++++++++++++++++++++----
 gcc/ira-int.h      |  11 +++
 gcc/ira-lives.cc   |  42 +++++++++-
 gcc/target.def     |  30 +++++++
 6 files changed, 295 insertions(+), 19 deletions(-)

diff --git a/gcc/doc/tm.texi b/gcc/doc/tm.texi
index dc022ae8d84..427b81bbf78 100644
--- a/gcc/doc/tm.texi
+++ b/gcc/doc/tm.texi
@@ -3093,6 +3093,32 @@ A target hook which can change allocno class for given 
pseudo from
   The default version of this target hook always returns given class.
 @end deftypefn
 
+@deftypefn {Target Hook} int TARGET_IRA_LOOP_REGION_FLATTEN_COST (int 
@var{call_freq}, int @var{border_freq}, int @var{border_edges})
+IRA removes a loop from its region tree when the register pressure in
+the loop and its parent is low.  If this hook is defined, IRA calls it
+before removing such a loop when more pseudos of some register class
+live through a call in the loop than the class has call-saved registers
+and the excess is covered by pseudos that live through the loop without
+being referenced in it.  Spilling those around the loop lets the others
+have call-saved registers.  IRA does not remove the loop for low
+pressure if the result is positive.  The loop can still be removed for
+other reasons, for example when there are more than
+@option{--param ira-max-loops-num} loops.
+
+@var{call_freq} is the frequency of the calls in the loop that the
+pseudos of such a class live through, summed over the pseudos live on
+the entry and exit edges of the loop.  @var{border_freq} is the
+frequency of those edges, summed over the pseudos live on them, and
+@var{border_edges} is their number.  Calls in a subloop that stays a
+region are not counted.  A target with many more call-clobbered than
+call-saved registers can use the hook to keep loops whose pseudos would
+otherwise be saved and restored around calls.
+
+IRA first calls the hook with @var{call_freq} 1 and the other arguments
+0 and does not examine the loops of a function if that result is not
+positive.
+@end deftypefn
+
 @deftypefn {Target Hook} bool TARGET_LRA_P (void)
 A target hook which returns true if we use LRA instead of reload pass.
 
diff --git a/gcc/doc/tm.texi.in b/gcc/doc/tm.texi.in
index 3cbaba910fb..bbd0b5d5835 100644
--- a/gcc/doc/tm.texi.in
+++ b/gcc/doc/tm.texi.in
@@ -2417,6 +2417,8 @@ in the reload pass.
 
 @hook TARGET_IRA_CHANGE_PSEUDO_ALLOCNO_CLASS
 
+@hook TARGET_IRA_LOOP_REGION_FLATTEN_COST
+
 @hook TARGET_LRA_P
 
 @hook TARGET_REGISTER_PRIORITY
diff --git a/gcc/ira-build.cc b/gcc/ira-build.cc
index d8c9da93192..9ddfe9e0ff5 100644
--- a/gcc/ira-build.cc
+++ b/gcc/ira-build.cc
@@ -55,6 +55,10 @@ ira_loop_tree_node_t ira_loop_nodes;
 /* And size of the ira_loop_nodes array.  */
 unsigned int ira_loop_nodes_count;
 
+/* True if the target can keep a loop of the current function with low
+   register pressure as a region (see flatten_cost_keep_p).  */
+bool ira_flatten_cost_p;
+
 /* Map regno -> allocnos with given regno (see comments for
    allocno member `next_regno_allocno').  */
 ira_allocno_t *ira_regno_allocno_map;
@@ -107,6 +111,8 @@ init_loop_tree_node (struct ira_loop_tree_node *node, int 
loop_num)
     = (ira_allocno_t *) ira_allocate (sizeof (ira_allocno_t) * max_regno);
   memset (node->regno_allocno_map, 0, sizeof (ira_allocno_t) * max_regno);
   memset (node->reg_pressure, 0, sizeof (node->reg_pressure));
+  memset (node->call_crossing_pressure, 0,
+         sizeof (node->call_crossing_pressure));
   node->all_allocnos = ira_allocate_bitmap ();
   node->modified_regnos = ira_allocate_bitmap ();
   node->border_allocnos = ira_allocate_bitmap ();
@@ -140,6 +146,8 @@ create_loop_tree_nodes (void)
       ira_bb_nodes[i].regno_allocno_map = NULL;
       memset (ira_bb_nodes[i].reg_pressure, 0,
              sizeof (ira_bb_nodes[i].reg_pressure));
+      memset (ira_bb_nodes[i].call_crossing_pressure, 0,
+             sizeof (ira_bb_nodes[i].call_crossing_pressure));
       ira_bb_nodes[i].all_allocnos = NULL;
       ira_bb_nodes[i].modified_regnos = NULL;
       ira_bb_nodes[i].border_allocnos = NULL;
@@ -2238,6 +2246,136 @@ low_pressure_loop_node_p (ira_loop_tree_node_t node)
   return true;
 }
 
+/* Return the frequency of the calls that the pseudo REGNO lives through
+   in LOOP_NODE and in its subloops marked for removal, counting only
+   calls that clobber a register of the pseudo's class.  Set *CROSS_P if
+   there is such a call.  Subloop allocnos are not yet propagated to the
+   parents here.  */
+static int
+region_call_freq (ira_loop_tree_node_t loop_node, int regno, bool *cross_p)
+{
+  int freq = 0;
+  ira_allocno_t a = loop_node->regno_allocno_map[regno];
+  if (a != NULL
+      && ALLOCNO_CALLS_CROSSED_NUM (a) != 0
+      && hard_reg_set_intersect_p (ira_need_caller_save_regs (a),
+                                  reg_class_contents[ALLOCNO_CLASS (a)]))
+    {
+      freq = ALLOCNO_CALL_FREQ (a);
+      *cross_p = true;
+    }
+  for (ira_loop_tree_node_t sub = loop_node->subloops; sub != NULL;
+       sub = sub->subloop_next)
+    if (sub->regno_allocno_map != NULL && sub->to_remove_p)
+      freq += region_call_freq (sub, regno, cross_p);
+  return freq;
+}
+
+/* Return true if the pseudo REGNO is referenced in LOOP_NODE or in one
+   of its subloops.  */
+static bool
+region_ref_p (ira_loop_tree_node_t loop_node, int regno)
+{
+  ira_allocno_t a = loop_node->regno_allocno_map[regno];
+  if (a != NULL && ALLOCNO_NREFS (a) != 0)
+    return true;
+  for (ira_loop_tree_node_t sub = loop_node->subloops; sub != NULL;
+       sub = sub->subloop_next)
+    if (sub->regno_allocno_map != NULL && region_ref_p (sub, regno))
+      return true;
+  return false;
+}
+
+/* Return TRUE if LOOP_NODE should be kept as a separate region although
+   its pressure is low.  This is possible only if more pseudos of some
+   class live through a call in the loop than the class has
+   call-preserved registers, and the pseudos living through the loop
+   without a reference in it cover the excess: spilling them around the
+   loop lets the others have call-preserved registers.  Keeping the loop
+   costs moves on the loop border.  Let the target compare the two.  The
+   subloops of LOOP_NODE are already marked.  */
+static bool
+flatten_cost_keep_p (ira_loop_tree_node_t loop_node)
+{
+  if (!ira_flatten_cost_p)
+    return false;
+
+  int excess[N_REG_CLASSES], through[N_REG_CLASSES];
+  memset (excess, 0, sizeof (excess));
+  memset (through, 0, sizeof (through));
+  bool any_excess_p = false;
+  for (int i = 0; i < ira_pressure_classes_num; i++)
+    {
+      enum reg_class cl = ira_pressure_classes[i];
+      int preserved_num = 0;
+      for (int j = 0; j < ira_class_hard_regs_num[cl]; j++)
+       if (!crtl->abi->clobbers_full_reg_p (ira_class_hard_regs[cl][j]))
+         preserved_num++;
+      if (preserved_num != 0)
+       excess[cl] = loop_node->call_crossing_pressure[cl] - preserved_num;
+      any_excess_p |= excess[cl] > 0;
+    }
+  if (!any_excess_p)
+    return false;
+
+  int call_freq = 0, border_freq = 0;
+  unsigned int i;
+  bitmap_iterator bi;
+  EXECUTE_IF_SET_IN_BITMAP (loop_node->border_allocnos, 0, i, bi)
+    {
+      ira_allocno_t a = ira_allocnos[i];
+      int regno = ALLOCNO_REGNO (a);
+      enum reg_class cl, aclass = ALLOCNO_CLASS (a);
+      enum reg_class pclass = ira_pressure_class_translate[aclass];
+      bool cross_p = false;
+      int freq = region_call_freq (loop_node, regno, &cross_p);
+      if (excess[pclass] > 0)
+       call_freq += freq;
+      if (cross_p && !region_ref_p (loop_node, regno))
+       for (int j = 0;
+            (cl = ira_reg_class_super_classes[pclass][j]) != LIM_REG_CLASSES;
+            j++)
+         through[cl] += ira_reg_class_max_nregs[aclass][ALLOCNO_MODE (a)];
+      border_freq += (ira_loop_edge_freq (loop_node, regno, false)
+                     + ira_loop_edge_freq (loop_node, regno, true));
+    }
+
+  bool covered_p = false;
+  for (int j = 0; j < ira_pressure_classes_num; j++)
+    {
+      enum reg_class cl = ira_pressure_classes[j];
+      if (excess[cl] > 0 && through[cl] >= excess[cl])
+       covered_p = true;
+    }
+  if (!covered_p)
+    {
+      if (internal_flag_ira_verbose > 1 && ira_dump_file != NULL)
+       fprintf (ira_dump_file,
+                "  Loop %d flatten-cost: excess not covered (allow flatten)\n",
+                loop_node->loop_num);
+      return false;
+    }
+
+  /* Count the entry and exit edges, not the back edges.  */
+  class loop *loop = loop_node->loop;
+  int border_edges = get_loop_exit_edges (loop).length ();
+  edge e;
+  edge_iterator ei;
+  FOR_EACH_EDGE (e, ei, loop->header->preds)
+    if (!flow_bb_inside_loop_p (loop, e->src))
+      border_edges++;
+
+  int bias = targetm.ira_loop_region_flatten_cost (call_freq, border_freq,
+                                                  border_edges);
+  if (internal_flag_ira_verbose > 1 && ira_dump_file != NULL)
+    fprintf (ira_dump_file,
+            "  Loop %d flatten-cost: call_freq=%d border_freq=%d"
+            " border_edges=%d bias=%d (%s)\n",
+            loop_node->loop_num, call_freq, border_freq, border_edges, bias,
+            bias > 0 ? "keep (veto flatten)" : "allow flatten");
+  return bias > 0;
+}
+
 #ifdef STACK_REGS
 /* Return TRUE if LOOP has a complex enter or exit edge.  We don't
    form a region from such loop if the target use stack register
@@ -2289,6 +2427,37 @@ loop_compare_func (const void *v1p, const void *v2p)
   return l1->loop_num - l2->loop_num;
 }
 
+/* Set to_remove_p of LOOP_NODE for mark_loops_for_removal.  */
+static void
+mark_low_pressure_loop_for_removal (ira_loop_tree_node_t loop_node)
+{
+  if (loop_node->parent == NULL)
+    {
+      /* Don't remove the root.  */
+      loop_node->to_remove_p = false;
+      return;
+    }
+  loop_node->to_remove_p
+    = ((low_pressure_loop_node_p (loop_node->parent)
+       && low_pressure_loop_node_p (loop_node)
+       && !flatten_cost_keep_p (loop_node))
+#ifdef STACK_REGS
+       || loop_with_complex_edge_p (loop_node->loop)
+#endif
+       );
+  if (!loop_node->to_remove_p)
+    return;
+  /* The calls of a removed loop become calls of its parent.  */
+  for (int i = 0; i < ira_pressure_classes_num; i++)
+    {
+      enum reg_class cl = ira_pressure_classes[i];
+      if (loop_node->parent->call_crossing_pressure[cl]
+         < loop_node->call_crossing_pressure[cl])
+       loop_node->parent->call_crossing_pressure[cl]
+         = loop_node->call_crossing_pressure[cl];
+    }
+}
+
 /* Mark loops which should be removed from regional allocation.  We
    remove a loop with low register pressure inside another loop with
    register pressure.  In this case a separate allocation of the loop
@@ -2312,27 +2481,18 @@ mark_loops_for_removal (void)
   sorted_loops
     = (ira_loop_tree_node_t *) ira_allocate (sizeof (ira_loop_tree_node_t)
                                             * number_of_loops (cfun));
+  /* Mark the subloops of a loop first: flatten_cost_keep_p needs to know
+     which of them become a part of the loop.  */
+  ira_traverse_loop_tree (false, ira_loop_tree_root, NULL,
+                         mark_low_pressure_loop_for_removal);
   for (n = i = 0; vec_safe_iterate (get_loops (cfun), i, &loop); i++)
-    if (ira_loop_nodes[i].regno_allocno_map != NULL)
-      {
-       if (ira_loop_nodes[i].parent == NULL)
-         {
-           /* Don't remove the root.  */
-           ira_loop_nodes[i].to_remove_p = false;
-           continue;
-         }
-       sorted_loops[n++] = &ira_loop_nodes[i];
-       ira_loop_nodes[i].to_remove_p
-         = ((low_pressure_loop_node_p (ira_loop_nodes[i].parent)
-             && low_pressure_loop_node_p (&ira_loop_nodes[i]))
-#ifdef STACK_REGS
-            || loop_with_complex_edge_p (ira_loop_nodes[i].loop)
-#endif
-            );
-      }
+    if (ira_loop_nodes[i].regno_allocno_map != NULL
+       && ira_loop_nodes[i].parent != NULL)
+      sorted_loops[n++] = &ira_loop_nodes[i];
   qsort (sorted_loops, n, sizeof (ira_loop_tree_node_t), loop_compare_func);
   for (i = 0; i < n - param_ira_max_loops_num; i++)
     {
+      bool removed_p = sorted_loops[i]->to_remove_p;
       sorted_loops[i]->to_remove_p = true;
       if (internal_flag_ira_verbose > 1 && ira_dump_file != NULL)
        fprintf
@@ -2341,7 +2501,8 @@ mark_loops_for_removal (void)
           sorted_loops[i]->loop_num, sorted_loops[i]->loop->header->index,
           sorted_loops[i]->loop->header->count.to_frequency (cfun),
           loop_depth (sorted_loops[i]->loop),
-          low_pressure_loop_node_p (sorted_loops[i]->parent)
+          removed_p
+          && low_pressure_loop_node_p (sorted_loops[i]->parent)
           && low_pressure_loop_node_p (sorted_loops[i])
           ? "low pressure" : "cheap loop");
     }
@@ -3514,6 +3675,12 @@ ira_build (void)
   create_allocnos ();
   ira_costs ();
   create_allocno_objects ();
+  /* The hook result for one call and no loop border tells whether the
+     target keeps any loop.  If it does not, the data for the hook is
+     not collected.  */
+  ira_flatten_cost_p
+    = (targetm.ira_loop_region_flatten_cost != NULL
+       && targetm.ira_loop_region_flatten_cost (1, 0, 0) > 0);
   ira_create_allocno_live_ranges ();
   remove_unnecessary_regions (false);
   ira_compress_allocno_live_ranges ();
diff --git a/gcc/ira-int.h b/gcc/ira-int.h
index 710801bd65c..3cd7c885aa3 100644
--- a/gcc/ira-int.h
+++ b/gcc/ira-int.h
@@ -124,6 +124,13 @@ struct ira_loop_tree_node
      (defined only for the pressure classes).  */
   int reg_pressure[N_REG_CLASSES];
 
+  /* Maximal number of registers of given register class needed by the
+     pseudos living through one call inside the loop, counting only
+     calls that clobber a register of the class (defined only for the
+     pressure classes).  Unlike REG_PRESSURE it does not include the
+     subloops until they are marked for removal.  */
+  int call_crossing_pressure[N_REG_CLASSES];
+
   /* Numbers of allocnos referred or living in the loop node (except
      for its subloops).  */
   bitmap all_allocnos;
@@ -172,6 +179,10 @@ extern ira_loop_tree_node_t ira_bb_nodes;
    array.  */
 extern ira_loop_tree_node_t ira_loop_nodes;
 
+/* True if the target can keep a loop of the current function with low
+   register pressure as a region (see flatten_cost_keep_p).  */
+extern bool ira_flatten_cost_p;
+
 /* Two access macros to the nodes representing loops.  */
 #if defined ENABLE_IRA_CHECKING && (GCC_VERSION >= 2007)
 #define IRA_LOOP_NODE_BY_INDEX(index) __extension__                    \
diff --git a/gcc/ira-lives.cc b/gcc/ira-lives.cc
index 98279bec6f1..af362e67129 100644
--- a/gcc/ira-lives.cc
+++ b/gcc/ira-lives.cc
@@ -1564,13 +1564,18 @@ process_bb_node_lives (ira_loop_tree_node_t 
loop_tree_node)
 
              last_call_num++;
              sparseset_clear (allocnos_processed);
+             function_abi callee_abi = insn_callee_abi (insn);
+             /* Registers of each class needed by the pseudos living
+                through this call.  */
+             int call_pressure[N_REG_CLASSES];
+             if (ira_flatten_cost_p)
+               memset (call_pressure, 0, sizeof (call_pressure));
              /* The current set of live allocnos are live across the call.  */
              EXECUTE_IF_SET_IN_SPARSESET (objects_live, i)
                {
                  ira_object_t obj = ira_object_id_map[i];
                  a = OBJECT_ALLOCNO (obj);
                  int num = ALLOCNO_NUM (a);
-                 function_abi callee_abi = insn_callee_abi (insn);
 
                  /* Don't allocate allocnos that cross setjmps or any
                     call, if this function receives a nonlocal
@@ -1591,6 +1596,25 @@ process_bb_node_lives (ira_loop_tree_node_t 
loop_tree_node)
                    continue;
                  sparseset_set_bit (allocnos_processed, num);
 
+                 if (ira_flatten_cost_p)
+                   {
+                     enum reg_class cl, aclass = ALLOCNO_CLASS (a);
+                     enum reg_class pclass
+                       = ira_pressure_class_translate[aclass];
+                     int nregs
+                       = ira_reg_class_max_nregs[aclass][ALLOCNO_MODE (a)];
+                     /* A call that clobbers no register of the class
+                        does not need a call-preserved register.  */
+                     if (hard_reg_set_intersect_p
+                         (callee_abi.full_and_partial_reg_clobbers (),
+                          reg_class_contents[pclass] & ~ira_no_alloc_regs))
+                       for (int j = 0;
+                            (cl = ira_reg_class_super_classes[pclass][j])
+                            != LIM_REG_CLASSES;
+                            j++)
+                         call_pressure[cl] += nregs;
+                   }
+
                  if (allocno_saved_at_call[num] != last_call_num)
                    /* Here we are mimicking caller-save.cc behavior
                       which does not save hard register at a call if
@@ -1608,6 +1632,15 @@ process_bb_node_lives (ira_loop_tree_node_t 
loop_tree_node)
                      && ALLOCNO_REGNO (a) == (int) REGNO (cheap_reg))
                    ALLOCNO_CHEAP_CALLS_CROSSED_NUM (a)++;
                }
+             if (ira_flatten_cost_p)
+               for (int j = 0; j < ira_pressure_classes_num; j++)
+                 {
+                   enum reg_class cl = ira_pressure_classes[j];
+                   if (curr_bb_node->call_crossing_pressure[cl]
+                       < call_pressure[cl])
+                     curr_bb_node->call_crossing_pressure[cl]
+                       = call_pressure[cl];
+                 }
            }
 
          /* See which defined values die here.  Note that we include
@@ -1726,6 +1759,13 @@ process_bb_node_lives (ira_loop_tree_node_t 
loop_tree_node)
            > loop_tree_node->parent->reg_pressure[pclass])
          loop_tree_node->parent->reg_pressure[pclass]
            = loop_tree_node->reg_pressure[pclass];
+       /* A loop gets the calls of its subloops when they are marked for
+          removal.  */
+       if (loop_tree_node->bb != NULL
+           && (loop_tree_node->call_crossing_pressure[pclass]
+               > loop_tree_node->parent->call_crossing_pressure[pclass]))
+         loop_tree_node->parent->call_crossing_pressure[pclass]
+           = loop_tree_node->call_crossing_pressure[pclass];
       }
 }
 
diff --git a/gcc/target.def b/gcc/target.def
index 029956b6268..72d9131b085 100644
--- a/gcc/target.def
+++ b/gcc/target.def
@@ -5938,6 +5938,36 @@ DEFHOOK
  reg_class_t, (int, reg_class_t, reg_class_t),
  default_ira_change_pseudo_allocno_class)
 
+/* Return the gain from keeping a low-pressure loop as a separate IRA
+   region.  */
+DEFHOOK
+(ira_loop_region_flatten_cost,
+ "IRA removes a loop from its region tree when the register pressure in\n\
+the loop and its parent is low.  If this hook is defined, IRA calls it\n\
+before removing such a loop when more pseudos of some register class\n\
+live through a call in the loop than the class has call-saved registers\n\
+and the excess is covered by pseudos that live through the loop without\n\
+being referenced in it.  Spilling those around the loop lets the others\n\
+have call-saved registers.  IRA does not remove the loop for low\n\
+pressure if the result is positive.  The loop can still be removed for\n\
+other reasons, for example when there are more than\n\
+@option{--param ira-max-loops-num} loops.\n\
+\n\
+@var{call_freq} is the frequency of the calls in the loop that the\n\
+pseudos of such a class live through, summed over the pseudos live on\n\
+the entry and exit edges of the loop.  @var{border_freq} is the\n\
+frequency of those edges, summed over the pseudos live on them, and\n\
+@var{border_edges} is their number.  Calls in a subloop that stays a\n\
+region are not counted.  A target with many more call-clobbered than\n\
+call-saved registers can use the hook to keep loops whose pseudos would\n\
+otherwise be saved and restored around calls.\n\
+\n\
+IRA first calls the hook with @var{call_freq} 1 and the other arguments\n\
+0 and does not examine the loops of a function if that result is not\n\
+positive.",
+ int, (int call_freq, int border_freq, int border_edges),
+ NULL)
+
 /* Return true if we use LRA instead of reload.  */
 DEFHOOK
 (lra_p,
-- 
2.31.1

Reply via email to