Add ix86_bundle_align_loops, an RTL pass that marks selected loop regions
with .bundle_lock flex N. At assembly time, padding is inserted only when
required to keep a region that fits in one cache line from crossing a line
boundary, or to minimize the cache lines spanned by an oversized region.
Candidates are formed from layout-backward loop edges and selected as a
disjoint set by weighted interval scheduling.
Enable the pass for Zen4 and later, add the target options and assembler
feature check, and add tests for candidate formation, union construction,
weight calculation, selection, and representative loop cases.
gcc/
* config/i386/i386-features.cc (struct bundle_cand): New.
(bundle_cand_cmp_hi, bundle_hwi_cmp, bundle_hwi_less)
(bundle_cand_hi_less, bundle_build_layout, bundle_compute_layout)
(bundle_collect_candidates, bundle_add_unions)
(bundle_compute_weights, bundle_select_and_emit)
(ix86_do_bundle_align_loops): New functions.
(class pass_bundle_align_loops): New.
(make_pass_bundle_align_loops): New function.
* config/i386/i386-protos.h (make_pass_bundle_align_loops): Declare.
* config/i386/i386-passes.def (pass_bundle_align_loops): Insert before
pass_shorten_branches.
* config/i386/i386-options.cc (ix86_default_bundle_align): New function.
(ix86_default_align): Add opts_set argument. Disable generic loop,
jump, and label alignment defaults when bundle alignment is enabled.
(ix86_override_options_after_change, ix86_option_override_internal):
Call ix86_default_bundle_align and pass opts_set to ix86_default_align.
* config/i386/i386.md (UNSPECV_BUNDLE_LOCK)
(UNSPECV_BUNDLE_UNLOCK): New unspecs.
(bundle_lock, bundle_unlock): New insns.
* config/i386/i386.opt (mbundle-align-loops): New option.
(x86-tight-loop-max-size): New param.
* config/i386/i386.opt.urls: Regenerate.
* config/i386/x86-tune.def (X86_TUNE_BUNDLE_ALIGN_LOOPS): New tune flag.
* configure.ac: Check for .bundle_lock flex N and .bundle_unlock flex.
* configure: Regenerate.
* config.in: Regenerate.
* doc/invoke.texi (-mbundle-align-loops, x86-tight-loop-max-size):
Document.
* doc/extend.texi (x86 Function Attributes): Document
bundle-align-loops.
* doc/sourcebuild.texi (gcc_bundle_align_loops): Document.
gcc/testsuite/
* gcc.target/i386/bundle-align-1.c: New test.
* gcc.target/i386/bundle-align-2.c: New test.
* gcc.target/i386/bundle-align-3.c: New test.
* gcc.target/i386/bundle-align-4.c: New test.
* gcc.target/i386/bundle-align-5.c: New test.
* gcc.target/i386/bundle-align-6.c: New test.
* gcc.target/i386/bundle-align-7.c: New test.
* lib/target-supports.exp
(check_effective_target_gcc_bundle_align_loops): New.
Co-authored-by: Branden Moore <[email protected]>
Co-authored-by: Venkataramanan Kumar <[email protected]>
Signed-off-by: Sarvesh Chandra <[email protected]>
---
Presented at GNU Tools Cauldron 2026. We would like to include this in GCC 17.
Thanks Hongtao and Richard for the comments.
Changes since RFC v2:
- Emit .bundle_lock flex N and .bundle_unlock flex, so a flex region is
self-contained and no longer needs .bundle_align_mode; update the
configure check to match.
- When the assembler lacks flex support, keep bundle alignment off and use
the normal alignment defaults.
- Walk only the blocks between the first and last loop block, and index the
offset arrays by layout position instead of basic-block number (which is
not layout order); early exit on a function with no loop.
- Scan for .bundle_lock flex N with its size operand and for
.bundle_unlock flex, so a dropped size or a plain unlock fails the test.
gcc/config.in | 6 +
gcc/config/i386/i386-features.cc | 376 +++++++++++++++++-
gcc/config/i386/i386-options.cc | 32 +-
gcc/config/i386/i386-passes.def | 6 +-
gcc/config/i386/i386-protos.h | 2 +-
gcc/config/i386/i386.md | 35 ++
gcc/config/i386/i386.opt | 8 +
gcc/config/i386/i386.opt.urls | 3 +
gcc/config/i386/x86-tune.def | 7 +
gcc/configure | 37 ++
gcc/configure.ac | 10 +
gcc/doc/extend.texi | 6 +
gcc/doc/invoke.texi | 17 +-
gcc/doc/sourcebuild.texi | 4 +
.../gcc.target/i386/bundle-align-1.c | 16 +
.../gcc.target/i386/bundle-align-2.c | 21 +
.../gcc.target/i386/bundle-align-3.c | 26 ++
.../gcc.target/i386/bundle-align-4.c | 22 +
.../gcc.target/i386/bundle-align-5.c | 17 +
.../gcc.target/i386/bundle-align-6.c | 24 ++
.../gcc.target/i386/bundle-align-7.c | 17 +
gcc/testsuite/lib/target-supports.exp | 16 +
22 files changed, 688 insertions(+), 20 deletions(-)
create mode 100644 gcc/testsuite/gcc.target/i386/bundle-align-1.c
create mode 100644 gcc/testsuite/gcc.target/i386/bundle-align-2.c
create mode 100644 gcc/testsuite/gcc.target/i386/bundle-align-3.c
create mode 100644 gcc/testsuite/gcc.target/i386/bundle-align-4.c
create mode 100644 gcc/testsuite/gcc.target/i386/bundle-align-5.c
create mode 100644 gcc/testsuite/gcc.target/i386/bundle-align-6.c
create mode 100644 gcc/testsuite/gcc.target/i386/bundle-align-7.c
diff --git a/gcc/config.in b/gcc/config.in
index d84f59931da..b5e59ad45d8 100644
--- a/gcc/config.in
+++ b/gcc/config.in
@@ -1448,6 +1448,12 @@
#endif
+/* Define if your assembler accepts the flex argument to .bundle_lock. */
+#ifndef USED_FOR_TARGET
+#undef HAVE_GAS_BUNDLE_FLEX_LOCK
+#endif
+
+
/* Define 0/1 if your assembler supports CFI directives. */
#undef HAVE_GAS_CFI_DIRECTIVE
diff --git a/gcc/config/i386/i386-features.cc b/gcc/config/i386/i386-features.cc
index f95bc41fb47..2e5571b1ab7 100644
--- a/gcc/config/i386/i386-features.cc
+++ b/gcc/config/i386/i386-features.cc
@@ -83,6 +83,7 @@ along with GCC; see the file COPYING3. If not see
#include "ifcvt.h"
#include "symbol-summary.h"
#include "sreal.h"
+#include "fibonacci_heap.h"
#include "ipa-cp.h"
#include "ipa-prop.h"
#include "ipa-fnsummary.h"
@@ -5664,12 +5665,359 @@ ix86_align_loops ()
free_dominance_info (CDI_DOMINATORS);
}
+/* ix86_bundle_align_loops pass.
+
+ 1. Compute layout offsets and identify feasible loop-backward edges.
+ 2. Add synthetic unions for candidates sharing a K-byte window.
+ 3. Weight candidates by the back-edge counts of loops contained in
+ them.
+ 4. Select a maximum-weight disjoint set with weighted interval
+ scheduling.
+ 5. Emit .bundle_lock flex for the selected regions.
+
+ K is param_x86_tight_loop_max_size, or ix86_cost->prefetch_block when
+ unset. */
+
+/* A candidate loop region: the byte interval [lo, hi] on the layout axis
+ (hi inclusive), weight w, interval-scheduling predecessor p, the lock and
+ unlock blocks, and a feasibility flag. A synthetic union has w == -1 until
+ the weight sweep replaces it with its contained weight. */
+
+struct bundle_cand
+{
+ class loop *loop;
+ basic_block lock, unlock;
+ HOST_WIDE_INT lo, hi;
+ gcov_type w;
+ int p;
+ bool feasible;
+};
+
+/* Sort by hi ascending. Break ties by lo descending (a contained candidate
+ before its container), then by real loops before a synthetic union of the
+ same span. */
+
+static int
+bundle_cand_cmp_hi (const void *a, const void *b)
+{
+ const bundle_cand *x = (const bundle_cand *)a, *y = (const bundle_cand *)b;
+ if (x->hi != y->hi)
+ return x->hi < y->hi ? -1 : 1;
+ if (x->lo != y->lo)
+ return x->lo > y->lo ? -1 : 1;
+ if ((x->w == -1) != (y->w == -1))
+ return x->w == -1 ? 1 : -1;
+ return 0;
+}
+
+/* qsort comparator for HOST_WIDE_INT, ascending (lo-value compression). */
+
+static int
+bundle_hwi_cmp (const void *a, const void *b)
+{
+ HOST_WIDE_INT x = *(const HOST_WIDE_INT *)a, y = *(const HOST_WIDE_INT *)b;
+ return x < y ? -1 : x > y ? 1 : 0;
+}
+
+/* Less-than on HOST_WIDE_INT, for vec::lower_bound over the compressed
+ lo-value list. */
+
+static bool
+bundle_hwi_less (const HOST_WIDE_INT &a, const HOST_WIDE_INT &b)
+{
+ return a < b;
+}
+
+/* Less-than on interval end, for vec::lower_bound in the WIS predecessor
+ search over the hi-sorted feasible set. */
+
+static bool
+bundle_cand_hi_less (const bundle_cand &a, const bundle_cand &b)
+{
+ return a.hi < b.hi;
+}
+
+/* Record, in layout order, the blocks from the first through the last that
+ lie in a real loop, mapping each to its position in LAYOUT_INDEX. A
+ candidate may span intervening blocks that are not themselves in a loop. */
+
+static void
+bundle_build_layout (auto_vec<basic_block> &layout,
+ hash_map<basic_block, unsigned> &layout_index)
+{
+ basic_block bb, last = NULL;
+
+ FOR_EACH_BB_FN (bb, cfun)
+ if (bb->loop_father
+ && bb->loop_father->latch != EXIT_BLOCK_PTR_FOR_FN (cfun))
+ last = bb;
+
+ bool started = false;
+ FOR_EACH_BB_FN (bb, cfun)
+ {
+ if (!started
+ && (!bb->loop_father
+ || bb->loop_father->latch == EXIT_BLOCK_PTR_FOR_FN (cfun)))
+ continue;
+ started = true;
+ layout_index.put (bb, layout.length ());
+ layout.safe_push (bb);
+ if (bb == last)
+ break;
+ }
+}
+
+static void
+bundle_compute_layout (const auto_vec<basic_block> &layout,
+ HOST_WIDE_INT *bb_lo, HOST_WIDE_INT *bb_hi,
+ int *asm_lo, int *asm_hi)
+{
+ HOST_WIDE_INT off = 0;
+ int nasm = 0;
+
+ for (unsigned index = 0; index < layout.length (); index++)
+ {
+ basic_block bb = layout[index];
+ rtx_insn *insn;
+ bb_lo[index] = off;
+ asm_lo[index] = nasm;
+ FOR_BB_INSNS (bb, insn)
+ if (NONDEBUG_INSN_P (insn))
+ {
+ /* Inline asm has no reliable min size; count its presence so any
+ candidate spanning this block can be excluded outright. */
+ if (asm_noperands (PATTERN (insn)) >= 0)
+ nasm++;
+ else
+ off += ix86_min_insn_size (insn);
+ }
+ bb_hi[index] = off;
+ asm_hi[index] = nasm;
+ }
+}
+
+static void
+bundle_collect_candidates (vec<bundle_cand> &feas, HOST_WIDE_INT k,
+ const auto_vec<basic_block> &layout,
+ hash_map<basic_block, unsigned> &layout_index,
+ HOST_WIDE_INT *bb_lo, HOST_WIDE_INT *bb_hi,
+ int *asm_lo, int *asm_hi)
+{
+ /* Step 1: identify feasible candidates from layout-backward loop edges. */
+ for (unsigned index = 0; index < layout.length (); index++)
+ {
+ basic_block bb = layout[index];
+ edge e;
+ edge_iterator ei;
+ FOR_EACH_EDGE (e, ei, bb->succs)
+ {
+ unsigned *dest_index = layout_index.get (e->dest);
+ if (!e->count ().initialized_p () || dest_index == NULL
+ || !(e->dest == bb || *dest_index < index))
+ continue;
+
+ bundle_cand c = {};
+ c.loop = bb->loop_father;
+ c.lock = e->dest;
+ c.unlock = bb;
+ c.lo = bb_lo[*dest_index];
+ c.hi = bb_hi[index];
+ c.w = e->count ().to_gcov_type ();
+ /* Both ends must lie in a real loop, not the function-wide fake
+ loop whose latch is EXIT. This rejects a layout-backward exit
+ edge whose destination is outside any loop. */
+ c.feasible = (c.loop
+ && c.loop->latch != EXIT_BLOCK_PTR_FOR_FN (cfun)
+ && e->dest->loop_father
+ && (e->dest->loop_father->latch
+ != EXIT_BLOCK_PTR_FOR_FN (cfun))
+ && asm_hi[index] == asm_lo[*dest_index]
+ && (c.hi - c.lo) <= k
+ && optimize_bb_for_speed_p (c.lock));
+ if (c.feasible)
+ feas.safe_push (c);
+ }
+ }
+}
+
+static void
+bundle_add_unions (vec<bundle_cand> &feas, HOST_WIDE_INT k)
+{
+ /* Step 2: add synthetic unions for candidates sharing a K-byte window. */
+ feas.qsort (bundle_cand_cmp_hi);
+ unsigned base = feas.length ();
+ /* At most one union is added per candidate, so reserve now; this keeps the
+ &feas[i] pointers held by the heap valid across the push below. */
+ feas.reserve (2 * base);
+ fibonacci_heap<HOST_WIDE_INT, bundle_cand> heap (HOST_WIDE_INT_MIN);
+ for (unsigned i = 0; i < base; i++)
+ {
+ heap.insert (feas[i].lo, &feas[i]);
+ HOST_WIDE_INT thr = feas[i].hi - k;
+ while (heap.min_key () < thr)
+ heap.extract_min ();
+ if (heap.nodes () >= 2)
+ {
+ bundle_cand *s = heap.min ();
+ bundle_cand c = {};
+ c.loop = s->loop;
+ c.lock = s->lock;
+ c.unlock = feas[i].unlock;
+ c.lo = s->lo;
+ c.hi = feas[i].hi;
+ c.w = -1; /* Union; the weight sweep fills this in. */
+ c.feasible = true;
+ feas.safe_push (c);
+ }
+ }
+}
+
+static void
+bundle_compute_weights (vec<bundle_cand> &feas)
+{
+ /* Step 3: compute containment weights with a Fenwick tree. */
+ feas.qsort (bundle_cand_cmp_hi);
+ unsigned n_all = feas.length ();
+ /* Compress lo values into a sorted, unique list; POS below maps a lo to its
+ reversed rank via lower_bound. */
+ auto_vec<HOST_WIDE_INT> los;
+ los.reserve (n_all);
+ for (unsigned p = 0; p < n_all; p++)
+ los.quick_push (feas[p].lo);
+ los.qsort (bundle_hwi_cmp);
+ int m = 0;
+ for (unsigned p = 0; p < n_all; p++)
+ if (m == 0 || los[m - 1] != los[p])
+ los[m++] = los[p];
+ los.truncate (m);
+ /* BIT[1..m] keyed by reversed lo-rank, so a prefix sum [1..pos] covers
+ exactly lo_j >= lo_i. */
+ gcov_type *bit = XCNEWVEC (gcov_type, m + 1);
+ for (unsigned p = 0; p < n_all; p++)
+ {
+ /* Reversed rank of feas[p].lo: LOWER_BOUND is the first LOS >= lo. */
+ int pos = m - (int) los.lower_bound (feas[p].lo, bundle_hwi_less);
+ /* A union is not a real loop, so do not insert it into the tree. Its
+ weight is the contained sum, less one to break ties toward it. */
+ bool u = feas[p].w == -1;
+ if (!u)
+ for (int i = pos; i <= m; i += i & -i)
+ bit[i] += feas[p].w;
+ gcov_type sum = 0;
+ for (int i = pos; i > 0; i -= i & -i)
+ sum += bit[i];
+ feas[p].w = u ? sum - 1 : sum;
+ }
+ XDELETEVEC (bit);
+}
+
+static void
+bundle_select_and_emit (vec<bundle_cand> &feas)
+{
+ /* Step 4: select the maximum-weight disjoint set. */
+ unsigned n = feas.length ();
+ /* dp[i+1] is the best weight over feas[0..i]. Whether i is taken is
+ recomputed while backtracking rather than stored. */
+ gcov_type *dp = XCNEWVEC (gcov_type, n + 1);
+
+ for (unsigned i = 0; i < n; i++)
+ {
+ /* Predecessor: the last candidate that ends before feas[i] starts.
+ hi is inclusive, so find the first candidate with hi >= lo. */
+ bundle_cand key = {};
+ key.hi = feas[i].lo;
+ int p = (int) feas.lower_bound (key, bundle_cand_hi_less) - 1;
+ feas[i].p = p;
+ gcov_type take = feas[i].w + dp[p + 1], skip = dp[i];
+ dp[i + 1] = take >= skip ? take : skip;
+ if (dump_file)
+ fprintf (dump_file,
+ "bundle-align: cand %u loop %d [bb%d..bb%d] "
+ "lo=%ld hi=%ld w=%ld p=%d take=%ld skip=%ld => %s\n",
+ i, feas[i].loop->num, feas[i].lock->index,
+ feas[i].unlock->index, (long) feas[i].lo,
+ (long) feas[i].hi, (long) feas[i].w, p,
+ (long) take, (long) skip, take >= skip ? "take" : "skip");
+ }
+
+ /* Step 5: emit the selected regions. */
+ for (int i = (int) n - 1; i >= 0;)
+ if (feas[i].w + dp[feas[i].p + 1] >= dp[i])
+ {
+ basic_block h = feas[i].lock, l = feas[i].unlock;
+ emit_insn_before (gen_bundle_lock (), BB_HEAD (h));
+ emit_insn_after (gen_bundle_unlock (), BB_END (l));
+ if (dump_file)
+ fprintf (dump_file,
+ "bundle-align: wrap loop %d [bb%d..bb%d] "
+ "w=%ld span=%ld\n", feas[i].loop->num, h->index,
+ l->index, (long) feas[i].w,
+ (long) (feas[i].hi - feas[i].lo));
+ i = feas[i].p;
+ }
+ else
+ i--;
+
+ XDELETEVEC (dp);
+}
+
+/* Wrap the selected disjoint hot loop regions in .bundle_lock/.bundle_unlock.
+ Each region is an interval on the layout byte axis, which handles rotated
+ (latch-before-header) loops uniformly. */
+
+static void
+ix86_do_bundle_align_loops ()
+{
+ if (ix86_cost->prefetch_block == 0)
+ return;
+
+ loop_optimizer_init (AVOID_CFG_MODIFICATIONS);
+ if (number_of_loops (cfun) <= 1)
+ {
+ loop_optimizer_finalize ();
+ return;
+ }
+
+ /* Region size cap: tuning param if set, else the CPU bundle size. */
+ HOST_WIDE_INT k = (param_x86_tight_loop_max_size > 0
+ ? (HOST_WIDE_INT)param_x86_tight_loop_max_size
+ : (HOST_WIDE_INT)ix86_cost->prefetch_block);
+
+ auto_vec<basic_block> layout;
+ hash_map<basic_block, unsigned> layout_index;
+ bundle_build_layout (layout, layout_index);
+
+ /* Per-block layout byte offsets [bb_lo, bb_hi). */
+ unsigned nbb = layout.length ();
+ HOST_WIDE_INT *bb_lo = XCNEWVEC (HOST_WIDE_INT, nbb);
+ HOST_WIDE_INT *bb_hi = XCNEWVEC (HOST_WIDE_INT, nbb);
+ /* Running count of inline-asm insns in layout order; [asm_lo, asm_hi) lets a
+ candidate test whether any block it spans contains an asm. */
+ int *asm_lo = XCNEWVEC (int, nbb);
+ int *asm_hi = XCNEWVEC (int, nbb);
+ bundle_compute_layout (layout, bb_lo, bb_hi, asm_lo, asm_hi);
+
+ auto_vec<bundle_cand> feas;
+ bundle_collect_candidates (feas, k, layout, layout_index, bb_lo, bb_hi,
+ asm_lo, asm_hi);
+ bundle_add_unions (feas, k);
+ bundle_compute_weights (feas);
+ if (!feas.is_empty ())
+ bundle_select_and_emit (feas);
+
+ XDELETEVEC (bb_lo);
+ XDELETEVEC (bb_hi);
+ XDELETEVEC (asm_lo);
+ XDELETEVEC (asm_hi);
+ loop_optimizer_finalize ();
+}
+
namespace {
-const pass_data pass_data_align_tight_loops =
+const pass_data pass_data_bundle_align_loops =
{
RTL_PASS, /* type */
- "align_tight_loops", /* name */
+ "bundle_align_loops", /* name */
OPTGROUP_NONE, /* optinfo_flags */
TV_MACH_DEP, /* tv_id */
0, /* properties_required */
@@ -5679,40 +6027,44 @@ const pass_data pass_data_align_tight_loops =
0, /* todo_flags_finish */
};
-class pass_align_tight_loops : public rtl_opt_pass
+class pass_bundle_align_loops : public rtl_opt_pass
{
public:
- pass_align_tight_loops (gcc::context *ctxt)
- : rtl_opt_pass (pass_data_align_tight_loops, ctxt)
+ pass_bundle_align_loops (gcc::context *ctxt)
+ : rtl_opt_pass (pass_data_bundle_align_loops, ctxt)
{}
/* opt_pass methods: */
bool gate (function *) final override
{
- return TARGET_ALIGN_TIGHT_LOOPS
- && optimize
- && optimize_function_for_speed_p (cfun);
+ return optimize
+ && optimize_function_for_speed_p (cfun)
+ && (ix86_bundle_align_loops > 0 || TARGET_ALIGN_TIGHT_LOOPS);
}
unsigned int execute (function *) final override
{
timevar_push (TV_MACH_DEP);
+ if (ix86_bundle_align_loops > 0)
+ ix86_do_bundle_align_loops ();
#ifdef ASM_OUTPUT_MAX_SKIP_ALIGN
- ix86_align_loops ();
+ else
+ ix86_align_loops ();
#endif
timevar_pop (TV_MACH_DEP);
return 0;
}
-}; // class pass_align_tight_loops
+}; // class pass_bundle_align_loops
} // anon namespace
rtl_opt_pass *
-make_pass_align_tight_loops (gcc::context *ctxt)
+make_pass_bundle_align_loops (gcc::context *ctxt)
{
- return new pass_align_tight_loops (ctxt);
+ return new pass_bundle_align_loops (ctxt);
}
+
/* This compares the priority of target features in function DECL1
and DECL2. It returns positive value if DECL1 is higher priority,
negative value if DECL2 is higher priority and 0 if they are the
diff --git a/gcc/config/i386/i386-options.cc b/gcc/config/i386/i386-options.cc
index ebd86f71f93..a8774769eb3 100644
--- a/gcc/config/i386/i386-options.cc
+++ b/gcc/config/i386/i386-options.cc
@@ -1874,8 +1874,16 @@ set_ix86_tune_features (struct gcc_options *opts,
/* Default align_* from the processor table. */
static void
-ix86_default_align (struct gcc_options *opts)
+ix86_default_align (struct gcc_options *opts,
+ struct gcc_options *opts_set)
{
+ /* Bundle alignment replaces generic loop, jump, and label padding. */
+ if (opts->x_ix86_bundle_align_loops > 0)
+ {
+ SET_OPTION_IF_UNSET (opts, opts_set, str_align_loops, "1");
+ SET_OPTION_IF_UNSET (opts, opts_set, str_align_jumps, "1");
+ SET_OPTION_IF_UNSET (opts, opts_set, str_align_labels, "1");
+ }
/* -falign-foo without argument: supply one. */
if (opts->x_flag_align_loops && !opts->x_str_align_loops)
opts->x_str_align_loops = processor_cost_table[ix86_tune]->align_loop;
@@ -1887,6 +1895,22 @@ ix86_default_align (struct gcc_options *opts)
opts->x_str_align_functions = processor_cost_table[ix86_tune]->align_func;
}
+/* Default -mbundle-align-loops from the active tune model. */
+
+static void
+ix86_default_bundle_align (struct gcc_options *opts)
+{
+#ifdef HAVE_GAS_BUNDLE_FLEX_LOCK
+ if (opts->x_ix86_bundle_align_loops == -1)
+ opts->x_ix86_bundle_align_loops
+ = ix86_tune_features[X86_TUNE_BUNDLE_ALIGN_LOOPS] ? 1 : 0;
+#else
+ /* Without assembler support for flex .bundle_lock the pass emits nothing,
+ so keep it off and let the normal alignment defaults apply. */
+ opts->x_ix86_bundle_align_loops = 0;
+#endif
+}
+
#ifndef USE_IX86_FRAME_POINTER
#define USE_IX86_FRAME_POINTER 0
#endif
@@ -1988,7 +2012,8 @@ ix86_override_options_after_change_1 (struct gcc_options
*opts,
void
ix86_override_options_after_change (void)
{
- ix86_default_align (&global_options);
+ ix86_default_bundle_align (&global_options);
+ ix86_default_align (&global_options, &global_options_set);
ix86_recompute_optlev_based_flags (&global_options, &global_options_set);
@@ -2621,7 +2646,8 @@ ix86_option_override_internal (bool main_args_p,
opts->x_ix86_regparm = REGPARM_MAX;
/* Default align_* from the processor table. */
- ix86_default_align (&global_options);
+ ix86_default_bundle_align (&global_options);
+ ix86_default_align (&global_options, &global_options_set);
/* Provide default for -mbranch-cost= value. */
SET_OPTION_IF_UNSET (opts, opts_set, ix86_branch_cost,
diff --git a/gcc/config/i386/i386-passes.def b/gcc/config/i386/i386-passes.def
index 4f461116cac..82d4d7411df 100644
--- a/gcc/config/i386/i386-passes.def
+++ b/gcc/config/i386/i386-passes.def
@@ -31,9 +31,9 @@ along with GCC; see the file COPYING3. If not see
INSERT_PASS_BEFORE (pass_cse2, 1, pass_stv, true /* timode_p */);
INSERT_PASS_BEFORE (pass_shorten_branches, 1,
pass_insert_endbr_and_patchable_area);
- /* pass_align_tight_loops must be after pass_insert_endbr_and_patchable_area.
- PR116174. */
- INSERT_PASS_BEFORE (pass_shorten_branches, 1, pass_align_tight_loops);
+ /* Tight/bundle loop aligner; after pass_insert_endbr_and_patchable_area
+ (PR116174) and before pass_shorten_branches. */
+ INSERT_PASS_BEFORE (pass_shorten_branches, 1, pass_bundle_align_loops);
INSERT_PASS_AFTER (pass_late_combine, 1, pass_x86_cse);
INSERT_PASS_AFTER (pass_late_combine, 1, pass_remove_partial_avx_dependency);
diff --git a/gcc/config/i386/i386-protos.h b/gcc/config/i386/i386-protos.h
index 23f82b6239e..1bb8432bce4 100644
--- a/gcc/config/i386/i386-protos.h
+++ b/gcc/config/i386/i386-protos.h
@@ -434,7 +434,7 @@ extern rtl_opt_pass *make_pass_remove_partial_avx_dependency
(gcc::context *);
extern rtl_opt_pass *make_pass_x86_cse (gcc::context *);
extern rtl_opt_pass *make_pass_apx_nf_convert (gcc::context *);
-extern rtl_opt_pass *make_pass_align_tight_loops (gcc::context *);
+extern rtl_opt_pass *make_pass_bundle_align_loops (gcc::context *);
extern bool ix86_has_no_direct_extern_access;
extern bool ix86_rpad_gate ();
diff --git a/gcc/config/i386/i386.md b/gcc/config/i386/i386.md
index 2f499722182..bb48d08c263 100644
--- a/gcc/config/i386/i386.md
+++ b/gcc/config/i386/i386.md
@@ -233,6 +233,8 @@
UNSPECV_STACK_PROBE
UNSPECV_PROBE_STACK_RANGE
UNSPECV_ALIGN
+ UNSPECV_BUNDLE_LOCK
+ UNSPECV_BUNDLE_UNLOCK
UNSPECV_PROLOGUE_USE
UNSPECV_SPLIT_STACK_RETURN
UNSPECV_CLD
@@ -21820,6 +21822,39 @@
}
[(set_attr "length" "16")])
+;; ix86_bundle_align_loops pass: let the assembler insert padding only when
+;; needed to keep the locked region within one cache line (prefetch_block), or
+;; to minimize the cache lines spanned by an oversized region. Final sizes may
+;; grow during assembly, so flex .bundle_lock accepts an oversized region
+;; instead of raising an error. Requires flex .bundle_lock support.
+(define_insn "bundle_lock"
+ [(unspec_volatile [(const_int 0)] UNSPECV_BUNDLE_LOCK)]
+ ""
+{
+#ifdef HAVE_GAS_BUNDLE_FLEX_LOCK
+ fprintf (asm_out_file, "\t.bundle_lock flex %d\n",
+ ceil_log2 (ix86_cost->prefetch_block));
+#endif
+ return "";
+}
+ [(set_attr "length" "0")
+ (set_attr "length_immediate" "0")
+ (set_attr "modrm" "0")])
+
+;; Close the flex bundle region.
+(define_insn "bundle_unlock"
+ [(unspec_volatile [(const_int 0)] UNSPECV_BUNDLE_UNLOCK)]
+ ""
+{
+#ifdef HAVE_GAS_BUNDLE_FLEX_LOCK
+ fputs ("\t.bundle_unlock flex\n", asm_out_file);
+#endif
+ return "";
+}
+ [(set_attr "length" "0")
+ (set_attr "length_immediate" "0")
+ (set_attr "modrm" "0")])
+
(define_expand "prologue"
[(const_int 0)]
""
diff --git a/gcc/config/i386/i386.opt b/gcc/config/i386/i386.opt
index f9aeeac08fa..c5e372414b4 100644
--- a/gcc/config/i386/i386.opt
+++ b/gcc/config/i386/i386.opt
@@ -240,6 +240,10 @@ malign-loops=
Target Undocumented RejectNegative Joined UInteger
Loop code aligned to this power of 2.
+mbundle-align-loops
+Target Var(ix86_bundle_align_loops) Init(-1) Save
+Wrap small hot loops with .bundle_lock flex so the assembler pads them only
when needed to reduce cache-line crossings.
+
malign-stringops
Target RejectNegative InverseMask(NO_ALIGN_STRINGOPS, ALIGN_STRINGOPS) Save
Align destination of the string operations.
@@ -1221,6 +1225,10 @@ Do not use GOT to access external symbols.
Target Joined UInteger Var(x86_stlf_window_ninsns) Init(64) Param
Instructions number above which STFL stall penalty can be compensated.
+-param=x86-tight-loop-max-size=
+Target Joined UInteger Var(param_x86_tight_loop_max_size) Init(0) Param
+Maximum body size in bytes for the i386 tight-loop aligner; 0 picks the active
tune prefetch_block.
+
mavxifma
Target Mask(ISA2_AVXIFMA) Var(ix86_isa_flags2) Save
Support MMX, SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2, AVX, AVX2, and
diff --git a/gcc/config/i386/i386.opt.urls b/gcc/config/i386/i386.opt.urls
index 5cfa30c07e3..1fdbd167f34 100644
--- a/gcc/config/i386/i386.opt.urls
+++ b/gcc/config/i386/i386.opt.urls
@@ -28,6 +28,9 @@ UrlSuffix(gcc/x86-Options.html#index-malign-double)
; skipping UrlSuffix for 'malign-loops=' due to finding no URLs
+mbundle-align-loops
+UrlSuffix(gcc/x86-Options.html#index-mbundle-align-loops)
+
malign-stringops
UrlSuffix(gcc/x86-Options.html#index-malign-stringops)
diff --git a/gcc/config/i386/x86-tune.def b/gcc/config/i386/x86-tune.def
index d4c27351ad7..4716f26b2e1 100644
--- a/gcc/config/i386/x86-tune.def
+++ b/gcc/config/i386/x86-tune.def
@@ -259,6 +259,13 @@ DEF_TUNE (X86_TUNE_ALIGN_TIGHT_LOOPS, "align_tight_loops",
~(m_ZHAOXIN | m_CASCADELAKE | m_SKYLAKE_AVX512
| m_CORE_ATOM))
+/* X86_TUNE_BUNDLE_ALIGN_LOOPS: when true, use the ix86_bundle_align_loops
+ pass with the assembler's .bundle_lock/.bundle_unlock directives instead
+ of static .p2align padding. Targets cores where loops straddling a 64-byte
+ instruction-cache line are particularly costly (AMD Zen4/Zen5/Zen6). */
+DEF_TUNE (X86_TUNE_BUNDLE_ALIGN_LOOPS, "bundle_align_loops",
+ m_ZNVER4 | m_ZNVER5 | m_ZNVER6)
+
/*****************************************************************************/
/* Integer instruction selection tuning */
/*****************************************************************************/
diff --git a/gcc/configure b/gcc/configure
index c68d0865fb6..eb4866a7bff 100755
--- a/gcc/configure
+++ b/gcc/configure
@@ -26215,6 +26215,43 @@ $as_echo "#define HAVE_GAS_MAX_SKIP_P2ALIGN 1"
>>confdefs.h
fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking assembler for .bundle_lock
flex argument" >&5
+$as_echo_n "checking assembler for .bundle_lock flex argument... " >&6; }
+if ${gcc_cv_as_bundle_flex_lock+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+ gcc_cv_as_bundle_flex_lock=no
+ if test x"$gcc_cv_as" != x; then
+ $as_echo '.bundle_lock flex 6
+.rept 100
+nop
+.endr
+.bundle_unlock flex' > conftest.s
+ if { ac_try='$gcc_cv_as $gcc_cv_as_flags -o conftest.o conftest.s >&5'
+ { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_try\""; } >&5
+ (eval $ac_try) 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; }
+ then
+ gcc_cv_as_bundle_flex_lock=yes
+ else
+ echo "configure: failed program was" >&5
+ cat conftest.s >&5
+ fi
+ rm -f conftest.o conftest.s
+ fi
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $gcc_cv_as_bundle_flex_lock"
>&5
+$as_echo "$gcc_cv_as_bundle_flex_lock" >&6; }
+if test $gcc_cv_as_bundle_flex_lock = yes; then
+
+$as_echo "#define HAVE_GAS_BUNDLE_FLEX_LOCK 1" >>confdefs.h
+
+fi
+
+
+
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking assembler for .literal16" >&5
$as_echo_n "checking assembler for .literal16... " >&6; }
if ${gcc_cv_as_literal16+:} false; then :
diff --git a/gcc/configure.ac b/gcc/configure.ac
index 37f04e1a507..0084da912bf 100644
--- a/gcc/configure.ac
+++ b/gcc/configure.ac
@@ -3204,6 +3204,16 @@ gcc_GAS_CHECK_FEATURE([.p2align with maximum skip],
gcc_cv_as_max_skip_p2align,,
[Define if your assembler supports specifying the maximum number
of bytes to skip when using the GAS .p2align command.])])
+gcc_GAS_CHECK_FEATURE([.bundle_lock flex argument],
gcc_cv_as_bundle_flex_lock,,
+ [.bundle_lock flex 6
+.rept 100
+nop
+.endr
+.bundle_unlock flex],,
+[AC_DEFINE(HAVE_GAS_BUNDLE_FLEX_LOCK, 1,
+ [Define if your assembler accepts the flex argument to .bundle_lock
+ that permits a locked region larger than one bundle.])])
+
gcc_GAS_CHECK_FEATURE([.literal16], gcc_cv_as_literal16,,
[.literal16],,
[AC_DEFINE(HAVE_GAS_LITERAL16, 1,
diff --git a/gcc/doc/extend.texi b/gcc/doc/extend.texi
index a35b95cc74f..f7aa2ebaa29 100644
--- a/gcc/doc/extend.texi
+++ b/gcc/doc/extend.texi
@@ -9607,6 +9607,12 @@ operations and calling the library routines for large
operations.
@itemx no-align-stringops
Do/do not align destination of inlined string operations.
+@atindex @code{target("bundle-align-loops")}, x86
+@item bundle-align-loops
+@itemx no-bundle-align-loops
+Enable/disable wrapping of small hot loops in assembler
+@code{.bundle_lock}/@code{.bundle_unlock} directives.
+
@atindex @code{target("recip")}, x86
@item recip
@itemx no-recip
diff --git a/gcc/doc/invoke.texi b/gcc/doc/invoke.texi
index 66d80f17e74..28753c51d31 100644
--- a/gcc/doc/invoke.texi
+++ b/gcc/doc/invoke.texi
@@ -1555,7 +1555,8 @@ See RS/6000 and PowerPC Options.
-mamx-complex -mavxvnniint16 -msm3 -msha512 -msm4 -mapxf
-musermsr -mavx10.1 -mavx10.2 -mamx-avx512 -mmovrs -mamx-movrs
-mavx512bmm -mavx10v2aux -macev1
--mcldemote -mms-bitfields -mno-align-stringops -minline-all-stringops
+-mcldemote -mms-bitfields -mbundle-align-loops -mno-align-stringops
+-minline-all-stringops
-minline-stringops-dynamically -mstringop-strategy=@var{alg}
-mkl -mwidekl
-mmemcpy-strategy=@var{strategy} -mmemset-strategy=@var{strategy}
@@ -37246,6 +37247,20 @@ Here, @code{t5} takes up 2 bytes.
@end enumerate
+@opindex mbundle-align-loops
+@opindex mno-bundle-align-loops
+@item -mbundle-align-loops
+@itemx -mno-bundle-align-loops
+Wrap selected hot loop regions in assembler @code{.bundle_lock flex}/
+@code{.bundle_unlock flex} directives. At assembly time, padding is inserted
+only when required to keep a region that fits in one instruction-cache line
+from crossing a line boundary. If a region exceeds one line, padding minimizes
+the number of cache lines it spans. Enabled by default for AMD Zen4 and later.
+It defaults @option{-falign-loops},
+@option{-falign-jumps}, and @option{-falign-labels} to 1 unless explicitly set.
+The maximum loop-body size considered is set by the
+@option{x86-tight-loop-max-size} parameter.
+
@opindex mno-align-stringops
@opindex malign-stringops
@item -mno-align-stringops
diff --git a/gcc/doc/sourcebuild.texi b/gcc/doc/sourcebuild.texi
index 3aa9fcbb577..ed228cc8384 100644
--- a/gcc/doc/sourcebuild.texi
+++ b/gcc/doc/sourcebuild.texi
@@ -3121,6 +3121,10 @@ Target supports @option{-fstack-protector}.
@item gas
Target uses GNU @command{as}.
+@item gcc_bundle_align_loops
+Target compiler and assembler support @option{-mbundle-align-loops} and
+@code{.bundle_lock flex}.
+
@item gc_sections
Target supports @option{--gc-sections}.
diff --git a/gcc/testsuite/gcc.target/i386/bundle-align-1.c
b/gcc/testsuite/gcc.target/i386/bundle-align-1.c
new file mode 100644
index 00000000000..e557ee83f13
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/bundle-align-1.c
@@ -0,0 +1,16 @@
+/* { dg-do compile { target { ! ia32 } } } */
+/* { dg-options "-O3 -march=znver3 -falign-loops=1 -falign-jumps=1
-falign-labels=1 -mbundle-align-loops" } */
+/* { dg-require-effective-target gcc_bundle_align_loops } */
+/* { dg-final { scan-assembler "\\.bundle_lock flex \[0-9\]+" } } */
+/* { dg-final { scan-assembler "\\.bundle_unlock flex" } } */
+
+typedef int (*callee_t) (int, int *);
+
+int
+indirect_sum (callee_t callee, int n, int *array, long calls)
+{
+ int sink = 0;
+ for (long call = 0; call < calls; ++call)
+ sink += callee (n, array);
+ return sink;
+}
diff --git a/gcc/testsuite/gcc.target/i386/bundle-align-2.c
b/gcc/testsuite/gcc.target/i386/bundle-align-2.c
new file mode 100644
index 00000000000..a9741eecba0
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/bundle-align-2.c
@@ -0,0 +1,21 @@
+/* { dg-do compile { target { ! ia32 } } } */
+/* { dg-options "-O3 -march=znver3 -falign-loops=1 -falign-jumps=1
-falign-labels=1 -mbundle-align-loops" } */
+/* { dg-require-effective-target gcc_bundle_align_loops } */
+/* { dg-final { scan-assembler "\\.bundle_lock flex \[0-9\]+" } } */
+/* { dg-final { scan-assembler "\\.bundle_unlock flex" } } */
+
+__attribute__((noinline,noipa))
+int
+f (int n, int *p)
+{
+ int i = 0, s = 0;
+ if (n & 1)
+ goto mid;
+ while (i < n)
+ {
+ s += p[i & 1023];
+ mid:
+ i++;
+ }
+ return s;
+}
diff --git a/gcc/testsuite/gcc.target/i386/bundle-align-3.c
b/gcc/testsuite/gcc.target/i386/bundle-align-3.c
new file mode 100644
index 00000000000..18a4ce60459
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/bundle-align-3.c
@@ -0,0 +1,26 @@
+/* { dg-do compile { target { ! ia32 } } } */
+/* { dg-options "-O3 -march=znver3 -falign-loops=1 -falign-jumps=1
-falign-labels=1 -mbundle-align-loops" } */
+/* { dg-require-effective-target gcc_bundle_align_loops } */
+/* { dg-final { scan-assembler "\\.bundle_lock flex \[0-9\]+" } } */
+/* { dg-final { scan-assembler "\\.bundle_unlock flex" } } */
+
+int a[1024];
+__attribute__((hot,noinline,noipa))
+int
+two_backedges (int n, int m)
+{
+ int i = 0, s = 0;
+outer_head:
+ {
+ int j = 0;
+ inner_head:
+ s += a[(i + j) & 1023];
+ if (__builtin_expect (s < 0, 0))
+ goto outer_head;
+ if (++j < m)
+ goto inner_head;
+ }
+ if (++i < n)
+ goto outer_head;
+ return s;
+}
diff --git a/gcc/testsuite/gcc.target/i386/bundle-align-4.c
b/gcc/testsuite/gcc.target/i386/bundle-align-4.c
new file mode 100644
index 00000000000..19bd71e3d3e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/bundle-align-4.c
@@ -0,0 +1,22 @@
+/* { dg-do compile { target { ! ia32 } } } */
+/* { dg-options "-O3 -march=znver5 -falign-loops=1 -falign-jumps=1
-falign-labels=1 -mbundle-align-loops" } */
+/* { dg-require-effective-target gcc_bundle_align_loops } */
+/* { dg-final { scan-assembler-times "\\.bundle_lock flex \[0-9\]+" 1 } } */
+/* { dg-final { scan-assembler-times "\\.bundle_unlock flex" 1 } } */
+
+volatile int sink;
+int a[1024];
+int
+classify (int n)
+{
+ int s = 0;
+ asm volatile (".rept 0\n\t.byte 0x90\n\t.endr" ::: "memory");
+ for (int i = 0; i < n; i++)
+ {
+ if (a[i] & 1) sink = i;
+ if (a[i] & 2) sink = i + 1;
+ if (a[i] & 4) s += 4;
+ s += a[i];
+ }
+ return s;
+}
diff --git a/gcc/testsuite/gcc.target/i386/bundle-align-5.c
b/gcc/testsuite/gcc.target/i386/bundle-align-5.c
new file mode 100644
index 00000000000..e4c3f2db50a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/bundle-align-5.c
@@ -0,0 +1,17 @@
+/* { dg-do compile { target { ! ia32 } } } */
+/* { dg-options "-O3 -march=znver3 -falign-loops=1 -falign-jumps=1
-falign-labels=1 -mbundle-align-loops" } */
+/* { dg-require-effective-target gcc_bundle_align_loops } */
+/* { dg-final { scan-assembler "\\.bundle_lock flex \[0-9\]+" } } */
+/* { dg-final { scan-assembler "\\.bundle_unlock flex" } } */
+
+void *mc (const void *s, int c, unsigned long n)
+{
+ const unsigned char *p = s;
+ while (n--)
+ {
+ if (*p == (unsigned char) c)
+ return (void *) p;
+ p++;
+ }
+ return 0;
+}
diff --git a/gcc/testsuite/gcc.target/i386/bundle-align-6.c
b/gcc/testsuite/gcc.target/i386/bundle-align-6.c
new file mode 100644
index 00000000000..9b72fc81c52
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/bundle-align-6.c
@@ -0,0 +1,24 @@
+/* { dg-do compile { target { ! ia32 } } } */
+/* { dg-options "-O3 -march=znver3 -falign-loops=1 -falign-jumps=1
-falign-labels=1 -mbundle-align-loops" } */
+/* { dg-require-effective-target gcc_bundle_align_loops } */
+/* { dg-final { scan-assembler-times "\\.bundle_lock flex \[0-9\]+" 1 } } */
+/* { dg-final { scan-assembler-times "\\.bundle_unlock flex" 1 } } */
+
+struct O { long rc; };
+extern void dealloc (struct O *);
+void
+f (struct O **v, long n)
+{
+ asm volatile (".rept 10\n\t.byte 0x90\n\t.endr" ::: "memory");
+ while (--n >= 0)
+ {
+ struct O *x = v[n];
+ if (x)
+ {
+ if (x->rc < 0)
+ ;
+ else if (--x->rc == 0)
+ dealloc (x);
+ }
+ }
+}
diff --git a/gcc/testsuite/gcc.target/i386/bundle-align-7.c
b/gcc/testsuite/gcc.target/i386/bundle-align-7.c
new file mode 100644
index 00000000000..67f8ba7f39f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/bundle-align-7.c
@@ -0,0 +1,17 @@
+/* { dg-do compile { target { ! ia32 } } } */
+/* { dg-options "-O1 -march=znver3 -falign-loops=1 -falign-jumps=1
-falign-labels=1 -mbundle-align-loops" } */
+/* { dg-require-effective-target gcc_bundle_align_loops } */
+/* { dg-final { scan-assembler "\\.bundle_lock flex \[0-9\]+" } } */
+/* { dg-final { scan-assembler "\\.bundle_unlock flex" } } */
+
+static int A[1 << 20];
+
+int
+nested_sum (int n, int m)
+{
+ int sum = 0;
+ for (int i = 0; i < n; ++i)
+ for (int j = 0; j < m; ++j)
+ sum += A[i * m + j];
+ return sum;
+}
diff --git a/gcc/testsuite/lib/target-supports.exp
b/gcc/testsuite/lib/target-supports.exp
index ae04b6a8daa..0f0e2d4c438 100644
--- a/gcc/testsuite/lib/target-supports.exp
+++ b/gcc/testsuite/lib/target-supports.exp
@@ -12527,6 +12527,22 @@ proc check_effective_target_gas { } {
return $use_gas_saved
}
+# Return 1 if the compiler was configured with assembler support for
+# the flex .bundle_lock argument used by -mbundle-align-loops.
+
+proc check_effective_target_gcc_bundle_align_loops { } {
+ return [check_no_messages_and_pattern gcc_bundle_align_loops \
+ {\.bundle_lock flex [0-9]+} assembly {
+ int a[1024];
+ int foo (int n) {
+ int s = 0;
+ for (int i = 0; i < n; i++)
+ s += a[i];
+ return s;
+ }
+ } "-O2 -march=znver4 -mbundle-align-loops -falign-loops=1"]
+}
+
# Return 1 if GNU ld is used.
proc check_effective_target_gld { } {
--
2.34.1
