On Sat, Oct 3, 2026 at 4:03 PM Sarvesh Chandra <[email protected]> wrote:
>
> Add ix86_bundle_align_loops, an RTL pass that marks selected loop regions
> with .bundle_lock flex. The assembler aligns each marked region after
> instruction relaxation, when the final instruction sizes are known, and
> flex mode accepts a region that ends up larger than one bundle.
>
> Candidates are formed from layout-backward loop edges and selected as a
> disjoint set by weighted interval scheduling.
>
> Enable the pass for Zen4 and later, add the target options and assembler
> feature check, and add tests for candidate formation, union construction,
> weight calculation, selection, and representative loop cases.
>
> gcc/
> * config/i386/i386-features.cc (struct bundle_cand): New.
> (bundle_cand_cmp_hi, bundle_hwi_cmp, bundle_hwi_less)
> (bundle_cand_hi_less, bundle_compute_layout)
> (bundle_collect_candidates, bundle_add_unions)
> (bundle_compute_weights, bundle_select_and_emit)
> (ix86_do_bundle_align_loops): New functions.
> (class pass_bundle_align_loops): New.
> (make_pass_bundle_align_loops): New function.
> * config/i386/i386-protos.h (make_pass_bundle_align_loops): Declare.
> * config/i386/i386-passes.def (pass_bundle_align_loops): Insert before
> pass_shorten_branches.
> * config/i386/i386-options.cc (ix86_default_bundle_align): New
> function.
> (ix86_default_align): Call it.
> * config/i386/i386.md (bundle_lock, bundle_unlock): New insns.
> * config/i386/i386.opt (mbundle-align-loops): New option.
> (x86-tight-loop-max-size, x86-bundle-align-branch-limit): New params.
> * config/i386/i386.opt.urls: Regenerate.
> * config/i386/x86-tune.def (X86_TUNE_BUNDLE_ALIGN_LOOPS): New tune
> flag.
> * configure.ac: Check for .bundle_lock flex support.
> * configure: Regenerate.
> * config.in: Regenerate.
> * doc/invoke.texi (-mbundle-align-loops, x86-tight-loop-max-size):
> Document.
> * doc/extend.texi (x86 Function Attributes): Document
> bundle-align-loops.
> * doc/sourcebuild.texi (gcc_bundle_align_loops): Document.
>
> gcc/testsuite/
> * gcc.target/i386/bundle-align-1.c: New test.
> * gcc.target/i386/bundle-align-2.c: New test.
> * gcc.target/i386/bundle-align-3.c: New test.
> * gcc.target/i386/bundle-align-4.c: New test.
> * gcc.target/i386/bundle-align-5.c: New test.
> * gcc.target/i386/bundle-align-6.c: New test.
> * gcc.target/i386/bundle-align-7.c: New test.
> * lib/target-supports.exp
> (check_effective_target_gcc_bundle_align_loops): New.
>
> Co-authored-by: Branden Moore <[email protected]>
> Co-authored-by: Venkataramanan Kumar <[email protected]>
> Signed-off-by: Sarvesh Chandra <[email protected]>
> ---
> [Presented at GNU Tools Cauldron 2026.]
>
> Cosmetic changes in v2:
> - Add co-authors Branden Moore and Venkataramanan Kumar.
> - Drop the slide references from the test comments. They pointed at
> the Cauldron presentation, but GCC does not allow attaching the PPT
> to the patch, so the references are irrelevant here.
> - Correct the i386.md comment: the simple flex mode accepts an
> oversized locked region and still aligns its start; it does not skip
> padding. An experimental post-relaxation patch does skip padding,
> but that is not part of this proposal.
> - bundle-align-5.c: rename the scan kernel instead of redefining
> memchr with -D.
> - bundle-align-6.c: drop the unneeded x86-tight-loop-max-size param.
> - Reduce the tests to the scanned kernel: drop the main drivers and
> headers.
>
> gcc/config.in | 6 +
> gcc/config/i386/i386-features.cc | 331 +++++++++++++++++-
> gcc/config/i386/i386-options.cc | 26 +-
> gcc/config/i386/i386-passes.def | 6 +-
> gcc/config/i386/i386-protos.h | 2 +-
> gcc/config/i386/i386.md | 35 ++
> gcc/config/i386/i386.opt | 12 +
> gcc/config/i386/i386.opt.urls | 3 +
> gcc/config/i386/x86-tune.def | 7 +
> gcc/configure | 38 ++
> gcc/configure.ac | 11 +
> gcc/doc/extend.texi | 6 +
> gcc/doc/invoke.texi | 15 +-
> gcc/doc/sourcebuild.texi | 4 +
> .../gcc.target/i386/bundle-align-1.c | 16 +
> .../gcc.target/i386/bundle-align-2.c | 21 ++
> .../gcc.target/i386/bundle-align-3.c | 26 ++
> .../gcc.target/i386/bundle-align-4.c | 22 ++
> .../gcc.target/i386/bundle-align-5.c | 17 +
> .../gcc.target/i386/bundle-align-6.c | 24 ++
> .../gcc.target/i386/bundle-align-7.c | 17 +
> gcc/testsuite/lib/target-supports.exp | 16 +
> 22 files changed, 641 insertions(+), 20 deletions(-)
> create mode 100644 gcc/testsuite/gcc.target/i386/bundle-align-1.c
> create mode 100644 gcc/testsuite/gcc.target/i386/bundle-align-2.c
> create mode 100644 gcc/testsuite/gcc.target/i386/bundle-align-3.c
> create mode 100644 gcc/testsuite/gcc.target/i386/bundle-align-4.c
> create mode 100644 gcc/testsuite/gcc.target/i386/bundle-align-5.c
> create mode 100644 gcc/testsuite/gcc.target/i386/bundle-align-6.c
> create mode 100644 gcc/testsuite/gcc.target/i386/bundle-align-7.c
>
> diff --git a/gcc/config.in b/gcc/config.in
> index b90caa0dd57..51dac287796 100644
> --- a/gcc/config.in
> +++ b/gcc/config.in
> @@ -1448,6 +1448,12 @@
> #endif
>
>
> +/* Define if your assembler accepts the flex argument to .bundle_lock. */
> +#ifndef USED_FOR_TARGET
> +#undef HAVE_GAS_BUNDLE_FLEX_LOCK
> +#endif
> +
> +
> /* Define 0/1 if your assembler supports CFI directives. */
> #undef HAVE_GAS_CFI_DIRECTIVE
>
> diff --git a/gcc/config/i386/i386-features.cc
> b/gcc/config/i386/i386-features.cc
> index eb5f82e42cb..6743c0ebcfd 100644
> --- a/gcc/config/i386/i386-features.cc
> +++ b/gcc/config/i386/i386-features.cc
> @@ -83,6 +83,7 @@ along with GCC; see the file COPYING3. If not see
> #include "ifcvt.h"
> #include "symbol-summary.h"
> #include "sreal.h"
> +#include "fibonacci_heap.h"
> #include "ipa-cp.h"
> #include "ipa-prop.h"
> #include "ipa-fnsummary.h"
> @@ -5646,12 +5647,314 @@ ix86_align_loops ()
> free_dominance_info (CDI_DOMINATORS);
> }
>
> +/* ix86_bundle_align_loops pass.
> +
> + 1. Compute layout offsets and identify feasible loop-backward edges.
> + 2. Add synthetic unions for candidates sharing a K-byte window.
> + 3. Weight candidates by the back-edge counts of loops contained in
> + them.
> + 4. Select a maximum-weight disjoint set with weighted interval
> + scheduling.
> + 5. Emit .bundle_lock flex for the selected regions.
> +
> + K is param_x86_tight_loop_max_size, or ix86_cost->prefetch_block when
> + unset. */
> +
> +/* A candidate loop region: the byte interval [lo, hi] on the layout axis
> + (hi inclusive), weight w, interval-scheduling predecessor p, the lock and
> + unlock blocks, and a feasibility flag. A synthetic union has w == -1
> until
> + the weight sweep replaces it with its contained weight. */
> +
> +struct bundle_cand
> +{
> + class loop *loop;
> + basic_block lock, unlock;
> + HOST_WIDE_INT lo, hi;
> + gcov_type w;
> + int p;
> + bool feasible;
> +};
> +
> +/* Sort by hi ascending. Break ties by lo descending (a contained candidate
> + before its container), then by real loops before a synthetic union of the
> + same span. */
> +
> +static int
> +bundle_cand_cmp_hi (const void *a, const void *b)
> +{
> + const bundle_cand *x = (const bundle_cand *)a, *y = (const bundle_cand *)b;
> + if (x->hi != y->hi)
> + return x->hi < y->hi ? -1 : 1;
> + if (x->lo != y->lo)
> + return x->lo > y->lo ? -1 : 1;
> + if ((x->w == -1) != (y->w == -1))
> + return x->w == -1 ? 1 : -1;
> + return 0;
> +}
> +
> +/* qsort comparator for HOST_WIDE_INT, ascending (lo-value compression). */
> +
> +static int
> +bundle_hwi_cmp (const void *a, const void *b)
> +{
> + HOST_WIDE_INT x = *(const HOST_WIDE_INT *)a, y = *(const HOST_WIDE_INT *)b;
> + return x < y ? -1 : x > y ? 1 : 0;
> +}
> +
> +/* Less-than on HOST_WIDE_INT, for vec::lower_bound over the compressed
> + lo-value list. */
> +
> +static bool
> +bundle_hwi_less (const HOST_WIDE_INT &a, const HOST_WIDE_INT &b)
> +{
> + return a < b;
> +}
> +
> +/* Less-than on interval end, for vec::lower_bound in the WIS predecessor
> + search over the hi-sorted feasible set. */
> +
> +static bool
> +bundle_cand_hi_less (const bundle_cand &a, const bundle_cand &b)
> +{
> + return a.hi < b.hi;
> +}
> +
> +static void
> +bundle_compute_layout (HOST_WIDE_INT *bb_lo, HOST_WIDE_INT *bb_hi,
> + int *asm_lo, int *asm_hi)
> +{
> + HOST_WIDE_INT off = 0;
> + int nasm = 0;
> + basic_block bb;
> + FOR_EACH_BB_FN (bb, cfun)
can you avoid doing this for non-candidate loop blocks?
> + {
> + rtx_insn *insn;
> + bb_lo[bb->index] = off;
> + asm_lo[bb->index] = nasm;
> + FOR_BB_INSNS (bb, insn)
> + if (NONDEBUG_INSN_P (insn))
> + {
> + /* Inline asm has no reliable min size; count its presence so any
> + candidate spanning this block can be excluded outright. */
> + if (asm_noperands (PATTERN (insn)) >= 0)
> + nasm++;
> + else
> + off += ix86_min_insn_size (insn);
> + }
> + bb_hi[bb->index] = off;
> + asm_hi[bb->index] = nasm;
> + }
> +}
> +
> +static void
> +bundle_collect_candidates (vec<bundle_cand> &feas, HOST_WIDE_INT k,
> + HOST_WIDE_INT *bb_lo, HOST_WIDE_INT *bb_hi,
> + int *asm_lo, int *asm_hi)
> +{
> + /* Step 1: identify feasible candidates from layout-backward loop edges.
> */
> + basic_block bb;
> + FOR_EACH_BB_FN (bb, cfun)
I wonder if you can instead walk the loop tree?
> + {
> + edge e;
> + edge_iterator ei;
> + FOR_EACH_EDGE (e, ei, bb->succs)
> + if (e->count ().initialized_p ()
> + && e->dest->index >= NUM_FIXED_BLOCKS
> + && (e->dest == bb
> + || bb_lo[e->dest->index] < bb_lo[bb->index]))
> + {
> + bundle_cand c = {};
> + c.loop = bb->loop_father;
> + c.lock = e->dest;
> + c.unlock = bb;
> + c.lo = bb_lo[e->dest->index];
> + c.hi = bb_hi[bb->index];
> + c.w = e->count ().to_gcov_type ();
> + /* Both ends must lie in a real loop, not the function-wide fake
> + loop whose latch is EXIT. This rejects a layout-backward exit
> + edge whose destination is outside any loop. */
> + c.feasible = (c.loop
> + && c.loop->latch != EXIT_BLOCK_PTR_FOR_FN (cfun)
> + && e->dest->loop_father
> + && (e->dest->loop_father->latch
> + != EXIT_BLOCK_PTR_FOR_FN (cfun))
> + && asm_hi[bb->index] == asm_lo[e->dest->index]
> + && c.hi - c.lo <= k
> + && optimize_bb_for_speed_p (c.lock));
> + if (c.feasible)
> + feas.safe_push (c);
> + }
> + }
> +}
> +
> +static void
> +bundle_add_unions (vec<bundle_cand> &feas, HOST_WIDE_INT k)
> +{
> + /* Step 2: add synthetic unions for candidates sharing a K-byte window. */
> + feas.qsort (bundle_cand_cmp_hi);
> + unsigned base = feas.length ();
> + /* At most one union is added per candidate, so reserve now; this keeps the
> + &feas[i] pointers held by the heap valid across the push below. */
> + feas.reserve (2 * base);
> + fibonacci_heap<HOST_WIDE_INT, bundle_cand> heap (HOST_WIDE_INT_MIN);
> + for (unsigned i = 0; i < base; i++)
> + {
> + heap.insert (feas[i].lo, &feas[i]);
> + HOST_WIDE_INT thr = feas[i].hi - k;
> + while (heap.min_key () < thr)
> + heap.extract_min ();
> + if (heap.nodes () >= 2)
> + {
> + bundle_cand *s = heap.min ();
> + bundle_cand c = {};
> + c.loop = s->loop;
> + c.lock = s->lock;
> + c.unlock = feas[i].unlock;
> + c.lo = s->lo;
> + c.hi = feas[i].hi;
> + c.w = -1; /* Union; the weight sweep fills this in. */
> + c.feasible = true;
> + feas.safe_push (c);
> + }
> + }
> +}
> +
> +static void
> +bundle_compute_weights (vec<bundle_cand> &feas)
> +{
> + /* Step 3: compute containment weights with a Fenwick tree. */
> + feas.qsort (bundle_cand_cmp_hi);
> + unsigned n_all = feas.length ();
> + /* Compress lo values into a sorted, unique list; POS below maps a lo to
> its
> + reversed rank via lower_bound. */
> + auto_vec<HOST_WIDE_INT> los;
> + los.reserve (n_all);
> + for (unsigned p = 0; p < n_all; p++)
> + los.quick_push (feas[p].lo);
> + los.qsort (bundle_hwi_cmp);
> + int m = 0;
> + for (unsigned p = 0; p < n_all; p++)
> + if (m == 0 || los[m - 1] != los[p])
> + los[m++] = los[p];
> + los.truncate (m);
> + /* BIT[1..m] keyed by reversed lo-rank, so a prefix sum [1..pos] covers
> + exactly lo_j >= lo_i. */
> + gcov_type *bit = XCNEWVEC (gcov_type, m + 1);
> + for (unsigned p = 0; p < n_all; p++)
> + {
> + /* Reversed rank of feas[p].lo: LOWER_BOUND is the first LOS >= lo. */
> + int pos = m - (int) los.lower_bound (feas[p].lo, bundle_hwi_less);
> + /* A union is not a real loop, so do not insert it into the tree. Its
> + weight is the contained sum, less one to break ties toward it. */
> + bool u = feas[p].w == -1;
> + if (!u)
> + for (int i = pos; i <= m; i += i & -i)
> + bit[i] += feas[p].w;
> + gcov_type sum = 0;
> + for (int i = pos; i > 0; i -= i & -i)
> + sum += bit[i];
> + feas[p].w = u ? sum - 1 : sum;
> + }
> + XDELETEVEC (bit);
> +}
> +
> +static void
> +bundle_select_and_emit (vec<bundle_cand> &feas)
> +{
> + /* Step 4: select the maximum-weight disjoint set. */
> + unsigned n = feas.length ();
> + /* dp[i+1] is the best weight over feas[0..i]. Whether i is taken is
> + recomputed while backtracking rather than stored. */
> + gcov_type *dp = XCNEWVEC (gcov_type, n + 1);
> +
> + for (unsigned i = 0; i < n; i++)
> + {
> + /* Predecessor: the last candidate that ends before feas[i] starts.
> + hi is inclusive, so find the first candidate with hi >= lo. */
> + bundle_cand key = {};
> + key.hi = feas[i].lo;
> + int p = (int) feas.lower_bound (key, bundle_cand_hi_less) - 1;
> + feas[i].p = p;
> + gcov_type take = feas[i].w + dp[p + 1], skip = dp[i];
> + dp[i + 1] = take >= skip ? take : skip;
> + if (dump_file)
> + fprintf (dump_file,
> + "bundle-align: cand %u loop %d [bb%d..bb%d] "
> + "lo=%ld hi=%ld w=%ld p=%d take=%ld skip=%ld => %s\n",
> + i, feas[i].loop->num, feas[i].lock->index,
> + feas[i].unlock->index, (long) feas[i].lo,
> + (long) feas[i].hi, (long) feas[i].w, p,
> + (long) take, (long) skip, take >= skip ? "take" : "skip");
> + }
> +
> + /* Step 5: emit the selected regions. */
> + for (int i = (int) n - 1; i >= 0;)
> + if (feas[i].w + dp[feas[i].p + 1] >= dp[i])
> + {
> + basic_block h = feas[i].lock, l = feas[i].unlock;
> + emit_insn_before (gen_bundle_lock (), BB_HEAD (h));
> + emit_insn_after (gen_bundle_unlock (), BB_END (l));
> + if (dump_file)
> + fprintf (dump_file,
> + "bundle-align: wrap loop %d [bb%d..bb%d] "
> + "w=%ld span=%ld\n", feas[i].loop->num, h->index,
> + l->index, (long) feas[i].w,
> + (long) (feas[i].hi - feas[i].lo));
> + i = feas[i].p;
> + }
> + else
> + i--;
> +
> + XDELETEVEC (dp);
> +}
> +
> +/* Wrap the selected disjoint hot loop regions in
> .bundle_lock/.bundle_unlock.
> + Each region is an interval on the layout byte axis, which handles rotated
> + (latch-before-header) loops uniformly. */
> +
> +static void
> +ix86_do_bundle_align_loops ()
> +{
> + if (ix86_cost->prefetch_block == 0)
|| number_of_loops (cfun) <= 1
> + return;
> +
> + /* Region size cap: tuning param if set, else the CPU bundle size. */
> + HOST_WIDE_INT k = (param_x86_tight_loop_max_size > 0
> + ? (HOST_WIDE_INT)param_x86_tight_loop_max_size
> + : (HOST_WIDE_INT)ix86_cost->prefetch_block);
> +
> + loop_optimizer_init (AVOID_CFG_MODIFICATIONS);
> +
> + /* Per-block layout byte offsets [bb_lo, bb_hi). */
> + int nbb = last_basic_block_for_fn (cfun);
> + HOST_WIDE_INT *bb_lo = XCNEWVEC (HOST_WIDE_INT, nbb);
> + HOST_WIDE_INT *bb_hi = XCNEWVEC (HOST_WIDE_INT, nbb);
> + /* Running count of inline-asm insns in layout order; [asm_lo, asm_hi)
> lets a
> + candidate test whether any block it spans contains an asm. */
> + int *asm_lo = XCNEWVEC (int, nbb);
> + int *asm_hi = XCNEWVEC (int, nbb);
> + bundle_compute_layout (bb_lo, bb_hi, asm_lo, asm_hi);
> +
> + auto_vec<bundle_cand> feas;
> + bundle_collect_candidates (feas, k, bb_lo, bb_hi, asm_lo, asm_hi);
> + bundle_add_unions (feas, k);
> + bundle_compute_weights (feas);
> + if (!feas.is_empty ())
> + bundle_select_and_emit (feas);
> +
> + XDELETEVEC (bb_lo);
> + XDELETEVEC (bb_hi);
> + XDELETEVEC (asm_lo);
> + XDELETEVEC (asm_hi);
> + loop_optimizer_finalize ();
> +}
> +
> namespace {
>
> -const pass_data pass_data_align_tight_loops =
> +const pass_data pass_data_bundle_align_loops =
> {
> RTL_PASS, /* type */
> - "align_tight_loops", /* name */
> + "bundle_align_loops", /* name */
> OPTGROUP_NONE, /* optinfo_flags */
> TV_MACH_DEP, /* tv_id */
> 0, /* properties_required */
> @@ -5661,40 +5964,44 @@ const pass_data pass_data_align_tight_loops =
> 0, /* todo_flags_finish */
> };
>
> -class pass_align_tight_loops : public rtl_opt_pass
> +class pass_bundle_align_loops : public rtl_opt_pass
> {
> public:
> - pass_align_tight_loops (gcc::context *ctxt)
> - : rtl_opt_pass (pass_data_align_tight_loops, ctxt)
> + pass_bundle_align_loops (gcc::context *ctxt)
> + : rtl_opt_pass (pass_data_bundle_align_loops, ctxt)
> {}
>
> /* opt_pass methods: */
> bool gate (function *) final override
> {
> - return TARGET_ALIGN_TIGHT_LOOPS
> - && optimize
> - && optimize_function_for_speed_p (cfun);
> + return optimize
> + && optimize_function_for_speed_p (cfun)
> + && (ix86_bundle_align_loops > 0 || TARGET_ALIGN_TIGHT_LOOPS);
> }
>
> unsigned int execute (function *) final override
> {
> timevar_push (TV_MACH_DEP);
> + if (ix86_bundle_align_loops > 0)
> + ix86_do_bundle_align_loops ();
> #ifdef ASM_OUTPUT_MAX_SKIP_ALIGN
> - ix86_align_loops ();
> + else
> + ix86_align_loops ();
> #endif
> timevar_pop (TV_MACH_DEP);
> return 0;
> }
> -}; // class pass_align_tight_loops
> +}; // class pass_bundle_align_loops
>
> } // anon namespace
>
> rtl_opt_pass *
> -make_pass_align_tight_loops (gcc::context *ctxt)
> +make_pass_bundle_align_loops (gcc::context *ctxt)
> {
> - return new pass_align_tight_loops (ctxt);
> + return new pass_bundle_align_loops (ctxt);
> }
>
> +
> /* This compares the priority of target features in function DECL1
> and DECL2. It returns positive value if DECL1 is higher priority,
> negative value if DECL2 is higher priority and 0 if they are the
> diff --git a/gcc/config/i386/i386-options.cc b/gcc/config/i386/i386-options.cc
> index e7f06154103..db0a9426b29 100644
> --- a/gcc/config/i386/i386-options.cc
> +++ b/gcc/config/i386/i386-options.cc
> @@ -1872,8 +1872,16 @@ set_ix86_tune_features (struct gcc_options *opts,
> /* Default align_* from the processor table. */
>
> static void
> -ix86_default_align (struct gcc_options *opts)
> +ix86_default_align (struct gcc_options *opts,
> + struct gcc_options *opts_set)
> {
> + /* Bundle alignment replaces generic loop, jump, and label padding. */
> + if (opts->x_ix86_bundle_align_loops > 0)
> + {
> + SET_OPTION_IF_UNSET (opts, opts_set, str_align_loops, "1");
> + SET_OPTION_IF_UNSET (opts, opts_set, str_align_jumps, "1");
> + SET_OPTION_IF_UNSET (opts, opts_set, str_align_labels, "1");
> + }
> /* -falign-foo without argument: supply one. */
> if (opts->x_flag_align_loops && !opts->x_str_align_loops)
> opts->x_str_align_loops = processor_cost_table[ix86_tune]->align_loop;
> @@ -1885,6 +1893,16 @@ ix86_default_align (struct gcc_options *opts)
> opts->x_str_align_functions =
> processor_cost_table[ix86_tune]->align_func;
> }
>
> +/* Default -mbundle-align-loops from the active tune model. */
> +
> +static void
> +ix86_default_bundle_align (struct gcc_options *opts)
> +{
> + if (opts->x_ix86_bundle_align_loops == -1)
> + opts->x_ix86_bundle_align_loops
> + = ix86_tune_features[X86_TUNE_BUNDLE_ALIGN_LOOPS] ? 1 : 0;
> +}
> +
> #ifndef USE_IX86_FRAME_POINTER
> #define USE_IX86_FRAME_POINTER 0
> #endif
> @@ -1986,7 +2004,8 @@ ix86_override_options_after_change_1 (struct
> gcc_options *opts,
> void
> ix86_override_options_after_change (void)
> {
> - ix86_default_align (&global_options);
> + ix86_default_bundle_align (&global_options);
> + ix86_default_align (&global_options, &global_options_set);
>
> ix86_recompute_optlev_based_flags (&global_options, &global_options_set);
>
> @@ -2619,7 +2638,8 @@ ix86_option_override_internal (bool main_args_p,
> opts->x_ix86_regparm = REGPARM_MAX;
>
> /* Default align_* from the processor table. */
> - ix86_default_align (&global_options);
> + ix86_default_bundle_align (&global_options);
> + ix86_default_align (&global_options, &global_options_set);
>
> /* Provide default for -mbranch-cost= value. */
> SET_OPTION_IF_UNSET (opts, opts_set, ix86_branch_cost,
> diff --git a/gcc/config/i386/i386-passes.def b/gcc/config/i386/i386-passes.def
> index 4f461116cac..82d4d7411df 100644
> --- a/gcc/config/i386/i386-passes.def
> +++ b/gcc/config/i386/i386-passes.def
> @@ -31,9 +31,9 @@ along with GCC; see the file COPYING3. If not see
> INSERT_PASS_BEFORE (pass_cse2, 1, pass_stv, true /* timode_p */);
>
> INSERT_PASS_BEFORE (pass_shorten_branches, 1,
> pass_insert_endbr_and_patchable_area);
> - /* pass_align_tight_loops must be after
> pass_insert_endbr_and_patchable_area.
> - PR116174. */
> - INSERT_PASS_BEFORE (pass_shorten_branches, 1, pass_align_tight_loops);
> + /* Tight/bundle loop aligner; after pass_insert_endbr_and_patchable_area
> + (PR116174) and before pass_shorten_branches. */
> + INSERT_PASS_BEFORE (pass_shorten_branches, 1, pass_bundle_align_loops);
>
> INSERT_PASS_AFTER (pass_late_combine, 1, pass_x86_cse);
> INSERT_PASS_AFTER (pass_late_combine, 1,
> pass_remove_partial_avx_dependency);
> diff --git a/gcc/config/i386/i386-protos.h b/gcc/config/i386/i386-protos.h
> index 2687de646b0..84d2a7ad74c 100644
> --- a/gcc/config/i386/i386-protos.h
> +++ b/gcc/config/i386/i386-protos.h
> @@ -434,7 +434,7 @@ extern rtl_opt_pass
> *make_pass_remove_partial_avx_dependency
> (gcc::context *);
> extern rtl_opt_pass *make_pass_x86_cse (gcc::context *);
> extern rtl_opt_pass *make_pass_apx_nf_convert (gcc::context *);
> -extern rtl_opt_pass *make_pass_align_tight_loops (gcc::context *);
> +extern rtl_opt_pass *make_pass_bundle_align_loops (gcc::context *);
>
> extern bool ix86_has_no_direct_extern_access;
> extern bool ix86_rpad_gate ();
> diff --git a/gcc/config/i386/i386.md b/gcc/config/i386/i386.md
> index 5768a93d5b0..f85010a41aa 100644
> --- a/gcc/config/i386/i386.md
> +++ b/gcc/config/i386/i386.md
> @@ -233,6 +233,8 @@
> UNSPECV_STACK_PROBE
> UNSPECV_PROBE_STACK_RANGE
> UNSPECV_ALIGN
> + UNSPECV_BUNDLE_LOCK
> + UNSPECV_BUNDLE_UNLOCK
> UNSPECV_PROLOGUE_USE
> UNSPECV_SPLIT_STACK_RETURN
> UNSPECV_CLD
> @@ -21756,6 +21758,39 @@
> }
> [(set_attr "length" "16")])
>
> +;; ix86_bundle_align_loops pass: align the locked region's start to the cache
> +;; line (prefetch_block). Final instruction sizes may grow during assembly
> +;; because of indirect calls, prefixes, and relaxation, so a selected region
> +;; can exceed one bundle; flex .bundle_lock accepts such a region instead of
> +;; raising an error. Requires flex .bundle_lock support.
> +(define_insn "bundle_lock"
> + [(unspec_volatile [(const_int 0)] UNSPECV_BUNDLE_LOCK)]
> + ""
> +{
> +#ifdef HAVE_GAS_BUNDLE_FLEX_LOCK
> + fprintf (asm_out_file, "\t.bundle_align_mode %d\n\t.bundle_lock flex\n",
> + ceil_log2 (ix86_cost->prefetch_block));
> +#endif
> + return "";
> +}
> + [(set_attr "length" "0")
> + (set_attr "length_immediate" "0")
> + (set_attr "modrm" "0")])
> +
> +;; Close the bundle and turn bundle alignment mode back off.
> +(define_insn "bundle_unlock"
> + [(unspec_volatile [(const_int 0)] UNSPECV_BUNDLE_UNLOCK)]
> + ""
> +{
> +#ifdef HAVE_GAS_BUNDLE_FLEX_LOCK
> + fputs ("\t.bundle_unlock\n\t.bundle_align_mode 0\n", asm_out_file);
> +#endif
> + return "";
> +}
> + [(set_attr "length" "0")
> + (set_attr "length_immediate" "0")
> + (set_attr "modrm" "0")])
> +
> (define_expand "prologue"
> [(const_int 0)]
> ""
> diff --git a/gcc/config/i386/i386.opt b/gcc/config/i386/i386.opt
> index 57ba3fbfa8f..148d26b6053 100644
> --- a/gcc/config/i386/i386.opt
> +++ b/gcc/config/i386/i386.opt
> @@ -240,6 +240,10 @@ malign-loops=
> Target Undocumented RejectNegative Joined UInteger
> Loop code aligned to this power of 2.
>
> +mbundle-align-loops
> +Target Var(ix86_bundle_align_loops) Init(-1) Save
> +Wrap small hot loops with .bundle_lock so the assembler keeps them inside
> one cache line.
> +
> malign-stringops
> Target RejectNegative InverseMask(NO_ALIGN_STRINGOPS, ALIGN_STRINGOPS) Save
> Align destination of the string operations.
> @@ -1221,6 +1225,14 @@ Do not use GOT to access external symbols.
> Target Joined UInteger Var(x86_stlf_window_ninsns) Init(64) Param
> Instructions number above which STFL stall penalty can be compensated.
>
> +-param=x86-tight-loop-max-size=
> +Target Joined UInteger Var(param_x86_tight_loop_max_size) Init(0) Param
> +Maximum body size in bytes for the i386 tight-loop aligner; 0 picks the
> active tune prefetch_block.
> +
> +-param=x86-bundle-align-branch-limit=
> +Target Joined UInteger Var(param_x86_bundle_align_branch_limit) Init(2) Param
> +Maximum internal conditional branches allowed in a bundle-locked loop region
> (the back-edge branch is not counted).
> +
> mavxifma
> Target Mask(ISA2_AVXIFMA) Var(ix86_isa_flags2) Save
> Support MMX, SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2, AVX, AVX2, and
> diff --git a/gcc/config/i386/i386.opt.urls b/gcc/config/i386/i386.opt.urls
> index d52e6de8014..20a638ea1f7 100644
> --- a/gcc/config/i386/i386.opt.urls
> +++ b/gcc/config/i386/i386.opt.urls
> @@ -28,6 +28,9 @@ UrlSuffix(gcc/x86-Options.html#index-malign-double)
>
> ; skipping UrlSuffix for 'malign-loops=' due to finding no URLs
>
> +mbundle-align-loops
> +UrlSuffix(gcc/x86-Options.html#index-mbundle-align-loops)
> +
> malign-stringops
> UrlSuffix(gcc/x86-Options.html#index-malign-stringops)
>
> diff --git a/gcc/config/i386/x86-tune.def b/gcc/config/i386/x86-tune.def
> index d4c27351ad7..4716f26b2e1 100644
> --- a/gcc/config/i386/x86-tune.def
> +++ b/gcc/config/i386/x86-tune.def
> @@ -259,6 +259,13 @@ DEF_TUNE (X86_TUNE_ALIGN_TIGHT_LOOPS,
> "align_tight_loops",
> ~(m_ZHAOXIN | m_CASCADELAKE | m_SKYLAKE_AVX512
> | m_CORE_ATOM))
>
> +/* X86_TUNE_BUNDLE_ALIGN_LOOPS: when true, use the ix86_bundle_align_loops
> + pass with the assembler's .bundle_lock/.bundle_unlock directives instead
> + of static .p2align padding. Targets cores where loops straddling a
> 64-byte
> + instruction-cache line are particularly costly (AMD Zen4/Zen5/Zen6). */
> +DEF_TUNE (X86_TUNE_BUNDLE_ALIGN_LOOPS, "bundle_align_loops",
> + m_ZNVER4 | m_ZNVER5 | m_ZNVER6)
> +
>
> /*****************************************************************************/
> /* Integer instruction selection tuning
> */
>
> /*****************************************************************************/
> diff --git a/gcc/configure b/gcc/configure
> index 43ca45144b8..dd5a0c0a539 100755
> --- a/gcc/configure
> +++ b/gcc/configure
> @@ -26306,6 +26306,44 @@ $as_echo "#define HAVE_GAS_MAX_SKIP_P2ALIGN 1"
> >>confdefs.h
> fi
>
>
> +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking assembler for .bundle_lock
> flex argument" >&5
> +$as_echo_n "checking assembler for .bundle_lock flex argument... " >&6; }
> +if ${gcc_cv_as_bundle_flex_lock+:} false; then :
> + $as_echo_n "(cached) " >&6
> +else
> + gcc_cv_as_bundle_flex_lock=no
> + if test x"$gcc_cv_as" != x; then
> + $as_echo '.bundle_align_mode 6
> +.bundle_lock flex
> +.rept 100
> +nop
> +.endr
> +.bundle_unlock' > conftest.s
> + if { ac_try='$gcc_cv_as $gcc_cv_as_flags -o conftest.o conftest.s >&5'
> + { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_try\""; } >&5
> + (eval $ac_try) 2>&5
> + ac_status=$?
> + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
> + test $ac_status = 0; }; }
> + then
> + gcc_cv_as_bundle_flex_lock=yes
> + else
> + echo "configure: failed program was" >&5
> + cat conftest.s >&5
> + fi
> + rm -f conftest.o conftest.s
> + fi
> +fi
> +{ $as_echo "$as_me:${as_lineno-$LINENO}: result:
> $gcc_cv_as_bundle_flex_lock" >&5
> +$as_echo "$gcc_cv_as_bundle_flex_lock" >&6; }
> +if test $gcc_cv_as_bundle_flex_lock = yes; then
> +
> +$as_echo "#define HAVE_GAS_BUNDLE_FLEX_LOCK 1" >>confdefs.h
> +
> +fi
> +
> +
> +
> { $as_echo "$as_me:${as_lineno-$LINENO}: checking assembler for .literal16"
> >&5
> $as_echo_n "checking assembler for .literal16... " >&6; }
> if ${gcc_cv_as_literal16+:} false; then :
> diff --git a/gcc/configure.ac b/gcc/configure.ac
> index 075b6b68ffa..346bf254de3 100644
> --- a/gcc/configure.ac
> +++ b/gcc/configure.ac
> @@ -3142,6 +3142,17 @@ gcc_GAS_CHECK_FEATURE([.p2align with maximum skip],
> gcc_cv_as_max_skip_p2align,,
> [Define if your assembler supports specifying the maximum number
> of bytes to skip when using the GAS .p2align command.])])
>
> +gcc_GAS_CHECK_FEATURE([.bundle_lock flex argument],
> gcc_cv_as_bundle_flex_lock,,
> + [.bundle_align_mode 6
> +.bundle_lock flex
> +.rept 100
> +nop
> +.endr
> +.bundle_unlock],,
> +[AC_DEFINE(HAVE_GAS_BUNDLE_FLEX_LOCK, 1,
> + [Define if your assembler accepts the flex argument to .bundle_lock
> + that permits bundles larger than .bundle_align_mode alignment.])])
> +
> gcc_GAS_CHECK_FEATURE([.literal16], gcc_cv_as_literal16,,
> [.literal16],,
> [AC_DEFINE(HAVE_GAS_LITERAL16, 1,
> diff --git a/gcc/doc/extend.texi b/gcc/doc/extend.texi
> index 397b05ab87d..9081317927d 100644
> --- a/gcc/doc/extend.texi
> +++ b/gcc/doc/extend.texi
> @@ -9598,6 +9598,12 @@ operations and calling the library routines for large
> operations.
> @itemx no-align-stringops
> Do/do not align destination of inlined string operations.
>
> +@atindex @code{target("bundle-align-loops")}, x86
> +@item bundle-align-loops
> +@itemx no-bundle-align-loops
> +Enable/disable wrapping of small hot loops in assembler
> +@code{.bundle_lock}/@code{.bundle_unlock} directives.
> +
> @atindex @code{target("recip")}, x86
> @item recip
> @itemx no-recip
> diff --git a/gcc/doc/invoke.texi b/gcc/doc/invoke.texi
> index e1870c9c227..619f92185fd 100644
> --- a/gcc/doc/invoke.texi
> +++ b/gcc/doc/invoke.texi
> @@ -1554,7 +1554,8 @@ See RS/6000 and PowerPC Options.
> -mamx-complex -mavxvnniint16 -msm3 -msha512 -msm4 -mapxf
> -musermsr -mavx10.1 -mavx10.2 -mamx-avx512 -mmovrs -mamx-movrs
> -mavx512bmm -mavx10v2aux
> --mcldemote -mms-bitfields -mno-align-stringops -minline-all-stringops
> +-mcldemote -mms-bitfields -mbundle-align-loops -mno-align-stringops
> +-minline-all-stringops
> -minline-stringops-dynamically -mstringop-strategy=@var{alg}
> -mkl -mwidekl
> -mmemcpy-strategy=@var{strategy} -mmemset-strategy=@var{strategy}
> @@ -37155,6 +37156,18 @@ Here, @code{t5} takes up 2 bytes.
> @end enumerate
>
>
> +@opindex mbundle-align-loops
> +@opindex mno-bundle-align-loops
> +@item -mbundle-align-loops
> +@itemx -mno-bundle-align-loops
> +Wrap small hot loops in assembler @code{.bundle_lock}/@code{.bundle_unlock}
> +directives so the assembler keeps each loop within a single instruction-cache
> +line, aligning the loop header without inserting executed padding. Enabled
> by
> +default for AMD Zen4 and later. It defaults @option{-falign-loops},
> +@option{-falign-jumps}, and @option{-falign-labels} to 1 unless explicitly
> set.
> +The maximum loop-body size considered is set by the
> +@option{x86-tight-loop-max-size} parameter.
> +
> @opindex mno-align-stringops
> @opindex malign-stringops
> @item -mno-align-stringops
> diff --git a/gcc/doc/sourcebuild.texi b/gcc/doc/sourcebuild.texi
> index aad51550eba..35324653699 100644
> --- a/gcc/doc/sourcebuild.texi
> +++ b/gcc/doc/sourcebuild.texi
> @@ -3114,6 +3114,10 @@ Target supports @option{-fstack-protector}.
> @item gas
> Target uses GNU @command{as}.
>
> +@item gcc_bundle_align_loops
> +Target compiler and assembler support @option{-mbundle-align-loops} and
> +@code{.bundle_lock flex}.
> +
> @item gc_sections
> Target supports @option{--gc-sections}.
>
> diff --git a/gcc/testsuite/gcc.target/i386/bundle-align-1.c
> b/gcc/testsuite/gcc.target/i386/bundle-align-1.c
> new file mode 100644
> index 00000000000..c7120731f5f
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/i386/bundle-align-1.c
> @@ -0,0 +1,16 @@
> +/* { dg-do compile { target { ! ia32 } } } */
> +/* { dg-options "-O3 -march=znver3 -falign-loops=1 -falign-jumps=1
> -falign-labels=1 -mbundle-align-loops" } */
> +/* { dg-require-effective-target gcc_bundle_align_loops } */
> +/* { dg-final { scan-assembler "\\.bundle_lock flex" } } */
> +/* { dg-final { scan-assembler "\\.bundle_unlock" } } */
> +
> +typedef int (*callee_t) (int, int *);
> +
> +int
> +indirect_sum (callee_t callee, int n, int *array, long calls)
> +{
> + int sink = 0;
> + for (long call = 0; call < calls; ++call)
> + sink += callee (n, array);
> + return sink;
> +}
> diff --git a/gcc/testsuite/gcc.target/i386/bundle-align-2.c
> b/gcc/testsuite/gcc.target/i386/bundle-align-2.c
> new file mode 100644
> index 00000000000..957e853dfb2
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/i386/bundle-align-2.c
> @@ -0,0 +1,21 @@
> +/* { dg-do compile { target { ! ia32 } } } */
> +/* { dg-options "-O3 -march=znver3 -falign-loops=1 -falign-jumps=1
> -falign-labels=1 -mbundle-align-loops" } */
> +/* { dg-require-effective-target gcc_bundle_align_loops } */
> +/* { dg-final { scan-assembler "\\.bundle_lock flex" } } */
> +/* { dg-final { scan-assembler "\\.bundle_unlock" } } */
> +
> +__attribute__((noinline,noipa))
> +int
> +f (int n, int *p)
> +{
> + int i = 0, s = 0;
> + if (n & 1)
> + goto mid;
> + while (i < n)
> + {
> + s += p[i & 1023];
> + mid:
> + i++;
> + }
> + return s;
> +}
> diff --git a/gcc/testsuite/gcc.target/i386/bundle-align-3.c
> b/gcc/testsuite/gcc.target/i386/bundle-align-3.c
> new file mode 100644
> index 00000000000..7cdfa8db985
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/i386/bundle-align-3.c
> @@ -0,0 +1,26 @@
> +/* { dg-do compile { target { ! ia32 } } } */
> +/* { dg-options "-O3 -march=znver3 -falign-loops=1 -falign-jumps=1
> -falign-labels=1 -mbundle-align-loops" } */
> +/* { dg-require-effective-target gcc_bundle_align_loops } */
> +/* { dg-final { scan-assembler "\\.bundle_lock flex" } } */
> +/* { dg-final { scan-assembler "\\.bundle_unlock" } } */
> +
> +int a[1024];
> +__attribute__((hot,noinline,noipa))
> +int
> +two_backedges (int n, int m)
> +{
> + int i = 0, s = 0;
> +outer_head:
> + {
> + int j = 0;
> + inner_head:
> + s += a[(i + j) & 1023];
> + if (__builtin_expect (s < 0, 0))
> + goto outer_head;
> + if (++j < m)
> + goto inner_head;
> + }
> + if (++i < n)
> + goto outer_head;
> + return s;
> +}
> diff --git a/gcc/testsuite/gcc.target/i386/bundle-align-4.c
> b/gcc/testsuite/gcc.target/i386/bundle-align-4.c
> new file mode 100644
> index 00000000000..80bca4ec200
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/i386/bundle-align-4.c
> @@ -0,0 +1,22 @@
> +/* { dg-do compile { target { ! ia32 } } } */
> +/* { dg-options "-O3 -march=znver5 -falign-loops=1 -falign-jumps=1
> -falign-labels=1 -mbundle-align-loops" } */
> +/* { dg-require-effective-target gcc_bundle_align_loops } */
> +/* { dg-final { scan-assembler-times "\\.bundle_lock flex" 1 } } */
> +/* { dg-final { scan-assembler-times "\\.bundle_unlock" 1 } } */
> +
> +volatile int sink;
> +int a[1024];
> +int
> +classify (int n)
> +{
> + int s = 0;
> + asm volatile (".rept 0\n\t.byte 0x90\n\t.endr" ::: "memory");
> + for (int i = 0; i < n; i++)
> + {
> + if (a[i] & 1) sink = i;
> + if (a[i] & 2) sink = i + 1;
> + if (a[i] & 4) s += 4;
> + s += a[i];
> + }
> + return s;
> +}
> diff --git a/gcc/testsuite/gcc.target/i386/bundle-align-5.c
> b/gcc/testsuite/gcc.target/i386/bundle-align-5.c
> new file mode 100644
> index 00000000000..7aa07effa6e
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/i386/bundle-align-5.c
> @@ -0,0 +1,17 @@
> +/* { dg-do compile { target { ! ia32 } } } */
> +/* { dg-options "-O3 -march=znver3 -falign-loops=1 -falign-jumps=1
> -falign-labels=1 -mbundle-align-loops" } */
> +/* { dg-require-effective-target gcc_bundle_align_loops } */
> +/* { dg-final { scan-assembler "\\.bundle_lock flex" } } */
> +/* { dg-final { scan-assembler "\\.bundle_unlock" } } */
> +
> +void *mc (const void *s, int c, unsigned long n)
> +{
> + const unsigned char *p = s;
> + while (n--)
> + {
> + if (*p == (unsigned char) c)
> + return (void *) p;
> + p++;
> + }
> + return 0;
> +}
> diff --git a/gcc/testsuite/gcc.target/i386/bundle-align-6.c
> b/gcc/testsuite/gcc.target/i386/bundle-align-6.c
> new file mode 100644
> index 00000000000..1f561e07caa
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/i386/bundle-align-6.c
> @@ -0,0 +1,24 @@
> +/* { dg-do compile { target { ! ia32 } } } */
> +/* { dg-options "-O3 -march=znver3 -falign-loops=1 -falign-jumps=1
> -falign-labels=1 -mbundle-align-loops" } */
> +/* { dg-require-effective-target gcc_bundle_align_loops } */
> +/* { dg-final { scan-assembler-times "\\.bundle_lock flex" 1 } } */
> +/* { dg-final { scan-assembler-times "\\.bundle_unlock" 1 } } */
> +
> +struct O { long rc; };
> +extern void dealloc (struct O *);
> +void
> +f (struct O **v, long n)
> +{
> + asm volatile (".rept 10\n\t.byte 0x90\n\t.endr" ::: "memory");
> + while (--n >= 0)
> + {
> + struct O *x = v[n];
> + if (x)
> + {
> + if (x->rc < 0)
> + ;
> + else if (--x->rc == 0)
> + dealloc (x);
> + }
> + }
> +}
> diff --git a/gcc/testsuite/gcc.target/i386/bundle-align-7.c
> b/gcc/testsuite/gcc.target/i386/bundle-align-7.c
> new file mode 100644
> index 00000000000..b713d25d0b7
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/i386/bundle-align-7.c
> @@ -0,0 +1,17 @@
> +/* { dg-do compile { target { ! ia32 } } } */
> +/* { dg-options "-O1 -march=znver3 -falign-loops=1 -falign-jumps=1
> -falign-labels=1 -mbundle-align-loops" } */
> +/* { dg-require-effective-target gcc_bundle_align_loops } */
> +/* { dg-final { scan-assembler "\\.bundle_lock flex" } } */
> +/* { dg-final { scan-assembler "\\.bundle_unlock" } } */
> +
> +static int A[1 << 20];
> +
> +int
> +nested_sum (int n, int m)
> +{
> + int sum = 0;
> + for (int i = 0; i < n; ++i)
> + for (int j = 0; j < m; ++j)
> + sum += A[i * m + j];
> + return sum;
> +}
> diff --git a/gcc/testsuite/lib/target-supports.exp
> b/gcc/testsuite/lib/target-supports.exp
> index d467e9fa3b7..3286c40b07d 100644
> --- a/gcc/testsuite/lib/target-supports.exp
> +++ b/gcc/testsuite/lib/target-supports.exp
> @@ -12491,6 +12491,22 @@ proc check_effective_target_gas { } {
> return $use_gas_saved
> }
>
> +# Return 1 if the compiler was configured with assembler support for
> +# the flex .bundle_lock argument used by -mbundle-align-loops.
> +
> +proc check_effective_target_gcc_bundle_align_loops { } {
> + return [check_no_messages_and_pattern gcc_bundle_align_loops \
> + "\\.bundle_lock flex" assembly {
> + int a[1024];
> + int foo (int n) {
> + int s = 0;
> + for (int i = 0; i < n; i++)
> + s += a[i];
> + return s;
> + }
> + } "-O2 -march=znver4 -mbundle-align-loops -falign-loops=1"]
> +}
> +
> # Return 1 if GNU ld is used.
>
> proc check_effective_target_gld { } {
> --
> 2.34.1
>
