On Sat Oct 3, 2026 at 7:02 AM PDT, Sarvesh Chandra wrote:
> Add ix86_bundle_align_loops, an RTL pass that marks selected loop regions
> with .bundle_lock flex. The assembler aligns each marked region after
> instruction relaxation, when the final instruction sizes are known, and
> flex mode accepts a region that ends up larger than one bundle.
>
> Candidates are formed from layout-backward loop edges and selected as a
> disjoint set by weighted interval scheduling.
>
> Enable the pass for Zen4 and later, add the target options and assembler
> feature check, and add tests for candidate formation, union construction,
> weight calculation, selection, and representative loop cases.
>
> gcc/
> * config/i386/i386-features.cc (struct bundle_cand): New.
> (bundle_cand_cmp_hi, bundle_hwi_cmp, bundle_hwi_less)
> (bundle_cand_hi_less, bundle_compute_layout)
> (bundle_collect_candidates, bundle_add_unions)
> (bundle_compute_weights, bundle_select_and_emit)
> (ix86_do_bundle_align_loops): New functions.
> (class pass_bundle_align_loops): New.
> (make_pass_bundle_align_loops): New function.
> * config/i386/i386-protos.h (make_pass_bundle_align_loops): Declare.
> * config/i386/i386-passes.def (pass_bundle_align_loops): Insert before
> pass_shorten_branches.
> * config/i386/i386-options.cc (ix86_default_bundle_align): New function.
> (ix86_default_align): Call it.
> * config/i386/i386.md (bundle_lock, bundle_unlock): New insns.
> * config/i386/i386.opt (mbundle-align-loops): New option.
> (x86-tight-loop-max-size, x86-bundle-align-branch-limit): New params.
> * config/i386/i386.opt.urls: Regenerate.
> * config/i386/x86-tune.def (X86_TUNE_BUNDLE_ALIGN_LOOPS): New tune flag.
> * configure.ac: Check for .bundle_lock flex support.
> * configure: Regenerate.
> * config.in: Regenerate.
> * doc/invoke.texi (-mbundle-align-loops, x86-tight-loop-max-size):
> Document.
> * doc/extend.texi (x86 Function Attributes): Document
> bundle-align-loops.
> * doc/sourcebuild.texi (gcc_bundle_align_loops): Document.
>
> gcc/testsuite/
> * gcc.target/i386/bundle-align-1.c: New test.
> * gcc.target/i386/bundle-align-2.c: New test.
> * gcc.target/i386/bundle-align-3.c: New test.
> * gcc.target/i386/bundle-align-4.c: New test.
> * gcc.target/i386/bundle-align-5.c: New test.
> * gcc.target/i386/bundle-align-6.c: New test.
> * gcc.target/i386/bundle-align-7.c: New test.
> * lib/target-supports.exp
> (check_effective_target_gcc_bundle_align_loops): New.
>
> Co-authored-by: Branden Moore <[email protected]>
> Co-authored-by: Venkataramanan Kumar <[email protected]>
> Signed-off-by: Sarvesh Chandra <[email protected]>
> ---
> [Presented at GNU Tools Cauldron 2026.]
>
> Cosmetic changes in v2:
> - Add co-authors Branden Moore and Venkataramanan Kumar.
> - Drop the slide references from the test comments. They pointed at
> the Cauldron presentation, but GCC does not allow attaching the PPT
> to the patch, so the references are irrelevant here.
> - Correct the i386.md comment: the simple flex mode accepts an
> oversized locked region and still aligns its start; it does not skip
> padding. An experimental post-relaxation patch does skip padding,
> but that is not part of this proposal.
> - bundle-align-5.c: rename the scan kernel instead of redefining
> memchr with -D.
> - bundle-align-6.c: drop the unneeded x86-tight-loop-max-size param.
> - Reduce the tests to the scanned kernel: drop the main drivers and
> headers.
>
> gcc/config.in | 6 +
> gcc/config/i386/i386-features.cc | 331 +++++++++++++++++-
> gcc/config/i386/i386-options.cc | 26 +-
> gcc/config/i386/i386-passes.def | 6 +-
> gcc/config/i386/i386-protos.h | 2 +-
> gcc/config/i386/i386.md | 35 ++
> gcc/config/i386/i386.opt | 12 +
> gcc/config/i386/i386.opt.urls | 3 +
> gcc/config/i386/x86-tune.def | 7 +
> gcc/configure | 38 ++
> gcc/configure.ac | 11 +
> gcc/doc/extend.texi | 6 +
> gcc/doc/invoke.texi | 15 +-
> gcc/doc/sourcebuild.texi | 4 +
> .../gcc.target/i386/bundle-align-1.c | 16 +
> .../gcc.target/i386/bundle-align-2.c | 21 ++
> .../gcc.target/i386/bundle-align-3.c | 26 ++
> .../gcc.target/i386/bundle-align-4.c | 22 ++
> .../gcc.target/i386/bundle-align-5.c | 17 +
> .../gcc.target/i386/bundle-align-6.c | 24 ++
> .../gcc.target/i386/bundle-align-7.c | 17 +
> gcc/testsuite/lib/target-supports.exp | 16 +
> 22 files changed, 641 insertions(+), 20 deletions(-)
> create mode 100644 gcc/testsuite/gcc.target/i386/bundle-align-1.c
> create mode 100644 gcc/testsuite/gcc.target/i386/bundle-align-2.c
> create mode 100644 gcc/testsuite/gcc.target/i386/bundle-align-3.c
> create mode 100644 gcc/testsuite/gcc.target/i386/bundle-align-4.c
> create mode 100644 gcc/testsuite/gcc.target/i386/bundle-align-5.c
> create mode 100644 gcc/testsuite/gcc.target/i386/bundle-align-6.c
> create mode 100644 gcc/testsuite/gcc.target/i386/bundle-align-7.c
>
> diff --git a/gcc/config.in b/gcc/config.in
> index b90caa0dd57..51dac287796 100644
> --- a/gcc/config.in
> +++ b/gcc/config.in
> @@ -1448,6 +1448,12 @@
> #endif
>
>
> +/* Define if your assembler accepts the flex argument to .bundle_lock. */
> +#ifndef USED_FOR_TARGET
> +#undef HAVE_GAS_BUNDLE_FLEX_LOCK
> +#endif
> +
> +
> /* Define 0/1 if your assembler supports CFI directives. */
> #undef HAVE_GAS_CFI_DIRECTIVE
>
> diff --git a/gcc/config/i386/i386-features.cc
> b/gcc/config/i386/i386-features.cc
> index eb5f82e42cb..6743c0ebcfd 100644
> --- a/gcc/config/i386/i386-features.cc
> +++ b/gcc/config/i386/i386-features.cc
> @@ -83,6 +83,7 @@ along with GCC; see the file COPYING3. If not see
> #include "ifcvt.h"
> #include "symbol-summary.h"
> #include "sreal.h"
> +#include "fibonacci_heap.h"
> #include "ipa-cp.h"
> #include "ipa-prop.h"
> #include "ipa-fnsummary.h"
> @@ -5646,12 +5647,314 @@ ix86_align_loops ()
> free_dominance_info (CDI_DOMINATORS);
> }
>
> +/* ix86_bundle_align_loops pass.
> +
> + 1. Compute layout offsets and identify feasible loop-backward edges.
> + 2. Add synthetic unions for candidates sharing a K-byte window.
> + 3. Weight candidates by the back-edge counts of loops contained in
> + them.
> + 4. Select a maximum-weight disjoint set with weighted interval
> + scheduling.
> + 5. Emit .bundle_lock flex for the selected regions.
> +
> + K is param_x86_tight_loop_max_size, or ix86_cost->prefetch_block when
> + unset. */
> +
> +/* A candidate loop region: the byte interval [lo, hi] on the layout axis
> + (hi inclusive), weight w, interval-scheduling predecessor p, the lock and
> + unlock blocks, and a feasibility flag. A synthetic union has w == -1
> until
> + the weight sweep replaces it with its contained weight. */
> +
> +struct bundle_cand
> +{
> + class loop *loop;
> + basic_block lock, unlock;
> + HOST_WIDE_INT lo, hi;
> + gcov_type w;
> + int p;
> + bool feasible;
> +};
> +
> +/* Sort by hi ascending. Break ties by lo descending (a contained candidate
> + before its container), then by real loops before a synthetic union of the
> + same span. */
> +
> +static int
> +bundle_cand_cmp_hi (const void *a, const void *b)
> +{
> + const bundle_cand *x = (const bundle_cand *)a, *y = (const bundle_cand *)b;
> + if (x->hi != y->hi)
> + return x->hi < y->hi ? -1 : 1;
> + if (x->lo != y->lo)
> + return x->lo > y->lo ? -1 : 1;
> + if ((x->w == -1) != (y->w == -1))
> + return x->w == -1 ? 1 : -1;
> + return 0;
> +}
> +
> +/* qsort comparator for HOST_WIDE_INT, ascending (lo-value compression). */
> +
> +static int
> +bundle_hwi_cmp (const void *a, const void *b)
> +{
> + HOST_WIDE_INT x = *(const HOST_WIDE_INT *)a, y = *(const HOST_WIDE_INT *)b;
> + return x < y ? -1 : x > y ? 1 : 0;
> +}
> +
> +/* Less-than on HOST_WIDE_INT, for vec::lower_bound over the compressed
> + lo-value list. */
> +
> +static bool
> +bundle_hwi_less (const HOST_WIDE_INT &a, const HOST_WIDE_INT &b)
> +{
> + return a < b;
> +}
> +
> +/* Less-than on interval end, for vec::lower_bound in the WIS predecessor
> + search over the hi-sorted feasible set. */
> +
> +static bool
> +bundle_cand_hi_less (const bundle_cand &a, const bundle_cand &b)
> +{
> + return a.hi < b.hi;
> +}
> +
> +static void
> +bundle_compute_layout (HOST_WIDE_INT *bb_lo, HOST_WIDE_INT *bb_hi,
> + int *asm_lo, int *asm_hi)
> +{
> + HOST_WIDE_INT off = 0;
> + int nasm = 0;
> + basic_block bb;
> + FOR_EACH_BB_FN (bb, cfun)
> + {
> + rtx_insn *insn;
> + bb_lo[bb->index] = off;
> + asm_lo[bb->index] = nasm;
> + FOR_BB_INSNS (bb, insn)
> + if (NONDEBUG_INSN_P (insn))
> + {
> + /* Inline asm has no reliable min size; count its presence so any
> + candidate spanning this block can be excluded outright. */
> + if (asm_noperands (PATTERN (insn)) >= 0)
> + nasm++;
> + else
> + off += ix86_min_insn_size (insn);
> + }
> + bb_hi[bb->index] = off;
> + asm_hi[bb->index] = nasm;
> + }
> +}
> +
> +static void
> +bundle_collect_candidates (vec<bundle_cand> &feas, HOST_WIDE_INT k,
> + HOST_WIDE_INT *bb_lo, HOST_WIDE_INT *bb_hi,
> + int *asm_lo, int *asm_hi)
> +{
> + /* Step 1: identify feasible candidates from layout-backward loop edges.
> */
> + basic_block bb;
> + FOR_EACH_BB_FN (bb, cfun)
> + {
> + edge e;
> + edge_iterator ei;
> + FOR_EACH_EDGE (e, ei, bb->succs)
> + if (e->count ().initialized_p ()
> + && e->dest->index >= NUM_FIXED_BLOCKS
> + && (e->dest == bb
> + || bb_lo[e->dest->index] < bb_lo[bb->index]))
> + {
> + bundle_cand c = {};
> + c.loop = bb->loop_father;
> + c.lock = e->dest;
> + c.unlock = bb;
> + c.lo = bb_lo[e->dest->index];
> + c.hi = bb_hi[bb->index];
> + c.w = e->count ().to_gcov_type ();
> + /* Both ends must lie in a real loop, not the function-wide fake
> + loop whose latch is EXIT. This rejects a layout-backward exit
> + edge whose destination is outside any loop. */
> + c.feasible = (c.loop
> + && c.loop->latch != EXIT_BLOCK_PTR_FOR_FN (cfun)
> + && e->dest->loop_father
> + && (e->dest->loop_father->latch
> + != EXIT_BLOCK_PTR_FOR_FN (cfun))
> + && asm_hi[bb->index] == asm_lo[e->dest->index]
> + && c.hi - c.lo <= k
> + && optimize_bb_for_speed_p (c.lock));
> + if (c.feasible)
> + feas.safe_push (c);
> + }
> + }
> +}
> +
> +static void
> +bundle_add_unions (vec<bundle_cand> &feas, HOST_WIDE_INT k)
> +{
> + /* Step 2: add synthetic unions for candidates sharing a K-byte window. */
> + feas.qsort (bundle_cand_cmp_hi);
> + unsigned base = feas.length ();
> + /* At most one union is added per candidate, so reserve now; this keeps the
> + &feas[i] pointers held by the heap valid across the push below. */
> + feas.reserve (2 * base);
> + fibonacci_heap<HOST_WIDE_INT, bundle_cand> heap (HOST_WIDE_INT_MIN);
> + for (unsigned i = 0; i < base; i++)
> + {
> + heap.insert (feas[i].lo, &feas[i]);
> + HOST_WIDE_INT thr = feas[i].hi - k;
> + while (heap.min_key () < thr)
> + heap.extract_min ();
> + if (heap.nodes () >= 2)
> + {
> + bundle_cand *s = heap.min ();
> + bundle_cand c = {};
> + c.loop = s->loop;
> + c.lock = s->lock;
> + c.unlock = feas[i].unlock;
> + c.lo = s->lo;
> + c.hi = feas[i].hi;
> + c.w = -1; /* Union; the weight sweep fills this in. */
> + c.feasible = true;
> + feas.safe_push (c);
> + }
> + }
> +}
> +
> +static void
> +bundle_compute_weights (vec<bundle_cand> &feas)
> +{
> + /* Step 3: compute containment weights with a Fenwick tree. */
> + feas.qsort (bundle_cand_cmp_hi);
> + unsigned n_all = feas.length ();
> + /* Compress lo values into a sorted, unique list; POS below maps a lo to
> its
> + reversed rank via lower_bound. */
> + auto_vec<HOST_WIDE_INT> los;
> + los.reserve (n_all);
> + for (unsigned p = 0; p < n_all; p++)
> + los.quick_push (feas[p].lo);
> + los.qsort (bundle_hwi_cmp);
> + int m = 0;
> + for (unsigned p = 0; p < n_all; p++)
> + if (m == 0 || los[m - 1] != los[p])
> + los[m++] = los[p];
> + los.truncate (m);
> + /* BIT[1..m] keyed by reversed lo-rank, so a prefix sum [1..pos] covers
> + exactly lo_j >= lo_i. */
> + gcov_type *bit = XCNEWVEC (gcov_type, m + 1);
> + for (unsigned p = 0; p < n_all; p++)
> + {
> + /* Reversed rank of feas[p].lo: LOWER_BOUND is the first LOS >= lo. */
> + int pos = m - (int) los.lower_bound (feas[p].lo, bundle_hwi_less);
> + /* A union is not a real loop, so do not insert it into the tree. Its
> + weight is the contained sum, less one to break ties toward it. */
> + bool u = feas[p].w == -1;
> + if (!u)
> + for (int i = pos; i <= m; i += i & -i)
> + bit[i] += feas[p].w;
> + gcov_type sum = 0;
> + for (int i = pos; i > 0; i -= i & -i)
> + sum += bit[i];
> + feas[p].w = u ? sum - 1 : sum;
> + }
> + XDELETEVEC (bit);
> +}
> +
> +static void
> +bundle_select_and_emit (vec<bundle_cand> &feas)
> +{
> + /* Step 4: select the maximum-weight disjoint set. */
> + unsigned n = feas.length ();
> + /* dp[i+1] is the best weight over feas[0..i]. Whether i is taken is
> + recomputed while backtracking rather than stored. */
> + gcov_type *dp = XCNEWVEC (gcov_type, n + 1);
> +
> + for (unsigned i = 0; i < n; i++)
> + {
> + /* Predecessor: the last candidate that ends before feas[i] starts.
> + hi is inclusive, so find the first candidate with hi >= lo. */
> + bundle_cand key = {};
> + key.hi = feas[i].lo;
> + int p = (int) feas.lower_bound (key, bundle_cand_hi_less) - 1;
> + feas[i].p = p;
> + gcov_type take = feas[i].w + dp[p + 1], skip = dp[i];
> + dp[i + 1] = take >= skip ? take : skip;
> + if (dump_file)
> + fprintf (dump_file,
> + "bundle-align: cand %u loop %d [bb%d..bb%d] "
> + "lo=%ld hi=%ld w=%ld p=%d take=%ld skip=%ld => %s\n",
> + i, feas[i].loop->num, feas[i].lock->index,
> + feas[i].unlock->index, (long) feas[i].lo,
> + (long) feas[i].hi, (long) feas[i].w, p,
> + (long) take, (long) skip, take >= skip ? "take" : "skip");
> + }
> +
> + /* Step 5: emit the selected regions. */
> + for (int i = (int) n - 1; i >= 0;)
> + if (feas[i].w + dp[feas[i].p + 1] >= dp[i])
> + {
> + basic_block h = feas[i].lock, l = feas[i].unlock;
> + emit_insn_before (gen_bundle_lock (), BB_HEAD (h));
> + emit_insn_after (gen_bundle_unlock (), BB_END (l));
> + if (dump_file)
> + fprintf (dump_file,
> + "bundle-align: wrap loop %d [bb%d..bb%d] "
> + "w=%ld span=%ld\n", feas[i].loop->num, h->index,
> + l->index, (long) feas[i].w,
> + (long) (feas[i].hi - feas[i].lo));
> + i = feas[i].p;
> + }
> + else
> + i--;
> +
> + XDELETEVEC (dp);
> +}
> +
> +/* Wrap the selected disjoint hot loop regions in
> .bundle_lock/.bundle_unlock.
> + Each region is an interval on the layout byte axis, which handles rotated
> + (latch-before-header) loops uniformly. */
> +
> +static void
> +ix86_do_bundle_align_loops ()
> +{
> + if (ix86_cost->prefetch_block == 0)
> + return;
> +
> + /* Region size cap: tuning param if set, else the CPU bundle size. */
> + HOST_WIDE_INT k = (param_x86_tight_loop_max_size > 0
> + ? (HOST_WIDE_INT)param_x86_tight_loop_max_size
> + : (HOST_WIDE_INT)ix86_cost->prefetch_block);
> +
> + loop_optimizer_init (AVOID_CFG_MODIFICATIONS);
> +
> + /* Per-block layout byte offsets [bb_lo, bb_hi). */
> + int nbb = last_basic_block_for_fn (cfun);
> + HOST_WIDE_INT *bb_lo = XCNEWVEC (HOST_WIDE_INT, nbb);
> + HOST_WIDE_INT *bb_hi = XCNEWVEC (HOST_WIDE_INT, nbb);
> + /* Running count of inline-asm insns in layout order; [asm_lo, asm_hi)
> lets a
> + candidate test whether any block it spans contains an asm. */
> + int *asm_lo = XCNEWVEC (int, nbb);
> + int *asm_hi = XCNEWVEC (int, nbb);
> + bundle_compute_layout (bb_lo, bb_hi, asm_lo, asm_hi);
> +
> + auto_vec<bundle_cand> feas;
> + bundle_collect_candidates (feas, k, bb_lo, bb_hi, asm_lo, asm_hi);
> + bundle_add_unions (feas, k);
> + bundle_compute_weights (feas);
> + if (!feas.is_empty ())
> + bundle_select_and_emit (feas);
> +
> + XDELETEVEC (bb_lo);
> + XDELETEVEC (bb_hi);
> + XDELETEVEC (asm_lo);
> + XDELETEVEC (asm_hi);
> + loop_optimizer_finalize ();
> +}
> +
> namespace {
>
> -const pass_data pass_data_align_tight_loops =
> +const pass_data pass_data_bundle_align_loops =
> {
> RTL_PASS, /* type */
> - "align_tight_loops", /* name */
> + "bundle_align_loops", /* name */
> OPTGROUP_NONE, /* optinfo_flags */
> TV_MACH_DEP, /* tv_id */
> 0, /* properties_required */
> @@ -5661,40 +5964,44 @@ const pass_data pass_data_align_tight_loops =
> 0, /* todo_flags_finish */
> };
>
> -class pass_align_tight_loops : public rtl_opt_pass
> +class pass_bundle_align_loops : public rtl_opt_pass
> {
> public:
> - pass_align_tight_loops (gcc::context *ctxt)
> - : rtl_opt_pass (pass_data_align_tight_loops, ctxt)
> + pass_bundle_align_loops (gcc::context *ctxt)
> + : rtl_opt_pass (pass_data_bundle_align_loops, ctxt)
> {}
>
> /* opt_pass methods: */
> bool gate (function *) final override
> {
> - return TARGET_ALIGN_TIGHT_LOOPS
> - && optimize
> - && optimize_function_for_speed_p (cfun);
> + return optimize
> + && optimize_function_for_speed_p (cfun)
> + && (ix86_bundle_align_loops > 0 || TARGET_ALIGN_TIGHT_LOOPS);
> }
>
> unsigned int execute (function *) final override
> {
> timevar_push (TV_MACH_DEP);
> + if (ix86_bundle_align_loops > 0)
> + ix86_do_bundle_align_loops ();
> #ifdef ASM_OUTPUT_MAX_SKIP_ALIGN
> - ix86_align_loops ();
> + else
> + ix86_align_loops ();
> #endif
> timevar_pop (TV_MACH_DEP);
> return 0;
> }
> -}; // class pass_align_tight_loops
> +}; // class pass_bundle_align_loops
>
> } // anon namespace
>
> rtl_opt_pass *
> -make_pass_align_tight_loops (gcc::context *ctxt)
> +make_pass_bundle_align_loops (gcc::context *ctxt)
> {
> - return new pass_align_tight_loops (ctxt);
> + return new pass_bundle_align_loops (ctxt);
> }
>
> +
> /* This compares the priority of target features in function DECL1
> and DECL2. It returns positive value if DECL1 is higher priority,
> negative value if DECL2 is higher priority and 0 if they are the
> diff --git a/gcc/config/i386/i386-options.cc b/gcc/config/i386/i386-options.cc
> index e7f06154103..db0a9426b29 100644
> --- a/gcc/config/i386/i386-options.cc
> +++ b/gcc/config/i386/i386-options.cc
> @@ -1872,8 +1872,16 @@ set_ix86_tune_features (struct gcc_options *opts,
> /* Default align_* from the processor table. */
>
> static void
> -ix86_default_align (struct gcc_options *opts)
> +ix86_default_align (struct gcc_options *opts,
> + struct gcc_options *opts_set)
> {
> + /* Bundle alignment replaces generic loop, jump, and label padding. */
> + if (opts->x_ix86_bundle_align_loops > 0)
> + {
> + SET_OPTION_IF_UNSET (opts, opts_set, str_align_loops, "1");
> + SET_OPTION_IF_UNSET (opts, opts_set, str_align_jumps, "1");
> + SET_OPTION_IF_UNSET (opts, opts_set, str_align_labels, "1");
If we just set loop/jump/label align to 1, and keep align tight loops, can we
get similar
code size reduction without performance drop?
It sounds to me bundle alignments is an enhancement to current tightloop
alignment with
dynamic decision in assembler instead of hard-coded .palign. so what's the real
gain of
the new pass against the original align_tight_loop?
The pass manually setting all other alignments to 1, if the assembler doesn't
support
the new directive .bundle_{lock, unlock}, do we have a way to fallback? or
just lose
all alignment with some performance loss?
> + }
> /* -falign-foo without argument: supply one. */
> if (opts->x_flag_align_loops && !opts->x_str_align_loops)
> opts->x_str_align_loops = processor_cost_table[ix86_tune]->align_loop;
> @@ -1885,6 +1893,16 @@ ix86_default_align (struct gcc_options *opts)
> opts->x_str_align_functions =
> processor_cost_table[ix86_tune]->align_func;
> }
>
> +/* Default -mbundle-align-loops from the active tune model. */
> +
> +static void
> +ix86_default_bundle_align (struct gcc_options *opts)
> +{
> + if (opts->x_ix86_bundle_align_loops == -1)
> + opts->x_ix86_bundle_align_loops
> + = ix86_tune_features[X86_TUNE_BUNDLE_ALIGN_LOOPS] ? 1 : 0;
> +}
> +
> #ifndef USE_IX86_FRAME_POINTER
> #define USE_IX86_FRAME_POINTER 0
> #endif
> @@ -1986,7 +2004,8 @@ ix86_override_options_after_change_1 (struct
> gcc_options *opts,
> void
> ix86_override_options_after_change (void)
> {
> - ix86_default_align (&global_options);
> + ix86_default_bundle_align (&global_options);
> + ix86_default_align (&global_options, &global_options_set);
>
> ix86_recompute_optlev_based_flags (&global_options, &global_options_set);
>
> @@ -2619,7 +2638,8 @@ ix86_option_override_internal (bool main_args_p,
> opts->x_ix86_regparm = REGPARM_MAX;
>
> /* Default align_* from the processor table. */
> - ix86_default_align (&global_options);
> + ix86_default_bundle_align (&global_options);
> + ix86_default_align (&global_options, &global_options_set);
>
> /* Provide default for -mbranch-cost= value. */
> SET_OPTION_IF_UNSET (opts, opts_set, ix86_branch_cost,
> diff --git a/gcc/config/i386/i386-passes.def b/gcc/config/i386/i386-passes.def
> index 4f461116cac..82d4d7411df 100644
> --- a/gcc/config/i386/i386-passes.def
> +++ b/gcc/config/i386/i386-passes.def
> @@ -31,9 +31,9 @@ along with GCC; see the file COPYING3. If not see
> INSERT_PASS_BEFORE (pass_cse2, 1, pass_stv, true /* timode_p */);
>
> INSERT_PASS_BEFORE (pass_shorten_branches, 1,
> pass_insert_endbr_and_patchable_area);
> - /* pass_align_tight_loops must be after
> pass_insert_endbr_and_patchable_area.
> - PR116174. */
> - INSERT_PASS_BEFORE (pass_shorten_branches, 1, pass_align_tight_loops);
> + /* Tight/bundle loop aligner; after pass_insert_endbr_and_patchable_area
> + (PR116174) and before pass_shorten_branches. */
> + INSERT_PASS_BEFORE (pass_shorten_branches, 1, pass_bundle_align_loops);
>
> INSERT_PASS_AFTER (pass_late_combine, 1, pass_x86_cse);
> INSERT_PASS_AFTER (pass_late_combine, 1,
> pass_remove_partial_avx_dependency);
> diff --git a/gcc/config/i386/i386-protos.h b/gcc/config/i386/i386-protos.h
> index 2687de646b0..84d2a7ad74c 100644
> --- a/gcc/config/i386/i386-protos.h
> +++ b/gcc/config/i386/i386-protos.h
> @@ -434,7 +434,7 @@ extern rtl_opt_pass
> *make_pass_remove_partial_avx_dependency
> (gcc::context *);
> extern rtl_opt_pass *make_pass_x86_cse (gcc::context *);
> extern rtl_opt_pass *make_pass_apx_nf_convert (gcc::context *);
> -extern rtl_opt_pass *make_pass_align_tight_loops (gcc::context *);
> +extern rtl_opt_pass *make_pass_bundle_align_loops (gcc::context *);
>
> extern bool ix86_has_no_direct_extern_access;
> extern bool ix86_rpad_gate ();
> diff --git a/gcc/config/i386/i386.md b/gcc/config/i386/i386.md
> index 5768a93d5b0..f85010a41aa 100644
> --- a/gcc/config/i386/i386.md
> +++ b/gcc/config/i386/i386.md
> @@ -233,6 +233,8 @@
> UNSPECV_STACK_PROBE
> UNSPECV_PROBE_STACK_RANGE
> UNSPECV_ALIGN
> + UNSPECV_BUNDLE_LOCK
> + UNSPECV_BUNDLE_UNLOCK
> UNSPECV_PROLOGUE_USE
> UNSPECV_SPLIT_STACK_RETURN
> UNSPECV_CLD
> @@ -21756,6 +21758,39 @@
> }
> [(set_attr "length" "16")])
>
> +;; ix86_bundle_align_loops pass: align the locked region's start to the cache
> +;; line (prefetch_block). Final instruction sizes may grow during assembly
> +;; because of indirect calls, prefixes, and relaxation, so a selected region
> +;; can exceed one bundle; flex .bundle_lock accepts such a region instead of
> +;; raising an error. Requires flex .bundle_lock support.
> +(define_insn "bundle_lock"
> + [(unspec_volatile [(const_int 0)] UNSPECV_BUNDLE_LOCK)]
> + ""
> +{
> +#ifdef HAVE_GAS_BUNDLE_FLEX_LOCK
> + fprintf (asm_out_file, "\t.bundle_align_mode %d\n\t.bundle_lock flex\n",
> + ceil_log2 (ix86_cost->prefetch_block));
> +#endif
> + return "";
> +}
> + [(set_attr "length" "0")
> + (set_attr "length_immediate" "0")
> + (set_attr "modrm" "0")])
> +
> +;; Close the bundle and turn bundle alignment mode back off.
> +(define_insn "bundle_unlock"
> + [(unspec_volatile [(const_int 0)] UNSPECV_BUNDLE_UNLOCK)]
> + ""
> +{
> +#ifdef HAVE_GAS_BUNDLE_FLEX_LOCK
> + fputs ("\t.bundle_unlock\n\t.bundle_align_mode 0\n", asm_out_file);
> +#endif
> + return "";
> +}
> + [(set_attr "length" "0")
> + (set_attr "length_immediate" "0")
> + (set_attr "modrm" "0")])
> +
> (define_expand "prologue"
> [(const_int 0)]
> ""
> diff --git a/gcc/config/i386/i386.opt b/gcc/config/i386/i386.opt
> index 57ba3fbfa8f..148d26b6053 100644
> --- a/gcc/config/i386/i386.opt
> +++ b/gcc/config/i386/i386.opt
> @@ -240,6 +240,10 @@ malign-loops=
> Target Undocumented RejectNegative Joined UInteger
> Loop code aligned to this power of 2.
>
> +mbundle-align-loops
> +Target Var(ix86_bundle_align_loops) Init(-1) Save
> +Wrap small hot loops with .bundle_lock so the assembler keeps them inside
> one cache line.
> +
> malign-stringops
> Target RejectNegative InverseMask(NO_ALIGN_STRINGOPS, ALIGN_STRINGOPS) Save
> Align destination of the string operations.
> @@ -1221,6 +1225,14 @@ Do not use GOT to access external symbols.
> Target Joined UInteger Var(x86_stlf_window_ninsns) Init(64) Param
> Instructions number above which STFL stall penalty can be compensated.
>
> +-param=x86-tight-loop-max-size=
> +Target Joined UInteger Var(param_x86_tight_loop_max_size) Init(0) Param
> +Maximum body size in bytes for the i386 tight-loop aligner; 0 picks the
> active tune prefetch_block.
> +
> +-param=x86-bundle-align-branch-limit=
> +Target Joined UInteger Var(param_x86_bundle_align_branch_limit) Init(2) Param
> +Maximum internal conditional branches allowed in a bundle-locked loop region
> (the back-edge branch is not counted).
> +
> mavxifma
> Target Mask(ISA2_AVXIFMA) Var(ix86_isa_flags2) Save
> Support MMX, SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2, AVX, AVX2, and
> diff --git a/gcc/config/i386/i386.opt.urls b/gcc/config/i386/i386.opt.urls
> index d52e6de8014..20a638ea1f7 100644
> --- a/gcc/config/i386/i386.opt.urls
> +++ b/gcc/config/i386/i386.opt.urls
> @@ -28,6 +28,9 @@ UrlSuffix(gcc/x86-Options.html#index-malign-double)
>
> ; skipping UrlSuffix for 'malign-loops=' due to finding no URLs
>
> +mbundle-align-loops
> +UrlSuffix(gcc/x86-Options.html#index-mbundle-align-loops)
> +
> malign-stringops
> UrlSuffix(gcc/x86-Options.html#index-malign-stringops)
>
> diff --git a/gcc/config/i386/x86-tune.def b/gcc/config/i386/x86-tune.def
> index d4c27351ad7..4716f26b2e1 100644
> --- a/gcc/config/i386/x86-tune.def
> +++ b/gcc/config/i386/x86-tune.def
> @@ -259,6 +259,13 @@ DEF_TUNE (X86_TUNE_ALIGN_TIGHT_LOOPS,
> "align_tight_loops",
> ~(m_ZHAOXIN | m_CASCADELAKE | m_SKYLAKE_AVX512
> | m_CORE_ATOM))
>
> +/* X86_TUNE_BUNDLE_ALIGN_LOOPS: when true, use the ix86_bundle_align_loops
> + pass with the assembler's .bundle_lock/.bundle_unlock directives instead
> + of static .p2align padding. Targets cores where loops straddling a
> 64-byte
> + instruction-cache line are particularly costly (AMD Zen4/Zen5/Zen6). */
> +DEF_TUNE (X86_TUNE_BUNDLE_ALIGN_LOOPS, "bundle_align_loops",
> + m_ZNVER4 | m_ZNVER5 | m_ZNVER6)
> +
>
> /*****************************************************************************/
> /* Integer instruction selection tuning
> */
>
> /*****************************************************************************/
> diff --git a/gcc/configure b/gcc/configure
> index 43ca45144b8..dd5a0c0a539 100755
> --- a/gcc/configure
> +++ b/gcc/configure
> @@ -26306,6 +26306,44 @@ $as_echo "#define HAVE_GAS_MAX_SKIP_P2ALIGN 1"
> >>confdefs.h
> fi
>
>
> +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking assembler for .bundle_lock
> flex argument" >&5
> +$as_echo_n "checking assembler for .bundle_lock flex argument... " >&6; }
> +if ${gcc_cv_as_bundle_flex_lock+:} false; then :
> + $as_echo_n "(cached) " >&6
> +else
> + gcc_cv_as_bundle_flex_lock=no
> + if test x"$gcc_cv_as" != x; then
> + $as_echo '.bundle_align_mode 6
> +.bundle_lock flex
> +.rept 100
> +nop
> +.endr
> +.bundle_unlock' > conftest.s
> + if { ac_try='$gcc_cv_as $gcc_cv_as_flags -o conftest.o conftest.s >&5'
> + { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_try\""; } >&5
> + (eval $ac_try) 2>&5
> + ac_status=$?
> + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
> + test $ac_status = 0; }; }
> + then
> + gcc_cv_as_bundle_flex_lock=yes
> + else
> + echo "configure: failed program was" >&5
> + cat conftest.s >&5
> + fi
> + rm -f conftest.o conftest.s
> + fi
> +fi
> +{ $as_echo "$as_me:${as_lineno-$LINENO}: result:
> $gcc_cv_as_bundle_flex_lock" >&5
> +$as_echo "$gcc_cv_as_bundle_flex_lock" >&6; }
> +if test $gcc_cv_as_bundle_flex_lock = yes; then
> +
> +$as_echo "#define HAVE_GAS_BUNDLE_FLEX_LOCK 1" >>confdefs.h
> +
> +fi
> +
> +
> +
> { $as_echo "$as_me:${as_lineno-$LINENO}: checking assembler for .literal16"
> >&5
> $as_echo_n "checking assembler for .literal16... " >&6; }
> if ${gcc_cv_as_literal16+:} false; then :
> diff --git a/gcc/configure.ac b/gcc/configure.ac
> index 075b6b68ffa..346bf254de3 100644
> --- a/gcc/configure.ac
> +++ b/gcc/configure.ac
> @@ -3142,6 +3142,17 @@ gcc_GAS_CHECK_FEATURE([.p2align with maximum skip],
> gcc_cv_as_max_skip_p2align,,
> [Define if your assembler supports specifying the maximum number
> of bytes to skip when using the GAS .p2align command.])])
>
> +gcc_GAS_CHECK_FEATURE([.bundle_lock flex argument],
> gcc_cv_as_bundle_flex_lock,,
> + [.bundle_align_mode 6
> +.bundle_lock flex
> +.rept 100
> +nop
> +.endr
> +.bundle_unlock],,
> +[AC_DEFINE(HAVE_GAS_BUNDLE_FLEX_LOCK, 1,
> + [Define if your assembler accepts the flex argument to .bundle_lock
> + that permits bundles larger than .bundle_align_mode alignment.])])
> +
> gcc_GAS_CHECK_FEATURE([.literal16], gcc_cv_as_literal16,,
> [.literal16],,
> [AC_DEFINE(HAVE_GAS_LITERAL16, 1,
> diff --git a/gcc/doc/extend.texi b/gcc/doc/extend.texi
> index 397b05ab87d..9081317927d 100644
> --- a/gcc/doc/extend.texi
> +++ b/gcc/doc/extend.texi
> @@ -9598,6 +9598,12 @@ operations and calling the library routines for large
> operations.
> @itemx no-align-stringops
> Do/do not align destination of inlined string operations.
>
> +@atindex @code{target("bundle-align-loops")}, x86
> +@item bundle-align-loops
> +@itemx no-bundle-align-loops
> +Enable/disable wrapping of small hot loops in assembler
> +@code{.bundle_lock}/@code{.bundle_unlock} directives.
> +
> @atindex @code{target("recip")}, x86
> @item recip
> @itemx no-recip
> diff --git a/gcc/doc/invoke.texi b/gcc/doc/invoke.texi
> index e1870c9c227..619f92185fd 100644
> --- a/gcc/doc/invoke.texi
> +++ b/gcc/doc/invoke.texi
> @@ -1554,7 +1554,8 @@ See RS/6000 and PowerPC Options.
> -mamx-complex -mavxvnniint16 -msm3 -msha512 -msm4 -mapxf
> -musermsr -mavx10.1 -mavx10.2 -mamx-avx512 -mmovrs -mamx-movrs
> -mavx512bmm -mavx10v2aux
> --mcldemote -mms-bitfields -mno-align-stringops -minline-all-stringops
> +-mcldemote -mms-bitfields -mbundle-align-loops -mno-align-stringops
> +-minline-all-stringops
> -minline-stringops-dynamically -mstringop-strategy=@var{alg}
> -mkl -mwidekl
> -mmemcpy-strategy=@var{strategy} -mmemset-strategy=@var{strategy}
> @@ -37155,6 +37156,18 @@ Here, @code{t5} takes up 2 bytes.
> @end enumerate
>
>
> +@opindex mbundle-align-loops
> +@opindex mno-bundle-align-loops
> +@item -mbundle-align-loops
> +@itemx -mno-bundle-align-loops
> +Wrap small hot loops in assembler @code{.bundle_lock}/@code{.bundle_unlock}
> +directives so the assembler keeps each loop within a single instruction-cache
> +line, aligning the loop header without inserting executed padding. Enabled
> by
> +default for AMD Zen4 and later. It defaults @option{-falign-loops},
> +@option{-falign-jumps}, and @option{-falign-labels} to 1 unless explicitly
> set.
> +The maximum loop-body size considered is set by the
> +@option{x86-tight-loop-max-size} parameter.
> +
> @opindex mno-align-stringops
> @opindex malign-stringops
> @item -mno-align-stringops
> diff --git a/gcc/doc/sourcebuild.texi b/gcc/doc/sourcebuild.texi
> index aad51550eba..35324653699 100644
> --- a/gcc/doc/sourcebuild.texi
> +++ b/gcc/doc/sourcebuild.texi
> @@ -3114,6 +3114,10 @@ Target supports @option{-fstack-protector}.
> @item gas
> Target uses GNU @command{as}.
>
> +@item gcc_bundle_align_loops
> +Target compiler and assembler support @option{-mbundle-align-loops} and
> +@code{.bundle_lock flex}.
> +
> @item gc_sections
> Target supports @option{--gc-sections}.
>
> diff --git a/gcc/testsuite/gcc.target/i386/bundle-align-1.c
> b/gcc/testsuite/gcc.target/i386/bundle-align-1.c
> new file mode 100644
> index 00000000000..c7120731f5f
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/i386/bundle-align-1.c
> @@ -0,0 +1,16 @@
> +/* { dg-do compile { target { ! ia32 } } } */
> +/* { dg-options "-O3 -march=znver3 -falign-loops=1 -falign-jumps=1
> -falign-labels=1 -mbundle-align-loops" } */
> +/* { dg-require-effective-target gcc_bundle_align_loops } */
> +/* { dg-final { scan-assembler "\\.bundle_lock flex" } } */
> +/* { dg-final { scan-assembler "\\.bundle_unlock" } } */
> +
> +typedef int (*callee_t) (int, int *);
> +
> +int
> +indirect_sum (callee_t callee, int n, int *array, long calls)
> +{
> + int sink = 0;
> + for (long call = 0; call < calls; ++call)
> + sink += callee (n, array);
> + return sink;
> +}
> diff --git a/gcc/testsuite/gcc.target/i386/bundle-align-2.c
> b/gcc/testsuite/gcc.target/i386/bundle-align-2.c
> new file mode 100644
> index 00000000000..957e853dfb2
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/i386/bundle-align-2.c
> @@ -0,0 +1,21 @@
> +/* { dg-do compile { target { ! ia32 } } } */
> +/* { dg-options "-O3 -march=znver3 -falign-loops=1 -falign-jumps=1
> -falign-labels=1 -mbundle-align-loops" } */
> +/* { dg-require-effective-target gcc_bundle_align_loops } */
> +/* { dg-final { scan-assembler "\\.bundle_lock flex" } } */
> +/* { dg-final { scan-assembler "\\.bundle_unlock" } } */
> +
> +__attribute__((noinline,noipa))
> +int
> +f (int n, int *p)
> +{
> + int i = 0, s = 0;
> + if (n & 1)
> + goto mid;
> + while (i < n)
> + {
> + s += p[i & 1023];
> + mid:
> + i++;
> + }
> + return s;
> +}
> diff --git a/gcc/testsuite/gcc.target/i386/bundle-align-3.c
> b/gcc/testsuite/gcc.target/i386/bundle-align-3.c
> new file mode 100644
> index 00000000000..7cdfa8db985
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/i386/bundle-align-3.c
> @@ -0,0 +1,26 @@
> +/* { dg-do compile { target { ! ia32 } } } */
> +/* { dg-options "-O3 -march=znver3 -falign-loops=1 -falign-jumps=1
> -falign-labels=1 -mbundle-align-loops" } */
> +/* { dg-require-effective-target gcc_bundle_align_loops } */
> +/* { dg-final { scan-assembler "\\.bundle_lock flex" } } */
> +/* { dg-final { scan-assembler "\\.bundle_unlock" } } */
> +
> +int a[1024];
> +__attribute__((hot,noinline,noipa))
> +int
> +two_backedges (int n, int m)
> +{
> + int i = 0, s = 0;
> +outer_head:
> + {
> + int j = 0;
> + inner_head:
> + s += a[(i + j) & 1023];
> + if (__builtin_expect (s < 0, 0))
> + goto outer_head;
> + if (++j < m)
> + goto inner_head;
> + }
> + if (++i < n)
> + goto outer_head;
> + return s;
> +}
> diff --git a/gcc/testsuite/gcc.target/i386/bundle-align-4.c
> b/gcc/testsuite/gcc.target/i386/bundle-align-4.c
> new file mode 100644
> index 00000000000..80bca4ec200
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/i386/bundle-align-4.c
> @@ -0,0 +1,22 @@
> +/* { dg-do compile { target { ! ia32 } } } */
> +/* { dg-options "-O3 -march=znver5 -falign-loops=1 -falign-jumps=1
> -falign-labels=1 -mbundle-align-loops" } */
> +/* { dg-require-effective-target gcc_bundle_align_loops } */
> +/* { dg-final { scan-assembler-times "\\.bundle_lock flex" 1 } } */
> +/* { dg-final { scan-assembler-times "\\.bundle_unlock" 1 } } */
> +
> +volatile int sink;
> +int a[1024];
> +int
> +classify (int n)
> +{
> + int s = 0;
> + asm volatile (".rept 0\n\t.byte 0x90\n\t.endr" ::: "memory");
> + for (int i = 0; i < n; i++)
> + {
> + if (a[i] & 1) sink = i;
> + if (a[i] & 2) sink = i + 1;
> + if (a[i] & 4) s += 4;
> + s += a[i];
> + }
> + return s;
> +}
> diff --git a/gcc/testsuite/gcc.target/i386/bundle-align-5.c
> b/gcc/testsuite/gcc.target/i386/bundle-align-5.c
> new file mode 100644
> index 00000000000..7aa07effa6e
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/i386/bundle-align-5.c
> @@ -0,0 +1,17 @@
> +/* { dg-do compile { target { ! ia32 } } } */
> +/* { dg-options "-O3 -march=znver3 -falign-loops=1 -falign-jumps=1
> -falign-labels=1 -mbundle-align-loops" } */
> +/* { dg-require-effective-target gcc_bundle_align_loops } */
> +/* { dg-final { scan-assembler "\\.bundle_lock flex" } } */
> +/* { dg-final { scan-assembler "\\.bundle_unlock" } } */
> +
> +void *mc (const void *s, int c, unsigned long n)
> +{
> + const unsigned char *p = s;
> + while (n--)
> + {
> + if (*p == (unsigned char) c)
> + return (void *) p;
> + p++;
> + }
> + return 0;
> +}
> diff --git a/gcc/testsuite/gcc.target/i386/bundle-align-6.c
> b/gcc/testsuite/gcc.target/i386/bundle-align-6.c
> new file mode 100644
> index 00000000000..1f561e07caa
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/i386/bundle-align-6.c
> @@ -0,0 +1,24 @@
> +/* { dg-do compile { target { ! ia32 } } } */
> +/* { dg-options "-O3 -march=znver3 -falign-loops=1 -falign-jumps=1
> -falign-labels=1 -mbundle-align-loops" } */
> +/* { dg-require-effective-target gcc_bundle_align_loops } */
> +/* { dg-final { scan-assembler-times "\\.bundle_lock flex" 1 } } */
> +/* { dg-final { scan-assembler-times "\\.bundle_unlock" 1 } } */
> +
> +struct O { long rc; };
> +extern void dealloc (struct O *);
> +void
> +f (struct O **v, long n)
> +{
> + asm volatile (".rept 10\n\t.byte 0x90\n\t.endr" ::: "memory");
> + while (--n >= 0)
> + {
> + struct O *x = v[n];
> + if (x)
> + {
> + if (x->rc < 0)
> + ;
> + else if (--x->rc == 0)
> + dealloc (x);
> + }
> + }
> +}
> diff --git a/gcc/testsuite/gcc.target/i386/bundle-align-7.c
> b/gcc/testsuite/gcc.target/i386/bundle-align-7.c
> new file mode 100644
> index 00000000000..b713d25d0b7
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/i386/bundle-align-7.c
> @@ -0,0 +1,17 @@
> +/* { dg-do compile { target { ! ia32 } } } */
> +/* { dg-options "-O1 -march=znver3 -falign-loops=1 -falign-jumps=1
> -falign-labels=1 -mbundle-align-loops" } */
> +/* { dg-require-effective-target gcc_bundle_align_loops } */
> +/* { dg-final { scan-assembler "\\.bundle_lock flex" } } */
> +/* { dg-final { scan-assembler "\\.bundle_unlock" } } */
> +
> +static int A[1 << 20];
> +
> +int
> +nested_sum (int n, int m)
> +{
> + int sum = 0;
> + for (int i = 0; i < n; ++i)
> + for (int j = 0; j < m; ++j)
> + sum += A[i * m + j];
> + return sum;
> +}
> diff --git a/gcc/testsuite/lib/target-supports.exp
> b/gcc/testsuite/lib/target-supports.exp
> index d467e9fa3b7..3286c40b07d 100644
> --- a/gcc/testsuite/lib/target-supports.exp
> +++ b/gcc/testsuite/lib/target-supports.exp
> @@ -12491,6 +12491,22 @@ proc check_effective_target_gas { } {
> return $use_gas_saved
> }
>
> +# Return 1 if the compiler was configured with assembler support for
> +# the flex .bundle_lock argument used by -mbundle-align-loops.
> +
> +proc check_effective_target_gcc_bundle_align_loops { } {
> + return [check_no_messages_and_pattern gcc_bundle_align_loops \
> + "\\.bundle_lock flex" assembly {
> + int a[1024];
> + int foo (int n) {
> + int s = 0;
> + for (int i = 0; i < n; i++)
> + s += a[i];
> + return s;
> + }
> + } "-O2 -march=znver4 -mbundle-align-loops -falign-loops=1"]
> +}
> +
> # Return 1 if GNU ld is used.
>
> proc check_effective_target_gld { } {
> --
> 2.34.1
