Targets that cannot guarantee arbitrary stack alignment cap stack slots at
MAX_SUPPORTED_STACK_ALIGNMENT. Win64 SEH caps MAX_STACK_ALIGNMENT at 128
bits; aarch64 does not define MAX_STACK_ALIGNMENT and is capped at
STACK_BOUNDARY, also 128 bits. assign_stack_local silently ignored any
alignment beyond that cap.
A 256-bit vector requires 32-byte alignment, but its slot could therefore
receive only 16-byte alignment while later MEM attributes still claimed 32.
x86 could select vmovaps from that MEM_ALIGN and fault on the under-aligned
slot (PR54412, MSYS2/MINGW-packages#1209). An escaped pointer has the same
problem even when the local move is unaligned, since its type still promises
the callee a 32-byte-aligned object.
Make assign_stack_local_1 honour an explicit alignment beyond the cap by
reserving enough space and aligning an address inside it at run time, as
assign_parm_setup_block already did for BLKmode parameters and
expand_stack_vars does for locals. Base the extra space on the alignment
guaranteed for VIRTUAL_STACK_VARS_REGNUM rather than on the maximum alignment
supported by the backend, since the frame itself is not guaranteed to have
the latter alignment on all targets.
Let a new internal assign_stack_local_2 return the address before run-time
alignment. Store it in the temp slot and realign the address on reuse, while
avoiding splitting or combining over-aligned slots.
Honour required temporary alignment at run time, but cap merely preferred
alignment. Preserve zero-size behavior and the target's preferred stack
boundary, and emit run-time alignment only during expansion to RTL.
Use ASLK_REDUCE_ALIGN where less alignment is acceptable: LRA spill slots and
the deliberately over-aligned SJLJ function context.
Regtested on x86_64-pc-linux-gnu and, under QEMU, on
aarch64-unknown-linux-gnu, with no new failures.
gcc/ChangeLog:
PR target/54412
* except.cc (sjlj_build_landing_pads): Pass ASLK_REDUCE_ALIGN.
* function.cc (assign_stack_local_2): New function, split from
assign_stack_local_1. Honour an explicit alignment beyond
MAX_SUPPORTED_STACK_ALIGNMENT by overallocating the slot and aligning
its address at run time. Optionally return the address before run-time
alignment.
(assign_stack_local_1): Wrap assign_stack_local_2.
(temp_slot): Add base_addr.
(assign_stack_temp_for_type): Distinguish required and preferred
alignment. Record the address before run-time alignment and recalculate
the aligned address when reusing an over-aligned slot. Do not split an
over-aligned BLKmode slot.
(combine_temp_slots): Do not combine over-aligned slots.
(assign_parm_setup_block): Let assign_stack_local align the slot.
* lra-spills.cc (assign_mem_slot): Pass ASLK_REDUCE_ALIGN.
gcc/testsuite/ChangeLog:
PR target/54412
* g++.target/i386/pr54412-highpart-extract-rt.C: Adjust the comment to
describe safe reuse of an over-aligned temporary.
Signed-off-by: Oleg Tolmatcev <[email protected]>
---
gcc/except.cc | 12 ++---
gcc/function.cc | 131 +++++++++++++++++++++++++++++++++++-----------
gcc/lra-spills.cc | 5 +-
3 files changed, 108 insertions(+), 40 deletions(-)
diff --git a/gcc/except.cc b/gcc/except.cc
index 98876833ee..e3b8645733 100644
--- a/gcc/except.cc
+++ b/gcc/except.cc
@@ -1478,9 +1478,9 @@ sjlj_build_landing_pads (void)
TYPE_MODE (sjlj_fc_type_node),
TYPE_ALIGN (sjlj_fc_type_node));
crtl->eh.sjlj_fc
- = assign_stack_local (TYPE_MODE (sjlj_fc_type_node),
- int_size_in_bytes (sjlj_fc_type_node),
- align);
+ = assign_stack_local_1 (TYPE_MODE (sjlj_fc_type_node),
+ int_size_in_bytes (sjlj_fc_type_node),
+ align, ASLK_RECORD_PAD | ASLK_REDUCE_ALIGN);
sjlj_mark_call_sites ();
sjlj_emit_function_enter (dispatch_label);
@@ -1496,9 +1496,9 @@ sjlj_build_landing_pads (void)
TYPE_MODE (sjlj_fc_type_node),
TYPE_ALIGN (sjlj_fc_type_node));
crtl->eh.sjlj_fc
- = assign_stack_local (TYPE_MODE (sjlj_fc_type_node),
- int_size_in_bytes (sjlj_fc_type_node),
- align);
+ = assign_stack_local_1 (TYPE_MODE (sjlj_fc_type_node),
+ int_size_in_bytes (sjlj_fc_type_node),
+ align, ASLK_RECORD_PAD | ASLK_REDUCE_ALIGN);
sjlj_mark_call_sites ();
sjlj_emit_function_enter (NULL);
diff --git a/gcc/function.cc b/gcc/function.cc
index 9358787b26..02796e2f7a 100644
--- a/gcc/function.cc
+++ b/gcc/function.cc
@@ -155,6 +155,7 @@ static bool contains (const rtx_insn *,
hash_table<insn_cache_hasher> *);
static void prepare_function_start (void);
static void do_clobber_return_reg (rtx, void *);
static void do_use_return_reg (rtx, void *);
+static rtx assign_stack_local_2 (machine_mode, poly_int64, int, int, rtx *);
/* Stack of nested functions. */
@@ -370,6 +371,10 @@ add_frame_space (poly_int64 start, poly_int64 end)
-2 means use BITS_PER_UNIT,
positive specifies alignment boundary in bits.
+ A positive ALIGN that exceeds MAX_SUPPORTED_STACK_ALIGNMENT is honoured
+ by overallocating the slot and aligning its address at run time, with
+ insns emitted at the current position, unless KIND has ASLK_REDUCE_ALIGN.
+
KIND has ASLK_REDUCE_ALIGN bit set if it is OK to reduce
alignment and ASLK_RECORD_PAD bit set if we should remember
extra space we allocated for alignment purposes. When we are
@@ -381,11 +386,24 @@ add_frame_space (poly_int64 start, poly_int64 end)
rtx
assign_stack_local_1 (machine_mode mode, poly_int64 size,
int align, int kind)
+{
+ return assign_stack_local_2 (mode, size, align, kind, NULL);
+}
+
+/* Like assign_stack_local_1. If BASE_ADDR is nonnull, set it to the slot
+ address before run-time alignment, or to NULL_RTX if none was needed. */
+
+static rtx
+assign_stack_local_2 (machine_mode mode, poly_int64 size,
+ int align, int kind, rtx *base_addr)
{
rtx x, addr;
poly_int64 bigend_correction = 0;
poly_int64 slot_offset = 0, old_frame_offset;
- unsigned int alignment, alignment_in_bits;
+ unsigned int alignment, alignment_in_bits, dynamic_align = 0;
+
+ if (base_addr)
+ *base_addr = NULL_RTX;
if (align == 0)
{
@@ -404,9 +422,23 @@ assign_stack_local_1 (machine_mode mode, poly_int64 size,
alignment_in_bits = alignment * BITS_PER_UNIT;
- /* Ignore alignment if it exceeds MAX_SUPPORTED_STACK_ALIGNMENT. */
+ /* The frame cannot be aligned beyond MAX_SUPPORTED_STACK_ALIGNMENT. If more
+ is requested, pad the slot so that an over-aligned object still fits. */
if (alignment_in_bits > MAX_SUPPORTED_STACK_ALIGNMENT)
{
+ if (align > 0 && !(kind & ASLK_REDUCE_ALIGN) && maybe_ne (size, 0))
+ {
+ gcc_assert (currently_expanding_to_rtl);
+ dynamic_align = alignment_in_bits;
+
+ if (crtl->preferred_stack_boundary < PREFERRED_STACK_BOUNDARY)
+ crtl->preferred_stack_boundary = PREFERRED_STACK_BOUNDARY;
+
+ /* Pad from the alignment the frame base is guaranteed to have. */
+ size += ((alignment_in_bits
+ - REGNO_POINTER_ALIGN (VIRTUAL_STACK_VARS_REGNUM))
+ / BITS_PER_UNIT);
+ }
alignment_in_bits = MAX_SUPPORTED_STACK_ALIGNMENT;
alignment = MAX_SUPPORTED_STACK_ALIGNMENT / BITS_PER_UNIT;
}
@@ -506,7 +538,7 @@ assign_stack_local_1 (machine_mode mode, poly_int64 size,
found_space:
/* On a big-endian machine, if we are allocating more space than we will use,
use the least significant bytes of those that are allocated. */
- if (mode != BLKmode)
+ if (mode != BLKmode && !dynamic_align)
{
/* The slot size can sometimes be smaller than the mode size;
e.g. the rs6000 port allocates slots with a vector mode
@@ -531,6 +563,15 @@ assign_stack_local_1 (machine_mode mode, poly_int64 size,
(slot_offset + bigend_correction,
Pmode));
+ if (dynamic_align)
+ {
+ if (base_addr)
+ *base_addr = addr;
+ addr = align_dynamic_address (addr, dynamic_align);
+ mark_reg_pointer (addr, dynamic_align);
+ alignment_in_bits = dynamic_align;
+ }
+
x = gen_rtx_MEM (mode, addr);
set_mem_align (x, alignment_in_bits);
MEM_NOTRAP_P (x) = 1;
@@ -592,6 +633,9 @@ public:
/* The size of the slot, including extra space for alignment. This
info is for combine_temp_slots. */
poly_int64 full_size;
+ /* The address before run-time alignment, or NULL_RTX if the slot does not
+ need run-time alignment. */
+ rtx base_addr;
};
/* Entry for the below hash table. */
@@ -803,6 +847,18 @@ assign_stack_temp_for_type (machine_mode mode, poly_int64
size, tree type)
align = get_stack_local_alignment (type, mode);
+ unsigned int required_align
+ = (mode == BLKmode ? BITS_PER_UNIT : GET_MODE_ALIGNMENT (mode));
+ if (type)
+ required_align = MAX (required_align, TYPE_ALIGN (type));
+
+ bool dynamic_align = required_align > MAX_SUPPORTED_STACK_ALIGNMENT;
+
+ if (dynamic_align)
+ align = required_align;
+ else
+ align = MAX (required_align, MIN (align, MAX_SUPPORTED_STACK_ALIGNMENT));
+
/* Try to find an available, already-allocated temporary of the proper
mode which meets the size and alignment requirements. Choose the
smallest one with the closest alignment.
@@ -844,7 +900,8 @@ assign_stack_temp_for_type (machine_mode mode, poly_int64
size, tree type)
/* If there are enough aligned bytes left over, make them into a new
temp_slot so that the extra bytes don't get wasted. Do this only
for BLKmode slots, so that we can be sure of the alignment. */
- if (GET_MODE (best_p->slot) == BLKmode)
+ if (GET_MODE (best_p->slot) == BLKmode
+ && best_p->align <= MAX_SUPPORTED_STACK_ALIGNMENT)
{
int alignment = best_p->align / BITS_PER_UNIT;
poly_int64 rounded_size = aligned_upper_bound (size, alignment);
@@ -852,6 +909,7 @@ assign_stack_temp_for_type (machine_mode mode, poly_int64
size, tree type)
if (known_ge (best_p->size - rounded_size, alignment))
{
p = ggc_alloc<temp_slot> ();
+ p->base_addr = NULL_RTX;
p->in_use = false;
p->size = best_p->size - rounded_size;
p->base_offset = best_p->base_offset + rounded_size;
@@ -869,12 +927,20 @@ assign_stack_temp_for_type (machine_mode mode, poly_int64
size, tree type)
}
}
+ bool reused_p = selected != 0;
+
/* If we still didn't find one, make a new temporary. */
if (selected == 0)
{
poly_int64 frame_offset_old = frame_offset;
p = ggc_alloc<temp_slot> ();
+ p->base_addr = NULL_RTX;
+
+ poly_int64 slot_size
+ = (mode == BLKmode
+ ? aligned_upper_bound (size, (int) align / BITS_PER_UNIT)
+ : size);
/* We are passing an explicit alignment request to assign_stack_local.
One side effect of that is assign_stack_local will not round SIZE
@@ -883,14 +949,15 @@ assign_stack_temp_for_type (machine_mode mode, poly_int64
size, tree type)
So for requests which depended on the rounding of SIZE, we go ahead
and round it now. We also make sure ALIGNMENT is at least
BIGGEST_ALIGNMENT. */
- gcc_assert (mode != BLKmode || align == BIGGEST_ALIGNMENT);
- p->slot = assign_stack_local_1 (mode,
- (mode == BLKmode
- ? aligned_upper_bound (size,
- (int) align
- / BITS_PER_UNIT)
- : size),
- align, 0);
+ gcc_assert (mode != BLKmode
+ || dynamic_align
+ || (align
+ >= MIN (BIGGEST_ALIGNMENT,
+ MAX_SUPPORTED_STACK_ALIGNMENT)));
+
+ rtx base_addr;
+ p->slot = assign_stack_local_2 (mode, slot_size, align, 0, &base_addr);
+ p->base_addr = base_addr;
p->align = align;
@@ -901,7 +968,9 @@ assign_stack_temp_for_type (machine_mode mode, poly_int64
size, tree type)
can be either above or below this stack slot depending on which
way the frame grows. We include the extra space if and only if it
is above this slot. */
- if (FRAME_GROWS_DOWNWARD)
+ if (dynamic_align)
+ p->size = slot_size;
+ else if (FRAME_GROWS_DOWNWARD)
p->size = frame_offset_old - frame_offset;
else
p->size = size;
@@ -929,10 +998,19 @@ assign_stack_temp_for_type (machine_mode mode, poly_int64
size, tree type)
pp = temp_slots_at_level (p->level);
insert_slot_to_list (p, pp);
- insert_temp_slot_address (XEXP (p->slot, 0), p);
+
+ rtx addr = XEXP (p->slot, 0);
+
+ if (reused_p && p->align > MAX_SUPPORTED_STACK_ALIGNMENT)
+ {
+ addr = align_dynamic_address (p->base_addr, p->align);
+ mark_reg_pointer (addr, p->align);
+ }
+
+ insert_temp_slot_address (addr, p);
/* Create a new MEM rtx to avoid clobbering MEM flags of old slots. */
- slot = gen_rtx_MEM (mode, XEXP (p->slot, 0));
+ slot = gen_rtx_MEM (mode, addr);
vec_safe_push (stack_slot_list, slot);
/* If we know the alias set for the memory that will be used, use
@@ -1061,7 +1139,8 @@ combine_temp_slots (void)
next = p->next;
- if (GET_MODE (p->slot) != BLKmode)
+ if (GET_MODE (p->slot) != BLKmode
+ || p->align > MAX_SUPPORTED_STACK_ALIGNMENT)
continue;
for (q = p->next; q; q = next_q)
@@ -1070,7 +1149,8 @@ combine_temp_slots (void)
next_q = q->next;
- if (GET_MODE (q->slot) != BLKmode)
+ if (GET_MODE (q->slot) != BLKmode
+ || q->align > MAX_SUPPORTED_STACK_ALIGNMENT)
continue;
if (known_eq (p->base_offset + p->full_size, q->base_offset))
@@ -2948,21 +3028,8 @@ assign_parm_setup_block (struct assign_parm_data_all
*all,
? MAX (DECL_ALIGN (parm), BITS_PER_WORD) : DECL_ALIGN (parm));
SET_DECL_ALIGN (parm, parm_align);
- if (DECL_ALIGN (parm) > MAX_SUPPORTED_STACK_ALIGNMENT)
- {
- rtx allocsize = gen_int_mode (size_stored, Pmode);
- get_dynamic_stack_size (&allocsize, 0, DECL_ALIGN (parm), NULL);
- stack_parm = assign_stack_local (BLKmode, UINTVAL (allocsize),
- MAX_SUPPORTED_STACK_ALIGNMENT);
- rtx addr = align_dynamic_address (XEXP (stack_parm, 0),
- DECL_ALIGN (parm));
- mark_reg_pointer (addr, DECL_ALIGN (parm));
- stack_parm = gen_rtx_MEM (GET_MODE (stack_parm), addr);
- MEM_NOTRAP_P (stack_parm) = 1;
- }
- else
- stack_parm = assign_stack_local (BLKmode, size_stored,
- DECL_ALIGN (parm));
+ stack_parm = assign_stack_local (BLKmode, size_stored,
+ DECL_ALIGN (parm));
if (known_eq (GET_MODE_SIZE (GET_MODE (entry_parm)), size))
PUT_MODE (stack_parm, GET_MODE (entry_parm));
set_mem_attributes (stack_parm, parm, 1);
diff --git a/gcc/lra-spills.cc b/gcc/lra-spills.cc
index 07801e39d5..f55839f50b 100644
--- a/gcc/lra-spills.cc
+++ b/gcc/lra-spills.cc
@@ -146,8 +146,9 @@ assign_mem_slot (int i)
x = slots[slot_num].mem;
if (!x)
{
- x = assign_stack_local (BLKmode, slots[slot_num].size,
- slots[slot_num].align);
+ x = assign_stack_local_1 (BLKmode, slots[slot_num].size,
+ slots[slot_num].align,
+ ASLK_RECORD_PAD | ASLK_REDUCE_ALIGN);
slots[slot_num].mem = x;
}
--
2.55.0.windows.5
