The cas insn is a single insn, and if expanded properly need not be split after reload. Use the proper inputs for the insn.
2020-04-16 Andre Vieira <andre.simoesdiasvie...@arm.com> Backport from mainline. 2018-10-31 Richard Henderson <richard.hender...@linaro.org> * config/aarch64/aarch64.c (aarch64_expand_compare_and_swap): Force oldval into the rval register for TARGET_LSE; emit the compare during initial expansion so that it may be deleted if unused. (aarch64_gen_atomic_cas): Remove. * config/aarch64/atomics.md (aarch64_compare_and_swap<SHORT>_lse): Change =&r to +r for operand 0; use match_dup for operand 2; remove is_weak and mod_f operands as unused. Drop the split and merge with... (aarch64_atomic_cas<SHORT>): ... this pattern's output; remove. (aarch64_compare_and_swap<GPI>_lse): Similarly. (aarch64_atomic_cas<GPI>): Similarly.
diff --git a/gcc/config/aarch64/aarch64-protos.h b/gcc/config/aarch64/aarch64-protos.h index cda2895d28e7496f8fd6c1b365c4bb497b54c323..a03565c3b4e13990dc1a0064f9cbbc38bb109795 100644 --- a/gcc/config/aarch64/aarch64-protos.h +++ b/gcc/config/aarch64/aarch64-protos.h @@ -496,7 +496,6 @@ rtx aarch64_load_tp (rtx); void aarch64_expand_compare_and_swap (rtx op[]); void aarch64_split_compare_and_swap (rtx op[]); -void aarch64_gen_atomic_cas (rtx, rtx, rtx, rtx, rtx); bool aarch64_atomic_ldop_supported_p (enum rtx_code); void aarch64_gen_atomic_ldop (enum rtx_code, rtx, rtx, rtx, rtx, rtx); diff --git a/gcc/config/aarch64/aarch64.c b/gcc/config/aarch64/aarch64.c index 20761578fb6051e600299cd58f245774bd457432..c83a9f7ae78d4ed3da6636fce4d1f57c27048756 100644 --- a/gcc/config/aarch64/aarch64.c +++ b/gcc/config/aarch64/aarch64.c @@ -14169,17 +14169,19 @@ aarch64_expand_compare_and_swap (rtx operands[]) { rtx bval, rval, mem, oldval, newval, is_weak, mod_s, mod_f, x; machine_mode mode, cmp_mode; - typedef rtx (*gen_cas_fn) (rtx, rtx, rtx, rtx, rtx, rtx, rtx); + typedef rtx (*gen_split_cas_fn) (rtx, rtx, rtx, rtx, rtx, rtx, rtx); + typedef rtx (*gen_atomic_cas_fn) (rtx, rtx, rtx, rtx); int idx; - gen_cas_fn gen; - const gen_cas_fn split_cas[] = + gen_split_cas_fn split_gen; + gen_atomic_cas_fn atomic_gen; + const gen_split_cas_fn split_cas[] = { gen_aarch64_compare_and_swapqi, gen_aarch64_compare_and_swaphi, gen_aarch64_compare_and_swapsi, gen_aarch64_compare_and_swapdi }; - const gen_cas_fn atomic_cas[] = + const gen_atomic_cas_fn atomic_cas[] = { gen_aarch64_compare_and_swapqi_lse, gen_aarch64_compare_and_swaphi_lse, @@ -14238,14 +14240,29 @@ aarch64_expand_compare_and_swap (rtx operands[]) gcc_unreachable (); } if (TARGET_LSE) - gen = atomic_cas[idx]; + { + atomic_gen = atomic_cas[idx]; + /* The CAS insn requires oldval and rval overlap, but we need to + have a copy of oldval saved across the operation to tell if + the operation is successful. */ + if (mode == QImode || mode == HImode) + rval = copy_to_mode_reg (SImode, gen_lowpart (SImode, oldval)); + else if (reg_overlap_mentioned_p (rval, oldval)) + rval = copy_to_mode_reg (mode, oldval); + else + emit_move_insn (rval, oldval); + emit_insn (atomic_gen (rval, mem, newval, mod_s)); + aarch64_gen_compare_reg (EQ, rval, oldval); + } else - gen = split_cas[idx]; - - emit_insn (gen (rval, mem, oldval, newval, is_weak, mod_s, mod_f)); + { + split_gen = split_cas[idx]; + emit_insn (split_gen (rval, mem, oldval, newval, is_weak, mod_s, mod_f)); + } if (mode == QImode || mode == HImode) - emit_move_insn (operands[1], gen_lowpart (mode, rval)); + rval = gen_lowpart (mode, rval); + emit_move_insn (operands[1], rval); x = gen_rtx_REG (CCmode, CC_REGNUM); x = gen_rtx_EQ (SImode, x, const0_rtx); @@ -14295,42 +14312,6 @@ aarch64_emit_post_barrier (enum memmodel model) } } -/* Emit an atomic compare-and-swap operation. RVAL is the destination register - for the data in memory. EXPECTED is the value expected to be in memory. - DESIRED is the value to store to memory. MEM is the memory location. MODEL - is the memory ordering to use. */ - -void -aarch64_gen_atomic_cas (rtx rval, rtx mem, - rtx expected, rtx desired, - rtx model) -{ - rtx (*gen) (rtx, rtx, rtx, rtx); - machine_mode mode; - - mode = GET_MODE (mem); - - switch (mode) - { - case E_QImode: gen = gen_aarch64_atomic_casqi; break; - case E_HImode: gen = gen_aarch64_atomic_cashi; break; - case E_SImode: gen = gen_aarch64_atomic_cassi; break; - case E_DImode: gen = gen_aarch64_atomic_casdi; break; - default: - gcc_unreachable (); - } - - /* Move the expected value into the CAS destination register. */ - emit_insn (gen_rtx_SET (rval, expected)); - - /* Emit the CAS. */ - emit_insn (gen (rval, mem, desired, model)); - - /* Compare the expected value with the value loaded by the CAS, to establish - whether the swap was made. */ - aarch64_gen_compare_reg (EQ, rval, expected); -} - /* Split a compare and swap pattern. */ void diff --git a/gcc/config/aarch64/atomics.md b/gcc/config/aarch64/atomics.md index fba5ec6db5832a184b0323e62041f9c473761bae..be970d105ffbe218afb044bef900494454dd8d37 100644 --- a/gcc/config/aarch64/atomics.md +++ b/gcc/config/aarch64/atomics.md @@ -85,56 +85,50 @@ } ) -(define_insn_and_split "aarch64_compare_and_swap<mode>_lse" - [(set (reg:CC CC_REGNUM) ;; bool out - (unspec_volatile:CC [(const_int 0)] UNSPECV_ATOMIC_CMPSW)) - (set (match_operand:SI 0 "register_operand" "=&r") ;; val out +(define_insn "aarch64_compare_and_swap<mode>_lse" + [(set (match_operand:SI 0 "register_operand" "+r") ;; val out (zero_extend:SI - (match_operand:SHORT 1 "aarch64_sync_memory_operand" "+Q"))) ;; memory + (match_operand:SHORT 1 "aarch64_sync_memory_operand" "+Q"))) ;; memory (set (match_dup 1) (unspec_volatile:SHORT - [(match_operand:SI 2 "aarch64_plus_operand" "rI") ;; expected - (match_operand:SHORT 3 "aarch64_reg_or_zero" "rZ") ;; desired - (match_operand:SI 4 "const_int_operand") ;; is_weak - (match_operand:SI 5 "const_int_operand") ;; mod_s - (match_operand:SI 6 "const_int_operand")] ;; mod_f + [(match_dup 0) ;; expected + (match_operand:SHORT 2 "aarch64_reg_or_zero" "rZ") ;; desired + (match_operand:SI 3 "const_int_operand")] ;; mod_s UNSPECV_ATOMIC_CMPSW))] "TARGET_LSE" - "#" - "&& reload_completed" - [(const_int 0)] - { - aarch64_gen_atomic_cas (operands[0], operands[1], - operands[2], operands[3], - operands[5]); - DONE; - } -) +{ + enum memmodel model = memmodel_from_int (INTVAL (operands[3])); + if (is_mm_relaxed (model)) + return "cas<atomic_sfx>\t%<w>0, %<w>2, %1"; + else if (is_mm_acquire (model) || is_mm_consume (model)) + return "casa<atomic_sfx>\t%<w>0, %<w>2, %1"; + else if (is_mm_release (model)) + return "casl<atomic_sfx>\t%<w>0, %<w>2, %1"; + else + return "casal<atomic_sfx>\t%<w>0, %<w>2, %1"; +}) -(define_insn_and_split "aarch64_compare_and_swap<mode>_lse" - [(set (reg:CC CC_REGNUM) ;; bool out - (unspec_volatile:CC [(const_int 0)] UNSPECV_ATOMIC_CMPSW)) - (set (match_operand:GPI 0 "register_operand" "=&r") ;; val out +(define_insn "aarch64_compare_and_swap<mode>_lse" + [(set (match_operand:GPI 0 "register_operand" "+r") ;; val out (match_operand:GPI 1 "aarch64_sync_memory_operand" "+Q")) ;; memory (set (match_dup 1) (unspec_volatile:GPI - [(match_operand:GPI 2 "aarch64_plus_operand" "rI") ;; expect - (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ") ;; desired - (match_operand:SI 4 "const_int_operand") ;; is_weak - (match_operand:SI 5 "const_int_operand") ;; mod_s - (match_operand:SI 6 "const_int_operand")] ;; mod_f + [(match_dup 0) ;; expected + (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ") ;; desired + (match_operand:SI 3 "const_int_operand")] ;; mod_s UNSPECV_ATOMIC_CMPSW))] "TARGET_LSE" - "#" - "&& reload_completed" - [(const_int 0)] - { - aarch64_gen_atomic_cas (operands[0], operands[1], - operands[2], operands[3], - operands[5]); - DONE; - } -) +{ + enum memmodel model = memmodel_from_int (INTVAL (operands[3])); + if (is_mm_relaxed (model)) + return "cas<atomic_sfx>\t%<w>0, %<w>2, %1"; + else if (is_mm_acquire (model) || is_mm_consume (model)) + return "casa<atomic_sfx>\t%<w>0, %<w>2, %1"; + else if (is_mm_release (model)) + return "casl<atomic_sfx>\t%<w>0, %<w>2, %1"; + else + return "casal<atomic_sfx>\t%<w>0, %<w>2, %1"; +}) (define_expand "atomic_exchange<mode>" [(match_operand:ALLI 0 "register_operand" "") @@ -607,55 +601,6 @@ return "swpal<atomic_sfx>\t%<w>2, %<w>0, %1"; }) -;; Atomic compare-and-swap: HI and smaller modes. - -(define_insn "aarch64_atomic_cas<mode>" - [(set (match_operand:SI 0 "register_operand" "+&r") ;; out - (zero_extend:SI - (match_operand:SHORT 1 "aarch64_sync_memory_operand" "+Q"))) ;; memory. - (set (match_dup 1) - (unspec_volatile:SHORT - [(match_dup 0) - (match_operand:SHORT 2 "aarch64_reg_or_zero" "rZ") ;; value. - (match_operand:SI 3 "const_int_operand" "")] ;; model. - UNSPECV_ATOMIC_CAS))] - "TARGET_LSE && reload_completed" -{ - enum memmodel model = memmodel_from_int (INTVAL (operands[3])); - if (is_mm_relaxed (model)) - return "cas<atomic_sfx>\t%<w>0, %<w>2, %1"; - else if (is_mm_acquire (model) || is_mm_consume (model)) - return "casa<atomic_sfx>\t%<w>0, %<w>2, %1"; - else if (is_mm_release (model)) - return "casl<atomic_sfx>\t%<w>0, %<w>2, %1"; - else - return "casal<atomic_sfx>\t%<w>0, %<w>2, %1"; -}) - -;; Atomic compare-and-swap: SI and larger modes. - -(define_insn "aarch64_atomic_cas<mode>" - [(set (match_operand:GPI 0 "register_operand" "+&r") ;; out - (match_operand:GPI 1 "aarch64_sync_memory_operand" "+Q")) ;; memory. - (set (match_dup 1) - (unspec_volatile:GPI - [(match_dup 0) - (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ") ;; value. - (match_operand:SI 3 "const_int_operand" "")] ;; model. - UNSPECV_ATOMIC_CAS))] - "TARGET_LSE && reload_completed" -{ - enum memmodel model = memmodel_from_int (INTVAL (operands[3])); - if (is_mm_relaxed (model)) - return "cas<atomic_sfx>\t%<w>0, %<w>2, %1"; - else if (is_mm_acquire (model) || is_mm_consume (model)) - return "casa<atomic_sfx>\t%<w>0, %<w>2, %1"; - else if (is_mm_release (model)) - return "casl<atomic_sfx>\t%<w>0, %<w>2, %1"; - else - return "casal<atomic_sfx>\t%<w>0, %<w>2, %1"; -}) - ;; Atomic load-op: Load data, operate, store result, keep data. (define_insn "aarch64_atomic_load<atomic_ldop><mode>"