diff --git a/gcc/builtins.c b/gcc/builtins.c index 7afe61d..cbef9ed 100644 --- a/gcc/builtins.c +++ b/gcc/builtins.c @@ -5339,7 +5339,7 @@ get_memmodel (tree exp) return MEMMODEL_SEQ_CST; op = expand_normal (exp); - if (INTVAL (op) < 0 || INTVAL (op) >= MEMMODEL_LAST) + if (INTVAL (op) < 0) { warning (OPT_Winvalid_memory_model, "invalid memory model argument to builtin"); @@ -5392,11 +5392,15 @@ expand_builtin_atomic_compare_exchange (enum machine_mode mode, tree exp, enum memmodel success, failure; tree weak; bool is_weak; + unsigned memmodel_last_mask = (1 << (sizeof(MEMMODEL_LAST) * 8 - 1 + - __builtin_clz (MEMMODEL_LAST))); + unsigned memmodel_mask = (memmodel_last_mask - 1) | memmodel_last_mask; success = get_memmodel (CALL_EXPR_ARG (exp, 4)); failure = get_memmodel (CALL_EXPR_ARG (exp, 5)); - if (failure == MEMMODEL_RELEASE || failure == MEMMODEL_ACQ_REL) + if ( (failure & memmodel_mask) == MEMMODEL_RELEASE + || (failure & memmodel_mask) == MEMMODEL_ACQ_REL) { error ("invalid failure memory model for %<__atomic_compare_exchange%>"); return NULL_RTX; diff --git a/gcc/config/i386/cpuid.h b/gcc/config/i386/cpuid.h index 6696b7a..a9d25c5 100644 --- a/gcc/config/i386/cpuid.h +++ b/gcc/config/i386/cpuid.h @@ -66,6 +66,7 @@ /* Extended Features (%eax == 7) */ #define bit_FSGSBASE (1 << 0) #define bit_BMI (1 << 3) +#define bit_HLE (1 << 4) #define bit_AVX2 (1 << 5) #define bit_BMI2 (1 << 8) #define bit_RTM (1 << 11) diff --git a/gcc/config/i386/driver-i386.c b/gcc/config/i386/driver-i386.c index 09de555..34cd096 100644 --- a/gcc/config/i386/driver-i386.c +++ b/gcc/config/i386/driver-i386.c @@ -397,6 +397,7 @@ const char *host_detect_local_cpu (int argc, const char **argv) unsigned int has_pclmul = 0, has_abm = 0, has_lwp = 0; unsigned int has_fma = 0, has_fma4 = 0, has_xop = 0; unsigned int has_bmi = 0, has_bmi2 = 0, has_tbm = 0, has_lzcnt = 0; + unsigned int has_hle = 0; bool arch; @@ -456,6 +457,7 @@ const char *host_detect_local_cpu (int argc, const char **argv) __cpuid_count (7, 0, eax, ebx, ecx, edx); has_bmi = ebx & bit_BMI; + has_hle = ebx & bit_HLE; has_avx2 = ebx & bit_AVX2; has_bmi2 = ebx & bit_BMI2; } @@ -726,10 +728,12 @@ const char *host_detect_local_cpu (int argc, const char **argv) const char *sse4_2 = has_sse4_2 ? " -msse4.2" : " -mno-sse4.2"; const char *sse4_1 = has_sse4_1 ? " -msse4.1" : " -mno-sse4.1"; const char *lzcnt = has_lzcnt ? " -mlzcnt" : " -mno-lzcnt"; + const char *hle = has_hle ? " -mhle" : "-mno-hle"; options = concat (options, cx16, sahf, movbe, ase, pclmul, popcnt, abm, lwp, fma, fma4, xop, bmi, bmi2, - tbm, avx, avx2, sse4_2, sse4_1, lzcnt, NULL); + tbm, avx, avx2, sse4_2, sse4_1, lzcnt, + hle, NULL); } done: diff --git a/gcc/config/i386/i386-c.c b/gcc/config/i386/i386-c.c index 8adb3b4..d26c998 100644 --- a/gcc/config/i386/i386-c.c +++ b/gcc/config/i386/i386-c.c @@ -279,6 +279,17 @@ ix86_target_macros_internal (HOST_WIDE_INT isa_flag, def_or_undef (parse_in, "__BMI2__"); if (isa_flag & OPTION_MASK_ISA_LZCNT) def_or_undef (parse_in, "__LZCNT__"); + if (isa_flag & OPTION_MASK_ISA_HLE) { + char buf[64]; + + sprintf (buf, "__ATOMIC_HLE_ACQUIRE=%d", IX86_HLE_ACQUIRE); + def_or_undef (parse_in, buf); + + sprintf (buf, "__ATOMIC_HLE_RELEASE=%d", IX86_HLE_RELEASE); + def_or_undef (parse_in, buf); + + def_or_undef (parse_in, "__HLE__"); + } if (isa_flag & OPTION_MASK_ISA_TBM) def_or_undef (parse_in, "__TBM__"); if (isa_flag & OPTION_MASK_ISA_POPCNT) diff --git a/gcc/config/i386/i386.c b/gcc/config/i386/i386.c index 8974ddc..6d68525 100644 --- a/gcc/config/i386/i386.c +++ b/gcc/config/i386/i386.c @@ -2679,6 +2679,7 @@ ix86_target_string (HOST_WIDE_INT isa, int flags, const char *arch, { "-mbmi", OPTION_MASK_ISA_BMI }, { "-mbmi2", OPTION_MASK_ISA_BMI2 }, { "-mlzcnt", OPTION_MASK_ISA_LZCNT }, + { "-mhle", OPTION_MASK_ISA_HLE }, { "-mtbm", OPTION_MASK_ISA_TBM }, { "-mpopcnt", OPTION_MASK_ISA_POPCNT }, { "-mmovbe", OPTION_MASK_ISA_MOVBE }, @@ -2954,6 +2955,7 @@ ix86_option_override_internal (bool main_args_p) #define PTA_AVX2 (HOST_WIDE_INT_1 << 30) #define PTA_BMI2 (HOST_WIDE_INT_1 << 31) #define PTA_RTM (HOST_WIDE_INT_1 << 32) +#define PTA_HLE (HOST_WIDE_INT_1 << 33) /* if this reaches 64, need to widen struct pta flags below */ static struct pta @@ -3012,7 +3014,7 @@ ix86_option_override_internal (bool main_args_p) | PTA_SSSE3 | PTA_SSE4_1 | PTA_SSE4_2 | PTA_AVX | PTA_AVX2 | PTA_CX16 | PTA_POPCNT | PTA_AES | PTA_PCLMUL | PTA_FSGSBASE | PTA_RDRND | PTA_F16C | PTA_BMI | PTA_BMI2 | PTA_LZCNT - | PTA_FMA | PTA_MOVBE | PTA_RTM}, + | PTA_FMA | PTA_MOVBE | PTA_RTM | PTA_HLE}, {"atom", PROCESSOR_ATOM, CPU_ATOM, PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3 | PTA_SSSE3 | PTA_CX16 | PTA_MOVBE}, @@ -3430,6 +3432,9 @@ ix86_option_override_internal (bool main_args_p) if (processor_alias_table[i].flags & PTA_RTM && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_RTM)) ix86_isa_flags |= OPTION_MASK_ISA_RTM; + if (processor_alias_table[i].flags & PTA_HLE + && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_HLE)) + ix86_isa_flags |= OPTION_MASK_ISA_HLE; if (processor_alias_table[i].flags & (PTA_PREFETCH_SSE | PTA_SSE)) x86_prefetch_sse = true; @@ -4251,6 +4256,7 @@ ix86_valid_target_attribute_inner_p (tree args, char *p_strings[], IX86_ATTR_ISA ("rdrnd", OPT_mrdrnd), IX86_ATTR_ISA ("f16c", OPT_mf16c), IX86_ATTR_ISA ("rtm", OPT_mrtm), + IX86_ATTR_ISA ("hle", OPT_mhle), /* enum options */ IX86_ATTR_ENUM ("fpmath=", OPT_mfpmath_), diff --git a/gcc/config/i386/i386.h b/gcc/config/i386/i386.h index 8942ea8..22c437e 100644 --- a/gcc/config/i386/i386.h +++ b/gcc/config/i386/i386.h @@ -75,6 +75,7 @@ see the files COPYING3 and COPYING.RUNTIME respectively. If not, see #define TARGET_RDRND OPTION_ISA_RDRND #define TARGET_F16C OPTION_ISA_F16C #define TARGET_RTM OPTION_ISA_RTM +#define TARGET_HLE OPTION_ISA_HLE #define TARGET_LP64 OPTION_ABI_64 #define TARGET_X32 OPTION_ABI_X32 @@ -2344,6 +2345,12 @@ extern void debug_dispatch_window (int); #define TARGET_RECIP_VEC_DIV ((recip_mask & RECIP_MASK_VEC_DIV) != 0) #define TARGET_RECIP_VEC_SQRT ((recip_mask & RECIP_MASK_VEC_SQRT) != 0) +#define IX86_HLE_ACQUIRE (1 << (sizeof(MEMMODEL_LAST) * 8 - \ + __builtin_clz (MEMMODEL_LAST))) + +#define IX86_HLE_RELEASE (1 << (sizeof(MEMMODEL_LAST) * 8 - \ + __builtin_clz (MEMMODEL_LAST) - 1)) + /* Local variables: version-control: t diff --git a/gcc/config/i386/i386.opt b/gcc/config/i386/i386.opt index bf50aed..1d16149 100644 --- a/gcc/config/i386/i386.opt +++ b/gcc/config/i386/i386.opt @@ -528,6 +528,10 @@ mlzcnt Target Report Mask(ISA_LZCNT) Var(ix86_isa_flags) Save Support LZCNT built-in function and code generation +mhle +Target Report Mask(ISA_HLE) Var(ix86_isa_flags) Save +Support Hardware Lock Elision prefixies + mtbm Target Report Mask(ISA_TBM) Var(ix86_isa_flags) Save Support TBM built-in functions and code generation diff --git a/gcc/config/i386/sync.md b/gcc/config/i386/sync.md index 18ccabf..f2b132a 100644 --- a/gcc/config/i386/sync.md +++ b/gcc/config/i386/sync.md @@ -315,8 +315,9 @@ (match_operand:SI 7 "const_int_operand")] ;; failure model "TARGET_CMPXCHG" { - emit_insn (gen_atomic_compare_and_swap_single - (operands[1], operands[2], operands[3], operands[4])); + emit_insn + (gen_atomic_compare_and_swap_single + (operands[1], operands[2], operands[3], operands[4], operands[6])); ix86_expand_setcc (operands[0], EQ, gen_rtx_REG (CCZmode, FLAGS_REG), const0_rtx); DONE; @@ -344,8 +345,9 @@ { if (mode == DImode && TARGET_64BIT) { - emit_insn (gen_atomic_compare_and_swap_singledi - (operands[1], operands[2], operands[3], operands[4])); + emit_insn + (gen_atomic_compare_and_swap_singledi + (operands[1], operands[2], operands[3], operands[4], operands[6])); } else { @@ -370,7 +372,7 @@ mem = replace_equiv_address (mem, force_reg (Pmode, XEXP (mem, 0))); emit_insn (gen_atomic_compare_and_swap_double - (lo_o, hi_o, mem, lo_e, hi_e, lo_n, hi_n)); + (lo_o, hi_o, mem, lo_e, hi_e, lo_n, hi_n, operands[6])); } ix86_expand_setcc (operands[0], EQ, gen_rtx_REG (CCZmode, FLAGS_REG), const0_rtx); @@ -382,14 +384,25 @@ (unspec_volatile:SWI [(match_operand:SWI 1 "memory_operand" "+m") (match_operand:SWI 2 "register_operand" "0") - (match_operand:SWI 3 "register_operand" "")] + (match_operand:SWI 3 "register_operand" "") + (match_operand:SI 4 "const_int_operand")] UNSPECV_CMPXCHG_1)) (set (match_dup 1) (unspec_volatile:SWI [(const_int 0)] UNSPECV_CMPXCHG_2)) (set (reg:CCZ FLAGS_REG) (unspec_volatile:CCZ [(const_int 0)] UNSPECV_CMPXCHG_3))] "TARGET_CMPXCHG" - "lock{%;} cmpxchg{}\t{%3, %1|%1, %3}") +{ + static char buf[128], hle[16]=""; + if (INTVAL (operands[4]) & IX86_HLE_ACQUIRE) + snprintf (hle, sizeof (hle), "xacquire "); + else if (INTVAL (operands[4]) & IX86_HLE_RELEASE) + snprintf (hle, sizeof (hle), "release "); + + snprintf (buf, sizeof (buf), + "lock{%%;} %scmpxchg{}\t{%%3, %%1|%%1, %%3}", hle); + return buf; +}) ;; For double-word compare and swap, we are obliged to play tricks with ;; the input newval (op5:op6) because the Intel register numbering does @@ -403,7 +416,8 @@ (match_operand: 3 "register_operand" "0") (match_operand: 4 "register_operand" "1") (match_operand: 5 "register_operand" "b") - (match_operand: 6 "register_operand" "c")] + (match_operand: 6 "register_operand" "c") + (match_operand:SI 7 "const_int_operand")] UNSPECV_CMPXCHG_1)) (set (match_operand: 1 "register_operand" "=d") (unspec_volatile: [(const_int 0)] UNSPECV_CMPXCHG_2)) @@ -412,7 +426,18 @@ (set (reg:CCZ FLAGS_REG) (unspec_volatile:CCZ [(const_int 0)] UNSPECV_CMPXCHG_4))] "" - "lock{%;} cmpxchgb\t%2") +{ + static char buf[128], hle[16]; + + if (INTVAL (operands[4]) & IX86_HLE_ACQUIRE) + snprintf (hle, sizeof (hle), "xacquire "); + else if (INTVAL (operands[4]) & IX86_HLE_RELEASE) + snprintf (hle, sizeof (hle), "release "); + + snprintf (buf, sizeof (buf), + "lock{%%;} %scmpxchgb\t%%2", hle); + return buf; +}) ;; Theoretically we'd like to use constraint "r" (any reg) for op5, ;; but that includes ecx. If op5 and op6 are the same (like when diff --git a/gcc/testsuite/gcc.target/i386/hle-cmpxchg-acq-1.c b/gcc/testsuite/gcc.target/i386/hle-cmpxchg-acq-1.c new file mode 100644 index 0000000..e508a76 --- /dev/null +++ b/gcc/testsuite/gcc.target/i386/hle-cmpxchg-acq-1.c @@ -0,0 +1,9 @@ +/* { dg-do compile } */ +/* { dg-options "-mhle -dp" } */ +/* { dg-final { scan-assembler "lock xacquire cmpxchg" } } */ + +int +hle_cmpxchg (int *p, int oldv, int newv) +{ + return __atomic_compare_exchange_n (p, &oldv, newv, 0, __ATOMIC_ACQUIRE | __ATOMIC_HLE_ACQUIRE, __ATOMIC_ACQUIRE); +} diff --git a/gcc/testsuite/gcc.target/i386/hle-cmpxchg-rel-1.c b/gcc/testsuite/gcc.target/i386/hle-cmpxchg-rel-1.c new file mode 100644 index 0000000..f48646b --- /dev/null +++ b/gcc/testsuite/gcc.target/i386/hle-cmpxchg-rel-1.c @@ -0,0 +1,9 @@ +/* { dg-do compile } */ +/* { dg-options "-mhle -dp" } */ +/* { dg-final { scan-assembler "lock release cmpxchg" } } */ + +int +hle_cmpxchg (int *p, int oldv, int newv) +{ + return __atomic_compare_exchange_n (p, &oldv, newv, 0, __ATOMIC_RELEASE | __ATOMIC_HLE_RELEASE, __ATOMIC_ACQUIRE); +}