the patch is revised to replace targetflags with HLE hint.
Hi eli.friedman, hfinkel, jfifield, chandlerc,
http://llvm-reviews.chandlerc.com/D440
CHANGE SINCE LAST DIFF
http://llvm-reviews.chandlerc.com/D440?vs=1056&id=1145#toc
Files:
lib/CodeGen/CGBuiltin.cpp
lib/CodeGen/CGExpr.cpp
test/CodeGen/atomic-ops-targetflags.c
Index: lib/CodeGen/CGBuiltin.cpp
===================================================================
--- lib/CodeGen/CGBuiltin.cpp
+++ lib/CodeGen/CGBuiltin.cpp
@@ -1075,7 +1075,9 @@
Value *NewVal = Builder.getInt8(1);
Value *Order = EmitScalarExpr(E->getArg(1));
if (isa<llvm::ConstantInt>(Order)) {
- int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
+ unsigned ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
+ unsigned flags = ord >> 16;
+ ord = ord & 0xFFFF; // Mask off target flags.
AtomicRMWInst *Result = 0;
switch (ord) {
case 0: // memory_order_relaxed
@@ -1107,6 +1109,19 @@
break;
}
Result->setVolatile(Volatile);
+ if (flags) {
+ llvm::Value *string = 0;
+
+ if (flags & 0x01)
+ string = llvm::MDString::get(getLLVMContext(), "acquire");
+ else if (flags & 0x02)
+ string = llvm::MDString::get(getLLVMContext(), "release");
+
+ if (string) {
+ llvm::MDNode *HLEFlags = llvm::MDNode::get(getLLVMContext(), string);
+ Result->setMetadata("hle.lock", HLEFlags);
+ }
+ }
return RValue::get(Builder.CreateIsNotNull(Result, "tobool"));
}
@@ -1124,7 +1139,7 @@
llvm::AcquireRelease, llvm::SequentiallyConsistent
};
- Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
+ Order = Builder.CreateIntCast(Order, Builder.getInt16Ty(), false);
llvm::SwitchInst *SI = Builder.CreateSwitch(Order, BBs[0]);
Builder.SetInsertPoint(ContBB);
@@ -1139,12 +1154,12 @@
Builder.CreateBr(ContBB);
}
- SI->addCase(Builder.getInt32(0), BBs[0]);
- SI->addCase(Builder.getInt32(1), BBs[1]);
- SI->addCase(Builder.getInt32(2), BBs[1]);
- SI->addCase(Builder.getInt32(3), BBs[2]);
- SI->addCase(Builder.getInt32(4), BBs[3]);
- SI->addCase(Builder.getInt32(5), BBs[4]);
+ SI->addCase(Builder.getInt16(0), BBs[0]);
+ SI->addCase(Builder.getInt16(1), BBs[1]);
+ SI->addCase(Builder.getInt16(2), BBs[1]);
+ SI->addCase(Builder.getInt16(3), BBs[2]);
+ SI->addCase(Builder.getInt16(4), BBs[3]);
+ SI->addCase(Builder.getInt16(5), BBs[4]);
Builder.SetInsertPoint(ContBB);
return RValue::get(Builder.CreateIsNotNull(Result, "tobool"));
@@ -1161,7 +1176,9 @@
Value *NewVal = Builder.getInt8(0);
Value *Order = EmitScalarExpr(E->getArg(1));
if (isa<llvm::ConstantInt>(Order)) {
- int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
+ unsigned ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
+ unsigned flags = ord >> 16;
+ ord = ord & 0xFFFF; // Mask off target flags.
StoreInst *Store = Builder.CreateStore(NewVal, Ptr, Volatile);
Store->setAlignment(1);
switch (ord) {
@@ -1176,6 +1193,19 @@
Store->setOrdering(llvm::SequentiallyConsistent);
break;
}
+ if (flags) {
+ llvm::Value *string = 0;
+
+ if (flags & 0x01)
+ string = llvm::MDString::get(getLLVMContext(), "acquire");
+ else if (flags & 0x02)
+ string = llvm::MDString::get(getLLVMContext(), "release");
+
+ if (string) {
+ llvm::MDNode *HLEFlags = llvm::MDNode::get(getLLVMContext(), string);
+ Store->setMetadata("hle.lock", HLEFlags);
+ }
+ }
return RValue::get(0);
}
@@ -1190,7 +1220,7 @@
llvm::Monotonic, llvm::Release, llvm::SequentiallyConsistent
};
- Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
+ Order = Builder.CreateIntCast(Order, Builder.getInt16Ty(), false);
llvm::SwitchInst *SI = Builder.CreateSwitch(Order, BBs[0]);
for (unsigned i = 0; i < 3; ++i) {
@@ -1201,9 +1231,9 @@
Builder.CreateBr(ContBB);
}
- SI->addCase(Builder.getInt32(0), BBs[0]);
- SI->addCase(Builder.getInt32(3), BBs[1]);
- SI->addCase(Builder.getInt32(5), BBs[2]);
+ SI->addCase(Builder.getInt16(0), BBs[0]);
+ SI->addCase(Builder.getInt16(3), BBs[1]);
+ SI->addCase(Builder.getInt16(5), BBs[2]);
Builder.SetInsertPoint(ContBB);
return RValue::get(0);
Index: lib/CodeGen/CGExpr.cpp
===================================================================
--- lib/CodeGen/CGExpr.cpp
+++ lib/CodeGen/CGExpr.cpp
@@ -3138,7 +3138,8 @@
static void
EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, llvm::Value *Dest,
llvm::Value *Ptr, llvm::Value *Val1, llvm::Value *Val2,
- uint64_t Size, unsigned Align, llvm::AtomicOrdering Order) {
+ uint64_t Size, unsigned Align, llvm::AtomicOrdering Order,
+ llvm::MDNode *HLEHint = 0) {
llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add;
llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0;
@@ -3159,6 +3160,8 @@
llvm::AtomicCmpXchgInst *CXI =
CGF.Builder.CreateAtomicCmpXchg(Ptr, LoadVal1, LoadVal2, Order);
CXI->setVolatile(E->isVolatile());
+ if (HLEHint)
+ CXI->setMetadata("hle.lock", HLEHint);
llvm::StoreInst *StoreVal1 = CGF.Builder.CreateStore(CXI, Val1);
StoreVal1->setAlignment(Align);
llvm::Value *Cmp = CGF.Builder.CreateICmpEQ(CXI, LoadVal1);
@@ -3173,6 +3176,8 @@
Load->setAtomic(Order);
Load->setAlignment(Size);
Load->setVolatile(E->isVolatile());
+ if (HLEHint)
+ Load->setMetadata("hle.lock", HLEHint);
llvm::StoreInst *StoreDest = CGF.Builder.CreateStore(Load, Dest);
StoreDest->setAlignment(Align);
return;
@@ -3188,6 +3193,8 @@
Store->setAtomic(Order);
Store->setAlignment(Size);
Store->setVolatile(E->isVolatile());
+ if (HLEHint)
+ Store->setMetadata("hle.lock", HLEHint);
return;
}
@@ -3250,6 +3257,8 @@
llvm::AtomicRMWInst *RMWI =
CGF.Builder.CreateAtomicRMW(Op, Ptr, LoadVal1, Order);
RMWI->setVolatile(E->isVolatile());
+ if (HLEHint)
+ RMWI->setMetadata("hle.lock", HLEHint);
// For __atomic_*_fetch operations, perform the operation again to
// determine the value which was written.
@@ -3505,34 +3514,46 @@
if (Dest && !E->isCmpXChg()) Dest = Builder.CreateBitCast(Dest, IPtrTy);
if (isa<llvm::ConstantInt>(Order)) {
- int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
+ unsigned ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
+ unsigned flags = ord >> 16;
+ ord = ord & 0xFFFF; // Mask off target flags.
+ llvm::MDNode *HLEHint = 0;
+ if (flags) {
+ llvm::Value *string = 0;
+ if (flags & 0x01)
+ string = llvm::MDString::get(getLLVMContext(), "acquire");
+ else if (flags & 0x02)
+ string = llvm::MDString::get(getLLVMContext(), "release");
+ if (string)
+ HLEHint = llvm::MDNode::get(getLLVMContext(), string);
+ }
switch (ord) {
case 0: // memory_order_relaxed
EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
- llvm::Monotonic);
+ llvm::Monotonic, HLEHint);
break;
case 1: // memory_order_consume
case 2: // memory_order_acquire
if (IsStore)
break; // Avoid crashing on code with undefined behavior
EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
- llvm::Acquire);
+ llvm::Acquire, HLEHint);
break;
case 3: // memory_order_release
if (IsLoad)
break; // Avoid crashing on code with undefined behavior
EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
- llvm::Release);
+ llvm::Release, HLEHint);
break;
case 4: // memory_order_acq_rel
if (IsLoad || IsStore)
break; // Avoid crashing on code with undefined behavior
EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
- llvm::AcquireRelease);
+ llvm::AcquireRelease, HLEHint);
break;
case 5: // memory_order_seq_cst
EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
- llvm::SequentiallyConsistent);
+ llvm::SequentiallyConsistent, HLEHint);
break;
default: // invalid order
// We should not ever get here normally, but it's hard to
@@ -3563,7 +3584,10 @@
// MonotonicBB is arbitrarily chosen as the default case; in practice, this
// doesn't matter unless someone is crazy enough to use something that
// doesn't fold to a constant for the ordering.
- Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
+ //
+ // Cast to i16 to mask off the target flags. So far, if order cannot be
+ // folded into a constant, target flags are ignored.
+ Order = Builder.CreateIntCast(Order, Builder.getInt16Ty(), false);
llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB);
// Emit all the different atomics
@@ -3576,28 +3600,28 @@
EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
llvm::Acquire);
Builder.CreateBr(ContBB);
- SI->addCase(Builder.getInt32(1), AcquireBB);
- SI->addCase(Builder.getInt32(2), AcquireBB);
+ SI->addCase(Builder.getInt16(1), AcquireBB);
+ SI->addCase(Builder.getInt16(2), AcquireBB);
}
if (!IsLoad) {
Builder.SetInsertPoint(ReleaseBB);
EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
llvm::Release);
Builder.CreateBr(ContBB);
- SI->addCase(Builder.getInt32(3), ReleaseBB);
+ SI->addCase(Builder.getInt16(3), ReleaseBB);
}
if (!IsLoad && !IsStore) {
Builder.SetInsertPoint(AcqRelBB);
EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
llvm::AcquireRelease);
Builder.CreateBr(ContBB);
- SI->addCase(Builder.getInt32(4), AcqRelBB);
+ SI->addCase(Builder.getInt16(4), AcqRelBB);
}
Builder.SetInsertPoint(SeqCstBB);
EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
llvm::SequentiallyConsistent);
Builder.CreateBr(ContBB);
- SI->addCase(Builder.getInt32(5), SeqCstBB);
+ SI->addCase(Builder.getInt16(5), SeqCstBB);
// Cleanup and return
Builder.SetInsertPoint(ContBB);
Index: test/CodeGen/atomic-ops-targetflags.c
===================================================================
--- /dev/null
+++ test/CodeGen/atomic-ops-targetflags.c
@@ -0,0 +1,199 @@
+// RUN: %clang_cc1 %s -emit-llvm -o - -triple=i686-apple-darwin9 | FileCheck %s
+
+// Also test serialization of atomic operations here, to avoid duplicating the
+// test.
+// RUN: %clang_cc1 %s -emit-pch -o %t -triple=i686-apple-darwin9
+// RUN: %clang_cc1 %s -include-pch %t -triple=i686-apple-darwin9 -emit-llvm -o - | FileCheck %s
+#ifndef ALREADY_INCLUDED
+#define ALREADY_INCLUDED
+
+// Basic IRGen tests for __c11_atomic_* and GNU __atomic_*
+
+typedef enum memory_order {
+ memory_order_relaxed, memory_order_consume, memory_order_acquire,
+ memory_order_release, memory_order_acq_rel, memory_order_seq_cst
+} memory_order;
+
+#define HLE_ACQ_FLAG (1 << 16)
+#define HLE_REL_FLAG (2 << 16)
+
+int fi1(_Atomic(int) *i) {
+ // CHECK: @fi1
+ // CHECK: load atomic i32* {{.*}} seq_cst, {{.*}}, !hle.lock !0
+ return __c11_atomic_load(i, memory_order_seq_cst | HLE_ACQ_FLAG);
+}
+
+int fi1a(int *i) {
+ // CHECK: @fi1a
+ // CHECK: load atomic i32* {{.*}} seq_cst, {{.*}}, !hle.lock !1
+ int v;
+ __atomic_load(i, &v, memory_order_seq_cst | HLE_REL_FLAG);
+ return v;
+}
+
+int fi1b(int *i) {
+ // CHECK: @fi1b
+ // CHECK: load atomic i32* {{.*}} seq_cst, {{.*}}, !hle.lock !0
+ return __atomic_load_n(i, memory_order_seq_cst | HLE_ACQ_FLAG);
+}
+
+void fi2(_Atomic(int) *i) {
+ // CHECK: @fi2
+ // CHECK: store atomic i32 {{.*}} seq_cst, {{.*}}, !hle.lock !1
+ __c11_atomic_store(i, 1, memory_order_seq_cst | HLE_REL_FLAG);
+}
+
+void fi2a(int *i) {
+ // CHECK: @fi2a
+ // CHECK: store atomic i32 {{.*}} seq_cst, {{.*}}, !hle.lock !0
+ int v = 1;
+ __atomic_store(i, &v, memory_order_seq_cst | HLE_ACQ_FLAG);
+}
+
+void fi2b(int *i) {
+ // CHECK: @fi2b
+ // CHECK: store atomic i32 {{.*}} seq_cst, {{.*}}, !hle.lock !1
+ __atomic_store_n(i, 1, memory_order_seq_cst | HLE_REL_FLAG);
+}
+
+int fi3(_Atomic(int) *i) {
+ // CHECK: @fi3
+ // CHECK: atomicrmw and i32* {{.*}} seq_cst, !hle.lock !0
+ // CHECK-NOT: and
+ return __c11_atomic_fetch_and(i, 1, memory_order_seq_cst | HLE_ACQ_FLAG);
+}
+
+int fi3a(int *i) {
+ // CHECK: @fi3a
+ // CHECK: atomicrmw xor i32* {{.*}} seq_cst, !hle.lock !1
+ // CHECK-NOT: xor
+ return __atomic_fetch_xor(i, 1, memory_order_seq_cst | HLE_REL_FLAG);
+}
+
+int fi3b(int *i) {
+ // CHECK: @fi3b
+ // CHECK: atomicrmw add i32* {{.*}} seq_cst, !hle.lock !0
+ // CHECK: add
+ return __atomic_add_fetch(i, 1, memory_order_seq_cst | HLE_ACQ_FLAG);
+}
+
+int fi3c(int *i) {
+ // CHECK: @fi3c
+ // CHECK: atomicrmw nand i32* {{.*}} seq_cst, !hle.lock !1
+ // CHECK-NOT: and
+ return __atomic_fetch_nand(i, 1, memory_order_seq_cst | HLE_REL_FLAG);
+}
+
+int fi3d(int *i) {
+ // CHECK: @fi3d
+ // CHECK: atomicrmw nand i32* {{.*}} seq_cst, !hle.lock !0
+ // CHECK: and
+ // CHECK: xor
+ return __atomic_nand_fetch(i, 1, memory_order_seq_cst | HLE_ACQ_FLAG);
+}
+
+_Bool fi4(_Atomic(int) *i) {
+ // CHECK: @fi4
+ // CHECK: cmpxchg i32* {{.*}} acquire, !hle.lock !1
+ int cmp = 0;
+ return __c11_atomic_compare_exchange_strong(i, &cmp, 1, memory_order_acquire | HLE_REL_FLAG, memory_order_acquire);
+}
+
+_Bool fi4a(int *i) {
+ // CHECK: @fi4
+ // CHECK: cmpxchg i32* {{.*}} acquire, !hle.lock !0
+ int cmp = 0;
+ int desired = 1;
+ return __atomic_compare_exchange(i, &cmp, &desired, 0, memory_order_acquire | HLE_ACQ_FLAG, memory_order_acquire);
+}
+
+_Bool fi4b(int *i) {
+ // CHECK: @fi4
+ // CHECK: cmpxchg i32* {{.*}} acquire, !hle.lock !1
+ int cmp = 0;
+ return __atomic_compare_exchange_n(i, &cmp, 1, 1, memory_order_acquire | HLE_REL_FLAG, memory_order_acquire);
+}
+
+float ff1(_Atomic(float) *d) {
+ // CHECK: @ff1
+ // CHECK: load atomic i32* {{.*}} monotonic, {{.*}}, !hle.lock !0
+ return __c11_atomic_load(d, memory_order_relaxed | HLE_ACQ_FLAG);
+}
+
+void ff2(_Atomic(float) *d) {
+ // CHECK: @ff2
+ // CHECK: store atomic i32 {{.*}} release, {{.*}}, !hle.lock !1
+ __c11_atomic_store(d, 1, memory_order_release | HLE_REL_FLAG);
+}
+
+float ff3(_Atomic(float) *d) {
+ // CHECK: @ff3
+ // CHECK: atomicrmw xchg i32* {{.*}} seq_cst, !hle.lock !0
+ return __c11_atomic_exchange(d, 2, memory_order_seq_cst | HLE_ACQ_FLAG);
+}
+
+int* fp1(_Atomic(int*) *p) {
+ // CHECK: @fp1
+ // CHECK: load atomic i32* {{.*}} seq_cst, {{.*}}, !hle.lock !1
+ return __c11_atomic_load(p, memory_order_seq_cst | HLE_REL_FLAG);
+}
+
+int* fp2(_Atomic(int*) *p) {
+ // CHECK: @fp2
+ // CHECK: store i32 4
+ // CHECK: atomicrmw add i32* {{.*}} monotonic, !hle.lock !0
+ return __c11_atomic_fetch_add(p, 1, memory_order_relaxed | HLE_ACQ_FLAG);
+}
+
+int *fp2a(int **p) {
+ // CHECK: @fp2a
+ // CHECK: store i32 4
+ // CHECK: atomicrmw sub i32* {{.*}} monotonic, !hle.lock !1
+ // Note, the GNU builtins do not multiply by sizeof(T)!
+ return __atomic_fetch_sub(p, 4, memory_order_relaxed | HLE_REL_FLAG);
+}
+
+_Complex float fc(_Atomic(_Complex float) *c) {
+ // CHECK: @fc
+ // CHECK: atomicrmw xchg i64* {{.*}} seq_cst, !hle.lock !0
+ return __c11_atomic_exchange(c, 2, memory_order_seq_cst | HLE_ACQ_FLAG);
+}
+
+typedef struct X { int x; } X;
+X fs(_Atomic(X) *c) {
+ // CHECK: @fs
+ // CHECK: atomicrmw xchg i32* {{.*}} seq_cst, !hle.lock !1
+ return __c11_atomic_exchange(c, (X){2}, memory_order_seq_cst | HLE_REL_FLAG);
+}
+
+X fsa(X *c, X *d) {
+ // CHECK: @fsa
+ // CHECK: atomicrmw xchg i32* {{.*}} seq_cst, !hle.lock !0
+ X ret;
+ __atomic_exchange(c, d, &ret, memory_order_seq_cst | HLE_ACQ_FLAG);
+ return ret;
+}
+
+_Bool fsb(_Bool *c) {
+ // CHECK: @fsb
+ // CHECK: atomicrmw xchg i8* {{.*}} seq_cst, !hle.lock !1
+ return __atomic_exchange_n(c, 1, memory_order_seq_cst | HLE_REL_FLAG);
+}
+
+char flag1;
+volatile char flag2;
+void test_and_set() {
+ // CHECK: atomicrmw xchg i8* @flag1, i8 1 seq_cst, !hle.lock !0
+ __atomic_test_and_set(&flag1, memory_order_seq_cst | HLE_ACQ_FLAG);
+ // CHECK: atomicrmw volatile xchg i8* @flag2, i8 1 acquire, !hle.lock !1
+ __atomic_test_and_set(&flag2, memory_order_acquire | HLE_REL_FLAG);
+ // CHECK: store atomic volatile i8 0, i8* @flag2 release, {{.*}}, !hle.lock !0
+ __atomic_clear(&flag2, memory_order_release | HLE_ACQ_FLAG);
+ // CHECK: store atomic i8 0, i8* @flag1 seq_cst, {{.*}}, !hle.lock !1
+ __atomic_clear(&flag1, memory_order_seq_cst | HLE_REL_FLAG);
+}
+
+// CHECK: !0 = metadata !{metadata !"acquire"}
+// CHECK: !1 = metadata !{metadata !"release"}
+
+#endif
_______________________________________________
cfe-commits mailing list
[email protected]
http://lists.cs.uiuc.edu/mailman/listinfo/cfe-commits
