It seems the mailing doesn't allow me to send message too big. Here is the patch 0001-Add-targetflags-in-AtomicSDNode-MemIntrinsicSDNode.patch
Yours - Michael On Tue, 2013-02-19 at 14:07 -0800, Michael Liao wrote: > Hi All, > > I'd like to add HLE support in LLVM/clang consistent to GCC's style [1]. > HLE from Intel TSX [2] is legacy compatible instruction set extension to > specify transactional region by adding XACQUIRE and XRELEASE prefixes. > To support that, GCC chooses the approach by extending the memory order > flag in __atomic_* builtins with target-specific memory model in high > bits (bit 31-16 for target-specific memory model, bit 15-0 for the > general memory model.) To follow the similar approach, I propose to > change LLVM/clang by adding: > > + a metadata 'targetflags' in LLVM atomic IR to pass this > target-specific memory model hint > > + one extra target flag in AtomicSDNode & MemIntrinsicSDNode to specify > XACQUIRE or XRELEASE hints > This extra target flag is embedded into the SubclassData fields. The > following is rationale how such target flags are embedded into > SubclassData in SDNode > > here is the current SDNode class hierarchy of memory related nodes > > SDNode -> MemSDNode -> LSBaseNode -> LoadSDNode > | + -> StoreSDNode > + -> AtomicSDNode > + -> MemIntrinsicSDNode > > here is the current SubclassData definitions: > > bit 0~1 : extension type used in LoadSDNode > bit 0 : truncating store in StoreSDNode > bit 2~4 : addressing mode in LSBaseNode > bit 5 : volatile bit in MemSDNode > bit 6 : non-temporal bit in MemSDNode > bit 7 : invariant bit in MemSDNode > bit 8~11: memory order in AtomicSDNode > bit 12 : synch scope in AtomicSDNode > > Considering the class hierarchy, we could safely reused bit 0~1 as the > target flags in AtomicSDNode/MemIntrinsicNode > > + X86 backend is modified to generate additional XACQUIRE/XRELEASE > prefix based on the specified target flag > > > The following are details of each patch: > > * 0001-Add-targetflags-in-AtomicSDNode-MemIntrinsicSDNode.patch > > This patch adds 'targetflags' support in AtomicSDNode and > MemIntrinsicSDNode. It will check metadata 'targetflags' and embedded > its value into SubclassData. Currently, only two bits are defined. > > * 0002-Add-HLE-target-feature.patch > > This patch adds HLE feature and auto-detection support > > * 0003-Add-XACQ-XREL-prefix-and-encoding-asm-printer-suppor.patch > > This patch adds XACQUIRE/XRELEASE prefix and its assembler/encoding > support > > * 0004-Enable-HLE-code-generation.patch > > This patch enables HLE code generation by extending the current logic to > handle 'targetflags'. > > * 0001-Add-target-flags-support-for-atomic-ops.patch > > This patch adds target flags support in __atomic_* builtins. It splits > the whole 32-bit order word into high and low 16-bit parts. The low > 16-bit is the original memory order and the high 16-bit will be > re-defined as target-specific flags and passed through 'targetflags' > metadata. > > * 0002-Add-mhle-option-support-and-populate-pre-defined-mac.patch > > It adds '-m[no]hle' option to turn on HLE feature or not. Once HLE > feature is turned on, two more macros (__ATOMIC_HLE_ACQUIRE and > __ATOMIC_HLE_RELEASE) are defined for developers to mark atomic > builtins. > > Thanks for your time to review! > > Yours > - Michael > --- > [1] http://gcc.gnu.org/ml/gcc-patches/2012-04/msg01073.html > [2] http://software.intel.com/sites/default/files/319433-014.pdf >
>From c2ed27488d773a6684e42adac9c61bff7f2badf8 Mon Sep 17 00:00:00 2001 From: Michael Liao <[email protected]> Date: Tue, 3 Jul 2012 23:28:17 -0700 Subject: [PATCH 1/4] Add targetflags in AtomicSDNode & MemIntrinsicSDNode - to pass HLE acquire/release hint to backend --- include/llvm/CodeGen/SelectionDAG.h | 22 ++++++----- include/llvm/CodeGen/SelectionDAGNodes.h | 34 ++++++++++++---- lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 2 + lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp | 14 +++++-- lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 44 +++++++++++---------- lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp | 23 +++++++++++ lib/Target/AArch64/AArch64ISelLowering.cpp | 2 + lib/Target/X86/X86ISelLowering.cpp | 22 +++++++---- 8 files changed, 114 insertions(+), 49 deletions(-) diff --git a/include/llvm/CodeGen/SelectionDAG.h b/include/llvm/CodeGen/SelectionDAG.h index c25497a..2ccda96 100644 --- a/include/llvm/CodeGen/SelectionDAG.h +++ b/include/llvm/CodeGen/SelectionDAG.h @@ -636,23 +636,24 @@ public: SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachinePointerInfo PtrInfo, unsigned Alignment, - AtomicOrdering Ordering, + AtomicOrdering Ordering, unsigned TargetFlags, SynchronizationScope SynchScope); SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachineMemOperand *MMO, - AtomicOrdering Ordering, + AtomicOrdering Ordering, unsigned TargetFlags, SynchronizationScope SynchScope); /// getAtomic - Gets a node for an atomic op, produces result (if relevant) /// and chain and takes 2 operands. SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, const Value* PtrVal, - unsigned Alignment, AtomicOrdering Ordering, + unsigned Alignment, + AtomicOrdering Ordering, unsigned TargetFlags, SynchronizationScope SynchScope); SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO, - AtomicOrdering Ordering, + AtomicOrdering Ordering, unsigned TargetFlags, SynchronizationScope SynchScope); /// getAtomic - Gets a node for an atomic op, produces result and chain and @@ -660,11 +661,11 @@ public: SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, EVT VT, SDValue Chain, SDValue Ptr, const Value* PtrVal, unsigned Alignment, - AtomicOrdering Ordering, + AtomicOrdering Ordering, unsigned TargetFlags, SynchronizationScope SynchScope); SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, EVT VT, SDValue Chain, SDValue Ptr, MachineMemOperand *MMO, - AtomicOrdering Ordering, + AtomicOrdering Ordering, unsigned TargetFlags, SynchronizationScope SynchScope); /// getMemIntrinsicNode - Creates a MemIntrinsicNode that may produce a @@ -676,17 +677,20 @@ public: const SDValue *Ops, unsigned NumOps, EVT MemVT, MachinePointerInfo PtrInfo, unsigned Align = 0, bool Vol = false, - bool ReadMem = true, bool WriteMem = true); + bool ReadMem = true, bool WriteMem = true, + unsigned TargetFlags = 0); SDValue getMemIntrinsicNode(unsigned Opcode, DebugLoc dl, SDVTList VTList, const SDValue *Ops, unsigned NumOps, EVT MemVT, MachinePointerInfo PtrInfo, unsigned Align = 0, bool Vol = false, - bool ReadMem = true, bool WriteMem = true); + bool ReadMem = true, bool WriteMem = true, + unsigned TargetFlags = 0); SDValue getMemIntrinsicNode(unsigned Opcode, DebugLoc dl, SDVTList VTList, const SDValue *Ops, unsigned NumOps, - EVT MemVT, MachineMemOperand *MMO); + EVT MemVT, MachineMemOperand *MMO, + unsigned TargetFlags = 0); /// getMergeValues - Create a MERGE_VALUES node from the given operands. SDValue getMergeValues(const SDValue *Ops, unsigned NumOps, DebugLoc dl); diff --git a/include/llvm/CodeGen/SelectionDAGNodes.h b/include/llvm/CodeGen/SelectionDAGNodes.h index 2c34b4f..8e88834 100644 --- a/include/llvm/CodeGen/SelectionDAGNodes.h +++ b/include/llvm/CodeGen/SelectionDAGNodes.h @@ -1013,15 +1013,20 @@ public: class AtomicSDNode : public MemSDNode { SDUse Ops[4]; - void InitAtomic(AtomicOrdering Ordering, SynchronizationScope SynchScope) { + void InitAtomic(AtomicOrdering Ordering, unsigned TargetFlags, + SynchronizationScope SynchScope) { // This must match encodeMemSDNodeFlags() in SelectionDAG.cpp. assert((Ordering & 15) == Ordering && "Ordering may not require more than 4 bits!"); + assert((TargetFlags & 3) == TargetFlags && + "TargetFlags may not require more than 2 bits!"); assert((SynchScope & 1) == SynchScope && "SynchScope may not require more than 1 bit!"); SubclassData |= Ordering << 8; + SubclassData |= TargetFlags; SubclassData |= SynchScope << 12; assert(getOrdering() == Ordering && "Ordering encoding error!"); + assert(getTargetFlags() == TargetFlags && "TargetFlags encoding error!"); assert(getSynchScope() == SynchScope && "Synch-scope encoding error!"); } @@ -1037,28 +1042,34 @@ public: AtomicSDNode(unsigned Opc, DebugLoc dl, SDVTList VTL, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachineMemOperand *MMO, - AtomicOrdering Ordering, SynchronizationScope SynchScope) + AtomicOrdering Ordering, unsigned TargetFlags, + SynchronizationScope SynchScope) : MemSDNode(Opc, dl, VTL, MemVT, MMO) { - InitAtomic(Ordering, SynchScope); + InitAtomic(Ordering, TargetFlags, SynchScope); InitOperands(Ops, Chain, Ptr, Cmp, Swp); } AtomicSDNode(unsigned Opc, DebugLoc dl, SDVTList VTL, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO, - AtomicOrdering Ordering, SynchronizationScope SynchScope) + AtomicOrdering Ordering, unsigned TargetFlags, + SynchronizationScope SynchScope) : MemSDNode(Opc, dl, VTL, MemVT, MMO) { - InitAtomic(Ordering, SynchScope); + InitAtomic(Ordering, TargetFlags, SynchScope); InitOperands(Ops, Chain, Ptr, Val); } AtomicSDNode(unsigned Opc, DebugLoc dl, SDVTList VTL, EVT MemVT, SDValue Chain, SDValue Ptr, MachineMemOperand *MMO, - AtomicOrdering Ordering, SynchronizationScope SynchScope) + AtomicOrdering Ordering, unsigned TargetFlags, + SynchronizationScope SynchScope) : MemSDNode(Opc, dl, VTL, MemVT, MMO) { - InitAtomic(Ordering, SynchScope); + InitAtomic(Ordering, TargetFlags, SynchScope); InitOperands(Ops, Chain, Ptr); } + /// getTargetFlags - Return target-specific flags. + unsigned getTargetFlags() const { return SubclassData & 3; } + const SDValue &getBasePtr() const { return getOperand(1); } const SDValue &getVal() const { return getOperand(2); } @@ -1094,10 +1105,17 @@ class MemIntrinsicSDNode : public MemSDNode { public: MemIntrinsicSDNode(unsigned Opc, DebugLoc dl, SDVTList VTs, const SDValue *Ops, unsigned NumOps, - EVT MemoryVT, MachineMemOperand *MMO) + EVT MemoryVT, MachineMemOperand *MMO, + unsigned TargetFlags = 0) : MemSDNode(Opc, dl, VTs, Ops, NumOps, MemoryVT, MMO) { + assert((TargetFlags & 3) == TargetFlags && + "TargetFlags may not require more than 2 bits!"); + SubclassData |= TargetFlags; } + /// getTargetFlags - Return target-specific flags. + unsigned getTargetFlags() const { return SubclassData & 3; } + // Methods to support isa and dyn_cast static bool classof(const SDNode *N) { // We lower some target intrinsics to their target opcode diff --git a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp index a9d40d0..18c1d16 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp @@ -2799,6 +2799,7 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node) { Node->getOperand(1), Zero, Zero, cast<AtomicSDNode>(Node)->getMemOperand(), cast<AtomicSDNode>(Node)->getOrdering(), + cast<AtomicSDNode>(Node)->getTargetFlags(), cast<AtomicSDNode>(Node)->getSynchScope()); Results.push_back(Swap.getValue(0)); Results.push_back(Swap.getValue(1)); @@ -2812,6 +2813,7 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node) { Node->getOperand(1), Node->getOperand(2), cast<AtomicSDNode>(Node)->getMemOperand(), cast<AtomicSDNode>(Node)->getOrdering(), + cast<AtomicSDNode>(Node)->getTargetFlags(), cast<AtomicSDNode>(Node)->getSynchScope()); Results.push_back(Swap.getValue(1)); break; diff --git a/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp index 182b7f3..a648940 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp @@ -169,7 +169,8 @@ SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) { SDValue Res = DAG.getAtomic(N->getOpcode(), N->getDebugLoc(), N->getMemoryVT(), ResVT, N->getChain(), N->getBasePtr(), - N->getMemOperand(), N->getOrdering(), + N->getMemOperand(), + N->getOrdering(), N->getTargetFlags(), N->getSynchScope()); // Legalized the chain result - switch anything that used the old chain to // use the new one. @@ -182,7 +183,8 @@ SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) { SDValue Res = DAG.getAtomic(N->getOpcode(), N->getDebugLoc(), N->getMemoryVT(), N->getChain(), N->getBasePtr(), - Op2, N->getMemOperand(), N->getOrdering(), + Op2, N->getMemOperand(), + N->getOrdering(), N->getTargetFlags(), N->getSynchScope()); // Legalized the chain result - switch anything that used the old chain to // use the new one. @@ -195,7 +197,8 @@ SDValue DAGTypeLegalizer::PromoteIntRes_Atomic2(AtomicSDNode *N) { SDValue Op3 = GetPromotedInteger(N->getOperand(3)); SDValue Res = DAG.getAtomic(N->getOpcode(), N->getDebugLoc(), N->getMemoryVT(), N->getChain(), N->getBasePtr(), - Op2, Op3, N->getMemOperand(), N->getOrdering(), + Op2, Op3, N->getMemOperand(), + N->getOrdering(), N->getTargetFlags(), N->getSynchScope()); // Legalized the chain result - switch anything that used the old chain to // use the new one. @@ -853,7 +856,8 @@ SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) { SDValue Op2 = GetPromotedInteger(N->getOperand(2)); return DAG.getAtomic(N->getOpcode(), N->getDebugLoc(), N->getMemoryVT(), N->getChain(), N->getBasePtr(), Op2, N->getMemOperand(), - N->getOrdering(), N->getSynchScope()); + N->getOrdering(), N->getTargetFlags(), + N->getSynchScope()); } SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) { @@ -2435,6 +2439,7 @@ void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N, N->getOperand(1), Zero, Zero, cast<AtomicSDNode>(N)->getMemOperand(), cast<AtomicSDNode>(N)->getOrdering(), + cast<AtomicSDNode>(N)->getTargetFlags(), cast<AtomicSDNode>(N)->getSynchScope()); ReplaceValueWith(SDValue(N, 0), Swap.getValue(0)); ReplaceValueWith(SDValue(N, 1), Swap.getValue(1)); @@ -2859,6 +2864,7 @@ SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) { N->getOperand(1), N->getOperand(2), cast<AtomicSDNode>(N)->getMemOperand(), cast<AtomicSDNode>(N)->getOrdering(), + cast<AtomicSDNode>(N)->getTargetFlags(), cast<AtomicSDNode>(N)->getSynchScope()); return Swap.getValue(1); } diff --git a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index 09885d8..68f417b 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -4061,7 +4061,7 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachinePointerInfo PtrInfo, unsigned Alignment, - AtomicOrdering Ordering, + AtomicOrdering Ordering, unsigned TargetFlags, SynchronizationScope SynchScope) { if (Alignment == 0) // Ensure that codegen never sees alignment 0 Alignment = getEVTAlignment(MemVT); @@ -4082,14 +4082,14 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment); return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Cmp, Swp, MMO, - Ordering, SynchScope); + Ordering, TargetFlags, SynchScope); } SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachineMemOperand *MMO, - AtomicOrdering Ordering, + AtomicOrdering Ordering, unsigned TargetFlags, SynchronizationScope SynchScope) { assert(Opcode == ISD::ATOMIC_CMP_SWAP && "Invalid Atomic Op"); assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types"); @@ -4109,7 +4109,7 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, } SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl, VTs, MemVT, Chain, Ptr, Cmp, Swp, MMO, Ordering, - SynchScope); + TargetFlags, SynchScope); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); return SDValue(N, 0); @@ -4120,7 +4120,7 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Ptr, SDValue Val, const Value* PtrVal, unsigned Alignment, - AtomicOrdering Ordering, + AtomicOrdering Ordering, unsigned TargetFlags, SynchronizationScope SynchScope) { if (Alignment == 0) // Ensure that codegen never sees alignment 0 Alignment = getEVTAlignment(MemVT); @@ -4143,14 +4143,14 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, MemVT.getStoreSize(), Alignment); return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Val, MMO, - Ordering, SynchScope); + Ordering, TargetFlags, SynchScope); } SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO, - AtomicOrdering Ordering, + AtomicOrdering Ordering, unsigned TargetFlags, SynchronizationScope SynchScope) { assert((Opcode == ISD::ATOMIC_LOAD_ADD || Opcode == ISD::ATOMIC_LOAD_SUB || @@ -4181,8 +4181,8 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, return SDValue(E, 0); } SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl, VTs, MemVT, Chain, - Ptr, Val, MMO, - Ordering, SynchScope); + Ptr, Val, MMO, Ordering, + TargetFlags, SynchScope); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); return SDValue(N, 0); @@ -4193,7 +4193,7 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Ptr, const Value* PtrVal, unsigned Alignment, - AtomicOrdering Ordering, + AtomicOrdering Ordering, unsigned TargetFlags, SynchronizationScope SynchScope) { if (Alignment == 0) // Ensure that codegen never sees alignment 0 Alignment = getEVTAlignment(MemVT); @@ -4216,14 +4216,14 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, MemVT.getStoreSize(), Alignment); return getAtomic(Opcode, dl, MemVT, VT, Chain, Ptr, MMO, - Ordering, SynchScope); + Ordering, TargetFlags, SynchScope); } SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, EVT VT, SDValue Chain, SDValue Ptr, MachineMemOperand *MMO, - AtomicOrdering Ordering, + AtomicOrdering Ordering, unsigned TargetFlags, SynchronizationScope SynchScope) { assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op"); @@ -4239,7 +4239,8 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, return SDValue(E, 0); } SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl, VTs, MemVT, Chain, - Ptr, MMO, Ordering, SynchScope); + Ptr, MMO, Ordering, TargetFlags, + SynchScope); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); return SDValue(N, 0); @@ -4265,10 +4266,11 @@ SelectionDAG::getMemIntrinsicNode(unsigned Opcode, DebugLoc dl, const SDValue *Ops, unsigned NumOps, EVT MemVT, MachinePointerInfo PtrInfo, unsigned Align, bool Vol, - bool ReadMem, bool WriteMem) { + bool ReadMem, bool WriteMem, + unsigned TargetFlags) { return getMemIntrinsicNode(Opcode, dl, makeVTList(VTs, NumVTs), Ops, NumOps, MemVT, PtrInfo, Align, Vol, - ReadMem, WriteMem); + ReadMem, WriteMem, TargetFlags); } SDValue @@ -4276,7 +4278,8 @@ SelectionDAG::getMemIntrinsicNode(unsigned Opcode, DebugLoc dl, SDVTList VTList, const SDValue *Ops, unsigned NumOps, EVT MemVT, MachinePointerInfo PtrInfo, unsigned Align, bool Vol, - bool ReadMem, bool WriteMem) { + bool ReadMem, bool WriteMem, + unsigned TargetFlags) { if (Align == 0) // Ensure that codegen never sees alignment 0 Align = getEVTAlignment(MemVT); @@ -4291,13 +4294,14 @@ SelectionDAG::getMemIntrinsicNode(unsigned Opcode, DebugLoc dl, SDVTList VTList, MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Align); - return getMemIntrinsicNode(Opcode, dl, VTList, Ops, NumOps, MemVT, MMO); + return getMemIntrinsicNode(Opcode, dl, VTList, Ops, NumOps, MemVT, MMO, TargetFlags); } SDValue SelectionDAG::getMemIntrinsicNode(unsigned Opcode, DebugLoc dl, SDVTList VTList, const SDValue *Ops, unsigned NumOps, - EVT MemVT, MachineMemOperand *MMO) { + EVT MemVT, MachineMemOperand *MMO, + unsigned TargetFlags) { assert((Opcode == ISD::INTRINSIC_VOID || Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::PREFETCH || @@ -4320,11 +4324,11 @@ SelectionDAG::getMemIntrinsicNode(unsigned Opcode, DebugLoc dl, SDVTList VTList, } N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl, VTList, Ops, NumOps, - MemVT, MMO); + MemVT, MMO, TargetFlags); CSEMap.InsertNode(N, IP); } else { N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl, VTList, Ops, NumOps, - MemVT, MMO); + MemVT, MMO, TargetFlags); } AllNodes.push_back(N); return SDValue(N, 0); diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 3a55696..0aa4be5 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -3423,9 +3423,25 @@ static SDValue InsertFenceForAtomic(SDValue Chain, AtomicOrdering Order, return DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops, 3); } +static unsigned GetAtomicTargetFlags(const Instruction &I) { + const MDNode* TargetFlagsInfo = I.getMetadata("targetflags"); + + if (!TargetFlagsInfo) + return 0; + + assert((TargetFlagsInfo->getNumOperands() > 0) && + "'targetflags' requires 1 operand!"); + const ConstantInt *CI = + dyn_cast<ConstantInt>(TargetFlagsInfo->getOperand(0)); + assert(CI && "'targetflags' not a constant integer!"); + + return CI->getZExtValue(); +} + void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) { DebugLoc dl = getCurDebugLoc(); AtomicOrdering Order = I.getOrdering(); + unsigned TargetFlags = GetAtomicTargetFlags(I); SynchronizationScope Scope = I.getSynchScope(); SDValue InChain = getRoot(); @@ -3443,6 +3459,7 @@ void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) { getValue(I.getNewValOperand()), MachinePointerInfo(I.getPointerOperand()), 0 /* Alignment */, TLI.getInsertFencesForAtomic() ? Monotonic : Order, + TargetFlags, Scope); SDValue OutChain = L.getValue(1); @@ -3473,6 +3490,7 @@ void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) { case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break; } AtomicOrdering Order = I.getOrdering(); + unsigned TargetFlags = GetAtomicTargetFlags(I); SynchronizationScope Scope = I.getSynchScope(); SDValue InChain = getRoot(); @@ -3489,6 +3507,7 @@ void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) { getValue(I.getValOperand()), I.getPointerOperand(), 0 /* Alignment */, TLI.getInsertFencesForAtomic() ? Monotonic : Order, + TargetFlags, Scope); SDValue OutChain = L.getValue(1); @@ -3513,6 +3532,7 @@ void SelectionDAGBuilder::visitFence(const FenceInst &I) { void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) { DebugLoc dl = getCurDebugLoc(); AtomicOrdering Order = I.getOrdering(); + unsigned TargetFlags = GetAtomicTargetFlags(I); SynchronizationScope Scope = I.getSynchScope(); SDValue InChain = getRoot(); @@ -3527,6 +3547,7 @@ void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) { getValue(I.getPointerOperand()), I.getPointerOperand(), I.getAlignment(), TLI.getInsertFencesForAtomic() ? Monotonic : Order, + TargetFlags, Scope); SDValue OutChain = L.getValue(1); @@ -3543,6 +3564,7 @@ void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) { DebugLoc dl = getCurDebugLoc(); AtomicOrdering Order = I.getOrdering(); + unsigned TargetFlags = GetAtomicTargetFlags(I); SynchronizationScope Scope = I.getSynchScope(); SDValue InChain = getRoot(); @@ -3563,6 +3585,7 @@ void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) { getValue(I.getValueOperand()), I.getPointerOperand(), I.getAlignment(), TLI.getInsertFencesForAtomic() ? Monotonic : Order, + TargetFlags, Scope); if (TLI.getInsertFencesForAtomic()) diff --git a/lib/Target/AArch64/AArch64ISelLowering.cpp b/lib/Target/AArch64/AArch64ISelLowering.cpp index cea7f91..66f6eec 100644 --- a/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -2414,6 +2414,7 @@ static SDValue PerformATOMIC_FENCECombine(SDNode *FenceNode, Chain, // Chain AtomicOp.getOperand(1), // Pointer AtomicNode->getMemOperand(), Acquire, + AtomicNode->getTargetFlags(), FenceScope); if (AtomicNode->getOpcode() == ISD::ATOMIC_LOAD) @@ -2447,6 +2448,7 @@ static SDValue PerformATOMIC_STORECombine(SDNode *N, AtomicNode->getOperand(1), // Pointer AtomicNode->getOperand(2), // Value AtomicNode->getMemOperand(), Release, + AtomicNode->getTargetFlags(), FenceScope); } diff --git a/lib/Target/X86/X86ISelLowering.cpp b/lib/Target/X86/X86ISelLowering.cpp index 9ed03cd..d525e3d 100644 --- a/lib/Target/X86/X86ISelLowering.cpp +++ b/lib/Target/X86/X86ISelLowering.cpp @@ -11926,9 +11926,10 @@ static SDValue LowerCMP_SWAP(SDValue Op, const X86Subtarget *Subtarget, DAG.getTargetConstant(size, MVT::i8), cpIn.getValue(1) }; SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue); - MachineMemOperand *MMO = cast<AtomicSDNode>(Op)->getMemOperand(); + const AtomicSDNode *AT = cast<AtomicSDNode>(Op); SDValue Result = DAG.getMemIntrinsicNode(X86ISD::LCMPXCHG_DAG, DL, Tys, - Ops, 5, T, MMO); + Ops, 5, T, AT->getMemOperand(), + AT->getTargetFlags()); SDValue cpOut = DAG.getCopyFromReg(Result.getValue(0), DL, Reg, T, Result.getValue(1)); return cpOut; @@ -11986,6 +11987,7 @@ static SDValue LowerLOAD_SUB(SDValue Op, SelectionDAG &DAG) { cast<AtomicSDNode>(Node)->getSrcValue(), cast<AtomicSDNode>(Node)->getAlignment(), cast<AtomicSDNode>(Node)->getOrdering(), + cast<AtomicSDNode>(Node)->getTargetFlags(), cast<AtomicSDNode>(Node)->getSynchScope()); } @@ -12007,6 +12009,7 @@ static SDValue LowerATOMIC_STORE(SDValue Op, SelectionDAG &DAG) { Node->getOperand(1), Node->getOperand(2), cast<AtomicSDNode>(Node)->getMemOperand(), cast<AtomicSDNode>(Node)->getOrdering(), + cast<AtomicSDNode>(Node)->getTargetFlags(), cast<AtomicSDNode>(Node)->getSynchScope()); return Swap.getValue(1); } @@ -12179,6 +12182,7 @@ static void ReplaceATOMIC_LOAD(SDNode *Node, Node->getOperand(1), Zero, Zero, cast<AtomicSDNode>(Node)->getMemOperand(), cast<AtomicSDNode>(Node)->getOrdering(), + cast<AtomicSDNode>(Node)->getTargetFlags(), cast<AtomicSDNode>(Node)->getSynchScope()); Results.push_back(Swap.getValue(0)); Results.push_back(Swap.getValue(1)); @@ -12199,9 +12203,10 @@ ReplaceATOMIC_BINARY_64(SDNode *Node, SmallVectorImpl<SDValue>&Results, Node->getOperand(2), DAG.getIntPtrConstant(1)); SDValue Ops[] = { Chain, In1, In2L, In2H }; SDVTList Tys = DAG.getVTList(MVT::i32, MVT::i32, MVT::Other); - SDValue Result = - DAG.getMemIntrinsicNode(NewOp, dl, Tys, Ops, 4, MVT::i64, - cast<MemSDNode>(Node)->getMemOperand()); + const AtomicSDNode *AT = cast<AtomicSDNode>(Node); + SDValue Result = DAG.getMemIntrinsicNode(NewOp, dl, Tys, Ops, 4, MVT::i64, + AT->getMemOperand(), + AT->getTargetFlags()); SDValue OpsF[] = { Result.getValue(0), Result.getValue(1)}; Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, OpsF, 2)); Results.push_back(Result.getValue(2)); @@ -12314,11 +12319,12 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N, N->getOperand(1), swapInH.getValue(1) }; SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue); - MachineMemOperand *MMO = cast<AtomicSDNode>(N)->getMemOperand(); + const AtomicSDNode *AT = cast<AtomicSDNode>(N); unsigned Opcode = Regs64bit ? X86ISD::LCMPXCHG16_DAG : X86ISD::LCMPXCHG8_DAG; - SDValue Result = DAG.getMemIntrinsicNode(Opcode, dl, Tys, - Ops, 3, T, MMO); + SDValue Result = DAG.getMemIntrinsicNode(Opcode, dl, Tys, Ops, 3, T, + AT->getMemOperand(), + AT->getTargetFlags()); SDValue cpOutL = DAG.getCopyFromReg(Result.getValue(0), dl, Regs64bit ? X86::RAX : X86::EAX, HalfT, Result.getValue(1)); -- 1.7.9.5
_______________________________________________ cfe-commits mailing list [email protected] http://lists.cs.uiuc.edu/mailman/listinfo/cfe-commits
