diff --git a/include/llvm/IntrinsicsARM.td b/include/llvm/IntrinsicsARM.td index 8bd2080..9c43f39 100644 --- a/include/llvm/IntrinsicsARM.td +++ b/include/llvm/IntrinsicsARM.td @@ -36,6 +36,16 @@ let TargetPrefix = "arm" in { // All intrinsics start with "llvm.arm.". } //===----------------------------------------------------------------------===// +// Load and Store exclusive doubleword + +let TargetPrefix = "arm" in { // All intrinsics start with "llvm.arm.". + def int_arm_strexd : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, + llvm_ptr_ty], [IntrReadWriteArgMem]>; + def int_arm_ldrexd : Intrinsic<[llvm_i32_ty, llvm_i32_ty], [llvm_ptr_ty], + [IntrReadArgMem]>; +} + +//===----------------------------------------------------------------------===// // VFP let TargetPrefix = "arm" in { // All intrinsics start with "llvm.arm.". diff --git a/lib/Target/ARM/ARMISelDAGToDAG.cpp b/lib/Target/ARM/ARMISelDAGToDAG.cpp index abe5a31..e272024 100644 --- a/lib/Target/ARM/ARMISelDAGToDAG.cpp +++ b/lib/Target/ARM/ARMISelDAGToDAG.cpp @@ -131,6 +131,7 @@ public: bool SelectAddrMode5(SDValue N, SDValue &Base, SDValue &Offset); bool SelectAddrMode6(SDNode *Parent, SDValue N, SDValue &Addr,SDValue &Align); + bool SelectAddrMode7(SDNode *Parent, SDValue N, SDValue &Addr); bool SelectAddrMode6Offset(SDNode *Op, SDValue N, SDValue &Offset); bool SelectAddrModePC(SDValue N, SDValue &Offset, SDValue &Label); @@ -859,6 +860,11 @@ bool ARMDAGToDAGISel::SelectAddrMode6(SDNode *Parent, SDValue N, SDValue &Addr, return true; } +bool ARMDAGToDAGISel::SelectAddrMode7(SDNode *Parent, SDValue N, SDValue &Addr){ + Addr = N; + return true; +} + bool ARMDAGToDAGISel::SelectAddrMode6Offset(SDNode *Op, SDValue N, SDValue &Offset) { LSBaseSDNode *LdSt = cast(Op); @@ -2691,6 +2697,42 @@ SDNode *ARMDAGToDAGISel::Select(SDNode *N) { default: break; + case Intrinsic::arm_ldrexd: { + SDValue MemAddr = N->getOperand(2); + DebugLoc dl = N->getDebugLoc(); + SDValue Chain = N->getOperand(0); + + unsigned NewOpc = ARM::LDREXD; + if (Subtarget->isThumb() && Subtarget->hasThumb2()) + NewOpc = ARM::t2LDREXD; + + // arm_ldrexd returns a i64 value in {i32, i32} + std::vector ResTys; + ResTys.push_back(MVT::i32); + ResTys.push_back(MVT::i32); + ResTys.push_back(MVT::Other); + + // place arguments in the right order + SmallVector Ops; + Ops.push_back(MemAddr); + Ops.push_back(getAL(CurDAG)); + Ops.push_back(CurDAG->getRegister(0, MVT::i32)); + Ops.push_back(Chain); + SDNode *Ld = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops.data(), + Ops.size()); + + // Transfer memoperands. + MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); + MemOp[0] = cast(N)->getMemOperand(); + cast(Ld)->setMemRefs(MemOp, MemOp + 1); + + // Remap uses. + ReplaceUses(SDValue(N, 0), SDValue(Ld, 0)); + ReplaceUses(SDValue(N, 1), SDValue(Ld, 1)); + ReplaceUses(SDValue(N, 2), SDValue(Ld, 2)); + return NULL; + } + case Intrinsic::arm_neon_vld1: { unsigned DOpcodes[] = { ARM::VLD1d8, ARM::VLD1d16, ARM::VLD1d32, ARM::VLD1d64 }; diff --git a/lib/Target/ARM/ARMISelLowering.cpp b/lib/Target/ARM/ARMISelLowering.cpp index 5c1cded..14434b6 100644 --- a/lib/Target/ARM/ARMISelLowering.cpp +++ b/lib/Target/ARM/ARMISelLowering.cpp @@ -7644,6 +7644,17 @@ bool ARMTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, Info.writeMem = true; return true; } + case Intrinsic::arm_ldrexd: { + Info.opc = ISD::INTRINSIC_W_CHAIN; + Info.memVT = MVT::i64; + Info.ptrVal = I.getArgOperand(0); + Info.offset = 0; + Info.align = 8; + Info.vol = false; + Info.readMem = true; + Info.writeMem = false; + return true; + } default: break; } diff --git a/lib/Target/ARM/ARMInstrInfo.td b/lib/Target/ARM/ARMInstrInfo.td index bfcafad..e42e74a 100644 --- a/lib/Target/ARM/ARMInstrInfo.td +++ b/lib/Target/ARM/ARMInstrInfo.td @@ -620,12 +620,12 @@ def MemMode7AsmOperand : AsmOperandClass { } // addrmode7 := reg -// Used by load/store exclusive instructions. Useful to enable right assembly -// parsing and printing. Not used for any codegen matching. +// Used by load/store exclusive instructions. // -def addrmode7 : Operand { +def addrmode7 : Operand, + ComplexPattern{ let PrintMethod = "printAddrMode7Operand"; - let MIOperandInfo = (ops GPR); + let MIOperandInfo = (ops GPR:$addr); let ParserMatchClass = MemMode7AsmOperand; } @@ -3371,8 +3371,9 @@ def LDREXH : AIldrex<0b11, (outs GPR:$Rt), (ins addrmode7:$addr), NoItinerary, "ldrexh", "\t$Rt, $addr", []>; def LDREX : AIldrex<0b00, (outs GPR:$Rt), (ins addrmode7:$addr), NoItinerary, "ldrex", "\t$Rt, $addr", []>; -def LDREXD : AIldrex<0b01, (outs GPR:$Rt, GPR:$Rt2), (ins addrmode7:$addr), - NoItinerary, "ldrexd", "\t$Rt, $Rt2, $addr", []>; +let hasExtraDefRegAllocReq = 1 in + def LDREXD : AIldrex<0b01, (outs GPR:$Rt, GPR:$Rt2), (ins addrmode7:$addr), + NoItinerary, "ldrexd", "\t$Rt, $Rt2, $addr", []>; } let mayStore = 1, Constraints = "@earlyclobber $Rd" in { @@ -3382,10 +3383,14 @@ def STREXH : AIstrex<0b11, (outs GPR:$Rd), (ins GPR:$Rt, addrmode7:$addr), NoItinerary, "strexh", "\t$Rd, $Rt, $addr", []>; def STREX : AIstrex<0b00, (outs GPR:$Rd), (ins GPR:$Rt, addrmode7:$addr), NoItinerary, "strex", "\t$Rd, $Rt, $addr", []>; +} + +let hasExtraSrcRegAllocReq = 1, Constraints = "@earlyclobber $Rd" in def STREXD : AIstrex<0b01, (outs GPR:$Rd), (ins GPR:$Rt, GPR:$Rt2, addrmode7:$addr), - NoItinerary, "strexd", "\t$Rd, $Rt, $Rt2, $addr", []>; -} + NoItinerary, "strexd", "\t$Rd, $Rt, $Rt2, $addr", + [(set GPR:$Rd, (int_arm_strexd GPR:$Rt, GPR:$Rt2, + addrmode7:$addr))]>; // Clear-Exclusive is for disassembly only. def CLREX : AXI<(outs), (ins), MiscFrm, NoItinerary, "clrex", diff --git a/lib/Target/ARM/ARMInstrThumb2.td b/lib/Target/ARM/ARMInstrThumb2.td index 08f7c92..59d9c3e 100644 --- a/lib/Target/ARM/ARMInstrThumb2.td +++ b/lib/Target/ARM/ARMInstrThumb2.td @@ -153,10 +153,10 @@ def t2addrmode_so_reg : Operand, } // t2addrmode_reg := reg -// Used by load/store exclusive instructions. Useful to enable right assembly -// parsing and printing. Not used for any codegen matching. +// Used by load/store exclusive instructions. // -def t2addrmode_reg : Operand { +def t2addrmode_reg : Operand, + ComplexPattern { let PrintMethod = "printAddrMode7Operand"; let MIOperandInfo = (ops GPR); let ParserMatchClass = MemMode7AsmOperand; @@ -2886,7 +2886,9 @@ def t2LDREX : Thumb2I<(outs rGPR:$Rt), (ins t2addrmode_reg:$addr), AddrModeNone let Inst{19-16} = addr; let Inst{15-12} = Rt; } -def t2LDREXD : T2I_ldrex<0b11, (outs rGPR:$Rt, rGPR:$Rt2), (ins t2addrmode_reg:$addr), +let hasExtraDefRegAllocReq = 1 in +def t2LDREXD : T2I_ldrex<0b11, (outs rGPR:$Rt, rGPR:$Rt2), + (ins t2addrmode_reg:$addr), AddrModeNone, Size4Bytes, NoItinerary, "ldrexd", "\t$Rt, $Rt2, $addr", "", [], {?, ?, ?, ?}> { @@ -2917,15 +2919,19 @@ def t2STREX : Thumb2I<(outs rGPR:$Rd), (ins rGPR:$Rt, t2addrmode_reg:$addr), let Inst{19-16} = addr; let Inst{15-12} = Rt; } +} + +let hasExtraSrcRegAllocReq = 1, Constraints = "@earlyclobber $Rd" in def t2STREXD : T2I_strex<0b11, (outs rGPR:$Rd), (ins rGPR:$Rt, rGPR:$Rt2, t2addrmode_reg:$addr), AddrModeNone, Size4Bytes, NoItinerary, - "strexd", "\t$Rd, $Rt, $Rt2, $addr", "", [], + "strexd", "\t$Rd, $Rt, $Rt2, $addr", "", + [(set rGPR:$Rd, (int_arm_strexd rGPR:$Rt, rGPR:$Rt2, + t2addrmode_reg:$addr))], {?, ?, ?, ?}> { bits<4> Rt2; let Inst{11-8} = Rt2; } -} // Clear-Exclusive is for disassembly only. def t2CLREX : T2XI<(outs), (ins), NoItinerary, "clrex", diff --git a/lib/Target/ARM/ARMLoadStoreOptimizer.cpp b/lib/Target/ARM/ARMLoadStoreOptimizer.cpp index 8b9a82e..40afb7a 100644 --- a/lib/Target/ARM/ARMLoadStoreOptimizer.cpp +++ b/lib/Target/ARM/ARMLoadStoreOptimizer.cpp @@ -111,6 +111,8 @@ namespace { SmallVector &Merges); void AdvanceRS(MachineBasicBlock &MBB, MemOpQueue &MemOps); + void FixInvalidLdStExclusiveRegPairOp(MachineBasicBlock &MBB, + MachineBasicBlock::iterator &MBBI); bool FixInvalidRegPairOp(MachineBasicBlock &MBB, MachineBasicBlock::iterator &MBBI); bool MergeBaseUpdateLoadStore(MachineBasicBlock &MBB, @@ -1062,10 +1064,102 @@ static void InsertLDR_STR(MachineBasicBlock &MBB, } } +void ARMLoadStoreOpt::FixInvalidLdStExclusiveRegPairOp(MachineBasicBlock &MBB, + MachineBasicBlock::iterator &MBBI) { + MachineInstr *MI = &*MBBI; + bool isLd = MI->getOpcode() == ARM::LDREXD; + + unsigned RegIdxStart = isLd ? 0 : 1; + unsigned EvenReg = MI->getOperand(RegIdxStart).getReg(); + unsigned OddReg = MI->getOperand(RegIdxStart+1).getReg(); + + // Check if the registers need to be fixed + unsigned EvenRegNum = TRI->getDwarfRegNum(EvenReg, false); + unsigned OddRegNum = TRI->getDwarfRegNum(OddReg, false); + if ((EvenRegNum & 1) == 0 && (EvenRegNum + 1) == OddRegNum) + return; + + DebugLoc DL = MBBI->getDebugLoc(); + MachineOperand &MOEven = MI->getOperand(RegIdxStart); + MachineOperand &MOOdd = MI->getOperand(RegIdxStart+1); + + // Keep track of the registers not used by the current MI. + BitVector UnusedRegs(TRI->getNumRegs()); + if (isLd) { + UnusedRegs.set(MI->getOperand(2).getReg()); // [$base] + } else { + UnusedRegs.set(MI->getOperand(0).getReg()); // [$Rd] + UnusedRegs.set(MI->getOperand(3).getReg()); // [$base] + } + UnusedRegs.flip(); + + // Use a very simple approach. Get a register pair $p0, $p1 where + // $p0 and $p1 are different from the $Rd and $base registers. + // Push $p0 and $p1 if needed + // Necessary copies if needed + // Exclusive instruction (LDREXD/STREXD) + // Necessary copies if needed + // Pop $p0 and $p1 if needed + // + unsigned CurReg = ARM::R0; + for (unsigned LastReg = ARM::R12; CurReg != LastReg; CurReg += 2) + if (UnusedRegs.test(CurReg) && UnusedRegs.test(CurReg+1)) + break; + + // Registers to be saved and copied to/from. + BitVector Available = RS->getRegsAvailable(ARM::GPRRegisterClass); + SmallVector, 2> RegsToCopy; + SmallVector RegsNeedRestore; + + if (CurReg != EvenReg) { + RegsToCopy.push_back(std::make_pair(CurReg, EvenReg)); + if (!Available.test(CurReg)) + RegsNeedRestore.push_back(CurReg); + } + if (CurReg+1 != OddReg) { + RegsToCopy.push_back(std::make_pair(CurReg+1, OddReg)); + if (!Available.test(CurReg+1)) + RegsNeedRestore.push_back(CurReg+1); + } + + // Push registers + MachineInstrBuilder MIBPush = + BuildMI(MBB, MBBI, DL, TII->get(ARM::STMDB_UPD), ARM::SP) + .addReg(ARM::SP).addImm((int64_t)ARMCC::AL).addReg(0); + for (int i = 0, e = RegsNeedRestore.size(); i < e; ++i) + MIBPush.addReg(RegsNeedRestore[i]); + + // Emit copies to/from the original used pair + for (int i = 0, e = RegsToCopy.size(); i < e; ++i) { + unsigned SrcReg = isLd ? RegsToCopy[i].first : RegsToCopy[i].second; + unsigned DstReg = isLd ? RegsToCopy[i].second : RegsToCopy[i].first; + MachineBasicBlock::iterator MBBIns = isLd ? llvm::next(MBBI) : MBBI; + BuildMI(MBB, MBBI, MBBI->getDebugLoc(), TII->get(ARM::MOVr), DstReg) + .addReg(SrcReg).addImm((unsigned)ARMCC::AL).addReg(0).addReg(0); + } + + // Pop registers + MachineInstrBuilder MIBPop = + BuildMI(MBB, llvm::next(MBBI), DL, TII->get(ARM::LDMIA_UPD), ARM::SP) + .addReg(ARM::SP).addImm((int64_t)ARMCC::AL).addReg(0); + for (int i = 0, e = RegsNeedRestore.size(); i < e; ++i) + MIBPop.addReg(RegsNeedRestore[i]); + + // Update MI with the current used pair + MOEven.setReg(CurReg); + MOOdd.setReg(CurReg+1); +} + bool ARMLoadStoreOpt::FixInvalidRegPairOp(MachineBasicBlock &MBB, MachineBasicBlock::iterator &MBBI) { MachineInstr *MI = &*MBBI; unsigned Opcode = MI->getOpcode(); + + if (Opcode == ARM::STREXD || Opcode == ARM::LDREXD) { + FixInvalidLdStExclusiveRegPairOp(MBB, MBBI); + return false; + } + if (Opcode == ARM::LDRD || Opcode == ARM::STRD || Opcode == ARM::t2LDRDi8 || Opcode == ARM::t2STRDi8) { unsigned EvenReg = MI->getOperand(0).getReg(); @@ -1428,6 +1522,7 @@ namespace { unsigned Base, bool isLd, DenseMap &MI2LocMap); bool RescheduleLoadStoreInstrs(MachineBasicBlock *MBB); + void CreateAllocHintForLdStExclusive(MachineInstr *MI); }; char ARMPreAllocLoadStoreOpt::ID = 0; } @@ -1448,6 +1543,7 @@ bool ARMPreAllocLoadStoreOpt::runOnMachineFunction(MachineFunction &Fn) { return Modified; } + static bool IsSafeAndProfitableToMove(bool isLd, unsigned Base, MachineBasicBlock::iterator I, MachineBasicBlock::iterator E, @@ -1495,6 +1591,18 @@ static bool IsSafeAndProfitableToMove(bool isLd, unsigned Base, return AddedRegPressure.size() <= MemRegs.size() * 2; } +void +ARMPreAllocLoadStoreOpt::CreateAllocHintForLdStExclusive(MachineInstr *MI) { + unsigned RegIdxStart = MI->getOpcode() == ARM::STREXD ? 1 : 0; + + unsigned EvenReg = MI->getOperand(RegIdxStart).getReg(); + unsigned OddReg = MI->getOperand(RegIdxStart+1).getReg(); + + // Add register allocation hints to form register pairs. + MRI->setRegAllocationHint(EvenReg, ARMRI::RegPairEven, OddReg); + MRI->setRegAllocationHint(OddReg, ARMRI::RegPairOdd, EvenReg); +} + bool ARMPreAllocLoadStoreOpt::CanFormLdStDWord(MachineInstr *Op0, MachineInstr *Op1, DebugLoc &dl, @@ -1743,6 +1851,7 @@ ARMPreAllocLoadStoreOpt::RescheduleLoadStoreInstrs(MachineBasicBlock *MBB) { for (; MBBI != E; ++MBBI) { MachineInstr *MI = MBBI; const TargetInstrDesc &TID = MI->getDesc(); + int Opc = MI->getOpcode(); if (TID.isCall() || TID.isTerminator()) { // Stop at barriers. ++MBBI; @@ -1751,14 +1860,20 @@ ARMPreAllocLoadStoreOpt::RescheduleLoadStoreInstrs(MachineBasicBlock *MBB) { if (!MI->isDebugValue()) MI2LocMap[MI] = ++Loc; - + // Load and Store doubleword exclusive instructions also need a + // pre-regalloc hint because both use pairs of GPR registers to + // load and store 64-bit data. Since they are already in the + // "double" form, they don't need to be rescheduled or re-written + if (Opc == ARM::STREXD || Opc == ARM::LDREXD) { + CreateAllocHintForLdStExclusive(MI); + continue; + } if (!isMemoryOp(MI)) continue; unsigned PredReg = 0; if (llvm::getInstrPredicate(MI, PredReg) != ARMCC::AL) continue; - int Opc = MI->getOpcode(); bool isLd = isi32Load(Opc) || Opc == ARM::VLDRS || Opc == ARM::VLDRD; unsigned Base = MI->getOperand(1).getReg(); int Offset = getMemoryOpOffset(MI); diff --git a/test/CodeGen/ARM/ldstrexd.ll b/test/CodeGen/ARM/ldstrexd.ll new file mode 100644 index 0000000..b0683cf --- /dev/null +++ b/test/CodeGen/ARM/ldstrexd.ll @@ -0,0 +1,53 @@ +; RUN: llc < %s -mtriple=armv7-apple-darwin | FileCheck %s -check-prefix=ARM +; RUN: llc < %s -mtriple=thumbv7-apple-darwin | FileCheck %s -check-prefix=T2 + +%0 = type { i32, i32 } + +; ARM: f0: +; ARM: ldrexd +; T2: f0: +; T2: ldrexd +define i64 @f0(i8* %p) nounwind readonly { +entry: + %ldrexd = tail call %0 @llvm.arm.ldrexd(i8* %p) + %0 = extractvalue %0 %ldrexd, 1 + %1 = extractvalue %0 %ldrexd, 0 + %2 = zext i32 %0 to i64 + %3 = zext i32 %1 to i64 + %shl = shl nuw i64 %2, 32 + %4 = or i64 %shl, %3 + ret i64 %4 +} + +; ARM: f1: +; ARM: strexd +; T2: f1: +; T2: strexd +define i32 @f1(i8* %ptr, i64 %val) nounwind { +entry: + %tmp4 = trunc i64 %val to i32 + %tmp6 = lshr i64 %val, 32 + %tmp7 = trunc i64 %tmp6 to i32 + %strexd = tail call i32 @llvm.arm.strexd(i32 %tmp4, i32 %tmp7, i8* %ptr) + ret i32 %strexd +} + +; ARM: f1_hint_fail: +; ARM: push +; ARM: push +; ARM: strexd +; ARM: pop +; ARM: pop +define i32 @f1_hint_fail(i64 %val, i64* %ptr) nounwind { +entry: + %tmp4.i = trunc i64 %val to i32 + %tmp6.i = lshr i64 %val, 32 + %tmp7.i = trunc i64 %tmp6.i to i32 + %0 = bitcast i64* %ptr to i8* + %strexd.i = tail call i32 @llvm.arm.strexd(i32 %tmp4.i, i32 %tmp7.i, i8* %0) nounwind + ret i32 %strexd.i +} + +declare %0 @llvm.arm.ldrexd(i8*) nounwind readonly +declare i32 @llvm.arm.strexd(i32, i32, i8*) nounwind +