Modified: trunk/Source/_javascript_Core/bytecode/CallLinkInfo.cpp (277837 => 277838)
--- trunk/Source/_javascript_Core/bytecode/CallLinkInfo.cpp 2021-05-20 23:39:30 UTC (rev 277837)
+++ trunk/Source/_javascript_Core/bytecode/CallLinkInfo.cpp 2021-05-21 00:14:50 UTC (rev 277838)
@@ -131,9 +131,12 @@
return m_doneLocation;
}
+static constexpr uintptr_t polymorphicCalleeMask = 1;
+
void CallLinkInfo::setMonomorphicCallee(VM& vm, JSCell* owner, JSObject* callee, MacroAssemblerCodePtr<JSEntryPtrTag> codePtr)
{
RELEASE_ASSERT(!isDirect());
+ RELEASE_ASSERT(!(bitwise_cast<uintptr_t>(callee) & polymorphicCalleeMask));
m_calleeOrCodeBlock.set(vm, owner, callee);
if (isDataIC())
@@ -157,6 +160,7 @@
JSObject* CallLinkInfo::callee()
{
RELEASE_ASSERT(!isDirect());
+ RELEASE_ASSERT(!(bitwise_cast<uintptr_t>(m_calleeOrCodeBlock.get()) & polymorphicCalleeMask));
return jsCast<JSObject*>(m_calleeOrCodeBlock.get());
}
@@ -250,18 +254,19 @@
pointerDump(codeBlock()), ").\n");
}
} else {
- if (callee()->type() == JSFunctionType) {
+ JSObject* callee = jsCast<JSObject*>(m_calleeOrCodeBlock.get());
+ if (callee->type() == JSFunctionType) {
if (UNLIKELY(Options::verboseOSR())) {
dataLog(
"Clearing call to ",
- RawPointer(callee()), " (",
- static_cast<JSFunction*>(callee())->executable()->hashFor(specializationKind()),
+ RawPointer(callee), " (",
+ static_cast<JSFunction*>(callee)->executable()->hashFor(specializationKind()),
").\n");
}
- handleSpecificCallee(static_cast<JSFunction*>(callee()));
+ handleSpecificCallee(static_cast<JSFunction*>(callee));
} else {
if (UNLIKELY(Options::verboseOSR()))
- dataLog("Clearing call to ", RawPointer(callee()), ".\n");
+ dataLog("Clearing call to ", RawPointer(callee), ".\n");
m_clearedByGC = true;
}
}
@@ -293,18 +298,6 @@
m_frameShuffleData->shrinkToFit();
}
-void CallLinkInfo::emitFirstInstructionForDataIC(CCallHelpers& jit, GPRReg callLinkInfoGPR)
-{
- GPRReg scratchGPR = jit.scratchRegister();
- DisallowMacroScratchRegisterUsage disallowScratch(jit);
- // When repatching, we just overwrite the first instruction back, since it'll have been replaced with a jump to the polymorphic call stub.
- size_t startSize = jit.m_assembler.buffer().codeSize();
- jit.loadPtr(CCallHelpers::Address(callLinkInfoGPR, offsetOfCallee()), scratchGPR);
- size_t loadSize = jit.m_assembler.buffer().codeSize() - startSize;
- if (loadSize < static_cast<size_t>(CCallHelpers::patchableJumpSize()))
- jit.emitNops(static_cast<size_t>(CCallHelpers::patchableJumpSize()) - loadSize);
-}
-
MacroAssembler::JumpList CallLinkInfo::emitFastPathImpl(CCallHelpers& jit, GPRReg calleeGPR, GPRReg callLinkInfoGPR, UseDataIC useDataIC, WTF::Function<void()> prepareForTailCall)
{
setUsesDataICs(useDataIC);
@@ -323,16 +316,21 @@
if (isDataIC()) {
GPRReg scratchGPR = jit.scratchRegister();
- emitFirstInstructionForDataIC(jit, callLinkInfoGPR);
+ jit.loadPtr(CCallHelpers::Address(callLinkInfoGPR, offsetOfCallee()), scratchGPR);
+ CCallHelpers::Jump goPolymorphic;
{
DisallowMacroScratchRegisterUsage disallowScratch(jit);
+ goPolymorphic = jit.branchTestPtr(CCallHelpers::NonZero, scratchGPR, CCallHelpers::TrustedImm32(polymorphicCalleeMask));
slowPath.append(jit.branchPtr(CCallHelpers::NotEqual, scratchGPR, calleeGPR));
}
if (isTailCall()) {
prepareForTailCall();
+ goPolymorphic.link(&jit); // Polymorphic stub handles tail call stack prep.
jit.farJump(CCallHelpers::Address(callLinkInfoGPR, offsetOfMonomorphicCallDestination()), JSEntryPtrTag);
- } else
+ } else {
+ goPolymorphic.link(&jit);
jit.call(CCallHelpers::Address(callLinkInfoGPR, offsetOfMonomorphicCallDestination()), JSEntryPtrTag);
+ }
} else {
CCallHelpers::DataLabelPtr calleeCheck;
slowPath.append(jit.branchPtrWithPatch(CCallHelpers::NotEqual, calleeGPR, calleeCheck, CCallHelpers::TrustedImmPtr(nullptr)));
@@ -464,14 +462,13 @@
// what in all likelihood fits in 24. So we just splat out the first instruction. Long term, we
// need something cleaner. But this works on arm64 for now.
- CCallHelpers::emitJITCodeOver(fastPathStart(), [&] (CCallHelpers& jit) {
- if (isDataIC())
- emitFirstInstructionForDataIC(jit, u.dataIC.m_callLinkInfoGPR);
- else {
- CCallHelpers::revertJumpReplacementToBranchPtrWithPatch(
- CCallHelpers::startOfBranchPtrWithPatchOnRegister(u.codeIC.m_calleeLocation), calleeGPR(), nullptr);
- }
- }, "Resetting PolymorphicCall stubbed CallLinkInfo");
+ if (isDataIC()) {
+ m_calleeOrCodeBlock.clear();
+ u.dataIC.m_monomorphicCallDestination = nullptr;
+ } else {
+ CCallHelpers::revertJumpReplacementToBranchPtrWithPatch(
+ CCallHelpers::startOfBranchPtrWithPatchOnRegister(u.codeIC.m_calleeLocation), calleeGPR(), nullptr);
+ }
}
void CallLinkInfo::setStub(Ref<PolymorphicCallStubRoutine>&& newStub)
@@ -479,13 +476,11 @@
clearStub();
m_stub = WTFMove(newStub);
+ m_calleeOrCodeBlock.clear();
+
if (isDataIC()) {
- CCallHelpers::emitJITCodeOver(fastPathStart(), [&] (CCallHelpers& jit) {
- auto jump = jit.jump();
- jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
- linkBuffer.link(jump, CodeLocationLabel<JITStubRoutinePtrTag>(m_stub->code().code()));
- });
- }, "Patching call to PolymorphicCallStubRoutine in CallLinkInfo");
+ *bitwise_cast<uintptr_t*>(m_calleeOrCodeBlock.slot()) = polymorphicCalleeMask;
+ u.dataIC.m_monomorphicCallDestination = m_stub->code().code().retagged<JSEntryPtrTag>();
} else {
MacroAssembler::replaceWithJump(
MacroAssembler::startOfBranchPtrWithPatchOnRegister(u.codeIC.m_calleeLocation),
Modified: trunk/Source/_javascript_Core/bytecode/CallLinkInfo.h (277837 => 277838)
--- trunk/Source/_javascript_Core/bytecode/CallLinkInfo.h 2021-05-20 23:39:30 UTC (rev 277837)
+++ trunk/Source/_javascript_Core/bytecode/CallLinkInfo.h 2021-05-21 00:14:50 UTC (rev 277838)
@@ -173,7 +173,6 @@
};
private:
- void emitFirstInstructionForDataIC(CCallHelpers&, GPRReg callLinkInfoGPR);
MacroAssembler::JumpList emitFastPathImpl(CCallHelpers&, GPRReg calleeGPR, GPRReg callLinkInfoGPR, UseDataIC, WTF::Function<void()> prepareForTailCall) WARN_UNUSED_RETURN;
public:
MacroAssembler::JumpList emitFastPath(CCallHelpers&, GPRReg calleeGPR, GPRReg callLinkInfoGPR, UseDataIC) WARN_UNUSED_RETURN;
Modified: trunk/Source/_javascript_Core/jit/Repatch.cpp (277837 => 277838)
--- trunk/Source/_javascript_Core/jit/Repatch.cpp 2021-05-20 23:39:30 UTC (rev 277837)
+++ trunk/Source/_javascript_Core/jit/Repatch.cpp 2021-05-21 00:14:50 UTC (rev 277838)
@@ -1456,6 +1456,7 @@
GPRReg calleeGPR = callLinkInfo.calleeGPR();
+ bool isDataIC = callLinkInfo.isDataIC();
CCallHelpers stubJit(callerCodeBlock);
std::unique_ptr<CallFrameShuffler> frameShuffler;
@@ -1538,6 +1539,8 @@
CCallHelpers::TrustedImm32(1),
CCallHelpers::Address(fastCountsBaseGPR, caseIndex * sizeof(uint32_t)));
}
+
+ bool needsDoneJump = false;
if (frameShuffler) {
CallFrameShuffler(stubJit, frameShuffler->snapshot()).prepareForTailCall();
calls[caseIndex].call = stubJit.nearTailCall();
@@ -1544,10 +1547,17 @@
} else if (callLinkInfo.isTailCall()) {
stubJit.prepareForTailCallSlow();
calls[caseIndex].call = stubJit.nearTailCall();
- } else
- calls[caseIndex].call = stubJit.nearCall();
+ } else {
+ if (isDataIC)
+ calls[caseIndex].call = stubJit.nearTailCall();
+ else {
+ calls[caseIndex].call = stubJit.nearCall();
+ needsDoneJump = true;
+ }
+ }
calls[caseIndex].codePtr = codePtr;
- done.append(stubJit.jump());
+ if (needsDoneJump)
+ done.append(stubJit.jump());
}
slowPath.link(&stubJit);
@@ -1571,6 +1581,12 @@
}
stubJit.move(CCallHelpers::TrustedImmPtr(globalObject), GPRInfo::regT3);
stubJit.move(CCallHelpers::TrustedImmPtr(&callLinkInfo), GPRInfo::regT2);
+ if (isDataIC && !callLinkInfo.isTailCall()) {
+ // We were called from the fast path, get rid of any remnants of that
+ // which may exist. This really only matters for x86, which adjusts
+ // SP for calls.
+ stubJit.preserveReturnAddressAfterCall(GPRInfo::regT4);
+ }
stubJit.move(CCallHelpers::TrustedImmPtr(callLinkInfo.doneLocation().untaggedExecutableAddress()), GPRInfo::regT4);
stubJit.restoreReturnAddressBeforeReturn(GPRInfo::regT4);
@@ -1594,7 +1610,9 @@
patchBuffer.link(callToCodePtr.call, FunctionPtr<JSEntryPtrTag>(callToCodePtr.codePtr));
#endif
}
- patchBuffer.link(done, callLinkInfo.doneLocation());
+
+ if (!done.empty())
+ patchBuffer.link(done, callLinkInfo.doneLocation());
patchBuffer.link(slow, CodeLocationLabel<JITThunkPtrTag>(vm.getCTIStub(linkPolymorphicCallThunkGenerator).code()));
auto stubRoutine = adoptRef(*new PolymorphicCallStubRoutine(
