Diff
Modified: trunk/Source/_javascript_Core/ChangeLog (287863 => 287864)
--- trunk/Source/_javascript_Core/ChangeLog 2022-01-10 23:48:40 UTC (rev 287863)
+++ trunk/Source/_javascript_Core/ChangeLog 2022-01-11 01:36:00 UTC (rev 287864)
@@ -1,3 +1,62 @@
+2022-01-10 Saam Barati <[email protected]>
+
+ Allow loop tier up to the Air tier
+ https://bugs.webkit.org/show_bug.cgi?id=234587
+ <rdar://problem/86968638>
+
+ Reviewed by Yusuke Suzuki.
+
+ This patch adds loop tier up from LLInt -> Air. To implement this, we use
+ EntrySwitch to point at each loop header, making each loop an entrypoint.
+ This is unlike BBQ->OMG tier up, where we compile a special OSR entry OMG
+ callee. This seems like a good architecture for the Air tier, since we might end
+ up with slightly worse throughput, but we won't need a different compilation
+ for loops vs call entrypoints.
+
+ This patch also fixes a bug in Air's O0 register allocation where it
+ didn't properly account for all named registers in an instruction. There
+ was a silly bug where we asked each arg if it were a temp, instead of
+ asking the Inst for each of its temps, since an Arg can be an address
+ but still use temps.
+
+ * b3/air/AirAllocateRegistersAndStackAndGenerateCode.cpp:
+ (JSC::B3::Air::GenerateAndAllocateRegisters::generate):
+ * wasm/WasmAirIRGenerator.cpp:
+ (JSC::Wasm::AirIRGenerator::emitLoad):
+ (JSC::Wasm::AirIRGenerator::AirIRGenerator):
+ (JSC::Wasm::AirIRGenerator::finalizeEntrypoints):
+ (JSC::Wasm::AirIRGenerator::emitLoopTierUpCheck):
+ (JSC::Wasm::AirIRGenerator::addLoop):
+ (JSC::Wasm::parseAndCompileAir):
+ * wasm/WasmB3IRGenerator.cpp:
+ (JSC::Wasm::B3IRGenerator::B3IRGenerator):
+ (JSC::Wasm::parseAndCompileB3):
+ (JSC::Wasm::parseAndCompile): Deleted.
+ * wasm/WasmB3IRGenerator.h:
+ * wasm/WasmBBQPlan.cpp:
+ (JSC::Wasm::BBQPlan::prepareImpl):
+ (JSC::Wasm::BBQPlan::work):
+ (JSC::Wasm::BBQPlan::compileFunction):
+ (JSC::Wasm::BBQPlan::didCompleteCompilation):
+ (JSC::Wasm::BBQPlan::initializeCallees):
+ * wasm/WasmBBQPlan.h:
+ * wasm/WasmCallee.h:
+ * wasm/WasmCalleeGroup.h:
+ * wasm/WasmFormat.h:
+ * wasm/WasmIRGeneratorHelpers.h:
+ (JSC::Wasm::computeExceptionHandlerAndLoopEntrypointLocations):
+ (JSC::Wasm::computeExceptionHandlerLocations):
+ * wasm/WasmLLIntPlan.cpp:
+ (JSC::Wasm::LLIntPlan::didCompleteCompilation):
+ * wasm/WasmOMGPlan.cpp:
+ (JSC::Wasm::OMGPlan::work):
+ * wasm/WasmOSREntryPlan.cpp:
+ (JSC::Wasm::OSREntryPlan::work):
+ * wasm/WasmSlowPaths.cpp:
+ (JSC::LLInt::WASM_SLOW_PATH_DECL):
+ * wasm/js/JSToWasm.cpp:
+ (JSC::Wasm::createJSToWasmWrapper):
+
2022-01-10 Alex Christensen <[email protected]>
Start using C++20
Modified: trunk/Source/_javascript_Core/b3/air/AirAllocateRegistersAndStackAndGenerateCode.cpp (287863 => 287864)
--- trunk/Source/_javascript_Core/b3/air/AirAllocateRegistersAndStackAndGenerateCode.cpp 2022-01-10 23:48:40 UTC (rev 287863)
+++ trunk/Source/_javascript_Core/b3/air/AirAllocateRegistersAndStackAndGenerateCode.cpp 2022-01-11 01:36:00 UTC (rev 287864)
@@ -581,11 +581,7 @@
})();
checkConsistency();
- inst.forEachArg([&] (Arg& arg, Arg::Role role, Bank, Width) {
- if (!arg.isTmp())
- return;
-
- Tmp tmp = arg.tmp();
+ inst.forEachTmp([&] (const Tmp& tmp, Arg::Role role, Bank, Width) {
if (tmp.isReg() && isDisallowedRegister(tmp.reg()))
return;
@@ -595,7 +591,14 @@
if (Arg::isAnyDef(role))
m_namedDefdRegs.set(tmp.reg());
}
+ });
+ inst.forEachArg([&] (Arg& arg, Arg::Role role, Bank, Width) {
+ if (!arg.isTmp())
+ return;
+
+ Tmp tmp = arg.tmp();
+
// We convert any cold uses that are already in the stack to just point to
// the canonical stack location.
if (!Arg::isColdUse(role))
@@ -693,34 +696,28 @@
// We rewind this Inst to be in its previous state, however, if any arg admits stack,
// we move to providing that arg in stack form. This will allow us to fully allocate
// this inst when we rewind.
- inst.forEachArg([&] (Arg& arg, Arg::Role, Bank, Width) {
- if (!arg.isTmp())
+ inst.forEachTmpFast([&] (Tmp& tmp) {
+ if (!tmp.isReg())
return;
-
- Tmp tmp = arg.tmp();
- if (tmp.isReg() && isDisallowedRegister(tmp.reg()))
+ if (isDisallowedRegister(tmp.reg()))
return;
-
- if (tmp.isReg()) {
- Tmp originalTmp = allocationSnapshot[tmp.reg()];
- if (originalTmp.isReg()) {
- ASSERT(tmp.reg() == originalTmp.reg());
- // This means this Inst referred to this reg directly. We leave these as is.
- return;
- }
- tmp = originalTmp;
+ Tmp originalTmp = allocationSnapshot[tmp.reg()];
+ if (originalTmp.isReg()) {
+ ASSERT(tmp.reg() == originalTmp.reg());
+ // This means this Inst referred to this reg directly. We leave these as is.
+ return;
}
+ tmp = originalTmp;
+ });
+ inst.forEachArg([&] (Arg& arg, Arg::Role, Bank, Width) {
+ if (arg.isTmp() && !arg.tmp().isReg() && inst.admitsStack(arg)) {
+ Tmp tmp = arg.tmp();
+ auto& entry = m_map[tmp];
+ if (Reg reg = entry.reg)
+ spill(tmp, reg);
- if (!inst.admitsStack(arg)) {
- arg = tmp;
- return;
+ arg = Arg::addr(Tmp(GPRInfo::callFrameRegister), entry.spillSlot->offsetFromFP());
}
-
- auto& entry = m_map[tmp];
- if (Reg reg = entry.reg)
- spill(tmp, reg);
-
- arg = Arg::addr(Tmp(GPRInfo::callFrameRegister), entry.spillSlot->offsetFromFP());
});
--instIndex;
Modified: trunk/Source/_javascript_Core/wasm/WasmAirIRGenerator.cpp (287863 => 287864)
--- trunk/Source/_javascript_Core/wasm/WasmAirIRGenerator.cpp 2022-01-10 23:48:40 UTC (rev 287863)
+++ trunk/Source/_javascript_Core/wasm/WasmAirIRGenerator.cpp 2022-01-11 01:36:00 UTC (rev 287864)
@@ -322,7 +322,7 @@
return fail(__VA_ARGS__); \
} while (0)
- AirIRGenerator(const ModuleInformation&, B3::Procedure&, InternalFunction*, Vector<UnlinkedWasmToWasmCall>&, MemoryMode, unsigned functionIndex, TierUpCount*, const Signature&);
+ AirIRGenerator(const ModuleInformation&, B3::Procedure&, InternalFunction*, Vector<UnlinkedWasmToWasmCall>&, MemoryMode, unsigned functionIndex, TierUpCount*, const Signature&, unsigned& osrEntryScratchBufferSize);
void finalizeEntrypoints();
@@ -783,7 +783,7 @@
}
}
- void emitLoad(Tmp base, size_t offset, TypedTmp result)
+ void emitLoad(Tmp base, size_t offset, const TypedTmp& result)
{
emitLoad(moveOpForValueType(result.type()), toB3Type(result.type()), base, offset, result.tmp());
}
@@ -791,7 +791,7 @@
void emitThrowException(CCallHelpers&, ExceptionType);
void emitEntryTierUpCheck();
- void emitLoopTierUpCheck(uint32_t loopIndex, const Stack& enclosingStack, const Stack& newStack);
+ void emitLoopTierUpCheck(uint32_t loopIndex, const Vector<TypedTmp>& liveValues);
void emitWriteBarrierForJSWrapper();
ExpressionType emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOp);
@@ -884,6 +884,9 @@
Checked<unsigned> m_callSiteIndex { 0 };
StackMaps m_stackmaps;
Vector<UnlinkedHandlerInfo> m_exceptionHandlers;
+
+ Vector<std::pair<BasicBlock*, Vector<TypedTmp>>> m_loopEntryVariableData;
+ unsigned& m_osrEntryScratchBufferSize;
};
// Memory accesses in WebAssembly have unsigned 32-bit offsets, whereas they have signed 32-bit offsets in B3.
@@ -929,7 +932,7 @@
emitPatchpoint(block, patchpoint, Tmp(), instance);
}
-AirIRGenerator::AirIRGenerator(const ModuleInformation& info, B3::Procedure& procedure, InternalFunction* compilation, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, MemoryMode mode, unsigned functionIndex, TierUpCount* tierUp, const Signature& signature)
+AirIRGenerator::AirIRGenerator(const ModuleInformation& info, B3::Procedure& procedure, InternalFunction* compilation, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, MemoryMode mode, unsigned functionIndex, TierUpCount* tierUp, const Signature& signature, unsigned& osrEntryScratchBufferSize)
: m_info(info)
, m_mode(mode)
, m_functionIndex(functionIndex)
@@ -938,6 +941,7 @@
, m_code(m_proc.code())
, m_unlinkedWasmToWasmCalls(unlinkedWasmToWasmCalls)
, m_numImportFunctions(info.importFunctionCount())
+ , m_osrEntryScratchBufferSize(osrEntryScratchBufferSize)
{
m_currentBlock = m_code.addBlock();
m_rootBlock = m_currentBlock;
@@ -967,7 +971,7 @@
GPRReg calleeGPR = wasmCallingConvention().prologueScratchGPRs[0];
auto moveLocation = jit.moveWithPatch(MacroAssembler::TrustedImmPtr(nullptr), calleeGPR);
jit.addLinkTask([compilation, moveLocation] (LinkBuffer& linkBuffer) {
- compilation->calleeMoveLocation = linkBuffer.locationOf<WasmEntryPtrTag>(moveLocation);
+ compilation->calleeMoveLocations.append(linkBuffer.locationOf<WasmEntryPtrTag>(moveLocation));
});
jit.emitPutToCallFrameHeader(calleeGPR, CallFrameSlot::callee);
jit.emitPutToCallFrameHeader(nullptr, CallFrameSlot::codeBlock);
@@ -1072,9 +1076,11 @@
void AirIRGenerator::finalizeEntrypoints()
{
- unsigned numEntrypoints = 1 + m_catchEntrypoints.size();
+ unsigned numEntrypoints = Checked<unsigned>(1) + m_catchEntrypoints.size() + m_loopEntryVariableData.size();
m_proc.setNumEntrypoints(numEntrypoints);
m_code.setPrologueForEntrypoint(0, Ref<B3::Air::PrologueGenerator>(*m_prologueGenerator));
+ for (unsigned i = 1 + m_catchEntrypoints.size(); i < numEntrypoints; ++i)
+ m_code.setPrologueForEntrypoint(i, Ref<B3::Air::PrologueGenerator>(*m_prologueGenerator));
if (m_catchEntrypoints.size()) {
Ref<B3::Air::PrologueGenerator> catchPrologueGenerator = createSharedTask<B3::Air::PrologueGeneratorFunction>([this] (CCallHelpers& jit, B3::Air::Code& code) {
@@ -1095,6 +1101,25 @@
successors.append(m_mainEntrypointStart);
successors.appendVector(m_catchEntrypoints);
+ for (auto& pair : m_loopEntryVariableData) {
+ BasicBlock* loopBody = pair.first;
+ BasicBlock* entry = m_code.addBlock();
+ successors.append(entry);
+ m_currentBlock = entry;
+
+ auto& temps = pair.second;
+ m_osrEntryScratchBufferSize = std::max(m_osrEntryScratchBufferSize, static_cast<unsigned>(temps.size()));
+ Tmp basePtr = Tmp(GPRInfo::argumentGPR0);
+
+ for (size_t i = 0; i < temps.size(); ++i) {
+ size_t offset = static_cast<size_t>(i) * sizeof(uint64_t);
+ emitLoad(basePtr, offset, temps[i]);
+ }
+
+ append(Jump);
+ entry->setSuccessors(loopBody);
+ }
+
RELEASE_ASSERT(numEntrypoints == successors.size());
m_rootBlock->successors() = successors;
}
@@ -3033,7 +3058,7 @@
emitPatchpoint(patch, Tmp(), countdownPtr);
}
-void AirIRGenerator::emitLoopTierUpCheck(uint32_t loopIndex, const Stack& enclosingStack, const Stack& newStack)
+void AirIRGenerator::emitLoopTierUpCheck(uint32_t loopIndex, const Vector<TypedTmp>& liveValues)
{
uint32_t outerLoopIndex = this->outerLoopIndex();
m_outerLoops.append(loopIndex);
@@ -3062,15 +3087,9 @@
Vector<ConstrainedTmp> patchArgs;
patchArgs.append(countdownPtr);
+ for (const TypedTmp& tmp : liveValues)
+ patchArgs.append(ConstrainedTmp(tmp.tmp(), B3::ValueRep::ColdAny));
- forEachLiveValue([&] (Tmp tmp) {
- patchArgs.append(ConstrainedTmp(tmp, B3::ValueRep::ColdAny));
- });
- for (TypedExpression value : enclosingStack)
- patchArgs.append(ConstrainedTmp(value.value(), B3::ValueRep::ColdAny));
- for (TypedExpression value : newStack)
- patchArgs.append(ConstrainedTmp(value.value(), B3::ValueRep::ColdAny));
-
TierUpCount::TriggerReason* forceEntryTrigger = &(m_tierUp->osrEntryTriggers().last());
static_assert(!static_cast<uint8_t>(TierUpCount::TriggerReason::DontTrigger), "the JIT code assumes non-zero means 'enter'");
static_assert(sizeof(TierUpCount::TriggerReason) == 1, "branchTest8 assumes this size");
@@ -3110,10 +3129,22 @@
auto AirIRGenerator::addLoop(BlockSignature signature, Stack& enclosingStack, ControlType& block, Stack& newStack, uint32_t loopIndex) -> PartialResult
{
+ RELEASE_ASSERT(loopIndex == m_loopEntryVariableData.size());
+
BasicBlock* body = m_code.addBlock();
BasicBlock* continuation = m_code.addBlock();
splitStack(signature, enclosingStack, newStack);
+
+ Vector<TypedTmp> liveValues;
+ forEachLiveValue([&] (TypedTmp tmp) {
+ liveValues.append(tmp);
+ });
+ for (auto variable : enclosingStack)
+ liveValues.append(variable);
+ for (auto variable : newStack)
+ liveValues.append(variable);
+
ResultList results;
results.reserveInitialCapacity(newStack.size());
for (auto item : newStack)
@@ -3124,8 +3155,10 @@
m_currentBlock->setSuccessors(body);
m_currentBlock = body;
- emitLoopTierUpCheck(loopIndex, enclosingStack, newStack);
+ emitLoopTierUpCheck(loopIndex, liveValues);
+ m_loopEntryVariableData.append(std::pair<BasicBlock*, Vector<TypedTmp>>(body, WTFMove(liveValues)));
+
return { };
}
@@ -3899,7 +3932,7 @@
procedure.setOptLevel(Options::webAssemblyBBQAirOptimizationLevel());
- AirIRGenerator irGenerator(info, procedure, result.get(), unlinkedWasmToWasmCalls, mode, functionIndex, tierUp, signature);
+ AirIRGenerator irGenerator(info, procedure, result.get(), unlinkedWasmToWasmCalls, mode, functionIndex, tierUp, signature, result->osrEntryScratchBufferSize);
FunctionParser<AirIRGenerator> parser(irGenerator, function.data.data(), function.data.size(), signature, info);
WASM_FAIL_IF_HELPER_FAILS(parser.parse());
Modified: trunk/Source/_javascript_Core/wasm/WasmB3IRGenerator.cpp (287863 => 287864)
--- trunk/Source/_javascript_Core/wasm/WasmB3IRGenerator.cpp 2022-01-10 23:48:40 UTC (rev 287863)
+++ trunk/Source/_javascript_Core/wasm/WasmB3IRGenerator.cpp 2022-01-11 01:36:00 UTC (rev 287864)
@@ -677,7 +677,7 @@
}
{
- auto* calleeMoveLocation = &compilation->calleeMoveLocation;
+ auto* calleeMoveLocations = &compilation->calleeMoveLocations;
static_assert(CallFrameSlot::codeBlock * sizeof(Register) < WasmCallingConvention::headerSizeInBytes, "We rely on this here for now.");
static_assert(CallFrameSlot::callee * sizeof(Register) < WasmCallingConvention::headerSizeInBytes, "We rely on this here for now.");
B3::PatchpointValue* getCalleePatchpoint = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Int64, Origin());
@@ -687,8 +687,8 @@
[=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
GPRReg result = params[0].gpr();
MacroAssembler::DataLabelPtr moveLocation = jit.moveWithPatch(MacroAssembler::TrustedImmPtr(nullptr), result);
- jit.addLinkTask([calleeMoveLocation, moveLocation] (LinkBuffer& linkBuffer) {
- *calleeMoveLocation = linkBuffer.locationOf<WasmEntryPtrTag>(moveLocation);
+ jit.addLinkTask([calleeMoveLocations, moveLocation] (LinkBuffer& linkBuffer) {
+ calleeMoveLocations->append(linkBuffer.locationOf<WasmEntryPtrTag>(moveLocation));
});
});
@@ -3138,7 +3138,7 @@
return dumpAllowlist->shouldDumpWasmFunction(functionIndex);
}
-Expected<std::unique_ptr<InternalFunction>, String> parseAndCompile(CompilationContext& compilationContext, const FunctionData& function, const Signature& signature, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, unsigned& osrEntryScratchBufferSize, const ModuleInformation& info, MemoryMode mode, CompilationMode compilationMode, uint32_t functionIndex, uint32_t loopIndexForOSREntry, TierUpCount* tierUp)
+Expected<std::unique_ptr<InternalFunction>, String> parseAndCompileB3(CompilationContext& compilationContext, const FunctionData& function, const Signature& signature, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, const ModuleInformation& info, MemoryMode mode, CompilationMode compilationMode, uint32_t functionIndex, uint32_t loopIndexForOSREntry, TierUpCount* tierUp)
{
auto result = makeUnique<InternalFunction>();
@@ -3175,7 +3175,7 @@
? Options::webAssemblyBBQB3OptimizationLevel()
: Options::webAssemblyOMGOptimizationLevel());
- B3IRGenerator irGenerator(info, procedure, result.get(), unlinkedWasmToWasmCalls, osrEntryScratchBufferSize, mode, compilationMode, functionIndex, loopIndexForOSREntry, tierUp);
+ B3IRGenerator irGenerator(info, procedure, result.get(), unlinkedWasmToWasmCalls, result->osrEntryScratchBufferSize, mode, compilationMode, functionIndex, loopIndexForOSREntry, tierUp);
FunctionParser<B3IRGenerator> parser(irGenerator, function.data.data(), function.data.size(), signature, info);
WASM_FAIL_IF_HELPER_FAILS(parser.parse());
Modified: trunk/Source/_javascript_Core/wasm/WasmB3IRGenerator.h (287863 => 287864)
--- trunk/Source/_javascript_Core/wasm/WasmB3IRGenerator.h 2022-01-10 23:48:40 UTC (rev 287863)
+++ trunk/Source/_javascript_Core/wasm/WasmB3IRGenerator.h 2022-01-11 01:36:00 UTC (rev 287864)
@@ -57,7 +57,7 @@
Box<PCToCodeOriginMap> pcToCodeOriginMap;
};
-Expected<std::unique_ptr<InternalFunction>, String> parseAndCompile(CompilationContext&, const FunctionData&, const Signature&, Vector<UnlinkedWasmToWasmCall>&, unsigned& osrEntryScratchBufferSize, const ModuleInformation&, MemoryMode, CompilationMode, uint32_t functionIndex, uint32_t loopIndexForOSREntry, TierUpCount* = nullptr);
+Expected<std::unique_ptr<InternalFunction>, String> parseAndCompileB3(CompilationContext&, const FunctionData&, const Signature&, Vector<UnlinkedWasmToWasmCall>&, const ModuleInformation&, MemoryMode, CompilationMode, uint32_t functionIndex, uint32_t loopIndexForOSREntry, TierUpCount* = nullptr);
void computePCToCodeOriginMap(CompilationContext&, LinkBuffer&);
Modified: trunk/Source/_javascript_Core/wasm/WasmBBQPlan.cpp (287863 => 287864)
--- trunk/Source/_javascript_Core/wasm/WasmBBQPlan.cpp 2022-01-10 23:48:40 UTC (rev 287863)
+++ trunk/Source/_javascript_Core/wasm/WasmBBQPlan.cpp 2022-01-11 01:36:00 UTC (rev 287864)
@@ -64,7 +64,8 @@
const auto& functions = m_moduleInformation->functions;
if (!tryReserveCapacity(m_wasmInternalFunctions, functions.size(), " WebAssembly functions")
|| !tryReserveCapacity(m_compilationContexts, functions.size(), " compilation contexts")
- || !tryReserveCapacity(m_tierUpCounts, functions.size(), " tier-up counts"))
+ || !tryReserveCapacity(m_tierUpCounts, functions.size(), " tier-up counts")
+ || !tryReserveCapacity(m_allLoopEntrypoints, functions.size(), " loop entrypoints"))
return false;
m_wasmInternalFunctions.resize(functions.size());
@@ -71,6 +72,7 @@
m_exceptionHandlerLocations.resize(functions.size());
m_compilationContexts.resize(functions.size());
m_tierUpCounts.resize(functions.size());
+ m_allLoopEntrypoints.resize(functions.size());
return true;
}
@@ -113,7 +115,8 @@
}
Vector<CodeLocationLabel<ExceptionHandlerPtrTag>> exceptionHandlerLocations;
- computeExceptionHandlerLocations(exceptionHandlerLocations, function.get(), context, linkBuffer);
+ Vector<CodeLocationLabel<WasmEntryPtrTag>> loopEntrypointLocations;
+ computeExceptionHandlerAndLoopEntrypointLocations(exceptionHandlerLocations, loopEntrypointLocations, function.get(), context, linkBuffer);
computePCToCodeOriginMap(context, linkBuffer);
@@ -126,8 +129,9 @@
MacroAssemblerCodePtr<WasmEntryPtrTag> entrypoint;
{
- Ref<BBQCallee> callee = BBQCallee::create(WTFMove(function->entrypoint), functionIndexSpace, m_moduleInformation->nameSection->get(functionIndexSpace), WTFMove(tierUp), WTFMove(unlinkedWasmToWasmCalls), WTFMove(function->stackmaps), WTFMove(function->exceptionHandlers), WTFMove(exceptionHandlerLocations));
- MacroAssembler::repatchPointer(function->calleeMoveLocation, CalleeBits::boxWasm(callee.ptr()));
+ Ref<BBQCallee> callee = BBQCallee::create(WTFMove(function->entrypoint), functionIndexSpace, m_moduleInformation->nameSection->get(functionIndexSpace), WTFMove(tierUp), WTFMove(unlinkedWasmToWasmCalls), WTFMove(function->stackmaps), WTFMove(function->exceptionHandlers), WTFMove(exceptionHandlerLocations), WTFMove(loopEntrypointLocations), function->osrEntryScratchBufferSize);
+ for (auto& moveLocation : function->calleeMoveLocations)
+ MacroAssembler::repatchPointer(moveLocation, CalleeBits::boxWasm(callee.ptr()));
entrypoint = callee->entrypoint();
if (context.pcToCodeOriginMap)
@@ -196,7 +200,6 @@
unsigned functionIndexSpace = m_moduleInformation->importFunctionCount() + functionIndex;
ASSERT_UNUSED(functionIndexSpace, m_moduleInformation->signatureIndexFromFunctionIndexSpace(functionIndexSpace) == signatureIndex);
Expected<std::unique_ptr<InternalFunction>, String> parseAndCompileResult;
- unsigned osrEntryScratchBufferSize = 0;
// FIXME: Some webpages use very large Wasm module, and it exhausts all executable memory in ARM64 devices since the size of executable memory region is only limited to 128MB.
// The long term solution should be to introduce a Wasm interpreter. But as a short term solution, we introduce heuristics to switch back to BBQ B3 at the sacrifice of start-up time,
@@ -208,7 +211,7 @@
forceUsingB3 = true;
if (forceUsingB3)
- parseAndCompileResult = parseAndCompile(context, function, signature, unlinkedWasmToWasmCalls, osrEntryScratchBufferSize, m_moduleInformation.get(), m_mode, CompilationMode::BBQMode, functionIndex, UINT32_MAX, tierUp);
+ parseAndCompileResult = parseAndCompileB3(context, function, signature, unlinkedWasmToWasmCalls, m_moduleInformation.get(), m_mode, CompilationMode::BBQMode, functionIndex, UINT32_MAX, tierUp);
else
parseAndCompileResult = parseAndCompileAir(context, function, signature, unlinkedWasmToWasmCalls, m_moduleInformation.get(), m_mode, functionIndex, tierUp);
@@ -241,7 +244,7 @@
return;
}
- computeExceptionHandlerLocations(m_exceptionHandlerLocations[functionIndex], function, context, linkBuffer);
+ computeExceptionHandlerAndLoopEntrypointLocations(m_exceptionHandlerLocations[functionIndex], m_allLoopEntrypoints[functionIndex], function, context, linkBuffer);
computePCToCodeOriginMap(context, linkBuffer);
@@ -284,14 +287,17 @@
RefPtr<EmbedderEntrypointCallee> embedderEntrypointCallee;
if (auto embedderToWasmFunction = m_embedderToWasmInternalFunctions.get(internalFunctionIndex)) {
embedderEntrypointCallee = EmbedderEntrypointCallee::create(WTFMove(embedderToWasmFunction->entrypoint));
- MacroAssembler::repatchPointer(embedderToWasmFunction->calleeMoveLocation, CalleeBits::boxWasm(embedderEntrypointCallee.get()));
+ for (auto& moveLocation : embedderToWasmFunction->calleeMoveLocations)
+ MacroAssembler::repatchPointer(moveLocation, CalleeBits::boxWasm(embedderEntrypointCallee.get()));
}
InternalFunction* function = m_wasmInternalFunctions[internalFunctionIndex].get();
size_t functionIndexSpace = internalFunctionIndex + m_moduleInformation->importFunctionCount();
- Ref<BBQCallee> wasmEntrypointCallee = BBQCallee::create(WTFMove(function->entrypoint), functionIndexSpace, m_moduleInformation->nameSection->get(functionIndexSpace), WTFMove(m_tierUpCounts[internalFunctionIndex]), WTFMove(m_unlinkedWasmToWasmCalls[internalFunctionIndex]), WTFMove(function->stackmaps), WTFMove(function->exceptionHandlers), WTFMove(m_exceptionHandlerLocations[internalFunctionIndex]));
- MacroAssembler::repatchPointer(function->calleeMoveLocation, CalleeBits::boxWasm(wasmEntrypointCallee.ptr()));
+ Ref<BBQCallee> wasmEntrypointCallee = BBQCallee::create(WTFMove(function->entrypoint), functionIndexSpace, m_moduleInformation->nameSection->get(functionIndexSpace), WTFMove(m_tierUpCounts[internalFunctionIndex]), WTFMove(m_unlinkedWasmToWasmCalls[internalFunctionIndex]), WTFMove(function->stackmaps), WTFMove(function->exceptionHandlers), WTFMove(m_exceptionHandlerLocations[internalFunctionIndex]), WTFMove(m_allLoopEntrypoints[internalFunctionIndex]), function->osrEntryScratchBufferSize);
+ for (auto& moveLocation : function->calleeMoveLocations)
+ MacroAssembler::repatchPointer(moveLocation, CalleeBits::boxWasm(wasmEntrypointCallee.ptr()));
+
if (m_compilationContexts[internalFunctionIndex].pcToCodeOriginMap)
CalleeRegistry::singleton().addPCToCodeOriginMap(wasmEntrypointCallee.ptr(), WTFMove(m_compilationContexts[internalFunctionIndex].pcToCodeOriginMap));
Modified: trunk/Source/_javascript_Core/wasm/WasmBBQPlan.h (287863 => 287864)
--- trunk/Source/_javascript_Core/wasm/WasmBBQPlan.h 2022-01-10 23:48:40 UTC (rev 287863)
+++ trunk/Source/_javascript_Core/wasm/WasmBBQPlan.h 2022-01-11 01:36:00 UTC (rev 287864)
@@ -87,6 +87,7 @@
HashMap<uint32_t, std::unique_ptr<InternalFunction>, DefaultHash<uint32_t>, WTF::UnsignedWithZeroKeyHashTraits<uint32_t>> m_embedderToWasmInternalFunctions;
Vector<CompilationContext> m_compilationContexts;
Vector<std::unique_ptr<TierUpCount>> m_tierUpCounts;
+ Vector<Vector<CodeLocationLabel<WasmEntryPtrTag>>> m_allLoopEntrypoints;
RefPtr<CalleeGroup> m_calleeGroup { nullptr };
uint32_t m_functionIndex;
Modified: trunk/Source/_javascript_Core/wasm/WasmCallee.h (287863 => 287864)
--- trunk/Source/_javascript_Core/wasm/WasmCallee.h 2022-01-10 23:48:40 UTC (rev 287863)
+++ trunk/Source/_javascript_Core/wasm/WasmCallee.h 2022-01-11 01:36:00 UTC (rev 287864)
@@ -172,9 +172,9 @@
class BBQCallee final : public OptimizingJITCallee {
public:
- static Ref<BBQCallee> create(Wasm::Entrypoint&& entrypoint, size_t index, std::pair<const Name*, RefPtr<NameSection>>&& name, std::unique_ptr<TierUpCount>&& tierUpCount, Vector<UnlinkedWasmToWasmCall>&& unlinkedCalls, StackMaps&& stackmaps, Vector<UnlinkedHandlerInfo>&& exceptionHandlers, Vector<CodeLocationLabel<ExceptionHandlerPtrTag>>&& exceptionHandlerLocations)
+ static Ref<BBQCallee> create(Wasm::Entrypoint&& entrypoint, size_t index, std::pair<const Name*, RefPtr<NameSection>>&& name, std::unique_ptr<TierUpCount>&& tierUpCount, Vector<UnlinkedWasmToWasmCall>&& unlinkedCalls, StackMaps&& stackmaps, Vector<UnlinkedHandlerInfo>&& exceptionHandlers, Vector<CodeLocationLabel<ExceptionHandlerPtrTag>>&& exceptionHandlerLocations, Vector<CodeLocationLabel<WasmEntryPtrTag>>&& loopEntrypoints, unsigned osrEntryScratchBufferSize)
{
- return adoptRef(*new BBQCallee(WTFMove(entrypoint), index, WTFMove(name), WTFMove(tierUpCount), WTFMove(unlinkedCalls), WTFMove(stackmaps), WTFMove(exceptionHandlers), WTFMove(exceptionHandlerLocations)));
+ return adoptRef(*new BBQCallee(WTFMove(entrypoint), index, WTFMove(name), WTFMove(tierUpCount), WTFMove(unlinkedCalls), WTFMove(stackmaps), WTFMove(exceptionHandlers), WTFMove(exceptionHandlerLocations), WTFMove(loopEntrypoints), osrEntryScratchBufferSize));
}
OSREntryCallee* osrEntryCallee() { return m_osrEntryCallee.get(); }
@@ -194,10 +194,16 @@
TierUpCount* tierUpCount() { return m_tierUpCount.get(); }
+ const Vector<CodeLocationLabel<WasmEntryPtrTag>>& loopEntrypoints() { return m_loopEntrypoints; }
+
+ unsigned osrEntryScratchBufferSize() const { return m_osrEntryScratchBufferSize; }
+
private:
- BBQCallee(Wasm::Entrypoint&& entrypoint, size_t index, std::pair<const Name*, RefPtr<NameSection>>&& name, std::unique_ptr<TierUpCount>&& tierUpCount, Vector<UnlinkedWasmToWasmCall>&& unlinkedCalls, StackMaps&& stackmaps, Vector<UnlinkedHandlerInfo>&& exceptionHandlers, Vector<CodeLocationLabel<ExceptionHandlerPtrTag>>&& exceptionHandlerLocations)
+ BBQCallee(Wasm::Entrypoint&& entrypoint, size_t index, std::pair<const Name*, RefPtr<NameSection>>&& name, std::unique_ptr<TierUpCount>&& tierUpCount, Vector<UnlinkedWasmToWasmCall>&& unlinkedCalls, StackMaps&& stackmaps, Vector<UnlinkedHandlerInfo>&& exceptionHandlers, Vector<CodeLocationLabel<ExceptionHandlerPtrTag>>&& exceptionHandlerLocations, Vector<CodeLocationLabel<WasmEntryPtrTag>>&& loopEntrypoints, unsigned osrEntryScratchBufferSize)
: OptimizingJITCallee(Wasm::CompilationMode::BBQMode, WTFMove(entrypoint), index, WTFMove(name), WTFMove(unlinkedCalls), WTFMove(stackmaps), WTFMove(exceptionHandlers), WTFMove(exceptionHandlerLocations))
, m_tierUpCount(WTFMove(tierUpCount))
+ , m_loopEntrypoints(WTFMove(loopEntrypoints))
+ , m_osrEntryScratchBufferSize(osrEntryScratchBufferSize)
{
}
@@ -204,6 +210,8 @@
RefPtr<OSREntryCallee> m_osrEntryCallee;
RefPtr<OMGCallee> m_replacement;
std::unique_ptr<TierUpCount> m_tierUpCount;
+ Vector<CodeLocationLabel<WasmEntryPtrTag>> m_loopEntrypoints;
+ unsigned m_osrEntryScratchBufferSize { 0 };
bool m_didStartCompilingOSREntryCallee { false };
};
#endif
@@ -267,7 +275,7 @@
#if ENABLE(WEBASSEMBLY_B3JIT)
JITCallee* replacement(MemoryMode mode) { return m_replacements[static_cast<uint8_t>(mode)].get(); }
- void setReplacement(Ref<JITCallee>&& replacement, MemoryMode mode)
+ void setReplacement(Ref<OptimizingJITCallee>&& replacement, MemoryMode mode)
{
m_replacements[static_cast<uint8_t>(mode)] = WTFMove(replacement);
}
@@ -302,7 +310,7 @@
FixedVector<JumpTable> m_jumpTables;
#if ENABLE(WEBASSEMBLY_B3JIT)
- RefPtr<JITCallee> m_replacements[Wasm::NumberOfMemoryModes];
+ RefPtr<OptimizingJITCallee> m_replacements[Wasm::NumberOfMemoryModes];
RefPtr<OSREntryCallee> m_osrEntryCallees[Wasm::NumberOfMemoryModes];
#endif
MacroAssemblerCodePtr<WasmEntryPtrTag> m_entrypoint;
Modified: trunk/Source/_javascript_Core/wasm/WasmCalleeGroup.h (287863 => 287864)
--- trunk/Source/_javascript_Core/wasm/WasmCalleeGroup.h 2022-01-10 23:48:40 UTC (rev 287863)
+++ trunk/Source/_javascript_Core/wasm/WasmCalleeGroup.h 2022-01-11 01:36:00 UTC (rev 287864)
@@ -183,6 +183,7 @@
RefPtr<EntryPlan> m_plan;
std::atomic<bool> m_compilationFinished { false };
String m_errorMessage;
+public:
Lock m_lock;
};
Modified: trunk/Source/_javascript_Core/wasm/WasmFormat.h (287863 => 287864)
--- trunk/Source/_javascript_Core/wasm/WasmFormat.h 2022-01-10 23:48:40 UTC (rev 287863)
+++ trunk/Source/_javascript_Core/wasm/WasmFormat.h 2022-01-11 01:36:00 UTC (rev 287864)
@@ -443,10 +443,11 @@
struct InternalFunction {
WTF_MAKE_STRUCT_FAST_ALLOCATED;
- CodeLocationDataLabelPtr<WasmEntryPtrTag> calleeMoveLocation;
+ Vector<CodeLocationDataLabelPtr<WasmEntryPtrTag>> calleeMoveLocations;
StackMaps stackmaps;
Vector<UnlinkedHandlerInfo> exceptionHandlers;
Entrypoint entrypoint;
+ unsigned osrEntryScratchBufferSize { 0 };
};
// WebAssembly direct calls and call_indirect use indices into "function index space". This space starts
Modified: trunk/Source/_javascript_Core/wasm/WasmIRGeneratorHelpers.h (287863 => 287864)
--- trunk/Source/_javascript_Core/wasm/WasmIRGeneratorHelpers.h 2022-01-10 23:48:40 UTC (rev 287863)
+++ trunk/Source/_javascript_Core/wasm/WasmIRGeneratorHelpers.h 2022-01-11 01:36:00 UTC (rev 287864)
@@ -63,26 +63,34 @@
};
-static inline void computeExceptionHandlerLocations(Vector<CodeLocationLabel<ExceptionHandlerPtrTag>>& handlers, const InternalFunction* function, const CompilationContext& context, LinkBuffer& linkBuffer)
+static inline void computeExceptionHandlerAndLoopEntrypointLocations(Vector<CodeLocationLabel<ExceptionHandlerPtrTag>>& handlers, Vector<CodeLocationLabel<WasmEntryPtrTag>>& loopEntrypoints, const InternalFunction* function, const CompilationContext& context, LinkBuffer& linkBuffer)
{
if (!context.procedure)
return;
- unsigned entrypointIndex = 0;
+ unsigned entrypointIndex = 1;
unsigned numEntrypoints = context.procedure->numEntrypoints();
for (const UnlinkedHandlerInfo& handlerInfo : function->exceptionHandlers) {
- RELEASE_ASSERT(entrypointIndex < numEntrypoints);
if (handlerInfo.m_type == HandlerType::Delegate) {
handlers.append({ });
continue;
}
+ RELEASE_ASSERT(entrypointIndex < numEntrypoints);
+ handlers.append(linkBuffer.locationOf<ExceptionHandlerPtrTag>(context.procedure->code().entrypointLabel(entrypointIndex)));
++entrypointIndex;
- handlers.append(linkBuffer.locationOf<ExceptionHandlerPtrTag>(context.procedure->code().entrypointLabel(entrypointIndex)));
}
- RELEASE_ASSERT(entrypointIndex == numEntrypoints - 1);
+
+ for (; entrypointIndex < numEntrypoints; ++entrypointIndex)
+ loopEntrypoints.append(linkBuffer.locationOf<WasmEntryPtrTag>(context.procedure->code().entrypointLabel(entrypointIndex)));
}
+static inline void computeExceptionHandlerLocations(Vector<CodeLocationLabel<ExceptionHandlerPtrTag>>& handlers, const InternalFunction* function, const CompilationContext& context, LinkBuffer& linkBuffer)
+{
+ Vector<CodeLocationLabel<WasmEntryPtrTag>> ignored;
+ computeExceptionHandlerAndLoopEntrypointLocations(handlers, ignored, function, context, linkBuffer);
+}
+
static inline void emitRethrowImpl(CCallHelpers& jit)
{
// Instance in argumentGPR0
Modified: trunk/Source/_javascript_Core/wasm/WasmLLIntPlan.cpp (287863 => 287864)
--- trunk/Source/_javascript_Core/wasm/WasmLLIntPlan.cpp 2022-01-10 23:48:40 UTC (rev 287863)
+++ trunk/Source/_javascript_Core/wasm/WasmLLIntPlan.cpp 2022-01-11 01:36:00 UTC (rev 287864)
@@ -168,8 +168,8 @@
Ref<EmbedderEntrypointCallee> callee = EmbedderEntrypointCallee::create(WTFMove(function->entrypoint));
// FIXME: remove this repatchPointer - just pass in the callee directly
// https://bugs.webkit.org/show_bug.cgi?id=166462
- if (function->calleeMoveLocation)
- MacroAssembler::repatchPointer(function->calleeMoveLocation, CalleeBits::boxWasm(callee.ptr()));
+ for (auto& moveLocation : function->calleeMoveLocations)
+ MacroAssembler::repatchPointer(moveLocation, CalleeBits::boxWasm(callee.ptr()));
auto result = m_embedderCallees.add(functionIndex, WTFMove(callee));
ASSERT_UNUSED(result, result.isNewEntry);
Modified: trunk/Source/_javascript_Core/wasm/WasmOMGPlan.cpp (287863 => 287864)
--- trunk/Source/_javascript_Core/wasm/WasmOMGPlan.cpp 2022-01-10 23:48:40 UTC (rev 287863)
+++ trunk/Source/_javascript_Core/wasm/WasmOMGPlan.cpp 2022-01-11 01:36:00 UTC (rev 287864)
@@ -72,9 +72,8 @@
const Signature& signature = SignatureInformation::get(signatureIndex);
Vector<UnlinkedWasmToWasmCall> unlinkedCalls;
- unsigned osrEntryScratchBufferSize;
CompilationContext context;
- auto parseAndCompileResult = parseAndCompile(context, function, signature, unlinkedCalls, osrEntryScratchBufferSize, m_moduleInformation.get(), m_mode, CompilationMode::OMGMode, m_functionIndex, UINT32_MAX);
+ auto parseAndCompileResult = parseAndCompileB3(context, function, signature, unlinkedCalls, m_moduleInformation.get(), m_mode, CompilationMode::OMGMode, m_functionIndex, UINT32_MAX);
if (UNLIKELY(!parseAndCompileResult)) {
Locker locker { m_lock };
@@ -106,7 +105,8 @@
{
ASSERT(m_calleeGroup.ptr() == m_module->calleeGroupFor(mode()));
Ref<OMGCallee> callee = OMGCallee::create(WTFMove(omgEntrypoint), functionIndexSpace, m_moduleInformation->nameSection->get(functionIndexSpace), WTFMove(unlinkedCalls), WTFMove(internalFunction->stackmaps), WTFMove(internalFunction->exceptionHandlers), WTFMove(exceptionHandlerLocations));
- MacroAssembler::repatchPointer(internalFunction->calleeMoveLocation, CalleeBits::boxWasm(callee.ptr()));
+ for (auto& moveLocation : internalFunction->calleeMoveLocations)
+ MacroAssembler::repatchPointer(moveLocation, CalleeBits::boxWasm(callee.ptr()));
entrypoint = callee->entrypoint();
if (context.pcToCodeOriginMap)
Modified: trunk/Source/_javascript_Core/wasm/WasmOSREntryPlan.cpp (287863 => 287864)
--- trunk/Source/_javascript_Core/wasm/WasmOSREntryPlan.cpp 2022-01-10 23:48:40 UTC (rev 287863)
+++ trunk/Source/_javascript_Core/wasm/WasmOSREntryPlan.cpp 2022-01-11 01:36:00 UTC (rev 287864)
@@ -77,8 +77,7 @@
Vector<UnlinkedWasmToWasmCall> unlinkedCalls;
CompilationContext context;
- unsigned osrEntryScratchBufferSize = 0;
- auto parseAndCompileResult = parseAndCompile(context, function, signature, unlinkedCalls, osrEntryScratchBufferSize, m_moduleInformation.get(), m_mode, targetCompilationMode, m_functionIndex, m_loopIndex);
+ auto parseAndCompileResult = parseAndCompileB3(context, function, signature, unlinkedCalls, m_moduleInformation.get(), m_mode, targetCompilationMode, m_functionIndex, m_loopIndex);
if (UNLIKELY(!parseAndCompileResult)) {
Locker locker { m_lock };
@@ -105,9 +104,10 @@
omgEntrypoint.calleeSaveRegisters = WTFMove(internalFunction->entrypoint.calleeSaveRegisters);
ASSERT(m_calleeGroup.ptr() == m_module->calleeGroupFor(mode()));
- Ref<OSREntryCallee> callee = OSREntryCallee::create(targetCompilationMode, WTFMove(omgEntrypoint), functionIndexSpace, m_moduleInformation->nameSection->get(functionIndexSpace), osrEntryScratchBufferSize, m_loopIndex, WTFMove(unlinkedCalls), WTFMove(internalFunction->stackmaps), WTFMove(internalFunction->exceptionHandlers), WTFMove(exceptionHandlerLocations));
+ Ref<OSREntryCallee> callee = OSREntryCallee::create(targetCompilationMode, WTFMove(omgEntrypoint), functionIndexSpace, m_moduleInformation->nameSection->get(functionIndexSpace), internalFunction->osrEntryScratchBufferSize, m_loopIndex, WTFMove(unlinkedCalls), WTFMove(internalFunction->stackmaps), WTFMove(internalFunction->exceptionHandlers), WTFMove(exceptionHandlerLocations));
{
- MacroAssembler::repatchPointer(internalFunction->calleeMoveLocation, CalleeBits::boxWasm(callee.ptr()));
+ for (auto& moveLocation : internalFunction->calleeMoveLocations)
+ MacroAssembler::repatchPointer(moveLocation, CalleeBits::boxWasm(callee.ptr()));
Locker locker { m_calleeGroup->m_lock };
for (auto& call : callee->wasmToWasmCallsites()) {
Modified: trunk/Source/_javascript_Core/wasm/WasmSlowPaths.cpp (287863 => 287864)
--- trunk/Source/_javascript_Core/wasm/WasmSlowPaths.cpp 2022-01-10 23:48:40 UTC (rev 287863)
+++ trunk/Source/_javascript_Core/wasm/WasmSlowPaths.cpp 2022-01-11 01:36:00 UTC (rev 287864)
@@ -180,63 +180,88 @@
dataLogLnIf(Options::verboseOSR(), *callee, ": Entered loop_osr with tierUpCounter = ", callee->tierUpCounter());
- unsigned loopOSREntryBytecodeOffset = callee->bytecodeOffset(pc);
- const auto& osrEntryData = tierUpCounter.osrEntryDataForLoop(loopOSREntryBytecodeOffset);
-
if (!tierUpCounter.checkIfOptimizationThresholdReached()) {
dataLogLnIf(Options::verboseOSR(), " JIT threshold should be lifted.");
WASM_RETURN_TWO(nullptr, nullptr);
}
- const auto doOSREntry = [&](Wasm::OSREntryCallee* osrEntryCallee) {
- if (osrEntryCallee->loopIndex() != osrEntryData.loopIndex)
+ unsigned loopOSREntryBytecodeOffset = callee->bytecodeOffset(pc);
+ const auto& osrEntryData = tierUpCounter.osrEntryDataForLoop(loopOSREntryBytecodeOffset);
+
+ if (Options::wasmLLIntTiersUpToBBQ()) {
+ if (!jitCompileAndSetHeuristics(callee, instance))
WASM_RETURN_TWO(nullptr, nullptr);
- size_t osrEntryScratchBufferSize = osrEntryCallee->osrEntryScratchBufferSize();
- RELEASE_ASSERT(osrEntryScratchBufferSize == osrEntryData.values.size());
+ Wasm::BBQCallee* bbqCallee;
+ {
+ Locker locker { instance->calleeGroup()->m_lock };
+ bbqCallee = instance->calleeGroup()->bbqCallee(locker, callee->functionIndex());
+ }
+ RELEASE_ASSERT(bbqCallee);
+
+ size_t osrEntryScratchBufferSize = bbqCallee->osrEntryScratchBufferSize();
+ RELEASE_ASSERT(osrEntryScratchBufferSize >= osrEntryData.values.size());
uint64_t* buffer = instance->context()->scratchBufferForSize(osrEntryScratchBufferSize);
if (!buffer)
WASM_RETURN_TWO(nullptr, nullptr);
+ RELEASE_ASSERT(osrEntryData.loopIndex < bbqCallee->loopEntrypoints().size());
uint32_t index = 0;
for (VirtualRegister reg : osrEntryData.values)
buffer[index++] = READ(reg).encodedJSValue();
- WASM_RETURN_TWO(buffer, osrEntryCallee->entrypoint().executableAddress());
- };
+ WASM_RETURN_TWO(buffer, bbqCallee->loopEntrypoints()[osrEntryData.loopIndex].executableAddress());
+ } else {
+ const auto doOSREntry = [&](Wasm::OSREntryCallee* osrEntryCallee) {
+ if (osrEntryCallee->loopIndex() != osrEntryData.loopIndex)
+ WASM_RETURN_TWO(nullptr, nullptr);
- if (auto* osrEntryCallee = callee->osrEntryCallee(instance->memory()->mode()))
- return doOSREntry(osrEntryCallee);
+ size_t osrEntryScratchBufferSize = osrEntryCallee->osrEntryScratchBufferSize();
+ RELEASE_ASSERT(osrEntryScratchBufferSize == osrEntryData.values.size());
+ uint64_t* buffer = instance->context()->scratchBufferForSize(osrEntryScratchBufferSize);
+ if (!buffer)
+ WASM_RETURN_TWO(nullptr, nullptr);
- bool compile = false;
- {
- Locker locker { tierUpCounter.m_lock };
- switch (tierUpCounter.m_loopCompilationStatus) {
- case Wasm::LLIntTierUpCounter::CompilationStatus::NotCompiled:
- compile = true;
- tierUpCounter.m_loopCompilationStatus = Wasm::LLIntTierUpCounter::CompilationStatus::Compiling;
- break;
- case Wasm::LLIntTierUpCounter::CompilationStatus::Compiling:
- tierUpCounter.optimizeAfterWarmUp();
- break;
- case Wasm::LLIntTierUpCounter::CompilationStatus::Compiled:
- break;
+ uint32_t index = 0;
+ for (VirtualRegister reg : osrEntryData.values)
+ buffer[index++] = READ(reg).encodedJSValue();
+
+ WASM_RETURN_TWO(buffer, osrEntryCallee->entrypoint().executableAddress());
+ };
+
+ if (auto* osrEntryCallee = callee->osrEntryCallee(instance->memory()->mode()))
+ return doOSREntry(osrEntryCallee);
+
+ bool compile = false;
+ {
+ Locker locker { tierUpCounter.m_lock };
+ switch (tierUpCounter.m_loopCompilationStatus) {
+ case Wasm::LLIntTierUpCounter::CompilationStatus::NotCompiled:
+ compile = true;
+ tierUpCounter.m_loopCompilationStatus = Wasm::LLIntTierUpCounter::CompilationStatus::Compiling;
+ break;
+ case Wasm::LLIntTierUpCounter::CompilationStatus::Compiling:
+ tierUpCounter.optimizeAfterWarmUp();
+ break;
+ case Wasm::LLIntTierUpCounter::CompilationStatus::Compiled:
+ break;
+ }
}
- }
- if (compile) {
- Ref<Wasm::Plan> plan = adoptRef(*static_cast<Wasm::Plan*>(new Wasm::OSREntryPlan(instance->context(), Ref<Wasm::Module>(instance->module()), Ref<Wasm::Callee>(*callee), callee->functionIndex(), osrEntryData.loopIndex, instance->memory()->mode(), Wasm::Plan::dontFinalize())));
- Wasm::ensureWorklist().enqueue(plan.copyRef());
- if (UNLIKELY(!Options::useConcurrentJIT()))
- plan->waitForCompletion();
- else
- tierUpCounter.optimizeAfterWarmUp();
- }
+ if (compile) {
+ Ref<Wasm::Plan> plan = adoptRef(*static_cast<Wasm::Plan*>(new Wasm::OSREntryPlan(instance->context(), Ref<Wasm::Module>(instance->module()), Ref<Wasm::Callee>(*callee), callee->functionIndex(), osrEntryData.loopIndex, instance->memory()->mode(), Wasm::Plan::dontFinalize())));
+ Wasm::ensureWorklist().enqueue(plan.copyRef());
+ if (UNLIKELY(!Options::useConcurrentJIT()))
+ plan->waitForCompletion();
+ else
+ tierUpCounter.optimizeAfterWarmUp();
+ }
- if (auto* osrEntryCallee = callee->osrEntryCallee(instance->memory()->mode()))
- return doOSREntry(osrEntryCallee);
+ if (auto* osrEntryCallee = callee->osrEntryCallee(instance->memory()->mode()))
+ return doOSREntry(osrEntryCallee);
- WASM_RETURN_TWO(nullptr, nullptr);
+ WASM_RETURN_TWO(nullptr, nullptr);
+ }
}
WASM_SLOW_PATH_DECL(epilogue_osr)
Modified: trunk/Source/_javascript_Core/wasm/js/JSToWasm.cpp (287863 => 287864)
--- trunk/Source/_javascript_Core/wasm/js/JSToWasm.cpp 2022-01-10 23:48:40 UTC (rev 287863)
+++ trunk/Source/_javascript_Core/wasm/js/JSToWasm.cpp 2022-01-11 01:36:00 UTC (rev 287864)
@@ -178,9 +178,9 @@
jit.emitZeroToCallFrameHeader(CallFrameSlot::codeBlock);
MacroAssembler::DataLabelPtr calleeMoveLocation = jit.moveWithPatch(MacroAssembler::TrustedImmPtr(nullptr), GPRInfo::nonPreservedNonReturnGPR);
jit.emitPutToCallFrameHeader(GPRInfo::nonPreservedNonReturnGPR, CallFrameSlot::callee);
- CodeLocationDataLabelPtr<WasmEntryPtrTag>* linkedCalleeMove = &result->calleeMoveLocation;
+ Vector<CodeLocationDataLabelPtr<WasmEntryPtrTag>>* linkedCalleeMove = &result->calleeMoveLocations;
jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
- *linkedCalleeMove = linkBuffer.locationOf<WasmEntryPtrTag>(calleeMoveLocation);
+ linkedCalleeMove->append(linkBuffer.locationOf<WasmEntryPtrTag>(calleeMoveLocation));
});
const PinnedRegisterInfo& pinnedRegs = PinnedRegisterInfo::get();
