Diff
Modified: branches/safari-613-branch/Source/_javascript_Core/CMakeLists.txt (288778 => 288779)
--- branches/safari-613-branch/Source/_javascript_Core/CMakeLists.txt 2022-01-29 03:27:40 UTC (rev 288778)
+++ branches/safari-613-branch/Source/_javascript_Core/CMakeLists.txt 2022-01-29 03:27:48 UTC (rev 288779)
@@ -593,6 +593,8 @@
assembler/RISCV64Assembler.h
assembler/RISCV64Registers.h
assembler/RegisterInfo.h
+ assembler/SecureARM64EHashPins.h
+ assembler/SecureARM64EHashPinsInlines.h
assembler/X86Assembler.h
assembler/X86Registers.h
assembler/X86_64Registers.h
@@ -1023,7 +1025,6 @@
runtime/InternalFunction.h
runtime/Intrinsic.h
runtime/IterationKind.h
- runtime/IterationStatus.h
runtime/IteratorOperations.h
runtime/IteratorPrototype.h
runtime/JSArray.h
Modified: branches/safari-613-branch/Source/_javascript_Core/_javascript_Core.xcodeproj/project.pbxproj (288778 => 288779)
--- branches/safari-613-branch/Source/_javascript_Core/_javascript_Core.xcodeproj/project.pbxproj 2022-01-29 03:27:40 UTC (rev 288778)
+++ branches/safari-613-branch/Source/_javascript_Core/_javascript_Core.xcodeproj/project.pbxproj 2022-01-29 03:27:48 UTC (rev 288779)
@@ -892,6 +892,8 @@
52C0611F1AA51E1C00B4ADBA /* RuntimeType.h in Headers */ = {isa = PBXBuildFile; fileRef = 52C0611D1AA51E1B00B4ADBA /* RuntimeType.h */; settings = {ATTRIBUTES = (Private, ); }; };
52C55566224C2AEA0099F5CC /* DFGValueRepReductionPhase.h in Headers */ = {isa = PBXBuildFile; fileRef = 52C55565224C2AE70099F5CC /* DFGValueRepReductionPhase.h */; };
52C952B719A289850069B386 /* TypeProfiler.h in Headers */ = {isa = PBXBuildFile; fileRef = 52C952B619A289850069B386 /* TypeProfiler.h */; settings = {ATTRIBUTES = (Private, ); }; };
+ 52CAEC752790B8FB00DDBAAF /* SecureARM64EHashPins.h in Headers */ = {isa = PBXBuildFile; fileRef = 52CAEC742790B8F600DDBAAF /* SecureARM64EHashPins.h */; settings = {ATTRIBUTES = (Private, ); }; };
+ 52CAEC762790B90100DDBAAF /* SecureARM64EHashPinsInlines.h in Headers */ = {isa = PBXBuildFile; fileRef = 52CAEC722790B8F600DDBAAF /* SecureARM64EHashPinsInlines.h */; settings = {ATTRIBUTES = (Private, ); }; };
52CD0F5D2242F569004A18A5 /* Foundation.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 51F0EB6105C86C6B00E6DF1B /* Foundation.framework */; };
52CD0F5E2242F569004A18A5 /* _javascript_Core.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 932F5BD90822A1C700736975 /* _javascript_Core.framework */; };
52CD0F682242F71C004A18A5 /* testdfg.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 52CD0F672242F71C004A18A5 /* testdfg.cpp */; };
@@ -2077,7 +2079,6 @@
FE48BD4423245E9300F136D0 /* JSCConfig.h in Headers */ = {isa = PBXBuildFile; fileRef = FE48BD4223245E8700F136D0 /* JSCConfig.h */; settings = {ATTRIBUTES = (Private, ); }; };
FE48E6381EB118D2005D7A96 /* ObjectInitializationScope.h in Headers */ = {isa = PBXBuildFile; fileRef = FE48E6361EB1188F005D7A96 /* ObjectInitializationScope.h */; settings = {ATTRIBUTES = (Private, ); }; };
FE4BFF2C1AD476E700088F87 /* FunctionOverrides.h in Headers */ = {isa = PBXBuildFile; fileRef = FE4BFF2A1AD476E700088F87 /* FunctionOverrides.h */; };
- FE4D55B81AE716CA0052E459 /* IterationStatus.h in Headers */ = {isa = PBXBuildFile; fileRef = FE4D55B71AE716CA0052E459 /* IterationStatus.h */; settings = {ATTRIBUTES = (Private, ); }; };
FE5068651AE246390009DAB7 /* DeferredSourceDump.h in Headers */ = {isa = PBXBuildFile; fileRef = FE5068641AE246390009DAB7 /* DeferredSourceDump.h */; settings = {ATTRIBUTES = (Private, ); }; };
FE533CA51F217DB30016A1FE /* Foundation.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 51F0EB6105C86C6B00E6DF1B /* Foundation.framework */; };
FE533CA61F217DB30016A1FE /* _javascript_Core.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 932F5BD90822A1C700736975 /* _javascript_Core.framework */; };
@@ -3750,6 +3751,9 @@
52C55565224C2AE70099F5CC /* DFGValueRepReductionPhase.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = DFGValueRepReductionPhase.h; path = dfg/DFGValueRepReductionPhase.h; sourceTree = "<group>"; };
52C952B619A289850069B386 /* TypeProfiler.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = TypeProfiler.h; sourceTree = "<group>"; };
52C952B819A28A1C0069B386 /* TypeProfiler.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = TypeProfiler.cpp; sourceTree = "<group>"; };
+ 52CAEC722790B8F600DDBAAF /* SecureARM64EHashPinsInlines.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = SecureARM64EHashPinsInlines.h; sourceTree = "<group>"; };
+ 52CAEC732790B8F600DDBAAF /* SecureARM64EHashPins.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = SecureARM64EHashPins.cpp; sourceTree = "<group>"; };
+ 52CAEC742790B8F600DDBAAF /* SecureARM64EHashPins.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = SecureARM64EHashPins.h; sourceTree = "<group>"; };
52CD0F642242F569004A18A5 /* testdfg */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = testdfg; sourceTree = BUILT_PRODUCTS_DIR; };
52CD0F672242F71C004A18A5 /* testdfg.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = testdfg.cpp; path = dfg/testdfg.cpp; sourceTree = "<group>"; };
52D1308F221CE03A009C836C /* foo.js */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode._javascript_; path = foo.js; sourceTree = "<group>"; };
@@ -5588,7 +5592,6 @@
FE48E6371EB118AD005D7A96 /* ObjectInitializationScope.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = ObjectInitializationScope.cpp; sourceTree = "<group>"; };
FE4BFF291AD476E700088F87 /* FunctionOverrides.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = FunctionOverrides.cpp; sourceTree = "<group>"; };
FE4BFF2A1AD476E700088F87 /* FunctionOverrides.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = FunctionOverrides.h; sourceTree = "<group>"; };
- FE4D55B71AE716CA0052E459 /* IterationStatus.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = IterationStatus.h; sourceTree = "<group>"; };
FE5068641AE246390009DAB7 /* DeferredSourceDump.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = DeferredSourceDump.h; sourceTree = "<group>"; };
FE5068661AE25E280009DAB7 /* DeferredSourceDump.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = DeferredSourceDump.cpp; sourceTree = "<group>"; };
FE533CA01F217C310016A1FE /* testmasm.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = testmasm.cpp; sourceTree = "<group>"; };
@@ -7776,7 +7779,6 @@
A38CA59B26DD84DE00C8D84C /* ISO8601.cpp */,
A38CA59C26DD84DE00C8D84C /* ISO8601.h */,
8B9F6D551D5912FA001C739F /* IterationKind.h */,
- FE4D55B71AE716CA0052E459 /* IterationStatus.h */,
70113D491A8DB093003848C4 /* IteratorOperations.cpp */,
70113D4A1A8DB093003848C4 /* IteratorOperations.h */,
70DC3E071B2DF2C700054299 /* IteratorPrototype.cpp */,
@@ -8763,6 +8765,9 @@
FE10AAE91F44D510009DEDC5 /* ProbeStack.cpp */,
FE10AAEA1F44D512009DEDC5 /* ProbeStack.h */,
9688CB140ED12B4E001D6499 /* RegisterInfo.h */,
+ 52CAEC732790B8F600DDBAAF /* SecureARM64EHashPins.cpp */,
+ 52CAEC742790B8F600DDBAAF /* SecureARM64EHashPins.h */,
+ 52CAEC722790B8F600DDBAAF /* SecureARM64EHashPinsInlines.h */,
FE533CA01F217C310016A1FE /* testmasm.cpp */,
9688CB140ED12B4E001D6492 /* X86_64Registers.h */,
9688CB140ED12B4E001D649F /* X86Assembler.h */,
@@ -9980,6 +9985,7 @@
0FBE0F7316C1DB050082C5E8 /* DFGCPSRethreadingPhase.h in Headers */,
A7D89CF617A0B8CC00773AD8 /* DFGCriticalEdgeBreakingPhase.h in Headers */,
0FFFC95A14EF90A900C72532 /* DFGCSEPhase.h in Headers */,
+ 52CAEC752790B8FB00DDBAAF /* SecureARM64EHashPins.h in Headers */,
0F2FC77316E12F740038D976 /* DFGDCEPhase.h in Headers */,
E3BFA5D021E853A1009C0EBA /* DFGDesiredGlobalProperty.h in Headers */,
0F8F2B9A172F0501007DBDA5 /* DFGDesiredIdentifiers.h in Headers */,
@@ -10437,7 +10443,6 @@
0F5E0FE72086AD480097F0DE /* IsoSubspacePerVM.h in Headers */,
8B9F6D561D5912FA001C739F /* IterationKind.h in Headers */,
53D41EC923C0081A00AE984B /* IterationModeMetadata.h in Headers */,
- FE4D55B81AE716CA0052E459 /* IterationStatus.h in Headers */,
70113D4C1A8DB093003848C4 /* IteratorOperations.h in Headers */,
70DC3E0A1B2DF2C700054299 /* IteratorPrototype.h in Headers */,
BC18C4130E16F5CD00B34460 /* _javascript_.h in Headers */,
@@ -11140,6 +11145,7 @@
E33BBE0925BFA0410053690F /* WasmStreamingCompiler.h in Headers */,
E3A0531A21342B680022EC14 /* WasmStreamingParser.h in Headers */,
E3C73A9125BFA73B00EFE303 /* WasmStreamingPlan.h in Headers */,
+ 52CAEC762790B90100DDBAAF /* SecureARM64EHashPinsInlines.h in Headers */,
AD5C36E61F69EC91000BCAAF /* WasmTable.h in Headers */,
14D01BF526DEEF5300CAE0D0 /* WasmTag.h in Headers */,
5250D2D21E8DA05A0029A932 /* WasmThunks.h in Headers */,
Modified: branches/safari-613-branch/Source/_javascript_Core/Sources.txt (288778 => 288779)
--- branches/safari-613-branch/Source/_javascript_Core/Sources.txt 2022-01-29 03:27:40 UTC (rev 288778)
+++ branches/safari-613-branch/Source/_javascript_Core/Sources.txt 2022-01-29 03:27:48 UTC (rev 288779)
@@ -61,6 +61,7 @@
assembler/Printer.cpp
assembler/ProbeContext.cpp
assembler/ProbeStack.cpp
+assembler/SecureARM64EHashPins.cpp
b3/air/AirAllocateRegistersAndStackAndGenerateCode.cpp
b3/air/AirAllocateRegistersAndStackByLinearScan.cpp
Modified: branches/safari-613-branch/Source/_javascript_Core/assembler/AssemblerBuffer.h (288778 => 288779)
--- branches/safari-613-branch/Source/_javascript_Core/assembler/AssemblerBuffer.h 2022-01-29 03:27:40 UTC (rev 288778)
+++ branches/safari-613-branch/Source/_javascript_Core/assembler/AssemblerBuffer.h 2022-01-29 03:27:48 UTC (rev 288779)
@@ -29,6 +29,7 @@
#include "ExecutableAllocator.h"
#include "JITCompilationEffort.h"
+#include "SecureARM64EHashPinsInlines.h"
#include "stdint.h"
#include <string.h>
#include <wtf/Assertions.h>
@@ -203,19 +204,48 @@
};
#if CPU(ARM64E)
+ enum class ShouldSign {
+ Yes,
+ No
+ };
+ template <ShouldSign shouldSign>
class ARM64EHash {
+ WTF_MAKE_NONCOPYABLE(ARM64EHash);
public:
- ARM64EHash(void* diversifier)
+ ARM64EHash()
{
- setUpdatedHash(0, 0, diversifier);
+ allocatePinForCurrentThreadAndInitializeHash();
}
- ALWAYS_INLINE uint32_t update(uint32_t instruction, uint32_t index, void* diversifier)
+ ~ARM64EHash()
{
- uint32_t currentHash = this->currentHash(index, diversifier);
+ deallocatePinForCurrentThread();
+ }
+
+ ALWAYS_INLINE void allocatePinForCurrentThreadAndInitializeHash()
+ {
+ if constexpr (shouldSign == ShouldSign::Yes) {
+ m_initializedPin = true;
+ g_jscConfig.arm64eHashPins.allocatePinForCurrentThread();
+ setUpdatedHash(0, 0);
+ } else
+ m_hash = 0;
+ }
+
+ void deallocatePinForCurrentThread()
+ {
+ if (m_initializedPin) {
+ g_jscConfig.arm64eHashPins.deallocatePinForCurrentThread();
+ m_initializedPin = false;
+ }
+ }
+
+ ALWAYS_INLINE uint32_t update(uint32_t instruction, uint32_t index)
+ {
+ uint32_t currentHash = this->currentHash(index);
uint64_t nextIndex = index + 1;
uint32_t output = nextValue(instruction, nextIndex, currentHash);
- setUpdatedHash(output, nextIndex, diversifier);
+ setUpdatedHash(output, nextIndex);
return output;
}
@@ -235,29 +265,28 @@
return (a >> 39) ^ (b >> 23);
}
- static ALWAYS_INLINE uint32_t bitsForDiversifier(void* diversifier)
+ static ALWAYS_INLINE uint32_t pin()
{
- return bitwise_cast<uintptr_t>(diversifier);
+ return g_jscConfig.arm64eHashPins.pinForCurrentThread();
}
- ALWAYS_INLINE uint32_t currentHash(uint32_t index, void* diversifier)
+ ALWAYS_INLINE uint32_t currentHash(uint32_t index)
{
- bool hashFieldIsTagged = index == 0;
- if (hashFieldIsTagged)
- return untagInt(m_hash, makeDiversifier(initializationNamespace, index, bitsForDiversifier(diversifier)));
+ if constexpr (shouldSign == ShouldSign::Yes)
+ return untagInt(m_hash, makeDiversifier(initializationNamespace, index, pin()));
return m_hash;
}
- ALWAYS_INLINE void setUpdatedHash(uint32_t value, uint32_t index, void* diversifier)
+ ALWAYS_INLINE void setUpdatedHash(uint32_t value, uint32_t index)
{
- bool shouldTagHashField = index == 0;
- if (shouldTagHashField)
- m_hash = tagInt(static_cast<uint64_t>(value), makeDiversifier(initializationNamespace, index, bitsForDiversifier(diversifier)));
+ if constexpr (shouldSign == ShouldSign::Yes)
+ m_hash = tagInt(static_cast<uint64_t>(value), makeDiversifier(initializationNamespace, index, pin()));
else
- m_hash = value;
+ m_hash = static_cast<uint64_t>(value);
}
uint64_t m_hash;
+ bool m_initializedPin { false };
};
#endif // CPU(ARM64E)
@@ -267,7 +296,7 @@
: m_storage()
, m_index(0)
#if CPU(ARM64E)
- , m_hash(this)
+ , m_hash()
, m_hashes()
#endif
{
@@ -409,6 +438,10 @@
void* data() const { return m_storage.buffer(); }
#endif
+#if CPU(ARM64E)
+ ARM64EHash<ShouldSign::Yes>& arm64eHash() { return m_hash; }
+#endif
+
protected:
template<typename IntegralType>
void putIntegral(IntegralType value)
@@ -425,7 +458,7 @@
#if CPU(ARM64)
static_assert(sizeof(value) == 4, "");
#if CPU(ARM64E)
- uint32_t hash = m_hash.update(value, m_index / sizeof(IntegralType), this);
+ uint32_t hash = m_hash.update(value, m_index / sizeof(IntegralType));
WTF::unalignedStore<uint32_t>(m_hashes.buffer() + m_index, hash);
#endif
#endif
@@ -459,7 +492,7 @@
AssemblerData m_storage;
unsigned m_index;
#if CPU(ARM64E)
- ARM64EHash m_hash;
+ ARM64EHash<ShouldSign::Yes> m_hash;
AssemblerData m_hashes;
#endif
};
Modified: branches/safari-613-branch/Source/_javascript_Core/assembler/LinkBuffer.cpp (288778 => 288779)
--- branches/safari-613-branch/Source/_javascript_Core/assembler/LinkBuffer.cpp 2022-01-29 03:27:40 UTC (rev 288778)
+++ branches/safari-613-branch/Source/_javascript_Core/assembler/LinkBuffer.cpp 2022-01-29 03:27:48 UTC (rev 288779)
@@ -221,8 +221,7 @@
m_assemblerStorage = macroAssembler.m_assembler.buffer().releaseAssemblerData();
uint8_t* inData = bitwise_cast<uint8_t*>(m_assemblerStorage.buffer());
#if CPU(ARM64E)
- void* bufferPtr = ¯oAssembler.m_assembler.buffer();
- ARM64EHash verifyUncompactedHash { bufferPtr };
+ ARM64EHash<ShouldSign::No> verifyUncompactedHash;
m_assemblerHashesStorage = macroAssembler.m_assembler.buffer().releaseAssemblerHashes();
uint32_t* inHashes = bitwise_cast<uint32_t*>(m_assemblerHashesStorage.buffer());
#endif
@@ -245,7 +244,7 @@
InstructionType value = *ptr;
#if CPU(ARM64E)
unsigned index = (bitwise_cast<uint8_t*>(ptr) - inData) / 4;
- uint32_t hash = verifyUncompactedHash.update(value, index, bufferPtr);
+ uint32_t hash = verifyUncompactedHash.update(value, index);
RELEASE_ASSERT(inHashes[index] == hash);
#endif
return value;
@@ -438,6 +437,10 @@
initialSize = macroAssembler.m_assembler.codeSize();
}
+#if CPU(ARM64E)
+ macroAssembler.m_assembler.buffer().arm64eHash().deallocatePinForCurrentThread();
+#endif
+
m_executableMemory = ExecutableAllocator::singleton().allocate(initialSize, effort);
if (!m_executableMemory)
return;
Added: branches/safari-613-branch/Source/_javascript_Core/assembler/SecureARM64EHashPins.cpp (0 => 288779)
--- branches/safari-613-branch/Source/_javascript_Core/assembler/SecureARM64EHashPins.cpp (rev 0)
+++ branches/safari-613-branch/Source/_javascript_Core/assembler/SecureARM64EHashPins.cpp 2022-01-29 03:27:48 UTC (rev 288779)
@@ -0,0 +1,275 @@
+/*
+ * Copyright (C) 2022 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "config.h"
+#include "SecureARM64EHashPins.h"
+
+#include "ExecutableAllocator.h"
+#include "FastJITPermissions.h"
+#include "SecureARM64EHashPinsInlines.h"
+#include <wtf/Condition.h>
+#include <wtf/Lock.h>
+
+namespace JSC {
+
+#if CPU(ARM64E) && ENABLE(JIT)
+
+struct WriteToJITRegionScope {
+ ALWAYS_INLINE WriteToJITRegionScope()
+ {
+ threadSelfRestrictRWXToRW();
+ }
+
+ ALWAYS_INLINE ~WriteToJITRegionScope()
+ {
+ threadSelfRestrictRWXToRX();
+ }
+};
+
+struct ValidateNonReentrancyScope {
+ ALWAYS_INLINE ValidateNonReentrancyScope(SecureARM64EHashPins::Metadata* metadata)
+ : m_metadata(metadata)
+ {
+ uint32_t validateNonReentrancy = m_metadata->assertNotReentrant.exchange(1, std::memory_order_seq_cst);
+ RELEASE_ASSERT(!validateNonReentrancy);
+ }
+
+ ALWAYS_INLINE ~ValidateNonReentrancyScope()
+ {
+ uint32_t validateNonReentrancy = m_metadata->assertNotReentrant.exchange(0, std::memory_order_seq_cst);
+ RELEASE_ASSERT(validateNonReentrancy == 1);
+ }
+ SecureARM64EHashPins::Metadata* m_metadata;
+};
+
+// This class is allocated once per process, so static lock is ok.
+static Lock hashPinsLock;
+
+ALWAYS_INLINE static ExecutableMemoryHandle::MemoryPtr allocateInExecutableMemory(size_t size)
+{
+ ExecutableMemoryHandle* handle = ExecutableAllocator::singleton().allocate(size, JITCompilationMustSucceed).leakRef();
+ RELEASE_ASSERT(handle);
+ void* memory = handle->start().untaggedPtr<char*>();
+ RELEASE_ASSERT(isJITPC(memory) && isJITPC(bitwise_cast<char*>(memory) + size - 1));
+ return handle->start();
+}
+
+ALWAYS_INLINE SecureARM64EHashPins::Page::Page()
+{
+ for (unsigned i = 0; i < numEntriesPerPage; ++i) {
+ auto& entry = this->entry(i);
+ entry.pin = 0;
+ entry.key = 0;
+ }
+}
+
+static ALWAYS_INLINE void initializePage(const WriteToJITRegionScope&, SecureARM64EHashPins::Page* page)
+{
+ new (page) SecureARM64EHashPins::Page;
+}
+
+#define VALIDATE_THIS_VALUE() RELEASE_ASSERT(this == &g_jscConfig.arm64eHashPins)
+
+ALWAYS_INLINE auto SecureARM64EHashPins::metadata() -> Metadata*
+{
+ VALIDATE_THIS_VALUE();
+ return bitwise_cast<Metadata*>(m_memory) - 1;
+}
+
+void SecureARM64EHashPins::initializeAtStartup()
+{
+ VALIDATE_THIS_VALUE();
+ RELEASE_ASSERT(!m_memory);
+
+ m_memory = allocateInExecutableMemory(sizeof(Metadata) + Page::allocationSize()).untaggedPtr<char*>() + sizeof(Metadata);
+
+ {
+ WriteToJITRegionScope writeScope;
+
+ new (metadata()) Metadata;
+ initializePage(writeScope, firstPage());
+ }
+}
+
+bool SecureARM64EHashPins::allocatePinForCurrentThreadImpl(const AbstractLocker&)
+{
+ VALIDATE_THIS_VALUE();
+
+ size_t newEntryIndex;
+ Page* newEntryPage;
+
+ bool found = false;
+ size_t baseIndex = 0;
+ forEachPage([&] (Page& page) {
+ size_t bit = page.findClearBit();
+ if (bit < numEntriesPerPage) {
+ found = true;
+ newEntryIndex = baseIndex + bit;
+ newEntryPage = &page;
+ return IterationStatus::Done;
+ }
+
+ baseIndex += numEntriesPerPage;
+ return IterationStatus::Continue;
+ });
+
+ if (!found)
+ return false;
+
+ auto findResult = findFirstEntry();
+ Entry* preexistingEntry = findResult.entry;
+ size_t preexistingEntryIndex = findResult.index;
+ Page* preexistingEntryPage = findResult.page;
+
+ {
+ auto* metadata = this->metadata();
+
+ WriteToJITRegionScope writeScope;
+ ValidateNonReentrancyScope validateNonReentrancy(metadata);
+
+ RELEASE_ASSERT(isJITPC(metadata));
+ uint64_t nextPin = metadata->nextPin.exchangeAdd(1, std::memory_order_seq_cst);
+ RELEASE_ASSERT(nextPin);
+
+ newEntryPage->setIsInUse(newEntryIndex);
+
+ if (preexistingEntry && preexistingEntryIndex < newEntryIndex) {
+ RELEASE_ASSERT(preexistingEntryPage->isInUse(preexistingEntryIndex));
+
+ newEntryPage->entry(newEntryIndex) = *preexistingEntry;
+ newEntryIndex = preexistingEntryIndex;
+ newEntryPage = preexistingEntryPage;
+ }
+
+ auto& entry = newEntryPage->entry(newEntryIndex);
+ entry.pin = nextPin;
+ entry.key = keyForCurrentThread();
+ }
+
+ return true;
+}
+
+void SecureARM64EHashPins::allocatePinForCurrentThread()
+{
+ VALIDATE_THIS_VALUE();
+
+ Locker locker { hashPinsLock };
+
+ if (allocatePinForCurrentThreadImpl(locker))
+ return;
+
+ // Allocate a new page
+ {
+ Page* lastPage = firstPage();
+ while (lastPage->next)
+ lastPage = lastPage->next;
+ RELEASE_ASSERT(!lastPage->next);
+
+ auto newPage = allocateInExecutableMemory(Page::allocationSize());
+
+ {
+ WriteToJITRegionScope writeScope;
+ ValidateNonReentrancyScope validateNonReentrancy(metadata());
+ initializePage(writeScope, newPage.untaggedPtr<Page*>());
+ // This can be read from concurrently. Make sure it's in a good state for that to happen.
+ WTF::storeStoreFence();
+ lastPage->next = newPage.untaggedPtr<Page*>();
+ }
+ }
+
+ bool success = allocatePinForCurrentThreadImpl(locker);
+ RELEASE_ASSERT(success);
+}
+
+void SecureARM64EHashPins::deallocatePinForCurrentThread()
+{
+ VALIDATE_THIS_VALUE();
+
+ Locker locker { hashPinsLock };
+
+ auto clear = [] (Entry& entry) {
+ entry.pin = 0;
+ entry.key = 0;
+ };
+
+ Page* removalPage;
+ size_t removalIndex;
+ {
+ auto findResult = findFirstEntry();
+ Entry& entry = *findResult.entry;
+ removalIndex = findResult.index;
+ removalPage = findResult.page;
+
+ WriteToJITRegionScope writeScope;
+ ValidateNonReentrancyScope validateNonReentrancy(metadata());
+
+ clear(entry);
+ removalPage->clearIsInUse(removalIndex);
+ }
+
+ // This implementation allows recursive uses of the MacroAssembler, forming a stack
+ // like data structure. The in use pin (top of the stack) will always be at the lowest
+ // page and lowest index in the bit vector. So when we deallocate the current top of the
+ // stack, we need to find the next entry in the stack, and ensure it's at the lowest index.
+ // So if there are other entries for this current thread, we find the one with the highest
+ // pin (next top of stack value), and replace it at the index we were using before.
+ // Allocation also maintains this invariant by always placing the newest entry for
+ // the current thread at the lowest index.
+ {
+ uint64_t key = keyForCurrentThread();
+ bool found = false;
+ uint64_t maxPin = 0;
+ size_t maxIndex = std::numeric_limits<size_t>::max();
+ Page* maxPage = nullptr;
+
+ forEachEntry([&] (Page& page, Entry& entry, size_t index) {
+ RELEASE_ASSERT(entry.pin);
+ if (entry.key == key && entry.pin > maxPin) {
+ found = true;
+ maxPin = entry.pin;
+ maxIndex = index;
+ maxPage = &page;
+ }
+
+ return IterationStatus::Continue;
+ });
+
+ if (found) {
+ RELEASE_ASSERT(removalIndex < maxIndex);
+
+ WriteToJITRegionScope writeScope;
+ ValidateNonReentrancyScope validateNonReentrancy(metadata());
+
+ removalPage->entry(removalIndex) = maxPage->entry(maxIndex);
+ clear(maxPage->entry(maxIndex));
+ maxPage->clearIsInUse(maxIndex);
+ removalPage->setIsInUse(removalIndex);
+ }
+ }
+}
+
+#endif // CPU(ARM64E) && ENABLE(JIT)
+
+} // namespace JSC
Added: branches/safari-613-branch/Source/_javascript_Core/assembler/SecureARM64EHashPins.h (0 => 288779)
--- branches/safari-613-branch/Source/_javascript_Core/assembler/SecureARM64EHashPins.h (rev 0)
+++ branches/safari-613-branch/Source/_javascript_Core/assembler/SecureARM64EHashPins.h 2022-01-29 03:27:48 UTC (rev 288779)
@@ -0,0 +1,135 @@
+/*
+ * Copyright (C) 2022 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+#if CPU(ARM64E) && ENABLE(JIT)
+
+#include <wtf/Bitmap.h>
+
+namespace JSC {
+
+class SecureARM64EHashPins {
+public:
+ JS_EXPORT_PRIVATE void initializeAtStartup();
+ JS_EXPORT_PRIVATE void allocatePinForCurrentThread();
+ JS_EXPORT_PRIVATE void deallocatePinForCurrentThread();
+ uint64_t pinForCurrentThread();
+
+ static constexpr size_t numEntriesPerPage = 64;
+
+ struct Entry {
+ uint64_t pin;
+ uint64_t key;
+ };
+
+ struct alignas(alignof(Entry)) Page {
+ Page();
+
+ static size_t allocationSize() { return sizeof(Page) + numEntriesPerPage * sizeof(Entry); }
+
+ static constexpr size_t mask = numEntriesPerPage - 1;
+ static_assert(hasOneBitSet(numEntriesPerPage));
+ static_assert(!!mask);
+ static_assert(!(mask & numEntriesPerPage));
+
+ ALWAYS_INLINE Entry& entry(size_t index)
+ {
+ return entries()[index & mask];
+ }
+
+ ALWAYS_INLINE Entry& fastEntryUnchecked(size_t index)
+ {
+ ASSERT((index & mask) == index);
+ return entries()[index];
+ }
+
+ ALWAYS_INLINE void setIsInUse(size_t index)
+ {
+ isInUseMap.set(index & mask);
+ }
+
+ ALWAYS_INLINE void clearIsInUse(size_t index)
+ {
+ isInUseMap.set(index & mask, false);
+ }
+
+ ALWAYS_INLINE bool isInUse(size_t index)
+ {
+ return isInUseMap.get(index & mask);
+ }
+
+ ALWAYS_INLINE size_t findClearBit()
+ {
+ return isInUseMap.findBit(0, false);
+ }
+
+ template <typename Function>
+ ALWAYS_INLINE void forEachSetBit(Function function)
+ {
+ isInUseMap.forEachSetBit(function);
+ }
+
+ Page* next { nullptr };
+ private:
+ Entry* entries() { return bitwise_cast<Entry*>(this + 1); }
+ Bitmap<numEntriesPerPage> isInUseMap;
+ };
+
+ static_assert(sizeof(Page) % alignof(Entry) == 0);
+
+ struct alignas(alignof(Page)) Metadata {
+ Atomic<uint64_t> nextPin { 1 };
+ Atomic<uint32_t> assertNotReentrant { 0 };
+ };
+
+ static_assert(sizeof(Metadata) % alignof(Page) == 0);
+
+private:
+ static uint64_t keyForCurrentThread();
+ bool allocatePinForCurrentThreadImpl(const AbstractLocker&);
+
+ struct FindResult {
+ Entry* entry { nullptr };
+ size_t index { std::numeric_limits<size_t>::max() };
+ Page* page { nullptr };
+ };
+ FindResult findFirstEntry();
+
+ Metadata* metadata();
+ inline Page* firstPage() { return bitwise_cast<Page*>(m_memory); }
+
+ template <typename Function>
+ void forEachPage(Function);
+
+ template <typename Function>
+ void forEachEntry(Function);
+
+ void* m_memory { nullptr };
+};
+
+} // namespace JSC
+
+#endif // CPU(ARM64E) && ENABLE(JIT)
Added: branches/safari-613-branch/Source/_javascript_Core/assembler/SecureARM64EHashPinsInlines.h (0 => 288779)
--- branches/safari-613-branch/Source/_javascript_Core/assembler/SecureARM64EHashPinsInlines.h (rev 0)
+++ branches/safari-613-branch/Source/_javascript_Core/assembler/SecureARM64EHashPinsInlines.h 2022-01-29 03:27:48 UTC (rev 288779)
@@ -0,0 +1,106 @@
+/*
+ * Copyright (C) 2022 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+#include "SecureARM64EHashPins.h"
+
+#if CPU(ARM64E) && ENABLE(JIT)
+
+namespace JSC {
+
+ALWAYS_INLINE uint64_t SecureARM64EHashPins::keyForCurrentThread()
+{
+ uint64_t result;
+ asm (
+ "mrs %x[result], TPIDRRO_EL0"
+ : [result] "=r" (result)
+ :
+ :
+ );
+ return result + 1;
+}
+
+template <typename Function>
+ALWAYS_INLINE void SecureARM64EHashPins::forEachPage(Function function)
+{
+ for (Page* page = firstPage(); page; page = page->next) {
+ RELEASE_ASSERT(isJITPC(page));
+ if (function(*page) == IterationStatus::Done)
+ return;
+ }
+}
+
+template <typename Function>
+ALWAYS_INLINE void SecureARM64EHashPins::forEachEntry(Function function)
+{
+ size_t baseIndex = 0;
+ forEachPage([&] (Page& page) {
+ IterationStatus iterationStatus = IterationStatus::Continue;
+ page.forEachSetBit([&] (size_t bitIndex) {
+ Entry& entry = page.fastEntryUnchecked(bitIndex);
+ ASSERT(isJITPC(&entry));
+ size_t index = baseIndex + bitIndex;
+ iterationStatus = function(page, entry, index);
+ return iterationStatus;
+ });
+ baseIndex += numEntriesPerPage;
+ return iterationStatus;
+ });
+}
+
+ALWAYS_INLINE auto SecureARM64EHashPins::findFirstEntry() -> FindResult
+{
+ uint64_t key = keyForCurrentThread();
+ // We can call this concurrently to the bit vector being modified
+ // since we either call this when we're locked, or we call it when
+ // we know the entry already exists. When the entry already exists,
+ // we know that the bit can't get flipped spuriously during concurrent
+ // modification, so we're guaranteed to find the value. We might see
+ // an entry as it's being written to, but that's also fine, since it
+ // won't have our same key. It'll either be zero, or a different key.
+
+ FindResult result;
+ forEachEntry([&] (Page& page, Entry& entry, size_t index) {
+ if (entry.key == key) {
+ result.entry = &entry;
+ result.page = &page;
+ result.index = index;
+ return IterationStatus::Done;
+ }
+ return IterationStatus::Continue;
+ });
+
+ return result;
+}
+
+ALWAYS_INLINE uint64_t SecureARM64EHashPins::pinForCurrentThread()
+{
+ return findFirstEntry().entry->pin;
+}
+
+} // namespace JSC
+
+#endif // CPU(ARM64E) && ENABLE(JIT)
Modified: branches/safari-613-branch/Source/_javascript_Core/heap/MarkedBlock.h (288778 => 288779)
--- branches/safari-613-branch/Source/_javascript_Core/heap/MarkedBlock.h 2022-01-29 03:27:40 UTC (rev 288778)
+++ branches/safari-613-branch/Source/_javascript_Core/heap/MarkedBlock.h 2022-01-29 03:27:48 UTC (rev 288779)
@@ -24,7 +24,6 @@
#include "CellAttributes.h"
#include "DestructionMode.h"
#include "HeapCell.h"
-#include "IterationStatus.h"
#include "WeakSet.h"
#include <algorithm>
#include <wtf/Atomics.h>
@@ -31,6 +30,7 @@
#include <wtf/Bitmap.h>
#include <wtf/CountingLock.h>
#include <wtf/HashFunctions.h>
+#include <wtf/IterationStatus.h>
#include <wtf/PageBlock.h>
#include <wtf/StdLibExtras.h>
Modified: branches/safari-613-branch/Source/_javascript_Core/heap/MarkedSpace.h (288778 => 288779)
--- branches/safari-613-branch/Source/_javascript_Core/heap/MarkedSpace.h 2022-01-29 03:27:40 UTC (rev 288778)
+++ branches/safari-613-branch/Source/_javascript_Core/heap/MarkedSpace.h 2022-01-29 03:27:48 UTC (rev 288779)
@@ -22,7 +22,6 @@
#pragma once
#include "BlockDirectory.h"
-#include "IterationStatus.h"
#include "MarkedBlock.h"
#include "MarkedBlockSet.h"
#include "PreciseAllocation.h"
@@ -29,6 +28,7 @@
#include <array>
#include <wtf/Bag.h>
#include <wtf/HashSet.h>
+#include <wtf/IterationStatus.h>
#include <wtf/Noncopyable.h>
#include <wtf/RetainPtr.h>
#include <wtf/SentinelLinkedList.h>
Modified: branches/safari-613-branch/Source/_javascript_Core/heap/SlotVisitor.h (288778 => 288779)
--- branches/safari-613-branch/Source/_javascript_Core/heap/SlotVisitor.h 2022-01-29 03:27:40 UTC (rev 288778)
+++ branches/safari-613-branch/Source/_javascript_Core/heap/SlotVisitor.h 2022-01-29 03:27:48 UTC (rev 288779)
@@ -27,8 +27,8 @@
#include "AbstractSlotVisitor.h"
#include "HandleTypes.h"
-#include "IterationStatus.h"
#include <wtf/Forward.h>
+#include <wtf/IterationStatus.h>
#include <wtf/MonotonicTime.h>
namespace JSC {
Modified: branches/safari-613-branch/Source/_javascript_Core/jit/BaselineJITPlan.cpp (288778 => 288779)
--- branches/safari-613-branch/Source/_javascript_Core/jit/BaselineJITPlan.cpp 2022-01-29 03:27:40 UTC (rev 288778)
+++ branches/safari-613-branch/Source/_javascript_Core/jit/BaselineJITPlan.cpp 2022-01-29 03:27:48 UTC (rev 288779)
@@ -34,11 +34,17 @@
: JITPlan(JITCompilationMode::Baseline, codeBlock)
, m_jit(codeBlock->vm(), codeBlock, loopOSREntryBytecodeIndex)
{
+#if CPU(ARM64E)
+ m_jit.m_assembler.buffer().arm64eHash().deallocatePinForCurrentThread();
+#endif
m_jit.doMainThreadPreparationBeforeCompile();
}
auto BaselineJITPlan::compileInThreadImpl() -> CompilationPath
{
+#if CPU(ARM64E)
+ m_jit.m_assembler.buffer().arm64eHash().allocatePinForCurrentThreadAndInitializeHash();
+#endif
m_jit.compileAndLinkWithoutFinalizing(JITCompilationCanFail);
return BaselinePath;
}
Modified: branches/safari-613-branch/Source/_javascript_Core/jit/ExecutableAllocator.cpp (288778 => 288779)
--- branches/safari-613-branch/Source/_javascript_Core/jit/ExecutableAllocator.cpp 2022-01-29 03:27:40 UTC (rev 288778)
+++ branches/safari-613-branch/Source/_javascript_Core/jit/ExecutableAllocator.cpp 2022-01-29 03:27:48 UTC (rev 288779)
@@ -29,11 +29,11 @@
#if ENABLE(JIT)
#include "ExecutableAllocationFuzz.h"
-#include "IterationStatus.h"
#include "JITOperationValidation.h"
#include "LinkBuffer.h"
#include <wtf/FastBitVector.h>
#include <wtf/FileSystem.h>
+#include <wtf/IterationStatus.h>
#include <wtf/PageReservation.h>
#include <wtf/ProcessID.h>
#include <wtf/RedBlackTree.h>
@@ -400,8 +400,8 @@
#endif
void* reservationEnd = reinterpret_cast<uint8_t*>(reservation.base) + reservation.size;
- g_jscConfig.startExecutableMemory = tagCodePtr<ExecutableMemoryPtrTag>(reservation.base);
- g_jscConfig.endExecutableMemory = tagCodePtr<ExecutableMemoryPtrTag>(reservationEnd);
+ g_jscConfig.startExecutableMemory = reservation.base;
+ g_jscConfig.endExecutableMemory = reservationEnd;
#if !USE(SYSTEM_MALLOC) && ENABLE(UNIFIED_AND_FREEZABLE_CONFIG_RECORD)
WebConfig::g_config[0] = bitwise_cast<uintptr_t>(reservation.base);
@@ -471,9 +471,8 @@
m_reservation.deallocate();
}
- void* memoryStart() { return untagCodePtr<ExecutableMemoryPtrTag>(g_jscConfig.startExecutableMemory); }
- void* memoryEnd() { return untagCodePtr<ExecutableMemoryPtrTag>(g_jscConfig.endExecutableMemory); }
- bool isJITPC(void* pc) { return memoryStart() <= pc && pc < memoryEnd(); }
+ void* memoryStart() { return g_jscConfig.startExecutableMemory; }
+ void* memoryEnd() { return g_jscConfig.endExecutableMemory; }
bool isValid() { return !!m_reservation; }
RefPtr<ExecutableMemoryHandle> allocate(size_t sizeInBytes)
@@ -1129,12 +1128,6 @@
return allocator->memoryEnd();
}
-bool isJITPC(void* pc)
-{
- FixedVMPoolExecutableAllocator* allocator = g_jscConfig.fixedVMPoolExecutableAllocator;
- return allocator && allocator->isJITPC(pc);
-}
-
void dumpJITMemory(const void* dst, const void* src, size_t size)
{
RELEASE_ASSERT(Options::dumpJITMemoryPath());
Modified: branches/safari-613-branch/Source/_javascript_Core/jit/ExecutableAllocator.h (288778 => 288779)
--- branches/safari-613-branch/Source/_javascript_Core/jit/ExecutableAllocator.h 2022-01-29 03:27:40 UTC (rev 288778)
+++ branches/safari-613-branch/Source/_javascript_Core/jit/ExecutableAllocator.h 2022-01-29 03:27:48 UTC (rev 288779)
@@ -108,7 +108,10 @@
return bitwise_cast<T>(endOfFixedExecutableMemoryPoolImpl());
}
-JS_EXPORT_PRIVATE bool isJITPC(void* pc);
+ALWAYS_INLINE bool isJITPC(void* pc)
+{
+ return g_jscConfig.startExecutableMemory <= pc && pc < g_jscConfig.endExecutableMemory;
+}
JS_EXPORT_PRIVATE void dumpJITMemory(const void*, const void*, size_t);
Modified: branches/safari-613-branch/Source/_javascript_Core/parser/Parser.h (288778 => 288779)
--- branches/safari-613-branch/Source/_javascript_Core/parser/Parser.h 2022-01-29 03:27:40 UTC (rev 288778)
+++ branches/safari-613-branch/Source/_javascript_Core/parser/Parser.h 2022-01-29 03:27:48 UTC (rev 288779)
@@ -23,7 +23,6 @@
#pragma once
#include "ExecutableInfo.h"
-#include "IterationStatus.h"
#include "Lexer.h"
#include "ModuleScopeData.h"
#include "Nodes.h"
@@ -38,6 +37,7 @@
#include "VariableEnvironment.h"
#include <wtf/FixedVector.h>
#include <wtf/Forward.h>
+#include <wtf/IterationStatus.h>
#include <wtf/Noncopyable.h>
#include <wtf/RefPtr.h>
Modified: branches/safari-613-branch/Source/_javascript_Core/runtime/InitializeThreading.cpp (288778 => 288779)
--- branches/safari-613-branch/Source/_javascript_Core/runtime/InitializeThreading.cpp 2022-01-29 03:27:40 UTC (rev 288778)
+++ branches/safari-613-branch/Source/_javascript_Core/runtime/InitializeThreading.cpp 2022-01-29 03:27:48 UTC (rev 288779)
@@ -71,6 +71,10 @@
if (!g_jscConfig.vm.canUseJIT) {
Options::useJIT() = false;
Options::recomputeDependentOptions();
+ } else {
+#if CPU(ARM64E) && ENABLE(JIT)
+ g_jscConfig.arm64eHashPins.initializeAtStartup();
+#endif
}
}
Options::finalize();
Deleted: branches/safari-613-branch/Source/_javascript_Core/runtime/IterationStatus.h (288778 => 288779)
--- branches/safari-613-branch/Source/_javascript_Core/runtime/IterationStatus.h 2022-01-29 03:27:40 UTC (rev 288778)
+++ branches/safari-613-branch/Source/_javascript_Core/runtime/IterationStatus.h 2022-01-29 03:27:48 UTC (rev 288779)
@@ -1,35 +0,0 @@
-/*
- * Copyright (C) 2015 Apple Inc. All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
- * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
- * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
- * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
- * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
- * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
- * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#pragma once
-
-namespace JSC {
-
-enum class IterationStatus {
- Continue,
- Done
-};
-
-} // namespace JSC
Modified: branches/safari-613-branch/Source/_javascript_Core/runtime/JSCConfig.h (288778 => 288779)
--- branches/safari-613-branch/Source/_javascript_Core/runtime/JSCConfig.h 2022-01-29 03:27:40 UTC (rev 288778)
+++ branches/safari-613-branch/Source/_javascript_Core/runtime/JSCConfig.h 2022-01-29 03:27:48 UTC (rev 288779)
@@ -28,6 +28,7 @@
#include "Gate.h"
#include "Opcode.h"
#include "OptionsList.h"
+#include "SecureARM64EHashPins.h"
#include <wtf/WTFConfig.h>
namespace JSC {
@@ -99,6 +100,10 @@
#if CPU(ARM64E) && ENABLE(PTRTAG_DEBUGGING)
WTF::PtrTagLookup ptrTagLookupRecord;
#endif
+
+#if CPU(ARM64E) && ENABLE(JIT)
+ SecureARM64EHashPins arm64eHashPins;
+#endif
};
#if ENABLE(UNIFIED_AND_FREEZABLE_CONFIG_RECORD)
Modified: branches/safari-613-branch/Source/_javascript_Core/wasm/WasmAirIRGenerator.cpp (288778 => 288779)
--- branches/safari-613-branch/Source/_javascript_Core/wasm/WasmAirIRGenerator.cpp 2022-01-29 03:27:40 UTC (rev 288778)
+++ branches/safari-613-branch/Source/_javascript_Core/wasm/WasmAirIRGenerator.cpp 2022-01-29 03:27:48 UTC (rev 288779)
@@ -3879,7 +3879,6 @@
{
auto result = makeUnique<InternalFunction>();
- compilationContext.embedderEntrypointJIT = makeUnique<CCallHelpers>();
compilationContext.wasmEntrypointJIT = makeUnique<CCallHelpers>();
compilationContext.procedure = makeUnique<B3::Procedure>();
Modified: branches/safari-613-branch/Source/_javascript_Core/wasm/WasmB3IRGenerator.cpp (288778 => 288779)
--- branches/safari-613-branch/Source/_javascript_Core/wasm/WasmB3IRGenerator.cpp 2022-01-29 03:27:40 UTC (rev 288778)
+++ branches/safari-613-branch/Source/_javascript_Core/wasm/WasmB3IRGenerator.cpp 2022-01-29 03:27:48 UTC (rev 288779)
@@ -3142,7 +3142,6 @@
{
auto result = makeUnique<InternalFunction>();
- compilationContext.embedderEntrypointJIT = makeUnique<CCallHelpers>();
compilationContext.wasmEntrypointJIT = makeUnique<CCallHelpers>();
compilationContext.procedure = makeUnique<Procedure>();
@@ -3150,7 +3149,6 @@
if (shouldDumpIRFor(functionIndex + info.importFunctionCount()))
procedure.setShouldDumpIR();
- compilationContext.wasmEntrypointJIT = makeUnique<CCallHelpers>();
if (Options::useSamplingProfiler()) {
// FIXME: We should do this based on VM relevant info.
Modified: branches/safari-613-branch/Source/_javascript_Core/wasm/WasmBBQPlan.cpp (288778 => 288779)
--- branches/safari-613-branch/Source/_javascript_Core/wasm/WasmBBQPlan.cpp 2022-01-29 03:27:40 UTC (rev 288778)
+++ branches/safari-613-branch/Source/_javascript_Core/wasm/WasmBBQPlan.cpp 2022-01-29 03:27:48 UTC (rev 288779)
@@ -63,11 +63,13 @@
{
const auto& functions = m_moduleInformation->functions;
if (!tryReserveCapacity(m_wasmInternalFunctions, functions.size(), " WebAssembly functions")
+ || !tryReserveCapacity(m_wasmInternalFunctionLinkBuffers, functions.size(), " compilation contexts")
|| !tryReserveCapacity(m_compilationContexts, functions.size(), " compilation contexts")
|| !tryReserveCapacity(m_tierUpCounts, functions.size(), " tier-up counts"))
return false;
m_wasmInternalFunctions.resize(functions.size());
+ m_wasmInternalFunctionLinkBuffers.resize(functions.size());
m_exceptionHandlerLocations.resize(functions.size());
m_compilationContexts.resize(functions.size());
m_tierUpCounts.resize(functions.size());
@@ -178,12 +180,22 @@
m_tierUpCounts[functionIndex] = nullptr;
m_wasmInternalFunctions[functionIndex] = compileFunction(functionIndex, m_compilationContexts[functionIndex], m_unlinkedWasmToWasmCalls[functionIndex], m_tierUpCounts[functionIndex].get());
+ {
+ auto linkBuffer = makeUnique<LinkBuffer>(*m_compilationContexts[functionIndex].wasmEntrypointJIT, nullptr, LinkBuffer::Profile::Wasm, JITCompilationCanFail);
+ if (linkBuffer->isValid())
+ m_wasmInternalFunctionLinkBuffers[functionIndex] = WTFMove(linkBuffer);
+ }
if (m_exportedFunctionIndices.contains(functionIndex) || m_moduleInformation->referencedFunctions().contains(functionIndex)) {
Locker locker { m_lock };
SignatureIndex signatureIndex = m_moduleInformation->internalFunctionSignatureIndices[functionIndex];
const Signature& signature = SignatureInformation::get(signatureIndex);
- auto result = m_embedderToWasmInternalFunctions.add(functionIndex, createJSToWasmWrapper(*m_compilationContexts[functionIndex].embedderEntrypointJIT, signature, &m_unlinkedWasmToWasmCalls[functionIndex], m_moduleInformation.get(), m_mode, functionIndex));
+
+ m_compilationContexts[functionIndex].embedderEntrypointJIT = makeUnique<CCallHelpers>();
+ auto embedderToWasmInternalFunction = createJSToWasmWrapper(*m_compilationContexts[functionIndex].embedderEntrypointJIT, signature, &m_unlinkedWasmToWasmCalls[functionIndex], m_moduleInformation.get(), m_mode, functionIndex);
+ auto linkBuffer = makeUnique<LinkBuffer>(*m_compilationContexts[functionIndex].embedderEntrypointJIT, nullptr, LinkBuffer::Profile::Wasm, JITCompilationCanFail);
+
+ auto result = m_embedderToWasmInternalFunctions.add(functionIndex, std::pair { WTFMove(linkBuffer), WTFMove(embedderToWasmInternalFunction) });
ASSERT_UNUSED(result, result.isNewEntry);
}
}
@@ -235,11 +247,12 @@
ASSERT(functionIndexSpace < m_moduleInformation->functionIndexSpaceSize());
{
InternalFunction* function = m_wasmInternalFunctions[functionIndex].get();
- LinkBuffer linkBuffer(*context.wasmEntrypointJIT, nullptr, LinkBuffer::Profile::Wasm, JITCompilationCanFail);
- if (UNLIKELY(linkBuffer.didFailToAllocate())) {
+ if (!m_wasmInternalFunctionLinkBuffers[functionIndex]) {
Base::fail(makeString("Out of executable memory in function at index ", String::number(functionIndex)));
return;
}
+
+ auto& linkBuffer = *m_wasmInternalFunctionLinkBuffers[functionIndex];
computeExceptionHandlerLocations(m_exceptionHandlerLocations[functionIndex], function, context, linkBuffer);
@@ -250,16 +263,21 @@
WTFMove(context.wasmEntrypointByproducts));
}
- if (const auto& embedderToWasmInternalFunction = m_embedderToWasmInternalFunctions.get(functionIndex)) {
- LinkBuffer linkBuffer(*context.embedderEntrypointJIT, nullptr, LinkBuffer::Profile::Wasm, JITCompilationCanFail);
- if (UNLIKELY(linkBuffer.didFailToAllocate())) {
- Base::fail(makeString("Out of executable memory in function entrypoint at index ", String::number(functionIndex)));
- return;
+ {
+ auto iter = m_embedderToWasmInternalFunctions.find(functionIndex);
+ if (iter != m_embedderToWasmInternalFunctions.end()) {
+ LinkBuffer& linkBuffer = *iter->value.first;
+ const auto& embedderToWasmInternalFunction = iter->value.second;
+
+ if (linkBuffer.didFailToAllocate()) {
+ Base::fail(makeString("Out of executable memory in function entrypoint at index ", String::number(functionIndex)));
+ return;
+ }
+
+ embedderToWasmInternalFunction->entrypoint.compilation = makeUnique<Compilation>(
+ FINALIZE_CODE(linkBuffer, JITCompilationPtrTag, "Embedder->WebAssembly entrypoint[%i] %s name %s", functionIndex, signature.toString().ascii().data(), makeString(IndexOrName(functionIndexSpace, m_moduleInformation->nameSection->get(functionIndexSpace))).ascii().data()),
+ nullptr);
}
-
- embedderToWasmInternalFunction->entrypoint.compilation = makeUnique<Compilation>(
- FINALIZE_CODE(linkBuffer, JITCompilationPtrTag, "Embedder->WebAssembly entrypoint[%i] %s name %s", functionIndex, signature.toString().ascii().data(), makeString(IndexOrName(functionIndexSpace, m_moduleInformation->nameSection->get(functionIndexSpace))).ascii().data()),
- nullptr);
}
}
@@ -280,11 +298,15 @@
{
ASSERT(!failed());
for (unsigned internalFunctionIndex = 0; internalFunctionIndex < m_wasmInternalFunctions.size(); ++internalFunctionIndex) {
+ RefPtr<EmbedderEntrypointCallee> embedderEntrypointCallee;
- RefPtr<EmbedderEntrypointCallee> embedderEntrypointCallee;
- if (auto embedderToWasmFunction = m_embedderToWasmInternalFunctions.get(internalFunctionIndex)) {
- embedderEntrypointCallee = EmbedderEntrypointCallee::create(WTFMove(embedderToWasmFunction->entrypoint));
- MacroAssembler::repatchPointer(embedderToWasmFunction->calleeMoveLocation, CalleeBits::boxWasm(embedderEntrypointCallee.get()));
+ {
+ auto iter = m_embedderToWasmInternalFunctions.find(internalFunctionIndex);
+ if (iter != m_embedderToWasmInternalFunctions.end()) {
+ const auto& embedderToWasmFunction = iter->value.second;
+ embedderEntrypointCallee = EmbedderEntrypointCallee::create(WTFMove(embedderToWasmFunction->entrypoint));
+ MacroAssembler::repatchPointer(embedderToWasmFunction->calleeMoveLocation, CalleeBits::boxWasm(embedderEntrypointCallee.get()));
+ }
}
InternalFunction* function = m_wasmInternalFunctions[internalFunctionIndex].get();
Modified: branches/safari-613-branch/Source/_javascript_Core/wasm/WasmBBQPlan.h (288778 => 288779)
--- branches/safari-613-branch/Source/_javascript_Core/wasm/WasmBBQPlan.h 2022-01-29 03:27:40 UTC (rev 288778)
+++ branches/safari-613-branch/Source/_javascript_Core/wasm/WasmBBQPlan.h 2022-01-29 03:27:48 UTC (rev 288779)
@@ -83,8 +83,9 @@
std::unique_ptr<InternalFunction> compileFunction(uint32_t functionIndex, CompilationContext&, Vector<UnlinkedWasmToWasmCall>&, TierUpCount*);
Vector<std::unique_ptr<InternalFunction>> m_wasmInternalFunctions;
+ Vector<std::unique_ptr<LinkBuffer>> m_wasmInternalFunctionLinkBuffers;
Vector<Vector<CodeLocationLabel<ExceptionHandlerPtrTag>>> m_exceptionHandlerLocations;
- HashMap<uint32_t, std::unique_ptr<InternalFunction>, DefaultHash<uint32_t>, WTF::UnsignedWithZeroKeyHashTraits<uint32_t>> m_embedderToWasmInternalFunctions;
+ HashMap<uint32_t, std::pair<std::unique_ptr<LinkBuffer>, std::unique_ptr<InternalFunction>>, DefaultHash<uint32_t>, WTF::UnsignedWithZeroKeyHashTraits<uint32_t>> m_embedderToWasmInternalFunctions;
Vector<CompilationContext> m_compilationContexts;
Vector<std::unique_ptr<TierUpCount>> m_tierUpCounts;
Modified: branches/safari-613-branch/Source/WTF/WTF.xcodeproj/project.pbxproj (288778 => 288779)
--- branches/safari-613-branch/Source/WTF/WTF.xcodeproj/project.pbxproj 2022-01-29 03:27:40 UTC (rev 288778)
+++ branches/safari-613-branch/Source/WTF/WTF.xcodeproj/project.pbxproj 2022-01-29 03:27:48 UTC (rev 288779)
@@ -418,6 +418,7 @@
51F1752A1F3D486000C74950 /* PersistentEncoder.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = PersistentEncoder.h; sourceTree = "<group>"; };
521CC6B324A277C2004377D6 /* TranslatedProcess.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = TranslatedProcess.cpp; sourceTree = "<group>"; };
526AEC911F6B4E5C00695F5D /* NoTailCalls.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = NoTailCalls.h; sourceTree = "<group>"; };
+ 5295B00427920C54006D746A /* IterationStatus.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = IterationStatus.h; sourceTree = "<group>"; };
52B228C12458DC8200753D91 /* ICUHelpers.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = ICUHelpers.h; sourceTree = "<group>"; };
530A63481FA3E3A00026A545 /* Scripts */ = {isa = PBXFileReference; lastKnownFileType = folder; path = Scripts; sourceTree = "<group>"; };
5311BD511EA71CAD00525281 /* Signals.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = Signals.cpp; sourceTree = "<group>"; };
@@ -1128,6 +1129,7 @@
E33667492722550900259122 /* Int128.cpp */,
F6D67D3226F90142006E0349 /* Int128.h */,
33FD4811265CB38000ABE4F4 /* InterferenceGraph.h */,
+ 5295B00427920C54006D746A /* IterationStatus.h */,
0F7EB85B1FA8FF4100F1ABCB /* IsoMalloc.h */,
0F7EB85C1FA8FF4200F1ABCB /* IsoMallocInlines.h */,
7CDD7FF7186D291E007433CD /* IteratorAdaptors.h */,
Modified: branches/safari-613-branch/Source/WTF/wtf/Bitmap.h (288778 => 288779)
--- branches/safari-613-branch/Source/WTF/wtf/Bitmap.h 2022-01-29 03:27:40 UTC (rev 288778)
+++ branches/safari-613-branch/Source/WTF/wtf/Bitmap.h 2022-01-29 03:27:48 UTC (rev 288779)
@@ -22,6 +22,7 @@
#include <array>
#include <wtf/Atomics.h>
#include <wtf/HashFunctions.h>
+#include <wtf/IterationStatus.h>
#include <wtf/MathExtras.h>
#include <wtf/PrintStream.h>
#include <wtf/StdIntExtras.h>
@@ -72,6 +73,8 @@
bool subsumes(const Bitmap&) const;
+ // If the lambda returns an IterationStatus, we use it. The lambda can also return
+ // void, in which case, we'll iterate every set bit.
template<typename Func>
void forEachSetBit(const Func&) const;
@@ -165,13 +168,13 @@
}
template<size_t bitmapSize, typename WordType>
-inline void Bitmap<bitmapSize, WordType>::set(size_t n)
+ALWAYS_INLINE void Bitmap<bitmapSize, WordType>::set(size_t n)
{
bits[n / wordSize] |= (one << (n % wordSize));
}
template<size_t bitmapSize, typename WordType>
-inline void Bitmap<bitmapSize, WordType>::set(size_t n, bool value)
+ALWAYS_INLINE void Bitmap<bitmapSize, WordType>::set(size_t n, bool value)
{
if (value)
set(n);
@@ -381,7 +384,7 @@
template<size_t bitmapSize, typename WordType>
template<typename Func>
-inline void Bitmap<bitmapSize, WordType>::forEachSetBit(const Func& func) const
+ALWAYS_INLINE void Bitmap<bitmapSize, WordType>::forEachSetBit(const Func& func) const
{
for (size_t i = 0; i < words; ++i) {
WordType word = bits[i];
@@ -389,8 +392,13 @@
continue;
size_t base = i * wordSize;
for (size_t j = 0; j < wordSize; ++j) {
- if (word & 1)
- func(base + j);
+ if (word & 1) {
+ if constexpr (std::is_same_v<IterationStatus, decltype(func(base + j))>) {
+ if (func(base + j) == IterationStatus::Done)
+ return;
+ } else
+ func(base + j);
+ }
word >>= 1;
}
}
Modified: branches/safari-613-branch/Source/WTF/wtf/CMakeLists.txt (288778 => 288779)
--- branches/safari-613-branch/Source/WTF/wtf/CMakeLists.txt 2022-01-29 03:27:40 UTC (rev 288778)
+++ branches/safari-613-branch/Source/WTF/wtf/CMakeLists.txt 2022-01-29 03:27:48 UTC (rev 288779)
@@ -110,6 +110,7 @@
InterferenceGraph.h
IsoMalloc.h
IsoMallocInlines.h
+ IterationStatus.h
IteratorAdaptors.h
IteratorRange.h
JSONValues.h
Copied: branches/safari-613-branch/Source/WTF/wtf/IterationStatus.h (from rev 288778, branches/safari-613-branch/Source/_javascript_Core/runtime/IterationStatus.h) (0 => 288779)
--- branches/safari-613-branch/Source/WTF/wtf/IterationStatus.h (rev 0)
+++ branches/safari-613-branch/Source/WTF/wtf/IterationStatus.h 2022-01-29 03:27:48 UTC (rev 288779)
@@ -0,0 +1,37 @@
+/*
+ * Copyright (C) 2022 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+namespace WTF {
+
+enum class IterationStatus {
+ Continue,
+ Done
+};
+
+} // namespace WTF
+
+using WTF::IterationStatus;
