- Revision
- 205210
- Author
- [email protected]
- Date
- 2016-08-30 17:15:50 -0700 (Tue, 30 Aug 2016)
Log Message
bmalloc: speed up the lock slow path
https://bugs.webkit.org/show_bug.cgi?id=161058
Unreviewed roll-in - with regression fixed.
Revert to using yield() instead of swtch() because very low priority
background tasks can cause priority inversion and deadlock. In the
network process, that happened with com.apple.WebKit.Cache.Storage.serialBackground.
Still a big speedup on MallocBench.
* bmalloc.xcodeproj/project.pbxproj:
* bmalloc/ScopeExit.h: Added.
(bmalloc::ScopeExit::ScopeExit):
(bmalloc::ScopeExit::~ScopeExit):
(bmalloc::makeScopeExit):
* bmalloc/StaticMutex.cpp:
(bmalloc::StaticMutex::lockSlowCase):
* bmalloc/StaticMutex.h:
(bmalloc::StaticMutex::init):
Modified Paths
Added Paths
Diff
Modified: trunk/Source/bmalloc/ChangeLog (205209 => 205210)
--- trunk/Source/bmalloc/ChangeLog 2016-08-31 00:11:33 UTC (rev 205209)
+++ trunk/Source/bmalloc/ChangeLog 2016-08-31 00:15:50 UTC (rev 205210)
@@ -1,3 +1,26 @@
+2016-08-30 Geoffrey Garen <[email protected]>
+
+ bmalloc: speed up the lock slow path
+ https://bugs.webkit.org/show_bug.cgi?id=161058
+
+ Unreviewed roll-in - with regression fixed.
+
+ Revert to using yield() instead of swtch() because very low priority
+ background tasks can cause priority inversion and deadlock. In the
+ network process, that happened with com.apple.WebKit.Cache.Storage.serialBackground.
+
+ Still a big speedup on MallocBench.
+
+ * bmalloc.xcodeproj/project.pbxproj:
+ * bmalloc/ScopeExit.h: Added.
+ (bmalloc::ScopeExit::ScopeExit):
+ (bmalloc::ScopeExit::~ScopeExit):
+ (bmalloc::makeScopeExit):
+ * bmalloc/StaticMutex.cpp:
+ (bmalloc::StaticMutex::lockSlowCase):
+ * bmalloc/StaticMutex.h:
+ (bmalloc::StaticMutex::init):
+
2016-08-26 Geoffrey Garen <[email protected]>
Unreviewed build fix.
Added: trunk/Source/bmalloc/bmalloc/ScopeExit.h (0 => 205210)
--- trunk/Source/bmalloc/bmalloc/ScopeExit.h (rev 0)
+++ trunk/Source/bmalloc/bmalloc/ScopeExit.h 2016-08-31 00:15:50 UTC (rev 205210)
@@ -0,0 +1,53 @@
+/*
+ * Copyright (C) 2016 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <type_traits>
+
+namespace bmalloc {
+
+template<typename ExitFunction>
+class ScopeExit {
+public:
+ explicit ScopeExit(ExitFunction&& exitFunction)
+ : m_exitFunction(exitFunction)
+ {
+ }
+
+ ~ScopeExit()
+ {
+ m_exitFunction();
+ }
+
+private:
+ ExitFunction m_exitFunction;
+};
+
+template<typename ExitFunction>
+ScopeExit<ExitFunction> makeScopeExit(ExitFunction&& exitFunction)
+{
+ return ScopeExit<ExitFunction>(std::forward<ExitFunction>(exitFunction));
+}
+
+} // namespace bmalloc
Modified: trunk/Source/bmalloc/bmalloc/StaticMutex.cpp (205209 => 205210)
--- trunk/Source/bmalloc/bmalloc/StaticMutex.cpp 2016-08-31 00:11:33 UTC (rev 205209)
+++ trunk/Source/bmalloc/bmalloc/StaticMutex.cpp 2016-08-31 00:15:50 UTC (rev 205210)
@@ -23,6 +23,7 @@
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
+#include "ScopeExit.h"
#include "StaticMutex.h"
#include <thread>
@@ -30,6 +31,21 @@
void StaticMutex::lockSlowCase()
{
+ // The longest critical section in bmalloc is much shorter than the
+ // time it takes to make a system call to yield to the OS scheduler.
+ // So, we try again a lot before we yield.
+ static const size_t aLot = 256;
+
+ if (!m_isSpinning.test_and_set()) {
+ auto clear = makeScopeExit([&] { m_isSpinning.clear(); });
+
+ for (size_t i = 0; i < aLot; ++i) {
+ if (try_lock())
+ return;
+ }
+ }
+
+ // Avoid spinning pathologically.
while (!try_lock())
std::this_thread::yield();
}
Modified: trunk/Source/bmalloc/bmalloc/StaticMutex.h (205209 => 205210)
--- trunk/Source/bmalloc/bmalloc/StaticMutex.h 2016-08-31 00:11:33 UTC (rev 205209)
+++ trunk/Source/bmalloc/bmalloc/StaticMutex.h 2016-08-31 00:15:50 UTC (rev 205210)
@@ -52,6 +52,7 @@
void lockSlowCase();
std::atomic_flag m_flag;
+ std::atomic_flag m_isSpinning;
};
static inline void sleep(
@@ -78,6 +79,7 @@
inline void StaticMutex::init()
{
m_flag.clear();
+ m_isSpinning.clear();
}
inline bool StaticMutex::try_lock()
Modified: trunk/Source/bmalloc/bmalloc.xcodeproj/project.pbxproj (205209 => 205210)
--- trunk/Source/bmalloc/bmalloc.xcodeproj/project.pbxproj 2016-08-31 00:11:33 UTC (rev 205209)
+++ trunk/Source/bmalloc/bmalloc.xcodeproj/project.pbxproj 2016-08-31 00:15:50 UTC (rev 205210)
@@ -24,6 +24,7 @@
147DC6E31CA5B70B00724E8D /* Chunk.h in Headers */ = {isa = PBXBuildFile; fileRef = 147DC6E21CA5B70B00724E8D /* Chunk.h */; settings = {ATTRIBUTES = (Private, ); }; };
14895D911A3A319C0006235D /* Environment.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 14895D8F1A3A319C0006235D /* Environment.cpp */; };
14895D921A3A319C0006235D /* Environment.h in Headers */ = {isa = PBXBuildFile; fileRef = 14895D901A3A319C0006235D /* Environment.h */; settings = {ATTRIBUTES = (Private, ); }; };
+ 148EFAE81D6B953B008E721E /* ScopeExit.h in Headers */ = {isa = PBXBuildFile; fileRef = 148EFAE61D6B953B008E721E /* ScopeExit.h */; };
14C8992B1CC485E70027A057 /* Map.h in Headers */ = {isa = PBXBuildFile; fileRef = 14C8992A1CC485E70027A057 /* Map.h */; settings = {ATTRIBUTES = (Private, ); }; };
14C8992D1CC578330027A057 /* LargeRange.h in Headers */ = {isa = PBXBuildFile; fileRef = 14C8992C1CC578330027A057 /* LargeRange.h */; settings = {ATTRIBUTES = (Private, ); }; };
14C919C918FCC59F0028DB43 /* BPlatform.h in Headers */ = {isa = PBXBuildFile; fileRef = 14C919C818FCC59F0028DB43 /* BPlatform.h */; settings = {ATTRIBUTES = (Private, ); }; };
@@ -110,6 +111,7 @@
1485656018A43DBA00ED6942 /* ObjectType.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = ObjectType.h; path = bmalloc/ObjectType.h; sourceTree = "<group>"; };
14895D8F1A3A319C0006235D /* Environment.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = Environment.cpp; path = bmalloc/Environment.cpp; sourceTree = "<group>"; };
14895D901A3A319C0006235D /* Environment.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = Environment.h; path = bmalloc/Environment.h; sourceTree = "<group>"; };
+ 148EFAE61D6B953B008E721E /* ScopeExit.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = ScopeExit.h; path = bmalloc/ScopeExit.h; sourceTree = "<group>"; };
14B650C518F39F4800751968 /* Base.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; path = Base.xcconfig; sourceTree = "<group>"; };
14B650C618F39F4800751968 /* bmalloc.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; path = bmalloc.xcconfig; sourceTree = "<group>"; };
14B650C718F39F4800751968 /* DebugRelease.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; path = DebugRelease.xcconfig; sourceTree = "<group>"; };
@@ -262,6 +264,7 @@
14446A0717A61FA400F9EA1D /* PerProcess.h */,
144469FD17A61F1F00F9EA1D /* PerThread.h */,
145F6878179E3A4400D65598 /* Range.h */,
+ 148EFAE61D6B953B008E721E /* ScopeExit.h */,
143CB81A19022BC900B16A45 /* StaticMutex.cpp */,
143CB81B19022BC900B16A45 /* StaticMutex.h */,
1417F64F18B7280C0076FA3F /* Syscall.h */,
@@ -316,6 +319,7 @@
14895D921A3A319C0006235D /* Environment.h in Headers */,
1400274A18F89C2300115C97 /* VMHeap.h in Headers */,
1400274918F89C1300115C97 /* Heap.h in Headers */,
+ 148EFAE81D6B953B008E721E /* ScopeExit.h in Headers */,
140FA00319CE429C00FFD3C8 /* BumpRange.h in Headers */,
4426E2811C838EE0008EB042 /* Logging.h in Headers */,
14DD78C518F48D7500950702 /* Algorithm.h in Headers */,
