- Revision
- 199453
- Author
- [email protected]
- Date
- 2016-04-13 02:39:26 -0700 (Wed, 13 Apr 2016)
Log Message
Merge r199115 - bmalloc: handle aligned allocations on the fast path
https://bugs.webkit.org/show_bug.cgi?id=156302
Reviewed by Michael Saboff.
This helps keep the _javascript_Core GC on the fast path, and it also
helps avoid fragmentation on our website stress test:
nimlang 209,584kB 198,076kB ^ 1.06x smaller
* bmalloc/Allocator.cpp:
(bmalloc::Allocator::allocate): Because we arrange for power-of-two size
classes to allocate at power-of-two alignments, we can allocate any
small aligned request on the small path.
* bmalloc/Chunk.h:
(bmalloc::Chunk::bytes):
(bmalloc::Chunk::lines):
(bmalloc::Chunk::pages):
(bmalloc::Chunk::boundaryTags):
(bmalloc::Chunk::objectType): Moved some code around to provide better
API.
(bmalloc::Chunk::Chunk): Moved this code to VMHeap.
(bmalloc::Chunk::offset):
(bmalloc::Chunk::object): Use our new bytes() helper function.
* bmalloc/VMHeap.cpp:
(bmalloc::VMHeap::allocateChunk): Moved code here from Chunk.
(bmalloc::VMHeap::allocateSmallChunk): Ensure that power-of-two page
sizes always begin allocation at the same alignment. Power-of-two object
sizes always request power-of-two page sizes (since that's the least
wasteful option), so if we also ensure that power-of-two page sizes get
power-of-two alignment, then everything is aligned for all small objects.
Modified Paths
Diff
Modified: releases/WebKitGTK/webkit-2.12/Source/bmalloc/ChangeLog (199452 => 199453)
--- releases/WebKitGTK/webkit-2.12/Source/bmalloc/ChangeLog 2016-04-13 09:39:10 UTC (rev 199452)
+++ releases/WebKitGTK/webkit-2.12/Source/bmalloc/ChangeLog 2016-04-13 09:39:26 UTC (rev 199453)
@@ -1,3 +1,42 @@
+2016-04-06 Geoffrey Garen <[email protected]>
+
+ bmalloc: handle aligned allocations on the fast path
+ https://bugs.webkit.org/show_bug.cgi?id=156302
+
+ Reviewed by Michael Saboff.
+
+ This helps keep the _javascript_Core GC on the fast path, and it also
+ helps avoid fragmentation on our website stress test:
+
+ nimlang 209,584kB 198,076kB ^ 1.06x smaller
+
+ * bmalloc/Allocator.cpp:
+ (bmalloc::Allocator::allocate): Because we arrange for power-of-two size
+ classes to allocate at power-of-two alignments, we can allocate any
+ small aligned request on the small path.
+
+ * bmalloc/Chunk.h:
+ (bmalloc::Chunk::bytes):
+ (bmalloc::Chunk::lines):
+ (bmalloc::Chunk::pages):
+ (bmalloc::Chunk::boundaryTags):
+ (bmalloc::Chunk::objectType): Moved some code around to provide better
+ API.
+
+ (bmalloc::Chunk::Chunk): Moved this code to VMHeap.
+
+ (bmalloc::Chunk::offset):
+ (bmalloc::Chunk::object): Use our new bytes() helper function.
+
+ * bmalloc/VMHeap.cpp:
+ (bmalloc::VMHeap::allocateChunk): Moved code here from Chunk.
+
+ (bmalloc::VMHeap::allocateSmallChunk): Ensure that power-of-two page
+ sizes always begin allocation at the same alignment. Power-of-two object
+ sizes always request power-of-two page sizes (since that's the least
+ wasteful option), so if we also ensure that power-of-two page sizes get
+ power-of-two alignment, then everything is aligned for all small objects.
+
2016-04-03 Geoffrey Garen <[email protected]>
bmalloc: segregate small and large objects again, and allocate more objects on the small path
Modified: releases/WebKitGTK/webkit-2.12/Source/bmalloc/bmalloc/Allocator.cpp (199452 => 199453)
--- releases/WebKitGTK/webkit-2.12/Source/bmalloc/bmalloc/Allocator.cpp 2016-04-13 09:39:10 UTC (rev 199452)
+++ releases/WebKitGTK/webkit-2.12/Source/bmalloc/bmalloc/Allocator.cpp 2016-04-13 09:39:26 UTC (rev 199453)
@@ -77,16 +77,13 @@
return nullptr;
return result;
}
-
- if (size <= smallMax && alignment <= smallLineSize) {
- size_t alignmentMask = alignment - 1;
- while (void* p = allocate(size)) {
- if (!test(p, alignmentMask))
- return p;
- m_deallocator.deallocate(p);
- }
- }
+ if (!size)
+ size = alignment;
+
+ if (size <= smallMax && alignment <= smallMax)
+ return allocate(roundUpToMultipleOf(alignment, size));
+
if (size <= largeMax && alignment <= largeMax) {
size = std::max(largeMin, roundUpToMultipleOf<largeAlignment>(size));
alignment = roundUpToMultipleOf<largeAlignment>(alignment);
Modified: releases/WebKitGTK/webkit-2.12/Source/bmalloc/bmalloc/Chunk.h (199452 => 199453)
--- releases/WebKitGTK/webkit-2.12/Source/bmalloc/bmalloc/Chunk.h 2016-04-13 09:39:10 UTC (rev 199452)
+++ releases/WebKitGTK/webkit-2.12/Source/bmalloc/bmalloc/Chunk.h 2016-04-13 09:39:26 UTC (rev 199453)
@@ -39,6 +39,23 @@
namespace bmalloc {
class Chunk {
+ // Our metadata layout includes a left and right edge sentinel.
+ // Metadata takes up enough space to leave at least the first two
+ // boundary tag slots unused.
+ //
+ // So, boundary tag space looks like this:
+ //
+ // [OOXXXXX...]
+ //
+ // And BoundaryTag::get subtracts one, producing:
+ //
+ // [OXXXXX...O].
+ //
+ // We use the X's for boundary tags and the O's for edge sentinels.
+
+ static const size_t boundaryTagCount = chunkSize / largeMin;
+ static_assert(boundaryTagCount > 2, "Chunk must have space for two sentinel boundary tags");
+
public:
static Chunk* get(void*);
@@ -53,33 +70,14 @@
SmallPage* page(size_t offset);
SmallLine* line(size_t offset);
+ char* bytes() { return reinterpret_cast<char*>(this); }
SmallLine* lines() { return m_lines.begin(); }
SmallPage* pages() { return m_pages.begin(); }
+ std::array<BoundaryTag, boundaryTagCount>& boundaryTags() { return m_boundaryTags; }
- char* begin() { return roundUpToMultipleOf(vmPageSizePhysical(), m_memory); }
- char* end() { return reinterpret_cast<char*>(this) + chunkSize; }
- size_t size() { return end() - begin(); }
-
ObjectType objectType() { return m_objectType; }
private:
- static const size_t boundaryTagCount = chunkSize / largeMin;
- static_assert(boundaryTagCount > 2, "Chunk must have space for two sentinel boundary tags");
-
- // Our metadata layout includes a left and right edge sentinel.
- // Metadata takes up enough space to leave at least the first two
- // boundary tag slots unused.
- //
- // So, boundary tag space looks like this:
- //
- // [OOXXXXX...]
- //
- // And BoundaryTag::get subtracts one, producing:
- //
- // [OXXXXX...O].
- //
- // We use the X's for boundary tags and the O's for edge sentinels.
-
union {
// The first few bytes of metadata cover the metadata region, so they're
// not used. We can steal them to store m_objectType.
@@ -93,7 +91,6 @@
std::array<SmallPage, chunkSize / smallPageSize> m_pages;
std::array<BoundaryTag, boundaryTagCount> m_boundaryTags;
};
- char m_memory[];
};
static_assert(sizeof(Chunk) + largeMax <= chunkSize, "largeMax is too big");
@@ -104,33 +101,6 @@
inline Chunk::Chunk(std::lock_guard<StaticMutex>&, ObjectType objectType)
: m_objectType(objectType)
{
- if (objectType != ObjectType::Large)
- return;
-
- Range range(begin(), size());
- BASSERT(range.size() <= largeObjectMax);
-
- BeginTag* beginTag = Chunk::beginTag(range.begin());
- beginTag->setRange(range);
- beginTag->setFree(true);
- beginTag->setVMState(VMState::Virtual);
-
- EndTag* endTag = Chunk::endTag(range.begin(), range.size());
- endTag->init(beginTag);
-
- // Mark the left and right edges of our range as allocated. This naturally
- // prevents merging logic from overflowing left (into metadata) or right
- // (beyond our chunk), without requiring special-case checks.
-
- EndTag* leftSentinel = beginTag->prev();
- BASSERT(leftSentinel >= m_boundaryTags.begin());
- BASSERT(leftSentinel < m_boundaryTags.end());
- leftSentinel->initSentinel();
-
- BeginTag* rightSentinel = endTag->next();
- BASSERT(rightSentinel >= m_boundaryTags.begin());
- BASSERT(rightSentinel < m_boundaryTags.end());
- rightSentinel->initSentinel();
}
inline Chunk* Chunk::get(void* object)
@@ -161,13 +131,13 @@
inline size_t Chunk::offset(void* object)
{
BASSERT(object >= this);
- BASSERT(object < reinterpret_cast<char*>(this) + chunkSize);
- return static_cast<char*>(object) - reinterpret_cast<char*>(this);
+ BASSERT(object < bytes() + chunkSize);
+ return static_cast<char*>(object) - bytes();
}
inline char* Chunk::object(size_t offset)
{
- return reinterpret_cast<char*>(this) + offset;
+ return bytes() + offset;
}
inline SmallPage* Chunk::page(size_t offset)
Modified: releases/WebKitGTK/webkit-2.12/Source/bmalloc/bmalloc/VMHeap.cpp (199452 => 199453)
--- releases/WebKitGTK/webkit-2.12/Source/bmalloc/bmalloc/VMHeap.cpp 2016-04-13 09:39:10 UTC (rev 199452)
+++ releases/WebKitGTK/webkit-2.12/Source/bmalloc/bmalloc/VMHeap.cpp 2016-04-13 09:39:26 UTC (rev 199453)
@@ -44,7 +44,35 @@
m_zone.addChunk(chunk);
#endif
- return LargeObject(chunk->begin());
+ size_t alignment = largeAlignment;
+ size_t metadataSize = roundUpToMultipleOf(alignment, sizeof(Chunk));
+
+ Range range(chunk->bytes() + metadataSize, chunkSize - metadataSize);
+ BASSERT(range.size() <= largeObjectMax);
+
+ BeginTag* beginTag = Chunk::beginTag(range.begin());
+ beginTag->setRange(range);
+ beginTag->setFree(true);
+ beginTag->setVMState(VMState::Virtual);
+
+ EndTag* endTag = Chunk::endTag(range.begin(), range.size());
+ endTag->init(beginTag);
+
+ // Mark the left and right edges of our range as allocated. This naturally
+ // prevents merging logic from overflowing left (into metadata) or right
+ // (beyond our chunk), without requiring special-case checks.
+
+ EndTag* leftSentinel = beginTag->prev();
+ BASSERT(leftSentinel >= chunk->boundaryTags().begin());
+ BASSERT(leftSentinel < chunk->boundaryTags().end());
+ leftSentinel->initSentinel();
+
+ BeginTag* rightSentinel = endTag->next();
+ BASSERT(rightSentinel >= chunk->boundaryTags().begin());
+ BASSERT(rightSentinel < chunk->boundaryTags().end());
+ rightSentinel->initSentinel();
+
+ return LargeObject(range.begin());
}
void VMHeap::allocateSmallChunk(std::lock_guard<StaticMutex>& lock, size_t pageClass)
@@ -59,9 +87,16 @@
size_t pageSize = bmalloc::pageSize(pageClass);
size_t smallPageCount = pageSize / smallPageSize;
- Object begin(chunk->begin());
- Object end(begin + chunk->size());
+ // If our page size is a power of two, we align to it in order to guarantee
+ // that we can service aligned allocation requests at the same power of two.
+ size_t alignment = vmPageSizePhysical();
+ if (isPowerOfTwo(pageSize))
+ alignment = pageSize;
+ size_t metadataSize = roundUpToMultipleOf(alignment, sizeof(Chunk));
+ Object begin(chunk, metadataSize);
+ Object end(chunk, chunkSize);
+
for (Object it = begin; it + pageSize <= end; it = it + pageSize) {
SmallPage* page = it.page();
new (page) SmallPage;
