- Revision
- 198829
- Author
- [email protected]
- Date
- 2016-03-29 23:42:36 -0700 (Tue, 29 Mar 2016)
Log Message
bmalloc: support physical page sizes that don't match the virtual page size (take 2)
https://bugs.webkit.org/show_bug.cgi?id=156003
Reviewed by Andreas Kling.
This is a memory savings on iOS devices where the virtual page size
is 16kB but the physical page size is 4kB.
Take 1 was a memory regression on 16kB virtual / 16kB physical systems
because it used a 4kB page size within a 16kB page size, allowing up to
4 different object types to mix within a physical page. Because objects
of the same type tend to deallocate at the same time, mixing objects of
different types made pages less likely to become completely empty.
(Take 1 also had a bug where it used a platform #ifdef that didn't exist.
Oops.)
Take 2 allocates units of SmallPages equal to the physical page size.
* bmalloc/Heap.cpp:
(bmalloc::Heap::Heap):
(bmalloc::Heap::initializeLineMetadata):
(bmalloc::Heap::allocateSmallBumpRanges):
(bmalloc::Heap::allocateSmallPage):
(bmalloc::Heap::allocateLarge):
(bmalloc::Heap::splitAndAllocate):
(bmalloc::Heap::tryAllocateXLarge):
(bmalloc::Heap::shrinkXLarge):
* bmalloc/Heap.h: Use the physical page size for our VM operations because
we're only concerned with returning physical pages to the OS.
* bmalloc/VMAllocate.h:
(bmalloc::vmPageSize):
(bmalloc::vmPageShift):
(bmalloc::vmSize):
(bmalloc::vmValidate):
(bmalloc::vmPageSizePhysical):
(bmalloc::vmValidatePhysical):
(bmalloc::tryVMAllocate):
(bmalloc::vmDeallocatePhysicalPages):
(bmalloc::vmAllocatePhysicalPages):
(bmalloc::vmDeallocatePhysicalPagesSloppy):
(bmalloc::vmAllocatePhysicalPagesSloppy): Use the physical page size.
Modified Paths
Diff
Modified: trunk/Source/bmalloc/ChangeLog (198828 => 198829)
--- trunk/Source/bmalloc/ChangeLog 2016-03-30 06:00:22 UTC (rev 198828)
+++ trunk/Source/bmalloc/ChangeLog 2016-03-30 06:42:36 UTC (rev 198829)
@@ -1,5 +1,51 @@
2016-03-29 Geoffrey Garen <[email protected]>
+ bmalloc: support physical page sizes that don't match the virtual page size (take 2)
+ https://bugs.webkit.org/show_bug.cgi?id=156003
+
+ Reviewed by Andreas Kling.
+
+ This is a memory savings on iOS devices where the virtual page size
+ is 16kB but the physical page size is 4kB.
+
+ Take 1 was a memory regression on 16kB virtual / 16kB physical systems
+ because it used a 4kB page size within a 16kB page size, allowing up to
+ 4 different object types to mix within a physical page. Because objects
+ of the same type tend to deallocate at the same time, mixing objects of
+ different types made pages less likely to become completely empty.
+
+ (Take 1 also had a bug where it used a platform #ifdef that didn't exist.
+ Oops.)
+
+ Take 2 allocates units of SmallPages equal to the physical page size.
+
+ * bmalloc/Heap.cpp:
+ (bmalloc::Heap::Heap):
+ (bmalloc::Heap::initializeLineMetadata):
+ (bmalloc::Heap::allocateSmallBumpRanges):
+ (bmalloc::Heap::allocateSmallPage):
+ (bmalloc::Heap::allocateLarge):
+ (bmalloc::Heap::splitAndAllocate):
+ (bmalloc::Heap::tryAllocateXLarge):
+ (bmalloc::Heap::shrinkXLarge):
+ * bmalloc/Heap.h: Use the physical page size for our VM operations because
+ we're only concerned with returning physical pages to the OS.
+
+ * bmalloc/VMAllocate.h:
+ (bmalloc::vmPageSize):
+ (bmalloc::vmPageShift):
+ (bmalloc::vmSize):
+ (bmalloc::vmValidate):
+ (bmalloc::vmPageSizePhysical):
+ (bmalloc::vmValidatePhysical):
+ (bmalloc::tryVMAllocate):
+ (bmalloc::vmDeallocatePhysicalPages):
+ (bmalloc::vmAllocatePhysicalPages):
+ (bmalloc::vmDeallocatePhysicalPagesSloppy):
+ (bmalloc::vmAllocatePhysicalPagesSloppy): Use the physical page size.
+
+2016-03-29 Geoffrey Garen <[email protected]>
+
bmalloc: page size should be configurable at runtime
https://bugs.webkit.org/show_bug.cgi?id=155993
Modified: trunk/Source/bmalloc/bmalloc/Heap.cpp (198828 => 198829)
--- trunk/Source/bmalloc/bmalloc/Heap.cpp 2016-03-30 06:00:22 UTC (rev 198828)
+++ trunk/Source/bmalloc/bmalloc/Heap.cpp 2016-03-30 06:42:36 UTC (rev 198829)
@@ -35,11 +35,15 @@
namespace bmalloc {
Heap::Heap(std::lock_guard<StaticMutex>&)
- : m_vmPageSize(vmPageSize())
+ : m_vmPageSizePhysical(vmPageSizePhysical())
, m_largeObjects(VMState::HasPhysical::True)
, m_isAllocatingPages(false)
, m_scavenger(*this, &Heap::concurrentScavenge)
{
+ RELEASE_BASSERT(vmPageSizePhysical() >= smallPageSize);
+ RELEASE_BASSERT(vmPageSize() >= vmPageSizePhysical());
+ RELEASE_BASSERT(xLargeAlignment >= vmPageSize());
+
initializeLineMetadata();
}
@@ -47,7 +51,7 @@
{
// We assume that m_smallLineMetadata is zero-filled.
- size_t smallLineCount = m_vmPageSize / smallLineSize;
+ size_t smallLineCount = m_vmPageSizePhysical / smallLineSize;
m_smallLineMetadata.grow(sizeClassCount * smallLineCount);
for (size_t sizeClass = 0; sizeClass < sizeClassCount; ++sizeClass) {
@@ -56,7 +60,7 @@
size_t object = 0;
size_t line = 0;
- while (object < m_vmPageSize) {
+ while (object < m_vmPageSizePhysical) {
line = object / smallLineSize;
size_t leftover = object % smallLineSize;
@@ -70,7 +74,7 @@
}
// Don't allow the last object in a page to escape the page.
- if (object > m_vmPageSize) {
+ if (object > m_vmPageSizePhysical) {
BASSERT(pageMetadata[line].objectCount);
--pageMetadata[line].objectCount;
}
@@ -152,7 +156,7 @@
SmallPage* page = allocateSmallPage(lock, sizeClass);
SmallLine* lines = page->begin();
BASSERT(page->hasFreeLines(lock));
- size_t smallLineCount = m_vmPageSize / smallLineSize;
+ size_t smallLineCount = m_vmPageSizePhysical / smallLineSize;
LineMetadata* pageMetadata = &m_smallLineMetadata[sizeClass * smallLineCount];
// Find a free line.
@@ -211,8 +215,8 @@
return page;
}
- size_t unalignedSize = largeMin + m_vmPageSize - largeAlignment + m_vmPageSize;
- LargeObject largeObject = allocateLarge(lock, m_vmPageSize, m_vmPageSize, unalignedSize);
+ size_t unalignedSize = largeMin + m_vmPageSizePhysical - largeAlignment + m_vmPageSizePhysical;
+ LargeObject largeObject = allocateLarge(lock, m_vmPageSizePhysical, m_vmPageSizePhysical, unalignedSize);
// Transform our large object into a small object page. We deref here
// because our small objects will keep their own line refcounts.
@@ -222,10 +226,10 @@
SmallPage* page = object.page();
page->setSizeClass(sizeClass);
- page->setSmallPageCount(m_vmPageSize / smallPageSize);
+ page->setSmallPageCount(m_vmPageSizePhysical / smallPageSize);
// Set a slide() value on intermediate SmallPages so they hash to their
- // vmPageSize-sized page.
+ // vmPageSizePhysical-sized page.
for (size_t i = 1; i < page->smallPageCount(); ++i)
page[i].setSlide(i);
@@ -326,7 +330,7 @@
BASSERT(size >= largeMin);
BASSERT(size == roundUpToMultipleOf<largeAlignment>(size));
- if (size <= m_vmPageSize)
+ if (size <= m_vmPageSizePhysical)
scavengeSmallPages(lock);
LargeObject largeObject = m_largeObjects.take(size);
@@ -357,7 +361,7 @@
BASSERT(alignment >= largeAlignment);
BASSERT(isPowerOfTwo(alignment));
- if (size <= m_vmPageSize)
+ if (size <= m_vmPageSizePhysical)
scavengeSmallPages(lock);
LargeObject largeObject = m_largeObjects.take(alignment, size, unalignedSize);
@@ -431,7 +435,7 @@
// in the allocated list. This is an important optimization because it
// keeps the free list short, speeding up allocation and merging.
- std::pair<XLargeRange, XLargeRange> allocated = range.split(roundUpToMultipleOf(m_vmPageSize, size));
+ std::pair<XLargeRange, XLargeRange> allocated = range.split(roundUpToMultipleOf(m_vmPageSizePhysical, size));
if (allocated.first.vmState().hasVirtual()) {
vmAllocatePhysicalPagesSloppy(allocated.first.begin(), allocated.first.size());
allocated.first.setVMState(VMState::Physical);
@@ -448,7 +452,7 @@
m_isAllocatingPages = true;
- size = std::max(m_vmPageSize, size);
+ size = std::max(m_vmPageSizePhysical, size);
alignment = roundUpToMultipleOf<xLargeAlignment>(alignment);
XLargeRange range = m_xLargeMap.takeFree(alignment, size);
@@ -475,7 +479,7 @@
{
BASSERT(object.size() > newSize);
- if (object.size() - newSize < m_vmPageSize)
+ if (object.size() - newSize < m_vmPageSizePhysical)
return;
XLargeRange range = m_xLargeMap.takeAllocated(object.begin());
Modified: trunk/Source/bmalloc/bmalloc/Heap.h (198828 => 198829)
--- trunk/Source/bmalloc/bmalloc/Heap.h 2016-03-30 06:00:22 UTC (rev 198828)
+++ trunk/Source/bmalloc/bmalloc/Heap.h 2016-03-30 06:42:36 UTC (rev 198829)
@@ -93,7 +93,7 @@
void scavengeLargeObjects(std::unique_lock<StaticMutex>&, std::chrono::milliseconds);
void scavengeXLargeObjects(std::unique_lock<StaticMutex>&, std::chrono::milliseconds);
- size_t m_vmPageSize;
+ size_t m_vmPageSizePhysical;
Vector<LineMetadata> m_smallLineMetadata;
Modified: trunk/Source/bmalloc/bmalloc/VMAllocate.h (198828 => 198829)
--- trunk/Source/bmalloc/bmalloc/VMAllocate.h 2016-03-30 06:00:22 UTC (rev 198828)
+++ trunk/Source/bmalloc/bmalloc/VMAllocate.h 2016-03-30 06:42:36 UTC (rev 198829)
@@ -36,6 +36,7 @@
#include <unistd.h>
#if BOS(DARWIN)
+#include <mach/vm_page_size.h>
#include <mach/vm_statistics.h>
#endif
@@ -49,12 +50,18 @@
inline size_t vmPageSize()
{
- return sysconf(_SC_PAGESIZE);
+ static size_t cached;
+ if (!cached)
+ cached = sysconf(_SC_PAGESIZE);
+ return cached;
}
inline size_t vmPageShift()
{
- return log2(vmPageSize());
+ static size_t cached;
+ if (!cached)
+ cached = log2(vmPageSize());
+ return cached;
}
inline size_t vmSize(size_t size)
@@ -78,6 +85,34 @@
BASSERT(p == mask(p, ~(vmPageSize() - 1)));
}
+inline size_t vmPageSizePhysical()
+{
+#if (BPLATFORM(IOS) && __IPHONE_OS_VERSION_MIN_REQUIRED >= 100000)
+ return vm_kernel_page_size;
+#else
+ static size_t cached;
+ if (!cached)
+ cached = sysconf(_SC_PAGESIZE);
+ return cached;
+#endif
+}
+
+inline void vmValidatePhysical(size_t vmSize)
+{
+ UNUSED(vmSize);
+ BASSERT(vmSize);
+ BASSERT(vmSize == roundUpToMultipleOf(vmPageSizePhysical(), vmSize));
+}
+
+inline void vmValidatePhysical(void* p, size_t vmSize)
+{
+ vmValidatePhysical(vmSize);
+
+ UNUSED(p);
+ BASSERT(p);
+ BASSERT(p == mask(p, ~(vmPageSizePhysical() - 1)));
+}
+
inline void* tryVMAllocate(size_t vmSize)
{
vmValidate(vmSize);
@@ -139,7 +174,7 @@
inline void vmDeallocatePhysicalPages(void* p, size_t vmSize)
{
- vmValidate(p, vmSize);
+ vmValidatePhysical(p, vmSize);
#if BOS(DARWIN)
SYSCALL(madvise(p, vmSize, MADV_FREE_REUSABLE));
#else
@@ -149,7 +184,7 @@
inline void vmAllocatePhysicalPages(void* p, size_t vmSize)
{
- vmValidate(p, vmSize);
+ vmValidatePhysical(p, vmSize);
#if BOS(DARWIN)
SYSCALL(madvise(p, vmSize, MADV_FREE_REUSE));
#else
@@ -160,8 +195,8 @@
// Trims requests that are un-page-aligned.
inline void vmDeallocatePhysicalPagesSloppy(void* p, size_t size)
{
- char* begin = roundUpToMultipleOf(vmPageSize(), static_cast<char*>(p));
- char* end = roundDownToMultipleOf(vmPageSize(), static_cast<char*>(p) + size);
+ char* begin = roundUpToMultipleOf(vmPageSizePhysical(), static_cast<char*>(p));
+ char* end = roundDownToMultipleOf(vmPageSizePhysical(), static_cast<char*>(p) + size);
if (begin >= end)
return;
@@ -172,8 +207,8 @@
// Expands requests that are un-page-aligned.
inline void vmAllocatePhysicalPagesSloppy(void* p, size_t size)
{
- char* begin = roundDownToMultipleOf(vmPageSize(), static_cast<char*>(p));
- char* end = roundUpToMultipleOf(vmPageSize(), static_cast<char*>(p) + size);
+ char* begin = roundDownToMultipleOf(vmPageSizePhysical(), static_cast<char*>(p));
+ char* end = roundUpToMultipleOf(vmPageSizePhysical(), static_cast<char*>(p) + size);
if (begin >= end)
return;
