Diff
Modified: trunk/Source/bmalloc/ChangeLog (278277 => 278278)
--- trunk/Source/bmalloc/ChangeLog 2021-05-31 15:02:54 UTC (rev 278277)
+++ trunk/Source/bmalloc/ChangeLog 2021-05-31 15:04:24 UTC (rev 278278)
@@ -1,3 +1,73 @@
+2021-05-31 Michael Saboff <[email protected]>
+
+ [bmalloc] Make adaptive scavenging more precise
+ https://bugs.webkit.org/show_bug.cgi?id=226237
+
+ Reviewed by Geoffrey Garen.
+
+ This patch re-enables the adaptive scavenger for macOS.
+ It is much more precise when calling madvise() by keeping track of the
+ maximum extent of physically mapped memory in a LargeRange. For example,
+ in the prior code, we'd return a small range back to a Gigacage LargeRange
+ whithout any physical pages. When we scavenge that LargeRange, we madvise()
+ the whole Gigacage range. Although this didn't cause correctness issues,
+ it was quite wasteful.
+
+ In the tests I did with this patch compared to the earlier adaptive change,
+ the number of madvise calls we made drops by ~18% and the time spent in those
+ calls dropped ~13% on an Apple silicon mac and ~30% on an x86 mac.
+ This was measured using dtrace running JetStream2 from the command line.
+ RAMification results improved ~1% over the adptive change.
+
+ There is the possible future optimization where we also keep track of the
+ first address of physically mapped memory in a LargeRange. Since bmalloc
+ allocates memory from lower addresses first, it is thought that the change
+ in this patch is sufficient to reduce not only the number of madvise calls,
+ but the time it takes to make those calls.
+
+ * bmalloc/BPlatform.h:
+ * bmalloc/Heap.cpp:
+ (bmalloc::Heap::decommitLargeRange):
+ (bmalloc::Heap::scavenge):
+ (bmalloc::Heap::allocateSmallChunk):
+ (bmalloc::Heap::deallocateSmallChunk):
+ (bmalloc::Heap::allocateSmallPage):
+ (bmalloc::Heap::splitAndAllocate):
+ (bmalloc::Heap::allocateLarge):
+ (bmalloc::Heap::tryAllocateLargeChunk):
+ (bmalloc::Heap::shrinkLarge):
+ (bmalloc::Heap::deallocateLarge):
+ (bmalloc::Heap::scavengeToHighWatermark): Deleted.
+ * bmalloc/Heap.h:
+ * bmalloc/IsoDirectory.h:
+ * bmalloc/IsoDirectoryInlines.h:
+ (bmalloc::passedNumPages>::takeFirstEligible):
+ (bmalloc::passedNumPages>::scavenge):
+ (bmalloc::passedNumPages>::scavengeToHighWatermark): Deleted.
+ * bmalloc/IsoHeapImpl.h:
+ * bmalloc/IsoHeapImplInlines.h:
+ (bmalloc::IsoHeapImpl<Config>::scavengeToHighWatermark): Deleted.
+ * bmalloc/LargeMap.cpp:
+ (bmalloc::LargeMap::add):
+ * bmalloc/LargeRange.h:
+ (bmalloc::LargeRange::LargeRange):
+ (bmalloc::LargeRange::physicalExtent const):
+ (bmalloc::LargeRange::setPhysicalExtent):
+ (bmalloc::LargeRange::resetPhysicalExtent):
+ (bmalloc::LargeRange::setUsedSinceLastScavenge):
+ (bmalloc::merge):
+ (bmalloc::LargeRange::split const):
+ (): Deleted.
+ * bmalloc/Scavenger.cpp:
+ (bmalloc::Scavenger::Scavenger):
+ (bmalloc::Scavenger::scavenge):
+ (bmalloc::Scavenger::threadRunLoop):
+ (bmalloc::Scavenger::timeSinceLastPartialScavenge): Deleted.
+ (bmalloc::Scavenger::partialScavenge): Deleted.
+ * bmalloc/Scavenger.h:
+ * bmalloc/SmallPage.h:
+ (bmalloc::SmallPage::setUsedSinceLastScavenge):
+
2021-05-21 Michael Saboff <[email protected]>
[bmalloc] Rollout r276266 because WebKit processes are spending much more time in madvise
Modified: trunk/Source/bmalloc/bmalloc/BPlatform.h (278277 => 278278)
--- trunk/Source/bmalloc/bmalloc/BPlatform.h 2021-05-31 15:02:54 UTC (rev 278277)
+++ trunk/Source/bmalloc/bmalloc/BPlatform.h 2021-05-31 15:04:24 UTC (rev 278278)
@@ -309,12 +309,6 @@
/* This is used for debugging when hacking on how bmalloc calculates its physical footprint. */
#define ENABLE_PHYSICAL_PAGE_MAP 0
-#if BPLATFORM(MAC)
-#define BUSE_PARTIAL_SCAVENGE 1
-#else
-#define BUSE_PARTIAL_SCAVENGE 0
-#endif
-
#if !defined(BUSE_PRECOMPUTED_CONSTANTS_VMPAGE4K)
#define BUSE_PRECOMPUTED_CONSTANTS_VMPAGE4K 1
#endif
Modified: trunk/Source/bmalloc/bmalloc/Heap.cpp (278277 => 278278)
--- trunk/Source/bmalloc/bmalloc/Heap.cpp 2021-05-31 15:02:54 UTC (rev 278277)
+++ trunk/Source/bmalloc/bmalloc/Heap.cpp 2021-05-31 15:04:24 UTC (rev 278278)
@@ -65,7 +65,7 @@
m_gigacageSize = size;
ptrdiff_t offset = roundDownToMultipleOf(vmPageSize(), random[1] % (gigacageSize - size));
void* base = reinterpret_cast<unsigned char*>(gigacageBasePtr) + offset;
- m_largeFree.add(LargeRange(base, size, 0, 0));
+ m_largeFree.add(LargeRange(base, size, 0, 0, base));
}
#endif
@@ -108,10 +108,13 @@
{
m_footprint -= range.totalPhysicalSize();
m_freeableMemory -= range.totalPhysicalSize();
- decommitter.addLazy(range.begin(), range.size());
- m_hasPendingDecommits = true;
+ if (range.totalPhysicalSize()) {
+ decommitter.addLazy(range.begin(), range.physicalEnd() - range.begin());
+ m_hasPendingDecommits = true;
+ }
range.setStartPhysicalSize(0);
range.setTotalPhysicalSize(0);
+ range.clearPhysicalEnd();
BASSERT(range.isEligibile());
range.setEligible(false);
#if ENABLE_PHYSICAL_PAGE_MAP
@@ -119,11 +122,7 @@
#endif
}
-#if BUSE(PARTIAL_SCAVENGE)
-void Heap::scavenge(UniqueLockHolder& lock, BulkDecommit& decommitter)
-#else
void Heap::scavenge(UniqueLockHolder& lock, BulkDecommit& decommitter, size_t& deferredDecommits)
-#endif
{
for (auto& list : m_freePages) {
for (auto* chunk : list) {
@@ -130,13 +129,11 @@
for (auto* page : chunk->freePages()) {
if (!page->hasPhysicalPages())
continue;
-#if !BUSE(PARTIAL_SCAVENGE)
if (page->usedSinceLastScavenge()) {
page->clearUsedSinceLastScavenge();
deferredDecommits++;
continue;
}
-#endif
size_t pageSize = bmalloc::pageSize(&list - &m_freePages[0]);
size_t decommitSize = physicalPageSizeSloppy(page->begin()->begin(), pageSize);
@@ -157,37 +154,15 @@
}
for (LargeRange& range : m_largeFree) {
-#if BUSE(PARTIAL_SCAVENGE)
- m_highWatermark = std::min(m_highWatermark, static_cast<void*>(range.begin()));
-#else
if (range.usedSinceLastScavenge()) {
range.clearUsedSinceLastScavenge();
deferredDecommits++;
continue;
}
-#endif
decommitLargeRange(lock, range, decommitter);
}
-
-#if BUSE(PARTIAL_SCAVENGE)
- m_freeableMemory = 0;
-#endif
}
-#if BUSE(PARTIAL_SCAVENGE)
-void Heap::scavengeToHighWatermark(UniqueLockHolder& lock, BulkDecommit& decommitter)
-{
- void* newHighWaterMark = nullptr;
- for (LargeRange& range : m_largeFree) {
- if (range.begin() <= m_highWatermark)
- newHighWaterMark = std::min(newHighWaterMark, static_cast<void*>(range.begin()));
- else
- decommitLargeRange(lock, range, decommitter);
- }
- m_highWatermark = newHighWaterMark;
-}
-#endif
-
void Heap::deallocateLineCache(UniqueLockHolder&, LineCache& lineCache)
{
for (auto& list : lineCache) {
@@ -218,26 +193,15 @@
m_objectTypes.set(lock, chunk, ObjectType::Small);
- size_t accountedInFreeable = 0;
forEachPage(chunk, pageSize, [&](SmallPage* page) {
page->setHasPhysicalPages(true);
-#if !BUSE(PARTIAL_SCAVENGE)
page->setUsedSinceLastScavenge();
-#endif
page->setHasFreeLines(lock, true);
chunk->freePages().push(page);
- accountedInFreeable += pageSize;
});
- m_freeableMemory += accountedInFreeable;
+ m_freeableMemory += chunkSize;
- auto metadataSize = Chunk::metadataSize(pageSize);
- vmDeallocatePhysicalPagesSloppy(chunk->address(sizeof(Chunk)), metadataSize - sizeof(Chunk));
-
- auto decommitSize = chunkSize - metadataSize - accountedInFreeable;
- if (decommitSize > 0)
- vmDeallocatePhysicalPagesSloppy(chunk->address(chunkSize - decommitSize), decommitSize);
-
m_scavenger->schedule(0);
return chunk;
@@ -253,24 +217,23 @@
size_t size = m_largeAllocated.remove(chunk);
size_t totalPhysicalSize = size;
+ size_t chunkPageSize = pageSize(pageClass);
+ SmallPage* firstPageWithoutPhysicalPages = nullptr;
- size_t accountedInFreeable = 0;
-
- bool hasPhysicalPages = true;
- forEachPage(chunk, pageSize(pageClass), [&](SmallPage* page) {
+ void* physicalEnd = chunk->address(chunk->metadataSize(chunkPageSize));
+ forEachPage(chunk, chunkPageSize, [&](SmallPage* page) {
size_t physicalSize = physicalPageSizeSloppy(page->begin()->begin(), pageSize(pageClass));
if (!page->hasPhysicalPages()) {
totalPhysicalSize -= physicalSize;
- hasPhysicalPages = false;
+ if (!firstPageWithoutPhysicalPages)
+ firstPageWithoutPhysicalPages = page;
} else
- accountedInFreeable += physicalSize;
+ physicalEnd = page->begin()->begin() + physicalSize;
});
- m_freeableMemory -= accountedInFreeable;
- m_freeableMemory += totalPhysicalSize;
+ size_t startPhysicalSize = firstPageWithoutPhysicalPages ? firstPageWithoutPhysicalPages->begin()->begin() - chunk->bytes() : size;
- size_t startPhysicalSize = hasPhysicalPages ? size : 0;
- m_largeFree.add(LargeRange(chunk, size, startPhysicalSize, totalPhysicalSize));
+ m_largeFree.add(LargeRange(chunk, size, startPhysicalSize, totalPhysicalSize, chunk->address(startPhysicalSize)));
}
SmallPage* Heap::allocateSmallPage(UniqueLockHolder& lock, size_t sizeClass, LineCache& lineCache, FailureAction action)
@@ -283,8 +246,6 @@
if (!m_lineCache[sizeClass].isEmpty())
return m_lineCache[sizeClass].popFront();
- m_scavenger->didStartGrowing();
-
SmallPage* page = [&]() -> SmallPage* {
size_t pageClass = m_constants.pageClass(sizeClass);
@@ -314,9 +275,7 @@
m_physicalPageMap.commit(page->begin()->begin(), pageSize);
#endif
}
-#if !BUSE(PARTIAL_SCAVENGE)
page->setUsedSinceLastScavenge();
-#endif
return page;
}();
@@ -525,6 +484,7 @@
vmAllocatePhysicalPagesSloppy(range.begin() + range.startPhysicalSize(), range.size() - range.startPhysicalSize());
range.setStartPhysicalSize(range.size());
range.setTotalPhysicalSize(range.size());
+ range.setPhysicalEnd(range.begin() + range.size());
#if ENABLE_PHYSICAL_PAGE_MAP
m_physicalPageMap.commit(range.begin(), range.size());
#endif
@@ -560,8 +520,6 @@
BASSERT(isPowerOfTwo(alignment));
- m_scavenger->didStartGrowing();
-
size_t roundedSize = size ? roundUpToMultipleOf(largeAlignment, size) : largeAlignment;
ASSERT_OR_RETURN_ON_FAILURE(roundedSize >= size); // Check for overflow
size = roundedSize;
@@ -590,9 +548,6 @@
m_freeableMemory -= range.totalPhysicalSize();
void* result = splitAndAllocate(lock, range, alignment, size).begin();
-#if BUSE(PARTIAL_SCAVENGE)
- m_highWatermark = std::max(m_highWatermark, result);
-#endif
ASSERT_OR_RETURN_ON_FAILURE(result);
return result;
@@ -621,7 +576,7 @@
PerProcess<Zone>::get()->addRange(Range(memory, size));
#endif
- return LargeRange(memory, size, 0, 0);
+ return LargeRange(memory, size, 0, 0, memory);
}
size_t Heap::largeSize(UniqueLockHolder&, void* object)
@@ -634,7 +589,7 @@
BASSERT(object.size() > newSize);
size_t size = m_largeAllocated.remove(object.begin());
- LargeRange range = LargeRange(object, size, size);
+ LargeRange range = LargeRange(object, size, size, object.begin() + size);
splitAndAllocate(lock, range, alignment, newSize);
m_scavenger->schedule(size);
@@ -643,7 +598,7 @@
void Heap::deallocateLarge(UniqueLockHolder&, void* object)
{
size_t size = m_largeAllocated.remove(object);
- m_largeFree.add(LargeRange(object, size, size, size));
+ m_largeFree.add(LargeRange(object, size, size, size, static_cast<char*>(object) + size));
m_freeableMemory += size;
m_scavenger->schedule(size);
}
Modified: trunk/Source/bmalloc/bmalloc/Heap.h (278277 => 278278)
--- trunk/Source/bmalloc/bmalloc/Heap.h 2021-05-31 15:02:54 UTC (rev 278277)
+++ trunk/Source/bmalloc/bmalloc/Heap.h 2021-05-31 15:04:24 UTC (rev 278278)
@@ -74,12 +74,7 @@
size_t largeSize(UniqueLockHolder&, void*);
void shrinkLarge(UniqueLockHolder&, const Range&, size_t);
-#if BUSE(PARTIAL_SCAVENGE)
- void scavengeToHighWatermark(UniqueLockHolder&, BulkDecommit&);
- void scavenge(UniqueLockHolder&, BulkDecommit&);
-#else
void scavenge(UniqueLockHolder&, BulkDecommit&, size_t& deferredDecommits);
-#endif
void scavenge(UniqueLockHolder&, BulkDecommit&, size_t& freed, size_t goal);
size_t freeableMemory(UniqueLockHolder&);
@@ -147,10 +142,6 @@
#if ENABLE_PHYSICAL_PAGE_MAP
PhysicalPageMap m_physicalPageMap;
#endif
-
-#if BUSE(PARTIAL_SCAVENGE)
- void* m_highWatermark { nullptr };
-#endif
};
inline void Heap::allocateSmallBumpRanges(
Modified: trunk/Source/bmalloc/bmalloc/IsoDirectory.h (278277 => 278278)
--- trunk/Source/bmalloc/bmalloc/IsoDirectory.h 2021-05-31 15:02:54 UTC (rev 278277)
+++ trunk/Source/bmalloc/bmalloc/IsoDirectory.h 2021-05-31 15:04:24 UTC (rev 278278)
@@ -76,9 +76,6 @@
// Iterate over all empty and committed pages, and put them into the vector. This also records the
// pages as being decommitted. It's the caller's job to do the actual decommitting.
void scavenge(const LockHolder&, Vector<DeferredDecommit>&);
-#if BUSE(PARTIAL_SCAVENGE)
- void scavengeToHighWatermark(const LockHolder&, Vector<DeferredDecommit>&);
-#endif
template<typename Func>
void forEachCommittedPage(const LockHolder&, const Func&);
@@ -93,9 +90,6 @@
Bits<numPages> m_empty;
Bits<numPages> m_committed;
unsigned m_firstEligibleOrDecommitted { 0 };
-#if BUSE(PARTIAL_SCAVENGE)
- unsigned m_highWatermark { 0 };
-#endif
};
} // namespace bmalloc
Modified: trunk/Source/bmalloc/bmalloc/IsoDirectoryInlines.h (278277 => 278278)
--- trunk/Source/bmalloc/bmalloc/IsoDirectoryInlines.h 2021-05-31 15:02:54 UTC (rev 278277)
+++ trunk/Source/bmalloc/bmalloc/IsoDirectoryInlines.h 2021-05-31 15:04:24 UTC (rev 278278)
@@ -50,12 +50,7 @@
if (pageIndex >= numPages)
return EligibilityKind::Full;
-#if BUSE(PARTIAL_SCAVENGE)
- m_highWatermark = std::max(pageIndex, m_highWatermark);
-#endif
-
Scavenger& scavenger = *Scavenger::get();
- scavenger.didStartGrowing();
IsoPage<Config>* page = m_pages[pageIndex].get();
@@ -146,25 +141,9 @@
[&] (size_t index) {
scavengePage(locker, index, decommits);
});
-#if BUSE(PARTIAL_SCAVENGE)
- m_highWatermark = 0;
-#endif
}
-#if BUSE(PARTIAL_SCAVENGE)
template<typename Config, unsigned passedNumPages>
-void IsoDirectory<Config, passedNumPages>::scavengeToHighWatermark(const LockHolder& locker, Vector<DeferredDecommit>& decommits)
-{
- (m_empty & m_committed).forEachSetBit(
- [&] (size_t index) {
- if (index > m_highWatermark)
- scavengePage(locker, index, decommits);
- });
- m_highWatermark = 0;
-}
-#endif
-
-template<typename Config, unsigned passedNumPages>
template<typename Func>
void IsoDirectory<Config, passedNumPages>::forEachCommittedPage(const LockHolder&, const Func& func)
{
Modified: trunk/Source/bmalloc/bmalloc/IsoHeapImpl.h (278277 => 278278)
--- trunk/Source/bmalloc/bmalloc/IsoHeapImpl.h 2021-05-31 15:02:54 UTC (rev 278277)
+++ trunk/Source/bmalloc/bmalloc/IsoHeapImpl.h 2021-05-31 15:04:24 UTC (rev 278278)
@@ -49,9 +49,6 @@
virtual ~IsoHeapImplBase();
virtual void scavenge(Vector<DeferredDecommit>&) = 0;
-#if BUSE(PARTIAL_SCAVENGE)
- virtual void scavengeToHighWatermark(Vector<DeferredDecommit>&) = 0;
-#endif
void scavengeNow();
static void finishScavenging(Vector<DeferredDecommit>&);
@@ -112,9 +109,6 @@
void didBecomeEligibleOrDecommited(const LockHolder&, IsoDirectory<Config, IsoDirectoryPage<Config>::numPages>*);
void scavenge(Vector<DeferredDecommit>&) override;
-#if BUSE(PARTIAL_SCAVENGE)
- void scavengeToHighWatermark(Vector<DeferredDecommit>&) override;
-#endif
unsigned allocatorOffset();
unsigned deallocatorOffset();
Modified: trunk/Source/bmalloc/bmalloc/IsoHeapImplInlines.h (278277 => 278278)
--- trunk/Source/bmalloc/bmalloc/IsoHeapImplInlines.h 2021-05-31 15:02:54 UTC (rev 278277)
+++ trunk/Source/bmalloc/bmalloc/IsoHeapImplInlines.h 2021-05-31 15:04:24 UTC (rev 278278)
@@ -121,21 +121,6 @@
m_directoryHighWatermark = 0;
}
-#if BUSE(PARTIAL_SCAVENGE)
-template<typename Config>
-void IsoHeapImpl<Config>::scavengeToHighWatermark(Vector<DeferredDecommit>& decommits)
-{
- LockHolder locker(this->lock);
- if (!m_directoryHighWatermark)
- m_inlineDirectory.scavengeToHighWatermark(locker, decommits);
- for (IsoDirectoryPage<Config>* page = m_headDirectory.get(); page; page = page->next) {
- if (page->index() >= m_directoryHighWatermark)
- page->payload.scavengeToHighWatermark(locker, decommits);
- }
- m_directoryHighWatermark = 0;
-}
-#endif
-
inline size_t IsoHeapImplBase::freeableMemory()
{
return m_freeableMemory;
Modified: trunk/Source/bmalloc/bmalloc/IsoSharedHeapInlines.h (278277 => 278278)
--- trunk/Source/bmalloc/bmalloc/IsoSharedHeapInlines.h 2021-05-31 15:02:54 UTC (rev 278277)
+++ trunk/Source/bmalloc/bmalloc/IsoSharedHeapInlines.h 2021-05-31 15:04:24 UTC (rev 278278)
@@ -63,7 +63,6 @@
BNO_INLINE void* IsoSharedHeap::allocateSlow(const LockHolder& locker, bool abortOnFailure)
{
Scavenger& scavenger = *Scavenger::get();
- scavenger.didStartGrowing();
scavenger.scheduleIfUnderMemoryPressure(IsoSharedPage::pageSize);
IsoSharedPage* page = IsoSharedPage::tryCreate();
Modified: trunk/Source/bmalloc/bmalloc/LargeMap.cpp (278277 => 278278)
--- trunk/Source/bmalloc/bmalloc/LargeMap.cpp 2021-05-31 15:02:54 UTC (rev 278277)
+++ trunk/Source/bmalloc/bmalloc/LargeMap.cpp 2021-05-31 15:04:24 UTC (rev 278278)
@@ -76,9 +76,7 @@
merged = merge(merged, m_free.pop(i--));
}
-#if !BUSE(PARTIAL_SCAVENGE)
merged.setUsedSinceLastScavenge();
-#endif
m_free.push(merged);
}
Modified: trunk/Source/bmalloc/bmalloc/LargeRange.h (278277 => 278278)
--- trunk/Source/bmalloc/bmalloc/LargeRange.h 2021-05-31 15:02:54 UTC (rev 278277)
+++ trunk/Source/bmalloc/bmalloc/LargeRange.h 2021-05-31 15:04:24 UTC (rev 278278)
@@ -37,40 +37,29 @@
: Range()
, m_startPhysicalSize(0)
, m_totalPhysicalSize(0)
-#if !BUSE(PARTIAL_SCAVENGE)
+ , m_physicalEnd(begin())
, m_isEligible(true)
, m_usedSinceLastScavenge(false)
-#endif
{
}
- LargeRange(const Range& other, size_t startPhysicalSize, size_t totalPhysicalSize)
+ LargeRange(const Range& other, size_t startPhysicalSize, size_t totalPhysicalSize, void* physicalEnd)
: Range(other)
, m_startPhysicalSize(startPhysicalSize)
, m_totalPhysicalSize(totalPhysicalSize)
-#if !BUSE(PARTIAL_SCAVENGE)
+ , m_physicalEnd(static_cast<char*>(physicalEnd))
, m_isEligible(true)
, m_usedSinceLastScavenge(false)
-#endif
{
BASSERT(this->size() >= this->totalPhysicalSize());
BASSERT(this->totalPhysicalSize() >= this->startPhysicalSize());
}
-#if BUSE(PARTIAL_SCAVENGE)
- LargeRange(void* begin, size_t size, size_t startPhysicalSize, size_t totalPhysicalSize)
+ LargeRange(void* begin, size_t size, size_t startPhysicalSize, size_t totalPhysicalSize, void* physicalEnd, bool usedSinceLastScavenge = false)
: Range(begin, size)
, m_startPhysicalSize(startPhysicalSize)
, m_totalPhysicalSize(totalPhysicalSize)
- {
- BASSERT(this->size() >= this->totalPhysicalSize());
- BASSERT(this->totalPhysicalSize() >= this->startPhysicalSize());
- }
-#else
- LargeRange(void* begin, size_t size, size_t startPhysicalSize, size_t totalPhysicalSize, bool usedSinceLastScavenge = false)
- : Range(begin, size)
- , m_startPhysicalSize(startPhysicalSize)
- , m_totalPhysicalSize(totalPhysicalSize)
+ , m_physicalEnd(static_cast<char*>(physicalEnd))
, m_isEligible(true)
, m_usedSinceLastScavenge(usedSinceLastScavenge)
{
@@ -77,7 +66,6 @@
BASSERT(this->size() >= this->totalPhysicalSize());
BASSERT(this->totalPhysicalSize() >= this->startPhysicalSize());
}
-#endif
// Returns a lower bound on physical size at the start of the range. Ranges that
// span non-physical fragments use this number to remember the physical size of
@@ -98,6 +86,12 @@
// doesn't really affect accuracy.
size_t totalPhysicalSize() const { return m_totalPhysicalSize; }
void setTotalPhysicalSize(size_t totalPhysicalSize) { m_totalPhysicalSize = totalPhysicalSize; }
+
+ // This is the address past the end of physical memory in this range.
+ // When decomitting this range, we decommitt [begin(), physicalEnd).
+ char* physicalEnd() const { return m_physicalEnd; }
+ void setPhysicalEnd(void* physicalEnd) { m_physicalEnd = static_cast<char*>(physicalEnd); }
+ void clearPhysicalEnd() { m_physicalEnd = begin(); }
std::pair<LargeRange, LargeRange> split(size_t) const;
@@ -104,11 +98,9 @@
void setEligible(bool eligible) { m_isEligible = eligible; }
bool isEligibile() const { return m_isEligible; }
-#if !BUSE(PARTIAL_SCAVENGE)
bool usedSinceLastScavenge() const { return m_usedSinceLastScavenge; }
void clearUsedSinceLastScavenge() { m_usedSinceLastScavenge = false; }
void setUsedSinceLastScavenge() { m_usedSinceLastScavenge = true; }
-#endif
bool operator<(const void* other) const { return begin() < other; }
bool operator<(const LargeRange& other) const { return begin() < other.begin(); }
@@ -116,12 +108,9 @@
private:
size_t m_startPhysicalSize;
size_t m_totalPhysicalSize;
-#if BUSE(PARTIAL_SCAVENGE)
- bool m_isEligible { true };
-#else
+ char* m_physicalEnd;
unsigned m_isEligible: 1;
unsigned m_usedSinceLastScavenge: 1;
-#endif
};
inline bool canMerge(const LargeRange& a, const LargeRange& b)
@@ -144,18 +133,17 @@
inline LargeRange merge(const LargeRange& a, const LargeRange& b)
{
const LargeRange& left = std::min(a, b);
-#if !BUSE(PARTIAL_SCAVENGE)
+ const LargeRange& right = std::max(a, b);
+ void* physicalEnd = right.totalPhysicalSize() ? right.physicalEnd() : left.physicalEnd();
bool mergedUsedSinceLastScavenge = a.usedSinceLastScavenge() || b.usedSinceLastScavenge();
-#endif
if (left.size() == left.startPhysicalSize()) {
return LargeRange(
left.begin(),
a.size() + b.size(),
a.startPhysicalSize() + b.startPhysicalSize(),
- a.totalPhysicalSize() + b.totalPhysicalSize()
-#if !BUSE(PARTIAL_SCAVENGE)
+ a.totalPhysicalSize() + b.totalPhysicalSize(),
+ physicalEnd
, mergedUsedSinceLastScavenge
-#endif
);
}
@@ -164,10 +152,9 @@
left.begin(),
a.size() + b.size(),
left.startPhysicalSize(),
- a.totalPhysicalSize() + b.totalPhysicalSize()
-#if !BUSE(PARTIAL_SCAVENGE)
+ a.totalPhysicalSize() + b.totalPhysicalSize(),
+ physicalEnd
, mergedUsedSinceLastScavenge
-#endif
);
}
@@ -175,11 +162,12 @@
{
BASSERT(leftSize <= this->size());
size_t rightSize = this->size() - leftSize;
+ char* physicalEnd = this->physicalEnd();
if (leftSize <= startPhysicalSize()) {
BASSERT(totalPhysicalSize() >= leftSize);
- LargeRange left(begin(), leftSize, leftSize, leftSize);
- LargeRange right(left.end(), rightSize, startPhysicalSize() - leftSize, totalPhysicalSize() - leftSize);
+ LargeRange left(begin(), leftSize, leftSize, leftSize, std::min(physicalEnd, begin() + leftSize));
+ LargeRange right(left.end(), rightSize, startPhysicalSize() - leftSize, totalPhysicalSize() - leftSize, std::max(physicalEnd, left.end()));
return std::make_pair(left, right);
}
@@ -194,8 +182,8 @@
rightTotalPhysicalSize = rightSize;
}
- LargeRange left(begin(), leftSize, startPhysicalSize(), leftTotalPhysicalSize);
- LargeRange right(left.end(), rightSize, 0, rightTotalPhysicalSize);
+ LargeRange left(begin(), leftSize, startPhysicalSize(), leftTotalPhysicalSize, std::min(physicalEnd, begin() + leftSize));
+ LargeRange right(left.end(), rightSize, 0, rightTotalPhysicalSize, std::max(physicalEnd, left.end()));
return std::make_pair(left, right);
}
Modified: trunk/Source/bmalloc/bmalloc/Scavenger.cpp (278277 => 278278)
--- trunk/Source/bmalloc/bmalloc/Scavenger.cpp 2021-05-31 15:02:54 UTC (rev 278277)
+++ trunk/Source/bmalloc/bmalloc/Scavenger.cpp 2021-05-31 15:04:24 UTC (rev 278278)
@@ -85,11 +85,7 @@
dispatch_resume(m_pressureHandlerDispatchSource);
dispatch_release(queue);
#endif
-#if BUSE(PARTIAL_SCAVENGE)
- m_waitTime = std::chrono::milliseconds(m_isInMiniMode ? 200 : 2000);
-#else
m_waitTime = std::chrono::milliseconds(10);
-#endif
m_thread = std::thread(&threadEntryPoint, this);
}
@@ -120,12 +116,6 @@
m_condition.notify_all();
}
-void Scavenger::didStartGrowing()
-{
- // We don't really need to lock here, since this is just a heuristic.
- m_isProbablyGrowing = true;
-}
-
void Scavenger::scheduleIfUnderMemoryPressure(size_t bytes)
{
LockHolder lock(mutex());
@@ -146,7 +136,6 @@
if (!isUnderMemoryPressure())
return;
- m_isProbablyGrowing = false;
run(lock);
}
@@ -158,7 +147,6 @@
if (willRunSoon())
return;
- m_isProbablyGrowing = false;
runSoon(lock);
}
@@ -187,14 +175,6 @@
return std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - m_lastFullScavengeTime);
}
-#if BUSE(PARTIAL_SCAVENGE)
-std::chrono::milliseconds Scavenger::timeSinceLastPartialScavenge()
-{
- UniqueLockHolder lock(mutex());
- return std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - m_lastPartialScavengeTime);
-}
-#endif
-
void Scavenger::enableMiniMode()
{
m_isInMiniMode = true; // We just store to this racily. The scavenger thread will eventually pick up the right value.
@@ -220,25 +200,17 @@
{
PrintTime printTime("\nfull scavenge under lock time");
-#if !BUSE(PARTIAL_SCAVENGE)
size_t deferredDecommits = 0;
-#endif
UniqueLockHolder lock(Heap::mutex());
for (unsigned i = numHeaps; i--;) {
if (!isActiveHeapKind(static_cast<HeapKind>(i)))
continue;
-#if BUSE(PARTIAL_SCAVENGE)
- PerProcess<PerHeapKind<Heap>>::get()->at(i).scavenge(lock, decommitter);
-#else
PerProcess<PerHeapKind<Heap>>::get()->at(i).scavenge(lock, decommitter, deferredDecommits);
-#endif
}
decommitter.processEager();
-#if !BUSE(PARTIAL_SCAVENGE)
if (deferredDecommits)
m_state = State::RunSoon;
-#endif
}
{
@@ -279,78 +251,6 @@
}
}
-#if BUSE(PARTIAL_SCAVENGE)
-void Scavenger::partialScavenge()
-{
- if (!m_isEnabled)
- return;
-
- UniqueLockHolder lock(m_scavengingMutex);
-
- if (verbose) {
- fprintf(stderr, "--------------------------------\n");
- fprintf(stderr, "--before partial scavenging--\n");
- dumpStats();
- }
-
- {
- BulkDecommit decommitter;
- {
- PrintTime printTime("\npartialScavenge under lock time");
- UniqueLockHolder lock(Heap::mutex());
- for (unsigned i = numHeaps; i--;) {
- if (!isActiveHeapKind(static_cast<HeapKind>(i)))
- continue;
- Heap& heap = PerProcess<PerHeapKind<Heap>>::get()->at(i);
- size_t freeableMemory = heap.freeableMemory(lock);
- if (freeableMemory < 4 * MB)
- continue;
- heap.scavengeToHighWatermark(lock, decommitter);
- }
-
- decommitter.processEager();
- }
-
- {
- PrintTime printTime("partialScavenge lazy decommit time");
- decommitter.processLazy();
- }
-
- {
- PrintTime printTime("partialScavenge mark all as eligible time");
- LockHolder lock(Heap::mutex());
- for (unsigned i = numHeaps; i--;) {
- if (!isActiveHeapKind(static_cast<HeapKind>(i)))
- continue;
- Heap& heap = PerProcess<PerHeapKind<Heap>>::get()->at(i);
- heap.markAllLargeAsEligibile(lock);
- }
- }
- }
-
- {
- RELEASE_BASSERT(!m_deferredDecommits.size());
- AllIsoHeaps::get()->forEach(
- [&] (IsoHeapImplBase& heap) {
- heap.scavengeToHighWatermark(m_deferredDecommits);
- });
- IsoHeapImplBase::finishScavenging(m_deferredDecommits);
- m_deferredDecommits.shrink(0);
- }
-
- if (verbose) {
- fprintf(stderr, "--after partial scavenging--\n");
- dumpStats();
- fprintf(stderr, "--------------------------------\n");
- }
-
- {
- UniqueLockHolder lock(mutex());
- m_lastPartialScavengeTime = std::chrono::steady_clock::now();
- }
-}
-#endif
-
size_t Scavenger::freeableMemory()
{
size_t result = 0;
@@ -432,69 +332,6 @@
fprintf(stderr, "--------------------------------\n");
}
-#if BUSE(PARTIAL_SCAVENGE)
- enum class ScavengeMode {
- None,
- Partial,
- Full
- };
-
- size_t freeableMemory = this->freeableMemory();
-
- ScavengeMode scavengeMode = [&] {
- auto timeSinceLastFullScavenge = this->timeSinceLastFullScavenge();
- auto timeSinceLastPartialScavenge = this->timeSinceLastPartialScavenge();
- auto timeSinceLastScavenge = std::min(timeSinceLastPartialScavenge, timeSinceLastFullScavenge);
-
- if (isUnderMemoryPressure() && freeableMemory > 1 * MB && timeSinceLastScavenge > std::chrono::milliseconds(5))
- return ScavengeMode::Full;
-
- if (!m_isProbablyGrowing) {
- if (timeSinceLastFullScavenge < std::chrono::milliseconds(1000) && !m_isInMiniMode)
- return ScavengeMode::Partial;
- return ScavengeMode::Full;
- }
-
- if (m_isInMiniMode) {
- if (timeSinceLastFullScavenge < std::chrono::milliseconds(200))
- return ScavengeMode::Partial;
- return ScavengeMode::Full;
- }
-
-#if BCPU(X86_64)
- auto partialScavengeInterval = std::chrono::milliseconds(12000);
-#else
- auto partialScavengeInterval = std::chrono::milliseconds(8000);
-#endif
- if (timeSinceLastScavenge < partialScavengeInterval) {
- // Rate limit partial scavenges.
- return ScavengeMode::None;
- }
- if (freeableMemory < 25 * MB)
- return ScavengeMode::None;
- if (5 * freeableMemory < footprint())
- return ScavengeMode::None;
- return ScavengeMode::Partial;
- }();
-
- m_isProbablyGrowing = false;
-
- switch (scavengeMode) {
- case ScavengeMode::None: {
- runSoon();
- break;
- }
- case ScavengeMode::Partial: {
- partialScavenge();
- runSoon();
- break;
- }
- case ScavengeMode::Full: {
- scavenge();
- break;
- }
- }
-#else
std::chrono::steady_clock::time_point start { std::chrono::steady_clock::now() };
scavenge();
@@ -509,14 +346,13 @@
// FIXME: We need to investigate mini-mode's adjustment.
// https://bugs.webkit.org/show_bug.cgi?id=203987
if (!m_isInMiniMode) {
- timeSpentScavenging *= 150;
+ timeSpentScavenging *= s_newWaitMultiplier;
std::chrono::milliseconds newWaitTime = std::chrono::duration_cast<std::chrono::milliseconds>(timeSpentScavenging);
- m_waitTime = std::min(std::max(newWaitTime, std::chrono::milliseconds(100)), std::chrono::milliseconds(10000));
+ m_waitTime = std::min(std::max(newWaitTime, std::chrono::milliseconds(s_minWaitTimeMilliseconds)), std::chrono::milliseconds(s_maxWaitTimeMilliseconds));
}
if (verbose)
fprintf(stderr, "new wait time %lldms\n", static_cast<long long int>(m_waitTime.count()));
-#endif
}
}
Modified: trunk/Source/bmalloc/bmalloc/Scavenger.h (278277 => 278278)
--- trunk/Source/bmalloc/bmalloc/Scavenger.h 2021-05-31 15:02:54 UTC (rev 278277)
+++ trunk/Source/bmalloc/bmalloc/Scavenger.h 2021-05-31 15:04:24 UTC (rev 278278)
@@ -59,7 +59,6 @@
bool willRunSoon() { return m_state > State::Sleep; }
void runSoon();
- BEXPORT void didStartGrowing();
BEXPORT void scheduleIfUnderMemoryPressure(size_t bytes);
BEXPORT void schedule(size_t bytes);
@@ -92,15 +91,10 @@
void setThreadName(const char*);
std::chrono::milliseconds timeSinceLastFullScavenge();
-#if BUSE(PARTIAL_SCAVENGE)
- std::chrono::milliseconds timeSinceLastPartialScavenge();
- void partialScavenge();
-#endif
std::atomic<State> m_state { State::Sleep };
size_t m_scavengerBytes { 0 };
std::chrono::milliseconds m_waitTime;
- bool m_isProbablyGrowing { false };
bool m_isInMiniMode { false };
Mutex m_scavengingMutex;
@@ -108,9 +102,6 @@
std::thread m_thread;
std::chrono::steady_clock::time_point m_lastFullScavengeTime { std::chrono::steady_clock::now() };
-#if BUSE(PARTIAL_SCAVENGE)
- std::chrono::steady_clock::time_point m_lastPartialScavengeTime { std::chrono::steady_clock::now() };
-#endif
#if BOS(DARWIN)
dispatch_source_t m_pressureHandlerDispatchSource;
@@ -117,6 +108,16 @@
qos_class_t m_requestedScavengerThreadQOSClass { QOS_CLASS_USER_INITIATED };
#endif
+#if BPLATFORM(MAC)
+ const unsigned s_newWaitMultiplier = 300;
+ const unsigned s_minWaitTimeMilliseconds = 750;
+ const unsigned s_maxWaitTimeMilliseconds = 20000;
+#else
+ const unsigned s_newWaitMultiplier = 150;
+ const unsigned s_minWaitTimeMilliseconds = 100;
+ const unsigned s_maxWaitTimeMilliseconds = 10000;
+#endif
+
Vector<DeferredDecommit> m_deferredDecommits;
bool m_isEnabled { true };
};
Modified: trunk/Source/bmalloc/bmalloc/SmallPage.h (278277 => 278278)
--- trunk/Source/bmalloc/bmalloc/SmallPage.h 2021-05-31 15:02:54 UTC (rev 278277)
+++ trunk/Source/bmalloc/bmalloc/SmallPage.h 2021-05-31 15:04:24 UTC (rev 278278)
@@ -51,11 +51,9 @@
bool hasPhysicalPages() { return m_hasPhysicalPages; }
void setHasPhysicalPages(bool hasPhysicalPages) { m_hasPhysicalPages = hasPhysicalPages; }
-#if !BUSE(PARTIAL_SCAVENGE)
bool usedSinceLastScavenge() { return m_usedSinceLastScavenge; }
void clearUsedSinceLastScavenge() { m_usedSinceLastScavenge = false; }
void setUsedSinceLastScavenge() { m_usedSinceLastScavenge = true; }
-#endif
SmallLine* begin();
@@ -65,9 +63,7 @@
private:
unsigned char m_hasFreeLines: 1;
unsigned char m_hasPhysicalPages: 1;
-#if !BUSE(PARTIAL_SCAVENGE)
unsigned char m_usedSinceLastScavenge: 1;
-#endif
unsigned char m_refCount: 7;
unsigned char m_sizeClass;
unsigned char m_slide;
