Diff
Modified: trunk/Source/_javascript_Core/ChangeLog (289589 => 289590)
--- trunk/Source/_javascript_Core/ChangeLog 2022-02-10 23:42:48 UTC (rev 289589)
+++ trunk/Source/_javascript_Core/ChangeLog 2022-02-10 23:49:33 UTC (rev 289590)
@@ -1,3 +1,15 @@
+2022-01-24 Filip Pizlo <fpi...@apple.com>
+
+ [libpas] jit_heap should support the segregated heap
+ https://bugs.webkit.org/show_bug.cgi?id=235497
+
+ Reviewed by Yusuke Suzuki.
+
+ Disable the jit_heap segregated heap if the executable pool size is too small.
+
+ * jit/ExecutableAllocator.cpp:
+ (JSC::initializeJITPageReservation):
+
2022-02-09 Yusuke Suzuki <ysuz...@apple.com>
Use local variable pointer for concurrently load value
Modified: trunk/Source/_javascript_Core/jit/ExecutableAllocator.cpp (289589 => 289590)
--- trunk/Source/_javascript_Core/jit/ExecutableAllocator.cpp 2022-02-10 23:42:48 UTC (rev 289589)
+++ trunk/Source/_javascript_Core/jit/ExecutableAllocator.cpp 2022-02-10 23:49:33 UTC (rev 289590)
@@ -1,5 +1,5 @@
/*
- * Copyright (C) 2008-2021 Apple Inc. All rights reserved.
+ * Copyright (C) 2008-2022 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -43,6 +43,7 @@
#if USE(LIBPAS_JIT_HEAP)
#include <bmalloc/jit_heap.h>
+#include <bmalloc/jit_heap_config.h>
#else
#include <wtf/MetaAllocator.h>
#endif
@@ -84,6 +85,10 @@
using namespace WTF;
+#if USE(LIBPAS_JIT_HEAP)
+static constexpr size_t minimumPoolSizeForSegregatedHeap = 256 * MB;
+#endif
+
#if defined(FIXED_EXECUTABLE_MEMORY_POOL_SIZE_IN_MB) && FIXED_EXECUTABLE_MEMORY_POOL_SIZE_IN_MB > 0
static constexpr size_t fixedExecutableMemoryPoolSize = FIXED_EXECUTABLE_MEMORY_POOL_SIZE_IN_MB * MB;
#elif CPU(ARM)
@@ -354,6 +359,11 @@
}
reservation.size = std::max(roundUpToMultipleOf(pageSize(), reservation.size), pageSize() * 2);
+#if USE(LIBPAS_JIT_HEAP)
+ if (reservation.size < minimumPoolSizeForSegregatedHeap)
+ jit_heap_runtime_config.max_segregated_object_size = 0;
+#endif
+
auto tryCreatePageReservation = [] (size_t reservationSize) {
#if OS(LINUX)
// If we use uncommitted reservation, mmap operation is recorded with small page size in perf command's output.
@@ -770,7 +780,7 @@
RELEASE_ASSERT(!m_start);
RELEASE_ASSERT(!m_end);
m_start = reinterpret_cast<uintptr_t>(start);
- m_end = m_start + sizeInBytes;
+ m_end = m_start + sizeInBytes;
jit_heap_add_fresh_memory(pas_range_create(m_start, m_end));
}
Modified: trunk/Source/_javascript_Core/jit/ExecutableMemoryHandle.h (289589 => 289590)
--- trunk/Source/_javascript_Core/jit/ExecutableMemoryHandle.h 2022-02-10 23:42:48 UTC (rev 289589)
+++ trunk/Source/_javascript_Core/jit/ExecutableMemoryHandle.h 2022-02-10 23:49:33 UTC (rev 289590)
@@ -28,7 +28,6 @@
#if USE(LIBPAS_JIT_HEAP) && ENABLE(JIT)
#include <wtf/MetaAllocatorPtr.h>
#include <wtf/ThreadSafeRefCounted.h>
-#include <bmalloc/jit_heap.h>
#else
#include <wtf/MetaAllocatorHandle.h>
#endif
Modified: trunk/Source/bmalloc/ChangeLog (289589 => 289590)
--- trunk/Source/bmalloc/ChangeLog 2022-02-10 23:42:48 UTC (rev 289589)
+++ trunk/Source/bmalloc/ChangeLog 2022-02-10 23:49:33 UTC (rev 289590)
@@ -1,3 +1,111 @@
+2022-01-23 Filip Pizlo <fpi...@apple.com>
+
+ [libpas] jit_heap should support the segregated heap
+ https://bugs.webkit.org/show_bug.cgi?id=235497
+
+ Reviewed by Yusuke Suzuki.
+
+ One of the things that libpas provides is a fast implementation of executable memory allocation, which
+ libpas calls "jit_heap". The initial implementation of this only used libpas's bitfit and large
+ algorithms. Bitfit was used for smaller objects and large was used for larger objects. The libpas
+ segregated heap was disabled in jit_heap. This made sense because:
+
+ - Bitfit and large support the shrink() call, where we shrink an allocation in-place. Executable
+ allocation supports this to aid branch compaction (we compact code after we copy it into its final
+ resting place, which is after we allocate the memory - so after we finish compaction, we can only
+ shrink the allocation in-place or not at all).
+
+ - Segregated heaps have some virtual address space overheads, and memory overheads, that the bitfit
+ and large heaps don't have. This happens because segreated heaps will dedicate each page to objects
+ of exactly one size forever.
+
+ However, segregated heaps are substantially faster than bitfit heaps. They are faster under serial
+ allocation thanks to the fact that no locks need to be acquired for a typical allocation or
+ deallocation. They are even more faster under parallel allocation thanks to thread-local caching.
+
+ This patch enables segregated heaps for the smallest jit_heap allocations (<=2000 bytes). The cutoff
+ threshold is runtime configurable, so it's possible to disable segregated heaps at runtime by setting
+ the cutoff to 0.
+
+ With the 2000 threshold, this appears to be a 0.3% Speedometer2 speed-up.
+
+ While there is a theoretical possibility of memory overhead, I haven't seen it. If we are using jump
+ islands then the virtual memory overhead of segregated heaps will just mean that we use jump islands
+ more frequently, which would manifest as a performance regression -- but I've not seen any such
+ regression. So, this disables the segregated heap if the JIT pool size is too small (see changes to
+ ExecutableAllocator.cpp in JSC).
+
+ * libpas/libpas.xcodeproj/project.pbxproj:
+ * libpas/src/libpas/jit_heap.c:
+ (jit_heap_shrink):
+ * libpas/src/libpas/jit_heap_config.c:
+ (jit_small_segregated_allocate_page):
+ (jit_small_segregated_create_page_header):
+ (jit_small_destroy_page_header):
+ (jit_small_segregated_shared_page_directory_selector):
+ (jit_small_bitfit_allocate_page):
+ (jit_small_bitfit_create_page_header):
+ (jit_medium_bitfit_create_page_header):
+ (jit_medium_destroy_page_header):
+ (jit_prepare_to_enumerate):
+ (jit_small_bitfit_destroy_page_header): Deleted.
+ (jit_medium_bitfit_destroy_page_header): Deleted.
+ * libpas/src/libpas/jit_heap_config.h:
+ (jit_small_page_header_for_boundary):
+ (jit_small_boundary_for_page_header):
+ (jit_medium_page_header_for_boundary):
+ (jit_medium_boundary_for_page_header):
+ (jit_heap_config_page_header):
+ (jit_small_bitfit_page_header_for_boundary): Deleted.
+ (jit_small_bitfit_boundary_for_page_header): Deleted.
+ (jit_medium_bitfit_page_header_for_boundary): Deleted.
+ (jit_medium_bitfit_boundary_for_page_header): Deleted.
+ * libpas/src/libpas/jit_heap_config_root_data.h:
+ * libpas/src/libpas/pas_bitfit_heap.c:
+ (pas_bitfit_heap_select_variant):
+ (pas_bitfit_heap_construct_and_insert_size_class):
+ * libpas/src/libpas/pas_bitfit_heap.h:
+ * libpas/src/libpas/pas_bitfit_page_config.h:
+ (pas_bitfit_page_config_is_enabled):
+ * libpas/src/libpas/pas_heap_config.h:
+ (pas_heap_config_segregated_heap_min_align_shift):
+ (pas_heap_config_segregated_heap_min_align):
+ * libpas/src/libpas/pas_segregated_heap.c:
+ (min_align_for_heap):
+ (min_object_size_for_heap):
+ (max_segregated_object_size_for_heap):
+ (max_bitfit_object_size_for_heap):
+ (max_object_size_for_heap):
+ (pas_segregated_heap_ensure_allocator_index):
+ (pas_segregated_heap_ensure_size_directory_for_size):
+ (min_object_size_for_heap_config): Deleted.
+ (max_segregated_object_size_for_heap_config): Deleted.
+ (max_bitfit_object_size_for_heap_config): Deleted.
+ (max_object_size_for_heap_config): Deleted.
+ * libpas/src/libpas/pas_segregated_heap.h:
+ (pas_segregated_heap_index_for_size):
+ (pas_segregated_heap_size_for_index):
+ * libpas/src/libpas/pas_segregated_page.c:
+ (pas_segregated_page_construct):
+ * libpas/src/libpas/pas_segregated_page_config.h:
+ (pas_segregated_page_config_is_enabled):
+ * libpas/src/libpas/pas_segregated_page_config_kind.def:
+ * libpas/src/libpas/pas_segregated_page_config_kind_and_role.h:
+ * libpas/src/libpas/pas_segregated_size_directory.c:
+ (pas_segregated_size_directory_create):
+ * libpas/src/test/IsoHeapChaosTests.cpp:
+ (std::addAllTests):
+ * libpas/src/test/JITHeapTests.cpp:
+ (std::testAllocationSize):
+ (addJITHeapTests):
+ * libpas/src/test/TestHarness.cpp:
+ (main):
+ * libpas/src/test/ViewCacheTests.cpp: Added.
+ (std::setupConfig):
+ (std::testDisableViewCacheUsingBoundForNoViewCache):
+ (std::testEnableViewCacheAtSomeBoundForNoViewCache):
+ (addViewCacheTests):
+
2022-01-18 Filip Pizlo <fpi...@apple.com>
[libpas] it should be possible to decommit unused parts of a thread_local_cache (update to 78508e79fa2797680344d43b94841ae2e903b15b)
Modified: trunk/Source/bmalloc/bmalloc.xcodeproj/project.pbxproj (289589 => 289590)
--- trunk/Source/bmalloc/bmalloc.xcodeproj/project.pbxproj 2022-02-10 23:42:48 UTC (rev 289589)
+++ trunk/Source/bmalloc/bmalloc.xcodeproj/project.pbxproj 2022-02-10 23:49:33 UTC (rev 289590)
@@ -38,9 +38,9 @@
0F5167741FAD685C008236A8 /* bmalloc.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 0F5167731FAD6852008236A8 /* bmalloc.cpp */; };
0F516EE52456184F004E2B8D /* pas_internal_config.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F516EE42456184F004E2B8D /* pas_internal_config.h */; settings = {ATTRIBUTES = (Private, ); }; };
0F516EE924561AE9004E2B8D /* iso_heap_innards.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F516EE824561AE8004E2B8D /* iso_heap_innards.h */; settings = {ATTRIBUTES = (Private, ); }; };
- 0F5193EE266C42AD00483A2C /* jit_heap_config.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F5193E8266C42AC00483A2C /* jit_heap_config.h */; };
+ 0F5193EE266C42AD00483A2C /* jit_heap_config.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F5193E8266C42AC00483A2C /* jit_heap_config.h */; settings = {ATTRIBUTES = (Private, ); }; };
0F5193EF266C42AD00483A2C /* jit_heap.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F5193E9266C42AC00483A2C /* jit_heap.h */; settings = {ATTRIBUTES = (Private, ); }; };
- 0F5193F0266C42AD00483A2C /* jit_heap_config_root_data.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F5193EA266C42AC00483A2C /* jit_heap_config_root_data.h */; };
+ 0F5193F0266C42AD00483A2C /* jit_heap_config_root_data.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F5193EA266C42AC00483A2C /* jit_heap_config_root_data.h */; settings = {ATTRIBUTES = (Private, ); }; };
0F5193F1266C42AD00483A2C /* jit_heap_config.c in Sources */ = {isa = PBXBuildFile; fileRef = 0F5193EB266C42AC00483A2C /* jit_heap_config.c */; };
0F5193F2266C42AD00483A2C /* jit_heap.c in Sources */ = {isa = PBXBuildFile; fileRef = 0F5193EC266C42AC00483A2C /* jit_heap.c */; };
0F5193F3266C42AD00483A2C /* pas_basic_heap_config_enumerator_data.c in Sources */ = {isa = PBXBuildFile; fileRef = 0F5193ED266C42AD00483A2C /* pas_basic_heap_config_enumerator_data.c */; };
Modified: trunk/Source/bmalloc/libpas/libpas.xcodeproj/project.pbxproj (289589 => 289590)
--- trunk/Source/bmalloc/libpas/libpas.xcodeproj/project.pbxproj 2022-02-10 23:42:48 UTC (rev 289589)
+++ trunk/Source/bmalloc/libpas/libpas.xcodeproj/project.pbxproj 2022-02-10 23:49:33 UTC (rev 289590)
@@ -554,6 +554,7 @@
2C11E88E2728A783002162D0 /* bmalloc_type.h in Headers */ = {isa = PBXBuildFile; fileRef = 2C11E88C2728A783002162D0 /* bmalloc_type.h */; };
2C11E88F2728A783002162D0 /* pas_simple_type.c in Sources */ = {isa = PBXBuildFile; fileRef = 2C11E88D2728A783002162D0 /* pas_simple_type.c */; };
2C11E8912728A893002162D0 /* bmalloc_type.c in Sources */ = {isa = PBXBuildFile; fileRef = 2C11E8902728A893002162D0 /* bmalloc_type.c */; };
+ 2C16DAE0279E07290042E919 /* ViewCacheTests.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 2C16DADF279E07290042E919 /* ViewCacheTests.cpp */; };
2C340000275815D0005565CB /* pas_segregated_page_config_kind_and_role.c in Sources */ = {isa = PBXBuildFile; fileRef = 2C34FFFF275815D0005565CB /* pas_segregated_page_config_kind_and_role.c */; };
2C34000227581687005565CB /* pas_page_base_config.c in Sources */ = {isa = PBXBuildFile; fileRef = 2C34000127581687005565CB /* pas_page_base_config.c */; };
2C34FFFA27571D2F005565CB /* pas_page_base_and_kind.h in Headers */ = {isa = PBXBuildFile; fileRef = 2C34FFF527571D2F005565CB /* pas_page_base_and_kind.h */; };
@@ -1255,6 +1256,7 @@
2C11E88C2728A783002162D0 /* bmalloc_type.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = bmalloc_type.h; sourceTree = "<group>"; };
2C11E88D2728A783002162D0 /* pas_simple_type.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = pas_simple_type.c; sourceTree = "<group>"; };
2C11E8902728A893002162D0 /* bmalloc_type.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = bmalloc_type.c; sourceTree = "<group>"; };
+ 2C16DADF279E07290042E919 /* ViewCacheTests.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = ViewCacheTests.cpp; sourceTree = "<group>"; };
2C34000127581687005565CB /* pas_page_base_config.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = pas_page_base_config.c; sourceTree = "<group>"; };
2C34FFF527571D2F005565CB /* pas_page_base_and_kind.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = pas_page_base_and_kind.h; sourceTree = "<group>"; };
2C34FFF627571D2F005565CB /* pas_get_page_base_and_kind_for_small_other_in_fast_megapage.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = pas_get_page_base_and_kind_for_small_other_in_fast_megapage.h; sourceTree = "<group>"; };
@@ -1416,6 +1418,7 @@
2C473161277D4C9E00B62C49 /* TLCDecommitTests.cpp */,
0F8700DE25B14C68000E1ABF /* TSDTests.cpp */,
0F53181222C9609D003F7B6A /* UtilsTests.cpp */,
+ 2C16DADF279E07290042E919 /* ViewCacheTests.cpp */,
);
path = test;
sourceTree = "<group>";
@@ -2643,6 +2646,7 @@
0FDEA461228B651B0085E340 /* BitfieldVectorTests.cpp in Sources */,
0F53181122C954ED003F7B6A /* BitvectorTests.cpp in Sources */,
0F31A66823E8B336002C0CA3 /* CartesianTreeTests.cpp in Sources */,
+ 2C16DAE0279E07290042E919 /* ViewCacheTests.cpp in Sources */,
0FC682382129D4EE003C6A13 /* CoalignTests.cpp in Sources */,
2C48132D273F4159006CAB55 /* ExpendableMemoryTests.cpp in Sources */,
0FC682312129CB46003C6A13 /* ExtendedGCDTests.cpp in Sources */,
Modified: trunk/Source/bmalloc/libpas/src/libpas/jit_heap.c (289589 => 289590)
--- trunk/Source/bmalloc/libpas/src/libpas/jit_heap.c 2022-02-10 23:42:48 UTC (rev 289589)
+++ trunk/Source/bmalloc/libpas/src/libpas/jit_heap.c 2022-02-10 23:49:33 UTC (rev 289590)
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2021 Apple Inc. All rights reserved.
+ * Copyright (c) 2021-2022 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -86,9 +86,10 @@
void jit_heap_shrink(void* object, size_t new_size)
{
- bool result;
- result = pas_try_shrink(object, new_size, JIT_HEAP_CONFIG);
- PAS_ASSERT(result);
+ /* NOTE: the shrink call will fail (return false) for segregated allocations, and that's fine because we
+ only use segregated allocations for smaller sizes (so the amount of potential memory savings from
+ shrinking is small). */
+ pas_try_shrink(object, new_size, JIT_HEAP_CONFIG);
}
size_t jit_heap_get_size(void* object)
Modified: trunk/Source/bmalloc/libpas/src/libpas/jit_heap_config.c (289589 => 289590)
--- trunk/Source/bmalloc/libpas/src/libpas/jit_heap_config.c 2022-02-10 23:42:48 UTC (rev 289589)
+++ trunk/Source/bmalloc/libpas/src/libpas/jit_heap_config.c 2022-02-10 23:49:33 UTC (rev 289590)
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2021 Apple Inc. All rights reserved.
+ * Copyright (c) 2021-2022 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -37,6 +37,7 @@
#include "pas_enumerable_page_malloc.h"
#include "pas_heap_config_inlines.h"
#include "pas_root.h"
+#include "pas_segregated_page_config_inlines.h"
#include "pas_stream.h"
#if defined(PAS_BMALLOC)
@@ -102,9 +103,9 @@
.provider_arg = NULL
};
-pas_page_header_table jit_small_bitfit_page_header_table =
+pas_page_header_table jit_small_page_header_table =
PAS_PAGE_HEADER_TABLE_INITIALIZER(JIT_SMALL_PAGE_SIZE);
-pas_page_header_table jit_medium_bitfit_page_header_table =
+pas_page_header_table jit_medium_page_header_table =
PAS_PAGE_HEADER_TABLE_INITIALIZER(JIT_MEDIUM_PAGE_SIZE);
pas_heap_runtime_config jit_heap_runtime_config = {
@@ -114,15 +115,15 @@
.is_part_of_heap = true,
.directory_size_bound_for_partial_views = 0,
.directory_size_bound_for_baseline_allocators = 0,
- .directory_size_bound_for_no_view_cache = UINT_MAX,
- .max_segregated_object_size = 0,
+ .directory_size_bound_for_no_view_cache = 0,
+ .max_segregated_object_size = 2000,
.max_bitfit_object_size = UINT_MAX,
- .view_cache_capacity_for_object_size = pas_heap_runtime_config_zero_view_cache_capacity
+ .view_cache_capacity_for_object_size = pas_heap_runtime_config_aggressive_view_cache_capacity
};
jit_heap_config_root_data jit_root_data = {
- .small_bitfit_page_header_table = &jit_small_bitfit_page_header_table,
- .medium_bitfit_page_header_table = &jit_medium_bitfit_page_header_table
+ .small_page_header_table = &jit_small_page_header_table,
+ .medium_page_header_table = &jit_medium_page_header_table
};
void jit_heap_config_activate(void)
@@ -142,6 +143,50 @@
return (pas_page_base*)pas_ptr_hash_map_get(&data->page_header_table, boundary).value;
}
+void* jit_small_segregated_allocate_page(
+ pas_segregated_heap* heap, pas_physical_memory_transaction* transaction, pas_segregated_page_role role)
+{
+ PAS_ASSERT(role == pas_segregated_page_exclusive_role);
+ PAS_UNUSED_PARAM(heap);
+ PAS_UNUSED_PARAM(transaction);
+ return (void*)allocate_from_fresh(
+ JIT_SMALL_PAGE_SIZE, pas_alignment_create_traditional(JIT_SMALL_PAGE_SIZE)).begin;
+}
+
+pas_page_base* jit_small_segregated_create_page_header(
+ void* boundary, pas_page_kind kind, pas_lock_hold_mode heap_lock_hold_mode)
+{
+ pas_page_base* result;
+ PAS_ASSERT(kind == pas_small_exclusive_segregated_page_kind);
+ pas_heap_lock_lock_conditionally(heap_lock_hold_mode);
+ result = pas_page_header_table_add(
+ &jit_small_page_header_table,
+ JIT_SMALL_PAGE_SIZE,
+ pas_segregated_page_header_size(JIT_HEAP_CONFIG.small_segregated_config,
+ pas_segregated_page_exclusive_role),
+ boundary);
+ pas_heap_lock_unlock_conditionally(heap_lock_hold_mode);
+ return result;
+}
+
+void jit_small_destroy_page_header(pas_page_base* page, pas_lock_hold_mode heap_lock_hold_mode)
+{
+ pas_heap_lock_lock_conditionally(heap_lock_hold_mode);
+ pas_page_header_table_remove(&jit_small_page_header_table,
+ JIT_SMALL_PAGE_SIZE,
+ page);
+ pas_heap_lock_unlock_conditionally(heap_lock_hold_mode);
+}
+
+pas_segregated_shared_page_directory* jit_small_segregated_shared_page_directory_selector(
+ pas_segregated_heap* heap, pas_segregated_size_directory* directory)
+{
+ PAS_UNUSED_PARAM(heap);
+ PAS_UNUSED_PARAM(directory);
+ PAS_ASSERT(!"Not implemented");
+ return NULL;
+}
+
void* jit_small_bitfit_allocate_page(
pas_segregated_heap* heap, pas_physical_memory_transaction* transaction)
{
@@ -157,7 +202,7 @@
pas_page_base* result;
PAS_ASSERT(kind == pas_small_bitfit_page_kind);
pas_heap_lock_lock_conditionally(heap_lock_hold_mode);
- result = pas_page_header_table_add(&jit_small_bitfit_page_header_table,
+ result = pas_page_header_table_add(&jit_small_page_header_table,
JIT_SMALL_PAGE_SIZE,
pas_bitfit_page_header_size(JIT_HEAP_CONFIG.small_bitfit_config),
boundary);
@@ -165,16 +210,6 @@
return result;
}
-void jit_small_bitfit_destroy_page_header(
- pas_page_base* page, pas_lock_hold_mode heap_lock_hold_mode)
-{
- pas_heap_lock_lock_conditionally(heap_lock_hold_mode);
- pas_page_header_table_remove(&jit_small_bitfit_page_header_table,
- JIT_SMALL_PAGE_SIZE,
- page);
- pas_heap_lock_unlock_conditionally(heap_lock_hold_mode);
-}
-
void* jit_medium_bitfit_allocate_page(
pas_segregated_heap* heap, pas_physical_memory_transaction* transaction)
{
@@ -190,7 +225,7 @@
pas_page_base* result;
PAS_ASSERT(kind == pas_medium_bitfit_page_kind);
pas_heap_lock_lock_conditionally(heap_lock_hold_mode);
- result = pas_page_header_table_add(&jit_medium_bitfit_page_header_table,
+ result = pas_page_header_table_add(&jit_medium_page_header_table,
JIT_MEDIUM_PAGE_SIZE,
pas_bitfit_page_header_size(JIT_HEAP_CONFIG.medium_bitfit_config),
boundary);
@@ -198,11 +233,11 @@
return result;
}
-void jit_medium_bitfit_destroy_page_header(
+void jit_medium_destroy_page_header(
pas_page_base* page, pas_lock_hold_mode heap_lock_hold_mode)
{
pas_heap_lock_lock_conditionally(heap_lock_hold_mode);
- pas_page_header_table_remove(&jit_medium_bitfit_page_header_table,
+ pas_page_header_table_remove(&jit_medium_page_header_table,
JIT_MEDIUM_PAGE_SIZE,
page);
pas_heap_lock_unlock_conditionally(heap_lock_hold_mode);
@@ -269,7 +304,7 @@
result,
enumerator,
(pas_page_header_table*)pas_enumerator_read(
- enumerator, root_data->small_bitfit_page_header_table, sizeof(pas_page_header_table))))
+ enumerator, root_data->small_page_header_table, sizeof(pas_page_header_table))))
return NULL;
if (!pas_basic_heap_config_enumerator_data_add_page_header_table(
@@ -276,7 +311,7 @@
result,
enumerator,
(pas_page_header_table*)pas_enumerator_read(
- enumerator, root_data->medium_bitfit_page_header_table, sizeof(pas_page_header_table))))
+ enumerator, root_data->medium_page_header_table, sizeof(pas_page_header_table))))
return NULL;
return result;
@@ -331,6 +366,8 @@
&jit_fresh_memory_heap, range.begin, range.end, pas_zero_mode_is_all_zero, &config);
}
+PAS_SEGREGATED_PAGE_CONFIG_SPECIALIZATION_DEFINITIONS(
+ jit_small_segregated_page_config, JIT_HEAP_CONFIG.small_segregated_config);
PAS_BITFIT_PAGE_CONFIG_SPECIALIZATION_DEFINITIONS(
jit_small_bitfit_page_config, JIT_HEAP_CONFIG.small_bitfit_config);
PAS_BITFIT_PAGE_CONFIG_SPECIALIZATION_DEFINITIONS(
Modified: trunk/Source/bmalloc/libpas/src/libpas/jit_heap_config.h (289589 => 289590)
--- trunk/Source/bmalloc/libpas/src/libpas/jit_heap_config.h 2022-02-10 23:42:48 UTC (rev 289589)
+++ trunk/Source/bmalloc/libpas/src/libpas/jit_heap_config.h 2022-02-10 23:49:33 UTC (rev 289590)
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2021 Apple Inc. All rights reserved.
+ * Copyright (c) 2021-2022 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -36,12 +36,14 @@
PAS_BEGIN_EXTERN_C;
-#define JIT_SMALL_MIN_ALIGN_SHIFT 2u
-#define JIT_SMALL_MIN_ALIGN (1u << JIT_SMALL_MIN_ALIGN_SHIFT)
+#define JIT_SMALL_SEGREGATED_MIN_ALIGN_SHIFT 4u
+#define JIT_SMALL_SEGREGATED_MIN_ALIGN (1u << JIT_SMALL_SEGREGATED_MIN_ALIGN_SHIFT)
+#define JIT_SMALL_BITFIT_MIN_ALIGN_SHIFT 2u
+#define JIT_SMALL_BITFIT_MIN_ALIGN (1u << JIT_SMALL_BITFIT_MIN_ALIGN_SHIFT)
#define JIT_SMALL_PAGE_SIZE 16384u
#define JIT_SMALL_GRANULE_SIZE 16384u
-#define JIT_MEDIUM_MIN_ALIGN_SHIFT 8u
-#define JIT_MEDIUM_MIN_ALIGN (1u << JIT_MEDIUM_MIN_ALIGN_SHIFT)
+#define JIT_MEDIUM_BITFIT_MIN_ALIGN_SHIFT 8u
+#define JIT_MEDIUM_BITFIT_MIN_ALIGN (1u << JIT_MEDIUM_BITFIT_MIN_ALIGN_SHIFT)
#define JIT_MEDIUM_PAGE_SIZE 131072u
#if PAS_ARM64
#define JIT_MEDIUM_GRANULE_SIZE 16384u
@@ -66,23 +68,33 @@
PAS_API pas_page_base* jit_page_header_for_boundary_remote(pas_enumerator* enumerator, void* boundary);
-static PAS_ALWAYS_INLINE pas_page_base* jit_small_bitfit_page_header_for_boundary(void* boundary);
-static PAS_ALWAYS_INLINE void* jit_small_bitfit_boundary_for_page_header(pas_page_base* page);
+static PAS_ALWAYS_INLINE pas_page_base* jit_small_page_header_for_boundary(void* boundary);
+static PAS_ALWAYS_INLINE void* jit_small_boundary_for_page_header(pas_page_base* page);
+PAS_API void* jit_small_segregated_allocate_page(
+ pas_segregated_heap* heap, pas_physical_memory_transaction* transaction, pas_segregated_page_role role);
+PAS_API pas_page_base* jit_small_segregated_create_page_header(
+ void* boundary, pas_page_kind kind, pas_lock_hold_mode heap_lock_hold_mode);
+PAS_API void jit_small_destroy_page_header(
+ pas_page_base* page, pas_lock_hold_mode heap_lock_hold_mode);
+PAS_API pas_segregated_shared_page_directory* jit_small_segregated_shared_page_directory_selector(
+ pas_segregated_heap* heap, pas_segregated_size_directory* directory);
+
+PAS_SEGREGATED_PAGE_CONFIG_SPECIALIZATION_DECLARATIONS(jit_small_segregated_page_config);
+
PAS_API void* jit_small_bitfit_allocate_page(
pas_segregated_heap* heap, pas_physical_memory_transaction* transaction);
PAS_API pas_page_base* jit_small_bitfit_create_page_header(
void* boundary, pas_page_kind kind, pas_lock_hold_mode heap_lock_hold_mode);
-PAS_API void jit_small_bitfit_destroy_page_header(
- pas_page_base* page, pas_lock_hold_mode heap_lock_hold_mode);
PAS_BITFIT_PAGE_CONFIG_SPECIALIZATION_DECLARATIONS(jit_small_bitfit_page_config);
-static PAS_ALWAYS_INLINE pas_page_base* jit_medium_bitfit_page_header_for_boundary(void* boundary);
+static PAS_ALWAYS_INLINE pas_page_base* jit_medium_page_header_for_boundary(void* boundary);
+static PAS_ALWAYS_INLINE void* jit_medium_boundary_for_page_header(pas_page_base* page);
PAS_API void* jit_medium_bitfit_allocate_page(
pas_segregated_heap* heap, pas_physical_memory_transaction* transaction);
PAS_API pas_page_base* jit_medium_bitfit_create_page_header(
void* boundary, pas_page_kind kind, pas_lock_hold_mode heap_lock_hold_mode);
-PAS_API void jit_medium_bitfit_destroy_page_header(
+PAS_API void jit_medium_destroy_page_header(
pas_page_base* page, pas_lock_hold_mode heap_lock_hold_mode);
PAS_BITFIT_PAGE_CONFIG_SPECIALIZATION_DECLARATIONS(jit_medium_bitfit_page_config);
@@ -121,16 +133,16 @@
.heap_config_ptr = &jit_heap_config, \
.page_config_ptr = &jit_heap_config.variant_lowercase ## _bitfit_config.base, \
.page_config_kind = pas_page_config_kind_bitfit, \
- .min_align_shift = JIT_ ## variant_uppercase ## _MIN_ALIGN_SHIFT, \
+ .min_align_shift = JIT_ ## variant_uppercase ## _BITFIT_MIN_ALIGN_SHIFT, \
.page_size = JIT_ ## variant_uppercase ## _PAGE_SIZE, \
.granule_size = JIT_ ## variant_uppercase ## _GRANULE_SIZE, \
.max_object_size = \
- PAS_BITFIT_MAX_FREE_MAX_VALID << JIT_ ## variant_uppercase ## _MIN_ALIGN_SHIFT, \
- .page_header_for_boundary = jit_ ## variant_lowercase ## _bitfit_page_header_for_boundary, \
- .boundary_for_page_header = jit_ ## variant_lowercase ## _bitfit_boundary_for_page_header, \
+ PAS_BITFIT_MAX_FREE_MAX_VALID << JIT_ ## variant_uppercase ## _BITFIT_MIN_ALIGN_SHIFT, \
+ .page_header_for_boundary = jit_ ## variant_lowercase ## _page_header_for_boundary, \
+ .boundary_for_page_header = jit_ ## variant_lowercase ## _boundary_for_page_header, \
.page_header_for_boundary_remote = jit_page_header_for_boundary_remote, \
.create_page_header = jit_ ## variant_lowercase ## _bitfit_create_page_header, \
- .destroy_page_header = jit_ ## variant_lowercase ## _bitfit_destroy_page_header \
+ .destroy_page_header = jit_ ## variant_lowercase ## _destroy_page_header \
}, \
.variant = pas_ ## variant_lowercase ## _bitfit_page_config_variant, \
.kind = pas_bitfit_page_config_kind_jit_ ## variant_lowercase ## _bitfit, \
@@ -147,11 +159,43 @@
.get_type_size = jit_type_size, \
.get_type_alignment = jit_type_alignment, \
.dump_type = jit_type_dump, \
- .large_alignment = JIT_SMALL_MIN_ALIGN, \
+ .large_alignment = PAS_MIN_CONST(JIT_SMALL_SEGREGATED_MIN_ALIGN, JIT_SMALL_BITFIT_MIN_ALIGN), \
.small_segregated_config = { \
.base = { \
- .is_enabled = false \
- } \
+ .is_enabled = true, \
+ .heap_config_ptr = &jit_heap_config, \
+ .page_config_ptr = &jit_heap_config.small_segregated_config.base, \
+ .page_config_kind = pas_page_config_kind_segregated, \
+ .min_align_shift = JIT_SMALL_SEGREGATED_MIN_ALIGN_SHIFT, \
+ .page_size = JIT_SMALL_PAGE_SIZE, \
+ .granule_size = JIT_SMALL_GRANULE_SIZE, \
+ .max_object_size = PAS_MAX_OBJECT_SIZE(JIT_SMALL_PAGE_SIZE), \
+ .page_header_for_boundary = jit_small_page_header_for_boundary, \
+ .boundary_for_page_header = jit_small_boundary_for_page_header, \
+ .page_header_for_boundary_remote = jit_page_header_for_boundary_remote, \
+ .create_page_header = jit_small_segregated_create_page_header, \
+ .destroy_page_header = jit_small_destroy_page_header \
+ }, \
+ .variant = pas_small_segregated_page_config_variant, \
+ .kind = pas_segregated_page_config_kind_jit_small_segregated, \
+ .wasteage_handicap = 1., \
+ .sharing_shift = PAS_SMALL_SHARING_SHIFT, \
+ .num_alloc_bits = PAS_BASIC_SEGREGATED_NUM_ALLOC_BITS(JIT_SMALL_SEGREGATED_MIN_ALIGN_SHIFT, \
+ JIT_SMALL_PAGE_SIZE), \
+ .shared_payload_offset = 0, \
+ .exclusive_payload_offset = 0, \
+ .shared_payload_size = 0, \
+ .exclusive_payload_size = JIT_SMALL_PAGE_SIZE, \
+ .shared_logging_mode = pas_segregated_deallocation_no_logging_mode, \
+ .exclusive_logging_mode = pas_segregated_deallocation_size_oblivious_logging_mode, \
+ .use_reversed_current_word = PAS_ARM64, \
+ .check_deallocation = false, \
+ .enable_empty_word_eligibility_optimization_for_shared = false, \
+ .enable_empty_word_eligibility_optimization_for_exclusive = true, \
+ .enable_view_cache = true, \
+ .page_allocator = jit_small_segregated_allocate_page, \
+ .shared_page_directory_selector = jit_small_segregated_shared_page_directory_selector, \
+ PAS_SEGREGATED_PAGE_CONFIG_SPECIALIZATIONS(jit_small_segregated_page_config) \
}, \
.medium_segregated_config = { \
.base = { \
@@ -191,33 +235,33 @@
PAS_API extern pas_simple_large_free_heap jit_fresh_memory_heap;
PAS_API extern pas_large_heap_physical_page_sharing_cache jit_large_fresh_memory_heap;
-PAS_API extern pas_page_header_table jit_small_bitfit_page_header_table;
-PAS_API extern pas_page_header_table jit_medium_bitfit_page_header_table;
+PAS_API extern pas_page_header_table jit_small_page_header_table;
+PAS_API extern pas_page_header_table jit_medium_page_header_table;
PAS_API extern pas_heap_runtime_config jit_heap_runtime_config;
PAS_API extern jit_heap_config_root_data jit_root_data;
-static PAS_ALWAYS_INLINE pas_page_base* jit_small_bitfit_page_header_for_boundary(void* boundary)
+static PAS_ALWAYS_INLINE pas_page_base* jit_small_page_header_for_boundary(void* boundary)
{
return pas_page_header_table_get_for_boundary(
- &jit_small_bitfit_page_header_table, JIT_SMALL_PAGE_SIZE, boundary);
+ &jit_small_page_header_table, JIT_SMALL_PAGE_SIZE, boundary);
}
-static PAS_ALWAYS_INLINE void* jit_small_bitfit_boundary_for_page_header(pas_page_base* page)
+static PAS_ALWAYS_INLINE void* jit_small_boundary_for_page_header(pas_page_base* page)
{
return pas_page_header_table_get_boundary(
- &jit_small_bitfit_page_header_table, JIT_SMALL_PAGE_SIZE, page);
+ &jit_small_page_header_table, JIT_SMALL_PAGE_SIZE, page);
}
-static PAS_ALWAYS_INLINE pas_page_base* jit_medium_bitfit_page_header_for_boundary(void* boundary)
+static PAS_ALWAYS_INLINE pas_page_base* jit_medium_page_header_for_boundary(void* boundary)
{
return pas_page_header_table_get_for_boundary(
- &jit_medium_bitfit_page_header_table, JIT_MEDIUM_PAGE_SIZE, boundary);
+ &jit_medium_page_header_table, JIT_MEDIUM_PAGE_SIZE, boundary);
}
-static PAS_ALWAYS_INLINE void* jit_medium_bitfit_boundary_for_page_header(pas_page_base* page)
+static PAS_ALWAYS_INLINE void* jit_medium_boundary_for_page_header(pas_page_base* page)
{
return pas_page_header_table_get_boundary(
- &jit_medium_bitfit_page_header_table, JIT_MEDIUM_PAGE_SIZE, page);
+ &jit_medium_page_header_table, JIT_MEDIUM_PAGE_SIZE, page);
}
static PAS_ALWAYS_INLINE pas_page_base* jit_heap_config_page_header(uintptr_t begin)
@@ -225,12 +269,12 @@
pas_page_base* result;
result = pas_page_header_table_get_for_address(
- &jit_small_bitfit_page_header_table, JIT_SMALL_PAGE_SIZE, (void*)begin);
+ &jit_small_page_header_table, JIT_SMALL_PAGE_SIZE, (void*)begin);
if (result)
return result;
return pas_page_header_table_get_for_address(
- &jit_medium_bitfit_page_header_table, JIT_MEDIUM_PAGE_SIZE, (void*)begin);
+ &jit_medium_page_header_table, JIT_MEDIUM_PAGE_SIZE, (void*)begin);
}
PAS_API void jit_heap_config_add_fresh_memory(pas_range range);
Modified: trunk/Source/bmalloc/libpas/src/libpas/jit_heap_config_root_data.h (289589 => 289590)
--- trunk/Source/bmalloc/libpas/src/libpas/jit_heap_config_root_data.h 2022-02-10 23:42:48 UTC (rev 289589)
+++ trunk/Source/bmalloc/libpas/src/libpas/jit_heap_config_root_data.h 2022-02-10 23:49:33 UTC (rev 289590)
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2021 Apple Inc. All rights reserved.
+ * Copyright (c) 2021-2022 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -36,8 +36,8 @@
typedef struct pas_page_header_table pas_page_header_table;
struct jit_heap_config_root_data {
- pas_page_header_table* small_bitfit_page_header_table;
- pas_page_header_table* medium_bitfit_page_header_table;
+ pas_page_header_table* small_page_header_table;
+ pas_page_header_table* medium_page_header_table;
};
PAS_END_EXTERN_C;
Modified: trunk/Source/bmalloc/libpas/src/libpas/pas_bitfit_heap.c (289589 => 289590)
--- trunk/Source/bmalloc/libpas/src/libpas/pas_bitfit_heap.c 2022-02-10 23:42:48 UTC (rev 289589)
+++ trunk/Source/bmalloc/libpas/src/libpas/pas_bitfit_heap.c 2022-02-10 23:49:33 UTC (rev 289590)
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020-2021 Apple Inc. All rights reserved.
+ * Copyright (c) 2020-2022 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -60,7 +60,8 @@
}
pas_bitfit_variant_selection pas_bitfit_heap_select_variant(size_t requested_object_size,
- pas_heap_config* config)
+ pas_heap_config* config,
+ pas_heap_runtime_config* runtime_config)
{
static const bool verbose = false;
@@ -79,7 +80,7 @@
size_t object_size;
page_config = *pas_heap_config_bitfit_page_config_ptr_for_variant(config, variant);
- if (!pas_bitfit_page_config_is_enabled(page_config))
+ if (!pas_bitfit_page_config_is_enabled(page_config, runtime_config))
continue;
if (verbose)
@@ -125,7 +126,8 @@
void pas_bitfit_heap_construct_and_insert_size_class(pas_bitfit_heap* heap,
pas_bitfit_size_class* size_class,
unsigned object_size,
- pas_heap_config* config)
+ pas_heap_config* config,
+ pas_heap_runtime_config* runtime_config)
{
static const bool verbose = false;
@@ -139,7 +141,7 @@
heap, size_class, object_size);
}
- best = pas_bitfit_heap_select_variant(object_size, config);
+ best = pas_bitfit_heap_select_variant(object_size, config, runtime_config);
pas_heap_lock_assert_held();
if (verbose) {
Modified: trunk/Source/bmalloc/libpas/src/libpas/pas_bitfit_heap.h (289589 => 289590)
--- trunk/Source/bmalloc/libpas/src/libpas/pas_bitfit_heap.h 2022-02-10 23:42:48 UTC (rev 289589)
+++ trunk/Source/bmalloc/libpas/src/libpas/pas_bitfit_heap.h 2022-02-10 23:49:33 UTC (rev 289590)
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020-2021 Apple Inc. All rights reserved.
+ * Copyright (c) 2020-2022 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -33,10 +33,12 @@
struct pas_bitfit_heap;
struct pas_heap_config;
+struct pas_heap_runtime_config;
struct pas_segregated_size_directory;
struct pas_segregated_heap;
typedef struct pas_bitfit_heap pas_bitfit_heap;
typedef struct pas_heap_config pas_heap_config;
+typedef struct pas_heap_runtime_config pas_heap_runtime_config;
typedef struct pas_segregated_size_directory pas_segregated_size_directory;
typedef struct pas_segregated_heap pas_segregated_heap;
@@ -61,12 +63,15 @@
} pas_bitfit_variant_selection;
PAS_API pas_bitfit_variant_selection
-pas_bitfit_heap_select_variant(size_t object_size, pas_heap_config* config);
+pas_bitfit_heap_select_variant(size_t object_size,
+ pas_heap_config* config,
+ pas_heap_runtime_config* runtime_config);
PAS_API void pas_bitfit_heap_construct_and_insert_size_class(pas_bitfit_heap* heap,
pas_bitfit_size_class* size_class,
unsigned object_size,
- pas_heap_config* config);
+ pas_heap_config* config,
+ pas_heap_runtime_config* runtime_config);
PAS_API pas_heap_summary pas_bitfit_heap_compute_summary(pas_bitfit_heap* heap);
Modified: trunk/Source/bmalloc/libpas/src/libpas/pas_bitfit_page_config.h (289589 => 289590)
--- trunk/Source/bmalloc/libpas/src/libpas/pas_bitfit_page_config.h 2022-02-10 23:42:48 UTC (rev 289589)
+++ trunk/Source/bmalloc/libpas/src/libpas/pas_bitfit_page_config.h 2022-02-10 23:49:33 UTC (rev 289590)
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020-2021 Apple Inc. All rights reserved.
+ * Copyright (c) 2020-2022 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -31,6 +31,7 @@
#include "pas_bitfit_page_config_variant.h"
#include "pas_bitvector.h"
#include "pas_fast_path_allocation_result.h"
+#include "pas_heap_runtime_config.h"
#include "pas_page_base_config.h"
#include "pas_page_malloc.h"
#include "pas_utils.h"
@@ -135,10 +136,18 @@
PAS_API void lower_case_page_config_name ## _specialized_page_shrink_with_page( \
pas_bitfit_page* page, uintptr_t begin, size_t new_size)
-static inline bool pas_bitfit_page_config_is_enabled(pas_bitfit_page_config config)
+static inline bool pas_bitfit_page_config_is_enabled(pas_bitfit_page_config config,
+ pas_heap_runtime_config* runtime_config)
{
if (!config.base.is_enabled)
return false;
+ /* Doing this check here is not super necessary, but it's sort of nice for cases where we have a heap
+ that sometimes uses bitfit exclusively or sometimes uses segregated exclusively and that's selected
+ by selecting or mutating runtime_configs. This is_enabled function is only called as part of the math
+ that sets up size classes, at least for now, so the implications of not doing this check are rather
+ tiny. */
+ if (!runtime_config->max_bitfit_object_size)
+ return false;
switch (config.variant) {
case pas_small_bitfit_page_config_variant:
return pas_small_bitfit_page_config_variant_is_enabled_override;
Modified: trunk/Source/bmalloc/libpas/src/libpas/pas_heap_config.h (289589 => 289590)
--- trunk/Source/bmalloc/libpas/src/libpas/pas_heap_config.h 2022-02-10 23:42:48 UTC (rev 289589)
+++ trunk/Source/bmalloc/libpas/src/libpas/pas_heap_config.h 2022-02-10 23:49:33 UTC (rev 289590)
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018-2021 Apple Inc. All rights reserved.
+ * Copyright (c) 2018-2022 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -308,6 +308,23 @@
return config.small_bitfit_config;
}
+static PAS_ALWAYS_INLINE size_t pas_heap_config_segregated_heap_min_align_shift(pas_heap_config config)
+{
+ size_t result;
+ result = SIZE_MAX;
+ if (config.small_bitfit_config.base.is_enabled)
+ result = PAS_MIN(result, config.small_bitfit_config.base.min_align_shift);
+ if (config.small_segregated_config.base.is_enabled)
+ result = PAS_MIN(result, config.small_segregated_config.base.min_align_shift);
+ PAS_ASSERT(result != SIZE_MAX);
+ return result;
+}
+
+static PAS_ALWAYS_INLINE size_t pas_heap_config_segregated_heap_min_align(pas_heap_config config)
+{
+ return (size_t)1 << pas_heap_config_segregated_heap_min_align_shift(config);
+}
+
/* Returns true if we were the first to active it. Must hold the heap lock to call this. This is
permissive of recursive initialization: in that case, it will just pretend that the config is
already initialized. */
Modified: trunk/Source/bmalloc/libpas/src/libpas/pas_segregated_heap.c (289589 => 289590)
--- trunk/Source/bmalloc/libpas/src/libpas/pas_segregated_heap.c 2022-02-10 23:42:48 UTC (rev 289589)
+++ trunk/Source/bmalloc/libpas/src/libpas/pas_segregated_heap.c 2022-02-10 23:49:33 UTC (rev 289590)
@@ -52,7 +52,8 @@
unsigned *cached_index,
const char* where);
-static size_t min_object_size_for_heap_config(pas_heap_config* config)
+static size_t min_align_for_heap(pas_segregated_heap* heap,
+ pas_heap_config* config)
{
pas_segregated_page_config_variant segregated_variant;
pas_bitfit_page_config_variant bitfit_variant;
@@ -60,7 +61,7 @@
pas_segregated_page_config* page_config;
page_config =
pas_heap_config_segregated_page_config_ptr_for_variant(config, segregated_variant);
- if (!pas_segregated_page_config_is_enabled(*page_config))
+ if (!pas_segregated_page_config_is_enabled(*page_config, heap->runtime_config))
continue;
return pas_segregated_page_config_min_align(*page_config);
}
@@ -70,7 +71,7 @@
pas_bitfit_page_config* page_config;
page_config =
pas_heap_config_bitfit_page_config_ptr_for_variant(config, bitfit_variant);
- if (!pas_bitfit_page_config_is_enabled(*page_config))
+ if (!pas_bitfit_page_config_is_enabled(*page_config, heap->runtime_config))
continue;
return pas_page_base_config_min_align(page_config->base);
}
@@ -79,6 +80,12 @@
return config->large_alignment;
}
+static size_t min_object_size_for_heap(pas_segregated_heap* heap,
+ pas_heap_config* config)
+{
+ return min_align_for_heap(heap, config);
+}
+
static size_t max_object_size_for_page_config(pas_heap* parent_heap,
pas_page_base_config* page_config)
{
@@ -98,9 +105,9 @@
return result;
}
-static size_t max_segregated_object_size_for_heap_config(pas_heap* parent_heap,
- pas_segregated_heap* heap,
- pas_heap_config* config)
+static size_t max_segregated_object_size_for_heap(pas_heap* parent_heap,
+ pas_segregated_heap* heap,
+ pas_heap_config* config)
{
static const bool verbose = false;
@@ -111,7 +118,7 @@
size_t result;
page_config = pas_heap_config_segregated_page_config_ptr_for_variant(config, variant);
- if (!pas_segregated_page_config_is_enabled(*page_config))
+ if (!pas_segregated_page_config_is_enabled(*page_config, heap->runtime_config))
continue;
max_size_per_config = max_object_size_for_page_config(parent_heap, &page_config->base);
@@ -125,9 +132,9 @@
return 0;
}
-static size_t max_bitfit_object_size_for_heap_config(pas_heap* parent_heap,
- pas_segregated_heap* heap,
- pas_heap_config* config)
+static size_t max_bitfit_object_size_for_heap(pas_heap* parent_heap,
+ pas_segregated_heap* heap,
+ pas_heap_config* config)
{
static const bool verbose = false;
@@ -145,7 +152,7 @@
pas_bitfit_page_config_kind_get_string(page_config->kind));
}
- if (!pas_bitfit_page_config_is_enabled(*page_config)) {
+ if (!pas_bitfit_page_config_is_enabled(*page_config, heap->runtime_config)) {
if (verbose) {
pas_log("Not considering %s because it's disabled.\n",
pas_bitfit_page_config_kind_get_string(page_config->kind));
@@ -182,12 +189,12 @@
return 0;
}
-static size_t max_object_size_for_heap_config(pas_heap* parent_heap,
- pas_segregated_heap* heap,
- pas_heap_config* config)
+static size_t max_object_size_for_heap(pas_heap* parent_heap,
+ pas_segregated_heap* heap,
+ pas_heap_config* config)
{
- return PAS_MAX(max_segregated_object_size_for_heap_config(parent_heap, heap, config),
- max_bitfit_object_size_for_heap_config(parent_heap, heap, config));
+ return PAS_MAX(max_segregated_object_size_for_heap(parent_heap, heap, config),
+ max_bitfit_object_size_for_heap(parent_heap, heap, config));
}
void pas_segregated_heap_construct(pas_segregated_heap* segregated_heap,
@@ -847,7 +854,7 @@
pas_heap_lock_assert_held();
- PAS_ASSERT(directory->object_size >= min_object_size_for_heap_config(config));
+ PAS_ASSERT(directory->object_size >= min_object_size_for_heap(heap, config));
rematerialize_size_lookup_if_necessary(heap, config, cached_index);
@@ -1328,6 +1335,8 @@
size_t object_size;
pas_segregated_heap_medium_directory_tuple* medium_tuple;
bool is_utility;
+ size_t dynamic_min_align;
+ size_t static_min_align;
size_t type_alignment;
pas_heap_lock_assert_held();
@@ -1423,8 +1432,15 @@
PAS_ASSERT(alignment >= type_alignment);
index = pas_segregated_heap_index_for_size(size, *config);
+
+ if (verbose)
+ pas_log("index = %zu\n", index);
+
object_size = pas_segregated_heap_size_for_index(index, *config);
+ if (verbose)
+ pas_log("object_size = %zu\n", object_size);
+
if (object_size < size) {
/* This'll only happen in certain kinds of overflows, in which case the size must be way too big. */
PAS_ASSERT(!object_size);
@@ -1431,14 +1447,22 @@
return NULL;
}
- object_size = PAS_MAX(min_object_size_for_heap_config(config), object_size);
+ object_size = PAS_MAX(min_object_size_for_heap(heap, config), object_size);
- PAS_ASSERT(pas_is_aligned(
- object_size, pas_segregated_page_config_min_align(config->small_segregated_config)));
+ if (verbose)
+ pas_log("object_size after accounting for min_object_size = %zu\n", object_size);
- alignment = PAS_MAX(alignment,
- pas_segregated_page_config_min_align(config->small_segregated_config));
+ static_min_align = pas_heap_config_segregated_heap_min_align(*config);
+ dynamic_min_align = min_align_for_heap(heap, config);
+ PAS_ASSERT(dynamic_min_align >= static_min_align);
+ PAS_ASSERT(pas_is_aligned(object_size, alignment));
+ PAS_ASSERT(pas_is_aligned(object_size, static_min_align));
+
+ object_size = pas_round_up_to_power_of_2(object_size, dynamic_min_align);
+
+ alignment = PAS_MAX(alignment, dynamic_min_align);
+
if (alignment > type_alignment) {
/* If the alignment is bigger than type_alignment then it means that this is a dynamically requested
alignment. So, we expect that the object size was already aligned to it by the allocator fast
@@ -1466,7 +1490,7 @@
/* If it's impossible to use the largest segregated heap to allocate the request then
immediately give up. Do this before ensure_size_lookup so that heaps that are only used
for large object allocation don't allocate any small heap meta-data. */
- if (object_size > max_object_size_for_heap_config(parent_heap, heap, config)) {
+ if (object_size > max_object_size_for_heap(parent_heap, heap, config)) {
if (verbose)
pas_log("It's too big.\n");
return NULL;
@@ -1704,7 +1728,7 @@
page_config_ptr =
pas_heap_config_segregated_page_config_ptr_for_variant(config, variant);
- if (!pas_segregated_page_config_is_enabled(*page_config_ptr))
+ if (!pas_segregated_page_config_is_enabled(*page_config_ptr, heap->runtime_config))
continue;
page_config = *page_config_ptr;
@@ -1748,9 +1772,9 @@
}
PAS_ASSERT(object_size <= heap->runtime_config->max_bitfit_object_size);
PAS_TESTING_ASSERT(
- object_size <= max_bitfit_object_size_for_heap_config(parent_heap, heap, config));
+ object_size <= max_bitfit_object_size_for_heap(parent_heap, heap, config));
best_bytes_dirtied_per_object =
- pas_bitfit_heap_select_variant(object_size, config).object_size;
+ pas_bitfit_heap_select_variant(object_size, config, heap->runtime_config).object_size;
PAS_ASSERT(!is_utility);
}
Modified: trunk/Source/bmalloc/libpas/src/libpas/pas_segregated_heap.h (289589 => 289590)
--- trunk/Source/bmalloc/libpas/src/libpas/pas_segregated_heap.h 2022-02-10 23:42:48 UTC (rev 289589)
+++ trunk/Source/bmalloc/libpas/src/libpas/pas_segregated_heap.h 2022-02-10 23:49:33 UTC (rev 289590)
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018-2021 Apple Inc. All rights reserved.
+ * Copyright (c) 2018-2022 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -155,14 +155,17 @@
static PAS_ALWAYS_INLINE size_t
pas_segregated_heap_index_for_size(size_t size, pas_heap_config config)
{
- return (size + pas_segregated_page_config_min_align(config.small_segregated_config) - 1)
- >> config.small_segregated_config.base.min_align_shift;
+ size_t min_align_shift;
+ size_t min_align;
+ min_align_shift = pas_heap_config_segregated_heap_min_align_shift(config);
+ min_align = (size_t)1 << min_align_shift;
+ return (size + min_align - 1) >> min_align_shift;
}
static PAS_ALWAYS_INLINE size_t
pas_segregated_heap_size_for_index(size_t index, pas_heap_config config)
{
- return index << config.small_segregated_config.base.min_align_shift;
+ return index << pas_heap_config_segregated_heap_min_align_shift(config);
}
/* These functions help with dealing with pas_heap_ref::allocator_index. Can pass a NULL cached_index, in which
Modified: trunk/Source/bmalloc/libpas/src/libpas/pas_segregated_page.c (289589 => 289590)
--- trunk/Source/bmalloc/libpas/src/libpas/pas_segregated_page.c 2022-02-10 23:42:48 UTC (rev 289589)
+++ trunk/Source/bmalloc/libpas/src/libpas/pas_segregated_page.c 2022-02-10 23:49:33 UTC (rev 289590)
@@ -243,7 +243,6 @@
if (pas_segregated_size_directory_view_cache_capacity(directory)) {
PAS_ASSERT(directory->view_cache_index);
- PAS_ASSERT(directory->view_cache_index < (pas_allocator_index)UINT_MAX);
page->view_cache_index = directory->view_cache_index;
} else
PAS_ASSERT(directory->view_cache_index == (pas_allocator_index)UINT_MAX);
Modified: trunk/Source/bmalloc/libpas/src/libpas/pas_segregated_page_config.h (289589 => 289590)
--- trunk/Source/bmalloc/libpas/src/libpas/pas_segregated_page_config.h 2022-02-10 23:42:48 UTC (rev 289589)
+++ trunk/Source/bmalloc/libpas/src/libpas/pas_segregated_page_config.h 2022-02-10 23:49:33 UTC (rev 289590)
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018-2021 Apple Inc. All rights reserved.
+ * Copyright (c) 2018-2022 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -29,13 +29,14 @@
#include "pas_allocation_result.h"
#include "pas_bitvector.h"
#include "pas_config.h"
+#include "pas_heap_runtime_config.h"
#include "pas_local_allocator_refill_mode.h"
#include "pas_lock.h"
-#include "pas_segregated_page_config_kind.h"
#include "pas_page_base_config.h"
#include "pas_page_granule_use_count.h"
#include "pas_page_sharing_mode.h"
#include "pas_segregated_deallocation_logging_mode.h"
+#include "pas_segregated_page_config_kind.h"
#include "pas_segregated_page_config_variant.h"
#include "pas_segregated_page_role.h"
#include "pas_segregated_view.h"
@@ -222,10 +223,18 @@
#define PAS_SEGREGATED_PAGE_CONFIG_GOOD_MAX_OBJECT_SIZE(object_payload_size, min_num_objects) \
((object_payload_size) / (min_num_objects))
-static inline bool pas_segregated_page_config_is_enabled(pas_segregated_page_config config)
+static inline bool pas_segregated_page_config_is_enabled(pas_segregated_page_config config,
+ pas_heap_runtime_config* runtime_config)
{
if (!config.base.is_enabled)
return false;
+ /* Doing this check here is not super necessary, but it's sort of nice for cases where we have a heap
+ that sometimes uses bitfit exclusively or sometimes uses segregated exclusively and that's selected
+ by selecting or mutating runtime_configs. This is_enabled function is only called as part of the math
+ that sets up size classes, at least for now, so the implications of not doing this check are rather
+ tiny. */
+ if (!runtime_config->max_segregated_object_size)
+ return false;
switch (config.variant) {
case pas_small_segregated_page_config_variant:
return pas_small_segregated_page_config_variant_is_enabled_override;
Modified: trunk/Source/bmalloc/libpas/src/libpas/pas_segregated_page_config_kind.def (289589 => 289590)
--- trunk/Source/bmalloc/libpas/src/libpas/pas_segregated_page_config_kind.def 2022-02-10 23:42:48 UTC (rev 289589)
+++ trunk/Source/bmalloc/libpas/src/libpas/pas_segregated_page_config_kind.def 2022-02-10 23:49:33 UTC (rev 289590)
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019-2021 Apple Inc. All rights reserved.
+ * Copyright (c) 2019-2022 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -104,3 +104,7 @@
HOTBIT_HEAP_CONFIG.medium_segregated_config)
#endif
+#if PAS_ENABLE_JIT
+PAS_DEFINE_SEGREGATED_PAGE_CONFIG_KIND(jit_small_segregated,
+ JIT_HEAP_CONFIG.small_segregated_config)
+#endif
Modified: trunk/Source/bmalloc/libpas/src/libpas/pas_segregated_page_config_kind_and_role.h (289589 => 289590)
--- trunk/Source/bmalloc/libpas/src/libpas/pas_segregated_page_config_kind_and_role.h 2022-02-10 23:42:48 UTC (rev 289589)
+++ trunk/Source/bmalloc/libpas/src/libpas/pas_segregated_page_config_kind_and_role.h 2022-02-10 23:49:33 UTC (rev 289590)
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2021 Apple Inc. All rights reserved.
+ * Copyright (c) 2021-2022 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -41,7 +41,7 @@
typedef enum pas_segregated_page_config_kind_and_role pas_segregated_page_config_kind_and_role;
-#define PAS_SEGREGATED_PAGE_CONFIG_KIND_AND_ROLE_NUM_BITS 5u
+#define PAS_SEGREGATED_PAGE_CONFIG_KIND_AND_ROLE_NUM_BITS 6u
#define PAS_SEGREGATED_PAGE_CONFIG_KIND_AND_ROLE_MASK \
((1u << PAS_SEGREGATED_PAGE_CONFIG_KIND_AND_ROLE_NUM_BITS) - 1u)
Modified: trunk/Source/bmalloc/libpas/src/libpas/pas_segregated_size_directory.c (289589 => 289590)
--- trunk/Source/bmalloc/libpas/src/libpas/pas_segregated_size_directory.c 2022-02-10 23:42:48 UTC (rev 289589)
+++ trunk/Source/bmalloc/libpas/src/libpas/pas_segregated_size_directory.c 2022-02-10 23:49:33 UTC (rev 289590)
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019-2021 Apple Inc. All rights reserved.
+ * Copyright (c) 2019-2022 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -130,7 +130,7 @@
bitfit_size_class = pas_segregated_size_directory_get_bitfit_size_class(result);
pas_bitfit_heap_construct_and_insert_size_class(
- bitfit_heap, bitfit_size_class, object_size, heap_config);
+ bitfit_heap, bitfit_size_class, object_size, heap_config, heap->runtime_config);
}
pas_compact_atomic_segregated_size_directory_ptr_store(&result->next_for_heap, NULL);
Modified: trunk/Source/bmalloc/libpas/src/test/IsoHeapChaosTests.cpp (289589 => 289590)
--- trunk/Source/bmalloc/libpas/src/test/IsoHeapChaosTests.cpp 2022-02-10 23:42:48 UTC (rev 289589)
+++ trunk/Source/bmalloc/libpas/src/test/IsoHeapChaosTests.cpp 2022-02-10 23:49:33 UTC (rev 289590)
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019-2021 Apple Inc. All rights reserved.
+ * Copyright (c) 2019-2022 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -1207,6 +1207,11 @@
BootJITHeap bootJITHeap;
ADD_GROUP(addTheTests(1, true));
+
+ {
+ ForceBitfit forceBitfit;
+ ADD_GROUP(addTheTests(1, true));
+ }
}
{
@@ -1227,6 +1232,11 @@
BootJITHeap bootJITHeap;
ADD_GROUP(addTheTests(1, false));
+
+ {
+ ForceBitfit forceBitfit;
+ ADD_GROUP(addTheTests(1, false));
+ }
}
#endif // PAS_ENABLE_HOTBIT
}
Modified: trunk/Source/bmalloc/libpas/src/test/JITHeapTests.cpp (289589 => 289590)
--- trunk/Source/bmalloc/libpas/src/test/JITHeapTests.cpp 2022-02-10 23:42:48 UTC (rev 289589)
+++ trunk/Source/bmalloc/libpas/src/test/JITHeapTests.cpp 2022-02-10 23:49:33 UTC (rev 289590)
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2021 Apple Inc. All rights reserved.
+ * Copyright (c) 2021-2022 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -69,6 +69,11 @@
jit_heap_deallocate(newPtr);
}
+void testAllocationSize(size_t requestedSize, size_t actualSize)
+{
+ CHECK_EQUAL(jit_heap_get_size(jit_heap_try_allocate(requestedSize)), actualSize);
+}
+
} // anonymous namespace
#endif // PAS_ENABLE_JIT
@@ -78,13 +83,30 @@
#if PAS_ENABLE_JIT
BootJITHeap bootJITHeap;
- ADD_TEST(testAllocateShrinkAndAllocate(0, 0, 0, 0, 0, 4));
- ADD_TEST(testAllocateShrinkAndAllocate(0, 0, 128, 64, 64, 4));
- ADD_TEST(testAllocateShrinkAndAllocate(32, 10, 128, 64, 64, 4));
- ADD_TEST(testAllocateShrinkAndAllocate(32, 10, 1000, 500, 1000, 4));
- ADD_TEST(testAllocateShrinkAndAllocate(0, 0, 2048, 512, 1100, 256));
- ADD_TEST(testAllocateShrinkAndAllocate(32, 10, 2048, 512, 1100, 256));
- ADD_TEST(testAllocateShrinkAndAllocate(1100, 10, 2048, 512, 1100, 256));
- ADD_TEST(testAllocateShrinkAndAllocate(0, 0, 100000, 10000, 80000, 4));
+ {
+ ForceBitfit forceBitfit;
+ ADD_TEST(testAllocateShrinkAndAllocate(0, 0, 0, 0, 0, 4));
+ ADD_TEST(testAllocateShrinkAndAllocate(0, 0, 128, 64, 64, 4));
+ ADD_TEST(testAllocateShrinkAndAllocate(32, 10, 128, 64, 64, 4));
+ ADD_TEST(testAllocateShrinkAndAllocate(32, 10, 1000, 500, 1000, 4));
+ ADD_TEST(testAllocateShrinkAndAllocate(0, 0, 2048, 512, 1100, 256));
+ ADD_TEST(testAllocateShrinkAndAllocate(32, 10, 2048, 512, 1100, 256));
+ ADD_TEST(testAllocateShrinkAndAllocate(1100, 10, 2048, 512, 1100, 256));
+ ADD_TEST(testAllocateShrinkAndAllocate(0, 0, 100000, 10000, 80000, 4));
+ }
+
+ ADD_TEST(testAllocationSize(4, 16));
+ ADD_TEST(testAllocationSize(8, 16));
+ ADD_TEST(testAllocationSize(12, 16));
+ ADD_TEST(testAllocationSize(16, 16));
+ ADD_TEST(testAllocationSize(20, 32));
+ {
+ ForceBitfit forceBitfit;
+ ADD_TEST(testAllocationSize(4, 4));
+ ADD_TEST(testAllocationSize(8, 8));
+ ADD_TEST(testAllocationSize(12, 12));
+ ADD_TEST(testAllocationSize(16, 16));
+ ADD_TEST(testAllocationSize(20, 20));
+ }
#endif // PAS_ENABLE_JIT
}
Modified: trunk/Source/bmalloc/libpas/src/test/TestHarness.cpp (289589 => 289590)
--- trunk/Source/bmalloc/libpas/src/test/TestHarness.cpp 2022-02-10 23:42:48 UTC (rev 289589)
+++ trunk/Source/bmalloc/libpas/src/test/TestHarness.cpp 2022-02-10 23:49:33 UTC (rev 289590)
@@ -378,6 +378,7 @@
void addTSDTests();
void addThingyAndUtilityHeapAllocationTests();
void addUtilsTests();
+void addViewCacheTests();
void testSucceeded()
{
@@ -744,6 +745,7 @@
ADD_SUITE(TLCDecommit);
ADD_SUITE(TSD);
ADD_SUITE(Utils);
+ ADD_SUITE(ViewCache);
string filter;
Added: trunk/Source/bmalloc/libpas/src/test/ViewCacheTests.cpp (0 => 289590)
--- trunk/Source/bmalloc/libpas/src/test/ViewCacheTests.cpp (rev 0)
+++ trunk/Source/bmalloc/libpas/src/test/ViewCacheTests.cpp 2022-02-10 23:49:33 UTC (rev 289590)
@@ -0,0 +1,110 @@
+/*
+ * Copyright (c) 2022 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "TestHarness.h"
+
+#if PAS_ENABLE_BMALLOC
+
+#include "bmalloc_heap.h"
+#include "bmalloc_heap_config.h"
+#include "pas_segregated_size_directory.h"
+#include <set>
+
+using namespace std;
+
+namespace {
+
+void setupConfig()
+{
+ // If these assertions ever fail we could just fix it by replacing them with code that mutates the
+ // config to have the settings we want.
+ CHECK_EQUAL(bmalloc_intrinsic_runtime_config.base.view_cache_capacity_for_object_size,
+ pas_heap_runtime_config_aggressive_view_cache_capacity);
+ CHECK_EQUAL(bmalloc_intrinsic_runtime_config.base.directory_size_bound_for_no_view_cache, 0);
+}
+
+void testDisableViewCacheUsingBoundForNoViewCache()
+{
+ setupConfig();
+
+ bmalloc_intrinsic_runtime_config.base.directory_size_bound_for_no_view_cache = UINT_MAX;
+
+ bmalloc_deallocate(bmalloc_allocate(42));
+}
+
+void testEnableViewCacheAtSomeBoundForNoViewCache(unsigned bound)
+{
+ setupConfig();
+
+ bmalloc_intrinsic_runtime_config.base.directory_size_bound_for_no_view_cache = bound;
+
+ void* ptr = bmalloc_allocate(42);
+ pas_segregated_view view = pas_segregated_view_for_object(
+ reinterpret_cast<uintptr_t>(ptr), &bmalloc_heap_config);
+ pas_segregated_size_directory* theDirectory = pas_segregated_view_get_size_directory(view);
+
+ CHECK_EQUAL(theDirectory->view_cache_index, UINT_MAX);
+
+ set<pas_segregated_view> views;
+ views.insert(view);
+
+ for (;;) {
+ ptr = bmalloc_allocate(42);
+ view = pas_segregated_view_for_object(reinterpret_cast<uintptr_t>(ptr), &bmalloc_heap_config);
+ pas_segregated_size_directory* directory = pas_segregated_view_get_size_directory(view);
+
+ CHECK_EQUAL(directory, theDirectory);
+
+ views.insert(view);
+
+ if (views.size() > bound) {
+ CHECK_EQUAL(views.size(), bound + 1);
+ CHECK_LESS(theDirectory->view_cache_index, UINT_MAX);
+ CHECK_GREATER(theDirectory->view_cache_index, 0);
+
+ pas_segregated_page* page = pas_segregated_view_get_page(view);
+ CHECK_EQUAL(page->view_cache_index, theDirectory->view_cache_index);
+ return;
+ }
+ }
+}
+
+} // anonymous namespace
+
+#endif // PAS_ENABLE_BMALLOC
+
+void addViewCacheTests()
+{
+#if PAS_ENABLE_BMALLOC
+ ForceExclusives forceExclusives;
+ ForceTLAs forceTLAs;
+
+ ADD_TEST(testDisableViewCacheUsingBoundForNoViewCache());
+ ADD_TEST(testEnableViewCacheAtSomeBoundForNoViewCache(1));
+ ADD_TEST(testEnableViewCacheAtSomeBoundForNoViewCache(10));
+ ADD_TEST(testEnableViewCacheAtSomeBoundForNoViewCache(100));
+#endif // PAS_ENABLE_BMALLOC
+}
+
