Diff
Modified: trunk/Source/bmalloc/ChangeLog (287967 => 287968)
--- trunk/Source/bmalloc/ChangeLog 2022-01-13 02:17:09 UTC (rev 287967)
+++ trunk/Source/bmalloc/ChangeLog 2022-01-13 02:52:00 UTC (rev 287968)
@@ -1,3 +1,66 @@
+2022-01-12 Filip Pizlo <fpi...@apple.com>
+
+ [libpas] thread_local_cache should not be allocated in the compact heap (cherry pick 11afcedfb5968f6894379ff1a41dd449ba7745f6)
+ https://bugs.webkit.org/show_bug.cgi?id=235096
+
+ Reviewed by Yusuke Suzuki.
+
+ Thread local caches can get quite large because of how libpas uses them, we can allocate one per
+ thread, and we reallocate them with exponential resizing, so there's a lot of wasted space and a
+ decent amount of fragmentation. This shows up as occasional crashes trying to allocate a thread local
+ cache out of the compact heap.
+
+ This moves thread local caches out of the compact heap. They were only ever there because partial views
+ sometimes need to point to the local_allocator's bitvector, but that's part of the thread local cache.
+ So, that means that either the partial views' bits pointer cannot be a compact pointer, or the
+ thread_local_cache needs to be in the compact heap. So, the thread_local_cache ended up in the compact
+ heap to keep that pointer small.
+
+ This change works around the problem: it's rare that the partial views' bits pointer points at
+ the local_allocator's bits, and none of the fast path cases where we access that pointer will ever see
+ it in that state. So, this makes the pointer either point to a utility-heap-allocated box that contains
+ the full pointer, or it points at the actual array allocated in the compact heap. The utility heap is
+ in the compact heap, so the compact pointer can point at either one. The implementation of this is
+ encapsulated as pas_lenient_compact_ptr. It's a bit gross; storing to it only works when you're holding
+ the heap lock, for example.
+
+ This is perf-neutral on Speedometer. This is perf-neutral on JS2 cli with full JSC isoheaps (i.e. the
+ patch from bug 231938). It's a 0.4% regression on RAMification with full JSC isoheaps, but I'm not
+ going to worry about that because trunk doesn't have full JSC isoheaps, and JSC isoheaps requires some
+ change like this to work reliably (currently it'll randomly run out of compact heap).
+
+ * bmalloc.xcodeproj/project.pbxproj:
+ * libpas/libpas.xcodeproj/project.pbxproj:
+ * libpas/src/libpas/pas_enumerate_segregated_heaps.c:
+ (enumerate_partial_view):
+ * libpas/src/libpas/pas_full_alloc_bits_inlines.h:
+ (pas_full_alloc_bits_create_for_partial_but_not_primordial):
+ (pas_full_alloc_bits_create_for_partial):
+ * libpas/src/libpas/pas_lenient_compact_ptr.h: Added.
+ * libpas/src/libpas/pas_lenient_compact_ptr_inlines.h: Added.
+ * libpas/src/libpas/pas_lenient_compact_unsigned_ptr.c: Added.
+ * libpas/src/libpas/pas_lenient_compact_unsigned_ptr.h: Added.
+ * libpas/src/libpas/pas_local_allocator.c:
+ (pas_local_allocator_move):
+ * libpas/src/libpas/pas_local_allocator_inlines.h:
+ (pas_local_allocator_set_up_free_bits):
+ (pas_local_allocator_start_allocating_in_primordial_partial_view):
+ (pas_local_allocator_bless_primordial_partial_view_before_stopping):
+ * libpas/src/libpas/pas_segregated_partial_view.c:
+ (pas_segregated_partial_view_create):
+ (compute_summary):
+ * libpas/src/libpas/pas_segregated_partial_view.h:
+ * libpas/src/libpas/pas_segregated_view_allocator_inlines.h:
+ (pas_segregated_view_will_start_allocating):
+ * libpas/src/libpas/pas_thread_local_cache.c:
+ (deallocate):
+ (allocate_cache):
+ * libpas/src/test/LotsOfHeapsAndThreads.cpp: Added.
+ (std::testLotsOfHeapsAndThreads):
+ (addLotsOfHeapsAndThreadsTests):
+ * libpas/src/test/TestHarness.cpp:
+ (main):
+
2022-01-12 Elliott Williams <e...@apple.com>
[Xcode] Configure each project for the legacy build system
Modified: trunk/Source/bmalloc/bmalloc.xcodeproj/project.pbxproj (287967 => 287968)
--- trunk/Source/bmalloc/bmalloc.xcodeproj/project.pbxproj 2022-01-13 02:17:09 UTC (rev 287967)
+++ trunk/Source/bmalloc/bmalloc.xcodeproj/project.pbxproj 2022-01-13 02:52:00 UTC (rev 287968)
@@ -616,6 +616,10 @@
2C48134427406A3E006CAB55 /* pas_expendable_memory.h in Headers */ = {isa = PBXBuildFile; fileRef = 2C48133E27406A3E006CAB55 /* pas_expendable_memory.h */; settings = {ATTRIBUTES = (Private, ); }; };
2C48134527406A3E006CAB55 /* pas_large_expendable_memory.h in Headers */ = {isa = PBXBuildFile; fileRef = 2C48133F27406A3E006CAB55 /* pas_large_expendable_memory.h */; settings = {ATTRIBUTES = (Private, ); }; };
2C91E552271CE47A00D67FF9 /* pas_size_lookup_mode.h in Headers */ = {isa = PBXBuildFile; fileRef = 2C91E551271CE47A00D67FF9 /* pas_size_lookup_mode.h */; settings = {ATTRIBUTES = (Private, ); }; };
+ 2C971D04278E798700C9E129 /* pas_lenient_compact_ptr_inlines.h in Headers */ = {isa = PBXBuildFile; fileRef = 2C971D00278E798700C9E129 /* pas_lenient_compact_ptr_inlines.h */; };
+ 2C971D05278E798700C9E129 /* pas_lenient_compact_ptr.h in Headers */ = {isa = PBXBuildFile; fileRef = 2C971D01278E798700C9E129 /* pas_lenient_compact_ptr.h */; settings = {ATTRIBUTES = (Private, ); }; };
+ 2C971D06278E798700C9E129 /* pas_lenient_compact_unsigned_ptr.h in Headers */ = {isa = PBXBuildFile; fileRef = 2C971D02278E798700C9E129 /* pas_lenient_compact_unsigned_ptr.h */; settings = {ATTRIBUTES = (Private, ); }; };
+ 2C971D07278E798700C9E129 /* pas_lenient_compact_unsigned_ptr.c in Sources */ = {isa = PBXBuildFile; fileRef = 2C971D03278E798700C9E129 /* pas_lenient_compact_unsigned_ptr.c */; };
2CE2AE2D27596DEB00D02BBC /* pas_segregated_page_role.h in Headers */ = {isa = PBXBuildFile; fileRef = 2CE2AE2627596DEA00D02BBC /* pas_segregated_page_role.h */; settings = {ATTRIBUTES = (Private, ); }; };
2CE2AE2E27596DEB00D02BBC /* pas_page_base_and_kind.h in Headers */ = {isa = PBXBuildFile; fileRef = 2CE2AE2727596DEA00D02BBC /* pas_page_base_and_kind.h */; settings = {ATTRIBUTES = (Private, ); }; };
2CE2AE2F27596DEB00D02BBC /* pas_page_base_config.c in Sources */ = {isa = PBXBuildFile; fileRef = 2CE2AE2827596DEA00D02BBC /* pas_page_base_config.c */; };
@@ -1275,6 +1279,10 @@
2C48133E27406A3E006CAB55 /* pas_expendable_memory.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = pas_expendable_memory.h; path = libpas/src/libpas/pas_expendable_memory.h; sourceTree = "<group>"; };
2C48133F27406A3E006CAB55 /* pas_large_expendable_memory.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = pas_large_expendable_memory.h; path = libpas/src/libpas/pas_large_expendable_memory.h; sourceTree = "<group>"; };
2C91E551271CE47A00D67FF9 /* pas_size_lookup_mode.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = pas_size_lookup_mode.h; path = libpas/src/libpas/pas_size_lookup_mode.h; sourceTree = "<group>"; };
+ 2C971D00278E798700C9E129 /* pas_lenient_compact_ptr_inlines.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = pas_lenient_compact_ptr_inlines.h; path = libpas/src/libpas/pas_lenient_compact_ptr_inlines.h; sourceTree = "<group>"; };
+ 2C971D01278E798700C9E129 /* pas_lenient_compact_ptr.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = pas_lenient_compact_ptr.h; path = libpas/src/libpas/pas_lenient_compact_ptr.h; sourceTree = "<group>"; };
+ 2C971D02278E798700C9E129 /* pas_lenient_compact_unsigned_ptr.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = pas_lenient_compact_unsigned_ptr.h; path = libpas/src/libpas/pas_lenient_compact_unsigned_ptr.h; sourceTree = "<group>"; };
+ 2C971D03278E798700C9E129 /* pas_lenient_compact_unsigned_ptr.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = pas_lenient_compact_unsigned_ptr.c; path = libpas/src/libpas/pas_lenient_compact_unsigned_ptr.c; sourceTree = "<group>"; };
2CE2AE2627596DEA00D02BBC /* pas_segregated_page_role.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = pas_segregated_page_role.h; path = libpas/src/libpas/pas_segregated_page_role.h; sourceTree = "<group>"; };
2CE2AE2727596DEA00D02BBC /* pas_page_base_and_kind.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = pas_page_base_and_kind.h; path = libpas/src/libpas/pas_page_base_and_kind.h; sourceTree = "<group>"; };
2CE2AE2827596DEA00D02BBC /* pas_page_base_config.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = pas_page_base_config.c; path = libpas/src/libpas/pas_page_base_config.c; sourceTree = "<group>"; };
@@ -1701,6 +1709,10 @@
0FC40A1D2451498400876DA0 /* pas_large_sharing_pool.h */,
0FC40AA32451498E00876DA0 /* pas_large_utility_free_heap.c */,
0FC40AF12451499300876DA0 /* pas_large_utility_free_heap.h */,
+ 2C971D00278E798700C9E129 /* pas_lenient_compact_ptr_inlines.h */,
+ 2C971D01278E798700C9E129 /* pas_lenient_compact_ptr.h */,
+ 2C971D03278E798700C9E129 /* pas_lenient_compact_unsigned_ptr.c */,
+ 2C971D02278E798700C9E129 /* pas_lenient_compact_unsigned_ptr.h */,
0F8A812425F83E2400790B4A /* pas_line_word_config.h */,
0FC40A7D2451498B00876DA0 /* pas_list_direction.h */,
0FC40A4C2451498800876DA0 /* pas_local_allocator_config_kind.h */,
@@ -2179,6 +2191,7 @@
0F7EB82D1F9541B000F1ABCB /* IsoTLSEntry.h in Headers */,
0F7EB8351F9541B000F1ABCB /* IsoTLSEntryInlines.h in Headers */,
0F7EB8461F9541B000F1ABCB /* IsoTLSInlines.h in Headers */,
+ 2C971D04278E798700C9E129 /* pas_lenient_compact_ptr_inlines.h in Headers */,
0F7EB8401F9541B000F1ABCB /* IsoTLSLayout.h in Headers */,
0F5193EF266C42AD00483A2C /* jit_heap.h in Headers */,
0F5193EE266C42AD00483A2C /* jit_heap_config.h in Headers */,
@@ -2186,6 +2199,7 @@
144C07F51C7B70260051BB6A /* LargeMap.h in Headers */,
14C8992D1CC578330027A057 /* LargeRange.h in Headers */,
140FA00519CE4B6800FFD3C8 /* LineMetadata.h in Headers */,
+ 2C971D06278E798700C9E129 /* pas_lenient_compact_unsigned_ptr.h in Headers */,
141D9B001C8E51C0000ABBA0 /* List.h in Headers */,
4426E2811C838EE0008EB042 /* Logging.h in Headers */,
14C8992B1CC485E70027A057 /* Map.h in Headers */,
@@ -2312,6 +2326,7 @@
0FC409EA2451496400876DA0 /* pas_ensure_heap_forced_into_reserved_memory.h in Headers */,
0FC409E92451496400876DA0 /* pas_ensure_heap_with_page_caches.h in Headers */,
0F87002C25AF89C9000E1ABF /* pas_enumerable_page_malloc.h in Headers */,
+ 2C971D05278E798700C9E129 /* pas_lenient_compact_ptr.h in Headers */,
0F87002E25AF89C9000E1ABF /* pas_enumerable_range_list.h in Headers */,
0F87FFEC25AF897C000E1ABF /* pas_enumerate_bitfit_heaps.h in Headers */,
0F87000A25AF897C000E1ABF /* pas_enumerate_initially_unaccounted_pages.h in Headers */,
@@ -2683,6 +2698,7 @@
14F271C818EA3990008C152F /* ObjectType.cpp in Sources */,
E378A9DF246B68720029C2BB /* ObjectTypeTable.cpp in Sources */,
0F18F84725C3467700721C2A /* pagesize64k_heap.c in Sources */,
+ 2C971D07278E798700C9E129 /* pas_lenient_compact_unsigned_ptr.c in Sources */,
0F18F84925C3467700721C2A /* pagesize64k_heap_config.c in Sources */,
0FC4095D2451494400876DA0 /* pas_alignment.c in Sources */,
0FC409642451494400876DA0 /* pas_all_heaps.c in Sources */,
Modified: trunk/Source/bmalloc/libpas/libpas.xcodeproj/project.pbxproj (287967 => 287968)
--- trunk/Source/bmalloc/libpas/libpas.xcodeproj/project.pbxproj 2022-01-13 02:17:09 UTC (rev 287967)
+++ trunk/Source/bmalloc/libpas/libpas.xcodeproj/project.pbxproj 2022-01-13 02:52:00 UTC (rev 287968)
@@ -571,6 +571,11 @@
2C48132D273F4159006CAB55 /* ExpendableMemoryTests.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 2C48132C273F4159006CAB55 /* ExpendableMemoryTests.cpp */; };
2C85DC4127128F0F00367905 /* pas_try_allocate_intrinsic.h in Headers */ = {isa = PBXBuildFile; fileRef = 2C85DC4027128F0F00367905 /* pas_try_allocate_intrinsic.h */; };
2C91E5502718DA9A00D67FF9 /* pas_size_lookup_mode.h in Headers */ = {isa = PBXBuildFile; fileRef = 2C91E54F2718DA9A00D67FF9 /* pas_size_lookup_mode.h */; };
+ 2C971CFA278E11C300C9E129 /* pas_lenient_compact_ptr.h in Headers */ = {isa = PBXBuildFile; fileRef = 2C971CF8278E11C300C9E129 /* pas_lenient_compact_ptr.h */; };
+ 2C971CFB278E11C300C9E129 /* pas_lenient_compact_unsigned_ptr.h in Headers */ = {isa = PBXBuildFile; fileRef = 2C971CF9278E11C300C9E129 /* pas_lenient_compact_unsigned_ptr.h */; };
+ 2C971CFE278E136300C9E129 /* pas_lenient_compact_unsigned_ptr.c in Sources */ = {isa = PBXBuildFile; fileRef = 2C971CFC278E136300C9E129 /* pas_lenient_compact_unsigned_ptr.c */; };
+ 2C971CFF278E136300C9E129 /* pas_lenient_compact_ptr_inlines.h in Headers */ = {isa = PBXBuildFile; fileRef = 2C971CFD278E136300C9E129 /* pas_lenient_compact_ptr_inlines.h */; };
+ 2CB9B153278F6C85003A8C1B /* LotsOfHeapsAndThreads.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 2CB9B152278F6C85003A8C1B /* LotsOfHeapsAndThreads.cpp */; };
2CE2AE35275A953E00D02BBC /* BitfitTests.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 2CE2AE34275A953E00D02BBC /* BitfitTests.cpp */; };
/* End PBXBuildFile section */
@@ -1259,6 +1264,11 @@
2C48132C273F4159006CAB55 /* ExpendableMemoryTests.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = ExpendableMemoryTests.cpp; sourceTree = "<group>"; };
2C85DC4027128F0F00367905 /* pas_try_allocate_intrinsic.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = pas_try_allocate_intrinsic.h; sourceTree = "<group>"; };
2C91E54F2718DA9A00D67FF9 /* pas_size_lookup_mode.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = pas_size_lookup_mode.h; sourceTree = "<group>"; };
+ 2C971CF8278E11C300C9E129 /* pas_lenient_compact_ptr.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = pas_lenient_compact_ptr.h; sourceTree = "<group>"; };
+ 2C971CF9278E11C300C9E129 /* pas_lenient_compact_unsigned_ptr.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = pas_lenient_compact_unsigned_ptr.h; sourceTree = "<group>"; };
+ 2C971CFC278E136300C9E129 /* pas_lenient_compact_unsigned_ptr.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = pas_lenient_compact_unsigned_ptr.c; sourceTree = "<group>"; };
+ 2C971CFD278E136300C9E129 /* pas_lenient_compact_ptr_inlines.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = pas_lenient_compact_ptr_inlines.h; sourceTree = "<group>"; };
+ 2CB9B152278F6C85003A8C1B /* LotsOfHeapsAndThreads.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = LotsOfHeapsAndThreads.cpp; sourceTree = "<group>"; };
2CE2AE34275A953E00D02BBC /* BitfitTests.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = BitfitTests.cpp; sourceTree = "<group>"; };
/* End PBXFileReference section */
@@ -1354,7 +1364,6 @@
0FDEA45D228B651B0085E340 /* BitfieldVectorTests.cpp */,
2CE2AE34275A953E00D02BBC /* BitfitTests.cpp */,
0F53181022C954ED003F7B6A /* BitvectorTests.cpp */,
- 2B2A589B2742D815005EE07C /* PGMTests.cpp */,
0F31A66723E8B336002C0CA3 /* CartesianTreeTests.cpp */,
0FC682362129D4EB003C6A13 /* CoalignTests.cpp */,
2C48132C273F4159006CAB55 /* ExpendableMemoryTests.cpp */,
@@ -1368,13 +1377,14 @@
0FD48B6623B589910026C46D /* IsoHeapPartialAndBaselineTests.cpp */,
0F5B6093235E919900CAE629 /* IsoHeapReservedMemoryTests.cpp */,
0F5193E6266AE5D400483A2C /* JITHeapTests.cpp */,
- 2B8BCAC62739A47900B1A46F /* PGMTests.cpp */,
0FC64190213745FA0040CE5E /* LargeFreeHeapTests.cpp */,
0FEB6666231C87BC009C001B /* LargeSharingPoolDump.cpp */,
0FEB6667231C87BC009C001B /* LargeSharingPoolDump.h */,
0FEB6669231DA86F009C001B /* LargeSharingPoolTests.cpp */,
0FEA45BE236CDAED00B5A375 /* LockFreeReadPtrPtrHashtableTests.cpp */,
+ 2CB9B152278F6C85003A8C1B /* LotsOfHeapsAndThreads.cpp */,
0FD22D0722CD8A7500B21841 /* MinHeapTests.cpp */,
+ 2B2A589B2742D815005EE07C /* PGMTests.cpp */,
0F5E483923D69F610046DA5C /* RaceTests.cpp */,
0FF08F3522A59DB300386575 /* RedBlackTreeTests.cpp */,
0FDE52552342B7C400A0808F /* SuspendScavenger.h */,
@@ -1410,8 +1420,6 @@
0F5FE7A325B614F2001859FC /* iso_heap_inlines.h */,
0F516EE624561A60004E2B8D /* iso_heap_innards.h */,
0F529DDE2455460C00385A8C /* iso_heap_ref.h */,
- 2B2A58982742D802005EE07C /* pas_probabilistic_guard_malloc_allocator.c */,
- 2B2A58972742D802005EE07C /* pas_probabilistic_guard_malloc_allocator.h */,
0FE7EE74229F010F004F4166 /* iso_heap.c */,
0FE7EE75229F010F004F4166 /* iso_heap.h */,
0F5E5CE823BEF35F00AA0B6F /* iso_test_heap_config.c */,
@@ -1685,6 +1693,10 @@
0FCFA498238352EB00CCD726 /* pas_large_sharing_pool.h */,
0F6D547523C573E000F40DBB /* pas_large_utility_free_heap.c */,
0F6D547223C573E000F40DBB /* pas_large_utility_free_heap.h */,
+ 2C971CFD278E136300C9E129 /* pas_lenient_compact_ptr_inlines.h */,
+ 2C971CF8278E11C300C9E129 /* pas_lenient_compact_ptr.h */,
+ 2C971CFC278E136300C9E129 /* pas_lenient_compact_unsigned_ptr.c */,
+ 2C971CF9278E11C300C9E129 /* pas_lenient_compact_unsigned_ptr.h */,
0F8A802725EC588800790B4A /* pas_line_word_config.h */,
0F31A64323E3509E002C0CA3 /* pas_list_direction.h */,
0FD48B0223A9ABB00026C46D /* pas_local_allocator_config_kind.h */,
@@ -1742,13 +1754,13 @@
0FD22CFD22CC01D300B21841 /* pas_page_sharing_pool.h */,
0F4F61C925A4CD3B008B4A82 /* pas_payload_reservation_page_list.c */,
0F4F61C825A4CD3B008B4A82 /* pas_payload_reservation_page_list.h */,
- 2B8BCAC32739A45900B1A46F /* pas_probabilistic_guard_malloc_allocator.c */,
- 2B8BCAC22739A45900B1A46F /* pas_probabilistic_guard_malloc_allocator.h */,
0F6D547323C573E000F40DBB /* pas_physical_memory_synchronization_style.h */,
0FF248F722FC9EDA0077202E /* pas_physical_memory_transaction.c */,
0F78088622FA2E4900F37451 /* pas_physical_memory_transaction.h */,
0FDEA578228E23450085E340 /* pas_primitive_heap_ref.c */,
0FDEA577228E23440085E340 /* pas_primitive_heap_ref.h */,
+ 2B2A58982742D802005EE07C /* pas_probabilistic_guard_malloc_allocator.c */,
+ 2B2A58972742D802005EE07C /* pas_probabilistic_guard_malloc_allocator.h */,
0F9A1CAD2559961300C8D11B /* pas_promote_intrinsic_heap.h */,
0F4F615A259BF6EC008B4A82 /* pas_ptr_hash_map.h */,
0F4F6146259BB109008B4A82 /* pas_ptr_hash_set.h */,
@@ -2050,6 +2062,7 @@
0F9A1D41255AF27900C8D11B /* pas_bitfit_view_inlines.h in Headers */,
0FE7EE0422960142004F4166 /* pas_bitvector.h in Headers */,
0FE7EE0522960142004F4166 /* pas_bootstrap_free_heap.h in Headers */,
+ 2C971CFB278E11C300C9E129 /* pas_lenient_compact_unsigned_ptr.h in Headers */,
0F5B6084235E572000CAE629 /* pas_bootstrap_heap_page_provider.h in Headers */,
0FE7EE0822960142004F4166 /* pas_cares_about_size_mode.h in Headers */,
0F31A66623E8A942002C0CA3 /* pas_cartesian_tree.h in Headers */,
@@ -2146,6 +2159,7 @@
0FD48B5823A9ABB30026C46D /* pas_full_alloc_bits.h in Headers */,
0FD48B3A23A9ABB30026C46D /* pas_full_alloc_bits_inlines.h in Headers */,
0FF08F3A22A5B40300386575 /* pas_generic_large_free_heap.h in Headers */,
+ 2C971CFA278E11C300C9E129 /* pas_lenient_compact_ptr.h in Headers */,
0FE7EE60229E14FA004F4166 /* pas_get_allocation_size.h in Headers */,
0FE7EE64229E14FA004F4166 /* pas_get_heap.h in Headers */,
0F9A1D4D255F2CD700C8D11B /* pas_get_object_kind.h in Headers */,
@@ -2253,6 +2267,7 @@
0F4F60FB25979BC8008B4A82 /* pas_root.h in Headers */,
0F19326C22F73E8500FBA713 /* pas_scavenger.h in Headers */,
0F68127222BD4BF40036A02B /* pas_segmented_vector.h in Headers */,
+ 2C971CFF278E136300C9E129 /* pas_lenient_compact_ptr_inlines.h in Headers */,
0F148710269B7518006887A9 /* pas_segregated_deallocation_mode.h in Headers */,
0FD48B5623A9ABB30026C46D /* pas_segregated_directory.h in Headers */,
0FD48B4723A9ABB30026C46D /* pas_segregated_directory_bit_reference.h in Headers */,
@@ -2599,6 +2614,7 @@
0FA18546236B3C82003609AD /* IsoHeapChaosTests.cpp in Sources */,
0FF248FD230CC4CB0077202E /* IsoHeapPageSharingTests.cpp in Sources */,
0FD48B6723B589910026C46D /* IsoHeapPartialAndBaselineTests.cpp in Sources */,
+ 2CB9B153278F6C85003A8C1B /* LotsOfHeapsAndThreads.cpp in Sources */,
0F5B6094235E919900CAE629 /* IsoHeapReservedMemoryTests.cpp in Sources */,
2CE2AE35275A953E00D02BBC /* BitfitTests.cpp in Sources */,
0F5193E7266AE5D400483A2C /* JITHeapTests.cpp in Sources */,
@@ -2681,6 +2697,7 @@
0F4F611B259838B1008B4A82 /* pas_enumerator_region.c in Sources */,
0F19325622ED04A600FBA713 /* pas_epoch.c in Sources */,
0FD48B3423A9ABB30026C46D /* pas_exclusive_view_template_memo_table.c in Sources */,
+ 2C971CFE278E136300C9E129 /* pas_lenient_compact_unsigned_ptr.c in Sources */,
2C481328273F341A006CAB55 /* pas_expendable_memory.c in Sources */,
0FE7EDBA22960142004F4166 /* pas_extended_gcd.c in Sources */,
0FF08F4322A627C100386575 /* pas_fast_large_free_heap.c in Sources */,
Modified: trunk/Source/bmalloc/libpas/src/libpas/pas_enumerate_segregated_heaps.c (287967 => 287968)
--- trunk/Source/bmalloc/libpas/src/libpas/pas_enumerate_segregated_heaps.c 2022-01-13 02:17:09 UTC (rev 287967)
+++ trunk/Source/bmalloc/libpas/src/libpas/pas_enumerate_segregated_heaps.c 2022-01-13 02:52:00 UTC (rev 287968)
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020-2021 Apple Inc. All rights reserved.
+ * Copyright (c) 2020-2022 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -515,7 +515,11 @@
if (verbose)
pas_log("Found allocator = %p\n", allocator);
- full_alloc_bits.bits = pas_compact_tagged_unsigned_ptr_load_remote(enumerator, &view->alloc_bits);
+ /* This is so weird: the size we pass is only valid when view->alloc_bits is pointing at the
+ local_allocator's bits. But that's the only time that load_remote will go down the path where it needs
+ to know the size. So, it's fine, I guess. */
+ full_alloc_bits.bits = pas_lenient_compact_unsigned_ptr_load_remote(
+ enumerator, &view->alloc_bits, pas_segregated_page_config_num_alloc_bytes(*page_config));
full_alloc_bits.word_index_begin = view->alloc_bits_offset;
full_alloc_bits.word_index_end = view->alloc_bits_offset + view->alloc_bits_size;
record_page_objects(
Modified: trunk/Source/bmalloc/libpas/src/libpas/pas_full_alloc_bits_inlines.h (287967 => 287968)
--- trunk/Source/bmalloc/libpas/src/libpas/pas_full_alloc_bits_inlines.h 2022-01-13 02:17:09 UTC (rev 287967)
+++ trunk/Source/bmalloc/libpas/src/libpas/pas_full_alloc_bits_inlines.h 2022-01-13 02:52:00 UTC (rev 287968)
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019-2020 Apple Inc. All rights reserved.
+ * Copyright (c) 2019-2022 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -49,6 +49,19 @@
}
static PAS_ALWAYS_INLINE pas_full_alloc_bits
+pas_full_alloc_bits_create_for_partial_but_not_primordial(pas_segregated_view view)
+{
+ pas_segregated_partial_view* partial_view;
+
+ partial_view = pas_segregated_view_get_partial(view);
+
+ return pas_full_alloc_bits_create(
+ pas_lenient_compact_unsigned_ptr_load_compact_non_null(&partial_view->alloc_bits),
+ partial_view->alloc_bits_offset,
+ partial_view->alloc_bits_offset + partial_view->alloc_bits_size);
+}
+
+static PAS_ALWAYS_INLINE pas_full_alloc_bits
pas_full_alloc_bits_create_for_partial(pas_segregated_view view)
{
pas_segregated_partial_view* partial_view;
@@ -56,7 +69,7 @@
partial_view = pas_segregated_view_get_partial(view);
return pas_full_alloc_bits_create(
- pas_compact_tagged_unsigned_ptr_load_non_null(&partial_view->alloc_bits),
+ pas_lenient_compact_unsigned_ptr_load(&partial_view->alloc_bits),
partial_view->alloc_bits_offset,
partial_view->alloc_bits_offset + partial_view->alloc_bits_size);
}
Added: trunk/Source/bmalloc/libpas/src/libpas/pas_lenient_compact_ptr.h (0 => 287968)
--- trunk/Source/bmalloc/libpas/src/libpas/pas_lenient_compact_ptr.h (rev 0)
+++ trunk/Source/bmalloc/libpas/src/libpas/pas_lenient_compact_ptr.h 2022-01-13 02:52:00 UTC (rev 287968)
@@ -0,0 +1,80 @@
+/*
+ * Copyright (c) 2022 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef PAS_LENIENT_COMPACT_PTR_H
+#define PAS_LENIENT_COMPACT_PTR_H
+
+#include "pas_compact_tagged_atomic_ptr.h"
+
+PAS_BEGIN_EXTERN_C;
+
+/* You can use this pointer to point at something that is likely to be in the compact heap but that
+ sometimes won't be. To make this work, it's necessary to be able to destruct the pointer, and it's
+ not legal to pass the pointer around by value. Also, the thing being pointed to must have the lowest
+ bit available (i.e. that bit must always be zero). */
+
+#define PAS_LENIENT_COMPACT_PTR_INITIALIZER { .ptr = PAS_COMPACT_TAGGED_ATOMIC_PTR_INITIALIZER }
+
+#define PAS_LENIENT_COMPACT_PTR_FULL_PTR_BIT ((uintptr_t)1)
+
+#define PAS_DECLARE_LENIENT_COMPACT_PTR(type, name) \
+ \
+ PAS_DEFINE_COMPACT_TAGGED_ATOMIC_PTR(type*, name ## _compact_tagged_atomic_ptr); \
+ \
+ struct name; \
+ typedef struct name name; \
+ \
+ struct name { \
+ name ## _compact_tagged_atomic_ptr ptr; \
+ }; \
+ \
+ PAS_API void name ## _destruct(name* ptr); \
+ PAS_API void name ## _store(name* ptr, type* value); \
+ PAS_API type* name ## _load(name* ptr); \
+ \
+ static inline type* name ## _load_compact(name* ptr) \
+ { \
+ type* result; \
+ result = name ## _compact_tagged_atomic_ptr_load(&ptr->ptr); \
+ PAS_TESTING_ASSERT(!((uintptr_t)result & PAS_LENIENT_COMPACT_PTR_FULL_PTR_BIT)); \
+ return result; \
+ } \
+ \
+ static inline type* name ## _load_compact_non_null(name* ptr) \
+ { \
+ type* result; \
+ result = name ## _compact_tagged_atomic_ptr_load_non_null(&ptr->ptr); \
+ PAS_TESTING_ASSERT(!((uintptr_t)result & PAS_LENIENT_COMPACT_PTR_FULL_PTR_BIT)); \
+ return result; \
+ } \
+ \
+ PAS_API type* name ## _load_remote(pas_enumerator* enumerator, name* ptr, size_t size); \
+ \
+ struct pas_dummy
+
+PAS_END_EXTERN_C;
+
+#endif /* PAS_LENIENT_COMPACT_PTR_H */
+
Added: trunk/Source/bmalloc/libpas/src/libpas/pas_lenient_compact_ptr_inlines.h (0 => 287968)
--- trunk/Source/bmalloc/libpas/src/libpas/pas_lenient_compact_ptr_inlines.h (rev 0)
+++ trunk/Source/bmalloc/libpas/src/libpas/pas_lenient_compact_ptr_inlines.h 2022-01-13 02:52:00 UTC (rev 287968)
@@ -0,0 +1,86 @@
+/*
+ * Copyright (c) 2022 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef PAS_LENIENT_COMPACT_PTR_INLINES_H
+#define PAS_LENIENT_COMPACT_PTR_INLINES_H
+
+#include "pas_lenient_compact_ptr.h"
+#include "pas_utility_heap.h"
+
+PAS_BEGIN_EXTERN_C;
+
+#define PAS_DEFINE_LENIENT_COMPACT_PTR(type, name) \
+ void name ## _destruct(name* ptr) \
+ { \
+ type* old_value; \
+ old_value = name ## _compact_tagged_atomic_ptr_load(&ptr->ptr); \
+ if ((uintptr_t)old_value & PAS_LENIENT_COMPACT_PTR_FULL_PTR_BIT) { \
+ pas_utility_heap_deallocate( \
+ (void*)((uintptr_t)old_value & ~PAS_LENIENT_COMPACT_PTR_FULL_PTR_BIT)); \
+ } \
+ } \
+ \
+ void name ## _store(name* ptr, type* value) \
+ { \
+ PAS_TESTING_ASSERT(!((uintptr_t)value & PAS_LENIENT_COMPACT_PTR_FULL_PTR_BIT)); \
+ name ## _destruct(ptr); \
+ if ((uintptr_t)value >= PAS_INTERNAL_MIN_ALIGN \
+ && (uintptr_t)value - pas_compact_heap_reservation_base >= pas_compact_heap_reservation_size) { \
+ type** box; \
+ box = pas_utility_heap_allocate(sizeof(type*), #name "/box"); \
+ *box = value; \
+ value = (type*)((uintptr_t)box | PAS_LENIENT_COMPACT_PTR_FULL_PTR_BIT); \
+ } \
+ name ## _compact_tagged_atomic_ptr_store(&ptr->ptr, value); \
+ } \
+ \
+ type* name ## _load(name* ptr) \
+ { \
+ type* result; \
+ result = name ## _compact_tagged_atomic_ptr_load(&ptr->ptr); \
+ if ((uintptr_t)result & PAS_LENIENT_COMPACT_PTR_FULL_PTR_BIT) \
+ return *(type**)((uintptr_t)result & ~PAS_LENIENT_COMPACT_PTR_FULL_PTR_BIT); \
+ return result; \
+ } \
+ \
+ type* name ## _load_remote(pas_enumerator* enumerator, name* ptr, size_t size) \
+ { \
+ type* result; \
+ result = name ## _compact_tagged_atomic_ptr_load_remote(enumerator, &ptr->ptr); \
+ if ((uintptr_t)result & PAS_LENIENT_COMPACT_PTR_FULL_PTR_BIT) { \
+ return (type*)pas_enumerator_read( \
+ enumerator, \
+ *(type**)((uintptr_t)result & ~PAS_LENIENT_COMPACT_PTR_FULL_PTR_BIT), \
+ size); \
+ } \
+ return result; \
+ } \
+ \
+ struct pas_dummy
+
+PAS_END_EXTERN_C;
+
+#endif /* PAS_LENIENT_COMPACT_PTR_INLINES_H */
+
Added: trunk/Source/bmalloc/libpas/src/libpas/pas_lenient_compact_unsigned_ptr.c (0 => 287968)
--- trunk/Source/bmalloc/libpas/src/libpas/pas_lenient_compact_unsigned_ptr.c (rev 0)
+++ trunk/Source/bmalloc/libpas/src/libpas/pas_lenient_compact_unsigned_ptr.c 2022-01-13 02:52:00 UTC (rev 287968)
@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2022 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "pas_config.h"
+
+#if LIBPAS_ENABLED
+
+#include "pas_lenient_compact_unsigned_ptr.h"
+
+#include "pas_lenient_compact_ptr_inlines.h"
+
+PAS_DEFINE_LENIENT_COMPACT_PTR(unsigned, pas_lenient_compact_unsigned_ptr);
+
+#endif /* LIBPAS_ENABLED */
+
+
Added: trunk/Source/bmalloc/libpas/src/libpas/pas_lenient_compact_unsigned_ptr.h (0 => 287968)
--- trunk/Source/bmalloc/libpas/src/libpas/pas_lenient_compact_unsigned_ptr.h (rev 0)
+++ trunk/Source/bmalloc/libpas/src/libpas/pas_lenient_compact_unsigned_ptr.h 2022-01-13 02:52:00 UTC (rev 287968)
@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2022 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef PAS_LENIENT_COMPACT_UNSIGNED_PTR_H
+#define PAS_LENIENT_COMPACT_UNSIGNED_PTR_H
+
+#include "pas_lenient_compact_ptr.h"
+
+PAS_BEGIN_EXTERN_C;
+
+PAS_DECLARE_LENIENT_COMPACT_PTR(unsigned, pas_lenient_compact_unsigned_ptr);
+
+PAS_END_EXTERN_C;
+
+#endif /* PAS_LENIENT_COMPACT_UNSIGNED_PTR_H */
+
Modified: trunk/Source/bmalloc/libpas/src/libpas/pas_local_allocator.c (287967 => 287968)
--- trunk/Source/bmalloc/libpas/src/libpas/pas_local_allocator.c 2022-01-13 02:17:09 UTC (rev 287967)
+++ trunk/Source/bmalloc/libpas/src/libpas/pas_local_allocator.c 2022-01-13 02:52:00 UTC (rev 287968)
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018-2021 Apple Inc. All rights reserved.
+ * Copyright (c) 2018-2022 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -118,6 +118,8 @@
pas_segregated_partial_view* partial_view;
pas_segregated_shared_view* shared_view;
+ pas_heap_lock_assert_held(); /* Needed to modify a lenient_compact_ptr. */
+
directory = pas_segregated_view_get_size_directory(src->view);
size = pas_segregated_size_directory_local_allocator_size(directory);
@@ -135,8 +137,8 @@
is the only client of partial_view->alloc_bits being right during primordial mode, and it
happens to hold the ownership lock. */
pas_lock_lock(&shared_view->ownership_lock);
- if (pas_compact_tagged_unsigned_ptr_load(&partial_view->alloc_bits) == (unsigned*)src->bits)
- pas_compact_tagged_unsigned_ptr_store(&partial_view->alloc_bits, (unsigned*)dst->bits);
+ if (pas_lenient_compact_unsigned_ptr_load(&partial_view->alloc_bits) == (unsigned*)src->bits)
+ pas_lenient_compact_unsigned_ptr_store(&partial_view->alloc_bits, (unsigned*)dst->bits);
pas_lock_unlock(&shared_view->ownership_lock);
}
Modified: trunk/Source/bmalloc/libpas/src/libpas/pas_local_allocator_inlines.h (287967 => 287968)
--- trunk/Source/bmalloc/libpas/src/libpas/pas_local_allocator_inlines.h 2022-01-13 02:17:09 UTC (rev 287967)
+++ trunk/Source/bmalloc/libpas/src/libpas/pas_local_allocator_inlines.h 2022-01-13 02:52:00 UTC (rev 287968)
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018-2021 Apple Inc. All rights reserved.
+ * Copyright (c) 2018-2022 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -352,7 +352,7 @@
partial_view_as_view = pas_segregated_partial_view_as_view((pas_segregated_partial_view*)view);
- full_alloc_bits = pas_full_alloc_bits_create_for_partial(partial_view_as_view);
+ full_alloc_bits = pas_full_alloc_bits_create_for_partial_but_not_primordial(partial_view_as_view);
allocator->view = partial_view_as_view;
@@ -599,21 +599,16 @@
shared_page_directory = page_config.shared_page_directory_selector(heap, size_directory);
shared_view = pas_segregated_shared_page_directory_find_first_eligible(
- shared_page_directory, size, alignment,
- pas_segregated_page_config_heap_lock_hold_mode(page_config));
+ shared_page_directory, size, alignment, pas_lock_is_not_held);
PAS_ASSERT(shared_view);
- pas_lock_lock_conditionally(
- &shared_view->commit_lock,
- pas_segregated_page_config_heap_lock_hold_mode(page_config));
+ pas_lock_lock(&shared_view->commit_lock);
handle = pas_segregated_shared_view_commit_page_if_necessary(
shared_view, heap, shared_page_directory, view, page_config);
if (!handle) {
- pas_lock_unlock_conditionally(
- &shared_view->commit_lock,
- pas_segregated_page_config_heap_lock_hold_mode(page_config));
+ pas_lock_unlock(&shared_view->commit_lock);
return false;
}
@@ -663,13 +658,6 @@
pas_zero_memory(allocator->bits, pas_segregated_page_config_num_alloc_bytes(page_config));
- /* Doing this helps heap introspection but isn't otherwise necessary. */
- pas_compact_tagged_unsigned_ptr_store(&view->alloc_bits, (unsigned*)allocator->bits);
- PAS_ASSERT(!view->alloc_bits_offset);
- PAS_ASSERT((uint8_t)pas_segregated_page_config_num_alloc_words(page_config)
- == pas_segregated_page_config_num_alloc_words(page_config));
- view->alloc_bits_size = (uint8_t)pas_segregated_page_config_num_alloc_words(page_config);
-
pas_local_allocator_set_up_primordial_bump(
allocator, view, handle, page, &held_lock, bump_result,
pas_local_allocator_primordial_bump_stash_whole_allocation,
@@ -679,11 +667,37 @@
view->is_attached_to_shared_handle = true;
+ pas_lock_unlock(&shared_view->commit_lock);
+
+ /* This code to set the view's alloc_bits is primarily for heap enumeration.
+
+ If the enumerator ran right now then (before we do this stuff), then it would see:
+
+ - A bunch of page alloc bits set for the objects we are about to allocate.
+ - No partial views say that they own those objects.
+ - Local allocator claims to own those objects.
+
+ So, we would report that those objects are not live right now. I believe they would show up
+ as "meta" allocations, not even as objects.
+
+ Immediately after settings the view's alloc_bits and alloc_bits_size, we will know that the
+ objects exist (view alloc bits are set), but they are dead (despite page alloc bits also being
+ set, the objects are part of the bump range). */
+ if (!pas_heap_lock_try_lock()) {
+ pas_lock_switch(&held_lock, NULL);
+ pas_heap_lock_lock();
+ pas_segregated_page_switch_lock(page, &held_lock, page_config);
+ }
+
+ pas_lenient_compact_unsigned_ptr_store(&view->alloc_bits, (unsigned*)allocator->bits);
+ PAS_ASSERT(!view->alloc_bits_offset);
+ PAS_ASSERT((uint8_t)pas_segregated_page_config_num_alloc_words(page_config)
+ == pas_segregated_page_config_num_alloc_words(page_config));
+ pas_compiler_fence();
+ view->alloc_bits_size = (uint8_t)pas_segregated_page_config_num_alloc_words(page_config);
pas_lock_switch(&held_lock, NULL);
- pas_lock_unlock_conditionally(
- &shared_view->commit_lock,
- pas_segregated_page_config_heap_lock_hold_mode(page_config));
-
+ pas_heap_lock_unlock();
+
return true;
}
}
@@ -775,30 +789,28 @@
PAS_ASSERT((uint8_t)alloc_bits_offset == alloc_bits_offset);
view->alloc_bits_offset = (uint8_t)alloc_bits_offset;
+
+ /* We hold the page lock. Lock ordering says that we cannot acquire the heap lock when
+ we are holding the page lock.
+
+ Luckily, this is a fine place to drop the page lock.
+
+ Therefore, we try-lock the heap lock. It's always safe to do that. Usually it will just
+ succeed. But if it fails, we will drop the page lock and then acquire both of them. */
+ if (!pas_heap_lock_try_lock_conditionally(heap_lock_hold_mode)) {
+ pas_segregated_page_unlock(page, page_config);
+ pas_heap_lock_lock();
+ pas_segregated_page_lock(page, page_config);
+ }
if (alloc_bits_size == 1)
alloc_bits = &view->inline_alloc_bits - alloc_bits_offset;
else {
- /* We hold the page lock. Lock ordering says that we cannot acquire the heap lock when
- we are holding the page lock.
-
- Luckily, this is a fine place to drop the page lock.
-
- Therefore, we try-lock the heap lock. It's always safe to do that. Usually it will just
- succeed. But if it fails, we will drop the page lock and then acquire both of them. */
- if (!pas_heap_lock_try_lock_conditionally(heap_lock_hold_mode)) {
- pas_segregated_page_unlock(page, page_config);
- pas_heap_lock_lock();
- pas_segregated_page_lock(page, page_config);
- }
-
alloc_bits = (unsigned*)pas_immortal_heap_allocate_with_manual_alignment(
alloc_bits_size * sizeof(unsigned),
sizeof(unsigned),
"pas_segregated_partial_view/alloc_bits",
pas_object_allocation) - alloc_bits_offset;
-
- pas_heap_lock_unlock_conditionally(heap_lock_hold_mode);
}
memcpy(alloc_bits + alloc_bits_offset,
@@ -807,7 +819,9 @@
pas_store_store_fence();
- pas_compact_tagged_unsigned_ptr_store(&view->alloc_bits, alloc_bits);
+ pas_lenient_compact_unsigned_ptr_store(&view->alloc_bits, alloc_bits);
+
+ pas_heap_lock_unlock_conditionally(heap_lock_hold_mode);
}
static PAS_ALWAYS_INLINE pas_allocation_result
Modified: trunk/Source/bmalloc/libpas/src/libpas/pas_segregated_partial_view.c (287967 => 287968)
--- trunk/Source/bmalloc/libpas/src/libpas/pas_segregated_partial_view.c 2022-01-13 02:17:09 UTC (rev 287967)
+++ trunk/Source/bmalloc/libpas/src/libpas/pas_segregated_partial_view.c 2022-01-13 02:52:00 UTC (rev 287968)
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019-2020 Apple Inc. All rights reserved.
+ * Copyright (c) 2019-2022 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -64,7 +64,7 @@
result->alloc_bits_offset = 0;
result->alloc_bits_size = 0;
- pas_compact_tagged_unsigned_ptr_store(&result->alloc_bits, NULL);
+ pas_lenient_compact_unsigned_ptr_store(&result->alloc_bits, NULL);
result->inline_alloc_bits = 0; /* There isn't a real big need to do this, but it helps keep
things sane. */
@@ -186,7 +186,7 @@
shared_view = pas_compact_segregated_shared_view_ptr_load_non_null(&view->shared_view);
- full_alloc_bits = pas_compact_tagged_unsigned_ptr_load_non_null(&view->alloc_bits);
+ full_alloc_bits = pas_lenient_compact_unsigned_ptr_load(&view->alloc_bits);
if (shared_view->is_owned) {
page_boundary = (uintptr_t)pas_shared_handle_or_page_boundary_get_page_boundary(
Modified: trunk/Source/bmalloc/libpas/src/libpas/pas_segregated_partial_view.h (287967 => 287968)
--- trunk/Source/bmalloc/libpas/src/libpas/pas_segregated_partial_view.h 2022-01-13 02:17:09 UTC (rev 287967)
+++ trunk/Source/bmalloc/libpas/src/libpas/pas_segregated_partial_view.h 2022-01-13 02:52:00 UTC (rev 287968)
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019-2020 Apple Inc. All rights reserved.
+ * Copyright (c) 2019-2022 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -28,9 +28,9 @@
#include "pas_compact_segregated_shared_view_ptr.h"
#include "pas_compact_segregated_size_directory_ptr.h"
-#include "pas_compact_tagged_unsigned_ptr.h"
+#include "pas_lenient_compact_unsigned_ptr.h"
+#include "pas_page_granule_use_count.h"
#include "pas_segregated_page_config.h"
-#include "pas_page_granule_use_count.h"
#include "pas_segregated_view.h"
#include "pas_utils.h"
@@ -48,7 +48,7 @@
pas_compact_segregated_shared_view_ptr shared_view;
pas_compact_segregated_size_directory_ptr directory;
- pas_compact_tagged_unsigned_ptr alloc_bits;
+ pas_lenient_compact_unsigned_ptr alloc_bits;
/* The index can be small since we would never create a high-indexed partial view. That would not
have a meaningful effect on the footprint of the directory.
Modified: trunk/Source/bmalloc/libpas/src/libpas/pas_segregated_view_allocator_inlines.h (287967 => 287968)
--- trunk/Source/bmalloc/libpas/src/libpas/pas_segregated_view_allocator_inlines.h 2022-01-13 02:17:09 UTC (rev 287967)
+++ trunk/Source/bmalloc/libpas/src/libpas/pas_segregated_view_allocator_inlines.h 2022-01-13 02:52:00 UTC (rev 287968)
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2021 Apple Inc. All rights reserved.
+ * Copyright (c) 2021-2022 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -262,7 +262,7 @@
unsigned end_word_index;
unsigned word_index;
- full_alloc_bits = pas_compact_tagged_unsigned_ptr_load_non_null(&partial->alloc_bits);
+ full_alloc_bits = pas_lenient_compact_unsigned_ptr_load_compact_non_null(&partial->alloc_bits);
begin_word_index = partial->alloc_bits_offset;
end_word_index = begin_word_index + partial->alloc_bits_size;
Modified: trunk/Source/bmalloc/libpas/src/libpas/pas_thread_local_cache.c (287967 => 287968)
--- trunk/Source/bmalloc/libpas/src/libpas/pas_thread_local_cache.c 2022-01-13 02:17:09 UTC (rev 287967)
+++ trunk/Source/bmalloc/libpas/src/libpas/pas_thread_local_cache.c 2022-01-13 02:52:00 UTC (rev 287968)
@@ -30,9 +30,9 @@
#include "pas_thread_local_cache.h"
#include "pas_all_heap_configs.h"
-#include "pas_compact_large_utility_free_heap.h"
#include "pas_debug_heap.h"
#include "pas_heap_lock.h"
+#include "pas_large_utility_free_heap.h"
#include "pas_log.h"
#include "pas_monotonic_time.h"
#include "pas_scavenger.h"
@@ -63,12 +63,12 @@
{
char* begin;
- pas_compact_large_utility_free_heap_deallocate(
+ pas_large_utility_free_heap_deallocate(
thread_local_cache->should_stop_bitvector,
PAS_BITVECTOR_NUM_BYTES(thread_local_cache->allocator_index_capacity));
begin = (char*)thread_local_cache;
- pas_compact_large_utility_free_heap_deallocate(
+ pas_large_utility_free_heap_deallocate(
begin,
pas_thread_local_cache_size_for_allocator_index_capacity(
thread_local_cache->allocator_index_capacity));
@@ -129,11 +129,11 @@
if (verbose)
printf("Cache size: %zu\n", size);
- result = (pas_thread_local_cache*)pas_compact_large_utility_free_heap_allocate(size, "pas_thread_local_cache");
+ result = (pas_thread_local_cache*)pas_large_utility_free_heap_allocate(size, "pas_thread_local_cache");
pas_zero_memory(result, size);
- result->should_stop_bitvector = (unsigned int*)pas_compact_large_utility_free_heap_allocate(
+ result->should_stop_bitvector = (unsigned int*)pas_large_utility_free_heap_allocate(
PAS_BITVECTOR_NUM_BYTES(allocator_index_capacity),
"pas_thread_local_cache/should_stop_bitvector");
Added: trunk/Source/bmalloc/libpas/src/test/LotsOfHeapsAndThreads.cpp (0 => 287968)
--- trunk/Source/bmalloc/libpas/src/test/LotsOfHeapsAndThreads.cpp (rev 0)
+++ trunk/Source/bmalloc/libpas/src/test/LotsOfHeapsAndThreads.cpp 2022-01-13 02:52:00 UTC (rev 287968)
@@ -0,0 +1,78 @@
+/*
+ * Copyright (c) 2022 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "TestHarness.h"
+
+#if PAS_ENABLE_BMALLOC
+
+#include "bmalloc_heap.h"
+#include "bmalloc_heap_config.h"
+#include "pas_get_heap.h"
+#include <thread>
+
+using namespace std;
+
+namespace {
+
+void testLotsOfHeapsAndThreads(unsigned numHeaps, unsigned numThreads, unsigned count)
+{
+ thread* threads = new thread[numThreads];
+ pas_heap_ref* heaps = new pas_heap_ref[numHeaps];
+
+ for (unsigned i = numHeaps; i--;)
+ heaps[i] = BMALLOC_HEAP_REF_INITIALIZER(new bmalloc_type(BMALLOC_TYPE_INITIALIZER(16, 16, "test")));
+
+ for (unsigned i = numThreads; i--;) {
+ threads[i] = thread([&] () {
+ for (unsigned j = count; j--;) {
+ for (unsigned k = numHeaps; k--;) {
+ void* ptr = bmalloc_iso_allocate(heaps + k);
+ CHECK_EQUAL(pas_get_heap(ptr, BMALLOC_HEAP_CONFIG),
+ bmalloc_heap_ref_get_heap(heaps + k));
+ bmalloc_deallocate(ptr);
+ }
+ }
+ });
+ }
+
+ for (unsigned i = numThreads; i--;)
+ threads[i].join();
+}
+
+} // anonymous namespace
+
+#endif // PAS_ENABLE_BMALLOC
+
+void addLotsOfHeapsAndThreadsTests()
+{
+#if PAS_ENABLE_BMALLOC
+ ForceTLAs forceTLAs;
+ ForcePartials forcePartials;
+
+ ADD_TEST(testLotsOfHeapsAndThreads(10000, 100, 10));
+ ADD_TEST(testLotsOfHeapsAndThreads(25000, 100, 10));
+ ADD_TEST(testLotsOfHeapsAndThreads(30000, 100, 10)); // This is about as high as we can reliably go right now!
+#endif // PAS_ENABLE_BMALLOC
+}
Modified: trunk/Source/bmalloc/libpas/src/test/TestHarness.cpp (287967 => 287968)
--- trunk/Source/bmalloc/libpas/src/test/TestHarness.cpp 2022-01-13 02:17:09 UTC (rev 287967)
+++ trunk/Source/bmalloc/libpas/src/test/TestHarness.cpp 2022-01-13 02:52:00 UTC (rev 287968)
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018-2021 Apple Inc. All rights reserved.
+ * Copyright (c) 2018-2022 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -353,6 +353,7 @@
void addLargeFreeHeapTests();
void addLargeSharingPoolTests();
void addLockFreeReadPtrPtrHashtableTests();
+void addLotsOfHeapsAndThreadsTests();
void addMinHeapTests();
void addPGMTests();
void addRaceTests();
@@ -717,6 +718,7 @@
ADD_SUITE(LargeFreeHeap);
ADD_SUITE(LargeSharingPool);
ADD_SUITE(LockFreeReadPtrPtrHashtable);
+ ADD_SUITE(LotsOfHeapsAndThreads);
ADD_SUITE(MinHeap);
ADD_SUITE(PGM);
ADD_SUITE(Race);
