Diff
Modified: branches/safari-613-branch/Source/bmalloc/CMakeLists.txt (290856 => 290857)
--- branches/safari-613-branch/Source/bmalloc/CMakeLists.txt 2022-03-05 01:06:41 UTC (rev 290856)
+++ branches/safari-613-branch/Source/bmalloc/CMakeLists.txt 2022-03-05 01:44:20 UTC (rev 290857)
@@ -126,7 +126,6 @@
libpas/src/libpas/pas_local_allocator.c
libpas/src/libpas/pas_local_allocator_scavenger_data.c
libpas/src/libpas/pas_local_view_cache.c
- libpas/src/libpas/pas_local_view_cache_node.c
libpas/src/libpas/pas_lock.c
libpas/src/libpas/pas_lock_free_read_ptr_ptr_hashtable.c
libpas/src/libpas/pas_log.c
@@ -505,7 +504,6 @@
libpas/src/libpas/pas_local_allocator_result.h
libpas/src/libpas/pas_local_allocator_scavenger_data.h
libpas/src/libpas/pas_local_view_cache.h
- libpas/src/libpas/pas_local_view_cache_node.h
libpas/src/libpas/pas_lock_free_read_ptr_ptr_hashtable.h
libpas/src/libpas/pas_lock.h
libpas/src/libpas/pas_log.h
Modified: branches/safari-613-branch/Source/bmalloc/ChangeLog (290856 => 290857)
--- branches/safari-613-branch/Source/bmalloc/ChangeLog 2022-03-05 01:06:41 UTC (rev 290856)
+++ branches/safari-613-branch/Source/bmalloc/ChangeLog 2022-03-05 01:44:20 UTC (rev 290857)
@@ -1,3 +1,85 @@
+2022-03-04 Russell Epstein <[email protected]>
+
+ Apply patch. rdar://problem/89071706
+
+ 2022-03-04 Yusuke Suzuki <[email protected]>
+
+ Unreviewed, fix heap enumeration
+ https://bugs.webkit.org/show_bug.cgi?id=236662
+
+ * libpas/src/libpas/pas_enumerate_segregated_heaps.c:
+ (pas_enumerate_segregated_heaps):
+
+2022-02-16 Yusuke Suzuki <[email protected]>
+
+ [libpas] Use segmented vector for TLC layout list
+ https://bugs.webkit.org/show_bug.cgi?id=236662
+
+ Reviewed by Saam Barati.
+
+ We add segmented vector for pas_thread_local_cache_layout_node iteration instead of using
+ linked-list of that to accelerate iteration performance. It does not affect on Speedometer2,
+ JetStream2, PLT5 etc., but we do this iteration frequently in scavenger thread, and reducing
+ time of this makes it efficient. While speeding up scavenger does not matter for throughput
+ of non scavenger threads since libpas works well without taking a lock in most times which
+ is blocked by scavenger, making scavenger faster is good for energy efficiency.
+
+ 1. Use segmented vector for (concurrently read) pas_thread_local_cache_layout_node list.
+ We still keep nice for-loop abstraction for list of pas_thread_local_cache_layout_node.
+ 2. Remove pas_local_view_cache_node since we no longer need it. It was introduced just to
+ make it linked-list node for pas_thread_local_cache_layout_node. But now, segmented vector
+ is holding it separately.
+ 3. Move pas_segregated_size_directory_data's allocator_index to pas_segregated_size_directory.
+ Because of (1) change, we can make pas_segregated_size_directory linked-list-node, which
+ saves 4 bytes. So we can move pas_segregated_size_directory_data's allocator_index to
+ pas_segregated_size_directory since it is heavily accessed from the scavenger.
+
+ * CMakeLists.txt:
+ * bmalloc.xcodeproj/project.pbxproj:
+ * libpas/libpas.xcodeproj/project.pbxproj:
+ * libpas/src/libpas/pas_designated_intrinsic_heap.c:
+ (set_up_range):
+ * libpas/src/libpas/pas_enumerate_segregated_heaps.c:
+ (pas_enumerate_segregated_heaps):
+ * libpas/src/libpas/pas_local_view_cache_node.c: Removed.
+ * libpas/src/libpas/pas_local_view_cache_node.h: Removed.
+ * libpas/src/libpas/pas_redundant_local_allocator_node.c:
+ (pas_redundant_local_allocator_node_create):
+ * libpas/src/libpas/pas_redundant_local_allocator_node.h:
+ * libpas/src/libpas/pas_root.c:
+ (pas_root_construct):
+ * libpas/src/libpas/pas_root.h:
+ * libpas/src/libpas/pas_segregated_heap.c:
+ (recompute_size_lookup):
+ (pas_segregated_heap_ensure_allocator_index):
+ * libpas/src/libpas/pas_segregated_size_directory.c:
+ (pas_segregated_size_directory_create):
+ (pas_segregated_size_directory_ensure_data):
+ (pas_segregated_size_directory_create_tlc_allocator):
+ (pas_segregated_size_directory_get_allocator_from_tlc):
+ * libpas/src/libpas/pas_segregated_size_directory.h:
+ (pas_segregated_size_directory_has_tlc_allocator):
+ (pas_segregated_size_directory_get_tlc_allocator_index):
+ * libpas/src/libpas/pas_thread_local_cache.c:
+ (decommit_allocator_range):
+ (pas_thread_local_cache_for_all):
+ * libpas/src/libpas/pas_thread_local_cache_layout.c:
+ (pas_thread_local_cache_layout_add_node):
+ (pas_thread_local_cache_layout_add_view_cache):
+ (pas_thread_local_cache_layout_get_last_node):
+ * libpas/src/libpas/pas_thread_local_cache_layout.h:
+ (pas_thread_local_cache_layout_segment_get_node):
+ (pas_thread_local_cache_layout_segment_next_node):
+ * libpas/src/libpas/pas_thread_local_cache_layout_node.c:
+ (pas_thread_local_cache_layout_node_get_directory):
+ (allocator_index_ptr):
+ (next_ptr): Deleted.
+ (pas_thread_local_cache_layout_node_get_next): Deleted.
+ (pas_thread_local_cache_layout_node_set_next): Deleted.
+ * libpas/src/libpas/pas_thread_local_cache_layout_node.h:
+ (pas_wrap_local_view_cache_node):
+ (pas_unwrap_local_view_cache_node):
+
2022-02-07 Russell Epstein <[email protected]>
Cherry-pick r289144. rdar://problem/88585404
Modified: branches/safari-613-branch/Source/bmalloc/bmalloc.xcodeproj/project.pbxproj (290856 => 290857)
--- branches/safari-613-branch/Source/bmalloc/bmalloc.xcodeproj/project.pbxproj 2022-03-05 01:06:41 UTC (rev 290856)
+++ branches/safari-613-branch/Source/bmalloc/bmalloc.xcodeproj/project.pbxproj 2022-03-05 01:44:20 UTC (rev 290857)
@@ -88,11 +88,9 @@
0F7C92E926E57F75006AF012 /* pas_heap_runtime_config.c in Sources */ = {isa = PBXBuildFile; fileRef = 0F7C92D926E57F75006AF012 /* pas_heap_runtime_config.c */; };
0F7C92EA26E57F75006AF012 /* pas_local_allocator_scavenger_data.c in Sources */ = {isa = PBXBuildFile; fileRef = 0F7C92DA26E57F75006AF012 /* pas_local_allocator_scavenger_data.c */; };
0F7C92EB26E57F75006AF012 /* pas_compact_segregated_size_directory_ptr.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F7C92DB26E57F75006AF012 /* pas_compact_segregated_size_directory_ptr.h */; settings = {ATTRIBUTES = (Private, ); }; };
- 0F7C92EC26E57F75006AF012 /* pas_local_view_cache_node.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F7C92DC26E57F75006AF012 /* pas_local_view_cache_node.h */; };
0F7C92ED26E57F75006AF012 /* pas_segregated_size_directory_creation_mode.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F7C92DD26E57F75006AF012 /* pas_segregated_size_directory_creation_mode.h */; settings = {ATTRIBUTES = (Private, ); }; };
0F7C92EE26E57F75006AF012 /* pas_local_view_cache.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F7C92DE26E57F75006AF012 /* pas_local_view_cache.h */; settings = {ATTRIBUTES = (Private, ); }; };
0F7C92F026E57F75006AF012 /* pas_local_allocator_kind.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F7C92E026E57F75006AF012 /* pas_local_allocator_kind.h */; settings = {ATTRIBUTES = (Private, ); }; };
- 0F7C92F126E57F75006AF012 /* pas_local_view_cache_node.c in Sources */ = {isa = PBXBuildFile; fileRef = 0F7C92E126E57F75006AF012 /* pas_local_view_cache_node.c */; };
0F7EB8231F9541B000F1ABCB /* EligibilityResult.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F7EB7FC1F9541AD00F1ABCB /* EligibilityResult.h */; settings = {ATTRIBUTES = (Private, ); }; };
0F7EB8241F9541B000F1ABCB /* IsoHeapImplInlines.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F7EB7FD1F9541AD00F1ABCB /* IsoHeapImplInlines.h */; settings = {ATTRIBUTES = (Private, ); }; };
0F7EB8251F9541B000F1ABCB /* DeferredTriggerInlines.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F7EB7FE1F9541AD00F1ABCB /* DeferredTriggerInlines.h */; settings = {ATTRIBUTES = (Private, ); }; };
@@ -747,11 +745,9 @@
0F7C92D926E57F75006AF012 /* pas_heap_runtime_config.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = pas_heap_runtime_config.c; path = libpas/src/libpas/pas_heap_runtime_config.c; sourceTree = "<group>"; };
0F7C92DA26E57F75006AF012 /* pas_local_allocator_scavenger_data.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = pas_local_allocator_scavenger_data.c; path = libpas/src/libpas/pas_local_allocator_scavenger_data.c; sourceTree = "<group>"; };
0F7C92DB26E57F75006AF012 /* pas_compact_segregated_size_directory_ptr.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = pas_compact_segregated_size_directory_ptr.h; path = libpas/src/libpas/pas_compact_segregated_size_directory_ptr.h; sourceTree = "<group>"; };
- 0F7C92DC26E57F75006AF012 /* pas_local_view_cache_node.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = pas_local_view_cache_node.h; path = libpas/src/libpas/pas_local_view_cache_node.h; sourceTree = "<group>"; };
0F7C92DD26E57F75006AF012 /* pas_segregated_size_directory_creation_mode.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = pas_segregated_size_directory_creation_mode.h; path = libpas/src/libpas/pas_segregated_size_directory_creation_mode.h; sourceTree = "<group>"; };
0F7C92DE26E57F75006AF012 /* pas_local_view_cache.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = pas_local_view_cache.h; path = libpas/src/libpas/pas_local_view_cache.h; sourceTree = "<group>"; };
0F7C92E026E57F75006AF012 /* pas_local_allocator_kind.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = pas_local_allocator_kind.h; path = libpas/src/libpas/pas_local_allocator_kind.h; sourceTree = "<group>"; };
- 0F7C92E126E57F75006AF012 /* pas_local_view_cache_node.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = pas_local_view_cache_node.c; path = libpas/src/libpas/pas_local_view_cache_node.c; sourceTree = "<group>"; };
0F7EB7FC1F9541AD00F1ABCB /* EligibilityResult.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = EligibilityResult.h; path = bmalloc/EligibilityResult.h; sourceTree = SOURCE_ROOT; };
0F7EB7FD1F9541AD00F1ABCB /* IsoHeapImplInlines.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = IsoHeapImplInlines.h; path = bmalloc/IsoHeapImplInlines.h; sourceTree = SOURCE_ROOT; };
0F7EB7FE1F9541AD00F1ABCB /* DeferredTriggerInlines.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = DeferredTriggerInlines.h; path = bmalloc/DeferredTriggerInlines.h; sourceTree = SOURCE_ROOT; };
@@ -1730,8 +1726,6 @@
0F7C92D426E57F74006AF012 /* pas_local_allocator_scavenger_data.h */,
0F7C92D626E57F75006AF012 /* pas_local_view_cache.c */,
0F7C92DE26E57F75006AF012 /* pas_local_view_cache.h */,
- 0F7C92E126E57F75006AF012 /* pas_local_view_cache_node.c */,
- 0F7C92DC26E57F75006AF012 /* pas_local_view_cache_node.h */,
0FC40BD9245243A400876DA0 /* pas_lock.c */,
0FC40A8B2451498C00876DA0 /* pas_lock.h */,
0FC40A562451498800876DA0 /* pas_lock_free_read_ptr_ptr_hashtable.c */,
@@ -2416,7 +2410,6 @@
0FC40B2E2451499400876DA0 /* pas_local_allocator_result.h in Headers */,
0F7C92E426E57F75006AF012 /* pas_local_allocator_scavenger_data.h in Headers */,
0F7C92EE26E57F75006AF012 /* pas_local_view_cache.h in Headers */,
- 0F7C92EC26E57F75006AF012 /* pas_local_view_cache_node.h in Headers */,
0FC40B672451499400876DA0 /* pas_lock.h in Headers */,
0FC40B7F2451499400876DA0 /* pas_lock_free_read_ptr_ptr_hashtable.h in Headers */,
0FC40B9C2451499400876DA0 /* pas_log.h in Headers */,
@@ -2776,7 +2769,6 @@
0FC40B072451499400876DA0 /* pas_local_allocator.c in Sources */,
0F7C92EA26E57F75006AF012 /* pas_local_allocator_scavenger_data.c in Sources */,
0F7C92E626E57F75006AF012 /* pas_local_view_cache.c in Sources */,
- 0F7C92F126E57F75006AF012 /* pas_local_view_cache_node.c in Sources */,
0FC40BDA245243A400876DA0 /* pas_lock.c in Sources */,
0FC40B342451499400876DA0 /* pas_lock_free_read_ptr_ptr_hashtable.c in Sources */,
0FC40BB22451499400876DA0 /* pas_log.c in Sources */,
Modified: branches/safari-613-branch/Source/bmalloc/libpas/libpas.xcodeproj/project.pbxproj (290856 => 290857)
--- branches/safari-613-branch/Source/bmalloc/libpas/libpas.xcodeproj/project.pbxproj 2022-03-05 01:06:41 UTC (rev 290856)
+++ branches/safari-613-branch/Source/bmalloc/libpas/libpas.xcodeproj/project.pbxproj 2022-03-05 01:44:20 UTC (rev 290857)
@@ -53,10 +53,8 @@
0F148713269B7518006887A9 /* pas_compact_segregated_exclusive_view_ptr.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F14870D269B7518006887A9 /* pas_compact_segregated_exclusive_view_ptr.h */; };
0F148714269B7518006887A9 /* pas_heap_runtime_config.c in Sources */ = {isa = PBXBuildFile; fileRef = 0F14870E269B7518006887A9 /* pas_heap_runtime_config.c */; };
0F148715269B7518006887A9 /* pas_compact_segregated_size_directory_ptr.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F14870F269B7518006887A9 /* pas_compact_segregated_size_directory_ptr.h */; };
- 0F14871F269F5FCA006887A9 /* pas_local_view_cache_node.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F14871A269F5FCA006887A9 /* pas_local_view_cache_node.h */; };
0F148720269F5FCA006887A9 /* pas_local_view_cache.c in Sources */ = {isa = PBXBuildFile; fileRef = 0F14871B269F5FCA006887A9 /* pas_local_view_cache.c */; };
0F148721269F5FCA006887A9 /* pas_segregated_size_directory_creation_mode.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F14871C269F5FCA006887A9 /* pas_segregated_size_directory_creation_mode.h */; };
- 0F148722269F5FCA006887A9 /* pas_local_view_cache_node.c in Sources */ = {isa = PBXBuildFile; fileRef = 0F14871D269F5FCA006887A9 /* pas_local_view_cache_node.c */; };
0F148723269F5FCA006887A9 /* pas_thread_local_cache_layout_node_kind.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F14871E269F5FCA006887A9 /* pas_thread_local_cache_layout_node_kind.h */; };
0F1518FD23441E3200832B5C /* pas_has_object.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F1518FC23441E3200832B5C /* pas_has_object.h */; };
0F18E63E2628BFB700F6903D /* pas_shared_page_directory_by_size.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F18E63C2628BFB600F6903D /* pas_shared_page_directory_by_size.h */; };
@@ -577,6 +575,8 @@
2C971CFF278E136300C9E129 /* pas_lenient_compact_ptr_inlines.h in Headers */ = {isa = PBXBuildFile; fileRef = 2C971CFD278E136300C9E129 /* pas_lenient_compact_ptr_inlines.h */; };
2CB9B153278F6C85003A8C1B /* LotsOfHeapsAndThreads.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 2CB9B152278F6C85003A8C1B /* LotsOfHeapsAndThreads.cpp */; };
2CE2AE35275A953E00D02BBC /* BitfitTests.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 2CE2AE34275A953E00D02BBC /* BitfitTests.cpp */; };
+ E3A7DC8F27BEF8EA004E76E9 /* pas_thread_suspend_lock.h in Headers */ = {isa = PBXBuildFile; fileRef = E3A7DC8D27BEF8EA004E76E9 /* pas_thread_suspend_lock.h */; };
+ E3A7DC9027BEF8EA004E76E9 /* pas_thread_suspend_lock.c in Sources */ = {isa = PBXBuildFile; fileRef = E3A7DC8E27BEF8EA004E76E9 /* pas_thread_suspend_lock.c */; };
/* End PBXBuildFile section */
/* Begin PBXContainerItemProxy section */
@@ -739,10 +739,8 @@
0F14870D269B7518006887A9 /* pas_compact_segregated_exclusive_view_ptr.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = pas_compact_segregated_exclusive_view_ptr.h; sourceTree = "<group>"; };
0F14870E269B7518006887A9 /* pas_heap_runtime_config.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = pas_heap_runtime_config.c; sourceTree = "<group>"; };
0F14870F269B7518006887A9 /* pas_compact_segregated_size_directory_ptr.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = pas_compact_segregated_size_directory_ptr.h; sourceTree = "<group>"; };
- 0F14871A269F5FCA006887A9 /* pas_local_view_cache_node.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = pas_local_view_cache_node.h; sourceTree = "<group>"; };
0F14871B269F5FCA006887A9 /* pas_local_view_cache.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = pas_local_view_cache.c; sourceTree = "<group>"; };
0F14871C269F5FCA006887A9 /* pas_segregated_size_directory_creation_mode.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = pas_segregated_size_directory_creation_mode.h; sourceTree = "<group>"; };
- 0F14871D269F5FCA006887A9 /* pas_local_view_cache_node.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = pas_local_view_cache_node.c; sourceTree = "<group>"; };
0F14871E269F5FCA006887A9 /* pas_thread_local_cache_layout_node_kind.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = pas_thread_local_cache_layout_node_kind.h; sourceTree = "<group>"; };
0F1518FC23441E3200832B5C /* pas_has_object.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = pas_has_object.h; sourceTree = "<group>"; };
0F18E63C2628BFB600F6903D /* pas_shared_page_directory_by_size.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = pas_shared_page_directory_by_size.h; sourceTree = "<group>"; };
@@ -1270,6 +1268,8 @@
2C971CFD278E136300C9E129 /* pas_lenient_compact_ptr_inlines.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = pas_lenient_compact_ptr_inlines.h; sourceTree = "<group>"; };
2CB9B152278F6C85003A8C1B /* LotsOfHeapsAndThreads.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = LotsOfHeapsAndThreads.cpp; sourceTree = "<group>"; };
2CE2AE34275A953E00D02BBC /* BitfitTests.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = BitfitTests.cpp; sourceTree = "<group>"; };
+ E3A7DC8D27BEF8EA004E76E9 /* pas_thread_suspend_lock.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = pas_thread_suspend_lock.h; sourceTree = "<group>"; };
+ E3A7DC8E27BEF8EA004E76E9 /* pas_thread_suspend_lock.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = pas_thread_suspend_lock.c; sourceTree = "<group>"; };
/* End PBXFileReference section */
/* Begin PBXFrameworksBuildPhase section */
@@ -1708,8 +1708,6 @@
0F45D5A826AB731400C35D93 /* pas_local_allocator_scavenger_data.h */,
0FDBB07122B875A6006BA5FC /* pas_local_allocator.c */,
0FDBB07222B875A6006BA5FC /* pas_local_allocator.h */,
- 0F14871D269F5FCA006887A9 /* pas_local_view_cache_node.c */,
- 0F14871A269F5FCA006887A9 /* pas_local_view_cache_node.h */,
0F14871B269F5FCA006887A9 /* pas_local_view_cache.c */,
0F14870C269B7518006887A9 /* pas_local_view_cache.h */,
0FEA45BA236CD5AC00B5A375 /* pas_lock_free_read_ptr_ptr_hashtable.c */,
@@ -1872,6 +1870,8 @@
0F7F51BD23159EC0007552AE /* pas_thread_local_cache_node.h */,
0FC681C9210F7C9F003C6A13 /* pas_thread_local_cache.c */,
0FC681C4210F7C9E003C6A13 /* pas_thread_local_cache.h */,
+ E3A7DC8E27BEF8EA004E76E9 /* pas_thread_suspend_lock.c */,
+ E3A7DC8D27BEF8EA004E76E9 /* pas_thread_suspend_lock.h */,
0F6D547123C573E000F40DBB /* pas_tiny_large_map_entry.h */,
0F31A66923E8E594002C0CA3 /* pas_tree_direction.h */,
0F8A800C25CF64C900790B4A /* pas_tri_state.h */,
@@ -2214,7 +2214,6 @@
0FE7EE0E22960142004F4166 /* pas_local_allocator_result.h in Headers */,
0F45D5AB26AB731400C35D93 /* pas_local_allocator_scavenger_data.h in Headers */,
0F148712269B7518006887A9 /* pas_local_view_cache.h in Headers */,
- 0F14871F269F5FCA006887A9 /* pas_local_view_cache_node.h in Headers */,
0FE7EE2122960142004F4166 /* pas_lock.h in Headers */,
0FEA45BD236CD5AC00B5A375 /* pas_lock_free_read_ptr_ptr_hashtable.h in Headers */,
0F9E945923452262009FAFDD /* pas_log.h in Headers */,
@@ -2290,6 +2289,7 @@
0F083F5A22C2979B00F3CDB3 /* pas_segregated_page_config_utils_inlines.h in Headers */,
0F9A1CEB2559961700C8D11B /* pas_segregated_page_config_variant.h in Headers */,
0FC4EC792351707B00B710A3 /* pas_segregated_page_emptiness_kind.h in Headers */,
+ E3A7DC8F27BEF8EA004E76E9 /* pas_thread_suspend_lock.h in Headers */,
0FF8D62922C166F500EC72FE /* pas_segregated_page_inlines.h in Headers */,
0FD48B3523A9ABB30026C46D /* pas_segregated_partial_view.h in Headers */,
0FD48B6523A9ABB30026C46D /* pas_segregated_partial_view_inlines.h in Headers */,
@@ -2727,7 +2727,6 @@
0FDBB07422B875A6006BA5FC /* pas_local_allocator.c in Sources */,
0F45D5AA26AB731400C35D93 /* pas_local_allocator_scavenger_data.c in Sources */,
0F148720269F5FCA006887A9 /* pas_local_view_cache.c in Sources */,
- 0F148722269F5FCA006887A9 /* pas_local_view_cache_node.c in Sources */,
0F04C21F2460DB6F001D31F0 /* pas_lock.c in Sources */,
0FEA45BC236CD5AC00B5A375 /* pas_lock_free_read_ptr_ptr_hashtable.c in Sources */,
0F9E945823452262009FAFDD /* pas_log.c in Sources */,
@@ -2764,6 +2763,7 @@
0FD48B5C23A9ABB30026C46D /* pas_segregated_size_directory.c in Sources */,
0FD48B3623A9ABB30026C46D /* pas_segregated_view.c in Sources */,
0F18E63F2628BFB700F6903D /* pas_shared_page_directory_by_size.c in Sources */,
+ E3A7DC9027BEF8EA004E76E9 /* pas_thread_suspend_lock.c in Sources */,
0F6D564623CAA1BA00F40DBB /* pas_simple_free_heap_helpers.c in Sources */,
0FF08F3E22A5DC6900386575 /* pas_simple_large_free_heap.c in Sources */,
2C11E88F2728A783002162D0 /* pas_simple_type.c in Sources */,
Modified: branches/safari-613-branch/Source/bmalloc/libpas/src/libpas/pas_designated_intrinsic_heap.c (290856 => 290857)
--- branches/safari-613-branch/Source/bmalloc/libpas/src/libpas/pas_designated_intrinsic_heap.c 2022-03-05 01:06:41 UTC (rev 290856)
+++ branches/safari-613-branch/Source/bmalloc/libpas/src/libpas/pas_designated_intrinsic_heap.c 2022-03-05 01:44:20 UTC (rev 290857)
@@ -63,11 +63,12 @@
PAS_ASSERT(designated_begin == data->next_index_to_set);
- if (pas_thread_local_cache_layout_last_node) {
+ pas_heap_lock_assert_held();
+
+ if (pas_thread_local_cache_layout_get_last_node()) {
PAS_ASSERT(
size
- > pas_thread_local_cache_layout_node_get_directory(
- pas_thread_local_cache_layout_last_node)->object_size);
+ > pas_thread_local_cache_layout_node_get_directory(pas_thread_local_cache_layout_get_last_node())->object_size);
} else
PAS_ASSERT(!designated_begin);
@@ -107,10 +108,9 @@
pas_segregated_size_directory_create_tlc_allocator(directory);
- PAS_ASSERT(
- pas_segregated_size_directory_data_ptr_load(&directory->data)->allocator_index
- == target_allocator_index);
+ PAS_ASSERT(directory->allocator_index == target_allocator_index);
} else {
+ pas_thread_local_cache_layout_node last_node;
pas_allocator_index resulting_index;
PAS_ASSERT(directory);
@@ -117,13 +117,11 @@
resulting_index = pas_thread_local_cache_layout_duplicate(directory);
PAS_ASSERT(resulting_index == target_allocator_index);
- PAS_ASSERT(pas_thread_local_cache_layout_node_get_directory(
- pas_thread_local_cache_layout_last_node) == directory);
- PAS_ASSERT(pas_thread_local_cache_layout_node_get_allocator_index_for_allocator(
- pas_thread_local_cache_layout_last_node) == target_allocator_index);
- PAS_ASSERT(
- pas_segregated_size_directory_data_ptr_load(&directory->data)->allocator_index
- == PAS_LOCAL_ALLOCATOR_UNSELECTED_NUM_INDICES + designated_begin * data->num_allocator_indices);
+
+ last_node = pas_thread_local_cache_layout_get_last_node();
+ PAS_ASSERT(pas_thread_local_cache_layout_node_get_directory(last_node) == directory);
+ PAS_ASSERT(pas_thread_local_cache_layout_node_get_allocator_index_for_allocator(last_node) == target_allocator_index);
+ PAS_ASSERT(directory->allocator_index == PAS_LOCAL_ALLOCATOR_UNSELECTED_NUM_INDICES + designated_begin * data->num_allocator_indices);
}
}
Modified: branches/safari-613-branch/Source/bmalloc/libpas/src/libpas/pas_enumerate_segregated_heaps.c (290856 => 290857)
--- branches/safari-613-branch/Source/bmalloc/libpas/src/libpas/pas_enumerate_segregated_heaps.c 2022-03-05 01:06:41 UTC (rev 290856)
+++ branches/safari-613-branch/Source/bmalloc/libpas/src/libpas/pas_enumerate_segregated_heaps.c 2022-03-05 01:44:20 UTC (rev 290857)
@@ -33,7 +33,6 @@
#include "pas_enumerator_internal.h"
#include "pas_full_alloc_bits.h"
#include "pas_hashtable.h"
-#include "pas_local_view_cache_node.h"
#include "pas_ptr_hash_set.h"
#include "pas_redundant_local_allocator_node.h"
#include "pas_root.h"
@@ -43,6 +42,7 @@
#include "pas_segregated_shared_handle.h"
#include "pas_segregated_shared_view.h"
#include "pas_shared_handle_or_page_boundary_inlines.h"
+#include "pas_thread_local_cache_layout.h"
#include "pas_thread_local_cache_node.h"
static const bool verbose = false;
@@ -629,7 +629,7 @@
{
pas_thread_local_cache_node** tlc_node_first_ptr;
pas_thread_local_cache_node* tlc_node;
- pas_thread_local_cache_layout_node* tlc_layout_first_node_ptr;
+ pas_thread_local_cache_layout_segment** tlc_layout_first_segment_ptr;
enumeration_context context;
pas_baseline_allocator** baseline_allocator_table_ptr;
size_t index;
@@ -644,15 +644,12 @@
if (!tlc_node_first_ptr)
return false;
- tlc_layout_first_node_ptr = pas_enumerator_read(enumerator,
- enumerator->root->thread_local_cache_layout_first_node,
- sizeof(pas_thread_local_cache_layout_node));
- if (!tlc_layout_first_node_ptr)
+ tlc_layout_first_segment_ptr = pas_enumerator_read(enumerator, enumerator->root->thread_local_cache_layout_first_segment, sizeof(pas_thread_local_cache_layout_segment*));
+ if (!tlc_layout_first_segment_ptr)
return false;
for (tlc_node = *tlc_node_first_ptr; tlc_node; tlc_node = tlc_node->next) {
pas_thread_local_cache* tlc;
- pas_thread_local_cache_layout_node layout_node;
unsigned index;
tlc_node = pas_enumerator_read(enumerator,
@@ -687,58 +684,78 @@
}
}
- layout_node = pas_enumerator_read_compact(enumerator, *tlc_layout_first_node_ptr);
- while (layout_node) {
- bool has_allocator;
- unsigned allocator_index;
-
- if (pas_is_wrapped_segregated_size_directory(layout_node)) {
- pas_segregated_size_directory* directory;
- pas_segregated_size_directory_data* data;
+ if (*tlc_layout_first_segment_ptr) {
+ pas_thread_local_cache_layout_segment** tlc_layout_segment_ptr;
+ uintptr_t node_index = 0;
+ pas_thread_local_cache_layout_node layout_node;
+ pas_compact_atomic_thread_local_cache_layout_node* layout_node_ptr;
- directory = pas_unwrap_segregated_size_directory(layout_node);
+ tlc_layout_segment_ptr = tlc_layout_first_segment_ptr;
- data = ""
- enumerator, &directory->data);
+ layout_node_ptr = pas_enumerator_read(enumerator, &((*tlc_layout_segment_ptr)->nodes[node_index]), sizeof(pas_compact_atomic_thread_local_cache_layout_node));
+ if (!layout_node_ptr)
+ return false;
+ layout_node = pas_compact_atomic_thread_local_cache_layout_node_load_remote(enumerator, layout_node_ptr);
+ while (1) {
+ bool has_allocator;
+ unsigned allocator_index;
- layout_node = pas_compact_atomic_thread_local_cache_layout_node_load_remote(
- enumerator, &data->next_for_layout);
+ if (!layout_node) {
+ tlc_layout_segment_ptr = pas_enumerator_read(enumerator, &((*tlc_layout_segment_ptr)->next), sizeof(pas_thread_local_cache_layout_segment*));
+ if (!tlc_layout_segment_ptr)
+ return false;
+ if (!*tlc_layout_segment_ptr)
+ break;
+ node_index = 0;
+ layout_node_ptr = pas_enumerator_read(enumerator, &((*tlc_layout_segment_ptr)->nodes[node_index]), sizeof(pas_compact_atomic_thread_local_cache_layout_node));
+ if (!layout_node_ptr)
+ return false;
+ layout_node = pas_compact_atomic_thread_local_cache_layout_node_load_remote(enumerator, layout_node_ptr);
+ if (!layout_node)
+ break;
+ }
+
+ if (pas_is_wrapped_segregated_size_directory(layout_node)) {
+ pas_segregated_size_directory* directory;
- allocator_index = data->allocator_index;
- has_allocator = true;
- } else if (pas_is_wrapped_redundant_local_allocator_node(layout_node)) {
- pas_redundant_local_allocator_node* redundant_node;
+ directory = pas_unwrap_segregated_size_directory(layout_node);
- redundant_node = pas_unwrap_redundant_local_allocator_node(layout_node);
+ allocator_index = directory->allocator_index;
+ has_allocator = true;
+ } else if (pas_is_wrapped_redundant_local_allocator_node(layout_node)) {
+ pas_redundant_local_allocator_node* redundant_node;
- layout_node = pas_compact_atomic_thread_local_cache_layout_node_load_remote(
- enumerator, &redundant_node->next);
+ redundant_node = pas_unwrap_redundant_local_allocator_node(layout_node);
- allocator_index = redundant_node->allocator_index;
- has_allocator = true;
- } else {
- pas_local_view_cache_node* cache_node;
-
- PAS_ASSERT(pas_is_wrapped_local_view_cache_node(layout_node));
+ allocator_index = redundant_node->allocator_index;
+ has_allocator = true;
+ } else {
+ pas_segregated_size_directory* directory;
+
+ PAS_ASSERT(pas_is_wrapped_local_view_cache_node(layout_node));
- cache_node = pas_unwrap_local_view_cache_node(layout_node);
+ directory = pas_unwrap_local_view_cache_node(layout_node);
- layout_node = pas_compact_atomic_thread_local_cache_layout_node_load_remote(
- enumerator, &cache_node->next);
+ allocator_index = 0;
+ has_allocator = false;
+ }
- allocator_index = 0;
- has_allocator = false;
- }
+ if (has_allocator) {
+ pas_local_allocator* allocator;
+
+ if (!allocator_index || allocator_index >= tlc->allocator_index_upper_bound)
+ break;
- if (has_allocator) {
- pas_local_allocator* allocator;
-
- if (!allocator_index || allocator_index >= tlc->allocator_index_upper_bound)
- break;
+ allocator = pas_thread_local_cache_get_local_allocator_direct(tlc, allocator_index);
+
+ consider_allocator(enumerator, &context, allocator);
+ }
- allocator = pas_thread_local_cache_get_local_allocator_direct(tlc, allocator_index);
-
- consider_allocator(enumerator, &context, allocator);
+ ++node_index;
+ layout_node_ptr = pas_enumerator_read(enumerator, &((*tlc_layout_segment_ptr)->nodes[node_index]), sizeof(pas_compact_atomic_thread_local_cache_layout_node));
+ if (!layout_node_ptr)
+ return false;
+ layout_node = pas_compact_atomic_thread_local_cache_layout_node_load_remote(enumerator, layout_node_ptr);
}
}
}
Deleted: branches/safari-613-branch/Source/bmalloc/libpas/src/libpas/pas_local_view_cache_node.c (290856 => 290857)
--- branches/safari-613-branch/Source/bmalloc/libpas/src/libpas/pas_local_view_cache_node.c 2022-03-05 01:06:41 UTC (rev 290856)
+++ branches/safari-613-branch/Source/bmalloc/libpas/src/libpas/pas_local_view_cache_node.c 2022-03-05 01:44:20 UTC (rev 290857)
@@ -1,50 +0,0 @@
-/*
- * Copyright (c) 2021 Apple Inc. All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
- * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
- * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
- * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
- * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
- * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
- * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include "pas_config.h"
-
-#if LIBPAS_ENABLED
-
-#include "pas_local_view_cache_node.h"
-
-#include "pas_immortal_heap.h"
-
-pas_local_view_cache_node*
-pas_local_view_cache_node_create(pas_segregated_size_directory* directory)
-{
- pas_local_view_cache_node* result;
-
- result = pas_immortal_heap_allocate(
- sizeof(pas_local_view_cache_node),
- "pas_local_view_cache_node",
- pas_object_allocation);
-
- pas_compact_atomic_thread_local_cache_layout_node_store(&result->next, NULL);
- pas_compact_segregated_size_directory_ptr_store(&result->directory, directory);
-
- return result;
-}
-
-#endif /* LIBPAS_ENABLED */
Deleted: branches/safari-613-branch/Source/bmalloc/libpas/src/libpas/pas_local_view_cache_node.h (290856 => 290857)
--- branches/safari-613-branch/Source/bmalloc/libpas/src/libpas/pas_local_view_cache_node.h 2022-03-05 01:06:41 UTC (rev 290856)
+++ branches/safari-613-branch/Source/bmalloc/libpas/src/libpas/pas_local_view_cache_node.h 2022-03-05 01:44:20 UTC (rev 290857)
@@ -1,49 +0,0 @@
-/*
- * Copyright (c) 2021 Apple Inc. All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
- * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
- * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
- * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
- * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
- * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
- * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef PAS_LOCAL_VIEW_CACHE_NODE_H
-#define PAS_LOCAL_VIEW_CACHE_NODE_H
-
-#include "pas_compact_segregated_size_directory_ptr.h"
-#include "pas_compact_atomic_thread_local_cache_layout_node.h"
-#include "pas_allocator_index.h"
-
-PAS_BEGIN_EXTERN_C;
-
-struct pas_local_view_cache_node;
-typedef struct pas_local_view_cache_node pas_local_view_cache_node;
-
-struct pas_local_view_cache_node {
- pas_compact_atomic_thread_local_cache_layout_node next;
- pas_compact_segregated_size_directory_ptr directory;
-};
-
-PAS_API pas_local_view_cache_node*
-pas_local_view_cache_node_create(pas_segregated_size_directory* directory);
-
-PAS_END_EXTERN_C;
-
-#endif /* PAS_LOCAL_VIEW_CACHE_NODE_H */
-
Modified: branches/safari-613-branch/Source/bmalloc/libpas/src/libpas/pas_redundant_local_allocator_node.c (290856 => 290857)
--- branches/safari-613-branch/Source/bmalloc/libpas/src/libpas/pas_redundant_local_allocator_node.c 2022-03-05 01:06:41 UTC (rev 290856)
+++ branches/safari-613-branch/Source/bmalloc/libpas/src/libpas/pas_redundant_local_allocator_node.c 2022-03-05 01:44:20 UTC (rev 290857)
@@ -41,7 +41,6 @@
"pas_redundant_local_allocator_node",
pas_object_allocation);
- pas_compact_atomic_thread_local_cache_layout_node_store(&result->next, NULL);
pas_compact_segregated_size_directory_ptr_store(&result->directory, directory);
result->allocator_index = 0;
Modified: branches/safari-613-branch/Source/bmalloc/libpas/src/libpas/pas_redundant_local_allocator_node.h (290856 => 290857)
--- branches/safari-613-branch/Source/bmalloc/libpas/src/libpas/pas_redundant_local_allocator_node.h 2022-03-05 01:06:41 UTC (rev 290856)
+++ branches/safari-613-branch/Source/bmalloc/libpas/src/libpas/pas_redundant_local_allocator_node.h 2022-03-05 01:44:20 UTC (rev 290857)
@@ -36,7 +36,6 @@
typedef struct pas_redundant_local_allocator_node pas_redundant_local_allocator_node;
struct pas_redundant_local_allocator_node {
- pas_compact_atomic_thread_local_cache_layout_node next;
pas_compact_segregated_size_directory_ptr directory;
pas_allocator_index allocator_index;
};
Modified: branches/safari-613-branch/Source/bmalloc/libpas/src/libpas/pas_root.c (290856 => 290857)
--- branches/safari-613-branch/Source/bmalloc/libpas/src/libpas/pas_root.c 2022-03-05 01:06:41 UTC (rev 290856)
+++ branches/safari-613-branch/Source/bmalloc/libpas/src/libpas/pas_root.c 2022-03-05 01:44:20 UTC (rev 290857)
@@ -95,7 +95,7 @@
&pas_large_heap_physical_page_sharing_cache_page_list;
root->payload_reservation_page_list = &pas_payload_reservation_page_list;
root->thread_local_cache_node_first = &pas_thread_local_cache_node_first;
- root->thread_local_cache_layout_first_node = &pas_thread_local_cache_layout_first_node;
+ root->thread_local_cache_layout_first_segment = &pas_thread_local_cache_layout_first_segment;
root->all_heaps_first_heap = &pas_all_heaps_first_heap;
root->num_static_heaps = 0;
Modified: branches/safari-613-branch/Source/bmalloc/libpas/src/libpas/pas_root.h (290856 => 290857)
--- branches/safari-613-branch/Source/bmalloc/libpas/src/libpas/pas_root.h 2022-03-05 01:06:41 UTC (rev 290856)
+++ branches/safari-613-branch/Source/bmalloc/libpas/src/libpas/pas_root.h 2022-03-05 01:44:20 UTC (rev 290857)
@@ -61,6 +61,7 @@
typedef struct pas_small_large_map_hashtable pas_small_large_map_hashtable;
typedef struct pas_small_large_map_hashtable_in_flux_stash pas_small_large_map_hashtable_in_flux_stash;
typedef struct pas_thread_local_cache_node pas_thread_local_cache_node;
+typedef struct pas_thread_local_cache_layout_segment pas_thread_local_cache_layout_segment;
typedef struct pas_tiny_large_map_hashtable pas_tiny_large_map_hashtable;
typedef struct pas_tiny_large_map_hashtable_in_flux_stash pas_tiny_large_map_hashtable_in_flux_stash;
typedef struct pas_tiny_large_map_second_level_hashtable_in_flux_stash pas_tiny_large_map_second_level_hashtable_in_flux_stash;
@@ -76,7 +77,7 @@
pas_enumerable_range_list* large_heap_physical_page_sharing_cache_page_list;
pas_enumerable_range_list* payload_reservation_page_list;
pas_thread_local_cache_node** thread_local_cache_node_first;
- pas_thread_local_cache_layout_node* thread_local_cache_layout_first_node;
+ pas_thread_local_cache_layout_segment** thread_local_cache_layout_first_segment;
pas_heap** all_heaps_first_heap;
pas_heap** static_heaps;
size_t num_static_heaps;
Modified: branches/safari-613-branch/Source/bmalloc/libpas/src/libpas/pas_segregated_heap.c (290856 => 290857)
--- branches/safari-613-branch/Source/bmalloc/libpas/src/libpas/pas_segregated_heap.c 2022-03-05 01:06:41 UTC (rev 290856)
+++ branches/safari-613-branch/Source/bmalloc/libpas/src/libpas/pas_segregated_heap.c 2022-03-05 01:44:20 UTC (rev 290857)
@@ -674,15 +674,10 @@
directory = pas_compact_atomic_segregated_size_directory_ptr_load(&directory->next_for_heap)) {
size_t index;
pas_allocator_index allocator_index;
- pas_segregated_size_directory_data* data;
size_t extra_index_for_allocator;
bool have_extra_index_for_allocator;
- data = ""
- if (data)
- allocator_index = data->allocator_index;
- else
- allocator_index = 0;
+ allocator_index = directory->allocator_index;
PAS_ASSERT(allocator_index != (pas_allocator_index)UINT_MAX);
@@ -736,14 +731,9 @@
(directory = size_directory_min_heap_take_min(&min_heap));
medium_tuple_index++) {
pas_segregated_heap_medium_directory_tuple tuple;
- pas_segregated_size_directory_data* data;
-
+
pas_compact_atomic_segregated_size_directory_ptr_store(&tuple.directory, directory);
- data = ""
- if (data)
- tuple.allocator_index = data->allocator_index;
- else
- tuple.allocator_index = 0;
+ tuple.allocator_index = directory->allocator_index;
PAS_ASSERT(tuple.allocator_index != (pas_allocator_index)UINT_MAX);
tuple.begin_index = pas_segregated_size_directory_min_index(directory);
PAS_ASSERT(tuple.begin_index);
@@ -865,8 +855,7 @@
if (verbose)
pas_log("index = %zu\n", index);
- allocator_index =
- pas_segregated_size_directory_data_ptr_load(&directory->data)->allocator_index;
+ allocator_index = directory->allocator_index;
PAS_ASSERT(allocator_index);
PAS_ASSERT((pas_allocator_index)allocator_index == allocator_index);
PAS_ASSERT(allocator_index < (unsigned)(pas_allocator_index)UINT_MAX);
Modified: branches/safari-613-branch/Source/bmalloc/libpas/src/libpas/pas_segregated_size_directory.c (290856 => 290857)
--- branches/safari-613-branch/Source/bmalloc/libpas/src/libpas/pas_segregated_size_directory.c 2022-03-05 01:06:41 UTC (rev 290856)
+++ branches/safari-613-branch/Source/bmalloc/libpas/src/libpas/pas_segregated_size_directory.c 2022-03-05 01:44:20 UTC (rev 290857)
@@ -117,6 +117,7 @@
pas_segregated_size_directory_encode_stuff(2 * PAS_NUM_BASELINE_ALLOCATORS, UINT_MAX);
result->view_cache_index = (pas_allocator_index)UINT_MAX;
+ result->allocator_index = 0;
pas_segregated_size_directory_data_ptr_store(&result->data, NULL);
@@ -204,8 +205,6 @@
data->offset_from_page_boundary_to_first_object = 0;
data->offset_from_page_boundary_to_end_of_last_object = 0;
data->full_num_non_empty_words = 0;
- data->allocator_index = 0;
- pas_compact_atomic_thread_local_cache_layout_node_store(&data->next_for_layout, NULL);
pas_fence();
@@ -233,24 +232,22 @@
void pas_segregated_size_directory_create_tlc_allocator(
pas_segregated_size_directory* directory)
{
- pas_segregated_size_directory_data* data;
-
pas_heap_lock_assert_held();
if (pas_segregated_page_config_kind_is_utility(directory->base.page_config_kind))
return;
- data = "" pas_lock_is_held);
+ pas_segregated_size_directory_ensure_data(directory, pas_lock_is_held);
- if (data->allocator_index) {
- PAS_ASSERT(data->allocator_index != (pas_allocator_index)UINT_MAX);
+ if (directory->allocator_index) {
+ PAS_ASSERT(directory->allocator_index != (pas_allocator_index)UINT_MAX);
return;
}
pas_thread_local_cache_layout_add(directory);
- PAS_ASSERT(data->allocator_index);
- PAS_ASSERT(data->allocator_index < (pas_allocator_index)UINT_MAX);
+ PAS_ASSERT(directory->allocator_index);
+ PAS_ASSERT(directory->allocator_index < (pas_allocator_index)UINT_MAX);
}
void pas_segregated_size_directory_create_tlc_view_cache(
@@ -947,7 +944,7 @@
tlc_result = pas_thread_local_cache_get_local_allocator_for_initialized_index(
pas_thread_local_cache_get(config),
- pas_segregated_size_directory_data_ptr_load_non_null(&directory->data)->allocator_index,
+ directory->allocator_index,
pas_lock_is_not_held);
PAS_ASSERT(tlc_result.did_succeed);
Modified: branches/safari-613-branch/Source/bmalloc/libpas/src/libpas/pas_segregated_size_directory.h (290856 => 290857)
--- branches/safari-613-branch/Source/bmalloc/libpas/src/libpas/pas_segregated_size_directory.h 2022-03-05 01:06:41 UTC (rev 290856)
+++ branches/safari-613-branch/Source/bmalloc/libpas/src/libpas/pas_segregated_size_directory.h 2022-03-05 01:44:20 UTC (rev 290857)
@@ -84,6 +84,9 @@
pas_allocator_index view_cache_index; /* This could be in the size_directory_data but we put it here
because it takes less space to put it here. */
+
+ pas_allocator_index allocator_index; /* This could be in the size_directory_data but we put it here
+ because it takes less space to put it here. */
pas_segregated_size_directory_data_ptr data;
@@ -95,12 +98,9 @@
unsigned offset_from_page_boundary_to_first_object; /* Cached to make refill fast. */
unsigned offset_from_page_boundary_to_end_of_last_object; /* Cached to make refill fast. */
- pas_allocator_index allocator_index;
-
uint8_t full_num_non_empty_words;
pas_compact_tagged_unsigned_ptr full_alloc_bits; /* Precomputed alloc bits in the case that a page is empty. */
- pas_compact_atomic_thread_local_cache_layout_node next_for_layout;
};
struct pas_extended_segregated_size_directory_data {
@@ -300,19 +300,13 @@
static inline bool pas_segregated_size_directory_has_tlc_allocator(
pas_segregated_size_directory* directory)
{
- pas_segregated_size_directory_data* data;
- data = ""
- return data && data->allocator_index;
+ return directory->allocator_index;
}
static inline pas_allocator_index pas_segregated_size_directory_get_tlc_allocator_index(
pas_segregated_size_directory* directory)
{
- pas_segregated_size_directory_data* data;
- data = ""
- if (data)
- return data->allocator_index;
- return 0;
+ return directory->allocator_index;
}
/* Call with heap lock held. */
Modified: branches/safari-613-branch/Source/bmalloc/libpas/src/libpas/pas_thread_local_cache.c (290856 => 290857)
--- branches/safari-613-branch/Source/bmalloc/libpas/src/libpas/pas_thread_local_cache.c 2022-03-05 01:06:41 UTC (rev 290856)
+++ branches/safari-613-branch/Source/bmalloc/libpas/src/libpas/pas_thread_local_cache.c 2022-03-05 01:44:20 UTC (rev 290857)
@@ -689,13 +689,17 @@
if (allocator_action != pas_allocator_scavenge_no_action) {
bool did_suspend;
+ pas_thread_local_cache_layout_segment* segment;
pas_thread_local_cache_layout_node layout_node;
+ uintptr_t node_index;
did_suspend = false;
PAS_UNUSED_PARAM(did_suspend);
- for (PAS_THREAD_LOCAL_CACHE_LAYOUT_EACH_ALLOCATOR(layout_node)) {
+ segment = pas_thread_local_cache_layout_first_segment;
+ node_index = 0;
+ for (PAS_THREAD_LOCAL_CACHE_LAYOUT_EACH_ALLOCATOR_WITH_SEGMENT_AND_INDEX(layout_node, segment, node_index)) {
pas_allocator_index allocator_index;
pas_local_allocator_scavenger_data* scavenger_data;
Modified: branches/safari-613-branch/Source/bmalloc/libpas/src/libpas/pas_thread_local_cache_layout.c (290856 => 290857)
--- branches/safari-613-branch/Source/bmalloc/libpas/src/libpas/pas_thread_local_cache_layout.c 2022-03-05 01:06:41 UTC (rev 290856)
+++ branches/safari-613-branch/Source/bmalloc/libpas/src/libpas/pas_thread_local_cache_layout.c 2022-03-05 01:44:20 UTC (rev 290857)
@@ -31,13 +31,15 @@
#include "pas_heap_lock.h"
#include "pas_large_utility_free_heap.h"
-#include "pas_local_view_cache_node.h"
#include "pas_redundant_local_allocator_node.h"
#include "pas_segregated_size_directory_inlines.h"
+#include "pas_utility_heap.h"
#include "pas_utils.h"
-pas_thread_local_cache_layout_node pas_thread_local_cache_layout_first_node = NULL;
-pas_thread_local_cache_layout_node pas_thread_local_cache_layout_last_node = NULL;
+pas_thread_local_cache_layout_segment* pas_thread_local_cache_layout_first_segment = NULL;
+static pas_thread_local_cache_layout_segment* pas_thread_local_cache_layout_last_segment = NULL;
+static unsigned pas_thread_local_cache_layout_last_segment_size = 0;
+
pas_allocator_index pas_thread_local_cache_layout_next_allocator_index =
PAS_LOCAL_ALLOCATOR_UNSELECTED_NUM_INDICES;
@@ -62,8 +64,6 @@
break;
}
- PAS_ASSERT(!pas_thread_local_cache_layout_node_get_next(node));
-
result = pas_thread_local_cache_layout_next_allocator_index;
PAS_ASSERT(result < (pas_allocator_index)UINT_MAX);
@@ -81,18 +81,30 @@
&pas_thread_local_cache_layout_next_allocator_index);
PAS_ASSERT(!did_overflow);
- pas_fence();
-
- if (!pas_thread_local_cache_layout_first_node) {
- PAS_ASSERT(!pas_thread_local_cache_layout_last_node);
- PAS_ASSERT(result == PAS_LOCAL_ALLOCATOR_UNSELECTED_NUM_INDICES);
- pas_thread_local_cache_layout_first_node = node;
- pas_thread_local_cache_layout_last_node = node;
+ if (!pas_thread_local_cache_layout_last_segment || pas_thread_local_cache_layout_last_segment_size == PAS_THREAD_LOCAL_CACHE_LAYOUT_SEGMENT_SIZE) {
+ pas_thread_local_cache_layout_segment* segment;
+
+ segment = (pas_thread_local_cache_layout_segment*)pas_utility_heap_allocate(sizeof(pas_thread_local_cache_layout_segment), "pas_thread_local_cache_layout_segment");
+ pas_zero_memory(segment, sizeof(pas_thread_local_cache_layout_segment));
+
+ pas_compact_atomic_thread_local_cache_layout_node_store(&segment->nodes[0], node);
+ pas_thread_local_cache_layout_last_segment_size = 1;
+
+ pas_fence();
+ if (!pas_thread_local_cache_layout_last_segment) {
+ PAS_ASSERT(!pas_thread_local_cache_layout_first_segment);
+ PAS_ASSERT(result == PAS_LOCAL_ALLOCATOR_UNSELECTED_NUM_INDICES);
+ pas_thread_local_cache_layout_first_segment = segment;
+ pas_thread_local_cache_layout_last_segment = segment;
+ } else {
+ pas_thread_local_cache_layout_last_segment->next = segment;
+ pas_thread_local_cache_layout_last_segment = segment;
+ }
} else {
- PAS_ASSERT(pas_thread_local_cache_layout_last_node);
+ PAS_ASSERT(pas_thread_local_cache_layout_last_segment);
PAS_ASSERT(result > PAS_LOCAL_ALLOCATOR_UNSELECTED_NUM_INDICES);
- pas_thread_local_cache_layout_node_set_next(pas_thread_local_cache_layout_last_node, node);
- pas_thread_local_cache_layout_last_node = node;
+ pas_fence();
+ pas_compact_atomic_thread_local_cache_layout_node_store(&pas_thread_local_cache_layout_last_segment->nodes[pas_thread_local_cache_layout_last_segment_size++], node);
}
return result;
@@ -127,11 +139,16 @@
pas_allocator_index pas_thread_local_cache_layout_add_view_cache(
pas_segregated_size_directory* directory)
{
- pas_local_view_cache_node* cache_node;
+ return pas_thread_local_cache_layout_add_node(pas_wrap_local_view_cache_node(directory));
+}
- cache_node = pas_local_view_cache_node_create(directory);
-
- return pas_thread_local_cache_layout_add_node(pas_wrap_local_view_cache_node(cache_node));
+pas_thread_local_cache_layout_node pas_thread_local_cache_layout_get_last_node()
+{
+ pas_heap_lock_assert_held();
+ if (!pas_thread_local_cache_layout_last_segment)
+ return NULL;
+ PAS_ASSERT(pas_thread_local_cache_layout_last_segment_size);
+ return pas_thread_local_cache_layout_segment_get_node(pas_thread_local_cache_layout_last_segment, pas_thread_local_cache_layout_last_segment_size - 1);
}
#endif /* LIBPAS_ENABLED */
Modified: branches/safari-613-branch/Source/bmalloc/libpas/src/libpas/pas_thread_local_cache_layout.h (290856 => 290857)
--- branches/safari-613-branch/Source/bmalloc/libpas/src/libpas/pas_thread_local_cache_layout.h 2022-03-05 01:06:41 UTC (rev 290856)
+++ branches/safari-613-branch/Source/bmalloc/libpas/src/libpas/pas_thread_local_cache_layout.h 2022-03-05 01:44:20 UTC (rev 290857)
@@ -27,6 +27,7 @@
#define PAS_THREAD_LOCAL_CACHE_LAYOUT_H
#include "pas_allocator_index.h"
+#include "pas_compact_atomic_thread_local_cache_layout_node.h"
#include "pas_config.h"
#include "pas_thread_local_cache_layout_node.h"
#include "pas_utils.h"
@@ -33,9 +34,16 @@
PAS_BEGIN_EXTERN_C;
-PAS_API extern pas_thread_local_cache_layout_node pas_thread_local_cache_layout_first_node;
-PAS_API extern pas_thread_local_cache_layout_node pas_thread_local_cache_layout_last_node;
+#define PAS_THREAD_LOCAL_CACHE_LAYOUT_SEGMENT_SIZE 257
+struct pas_thread_local_cache_layout_segment {
+ pas_compact_atomic_thread_local_cache_layout_node nodes[PAS_THREAD_LOCAL_CACHE_LAYOUT_SEGMENT_SIZE];
+ pas_compact_atomic_thread_local_cache_layout_node sentinel;
+ struct pas_thread_local_cache_layout_segment* next;
+};
+typedef struct pas_thread_local_cache_layout_segment pas_thread_local_cache_layout_segment;
+PAS_API extern pas_thread_local_cache_layout_segment* pas_thread_local_cache_layout_first_segment;
+
/* Clients can use this to force the next call to add to go to this index. */
PAS_API extern pas_allocator_index pas_thread_local_cache_layout_next_allocator_index;
@@ -51,11 +59,39 @@
PAS_API pas_allocator_index pas_thread_local_cache_layout_add_view_cache(
pas_segregated_size_directory* directory);
+PAS_API pas_thread_local_cache_layout_node pas_thread_local_cache_layout_get_last_node(void);
+
+static PAS_ALWAYS_INLINE pas_thread_local_cache_layout_node pas_thread_local_cache_layout_segment_get_node(pas_thread_local_cache_layout_segment* segment, uintptr_t index)
+{
+ if (!segment)
+ return NULL;
+ return pas_compact_atomic_thread_local_cache_layout_node_load(&segment->nodes[index]);
+}
+
+static PAS_ALWAYS_INLINE pas_thread_local_cache_layout_node pas_thread_local_cache_layout_segment_next_node(pas_thread_local_cache_layout_segment** segment, uintptr_t* index)
+{
+ PAS_TESTING_ASSERT(segment);
+ PAS_TESTING_ASSERT(*segment);
+ uintptr_t next_index = *index + 1;
+ pas_thread_local_cache_layout_node next = pas_thread_local_cache_layout_segment_get_node(*segment, next_index);
+ if (next) {
+ *index = next_index;
+ return next;
+ }
+ *segment = (*segment)->next;
+ *index = 0;
+ return pas_thread_local_cache_layout_segment_get_node(*segment, 0);
+}
+
#define PAS_THREAD_LOCAL_CACHE_LAYOUT_EACH_ALLOCATOR(node) \
- node = pas_thread_local_cache_layout_first_node; \
+ uintptr_t internal_node_index = 0, internal_segment = (uintptr_t)pas_thread_local_cache_layout_first_segment, internal_node = (uintptr_t)pas_thread_local_cache_layout_segment_get_node((pas_thread_local_cache_layout_segment*)internal_segment, internal_node_index); \
+ (node = ((pas_thread_local_cache_layout_node)internal_node)); \
+ internal_node = (uintptr_t)pas_thread_local_cache_layout_segment_next_node((pas_thread_local_cache_layout_segment**)&internal_segment, &internal_node_index)
+
+#define PAS_THREAD_LOCAL_CACHE_LAYOUT_EACH_ALLOCATOR_WITH_SEGMENT_AND_INDEX(node, segment, index) \
+ node = pas_thread_local_cache_layout_segment_get_node(segment, index); \
node; \
- node = pas_thread_local_cache_layout_node_get_next(node)
-
+ node = pas_thread_local_cache_layout_segment_next_node(&segment, &index)
PAS_END_EXTERN_C;
Modified: branches/safari-613-branch/Source/bmalloc/libpas/src/libpas/pas_thread_local_cache_layout_node.c (290856 => 290857)
--- branches/safari-613-branch/Source/bmalloc/libpas/src/libpas/pas_thread_local_cache_layout_node.c 2022-03-05 01:06:41 UTC (rev 290856)
+++ branches/safari-613-branch/Source/bmalloc/libpas/src/libpas/pas_thread_local_cache_layout_node.c 2022-03-05 01:44:20 UTC (rev 290857)
@@ -30,7 +30,6 @@
#include "pas_thread_local_cache_layout_node.h"
#include "pas_local_view_cache.h"
-#include "pas_local_view_cache_node.h"
#include "pas_redundant_local_allocator_node.h"
#include "pas_segregated_size_directory_inlines.h"
@@ -45,8 +44,7 @@
&pas_unwrap_redundant_local_allocator_node(node)->directory);
}
- return pas_compact_segregated_size_directory_ptr_load_non_null(
- &pas_unwrap_local_view_cache_node(node)->directory);
+ return pas_unwrap_local_view_cache_node(node);
}
pas_allocator_index
@@ -67,15 +65,13 @@
allocator_index_ptr(pas_thread_local_cache_layout_node node)
{
if (pas_is_wrapped_segregated_size_directory(node)) {
- return &pas_segregated_size_directory_data_ptr_load(
- &pas_unwrap_segregated_size_directory(node)->data)->allocator_index;
+ return &pas_unwrap_segregated_size_directory(node)->allocator_index;
}
if (pas_is_wrapped_redundant_local_allocator_node(node))
return &pas_unwrap_redundant_local_allocator_node(node)->allocator_index;
- return &pas_compact_segregated_size_directory_ptr_load_non_null(
- &pas_unwrap_local_view_cache_node(node)->directory)->view_cache_index;
+ return &pas_unwrap_local_view_cache_node(node)->view_cache_index;
}
pas_allocator_index
@@ -105,32 +101,6 @@
*allocator_index_ptr(node) = index;
}
-static pas_compact_atomic_thread_local_cache_layout_node*
-next_ptr(pas_thread_local_cache_layout_node node)
-{
- if (pas_is_wrapped_segregated_size_directory(node)) {
- return &pas_segregated_size_directory_data_ptr_load(
- &pas_unwrap_segregated_size_directory(node)->data)->next_for_layout;
- }
-
- if (pas_is_wrapped_redundant_local_allocator_node(node))
- return &pas_unwrap_redundant_local_allocator_node(node)->next;
-
- return &pas_unwrap_local_view_cache_node(node)->next;
-}
-
-pas_thread_local_cache_layout_node
-pas_thread_local_cache_layout_node_get_next(pas_thread_local_cache_layout_node node)
-{
- return pas_compact_atomic_thread_local_cache_layout_node_load(next_ptr(node));
-}
-
-void pas_thread_local_cache_layout_node_set_next(pas_thread_local_cache_layout_node node,
- pas_thread_local_cache_layout_node next_node)
-{
- pas_compact_atomic_thread_local_cache_layout_node_store(next_ptr(node), next_node);
-}
-
void pas_thread_local_cache_layout_node_construct(pas_thread_local_cache_layout_node node,
pas_thread_local_cache* cache)
{
Modified: branches/safari-613-branch/Source/bmalloc/libpas/src/libpas/pas_thread_local_cache_layout_node.h (290856 => 290857)
--- branches/safari-613-branch/Source/bmalloc/libpas/src/libpas/pas_thread_local_cache_layout_node.h 2022-03-05 01:06:41 UTC (rev 290856)
+++ branches/safari-613-branch/Source/bmalloc/libpas/src/libpas/pas_thread_local_cache_layout_node.h 2022-03-05 01:44:20 UTC (rev 290857)
@@ -33,12 +33,10 @@
PAS_BEGIN_EXTERN_C;
-struct pas_local_view_cache_node;
struct pas_redundant_local_allocator_node;
struct pas_segregated_size_directory;
struct pas_thread_local_cache;
struct pas_thread_local_cache_layout_node_opaque;
-typedef struct pas_local_view_cache_node pas_local_view_cache_node;
typedef struct pas_redundant_local_allocator_node pas_redundant_local_allocator_node;
typedef struct pas_segregated_size_directory pas_segregated_size_directory;
typedef struct pas_thread_local_cache pas_thread_local_cache;
@@ -87,10 +85,10 @@
}
static inline pas_thread_local_cache_layout_node
-pas_wrap_local_view_cache_node(pas_local_view_cache_node* node)
+pas_wrap_local_view_cache_node(pas_segregated_size_directory* directory)
{
return pas_thread_local_cache_layout_node_create(
- pas_thread_local_cache_layout_local_view_cache_node_kind, node);
+ pas_thread_local_cache_layout_local_view_cache_node_kind, directory);
}
static inline bool
@@ -128,11 +126,11 @@
return (pas_redundant_local_allocator_node*)pas_thread_local_cache_layout_node_get_ptr(node);
}
-static inline pas_local_view_cache_node*
+static inline pas_segregated_size_directory*
pas_unwrap_local_view_cache_node(pas_thread_local_cache_layout_node node)
{
PAS_ASSERT(pas_is_wrapped_local_view_cache_node(node));
- return (pas_local_view_cache_node*)pas_thread_local_cache_layout_node_get_ptr(node);
+ return (pas_segregated_size_directory*)pas_thread_local_cache_layout_node_get_ptr(node);
}
PAS_API pas_segregated_size_directory*
@@ -169,13 +167,6 @@
pas_thread_local_cache_layout_node_set_allocator_index(pas_thread_local_cache_layout_node node,
pas_allocator_index index);
-PAS_API pas_thread_local_cache_layout_node
-pas_thread_local_cache_layout_node_get_next(pas_thread_local_cache_layout_node node);
-
-PAS_API void
-pas_thread_local_cache_layout_node_set_next(pas_thread_local_cache_layout_node node,
- pas_thread_local_cache_layout_node next_node);
-
PAS_API void pas_thread_local_cache_layout_node_construct(pas_thread_local_cache_layout_node node,
pas_thread_local_cache* cache);
