Diff
Modified: branches/safari-613.1.17.1-branch/Source/bmalloc/CMakeLists.txt (290157 => 290158)
--- branches/safari-613.1.17.1-branch/Source/bmalloc/CMakeLists.txt 2022-02-18 21:12:51 UTC (rev 290157)
+++ branches/safari-613.1.17.1-branch/Source/bmalloc/CMakeLists.txt 2022-02-18 21:13:01 UTC (rev 290158)
@@ -126,6 +126,7 @@
libpas/src/libpas/pas_local_allocator.c
libpas/src/libpas/pas_local_allocator_scavenger_data.c
libpas/src/libpas/pas_local_view_cache.c
+ libpas/src/libpas/pas_local_view_cache_node.c
libpas/src/libpas/pas_lock.c
libpas/src/libpas/pas_lock_free_read_ptr_ptr_hashtable.c
libpas/src/libpas/pas_log.c
@@ -504,6 +505,7 @@
libpas/src/libpas/pas_local_allocator_result.h
libpas/src/libpas/pas_local_allocator_scavenger_data.h
libpas/src/libpas/pas_local_view_cache.h
+ libpas/src/libpas/pas_local_view_cache_node.h
libpas/src/libpas/pas_lock_free_read_ptr_ptr_hashtable.h
libpas/src/libpas/pas_lock.h
libpas/src/libpas/pas_log.h
Modified: branches/safari-613.1.17.1-branch/Source/bmalloc/ChangeLog (290157 => 290158)
--- branches/safari-613.1.17.1-branch/Source/bmalloc/ChangeLog 2022-02-18 21:12:51 UTC (rev 290157)
+++ branches/safari-613.1.17.1-branch/Source/bmalloc/ChangeLog 2022-02-18 21:13:01 UTC (rev 290158)
@@ -1,85 +1,3 @@
-2022-02-18 Russell Epstein <repst...@apple.com>
-
- Apply patch. rdar://problem/89071706
-
- 2022-02-18 Yusuke Suzuki <ysuz...@apple.com>
-
- Unreviewed, fix heap enumeration
- https://bugs.webkit.org/show_bug.cgi?id=236662
-
- * libpas/src/libpas/pas_enumerate_segregated_heaps.c:
- (pas_enumerate_segregated_heaps):
-
-2022-02-16 Yusuke Suzuki <ysuz...@apple.com>
-
- [libpas] Use segmented vector for TLC layout list
- https://bugs.webkit.org/show_bug.cgi?id=236662
-
- Reviewed by Saam Barati.
-
- We add segmented vector for pas_thread_local_cache_layout_node iteration instead of using
- linked-list of that to accelerate iteration performance. It does not affect on Speedometer2,
- JetStream2, PLT5 etc., but we do this iteration frequently in scavenger thread, and reducing
- time of this makes it efficient. While speeding up scavenger does not matter for throughput
- of non scavenger threads since libpas works well without taking a lock in most times which
- is blocked by scavenger, making scavenger faster is good for energy efficiency.
-
- 1. Use segmented vector for (concurrently read) pas_thread_local_cache_layout_node list.
- We still keep nice for-loop abstraction for list of pas_thread_local_cache_layout_node.
- 2. Remove pas_local_view_cache_node since we no longer need it. It was introduced just to
- make it linked-list node for pas_thread_local_cache_layout_node. But now, segmented vector
- is holding it separately.
- 3. Move pas_segregated_size_directory_data's allocator_index to pas_segregated_size_directory.
- Because of (1) change, we can make pas_segregated_size_directory linked-list-node, which
- saves 4 bytes. So we can move pas_segregated_size_directory_data's allocator_index to
- pas_segregated_size_directory since it is heavily accessed from the scavenger.
-
- * CMakeLists.txt:
- * bmalloc.xcodeproj/project.pbxproj:
- * libpas/libpas.xcodeproj/project.pbxproj:
- * libpas/src/libpas/pas_designated_intrinsic_heap.c:
- (set_up_range):
- * libpas/src/libpas/pas_enumerate_segregated_heaps.c:
- (pas_enumerate_segregated_heaps):
- * libpas/src/libpas/pas_local_view_cache_node.c: Removed.
- * libpas/src/libpas/pas_local_view_cache_node.h: Removed.
- * libpas/src/libpas/pas_redundant_local_allocator_node.c:
- (pas_redundant_local_allocator_node_create):
- * libpas/src/libpas/pas_redundant_local_allocator_node.h:
- * libpas/src/libpas/pas_root.c:
- (pas_root_construct):
- * libpas/src/libpas/pas_root.h:
- * libpas/src/libpas/pas_segregated_heap.c:
- (recompute_size_lookup):
- (pas_segregated_heap_ensure_allocator_index):
- * libpas/src/libpas/pas_segregated_size_directory.c:
- (pas_segregated_size_directory_create):
- (pas_segregated_size_directory_ensure_data):
- (pas_segregated_size_directory_create_tlc_allocator):
- (pas_segregated_size_directory_get_allocator_from_tlc):
- * libpas/src/libpas/pas_segregated_size_directory.h:
- (pas_segregated_size_directory_has_tlc_allocator):
- (pas_segregated_size_directory_get_tlc_allocator_index):
- * libpas/src/libpas/pas_thread_local_cache.c:
- (decommit_allocator_range):
- (pas_thread_local_cache_for_all):
- * libpas/src/libpas/pas_thread_local_cache_layout.c:
- (pas_thread_local_cache_layout_add_node):
- (pas_thread_local_cache_layout_add_view_cache):
- (pas_thread_local_cache_layout_get_last_node):
- * libpas/src/libpas/pas_thread_local_cache_layout.h:
- (pas_thread_local_cache_layout_segment_get_node):
- (pas_thread_local_cache_layout_segment_next_node):
- * libpas/src/libpas/pas_thread_local_cache_layout_node.c:
- (pas_thread_local_cache_layout_node_get_directory):
- (allocator_index_ptr):
- (next_ptr): Deleted.
- (pas_thread_local_cache_layout_node_get_next): Deleted.
- (pas_thread_local_cache_layout_node_set_next): Deleted.
- * libpas/src/libpas/pas_thread_local_cache_layout_node.h:
- (pas_wrap_local_view_cache_node):
- (pas_unwrap_local_view_cache_node):
-
2022-02-07 Russell Epstein <repst...@apple.com>
Cherry-pick r289144. rdar://problem/88585404
Modified: branches/safari-613.1.17.1-branch/Source/bmalloc/bmalloc.xcodeproj/project.pbxproj (290157 => 290158)
--- branches/safari-613.1.17.1-branch/Source/bmalloc/bmalloc.xcodeproj/project.pbxproj 2022-02-18 21:12:51 UTC (rev 290157)
+++ branches/safari-613.1.17.1-branch/Source/bmalloc/bmalloc.xcodeproj/project.pbxproj 2022-02-18 21:13:01 UTC (rev 290158)
@@ -88,9 +88,11 @@
0F7C92E926E57F75006AF012 /* pas_heap_runtime_config.c in Sources */ = {isa = PBXBuildFile; fileRef = 0F7C92D926E57F75006AF012 /* pas_heap_runtime_config.c */; };
0F7C92EA26E57F75006AF012 /* pas_local_allocator_scavenger_data.c in Sources */ = {isa = PBXBuildFile; fileRef = 0F7C92DA26E57F75006AF012 /* pas_local_allocator_scavenger_data.c */; };
0F7C92EB26E57F75006AF012 /* pas_compact_segregated_size_directory_ptr.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F7C92DB26E57F75006AF012 /* pas_compact_segregated_size_directory_ptr.h */; settings = {ATTRIBUTES = (Private, ); }; };
+ 0F7C92EC26E57F75006AF012 /* pas_local_view_cache_node.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F7C92DC26E57F75006AF012 /* pas_local_view_cache_node.h */; };
0F7C92ED26E57F75006AF012 /* pas_segregated_size_directory_creation_mode.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F7C92DD26E57F75006AF012 /* pas_segregated_size_directory_creation_mode.h */; settings = {ATTRIBUTES = (Private, ); }; };
0F7C92EE26E57F75006AF012 /* pas_local_view_cache.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F7C92DE26E57F75006AF012 /* pas_local_view_cache.h */; settings = {ATTRIBUTES = (Private, ); }; };
0F7C92F026E57F75006AF012 /* pas_local_allocator_kind.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F7C92E026E57F75006AF012 /* pas_local_allocator_kind.h */; settings = {ATTRIBUTES = (Private, ); }; };
+ 0F7C92F126E57F75006AF012 /* pas_local_view_cache_node.c in Sources */ = {isa = PBXBuildFile; fileRef = 0F7C92E126E57F75006AF012 /* pas_local_view_cache_node.c */; };
0F7EB8231F9541B000F1ABCB /* EligibilityResult.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F7EB7FC1F9541AD00F1ABCB /* EligibilityResult.h */; settings = {ATTRIBUTES = (Private, ); }; };
0F7EB8241F9541B000F1ABCB /* IsoHeapImplInlines.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F7EB7FD1F9541AD00F1ABCB /* IsoHeapImplInlines.h */; settings = {ATTRIBUTES = (Private, ); }; };
0F7EB8251F9541B000F1ABCB /* DeferredTriggerInlines.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F7EB7FE1F9541AD00F1ABCB /* DeferredTriggerInlines.h */; settings = {ATTRIBUTES = (Private, ); }; };
@@ -745,9 +747,11 @@
0F7C92D926E57F75006AF012 /* pas_heap_runtime_config.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = pas_heap_runtime_config.c; path = libpas/src/libpas/pas_heap_runtime_config.c; sourceTree = "<group>"; };
0F7C92DA26E57F75006AF012 /* pas_local_allocator_scavenger_data.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = pas_local_allocator_scavenger_data.c; path = libpas/src/libpas/pas_local_allocator_scavenger_data.c; sourceTree = "<group>"; };
0F7C92DB26E57F75006AF012 /* pas_compact_segregated_size_directory_ptr.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = pas_compact_segregated_size_directory_ptr.h; path = libpas/src/libpas/pas_compact_segregated_size_directory_ptr.h; sourceTree = "<group>"; };
+ 0F7C92DC26E57F75006AF012 /* pas_local_view_cache_node.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = pas_local_view_cache_node.h; path = libpas/src/libpas/pas_local_view_cache_node.h; sourceTree = "<group>"; };
0F7C92DD26E57F75006AF012 /* pas_segregated_size_directory_creation_mode.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = pas_segregated_size_directory_creation_mode.h; path = libpas/src/libpas/pas_segregated_size_directory_creation_mode.h; sourceTree = "<group>"; };
0F7C92DE26E57F75006AF012 /* pas_local_view_cache.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = pas_local_view_cache.h; path = libpas/src/libpas/pas_local_view_cache.h; sourceTree = "<group>"; };
0F7C92E026E57F75006AF012 /* pas_local_allocator_kind.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = pas_local_allocator_kind.h; path = libpas/src/libpas/pas_local_allocator_kind.h; sourceTree = "<group>"; };
+ 0F7C92E126E57F75006AF012 /* pas_local_view_cache_node.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = pas_local_view_cache_node.c; path = libpas/src/libpas/pas_local_view_cache_node.c; sourceTree = "<group>"; };
0F7EB7FC1F9541AD00F1ABCB /* EligibilityResult.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = EligibilityResult.h; path = bmalloc/EligibilityResult.h; sourceTree = SOURCE_ROOT; };
0F7EB7FD1F9541AD00F1ABCB /* IsoHeapImplInlines.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = IsoHeapImplInlines.h; path = bmalloc/IsoHeapImplInlines.h; sourceTree = SOURCE_ROOT; };
0F7EB7FE1F9541AD00F1ABCB /* DeferredTriggerInlines.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = DeferredTriggerInlines.h; path = bmalloc/DeferredTriggerInlines.h; sourceTree = SOURCE_ROOT; };
@@ -1726,6 +1730,8 @@
0F7C92D426E57F74006AF012 /* pas_local_allocator_scavenger_data.h */,
0F7C92D626E57F75006AF012 /* pas_local_view_cache.c */,
0F7C92DE26E57F75006AF012 /* pas_local_view_cache.h */,
+ 0F7C92E126E57F75006AF012 /* pas_local_view_cache_node.c */,
+ 0F7C92DC26E57F75006AF012 /* pas_local_view_cache_node.h */,
0FC40BD9245243A400876DA0 /* pas_lock.c */,
0FC40A8B2451498C00876DA0 /* pas_lock.h */,
0FC40A562451498800876DA0 /* pas_lock_free_read_ptr_ptr_hashtable.c */,
@@ -2410,6 +2416,7 @@
0FC40B2E2451499400876DA0 /* pas_local_allocator_result.h in Headers */,
0F7C92E426E57F75006AF012 /* pas_local_allocator_scavenger_data.h in Headers */,
0F7C92EE26E57F75006AF012 /* pas_local_view_cache.h in Headers */,
+ 0F7C92EC26E57F75006AF012 /* pas_local_view_cache_node.h in Headers */,
0FC40B672451499400876DA0 /* pas_lock.h in Headers */,
0FC40B7F2451499400876DA0 /* pas_lock_free_read_ptr_ptr_hashtable.h in Headers */,
0FC40B9C2451499400876DA0 /* pas_log.h in Headers */,
@@ -2769,6 +2776,7 @@
0FC40B072451499400876DA0 /* pas_local_allocator.c in Sources */,
0F7C92EA26E57F75006AF012 /* pas_local_allocator_scavenger_data.c in Sources */,
0F7C92E626E57F75006AF012 /* pas_local_view_cache.c in Sources */,
+ 0F7C92F126E57F75006AF012 /* pas_local_view_cache_node.c in Sources */,
0FC40BDA245243A400876DA0 /* pas_lock.c in Sources */,
0FC40B342451499400876DA0 /* pas_lock_free_read_ptr_ptr_hashtable.c in Sources */,
0FC40BB22451499400876DA0 /* pas_log.c in Sources */,
Modified: branches/safari-613.1.17.1-branch/Source/bmalloc/libpas/libpas.xcodeproj/project.pbxproj (290157 => 290158)
--- branches/safari-613.1.17.1-branch/Source/bmalloc/libpas/libpas.xcodeproj/project.pbxproj 2022-02-18 21:12:51 UTC (rev 290157)
+++ branches/safari-613.1.17.1-branch/Source/bmalloc/libpas/libpas.xcodeproj/project.pbxproj 2022-02-18 21:13:01 UTC (rev 290158)
@@ -53,8 +53,10 @@
0F148713269B7518006887A9 /* pas_compact_segregated_exclusive_view_ptr.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F14870D269B7518006887A9 /* pas_compact_segregated_exclusive_view_ptr.h */; };
0F148714269B7518006887A9 /* pas_heap_runtime_config.c in Sources */ = {isa = PBXBuildFile; fileRef = 0F14870E269B7518006887A9 /* pas_heap_runtime_config.c */; };
0F148715269B7518006887A9 /* pas_compact_segregated_size_directory_ptr.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F14870F269B7518006887A9 /* pas_compact_segregated_size_directory_ptr.h */; };
+ 0F14871F269F5FCA006887A9 /* pas_local_view_cache_node.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F14871A269F5FCA006887A9 /* pas_local_view_cache_node.h */; };
0F148720269F5FCA006887A9 /* pas_local_view_cache.c in Sources */ = {isa = PBXBuildFile; fileRef = 0F14871B269F5FCA006887A9 /* pas_local_view_cache.c */; };
0F148721269F5FCA006887A9 /* pas_segregated_size_directory_creation_mode.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F14871C269F5FCA006887A9 /* pas_segregated_size_directory_creation_mode.h */; };
+ 0F148722269F5FCA006887A9 /* pas_local_view_cache_node.c in Sources */ = {isa = PBXBuildFile; fileRef = 0F14871D269F5FCA006887A9 /* pas_local_view_cache_node.c */; };
0F148723269F5FCA006887A9 /* pas_thread_local_cache_layout_node_kind.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F14871E269F5FCA006887A9 /* pas_thread_local_cache_layout_node_kind.h */; };
0F1518FD23441E3200832B5C /* pas_has_object.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F1518FC23441E3200832B5C /* pas_has_object.h */; };
0F18E63E2628BFB700F6903D /* pas_shared_page_directory_by_size.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F18E63C2628BFB600F6903D /* pas_shared_page_directory_by_size.h */; };
@@ -575,8 +577,6 @@
2C971CFF278E136300C9E129 /* pas_lenient_compact_ptr_inlines.h in Headers */ = {isa = PBXBuildFile; fileRef = 2C971CFD278E136300C9E129 /* pas_lenient_compact_ptr_inlines.h */; };
2CB9B153278F6C85003A8C1B /* LotsOfHeapsAndThreads.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 2CB9B152278F6C85003A8C1B /* LotsOfHeapsAndThreads.cpp */; };
2CE2AE35275A953E00D02BBC /* BitfitTests.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 2CE2AE34275A953E00D02BBC /* BitfitTests.cpp */; };
- E3A7DC8F27BEF8EA004E76E9 /* pas_thread_suspend_lock.h in Headers */ = {isa = PBXBuildFile; fileRef = E3A7DC8D27BEF8EA004E76E9 /* pas_thread_suspend_lock.h */; };
- E3A7DC9027BEF8EA004E76E9 /* pas_thread_suspend_lock.c in Sources */ = {isa = PBXBuildFile; fileRef = E3A7DC8E27BEF8EA004E76E9 /* pas_thread_suspend_lock.c */; };
/* End PBXBuildFile section */
/* Begin PBXContainerItemProxy section */
@@ -739,8 +739,10 @@
0F14870D269B7518006887A9 /* pas_compact_segregated_exclusive_view_ptr.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = pas_compact_segregated_exclusive_view_ptr.h; sourceTree = "<group>"; };
0F14870E269B7518006887A9 /* pas_heap_runtime_config.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = pas_heap_runtime_config.c; sourceTree = "<group>"; };
0F14870F269B7518006887A9 /* pas_compact_segregated_size_directory_ptr.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = pas_compact_segregated_size_directory_ptr.h; sourceTree = "<group>"; };
+ 0F14871A269F5FCA006887A9 /* pas_local_view_cache_node.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = pas_local_view_cache_node.h; sourceTree = "<group>"; };
0F14871B269F5FCA006887A9 /* pas_local_view_cache.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = pas_local_view_cache.c; sourceTree = "<group>"; };
0F14871C269F5FCA006887A9 /* pas_segregated_size_directory_creation_mode.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = pas_segregated_size_directory_creation_mode.h; sourceTree = "<group>"; };
+ 0F14871D269F5FCA006887A9 /* pas_local_view_cache_node.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = pas_local_view_cache_node.c; sourceTree = "<group>"; };
0F14871E269F5FCA006887A9 /* pas_thread_local_cache_layout_node_kind.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = pas_thread_local_cache_layout_node_kind.h; sourceTree = "<group>"; };
0F1518FC23441E3200832B5C /* pas_has_object.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = pas_has_object.h; sourceTree = "<group>"; };
0F18E63C2628BFB600F6903D /* pas_shared_page_directory_by_size.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = pas_shared_page_directory_by_size.h; sourceTree = "<group>"; };
@@ -1268,8 +1270,6 @@
2C971CFD278E136300C9E129 /* pas_lenient_compact_ptr_inlines.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = pas_lenient_compact_ptr_inlines.h; sourceTree = "<group>"; };
2CB9B152278F6C85003A8C1B /* LotsOfHeapsAndThreads.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = LotsOfHeapsAndThreads.cpp; sourceTree = "<group>"; };
2CE2AE34275A953E00D02BBC /* BitfitTests.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = BitfitTests.cpp; sourceTree = "<group>"; };
- E3A7DC8D27BEF8EA004E76E9 /* pas_thread_suspend_lock.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = pas_thread_suspend_lock.h; sourceTree = "<group>"; };
- E3A7DC8E27BEF8EA004E76E9 /* pas_thread_suspend_lock.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = pas_thread_suspend_lock.c; sourceTree = "<group>"; };
/* End PBXFileReference section */
/* Begin PBXFrameworksBuildPhase section */
@@ -1708,6 +1708,8 @@
0F45D5A826AB731400C35D93 /* pas_local_allocator_scavenger_data.h */,
0FDBB07122B875A6006BA5FC /* pas_local_allocator.c */,
0FDBB07222B875A6006BA5FC /* pas_local_allocator.h */,
+ 0F14871D269F5FCA006887A9 /* pas_local_view_cache_node.c */,
+ 0F14871A269F5FCA006887A9 /* pas_local_view_cache_node.h */,
0F14871B269F5FCA006887A9 /* pas_local_view_cache.c */,
0F14870C269B7518006887A9 /* pas_local_view_cache.h */,
0FEA45BA236CD5AC00B5A375 /* pas_lock_free_read_ptr_ptr_hashtable.c */,
@@ -1870,8 +1872,6 @@
0F7F51BD23159EC0007552AE /* pas_thread_local_cache_node.h */,
0FC681C9210F7C9F003C6A13 /* pas_thread_local_cache.c */,
0FC681C4210F7C9E003C6A13 /* pas_thread_local_cache.h */,
- E3A7DC8E27BEF8EA004E76E9 /* pas_thread_suspend_lock.c */,
- E3A7DC8D27BEF8EA004E76E9 /* pas_thread_suspend_lock.h */,
0F6D547123C573E000F40DBB /* pas_tiny_large_map_entry.h */,
0F31A66923E8E594002C0CA3 /* pas_tree_direction.h */,
0F8A800C25CF64C900790B4A /* pas_tri_state.h */,
@@ -2214,6 +2214,7 @@
0FE7EE0E22960142004F4166 /* pas_local_allocator_result.h in Headers */,
0F45D5AB26AB731400C35D93 /* pas_local_allocator_scavenger_data.h in Headers */,
0F148712269B7518006887A9 /* pas_local_view_cache.h in Headers */,
+ 0F14871F269F5FCA006887A9 /* pas_local_view_cache_node.h in Headers */,
0FE7EE2122960142004F4166 /* pas_lock.h in Headers */,
0FEA45BD236CD5AC00B5A375 /* pas_lock_free_read_ptr_ptr_hashtable.h in Headers */,
0F9E945923452262009FAFDD /* pas_log.h in Headers */,
@@ -2289,7 +2290,6 @@
0F083F5A22C2979B00F3CDB3 /* pas_segregated_page_config_utils_inlines.h in Headers */,
0F9A1CEB2559961700C8D11B /* pas_segregated_page_config_variant.h in Headers */,
0FC4EC792351707B00B710A3 /* pas_segregated_page_emptiness_kind.h in Headers */,
- E3A7DC8F27BEF8EA004E76E9 /* pas_thread_suspend_lock.h in Headers */,
0FF8D62922C166F500EC72FE /* pas_segregated_page_inlines.h in Headers */,
0FD48B3523A9ABB30026C46D /* pas_segregated_partial_view.h in Headers */,
0FD48B6523A9ABB30026C46D /* pas_segregated_partial_view_inlines.h in Headers */,
@@ -2727,6 +2727,7 @@
0FDBB07422B875A6006BA5FC /* pas_local_allocator.c in Sources */,
0F45D5AA26AB731400C35D93 /* pas_local_allocator_scavenger_data.c in Sources */,
0F148720269F5FCA006887A9 /* pas_local_view_cache.c in Sources */,
+ 0F148722269F5FCA006887A9 /* pas_local_view_cache_node.c in Sources */,
0F04C21F2460DB6F001D31F0 /* pas_lock.c in Sources */,
0FEA45BC236CD5AC00B5A375 /* pas_lock_free_read_ptr_ptr_hashtable.c in Sources */,
0F9E945823452262009FAFDD /* pas_log.c in Sources */,
@@ -2763,7 +2764,6 @@
0FD48B5C23A9ABB30026C46D /* pas_segregated_size_directory.c in Sources */,
0FD48B3623A9ABB30026C46D /* pas_segregated_view.c in Sources */,
0F18E63F2628BFB700F6903D /* pas_shared_page_directory_by_size.c in Sources */,
- E3A7DC9027BEF8EA004E76E9 /* pas_thread_suspend_lock.c in Sources */,
0F6D564623CAA1BA00F40DBB /* pas_simple_free_heap_helpers.c in Sources */,
0FF08F3E22A5DC6900386575 /* pas_simple_large_free_heap.c in Sources */,
2C11E88F2728A783002162D0 /* pas_simple_type.c in Sources */,
Modified: branches/safari-613.1.17.1-branch/Source/bmalloc/libpas/src/libpas/pas_designated_intrinsic_heap.c (290157 => 290158)
--- branches/safari-613.1.17.1-branch/Source/bmalloc/libpas/src/libpas/pas_designated_intrinsic_heap.c 2022-02-18 21:12:51 UTC (rev 290157)
+++ branches/safari-613.1.17.1-branch/Source/bmalloc/libpas/src/libpas/pas_designated_intrinsic_heap.c 2022-02-18 21:13:01 UTC (rev 290158)
@@ -63,12 +63,11 @@
PAS_ASSERT(designated_begin == data->next_index_to_set);
- pas_heap_lock_assert_held();
-
- if (pas_thread_local_cache_layout_get_last_node()) {
+ if (pas_thread_local_cache_layout_last_node) {
PAS_ASSERT(
size
- > pas_thread_local_cache_layout_node_get_directory(pas_thread_local_cache_layout_get_last_node())->object_size);
+ > pas_thread_local_cache_layout_node_get_directory(
+ pas_thread_local_cache_layout_last_node)->object_size);
} else
PAS_ASSERT(!designated_begin);
@@ -108,9 +107,10 @@
pas_segregated_size_directory_create_tlc_allocator(directory);
- PAS_ASSERT(directory->allocator_index == target_allocator_index);
+ PAS_ASSERT(
+ pas_segregated_size_directory_data_ptr_load(&directory->data)->allocator_index
+ == target_allocator_index);
} else {
- pas_thread_local_cache_layout_node last_node;
pas_allocator_index resulting_index;
PAS_ASSERT(directory);
@@ -117,11 +117,13 @@
resulting_index = pas_thread_local_cache_layout_duplicate(directory);
PAS_ASSERT(resulting_index == target_allocator_index);
-
- last_node = pas_thread_local_cache_layout_get_last_node();
- PAS_ASSERT(pas_thread_local_cache_layout_node_get_directory(last_node) == directory);
- PAS_ASSERT(pas_thread_local_cache_layout_node_get_allocator_index_for_allocator(last_node) == target_allocator_index);
- PAS_ASSERT(directory->allocator_index == PAS_LOCAL_ALLOCATOR_UNSELECTED_NUM_INDICES + designated_begin * data->num_allocator_indices);
+ PAS_ASSERT(pas_thread_local_cache_layout_node_get_directory(
+ pas_thread_local_cache_layout_last_node) == directory);
+ PAS_ASSERT(pas_thread_local_cache_layout_node_get_allocator_index_for_allocator(
+ pas_thread_local_cache_layout_last_node) == target_allocator_index);
+ PAS_ASSERT(
+ pas_segregated_size_directory_data_ptr_load(&directory->data)->allocator_index
+ == PAS_LOCAL_ALLOCATOR_UNSELECTED_NUM_INDICES + designated_begin * data->num_allocator_indices);
}
}
Modified: branches/safari-613.1.17.1-branch/Source/bmalloc/libpas/src/libpas/pas_enumerate_segregated_heaps.c (290157 => 290158)
--- branches/safari-613.1.17.1-branch/Source/bmalloc/libpas/src/libpas/pas_enumerate_segregated_heaps.c 2022-02-18 21:12:51 UTC (rev 290157)
+++ branches/safari-613.1.17.1-branch/Source/bmalloc/libpas/src/libpas/pas_enumerate_segregated_heaps.c 2022-02-18 21:13:01 UTC (rev 290158)
@@ -33,6 +33,7 @@
#include "pas_enumerator_internal.h"
#include "pas_full_alloc_bits.h"
#include "pas_hashtable.h"
+#include "pas_local_view_cache_node.h"
#include "pas_ptr_hash_set.h"
#include "pas_redundant_local_allocator_node.h"
#include "pas_root.h"
@@ -42,7 +43,6 @@
#include "pas_segregated_shared_handle.h"
#include "pas_segregated_shared_view.h"
#include "pas_shared_handle_or_page_boundary_inlines.h"
-#include "pas_thread_local_cache_layout.h"
#include "pas_thread_local_cache_node.h"
static const bool verbose = false;
@@ -629,7 +629,7 @@
{
pas_thread_local_cache_node** tlc_node_first_ptr;
pas_thread_local_cache_node* tlc_node;
- pas_thread_local_cache_layout_segment** tlc_layout_first_segment_ptr;
+ pas_thread_local_cache_layout_node* tlc_layout_first_node_ptr;
enumeration_context context;
pas_baseline_allocator** baseline_allocator_table_ptr;
size_t index;
@@ -644,12 +644,15 @@
if (!tlc_node_first_ptr)
return false;
- tlc_layout_first_segment_ptr = pas_enumerator_read(enumerator, enumerator->root->thread_local_cache_layout_first_segment, sizeof(pas_thread_local_cache_layout_segment*));
- if (!tlc_layout_first_segment_ptr)
+ tlc_layout_first_node_ptr = pas_enumerator_read(enumerator,
+ enumerator->root->thread_local_cache_layout_first_node,
+ sizeof(pas_thread_local_cache_layout_node));
+ if (!tlc_layout_first_node_ptr)
return false;
for (tlc_node = *tlc_node_first_ptr; tlc_node; tlc_node = tlc_node->next) {
pas_thread_local_cache* tlc;
+ pas_thread_local_cache_layout_node layout_node;
unsigned index;
tlc_node = pas_enumerator_read(enumerator,
@@ -684,78 +687,58 @@
}
}
- if (*tlc_layout_first_segment_ptr) {
- pas_thread_local_cache_layout_segment** tlc_layout_segment_ptr;
- uintptr_t node_index = 0;
- pas_thread_local_cache_layout_node layout_node;
- pas_compact_atomic_thread_local_cache_layout_node* layout_node_ptr;
+ layout_node = pas_enumerator_read_compact(enumerator, *tlc_layout_first_node_ptr);
+ while (layout_node) {
+ bool has_allocator;
+ unsigned allocator_index;
+
+ if (pas_is_wrapped_segregated_size_directory(layout_node)) {
+ pas_segregated_size_directory* directory;
+ pas_segregated_size_directory_data* data;
- tlc_layout_segment_ptr = tlc_layout_first_segment_ptr;
+ directory = pas_unwrap_segregated_size_directory(layout_node);
- layout_node_ptr = pas_enumerator_read(enumerator, &((*tlc_layout_segment_ptr)->nodes[node_index]), sizeof(pas_compact_atomic_thread_local_cache_layout_node));
- if (!layout_node_ptr)
- return false;
- layout_node = pas_compact_atomic_thread_local_cache_layout_node_load_remote(enumerator, layout_node_ptr);
- while (1) {
- bool has_allocator;
- unsigned allocator_index;
+ data = ""
+ enumerator, &directory->data);
- if (!layout_node) {
- tlc_layout_segment_ptr = pas_enumerator_read(enumerator, &((*tlc_layout_segment_ptr)->next), sizeof(pas_thread_local_cache_layout_segment*));
- if (!tlc_layout_segment_ptr)
- return false;
- if (!*tlc_layout_segment_ptr)
- break;
- node_index = 0;
- layout_node_ptr = pas_enumerator_read(enumerator, &((*tlc_layout_segment_ptr)->nodes[node_index]), sizeof(pas_compact_atomic_thread_local_cache_layout_node));
- if (!layout_node_ptr)
- return false;
- layout_node = pas_compact_atomic_thread_local_cache_layout_node_load_remote(enumerator, layout_node_ptr);
- if (!layout_node)
- break;
- }
-
- if (pas_is_wrapped_segregated_size_directory(layout_node)) {
- pas_segregated_size_directory* directory;
+ layout_node = pas_compact_atomic_thread_local_cache_layout_node_load_remote(
+ enumerator, &data->next_for_layout);
- directory = pas_unwrap_segregated_size_directory(layout_node);
+ allocator_index = data->allocator_index;
+ has_allocator = true;
+ } else if (pas_is_wrapped_redundant_local_allocator_node(layout_node)) {
+ pas_redundant_local_allocator_node* redundant_node;
- allocator_index = directory->allocator_index;
- has_allocator = true;
- } else if (pas_is_wrapped_redundant_local_allocator_node(layout_node)) {
- pas_redundant_local_allocator_node* redundant_node;
+ redundant_node = pas_unwrap_redundant_local_allocator_node(layout_node);
- redundant_node = pas_unwrap_redundant_local_allocator_node(layout_node);
+ layout_node = pas_compact_atomic_thread_local_cache_layout_node_load_remote(
+ enumerator, &redundant_node->next);
- allocator_index = redundant_node->allocator_index;
- has_allocator = true;
- } else {
- pas_segregated_size_directory* directory;
-
- PAS_ASSERT(pas_is_wrapped_local_view_cache_node(layout_node));
+ allocator_index = redundant_node->allocator_index;
+ has_allocator = true;
+ } else {
+ pas_local_view_cache_node* cache_node;
+
+ PAS_ASSERT(pas_is_wrapped_local_view_cache_node(layout_node));
- directory = pas_unwrap_local_view_cache_node(layout_node);
+ cache_node = pas_unwrap_local_view_cache_node(layout_node);
- allocator_index = 0;
- has_allocator = false;
- }
+ layout_node = pas_compact_atomic_thread_local_cache_layout_node_load_remote(
+ enumerator, &cache_node->next);
- if (has_allocator) {
- pas_local_allocator* allocator;
-
- if (!allocator_index || allocator_index >= tlc->allocator_index_upper_bound)
- break;
+ allocator_index = 0;
+ has_allocator = false;
+ }
- allocator = pas_thread_local_cache_get_local_allocator_direct(tlc, allocator_index);
-
- consider_allocator(enumerator, &context, allocator);
- }
+ if (has_allocator) {
+ pas_local_allocator* allocator;
+
+ if (!allocator_index || allocator_index >= tlc->allocator_index_upper_bound)
+ break;
- ++node_index;
- layout_node_ptr = pas_enumerator_read(enumerator, &((*tlc_layout_segment_ptr)->nodes[node_index]), sizeof(pas_compact_atomic_thread_local_cache_layout_node));
- if (!layout_node_ptr)
- return false;
- layout_node = pas_compact_atomic_thread_local_cache_layout_node_load_remote(enumerator, layout_node_ptr);
+ allocator = pas_thread_local_cache_get_local_allocator_direct(tlc, allocator_index);
+
+ consider_allocator(enumerator, &context, allocator);
}
}
}
Copied: branches/safari-613.1.17.1-branch/Source/bmalloc/libpas/src/libpas/pas_local_view_cache_node.c (from rev 290136, branches/safari-613.1.17.1-branch/Source/bmalloc/libpas/src/libpas/pas_redundant_local_allocator_node.c) (0 => 290158)
--- branches/safari-613.1.17.1-branch/Source/bmalloc/libpas/src/libpas/pas_local_view_cache_node.c (rev 0)
+++ branches/safari-613.1.17.1-branch/Source/bmalloc/libpas/src/libpas/pas_local_view_cache_node.c 2022-02-18 21:13:01 UTC (rev 290158)
@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) 2021 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "pas_config.h"
+
+#if LIBPAS_ENABLED
+
+#include "pas_local_view_cache_node.h"
+
+#include "pas_immortal_heap.h"
+
+pas_local_view_cache_node*
+pas_local_view_cache_node_create(pas_segregated_size_directory* directory)
+{
+ pas_local_view_cache_node* result;
+
+ result = pas_immortal_heap_allocate(
+ sizeof(pas_local_view_cache_node),
+ "pas_local_view_cache_node",
+ pas_object_allocation);
+
+ pas_compact_atomic_thread_local_cache_layout_node_store(&result->next, NULL);
+ pas_compact_segregated_size_directory_ptr_store(&result->directory, directory);
+
+ return result;
+}
+
+#endif /* LIBPAS_ENABLED */
Copied: branches/safari-613.1.17.1-branch/Source/bmalloc/libpas/src/libpas/pas_local_view_cache_node.h (from rev 290136, branches/safari-613.1.17.1-branch/Source/bmalloc/libpas/src/libpas/pas_redundant_local_allocator_node.h) (0 => 290158)
--- branches/safari-613.1.17.1-branch/Source/bmalloc/libpas/src/libpas/pas_local_view_cache_node.h (rev 0)
+++ branches/safari-613.1.17.1-branch/Source/bmalloc/libpas/src/libpas/pas_local_view_cache_node.h 2022-02-18 21:13:01 UTC (rev 290158)
@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2021 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef PAS_LOCAL_VIEW_CACHE_NODE_H
+#define PAS_LOCAL_VIEW_CACHE_NODE_H
+
+#include "pas_compact_segregated_size_directory_ptr.h"
+#include "pas_compact_atomic_thread_local_cache_layout_node.h"
+#include "pas_allocator_index.h"
+
+PAS_BEGIN_EXTERN_C;
+
+struct pas_local_view_cache_node;
+typedef struct pas_local_view_cache_node pas_local_view_cache_node;
+
+struct pas_local_view_cache_node {
+ pas_compact_atomic_thread_local_cache_layout_node next;
+ pas_compact_segregated_size_directory_ptr directory;
+};
+
+PAS_API pas_local_view_cache_node*
+pas_local_view_cache_node_create(pas_segregated_size_directory* directory);
+
+PAS_END_EXTERN_C;
+
+#endif /* PAS_LOCAL_VIEW_CACHE_NODE_H */
+
Modified: branches/safari-613.1.17.1-branch/Source/bmalloc/libpas/src/libpas/pas_redundant_local_allocator_node.c (290157 => 290158)
--- branches/safari-613.1.17.1-branch/Source/bmalloc/libpas/src/libpas/pas_redundant_local_allocator_node.c 2022-02-18 21:12:51 UTC (rev 290157)
+++ branches/safari-613.1.17.1-branch/Source/bmalloc/libpas/src/libpas/pas_redundant_local_allocator_node.c 2022-02-18 21:13:01 UTC (rev 290158)
@@ -41,6 +41,7 @@
"pas_redundant_local_allocator_node",
pas_object_allocation);
+ pas_compact_atomic_thread_local_cache_layout_node_store(&result->next, NULL);
pas_compact_segregated_size_directory_ptr_store(&result->directory, directory);
result->allocator_index = 0;
Modified: branches/safari-613.1.17.1-branch/Source/bmalloc/libpas/src/libpas/pas_redundant_local_allocator_node.h (290157 => 290158)
--- branches/safari-613.1.17.1-branch/Source/bmalloc/libpas/src/libpas/pas_redundant_local_allocator_node.h 2022-02-18 21:12:51 UTC (rev 290157)
+++ branches/safari-613.1.17.1-branch/Source/bmalloc/libpas/src/libpas/pas_redundant_local_allocator_node.h 2022-02-18 21:13:01 UTC (rev 290158)
@@ -36,6 +36,7 @@
typedef struct pas_redundant_local_allocator_node pas_redundant_local_allocator_node;
struct pas_redundant_local_allocator_node {
+ pas_compact_atomic_thread_local_cache_layout_node next;
pas_compact_segregated_size_directory_ptr directory;
pas_allocator_index allocator_index;
};
Modified: branches/safari-613.1.17.1-branch/Source/bmalloc/libpas/src/libpas/pas_root.c (290157 => 290158)
--- branches/safari-613.1.17.1-branch/Source/bmalloc/libpas/src/libpas/pas_root.c 2022-02-18 21:12:51 UTC (rev 290157)
+++ branches/safari-613.1.17.1-branch/Source/bmalloc/libpas/src/libpas/pas_root.c 2022-02-18 21:13:01 UTC (rev 290158)
@@ -95,7 +95,7 @@
&pas_large_heap_physical_page_sharing_cache_page_list;
root->payload_reservation_page_list = &pas_payload_reservation_page_list;
root->thread_local_cache_node_first = &pas_thread_local_cache_node_first;
- root->thread_local_cache_layout_first_segment = &pas_thread_local_cache_layout_first_segment;
+ root->thread_local_cache_layout_first_node = &pas_thread_local_cache_layout_first_node;
root->all_heaps_first_heap = &pas_all_heaps_first_heap;
root->num_static_heaps = 0;
Modified: branches/safari-613.1.17.1-branch/Source/bmalloc/libpas/src/libpas/pas_root.h (290157 => 290158)
--- branches/safari-613.1.17.1-branch/Source/bmalloc/libpas/src/libpas/pas_root.h 2022-02-18 21:12:51 UTC (rev 290157)
+++ branches/safari-613.1.17.1-branch/Source/bmalloc/libpas/src/libpas/pas_root.h 2022-02-18 21:13:01 UTC (rev 290158)
@@ -61,7 +61,6 @@
typedef struct pas_small_large_map_hashtable pas_small_large_map_hashtable;
typedef struct pas_small_large_map_hashtable_in_flux_stash pas_small_large_map_hashtable_in_flux_stash;
typedef struct pas_thread_local_cache_node pas_thread_local_cache_node;
-typedef struct pas_thread_local_cache_layout_segment pas_thread_local_cache_layout_segment;
typedef struct pas_tiny_large_map_hashtable pas_tiny_large_map_hashtable;
typedef struct pas_tiny_large_map_hashtable_in_flux_stash pas_tiny_large_map_hashtable_in_flux_stash;
typedef struct pas_tiny_large_map_second_level_hashtable_in_flux_stash pas_tiny_large_map_second_level_hashtable_in_flux_stash;
@@ -77,7 +76,7 @@
pas_enumerable_range_list* large_heap_physical_page_sharing_cache_page_list;
pas_enumerable_range_list* payload_reservation_page_list;
pas_thread_local_cache_node** thread_local_cache_node_first;
- pas_thread_local_cache_layout_segment** thread_local_cache_layout_first_segment;
+ pas_thread_local_cache_layout_node* thread_local_cache_layout_first_node;
pas_heap** all_heaps_first_heap;
pas_heap** static_heaps;
size_t num_static_heaps;
Modified: branches/safari-613.1.17.1-branch/Source/bmalloc/libpas/src/libpas/pas_segregated_heap.c (290157 => 290158)
--- branches/safari-613.1.17.1-branch/Source/bmalloc/libpas/src/libpas/pas_segregated_heap.c 2022-02-18 21:12:51 UTC (rev 290157)
+++ branches/safari-613.1.17.1-branch/Source/bmalloc/libpas/src/libpas/pas_segregated_heap.c 2022-02-18 21:13:01 UTC (rev 290158)
@@ -674,10 +674,15 @@
directory = pas_compact_atomic_segregated_size_directory_ptr_load(&directory->next_for_heap)) {
size_t index;
pas_allocator_index allocator_index;
+ pas_segregated_size_directory_data* data;
size_t extra_index_for_allocator;
bool have_extra_index_for_allocator;
- allocator_index = directory->allocator_index;
+ data = ""
+ if (data)
+ allocator_index = data->allocator_index;
+ else
+ allocator_index = 0;
PAS_ASSERT(allocator_index != (pas_allocator_index)UINT_MAX);
@@ -731,9 +736,14 @@
(directory = size_directory_min_heap_take_min(&min_heap));
medium_tuple_index++) {
pas_segregated_heap_medium_directory_tuple tuple;
-
+ pas_segregated_size_directory_data* data;
+
pas_compact_atomic_segregated_size_directory_ptr_store(&tuple.directory, directory);
- tuple.allocator_index = directory->allocator_index;
+ data = ""
+ if (data)
+ tuple.allocator_index = data->allocator_index;
+ else
+ tuple.allocator_index = 0;
PAS_ASSERT(tuple.allocator_index != (pas_allocator_index)UINT_MAX);
tuple.begin_index = pas_segregated_size_directory_min_index(directory);
PAS_ASSERT(tuple.begin_index);
@@ -855,7 +865,8 @@
if (verbose)
pas_log("index = %zu\n", index);
- allocator_index = directory->allocator_index;
+ allocator_index =
+ pas_segregated_size_directory_data_ptr_load(&directory->data)->allocator_index;
PAS_ASSERT(allocator_index);
PAS_ASSERT((pas_allocator_index)allocator_index == allocator_index);
PAS_ASSERT(allocator_index < (unsigned)(pas_allocator_index)UINT_MAX);
Modified: branches/safari-613.1.17.1-branch/Source/bmalloc/libpas/src/libpas/pas_segregated_size_directory.c (290157 => 290158)
--- branches/safari-613.1.17.1-branch/Source/bmalloc/libpas/src/libpas/pas_segregated_size_directory.c 2022-02-18 21:12:51 UTC (rev 290157)
+++ branches/safari-613.1.17.1-branch/Source/bmalloc/libpas/src/libpas/pas_segregated_size_directory.c 2022-02-18 21:13:01 UTC (rev 290158)
@@ -117,7 +117,6 @@
pas_segregated_size_directory_encode_stuff(2 * PAS_NUM_BASELINE_ALLOCATORS, UINT_MAX);
result->view_cache_index = (pas_allocator_index)UINT_MAX;
- result->allocator_index = 0;
pas_segregated_size_directory_data_ptr_store(&result->data, NULL);
@@ -205,6 +204,8 @@
data->offset_from_page_boundary_to_first_object = 0;
data->offset_from_page_boundary_to_end_of_last_object = 0;
data->full_num_non_empty_words = 0;
+ data->allocator_index = 0;
+ pas_compact_atomic_thread_local_cache_layout_node_store(&data->next_for_layout, NULL);
pas_fence();
@@ -232,22 +233,24 @@
void pas_segregated_size_directory_create_tlc_allocator(
pas_segregated_size_directory* directory)
{
+ pas_segregated_size_directory_data* data;
+
pas_heap_lock_assert_held();
if (pas_segregated_page_config_kind_is_utility(directory->base.page_config_kind))
return;
- pas_segregated_size_directory_ensure_data(directory, pas_lock_is_held);
+ data = "" pas_lock_is_held);
- if (directory->allocator_index) {
- PAS_ASSERT(directory->allocator_index != (pas_allocator_index)UINT_MAX);
+ if (data->allocator_index) {
+ PAS_ASSERT(data->allocator_index != (pas_allocator_index)UINT_MAX);
return;
}
pas_thread_local_cache_layout_add(directory);
- PAS_ASSERT(directory->allocator_index);
- PAS_ASSERT(directory->allocator_index < (pas_allocator_index)UINT_MAX);
+ PAS_ASSERT(data->allocator_index);
+ PAS_ASSERT(data->allocator_index < (pas_allocator_index)UINT_MAX);
}
void pas_segregated_size_directory_create_tlc_view_cache(
@@ -944,7 +947,7 @@
tlc_result = pas_thread_local_cache_get_local_allocator_for_initialized_index(
pas_thread_local_cache_get(config),
- directory->allocator_index,
+ pas_segregated_size_directory_data_ptr_load_non_null(&directory->data)->allocator_index,
pas_lock_is_not_held);
PAS_ASSERT(tlc_result.did_succeed);
Modified: branches/safari-613.1.17.1-branch/Source/bmalloc/libpas/src/libpas/pas_segregated_size_directory.h (290157 => 290158)
--- branches/safari-613.1.17.1-branch/Source/bmalloc/libpas/src/libpas/pas_segregated_size_directory.h 2022-02-18 21:12:51 UTC (rev 290157)
+++ branches/safari-613.1.17.1-branch/Source/bmalloc/libpas/src/libpas/pas_segregated_size_directory.h 2022-02-18 21:13:01 UTC (rev 290158)
@@ -84,9 +84,6 @@
pas_allocator_index view_cache_index; /* This could be in the size_directory_data but we put it here
because it takes less space to put it here. */
-
- pas_allocator_index allocator_index; /* This could be in the size_directory_data but we put it here
- because it takes less space to put it here. */
pas_segregated_size_directory_data_ptr data;
@@ -98,9 +95,12 @@
unsigned offset_from_page_boundary_to_first_object; /* Cached to make refill fast. */
unsigned offset_from_page_boundary_to_end_of_last_object; /* Cached to make refill fast. */
+ pas_allocator_index allocator_index;
+
uint8_t full_num_non_empty_words;
pas_compact_tagged_unsigned_ptr full_alloc_bits; /* Precomputed alloc bits in the case that a page is empty. */
+ pas_compact_atomic_thread_local_cache_layout_node next_for_layout;
};
struct pas_extended_segregated_size_directory_data {
@@ -300,13 +300,19 @@
static inline bool pas_segregated_size_directory_has_tlc_allocator(
pas_segregated_size_directory* directory)
{
- return directory->allocator_index;
+ pas_segregated_size_directory_data* data;
+ data = ""
+ return data && data->allocator_index;
}
static inline pas_allocator_index pas_segregated_size_directory_get_tlc_allocator_index(
pas_segregated_size_directory* directory)
{
- return directory->allocator_index;
+ pas_segregated_size_directory_data* data;
+ data = ""
+ if (data)
+ return data->allocator_index;
+ return 0;
}
/* Call with heap lock held. */
Modified: branches/safari-613.1.17.1-branch/Source/bmalloc/libpas/src/libpas/pas_thread_local_cache.c (290157 => 290158)
--- branches/safari-613.1.17.1-branch/Source/bmalloc/libpas/src/libpas/pas_thread_local_cache.c 2022-02-18 21:12:51 UTC (rev 290157)
+++ branches/safari-613.1.17.1-branch/Source/bmalloc/libpas/src/libpas/pas_thread_local_cache.c 2022-02-18 21:13:01 UTC (rev 290158)
@@ -689,17 +689,13 @@
if (allocator_action != pas_allocator_scavenge_no_action) {
bool did_suspend;
- pas_thread_local_cache_layout_segment* segment;
pas_thread_local_cache_layout_node layout_node;
- uintptr_t node_index;
did_suspend = false;
PAS_UNUSED_PARAM(did_suspend);
- segment = pas_thread_local_cache_layout_first_segment;
- node_index = 0;
- for (PAS_THREAD_LOCAL_CACHE_LAYOUT_EACH_ALLOCATOR_WITH_SEGMENT_AND_INDEX(layout_node, segment, node_index)) {
+ for (PAS_THREAD_LOCAL_CACHE_LAYOUT_EACH_ALLOCATOR(layout_node)) {
pas_allocator_index allocator_index;
pas_local_allocator_scavenger_data* scavenger_data;
Modified: branches/safari-613.1.17.1-branch/Source/bmalloc/libpas/src/libpas/pas_thread_local_cache_layout.c (290157 => 290158)
--- branches/safari-613.1.17.1-branch/Source/bmalloc/libpas/src/libpas/pas_thread_local_cache_layout.c 2022-02-18 21:12:51 UTC (rev 290157)
+++ branches/safari-613.1.17.1-branch/Source/bmalloc/libpas/src/libpas/pas_thread_local_cache_layout.c 2022-02-18 21:13:01 UTC (rev 290158)
@@ -31,15 +31,13 @@
#include "pas_heap_lock.h"
#include "pas_large_utility_free_heap.h"
+#include "pas_local_view_cache_node.h"
#include "pas_redundant_local_allocator_node.h"
#include "pas_segregated_size_directory_inlines.h"
-#include "pas_utility_heap.h"
#include "pas_utils.h"
-pas_thread_local_cache_layout_segment* pas_thread_local_cache_layout_first_segment = NULL;
-static pas_thread_local_cache_layout_segment* pas_thread_local_cache_layout_last_segment = NULL;
-static unsigned pas_thread_local_cache_layout_last_segment_size = 0;
-
+pas_thread_local_cache_layout_node pas_thread_local_cache_layout_first_node = NULL;
+pas_thread_local_cache_layout_node pas_thread_local_cache_layout_last_node = NULL;
pas_allocator_index pas_thread_local_cache_layout_next_allocator_index =
PAS_LOCAL_ALLOCATOR_UNSELECTED_NUM_INDICES;
@@ -64,6 +62,8 @@
break;
}
+ PAS_ASSERT(!pas_thread_local_cache_layout_node_get_next(node));
+
result = pas_thread_local_cache_layout_next_allocator_index;
PAS_ASSERT(result < (pas_allocator_index)UINT_MAX);
@@ -81,30 +81,18 @@
&pas_thread_local_cache_layout_next_allocator_index);
PAS_ASSERT(!did_overflow);
- if (!pas_thread_local_cache_layout_last_segment || pas_thread_local_cache_layout_last_segment_size == PAS_THREAD_LOCAL_CACHE_LAYOUT_SEGMENT_SIZE) {
- pas_thread_local_cache_layout_segment* segment;
-
- segment = (pas_thread_local_cache_layout_segment*)pas_utility_heap_allocate(sizeof(pas_thread_local_cache_layout_segment), "pas_thread_local_cache_layout_segment");
- pas_zero_memory(segment, sizeof(pas_thread_local_cache_layout_segment));
-
- pas_compact_atomic_thread_local_cache_layout_node_store(&segment->nodes[0], node);
- pas_thread_local_cache_layout_last_segment_size = 1;
-
- pas_fence();
- if (!pas_thread_local_cache_layout_last_segment) {
- PAS_ASSERT(!pas_thread_local_cache_layout_first_segment);
- PAS_ASSERT(result == PAS_LOCAL_ALLOCATOR_UNSELECTED_NUM_INDICES);
- pas_thread_local_cache_layout_first_segment = segment;
- pas_thread_local_cache_layout_last_segment = segment;
- } else {
- pas_thread_local_cache_layout_last_segment->next = segment;
- pas_thread_local_cache_layout_last_segment = segment;
- }
+ pas_fence();
+
+ if (!pas_thread_local_cache_layout_first_node) {
+ PAS_ASSERT(!pas_thread_local_cache_layout_last_node);
+ PAS_ASSERT(result == PAS_LOCAL_ALLOCATOR_UNSELECTED_NUM_INDICES);
+ pas_thread_local_cache_layout_first_node = node;
+ pas_thread_local_cache_layout_last_node = node;
} else {
- PAS_ASSERT(pas_thread_local_cache_layout_last_segment);
+ PAS_ASSERT(pas_thread_local_cache_layout_last_node);
PAS_ASSERT(result > PAS_LOCAL_ALLOCATOR_UNSELECTED_NUM_INDICES);
- pas_fence();
- pas_compact_atomic_thread_local_cache_layout_node_store(&pas_thread_local_cache_layout_last_segment->nodes[pas_thread_local_cache_layout_last_segment_size++], node);
+ pas_thread_local_cache_layout_node_set_next(pas_thread_local_cache_layout_last_node, node);
+ pas_thread_local_cache_layout_last_node = node;
}
return result;
@@ -139,16 +127,11 @@
pas_allocator_index pas_thread_local_cache_layout_add_view_cache(
pas_segregated_size_directory* directory)
{
- return pas_thread_local_cache_layout_add_node(pas_wrap_local_view_cache_node(directory));
-}
+ pas_local_view_cache_node* cache_node;
-pas_thread_local_cache_layout_node pas_thread_local_cache_layout_get_last_node()
-{
- pas_heap_lock_assert_held();
- if (!pas_thread_local_cache_layout_last_segment)
- return NULL;
- PAS_ASSERT(pas_thread_local_cache_layout_last_segment_size);
- return pas_thread_local_cache_layout_segment_get_node(pas_thread_local_cache_layout_last_segment, pas_thread_local_cache_layout_last_segment_size - 1);
+ cache_node = pas_local_view_cache_node_create(directory);
+
+ return pas_thread_local_cache_layout_add_node(pas_wrap_local_view_cache_node(cache_node));
}
#endif /* LIBPAS_ENABLED */
Modified: branches/safari-613.1.17.1-branch/Source/bmalloc/libpas/src/libpas/pas_thread_local_cache_layout.h (290157 => 290158)
--- branches/safari-613.1.17.1-branch/Source/bmalloc/libpas/src/libpas/pas_thread_local_cache_layout.h 2022-02-18 21:12:51 UTC (rev 290157)
+++ branches/safari-613.1.17.1-branch/Source/bmalloc/libpas/src/libpas/pas_thread_local_cache_layout.h 2022-02-18 21:13:01 UTC (rev 290158)
@@ -27,7 +27,6 @@
#define PAS_THREAD_LOCAL_CACHE_LAYOUT_H
#include "pas_allocator_index.h"
-#include "pas_compact_atomic_thread_local_cache_layout_node.h"
#include "pas_config.h"
#include "pas_thread_local_cache_layout_node.h"
#include "pas_utils.h"
@@ -34,16 +33,9 @@
PAS_BEGIN_EXTERN_C;
-#define PAS_THREAD_LOCAL_CACHE_LAYOUT_SEGMENT_SIZE 257
-struct pas_thread_local_cache_layout_segment {
- pas_compact_atomic_thread_local_cache_layout_node nodes[PAS_THREAD_LOCAL_CACHE_LAYOUT_SEGMENT_SIZE];
- pas_compact_atomic_thread_local_cache_layout_node sentinel;
- struct pas_thread_local_cache_layout_segment* next;
-};
-typedef struct pas_thread_local_cache_layout_segment pas_thread_local_cache_layout_segment;
+PAS_API extern pas_thread_local_cache_layout_node pas_thread_local_cache_layout_first_node;
+PAS_API extern pas_thread_local_cache_layout_node pas_thread_local_cache_layout_last_node;
-PAS_API extern pas_thread_local_cache_layout_segment* pas_thread_local_cache_layout_first_segment;
-
/* Clients can use this to force the next call to add to go to this index. */
PAS_API extern pas_allocator_index pas_thread_local_cache_layout_next_allocator_index;
@@ -59,39 +51,11 @@
PAS_API pas_allocator_index pas_thread_local_cache_layout_add_view_cache(
pas_segregated_size_directory* directory);
-PAS_API pas_thread_local_cache_layout_node pas_thread_local_cache_layout_get_last_node(void);
-
-static PAS_ALWAYS_INLINE pas_thread_local_cache_layout_node pas_thread_local_cache_layout_segment_get_node(pas_thread_local_cache_layout_segment* segment, uintptr_t index)
-{
- if (!segment)
- return NULL;
- return pas_compact_atomic_thread_local_cache_layout_node_load(&segment->nodes[index]);
-}
-
-static PAS_ALWAYS_INLINE pas_thread_local_cache_layout_node pas_thread_local_cache_layout_segment_next_node(pas_thread_local_cache_layout_segment** segment, uintptr_t* index)
-{
- PAS_TESTING_ASSERT(segment);
- PAS_TESTING_ASSERT(*segment);
- uintptr_t next_index = *index + 1;
- pas_thread_local_cache_layout_node next = pas_thread_local_cache_layout_segment_get_node(*segment, next_index);
- if (next) {
- *index = next_index;
- return next;
- }
- *segment = (*segment)->next;
- *index = 0;
- return pas_thread_local_cache_layout_segment_get_node(*segment, 0);
-}
-
#define PAS_THREAD_LOCAL_CACHE_LAYOUT_EACH_ALLOCATOR(node) \
- uintptr_t internal_node_index = 0, internal_segment = (uintptr_t)pas_thread_local_cache_layout_first_segment, internal_node = (uintptr_t)pas_thread_local_cache_layout_segment_get_node((pas_thread_local_cache_layout_segment*)internal_segment, internal_node_index); \
- (node = ((pas_thread_local_cache_layout_node)internal_node)); \
- internal_node = (uintptr_t)pas_thread_local_cache_layout_segment_next_node((pas_thread_local_cache_layout_segment**)&internal_segment, &internal_node_index)
-
-#define PAS_THREAD_LOCAL_CACHE_LAYOUT_EACH_ALLOCATOR_WITH_SEGMENT_AND_INDEX(node, segment, index) \
- node = pas_thread_local_cache_layout_segment_get_node(segment, index); \
+ node = pas_thread_local_cache_layout_first_node; \
node; \
- node = pas_thread_local_cache_layout_segment_next_node(&segment, &index)
+ node = pas_thread_local_cache_layout_node_get_next(node)
+
PAS_END_EXTERN_C;
Modified: branches/safari-613.1.17.1-branch/Source/bmalloc/libpas/src/libpas/pas_thread_local_cache_layout_node.c (290157 => 290158)
--- branches/safari-613.1.17.1-branch/Source/bmalloc/libpas/src/libpas/pas_thread_local_cache_layout_node.c 2022-02-18 21:12:51 UTC (rev 290157)
+++ branches/safari-613.1.17.1-branch/Source/bmalloc/libpas/src/libpas/pas_thread_local_cache_layout_node.c 2022-02-18 21:13:01 UTC (rev 290158)
@@ -30,6 +30,7 @@
#include "pas_thread_local_cache_layout_node.h"
#include "pas_local_view_cache.h"
+#include "pas_local_view_cache_node.h"
#include "pas_redundant_local_allocator_node.h"
#include "pas_segregated_size_directory_inlines.h"
@@ -44,7 +45,8 @@
&pas_unwrap_redundant_local_allocator_node(node)->directory);
}
- return pas_unwrap_local_view_cache_node(node);
+ return pas_compact_segregated_size_directory_ptr_load_non_null(
+ &pas_unwrap_local_view_cache_node(node)->directory);
}
pas_allocator_index
@@ -65,13 +67,15 @@
allocator_index_ptr(pas_thread_local_cache_layout_node node)
{
if (pas_is_wrapped_segregated_size_directory(node)) {
- return &pas_unwrap_segregated_size_directory(node)->allocator_index;
+ return &pas_segregated_size_directory_data_ptr_load(
+ &pas_unwrap_segregated_size_directory(node)->data)->allocator_index;
}
if (pas_is_wrapped_redundant_local_allocator_node(node))
return &pas_unwrap_redundant_local_allocator_node(node)->allocator_index;
- return &pas_unwrap_local_view_cache_node(node)->view_cache_index;
+ return &pas_compact_segregated_size_directory_ptr_load_non_null(
+ &pas_unwrap_local_view_cache_node(node)->directory)->view_cache_index;
}
pas_allocator_index
@@ -101,6 +105,32 @@
*allocator_index_ptr(node) = index;
}
+static pas_compact_atomic_thread_local_cache_layout_node*
+next_ptr(pas_thread_local_cache_layout_node node)
+{
+ if (pas_is_wrapped_segregated_size_directory(node)) {
+ return &pas_segregated_size_directory_data_ptr_load(
+ &pas_unwrap_segregated_size_directory(node)->data)->next_for_layout;
+ }
+
+ if (pas_is_wrapped_redundant_local_allocator_node(node))
+ return &pas_unwrap_redundant_local_allocator_node(node)->next;
+
+ return &pas_unwrap_local_view_cache_node(node)->next;
+}
+
+pas_thread_local_cache_layout_node
+pas_thread_local_cache_layout_node_get_next(pas_thread_local_cache_layout_node node)
+{
+ return pas_compact_atomic_thread_local_cache_layout_node_load(next_ptr(node));
+}
+
+void pas_thread_local_cache_layout_node_set_next(pas_thread_local_cache_layout_node node,
+ pas_thread_local_cache_layout_node next_node)
+{
+ pas_compact_atomic_thread_local_cache_layout_node_store(next_ptr(node), next_node);
+}
+
void pas_thread_local_cache_layout_node_construct(pas_thread_local_cache_layout_node node,
pas_thread_local_cache* cache)
{
Modified: branches/safari-613.1.17.1-branch/Source/bmalloc/libpas/src/libpas/pas_thread_local_cache_layout_node.h (290157 => 290158)
--- branches/safari-613.1.17.1-branch/Source/bmalloc/libpas/src/libpas/pas_thread_local_cache_layout_node.h 2022-02-18 21:12:51 UTC (rev 290157)
+++ branches/safari-613.1.17.1-branch/Source/bmalloc/libpas/src/libpas/pas_thread_local_cache_layout_node.h 2022-02-18 21:13:01 UTC (rev 290158)
@@ -33,10 +33,12 @@
PAS_BEGIN_EXTERN_C;
+struct pas_local_view_cache_node;
struct pas_redundant_local_allocator_node;
struct pas_segregated_size_directory;
struct pas_thread_local_cache;
struct pas_thread_local_cache_layout_node_opaque;
+typedef struct pas_local_view_cache_node pas_local_view_cache_node;
typedef struct pas_redundant_local_allocator_node pas_redundant_local_allocator_node;
typedef struct pas_segregated_size_directory pas_segregated_size_directory;
typedef struct pas_thread_local_cache pas_thread_local_cache;
@@ -85,10 +87,10 @@
}
static inline pas_thread_local_cache_layout_node
-pas_wrap_local_view_cache_node(pas_segregated_size_directory* directory)
+pas_wrap_local_view_cache_node(pas_local_view_cache_node* node)
{
return pas_thread_local_cache_layout_node_create(
- pas_thread_local_cache_layout_local_view_cache_node_kind, directory);
+ pas_thread_local_cache_layout_local_view_cache_node_kind, node);
}
static inline bool
@@ -126,11 +128,11 @@
return (pas_redundant_local_allocator_node*)pas_thread_local_cache_layout_node_get_ptr(node);
}
-static inline pas_segregated_size_directory*
+static inline pas_local_view_cache_node*
pas_unwrap_local_view_cache_node(pas_thread_local_cache_layout_node node)
{
PAS_ASSERT(pas_is_wrapped_local_view_cache_node(node));
- return (pas_segregated_size_directory*)pas_thread_local_cache_layout_node_get_ptr(node);
+ return (pas_local_view_cache_node*)pas_thread_local_cache_layout_node_get_ptr(node);
}
PAS_API pas_segregated_size_directory*
@@ -167,6 +169,13 @@
pas_thread_local_cache_layout_node_set_allocator_index(pas_thread_local_cache_layout_node node,
pas_allocator_index index);
+PAS_API pas_thread_local_cache_layout_node
+pas_thread_local_cache_layout_node_get_next(pas_thread_local_cache_layout_node node);
+
+PAS_API void
+pas_thread_local_cache_layout_node_set_next(pas_thread_local_cache_layout_node node,
+ pas_thread_local_cache_layout_node next_node);
+
PAS_API void pas_thread_local_cache_layout_node_construct(pas_thread_local_cache_layout_node node,
pas_thread_local_cache* cache);
