Modified: trunk/Source/bmalloc/libpas/src/libpas/pas_scavenger.c (292512 => 292513)
--- trunk/Source/bmalloc/libpas/src/libpas/pas_scavenger.c 2022-04-06 22:41:39 UTC (rev 292512)
+++ trunk/Source/bmalloc/libpas/src/libpas/pas_scavenger.c 2022-04-06 22:47:34 UTC (rev 292513)
@@ -69,6 +69,10 @@
uint64_t pas_scavenger_max_epoch_delta = 300ll * 1000ll * 1000ll;
#endif
+static uint32_t pas_scavenger_tick_count = 0;
+/* Run thread-local-cache decommit once a N. It should be power of two. */
+#define PAS_THREAD_LOCAL_CACHE_DECOMMIT_PERIOD_COUNT 128 /* Roughly speaking, it runs once per 13 seconds. */
+
#if PAS_OS(DARWIN)
static _Atomic qos_class_t pas_scavenger_requested_qos_class = QOS_CLASS_USER_INITIATED;
@@ -192,6 +196,7 @@
double time_in_milliseconds;
double absolute_timeout_in_milliseconds_for_period_sleep;
pas_scavenger_activity_callback completion_callback;
+ pas_thread_local_cache_decommit_action thread_local_cache_decommit_action;
bool should_go_again;
uint64_t epoch;
uint64_t delta;
@@ -208,6 +213,7 @@
pthread_set_qos_class_self_np(configured_qos_class, 0);
}
#endif
+ ++pas_scavenger_tick_count;
should_go_again = false;
@@ -229,9 +235,16 @@
pas_utility_heap_for_all_allocators(pas_allocator_scavenge_request_stop_action,
pas_lock_is_not_held);
+ thread_local_cache_decommit_action = pas_thread_local_cache_decommit_no_action;
+ if ((pas_scavenger_tick_count % PAS_THREAD_LOCAL_CACHE_DECOMMIT_PERIOD_COUNT) == 0) {
+ if (verbose)
+ printf("Attempt to decommit unused TLC\n");
+ thread_local_cache_decommit_action = pas_thread_local_cache_decommit_if_possible_action;
+ }
should_go_again |=
pas_thread_local_cache_for_all(pas_allocator_scavenge_request_stop_action,
- pas_deallocator_scavenge_flush_log_if_clean_action);
+ pas_deallocator_scavenge_flush_log_if_clean_action,
+ thread_local_cache_decommit_action);
should_go_again |= handle_expendable_memory(pas_expendable_memory_scavenge_periodic);
@@ -502,7 +515,8 @@
pas_scavenger_clear_all_caches_except_remote_tlcs();
pas_thread_local_cache_for_all(pas_allocator_scavenge_force_stop_action,
- pas_deallocator_scavenge_flush_log_action);
+ pas_deallocator_scavenge_flush_log_action,
+ pas_thread_local_cache_decommit_if_possible_action);
}
void pas_scavenger_decommit_expendable_memory(void)
Modified: trunk/Source/bmalloc/libpas/src/libpas/pas_thread_local_cache.c (292512 => 292513)
--- trunk/Source/bmalloc/libpas/src/libpas/pas_thread_local_cache.c 2022-04-06 22:41:39 UTC (rev 292512)
+++ trunk/Source/bmalloc/libpas/src/libpas/pas_thread_local_cache.c 2022-04-06 22:47:34 UTC (rev 292513)
@@ -1022,7 +1022,8 @@
}
bool pas_thread_local_cache_for_all(pas_allocator_scavenge_action allocator_action,
- pas_deallocator_scavenge_action deallocator_action)
+ pas_deallocator_scavenge_action deallocator_action,
+ pas_thread_local_cache_decommit_action thread_local_cache_decommit_action)
{
pas_thread_local_cache_node* node;
bool result;
@@ -1087,8 +1088,6 @@
pas_thread_local_cache_layout_segment* segment;
pas_thread_local_cache_layout_segment* begin_segment;
uintptr_t start_of_possible_decommit;
- uintptr_t begin_page_index;
- uintptr_t end_page_index;
uintptr_t page_index;
uintptr_t node_index;
uintptr_t begin_node_index;
@@ -1106,9 +1105,7 @@
node_index = 0;
for (PAS_THREAD_LOCAL_CACHE_LAYOUT_EACH_ALLOCATOR_WITH_SEGMENT_AND_INDEX(layout_node, segment, node_index)) {
pas_allocator_index allocator_index;
- pas_allocator_index end_allocator_index;
pas_local_allocator_scavenger_data* scavenger_data;
- pas_decommit_exclusion_range decommit_exclusion_range;
allocator_index =
pas_thread_local_cache_layout_node_get_allocator_index_generic(layout_node);
@@ -1116,45 +1113,44 @@
if (allocator_index >= cache->allocator_index_upper_bound)
break;
- end_allocator_index =
- allocator_index + pas_thread_local_cache_layout_node_num_allocator_indices(layout_node);
-
scavenger_data = (pas_local_allocator_scavenger_data*)
pas_thread_local_cache_get_local_allocator_direct(cache, allocator_index);
-
- decommit_exclusion_range = pas_thread_local_cache_compute_decommit_exclusion_range(
- cache, allocator_index, end_allocator_index);
- PAS_ASSERT(start_of_possible_decommit
- <= decommit_exclusion_range.start_of_possible_decommit);
- PAS_ASSERT(start_of_possible_decommit
- <= decommit_exclusion_range.end_of_possible_decommit);
- if (pas_decommit_exclusion_range_is_contiguous(decommit_exclusion_range)) {
- stop_allocator(
- cache, allocator_action, allocator_index, scavenger_data, &result, &thread_suspend_data);
- if (pas_local_allocator_scavenger_data_is_stopped(scavenger_data)) {
- if (!begin_segment) {
- begin_segment = segment;
- begin_node_index = node_index;
+ if (thread_local_cache_decommit_action == pas_thread_local_cache_decommit_if_possible_action) {
+ pas_allocator_index end_allocator_index = allocator_index + pas_thread_local_cache_layout_node_num_allocator_indices(layout_node);
+ pas_decommit_exclusion_range decommit_exclusion_range = pas_thread_local_cache_compute_decommit_exclusion_range(cache, allocator_index, end_allocator_index);
+ PAS_ASSERT(start_of_possible_decommit <= decommit_exclusion_range.start_of_possible_decommit);
+ PAS_ASSERT(start_of_possible_decommit <= decommit_exclusion_range.end_of_possible_decommit);
+
+ if (pas_decommit_exclusion_range_is_contiguous(decommit_exclusion_range)) {
+ stop_allocator(
+ cache, allocator_action, allocator_index, scavenger_data, &result, &thread_suspend_data);
+
+ if (pas_local_allocator_scavenger_data_is_stopped(scavenger_data)) {
+ if (!begin_segment) {
+ begin_segment = segment;
+ begin_node_index = node_index;
+ } else {
+ PAS_TESTING_ASSERT(
+ pas_thread_local_cache_layout_node_get_allocator_index_generic(pas_thread_local_cache_layout_segment_get_node(begin_segment, begin_node_index))
+ < allocator_index);
+ }
} else {
- PAS_TESTING_ASSERT(
- pas_thread_local_cache_layout_node_get_allocator_index_generic(pas_thread_local_cache_layout_segment_get_node(begin_segment, begin_node_index))
- < allocator_index);
+ decommit_exclusion_range =
+ pas_decommit_exclusion_range_create_inverted(decommit_exclusion_range);
}
- } else {
- decommit_exclusion_range =
- pas_decommit_exclusion_range_create_inverted(decommit_exclusion_range);
}
- }
- if (pas_decommit_exclusion_range_is_inverted(decommit_exclusion_range)) {
- decommit_allocator_range(
- cache, begin_segment, begin_node_index, start_of_possible_decommit,
- decommit_exclusion_range.end_of_possible_decommit, segment, node_index);
- start_of_possible_decommit = decommit_exclusion_range.start_of_possible_decommit;
- begin_segment = NULL;
- begin_node_index = 0;
- }
+ if (pas_decommit_exclusion_range_is_inverted(decommit_exclusion_range)) {
+ decommit_allocator_range(
+ cache, begin_segment, begin_node_index, start_of_possible_decommit,
+ decommit_exclusion_range.end_of_possible_decommit, segment, node_index);
+ start_of_possible_decommit = decommit_exclusion_range.start_of_possible_decommit;
+ begin_segment = NULL;
+ begin_node_index = 0;
+ }
+ } else
+ stop_allocator(cache, allocator_action, allocator_index, scavenger_data, &result, &thread_suspend_data);
}
#if PAS_OS(DARWIN)
@@ -1162,29 +1158,31 @@
resume(cache, &thread_suspend_data);
#endif
- begin_page_index =
- pas_round_up_to_power_of_2(start_of_possible_decommit, page_size)
- >> page_size_shift;
- end_page_index =
- pas_thread_local_cache_size_for_allocator_index_capacity(cache->allocator_index_capacity)
- >> page_size_shift;
+ if (thread_local_cache_decommit_action == pas_thread_local_cache_decommit_if_possible_action) {
+ uintptr_t begin_page_index =
+ pas_round_up_to_power_of_2(start_of_possible_decommit, page_size)
+ >> page_size_shift;
+ uintptr_t end_page_index =
+ pas_thread_local_cache_size_for_allocator_index_capacity(cache->allocator_index_capacity)
+ >> page_size_shift;
- for (page_index = begin_page_index; page_index < end_page_index; ++page_index) {
- if (!pas_bitvector_get(cache->pages_committed, page_index)) {
- decommit_allocator_range(
- cache, begin_segment, begin_node_index, start_of_possible_decommit,
- page_index << page_size_shift,
- segment, node_index);
- start_of_possible_decommit = (page_index + 1) << page_size_shift;
- begin_segment = NULL;
- begin_node_index = 0;
+ for (page_index = begin_page_index; page_index < end_page_index; ++page_index) {
+ if (!pas_bitvector_get(cache->pages_committed, page_index)) {
+ decommit_allocator_range(
+ cache, begin_segment, begin_node_index, start_of_possible_decommit,
+ page_index << page_size_shift,
+ segment, node_index);
+ start_of_possible_decommit = (page_index + 1) << page_size_shift;
+ begin_segment = NULL;
+ begin_node_index = 0;
+ }
}
+
+ decommit_allocator_range(
+ cache, begin_segment, begin_node_index, start_of_possible_decommit,
+ pas_thread_local_cache_size_for_allocator_index_capacity(cache->allocator_index_capacity),
+ segment, node_index);
}
-
- decommit_allocator_range(
- cache, begin_segment, begin_node_index, start_of_possible_decommit,
- pas_thread_local_cache_size_for_allocator_index_capacity(cache->allocator_index_capacity),
- segment, node_index);
}
pas_lock_unlock(&node->scavenger_lock);
Modified: trunk/Source/bmalloc/libpas/src/libpas/pas_thread_local_cache.h (292512 => 292513)
--- trunk/Source/bmalloc/libpas/src/libpas/pas_thread_local_cache.h 2022-04-06 22:41:39 UTC (rev 292512)
+++ trunk/Source/bmalloc/libpas/src/libpas/pas_thread_local_cache.h 2022-04-06 22:47:34 UTC (rev 292513)
@@ -372,10 +372,17 @@
pas_thread_local_cache* thread_local_cache,
pas_lock_hold_mode heap_lock_hold_mode);
+enum pas_thread_local_cache_decommit_action {
+ pas_thread_local_cache_decommit_no_action,
+ pas_thread_local_cache_decommit_if_possible_action,
+};
+typedef enum pas_thread_local_cache_decommit_action pas_thread_local_cache_decommit_action;
+
/* Returns true if it's possible that we'll be able to return more memory if this was called
again. */
PAS_API bool pas_thread_local_cache_for_all(pas_allocator_scavenge_action allocator_action,
- pas_deallocator_scavenge_action deallocator_action);
+ pas_deallocator_scavenge_action deallocator_action,
+ pas_thread_local_cache_decommit_action thread_local_cache_decommit_action);
PAS_API PAS_NEVER_INLINE void pas_thread_local_cache_append_deallocation_slow(
pas_thread_local_cache* thread_local_cache,
Modified: trunk/Source/bmalloc/libpas/src/test/TLCDecommitTests.cpp (292512 => 292513)
--- trunk/Source/bmalloc/libpas/src/test/TLCDecommitTests.cpp 2022-04-06 22:41:39 UTC (rev 292512)
+++ trunk/Source/bmalloc/libpas/src/test/TLCDecommitTests.cpp 2022-04-06 22:47:34 UTC (rev 292513)
@@ -157,7 +157,8 @@
pas_heap_lock_unlock();
pas_thread_local_cache_for_all(pas_allocator_scavenge_request_stop_action,
- pas_deallocator_scavenge_no_action);
+ pas_deallocator_scavenge_no_action,
+ pas_thread_local_cache_decommit_if_possible_action);
CHECK_EQUAL(numCommittedPagesInTLC(), numCommittedPagesAfterDecommit);
@@ -296,7 +297,8 @@
CHECK(pas_thread_local_cache_try_get()->deallocation_log_index);
pas_thread_local_cache_for_all(pas_allocator_scavenge_force_stop_action,
- pas_deallocator_scavenge_no_action);
+ pas_deallocator_scavenge_no_action,
+ pas_thread_local_cache_decommit_if_possible_action);
CHECK_EQUAL(numCommittedPagesInTLC(), 1);
CHECK(pas_thread_local_cache_try_get()->deallocation_log_index);
@@ -340,7 +342,8 @@
CHECK(!pas_thread_local_cache_try_get()->deallocation_log_index);
pas_thread_local_cache_for_all(pas_allocator_scavenge_force_stop_action,
- pas_deallocator_scavenge_no_action);
+ pas_deallocator_scavenge_no_action,
+ pas_thread_local_cache_decommit_if_possible_action);
for (void* object : objects)
bmalloc_deallocate(object);
