wingo pushed a commit to branch wip-whippet
in repository guile.
commit 1944b54a192f104d6c32bcc2ac7579ceac1d58c6
Author: Andy Wingo <wi...@igalia.com>
AuthorDate: Wed Sep 21 10:55:26 2022 +0200
Whippet can trace conservative roots
Next up, enabling it via the makefiles.
---
conservative-roots-embedder.h | 34 ++++--
gc-conservative-ref.h | 17 +++
gc-embedder-api.h | 16 ++-
large-object-space.h | 38 +++++++
precise-roots-embedder.h | 26 +++--
semi.c | 2 +-
simple-gc-embedder.h | 6 ++
whippet.c | 245 +++++++++++++++++++++++++++++++++---------
8 files changed, 311 insertions(+), 73 deletions(-)
diff --git a/conservative-roots-embedder.h b/conservative-roots-embedder.h
index 2ac2d2b78..15447a2c4 100644
--- a/conservative-roots-embedder.h
+++ b/conservative-roots-embedder.h
@@ -16,16 +16,34 @@ static inline int gc_has_conservative_intraheap_edges(void)
{
return 0;
}
-static inline void gc_trace_precise_mutator_roots(struct gc_mutator_roots
*roots,
- void (*trace_edge)(struct
gc_edge edge,
- void
*trace_data),
- void *trace_data) {
+static inline int
+gc_is_valid_conservative_ref_displacement(uintptr_t displacement) {
+ // Here is where you would allow tagged heap object references.
+ return displacement == 0;
+}
+static inline int
+gc_conservative_ref_might_be_a_heap_object(struct gc_conservative_ref ref,
+ int possibly_interior) {
+ // Assume that the minimum page size is 4096, and that the first page
+ // will contain no heap objects.
+ if (gc_conservative_ref_value(ref) < 4096)
+ return 0;
+ if (possibly_interior)
+ return 1;
+ return gc_is_valid_conservative_ref_displacement
+ (gc_conservative_ref_value(ref) & (sizeof(uintptr_t) - 1));
+}
+
+static inline void gc_trace_mutator_roots(struct gc_mutator_roots *roots,
+ void (*trace_edge)(struct gc_edge
edge,
+ void *trace_data),
+ void *trace_data) {
}
-static inline void gc_trace_precise_heap_roots(struct gc_heap_roots *roots,
- void (*trace_edge)(struct
gc_edge edge,
- void
*trace_data),
- void *trace_data) {
+static inline void gc_trace_heap_roots(struct gc_heap_roots *roots,
+ void (*trace_edge)(struct gc_edge edge,
+ void *trace_data),
+ void *trace_data) {
}
#endif // CONSERVATIVE_ROOTS_EMBEDDER_H
diff --git a/gc-conservative-ref.h b/gc-conservative-ref.h
new file mode 100644
index 000000000..a2b260384
--- /dev/null
+++ b/gc-conservative-ref.h
@@ -0,0 +1,17 @@
+#ifndef GC_CONSERVATIVE_REF_H
+#define GC_CONSERVATIVE_REF_H
+
+#include <stdint.h>
+
+struct gc_conservative_ref {
+ uintptr_t value;
+};
+
+static inline struct gc_conservative_ref gc_conservative_ref(uintptr_t value) {
+ return (struct gc_conservative_ref){value};
+}
+static inline uintptr_t gc_conservative_ref_value(struct gc_conservative_ref
ref) {
+ return ref.value;
+}
+
+#endif // GC_CONSERVATIVE_REF_H
diff --git a/gc-embedder-api.h b/gc-embedder-api.h
index b24483245..2d74ed0a4 100644
--- a/gc-embedder-api.h
+++ b/gc-embedder-api.h
@@ -1,6 +1,7 @@
#ifndef GC_EMBEDDER_API_H
#define GC_EMBEDDER_API_H
+#include "gc-conservative-ref.h"
#include "gc-edge.h"
#include "gc-forwarding.h"
@@ -17,19 +18,24 @@ GC_EMBEDDER_API inline int
gc_has_global_conservative_roots(void);
GC_EMBEDDER_API inline int gc_has_conservative_intraheap_edges(void);
GC_EMBEDDER_API inline int gc_mutator_conservative_roots_may_be_interior(void);
+GC_EMBEDDER_API inline int gc_is_valid_conservative_ref_displacement(uintptr_t
displacement);
+GC_EMBEDDER_API inline int gc_conservative_ref_might_be_a_heap_object(struct
gc_conservative_ref,
+ int
possibly_interior);
+
GC_EMBEDDER_API inline void gc_trace_object(struct gc_ref ref,
void (*trace_edge)(struct gc_edge
edge,
void
*trace_data),
void *trace_data,
size_t *size) GC_ALWAYS_INLINE;
-GC_EMBEDDER_API inline void gc_trace_precise_mutator_roots(struct
gc_mutator_roots *roots,
+
+GC_EMBEDDER_API inline void gc_trace_mutator_roots(struct gc_mutator_roots
*roots,
void (*trace_edge)(struct
gc_edge edge,
void
*trace_data),
void *trace_data);
-GC_EMBEDDER_API inline void gc_trace_precise_heap_roots(struct gc_heap_roots
*roots,
- void
(*trace_edge)(struct gc_edge edge,
-
void *trace_data),
- void *trace_data);
+GC_EMBEDDER_API inline void gc_trace_heap_roots(struct gc_heap_roots *roots,
+ void (*trace_edge)(struct
gc_edge edge,
+ void
*trace_data),
+ void *trace_data);
GC_EMBEDDER_API inline uintptr_t gc_object_forwarded_nonatomic(struct gc_ref
ref);
GC_EMBEDDER_API inline void gc_object_forward_nonatomic(struct gc_ref ref,
diff --git a/large-object-space.h b/large-object-space.h
index 01bc4cfc7..6bb7a5af7 100644
--- a/large-object-space.h
+++ b/large-object-space.h
@@ -10,6 +10,7 @@
#include <unistd.h>
#include "gc-ref.h"
+#include "gc-conservative-ref.h"
#include "address-map.h"
#include "address-set.h"
@@ -90,6 +91,11 @@ done:
return copied;
}
+static int large_object_space_mark_object(struct large_object_space *space,
+ struct gc_ref ref) {
+ return large_object_space_copy(space, ref);
+}
+
static void large_object_space_reclaim_one(uintptr_t addr, void *data) {
struct large_object_space *space = data;
size_t npages = address_map_lookup(&space->object_pages, addr, 0);
@@ -145,6 +151,38 @@ static void large_object_space_finish_gc(struct
large_object_space *space,
pthread_mutex_unlock(&space->lock);
}
+static inline struct gc_ref
+large_object_space_mark_conservative_ref(struct large_object_space *space,
+ struct gc_conservative_ref ref,
+ int possibly_interior) {
+ uintptr_t addr = gc_conservative_ref_value(ref);
+
+ if (possibly_interior) {
+ // FIXME: This only allows interior pointers within the first page.
+ // BDW-GC doesn't have all-interior-pointers on for intraheap edges
+ // or edges originating in static data but by default does allow
+ // them from stack edges; probably we should too.
+ addr &= ~(space->page_size - 1);
+ } else {
+ // Addr not aligned on page boundary? Not a large object.
+ uintptr_t displacement = addr & (space->page_size - 1);
+ if (!gc_is_valid_conservative_ref_displacement(displacement))
+ return gc_ref_null();
+ addr -= displacement;
+ }
+
+ pthread_mutex_lock(&space->lock);
+ // ptr might be in fromspace or tospace. Just check the object_pages table,
which
+ // contains both, as well as object_pages for free blocks.
+ int found = address_map_contains(&space->object_pages, addr);
+ pthread_mutex_unlock(&space->lock);
+
+ if (found && large_object_space_copy(space, gc_ref(addr)))
+ return gc_ref(addr);
+
+ return gc_ref_null();
+}
+
static inline int large_object_space_contains(struct large_object_space *space,
struct gc_ref ref) {
pthread_mutex_lock(&space->lock);
diff --git a/precise-roots-embedder.h b/precise-roots-embedder.h
index cf649e14d..bde6be36e 100644
--- a/precise-roots-embedder.h
+++ b/precise-roots-embedder.h
@@ -18,6 +18,16 @@ static inline int gc_has_conservative_intraheap_edges(void) {
return 0;
}
+static inline int
+gc_is_valid_conservative_ref_displacement(uintptr_t displacement) {
+ GC_CRASH();
+}
+static inline int
+gc_conservative_ref_might_be_a_heap_object(struct gc_conservative_ref ref,
+ int possibly_interior) {
+ GC_CRASH();
+}
+
static inline void visit_roots(struct handle *roots,
void (*trace_edge)(struct gc_edge edge,
void *trace_data),
@@ -26,18 +36,18 @@ static inline void visit_roots(struct handle *roots,
trace_edge(gc_edge(&h->v), trace_data);
}
-static inline void gc_trace_precise_mutator_roots(struct gc_mutator_roots
*roots,
- void (*trace_edge)(struct
gc_edge edge,
- void
*trace_data),
- void *trace_data) {
+static inline void gc_trace_mutator_roots(struct gc_mutator_roots *roots,
+ void (*trace_edge)(struct gc_edge
edge,
+ void *trace_data),
+ void *trace_data) {
if (roots)
visit_roots(roots->roots, trace_edge, trace_data);
}
-static inline void gc_trace_precise_heap_roots(struct gc_heap_roots *roots,
- void (*trace_edge)(struct
gc_edge edge,
- void
*trace_data),
- void *trace_data) {
+static inline void gc_trace_heap_roots(struct gc_heap_roots *roots,
+ void (*trace_edge)(struct gc_edge edge,
+ void *trace_data),
+ void *trace_data) {
if (roots)
visit_roots(roots->roots, trace_edge, trace_data);
}
diff --git a/semi.c b/semi.c
index e771f7f3d..3ad765416 100644
--- a/semi.c
+++ b/semi.c
@@ -161,7 +161,7 @@ static void collect(struct gc_mutator *mut) {
flip(semi);
uintptr_t grey = semi->hp;
if (mut->roots)
- gc_trace_precise_mutator_roots(mut->roots, visit, heap);
+ gc_trace_mutator_roots(mut->roots, visit, heap);
// fprintf(stderr, "pushed %zd bytes in roots\n", space->hp - grey);
while(grey < semi->hp)
grey = scan(heap, gc_ref(grey));
diff --git a/simple-gc-embedder.h b/simple-gc-embedder.h
index 71255256d..457d9b09e 100644
--- a/simple-gc-embedder.h
+++ b/simple-gc-embedder.h
@@ -104,3 +104,9 @@ gc_atomic_forward_address(struct gc_atomic_forward *fwd) {
GC_ASSERT(fwd->state == GC_FORWARDING_STATE_FORWARDED);
return fwd->data;
}
+
+static inline uintptr_t
+gc_conservative_ref_heap_address(struct gc_conservative_ref ref) {
+ // The specific spaces are responsible for checking alignment.
+ return gc_conservative_ref_value(ref);
+}
diff --git a/whippet.c b/whippet.c
index ad1e02260..687d6e307 100644
--- a/whippet.c
+++ b/whippet.c
@@ -15,6 +15,7 @@
#include "debug.h"
#include "gc-align.h"
#include "gc-inline.h"
+#include "gc-platform.h"
#include "gc-stack.h"
#include "large-object-space.h"
#if GC_PARALLEL
@@ -597,15 +598,19 @@ static inline int
mark_space_evacuate_or_mark_object(struct mark_space *space,
return 1;
}
-static inline int mark_space_contains(struct mark_space *space,
- struct gc_ref ref) {
- uintptr_t addr = gc_ref_value(ref);
+static inline int mark_space_contains_address(struct mark_space *space,
+ uintptr_t addr) {
return addr - space->low_addr < space->extent;
}
-static inline int large_object_space_mark_object(struct large_object_space
*space,
- struct gc_ref ref) {
- return large_object_space_copy(space, ref);
+static inline int mark_space_contains_conservative_ref(struct mark_space
*space,
+ struct
gc_conservative_ref ref) {
+ return mark_space_contains_address(space, gc_conservative_ref_value(ref));
+}
+
+static inline int mark_space_contains(struct mark_space *space,
+ struct gc_ref ref) {
+ return mark_space_contains_address(space, gc_ref_value(ref));
}
static inline int trace_edge(struct gc_heap *heap, struct gc_edge edge) {
@@ -625,18 +630,84 @@ static inline int trace_edge(struct gc_heap *heap, struct
gc_edge edge) {
GC_CRASH();
}
-static inline int trace_ref(struct gc_heap *heap, struct gc_ref ref) {
- if (!gc_ref_is_heap_object(ref))
- return 0;
- if (GC_LIKELY(mark_space_contains(heap_mark_space(heap), ref))) {
- GC_ASSERT(!heap_mark_space(heap)->evacuating);
- return mark_space_mark_object(heap_mark_space(heap), ref);
+static inline struct gc_ref mark_space_mark_conservative_ref(struct mark_space
*space,
+ struct
gc_conservative_ref ref,
+ int
possibly_interior) {
+ uintptr_t addr = gc_conservative_ref_value(ref);
+
+ if (possibly_interior) {
+ addr = align_down(addr, GRANULE_SIZE);
+ } else {
+ // Addr not an aligned granule? Not an object.
+ uintptr_t displacement = addr & (GRANULE_SIZE - 1);
+ if (!gc_is_valid_conservative_ref_displacement(displacement))
+ return gc_ref_null();
+ addr -= displacement;
}
- else if (large_object_space_contains(heap_large_object_space(heap), ref))
- return large_object_space_mark_object(heap_large_object_space(heap),
- ref);
+
+ // Addr in meta block? Not an object.
+ if ((addr & (SLAB_SIZE - 1)) < META_BLOCKS_PER_SLAB * BLOCK_SIZE)
+ return gc_ref_null();
+
+ // Addr in block that has been paged out? Not an object.
+ struct block_summary *summary = block_summary_for_addr(addr);
+ if (block_summary_has_flag(summary, BLOCK_UNAVAILABLE))
+ return gc_ref_null();
+
+ uint8_t *loc = metadata_byte_for_addr(addr);
+ uint8_t byte = atomic_load_explicit(loc, memory_order_relaxed);
+
+ // Already marked object? Nothing to do.
+ if (byte & space->marked_mask)
+ return gc_ref_null();
+
+ // Addr is the not start of an unmarked object? Search backwards if
+ // we have interior pointers, otherwise not an object.
+ uint8_t object_start_mask = space->live_mask | METADATA_BYTE_YOUNG;
+ if (!(byte & object_start_mask)) {
+ if (!possibly_interior)
+ return gc_ref_null();
+
+ uintptr_t block_base = align_down(addr, BLOCK_SIZE);
+ uint8_t *loc_base = metadata_byte_for_addr(block_base);
+ do {
+ // Searched past block? Not an object.
+ if (loc-- == loc_base)
+ return gc_ref_null();
+
+ byte = atomic_load_explicit(loc, memory_order_relaxed);
+
+ // Ran into the end of some other allocation? Not an object, then.
+ if (byte & METADATA_BYTE_END)
+ return gc_ref_null();
+
+ // Continue until we find object start.
+ } while (!(byte & object_start_mask));
+
+ // Found object start, and object is unmarked; adjust addr.
+ addr = block_base + (loc - loc_base) * GRANULE_SIZE;
+ }
+
+ uint8_t mask = METADATA_BYTE_YOUNG | METADATA_BYTE_MARK_0
+ | METADATA_BYTE_MARK_1 | METADATA_BYTE_MARK_2;
+ atomic_store_explicit(loc, (byte & ~mask) | space->marked_mask,
+ memory_order_relaxed);
+
+ return gc_ref(addr);
+}
+
+static inline struct gc_ref trace_conservative_ref(struct gc_heap *heap,
+ struct gc_conservative_ref
ref,
+ int possibly_interior) {
+ if (!gc_conservative_ref_might_be_a_heap_object(ref, possibly_interior))
+ return gc_ref_null();
+
+ if (GC_LIKELY(mark_space_contains_conservative_ref(heap_mark_space(heap),
ref)))
+ return mark_space_mark_conservative_ref(heap_mark_space(heap), ref,
+ possibly_interior);
else
- GC_CRASH();
+ return
large_object_space_mark_conservative_ref(heap_large_object_space(heap),
+ ref, possibly_interior);
}
static inline void trace_one(struct gc_ref ref, void *mark_data) {
@@ -894,10 +965,24 @@ static void trace_and_enqueue_locally(struct gc_edge
edge, void *data) {
mutator_mark_buf_push(&mut->mark_buf, gc_edge_ref(edge));
}
-static void trace_ref_and_enqueue_locally(struct gc_ref ref, void *data) {
+static inline void do_trace_conservative_ref_and_enqueue_locally(struct
gc_conservative_ref ref,
+ void *data,
+ int
possibly_interior) {
struct gc_mutator *mut = data;
- if (trace_ref(mutator_heap(mut), ref))
- mutator_mark_buf_push(&mut->mark_buf, ref);
+ struct gc_ref object = trace_conservative_ref(mutator_heap(mut), ref,
+ possibly_interior);
+ if (gc_ref_is_heap_object(object))
+ mutator_mark_buf_push(&mut->mark_buf, object);
+}
+
+static void trace_possibly_interior_conservative_ref_and_enqueue_locally
+ (struct gc_conservative_ref ref, void *data) {
+ return do_trace_conservative_ref_and_enqueue_locally(ref, data, 1);
+}
+
+static void trace_conservative_ref_and_enqueue_locally
+ (struct gc_conservative_ref ref, void *data) {
+ return do_trace_conservative_ref_and_enqueue_locally(ref, data, 0);
}
static void trace_and_enqueue_globally(struct gc_edge edge, void *data) {
@@ -906,40 +991,105 @@ static void trace_and_enqueue_globally(struct gc_edge
edge, void *data) {
tracer_enqueue_root(&heap->tracer, gc_edge_ref(edge));
}
-static void trace_ref_and_enqueue_globally(struct gc_ref ref, void *data) {
+static inline void do_trace_conservative_ref_and_enqueue_globally(struct
gc_conservative_ref ref,
+ void *data,
+ int
possibly_interior) {
struct gc_heap *heap = data;
- if (trace_ref(heap, ref))
- tracer_enqueue_root(&heap->tracer, ref);
+ struct gc_ref object = trace_conservative_ref(heap, ref, possibly_interior);
+ if (gc_ref_is_heap_object(object))
+ tracer_enqueue_root(&heap->tracer, object);
+}
+
+static void
trace_possibly_interior_conservative_ref_and_enqueue_globally(struct
gc_conservative_ref ref,
+ void
*data) {
+ return do_trace_conservative_ref_and_enqueue_globally(ref, data, 1);
+}
+
+static void trace_conservative_ref_and_enqueue_globally(struct
gc_conservative_ref ref,
+ void *data) {
+ return do_trace_conservative_ref_and_enqueue_globally(ref, data, 0);
+}
+
+static inline struct gc_conservative_ref
+load_conservative_ref(uintptr_t addr) {
+ GC_ASSERT((addr & (sizeof(uintptr_t) - 1)) == 0);
+ uintptr_t val;
+ memcpy(&val, (char*)addr, sizeof(uintptr_t));
+ return gc_conservative_ref(val);
+}
+
+static inline void
+trace_conservative_edges(uintptr_t low,
+ uintptr_t high,
+ void (*trace)(struct gc_conservative_ref, void *),
+ void *data) {
+ GC_ASSERT(low == align_down(low, sizeof(uintptr_t)));
+ GC_ASSERT(high == align_down(high, sizeof(uintptr_t)));
+ for (uintptr_t addr = low; addr < high; addr += sizeof(uintptr_t))
+ trace(load_conservative_ref(addr), data);
+}
+
+static void
+mark_and_globally_enqueue_mutator_conservative_roots(uintptr_t low,
+ uintptr_t high,
+ void *data) {
+ trace_conservative_edges(low, high,
+ gc_mutator_conservative_roots_may_be_interior()
+ ?
trace_possibly_interior_conservative_ref_and_enqueue_globally
+ : trace_conservative_ref_and_enqueue_globally,
+ data);
+}
+
+static void
+mark_and_globally_enqueue_heap_conservative_roots(uintptr_t low,
+ uintptr_t high,
+ void *data) {
+ trace_conservative_edges(low, high,
+ trace_conservative_ref_and_enqueue_globally,
+ data);
+}
+
+static void
+mark_and_locally_enqueue_mutator_conservative_roots(uintptr_t low,
+ uintptr_t high,
+ void *data) {
+ trace_conservative_edges(low, high,
+ gc_mutator_conservative_roots_may_be_interior()
+ ?
trace_possibly_interior_conservative_ref_and_enqueue_locally
+ : trace_conservative_ref_and_enqueue_locally,
+ data);
+}
+
+static inline void
+trace_mutator_conservative_roots(struct gc_mutator *mut,
+ void (*trace_range)(uintptr_t low,
+ uintptr_t high,
+ void *data),
+ void *data) {
+ if (gc_has_mutator_conservative_roots())
+ gc_stack_visit(&mut->stack, trace_range, data);
}
// Mark the roots of a mutator that is stopping for GC. We can't
// enqueue them directly, so we send them to the controller in a buffer.
static void trace_stopping_mutator_roots(struct gc_mutator *mut) {
GC_ASSERT(mutator_should_mark_while_stopping(mut));
- /*
trace_mutator_conservative_roots(mut,
- mark_and_locally_enqueue_conservative_roots,
+
mark_and_locally_enqueue_mutator_conservative_roots,
mut);
- */
- gc_trace_precise_mutator_roots(mut->roots, trace_and_enqueue_locally, mut);
-}
-
-static void trace_precise_mutator_roots_with_lock(struct gc_mutator *mut) {
- gc_trace_precise_mutator_roots(mut->roots, trace_and_enqueue_globally,
- mutator_heap(mut));
+ gc_trace_mutator_roots(mut->roots, trace_and_enqueue_locally, mut);
}
static void trace_mutator_conservative_roots_with_lock(struct gc_mutator *mut)
{
- /*
trace_mutator_conservative_roots(mut,
-
mark_and_globally_enqueue_conservative_roots,
+
mark_and_globally_enqueue_mutator_conservative_roots,
mutator_heap(mut));
- */
}
static void trace_mutator_roots_with_lock(struct gc_mutator *mut) {
trace_mutator_conservative_roots_with_lock(mut);
- trace_precise_mutator_roots_with_lock(mut);
+ gc_trace_mutator_roots(mut->roots, trace_and_enqueue_globally,
+ mutator_heap(mut));
}
static void trace_mutator_roots_with_lock_before_stop(struct gc_mutator *mut) {
@@ -965,12 +1115,11 @@ static void finish_sweeping_in_block(struct gc_mutator
*mut);
static void trace_mutator_conservative_roots_after_stop(struct gc_heap *heap) {
int active_mutators_already_marked = heap_should_mark_while_stopping(heap);
- if (!active_mutators_already_marked) {
+ if (!active_mutators_already_marked)
for (struct gc_mutator *mut = atomic_load(&heap->mutator_trace_list);
mut;
mut = mut->next)
trace_mutator_conservative_roots_with_lock(mut);
- }
for (struct gc_mutator *mut = heap->deactivated_mutators;
mut;
@@ -978,7 +1127,7 @@ static void
trace_mutator_conservative_roots_after_stop(struct gc_heap *heap) {
trace_mutator_conservative_roots_with_lock(mut);
}
-static void trace_precise_mutator_roots_after_stop(struct gc_heap *heap) {
+static void trace_mutator_roots_after_stop(struct gc_heap *heap) {
struct gc_mutator *mut = atomic_load(&heap->mutator_trace_list);
int active_mutators_already_marked = heap_should_mark_while_stopping(heap);
while (mut) {
@@ -988,7 +1137,7 @@ static void trace_precise_mutator_roots_after_stop(struct
gc_heap *heap) {
tracer_enqueue_roots(&heap->tracer, mut->mark_buf.objects,
mut->mark_buf.size);
else
- trace_precise_mutator_roots_with_lock(mut);
+ trace_mutator_roots_with_lock(mut);
// Also unlink mutator_trace_list chain.
struct gc_mutator *next = mut->next;
mut->next = NULL;
@@ -998,20 +1147,14 @@ static void
trace_precise_mutator_roots_after_stop(struct gc_heap *heap) {
for (struct gc_mutator *mut = heap->deactivated_mutators; mut; mut =
mut->next) {
finish_sweeping_in_block(mut);
- trace_precise_mutator_roots_with_lock(mut);
+ trace_mutator_roots_with_lock(mut);
}
}
-static void trace_precise_global_roots(struct gc_heap *heap) {
- gc_trace_precise_heap_roots(heap->roots, trace_and_enqueue_globally, heap);
-}
-
static void trace_global_conservative_roots(struct gc_heap *heap) {
- /*
if (gc_has_global_conservative_roots())
gc_platform_visit_global_conservative_roots
- (mark_and_globally_enqueue_conservative_roots, heap);
- */
+ (mark_and_globally_enqueue_heap_conservative_roots, heap);
}
static inline uint64_t load_eight_aligned_bytes(uint8_t *mark) {
@@ -1456,9 +1599,9 @@ static void trace_pinned_roots_after_stop(struct gc_heap
*heap) {
trace_conservative_roots_after_stop(heap);
}
-static void trace_precise_roots_after_stop(struct gc_heap *heap) {
- trace_precise_mutator_roots_after_stop(heap);
- trace_precise_global_roots(heap);
+static void trace_roots_after_stop(struct gc_heap *heap) {
+ trace_mutator_roots_after_stop(heap);
+ gc_trace_heap_roots(heap->roots, trace_and_enqueue_globally, heap);
trace_generational_roots(heap);
}
@@ -1494,7 +1637,7 @@ static void collect(struct gc_mutator *mut) {
detect_out_of_memory(heap);
trace_pinned_roots_after_stop(heap);
prepare_for_evacuation(heap);
- trace_precise_roots_after_stop(heap);
+ trace_roots_after_stop(heap);
tracer_trace(heap);
tracer_release(heap);
mark_space_finish_gc(space, gc_kind);
