wingo pushed a commit to branch wip-whippet
in repository guile.
commit b5c36b9fd857674f951dcd4892d98ddd8f429d36
Author: Andy Wingo <wi...@igalia.com>
AuthorDate: Fri Oct 4 11:40:09 2024 +0200
Explicitly support immediate values
Because we have to deref edges ourselves, as part of generational
marking, we need to ignore edges that don't point to heap objects.
---
api/gc-config.h | 4 ++++
api/gc-ref.h | 15 ++++++++++++++-
src/field-set.h | 12 ++++++------
src/gc-finalizer.c | 5 ++---
src/mmc.c | 28 +++++++++++++++-------------
src/nofl-space.h | 2 +-
src/parallel-tracer.h | 8 ++++----
src/pcc.c | 10 ++++++----
src/semi.c | 5 ++++-
src/serial-tracer.h | 2 +-
10 files changed, 57 insertions(+), 34 deletions(-)
diff --git a/api/gc-config.h b/api/gc-config.h
index 91dd555e2..ca1b38d14 100644
--- a/api/gc-config.h
+++ b/api/gc-config.h
@@ -5,6 +5,10 @@
#define GC_DEBUG 0
#endif
+#ifndef GC_HAS_IMMEDIATES
+#define GC_HAS_IMMEDIATES 1
+#endif
+
#ifndef GC_PARALLEL
#define GC_PARALLEL 0
#endif
diff --git a/api/gc-ref.h b/api/gc-ref.h
index 33ac5e73b..29e1a3853 100644
--- a/api/gc-ref.h
+++ b/api/gc-ref.h
@@ -2,6 +2,7 @@
#define GC_REF_H
#include "gc-assert.h"
+#include "gc-config.h"
#include <stdint.h>
@@ -19,8 +20,20 @@ static inline uintptr_t gc_ref_value(struct gc_ref ref) {
static inline struct gc_ref gc_ref_null(void) {
return gc_ref(0);
}
+static inline int gc_ref_is_null(struct gc_ref ref) {
+ return ref.value == 0;
+}
+static inline int gc_ref_is_immediate(struct gc_ref ref) {
+ GC_ASSERT(!gc_ref_is_null(ref));
+ return GC_HAS_IMMEDIATES && (ref.value & (sizeof(void*) - 1));
+}
+static inline struct gc_ref gc_ref_immediate(uintptr_t val) {
+ GC_ASSERT(val & (sizeof(void*) - 1));
+ GC_ASSERT(GC_HAS_IMMEDIATES);
+ return gc_ref(val);
+}
static inline int gc_ref_is_heap_object(struct gc_ref ref) {
- return ref.value != 0;
+ return !gc_ref_is_immediate(ref);
}
static inline struct gc_ref gc_ref_from_heap_object_or_null(void *obj) {
return gc_ref((uintptr_t) obj);
diff --git a/src/field-set.h b/src/field-set.h
index c7ddffd08..2c93232c1 100644
--- a/src/field-set.h
+++ b/src/field-set.h
@@ -162,15 +162,15 @@ gc_field_set_clear(struct gc_field_set *set,
}
static inline void
-gc_field_set_trace_edge_buffer(struct gc_field_set *set,
+gc_field_set_visit_edge_buffer(struct gc_field_set *set,
struct gc_edge_buffer *buf,
- void (*tracer_visit)(struct gc_edge,
- struct gc_heap*,
- void *data),
+ void (*visit)(struct gc_edge,
+ struct gc_heap*,
+ void *data),
struct gc_heap *heap,
- struct gc_trace_worker *worker) {
+ void *data) {
for (size_t i = 0; i < buf->size; i++)
- tracer_visit(buf->edges[i], heap, worker);
+ visit(buf->edges[i], heap, data);
}
static void
diff --git a/src/gc-finalizer.c b/src/gc-finalizer.c
index c0e5831bf..5365899fc 100644
--- a/src/gc-finalizer.c
+++ b/src/gc-finalizer.c
@@ -164,8 +164,7 @@ void gc_finalizer_init_internal(struct gc_finalizer *f,
// value.
if (f->state != FINALIZER_STATE_INIT)
GC_CRASH();
- if (gc_ref_is_heap_object(f->object))
- GC_CRASH();
+ GC_ASSERT(gc_ref_is_null(f->object));
f->object = object;
f->closure = closure;
}
@@ -179,7 +178,7 @@ void gc_finalizer_attach_internal(struct gc_finalizer_state
*state,
// value.
if (f->state != FINALIZER_STATE_INIT)
GC_CRASH();
- if (!gc_ref_is_heap_object(f->object))
+ if (gc_ref_is_null(f->object))
GC_CRASH();
f->state = FINALIZER_STATE_ACTIVE;
diff --git a/src/mmc.c b/src/mmc.c
index b1f4238a7..e5d5a95be 100644
--- a/src/mmc.c
+++ b/src/mmc.c
@@ -121,8 +121,6 @@ gc_trace_worker_call_with_data(void (*f)(struct gc_tracer
*tracer,
static inline int
do_trace(struct gc_heap *heap, struct gc_edge edge, struct gc_ref ref,
struct gc_trace_worker_data *data) {
- if (!gc_ref_is_heap_object(ref))
- return 0;
if (GC_LIKELY(nofl_space_contains(heap_nofl_space(heap), ref)))
return nofl_space_evacuate_or_mark_object(heap_nofl_space(heap), edge, ref,
&data->allocator);
@@ -137,6 +135,9 @@ static inline int
trace_edge(struct gc_heap *heap, struct gc_edge edge,
struct gc_trace_worker_data *data) {
struct gc_ref ref = gc_edge_ref(edge);
+ if (gc_ref_is_null(ref) || gc_ref_is_immediate(ref))
+ return 0;
+
int is_new = do_trace(heap, edge, ref, data);
if (is_new &&
@@ -150,9 +151,10 @@ trace_edge(struct gc_heap *heap, struct gc_edge edge,
int
gc_visit_ephemeron_key(struct gc_edge edge, struct gc_heap *heap) {
struct gc_ref ref = gc_edge_ref(edge);
- if (!gc_ref_is_heap_object(ref))
- return 0;
-
+ GC_ASSERT(!gc_ref_is_null(ref));
+ if (gc_ref_is_immediate(ref))
+ return 1;
+ GC_ASSERT(gc_ref_is_heap_object(ref));
struct nofl_space *nofl_space = heap_nofl_space(heap);
if (GC_LIKELY(nofl_space_contains(nofl_space, ref)))
return nofl_space_forward_or_mark_if_traced(nofl_space, edge, ref);
@@ -271,11 +273,11 @@ static inline struct gc_ref
trace_conservative_ref(struct gc_heap *heap, struct gc_conservative_ref ref,
int possibly_interior) {
struct gc_ref ret = do_trace_conservative_ref(heap, ref, possibly_interior);
-
- if (gc_ref_is_heap_object(ret) &&
- GC_UNLIKELY(atomic_load_explicit(&heap->check_pending_ephemerons,
- memory_order_relaxed)))
- gc_resolve_pending_ephemerons(ret, heap);
+ if (!gc_ref_is_null(ret)) {
+ if (GC_UNLIKELY(atomic_load_explicit(&heap->check_pending_ephemerons,
+ memory_order_relaxed)))
+ gc_resolve_pending_ephemerons(ret, heap);
+ }
return ret;
}
@@ -286,7 +288,7 @@ tracer_trace_conservative_ref(struct gc_conservative_ref
ref,
struct gc_trace_worker *worker,
int possibly_interior) {
struct gc_ref resolved = trace_conservative_ref(heap, ref,
possibly_interior);
- if (gc_ref_is_heap_object(resolved))
+ if (!gc_ref_is_null(resolved))
gc_trace_worker_enqueue(worker, resolved);
}
@@ -367,7 +369,7 @@ trace_root(struct gc_root root, struct gc_heap *heap,
tracer_visit(root.edge, heap, worker);
break;
case GC_ROOT_KIND_EDGE_BUFFER:
- gc_field_set_trace_edge_buffer(&heap->remembered_set, root.edge_buffer,
+ gc_field_set_visit_edge_buffer(&heap->remembered_set, root.edge_buffer,
tracer_visit, heap, worker);
break;
default:
@@ -910,7 +912,7 @@ void
gc_write_barrier_slow(struct gc_mutator *mut, struct gc_ref obj,
size_t obj_size, struct gc_edge edge,
struct gc_ref new_val) {
- GC_ASSERT(gc_ref_is_heap_object(new_val));
+ GC_ASSERT(!gc_ref_is_null(new_val));
if (!GC_GENERATIONAL) return;
if (gc_object_is_old_generation_slow(mut, new_val))
return;
diff --git a/src/nofl-space.h b/src/nofl-space.h
index 9759b3d8e..bc5144205 100644
--- a/src/nofl-space.h
+++ b/src/nofl-space.h
@@ -1483,7 +1483,7 @@ nofl_space_evacuate(struct nofl_space *space, uint8_t
*metadata, uint8_t byte,
size_t object_granules = nofl_space_live_object_granules(metadata);
struct gc_ref new_ref = nofl_evacuation_allocate(evacuate, space,
object_granules);
- if (gc_ref_is_heap_object(new_ref)) {
+ if (!gc_ref_is_null(new_ref)) {
// Copy object contents before committing, as we don't know what
// part of the object (if any) will be overwritten by the
// commit.
diff --git a/src/parallel-tracer.h b/src/parallel-tracer.h
index 20d66730f..8115c369d 100644
--- a/src/parallel-tracer.h
+++ b/src/parallel-tracer.h
@@ -207,7 +207,7 @@ trace_worker_steal_from_any(struct gc_trace_worker *worker,
for (size_t i = 0; i < tracer->worker_count; i++) {
LOG("tracer #%zu: stealing from #%zu\n", worker->id, worker->steal_id);
struct gc_ref obj = tracer_steal_from_worker(tracer, worker->steal_id);
- if (gc_ref_is_heap_object(obj)) {
+ if (!gc_ref_is_null(obj)) {
LOG("tracer #%zu: stealing got %p\n", worker->id,
gc_ref_heap_object(obj));
return obj;
@@ -281,13 +281,13 @@ trace_worker_steal(struct gc_trace_worker *worker) {
{
LOG("tracer #%zu: trying to pop worker's own deque\n", worker->id);
struct gc_ref obj = shared_worklist_try_pop(&worker->shared);
- if (gc_ref_is_heap_object(obj))
+ if (!gc_ref_is_null(obj))
return obj;
}
LOG("tracer #%zu: trying to steal\n", worker->id);
struct gc_ref obj = trace_worker_steal_from_any(worker, tracer);
- if (gc_ref_is_heap_object(obj))
+ if (!gc_ref_is_null(obj))
return obj;
return gc_ref_null();
@@ -337,7 +337,7 @@ trace_with_data(struct gc_tracer *tracer,
ref = local_worklist_pop(&worker->local);
} else {
ref = trace_worker_steal(worker);
- if (!gc_ref_is_heap_object(ref))
+ if (gc_ref_is_null(ref))
break;
}
trace_one(ref, heap, worker);
diff --git a/src/pcc.c b/src/pcc.c
index 593f43fc6..6b8a55c03 100644
--- a/src/pcc.c
+++ b/src/pcc.c
@@ -103,8 +103,6 @@ gc_trace_worker_call_with_data(void (*f)(struct gc_tracer
*tracer,
static inline int do_trace(struct gc_heap *heap, struct gc_edge edge,
struct gc_ref ref,
struct gc_trace_worker_data *data) {
- if (!gc_ref_is_heap_object(ref))
- return 0;
if (GC_LIKELY(copy_space_contains(heap_copy_space(heap), ref)))
return copy_space_forward(heap_copy_space(heap), edge, ref,
&data->allocator);
@@ -117,6 +115,8 @@ static inline int do_trace(struct gc_heap *heap, struct
gc_edge edge,
static inline int trace_edge(struct gc_heap *heap, struct gc_edge edge,
struct gc_trace_worker *worker) {
struct gc_ref ref = gc_edge_ref(edge);
+ if (gc_ref_is_null(ref) || gc_ref_is_immediate(ref))
+ return 0;
struct gc_trace_worker_data *data = gc_trace_worker_data(worker);
int is_new = do_trace(heap, edge, ref, data);
@@ -130,8 +130,10 @@ static inline int trace_edge(struct gc_heap *heap, struct
gc_edge edge,
int gc_visit_ephemeron_key(struct gc_edge edge, struct gc_heap *heap) {
struct gc_ref ref = gc_edge_ref(edge);
- if (!gc_ref_is_heap_object(ref))
- return 0;
+ GC_ASSERT(!gc_ref_is_null(ref));
+ if (gc_ref_is_immediate(ref))
+ return 1;
+ GC_ASSERT(gc_ref_is_heap_object(ref));
if (GC_LIKELY(copy_space_contains(heap_copy_space(heap), ref)))
return copy_space_forward_if_traced(heap_copy_space(heap), edge, ref);
if (large_object_space_contains(heap_large_object_space(heap), ref))
diff --git a/src/semi.c b/src/semi.c
index ca7a31607..239367999 100644
--- a/src/semi.c
+++ b/src/semi.c
@@ -233,7 +233,7 @@ static void visit_external_object(struct gc_heap *heap,
static void visit(struct gc_edge edge, struct gc_heap *heap) {
struct gc_ref ref = gc_edge_ref(edge);
- if (!gc_ref_is_heap_object(ref))
+ if (gc_ref_is_null(ref) || gc_ref_is_immediate(ref))
return;
if (semi_space_contains(heap_semi_space(heap), ref))
visit_semi_space(heap, heap_semi_space(heap), edge, ref);
@@ -250,6 +250,9 @@ gc_heap_pending_ephemerons(struct gc_heap *heap) {
int gc_visit_ephemeron_key(struct gc_edge edge, struct gc_heap *heap) {
struct gc_ref ref = gc_edge_ref(edge);
+ GC_ASSERT(!gc_ref_is_null(ref));
+ if (gc_ref_is_immediate(ref))
+ return 1;
GC_ASSERT(gc_ref_is_heap_object(ref));
if (semi_space_contains(heap_semi_space(heap), ref)) {
uintptr_t forwarded = gc_object_forwarded_nonatomic(ref);
diff --git a/src/serial-tracer.h b/src/serial-tracer.h
index b9575fddb..a3289e30c 100644
--- a/src/serial-tracer.h
+++ b/src/serial-tracer.h
@@ -65,7 +65,7 @@ tracer_trace_with_data(struct gc_tracer *tracer, struct
gc_heap *heap,
if (!tracer->trace_roots_only) {
do {
struct gc_ref obj = simple_worklist_pop(&tracer->worklist);
- if (!gc_ref_is_heap_object(obj))
+ if (gc_ref_is_null(obj))
break;
trace_one(obj, heap, worker);
} while (1);
