wingo pushed a commit to branch wip-whippet
in repository guile.
commit e27029024206380c1def273a5f902fbccd4927e7
Author: Andy Wingo <wi...@igalia.com>
AuthorDate: Wed Mar 15 09:34:12 2023 +0100
Allow large object space to be part of remembered set
---
bdw-attrs.h | 6 +++---
bdw.c | 4 ++++
gc-api.h | 18 +++++++++++++++---
gc-attrs.h | 9 +++++----
gc-embedder-api.h | 8 ++++++++
large-object-space.h | 43 +++++++++++++++++++++++++++++++++++++++++++
mt-gcbench.c | 6 +++---
semi-attrs.h | 6 +++---
semi.c | 4 ++++
simple-gc-embedder.h | 19 +++++++++++++++++++
simple-tagging-scheme.h | 8 +++++---
whippet-attrs.h | 13 ++++++++-----
whippet.c | 16 +++++++++++++++-
13 files changed, 135 insertions(+), 25 deletions(-)
diff --git a/bdw-attrs.h b/bdw-attrs.h
index 960e543b0..e7a08100d 100644
--- a/bdw-attrs.h
+++ b/bdw-attrs.h
@@ -40,13 +40,13 @@ static inline int gc_allocator_needs_clear(void) {
return 0;
}
-static inline enum gc_write_barrier_kind gc_small_write_barrier_kind(void) {
+static inline enum gc_write_barrier_kind gc_write_barrier_kind(size_t) {
return GC_WRITE_BARRIER_NONE;
}
-static inline size_t gc_small_write_barrier_card_table_alignment(void) {
+static inline size_t gc_write_barrier_card_table_alignment(void) {
GC_CRASH();
}
-static inline size_t gc_small_write_barrier_card_size(void) {
+static inline size_t gc_write_barrier_card_size(void) {
GC_CRASH();
}
diff --git a/bdw.c b/bdw.c
index 29958f133..cf17f19e7 100644
--- a/bdw.c
+++ b/bdw.c
@@ -115,6 +115,10 @@ void gc_collect(struct gc_mutator *mut) {
GC_gcollect();
}
+void gc_write_barrier_extern(struct gc_ref obj, size_t obj_size,
+ struct gc_edge edge, struct gc_ref new_val) {
+}
+
// In BDW-GC, we can't hook into the mark phase to call
// gc_trace_ephemerons_for_object, so the advertised ephemeron strategy
// doesn't really work. The primitives that we have are mark functions,
diff --git a/gc-api.h b/gc-api.h
index 5f0d3c1aa..6cf783703 100644
--- a/gc-api.h
+++ b/gc-api.h
@@ -177,18 +177,30 @@ static inline void gc_small_write_barrier(struct gc_ref
obj, struct gc_edge edge
struct gc_ref new_val)
GC_ALWAYS_INLINE;
static inline void gc_small_write_barrier(struct gc_ref obj, struct gc_edge
edge,
struct gc_ref new_val) {
- switch (gc_small_write_barrier_kind()) {
+}
+
+GC_API_ void gc_write_barrier_extern(struct gc_ref obj, size_t obj_size,
+ struct gc_edge edge, struct gc_ref
new_val) GC_NEVER_INLINE;
+
+static inline void gc_write_barrier(struct gc_ref obj, size_t obj_size,
+ struct gc_edge edge, struct gc_ref
new_val) GC_ALWAYS_INLINE;
+static inline void gc_write_barrier(struct gc_ref obj, size_t obj_size,
+ struct gc_edge edge, struct gc_ref
new_val) {
+ switch (gc_write_barrier_kind(obj_size)) {
case GC_WRITE_BARRIER_NONE:
return;
case GC_WRITE_BARRIER_CARD: {
- size_t card_table_alignment =
gc_small_write_barrier_card_table_alignment();
- size_t card_size = gc_small_write_barrier_card_size();
+ size_t card_table_alignment = gc_write_barrier_card_table_alignment();
+ size_t card_size = gc_write_barrier_card_size();
uintptr_t addr = gc_ref_value(obj);
uintptr_t base = addr & ~(card_table_alignment - 1);
uintptr_t card = (addr & (card_table_alignment - 1)) / card_size;
atomic_store_explicit((uint8_t*)(base + card), 1, memory_order_relaxed);
return;
}
+ case GC_WRITE_BARRIER_EXTERN:
+ gc_write_barrier_extern(obj, obj_size, edge, new_val);
+ return;
default:
GC_CRASH();
}
diff --git a/gc-attrs.h b/gc-attrs.h
index 17ff2add5..60d8e3351 100644
--- a/gc-attrs.h
+++ b/gc-attrs.h
@@ -29,11 +29,12 @@ static inline int gc_allocator_needs_clear(void)
GC_ALWAYS_INLINE;
enum gc_write_barrier_kind {
GC_WRITE_BARRIER_NONE,
- GC_WRITE_BARRIER_CARD
+ GC_WRITE_BARRIER_CARD,
+ GC_WRITE_BARRIER_EXTERN
};
-static inline enum gc_write_barrier_kind gc_small_write_barrier_kind(void)
GC_ALWAYS_INLINE;
-static inline size_t gc_small_write_barrier_card_table_alignment(void)
GC_ALWAYS_INLINE;
-static inline size_t gc_small_write_barrier_card_size(void) GC_ALWAYS_INLINE;
+static inline enum gc_write_barrier_kind gc_write_barrier_kind(size_t
obj_size) GC_ALWAYS_INLINE;
+static inline size_t gc_write_barrier_card_table_alignment(void)
GC_ALWAYS_INLINE;
+static inline size_t gc_write_barrier_card_size(void) GC_ALWAYS_INLINE;
#endif // GC_ATTRS_H
diff --git a/gc-embedder-api.h b/gc-embedder-api.h
index 8ae45ef61..6e39f05ea 100644
--- a/gc-embedder-api.h
+++ b/gc-embedder-api.h
@@ -41,6 +41,14 @@ GC_EMBEDDER_API inline void gc_trace_heap_roots(struct
gc_heap_roots *roots,
struct gc_heap *heap,
void *trace_data);
+// Some heap objects have space for a "remembered" bit, indicating they
+// are in the remembered set. Large or potentially large objects
+// (e.g. a vector whose size is a run-time property) must have a
+// remembered set bit. Small objects may or may not have such a bit.
+GC_EMBEDDER_API inline void gc_object_set_remembered(struct gc_ref ref);
+GC_EMBEDDER_API inline int gc_object_is_remembered_nonatomic(struct gc_ref
ref);
+GC_EMBEDDER_API inline void gc_object_clear_remembered_nonatomic(struct gc_ref
ref);
+
GC_EMBEDDER_API inline uintptr_t gc_object_forwarded_nonatomic(struct gc_ref
ref);
GC_EMBEDDER_API inline void gc_object_forward_nonatomic(struct gc_ref ref,
struct gc_ref new_ref);
diff --git a/large-object-space.h b/large-object-space.h
index de41dea60..9d8d0d06a 100644
--- a/large-object-space.h
+++ b/large-object-space.h
@@ -58,6 +58,49 @@ static size_t large_object_space_npages(struct
large_object_space *space,
return (bytes + space->page_size - 1) >> space->page_size_log2;
}
+static void large_object_space_clear_one_remembered(uintptr_t addr,
+ void *unused) {
+ struct gc_ref ref = gc_ref(addr);
+ if (gc_object_is_remembered_nonatomic(ref))
+ gc_object_clear_remembered_nonatomic(ref);
+}
+
+static void
+large_object_space_clear_remembered_set(struct large_object_space *space) {
+ if (!GC_GENERATIONAL)
+ return;
+ address_set_for_each(&space->to_space,
+ large_object_space_clear_one_remembered, NULL);
+}
+
+struct large_object_space_trace_remembered_data {
+ void (*trace)(struct gc_ref, struct gc_heap*);
+ struct gc_heap *heap;
+};
+
+static void large_object_space_trace_one_remembered(uintptr_t addr,
+ void *data) {
+ struct gc_ref ref = gc_ref(addr);
+ if (gc_object_is_remembered_nonatomic(ref)) {
+ gc_object_clear_remembered_nonatomic(ref);
+ struct large_object_space_trace_remembered_data *vdata = data;
+ vdata->trace(ref, vdata->heap);
+ }
+}
+
+static void
+large_object_space_trace_remembered_set(struct large_object_space *space,
+ void (*trace)(struct gc_ref,
+ struct gc_heap*),
+ struct gc_heap *heap) {
+ struct large_object_space_trace_remembered_data vdata = { trace, heap };
+
+ if (!GC_GENERATIONAL)
+ return;
+ address_set_for_each(&space->to_space,
+ large_object_space_trace_one_remembered, &vdata);
+}
+
static void large_object_space_start_gc(struct large_object_space *space,
int is_minor_gc) {
if (is_minor_gc)
diff --git a/mt-gcbench.c b/mt-gcbench.c
index f72dac66e..e7e7d8a58 100644
--- a/mt-gcbench.c
+++ b/mt-gcbench.c
@@ -144,9 +144,9 @@ static void allocate_garbage(struct thread *t) {
}
static void set_field(Node *obj, Node **field, Node *val) {
- gc_small_write_barrier(gc_ref_from_heap_object(obj),
- gc_edge(field),
- gc_ref_from_heap_object(val));
+ gc_write_barrier(gc_ref_from_heap_object(obj), sizeof(Node),
+ gc_edge(field),
+ gc_ref_from_heap_object(val));
*field = val;
}
diff --git a/semi-attrs.h b/semi-attrs.h
index e6b429178..3bf9584b8 100644
--- a/semi-attrs.h
+++ b/semi-attrs.h
@@ -42,13 +42,13 @@ static inline uint8_t
gc_allocator_alloc_table_end_pattern(void) {
GC_CRASH();
}
-static inline enum gc_write_barrier_kind gc_small_write_barrier_kind(void) {
+static inline enum gc_write_barrier_kind gc_write_barrier_kind(size_t) {
return GC_WRITE_BARRIER_NONE;
}
-static inline size_t gc_small_write_barrier_card_table_alignment(void) {
+static inline size_t gc_write_barrier_card_table_alignment(void) {
GC_CRASH();
}
-static inline size_t gc_small_write_barrier_card_size(void) {
+static inline size_t gc_write_barrier_card_size(void) {
GC_CRASH();
}
diff --git a/semi.c b/semi.c
index 5d85ec8c6..11b74ec5f 100644
--- a/semi.c
+++ b/semi.c
@@ -375,6 +375,10 @@ void gc_collect(struct gc_mutator *mut) {
collect(mut, 0);
}
+void gc_write_barrier_extern(struct gc_ref obj, size_t obj_size,
+ struct gc_edge edge, struct gc_ref new_val) {
+}
+
static void collect_for_large_alloc(struct gc_mutator *mut, size_t npages) {
collect_for_alloc(mut, npages * mutator_semi_space(mut)->page_size);
}
diff --git a/simple-gc-embedder.h b/simple-gc-embedder.h
index 70fd5c7a8..14fb142e7 100644
--- a/simple-gc-embedder.h
+++ b/simple-gc-embedder.h
@@ -86,6 +86,25 @@ static inline void gc_object_forward_nonatomic(struct gc_ref
ref,
*tag_word(ref) = gc_ref_value(new_ref);
}
+static inline void gc_object_set_remembered(struct gc_ref ref) {
+ uintptr_t *loc = tag_word(ref);
+ uintptr_t tag = *loc;
+ while (!(tag & gcobj_remembered_bit))
+ atomic_compare_exchange_weak(loc, &tag, tag | gcobj_remembered_bit);
+}
+
+static inline int gc_object_is_remembered_nonatomic(struct gc_ref ref) {
+ uintptr_t *loc = tag_word(ref);
+ uintptr_t tag = *loc;
+ return tag & gcobj_remembered_bit;
+}
+
+static inline void gc_object_clear_remembered_nonatomic(struct gc_ref ref) {
+ uintptr_t *loc = tag_word(ref);
+ uintptr_t tag = *loc;
+ *loc = tag & ~(uintptr_t)gcobj_remembered_bit;
+}
+
static inline struct gc_atomic_forward
gc_atomic_forward_begin(struct gc_ref ref) {
uintptr_t tag = atomic_load_explicit(tag_word(ref), memory_order_acquire);
diff --git a/simple-tagging-scheme.h b/simple-tagging-scheme.h
index b6b8a924c..aa0b707e4 100644
--- a/simple-tagging-scheme.h
+++ b/simple-tagging-scheme.h
@@ -7,9 +7,11 @@ struct gc_header {
uintptr_t tag;
};
-// Alloc kind is in bits 1-7, for live objects.
-static const uintptr_t gcobj_alloc_kind_mask = 0x7f;
-static const uintptr_t gcobj_alloc_kind_shift = 1;
+// Alloc kind is in bits 2-7, for live objects.
+static const uintptr_t gcobj_alloc_kind_mask = 0x3f;
+static const uintptr_t gcobj_alloc_kind_shift = 2;
+static const uintptr_t gcobj_remembered_mask = 0x2;
+static const uintptr_t gcobj_remembered_bit = 0x2;
static const uintptr_t gcobj_forwarded_mask = 0x1;
static const uintptr_t gcobj_not_forwarded_bit = 0x1;
static const uintptr_t gcobj_busy = 0;
diff --git a/whippet-attrs.h b/whippet-attrs.h
index bfecc44db..b26d79ad3 100644
--- a/whippet-attrs.h
+++ b/whippet-attrs.h
@@ -40,16 +40,19 @@ static inline int gc_allocator_needs_clear(void) {
return 0;
}
-static inline enum gc_write_barrier_kind gc_small_write_barrier_kind(void) {
- if (GC_GENERATIONAL)
- return GC_WRITE_BARRIER_CARD;
+static inline enum gc_write_barrier_kind gc_write_barrier_kind(size_t
obj_size) {
+ if (GC_GENERATIONAL) {
+ if (obj_size <= gc_allocator_large_threshold())
+ return GC_WRITE_BARRIER_CARD;
+ return GC_WRITE_BARRIER_EXTERN;
+ }
return GC_WRITE_BARRIER_NONE;
}
-static inline size_t gc_small_write_barrier_card_table_alignment(void) {
+static inline size_t gc_write_barrier_card_table_alignment(void) {
GC_ASSERT(GC_GENERATIONAL);
return 4 * 1024 * 1024;
}
-static inline size_t gc_small_write_barrier_card_size(void) {
+static inline size_t gc_write_barrier_card_size(void) {
GC_ASSERT(GC_GENERATIONAL);
return 256;
}
diff --git a/whippet.c b/whippet.c
index 5d692b728..4771e37e9 100644
--- a/whippet.c
+++ b/whippet.c
@@ -1333,6 +1333,10 @@ static void trace_global_conservative_roots(struct
gc_heap *heap) {
(mark_and_globally_enqueue_heap_conservative_roots, heap, NULL);
}
+static void enqueue_generational_root(struct gc_ref ref, struct gc_heap *heap)
{
+ tracer_enqueue_root(&heap->tracer, ref);
+}
+
// Note that it's quite possible (and even likely) that any given remset
// byte doesn't hold any roots, if all stores were to nursery objects.
STATIC_ASSERT_EQ(GRANULES_PER_REMSET_BYTE % 8, 0);
@@ -1352,7 +1356,7 @@ static void mark_space_trace_card(struct mark_space
*space,
size_t granule = granule_base + granule_offset;
uintptr_t addr = first_addr_in_slab + granule * GRANULE_SIZE;
GC_ASSERT(metadata_byte_for_addr(addr) == &slab->metadata[granule]);
- tracer_enqueue_root(&heap->tracer, gc_ref(addr));
+ enqueue_generational_root(gc_ref(addr), heap);
}
}
}
@@ -1385,12 +1389,22 @@ static void mark_space_clear_remembered_set(struct
mark_space *space) {
}
}
+void gc_write_barrier_extern(struct gc_ref obj, size_t obj_size,
+ struct gc_edge edge, struct gc_ref new_val) {
+ GC_ASSERT(size > gc_allocator_large_threshold());
+ gc_object_set_remembered(obj);
+}
+
static void trace_generational_roots(struct gc_heap *heap) {
// TODO: Add lospace nursery.
if (atomic_load(&heap->gc_kind) & GC_KIND_FLAG_MINOR) {
mark_space_trace_remembered_set(heap_mark_space(heap), heap);
+ large_object_space_trace_remembered_set(heap_large_object_space(heap),
+ enqueue_generational_root,
+ heap);
} else {
mark_space_clear_remembered_set(heap_mark_space(heap));
+ large_object_space_clear_remembered_set(heap_large_object_space(heap));
}
}
