wingo pushed a commit to branch wip-whippet
in repository guile.
commit 1f4e3bdf3731a48f9758d0cf1f304a77aacf4de0
Author: Andy Wingo <wi...@igalia.com>
AuthorDate: Tue Oct 1 10:34:27 2024 +0200
Add field-logging write barrier (fast path only)
Add a new kind of write barrier, one which has a bit per field; the
mutator that sets the bit will need to add the field's location (the
edge) to a remembered set. Here we just have the fast-path
implementation.
---
api/bdw-attrs.h | 9 +++++++++
api/gc-api.h | 37 ++++++++++++++++++++++++++++---------
api/gc-attrs.h | 6 +++++-
api/mmc-attrs.h | 14 +++++++++++++-
api/pcc-attrs.h | 9 +++++++++
api/semi-attrs.h | 9 +++++++++
benchmarks/mt-gcbench.c | 9 +++++----
src/bdw.c | 5 +++--
src/mmc.c | 5 +++--
src/pcc.c | 5 +++--
src/semi.c | 5 +++--
11 files changed, 90 insertions(+), 23 deletions(-)
diff --git a/api/bdw-attrs.h b/api/bdw-attrs.h
index af1042af3..51b4e72f1 100644
--- a/api/bdw-attrs.h
+++ b/api/bdw-attrs.h
@@ -49,6 +49,15 @@ static inline size_t
gc_write_barrier_card_table_alignment(void) {
static inline size_t gc_write_barrier_card_size(void) {
GC_CRASH();
}
+static inline size_t gc_write_barrier_field_table_alignment(void) {
+ GC_CRASH();
+}
+static inline size_t gc_write_barrier_field_fields_per_byte(void) {
+ GC_CRASH();
+}
+static inline uint8_t gc_write_barrier_field_first_bit_pattern(void) {
+ GC_CRASH();
+}
static inline enum gc_safepoint_mechanism gc_safepoint_mechanism(void) {
return GC_SAFEPOINT_MECHANISM_SIGNAL;
diff --git a/api/gc-api.h b/api/gc-api.h
index 63921f628..cb1e4e819 100644
--- a/api/gc-api.h
+++ b/api/gc-api.h
@@ -179,13 +179,16 @@ static inline void* gc_allocate(struct gc_mutator *mut,
size_t size) {
// FIXME: remove :P
GC_API_ void* gc_allocate_pointerless(struct gc_mutator *mut, size_t bytes);
-GC_API_ void gc_write_barrier_extern(struct gc_ref obj, size_t obj_size,
- struct gc_edge edge, struct gc_ref
new_val) GC_NEVER_INLINE;
-
-static inline void gc_write_barrier(struct gc_ref obj, size_t obj_size,
- struct gc_edge edge, struct gc_ref
new_val) GC_ALWAYS_INLINE;
-static inline void gc_write_barrier(struct gc_ref obj, size_t obj_size,
- struct gc_edge edge, struct gc_ref
new_val) {
+GC_API_ void gc_write_barrier_slow(struct gc_mutator *mut, struct gc_ref obj,
+ size_t obj_size, struct gc_edge edge,
+ struct gc_ref new_val) GC_NEVER_INLINE;
+
+static inline void gc_write_barrier(struct gc_mutator *mut, struct gc_ref obj,
+ size_t obj_size, struct gc_edge edge,
+ struct gc_ref new_val) GC_ALWAYS_INLINE;
+static inline void gc_write_barrier(struct gc_mutator *mut, struct gc_ref obj,
+ size_t obj_size, struct gc_edge edge,
+ struct gc_ref new_val) {
switch (gc_write_barrier_kind(obj_size)) {
case GC_WRITE_BARRIER_NONE:
return;
@@ -198,8 +201,24 @@ static inline void gc_write_barrier(struct gc_ref obj,
size_t obj_size,
atomic_store_explicit((uint8_t*)(base + card), 1, memory_order_relaxed);
return;
}
- case GC_WRITE_BARRIER_EXTERN:
- gc_write_barrier_extern(obj, obj_size, edge, new_val);
+ case GC_WRITE_BARRIER_FIELD: {
+ size_t field_table_alignment = gc_write_barrier_field_table_alignment();
+ size_t fields_per_byte = gc_write_barrier_field_fields_per_byte();
+ uint8_t first_bit_pattern = gc_write_barrier_field_first_bit_pattern();
+
+ uintptr_t addr = (uintptr_t) gc_edge_loc(edge);
+ uintptr_t base = addr & ~(field_table_alignment - 1);
+ uintptr_t field = (addr & (field_table_alignment - 1)) / sizeof(uintptr_t);
+ uintptr_t log_byte = field / fields_per_byte;
+ uint8_t log_bit = first_bit_pattern << (field % fields_per_byte);
+ uint8_t *byte_loc = (uint8_t*)(base + log_byte);
+ uint8_t byte = atomic_load_explicit(byte_loc, memory_order_relaxed);
+ if (!(byte & log_bit))
+ gc_write_barrier_slow(mut, obj, obj_size, edge, new_val);
+ return;
+ }
+ case GC_WRITE_BARRIER_SLOW:
+ gc_write_barrier_slow(mut, obj, obj_size, edge, new_val);
return;
default:
GC_CRASH();
diff --git a/api/gc-attrs.h b/api/gc-attrs.h
index b6acb4302..39faceae2 100644
--- a/api/gc-attrs.h
+++ b/api/gc-attrs.h
@@ -30,12 +30,16 @@ static inline int gc_allocator_needs_clear(void)
GC_ALWAYS_INLINE;
enum gc_write_barrier_kind {
GC_WRITE_BARRIER_NONE,
GC_WRITE_BARRIER_CARD,
- GC_WRITE_BARRIER_EXTERN
+ GC_WRITE_BARRIER_FIELD,
+ GC_WRITE_BARRIER_SLOW
};
static inline enum gc_write_barrier_kind gc_write_barrier_kind(size_t
obj_size) GC_ALWAYS_INLINE;
static inline size_t gc_write_barrier_card_table_alignment(void)
GC_ALWAYS_INLINE;
static inline size_t gc_write_barrier_card_size(void) GC_ALWAYS_INLINE;
+static inline size_t gc_write_barrier_field_table_alignment(void)
GC_ALWAYS_INLINE;
+static inline size_t gc_write_barrier_field_fields_per_byte(void)
GC_ALWAYS_INLINE;
+static inline uint8_t gc_write_barrier_field_first_bit_pattern(void)
GC_ALWAYS_INLINE;
enum gc_safepoint_mechanism {
GC_SAFEPOINT_MECHANISM_COOPERATIVE,
diff --git a/api/mmc-attrs.h b/api/mmc-attrs.h
index e5757f6d1..6da4b2a68 100644
--- a/api/mmc-attrs.h
+++ b/api/mmc-attrs.h
@@ -44,7 +44,7 @@ static inline enum gc_write_barrier_kind
gc_write_barrier_kind(size_t obj_size)
if (GC_GENERATIONAL) {
if (obj_size <= gc_allocator_large_threshold())
return GC_WRITE_BARRIER_CARD;
- return GC_WRITE_BARRIER_EXTERN;
+ return GC_WRITE_BARRIER_SLOW;
}
return GC_WRITE_BARRIER_NONE;
}
@@ -56,6 +56,18 @@ static inline size_t gc_write_barrier_card_size(void) {
GC_ASSERT(GC_GENERATIONAL);
return 256;
}
+static inline size_t gc_write_barrier_field_table_alignment(void) {
+ GC_ASSERT(GC_GENERATIONAL);
+ return 4 * 1024 * 1024;
+}
+static inline size_t gc_write_barrier_field_fields_per_byte(void) {
+ GC_ASSERT(GC_GENERATIONAL);
+ return 2;
+}
+static inline uint8_t gc_write_barrier_field_first_bit_pattern(void) {
+ GC_ASSERT(GC_GENERATIONAL);
+ return 64; // NOFL_METADATA_BYTE_LOGGED_0
+}
static inline enum gc_safepoint_mechanism gc_safepoint_mechanism(void) {
return GC_SAFEPOINT_MECHANISM_COOPERATIVE;
diff --git a/api/pcc-attrs.h b/api/pcc-attrs.h
index 2f02640ea..5dd38c42f 100644
--- a/api/pcc-attrs.h
+++ b/api/pcc-attrs.h
@@ -52,6 +52,15 @@ static inline size_t
gc_write_barrier_card_table_alignment(void) {
static inline size_t gc_write_barrier_card_size(void) {
GC_CRASH();
}
+static inline size_t gc_write_barrier_field_table_alignment(void) {
+ GC_CRASH();
+}
+static inline size_t gc_write_barrier_field_fields_per_byte(void) {
+ GC_CRASH();
+}
+static inline uint8_t gc_write_barrier_field_first_bit_pattern(void) {
+ GC_CRASH();
+}
static inline enum gc_safepoint_mechanism gc_safepoint_mechanism(void) {
return GC_SAFEPOINT_MECHANISM_COOPERATIVE;
diff --git a/api/semi-attrs.h b/api/semi-attrs.h
index bcd8e89e0..94e2dc814 100644
--- a/api/semi-attrs.h
+++ b/api/semi-attrs.h
@@ -51,6 +51,15 @@ static inline size_t
gc_write_barrier_card_table_alignment(void) {
static inline size_t gc_write_barrier_card_size(void) {
GC_CRASH();
}
+static inline size_t gc_write_barrier_field_table_alignment(void) {
+ GC_CRASH();
+}
+static inline size_t gc_write_barrier_field_fields_per_byte(void) {
+ GC_CRASH();
+}
+static inline uint8_t gc_write_barrier_field_first_bit_pattern(void) {
+ GC_CRASH();
+}
static inline enum gc_safepoint_mechanism gc_safepoint_mechanism(void) {
return GC_SAFEPOINT_MECHANISM_COOPERATIVE;
diff --git a/benchmarks/mt-gcbench.c b/benchmarks/mt-gcbench.c
index 05ae887d0..7f342fe90 100644
--- a/benchmarks/mt-gcbench.c
+++ b/benchmarks/mt-gcbench.c
@@ -144,8 +144,9 @@ static void allocate_garbage(struct thread *t) {
}
}
-static void set_field(Node *obj, Node **field, Node *val) {
- gc_write_barrier(gc_ref_from_heap_object(obj), sizeof(Node),
+static void set_field(struct gc_mutator *mut, Node *obj,
+ Node **field, Node *val) {
+ gc_write_barrier(mut, gc_ref_from_heap_object(obj), sizeof(Node),
gc_edge(field),
gc_ref_from_heap_object(val));
*field = val;
@@ -166,8 +167,8 @@ static void populate(struct thread *t, int depth, Node
*node) {
NodeHandle r = { allocate_node(mut) };
PUSH_HANDLE(t, r);
- set_field(HANDLE_REF(self), &HANDLE_REF(self)->left, HANDLE_REF(l));
- set_field(HANDLE_REF(self), &HANDLE_REF(self)->right, HANDLE_REF(r));
+ set_field(mut, HANDLE_REF(self), &HANDLE_REF(self)->left, HANDLE_REF(l));
+ set_field(mut, HANDLE_REF(self), &HANDLE_REF(self)->right, HANDLE_REF(r));
// i is 0 because the memory is zeroed.
HANDLE_REF(self)->j = depth;
diff --git a/src/bdw.c b/src/bdw.c
index df579dde7..d4cbe4c41 100644
--- a/src/bdw.c
+++ b/src/bdw.c
@@ -149,8 +149,9 @@ void gc_collect(struct gc_mutator *mut,
}
}
-void gc_write_barrier_extern(struct gc_ref obj, size_t obj_size,
- struct gc_edge edge, struct gc_ref new_val) {
+void gc_write_barrier_slow(struct gc_mutator *mut, struct gc_ref obj,
+ size_t obj_size, struct gc_edge edge,
+ struct gc_ref new_val) {
}
int* gc_safepoint_flag_loc(struct gc_mutator *mut) { GC_CRASH(); }
diff --git a/src/mmc.c b/src/mmc.c
index 48d8eae59..63778d669 100644
--- a/src/mmc.c
+++ b/src/mmc.c
@@ -876,8 +876,9 @@ gc_pin_object(struct gc_mutator *mut, struct gc_ref ref) {
}
void
-gc_write_barrier_extern(struct gc_ref obj, size_t obj_size,
- struct gc_edge edge, struct gc_ref new_val) {
+gc_write_barrier_slow(struct gc_mutator *mut, struct gc_ref obj,
+ size_t obj_size, struct gc_edge edge,
+ struct gc_ref new_val) {
GC_ASSERT(obj_size > gc_allocator_large_threshold());
gc_object_set_remembered(obj);
}
diff --git a/src/pcc.c b/src/pcc.c
index 90194e96a..10d5f8382 100644
--- a/src/pcc.c
+++ b/src/pcc.c
@@ -483,8 +483,9 @@ void gc_pin_object(struct gc_mutator *mut, struct gc_ref
ref) {
GC_CRASH();
}
-void gc_write_barrier_extern(struct gc_ref obj, size_t obj_size,
- struct gc_edge edge, struct gc_ref new_val) {
+void gc_write_barrier_slow(struct gc_mutator *mut, struct gc_ref obj,
+ size_t obj_size, struct gc_edge edge,
+ struct gc_ref new_val) {
}
int* gc_safepoint_flag_loc(struct gc_mutator *mut) {
diff --git a/src/semi.c b/src/semi.c
index 7958b5898..aab5af233 100644
--- a/src/semi.c
+++ b/src/semi.c
@@ -453,8 +453,9 @@ void gc_collect(struct gc_mutator *mut,
collect(mut, 0);
}
-void gc_write_barrier_extern(struct gc_ref obj, size_t obj_size,
- struct gc_edge edge, struct gc_ref new_val) {
+void gc_write_barrier_slow(struct gc_mutator *mut, struct gc_ref obj,
+ size_t obj_size, struct gc_edge edge,
+ struct gc_ref new_val) {
}
int* gc_safepoint_flag_loc(struct gc_mutator *mut) { GC_CRASH(); }
