Author: Remi Meier <[email protected]>
Branch: c8-small-uniform
Changeset: r1387:22150f9efdc0
Date: 2014-09-16 14:27 +0200
http://bitbucket.org/pypy/stmgc/changeset/22150f9efdc0/
Log: add synchronization to other segments on allocation of new (old)
objects
diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -5,6 +5,19 @@
#include <signal.h>
+#ifdef NDEBUG
+#define EVENTUALLY(condition) {/* nothing */}
+#else
+#define EVENTUALLY(condition) \
+ { \
+ if (!(condition)) { \
+ acquire_privatization_lock(STM_SEGMENT->segment_num);\
+ if (!(condition)) \
+ stm_fatalerror("fails: " #condition); \
+ release_privatization_lock(STM_SEGMENT->segment_num);\
+ } \
+ }
+#endif
/* ############# commit log ############# */
@@ -616,3 +629,102 @@
synchronize_all_threads(STOP_OTHERS_AND_BECOME_GLOBALLY_UNIQUE);
s_mutex_unlock();
}
+
+
+
+static inline void _synchronize_fragment(stm_char *frag, ssize_t frag_size)
+{
+ /* double-check that the result fits in one page */
+ assert(frag_size > 0);
+ assert(frag_size + ((uintptr_t)frag & 4095) <= 4096);
+
+ /* if the page of the fragment is fully shared, nothing to do */
+ assert(STM_PSEGMENT->privatization_lock);
+ if (is_shared_log_page((uintptr_t)frag / 4096))
+ return; /* nothing to do */
+
+ /* Enqueue this object (or fragemnt of object) */
+ if (STM_PSEGMENT->sq_len == SYNC_QUEUE_SIZE)
+ synchronize_objects_flush();
+ STM_PSEGMENT->sq_fragments[STM_PSEGMENT->sq_len] = frag;
+ STM_PSEGMENT->sq_fragsizes[STM_PSEGMENT->sq_len] = frag_size;
+ ++STM_PSEGMENT->sq_len;
+}
+
+static void synchronize_object_enqueue(object_t *obj)
+{
+ assert(!_is_young(obj));
+ assert(STM_PSEGMENT->privatization_lock);
+ assert(obj->stm_flags & GCFLAG_WRITE_BARRIER);
+ ssize_t obj_size = stmcb_size_rounded_up(
+ (struct object_s *)REAL_ADDRESS(STM_SEGMENT->segment_base, obj));
+ OPT_ASSERT(obj_size >= 16);
+
+ if (LIKELY(is_small_uniform(obj))) {
+ _synchronize_fragment((stm_char *)obj, obj_size);
+ return;
+ }
+
+ /* else, a more complicated case for large objects, to copy
+ around data only within the needed pages
+ */
+ uintptr_t start = (uintptr_t)obj;
+ uintptr_t end = start + obj_size;
+
+ do {
+ uintptr_t copy_up_to = (start + 4096) & ~4095; /* end of page */
+ if (copy_up_to >= end) {
+ copy_up_to = end; /* this is the last fragment */
+ }
+ uintptr_t copy_size = copy_up_to - start;
+
+ _synchronize_fragment((stm_char *)start, copy_size);
+
+ start = copy_up_to;
+ } while (start != end);
+}
+
+static void synchronize_objects_flush(void)
+{
+
+ /* Do a full memory barrier. We must make sure that other
+ CPUs see the changes we did to the shared page ("S", in
+ synchronize_object_enqueue()) before we check the other segments
+ with is_private_page() (below). Otherwise, we risk the
+ following: this CPU writes "S" but the writes are not visible yet;
+ then it checks is_private_page() and gets false, and does nothing
+ more; just afterwards another CPU sets its own private_page bit
+ and copies the page; but it risks doing so before seeing the "S"
+ writes.
+ */
+ /* XXX: not sure this applies anymore. */
+ long j = STM_PSEGMENT->sq_len;
+ if (j == 0)
+ return;
+ STM_PSEGMENT->sq_len = 0;
+
+ __sync_synchronize();
+
+ long i, myself = STM_SEGMENT->segment_num;
+ do {
+ --j;
+ stm_char *frag = STM_PSEGMENT->sq_fragments[j];
+ uintptr_t page = ((uintptr_t)frag) / 4096UL;
+ if (is_shared_log_page(page))
+ continue;
+
+ ssize_t frag_size = STM_PSEGMENT->sq_fragsizes[j];
+
+ char *src = REAL_ADDRESS(STM_SEGMENT->segment_base, frag);
+ for (i = 0; i < NB_SEGMENTS; i++) {
+ if (i == myself)
+ continue;
+
+ char *dst = REAL_ADDRESS(get_segment_base(i), frag);
+ if (is_private_log_page_in(i, page))
+ memcpy(dst, src, frag_size);
+ else
+ EVENTUALLY(memcmp(dst, src, frag_size) == 0); /* same page */
+ }
+ } while (j > 0);
+}
diff --git a/c8/stm/core.h b/c8/stm/core.h
--- a/c8/stm/core.h
+++ b/c8/stm/core.h
@@ -38,6 +38,10 @@
};
+
+#define SYNC_QUEUE_SIZE 31
+
+
/************************************************************/
@@ -72,8 +76,15 @@
/* This is for smallmalloc.c */
struct small_malloc_data_s small_malloc_data;
+
+ /* The sync queue used to synchronize newly allocated objs to
+ other segments */
+ stm_char *sq_fragments[SYNC_QUEUE_SIZE];
+ int sq_fragsizes[SYNC_QUEUE_SIZE];
+ int sq_len;
};
+
enum /* safe_point */ {
SP_NO_TRANSACTION=0,
SP_RUNNING,
@@ -138,6 +149,8 @@
static stm_thread_local_t *abort_with_mutex_no_longjmp(void);
static void abort_data_structures_from_segment_num(int segment_num);
+static void synchronize_object_enqueue(object_t *obj);
+static void synchronize_objects_flush(void);
static inline void _duck(void) {
diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c
--- a/c8/stm/nursery.c
+++ b/c8/stm/nursery.c
@@ -47,11 +47,14 @@
/************************************************************/
static object_t *find_existing_shadow(object_t *obj);
#define GCWORD_MOVED ((object_t *) -1)
+#define FLAG_SYNC_LARGE 0x01
+
static void minor_trace_if_young(object_t **pobj)
{
/* takes a normal pointer to a thread-local pointer to an object */
object_t *obj = *pobj;
+ uintptr_t nobj_sync_now;
object_t *nobj;
char *realobj;
size_t size;
@@ -97,7 +100,6 @@
else {
/* case "small enough" */
char *allocated = allocate_outside_nursery_small(size);
- dprintf(("outside small %p or %p, sz=%lu\n", allocated, allocated
- stm_object_pages, size));
nobj = (object_t *)(allocated - stm_object_pages);
}
@@ -106,6 +108,8 @@
char *realnobj = REAL_ADDRESS(STM_SEGMENT->segment_base, nobj);
memcpy(realnobj, realobj, size);
+ nobj_sync_now = ((uintptr_t)nobj) | FLAG_SYNC_LARGE;
+
pforwarded_array[0] = GCWORD_MOVED;
pforwarded_array[1] = nobj;
*pobj = nobj;
@@ -119,10 +123,12 @@
/* a young object outside the nursery */
nobj = obj;
tree_delete_item(STM_PSEGMENT->young_outside_nursery, (uintptr_t)nobj);
+
+ nobj_sync_now = ((uintptr_t)nobj) | FLAG_SYNC_LARGE;
}
/* Must trace the object later */
- LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, (uintptr_t)nobj);
+ LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, nobj_sync_now);
}
@@ -174,13 +180,24 @@
struct list_s *lst = STM_PSEGMENT->objects_pointing_to_nursery;
while (!list_is_empty(lst)) {
- object_t *obj = (object_t *)list_pop_item(lst);;
+ uintptr_t obj_sync_now = list_pop_item(lst);
+ object_t *obj = (object_t *)(obj_sync_now & ~FLAG_SYNC_LARGE);
_collect_now(obj);
+ if (obj_sync_now & FLAG_SYNC_LARGE) {
+ /* this is a newly allocated object. We must synchronize it
+ to other segments (after we added WRITE_BARRIER). */
+ acquire_privatization_lock(STM_SEGMENT->segment_num);
+ synchronize_object_enqueue(obj);
+ release_privatization_lock(STM_SEGMENT->segment_num);
+ }
+
/* the list could have moved while appending */
lst = STM_PSEGMENT->objects_pointing_to_nursery;
}
+
+ synchronize_objects_flush();
}
diff --git a/c8/test/test_nursery.py b/c8/test/test_nursery.py
--- a/c8/test/test_nursery.py
+++ b/c8/test/test_nursery.py
@@ -274,7 +274,6 @@
self.push_root(new2)
self.commit_transaction()
new2 = self.pop_root()
- print "new2", new2
# check that this new object was correctly sychronized
self.switch(1)
diff --git a/c8/test/test_random.py b/c8/test/test_random.py
--- a/c8/test/test_random.py
+++ b/c8/test/test_random.py
@@ -361,7 +361,7 @@
def op_allocate(ex, global_state, thread_state):
size = global_state.rnd.choice([
- "16",
+ "16", "48", "288",
str(4096+16),
str(80*1024+16),
#"SOME_MEDIUM_SIZE+16",
_______________________________________________
pypy-commit mailing list
[email protected]
https://mail.python.org/mailman/listinfo/pypy-commit
