Author: Remi Meier
Branch: c7
Changeset: r673:31b7b6a45064
Date: 2014-01-24 15:46 +0100
http://bitbucket.org/pypy/stmgc/changeset/31b7b6a45064/
Log: use only largemalloc to allocate old objects
diff --git a/c7/core.c b/c7/core.c
--- a/c7/core.c
+++ b/c7/core.c
@@ -68,11 +68,9 @@
uintptr_t lock_idx = (((uintptr_t)item) >> 4) - READMARKER_START;
assert(write_locks[lock_idx]);
write_locks[lock_idx] = 0;
-
- char *src = REAL_ADDRESS(local_base, item);
- char *dst = REAL_ADDRESS(remote_base, item);
- size_t size = stmcb_size((struct object_s*)src);
- memcpy(dst, src, size);
+
+ _stm_move_object(REAL_ADDRESS(local_base, item),
+ REAL_ADDRESS(remote_base, item));
}));
if (conflicted) {
@@ -88,23 +86,24 @@
{
uintptr_t pagenum = ((uintptr_t)obj) / 4096;
assert(pagenum < NB_PAGES);
+ assert(!_stm_is_young(obj));
LIST_APPEND(_STM_TL->old_objects_to_trace, obj);
/* for old objects from the same transaction we don't need
- to privatize the page */
- if ((stm_get_page_flag(pagenum) == UNCOMMITTED_SHARED_PAGE)
- || (obj->stm_flags & GCFLAG_NOT_COMMITTED)) {
+ to privatize the pages */
+ if (obj->stm_flags & GCFLAG_NOT_COMMITTED) {
obj->stm_flags &= ~GCFLAG_WRITE_BARRIER;
return;
}
/* privatize if SHARED_PAGE */
- /* xxx stmcb_size() is probably too slow, maybe add a GCFLAG_LARGE for
- objs with more than 1 page */
- int pages = stmcb_size(real_address(obj)) / 4096;
- for (; pages >= 0; pages--)
- stm_pages_privatize(pagenum + pages);
+ uintptr_t pagenum2, pages;
+ _stm_chunk_pages((struct object_s*)REAL_ADDRESS(get_thread_base(0), obj),
+ &pagenum2, &pages);
+ assert(pagenum == pagenum2);
+ for (pagenum2 += pages - 1; pagenum2 >= pagenum; pagenum2--)
+ stm_pages_privatize(pagenum2);
/* claim the write-lock for this object */
uintptr_t lock_idx = (((uintptr_t)obj) >> 4) - READMARKER_START;
@@ -226,7 +225,6 @@
_STM_TL->shadow_stack_base = _STM_TL->shadow_stack;
_STM_TL->old_objects_to_trace = stm_list_create();
- _STM_TL->uncommitted_pages = stm_list_create();
_STM_TL->modified_objects = stm_list_create();
_STM_TL->uncommitted_objects = stm_list_create();
@@ -247,7 +245,6 @@
assert(stm_list_is_empty(_STM_TL->uncommitted_objects));
stm_list_free(_STM_TL->uncommitted_objects);
- _STM_TL->uncommitted_objects = NULL;
assert(_STM_TL->shadow_stack == _STM_TL->shadow_stack_base);
free(_STM_TL->shadow_stack);
@@ -255,9 +252,6 @@
assert(_STM_TL->old_objects_to_trace->count == 0);
stm_list_free(_STM_TL->old_objects_to_trace);
- assert(_STM_TL->uncommitted_pages->count == 0);
- stm_list_free(_STM_TL->uncommitted_pages);
-
set_gs_register(INVALID_GS_VALUE);
}
@@ -341,9 +335,7 @@
_STM_TL->jmpbufptr = NULL; /* cannot abort any more */
- /* push uncommitted objects to other threads,
- make completely uncommitted pages SHARED,
- */
+ /* push uncommitted objects to other threads */
nursery_on_commit();
/* copy modified object versions to other threads */
diff --git a/c7/core.h b/c7/core.h
--- a/c7/core.h
+++ b/c7/core.h
@@ -49,7 +49,6 @@
typedef TLPREFIX struct object_s object_t;
typedef TLPREFIX struct read_marker_s read_marker_t;
typedef TLPREFIX char localchar_t;
-typedef TLPREFIX struct alloc_for_size_s alloc_for_size_t;
typedef void* jmpbufptr_t[5]; /* for use with __builtin_setjmp() */
/* Structure of objects
@@ -80,12 +79,6 @@
uint8_t rm;
};
-struct alloc_for_size_s {
- localchar_t *next;
- uint16_t start, stop;
- bool flag_partial_page;
-};
-
struct _thread_local1_s {
jmpbufptr_t *jmpbufptr;
@@ -101,13 +94,8 @@
object_t **shadow_stack;
object_t **shadow_stack_base;
- struct alloc_for_size_s alloc[LARGE_OBJECT_WORDS];
struct stm_list_s *uncommitted_objects;
- /* pages newly allocated in the current transaction only containing
- uncommitted objects */
- struct stm_list_s *uncommitted_pages;
-
localchar_t *nursery_current;
struct stm_list_s *old_objects_to_trace;
};
diff --git a/c7/largemalloc.c b/c7/largemalloc.c
--- a/c7/largemalloc.c
+++ b/c7/largemalloc.c
@@ -5,11 +5,13 @@
or medium-block support that are also present in the GNU C Library.
*/
+#include <string.h>
#include <stdio.h>
#include <stddef.h>
#include <assert.h>
#include "largemalloc.h"
-
+#include "pages.h"
+#include "pagecopy.h"
#define MMAP_LIMIT (1280*1024)
@@ -89,20 +91,45 @@
static dlist_t largebins[N_BINS];
static mchunk_t *first_chunk, *last_chunk;
-void _stm_chunk_pages(object_t *tldata, intptr_t *start, intptr_t *num)
+void _stm_chunk_pages(struct object_s *data, uintptr_t *start, uintptr_t *num)
{
- char *data = _stm_real_address(tldata);
- mchunk_t *chunk = data2chunk(data);
- *start = (((char*)chunk) - get_thread_base(0)) / 4096UL;
- size_t offset_into_page = ((uintptr_t)chunk) & 4095UL; // % 4096
- *num = ((chunk->size & ~CHUNK_HEADER_SIZE) + CHUNK_HEADER_SIZE +
offset_into_page + 4095) / 4096UL;
+ /* returns the start page and number of pages that the *payload*
+ spans over. the CHUNK_HEADER is not included in the calculations */
+ mchunk_t *chunk = data2chunk((char*)data);
+ *start = (((char*)data) - get_thread_base(0)) / 4096UL;
+ size_t offset_into_page = ((uintptr_t)data) & 4095UL; // % 4096
+ *num = ((chunk->size & ~FLAG_SORTED) + offset_into_page + 4095) / 4096UL;
}
-size_t _stm_data_size(object_t *tldata)
+size_t _stm_data_size(struct object_s *data)
{
- char *data = _stm_real_address(tldata);
- mchunk_t *chunk = data2chunk(data);
- return chunk->size & ~CHUNK_HEADER_SIZE;
+ mchunk_t *chunk = data2chunk((char*)data);
+ return chunk->size & ~FLAG_SORTED;
+}
+
+void _stm_move_object(char *src, char *dst)
+{
+ /* only copies if page is PRIVATE
+ XXX: various optimizations for objects with
+ multiple pages. E.g. using pagecopy or
+ memcpy over multiple PRIVATE pages. */
+ char *end = src + _stm_data_size((struct object_s*)src);
+ uintptr_t pagenum, num;
+ struct object_s *t0_obj = (struct
object_s*)REAL_ADDRESS(get_thread_base(0), _stm_tl_address(src));
+ _stm_chunk_pages(t0_obj, &pagenum, &num);
+
+ while (src < end) {
+ size_t to_copy = 4096UL - ((uintptr_t)src & 4095UL);
+ if (to_copy > end - src)
+ to_copy = end - src;
+ if (stm_get_page_flag(pagenum) == PRIVATE_PAGE) {
+ memcpy(dst, src, to_copy);
+ }
+
+ pagenum++;
+ src += to_copy;
+ dst += to_copy;
+ }
}
static void insert_unsorted(mchunk_t *new)
diff --git a/c7/largemalloc.h b/c7/largemalloc.h
--- a/c7/largemalloc.h
+++ b/c7/largemalloc.h
@@ -8,6 +8,10 @@
void stm_large_free(object_t *data);
void _stm_large_dump(void);
-void _stm_chunk_pages(object_t *tldata, intptr_t *start, intptr_t *num);
-size_t _stm_data_size(object_t *tldata);
char *_stm_largemalloc_data_start(void);
+
+void _stm_move_object(char *src, char *dst);
+size_t _stm_data_size(struct object_s *data);
+void _stm_chunk_pages(struct object_s *data, uintptr_t *start, uintptr_t *num);
+
+
diff --git a/c7/nursery.c b/c7/nursery.c
--- a/c7/nursery.c
+++ b/c7/nursery.c
@@ -15,6 +15,7 @@
#include "nursery.h"
#include "pages.h"
#include "stmsync.h"
+#include "largemalloc.h"
void stm_major_collection(void)
{
@@ -30,18 +31,9 @@
}
-void mark_page_as_uncommitted(uintptr_t pagenum)
-{
- stm_set_page_flag(pagenum, UNCOMMITTED_SHARED_PAGE);
- LIST_APPEND(_STM_TL->uncommitted_pages, (object_t*)pagenum);
-}
-
object_t *_stm_allocate_old(size_t size)
{
- int pages = (size + 4095) / 4096;
- localchar_t* addr = (localchar_t*)(stm_pages_reserve(pages) * 4096);
-
- object_t* o = (object_t*)addr;
+ object_t* o = stm_large_malloc(size);
o->stm_flags |= GCFLAG_WRITE_BARRIER;
return o;
}
@@ -51,84 +43,6 @@
return _stm_allocate_old(size); /* XXX */
}
-localchar_t *_stm_alloc_next_page(size_t size_class)
-{
- /* may return uninitialized pages */
-
- /* 'alloc->next' points to where the next allocation should go. The
- present function is called instead when this next allocation is
- equal to 'alloc->stop'. As we know that 'start', 'next' and
- 'stop' are always nearby pointers, we play tricks and only store
- the lower 16 bits of 'start' and 'stop', so that the three
- variables plus some flags fit in 16 bytes.
- */
- uintptr_t page;
- localchar_t *result;
- alloc_for_size_t *alloc = &_STM_TL->alloc[size_class];
- size_t size = size_class * 8;
-
- /* reserve a fresh new page */
- page = stm_pages_reserve(1);
-
- /* mark as UNCOMMITTED_... */
- mark_page_as_uncommitted(page);
-
- result = (localchar_t *)(page * 4096UL);
- alloc->start = (uintptr_t)result;
- alloc->stop = alloc->start + (4096 / size) * size;
- alloc->next = result + size;
- alloc->flag_partial_page = false;
- return result;
-}
-
-
-
-
-object_t *_stm_alloc_old(size_t size)
-{
- /* may return uninitialized objects. except for the
- GCFLAG_NOT_COMMITTED, it is set exactly if
- we allocated the object in a SHARED and partially
- committed page. (XXX: add the flag in some other place)
- */
- object_t *result;
- size_t size_class = size / 8;
- assert(size_class >= 2);
-
- if (size_class >= LARGE_OBJECT_WORDS) {
- result = _stm_allocate_old(size);
- result->stm_flags &= ~GCFLAG_NOT_COMMITTED; /* page may be non-zeroed
*/
-
- int page = ((uintptr_t)result) / 4096;
- int pages = (size + 4095) / 4096;
- int i;
- for (i = 0; i < pages; i++) {
- mark_page_as_uncommitted(page + i);
- }
- /* make sure the flag is not set (page is not zeroed!) */
- result->stm_flags &= ~GCFLAG_NOT_COMMITTED;
- } else {
- alloc_for_size_t *alloc = &_STM_TL->alloc[size_class];
-
- if ((uint16_t)((uintptr_t)alloc->next) == alloc->stop) {
- result = (object_t *)_stm_alloc_next_page(size_class);
- } else {
- result = (object_t *)alloc->next;
- alloc->next += size;
- if (alloc->flag_partial_page) {
- LIST_APPEND(_STM_TL->uncommitted_objects, result);
- result->stm_flags |= GCFLAG_NOT_COMMITTED;
- } else {
- /* make sure the flag is not set (page is not zeroed!) */
- result->stm_flags &= ~GCFLAG_NOT_COMMITTED;
- }
- }
- }
- return result;
-}
-
-
-
void trace_if_young(object_t **pobj)
{
@@ -147,16 +61,16 @@
/* move obj to somewhere else */
size_t size = stmcb_size(real_address(*pobj));
- object_t *moved = (object_t*)_stm_alloc_old(size);
+ object_t *moved = stm_large_malloc(size);
- if (moved->stm_flags & GCFLAG_NOT_COMMITTED)
- (*pobj)->stm_flags |= GCFLAG_NOT_COMMITTED; /* XXX: memcpy below
overwrites this otherwise.
- find better solution.*/
-
memcpy((void*)real_address(moved),
(void*)real_address(*pobj),
size);
+ /* object is not committed yet */
+ moved->stm_flags |= GCFLAG_NOT_COMMITTED;
+ LIST_APPEND(_STM_TL->uncommitted_objects, moved);
+
(*pobj)->stm_flags |= GCFLAG_MOVED;
*pforwarded = moved;
*pobj = moved;
@@ -251,22 +165,15 @@
/* remove the flag (they are now committed) */
item->stm_flags &= ~GCFLAG_NOT_COMMITTED;
-
- uintptr_t pagenum = ((uintptr_t)item) / 4096UL;
- if (stm_get_page_flag(pagenum) == PRIVATE_PAGE) {
- /* page was privatized... */
- char *src = REAL_ADDRESS(local_base, item);
- char *dst = REAL_ADDRESS(remote_base, item);
- size_t size = stmcb_size((struct object_s*)src);
- memcpy(dst, src, size);
- }
+
+ _stm_move_object(REAL_ADDRESS(local_base, item),
+ REAL_ADDRESS(remote_base, item));
}));
}
void nursery_on_start()
{
assert(stm_list_is_empty(_STM_TL->old_objects_to_trace));
- stm_list_clear(_STM_TL->uncommitted_pages);
_STM_TL->old_shadow_stack = _STM_TL->shadow_stack;
}
@@ -277,37 +184,9 @@
the caller (optimization) */
/* minor_collect(); */
- /* uncommitted objects / partially COMMITTED pages */
+ /* uncommitted objects */
push_uncommitted_to_other_threads();
stm_list_clear(_STM_TL->uncommitted_objects);
-
- /* uncommitted_pages */
- long j;
- for (j = 2; j < LARGE_OBJECT_WORDS; j++) {
- alloc_for_size_t *alloc = &_STM_TL->alloc[j];
- uint16_t start = alloc->start;
- uint16_t cur = (uintptr_t)alloc->next;
-
- if (start == cur)
- continue; /* page full -> will be replaced automatically
*/
-
- alloc->start = cur; /* next transaction has different 'start' to
- reset in case of an abort */
-
- uintptr_t pagenum = ((uintptr_t)(alloc->next - 1)) / 4096UL;
- if (stm_get_page_flag(pagenum) == UNCOMMITTED_SHARED_PAGE) {
- /* becomes a SHARED (done below) partially used page */
- alloc->flag_partial_page = 1;
- }
- }
-
- STM_LIST_FOREACH(
- _STM_TL->uncommitted_pages,
- ({
- uintptr_t pagenum = (uintptr_t)item;
- stm_set_page_flag(pagenum, SHARED_PAGE);
- }));
- stm_list_clear(_STM_TL->uncommitted_pages);
}
void nursery_on_abort()
@@ -324,36 +203,16 @@
_STM_TL->nursery_current = nursery_base;
- /* forget about GCFLAG_NOT_COMMITTED objects by
- resetting alloc-pages */
- long j;
- for (j = 2; j < LARGE_OBJECT_WORDS; j++) {
- alloc_for_size_t *alloc = &_STM_TL->alloc[j];
- uint16_t num_allocated = ((uintptr_t)alloc->next) - alloc->start;
- uintptr_t next = (uintptr_t)alloc->next;
-
- if (num_allocated) {
- /* forget about all non-committed objects */
- alloc->next -= num_allocated;
-
- uintptr_t pagenum = ((uintptr_t)(next - 1)) / 4096UL;
- if (stm_get_page_flag(pagenum) == UNCOMMITTED_SHARED_PAGE) {
- /* the page will be freed below, we need a new one for the
- next allocation */
- alloc->next = 0;
- alloc->stop = 0;
- alloc->start = 0;
- }
- }
- }
+ /* free uncommitted objects */
+ struct stm_list_s *uncommitted = _STM_TL->uncommitted_objects;
- /* unreserve uncommitted_pages and mark them as SHARED again
- IFF they are not in alloc[] */
- STM_LIST_FOREACH(_STM_TL->uncommitted_pages, ({
- stm_pages_unreserve((uintptr_t)item);
- }));
- stm_list_clear(_STM_TL->uncommitted_pages);
-
+ STM_LIST_FOREACH(
+ uncommitted,
+ ({
+ stm_large_free(item);
+ }));
+
+ stm_list_clear(uncommitted);
}
diff --git a/c7/pages.h b/c7/pages.h
--- a/c7/pages.h
+++ b/c7/pages.h
@@ -7,11 +7,6 @@
/* page private for each thread */
PRIVATE_PAGE,
-
- /* set for SHARED pages that only contain objects belonging
- to the current transaction, so the whole page is not
- visible yet for other threads */
- UNCOMMITTED_SHARED_PAGE,
}; /* flag_page_private */
diff --git a/c7/test/support.py b/c7/test/support.py
--- a/c7/test/support.py
+++ b/c7/test/support.py
@@ -86,7 +86,6 @@
SHARED_PAGE=0,
REMAPPING_PAGE,
PRIVATE_PAGE,
- UNCOMMITTED_SHARED_PAGE,
}; /* flag_page_private */
enum {
@@ -103,10 +102,12 @@
void stm_large_free(object_t *data);
void _stm_large_dump(void);
-void _stm_chunk_pages(object_t *tldata, intptr_t *start, intptr_t *num);
-size_t _stm_data_size(object_t *tldata);
char *_stm_largemalloc_data_start(void);
+void _stm_move_object(char *src, char *dst);
+size_t _stm_data_size(struct object_s *data);
+void _stm_chunk_pages(struct object_s *data, uintptr_t *start, uintptr_t *num);
+
""")
lib = ffi.verify('''
diff --git a/c7/test/test_basic.py b/c7/test/test_basic.py
--- a/c7/test/test_basic.py
+++ b/c7/test/test_basic.py
@@ -353,8 +353,8 @@
new = stm_pop_root()
assert len(stm_get_obj_pages(new)) == 2
- assert ([stm_get_page_flag(p) for p in stm_get_obj_pages(new)]
- == [lib.UNCOMMITTED_SHARED_PAGE]*2)
+ # assert ([stm_get_page_flag(p) for p in stm_get_obj_pages(new)]
+ # == [lib.UNCOMMITTED_SHARED_PAGE]*2)
assert not is_in_nursery(new)
@@ -397,8 +397,8 @@
stm_push_root(new)
stm_minor_collect()
new = stm_pop_root()
- assert stm_get_page_flag(stm_get_obj_pages(new)[0]) ==
lib.UNCOMMITTED_SHARED_PAGE
- assert not (stm_get_flags(new) & lib.GCFLAG_NOT_COMMITTED)
+ # assert stm_get_page_flag(stm_get_obj_pages(new)[0]) ==
lib.UNCOMMITTED_SHARED_PAGE
+ # assert not (stm_get_flags(new) & lib.GCFLAG_NOT_COMMITTED)
stm_stop_transaction()
assert stm_get_page_flag(stm_get_obj_pages(new)[0]) == lib.SHARED_PAGE
@@ -446,7 +446,7 @@
stm_push_root(new)
stm_minor_collect()
new = stm_pop_root()
- assert stm_get_page_flag(stm_get_obj_pages(new)[0]) ==
lib.UNCOMMITTED_SHARED_PAGE
+ # assert stm_get_page_flag(stm_get_obj_pages(new)[0]) ==
lib.UNCOMMITTED_SHARED_PAGE
stm_abort_transaction()
stm_start_transaction()
_______________________________________________
pypy-commit mailing list
[email protected]
https://mail.python.org/mailman/listinfo/pypy-commit
