Author: Remi Meier <[email protected]>
Branch: cl-collector-thread
Changeset: r1709:caba34127c17
Date: 2015-03-09 15:47 +0100
http://bitbucket.org/pypy/stmgc/changeset/caba34127c17/
Log: fixes, call CL-collection on segment release, do validation of non-
running segments in CL-collection
diff --git a/c8/stm/commitlog.c b/c8/stm/commitlog.c
--- a/c8/stm/commitlog.c
+++ b/c8/stm/commitlog.c
@@ -25,7 +25,13 @@
__sync_add_and_fetch(&cle_allocated, add_or_remove);
}
-uint64_t _stm_cle_allocated(void) {
+static bool is_cle_collection_requested(void)
+{
+ return cle_allocated > CLE_COLLECT_BOUND;
+}
+
+uint64_t _stm_cle_allocated(void)
+{
return cle_allocated;
}
@@ -53,6 +59,7 @@
static void free_cle(struct stm_commit_log_entry_s *e)
{
+ dprintf(("free_cle(%p): written=%lu\n", e, e->written_count));
size_t byte_len = sizeof(struct stm_commit_log_entry_s) +
e->written_count * sizeof(struct stm_undo_s);
add_cle_allocated(-byte_len);
@@ -85,3 +92,110 @@
}
}
}
+
+
+
+void free_commit_log_entries_up_to(struct stm_commit_log_entry_s *end)
+{
+ struct stm_commit_log_entry_s *cl, *next;
+
+ cl = &commit_log_root;
+ assert(cl->next != NULL && cl->next != INEV_RUNNING);
+ assert(end->next != NULL && end->next != INEV_RUNNING);
+ assert(cl != end);
+
+ uint64_t rev_num = -1;
+ next = cl->next; /* guaranteed to exist */
+ do {
+ cl = next;
+ rev_num = cl->rev_num;
+
+ /* free bk copies of entries: */
+ long count = cl->written_count;
+ while (count-->0) {
+ free_bk(&cl->written[count]);
+ }
+
+ next = cl->next;
+ free_cle(cl);
+
+ assert(next != INEV_RUNNING);
+ if (cl == end) {
+ /* was the last one to free */
+ break;
+ }
+ } while (next != NULL);
+
+ /* set the commit_log_root to the last, common cl entry: */
+ commit_log_root.next = next;
+ commit_log_root.rev_num = rev_num;
+}
+
+void maybe_collect_commit_log()
+{
+ /* XXX: maybe use other lock, but right now we must make sure
+ that we do not run a major GC in parallel, since we do a
+ validate in some segments. */
+ assert(_has_mutex());
+
+ if (!is_cle_collection_requested())
+ return;
+
+ /* do validation in segments with no threads running, as some
+ of them may rarely run a thread and thus rarely advance in
+ the commit log. */
+ int original_num = STM_SEGMENT->segment_num;
+ for (long i = 0; i < NB_SEGMENTS; i++) {
+ struct stm_priv_segment_info_s *pseg = get_priv_segment(i);
+
+ if (pseg->pub.running_thread == NULL) {
+ assert(pseg->transaction_state == TS_NONE);
+
+ set_gs_register(get_segment_base(i));
+ bool ok = _stm_validate();
+ OPT_ASSERT(IMPLY(STM_PSEGMENT->transaction_state != TS_NONE,
+ ok));
+ /* only TS_NONE segments may have stale read-marker data
+ that reports a conflict here. but it is fine to ignore it. */
+ }
+ }
+ set_gs_register(get_segment_base(original_num));
+
+
+ /* look for the last common commit log entry. However,
+ don't free the last common one, as it may still be
+ used by running threads. */
+ struct stm_commit_log_entry_s *cl, *next;
+
+ cl = &commit_log_root;
+ next = cl;
+ do {
+ bool found = false;
+ for (int i = 0; i < NB_SEGMENTS; i++) {
+ struct stm_commit_log_entry_s *tmp;
+ tmp = get_priv_segment(i)->last_commit_log_entry;
+ if (next == tmp) {
+ found = true;
+ break;
+ }
+ }
+
+ if (found) {
+ /* cl is the last one to consider */
+ break;
+ } else {
+ cl = next;
+ }
+ } while ((next = cl->next) && next != INEV_RUNNING);
+
+
+ if (cl == &commit_log_root) {
+ dprintf(("WARN: triggered CLE collection w/o anything to do\n"));
+ return; /* none found that is newer */
+ }
+
+ /* cl should never be the last (common) one */
+ assert(cl->next != NULL && cl->next != INEV_RUNNING);
+
+ free_commit_log_entries_up_to(cl);
+}
diff --git a/c8/stm/commitlog.h b/c8/stm/commitlog.h
--- a/c8/stm/commitlog.h
+++ b/c8/stm/commitlog.h
@@ -1,7 +1,11 @@
/* when to trigger a CLE collection */
-#define CLE_COLLECT_BOUND (1*1024*1024) /* 1 MiB */
+#ifndef STM_TESTS
+#define CLE_COLLECT_BOUND (30*1024*1024) /* 30 MiB */
+#else
+#define CLE_COLLECT_BOUND 0 /* ASAP! */
+#endif
/* Commit Log things */
@@ -37,6 +41,8 @@
static struct stm_commit_log_entry_s commit_log_root;
+static bool is_cle_collection_requested(void);
+
static char *malloc_bk(size_t bk_size);
static void free_bk(struct stm_undo_s *undo);
static struct stm_commit_log_entry_s *malloc_cle(long entries);
diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c
--- a/c8/stm/gcpage.c
+++ b/c8/stm/gcpage.c
@@ -573,62 +573,6 @@
_stm_smallmalloc_sweep();
}
-static void clean_up_commit_log_entries()
-{
- struct stm_commit_log_entry_s *cl, *next;
-
-#ifndef NDEBUG
- /* check that all segments are at the same revision: */
- cl = get_priv_segment(0)->last_commit_log_entry;
- for (long i = 0; i < NB_SEGMENTS; i++) {
- struct stm_priv_segment_info_s *pseg = get_priv_segment(i);
- assert(pseg->last_commit_log_entry == cl);
- }
-#endif
-
- /* if there is only one element, we don't have to do anything: */
- cl = &commit_log_root;
-
- if (cl->next == NULL || cl->next == INEV_RUNNING) {
- assert(get_priv_segment(0)->last_commit_log_entry == cl);
- return;
- }
-
- bool was_inev = false;
- uint64_t rev_num = -1;
-
- next = cl->next; /* guaranteed to exist */
- do {
- cl = next;
- rev_num = cl->rev_num;
-
- /* free bk copies of entries: */
- long count = cl->written_count;
- while (count-->0) {
- free_bk(&cl->written[count]);
- }
-
- next = cl->next;
- free_cle(cl);
- if (next == INEV_RUNNING) {
- was_inev = true;
- break;
- }
- } while (next != NULL);
-
- /* set the commit_log_root to the last, common cl entry: */
- commit_log_root.next = was_inev ? INEV_RUNNING : NULL;
- commit_log_root.rev_num = rev_num;
-
- /* update in all segments: */
- for (long i = 0; i < NB_SEGMENTS; i++) {
- get_priv_segment(i)->last_commit_log_entry = &commit_log_root;
- }
-
- assert(_stm_count_cl_entries() == 0);
-}
-
-
static void major_collection_now_at_safe_point(void)
{
@@ -644,37 +588,6 @@
dprintf((" | commit log entries before: %ld\n",
_stm_count_cl_entries()));
-
- /* free all commit log entries. all segments are on the most recent
- revision now. */
- uint64_t allocd_before = pages_ctl.total_allocated;
- clean_up_commit_log_entries();
- /* check if freeing the log entries actually freed a considerable
- amount itself. Then we don't want to also trace the whole heap
- and just leave major gc right here.
- The problem is apparent from raytrace.py, but may disappear if
- we have card marking that also reduces the size of commit log
- entries */
- if ((pages_ctl.total_allocated < pages_ctl.total_allocated_bound)
- && (allocd_before - pages_ctl.total_allocated > 0.3 * allocd_before)) {
- /* 0.3 should mean that we are at about 50% of the way to the
- allocated_bound again */
-#ifndef STM_TESTS
- /* we freed a considerable amount just by freeing commit log entries */
- pages_ctl.major_collection_requested = false; // reset_m_gc_requested
-
- dprintf(("STOP AFTER FREEING CL ENTRIES: -%ld\n",
- (long)(allocd_before - pages_ctl.total_allocated)));
- dprintf((" | used after collection: %ld\n",
- (long)pages_ctl.total_allocated));
- dprintf((" `----------------------------------------------\n"));
- if (must_abort())
- abort_with_mutex();
-
- return;
-#endif
- }
-
/* only necessary because of assert that fails otherwise (XXX) */
acquire_all_privatization_locks();
diff --git a/c8/stm/sync.c b/c8/stm/sync.c
--- a/c8/stm/sync.c
+++ b/c8/stm/sync.c
@@ -166,6 +166,10 @@
assert(sync_ctl.in_use1[tl->last_associated_segment_num] == 1);
sync_ctl.in_use1[tl->last_associated_segment_num] = 0;
+
+ /* check for cle collection here after we released the segment
+ while we still hold the mutex */
+ maybe_collect_commit_log();
}
__attribute__((unused))
diff --git a/c8/test/support.py b/c8/test/support.py
--- a/c8/test/support.py
+++ b/c8/test/support.py
@@ -533,7 +533,6 @@
def stm_major_collect():
res = lib._check_stm_collect(1)
- assert count_commit_log_entries() == 0
if res == 1:
raise Conflict()
return res
diff --git a/c8/test/test_basic.py b/c8/test/test_basic.py
--- a/c8/test/test_basic.py
+++ b/c8/test/test_basic.py
@@ -40,7 +40,7 @@
self.commit_transaction()
assert last_commit_log_entry_objs() == []
- assert count_commit_log_entries() == 2
+ assert count_commit_log_entries() == 1
def test_simple_read(self):
self.start_transaction()
diff --git a/c8/test/test_gcpage.py b/c8/test/test_gcpage.py
--- a/c8/test/test_gcpage.py
+++ b/c8/test/test_gcpage.py
@@ -150,7 +150,8 @@
assert last_commit_log_entry_objs() == []
# 2 CLEs, 1 old object
assert lib._stm_total_allocated() == 5008 + LMO
- assert lib._stm_cle_allocated() == 2*CLEO
+ # however, on commit, we could free 1/2 CLE
+ assert lib._stm_cle_allocated() == 1*CLEO
self.start_transaction()
o = self.pop_root()
@@ -158,18 +159,18 @@
self.push_root(o)
self.commit_transaction()
assert last_commit_log_entry_objs() == [o]*2
- # 3 CLEs, 1 old object
+ # 2 CLEs, 1 old object
# also, 2 slices of bk_copy and thus 2 CLE entries
assert lib._stm_total_allocated() == 5008+LMO
- assert lib._stm_cle_allocated() == 3*CLEO + (5008 + CLEEO*2)
+ # however, on commit, we could free 1/2 CLE
+ assert lib._stm_cle_allocated() == 1*CLEO + (5008 + CLEEO*2)
self.start_transaction()
assert lib._stm_total_allocated() == 5008+LMO
- assert lib._stm_cle_allocated() == 3*CLEO + (5008 + CLEEO*2)
+ assert lib._stm_cle_allocated() == 1*CLEO + (5008 + CLEEO*2)
stm_major_collect()
- # all CLE and CLE entries freed:
assert lib._stm_total_allocated() == (5008+LMO)
- assert lib._stm_cle_allocated() == 0
+ assert lib._stm_cle_allocated() == 1*CLEO + (5008 + CLEEO*2)
self.commit_transaction()
@@ -456,14 +457,16 @@
self.start_transaction()
stm_major_collect()
self.commit_transaction()
+ assert count_commit_log_entries() == 1
self.switch(1)
self.start_transaction()
self.commit_transaction()
+ assert count_commit_log_entries() == 1
+ py.test.xfail("XXX: we never completely free the CLEs anymore")
self.switch(0)
self.start_transaction()
- assert count_commit_log_entries() == 2
stm_major_collect()
assert count_commit_log_entries() == 0
_______________________________________________
pypy-commit mailing list
[email protected]
https://mail.python.org/mailman/listinfo/pypy-commit
