Author: mmacy
Date: Thu May 17 19:50:55 2018
New Revision: 333759
URL: https://svnweb.freebsd.org/changeset/base/333759
Log:
epoch(9): eliminate the need to wait when polling for callbacks to run
by using ck's own callback handling mechanism we can simply check which
callbacks have had a grace period elapse
Approved by: sbruno
Modified:
head/sys/kern/subr_epoch.c
Modified: head/sys/kern/subr_epoch.c
==============================================================================
--- head/sys/kern/subr_epoch.c Thu May 17 19:41:58 2018 (r333758)
+++ head/sys/kern/subr_epoch.c Thu May 17 19:50:55 2018 (r333759)
@@ -64,6 +64,7 @@ static MALLOC_DEFINE(M_EPOCH, "epoch", "epoch based re
#endif
CTASSERT(sizeof(epoch_section_t) == sizeof(ck_epoch_section_t));
+CTASSERT(sizeof(ck_epoch_entry_t) == sizeof(struct epoch_context));
SYSCTL_NODE(_kern, OID_AUTO, epoch, CTLFLAG_RW, 0, "epoch information");
SYSCTL_NODE(_kern_epoch, OID_AUTO, stats, CTLFLAG_RW, 0, "epoch stats");
@@ -84,13 +85,11 @@ static counter_u64_t switch_count;
SYSCTL_COUNTER_U64(_kern_epoch_stats, OID_AUTO, switches, CTLFLAG_RW,
&switch_count, "# of times a thread
voluntarily context switched in epoch_wait");
-typedef struct epoch_cb {
- void (*ec_callback)(epoch_context_t);
- STAILQ_ENTRY(epoch_cb) ec_link;
-} *epoch_cb_t;
-
TAILQ_HEAD(threadlist, thread);
+CK_STACK_CONTAINER(struct ck_epoch_entry, stack_entry,
+ ck_epoch_entry_container)
+
typedef struct epoch_record {
ck_epoch_record_t er_record;
volatile struct threadlist er_tdlist;
@@ -100,7 +99,6 @@ typedef struct epoch_record {
struct epoch_pcpu_state {
struct epoch_record eps_record;
- STAILQ_HEAD(, epoch_cb) eps_cblist;
} __aligned(EPOCH_ALIGN);
struct epoch {
@@ -179,7 +177,6 @@ epoch_init_numa(epoch_t epoch)
for (int i = 0; i < domcount[domain]; i++, eps++) {
epoch->e_pcpu[cpu_offset + i] = eps;
er = &eps->eps_record;
- STAILQ_INIT(&eps->eps_cblist);
ck_epoch_register(&epoch->e_epoch, &er->er_record,
NULL);
TAILQ_INIT((struct threadlist
*)(uintptr_t)&er->er_tdlist);
er->er_cpuid = cpu_offset + i;
@@ -200,7 +197,6 @@ epoch_init_legacy(epoch_t epoch)
er = &eps->eps_record;
ck_epoch_register(&epoch->e_epoch, &er->er_record, NULL);
TAILQ_INIT((struct threadlist *)(uintptr_t)&er->er_tdlist);
- STAILQ_INIT(&eps->eps_cblist);
er->er_cpuid = i;
}
}
@@ -525,25 +521,24 @@ void
epoch_call(epoch_t epoch, epoch_context_t ctx, void (*callback)
(epoch_context_t))
{
struct epoch_pcpu_state *eps;
- epoch_cb_t cb;
+ ck_epoch_entry_t *cb;
cb = (void *)ctx;
MPASS(callback);
/* too early in boot to have epoch set up */
- if (__predict_false(epoch == NULL)) {
- callback(ctx);
- return;
- }
- MPASS(cb->ec_callback == NULL);
- MPASS(cb->ec_link.stqe_next == NULL);
- cb->ec_callback = callback;
+ if (__predict_false(epoch == NULL))
+ goto boottime;
+
counter_u64_add(epoch->e_frees, 1);
critical_enter();
eps = epoch->e_pcpu[curcpu];
- STAILQ_INSERT_HEAD(&eps->eps_cblist, cb, ec_link);
+ ck_epoch_call(&eps->eps_record.er_record, cb, (ck_epoch_cb_t*)callback);
critical_exit();
+ return;
+ boottime:
+ callback(ctx);
}
static void
@@ -551,28 +546,26 @@ epoch_call_task(void *context)
{
struct epoch_pcpu_state *eps;
epoch_t epoch;
- epoch_cb_t cb;
struct thread *td;
+ ck_stack_entry_t *cursor;
+ ck_stack_t deferred;
int cpu;
- STAILQ_HEAD(, epoch_cb) tmp_head;
epoch = context;
- STAILQ_INIT(&tmp_head);
td = curthread;
+ ck_stack_init(&deferred);
thread_lock(td);
CPU_FOREACH(cpu) {
sched_bind(td, cpu);
eps = epoch->e_pcpu[cpu];
- if (!STAILQ_EMPTY(&eps->eps_cblist))
- STAILQ_CONCAT(&tmp_head, &eps->eps_cblist);
+ ck_epoch_poll_deferred(&eps->eps_record.er_record, &deferred);
}
sched_unbind(td);
thread_unlock(td);
- epoch_wait(epoch);
-
- while ((cb = STAILQ_FIRST(&tmp_head)) != NULL) {
- STAILQ_REMOVE_HEAD(&tmp_head, ec_link);
- cb->ec_callback((void*)cb);
+ while((cursor = ck_stack_pop_npsc(&deferred)) != NULL) {
+ struct ck_epoch_entry *entry =
+ ck_epoch_entry_container(cursor);
+ entry->function(entry);
}
}
_______________________________________________
svn-src-all@freebsd.org mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-all
To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"
