Author: mmacy
Date: Wed May 30 03:39:57 2018
New Revision: 334368
URL: https://svnweb.freebsd.org/changeset/base/334368
Log:
epoch(9): make epoch closer to style(9)
Modified:
head/sys/kern/subr_epoch.c
head/sys/sys/epoch.h
Modified: head/sys/kern/subr_epoch.c
==============================================================================
--- head/sys/kern/subr_epoch.c Wed May 30 03:00:57 2018 (r334367)
+++ head/sys/kern/subr_epoch.c Wed May 30 03:39:57 2018 (r334368)
@@ -72,49 +72,54 @@ SYSCTL_NODE(_kern_epoch, OID_AUTO, stats, CTLFLAG_RW,
/* Stats. */
static counter_u64_t block_count;
+
SYSCTL_COUNTER_U64(_kern_epoch_stats, OID_AUTO, nblocked, CTLFLAG_RW,
- &block_count, "# of times a thread was in an
epoch when epoch_wait was called");
+ &block_count, "# of times a thread was in an epoch when epoch_wait was
called");
static counter_u64_t migrate_count;
+
SYSCTL_COUNTER_U64(_kern_epoch_stats, OID_AUTO, migrations, CTLFLAG_RW,
- &migrate_count, "# of times thread was
migrated to another CPU in epoch_wait");
+ &migrate_count, "# of times thread was migrated to another CPU in
epoch_wait");
static counter_u64_t turnstile_count;
+
SYSCTL_COUNTER_U64(_kern_epoch_stats, OID_AUTO, ncontended, CTLFLAG_RW,
- &turnstile_count, "# of times a thread was
blocked on a lock in an epoch during an epoch_wait");
+ &turnstile_count, "# of times a thread was blocked on a lock in an epoch
during an epoch_wait");
static counter_u64_t switch_count;
+
SYSCTL_COUNTER_U64(_kern_epoch_stats, OID_AUTO, switches, CTLFLAG_RW,
- &switch_count, "# of times a thread
voluntarily context switched in epoch_wait");
+ &switch_count, "# of times a thread voluntarily context switched in
epoch_wait");
static counter_u64_t epoch_call_count;
+
SYSCTL_COUNTER_U64(_kern_epoch_stats, OID_AUTO, epoch_calls, CTLFLAG_RW,
- &epoch_call_count, "# of times a callback
was deferred");
+ &epoch_call_count, "# of times a callback was deferred");
static counter_u64_t epoch_call_task_count;
+
SYSCTL_COUNTER_U64(_kern_epoch_stats, OID_AUTO, epoch_call_tasks, CTLFLAG_RW,
- &epoch_call_task_count, "# of times a
callback task was run");
+ &epoch_call_task_count, "# of times a callback task was run");
-TAILQ_HEAD(threadlist, thread);
+TAILQ_HEAD (threadlist, thread);
CK_STACK_CONTAINER(struct ck_epoch_entry, stack_entry,
ck_epoch_entry_container)
-
typedef struct epoch_record {
ck_epoch_record_t er_record;
volatile struct threadlist er_tdlist;
volatile uint32_t er_gen;
uint32_t er_cpuid;
-} *epoch_record_t;
+} *epoch_record_t;
struct epoch_pcpu_state {
struct epoch_record eps_record;
-} __aligned(EPOCH_ALIGN);
+} __aligned(EPOCH_ALIGN);
struct epoch {
struct ck_epoch e_epoch __aligned(EPOCH_ALIGN);
struct epoch_pcpu_state *e_pcpu_dom[MAXMEMDOM] __aligned(EPOCH_ALIGN);
- int e_idx;
- int e_flags;
+ int e_idx;
+ int e_flags;
struct epoch_pcpu_state *e_pcpu[0];
};
-epoch_t allepochs[MAX_EPOCHS];
+epoch_t allepochs[MAX_EPOCHS];
DPCPU_DEFINE(struct grouptask, epoch_cb_task);
DPCPU_DEFINE(int, epoch_cb_count);
@@ -154,7 +159,7 @@ epoch_init(void *arg __unused)
printf("domcount[%d] %d\n", domain, domcount[domain]);
}
for (domain = 1; domain < vm_ndomains; domain++)
- domoffsets[domain] = domoffsets[domain-1] + domcount[domain-1];
+ domoffsets[domain] = domoffsets[domain - 1] + domcount[domain -
1];
for (domain = 0; domain < vm_ndomains; domain++) {
if (domcount[domain] == 0) {
@@ -162,7 +167,7 @@ epoch_init(void *arg __unused)
break;
}
}
- done:
+done:
CPU_FOREACH(cpu) {
GROUPTASK_INIT(DPCPU_ID_PTR(cpu, epoch_cb_task), 0,
epoch_call_task, NULL);
taskqgroup_attach_cpu(qgroup_softirq, DPCPU_ID_PTR(cpu,
epoch_cb_task), NULL, cpu, -1, "epoch call task");
@@ -171,6 +176,7 @@ epoch_init(void *arg __unused)
global_epoch = epoch_alloc(0);
global_epoch_preempt = epoch_alloc(EPOCH_PREEMPT);
}
+
SYSINIT(epoch, SI_SUB_TASKQ + 1, SI_ORDER_FIRST, epoch_init, NULL);
static void
@@ -181,8 +187,8 @@ epoch_init_numa(epoch_t epoch)
epoch_record_t er;
for (domain = 0; domain < vm_ndomains; domain++) {
- eps = malloc_domain(sizeof(*eps)*domcount[domain], M_EPOCH,
- domain,
M_ZERO|M_WAITOK);
+ eps = malloc_domain(sizeof(*eps) * domcount[domain], M_EPOCH,
+ domain, M_ZERO | M_WAITOK);
epoch->e_pcpu_dom[domain] = eps;
cpu_offset = domoffsets[domain];
for (int i = 0; i < domcount[domain]; i++, eps++) {
@@ -201,7 +207,7 @@ epoch_init_legacy(epoch_t epoch)
struct epoch_pcpu_state *eps;
epoch_record_t er;
- eps = malloc(sizeof(*eps)*mp_ncpus, M_EPOCH, M_ZERO|M_WAITOK);
+ eps = malloc(sizeof(*eps) * mp_ncpus, M_EPOCH, M_ZERO | M_WAITOK);
epoch->e_pcpu_dom[0] = eps;
for (int i = 0; i < mp_ncpus; i++, eps++) {
epoch->e_pcpu[i] = eps;
@@ -219,14 +225,14 @@ epoch_alloc(int flags)
if (__predict_false(!inited))
panic("%s called too early in boot", __func__);
- epoch = malloc(sizeof(struct epoch) + mp_ncpus*sizeof(void*),
- M_EPOCH, M_ZERO|M_WAITOK);
+ epoch = malloc(sizeof(struct epoch) + mp_ncpus * sizeof(void *),
+ M_EPOCH, M_ZERO | M_WAITOK);
ck_epoch_init(&epoch->e_epoch);
if (usedomains)
epoch_init_numa(epoch);
else
epoch_init_legacy(epoch);
- MPASS(epoch_count < MAX_EPOCHS-2);
+ MPASS(epoch_count < MAX_EPOCHS - 2);
epoch->e_flags = flags;
epoch->e_idx = epoch_count;
allepochs[epoch_count++] = epoch;
@@ -280,8 +286,8 @@ epoch_enter_preempt_internal(epoch_t epoch, struct thr
int found = 0;
TAILQ_FOREACH(curtd, &eps->eps_record.er_tdlist, td_epochq)
- if (curtd == td)
- found = 1;
+ if (curtd == td)
+ found = 1;
KASSERT(found, ("recursing on a second epoch"));
critical_exit();
return;
@@ -289,7 +295,7 @@ epoch_enter_preempt_internal(epoch_t epoch, struct thr
#endif
TAILQ_INSERT_TAIL(&eps->eps_record.er_tdlist, td, td_epochq);
sched_pin();
- ck_epoch_begin(&eps->eps_record.er_record,
(ck_epoch_section_t*)&td->td_epoch_section);
+ ck_epoch_begin(&eps->eps_record.er_record, (ck_epoch_section_t
*)&td->td_epoch_section);
critical_exit();
}
@@ -320,7 +326,7 @@ epoch_exit_preempt_internal(epoch_t epoch, struct thre
eps = epoch->e_pcpu[curcpu];
MPASS(epoch->e_flags & EPOCH_PREEMPT);
- ck_epoch_end(&eps->eps_record.er_record,
(ck_epoch_section_t*)&td->td_epoch_section);
+ ck_epoch_end(&eps->eps_record.er_record, (ck_epoch_section_t
*)&td->td_epoch_section);
TAILQ_REMOVE(&eps->eps_record.er_tdlist, td, td_epochq);
eps->eps_record.er_gen++;
sched_unpin();
@@ -351,7 +357,7 @@ epoch_exit(epoch_t epoch)
*/
static void
epoch_block_handler_preempt(struct ck_epoch *global __unused,
ck_epoch_record_t *cr,
- void *arg __unused)
+ void *arg __unused)
{
epoch_record_t record;
struct thread *td, *tdwait, *owner;
@@ -370,18 +376,17 @@ epoch_block_handler_preempt(struct ck_epoch *global __
* overhead of a migration
*/
if ((tdwait = TAILQ_FIRST(&record->er_tdlist)) != NULL &&
- TD_IS_RUNNING(tdwait)) {
+ TD_IS_RUNNING(tdwait)) {
gen = record->er_gen;
thread_unlock(td);
do {
cpu_spinwait();
} while (tdwait == TAILQ_FIRST(&record->er_tdlist) &&
- gen == record->er_gen &&
TD_IS_RUNNING(tdwait) &&
- spincount++ < MAX_ADAPTIVE_SPIN);
+ gen == record->er_gen && TD_IS_RUNNING(tdwait) &&
+ spincount++ < MAX_ADAPTIVE_SPIN);
thread_lock(td);
return;
}
-
/*
* Being on the same CPU as that of the record on which
* we need to wait allows us access to the thread
@@ -403,7 +408,7 @@ epoch_block_handler_preempt(struct ck_epoch *global __
return;
}
/*
- * Try to find a thread in an epoch section on this CPU
+ * Try to find a thread in an epoch section on this CPU
* waiting on a turnstile. Otherwise find the lowest
* priority thread (highest prio value) and drop our priority
* to match to allow it to run.
@@ -424,7 +429,7 @@ epoch_block_handler_preempt(struct ck_epoch *global __
critical_exit();
}
if (TD_IS_INHIBITED(tdwait) && TD_ON_LOCK(tdwait) &&
- ((ts = tdwait->td_blocked) != NULL)) {
+ ((ts = tdwait->td_blocked) != NULL)) {
/*
* We unlock td to allow turnstile_wait to reacquire the
* the thread lock. Before unlocking it we enter a
critical
@@ -455,7 +460,7 @@ epoch_block_handler_preempt(struct ck_epoch *global __
thread_lock(td);
critical_exit();
KASSERT(td->td_locks == 0,
- ("%d locks held", td->td_locks));
+ ("%d locks held", td->td_locks));
}
}
/*
@@ -529,12 +534,12 @@ epoch_wait_preempt(epoch_t epoch)
thread_unlock(td);
PICKUP_GIANT();
KASSERT(td->td_locks == locks,
- ("%d residual locks held", td->td_locks - locks));
+ ("%d residual locks held", td->td_locks - locks));
}
static void
epoch_block_handler(struct ck_epoch *g __unused, ck_epoch_record_t *c __unused,
- void *arg __unused)
+ void *arg __unused)
{
cpu_spinwait();
}
@@ -567,10 +572,10 @@ epoch_call(epoch_t epoch, epoch_context_t ctx, void (*
critical_enter();
*DPCPU_PTR(epoch_cb_count) += 1;
eps = epoch->e_pcpu[curcpu];
- ck_epoch_call(&eps->eps_record.er_record, cb, (ck_epoch_cb_t*)callback);
+ ck_epoch_call(&eps->eps_record.er_record, cb, (ck_epoch_cb_t
*)callback);
critical_exit();
return;
- boottime:
+boottime:
callback(ctx);
}
@@ -605,7 +610,8 @@ epoch_call_task(void *arg __unused)
head = ck_stack_batch_pop_npsc(&cb_stack);
for (cursor = head; cursor != NULL; cursor = next) {
struct ck_epoch_entry *entry =
- ck_epoch_entry_container(cursor);
+ ck_epoch_entry_container(cursor);
+
next = CK_STACK_NEXT(cursor);
entry->function(entry);
}
Modified: head/sys/sys/epoch.h
==============================================================================
--- head/sys/sys/epoch.h Wed May 30 03:00:57 2018 (r334367)
+++ head/sys/sys/epoch.h Wed May 30 03:39:57 2018 (r334368)
@@ -44,21 +44,21 @@ extern epoch_t global_epoch;
extern epoch_t global_epoch_preempt;
struct epoch_context {
- void *data[2];
-} __aligned(sizeof(void *));
+ void *data[2];
+} __aligned(sizeof(void *));
typedef struct epoch_context *epoch_context_t;
-epoch_t epoch_alloc(int flags);
-void epoch_free(epoch_t epoch);
-void epoch_enter(epoch_t epoch);
-void epoch_enter_preempt_internal(epoch_t epoch, struct thread *td);
-void epoch_exit(epoch_t epoch);
-void epoch_exit_preempt_internal(epoch_t epoch, struct thread *td);
-void epoch_wait(epoch_t epoch);
-void epoch_wait_preempt(epoch_t epoch);
-void epoch_call(epoch_t epoch, epoch_context_t ctx, void (*callback)
(epoch_context_t));
-int in_epoch(void);
+epoch_t epoch_alloc(int flags);
+void epoch_free(epoch_t epoch);
+void epoch_enter(epoch_t epoch);
+void epoch_enter_preempt_internal(epoch_t epoch, struct thread *td);
+void epoch_exit(epoch_t epoch);
+void epoch_exit_preempt_internal(epoch_t epoch, struct thread *td);
+void epoch_wait(epoch_t epoch);
+void epoch_wait_preempt(epoch_t epoch);
+void epoch_call(epoch_t epoch, epoch_context_t ctx, void (*callback)
(epoch_context_t));
+int in_epoch(void);
#ifdef _KERNEL
DPCPU_DECLARE(int, epoch_cb_count);
@@ -88,5 +88,5 @@ epoch_exit_preempt(epoch_t epoch)
if (td->td_epochnest-- == 1)
epoch_exit_preempt_internal(epoch, td);
}
-#endif /* _KERNEL */
+#endif /* _KERNEL */
#endif
_______________________________________________
svn-src-head@freebsd.org mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-head
To unsubscribe, send any mail to "svn-src-head-unsubscr...@freebsd.org"
