Re: svn commit: r333775 - in head/sys: kern net sys tests/epoch

2018-05-17 Thread Matthew Macy 
>   - epoch_enter_critical() - can be called inside a different epoch,
> starts a section that will acquire any MTX_DEF mutexes or do
> anything that might sleep.

Should read will _not_ acquire ...

>   - epoch_exit_critical() - corresponding exit call
>   - epoch_wait_critical() - wait variant that is guaranteed that any
> threads in a section are running.
>   - epoch_global_critical - an epoch_wait_critical safe epoch instance
>
>   Requested by:   markj
>   Approved by:  sbruno
>
> Modified:
>   head/sys/kern/subr_epoch.c
>   head/sys/net/if.c
>   head/sys/sys/epoch.h
>   head/sys/tests/epoch/epoch_test.c
>
> Modified: head/sys/kern/subr_epoch.c
> ==
> --- head/sys/kern/subr_epoch.c  Thu May 17 23:59:56 2018(r333774)
> +++ head/sys/kern/subr_epoch.c  Fri May 18 01:52:51 2018(r333775)
> @@ -83,6 +83,12 @@ SYSCTL_COUNTER_U64(_kern_epoch_stats, OID_AUTO, nconte
>  static counter_u64_t switch_count;
>  SYSCTL_COUNTER_U64(_kern_epoch_stats, OID_AUTO, switches, CTLFLAG_RW,
>_count, "# of times a thread 
> voluntarily context switched in epoch_wait");
> +static counter_u64_t epoch_call_count;
> +SYSCTL_COUNTER_U64(_kern_epoch_stats, OID_AUTO, epoch_calls, CTLFLAG_RW,
> +  _call_count, "# of times a callback 
> was deferred");
> +static counter_u64_t epoch_call_task_count;
> +SYSCTL_COUNTER_U64(_kern_epoch_stats, OID_AUTO, epoch_call_tasks, CTLFLAG_RW,
> +  _call_task_count, "# of times a 
> callback task was run");
>
>  TAILQ_HEAD(threadlist, thread);
>
> @@ -103,9 +109,8 @@ struct epoch_pcpu_state {
>  struct epoch {
> struct ck_epoch e_epoch __aligned(EPOCH_ALIGN);
> struct epoch_pcpu_state *e_pcpu_dom[MAXMEMDOM] __aligned(EPOCH_ALIGN);
> -   counter_u64_t e_frees;
> -   uint64_t e_free_last;
> int e_idx;
> +   int e_flags;
> struct epoch_pcpu_state *e_pcpu[0];
>  };
>
> @@ -119,7 +124,7 @@ static __read_mostly int domoffsets[MAXMEMDOM];
>  static __read_mostly int inited;
>  static __read_mostly int epoch_count;
>  __read_mostly epoch_t global_epoch;
> -static __read_mostly epoch_t private_epoch;
> +__read_mostly epoch_t global_epoch_critical;
>
>  static void epoch_call_task(void *context __unused);
>
> @@ -161,8 +166,8 @@ epoch_init(void *arg __unused)
> taskqgroup_attach_cpu(qgroup_softirq, DPCPU_ID_PTR(cpu, 
> epoch_cb_task), NULL, cpu, -1, "epoch call task");
> }
> inited = 1;
> -   global_epoch = epoch_alloc();
> -   private_epoch = epoch_alloc();
> +   global_epoch = epoch_alloc(0);
> +   global_epoch_critical = epoch_alloc(EPOCH_CRITICAL);
>  }
>  SYSINIT(epoch, SI_SUB_TASKQ + 1, SI_ORDER_FIRST, epoch_init, NULL);
>
> @@ -206,7 +211,7 @@ epoch_init_legacy(epoch_t epoch)
>  }
>
>  epoch_t
> -epoch_alloc(void)
> +epoch_alloc(int flags)
>  {
> epoch_t epoch;
>
> @@ -215,12 +220,12 @@ epoch_alloc(void)
> epoch = malloc(sizeof(struct epoch) + mp_ncpus*sizeof(void*),
>M_EPOCH, M_ZERO|M_WAITOK);
> ck_epoch_init(>e_epoch);
> -   epoch->e_frees = counter_u64_alloc(M_WAITOK);
> if (usedomains)
> epoch_init_numa(epoch);
> else
> epoch_init_legacy(epoch);
> MPASS(epoch_count < MAX_EPOCHS-2);
> +   epoch->e_flags = flags;
> epoch->e_idx = epoch_count;
> allepochs[epoch_count++] = epoch;
> return (epoch);
> @@ -240,11 +245,7 @@ epoch_free(epoch_t epoch)
> }
>  #endif
> allepochs[epoch->e_idx] = NULL;
> -   epoch_wait(private_epoch);
> -   /*
> -* Execute any lingering callbacks
> -*/
> -   counter_u64_free(epoch->e_frees);
> +   epoch_wait_critical(global_epoch_critical);
> if (usedomains)
> for (domain = 0; domain < vm_ndomains; domain++)
> free_domain(epoch->e_pcpu_dom[domain], M_EPOCH);
> @@ -289,14 +290,21 @@ epoch_enter_internal(epoch_t epoch, struct thread *td)
>  }
>
>
> -static void
> -epoch_enter_private(ck_epoch_section_t *section)
> +void
> +epoch_enter_critical(epoch_t epoch)
>  {
> -   struct epoch_pcpu_state *eps;
> +   ck_epoch_record_t *record;
> +   ck_epoch_section_t *section;
> +   struct thread *td;
>
> -   MPASS(curthread->td_critnest);
> -   eps = private_epoch->e_pcpu[curcpu];
> -   ck_epoch_begin(>eps_record.er_record, section);
> +   section = NULL;
> +   td = curthread;
> +   critical_enter();
> +   if (__predict_true(td->td_epochnest++ == 0))
> +   section = (ck_epoch_section_t*)>td_epoch_section;
> +
> +   record = >e_pcpu[curcpu]->eps_record.er_record;
> +   ck_epoch_begin(record, section);
>  }
>
>  void
> @@ -321,14 +329,21 @@ epoch_exit_internal(epoch_t epoch, struct thread *td)
>

svn commit: r333775 - in head/sys: kern net sys tests/epoch

2018-05-17 Thread Matt Macy 
Author: mmacy
Date: Fri May 18 01:52:51 2018
New Revision: 333775
URL: https://svnweb.freebsd.org/changeset/base/333775

Log:
  epoch: add non-preemptible "critical" variant
  
  adds:
  - epoch_enter_critical() - can be called inside a different epoch,
starts a section that will acquire any MTX_DEF mutexes or do
anything that might sleep.
  - epoch_exit_critical() - corresponding exit call
  - epoch_wait_critical() - wait variant that is guaranteed that any
threads in a section are running.
  - epoch_global_critical - an epoch_wait_critical safe epoch instance
  
  Requested by:   markj
  Approved by:  sbruno

Modified:
  head/sys/kern/subr_epoch.c
  head/sys/net/if.c
  head/sys/sys/epoch.h
  head/sys/tests/epoch/epoch_test.c

Modified: head/sys/kern/subr_epoch.c
==
--- head/sys/kern/subr_epoch.c  Thu May 17 23:59:56 2018(r333774)
+++ head/sys/kern/subr_epoch.c  Fri May 18 01:52:51 2018(r333775)
@@ -83,6 +83,12 @@ SYSCTL_COUNTER_U64(_kern_epoch_stats, OID_AUTO, nconte
 static counter_u64_t switch_count;
 SYSCTL_COUNTER_U64(_kern_epoch_stats, OID_AUTO, switches, CTLFLAG_RW,
   _count, "# of times a thread 
voluntarily context switched in epoch_wait");
+static counter_u64_t epoch_call_count;
+SYSCTL_COUNTER_U64(_kern_epoch_stats, OID_AUTO, epoch_calls, CTLFLAG_RW,
+  _call_count, "# of times a callback 
was deferred");
+static counter_u64_t epoch_call_task_count;
+SYSCTL_COUNTER_U64(_kern_epoch_stats, OID_AUTO, epoch_call_tasks, CTLFLAG_RW,
+  _call_task_count, "# of times a 
callback task was run");
 
 TAILQ_HEAD(threadlist, thread);
 
@@ -103,9 +109,8 @@ struct epoch_pcpu_state {
 struct epoch {
struct ck_epoch e_epoch __aligned(EPOCH_ALIGN);
struct epoch_pcpu_state *e_pcpu_dom[MAXMEMDOM] __aligned(EPOCH_ALIGN);
-   counter_u64_t e_frees;
-   uint64_t e_free_last;
int e_idx;
+   int e_flags;
struct epoch_pcpu_state *e_pcpu[0];
 };
 
@@ -119,7 +124,7 @@ static __read_mostly int domoffsets[MAXMEMDOM];
 static __read_mostly int inited;
 static __read_mostly int epoch_count;
 __read_mostly epoch_t global_epoch;
-static __read_mostly epoch_t private_epoch;
+__read_mostly epoch_t global_epoch_critical;
 
 static void epoch_call_task(void *context __unused);
 
@@ -161,8 +166,8 @@ epoch_init(void *arg __unused)
taskqgroup_attach_cpu(qgroup_softirq, DPCPU_ID_PTR(cpu, 
epoch_cb_task), NULL, cpu, -1, "epoch call task");
}
inited = 1;
-   global_epoch = epoch_alloc();
-   private_epoch = epoch_alloc();
+   global_epoch = epoch_alloc(0);
+   global_epoch_critical = epoch_alloc(EPOCH_CRITICAL);
 }
 SYSINIT(epoch, SI_SUB_TASKQ + 1, SI_ORDER_FIRST, epoch_init, NULL);
 
@@ -206,7 +211,7 @@ epoch_init_legacy(epoch_t epoch)
 }
 
 epoch_t
-epoch_alloc(void)
+epoch_alloc(int flags)
 {
epoch_t epoch;
 
@@ -215,12 +220,12 @@ epoch_alloc(void)
epoch = malloc(sizeof(struct epoch) + mp_ncpus*sizeof(void*),
   M_EPOCH, M_ZERO|M_WAITOK);
ck_epoch_init(>e_epoch);
-   epoch->e_frees = counter_u64_alloc(M_WAITOK);
if (usedomains)
epoch_init_numa(epoch);
else
epoch_init_legacy(epoch);
MPASS(epoch_count < MAX_EPOCHS-2);
+   epoch->e_flags = flags;
epoch->e_idx = epoch_count;
allepochs[epoch_count++] = epoch;
return (epoch);
@@ -240,11 +245,7 @@ epoch_free(epoch_t epoch)
}
 #endif
allepochs[epoch->e_idx] = NULL;
-   epoch_wait(private_epoch);
-   /*
-* Execute any lingering callbacks
-*/
-   counter_u64_free(epoch->e_frees);
+   epoch_wait_critical(global_epoch_critical);
if (usedomains)
for (domain = 0; domain < vm_ndomains; domain++)
free_domain(epoch->e_pcpu_dom[domain], M_EPOCH);
@@ -289,14 +290,21 @@ epoch_enter_internal(epoch_t epoch, struct thread *td)
 }
 
 
-static void
-epoch_enter_private(ck_epoch_section_t *section)
+void
+epoch_enter_critical(epoch_t epoch)
 {
-   struct epoch_pcpu_state *eps;
+   ck_epoch_record_t *record;
+   ck_epoch_section_t *section;
+   struct thread *td;
 
-   MPASS(curthread->td_critnest);
-   eps = private_epoch->e_pcpu[curcpu];
-   ck_epoch_begin(>eps_record.er_record, section);
+   section = NULL;
+   td = curthread;
+   critical_enter();
+   if (__predict_true(td->td_epochnest++ == 0))
+   section = (ck_epoch_section_t*)>td_epoch_section;
+
+   record = >e_pcpu[curcpu]->eps_record.er_record;
+   ck_epoch_begin(record, section);
 }
 
 void
@@ -321,14 +329,21 @@ epoch_exit_internal(epoch_t epoch, struct thread *td)
critical_exit();
 }
 
-static void
-epoch_exit_private(ck_epoch_section_t *section)
+void