Author: glebius Date: Tue Feb 11 18:48:07 2020 New Revision: 357771 URL: https://svnweb.freebsd.org/changeset/base/357771
Log: Add flag to struct task to mark the task as requiring network epoch. When processing a taskqueue and a task has associated epoch, then enter for duration of the task. If consecutive tasks belong to the same epoch, batch them. Now we are talking about the network epoch only. Shrink the ta_priority size to 8-bits. No current consumers use a priority that won't fit into 8 bits. Also complexity of taskqueue_enqueue() is a square of maximum value of priority, so we unlikely ever want to go over UCHAR_MAX here. Reviewed by: hselasky Differential Revision: https://reviews.freebsd.org/D23518 Modified: head/sys/kern/subr_gtaskqueue.c head/sys/kern/subr_taskqueue.c head/sys/sys/_task.h head/sys/sys/epoch.h head/sys/sys/gtaskqueue.h head/sys/sys/taskqueue.h Modified: head/sys/kern/subr_gtaskqueue.c ============================================================================== --- head/sys/kern/subr_gtaskqueue.c Tue Feb 11 18:19:56 2020 (r357770) +++ head/sys/kern/subr_gtaskqueue.c Tue Feb 11 18:48:07 2020 (r357771) @@ -41,6 +41,7 @@ __FBSDID("$FreeBSD$"); #include <sys/malloc.h> #include <sys/mutex.h> #include <sys/proc.h> +#include <sys/epoch.h> #include <sys/sched.h> #include <sys/smp.h> #include <sys/gtaskqueue.h> @@ -342,13 +343,16 @@ gtaskqueue_unblock(struct gtaskqueue *queue) static void gtaskqueue_run_locked(struct gtaskqueue *queue) { + struct epoch_tracker et; struct gtaskqueue_busy tb; struct gtask *gtask; + bool in_net_epoch; KASSERT(queue != NULL, ("tq is NULL")); TQ_ASSERT_LOCKED(queue); tb.tb_running = NULL; LIST_INSERT_HEAD(&queue->tq_active, &tb, tb_link); + in_net_epoch = false; while ((gtask = STAILQ_FIRST(&queue->tq_queue)) != NULL) { STAILQ_REMOVE_HEAD(&queue->tq_queue, ta_link); @@ -358,11 +362,20 @@ gtaskqueue_run_locked(struct gtaskqueue *queue) TQ_UNLOCK(queue); KASSERT(gtask->ta_func != NULL, ("task->ta_func is NULL")); + if (!in_net_epoch && TASK_IS_NET(gtask)) { + in_net_epoch = true; + NET_EPOCH_ENTER(et); + } else if (in_net_epoch && !TASK_IS_NET(gtask)) { + NET_EPOCH_EXIT(et); + in_net_epoch = false; + } gtask->ta_func(gtask->ta_context); TQ_LOCK(queue); wakeup(gtask); } + if (in_net_epoch) + NET_EPOCH_EXIT(et); LIST_REMOVE(&tb, tb_link); } Modified: head/sys/kern/subr_taskqueue.c ============================================================================== --- head/sys/kern/subr_taskqueue.c Tue Feb 11 18:19:56 2020 (r357770) +++ head/sys/kern/subr_taskqueue.c Tue Feb 11 18:48:07 2020 (r357771) @@ -42,6 +42,7 @@ __FBSDID("$FreeBSD$"); #include <sys/malloc.h> #include <sys/mutex.h> #include <sys/proc.h> +#include <sys/epoch.h> #include <sys/sched.h> #include <sys/smp.h> #include <sys/taskqueue.h> @@ -371,7 +372,7 @@ taskqueue_drain_tq_queue(struct taskqueue *queue) * anyway) so just insert it at tail while we have the * queue lock. */ - TASK_INIT(&t_barrier, USHRT_MAX, taskqueue_task_nop_fn, &t_barrier); + TASK_INIT(&t_barrier, UCHAR_MAX, taskqueue_task_nop_fn, &t_barrier); STAILQ_INSERT_TAIL(&queue->tq_queue, &t_barrier, ta_link); queue->tq_hint = &t_barrier; t_barrier.ta_pending = 1; @@ -442,14 +443,17 @@ taskqueue_unblock(struct taskqueue *queue) static void taskqueue_run_locked(struct taskqueue *queue) { + struct epoch_tracker et; struct taskqueue_busy tb; struct task *task; + bool in_net_epoch; int pending; KASSERT(queue != NULL, ("tq is NULL")); TQ_ASSERT_LOCKED(queue); tb.tb_running = NULL; LIST_INSERT_HEAD(&queue->tq_active, &tb, tb_link); + in_net_epoch = false; while ((task = STAILQ_FIRST(&queue->tq_queue)) != NULL) { STAILQ_REMOVE_HEAD(&queue->tq_queue, ta_link); @@ -462,11 +466,20 @@ taskqueue_run_locked(struct taskqueue *queue) TQ_UNLOCK(queue); KASSERT(task->ta_func != NULL, ("task->ta_func is NULL")); + if (!in_net_epoch && TASK_IS_NET(task)) { + in_net_epoch = true; + NET_EPOCH_ENTER(et); + } else if (in_net_epoch && !TASK_IS_NET(task)) { + NET_EPOCH_EXIT(et); + in_net_epoch = false; + } task->ta_func(task->ta_context, pending); TQ_LOCK(queue); wakeup(task); } + if (in_net_epoch) + NET_EPOCH_EXIT(et); LIST_REMOVE(&tb, tb_link); } Modified: head/sys/sys/_task.h ============================================================================== --- head/sys/sys/_task.h Tue Feb 11 18:19:56 2020 (r357770) +++ head/sys/sys/_task.h Tue Feb 11 18:48:07 2020 (r357771) @@ -48,10 +48,17 @@ typedef void task_fn_t(void *context, int pending); struct task { STAILQ_ENTRY(task) ta_link; /* (q) link for queue */ uint16_t ta_pending; /* (q) count times queued */ - u_short ta_priority; /* (c) Priority */ + uint8_t ta_priority; /* (c) Priority */ + uint8_t ta_flags; /* (c) Flags */ task_fn_t *ta_func; /* (c) task handler */ void *ta_context; /* (c) argument for handler */ }; + +#define TASK_ENQUEUED 0x1 +#define TASK_NOENQUEUE 0x2 +#define TASK_NETWORK 0x4 + +#define TASK_IS_NET(ta) ((ta)->ta_flags & TASK_NETWORK) #ifdef _KERNEL Modified: head/sys/sys/epoch.h ============================================================================== --- head/sys/sys/epoch.h Tue Feb 11 18:19:56 2020 (r357770) +++ head/sys/sys/epoch.h Tue Feb 11 18:48:07 2020 (r357771) @@ -104,6 +104,9 @@ extern epoch_t net_epoch_preempt; #define NET_EPOCH_WAIT() epoch_wait_preempt(net_epoch_preempt) #define NET_EPOCH_CALL(f, c) epoch_call(net_epoch_preempt, (f), (c)) #define NET_EPOCH_ASSERT() MPASS(in_epoch(net_epoch_preempt)) +#define NET_TASK_INIT(t, p, f, c) TASK_INIT_FLAGS(t, p, f, c, TASK_NETWORK) +#define NET_GROUPTASK_INIT(gtask, prio, func, ctx) \ + GTASK_INIT(&(gtask)->gt_task, TASK_NETWORK, (prio), (func), (ctx)) #endif /* _KERNEL */ #endif /* _SYS_EPOCH_H_ */ Modified: head/sys/sys/gtaskqueue.h ============================================================================== --- head/sys/sys/gtaskqueue.h Tue Feb 11 18:19:56 2020 (r357770) +++ head/sys/sys/gtaskqueue.h Tue Feb 11 18:48:07 2020 (r357771) @@ -84,10 +84,6 @@ void taskqgroup_config_gtask_init(void *ctx, struct gr gtask_fn_t *fn, const char *name); void taskqgroup_config_gtask_deinit(struct grouptask *gtask); -#define TASK_ENQUEUED 0x1 -#define TASK_SKIP_WAKEUP 0x2 -#define TASK_NOENQUEUE 0x4 - #define GTASK_INIT(gtask, flags, priority, func, context) do { \ (gtask)->ta_flags = flags; \ (gtask)->ta_priority = (priority); \ @@ -96,7 +92,7 @@ void taskqgroup_config_gtask_deinit(struct grouptask * } while (0) #define GROUPTASK_INIT(gtask, priority, func, context) \ - GTASK_INIT(&(gtask)->gt_task, TASK_SKIP_WAKEUP, priority, func, context) + GTASK_INIT(&(gtask)->gt_task, 0, priority, func, context) #define GROUPTASK_ENQUEUE(gtask) \ grouptaskqueue_enqueue((gtask)->gt_taskqueue, &(gtask)->gt_task) Modified: head/sys/sys/taskqueue.h ============================================================================== --- head/sys/sys/taskqueue.h Tue Feb 11 18:19:56 2020 (r357770) +++ head/sys/sys/taskqueue.h Tue Feb 11 18:48:07 2020 (r357771) @@ -107,8 +107,7 @@ void taskqueue_set_callback(struct taskqueue *queue, taskqueue_callback_fn callback, void *context); #define TASK_INITIALIZER(priority, func, context) \ - { .ta_pending = 0, \ - .ta_priority = (priority), \ + { .ta_priority = (priority), \ .ta_func = (func), \ .ta_context = (context) } @@ -121,18 +120,25 @@ void taskqueue_thread_enqueue(void *context); /* * Initialise a task structure. */ -#define TASK_INIT(task, priority, func, context) do { \ - (task)->ta_pending = 0; \ - (task)->ta_priority = (priority); \ - (task)->ta_func = (func); \ - (task)->ta_context = (context); \ +#define TASK_INIT_FLAGS(task, priority, func, context, flags) do { \ + MPASS((priority) >= 0 && (priority) <= 255); \ + (task)->ta_pending = 0; \ + (task)->ta_priority = (priority); \ + (task)->ta_flags = (flags); \ + (task)->ta_func = (func); \ + (task)->ta_context = (context); \ } while (0) +#define TASK_INIT(t, p, f, c) TASK_INIT_FLAGS(t, p, f, c, 0) + void _timeout_task_init(struct taskqueue *queue, struct timeout_task *timeout_task, int priority, task_fn_t func, void *context); -#define TIMEOUT_TASK_INIT(queue, timeout_task, priority, func, context) \ - _timeout_task_init(queue, timeout_task, priority, func, context); +#define TIMEOUT_TASK_INIT(queue, timeout_task, priority, func, context) do { \ + _Static_assert((priority) >= 0 && (priority) <= 255, \ + "struct task priority is 8 bit in size"); \ + _timeout_task_init(queue, timeout_task, priority, func, context); \ +} while (0) /* * Declare a reference to a taskqueue. _______________________________________________ svn-src-all@freebsd.org mailing list https://lists.freebsd.org/mailman/listinfo/svn-src-all To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"
