Author: mav
Date: Sun Mar 20 14:37:37 2016
New Revision: 297066
URL: https://svnweb.freebsd.org/changeset/base/297066
Log:
MFC r256613, r256862: MFprojects/camlock r254763:
Move tq_enqueue() call out of the queue lock for known handlers (actually
I have found no others in the base system). This reduces queue lock hold
time and congestion spinning under active multithreaded enqueuing.
Modified:
stable/10/sys/kern/subr_taskqueue.c
Directory Properties:
stable/10/ (props changed)
Modified: stable/10/sys/kern/subr_taskqueue.c
==============================================================================
--- stable/10/sys/kern/subr_taskqueue.c Sun Mar 20 14:21:07 2016
(r297065)
+++ stable/10/sys/kern/subr_taskqueue.c Sun Mar 20 14:37:37 2016
(r297066)
@@ -46,6 +46,9 @@ __FBSDID("$FreeBSD$");
static MALLOC_DEFINE(M_TASKQUEUE, "taskqueue", "Task Queues");
static void *taskqueue_giant_ih;
static void *taskqueue_ih;
+static void taskqueue_fast_enqueue(void *);
+static void taskqueue_swi_enqueue(void *);
+static void taskqueue_swi_giant_enqueue(void *);
struct taskqueue_busy {
struct task *tb_running;
@@ -69,6 +72,7 @@ struct taskqueue {
#define TQ_FLAGS_ACTIVE (1 << 0)
#define TQ_FLAGS_BLOCKED (1 << 1)
+#define TQ_FLAGS_UNLOCKED_ENQUEUE (1 << 2)
#define DT_CALLOUT_ARMED (1 << 0)
@@ -96,7 +100,8 @@ _timeout_task_init(struct taskqueue *que
{
TASK_INIT(&timeout_task->t, priority, func, context);
- callout_init_mtx(&timeout_task->c, &queue->tq_mutex, 0);
+ callout_init_mtx(&timeout_task->c, &queue->tq_mutex,
+ CALLOUT_RETURNUNLOCKED);
timeout_task->q = queue;
timeout_task->f = 0;
}
@@ -127,6 +132,11 @@ _taskqueue_create(const char *name __unu
queue->tq_context = context;
queue->tq_spin = (mtxflags & MTX_SPIN) != 0;
queue->tq_flags |= TQ_FLAGS_ACTIVE;
+ if (enqueue == taskqueue_fast_enqueue ||
+ enqueue == taskqueue_swi_enqueue ||
+ enqueue == taskqueue_swi_giant_enqueue ||
+ enqueue == taskqueue_thread_enqueue)
+ queue->tq_flags |= TQ_FLAGS_UNLOCKED_ENQUEUE;
mtx_init(&queue->tq_mutex, mtxname, NULL, mtxflags);
return queue;
@@ -196,6 +206,7 @@ taskqueue_enqueue_locked(struct taskqueu
if (task->ta_pending) {
if (task->ta_pending < USHRT_MAX)
task->ta_pending++;
+ TQ_UNLOCK(queue);
return (0);
}
@@ -219,9 +230,14 @@ taskqueue_enqueue_locked(struct taskqueu
}
task->ta_pending = 1;
+ if ((queue->tq_flags & TQ_FLAGS_UNLOCKED_ENQUEUE) != 0)
+ TQ_UNLOCK(queue);
if ((queue->tq_flags & TQ_FLAGS_BLOCKED) == 0)
queue->tq_enqueue(queue->tq_context);
+ if ((queue->tq_flags & TQ_FLAGS_UNLOCKED_ENQUEUE) == 0)
+ TQ_UNLOCK(queue);
+ /* Return with lock released. */
return (0);
}
int
@@ -231,7 +247,7 @@ taskqueue_enqueue(struct taskqueue *queu
TQ_LOCK(queue);
res = taskqueue_enqueue_locked(queue, task);
- TQ_UNLOCK(queue);
+ /* The lock is released inside. */
return (res);
}
@@ -248,6 +264,7 @@ taskqueue_timeout_func(void *arg)
timeout_task->f &= ~DT_CALLOUT_ARMED;
queue->tq_callouts--;
taskqueue_enqueue_locked(timeout_task->q, &timeout_task->t);
+ /* The lock is released inside. */
}
int
@@ -264,6 +281,7 @@ taskqueue_enqueue_timeout(struct taskque
res = timeout_task->t.ta_pending;
if (ticks == 0) {
taskqueue_enqueue_locked(queue, &timeout_task->t);
+ /* The lock is released inside. */
} else {
if ((timeout_task->f & DT_CALLOUT_ARMED) != 0) {
res++;
@@ -277,8 +295,8 @@ taskqueue_enqueue_timeout(struct taskque
callout_reset(&timeout_task->c, ticks,
taskqueue_timeout_func, timeout_task);
}
+ TQ_UNLOCK(queue);
}
- TQ_UNLOCK(queue);
return (res);
}
@@ -591,7 +609,6 @@ taskqueue_thread_enqueue(void *context)
tqp = context;
tq = *tqp;
- TQ_ASSERT_LOCKED(tq);
wakeup_one(tq);
}
_______________________________________________
svn-src-stable-10@freebsd.org mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-stable-10
To unsubscribe, send any mail to "svn-src-stable-10-unsubscr...@freebsd.org"
