When the first queued waiter is a reader, wake all readers instead of just those that are at the front of the queue. This should result in increased parallelism for workloads that mix readers and writers.
Signed-off-by: Michel Lespinasse <[email protected]> --- lib/rwsem-spinlock.c | 39 ++++++++++++++++----------------------- 1 file changed, 16 insertions(+), 23 deletions(-) diff --git a/lib/rwsem-spinlock.c b/lib/rwsem-spinlock.c index 5f117f37ac0a..d22948eeaeae 100644 --- a/lib/rwsem-spinlock.c +++ b/lib/rwsem-spinlock.c @@ -70,36 +70,29 @@ __rwsem_do_wake(struct rw_semaphore *sem, int wakewrite) waiter = list_entry(sem->wait_list.next, struct rwsem_waiter, list); - if (!wakewrite) { - if (waiter->type == RWSEM_WAITING_FOR_WRITE) - goto out; - goto dont_wake_writers; - } - - /* - * as we support write lock stealing, we can't set sem->activity - * to -1 here to indicate we get the lock. Instead, we wake it up - * to let it go get it again. - */ if (waiter->type == RWSEM_WAITING_FOR_WRITE) { - wake_up_process(waiter->task); + if (wakewrite) + /* Wake up a writer. Note that we do not grant it the + * lock - it will have to acquire it when it runs. */ + wake_up_process(waiter->task); goto out; } - /* grant an infinite number of read locks to the front of the queue */ - dont_wake_writers: + /* grant read locks to all queued readers. */ woken = 0; - while (waiter->type == RWSEM_WAITING_FOR_READ) { + while (true) { struct list_head *next = waiter->list.next; - list_del(&waiter->list); - tsk = waiter->task; - smp_mb(); - waiter->task = NULL; - wake_up_process(tsk); - put_task_struct(tsk); - woken++; - if (list_empty(&sem->wait_list)) + if (waiter->type != RWSEM_WAITING_FOR_WRITE) { + list_del(&waiter->list); + tsk = waiter->task; + smp_mb(); + waiter->task = NULL; + wake_up_process(tsk); + put_task_struct(tsk); + woken++; + } + if (next == &sem->wait_list) break; waiter = list_entry(next, struct rwsem_waiter, list); } -- 1.8.1.3 -- To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html Please read the FAQ at http://www.tux.org/lkml/
