[PATCH 3/3] task_work: use TIF_TASKWORK if available

[Jens Axboe]

If the arch supports TIF_TASKWORK, then use that for TWA_SIGNAL as
it's more efficient than using the signal delivery method. This is
especially true on threaded applications, where ->sighand is shared
across threads.

Signed-off-by: Jens Axboe <ax...@kernel.dk>
---
 kernel/task_work.c | 48 ++++++++++++++++++++++++++++++++++------------
 1 file changed, 36 insertions(+), 12 deletions(-)

diff --git a/kernel/task_work.c b/kernel/task_work.c
index 613b2d634af8..ae317cfe86b8 100644
--- a/kernel/task_work.c
+++ b/kernel/task_work.c
@@ -5,6 +5,39 @@
 
 static struct callback_head work_exited; /* all we need is ->next == NULL */
 
+/*
+ * TWA_SIGNAL signaling - use TIF_TASKWORK, if available.
+ */
+static void task_work_signal(struct task_struct *task)
+{
+#ifndef TIF_TASKWORK
+       unsigned long flags;
+
+       /*
+        * Only grab the sighand lock if we don't already have some
+        * task_work pending. This pairs with the smp_store_mb()
+        * in get_signal(), see comment there.
+        */
+       if (!(READ_ONCE(task->jobctl) & JOBCTL_TASK_WORK) &&
+           lock_task_sighand(task, &flags)) {
+               task->jobctl |= JOBCTL_TASK_WORK;
+               signal_wake_up(task, 0);
+               unlock_task_sighand(task, &flags);
+       }
+#else
+       set_tsk_thread_flag(task, TIF_TASKWORK);
+       set_notify_resume(task);
+#endif
+}
+
+static inline void clear_tsk_taskwork(struct task_struct *task)
+{
+#ifdef TIF_TASKWORK
+       if (test_tsk_thread_flag(task, TIF_TASKWORK))
+               clear_tsk_thread_flag(task, TIF_TASKWORK);
+#endif
+}
+
 /**
  * task_work_add - ask the @task to execute @work->func()
  * @task: the task which should run the callback
@@ -28,7 +61,6 @@ int
 task_work_add(struct task_struct *task, struct callback_head *work, int notify)
 {
        struct callback_head *head;
-       unsigned long flags;
 
        do {
                head = READ_ONCE(task->task_works);
@@ -42,17 +74,7 @@ task_work_add(struct task_struct *task, struct callback_head 
*work, int notify)
                set_notify_resume(task);
                break;
        case TWA_SIGNAL:
-               /*
-                * Only grab the sighand lock if we don't already have some
-                * task_work pending. This pairs with the smp_store_mb()
-                * in get_signal(), see comment there.
-                */
-               if (!(READ_ONCE(task->jobctl) & JOBCTL_TASK_WORK) &&
-                   lock_task_sighand(task, &flags)) {
-                       task->jobctl |= JOBCTL_TASK_WORK;
-                       signal_wake_up(task, 0);
-                       unlock_task_sighand(task, &flags);
-               }
+               task_work_signal(task);
                break;
        }
 
@@ -110,6 +132,8 @@ void task_work_run(void)
        struct task_struct *task = current;
        struct callback_head *work, *head, *next;
 
+       clear_tsk_taskwork(task);
+
        for (;;) {
                /*
                 * work->func() can do task_work_add(), do not set
-- 
2.28.0