[PATCH -v3 5/8] sched/rtmutex: Refactor rt_mutex_setprio()

[Peter Zijlstra]

With the introduction of SCHED_DEADLINE the whole notion that priority
is a single number is gone, therefore the @prio argument to
rt_mutex_setprio() doesn't make sense anymore.

So rework the code to pass a pi_task instead.

Note this also fixes a problem with pi_top_task caching; previously we
would not set the pointer (call rt_mutex_update_top_task) if the
priority didn't change, this could lead to a stale pointer.

As for the XXX, I think its fine to use pi_task->prio, because if it
differs from waiter->prio, a PI chain update is immenent.

Signed-off-by: Peter Zijlstra (Intel) 
---
 include/linux/sched/rt.h |   24 +++---
 kernel/locking/rtmutex.c |  112 ---
 kernel/sched/core.c  |   66 ++-
 3 files changed, 91 insertions(+), 111 deletions(-)

--- a/include/linux/sched/rt.h
+++ b/include/linux/sched/rt.h
@@ -18,28 +18,20 @@ static inline int rt_task(struct task_st
 }
 
 #ifdef CONFIG_RT_MUTEXES
-extern int rt_mutex_getprio(struct task_struct *p);
-extern void rt_mutex_setprio(struct task_struct *p, int prio);
-extern int rt_mutex_get_effective_prio(struct task_struct *task, int newprio);
-extern void rt_mutex_update_top_task(struct task_struct *p);
-extern struct task_struct *rt_mutex_get_top_task(struct task_struct *task);
+/*
+ * Must hold either p->pi_lock or task_rq(p)->lock.
+ */
+static inline struct task_struct *rt_mutex_get_top_task(struct task_struct *p)
+{
+   return p->pi_top_task;
+}
+extern void rt_mutex_setprio(struct task_struct *p, struct task_struct 
*pi_task);
 extern void rt_mutex_adjust_pi(struct task_struct *p);
 static inline bool tsk_is_pi_blocked(struct task_struct *tsk)
 {
return tsk->pi_blocked_on != NULL;
 }
 #else
-static inline int rt_mutex_getprio(struct task_struct *p)
-{
-   return p->normal_prio;
-}
-
-static inline int rt_mutex_get_effective_prio(struct task_struct *task,
- int newprio)
-{
-   return newprio;
-}
-
 static inline struct task_struct *rt_mutex_get_top_task(struct task_struct 
*task)
 {
return NULL;
--- a/kernel/locking/rtmutex.c
+++ b/kernel/locking/rtmutex.c
@@ -322,67 +322,16 @@ rt_mutex_dequeue_pi(struct task_struct *
RB_CLEAR_NODE(>pi_tree_entry);
 }
 
-/*
- * Must hold both p->pi_lock and task_rq(p)->lock.
- */
-void rt_mutex_update_top_task(struct task_struct *p)
-{
-   if (!task_has_pi_waiters(p)) {
-   p->pi_top_task = NULL;
-   return;
-   }
-
-   p->pi_top_task = task_top_pi_waiter(p)->task;
-}
-
-/*
- * Calculate task priority from the waiter tree priority
- *
- * Return task->normal_prio when the waiter tree is empty or when
- * the waiter is not allowed to do priority boosting
- */
-int rt_mutex_getprio(struct task_struct *task)
-{
-   if (likely(!task_has_pi_waiters(task)))
-   return task->normal_prio;
-
-   return min(task_top_pi_waiter(task)->prio,
-  task->normal_prio);
-}
-
-/*
- * Must hold either p->pi_lock or task_rq(p)->lock.
- */
-struct task_struct *rt_mutex_get_top_task(struct task_struct *task)
-{
-   return task->pi_top_task;
-}
-
-/*
- * Called by sched_setscheduler() to get the priority which will be
- * effective after the change.
- */
-int rt_mutex_get_effective_prio(struct task_struct *task, int newprio)
+static void rt_mutex_adjust_prio(struct task_struct *p)
 {
-   struct task_struct *top_task = rt_mutex_get_top_task(task);
+   struct task_struct *pi_task = NULL;
 
-   if (!top_task)
-   return newprio;
+   lockdep_assert_held(>pi_lock);
 
-   return min(top_task->prio, newprio);
-}
+   if (task_has_pi_waiters(p))
+   pi_task = task_top_pi_waiter(p)->task;
 
-/*
- * Adjust the priority of a task, after its pi_waiters got modified.
- *
- * This can be both boosting and unboosting. task->pi_lock must be held.
- */
-static void __rt_mutex_adjust_prio(struct task_struct *task)
-{
-   int prio = rt_mutex_getprio(task);
-
-   if (task->prio != prio || dl_prio(prio))
-   rt_mutex_setprio(task, prio);
+   rt_mutex_setprio(p, pi_task);
 }
 
 /*
@@ -742,7 +691,7 @@ static int rt_mutex_adjust_prio_chain(st
 */
rt_mutex_dequeue_pi(task, prerequeue_top_waiter);
rt_mutex_enqueue_pi(task, waiter);
-   __rt_mutex_adjust_prio(task);
+   rt_mutex_adjust_prio(task);
 
} else if (prerequeue_top_waiter == waiter) {
/*
@@ -758,7 +707,7 @@ static int rt_mutex_adjust_prio_chain(st
rt_mutex_dequeue_pi(task, waiter);
waiter = rt_mutex_top_waiter(lock);
rt_mutex_enqueue_pi(task, waiter);
-   __rt_mutex_adjust_prio(task);
+   rt_mutex_adjust_prio(task);
} else {
/*
 * Nothing changed. No need to do any priority
@@ -966,7 

[PATCH -v3 5/8] sched/rtmutex: Refactor rt_mutex_setprio()

[Peter Zijlstra]

With the introduction of SCHED_DEADLINE the whole notion that priority
is a single number is gone, therefore the @prio argument to
rt_mutex_setprio() doesn't make sense anymore.

So rework the code to pass a pi_task instead.

Note this also fixes a problem with pi_top_task caching; previously we
would not set the pointer (call rt_mutex_update_top_task) if the
priority didn't change, this could lead to a stale pointer.

As for the XXX, I think its fine to use pi_task->prio, because if it
differs from waiter->prio, a PI chain update is immenent.

Signed-off-by: Peter Zijlstra (Intel) 
---
 include/linux/sched/rt.h |   24 +++---
 kernel/locking/rtmutex.c |  112 ---
 kernel/sched/core.c  |   66 ++-
 3 files changed, 91 insertions(+), 111 deletions(-)

--- a/include/linux/sched/rt.h
+++ b/include/linux/sched/rt.h
@@ -18,28 +18,20 @@ static inline int rt_task(struct task_st
 }
 
 #ifdef CONFIG_RT_MUTEXES
-extern int rt_mutex_getprio(struct task_struct *p);
-extern void rt_mutex_setprio(struct task_struct *p, int prio);
-extern int rt_mutex_get_effective_prio(struct task_struct *task, int newprio);
-extern void rt_mutex_update_top_task(struct task_struct *p);
-extern struct task_struct *rt_mutex_get_top_task(struct task_struct *task);
+/*
+ * Must hold either p->pi_lock or task_rq(p)->lock.
+ */
+static inline struct task_struct *rt_mutex_get_top_task(struct task_struct *p)
+{
+   return p->pi_top_task;
+}
+extern void rt_mutex_setprio(struct task_struct *p, struct task_struct 
*pi_task);
 extern void rt_mutex_adjust_pi(struct task_struct *p);
 static inline bool tsk_is_pi_blocked(struct task_struct *tsk)
 {
return tsk->pi_blocked_on != NULL;
 }
 #else
-static inline int rt_mutex_getprio(struct task_struct *p)
-{
-   return p->normal_prio;
-}
-
-static inline int rt_mutex_get_effective_prio(struct task_struct *task,
- int newprio)
-{
-   return newprio;
-}
-
 static inline struct task_struct *rt_mutex_get_top_task(struct task_struct 
*task)
 {
return NULL;
--- a/kernel/locking/rtmutex.c
+++ b/kernel/locking/rtmutex.c
@@ -322,67 +322,16 @@ rt_mutex_dequeue_pi(struct task_struct *
RB_CLEAR_NODE(>pi_tree_entry);
 }
 
-/*
- * Must hold both p->pi_lock and task_rq(p)->lock.
- */
-void rt_mutex_update_top_task(struct task_struct *p)
-{
-   if (!task_has_pi_waiters(p)) {
-   p->pi_top_task = NULL;
-   return;
-   }
-
-   p->pi_top_task = task_top_pi_waiter(p)->task;
-}
-
-/*
- * Calculate task priority from the waiter tree priority
- *
- * Return task->normal_prio when the waiter tree is empty or when
- * the waiter is not allowed to do priority boosting
- */
-int rt_mutex_getprio(struct task_struct *task)
-{
-   if (likely(!task_has_pi_waiters(task)))
-   return task->normal_prio;
-
-   return min(task_top_pi_waiter(task)->prio,
-  task->normal_prio);
-}
-
-/*
- * Must hold either p->pi_lock or task_rq(p)->lock.
- */
-struct task_struct *rt_mutex_get_top_task(struct task_struct *task)
-{
-   return task->pi_top_task;
-}
-
-/*
- * Called by sched_setscheduler() to get the priority which will be
- * effective after the change.
- */
-int rt_mutex_get_effective_prio(struct task_struct *task, int newprio)
+static void rt_mutex_adjust_prio(struct task_struct *p)
 {
-   struct task_struct *top_task = rt_mutex_get_top_task(task);
+   struct task_struct *pi_task = NULL;
 
-   if (!top_task)
-   return newprio;
+   lockdep_assert_held(>pi_lock);
 
-   return min(top_task->prio, newprio);
-}
+   if (task_has_pi_waiters(p))
+   pi_task = task_top_pi_waiter(p)->task;
 
-/*
- * Adjust the priority of a task, after its pi_waiters got modified.
- *
- * This can be both boosting and unboosting. task->pi_lock must be held.
- */
-static void __rt_mutex_adjust_prio(struct task_struct *task)
-{
-   int prio = rt_mutex_getprio(task);
-
-   if (task->prio != prio || dl_prio(prio))
-   rt_mutex_setprio(task, prio);
+   rt_mutex_setprio(p, pi_task);
 }
 
 /*
@@ -742,7 +691,7 @@ static int rt_mutex_adjust_prio_chain(st
 */
rt_mutex_dequeue_pi(task, prerequeue_top_waiter);
rt_mutex_enqueue_pi(task, waiter);
-   __rt_mutex_adjust_prio(task);
+   rt_mutex_adjust_prio(task);
 
} else if (prerequeue_top_waiter == waiter) {
/*
@@ -758,7 +707,7 @@ static int rt_mutex_adjust_prio_chain(st
rt_mutex_dequeue_pi(task, waiter);
waiter = rt_mutex_top_waiter(lock);
rt_mutex_enqueue_pi(task, waiter);
-   __rt_mutex_adjust_prio(task);
+   rt_mutex_adjust_prio(task);
} else {
/*
 * Nothing changed. No need to do any priority
@@ -966,7 +915,7 @@ static int