In preparation for adding diagnostic checks to catch missing calls to
update_rq_clock(), provide wrappers for (re)pinning and unpinning
rq->lock.

Because the pending diagnostic checks allow state to be maintained in
rq_flags across pin contexts, swap the 'struct pin_cookie' arguments
for 'struct rq_flags *'.

Cc: Peter Zijlstra <pet...@infradead.org>
Cc: Ingo Molnar <mi...@kernel.org>
Cc: Mike Galbraith <umgwanakikb...@gmail.com>
Cc: Mel Gorman <mgor...@techsingularity.net>
Signed-off-by: Matt Fleming <m...@codeblueprint.co.uk>
---
 kernel/sched/core.c      | 80 ++++++++++++++++++++++++------------------------
 kernel/sched/deadline.c  | 10 +++---
 kernel/sched/fair.c      |  6 ++--
 kernel/sched/idle_task.c |  2 +-
 kernel/sched/rt.c        |  6 ++--
 kernel/sched/sched.h     | 31 ++++++++++++++-----
 kernel/sched/stop_task.c |  2 +-
 7 files changed, 76 insertions(+), 61 deletions(-)

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 860070fba814..7950c372fca0 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -185,7 +185,7 @@ struct rq *__task_rq_lock(struct task_struct *p, struct 
rq_flags *rf)
                rq = task_rq(p);
                raw_spin_lock(&rq->lock);
                if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) {
-                       rf->cookie = lockdep_pin_lock(&rq->lock);
+                       rq_pin_lock(rq, rf);
                        return rq;
                }
                raw_spin_unlock(&rq->lock);
@@ -225,7 +225,7 @@ struct rq *task_rq_lock(struct task_struct *p, struct 
rq_flags *rf)
                 * pair with the WMB to ensure we must then also see migrating.
                 */
                if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) {
-                       rf->cookie = lockdep_pin_lock(&rq->lock);
+                       rq_pin_lock(rq, rf);
                        return rq;
                }
                raw_spin_unlock(&rq->lock);
@@ -1184,9 +1184,9 @@ static int __set_cpus_allowed_ptr(struct task_struct *p,
                 * OK, since we're going to drop the lock immediately
                 * afterwards anyway.
                 */
-               lockdep_unpin_lock(&rq->lock, rf.cookie);
+               rq_unpin_lock(rq, &rf);
                rq = move_queued_task(rq, p, dest_cpu);
-               lockdep_repin_lock(&rq->lock, rf.cookie);
+               rq_repin_lock(rq, &rf);
        }
 out:
        task_rq_unlock(rq, p, &rf);
@@ -1679,7 +1679,7 @@ static inline void ttwu_activate(struct rq *rq, struct 
task_struct *p, int en_fl
  * Mark the task runnable and perform wakeup-preemption.
  */
 static void ttwu_do_wakeup(struct rq *rq, struct task_struct *p, int 
wake_flags,
-                          struct pin_cookie cookie)
+                          struct rq_flags *rf)
 {
        check_preempt_curr(rq, p, wake_flags);
        p->state = TASK_RUNNING;
@@ -1691,9 +1691,9 @@ static void ttwu_do_wakeup(struct rq *rq, struct 
task_struct *p, int wake_flags,
                 * Our task @p is fully woken up and running; so its safe to
                 * drop the rq->lock, hereafter rq is only used for statistics.
                 */
-               lockdep_unpin_lock(&rq->lock, cookie);
+               rq_unpin_lock(rq, rf);
                p->sched_class->task_woken(rq, p);
-               lockdep_repin_lock(&rq->lock, cookie);
+               rq_repin_lock(rq, rf);
        }
 
        if (rq->idle_stamp) {
@@ -1712,7 +1712,7 @@ static void ttwu_do_wakeup(struct rq *rq, struct 
task_struct *p, int wake_flags,
 
 static void
 ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags,
-                struct pin_cookie cookie)
+                struct rq_flags *rf)
 {
        int en_flags = ENQUEUE_WAKEUP;
 
@@ -1727,7 +1727,7 @@ ttwu_do_activate(struct rq *rq, struct task_struct *p, 
int wake_flags,
 #endif
 
        ttwu_activate(rq, p, en_flags);
-       ttwu_do_wakeup(rq, p, wake_flags, cookie);
+       ttwu_do_wakeup(rq, p, wake_flags, rf);
 }
 
 /*
@@ -1746,7 +1746,7 @@ static int ttwu_remote(struct task_struct *p, int 
wake_flags)
        if (task_on_rq_queued(p)) {
                /* check_preempt_curr() may use rq clock */
                update_rq_clock(rq);
-               ttwu_do_wakeup(rq, p, wake_flags, rf.cookie);
+               ttwu_do_wakeup(rq, p, wake_flags, &rf);
                ret = 1;
        }
        __task_rq_unlock(rq, &rf);
@@ -1759,15 +1759,15 @@ void sched_ttwu_pending(void)
 {
        struct rq *rq = this_rq();
        struct llist_node *llist = llist_del_all(&rq->wake_list);
-       struct pin_cookie cookie;
        struct task_struct *p;
        unsigned long flags;
+       struct rq_flags rf;
 
        if (!llist)
                return;
 
        raw_spin_lock_irqsave(&rq->lock, flags);
-       cookie = lockdep_pin_lock(&rq->lock);
+       rq_pin_lock(rq, &rf);
 
        while (llist) {
                int wake_flags = 0;
@@ -1778,10 +1778,10 @@ void sched_ttwu_pending(void)
                if (p->sched_remote_wakeup)
                        wake_flags = WF_MIGRATED;
 
-               ttwu_do_activate(rq, p, wake_flags, cookie);
+               ttwu_do_activate(rq, p, wake_flags, &rf);
        }
 
-       lockdep_unpin_lock(&rq->lock, cookie);
+       rq_unpin_lock(rq, &rf);
        raw_spin_unlock_irqrestore(&rq->lock, flags);
 }
 
@@ -1870,7 +1870,7 @@ bool cpus_share_cache(int this_cpu, int that_cpu)
 static void ttwu_queue(struct task_struct *p, int cpu, int wake_flags)
 {
        struct rq *rq = cpu_rq(cpu);
-       struct pin_cookie cookie;
+       struct rq_flags rf;
 
 #if defined(CONFIG_SMP)
        if (sched_feat(TTWU_QUEUE) && !cpus_share_cache(smp_processor_id(), 
cpu)) {
@@ -1881,9 +1881,9 @@ static void ttwu_queue(struct task_struct *p, int cpu, 
int wake_flags)
 #endif
 
        raw_spin_lock(&rq->lock);
-       cookie = lockdep_pin_lock(&rq->lock);
-       ttwu_do_activate(rq, p, wake_flags, cookie);
-       lockdep_unpin_lock(&rq->lock, cookie);
+       rq_pin_lock(rq, &rf);
+       ttwu_do_activate(rq, p, wake_flags, &rf);
+       rq_unpin_lock(rq, &rf);
        raw_spin_unlock(&rq->lock);
 }
 
@@ -2099,7 +2099,7 @@ out:
  * ensure that this_rq() is locked, @p is bound to this_rq() and not
  * the current task.
  */
-static void try_to_wake_up_local(struct task_struct *p, struct pin_cookie 
cookie)
+static void try_to_wake_up_local(struct task_struct *p, struct rq_flags *rf)
 {
        struct rq *rq = task_rq(p);
 
@@ -2116,11 +2116,11 @@ static void try_to_wake_up_local(struct task_struct *p, 
struct pin_cookie cookie
                 * disabled avoiding further scheduler activity on it and we've
                 * not yet picked a replacement task.
                 */
-               lockdep_unpin_lock(&rq->lock, cookie);
+               rq_unpin_lock(rq, rf);
                raw_spin_unlock(&rq->lock);
                raw_spin_lock(&p->pi_lock);
                raw_spin_lock(&rq->lock);
-               lockdep_repin_lock(&rq->lock, cookie);
+               rq_repin_lock(rq, rf);
        }
 
        if (!(p->state & TASK_NORMAL))
@@ -2131,7 +2131,7 @@ static void try_to_wake_up_local(struct task_struct *p, 
struct pin_cookie cookie
        if (!task_on_rq_queued(p))
                ttwu_activate(rq, p, ENQUEUE_WAKEUP);
 
-       ttwu_do_wakeup(rq, p, 0, cookie);
+       ttwu_do_wakeup(rq, p, 0, rf);
        ttwu_stat(p, smp_processor_id(), 0);
 out:
        raw_spin_unlock(&p->pi_lock);
@@ -2578,9 +2578,9 @@ void wake_up_new_task(struct task_struct *p)
                 * Nothing relies on rq->lock after this, so its fine to
                 * drop it.
                 */
-               lockdep_unpin_lock(&rq->lock, rf.cookie);
+               rq_unpin_lock(rq, &rf);
                p->sched_class->task_woken(rq, p);
-               lockdep_repin_lock(&rq->lock, rf.cookie);
+               rq_repin_lock(rq, &rf);
        }
 #endif
        task_rq_unlock(rq, p, &rf);
@@ -2845,7 +2845,7 @@ asmlinkage __visible void schedule_tail(struct 
task_struct *prev)
  */
 static __always_inline struct rq *
 context_switch(struct rq *rq, struct task_struct *prev,
-              struct task_struct *next, struct pin_cookie cookie)
+              struct task_struct *next, struct rq_flags *rf)
 {
        struct mm_struct *mm, *oldmm;
 
@@ -2877,7 +2877,7 @@ context_switch(struct rq *rq, struct task_struct *prev,
         * of the scheduler it's an obvious special-case), so we
         * do an early lockdep release here:
         */
-       lockdep_unpin_lock(&rq->lock, cookie);
+       rq_unpin_lock(rq, rf);
        spin_release(&rq->lock.dep_map, 1, _THIS_IP_);
 
        /* Here we just switch the register state and the stack. */
@@ -3241,7 +3241,7 @@ static inline void schedule_debug(struct task_struct 
*prev)
  * Pick up the highest-prio task:
  */
 static inline struct task_struct *
-pick_next_task(struct rq *rq, struct task_struct *prev, struct pin_cookie 
cookie)
+pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
 {
        const struct sched_class *class = &fair_sched_class;
        struct task_struct *p;
@@ -3252,20 +3252,20 @@ pick_next_task(struct rq *rq, struct task_struct *prev, 
struct pin_cookie cookie
         */
        if (likely(prev->sched_class == class &&
                   rq->nr_running == rq->cfs.h_nr_running)) {
-               p = fair_sched_class.pick_next_task(rq, prev, cookie);
+               p = fair_sched_class.pick_next_task(rq, prev, rf);
                if (unlikely(p == RETRY_TASK))
                        goto again;
 
                /* assumes fair_sched_class->next == idle_sched_class */
                if (unlikely(!p))
-                       p = idle_sched_class.pick_next_task(rq, prev, cookie);
+                       p = idle_sched_class.pick_next_task(rq, prev, rf);
 
                return p;
        }
 
 again:
        for_each_class(class) {
-               p = class->pick_next_task(rq, prev, cookie);
+               p = class->pick_next_task(rq, prev, rf);
                if (p) {
                        if (unlikely(p == RETRY_TASK))
                                goto again;
@@ -3319,7 +3319,7 @@ static void __sched notrace __schedule(bool preempt)
 {
        struct task_struct *prev, *next;
        unsigned long *switch_count;
-       struct pin_cookie cookie;
+       struct rq_flags rf;
        struct rq *rq;
        int cpu;
 
@@ -3353,7 +3353,7 @@ static void __sched notrace __schedule(bool preempt)
         */
        smp_mb__before_spinlock();
        raw_spin_lock(&rq->lock);
-       cookie = lockdep_pin_lock(&rq->lock);
+       rq_pin_lock(rq, &rf);
 
        rq->clock_skip_update <<= 1; /* promote REQ to ACT */
 
@@ -3375,7 +3375,7 @@ static void __sched notrace __schedule(bool preempt)
 
                                to_wakeup = wq_worker_sleeping(prev);
                                if (to_wakeup)
-                                       try_to_wake_up_local(to_wakeup, cookie);
+                                       try_to_wake_up_local(to_wakeup, &rf);
                        }
                }
                switch_count = &prev->nvcsw;
@@ -3384,7 +3384,7 @@ static void __sched notrace __schedule(bool preempt)
        if (task_on_rq_queued(prev))
                update_rq_clock(rq);
 
-       next = pick_next_task(rq, prev, cookie);
+       next = pick_next_task(rq, prev, &rf);
        clear_tsk_need_resched(prev);
        clear_preempt_need_resched();
        rq->clock_skip_update = 0;
@@ -3395,9 +3395,9 @@ static void __sched notrace __schedule(bool preempt)
                ++*switch_count;
 
                trace_sched_switch(preempt, prev, next);
-               rq = context_switch(rq, prev, next, cookie); /* unlocks the rq 
*/
+               rq = context_switch(rq, prev, next, &rf); /* unlocks the rq */
        } else {
-               lockdep_unpin_lock(&rq->lock, cookie);
+               rq_unpin_lock(rq, &rf);
                raw_spin_unlock_irq(&rq->lock);
        }
 
@@ -5487,7 +5487,7 @@ static void migrate_tasks(struct rq *dead_rq)
 {
        struct rq *rq = dead_rq;
        struct task_struct *next, *stop = rq->stop;
-       struct pin_cookie cookie;
+       struct rq_flags rf;
        int dest_cpu;
 
        /*
@@ -5519,8 +5519,8 @@ static void migrate_tasks(struct rq *dead_rq)
                /*
                 * pick_next_task assumes pinned rq->lock.
                 */
-               cookie = lockdep_pin_lock(&rq->lock);
-               next = pick_next_task(rq, &fake_task, cookie);
+               rq_pin_lock(rq, &rf);
+               next = pick_next_task(rq, &fake_task, &rf);
                BUG_ON(!next);
                next->sched_class->put_prev_task(rq, next);
 
@@ -5533,7 +5533,7 @@ static void migrate_tasks(struct rq *dead_rq)
                 * because !cpu_active at this point, which means load-balance
                 * will not interfere. Also, stop-machine.
                 */
-               lockdep_unpin_lock(&rq->lock, cookie);
+               rq_unpin_lock(rq, &rf);
                raw_spin_unlock(&rq->lock);
                raw_spin_lock(&next->pi_lock);
                raw_spin_lock(&rq->lock);
diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c
index 0c75bc656178..9c7abab9f61e 100644
--- a/kernel/sched/deadline.c
+++ b/kernel/sched/deadline.c
@@ -663,9 +663,9 @@ static enum hrtimer_restart dl_task_timer(struct hrtimer 
*timer)
                 * Nothing relies on rq->lock after this, so its safe to drop
                 * rq->lock.
                 */
-               lockdep_unpin_lock(&rq->lock, rf.cookie);
+               rq_unpin_lock(rq, &rf);
                push_dl_task(rq);
-               lockdep_repin_lock(&rq->lock, rf.cookie);
+               rq_repin_lock(rq, &rf);
        }
 #endif
 
@@ -1119,7 +1119,7 @@ static struct sched_dl_entity *pick_next_dl_entity(struct 
rq *rq,
 }
 
 struct task_struct *
-pick_next_task_dl(struct rq *rq, struct task_struct *prev, struct pin_cookie 
cookie)
+pick_next_task_dl(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
 {
        struct sched_dl_entity *dl_se;
        struct task_struct *p;
@@ -1134,9 +1134,9 @@ pick_next_task_dl(struct rq *rq, struct task_struct 
*prev, struct pin_cookie coo
                 * disabled avoiding further scheduler activity on it and we're
                 * being very careful to re-start the picking loop.
                 */
-               lockdep_unpin_lock(&rq->lock, cookie);
+               rq_unpin_lock(rq, rf);
                pull_dl_task(rq);
-               lockdep_repin_lock(&rq->lock, cookie);
+               rq_repin_lock(rq, rf);
                /*
                 * pull_rt_task() can drop (and re-acquire) rq->lock; this
                 * means a stop task can slip in, in which case we need to
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 85ca4ddab0d3..f0032827fb79 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -5718,7 +5718,7 @@ preempt:
 }
 
 static struct task_struct *
-pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct pin_cookie 
cookie)
+pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags 
*rf)
 {
        struct cfs_rq *cfs_rq = &rq->cfs;
        struct sched_entity *se;
@@ -5831,9 +5831,9 @@ idle:
         * further scheduler activity on it and we're being very careful to
         * re-start the picking loop.
         */
-       lockdep_unpin_lock(&rq->lock, cookie);
+       rq_unpin_lock(rq, rf);
        new_tasks = idle_balance(rq);
-       lockdep_repin_lock(&rq->lock, cookie);
+       rq_repin_lock(rq, rf);
        /*
         * Because idle_balance() releases (and re-acquires) rq->lock, it is
         * possible for any higher priority task to appear. In that case we
diff --git a/kernel/sched/idle_task.c b/kernel/sched/idle_task.c
index dedc81ecbb2e..5a84cc97118f 100644
--- a/kernel/sched/idle_task.c
+++ b/kernel/sched/idle_task.c
@@ -24,7 +24,7 @@ static void check_preempt_curr_idle(struct rq *rq, struct 
task_struct *p, int fl
 }
 
 static struct task_struct *
-pick_next_task_idle(struct rq *rq, struct task_struct *prev, struct pin_cookie 
cookie)
+pick_next_task_idle(struct rq *rq, struct task_struct *prev, struct rq_flags 
*rf)
 {
        put_prev_task(rq, prev);
 
diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c
index d5690b722691..ab79812ee33e 100644
--- a/kernel/sched/rt.c
+++ b/kernel/sched/rt.c
@@ -1524,7 +1524,7 @@ static struct task_struct *_pick_next_task_rt(struct rq 
*rq)
 }
 
 static struct task_struct *
-pick_next_task_rt(struct rq *rq, struct task_struct *prev, struct pin_cookie 
cookie)
+pick_next_task_rt(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
 {
        struct task_struct *p;
        struct rt_rq *rt_rq = &rq->rt;
@@ -1536,9 +1536,9 @@ pick_next_task_rt(struct rq *rq, struct task_struct 
*prev, struct pin_cookie coo
                 * disabled avoiding further scheduler activity on it and we're
                 * being very careful to re-start the picking loop.
                 */
-               lockdep_unpin_lock(&rq->lock, cookie);
+               rq_unpin_lock(rq, rf);
                pull_rt_task(rq);
-               lockdep_repin_lock(&rq->lock, cookie);
+               rq_repin_lock(rq, rf);
                /*
                 * pull_rt_task() can drop (and re-acquire) rq->lock; this
                 * means a dl or stop task can slip in, in which case we need
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 420c05d099c3..bf48e7975c23 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -761,6 +761,26 @@ static inline void rq_clock_skip_update(struct rq *rq, 
bool skip)
                rq->clock_skip_update &= ~RQCF_REQ_SKIP;
 }
 
+struct rq_flags {
+       unsigned long flags;
+       struct pin_cookie cookie;
+};
+
+static inline void rq_pin_lock(struct rq *rq, struct rq_flags *rf)
+{
+       rf->cookie = lockdep_pin_lock(&rq->lock);
+}
+
+static inline void rq_unpin_lock(struct rq *rq, struct rq_flags *rf)
+{
+       lockdep_unpin_lock(&rq->lock, rf->cookie);
+}
+
+static inline void rq_repin_lock(struct rq *rq, struct rq_flags *rf)
+{
+       lockdep_repin_lock(&rq->lock, rf->cookie);
+}
+
 #ifdef CONFIG_NUMA
 enum numa_topology_type {
        NUMA_DIRECT,
@@ -1212,7 +1232,7 @@ struct sched_class {
         */
        struct task_struct * (*pick_next_task) (struct rq *rq,
                                                struct task_struct *prev,
-                                               struct pin_cookie cookie);
+                                               struct rq_flags *rf);
        void (*put_prev_task) (struct rq *rq, struct task_struct *p);
 
 #ifdef CONFIG_SMP
@@ -1463,11 +1483,6 @@ static inline void sched_rt_avg_update(struct rq *rq, 
u64 rt_delta) { }
 static inline void sched_avg_update(struct rq *rq) { }
 #endif
 
-struct rq_flags {
-       unsigned long flags;
-       struct pin_cookie cookie;
-};
-
 struct rq *__task_rq_lock(struct task_struct *p, struct rq_flags *rf)
        __acquires(rq->lock);
 struct rq *task_rq_lock(struct task_struct *p, struct rq_flags *rf)
@@ -1477,7 +1492,7 @@ struct rq *task_rq_lock(struct task_struct *p, struct 
rq_flags *rf)
 static inline void __task_rq_unlock(struct rq *rq, struct rq_flags *rf)
        __releases(rq->lock)
 {
-       lockdep_unpin_lock(&rq->lock, rf->cookie);
+       rq_unpin_lock(rq, rf);
        raw_spin_unlock(&rq->lock);
 }
 
@@ -1486,7 +1501,7 @@ task_rq_unlock(struct rq *rq, struct task_struct *p, 
struct rq_flags *rf)
        __releases(rq->lock)
        __releases(p->pi_lock)
 {
-       lockdep_unpin_lock(&rq->lock, rf->cookie);
+       rq_unpin_lock(rq, rf);
        raw_spin_unlock(&rq->lock);
        raw_spin_unlock_irqrestore(&p->pi_lock, rf->flags);
 }
diff --git a/kernel/sched/stop_task.c b/kernel/sched/stop_task.c
index 604297a08b3a..9f69fb630853 100644
--- a/kernel/sched/stop_task.c
+++ b/kernel/sched/stop_task.c
@@ -24,7 +24,7 @@ check_preempt_curr_stop(struct rq *rq, struct task_struct *p, 
int flags)
 }
 
 static struct task_struct *
-pick_next_task_stop(struct rq *rq, struct task_struct *prev, struct pin_cookie 
cookie)
+pick_next_task_stop(struct rq *rq, struct task_struct *prev, struct rq_flags 
*rf)
 {
        struct task_struct *stop = rq->stop;
 
-- 
2.9.3

Reply via email to