Use the spin_begin/spin_cpu_relax/spin_end APIs in qspinlock, which helps
to prevent threads issuing a lot of expensive priority nops which may not
have much effect due to immediately executing low then medium priority.

Signed-off-by: Nicholas Piggin <[email protected]>
---
 arch/powerpc/lib/qspinlock.c | 41 ++++++++++++++++++++++++++++++++----
 1 file changed, 37 insertions(+), 4 deletions(-)

diff --git a/arch/powerpc/lib/qspinlock.c b/arch/powerpc/lib/qspinlock.c
index 830a90a66f5f..ea8886e2922b 100644
--- a/arch/powerpc/lib/qspinlock.c
+++ b/arch/powerpc/lib/qspinlock.c
@@ -203,6 +203,7 @@ static struct qnode *get_tail_qnode(struct qspinlock *lock, 
u32 val)
        BUG();
 }
 
+/* Called inside spin_begin() */
 static __always_inline void __yield_to_locked_owner(struct qspinlock *lock, 
u32 val, bool paravirt, bool mustq)
 {
        int owner;
@@ -222,6 +223,8 @@ static __always_inline void __yield_to_locked_owner(struct 
qspinlock *lock, u32
        if ((yield_count & 1) == 0)
                goto relax; /* owner vcpu is running */
 
+       spin_end();
+
        /*
         * Read the lock word after sampling the yield count. On the other side
         * there may a wmb because the yield count update is done by the
@@ -237,18 +240,22 @@ static __always_inline void 
__yield_to_locked_owner(struct qspinlock *lock, u32
                yield_to_preempted(owner, yield_count);
                if (mustq)
                        set_mustq(lock);
+               spin_begin();
                /* Don't relax if we yielded. Maybe we should? */
                return;
        }
+       spin_begin();
 relax:
-       cpu_relax();
+       spin_cpu_relax();
 }
 
+/* Called inside spin_begin() */
 static __always_inline void yield_to_locked_owner(struct qspinlock *lock, u32 
val, bool paravirt)
 {
        __yield_to_locked_owner(lock, val, paravirt, false);
 }
 
+/* Called inside spin_begin() */
 static __always_inline void yield_head_to_locked_owner(struct qspinlock *lock, 
u32 val, bool paravirt)
 {
        bool mustq = false;
@@ -286,6 +293,7 @@ static __always_inline void propagate_yield_cpu(struct 
qnode *node, u32 val, int
        }
 }
 
+/* Called inside spin_begin() */
 static __always_inline void yield_to_prev(struct qspinlock *lock, struct qnode 
*node, u32 val, bool paravirt)
 {
        int prev_cpu = decode_tail_cpu(val);
@@ -310,14 +318,18 @@ static __always_inline void yield_to_prev(struct 
qspinlock *lock, struct qnode *
        if ((yield_count & 1) == 0)
                goto yield_prev; /* owner vcpu is running */
 
+       spin_end();
+
        smp_rmb();
 
        if (yield_cpu == node->yield_cpu) {
                if (node->next && node->next->yield_cpu != yield_cpu)
                        node->next->yield_cpu = yield_cpu;
                yield_to_preempted(yield_cpu, yield_count);
+               spin_begin();
                return;
        }
+       spin_begin();
 
 yield_prev:
        if (!pv_yield_prev)
@@ -327,15 +339,19 @@ static __always_inline void yield_to_prev(struct 
qspinlock *lock, struct qnode *
        if ((yield_count & 1) == 0)
                goto relax; /* owner vcpu is running */
 
+       spin_end();
+
        smp_rmb(); /* See __yield_to_locked_owner comment */
 
        if (!node->locked) {
                yield_to_preempted(prev_cpu, yield_count);
+               spin_begin();
                return;
        }
+       spin_begin();
 
 relax:
-       cpu_relax();
+       spin_cpu_relax();
 }
 
 
@@ -347,6 +363,8 @@ static __always_inline bool try_to_steal_lock(struct 
qspinlock *lock, bool parav
                return false;
 
        /* Attempt to steal the lock */
+       spin_begin();
+
        do {
                u32 val = READ_ONCE(lock->val);
 
@@ -354,8 +372,10 @@ static __always_inline bool try_to_steal_lock(struct 
qspinlock *lock, bool parav
                        break;
 
                if (unlikely(!(val & _Q_LOCKED_VAL))) {
+                       spin_end();
                        if (trylock_with_tail_cpu(lock, val))
                                return true;
+                       spin_begin();
                } else {
                        yield_to_locked_owner(lock, val, paravirt);
                }
@@ -363,6 +383,8 @@ static __always_inline bool try_to_steal_lock(struct 
qspinlock *lock, bool parav
                iters++;
        } while (iters < get_steal_spins(paravirt));
 
+       spin_end();
+
        return false;
 }
 
@@ -411,8 +433,10 @@ static __always_inline void 
queued_spin_lock_mcs_queue(struct qspinlock *lock, b
                WRITE_ONCE(prev->next, node);
 
                /* Wait for mcs node lock to be released */
+               spin_begin();
                while (!node->locked)
                        yield_to_prev(lock, node, old, paravirt);
+               spin_end();
 
                /* Clear out stale propagated yield_cpu */
                if (paravirt && pv_yield_propagate_owner && node->yield_cpu != 
-1)
@@ -425,6 +449,7 @@ static __always_inline void 
queued_spin_lock_mcs_queue(struct qspinlock *lock, b
                int set_yield_cpu = -1;
 
                /* We're at the head of the waitqueue, wait for the lock. */
+               spin_begin();
                for (;;) {
                        val = READ_ONCE(lock->val);
                        if (!(val & _Q_LOCKED_VAL))
@@ -433,6 +458,7 @@ static __always_inline void 
queued_spin_lock_mcs_queue(struct qspinlock *lock, b
                        propagate_yield_cpu(node, val, &set_yield_cpu, 
paravirt);
                        yield_head_to_locked_owner(lock, val, paravirt);
                }
+               spin_end();
 
                /* If we're the last queued, must clean up the tail. */
                if ((val & _Q_TAIL_CPU_MASK) == tail) {
@@ -450,6 +476,7 @@ static __always_inline void 
queued_spin_lock_mcs_queue(struct qspinlock *lock, b
 
 again:
                /* We're at the head of the waitqueue, wait for the lock. */
+               spin_begin();
                for (;;) {
                        val = READ_ONCE(lock->val);
                        if (!(val & _Q_LOCKED_VAL))
@@ -465,6 +492,7 @@ static __always_inline void 
queued_spin_lock_mcs_queue(struct qspinlock *lock, b
                                val |= _Q_MUST_Q_VAL;
                        }
                }
+               spin_end();
 
                /* If we're the last queued, must clean up the tail. */
                if ((val & _Q_TAIL_CPU_MASK) == tail) {
@@ -480,8 +508,13 @@ static __always_inline void 
queued_spin_lock_mcs_queue(struct qspinlock *lock, b
 
 unlock_next:
        /* contended path; must wait for next != NULL (MCS protocol) */
-       while (!(next = READ_ONCE(node->next)))
-               cpu_relax();
+       next = READ_ONCE(node->next);
+       if (!next) {
+               spin_begin();
+               while (!(next = READ_ONCE(node->next)))
+                       cpu_relax();
+               spin_end();
+       }
 
        /*
         * Unlock the next mcs waiter node. Release barrier is not required
-- 
2.37.2

Reply via email to