The trampoline's ldstub modifies the queue-tail byte of prom_entry_lock,
not its locked byte, because SPARC64 uses big-endian queued spinlocks.
Acquire the word with CASA, release byte three and poll while contended.
Retain the predicted-branch barriers for Spitfire erratum 51.
See qspinlock_types.h for the layout and JPS2 A.10/A.35 for CASA/MEMBAR.
Fixes: 145d97858597 ("arch/sparc: Enable queued spinlock support for SPARC")
Cc: [email protected]
Signed-off-by: Magnus Lindholm <[email protected]>
---
arch/sparc/kernel/trampoline_64.S | 34 +++++++++++++++++++++++++++----
1 file changed, 30 insertions(+), 4 deletions(-)
diff --git a/arch/sparc/kernel/trampoline_64.S
b/arch/sparc/kernel/trampoline_64.S
index 51bf1eb92a36..62f404b3079a 100644
--- a/arch/sparc/kernel/trampoline_64.S
+++ b/arch/sparc/kernel/trampoline_64.S
@@ -106,9 +106,19 @@ startup_continue:
* We lock 'num_kernel_image_mappings' consequetive entries.
*/
sethi %hi(prom_entry_lock), %g2
-1: ldstub [%g2 + %lo(prom_entry_lock)], %g1
+ /* Big-endian qspinlock: acquire the whole 32-bit value, not byte 0. */
+ or %g2, %lo(prom_entry_lock), %g2
+1: lduw [%g2], %g1
brnz,pn %g1, 1b
nop
+ mov 1, %g1
+ casa [%g2] ASI_P, %g0, %g1
+ brnz,pn %g1, 1b
+ nop
+ /* Spitfire erratum 51: barrier in a predicted-taken delay slot. */
+ ba,pt %xcc, 2f
+ membar #Sync
+2:
/* Get onto temporary stack which will be in the locked
* kernel image.
@@ -212,7 +222,10 @@ startup_continue:
nop
sethi %hi(prom_entry_lock), %g2
- stb %g0, [%g2 + %lo(prom_entry_lock)]
+ /* Match membar_safe() for Spitfire erratum 51. */
+ ba,pt %xcc, 2f
+ membar #StoreStore | #LoadStore
+2: stb %g0, [%g2 + %lo(prom_entry_lock) + 3]
ba,pt %xcc, after_lock_tlb
nop
@@ -326,9 +339,19 @@ after_lock_tlb:
wrpr %g0, 0, %wstate
sethi %hi(prom_entry_lock), %g2
-1: ldstub [%g2 + %lo(prom_entry_lock)], %g1
+ /* Big-endian qspinlock: acquire the whole 32-bit value, not byte 0. */
+ or %g2, %lo(prom_entry_lock), %g2
+1: lduw [%g2], %g1
brnz,pn %g1, 1b
nop
+ mov 1, %g1
+ casa [%g2] ASI_P, %g0, %g1
+ brnz,pn %g1, 1b
+ nop
+ /* Spitfire erratum 51: barrier in a predicted-taken delay slot. */
+ ba,pt %xcc, 2f
+ membar #Sync
+2:
/* As a hack, put &init_thread_union into %g6.
* prom_world() loads from here to restore the %asi
@@ -389,7 +412,10 @@ after_lock_tlb:
add %sp, (2047 + 128), %o0
3: sethi %hi(prom_entry_lock), %g2
- stb %g0, [%g2 + %lo(prom_entry_lock)]
+ /* Match membar_safe() for Spitfire erratum 51. */
+ ba,pt %xcc, 2f
+ membar #StoreStore | #LoadStore
+2: stb %g0, [%g2 + %lo(prom_entry_lock) + 3]
ldx [%l0], %g6
ldx [%g6 + TI_TASK], %g4
