ChangeSet 1.2226.2.1, 2005/04/01 14:35:01-08:00, [EMAIL PROTECTED]
[SPARC64]: Store per-cpu pointer in IMMU TSB register.
This trick requires that we PAGE_SIZE align the per-cpu
areas, but that is fine. Also, reduce the __per_cpu_offsets[]
array into just two values, a base and a shift.
Signed-off-by: David S. Miller <[EMAIL PROTECTED]>
arch/sparc64/kernel/etrap.S | 14 ++------
arch/sparc64/kernel/head.S | 2 +
arch/sparc64/kernel/rtrap.S | 6 ++-
arch/sparc64/kernel/smp.c | 60 +++++++++++++++++++++++++++++++++-----
arch/sparc64/kernel/vmlinux.lds.S | 2 -
arch/sparc64/kernel/winfixup.S | 21 +++----------
include/asm-sparc64/percpu.h | 12 +++++--
7 files changed, 78 insertions(+), 39 deletions(-)
diff -Nru a/arch/sparc64/kernel/etrap.S b/arch/sparc64/kernel/etrap.S
--- a/arch/sparc64/kernel/etrap.S 2005-04-03 21:22:20 -07:00
+++ b/arch/sparc64/kernel/etrap.S 2005-04-03 21:22:20 -07:00
@@ -103,11 +103,8 @@
wrpr %g0, ETRAP_PSTATE2, %pstate
mov %l6, %g6
#ifdef CONFIG_SMP
- ldub [%g6 + TI_CPU], %g3
- sethi %hi(__per_cpu_offset), %g2
- or %g2, %lo(__per_cpu_offset), %g2
- sllx %g3, 3, %g3
- ldx [%g2 + %g3], %g5
+ mov TSB_REG, %g3
+ ldxa [%g3] ASI_IMMU, %g5
#endif
jmpl %l2 + 0x4, %g0
ldx [%g6 + TI_TASK], %g4
@@ -259,11 +256,8 @@
mov %l6, %g6
stx %i7, [%sp + PTREGS_OFF + PT_V9_I7]
#ifdef CONFIG_SMP
- ldub [%g6 + TI_CPU], %g3
- sethi %hi(__per_cpu_offset), %g2
- or %g2, %lo(__per_cpu_offset), %g2
- sllx %g3, 3, %g3
- ldx [%g2 + %g3], %g5
+ mov TSB_REG, %g3
+ ldxa [%g3] ASI_IMMU, %g5
#endif
ldx [%g6 + TI_TASK], %g4
done
diff -Nru a/arch/sparc64/kernel/head.S b/arch/sparc64/kernel/head.S
--- a/arch/sparc64/kernel/head.S 2005-04-03 21:22:20 -07:00
+++ b/arch/sparc64/kernel/head.S 2005-04-03 21:22:20 -07:00
@@ -657,6 +657,8 @@
mov TSB_REG, %g1
stxa %g0, [%g1] ASI_DMMU
membar #Sync
+ stxa %g0, [%g1] ASI_IMMU
+ membar #Sync
mov TLB_SFSR, %g1
sethi %uhi(KERN_HIGHBITS), %g2
or %g2, %ulo(KERN_HIGHBITS), %g2
diff -Nru a/arch/sparc64/kernel/rtrap.S b/arch/sparc64/kernel/rtrap.S
--- a/arch/sparc64/kernel/rtrap.S 2005-04-03 21:22:20 -07:00
+++ b/arch/sparc64/kernel/rtrap.S 2005-04-03 21:22:20 -07:00
@@ -222,8 +222,10 @@
ldx [%sp + PTREGS_OFF + PT_V9_G3], %g3
ldx [%sp + PTREGS_OFF + PT_V9_G4], %g4
- brz,a,pn %l3, 1f
- ldx [%sp + PTREGS_OFF + PT_V9_G5], %g5
+ ldx [%sp + PTREGS_OFF + PT_V9_G5], %g5
+ mov TSB_REG, %g6
+ brnz,a,pn %l3, 1f
+ ldxa [%g6] ASI_IMMU, %g5
1: ldx [%sp + PTREGS_OFF + PT_V9_G6], %g6
ldx [%sp + PTREGS_OFF + PT_V9_G7], %g7
wrpr %g0, RTRAP_PSTATE_AG_IRQOFF, %pstate
diff -Nru a/arch/sparc64/kernel/smp.c b/arch/sparc64/kernel/smp.c
--- a/arch/sparc64/kernel/smp.c 2005-04-03 21:22:20 -07:00
+++ b/arch/sparc64/kernel/smp.c 2005-04-03 21:22:20 -07:00
@@ -3,6 +3,7 @@
* Copyright (C) 1997 David S. Miller ([EMAIL PROTECTED])
*/
+#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/sched.h>
#include <linux/mm.h>
@@ -20,6 +21,7 @@
#include <linux/cache.h>
#include <linux/jiffies.h>
#include <linux/profile.h>
+#include <linux/bootmem.h>
#include <asm/head.h>
#include <asm/ptrace.h>
@@ -99,6 +101,16 @@
extern void inherit_locked_prom_mappings(int save_p);
+static inline void cpu_setup_percpu_base(unsigned long cpu_id)
+{
+ __asm__ __volatile__("mov %0, %%g5\n\t"
+ "stxa %0, [%1] %2\n\t"
+ "membar #Sync"
+ : /* no outputs */
+ : "r" (__per_cpu_offset(cpu_id)),
+ "r" (TSB_REG), "i" (ASI_IMMU));
+}
+
void __init smp_callin(void)
{
int cpuid = hard_smp_processor_id();
@@ -107,9 +119,7 @@
__flush_tlb_all();
- __asm__ __volatile__("mov %0, %%g5\n\t"
- : /* no outputs */
- : "r" (__per_cpu_offset[cpuid]));
+ cpu_setup_percpu_base(cpuid);
smp_setup_percpu_timer();
@@ -1120,10 +1130,6 @@
current_thread_info()->cpu = hard_smp_processor_id();
- __asm__ __volatile__("mov %0, %%g5\n\t"
- : /* no outputs */
- : "r" (__per_cpu_offset[smp_processor_id()]));
-
cpu_set(smp_processor_id(), cpu_online_map);
cpu_set(smp_processor_id(), phys_cpu_present_map);
}
@@ -1184,3 +1190,43 @@
{
}
+unsigned long __per_cpu_base;
+unsigned long __per_cpu_shift;
+
+EXPORT_SYMBOL(__per_cpu_base);
+EXPORT_SYMBOL(__per_cpu_shift);
+
+void __init setup_per_cpu_areas(void)
+{
+ unsigned long goal, size, i;
+ char *ptr;
+ /* Created by linker magic */
+ extern char __per_cpu_start[], __per_cpu_end[];
+
+ /* Copy section for each CPU (we discard the original) */
+ goal = ALIGN(__per_cpu_end - __per_cpu_start, PAGE_SIZE);
+
+#ifdef CONFIG_MODULES
+ if (goal < PERCPU_ENOUGH_ROOM)
+ goal = PERCPU_ENOUGH_ROOM;
+#endif
+ __per_cpu_shift = 0;
+ for (size = 1UL; size < goal; size <<= 1UL)
+ __per_cpu_shift++;
+
+ ptr = alloc_bootmem_pages(size * NR_CPUS);
+
+ __per_cpu_base = ptr - __per_cpu_start;
+
+ for (i = 0; i < NR_CPUS; i++, ptr += size)
+ memcpy(ptr, __per_cpu_start, __per_cpu_end - __per_cpu_start);
+
+ /* Finally, load in the boot cpu's base value.
+ * We abuse the IMMU TSB register for trap handler
+ * entry and exit loading of %g5. That is why it
+ * has to be page aligned.
+ */
+ BUG_ON((__per_cpu_shift < PAGE_SHIFT) ||
+ (__per_cpu_base & ~PAGE_MASK));
+ cpu_setup_percpu_base(hard_smp_processor_id());
+}
diff -Nru a/arch/sparc64/kernel/vmlinux.lds.S
b/arch/sparc64/kernel/vmlinux.lds.S
--- a/arch/sparc64/kernel/vmlinux.lds.S 2005-04-03 21:22:20 -07:00
+++ b/arch/sparc64/kernel/vmlinux.lds.S 2005-04-03 21:22:20 -07:00
@@ -72,7 +72,7 @@
__initramfs_start = .;
.init.ramfs : { *(.init.ramfs) }
__initramfs_end = .;
- . = ALIGN(32);
+ . = ALIGN(8192);
__per_cpu_start = .;
.data.percpu : { *(.data.percpu) }
__per_cpu_end = .;
diff -Nru a/arch/sparc64/kernel/winfixup.S b/arch/sparc64/kernel/winfixup.S
--- a/arch/sparc64/kernel/winfixup.S 2005-04-03 21:22:20 -07:00
+++ b/arch/sparc64/kernel/winfixup.S 2005-04-03 21:22:20 -07:00
@@ -94,11 +94,8 @@
mov %o7, %g6
ldx [%g6 + TI_TASK], %g4
#ifdef CONFIG_SMP
- ldub [%g6 + TI_CPU], %g1
- sethi %hi(__per_cpu_offset), %g2
- or %g2, %lo(__per_cpu_offset), %g2
- sllx %g1, 3, %g1
- ldx [%g2 + %g1], %g5
+ mov TSB_REG, %g1
+ ldxa [%g1] ASI_IMMU, %g5
#endif
/* This is the same as below, except we handle this a bit special
@@ -221,11 +218,8 @@
mov %o7, %g6 ! Get current back.
ldx [%g6 + TI_TASK], %g4 ! Finish it.
#ifdef CONFIG_SMP
- ldub [%g6 + TI_CPU], %g1
- sethi %hi(__per_cpu_offset), %g2
- or %g2, %lo(__per_cpu_offset), %g2
- sllx %g1, 3, %g1
- ldx [%g2 + %g1], %g5
+ mov TSB_REG, %g1
+ ldxa [%g1] ASI_IMMU, %g5
#endif
call mem_address_unaligned
add %sp, PTREGS_OFF, %o0
@@ -333,11 +327,8 @@
mov %o7, %g6 ! Get current back.
ldx [%g6 + TI_TASK], %g4 ! Finish it.
#ifdef CONFIG_SMP
- ldub [%g6 + TI_CPU], %g1
- sethi %hi(__per_cpu_offset), %g2
- or %g2, %lo(__per_cpu_offset), %g2
- sllx %g1, 3, %g1
- ldx [%g2 + %g1], %g5
+ mov TSB_REG, %g1
+ ldxa [%g1] ASI_IMMU, %g5
#endif
call data_access_exception
add %sp, PTREGS_OFF, %o0
diff -Nru a/include/asm-sparc64/percpu.h b/include/asm-sparc64/percpu.h
--- a/include/asm-sparc64/percpu.h 2005-04-03 21:22:20 -07:00
+++ b/include/asm-sparc64/percpu.h 2005-04-03 21:22:20 -07:00
@@ -3,10 +3,14 @@
#include <linux/compiler.h>
-#define __GENERIC_PER_CPU
#ifdef CONFIG_SMP
-extern unsigned long __per_cpu_offset[NR_CPUS];
+extern void setup_per_cpu_areas(void);
+
+extern unsigned long __per_cpu_base;
+extern unsigned long __per_cpu_shift;
+#define __per_cpu_offset(__cpu) \
+ (__per_cpu_base + ((unsigned long)(__cpu) << __per_cpu_shift))
/* Separate out the type, so (int[3], foo) works. */
#define DEFINE_PER_CPU(type, name) \
@@ -15,7 +19,7 @@
register unsigned long __local_per_cpu_offset asm("g5");
/* var is in discarded region: offset to particular copy we want */
-#define per_cpu(var, cpu) (*RELOC_HIDE(&per_cpu__##var, __per_cpu_offset[cpu]))
+#define per_cpu(var, cpu) (*RELOC_HIDE(&per_cpu__##var, __per_cpu_offset(cpu)))
#define __get_cpu_var(var) (*RELOC_HIDE(&per_cpu__##var,
__local_per_cpu_offset))
/* A macro to avoid #include hell... */
@@ -24,7 +28,7 @@
unsigned int __i; \
for (__i = 0; __i < NR_CPUS; __i++) \
if (cpu_possible(__i)) \
- memcpy((pcpudst)+__per_cpu_offset[__i], \
+ memcpy((pcpudst)+__per_cpu_offset(__i), \
(src), (size)); \
} while (0)
#else /* ! SMP */
-
To unsubscribe from this list: send the line "unsubscribe bk-commits-head" in
the body of a message to [EMAIL PROTECTED]
More majordomo info at http://vger.kernel.org/majordomo-info.html
