This patch gets things to the point that each secondary processor can
call the same C routine that the primary processor did to invalidate the
segment table and initialize the HID registers. It prints the following
and boots up to nfsroot on a JS21 blade:
(XEN) CPU #1 is waiting for a stack ...
(XEN) CPU #1: Hello World!
(XEN) CPU #1: stack is here: 000000000376c000
(XEN) CPU #1: hid0: 0x0051118180000000
(XEN) CPU #2 is waiting for a stack ...
(XEN) CPU #2: Hello World!
(XEN) CPU #2: stack is here: 0000000000070000
(XEN) CPU #2: hid0: 0x0051118180000000
(XEN) CPU #3 is waiting for a stack ...
(XEN) CPU #3: Hello World!
(XEN) CPU #3: stack is here: 000000000006c000
(XEN) CPU #3: hid0: 0x0051118180000000
The logic needs a lot of cleanup, but right now I am worrying about
whether we need one idle domain per processor. We surely can't do this
from more than one processor ...
set_current(idle_domain->vcpu[0]);
idle_vcpu[0] = current;
Any comments would be appreciated.
---
arch/powerpc/boot_of.c | 70 ++++++++++++++++++++++++++++++++----
arch/powerpc/of-devtree.h | 7 +++
arch/powerpc/powerpc64/exceptions.S | 38 +++++++++++++++++++
arch/powerpc/powerpc64/ppc970.c | 7 ++-
arch/powerpc/setup.c | 35 +++++++++++++++++-
include/asm-powerpc/config.h | 2 -
include/asm-powerpc/processor.h | 2 -
7 files changed, 149 insertions(+), 12 deletions(-)
diff -r bb510c274af8 xen/arch/powerpc/boot_of.c
--- a/xen/arch/powerpc/boot_of.c Fri Aug 11 13:30:48 2006 -0400
+++ b/xen/arch/powerpc/boot_of.c Fri Aug 11 21:58:19 2006 -0400
@@ -30,6 +30,9 @@
#include <asm/io.h>
#include "exceptions.h"
#include "of-devtree.h"
+
+/* Secondary processors index into this by their processor id. */
+volatile struct per_cpu __per_cpu_init[NR_CPUS];
static ulong of_vec;
static ulong of_msr;
@@ -926,8 +929,8 @@ static void boot_of_module(ulong r3, ulo
static int __init boot_of_cpus(void)
{
- int cpus;
- int cpu;
+ int i, cpus;
+ int cpu, bootcpu;
int result;
u32 cpu_clock[2];
@@ -952,10 +955,53 @@ static int __init boot_of_cpus(void)
cpu_khz /= 1000;
of_printf("OF: clock-frequency = %ld KHz\n", cpu_khz);
- /* FIXME: should not depend on the boot CPU bring the first child */
- cpu = of_getpeer(cpu);
- while (cpu > 0) {
- of_start_cpu(cpu, (ulong)spin_start, 0);
+ /* Look up which CPU we are running on right now. */
+ result = of_getprop(bof_chosen, "cpu", &bootcpu, sizeof (bootcpu));
+ if (result == OF_FAILURE)
+ of_panic("Failed to look up boot cpu\n");
+
+ for (i = 0, cpu = of_getpeer(cpu); i < NR_CPUS && cpu > 0; i++) {
+ unsigned int cpuid, ping, pong;
+ unsigned long now, then, timeout;
+
+ if (cpu == bootcpu) {
+ of_printf("skipping boot cpu!\n");
+ continue;
+ }
+
+ result = of_getprop(cpu, "reg", &cpuid, sizeof(cpuid));
+ if (result == OF_FAILURE)
+ of_panic("cpuid lookup failed\n");
+
+ of_printf("spinning up secondary processor #%d: ", cpuid);
+
+ __per_cpu_init[cpuid].sp = 0x0;
+ __per_cpu_init[cpuid].id = ~0x0;
+ ping = __per_cpu_init[cpuid].id;
+ pong = __per_cpu_init[cpuid].id;
+ of_printf("ping = 0x%x: ", ping);
+
+ mb();
+ result = of_start_cpu(cpu, (ulong)spin_start, cpuid);
+ if (result == OF_FAILURE)
+ of_panic("start cpu failed\n");
+
+ /* We will give the secondary processor five seconds to reply. */
+ then = mftb();
+ timeout = then + (5 * timebase_freq);
+
+ do {
+ now = mftb();
+ if (now >= timeout) {
+ of_printf("BROKEN: ");
+ break;
+ }
+
+ mb();
+ pong = __per_cpu_init[cpuid].id;
+ } while (pong == ping);
+ of_printf("pong = 0x%x\n", pong);
+
cpu = of_getpeer(cpu);
}
return 1;
@@ -1003,9 +1049,21 @@ multiboot_info_t __init *boot_of_init(
boot_of_rtas();
/* end of OF */
+ of_printf("Quiescing Open Firmware ...\n");
of_call("quiesce", 0, 0, NULL);
return &mbi;
+}
+
+int secondary_cpu_init(int cpuid);
+
+int secondary_cpu_init(int cpuid)
+{
+ printk("CPU #%d: Hello World!\n", cpuid);
+
+ cpu_initialize(cpuid);
+
+ return 0;
}
/*
diff -r bb510c274af8 xen/arch/powerpc/of-devtree.h
--- a/xen/arch/powerpc/of-devtree.h Fri Aug 11 13:30:48 2006 -0400
+++ b/xen/arch/powerpc/of-devtree.h Fri Aug 11 19:55:51 2006 -0400
@@ -23,6 +23,13 @@
#include <xen/types.h>
#include <public/xen.h>
+
+/* Data structure used by secondary processors to discover their stack. */
+struct per_cpu {
+ int id;
+ void *sp;
+ void *toc;
+};
enum {
OF_FAILURE = -1,
diff -r bb510c274af8 xen/arch/powerpc/powerpc64/exceptions.S
--- a/xen/arch/powerpc/powerpc64/exceptions.S Fri Aug 11 13:30:48 2006 -0400
+++ b/xen/arch/powerpc/powerpc64/exceptions.S Fri Aug 11 21:11:00 2006 -0400
@@ -514,6 +514,44 @@ _GLOBAL(sleep)
mtmsrd r3
blr
+/* Get these from asm-offsets, talk to Jimi about int->long alignment hole. */
+#define ID_OFFSET 0
+#define SP_OFFSET 8
+#define TC_OFFSET 16
+#define PER_CPU_SIZE 24
+
+/* Firmware is told to spin up secondary processors at this address. We
+ * only need a function entry point instead of a descriptor since this is
+ * never called from C code.
+ */
.globl spin_start
spin_start:
+ /* Our physical cpu number is passed in r3, so index into array. */
+ muli r5, r3, PER_CPU_SIZE
+ LOADADDR(r27, __per_cpu_init)
+ add r27, r27, r5
+ stw r3, ID_OFFSET(r27)
+ sync
+ /* At some point these should go in different cache lines. */
+1: ld r6, SP_OFFSET(r27)
+ cmpldi r6, 0
+ beq 1b
+ /* The primary cpu has an allocator now and wants our attention. */
+ li r6, 0x0
+ std r6, SP_OFFSET(r27)
+ sync
+ /* Wait for the primary cpu to give us our stack address. */
+2: ld r6, SP_OFFSET(r27)
+ cmpldi r6, 0
+ beq 2b
+ /* The primary cpu has allocated a stack for us, so rock and roll. */
+ mr r1, r6
+ ld r2, TC_OFFSET(r27)
+ LOADADDR(r8, .secondary_cpu_init)
+ mtctr r8
+ bctrl
+ /* For now, the primary processor is waiting for us to return from C. */
+ li r3, 0
+ std r3, SP_OFFSET(r27)
+ sync
b .
diff -r bb510c274af8 xen/arch/powerpc/powerpc64/ppc970.c
--- a/xen/arch/powerpc/powerpc64/ppc970.c Fri Aug 11 13:30:48 2006 -0400
+++ b/xen/arch/powerpc/powerpc64/ppc970.c Fri Aug 11 21:56:15 2006 -0400
@@ -37,10 +37,11 @@ unsigned int cpu_rma_order(void)
return 14; /* 1<<14<<PAGE_SIZE = 64M */
}
-void cpu_initialize(void)
+void cpu_initialize(int cpuid)
{
ulong stack;
+ /* This is SMP safe because the compiler must use r13 for it. */
parea = xmalloc(struct processor_area);
ASSERT(parea != NULL);
@@ -48,7 +49,7 @@ void cpu_initialize(void)
ASSERT(stack != 0);
parea->hyp_stack_base = (void *)(stack + STACK_SIZE);
- printk("stack is here: %p\n", parea->hyp_stack_base);
+ printk("CPU #%d: stack is here: %p\n", cpuid, parea->hyp_stack_base);
mthsprg0((ulong)parea); /* now ready for exceptions */
@@ -78,7 +79,7 @@ void cpu_initialize(void)
s |= 1UL << (63-3); /* ser-gp */
hid0.word |= s;
#endif
- printk("hid0: 0x%016lx\n", hid0.word);
+ printk("CPU #%d: hid0: 0x%016lx\n", cpuid, hid0.word);
mthid0(hid0.word);
union hid1 hid1;
diff -r bb510c274af8 xen/arch/powerpc/setup.c
--- a/xen/arch/powerpc/setup.c Fri Aug 11 13:30:48 2006 -0400
+++ b/xen/arch/powerpc/setup.c Fri Aug 11 21:55:40 2006 -0400
@@ -75,6 +75,8 @@ extern void idle_loop(void);
/* move us to a header file */
extern void initialize_keytable(void);
+extern volatile struct per_cpu __per_cpu_init[];
+
int is_kernel_text(unsigned long addr)
{
if (addr >= (unsigned long) &_start &&
@@ -314,7 +316,7 @@ static void __init __start_xen(multiboot
printk("Xen heap: %luMB (%lukB)\n", heap_size >> 20, heap_size >> 10);
- cpu_initialize();
+ cpu_initialize(0);
#ifdef CONFIG_GDB
initialise_gdb();
@@ -323,6 +325,37 @@ static void __init __start_xen(multiboot
#endif
start_of_day();
+
+ {
+ int i;
+
+ for (i = 1; i <= NR_CPUS; i++) {
+ void *stack, *toc;
+ void *syn = (void *)~0x0;
+
+ if (__per_cpu_init[i].id > 0) {
+ printk("CPU #%d is waiting for a stack ...\n", i);
+
+ __per_cpu_init[i].sp = syn;
+ do {
+ mb();
+ } while (__per_cpu_init[i].sp == syn);
+
+ stack = xmalloc_bytes(STACK_SIZE);
+ if (stack == NULL)
+ panic("failed to allocate stack\n");
+
+ __per_cpu_init[i].sp = stack;
+ asm("mr %0, 2" : "=r" (toc));
+ __per_cpu_init[i].toc = toc;
+ mb();
+
+ do {
+ mb();
+ } while (__per_cpu_init[i].sp != 0x0);
+ }
+ }
+ }
/* Create initial domain 0. */
dom0 = domain_create(0);
diff -r bb510c274af8 xen/include/asm-powerpc/config.h
--- a/xen/include/asm-powerpc/config.h Fri Aug 11 13:30:48 2006 -0400
+++ b/xen/include/asm-powerpc/config.h Fri Aug 11 16:56:58 2006 -0400
@@ -51,7 +51,7 @@ extern char __bss_start[];
#define CONFIG_GDB 1
#define CONFIG_SMP 1
#define CONFIG_PCI 1
-#define NR_CPUS 1
+#define NR_CPUS 6
#ifndef ELFSIZE
#define ELFSIZE 64
diff -r bb510c274af8 xen/include/asm-powerpc/processor.h
--- a/xen/include/asm-powerpc/processor.h Fri Aug 11 13:30:48 2006 -0400
+++ b/xen/include/asm-powerpc/processor.h Fri Aug 11 21:54:51 2006 -0400
@@ -40,7 +40,7 @@ extern void show_registers(struct cpu_us
extern void show_registers(struct cpu_user_regs *);
extern void show_execution_state(struct cpu_user_regs *);
extern unsigned int cpu_rma_order(void);
-extern void cpu_initialize(void);
+extern void cpu_initialize(int cpuid);
extern void cpu_init_vcpu(struct vcpu *);
extern void save_cpu_sprs(struct vcpu *);
extern void load_cpu_sprs(struct vcpu *);
_______________________________________________
Xen-ppc-devel mailing list
Xen-ppc-devel@lists.xensource.com
http://lists.xensource.com/xen-ppc-devel
