This patch brings the secondary processors up to the point where they
have a per-processor structure pointer in r13, at which point they just
enter a spinloop.  It differs from take one in that it has an important
shadowed variable bug fix.  If you test it, you should see two
additional pieces of output on your console:
 
 spinning up secondary processor #1: ping = 0xffffffff: pong = 0x1
 spinning up secondary processor #2: ping = 0xffffffff: pong = 0x2
 spinning up secondary processor #3: ping = 0xffffffff: pong = 0x3
 
 (XEN) CPU #1: Hello World! SP = 1fe60 TOC = 757e78 HID0 = 51118180000000
 (XEN) CPU #2: Hello World! SP = 1be60 TOC = 757e78 HID0 = 51118180000000
 (XEN) CPU #3: Hello World! SP = 79fe60 TOC = 757e78 HID0 = 51118180000000
 
I think this patch is a candidate for merging.  I will submit a patch
next for smp_processor_id, and then one for making the secondary
processors join the idle domain, but I would like to have any necessary
discussion about this portion first.
 
Note that this patch also makes arch/powerpc build with -Wshadow, since
otherwise it is way too easy to accidentaly shadow the global parea
variable which is supposed to only be referred to in r13.

Tested on systemsim-gpul with up to four simulated processors, Maple-D
with two processors, JS20 with two processors, and JS21 with four processors.

Signed-off-by: Amos Waterland <[EMAIL PROTECTED]>
 
---

 arch/powerpc/Makefile               |    2 -
 arch/powerpc/boot_of.c              |   53 +++++++++++++++++++++++++++++++++---
 arch/powerpc/mpic.c                 |    2 -
 arch/powerpc/ofd_fixup.c            |    4 +-
 arch/powerpc/powerpc64/exceptions.S |   31 +++++++++++++++++++++
 arch/powerpc/powerpc64/ppc970.c     |   22 ++++++++------
 arch/powerpc/setup.c                |   52 ++++++++++++++++++++++++++++++++++-
 include/asm-powerpc/config.h        |    2 -
 include/asm-powerpc/current.h       |    2 -
 include/asm-powerpc/processor.h     |    2 -
 10 files changed, 151 insertions(+), 21 deletions(-)

diff -r 7855f2e80748 xen/arch/powerpc/Makefile
--- a/xen/arch/powerpc/Makefile Thu Aug 17 20:38:08 2006 -0400
+++ b/xen/arch/powerpc/Makefile Thu Aug 17 22:44:27 2006 -0400
@@ -46,7 +46,7 @@ obj-y += elf32.o
 
 # These are extra warnings like for the arch/ppc directory but may not
 # allow the rest of the tree to build.
-PPC_C_WARNINGS += -Wundef -Wmissing-prototypes -Wmissing-declarations
+PPC_C_WARNINGS += -Wundef -Wmissing-prototypes -Wmissing-declarations -Wshadow
 CFLAGS += $(PPC_C_WARNINGS)
 
 LINK=0x400000
diff -r 7855f2e80748 xen/arch/powerpc/boot_of.c
--- a/xen/arch/powerpc/boot_of.c        Thu Aug 17 20:38:08 2006 -0400
+++ b/xen/arch/powerpc/boot_of.c        Thu Aug 17 22:44:27 2006 -0400
@@ -31,6 +31,9 @@
 #include <asm/io.h>
 #include "exceptions.h"
 #include "of-devtree.h"
+
+/* Secondary processors use this for handshaking with main processor.  */
+volatile unsigned int __spin_ack;
 
 static ulong of_vec;
 static ulong of_msr;
@@ -955,7 +958,7 @@ static int __init boot_of_cpus(void)
 static int __init boot_of_cpus(void)
 {
     int cpus;
-    int cpu;
+    int cpu, bootcpu;
     int result;
     u32 cpu_clock[2];
 
@@ -980,10 +983,53 @@ static int __init boot_of_cpus(void)
     cpu_khz /= 1000;
     of_printf("OF: clock-frequency = %ld KHz\n", cpu_khz);
 
-    /* FIXME: should not depend on the boot CPU bring the first child */
+    /* Look up which CPU we are running on right now.  */
+    result = of_getprop(bof_chosen, "cpu", &bootcpu, sizeof (bootcpu));
+    if (result == OF_FAILURE)
+        of_panic("Failed to look up boot cpu\n");
+
     cpu = of_getpeer(cpu);
     while (cpu > 0) {
-        of_start_cpu(cpu, (ulong)spin_start, 0);
+        unsigned int cpuid, ping, pong;
+        unsigned long now, then, timeout;
+
+        if (cpu == bootcpu) {
+            of_printf("skipping boot cpu!\n");
+            continue;
+        }
+
+        result = of_getprop(cpu, "reg", &cpuid, sizeof(cpuid));
+        if (result == OF_FAILURE)
+            of_panic("cpuid lookup failed\n");
+
+        of_printf("spinning up secondary processor #%d: ", cpuid);
+
+        __spin_ack = ~0x0;
+        ping = __spin_ack;
+        pong = __spin_ack;
+        of_printf("ping = 0x%x: ", ping);
+
+        mb();
+        result = of_start_cpu(cpu, (ulong)spin_start, cpuid);
+        if (result == OF_FAILURE)
+            of_panic("start cpu failed\n");
+
+        /* We will give the secondary processor five seconds to reply.  */
+        then = mftb();
+        timeout = then + (5 * timebase_freq);
+
+        do {
+            now = mftb();
+            if (now >= timeout) {
+                of_printf("BROKEN: ");
+                break;
+            }
+
+            mb();
+            pong = __spin_ack;
+        } while (pong == ping);
+        of_printf("pong = 0x%x\n", pong);
+
         cpu = of_getpeer(cpu);
     }
     return 1;
@@ -1031,6 +1077,7 @@ multiboot_info_t __init *boot_of_init(
     boot_of_rtas();
 
     /* end of OF */
+    of_printf("Quiescing Open Firmware ...\n");
     of_call("quiesce", 0, 0, NULL);
 
     return &mbi;
diff -r 7855f2e80748 xen/arch/powerpc/mpic.c
--- a/xen/arch/powerpc/mpic.c   Thu Aug 17 20:38:08 2006 -0400
+++ b/xen/arch/powerpc/mpic.c   Thu Aug 17 22:44:27 2006 -0400
@@ -498,7 +498,7 @@ static void mpic_enable_irq(unsigned int
 
 #ifdef CONFIG_MPIC_BROKEN_U3
        if (mpic->flags & MPIC_BROKEN_U3) {
-               unsigned int src = irq - mpic->irq_offset;
+               src = irq - mpic->irq_offset;
                if (mpic_is_ht_interrupt(mpic, src) &&
                    (irq_desc[irq].status & IRQ_LEVEL))
                        mpic_ht_end_irq(mpic, src);
diff -r 7855f2e80748 xen/arch/powerpc/ofd_fixup.c
--- a/xen/arch/powerpc/ofd_fixup.c      Thu Aug 17 20:38:08 2006 -0400
+++ b/xen/arch/powerpc/ofd_fixup.c      Fri Aug 18 04:15:26 2006 -0400
@@ -497,8 +497,8 @@ int ofd_dom0_fixup(struct domain *d, ulo
     r = ofd_prop_add(m, n, "ibm,partition-no", &did, sizeof(did));
     ASSERT( r > 0 );
 
-    const char dom0[] = "dom0";
-    r = ofd_prop_add(m, n, "ibm,partition-name", dom0, sizeof (dom0));
+    const char dom[] = "dom0";
+    r = ofd_prop_add(m, n, "ibm,partition-name", dom, sizeof (dom));
     ASSERT( r > 0 );
 
 
diff -r 7855f2e80748 xen/arch/powerpc/powerpc64/exceptions.S
--- a/xen/arch/powerpc/powerpc64/exceptions.S   Thu Aug 17 20:38:08 2006 -0400
+++ b/xen/arch/powerpc/powerpc64/exceptions.S   Thu Aug 17 22:44:27 2006 -0400
@@ -514,6 +514,37 @@ _GLOBAL(sleep)
     mtmsrd r3
     blr
 
+/* The primary processor issues a firmware call to spin us up at this
+ * address, passing our CPU number in r3.  We only need a function
+ * entry point instead of a descriptor since this is never called from
+ * C code.
+ */    
     .globl spin_start
 spin_start:
+    /* Write our processor number as an acknowledgment that we're alive.  */
+    LOADADDR(r14, __spin_ack)
+    stw r3, 0(r14)
+    sync
+    /* Find our index in the array of processor_area struct pointers.  */
+    LOADADDR(r14, global_cpu_table)
+    muli r15, r3, 8
+    add r14, r14, r15
+    /* Spin until the pointer for our processor goes valid.  */
+1:  ld r15, 0(r14)
+    cmpldi r15, 0
+    beq 1b
+    /* Dereference the pointer and load our stack pointer.  */
+    isync
+    ld r1, PAREA_stack(r15)
+    li r14, STACK_FRAME_OVERHEAD
+    sub r1, r1, r14
+    /* We must not speculatively execute beyond these loads.  */
+    LOADADDR(r14, secondary_cpu_init)
+    ld r2, 8(r14)
+    ld r11, 0(r14)
+    mtctr r11
+    isync
+    /* Jump into C code now.  */
+    bctrl
+    nop
     b .
diff -r 7855f2e80748 xen/arch/powerpc/powerpc64/ppc970.c
--- a/xen/arch/powerpc/powerpc64/ppc970.c       Thu Aug 17 20:38:08 2006 -0400
+++ b/xen/arch/powerpc/powerpc64/ppc970.c       Thu Aug 17 22:44:27 2006 -0400
@@ -31,6 +31,8 @@
 
 #undef SERIALIZE
 
+extern volatile struct processor_area * volatile global_cpu_table[];
+
 unsigned int cpu_rma_order(void)
 {
     /* XXX what about non-HV mode? */
@@ -38,18 +40,15 @@ unsigned int cpu_rma_order(void)
     return rma_log_size - PAGE_SHIFT;
 }
 
-void cpu_initialize(void)
+void cpu_initialize(int cpuid)
 {
-    ulong stack;
+    ulong r1, r2;
+    __asm__ __volatile__ ("mr %0, 1" : "=r" (r1));
+    __asm__ __volatile__ ("mr %0, 2" : "=r" (r2));
 
-    parea = xmalloc(struct processor_area);
+    /* This is SMP safe because the compiler must use r13 for it.  */
+    parea = global_cpu_table[cpuid];
     ASSERT(parea != NULL);
-
-    stack = (ulong)alloc_xenheap_pages(STACK_ORDER);
-
-    ASSERT(stack != 0);
-    parea->hyp_stack_base = (void *)(stack + STACK_SIZE);
-    printk("stack is here: %p\n", parea->hyp_stack_base);
 
     mthsprg0((ulong)parea); /* now ready for exceptions */
 
@@ -79,7 +78,10 @@ void cpu_initialize(void)
     s |= 1UL << (63-3);     /* ser-gp */
     hid0.word |= s;
 #endif
-    printk("hid0: 0x%016lx\n", hid0.word);
+
+    printk("CPU #%d: Hello World! SP = %lx TOC = %lx HID0 = %lx\n", 
+           cpuid, r1, r2, hid0.word);
+
     mthid0(hid0.word);
 
     union hid1 hid1;
diff -r 7855f2e80748 xen/arch/powerpc/setup.c
--- a/xen/arch/powerpc/setup.c  Thu Aug 17 20:38:08 2006 -0400
+++ b/xen/arch/powerpc/setup.c  Thu Aug 17 22:44:27 2006 -0400
@@ -78,6 +78,13 @@ extern void idle_loop(void);
 /* move us to a header file */
 extern void initialize_keytable(void);
 
+static void init_parea(int cpuid);
+
+volatile struct processor_area * volatile global_cpu_table[NR_CPUS];
+
+static int kick_secondary_cpus(void);
+int secondary_cpu_init(int cpuid, unsigned long);
+
 int is_kernel_text(unsigned long addr)
 {
     if (addr >= (unsigned long) &_start &&
@@ -358,13 +365,16 @@ static void __init __start_xen(multiboot
 
     percpu_init_areas();
 
-    cpu_initialize();
+    init_parea(0);
+    cpu_initialize(0);
 
 #ifdef CONFIG_GDB
     initialise_gdb();
     if (opt_earlygdb)
         debugger_trap_immediate();
 #endif
+
+    kick_secondary_cpus();
 
     start_of_day();
 
@@ -441,6 +451,46 @@ extern void arch_get_xen_caps(xen_capabi
 extern void arch_get_xen_caps(xen_capabilities_info_t info);
 void arch_get_xen_caps(xen_capabilities_info_t info)
 {
+}
+
+static void init_parea(int cpuid)
+{
+    /* Be careful not to shadow the global variable.  */
+    volatile struct processor_area *pa;
+    void *stack;
+
+    pa = xmalloc(struct processor_area);
+    if (pa == NULL)
+        panic("failed to allocate parea\n");
+
+    stack = alloc_xenheap_pages(STACK_ORDER);
+    if (stack == NULL)
+        panic("failed to allocate stack\n");
+
+    pa->hyp_stack_base = (void *)((ulong)stack + STACK_SIZE);
+
+    /* This store has the effect of invoking secondary_cpu_init.  */
+    global_cpu_table[cpuid] = pa;
+    mb();
+}
+
+static int kick_secondary_cpus()
+{
+    int i;
+
+    for (i = 1; i < NR_CPUS; i++) {
+        init_parea(i);
+    }
+
+    return 0;
+}
+
+/* This is the first C code that secondary processors invoke.  */
+int secondary_cpu_init(int cpuid, unsigned long r4)
+{
+    cpu_initialize(cpuid);
+
+    while(1);
 }
 
 /*
diff -r 7855f2e80748 xen/include/asm-powerpc/config.h
--- a/xen/include/asm-powerpc/config.h  Thu Aug 17 20:38:08 2006 -0400
+++ b/xen/include/asm-powerpc/config.h  Thu Aug 17 22:44:27 2006 -0400
@@ -51,7 +51,7 @@ extern char __bss_start[];
 #define CONFIG_GDB 1
 #define CONFIG_SMP 1
 #define CONFIG_PCI 1
-#define NR_CPUS 1
+#define NR_CPUS 4
 
 #ifndef ELFSIZE
 #define ELFSIZE 64
diff -r 7855f2e80748 xen/include/asm-powerpc/current.h
--- a/xen/include/asm-powerpc/current.h Thu Aug 17 20:38:08 2006 -0400
+++ b/xen/include/asm-powerpc/current.h Fri Aug 18 04:19:41 2006 -0400
@@ -27,7 +27,7 @@
 
 struct vcpu;
 
-register struct processor_area *parea asm("r13");
+register volatile struct processor_area *parea asm("r13");
 
 static inline struct vcpu *get_current(void)
 {
diff -r 7855f2e80748 xen/include/asm-powerpc/processor.h
--- a/xen/include/asm-powerpc/processor.h       Thu Aug 17 20:38:08 2006 -0400
+++ b/xen/include/asm-powerpc/processor.h       Thu Aug 17 22:44:27 2006 -0400
@@ -40,7 +40,7 @@ extern void show_registers(struct cpu_us
 extern void show_registers(struct cpu_user_regs *);
 extern void show_execution_state(struct cpu_user_regs *);
 extern unsigned int cpu_rma_order(void);
-extern void cpu_initialize(void);
+extern void cpu_initialize(int cpuid);
 extern void cpu_init_vcpu(struct vcpu *);
 extern void save_cpu_sprs(struct vcpu *);
 extern void load_cpu_sprs(struct vcpu *);

_______________________________________________
Xen-ppc-devel mailing list
Xen-ppc-devel@lists.xensource.com
http://lists.xensource.com/xen-ppc-devel

Reply via email to