On Fri, Aug 11, 2006 at 08:40:19PM -0400, Amos Waterland wrote:
> I have screwed up the link register or something however, because the
> logic right after the bctrl never seems to run.  

It turned out to be the fact that I was using r4 for my per-cpu struct
pointer, and the called C code has no obligation to preserve it :)

The code runs fine now.  Corrected patch below.

---

 arch/powerpc/boot_of.c              |   72 +++++++++++++++++++++++++++++++++---
 arch/powerpc/of-devtree.h           |    7 +++
 arch/powerpc/powerpc64/exceptions.S |   39 +++++++++++++++++++
 arch/powerpc/setup.c                |   40 ++++++++++++++++++++
 include/asm-powerpc/config.h        |    2 -
 5 files changed, 153 insertions(+), 7 deletions(-)

diff -r bb510c274af8 xen/arch/powerpc/boot_of.c
--- a/xen/arch/powerpc/boot_of.c        Fri Aug 11 13:30:48 2006 -0400
+++ b/xen/arch/powerpc/boot_of.c        Fri Aug 11 20:32:17 2006 -0400
@@ -30,6 +30,9 @@
 #include <asm/io.h>
 #include "exceptions.h"
 #include "of-devtree.h"
+
+/* Secondary processors index into this by their processor id.  */
+volatile struct per_cpu __per_cpu_init[NR_CPUS];
 
 static ulong of_vec;
 static ulong of_msr;
@@ -926,8 +929,8 @@ static void boot_of_module(ulong r3, ulo
 
 static int __init boot_of_cpus(void)
 {
-    int cpus;
-    int cpu;
+    int i, cpus;
+    int cpu, bootcpu;
     int result;
     u32 cpu_clock[2];
 
@@ -952,10 +955,53 @@ static int __init boot_of_cpus(void)
     cpu_khz /= 1000;
     of_printf("OF: clock-frequency = %ld KHz\n", cpu_khz);
 
-    /* FIXME: should not depend on the boot CPU bring the first child */
-    cpu = of_getpeer(cpu);
-    while (cpu > 0) {
-        of_start_cpu(cpu, (ulong)spin_start, 0);
+    /* Look up which CPU we are running on right now.  */
+    result = of_getprop(bof_chosen, "cpu", &bootcpu, sizeof (bootcpu));
+    if (result == OF_FAILURE)
+        of_panic("Failed to look up boot cpu\n");
+
+    for (i = 0, cpu = of_getpeer(cpu); i < NR_CPUS && cpu > 0; i++) {
+        unsigned int cpuid, ping, pong;
+        unsigned long now, then, timeout;
+
+        if (cpu == bootcpu) {
+            of_printf("skipping boot cpu!\n");
+            continue;
+        }
+
+        result = of_getprop(cpu, "reg", &cpuid, sizeof(cpuid));
+        if (result == OF_FAILURE)
+            of_panic("cpuid lookup failed\n");
+
+        of_printf("spinning up secondary processor #%d: ", cpuid);
+
+        __per_cpu_init[cpuid].sp = 0x0;
+        __per_cpu_init[cpuid].id = ~0x0;
+        ping = __per_cpu_init[cpuid].id;
+        pong = __per_cpu_init[cpuid].id;
+        of_printf("ping = 0x%x: ", ping);
+
+        mb();
+        result = of_start_cpu(cpu, (ulong)spin_start, cpuid);
+        if (result == OF_FAILURE)
+            of_panic("start cpu failed\n");
+
+        /* We will give the secondary processor five seconds to reply.  */
+        then = mftb();
+        timeout = then + (5 * timebase_freq);
+
+        do {
+            now = mftb();
+            if (now >= timeout) {
+                of_printf("BROKEN: ");
+                break;
+            }
+
+            mb();
+            pong = __per_cpu_init[cpuid].id;
+        } while (pong == ping);
+        of_printf("pong = 0x%x\n", pong);
+
         cpu = of_getpeer(cpu);
     }
     return 1;
@@ -1003,9 +1049,23 @@ multiboot_info_t __init *boot_of_init(
     boot_of_rtas();
 
     /* end of OF */
+    of_printf("Quiescing Open Firmware ...\n");
     of_call("quiesce", 0, 0, NULL);
 
     return &mbi;
+}
+
+int hello(int depth);
+
+int hello(int depth)
+{
+    of_printf(" -> Hello from C #%d\n", depth);
+
+    if (depth > 0)
+        return hello(depth - 1);
+
+    of_printf(" <- Goodbye from C\n");
+    return 0;
 }
 
 /*
diff -r bb510c274af8 xen/arch/powerpc/of-devtree.h
--- a/xen/arch/powerpc/of-devtree.h     Fri Aug 11 13:30:48 2006 -0400
+++ b/xen/arch/powerpc/of-devtree.h     Fri Aug 11 19:55:51 2006 -0400
@@ -23,6 +23,13 @@
 
 #include <xen/types.h>
 #include <public/xen.h>
+
+/* Data structure used by secondary processors to discover their stack.  */
+struct per_cpu {
+    int id;
+    void *sp;
+    void *toc;
+};
 
 enum {
     OF_FAILURE = -1,
diff -r bb510c274af8 xen/arch/powerpc/powerpc64/exceptions.S
--- a/xen/arch/powerpc/powerpc64/exceptions.S   Fri Aug 11 13:30:48 2006 -0400
+++ b/xen/arch/powerpc/powerpc64/exceptions.S   Fri Aug 11 20:50:03 2006 -0400
@@ -514,6 +514,45 @@ _GLOBAL(sleep)
     mtmsrd r3
     blr
 
+/* Get these from asm-offsets, talk to Jimi about int->long alignment hole.  */
+#define ID_OFFSET    0
+#define SP_OFFSET    8
+#define TC_OFFSET    16
+#define PER_CPU_SIZE 24
+       
+/* Firmware is told to spin up secondary processors at this address.  We
+ * only need a function entry point instead of a descriptor since this is
+ * never called from C code.
+ */
     .globl spin_start
 spin_start:
+    /* Our physical cpu number is passed in r3, so index into array.  */
+    muli r5, r3, PER_CPU_SIZE
+    LOADADDR(r27, __per_cpu_init)
+    add r27, r27, r5
+    stw r3, ID_OFFSET(r27)
+    sync
+    /* At some point these should go in different cache lines.  */
+1:  ld r6, SP_OFFSET(r27)
+    cmpldi r6, 0
+    beq 1b
+    /* The primary cpu has an allocator now and wants our attention.  */
+    li r6, 0x0
+    std r6, SP_OFFSET(r27)
+    sync
+    /* Wait for the primary cpu to give us our stack address.  */      
+2:  ld r6, SP_OFFSET(r27)
+    cmpldi r6, 0
+    beq 2b
+    /* The primary cpu has allocated a stack for us, so rock and roll.  */
+    mr r1, r6
+    ld r2, TC_OFFSET(r27)
+    li r3, 8
+    LOADADDR(r8, .hello)
+    mtctr r8
+    bctrl
+    std r2, TC_OFFSET(r27)
+    li r3, 0
+    std r3, SP_OFFSET(r27)
+    sync
     b .
diff -r bb510c274af8 xen/arch/powerpc/setup.c
--- a/xen/arch/powerpc/setup.c  Fri Aug 11 13:30:48 2006 -0400
+++ b/xen/arch/powerpc/setup.c  Fri Aug 11 20:06:46 2006 -0400
@@ -75,6 +75,8 @@ extern void idle_loop(void);
 /* move us to a header file */
 extern void initialize_keytable(void);
 
+extern volatile struct per_cpu __per_cpu_init[];
+
 int is_kernel_text(unsigned long addr)
 {
     if (addr >= (unsigned long) &_start &&
@@ -323,6 +325,44 @@ static void __init __start_xen(multiboot
 #endif
 
     start_of_day();
+
+    {
+        int i;
+
+        for (i = 1; i <= NR_CPUS; i++) {
+            void *stack, *toc;
+            void *syn = (void *)~0x0;
+
+            if (__per_cpu_init[i].id > 0) {
+                printk("CPU #%d is waiting for a stack ...\n", i);
+
+                __per_cpu_init[i].sp = syn;
+                do {
+                    mb();
+                } while (__per_cpu_init[i].sp == syn);
+
+                stack = xmalloc_bytes(STACK_SIZE);
+                if (stack == NULL)
+                    panic("failed to allocate stack\n");
+
+                __per_cpu_init[i].sp = stack;
+                asm("mr %0, 2" : "=r" (toc));
+                __per_cpu_init[i].toc = toc;
+                mb();
+
+                do {
+                    mb();
+                } while (__per_cpu_init[i].sp != 0x0);
+
+                printk(" id: 0x%i\n"
+                       " sp: 0x%p\n"
+                       " tc: 0x%p (0x%p)\n",
+                       __per_cpu_init[i].id,
+                       __per_cpu_init[i].sp,
+                       __per_cpu_init[i].toc, toc);
+            }
+        }
+    }
 
     /* Create initial domain 0. */
     dom0 = domain_create(0);
diff -r bb510c274af8 xen/include/asm-powerpc/config.h
--- a/xen/include/asm-powerpc/config.h  Fri Aug 11 13:30:48 2006 -0400
+++ b/xen/include/asm-powerpc/config.h  Fri Aug 11 16:56:58 2006 -0400
@@ -51,7 +51,7 @@ extern char __bss_start[];
 #define CONFIG_GDB 1
 #define CONFIG_SMP 1
 #define CONFIG_PCI 1
-#define NR_CPUS 1
+#define NR_CPUS 6
 
 #ifndef ELFSIZE
 #define ELFSIZE 64

_______________________________________________
Xen-ppc-devel mailing list
Xen-ppc-devel@lists.xensource.com
http://lists.xensource.com/xen-ppc-devel

Reply via email to