Re: [XenPPC] [PATCH/RFC] Run C code from secondary processors
On Fri, Aug 11, 2006 at 08:40:19PM -0400, Amos Waterland wrote:
> I have screwed up the link register or something however, because the
> logic right after the bctrl never seems to run.
It turned out to be the fact that I was using r4 for my per-cpu struct
pointer, and the called C code has no obligation to preserve it :)
The code runs fine now. Corrected patch below.
---
arch/powerpc/boot_of.c | 72 +---
arch/powerpc/of-devtree.h |7 +++
arch/powerpc/powerpc64/exceptions.S | 39 +++
arch/powerpc/setup.c| 40
include/asm-powerpc/config.h|2 -
5 files changed, 153 insertions(+), 7 deletions(-)
diff -r bb510c274af8 xen/arch/powerpc/boot_of.c
--- a/xen/arch/powerpc/boot_of.cFri Aug 11 13:30:48 2006 -0400
+++ b/xen/arch/powerpc/boot_of.cFri Aug 11 20:32:17 2006 -0400
@@ -30,6 +30,9 @@
#include
#include "exceptions.h"
#include "of-devtree.h"
+
+/* Secondary processors index into this by their processor id. */
+volatile struct per_cpu __per_cpu_init[NR_CPUS];
static ulong of_vec;
static ulong of_msr;
@@ -926,8 +929,8 @@ static void boot_of_module(ulong r3, ulo
static int __init boot_of_cpus(void)
{
-int cpus;
-int cpu;
+int i, cpus;
+int cpu, bootcpu;
int result;
u32 cpu_clock[2];
@@ -952,10 +955,53 @@ static int __init boot_of_cpus(void)
cpu_khz /= 1000;
of_printf("OF: clock-frequency = %ld KHz\n", cpu_khz);
-/* FIXME: should not depend on the boot CPU bring the first child */
-cpu = of_getpeer(cpu);
-while (cpu > 0) {
-of_start_cpu(cpu, (ulong)spin_start, 0);
+/* Look up which CPU we are running on right now. */
+result = of_getprop(bof_chosen, "cpu", &bootcpu, sizeof (bootcpu));
+if (result == OF_FAILURE)
+of_panic("Failed to look up boot cpu\n");
+
+for (i = 0, cpu = of_getpeer(cpu); i < NR_CPUS && cpu > 0; i++) {
+unsigned int cpuid, ping, pong;
+unsigned long now, then, timeout;
+
+if (cpu == bootcpu) {
+of_printf("skipping boot cpu!\n");
+continue;
+}
+
+result = of_getprop(cpu, "reg", &cpuid, sizeof(cpuid));
+if (result == OF_FAILURE)
+of_panic("cpuid lookup failed\n");
+
+of_printf("spinning up secondary processor #%d: ", cpuid);
+
+__per_cpu_init[cpuid].sp = 0x0;
+__per_cpu_init[cpuid].id = ~0x0;
+ping = __per_cpu_init[cpuid].id;
+pong = __per_cpu_init[cpuid].id;
+of_printf("ping = 0x%x: ", ping);
+
+mb();
+result = of_start_cpu(cpu, (ulong)spin_start, cpuid);
+if (result == OF_FAILURE)
+of_panic("start cpu failed\n");
+
+/* We will give the secondary processor five seconds to reply. */
+then = mftb();
+timeout = then + (5 * timebase_freq);
+
+do {
+now = mftb();
+if (now >= timeout) {
+of_printf("BROKEN: ");
+break;
+}
+
+mb();
+pong = __per_cpu_init[cpuid].id;
+} while (pong == ping);
+of_printf("pong = 0x%x\n", pong);
+
cpu = of_getpeer(cpu);
}
return 1;
@@ -1003,9 +1049,23 @@ multiboot_info_t __init *boot_of_init(
boot_of_rtas();
/* end of OF */
+of_printf("Quiescing Open Firmware ...\n");
of_call("quiesce", 0, 0, NULL);
return &mbi;
+}
+
+int hello(int depth);
+
+int hello(int depth)
+{
+of_printf(" -> Hello from C #%d\n", depth);
+
+if (depth > 0)
+return hello(depth - 1);
+
+of_printf(" <- Goodbye from C\n");
+return 0;
}
/*
diff -r bb510c274af8 xen/arch/powerpc/of-devtree.h
--- a/xen/arch/powerpc/of-devtree.h Fri Aug 11 13:30:48 2006 -0400
+++ b/xen/arch/powerpc/of-devtree.h Fri Aug 11 19:55:51 2006 -0400
@@ -23,6 +23,13 @@
#include
#include
+
+/* Data structure used by secondary processors to discover their stack. */
+struct per_cpu {
+int id;
+void *sp;
+void *toc;
+};
enum {
OF_FAILURE = -1,
diff -r bb510c274af8 xen/arch/powerpc/powerpc64/exceptions.S
--- a/xen/arch/powerpc/powerpc64/exceptions.S Fri Aug 11 13:30:48 2006 -0400
+++ b/xen/arch/powerpc/powerpc64/exceptions.S Fri Aug 11 20:50:03 2006 -0400
@@ -514,6 +514,45 @@ _GLOBAL(sleep)
mtmsrd r3
blr
+/* Get these from asm-offsets, talk to Jimi about int->long alignment hole. */
+#define ID_OFFSET0
+#define SP_OFFSET8
+#define TC_OFFSET16
+#define PER_CPU_SIZE 24
+
+/* Firmware is told to spin up secondary processors at this address. We
+ * only need a function entry point instead of a descriptor since this is
+ * never called from C code.
+ */
.globl spin_start
spin_start:
+/* Our physical cpu number is passed in r3, so index into array. */
+muli r5, r3, PER_CPU_SIZE
+LOADADDR(r27, __per_cpu_init)
+add r27, r27, r5
+s
[XenPPC] [PATCH/RFC] Run C code from secondary processors
This is a very rough first attempt to set up the stack and TOC for each
secondary processor such that they can invoke C code. It supercedes the
previous handshaking patches because it moves all the logic to a set of
per-cpu structures. This paves the way for bringing up processors in
parallel, but the most important reason is to give us a place to notify
the secondary processors what their stack is once the primary processor
has gotten far enough along to allocate from the heap.
This patch "works" in that the first secondary processor is able to
print this:
(XEN) xen_mpic_init: success
(XEN) Using scheduler: SMP Credit Scheduler (credit)
-> Hello from C #8
-> Hello from C #7
-> Hello from C #6
-> Hello from C #5
-> Hello from C #4
-> Hello from C #3
-> Hello from C #2
-> Hello from C #1
-> Hello from C #0
<- Goodbye from C
I have screwed up the link register or something however, because the
logic right after the bctrl never seems to run. Any comments on this
specific issue, or the approach in general?
---
arch/powerpc/boot_of.c | 72 +---
arch/powerpc/of-devtree.h |7 +++
arch/powerpc/powerpc64/exceptions.S | 39 +++
arch/powerpc/setup.c| 40
include/asm-powerpc/config.h|2 -
5 files changed, 153 insertions(+), 7 deletions(-)
diff -r bb510c274af8 xen/arch/powerpc/boot_of.c
--- a/xen/arch/powerpc/boot_of.cFri Aug 11 13:30:48 2006 -0400
+++ b/xen/arch/powerpc/boot_of.cFri Aug 11 20:32:17 2006 -0400
@@ -30,6 +30,9 @@
#include
#include "exceptions.h"
#include "of-devtree.h"
+
+/* Secondary processors index into this by their processor id. */
+volatile struct per_cpu __per_cpu_init[NR_CPUS];
static ulong of_vec;
static ulong of_msr;
@@ -926,8 +929,8 @@ static void boot_of_module(ulong r3, ulo
static int __init boot_of_cpus(void)
{
-int cpus;
-int cpu;
+int i, cpus;
+int cpu, bootcpu;
int result;
u32 cpu_clock[2];
@@ -952,10 +955,53 @@ static int __init boot_of_cpus(void)
cpu_khz /= 1000;
of_printf("OF: clock-frequency = %ld KHz\n", cpu_khz);
-/* FIXME: should not depend on the boot CPU bring the first child */
-cpu = of_getpeer(cpu);
-while (cpu > 0) {
-of_start_cpu(cpu, (ulong)spin_start, 0);
+/* Look up which CPU we are running on right now. */
+result = of_getprop(bof_chosen, "cpu", &bootcpu, sizeof (bootcpu));
+if (result == OF_FAILURE)
+of_panic("Failed to look up boot cpu\n");
+
+for (i = 0, cpu = of_getpeer(cpu); i < NR_CPUS && cpu > 0; i++) {
+unsigned int cpuid, ping, pong;
+unsigned long now, then, timeout;
+
+if (cpu == bootcpu) {
+of_printf("skipping boot cpu!\n");
+continue;
+}
+
+result = of_getprop(cpu, "reg", &cpuid, sizeof(cpuid));
+if (result == OF_FAILURE)
+of_panic("cpuid lookup failed\n");
+
+of_printf("spinning up secondary processor #%d: ", cpuid);
+
+__per_cpu_init[cpuid].sp = 0x0;
+__per_cpu_init[cpuid].id = ~0x0;
+ping = __per_cpu_init[cpuid].id;
+pong = __per_cpu_init[cpuid].id;
+of_printf("ping = 0x%x: ", ping);
+
+mb();
+result = of_start_cpu(cpu, (ulong)spin_start, cpuid);
+if (result == OF_FAILURE)
+of_panic("start cpu failed\n");
+
+/* We will give the secondary processor five seconds to reply. */
+then = mftb();
+timeout = then + (5 * timebase_freq);
+
+do {
+now = mftb();
+if (now >= timeout) {
+of_printf("BROKEN: ");
+break;
+}
+
+mb();
+pong = __per_cpu_init[cpuid].id;
+} while (pong == ping);
+of_printf("pong = 0x%x\n", pong);
+
cpu = of_getpeer(cpu);
}
return 1;
@@ -1003,9 +1049,23 @@ multiboot_info_t __init *boot_of_init(
boot_of_rtas();
/* end of OF */
+of_printf("Quiescing Open Firmware ...\n");
of_call("quiesce", 0, 0, NULL);
return &mbi;
+}
+
+int hello(int depth);
+
+int hello(int depth)
+{
+of_printf(" -> Hello from C #%d\n", depth);
+
+if (depth > 0)
+return hello(depth - 1);
+
+of_printf(" <- Goodbye from C\n");
+return 0;
}
/*
diff -r bb510c274af8 xen/arch/powerpc/of-devtree.h
--- a/xen/arch/powerpc/of-devtree.h Fri Aug 11 13:30:48 2006 -0400
+++ b/xen/arch/powerpc/of-devtree.h Fri Aug 11 19:55:51 2006 -0400
@@ -23,6 +23,13 @@
#include
#include
+
+/* Data structure used by secondary processors to discover their stack. */
+struct per_cpu {
+int id;
+void *sp;
+void *toc;
+};
enum {
OF_FAILURE = -1,
diff -r bb510c274af8 xen/arch/powerpc/powerpc64/exceptions.S
--- a/xen/arch/powerpc/powerpc64/exceptions.S Fri Aug 11 13:30:48 2006 -0400
+++ b/xen/arch/powerpc/powerpc64/exceptions.S
