When using the register dump feature of Xen, one will sometimes see a
message about an IPI finish stall. This is because of an int to long
comparison bug, so fix it by doing proper nanosecond based time accounting.
As a side note, our IPI remote function call latency of completion on
a JS21 blade is: min = 34 ticks, max = 119 ticks, mean = 2691ns.
Signed-off-by: Amos Waterland <[EMAIL PROTECTED]>
---
smp.c | 39 +++++++++++++++++++++++++--------------
1 file changed, 25 insertions(+), 14 deletions(-)
diff -r e01e08ca629b xen/arch/powerpc/smp.c
--- a/xen/arch/powerpc/smp.c Tue Nov 28 17:01:00 2006 -0500
+++ b/xen/arch/powerpc/smp.c Tue Nov 28 17:40:50 2006 -0500
@@ -90,7 +90,8 @@ int on_selected_cpus(
int retry,
int wait)
{
- int t, retval = 0, nr_cpus = cpus_weight(selected);
+ int retval = 0, nr_cpus = cpus_weight(selected);
+ unsigned long start, stall = tb_to_ns(timebase_freq);
spin_lock(&call_lock);
@@ -104,19 +105,21 @@ int on_selected_cpus(
send_IPI_mask(selected, CALL_FUNCTION_VECTOR);
/* We always wait for an initiation ACK from remote CPU. */
- for (t = 0; atomic_read(&call_data.started) != nr_cpus; t++) {
- if (t && t % timebase_freq == 0) {
+ for (start = NOW(); atomic_read(&call_data.started) != nr_cpus; ) {
+ if (NOW() > start + stall) {
printk("IPI start stall: %d ACKS to %d SYNS\n",
atomic_read(&call_data.started), nr_cpus);
+ start = NOW();
}
}
/* If told to, we wait for a completion ACK from remote CPU. */
if (wait) {
- for (t = 0; atomic_read(&call_data.finished) != nr_cpus; t++) {
- if (t > timebase_freq && t % timebase_freq == 0) {
+ for (start = NOW(); atomic_read(&call_data.finished) != nr_cpus; ) {
+ if (NOW() > start + stall) {
printk("IPI finish stall: %d ACKS to %d SYNS\n",
atomic_read(&call_data.finished), nr_cpus);
+ start = NOW();
}
}
}
@@ -168,6 +171,11 @@ void smp_message_recv(int msg, struct cp
#ifdef DEBUG_IPI
static void debug_ipi_ack(void *info)
{
+ if (info) {
+ unsigned long start, stall = tb_to_ns(5 * timebase_freq);
+ for (start = NOW(); NOW() < start + stall; );
+ printk("IPI recv on cpu #%d: %s\n", smp_processor_id(), (char *)info);
+ }
return;
}
@@ -175,12 +183,12 @@ void ipi_torture_test(void)
{
int cpu;
unsigned long before, after, delta;
- unsigned long min = ~0, max = 0, mean = 0, sum = 0, tick = 0;
+ unsigned long min = ~0, max = 0, mean = 0, sum = 0, trials = 0;
cpumask_t mask;
cpus_clear(mask);
- while (tick < 1000000) {
+ while (trials < 1000000) {
for_each_online_cpu(cpu) {
cpu_set(cpu, mask);
before = mftb();
@@ -192,12 +200,15 @@ void ipi_torture_test(void)
if (delta > max) max = delta;
if (delta < min) min = delta;
sum += delta;
- tick++;
- }
- }
-
- mean = sum / tick;
-
- printk("IPI tb ticks: min = %ld max = %ld mean = %ld\n", min, max, mean);
+ trials++;
+ }
+ }
+
+ mean = tb_to_ns(sum / trials);
+
+ printk("IPI latency: min = %ld ticks, max = %ld ticks, mean = %ldns\n",
+ min, max, mean);
+
+ smp_call_function(debug_ipi_ack, "Hi", 0, 1);
}
#endif
_______________________________________________
Xen-ppc-devel mailing list
Xen-ppc-devel@lists.xensource.com
http://lists.xensource.com/xen-ppc-devel
