[PATCH 2/2] powerpc: Add ppc64 hard lockup detector support
The hard lockup detector uses a PMU event as a periodic NMI to
detect if we are stuck (where stuck means no timer interrupts have
occurred).
Ben's rework of the ppc64 soft disable code has made ppc64 PMU
exceptions a partial NMI. They can get disabled if an external
interrupt comes in, but otherwise PMU interrupts will fire in
interrupt disabled regions.
We disable the hard lockup detector by default for a few reasons:
- It breaks userspace event based branches on POWER8.
- It is likely to produce false positives on KVM guests.
- Since PMCs can only count to 2^31, counting cycles means we might
take multiple PMU exceptions per second per hardware thread even
if our hard lockup timeout is 10 seconds.
It can be enabled via a boot option, or via procfs.
Signed-off-by: Anton Blanchard an...@samba.org
---
arch/powerpc/Kconfig | 1 +
arch/powerpc/include/asm/nmi.h | 4
arch/powerpc/kernel/setup_64.c | 20
3 files changed, 25 insertions(+)
create mode 100644 arch/powerpc/include/asm/nmi.h
diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig
index c102668..716c9e6 100644
--- a/arch/powerpc/Kconfig
+++ b/arch/powerpc/Kconfig
@@ -152,6 +152,7 @@ config PPC
select DCACHE_WORD_ACCESS if PPC64 CPU_LITTLE_ENDIAN
select NO_BOOTMEM
select HAVE_GENERIC_RCU_GUP
+ select HAVE_PERF_EVENTS_NMI if PPC64
config GENERIC_CSUM
def_bool CPU_LITTLE_ENDIAN
diff --git a/arch/powerpc/include/asm/nmi.h b/arch/powerpc/include/asm/nmi.h
new file mode 100644
index 000..ff1ccb3
--- /dev/null
+++ b/arch/powerpc/include/asm/nmi.h
@@ -0,0 +1,4 @@
+#ifndef _ASM_NMI_H
+#define _ASM_NMI_H
+
+#endif /* _ASM_NMI_H */
diff --git a/arch/powerpc/kernel/setup_64.c b/arch/powerpc/kernel/setup_64.c
index 300be7d..ae39ec6 100644
--- a/arch/powerpc/kernel/setup_64.c
+++ b/arch/powerpc/kernel/setup_64.c
@@ -37,6 +37,7 @@
#include linux/memblock.h
#include linux/hugetlb.h
#include linux/memory.h
+#include linux/nmi.h
#include asm/io.h
#include asm/kdump.h
@@ -779,3 +780,22 @@ unsigned long memory_block_size_bytes(void)
struct ppc_pci_io ppc_pci_io;
EXPORT_SYMBOL(ppc_pci_io);
#endif
+
+#ifdef CONFIG_HARDLOCKUP_DETECTOR
+u64 hw_nmi_get_sample_period(int watchdog_thresh)
+{
+ return ppc_proc_freq * watchdog_thresh;
+}
+
+/*
+ * The hardlockup detector breaks PMU event based branches and is likely
+ * to get false positives in KVM guests, so disable it by default.
+ */
+static int __init disable_hardlockup_detector(void)
+{
+ watchdog_enable_hardlockup_detector(false);
+
+ return 0;
+}
+early_initcall(disable_hardlockup_detector);
+#endif
--
2.1.0
___
Linuxppc-dev mailing list
Linuxppc-dev@lists.ozlabs.org
https://lists.ozlabs.org/listinfo/linuxppc-dev
[PATCH 2/2] powerpc: Add ppc64 hard lockup detector support
The hard lockup detector uses a PMU event as a periodic NMI to
detect if we are stuck (where stuck means no timer interrupts have
occurred).
Ben's rework of the ppc64 soft disable code has made ppc64 PMU
exceptions a partial NMI. They can get disabled if an external
interrupt comes in, but otherwise PMU interrupts will fire in
interrupt disabled regions.
We disable the hard lockup detector by default for a few reasons:
- It breaks userspace event based branches on POWER8.
- It is likely to produce false positives on KVM guests.
- Since PMCs can only count to 2^31, counting cycles means we might
take multiple PMU exceptions per second per hardware thread even
if our hard lockup timeout is 10 seconds.
It can be enabled via a boot option, or via procfs.
Signed-off-by: Anton Blanchard an...@samba.org
---
arch/powerpc/Kconfig | 1 +
arch/powerpc/include/asm/nmi.h | 4
arch/powerpc/kernel/setup_64.c | 20
3 files changed, 25 insertions(+)
create mode 100644 arch/powerpc/include/asm/nmi.h
diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig
index a2a168e..f6f4734 100644
--- a/arch/powerpc/Kconfig
+++ b/arch/powerpc/Kconfig
@@ -152,6 +152,7 @@ config PPC
select DCACHE_WORD_ACCESS if PPC64 CPU_LITTLE_ENDIAN
select NO_BOOTMEM
select HAVE_GENERIC_RCU_GUP
+ select HAVE_PERF_EVENTS_NMI if PPC64
config GENERIC_CSUM
def_bool CPU_LITTLE_ENDIAN
diff --git a/arch/powerpc/include/asm/nmi.h b/arch/powerpc/include/asm/nmi.h
new file mode 100644
index 000..ff1ccb3
--- /dev/null
+++ b/arch/powerpc/include/asm/nmi.h
@@ -0,0 +1,4 @@
+#ifndef _ASM_NMI_H
+#define _ASM_NMI_H
+
+#endif /* _ASM_NMI_H */
diff --git a/arch/powerpc/kernel/setup_64.c b/arch/powerpc/kernel/setup_64.c
index 300be7d..ae39ec6 100644
--- a/arch/powerpc/kernel/setup_64.c
+++ b/arch/powerpc/kernel/setup_64.c
@@ -37,6 +37,7 @@
#include linux/memblock.h
#include linux/hugetlb.h
#include linux/memory.h
+#include linux/nmi.h
#include asm/io.h
#include asm/kdump.h
@@ -779,3 +780,22 @@ unsigned long memory_block_size_bytes(void)
struct ppc_pci_io ppc_pci_io;
EXPORT_SYMBOL(ppc_pci_io);
#endif
+
+#ifdef CONFIG_HARDLOCKUP_DETECTOR
+u64 hw_nmi_get_sample_period(int watchdog_thresh)
+{
+ return ppc_proc_freq * watchdog_thresh;
+}
+
+/*
+ * The hardlockup detector breaks PMU event based branches and is likely
+ * to get false positives in KVM guests, so disable it by default.
+ */
+static int __init disable_hardlockup_detector(void)
+{
+ watchdog_enable_hardlockup_detector(false);
+
+ return 0;
+}
+early_initcall(disable_hardlockup_detector);
+#endif
--
2.1.0
___
Linuxppc-dev mailing list
Linuxppc-dev@lists.ozlabs.org
https://lists.ozlabs.org/listinfo/linuxppc-dev
[PATCH 2/2] powerpc: Add ppc64 hard lockup detector support
The hard lockup detector uses a PMU event as a periodic NMI to
detect if we are stuck (where stuck means no timer interrupts have
occurred).
Ben's rework of the ppc64 soft disable code has made ppc64 PMU
exceptions a partial NMI. They can get disabled if an external interrupt
comes in, but otherwise PMU interrupts will fire in interrupt disabled
regions.
I wrote a kernel module to test this patch and noticed we sometimes
missed hard lockup warnings. The RCU code detected the stall first and
issued an IPI to backtrace all CPUs. Unfortunately an IPI is an external
interrupt and that will hard disable interrupts, preventing the hard
lockup detector from going off.
If I reduced the hard lockup threshold to 5 seconds:
echo 5 /proc/sys/kernel/watchdog_thresh
Then it would beat the RCU code in detecting a stall and get a
correct backtrace out.
Another downside is that our PMCs can only count to 2^31, so even when
we ask for 10 seconds of processor cycles, we end up taking a couple
of PMU exceptions a second.
Signed-off-by: Anton Blanchard an...@samba.org
---
v2: Mikey noticed a build issue with oprofile. Since our NMI is just
the PMU hardware it doesn't make any sense for oprofile to try and
use it.
Index: b/arch/powerpc/Kconfig
===
--- a/arch/powerpc/Kconfig
+++ b/arch/powerpc/Kconfig
@@ -145,6 +145,7 @@ config PPC
select HAVE_IRQ_EXIT_ON_IRQ_STACK
select ARCH_USE_CMPXCHG_LOCKREF if PPC64
select HAVE_ARCH_AUDITSYSCALL
+ select HAVE_PERF_EVENTS_NMI if PPC64
config GENERIC_CSUM
def_bool CPU_LITTLE_ENDIAN
Index: b/arch/powerpc/include/asm/nmi.h
===
--- /dev/null
+++ b/arch/powerpc/include/asm/nmi.h
@@ -0,0 +1,4 @@
+#ifndef _ASM_NMI_H
+#define _ASM_NMI_H
+
+#endif /* _ASM_NMI_H */
Index: b/arch/powerpc/kernel/setup_64.c
===
--- a/arch/powerpc/kernel/setup_64.c
+++ b/arch/powerpc/kernel/setup_64.c
@@ -800,3 +800,10 @@ unsigned long memory_block_size_bytes(vo
struct ppc_pci_io ppc_pci_io;
EXPORT_SYMBOL(ppc_pci_io);
#endif
+
+#ifdef CONFIG_HARDLOCKUP_DETECTOR
+u64 hw_nmi_get_sample_period(int watchdog_thresh)
+{
+ return ppc_proc_freq * watchdog_thresh;
+}
+#endif
Index: b/arch/Kconfig
===
--- a/arch/Kconfig
+++ b/arch/Kconfig
@@ -32,7 +32,7 @@ config HAVE_OPROFILE
config OPROFILE_NMI_TIMER
def_bool y
- depends on PERF_EVENTS HAVE_PERF_EVENTS_NMI
+ depends on (PERF_EVENTS HAVE_PERF_EVENTS_NMI) !PPC
config KPROBES
bool Kprobes
___
Linuxppc-dev mailing list
Linuxppc-dev@lists.ozlabs.org
https://lists.ozlabs.org/listinfo/linuxppc-dev
Re: [PATCH 2/2] powerpc: Add ppc64 hard lockup detector support
On Tue, Aug 12, 2014 at 09:31:37AM +1000, Anton Blanchard wrote:
The hard lockup detector uses a PMU event as a periodic NMI to
detect if we are stuck (where stuck means no timer interrupts have
occurred).
Ben's rework of the ppc64 soft disable code has made ppc64 PMU
exceptions a partial NMI. They can get disabled if an external interrupt
comes in, but otherwise PMU interrupts will fire in interrupt disabled
regions.
I wrote a kernel module to test this patch and noticed we sometimes
missed hard lockup warnings. The RCU code detected the stall first and
issued an IPI to backtrace all CPUs. Unfortunately an IPI is an external
interrupt and that will hard disable interrupts, preventing the hard
lockup detector from going off.
If it helps, commit bc1dce514e9b (rcu: Don't use NMIs to dump other
CPUs' stacks) makes RCU avoid this behavior. It instead reads the
stacks out remotely when this commit is applied. It is in -tip, and
should make mainline this merge window. Corresponding patch below.
Thanx, Paul
rcu: Don't use NMIs to dump other CPUs' stacks
Although NMI-based stack dumps are in principle more accurate, they are
also more likely to trigger deadlocks. This commit therefore replaces
all uses of trigger_all_cpu_backtrace() with rcu_dump_cpu_stacks(), so
that the CPU detecting an RCU CPU stall does the stack dumping.
Signed-off-by: Paul E. McKenney paul...@linux.vnet.ibm.com
Reviewed-by: Lai Jiangshan la...@cn.fujitsu.com
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 3f93033d3c61..8f3e4d43d736 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -1013,10 +1013,7 @@ static void record_gp_stall_check_time(struct rcu_state
*rsp)
}
/*
- * Dump stacks of all tasks running on stalled CPUs. This is a fallback
- * for architectures that do not implement trigger_all_cpu_backtrace().
- * The NMI-triggered stack traces are more accurate because they are
- * printed by the target CPU.
+ * Dump stacks of all tasks running on stalled CPUs.
*/
static void rcu_dump_cpu_stacks(struct rcu_state *rsp)
{
@@ -1094,7 +1091,7 @@ static void print_other_cpu_stall(struct rcu_state *rsp)
(long)rsp-gpnum, (long)rsp-completed, totqlen);
if (ndetected == 0)
pr_err(INFO: Stall ended before state dump start\n);
- else if (!trigger_all_cpu_backtrace())
+ else
rcu_dump_cpu_stacks(rsp);
/* Complain about tasks blocking the grace period. */
@@ -1125,8 +1122,7 @@ static void print_cpu_stall(struct rcu_state *rsp)
pr_cont( (t=%lu jiffies g=%ld c=%ld q=%lu)\n,
jiffies - rsp-gp_start,
(long)rsp-gpnum, (long)rsp-completed, totqlen);
- if (!trigger_all_cpu_backtrace())
- dump_stack();
+ rcu_dump_cpu_stacks(rsp);
raw_spin_lock_irqsave(rnp-lock, flags);
if (ULONG_CMP_GE(jiffies, ACCESS_ONCE(rsp-jiffies_stall)))
___
Linuxppc-dev mailing list
Linuxppc-dev@lists.ozlabs.org
https://lists.ozlabs.org/listinfo/linuxppc-dev
[PATCH 2/2] powerpc: Add ppc64 hard lockup detector support
The hard lockup detector uses a PMU event as a periodic NMI to
detect if we are stuck (where stuck means no timer interrupts have
occurred).
Ben's rework of the ppc64 soft disable code has made ppc64 PMU
exceptions a partial NMI. They can get disabled if an external interrupt
comes in, but otherwise PMU interrupts will fire in interrupt disabled
regions.
I wrote a kernel module to test this patch and noticed we sometimes
missed hard lockup warnings. The RCU code detected the stall first and
issued an IPI to backtrace all CPUs. Unfortunately an IPI is an external
interrupt and that will hard disable interrupts, preventing the hard
lockup detector from going off.
If I reduced the hard lockup threshold to 5 seconds:
echo 5 /proc/sys/kernel/watchdog_thresh
Then it would beat the RCU code in detecting a stall and get a
correct backtrace out.
Another downside is that our PMCs can only count to 2^31, so even when
we ask for 10 seconds of processor cycles, we end up taking a couple
of PMU exceptions a second.
Signed-off-by: Anton Blanchard an...@samba.org
---
Index: b/arch/powerpc/Kconfig
===
--- a/arch/powerpc/Kconfig
+++ b/arch/powerpc/Kconfig
@@ -145,6 +145,7 @@ config PPC
select HAVE_IRQ_EXIT_ON_IRQ_STACK
select ARCH_USE_CMPXCHG_LOCKREF if PPC64
select HAVE_ARCH_AUDITSYSCALL
+ select HAVE_PERF_EVENTS_NMI if PPC64
config GENERIC_CSUM
def_bool CPU_LITTLE_ENDIAN
Index: b/arch/powerpc/include/asm/nmi.h
===
--- /dev/null
+++ b/arch/powerpc/include/asm/nmi.h
@@ -0,0 +1,4 @@
+#ifndef _ASM_NMI_H
+#define _ASM_NMI_H
+
+#endif /* _ASM_NMI_H */
Index: b/arch/powerpc/kernel/setup_64.c
===
--- a/arch/powerpc/kernel/setup_64.c
+++ b/arch/powerpc/kernel/setup_64.c
@@ -796,3 +796,10 @@ unsigned long memory_block_size_bytes(vo
struct ppc_pci_io ppc_pci_io;
EXPORT_SYMBOL(ppc_pci_io);
#endif
+
+#ifdef CONFIG_HARDLOCKUP_DETECTOR
+u64 hw_nmi_get_sample_period(int watchdog_thresh)
+{
+ return ppc_proc_freq * watchdog_thresh;
+}
+#endif
___
Linuxppc-dev mailing list
Linuxppc-dev@lists.ozlabs.org
https://lists.ozlabs.org/listinfo/linuxppc-dev
