commit a3af84b0fa00ead01fcd0e28b5d773ff25990a0d upstream.
Enable hardening against JIT spraying when Spectre-v2 mitigations are in
use. Specifically, issue an IBPB flush on BPF JIT memory reuse. Skip
enabling the IBPB flush if the BPF dispatcher is already using a retpoline
sequence.
This hardening applies only when BPF-JIT is in use. Guard the enabling
under CONFIG_BPF_JIT so that bugs.c still builds with CONFIG_BPF_JIT=n.
[ pawan: Use entry_ibpb() instead of write_ibpb(). JIT hardening enable
moved to spectre_v2_select_mitigation() because there is no
spectre_v2_apply_mitigation()]
Signed-off-by: Pawan Gupta <[email protected]>
Acked-by: Daniel Borkmann <[email protected]>
Acked-by: Dave Hansen <[email protected]>
Signed-off-by: Daniel Borkmann <[email protected]>
---
arch/x86/include/asm/nospec-branch.h | 4 +++
arch/x86/kernel/cpu/bugs.c | 50 ++++++++++++++++++++++++++++++++----
2 files changed, 49 insertions(+), 5 deletions(-)
diff --git a/arch/x86/include/asm/nospec-branch.h
b/arch/x86/include/asm/nospec-branch.h
index fb469ace3839..c7d019b0ef4a 100644
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -418,6 +418,10 @@ extern void srso_alias_untrain_ret(void);
extern void entry_untrain_ret(void);
extern void entry_ibpb(void);
+#ifdef CONFIG_BPF_JIT
+extern void bpf_arch_ibpb(void);
+#endif
+
#ifdef CONFIG_X86_64
extern void clear_bhb_loop(void);
#endif
diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
index ef1d3a5024ed..32a27c19acde 100644
--- a/arch/x86/kernel/cpu/bugs.c
+++ b/arch/x86/kernel/cpu/bugs.c
@@ -16,6 +16,7 @@
#include <linux/sched/smt.h>
#include <linux/pgtable.h>
#include <linux/bpf.h>
+#include <linux/filter.h>
#include <asm/spec-ctrl.h>
#include <asm/cmdline.h>
@@ -1360,8 +1361,21 @@ static inline const char *spectre_v2_module_string(void)
{
return spectre_v2_bad_module ? " - vulnerable module loaded" : "";
}
+
+/*
+ * The "retpoline sequence" is the "call;mov;ret" sequence that
+ * replaces normal indirect branch instructions. Differentiate
+ * *the* retpoline sequence from the LFENCE-prefixed indirect
+ * branches that simply use the retpoline infrastructure.
+ */
+static inline bool retpoline_seq_enabled(void)
+{
+ return boot_cpu_has(X86_FEATURE_RETPOLINE) &&
!boot_cpu_has(X86_FEATURE_RETPOLINE_LFENCE);
+}
+
#else
static inline const char *spectre_v2_module_string(void) { return ""; }
+static inline bool retpoline_seq_enabled(void) { return false; }
#endif
#define SPECTRE_V2_LFENCE_MSG "WARNING: LFENCE mitigation is not recommended
for this CPU, data leaks possible!\n"
@@ -1835,8 +1849,7 @@ static void __init bhi_select_mitigation(void)
return;
/* Retpoline mitigates against BHI unless the CPU has RRSBA behavior */
- if (boot_cpu_has(X86_FEATURE_RETPOLINE) &&
- !boot_cpu_has(X86_FEATURE_RETPOLINE_LFENCE)) {
+ if (retpoline_seq_enabled()) {
spec_ctrl_disable_kernel_rrsba();
if (rrsba_disabled)
return;
@@ -1858,6 +1871,27 @@ static void __init bhi_select_mitigation(void)
pr_info("Spectre BHI mitigation: SW BHB clearing on syscall\n");
}
+#ifdef CONFIG_BPF_JIT
+static void __bpf_arch_ibpb(void *unused)
+{
+ entry_ibpb();
+}
+
+void bpf_arch_ibpb(void)
+{
+ on_each_cpu(__bpf_arch_ibpb, NULL, 1);
+}
+
+static bool __init cpu_wants_ibpb_bpf(void)
+{
+ /* A genuine retpoline already neutralizes ring0 indirect predictions */
+ if (retpoline_seq_enabled())
+ return false;
+
+ return boot_cpu_has(X86_FEATURE_IBPB);
+}
+#endif
+
static void __init spectre_v2_select_mitigation(void)
{
enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline();
@@ -2041,6 +2075,14 @@ static void __init spectre_v2_select_mitigation(void)
pr_info("Enabling Restricted Speculation for firmware calls\n");
}
+#ifdef CONFIG_BPF_JIT
+ if (cpu_wants_ibpb_bpf()) {
+ static_call_update(bpf_arch_pred_flush, bpf_arch_ibpb);
+ static_branch_enable(&bpf_pred_flush_enabled);
+ pr_info("Enabling IBPB for BPF\n");
+ }
+#endif
+
/* Set up IBPB and STIBP depending on the general spectre V2 command */
spectre_v2_cmd = cmd;
}
@@ -3210,9 +3252,7 @@ static const char *spectre_bhi_state(void)
return "; BHI: BHI_DIS_S";
else if (boot_cpu_has(X86_FEATURE_CLEAR_BHB_LOOP))
return "; BHI: SW loop, KVM: SW loop";
- else if (boot_cpu_has(X86_FEATURE_RETPOLINE) &&
- !boot_cpu_has(X86_FEATURE_RETPOLINE_LFENCE) &&
- rrsba_disabled)
+ else if (retpoline_seq_enabled() && rrsba_disabled)
return "; BHI: Retpoline";
else if (boot_cpu_has(X86_FEATURE_CLEAR_BHB_LOOP_ON_VMEXIT))
return "; BHI: Vulnerable, KVM: SW loop";
--
2.43.0
