When entering the kernel with CR3 pointing to the kernel's PGD, there's no need to set it again. This will avoid a TLB flush on syscalls for tasks running with the kernel's PGD (see next patch).
Signed-off-by: Willy Tarreau <[email protected]> Cc: Andy Lutomirski <[email protected]> Cc: Borislav Petkov <[email protected]> Cc: Brian Gerst <[email protected]> Cc: Dave Hansen <[email protected]> Cc: Ingo Molnar <[email protected]> Cc: Linus Torvalds <[email protected]> Cc: Peter Zijlstra <[email protected]> Cc: Thomas Gleixner <[email protected]> Cc: Josh Poimboeuf <[email protected]> Cc: "H. Peter Anvin" <[email protected]> Cc: Kees Cook <[email protected]> v2: - updated comments according to Ingo's suggestions - split the code to keep only the CR3 changes here --- arch/x86/entry/calling.h | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/arch/x86/entry/calling.h b/arch/x86/entry/calling.h index 45a63e0..19c6790 100644 --- a/arch/x86/entry/calling.h +++ b/arch/x86/entry/calling.h @@ -214,6 +214,11 @@ .macro SWITCH_TO_KERNEL_CR3 scratch_reg:req ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI mov %cr3, \scratch_reg + + /* If we're already on the kernel PGD, we don't switch */ + testq $(PTI_SWITCH_PGTABLES_MASK), \scratch_reg + jz .Lend_\@ + ADJUST_KERNEL_CR3 \scratch_reg mov \scratch_reg, %cr3 .Lend_\@: @@ -262,6 +267,14 @@ ALTERNATIVE "jmp .Ldone_\@", "", X86_FEATURE_PTI movq %cr3, \scratch_reg movq \scratch_reg, \save_reg + + /* + * If we're already on the kernel PGD, we don't switch, + * we just save the current CR3. + */ + testq $(PTI_SWITCH_PGTABLES_MASK), \scratch_reg + jz .Ldone_\@ + /* * Is the "switch mask" all zero? That means that both of * these are zero: @@ -284,6 +297,13 @@ .macro RESTORE_CR3 scratch_reg:req save_reg:req ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI + /* + * If we saved a kernel context on entry, we didn't switch the CR3, + * so we don't need to restore it on the way out either: + */ + testq $(PTI_SWITCH_PGTABLES_MASK), \save_reg + jz .Lend_\@ + ALTERNATIVE "jmp .Lwrcr3_\@", "", X86_FEATURE_PCID /* -- 1.7.12.1
