[tip:x86/pti] x86/pti: Enable global pages for shared areas

[tip-bot for Dave Hansen]

Commit-ID:  0f561fce4d6979a50415616896512f87a6d1d5c8
Gitweb: https://git.kernel.org/tip/0f561fce4d6979a50415616896512f87a6d1d5c8
Author: Dave Hansen 
AuthorDate: Fri, 6 Apr 2018 13:55:15 -0700
Committer:  Ingo Molnar 
CommitDate: Thu, 12 Apr 2018 09:05:59 +0200

x86/pti: Enable global pages for shared areas

The entry/exit text and cpu_entry_area are mapped into userspace and
the kernel.  But, they are not _PAGE_GLOBAL.  This creates unnecessary
TLB misses.

Add the _PAGE_GLOBAL flag for these areas.

Signed-off-by: Dave Hansen 
Cc: Andrea Arcangeli 
Cc: Andy Lutomirski 
Cc: Arjan van de Ven 
Cc: Borislav Petkov 
Cc: Dan Williams 
Cc: David Woodhouse 
Cc: Greg Kroah-Hartman 
Cc: Hugh Dickins 
Cc: Josh Poimboeuf 
Cc: Juergen Gross 
Cc: Kees Cook 
Cc: Linus Torvalds 
Cc: Nadav Amit 
Cc: Peter Zijlstra 
Cc: Thomas Gleixner 
Cc: linux...@kvack.org
Link: http://lkml.kernel.org/r/20180406205515.2977e...@viggo.jf.intel.com
Signed-off-by: Ingo Molnar 
---
 arch/x86/mm/cpu_entry_area.c | 14 +-
 arch/x86/mm/pti.c| 23 ++-
 2 files changed, 35 insertions(+), 2 deletions(-)

diff --git a/arch/x86/mm/cpu_entry_area.c b/arch/x86/mm/cpu_entry_area.c
index 476d810639a8..b45f5aaefd74 100644
--- a/arch/x86/mm/cpu_entry_area.c
+++ b/arch/x86/mm/cpu_entry_area.c
@@ -27,8 +27,20 @@ EXPORT_SYMBOL(get_cpu_entry_area);
 void cea_set_pte(void *cea_vaddr, phys_addr_t pa, pgprot_t flags)
 {
unsigned long va = (unsigned long) cea_vaddr;
+   pte_t pte = pfn_pte(pa >> PAGE_SHIFT, flags);
 
-   set_pte_vaddr(va, pfn_pte(pa >> PAGE_SHIFT, flags));
+   /*
+* The cpu_entry_area is shared between the user and kernel
+* page tables.  All of its ptes can safely be global.
+* _PAGE_GLOBAL gets reused to help indicate PROT_NONE for
+* non-present PTEs, so be careful not to set it in that
+* case to avoid confusion.
+*/
+   if (boot_cpu_has(X86_FEATURE_PGE) &&
+   (pgprot_val(flags) & _PAGE_PRESENT))
+   pte = pte_set_flags(pte, _PAGE_GLOBAL);
+
+   set_pte_vaddr(va, pte);
 }
 
 static void __init
diff --git a/arch/x86/mm/pti.c b/arch/x86/mm/pti.c
index 631507f0c198..8082f8b0c10e 100644
--- a/arch/x86/mm/pti.c
+++ b/arch/x86/mm/pti.c
@@ -299,6 +299,27 @@ pti_clone_pmds(unsigned long start, unsigned long end, 
pmdval_t clear)
if (WARN_ON(!target_pmd))
return;
 
+   /*
+* Only clone present PMDs.  This ensures only setting
+* _PAGE_GLOBAL on present PMDs.  This should only be
+* called on well-known addresses anyway, so a non-
+* present PMD would be a surprise.
+*/
+   if (WARN_ON(!(pmd_flags(*pmd) & _PAGE_PRESENT)))
+   return;
+
+   /*
+* Setting 'target_pmd' below creates a mapping in both
+* the user and kernel page tables.  It is effectively
+* global, so set it as global in both copies.  Note:
+* the X86_FEATURE_PGE check is not _required_ because
+* the CPU ignores _PAGE_GLOBAL when PGE is not
+* supported.  The check keeps consistentency with
+* code that only set this bit when supported.
+*/
+   if (boot_cpu_has(X86_FEATURE_PGE))
+   *pmd = pmd_set_flags(*pmd, _PAGE_GLOBAL);
+
/*
 * Copy the PMD.  That is, the kernelmode and usermode
 * tables will share the last-level page tables of this
@@ -348,7 +369,7 @@ static void __init pti_clone_entry_text(void)
 {
pti_clone_pmds((unsigned long) __entry_text_start,
(unsigned long) __irqentry_text_end,
-  _PAGE_RW | _PAGE_GLOBAL);
+  _PAGE_RW);
 }
 
 /*

[tip:x86/pti] x86/pti: Enable global pages for shared areas

[tip-bot for Dave Hansen]

Commit-ID:  0f561fce4d6979a50415616896512f87a6d1d5c8
Gitweb: https://git.kernel.org/tip/0f561fce4d6979a50415616896512f87a6d1d5c8
Author: Dave Hansen 
AuthorDate: Fri, 6 Apr 2018 13:55:15 -0700
Committer:  Ingo Molnar 
CommitDate: Thu, 12 Apr 2018 09:05:59 +0200

x86/pti: Enable global pages for shared areas

The entry/exit text and cpu_entry_area are mapped into userspace and
the kernel.  But, they are not _PAGE_GLOBAL.  This creates unnecessary
TLB misses.

Add the _PAGE_GLOBAL flag for these areas.

Signed-off-by: Dave Hansen 
Cc: Andrea Arcangeli 
Cc: Andy Lutomirski 
Cc: Arjan van de Ven 
Cc: Borislav Petkov 
Cc: Dan Williams 
Cc: David Woodhouse 
Cc: Greg Kroah-Hartman 
Cc: Hugh Dickins 
Cc: Josh Poimboeuf 
Cc: Juergen Gross 
Cc: Kees Cook 
Cc: Linus Torvalds 
Cc: Nadav Amit 
Cc: Peter Zijlstra 
Cc: Thomas Gleixner 
Cc: linux...@kvack.org
Link: http://lkml.kernel.org/r/20180406205515.2977e...@viggo.jf.intel.com
Signed-off-by: Ingo Molnar 
---
 arch/x86/mm/cpu_entry_area.c | 14 +-
 arch/x86/mm/pti.c| 23 ++-
 2 files changed, 35 insertions(+), 2 deletions(-)

diff --git a/arch/x86/mm/cpu_entry_area.c b/arch/x86/mm/cpu_entry_area.c
index 476d810639a8..b45f5aaefd74 100644
--- a/arch/x86/mm/cpu_entry_area.c
+++ b/arch/x86/mm/cpu_entry_area.c
@@ -27,8 +27,20 @@ EXPORT_SYMBOL(get_cpu_entry_area);
 void cea_set_pte(void *cea_vaddr, phys_addr_t pa, pgprot_t flags)
 {
unsigned long va = (unsigned long) cea_vaddr;
+   pte_t pte = pfn_pte(pa >> PAGE_SHIFT, flags);
 
-   set_pte_vaddr(va, pfn_pte(pa >> PAGE_SHIFT, flags));
+   /*
+* The cpu_entry_area is shared between the user and kernel
+* page tables.  All of its ptes can safely be global.
+* _PAGE_GLOBAL gets reused to help indicate PROT_NONE for
+* non-present PTEs, so be careful not to set it in that
+* case to avoid confusion.
+*/
+   if (boot_cpu_has(X86_FEATURE_PGE) &&
+   (pgprot_val(flags) & _PAGE_PRESENT))
+   pte = pte_set_flags(pte, _PAGE_GLOBAL);
+
+   set_pte_vaddr(va, pte);
 }
 
 static void __init
diff --git a/arch/x86/mm/pti.c b/arch/x86/mm/pti.c
index 631507f0c198..8082f8b0c10e 100644
--- a/arch/x86/mm/pti.c
+++ b/arch/x86/mm/pti.c
@@ -299,6 +299,27 @@ pti_clone_pmds(unsigned long start, unsigned long end, 
pmdval_t clear)
if (WARN_ON(!target_pmd))
return;
 
+   /*
+* Only clone present PMDs.  This ensures only setting
+* _PAGE_GLOBAL on present PMDs.  This should only be
+* called on well-known addresses anyway, so a non-
+* present PMD would be a surprise.
+*/
+   if (WARN_ON(!(pmd_flags(*pmd) & _PAGE_PRESENT)))
+   return;
+
+   /*
+* Setting 'target_pmd' below creates a mapping in both
+* the user and kernel page tables.  It is effectively
+* global, so set it as global in both copies.  Note:
+* the X86_FEATURE_PGE check is not _required_ because
+* the CPU ignores _PAGE_GLOBAL when PGE is not
+* supported.  The check keeps consistentency with
+* code that only set this bit when supported.
+*/
+   if (boot_cpu_has(X86_FEATURE_PGE))
+   *pmd = pmd_set_flags(*pmd, _PAGE_GLOBAL);
+
/*
 * Copy the PMD.  That is, the kernelmode and usermode
 * tables will share the last-level page tables of this
@@ -348,7 +369,7 @@ static void __init pti_clone_entry_text(void)
 {
pti_clone_pmds((unsigned long) __entry_text_start,
(unsigned long) __irqentry_text_end,
-  _PAGE_RW | _PAGE_GLOBAL);
+  _PAGE_RW);
 }
 
 /*

[tip:x86/pti] x86/pti: Enable global pages for shared areas

[tip-bot for Dave Hansen]

Commit-ID:  e0bb456e32505b08e42477714169111fbdbff95b
Gitweb: https://git.kernel.org/tip/e0bb456e32505b08e42477714169111fbdbff95b
Author: Dave Hansen 
AuthorDate: Fri, 6 Apr 2018 13:55:15 -0700
Committer:  Ingo Molnar 
CommitDate: Mon, 9 Apr 2018 18:27:34 +0200

x86/pti: Enable global pages for shared areas

The entry/exit text and cpu_entry_area are mapped into userspace and
the kernel.  But, they are not _PAGE_GLOBAL.  This creates unnecessary
TLB misses.

Add the _PAGE_GLOBAL flag for these areas.

Signed-off-by: Dave Hansen 
Cc: Andrea Arcangeli 
Cc: Andy Lutomirski 
Cc: Arjan van de Ven 
Cc: Borislav Petkov 
Cc: Dan Williams 
Cc: David Woodhouse 
Cc: Greg Kroah-Hartman 
Cc: Hugh Dickins 
Cc: Josh Poimboeuf 
Cc: Juergen Gross 
Cc: Kees Cook 
Cc: Linus Torvalds 
Cc: Nadav Amit 
Cc: Peter Zijlstra 
Cc: Thomas Gleixner 
Cc: linux...@kvack.org
Link: http://lkml.kernel.org/r/20180406205515.2977e...@viggo.jf.intel.com
Signed-off-by: Ingo Molnar 
---
 arch/x86/mm/cpu_entry_area.c | 14 +-
 arch/x86/mm/pti.c| 23 ++-
 2 files changed, 35 insertions(+), 2 deletions(-)

diff --git a/arch/x86/mm/cpu_entry_area.c b/arch/x86/mm/cpu_entry_area.c
index 476d810639a8..b45f5aaefd74 100644
--- a/arch/x86/mm/cpu_entry_area.c
+++ b/arch/x86/mm/cpu_entry_area.c
@@ -27,8 +27,20 @@ EXPORT_SYMBOL(get_cpu_entry_area);
 void cea_set_pte(void *cea_vaddr, phys_addr_t pa, pgprot_t flags)
 {
unsigned long va = (unsigned long) cea_vaddr;
+   pte_t pte = pfn_pte(pa >> PAGE_SHIFT, flags);
 
-   set_pte_vaddr(va, pfn_pte(pa >> PAGE_SHIFT, flags));
+   /*
+* The cpu_entry_area is shared between the user and kernel
+* page tables.  All of its ptes can safely be global.
+* _PAGE_GLOBAL gets reused to help indicate PROT_NONE for
+* non-present PTEs, so be careful not to set it in that
+* case to avoid confusion.
+*/
+   if (boot_cpu_has(X86_FEATURE_PGE) &&
+   (pgprot_val(flags) & _PAGE_PRESENT))
+   pte = pte_set_flags(pte, _PAGE_GLOBAL);
+
+   set_pte_vaddr(va, pte);
 }
 
 static void __init
diff --git a/arch/x86/mm/pti.c b/arch/x86/mm/pti.c
index 631507f0c198..8082f8b0c10e 100644
--- a/arch/x86/mm/pti.c
+++ b/arch/x86/mm/pti.c
@@ -299,6 +299,27 @@ pti_clone_pmds(unsigned long start, unsigned long end, 
pmdval_t clear)
if (WARN_ON(!target_pmd))
return;
 
+   /*
+* Only clone present PMDs.  This ensures only setting
+* _PAGE_GLOBAL on present PMDs.  This should only be
+* called on well-known addresses anyway, so a non-
+* present PMD would be a surprise.
+*/
+   if (WARN_ON(!(pmd_flags(*pmd) & _PAGE_PRESENT)))
+   return;
+
+   /*
+* Setting 'target_pmd' below creates a mapping in both
+* the user and kernel page tables.  It is effectively
+* global, so set it as global in both copies.  Note:
+* the X86_FEATURE_PGE check is not _required_ because
+* the CPU ignores _PAGE_GLOBAL when PGE is not
+* supported.  The check keeps consistentency with
+* code that only set this bit when supported.
+*/
+   if (boot_cpu_has(X86_FEATURE_PGE))
+   *pmd = pmd_set_flags(*pmd, _PAGE_GLOBAL);
+
/*
 * Copy the PMD.  That is, the kernelmode and usermode
 * tables will share the last-level page tables of this
@@ -348,7 +369,7 @@ static void __init pti_clone_entry_text(void)
 {
pti_clone_pmds((unsigned long) __entry_text_start,
(unsigned long) __irqentry_text_end,
-  _PAGE_RW | _PAGE_GLOBAL);
+  _PAGE_RW);
 }
 
 /*

[tip:x86/pti] x86/pti: Enable global pages for shared areas

[tip-bot for Dave Hansen]

Commit-ID:  e0bb456e32505b08e42477714169111fbdbff95b
Gitweb: https://git.kernel.org/tip/e0bb456e32505b08e42477714169111fbdbff95b
Author: Dave Hansen 
AuthorDate: Fri, 6 Apr 2018 13:55:15 -0700
Committer:  Ingo Molnar 
CommitDate: Mon, 9 Apr 2018 18:27:34 +0200

x86/pti: Enable global pages for shared areas

The entry/exit text and cpu_entry_area are mapped into userspace and
the kernel.  But, they are not _PAGE_GLOBAL.  This creates unnecessary
TLB misses.

Add the _PAGE_GLOBAL flag for these areas.

Signed-off-by: Dave Hansen 
Cc: Andrea Arcangeli 
Cc: Andy Lutomirski 
Cc: Arjan van de Ven 
Cc: Borislav Petkov 
Cc: Dan Williams 
Cc: David Woodhouse 
Cc: Greg Kroah-Hartman 
Cc: Hugh Dickins 
Cc: Josh Poimboeuf 
Cc: Juergen Gross 
Cc: Kees Cook 
Cc: Linus Torvalds 
Cc: Nadav Amit 
Cc: Peter Zijlstra 
Cc: Thomas Gleixner 
Cc: linux...@kvack.org
Link: http://lkml.kernel.org/r/20180406205515.2977e...@viggo.jf.intel.com
Signed-off-by: Ingo Molnar 
---
 arch/x86/mm/cpu_entry_area.c | 14 +-
 arch/x86/mm/pti.c| 23 ++-
 2 files changed, 35 insertions(+), 2 deletions(-)

diff --git a/arch/x86/mm/cpu_entry_area.c b/arch/x86/mm/cpu_entry_area.c
index 476d810639a8..b45f5aaefd74 100644
--- a/arch/x86/mm/cpu_entry_area.c
+++ b/arch/x86/mm/cpu_entry_area.c
@@ -27,8 +27,20 @@ EXPORT_SYMBOL(get_cpu_entry_area);
 void cea_set_pte(void *cea_vaddr, phys_addr_t pa, pgprot_t flags)
 {
unsigned long va = (unsigned long) cea_vaddr;
+   pte_t pte = pfn_pte(pa >> PAGE_SHIFT, flags);
 
-   set_pte_vaddr(va, pfn_pte(pa >> PAGE_SHIFT, flags));
+   /*
+* The cpu_entry_area is shared between the user and kernel
+* page tables.  All of its ptes can safely be global.
+* _PAGE_GLOBAL gets reused to help indicate PROT_NONE for
+* non-present PTEs, so be careful not to set it in that
+* case to avoid confusion.
+*/
+   if (boot_cpu_has(X86_FEATURE_PGE) &&
+   (pgprot_val(flags) & _PAGE_PRESENT))
+   pte = pte_set_flags(pte, _PAGE_GLOBAL);
+
+   set_pte_vaddr(va, pte);
 }
 
 static void __init
diff --git a/arch/x86/mm/pti.c b/arch/x86/mm/pti.c
index 631507f0c198..8082f8b0c10e 100644
--- a/arch/x86/mm/pti.c
+++ b/arch/x86/mm/pti.c
@@ -299,6 +299,27 @@ pti_clone_pmds(unsigned long start, unsigned long end, 
pmdval_t clear)
if (WARN_ON(!target_pmd))
return;
 
+   /*
+* Only clone present PMDs.  This ensures only setting
+* _PAGE_GLOBAL on present PMDs.  This should only be
+* called on well-known addresses anyway, so a non-
+* present PMD would be a surprise.
+*/
+   if (WARN_ON(!(pmd_flags(*pmd) & _PAGE_PRESENT)))
+   return;
+
+   /*
+* Setting 'target_pmd' below creates a mapping in both
+* the user and kernel page tables.  It is effectively
+* global, so set it as global in both copies.  Note:
+* the X86_FEATURE_PGE check is not _required_ because
+* the CPU ignores _PAGE_GLOBAL when PGE is not
+* supported.  The check keeps consistentency with
+* code that only set this bit when supported.
+*/
+   if (boot_cpu_has(X86_FEATURE_PGE))
+   *pmd = pmd_set_flags(*pmd, _PAGE_GLOBAL);
+
/*
 * Copy the PMD.  That is, the kernelmode and usermode
 * tables will share the last-level page tables of this
@@ -348,7 +369,7 @@ static void __init pti_clone_entry_text(void)
 {
pti_clone_pmds((unsigned long) __entry_text_start,
(unsigned long) __irqentry_text_end,
-  _PAGE_RW | _PAGE_GLOBAL);
+  _PAGE_RW);
 }
 
 /*