Re: [PATCH 4/4] x86/mm: Only use tlb_remove_table() for paravirt
Hey Peter, On Wed, Aug 22, 2018 at 05:30:16PM +0200, Peter Zijlstra wrote: > If we don't use paravirt; don't play unnecessary and complicated games > to free page-tables. > > Suggested-by: Linus Torvalds > Signed-off-by: Peter Zijlstra (Intel) > --- > arch/x86/Kconfig |2 +- > arch/x86/hyperv/mmu.c |2 ++ > arch/x86/include/asm/paravirt.h |5 + > arch/x86/include/asm/paravirt_types.h |3 +++ > arch/x86/include/asm/tlbflush.h |3 +++ > arch/x86/kernel/kvm.c |5 - > arch/x86/kernel/paravirt.c|2 ++ > arch/x86/mm/pgtable.c |8 > arch/x86/xen/mmu_pv.c |1 + > 9 files changed, 25 insertions(+), 6 deletions(-) > > --- a/arch/x86/xen/mmu_pv.c > +++ b/arch/x86/xen/mmu_pv.c > @@ -2397,6 +2397,7 @@ static const struct pv_mmu_ops xen_mmu_o > .flush_tlb_kernel = xen_flush_tlb, > .flush_tlb_one_user = xen_flush_tlb_one_user, > .flush_tlb_others = xen_flush_tlb_others, > + .tlb_remove_table = tlb_remove_table, I tried compiling the series, and I am getting the following compilation error: CC arch/x86/xen/mmu_pv.o arch/x86/xen/mmu_pv.c:2400:22: error: ‘tlb_remove_table’ undeclared here (not in a function); did you mean ‘bus_remove_file’? .tlb_remove_table = tlb_remove_table, ^~~~ bus_remove_file make[4]: *** [arch/x86/xen/mmu_pv.o] Error 1 which I solved by: diff --git a/arch/x86/xen/mmu_pv.c b/arch/x86/xen/mmu_pv.c index a21988ec30d3..3114b7c76970 100644 --- a/arch/x86/xen/mmu_pv.c +++ b/arch/x86/xen/mmu_pv.c @@ -67,6 +67,7 @@ #include #include #include +#include #include #include > > .pgd_alloc = xen_pgd_alloc, > .pgd_free = xen_pgd_free, > > > -- All the best, Eduardo Valentin
Re: [PATCH 4/4] x86/mm: Only use tlb_remove_table() for paravirt
Hey Peter, On Wed, Aug 22, 2018 at 05:30:16PM +0200, Peter Zijlstra wrote: > If we don't use paravirt; don't play unnecessary and complicated games > to free page-tables. > > Suggested-by: Linus Torvalds > Signed-off-by: Peter Zijlstra (Intel) > --- > arch/x86/Kconfig |2 +- > arch/x86/hyperv/mmu.c |2 ++ > arch/x86/include/asm/paravirt.h |5 + > arch/x86/include/asm/paravirt_types.h |3 +++ > arch/x86/include/asm/tlbflush.h |3 +++ > arch/x86/kernel/kvm.c |5 - > arch/x86/kernel/paravirt.c|2 ++ > arch/x86/mm/pgtable.c |8 > arch/x86/xen/mmu_pv.c |1 + > 9 files changed, 25 insertions(+), 6 deletions(-) > > --- a/arch/x86/xen/mmu_pv.c > +++ b/arch/x86/xen/mmu_pv.c > @@ -2397,6 +2397,7 @@ static const struct pv_mmu_ops xen_mmu_o > .flush_tlb_kernel = xen_flush_tlb, > .flush_tlb_one_user = xen_flush_tlb_one_user, > .flush_tlb_others = xen_flush_tlb_others, > + .tlb_remove_table = tlb_remove_table, I tried compiling the series, and I am getting the following compilation error: CC arch/x86/xen/mmu_pv.o arch/x86/xen/mmu_pv.c:2400:22: error: ‘tlb_remove_table’ undeclared here (not in a function); did you mean ‘bus_remove_file’? .tlb_remove_table = tlb_remove_table, ^~~~ bus_remove_file make[4]: *** [arch/x86/xen/mmu_pv.o] Error 1 which I solved by: diff --git a/arch/x86/xen/mmu_pv.c b/arch/x86/xen/mmu_pv.c index a21988ec30d3..3114b7c76970 100644 --- a/arch/x86/xen/mmu_pv.c +++ b/arch/x86/xen/mmu_pv.c @@ -67,6 +67,7 @@ #include #include #include +#include #include #include > > .pgd_alloc = xen_pgd_alloc, > .pgd_free = xen_pgd_free, > > > -- All the best, Eduardo Valentin
[PATCH 4/4] x86/mm: Only use tlb_remove_table() for paravirt
If we don't use paravirt; don't play unnecessary and complicated games
to free page-tables.
Suggested-by: Linus Torvalds
Signed-off-by: Peter Zijlstra (Intel)
---
arch/x86/Kconfig |2 +-
arch/x86/hyperv/mmu.c |2 ++
arch/x86/include/asm/paravirt.h |5 +
arch/x86/include/asm/paravirt_types.h |3 +++
arch/x86/include/asm/tlbflush.h |3 +++
arch/x86/kernel/kvm.c |5 -
arch/x86/kernel/paravirt.c|2 ++
arch/x86/mm/pgtable.c |8
arch/x86/xen/mmu_pv.c |1 +
9 files changed, 25 insertions(+), 6 deletions(-)
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -179,7 +179,7 @@ config X86
select HAVE_HARDLOCKUP_DETECTOR_PERFif PERF_EVENTS &&
HAVE_PERF_EVENTS_NMI
select HAVE_PERF_REGS
select HAVE_PERF_USER_STACK_DUMP
- select HAVE_RCU_TABLE_FREE
+ select HAVE_RCU_TABLE_FREE if PARAVIRT
select HAVE_RCU_TABLE_INVALIDATEif HAVE_RCU_TABLE_FREE
select HAVE_REGS_AND_STACK_ACCESS_API
select HAVE_RELIABLE_STACKTRACE if X86_64 &&
(UNWINDER_FRAME_POINTER || UNWINDER_ORC) && STACK_VALIDATION
--- a/arch/x86/hyperv/mmu.c
+++ b/arch/x86/hyperv/mmu.c
@@ -9,6 +9,7 @@
#include
#include
#include
+#include
#define CREATE_TRACE_POINTS
#include
@@ -231,4 +232,5 @@ void hyperv_setup_mmu_ops(void)
pr_info("Using hypercall for remote TLB flush\n");
pv_mmu_ops.flush_tlb_others = hyperv_flush_tlb_others;
+ pv_mmu_ops.tlb_remove_table = tlb_remove_table;
}
--- a/arch/x86/include/asm/paravirt.h
+++ b/arch/x86/include/asm/paravirt.h
@@ -309,6 +309,11 @@ static inline void flush_tlb_others(cons
PVOP_VCALL2(pv_mmu_ops.flush_tlb_others, cpumask, info);
}
+static inline void paravirt_tlb_remove_table(struct mmu_gather *tlb, void
*table)
+{
+ PVOP_VCALL2(pv_mmu_ops.tlb_remove_table, tlb, table);
+}
+
static inline int paravirt_pgd_alloc(struct mm_struct *mm)
{
return PVOP_CALL1(int, pv_mmu_ops.pgd_alloc, mm);
--- a/arch/x86/include/asm/paravirt_types.h
+++ b/arch/x86/include/asm/paravirt_types.h
@@ -54,6 +54,7 @@ struct desc_struct;
struct task_struct;
struct cpumask;
struct flush_tlb_info;
+struct mmu_gather;
/*
* Wrapper type for pointers to code which uses the non-standard
@@ -222,6 +223,8 @@ struct pv_mmu_ops {
void (*flush_tlb_others)(const struct cpumask *cpus,
const struct flush_tlb_info *info);
+ void (*tlb_remove_table)(struct mmu_gather *tlb, void *table);
+
/* Hooks for allocating and freeing a pagetable top-level */
int (*pgd_alloc)(struct mm_struct *mm);
void (*pgd_free)(struct mm_struct *mm, pgd_t *pgd);
--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -552,6 +552,9 @@ extern void arch_tlbbatch_flush(struct a
#ifndef CONFIG_PARAVIRT
#define flush_tlb_others(mask, info) \
native_flush_tlb_others(mask, info)
+
+#define paravirt_tlb_remove_table(tlb, page) \
+ tlb_remove_page(tlb, (void *)(page))
#endif
#endif /* _ASM_X86_TLBFLUSH_H */
--- a/arch/x86/kernel/kvm.c
+++ b/arch/x86/kernel/kvm.c
@@ -45,6 +45,7 @@
#include
#include
#include
+#include
static int kvmapf = 1;
@@ -636,8 +637,10 @@ static void __init kvm_guest_init(void)
if (kvm_para_has_feature(KVM_FEATURE_PV_TLB_FLUSH) &&
!kvm_para_has_hint(KVM_HINTS_REALTIME) &&
- kvm_para_has_feature(KVM_FEATURE_STEAL_TIME))
+ kvm_para_has_feature(KVM_FEATURE_STEAL_TIME)) {
pv_mmu_ops.flush_tlb_others = kvm_flush_tlb_others;
+ pv_mmu_ops.tlb_remove_table = tlb_remove_table;
+ }
if (kvm_para_has_feature(KVM_FEATURE_PV_EOI))
apic_set_eoi_write(kvm_guest_apic_eoi_write);
--- a/arch/x86/kernel/paravirt.c
+++ b/arch/x86/kernel/paravirt.c
@@ -41,6 +41,7 @@
#include
#include
#include
+#include
/*
* nop stub, which must not clobber anything *including the stack* to
@@ -409,6 +410,7 @@ struct pv_mmu_ops pv_mmu_ops __ro_after_
.flush_tlb_kernel = native_flush_tlb_global,
.flush_tlb_one_user = native_flush_tlb_one_user,
.flush_tlb_others = native_flush_tlb_others,
+ .tlb_remove_table = (void (*)(struct mmu_gather *, void
*))tlb_remove_page,
.pgd_alloc = __paravirt_pgd_alloc,
.pgd_free = paravirt_nop,
--- a/arch/x86/mm/pgtable.c
+++ b/arch/x86/mm/pgtable.c
@@ -63,7 +63,7 @@ void ___pte_free_tlb(struct mmu_gather *
{
pgtable_page_dtor(pte);
paravirt_release_pte(page_to_pfn(pte));
- tlb_remove_table(tlb, pte);
+ paravirt_tlb_remove_table(tlb, pte);
}
#if CONFIG_PGTABLE_LEVELS > 2
@@ -79,21 +79,21 @@ void ___pmd_free_tlb(struct mmu_gather *
tlb->need_flush_all = 1;
#endif
pgtable_pmd_page_dtor(page);
- tlb_remove_table(tlb, page);
+
[PATCH 4/4] x86/mm: Only use tlb_remove_table() for paravirt
If we don't use paravirt; don't play unnecessary and complicated games
to free page-tables.
Suggested-by: Linus Torvalds
Signed-off-by: Peter Zijlstra (Intel)
---
arch/x86/Kconfig |2 +-
arch/x86/hyperv/mmu.c |2 ++
arch/x86/include/asm/paravirt.h |5 +
arch/x86/include/asm/paravirt_types.h |3 +++
arch/x86/include/asm/tlbflush.h |3 +++
arch/x86/kernel/kvm.c |5 -
arch/x86/kernel/paravirt.c|2 ++
arch/x86/mm/pgtable.c |8
arch/x86/xen/mmu_pv.c |1 +
9 files changed, 25 insertions(+), 6 deletions(-)
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -179,7 +179,7 @@ config X86
select HAVE_HARDLOCKUP_DETECTOR_PERFif PERF_EVENTS &&
HAVE_PERF_EVENTS_NMI
select HAVE_PERF_REGS
select HAVE_PERF_USER_STACK_DUMP
- select HAVE_RCU_TABLE_FREE
+ select HAVE_RCU_TABLE_FREE if PARAVIRT
select HAVE_RCU_TABLE_INVALIDATEif HAVE_RCU_TABLE_FREE
select HAVE_REGS_AND_STACK_ACCESS_API
select HAVE_RELIABLE_STACKTRACE if X86_64 &&
(UNWINDER_FRAME_POINTER || UNWINDER_ORC) && STACK_VALIDATION
--- a/arch/x86/hyperv/mmu.c
+++ b/arch/x86/hyperv/mmu.c
@@ -9,6 +9,7 @@
#include
#include
#include
+#include
#define CREATE_TRACE_POINTS
#include
@@ -231,4 +232,5 @@ void hyperv_setup_mmu_ops(void)
pr_info("Using hypercall for remote TLB flush\n");
pv_mmu_ops.flush_tlb_others = hyperv_flush_tlb_others;
+ pv_mmu_ops.tlb_remove_table = tlb_remove_table;
}
--- a/arch/x86/include/asm/paravirt.h
+++ b/arch/x86/include/asm/paravirt.h
@@ -309,6 +309,11 @@ static inline void flush_tlb_others(cons
PVOP_VCALL2(pv_mmu_ops.flush_tlb_others, cpumask, info);
}
+static inline void paravirt_tlb_remove_table(struct mmu_gather *tlb, void
*table)
+{
+ PVOP_VCALL2(pv_mmu_ops.tlb_remove_table, tlb, table);
+}
+
static inline int paravirt_pgd_alloc(struct mm_struct *mm)
{
return PVOP_CALL1(int, pv_mmu_ops.pgd_alloc, mm);
--- a/arch/x86/include/asm/paravirt_types.h
+++ b/arch/x86/include/asm/paravirt_types.h
@@ -54,6 +54,7 @@ struct desc_struct;
struct task_struct;
struct cpumask;
struct flush_tlb_info;
+struct mmu_gather;
/*
* Wrapper type for pointers to code which uses the non-standard
@@ -222,6 +223,8 @@ struct pv_mmu_ops {
void (*flush_tlb_others)(const struct cpumask *cpus,
const struct flush_tlb_info *info);
+ void (*tlb_remove_table)(struct mmu_gather *tlb, void *table);
+
/* Hooks for allocating and freeing a pagetable top-level */
int (*pgd_alloc)(struct mm_struct *mm);
void (*pgd_free)(struct mm_struct *mm, pgd_t *pgd);
--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -552,6 +552,9 @@ extern void arch_tlbbatch_flush(struct a
#ifndef CONFIG_PARAVIRT
#define flush_tlb_others(mask, info) \
native_flush_tlb_others(mask, info)
+
+#define paravirt_tlb_remove_table(tlb, page) \
+ tlb_remove_page(tlb, (void *)(page))
#endif
#endif /* _ASM_X86_TLBFLUSH_H */
--- a/arch/x86/kernel/kvm.c
+++ b/arch/x86/kernel/kvm.c
@@ -45,6 +45,7 @@
#include
#include
#include
+#include
static int kvmapf = 1;
@@ -636,8 +637,10 @@ static void __init kvm_guest_init(void)
if (kvm_para_has_feature(KVM_FEATURE_PV_TLB_FLUSH) &&
!kvm_para_has_hint(KVM_HINTS_REALTIME) &&
- kvm_para_has_feature(KVM_FEATURE_STEAL_TIME))
+ kvm_para_has_feature(KVM_FEATURE_STEAL_TIME)) {
pv_mmu_ops.flush_tlb_others = kvm_flush_tlb_others;
+ pv_mmu_ops.tlb_remove_table = tlb_remove_table;
+ }
if (kvm_para_has_feature(KVM_FEATURE_PV_EOI))
apic_set_eoi_write(kvm_guest_apic_eoi_write);
--- a/arch/x86/kernel/paravirt.c
+++ b/arch/x86/kernel/paravirt.c
@@ -41,6 +41,7 @@
#include
#include
#include
+#include
/*
* nop stub, which must not clobber anything *including the stack* to
@@ -409,6 +410,7 @@ struct pv_mmu_ops pv_mmu_ops __ro_after_
.flush_tlb_kernel = native_flush_tlb_global,
.flush_tlb_one_user = native_flush_tlb_one_user,
.flush_tlb_others = native_flush_tlb_others,
+ .tlb_remove_table = (void (*)(struct mmu_gather *, void
*))tlb_remove_page,
.pgd_alloc = __paravirt_pgd_alloc,
.pgd_free = paravirt_nop,
--- a/arch/x86/mm/pgtable.c
+++ b/arch/x86/mm/pgtable.c
@@ -63,7 +63,7 @@ void ___pte_free_tlb(struct mmu_gather *
{
pgtable_page_dtor(pte);
paravirt_release_pte(page_to_pfn(pte));
- tlb_remove_table(tlb, pte);
+ paravirt_tlb_remove_table(tlb, pte);
}
#if CONFIG_PGTABLE_LEVELS > 2
@@ -79,21 +79,21 @@ void ___pmd_free_tlb(struct mmu_gather *
tlb->need_flush_all = 1;
#endif
pgtable_pmd_page_dtor(page);
- tlb_remove_table(tlb, page);
+
