Partially revert 3db6d5a5eca ("x86/mm/tlb: Remove 'struct
flush_tlb_info' from the stack").
Now that we copy flush_tlb_info and inline it with the IPI information,
we can put it back onto the stack. This simplifies the code and should be
slightly more robust. The stack is also a bit more likely to be cached
than a global variable.
Cc: Peter Zijlstra <pet...@infradead.org>
Cc: Andy Lutomirski <l...@kernel.org>
Cc: Borislav Petkov <b...@alien8.de>
Cc: Brian Gerst <brge...@gmail.com>
Cc: Dave Hansen <dave.han...@intel.com>
Cc: H. Peter Anvin <h...@zytor.com>
Cc: Rik van Riel <r...@surriel.com>
Cc: Thomas Gleixner <t...@linutronix.de>
Signed-off-by: Nadav Amit <na...@vmware.com>
---
arch/x86/mm/tlb.c | 46 +++++++++++++++++++++++++---------------------
1 file changed, 25 insertions(+), 21 deletions(-)
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index fd7e90adbe43..81170fd6b1dd 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -827,15 +827,17 @@ static inline void put_flush_tlb_info(void)
static void flush_tlb_on_cpus(const cpumask_t *cpumask,
const struct flush_tlb_info *info)
{
- int this_cpu = smp_processor_id();
bool flush_others = false;
+ int this_cpu;
+
+ this_cpu = get_cpu();
if (cpumask_any_but(cpumask, this_cpu) < nr_cpu_ids)
flush_others = true;
if (static_branch_likely(&flush_tlb_multi_enabled) && flush_others) {
flush_tlb_multi(cpumask, info);
- return;
+ goto out;
}
if (cpumask_test_cpu(this_cpu, cpumask)) {
@@ -847,27 +849,33 @@ static void flush_tlb_on_cpus(const cpumask_t *cpumask,
if (flush_others)
flush_tlb_others(cpumask, info);
+
+out:
+ put_cpu();
}
void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
unsigned long end, unsigned int stride_shift,
bool freed_tables)
{
- struct flush_tlb_info *info;
- u64 new_tlb_gen;
+ struct flush_tlb_info info = {
+ .mm = mm,
+ .stride_shift = stride_shift,
+ .freed_tables = freed_tables,
+ };
/* Should we flush just the requested range? */
if ((end == TLB_FLUSH_ALL) ||
((end - start) >> stride_shift) > tlb_single_page_flush_ceiling) {
- start = 0;
- end = TLB_FLUSH_ALL;
+ info.start = 0;
+ info.end = TLB_FLUSH_ALL;
+ } else {
+ info.start = start;
+ info.end = end;
}
/* This is also a barrier that synchronizes with switch_mm(). */
- new_tlb_gen = inc_mm_tlb_gen(mm);
-
- info = get_flush_tlb_info(mm, start, end, stride_shift, freed_tables,
- new_tlb_gen);
+ info.new_tlb_gen = inc_mm_tlb_gen(mm);
/*
* Assert that mm_cpumask() corresponds with the loaded mm. We got one
@@ -878,9 +886,7 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long
start,
(mm == this_cpu_read(cpu_tlbstate.loaded_mm)) ||
mm == &init_mm);
- flush_tlb_on_cpus(mm_cpumask(mm), info);
-
- put_flush_tlb_info();
+ flush_tlb_on_cpus(mm_cpumask(mm), &info);
}
@@ -913,18 +919,18 @@ void flush_tlb_kernel_range(unsigned long start, unsigned
long end)
(end - start) > tlb_single_page_flush_ceiling << PAGE_SHIFT) {
on_each_cpu(do_flush_tlb_all, NULL, 1);
} else {
- struct flush_tlb_info *info;
-
- info = get_flush_tlb_info(NULL, start, end, 0, false, 0);
+ struct flush_tlb_info info = {
+ .mm = NULL,
+ .start = start,
+ .end = end,
+ };
/*
* We have to wait for the remote shootdown to be done since it
* is kernel space.
*/
__on_each_cpu_mask(cpu_online_mask, do_kernel_range_flush,
- info, sizeof(*info), 1);
-
- put_flush_tlb_info();
+ &info, sizeof(info), 1);
}
}
@@ -942,9 +948,7 @@ static const struct flush_tlb_info full_flush_tlb_info = {
void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch)
{
- preempt_disable();
flush_tlb_on_cpus(&batch->cpumask, &full_flush_tlb_info);
- preempt_enable();
cpumask_clear(&batch->cpumask);
}
--
2.20.1
