From: Johannes Berg <[email protected]> Since this approach shares more code with the MMU version in nommu, further move some code around to be able to use it in nommu (report_enomem) or be able to replace it (this applies to mm_id, stack, sync). Also add empty inlines for the TLB flushing and decouple trap.c code.
Signed-off-by: Johannes Berg <[email protected]> --- arch/um/include/asm/tlbflush.h | 20 +++ arch/um/kernel/Makefile | 2 +- arch/um/kernel/physmem.c | 7 + arch/um/kernel/skas/Makefile | 3 +- arch/um/kernel/skas/mmu.c | 25 ++++ arch/um/kernel/skas/process.c | 26 ---- arch/um/kernel/tlb.c | 7 - arch/um/kernel/trap-mmu.c | 238 +++++++++++++++++++++++++++++++++ arch/um/kernel/trap.c | 219 ------------------------------ 9 files changed, 293 insertions(+), 254 deletions(-) create mode 100644 arch/um/kernel/trap-mmu.c diff --git a/arch/um/include/asm/tlbflush.h b/arch/um/include/asm/tlbflush.h index 13a3009942be..f73b568a0859 100644 --- a/arch/um/include/asm/tlbflush.h +++ b/arch/um/include/asm/tlbflush.h @@ -30,6 +30,8 @@ * - flush_tlb_kernel_range(start, end) flushes a range of kernel pages */ +#ifdef CONFIG_MMU + extern int um_tlb_sync(struct mm_struct *mm); extern void flush_tlb_all(void); @@ -56,4 +58,22 @@ static inline void flush_tlb_kernel_range(unsigned long start, um_tlb_sync(&init_mm); } +#else /* !CONFIG_MMU */ + +/* + * With NOMMU the kernel and userspace share a single host address space, + * so there is nothing to synchronise and all TLB flushes are no-ops. + */ +static inline int um_tlb_sync(struct mm_struct *mm) { return 0; } +static inline void flush_tlb_all(void) { } +static inline void flush_tlb_mm(struct mm_struct *mm) { } +static inline void flush_tlb_page(struct vm_area_struct *vma, + unsigned long address) { } +static inline void flush_tlb_range(struct vm_area_struct *vma, + unsigned long start, unsigned long end) { } +static inline void flush_tlb_kernel_range(unsigned long start, + unsigned long end) { } + +#endif /* CONFIG_MMU */ + #endif diff --git a/arch/um/kernel/Makefile b/arch/um/kernel/Makefile index d56fe6d829cc..5cd1bd4b23ac 100644 --- a/arch/um/kernel/Makefile +++ b/arch/um/kernel/Makefile @@ -19,7 +19,7 @@ obj-y = config.o exec.o exitcode.o irq.o ksyms.o mem.o \ signal.o sysrq.o time.o trap.o \ um_arch.o umid.o kmsg_dump.o capflags.o skas/ obj-y += load_file.o -obj-$(CONFIG_MMU) += mem-pgtable.o tlb.o +obj-$(CONFIG_MMU) += mem-pgtable.o tlb.o trap-mmu.o obj-$(CONFIG_BLK_DEV_INITRD) += initrd.o obj-$(CONFIG_GPROF) += gprof_syms.o diff --git a/arch/um/kernel/physmem.c b/arch/um/kernel/physmem.c index ae6ca373c261..ae9b8f3144b8 100644 --- a/arch/um/kernel/physmem.c +++ b/arch/um/kernel/physmem.c @@ -110,6 +110,13 @@ int phys_mapping(unsigned long phys, unsigned long long *offset_out) } EXPORT_SYMBOL(phys_mapping); +void report_enomem(void) +{ + printk(KERN_ERR "UML ran out of memory on the host side! " + "This can happen due to a memory limitation or " + "vm.max_map_count has been reached.\n"); +} + static int __init uml_mem_setup(char *line, int *add) { char *retptr; diff --git a/arch/um/kernel/skas/Makefile b/arch/um/kernel/skas/Makefile index 3384be42691f..28dd33580abc 100644 --- a/arch/um/kernel/skas/Makefile +++ b/arch/um/kernel/skas/Makefile @@ -3,8 +3,9 @@ # Copyright (C) 2002 - 2007 Jeff Dike (jdike@{addtoit,linux.intel}.com) # -obj-y := stub.o mmu.o process.o syscall.o uaccess.o \ +obj-y := stub.o process.o syscall.o \ stub_exe_embed.o +obj-$(CONFIG_MMU) += mmu.o uaccess.o # Stub executable diff --git a/arch/um/kernel/skas/mmu.c b/arch/um/kernel/skas/mmu.c index b5017096028b..facb88d1e393 100644 --- a/arch/um/kernel/skas/mmu.c +++ b/arch/um/kernel/skas/mmu.c @@ -12,6 +12,7 @@ #include <asm/pgalloc.h> #include <asm/sections.h> #include <asm/mmu_context.h> +#include <asm/tlbflush.h> #include <as-layout.h> #include <os.h> #include <skas.h> @@ -40,6 +41,30 @@ void exit_turnstile(struct mm_id *mm_id) mutex_unlock(__get_turnstile(mm_id)); } +unsigned long current_stub_stack(void) +{ + if (current->mm == NULL) + return 0; + + return current->mm->context.id.stack; +} + +struct mm_id *current_mm_id(void) +{ + if (current->mm == NULL) + return NULL; + + return ¤t->mm->context.id; +} + +void current_mm_sync(void) +{ + if (current->mm == NULL) + return; + + um_tlb_sync(current->mm); +} + int init_new_context(struct task_struct *task, struct mm_struct *mm) { struct mm_id *new_id = &mm->context.id; diff --git a/arch/um/kernel/skas/process.c b/arch/um/kernel/skas/process.c index 4a7673b0261a..dcdce50b4595 100644 --- a/arch/um/kernel/skas/process.c +++ b/arch/um/kernel/skas/process.c @@ -9,8 +9,6 @@ #include <linux/sched/task.h> #include <linux/smp-internal.h> -#include <asm/tlbflush.h> - #include <as-layout.h> #include <kern.h> #include <os.h> @@ -42,30 +40,6 @@ int __init start_uml(void) &init_task.thread.switch_buf); } -unsigned long current_stub_stack(void) -{ - if (current->mm == NULL) - return 0; - - return current->mm->context.id.stack; -} - -struct mm_id *current_mm_id(void) -{ - if (current->mm == NULL) - return NULL; - - return ¤t->mm->context.id; -} - -void current_mm_sync(void) -{ - if (current->mm == NULL) - return; - - um_tlb_sync(current->mm); -} - static DEFINE_SPINLOCK(initial_jmpbuf_spinlock); void initial_jmpbuf_lock(void) diff --git a/arch/um/kernel/tlb.c b/arch/um/kernel/tlb.c index 1f175716b474..012bcc76168d 100644 --- a/arch/um/kernel/tlb.c +++ b/arch/um/kernel/tlb.c @@ -40,13 +40,6 @@ static int kern_unmap(struct mm_id *mm_idp, return os_unmap_memory((void *)virt, len); } -void report_enomem(void) -{ - printk(KERN_ERR "UML ran out of memory on the host side! " - "This can happen due to a memory limitation or " - "vm.max_map_count has been reached.\n"); -} - static inline int update_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, struct vm_ops *ops) diff --git a/arch/um/kernel/trap-mmu.c b/arch/um/kernel/trap-mmu.c new file mode 100644 index 000000000000..a233a61e1801 --- /dev/null +++ b/arch/um/kernel/trap-mmu.c @@ -0,0 +1,238 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2000 - 2007 Jeff Dike (jdike@{addtoit,linux.intel}.com) + */ + +#include <linux/mm.h> +#include <linux/sched/signal.h> +#include <linux/hardirq.h> +#include <linux/module.h> +#include <linux/uaccess.h> +#include <linux/sched/debug.h> +#include <asm/current.h> +#include <asm/tlbflush.h> +#include <arch.h> +#include <as-layout.h> +#include <kern_util.h> +#include <os.h> +#include <skas.h> + +/* + * NOTE: UML does not have exception tables. As such, this is almost a copy + * of the code in mm/memory.c, only adjusting the logic to simply check whether + * we are coming from the kernel instead of doing an additional lookup in the + * exception table. + * We can do this simplification because we never get here if the exception was + * fixable. + */ +static inline bool get_mmap_lock_carefully(struct mm_struct *mm, bool is_user) +{ + if (likely(mmap_read_trylock(mm))) + return true; + + if (!is_user) + return false; + + return !mmap_read_lock_killable(mm); +} + +static inline bool mmap_upgrade_trylock(struct mm_struct *mm) +{ + /* + * We don't have this operation yet. + * + * It should be easy enough to do: it's basically a + * atomic_long_try_cmpxchg_acquire() + * from RWSEM_READER_BIAS -> RWSEM_WRITER_LOCKED, but + * it also needs the proper lockdep magic etc. + */ + return false; +} + +static inline bool upgrade_mmap_lock_carefully(struct mm_struct *mm, bool is_user) +{ + mmap_read_unlock(mm); + if (!is_user) + return false; + + return !mmap_write_lock_killable(mm); +} + +/* + * Helper for page fault handling. + * + * This is kind of equivalend to "mmap_read_lock()" followed + * by "find_extend_vma()", except it's a lot more careful about + * the locking (and will drop the lock on failure). + * + * For example, if we have a kernel bug that causes a page + * fault, we don't want to just use mmap_read_lock() to get + * the mm lock, because that would deadlock if the bug were + * to happen while we're holding the mm lock for writing. + * + * So this checks the exception tables on kernel faults in + * order to only do this all for instructions that are actually + * expected to fault. + * + * We can also actually take the mm lock for writing if we + * need to extend the vma, which helps the VM layer a lot. + */ +static struct vm_area_struct * +um_lock_mm_and_find_vma(struct mm_struct *mm, + unsigned long addr, bool is_user) +{ + struct vm_area_struct *vma; + + if (!get_mmap_lock_carefully(mm, is_user)) + return NULL; + + vma = find_vma(mm, addr); + if (likely(vma && (vma->vm_start <= addr))) + return vma; + + /* + * Well, dang. We might still be successful, but only + * if we can extend a vma to do so. + */ + if (!vma || !(vma->vm_flags & VM_GROWSDOWN)) { + mmap_read_unlock(mm); + return NULL; + } + + /* + * We can try to upgrade the mmap lock atomically, + * in which case we can continue to use the vma + * we already looked up. + * + * Otherwise we'll have to drop the mmap lock and + * re-take it, and also look up the vma again, + * re-checking it. + */ + if (!mmap_upgrade_trylock(mm)) { + if (!upgrade_mmap_lock_carefully(mm, is_user)) + return NULL; + + vma = find_vma(mm, addr); + if (!vma) + goto fail; + if (vma->vm_start <= addr) + goto success; + if (!(vma->vm_flags & VM_GROWSDOWN)) + goto fail; + } + + if (expand_stack_locked(vma, addr)) + goto fail; + +success: + mmap_write_downgrade(mm); + return vma; + +fail: + mmap_write_unlock(mm); + return NULL; +} + +/* + * Note this is constrained to return 0, -EFAULT, -EACCES, -ENOMEM by + * segv(). + */ +int handle_page_fault(unsigned long address, unsigned long ip, + int is_write, int is_user, int *code_out) +{ + struct mm_struct *mm = current->mm; + struct vm_area_struct *vma; + pmd_t *pmd; + pte_t *pte; + int err = -EFAULT; + unsigned int flags = FAULT_FLAG_DEFAULT; + + *code_out = SEGV_MAPERR; + + /* + * If the fault was with pagefaults disabled, don't take the fault, just + * fail. + */ + if (faulthandler_disabled()) + goto out_nosemaphore; + + if (is_user) + flags |= FAULT_FLAG_USER; +retry: + vma = um_lock_mm_and_find_vma(mm, address, is_user); + if (!vma) + goto out_nosemaphore; + + *code_out = SEGV_ACCERR; + if (is_write) { + if (!(vma->vm_flags & VM_WRITE)) + goto out; + flags |= FAULT_FLAG_WRITE; + } else { + /* Don't require VM_READ|VM_EXEC for write faults! */ + if (!(vma->vm_flags & (VM_READ | VM_EXEC))) + goto out; + } + + do { + vm_fault_t fault; + + fault = handle_mm_fault(vma, address, flags, NULL); + + if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) + goto out_nosemaphore; + + /* The fault is fully completed (including releasing mmap lock) */ + if (fault & VM_FAULT_COMPLETED) + return 0; + + if (unlikely(fault & VM_FAULT_ERROR)) { + if (fault & VM_FAULT_OOM) { + goto out_of_memory; + } else if (fault & VM_FAULT_SIGSEGV) { + goto out; + } else if (fault & VM_FAULT_SIGBUS) { + err = -EACCES; + goto out; + } + BUG(); + } + if (fault & VM_FAULT_RETRY) { + flags |= FAULT_FLAG_TRIED; + + goto retry; + } + + pmd = pmd_off(mm, address); + pte = pte_offset_kernel(pmd, address); + } while (!pte_present(*pte)); + err = 0; + /* + * The below warning was added in place of + * pte_mkyoung(); if (is_write) pte_mkdirty(); + * If it's triggered, we'd see normally a hang here (a clean pte is + * marked read-only to emulate the dirty bit). + * However, the generic code can mark a PTE writable but clean on a + * concurrent read fault, triggering this harmlessly. So comment it out. + */ +#if 0 + WARN_ON(!pte_young(*pte) || (is_write && !pte_dirty(*pte))); +#endif + +out: + mmap_read_unlock(mm); +out_nosemaphore: + return err; + +out_of_memory: + /* + * We ran out of memory, call the OOM killer, and return the userspace + * (which will retry the fault, or kill us if we got oom-killed). + */ + mmap_read_unlock(mm); + if (!is_user) + goto out_nosemaphore; + pagefault_out_of_memory(); + return 0; +} + diff --git a/arch/um/kernel/trap.c b/arch/um/kernel/trap.c index 177615820a4c..ab60bab8d987 100644 --- a/arch/um/kernel/trap.c +++ b/arch/um/kernel/trap.c @@ -17,225 +17,6 @@ #include <os.h> #include <skas.h> -/* - * NOTE: UML does not have exception tables. As such, this is almost a copy - * of the code in mm/memory.c, only adjusting the logic to simply check whether - * we are coming from the kernel instead of doing an additional lookup in the - * exception table. - * We can do this simplification because we never get here if the exception was - * fixable. - */ -static inline bool get_mmap_lock_carefully(struct mm_struct *mm, bool is_user) -{ - if (likely(mmap_read_trylock(mm))) - return true; - - if (!is_user) - return false; - - return !mmap_read_lock_killable(mm); -} - -static inline bool mmap_upgrade_trylock(struct mm_struct *mm) -{ - /* - * We don't have this operation yet. - * - * It should be easy enough to do: it's basically a - * atomic_long_try_cmpxchg_acquire() - * from RWSEM_READER_BIAS -> RWSEM_WRITER_LOCKED, but - * it also needs the proper lockdep magic etc. - */ - return false; -} - -static inline bool upgrade_mmap_lock_carefully(struct mm_struct *mm, bool is_user) -{ - mmap_read_unlock(mm); - if (!is_user) - return false; - - return !mmap_write_lock_killable(mm); -} - -/* - * Helper for page fault handling. - * - * This is kind of equivalend to "mmap_read_lock()" followed - * by "find_extend_vma()", except it's a lot more careful about - * the locking (and will drop the lock on failure). - * - * For example, if we have a kernel bug that causes a page - * fault, we don't want to just use mmap_read_lock() to get - * the mm lock, because that would deadlock if the bug were - * to happen while we're holding the mm lock for writing. - * - * So this checks the exception tables on kernel faults in - * order to only do this all for instructions that are actually - * expected to fault. - * - * We can also actually take the mm lock for writing if we - * need to extend the vma, which helps the VM layer a lot. - */ -static struct vm_area_struct * -um_lock_mm_and_find_vma(struct mm_struct *mm, - unsigned long addr, bool is_user) -{ - struct vm_area_struct *vma; - - if (!get_mmap_lock_carefully(mm, is_user)) - return NULL; - - vma = find_vma(mm, addr); - if (likely(vma && (vma->vm_start <= addr))) - return vma; - - /* - * Well, dang. We might still be successful, but only - * if we can extend a vma to do so. - */ - if (!vma || !(vma->vm_flags & VM_GROWSDOWN)) { - mmap_read_unlock(mm); - return NULL; - } - - /* - * We can try to upgrade the mmap lock atomically, - * in which case we can continue to use the vma - * we already looked up. - * - * Otherwise we'll have to drop the mmap lock and - * re-take it, and also look up the vma again, - * re-checking it. - */ - if (!mmap_upgrade_trylock(mm)) { - if (!upgrade_mmap_lock_carefully(mm, is_user)) - return NULL; - - vma = find_vma(mm, addr); - if (!vma) - goto fail; - if (vma->vm_start <= addr) - goto success; - if (!(vma->vm_flags & VM_GROWSDOWN)) - goto fail; - } - - if (expand_stack_locked(vma, addr)) - goto fail; - -success: - mmap_write_downgrade(mm); - return vma; - -fail: - mmap_write_unlock(mm); - return NULL; -} - -/* - * Note this is constrained to return 0, -EFAULT, -EACCES, -ENOMEM by - * segv(). - */ -int handle_page_fault(unsigned long address, unsigned long ip, - int is_write, int is_user, int *code_out) -{ - struct mm_struct *mm = current->mm; - struct vm_area_struct *vma; - pmd_t *pmd; - pte_t *pte; - int err = -EFAULT; - unsigned int flags = FAULT_FLAG_DEFAULT; - - *code_out = SEGV_MAPERR; - - /* - * If the fault was with pagefaults disabled, don't take the fault, just - * fail. - */ - if (faulthandler_disabled()) - goto out_nosemaphore; - - if (is_user) - flags |= FAULT_FLAG_USER; -retry: - vma = um_lock_mm_and_find_vma(mm, address, is_user); - if (!vma) - goto out_nosemaphore; - - *code_out = SEGV_ACCERR; - if (is_write) { - if (!(vma->vm_flags & VM_WRITE)) - goto out; - flags |= FAULT_FLAG_WRITE; - } else { - /* Don't require VM_READ|VM_EXEC for write faults! */ - if (!(vma->vm_flags & (VM_READ | VM_EXEC))) - goto out; - } - - do { - vm_fault_t fault; - - fault = handle_mm_fault(vma, address, flags, NULL); - - if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) - goto out_nosemaphore; - - /* The fault is fully completed (including releasing mmap lock) */ - if (fault & VM_FAULT_COMPLETED) - return 0; - - if (unlikely(fault & VM_FAULT_ERROR)) { - if (fault & VM_FAULT_OOM) { - goto out_of_memory; - } else if (fault & VM_FAULT_SIGSEGV) { - goto out; - } else if (fault & VM_FAULT_SIGBUS) { - err = -EACCES; - goto out; - } - BUG(); - } - if (fault & VM_FAULT_RETRY) { - flags |= FAULT_FLAG_TRIED; - - goto retry; - } - - pmd = pmd_off(mm, address); - pte = pte_offset_kernel(pmd, address); - } while (!pte_present(*pte)); - err = 0; - /* - * The below warning was added in place of - * pte_mkyoung(); if (is_write) pte_mkdirty(); - * If it's triggered, we'd see normally a hang here (a clean pte is - * marked read-only to emulate the dirty bit). - * However, the generic code can mark a PTE writable but clean on a - * concurrent read fault, triggering this harmlessly. So comment it out. - */ -#if 0 - WARN_ON(!pte_young(*pte) || (is_write && !pte_dirty(*pte))); -#endif - -out: - mmap_read_unlock(mm); -out_nosemaphore: - return err; - -out_of_memory: - /* - * We ran out of memory, call the OOM killer, and return the userspace - * (which will retry the fault, or kill us if we got oom-killed). - */ - mmap_read_unlock(mm); - if (!is_user) - goto out_nosemaphore; - pagefault_out_of_memory(); - return 0; -} - static void show_segv_info(struct uml_pt_regs *regs) { struct task_struct *tsk = current; -- 2.53.0
