From: Nick Piggin <[EMAIL PROTECTED]>
Provide a lockless pagetable walk function without fallback to mmap_sem
on error.
Index: kvm/arch/x86/mm/gup.c
===================================================================
--- kvm.orig/arch/x86/mm/gup.c
+++ kvm/arch/x86/mm/gup.c
@@ -8,6 +8,7 @@
#include <linux/mm.h>
#include <linux/vmstat.h>
#include <linux/highmem.h>
+#include <linux/module.h>
#include <asm/pgtable.h>
@@ -219,7 +220,7 @@ static int gup_pud_range(pgd_t pgd, unsi
return 1;
}
-int get_user_pages_fast(unsigned long start, int nr_pages, int write,
+int get_user_pages_fast_atomic(unsigned long start, int nr_pages, int write,
struct page **pages)
{
struct mm_struct *mm = current->mm;
@@ -234,7 +235,7 @@ int get_user_pages_fast(unsigned long st
end = start + len;
if (unlikely(!access_ok(write ? VERIFY_WRITE : VERIFY_READ,
start, len)))
- goto slow_irqon;
+ return -EFAULT;
/*
* XXX: batch / limit 'nr', to avoid large irq off latency
@@ -261,38 +262,49 @@ int get_user_pages_fast(unsigned long st
next = pgd_addr_end(addr, end);
if (pgd_none(pgd))
- goto slow;
+ goto out_short;
if (!gup_pud_range(pgd, addr, next, write, pages, &nr))
- goto slow;
+ goto out_short;
} while (pgdp++, addr = next, addr != end);
- local_irq_enable();
VM_BUG_ON(nr != (end - start) >> PAGE_SHIFT);
+out_short:
+ local_irq_enable();
return nr;
+}
- {
- int ret;
-
-slow:
- local_irq_enable();
-slow_irqon:
- /* Try to get the remaining pages with get_user_pages */
- start += nr << PAGE_SHIFT;
- pages += nr;
-
- down_read(&mm->mmap_sem);
- ret = get_user_pages(current, mm, start,
- (end - start) >> PAGE_SHIFT, write, 0, pages, NULL);
- up_read(&mm->mmap_sem);
-
- /* Have to be a bit careful with return values */
- if (nr > 0) {
- if (ret < 0)
- ret = nr;
- else
- ret += nr;
- }
+int get_user_pages_fast(unsigned long start, int nr_pages, int write,
+ struct page **pages)
+{
+ struct mm_struct *mm = current->mm;
+ int nr = 0;
+ int ret;
- return ret;
+ nr = get_user_pages_fast_atomic(start, nr_pages, write, pages);
+ if (likely(nr == nr_pages))
+ return nr;
+
+ if (unlikely(nr < 0))
+ return nr;
+
+ start += nr << PAGE_SHIFT;
+ pages += nr;
+ nr_pages -= nr;
+
+ down_read(&mm->mmap_sem);
+ ret = get_user_pages(current, mm, start,
+ nr_pages, write, 0, pages, NULL);
+ up_read(&mm->mmap_sem);
+
+ /* Have to be a bit careful with return values */
+ if (nr > 0) {
+ if (ret < 0)
+ ret = nr;
+ else
+ ret += nr;
}
+
+ return ret;
}
+
+EXPORT_SYMBOL_GPL(get_user_pages_fast_atomic);
Index: kvm/include/asm-x86/uaccess.h
===================================================================
--- kvm.orig/include/asm-x86/uaccess.h
+++ kvm/include/asm-x86/uaccess.h
@@ -443,6 +443,9 @@ extern struct movsl_mask {
#define ARCH_HAS_NOCACHE_UACCESS 1
+int get_user_pages_fast_atomic(unsigned long start, int nr_pages, int write,
+ struct page **pages);
+
#ifdef CONFIG_X86_32
# include "uaccess_32.h"
#else
--
--
To unsubscribe from this list: send the line "unsubscribe kvm" in
the body of a message to [EMAIL PROTECTED]
More majordomo info at http://vger.kernel.org/majordomo-info.html
