Check whether the starting address in the fault-around handler or the faulty address in the fault handler is part of a huge page in order to attempt a PMD sized PFN insertion into the VMA.
On builds with CONFIG_TRANSPARENT_HUGEPAGE enabled, if the mmap() user address is PMD size aligned, if the GEM object is backed by shmem buffers on mount points setting the 'huge=' option and if the shmem backing store manages to allocate a huge folio, CPU mapping would then benefit from significantly increased memcpy() performance. When these conditions are met on a system with 2 MiB huge pages, an aligned copy of 2 MiB would raise a single page fault instead of 4096. v4: - Implement map_pages instead of huge_fault Signed-off-by: Loïc Molinari <[email protected]> --- drivers/gpu/drm/drm_gem_shmem_helper.c | 57 +++++++++++++++++++++----- 1 file changed, 46 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c index e151262332f9..5d11bea573fe 100644 --- a/drivers/gpu/drm/drm_gem_shmem_helper.c +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c @@ -580,6 +580,26 @@ static bool drm_gem_shmem_fault_is_valid(struct drm_gem_object *obj, return true; } +static bool drm_gem_shmem_map_pmd(struct vm_fault *vmf, unsigned long addr, + struct page *page) +{ +#ifdef CONFIG_ARCH_SUPPORTS_PMD_PFNMAP + unsigned long pfn = page_to_pfn(page); + unsigned long paddr = pfn << PAGE_SHIFT; + bool aligned = (addr & ~PMD_MASK) == (paddr & ~PMD_MASK); + + if (aligned && + pmd_none(*vmf->pmd) && + folio_test_pmd_mappable(page_folio(page))) { + pfn &= PMD_MASK >> PAGE_SHIFT; + if (vmf_insert_pfn_pmd(vmf, pfn, false) == VM_FAULT_NOPAGE) + return true; + } +#endif + + return false; +} + static vm_fault_t drm_gem_shmem_map_pages(struct vm_fault *vmf, pgoff_t start_pgoff, pgoff_t end_pgoff) @@ -587,6 +607,7 @@ static vm_fault_t drm_gem_shmem_map_pages(struct vm_fault *vmf, struct vm_area_struct *vma = vmf->vma; struct drm_gem_object *obj = vma->vm_private_data; struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj); + struct page **pages = shmem->pages; unsigned long addr, pfn; vm_fault_t ret; @@ -598,15 +619,22 @@ static vm_fault_t drm_gem_shmem_map_pages(struct vm_fault *vmf, if (unlikely(!drm_gem_shmem_fault_is_valid(obj, start_pgoff))) { ret = VM_FAULT_SIGBUS; - } else { - /* Map a range of pages around the faulty address. */ - do { - pfn = page_to_pfn(shmem->pages[start_pgoff]); - ret = vmf_insert_pfn(vma, addr, pfn); - addr += PAGE_SIZE; - } while (++start_pgoff <= end_pgoff && ret == VM_FAULT_NOPAGE); + goto out; } + if (drm_gem_shmem_map_pmd(vmf, addr, pages[start_pgoff])) { + ret = VM_FAULT_NOPAGE; + goto out; + } + + /* Map a range of pages around the faulty address. */ + do { + pfn = page_to_pfn(pages[start_pgoff]); + ret = vmf_insert_pfn(vma, addr, pfn); + addr += PAGE_SIZE; + } while (++start_pgoff <= end_pgoff && ret == VM_FAULT_NOPAGE); + + out: dma_resv_unlock(shmem->base.resv); return ret; @@ -617,8 +645,9 @@ static vm_fault_t drm_gem_shmem_fault(struct vm_fault *vmf) struct vm_area_struct *vma = vmf->vma; struct drm_gem_object *obj = vma->vm_private_data; struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj); - struct page *page; + struct page **pages = shmem->pages; pgoff_t page_offset; + unsigned long pfn; vm_fault_t ret; /* Offset to faulty address in the VMA (without the fake offset). */ @@ -628,12 +657,18 @@ static vm_fault_t drm_gem_shmem_fault(struct vm_fault *vmf) if (unlikely(!drm_gem_shmem_fault_is_valid(obj, page_offset))) { ret = VM_FAULT_SIGBUS; - } else { - page = shmem->pages[page_offset]; + goto out; + } - ret = vmf_insert_pfn(vma, vmf->address, page_to_pfn(page)); + if (drm_gem_shmem_map_pmd(vmf, vmf->address, pages[page_offset])) { + ret = VM_FAULT_NOPAGE; + goto out; } + pfn = page_to_pfn(pages[page_offset]); + ret = vmf_insert_pfn(vma, vmf->address, pfn); + + out: dma_resv_unlock(shmem->base.resv); return ret; -- 2.47.3
