Split drm_pagemap_migrate_map_pages into device / system helpers clearly seperating these operations. Will help with upcoming changes to split IOVA allocation steps.
Signed-off-by: Matthew Brost <[email protected]> --- drivers/gpu/drm/drm_pagemap.c | 146 ++++++++++++++++++++++------------ 1 file changed, 96 insertions(+), 50 deletions(-) diff --git a/drivers/gpu/drm/drm_pagemap.c b/drivers/gpu/drm/drm_pagemap.c index 2c67aabd8d65..4b79d4019453 100644 --- a/drivers/gpu/drm/drm_pagemap.c +++ b/drivers/gpu/drm/drm_pagemap.c @@ -205,7 +205,7 @@ static void drm_pagemap_get_devmem_page(struct page *page, } /** - * drm_pagemap_migrate_map_pages() - Map migration pages for GPU SVM migration + * drm_pagemap_migrate_map_device_pages() - Map device migration pages for GPU SVM migration * @dev: The device performing the migration. * @local_dpagemap: The drm_pagemap local to the migrating device. * @pagemap_addr: Array to store DMA information corresponding to mapped pages. @@ -221,19 +221,22 @@ static void drm_pagemap_get_devmem_page(struct page *page, * * Returns: 0 on success, -EFAULT if an error occurs during mapping. */ -static int drm_pagemap_migrate_map_pages(struct device *dev, - struct drm_pagemap *local_dpagemap, - struct drm_pagemap_addr *pagemap_addr, - unsigned long *migrate_pfn, - unsigned long npages, - enum dma_data_direction dir, - const struct drm_pagemap_migrate_details *mdetails) +static int +drm_pagemap_migrate_map_device_pages(struct device *dev, + struct drm_pagemap *local_dpagemap, + struct drm_pagemap_addr *pagemap_addr, + unsigned long *migrate_pfn, + unsigned long npages, + enum dma_data_direction dir, + const struct drm_pagemap_migrate_details *mdetails) { unsigned long num_peer_pages = 0, num_local_pages = 0, i; for (i = 0; i < npages;) { struct page *page = migrate_pfn_to_page(migrate_pfn[i]); - dma_addr_t dma_addr; + struct drm_pagemap_zdd *zdd; + struct drm_pagemap *dpagemap; + struct drm_pagemap_addr addr; struct folio *folio; unsigned int order = 0; @@ -243,36 +246,26 @@ static int drm_pagemap_migrate_map_pages(struct device *dev, folio = page_folio(page); order = folio_order(folio); - if (is_device_private_page(page)) { - struct drm_pagemap_zdd *zdd = drm_pagemap_page_zone_device_data(page); - struct drm_pagemap *dpagemap = zdd->dpagemap; - struct drm_pagemap_addr addr; - - if (dpagemap == local_dpagemap) { - if (!mdetails->can_migrate_same_pagemap) - goto next; + WARN_ON_ONCE(!is_device_private_page(page)); - num_local_pages += NR_PAGES(order); - } else { - num_peer_pages += NR_PAGES(order); - } + zdd = drm_pagemap_page_zone_device_data(page); + dpagemap = zdd->dpagemap; - addr = dpagemap->ops->device_map(dpagemap, dev, page, order, dir); - if (dma_mapping_error(dev, addr.addr)) - return -EFAULT; + if (dpagemap == local_dpagemap) { + if (!mdetails->can_migrate_same_pagemap) + goto next; - pagemap_addr[i] = addr; + num_local_pages += NR_PAGES(order); } else { - dma_addr = dma_map_page(dev, page, 0, page_size(page), dir); - if (dma_mapping_error(dev, dma_addr)) - return -EFAULT; - - pagemap_addr[i] = - drm_pagemap_addr_encode(dma_addr, - DRM_INTERCONNECT_SYSTEM, - order, dir); + num_peer_pages += NR_PAGES(order); } + addr = dpagemap->ops->device_map(dpagemap, dev, page, order, dir); + if (dma_mapping_error(dev, addr.addr)) + return -EFAULT; + + pagemap_addr[i] = addr; + next: i += NR_PAGES(order); } @@ -287,6 +280,59 @@ static int drm_pagemap_migrate_map_pages(struct device *dev, return 0; } +/** + * drm_pagemap_migrate_map_system_pages() - Map system migration pages for GPU SVM migration + * @dev: The device performing the migration. + * @pagemap_addr: Array to store DMA information corresponding to mapped pages. + * @migrate_pfn: Array of page frame numbers of system pages or peer pages to map. + * @npages: Number of system pages or peer pages to map. + * @dir: Direction of data transfer (e.g., DMA_BIDIRECTIONAL) + * + * This function maps pages of memory for migration usage in GPU SVM. It + * iterates over each page frame number provided in @migrate_pfn, maps the + * corresponding page, and stores the DMA address in the provided @dma_addr + * array. + * + * Returns: 0 on success, -EFAULT if an error occurs during mapping. + */ +static int +drm_pagemap_migrate_map_system_pages(struct device *dev, + struct drm_pagemap_addr *pagemap_addr, + unsigned long *migrate_pfn, + unsigned long npages, + enum dma_data_direction dir) +{ + unsigned long i; + + for (i = 0; i < npages;) { + struct page *page = migrate_pfn_to_page(migrate_pfn[i]); + dma_addr_t dma_addr; + struct folio *folio; + unsigned int order = 0; + + if (!page) + goto next; + + WARN_ON_ONCE(is_device_private_page(page)); + folio = page_folio(page); + order = folio_order(folio); + + dma_addr = dma_map_page(dev, page, 0, page_size(page), dir); + if (dma_mapping_error(dev, dma_addr)) + return -EFAULT; + + pagemap_addr[i] = + drm_pagemap_addr_encode(dma_addr, + DRM_INTERCONNECT_SYSTEM, + order, dir); + +next: + i += NR_PAGES(order); + } + + return 0; +} + /** * drm_pagemap_migrate_unmap_pages() - Unmap pages previously mapped for GPU SVM migration * @dev: The device for which the pages were mapped @@ -347,9 +393,11 @@ drm_pagemap_migrate_remote_to_local(struct drm_pagemap_devmem *devmem, const struct drm_pagemap_migrate_details *mdetails) { - int err = drm_pagemap_migrate_map_pages(remote_device, remote_dpagemap, - pagemap_addr, local_pfns, - npages, DMA_FROM_DEVICE, mdetails); + int err = drm_pagemap_migrate_map_device_pages(remote_device, + remote_dpagemap, + pagemap_addr, local_pfns, + npages, DMA_FROM_DEVICE, + mdetails); if (err) goto out; @@ -368,12 +416,11 @@ drm_pagemap_migrate_sys_to_dev(struct drm_pagemap_devmem *devmem, struct page *local_pages[], struct drm_pagemap_addr pagemap_addr[], unsigned long npages, - const struct drm_pagemap_devmem_ops *ops, - const struct drm_pagemap_migrate_details *mdetails) + const struct drm_pagemap_devmem_ops *ops) { - int err = drm_pagemap_migrate_map_pages(devmem->dev, devmem->dpagemap, - pagemap_addr, sys_pfns, npages, - DMA_TO_DEVICE, mdetails); + int err = drm_pagemap_migrate_map_system_pages(devmem->dev, + pagemap_addr, sys_pfns, + npages, DMA_TO_DEVICE); if (err) goto out; @@ -437,7 +484,7 @@ static int drm_pagemap_migrate_range(struct drm_pagemap_devmem *devmem, &pages[last->start], &pagemap_addr[last->start], cur->start - last->start, - last->ops, mdetails); + last->ops); out: *last = *cur; @@ -954,7 +1001,6 @@ EXPORT_SYMBOL(drm_pagemap_put); int drm_pagemap_evict_to_ram(struct drm_pagemap_devmem *devmem_allocation) { const struct drm_pagemap_devmem_ops *ops = devmem_allocation->ops; - struct drm_pagemap_migrate_details mdetails = {}; unsigned long npages, mpages = 0; struct page **pages; unsigned long *src, *dst; @@ -993,10 +1039,10 @@ int drm_pagemap_evict_to_ram(struct drm_pagemap_devmem *devmem_allocation) if (err || !mpages) goto err_finalize; - err = drm_pagemap_migrate_map_pages(devmem_allocation->dev, - devmem_allocation->dpagemap, pagemap_addr, - dst, npages, DMA_FROM_DEVICE, - &mdetails); + err = drm_pagemap_migrate_map_system_pages(devmem_allocation->dev, + pagemap_addr, + dst, npages, + DMA_FROM_DEVICE); if (err) goto err_finalize; @@ -1057,7 +1103,6 @@ static int __drm_pagemap_migrate_to_ram(struct vm_area_struct *vas, MIGRATE_VMA_SELECT_DEVICE_COHERENT, .fault_page = page, }; - struct drm_pagemap_migrate_details mdetails = {}; struct drm_pagemap_zdd *zdd; const struct drm_pagemap_devmem_ops *ops; struct device *dev = NULL; @@ -1115,8 +1160,9 @@ static int __drm_pagemap_migrate_to_ram(struct vm_area_struct *vas, if (err) goto err_finalize; - err = drm_pagemap_migrate_map_pages(dev, zdd->dpagemap, pagemap_addr, migrate.dst, npages, - DMA_FROM_DEVICE, &mdetails); + err = drm_pagemap_migrate_map_system_pages(dev, pagemap_addr, + migrate.dst, npages, + DMA_FROM_DEVICE); if (err) goto err_finalize; -- 2.34.1
