Vulkan sparse residency requires that an access to an unbound part of a sparse resource does not fault, so a range has to be mappable without naming any buffer object.
Add DRM_PVR_VM_BIND_OP_MAP_SPARSE, which maps one dummy object over and over across the range. To save memory, one dummy is shared by all sparse mappings, but per file rather than per device: writes to a sparse range land in the dummy, so per file keeps clients isolated from each other. The dummy is 2MiB, so pvr_mmu_map() runs once per 2MiB, but the mapping itself is still done per page. Signed-off-by: Gyeyoung Baek <[email protected]> --- drivers/gpu/drm/imagination/pvr_device.h | 10 ++++ drivers/gpu/drm/imagination/pvr_drv.c | 63 +++++++++++++++++++++++-- drivers/gpu/drm/imagination/pvr_vm.c | 81 +++++++++++++++++++++++++++++--- drivers/gpu/drm/imagination/pvr_vm.h | 8 ++++ include/uapi/drm/pvr_drm.h | 21 +++++++-- 5 files changed, 169 insertions(+), 14 deletions(-) diff --git a/drivers/gpu/drm/imagination/pvr_device.h b/drivers/gpu/drm/imagination/pvr_device.h index c246122b4ca..c8a7a949fdc 100644 --- a/drivers/gpu/drm/imagination/pvr_device.h +++ b/drivers/gpu/drm/imagination/pvr_device.h @@ -380,6 +380,16 @@ struct pvr_file { */ struct xarray hwrt_handles; + /** + * @sparse_dummy_bo: Object the file's sparse mappings repeat. Pins its + * pages, so it is created on first use, under @sparse_dummy_lock. Per + * file, so that sparse writes stay out of other clients' view. + */ + struct pvr_gem_object *sparse_dummy_bo; + + /** @sparse_dummy_lock: Serialises the creation of @sparse_dummy_bo. */ + struct mutex sparse_dummy_lock; + /** * @vm_ctx_handles: Array of VM contexts belonging to this file. Array * members are of type "struct pvr_vm_context *". diff --git a/drivers/gpu/drm/imagination/pvr_drv.c b/drivers/gpu/drm/imagination/pvr_drv.c index c35d437aef7..377fa8d936a 100644 --- a/drivers/gpu/drm/imagination/pvr_drv.c +++ b/drivers/gpu/drm/imagination/pvr_drv.c @@ -1137,6 +1137,37 @@ pvr_ioctl_vm_unmap(struct drm_device *drm_dev, void *raw_args, return err; } +/** + * pvr_file_get_sparse_dummy() - Get a file's sparse dummy object. + * @pvr_file: Target PowerVR file. + * + * Return: The object, with a reference for the caller, or an ERR_PTR on + * failure. + */ +static struct pvr_gem_object * +pvr_file_get_sparse_dummy(struct pvr_file *pvr_file) +{ + struct pvr_gem_object *dummy; + + mutex_lock(&pvr_file->sparse_dummy_lock); + + dummy = pvr_file->sparse_dummy_bo; + if (!dummy) { + dummy = pvr_vm_sparse_dummy_create(pvr_file->pvr_dev); + if (IS_ERR(dummy)) + goto err_unlock; + + pvr_file->sparse_dummy_bo = dummy; + } + + pvr_gem_object_get(dummy); + +err_unlock: + mutex_unlock(&pvr_file->sparse_dummy_lock); + + return dummy; +} + /** * pvr_vm_bind_desc_from_uapi() - Translate a userspace bind operation. * @desc: Description to fill in. @@ -1150,6 +1181,8 @@ pvr_vm_bind_desc_from_uapi(struct pvr_vm_bind_desc *desc, struct pvr_file *pvr_file, const struct drm_pvr_vm_bind_op *uapi_op) { + const bool sparse = uapi_op->flags & DRM_PVR_VM_BIND_OP_MAP_SPARSE; + if (uapi_op->flags & ~DRM_PVR_VM_BIND_OP_FLAGS_MASK) return -EINVAL; @@ -1157,19 +1190,35 @@ pvr_vm_bind_desc_from_uapi(struct pvr_vm_bind_desc *desc, .offset = uapi_op->offset, .device_addr = uapi_op->device_addr, .size = uapi_op->size, + .sparse = sparse, }; switch (uapi_op->flags & DRM_PVR_VM_BIND_OP_TYPE_MASK) { case DRM_PVR_VM_BIND_OP_TYPE_MAP: desc->type = PVR_VM_BIND_TYPE_MAP; - desc->obj = pvr_gem_object_from_handle(pvr_file, uapi_op->handle); - if (!desc->obj) - return -ENOENT; + + if (sparse) { + struct pvr_gem_object *dummy; + + if (uapi_op->handle || uapi_op->offset) + return -EINVAL; + + dummy = pvr_file_get_sparse_dummy(pvr_file); + if (IS_ERR(dummy)) + return PTR_ERR(dummy); + + desc->obj = dummy; + } else { + desc->obj = pvr_gem_object_from_handle(pvr_file, + uapi_op->handle); + if (!desc->obj) + return -ENOENT; + } return 0; case DRM_PVR_VM_BIND_OP_TYPE_UNMAP: - if (uapi_op->handle || uapi_op->offset) + if (uapi_op->handle || uapi_op->offset || sparse) return -EINVAL; desc->type = PVR_VM_BIND_TYPE_UNMAP; @@ -1615,6 +1664,8 @@ pvr_drm_driver_open(struct drm_device *drm_dev, struct drm_file *file) INIT_LIST_HEAD(&pvr_file->contexts); + mutex_init(&pvr_file->sparse_dummy_lock); + xa_init_flags(&pvr_file->ctx_handles, XA_FLAGS_ALLOC1); xa_init_flags(&pvr_file->free_list_handles, XA_FLAGS_ALLOC1); xa_init_flags(&pvr_file->hwrt_handles, XA_FLAGS_ALLOC1); @@ -1653,6 +1704,10 @@ pvr_drm_driver_postclose(__always_unused struct drm_device *drm_dev, pvr_destroy_hwrt_datasets_for_file(pvr_file); pvr_destroy_vm_contexts_for_file(pvr_file); + if (pvr_file->sparse_dummy_bo) + pvr_gem_object_put(pvr_file->sparse_dummy_bo); + mutex_destroy(&pvr_file->sparse_dummy_lock); + kfree(pvr_file); file->driver_priv = NULL; } diff --git a/drivers/gpu/drm/imagination/pvr_vm.c b/drivers/gpu/drm/imagination/pvr_vm.c index 4e58ea8c50c..0c22c63e498 100644 --- a/drivers/gpu/drm/imagination/pvr_vm.c +++ b/drivers/gpu/drm/imagination/pvr_vm.c @@ -22,8 +22,10 @@ #include <linux/errno.h> #include <linux/gfp_types.h> #include <linux/kref.h> +#include <linux/minmax.h> #include <linux/mutex.h> #include <linux/sched.h> +#include <linux/sizes.h> #include <linux/slab.h> #include <linux/stddef.h> #include <linux/workqueue.h> @@ -188,6 +190,9 @@ struct pvr_vm_bind_op { */ struct pvr_vm_gpuva *next_va; + /** @sparse: See &pvr_vm_bind_desc.sparse. */ + bool sparse; + /** @offset: Offset into @pvr_obj to begin mapping from. */ u64 offset; @@ -280,11 +285,19 @@ pvr_vm_bind_op_map_init(struct pvr_vm_bind_op *bind_op, if (!pvr_device_addr_and_size_are_valid(vm_ctx, desc->device_addr, desc->size) || - desc->offset & ~PAGE_MASK || desc->size & ~PAGE_MASK || - desc->offset >= pvr_obj_size || offset_plus_size > pvr_obj_size) + desc->offset & ~PAGE_MASK || desc->size & ~PAGE_MASK) + return -EINVAL; + + /* + * A sparse mapping repeats @obj over a range wider than it, so only + * ordinary mappings must fit within @obj. + */ + if (!desc->sparse && + (desc->offset >= pvr_obj_size || offset_plus_size > pvr_obj_size)) return -EINVAL; bind_op->type = PVR_VM_BIND_TYPE_MAP; + bind_op->sparse = desc->sparse; bind_op->gpuvm_bo = drm_gpuvm_bo_create(&vm_ctx->gpuvm_mgr, obj); if (!bind_op->gpuvm_bo) @@ -373,6 +386,58 @@ pvr_vm_bind_op_unmap_init(struct pvr_vm_bind_op *bind_op, return err; } +/** + * pvr_vm_sparse_dummy_create() - Create the object sparse mappings repeat. + * @pvr_dev: Target PowerVR device. + * + * It covers a whole level 0 page table, so that each chunk mapped by + * pvr_vm_mmu_map_sparse() fills one. + * + * Return: The new object, or an ERR_PTR on failure. + */ +struct pvr_gem_object *pvr_vm_sparse_dummy_create(struct pvr_device *pvr_dev) +{ + return pvr_gem_object_create(pvr_dev, SZ_2M, + DRM_PVR_BO_BYPASS_DEVICE_CACHE); +} + +/** + * pvr_vm_mmu_map_sparse() - Map a dummy object over and over across a range. + * @mmu_op_ctx: Target MMU op context, created for @dummy. + * @dummy: Object to repeat. + * @device_addr: Virtual device address at the start of the range. + * @size: Size of the range. + * + * Maps @dummy repeatedly, in chunks aligned to its size, which must be a power + * of two. + * + * Return: 0 on success, or a negative error code with nothing left mapped. + */ +static int +pvr_vm_mmu_map_sparse(struct pvr_mmu_op_context *mmu_op_ctx, + struct pvr_gem_object *dummy, u64 device_addr, u64 size) +{ + const u64 dummy_size = pvr_gem_object_size(dummy); + u64 mapped = 0; + int err; + + while (mapped < size) { + const u64 addr = device_addr + mapped; + const u64 chunk = min(size - mapped, + dummy_size - (addr & (dummy_size - 1))); + + err = pvr_mmu_map(mmu_op_ctx, chunk, dummy->flags, addr); + if (err) { + pvr_mmu_unmap(mmu_op_ctx, device_addr, mapped); + return err; + } + + mapped += chunk; + } + + return 0; +} + /** * pvr_vm_gpuva_map() - Insert a mapping into a memory context. * @op: gpuva op containing the remap details. @@ -383,7 +448,7 @@ pvr_vm_bind_op_unmap_init(struct pvr_vm_bind_op *bind_op, * * Return: * * 0 on success, or - * * Any error returned by pvr_mmu_map(). + * * Any error returned by pvr_mmu_map() or pvr_vm_mmu_map_sparse(). */ static int pvr_vm_gpuva_map(struct drm_gpuva_op *op, void *op_ctx) @@ -395,8 +460,12 @@ pvr_vm_gpuva_map(struct drm_gpuva_op *op, void *op_ctx) if ((op->map.gem.offset | op->map.va.range) & ~PVR_DEVICE_PAGE_MASK) return -EINVAL; - err = pvr_mmu_map(ctx->mmu_op_ctx, op->map.va.range, pvr_gem->flags, - op->map.va.addr); + if (ctx->sparse) + err = pvr_vm_mmu_map_sparse(ctx->mmu_op_ctx, pvr_gem, + op->map.va.addr, op->map.va.range); + else + err = pvr_mmu_map(ctx->mmu_op_ctx, op->map.va.range, + pvr_gem->flags, op->map.va.addr); if (err) return err; @@ -741,7 +810,7 @@ pvr_vm_bind_op_init(struct pvr_vm_bind_op *bind_op, return pvr_vm_bind_op_map_init(bind_op, vm_ctx, desc); case PVR_VM_BIND_TYPE_UNMAP: - if (WARN_ON(desc->obj)) + if (WARN_ON(desc->obj || desc->sparse)) return -EINVAL; return pvr_vm_bind_op_unmap_init(bind_op, vm_ctx, desc); diff --git a/drivers/gpu/drm/imagination/pvr_vm.h b/drivers/gpu/drm/imagination/pvr_vm.h index df74b405960..68d5ffa5e88 100644 --- a/drivers/gpu/drm/imagination/pvr_vm.h +++ b/drivers/gpu/drm/imagination/pvr_vm.h @@ -80,6 +80,12 @@ struct pvr_vm_bind_desc { /** @size: Size of the range. */ u64 size; + + /** + * @sparse: Repeat @obj over the range rather than covering it once. + * Map only. + */ + bool sparse; }; int pvr_vm_bind(struct pvr_vm_context *vm_ctx, @@ -93,6 +99,8 @@ void pvr_vm_unmap_all(struct pvr_vm_context *vm_ctx); bool pvr_vm_context_is_unusable(struct pvr_vm_context *vm_ctx); +struct pvr_gem_object *pvr_vm_sparse_dummy_create(struct pvr_device *pvr_dev); + dma_addr_t pvr_vm_get_page_table_root_addr(struct pvr_vm_context *vm_ctx); struct dma_resv *pvr_vm_get_dma_resv(struct pvr_vm_context *vm_ctx); diff --git a/include/uapi/drm/pvr_drm.h b/include/uapi/drm/pvr_drm.h index 57c22f5f215..a7c2801cc70 100644 --- a/include/uapi/drm/pvr_drm.h +++ b/include/uapi/drm/pvr_drm.h @@ -1333,7 +1333,17 @@ struct drm_pvr_ioctl_submit_jobs_args { /** * DOC: Flags for VM_BIND operations. * - * Set exactly one operation type in &drm_pvr_vm_bind_op.flags. + * Set exactly one operation type in &drm_pvr_vm_bind_op.flags, optionally + * ORed with modifiers. + * + * .. c:macro:: DRM_PVR_VM_BIND_OP_MAP_SPARSE + * + * Sparsely map a range without naming a buffer object. Only valid with + * %DRM_PVR_VM_BIND_OP_TYPE_MAP. + * + * GPU accesses to the range do not fault. Nothing else is guaranteed: + * reads return undefined values, and writes may become visible through + * other sparse mappings. * * .. c:macro:: DRM_PVR_VM_BIND_OP_TYPE_MAP * @@ -1349,11 +1359,14 @@ struct drm_pvr_ioctl_submit_jobs_args { * * Mask used to extract the operation type. */ +#define DRM_PVR_VM_BIND_OP_MAP_SPARSE _BITUL(0) + #define DRM_PVR_VM_BIND_OP_TYPE_MAP (0u << 28) #define DRM_PVR_VM_BIND_OP_TYPE_UNMAP (1u << 28) #define DRM_PVR_VM_BIND_OP_TYPE_MASK (0xfu << 28) -#define DRM_PVR_VM_BIND_OP_FLAGS_MASK DRM_PVR_VM_BIND_OP_TYPE_MASK +#define DRM_PVR_VM_BIND_OP_FLAGS_MASK \ + (DRM_PVR_VM_BIND_OP_MAP_SPARSE | DRM_PVR_VM_BIND_OP_TYPE_MASK) /** * struct drm_pvr_vm_bind_op - A single VM bind operation. @@ -1366,13 +1379,13 @@ struct drm_pvr_vm_bind_op { * @handle: [IN] Handle of the target buffer object. * * Must be a valid handle returned by %DRM_IOCTL_PVR_CREATE_BO for map - * operations. MBZ for unmap operations. + * operations. MBZ for unmap and sparse map operations. */ __u32 handle; /** * @offset: [IN] Offset into the target buffer object from which to - * begin the mapping. MBZ for unmap operations. + * begin the mapping. MBZ for unmap and sparse map operations. */ __u64 offset; -- 2.43.0
