From: Honglei Huang <honglei1.hu...@amd.com>
Add mmu notifier, there are some benefits:
- UMD do not need manage the userptrs, just alloc and free user space
memory, with the MMU notifier userpters can be managed by kernel.
- Can achieve a performance improvement of 20%~30%. With the MMU notifier
UMD like OpenCL can achieve 98% performance compared to bare metal in
some bench marks like Geekbench and CLpeak.
Signed-off-by: Honglei Huang <honglei1.hu...@amd.com>
---
drivers/gpu/drm/virtio/virtgpu_drv.h | 47 ++-
drivers/gpu/drm/virtio/virtgpu_ioctl.c | 4 +-
drivers/gpu/drm/virtio/virtgpu_kms.c | 2 +
drivers/gpu/drm/virtio/virtgpu_userptr.c | 423 ++++++++++++++++++++++-
4 files changed, 469 insertions(+), 7 deletions(-)
diff --git a/drivers/gpu/drm/virtio/virtgpu_drv.h
b/drivers/gpu/drm/virtio/virtgpu_drv.h
index fa5dd46e3732..6fa6dd9d1738 100644
--- a/drivers/gpu/drm/virtio/virtgpu_drv.h
+++ b/drivers/gpu/drm/virtio/virtgpu_drv.h
@@ -42,6 +42,7 @@
#include <drm/drm_ioctl.h>
#include <drm/drm_probe_helper.h>
#include <drm/virtgpu_drm.h>
+#include <linux/mmu_notifier.h>
#define DRIVER_NAME "virtio_gpu"
#define DRIVER_DESC "virtio GPU"
@@ -121,9 +122,33 @@ struct virtio_gpu_object_userptr_ops {
int (*get_pages)(struct virtio_gpu_object_userptr *userptr);
void (*put_pages)(struct virtio_gpu_object_userptr *userptr);
void (*release)(struct virtio_gpu_object_userptr *userptr);
- int (*insert)(struct virtio_gpu_object_userptr *userptr, struct
virtio_gpu_fpriv *fpriv);
- int (*remove)(struct virtio_gpu_object_userptr *userptr, struct
virtio_gpu_fpriv *fpriv);
+ int (*insert)(struct virtio_gpu_object_userptr *userptr,
+ struct virtio_gpu_fpriv *fpriv);
+ int (*remove)(struct virtio_gpu_object_userptr *userptr,
+ struct virtio_gpu_fpriv *fpriv);
+ bool (*valid)(struct virtio_gpu_object_userptr *userptr);
+ void (*notifier_init)(struct virtio_gpu_object_userptr *userptr,
+ struct mm_struct *mm);
+ int (*notifier_add)(struct virtio_gpu_object_userptr *userptr,
+ unsigned long start, unsigned long length);
+ void (*notifier_remove)(struct virtio_gpu_object_userptr *userptr);
+ int (*split)(struct virtio_gpu_object_userptr *userptr,
+ unsigned long start, unsigned long last,
+ struct virtio_gpu_object_userptr **pnew);
+ void (*evict)(struct virtio_gpu_object_userptr *userptr);
+ void (*update)(struct virtio_gpu_object_userptr *userptr);
+ struct virtio_gpu_object_userptr *(*split_new)(
+ struct virtio_gpu_object_userptr *userptr, unsigned long start,
+ unsigned long last);
};
+
+enum userptr_work_list_ops {
+ USERPTR_OP_NULL,
+ USERPTR_OP_UNMAP,
+ USERPTR_OP_UPDATE,
+ USERPTR_OP_EVICT,
+};
+
struct virtio_gpu_object_userptr {
struct virtio_gpu_object base;
const struct virtio_gpu_object_userptr_ops *ops;
@@ -142,6 +167,16 @@ struct virtio_gpu_object_userptr {
struct sg_table *sgt;
struct interval_tree_node it_node;
+
+#ifdef CONFIG_MMU_NOTIFIER
+ struct list_head work_list;
+ enum userptr_work_list_ops op;
+ atomic_t in_release;
+ struct mm_struct *mm;
+ uint64_t notifier_start;
+ uint64_t notifier_last;
+ struct mmu_interval_notifier notifier;
+#endif
};
#define to_virtio_gpu_shmem(virtio_gpu_object) \
@@ -317,6 +352,12 @@ struct virtio_gpu_fpriv {
bool explicit_debug_name;
struct rb_root_cached userptrs_tree;
struct mutex userptrs_tree_lock;
+
+#ifdef CONFIG_MMU_NOTIFIER
+ struct work_struct userptr_work;
+ struct list_head userptr_work_list;
+ spinlock_t userptr_work_list_lock;
+#endif
};
/* virtgpu_ioctl.c */
@@ -536,4 +577,6 @@ bool virtio_gpu_is_userptr(struct virtio_gpu_object *bo);
void virtio_gpu_userptr_interval_tree_init(struct virtio_gpu_fpriv *vfpriv);
void virtio_gpu_userptr_set_handle(struct virtio_gpu_object *qobj,
uint32_t handle);
+uint32_t virtio_gpu_userptr_get_handle(struct virtio_gpu_object *qobj);
+void virtio_gpu_userptr_list_work_init(struct virtio_gpu_fpriv *vfpriv);
#endif
diff --git a/drivers/gpu/drm/virtio/virtgpu_ioctl.c
b/drivers/gpu/drm/virtio/virtgpu_ioctl.c
index ad1ac8d0eadf..14326fd8fee9 100644
--- a/drivers/gpu/drm/virtio/virtgpu_ioctl.c
+++ b/drivers/gpu/drm/virtio/virtgpu_ioctl.c
@@ -697,8 +697,10 @@ static int virtio_gpu_context_init_ioctl(struct drm_device
*dev,
}
}
- if (vfpriv->context_init & VIRTIO_GPU_CAPSET_HSAKMT)
+ if (vfpriv->context_init & VIRTIO_GPU_CAPSET_HSAKMT) {
+ virtio_gpu_userptr_list_work_init(vfpriv);
virtio_gpu_userptr_interval_tree_init(vfpriv);
+ }
virtio_gpu_create_context_locked(vgdev, vfpriv);
virtio_gpu_notify(vgdev);
diff --git a/drivers/gpu/drm/virtio/virtgpu_kms.c
b/drivers/gpu/drm/virtio/virtgpu_kms.c
index 3d5158caef46..3dc44eb16fb8 100644
--- a/drivers/gpu/drm/virtio/virtgpu_kms.c
+++ b/drivers/gpu/drm/virtio/virtgpu_kms.c
@@ -345,6 +345,8 @@ void virtio_gpu_driver_postclose(struct drm_device *dev,
struct drm_file *file)
return;
if (vfpriv->context_created) {
+ if (vfpriv->context_init & VIRTIO_GPU_CAPSET_HSAKMT)
+ flush_work(&vfpriv->userptr_work);
virtio_gpu_cmd_context_destroy(vgdev, vfpriv->ctx_id);
virtio_gpu_notify(vgdev);
}
diff --git a/drivers/gpu/drm/virtio/virtgpu_userptr.c
b/drivers/gpu/drm/virtio/virtgpu_userptr.c
index 03398c3b9f30..10264227f3e7 100644
--- a/drivers/gpu/drm/virtio/virtgpu_userptr.c
+++ b/drivers/gpu/drm/virtio/virtgpu_userptr.c
@@ -3,6 +3,7 @@
#include <linux/mm.h>
#include <linux/pid.h>
#include <linux/vmalloc.h>
+#include <linux/mmu_notifier.h>
#include "virtgpu_drv.h"
#include "drm/drm_gem.h"
@@ -13,21 +14,422 @@ virtio_gpu_userptr_get_sg_table(struct drm_gem_object
*obj);
static int virtio_gpu_userptr_insert(struct virtio_gpu_object_userptr *userptr,
struct virtio_gpu_fpriv *vfpriv)
{
+ int ret;
+
if (!userptr->ops->insert)
return -EINVAL;
- return userptr->ops->insert(userptr, vfpriv);
+ ret = userptr->ops->insert(userptr, vfpriv);
+ if (ret)
+ return ret;
+
+ if (userptr->ops->notifier_add)
+ ret = userptr->ops->notifier_add(userptr, userptr->start,
+ userptr->last -
+ userptr->start + 1UL);
+
+ return ret;
}
static int virtio_gpu_userptr_remove(struct virtio_gpu_object_userptr *userptr,
struct virtio_gpu_fpriv *vfpriv)
{
+ int ret;
+
if (!userptr->ops->remove)
return -EINVAL;
- return userptr->ops->remove(userptr, vfpriv);
+ ret = userptr->ops->remove(userptr, vfpriv);
+ if (ret)
+ return ret;
+
+ if (userptr->ops->notifier_remove)
+ userptr->ops->notifier_remove(userptr);
+
+ return ret;
+}
+
+static bool virtio_gpu_userptr_valid(struct virtio_gpu_object_userptr *userptr)
+{
+ if (userptr->ops->valid)
+ return userptr->ops->valid(userptr);
+
+ return true;
+}
+
+#ifdef CONFIG_MMU_NOTIFIER
+
+static bool
+virtio_gpu_userptr_invalidate(struct mmu_interval_notifier *mn,
+ const struct mmu_notifier_range *range,
+ unsigned long cur_seq);
+
+static const struct mmu_interval_notifier_ops virtio_gpu_userptr_mn_ops = {
+ .invalidate = virtio_gpu_userptr_invalidate,
+};
+
+static int
+virtio_gpu_userptr_add_notifier(struct virtio_gpu_object_userptr *userptr,
+ unsigned long start, unsigned long length)
+{
+ if (!start || !length)
+ return -EINVAL;
+
+ return mmu_interval_notifier_insert(&userptr->notifier, userptr->mm,
+ start, length,
+ &virtio_gpu_userptr_mn_ops);
+}
+
+static void
+virtio_gpu_userptr_remove_notifier(struct virtio_gpu_object_userptr *userptr)
+{
+ mmu_interval_notifier_remove(&userptr->notifier);
+}
+
+static void virtio_gpu_userptr_unmap(struct virtio_gpu_object_userptr *userptr)
+{
+ drm_gem_handle_delete(userptr->file, userptr->bo_handle);
+}
+
+static void virtio_gpu_userptr_update_notifier_and_interval_tree(
+ struct virtio_gpu_object_userptr *userptr)
+{
+ unsigned long start = userptr->notifier.interval_tree.start;
+ unsigned long last = userptr->notifier.interval_tree.last;
+
+ if (userptr->start == start && userptr->last == last)
+ return;
+
+ if (start != 0 && last != 0)
+ virtio_gpu_userptr_remove(userptr, userptr->file->driver_priv);
+
+ virtio_gpu_userptr_insert(userptr, userptr->file->driver_priv);
+ userptr->op = 0;
}
+static int virtio_gpu_userptr_split(struct virtio_gpu_object_userptr *userptr,
+ unsigned long valid_start,
+ unsigned long valid_last,
+ struct virtio_gpu_object_userptr **new)
+{
+ uint64_t old_start = userptr->start;
+ uint64_t old_last = userptr->last;
+
+ if (old_start != valid_start && old_last != valid_last)
+ return -EINVAL;
+ if (valid_start < old_start || valid_last > old_last)
+ return -EINVAL;
+
+ if (userptr->ops->split_new)
+ *new = userptr->ops->split_new(userptr, valid_start,
+ valid_last);
+
+ userptr->start = valid_start;
+ userptr->last = valid_last;
+
+ return 0;
+}
+
+static void
+virtio_gpu_userptr_update_split(struct virtio_gpu_object_userptr *userptr,
+ unsigned long mn_start, unsigned long mn_last)
+{
+ struct virtio_gpu_object_userptr *head;
+ struct virtio_gpu_object_userptr *tail;
+
+ if (!userptr->ops->split)
+ return;
+ if (userptr->op == USERPTR_OP_UNMAP)
+ return;
+
+ if (mn_start > userptr->last || mn_last < userptr->start)
+ return;
+
+ head = tail = userptr;
+ if (mn_start > userptr->start)
+ userptr->ops->split(userptr, userptr->start, mn_start - 1UL,
+ &tail);
+ else if (mn_last < userptr->last)
+ userptr->ops->split(userptr, mn_last + 1UL, userptr->last,
+ &head);
+}
+
+static void
+virtio_gpu_userptr_add_list_work(struct virtio_gpu_object_userptr *userptr,
+ int op)
+{
+ struct virtio_gpu_fpriv *vfpriv = userptr->file->driver_priv;
+
+ spin_lock(&vfpriv->userptr_work_list_lock);
+
+ if (!list_empty(&userptr->work_list)) {
+ if (op != USERPTR_OP_NULL && userptr->op != USERPTR_OP_UNMAP)
+ userptr->op = op;
+ } else {
+ userptr->op = op;
+ list_add_tail(&userptr->work_list, &vfpriv->userptr_work_list);
+ }
+
+ spin_unlock(&vfpriv->userptr_work_list_lock);
+}
+
+static int virtio_gpu_follow_pfn(struct vm_area_struct *vma, uint64_t addr,
+ unsigned long *pfn)
+{
+ struct follow_pfnmap_args args = { .vma = vma, .address = addr };
+
+ if (follow_pfnmap_start(&args))
+ return -EINVAL;
+
+ *pfn = args.pfn;
+ follow_pfnmap_end(&args);
+
+ return 0;
+}
+
+static int virtio_gpu_userptr_check(struct virtio_gpu_object_userptr *userptr,
+ struct vm_area_struct *vma, uint64_t start,
+ uint64_t end)
+{
+ uint64_t addr;
+ int ret;
+ unsigned long pfn;
+
+ for (addr = start; addr < end; addr += PAGE_SIZE) {
+ ret = virtio_gpu_follow_pfn(vma, addr, &pfn);
+ if (ret)
+ return -EINVAL;
+
+ if (page_to_pfn(userptr->pages[(addr - userptr->start) >>
+ PAGE_SHIFT]) != pfn)
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int
+virtio_gpu_userptr_check_range(struct virtio_gpu_object_userptr *userptr,
+ uint64_t notifier_start, uint64_t notifier_last)
+{
+ uint64_t start, end, addr;
+ int r = 0;
+
+ start = notifier_start;
+ end = notifier_last + (1UL << PAGE_SHIFT);
+
+ for (addr = start; !r && addr < end;) {
+ struct vm_area_struct *vma;
+ uint64_t next = 0;
+
+ vma = vma_lookup(userptr->mm, addr);
+
+ if (vma) {
+ next = min(vma->vm_end, end);
+ r = virtio_gpu_userptr_check(userptr, vma, start, next);
+ if (r)
+ break;
+ } else {
+ r = -EFAULT;
+ break;
+ }
+
+ addr = next;
+ }
+
+ return r;
+}
+
+static void
+virtio_gpu_update_or_remove_userptr(struct virtio_gpu_object_userptr *userptr,
+ unsigned long start, unsigned long last)
+{
+ if ((userptr->start) >= start && (userptr->last) <= last) {
+ if (atomic_xchg(&userptr->in_release, 1) == 0) {
+ virtio_gpu_userptr_add_list_work(userptr,
+ USERPTR_OP_UNMAP);
+ }
+ } else {
+ virtio_gpu_userptr_update_split(userptr, start, last);
+ virtio_gpu_userptr_add_list_work(userptr, USERPTR_OP_UPDATE);
+ }
+}
+
+static void virtio_gpu_userptr_evict(struct virtio_gpu_object_userptr *userptr)
+{
+ if (!userptr->notifier_start || !userptr->notifier_last)
+ return;
+
+ if (userptr->notifier_start < userptr->start ||
+ userptr->notifier_last > userptr->last)
+ return;
+
+ if (virtio_gpu_userptr_check_range(userptr, userptr->notifier_start,
+ userptr->notifier_last)) {
+ virtio_gpu_update_or_remove_userptr(
+ userptr, userptr->notifier_start,
+ userptr->notifier_last + (1UL << PAGE_SHIFT) - 1UL);
+ }
+
+ userptr->notifier_start = 0;
+ userptr->notifier_last = 0;
+}
+
+static void
+virtio_gpu_userptr_handle_list_work(struct virtio_gpu_object_userptr *userptr)
+{
+ switch (userptr->op) {
+ case USERPTR_OP_NULL:
+ break;
+ case USERPTR_OP_UNMAP:
+ virtio_gpu_userptr_unmap(userptr);
+ break;
+ case USERPTR_OP_UPDATE:
+ if (userptr->ops->update)
+ userptr->ops->update(userptr);
+ break;
+ case USERPTR_OP_EVICT:
+ if (userptr->ops->evict)
+ userptr->ops->evict(userptr);
+ break;
+ default:
+ break;
+ }
+}
+
+static void virtio_gpu_userptr_invalidate_work(struct work_struct *work)
+{
+ struct virtio_gpu_fpriv *vfpriv;
+ struct virtio_gpu_object_userptr *userptr;
+
+ vfpriv = container_of(work, struct virtio_gpu_fpriv, userptr_work);
+
+ spin_lock(&vfpriv->userptr_work_list_lock);
+ while (!list_empty(&vfpriv->userptr_work_list)) {
+ userptr = list_first_entry(&vfpriv->userptr_work_list,
+ struct virtio_gpu_object_userptr,
+ work_list);
+ spin_unlock(&vfpriv->userptr_work_list_lock);
+
+ mmap_write_lock(userptr->mm);
+
+ spin_lock(&vfpriv->userptr_work_list_lock);
+ list_del_init(&userptr->work_list);
+ spin_unlock(&vfpriv->userptr_work_list_lock);
+
+ mutex_lock(&vfpriv->userptrs_tree_lock);
+
+ virtio_gpu_userptr_handle_list_work(userptr);
+
+ mutex_unlock(&vfpriv->userptrs_tree_lock);
+ mmap_write_unlock(userptr->mm);
+
+ spin_lock(&vfpriv->userptr_work_list_lock);
+ }
+ spin_unlock(&vfpriv->userptr_work_list_lock);
+}
+
+void virtio_gpu_userptr_list_work_init(struct virtio_gpu_fpriv *vfpriv)
+{
+ INIT_WORK(&vfpriv->userptr_work, virtio_gpu_userptr_invalidate_work);
+ INIT_LIST_HEAD(&vfpriv->userptr_work_list);
+ spin_lock_init(&vfpriv->userptr_work_list_lock);
+}
+
+static void
+virtio_gpu_userptr_schedule_list_work(struct virtio_gpu_fpriv *vfpriv)
+{
+ spin_lock(&vfpriv->userptr_work_list_lock);
+ if (!list_empty(&vfpriv->userptr_work_list))
+ schedule_work(&vfpriv->userptr_work);
+ spin_unlock(&vfpriv->userptr_work_list_lock);
+}
+
+static bool
+virtio_gpu_userptr_invalidate(struct mmu_interval_notifier *mn,
+ const struct mmu_notifier_range *range,
+ unsigned long cur_seq)
+{
+ struct virtio_gpu_object_userptr *userptr;
+ unsigned long start;
+ unsigned long last;
+
+ if (range->event == MMU_NOTIFY_RELEASE)
+ return true;
+ if (!mmget_not_zero(mn->mm))
+ return true;
+
+ start = mn->interval_tree.start;
+ last = mn->interval_tree.last;
+ start = (max(start, range->start) >> PAGE_SHIFT) << PAGE_SHIFT;
+ last = (min(last, range->end - 1UL) >> PAGE_SHIFT) << PAGE_SHIFT;
+
+ userptr = container_of(mn, struct virtio_gpu_object_userptr, notifier);
+ userptr->mm = mn->mm;
+
+ mutex_lock(&userptr->lock);
+ mmu_interval_set_seq(mn, cur_seq);
+
+ if (userptr->op != USERPTR_OP_UNMAP) {
+ switch (range->event) {
+ case MMU_NOTIFY_UNMAP:
+ virtio_gpu_update_or_remove_userptr(
+ userptr, start,
+ last + (1UL << PAGE_SHIFT) - 1UL);
+ break;
+ default:
+ userptr->notifier_start = start;
+ userptr->notifier_last = last;
+ virtio_gpu_userptr_add_list_work(userptr,
+ USERPTR_OP_EVICT);
+ break;
+ }
+ }
+
+ virtio_gpu_userptr_schedule_list_work(userptr->file->driver_priv);
+
+ mutex_unlock(&userptr->lock);
+ mmput(mn->mm);
+ return true;
+}
+
+static void
+virtio_gpu_userptr_lock_and_flush_work(struct virtio_gpu_fpriv *vfpriv)
+{
+retry_flush_work:
+ flush_work(&vfpriv->userptr_work);
+
+ if (list_empty(&vfpriv->userptr_work_list))
+ return;
+
+ goto retry_flush_work;
+}
+
+static bool virtio_gpu_userptr_valid_with_notifier(
+ struct virtio_gpu_object_userptr *userptr)
+{
+ return (!atomic_read(&userptr->in_release)) && (!userptr->op);
+}
+
+static void
+virtio_gpu_userptr_notifier_init(struct virtio_gpu_object_userptr *userptr,
+ struct mm_struct *mm)
+{
+ userptr->notifier_start = 0;
+ userptr->notifier_last = 0;
+ atomic_set(&userptr->in_release, 0);
+ INIT_LIST_HEAD(&userptr->work_list);
+ mutex_init(&userptr->lock);
+ userptr->mm = mm;
+}
+
+#else
+static void
+virtio_gpu_userptr_lock_and_flush_work(struct virtio_gpu_fpriv *vfpriv)
+{
+}
+#endif /* CONFIG_MMU_NOTIFIER */
+
static uint64_t virtio_gpu_userptr_get_offset(struct virtio_gpu_object *qobj,
uint64_t addr)
{
@@ -52,7 +454,8 @@ virtio_gpu_userptr_from_addr_range(struct virtio_gpu_fpriv
*vfpriv,
userptr = container_of(node, struct virtio_gpu_object_userptr,
it_node);
- if (start >= userptr->start && last <= userptr->last) {
+ if (start >= userptr->start && last <= userptr->last &&
+ virtio_gpu_userptr_valid(userptr)) {
ret = userptr;
return ret;
}
@@ -92,7 +495,6 @@ void virtio_gpu_userptr_set_handle(struct virtio_gpu_object
*qobj,
uint32_t handle)
{
struct virtio_gpu_object_userptr *userptr = to_virtio_gpu_userptr(qobj);
-
userptr->bo_handle = handle;
}
@@ -254,6 +656,9 @@ virtio_gpu_userptr_init(struct drm_device *dev, struct
drm_file *file,
obj = &userptr->base.base.base;
obj->funcs = &virtio_gpu_userptr_funcs;
+ if (userptr->ops->notifier_init)
+ userptr->ops->notifier_init(userptr, current->mm);
+
drm_gem_private_object_init(dev, obj, aligned_size);
ret = virtio_gpu_resource_id_get(userptr->vgdev,
@@ -268,6 +673,15 @@ static const struct virtio_gpu_object_userptr_ops
virtio_gpu_userptr_ops = {
.release = virtio_gpu_userptr_release,
.insert = virtio_gpu_userptr_insert_interval_tree,
.remove = virtio_gpu_userptr_remove_interval_tree,
+#ifdef CONFIG_MMU_NOTIFIER
+ .valid = virtio_gpu_userptr_valid_with_notifier,
+ .notifier_init = virtio_gpu_userptr_notifier_init,
+ .notifier_add = virtio_gpu_userptr_add_notifier,
+ .notifier_remove = virtio_gpu_userptr_remove_notifier,
+ .split = virtio_gpu_userptr_split,
+ .update = virtio_gpu_userptr_update_notifier_and_interval_tree,
+ .evict = virtio_gpu_userptr_evict,
+#endif
};
int virtio_gpu_userptr_create(struct virtio_gpu_device *vgdev,
@@ -290,6 +704,7 @@ int virtio_gpu_userptr_create(struct virtio_gpu_device
*vgdev,
params->size))
return -EFAULT;
+ virtio_gpu_userptr_lock_and_flush_work(vfpriv);
mutex_lock(&vfpriv->userptrs_tree_lock);
userptr = virtio_gpu_userptr_from_addr_range(
--
2.34.1
