From: Philip Yang
When application explicitly call unmap or unmap from mmput when
application exit, driver will receive MMU_NOTIFY_UNMAP event to remove
svm range from process svms object tree and list first, unmap from GPUs
(in the following patch).
Split the svm ranges to handle partial unmapping of svm ranges. To
avoid deadlocks, updating MMU notifiers, range lists and interval trees
is done in a deferred worker. New child ranges are attached to their
parent range's child_list until the worker can update the
svm_range_list. svm_range_set_attr flushes deferred work and takes the
mmap_write_lock to guarantee that it has an up-to-date svm_range_list.
Signed-off-by: Philip Yang
Signed-off-by: Alex Sierra
Signed-off-by: Felix Kuehling
---
drivers/gpu/drm/amd/amdkfd/kfd_priv.h | 3 +
drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 285 +-
drivers/gpu/drm/amd/amdkfd/kfd_svm.h | 18 ++
3 files changed, 305 insertions(+), 1 deletion(-)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index 7c1d7789b91e..0d19a13fc227 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -735,6 +735,9 @@ struct svm_range_list {
struct mutexlock;
struct rb_root_cached objects;
struct list_headlist;
+ struct work_struct deferred_list_work;
+ struct list_headdeferred_range_list;
+ spinlock_t deferred_list_lock;
};
/* Process data */
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
index c6a9766b97a6..ddb1e2a29881 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
@@ -136,6 +136,8 @@ svm_range *svm_range_new(struct svm_range_list *svms,
uint64_t start,
INIT_LIST_HEAD(>update_list);
INIT_LIST_HEAD(>remove_list);
INIT_LIST_HEAD(>insert_list);
+ INIT_LIST_HEAD(>deferred_list);
+ INIT_LIST_HEAD(>child_list);
mutex_init(>lock);
svm_range_set_default_attributes(>preferred_loc,
>prefetch_loc,
@@ -412,6 +414,17 @@ svm_range_split_head(struct svm_range *prange, struct
svm_range *new,
return r;
}
+void svm_range_add_child(struct svm_range *prange, struct mm_struct *mm,
+struct svm_range *pchild, enum svm_work_list_ops op)
+{
+ pr_debug("add child 0x%p [0x%lx 0x%lx] to prange 0x%p child list %d\n",
+pchild, pchild->start, pchild->last, prange, op);
+
+ pchild->work_item.mm = mm;
+ pchild->work_item.op = op;
+ list_add_tail(>child_list, >child_list);
+}
+
/*
* Validation+GPU mapping with concurrent invalidation (MMU notifiers)
*
@@ -471,6 +484,30 @@ static int svm_range_validate_and_map(struct mm_struct *mm,
return r;
}
+/**
+ * svm_range_list_lock_and_flush_work - flush pending deferred work
+ *
+ * @svms: the svm range list
+ * @mm: the mm structure
+ *
+ * Context: Returns with mmap write lock held, pending deferred work flushed
+ *
+ */
+static void
+svm_range_list_lock_and_flush_work(struct svm_range_list *svms,
+ struct mm_struct *mm)
+{
+retry_flush_work:
+ flush_work(>deferred_list_work);
+ mmap_write_lock(mm);
+
+ if (list_empty(>deferred_range_list))
+ return;
+ mmap_write_unlock(mm);
+ pr_debug("retry flush\n");
+ goto retry_flush_work;
+}
+
struct svm_range *svm_range_clone(struct svm_range *old)
{
struct svm_range *new;
@@ -611,15 +648,255 @@ svm_range_handle_overlap(struct svm_range_list *svms,
struct svm_range *new,
return r;
}
+static void
+svm_range_update_notifier_and_interval_tree(struct mm_struct *mm,
+ struct svm_range *prange)
+{
+ unsigned long start;
+ unsigned long last;
+
+ start = prange->notifier.interval_tree.start >> PAGE_SHIFT;
+ last = prange->notifier.interval_tree.last >> PAGE_SHIFT;
+
+ if (prange->start == start && prange->last == last)
+ return;
+
+ pr_debug("up notifier 0x%p prange 0x%p [0x%lx 0x%lx] [0x%lx 0x%lx]\n",
+ prange->svms, prange, start, last, prange->start,
+ prange->last);
+
+ if (start != 0 && last != 0) {
+ interval_tree_remove(>it_node, >svms->objects);
+ svm_range_remove_notifier(prange);
+ }
+ prange->it_node.start = prange->start;
+ prange->it_node.last = prange->last;
+
+ interval_tree_insert(>it_node, >svms->objects);
+ svm_range_add_notifier_locked(mm, prange);
+}
+
+static void
+svm_range_handle_list_op(struct svm_range_list *svms, struct svm_range *prange)
+{
+ struct mm_struct *mm = prange->work_item.mm;
+
+ switch (prange->work_item.op) {
+ case SVM_OP_NULL:
+
