From: Christian König <christian.koe...@amd.com>
Just wait for any fence to become available, instead
of waiting for the last entry of the LRU.
Signed-off-by: Christian König <christian.koenig at amd.com>
---
drivers/gpu/drm/amd/amdgpu/amdgpu.h | 10 +-
drivers/gpu/drm/amd/amdgpu/amdgpu_job.c | 2 +-
drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c | 69 +++-----------
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 155 +++++++++++++++++++------------
4 files changed, 117 insertions(+), 119 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index f154d9f..52326d3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -597,10 +597,8 @@ int amdgpu_sync_resv(struct amdgpu_device *adev,
struct amdgpu_sync *sync,
struct reservation_object *resv,
void *owner);
-bool amdgpu_sync_is_idle(struct amdgpu_sync *sync,
- struct amdgpu_ring *ring);
-int amdgpu_sync_cycle_fences(struct amdgpu_sync *dst, struct amdgpu_sync *src,
- struct fence *fence);
+struct fence *amdgpu_sync_peek_fence(struct amdgpu_sync *sync,
+ struct amdgpu_ring *ring);
struct fence *amdgpu_sync_get_fence(struct amdgpu_sync *sync);
void amdgpu_sync_free(struct amdgpu_sync *sync);
int amdgpu_sync_init(void);
@@ -910,6 +908,10 @@ struct amdgpu_vm_manager {
struct list_head ids_lru;
struct amdgpu_vm_id ids[AMDGPU_NUM_VM];
+ /* Handling of VM fences */
+ u64 fence_context;
+ unsigned seqno[AMDGPU_MAX_RINGS];
+
uint32_t max_pfn;
/* vram base address for page table entry */
u64 vram_base_offset;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
index e395bbe..b50a845 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
@@ -166,7 +166,7 @@ static struct fence *amdgpu_job_run(struct amd_sched_job
*sched_job)
}
job = to_amdgpu_job(sched_job);
- BUG_ON(!amdgpu_sync_is_idle(&job->sync, NULL));
+ BUG_ON(amdgpu_sync_peek_fence(&job->sync, NULL));
trace_amdgpu_sched_run_job(job);
r = amdgpu_ib_schedule(job->ring, job->num_ibs, job->ibs,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
index a2766d7..5c8d302 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
@@ -223,16 +223,16 @@ int amdgpu_sync_resv(struct amdgpu_device *adev,
}
/**
- * amdgpu_sync_is_idle - test if all fences are signaled
+ * amdgpu_sync_peek_fence - get the next fence not signaled yet
*
* @sync: the sync object
* @ring: optional ring to use for test
*
- * Returns true if all fences in the sync object are signaled or scheduled to
- * the ring (if provided).
+ * Returns the next fence not signaled yet without removing it from the sync
+ * object.
*/
-bool amdgpu_sync_is_idle(struct amdgpu_sync *sync,
- struct amdgpu_ring *ring)
+struct fence *amdgpu_sync_peek_fence(struct amdgpu_sync *sync,
+ struct amdgpu_ring *ring)
{
struct amdgpu_sync_entry *e;
struct hlist_node *tmp;
@@ -246,68 +246,25 @@ bool amdgpu_sync_is_idle(struct amdgpu_sync *sync,
/* For fences from the same ring it is sufficient
* when they are scheduled.
*/
- if (s_fence->sched == &ring->sched &&
- fence_is_signaled(&s_fence->scheduled))
- continue;
- }
+ if (s_fence->sched == &ring->sched) {
+ if (fence_is_signaled(&s_fence->scheduled))
+ continue;
- if (fence_is_signaled(f)) {
- hash_del(&e->node);
- fence_put(f);
- kmem_cache_free(amdgpu_sync_slab, e);
- continue;
+ return &s_fence->scheduled;
+ }
}
- return false;
- }
-
- return true;
-}
-
-/**
- * amdgpu_sync_cycle_fences - move fences from one sync object into another
- *
- * @dst: the destination sync object
- * @src: the source sync object
- * @fence: fence to add to source
- *
- * Remove all fences from source and put them into destination and add
- * fence as new one into source.
- */
-int amdgpu_sync_cycle_fences(struct amdgpu_sync *dst, struct amdgpu_sync *src,
- struct fence *fence)
-{
- struct amdgpu_sync_entry *e, *newone;
- struct hlist_node *tmp;
- int i;
-
- /* Allocate the new entry before moving the old ones */
- newone = kmem_cache_alloc(amdgpu_sync_slab, GFP_KERNEL);
- if (!newone)
- return -ENOMEM;
-
- hash_for_each_safe(src->fences, i, tmp, e, node) {
- struct fence *f = e->fence;
-
- hash_del(&e->node);
if (fence_is_signaled(f)) {
+ hash_del(&e->node);
fence_put(f);
kmem_cache_free(amdgpu_sync_slab, e);
continue;
}
- if (amdgpu_sync_add_later(dst, f)) {
- kmem_cache_free(amdgpu_sync_slab, e);
- continue;
- }
-
- hash_add(dst->fences, &e->node, f->context);
+ return f;
}
- hash_add(src->fences, &newone->node, fence->context);
- newone->fence = fence_get(fence);
-
- return 0;
+ return NULL;
}
/**
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index f206820..8ea1c73 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -25,6 +25,7 @@
* Alex Deucher
* Jerome Glisse
*/
+#include <linux/fence-array.h>
#include <drm/drmP.h>
#include <drm/amdgpu_drm.h>
#include "amdgpu.h"
@@ -180,82 +181,116 @@ int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct
amdgpu_ring *ring,
struct amdgpu_device *adev = ring->adev;
struct fence *updates = sync->last_vm_update;
struct amdgpu_vm_id *id, *idle;
- unsigned i = ring->idx;
- int r;
+ struct fence **fences;
+ unsigned i;
+ int r = 0;
+
+ fences = kmalloc_array(sizeof(void *), adev->vm_manager.num_ids,
+ GFP_KERNEL);
+ if (!fences)
+ return -ENOMEM;
mutex_lock(&adev->vm_manager.lock);
/* Check if we have an idle VMID */
+ i = 0;
list_for_each_entry(idle, &adev->vm_manager.ids_lru, list) {
- if (amdgpu_sync_is_idle(&idle->active, ring))
+ fences[i] = amdgpu_sync_peek_fence(&idle->active, ring);
+ if (!fences[i])
break;
-
+ ++i;
}
- /* If we can't find a idle VMID to use, just wait for the oldest */
+ /* If we can't find a idle VMID to use, wait till one becomes available
*/
if (&idle->list == &adev->vm_manager.ids_lru) {
- id = list_first_entry(&adev->vm_manager.ids_lru,
- struct amdgpu_vm_id,
- list);
- } else {
- /* Check if we can use a VMID already assigned to this VM */
- do {
- struct fence *flushed;
-
- id = vm->ids[i++];
- if (i == AMDGPU_MAX_RINGS)
- i = 0;
-
- /* Check all the prerequisites to using this VMID */
- if (!id)
- continue;
-
- if (atomic64_read(&id->owner) != vm->client_id)
- continue;
-
- if (pd_addr != id->pd_gpu_addr)
- continue;
-
- if (id->last_user != ring && (!id->last_flush ||
- !fence_is_signaled(id->last_flush)))
- continue;
-
- flushed = id->flushed_updates;
- if (updates && (!flushed ||
- fence_is_later(updates, flushed)))
- continue;
-
- /* Good we can use this VMID */
- if (id->last_user == ring) {
- r = amdgpu_sync_fence(ring->adev, sync,
- id->first);
- if (r)
- goto error;
- }
+ u64 fence_context = adev->vm_manager.fence_context + ring->idx;
+ unsigned seqno = ++adev->vm_manager.seqno[ring->idx];
+ struct fence_array *array;
+ unsigned j;
+
+ for (j = 0; j < i; ++j)
+ fence_get(fences[j]);
+
+ array = fence_array_create(i, fences, fence_context,
+ seqno, true);
+ if (!array) {
+ for (j = 0; j < i; ++j)
+ fence_put(fences[j]);
+ kfree(fences);
+ r = -ENOMEM;
+ goto error;
+ }
+
+
+ r = amdgpu_sync_fence(ring->adev, sync, &array->base);
+ fence_put(&array->base);
+ if (r)
+ goto error;
+
+ mutex_unlock(&adev->vm_manager.lock);
+ return 0;
+
+ }
+ kfree(fences);
+
+ /* Check if we can use a VMID already assigned to this VM */
+ i = ring->idx;
+ do {
+ struct fence *flushed;
+
+ id = vm->ids[i++];
+ if (i == AMDGPU_MAX_RINGS)
+ i = 0;
- /* And remember this submission as user of the VMID */
- r = amdgpu_sync_fence(ring->adev, &id->active, fence);
+ /* Check all the prerequisites to using this VMID */
+ if (!id)
+ continue;
+
+ if (atomic64_read(&id->owner) != vm->client_id)
+ continue;
+
+ if (pd_addr != id->pd_gpu_addr)
+ continue;
+
+ if (id->last_user != ring &&
+ (!id->last_flush || !fence_is_signaled(id->last_flush)))
+ continue;
+
+ flushed = id->flushed_updates;
+ if (updates &&
+ (!flushed || fence_is_later(updates, flushed)))
+ continue;
+
+ /* Good we can use this VMID */
+ if (id->last_user == ring) {
+ r = amdgpu_sync_fence(ring->adev, sync,
+ id->first);
if (r)
goto error;
+ }
+
+ /* And remember this submission as user of the VMID */
+ r = amdgpu_sync_fence(ring->adev, &id->active, fence);
+ if (r)
+ goto error;
- list_move_tail(&id->list, &adev->vm_manager.ids_lru);
- vm->ids[ring->idx] = id;
+ list_move_tail(&id->list, &adev->vm_manager.ids_lru);
+ vm->ids[ring->idx] = id;
- *vm_id = id - adev->vm_manager.ids;
- *vm_pd_addr = AMDGPU_VM_NO_FLUSH;
- trace_amdgpu_vm_grab_id(vm, ring->idx, *vm_id,
- *vm_pd_addr);
+ *vm_id = id - adev->vm_manager.ids;
+ *vm_pd_addr = AMDGPU_VM_NO_FLUSH;
+ trace_amdgpu_vm_grab_id(vm, ring->idx, *vm_id, *vm_pd_addr);
- mutex_unlock(&adev->vm_manager.lock);
- return 0;
+ mutex_unlock(&adev->vm_manager.lock);
+ return 0;
- } while (i != ring->idx);
+ } while (i != ring->idx);
- /* Still no ID to use? Then use the idle one found earlier */
- id = idle;
- }
+ /* Still no ID to use? Then use the idle one found earlier */
+ id = idle;
- r = amdgpu_sync_cycle_fences(sync, &id->active, fence);
+ /* Remember this submission as user of the VMID */
+ r = amdgpu_sync_fence(ring->adev, &id->active, fence);
if (r)
goto error;
@@ -1515,6 +1550,10 @@ void amdgpu_vm_manager_init(struct amdgpu_device *adev)
&adev->vm_manager.ids_lru);
}
+ adev->vm_manager.fence_context = fence_context_alloc(AMDGPU_MAX_RINGS);
+ for (i = 0; i < AMDGPU_MAX_RINGS; ++i)
+ adev->vm_manager.seqno[i] = 0;
+
atomic_set(&adev->vm_manager.vm_pte_next_ring, 0);
atomic64_set(&adev->vm_manager.client_counter, 0);
}
--
2.5.0
