---
.../drm/amd/amdkfd/kfd_device_queue_manager.c | 106 ++++++++++++++++++
drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h | 8 ++
.../gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c | 40 +++++++
drivers/gpu/drm/amd/amdkfd/kfd_priv.h | 6 +
4 files changed, 160 insertions(+)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
index ea9d87450eae..a23384571193 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
@@ -1257,6 +1257,99 @@ static int resume_single_queue(struct
device_queue_manager *dqm,
return 0;
}
+/* Unpin the MQD BO at S4 suspend so it is evicted into the hibernation image;
+ * dqm_repin_mqd_bo() pins it back on resume. Gated on adev->in_s4 so runtime
+ * eviction is untouched.
+ */
+static void dqm_evict_mqd_bo(struct device_queue_manager *dqm, struct queue *q)
+{
+ struct mqd_manager *mqd_mgr;
+ struct amdgpu_bo *bo;
+
+ if (!dqm->dev->adev->in_s4)
+ return;
+ if (!mqd_on_vram(dqm->dev->adev))
+ return;
+ if (q->properties.type != KFD_QUEUE_TYPE_COMPUTE)
+ return;
+ if (!q->mqd_mem_obj || !q->mqd_mem_obj->mem)
+ return;
+
+ /* Without update_mqd_gpu_addr() the MQD self-address cannot be fixed up
+ * after a repin, so skip eviction (with a warning) instead of faulting.
+ */
+ mqd_mgr =
dqm->mqd_mgrs[get_mqd_type_from_queue_type(q->properties.type)];
+ if (!mqd_mgr->update_mqd_gpu_addr) {
+ dev_warn_once(dqm->dev->adev->dev,
+ "MQD is in VRAM but update_mqd_gpu_addr is not
implemented; skipping hibernation eviction\n");
+ return;
+ }
+
+ bo = q->mqd_mem_obj->mem;
+ if (amdgpu_bo_reserve(bo, false))
+ return;
+
+ amdgpu_bo_unpin(bo);
+ amdgpu_bo_unreserve(bo);
+ q->mqd = NULL;
+ q->needs_mqd_repin = true;
+}
+
+/* Repin the MQD BO to VRAM and refresh the cached mapping and GPU addresses.
+ * Used both on resume and when a queue is destroyed before resume has repinned
+ * it. A no-op unless a repin is owed (needs_mqd_repin set).
+ */
+static int dqm_repin_mqd_bo(struct device_queue_manager *dqm, struct queue *q)
+{
+ struct mqd_manager *mqd_mgr;
+ struct amdgpu_bo *bo;
+ void *cpu_ptr;
+ int r;
+
+ if (!q->needs_mqd_repin)
+ return 0;
+ if (!q->mqd_mem_obj || !q->mqd_mem_obj->mem)
+ return 0;
+
+ bo = q->mqd_mem_obj->mem;
+ r = amdgpu_bo_reserve(bo, false);
+ if (r)
+ return r;
+ r = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_VRAM);
+ if (r) {
+ amdgpu_bo_unreserve(bo);
+ dev_err(dqm->dev->adev->dev,
+ "Failed to repin MQD of queue %d to VRAM: %d\n",
+ q->properties.queue_id, r);
+ return r;
+ }
+ /* The BO may have moved; refresh the kernel mapping and gpu address. */
+ amdgpu_bo_kunmap(bo);
+ r = amdgpu_bo_kmap(bo, &cpu_ptr);
+ amdgpu_bo_unreserve(bo);
+ if (r) {
+ dev_err(dqm->dev->adev->dev,
+ "Failed to remap MQD of queue %d: %d\n",
+ q->properties.queue_id, r);
+ return r;
+ }
+
+ q->mqd_mem_obj->cpu_ptr = cpu_ptr;
+ q->mqd_mem_obj->gpu_addr = amdgpu_bo_gpu_offset(bo);
+ q->gart_mqd_addr = q->mqd_mem_obj->gpu_addr;
+ q->mqd = cpu_ptr;
+
+ mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
+ q->properties.type)];
+ if (mqd_mgr->update_mqd_gpu_addr)
+ mqd_mgr->update_mqd_gpu_addr(mqd_mgr, q->mqd,
+ q->mqd_mem_obj,
+ &q->properties);
+
+ q->needs_mqd_repin = false;
+ return 0;
+}
+
static int evict_process_queues_nocpsch(struct device_queue_manager *dqm,
struct qcm_process_device *qpd)
{
@@ -1353,6 +1446,8 @@ static int evict_process_queues_cpsch(struct
device_queue_manager *dqm,
goto out;
}
}
+
+ dqm_evict_mqd_bo(dqm, q);
}
if (!dqm->dev->kfd->shared_resources.enable_mes) {
@@ -1492,6 +1587,13 @@ static int restore_process_queues_cpsch(struct
device_queue_manager *dqm,
q->properties.is_active = true;
increment_queue_count(dqm, &pdd->qpd, q);
+ retval = dqm_repin_mqd_bo(dqm, q);
+ if (retval) {
+ dev_err(dev, "Failed to repin MQD for queue %d\n",
+ q->properties.queue_id);
+ goto out;
+ }
+
if (dqm->dev->kfd->shared_resources.enable_mes) {
retval = add_queue_mes(dqm, q, qpd);
if (retval) {
@@ -2763,6 +2865,8 @@ static int destroy_queue_cpsch(struct
device_queue_manager *dqm,
qpd->pqm->process, q->device,
-1, false, NULL, 0);
+ /* Repin the MQD BO if still evicted for hibernation, before it is freed. */
+ dqm_repin_mqd_bo(dqm, q);
mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
return retval;
@@ -3020,6 +3124,8 @@ static int process_termination_cpsch(struct
device_queue_manager *dqm,
list_del(&q->list);
qpd->queue_count--;
dqm_unlock(dqm);
+ /* Repin the MQD BO if still evicted for hibernation, before
free. */
+ dqm_repin_mqd_bo(dqm, q);
mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
dqm_lock(dqm);
}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h
b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h
index 59eff3389d39..38b46b696243 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h
@@ -117,6 +117,14 @@ struct mqd_manager {
const void *ctl_stack_src,
const u32 ctl_stack_size);
+ /* Patch the MQD's cached self GPU address after the MQD BO has moved
+ * (e.g. repinned to a new VRAM location on hibernation resume). The MQD
+ * contents are otherwise preserved.
+ */
+ void (*update_mqd_gpu_addr)(struct mqd_manager *mm, void *mqd,
+ struct kfd_mem_obj *mqd_mem_obj,
+ struct queue_properties *p);
+
#if defined(CONFIG_DEBUG_FS)
int (*debugfs_show_mqd)(struct seq_file *m, void *data);
#endif
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
index 75e5a9f67d50..b95720198e28 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
@@ -476,6 +476,20 @@ static void restore_mqd(struct mqd_manager *mm, void **mqd,
qp->is_active = 0;
}
+static void update_mqd_gpu_addr(struct mqd_manager *mm, void *mqd,
+ struct kfd_mem_obj *mqd_mem_obj,
+ struct queue_properties *qp)
+{
+ struct v9_mqd *m = get_mqd(mqd);
+ uint64_t addr = mqd_mem_obj->gpu_addr;
+
+ m->cp_mqd_base_addr_lo = lower_32_bits(addr);
+ m->cp_mqd_base_addr_hi = upper_32_bits(addr);
+
+ if (mqd_on_vram(mm->dev->adev))
+ amdgpu_device_flush_hdp(mm->dev->adev, NULL);
+}
+
static void init_mqd_hiq(struct mqd_manager *mm, void **mqd,
struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
struct queue_properties *q)
@@ -860,6 +874,30 @@ static void restore_mqd_v9_4_3(struct mqd_manager *mm,
void **mqd,
if (mqd_on_vram(mm->dev->adev))
amdgpu_device_flush_hdp(mm->dev->adev, NULL);
}
+
+static void update_mqd_gpu_addr_v9_4_3(struct mqd_manager *mm, void *mqd,
+ struct kfd_mem_obj *mqd_mem_obj,
+ struct queue_properties *qp)
+{
+ struct kfd_mem_obj xcc_mqd_mem_obj;
+ uint64_t offset = mm->mqd_stride(mm, qp);
+ u32 num_xcc = NUM_XCC(mm->dev->xcc_mask);
+ struct v9_mqd *m;
+ int xcc;
+
+ memset(&xcc_mqd_mem_obj, 0x0, sizeof(struct kfd_mem_obj));
+
+ for (xcc = 0; xcc < num_xcc; xcc++) {
+ get_xcc_mqd(mqd_mem_obj, &xcc_mqd_mem_obj, offset * xcc);
+ m = get_mqd(mqd + offset * xcc);
+ m->cp_mqd_base_addr_lo =
lower_32_bits(xcc_mqd_mem_obj.gpu_addr);
+ m->cp_mqd_base_addr_hi =
upper_32_bits(xcc_mqd_mem_obj.gpu_addr);
+ }
+
+ if (mqd_on_vram(mm->dev->adev))
+ amdgpu_device_flush_hdp(mm->dev->adev, NULL);
+}
+
static int destroy_mqd_v9_4_3(struct mqd_manager *mm, void *mqd,
enum kfd_preempt_type type, unsigned int timeout,
uint32_t pipe_id, uint32_t queue_id)
@@ -1017,6 +1055,7 @@ struct mqd_manager *mqd_manager_init_v9(enum KFD_MQD_TYPE
type,
mqd->get_wave_state = get_wave_state_v9_4_3;
mqd->checkpoint_mqd = checkpoint_mqd_v9_4_3;
mqd->restore_mqd = restore_mqd_v9_4_3;
+ mqd->update_mqd_gpu_addr = update_mqd_gpu_addr_v9_4_3;
} else {
mqd->init_mqd = init_mqd;
mqd->load_mqd = load_mqd;
@@ -1025,6 +1064,7 @@ struct mqd_manager *mqd_manager_init_v9(enum KFD_MQD_TYPE
type,
mqd->get_wave_state = get_wave_state;
mqd->checkpoint_mqd = checkpoint_mqd;
mqd->restore_mqd = restore_mqd;
+ mqd->update_mqd_gpu_addr = update_mqd_gpu_addr;
}
break;
case KFD_MQD_TYPE_HIQ:
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index 2ea1cfd330a9..d8631847f0eb 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -638,6 +638,12 @@ struct queue {
uint32_t gang_ctx_array_index;
struct amdgpu_bo *wptr_bo_gart;
+
+ /* The VRAM-resident MQD BO (mqd_on_vram()) is unpinned at S4 suspend so
+ * TTM evicts it into the hibernation image, and repinned on resume. Set
+ * while the BO is unpinned so the resume path knows to repin it.
+ */
+ bool needs_mqd_repin;
};
enum KFD_MQD_TYPE {
