Normal SDMA queues and all other GPU generations are unaffected.
Assisted-by: Claude:Sonnet 4.6
Signed-off-by: Sreekant Somasekharan <[email protected]>
---
.../drm/amd/amdkfd/kfd_device_queue_manager.c | 54 +++++++++++++++----
.../drm/amd/amdkfd/kfd_device_queue_manager.h | 4 ++
drivers/gpu/drm/amd/amdkfd/kfd_priv.h | 1 +
3 files changed, 48 insertions(+), 11 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
index a23384571193..10a672b87ca5 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
@@ -556,8 +556,9 @@ static int allocate_doorbell(struct qcm_process_device *qpd,
return -EINVAL;
q->doorbell_id = q->properties.queue_id;
- } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
- q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
+ } else if ((q->properties.type == KFD_QUEUE_TYPE_SDMA ||
+ q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) &&
+ !q->properties.is_oversubscribed) {
/* For SDMA queues on SOC15 with 8-byte doorbell, use static
* doorbell assignments based on the engine and queue id.
* The doobell index distance between RLC (2*i) and (2*i+1)
@@ -583,7 +584,10 @@ static int allocate_doorbell(struct qcm_process_device
*qpd,
return -EINVAL;
q->doorbell_id = valid_id;
} else {
- /* For CP queues on SOC15 */
+ /* For CP queues on SOC15, and oversubscribed SDMA queues which
+ * reuse HW queue ids and so need a unique doorbell allocated
+ * from the per-process bitmap pool.
+ */
if (restore_id) {
if (*restore_id >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS)
return -EINVAL;
@@ -621,9 +625,14 @@ static void deallocate_doorbell(struct qcm_process_device
*qpd,
unsigned int old;
struct kfd_node *dev = qpd->dqm->dev;
+ /* Oversubscribed SDMA queues allocate a doorbell from the bitmap pool
+ * (like CP queues), so they must be freed; normal SDMA queues use
+ * static doorbell assignments and must not be.
+ */
if (!KFD_IS_SOC15(dev) ||
- q->properties.type == KFD_QUEUE_TYPE_SDMA ||
- q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
+ ((q->properties.type == KFD_QUEUE_TYPE_SDMA ||
+ q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) &&
+ !q->properties.is_oversubscribed))
return;
old = test_and_clear_bit(q->doorbell_id, qpd->doorbell_bitmap);
@@ -1800,15 +1809,33 @@ static int allocate_sdma_queue(struct
device_queue_manager *dqm,
{
struct device *dev = dqm->dev->adev->dev;
int bit;
+ bool is_oversubscribed = false;
if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
if (bitmap_empty(dqm->sdma_bitmap, get_num_sdma_queues(dqm))) {
- dev_warn(dev, "No more SDMA queue to allocate (%d total
queues)\n",
- get_num_sdma_queues(dqm));
- return -ENOMEM;
- }
+ /* All HW queues are busy. On GFX 12.0.x under MES,
+ * oversubscribe by reusing a HW queue id round-robin;
+ * MES multiplexes the SW queues via their unique
+ * doorbells. The non-MES HWS runlist path cannot
express
+ * duplicate sdma_ids, so it still fails with -ENOMEM.
+ */
+ unsigned int num_reserved, num_available;
+
+ if (KFD_GC_VERSION(dqm->dev) < IP_VERSION(12, 0, 0) ||
+ KFD_GC_VERSION(dqm->dev) >= IP_VERSION(12, 1, 0) ||
+ !dqm->dev->kfd->shared_resources.enable_mes) {
+ dev_warn(dev, "No more SDMA queue to allocate (%d
total queues)\n",
+ get_num_sdma_queues(dqm));
+ return -ENOMEM;
+ }
- if (restore_sdma_id) {
+ num_reserved = kfd_get_num_sdma_engines(dqm->dev) *
+
dqm->dev->kfd->device_info.num_reserved_sdma_queues_per_engine;
+ num_available = get_num_sdma_queues(dqm) - num_reserved;
+ q->sdma_id = num_reserved +
+ dqm->sdma_oversub_rotor++ % num_available;
+ is_oversubscribed = true;
+ } else if (restore_sdma_id) {
if (*restore_sdma_id >= get_num_sdma_queues(dqm))
return -EINVAL;
@@ -1901,6 +1928,7 @@ static int allocate_sdma_queue(struct device_queue_manager *dqm,
return -ENOMEM;
}
}
+ q->properties.is_oversubscribed = is_oversubscribed;
pr_debug("SDMA engine id: %d\n", q->properties.sdma_engine_id);
pr_debug("SDMA queue id: %d\n", q->properties.sdma_queue_id);
@@ -1914,7 +1942,11 @@ static void deallocate_sdma_queue(struct
device_queue_manager *dqm,
if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
if (q->sdma_id >= get_num_sdma_queues(dqm))
return;
- set_bit(q->sdma_id, dqm->sdma_bitmap);
+ /* Don't return oversubscribed queue IDs to bitmap.
+ * They are shared by multiple queues and managed via
round-robin.
+ */
+ if (!q->properties.is_oversubscribed)
+ set_bit(q->sdma_id, dqm->sdma_bitmap);
} else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
if (q->sdma_id >= get_num_xgmi_sdma_queues(dqm))
return;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
index c9f9f7a87111..81626e657055 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
@@ -255,6 +255,10 @@ struct device_queue_manager {
unsigned int gws_queue_count;
unsigned int total_queue_count;
unsigned int next_pipe_to_allocate;
+ /* Monotonically increasing counter used to round-robin oversubscribed
+ * SDMA queues across the available HW queue slots.
+ */
+ unsigned int sdma_oversub_rotor;
unsigned int *allocated_queues;
DECLARE_BITMAP(sdma_bitmap, KFD_MAX_SDMA_QUEUES);
DECLARE_BITMAP(xgmi_sdma_bitmap, KFD_MAX_SDMA_QUEUES);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index d8631847f0eb..5fff5e55fb09 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -531,6 +531,7 @@ struct queue_properties {
bool is_dbg_wa;
bool is_user_cu_masked;
bool is_reset;
+ bool is_oversubscribed;
/* Not relevant for user mode queues in cp scheduling */
unsigned int vmid;
/* Relevant only for sdma queues*/
