From: Lijo Lazar <[email protected]> Add user save area parameters to the MQD properties for queue creation and validate the parameters before using them for MQD initialization.
The CWSR parameter validation walks the VM mapping tree and updates the user queue VA mapping state. Reserve the VM root BO around the validation so the required VM reservation is held while accessing the mapping tree. Signed-off-by: Lijo Lazar <[email protected]> Co-developed-by: James Zhu <[email protected]> Signed-off-by: James Zhu <[email protected]> Change-Id: I8b6f3a38a64280f043a060ebce958d7a425b2199 --- drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h | 8 +++ drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c | 25 ++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h | 8 ++- drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c | 14 +++++ drivers/gpu/drm/amd/amdgpu/mes_userqueue.c | 73 ++++++++++++++++++---- 5 files changed, 114 insertions(+), 14 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h index 2b13a5b37539..b3df09dbcbad 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h @@ -527,6 +527,14 @@ struct amdgpu_mqd_prop { uint32_t cu_mask_count; uint32_t cu_flags; bool is_user_cu_masked; + bool aql_queue; + uint32_t queue_percentage; + /* used in gfx9 and gfx12.1 */ + uint32_t pm4_target_xcc; + /* cwsr params */ + uint64_t ctx_save_area_addr; + uint32_t ctx_save_area_size; + uint32_t ctl_stack_size; }; struct amdgpu_mqd { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c index e29823ae8672..6eea0ca40556 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c @@ -35,6 +35,7 @@ #include "amdgpu_hmm.h" #include "amdgpu_userq_fence.h" #include "amdgpu_trace.h" +#include "amdgpu_cwsr.h" u32 amdgpu_userq_get_supported_ip_mask(struct amdgpu_device *adev) { @@ -266,6 +267,30 @@ int amdgpu_userq_input_va_validate(struct amdgpu_device *adev, return -EINVAL; } +int amdgpu_userq_input_cwsr_params_validate( + struct amdgpu_usermode_queue *queue, + struct amdgpu_cwsr_params *cwsr_params) +{ + struct amdgpu_fpriv *fpriv = uq_mgr_to_fpriv(queue->userq_mgr); + struct amdgpu_device *adev = queue->userq_mgr->adev; + uint32_t cwsr_size; + int num_xcc; + int r; + + num_xcc = amdgpu_xcp_get_num_xcc(adev->xcp_mgr, fpriv->xcp_id); + r = amdgpu_cwsr_validate_params(queue->userq_mgr->adev, cwsr_params, + num_xcc); + if (r) + return r; + cwsr_size = amdgpu_cwsr_size_needed(queue->userq_mgr->adev, num_xcc); + if (!cwsr_size) + return -EOPNOTSUPP; + + return amdgpu_userq_input_va_validate( + adev, queue, cwsr_params->ctx_save_area_address, cwsr_size, + &queue->userq_vas.va.cwsr); +} + static bool amdgpu_userq_buffer_va_mapped(struct amdgpu_vm *vm, u64 addr) { struct amdgpu_bo_va_mapping *mapping; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h index 488dc21d7c81..727f60c40bbf 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h @@ -41,6 +41,7 @@ enum amdgpu_userq_state { }; struct amdgpu_mqd_prop; +struct amdgpu_cwsr_params; struct amdgpu_userq_obj { void *cpu_ptr; @@ -97,8 +98,9 @@ struct amdgpu_usermode_queue { u64 eop; u64 shadow; u64 csa; + u64 cwsr; } va; - u64 va_array[6]; + u64 va_array[7]; } userq_vas; uint32_t gang_ctx_array_index; @@ -200,4 +202,8 @@ int amdgpu_userq_input_va_validate(struct amdgpu_device *adev, void amdgpu_userq_gem_va_unmap_validate(struct amdgpu_device *adev, struct amdgpu_bo_va_mapping *mapping); +int amdgpu_userq_input_cwsr_params_validate( + struct amdgpu_usermode_queue *queue, + struct amdgpu_cwsr_params *cwsr_params); + #endif diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c index 420ce997bebc..b5326c2067db 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c @@ -3327,6 +3327,20 @@ static int gfx_v12_0_compute_mqd_init(struct amdgpu_device *adev, void *m, /* set CU mask */ gfx_v12_0_compute_mqd_set_cu_mask(adev, mqd, prop); + /* If non-zero, assume cwsr is enabled */ + if (prop->ctx_save_area_addr) { + mqd->cp_hqd_persistent_state |= + (1 << CP_HQD_PERSISTENT_STATE__QSWITCH_MODE__SHIFT); + mqd->cp_hqd_ctx_save_base_addr_lo = + lower_32_bits(prop->ctx_save_area_addr); + mqd->cp_hqd_ctx_save_base_addr_hi = + upper_32_bits(prop->ctx_save_area_addr); + mqd->cp_hqd_ctx_save_size = prop->ctx_save_area_size; + mqd->cp_hqd_cntl_stack_size = prop->ctl_stack_size; + mqd->cp_hqd_cntl_stack_offset = prop->ctl_stack_size; + mqd->cp_hqd_wg_state_offset = prop->ctl_stack_size; + } + return 0; } diff --git a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c index 89a35fd53cba..3b7a1479fbf7 100644 --- a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c +++ b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c @@ -415,6 +415,7 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue, if (queue->queue_type == AMDGPU_HW_IP_COMPUTE) { struct drm_amdgpu_userq_mqd_compute_gfx11 *compute_mqd; + struct amdgpu_cwsr_params cwsr_params; if (mqd_user->mqd_size != sizeof(*compute_mqd)) { DRM_ERROR("Invalid compute IP MQD size\n"); @@ -422,7 +423,8 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue, goto free_mqd; } - compute_mqd = memdup_user(u64_to_user_ptr(mqd_user->mqd), mqd_user->mqd_size); + compute_mqd = memdup_user(u64_to_user_ptr(mqd_user->mqd), + mqd_user->mqd_size); if (IS_ERR(compute_mqd)) { r = PTR_ERR(compute_mqd); goto free_mqd; @@ -433,6 +435,7 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue, kfree(compute_mqd); goto free_mqd; } + r = amdgpu_userq_input_va_validate(adev, queue, compute_mqd->eop_va, 2048, &queue->userq_vas.va.eop); @@ -444,29 +447,63 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue, userq_props->eop_gpu_addr = compute_mqd->eop_va; userq_props->hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_NORMAL; - userq_props->hqd_queue_priority = AMDGPU_GFX_QUEUE_PRIORITY_MINIMUM; + userq_props->hqd_queue_priority = + AMDGPU_GFX_QUEUE_PRIORITY_MINIMUM; userq_props->hqd_active = false; userq_props->tmz_queue = mqd_user->flags & AMDGPU_USERQ_CREATE_FLAGS_QUEUE_SECURE; + + if (amdgpu_cwsr_is_enabled(adev)) { + struct amdgpu_fpriv *fpriv = + uq_mgr_to_fpriv(queue->userq_mgr); + int num_xcc; + + num_xcc = amdgpu_xcp_get_num_xcc(adev->xcp_mgr, + fpriv->xcp_id); + cwsr_params.ctx_save_area_address = + userq_props->ctx_save_area_addr; + cwsr_params.cwsr_sz = + userq_props->ctx_save_area_size; + userq_props->ctl_stack_size = + adev->cwsr_info->xcc_ctl_stack_sz * num_xcc; + + r = amdgpu_bo_reserve(queue->vm->root.bo, false); + if (r) { + kfree(compute_mqd); + goto free_mqd; + } + + r = amdgpu_userq_input_cwsr_params_validate( + queue, &cwsr_params); + amdgpu_bo_unreserve(queue->vm->root.bo); + if (r) { + kfree(compute_mqd); + goto free_mqd; + } + } + kfree(compute_mqd); } else if (queue->queue_type == AMDGPU_HW_IP_GFX) { struct drm_amdgpu_userq_mqd_gfx11 *mqd_gfx_v11; struct amdgpu_gfx_shadow_info shadow_info; if (adev->gfx.funcs->get_gfx_shadow_info) { - adev->gfx.funcs->get_gfx_shadow_info(adev, &shadow_info, true); + adev->gfx.funcs->get_gfx_shadow_info(adev, + &shadow_info, true); } else { r = -EINVAL; goto free_mqd; } - if (mqd_user->mqd_size != sizeof(*mqd_gfx_v11) || !mqd_user->mqd) { + if (mqd_user->mqd_size != sizeof(*mqd_gfx_v11) || + !mqd_user->mqd) { DRM_ERROR("Invalid GFX MQD\n"); r = -EINVAL; goto free_mqd; } - mqd_gfx_v11 = memdup_user(u64_to_user_ptr(mqd_user->mqd), mqd_user->mqd_size); + mqd_gfx_v11 = memdup_user(u64_to_user_ptr(mqd_user->mqd), + mqd_user->mqd_size); if (IS_ERR(mqd_gfx_v11)) { r = PTR_ERR(mqd_gfx_v11); goto free_mqd; @@ -482,7 +519,9 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue, kfree(mqd_gfx_v11); goto free_mqd; } - r = amdgpu_userq_input_va_validate(adev, queue, mqd_gfx_v11->shadow_va, + + r = amdgpu_userq_input_va_validate(adev, queue, + mqd_gfx_v11->shadow_va, shadow_info.shadow_size, &queue->userq_vas.va.shadow); if (r) { @@ -491,7 +530,8 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue, goto free_mqd; } - r = amdgpu_userq_input_va_validate(adev, queue, mqd_gfx_v11->csa_va, + r = amdgpu_userq_input_va_validate(adev, queue, + mqd_gfx_v11->csa_va, shadow_info.csa_size, &queue->userq_vas.va.csa); amdgpu_bo_unreserve(queue->vm->root.bo); @@ -504,13 +544,15 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue, } else if (queue->queue_type == AMDGPU_HW_IP_DMA) { struct drm_amdgpu_userq_mqd_sdma_gfx11 *mqd_sdma_v11; - if (mqd_user->mqd_size != sizeof(*mqd_sdma_v11) || !mqd_user->mqd) { + if (mqd_user->mqd_size != sizeof(*mqd_sdma_v11) || + !mqd_user->mqd) { DRM_ERROR("Invalid SDMA MQD\n"); r = -EINVAL; goto free_mqd; } - mqd_sdma_v11 = memdup_user(u64_to_user_ptr(mqd_user->mqd), mqd_user->mqd_size); + mqd_sdma_v11 = memdup_user(u64_to_user_ptr(mqd_user->mqd), + mqd_user->mqd_size); if (IS_ERR(mqd_sdma_v11)) { r = PTR_ERR(mqd_sdma_v11); goto free_mqd; @@ -521,7 +563,9 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue, kfree(mqd_sdma_v11); goto free_mqd; } - r = amdgpu_userq_input_va_validate(adev, queue, mqd_sdma_v11->csa_va, + + r = amdgpu_userq_input_va_validate(adev, queue, + mqd_sdma_v11->csa_va, 32, &queue->userq_vas.va.csa); amdgpu_bo_unreserve(queue->vm->root.bo); @@ -536,7 +580,8 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue, queue->userq_prop = userq_props; - r = mqd_hw_default->init_mqd(adev, (void *)queue->mqd.cpu_ptr, userq_props); + r = mqd_hw_default->init_mqd(adev, (void *)queue->mqd.cpu_ptr, + userq_props); if (r) { DRM_ERROR("Failed to initialize MQD for userqueue\n"); goto free_mqd; @@ -545,7 +590,8 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue, /* Create per-process MES process context BO */ r = mes_userq_create_proc_ctx_space(uq_mgr); if (r) { - DRM_ERROR("Failed to allocate MES process context space bo, error: %d\n", r); + DRM_ERROR("Failed to allocate MES process context space bo, error: %d\n", + r); goto free_mqd; } @@ -557,7 +603,8 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue, } /* FW expects WPTR BOs to be mapped into GART */ - r = mes_userq_create_wptr_mapping(adev, uq_mgr, queue, userq_props->wptr_gpu_addr); + r = mes_userq_create_wptr_mapping(adev, uq_mgr, queue, + userq_props->wptr_gpu_addr); if (r) { DRM_ERROR("Failed to create WPTR mapping\n"); goto free_ctx; -- 2.34.1
