From: Mukul Joshi <[email protected]> The optimized QUE_SLEEP was only programmed on the HWS packet-manager path via a PM4 WRITE_DATA packet, which is skipped when MES is enabled.
Add gfx12.1 get_iq_wait_times/build_dequeue_wait_counts_packet_info callbacks and program CP_IQ_WAIT_TIME2 through the MES WRITE_REG op at start_cpsch, per XCC. Signed-off-by: Mukul Joshi <[email protected]> Reviewed-by: Harish Kasiviswanathan <[email protected]> Signed-off-by: Alex Deucher <[email protected]> --- .../drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c | 3 +- .../drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.h | 3 +- .../drm/amd/amdgpu/amdgpu_amdkfd_gfx_v12_1.c | 36 ++++++++++++++++- .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c | 3 +- .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h | 3 +- .../drm/amd/amdkfd/kfd_device_queue_manager.c | 40 +++++++++++++++++++ .../drm/amd/amdkfd/kfd_packet_manager_v9.c | 3 +- .../gpu/drm/amd/include/kgd_kfd_interface.h | 3 +- 8 files changed, 87 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c index 88acf75f0edd6..bc48b2ba30d11 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c @@ -1026,7 +1026,8 @@ void kgd_gfx_v10_build_dequeue_wait_counts_packet_info(struct amdgpu_device *ade uint32_t sch_wave, uint32_t que_sleep, uint32_t *reg_offset, - uint32_t *reg_data) + uint32_t *reg_data, + uint32_t inst) { *reg_data = wait_times; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.h index a4c607c881783..df350ea67c306 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.h @@ -56,7 +56,8 @@ void kgd_gfx_v10_build_dequeue_wait_counts_packet_info(struct amdgpu_device *ade uint32_t sch_wave, uint32_t que_sleep, uint32_t *reg_offset, - uint32_t *reg_data); + uint32_t *reg_data, + uint32_t inst); uint64_t kgd_gfx_v10_hqd_get_pq_addr(struct amdgpu_device *adev, uint32_t pipe_id, uint32_t queue_id, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v12_1.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v12_1.c index 070001fd34b0f..0694bff871d2d 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v12_1.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v12_1.c @@ -518,6 +518,37 @@ static void kgd_gfx_v12_1_get_cu_occupancy(struct amdgpu_device *adev, adev->gfx.cu_info.max_waves_per_simd; } +static void kgd_gfx_v12_1_get_iq_wait_times(struct amdgpu_device *adev, + uint32_t *wait_times, + uint32_t inst) +{ + *wait_times = RREG32_SOC15(GC, GET_INST(GC, inst), regCP_IQ_WAIT_TIME2); +} + +static void kgd_gfx_v12_1_build_dequeue_wait_counts_packet_info(struct amdgpu_device *adev, + uint32_t wait_times, + uint32_t sch_wave, + uint32_t que_sleep, + uint32_t *reg_offset, + uint32_t *reg_data, + uint32_t inst) +{ + *reg_data = wait_times; + + if (sch_wave) + *reg_data = REG_SET_FIELD(*reg_data, + CP_IQ_WAIT_TIME2, + SCH_WAVE, + sch_wave); + if (que_sleep) + *reg_data = REG_SET_FIELD(*reg_data, + CP_IQ_WAIT_TIME2, + QUE_SLEEP, + que_sleep); + + *reg_offset = SOC15_REG_OFFSET(GC, GET_INST(GC, inst), regCP_IQ_WAIT_TIME2); +} + const struct kfd2kgd_calls gfx_v12_1_kfd2kgd = { .init_interrupts = init_interrupts_v12_1, .hqd_dump = hqd_dump_v12_1, @@ -532,5 +563,8 @@ const struct kfd2kgd_calls gfx_v12_1_kfd2kgd = { .set_address_watch = kgd_gfx_v12_1_set_address_watch, .clear_address_watch = kgd_gfx_v12_1_clear_address_watch, .hqd_sdma_get_doorbell = kgd_gfx_v12_1_hqd_sdma_get_doorbell, - .get_cu_occupancy = kgd_gfx_v12_1_get_cu_occupancy + .get_cu_occupancy = kgd_gfx_v12_1_get_cu_occupancy, + .get_iq_wait_times = kgd_gfx_v12_1_get_iq_wait_times, + .build_dequeue_wait_counts_packet_info = + kgd_gfx_v12_1_build_dequeue_wait_counts_packet_info, }; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c index 2e116c06d5be3..b30ad9701bcef 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c @@ -1082,7 +1082,8 @@ void kgd_gfx_v9_build_dequeue_wait_counts_packet_info(struct amdgpu_device *adev uint32_t sch_wave, uint32_t que_sleep, uint32_t *reg_offset, - uint32_t *reg_data) + uint32_t *reg_data, + uint32_t inst) { *reg_data = wait_times; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h index 704452ca62f8e..9f43f16a1acea 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h @@ -102,7 +102,8 @@ void kgd_gfx_v9_build_dequeue_wait_counts_packet_info(struct amdgpu_device *adev uint32_t sch_wave, uint32_t que_sleep, uint32_t *reg_offset, - uint32_t *reg_data); + uint32_t *reg_data, + uint32_t inst); uint64_t kgd_gfx_v9_hqd_get_pq_addr(struct amdgpu_device *adev, uint32_t pipe_id, uint32_t queue_id, diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c index a233845711937..f6daef0e5ec09 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c @@ -2046,6 +2046,43 @@ static int unhalt_cpsch(struct device_queue_manager *dqm) return ret; } +/* Program CP_IQ_WAIT_TIME2 via the MES WRITE_REG op (MES has no HIQ). */ +static int init_dequeue_wait_counts_mes(struct device_queue_manager *dqm) +{ + struct kfd_node *dev = dqm->dev; + struct amdgpu_device *adev = dev->adev; + uint32_t sch_wave = 0, que_sleep = 1; + int inst, ret; + + if (!dev->kfd2kgd->build_dequeue_wait_counts_packet_info) + return 0; + + if (KFD_GC_VERSION(dev) != IP_VERSION(12, 1, 0)) + return 0; + + /* CP_IQ_WAIT_TIME2 is per-XCC; the offset must encode the target + * XCC so MES routes the write to that XCC's local register. + */ + for_each_inst(inst, dev->xcc_mask) { + uint32_t reg_offset = 0, reg_data = 0; + + dev->kfd2kgd->build_dequeue_wait_counts_packet_info( + adev, dqm->wait_times, sch_wave, que_sleep, + ®_offset, ®_data, inst); + + ret = amdgpu_mes_wreg(adev, reg_offset, reg_data, inst); + if (ret) { + dev_err(adev->dev, + "Failed to set optimized dequeue wait via MES on xcc %d\n", + inst); + return ret; + } + } + + update_dqm_wait_times(dqm); + return 0; +} + static int start_cpsch(struct device_queue_manager *dqm) { struct device *dev = dqm->dev->adev->dev; @@ -2087,6 +2124,9 @@ static int start_cpsch(struct device_queue_manager *dqm) KFD_DEQUEUE_WAIT_INIT, 0 /* unused */)) dev_err(dev, "Setting optimized dequeue wait failed. Using default values\n"); execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD); + } else { + if (init_dequeue_wait_counts_mes(dqm)) + dev_err(dev, "Setting optimized dequeue wait failed. Using default values\n"); } /* setup per-queue reset detection buffer */ diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager_v9.c index 54868b81086e7..55d573fa3de38 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager_v9.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager_v9.c @@ -306,7 +306,8 @@ static inline void pm_build_dequeue_wait_counts_packet_info(struct packet_manage sch_value, que_sleep, reg_offset, - reg_data); + reg_data, + 0); } /* pm_grace_period_0_supported - whether firmware tolerates a CWSR grace diff --git a/drivers/gpu/drm/amd/include/kgd_kfd_interface.h b/drivers/gpu/drm/amd/include/kgd_kfd_interface.h index 965b50c8ca303..01c2631bbdff8 100644 --- a/drivers/gpu/drm/amd/include/kgd_kfd_interface.h +++ b/drivers/gpu/drm/amd/include/kgd_kfd_interface.h @@ -319,7 +319,8 @@ struct kfd2kgd_calls { uint32_t sch_wave, uint32_t que_sleep, uint32_t *reg_offset, - uint32_t *reg_data); + uint32_t *reg_data, + uint32_t inst); void (*get_cu_occupancy)(struct amdgpu_device *adev, struct kfd_cu_occupancy *cu_occupancy, int *max_waves_per_cu, uint32_t inst); -- 2.55.0
