Fix a hardware RTL bug in the TCP XNACK scoreboard reset path where a wave receiving a no-retry FAULT from UTC and subsequently terminating may leave its scoreboard entry uncleared when reset events arrive back-to-back.
This can cause a later, unrelated wave using the same slot to incorrectly inherit the XNACK error state, resulting in its VMEM operations being dropped even though the wave never faulted. Signed-off-by: Gang Ba <[email protected]> --- .../drm/amd/amdgpu/amdgpu_amdkfd_aldebaran.c | 3 +- .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c | 10 ++- .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h | 1 + drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.c | 83 +++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.h | 1 + .../gpu/drm/amd/amdkfd/kfd_int_process_v9.c | 54 ++++++++++++ .../gpu/drm/amd/include/kgd_kfd_interface.h | 1 + 7 files changed, 150 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_aldebaran.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_aldebaran.c index 7e9f7a280c1b..2aaa9826c8f7 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_aldebaran.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_aldebaran.c @@ -193,5 +193,6 @@ const struct kfd2kgd_calls aldebaran_kfd2kgd = { .program_trap_handler_settings = kgd_gfx_v9_program_trap_handler_settings, .hqd_get_pq_addr = kgd_gfx_v9_hqd_get_pq_addr, .hqd_reset = kgd_gfx_v9_hqd_reset, - .hqd_sdma_get_doorbell = kgd_gfx_v9_hqd_sdma_get_doorbell + .hqd_sdma_get_doorbell = kgd_gfx_v9_hqd_sdma_get_doorbell, + .hqd_gfx_cleaner = kgd_gfx_v9_cleaner }; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c index b30ad9701bce..81f1945c4090 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c @@ -37,6 +37,7 @@ #include "soc15.h" #include "soc15d.h" #include "gfx_v9_0.h" +#include "gfx_v9_4_2.h" #include "amdgpu_amdkfd_gfx_v9.h" #include <uapi/linux/kfd_ioctl.h> @@ -1226,11 +1227,15 @@ uint64_t kgd_gfx_v9_hqd_reset(struct amdgpu_device *adev, uint32_t kgd_gfx_v9_hqd_sdma_get_doorbell(struct amdgpu_device *adev, int engine, int queue) - { return 0; } +void kgd_gfx_v9_cleaner(struct amdgpu_device *adev) +{ + gfx_v9_4_2_shader_cleaner_workarounds(adev); +} + const struct kfd2kgd_calls gfx_v9_kfd2kgd = { .program_sh_mem_settings = kgd_gfx_v9_program_sh_mem_settings, .set_pasid_vmid_mapping = kgd_gfx_v9_set_pasid_vmid_mapping, @@ -1261,5 +1266,6 @@ const struct kfd2kgd_calls gfx_v9_kfd2kgd = { .program_trap_handler_settings = kgd_gfx_v9_program_trap_handler_settings, .hqd_get_pq_addr = kgd_gfx_v9_hqd_get_pq_addr, .hqd_reset = kgd_gfx_v9_hqd_reset, - .hqd_sdma_get_doorbell = kgd_gfx_v9_hqd_sdma_get_doorbell + .hqd_sdma_get_doorbell = kgd_gfx_v9_hqd_sdma_get_doorbell, + .hqd_gfx_cleaner = kgd_gfx_v9_cleaner }; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h index 9f43f16a1ace..3b8b7bd2377f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h @@ -115,3 +115,4 @@ uint64_t kgd_gfx_v9_hqd_reset(struct amdgpu_device *adev, unsigned int utimeout); uint32_t kgd_gfx_v9_hqd_sdma_get_doorbell(struct amdgpu_device *adev, int engine, int queue); +void kgd_gfx_v9_cleaner(struct amdgpu_device *adev); diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.c index 8b84bba58752..d3e008134afb 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.c @@ -343,6 +343,28 @@ const struct soc15_reg_entry sgpr64_init_regs_aldebaran[] = { { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff }, }; +static const u32 fault_cleaner_compute_shader_aldebaran[] = { + 0xbe8000ff, 0x00001000, 0x80808100, 0x86000000, 0xbf85fffd, 0xbf810000 +}; + +const struct soc15_reg_entry fault_cleaner_init_regs_aldebaran[] = { + { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, + { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_NUM_THREAD_X), 0x40 }, + { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_NUM_THREAD_Y), 1 }, + { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_NUM_THREAD_Z), 1 }, + { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_PGM_RSRC1), 0xaf0000 }, + { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_PGM_RSRC2), 0xc8 }, + { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_PGM_RSRC3), 0xea4fac }, + { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff }, + { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff }, + { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff }, + { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff }, + { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff }, + { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff }, + { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff }, + { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff }, +}; + static int gfx_v9_4_2_run_shader(struct amdgpu_device *adev, struct amdgpu_ring *ring, const u32 *shader_ptr, u32 shader_size, @@ -692,6 +714,67 @@ static int gfx_v9_4_2_do_vgprs_init(struct amdgpu_device *adev) return r; } +static int gfx_v9_4_2_do_cleaner(struct amdgpu_device *adev) +{ + int r; + /* CU_ID: 0~15, SIMD_ID: 0~3, WAVE_ID: 0 ~ 9 */ + int wb_size = adev->gfx.config.max_shader_engines * + CU_ID_MAX * SIMD_ID_MAX * WAVE_ID_MAX; + struct amdgpu_ib wb_ib; + struct dma_fence *fence = NULL; + u32 pattern = 0xa; + + /* bail if the compute ring is not ready */ + if (!adev->gfx.compute_ring[0].sched.ready) + return 0; + + /* allocate the write-back buffer from IB */ + memset(&wb_ib, 0, sizeof(wb_ib)); + r = amdgpu_ib_get(adev, NULL, (1 + wb_size) * sizeof(uint32_t), + AMDGPU_IB_POOL_DIRECT, &wb_ib); + if (r) { + dev_err(adev->dev, "failed to get ib (%d) for wb.\n", r); + return r; + } + memset(wb_ib.ptr, 0, (1 + wb_size) * sizeof(uint32_t)); + + r = gfx_v9_4_2_run_shader(adev, + &adev->gfx.compute_ring[0], + fault_cleaner_compute_shader_aldebaran, + sizeof(fault_cleaner_compute_shader_aldebaran), + fault_cleaner_init_regs_aldebaran, + ARRAY_SIZE(fault_cleaner_init_regs_aldebaran), + adev->gfx.cu_info.number * 32, + wb_ib.gpu_addr, pattern, &fence); + + if (r) { + dev_err(adev->dev, "failed to clear MI200\n"); + goto pro_end; + } + + /* wait for the GPU to finish processing the IB */ + r = dma_fence_wait(fence, false); + if (r) + dev_err(adev->dev, "timeout to clear MI200\n"); + +pro_end: + if (fence) + dma_fence_put(fence); + amdgpu_ib_free(&wb_ib, NULL); + + if (r) + dev_dbg(adev->dev, "Clean MI200 Failed\n"); + else + dev_dbg(adev->dev, "Clean MI200 Successfully\n"); + + return r; +} + +void gfx_v9_4_2_shader_cleaner_workarounds(struct amdgpu_device *adev) +{ + gfx_v9_4_2_do_cleaner(adev); +} + int gfx_v9_4_2_do_edc_gpr_workarounds(struct amdgpu_device *adev) { /* only support when RAS is enabled */ diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.h b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.h index a603724c1dfc..5c911f0f5d01 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.h +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.h @@ -30,6 +30,7 @@ void gfx_v9_4_2_init_golden_registers(struct amdgpu_device *adev, uint32_t die_id); void gfx_v9_4_2_init_sq(struct amdgpu_device *adev); void gfx_v9_4_2_set_power_brake_sequence(struct amdgpu_device *adev); +void gfx_v9_4_2_shader_cleaner_workarounds(struct amdgpu_device *adev); int gfx_v9_4_2_do_edc_gpr_workarounds(struct amdgpu_device *adev); extern struct amdgpu_gfx_ras gfx_v9_4_2_ras; diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c index 1688d8e595f2..3a963e2da649 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c @@ -359,6 +359,57 @@ static bool event_interrupt_isr_v9(struct kfd_node *dev, !amdgpu_no_queue_eviction_on_vm_fault); } +static void gfx942_workaround_cleaner(struct kfd_node *dev, uint16_t pasid) +{ + struct kfd_process *p; + struct kfd_process_device *pdd = NULL; + struct device_queue_manager *dqm; + int i, ret; + + p = kfd_lookup_process_by_pasid(pasid, NULL); + if (!p) + return; + + /* Evict all process devices before running the workaround */ + for (i = 0; i < p->n_pdds; i++) { + pdd = p->pdds[i]; + kfd_evict_process_device(pdd); + } + + /* Run the GFX cleaner workaround on the faulting device. + * Only run once per page fault (evicted == 1 after first evict). + */ + pdd = kfd_get_process_device_data(dev, p); + if (pdd) { + /* Only call cleaner once per page fault. Run the workaround + * only on the first eviction (evicted == 1 after evict above). + */ + dqm = pdd->dev->dqm; + dqm_lock(dqm); + if (pdd->qpd.evicted == 1 && dqm->dev->kfd2kgd->hqd_gfx_cleaner) + dqm->dev->kfd2kgd->hqd_gfx_cleaner(dqm->dev->adev); + dqm_unlock(dqm); + } + + /* Restore all process devices after workaround completes */ + for (i = 0; i < p->n_pdds; i++) { + pdd = p->pdds[i]; + if (!pdd->dev->dqm) { + dev_err(dev->adev->dev, + "Invalid dqm for process %d device %d\n", pasid, i); + continue; + } + ret = pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm, + &pdd->qpd); + if (ret) + dev_err(dev->adev->dev, + "Failed to restore queues for process %d device %d: %d\n", + pasid, i, ret); + } + + kfd_unref_process(p); +} + static void event_interrupt_wq_v9(struct kfd_node *dev, const uint32_t *ih_ring_entry) { @@ -547,6 +598,9 @@ static void event_interrupt_wq_v9(struct kfd_node *dev, return; } + if (KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 2)) + gfx942_workaround_cleaner(dev, pasid); + info.vmid = vmid; info.mc_id = client_id; info.page_addr = ih_ring_entry[4] | diff --git a/drivers/gpu/drm/amd/include/kgd_kfd_interface.h b/drivers/gpu/drm/amd/include/kgd_kfd_interface.h index 01c2631bbdff..a82afc613ef3 100644 --- a/drivers/gpu/drm/amd/include/kgd_kfd_interface.h +++ b/drivers/gpu/drm/amd/include/kgd_kfd_interface.h @@ -343,6 +343,7 @@ struct kfd2kgd_calls { int (*hqd_sdma_get_counter)(struct amdgpu_device *adev, void *mqd, uint32_t num_sdma_queues_per_eng, uint64_t *val); + void (*hqd_gfx_cleaner)(struct amdgpu_device *adev); }; #endif /* KGD_KFD_INTERFACE_H_INCLUDED */ -- 2.55.0
