AMD General +Jay
> -----Original Message----- > From: Ba, Gang <[email protected]> > Sent: August 31, 2026 4:51 PM > To: [email protected] > Cc: Russell, Kent <[email protected]> > Subject: [PATCH] drm/amdkfd: Fix TCP XNACK scoreboard reset race > > Fix a hardware RTL bug in the TCP XNACK scoreboard reset path where a wave > receiving a no-retry FAULT from UTC and subsequently terminating may leave > its scoreboard entry uncleared when reset events arrive back-to-back. > > This can cause a later, unrelated wave using the same slot to incorrectly > inherit the XNACK error state, resulting in its VMEM operations being > dropped even though the wave never faulted. > > Signed-off-by: Gang Ba <[email protected]> > --- > .../drm/amd/amdgpu/amdgpu_amdkfd_aldebaran.c | 3 +- > .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c | 10 ++- > .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h | 1 + > drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.c | 83 +++++++++++++++++++ > drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.h | 1 + > .../gpu/drm/amd/amdkfd/kfd_int_process_v9.c | 54 ++++++++++++ > .../gpu/drm/amd/include/kgd_kfd_interface.h | 1 + > 7 files changed, 150 insertions(+), 3 deletions(-) > > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_aldebaran.c > b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_aldebaran.c > index 7e9f7a280c1b..2aaa9826c8f7 100644 > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_aldebaran.c > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_aldebaran.c > @@ -193,5 +193,6 @@ const struct kfd2kgd_calls aldebaran_kfd2kgd = { > .program_trap_handler_settings = > kgd_gfx_v9_program_trap_handler_settings, > .hqd_get_pq_addr = kgd_gfx_v9_hqd_get_pq_addr, > .hqd_reset = kgd_gfx_v9_hqd_reset, > - .hqd_sdma_get_doorbell = kgd_gfx_v9_hqd_sdma_get_doorbell > + .hqd_sdma_get_doorbell = kgd_gfx_v9_hqd_sdma_get_doorbell, > + .hqd_gfx_cleaner = kgd_gfx_v9_cleaner > }; > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c > b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c > index b30ad9701bce..81f1945c4090 100644 > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c > @@ -37,6 +37,7 @@ > #include "soc15.h" > #include "soc15d.h" > #include "gfx_v9_0.h" > +#include "gfx_v9_4_2.h" > #include "amdgpu_amdkfd_gfx_v9.h" > #include <uapi/linux/kfd_ioctl.h> > > @@ -1226,11 +1227,15 @@ uint64_t kgd_gfx_v9_hqd_reset(struct > amdgpu_device *adev, > > uint32_t kgd_gfx_v9_hqd_sdma_get_doorbell(struct amdgpu_device *adev, > int engine, int queue) > - > { > return 0; > } > > +void kgd_gfx_v9_cleaner(struct amdgpu_device *adev) > +{ > + gfx_v9_4_2_shader_cleaner_workarounds(adev); > +} > + > const struct kfd2kgd_calls gfx_v9_kfd2kgd = { > .program_sh_mem_settings = kgd_gfx_v9_program_sh_mem_settings, > .set_pasid_vmid_mapping = kgd_gfx_v9_set_pasid_vmid_mapping, > @@ -1261,5 +1266,6 @@ const struct kfd2kgd_calls gfx_v9_kfd2kgd = { > .program_trap_handler_settings = > kgd_gfx_v9_program_trap_handler_settings, > .hqd_get_pq_addr = kgd_gfx_v9_hqd_get_pq_addr, > .hqd_reset = kgd_gfx_v9_hqd_reset, > - .hqd_sdma_get_doorbell = kgd_gfx_v9_hqd_sdma_get_doorbell > + .hqd_sdma_get_doorbell = kgd_gfx_v9_hqd_sdma_get_doorbell, > + .hqd_gfx_cleaner = kgd_gfx_v9_cleaner > }; > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h > b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h > index 9f43f16a1ace..3b8b7bd2377f 100644 > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h > @@ -115,3 +115,4 @@ uint64_t kgd_gfx_v9_hqd_reset(struct amdgpu_device > *adev, > unsigned int utimeout); > uint32_t kgd_gfx_v9_hqd_sdma_get_doorbell(struct amdgpu_device *adev, > int engine, int queue); > +void kgd_gfx_v9_cleaner(struct amdgpu_device *adev); > diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.c > b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.c > index 8b84bba58752..d3e008134afb 100644 > --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.c > +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.c > @@ -343,6 +343,28 @@ const struct soc15_reg_entry > sgpr64_init_regs_aldebaran[] = { > { SOC15_REG_ENTRY(GC, 0, > regCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff }, > }; > > +static const u32 fault_cleaner_compute_shader_aldebaran[] = { > + 0xbe8000ff, 0x00001000, 0x80808100, 0x86000000, 0xbf85fffd, > 0xbf810000 > +}; > + > +const struct soc15_reg_entry fault_cleaner_init_regs_aldebaran[] = { > + { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_RESOURCE_LIMITS), > 0x0000000 }, > + { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_NUM_THREAD_X), 0x40 }, > + { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_NUM_THREAD_Y), 1 }, > + { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_NUM_THREAD_Z), 1 }, > + { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_PGM_RSRC1), 0xaf0000 }, > + { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_PGM_RSRC2), 0xc8 }, > + { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_PGM_RSRC3), 0xea4fac }, > + { SOC15_REG_ENTRY(GC, 0, > regCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff }, > + { SOC15_REG_ENTRY(GC, 0, > regCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff }, > + { SOC15_REG_ENTRY(GC, 0, > regCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff }, > + { SOC15_REG_ENTRY(GC, 0, > regCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff }, > + { SOC15_REG_ENTRY(GC, 0, > regCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff }, > + { SOC15_REG_ENTRY(GC, 0, > regCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff }, > + { SOC15_REG_ENTRY(GC, 0, > regCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff }, > + { SOC15_REG_ENTRY(GC, 0, > regCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff }, > +}; > + > static int gfx_v9_4_2_run_shader(struct amdgpu_device *adev, > struct amdgpu_ring *ring, > const u32 *shader_ptr, u32 shader_size, > @@ -692,6 +714,67 @@ static int gfx_v9_4_2_do_vgprs_init(struct > amdgpu_device *adev) > return r; > } > > +static int gfx_v9_4_2_do_cleaner(struct amdgpu_device *adev) > +{ > + int r; > + /* CU_ID: 0~15, SIMD_ID: 0~3, WAVE_ID: 0 ~ 9 */ > + int wb_size = adev->gfx.config.max_shader_engines * > + CU_ID_MAX * SIMD_ID_MAX * WAVE_ID_MAX; > + struct amdgpu_ib wb_ib; > + struct dma_fence *fence = NULL; > + u32 pattern = 0xa; > + > + /* bail if the compute ring is not ready */ > + if (!adev->gfx.compute_ring[0].sched.ready) > + return 0; > + > + /* allocate the write-back buffer from IB */ > + memset(&wb_ib, 0, sizeof(wb_ib)); > + r = amdgpu_ib_get(adev, NULL, (1 + wb_size) * sizeof(uint32_t), > + AMDGPU_IB_POOL_DIRECT, &wb_ib); > + if (r) { > + dev_err(adev->dev, "failed to get ib (%d) for wb.\n", r); > + return r; > + } > + memset(wb_ib.ptr, 0, (1 + wb_size) * sizeof(uint32_t)); > + > + r = gfx_v9_4_2_run_shader(adev, > + &adev->gfx.compute_ring[0], > + fault_cleaner_compute_shader_aldebaran, > + sizeof(fault_cleaner_compute_shader_aldebaran), > + fault_cleaner_init_regs_aldebaran, > + ARRAY_SIZE(fault_cleaner_init_regs_aldebaran), > + adev->gfx.cu_info.number * 32, > + wb_ib.gpu_addr, pattern, &fence); > + > + if (r) { > + dev_err(adev->dev, "failed to clear MI200\n"); > + goto pro_end; > + } > + > + /* wait for the GPU to finish processing the IB */ > + r = dma_fence_wait(fence, false); > + if (r) > + dev_err(adev->dev, "timeout to clear MI200\n"); > + > +pro_end: > + if (fence) > + dma_fence_put(fence); > + amdgpu_ib_free(&wb_ib, NULL); > + > + if (r) > + dev_dbg(adev->dev, "Clean MI200 Failed\n"); > + else > + dev_dbg(adev->dev, "Clean MI200 Successfully\n"); > + > + return r; > +} > + > +void gfx_v9_4_2_shader_cleaner_workarounds(struct amdgpu_device *adev) > +{ > + gfx_v9_4_2_do_cleaner(adev); > +} > + > int gfx_v9_4_2_do_edc_gpr_workarounds(struct amdgpu_device *adev) > { > /* only support when RAS is enabled */ > diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.h > b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.h > index a603724c1dfc..5c911f0f5d01 100644 > --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.h > +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.h > @@ -30,6 +30,7 @@ void gfx_v9_4_2_init_golden_registers(struct > amdgpu_device *adev, > uint32_t die_id); > void gfx_v9_4_2_init_sq(struct amdgpu_device *adev); > void gfx_v9_4_2_set_power_brake_sequence(struct amdgpu_device *adev); > +void gfx_v9_4_2_shader_cleaner_workarounds(struct amdgpu_device *adev); > int gfx_v9_4_2_do_edc_gpr_workarounds(struct amdgpu_device *adev); > > extern struct amdgpu_gfx_ras gfx_v9_4_2_ras; > diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c > b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c > index 1688d8e595f2..3a963e2da649 100644 > --- a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c > +++ b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c > @@ -359,6 +359,57 @@ static bool event_interrupt_isr_v9(struct kfd_node *dev, > !amdgpu_no_queue_eviction_on_vm_fault); > } > > +static void gfx942_workaround_cleaner(struct kfd_node *dev, uint16_t pasid) > +{ > + struct kfd_process *p; > + struct kfd_process_device *pdd = NULL; > + struct device_queue_manager *dqm; > + int i, ret; > + > + p = kfd_lookup_process_by_pasid(pasid, NULL); > + if (!p) > + return; > + > + /* Evict all process devices before running the workaround */ > + for (i = 0; i < p->n_pdds; i++) { > + pdd = p->pdds[i]; > + kfd_evict_process_device(pdd); > + } > + > + /* Run the GFX cleaner workaround on the faulting device. > + * Only run once per page fault (evicted == 1 after first evict). > + */ > + pdd = kfd_get_process_device_data(dev, p); > + if (pdd) { > + /* Only call cleaner once per page fault. Run the workaround > + * only on the first eviction (evicted == 1 after evict above). > + */ > + dqm = pdd->dev->dqm; > + dqm_lock(dqm); > + if (pdd->qpd.evicted == 1 && dqm->dev->kfd2kgd- > >hqd_gfx_cleaner) > + dqm->dev->kfd2kgd->hqd_gfx_cleaner(dqm->dev- > >adev); > + dqm_unlock(dqm); > + } > + > + /* Restore all process devices after workaround completes */ > + for (i = 0; i < p->n_pdds; i++) { > + pdd = p->pdds[i]; > + if (!pdd->dev->dqm) { > + dev_err(dev->adev->dev, > + "Invalid dqm for process %d device %d\n", > pasid, i); > + continue; > + } > + ret = pdd->dev->dqm->ops.restore_process_queues(pdd->dev- > >dqm, > + &pdd->qpd); > + if (ret) > + dev_err(dev->adev->dev, > + "Failed to restore queues for process %d device > %d: %d\n", > + pasid, i, ret); > + } > + > + kfd_unref_process(p); > +} > + > static void event_interrupt_wq_v9(struct kfd_node *dev, > const uint32_t *ih_ring_entry) > { > @@ -547,6 +598,9 @@ static void event_interrupt_wq_v9(struct kfd_node *dev, > return; > } > > + if (KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 2)) > + gfx942_workaround_cleaner(dev, pasid); > + > info.vmid = vmid; > info.mc_id = client_id; > info.page_addr = ih_ring_entry[4] | > diff --git a/drivers/gpu/drm/amd/include/kgd_kfd_interface.h > b/drivers/gpu/drm/amd/include/kgd_kfd_interface.h > index 01c2631bbdff..a82afc613ef3 100644 > --- a/drivers/gpu/drm/amd/include/kgd_kfd_interface.h > +++ b/drivers/gpu/drm/amd/include/kgd_kfd_interface.h > @@ -343,6 +343,7 @@ struct kfd2kgd_calls { > int (*hqd_sdma_get_counter)(struct amdgpu_device *adev, > void *mqd, uint32_t > num_sdma_queues_per_eng, > uint64_t *val); > + void (*hqd_gfx_cleaner)(struct amdgpu_device *adev); > }; > > #endif /* KGD_KFD_INTERFACE_H_INCLUDED */ > -- > 2.55.0
