On 10/8/26 09:05, Srinivasan Shanmugam wrote: > User queues get their first-level TBA and TMA from MES/KIQ when the > queue is created. Kernel-managed VMIDs do not use this path, so program > their trap registers from the gfxhub setup path. > > Get the reserved first-level TBA and TMA virtual addresses and program > them for each kernel-managed VMID. > > The trap registers use the SRBM selector, so keep the programming under > srbm_mutex and restore the VMID selector after the registers are > updated. > > Trap enablement differs across GPU generations. On GFX9, enable traps > for kernel-managed VMIDs through the debug trap configuration. On > GFX9.4.3, enable the per-VMID trap control on each XCC. On generations > where trap enable is encoded in SQ_SHADER_TBA_HI, program TRAP_EN > together with the TBA address. > > Mask the TBA and TMA address fields before writing the registers. Keep > TRAP_EN disabled on the affected GFX11 ASICs to preserve the existing > trap workaround. > > On multi-XCC hardware, program the requested XCCs and restore the VMID > selector on each XCC afterwards. > > Keep the first-level trap VA layout inside amdgpu_trap. The gfxhub code > only needs the final TBA and TMA addresses. > > Suggested-by: Christian König <[email protected]> > Suggested-by: Alex Deucher <[email protected]> > Suggested-by: Lijo Lazar <[email protected]> > Cc: Felix Kuehling <[email protected]> > Cc: Lancelot Six <[email protected]> > Cc: James Zhu <[email protected]> > Cc: Samuel Pitoiset <[email protected]> > Cc: Natalie Vock <[email protected]> > Cc: Timur Kristóf <[email protected]> > Cc: Pierre-Eric Pelloux-Prayer <[email protected]> > Signed-off-by: Srinivasan Shanmugam <[email protected]>
This conflicts a bit with Sunils works since he is currently replacing the VMID ranges with a VMID mask. I suggest that Sunils work goes in first and then this here is rebased on top. But it is mostly just rebasing and using different parameters for the function (mask instead of first/last) and then a different for loop. Lijo or somebody else with more HW background needs to take a look as well I think. Acked-by: Christian König <[email protected]> Regards, Christian. > --- > drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c | 9 +++- > drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h | 5 ++ > drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c | 29 ++++++++--- > drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c | 16 +++++++ > drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c | 38 +++++++++++++++ > drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c | 38 +++++++++++++++ > drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c | 53 +++++++++++++++++++++ > drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c | 41 ++++++++++++++++ > drivers/gpu/drm/amd/amdgpu/gfxhub_v1_2.c | 53 +++++++++++++++++++++ > drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c | 38 +++++++++++++++ > drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c | 42 ++++++++++++++++ > drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c | 46 ++++++++++++++++++ > drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0_3.c | 46 ++++++++++++++++++ > 13 files changed, 446 insertions(+), 8 deletions(-) > > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c > b/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c > index 89968b5b7ecb..6e7dfceeee10 100644 > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c > @@ -63,7 +63,14 @@ static inline uint64_t amdgpu_trap_tba_vaddr(struct > amdgpu_device *adev) > return addr; > } > > -static inline bool amdgpu_trap_is_supported(struct amdgpu_device *adev) > +void amdgpu_trap_get_l1_addr(struct amdgpu_device *adev, > + u64 *tba_addr, u64 *tma_addr) > +{ > + *tba_addr = amdgpu_trap_tba_vaddr(adev); > + *tma_addr = *tba_addr + AMDGPU_TRAP_TMA_OFFSET; > +} > + > +bool amdgpu_trap_is_supported(struct amdgpu_device *adev) > { > uint32_t gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0); > > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h > b/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h > index c78c60e3d18d..523f65cb3d3d 100644 > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h > @@ -76,6 +76,11 @@ struct amdgpu_trap_usr_addr { > int amdgpu_trap_init(struct amdgpu_device *adev); > void amdgpu_trap_fini(struct amdgpu_device *adev); > > +bool amdgpu_trap_is_supported(struct amdgpu_device *adev); > + > +void amdgpu_trap_get_l1_addr(struct amdgpu_device *adev, > + u64 *tba_addr, u64 *tma_addr); > + > int amdgpu_trap_alloc(struct amdgpu_device *adev, struct amdgpu_vm *vm, > struct amdgpu_trap_obj **trap_obj); > > diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c > b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c > index 1678015ce5a9..07e249500254 100644 > --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c > +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c > @@ -29,6 +29,7 @@ > > #include "amdgpu.h" > #include "amdgpu_gfx.h" > +#include "amdgpu_trap.h" > #include "soc15.h" > #include "soc15d.h" > #include "amdgpu_atomfirmware.h" > @@ -2568,21 +2569,26 @@ static void gfx_v9_0_setup_rb(struct amdgpu_device > *adev) > } > > static void gfx_v9_0_debug_trap_config_init(struct amdgpu_device *adev, > - uint32_t first_vmid, > - uint32_t last_vmid) > + u32 first_vmid, > + uint32_t last_vmid) > { > uint32_t data; > - uint32_t trap_config_vmid_mask = 0; > + u32 trap_config_vmid_mask; > int i; > > - /* Calculate trap config vmid mask */ > + data = RREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_CONFIG)); > + trap_config_vmid_mask = > + REG_GET_FIELD(data, SPI_GDBG_TRAP_CONFIG, VMID_SEL); > + > + /* Add the requested VMIDs to the existing trap configuration. */ > for (i = first_vmid; i < last_vmid; i++) > trap_config_vmid_mask |= (1 << i); > > - data = REG_SET_FIELD(0, SPI_GDBG_TRAP_CONFIG, > - VMID_SEL, trap_config_vmid_mask); > data = REG_SET_FIELD(data, SPI_GDBG_TRAP_CONFIG, > - TRAP_EN, 1); > + VMID_SEL, trap_config_vmid_mask); > + data = REG_SET_FIELD(data, SPI_GDBG_TRAP_CONFIG, > + TRAP_EN, 1); > + > WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_CONFIG), data); > WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_MASK), 0); > > @@ -4051,6 +4057,15 @@ static int gfx_v9_0_hw_init(struct amdgpu_ip_block > *ip_block) > if (r) > return r; > > + if (amdgpu_trap_is_supported(adev)) { > + if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2)) > + gfx_v9_4_2_debug_trap_config_init(adev, 1, > + > adev->vm_manager.first_kfd_vmid); > + else > + gfx_v9_0_debug_trap_config_init(adev, 1, > + > adev->vm_manager.first_kfd_vmid); > + } > + > if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2) && > !amdgpu_sriov_vf(adev)) > gfx_v9_4_2_set_power_brake_sequence(adev); > diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c > b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c > index 01ed1701c781..69a1e64620cd 100644 > --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c > +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c > @@ -25,6 +25,7 @@ > #include "amdgpu.h" > #include "amdgpu_ip.h" > #include "amdgpu_gfx.h" > +#include "amdgpu_trap.h" > #include "soc15.h" > #include "soc15d.h" > #include "soc15_common.h" > @@ -1173,6 +1174,21 @@ static void gfx_v9_4_3_xcc_init_compute_vmid(struct > amdgpu_device *adev, > SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT; > > mutex_lock(&adev->srbm_mutex); > + > + if (amdgpu_trap_is_supported(adev)) { > + for (i = 1; i < adev->vm_manager.first_kfd_vmid; i++) { > + soc15_grbm_select(adev, 0, 0, 0, i, > + GET_INST(GC, xcc_id)); > + > + data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), > + regSPI_GDBG_PER_VMID_CNTL); > + data = REG_SET_FIELD(data, SPI_GDBG_PER_VMID_CNTL, > + TRAP_EN, 1); > + WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), > + regSPI_GDBG_PER_VMID_CNTL, data); > + } > + } > + > for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) { > soc15_grbm_select(adev, 0, 0, 0, i, GET_INST(GC, xcc_id)); > /* CP and shaders */ > diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c > b/drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c > index 8be912d50601..e635d83488af 100644 > --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c > +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c > @@ -22,6 +22,7 @@ > */ > > #include "amdgpu.h" > +#include "amdgpu_trap.h" > #include "gfxhub_v11_5_0.h" > > #include "gc/gc_11_5_0_offset.h" > @@ -295,6 +296,40 @@ static void > gfxhub_v11_5_0_disable_identity_aperture(struct amdgpu_device *adev) > > } > > +static void > +gfxhub_v11_5_0_program_kernel_trap_tba_tma(struct amdgpu_device *adev) > +{ > + u64 tba_addr, tma_addr; > + unsigned int vmid; > + > + if (!amdgpu_trap_is_supported(adev) || > + !adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q) > + return; > + > + amdgpu_trap_get_l1_addr(adev, &tba_addr, &tma_addr); > + > + WARN_ON(!IS_ALIGNED(tba_addr, 256)); > + WARN_ON(!IS_ALIGNED(tma_addr, 256)); > + > + mutex_lock(&adev->srbm_mutex); > + for (vmid = 1; vmid < adev->vm_manager.first_kfd_vmid; vmid++) { > + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, vmid, 0); > + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_LO, > + lower_32_bits(tba_addr >> 8)); > + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_HI, > + (upper_32_bits(tba_addr >> 8) & > + SQ_SHADER_TBA_HI__ADDR_HI_MASK) | > + (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT)); > + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_LO, > + lower_32_bits(tma_addr >> 8)); > + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_HI, > + upper_32_bits(tma_addr >> 8) & > + SQ_SHADER_TMA_HI__ADDR_HI_MASK); > + } > + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0); > + mutex_unlock(&adev->srbm_mutex); > +} > + > static void gfxhub_v11_5_0_setup_vmid_config(struct amdgpu_device *adev) > { > struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)]; > @@ -342,6 +377,9 @@ static void gfxhub_v11_5_0_setup_vmid_config(struct > amdgpu_device *adev) > } > > hub->vm_cntx_cntl = tmp; > + > + if (amdgpu_trap_is_supported(adev)) > + gfxhub_v11_5_0_program_kernel_trap_tba_tma(adev); > } > > static void gfxhub_v11_5_0_program_invalidation(struct amdgpu_device *adev) > diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c > b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c > index 4144c1c2ee13..dc7fde6962ca 100644 > --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c > +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c > @@ -22,6 +22,7 @@ > */ > > #include "amdgpu.h" > +#include "amdgpu_trap.h" > #include "gfxhub_v12_0.h" > > #include "gc/gc_12_0_0_offset.h" > @@ -300,6 +301,40 @@ static void > gfxhub_v12_0_disable_identity_aperture(struct amdgpu_device *adev) > > } > > +static void > +gfxhub_v12_0_program_kernel_trap_tba_tma(struct amdgpu_device *adev) > +{ > + u64 tba_addr, tma_addr; > + unsigned int vmid; > + > + if (!amdgpu_trap_is_supported(adev) || > + !adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q) > + return; > + > + amdgpu_trap_get_l1_addr(adev, &tba_addr, &tma_addr); > + > + WARN_ON(!IS_ALIGNED(tba_addr, 256)); > + WARN_ON(!IS_ALIGNED(tma_addr, 256)); > + > + mutex_lock(&adev->srbm_mutex); > + for (vmid = 1; vmid < adev->vm_manager.first_kfd_vmid; vmid++) { > + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, vmid, 0); > + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_LO, > + lower_32_bits(tba_addr >> 8)); > + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_HI, > + (upper_32_bits(tba_addr >> 8) & > + SQ_SHADER_TBA_HI__ADDR_HI_MASK) | > + (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT)); > + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_LO, > + lower_32_bits(tma_addr >> 8)); > + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_HI, > + upper_32_bits(tma_addr >> 8) & > + SQ_SHADER_TMA_HI__ADDR_HI_MASK); > + } > + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0); > + mutex_unlock(&adev->srbm_mutex); > +} > + > static void gfxhub_v12_0_setup_vmid_config(struct amdgpu_device *adev) > { > struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)]; > @@ -347,6 +382,9 @@ static void gfxhub_v12_0_setup_vmid_config(struct > amdgpu_device *adev) > } > > hub->vm_cntx_cntl = tmp; > + > + if (amdgpu_trap_is_supported(adev)) > + gfxhub_v12_0_program_kernel_trap_tba_tma(adev); > } > > static void gfxhub_v12_0_program_invalidation(struct amdgpu_device *adev) > diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c > b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c > index b13967a1c8fb..78088e72606c 100644 > --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c > +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c > @@ -21,6 +21,7 @@ > * > */ > #include "amdgpu.h" > +#include "amdgpu_trap.h" > #include "amdgpu_xcp.h" > #include "gfxhub_v12_1.h" > > @@ -392,6 +393,55 @@ static void > gfxhub_v12_1_xcc_disable_identity_aperture(struct amdgpu_device *ade > } > } > > +static void > +gfxhub_v12_1_program_kernel_trap_tba_tma(struct amdgpu_device *adev, > + uint32_t xcc_mask) > +{ > + u64 tba_addr, tma_addr; > + unsigned int vmid; > + int xcc; > + > + if (!amdgpu_trap_is_supported(adev) || > + !adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q) > + return; > + > + amdgpu_trap_get_l1_addr(adev, &tba_addr, &tma_addr); > + > + WARN_ON(!IS_ALIGNED(tba_addr, 256)); > + WARN_ON(!IS_ALIGNED(tma_addr, 256)); > + > + mutex_lock(&adev->srbm_mutex); > + for_each_inst(xcc, xcc_mask) { > + for (vmid = 1; > + vmid < adev->vm_manager.first_kfd_vmid; > + vmid++) { > + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, > + vmid, xcc); > + WREG32_SOC15(GC, GET_INST(GC, xcc), > + regSQ_SHADER_TBA_LO, > + lower_32_bits(tba_addr >> 8)); > + WREG32_SOC15(GC, GET_INST(GC, xcc), > + regSQ_SHADER_TBA_HI, > + (upper_32_bits(tba_addr >> 8) & > + SQ_SHADER_TBA_HI__ADDR_HI_MASK) | > + (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT)); > + WREG32_SOC15(GC, GET_INST(GC, xcc), > + regSQ_SHADER_TMA_LO, > + lower_32_bits(tma_addr >> 8)); > + WREG32_SOC15(GC, GET_INST(GC, xcc), > + regSQ_SHADER_TMA_HI, > + upper_32_bits(tma_addr >> 8) & > + SQ_SHADER_TMA_HI__ADDR_HI_MASK); > + } > + } > + > + /* Reset the VMID selector on every XCC instance. */ > + for_each_inst(xcc, xcc_mask) > + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, xcc); > + > + mutex_unlock(&adev->srbm_mutex); > +} > + > static void gfxhub_v12_1_xcc_setup_vmid_config(struct amdgpu_device *adev, > uint32_t xcc_mask) > { > @@ -457,6 +507,9 @@ static void gfxhub_v12_1_xcc_setup_vmid_config(struct > amdgpu_device *adev, > > hub->vm_cntx_cntl = tmp; > } > + > + if (amdgpu_trap_is_supported(adev)) > + gfxhub_v12_1_program_kernel_trap_tba_tma(adev, xcc_mask); > } > > static void gfxhub_v12_1_xcc_program_invalidation(struct amdgpu_device *adev, > diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c > b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c > index c35856180c3c..dbd993ae1d0e 100644 > --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c > +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c > @@ -21,6 +21,7 @@ > * > */ > #include "amdgpu.h" > +#include "amdgpu_trap.h" > #include "gfxhub_v1_0.h" > #include "gfxhub_v1_1.h" > > @@ -252,6 +253,43 @@ static void gfxhub_v1_0_disable_identity_aperture(struct > amdgpu_device *adev) > > } > > +static void > +gfxhub_v1_0_program_kernel_trap_tba_tma(struct amdgpu_device *adev) > +{ > + u64 tba_addr, tma_addr; > + unsigned int vmid; > + > + if (!amdgpu_trap_is_supported(adev) || > + !adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q) > + return; > + > + amdgpu_trap_get_l1_addr(adev, &tba_addr, &tma_addr); > + > + WARN_ON(!IS_ALIGNED(tba_addr, 256)); > + WARN_ON(!IS_ALIGNED(tma_addr, 256)); > + > + mutex_lock(&adev->srbm_mutex); > + > + for (vmid = 1; vmid < adev->vm_manager.first_kfd_vmid; vmid++) { > + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, vmid, 0); > + > + WREG32_SOC15(GC, 0, mmSQ_SHADER_TBA_LO, > + lower_32_bits(tba_addr >> 8)); > + WREG32_SOC15(GC, 0, mmSQ_SHADER_TBA_HI, > + upper_32_bits(tba_addr >> 8) & > + SQ_SHADER_TBA_HI__ADDR_HI_MASK); > + WREG32_SOC15(GC, 0, mmSQ_SHADER_TMA_LO, > + lower_32_bits(tma_addr >> 8)); > + WREG32_SOC15(GC, 0, mmSQ_SHADER_TMA_HI, > + upper_32_bits(tma_addr >> 8) & > + SQ_SHADER_TMA_HI__ADDR_HI_MASK); > + } > + > + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0); > + > + mutex_unlock(&adev->srbm_mutex); > +} > + > static void gfxhub_v1_0_setup_vmid_config(struct amdgpu_device *adev) > { > struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)]; > @@ -310,6 +348,9 @@ static void gfxhub_v1_0_setup_vmid_config(struct > amdgpu_device *adev) > i * hub->ctx_addr_distance, > upper_32_bits(adev->vm_manager.max_pfn - > 1)); > } > + > + if (amdgpu_trap_is_supported(adev)) > + gfxhub_v1_0_program_kernel_trap_tba_tma(adev); > } > > static void gfxhub_v1_0_program_invalidation(struct amdgpu_device *adev) > diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_2.c > b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_2.c > index e89813590539..76c2b6b74903 100644 > --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_2.c > +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_2.c > @@ -22,6 +22,7 @@ > */ > #include "amdgpu.h" > #include "amdgpu_ip.h" > +#include "amdgpu_trap.h" > #include "amdgpu_xcp.h" > #include "gfxhub_v1_2.h" > #include "gfxhub_v1_1.h" > @@ -328,6 +329,55 @@ gfxhub_v1_2_per_process_xnack_support(struct > amdgpu_device *adev) > return !!adev->aid_mask; > } > > +static void > +gfxhub_v1_2_program_kernel_trap_tba_tma(struct amdgpu_device *adev, > + uint32_t xcc_mask) > +{ > + u64 tba_addr, tma_addr; > + unsigned int vmid; > + int xcc; > + > + if (!amdgpu_trap_is_supported(adev) || > + !adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q) > + return; > + > + amdgpu_trap_get_l1_addr(adev, &tba_addr, &tma_addr); > + > + WARN_ON(!IS_ALIGNED(tba_addr, 256)); > + WARN_ON(!IS_ALIGNED(tma_addr, 256)); > + > + mutex_lock(&adev->srbm_mutex); > + > + for_each_inst(xcc, xcc_mask) { > + for (vmid = 1; > + vmid < adev->vm_manager.first_kfd_vmid; > + vmid++) { > + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, > + vmid, xcc); > + > + WREG32_SOC15(GC, GET_INST(GC, xcc), > + regSQ_SHADER_TBA_LO, > + lower_32_bits(tba_addr >> 8)); > + WREG32_SOC15(GC, GET_INST(GC, xcc), > + regSQ_SHADER_TBA_HI, > + upper_32_bits(tba_addr >> 8) & > + SQ_SHADER_TBA_HI__ADDR_HI_MASK); > + WREG32_SOC15(GC, GET_INST(GC, xcc), > + regSQ_SHADER_TMA_LO, > + lower_32_bits(tma_addr >> 8)); > + WREG32_SOC15(GC, GET_INST(GC, xcc), > + regSQ_SHADER_TMA_HI, > + upper_32_bits(tma_addr >> 8) & > + SQ_SHADER_TMA_HI__ADDR_HI_MASK); > + } > + } > + > + for_each_inst(xcc, xcc_mask) > + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, xcc); > + > + mutex_unlock(&adev->srbm_mutex); > +} > + > static void gfxhub_v1_2_xcc_setup_vmid_config(struct amdgpu_device *adev, > uint32_t xcc_mask) > { > @@ -398,6 +448,9 @@ static void gfxhub_v1_2_xcc_setup_vmid_config(struct > amdgpu_device *adev, > > upper_32_bits(adev->vm_manager.max_pfn - 1)); > } > } > + > + if (amdgpu_trap_is_supported(adev)) > + gfxhub_v1_2_program_kernel_trap_tba_tma(adev, xcc_mask); > } > > static void gfxhub_v1_2_xcc_program_invalidation(struct amdgpu_device *adev, > diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c > b/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c > index 04778f9f95bd..da4e1fa34d8d 100644 > --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c > +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c > @@ -22,6 +22,7 @@ > */ > > #include "amdgpu.h" > +#include "amdgpu_trap.h" > #include "gfxhub_v2_0.h" > > #include "gc/gc_10_1_0_offset.h" > @@ -285,6 +286,40 @@ static void gfxhub_v2_0_disable_identity_aperture(struct > amdgpu_device *adev) > > } > > +static void > +gfxhub_v2_0_program_kernel_trap_tba_tma(struct amdgpu_device *adev) > +{ > + u64 tba_addr, tma_addr; > + unsigned int vmid; > + > + if (!amdgpu_trap_is_supported(adev) || > + !adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q) > + return; > + > + amdgpu_trap_get_l1_addr(adev, &tba_addr, &tma_addr); > + > + WARN_ON(!IS_ALIGNED(tba_addr, 256)); > + WARN_ON(!IS_ALIGNED(tma_addr, 256)); > + > + mutex_lock(&adev->srbm_mutex); > + for (vmid = 1; vmid < adev->vm_manager.first_kfd_vmid; vmid++) { > + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, vmid, 0); > + WREG32_SOC15(GC, 0, mmSQ_SHADER_TBA_LO, > + lower_32_bits(tba_addr >> 8)); > + WREG32_SOC15(GC, 0, mmSQ_SHADER_TBA_HI, > + (upper_32_bits(tba_addr >> 8) & > + SQ_SHADER_TBA_HI__ADDR_HI_MASK) | > + (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT)); > + WREG32_SOC15(GC, 0, mmSQ_SHADER_TMA_LO, > + lower_32_bits(tma_addr >> 8)); > + WREG32_SOC15(GC, 0, mmSQ_SHADER_TMA_HI, > + upper_32_bits(tma_addr >> 8) & > + SQ_SHADER_TMA_HI__ADDR_HI_MASK); > + } > + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0); > + mutex_unlock(&adev->srbm_mutex); > +} > + > static void gfxhub_v2_0_setup_vmid_config(struct amdgpu_device *adev) > { > struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)]; > @@ -332,6 +367,9 @@ static void gfxhub_v2_0_setup_vmid_config(struct > amdgpu_device *adev) > } > > hub->vm_cntx_cntl = tmp; > + > + if (amdgpu_trap_is_supported(adev)) > + gfxhub_v2_0_program_kernel_trap_tba_tma(adev); > } > > static void gfxhub_v2_0_program_invalidation(struct amdgpu_device *adev) > diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c > b/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c > index 6c77c17878f8..c0ff251bcbff 100644 > --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c > +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c > @@ -22,6 +22,7 @@ > */ > > #include "amdgpu.h" > +#include "amdgpu_trap.h" > #include "gfxhub_v2_1.h" > > #include "gc/gc_10_3_0_offset.h" > @@ -297,6 +298,44 @@ static void gfxhub_v2_1_disable_identity_aperture(struct > amdgpu_device *adev) > > } > > +static void > +gfxhub_v2_1_program_kernel_trap_tba_tma(struct amdgpu_device *adev) > +{ > + u64 tba_addr, tma_addr; > + unsigned int vmid; > + > + if (!amdgpu_trap_is_supported(adev) || > + !adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q) > + return; > + > + amdgpu_trap_get_l1_addr(adev, &tba_addr, &tma_addr); > + > + WARN_ON(!IS_ALIGNED(tba_addr, 256)); > + WARN_ON(!IS_ALIGNED(tma_addr, 256)); > + > + mutex_lock(&adev->srbm_mutex); > + > + for (vmid = 1; vmid < adev->vm_manager.first_kfd_vmid; vmid++) { > + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, vmid, 0); > + > + WREG32_SOC15(GC, 0, mmSQ_SHADER_TBA_LO, > + lower_32_bits(tba_addr >> 8)); > + WREG32_SOC15(GC, 0, mmSQ_SHADER_TBA_HI, > + (upper_32_bits(tba_addr >> 8) & > + SQ_SHADER_TBA_HI__ADDR_HI_MASK) | > + (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT)); > + WREG32_SOC15(GC, 0, mmSQ_SHADER_TMA_LO, > + lower_32_bits(tma_addr >> 8)); > + WREG32_SOC15(GC, 0, mmSQ_SHADER_TMA_HI, > + upper_32_bits(tma_addr >> 8) & > + SQ_SHADER_TMA_HI__ADDR_HI_MASK); > + } > + > + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0); > + > + mutex_unlock(&adev->srbm_mutex); > +} > + > static void gfxhub_v2_1_setup_vmid_config(struct amdgpu_device *adev) > { > struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)]; > @@ -344,6 +383,9 @@ static void gfxhub_v2_1_setup_vmid_config(struct > amdgpu_device *adev) > } > > hub->vm_cntx_cntl = tmp; > + > + if (amdgpu_trap_is_supported(adev)) > + gfxhub_v2_1_program_kernel_trap_tba_tma(adev); > } > > static void gfxhub_v2_1_program_invalidation(struct amdgpu_device *adev) > diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c > b/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c > index e2b9fd0c8af0..1d9d4d41c42c 100644 > --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c > +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c > @@ -22,6 +22,7 @@ > */ > > #include "amdgpu.h" > +#include "amdgpu_trap.h" > #include "gfxhub_v3_0.h" > > #include "gc/gc_11_0_0_offset.h" > @@ -292,6 +293,48 @@ static void gfxhub_v3_0_disable_identity_aperture(struct > amdgpu_device *adev) > > } > > +static void > +gfxhub_v3_0_program_kernel_trap_tba_tma(struct amdgpu_device *adev) > +{ > + u64 tba_addr, tma_addr; > + u32 tba_hi; > + unsigned int vmid; > + > + if (!amdgpu_trap_is_supported(adev) || > + !adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q) > + return; > + > + amdgpu_trap_get_l1_addr(adev, &tba_addr, &tma_addr); > + > + WARN_ON(!IS_ALIGNED(tba_addr, 256)); > + WARN_ON(!IS_ALIGNED(tma_addr, 256)); > + > + tba_hi = upper_32_bits(tba_addr >> 8) & > + SQ_SHADER_TBA_HI__ADDR_HI_MASK; > + > + if (!amdgpu_trap_has_dbg_wa(adev)) > + tba_hi |= 1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT; > + > + mutex_lock(&adev->srbm_mutex); > + > + for (vmid = 1; vmid < adev->vm_manager.first_kfd_vmid; vmid++) { > + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, vmid, 0); > + > + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_LO, > + lower_32_bits(tba_addr >> 8)); > + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_HI, tba_hi); > + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_LO, > + lower_32_bits(tma_addr >> 8)); > + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_HI, > + upper_32_bits(tma_addr >> 8) & > + SQ_SHADER_TMA_HI__ADDR_HI_MASK); > + } > + > + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0); > + > + mutex_unlock(&adev->srbm_mutex); > +} > + > static void gfxhub_v3_0_setup_vmid_config(struct amdgpu_device *adev) > { > struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)]; > @@ -339,6 +382,9 @@ static void gfxhub_v3_0_setup_vmid_config(struct > amdgpu_device *adev) > } > > hub->vm_cntx_cntl = tmp; > + > + if (amdgpu_trap_is_supported(adev)) > + gfxhub_v3_0_program_kernel_trap_tba_tma(adev); > } > > static void gfxhub_v3_0_program_invalidation(struct amdgpu_device *adev) > diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0_3.c > b/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0_3.c > index fa8b966b7ea1..7f2a0b21cd4f 100644 > --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0_3.c > +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0_3.c > @@ -23,6 +23,7 @@ > > #include "amdgpu.h" > #include "gfxhub_v3_0_3.h" > +#include "amdgpu_trap.h" > > #include "gc/gc_11_0_3_offset.h" > #include "gc/gc_11_0_3_sh_mask.h" > @@ -297,6 +298,48 @@ static void > gfxhub_v3_0_3_disable_identity_aperture(struct amdgpu_device *adev) > > } > > +static void > +gfxhub_v3_0_3_program_kernel_trap_tba_tma(struct amdgpu_device *adev) > +{ > + u64 tba_addr, tma_addr; > + u32 tba_hi; > + unsigned int vmid; > + > + if (!amdgpu_trap_is_supported(adev) || > + !adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q) > + return; > + > + amdgpu_trap_get_l1_addr(adev, &tba_addr, &tma_addr); > + > + WARN_ON(!IS_ALIGNED(tba_addr, 256)); > + WARN_ON(!IS_ALIGNED(tma_addr, 256)); > + > + tba_hi = upper_32_bits(tba_addr >> 8) & > + SQ_SHADER_TBA_HI__ADDR_HI_MASK; > + > + if (!amdgpu_trap_has_dbg_wa(adev)) > + tba_hi |= 1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT; > + > + mutex_lock(&adev->srbm_mutex); > + > + for (vmid = 1; vmid < adev->vm_manager.first_kfd_vmid; vmid++) { > + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, vmid, 0); > + > + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_LO, > + lower_32_bits(tba_addr >> 8)); > + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_HI, tba_hi); > + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_LO, > + lower_32_bits(tma_addr >> 8)); > + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_HI, > + upper_32_bits(tma_addr >> 8) & > + SQ_SHADER_TMA_HI__ADDR_HI_MASK); > + } > + > + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0); > + > + mutex_unlock(&adev->srbm_mutex); > +} > + > static void gfxhub_v3_0_3_setup_vmid_config(struct amdgpu_device *adev) > { > struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)]; > @@ -344,6 +387,9 @@ static void gfxhub_v3_0_3_setup_vmid_config(struct > amdgpu_device *adev) > } > > hub->vm_cntx_cntl = tmp; > + > + if (amdgpu_trap_is_supported(adev)) > + gfxhub_v3_0_3_program_kernel_trap_tba_tma(adev); > } > > static void gfxhub_v3_0_3_program_invalidation(struct amdgpu_device *adev)
