User queues get their first-level TBA and TMA from MES/KIQ when the queue is created. Kernel-managed VMIDs do not use this path, so program their trap registers from the gfxhub setup path.
Get the TBA and TMA addresses from the first-level trap mapping and program them for each kernel-managed VMID. The trap registers use the SRBM selector, so keep the programming under srbm_mutex and restore the VMID selector after the registers are updated. Mask the TBA and TMA address fields before writing the registers. Keep TRAP_EN disabled on the affected GFX11 ASICs to preserve the existing trap workaround. On multi-XCC hardware, program the requested XCCs and restore the VMID selector on each XCC afterwards. Keep the first-level trap VA layout inside amdgpu_trap. The gfxhub code only needs the final TBA and TMA addresses. Suggested-by: Christian König <[email protected]> Suggested-by: Alex Deucher <[email protected]> Suggested-by: Lijo Lazar <[email protected]> Cc: Felix Kuehling <[email protected]> Cc: Lancelot Six <[email protected]> Cc: James Zhu <[email protected]> Cc: Samuel Pitoiset <[email protected]> Cc: Natalie Vock <[email protected]> Cc: Timur Kristóf <[email protected]> Cc: Pierre-Eric Pelloux-Prayer <[email protected]> Signed-off-by: Srinivasan Shanmugam <[email protected]> --- Changes in v3: - Keep the separate reserved first-level trap VA range. - Add kernel VMID trap programming for gfxhub_v3_0_3. - Mask the TBA and TMA address fields before programming the registers. - Keep TRAP_EN clear on affected GFX11 ASICs. - Improve multi-XCC handling and restore the VMID selector on each XCC. drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c | 7 +++ drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h | 3 ++ drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c | 38 +++++++++++++++ drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c | 38 +++++++++++++++ drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c | 53 +++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c | 38 +++++++++++++++ drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c | 46 ++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0_3.c | 46 ++++++++++++++++++ 8 files changed, 269 insertions(+) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c index 02d29e884206..6f0dbf1c6098 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c @@ -63,6 +63,13 @@ static inline uint64_t amdgpu_trap_tba_vaddr(struct amdgpu_device *adev) return addr; } +void amdgpu_trap_get_l1_addr(struct amdgpu_device *adev, + u64 *tba_addr, u64 *tma_addr) +{ + *tba_addr = amdgpu_trap_tba_vaddr(adev); + *tma_addr = *tba_addr + AMDGPU_TRAP_TMA_OFFSET; +} + static inline bool amdgpu_trap_is_supported(struct amdgpu_device *adev) { uint32_t gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h index c78c60e3d18d..c43e2c6338e0 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h @@ -76,6 +76,9 @@ struct amdgpu_trap_usr_addr { int amdgpu_trap_init(struct amdgpu_device *adev); void amdgpu_trap_fini(struct amdgpu_device *adev); +void amdgpu_trap_get_l1_addr(struct amdgpu_device *adev, + u64 *tba_addr, u64 *tma_addr); + int amdgpu_trap_alloc(struct amdgpu_device *adev, struct amdgpu_vm *vm, struct amdgpu_trap_obj **trap_obj); diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c index 652eea6eae4a..6b5ca31074c7 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c @@ -22,6 +22,7 @@ */ #include "amdgpu.h" +#include "amdgpu_trap.h" #include "gfxhub_v11_5_0.h" #include "gc/gc_11_5_0_offset.h" @@ -290,6 +291,40 @@ static void gfxhub_v11_5_0_disable_identity_aperture(struct amdgpu_device *adev) } +static void +gfxhub_v11_5_0_program_kernel_trap_tba_tma(struct amdgpu_device *adev) +{ + u64 tba_addr, tma_addr; + unsigned int vmid; + + if (!amdgpu_trap_is_enabled(adev) || + !adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q) + return; + + amdgpu_trap_get_l1_addr(adev, &tba_addr, &tma_addr); + + WARN_ON(!IS_ALIGNED(tba_addr, 256)); + WARN_ON(!IS_ALIGNED(tma_addr, 256)); + + mutex_lock(&adev->srbm_mutex); + for (vmid = 1; vmid < adev->vm_manager.first_kfd_vmid; vmid++) { + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, vmid, 0); + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_LO, + lower_32_bits(tba_addr >> 8)); + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_HI, + (upper_32_bits(tba_addr >> 8) & + SQ_SHADER_TBA_HI__ADDR_HI_MASK) | + (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT)); + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_LO, + lower_32_bits(tma_addr >> 8)); + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_HI, + upper_32_bits(tma_addr >> 8) & + SQ_SHADER_TMA_HI__ADDR_HI_MASK); + } + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0); + mutex_unlock(&adev->srbm_mutex); +} + static void gfxhub_v11_5_0_setup_vmid_config(struct amdgpu_device *adev) { struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)]; @@ -337,6 +372,9 @@ static void gfxhub_v11_5_0_setup_vmid_config(struct amdgpu_device *adev) } hub->vm_cntx_cntl = tmp; + + if (amdgpu_trap_is_enabled(adev)) + gfxhub_v11_5_0_program_kernel_trap_tba_tma(adev); } static void gfxhub_v11_5_0_program_invalidation(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c index 6cbf837d50dd..8f3dda85e870 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c @@ -22,6 +22,7 @@ */ #include "amdgpu.h" +#include "amdgpu_trap.h" #include "gfxhub_v12_0.h" #include "gc/gc_12_0_0_offset.h" @@ -295,6 +296,40 @@ static void gfxhub_v12_0_disable_identity_aperture(struct amdgpu_device *adev) } +static void +gfxhub_v12_0_program_kernel_trap_tba_tma(struct amdgpu_device *adev) +{ + u64 tba_addr, tma_addr; + unsigned int vmid; + + if (!amdgpu_trap_is_enabled(adev) || + !adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q) + return; + + amdgpu_trap_get_l1_addr(adev, &tba_addr, &tma_addr); + + WARN_ON(!IS_ALIGNED(tba_addr, 256)); + WARN_ON(!IS_ALIGNED(tma_addr, 256)); + + mutex_lock(&adev->srbm_mutex); + for (vmid = 1; vmid < adev->vm_manager.first_kfd_vmid; vmid++) { + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, vmid, 0); + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_LO, + lower_32_bits(tba_addr >> 8)); + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_HI, + (upper_32_bits(tba_addr >> 8) & + SQ_SHADER_TBA_HI__ADDR_HI_MASK) | + (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT)); + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_LO, + lower_32_bits(tma_addr >> 8)); + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_HI, + upper_32_bits(tma_addr >> 8) & + SQ_SHADER_TMA_HI__ADDR_HI_MASK); + } + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0); + mutex_unlock(&adev->srbm_mutex); +} + static void gfxhub_v12_0_setup_vmid_config(struct amdgpu_device *adev) { struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)]; @@ -342,6 +377,9 @@ static void gfxhub_v12_0_setup_vmid_config(struct amdgpu_device *adev) } hub->vm_cntx_cntl = tmp; + + if (amdgpu_trap_is_enabled(adev)) + gfxhub_v12_0_program_kernel_trap_tba_tma(adev); } static void gfxhub_v12_0_program_invalidation(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c index 4c2fd1e6616e..d1c97f9ab554 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c @@ -21,6 +21,7 @@ * */ #include "amdgpu.h" +#include "amdgpu_trap.h" #include "amdgpu_xcp.h" #include "gfxhub_v12_1.h" @@ -406,6 +407,55 @@ static void gfxhub_v12_1_xcc_disable_identity_aperture(struct amdgpu_device *ade } } +static void +gfxhub_v12_1_program_kernel_trap_tba_tma(struct amdgpu_device *adev, + uint32_t xcc_mask) +{ + u64 tba_addr, tma_addr; + unsigned int vmid; + int xcc; + + if (!amdgpu_trap_is_enabled(adev) || + !adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q) + return; + + amdgpu_trap_get_l1_addr(adev, &tba_addr, &tma_addr); + + WARN_ON(!IS_ALIGNED(tba_addr, 256)); + WARN_ON(!IS_ALIGNED(tma_addr, 256)); + + mutex_lock(&adev->srbm_mutex); + for_each_inst(xcc, xcc_mask) { + for (vmid = 1; + vmid < adev->vm_manager.first_kfd_vmid; + vmid++) { + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, + vmid, xcc); + WREG32_SOC15(GC, GET_INST(GC, xcc), + regSQ_SHADER_TBA_LO, + lower_32_bits(tba_addr >> 8)); + WREG32_SOC15(GC, GET_INST(GC, xcc), + regSQ_SHADER_TBA_HI, + (upper_32_bits(tba_addr >> 8) & + SQ_SHADER_TBA_HI__ADDR_HI_MASK) | + (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT)); + WREG32_SOC15(GC, GET_INST(GC, xcc), + regSQ_SHADER_TMA_LO, + lower_32_bits(tma_addr >> 8)); + WREG32_SOC15(GC, GET_INST(GC, xcc), + regSQ_SHADER_TMA_HI, + upper_32_bits(tma_addr >> 8) & + SQ_SHADER_TMA_HI__ADDR_HI_MASK); + } + } + + /* Reset the VMID selector on every XCC instance. */ + for_each_inst(xcc, xcc_mask) + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, xcc); + + mutex_unlock(&adev->srbm_mutex); +} + static void gfxhub_v12_1_xcc_setup_vmid_config(struct amdgpu_device *adev, uint32_t xcc_mask) { @@ -468,6 +518,9 @@ static void gfxhub_v12_1_xcc_setup_vmid_config(struct amdgpu_device *adev, hub->vm_cntx_cntl = tmp; } + + if (amdgpu_trap_is_enabled(adev)) + gfxhub_v12_1_program_kernel_trap_tba_tma(adev, xcc_mask); } static void gfxhub_v12_1_xcc_program_invalidation(struct amdgpu_device *adev, diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c index 9ea593e2c719..5fd1abda896b 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c @@ -22,6 +22,7 @@ */ #include "amdgpu.h" +#include "amdgpu_trap.h" #include "gfxhub_v2_0.h" #include "gc/gc_10_1_0_offset.h" @@ -280,6 +281,40 @@ static void gfxhub_v2_0_disable_identity_aperture(struct amdgpu_device *adev) } +static void +gfxhub_v2_0_program_kernel_trap_tba_tma(struct amdgpu_device *adev) +{ + u64 tba_addr, tma_addr; + unsigned int vmid; + + if (!amdgpu_trap_is_enabled(adev) || + !adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q) + return; + + amdgpu_trap_get_l1_addr(adev, &tba_addr, &tma_addr); + + WARN_ON(!IS_ALIGNED(tba_addr, 256)); + WARN_ON(!IS_ALIGNED(tma_addr, 256)); + + mutex_lock(&adev->srbm_mutex); + for (vmid = 1; vmid < adev->vm_manager.first_kfd_vmid; vmid++) { + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, vmid, 0); + WREG32_SOC15(GC, 0, mmSQ_SHADER_TBA_LO, + lower_32_bits(tba_addr >> 8)); + WREG32_SOC15(GC, 0, mmSQ_SHADER_TBA_HI, + (upper_32_bits(tba_addr >> 8) & + SQ_SHADER_TBA_HI__ADDR_HI_MASK) | + (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT)); + WREG32_SOC15(GC, 0, mmSQ_SHADER_TMA_LO, + lower_32_bits(tma_addr >> 8)); + WREG32_SOC15(GC, 0, mmSQ_SHADER_TMA_HI, + upper_32_bits(tma_addr >> 8) & + SQ_SHADER_TMA_HI__ADDR_HI_MASK); + } + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0); + mutex_unlock(&adev->srbm_mutex); +} + static void gfxhub_v2_0_setup_vmid_config(struct amdgpu_device *adev) { struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)]; @@ -327,6 +362,9 @@ static void gfxhub_v2_0_setup_vmid_config(struct amdgpu_device *adev) } hub->vm_cntx_cntl = tmp; + + if (amdgpu_trap_is_enabled(adev)) + gfxhub_v2_0_program_kernel_trap_tba_tma(adev); } static void gfxhub_v2_0_program_invalidation(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c index 9e6a6e13dec0..29d712843dd3 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c @@ -22,6 +22,7 @@ */ #include "amdgpu.h" +#include "amdgpu_trap.h" #include "gfxhub_v3_0.h" #include "gc/gc_11_0_0_offset.h" @@ -287,6 +288,48 @@ static void gfxhub_v3_0_disable_identity_aperture(struct amdgpu_device *adev) } +static void +gfxhub_v3_0_program_kernel_trap_tba_tma(struct amdgpu_device *adev) +{ + u64 tba_addr, tma_addr; + u32 tba_hi; + unsigned int vmid; + + if (!amdgpu_trap_is_enabled(adev) || + !adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q) + return; + + amdgpu_trap_get_l1_addr(adev, &tba_addr, &tma_addr); + + WARN_ON(!IS_ALIGNED(tba_addr, 256)); + WARN_ON(!IS_ALIGNED(tma_addr, 256)); + + tba_hi = upper_32_bits(tba_addr >> 8) & + SQ_SHADER_TBA_HI__ADDR_HI_MASK; + + if (!amdgpu_trap_has_dbg_wa(adev)) + tba_hi |= 1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT; + + mutex_lock(&adev->srbm_mutex); + + for (vmid = 1; vmid < adev->vm_manager.first_kfd_vmid; vmid++) { + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, vmid, 0); + + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_LO, + lower_32_bits(tba_addr >> 8)); + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_HI, tba_hi); + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_LO, + lower_32_bits(tma_addr >> 8)); + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_HI, + upper_32_bits(tma_addr >> 8) & + SQ_SHADER_TMA_HI__ADDR_HI_MASK); + } + + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0); + + mutex_unlock(&adev->srbm_mutex); +} + static void gfxhub_v3_0_setup_vmid_config(struct amdgpu_device *adev) { struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)]; @@ -334,6 +377,9 @@ static void gfxhub_v3_0_setup_vmid_config(struct amdgpu_device *adev) } hub->vm_cntx_cntl = tmp; + + if (amdgpu_trap_is_enabled(adev)) + gfxhub_v3_0_program_kernel_trap_tba_tma(adev); } static void gfxhub_v3_0_program_invalidation(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0_3.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0_3.c index b3b1085c7cd3..1db1421ae51c 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0_3.c +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0_3.c @@ -23,6 +23,7 @@ #include "amdgpu.h" #include "gfxhub_v3_0_3.h" +#include "amdgpu_trap.h" #include "gc/gc_11_0_3_offset.h" #include "gc/gc_11_0_3_sh_mask.h" @@ -292,6 +293,48 @@ static void gfxhub_v3_0_3_disable_identity_aperture(struct amdgpu_device *adev) } +static void +gfxhub_v3_0_3_program_kernel_trap_tba_tma(struct amdgpu_device *adev) +{ + u64 tba_addr, tma_addr; + u32 tba_hi; + unsigned int vmid; + + if (!amdgpu_trap_is_enabled(adev) || + !adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q) + return; + + amdgpu_trap_get_l1_addr(adev, &tba_addr, &tma_addr); + + WARN_ON(!IS_ALIGNED(tba_addr, 256)); + WARN_ON(!IS_ALIGNED(tma_addr, 256)); + + tba_hi = upper_32_bits(tba_addr >> 8) & + SQ_SHADER_TBA_HI__ADDR_HI_MASK; + + if (!amdgpu_trap_has_dbg_wa(adev)) + tba_hi |= 1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT; + + mutex_lock(&adev->srbm_mutex); + + for (vmid = 1; vmid < adev->vm_manager.first_kfd_vmid; vmid++) { + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, vmid, 0); + + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_LO, + lower_32_bits(tba_addr >> 8)); + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_HI, tba_hi); + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_LO, + lower_32_bits(tma_addr >> 8)); + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_HI, + upper_32_bits(tma_addr >> 8) & + SQ_SHADER_TMA_HI__ADDR_HI_MASK); + } + + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0); + + mutex_unlock(&adev->srbm_mutex); +} + static void gfxhub_v3_0_3_setup_vmid_config(struct amdgpu_device *adev) { struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)]; @@ -339,6 +382,9 @@ static void gfxhub_v3_0_3_setup_vmid_config(struct amdgpu_device *adev) } hub->vm_cntx_cntl = tmp; + + if (amdgpu_trap_is_enabled(adev)) + gfxhub_v3_0_3_program_kernel_trap_tba_tma(adev); } static void gfxhub_v3_0_3_program_invalidation(struct amdgpu_device *adev) -- 2.34.1
