User queues get their first-level TBA and TMA from MES/KIQ when the queue is created. Kernel-managed VMIDs do not use this path, so program their trap registers from the gfxhub setup path.
Get the reserved first-level TBA and TMA virtual addresses and program them for kernel-managed VMIDs. User-queue VMIDs continue to receive their trap state through the existing MES/KIQ programming path. The trap registers use the SRBM selector, so keep the programming under srbm_mutex and restore the VMID selector after the registers are updated. Trap enablement differs across GPU generations. On GFX9, enable traps for user-queue VMIDs through the debug trap configuration and program kernel-managed VMIDs through the gfxhub path. On GFX9.4.3, enable the per-VMID trap control on each XCC. On generations where trap enable is encoded in SQ_SHADER_TBA_HI, program TRAP_EN together with the TBA address. Mask the TBA and TMA address fields before writing the registers. Keep TRAP_EN disabled on the affected GFX11 ASICs to preserve the existing trap workaround. On multi-XCC hardware, program the requested XCCs and restore the VMID selector on each XCC afterwards. Keep the first-level trap VA layout inside amdgpu_trap. The gfxhub code only needs the final TBA and TMA addresses. Suggested-by: Christian König <[email protected]> Suggested-by: Alex Deucher <[email protected]> Suggested-by: Lijo Lazar <[email protected]> Cc: Felix Kuehling <[email protected]> Cc: Lancelot Six <[email protected]> Cc: James Zhu <[email protected]> Cc: Samuel Pitoiset <[email protected]> Cc: Natalie Vock <[email protected]> Cc: Timur Kristóf <[email protected]> Cc: Pierre-Eric Pelloux-Prayer <[email protected]> Signed-off-by: Srinivasan Shanmugam <[email protected]> Acked-by: Christian König <[email protected]> --- drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c | 9 ++- drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h | 5 ++ drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c | 28 ++++++-- drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c | 75 +++++++++++++++++---- drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c | 50 ++++++++++++++ drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c | 50 ++++++++++++++ drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c | 61 +++++++++++++++++ drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c | 47 +++++++++++++ drivers/gpu/drm/amd/amdgpu/gfxhub_v1_2.c | 59 ++++++++++++++++ drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c | 50 ++++++++++++++ drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c | 50 ++++++++++++++ drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c | 55 +++++++++++++++ drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0_3.c | 55 +++++++++++++++ 13 files changed, 576 insertions(+), 18 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c index 20d1dcc14160..0e37770e660f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c @@ -63,7 +63,14 @@ static inline uint64_t amdgpu_trap_tba_vaddr(struct amdgpu_device *adev) return addr; } -static inline bool amdgpu_trap_is_supported(struct amdgpu_device *adev) +void amdgpu_trap_get_l1_addr(struct amdgpu_device *adev, + u64 *tba_addr, u64 *tma_addr) +{ + *tba_addr = amdgpu_trap_tba_vaddr(adev); + *tma_addr = *tba_addr + AMDGPU_TRAP_TMA_OFFSET; +} + +bool amdgpu_trap_is_supported(struct amdgpu_device *adev) { uint32_t gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h index c78c60e3d18d..523f65cb3d3d 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h @@ -76,6 +76,11 @@ struct amdgpu_trap_usr_addr { int amdgpu_trap_init(struct amdgpu_device *adev); void amdgpu_trap_fini(struct amdgpu_device *adev); +bool amdgpu_trap_is_supported(struct amdgpu_device *adev); + +void amdgpu_trap_get_l1_addr(struct amdgpu_device *adev, + u64 *tba_addr, u64 *tma_addr); + int amdgpu_trap_alloc(struct amdgpu_device *adev, struct amdgpu_vm *vm, struct amdgpu_trap_obj **trap_obj); diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c index 2e2673a6dbbf..7490f366ac49 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c @@ -29,6 +29,7 @@ #include "amdgpu.h" #include "amdgpu_gfx.h" +#include "amdgpu_trap.h" #include "soc15.h" #include "soc15d.h" #include "amdgpu_atomfirmware.h" @@ -2570,14 +2571,25 @@ static void gfx_v9_0_setup_rb(struct amdgpu_device *adev) } static void gfx_v9_0_debug_trap_config_init(struct amdgpu_device *adev, - uint32_t vmid_mask) + uint32_t vmid_mask) { uint32_t data; + u32 trap_config_vmid_mask; + + data = RREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_CONFIG)); + + trap_config_vmid_mask = + REG_GET_FIELD(data, SPI_GDBG_TRAP_CONFIG, VMID_SEL); + + /* Add the requested VMIDs to the existing trap configuration. */ + trap_config_vmid_mask |= vmid_mask; + + data = REG_SET_FIELD(data, SPI_GDBG_TRAP_CONFIG, + VMID_SEL, trap_config_vmid_mask); - data = REG_SET_FIELD(0, SPI_GDBG_TRAP_CONFIG, - VMID_SEL, vmid_mask); data = REG_SET_FIELD(data, SPI_GDBG_TRAP_CONFIG, - TRAP_EN, 1); + TRAP_EN, 1); + WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_CONFIG), data); WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_MASK), 0); @@ -4046,6 +4058,14 @@ static int gfx_v9_0_hw_init(struct amdgpu_ip_block *ip_block) if (r) return r; + if (amdgpu_trap_is_supported(adev)) { + if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2)) + gfx_v9_4_2_debug_trap_config_init(adev); + else + gfx_v9_0_debug_trap_config_init(adev, + adev->vm_manager.vmid_uq_mask_gfxhub); + } + if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2) && !amdgpu_sriov_vf(adev)) gfx_v9_4_2_set_power_brake_sequence(adev); diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c index da031caa4e9e..06d7f73c58ad 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c @@ -25,6 +25,7 @@ #include "amdgpu.h" #include "amdgpu_ip.h" #include "amdgpu_gfx.h" +#include "amdgpu_trap.h" #include "soc15.h" #include "soc15d.h" #include "soc15_common.h" @@ -1168,25 +1169,66 @@ static void gfx_v9_4_3_xcc_init_compute_vmid(struct amdgpu_device *adev, * Scratch: 0x60000001'00000000 - 0x60000002'00000000 (4GB) * GPUVM: 0x60010000'00000000 - 0x60020000'00000000 (1TB) */ - sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16); + sh_mem_bases = DEFAULT_SH_MEM_BASES | + (DEFAULT_SH_MEM_BASES << 16); sh_mem_config = SH_MEM_ADDRESS_MODE_64 | SH_MEM_ALIGNMENT_MODE_UNALIGNED << SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT; mutex_lock(&adev->srbm_mutex); + + /* + * Enable traps for kernel-managed VMIDs. + */ + if (amdgpu_trap_is_supported(adev)) { + for (i = 1; i < AMDGPU_NUM_VMID; i++) { + if (adev->vm_manager.vmid_uq_mask_gfxhub & BIT(i)) + continue; + + soc15_grbm_select(adev, 0, 0, 0, i, + GET_INST(GC, xcc_id)); + + data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), + regSPI_GDBG_PER_VMID_CNTL); + + data = REG_SET_FIELD(data, + SPI_GDBG_PER_VMID_CNTL, + TRAP_EN, 1); + + WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), + regSPI_GDBG_PER_VMID_CNTL, + data); + } + } + for_each_uq_gfx_vmid(i, adev) { - soc15_grbm_select(adev, 0, 0, 0, i, GET_INST(GC, xcc_id)); + soc15_grbm_select(adev, 0, 0, 0, i, + GET_INST(GC, xcc_id)); + /* CP and shaders */ - WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regSH_MEM_CONFIG, sh_mem_config); - WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regSH_MEM_BASES, sh_mem_bases); + WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), + regSH_MEM_CONFIG, sh_mem_config); + + WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), + regSH_MEM_BASES, sh_mem_bases); - /* Enable trap for each kfd vmid. */ - data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regSPI_GDBG_PER_VMID_CNTL); - data = REG_SET_FIELD(data, SPI_GDBG_PER_VMID_CNTL, TRAP_EN, 1); - WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regSPI_GDBG_PER_VMID_CNTL, data); + /* Enable trap for each user queue VMID. */ + data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), + regSPI_GDBG_PER_VMID_CNTL); + + data = REG_SET_FIELD(data, + SPI_GDBG_PER_VMID_CNTL, + TRAP_EN, 1); + + WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), + regSPI_GDBG_PER_VMID_CNTL, + data); } - soc15_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id)); + + soc15_grbm_select(adev, 0, 0, 0, 0, + GET_INST(GC, xcc_id)); + mutex_unlock(&adev->srbm_mutex); /* @@ -1194,10 +1236,17 @@ static void gfx_v9_4_3_xcc_init_compute_vmid(struct amdgpu_device *adev, * access. These should be enabled by FW for target VMIDs. */ for_each_uq_gfx_vmid(i, adev) { - WREG32_SOC15_OFFSET(GC, GET_INST(GC, xcc_id), regGDS_VMID0_BASE, 2 * i, 0); - WREG32_SOC15_OFFSET(GC, GET_INST(GC, xcc_id), regGDS_VMID0_SIZE, 2 * i, 0); - WREG32_SOC15_OFFSET(GC, GET_INST(GC, xcc_id), regGDS_GWS_VMID0, i, 0); - WREG32_SOC15_OFFSET(GC, GET_INST(GC, xcc_id), regGDS_OA_VMID0, i, 0); + WREG32_SOC15_OFFSET(GC, GET_INST(GC, xcc_id), + regGDS_VMID0_BASE, 2 * i, 0); + + WREG32_SOC15_OFFSET(GC, GET_INST(GC, xcc_id), + regGDS_VMID0_SIZE, 2 * i, 0); + + WREG32_SOC15_OFFSET(GC, GET_INST(GC, xcc_id), + regGDS_GWS_VMID0, i, 0); + + WREG32_SOC15_OFFSET(GC, GET_INST(GC, xcc_id), + regGDS_OA_VMID0, i, 0); } } diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c index 8be912d50601..5b9efd09493a 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c @@ -22,6 +22,7 @@ */ #include "amdgpu.h" +#include "amdgpu_trap.h" #include "gfxhub_v11_5_0.h" #include "gc/gc_11_5_0_offset.h" @@ -295,6 +296,52 @@ static void gfxhub_v11_5_0_disable_identity_aperture(struct amdgpu_device *adev) } +static void +gfxhub_v11_5_0_program_kernel_trap_tba_tma(struct amdgpu_device *adev) +{ + u64 tba_addr, tma_addr; + unsigned int vmid; + + if (!amdgpu_trap_is_supported(adev) || + !adev->gfx.funcs || + !adev->gfx.funcs->select_me_pipe_q) + return; + + amdgpu_trap_get_l1_addr(adev, &tba_addr, &tma_addr); + + WARN_ON(!IS_ALIGNED(tba_addr, 256)); + WARN_ON(!IS_ALIGNED(tma_addr, 256)); + + mutex_lock(&adev->srbm_mutex); + + for (vmid = 1; vmid < AMDGPU_NUM_VMID; vmid++) { + /* Program only kernel-managed VMIDs. */ + if (adev->vm_manager.vmid_uq_mask_gfxhub & BIT(vmid)) + continue; + + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, vmid, 0); + + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_LO, + lower_32_bits(tba_addr >> 8)); + + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_HI, + (upper_32_bits(tba_addr >> 8) & + SQ_SHADER_TBA_HI__ADDR_HI_MASK) | + (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT)); + + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_LO, + lower_32_bits(tma_addr >> 8)); + + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_HI, + upper_32_bits(tma_addr >> 8) & + SQ_SHADER_TMA_HI__ADDR_HI_MASK); + } + + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0); + + mutex_unlock(&adev->srbm_mutex); +} + static void gfxhub_v11_5_0_setup_vmid_config(struct amdgpu_device *adev) { struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)]; @@ -342,6 +389,9 @@ static void gfxhub_v11_5_0_setup_vmid_config(struct amdgpu_device *adev) } hub->vm_cntx_cntl = tmp; + + if (amdgpu_trap_is_supported(adev)) + gfxhub_v11_5_0_program_kernel_trap_tba_tma(adev); } static void gfxhub_v11_5_0_program_invalidation(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c index 4144c1c2ee13..f0ca90f8b50e 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c @@ -22,6 +22,7 @@ */ #include "amdgpu.h" +#include "amdgpu_trap.h" #include "gfxhub_v12_0.h" #include "gc/gc_12_0_0_offset.h" @@ -300,6 +301,52 @@ static void gfxhub_v12_0_disable_identity_aperture(struct amdgpu_device *adev) } +static void +gfxhub_v12_0_program_kernel_trap_tba_tma(struct amdgpu_device *adev) +{ + u64 tba_addr, tma_addr; + unsigned int vmid; + + if (!amdgpu_trap_is_supported(adev) || + !adev->gfx.funcs || + !adev->gfx.funcs->select_me_pipe_q) + return; + + amdgpu_trap_get_l1_addr(adev, &tba_addr, &tma_addr); + + WARN_ON(!IS_ALIGNED(tba_addr, 256)); + WARN_ON(!IS_ALIGNED(tma_addr, 256)); + + mutex_lock(&adev->srbm_mutex); + + for (vmid = 1; vmid < AMDGPU_NUM_VMID; vmid++) { + /* Program only kernel-managed VMIDs. */ + if (adev->vm_manager.vmid_uq_mask_gfxhub & BIT(vmid)) + continue; + + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, vmid, 0); + + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_LO, + lower_32_bits(tba_addr >> 8)); + + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_HI, + (upper_32_bits(tba_addr >> 8) & + SQ_SHADER_TBA_HI__ADDR_HI_MASK) | + (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT)); + + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_LO, + lower_32_bits(tma_addr >> 8)); + + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_HI, + upper_32_bits(tma_addr >> 8) & + SQ_SHADER_TMA_HI__ADDR_HI_MASK); + } + + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0); + + mutex_unlock(&adev->srbm_mutex); +} + static void gfxhub_v12_0_setup_vmid_config(struct amdgpu_device *adev) { struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)]; @@ -347,6 +394,9 @@ static void gfxhub_v12_0_setup_vmid_config(struct amdgpu_device *adev) } hub->vm_cntx_cntl = tmp; + + if (amdgpu_trap_is_supported(adev)) + gfxhub_v12_0_program_kernel_trap_tba_tma(adev); } static void gfxhub_v12_0_program_invalidation(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c index b13967a1c8fb..a35ca7fcb198 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c @@ -21,6 +21,7 @@ * */ #include "amdgpu.h" +#include "amdgpu_trap.h" #include "amdgpu_xcp.h" #include "gfxhub_v12_1.h" @@ -392,6 +393,63 @@ static void gfxhub_v12_1_xcc_disable_identity_aperture(struct amdgpu_device *ade } } +static void +gfxhub_v12_1_program_kernel_trap_tba_tma(struct amdgpu_device *adev, + uint32_t xcc_mask) +{ + u64 tba_addr, tma_addr; + unsigned int vmid; + int xcc; + + if (!amdgpu_trap_is_supported(adev) || + !adev->gfx.funcs || + !adev->gfx.funcs->select_me_pipe_q) + return; + + amdgpu_trap_get_l1_addr(adev, &tba_addr, &tma_addr); + + WARN_ON(!IS_ALIGNED(tba_addr, 256)); + WARN_ON(!IS_ALIGNED(tma_addr, 256)); + + mutex_lock(&adev->srbm_mutex); + + for_each_inst(xcc, xcc_mask) { + for (vmid = 1; vmid < AMDGPU_NUM_VMID; vmid++) { + /* Program only kernel-managed VMIDs. */ + if (adev->vm_manager.vmid_uq_mask_gfxhub & BIT(vmid)) + continue; + + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, + vmid, xcc); + + WREG32_SOC15(GC, GET_INST(GC, xcc), + regSQ_SHADER_TBA_LO, + lower_32_bits(tba_addr >> 8)); + + WREG32_SOC15(GC, GET_INST(GC, xcc), + regSQ_SHADER_TBA_HI, + (upper_32_bits(tba_addr >> 8) & + SQ_SHADER_TBA_HI__ADDR_HI_MASK) | + (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT)); + + WREG32_SOC15(GC, GET_INST(GC, xcc), + regSQ_SHADER_TMA_LO, + lower_32_bits(tma_addr >> 8)); + + WREG32_SOC15(GC, GET_INST(GC, xcc), + regSQ_SHADER_TMA_HI, + upper_32_bits(tma_addr >> 8) & + SQ_SHADER_TMA_HI__ADDR_HI_MASK); + } + } + + /* Reset the VMID selector on every XCC instance. */ + for_each_inst(xcc, xcc_mask) + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, xcc); + + mutex_unlock(&adev->srbm_mutex); +} + static void gfxhub_v12_1_xcc_setup_vmid_config(struct amdgpu_device *adev, uint32_t xcc_mask) { @@ -457,6 +515,9 @@ static void gfxhub_v12_1_xcc_setup_vmid_config(struct amdgpu_device *adev, hub->vm_cntx_cntl = tmp; } + + if (amdgpu_trap_is_supported(adev)) + gfxhub_v12_1_program_kernel_trap_tba_tma(adev, xcc_mask); } static void gfxhub_v12_1_xcc_program_invalidation(struct amdgpu_device *adev, diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c index c35856180c3c..6cb7bae80f3d 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c @@ -21,6 +21,7 @@ * */ #include "amdgpu.h" +#include "amdgpu_trap.h" #include "gfxhub_v1_0.h" #include "gfxhub_v1_1.h" @@ -252,6 +253,49 @@ static void gfxhub_v1_0_disable_identity_aperture(struct amdgpu_device *adev) } +static void +gfxhub_v1_0_program_kernel_trap_tba_tma(struct amdgpu_device *adev) +{ + u64 tba_addr, tma_addr; + unsigned int vmid; + + if (!amdgpu_trap_is_supported(adev) || + !adev->gfx.funcs || + !adev->gfx.funcs->select_me_pipe_q) + return; + + amdgpu_trap_get_l1_addr(adev, &tba_addr, &tma_addr); + + WARN_ON(!IS_ALIGNED(tba_addr, 256)); + WARN_ON(!IS_ALIGNED(tma_addr, 256)); + + mutex_lock(&adev->srbm_mutex); + + for (vmid = 1; vmid < AMDGPU_NUM_VMID; vmid++) { + /* Skip user-queue VMIDs. Program only kernel-managed VMIDs. */ + if (adev->vm_manager.vmid_uq_mask_gfxhub & BIT(vmid)) + continue; + + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, vmid, 0); + + WREG32_SOC15(GC, 0, mmSQ_SHADER_TBA_LO, + lower_32_bits(tba_addr >> 8)); + WREG32_SOC15(GC, 0, mmSQ_SHADER_TBA_HI, + upper_32_bits(tba_addr >> 8) & + SQ_SHADER_TBA_HI__ADDR_HI_MASK); + + WREG32_SOC15(GC, 0, mmSQ_SHADER_TMA_LO, + lower_32_bits(tma_addr >> 8)); + WREG32_SOC15(GC, 0, mmSQ_SHADER_TMA_HI, + upper_32_bits(tma_addr >> 8) & + SQ_SHADER_TMA_HI__ADDR_HI_MASK); + } + + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0); + + mutex_unlock(&adev->srbm_mutex); +} + static void gfxhub_v1_0_setup_vmid_config(struct amdgpu_device *adev) { struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)]; @@ -310,6 +354,9 @@ static void gfxhub_v1_0_setup_vmid_config(struct amdgpu_device *adev) i * hub->ctx_addr_distance, upper_32_bits(adev->vm_manager.max_pfn - 1)); } + + if (amdgpu_trap_is_supported(adev)) + gfxhub_v1_0_program_kernel_trap_tba_tma(adev); } static void gfxhub_v1_0_program_invalidation(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_2.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_2.c index e89813590539..b06edea271af 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_2.c +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_2.c @@ -22,6 +22,7 @@ */ #include "amdgpu.h" #include "amdgpu_ip.h" +#include "amdgpu_trap.h" #include "amdgpu_xcp.h" #include "gfxhub_v1_2.h" #include "gfxhub_v1_1.h" @@ -328,6 +329,61 @@ gfxhub_v1_2_per_process_xnack_support(struct amdgpu_device *adev) return !!adev->aid_mask; } +static void +gfxhub_v1_2_program_kernel_trap_tba_tma(struct amdgpu_device *adev, + uint32_t xcc_mask) +{ + u64 tba_addr, tma_addr; + unsigned int vmid; + int xcc; + + if (!amdgpu_trap_is_supported(adev) || + !adev->gfx.funcs || + !adev->gfx.funcs->select_me_pipe_q) + return; + + amdgpu_trap_get_l1_addr(adev, &tba_addr, &tma_addr); + + WARN_ON(!IS_ALIGNED(tba_addr, 256)); + WARN_ON(!IS_ALIGNED(tma_addr, 256)); + + mutex_lock(&adev->srbm_mutex); + + for_each_inst(xcc, xcc_mask) { + for (vmid = 1; vmid < AMDGPU_NUM_VMID; vmid++) { + /* Program only kernel-managed VMIDs. */ + if (adev->vm_manager.vmid_uq_mask_gfxhub & BIT(vmid)) + continue; + + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, + vmid, xcc); + + WREG32_SOC15(GC, GET_INST(GC, xcc), + regSQ_SHADER_TBA_LO, + lower_32_bits(tba_addr >> 8)); + + WREG32_SOC15(GC, GET_INST(GC, xcc), + regSQ_SHADER_TBA_HI, + upper_32_bits(tba_addr >> 8) & + SQ_SHADER_TBA_HI__ADDR_HI_MASK); + + WREG32_SOC15(GC, GET_INST(GC, xcc), + regSQ_SHADER_TMA_LO, + lower_32_bits(tma_addr >> 8)); + + WREG32_SOC15(GC, GET_INST(GC, xcc), + regSQ_SHADER_TMA_HI, + upper_32_bits(tma_addr >> 8) & + SQ_SHADER_TMA_HI__ADDR_HI_MASK); + } + } + + for_each_inst(xcc, xcc_mask) + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, xcc); + + mutex_unlock(&adev->srbm_mutex); +} + static void gfxhub_v1_2_xcc_setup_vmid_config(struct amdgpu_device *adev, uint32_t xcc_mask) { @@ -398,6 +454,9 @@ static void gfxhub_v1_2_xcc_setup_vmid_config(struct amdgpu_device *adev, upper_32_bits(adev->vm_manager.max_pfn - 1)); } } + + if (amdgpu_trap_is_supported(adev)) + gfxhub_v1_2_program_kernel_trap_tba_tma(adev, xcc_mask); } static void gfxhub_v1_2_xcc_program_invalidation(struct amdgpu_device *adev, diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c index 04778f9f95bd..b7364d1b52c9 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c @@ -22,6 +22,7 @@ */ #include "amdgpu.h" +#include "amdgpu_trap.h" #include "gfxhub_v2_0.h" #include "gc/gc_10_1_0_offset.h" @@ -285,6 +286,52 @@ static void gfxhub_v2_0_disable_identity_aperture(struct amdgpu_device *adev) } +static void +gfxhub_v2_0_program_kernel_trap_tba_tma(struct amdgpu_device *adev) +{ + u64 tba_addr, tma_addr; + unsigned int vmid; + + if (!amdgpu_trap_is_supported(adev) || + !adev->gfx.funcs || + !adev->gfx.funcs->select_me_pipe_q) + return; + + amdgpu_trap_get_l1_addr(adev, &tba_addr, &tma_addr); + + WARN_ON(!IS_ALIGNED(tba_addr, 256)); + WARN_ON(!IS_ALIGNED(tma_addr, 256)); + + mutex_lock(&adev->srbm_mutex); + + for (vmid = 1; vmid < AMDGPU_NUM_VMID; vmid++) { + /* Program only kernel-managed VMIDs. */ + if (adev->vm_manager.vmid_uq_mask_gfxhub & BIT(vmid)) + continue; + + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, vmid, 0); + + WREG32_SOC15(GC, 0, mmSQ_SHADER_TBA_LO, + lower_32_bits(tba_addr >> 8)); + + WREG32_SOC15(GC, 0, mmSQ_SHADER_TBA_HI, + (upper_32_bits(tba_addr >> 8) & + SQ_SHADER_TBA_HI__ADDR_HI_MASK) | + (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT)); + + WREG32_SOC15(GC, 0, mmSQ_SHADER_TMA_LO, + lower_32_bits(tma_addr >> 8)); + + WREG32_SOC15(GC, 0, mmSQ_SHADER_TMA_HI, + upper_32_bits(tma_addr >> 8) & + SQ_SHADER_TMA_HI__ADDR_HI_MASK); + } + + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0); + + mutex_unlock(&adev->srbm_mutex); +} + static void gfxhub_v2_0_setup_vmid_config(struct amdgpu_device *adev) { struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)]; @@ -332,6 +379,9 @@ static void gfxhub_v2_0_setup_vmid_config(struct amdgpu_device *adev) } hub->vm_cntx_cntl = tmp; + + if (amdgpu_trap_is_supported(adev)) + gfxhub_v2_0_program_kernel_trap_tba_tma(adev); } static void gfxhub_v2_0_program_invalidation(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c index 6c77c17878f8..c00c7b23e6e6 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c @@ -22,6 +22,7 @@ */ #include "amdgpu.h" +#include "amdgpu_trap.h" #include "gfxhub_v2_1.h" #include "gc/gc_10_3_0_offset.h" @@ -297,6 +298,52 @@ static void gfxhub_v2_1_disable_identity_aperture(struct amdgpu_device *adev) } +static void +gfxhub_v2_1_program_kernel_trap_tba_tma(struct amdgpu_device *adev) +{ + u64 tba_addr, tma_addr; + unsigned int vmid; + + if (!amdgpu_trap_is_supported(adev) || + !adev->gfx.funcs || + !adev->gfx.funcs->select_me_pipe_q) + return; + + amdgpu_trap_get_l1_addr(adev, &tba_addr, &tma_addr); + + WARN_ON(!IS_ALIGNED(tba_addr, 256)); + WARN_ON(!IS_ALIGNED(tma_addr, 256)); + + mutex_lock(&adev->srbm_mutex); + + for (vmid = 1; vmid < AMDGPU_NUM_VMID; vmid++) { + /* Program only kernel-managed VMIDs. */ + if (adev->vm_manager.vmid_uq_mask_gfxhub & BIT(vmid)) + continue; + + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, vmid, 0); + + WREG32_SOC15(GC, 0, mmSQ_SHADER_TBA_LO, + lower_32_bits(tba_addr >> 8)); + + WREG32_SOC15(GC, 0, mmSQ_SHADER_TBA_HI, + (upper_32_bits(tba_addr >> 8) & + SQ_SHADER_TBA_HI__ADDR_HI_MASK) | + (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT)); + + WREG32_SOC15(GC, 0, mmSQ_SHADER_TMA_LO, + lower_32_bits(tma_addr >> 8)); + + WREG32_SOC15(GC, 0, mmSQ_SHADER_TMA_HI, + upper_32_bits(tma_addr >> 8) & + SQ_SHADER_TMA_HI__ADDR_HI_MASK); + } + + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0); + + mutex_unlock(&adev->srbm_mutex); +} + static void gfxhub_v2_1_setup_vmid_config(struct amdgpu_device *adev) { struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)]; @@ -344,6 +391,9 @@ static void gfxhub_v2_1_setup_vmid_config(struct amdgpu_device *adev) } hub->vm_cntx_cntl = tmp; + + if (amdgpu_trap_is_supported(adev)) + gfxhub_v2_1_program_kernel_trap_tba_tma(adev); } static void gfxhub_v2_1_program_invalidation(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c index e2b9fd0c8af0..fd7a0f1f369b 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c @@ -22,6 +22,7 @@ */ #include "amdgpu.h" +#include "amdgpu_trap.h" #include "gfxhub_v3_0.h" #include "gc/gc_11_0_0_offset.h" @@ -292,6 +293,57 @@ static void gfxhub_v3_0_disable_identity_aperture(struct amdgpu_device *adev) } +static void +gfxhub_v3_0_program_kernel_trap_tba_tma(struct amdgpu_device *adev) +{ + u64 tba_addr, tma_addr; + u32 tba_hi; + unsigned int vmid; + + if (!amdgpu_trap_is_supported(adev) || + !adev->gfx.funcs || + !adev->gfx.funcs->select_me_pipe_q) + return; + + amdgpu_trap_get_l1_addr(adev, &tba_addr, &tma_addr); + + WARN_ON(!IS_ALIGNED(tba_addr, 256)); + WARN_ON(!IS_ALIGNED(tma_addr, 256)); + + tba_hi = upper_32_bits(tba_addr >> 8) & + SQ_SHADER_TBA_HI__ADDR_HI_MASK; + + if (!amdgpu_trap_has_dbg_wa(adev)) + tba_hi |= 1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT; + + mutex_lock(&adev->srbm_mutex); + + for (vmid = 1; vmid < AMDGPU_NUM_VMID; vmid++) { + /* Program only kernel-managed VMIDs. */ + if (adev->vm_manager.vmid_uq_mask_gfxhub & BIT(vmid)) + continue; + + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, vmid, 0); + + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_LO, + lower_32_bits(tba_addr >> 8)); + + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_HI, + tba_hi); + + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_LO, + lower_32_bits(tma_addr >> 8)); + + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_HI, + upper_32_bits(tma_addr >> 8) & + SQ_SHADER_TMA_HI__ADDR_HI_MASK); + } + + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0); + + mutex_unlock(&adev->srbm_mutex); +} + static void gfxhub_v3_0_setup_vmid_config(struct amdgpu_device *adev) { struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)]; @@ -339,6 +391,9 @@ static void gfxhub_v3_0_setup_vmid_config(struct amdgpu_device *adev) } hub->vm_cntx_cntl = tmp; + + if (amdgpu_trap_is_supported(adev)) + gfxhub_v3_0_program_kernel_trap_tba_tma(adev); } static void gfxhub_v3_0_program_invalidation(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0_3.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0_3.c index fa8b966b7ea1..6c4b110f8048 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0_3.c +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0_3.c @@ -23,6 +23,7 @@ #include "amdgpu.h" #include "gfxhub_v3_0_3.h" +#include "amdgpu_trap.h" #include "gc/gc_11_0_3_offset.h" #include "gc/gc_11_0_3_sh_mask.h" @@ -297,6 +298,57 @@ static void gfxhub_v3_0_3_disable_identity_aperture(struct amdgpu_device *adev) } +static void +gfxhub_v3_0_3_program_kernel_trap_tba_tma(struct amdgpu_device *adev) +{ + u64 tba_addr, tma_addr; + u32 tba_hi; + unsigned int vmid; + + if (!amdgpu_trap_is_supported(adev) || + !adev->gfx.funcs || + !adev->gfx.funcs->select_me_pipe_q) + return; + + amdgpu_trap_get_l1_addr(adev, &tba_addr, &tma_addr); + + WARN_ON(!IS_ALIGNED(tba_addr, 256)); + WARN_ON(!IS_ALIGNED(tma_addr, 256)); + + tba_hi = upper_32_bits(tba_addr >> 8) & + SQ_SHADER_TBA_HI__ADDR_HI_MASK; + + if (!amdgpu_trap_has_dbg_wa(adev)) + tba_hi |= 1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT; + + mutex_lock(&adev->srbm_mutex); + + for (vmid = 1; vmid < AMDGPU_NUM_VMID; vmid++) { + /* Program only kernel-managed VMIDs. */ + if (adev->vm_manager.vmid_uq_mask_gfxhub & BIT(vmid)) + continue; + + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, vmid, 0); + + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_LO, + lower_32_bits(tba_addr >> 8)); + + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_HI, + tba_hi); + + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_LO, + lower_32_bits(tma_addr >> 8)); + + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_HI, + upper_32_bits(tma_addr >> 8) & + SQ_SHADER_TMA_HI__ADDR_HI_MASK); + } + + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0); + + mutex_unlock(&adev->srbm_mutex); +} + static void gfxhub_v3_0_3_setup_vmid_config(struct amdgpu_device *adev) { struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)]; @@ -344,6 +396,9 @@ static void gfxhub_v3_0_3_setup_vmid_config(struct amdgpu_device *adev) } hub->vm_cntx_cntl = tmp; + + if (amdgpu_trap_is_supported(adev)) + gfxhub_v3_0_3_program_kernel_trap_tba_tma(adev); } static void gfxhub_v3_0_3_program_invalidation(struct amdgpu_device *adev) -- 2.34.1
