User queues get their first-level TBA and TMA from MES/KIQ when the queue is created. Kernel-managed VMIDs do not use this path, so program their trap registers from the gfxhub setup path.
Get the reserved first-level TBA and TMA virtual addresses and program them for each kernel-managed VMID. The trap registers use the SRBM selector, so keep the programming under srbm_mutex and restore the VMID selector after the registers are updated. Trap enablement differs across GPU generations. On GFX9, enable traps for kernel-managed VMIDs through the debug trap configuration. On GFX9.4.3, enable the per-VMID trap control on each XCC. On generations where trap enable is encoded in SQ_SHADER_TBA_HI, program TRAP_EN together with the TBA address. Mask the TBA and TMA address fields before writing the registers. Keep TRAP_EN disabled on the affected GFX11 ASICs to preserve the existing trap workaround. On multi-XCC hardware, program the requested XCCs and restore the VMID selector on each XCC afterwards. Keep the first-level trap VA layout inside amdgpu_trap. The gfxhub code only needs the final TBA and TMA addresses. Suggested-by: Christian König <[email protected]> Suggested-by: Alex Deucher <[email protected]> Suggested-by: Lijo Lazar <[email protected]> Cc: Felix Kuehling <[email protected]> Cc: Lancelot Six <[email protected]> Cc: James Zhu <[email protected]> Cc: Samuel Pitoiset <[email protected]> Cc: Natalie Vock <[email protected]> Cc: Timur Kristóf <[email protected]> Cc: Pierre-Eric Pelloux-Prayer <[email protected]> Signed-off-by: Srinivasan Shanmugam <[email protected]> --- drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c | 9 +++- drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h | 5 ++ drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c | 29 ++++++++--- drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c | 16 +++++++ drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c | 38 +++++++++++++++ drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c | 38 +++++++++++++++ drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c | 53 +++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c | 41 ++++++++++++++++ drivers/gpu/drm/amd/amdgpu/gfxhub_v1_2.c | 53 +++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c | 38 +++++++++++++++ drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c | 42 ++++++++++++++++ drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c | 46 ++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0_3.c | 46 ++++++++++++++++++ 13 files changed, 446 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c index 89968b5b7ecb..6e7dfceeee10 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c @@ -63,7 +63,14 @@ static inline uint64_t amdgpu_trap_tba_vaddr(struct amdgpu_device *adev) return addr; } -static inline bool amdgpu_trap_is_supported(struct amdgpu_device *adev) +void amdgpu_trap_get_l1_addr(struct amdgpu_device *adev, + u64 *tba_addr, u64 *tma_addr) +{ + *tba_addr = amdgpu_trap_tba_vaddr(adev); + *tma_addr = *tba_addr + AMDGPU_TRAP_TMA_OFFSET; +} + +bool amdgpu_trap_is_supported(struct amdgpu_device *adev) { uint32_t gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h index c78c60e3d18d..523f65cb3d3d 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h @@ -76,6 +76,11 @@ struct amdgpu_trap_usr_addr { int amdgpu_trap_init(struct amdgpu_device *adev); void amdgpu_trap_fini(struct amdgpu_device *adev); +bool amdgpu_trap_is_supported(struct amdgpu_device *adev); + +void amdgpu_trap_get_l1_addr(struct amdgpu_device *adev, + u64 *tba_addr, u64 *tma_addr); + int amdgpu_trap_alloc(struct amdgpu_device *adev, struct amdgpu_vm *vm, struct amdgpu_trap_obj **trap_obj); diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c index 1678015ce5a9..07e249500254 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c @@ -29,6 +29,7 @@ #include "amdgpu.h" #include "amdgpu_gfx.h" +#include "amdgpu_trap.h" #include "soc15.h" #include "soc15d.h" #include "amdgpu_atomfirmware.h" @@ -2568,21 +2569,26 @@ static void gfx_v9_0_setup_rb(struct amdgpu_device *adev) } static void gfx_v9_0_debug_trap_config_init(struct amdgpu_device *adev, - uint32_t first_vmid, - uint32_t last_vmid) + u32 first_vmid, + uint32_t last_vmid) { uint32_t data; - uint32_t trap_config_vmid_mask = 0; + u32 trap_config_vmid_mask; int i; - /* Calculate trap config vmid mask */ + data = RREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_CONFIG)); + trap_config_vmid_mask = + REG_GET_FIELD(data, SPI_GDBG_TRAP_CONFIG, VMID_SEL); + + /* Add the requested VMIDs to the existing trap configuration. */ for (i = first_vmid; i < last_vmid; i++) trap_config_vmid_mask |= (1 << i); - data = REG_SET_FIELD(0, SPI_GDBG_TRAP_CONFIG, - VMID_SEL, trap_config_vmid_mask); data = REG_SET_FIELD(data, SPI_GDBG_TRAP_CONFIG, - TRAP_EN, 1); + VMID_SEL, trap_config_vmid_mask); + data = REG_SET_FIELD(data, SPI_GDBG_TRAP_CONFIG, + TRAP_EN, 1); + WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_CONFIG), data); WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_MASK), 0); @@ -4051,6 +4057,15 @@ static int gfx_v9_0_hw_init(struct amdgpu_ip_block *ip_block) if (r) return r; + if (amdgpu_trap_is_supported(adev)) { + if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2)) + gfx_v9_4_2_debug_trap_config_init(adev, 1, + adev->vm_manager.first_kfd_vmid); + else + gfx_v9_0_debug_trap_config_init(adev, 1, + adev->vm_manager.first_kfd_vmid); + } + if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2) && !amdgpu_sriov_vf(adev)) gfx_v9_4_2_set_power_brake_sequence(adev); diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c index 01ed1701c781..69a1e64620cd 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c @@ -25,6 +25,7 @@ #include "amdgpu.h" #include "amdgpu_ip.h" #include "amdgpu_gfx.h" +#include "amdgpu_trap.h" #include "soc15.h" #include "soc15d.h" #include "soc15_common.h" @@ -1173,6 +1174,21 @@ static void gfx_v9_4_3_xcc_init_compute_vmid(struct amdgpu_device *adev, SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT; mutex_lock(&adev->srbm_mutex); + + if (amdgpu_trap_is_supported(adev)) { + for (i = 1; i < adev->vm_manager.first_kfd_vmid; i++) { + soc15_grbm_select(adev, 0, 0, 0, i, + GET_INST(GC, xcc_id)); + + data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), + regSPI_GDBG_PER_VMID_CNTL); + data = REG_SET_FIELD(data, SPI_GDBG_PER_VMID_CNTL, + TRAP_EN, 1); + WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), + regSPI_GDBG_PER_VMID_CNTL, data); + } + } + for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) { soc15_grbm_select(adev, 0, 0, 0, i, GET_INST(GC, xcc_id)); /* CP and shaders */ diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c index 8be912d50601..e635d83488af 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c @@ -22,6 +22,7 @@ */ #include "amdgpu.h" +#include "amdgpu_trap.h" #include "gfxhub_v11_5_0.h" #include "gc/gc_11_5_0_offset.h" @@ -295,6 +296,40 @@ static void gfxhub_v11_5_0_disable_identity_aperture(struct amdgpu_device *adev) } +static void +gfxhub_v11_5_0_program_kernel_trap_tba_tma(struct amdgpu_device *adev) +{ + u64 tba_addr, tma_addr; + unsigned int vmid; + + if (!amdgpu_trap_is_supported(adev) || + !adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q) + return; + + amdgpu_trap_get_l1_addr(adev, &tba_addr, &tma_addr); + + WARN_ON(!IS_ALIGNED(tba_addr, 256)); + WARN_ON(!IS_ALIGNED(tma_addr, 256)); + + mutex_lock(&adev->srbm_mutex); + for (vmid = 1; vmid < adev->vm_manager.first_kfd_vmid; vmid++) { + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, vmid, 0); + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_LO, + lower_32_bits(tba_addr >> 8)); + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_HI, + (upper_32_bits(tba_addr >> 8) & + SQ_SHADER_TBA_HI__ADDR_HI_MASK) | + (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT)); + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_LO, + lower_32_bits(tma_addr >> 8)); + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_HI, + upper_32_bits(tma_addr >> 8) & + SQ_SHADER_TMA_HI__ADDR_HI_MASK); + } + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0); + mutex_unlock(&adev->srbm_mutex); +} + static void gfxhub_v11_5_0_setup_vmid_config(struct amdgpu_device *adev) { struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)]; @@ -342,6 +377,9 @@ static void gfxhub_v11_5_0_setup_vmid_config(struct amdgpu_device *adev) } hub->vm_cntx_cntl = tmp; + + if (amdgpu_trap_is_supported(adev)) + gfxhub_v11_5_0_program_kernel_trap_tba_tma(adev); } static void gfxhub_v11_5_0_program_invalidation(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c index 4144c1c2ee13..dc7fde6962ca 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c @@ -22,6 +22,7 @@ */ #include "amdgpu.h" +#include "amdgpu_trap.h" #include "gfxhub_v12_0.h" #include "gc/gc_12_0_0_offset.h" @@ -300,6 +301,40 @@ static void gfxhub_v12_0_disable_identity_aperture(struct amdgpu_device *adev) } +static void +gfxhub_v12_0_program_kernel_trap_tba_tma(struct amdgpu_device *adev) +{ + u64 tba_addr, tma_addr; + unsigned int vmid; + + if (!amdgpu_trap_is_supported(adev) || + !adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q) + return; + + amdgpu_trap_get_l1_addr(adev, &tba_addr, &tma_addr); + + WARN_ON(!IS_ALIGNED(tba_addr, 256)); + WARN_ON(!IS_ALIGNED(tma_addr, 256)); + + mutex_lock(&adev->srbm_mutex); + for (vmid = 1; vmid < adev->vm_manager.first_kfd_vmid; vmid++) { + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, vmid, 0); + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_LO, + lower_32_bits(tba_addr >> 8)); + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_HI, + (upper_32_bits(tba_addr >> 8) & + SQ_SHADER_TBA_HI__ADDR_HI_MASK) | + (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT)); + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_LO, + lower_32_bits(tma_addr >> 8)); + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_HI, + upper_32_bits(tma_addr >> 8) & + SQ_SHADER_TMA_HI__ADDR_HI_MASK); + } + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0); + mutex_unlock(&adev->srbm_mutex); +} + static void gfxhub_v12_0_setup_vmid_config(struct amdgpu_device *adev) { struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)]; @@ -347,6 +382,9 @@ static void gfxhub_v12_0_setup_vmid_config(struct amdgpu_device *adev) } hub->vm_cntx_cntl = tmp; + + if (amdgpu_trap_is_supported(adev)) + gfxhub_v12_0_program_kernel_trap_tba_tma(adev); } static void gfxhub_v12_0_program_invalidation(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c index b13967a1c8fb..78088e72606c 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c @@ -21,6 +21,7 @@ * */ #include "amdgpu.h" +#include "amdgpu_trap.h" #include "amdgpu_xcp.h" #include "gfxhub_v12_1.h" @@ -392,6 +393,55 @@ static void gfxhub_v12_1_xcc_disable_identity_aperture(struct amdgpu_device *ade } } +static void +gfxhub_v12_1_program_kernel_trap_tba_tma(struct amdgpu_device *adev, + uint32_t xcc_mask) +{ + u64 tba_addr, tma_addr; + unsigned int vmid; + int xcc; + + if (!amdgpu_trap_is_supported(adev) || + !adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q) + return; + + amdgpu_trap_get_l1_addr(adev, &tba_addr, &tma_addr); + + WARN_ON(!IS_ALIGNED(tba_addr, 256)); + WARN_ON(!IS_ALIGNED(tma_addr, 256)); + + mutex_lock(&adev->srbm_mutex); + for_each_inst(xcc, xcc_mask) { + for (vmid = 1; + vmid < adev->vm_manager.first_kfd_vmid; + vmid++) { + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, + vmid, xcc); + WREG32_SOC15(GC, GET_INST(GC, xcc), + regSQ_SHADER_TBA_LO, + lower_32_bits(tba_addr >> 8)); + WREG32_SOC15(GC, GET_INST(GC, xcc), + regSQ_SHADER_TBA_HI, + (upper_32_bits(tba_addr >> 8) & + SQ_SHADER_TBA_HI__ADDR_HI_MASK) | + (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT)); + WREG32_SOC15(GC, GET_INST(GC, xcc), + regSQ_SHADER_TMA_LO, + lower_32_bits(tma_addr >> 8)); + WREG32_SOC15(GC, GET_INST(GC, xcc), + regSQ_SHADER_TMA_HI, + upper_32_bits(tma_addr >> 8) & + SQ_SHADER_TMA_HI__ADDR_HI_MASK); + } + } + + /* Reset the VMID selector on every XCC instance. */ + for_each_inst(xcc, xcc_mask) + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, xcc); + + mutex_unlock(&adev->srbm_mutex); +} + static void gfxhub_v12_1_xcc_setup_vmid_config(struct amdgpu_device *adev, uint32_t xcc_mask) { @@ -457,6 +507,9 @@ static void gfxhub_v12_1_xcc_setup_vmid_config(struct amdgpu_device *adev, hub->vm_cntx_cntl = tmp; } + + if (amdgpu_trap_is_supported(adev)) + gfxhub_v12_1_program_kernel_trap_tba_tma(adev, xcc_mask); } static void gfxhub_v12_1_xcc_program_invalidation(struct amdgpu_device *adev, diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c index c35856180c3c..dbd993ae1d0e 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c @@ -21,6 +21,7 @@ * */ #include "amdgpu.h" +#include "amdgpu_trap.h" #include "gfxhub_v1_0.h" #include "gfxhub_v1_1.h" @@ -252,6 +253,43 @@ static void gfxhub_v1_0_disable_identity_aperture(struct amdgpu_device *adev) } +static void +gfxhub_v1_0_program_kernel_trap_tba_tma(struct amdgpu_device *adev) +{ + u64 tba_addr, tma_addr; + unsigned int vmid; + + if (!amdgpu_trap_is_supported(adev) || + !adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q) + return; + + amdgpu_trap_get_l1_addr(adev, &tba_addr, &tma_addr); + + WARN_ON(!IS_ALIGNED(tba_addr, 256)); + WARN_ON(!IS_ALIGNED(tma_addr, 256)); + + mutex_lock(&adev->srbm_mutex); + + for (vmid = 1; vmid < adev->vm_manager.first_kfd_vmid; vmid++) { + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, vmid, 0); + + WREG32_SOC15(GC, 0, mmSQ_SHADER_TBA_LO, + lower_32_bits(tba_addr >> 8)); + WREG32_SOC15(GC, 0, mmSQ_SHADER_TBA_HI, + upper_32_bits(tba_addr >> 8) & + SQ_SHADER_TBA_HI__ADDR_HI_MASK); + WREG32_SOC15(GC, 0, mmSQ_SHADER_TMA_LO, + lower_32_bits(tma_addr >> 8)); + WREG32_SOC15(GC, 0, mmSQ_SHADER_TMA_HI, + upper_32_bits(tma_addr >> 8) & + SQ_SHADER_TMA_HI__ADDR_HI_MASK); + } + + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0); + + mutex_unlock(&adev->srbm_mutex); +} + static void gfxhub_v1_0_setup_vmid_config(struct amdgpu_device *adev) { struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)]; @@ -310,6 +348,9 @@ static void gfxhub_v1_0_setup_vmid_config(struct amdgpu_device *adev) i * hub->ctx_addr_distance, upper_32_bits(adev->vm_manager.max_pfn - 1)); } + + if (amdgpu_trap_is_supported(adev)) + gfxhub_v1_0_program_kernel_trap_tba_tma(adev); } static void gfxhub_v1_0_program_invalidation(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_2.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_2.c index e89813590539..76c2b6b74903 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_2.c +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_2.c @@ -22,6 +22,7 @@ */ #include "amdgpu.h" #include "amdgpu_ip.h" +#include "amdgpu_trap.h" #include "amdgpu_xcp.h" #include "gfxhub_v1_2.h" #include "gfxhub_v1_1.h" @@ -328,6 +329,55 @@ gfxhub_v1_2_per_process_xnack_support(struct amdgpu_device *adev) return !!adev->aid_mask; } +static void +gfxhub_v1_2_program_kernel_trap_tba_tma(struct amdgpu_device *adev, + uint32_t xcc_mask) +{ + u64 tba_addr, tma_addr; + unsigned int vmid; + int xcc; + + if (!amdgpu_trap_is_supported(adev) || + !adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q) + return; + + amdgpu_trap_get_l1_addr(adev, &tba_addr, &tma_addr); + + WARN_ON(!IS_ALIGNED(tba_addr, 256)); + WARN_ON(!IS_ALIGNED(tma_addr, 256)); + + mutex_lock(&adev->srbm_mutex); + + for_each_inst(xcc, xcc_mask) { + for (vmid = 1; + vmid < adev->vm_manager.first_kfd_vmid; + vmid++) { + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, + vmid, xcc); + + WREG32_SOC15(GC, GET_INST(GC, xcc), + regSQ_SHADER_TBA_LO, + lower_32_bits(tba_addr >> 8)); + WREG32_SOC15(GC, GET_INST(GC, xcc), + regSQ_SHADER_TBA_HI, + upper_32_bits(tba_addr >> 8) & + SQ_SHADER_TBA_HI__ADDR_HI_MASK); + WREG32_SOC15(GC, GET_INST(GC, xcc), + regSQ_SHADER_TMA_LO, + lower_32_bits(tma_addr >> 8)); + WREG32_SOC15(GC, GET_INST(GC, xcc), + regSQ_SHADER_TMA_HI, + upper_32_bits(tma_addr >> 8) & + SQ_SHADER_TMA_HI__ADDR_HI_MASK); + } + } + + for_each_inst(xcc, xcc_mask) + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, xcc); + + mutex_unlock(&adev->srbm_mutex); +} + static void gfxhub_v1_2_xcc_setup_vmid_config(struct amdgpu_device *adev, uint32_t xcc_mask) { @@ -398,6 +448,9 @@ static void gfxhub_v1_2_xcc_setup_vmid_config(struct amdgpu_device *adev, upper_32_bits(adev->vm_manager.max_pfn - 1)); } } + + if (amdgpu_trap_is_supported(adev)) + gfxhub_v1_2_program_kernel_trap_tba_tma(adev, xcc_mask); } static void gfxhub_v1_2_xcc_program_invalidation(struct amdgpu_device *adev, diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c index 04778f9f95bd..da4e1fa34d8d 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c @@ -22,6 +22,7 @@ */ #include "amdgpu.h" +#include "amdgpu_trap.h" #include "gfxhub_v2_0.h" #include "gc/gc_10_1_0_offset.h" @@ -285,6 +286,40 @@ static void gfxhub_v2_0_disable_identity_aperture(struct amdgpu_device *adev) } +static void +gfxhub_v2_0_program_kernel_trap_tba_tma(struct amdgpu_device *adev) +{ + u64 tba_addr, tma_addr; + unsigned int vmid; + + if (!amdgpu_trap_is_supported(adev) || + !adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q) + return; + + amdgpu_trap_get_l1_addr(adev, &tba_addr, &tma_addr); + + WARN_ON(!IS_ALIGNED(tba_addr, 256)); + WARN_ON(!IS_ALIGNED(tma_addr, 256)); + + mutex_lock(&adev->srbm_mutex); + for (vmid = 1; vmid < adev->vm_manager.first_kfd_vmid; vmid++) { + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, vmid, 0); + WREG32_SOC15(GC, 0, mmSQ_SHADER_TBA_LO, + lower_32_bits(tba_addr >> 8)); + WREG32_SOC15(GC, 0, mmSQ_SHADER_TBA_HI, + (upper_32_bits(tba_addr >> 8) & + SQ_SHADER_TBA_HI__ADDR_HI_MASK) | + (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT)); + WREG32_SOC15(GC, 0, mmSQ_SHADER_TMA_LO, + lower_32_bits(tma_addr >> 8)); + WREG32_SOC15(GC, 0, mmSQ_SHADER_TMA_HI, + upper_32_bits(tma_addr >> 8) & + SQ_SHADER_TMA_HI__ADDR_HI_MASK); + } + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0); + mutex_unlock(&adev->srbm_mutex); +} + static void gfxhub_v2_0_setup_vmid_config(struct amdgpu_device *adev) { struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)]; @@ -332,6 +367,9 @@ static void gfxhub_v2_0_setup_vmid_config(struct amdgpu_device *adev) } hub->vm_cntx_cntl = tmp; + + if (amdgpu_trap_is_supported(adev)) + gfxhub_v2_0_program_kernel_trap_tba_tma(adev); } static void gfxhub_v2_0_program_invalidation(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c index 6c77c17878f8..c0ff251bcbff 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c @@ -22,6 +22,7 @@ */ #include "amdgpu.h" +#include "amdgpu_trap.h" #include "gfxhub_v2_1.h" #include "gc/gc_10_3_0_offset.h" @@ -297,6 +298,44 @@ static void gfxhub_v2_1_disable_identity_aperture(struct amdgpu_device *adev) } +static void +gfxhub_v2_1_program_kernel_trap_tba_tma(struct amdgpu_device *adev) +{ + u64 tba_addr, tma_addr; + unsigned int vmid; + + if (!amdgpu_trap_is_supported(adev) || + !adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q) + return; + + amdgpu_trap_get_l1_addr(adev, &tba_addr, &tma_addr); + + WARN_ON(!IS_ALIGNED(tba_addr, 256)); + WARN_ON(!IS_ALIGNED(tma_addr, 256)); + + mutex_lock(&adev->srbm_mutex); + + for (vmid = 1; vmid < adev->vm_manager.first_kfd_vmid; vmid++) { + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, vmid, 0); + + WREG32_SOC15(GC, 0, mmSQ_SHADER_TBA_LO, + lower_32_bits(tba_addr >> 8)); + WREG32_SOC15(GC, 0, mmSQ_SHADER_TBA_HI, + (upper_32_bits(tba_addr >> 8) & + SQ_SHADER_TBA_HI__ADDR_HI_MASK) | + (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT)); + WREG32_SOC15(GC, 0, mmSQ_SHADER_TMA_LO, + lower_32_bits(tma_addr >> 8)); + WREG32_SOC15(GC, 0, mmSQ_SHADER_TMA_HI, + upper_32_bits(tma_addr >> 8) & + SQ_SHADER_TMA_HI__ADDR_HI_MASK); + } + + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0); + + mutex_unlock(&adev->srbm_mutex); +} + static void gfxhub_v2_1_setup_vmid_config(struct amdgpu_device *adev) { struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)]; @@ -344,6 +383,9 @@ static void gfxhub_v2_1_setup_vmid_config(struct amdgpu_device *adev) } hub->vm_cntx_cntl = tmp; + + if (amdgpu_trap_is_supported(adev)) + gfxhub_v2_1_program_kernel_trap_tba_tma(adev); } static void gfxhub_v2_1_program_invalidation(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c index e2b9fd0c8af0..1d9d4d41c42c 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c @@ -22,6 +22,7 @@ */ #include "amdgpu.h" +#include "amdgpu_trap.h" #include "gfxhub_v3_0.h" #include "gc/gc_11_0_0_offset.h" @@ -292,6 +293,48 @@ static void gfxhub_v3_0_disable_identity_aperture(struct amdgpu_device *adev) } +static void +gfxhub_v3_0_program_kernel_trap_tba_tma(struct amdgpu_device *adev) +{ + u64 tba_addr, tma_addr; + u32 tba_hi; + unsigned int vmid; + + if (!amdgpu_trap_is_supported(adev) || + !adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q) + return; + + amdgpu_trap_get_l1_addr(adev, &tba_addr, &tma_addr); + + WARN_ON(!IS_ALIGNED(tba_addr, 256)); + WARN_ON(!IS_ALIGNED(tma_addr, 256)); + + tba_hi = upper_32_bits(tba_addr >> 8) & + SQ_SHADER_TBA_HI__ADDR_HI_MASK; + + if (!amdgpu_trap_has_dbg_wa(adev)) + tba_hi |= 1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT; + + mutex_lock(&adev->srbm_mutex); + + for (vmid = 1; vmid < adev->vm_manager.first_kfd_vmid; vmid++) { + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, vmid, 0); + + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_LO, + lower_32_bits(tba_addr >> 8)); + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_HI, tba_hi); + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_LO, + lower_32_bits(tma_addr >> 8)); + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_HI, + upper_32_bits(tma_addr >> 8) & + SQ_SHADER_TMA_HI__ADDR_HI_MASK); + } + + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0); + + mutex_unlock(&adev->srbm_mutex); +} + static void gfxhub_v3_0_setup_vmid_config(struct amdgpu_device *adev) { struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)]; @@ -339,6 +382,9 @@ static void gfxhub_v3_0_setup_vmid_config(struct amdgpu_device *adev) } hub->vm_cntx_cntl = tmp; + + if (amdgpu_trap_is_supported(adev)) + gfxhub_v3_0_program_kernel_trap_tba_tma(adev); } static void gfxhub_v3_0_program_invalidation(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0_3.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0_3.c index fa8b966b7ea1..7f2a0b21cd4f 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0_3.c +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0_3.c @@ -23,6 +23,7 @@ #include "amdgpu.h" #include "gfxhub_v3_0_3.h" +#include "amdgpu_trap.h" #include "gc/gc_11_0_3_offset.h" #include "gc/gc_11_0_3_sh_mask.h" @@ -297,6 +298,48 @@ static void gfxhub_v3_0_3_disable_identity_aperture(struct amdgpu_device *adev) } +static void +gfxhub_v3_0_3_program_kernel_trap_tba_tma(struct amdgpu_device *adev) +{ + u64 tba_addr, tma_addr; + u32 tba_hi; + unsigned int vmid; + + if (!amdgpu_trap_is_supported(adev) || + !adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q) + return; + + amdgpu_trap_get_l1_addr(adev, &tba_addr, &tma_addr); + + WARN_ON(!IS_ALIGNED(tba_addr, 256)); + WARN_ON(!IS_ALIGNED(tma_addr, 256)); + + tba_hi = upper_32_bits(tba_addr >> 8) & + SQ_SHADER_TBA_HI__ADDR_HI_MASK; + + if (!amdgpu_trap_has_dbg_wa(adev)) + tba_hi |= 1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT; + + mutex_lock(&adev->srbm_mutex); + + for (vmid = 1; vmid < adev->vm_manager.first_kfd_vmid; vmid++) { + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, vmid, 0); + + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_LO, + lower_32_bits(tba_addr >> 8)); + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_HI, tba_hi); + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_LO, + lower_32_bits(tma_addr >> 8)); + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_HI, + upper_32_bits(tma_addr >> 8) & + SQ_SHADER_TMA_HI__ADDR_HI_MASK); + } + + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0); + + mutex_unlock(&adev->srbm_mutex); +} + static void gfxhub_v3_0_3_setup_vmid_config(struct amdgpu_device *adev) { struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)]; @@ -344,6 +387,9 @@ static void gfxhub_v3_0_3_setup_vmid_config(struct amdgpu_device *adev) } hub->vm_cntx_cntl = tmp; + + if (amdgpu_trap_is_supported(adev)) + gfxhub_v3_0_3_program_kernel_trap_tba_tma(adev); } static void gfxhub_v3_0_3_program_invalidation(struct amdgpu_device *adev) -- 2.34.1
