User-queue VMIDs receive their trap register state from MES/KIQ at queue creation. Kernel-managed VMIDs do not; program them here.
Add program_kernel_trap_tba_tma() to each gfxhub generation and call it from setup_vmid_config(). No vmhub callback is introduced: the operation is generation-specific and belongs in each gfxhub's own setup path. Both TBA and TMA addresses are GPU virtual addresses taken from the common trap mapping established at vm_init. TRAP_EN is safe to set at this point: any GPUVM assigned to a kernel-managed VMID already has a valid trap mapping before it can execute work. TBA and TMA are mapped at GPU-page-aligned virtual addresses within the common trap VA reservation. Keep the first-level trap layout internal to amdgpu_trap and expose only the absolute TBA/TMA addresses to the gfxhub programming code. The helper is named program_kernel_trap_tba_tma() because it selects each kernel-managed VMID and writes its TBA/TMA registers; it does not configure the VMIDs themselves. Suggested-by: Christian König <[email protected]> Suggested-by: Alex Deucher <[email protected]> Cc: Felix Kuehling <[email protected]> Cc: Lancelot Six <[email protected]> Cc: James Zhu <[email protected]> Cc: Lijo Lazar <[email protected]> Cc: Samuel Pitoiset <[email protected]> Cc: Natalie Vock <[email protected]> Cc: Timur Kristóf <[email protected]> Cc: Pierre-Eric Pelloux-Prayer <[email protected]> Signed-off-by: Srinivasan Shanmugam <[email protected]> --- drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c | 9 ++++- drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h | 3 ++ drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h | 12 ++----- drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c | 36 +++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c | 36 +++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c | 40 +++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c | 36 +++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c | 36 +++++++++++++++++++ 8 files changed, 198 insertions(+), 10 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c index 05d8f48e06cb..0dd75832f351 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c @@ -57,13 +57,20 @@ enum amdgpu_trap_region { static inline uint64_t amdgpu_trap_tba_vaddr(struct amdgpu_device *adev) { - uint64_t addr = AMDGPU_VA_RESERVED_TRAP_UQ_START(adev); + u64 addr = AMDGPU_VA_RESERVED_TRAP_START(adev); addr = amdgpu_gmc_sign_extend(addr); return addr; } +void amdgpu_trap_get_l1_addr(struct amdgpu_device *adev, + u64 *tba_addr, u64 *tma_addr) +{ + *tba_addr = amdgpu_trap_tba_vaddr(adev); + *tma_addr = *tba_addr + AMDGPU_TRAP_TMA_OFFSET; +} + static inline bool amdgpu_trap_is_supported(struct amdgpu_device *adev) { uint32_t gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h index 323c49a53dad..92ff3d2aee80 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h @@ -77,6 +77,9 @@ struct amdgpu_trap_usr_addr { int amdgpu_trap_init(struct amdgpu_device *adev); void amdgpu_trap_fini(struct amdgpu_device *adev); +void amdgpu_trap_get_l1_addr(struct amdgpu_device *adev, + u64 *tba_addr, u64 *tma_addr); + int amdgpu_trap_alloc(struct amdgpu_device *adev, struct amdgpu_vm *vm, struct amdgpu_trap_obj **trap_obj); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h index bbf4aa96e15a..cc54847f11de 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h @@ -177,17 +177,11 @@ struct amdgpu_trap_obj; #define AMDGPU_VA_RESERVED_TRAP_SIZE (1ULL << 16) #define AMDGPU_VA_RESERVED_TRAP_START(adev) (AMDGPU_VA_RESERVED_SEQ64_START(adev) \ - AMDGPU_VA_RESERVED_TRAP_SIZE) -/* TBD: Ideally, existing TRAP VA should do. There is a conflict with KFD mapping that needs to be - * resolved. Revisit later. - */ -#define AMDGPU_VA_RESERVED_TRAP_UQ_SIZE (3ULL << 12) -#define AMDGPU_VA_RESERVED_TRAP_UQ_START(adev) \ - (AMDGPU_VA_RESERVED_TRAP_START(adev) - AMDGPU_VA_RESERVED_TRAP_UQ_SIZE) #define AMDGPU_VA_RESERVED_BOTTOM (1ULL << 16) -#define AMDGPU_VA_RESERVED_TOP \ - (AMDGPU_VA_RESERVED_TRAP_UQ_SIZE + AMDGPU_VA_RESERVED_TRAP_SIZE + \ - AMDGPU_VA_RESERVED_SEQ64_SIZE + AMDGPU_VA_RESERVED_CSA_SIZE) +#define AMDGPU_VA_RESERVED_TOP \ + (AMDGPU_VA_RESERVED_TRAP_SIZE + AMDGPU_VA_RESERVED_SEQ64_SIZE + \ + AMDGPU_VA_RESERVED_CSA_SIZE) /* See vm_update_mode */ #define AMDGPU_VM_USE_CPU_FOR_GFX (1 << 0) diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c index 652eea6eae4a..b02970c4ab34 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c @@ -22,6 +22,7 @@ */ #include "amdgpu.h" +#include "amdgpu_trap.h" #include "gfxhub_v11_5_0.h" #include "gc/gc_11_5_0_offset.h" @@ -290,6 +291,38 @@ static void gfxhub_v11_5_0_disable_identity_aperture(struct amdgpu_device *adev) } +static void +gfxhub_v11_5_0_program_kernel_trap_tba_tma(struct amdgpu_device *adev) +{ + u64 tba_addr, tma_addr; + int vmid; + + if (!amdgpu_trap_is_enabled(adev) || + !adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q) + return; + + amdgpu_trap_get_l1_addr(adev, &tba_addr, &tma_addr); + + WARN_ON(!IS_ALIGNED(tba_addr, 256)); + WARN_ON(!IS_ALIGNED(tma_addr, 256)); + + mutex_lock(&adev->srbm_mutex); + for (vmid = 1; vmid < adev->vm_manager.first_kfd_vmid; vmid++) { + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, vmid, 0); + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_LO, + lower_32_bits(tba_addr >> 8)); + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_HI, + upper_32_bits(tba_addr >> 8) | + (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT)); + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_LO, + lower_32_bits(tma_addr >> 8)); + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_HI, + upper_32_bits(tma_addr >> 8)); + } + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0); + mutex_unlock(&adev->srbm_mutex); +} + static void gfxhub_v11_5_0_setup_vmid_config(struct amdgpu_device *adev) { struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)]; @@ -337,6 +370,9 @@ static void gfxhub_v11_5_0_setup_vmid_config(struct amdgpu_device *adev) } hub->vm_cntx_cntl = tmp; + + if (amdgpu_trap_is_enabled(adev)) + gfxhub_v11_5_0_program_kernel_trap_tba_tma(adev); } static void gfxhub_v11_5_0_program_invalidation(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c index 6cbf837d50dd..3f402acf20bf 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c @@ -22,6 +22,7 @@ */ #include "amdgpu.h" +#include "amdgpu_trap.h" #include "gfxhub_v12_0.h" #include "gc/gc_12_0_0_offset.h" @@ -295,6 +296,38 @@ static void gfxhub_v12_0_disable_identity_aperture(struct amdgpu_device *adev) } +static void +gfxhub_v12_0_program_kernel_trap_tba_tma(struct amdgpu_device *adev) +{ + u64 tba_addr, tma_addr; + int vmid; + + if (!amdgpu_trap_is_enabled(adev) || + !adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q) + return; + + amdgpu_trap_get_l1_addr(adev, &tba_addr, &tma_addr); + + WARN_ON(!IS_ALIGNED(tba_addr, 256)); + WARN_ON(!IS_ALIGNED(tma_addr, 256)); + + mutex_lock(&adev->srbm_mutex); + for (vmid = 1; vmid < adev->vm_manager.first_kfd_vmid; vmid++) { + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, vmid, 0); + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_LO, + lower_32_bits(tba_addr >> 8)); + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_HI, + upper_32_bits(tba_addr >> 8) | + (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT)); + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_LO, + lower_32_bits(tma_addr >> 8)); + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_HI, + upper_32_bits(tma_addr >> 8)); + } + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0); + mutex_unlock(&adev->srbm_mutex); +} + static void gfxhub_v12_0_setup_vmid_config(struct amdgpu_device *adev) { struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)]; @@ -342,6 +375,9 @@ static void gfxhub_v12_0_setup_vmid_config(struct amdgpu_device *adev) } hub->vm_cntx_cntl = tmp; + + if (amdgpu_trap_is_enabled(adev)) + gfxhub_v12_0_program_kernel_trap_tba_tma(adev); } static void gfxhub_v12_0_program_invalidation(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c index 4c2fd1e6616e..bb5b4ffed065 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c @@ -21,6 +21,7 @@ * */ #include "amdgpu.h" +#include "amdgpu_trap.h" #include "amdgpu_xcp.h" #include "gfxhub_v12_1.h" @@ -406,6 +407,42 @@ static void gfxhub_v12_1_xcc_disable_identity_aperture(struct amdgpu_device *ade } } +static void +gfxhub_v12_1_program_kernel_trap_tba_tma(struct amdgpu_device *adev) +{ + u64 tba_addr, tma_addr; + int xcc, vmid; + + if (!amdgpu_trap_is_enabled(adev) || + !adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q) + return; + + amdgpu_trap_get_l1_addr(adev, &tba_addr, &tma_addr); + + WARN_ON(!IS_ALIGNED(tba_addr, 256)); + WARN_ON(!IS_ALIGNED(tma_addr, 256)); + + mutex_lock(&adev->srbm_mutex); + for_each_inst(xcc, adev->gfx.xcc_mask) { + for (vmid = 1; vmid < adev->vm_manager.first_kfd_vmid; vmid++) { + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, vmid, xcc); + WREG32_SOC15(GC, GET_INST(GC, xcc), regSQ_SHADER_TBA_LO, + lower_32_bits(tba_addr >> 8)); + WREG32_SOC15(GC, GET_INST(GC, xcc), regSQ_SHADER_TBA_HI, + upper_32_bits(tba_addr >> 8) | + (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT)); + WREG32_SOC15(GC, GET_INST(GC, xcc), regSQ_SHADER_TMA_LO, + lower_32_bits(tma_addr >> 8)); + WREG32_SOC15(GC, GET_INST(GC, xcc), regSQ_SHADER_TMA_HI, + upper_32_bits(tma_addr >> 8)); + } + } + /* Reset the VMID selector on every XCC instance. */ + for_each_inst(xcc, adev->gfx.xcc_mask) + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, xcc); + mutex_unlock(&adev->srbm_mutex); +} + static void gfxhub_v12_1_xcc_setup_vmid_config(struct amdgpu_device *adev, uint32_t xcc_mask) { @@ -468,6 +505,9 @@ static void gfxhub_v12_1_xcc_setup_vmid_config(struct amdgpu_device *adev, hub->vm_cntx_cntl = tmp; } + + if (amdgpu_trap_is_enabled(adev)) + gfxhub_v12_1_program_kernel_trap_tba_tma(adev); } static void gfxhub_v12_1_xcc_program_invalidation(struct amdgpu_device *adev, diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c index 9ea593e2c719..6cca45b374a3 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c @@ -22,6 +22,7 @@ */ #include "amdgpu.h" +#include "amdgpu_trap.h" #include "gfxhub_v2_0.h" #include "gc/gc_10_1_0_offset.h" @@ -280,6 +281,38 @@ static void gfxhub_v2_0_disable_identity_aperture(struct amdgpu_device *adev) } +static void +gfxhub_v2_0_program_kernel_trap_tba_tma(struct amdgpu_device *adev) +{ + u64 tba_addr, tma_addr; + int vmid; + + if (!amdgpu_trap_is_enabled(adev) || + !adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q) + return; + + amdgpu_trap_get_l1_addr(adev, &tba_addr, &tma_addr); + + WARN_ON(!IS_ALIGNED(tba_addr, 256)); + WARN_ON(!IS_ALIGNED(tma_addr, 256)); + + mutex_lock(&adev->srbm_mutex); + for (vmid = 1; vmid < adev->vm_manager.first_kfd_vmid; vmid++) { + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, vmid, 0); + WREG32_SOC15(GC, 0, mmSQ_SHADER_TBA_LO, + lower_32_bits(tba_addr >> 8)); + WREG32_SOC15(GC, 0, mmSQ_SHADER_TBA_HI, + upper_32_bits(tba_addr >> 8) | + (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT)); + WREG32_SOC15(GC, 0, mmSQ_SHADER_TMA_LO, + lower_32_bits(tma_addr >> 8)); + WREG32_SOC15(GC, 0, mmSQ_SHADER_TMA_HI, + upper_32_bits(tma_addr >> 8)); + } + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0); + mutex_unlock(&adev->srbm_mutex); +} + static void gfxhub_v2_0_setup_vmid_config(struct amdgpu_device *adev) { struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)]; @@ -327,6 +360,9 @@ static void gfxhub_v2_0_setup_vmid_config(struct amdgpu_device *adev) } hub->vm_cntx_cntl = tmp; + + if (amdgpu_trap_is_enabled(adev)) + gfxhub_v2_0_program_kernel_trap_tba_tma(adev); } static void gfxhub_v2_0_program_invalidation(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c index 9e6a6e13dec0..680b171c4c63 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c @@ -22,6 +22,7 @@ */ #include "amdgpu.h" +#include "amdgpu_trap.h" #include "gfxhub_v3_0.h" #include "gc/gc_11_0_0_offset.h" @@ -287,6 +288,38 @@ static void gfxhub_v3_0_disable_identity_aperture(struct amdgpu_device *adev) } +static void +gfxhub_v3_0_program_kernel_trap_tba_tma(struct amdgpu_device *adev) +{ + u64 tba_addr, tma_addr; + int vmid; + + if (!amdgpu_trap_is_enabled(adev) || + !adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q) + return; + + amdgpu_trap_get_l1_addr(adev, &tba_addr, &tma_addr); + + WARN_ON(!IS_ALIGNED(tba_addr, 256)); + WARN_ON(!IS_ALIGNED(tma_addr, 256)); + + mutex_lock(&adev->srbm_mutex); + for (vmid = 1; vmid < adev->vm_manager.first_kfd_vmid; vmid++) { + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, vmid, 0); + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_LO, + lower_32_bits(tba_addr >> 8)); + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_HI, + upper_32_bits(tba_addr >> 8) | + (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT)); + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_LO, + lower_32_bits(tma_addr >> 8)); + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_HI, + upper_32_bits(tma_addr >> 8)); + } + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0); + mutex_unlock(&adev->srbm_mutex); +} + static void gfxhub_v3_0_setup_vmid_config(struct amdgpu_device *adev) { struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)]; @@ -334,6 +367,9 @@ static void gfxhub_v3_0_setup_vmid_config(struct amdgpu_device *adev) } hub->vm_cntx_cntl = tmp; + + if (amdgpu_trap_is_enabled(adev)) + gfxhub_v3_0_program_kernel_trap_tba_tma(adev); } static void gfxhub_v3_0_program_invalidation(struct amdgpu_device *adev) -- 2.34.1
