User-queue VMIDs receive their trap register state from MES/KIQ at queue creation. Kernel-managed VMIDs do not; program them here.
Add program_kernel_trap_tba_tma() to each gfxhub generation and call it from setup_vmid_config(). No vmhub callback is introduced: the operation is generation-specific and belongs in each gfxhub's own setup path. Both TBA and TMA addresses are GPU virtual addresses taken from the common trap mapping established at vm_init. TRAP_EN is safe to set at this point: any GPUVM assigned to a kernel-managed VMID already has a valid trap mapping before it can execute work. TBA is the shared device-level ISA BO mapped into each VM at AMDGPU_VA_RESERVED_TRAP_START. TMA starts at offset 0x1800 from the same base, matching KFD_CWSR_TMA_OFFSET so both KFD and render-node VMs use the same per-VM buffer layout. The helper is named program_kernel_trap_tba_tma() because it selects each kernel-managed VMID and writes its TBA/TMA registers; it does not configure the VMIDs themselves. Signed-off-by: Srinivasan Shanmugam <[email protected]> Cc: Christian König <[email protected]> Cc: Alex Deucher <[email protected]> Cc: Felix Kuehling <[email protected]> Cc: Lijo Lazar <[email protected]> --- drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c | 4 --- drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h | 9 +++++ drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c | 35 ++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c | 35 ++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c | 39 +++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c | 35 ++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c | 35 ++++++++++++++++++ 7 files changed, 188 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c index 3313bc099cf4..d17f54ccadb0 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.c @@ -38,10 +38,6 @@ extern int cwsr_enable; -#define AMDGPU_TRAP_TBA_MAX_SIZE (2 * AMDGPU_GPU_PAGE_SIZE) -#define AMDGPU_TRAP_TMA_MAX_SIZE (AMDGPU_GPU_PAGE_SIZE) -#define AMDGPU_TRAP_TMA_OFFSET (AMDGPU_TRAP_TBA_MAX_SIZE) - #define SGPR_SIZE_PER_CU 0x4000 #define LDS_SIZE_PER_CU 0x10000 #define HWREG_SIZE_PER_CU 0x1000 diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h index 2d855433308a..5769c5acb5c0 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trap.h @@ -26,6 +26,15 @@ #include <linux/types.h> #include <linux/iosys-map.h> +/* + * TBA occupies one page plus 2 KiB (0x1800). TMA starts at that offset. + * Matches KFD_CWSR_TMA_OFFSET so both KFD and render-node paths share + * the same per-VM buffer layout. + */ +#define AMDGPU_TRAP_TBA_MAX_SIZE (AMDGPU_GPU_PAGE_SIZE + 2048) /* 0x1800 */ +#define AMDGPU_TRAP_TMA_MAX_SIZE (AMDGPU_GPU_PAGE_SIZE) +#define AMDGPU_TRAP_TMA_OFFSET (AMDGPU_TRAP_TBA_MAX_SIZE) /* 0x1800 */ + struct amdgpu_bo; struct amdgpu_bo_va; struct amdgpu_device; diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c index 652eea6eae4a..687268e23344 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c @@ -22,6 +22,7 @@ */ #include "amdgpu.h" +#include "amdgpu_trap.h" #include "gfxhub_v11_5_0.h" #include "gc/gc_11_5_0_offset.h" @@ -290,6 +291,37 @@ static void gfxhub_v11_5_0_disable_identity_aperture(struct amdgpu_device *adev) } +static void +gfxhub_v11_5_0_program_kernel_trap_tba_tma(struct amdgpu_device *adev) +{ + u64 tba_addr = AMDGPU_VA_RESERVED_TRAP_START(adev); + u64 tma_addr = tba_addr + AMDGPU_TRAP_TMA_OFFSET; /* tba_addr + 0x1800 */ + int vmid; + + if (!amdgpu_trap_is_enabled(adev) || + !adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q) + return; + + WARN_ON(!IS_ALIGNED(tba_addr, 256)); + WARN_ON(!IS_ALIGNED(tma_addr, 256)); + + mutex_lock(&adev->srbm_mutex); + for (vmid = 1; vmid < adev->vm_manager.first_kfd_vmid; vmid++) { + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, vmid, 0); + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_LO, + lower_32_bits(tba_addr >> 8)); + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_HI, + upper_32_bits(tba_addr >> 8) | + (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT)); + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_LO, + lower_32_bits(tma_addr >> 8)); + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_HI, + upper_32_bits(tma_addr >> 8)); + } + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0); + mutex_unlock(&adev->srbm_mutex); +} + static void gfxhub_v11_5_0_setup_vmid_config(struct amdgpu_device *adev) { struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)]; @@ -337,6 +369,9 @@ static void gfxhub_v11_5_0_setup_vmid_config(struct amdgpu_device *adev) } hub->vm_cntx_cntl = tmp; + + if (amdgpu_trap_is_enabled(adev)) + gfxhub_v11_5_0_program_kernel_trap_tba_tma(adev); } static void gfxhub_v11_5_0_program_invalidation(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c index 6cbf837d50dd..b521a6c6eaf6 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c @@ -22,6 +22,7 @@ */ #include "amdgpu.h" +#include "amdgpu_trap.h" #include "gfxhub_v12_0.h" #include "gc/gc_12_0_0_offset.h" @@ -295,6 +296,37 @@ static void gfxhub_v12_0_disable_identity_aperture(struct amdgpu_device *adev) } +static void +gfxhub_v12_0_program_kernel_trap_tba_tma(struct amdgpu_device *adev) +{ + u64 tba_addr = AMDGPU_VA_RESERVED_TRAP_START(adev); + u64 tma_addr = tba_addr + AMDGPU_TRAP_TMA_OFFSET; /* tba_addr + 0x1800 */ + int vmid; + + if (!amdgpu_trap_is_enabled(adev) || + !adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q) + return; + + WARN_ON(!IS_ALIGNED(tba_addr, 256)); + WARN_ON(!IS_ALIGNED(tma_addr, 256)); + + mutex_lock(&adev->srbm_mutex); + for (vmid = 1; vmid < adev->vm_manager.first_kfd_vmid; vmid++) { + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, vmid, 0); + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_LO, + lower_32_bits(tba_addr >> 8)); + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_HI, + upper_32_bits(tba_addr >> 8) | + (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT)); + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_LO, + lower_32_bits(tma_addr >> 8)); + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_HI, + upper_32_bits(tma_addr >> 8)); + } + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0); + mutex_unlock(&adev->srbm_mutex); +} + static void gfxhub_v12_0_setup_vmid_config(struct amdgpu_device *adev) { struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)]; @@ -342,6 +374,9 @@ static void gfxhub_v12_0_setup_vmid_config(struct amdgpu_device *adev) } hub->vm_cntx_cntl = tmp; + + if (amdgpu_trap_is_enabled(adev)) + gfxhub_v12_0_program_kernel_trap_tba_tma(adev); } static void gfxhub_v12_0_program_invalidation(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c index 4c2fd1e6616e..ac52ccda713d 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c @@ -21,6 +21,7 @@ * */ #include "amdgpu.h" +#include "amdgpu_trap.h" #include "amdgpu_xcp.h" #include "gfxhub_v12_1.h" @@ -406,6 +407,41 @@ static void gfxhub_v12_1_xcc_disable_identity_aperture(struct amdgpu_device *ade } } +static void +gfxhub_v12_1_program_kernel_trap_tba_tma(struct amdgpu_device *adev) +{ + u64 tba_addr = AMDGPU_VA_RESERVED_TRAP_START(adev); + u64 tma_addr = tba_addr + AMDGPU_TRAP_TMA_OFFSET; /* tba_addr + 0x1800 */ + int xcc, vmid; + + if (!amdgpu_trap_is_enabled(adev) || + !adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q) + return; + + WARN_ON(!IS_ALIGNED(tba_addr, 256)); + WARN_ON(!IS_ALIGNED(tma_addr, 256)); + + mutex_lock(&adev->srbm_mutex); + for_each_inst(xcc, adev->gfx.xcc_mask) { + for (vmid = 1; vmid < adev->vm_manager.first_kfd_vmid; vmid++) { + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, vmid, xcc); + WREG32_SOC15(GC, GET_INST(GC, xcc), regSQ_SHADER_TBA_LO, + lower_32_bits(tba_addr >> 8)); + WREG32_SOC15(GC, GET_INST(GC, xcc), regSQ_SHADER_TBA_HI, + upper_32_bits(tba_addr >> 8) | + (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT)); + WREG32_SOC15(GC, GET_INST(GC, xcc), regSQ_SHADER_TMA_LO, + lower_32_bits(tma_addr >> 8)); + WREG32_SOC15(GC, GET_INST(GC, xcc), regSQ_SHADER_TMA_HI, + upper_32_bits(tma_addr >> 8)); + } + } + /* Reset the VMID selector on every XCC instance. */ + for_each_inst(xcc, adev->gfx.xcc_mask) + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, xcc); + mutex_unlock(&adev->srbm_mutex); +} + static void gfxhub_v12_1_xcc_setup_vmid_config(struct amdgpu_device *adev, uint32_t xcc_mask) { @@ -468,6 +504,9 @@ static void gfxhub_v12_1_xcc_setup_vmid_config(struct amdgpu_device *adev, hub->vm_cntx_cntl = tmp; } + + if (amdgpu_trap_is_enabled(adev)) + gfxhub_v12_1_program_kernel_trap_tba_tma(adev); } static void gfxhub_v12_1_xcc_program_invalidation(struct amdgpu_device *adev, diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c index 9ea593e2c719..ba2d5d719474 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c @@ -22,6 +22,7 @@ */ #include "amdgpu.h" +#include "amdgpu_trap.h" #include "gfxhub_v2_0.h" #include "gc/gc_10_1_0_offset.h" @@ -280,6 +281,37 @@ static void gfxhub_v2_0_disable_identity_aperture(struct amdgpu_device *adev) } +static void +gfxhub_v2_0_program_kernel_trap_tba_tma(struct amdgpu_device *adev) +{ + u64 tba_addr = AMDGPU_VA_RESERVED_TRAP_START(adev); + u64 tma_addr = tba_addr + AMDGPU_TRAP_TMA_OFFSET; /* tba_addr + 0x1800 */ + int vmid; + + if (!amdgpu_trap_is_enabled(adev) || + !adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q) + return; + + WARN_ON(!IS_ALIGNED(tba_addr, 256)); + WARN_ON(!IS_ALIGNED(tma_addr, 256)); + + mutex_lock(&adev->srbm_mutex); + for (vmid = 1; vmid < adev->vm_manager.first_kfd_vmid; vmid++) { + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, vmid, 0); + WREG32_SOC15(GC, 0, mmSQ_SHADER_TBA_LO, + lower_32_bits(tba_addr >> 8)); + WREG32_SOC15(GC, 0, mmSQ_SHADER_TBA_HI, + upper_32_bits(tba_addr >> 8) | + (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT)); + WREG32_SOC15(GC, 0, mmSQ_SHADER_TMA_LO, + lower_32_bits(tma_addr >> 8)); + WREG32_SOC15(GC, 0, mmSQ_SHADER_TMA_HI, + upper_32_bits(tma_addr >> 8)); + } + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0); + mutex_unlock(&adev->srbm_mutex); +} + static void gfxhub_v2_0_setup_vmid_config(struct amdgpu_device *adev) { struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)]; @@ -327,6 +359,9 @@ static void gfxhub_v2_0_setup_vmid_config(struct amdgpu_device *adev) } hub->vm_cntx_cntl = tmp; + + if (amdgpu_trap_is_enabled(adev)) + gfxhub_v2_0_program_kernel_trap_tba_tma(adev); } static void gfxhub_v2_0_program_invalidation(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c index 9e6a6e13dec0..204a8817df30 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c @@ -22,6 +22,7 @@ */ #include "amdgpu.h" +#include "amdgpu_trap.h" #include "gfxhub_v3_0.h" #include "gc/gc_11_0_0_offset.h" @@ -287,6 +288,37 @@ static void gfxhub_v3_0_disable_identity_aperture(struct amdgpu_device *adev) } +static void +gfxhub_v3_0_program_kernel_trap_tba_tma(struct amdgpu_device *adev) +{ + u64 tba_addr = AMDGPU_VA_RESERVED_TRAP_START(adev); + u64 tma_addr = tba_addr + AMDGPU_TRAP_TMA_OFFSET; /* tba_addr + 0x1800 */ + int vmid; + + if (!amdgpu_trap_is_enabled(adev) || + !adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q) + return; + + WARN_ON(!IS_ALIGNED(tba_addr, 256)); + WARN_ON(!IS_ALIGNED(tma_addr, 256)); + + mutex_lock(&adev->srbm_mutex); + for (vmid = 1; vmid < adev->vm_manager.first_kfd_vmid; vmid++) { + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, vmid, 0); + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_LO, + lower_32_bits(tba_addr >> 8)); + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_HI, + upper_32_bits(tba_addr >> 8) | + (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT)); + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_LO, + lower_32_bits(tma_addr >> 8)); + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_HI, + upper_32_bits(tma_addr >> 8)); + } + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0); + mutex_unlock(&adev->srbm_mutex); +} + static void gfxhub_v3_0_setup_vmid_config(struct amdgpu_device *adev) { struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)]; @@ -334,6 +366,9 @@ static void gfxhub_v3_0_setup_vmid_config(struct amdgpu_device *adev) } hub->vm_cntx_cntl = tmp; + + if (amdgpu_trap_is_enabled(adev)) + gfxhub_v3_0_program_kernel_trap_tba_tma(adev); } static void gfxhub_v3_0_program_invalidation(struct amdgpu_device *adev) -- 2.34.1
