On Sat, Sep 5, 2026 at 5:15 AM Srinivasan Shanmugam
<[email protected]> wrote:
>
> Implement the program_kernel_trap_vmids() vmhub callback for all
> supported hardware generations. Each implementation programs
> SQ_SHADER_TBA/TMA for kernel queue VMIDs (1..first_kfd_vmid-1)
> via SRBM select.
>
> TBA points to the device-level CWSR ISA BO — same binary for all VMIDs.
> TMA is set to AMDGPU_VA_RESERVED_TRAP_START, the fixed virtual address
> where each VM's kq_tma_bo is mapped. The hardware resolves this VA
> through the active VMID's page tables, giving each process its own
> physical TMA buffer without requiring per-submission register updates.
>
> User queue VMIDs (first_kfd_vmid..15) must not be touched here — they
> are programmed by MES via the ADD_QUEUE packet's trap_handler_addr field.
>
> The function is called at two points:
> 1. amdgpu_trap_init() — first boot, after ISA BO is ready
> 2. setup_vmid_config() — GPU resume, after GART is restored
>
> GFX versions covered:
> GFX10 (gfxhub_v2_0): mm-prefixed registers, single XCC
> GFX11 (gfxhub_v3_0): reg-prefixed registers, single XCC
> GFX11.5 (gfxhub_v11_5_0): reg-prefixed registers, single XCC
> GFX12 (gfxhub_v12_0): reg-prefixed registers, single XCC
> GFX12.1 (gfxhub_v12_1): reg-prefixed registers, multi-XCC
>
> Suggested-by: Christian König <[email protected]>
> Suggested-by: Alexander Deucher <[email protected]>
> Cc: Lijo Lazar <[email protected]>
> Cc: Timur Kristóf <[email protected]>
> Cc: Samuel Pitoiset <[email protected]>
> Cc: Natalie Vock <[email protected]>
> Signed-off-by: Srinivasan Shanmugam <[email protected]>
> Change-Id: I2fa05a8ac96e5fe4b6e5f79dbdf62f32b8dbf96f
> ---
> drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c | 38 +++++++++++++++++++++
> drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c | 38 +++++++++++++++++++++
> drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c | 36 +++++++++++++++++++
> drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c | 38 +++++++++++++++++++++
> drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c | 33 ++++++++++++++++++
> 5 files changed, 183 insertions(+)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c
> b/drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c
> index 652eea6eae4a..b6f43c8274d4 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c
> @@ -22,6 +22,7 @@
> */
>
> #include "amdgpu.h"
> +#include "amdgpu_trap.h"
> #include "gfxhub_v11_5_0.h"
>
> #include "gc/gc_11_5_0_offset.h"
> @@ -290,6 +291,39 @@ static void
> gfxhub_v11_5_0_disable_identity_aperture(struct amdgpu_device *adev)
>
> }
>
> +/*
> + * MES owns kernel VMIDs but does not program trap handler registers.
> + * Program SQ_SHADER_TBA/TMA directly via SRBM select so the first-level
> + * CWSR handler is active for kernel queue VMIDs.
> + */
> +static void gfxhub_v11_5_0_program_kernel_trap_vmids(struct amdgpu_device
> *adev)
I'd suggest renaming it to something like
program_kernel_trap_tba_tma() since it's not really programming the
vmids, they just happen to be per vmid.
> +{
> + u64 tba_addr = amdgpu_bo_gpu_offset(adev->trap_info->isa_bo);
> + u64 tma_addr = AMDGPU_VA_RESERVED_TRAP_START(adev);
> + int i;
> +
> + if (!adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q)
> + return;
> +
> + WARN_ON(!IS_ALIGNED(tba_addr, 256));
> +
> + mutex_lock(&adev->srbm_mutex);
> + for (i = 1; i < adev->vm_manager.first_kfd_vmid; i++) {
> + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, i, 0);
> + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_LO,
> + lower_32_bits(tba_addr >> 8));
> + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_HI,
> + upper_32_bits(tba_addr >> 8) |
> + (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT));
If we set the enable bit, we need to make sure trap handler is valid
for each vmid.
> + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_LO,
> + lower_32_bits(tma_addr >> 8));
> + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_HI,
> + upper_32_bits(tma_addr >> 8));
> + }
> + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0);
> + mutex_unlock(&adev->srbm_mutex);
> +}
> +
> static void gfxhub_v11_5_0_setup_vmid_config(struct amdgpu_device *adev)
> {
> struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)];
> @@ -337,6 +371,9 @@ static void gfxhub_v11_5_0_setup_vmid_config(struct
> amdgpu_device *adev)
> }
>
> hub->vm_cntx_cntl = tmp;
> +
> + if (amdgpu_trap_is_enabled(adev))
> + gfxhub_v11_5_0_program_kernel_trap_vmids(adev);
> }
>
> static void gfxhub_v11_5_0_program_invalidation(struct amdgpu_device *adev)
> @@ -459,6 +496,7 @@ static void
> gfxhub_v11_5_0_set_fault_enable_default(struct amdgpu_device *adev,
> static const struct amdgpu_vmhub_funcs gfxhub_v11_5_0_vmhub_funcs = {
> .print_l2_protection_fault_status =
> gfxhub_v11_5_0_print_l2_protection_fault_status,
> .get_invalidate_req = gfxhub_v11_5_0_get_invalidate_req,
> + .program_kernel_trap_vmids = gfxhub_v11_5_0_program_kernel_trap_vmids,
What do we need the callback for?
Alex
> };
>
> static void gfxhub_v11_5_0_init(struct amdgpu_device *adev)
> diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c
> b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c
> index 6cbf837d50dd..5b2f0f59b895 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c
> @@ -22,6 +22,7 @@
> */
>
> #include "amdgpu.h"
> +#include "amdgpu_trap.h"
> #include "gfxhub_v12_0.h"
>
> #include "gc/gc_12_0_0_offset.h"
> @@ -295,6 +296,39 @@ static void
> gfxhub_v12_0_disable_identity_aperture(struct amdgpu_device *adev)
>
> }
>
> +/*
> + * MES owns kernel VMIDs but does not program trap handler registers.
> + * Program SQ_SHADER_TBA/TMA directly via SRBM select so the first-level
> + * CWSR handler is active for kernel queue VMIDs.
> + */
> +static void gfxhub_v12_0_program_kernel_trap_vmids(struct amdgpu_device
> *adev)
> +{
> + u64 tba_addr = amdgpu_bo_gpu_offset(adev->trap_info->isa_bo);
> + u64 tma_addr = AMDGPU_VA_RESERVED_TRAP_START(adev);
> + int i;
> +
> + if (!adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q)
> + return;
> +
> + WARN_ON(!IS_ALIGNED(tba_addr, 256));
> +
> + mutex_lock(&adev->srbm_mutex);
> + for (i = 1; i < adev->vm_manager.first_kfd_vmid; i++) {
> + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, i, 0);
> + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_LO,
> + lower_32_bits(tba_addr >> 8));
> + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_HI,
> + upper_32_bits(tba_addr >> 8) |
> + (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT));
> + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_LO,
> + lower_32_bits(tma_addr >> 8));
> + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_HI,
> + upper_32_bits(tma_addr >> 8));
> + }
> + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0);
> + mutex_unlock(&adev->srbm_mutex);
> +}
> +
> static void gfxhub_v12_0_setup_vmid_config(struct amdgpu_device *adev)
> {
> struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)];
> @@ -342,6 +376,9 @@ static void gfxhub_v12_0_setup_vmid_config(struct
> amdgpu_device *adev)
> }
>
> hub->vm_cntx_cntl = tmp;
> +
> + if (amdgpu_trap_is_enabled(adev))
> + gfxhub_v12_0_program_kernel_trap_vmids(adev);
> }
>
> static void gfxhub_v12_0_program_invalidation(struct amdgpu_device *adev)
> @@ -464,6 +501,7 @@ static void gfxhub_v12_0_set_fault_enable_default(struct
> amdgpu_device *adev,
> static const struct amdgpu_vmhub_funcs gfxhub_v12_0_vmhub_funcs = {
> .print_l2_protection_fault_status =
> gfxhub_v12_0_print_l2_protection_fault_status,
> .get_invalidate_req = gfxhub_v12_0_get_invalidate_req,
> + .program_kernel_trap_vmids = gfxhub_v12_0_program_kernel_trap_vmids,
> };
>
> static void gfxhub_v12_0_init(struct amdgpu_device *adev)
> diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c
> b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c
> index 4c2fd1e6616e..528917913950 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c
> @@ -21,6 +21,7 @@
> *
> */
> #include "amdgpu.h"
> +#include "amdgpu_trap.h"
> #include "amdgpu_xcp.h"
> #include "gfxhub_v12_1.h"
>
> @@ -406,6 +407,37 @@ static void
> gfxhub_v12_1_xcc_disable_identity_aperture(struct amdgpu_device *ade
> }
> }
>
> +static void gfxhub_v12_1_program_kernel_trap_vmids(struct amdgpu_device
> *adev)
> +{
> + u64 tba_addr = amdgpu_bo_gpu_offset(adev->trap_info->isa_bo);
> + u64 tma_addr = AMDGPU_VA_RESERVED_TRAP_START(adev);
> + u32 xcc_mask = GENMASK(NUM_XCC(adev->gfx.xcc_mask) - 1, 0);
> + int i, j;
> +
> + if (!adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q)
> + return;
> +
> + WARN_ON(!IS_ALIGNED(tba_addr, 256));
> +
> + for_each_inst(j, xcc_mask) {
> + mutex_lock(&adev->srbm_mutex);
> + for (i = 1; i < adev->vm_manager.first_kfd_vmid; i++) {
> + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, i, j);
> + WREG32_SOC15(GC, GET_INST(GC, j), regSQ_SHADER_TBA_LO,
> + lower_32_bits(tba_addr >> 8));
> + WREG32_SOC15(GC, GET_INST(GC, j), regSQ_SHADER_TBA_HI,
> + upper_32_bits(tba_addr >> 8) |
> + (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT));
> + WREG32_SOC15(GC, GET_INST(GC, j), regSQ_SHADER_TMA_LO,
> + lower_32_bits(tma_addr >> 8));
> + WREG32_SOC15(GC, GET_INST(GC, j), regSQ_SHADER_TMA_HI,
> + upper_32_bits(tma_addr >> 8));
> + }
> + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, j);
> + mutex_unlock(&adev->srbm_mutex);
> + }
> +}
> +
> static void gfxhub_v12_1_xcc_setup_vmid_config(struct amdgpu_device *adev,
> uint32_t xcc_mask)
> {
> @@ -468,6 +500,9 @@ static void gfxhub_v12_1_xcc_setup_vmid_config(struct
> amdgpu_device *adev,
>
> hub->vm_cntx_cntl = tmp;
> }
> +
> + if (amdgpu_trap_is_enabled(adev))
> + gfxhub_v12_1_program_kernel_trap_vmids(adev);
> }
>
> static void gfxhub_v12_1_xcc_program_invalidation(struct amdgpu_device *adev,
> @@ -751,6 +786,7 @@ static void
> gfxhub_v12_1_print_l2_protection_fault_status(struct amdgpu_device *
> static const struct amdgpu_vmhub_funcs gfxhub_v12_1_vmhub_funcs = {
> .print_l2_protection_fault_status =
> gfxhub_v12_1_print_l2_protection_fault_status,
> .get_invalidate_req = gfxhub_v12_1_get_invalidate_req,
> + .program_kernel_trap_vmids = gfxhub_v12_1_program_kernel_trap_vmids,
> };
>
> static void gfxhub_v12_1_xcc_init(struct amdgpu_device *adev, uint32_t
> xcc_mask)
> diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c
> b/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c
> index 9ea593e2c719..47a25df2a0c2 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c
> @@ -22,6 +22,7 @@
> */
>
> #include "amdgpu.h"
> +#include "amdgpu_trap.h"
> #include "gfxhub_v2_0.h"
>
> #include "gc/gc_10_1_0_offset.h"
> @@ -280,6 +281,39 @@ static void gfxhub_v2_0_disable_identity_aperture(struct
> amdgpu_device *adev)
>
> }
>
> +/*
> + * MES owns kernel VMIDs but does not program trap handler registers.
> + * Program SQ_SHADER_TBA/TMA directly via SRBM select so the first-level
> + * CWSR handler is active for kernel queue VMIDs.
> + */
> +static void gfxhub_v2_0_program_kernel_trap_vmids(struct amdgpu_device *adev)
> +{
> + u64 tba_addr = amdgpu_bo_gpu_offset(adev->trap_info->isa_bo);
> + u64 tma_addr = AMDGPU_VA_RESERVED_TRAP_START(adev);
> + int i;
> +
> + if (!adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q)
> + return;
> +
> + WARN_ON(!IS_ALIGNED(tba_addr, 256));
> +
> + mutex_lock(&adev->srbm_mutex);
> + for (i = 1; i < adev->vm_manager.first_kfd_vmid; i++) {
> + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, i, 0);
> + WREG32_SOC15(GC, 0, mmSQ_SHADER_TBA_LO,
> + lower_32_bits(tba_addr >> 8));
> + WREG32_SOC15(GC, 0, mmSQ_SHADER_TBA_HI,
> + upper_32_bits(tba_addr >> 8) |
> + (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT));
> + WREG32_SOC15(GC, 0, mmSQ_SHADER_TMA_LO,
> + lower_32_bits(tma_addr >> 8));
> + WREG32_SOC15(GC, 0, mmSQ_SHADER_TMA_HI,
> + upper_32_bits(tma_addr >> 8));
> + }
> + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0);
> + mutex_unlock(&adev->srbm_mutex);
> +}
> +
> static void gfxhub_v2_0_setup_vmid_config(struct amdgpu_device *adev)
> {
> struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)];
> @@ -327,6 +361,9 @@ static void gfxhub_v2_0_setup_vmid_config(struct
> amdgpu_device *adev)
> }
>
> hub->vm_cntx_cntl = tmp;
> +
> + if (amdgpu_trap_is_enabled(adev))
> + gfxhub_v2_0_program_kernel_trap_vmids(adev);
> }
>
> static void gfxhub_v2_0_program_invalidation(struct amdgpu_device *adev)
> @@ -428,6 +465,7 @@ static void gfxhub_v2_0_set_fault_enable_default(struct
> amdgpu_device *adev,
> static const struct amdgpu_vmhub_funcs gfxhub_v2_0_vmhub_funcs = {
> .print_l2_protection_fault_status =
> gfxhub_v2_0_print_l2_protection_fault_status,
> .get_invalidate_req = gfxhub_v2_0_get_invalidate_req,
> + .program_kernel_trap_vmids = gfxhub_v2_0_program_kernel_trap_vmids,
> };
>
> static void gfxhub_v2_0_init(struct amdgpu_device *adev)
> diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c
> b/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c
> index 9e6a6e13dec0..2f59de6e9407 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c
> @@ -22,6 +22,7 @@
> */
>
> #include "amdgpu.h"
> +#include "amdgpu_trap.h"
> #include "gfxhub_v3_0.h"
>
> #include "gc/gc_11_0_0_offset.h"
> @@ -287,6 +288,34 @@ static void gfxhub_v3_0_disable_identity_aperture(struct
> amdgpu_device *adev)
>
> }
>
> +static void gfxhub_v3_0_program_kernel_trap_vmids(struct amdgpu_device *adev)
> +{
> + u64 tba_addr = amdgpu_bo_gpu_offset(adev->trap_info->isa_bo);
> + u64 tma_addr = AMDGPU_VA_RESERVED_TRAP_START(adev);
> + int i;
> +
> + if (!adev->gfx.funcs || !adev->gfx.funcs->select_me_pipe_q)
> + return;
> +
> + WARN_ON(!IS_ALIGNED(tba_addr, 256));
> +
> + mutex_lock(&adev->srbm_mutex);
> + for (i = 1; i < adev->vm_manager.first_kfd_vmid; i++) {
> + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, i, 0);
> + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_LO,
> + lower_32_bits(tba_addr >> 8));
> + WREG32_SOC15(GC, 0, regSQ_SHADER_TBA_HI,
> + upper_32_bits(tba_addr >> 8) |
> + (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT));
> + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_LO,
> + lower_32_bits(tma_addr >> 8));
> + WREG32_SOC15(GC, 0, regSQ_SHADER_TMA_HI,
> + upper_32_bits(tma_addr >> 8));
> + }
> + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0);
> + mutex_unlock(&adev->srbm_mutex);
> +}
> +
> static void gfxhub_v3_0_setup_vmid_config(struct amdgpu_device *adev)
> {
> struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)];
> @@ -334,6 +363,9 @@ static void gfxhub_v3_0_setup_vmid_config(struct
> amdgpu_device *adev)
> }
>
> hub->vm_cntx_cntl = tmp;
> +
> + if (amdgpu_trap_is_enabled(adev))
> + gfxhub_v3_0_program_kernel_trap_vmids(adev);
> }
>
> static void gfxhub_v3_0_program_invalidation(struct amdgpu_device *adev)
> @@ -456,6 +488,7 @@ static void gfxhub_v3_0_set_fault_enable_default(struct
> amdgpu_device *adev,
> static const struct amdgpu_vmhub_funcs gfxhub_v3_0_vmhub_funcs = {
> .print_l2_protection_fault_status =
> gfxhub_v3_0_print_l2_protection_fault_status,
> .get_invalidate_req = gfxhub_v3_0_get_invalidate_req,
> + .program_kernel_trap_vmids = gfxhub_v3_0_program_kernel_trap_vmids,
> };
>
> static void gfxhub_v3_0_init(struct amdgpu_device *adev)
> --
> 2.34.1
>
