AMD General
> -----Original Message-----
> From: amd-gfx <[email protected]> On Behalf Of Alex
> Deucher
> Sent: Wednesday, September 2, 2026 4:10 AM
> To: [email protected]; Koenig, Christian
> <[email protected]>
> Cc: Deucher, Alexander <[email protected]>
> Subject: [PATCH 22/31] drm/amdgpu/gmc: rework pasid flushing
>
> Split out all of the various flush methods and use the new pasid flush method
> enum
> to determine which one to use.
>
> Signed-off-by: Alex Deucher <[email protected]>
> ---
> drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c | 247 +++++++++++++++++++-----
> drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c | 2 +
> drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c | 2 +
> drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c | 2 +
> 4 files changed, 201 insertions(+), 52 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
> index 8a975eddd75c7..2800eebe50649 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
> @@ -788,9 +788,66 @@ void amdgpu_gmc_flush_gpu_tlb_gart(struct
> amdgpu_device *adev,
> dev_err(adev->dev, "Error flushing GPU TLB using the SDMA (%d)!\n", r);
> }
>
> -int amdgpu_gmc_flush_gpu_tlb_pasid(struct amdgpu_device *adev, uint16_t
> pasid,
> - uint32_t flush_type, bool all_hub,
> - uint32_t inst)
> +static int amdgpu_gmc_flush_gpu_tlb_sdma_helper(struct amdgpu_device *adev,
> + uint16_t pasid, uint32_t
> flush_type,
> + bool all_hub, uint32_t inst)
> +{
> + struct amdgpu_ring *ring;
> + struct dma_fence *fence;
> + struct amdgpu_job *job;
> + uint16_t queried;
> + /* Use register 17 for GART */
> + u32 eng = 17;
> + int vmid, r, i, ndw;
> +
> + /* flush hdp cache */
> + amdgpu_device_flush_hdp(adev, NULL);
> +
> + ndw = ALIGN(adev->mman.buffer_funcs->tlb_inv_num_dw * 16 *
> AMDGPU_MAX_VMHUBS, 8);
> + ring = to_amdgpu_ring(adev->mman.buffer_funcs_scheds[0]);
> +
> + mutex_lock(&adev->mman.default_entity.lock);
> + r = amdgpu_job_alloc_with_ib(ring->adev, &adev-
> >mman.default_entity.base,
> + AMDGPU_FENCE_OWNER_UNDEFINED,
> + ndw * 4, AMDGPU_IB_POOL_IMMEDIATE,
> + AMDGPU_KERNEL_JOB_ID_VM_UPDATE,
> + &job);
> + if (r)
> + goto exit;
> +
> + for (vmid = 1; vmid < 16; vmid++) {
> + bool valid;
> +
> + valid = adev->gmc.gmc_funcs->get_vmid_pasid_mapping_info(adev,
> vmid, inst,
> +
> &queried);
> + if (!valid || queried != pasid)
> + continue;
> +
> + if (all_hub) {
> + for_each_set_bit(i, adev->vmhubs_mask,
> AMDGPU_MAX_VMHUBS)
> + amdgpu_emit_tlb_inv(adev, &job->ibs[0], vmid,
> i, eng,
> + flush_type, inst);
> + } else {
> + amdgpu_emit_tlb_inv(adev, &job->ibs[0], vmid,
> AMDGPU_GFXHUB(inst), eng,
> + flush_type, inst);
> + }
> + }
> + amdgpu_ring_pad_ib(ring, &job->ibs[0]);
> + fence = amdgpu_job_submit(job);
> + mutex_unlock(&adev->mman.default_entity.lock);
It should be double unlock, and it unlocks again at exit. we can
remove it .
Regards
Jesse
> +
> + dma_fence_wait(fence, false);
> + dma_fence_put(fence);
> +
> +exit:
> + mutex_unlock(&adev->mman.default_entity.lock);
> +
> + return r;
> +}
> +
> +static int amdgpu_gmc_flush_gpu_tlb_kiq_helper(struct amdgpu_device *adev,
> + uint16_t pasid, uint32_t
> flush_type,
> + bool all_hub, uint32_t inst)
> {
> struct amdgpu_ring *ring = &adev->gfx.kiq[inst].ring;
> struct amdgpu_kiq *kiq = &adev->gfx.kiq[inst]; @@ -798,6 +855,110 @@
> int amdgpu_gmc_flush_gpu_tlb_pasid(struct amdgpu_device *adev, uint16_t pasid,
> int r, cnt = 0;
> uint32_t seq;
>
> + /* 2 dwords flush + 8 dwords fence */
> + ndw = kiq->pmf->invalidate_tlbs_size + 8;
> +
> + if (adev->gmc.flush_tlb_needs_extra_type_2)
> + ndw += kiq->pmf->invalidate_tlbs_size;
> +
> + if (adev->gmc.flush_tlb_needs_extra_type_0)
> + ndw += kiq->pmf->invalidate_tlbs_size;
> +
> + spin_lock(&adev->gfx.kiq[inst].ring_lock);
> + r = amdgpu_ring_alloc(ring, ndw);
> + if (r) {
> + spin_unlock(&adev->gfx.kiq[inst].ring_lock);
> + return r;
> + }
> + if (adev->gmc.flush_tlb_needs_extra_type_2)
> + kiq->pmf->kiq_invalidate_tlbs(ring, pasid, 2, all_hub);
> +
> + if (flush_type == 2 && adev->gmc.flush_tlb_needs_extra_type_0)
> + kiq->pmf->kiq_invalidate_tlbs(ring, pasid, 0, all_hub);
> +
> + kiq->pmf->kiq_invalidate_tlbs(ring, pasid, flush_type, all_hub);
> + r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT);
> + if (r) {
> + amdgpu_ring_undo(ring);
> + spin_unlock(&adev->gfx.kiq[inst].ring_lock);
> + return r;
> + }
> +
> + amdgpu_ring_commit(ring);
> + spin_unlock(&adev->gfx.kiq[inst].ring_lock);
> +
> + r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
> +
> + might_sleep();
> + while (r < 1 && cnt++ < MAX_KIQ_REG_TRY &&
> + !amdgpu_reset_pending(adev->reset_domain)) {
> + msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
> + r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
> + }
> +
> + if (cnt > MAX_KIQ_REG_TRY) {
> + dev_err(adev->dev, "timeout waiting for kiq fence\n");
> + r = -ETIME;
> + } else
> + r = 0;
> +
> + return r;
> +}
> +
> +static int amdgpu_gmc_flush_gpu_tlb_mes_helper(struct amdgpu_device *adev,
> + uint16_t pasid, uint32_t
> flush_type,
> + bool all_hub, uint32_t inst) {
> + struct mes_inv_tlbs_pasid_input input = {0};
> + int r;
> +
> + input.xcc_id = inst;
> + input.pasid = pasid;
> + input.flush_type = flush_type;
> +
> + /* MES will invalidate hubs for the device(including slave xcc)
> + * from master, ignore request from slave
> + */
> + if (!amdgpu_gfx_is_master_xcc(adev, inst))
> + return -EINVAL;
> +
> + input.hub_id = AMDGPU_GFXHUB(0);
> + amdgpu_mes_lock(&adev->mes);
> + r = adev->mes.funcs->invalidate_tlbs_pasid(&adev->mes, &input);
> + amdgpu_mes_unlock(&adev->mes);
> + if (r)
> + return r;
> +
> + if (all_hub) {
> + /* invalidate mm_hub */
> + if (test_bit(AMDGPU_MMHUB0(0), adev->vmhubs_mask)) {
> + input.hub_id = AMDGPU_MMHUB0(0);
> + amdgpu_mes_lock(&adev->mes);
> + r = adev->mes.funcs->invalidate_tlbs_pasid(&adev->mes,
> &input);
> + amdgpu_mes_unlock(&adev->mes);
> + if (r)
> + return r;
> + }
> + if (test_bit(AMDGPU_MMHUB1(0), adev->vmhubs_mask)) {
> + input.hub_id = AMDGPU_MMHUB1(0);
> + amdgpu_mes_lock(&adev->mes);
> + r = adev->mes.funcs->invalidate_tlbs_pasid(&adev->mes,
> &input);
> + amdgpu_mes_unlock(&adev->mes);
> + if (r)
> + return r;
> + }
> + }
> + return 0;
> +}
> +
> +int amdgpu_gmc_flush_gpu_tlb_pasid(struct amdgpu_device *adev, uint16_t
> pasid,
> + uint32_t flush_type, bool all_hub,
> + uint32_t inst)
> +{
> + struct amdgpu_ring *ring;
> + bool use_mmio = false;
> + int r;
> +
> /*
> * A GPU reset should flush all TLBs anyway, so no need to do
> * this while one is ongoing.
> @@ -805,8 +966,38 @@ int amdgpu_gmc_flush_gpu_tlb_pasid(struct
> amdgpu_device *adev, uint16_t pasid,
> if (!down_read_trylock(&adev->reset_domain->sem))
> return 0;
>
> - if (!adev->gmc.flush_pasid_uses_kiq || !ring->sched.ready) {
> + switch (adev->gmc.pasid_inv_method) {
> + case AMDGPU_TLB_INV_METHOD_MMIO:
> + default:
> + use_mmio = true;
> + break;
> + case AMDGPU_TLB_INV_METHOD_SDMA:
> + ring = to_amdgpu_ring(adev->mman.buffer_funcs_scheds[0]);
> + if (!ring->sched.ready)
> + use_mmio = true;
> + else
> + r = amdgpu_gmc_flush_gpu_tlb_sdma_helper(adev, pasid,
> flush_type,
> + all_hub, inst);
> + break;
> + case AMDGPU_TLB_INV_METHOD_KIQ:
> + ring = &adev->gfx.kiq[inst].ring;
> + if (!adev->gmc.flush_pasid_uses_kiq || !ring->sched.ready)
> + use_mmio = true;
> + else
> + r = amdgpu_gmc_flush_gpu_tlb_kiq_helper(adev, pasid,
> flush_type,
> + all_hub, inst);
> + break;
> + case AMDGPU_TLB_INV_METHOD_MES:
> + ring = &adev->mes.ring[MES_PIPE_INST(inst, 0)];
> + if (!ring->sched.ready)
> + use_mmio = true;
> + else
> + r = amdgpu_gmc_flush_gpu_tlb_mes_helper(adev, pasid,
> flush_type,
> + all_hub, inst);
> + break;
> + }
>
> + if (use_mmio) {
> if (!adev->gmc.gmc_funcs->flush_gpu_tlb_pasid) {
> r = 0;
> goto error_unlock_reset;
> @@ -825,54 +1016,6 @@ int amdgpu_gmc_flush_gpu_tlb_pasid(struct
> amdgpu_device *adev, uint16_t pasid,
> adev->gmc.gmc_funcs->flush_gpu_tlb_pasid(adev, pasid,
> flush_type, all_hub,
> inst);
> - r = 0;
> - } else {
> - /* 2 dwords flush + 8 dwords fence */
> - ndw = kiq->pmf->invalidate_tlbs_size + 8;
> -
> - if (adev->gmc.flush_tlb_needs_extra_type_2)
> - ndw += kiq->pmf->invalidate_tlbs_size;
> -
> - if (adev->gmc.flush_tlb_needs_extra_type_0)
> - ndw += kiq->pmf->invalidate_tlbs_size;
> -
> - spin_lock(&adev->gfx.kiq[inst].ring_lock);
> - r = amdgpu_ring_alloc(ring, ndw);
> - if (r) {
> - spin_unlock(&adev->gfx.kiq[inst].ring_lock);
> - goto error_unlock_reset;
> - }
> - if (adev->gmc.flush_tlb_needs_extra_type_2)
> - kiq->pmf->kiq_invalidate_tlbs(ring, pasid, 2, all_hub);
> -
> - if (flush_type == 2 && adev->gmc.flush_tlb_needs_extra_type_0)
> - kiq->pmf->kiq_invalidate_tlbs(ring, pasid, 0, all_hub);
> -
> - kiq->pmf->kiq_invalidate_tlbs(ring, pasid, flush_type, all_hub);
> - r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT);
> - if (r) {
> - amdgpu_ring_undo(ring);
> - spin_unlock(&adev->gfx.kiq[inst].ring_lock);
> - goto error_unlock_reset;
> - }
> -
> - amdgpu_ring_commit(ring);
> - spin_unlock(&adev->gfx.kiq[inst].ring_lock);
> -
> - r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
> -
> - might_sleep();
> - while (r < 1 && cnt++ < MAX_KIQ_REG_TRY &&
> - !amdgpu_reset_pending(adev->reset_domain)) {
> - msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
> - r = amdgpu_fence_wait_polling(ring, seq,
> MAX_KIQ_REG_WAIT);
> - }
> -
> - if (cnt > MAX_KIQ_REG_TRY) {
> - dev_err(adev->dev, "timeout waiting for kiq fence\n");
> - r = -ETIME;
> - } else
> - r = 0;
> }
>
> error_unlock_reset:
> diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
> b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
> index 4bb10a89ce582..e7c529620199b 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
> @@ -974,6 +974,8 @@ static int gmc_v10_0_hw_init(struct amdgpu_ip_block
> *ip_block)
> int r;
>
> adev->gmc.flush_pasid_uses_kiq = !amdgpu_emu_mode;
> + if (adev->gmc.flush_pasid_uses_kiq)
> + adev->gmc.pasid_inv_method =
> AMDGPU_TLB_INV_METHOD_KIQ;
>
> /* The sequence of these two function calls matters.*/
> gmc_v10_0_init_golden_registers(adev);
> diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c
> b/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c
> index acab9f4180da6..a35c84cc7385f 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c
> @@ -950,6 +950,8 @@ static int gmc_v11_0_hw_init(struct amdgpu_ip_block
> *ip_block)
> int r;
>
> adev->gmc.flush_pasid_uses_kiq = !amdgpu_emu_mode;
> + if (adev->gmc.flush_pasid_uses_kiq)
> + adev->gmc.pasid_inv_method =
> AMDGPU_TLB_INV_METHOD_KIQ;
>
> /* The sequence of these two function calls matters.*/
> gmc_v11_0_init_golden_registers(adev);
> diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
> b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
> index 3c13a6920171a..317b44412b5a6 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
> @@ -2162,6 +2162,8 @@ static int gmc_v9_0_hw_init(struct amdgpu_ip_block
> *ip_block)
> int i, r;
>
> adev->gmc.flush_pasid_uses_kiq = true;
> + if (adev->gmc.flush_pasid_uses_kiq)
> + adev->gmc.pasid_inv_method =
> AMDGPU_TLB_INV_METHOD_KIQ;
>
> /* Vega20+XGMI caches PTEs in TC and TLB. Add a heavy-weight TLB
> flush
> * (type 2), which flushes both. Due to a race condition with
> --
> 2.55.0
