Split out all of the various flush methods and use the new pasid flush method enum to determine which one to use. No intended change in behavior.
Signed-off-by: Alex Deucher <[email protected]> --- drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c | 247 +++++++++++++++++++----- drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c | 2 + drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c | 2 + drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c | 16 -- drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c | 28 --- drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c | 2 + drivers/gpu/drm/amd/amdgpu/mes_v12_0.c | 4 + drivers/gpu/drm/amd/amdgpu/mes_v12_1.c | 4 + 8 files changed, 209 insertions(+), 96 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c index f1c7bb37cfb8e..ed132878e390f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c @@ -794,9 +794,66 @@ void amdgpu_gmc_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid, dev_err(adev->dev, "Error flushing GPU TLB using the SDMA (%d)!\n", r); } -int amdgpu_gmc_flush_gpu_tlb_pasid(struct amdgpu_device *adev, uint16_t pasid, - uint32_t flush_type, bool all_hub, - uint32_t inst) +static int amdgpu_gmc_flush_gpu_tlb_sdma_helper(struct amdgpu_device *adev, + uint16_t pasid, uint32_t flush_type, + bool all_hub, uint32_t inst) +{ + struct amdgpu_ring *ring; + struct dma_fence *fence; + struct amdgpu_job *job; + uint16_t queried; + /* Use register 17 for GART */ + u32 eng = 17; + int vmid, r, i, ndw; + + /* flush hdp cache */ + amdgpu_device_flush_hdp(adev, NULL); + + ndw = ALIGN(adev->mman.buffer_funcs->tlb_inv_num_dw * 16 * AMDGPU_MAX_VMHUBS, 8); + ring = to_amdgpu_ring(adev->mman.buffer_funcs_scheds[0]); + + mutex_lock(&adev->mman.default_entity.lock); + r = amdgpu_job_alloc_with_ib(ring->adev, &adev->mman.default_entity.base, + AMDGPU_FENCE_OWNER_UNDEFINED, + ndw * 4, AMDGPU_IB_POOL_IMMEDIATE, + AMDGPU_KERNEL_JOB_ID_VM_UPDATE, + &job); + if (r) + goto exit; + + for (vmid = 1; vmid < 16; vmid++) { + bool valid; + + valid = adev->gmc.gmc_funcs->get_vmid_pasid_mapping_info(adev, vmid, inst, + &queried); + if (!valid || queried != pasid) + continue; + + if (all_hub) { + for_each_set_bit(i, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS) + amdgpu_emit_tlb_inv(adev, &job->ibs[0], vmid, i, eng, + flush_type, inst); + } else { + amdgpu_emit_tlb_inv(adev, &job->ibs[0], vmid, AMDGPU_GFXHUB(inst), eng, + flush_type, inst); + } + } + amdgpu_ring_pad_ib(ring, &job->ibs[0]); + fence = amdgpu_job_submit(job); + mutex_unlock(&adev->mman.default_entity.lock); + + dma_fence_wait(fence, false); + dma_fence_put(fence); + +exit: + mutex_unlock(&adev->mman.default_entity.lock); + + return r; +} + +static int amdgpu_gmc_flush_gpu_tlb_kiq_helper(struct amdgpu_device *adev, + uint16_t pasid, uint32_t flush_type, + bool all_hub, uint32_t inst) { struct amdgpu_ring *ring = &adev->gfx.kiq[inst].ring; struct amdgpu_kiq *kiq = &adev->gfx.kiq[inst]; @@ -804,6 +861,110 @@ int amdgpu_gmc_flush_gpu_tlb_pasid(struct amdgpu_device *adev, uint16_t pasid, int r, cnt = 0; uint32_t seq; + /* 2 dwords flush + 8 dwords fence */ + ndw = kiq->pmf->invalidate_tlbs_size + 8; + + if (adev->gmc.flush_tlb_needs_extra_type_2) + ndw += kiq->pmf->invalidate_tlbs_size; + + if (adev->gmc.flush_tlb_needs_extra_type_0) + ndw += kiq->pmf->invalidate_tlbs_size; + + spin_lock(&adev->gfx.kiq[inst].ring_lock); + r = amdgpu_ring_alloc(ring, ndw); + if (r) { + spin_unlock(&adev->gfx.kiq[inst].ring_lock); + return r; + } + if (adev->gmc.flush_tlb_needs_extra_type_2) + kiq->pmf->kiq_invalidate_tlbs(ring, pasid, 2, all_hub); + + if (flush_type == 2 && adev->gmc.flush_tlb_needs_extra_type_0) + kiq->pmf->kiq_invalidate_tlbs(ring, pasid, 0, all_hub); + + kiq->pmf->kiq_invalidate_tlbs(ring, pasid, flush_type, all_hub); + r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT); + if (r) { + amdgpu_ring_undo(ring); + spin_unlock(&adev->gfx.kiq[inst].ring_lock); + return r; + } + + amdgpu_ring_commit(ring); + spin_unlock(&adev->gfx.kiq[inst].ring_lock); + + r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); + + might_sleep(); + while (r < 1 && cnt++ < MAX_KIQ_REG_TRY && + !amdgpu_reset_pending(adev->reset_domain)) { + msleep(MAX_KIQ_REG_BAILOUT_INTERVAL); + r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); + } + + if (cnt > MAX_KIQ_REG_TRY) { + dev_err(adev->dev, "timeout waiting for kiq fence\n"); + r = -ETIME; + } else + r = 0; + + return r; +} + +static int amdgpu_gmc_flush_gpu_tlb_mes_helper(struct amdgpu_device *adev, + uint16_t pasid, uint32_t flush_type, + bool all_hub, uint32_t inst) +{ + struct mes_inv_tlbs_pasid_input input = {0}; + int r; + + input.xcc_id = inst; + input.pasid = pasid; + input.flush_type = flush_type; + + /* MES will invalidate hubs for the device(including slave xcc) + * from master, ignore request from slave + */ + if (!amdgpu_gfx_is_master_xcc(adev, inst)) + return -EINVAL; + + input.hub_id = AMDGPU_GFXHUB(0); + amdgpu_mes_lock(&adev->mes); + r = adev->mes.funcs->invalidate_tlbs_pasid(&adev->mes, &input); + amdgpu_mes_unlock(&adev->mes); + if (r) + return r; + + if (all_hub) { + /* invalidate mm_hub */ + if (test_bit(AMDGPU_MMHUB0(0), adev->vmhubs_mask)) { + input.hub_id = AMDGPU_MMHUB0(0); + amdgpu_mes_lock(&adev->mes); + r = adev->mes.funcs->invalidate_tlbs_pasid(&adev->mes, &input); + amdgpu_mes_unlock(&adev->mes); + if (r) + return r; + } + if (test_bit(AMDGPU_MMHUB1(0), adev->vmhubs_mask)) { + input.hub_id = AMDGPU_MMHUB1(0); + amdgpu_mes_lock(&adev->mes); + r = adev->mes.funcs->invalidate_tlbs_pasid(&adev->mes, &input); + amdgpu_mes_unlock(&adev->mes); + if (r) + return r; + } + } + return 0; +} + +int amdgpu_gmc_flush_gpu_tlb_pasid(struct amdgpu_device *adev, uint16_t pasid, + uint32_t flush_type, bool all_hub, + uint32_t inst) +{ + struct amdgpu_ring *ring; + bool use_mmio = false; + int r; + /* * A GPU reset should flush all TLBs anyway, so no need to do * this while one is ongoing. @@ -811,8 +972,38 @@ int amdgpu_gmc_flush_gpu_tlb_pasid(struct amdgpu_device *adev, uint16_t pasid, if (!down_read_trylock(&adev->reset_domain->sem)) return 0; - if (!adev->gmc.flush_pasid_uses_kiq || !ring->sched.ready) { + switch (adev->gmc.pasid_inv_method) { + case AMDGPU_TLB_INV_METHOD_MMIO: + default: + use_mmio = true; + break; + case AMDGPU_TLB_INV_METHOD_SDMA: + ring = to_amdgpu_ring(adev->mman.buffer_funcs_scheds[0]); + if (!ring->sched.ready) + use_mmio = true; + else + r = amdgpu_gmc_flush_gpu_tlb_sdma_helper(adev, pasid, flush_type, + all_hub, inst); + break; + case AMDGPU_TLB_INV_METHOD_KIQ: + ring = &adev->gfx.kiq[inst].ring; + if (!adev->gmc.flush_pasid_uses_kiq || !ring->sched.ready) + use_mmio = true; + else + r = amdgpu_gmc_flush_gpu_tlb_kiq_helper(adev, pasid, flush_type, + all_hub, inst); + break; + case AMDGPU_TLB_INV_METHOD_MES: + ring = &adev->mes.ring[MES_PIPE_INST(inst, 0)]; + if (!ring->sched.ready) + use_mmio = true; + else + r = amdgpu_gmc_flush_gpu_tlb_mes_helper(adev, pasid, flush_type, + all_hub, inst); + break; + } + if (use_mmio) { if (!adev->gmc.gmc_funcs->flush_gpu_tlb_pasid) { r = 0; goto error_unlock_reset; @@ -831,54 +1022,6 @@ int amdgpu_gmc_flush_gpu_tlb_pasid(struct amdgpu_device *adev, uint16_t pasid, adev->gmc.gmc_funcs->flush_gpu_tlb_pasid(adev, pasid, flush_type, all_hub, inst); - r = 0; - } else { - /* 2 dwords flush + 8 dwords fence */ - ndw = kiq->pmf->invalidate_tlbs_size + 8; - - if (adev->gmc.flush_tlb_needs_extra_type_2) - ndw += kiq->pmf->invalidate_tlbs_size; - - if (adev->gmc.flush_tlb_needs_extra_type_0) - ndw += kiq->pmf->invalidate_tlbs_size; - - spin_lock(&adev->gfx.kiq[inst].ring_lock); - r = amdgpu_ring_alloc(ring, ndw); - if (r) { - spin_unlock(&adev->gfx.kiq[inst].ring_lock); - goto error_unlock_reset; - } - if (adev->gmc.flush_tlb_needs_extra_type_2) - kiq->pmf->kiq_invalidate_tlbs(ring, pasid, 2, all_hub); - - if (flush_type == 2 && adev->gmc.flush_tlb_needs_extra_type_0) - kiq->pmf->kiq_invalidate_tlbs(ring, pasid, 0, all_hub); - - kiq->pmf->kiq_invalidate_tlbs(ring, pasid, flush_type, all_hub); - r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT); - if (r) { - amdgpu_ring_undo(ring); - spin_unlock(&adev->gfx.kiq[inst].ring_lock); - goto error_unlock_reset; - } - - amdgpu_ring_commit(ring); - spin_unlock(&adev->gfx.kiq[inst].ring_lock); - - r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); - - might_sleep(); - while (r < 1 && cnt++ < MAX_KIQ_REG_TRY && - !amdgpu_reset_pending(adev->reset_domain)) { - msleep(MAX_KIQ_REG_BAILOUT_INTERVAL); - r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); - } - - if (cnt > MAX_KIQ_REG_TRY) { - dev_err(adev->dev, "timeout waiting for kiq fence\n"); - r = -ETIME; - } else - r = 0; } error_unlock_reset: diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c index 4b3303ab4bff9..1e1339f36c2a0 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c @@ -975,6 +975,8 @@ static int gmc_v10_0_hw_init(struct amdgpu_ip_block *ip_block) int r; adev->gmc.flush_pasid_uses_kiq = !amdgpu_emu_mode; + if (adev->gmc.flush_pasid_uses_kiq) + adev->gmc.pasid_inv_method = AMDGPU_TLB_INV_METHOD_KIQ; /* The sequence of these two function calls matters.*/ gmc_v10_0_init_golden_registers(adev); diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c index 2d7ef1e1fd8cc..8f5587743c2be 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c @@ -955,6 +955,8 @@ static int gmc_v11_0_hw_init(struct amdgpu_ip_block *ip_block) int r; adev->gmc.flush_pasid_uses_kiq = !amdgpu_emu_mode; + if (adev->gmc.flush_pasid_uses_kiq) + adev->gmc.pasid_inv_method = AMDGPU_TLB_INV_METHOD_KIQ; /* The sequence of these two function calls matters.*/ gmc_v11_0_init_golden_registers(adev); diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c index 1da76d19eb3c5..89e271b0e9653 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c @@ -359,22 +359,6 @@ static void gmc_v12_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev, uint16_t queried; int vmid, i; - if (adev->enable_uni_mes && adev->mes.ring[AMDGPU_MES_SCHED_PIPE].sched.ready && - (adev->mes.sched_version & AMDGPU_MES_VERSION_MASK) >= 0x84) { - struct mes_inv_tlbs_pasid_input input = {0}; - input.pasid = pasid; - input.flush_type = flush_type; - input.hub_id = AMDGPU_GFXHUB(0); - /* MES will invalidate all gc_hub for the device from master */ - adev->mes.funcs->invalidate_tlbs_pasid(&adev->mes, &input); - if (all_hub) { - /* Only need to invalidate mm_hub now, gfx12 only support one mmhub */ - input.hub_id = AMDGPU_MMHUB0(0); - adev->mes.funcs->invalidate_tlbs_pasid(&adev->mes, &input); - } - return; - } - for (vmid = 1; vmid < 16; vmid++) { bool valid; diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c b/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c index 1c0d0121ad8ab..6561129022864 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c @@ -442,34 +442,6 @@ static void gmc_v12_1_flush_gpu_tlb_pasid(struct amdgpu_device *adev, uint16_t queried; int vmid, i; - if (adev->mes.ring[0].sched.ready && - (adev->mes.sched_version & AMDGPU_MES_VERSION_MASK) >= 0x6f) { - struct mes_inv_tlbs_pasid_input input = {0}; - input.xcc_id = inst; - input.pasid = pasid; - input.flush_type = flush_type; - - /* MES will invalidate hubs for the device(including slave xcc) from master, ignore request from slave */ - if (!amdgpu_gfx_is_master_xcc(adev, inst)) - return; - - input.hub_id = AMDGPU_GFXHUB(0); - adev->mes.funcs->invalidate_tlbs_pasid(&adev->mes, &input); - - if (all_hub) { - /* invalidate mm_hub */ - if (test_bit(AMDGPU_MMHUB0(0), adev->vmhubs_mask)) { - input.hub_id = AMDGPU_MMHUB0(0); - adev->mes.funcs->invalidate_tlbs_pasid(&adev->mes, &input); - } - if (test_bit(AMDGPU_MMHUB1(0), adev->vmhubs_mask)) { - input.hub_id = AMDGPU_MMHUB1(0); - adev->mes.funcs->invalidate_tlbs_pasid(&adev->mes, &input); - } - } - return; - } - for (vmid = 1; vmid < 16; vmid++) { bool valid; diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c index 8e2b89c36a4da..047d0ce5c5ab4 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c @@ -2164,6 +2164,8 @@ static int gmc_v9_0_hw_init(struct amdgpu_ip_block *ip_block) int i, r; adev->gmc.flush_pasid_uses_kiq = true; + if (adev->gmc.flush_pasid_uses_kiq) + adev->gmc.pasid_inv_method = AMDGPU_TLB_INV_METHOD_KIQ; /* Vega20+XGMI caches PTEs in TC and TLB. Add a heavy-weight TLB flush * (type 2), which flushes both. Due to a race condition with diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c b/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c index ee72afdb0a1d3..dd68db14c43f6 100644 --- a/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c +++ b/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c @@ -2097,6 +2097,10 @@ static int mes_v12_0_hw_init(struct amdgpu_ip_block *ip_block) adev->gfx.kiq[0].ring.sched.ready = false; adev->mes.ring[0].sched.ready = true; + if (adev->enable_uni_mes && + (adev->mes.sched_version & AMDGPU_MES_VERSION_MASK) >= 0x84) + adev->gmc.pasid_inv_method = AMDGPU_TLB_INV_METHOD_MES; + return 0; failure: diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c b/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c index a69f9b9c5cd67..3781d088d283f 100644 --- a/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c +++ b/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c @@ -1817,6 +1817,10 @@ static int mes_v12_1_hw_init(struct amdgpu_ip_block *ip_block) return r; } + if (adev->enable_uni_mes && + (adev->mes.sched_version & AMDGPU_MES_VERSION_MASK) >= 0x6f) + adev->gmc.pasid_inv_method = AMDGPU_TLB_INV_METHOD_MES; + return 0; } -- 2.55.0
