Split out all of the various flush methods and use the new pasid flush method enum to determine which one to use.
Signed-off-by: Alex Deucher <[email protected]> --- drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c | 247 +++++++++++++++++++----- drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c | 2 + drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c | 2 + drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c | 2 + 4 files changed, 201 insertions(+), 52 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c index 8a975eddd75c7..2800eebe50649 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c @@ -788,9 +788,66 @@ void amdgpu_gmc_flush_gpu_tlb_gart(struct amdgpu_device *adev, dev_err(adev->dev, "Error flushing GPU TLB using the SDMA (%d)!\n", r); } -int amdgpu_gmc_flush_gpu_tlb_pasid(struct amdgpu_device *adev, uint16_t pasid, - uint32_t flush_type, bool all_hub, - uint32_t inst) +static int amdgpu_gmc_flush_gpu_tlb_sdma_helper(struct amdgpu_device *adev, + uint16_t pasid, uint32_t flush_type, + bool all_hub, uint32_t inst) +{ + struct amdgpu_ring *ring; + struct dma_fence *fence; + struct amdgpu_job *job; + uint16_t queried; + /* Use register 17 for GART */ + u32 eng = 17; + int vmid, r, i, ndw; + + /* flush hdp cache */ + amdgpu_device_flush_hdp(adev, NULL); + + ndw = ALIGN(adev->mman.buffer_funcs->tlb_inv_num_dw * 16 * AMDGPU_MAX_VMHUBS, 8); + ring = to_amdgpu_ring(adev->mman.buffer_funcs_scheds[0]); + + mutex_lock(&adev->mman.default_entity.lock); + r = amdgpu_job_alloc_with_ib(ring->adev, &adev->mman.default_entity.base, + AMDGPU_FENCE_OWNER_UNDEFINED, + ndw * 4, AMDGPU_IB_POOL_IMMEDIATE, + AMDGPU_KERNEL_JOB_ID_VM_UPDATE, + &job); + if (r) + goto exit; + + for (vmid = 1; vmid < 16; vmid++) { + bool valid; + + valid = adev->gmc.gmc_funcs->get_vmid_pasid_mapping_info(adev, vmid, inst, + &queried); + if (!valid || queried != pasid) + continue; + + if (all_hub) { + for_each_set_bit(i, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS) + amdgpu_emit_tlb_inv(adev, &job->ibs[0], vmid, i, eng, + flush_type, inst); + } else { + amdgpu_emit_tlb_inv(adev, &job->ibs[0], vmid, AMDGPU_GFXHUB(inst), eng, + flush_type, inst); + } + } + amdgpu_ring_pad_ib(ring, &job->ibs[0]); + fence = amdgpu_job_submit(job); + mutex_unlock(&adev->mman.default_entity.lock); + + dma_fence_wait(fence, false); + dma_fence_put(fence); + +exit: + mutex_unlock(&adev->mman.default_entity.lock); + + return r; +} + +static int amdgpu_gmc_flush_gpu_tlb_kiq_helper(struct amdgpu_device *adev, + uint16_t pasid, uint32_t flush_type, + bool all_hub, uint32_t inst) { struct amdgpu_ring *ring = &adev->gfx.kiq[inst].ring; struct amdgpu_kiq *kiq = &adev->gfx.kiq[inst]; @@ -798,6 +855,110 @@ int amdgpu_gmc_flush_gpu_tlb_pasid(struct amdgpu_device *adev, uint16_t pasid, int r, cnt = 0; uint32_t seq; + /* 2 dwords flush + 8 dwords fence */ + ndw = kiq->pmf->invalidate_tlbs_size + 8; + + if (adev->gmc.flush_tlb_needs_extra_type_2) + ndw += kiq->pmf->invalidate_tlbs_size; + + if (adev->gmc.flush_tlb_needs_extra_type_0) + ndw += kiq->pmf->invalidate_tlbs_size; + + spin_lock(&adev->gfx.kiq[inst].ring_lock); + r = amdgpu_ring_alloc(ring, ndw); + if (r) { + spin_unlock(&adev->gfx.kiq[inst].ring_lock); + return r; + } + if (adev->gmc.flush_tlb_needs_extra_type_2) + kiq->pmf->kiq_invalidate_tlbs(ring, pasid, 2, all_hub); + + if (flush_type == 2 && adev->gmc.flush_tlb_needs_extra_type_0) + kiq->pmf->kiq_invalidate_tlbs(ring, pasid, 0, all_hub); + + kiq->pmf->kiq_invalidate_tlbs(ring, pasid, flush_type, all_hub); + r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT); + if (r) { + amdgpu_ring_undo(ring); + spin_unlock(&adev->gfx.kiq[inst].ring_lock); + return r; + } + + amdgpu_ring_commit(ring); + spin_unlock(&adev->gfx.kiq[inst].ring_lock); + + r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); + + might_sleep(); + while (r < 1 && cnt++ < MAX_KIQ_REG_TRY && + !amdgpu_reset_pending(adev->reset_domain)) { + msleep(MAX_KIQ_REG_BAILOUT_INTERVAL); + r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); + } + + if (cnt > MAX_KIQ_REG_TRY) { + dev_err(adev->dev, "timeout waiting for kiq fence\n"); + r = -ETIME; + } else + r = 0; + + return r; +} + +static int amdgpu_gmc_flush_gpu_tlb_mes_helper(struct amdgpu_device *adev, + uint16_t pasid, uint32_t flush_type, + bool all_hub, uint32_t inst) +{ + struct mes_inv_tlbs_pasid_input input = {0}; + int r; + + input.xcc_id = inst; + input.pasid = pasid; + input.flush_type = flush_type; + + /* MES will invalidate hubs for the device(including slave xcc) + * from master, ignore request from slave + */ + if (!amdgpu_gfx_is_master_xcc(adev, inst)) + return -EINVAL; + + input.hub_id = AMDGPU_GFXHUB(0); + amdgpu_mes_lock(&adev->mes); + r = adev->mes.funcs->invalidate_tlbs_pasid(&adev->mes, &input); + amdgpu_mes_unlock(&adev->mes); + if (r) + return r; + + if (all_hub) { + /* invalidate mm_hub */ + if (test_bit(AMDGPU_MMHUB0(0), adev->vmhubs_mask)) { + input.hub_id = AMDGPU_MMHUB0(0); + amdgpu_mes_lock(&adev->mes); + r = adev->mes.funcs->invalidate_tlbs_pasid(&adev->mes, &input); + amdgpu_mes_unlock(&adev->mes); + if (r) + return r; + } + if (test_bit(AMDGPU_MMHUB1(0), adev->vmhubs_mask)) { + input.hub_id = AMDGPU_MMHUB1(0); + amdgpu_mes_lock(&adev->mes); + r = adev->mes.funcs->invalidate_tlbs_pasid(&adev->mes, &input); + amdgpu_mes_unlock(&adev->mes); + if (r) + return r; + } + } + return 0; +} + +int amdgpu_gmc_flush_gpu_tlb_pasid(struct amdgpu_device *adev, uint16_t pasid, + uint32_t flush_type, bool all_hub, + uint32_t inst) +{ + struct amdgpu_ring *ring; + bool use_mmio = false; + int r; + /* * A GPU reset should flush all TLBs anyway, so no need to do * this while one is ongoing. @@ -805,8 +966,38 @@ int amdgpu_gmc_flush_gpu_tlb_pasid(struct amdgpu_device *adev, uint16_t pasid, if (!down_read_trylock(&adev->reset_domain->sem)) return 0; - if (!adev->gmc.flush_pasid_uses_kiq || !ring->sched.ready) { + switch (adev->gmc.pasid_inv_method) { + case AMDGPU_TLB_INV_METHOD_MMIO: + default: + use_mmio = true; + break; + case AMDGPU_TLB_INV_METHOD_SDMA: + ring = to_amdgpu_ring(adev->mman.buffer_funcs_scheds[0]); + if (!ring->sched.ready) + use_mmio = true; + else + r = amdgpu_gmc_flush_gpu_tlb_sdma_helper(adev, pasid, flush_type, + all_hub, inst); + break; + case AMDGPU_TLB_INV_METHOD_KIQ: + ring = &adev->gfx.kiq[inst].ring; + if (!adev->gmc.flush_pasid_uses_kiq || !ring->sched.ready) + use_mmio = true; + else + r = amdgpu_gmc_flush_gpu_tlb_kiq_helper(adev, pasid, flush_type, + all_hub, inst); + break; + case AMDGPU_TLB_INV_METHOD_MES: + ring = &adev->mes.ring[MES_PIPE_INST(inst, 0)]; + if (!ring->sched.ready) + use_mmio = true; + else + r = amdgpu_gmc_flush_gpu_tlb_mes_helper(adev, pasid, flush_type, + all_hub, inst); + break; + } + if (use_mmio) { if (!adev->gmc.gmc_funcs->flush_gpu_tlb_pasid) { r = 0; goto error_unlock_reset; @@ -825,54 +1016,6 @@ int amdgpu_gmc_flush_gpu_tlb_pasid(struct amdgpu_device *adev, uint16_t pasid, adev->gmc.gmc_funcs->flush_gpu_tlb_pasid(adev, pasid, flush_type, all_hub, inst); - r = 0; - } else { - /* 2 dwords flush + 8 dwords fence */ - ndw = kiq->pmf->invalidate_tlbs_size + 8; - - if (adev->gmc.flush_tlb_needs_extra_type_2) - ndw += kiq->pmf->invalidate_tlbs_size; - - if (adev->gmc.flush_tlb_needs_extra_type_0) - ndw += kiq->pmf->invalidate_tlbs_size; - - spin_lock(&adev->gfx.kiq[inst].ring_lock); - r = amdgpu_ring_alloc(ring, ndw); - if (r) { - spin_unlock(&adev->gfx.kiq[inst].ring_lock); - goto error_unlock_reset; - } - if (adev->gmc.flush_tlb_needs_extra_type_2) - kiq->pmf->kiq_invalidate_tlbs(ring, pasid, 2, all_hub); - - if (flush_type == 2 && adev->gmc.flush_tlb_needs_extra_type_0) - kiq->pmf->kiq_invalidate_tlbs(ring, pasid, 0, all_hub); - - kiq->pmf->kiq_invalidate_tlbs(ring, pasid, flush_type, all_hub); - r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT); - if (r) { - amdgpu_ring_undo(ring); - spin_unlock(&adev->gfx.kiq[inst].ring_lock); - goto error_unlock_reset; - } - - amdgpu_ring_commit(ring); - spin_unlock(&adev->gfx.kiq[inst].ring_lock); - - r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); - - might_sleep(); - while (r < 1 && cnt++ < MAX_KIQ_REG_TRY && - !amdgpu_reset_pending(adev->reset_domain)) { - msleep(MAX_KIQ_REG_BAILOUT_INTERVAL); - r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); - } - - if (cnt > MAX_KIQ_REG_TRY) { - dev_err(adev->dev, "timeout waiting for kiq fence\n"); - r = -ETIME; - } else - r = 0; } error_unlock_reset: diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c index 4bb10a89ce582..e7c529620199b 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c @@ -974,6 +974,8 @@ static int gmc_v10_0_hw_init(struct amdgpu_ip_block *ip_block) int r; adev->gmc.flush_pasid_uses_kiq = !amdgpu_emu_mode; + if (adev->gmc.flush_pasid_uses_kiq) + adev->gmc.pasid_inv_method = AMDGPU_TLB_INV_METHOD_KIQ; /* The sequence of these two function calls matters.*/ gmc_v10_0_init_golden_registers(adev); diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c index acab9f4180da6..a35c84cc7385f 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c @@ -950,6 +950,8 @@ static int gmc_v11_0_hw_init(struct amdgpu_ip_block *ip_block) int r; adev->gmc.flush_pasid_uses_kiq = !amdgpu_emu_mode; + if (adev->gmc.flush_pasid_uses_kiq) + adev->gmc.pasid_inv_method = AMDGPU_TLB_INV_METHOD_KIQ; /* The sequence of these two function calls matters.*/ gmc_v11_0_init_golden_registers(adev); diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c index 13423258d33ed..e9e5a1e183231 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c @@ -2181,6 +2181,8 @@ static int gmc_v9_0_hw_init(struct amdgpu_ip_block *ip_block) int i, r; adev->gmc.flush_pasid_uses_kiq = true; + if (adev->gmc.flush_pasid_uses_kiq) + adev->gmc.pasid_inv_method = AMDGPU_TLB_INV_METHOD_KIQ; /* Vega20+XGMI caches PTEs in TC and TLB. Add a heavy-weight TLB flush * (type 2), which flushes both. Due to a race condition with -- 2.55.0
