Move more of the common code into the gmc.
Signed-off-by: Alex Deucher <[email protected]>
---
drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c | 172 +++++++++++++++++++++++-
drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h | 9 +-
2 files changed, 174 insertions(+), 7 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
index 5c1237a30b865..bbdedcaff4086 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
@@ -928,16 +928,158 @@ void amdgpu_gmc_fw_reg_write_reg_wait(struct
amdgpu_device *adev,
dev_err(adev->dev, "failed to write reg %x wait reg %x\n", reg0, reg1);
}
-int amdgpu_gmc_flush_gpu_tlb_helper(struct amdgpu_device *adev, uint32_t vmid,
- uint32_t vmhub, uint32_t flush_type)
+static bool amdgpu_gmc_use_invalidate_semaphore(struct amdgpu_device *adev,
+ uint32_t vmhub)
+{
+ return (!AMDGPU_IS_GFXHUB(vmhub) &&
+ !amdgpu_sriov_vf(adev));
+}
+
+uint64_t amdgpu_gmc_emit_flush_gpu_tlb_helper(struct amdgpu_ring *ring,
+ unsigned vmid, uint64_t pd_addr)
+{
+ bool use_semaphore = amdgpu_gmc_use_invalidate_semaphore(ring->adev,
ring->vm_hub);
+ struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub];
+ uint32_t req = hub->vmhub_funcs->get_invalidate_req(vmid, 0);
+ unsigned eng = ring->vm_inv_eng;
+
+ /*
+ * It may lose gpuvm invalidate acknowldege state across power-gating
+ * off cycle, add semaphore acquire before invalidation and semaphore
+ * release after invalidation to avoid entering power gated state
+ * to WA the Issue
+ */
+
+ if (use_semaphore)
+ /* a read return value of 1 means semaphore acuqire */
+ amdgpu_ring_emit_reg_wait(ring,
+ hub->vm_inv_eng0_sem +
+ hub->eng_distance * eng, 0x1, 0x1);
+
+ amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 +
+ (hub->ctx_addr_distance * vmid),
+ lower_32_bits(pd_addr));
+
+ amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 +
+ (hub->ctx_addr_distance * vmid),
+ upper_32_bits(pd_addr));
+
+ amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req +
+ hub->eng_distance * eng,
+ hub->vm_inv_eng0_ack +
+ hub->eng_distance * eng,
+ req, 1 << vmid);
+
+ if (use_semaphore)
+ /*
+ * add semaphore release after invalidation,
+ * write with 0 means semaphore release
+ */
+ amdgpu_ring_emit_wreg(ring, hub->vm_inv_eng0_sem +
+ hub->eng_distance * eng, 0);
+
+ return pd_addr;
+}
+
+static void amdgpu_gmc_flush_vm_hub_mmio(struct amdgpu_device *adev,
+ uint32_t vmid, unsigned int vmhub,
+ uint32_t flush_type)
+{
+ bool use_semaphore = amdgpu_gmc_use_invalidate_semaphore(adev, vmhub);
+ struct amdgpu_vmhub *hub = &adev->vmhub[vmhub];
+ u32 inv_req = hub->vmhub_funcs->get_invalidate_req(vmid, flush_type);
+ u32 tmp;
+ /* Use register 17 for GART */
+ const unsigned eng = 17;
+ unsigned int i;
+ unsigned char hub_ip = 0;
+
+ hub_ip = AMDGPU_IS_GFXHUB(vmhub) ?
+ GC_HWIP : MMHUB_HWIP;
+
+ /* disabllow gfxoff when we invalidate */
+ if (hub_ip == GC_HWIP)
+ amdgpu_gfx_off_ctrl(adev, false);
+
+ spin_lock(&adev->gmc.invalidate_lock);
+ /*
+ * It may lose gpuvm invalidate acknowldege state across power-gating
+ * off cycle, add semaphore acquire before invalidation and semaphore
+ * release after invalidation to avoid entering power gated state
+ * to WA the Issue
+ */
+ if (use_semaphore) {
+ for (i = 0; i < adev->usec_timeout; i++) {
+ /* a read return value of 1 means semaphore acuqire */
+ tmp = RREG32_RLC_NO_KIQ(hub->vm_inv_eng0_sem +
+ hub->eng_distance * eng, hub_ip);
+ if (tmp & 0x1)
+ break;
+ udelay(1);
+ }
+
+ if (i >= adev->usec_timeout)
+ dev_err(adev->dev,
+ "Timeout waiting for sem acquire in VM
flush!\n");
+ }
+
+ WREG32_RLC_NO_KIQ(hub->vm_inv_eng0_req + hub->eng_distance * eng,
+ inv_req, hub_ip);
+
+ /* Wait for ACK with a delay.*/
+ for (i = 0; i < adev->usec_timeout; i++) {
+ tmp = RREG32_RLC_NO_KIQ(hub->vm_inv_eng0_ack +
+ hub->eng_distance * eng, hub_ip);
+ tmp &= 1 << vmid;
+ if (tmp)
+ break;
+
+ udelay(1);
+ }
+
+ if (use_semaphore)
+ /*
+ * add semaphore release after invalidation,
+ * write with 0 means semaphore release
+ */
+ WREG32_RLC_NO_KIQ(hub->vm_inv_eng0_sem +
+ hub->eng_distance * eng, 0, hub_ip);
+
+ /* Issue additional private vm invalidation to MMHUB */
+ if (!AMDGPU_IS_GFXHUB(vmhub) &&
+ hub->vm_l2_bank_select_reserved_cid2 &&
+ !amdgpu_sriov_vf(adev)) {
+ inv_req = RREG32_NO_KIQ(hub->vm_l2_bank_select_reserved_cid2);
+ /* bit 25: RSERVED_CACHE_PRIVATE_INVALIDATION */
+ inv_req |= (1 << 25);
+ /* Issue private invalidation */
+ WREG32_NO_KIQ(hub->vm_l2_bank_select_reserved_cid2, inv_req);
+ /* Read back to ensure invalidation is done*/
+ RREG32_NO_KIQ(hub->vm_l2_bank_select_reserved_cid2);
+ }
+
+ spin_unlock(&adev->gmc.invalidate_lock);
+
+ if (hub_ip == GC_HWIP)
+ amdgpu_gfx_off_ctrl(adev, true);
+
+ if (i < adev->usec_timeout)
+ return;
+
+ dev_err(adev->dev, "Timeout waiting for VM flush ACK!\n");
+}
+
+void amdgpu_gmc_flush_gpu_tlb_helper(struct amdgpu_device *adev, uint32_t vmid,
+ uint32_t vmhub, uint32_t flush_type)
{
struct dma_fence *fence;
/* Use register 17 for GART */
- u32 inst, eng = 17;
+ const unsigned eng = 17;
+ u32 inst;
int r;
if (AMDGPU_IS_GFXHUB(vmhub) && !adev->gfx.is_poweron)
- return 0;
+ return;
if (vmhub >= AMDGPU_MMHUB0(0))
inst = 0;
@@ -947,6 +1089,7 @@ int amdgpu_gmc_flush_gpu_tlb_helper(struct amdgpu_device
*adev, uint32_t vmid,
/* flush hdp cache */
amdgpu_device_flush_hdp(adev, NULL);
+ /* use SDMA */
mutex_lock(&adev->mman.default_entity.lock);
r = amdgpu_ttm_tlb_inv(adev, vmid, vmhub, eng, flush_type, inst,
&adev->mman.default_entity,
@@ -961,7 +1104,26 @@ int amdgpu_gmc_flush_gpu_tlb_helper(struct amdgpu_device
*adev, uint32_t vmid,
exit:
mutex_unlock(&adev->mman.default_entity.lock);
- return r;
+ /* if SDMA fails, use MMIO */
+ if (r)
+ amdgpu_gmc_flush_vm_hub_mmio(adev, vmid, vmhub, flush_type);
+}
+
+void amdgpu_gmc_flush_gpu_tlb_pasid_helper(struct amdgpu_device *adev,
+ unsigned int vmid, uint32_t
flush_type,
+ bool all_hub, uint32_t inst)
+{
+ int i;
+
+ if (all_hub) {
+ for_each_set_bit(i, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS)
+ amdgpu_gmc_flush_gpu_tlb_helper(adev, vmid, i,
+ flush_type);
+ } else {
+ amdgpu_gmc_flush_gpu_tlb_helper(adev, vmid,
+ AMDGPU_GFXHUB(inst),
+ flush_type);
+ }
}
/**
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
index 0135be1418a91..64d02d38044bb 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
@@ -450,8 +450,13 @@ void amdgpu_gmc_fw_reg_write_reg_wait(struct amdgpu_device
*adev,
uint32_t reg0, uint32_t reg1,
uint32_t ref, uint32_t mask,
uint32_t xcc_inst);
-int amdgpu_gmc_flush_gpu_tlb_helper(struct amdgpu_device *adev, uint32_t vmid,
- uint32_t vmhub, uint32_t flush_type);
+uint64_t amdgpu_gmc_emit_flush_gpu_tlb_helper(struct amdgpu_ring *ring,
+ unsigned vmid, uint64_t pd_addr);
+void amdgpu_gmc_flush_gpu_tlb_helper(struct amdgpu_device *adev, uint32_t vmid,
+ uint32_t vmhub, uint32_t flush_type);
+void amdgpu_gmc_flush_gpu_tlb_pasid_helper(struct amdgpu_device *adev,
+ unsigned int vmid, uint32_t
flush_type,
+ bool all_hub, uint32_t inst);
extern void amdgpu_gmc_tmz_set(struct amdgpu_device *adev);
extern void amdgpu_gmc_noretry_set(struct amdgpu_device *adev);
--
2.55.0
