Use the new gmc helpers for tlb invalidations.
Signed-off-by: Alex Deucher <[email protected]>
---
drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c | 213 +------------------------
1 file changed, 5 insertions(+), 208 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c
b/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c
index dba83a90550da..54686066094f0 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c
@@ -180,20 +180,6 @@ static void gmc_v12_0_set_irq_funcs(struct amdgpu_device
*adev)
}
}
-/**
- * gmc_v12_0_use_invalidate_semaphore - judge whether to use semaphore
- *
- * @adev: amdgpu_device pointer
- * @vmhub: vmhub type
- *
- */
-static bool gmc_v12_0_use_invalidate_semaphore(struct amdgpu_device *adev,
- uint32_t vmhub)
-{
- return ((vmhub == AMDGPU_MMHUB0(0)) &&
- (!amdgpu_sriov_vf(adev)));
-}
-
static bool gmc_v12_0_get_vmid_pasid_mapping_info(
struct amdgpu_device *adev,
uint8_t vmid, uint16_t *p_pasid)
@@ -203,140 +189,6 @@ static bool gmc_v12_0_get_vmid_pasid_mapping_info(
return !!(*p_pasid);
}
-/*
- * GART
- * VMID 0 is the physical GPU addresses as used by the kernel.
- * VMIDs 1-15 are used for userspace clients and are handled
- * by the amdgpu vm/hsa code.
- */
-
-static void gmc_v12_0_flush_vm_hub(struct amdgpu_device *adev, uint32_t vmid,
- unsigned int vmhub, uint32_t flush_type)
-{
- bool use_semaphore = gmc_v12_0_use_invalidate_semaphore(adev, vmhub);
- struct amdgpu_vmhub *hub = &adev->vmhub[vmhub];
- u32 inv_req = hub->vmhub_funcs->get_invalidate_req(vmid, flush_type);
- u32 tmp;
- /* Use register 17 for GART */
- const unsigned eng = 17;
- unsigned int i;
- unsigned char hub_ip = 0;
-
- hub_ip = (vmhub == AMDGPU_GFXHUB(0)) ?
- GC_HWIP : MMHUB_HWIP;
-
- spin_lock(&adev->gmc.invalidate_lock);
- /*
- * It may lose gpuvm invalidate acknowldege state across power-gating
- * off cycle, add semaphore acquire before invalidation and semaphore
- * release after invalidation to avoid entering power gated state
- * to WA the Issue
- */
-
- /* TODO: It needs to continue working on debugging with semaphore for
GFXHUB as well. */
- if (use_semaphore) {
- for (i = 0; i < adev->usec_timeout; i++) {
- /* a read return value of 1 means semaphore acuqire */
- tmp = RREG32_RLC_NO_KIQ(hub->vm_inv_eng0_sem +
- hub->eng_distance * eng, hub_ip);
- if (tmp & 0x1)
- break;
- udelay(1);
- }
-
- if (i >= adev->usec_timeout)
- dev_err(adev->dev,
- "Timeout waiting for sem acquire in VM
flush!\n");
- }
-
- WREG32_RLC_NO_KIQ(hub->vm_inv_eng0_req + hub->eng_distance * eng,
inv_req, hub_ip);
-
- /* Wait for ACK with a delay.*/
- for (i = 0; i < adev->usec_timeout; i++) {
- tmp = RREG32_RLC_NO_KIQ(hub->vm_inv_eng0_ack +
- hub->eng_distance * eng, hub_ip);
- tmp &= 1 << vmid;
- if (tmp)
- break;
-
- udelay(1);
- }
-
- /* TODO: It needs to continue working on debugging with semaphore for
GFXHUB as well. */
- if (use_semaphore)
- /*
- * add semaphore release after invalidation,
- * write with 0 means semaphore release
- */
- WREG32_RLC_NO_KIQ(hub->vm_inv_eng0_sem +
- hub->eng_distance * eng, 0, hub_ip);
-
- /* Issue additional private vm invalidation to MMHUB */
- if ((vmhub != AMDGPU_GFXHUB(0)) &&
- (hub->vm_l2_bank_select_reserved_cid2) &&
- !amdgpu_sriov_vf(adev)) {
- inv_req = RREG32_NO_KIQ(hub->vm_l2_bank_select_reserved_cid2);
- /* bit 25: RSERVED_CACHE_PRIVATE_INVALIDATION */
- inv_req |= (1 << 25);
- /* Issue private invalidation */
- WREG32_NO_KIQ(hub->vm_l2_bank_select_reserved_cid2, inv_req);
- /* Read back to ensure invalidation is done*/
- RREG32_NO_KIQ(hub->vm_l2_bank_select_reserved_cid2);
- }
-
- spin_unlock(&adev->gmc.invalidate_lock);
-
- if (i < adev->usec_timeout)
- return;
-
- dev_err(adev->dev, "Timeout waiting for VM flush ACK!\n");
-}
-
-/**
- * gmc_v12_0_flush_gpu_tlb - gart tlb flush callback
- *
- * @adev: amdgpu_device pointer
- * @vmid: vm instance to flush
- * @vmhub: which hub to flush
- * @flush_type: the flush type
- *
- * Flush the TLB for the requested page table.
- */
-static void gmc_v12_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
- uint32_t vmhub, uint32_t flush_type)
-{
- if ((vmhub == AMDGPU_GFXHUB(0)) && !adev->gfx.is_poweron)
- return;
-
- /* flush hdp cache */
- amdgpu_device_flush_hdp(adev, NULL);
-
- /* This is necessary for SRIOV as well as for GFXOFF to function
- * properly under bare metal
- */
- if ((adev->gfx.kiq[0].ring.sched.ready ||
adev->mes.ring[0].sched.ready) &&
- (amdgpu_sriov_runtime(adev) || !amdgpu_sriov_vf(adev))) {
- struct amdgpu_vmhub *hub = &adev->vmhub[vmhub];
- const unsigned eng = 17;
- u32 inv_req = hub->vmhub_funcs->get_invalidate_req(vmid,
flush_type);
- u32 req = hub->vm_inv_eng0_req + hub->eng_distance * eng;
- u32 ack = hub->vm_inv_eng0_ack + hub->eng_distance * eng;
-
- amdgpu_gmc_fw_reg_write_reg_wait(adev, req, ack, inv_req,
- 1 << vmid, GET_INST(GC, 0));
- return;
- }
-
- /* disabllow gfxoff when we invalidate */
- if (vmhub == AMDGPU_GFXHUB(0))
- amdgpu_gfx_off_ctrl(adev, false);
-
- gmc_v12_0_flush_vm_hub(adev, vmid, vmhub, 0);
-
- if (vmhub == AMDGPU_GFXHUB(0))
- amdgpu_gfx_off_ctrl(adev, true);
-}
-
/**
* gmc_v12_0_flush_gpu_tlb_pasid - tlb flush via pasid
*
@@ -353,7 +205,7 @@ static void gmc_v12_0_flush_gpu_tlb_pasid(struct
amdgpu_device *adev,
bool all_hub, uint32_t inst)
{
uint16_t queried;
- int vmid, i;
+ int vmid;
if (adev->enable_uni_mes &&
adev->mes.ring[AMDGPU_MES_SCHED_PIPE].sched.ready &&
(adev->mes.sched_version & AMDGPU_MES_VERSION_MASK) >= 0x84) {
@@ -379,66 +231,11 @@ static void gmc_v12_0_flush_gpu_tlb_pasid(struct
amdgpu_device *adev,
if (!valid || queried != pasid)
continue;
- if (all_hub) {
- for_each_set_bit(i, adev->vmhubs_mask,
- AMDGPU_MAX_VMHUBS)
- gmc_v12_0_flush_gpu_tlb(adev, vmid, i,
- flush_type);
- } else {
- gmc_v12_0_flush_gpu_tlb(adev, vmid, AMDGPU_GFXHUB(0),
- flush_type);
- }
+ amdgpu_gmc_flush_gpu_tlb_pasid_helper(adev, vmid, flush_type,
+ all_hub, inst);
}
}
-static uint64_t gmc_v12_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring,
- unsigned vmid, uint64_t pd_addr)
-{
- bool use_semaphore = gmc_v12_0_use_invalidate_semaphore(ring->adev,
ring->vm_hub);
- struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub];
- uint32_t req = hub->vmhub_funcs->get_invalidate_req(vmid, 0);
- unsigned eng = ring->vm_inv_eng;
-
- /*
- * It may lose gpuvm invalidate acknowldege state across power-gating
- * off cycle, add semaphore acquire before invalidation and semaphore
- * release after invalidation to avoid entering power gated state
- * to WA the Issue
- */
-
- /* TODO: It needs to continue working on debugging with semaphore for
GFXHUB as well. */
- if (use_semaphore)
- /* a read return value of 1 means semaphore acuqire */
- amdgpu_ring_emit_reg_wait(ring,
- hub->vm_inv_eng0_sem +
- hub->eng_distance * eng, 0x1, 0x1);
-
- amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 +
- (hub->ctx_addr_distance * vmid),
- lower_32_bits(pd_addr));
-
- amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 +
- (hub->ctx_addr_distance * vmid),
- upper_32_bits(pd_addr));
-
- amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req +
- hub->eng_distance * eng,
- hub->vm_inv_eng0_ack +
- hub->eng_distance * eng,
- req, 1 << vmid);
-
- /* TODO: It needs to continue working on debugging with semaphore for
GFXHUB as well. */
- if (use_semaphore)
- /*
- * add semaphore release after invalidation,
- * write with 0 means semaphore release
- */
- amdgpu_ring_emit_wreg(ring, hub->vm_inv_eng0_sem +
- hub->eng_distance * eng, 0);
-
- return pd_addr;
-}
-
static void gmc_v12_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned
vmid,
unsigned pasid)
{
@@ -577,9 +374,9 @@ static unsigned int gmc_v12_0_get_dcc_alignment(struct
amdgpu_device *adev)
}
static const struct amdgpu_gmc_funcs gmc_v12_0_gmc_funcs = {
- .flush_gpu_tlb = gmc_v12_0_flush_gpu_tlb,
+ .flush_gpu_tlb = amdgpu_gmc_flush_gpu_tlb_helper,
.flush_gpu_tlb_pasid = gmc_v12_0_flush_gpu_tlb_pasid,
- .emit_flush_gpu_tlb = gmc_v12_0_emit_flush_gpu_tlb,
+ .emit_flush_gpu_tlb = amdgpu_gmc_emit_flush_gpu_tlb_helper,
.emit_pasid_mapping = gmc_v12_0_emit_pasid_mapping,
.get_vm_pde = gmc_v12_0_get_vm_pde,
.get_vm_pte = gmc_v12_0_get_vm_pte,
--
2.55.0
