---
drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c | 226 +++++++++++++++++-------
drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h | 3 +
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 10 +-
3 files changed, 172 insertions(+), 67 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
index 7c0554506334..6a9bcd28b43c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
@@ -780,15 +780,108 @@ void amdgpu_gmc_flush_gpu_tlb(struct amdgpu_device
*adev, uint32_t vmid,
dev_err(adev->dev, "Error flushing GPU TLB using the SDMA (%d)!\n", r);
}
+static void amdgpu_gmc_flush_pasid_regs(struct amdgpu_device *adev, uint16_t pasid,
+ uint32_t flush_type, bool all_hub,
+ uint32_t inst)
+{
+ if (!adev->gmc.gmc_funcs->flush_gpu_tlb_pasid)
+ return;
+
+ if (adev->gmc.flush_tlb_needs_extra_type_2)
+ adev->gmc.gmc_funcs->flush_gpu_tlb_pasid(adev, pasid, 2,
all_hub,
+ inst);
+
+ if (adev->gmc.flush_tlb_needs_extra_type_0 && flush_type == 2)
+ adev->gmc.gmc_funcs->flush_gpu_tlb_pasid(adev, pasid, 0,
all_hub,
+ inst);
+
+ adev->gmc.gmc_funcs->flush_gpu_tlb_pasid(adev, pasid, flush_type,
all_hub,
+ inst);
+}
+
+/*
+ * Queue the invalidation on one XCC and return its fence without waiting, so
+ * that a caller flushing several XCCs can have them in flight together. The
+ * KIQ ring, its lock and its fence sequence are per XCC, so the submissions do
+ * not interfere with each other.
+ */
+static int amdgpu_gmc_flush_pasid_kiq_submit(struct amdgpu_device *adev,
+ uint16_t pasid, uint32_t
flush_type,
+ bool all_hub, uint32_t inst,
+ uint32_t *seq)
+{
+ struct amdgpu_kiq *kiq = &adev->gfx.kiq[inst];
+ struct amdgpu_ring *ring = &kiq->ring;
+ unsigned int ndw;
+ int r;
+
+ /* 2 dwords flush + 8 dwords fence */
+ ndw = kiq->pmf->invalidate_tlbs_size + 8;
+
+ if (adev->gmc.flush_tlb_needs_extra_type_2)
+ ndw += kiq->pmf->invalidate_tlbs_size;
+
+ if (adev->gmc.flush_tlb_needs_extra_type_0)
+ ndw += kiq->pmf->invalidate_tlbs_size;
+
+ spin_lock(&kiq->ring_lock);
+ r = amdgpu_ring_alloc(ring, ndw);
+ if (r) {
+ spin_unlock(&kiq->ring_lock);
+ return r;
+ }
+
+ if (adev->gmc.flush_tlb_needs_extra_type_2)
+ kiq->pmf->kiq_invalidate_tlbs(ring, pasid, 2, all_hub);
+
+ if (flush_type == 2 && adev->gmc.flush_tlb_needs_extra_type_0)
+ kiq->pmf->kiq_invalidate_tlbs(ring, pasid, 0, all_hub);
+
+ kiq->pmf->kiq_invalidate_tlbs(ring, pasid, flush_type, all_hub);
+ r = amdgpu_fence_emit_polling(ring, seq, MAX_KIQ_REG_WAIT);
+ if (r) {
+ amdgpu_ring_undo(ring);
+ spin_unlock(&kiq->ring_lock);
+ return r;
+ }
+
+ amdgpu_ring_commit(ring);
+ spin_unlock(&kiq->ring_lock);
+
+ return 0;
+}
+
+static int amdgpu_gmc_flush_pasid_kiq_wait(struct amdgpu_device *adev,
+ uint32_t inst, uint32_t seq)
+{
+ struct amdgpu_ring *ring = &adev->gfx.kiq[inst].ring;
+ int cnt = 0;
+ signed long r;
+
+ r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
+
+ might_sleep();
+ while (r < 1 && cnt++ < MAX_KIQ_REG_TRY &&
+ !amdgpu_reset_pending(adev->reset_domain)) {
+ msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
+ r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
+ }
+
+ if (cnt > MAX_KIQ_REG_TRY) {
+ dev_err(adev->dev, "timeout waiting for kiq fence\n");
+ return -ETIME;
+ }
+
+ return 0;
+}
+
int amdgpu_gmc_flush_gpu_tlb_pasid(struct amdgpu_device *adev, uint16_t pasid,
uint32_t flush_type, bool all_hub,
uint32_t inst)
{
struct amdgpu_ring *ring = &adev->gfx.kiq[inst].ring;
- struct amdgpu_kiq *kiq = &adev->gfx.kiq[inst];
- unsigned int ndw;
- int r, cnt = 0;
uint32_t seq;
+ int r;
/*
* A GPU reset should flush all TLBs anyway, so no need to do
@@ -798,76 +891,91 @@ int amdgpu_gmc_flush_gpu_tlb_pasid(struct amdgpu_device
*adev, uint16_t pasid,
return 0;
if (!adev->gmc.flush_pasid_uses_kiq || !ring->sched.ready) {
+ amdgpu_gmc_flush_pasid_regs(adev, pasid, flush_type, all_hub,
inst);
+ r = 0;
+ goto error_unlock_reset;
+ }
- if (!adev->gmc.gmc_funcs->flush_gpu_tlb_pasid) {
- r = 0;
- goto error_unlock_reset;
- }
-
- if (adev->gmc.flush_tlb_needs_extra_type_2)
- adev->gmc.gmc_funcs->flush_gpu_tlb_pasid(adev, pasid,
- 2, all_hub,
- inst);
+ r = amdgpu_gmc_flush_pasid_kiq_submit(adev, pasid, flush_type, all_hub,
+ inst, &seq);
+ if (r)
+ goto error_unlock_reset;
- if (adev->gmc.flush_tlb_needs_extra_type_0 && flush_type == 2)
- adev->gmc.gmc_funcs->flush_gpu_tlb_pasid(adev, pasid,
- 0, all_hub,
- inst);
+ r = amdgpu_gmc_flush_pasid_kiq_wait(adev, inst, seq);
- adev->gmc.gmc_funcs->flush_gpu_tlb_pasid(adev, pasid,
- flush_type, all_hub,
- inst);
- r = 0;
- } else {
- /* 2 dwords flush + 8 dwords fence */
- ndw = kiq->pmf->invalidate_tlbs_size + 8;
+error_unlock_reset:
+ up_read(&adev->reset_domain->sem);
+ return r;
+}
- if (adev->gmc.flush_tlb_needs_extra_type_2)
- ndw += kiq->pmf->invalidate_tlbs_size;
+/**
+ * amdgpu_gmc_flush_gpu_tlb_pasid_xccs - flush a pasid on several XCCs at once
+ *
+ * @adev: amdgpu_device pointer
+ * @pasid: pasid to be flushed
+ * @flush_type: the flush type
+ * @all_hub: flush all hubs
+ * @xcc_mask: mask of the XCCs to flush
+ *
+ * Submits the invalidation to every XCC in @xcc_mask before waiting for any of
+ * them, so the round trips overlap instead of running back to back. Flushing
+ * one XCC at a time costs num_xcc times the latency of a single one, which on
+ * a partition holding every XCC of the device is most of the cost of a compute
+ * TLB flush.
+ *
+ * The XCCs are independent of each other here: the page tables are already
+ * updated before any invalidation is issued, and nothing in an invalidation
+ * depends on another XCC having completed its own.
+ *
+ * Returns:
+ * 0 for success, the first error otherwise. Every XCC is flushed even if one
+ * of them fails.
+ */
+int amdgpu_gmc_flush_gpu_tlb_pasid_xccs(struct amdgpu_device *adev, uint16_t
pasid,
+ uint32_t flush_type, bool all_hub,
+ uint32_t xcc_mask)
+{
+ uint32_t seq[AMDGPU_MAX_GC_INSTANCES];
+ unsigned long pending = 0;
+ int xcc, r = 0, tmp;
- if (adev->gmc.flush_tlb_needs_extra_type_0)
- ndw += kiq->pmf->invalidate_tlbs_size;
+ /*
+ * A GPU reset should flush all TLBs anyway, so no need to do
+ * this while one is ongoing.
+ */
+ if (!down_read_trylock(&adev->reset_domain->sem))
+ return 0;
- spin_lock(&adev->gfx.kiq[inst].ring_lock);
- r = amdgpu_ring_alloc(ring, ndw);
- if (r) {
- spin_unlock(&adev->gfx.kiq[inst].ring_lock);
- goto error_unlock_reset;
- }
- if (adev->gmc.flush_tlb_needs_extra_type_2)
- kiq->pmf->kiq_invalidate_tlbs(ring, pasid, 2, all_hub);
+ for_each_inst(xcc, xcc_mask) {
+ struct amdgpu_ring *ring;
- if (flush_type == 2 && adev->gmc.flush_tlb_needs_extra_type_0)
- kiq->pmf->kiq_invalidate_tlbs(ring, pasid, 0, all_hub);
+ if (xcc >= AMDGPU_MAX_GC_INSTANCES)
+ break;
- kiq->pmf->kiq_invalidate_tlbs(ring, pasid, flush_type, all_hub);
- r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT);
- if (r) {
- amdgpu_ring_undo(ring);
- spin_unlock(&adev->gfx.kiq[inst].ring_lock);
- goto error_unlock_reset;
+ ring = &adev->gfx.kiq[xcc].ring;
+ if (!adev->gmc.flush_pasid_uses_kiq || !ring->sched.ready) {
+ amdgpu_gmc_flush_pasid_regs(adev, pasid, flush_type,
+ all_hub, xcc);
+ continue;
}
- amdgpu_ring_commit(ring);
- spin_unlock(&adev->gfx.kiq[inst].ring_lock);
-
- r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
-
- might_sleep();
- while (r < 1 && cnt++ < MAX_KIQ_REG_TRY &&
- !amdgpu_reset_pending(adev->reset_domain)) {
- msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
- r = amdgpu_fence_wait_polling(ring, seq,
MAX_KIQ_REG_WAIT);
+ tmp = amdgpu_gmc_flush_pasid_kiq_submit(adev, pasid, flush_type,
+ all_hub, xcc,
&seq[xcc]);
+ if (tmp) {
+ if (!r)
+ r = tmp;
+ continue;
}
- if (cnt > MAX_KIQ_REG_TRY) {
- dev_err(adev->dev, "timeout waiting for kiq fence\n");
- r = -ETIME;
- } else
- r = 0;
+ __set_bit(xcc, &pending);
+ }
+
+ for_each_set_bit(xcc, &pending, AMDGPU_MAX_GC_INSTANCES) {
+ tmp = amdgpu_gmc_flush_pasid_kiq_wait(adev, xcc, seq[xcc]);
+ if (tmp && !r)
+ r = tmp;
}
-error_unlock_reset:
up_read(&adev->reset_domain->sem);
return r;
}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
index cffd5ca3fc66..5ea237ce133b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
@@ -447,6 +447,9 @@ void amdgpu_gmc_flush_gpu_tlb(struct amdgpu_device *adev,
uint32_t vmid,
int amdgpu_gmc_flush_gpu_tlb_pasid(struct amdgpu_device *adev, uint16_t pasid,
uint32_t flush_type, bool all_hub,
uint32_t inst);
+int amdgpu_gmc_flush_gpu_tlb_pasid_xccs(struct amdgpu_device *adev, uint16_t
pasid,
+ uint32_t flush_type, bool all_hub,
+ uint32_t xcc_mask);
void amdgpu_gmc_fw_reg_write_reg_wait(struct amdgpu_device *adev,
uint32_t reg0, uint32_t reg1,
uint32_t ref, uint32_t mask,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index 47cde5459469..aae92ca87914 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -1724,7 +1724,6 @@ int amdgpu_vm_flush_compute_tlb(struct amdgpu_device
*adev,
{
uint64_t tlb_seq = amdgpu_vm_tlb_seq(vm);
bool all_hub = false;
- int xcc = 0, r = 0;
WARN_ON_ONCE(!vm->is_compute_context);
@@ -1740,13 +1739,8 @@ int amdgpu_vm_flush_compute_tlb(struct amdgpu_device *adev,
adev->family == AMDGPU_FAMILY_RV)
all_hub = true;
- for_each_inst(xcc, xcc_mask) {
- r = amdgpu_gmc_flush_gpu_tlb_pasid(adev, vm->pasid, flush_type,
- all_hub, xcc);
- if (r)
- break;
- }
- return r;
+ return amdgpu_gmc_flush_gpu_tlb_pasid_xccs(adev, vm->pasid, flush_type,
+ all_hub, xcc_mask);
}
/**
