Pre-GFX12 SPM has a hardware bug: when the ring fills, hardware keeps writing past the end instead of wrapping back to the start. Work around it by reserving a region past the ring end, detecting how much of it was written, and moving that data to where hardware should have put it.
Add spm_overflow_reserved to amdgpu_spm_mgr, 0x400 on pre-GFX12 and 0 on GFX12+, and subtract it from spm->ring_size before amdgpu_rlc_spm_acquire() so hardware is programmed with a shorter ring. The reserve is carved out of the existing allocation, so the buffer object size is unchanged. amdgpu_spm_preset() fills the reserve with AMDGPU_SPM_OVERFLOW_MAGIC, one sentinel per 0x20 block: acquire arms the whole region, each drain re-arms only the part it consumed. On the wrap branch the drain scans at the same stride for the first surviving sentinel, and that offset is the overflow size. The data is copied to the start of the ring, which is safe because overflow and wrap are mutually exclusive. A dma_wmb() after re-arming keeps the sentinels visible before the read pointer hands the space back to hardware. Counter data matching the magic on a 0x20 boundary ends the scan early and under-reports the overflow; this is accepted as unlikely. Signed-off-by: James Zhu <[email protected]> Reviewed-by: Bing Ma <[email protected]> --- drivers/gpu/drm/amd/amdgpu/amdgpu_spm.c | 71 +++++++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_spm.h | 4 ++ 2 files changed, 75 insertions(+) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_spm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_spm.c index d17fb2963981..532d1bc34f89 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_spm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_spm.c @@ -39,6 +39,9 @@ #define AMDGPU_SPM_RING_SIZE (4 * 1024 * 1024) #define AMDGPU_SPM_META_SIZE 0x20 +/* used to detect SPM overflow */ +#define AMDGPU_SPM_OVERFLOW_MAGIC 0xBEEFABCDDEADABCDULL + static int amdgpu_spm_acquire(struct amdgpu_spm_mgr *spm_mgr, struct drm_file *filp); static int amdgpu_spm_release(struct amdgpu_spm_mgr *spm_mgr, struct drm_file *filp); static void _amdgpu_spm_release(struct amdgpu_spm_mgr *spm_mgr, int inst, struct drm_file *filp); @@ -51,6 +54,34 @@ static bool amdgpu_spm_is_owner(struct amdgpu_spm_mgr *spm_mgr, struct drm_file return spm_mgr->file == filp; } +/* Re-arm overflow detection region with magic values. + * + * Called with size = spm_overflow_reserved at acquire time to arm the entire + * reserved region. Called with size = overflow_size on each drain: the + * sentinel scan defines overflow_size as how far HW wrote into the reserve, + * so [overflow_size, spm_overflow_reserved) still holds its acquire-time + * magic and only the consumed portion needs re-arming. + */ +static void amdgpu_spm_preset(struct amdgpu_spm_base *spm, u32 size) +{ + u64 *overflow_ptr, *overflow_end_ptr; + + overflow_ptr = (u64 *)((u8 *)spm->cpu_addr + + spm->ring_size + AMDGPU_SPM_META_SIZE); + overflow_end_ptr = overflow_ptr + (size >> 3); + /* Write one magic sentinel per 0x20-byte block. + * overflow_ptr is u64*, so += 4 advances 0x20 bytes (4 * 8 = 32). + * Matches the 0x20 detection stride in overflow scanning. + */ + for (; overflow_ptr < overflow_end_ptr; overflow_ptr += 4) + *overflow_ptr = AMDGPU_SPM_OVERFLOW_MAGIC; + + /* Sentinels must be globally visible before HW may write the reserve + * again, i.e. before the rdptr update or the SPM enable that follows. + */ + dma_wmb(); +} + static int amdgpu_spm_data_copy(struct amdgpu_spm_mgr *spm_mgr, struct amdgpu_spm_cntr *spm_cntr, u32 ring_rptr, u32 size_to_copy, int inst) @@ -107,6 +138,7 @@ static int amdgpu_spm_read_ring_buffer(struct amdgpu_spm_mgr *spm_mgr, struct amdgpu_spm_base *spm = &spm_cntr->spm[inst]; struct amdgpu_device *adev = prof_xcp_mgr->adev; unsigned long flags; + u32 overflow_size = 0; u32 size_to_copy; u32 ring_wptr; u32 ring_rptr; @@ -163,6 +195,34 @@ static int amdgpu_spm_read_ring_buffer(struct amdgpu_spm_mgr *spm_mgr, size_to_copy = ring_wptr - ring_rptr; ret = amdgpu_spm_data_copy(spm_mgr, spm_cntr, ring_rptr, size_to_copy, inst); } else { + u64 *ring_start, *ring_end; + + ring_start = (u64 *)((u8 *)spm->cpu_addr + AMDGPU_SPM_META_SIZE); + ring_end = ring_start + (spm->ring_size >> 3); + /* Detect overflow size by scanning for first magic sentinel. + * Heuristic limitation: if genuine counter data at a 0x20 boundary + * equals AMDGPU_SPM_OVERFLOW_MAGIC (0xBEEFABCDDEADABCD), we stop + * early and under-report overflow. Low probability, but acknowledged + * trade-off. + */ + for ( ; overflow_size < spm_mgr->spm_overflow_reserved; + overflow_size += AMDGPU_SPM_META_SIZE) { + u64 *overflow_ptr = ring_end + (overflow_size >> 3); + + if (READ_ONCE(*overflow_ptr) == AMDGPU_SPM_OVERFLOW_MAGIC) + break; + } + if (overflow_size) { + dev_dbg(adev->dev, "SPM ring buffer overflow size 0x%x\n", overflow_size); + /* HW BUG WORKAROUND: On pre-GFX12, when the ring fills, hardware + * incorrectly writes overflow data to [ring_size..ring_size+overflow_size) + * instead of wrapping to [0..overflow_size). Copy it to where it should + * have been. Overflow and wrap are mutually exclusive: if overflow occurs, + * hardware failed to wrap, so no valid data exists at [0..overflow_size). + */ + memcpy(ring_start, ring_end, overflow_size); + } + /* Copy tail segment [rptr..ring_size) */ size_to_copy = spm->ring_size - ring_rptr; ret = amdgpu_spm_data_copy(spm_mgr, spm_cntr, ring_rptr, size_to_copy, inst); @@ -189,6 +249,7 @@ static int amdgpu_spm_read_ring_buffer(struct amdgpu_spm_mgr *spm_mgr, } exit: + amdgpu_spm_preset(spm, overflow_size); /* Read rptr under spm_irq_lock to pair with the snapshot above. */ spin_lock_irqsave(&spm_mgr->spm_irq_lock, flags); ring_rptr = spm->ring_rptr; @@ -302,6 +363,13 @@ int amdgpu_spm_ioctl(struct drm_device *dev, struct drm_amdgpu_spm_args *args, static void amdgpu_spm_init_device(struct amdgpu_spm_mgr *spm_mgr) { + struct amdgpu_profiler_xcp_mgr *prof_xcp_mgr = to_prof_xcp_mgr(spm_mgr, spm_mgr); + struct amdgpu_device *adev = prof_xcp_mgr->adev; + + /* pre-gfx12 spm has a hardware bug to cause overflow */ + if (amdgpu_ip_version(adev, GC_HWIP, 0) < IP_VERSION(12, 0, 0)) + spm_mgr->spm_overflow_reserved = 0x400; + spm_mgr->spm_cntr = NULL; spin_lock_init(&spm_mgr->spm_irq_lock); /* spm_work lives for the device lifetime; initialize it once here so @@ -334,6 +402,8 @@ static int _amdgpu_spm_acquire(struct amdgpu_spm_mgr *spm_mgr, int inst, struct goto out; } + /* reserve space to fix spm overflow */ + spm->ring_size -= spm_mgr->spm_overflow_reserved; ret = amdgpu_rlc_spm_acquire(adev, inst, drm_priv_to_vm(filp), spm->gpu_addr, spm->ring_size); if (ret) @@ -344,6 +414,7 @@ static int _amdgpu_spm_acquire(struct amdgpu_spm_mgr *spm_mgr, int inst, struct * the wptr there); the ring itself starts at AMDGPU_SPM_META_SIZE. */ spm->ring_size -= AMDGPU_SPM_META_SIZE; + amdgpu_spm_preset(spm, spm_mgr->spm_overflow_reserved); goto out; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_spm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_spm.h index 760fc04e7a6e..552c5701e08c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_spm.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_spm.h @@ -77,6 +77,10 @@ struct amdgpu_spm_mgr { struct work_struct spm_work; /* Protects spm_cntr publication and the SPM state read in the IRQ path */ spinlock_t spm_irq_lock; + /* 0x400 for pre-GFX12 overflow workaround, 0 for GFX12+. + * Must be 0x20-aligned to match preset/scan stride. + */ + u32 spm_overflow_reserved; }; int amdgpu_spm_ioctl(struct drm_device *dev, struct drm_amdgpu_spm_args *args, -- 2.34.1
