From: Honglei Huang <[email protected]> Add invalidate_ranges callback and checkpoint timestamp: - amdgpu_svm_capture_checkpoint_ts: capture interrupt handler write pointer timestamp for stale retry fault filtering - amdgpu_svm_range_invalidate: invalidate_ranges callback dispatching per range notifier begin/end with TLB flush batching, checkpoint timestamp capture on MMU_NOTIFY_UNMAP events
Signed-off-by: Honglei Huang <[email protected]> --- drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c | 71 +++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.h | 6 ++ 2 files changed, 77 insertions(+) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c index 9e348390795a6..ed057545d0964 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c @@ -728,3 +728,74 @@ void amdgpu_svm_range_unqueue(struct amdgpu_svm *svm, if (put) drm_gpusvm_range_put(&range->base); } + +/** + * amdgpu_svm_capture_checkpoint_ts() - Record the IH ring write pointer time + * @svm: The SVM context. + * + * Capture the timestamp of the most recent interrupt handler write pointer + * into @svm->checkpoint_ts. Retry faults whose timestamp predates this + * checkpoint are stale and are dropped by the fault handler. + */ +void amdgpu_svm_capture_checkpoint_ts(struct amdgpu_svm *svm) +{ + struct amdgpu_device *adev = svm->adev; + struct amdgpu_ih_ring *ih; + uint32_t checkpoint_wptr; + + if (!adev->irq.retry_cam_enabled && adev->irq.ih1.ring_size) { + ih = &adev->irq.ih1; + checkpoint_wptr = amdgpu_ih_get_wptr(adev, ih); + if (ih->rptr != checkpoint_wptr) { + WRITE_ONCE(svm->checkpoint_ts, + amdgpu_ih_decode_iv_ts(adev, ih, + checkpoint_wptr, -1)); + return; + } + } + + ih = &adev->irq.ih_soft; + checkpoint_wptr = amdgpu_ih_get_wptr(adev, ih); + if (ih->rptr != checkpoint_wptr) + WRITE_ONCE(svm->checkpoint_ts, + amdgpu_ih_decode_iv_ts(adev, ih, + checkpoint_wptr, -1)); +} + +/** + * amdgpu_svm_range_invalidate() - MMU notifier invalidate_ranges callback + * @svm: The SVM context. + * @notifier: The GPU SVM notifier covering the invalidated ranges. + * @mmu_range: The MMU notifier range describing the event. + * @first: First GPU SVM range in the affected interval. + * @adj_start: Adjusted start address of the affected interval. + * @adj_end: Adjusted end address of the affected interval. + * + * Dispatch the two phase notifier handling across every range in + * [@adj_start, @adj_end): run the begin phase for all ranges, then + * the end phase. On an unmap event, also capture a checkpoint timestamp + * so stale retry faults can be filtered. + */ +void amdgpu_svm_range_invalidate(struct amdgpu_svm *svm, + struct drm_gpusvm_notifier *notifier, + const struct mmu_notifier_range *mmu_range, + struct drm_gpusvm_range *first, + uint64_t adj_start, uint64_t adj_end) +{ + struct drm_gpusvm_range *r; + bool needs_flush = false; + + if (mmu_range->event == MMU_NOTIFY_UNMAP) + amdgpu_svm_capture_checkpoint_ts(svm); + + r = first; + drm_gpusvm_for_each_range(r, notifier, adj_start, adj_end) + needs_flush |= amdgpu_svm_range_notifier_event_begin(svm, r, + mmu_range); + if (needs_flush) + amdgpu_svm_flush_tlb(svm); + + r = first; + drm_gpusvm_for_each_range(r, notifier, adj_start, adj_end) + amdgpu_svm_range_notifier_event_end(svm, r, mmu_range); +} diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.h index f423be277a85a..5e49d774711ce 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.h @@ -175,6 +175,12 @@ amdgpu_svm_range_find_or_insert(struct amdgpu_svm *svm, unsigned long addr, int amdgpu_svm_range_get_pages(struct amdgpu_svm *svm, struct drm_gpusvm_range *range, struct drm_gpusvm_ctx *ctx); +void amdgpu_svm_capture_checkpoint_ts(struct amdgpu_svm *svm); +void amdgpu_svm_range_invalidate(struct amdgpu_svm *svm, + struct drm_gpusvm_notifier *notifier, + const struct mmu_notifier_range *mmu_range, + struct drm_gpusvm_range *first, + uint64_t adj_start, uint64_t adj_end); bool amdgpu_svm_range_notifier_event_begin(struct amdgpu_svm *svm, struct drm_gpusvm_range *range, const struct mmu_notifier_range *mmu_range); -- 2.53.0
