From: Mukul Joshi <[email protected]> Add implementation for freeing up exporter GPU's UALink handles and the corresponding xarray entries. Before freeing, the exporter GPU would send the NPA-REVOKE message to all the importers and wait for the NPA-RELEASE messages to come back before freeing the xarray entry.
Signed-off-by: Mukul Joshi <[email protected]> Reviewed-by: Felix Kuehling <[email protected]> Signed-off-by: Alex Deucher <[email protected]> --- drivers/gpu/drm/amd/amdgpu/amdgpu_ualink.c | 343 +++++++++++++++++++++ 1 file changed, 343 insertions(+) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ualink.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ualink.c index 82a362e44a3fe..d87b66d1d51d3 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ualink.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ualink.c @@ -1272,6 +1272,13 @@ static void amdgpu_generate_ualink_handle(struct amdgpu_device *adev, static void amdgpu_ualink_cleanup_exp_xa_node(struct kref *ref) { + struct amdgpu_ualink_exp_xa_node *exp_xa_node; + struct amdgpu_device *adev; + + exp_xa_node = container_of(ref, struct amdgpu_ualink_exp_xa_node, + refcount); + adev = amdgpu_ttm_adev(exp_xa_node->bo->tbo.bdev); + queue_work(adev->ualink.npa_wq, &exp_xa_node->cleanup_work); } static void amdgpu_ualink_cleanup_imp_xa_node(struct kref *ref) @@ -1751,8 +1758,344 @@ static int amdgpu_ualink_setup_connection(struct amdgpu_device *adev, return r; } +/* Set PTE.X = 1 for all importer entries to retry RPCs. */ +static void amdgpu_ualink_force_retry_rpcs(struct amdgpu_device *adev, + struct amdgpu_ualink_exp_xa_node *exp_xa_node) +{ + u32 addr_mode = adev->ualink.info->vpod.addr_mode; + struct amdgpu_ualink_importer_entry *imp_entry; + struct amdgpu_vm *vm = &adev->ualink.npa_vm; + u64 pte_flags, npa_addr, size; + struct dma_fence *fence = NULL; + struct amdgpu_bo *bo; + struct drm_exec exec; + u32 remote_acc_id; + int r; + + bo = exp_xa_node->bo; + size = amdgpu_bo_ngpu_pages(bo); + + amdgpu_ualink_reserve_npa_vm_and_bos(adev, &bo, 1, &exec, false); + + for_each_set_bit(remote_acc_id, exp_xa_node->importers_bitmap, + AMDGPU_UALINK_ACCEL_MAX) { + if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS) + imp_entry = &exp_xa_node->importer_entries[0]; + else + imp_entry = &exp_xa_node->importer_entries[remote_acc_id]; + npa_addr = imp_entry->npa_addr; + + /* If the connection state changed while we are freeing + * the BO, then ignore this importer. We will unmap this + * address eventually in amdgpu_ualink_unmap_all_npa_addr() + * function. Also, we will clear the corresponding bit in + * the importer_bitmap in the same function. + */ + if (!amdgpu_ualink_check_conn_ready(adev, remote_acc_id, + imp_entry->generation_count)) + continue; + + /* Set PTE.X = 1 */ + pte_flags = amdgpu_ualink_get_export_pte_flags(adev, bo, + AMDGPU_VM_PAGE_EXECUTABLE); + dev_dbg(adev->dev, + "RETRY-RPC: setting PTE.X=1 for NPA:%llx remote:%u pte:0x%llx\n", + npa_addr, remote_acc_id, pte_flags); + + r = amdgpu_vm_update_range(adev, vm, false, false, true, + false, NULL, npa_addr, npa_addr + size - 1, + pte_flags, 0, adev->vm_manager.vram_base_offset, + bo->tbo.resource, NULL, &vm->last_update); + + if (r) + dev_warn(adev->dev, + "RETRY-RPC: PTE.X update failed for NPA:%llx remote:%u, r: %d\n", + npa_addr, remote_acc_id, r); + + if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS) + break; + } + + r = amdgpu_vm_update_pdes(adev, vm, false); + if (r) { + dev_err(adev->dev, + "Failed %d to update page directories during force retry rpcs\n", + r); + amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec); + return; + } + + fence = dma_fence_get(vm->last_update); + if (fence) { + r = dma_fence_wait(fence, false); + dma_fence_put(fence); + fence = NULL; + if (r) + dev_dbg(adev->dev, "RETRY-RPC: dma fence wait failed err:%d\n", r); + } + + amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec); + + amdgpu_ualink_flush_tlb(adev, TLB_FLUSH_HEAVYWEIGHT); +} + +/* Unmap all NPA addresses associated with a BO (UALink handle). This function is used + * only in Source Identification mode. + */ +static void amdgpu_ualink_unmap_all_npa_addr(struct amdgpu_device *adev, + struct amdgpu_ualink_exp_xa_node *exp_xa_node) +{ + u32 addr_mode = adev->ualink.info->vpod.addr_mode; + struct amdgpu_ualink_importer_entry *imp_entry; + u64 pte_value = adev->gmc.noretry_flags; + struct dma_fence *fence = NULL; + struct drm_exec exec; + u64 npa_addr, size; + u32 remote_acc_id; + int r; + + size = amdgpu_bo_ngpu_pages(exp_xa_node->bo); + + amdgpu_ualink_reserve_npa_vm_and_bos(adev, &exp_xa_node->bo, 1, + &exec, false); + + for_each_set_bit(remote_acc_id, exp_xa_node->importers_bitmap, + AMDGPU_UALINK_ACCEL_MAX) { + if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS) + imp_entry = &exp_xa_node->importer_entries[0]; + else + imp_entry = &exp_xa_node->importer_entries[remote_acc_id]; + npa_addr = imp_entry->npa_addr; + + if (!amdgpu_ualink_check_conn_ready(adev, remote_acc_id, + imp_entry->generation_count)) { + clear_bit(remote_acc_id, + exp_xa_node->importers_bitmap); + continue; + } + + dev_dbg(adev->dev, + "UNMAP-NPA: Unmapping NPA:%llx, handle:%llx:%llx remote:%u pte:0x%llx\n", + npa_addr, exp_xa_node->handle.handle_hi, exp_xa_node->handle.handle_lo, + remote_acc_id, pte_value); + + r = amdgpu_vm_update_range(adev, &adev->ualink.npa_vm, false, + false, true, false, NULL, npa_addr, + npa_addr + size - 1, pte_value, 0, + 0, NULL, NULL, &fence); + + if (r) + dev_err(adev->dev, + "UNMAP-NPA: Unmap failed NPA:%llx, handle:%llx:%llx remote:%u\n", + npa_addr, exp_xa_node->handle.handle_hi, + exp_xa_node->handle.handle_lo, remote_acc_id); + + if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS) + break; + } + + r = amdgpu_vm_update_pdes(adev, &adev->ualink.npa_vm, false); + if (r) { + dev_err(adev->dev, + "Failed %d to update page directories during all NPA addresses unmapping\n", + r); + amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec); + return; + } + + if (fence) { + r = dma_fence_wait(fence, false); + dma_fence_put(fence); + fence = NULL; + if (r) + dev_err(adev->dev, + "UNMAP-NPA: dma fence wait failed\n"); + } + + amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec); + + amdgpu_ualink_flush_tlb(adev, TLB_FLUSH_HEAVYWEIGHT); +} + +static void amdgpu_ualink_free_all_npa_va(struct amdgpu_device *adev, + struct amdgpu_ualink_exp_xa_node *exp_xa_node, + unsigned long *importers_bitmap) +{ + u32 addr_mode = adev->ualink.info->vpod.addr_mode; + struct drm_mm_node *mm_node; + u32 remote_acc_id; + + for_each_set_bit(remote_acc_id, importers_bitmap, AMDGPU_UALINK_ACCEL_MAX) { + if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS) + mm_node = exp_xa_node->importer_entries[0].mm_node; + else + mm_node = exp_xa_node->importer_entries[remote_acc_id].mm_node; + + if (!mm_node) { + if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS) + break; + continue; + } + + dev_dbg(adev->dev, + "FREE-NPA: freeing NPA address:%llx, handle:%llx:%llx remote:%u\n", + mm_node->start, exp_xa_node->handle.handle_hi, + exp_xa_node->handle.handle_lo, remote_acc_id); + + amdgpu_ualink_npa_free_va(adev, mm_node); + kfree(mm_node); + + if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS) + break; + } +} + static void amdgpu_ualink_exp_cleanup_worker(struct work_struct *work) { + DECLARE_BITMAP(orig_importers_bitmap, AMDGPU_UALINK_ACCEL_MAX); + struct amdgpu_ualink_importer_entry *imp_entry; + struct amdgpu_ualink_exp_xa_node *exp_xa_node; + struct amdgpu_ualink_handle handle; + struct amdgpu_device *adev; + struct amdgpu_bo *bo; + u32 remote_acc_id; + int r; + + exp_xa_node = container_of(work, struct amdgpu_ualink_exp_xa_node, + cleanup_work); + bo = exp_xa_node->bo; + adev = amdgpu_ttm_adev(bo->tbo.bdev); + handle = exp_xa_node->handle; + + /* Revoking access to an exported memory follows the steps: + * 1. Set PTE.X = 1 to retry for RPCs. + * 2. Send Remote TLB Shootdowns to all importers. + * 3. Unmap the NPA address from NPA VM. + * 4. Drop the ref count for the BO. + * 5. Send NPA_REVOKE to all importers. + * 6. Wait for NPA_RELEASE from all importers. + * 7. Free the NPA address once all responses are received. + */ + /* If there are no importers for this BO/handle */ + if (bitmap_empty(exp_xa_node->importers_bitmap, + AMDGPU_UALINK_ACCEL_MAX)) { + /* Drop the BO reference so it can be freed. */ + amdgpu_bo_unref(&bo); + exp_xa_node->bo = NULL; + goto free_node; + } + + dev_dbg(adev->dev, + "EXP-CLEANUP: handle:%llx:%llx importers bitmap: %*pbl\n", + handle.handle_hi, handle.handle_lo, + AMDGPU_UALINK_ACCEL_MAX, exp_xa_node->importers_bitmap); + + bitmap_copy(orig_importers_bitmap, exp_xa_node->importers_bitmap, + AMDGPU_UALINK_ACCEL_MAX); + + /* Set PTE.X = 1 for NPA addresses from all importers*/ + amdgpu_ualink_force_retry_rpcs(adev, exp_xa_node); + + /* Send TLB-shootdown to all importer GPUs */ + for_each_set_bit(remote_acc_id, exp_xa_node->importers_bitmap, + AMDGPU_UALINK_ACCEL_MAX) { + dev_dbg(adev->dev, + "EXP-CLEANUP: Sending TLB-shootdown to remote:%u\n", + remote_acc_id); + r = amdgpu_ualink_send_tlb_shootdown(adev, remote_acc_id); + if (r) + dev_err(adev->dev, + "EXP-CLEANUP: TLB shootdown send failed to remote:%u\n", + remote_acc_id); + } + + /* Unmap all NPA addresses for this BO from NPA VM */ + amdgpu_ualink_unmap_all_npa_addr(adev, exp_xa_node); + + dev_dbg(adev->dev, "EXP-CLEANUP: handle:%llx:%llx Unpin BO, pin_count:%u, importers:%u\n", + handle.handle_hi, handle.handle_lo, bo->tbo.pin_count, + bitmap_weight(orig_importers_bitmap, AMDGPU_UALINK_ACCEL_MAX)); + WARN_ON(bo->tbo.pin_count < bitmap_weight(orig_importers_bitmap, + AMDGPU_UALINK_ACCEL_MAX)); + + /* Unpin the BO */ + if (likely(!amdgpu_bo_reserve(bo, true))) { + bo->ualink_handle_lo = 0ULL; + for_each_set_bit(remote_acc_id, orig_importers_bitmap, + AMDGPU_UALINK_ACCEL_MAX) + amdgpu_bo_unpin(bo); + amdgpu_bo_unreserve(bo); + } else { + dev_warn(adev->dev, + "EXP-CLEANUP: BO reserve to unpin failed for handle:%llx:%llx\n", + handle.handle_hi, handle.handle_lo); + } + + /* Free the DMABuf */ + dma_buf_put(exp_xa_node->dmabuf); + /* Drop the reference to the BO so it can be freed. */ + amdgpu_bo_unref(&bo); + exp_xa_node->bo = NULL; + + /* Send NPA-REVOKE to all importers which have imported this memory */ + for_each_set_bit(remote_acc_id, exp_xa_node->importers_bitmap, + AMDGPU_UALINK_ACCEL_MAX) { + imp_entry = &exp_xa_node->importer_entries[remote_acc_id]; + if (!amdgpu_ualink_check_conn_ready(adev, remote_acc_id, + imp_entry->generation_count)) { + clear_bit(remote_acc_id, + exp_xa_node->importers_bitmap); + continue; + } + + dev_dbg(adev->dev, + "EXP-CLEANUP: Sending NPA-REVOKE to remote:%u\n", + remote_acc_id); + set_bit(remote_acc_id, exp_xa_node->npa_release_bitmap); + r = amdgpu_ualink_send_npa_revoke_msg(adev, remote_acc_id, handle); + if (r) { + dev_err(adev->dev, + "EXP-CLEANUP: NPA-REVOKE send failed to remote:%u\n", + remote_acc_id); + clear_bit(remote_acc_id, exp_xa_node->npa_release_bitmap); + } + } + + if (!bitmap_empty(exp_xa_node->importers_bitmap, + AMDGPU_UALINK_ACCEL_MAX)) { + dev_dbg(adev->dev, + "EXP-CLEANUP: handle:%llx:%llx NPA-RELEASE bitmap: %*pbl\n", + handle.handle_hi, handle.handle_lo, + AMDGPU_UALINK_ACCEL_MAX, exp_xa_node->npa_release_bitmap); + + /* Wait for the NPA_RELEASE to come back from all importers */ + r = wait_for_completion_timeout(&exp_xa_node->npa_done, + msecs_to_jiffies(AMDGPU_UALINK_RESP_TIMEOUT)); + + if (r == 0) + dev_warn(adev->dev, + "EXP-CLEANUP: NPA-RELEASE timeout for handle:%llx:%llx\n", + handle.handle_hi, handle.handle_lo); + } + + /* Free the NPA addresses given to all the importers */ + amdgpu_ualink_free_all_npa_va(adev, exp_xa_node, + orig_importers_bitmap); + + /* Warn about all importers that didn't respond back with + * NPA-RELEASE message. This will trigger connection timeout + * handling which is added later. + */ + for_each_set_bit(remote_acc_id, exp_xa_node->npa_release_bitmap, + AMDGPU_UALINK_ACCEL_MAX) + dev_warn(adev->dev, + "EXP-CLEANUP: handle:%llx:%llx NPA-RELEASE timeout from remote:%u\n", + handle.handle_hi, handle.handle_lo, remote_acc_id); + +free_node: + xa_erase(&adev->ualink.handle_invalid_xa, handle.handle_lo); + mutex_destroy(&exp_xa_node->node_lock); + kfree(exp_xa_node); } /* This function is a copy of amdgpu_dma_buf_move_notify() function. -- 2.55.0
