AMD General
A new VRAM allocation needs to meet:
adev->kfd.vram_used + vram_needed <= vram_available
kfd.vram_used already includes BOs that are later pinned.
Pinning does not consume extra VRAM. Subtracting vram_pin_size without adding
back kfd.vram_pinned counts those bytes twice and causes fake-OOM while VRAM is
still free.
Add kfd.vram_pinned so reserve_mem_limit() matches get_available_memory().
kfd.vram_pinned is only updated in gpuvm_pin_bo(). dma-buf pin and vmap call
amdgpu_bo_pin() only, so the counter stays zero and those BOs are still charged
twice. Update the counter on first pin and last unpin in dma-buf pin/vmap and
gpuvm_pin_bo.
Signed-off-by: Zhen Han <[email protected]>
---
v3:
- update kfd.vram_pinned from dma-buf pin/vmap and gpuvm_pin_bo
on first pin / last unpin in amdgpu_dma_buf.c
v2:
- keep vram_pin_size; add kfd.vram_pinned in reserve_mem_limit (Mukul Joshi)
- leave get_available_memory unchanged
- drop former 2/2 pin/unpin changes in amdgpu_object.c (Christian König)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h | 2 +
.../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 38 +++++++++++++++----
drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c | 22 ++++++++++-
3 files changed, 53 insertions(+), 9 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
index 4541542a28df..9ebae5ffd2ed 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
@@ -378,6 +378,8 @@ void amdgpu_amdkfd_gpuvm_put_bo_ref(struct amdgpu_bo *bo);
* Return: ZERO if successful in pinning, Non-Zero in case of error.
*/
int amdgpu_amdkfd_gpuvm_pin_bo(struct amdgpu_bo *bo, u32 domain);
+void amdgpu_amdkfd_account_vram_pin(struct amdgpu_bo *bo); void
+amdgpu_amdkfd_account_vram_unpin(struct amdgpu_bo *bo);
/**
* amdgpu_amdkfd_gpuvm_unpin_bo() - Unpins BO using following criteria diff
--git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index 89958d0f7e59..191985dbc569 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -237,7 +237,8 @@ int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device
*adev,
if (adev && xcp_id >= 0 && (!adev->apu_prefer_gtt ||
adev->gmc.is_app_apu)) {
uint64_t vram_available =
vram_size - reserved_for_pt - reserved_for_ras -
- atomic64_read(&adev->vram_pin_size);
+ atomic64_read(&adev->vram_pin_size) +
+ atomic64_read(&adev->kfd.vram_pinned);
if (adev->kfd.vram_used[xcp_id] + vram_needed > vram_available)
{
ret = -ENOMEM;
goto release;
@@ -1584,6 +1585,28 @@ static int init_kfd_vm(struct amdgpu_vm *vm, void
**process_info,
return ret;
}
+void amdgpu_amdkfd_account_vram_pin(struct amdgpu_bo *bo) {
+ if (!bo->kfd_bo || !bo->tbo.resource)
+ return;
+ if (bo->tbo.resource->mem_type != TTM_PL_VRAM)
+ return;
+
+ atomic64_add(amdgpu_bo_size(bo),
+ &amdgpu_ttm_adev(bo->tbo.bdev)->kfd.vram_pinned);
+}
+
+void amdgpu_amdkfd_account_vram_unpin(struct amdgpu_bo *bo) {
+ if (!bo->kfd_bo || !bo->tbo.resource)
+ return;
+ if (bo->tbo.resource->mem_type != TTM_PL_VRAM)
+ return;
+
+ atomic64_sub(amdgpu_bo_size(bo),
+ &amdgpu_ttm_adev(bo->tbo.bdev)->kfd.vram_pinned);
+}
+
/**
* amdgpu_amdkfd_gpuvm_pin_bo() - Pins a BO using following criteria
* @bo: Handle of buffer object being pinned @@ -1598,11 +1621,14 @@ static
int init_kfd_vm(struct amdgpu_vm *vm, void **process_info, int
amdgpu_amdkfd_gpuvm_pin_bo(struct amdgpu_bo *bo, u32 domain) {
int ret = 0;
+ bool first_pin;
ret = amdgpu_bo_reserve(bo, false);
if (unlikely(ret))
return ret;
+ first_pin = !bo->tbo.pin_count;
+
if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS) {
/*
* If bo is not contiguous on VRAM, move to system memory first
to ensure @@ -1626,9 +1652,8 @@ int amdgpu_amdkfd_gpuvm_pin_bo(struct amdgpu_bo
*bo, u32 domain)
amdgpu_bo_sync_wait(bo, AMDGPU_FENCE_OWNER_KFD, false);
- if (!ret && bo->tbo.resource->mem_type == TTM_PL_VRAM)
- atomic64_add(amdgpu_bo_size(bo),
- &amdgpu_ttm_adev(bo->tbo.bdev)->kfd.vram_pinned);
+ if (!ret && first_pin)
+ amdgpu_amdkfd_account_vram_pin(bo);
out:
amdgpu_bo_unreserve(bo);
@@ -1653,9 +1678,8 @@ void amdgpu_amdkfd_gpuvm_unpin_bo(struct amdgpu_bo *bo)
amdgpu_bo_unpin(bo);
- if (bo->tbo.resource->mem_type == TTM_PL_VRAM)
- atomic64_sub(amdgpu_bo_size(bo),
- &amdgpu_ttm_adev(bo->tbo.bdev)->kfd.vram_pinned);
+ if (!bo->tbo.pin_count)
+ amdgpu_amdkfd_account_vram_unpin(bo);
amdgpu_bo_unreserve(bo);
}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
index 79828c6f694b..e2674540761d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
@@ -303,6 +303,8 @@ static int amdgpu_dma_buf_pin(struct dma_buf_attachment
*attach)
struct dma_buf *dmabuf = attach->dmabuf;
struct amdgpu_bo *bo = gem_to_amdgpu_bo(dmabuf->priv);
u32 domains = bo->allowed_domains;
+ bool first_pin;
+ int r;
dma_resv_assert_held(dmabuf->resv);
@@ -328,7 +330,12 @@ static int amdgpu_dma_buf_pin(struct dma_buf_attachment
*attach)
if (WARN_ON(!domains))
return -EINVAL;
- return amdgpu_bo_pin(bo, domains);
+ first_pin = !bo->tbo.pin_count;
+ r = amdgpu_bo_pin(bo, domains);
+ if (!r && first_pin)
+ amdgpu_amdkfd_account_vram_pin(bo);
+
+ return r;
}
/**
@@ -344,6 +351,8 @@ static void amdgpu_dma_buf_unpin(struct dma_buf_attachment
*attach)
struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);
amdgpu_bo_unpin(bo);
+ if (!bo->tbo.pin_count)
+ amdgpu_amdkfd_account_vram_unpin(bo);
}
#endif
@@ -516,6 +525,7 @@ static int amdgpu_dma_buf_vmap(struct dma_buf *dma_buf,
struct iosys_map *map) {
struct drm_gem_object *obj = dma_buf->priv;
struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);
+ bool first_pin;
int ret;
/*
@@ -523,12 +533,18 @@ static int amdgpu_dma_buf_vmap(struct dma_buf *dma_buf,
struct iosys_map *map)
* domain is not that important as long as it's mapable. Using
* GTT and VRAM should be compatible with most use cases.
*/
+ first_pin = !bo->tbo.pin_count;
ret = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT | AMDGPU_GEM_DOMAIN_VRAM);
if (ret)
return ret;
+ if (first_pin)
+ amdgpu_amdkfd_account_vram_pin(bo);
ret = drm_gem_dmabuf_vmap(dma_buf, map);
- if (ret)
+ if (ret) {
amdgpu_bo_unpin(bo);
+ if (!bo->tbo.pin_count)
+ amdgpu_amdkfd_account_vram_unpin(bo);
+ }
return ret;
}
@@ -540,6 +556,8 @@ static void amdgpu_dma_buf_vunmap(struct dma_buf *dma_buf,
struct iosys_map *map
drm_gem_dmabuf_vunmap(dma_buf, map);
amdgpu_bo_unpin(bo);
+ if (!bo->tbo.pin_count)
+ amdgpu_amdkfd_account_vram_unpin(bo);
}
#endif
--
2.43.7
