AMD General
reserve_mem_limit() computes available VRAM as
vram_size - reserved - vram_pin_size - kfd.vram_used
However, kfd.vram_used already includes BOs that are later pinned.
Pinning does not consume extra VRAM. Subtracting vram_pin_size counts those
bytes twice and causes fake-OOM while VRAM is still free.
New design reuses get_available_memory() for getting available VRAM in
reserve_mem_limit(). Move ALIGN_DOWN to the ioctl so the userspace ABI is
unchanged.
kfd.vram_pinned is only updated in gpuvm_pin_bo(). dma-buf pin calls
amdgpu_bo_pin() only, so the counter stays zero.
Update it from amdgpu_bo_pin() through a KFD helper, on first pin and last
unpin, matching vram_pin_size.
Signed-off-by: Zhen Han <[email protected]>
---
v4:
- use get_available_memory() in reserve_mem_limit()
- move ALIGN_DOWN from get_available_memory() to the ioctl
- drop v3 extra call in amdgpu_dma_buf.c
- update kfd.vram_pinned from amdgpu_bo_pin() via a KFD helper
v3:
- update kfd.vram_pinned from dma-buf pin/vmap and gpuvm_pin_bo
on first pin / last unpin in amdgpu_dma_buf.c
v2:
- keep vram_pin_size; add kfd.vram_pinned in reserve_mem_limit (Mukul Joshi)
- leave get_available_memory unchanged
- drop former 2/2 pin/unpin changes in amdgpu_object.c
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h | 18 +++++
.../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 81 +++++++++++--------
drivers/gpu/drm/amd/amdgpu/amdgpu_object.c | 2 +
drivers/gpu/drm/amd/amdkfd/kfd_chardev.c | 12 ++-
4 files changed, 76 insertions(+), 37 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
index 4541542a28df..ff6aab8a8657 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
@@ -42,6 +42,12 @@
extern uint64_t amdgpu_amdkfd_total_mem_size;
+/*
+ * Align VRAM availability to 2MB to avoid fragmentation caused by 4K
+ * allocations in the tail 2MB BO chunk.
+ */
+#define VRAM_AVAILABLITY_ALIGN (1 << 21)
+
enum TLB_FLUSH_TYPE {
TLB_FLUSH_LEGACY = 0,
TLB_FLUSH_LIGHTWEIGHT,
@@ -456,6 +462,8 @@ void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device
*adev,
* Allows KFD to release its resources associated with the GEM object.
*/
void amdgpu_amdkfd_release_notify(struct amdgpu_bo *bo);
+void amdgpu_amdkfd_account_vram_pin(struct amdgpu_bo *bo); void
+amdgpu_amdkfd_account_vram_unpin(struct amdgpu_bo *bo);
void amdgpu_amdkfd_reserve_system_mem(uint64_t size); #else static inline @@
-473,6 +481,16 @@ static inline void amdgpu_amdkfd_release_notify(struct
amdgpu_bo *bo) { }
+
+static inline
+void amdgpu_amdkfd_account_vram_pin(struct amdgpu_bo *bo) { }
+
+static inline
+void amdgpu_amdkfd_account_vram_unpin(struct amdgpu_bo *bo) { }
#endif
void amdgpu_amdkfd_rlc_spm_cntl(struct amdgpu_device *adev, int xcc_id, bool
cntl);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index 89958d0f7e59..eb894a6fe5ef 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -48,12 +48,6 @@
#define AMDGPU_USERPTR_RESTORE_DELAY_MS 1
#define AMDGPU_RESERVE_MEM_LIMIT (3UL << 29)
-/*
- * Align VRAM availability to 2MB to avoid fragmentation caused by 4K
allocations in the tail 2MB
- * BO chunk
- */
-#define VRAM_AVAILABLITY_ALIGN (1 << 21)
-
/* Impose limit on how much memory KFD can use */
static struct {
uint64_t max_system_mem_limit;
@@ -75,6 +69,8 @@ static const char * const domain_bit_to_string[] = {
#define domain_string(domain) domain_bit_to_string[ffs(domain)-1]
static void amdgpu_amdkfd_restore_userptr_worker(struct work_struct *work);
+static size_t __get_available_memory(struct amdgpu_device *adev,
+ uint8_t xcp_id);
static bool kfd_mem_is_attached(struct amdgpu_vm *avm,
struct kgd_mem *mem)
@@ -171,13 +167,8 @@ void amdgpu_amdkfd_reserve_system_mem(uint64_t size)
int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev,
uint64_t size, u32 alloc_flag, int8_t xcp_id)
{
- uint64_t reserved_for_pt =
- ESTIMATE_PT_SIZE(amdgpu_amdkfd_total_mem_size);
- struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
- uint64_t reserved_for_ras = (con ? con->reserved_pages_in_bytes : 0);
size_t system_mem_needed, ttm_mem_needed, vram_needed;
int ret = 0;
- uint64_t vram_size = 0;
system_mem_needed = 0;
ttm_mem_needed = 0;
@@ -198,7 +189,6 @@ int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device
*adev,
if (WARN_ONCE(xcp_id < 0, "invalid XCP ID %d", xcp_id))
return -EINVAL;
- vram_size = KFD_XCP_MEMORY_SIZE(adev, xcp_id);
if (adev->apu_prefer_gtt) {
system_mem_needed = size;
ttm_mem_needed = size;
@@ -234,14 +224,10 @@ int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device
*adev,
* VRAM check since ttm_mem_limit check already cover this allocation
*/
- if (adev && xcp_id >= 0 && (!adev->apu_prefer_gtt ||
adev->gmc.is_app_apu)) {
- uint64_t vram_available =
- vram_size - reserved_for_pt - reserved_for_ras -
- atomic64_read(&adev->vram_pin_size);
- if (adev->kfd.vram_used[xcp_id] + vram_needed > vram_available)
{
- ret = -ENOMEM;
- goto release;
- }
+ if (adev && xcp_id >= 0 && (!adev->apu_prefer_gtt ||
adev->gmc.is_app_apu) &&
+ vram_needed > __get_available_memory(adev, xcp_id)) {
+ ret = -ENOMEM;
+ goto release;
}
/* Update memory accounting by decreasing available system
@@ -320,6 +306,28 @@ void amdgpu_amdkfd_release_notify(struct amdgpu_bo *bo)
kfree(bo->kfd_bo);
}
+void amdgpu_amdkfd_account_vram_pin(struct amdgpu_bo *bo)
+{
+ if (!bo->kfd_bo || !bo->tbo.resource)
+ return;
+ if (bo->tbo.resource->mem_type != TTM_PL_VRAM)
+ return;
+
+ atomic64_add(amdgpu_bo_size(bo),
+ &amdgpu_ttm_adev(bo->tbo.bdev)->kfd.vram_pinned);
+}
+
+void amdgpu_amdkfd_account_vram_unpin(struct amdgpu_bo *bo)
+{
+ if (!bo->kfd_bo || !bo->tbo.resource)
+ return;
+ if (bo->tbo.resource->mem_type != TTM_PL_VRAM)
+ return;
+
+ atomic64_sub(amdgpu_bo_size(bo),
+ &amdgpu_ttm_adev(bo->tbo.bdev)->kfd.vram_pinned);
+}
+
/**
* create_dmamap_sg_bo() - Creates a amdgpu_bo object to reflect information
* about USERPTR or DOOREBELL or MMIO BO.
@@ -1626,10 +1634,6 @@ int amdgpu_amdkfd_gpuvm_pin_bo(struct amdgpu_bo *bo, u32
domain)
amdgpu_bo_sync_wait(bo, AMDGPU_FENCE_OWNER_KFD, false);
- if (!ret && bo->tbo.resource->mem_type == TTM_PL_VRAM)
- atomic64_add(amdgpu_bo_size(bo),
- &amdgpu_ttm_adev(bo->tbo.bdev)->kfd.vram_pinned);
-
out:
amdgpu_bo_unreserve(bo);
return ret;
@@ -1653,10 +1657,6 @@ void amdgpu_amdkfd_gpuvm_unpin_bo(struct amdgpu_bo *bo)
amdgpu_bo_unpin(bo);
- if (bo->tbo.resource->mem_type == TTM_PL_VRAM)
- atomic64_sub(amdgpu_bo_size(bo),
- &amdgpu_ttm_adev(bo->tbo.bdev)->kfd.vram_pinned);
-
amdgpu_bo_unreserve(bo);
}
@@ -1760,8 +1760,9 @@ int amdgpu_amdkfd_criu_resume(void *p)
return ret;
}
-size_t amdgpu_amdkfd_get_available_memory(struct amdgpu_device *adev,
- uint8_t xcp_id)
+/* Caller must hold kfd_mem_limit.mem_limit_lock. */
+static size_t __get_available_memory(struct amdgpu_device *adev,
+ uint8_t xcp_id)
{
uint64_t reserved_for_pt =
ESTIMATE_PT_SIZE(amdgpu_amdkfd_total_mem_size);
@@ -1770,7 +1771,8 @@ size_t amdgpu_amdkfd_get_available_memory(struct
amdgpu_device *adev,
ssize_t available;
uint64_t vram_available, system_mem_available, ttm_mem_available;
- spin_lock(&kfd_mem_limit.mem_limit_lock);
+ lockdep_assert_held(&kfd_mem_limit.mem_limit_lock);
+
if (adev->apu_prefer_gtt && !adev->gmc.is_app_apu)
vram_available = KFD_XCP_MEMORY_SIZE(adev, xcp_id)
- adev->kfd.vram_used_aligned[xcp_id];
@@ -1793,19 +1795,28 @@ size_t amdgpu_amdkfd_get_available_memory(struct
amdgpu_device *adev,
available = min3(system_mem_available, ttm_mem_available,
vram_available);
- available = ALIGN_DOWN(available, PAGE_SIZE);
} else {
- available = ALIGN_DOWN(vram_available, VRAM_AVAILABLITY_ALIGN);
+ available = vram_available;
}
- spin_unlock(&kfd_mem_limit.mem_limit_lock);
-
if (available < 0)
available = 0;
return available;
}
+size_t amdgpu_amdkfd_get_available_memory(struct amdgpu_device *adev,
+ uint8_t xcp_id)
+{
+ size_t available;
+
+ spin_lock(&kfd_mem_limit.mem_limit_lock);
+ available = __get_available_memory(adev, xcp_id);
+ spin_unlock(&kfd_mem_limit.mem_limit_lock);
+
+ return available;
+}
+
int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
struct amdgpu_device *adev, uint64_t va, uint64_t size,
void *drm_priv, struct kgd_mem **mem,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
index b66a0eeda788..233e94db930e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
@@ -1004,6 +1004,7 @@ int amdgpu_bo_pin(struct amdgpu_bo *bo, u32 domain)
atomic64_add(amdgpu_bo_size(bo), &adev->vram_pin_size);
atomic64_add(amdgpu_vram_mgr_bo_visible_size(bo),
&adev->visible_pin_size);
+ amdgpu_amdkfd_account_vram_pin(bo);
} else if (bo->tbo.resource->mem_type == TTM_PL_TT) {
atomic64_add(amdgpu_bo_size(bo), &adev->gart_pin_size);
}
@@ -1039,6 +1040,7 @@ void amdgpu_bo_unpin(struct amdgpu_bo *bo)
atomic64_sub(amdgpu_bo_size(bo), &adev->vram_pin_size);
atomic64_sub(amdgpu_vram_mgr_bo_visible_size(bo),
&adev->visible_pin_size);
+ amdgpu_amdkfd_account_vram_unpin(bo);
} else if (bo->tbo.resource->mem_type == TTM_PL_TT) {
atomic64_sub(amdgpu_bo_size(bo), &adev->gart_pin_size);
}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
index 9751a912177c..1baa5d95f71a 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
@@ -1073,11 +1073,19 @@ static int kfd_ioctl_get_available_memory(struct file
*filep,
{
struct kfd_ioctl_get_available_memory_args *args = data;
struct kfd_process_device *pdd = kfd_lock_pdd_by_id(p, args->gpu_id);
+ struct amdgpu_device *adev;
+ size_t available;
if (!pdd)
return -EINVAL;
- args->available = amdgpu_amdkfd_get_available_memory(pdd->dev->adev,
- pdd->dev->node_id);
+
+ adev = pdd->dev->adev;
+ available = amdgpu_amdkfd_get_available_memory(adev, pdd->dev->node_id);
+ if (adev->apu_prefer_gtt)
+ args->available = ALIGN_DOWN(available, PAGE_SIZE);
+ else
+ args->available = ALIGN_DOWN(available, VRAM_AVAILABLITY_ALIGN);
+
kfd_unlock_pdd(pdd);
return 0;
}
--
2.43.7
