Makes sense, thanks. I won't respin this one then.
Vitaly, for the reorder, here is a deterministic reproducer so you can confirm
the splat on your side without a round-trip. It arms an mmu_interval_notifier
via GEM_USERPTR over anonymous memory, then forces reclaim of that exact range
with madvise(MADV_PAGEOUT), so amdgpu_hmm_invalidate_gfx() takes notifier_lock
under fs_reclaim in the calling thread. Needs CONFIG_PROVE_LOCKING and a fresh
boot; build/run notes are in the header. Happy to give Tested-by once you post.
// SPDX-License-Identifier: MIT
/*
* amdgpu-notifier-reclaim-repro.c
*
* Deterministic reproducer for the false circular-locking-dependency splat
* produced by drivers/gpu/drm/amd/amdgpu/amdgpu_lockdep.c.
*
* amdgpu_lockdep_init() (run at module load) calls fs_reclaim_acquire()
* while holding the dummy notifier_lock, teaching lockdep that it is legal
* to enter reclaim with the MMU-notifier lock held. The real notifier lock
* is taken in amdgpu_hmm_invalidate_gfx(), which mm/ calls from inside
* reclaim, so the reverse edge fs_reclaim -> mmu_notifier -> notifier_lock
* is mandatory. The cycle is closed the first time reclaim unmaps a page
* covered by an amdgpu userptr interval notifier.
*
* This program installs such a notifier (GEM_USERPTR) over anonymous memory
* and then forces synchronous reclaim of that exact range with
* MADV_PAGEOUT, which runs try_to_unmap()->invalidate_range_start() with
* fs_reclaim held in the calling thread, closing the loop on demand.
*
* Requirements:
* - kernel built with CONFIG_PROVE_LOCKING (lockdep)
* - amdgpu loaded; run from a FRESH boot (the first lockdep splat of any
* kind calls debug_locks_off() and silences all later reports)
*
* Build: cc -O2 -o repro amdgpu-notifier-reclaim-repro.c
* Run: ./repro # picks the first amdgpu render node
* ./repro /dev/dri/renderD129
* Watch: sudo dmesg -w
*/
#define _GNU_SOURCE
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdint.h>
#include <errno.h>
#include <fcntl.h>
#include <unistd.h>
#include <dirent.h>
#include <sys/ioctl.h>
#include <sys/mman.h>
/* --- minimal amdgpu uapi (self-contained, no libdrm needed) ----------- */
#ifndef DRM_IOCTL_BASE
#define DRM_IOCTL_BASE 'd'
#endif
#define DRM_COMMAND_BASE 0x40
#define DRM_AMDGPU_GEM_USERPTR 0x11
struct drm_amdgpu_gem_userptr {
uint64_t addr;
uint64_t size;
uint32_t flags;
uint32_t handle;
};
#define DRM_IOCTL_AMDGPU_GEM_USERPTR \
_IOWR(DRM_IOCTL_BASE, DRM_COMMAND_BASE + DRM_AMDGPU_GEM_USERPTR, \
struct drm_amdgpu_gem_userptr)
#define AMDGPU_GEM_USERPTR_READONLY (1 << 0)
#define AMDGPU_GEM_USERPTR_ANONONLY (1 << 1)
#define AMDGPU_GEM_USERPTR_VALIDATE (1 << 2)
#define AMDGPU_GEM_USERPTR_REGISTER (1 << 3)
#ifndef MADV_PAGEOUT
#define MADV_PAGEOUT 21
#endif
/* ------------------------------------------------------------ */
#define BUF_SIZE (64ull * 1024 * 1024) /* 64 MiB, page aligned by mmap */
static int open_amdgpu_render(const char *forced)
{
if (forced) {
int fd = open(forced, O_RDWR | O_CLOEXEC);
if (fd < 0)
perror(forced);
return fd;
}
/* try renderD128..renderD143 and keep the first that accepts
GEM_USERPTR */
for (int i = 128; i < 144; i++) {
char path[64];
snprintf(path, sizeof(path), "/dev/dri/renderD%d", i);
int fd = open(path, O_RDWR | O_CLOEXEC);
if (fd < 0)
continue;
/* probe: a zero-size userptr returns -EINVAL on amdgpu but
* -ENOTTY/-ENODEV on a non-amdgpu driver, which lets us tell
* the nodes apart without pulling in libdrm version ioctls. */
struct drm_amdgpu_gem_userptr probe = { 0 };
errno = 0;
ioctl(fd, DRM_IOCTL_AMDGPU_GEM_USERPTR, &probe);
if (errno == ENOTTY || errno == ENODEV || errno == EOPNOTSUPP) {
close(fd);
continue;
}
fprintf(stderr, "using %s\n", path);
return fd;
}
fprintf(stderr, "no amdgpu render node found under /dev/dri\n");
return -1;
}
int main(int argc, char **argv)
{
int fd = open_amdgpu_render(argc > 1 ? argv[1] : NULL);
if (fd < 0)
return 1;
/* anonymous, private, page-aligned region for the userptr */
void *buf = mmap(NULL, BUF_SIZE, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
if (buf == MAP_FAILED) {
perror("mmap");
return 1;
}
memset(buf, 0xa5, BUF_SIZE); /* fault every page in */
/* REGISTER installs the mmu_interval_notifier (amdgpu_hmm_register());
* VALIDATE additionally faults the pages via hmm_range_fault() and
* binds them into GTT. ANONONLY matches our MAP_ANONYMOUS region. */
struct drm_amdgpu_gem_userptr up = {
.addr = (uint64_t)(uintptr_t)buf,
.size = BUF_SIZE,
.flags = AMDGPU_GEM_USERPTR_ANONONLY |
AMDGPU_GEM_USERPTR_REGISTER |
AMDGPU_GEM_USERPTR_VALIDATE,
};
if (ioctl(fd, DRM_IOCTL_AMDGPU_GEM_USERPTR, &up)) {
perror("GEM_USERPTR (VALIDATE)");
/* retry without VALIDATE: the notifier is still registered, the
* pages are present from the memset, MADV_PAGEOUT still works
*/
up.flags = AMDGPU_GEM_USERPTR_ANONONLY |
AMDGPU_GEM_USERPTR_REGISTER;
if (ioctl(fd, DRM_IOCTL_AMDGPU_GEM_USERPTR, &up)) {
perror("GEM_USERPTR (REGISTER)");
return 1;
}
}
fprintf(stderr, "userptr handle=%u, interval notifier armed over
%p..%p\n",
up.handle, buf, (char *)buf + BUF_SIZE);
/* Force synchronous reclaim of the notifier-covered range. MADV_PAGEOUT
* runs shrink_folio_list()->try_to_unmap()->invalidate_range_start()
* with fs_reclaim held in THIS thread, so amdgpu_hmm_invalidate_gfx()
* takes notifier_lock under fs_reclaim and lockdep closes the cycle.
*
* A few iterations (re-touching in between) cover the race where pages
* are already paged out on the first pass. */
for (int it = 0; it < 8; it++) {
if (madvise(buf, BUF_SIZE, MADV_PAGEOUT))
perror("madvise(MADV_PAGEOUT)");
usleep(50 * 1000);
memset(buf, 0xa5, BUF_SIZE); /* fault back in for the next pass
*/
}
fprintf(stderr,
"done: check dmesg for "
"\"possible circular locking dependency\" /
amdgpu_hmm_invalidate_gfx\n");
munmap(buf, BUF_SIZE);
close(fd);
return 0;
}
