The branch main has been updated by dougm:
URL:
https://cgit.FreeBSD.org/src/commit/?id=f06cbfc5592e58104fe01f3f458928ee09be24fc
commit f06cbfc5592e58104fe01f3f458928ee09be24fc
Author: Doug Moore <do...@freebsd.org>
AuthorDate: 2025-04-10 07:19:06 +0000
Commit: Doug Moore <do...@freebsd.org>
CommitDate: 2025-04-10 07:19:06 +0000
vm_fault: use iterator for allocation
Pass an iterator to vm_fault_allocate, so that it can invoke
vm_page_alloc_after. In vm_fault_object, use an iterator for a page
lookup so that the search can benefit the allocation in
vm_fault_allocate. In vm_fault, define an iterator, initialize it only
when it might be used, and pass it on to vm_fault_allocate.
Reviewed by: kib
Differential Revision: https://reviews.freebsd.org/D49741
---
sys/vm/vm_fault.c | 18 ++++++++++++------
sys/vm/vm_radix.h | 13 +++++++++++++
2 files changed, 25 insertions(+), 6 deletions(-)
diff --git a/sys/vm/vm_fault.c b/sys/vm/vm_fault.c
index 48a7a47e4c59..81631b672040 100644
--- a/sys/vm/vm_fault.c
+++ b/sys/vm/vm_fault.c
@@ -105,6 +105,7 @@
#include <vm/vm_pageout.h>
#include <vm/vm_kern.h>
#include <vm/vm_pager.h>
+#include <vm/vm_radix.h>
#include <vm/vm_extern.h>
#include <vm/vm_reserv.h>
@@ -1230,7 +1231,7 @@ vm_fault_allocate_oom(struct faultstate *fs)
* Allocate a page directly or via the object populate method.
*/
static enum fault_status
-vm_fault_allocate(struct faultstate *fs)
+vm_fault_allocate(struct faultstate *fs, struct pctrie_iter *pages)
{
struct domainset *dset;
enum fault_status res;
@@ -1291,8 +1292,9 @@ vm_fault_allocate(struct faultstate *fs)
vm_fault_unlock_and_deallocate(fs);
return (FAULT_FAILURE);
}
- fs->m = vm_page_alloc(fs->object, fs->pindex,
- P_KILLED(curproc) ? VM_ALLOC_SYSTEM : 0);
+ fs->m = vm_page_alloc_after(fs->object, fs->pindex,
+ P_KILLED(curproc) ? VM_ALLOC_SYSTEM : 0,
+ vm_radix_iter_lookup_lt(pages, fs->pindex));
}
if (fs->m == NULL) {
if (vm_fault_allocate_oom(fs))
@@ -1459,6 +1461,7 @@ vm_fault_busy_sleep(struct faultstate *fs)
static enum fault_status
vm_fault_object(struct faultstate *fs, int *behindp, int *aheadp)
{
+ struct pctrie_iter pages;
enum fault_status res;
bool dead;
@@ -1484,7 +1487,8 @@ vm_fault_object(struct faultstate *fs, int *behindp, int
*aheadp)
/*
* See if the page is resident.
*/
- fs->m = vm_page_lookup(fs->object, fs->pindex);
+ vm_page_iter_init(&pages, fs->object);
+ fs->m = vm_radix_iter_lookup(&pages, fs->pindex);
if (fs->m != NULL) {
if (!vm_page_tryxbusy(fs->m)) {
vm_fault_busy_sleep(fs);
@@ -1514,7 +1518,7 @@ vm_fault_object(struct faultstate *fs, int *behindp, int
*aheadp)
vm_fault_unlock_and_deallocate(fs);
return (FAULT_RESTART);
}
- res = vm_fault_allocate(fs);
+ res = vm_fault_allocate(fs, &pages);
if (res != FAULT_CONTINUE)
return (res);
}
@@ -1549,6 +1553,7 @@ int
vm_fault(vm_map_t map, vm_offset_t vaddr, vm_prot_t fault_type,
int fault_flags, vm_page_t *m_hold)
{
+ struct pctrie_iter pages;
struct faultstate fs;
int ahead, behind, faultcount, rv;
enum fault_status res;
@@ -1603,6 +1608,7 @@ RetryFault:
}
VM_OBJECT_ASSERT_WLOCKED(fs.first_object);
} else {
+ vm_page_iter_init(&pages, fs.first_object);
VM_OBJECT_WLOCK(fs.first_object);
}
@@ -1627,7 +1633,7 @@ RetryFault:
fs.pindex = fs.first_pindex;
if ((fs.entry->eflags & MAP_ENTRY_SPLIT_BOUNDARY_MASK) != 0) {
- res = vm_fault_allocate(&fs);
+ res = vm_fault_allocate(&fs, &pages);
switch (res) {
case FAULT_RESTART:
goto RetryFault;
diff --git a/sys/vm/vm_radix.h b/sys/vm/vm_radix.h
index 561e36f2e164..e9ee3e7f3911 100644
--- a/sys/vm/vm_radix.h
+++ b/sys/vm/vm_radix.h
@@ -270,6 +270,19 @@ vm_radix_iter_lookup_le(struct pctrie_iter *pages,
vm_pindex_t index)
return (VM_RADIX_PCTRIE_ITER_LOOKUP_LE(pages, index));
}
+/*
+ * Initialize an iterator pointing to the page with the greatest pindex that is
+ * less than to the specified pindex, or NULL if there are no such
+ * pages. Return the page.
+ *
+ * Requires that access be externally synchronized by a lock.
+ */
+static __inline vm_page_t
+vm_radix_iter_lookup_lt(struct pctrie_iter *pages, vm_pindex_t index)
+{
+ return (index == 0 ? NULL : vm_radix_iter_lookup_le(pages, index - 1));
+}
+
/*
* Update the iterator to point to the page with the pindex that is one greater
* than the current pindex, or NULL if there is no such page. Return the page.
