Module Name:    src
Committed By:   ad
Date:           Sun Dec 22 16:37:36 UTC 2019

Modified Files:
        src/sys/uvm: uvm_page.c

Log Message:
uvm_pagealloc_strat(): Tweak the locking to allow for lazy dequeue of pages
in the pdpolicy code.  This means taking pg->interlock if assigning to
an object.  The remaining barrier to lazy dequeue is having a dedicated
TAILQ_ENTRY in the page (it's currently shared with the page allocator).


To generate a diff of this commit:
cvs rdiff -u -r1.211 -r1.212 src/sys/uvm/uvm_page.c

Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.

Modified files:

Index: src/sys/uvm/uvm_page.c
diff -u src/sys/uvm/uvm_page.c:1.211 src/sys/uvm/uvm_page.c:1.212
--- src/sys/uvm/uvm_page.c:1.211	Sat Dec 21 15:16:14 2019
+++ src/sys/uvm/uvm_page.c	Sun Dec 22 16:37:36 2019
@@ -1,4 +1,4 @@
-/*	$NetBSD: uvm_page.c,v 1.211 2019/12/21 15:16:14 ad Exp $	*/
+/*	$NetBSD: uvm_page.c,v 1.212 2019/12/22 16:37:36 ad Exp $	*/
 
 /*
  * Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -66,7 +66,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_page.c,v 1.211 2019/12/21 15:16:14 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_page.c,v 1.212 2019/12/22 16:37:36 ad Exp $");
 
 #include "opt_ddb.h"
 #include "opt_uvm.h"
@@ -829,10 +829,6 @@ uvm_pagealloc_pgfl(struct uvm_cpu *ucpu,
 	LIST_REMOVE(pg, listq.list);	/* per-cpu list */
 	uvmexp.free--;
 
-	/* update zero'd page count */
-	if (pg->flags & PG_ZERO)
-	    	CPU_COUNT(CPU_COUNT_ZEROPAGES, -1);
-
 	if (color == trycolor)
 	    	CPU_COUNT(CPU_COUNT_COLORHIT, 1);
 	else {
@@ -996,34 +992,43 @@ uvm_pagealloc_strat(struct uvm_object *o
 			ucpu->page_idle_zero = vm_page_zero_enable;
 		}
 	}
+	if (pg->flags & PG_ZERO) {
+	    	CPU_COUNT(CPU_COUNT_ZEROPAGES, -1);
+	}
+	if (anon) {
+		CPU_COUNT(CPU_COUNT_ANONPAGES, 1);
+	}
 	KASSERT((pg->flags & ~(PG_ZERO|PG_FREE)) == 0);
 
-	/*
-	 * For now check this - later on we may do lazy dequeue, but need
-	 * to get page.queue used only by the pagedaemon policy first.
-	 */
-	KASSERT(!uvmpdpol_pageisqueued_p(pg));
+	/* mark the page as allocated and then drop uvm_fpageqlock. */
+	pg->flags &= ~PG_FREE;
+	mutex_spin_exit(&uvm_fpageqlock);
 
 	/*
-	 * assign the page to the object.  we don't need to lock the page's
-	 * identity to do this, as the caller holds the objects locked, and
-	 * the page is not on any paging queues at this time.
+	 * assign the page to the object.  as the page was free, we know
+	 * that pg->uobject and pg->uanon are NULL.  we only need to take
+	 * the page's interlock if we are changing the values.
 	 */
+	if (anon != NULL || obj != NULL) {
+		mutex_enter(&pg->interlock);
+	}
 	pg->offset = off;
 	pg->uobject = obj;
 	pg->uanon = anon;
 	KASSERT(uvm_page_locked_p(pg));
 	pg->flags = PG_BUSY|PG_CLEAN|PG_FAKE;
-	mutex_spin_exit(&uvm_fpageqlock);
 	if (anon) {
 		anon->an_page = pg;
 		pg->flags |= PG_ANON;
-		cpu_count(CPU_COUNT_ANONPAGES, 1);
+		mutex_exit(&pg->interlock);
 	} else if (obj) {
 		uvm_pageinsert_object(obj, pg);
+		mutex_exit(&pg->interlock);
 		error = uvm_pageinsert_tree(obj, pg);
 		if (error != 0) {
+			mutex_enter(&pg->interlock);
 			uvm_pageremove_object(obj, pg);
+			mutex_exit(&pg->interlock);
 			uvm_pagefree(pg);
 			return NULL;
 		}

Reply via email to