Author: glebius
Date: Thu Nov 17 21:02:55 2016
New Revision: 308783
URL: https://svnweb.freebsd.org/changeset/base/308783
Log:
Use bogus_page to properly reduce number of I/Os in sendfile(2). The new
sendfile_swapin() loop works this way:
- Find first invalid page in the request.
- Do vm_pager_has_page() and get count of pages, that can be taken in
single I/O.
- Trim valid pages from the end of the request.
- Cycle through the request and substitute to bogus_page all valid
pages that are in the middle of the request.
- After I/O launched (pager copies array of pages into buf(9), it
is important to restore proper page pointers with help vm_page_lookup().
Count bogus pages used and report them in sendfile stats.
Modified:
head/sys/kern/kern_sendfile.c
head/sys/sys/sf_buf.h
head/usr.bin/netstat/mbuf.c
Modified: head/sys/kern/kern_sendfile.c
==============================================================================
--- head/sys/kern/kern_sendfile.c Thu Nov 17 21:01:27 2016
(r308782)
+++ head/sys/kern/kern_sendfile.c Thu Nov 17 21:02:55 2016
(r308783)
@@ -62,6 +62,8 @@ __FBSDID("$FreeBSD$");
#include <vm/vm_object.h>
#include <vm/vm_pager.h>
+extern vm_page_t bogus_page;
+
/*
* Structure describing a single sendfile(2) I/O, which may consist of
* several underlying pager I/Os.
@@ -258,7 +260,8 @@ sendfile_iodone(void *arg, vm_page_t *pg
struct socket *so;
for (int i = 0; i < count; i++)
- vm_page_xunbusy(pg[i]);
+ if (pg[i] != bogus_page)
+ vm_page_xunbusy(pg[i]);
if (error)
sfio->error = error;
@@ -341,51 +344,53 @@ sendfile_swapin(vm_object_t obj, struct
}
/*
- * Now 'i' points to first invalid page, iterate further
- * to make 'j' point at first valid after a bunch of
- * invalid ones.
- */
- for (j = i + 1; j < npages; j++)
- if (vm_page_is_valid(pa[j], vmoff(j, off) & PAGE_MASK,
- xfsize(j, npages, off, len))) {
- SFSTAT_INC(sf_pages_valid);
- break;
- }
-
- /*
- * Now we got region of invalid pages between 'i' and 'j'.
- * Check that they belong to pager. They may not be there,
- * which is a regular situation for shmem pager. For vnode
- * pager this happens only in case of sparse file.
+ * Next page is invalid. Check if it belongs to pager. It
+ * may not be there, which is a regular situation for shmem
+ * pager. For vnode pager this happens only in case of
+ * a sparse file.
*
* Important feature of vm_pager_has_page() is the hint
* stored in 'a', about how many pages we can pagein after
* this page in a single I/O.
*/
- while (!vm_pager_has_page(obj, OFF_TO_IDX(vmoff(i, off)),
- NULL, &a) && i < j) {
+ if (!vm_pager_has_page(obj, OFF_TO_IDX(vmoff(i, off)), NULL,
+ &a)) {
pmap_zero_page(pa[i]);
pa[i]->valid = VM_PAGE_BITS_ALL;
pa[i]->dirty = 0;
vm_page_xunbusy(pa[i]);
i++;
- }
- if (i == j)
continue;
+ }
/*
* We want to pagein as many pages as possible, limited only
* by the 'a' hint and actual request.
- *
- * We should not pagein into already valid page, thus if
- * 'j' didn't reach last page, trim by that page.
- *
- * When the pagein fulfils the request, also specify readahead.
*/
- if (j < npages)
- a = min(a, j - i - 1);
count = min(a + 1, npages - i);
+ /*
+ * We should not pagein into a valid page, thus we first trim
+ * any valid pages off the end of request, and substitute
+ * to bogus_page those, that are in the middle.
+ */
+ for (j = i + count - 1; j > i; j--) {
+ if (vm_page_is_valid(pa[j], vmoff(j, off) & PAGE_MASK,
+ xfsize(j, npages, off, len))) {
+ count--;
+ rhpages = 0;
+ } else
+ break;
+ }
+ for (j = i + 1; j < i + count - 1; j++)
+ if (vm_page_is_valid(pa[j], vmoff(j, off) & PAGE_MASK,
+ xfsize(j, npages, off, len))) {
+ vm_page_xunbusy(pa[j]);
+ SFSTAT_INC(sf_pages_valid);
+ SFSTAT_INC(sf_pages_bogus);
+ pa[j] = bogus_page;
+ }
+
refcount_acquire(&sfio->nios);
rv = vm_pager_get_pages_async(obj, pa + i, count, NULL,
i + count == npages ? &rhpages : NULL,
@@ -398,13 +403,18 @@ sendfile_swapin(vm_object_t obj, struct
if (i + count == npages)
SFSTAT_ADD(sf_rhpages_read, rhpages);
-#ifdef INVARIANTS
- for (j = i; j < i + count && j < npages; j++)
- KASSERT(pa[j] == vm_page_lookup(obj,
- OFF_TO_IDX(vmoff(j, off))),
- ("pa[j] %p lookup %p\n", pa[j],
- vm_page_lookup(obj, OFF_TO_IDX(vmoff(j, off)))));
-#endif
+ /*
+ * Restore the valid page pointers. They are already
+ * unbusied, but still wired.
+ */
+ for (j = i; j < i + count; j++)
+ if (pa[j] == bogus_page) {
+ pa[j] = vm_page_lookup(obj,
+ OFF_TO_IDX(vmoff(j, off)));
+ KASSERT(pa[j], ("%s: page %p[%d] disappeared",
+ __func__, pa, j));
+
+ }
i += count;
nios++;
}
Modified: head/sys/sys/sf_buf.h
==============================================================================
--- head/sys/sys/sf_buf.h Thu Nov 17 21:01:27 2016 (r308782)
+++ head/sys/sys/sf_buf.h Thu Nov 17 21:02:55 2016 (r308783)
@@ -41,6 +41,7 @@ struct sfstat { /* sendfile
statistic
uint64_t sf_busy; /* times aborted on a busy page */
uint64_t sf_allocfail; /* times sfbuf allocation failed */
uint64_t sf_allocwait; /* times sfbuf allocation had to wait */
+ uint64_t sf_pages_bogus; /* times bogus page was used */
};
#ifdef _KERNEL
Modified: head/usr.bin/netstat/mbuf.c
==============================================================================
--- head/usr.bin/netstat/mbuf.c Thu Nov 17 21:01:27 2016 (r308782)
+++ head/usr.bin/netstat/mbuf.c Thu Nov 17 21:02:55 2016 (r308783)
@@ -340,6 +340,9 @@ mbpr(void *kvmd, u_long mbaddr)
xo_emit("{:sendfile-pages-valid/%ju} "
"{N:pages were valid at time of a sendfile request}\n",
(uintmax_t)sfstat.sf_pages_valid);
+ xo_emit("{:sendfile-pages-bogus/%ju} "
+ "{N:pages were valid and substituted to bogus page}\n",
+ (uintmax_t)sfstat.sf_pages_bogus);
xo_emit("{:sendfile-requested-readahead/%ju} "
"{N:pages were requested for read ahead by applications}\n",
(uintmax_t)sfstat.sf_rhpages_requested);
_______________________________________________
svn-src-head@freebsd.org mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-head
To unsubscribe, send any mail to "svn-src-head-unsubscr...@freebsd.org"
