Author: ian
Date: Fri Apr 4 17:57:49 2014
New Revision: 264130
URL: http://svnweb.freebsd.org/changeset/base/264130
Log:
Allocate per-cpu resources for doing pmap_zero_page() and pmap_copy_page().
This is performance enhancement rather than bugfix.
Modified:
head/sys/arm/arm/pmap-v6.c
Modified: head/sys/arm/arm/pmap-v6.c
==============================================================================
--- head/sys/arm/arm/pmap-v6.c Fri Apr 4 17:45:39 2014 (r264129)
+++ head/sys/arm/arm/pmap-v6.c Fri Apr 4 17:57:49 2014 (r264130)
@@ -265,9 +265,18 @@ vm_offset_t vm_max_kernel_address;
struct pmap kernel_pmap_store;
-static pt_entry_t *csrc_pte, *cdst_pte;
-static vm_offset_t csrcp, cdstp;
-static struct mtx cmtx;
+/*
+ * Resources for quickly copying and zeroing pages using virtual address space
+ * and page table entries that are pre-allocated per-CPU by pmap_init().
+ */
+struct czpages {
+ struct mtx lock;
+ pt_entry_t *srcptep;
+ pt_entry_t *dstptep;
+ vm_offset_t srcva;
+ vm_offset_t dstva;
+};
+static struct czpages cpu_czpages[MAXCPU];
static void pmap_init_l1(struct l1_ttable *, pd_entry_t *);
/*
@@ -1803,13 +1812,14 @@ pmap_bootstrap(vm_offset_t firstaddr, st
struct l1_ttable *l1 = &static_l1;
struct l2_dtable *l2;
struct l2_bucket *l2b;
+ struct czpages *czp;
pd_entry_t pde;
pd_entry_t *kernel_l1pt = (pd_entry_t *)l1pt->pv_va;
pt_entry_t *ptep;
vm_paddr_t pa;
vm_offset_t va;
vm_size_t size;
- int l1idx, l2idx, l2next = 0;
+ int i, l1idx, l2idx, l2next = 0;
PDEBUG(1, printf("firstaddr = %08x, lastaddr = %08x\n",
firstaddr, vm_max_kernel_address));
@@ -1921,13 +1931,16 @@ pmap_bootstrap(vm_offset_t firstaddr, st
/*
* Reserve some special page table entries/VA space for temporary
- * mapping of pages.
+ * mapping of pages that are being copied or zeroed.
*/
+ for (czp = cpu_czpages, i = 0; i < MAXCPU; ++i, ++czp) {
+ mtx_init(&czp->lock, "czpages", NULL, MTX_DEF);
+ pmap_alloc_specials(&virtual_avail, 1, &czp->srcva,
&czp->srcptep);
+ pmap_set_pt_cache_mode(kernel_l1pt, (vm_offset_t)czp->srcptep);
+ pmap_alloc_specials(&virtual_avail, 1, &czp->dstva,
&czp->dstptep);
+ pmap_set_pt_cache_mode(kernel_l1pt, (vm_offset_t)czp->dstptep);
+ }
- pmap_alloc_specials(&virtual_avail, 1, &csrcp, &csrc_pte);
- pmap_set_pt_cache_mode(kernel_l1pt, (vm_offset_t)csrc_pte);
- pmap_alloc_specials(&virtual_avail, 1, &cdstp, &cdst_pte);
- pmap_set_pt_cache_mode(kernel_l1pt, (vm_offset_t)cdst_pte);
size = ((vm_max_kernel_address - pmap_curmaxkvaddr) + L1_S_OFFSET) /
L1_S_SIZE;
pmap_alloc_specials(&virtual_avail,
@@ -1955,7 +1968,6 @@ pmap_bootstrap(vm_offset_t firstaddr, st
virtual_avail = round_page(virtual_avail);
virtual_end = vm_max_kernel_address;
kernel_vm_end = pmap_curmaxkvaddr;
- mtx_init(&cmtx, "TMP mappings mtx", NULL, MTX_DEF);
pmap_set_pcb_pagedir(kernel_pmap, thread0.td_pcb);
}
@@ -4439,39 +4451,42 @@ pmap_remove(pmap_t pmap, vm_offset_t sva
static void
pmap_zero_page_gen(vm_page_t m, int off, int size)
{
+ struct czpages *czp;
+
+ KASSERT(TAILQ_EMPTY(&m->md.pv_list),
+ ("pmap_zero_page_gen: page has mappings"));
vm_paddr_t phys = VM_PAGE_TO_PHYS(m);
- if (!TAILQ_EMPTY(&m->md.pv_list))
- panic("pmap_zero_page: page has mappings");
- mtx_lock(&cmtx);
+ sched_pin();
+ czp = &cpu_czpages[PCPU_GET(cpuid)];
+ mtx_lock(&czp->lock);
+
/*
- * Hook in the page, zero it, invalidate the TLB as needed.
- *
- * Note the temporary zero-page mapping must be a non-cached page in
- * order to work without corruption when write-allocate is enabled.
+ * Hook in the page, zero it.
*/
- *cdst_pte = L2_S_PROTO | phys | pte_l2_s_cache_mode | L2_S_REF;
- pmap_set_prot(cdst_pte, VM_PROT_WRITE, 0);
- PTE_SYNC(cdst_pte);
- cpu_tlb_flushD_SE(cdstp);
+ *czp->dstptep = L2_S_PROTO | phys | pte_l2_s_cache_mode | L2_S_REF;
+ pmap_set_prot(czp->dstptep, VM_PROT_WRITE, 0);
+ PTE_SYNC(czp->dstptep);
+ cpu_tlb_flushD_SE(czp->dstva);
cpu_cpwait();
+
if (off || size != PAGE_SIZE)
- bzero((void *)(cdstp + off), size);
+ bzero((void *)(czp->dstva + off), size);
else
- bzero_page(cdstp);
+ bzero_page(czp->dstva);
/*
- * Although aliasing is not possible if we use
- * cdstp temporary mappings with memory that
- * will be mapped later as non-cached or with write-through
- * caches we might end up overwriting it when calling wbinv_all
- * So make sure caches are clean after copy operation
+ * Although aliasing is not possible, if we use temporary mappings with
+ * memory that will be mapped later as non-cached or with write-through
+ * caches, we might end up overwriting it when calling wbinv_all. So
+ * make sure caches are clean after the operation.
*/
- cpu_idcache_wbinv_range(cdstp, size);
- pmap_l2cache_wbinv_range(cdstp, phys, size);
+ cpu_idcache_wbinv_range(czp->dstva, size);
+ pmap_l2cache_wbinv_range(czp->dstva, phys, size);
- mtx_unlock(&cmtx);
+ mtx_unlock(&czp->lock);
+ sched_unpin();
}
/*
@@ -4529,45 +4544,39 @@ pmap_zero_page_idle(vm_page_t m)
void
pmap_copy_page_generic(vm_paddr_t src, vm_paddr_t dst)
{
- /*
- * Hold the source page's lock for the duration of the copy
- * so that no other mappings can be created while we have a
- * potentially aliased mapping.
- * Map the pages into the page hook points, copy them, and purge
- * the cache for the appropriate page. Invalidate the TLB
- * as required.
- */
- mtx_lock(&cmtx);
-
- /* For ARMv6 using System bit is deprecated and mapping with AP
- * bits set to 0x0 makes page not accessible. csrc_pte is mapped
- * read/write until proper mapping defines are created for ARMv6.
- */
- *csrc_pte = L2_S_PROTO | src | pte_l2_s_cache_mode | L2_S_REF;
- pmap_set_prot(csrc_pte, VM_PROT_READ, 0);
- PTE_SYNC(csrc_pte);
-
- *cdst_pte = L2_S_PROTO | dst | pte_l2_s_cache_mode | L2_S_REF;
- pmap_set_prot(cdst_pte, VM_PROT_READ | VM_PROT_WRITE, 0);
- PTE_SYNC(cdst_pte);
+ struct czpages *czp;
- cpu_tlb_flushD_SE(csrcp);
- cpu_tlb_flushD_SE(cdstp);
+ sched_pin();
+ czp = &cpu_czpages[PCPU_GET(cpuid)];
+ mtx_lock(&czp->lock);
+
+ /*
+ * Map the pages into the page hook points, copy them, and purge the
+ * cache for the appropriate page.
+ */
+ *czp->srcptep = L2_S_PROTO | src | pte_l2_s_cache_mode | L2_S_REF;
+ pmap_set_prot(czp->srcptep, VM_PROT_READ, 0);
+ PTE_SYNC(czp->srcptep);
+ cpu_tlb_flushD_SE(czp->srcva);
+ *czp->dstptep = L2_S_PROTO | dst | pte_l2_s_cache_mode | L2_S_REF;
+ pmap_set_prot(czp->dstptep, VM_PROT_READ | VM_PROT_WRITE, 0);
+ PTE_SYNC(czp->dstptep);
+ cpu_tlb_flushD_SE(czp->dstva);
cpu_cpwait();
+ bcopy_page(czp->srcva, czp->dstva);
+
/*
- * Although aliasing is not possible if we use
- * cdstp temporary mappings with memory that
- * will be mapped later as non-cached or with write-through
- * caches we might end up overwriting it when calling wbinv_all
- * So make sure caches are clean after copy operation
+ * Although aliasing is not possible, if we use temporary mappings with
+ * memory that will be mapped later as non-cached or with write-through
+ * caches, we might end up overwriting it when calling wbinv_all. So
+ * make sure caches are clean after the operation.
*/
- bcopy_page(csrcp, cdstp);
-
- cpu_idcache_wbinv_range(cdstp, PAGE_SIZE);
- pmap_l2cache_wbinv_range(cdstp, dst, PAGE_SIZE);
+ cpu_idcache_wbinv_range(czp->dstva, PAGE_SIZE);
+ pmap_l2cache_wbinv_range(czp->dstva, dst, PAGE_SIZE);
- mtx_unlock(&cmtx);
+ mtx_unlock(&czp->lock);
+ sched_unpin();
}
int unmapped_buf_allowed = 1;
@@ -4579,8 +4588,12 @@ pmap_copy_pages(vm_page_t ma[], vm_offse
vm_page_t a_pg, b_pg;
vm_offset_t a_pg_offset, b_pg_offset;
int cnt;
+ struct czpages *czp;
+
+ sched_pin();
+ czp = &cpu_czpages[PCPU_GET(cpuid)];
+ mtx_lock(&czp->lock);
- mtx_lock(&cmtx);
while (xfersize > 0) {
a_pg = ma[a_offset >> PAGE_SHIFT];
a_pg_offset = a_offset & PAGE_MASK;
@@ -4588,27 +4601,29 @@ pmap_copy_pages(vm_page_t ma[], vm_offse
b_pg = mb[b_offset >> PAGE_SHIFT];
b_pg_offset = b_offset & PAGE_MASK;
cnt = min(cnt, PAGE_SIZE - b_pg_offset);
- *csrc_pte = L2_S_PROTO | VM_PAGE_TO_PHYS(a_pg) |
+ *czp->srcptep = L2_S_PROTO | VM_PAGE_TO_PHYS(a_pg) |
pte_l2_s_cache_mode | L2_S_REF;
- pmap_set_prot(csrc_pte, VM_PROT_READ, 0);
- PTE_SYNC(csrc_pte);
- *cdst_pte = L2_S_PROTO | VM_PAGE_TO_PHYS(b_pg) |
+ pmap_set_prot(czp->srcptep, VM_PROT_READ, 0);
+ PTE_SYNC(czp->srcptep);
+ cpu_tlb_flushD_SE(czp->srcva);
+ *czp->dstptep = L2_S_PROTO | VM_PAGE_TO_PHYS(b_pg) |
pte_l2_s_cache_mode | L2_S_REF;
- pmap_set_prot(cdst_pte, VM_PROT_READ | VM_PROT_WRITE, 0);
- PTE_SYNC(cdst_pte);
- cpu_tlb_flushD_SE(csrcp);
- cpu_tlb_flushD_SE(cdstp);
+ pmap_set_prot(czp->dstptep, VM_PROT_READ | VM_PROT_WRITE, 0);
+ PTE_SYNC(czp->dstptep);
+ cpu_tlb_flushD_SE(czp->dstva);
cpu_cpwait();
- bcopy((char *)csrcp + a_pg_offset, (char *)cdstp + b_pg_offset,
+ bcopy((char *)czp->srcva + a_pg_offset, (char *)czp->dstva +
b_pg_offset,
cnt);
- cpu_idcache_wbinv_range(cdstp + b_pg_offset, cnt);
- pmap_l2cache_wbinv_range(cdstp + b_pg_offset,
+ cpu_idcache_wbinv_range(czp->dstva + b_pg_offset, cnt);
+ pmap_l2cache_wbinv_range(czp->dstva + b_pg_offset,
VM_PAGE_TO_PHYS(b_pg) + b_pg_offset, cnt);
xfersize -= cnt;
a_offset += cnt;
b_offset += cnt;
}
- mtx_unlock(&cmtx);
+
+ mtx_unlock(&czp->lock);
+ sched_unpin();
}
void
_______________________________________________
[email protected] mailing list
http://lists.freebsd.org/mailman/listinfo/svn-src-head
To unsubscribe, send any mail to "[email protected]"
