Module Name: src
Committed By: scole
Date: Sat Apr 8 18:08:33 UTC 2017
Modified Files:
src/sys/arch/ia64/ia64: pmap.c
src/sys/arch/ia64/include: pmap.h
Log Message:
Attempted port over from FreeBSD with suggestions from <chs>. Still
more work needed, but at least now the ski simulator and hardware die
at the same place.
To generate a diff of this commit:
cvs rdiff -u -r1.34 -r1.35 src/sys/arch/ia64/ia64/pmap.c
cvs rdiff -u -r1.7 -r1.8 src/sys/arch/ia64/include/pmap.h
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/ia64/ia64/pmap.c
diff -u src/sys/arch/ia64/ia64/pmap.c:1.34 src/sys/arch/ia64/ia64/pmap.c:1.35
--- src/sys/arch/ia64/ia64/pmap.c:1.34 Fri Dec 23 17:26:43 2016
+++ src/sys/arch/ia64/ia64/pmap.c Sat Apr 8 18:08:33 2017
@@ -1,5 +1,4 @@
-/* $NetBSD: pmap.c,v 1.34 2016/12/23 17:26:43 scole Exp $ */
-
+/* $NetBSD: pmap.c,v 1.35 2017/04/08 18:08:33 scole Exp $ */
/*-
* Copyright (c) 1998, 1999, 2000, 2001 The NetBSD Foundation, Inc.
@@ -80,18 +79,18 @@
/* __FBSDID("$FreeBSD: src/sys/ia64/ia64/pmap.c,v 1.172 2005/11/20 06:09:48 alc Exp $"); */
-
-/* XXX: This module is a mess. Need to clean up Locking, list traversal. etc....... */
-
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.34 2016/12/23 17:26:43 scole Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.35 2017/04/08 18:08:33 scole Exp $");
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/buf.h>
#include <sys/reboot.h>
#include <sys/lock.h>
+#include <sys/pool.h>
+#include <sys/sched.h>
+#include <sys/bitops.h>
#include <uvm/uvm.h>
#include <uvm/uvm_physseg.h>
@@ -99,47 +98,158 @@ __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.3
#include <machine/pal.h>
#include <machine/atomic.h>
#include <machine/pte.h>
-#include <sys/sched.h>
#include <machine/cpufunc.h>
#include <machine/md_var.h>
+#include <machine/vmparam.h>
+
+/*
+ * Manages physical address maps.
+ *
+ * Since the information managed by this module is
+ * also stored by the logical address mapping module,
+ * this module may throw away valid virtual-to-physical
+ * mappings at almost any time. However, invalidations
+ * of virtual-to-physical mappings must be done as
+ * requested.
+ *
+ * In order to cope with hardware architectures which
+ * make virtual-to-physical map invalidates expensive,
+ * this module may delay invalidate or reduced protection
+ * operations until such time as they are actually
+ * necessary. This module is given full information as
+ * to which processors are currently using which maps,
+ * and to when physical maps must be made correct.
+ */
+
+/*
+ * Following the Linux model, region IDs are allocated in groups of
+ * eight so that a single region ID can be used for as many RRs as we
+ * want by encoding the RR number into the low bits of the ID.
+ *
+ * We reserve region ID 0 for the kernel and allocate the remaining
+ * IDs for user pmaps.
+ *
+ * Region 0-3: User virtually mapped
+ * Region 4: PBVM and special mappings
+ * Region 5: Kernel virtual memory
+ * Region 6: Direct-mapped uncacheable
+ * Region 7: Direct-mapped cacheable
+ */
+
+#if !defined(DIAGNOSTIC)
+#define PMAP_INLINE __inline
+#else
+#define PMAP_INLINE
+#endif
+
+#ifdef PV_STATS
+#define PV_STAT(x) do { x ; } while (0)
+#else
+#define PV_STAT(x) do { } while (0)
+#endif
+
+#define pmap_accessed(lpte) ((lpte)->pte & PTE_ACCESSED)
+#define pmap_dirty(lpte) ((lpte)->pte & PTE_DIRTY)
+#define pmap_exec(lpte) ((lpte)->pte & PTE_AR_RX)
+#define pmap_managed(lpte) ((lpte)->pte & PTE_MANAGED)
+#define pmap_ppn(lpte) ((lpte)->pte & PTE_PPN_MASK)
+#define pmap_present(lpte) ((lpte)->pte & PTE_PRESENT)
+#define pmap_prot(lpte) (((lpte)->pte & PTE_PROT_MASK) >> 56)
+#define pmap_wired(lpte) ((lpte)->pte & PTE_WIRED)
+
+#define pmap_clear_accessed(lpte) (lpte)->pte &= ~PTE_ACCESSED
+#define pmap_clear_dirty(lpte) (lpte)->pte &= ~PTE_DIRTY
+#define pmap_clear_present(lpte) (lpte)->pte &= ~PTE_PRESENT
+#define pmap_clear_wired(lpte) (lpte)->pte &= ~PTE_WIRED
+
+#define pmap_set_wired(lpte) (lpte)->pte |= PTE_WIRED
+
+/*
+ * Individual PV entries are stored in per-pmap chunks. This saves
+ * space by eliminating the need to record the pmap within every PV
+ * entry.
+ */
+#if PAGE_SIZE == 8192
+#define _NPCM 6
+#define _NPCPV 337
+#define _NPCS 2
+#elif PAGE_SIZE == 16384
+#define _NPCM 11
+#define _NPCPV 677
+#define _NPCS 1
+#else
+#error "invalid page size"
+#endif
+
+struct pv_chunk {
+ pmap_t pc_pmap;
+ TAILQ_ENTRY(pv_chunk) pc_list;
+ u_long pc_map[_NPCM]; /* bitmap; 1 = free */
+ TAILQ_ENTRY(pv_chunk) pc_lru;
+ u_long pc_spare[_NPCS];
+ struct pv_entry pc_pventry[_NPCPV];
+};
+
+/*
+ * The VHPT bucket head structure.
+ */
+struct ia64_bucket {
+ uint64_t chain;
+ kmutex_t mutex;
+ u_int length;
+};
+
+/*
+ * Statically allocated kernel pmap
+ */
+static struct pmap kernel_pmap_store;/* the kernel's pmap (proc0) */
+struct pmap *const kernel_pmap_ptr = &kernel_pmap_store;
+vaddr_t virtual_avail; /* VA of first avail page (after kernel bss) */
+vaddr_t virtual_end; /* VA of last avail page (end of kernel AS) */
+
+/* XXX freebsd, needs to be sorted out */
+#define kernel_pmap pmap_kernel()
+#define critical_enter() kpreempt_disable()
+#define critical_exit() kpreempt_enable()
+/* flags the entire page as dirty */
+#define vm_page_dirty(page) (page->flags &= ~PG_CLEAN)
+#define vm_page_is_managed(page) (pmap_initialized && uvm_pageismanaged(VM_PAGE_TO_PHYS(page)))
/*
* Kernel virtual memory management.
*/
static int nkpt;
-struct ia64_lpte **ia64_kptdir;
-#define KPTE_DIR_INDEX(va) \
- ((va >> (2*PAGE_SHIFT-5)) & ((1<<(PAGE_SHIFT-3))-1))
+
+extern struct ia64_lpte ***ia64_kptdir;
+
+#define KPTE_DIR0_INDEX(va) \
+ (((va) >> (3*PAGE_SHIFT-8)) & ((1<<(PAGE_SHIFT-3))-1))
+#define KPTE_DIR1_INDEX(va) \
+ (((va) >> (2*PAGE_SHIFT-5)) & ((1<<(PAGE_SHIFT-3))-1))
#define KPTE_PTE_INDEX(va) \
- ((va >> PAGE_SHIFT) & ((1<<(PAGE_SHIFT-5))-1))
+ (((va) >> PAGE_SHIFT) & ((1<<(PAGE_SHIFT-5))-1))
+
#define NKPTEPG (PAGE_SIZE / sizeof(struct ia64_lpte))
+vaddr_t kernel_vm_end;
-/* Values for ptc.e. XXX values for SKI. */
+/* Defaults for ptc.e. */
+/*
+static uint64_t pmap_ptc_e_base = 0;
+static uint32_t pmap_ptc_e_count1 = 1;
+static uint32_t pmap_ptc_e_count2 = 1;
+static uint32_t pmap_ptc_e_stride1 = 0;
+static uint32_t pmap_ptc_e_stride2 = 0;
+*/
+/* Values for ptc.e. XXX values for SKI, add SKI kernel option methinks */
static uint64_t pmap_ptc_e_base = 0x100000000;
static uint64_t pmap_ptc_e_count1 = 3;
static uint64_t pmap_ptc_e_count2 = 2;
static uint64_t pmap_ptc_e_stride1 = 0x2000;
static uint64_t pmap_ptc_e_stride2 = 0x100000000;
-kmutex_t pmap_ptc_lock; /* Global PTC lock */
-
-/* VHPT Base */
-
-vaddr_t vhpt_base;
-vaddr_t pmap_vhpt_log2size;
-
-struct ia64_bucket *pmap_vhpt_bucket;
-int pmap_vhpt_nbuckets;
-kmutex_t pmap_vhptlock; /* VHPT collision chain lock */
-
-int pmap_vhpt_inserts;
-int pmap_vhpt_resident;
-int pmap_vhpt_collisions;
-#ifdef DEBUG
-static void dump_vhpt(void);
-#endif
+kmutex_t pmap_ptc_mutex;
/*
* Data for the RID allocator
@@ -149,1080 +259,1280 @@ static int pmap_rididx;
static int pmap_ridmapsz;
static int pmap_ridmax;
static uint64_t *pmap_ridmap;
-kmutex_t pmap_rid_lock; /* RID allocator lock */
-
-
-bool pmap_initialized; /* Has pmap_init completed? */
-u_long pmap_pages_stolen; /* instrumentation */
+kmutex_t pmap_ridmutex;
-static struct pmap kernel_pmap_store; /* the kernel's pmap (proc0) */
-struct pmap *const kernel_pmap_ptr = &kernel_pmap_store;
+static krwlock_t pvh_global_lock __attribute__ ((aligned (128)));
-static vaddr_t kernel_vm_end; /* VA of last avail page ( end of kernel Address Space ) */
+static pool_cache_t pmap_pool_cache;
/*
- * This variable contains the number of CPU IDs we need to allocate
- * space for when allocating the pmap structure. It is used to
- * size a per-CPU array of ASN and ASN Generation number.
+ * Data for the pv entry allocation mechanism
*/
-u_long pmap_ncpuids;
-
-#ifndef PMAP_PV_LOWAT
-#define PMAP_PV_LOWAT 16
-#endif
-int pmap_pv_lowat = PMAP_PV_LOWAT;
+static TAILQ_HEAD(pch, pv_chunk) pv_chunks = TAILQ_HEAD_INITIALIZER(pv_chunks);
+static int pv_entry_count;
/*
- * PV table management functions.
+ * Data for allocating PTEs for user processes.
*/
-void *pmap_pv_page_alloc(struct pool *, int);
-void pmap_pv_page_free(struct pool *, void *);
+static pool_cache_t pte_pool_cache;
-struct pool_allocator pmap_pv_page_allocator = {
- pmap_pv_page_alloc, pmap_pv_page_free, 0,
-};
+struct ia64_bucket *pmap_vhpt_bucket;
-bool pmap_poolpage_alloc(paddr_t *);
-void pmap_poolpage_free(paddr_t);
+/* XXX For freebsd, these are sysctl variables */
+static uint64_t pmap_vhpt_nbuckets = 0;
+uint64_t pmap_vhpt_log2size = 0;
+static uint64_t pmap_vhpt_inserts = 0;
-/*
- * List of all pmaps, used to update them when e.g. additional kernel
- * page tables are allocated. This list is kept LRU-ordered by
- * pmap_activate(). XXX: Check on this.....
- */
-TAILQ_HEAD(, pmap) pmap_all_pmaps;
+static bool pmap_initialized = false; /* Has pmap_init completed? */
+static uint64_t pmap_pages_stolen = 0; /* instrumentation */
-/*
- * The pools from which pmap structures and sub-structures are allocated.
- */
-struct pool pmap_pmap_pool;
-struct pool pmap_ia64_lpte_pool;
-struct pool pmap_pv_pool;
+static struct ia64_lpte *pmap_find_vhpt(vaddr_t va);
-kmutex_t pmap_main_lock;
-kmutex_t pmap_all_pmaps_slock;
+static void free_pv_chunk(struct pv_chunk *pc);
+static void free_pv_entry(pmap_t pmap, pv_entry_t pv);
+static pv_entry_t get_pv_entry(pmap_t pmap, bool try);
+static struct vm_page *pmap_pv_reclaim(pmap_t locked_pmap);
-#if defined(MULTIPROCESSOR) || defined(LOCKDEBUG)
-/* XXX(kochi) need to use only spin lock? */
-#define PMAP_MAP_TO_HEAD_LOCK() \
- spinlockmgr(&pmap_main_lock, LK_SHARED, NULL)
-#define PMAP_MAP_TO_HEAD_UNLOCK() \
- spinlockmgr(&pmap_main_lock, LK_RELEASE, NULL)
-#define PMAP_HEAD_TO_MAP_LOCK() \
- spinlockmgr(&pmap_main_lock, LK_EXCLUSIVE, NULL)
-#define PMAP_HEAD_TO_MAP_UNLOCK() \
- spinlockmgr(&pmap_main_lock, LK_RELEASE, NULL)
-#else
-#define PMAP_MAP_TO_HEAD_LOCK() /* nothing */
-#define PMAP_MAP_TO_HEAD_UNLOCK() /* nothing */
-#define PMAP_HEAD_TO_MAP_LOCK() /* nothing */
-#define PMAP_HEAD_TO_MAP_UNLOCK() /* nothing */
-#endif /* MULTIPROCESSOR || LOCKDEBUG */
-
-
-#define pmap_accessed(lpte) ((lpte)->pte & PTE_ACCESSED)
-#define pmap_dirty(lpte) ((lpte)->pte & PTE_DIRTY)
-#define pmap_managed(lpte) ((lpte)->pte & PTE_MANAGED)
-#define pmap_ppn(lpte) ((lpte)->pte & PTE_PPN_MASK)
-#define pmap_present(lpte) ((lpte)->pte & PTE_PRESENT)
-#define pmap_prot(lpte) (((lpte)->pte & PTE_PROT_MASK) >> 56)
-#define pmap_wired(lpte) ((lpte)->pte & PTE_WIRED)
-
-#define pmap_clear_accessed(lpte) (lpte)->pte &= ~PTE_ACCESSED
-#define pmap_clear_dirty(lpte) (lpte)->pte &= ~PTE_DIRTY
-#define pmap_clear_present(lpte) (lpte)->pte &= ~PTE_PRESENT
-#define pmap_clear_wired(lpte) (lpte)->pte &= ~PTE_WIRED
+static void pmap_free_pte(struct ia64_lpte *pte, vaddr_t va);
+static int pmap_remove_pte(pmap_t pmap, struct ia64_lpte *pte,
+ vaddr_t va, pv_entry_t pv, int freepte);
+static int pmap_remove_vhpt(vaddr_t va);
-#define pmap_set_wired(lpte) (lpte)->pte |= PTE_WIRED
+static vaddr_t pmap_steal_vhpt_memory(vsize_t);
+static struct vm_page *vm_page_alloc1(void);
+static void vm_page_free1(struct vm_page *pg);
-/*
- * The VHPT bucket head structure.
- */
-struct ia64_bucket {
- uint64_t chain;
- kmutex_t lock;
- u_int length;
-};
+static vm_memattr_t pmap_flags_to_memattr(u_int flags);
+#if DEBUG
+static void pmap_testout(void);
+#endif
-/* Local Helper functions */
+static void
+pmap_initialize_vhpt(vaddr_t vhpt)
+{
+ struct ia64_lpte *pte;
+ u_int i;
-static void pmap_invalidate_all(pmap_t);
-static void pmap_invalidate_page(pmap_t, vaddr_t);
+ pte = (struct ia64_lpte *)vhpt;
+ for (i = 0; i < pmap_vhpt_nbuckets; i++) {
+ pte[i].pte = 0;
+ pte[i].itir = 0;
+ pte[i].tag = 1UL << 63; /* Invalid tag */
+ pte[i].chain = (uintptr_t)(pmap_vhpt_bucket + i);
+ }
+}
-static pmap_t pmap_switch(pmap_t pm);
-static pmap_t pmap_install(pmap_t);
+#ifdef MULTIPROCESSOR
+vaddr_t
+pmap_alloc_vhpt(void)
+{
+ vaddr_t vhpt;
+ struct vm_page *m;
+ vsize_t size;
-static struct ia64_lpte *pmap_find_kpte(vaddr_t);
+ size = 1UL << pmap_vhpt_log2size;
+ m = vm_page_alloc_contig(NULL, 0, VM_ALLOC_SYSTEM | VM_ALLOC_NOOBJ |
+ VM_ALLOC_WIRED, atop(size), 0UL, ~0UL, size, 0UL,
+ VM_MEMATTR_DEFAULT);
+ if (m != NULL) {
+ vhpt = IA64_PHYS_TO_RR7(VM_PAGE_TO_PHYS(m));
+ pmap_initialize_vhpt(vhpt);
+ return (vhpt);
+ }
+ return (0);
+}
+#endif
-static void
-pmap_set_pte(struct ia64_lpte *, vaddr_t, vaddr_t, bool, bool);
-static void
-pmap_free_pte(struct ia64_lpte *pte, vaddr_t va);
+/*
+ * Bootstrap the system enough to run with virtual memory.
+ */
+void
+pmap_bootstrap(void)
+{
+ struct ia64_pal_result res;
+ vaddr_t base;
+ size_t size;
+ vsize_t bufsz;
+ int i, ridbits;
-static __inline void
-pmap_pte_prot(pmap_t pm, struct ia64_lpte *pte, vm_prot_t prot);
-static int
-pmap_remove_pte(pmap_t pmap, struct ia64_lpte *pte, vaddr_t va,
- pv_entry_t pv, int freepte);
+ /*
+ * Query the PAL Code to find the loop parameters for the
+ * ptc.e instruction.
+ */
+ res = ia64_call_pal_static(PAL_PTCE_INFO, 0, 0, 0);
+ if (res.pal_status != 0)
+ panic("Can't configure ptc.e parameters");
+ pmap_ptc_e_base = res.pal_result[0];
+ pmap_ptc_e_count1 = res.pal_result[1] >> 32;
+ pmap_ptc_e_count2 = res.pal_result[1] & ((1L<<32) - 1);
+ pmap_ptc_e_stride1 = res.pal_result[2] >> 32;
+ pmap_ptc_e_stride2 = res.pal_result[2] & ((1L<<32) - 1);
+ if (bootverbose)
+ printf("ptc.e base=0x%lx, count1=%ld, count2=%ld, "
+ "stride1=0x%lx, stride2=0x%lx\n",
+ pmap_ptc_e_base,
+ pmap_ptc_e_count1,
+ pmap_ptc_e_count2,
+ pmap_ptc_e_stride1,
+ pmap_ptc_e_stride2);
-static struct ia64_lpte *
-pmap_find_pte(vaddr_t va);
-static int
-pmap_remove_entry(pmap_t pmap, struct vm_page * pg, vaddr_t va, pv_entry_t pv);
-static void
-pmap_insert_entry(pmap_t pmap, vaddr_t va, struct vm_page *pg);
+ mutex_init(&pmap_ptc_mutex, MUTEX_DEFAULT, IPL_VM);
-static void
-pmap_enter_vhpt(struct ia64_lpte *, vaddr_t);
-static int pmap_remove_vhpt(vaddr_t);
-static struct ia64_lpte *
-pmap_find_vhpt(vaddr_t);
-void
-pmap_page_purge(struct vm_page * pg);
-static void
-pmap_remove_page(pmap_t pmap, vaddr_t va);
+ /*
+ * Setup RIDs. RIDs 0..7 are reserved for the kernel.
+ *
+ * We currently need at least 19 bits in the RID because PID_MAX
+ * can only be encoded in 17 bits and we need RIDs for 4 regions
+ * per process. With PID_MAX equalling 99999 this means that we
+ * need to be able to encode 399996 (=4*PID_MAX).
+ * The Itanium processor only has 18 bits and the architected
+ * minimum is exactly that. So, we cannot use a PID based scheme
+ * in those cases. Enter pmap_ridmap...
+ * We should avoid the map when running on a processor that has
+ * implemented enough bits. This means that we should pass the
+ * process/thread ID to pmap. This we currently don't do, so we
+ * use the map anyway. However, we don't want to allocate a map
+ * that is large enough to cover the range dictated by the number
+ * of bits in the RID, because that may result in a RID map of
+ * 2MB in size for a 24-bit RID. A 64KB map is enough.
+ * The bottomline: we create a 32KB map when the processor only
+ * implements 18 bits (or when we can't figure it out). Otherwise
+ * we create a 64KB map.
+ */
+ res = ia64_call_pal_static(PAL_VM_SUMMARY, 0, 0, 0);
+ if (res.pal_status != 0) {
+ if (bootverbose)
+ printf("Can't read VM Summary - assuming 18 Region ID bits\n");
+ ridbits = 18; /* guaranteed minimum */
+ } else {
+ ridbits = (res.pal_result[1] >> 8) & 0xff;
+ if (bootverbose)
+ printf("Processor supports %d Region ID bits\n",
+ ridbits);
+ }
+ if (ridbits > 19)
+ ridbits = 19;
+ pmap_ridmax = (1 << ridbits);
+ pmap_ridmapsz = pmap_ridmax / 64;
+ pmap_ridmap = (uint64_t *)uvm_pageboot_alloc(pmap_ridmax / 8);
+ pmap_ridmap[0] |= 0xff;
+ pmap_rididx = 0;
+ pmap_ridcount = 8;
+ mutex_init(&pmap_ridmutex, MUTEX_DEFAULT, IPL_VM);
-static uint32_t pmap_allocate_rid(void);
-static void pmap_free_rid(uint32_t rid);
+ /*
+ * Compute the number of pages kmem_map will have.
+ */
+ kmeminit_nkmempages();
-static vaddr_t
-pmap_steal_vhpt_memory(vsize_t);
+ /*
+ * Figure out how many initial PTE's are necessary to map the
+ * kernel. We also reserve space for kmem_alloc_pageable()
+ * for vm_fork().
+ */
-/*
- * pmap_steal_memory: [ INTERFACE ]
- *
- * Bootstrap memory allocator (alternative to uvm_pageboot_alloc()).
- * This function allows for early dynamic memory allocation until the
- * virtual memory system has been bootstrapped. After that point, either
- * kmem_alloc or malloc should be used. This function works by stealing
- * pages from the (to be) managed page pool, then implicitly mapping the
- * pages (by using their RR7 addresses) and zeroing them.
- *
- * It may be used once the physical memory segments have been pre-loaded
- * into the vm_physmem[] array. Early memory allocation MUST use this
- * interface! This cannot be used after uvm_page_init(), and will
- * generate a panic if tried.
- *
- * Note that this memory will never be freed, and in essence it is wired
- * down.
- *
- * We must adjust *vstartp and/or *vendp iff we use address space
- * from the kernel virtual address range defined by pmap_virtual_space().
- *
- * Note: no locking is necessary in this function.
- */
-vaddr_t
-pmap_steal_memory(vsize_t size, vaddr_t *vstartp, vaddr_t *vendp)
-{
- int npgs;
- uvm_physseg_t upm;
- vaddr_t va;
- paddr_t pa;
+ /* Get size of buffer cache and set an upper limit */
+ bufsz = buf_memcalc();
+ buf_setvalimit(bufsz);
- size = round_page(size);
- npgs = atop(size);
+ nkpt = (((ubc_nwins << ubc_winshift) + uvm_emap_size +
+ bufsz + 16 * NCARGS + pager_map_size) / PAGE_SIZE +
+ USRIOSIZE + (maxproc * UPAGES) + nkmempages) / NKPTEPG;
- for (upm = uvm_physseg_get_first();
- uvm_physseg_valid_p(upm);
- upm = uvm_physseg_get_next(upm)) {
- if (uvm.page_init_done == true)
- panic("pmap_steal_memory: called _after_ bootstrap");
+ /*
+ * Allocate some memory for initial kernel 'page tables'.
+ */
+ ia64_kptdir = (void *)uvm_pageboot_alloc(PAGE_SIZE);
+ memset((void *)ia64_kptdir, 0, PAGE_SIZE);
+ nkpt = 0;
+ kernel_vm_end = VM_INIT_KERNEL_ADDRESS;
+
+ /*
+ * Determine a valid (mappable) VHPT size.
+ */
+ if (pmap_vhpt_log2size == 0)
+ pmap_vhpt_log2size = 20;
+ else if (pmap_vhpt_log2size < 16)
+ pmap_vhpt_log2size = 16;
+ else if (pmap_vhpt_log2size > 28)
+ pmap_vhpt_log2size = 28;
+ if (pmap_vhpt_log2size & 1)
+ pmap_vhpt_log2size--;
- if (uvm_physseg_get_avail_start(upm) != uvm_physseg_get_start(upm) ||
- uvm_physseg_get_avail_start(upm) >= uvm_physseg_get_avail_end(upm))
- continue;
+ size = 1UL << pmap_vhpt_log2size;
+ /* XXX add some retries here */
+ base = pmap_steal_vhpt_memory(size);
+
+ curcpu()->ci_vhpt = base;
- if ((uvm_physseg_get_avail_end(upm) - uvm_physseg_get_avail_start(upm))
- < npgs)
- continue;
+ if (base == 0)
+ panic("Unable to allocate VHPT");
- /*
- * There are enough pages here; steal them!
- */
- pa = ptoa(uvm_physseg_get_start(upm));
- uvm_physseg_unplug(atop(pa), npgs);
+ if (bootverbose)
+ printf("VHPT: address=%#lx, size=%#lx\n", base, size);
- va = IA64_PHYS_TO_RR7(pa);
- memset((void *)va, 0, size);
- pmap_pages_stolen += npgs;
- return va;
+ pmap_vhpt_nbuckets = size / sizeof(struct ia64_lpte);
+ pmap_vhpt_bucket = (void *)uvm_pageboot_alloc(pmap_vhpt_nbuckets *
+ sizeof(struct ia64_bucket));
+ for (i = 0; i < pmap_vhpt_nbuckets; i++) {
+ /* Stolen memory is zeroed. */
+ mutex_init(&pmap_vhpt_bucket[i].mutex, MUTEX_DEFAULT, IPL_VM);
}
+ pmap_initialize_vhpt(base);
+ map_vhpt(base);
+ ia64_set_pta(base + (1 << 8) + (pmap_vhpt_log2size << 2) + 1);
+ ia64_srlz_i();
+
+ /* XXX
+ virtual_avail = VM_INIT_KERNEL_ADDRESS;
+ virtual_end = VM_MAX_KERNEL_ADDRESS;
+ */
+
/*
- * If we got here, this was no memory left.
+ * Initialize the kernel pmap (which is statically allocated).
*/
- panic("pmap_steal_memory: no memory to steal");
+ PMAP_LOCK_INIT(kernel_pmap);
+ for (i = 0; i < IA64_VM_MINKERN_REGION; i++)
+ kernel_pmap->pm_rid[i] = 0;
+ TAILQ_INIT(&kernel_pmap->pm_pvchunk);
+
+ bzero(&kernel_pmap->pm_stats, sizeof kernel_pmap->pm_stats);
+ kernel_pmap->pm_refcount = 1;
+
+ curcpu()->ci_pmap = kernel_pmap;
+
+ /*
+ * Initialize the global pv list lock.
+ */
+ rw_init(&pvh_global_lock);
+
+ /* Region 5 is mapped via the VHPT. */
+ ia64_set_rr(IA64_RR_BASE(5), (5 << 8) | (PAGE_SHIFT << 2) | 1);
+
+ /*
+ * Clear out any random TLB entries left over from booting.
+ */
+ pmap_invalidate_all();
+
+ map_gateway_page();
}
-/*
- * pmap_steal_vhpt_memory: Derived from alpha/pmap.c:pmap_steal_memory()
- * Note: This function is not visible outside the pmap module.
- * Based on pmap_steal_memory();
- * Assumptions: size is always a power of 2.
- * Returns: Allocated memory at a naturally aligned address
- */
-static vaddr_t
-pmap_steal_vhpt_memory(vsize_t size)
+vaddr_t
+pmap_page_to_va(struct vm_page *m)
{
- int npgs;
- uvm_physseg_t upm;
+ paddr_t pa;
vaddr_t va;
- paddr_t pa = 0;
- paddr_t vhpt_start = 0, start1, start2, end1, end2;
- size = round_page(size);
- npgs = atop(size);
-
- for (upm = uvm_physseg_get_first();
- uvm_physseg_valid_p(upm);
- upm = uvm_physseg_get_next(upm)) {
- if (uvm.page_init_done == true)
- panic("pmap_vhpt_steal_memory: called _after_ bootstrap");
-
- if (uvm_physseg_get_avail_start(upm) != uvm_physseg_get_start(upm) || /* XXX: ??? */
- uvm_physseg_get_avail_start(upm) >= uvm_physseg_get_avail_end(upm))
- continue;
+ pa = VM_PAGE_TO_PHYS(m);
+ va = (m->mdpage.memattr == VM_MEMATTR_UNCACHEABLE) ? IA64_PHYS_TO_RR6(pa) :
+ IA64_PHYS_TO_RR7(pa);
+ return (va);
+}
- /* Break off a VHPT sized, aligned chunk off this segment. */
+/***************************************************
+ * Manipulate TLBs for a pmap
+ ***************************************************/
- start1 = uvm_physseg_get_avail_start(upm);
+static void
+pmap_invalidate_page(vaddr_t va)
+{
+ struct ia64_lpte *pte;
+ //struct pcpu *pc;
+ uint64_t tag;
+ u_int vhpt_ofs;
+ struct cpu_info *ci;
+ CPU_INFO_ITERATOR cii;
+
+ critical_enter();
- /* Align requested start address on requested size boundary */
- end1 = vhpt_start = roundup(start1, npgs);
+ vhpt_ofs = ia64_thash(va) - curcpu()->ci_vhpt;
+
+ tag = ia64_ttag(va);
- start2 = vhpt_start + npgs;
- end2 = uvm_physseg_get_avail_end(upm);
+ for (CPU_INFO_FOREACH(cii,ci)) {
+ pte = (struct ia64_lpte *)(ci->ci_vhpt + vhpt_ofs);
+ atomic_cmpset_64(&pte->tag, tag, 1UL << 63);
+ }
+
+ mutex_spin_enter(&pmap_ptc_mutex);
- /* Case 1: Doesn't fit. skip this segment */
+ ia64_ptc_ga(va, PAGE_SHIFT << 2);
+ ia64_mf();
+ ia64_srlz_i();
- if (start2 > end2) {
- vhpt_start = 0;
- continue;
- }
+ mutex_spin_exit(&pmap_ptc_mutex);
+
+ ia64_invala();
- /* For all cases of fit:
- * - Remove segment.
- * - Re-insert fragments via uvm_page_physload();
- */
+ critical_exit();
+}
- /*
- * We _fail_ on a vhpt request which exhausts memory.
- */
- if (start1 == end1 &&
- start2 == end2 &&
- uvm_physseg_get_first() == uvm_physseg_get_last() /* single segment */) {
-#ifdef DEBUG
- printf("pmap_vhpt_steal_memory: out of memory!");
-#endif
- return -1;
- }
+void
+pmap_invalidate_all(void)
+{
+ uint64_t addr;
+ int i, j;
- /* Remove this segment from the list. */
- if (uvm_physseg_unplug(uvm_physseg_get_start(upm),
- uvm_physseg_get_end(upm) - uvm_physseg_get_start(upm)) == false) {
- panic("%s: uvm_physseg_unplug(%"PRIxPADDR", %"PRIxPADDR") failed\n",
- __func__, uvm_physseg_get_start(upm),
- uvm_physseg_get_end(upm) - uvm_physseg_get_start(upm));
+ UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
+
+ addr = pmap_ptc_e_base;
+ for (i = 0; i < pmap_ptc_e_count1; i++) {
+ for (j = 0; j < pmap_ptc_e_count2; j++) {
+ ia64_ptc_e(addr);
+ addr += pmap_ptc_e_stride2;
}
-
- /* Case 2: Perfect fit - skip segment reload. */
-
- if (start1 == end1 && start2 == end2) break;
-
- /* Case 3: Left unfit - reload it.
- */
-
- if (start1 != end1)
- uvm_page_physload(start1, end1, start1, end1,
- VM_FREELIST_DEFAULT);
-
- /* Case 4: Right unfit - reload it. */
-
- if (start2 != end2)
- uvm_page_physload(start2, end2, start2, end2,
- VM_FREELIST_DEFAULT);
-
- /* Case 5: Both unfit - Redundant, isn't it ? */
- break;
- }
-
- /*
- * If we got here, we couldn't find a fit.
- */
- if (vhpt_start == 0) {
-#ifdef DEBUG
- printf("pmap_steal_vhpt_memory: no VHPT aligned fit found.");
-#endif
- return -1;
+ addr += pmap_ptc_e_stride1;
}
-
- /*
- * There are enough pages here; steal them!
- */
- pa = ptoa(vhpt_start);
- va = IA64_PHYS_TO_RR7(pa);
- memset((void *)va, 0, size);
- pmap_pages_stolen += npgs;
- return va;
+ ia64_srlz_i();
}
-/*
- * pmap_bootstrap:
- *
- * Bootstrap the system to run with virtual memory.
- *
- * Note: no locking is necessary in this function.
- */
-void
-pmap_bootstrap(void)
+static uint32_t
+pmap_allocate_rid(void)
{
- struct ia64_pal_result res;
- vaddr_t base, limit;
- size_t size;
- vsize_t bufsz;
-
- int i, ridbits;
+ uint64_t bit, bits;
+ int rid;
- /*
- * Query the PAL Code to find the loop parameters for the
- * ptc.e instruction.
- */
- res = ia64_call_pal_static(PAL_PTCE_INFO, 0, 0, 0);
- if (res.pal_status != 0)
- panic("Can't configure ptc.e parameters");
- pmap_ptc_e_base = res.pal_result[0];
- pmap_ptc_e_count1 = res.pal_result[1] >> 32;
- pmap_ptc_e_count2 = res.pal_result[1] & ((1L<<32) - 1);
- pmap_ptc_e_stride1 = res.pal_result[2] >> 32;
- pmap_ptc_e_stride2 = res.pal_result[2] & ((1L<<32) - 1);
- if (bootverbose)
- printf("ptc.e base=0x%lx, count1=%ld, count2=%ld, "
- "stride1=0x%lx, stride2=0x%lx\n",
- pmap_ptc_e_base,
- pmap_ptc_e_count1,
- pmap_ptc_e_count2,
- pmap_ptc_e_stride1,
- pmap_ptc_e_stride2);
- mutex_init(&pmap_ptc_lock, MUTEX_DEFAULT, IPL_VM);
+ mutex_enter(&pmap_ridmutex);
+
+ if (pmap_ridcount == pmap_ridmax)
+ panic("pmap_allocate_rid: All Region IDs used");
- /*
- * Setup RIDs. RIDs 0..7 are reserved for the kernel.
- *
- * We currently need at least 19 bits in the RID because PID_MAX
- * can only be encoded in 17 bits and we need RIDs for 5 regions
- * per process. With PID_MAX equalling 99999 this means that we
- * need to be able to encode 499995 (=5*PID_MAX).
- * The Itanium processor only has 18 bits and the architected
- * minimum is exactly that. So, we cannot use a PID based scheme
- * in those cases. Enter pmap_ridmap...
- * We should avoid the map when running on a processor that has
- * implemented enough bits. This means that we should pass the
- * process/thread ID to pmap. This we currently don't do, so we
- * use the map anyway. However, we don't want to allocate a map
- * that is large enough to cover the range dictated by the number
- * of bits in the RID, because that may result in a RID map of
- * 2MB in size for a 24-bit RID. A 64KB map is enough.
- * The bottomline: we create a 32KB map when the processor only
- * implements 18 bits (or when we can't figure it out). Otherwise
- * we create a 64KB map.
- */
- res = ia64_call_pal_static(PAL_VM_SUMMARY, 0, 0, 0);
- if (res.pal_status != 0) {
- if (bootverbose)
- printf("Can't read VM Summary - assuming 18 Region ID bits\n");
- ridbits = 18; /* guaranteed minimum */
- } else {
- ridbits = (res.pal_result[1] >> 8) & 0xff;
- if (bootverbose)
- printf("Processor supports %d Region ID bits\n",
- ridbits);
+ /* Find an index with a free bit. */
+ while ((bits = pmap_ridmap[pmap_rididx]) == ~0UL) {
+ pmap_rididx++;
+ if (pmap_rididx == pmap_ridmapsz)
+ pmap_rididx = 0;
}
- if (ridbits > 19)
- ridbits = 19;
+ rid = pmap_rididx * 64;
- pmap_ridmax = (1 << ridbits);
- pmap_ridmapsz = pmap_ridmax / 64;
- pmap_ridmap = (uint64_t *)uvm_pageboot_alloc(pmap_ridmax / 8);
- pmap_ridmap[0] |= 0xff;
- pmap_rididx = 0;
- pmap_ridcount = 8;
+ /* Find a free bit. */
+ bit = 1UL;
+ while (bits & bit) {
+ rid++;
+ bit <<= 1;
+ }
- /* XXX: The FreeBSD pmap.c defines initialises this like this:
- * mtx_init(&pmap_ridmutex, "RID allocator lock", NULL, MTX_DEF);
- * MTX_DEF can *sleep*.
- */
- mutex_init(&pmap_rid_lock, MUTEX_DEFAULT, IPL_VM);
+ pmap_ridmap[pmap_rididx] |= bit;
+ pmap_ridcount++;
- /*
- * Compute the number of pages kmem_map will have.
- */
- kmeminit_nkmempages();
+ mutex_exit(&pmap_ridmutex);
+
+ return rid;
+}
- /*
- * Figure out how many initial PTE's are necessary to map the
- * kernel. We also reserve space for kmem_alloc_pageable()
- * for vm_fork().
- */
+static void
+pmap_free_rid(uint32_t rid)
+{
+ uint64_t bit;
+ int idx;
- /* Get size of buffer cache and set an upper limit */
- bufsz = buf_memcalc();
- buf_setvalimit(bufsz);
+ idx = rid / 64;
+ bit = ~(1UL << (rid & 63));
- nkpt = (((ubc_nwins << ubc_winshift) + uvm_emap_size +
- bufsz + 16 * NCARGS + pager_map_size) / PAGE_SIZE +
- USRIOSIZE + (maxproc * UPAGES) + nkmempages) / NKPTEPG;
+ mutex_enter(&pmap_ridmutex);
+ pmap_ridmap[idx] &= bit;
+ pmap_ridcount--;
- /*
- * Allocate some memory for initial kernel 'page tables'.
- */
- ia64_kptdir = (void *)uvm_pageboot_alloc((nkpt + 1) * PAGE_SIZE);
- for (i = 0; i < nkpt; i++)
- ia64_kptdir[i] =
- (void*)((vaddr_t)ia64_kptdir + PAGE_SIZE * (i + 1));
+ mutex_exit(&pmap_ridmutex);
+}
- kernel_vm_end = nkpt * PAGE_SIZE * NKPTEPG + VM_MIN_KERNEL_ADDRESS -
- VM_GATEWAY_SIZE;
+/***************************************************
+ * Page table page management routines.....
+ ***************************************************/
+CTASSERT(sizeof(struct pv_chunk) == PAGE_SIZE);
- /*
- * Initialize the pmap pools and list.
- */
- pmap_ncpuids = pmap_ridmax;
- pool_init(&pmap_pmap_pool, sizeof(struct pmap), 0, 0, 0, "pmappl",
- &pool_allocator_nointr, IPL_NONE); /* This may block. */
+static __inline struct pv_chunk *
+pv_to_chunk(pv_entry_t pv)
+{
+ return ((struct pv_chunk *)((uintptr_t)pv & ~(uintptr_t)PAGE_MASK));
+}
- /* XXX: Need to convert ia64_kptdir[][] to a pool. ????*/
+#define PV_PMAP(pv) (pv_to_chunk(pv)->pc_pmap)
- /* The default pool allocator uses uvm_km_alloc & friends.
- * XXX: We should be using regular vm_alloced mem for regular,
- * non-kernel ptesl
- */
+#define PC_FREE_FULL 0xfffffffffffffffful
+#define PC_FREE_PARTIAL \
+ ((1UL << (_NPCPV - sizeof(u_long) * 8 * (_NPCM - 1))) - 1)
- pool_init(&pmap_ia64_lpte_pool, sizeof (struct ia64_lpte),
- sizeof(void *), 0, 0, "ptpl", NULL, IPL_NONE);
+#if PAGE_SIZE == 8192
+static const u_long pc_freemask[_NPCM] = {
+ PC_FREE_FULL, PC_FREE_FULL, PC_FREE_FULL,
+ PC_FREE_FULL, PC_FREE_FULL, PC_FREE_PARTIAL
+};
+#elif PAGE_SIZE == 16384
+static const u_long pc_freemask[_NPCM] = {
+ PC_FREE_FULL, PC_FREE_FULL, PC_FREE_FULL,
+ PC_FREE_FULL, PC_FREE_FULL, PC_FREE_FULL,
+ PC_FREE_FULL, PC_FREE_FULL, PC_FREE_FULL,
+ PC_FREE_FULL, PC_FREE_PARTIAL
+};
+#endif
- pool_init(&pmap_pv_pool, sizeof (struct pv_entry), sizeof(void *),
- 0, 0, "pvpl", &pmap_pv_page_allocator, IPL_NONE);
+#ifdef PV_STATS
+static int pc_chunk_count, pc_chunk_allocs, pc_chunk_frees, pc_chunk_tryfail;
+static long pv_entry_frees, pv_entry_allocs;
+static int pv_entry_spare;
+#endif
- TAILQ_INIT(&pmap_all_pmaps);
+/*
+ * We are in a serious low memory condition. Resort to
+ * drastic measures to free some pages so we can allocate
+ * another pv entry chunk.
+ */
+static struct vm_page
+*pmap_pv_reclaim(pmap_t locked_pmap)
+{
+ struct pch newtail;
+ struct pv_chunk *pc;
+ struct ia64_lpte *pte;
+ pmap_t pmap;
+ pv_entry_t pv;
+ vaddr_t va;
+ struct vm_page *m;
+ struct vm_page *m_pc;
+ u_long inuse;
+ int bit, field, freed, idx;
+
+ PMAP_LOCK_ASSERT(locked_pmap);
+ pmap = NULL;
+ m_pc = NULL;
+ TAILQ_INIT(&newtail);
+ while ((pc = TAILQ_FIRST(&pv_chunks)) != NULL) {
+ TAILQ_REMOVE(&pv_chunks, pc, pc_lru);
+ if (pmap != pc->pc_pmap) {
+ if (pmap != NULL) {
+ if (pmap != locked_pmap) {
+ pmap_switch(locked_pmap);
+ PMAP_UNLOCK(pmap);
+ }
+ }
+ pmap = pc->pc_pmap;
+ /* Avoid deadlock and lock recursion. */
+ if (pmap > locked_pmap)
+ PMAP_LOCK(pmap);
+ else if (pmap != locked_pmap && !PMAP_TRYLOCK(pmap)) {
+ pmap = NULL;
+ TAILQ_INSERT_TAIL(&newtail, pc, pc_lru);
+ continue;
+ }
+ pmap_switch(pmap);
+ }
- /*
- * Figure out a useful size for the VHPT, based on the size of
- * physical memory and try to locate a region which is large
- * enough to contain the VHPT (which must be a power of two in
- * size and aligned to a natural boundary).
- * We silently bump up the VHPT size to the minimum size if the
- * user has set the tunable too small. Likewise, the VHPT size
- * is silently capped to the maximum allowed.
- */
+ /*
+ * Destroy every non-wired, 8 KB page mapping in the chunk.
+ */
+ freed = 0;
+ for (field = 0; field < _NPCM; field++) {
+ for (inuse = ~pc->pc_map[field] & pc_freemask[field];
+ inuse != 0; inuse &= ~(1UL << bit)) {
+ bit = ffs64(inuse) - 1;
+ idx = field * sizeof(inuse) * NBBY + bit;
+ pv = &pc->pc_pventry[idx];
+ va = pv->pv_va;
+ pte = pmap_find_vhpt(va);
+ KASSERTMSG(pte != NULL, "pte");
+ if (pmap_wired(pte))
+ continue;
+ pmap_remove_vhpt(va);
+ pmap_invalidate_page(va);
+ m = PHYS_TO_VM_PAGE(pmap_ppn(pte));
+ if (pmap_dirty(pte))
+ vm_page_dirty(m);
+ pmap_free_pte(pte, va);
+ TAILQ_REMOVE(&m->mdpage.pv_list, pv, pv_list);
+ /* XXX
+ if (TAILQ_EMPTY(&m->mdpage.pv_list))
+ //vm_page_aflag_clear(m, PGA_WRITEABLE);
+ m->flags |= PG_RDONLY;
+ */
+ pc->pc_map[field] |= 1UL << bit;
+ freed++;
+ }
+ }
+ if (freed == 0) {
+ TAILQ_INSERT_TAIL(&newtail, pc, pc_lru);
+ continue;
+ }
+ /* Every freed mapping is for a 8 KB page. */
+ pmap->pm_stats.resident_count -= freed;
+ PV_STAT(pv_entry_frees += freed);
+ PV_STAT(pv_entry_spare += freed);
+ pv_entry_count -= freed;
+ TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
+ for (field = 0; field < _NPCM; field++)
+ if (pc->pc_map[field] != pc_freemask[field]) {
+ TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc,
+ pc_list);
+ TAILQ_INSERT_TAIL(&newtail, pc, pc_lru);
+
+ /*
+ * One freed pv entry in locked_pmap is
+ * sufficient.
+ */
+ if (pmap == locked_pmap)
+ goto out;
+ break;
+ }
+ if (field == _NPCM) {
+ PV_STAT(pv_entry_spare -= _NPCPV);
+ PV_STAT(pc_chunk_count--);
+ PV_STAT(pc_chunk_frees++);
+ /* Entire chunk is free; return it. */
+ m_pc = PHYS_TO_VM_PAGE(IA64_RR_MASK((vaddr_t)pc));
+ break;
+ }
+ }
+out:
+ TAILQ_CONCAT(&pv_chunks, &newtail, pc_lru);
+ if (pmap != NULL) {
+ if (pmap != locked_pmap) {
+ pmap_switch(locked_pmap);
+ PMAP_UNLOCK(pmap);
+ }
+ }
+ return (m_pc);
+}
- pmap_vhpt_log2size = PMAP_VHPT_LOG2SIZE;
+/*
+ * free the pv_entry back to the free list
+ */
+static void
+free_pv_entry(pmap_t pmap, pv_entry_t pv)
+{
+ struct pv_chunk *pc;
+ int bit, field, idx;
- if (pmap_vhpt_log2size == 0) {
- pmap_vhpt_log2size = 15;
- size = 1UL << pmap_vhpt_log2size;
- while (size < physmem * 32) {
- pmap_vhpt_log2size++;
- size <<= 1;
+ KASSERT(rw_write_held(&pvh_global_lock));
+ PMAP_LOCK_ASSERT(pmap);
+ PV_STAT(pv_entry_frees++);
+ PV_STAT(pv_entry_spare++);
+ pv_entry_count--;
+ pc = pv_to_chunk(pv);
+ idx = pv - &pc->pc_pventry[0];
+ field = idx / (sizeof(u_long) * NBBY);
+ bit = idx % (sizeof(u_long) * NBBY);
+ pc->pc_map[field] |= 1ul << bit;
+ for (idx = 0; idx < _NPCM; idx++)
+ if (pc->pc_map[idx] != pc_freemask[idx]) {
+ /*
+ * 98% of the time, pc is already at the head of the
+ * list. If it isn't already, move it to the head.
+ */
+ if (__predict_false(TAILQ_FIRST(&pmap->pm_pvchunk) !=
+ pc)) {
+ TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
+ TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc,
+ pc_list);
+ }
+ return;
}
- } else
- if (pmap_vhpt_log2size < 15)
- pmap_vhpt_log2size = 15;
+ TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
+ free_pv_chunk(pc);
+}
- if (pmap_vhpt_log2size > 61) pmap_vhpt_log2size = 61;
+static void
+free_pv_chunk(struct pv_chunk *pc)
+{
+ struct vm_page *m;
- vhpt_base = 0;
- base = limit = 0;
- size = 1UL << pmap_vhpt_log2size;
- while (vhpt_base == 0 && size) {
- if (bootverbose)
- printf("Trying VHPT size 0x%lx\n", size);
+ TAILQ_REMOVE(&pv_chunks, pc, pc_lru);
+ PV_STAT(pv_entry_spare -= _NPCPV);
+ PV_STAT(pc_chunk_count--);
+ PV_STAT(pc_chunk_frees++);
+ /* entire chunk is free, return it */
+ m = PHYS_TO_VM_PAGE(IA64_RR_MASK((vaddr_t)pc));
+#if 0
+ /* XXX freebsd these move pages around in queue */
+ vm_page_unwire(m, 0); // releases one wiring and moves page back active/inactive queue
+ vm_page_free(m); // moves page to "free" queue & disassociate with object
+
+ /* XXX might to need locks/other checks here, uvm_unwire, pmap_kremove... */
+ uvm_pagefree(m);
+#endif
+ vm_page_free1(m);
+}
- /* allocate size bytes aligned at size */
- /* #ifdef MULTIPROCESSOR, then (size * MAXCPU) bytes */
- base = pmap_steal_vhpt_memory(size);
-
- if (!base) {
- /* Can't fit, try next smaller size. */
- pmap_vhpt_log2size--;
- size >>= 1;
- } else
- vhpt_base = IA64_PHYS_TO_RR7(base);
- }
- if (pmap_vhpt_log2size < 15)
- panic("Can't find space for VHPT");
+/*
+ * get a new pv_entry, allocating a block from the system
+ * when needed.
+ */
+static pv_entry_t
+get_pv_entry(pmap_t pmap, bool try)
+{
+ struct pv_chunk *pc;
+ pv_entry_t pv;
+ struct vm_page *m;
+ int bit, field, idx;
- if (bootverbose)
- printf("Putting VHPT at 0x%lx\n", base);
+ KASSERT(rw_write_held(&pvh_global_lock));
+ PMAP_LOCK_ASSERT(pmap);
+ PV_STAT(pv_entry_allocs++);
+ pv_entry_count++;
+retry:
+ pc = TAILQ_FIRST(&pmap->pm_pvchunk);
+ if (pc != NULL) {
+ for (field = 0; field < _NPCM; field++) {
+ if (pc->pc_map[field]) {
+ bit = ffs64(pc->pc_map[field]) - 1;
+ break;
+ }
+ }
+ if (field < _NPCM) {
+ idx = field * sizeof(pc->pc_map[field]) * NBBY + bit;
+ pv = &pc->pc_pventry[idx];
+ pc->pc_map[field] &= ~(1ul << bit);
+ /* If this was the last item, move it to tail */
+ for (field = 0; field < _NPCM; field++)
+ if (pc->pc_map[field] != 0) {
+ PV_STAT(pv_entry_spare--);
+ return (pv); /* not full, return */
+ }
+ TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
+ TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);
+ PV_STAT(pv_entry_spare--);
+ return (pv);
+ }
+ }
- mutex_init(&pmap_vhptlock, MUTEX_DEFAULT, IPL_VM);
+ /* No free items, allocate another chunk */
+ m = vm_page_alloc1();
+
+ if (m == NULL) {
+ if (try) {
+ pv_entry_count--;
+ PV_STAT(pc_chunk_tryfail++);
+ return (NULL);
+ }
+ m = pmap_pv_reclaim(pmap);
+ if (m == NULL)
+ goto retry;
+ }
+
+ PV_STAT(pc_chunk_count++);
+ PV_STAT(pc_chunk_allocs++);
+ pc = (struct pv_chunk *)IA64_PHYS_TO_RR7(VM_PAGE_TO_PHYS(m));
+ pc->pc_pmap = pmap;
+ pc->pc_map[0] = pc_freemask[0] & ~1ul; /* preallocated bit 0 */
+ for (field = 1; field < _NPCM; field++)
+ pc->pc_map[field] = pc_freemask[field];
+ TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru);
+ pv = &pc->pc_pventry[0];
+ TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
+ PV_STAT(pv_entry_spare += _NPCPV - 1);
+ return (pv);
+}
- __asm __volatile("mov cr.pta=%0;; srlz.i;;" ::
- "r" (vhpt_base + (1<<8) + (pmap_vhpt_log2size<<2) + 1));
+/*
+ * Conditionally create a pv entry.
+ */
+#if 0
+static bool
+pmap_try_insert_pv_entry(pmap_t pmap, vaddr_t va, struct vm_page *m)
+{
+ pv_entry_t pv;
-#ifdef DEBUG
- dump_vhpt();
+ PMAP_LOCK_ASSERT(pmap);
+ KASSERT(rw_write_held(&pvh_global_lock));
+ if ((pv = get_pv_entry(pmap, true)) != NULL) {
+ pv->pv_va = va;
+ TAILQ_INSERT_TAIL(&m->mdpage.pv_list, pv, pv_list);
+ return (true);
+ } else
+ return (false);
+}
#endif
- /*
- * Initialise vhpt pte entries.
- */
-
- pmap_vhpt_nbuckets = size / sizeof(struct ia64_lpte);
-
- pmap_vhpt_bucket = (void *)uvm_pageboot_alloc(pmap_vhpt_nbuckets *
- sizeof(struct ia64_bucket));
+/*
+ * Add an ia64_lpte to the VHPT.
+ */
+static void
+pmap_enter_vhpt(struct ia64_lpte *pte, vaddr_t va)
+{
+ struct ia64_bucket *bckt;
+ struct ia64_lpte *vhpte;
+ uint64_t pte_pa;
- struct ia64_lpte *pte;
+ /* Can fault, so get it out of the way. */
+ pte_pa = ia64_tpa((vaddr_t)pte);
- pte = (struct ia64_lpte *)vhpt_base;
- for (i = 0; i < pmap_vhpt_nbuckets; i++) {
- pte[i].pte = 0;
- pte[i].itir = 0;
- pte[i].tag = 1UL << 63; /* Invalid tag */
- pte[i].chain = (uintptr_t)(pmap_vhpt_bucket + i);
- /* Stolen memory is zeroed! */
- mutex_init(&pmap_vhpt_bucket[i].lock, MUTEX_DEFAULT, IPL_VM);
- }
+ vhpte = (struct ia64_lpte *)ia64_thash(va);
+ bckt = (struct ia64_bucket *)vhpte->chain;
- /*
- * Initialize the locks.
- */
- mutex_init(&pmap_main_lock, MUTEX_DEFAULT, IPL_VM);
- mutex_init(&pmap_all_pmaps_slock, MUTEX_DEFAULT, IPL_VM);
+ mutex_spin_enter(&bckt->mutex);
+ pte->chain = bckt->chain;
+ ia64_mf();
+ bckt->chain = pte_pa;
- /*
- * Initialize the kernel pmap (which is statically allocated).
- */
- memset(pmap_kernel(), 0, sizeof(struct pmap));
+ pmap_vhpt_inserts++;
+ bckt->length++;
+ mutex_spin_exit(&bckt->mutex);
+}
- mutex_init(&pmap_kernel()->pm_slock, MUTEX_DEFAULT, IPL_VM);
- for (i = 0; i < 5; i++)
- pmap_kernel()->pm_rid[i] = 0;
- pmap_kernel()->pm_active = 1;
- TAILQ_INIT(&pmap_kernel()->pm_pvlist);
+/*
+ * Remove the ia64_lpte matching va from the VHPT. Return zero if it
+ * worked or an appropriate error code otherwise.
+ */
+static int
+pmap_remove_vhpt(vaddr_t va)
+{
+ struct ia64_bucket *bckt;
+ struct ia64_lpte *pte;
+ struct ia64_lpte *lpte;
+ struct ia64_lpte *vhpte;
+ uint64_t chain, tag;
- TAILQ_INSERT_TAIL(&pmap_all_pmaps, pmap_kernel(), pm_list);
+ tag = ia64_ttag(va);
+ vhpte = (struct ia64_lpte *)ia64_thash(va);
+ bckt = (struct ia64_bucket *)vhpte->chain;
- /*
- * Region 5 is mapped via the vhpt.
- */
- ia64_set_rr(IA64_RR_BASE(5), (5 << 8) | (PAGE_SHIFT << 2) | 1);
+ lpte = NULL;
+ mutex_spin_enter(&bckt->mutex);
+ chain = bckt->chain;
+ pte = (struct ia64_lpte *)IA64_PHYS_TO_RR7(chain);
+ while (chain != 0 && pte->tag != tag) {
+ lpte = pte;
+ chain = pte->chain;
+ pte = (struct ia64_lpte *)IA64_PHYS_TO_RR7(chain);
+ }
+ if (chain == 0) {
+ mutex_spin_exit(&bckt->mutex);
+ return (ENOENT);
+ }
- /*
- * Region 6 is direct mapped UC and region 7 is direct mapped
- * WC. The details of this is controlled by the Alt {I,D}TLB
- * handlers. Here we just make sure that they have the largest
- * possible page size to minimise TLB usage.
- */
- ia64_set_rr(IA64_RR_BASE(6), (6 << 8) | (IA64_ID_PAGE_SHIFT << 2));
- ia64_set_rr(IA64_RR_BASE(7), (7 << 8) | (IA64_ID_PAGE_SHIFT << 2));
- ia64_srlz_d();
+ /* Snip this pv_entry out of the collision chain. */
+ if (lpte == NULL)
+ bckt->chain = pte->chain;
+ else
+ lpte->chain = pte->chain;
+ ia64_mf();
- /*
- * Clear out any random TLB entries left over from booting.
- */
- pmap_invalidate_all(pmap_kernel());
+ bckt->length--;
- map_gateway_page();
+ mutex_spin_exit(&bckt->mutex);
+ return (0);
}
/*
- * pmap_init: [ INTERFACE ]
- *
- * Initialize the pmap module. Called by vm_init(), to initialize any
- * structures that the pmap system needs to map virtual memory.
- *
- * Note: no locking is necessary in this function.
+ * Find the ia64_lpte for the given va, if any.
*/
-void
-pmap_init(void)
+static struct ia64_lpte *
+pmap_find_vhpt(vaddr_t va)
{
+ struct ia64_bucket *bckt;
+ struct ia64_lpte *pte;
+ uint64_t chain, tag;
- /*
- * Set a low water mark on the pv_entry pool, so that we are
- * more likely to have these around even in extreme memory
- * starvation.
- */
- pool_setlowat(&pmap_pv_pool, pmap_pv_lowat);
+ tag = ia64_ttag(va);
+ pte = (struct ia64_lpte *)ia64_thash(va);
+ bckt = (struct ia64_bucket *)pte->chain;
- /*
- * Now it is safe to enable pv entry recording.
- */
- pmap_initialized = true;
+ mutex_spin_enter(&bckt->mutex);
+ chain = bckt->chain;
+ pte = (struct ia64_lpte *)IA64_PHYS_TO_RR7(chain);
+ while (chain != 0 && pte->tag != tag) {
+ chain = pte->chain;
+ pte = (struct ia64_lpte *)IA64_PHYS_TO_RR7(chain);
+ }
+ mutex_spin_exit(&bckt->mutex);
+ return ((chain != 0) ? pte : NULL);
}
/*
- * vtophys: virtual address to physical address. For use by
- * machine-dependent code only.
+ * Remove an entry from the list of managed mappings.
*/
-
-paddr_t
-vtophys(vaddr_t va)
+static int
+pmap_remove_entry(pmap_t pmap, struct vm_page *m, vaddr_t va, pv_entry_t pv)
{
- paddr_t pa;
- if (pmap_extract(pmap_kernel(), va, &pa) == true)
- return pa;
- return 0;
+ KASSERT(rw_write_held(&pvh_global_lock));
+ if (!pv) {
+ TAILQ_FOREACH(pv, &m->mdpage.pv_list, pv_list) {
+ if (pmap == PV_PMAP(pv) && va == pv->pv_va)
+ break;
+ }
+ }
+
+ if (pv) {
+ TAILQ_REMOVE(&m->mdpage.pv_list, pv, pv_list);
+ /* XXX
+ if (TAILQ_FIRST(&m->mdpage.pv_list) == NULL)
+ //vm_page_aflag_clear(m, PGA_WRITEABLE);
+ m->flags |= PG_RDONLY;
+ */
+ free_pv_entry(pmap, pv);
+ return 0;
+ } else {
+ return ENOENT;
+ }
}
/*
- * pmap_virtual_space: [ INTERFACE ]
- *
- * Define the initial bounds of the kernel virtual address space.
+ * Create a pv entry for page at pa for
+ * (pmap, va).
*/
-void
-pmap_virtual_space(vaddr_t *vstartp, vaddr_t *vendp)
+static void
+pmap_insert_entry(pmap_t pmap, vaddr_t va, struct vm_page *m)
{
+ pv_entry_t pv;
- *vstartp = VM_MIN_KERNEL_ADDRESS;
- *vendp = VM_MAX_KERNEL_ADDRESS;
+ KASSERT(rw_write_held(&pvh_global_lock));
+ pv = get_pv_entry(pmap, false);
+ pv->pv_va = va;
+ TAILQ_INSERT_TAIL(&m->mdpage.pv_list, pv, pv_list);
}
+/***************************************************
+ * Low level mapping routines.....
+ ***************************************************/
+
/*
- * pmap_remove_all: [ INTERFACE ]
- *
- * This function is a hint to the pmap implementation that all
- * entries in pmap will be removed before any more entries are
- * entered.
+ * Find the kernel lpte for mapping the given virtual address, which
+ * must be in the part of region 5 which we can cover with our kernel
+ * 'page tables'.
*/
-void
-pmap_remove_all(pmap_t pmap)
+static struct ia64_lpte *
+pmap_find_kpte(vaddr_t va)
{
- /* Nothing Yet */
+ struct ia64_lpte **dir1;
+ struct ia64_lpte *leaf;
+
+ UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
+ UVMHIST_LOG(maphist, "(va=%p)", va, 0, 0, 0);
+
+ KASSERTMSG((va >> 61) == 5, "kernel mapping 0x%lx not in region 5", va);
+
+ KASSERTMSG(va < kernel_vm_end, "kernel mapping 0x%lx out of range", va);
+
+ dir1 = ia64_kptdir[KPTE_DIR0_INDEX(va)];
+ leaf = dir1[KPTE_DIR1_INDEX(va)];
+
+ UVMHIST_LOG(maphist, "(kpte_dir0=%#lx, kpte_dir1=%#lx, kpte_pte=%#lx)",
+ KPTE_DIR0_INDEX(va), KPTE_DIR1_INDEX(va), KPTE_PTE_INDEX(va), 0);
+ UVMHIST_LOG(maphist, "(dir1=%p, leaf=%p ret=%p)",
+ dir1, leaf, &leaf[KPTE_PTE_INDEX(va)], 0);
+
+ return (&leaf[KPTE_PTE_INDEX(va)]);
}
/*
- * pmap_remove: [ INTERFACE ]
- *
- * Remove the given range of addresses from the specified map.
- *
- * It is assumed that the start and end are properly
- * rounded to the page size.
+ * Find a pte suitable for mapping a user-space address. If one exists
+ * in the VHPT, that one will be returned, otherwise a new pte is
+ * allocated.
*/
-void
-pmap_remove(pmap_t pmap, vaddr_t sva, vaddr_t eva)
+static struct ia64_lpte *
+pmap_find_pte(vaddr_t va)
{
- pmap_t oldpmap;
- vaddr_t va;
- pv_entry_t pv;
struct ia64_lpte *pte;
- if (pmap->pm_stats.resident_count == 0)
- return;
-
- PMAP_MAP_TO_HEAD_LOCK();
- PMAP_LOCK(pmap);
- oldpmap = pmap_install(pmap);
-
- /*
- * special handling of removing one page. a very
- * common operation and easy to short circuit some
- * code.
- */
- if (sva + PAGE_SIZE == eva) {
- pmap_remove_page(pmap, sva);
- goto out;
- }
-
- if (pmap->pm_stats.resident_count < ((eva - sva) >> PAGE_SHIFT)) {
- TAILQ_FOREACH(pv, &pmap->pm_pvlist, pv_plist) {
- va = pv->pv_va;
- if (va >= sva && va < eva) {
- pte = pmap_find_vhpt(va);
- KASSERT(pte != NULL);
- pmap_remove_pte(pmap, pte, va, pv, 1);
- pmap_invalidate_page(pmap, va);
- }
- }
+ if (va >= VM_MAXUSER_ADDRESS)
+ return pmap_find_kpte(va);
- } else {
- for (va = sva; va < eva; va += PAGE_SIZE) {
- pte = pmap_find_vhpt(va);
- if (pte != NULL) {
- pmap_remove_pte(pmap, pte, va, 0, 1);
- pmap_invalidate_page(pmap, va);
- }
+ pte = pmap_find_vhpt(va);
+ if (pte == NULL) {
+ pte = pool_cache_get(pte_pool_cache, PR_NOWAIT);
+ if (pte != NULL) {
+ memset((void *)pte, 0, sizeof(struct ia64_lpte));
+ pte->tag = 1UL << 63;
}
}
-out:
- pmap_install(oldpmap);
- PMAP_UNLOCK(pmap);
- PMAP_MAP_TO_HEAD_UNLOCK();
-
+ return (pte);
}
/*
- * pmap_zero_page: [ INTERFACE ]
- *
- * Zero the specified (machine independent) page by mapping the page
- * into virtual memory and clear its contents, one machine dependent
- * page at a time.
- *
- * Note: no locking is necessary in this function.
+ * Free a pte which is now unused. This simply returns it to the zone
+ * allocator if it is a user mapping. For kernel mappings, clear the
+ * valid bit to make it clear that the mapping is not currently used.
*/
-void
-pmap_zero_page(paddr_t phys)
+static void
+pmap_free_pte(struct ia64_lpte *pte, vaddr_t va)
{
- vaddr_t va = IA64_PHYS_TO_RR7(phys);
-
- memset((void *) va, 0, PAGE_SIZE);
+ if (va < VM_MAXUSER_ADDRESS)
+ pool_cache_put(pte_pool_cache, pte);
+ else
+ pmap_clear_present(pte);
}
-/*
- * pmap_copy_page: [ INTERFACE ]
- *
- * Copy the specified (machine independent) page by mapping the page
- * into virtual memory and using memcpy to copy the page, one machine
- * dependent page at a time.
- *
- * Note: no locking is necessary in this function.
- */
-void
-pmap_copy_page(paddr_t psrc, paddr_t pdst)
+static PMAP_INLINE void
+pmap_pte_prot(pmap_t pm, struct ia64_lpte *pte, vm_prot_t prot)
{
- vaddr_t vsrc = IA64_PHYS_TO_RR7(psrc);
- vaddr_t vdst = IA64_PHYS_TO_RR7(pdst);
+ static long prot2ar[4] = {
+ PTE_AR_R, /* VM_PROT_NONE */
+ PTE_AR_RW, /* VM_PROT_WRITE */
+ PTE_AR_RX|PTE_ED, /* VM_PROT_EXECUTE */
+ PTE_AR_RWX|PTE_ED /* VM_PROT_WRITE|VM_PROT_EXECUTE */
+ };
+
+ pte->pte &= ~(PTE_PROT_MASK | PTE_PL_MASK | PTE_AR_MASK | PTE_ED);
+ pte->pte |= (uint64_t)(prot & VM_PROT_ALL) << 56;
+ pte->pte |= (prot == VM_PROT_NONE || pm == kernel_pmap)
+ ? PTE_PL_KERN : PTE_PL_USER;
+ pte->pte |= prot2ar[(prot & VM_PROT_ALL) >> 1];
+}
- memcpy((void *) vdst, (void *) vsrc, PAGE_SIZE);
+static PMAP_INLINE void
+pmap_pte_attr(struct ia64_lpte *pte, vm_memattr_t ma)
+{
+ pte->pte &= ~PTE_MA_MASK;
+ pte->pte |= (ma & PTE_MA_MASK);
}
/*
- * pmap_unwire: [ INTERFACE ]
- *
- * Clear the wired attribute for a map/virtual-address pair.
- *
- * The mapping must already exist in the pmap.
+ * Set a pte to contain a valid mapping and enter it in the VHPT. If
+ * the pte was orginally valid, then its assumed to already be in the
+ * VHPT.
+ * This functions does not set the protection bits. It's expected
+ * that those have been set correctly prior to calling this function.
*/
-void
-pmap_unwire(pmap_t pmap, vaddr_t va)
+static void
+pmap_set_pte(struct ia64_lpte *pte, vaddr_t va, vaddr_t pa,
+ bool wired, bool managed)
{
- pmap_t oldpmap;
- struct ia64_lpte *pte;
+ pte->pte &= PTE_PROT_MASK | PTE_MA_MASK | PTE_PL_MASK |
+ PTE_AR_MASK | PTE_ED;
+ pte->pte |= PTE_PRESENT;
+ pte->pte |= (managed) ? PTE_MANAGED : (PTE_DIRTY | PTE_ACCESSED);
+ pte->pte |= (wired) ? PTE_WIRED : 0;
+ pte->pte |= pa & PTE_PPN_MASK;
- if (pmap == NULL)
- return;
+ pte->itir = PAGE_SHIFT << 2;
- PMAP_LOCK(pmap);
- oldpmap = pmap_install(pmap);
+ ia64_mf();
- pte = pmap_find_vhpt(va);
+ pte->tag = ia64_ttag(va);
+}
- KASSERT(pte != NULL);
+/*
+ * Remove the (possibly managed) mapping represented by pte from the
+ * given pmap.
+ */
+static int
+pmap_remove_pte(pmap_t pmap, struct ia64_lpte *pte, vaddr_t va,
+ pv_entry_t pv, int freepte)
+{
+ int error;
+ struct vm_page *m;
/*
- * If wiring actually changed (always?) clear the wire bit and
- * update the wire count. Note that wiring is not a hardware
- * characteristic so there is no need to invalidate the TLB.
+ * First remove from the VHPT.
*/
+ error = pmap_remove_vhpt(va);
+ KASSERTMSG(error == 0, "%s: pmap_remove_vhpt returned %d",__func__, error);
+
+ pmap_invalidate_page(va);
- if (pmap_wired(pte)) {
- pmap->pm_stats.wired_count--;
- pmap_clear_wired(pte);
- }
-#ifdef DIAGNOSTIC
- else {
- printf("pmap_unwire: wiring for pmap %p va 0x%lx "
- "didn't change!\n", pmap, va);
+ if (pmap_wired(pte))
+ pmap->pm_stats.wired_count -= 1;
+
+ pmap->pm_stats.resident_count -= 1;
+ if (pmap_managed(pte)) {
+ m = PHYS_TO_VM_PAGE(pmap_ppn(pte));
+ if (pmap_dirty(pte))
+ vm_page_dirty(m);
+
+ error = pmap_remove_entry(pmap, m, va, pv);
}
-#endif
- pmap_install(oldpmap);
- PMAP_UNLOCK(pmap);
+ if (freepte)
+ pmap_free_pte(pte, va);
+
+ return (error);
}
/*
- * pmap_kenter_pa: [ INTERFACE ]
+ * pmap_init: [ INTERFACE ]
*
- * Enter a va -> pa mapping into the kernel pmap without any
- * physical->virtual tracking.
+ * Initialize the pmap module. Called by vm_init(), to initialize any
+ * structures that the pmap system needs to map virtual memory.
*
* Note: no locking is necessary in this function.
*/
void
-pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags)
+pmap_init(void)
{
- struct ia64_lpte *pte;
+ UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
+
+ pmap_pool_cache = pool_cache_init(sizeof(struct pmap), 0, 0, 0,
+ "pmap_pool_cache", NULL, IPL_VM,
+ NULL, NULL, NULL);
+ if (pmap_pool_cache == NULL)
+ panic("%s cannot allocate pmap pool", __func__);
+
+ pte_pool_cache = pool_cache_init(sizeof(struct ia64_lpte), 0, 0, 0,
+ "pte_pool_cache", NULL, IPL_VM,
+ NULL, NULL, NULL);
+ if (pte_pool_cache == NULL)
+ panic("%s cannot allocate pte pool", __func__);
- pte = pmap_find_kpte(va);
- if (pmap_present(pte))
- pmap_invalidate_page(pmap_kernel(), va);
- else
- pmap_enter_vhpt(pte, va);
- pmap_pte_prot(pmap_kernel(), pte, prot);
- pmap_set_pte(pte, va, pa, false, false);
+
+ pmap_initialized = true;
+
+#if DEBUG
+ if (0) pmap_testout();
+#endif
}
/*
- * pmap_kremove: [ INTERFACE ]
+ * pmap_virtual_space: [ INTERFACE ]
*
- * Remove a mapping entered with pmap_kenter_pa() starting at va,
- * for size bytes (assumed to be page rounded).
+ * Define the initial bounds of the kernel virtual address space.
*/
void
-pmap_kremove(vaddr_t va, vsize_t size)
+pmap_virtual_space(vaddr_t *vstartp, vaddr_t *vendp)
{
- struct ia64_lpte *pte;
-
- while (size > 0) {
- pte = pmap_find_kpte(va);
- if (pmap_present(pte)) {
- pmap_remove_vhpt(va);
- pmap_invalidate_page(pmap_kernel(), va);
- pmap_clear_present(pte);
- }
- va += PAGE_SIZE;
- size -= PAGE_SIZE;
- }
+ *vstartp = VM_MIN_KERNEL_ADDRESS;
+ *vendp = VM_MAX_KERNEL_ADDRESS;
}
/*
- * pmap_create: [ INTERFACE ]
+ * pmap_steal_memory: [ INTERFACE ]
*
- * Create and return a physical map.
+ * Bootstrap memory allocator (alternative to uvm_pageboot_alloc()).
+ * This function allows for early dynamic memory allocation until the
+ * virtual memory system has been bootstrapped. After that point, either
+ * kmem_alloc or malloc should be used. This function works by stealing
+ * pages from the (to be) managed page pool, then implicitly mapping the
+ * pages (by using their RR7 addresses) and zeroing them.
+ *
+ * It may be used once the physical memory segments have been pre-loaded
+ * into the vm_physmem[] array. Early memory allocation MUST use this
+ * interface! This cannot be used after uvm_page_init(), and will
+ * generate a panic if tried.
+ *
+ * Note that this memory will never be freed, and in essence it is wired
+ * down.
+ *
+ * We must adjust *vstartp and/or *vendp iff we use address space
+ * from the kernel virtual address range defined by pmap_virtual_space().
*
* Note: no locking is necessary in this function.
*/
-pmap_t
-pmap_create(void)
+vaddr_t
+pmap_steal_memory(vsize_t size, vaddr_t *vstartp, vaddr_t *vendp)
{
- pmap_t pmap;
- int i;
+ int npgs;
+ uvm_physseg_t upm;
+ vaddr_t va;
+ paddr_t pa;
-#ifdef DEBUG
- printf("pmap_create()\n");
-#endif
+ size = round_page(size);
+ npgs = atop(size);
- pmap = pool_get(&pmap_pmap_pool, PR_WAITOK);
- memset(pmap, 0, sizeof(*pmap));
+ for (upm = uvm_physseg_get_first();
+ uvm_physseg_valid_p(upm);
+ upm = uvm_physseg_get_next(upm)) {
+ if (uvm.page_init_done == true)
+ panic("pmap_steal_memory: called _after_ bootstrap");
- for (i = 0; i < 5; i++)
- pmap->pm_rid[i] = pmap_allocate_rid();
- pmap->pm_active = 0;
- TAILQ_INIT(&pmap->pm_pvlist);
- memset(&pmap->pm_stats, 0, sizeof (pmap->pm_stats) );
-
- mutex_init(&pmap->pm_slock, MUTEX_DEFAULT, IPL_VM);
-
- mutex_enter(&pmap_all_pmaps_slock);
- TAILQ_INSERT_TAIL(&pmap_all_pmaps, pmap, pm_list);
- mutex_exit(&pmap_all_pmaps_slock);
+ if (uvm_physseg_get_avail_start(upm) != uvm_physseg_get_start(upm) ||
+ uvm_physseg_get_avail_start(upm) >= uvm_physseg_get_avail_end(upm))
+ continue;
- return pmap;
-}
+ if ((uvm_physseg_get_avail_end(upm) - uvm_physseg_get_avail_start(upm))
+ < npgs)
+ continue;
-/*
- * pmap_destroy: [ INTERFACE ]
- *
- * Drop the reference count on the specified pmap, releasing
- * all resources if the reference count drops to zero.
- */
-void
-pmap_destroy(pmap_t pmap)
-{
- int i;
+ /*
+ * There are enough pages here; steal them!
+ */
+ pa = ptoa(uvm_physseg_get_start(upm));
+ uvm_physseg_unplug(atop(pa), npgs);
-#ifdef DEBUG
- printf("pmap_destroy(%p)\n", pmap);
-#endif
+ va = IA64_PHYS_TO_RR7(pa);
+ memset((void *)va, 0, size);
+ pmap_pages_stolen += npgs;
+ return va;
+ }
- for (i = 0; i < 5; i++)
- if (pmap->pm_rid[i])
- pmap_free_rid(pmap->pm_rid[i]);
/*
- * Remove it from the global list of all pmaps.
+ * If we got here, this was no memory left.
*/
- mutex_enter(&pmap_all_pmaps_slock);
- TAILQ_REMOVE(&pmap_all_pmaps, pmap, pm_list);
- mutex_exit(&pmap_all_pmaps_slock);
-
- pool_put(&pmap_pmap_pool, pmap);
-
+ panic("pmap_steal_memory: no memory to steal");
}
/*
- * pmap_activate: [ INTERFACE ]
- *
- * Activate the pmap used by the specified process. This includes
- * reloading the MMU context if the current process, and marking
- * the pmap in use by the processor.
- *
- * Note: We may use only spin locks here, since we are called
- * by a critical section in cpu_switch()!
+ * pmap_steal_vhpt_memory: Derived from alpha/pmap.c:pmap_steal_memory()
+ * Note: This function is not visible outside the pmap module.
+ * Based on pmap_steal_memory();
+ * Assumptions: size is always a power of 2.
+ * Returns: Allocated memory at a naturally aligned address
*/
-void
-pmap_activate(struct lwp *l)
+static vaddr_t
+pmap_steal_vhpt_memory(vsize_t size)
{
+ int npgs;
+ uvm_physseg_t upm;
+ vaddr_t va;
+ paddr_t pa = 0;
+ paddr_t vhpt_start = 0, start1, start2, end1, end2;
- pmap_install(vm_map_pmap(&l->l_proc->p_vmspace->vm_map));
-}
+ size = round_page(size);
+ npgs = atop(size);
-/*
- * pmap_deactivate: [ INTERFACE ]
- *
- * Mark that the pmap used by the specified process is no longer
- * in use by the processor.
- *
- */
+ for (upm = uvm_physseg_get_first();
+ uvm_physseg_valid_p(upm);
+ upm = uvm_physseg_get_next(upm)) {
+ if (uvm.page_init_done == true)
+ panic("pmap_vhpt_steal_memory: called _after_ bootstrap");
-void
-pmap_deactivate(struct lwp *l)
-{
-}
+ if (uvm_physseg_get_avail_start(upm) != uvm_physseg_get_start(upm) || /* XXX: ??? */
+ uvm_physseg_get_avail_start(upm) >= uvm_physseg_get_avail_end(upm))
+ continue;
+
+ /* Break off a VHPT sized, aligned chunk off this segment. */
-/*
- * pmap_protect: [ INTERFACE ]
- *
- * Set the physical protection on the specified range of this map
- * as requested.
- */
-/*
- * Set the physical protection on the
- * specified range of this map as requested.
- */
-void
-pmap_protect(pmap_t pmap, vaddr_t sva, vaddr_t eva, vm_prot_t prot)
-{
- pmap_t oldpmap;
- struct ia64_lpte *pte;
+ start1 = uvm_physseg_get_avail_start(upm);
- if ((prot & VM_PROT_READ) == VM_PROT_NONE) {
- pmap_remove(pmap, sva, eva);
- return;
- }
+ /* Align requested start address on requested size boundary */
+ end1 = vhpt_start = roundup(start1, npgs);
+
+ start2 = vhpt_start + npgs;
+ end2 = uvm_physseg_get_avail_end(upm);
+
+ /* Case 1: Doesn't fit. skip this segment */
+
+ if (start2 > end2) {
+ vhpt_start = 0;
+ continue;
+ }
+
+ /* For all cases of fit:
+ * - Remove segment.
+ * - Re-insert fragments via uvm_page_physload();
+ */
+
+ /*
+ * We _fail_ on a vhpt request which exhausts memory.
+ */
+ if (start1 == end1 &&
+ start2 == end2 &&
+ uvm_physseg_get_first() == uvm_physseg_get_last() /* single segment */) {
+#ifdef DEBUG
+ printf("pmap_vhpt_steal_memory: out of memory!");
+#endif
+ return -1;
+ }
+
+ /* Remove this segment from the list. */
+ if (uvm_physseg_unplug(uvm_physseg_get_start(upm),
+ uvm_physseg_get_end(upm) - uvm_physseg_get_start(upm)) == false) {
+ panic("%s: uvm_physseg_unplug(%"PRIxPADDR", %"PRIxPADDR") failed\n",
+ __func__, uvm_physseg_get_start(upm),
+ uvm_physseg_get_end(upm) - uvm_physseg_get_start(upm));
+ }
- if (prot & VM_PROT_WRITE)
- return;
+ /* Case 2: Perfect fit - skip segment reload. */
- if ((sva & PAGE_MASK) || (eva & PAGE_MASK))
- panic("pmap_protect: unaligned addresses");
+ if (start1 == end1 && start2 == end2) break;
- PMAP_LOCK(pmap);
- oldpmap = pmap_install(pmap);
- while (sva < eva) {
- /*
- * If page is invalid, skip this page
+ /* Case 3: Left unfit - reload it.
*/
- pte = pmap_find_vhpt(sva);
- if (pte == NULL) {
- sva += PAGE_SIZE;
- continue;
- }
- if (pmap_prot(pte) != prot) {
- if (pmap_managed(pte)) {
- if (pmap_dirty(pte))
- pmap_clear_dirty(pte);
- if (pmap_accessed(pte)) {
- pmap_clear_accessed(pte);
- }
- }
- pmap_pte_prot(pmap, pte, prot);
- pmap_invalidate_page(pmap, sva);
- }
+ if (start1 != end1)
+ uvm_page_physload(start1, end1, start1, end1,
+ VM_FREELIST_DEFAULT);
- sva += PAGE_SIZE;
- }
- pmap_install(oldpmap);
- PMAP_UNLOCK(pmap);
-}
+ /* Case 4: Right unfit - reload it. */
-/*
- * pmap_extract: [ INTERFACE ]
- *
- * Extract the physical address associated with the given
- * pmap/virtual address pair.
- */
-bool
-pmap_extract(pmap_t pmap, vaddr_t va, paddr_t *pap)
-{
- struct ia64_lpte *pte;
- pmap_t oldpmap;
+ if (start2 != end2)
+ uvm_page_physload(start2, end2, start2, end2,
+ VM_FREELIST_DEFAULT);
- mutex_enter(&pmap->pm_slock);
- oldpmap = pmap_install(pmap); /*XXX: isn't this a little inefficient ?*/
- pte = pmap_find_vhpt(va);
- if (pte != NULL && pmap_present(pte)) {
- if (pap != NULL)
- *pap = pmap_ppn(pte);
- } else {
- mutex_exit(&pmap->pm_slock);
- return false;
+ /* Case 5: Both unfit - Redundant, isn't it ? */
+ break;
+ }
+
+ /*
+ * If we got here, we couldn't find a fit.
+ */
+ if (vhpt_start == 0) {
+#ifdef DEBUG
+ printf("pmap_steal_vhpt_memory: no VHPT aligned fit found.");
+#endif
+ return -1;
}
- pmap_install(oldpmap);
- mutex_exit(&pmap->pm_slock);
- return true;
+ /*
+ * There are enough pages here; steal them!
+ */
+ pa = ptoa(vhpt_start);
+ va = IA64_PHYS_TO_RR7(pa);
+ memset((void *)va, 0, size);
+ pmap_pages_stolen += npgs;
+ return va;
}
/*
- * pmap_clear_modify: [ INTERFACE ]
+ * pmap_create: [ INTERFACE ]
*
- * Clear the modify bits on the specified physical page.
+ * Create and return a physical map.
+ *
+ * Note: no locking is necessary in this function.
*/
-bool
-pmap_clear_modify(struct vm_page *pg)
+pmap_t
+pmap_create(void)
{
- struct vm_page_md * const md = VM_PAGE_TO_MD(pg);
- bool rv = false;
- struct ia64_lpte *pte;
- pmap_t oldpmap;
- pv_entry_t pv;
+ pmap_t pmap;
+ int i;
- TAILQ_FOREACH(pv, &md->pv_list, pv_list) {
- PMAP_LOCK(pv->pv_pmap);
- oldpmap = pmap_install(pv->pv_pmap);
- pte = pmap_find_vhpt(pv->pv_va);
- KASSERT(pte != NULL);
- if (pmap_dirty(pte)) {
- rv = true;
- pmap_clear_dirty(pte);
- pmap_invalidate_page(pv->pv_pmap, pv->pv_va);
- }
- pmap_install(oldpmap);
- PMAP_UNLOCK(pv->pv_pmap);
- }
- return rv;
+ pmap = pool_cache_get(pmap_pool_cache, PR_WAITOK);
+
+ if (pmap == NULL)
+ panic("%s no pool", __func__);
+
+ PMAP_LOCK_INIT(pmap);
+
+ for (i = 0; i < IA64_VM_MINKERN_REGION; i++)
+ pmap->pm_rid[i] = pmap_allocate_rid();
+
+ TAILQ_INIT(&pmap->pm_pvchunk);
+
+ pmap->pm_stats.resident_count = 0;
+ pmap->pm_stats.wired_count = 0;
+
+ pmap->pm_refcount = 1;
+
+ UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
+ UVMHIST_LOG(maphist, "(pm=%p)", pmap, 0, 0, 0);
+
+ return pmap;
}
/*
- * pmap_page_protect: [ INTERFACE ]
+ * pmap_destroy: [ INTERFACE ]
*
- * Lower the permission for all mappings to a given page to
- * the permissions specified.
+ * Drop the reference count on the specified pmap, releasing
+ * all resources if the reference count drops to zero.
*/
void
-pmap_page_protect(struct vm_page *pg, vm_prot_t prot)
+pmap_destroy(pmap_t pmap)
{
- struct vm_page_md * const md = VM_PAGE_TO_MD(pg);
- struct ia64_lpte *pte;
- pmap_t oldpmap, pmap;
- pv_entry_t pv;
+ int i;
- if ((prot & VM_PROT_WRITE) != 0)
+ UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
+ UVMHIST_LOG(maphist, "(pm=%p)", pmap, 0, 0, 0);
+
+ if (atomic_dec_64_nv(&pmap->pm_refcount) > 0)
return;
- if (prot & (VM_PROT_READ | VM_PROT_EXECUTE)) {
- if (pg->flags & PG_RDONLY)
- return;
- TAILQ_FOREACH(pv, &md->pv_list, pv_list) {
- pmap = pv->pv_pmap;
- PMAP_LOCK(pmap);
- oldpmap = pmap_install(pmap);
- pte = pmap_find_vhpt(pv->pv_va);
- KASSERT(pte != NULL);
- pmap_pte_prot(pmap, pte, prot);
- pmap_invalidate_page(pmap, pv->pv_va);
- pmap_install(oldpmap);
- PMAP_UNLOCK(pmap);
- }
- pg->flags |= PG_RDONLY;
- } else
- pmap_page_purge(pg);
+ KASSERT(pmap->pm_stats.resident_count == 0);
+ KASSERT(pmap->pm_stats.wired_count == 0);
+
+ KASSERT(TAILQ_EMPTY(&pmap->pm_pvchunk)); /* XXX hmmm */
+ KASSERT(!PMAP_LOCKED(pmap)); /* XXX hmmm */
+ /*PMAP_LOCK(pmap); */ /* XXX overkill */
+
+ for (i = 0; i < IA64_VM_MINKERN_REGION; i++)
+ if (pmap->pm_rid[i])
+ pmap_free_rid(pmap->pm_rid[i]);
+
+ /*PMAP_UNLOCK(pmap);*/ /* XXX hmm */
+ PMAP_LOCK_DESTROY(pmap);
+
+ pool_cache_put(pmap_pool_cache, pmap);
}
/*
@@ -1233,98 +1543,165 @@ pmap_page_protect(struct vm_page *pg, vm
void
pmap_reference(pmap_t pmap)
{
-
-#ifdef DEBUG
- printf("pmap_reference(%p)\n", pmap);
-#endif
-
- PMAP_LOCK(pmap);
- pmap->pm_count++;
- PMAP_UNLOCK(pmap);
+ atomic_inc_64(&pmap->pm_refcount);
}
/*
- * pmap_clear_reference: [ INTERFACE ]
+ * pmap_resident_count: [ INTERFACE ]
*
- * Clear the reference bit on the specified physical page.
+ * Query the ``resident pages'' statistic for pmap.
*/
-bool
-pmap_clear_reference(struct vm_page *pg)
+long
+pmap_resident_count(pmap_t pmap)
{
-
- return false;
+ return (pmap->pm_stats.resident_count);
}
/*
- * pmap_phys_address: [ INTERFACE ]
+ * pmap_wired_count: [ INTERFACE ]
*
- * Return the physical address corresponding to the specified
- * cookie. Used by the device pager to decode a device driver's
- * mmap entry point return value.
+ * Query the ``wired pages'' statistic for pmap.
*
- * Note: no locking is necessary in this function.
*/
-paddr_t
-pmap_phys_address(paddr_t ppn)
+long
+pmap_wired_count(pmap_t pmap)
{
+ return (pmap->pm_stats.wired_count);
+}
- return ia64_ptob(ppn);
+/*
+ * pmap_growkernel: [ INTERFACE ]
+ *
+ */
+vaddr_t
+pmap_growkernel(vaddr_t maxkvaddr)
+{
+ struct ia64_lpte **dir1;
+ struct ia64_lpte *leaf;
+ struct vm_page *pg;
+#if 0
+ struct vm_page *nkpg;
+ paddr_t pa;
+ vaddr_t va;
+#endif
+
+ /* XXX this function may still need serious fixin' */
+ UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
+ UVMHIST_LOG(maphist, "(va=%#lx, nkpt=%ld, kvm_end=%#lx before)", maxkvaddr, nkpt, kernel_vm_end, 0);
+
+ vaddr_t addr = maxkvaddr;
+
+ /* XXX use uvm_pageboot_alloc if not done? */
+ if (!uvm.page_init_done)
+ panic("uvm_page init not done");
+
+ while (kernel_vm_end <= addr) {
+ if (nkpt == PAGE_SIZE/8 + PAGE_SIZE*PAGE_SIZE/64)
+ panic("%s: out of kernel address space", __func__);
+ dir1 = ia64_kptdir[KPTE_DIR0_INDEX(kernel_vm_end)];
+
+ if (dir1 == NULL) {
+#if 0
+ /* FreeBSD does it this way... */
+ nkpg = vm_page_alloc(NULL, nkpt++, VM_ALLOC_NOOBJ|VM_ALLOC_INTERRUPT|VM_ALLOC_WIRED);
+ pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_USERESERVE|UVM_PGA_ZERO);
+#endif
+ pg = vm_page_alloc1();
+ if (!pg)
+ panic("%s: cannot add dir1 page", __func__);
+ nkpt++;
+
+#if 0
+ dir1 = (struct ia64_lpte **)pmap_page_to_va(nkpg);
+ bzero(dir1, PAGE_SIZE);
+#endif
+ dir1 = (struct ia64_lpte **)pmap_page_to_va(pg);
+
+ ia64_kptdir[KPTE_DIR0_INDEX(kernel_vm_end)] = dir1;
+ }
+
+#if 0
+ nkpg = vm_page_alloc(NULL, nkpt++, VM_ALLOC_NOOBJ|VM_ALLOC_INTERRUPT|VM_ALLOC_WIRED);
+ pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_USERESERVE|UVM_PGA_ZERO);
+#endif
+ pg = vm_page_alloc1();
+ if (!pg)
+ panic("%s: cannot add PTE page", __func__);
+ nkpt++;
+#if 0
+ leaf = (struct ia64_lpte *)pmap_page_to_va(nkpg);
+ bzero(leaf, PAGE_SIZE);
+#endif
+ leaf = (struct ia64_lpte *)pmap_page_to_va(pg);
+
+ dir1[KPTE_DIR1_INDEX(kernel_vm_end)] = leaf;
+
+ kernel_vm_end += PAGE_SIZE * NKPTEPG;
+ }
+
+ UVMHIST_LOG(maphist, "(va=%#lx, nkpt=%ld, kvm_end=%#lx after)", maxkvaddr, nkpt, kernel_vm_end, 0);
+
+ /* XXX fix */
+ return kernel_vm_end;
}
/*
* pmap_enter: [ INTERFACE ]
*
- * Insert the given physical page (p) at
- * the specified virtual address (v) in the
- * target physical map with the protection requested.
- *
- * If specified, the page will be wired down, meaning
- * that the related pte can not be reclaimed.
- *
- * Note: This is the only routine which MAY NOT lazy-evaluate
- * or lose information. That is, this routine must actually
- * insert this page into the given map NOW.
+ * Create a mapping in physical map pmap for the physical
+ * address pa at the virtual address va with protection speci-
+ * fied by bits in prot.
*/
int
pmap_enter(pmap_t pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags)
{
pmap_t oldpmap;
+ struct vm_page *m;
vaddr_t opa;
struct ia64_lpte origpte;
struct ia64_lpte *pte;
- bool managed, wired;
- struct vm_page *pg;
- int error = 0;
+ bool icache_inval, managed, wired, canfail;
+ /* vm_memattr_t ma; */
- PMAP_MAP_TO_HEAD_LOCK();
+ /* XXX this needs work */
+
+ UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
+ UVMHIST_LOG(maphist, "(pm=%p, va=%#lx, pa=%p, prot=%#x)", pmap, va, pa, prot);
+
+ /* wired = (flags & PMAP_ENTER_WIRED) != 0; */
+ wired = (flags & PMAP_WIRED) != 0;
+ canfail = (flags & PMAP_CANFAIL) != 0;
+
+ /* ma = pmap_flags_to_memattr(flags); */
+
+ rw_enter(&pvh_global_lock, RW_WRITER);
PMAP_LOCK(pmap);
- oldpmap = pmap_install(pmap);
+ oldpmap = pmap_switch(pmap);
va &= ~PAGE_MASK;
+ KASSERTMSG(va <= VM_MAX_KERNEL_ADDRESS, "%s: toobig", __func__);
- managed = false;
-
- wired = (flags & PMAP_WIRED) !=0;
-
- pg = PHYS_TO_VM_PAGE(pa);
-
-#ifdef DIAGNOSTIC
- if (va > VM_MAX_KERNEL_ADDRESS)
- panic("pmap_enter: toobig");
-#endif
+ /* XXX
+ if ((m->oflags & VPO_UNMANAGED) == 0 && !vm_page_xbusied(m))
+ VM_OBJECT_ASSERT_LOCKED(m->object);
+ */
/*
* Find (or create) a pte for the given mapping.
*/
- while ((pte = pmap_find_pte(va)) == NULL) {
- pmap_install(oldpmap);
+ pte = pmap_find_pte(va);
+
+ if (pte == NULL) {
+ pmap_switch(oldpmap);
PMAP_UNLOCK(pmap);
- PMAP_MAP_TO_HEAD_UNLOCK();
- uvm_kick_pdaemon();
- PMAP_MAP_TO_HEAD_LOCK();
- PMAP_LOCK(pmap);
- oldpmap = pmap_install(pmap);
+ rw_exit(&pvh_global_lock);
+
+ if (canfail)
+ return (ENOMEM);
+ else
+ panic("%s: no pte available", __func__);
}
+
origpte = *pte;
if (!pmap_present(pte)) {
opa = ~0UL;
@@ -1332,6 +1709,18 @@ pmap_enter(pmap_t pmap, vaddr_t va, padd
} else
opa = pmap_ppn(pte);
+ managed = false;
+ /* XXX hmm
+ pa = VM_PAGE_TO_PHYS(m);
+ */
+
+ m = PHYS_TO_VM_PAGE(pa);
+ if (m == NULL) {
+ /* implies page not managed? */
+ panic("%s: new page needed", __func__);
+ }
+ icache_inval = (prot & VM_PROT_EXECUTE) ? true : false;
+
/*
* Mapping has not changed, must be protection or wiring change.
*/
@@ -1349,15 +1738,18 @@ pmap_enter(pmap_t pmap, vaddr_t va, padd
managed = (pmap_managed(&origpte)) ? true : false;
-
/*
* We might be turning off write access to the page,
- * so we go ahead and sense modify status.
+ * so we go ahead and sense modify status. Otherwise,
+ * we can avoid I-cache invalidation if the page
+ * already allowed execution.
*/
if (managed && pmap_dirty(&origpte))
- pg->flags &= ~PG_CLEAN;
+ vm_page_dirty(m);
+ else if (pmap_exec(&origpte))
+ icache_inval = false;
- pmap_invalidate_page(pmap, va);
+ pmap_invalidate_page(va);
goto validate;
}
@@ -1373,9 +1765,12 @@ pmap_enter(pmap_t pmap, vaddr_t va, padd
/*
* Enter on the PV list if part of our managed memory.
*/
-
- if (pg != NULL) {
- pmap_insert_entry(pmap, va, pg);
+ if (vm_page_is_managed(m)) {
+#if 0
+ KASSERTMSG(va < kmi.clean_sva || va >= kmi.clean_eva,
+ ("pmap_enter: managed mapping within the clean submap"));
+#endif
+ pmap_insert_entry(pmap, va, m);
managed = true;
}
@@ -1393,672 +1788,1128 @@ validate:
* adds the pte to the VHPT if necessary.
*/
pmap_pte_prot(pmap, pte, prot);
+ pmap_pte_attr(pte, m->mdpage.memattr);
pmap_set_pte(pte, va, pa, wired, managed);
- PMAP_MAP_TO_HEAD_UNLOCK();
- pmap_install(oldpmap);
+ /* Invalidate the I-cache when needed. */
+ if (icache_inval)
+ ia64_sync_icache(va, PAGE_SIZE);
+
+ /* XXX
+ if ((prot & VM_PROT_WRITE) != 0 && managed)
+ //vm_page_aflag_set(m, PGA_WRITEABLE);
+ m->flags &= ~PG_RDONLY;
+ */
+ rw_exit(&pvh_global_lock);
+ pmap_switch(oldpmap);
PMAP_UNLOCK(pmap);
-
- return error; /* XXX: Look into this. */
+ return (0);
}
/*
- * Routine: pmap_page_purge
- * Function:
- * Removes this physical page from
- * all physical maps in which it resides.
- * Reflects back modify bits to the pager.
+ * pmap_remove: [ INTERFACE ]
*
- * Notes:
- * Original versions of this routine were very
- * inefficient because they iteratively called
- * pmap_remove (slow...)
+ * Remove the given range of addresses from the specified map.
+ *
+ * It is assumed that the start and end are properly
+ * rounded to the page size.
+ *
+ * Sparsely used ranges are inefficiently removed. The VHPT is
+ * probed for every page within the range. XXX
*/
-
void
-pmap_page_purge(struct vm_page *pg)
+pmap_remove(pmap_t pmap, vaddr_t sva, vaddr_t eva)
{
- struct vm_page_md * const md = VM_PAGE_TO_MD(pg);
pmap_t oldpmap;
- pv_entry_t pv;
+ vaddr_t va;
+ struct ia64_lpte *pte;
- while ((pv = TAILQ_FIRST(&md->pv_list)) != NULL) {
- struct ia64_lpte *pte;
- pmap_t pmap = pv->pv_pmap;
- vaddr_t va = pv->pv_va;
+ UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
+ UVMHIST_LOG(maphist, "(pm=%p, sva=%#lx, eva=%#lx)", pmap, sva, eva, 0);
- PMAP_LOCK(pmap);
- oldpmap = pmap_install(pmap);
+ /*
+ * Perform an unsynchronized read. This is, however, safe.
+ */
+ if (pmap->pm_stats.resident_count == 0)
+ return;
+
+ rw_enter(&pvh_global_lock, RW_WRITER);
+ PMAP_LOCK(pmap);
+ oldpmap = pmap_switch(pmap);
+ for (va = sva; va < eva; va += PAGE_SIZE) {
pte = pmap_find_vhpt(va);
- KASSERT(pte != NULL);
- if (pmap_ppn(pte) != VM_PAGE_TO_PHYS(pg))
- panic("pmap_remove_all:"
- "pv_table for %lx is inconsistent",
- VM_PAGE_TO_PHYS(pg));
- pmap_remove_pte(pmap, pte, va, pv, 1);
- pmap_install(oldpmap);
- PMAP_UNLOCK(pmap);
+ if (pte != NULL)
+ pmap_remove_pte(pmap, pte, va, 0, 1);
}
- pg->flags |= PG_RDONLY;
-
+ rw_exit(&pvh_global_lock);
+ pmap_switch(oldpmap);
+ PMAP_UNLOCK(pmap);
}
-pmap_t
-pmap_switch(pmap_t pm)
+/*
+ * pmap_remove_all: [ INTERFACE ]
+ *
+ * This function is a hint to the pmap implementation that all
+ * entries in pmap will be removed before any more entries are
+ * entered.
+ */
+void
+pmap_remove_all(pmap_t pmap)
{
- pmap_t prevpm;
- int i;
-
- prevpm = curcpu()->ci_pmap;
- if (prevpm == pm)
- return prevpm;
- if (pm == NULL)
- for (i = 0; i < 5; i++)
- ia64_set_rr(IA64_RR_BASE(i),
- (i << 8)|(PAGE_SHIFT << 2)|1);
- else
- for (i = 0; i < 5; i++)
- ia64_set_rr(IA64_RR_BASE(i),
- (pm->pm_rid[i] << 8)|(PAGE_SHIFT << 2)|1);
- curcpu()->ci_pmap = pm;
- ia64_srlz_d();
- return prevpm;
+ /* XXX do nothing */
}
-static pmap_t
-pmap_install(pmap_t pm)
+/*
+ * pmap_protect: [ INTERFACE ]
+ *
+ * Set the physical protection on the specified range of this map
+ * as requested.
+ */
+void
+pmap_protect(pmap_t pmap, vaddr_t sva, vaddr_t eva, vm_prot_t prot)
{
- pmap_t prevpm;
- int splsched;
+ pmap_t oldpmap;
+ struct ia64_lpte *pte;
+
+ UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
+ UVMHIST_LOG(maphist, "(pm=%p, sva=%#lx, eva=%#lx, prot=%#x)",
+ pmap, sva, eva, prot);
+ UVMHIST_LOG(maphist, "(VM_PROT_READ=%u, VM_PROT_WRITE=%u, VM_PROT_EXECUTE=%u)",
+ (prot & VM_PROT_READ) != 0, (prot & VM_PROT_WRITE) != 0,
+ (prot & VM_PROT_EXECUTE) != 0, 0);
+
+ if ((prot & VM_PROT_READ) == VM_PROT_NONE) {
+ pmap_remove(pmap, sva, eva);
+ return;
+ }
+
+ if ((prot & (VM_PROT_WRITE|VM_PROT_EXECUTE)) ==
+ (VM_PROT_WRITE|VM_PROT_EXECUTE))
+ return;
+
+ if ((sva & PAGE_MASK) || (eva & PAGE_MASK))
+ panic("pmap_protect: unaligned addresses");
+ sva = trunc_page(sva);
+ eva = round_page(eva) - 1;
+
+ PMAP_LOCK(pmap);
+ oldpmap = pmap_switch(pmap);
+ for ( ; sva < eva; sva += PAGE_SIZE) {
+ /* If page is invalid, skip this page */
+ pte = pmap_find_vhpt(sva);
+ if (pte == NULL)
+ continue;
+
+ /* If there's no change, skip it too */
+ if (pmap_prot(pte) == prot)
+ continue;
+
+ /* wired pages unaffected by prot changes */
+ if (pmap_wired(pte))
+ continue;
+
+ if ((prot & VM_PROT_WRITE) == 0 &&
+ pmap_managed(pte) && pmap_dirty(pte)) {
+ paddr_t pa = pmap_ppn(pte);
+ struct vm_page *m = PHYS_TO_VM_PAGE(pa);
+
+ vm_page_dirty(m);
+ pmap_clear_dirty(pte);
+ }
+
+ if (prot & VM_PROT_EXECUTE)
+ ia64_sync_icache(sva, PAGE_SIZE);
- splsched = splsched();
- prevpm = pmap_switch(pm);
- splx(splsched);
- return prevpm;
+ pmap_pte_prot(pmap, pte, prot);
+ pmap_invalidate_page(sva);
+ }
+ pmap_switch(oldpmap);
+ PMAP_UNLOCK(pmap);
}
-static uint32_t
-pmap_allocate_rid(void)
+/*
+ * pmap_unwire: [ INTERFACE ]
+ *
+ * Clear the wired attribute for a map/virtual-address pair.
+ *
+ * The wired attribute of the page table entry is not a hardware feature,
+ * so there is no need to invalidate any TLB entries.
+ *
+ * The mapping must already exist in the pmap.
+ */
+void
+pmap_unwire(pmap_t pmap, vaddr_t va)
{
- uint64_t bit, bits;
- int rid;
-
- mutex_enter(&pmap_rid_lock);
- if (pmap_ridcount == pmap_ridmax)
- panic("pmap_allocate_rid: All Region IDs used");
+ pmap_t oldpmap;
+ struct ia64_lpte *pte;
- /* Find an index with a free bit. */
- while ((bits = pmap_ridmap[pmap_rididx]) == ~0UL) {
- pmap_rididx++;
- if (pmap_rididx == pmap_ridmapsz)
- pmap_rididx = 0;
- }
- rid = pmap_rididx * 64;
+ UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
+ UVMHIST_LOG(maphist, "(pm=%p, va=%#x)", pmap, va, 0, 0);
- /* Find a free bit. */
- bit = 1UL;
- while (bits & bit) {
- rid++;
- bit <<= 1;
- }
+ PMAP_LOCK(pmap);
+ oldpmap = pmap_switch(pmap);
- pmap_ridmap[pmap_rididx] |= bit;
- pmap_ridcount++;
- mutex_exit(&pmap_rid_lock);
+ pte = pmap_find_vhpt(va);
- return rid;
+ /* XXX panic if no pte or not wired? */
+ if (pte == NULL)
+ panic("pmap_unwire: %lx not found in vhpt", va);
+
+ if (!pmap_wired(pte))
+ panic("pmap_unwire: pte %p isn't wired", pte);
+
+ pmap->pm_stats.wired_count--;
+ pmap_clear_wired(pte);
+
+ pmap_switch(oldpmap);
+ PMAP_UNLOCK(pmap);
}
-static void
-pmap_free_rid(uint32_t rid)
+/*
+ * pmap_extract: [ INTERFACE ]
+ *
+ * Extract the physical address associated with the given
+ * pmap/virtual address pair.
+ */
+bool
+pmap_extract(pmap_t pmap, vaddr_t va, paddr_t *pap)
{
- uint64_t bit;
- int idx;
+ struct ia64_lpte *pte;
+ pmap_t oldpmap;
+ paddr_t pa;
- idx = rid / 64;
- bit = ~(1UL << (rid & 63));
+ UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
+ UVMHIST_LOG(maphist, "(pm=%p, va=%#lx, pap=%#lx)", pmap, va, pap, 0);
- mutex_enter(&pmap_rid_lock);
- pmap_ridmap[idx] &= bit;
- pmap_ridcount--;
- mutex_exit(&pmap_rid_lock);
-}
+ pa = 0;
-/***************************************************
- * Manipulate TLBs for a pmap
- ***************************************************/
+ PMAP_LOCK(pmap);
+ oldpmap = pmap_switch(pmap);
+ pte = pmap_find_vhpt(va);
+ if (pte != NULL && pmap_present(pte))
+ pa = pmap_ppn(pte);
+ pmap_switch(oldpmap);
+ PMAP_UNLOCK(pmap);
-static void
-pmap_invalidate_page(pmap_t pmap, vaddr_t va)
-{
+ if (pa && (pap != NULL))
+ *pap = pa;
- KASSERT((pmap == pmap_kernel() || pmap == curcpu()->ci_pmap));
- ia64_ptc_g(va, PAGE_SHIFT << 2);
+ UVMHIST_LOG(maphist, "(pa=%#lx)", pa, 0, 0, 0);
+
+ return (pa != 0);
}
-
-static void
-pmap_invalidate_all_1(void *arg)
+/*
+ * Extract the physical page address associated with a kernel
+ * virtual address.
+ */
+paddr_t
+pmap_kextract(vaddr_t va)
{
- uint64_t addr;
- int i, j;
- register_t psr;
+ struct ia64_lpte *pte;
+ /*uint64_t *pbvm_pgtbl;*/
+ paddr_t pa;
+ /*u_int idx;*/
- psr = intr_disable();
- addr = pmap_ptc_e_base;
- for (i = 0; i < pmap_ptc_e_count1; i++) {
- for (j = 0; j < pmap_ptc_e_count2; j++) {
- ia64_ptc_e(addr);
- addr += pmap_ptc_e_stride2;
- }
- addr += pmap_ptc_e_stride1;
+ UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
+ UVMHIST_LOG(maphist, "(va=%#lx)", va, 0, 0, 0);
+
+ KASSERTMSG(va >= VM_MAXUSER_ADDRESS, "Must be kernel VA");
+
+ /* Regions 6 and 7 are direct mapped. */
+ if (va >= IA64_RR_BASE(6)) {
+ pa = IA64_RR_MASK(va);
+ goto out;
}
- intr_restore(psr);
-}
-
-static void
-pmap_invalidate_all(pmap_t pmap)
-{
- KASSERT(pmap == pmap_kernel() || pmap == curcpu()->ci_pmap);
+ /* Region 5 is our KVA. Bail out if the VA is beyond our limits. */
+ if (va >= kernel_vm_end)
+ goto err_out;
+ if (va >= VM_INIT_KERNEL_ADDRESS) {
+ pte = pmap_find_kpte(va);
+ pa = pmap_present(pte) ? pmap_ppn(pte) | (va & PAGE_MASK) : 0;
+ goto out;
+ }
+#if 0 /* XXX fix */
+ /* The PBVM page table. */
+ if (va >= IA64_PBVM_PGTBL + bootinfo->bi_pbvm_pgtblsz)
+ goto err_out;
+ if (va >= IA64_PBVM_PGTBL) {
+ pa = (va - IA64_PBVM_PGTBL) + bootinfo->bi_pbvm_pgtbl;
+ goto out;
+ }
-#ifdef MULTIPROCESSOR
- smp_rendezvous(0, pmap_invalidate_all_1, 0, 0);
-#else
- pmap_invalidate_all_1(0);
+ /* The PBVM itself. */
+ if (va >= IA64_PBVM_BASE) {
+ pbvm_pgtbl = (void *)IA64_PBVM_PGTBL;
+ idx = (va - IA64_PBVM_BASE) >> IA64_PBVM_PAGE_SHIFT;
+ if (idx >= (bootinfo->bi_pbvm_pgtblsz >> 3))
+ goto err_out;
+ if ((pbvm_pgtbl[idx] & PTE_PRESENT) == 0)
+ goto err_out;
+ pa = (pbvm_pgtbl[idx] & PTE_PPN_MASK) +
+ (va & IA64_PBVM_PAGE_MASK);
+ goto out;
+ }
#endif
-}
-/***************************************************
- * Low level mapping routines.....
- ***************************************************/
+ err_out:
+ KASSERT(1);
+ printf("XXX: %s: va=%#lx is invalid\n", __func__, va);
+ pa = 0;
+ /* FALLTHROUGH */
+
+ out:
+ UVMHIST_LOG(maphist, "(pa=%#lx)", pa, 0, 0, 0);
+ return (pa);
+}
/*
- * Find the kernel lpte for mapping the given virtual address, which
- * must be in the part of region 5 which we can cover with our kernel
- * 'page tables'.
+ * pmap_kenter_pa: [ INTERFACE ]
+ *
+ * Enter a va -> pa mapping into the kernel pmap without any
+ * physical->virtual tracking.
+ *
+ * Note: no locking is necessary in this function.
*/
-static struct ia64_lpte *
-pmap_find_kpte(vaddr_t va)
+void
+pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags)
{
+ struct ia64_lpte *pte;
+ vm_memattr_t attr;
+ const bool managed = false; /* don't gather ref/mod info */
+ const bool wired = true; /* pmap_kenter_pa always wired */
+
+ UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
+ UVMHIST_LOG(maphist, "(va=%#lx, pa=%#lx, prot=%p, flags=%p)", va, pa, prot, flags);
+
+ KASSERT(va >= VM_MIN_KERNEL_ADDRESS && va < VM_MAX_KERNEL_ADDRESS);
+
+ attr = pmap_flags_to_memattr(flags);
- KASSERT((va >> 61) == 5);
- KASSERT(IA64_RR_MASK(va) < (nkpt * PAGE_SIZE * NKPTEPG));
- return &ia64_kptdir[KPTE_DIR_INDEX(va)][KPTE_PTE_INDEX(va)];
-}
-
+ pte = pmap_find_kpte(va);
+ if (pmap_present(pte))
+ pmap_invalidate_page(va);
+ else
+ pmap_enter_vhpt(pte, va);
-/***************************************************
- * Low level helper routines.....
- ***************************************************/
+ pmap_pte_prot(kernel_pmap, pte, VM_PROT_ALL);
+ pmap_pte_attr(pte, attr);
+ pmap_set_pte(pte, va, pa, wired, managed);
+}
/*
- * Find a pte suitable for mapping a user-space address. If one exists
- * in the VHPT, that one will be returned, otherwise a new pte is
- * allocated.
+ * pmap_kremove: [ INTERFACE ]
+ *
+ * Remove all mappings starting at virtual address va for size
+ * bytes from the kernel physical map. All mappings that are
+ * removed must be the ``unmanaged'' type created with
+ * pmap_kenter_pa(). The implementation may assert this.
*/
-static struct ia64_lpte *
-pmap_find_pte(vaddr_t va)
+void
+pmap_kremove(vaddr_t va, vsize_t size)
{
struct ia64_lpte *pte;
- if (va >= VM_MAXUSER_ADDRESS)
- return pmap_find_kpte(va);
+ UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
+ UVMHIST_LOG(maphist, "(va=%#lx)", va, 0, 0, 0);
- pte = pmap_find_vhpt(va);
- if (pte == NULL) {
- pte = pool_get(&pmap_ia64_lpte_pool, PR_NOWAIT);
- pte->tag = 1UL << 63;
+ while (size > 0) {
+ pte = pmap_find_kpte(va);
+ if (pmap_present(pte)) {
+ KASSERT(pmap_managed(pte) != 0);
+ pmap_remove_vhpt(va);
+ pmap_invalidate_page(va);
+ pmap_clear_present(pte);
+ }
+ va += PAGE_SIZE;
+ size -= PAGE_SIZE;
}
-
- return pte;
}
-static __inline void
-pmap_pte_prot(pmap_t pm, struct ia64_lpte *pte, vm_prot_t prot)
+/*
+ * pmap_copy: [ INTERFACE ]
+ *
+ * This function copies the mappings starting at src_addr in
+ * src_map for len bytes into dst_map starting at dst_addr.
+ *
+ * Note that while this function is required to be provided by
+ * a pmap implementation, it is not actually required to do
+ * anything. pmap_copy() is merely advisory (it is used in
+ * the fork(2) path to ``pre-fault'' the child's address
+ * space).
+ */
+void
+pmap_copy(pmap_t dst_map, pmap_t src_map, vaddr_t dst_addr, vsize_t len,
+ vaddr_t src_addr)
{
- static int prot2ar[4] = {
- PTE_AR_R, /* VM_PROT_NONE */
- PTE_AR_RW, /* VM_PROT_WRITE */
- PTE_AR_RX, /* VM_PROT_EXECUTE */
- PTE_AR_RWX /* VM_PROT_WRITE|VM_PROT_EXECUTE */
- };
-
- pte->pte &= ~(PTE_PROT_MASK | PTE_PL_MASK | PTE_AR_MASK);
- pte->pte |= (uint64_t)(prot & VM_PROT_ALL) << 56;
- pte->pte |= (prot == VM_PROT_NONE || pm == pmap_kernel())
- ? PTE_PL_KERN : PTE_PL_USER;
- pte->pte |= prot2ar[(prot & VM_PROT_ALL) >> 1];
+ /* nothing required */
}
-
-
/*
- * Set a pte to contain a valid mapping and enter it in the VHPT. If
- * the pte was orginally valid, then its assumed to already be in the
- * VHPT.
- * This functions does not set the protection bits. It's expected
- * that those have been set correctly prior to calling this function.
+ * pmap_update: [ INTERFACE ]
+ *
+ * If a pmap implementation does not delay virtual-to-physical
+ * mapping updates, pmap_update() has no operation.
*/
-static void
-pmap_set_pte(struct ia64_lpte *pte, vaddr_t va, vaddr_t pa,
- bool wired, bool managed)
+void
+pmap_update(pmap_t pmap)
{
-
- pte->pte &= PTE_PROT_MASK | PTE_PL_MASK | PTE_AR_MASK;
- pte->pte |= PTE_PRESENT | PTE_MA_WB;
- pte->pte |= (managed) ? PTE_MANAGED : (PTE_DIRTY | PTE_ACCESSED);
- pte->pte |= (wired) ? PTE_WIRED : 0;
- pte->pte |= pa & PTE_PPN_MASK;
-
- pte->itir = PAGE_SHIFT << 2;
-
- pte->tag = ia64_ttag(va);
+ /* nothing required */
}
/*
- * Remove the (possibly managed) mapping represented by pte from the
- * given pmap.
+ * pmap_activate: [ INTERFACE ]
+ *
+ * Activate the pmap used by the specified process. This includes
+ * reloading the MMU context if the current process, and marking
+ * the pmap in use by the processor.
+ *
+ * Note: We may use only spin locks here, since we are called
+ * by a critical section in cpu_switch()!
*/
-static int
-pmap_remove_pte(pmap_t pmap, struct ia64_lpte *pte, vaddr_t va,
- pv_entry_t pv, int freepte)
+void
+pmap_activate(struct lwp *l)
{
- int error;
- struct vm_page *pg;
-
- KASSERT(pmap == pmap_kernel() || pmap == curcpu()->ci_pmap);
-
- /*
- * First remove from the VHPT.
- */
- error = pmap_remove_vhpt(va);
- if (error)
- return error;
-
- pmap_invalidate_page(pmap, va);
-
- if (pmap_wired(pte))
- pmap->pm_stats.wired_count -= 1;
+ UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
+ UVMHIST_LOG(maphist, "(lwp=%p)", l, 0, 0, 0);
+ KASSERT(l == curlwp);
+#if 0
+ /* pmap_switch(vmspace_pmap(td->td_proc->p_vmspace)); */
+ pmap_switch(vm_map_pmap(&l->l_proc->p_vmspace->vm_map));
+#else
+ struct pmap *pmap = l->l_proc->p_vmspace->vm_map.pmap;
- pmap->pm_stats.resident_count -= 1;
- if (pmap_managed(pte)) {
- pg = PHYS_TO_VM_PAGE(pmap_ppn(pte));
- if (pmap_dirty(pte))
- pg->flags &= ~(PG_CLEAN);
- if (pmap_accessed(pte))
- pg->flags &= ~PG_CLEAN; /* XXX: Do we need this ? */
+ if (pmap == pmap_kernel()) {
+ return;
+ }
- if (freepte)
- pmap_free_pte(pte, va);
+ if (l != curlwp) {
+ return;
+ }
- error = pmap_remove_entry(pmap, pg, va, pv);
+ pmap_switch(pmap);
+#endif
+}
- }
- if (freepte)
- pmap_free_pte(pte, va);
- return 0;
+/*
+ * pmap_deactivate: [ INTERFACE ]
+ *
+ * Deactivate the physical map used by the process behind lwp
+ * l. It is generally used in conjunction with
+ * pmap_activate(). Like pmap_activate(), pmap_deactivate()
+ * may not always be called when l is the current lwp.
+ *
+ */
+void
+pmap_deactivate(struct lwp *l)
+{
+ /* XXX ? */
}
/*
- * Free a pte which is now unused. This simply returns it to the zone
- * allocator if it is a user mapping. For kernel mappings, clear the
- * valid bit to make it clear that the mapping is not currently used.
+ * pmap_zero_page: [ INTERFACE ]
+ *
+ * Zero the specified (machine independent) page by mapping the page
+ * into virtual memory and clear its contents, one machine dependent
+ * page at a time.
+ *
+ * Note: no locking is necessary in this function.
*/
-static void
-pmap_free_pte(struct ia64_lpte *pte, vaddr_t va)
+void
+pmap_zero_page(paddr_t phys)
{
+ struct vm_page *m;
+ vaddr_t va;
+
+ UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
+ UVMHIST_LOG(maphist, "(pa=%p)", phys, 0, 0, 0);
- if (va >= VM_MAXUSER_ADDRESS)
- pmap_clear_present(pte);
-}
+ m = PHYS_TO_VM_PAGE(phys);
+ KASSERT(m != NULL);
+ va = pmap_page_to_va(m);
+ KASSERT(trunc_page(va) == va);
-/***************************************************
- * page management routines.
- ***************************************************/
+ UVMHIST_LOG(maphist, "(pa=%p, va=%p)", phys, va, 0, 0);
+ memset((void *)va, 0, PAGE_SIZE);
+}
/*
- * get a new pv_entry, allocating a block from the system
- * when needed.
- * the memory allocation is performed bypassing the malloc code
- * because of the possibility of allocations at interrupt time.
- */
-/*
- * get a new pv_entry, allocating a block from the system
- * when needed.
+ * pmap_copy_page: [ INTERFACE ]
+ *
+ * Copy the specified (machine independent) page by mapping the page
+ * into virtual memory and using memcpy to copy the page, one machine
+ * dependent page at a time.
+ *
+ * Note: no locking is necessary in this function.
*/
-static pv_entry_t
-get_pv_entry(pmap_t locked_pmap)
+void
+pmap_copy_page(paddr_t psrc, paddr_t pdst)
{
- pv_entry_t allocated_pv;
+ struct vm_page *md, *ms;
+ vaddr_t dst_va, src_va;
- allocated_pv = pool_get(&pmap_pv_pool, PR_NOWAIT);
- return allocated_pv;
+ UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
+ UVMHIST_LOG(maphist, "(sp=%p, dp=%p)", psrc, pdst, 0, 0);
- /* XXX: Nice to have all this stuff later:
- * Reclaim pv entries: At first, destroy mappings to inactive
- * pages. After that, if a pv entry is still needed, destroy
- * mappings to active pages.
- */
+ md = PHYS_TO_VM_PAGE(pdst);
+ ms = PHYS_TO_VM_PAGE(psrc);
+ KASSERT(md != NULL && ms != NULL);
+
+ dst_va = pmap_page_to_va(md);
+ src_va = pmap_page_to_va(ms);
+ KASSERT(trunc_page(dst_va) == dst_va && trunc_page(src_va) == src_va);
+
+ memcpy((void *)dst_va, (void *)src_va, PAGE_SIZE);
}
/*
- * free the pv_entry back to the free list
+ * pmap_page_protect: [ INTERFACE ]
+ *
+ * Lower the permissions for all mappings of the page pg to
+ * prot. This function is used by the virtual memory system
+ * to implement copy-on-write (called with VM_PROT_READ set in
+ * prot) and to revoke all mappings when cleaning a page
+ * (called with no bits set in prot). Access permissions must
+ * never be added to a page as a result of this call.
*/
-static __inline void
-free_pv_entry(pv_entry_t pv)
+void
+pmap_page_protect(struct vm_page *pg, vm_prot_t prot)
{
+ //struct ia64_lpte *pte;
+ pmap_t pmap;
+ pv_entry_t pv;
+ vaddr_t va;
+
+ UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
+ UVMHIST_LOG(maphist, "(m=%p, prot=%p)", pg, prot, 0, 0);
- pool_put(&pmap_pv_pool, pv);
+ if ((prot & VM_PROT_WRITE) != 0)
+ return;
+ if (prot & (VM_PROT_READ | VM_PROT_EXECUTE)) {
+ /* XXX FreeBSD
+ if ((pg->mdpage.aflags & PGA_WRITEABLE) == 0)
+ return;
+ */
+ if (pg->flags & PG_RDONLY)
+ return;
+ rw_enter(&pvh_global_lock, RW_WRITER);
+ TAILQ_FOREACH(pv, &pg->mdpage.pv_list, pv_list) {
+ pmap = PV_PMAP(pv);
+ va = pv->pv_va;
+ // locking of pmap done in pmap_protect
+ pmap_protect(pmap, va, va + PAGE_SIZE, prot);
+ }
+ /* XXX
+ vm_page_aflag_clear(pg, PGA_WRITEABLE);
+ */
+ pg->flags |= PG_RDONLY;
+
+ rw_exit(&pvh_global_lock);
+ } else {
+ pmap_remove_all_phys(pg);
+ }
}
/*
- * Add an ia64_lpte to the VHPT.
+ * pmap_clear_modify: [ INTERFACE ]
+ *
+ * Clear the modify bits on the specified physical page.
*/
-static void
-pmap_enter_vhpt(struct ia64_lpte *pte, vaddr_t va)
+bool
+pmap_clear_modify(struct vm_page *pg)
{
- struct ia64_bucket *bckt;
- struct ia64_lpte *vhpte;
- uint64_t pte_pa;
+ struct ia64_lpte *pte;
+ pmap_t oldpmap, pmap;
+ pv_entry_t pv;
+ bool rv;
- /* Can fault, so get it out of the way. */
- pte_pa = ia64_tpa((vaddr_t)pte);
+ UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
+ UVMHIST_LOG(maphist, "(m=%p)", pg, 0, 0, 0);
- vhpte = (struct ia64_lpte *)ia64_thash(va);
- bckt = (struct ia64_bucket *)vhpte->chain;
- /* XXX: fixme */
- mutex_enter(&bckt->lock);
- pte->chain = bckt->chain;
- ia64_mf();
- bckt->chain = pte_pa;
+ KASSERTMSG(vm_page_is_managed(pg), "%s : page %p not managed",
+ __func__, pg);
- pmap_vhpt_inserts++;
- bckt->length++;
- /*XXX : fixme */
- mutex_exit(&bckt->lock);
+ rv = false;
+
+ //VM_OBJECT_ASSERT_WLOCKED(m->object);
+ //KASSERT(!vm_page_xbusied(m),
+ // ("pmap_clear_modify: page %p is exclusive busied", m));
+
+ /*
+ * If the page is not PGA_WRITEABLE, then no PTEs can be modified.
+ * If the object containing the page is locked and the page is not
+ * exclusive busied, then PGA_WRITEABLE cannot be concurrently set.
+ */
+
+#if 0 /* XXX freebsd does this, looks faster but not required */
+ if ((m->aflags & PGA_WRITEABLE) == 0)
+ return;
+ if (pg->flags & PG_RDONLY)
+ return (rv);
+#endif
+ rw_enter(&pvh_global_lock, RW_WRITER);
+ TAILQ_FOREACH(pv, &pg->mdpage.pv_list, pv_list) {
+ pmap = PV_PMAP(pv);
+ PMAP_LOCK(pmap);
+ oldpmap = pmap_switch(pmap);
+ pte = pmap_find_vhpt(pv->pv_va);
+ KASSERTMSG(pte != NULL, "pte");
+ if (pmap_dirty(pte)) {
+ pmap_clear_dirty(pte);
+ pmap_invalidate_page(pv->pv_va);
+ rv = true;
+ }
+ pmap_switch(oldpmap);
+ PMAP_UNLOCK(pmap);
+ }
+
+ rw_exit(&pvh_global_lock);
+
+ return (rv);
}
/*
- * Remove the ia64_lpte matching va from the VHPT. Return zero if it
- * worked or an appropriate error code otherwise.
+ * pmap_clear_reference: [ INTERFACE ]
+ *
+ * Clear the reference bit on the specified physical page.
+ *
+ * returns true or false
+ * indicating whether or not the ``referenced'' attribute was
+ * set on the page before it was cleared
*/
-static int
-pmap_remove_vhpt(vaddr_t va)
+bool
+pmap_clear_reference(struct vm_page *pg)
{
- struct ia64_bucket *bckt;
struct ia64_lpte *pte;
- struct ia64_lpte *lpte;
- struct ia64_lpte *vhpte;
- uint64_t chain, tag;
+ pmap_t oldpmap, pmap;
+ pv_entry_t pv;
+ bool rv;
- tag = ia64_ttag(va);
- vhpte = (struct ia64_lpte *)ia64_thash(va);
- bckt = (struct ia64_bucket *)vhpte->chain;
+ UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
+ UVMHIST_LOG(maphist, "(m=%p)", pg, 0, 0, 0);
- lpte = NULL;
- mutex_enter(&bckt->lock);
+ KASSERTMSG(vm_page_is_managed(pg), "%s: page %p is not managed",
+ __func__, pg);
- chain = bckt->chain;
- pte = (struct ia64_lpte *)IA64_PHYS_TO_RR7(chain);
- while (chain != 0 && pte->tag != tag) {
- lpte = pte;
- chain = pte->chain;
- pte = (struct ia64_lpte *)IA64_PHYS_TO_RR7(chain);
- }
- if (chain == 0) {
- mutex_exit(&bckt->lock);
- return ENOENT;
+ rv = false;
+
+ rw_enter(&pvh_global_lock, RW_WRITER);
+ TAILQ_FOREACH(pv, &pg->mdpage.pv_list, pv_list) {
+ pmap = PV_PMAP(pv);
+ PMAP_LOCK(pmap);
+ oldpmap = pmap_switch(pmap);
+ pte = pmap_find_vhpt(pv->pv_va);
+ KASSERTMSG(pte != NULL, "pte");
+ if (pmap_accessed(pte)) {
+ rv = true;
+ pmap_clear_accessed(pte);
+ pmap_invalidate_page(pv->pv_va);
+ }
+ pmap_switch(oldpmap);
+ PMAP_UNLOCK(pmap);
}
- /* Snip this pv_entry out of the collision chain. */
- if (lpte == NULL)
- bckt->chain = pte->chain;
- else
- lpte->chain = pte->chain;
- ia64_mf();
-
- bckt->length--;
- mutex_exit(&bckt->lock);
- return 0;
+ rw_exit(&pvh_global_lock);
+ return (rv);
}
/*
- * Find the ia64_lpte for the given va, if any.
+ * pmap_is_modified: [ INTERFACE ]
+ *
*/
-static struct ia64_lpte *
-pmap_find_vhpt(vaddr_t va)
+bool
+pmap_is_modified(struct vm_page *pg)
{
- struct ia64_bucket *bckt;
struct ia64_lpte *pte;
- uint64_t chain, tag;
+ pmap_t oldpmap, pmap;
+ pv_entry_t pv;
+ bool rv;
- tag = ia64_ttag(va);
- pte = (struct ia64_lpte *)ia64_thash(va);
- bckt = (struct ia64_bucket *)pte->chain;
+ UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
+ UVMHIST_LOG(maphist, "(m=%p)", pg, 0, 0, 0);
- mutex_enter(&bckt->lock);
- chain = bckt->chain;
- pte = (struct ia64_lpte *)IA64_PHYS_TO_RR7(chain);
- while (chain != 0 && pte->tag != tag) {
- chain = pte->chain;
- pte = (struct ia64_lpte *)IA64_PHYS_TO_RR7(chain);
+ KASSERTMSG(vm_page_is_managed(pg), "%s: page %p is not managed",
+ __func__, pg);
+ rv = false;
+
+ /*
+ * If the page is not exclusive busied, then PGA_WRITEABLE cannot be
+ * concurrently set while the object is locked. Thus, if PGA_WRITEABLE
+ * is clear, no PTEs can be dirty.
+ */
+ /* XXX freebsd
+ VM_OBJECT_ASSERT_WLOCKED(m->object);
+ if (!vm_page_xbusied(m) && (m->aflags & PGA_WRITEABLE) == 0)
+ return (rv);
+ */
+ rw_enter(&pvh_global_lock, RW_WRITER);
+ TAILQ_FOREACH(pv, &pg->mdpage.pv_list, pv_list) {
+ pmap = PV_PMAP(pv);
+ PMAP_LOCK(pmap);
+ oldpmap = pmap_switch(pmap);
+ pte = pmap_find_vhpt(pv->pv_va);
+ pmap_switch(oldpmap);
+ KASSERTMSG(pte != NULL, "pte");
+
+ rv = pmap_dirty(pte) ? true : false;
+ PMAP_UNLOCK(pmap);
+ if (rv)
+ break;
}
- mutex_exit(&bckt->lock);
- return (chain != 0) ? pte : NULL;
+
+ rw_exit(&pvh_global_lock);
+ return (rv);
}
/*
- * Remove an entry from the list of managed mappings.
+ * pmap_is_referenced: [ INTERFACE ]
+ *
+ * Test whether or not the ``referenced'' attribute is set on
+ * page pg.
+ *
*/
-static int
-pmap_remove_entry(pmap_t pmap, struct vm_page *pg, vaddr_t va, pv_entry_t pv)
+bool
+pmap_is_referenced(struct vm_page *pg)
{
- struct vm_page_md * const md = VM_PAGE_TO_MD(pg);
+ struct ia64_lpte *pte;
+ pmap_t oldpmap, pmap;
+ pv_entry_t pv;
+ bool rv;
- if (!pv) {
- if (md->pv_list_count < pmap->pm_stats.resident_count) {
- TAILQ_FOREACH(pv, &md->pv_list, pv_list)
- if (pmap == pv->pv_pmap && va == pv->pv_va)
- break;
- } else {
- TAILQ_FOREACH(pv, &pmap->pm_pvlist, pv_plist)
- if (va == pv->pv_va)
- break;
- }
- }
+ UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
+ UVMHIST_LOG(maphist, "(m=%p)", pg, 0, 0, 0);
- if (pv) {
- TAILQ_REMOVE(&md->pv_list, pv, pv_list);
- md->pv_list_count--;
- if (TAILQ_FIRST(&md->pv_list) == NULL)
- pg->flags |= PG_RDONLY;
+ KASSERTMSG(vm_page_is_managed(pg), "%s: page %p is not managed",
+ __func__, pg);
+
+ rv = false;
+ rw_enter(&pvh_global_lock, RW_WRITER);
+ TAILQ_FOREACH(pv, &pg->mdpage.pv_list, pv_list) {
+ pmap = PV_PMAP(pv);
+ PMAP_LOCK(pmap);
+ oldpmap = pmap_switch(pmap);
+ pte = pmap_find_vhpt(pv->pv_va);
+ KASSERTMSG(pte != NULL, "pte");
+ rv = pmap_accessed(pte) ? true : false;
+ pmap_switch(oldpmap);
+ PMAP_UNLOCK(pmap);
+ if (rv)
+ break;
+ }
- TAILQ_REMOVE(&pmap->pm_pvlist, pv, pv_plist);
- free_pv_entry(pv);
- return 0;
- } else
- return ENOENT;
+ rw_exit(&pvh_global_lock);
+ return (rv);
}
/*
- * Create a pv entry for page at pa for
- * (pmap, va).
+ * pmap_phys_address: [ INTERFACE ]
+ *
+ * Return the physical address corresponding to the specified
+ * cookie. Used by the device pager to decode a device driver's
+ * mmap entry point return value.
+ *
+ * Note: no locking is necessary in this function.
*/
-static void
-pmap_insert_entry(pmap_t pmap, vaddr_t va, struct vm_page *pg)
+paddr_t
+pmap_phys_address(paddr_t ppn)
{
- struct vm_page_md * const md = VM_PAGE_TO_MD(pg);
- pv_entry_t pv;
-
- pv = get_pv_entry(pmap);
- pv->pv_pmap = pmap;
- pv->pv_va = va;
-
- TAILQ_INSERT_TAIL(&pmap->pm_pvlist, pv, pv_plist);
- TAILQ_INSERT_TAIL(&md->pv_list, pv, pv_list);
- md->pv_list_count++;
+ return ia64_ptob(ppn);
}
-/*
- * Remove a single page from a process address space
- */
-static void
-pmap_remove_page(pmap_t pmap, vaddr_t va)
+pmap_t
+pmap_switch(pmap_t pm)
{
- struct ia64_lpte *pte;
+ pmap_t prevpm;
+ int i;
- KASSERT(pmap == pmap_kernel() || pmap == curcpu()->ci_pmap);
+ critical_enter();
- pte = pmap_find_vhpt(va);
- if (pte) {
- pmap_remove_pte(pmap, pte, va, 0, 1);
- pmap_invalidate_page(pmap, va);
+ prevpm = curcpu()->ci_pmap;
+ if (prevpm == pm)
+ goto out;
+ if (pm == NULL) {
+ for (i = 0; i < IA64_VM_MINKERN_REGION; i++) {
+ ia64_set_rr(IA64_RR_BASE(i),
+ (i << 8)|(PAGE_SHIFT << 2)|1);
+ }
+ } else {
+ for (i = 0; i < IA64_VM_MINKERN_REGION; i++) {
+ ia64_set_rr(IA64_RR_BASE(i),
+ (pm->pm_rid[i] << 8)|(PAGE_SHIFT << 2)|1);
+ }
}
- return;
+
+ curcpu()->ci_pmap = pm;
+ ia64_srlz_d();
+
+out:
+ critical_exit();
+ return (prevpm);
}
+
/*
- * pmap_pv_page_alloc:
- *
- * Allocate a page for the pv_entry pool.
+ * Synchronize CPU instruction caches of the specified range.
+ * Same as freebsd
+ * void pmap_sync_icache(pmap_t pm, vaddr_t va, vsize_t sz)
*/
-void *
-pmap_pv_page_alloc(struct pool *pp, int flags)
+void
+pmap_procwr(struct proc *p, vaddr_t va, vsize_t sz)
{
- paddr_t pg;
+ pmap_t oldpm;
+ struct ia64_lpte *pte;
+ vaddr_t lim;
+ vsize_t len;
- if (pmap_poolpage_alloc(&pg))
- return (void *)IA64_PHYS_TO_RR7(pg);
- return NULL;
+ struct pmap * const pm = p->p_vmspace->vm_map.pmap;
+
+ UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
+ UVMHIST_LOG(maphist, "(pm=%p, va=%#lx, sz=%#lx)", pm, va, sz, 0);
+
+ sz += va & 31;
+ va &= ~31;
+ sz = (sz + 31) & ~31;
+
+ PMAP_LOCK(pm);
+ oldpm = pmap_switch(pm);
+ while (sz > 0) {
+ lim = round_page(va);
+ len = MIN(lim - va, sz);
+ pte = pmap_find_vhpt(va);
+ if (pte != NULL && pmap_present(pte))
+ ia64_sync_icache(va, len);
+ va += len;
+ sz -= len;
+ }
+ pmap_switch(oldpm);
+ PMAP_UNLOCK(pm);
}
/*
- * pmap_pv_page_free:
+ * Routine: pmap_remove_all_phys
+ * Function:
+ * Removes this physical page from
+ * all physical maps in which it resides.
+ * Reflects back modify bits to the pager.
*
- * Free a pv_entry pool page.
+ * Notes:
+ * Original versions of this routine were very
+ * inefficient because they iteratively called
+ * pmap_remove (slow...)
*/
void
-pmap_pv_page_free(struct pool *pp, void *v)
+pmap_remove_all_phys(struct vm_page *m)
{
+ pmap_t oldpmap;
+ pv_entry_t pv;
- pmap_poolpage_free(IA64_RR_MASK((vaddr_t)v));
-}
+ UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
+ UVMHIST_LOG(maphist, "(m=%p)", m, 0, 0, 0);
+
+ KASSERTMSG(vm_page_is_managed(m), "%s: page %p is not managed",
+ __func__, m);
+
+ rw_enter(&pvh_global_lock, RW_WRITER);
+ while ((pv = TAILQ_FIRST(&m->mdpage.pv_list)) != NULL) {
+ struct ia64_lpte *pte;
+ pmap_t pmap = PV_PMAP(pv);
+ vaddr_t va = pv->pv_va;
-/******************** misc. functions ********************/
+ PMAP_LOCK(pmap);
+ oldpmap = pmap_switch(pmap);
+ pte = pmap_find_vhpt(va);
+ KASSERTMSG(pte != NULL, "pte");
+ if (pmap_ppn(pte) != VM_PAGE_TO_PHYS(m))
+ panic("%s: pv_table for %lx is inconsistent",
+ __func__, VM_PAGE_TO_PHYS(m));
+ pmap_remove_pte(pmap, pte, va, pv, 1);
+ pmap_switch(oldpmap);
+ PMAP_UNLOCK(pmap);
+ }
+ /* XXX freebsd
+ vm_page_aflag_clear(m, PGA_WRITEABLE);
+ */
+ m->flags |= PG_RDONLY;
+
+ rw_exit(&pvh_global_lock);
+}
/*
- * pmap_poolpage_alloc: based on alpha/pmap_physpage_alloc
+ * vm_page_alloc1:
*
- * Allocate a single page from the VM system and return the
- * physical address for that page.
+ * Allocate and return a memory cell with no associated object.
*/
-bool
-pmap_poolpage_alloc(paddr_t *pap)
+static struct vm_page
+*vm_page_alloc1(void)
{
struct vm_page *pg;
- paddr_t pa;
-
+
pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_USERESERVE|UVM_PGA_ZERO);
- if (pg != NULL) {
- pa = VM_PAGE_TO_PHYS(pg);
-
-#ifdef DEBUG
- struct vm_page_md * const md = VM_PAGE_TO_MD(pg);
- mutex_enter(&md->pv_mutex);
- if (pg->wire_count != 0) {
- printf("pmap_physpage_alloc: page 0x%lx has "
- "%d references\n", pa, pg->wire_count);
- panic("pmap_physpage_alloc");
- }
- mutex_exit(&md->pv_mutex);
-#endif
- *pap = pa;
- return true;
+ if (pg) {
+ pg->wire_count = 1; /* no mappings yet */
+ pg->flags &= ~PG_BUSY; /* never busy */
}
- return false;
+ return pg;
}
/*
- * pmap_poolpage_free: based on alpha/pmap_physpage_free:
+ * vm_page_free1:
+ *
+ * Returns the given page to the free list,
+ * disassociating it with any VM object.
*
- * Free the single page table page at the specified physical address.
+ * Object and page must be locked prior to entry.
*/
-void
-pmap_poolpage_free(paddr_t pa)
+static void
+vm_page_free1(struct vm_page *pg)
{
- struct vm_page *pg;
-
- if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL)
- panic("pmap_physpage_free: bogus physical page address");
-
-#ifdef DEBUG
- struct vm_page_md * const md = VM_PAGE_TO_MD(pg);
- mutex_enter(&md->pv_mutex);
- if (pg->wire_count != 0)
- panic("pmap_physpage_free: page still has references");
- mutex_exit(&md->pv_mutex);
-#endif
+ KASSERT(pg->flags != (PG_CLEAN|PG_FAKE)); /* Freeing invalid page */
+ pg->flags |= PG_BUSY;
+ pg->wire_count = 0;
uvm_pagefree(pg);
}
-#ifdef DEBUG
-
-static void dump_vhpt(void)
+/*
+ * pmap_flag_to_attr
+ *
+ * Convert pmap_enter/pmap_kenter_pa flags to memory attributes
+ */
+static vm_memattr_t
+pmap_flags_to_memattr(u_int flags)
{
+ u_int cacheflags = flags & PMAP_CACHE_MASK;
- vaddr_t base;
- vsize_t size, i;
- struct ia64_lpte *pte;
-
- __asm __volatile("mov %0=cr.pta;; srlz.i;;" : "=r" (base));
-
-#define VHPTBASE(x) ( (x) & (~0x7fffUL) )
-#define VHPTSIZE(x) ( (vsize_t) (1 << (((x) & 0x7cUL) >> 2)))
-
- size = VHPTSIZE(base);
- base = VHPTBASE(base);
-
- pte = (void *) base;
-
- printf("vhpt base = %lx \n", base);
- printf("vhpt size = %lx \n", size);
-
- for(i = 0; i < size/sizeof(struct ia64_lpte);i++ )
- if(pte[i].pte & PTE_PRESENT) {
- printf("PTE_PRESENT ");
+#if 0
+ UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
+ UVMHIST_LOG(maphist, "(PMAP_NOCACHE=%u, PMAP_WRITE_COMBINE=%u, "
+ "PMAP_WRITE_BACK=%u, PMAP_NOCACHE_OVR=%u)",
+ (flags & PMAP_NOCACHE) != 0, (flags & PMAP_WRITE_COMBINE) != 0,
+ (flags & PMAP_WRITE_BACK) != 0, (flags & PMAP_NOCACHE_OVR) != 0);
+#endif
+ switch (cacheflags) {
+ case PMAP_NOCACHE:
+ return VM_MEMATTR_UNCACHEABLE;
+ case PMAP_WRITE_COMBINE:
+ /* XXX implement if possible */
+ KASSERT(1);
+ return VM_MEMATTR_WRITE_COMBINING;
+ case PMAP_WRITE_BACK:
+ return VM_MEMATTR_WRITE_BACK;
+ case PMAP_NOCACHE_OVR:
+ default:
+ return VM_MEMATTR_DEFAULT;
+ }
+}
- if (pte[i].pte & PTE_MA_MASK) printf("MA: ");
- if (pte[i].pte & PTE_MA_WB) printf("WB ");
- if (pte[i].pte & PTE_MA_UC) printf("UC ");
- if (pte[i].pte & PTE_MA_UCE) printf("UCE ");
- if (pte[i].pte & PTE_MA_WC) printf("WC ");
- if (pte[i].pte & PTE_MA_NATPAGE)printf("NATPAGE ");
+#ifdef DEBUG
+/*
+ * Test ref/modify handling.
+ */
- if (pte[i].pte & PTE_ACCESSED) printf("PTE_ACCESSED ");
- if (pte[i].pte & PTE_DIRTY) printf("PTE_DIRTY ");
+static void
+pmap_testout(void)
+{
+ vaddr_t va;
+ volatile int *loc;
+ int val = 0;
+ paddr_t pa;
+ struct vm_page *pg;
+ int ref, mod;
+ bool extracted;
+
+ /* Allocate a page */
+ va = (vaddr_t)uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_WIRED | UVM_KMF_ZERO);
+ KASSERT(va != 0);
+ loc = (int*)va;
+
+ extracted = pmap_extract(pmap_kernel(), va, &pa);
+ printf("va %p pa %lx extracted %u\n", (void *)(u_long)va, pa, extracted);
+ printf("kextract %lx\n", pmap_kextract(va));
- if (pte[i].pte & PTE_PL_MASK) printf("PL: ");
- if (pte[i].pte & PTE_PL_KERN) printf("KERN");
- if (pte[i].pte & PTE_PL_USER) printf("USER");
+ pg = PHYS_TO_VM_PAGE(pa);
+ pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0);
+ pmap_update(pmap_kernel());
- if (pte[i].pte & PTE_AR_MASK) printf("AR: ");
- if (pte[i].pte & PTE_AR_R) printf("R ");
- if (pte[i].pte & PTE_AR_RX) printf("RX ");
- if (pte[i].pte & PTE_AR_RWX) printf("RWX ");
- if (pte[i].pte & PTE_AR_R_RW) printf("R RW ");
- if (pte[i].pte & PTE_AR_RX_RWX) printf("RX RWX ");
+ ref = pmap_is_referenced(pg);
+ mod = pmap_is_modified(pg);
- printf("ppn = %lx", (pte[i].pte & PTE_PPN_MASK) >> 12);
+ printf("Entered page va %p pa %lx: ref %d, mod %d\n",
+ (void *)(u_long)va, (long)pa, ref, mod);
- if (pte[i].pte & PTE_ED) printf("ED ");
+ /* Now clear reference and modify */
+ ref = pmap_clear_reference(pg);
+ mod = pmap_clear_modify(pg);
+ printf("Clearing page va %p pa %lx: ref %d, mod %d\n",
+ (void *)(u_long)va, (long)pa,
+ ref, mod);
+
+ /* Check it's properly cleared */
+ ref = pmap_is_referenced(pg);
+ mod = pmap_is_modified(pg);
+ printf("Checking cleared page: ref %d, mod %d\n",
+ ref, mod);
+
+ /* Reference page */
+ val = *loc;
+
+ ref = pmap_is_referenced(pg);
+ mod = pmap_is_modified(pg);
+ printf("Referenced page: ref %d, mod %d val %x\n",
+ ref, mod, val);
+
+ /* Now clear reference and modify */
+ ref = pmap_clear_reference(pg);
+ mod = pmap_clear_modify(pg);
+ printf("Clearing page va %p pa %lx: ref %d, mod %d\n",
+ (void *)(u_long)va, (long)pa,
+ ref, mod);
+
+ /* Modify page */
+ *loc = 1;
+
+ ref = pmap_is_referenced(pg);
+ mod = pmap_is_modified(pg);
+ printf("Modified page: ref %d, mod %d\n",
+ ref, mod);
+
+ /* Now clear reference and modify */
+ ref = pmap_clear_reference(pg);
+ mod = pmap_clear_modify(pg);
+ printf("Clearing page va %p pa %lx: ref %d, mod %d\n",
+ (void *)(u_long)va, (long)pa,
+ ref, mod);
+
+ /* Check it's properly cleared */
+ ref = pmap_is_referenced(pg);
+ mod = pmap_is_modified(pg);
+ printf("Checking cleared page: ref %d, mod %d\n",
+ ref, mod);
+
+ /* Modify page */
+ *loc = 1;
+
+ ref = pmap_is_referenced(pg);
+ mod = pmap_is_modified(pg);
+ printf("Modified page: ref %d, mod %d\n",
+ ref, mod);
+
+ /* Check pmap_protect() */
+ pmap_protect(pmap_kernel(), va, va+1, VM_PROT_READ);
+ pmap_update(pmap_kernel());
+ ref = pmap_is_referenced(pg);
+ mod = pmap_is_modified(pg);
+ printf("pmap_protect(VM_PROT_READ): ref %d, mod %d\n",
+ ref, mod);
+
+ /* Now clear reference and modify */
+ ref = pmap_clear_reference(pg);
+ mod = pmap_clear_modify(pg);
+ printf("Clearing page va %p pa %lx: ref %d, mod %d\n",
+ (void *)(u_long)va, (long)pa,
+ ref, mod);
+
+ /* Modify page */
+ pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0);
+ pmap_update(pmap_kernel());
+ *loc = 1;
+
+ ref = pmap_is_referenced(pg);
+ mod = pmap_is_modified(pg);
+ printf("Modified page: ref %d, mod %d\n",
+ ref, mod);
+
+ /* Check pmap_protect() */
+ pmap_protect(pmap_kernel(), va, va+1, VM_PROT_NONE);
+ pmap_update(pmap_kernel());
+ ref = pmap_is_referenced(pg);
+ mod = pmap_is_modified(pg);
+ printf("pmap_protect(VM_PROT_READ): ref %d, mod %d\n",
+ ref, mod);
+
+ /* Now clear reference and modify */
+ ref = pmap_clear_reference(pg);
+ mod = pmap_clear_modify(pg);
+ printf("Clearing page va %p pa %lx: ref %d, mod %d\n",
+ (void *)(u_long)va, (long)pa,
+ ref, mod);
+
+ /* Modify page */
+ pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0);
+ pmap_update(pmap_kernel());
+ *loc = 1;
+
+ ref = pmap_is_referenced(pg);
+ mod = pmap_is_modified(pg);
+ printf("Modified page: ref %d, mod %d\n",
+ ref, mod);
+
+ /* Check pmap_pag_protect() */
+ pmap_page_protect(pg, VM_PROT_READ);
+ ref = pmap_is_referenced(pg);
+ mod = pmap_is_modified(pg);
+ printf("pmap_protect(): ref %d, mod %d\n",
+ ref, mod);
+
+ /* Now clear reference and modify */
+ ref = pmap_clear_reference(pg);
+ mod = pmap_clear_modify(pg);
+ printf("Clearing page va %p pa %lx: ref %d, mod %d\n",
+ (void *)(u_long)va, (long)pa,
+ ref, mod);
+
+
+ /* Modify page */
+ pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0);
+ pmap_update(pmap_kernel());
+ *loc = 1;
+
+ ref = pmap_is_referenced(pg);
+ mod = pmap_is_modified(pg);
+ printf("Modified page: ref %d, mod %d\n",
+ ref, mod);
+
+ /* Check pmap_pag_protect() */
+ pmap_page_protect(pg, VM_PROT_NONE);
+ ref = pmap_is_referenced(pg);
+ mod = pmap_is_modified(pg);
+ printf("pmap_protect(): ref %d, mod %d\n",
+ ref, mod);
+
+ /* Now clear reference and modify */
+ ref = pmap_clear_reference(pg);
+ mod = pmap_clear_modify(pg);
+ printf("Clearing page va %p pa %lx: ref %d, mod %d\n",
+ (void *)(u_long)va, (long)pa,
+ ref, mod);
+
+ /* Unmap page */
+ pmap_remove(pmap_kernel(), va, va+1);
+ pmap_update(pmap_kernel());
+ ref = pmap_is_referenced(pg);
+ mod = pmap_is_modified(pg);
+ printf("Unmapped page: ref %d, mod %d\n", ref, mod);
+
+ /* Now clear reference and modify */
+ ref = pmap_clear_reference(pg);
+ mod = pmap_clear_modify(pg);
+ printf("Clearing page va %p pa %lx: ref %d, mod %d\n",
+ (void *)(u_long)va, (long)pa, ref, mod);
+
+ /* Check it's properly cleared */
+ ref = pmap_is_referenced(pg);
+ mod = pmap_is_modified(pg);
+ printf("Checking cleared page: ref %d, mod %d\n",
+ ref, mod);
+
+ pmap_remove(pmap_kernel(), va, va+1);
+ pmap_update(pmap_kernel());
+ //pmap_free_page(pa, cpus_active);
+ uvm_km_free(kernel_map, (vaddr_t)va, PAGE_SIZE,
+ UVM_KMF_WIRED | UVM_KMF_ZERO);
- if (pte[i].pte & PTE_IG_MASK) printf("OS: ");
- if (pte[i].pte & PTE_WIRED) printf("WIRED ");
- if (pte[i].pte & PTE_MANAGED) printf("MANAGED ");
- printf("\n");
- }
+ panic("end of testout");
}
#endif
Index: src/sys/arch/ia64/include/pmap.h
diff -u src/sys/arch/ia64/include/pmap.h:1.7 src/sys/arch/ia64/include/pmap.h:1.8
--- src/sys/arch/ia64/include/pmap.h:1.7 Sun Nov 14 13:33:22 2010
+++ src/sys/arch/ia64/include/pmap.h Sat Apr 8 18:08:33 2017
@@ -69,67 +69,93 @@
* from: hp300: @(#)pmap.h 7.2 (Berkeley) 12/16/90
* from: @(#)pmap.h 7.4 (Berkeley) 5/12/91
* from: i386 pmap.h,v 1.54 1997/11/20 19:30:35 bde Exp
- * $FreeBSD: src/sys/ia64/include/pmap.h,v 1.25 2005/09/03 23:53:50 marcel Exp $
+ * $FreeBSD: releng/10.1/sys/ia64/include/pmap.h 268201 2014-07-02 23:57:55Z marcel $
*/
-#ifndef _PMAP_MACHINE_
-#define _PMAP_MACHINE_
+#ifndef _IA64_PMAP_H_
+#define _IA64_PMAP_H_
+#include <sys/types.h>
+#include <sys/queue.h>
#include <sys/lock.h>
+#include <sys/mutex.h>
+
+#include <machine/pte.h>
+#include <machine/vmparam.h>
+
+#include <machine/md_var.h>
+
+typedef char vm_memattr_t;
-paddr_t vtophys(vaddr_t);
+#define VM_MEMATTR_WRITE_BACK ((vm_memattr_t)PTE_MA_WB)
+#define VM_MEMATTR_UNCACHEABLE ((vm_memattr_t)PTE_MA_UC)
+#define VM_MEMATTR_UNCACHEABLE_EXPORTED ((vm_memattr_t)PTE_MA_UCE)
+#define VM_MEMATTR_WRITE_COMBINING ((vm_memattr_t)PTE_MA_WC)
+#define VM_MEMATTR_NATPAGE ((vm_memattr_t)PTE_MA_NATPAGE)
+#define VM_MEMATTR_DEFAULT VM_MEMATTR_WRITE_BACK
-struct pv_entry; /* Forward declaration. */
+#ifdef _KERNEL
+
+#define MAXKPT (PAGE_SIZE/sizeof(vaddr_t))
+
+#define vtophys(va) pmap_kextract((vaddr_t)(va))
+
+#endif /* _KERNEL */
+
+/*
+ * Pmap stuff
+ */
+struct pv_entry;
+struct pv_chunk;
struct pmap {
- TAILQ_ENTRY(pmap) pm_list; /* list of all pmaps */
- TAILQ_HEAD(,pv_entry) pm_pvlist; /* list of mappings in pmap */
- int pm_count; /* pmap reference count */
- kmutex_t pm_slock; /* lock on pmap */
- uint32_t pm_rid[5]; /* base RID for pmap */
- int pm_active; /* active flag */
+ kmutex_t pm_mtx;
+ TAILQ_HEAD(,pv_chunk) pm_pvchunk; /* list of mappings in pmap */
+ uint32_t pm_rid[IA64_VM_MINKERN_REGION];
struct pmap_statistics pm_stats; /* pmap statistics */
- unsigned long pm_cpus; /* mask of CPUs using pmap */
-
+ uint64_t pm_refcount; /* pmap reference count, atomic */
};
+typedef struct pmap *pmap_t;
+
+#ifdef _KERNEL
+
+#define PMAP_LOCK(pmap) mutex_enter(&(pmap)->pm_mtx)
+#define PMAP_LOCK_ASSERT(pmap) KASSERT(mutex_owned(&(pmap)->pm_mtx))
+#define PMAP_LOCK_DESTROY(pmap) mutex_destroy(&(pmap)->pm_mtx)
+#define PMAP_LOCK_INIT(pmap) mutex_init(&(pmap)->pm_mtx, MUTEX_DEFAULT, IPL_NONE)
+#define PMAP_LOCKED(pmap) mutex_owned(&(pmap)->pm_mtx)
+#define PMAP_MTX(pmap) (&(pmap)->pm_mtx)
+#define PMAP_TRYLOCK(pmap) mutex_tryenter(&(pmap)->pm_mtx)
+#define PMAP_UNLOCK(pmap) mutex_exit(&(pmap)->pm_mtx)
+
+#endif
+
/*
* For each vm_page_t, there is a list of all currently valid virtual
- * mappings of that page. An entry is a pv_entry_t, the list is pv_pvlist.
+ * mappings of that page. An entry is a pv_entry_t, the list is pv_list.
*/
typedef struct pv_entry {
- pmap_t pv_pmap; /* pmap where mapping lies */
- vaddr_t pv_va; /* virtual address for mapping */
+ vaddr_t pv_va; /* virtual address for mapping */
TAILQ_ENTRY(pv_entry) pv_list;
- TAILQ_ENTRY(pv_entry) pv_plist;
} *pv_entry_t;
-/* pvh_attrs */
-#define PGA_MODIFIED 0x01 /* modified */
-#define PGA_REFERENCED 0x02 /* referenced */
-
-
-#define pmap_resident_count(pmap) ((pmap)->pm_stats.resident_count)
-#define pmap_wired_count(pmap) ((pmap)->pm_stats.wired_count)
-
-#define pmap_copy(dp, sp, da, l, sa) /* nothing */
-#define pmap_update(pmap) /* nothing (yet) */
-
void pmap_bootstrap(void);
-#define pmap_is_referenced(pg) \
- (((pg)->mdpage.pvh_attrs & PGA_REFERENCED) != 0)
-#define pmap_is_modified(pg) \
- (((pg)->mdpage.pvh_attrs & PGA_MODIFIED) != 0)
-
+/* optional pmap API functions, according to pmap(9) */
+#define PMAP_STEAL_MEMORY
+#define PMAP_GROWKERNEL
-#define PMAP_STEAL_MEMORY /* enable pmap_steal_memory() */
+#define PMAP_NEED_PROCWR
+void pmap_procwr(struct proc *, vaddr_t, vsize_t);
/*
* Alternate mapping hooks for pool pages. Avoids thrashing the TLB.
*/
+/* XXX
#define PMAP_MAP_POOLPAGE(pa) IA64_PHYS_TO_RR7((pa))
#define PMAP_UNMAP_POOLPAGE(va) IA64_RR_MASK((va))
+*/
/*
* Macros for locking pmap structures.
@@ -139,30 +165,46 @@ void pmap_bootstrap(void);
* operations, locking the kernel pmap is not necessary. Therefore,
* it is not necessary to block interrupts when locking pmap strucutres.
*/
+/* XXX
#define PMAP_LOCK(pmap) mutex_enter(&(pmap)->pm_slock)
#define PMAP_UNLOCK(pmap) mutex_exit(&(pmap)->pm_slock)
+*/
-
-#define PMAP_VHPT_LOG2SIZE 16
-
-
-#include <sys/queue.h>
-#include <sys/mutex.h>
/*
* pmap-specific data store in the vm_page structure.
*/
#define __HAVE_VM_PAGE_MD
+
struct vm_page_md {
- TAILQ_HEAD(,pv_entry) pv_list; /* pv_entry list */
- int pv_list_count;
- kmutex_t pv_mutex; /* lock on this head */
- int pvh_attrs; /* page attributes */
+ TAILQ_HEAD(,pv_entry) pv_list;
+ vm_memattr_t memattr;
+#if 0 /* XXX freebsd */
+ uint8_t pv_flags;
+ uint8_t aflags;
+#endif
};
#define VM_MDPAGE_INIT(pg) \
do { \
TAILQ_INIT(&(pg)->mdpage.pv_list); \
- mutex_init(&(pg)->mdpage.pv_mutex, MUTEX_DEFAULT, IPL_NONE); \
+ (pg)->mdpage.memattr = VM_MEMATTR_DEFAULT; \
} while (/*CONSTCOND*/0)
-#endif /* _PMAP_MACHINE_ */
+#ifdef _KERNEL
+
+extern uint64_t pmap_vhpt_base[];
+extern uint64_t pmap_vhpt_log2size;
+
+vaddr_t pmap_page_to_va(struct vm_page*);
+
+/* Machine-architecture private */
+vaddr_t pmap_alloc_vhpt(void);
+void pmap_bootstrap(void);
+void pmap_invalidate_all(void);
+paddr_t pmap_kextract(vaddr_t va);
+struct pmap *pmap_switch(struct pmap *pmap);
+void pmap_remove_all_phys(struct vm_page*); /* used in only pmap_page_protect */
+
+#endif /* _KERNEL */
+
+#endif /* _IA64_PMAP_H_ */
