Author: markj
Date: Wed Jul 25 15:40:27 2018
New Revision: 336711
URL: https://svnweb.freebsd.org/changeset/base/336711
Log:
MFC r336460:
Port r324665 and r325285 to arm64.
Modified:
stable/11/sys/arm64/arm64/pmap.c
stable/11/sys/arm64/include/pmap.h
Directory Properties:
stable/11/ (props changed)
Modified: stable/11/sys/arm64/arm64/pmap.c
==============================================================================
--- stable/11/sys/arm64/arm64/pmap.c Wed Jul 25 15:14:35 2018
(r336710)
+++ stable/11/sys/arm64/arm64/pmap.c Wed Jul 25 15:40:27 2018
(r336711)
@@ -1851,11 +1851,11 @@ SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_spare, CTLFLAG
static vm_page_t
reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp)
{
- struct pch new_tail;
- struct pv_chunk *pc;
+ struct pv_chunk *pc, *pc_marker, *pc_marker_end;
+ struct pv_chunk_header pc_marker_b, pc_marker_end_b;
struct md_page *pvh;
pd_entry_t *pde;
- pmap_t pmap;
+ pmap_t next_pmap, pmap;
pt_entry_t *pte, tpte;
pv_entry_t pv;
vm_offset_t va;
@@ -1863,31 +1863,65 @@ reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **l
struct spglist free;
uint64_t inuse;
int bit, field, freed, lvl;
+ static int active_reclaims = 0;
PMAP_LOCK_ASSERT(locked_pmap, MA_OWNED);
KASSERT(lockp != NULL, ("reclaim_pv_chunk: lockp is NULL"));
+
pmap = NULL;
m_pc = NULL;
SLIST_INIT(&free);
- TAILQ_INIT(&new_tail);
+ bzero(&pc_marker_b, sizeof(pc_marker_b));
+ bzero(&pc_marker_end_b, sizeof(pc_marker_end_b));
+ pc_marker = (struct pv_chunk *)&pc_marker_b;
+ pc_marker_end = (struct pv_chunk *)&pc_marker_end_b;
+
mtx_lock(&pv_chunks_mutex);
- while ((pc = TAILQ_FIRST(&pv_chunks)) != NULL && SLIST_EMPTY(&free)) {
- TAILQ_REMOVE(&pv_chunks, pc, pc_lru);
+ active_reclaims++;
+ TAILQ_INSERT_HEAD(&pv_chunks, pc_marker, pc_lru);
+ TAILQ_INSERT_TAIL(&pv_chunks, pc_marker_end, pc_lru);
+ while ((pc = TAILQ_NEXT(pc_marker, pc_lru)) != pc_marker_end &&
+ SLIST_EMPTY(&free)) {
+ next_pmap = pc->pc_pmap;
+ if (next_pmap == NULL) {
+ /*
+ * The next chunk is a marker. However, it is
+ * not our marker, so active_reclaims must be
+ * > 1. Consequently, the next_chunk code
+ * will not rotate the pv_chunks list.
+ */
+ goto next_chunk;
+ }
mtx_unlock(&pv_chunks_mutex);
- if (pmap != pc->pc_pmap) {
+
+ /*
+ * A pv_chunk can only be removed from the pc_lru list
+ * when both pv_chunks_mutex is owned and the
+ * corresponding pmap is locked.
+ */
+ if (pmap != next_pmap) {
if (pmap != NULL && pmap != locked_pmap)
PMAP_UNLOCK(pmap);
- pmap = pc->pc_pmap;
+ pmap = next_pmap;
/* Avoid deadlock and lock recursion. */
if (pmap > locked_pmap) {
RELEASE_PV_LIST_LOCK(lockp);
PMAP_LOCK(pmap);
- } else if (pmap != locked_pmap &&
- !PMAP_TRYLOCK(pmap)) {
- pmap = NULL;
- TAILQ_INSERT_TAIL(&new_tail, pc, pc_lru);
mtx_lock(&pv_chunks_mutex);
continue;
+ } else if (pmap != locked_pmap) {
+ if (PMAP_TRYLOCK(pmap)) {
+ mtx_lock(&pv_chunks_mutex);
+ continue;
+ } else {
+ pmap = NULL; /* pmap is not locked */
+ mtx_lock(&pv_chunks_mutex);
+ pc = TAILQ_NEXT(pc_marker, pc_lru);
+ if (pc == NULL ||
+ pc->pc_pmap != next_pmap)
+ continue;
+ goto next_chunk;
+ }
}
}
@@ -1933,9 +1967,8 @@ reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **l
}
}
if (freed == 0) {
- TAILQ_INSERT_TAIL(&new_tail, pc, pc_lru);
mtx_lock(&pv_chunks_mutex);
- continue;
+ goto next_chunk;
}
/* Every freed mapping is for a 4 KB page. */
pmap_resident_count_dec(pmap, freed);
@@ -1952,16 +1985,36 @@ reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **l
m_pc = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pc));
dump_drop_page(m_pc->phys_addr);
mtx_lock(&pv_chunks_mutex);
+ TAILQ_REMOVE(&pv_chunks, pc, pc_lru);
break;
}
TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
- TAILQ_INSERT_TAIL(&new_tail, pc, pc_lru);
mtx_lock(&pv_chunks_mutex);
/* One freed pv entry in locked_pmap is sufficient. */
if (pmap == locked_pmap)
break;
+
+next_chunk:
+ TAILQ_REMOVE(&pv_chunks, pc_marker, pc_lru);
+ TAILQ_INSERT_AFTER(&pv_chunks, pc, pc_marker, pc_lru);
+ if (active_reclaims == 1 && pmap != NULL) {
+ /*
+ * Rotate the pv chunks list so that we do not
+ * scan the same pv chunks that could not be
+ * freed (because they contained a wired
+ * and/or superpage mapping) on every
+ * invocation of reclaim_pv_chunk().
+ */
+ while ((pc = TAILQ_FIRST(&pv_chunks)) != pc_marker) {
+ MPASS(pc->pc_pmap != NULL);
+ TAILQ_REMOVE(&pv_chunks, pc, pc_lru);
+ TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru);
+ }
+ }
}
- TAILQ_CONCAT(&pv_chunks, &new_tail, pc_lru);
+ TAILQ_REMOVE(&pv_chunks, pc_marker, pc_lru);
+ TAILQ_REMOVE(&pv_chunks, pc_marker_end, pc_lru);
+ active_reclaims--;
mtx_unlock(&pv_chunks_mutex);
if (pmap != NULL && pmap != locked_pmap)
PMAP_UNLOCK(pmap);
Modified: stable/11/sys/arm64/include/pmap.h
==============================================================================
--- stable/11/sys/arm64/include/pmap.h Wed Jul 25 15:14:35 2018
(r336710)
+++ stable/11/sys/arm64/include/pmap.h Wed Jul 25 15:40:27 2018
(r336711)
@@ -84,6 +84,7 @@ struct pmap {
TAILQ_HEAD(,pv_chunk) pm_pvchunk; /* list of mappings in pmap */
struct vm_radix pm_root; /* spare page table pages */
};
+typedef struct pmap *pmap_t;
typedef struct pv_entry {
vm_offset_t pv_va; /* virtual address for mapping */
@@ -96,15 +97,20 @@ typedef struct pv_entry {
*/
#define _NPCM 3
#define _NPCPV 168
-struct pv_chunk {
- struct pmap * pc_pmap;
- TAILQ_ENTRY(pv_chunk) pc_list;
- uint64_t pc_map[_NPCM]; /* bitmap; 1 = free */
+#define PV_CHUNK_HEADER
\
+ pmap_t pc_pmap; \
+ TAILQ_ENTRY(pv_chunk) pc_list; \
+ uint64_t pc_map[_NPCM]; /* bitmap; 1 = free */ \
TAILQ_ENTRY(pv_chunk) pc_lru;
- struct pv_entry pc_pventry[_NPCPV];
+
+struct pv_chunk_header {
+ PV_CHUNK_HEADER
};
-typedef struct pmap *pmap_t;
+struct pv_chunk {
+ PV_CHUNK_HEADER
+ struct pv_entry pc_pventry[_NPCPV];
+};
#ifdef _KERNEL
extern struct pmap kernel_pmap_store;
_______________________________________________
svn-src-all@freebsd.org mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-all
To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"
