Module Name: src
Committed By: snj
Date: Wed Mar 17 03:10:40 UTC 2010
Modified Files:
src/sys/arch/sparc64/include [netbsd-5]: pmap.h
src/sys/arch/sparc64/sparc64 [netbsd-5]: cache.h ipifuncs.c locore.s
machdep.c pmap.c
Log Message:
Apply patch (requested by mrg in ticket #1343):
- flush the dcache around pmap_{zero,copy}_page()
- convert all blast_dcache() / dcache_flush_page() calls to
properly handle flushes in all cpus as necessary
To generate a diff of this commit:
cvs rdiff -u -r1.40 -r1.40.14.1 src/sys/arch/sparc64/include/pmap.h
cvs rdiff -u -r1.10 -r1.10.66.1 src/sys/arch/sparc64/sparc64/cache.h
cvs rdiff -u -r1.22 -r1.22.8.1 src/sys/arch/sparc64/sparc64/ipifuncs.c
cvs rdiff -u -r1.286.2.1 -r1.286.2.2 src/sys/arch/sparc64/sparc64/locore.s
cvs rdiff -u -r1.227.4.1 -r1.227.4.2 src/sys/arch/sparc64/sparc64/machdep.c
cvs rdiff -u -r1.225.4.1 -r1.225.4.2 src/sys/arch/sparc64/sparc64/pmap.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/sparc64/include/pmap.h
diff -u src/sys/arch/sparc64/include/pmap.h:1.40 src/sys/arch/sparc64/include/pmap.h:1.40.14.1
--- src/sys/arch/sparc64/include/pmap.h:1.40 Fri Mar 14 15:40:02 2008
+++ src/sys/arch/sparc64/include/pmap.h Wed Mar 17 03:10:39 2010
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.h,v 1.40 2008/03/14 15:40:02 nakayama Exp $ */
+/* $NetBSD: pmap.h,v 1.40.14.1 2010/03/17 03:10:39 snj Exp $ */
/*-
* Copyright (C) 1995, 1996 Wolfgang Solfrank.
@@ -212,6 +212,9 @@
void sp_tlb_flush_ctx(int);
void sp_tlb_flush_all(void);
+void pmap_copy_page_phys(paddr_t, paddr_t);
+void pmap_zero_page_phys(paddr_t);
+
#ifdef MULTIPROCESSOR
void smp_tlb_flush_pte(vaddr_t, pmap_t);
void smp_tlb_flush_ctx(pmap_t);
Index: src/sys/arch/sparc64/sparc64/cache.h
diff -u src/sys/arch/sparc64/sparc64/cache.h:1.10 src/sys/arch/sparc64/sparc64/cache.h:1.10.66.1
--- src/sys/arch/sparc64/sparc64/cache.h:1.10 Sat Oct 21 23:49:29 2006
+++ src/sys/arch/sparc64/sparc64/cache.h Wed Mar 17 03:10:39 2010
@@ -1,4 +1,4 @@
-/* $NetBSD: cache.h,v 1.10 2006/10/21 23:49:29 mrg Exp $ */
+/* $NetBSD: cache.h,v 1.10.66.1 2010/03/17 03:10:39 snj Exp $ */
/*
* Copyright (c) 1996
@@ -75,11 +75,21 @@
/* The following are for I$ and D$ flushes and are in locore.s */
void dcache_flush_page(paddr_t); /* flush page from D$ */
void icache_flush_page(paddr_t); /* flush page from I$ */
-void blast_dcache(void); /* Clear entire D$ */
+void sp_blast_dcache(void); /* Clear entire D$ */
void blast_icache(void); /* Clear entire I$ */
/* The following flush a range from the D$ and I$ but not E$. */
void cache_flush_phys(paddr_t, psize_t, int);
+#ifdef MULTIPROCESSOR
+void smp_blast_dcache(sparc64_cpuset_t);
+void smp_dcache_flush_page_all(paddr_t pa);
+#define dcache_flush_page_all(pa) smp_dcache_flush_page_all(pa)
+#define blast_dcache() smp_blast_dcache(cpus_active)
+#else
+#define dcache_flush_page_all(pa) dcache_flush_page(pa)
+#define blast_dcache() sp_blast_dcache()
+#endif
+
/* Smallest E$ line size. */
extern int ecache_min_line_size;
Index: src/sys/arch/sparc64/sparc64/ipifuncs.c
diff -u src/sys/arch/sparc64/sparc64/ipifuncs.c:1.22 src/sys/arch/sparc64/sparc64/ipifuncs.c:1.22.8.1
--- src/sys/arch/sparc64/sparc64/ipifuncs.c:1.22 Sat May 31 08:00:34 2008
+++ src/sys/arch/sparc64/sparc64/ipifuncs.c Wed Mar 17 03:10:39 2010
@@ -1,4 +1,4 @@
-/* $NetBSD: ipifuncs.c,v 1.22 2008/05/31 08:00:34 nakayama Exp $ */
+/* $NetBSD: ipifuncs.c,v 1.22.8.1 2010/03/17 03:10:39 snj Exp $ */
/*-
* Copyright (c) 2004 The NetBSD Foundation, Inc.
@@ -27,7 +27,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ipifuncs.c,v 1.22 2008/05/31 08:00:34 nakayama Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ipifuncs.c,v 1.22.8.1 2010/03/17 03:10:39 snj Exp $");
#include "opt_ddb.h"
@@ -44,6 +44,8 @@
#include <machine/pmap.h>
#include <machine/sparc64.h>
+#include <sparc64/sparc64/cache.h>
+
#if defined(DDB) || defined(KGDB)
#ifdef DDB
#include <ddb/db_command.h>
@@ -69,6 +71,8 @@
void sparc64_ipi_flush_pte(void *);
void sparc64_ipi_flush_ctx(void *);
void sparc64_ipi_flush_all(void *);
+void sparc64_ipi_dcache_flush_page(void *);
+void sparc64_ipi_blast_dcache(void *);
/*
* Process cpu stop-self event.
@@ -412,6 +416,34 @@
sparc64_broadcast_ipi(sparc64_ipi_flush_all, 0, 0);
}
+/* XXX Spitfire specific for netbsd-5 branch */
+#define dcache_line_size 32
+#define dcache_size (16 * 1024)
+
+/*
+ * Make sure this page is flushed from all CPUs.
+ */
+void
+smp_dcache_flush_page_all(paddr_t pa)
+{
+
+ sparc64_broadcast_ipi(sparc64_ipi_dcache_flush_page, pa,
+ dcache_line_size);
+ dcache_flush_page(pa);
+}
+
+/*
+ * Flush the D$ on this set of CPUs.
+ */
+void
+smp_blast_dcache(sparc64_cpuset_t activecpus)
+{
+
+ sparc64_multicast_ipi(activecpus, sparc64_ipi_blast_dcache,
+ dcache_size, dcache_line_size);
+ sp_blast_dcache();
+}
+
/*
* Print an error message.
*/
Index: src/sys/arch/sparc64/sparc64/locore.s
diff -u src/sys/arch/sparc64/sparc64/locore.s:1.286.2.1 src/sys/arch/sparc64/sparc64/locore.s:1.286.2.2
--- src/sys/arch/sparc64/sparc64/locore.s:1.286.2.1 Fri Jun 5 18:28:34 2009
+++ src/sys/arch/sparc64/sparc64/locore.s Wed Mar 17 03:10:39 2010
@@ -1,4 +1,4 @@
-/* $NetBSD: locore.s,v 1.286.2.1 2009/06/05 18:28:34 snj Exp $ */
+/* $NetBSD: locore.s,v 1.286.2.2 2010/03/17 03:10:39 snj Exp $ */
/*
* Copyright (c) 1996-2002 Eduardo Horvath
@@ -5648,14 +5648,14 @@
#endif
/*
- * blast_dcache()
+ * sp_blast_dcache()
*
* Clear out all of D$ regardless of contents
* Does not modify %o0
*
*/
.align 8
-ENTRY(blast_dcache)
+ENTRY(sp_blast_dcache)
/*
* We turn off interrupts for the duration to prevent RED exceptions.
*/
@@ -5683,6 +5683,33 @@
wrpr %o3, %pstate
#endif
+#ifdef MULTIPROCESSOR
+/*
+ * void sparc64_ipi_blast_dcache(int dcache_size, int dcache_line_size)
+ *
+ * Clear out all of D$ regardless of contents
+ *
+ * On entry:
+ * %g2 = dcache_size
+ * %g3 = dcache_line_size
+ */
+ .align 8
+ENTRY(sparc64_ipi_blast_dcache)
+ sub %g2, %g3, %g2
+1:
+ stxa %g0, [%g2] ASI_DCACHE_TAG
+ membar #Sync
+ brnz,pt %g2, 1b
+ sub %g2, %g3, %g2
+
+ sethi %hi(KERNBASE), %g5
+ flush %g5
+ membar #Sync
+
+ ba,a ret_from_intr_vector
+ nop
+#endif /* MULTIPROCESSOR */
+
/*
* blast_icache()
*
@@ -7112,22 +7139,19 @@
membar #StoreStore|#StoreLoad
/*
- * pmap_zero_page(pa)
+ * pmap_zero_page_phys(pa)
*
* Zero one page physically addressed
*
* Block load/store ASIs do not exist for physical addresses,
* so we won't use them.
*
- * While we do the zero operation, we also need to blast away
- * the contents of the D$. We will execute a flush at the end
- * to sync the I$.
+ * We will execute a flush at the end to sync the I$.
+ *
+ * This version expects to have the dcache_flush_page_all(pa)
+ * to have been called before calling into here.
*/
- .data
-paginuse:
- .word 0
- .text
-ENTRY(pmap_zero_page)
+ENTRY(pmap_zero_page_phys)
#ifndef _LP64
COMBINE(%o0, %o1, %o0)
#endif
@@ -7171,7 +7195,7 @@
wr %g0, ASI_PRIMARY_NOFAULT, %asi ! Make C code happy
/*
- * pmap_copy_page(paddr_t src, paddr_t dst)
+ * pmap_copy_page_phys(paddr_t src, paddr_t dst)
*
* Copy one page physically addressed
* We need to use a global reg for ldxa/stxa
@@ -7180,10 +7204,11 @@
* 32-bit stack. We will unroll the loop by 4 to
* improve performance.
*
- * XXX We also need to blast the D$ and flush like
- * XXX pmap_zero_page.
+ * This version expects to have the dcache_flush_page_all(pa)
+ * to have been called before calling into here.
+ *
*/
-ENTRY(pmap_copy_page)
+ENTRY(pmap_copy_page_phys)
#ifndef _LP64
COMBINE(%o0, %o1, %o0)
COMBINE(%o2, %o3, %o1)
@@ -7239,6 +7264,7 @@
retl
mov %o4, %g1 ! Restore g1
#endif
+
/*
* extern int64_t pseg_get(struct pmap *pm, vaddr_t addr);
*
@@ -9592,6 +9618,48 @@
IPIEVC_INC(IPI_EVCNT_FPU_FLUSH,%g2,%g3)
ba,a ret_from_intr_vector
nop
+
+/*
+ * IPI handler to drop the current FPU state.
+ * void sparc64_ipi_dcache_flush_page(paddr_t pa, int line_size)
+ *
+ * On entry:
+ * %g2 = pa
+ * %g3 = line_size
+ */
+ENTRY(sparc64_ipi_dcache_flush_page)
+ mov -1, %g1 ! Generate mask for tag: bits [29..2]
+ srlx %g2, 13-2, %g5 ! Tag is PA bits <40:13> in bits <29:2>
+ clr %g4
+ srl %g1, 2, %g1 ! Now we have bits <29:0> set
+ set (2*NBPG), %g7
+ ba,pt %icc, 1f
+ andn %g1, 3, %g1 ! Now we have bits <29:2> set
+
+ .align 8
+1:
+ ldxa [%g4] ASI_DCACHE_TAG, %g6
+ mov %g4, %g2
+ deccc 32, %g7
+ bl,pn %icc, 2f
+ inc 32, %g4
+
+ xor %g6, %g5, %g6
+ andcc %g6, %g1, %g0
+ bne,pt %xcc, 1b
+ membar #LoadStore
+
+ stxa %g0, [%g2] ASI_DCACHE_TAG
+ ba,pt %icc, 1b
+ membar #StoreLoad
+2:
+
+ sethi %hi(KERNBASE), %g5
+ flush %g5
+ membar #Sync
+ ba,a ret_from_intr_vector
+ nop
+
#endif
/*
Index: src/sys/arch/sparc64/sparc64/machdep.c
diff -u src/sys/arch/sparc64/sparc64/machdep.c:1.227.4.1 src/sys/arch/sparc64/sparc64/machdep.c:1.227.4.2
--- src/sys/arch/sparc64/sparc64/machdep.c:1.227.4.1 Mon Feb 2 03:30:33 2009
+++ src/sys/arch/sparc64/sparc64/machdep.c Wed Mar 17 03:10:39 2010
@@ -1,4 +1,4 @@
-/* $NetBSD: machdep.c,v 1.227.4.1 2009/02/02 03:30:33 snj Exp $ */
+/* $NetBSD: machdep.c,v 1.227.4.2 2010/03/17 03:10:39 snj Exp $ */
/*-
* Copyright (c) 1996, 1997, 1998 The NetBSD Foundation, Inc.
@@ -71,7 +71,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.227.4.1 2009/02/02 03:30:33 snj Exp $");
+__KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.227.4.2 2010/03/17 03:10:39 snj Exp $");
#include "opt_ddb.h"
#include "opt_multiprocessor.h"
@@ -1291,7 +1291,7 @@
* We should be flushing a subrange, but we
* don't know where the segments starts.
*/
- dcache_flush_page(pa);
+ dcache_flush_page_all(pa);
}
}
Index: src/sys/arch/sparc64/sparc64/pmap.c
diff -u src/sys/arch/sparc64/sparc64/pmap.c:1.225.4.1 src/sys/arch/sparc64/sparc64/pmap.c:1.225.4.2
--- src/sys/arch/sparc64/sparc64/pmap.c:1.225.4.1 Sat Nov 22 16:44:25 2008
+++ src/sys/arch/sparc64/sparc64/pmap.c Wed Mar 17 03:10:40 2010
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.c,v 1.225.4.1 2008/11/22 16:44:25 snj Exp $ */
+/* $NetBSD: pmap.c,v 1.225.4.2 2010/03/17 03:10:40 snj Exp $ */
/*
*
* Copyright (C) 1996-1999 Eduardo Horvath.
@@ -26,7 +26,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.225.4.1 2008/11/22 16:44:25 snj Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.225.4.2 2010/03/17 03:10:40 snj Exp $");
#undef NO_VCACHE /* Don't forget the locked TLB in dostart */
#define HWREF
@@ -1401,6 +1401,7 @@
nextpg = TAILQ_NEXT(pg, listq.queue);
TAILQ_REMOVE(&pm->pm_obj.memq, pg, listq.queue);
KASSERT(pg->mdpage.mdpg_pvh.pv_pmap == NULL);
+ dcache_flush_page_all(VM_PAGE_TO_PHYS(pg));
uvm_pagefree(pg);
}
pmap_free_page((paddr_t)(u_long)pm->pm_segs);
@@ -1681,10 +1682,10 @@
*/
tlb_flush_pte(va, pm);
+ dcache_flush_page_all(pa);
}
if (flush) {
REMOVE_STAT(flushes);
- blast_dcache();
}
}
@@ -1967,6 +1968,7 @@
{
#ifdef MULTIPROCESSOR
struct cpu_info *ci;
+ sparc64_cpuset_t pmap_cpus_active;
#endif
if (pm == pmap_kernel()) {
@@ -1975,17 +1977,25 @@
write_user_windows();
pm->pm_refs = 0;
#ifdef MULTIPROCESSOR
+ CPUSET_CLEAR(pmap_cpus_active);
mutex_enter(&pmap_lock);
for (ci = cpus; ci != NULL; ci = ci->ci_next) {
- if (CPUSET_HAS(cpus_active, ci->ci_index))
+ if (CPUSET_HAS(cpus_active, ci->ci_index)) {
+ if (pm->pm_ctx[ci->ci_index] > 0)
+ CPUSET_ADD(pmap_cpus_active, ci->ci_index);
ctx_free(pm, ci);
+ }
}
mutex_exit(&pmap_lock);
#else
ctx_free(pm);
#endif
REMOVE_STAT(flushes);
- blast_dcache();
+#ifdef MULTIPROCESSOR
+ smp_blast_dcache(pmap_cpus_active);
+#else
+ sp_blast_dcache();
+#endif
}
/*
@@ -2079,10 +2089,10 @@
tsb_invalidate(va, pm);
REMOVE_STAT(tflushes);
tlb_flush_pte(va, pm);
+ dcache_flush_page_all(pa);
}
if (flush && pm->pm_refs) {
REMOVE_STAT(flushes);
- blast_dcache();
}
DPRINTF(PDB_REMOVE, ("\n"));
pv_check();
@@ -2647,7 +2657,7 @@
}
}
}
- dcache_flush_page(VM_PAGE_TO_PHYS(pg));
+ dcache_flush_page_all(VM_PAGE_TO_PHYS(pg));
pv_check();
#ifdef DEBUG
if (pmap_is_referenced_locked(pg)) {
@@ -3020,9 +3030,8 @@
pv->pv_next = NULL;
}
}
- if (needflush) {
- dcache_flush_page(VM_PAGE_TO_PHYS(pg));
- }
+ if (needflush)
+ dcache_flush_page_all(VM_PAGE_TO_PHYS(pg));
}
/* We should really only flush the pages we demapped. */
pv_check();
@@ -3481,6 +3490,7 @@
{
struct vm_page *pg = PHYS_TO_VM_PAGE(pa);
+ dcache_flush_page_all(pa);
uvm_pagefree(pg);
}
@@ -3717,3 +3727,28 @@
pmap->pm_refs = 1;
pmap_activate_pmap(pmap);
}
+
+/*
+ * pmap_copy_page()/pmap_zero_page()
+ *
+ * we make sure that the destination page is flushed from all D$'s
+ * before we perform the copy/zero.
+ */
+extern int cold;
+void
+pmap_copy_page(paddr_t src, paddr_t dst)
+{
+
+ if (!cold)
+ dcache_flush_page_all(dst);
+ pmap_copy_page_phys(src, dst);
+}
+
+void
+pmap_zero_page(paddr_t pa)
+{
+
+ if (!cold)
+ dcache_flush_page_all(pa);
+ pmap_zero_page_phys(pa);
+}
