Module Name: src
Committed By: maxv
Date: Thu Feb 22 13:27:18 UTC 2018
Modified Files:
src/sys/arch/amd64/amd64: machdep.c
src/sys/arch/amd64/include: pmap.h
src/sys/arch/x86/x86: cpu.c svs.c x86_machdep.c
Log Message:
Remove svs_pgg_update(). Instead of manually changing PG_G on each page,
we can disable the global-paging mechanism in %cr4 with CR4_PGE. Do that.
In addition, install CR4_PGE when SVS is disabled manually (via the
sysctl).
Now, doing "sysctl -w machdep.svs_enabled=0" restores the performance
completely, exactly as if SVS hadn't been enabled in the first place.
To generate a diff of this commit:
cvs rdiff -u -r1.300 -r1.301 src/sys/arch/amd64/amd64/machdep.c
cvs rdiff -u -r1.44 -r1.45 src/sys/arch/amd64/include/pmap.h
cvs rdiff -u -r1.148 -r1.149 src/sys/arch/x86/x86/cpu.c
cvs rdiff -u -r1.7 -r1.8 src/sys/arch/x86/x86/svs.c
cvs rdiff -u -r1.106 -r1.107 src/sys/arch/x86/x86/x86_machdep.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/amd64/amd64/machdep.c
diff -u src/sys/arch/amd64/amd64/machdep.c:1.300 src/sys/arch/amd64/amd64/machdep.c:1.301
--- src/sys/arch/amd64/amd64/machdep.c:1.300 Thu Feb 22 10:26:32 2018
+++ src/sys/arch/amd64/amd64/machdep.c Thu Feb 22 13:27:17 2018
@@ -1,4 +1,4 @@
-/* $NetBSD: machdep.c,v 1.300 2018/02/22 10:26:32 maxv Exp $ */
+/* $NetBSD: machdep.c,v 1.301 2018/02/22 13:27:17 maxv Exp $ */
/*
* Copyright (c) 1996, 1997, 1998, 2000, 2006, 2007, 2008, 2011
@@ -110,7 +110,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.300 2018/02/22 10:26:32 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.301 2018/02/22 13:27:17 maxv Exp $");
/* #define XENDEBUG_LOW */
@@ -1599,7 +1599,7 @@ init_x86_64(paddr_t first_avail)
cpu_probe(&cpu_info_primary);
#ifdef SVS
- svs_init(true);
+ svs_init();
#endif
cpu_init_msrs(&cpu_info_primary, true);
Index: src/sys/arch/amd64/include/pmap.h
diff -u src/sys/arch/amd64/include/pmap.h:1.44 src/sys/arch/amd64/include/pmap.h:1.45
--- src/sys/arch/amd64/include/pmap.h:1.44 Thu Feb 22 09:41:06 2018
+++ src/sys/arch/amd64/include/pmap.h Thu Feb 22 13:27:18 2018
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.h,v 1.44 2018/02/22 09:41:06 maxv Exp $ */
+/* $NetBSD: pmap.h,v 1.45 2018/02/22 13:27:18 maxv Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -221,7 +221,7 @@
void svs_pmap_sync(struct pmap *, int);
void svs_lwp_switch(struct lwp *, struct lwp *);
void svs_pdir_switch(struct pmap *);
-void svs_init(bool);
+void svs_init(void);
extern bool svs_enabled;
#include <x86/pmap.h>
Index: src/sys/arch/x86/x86/cpu.c
diff -u src/sys/arch/x86/x86/cpu.c:1.148 src/sys/arch/x86/x86/cpu.c:1.149
--- src/sys/arch/x86/x86/cpu.c:1.148 Thu Feb 22 08:56:52 2018
+++ src/sys/arch/x86/x86/cpu.c Thu Feb 22 13:27:18 2018
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.c,v 1.148 2018/02/22 08:56:52 maxv Exp $ */
+/* $NetBSD: cpu.c,v 1.149 2018/02/22 13:27:18 maxv Exp $ */
/*
* Copyright (c) 2000-2012 NetBSD Foundation, Inc.
@@ -62,7 +62,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.148 2018/02/22 08:56:52 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.149 2018/02/22 13:27:18 maxv Exp $");
#include "opt_ddb.h"
#include "opt_mpbios.h" /* for MPDEBUG */
@@ -589,6 +589,9 @@ cpu_init(struct cpu_info *ci)
* hardware supports it.
*/
if (cpu_feature[0] & CPUID_PGE)
+#ifdef SVS
+ if (!svs_enabled)
+#endif
cr4 |= CR4_PGE; /* enable global TLB caching */
/*
Index: src/sys/arch/x86/x86/svs.c
diff -u src/sys/arch/x86/x86/svs.c:1.7 src/sys/arch/x86/x86/svs.c:1.8
--- src/sys/arch/x86/x86/svs.c:1.7 Thu Feb 22 11:57:39 2018
+++ src/sys/arch/x86/x86/svs.c Thu Feb 22 13:27:18 2018
@@ -1,4 +1,4 @@
-/* $NetBSD: svs.c,v 1.7 2018/02/22 11:57:39 maxv Exp $ */
+/* $NetBSD: svs.c,v 1.8 2018/02/22 13:27:18 maxv Exp $ */
/*
* Copyright (c) 2018 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: svs.c,v 1.7 2018/02/22 11:57:39 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: svs.c,v 1.8 2018/02/22 13:27:18 maxv Exp $");
#include "opt_svs.h"
@@ -437,77 +437,6 @@ svs_pdir_switch(struct pmap *pmap)
}
static void
-svs_pgg_scanlvl(bool enable, int lvl, vaddr_t *levels)
-{
- pd_entry_t *pde = (pd_entry_t *)(levels[lvl-1]);
- pt_entry_t set, rem;
- size_t i, start;
- paddr_t pa;
- int nlvl;
-
- set = enable ? PG_G : 0;
- rem = enable ? 0 : PG_G;
-
- start = (lvl == 4) ? 256 : 0;
-
- for (i = start; i < 512; i++) {
- if (!pmap_valid_entry(pde[i])) {
- continue;
- }
- if (lvl == 1) {
- pde[i] = (pde[i] & ~rem) | set;
- } else if (pde[i] & PG_PS) {
- pde[i] = (pde[i] & ~rem) | set;
- } else {
- pa = (paddr_t)(pde[i] & PG_FRAME);
- nlvl = lvl - 1;
-
- /* remove the previous mapping */
- pmap_kremove_local(levels[nlvl-1], PAGE_SIZE);
-
- /* kenter the lower level */
- pmap_kenter_pa(levels[nlvl-1], pa,
- VM_PROT_READ|VM_PROT_WRITE, 0);
- pmap_update(pmap_kernel());
-
- /* go to the lower level */
- svs_pgg_scanlvl(enable, nlvl, levels);
- }
- }
-}
-
-static void
-svs_pgg_update(bool enable)
-{
- const paddr_t pa = pmap_pdirpa(pmap_kernel(), 0);
- vaddr_t levels[4];
- size_t i;
-
- if (!(cpu_feature[0] & CPUID_PGE)) {
- return;
- }
-
- pmap_pg_g = enable ? PG_G : 0;
-
- for (i = 0; i < 4; i++) {
- levels[i] = uvm_km_alloc(kernel_map, PAGE_SIZE, 0,
- UVM_KMF_VAONLY);
- }
-
- pmap_kenter_pa(levels[3], pa, VM_PROT_READ|VM_PROT_WRITE, 0);
- pmap_update(pmap_kernel());
-
- svs_pgg_scanlvl(enable, 4, levels);
-
- for (i = 0; i < 4; i++) {
- pmap_kremove_local(levels[i], PAGE_SIZE);
- uvm_km_free(kernel_map, levels[i], PAGE_SIZE, UVM_KMF_VAONLY);
- }
-
- tlbflushg();
-}
-
-static void
svs_enable(void)
{
extern uint8_t svs_enter, svs_enter_end;
@@ -604,6 +533,10 @@ svs_disable_cpu(void *arg1, void *arg2)
/* put back the non-SVS syscall entry point */
wrmsr(MSR_LSTAR, (uint64_t)Xsyscall);
+ /* enable global pages */
+ if (cpu_feature[0] & CPUID_PGE)
+ lcr4(rcr4() | CR4_PGE);
+
atomic_dec_ulong(&svs_cpu_barrier2);
while (atomic_cas_ulong(&svs_cpu_barrier2, 0, 0) != 0) {
x86_pause();
@@ -641,16 +574,10 @@ svs_disable(void)
svs_cpu_barrier1 = ncpu;
svs_cpu_barrier2 = ncpu;
- printf("[+] Disabling SVS\n");
+ printf("[+] Disabling SVS...");
xc = xc_broadcast(0, svs_disable_cpu, NULL, NULL);
xc_wait(xc);
-
- /*
- * XXX printf("[+] Installing PG_G\n");
- * XXX svs_pgg_update(true);
- */
-
- printf("[+] Done\n");
+ printf(" done!\n");
mutex_exit(&cpu_lock);
@@ -687,19 +614,10 @@ sysctl_machdep_svs_enabled(SYSCTLFN_ARGS
}
void
-svs_init(bool early)
+svs_init(void)
{
- /*
- * When early, declare that we want to use SVS, and hotpatch the
- * entry points. When late, remove PG_G from the page tables.
- */
- if (early) {
- if (cpu_vendor != CPUVENDOR_INTEL) {
- return;
- }
- svs_enable();
- } else if (svs_enabled) {
- svs_pgg_update(false);
+ if (cpu_vendor != CPUVENDOR_INTEL) {
+ return;
}
+ svs_enable();
}
-
Index: src/sys/arch/x86/x86/x86_machdep.c
diff -u src/sys/arch/x86/x86/x86_machdep.c:1.106 src/sys/arch/x86/x86/x86_machdep.c:1.107
--- src/sys/arch/x86/x86/x86_machdep.c:1.106 Thu Feb 22 10:42:11 2018
+++ src/sys/arch/x86/x86/x86_machdep.c Thu Feb 22 13:27:18 2018
@@ -1,4 +1,4 @@
-/* $NetBSD: x86_machdep.c,v 1.106 2018/02/22 10:42:11 maxv Exp $ */
+/* $NetBSD: x86_machdep.c,v 1.107 2018/02/22 13:27:18 maxv Exp $ */
/*-
* Copyright (c) 2002, 2006, 2007 YAMAMOTO Takashi,
@@ -31,7 +31,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: x86_machdep.c,v 1.106 2018/02/22 10:42:11 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: x86_machdep.c,v 1.107 2018/02/22 13:27:18 maxv Exp $");
#include "opt_modular.h"
#include "opt_physmem.h"
@@ -1091,9 +1091,6 @@ machdep_init(void)
void
x86_startup(void)
{
-#if SVS
- svs_init(false);
-#endif
#if !defined(XEN)
nmi_init();
pmc_init();
