Module Name: src
Committed By: maxv
Date: Thu Mar 14 20:29:53 UTC 2019
Modified Files:
src/sys/dev/nvmm/x86: nvmm_x86_vmx.c
Log Message:
Optimize NVMM-Intel: keep the VMCS active on the host CPU, and lazy-switch
it on demand only when needed. This allows the CPU to use the cached
version of the guest state, rather than the in-memory copy of it. This is
much more performant.
A VMCS must be active on only one CPU, but one CPU can have several active
VMCSs at the same time.
We keep track of which CPU each VMCS is active on. When we want to execute
a VCPU, we determine whether its VMCS is loaded on another CPU, and if so
send an IPI to ask it to unbusy that VMCS. In most cases the VMCS is
already active on the current CPU, so we don't have to do anything and can
proceed with a fast VMRESUME.
We send IPIs with kpreemption enabled but with a bound LWP, because we
don't want to get context-switched to the CPU we just sent an IPI to.
Overall, with this in place, I see a ~15% performance increase in the
guests on NVMM-Intel.
To generate a diff of this commit:
cvs rdiff -u -r1.18 -r1.19 src/sys/dev/nvmm/x86/nvmm_x86_vmx.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/dev/nvmm/x86/nvmm_x86_vmx.c
diff -u src/sys/dev/nvmm/x86/nvmm_x86_vmx.c:1.18 src/sys/dev/nvmm/x86/nvmm_x86_vmx.c:1.19
--- src/sys/dev/nvmm/x86/nvmm_x86_vmx.c:1.18 Thu Mar 14 19:26:44 2019
+++ src/sys/dev/nvmm/x86/nvmm_x86_vmx.c Thu Mar 14 20:29:53 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: nvmm_x86_vmx.c,v 1.18 2019/03/14 19:26:44 maxv Exp $ */
+/* $NetBSD: nvmm_x86_vmx.c,v 1.19 2019/03/14 20:29:53 maxv Exp $ */
/*
* Copyright (c) 2018 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: nvmm_x86_vmx.c,v 1.18 2019/03/14 19:26:44 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: nvmm_x86_vmx.c,v 1.19 2019/03/14 20:29:53 maxv Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -648,6 +648,8 @@ struct vmx_cpudata {
struct vmcs *vmcs;
paddr_t vmcs_pa;
size_t vmcs_refcnt;
+ struct cpu_info *vmcs_ci;
+ bool vmcs_launched;
/* MSR bitmap */
uint8_t *msrbm;
@@ -762,9 +764,35 @@ vmx_get_revision(void)
}
static void
+vmx_vmclear_ipi(void *arg1, void *arg2)
+{
+ paddr_t vmcs_pa = (paddr_t)arg1;
+ vmx_vmclear(&vmcs_pa);
+}
+
+static void
+vmx_vmclear_remote(struct cpu_info *ci, paddr_t vmcs_pa)
+{
+ uint64_t xc;
+ int bound;
+
+ KASSERT(kpreempt_disabled());
+
+ bound = curlwp_bind();
+ kpreempt_enable();
+
+ xc = xc_unicast(XC_HIGHPRI, vmx_vmclear_ipi, (void *)vmcs_pa, NULL, ci);
+ xc_wait(xc);
+
+ kpreempt_disable();
+ curlwp_bindx(bound);
+}
+
+static void
vmx_vmcs_enter(struct nvmm_cpu *vcpu)
{
struct vmx_cpudata *cpudata = vcpu->cpudata;
+ struct cpu_info *vmcs_ci;
paddr_t oldpa __diagused;
cpudata->vmcs_refcnt++;
@@ -777,12 +805,22 @@ vmx_vmcs_enter(struct nvmm_cpu *vcpu)
return;
}
+ vmcs_ci = cpudata->vmcs_ci;
+ cpudata->vmcs_ci = (void *)0x00FFFFFFFFFFFFFF; /* clobber */
+
kpreempt_disable();
-#ifdef DIAGNOSTIC
- vmx_vmptrst(&oldpa);
- KASSERT(oldpa == 0xFFFFFFFFFFFFFFFF);
-#endif
+ if (vmcs_ci == NULL) {
+ /* This VMCS is loaded for the first time. */
+ vmx_vmclear(&cpudata->vmcs_pa);
+ cpudata->vmcs_launched = false;
+ } else if (vmcs_ci != curcpu()) {
+ /* This VMCS is active on a remote CPU. */
+ vmx_vmclear_remote(vmcs_ci, cpudata->vmcs_pa);
+ cpudata->vmcs_launched = false;
+ } else {
+ /* This VMCS is active on curcpu, nothing to do. */
+ }
vmx_vmptrld(&cpudata->vmcs_pa);
}
@@ -805,6 +843,24 @@ vmx_vmcs_leave(struct nvmm_cpu *vcpu)
return;
}
+ cpudata->vmcs_ci = curcpu();
+ kpreempt_enable();
+}
+
+static void
+vmx_vmcs_destroy(struct nvmm_cpu *vcpu)
+{
+ struct vmx_cpudata *cpudata = vcpu->cpudata;
+ paddr_t oldpa __diagused;
+
+ KASSERT(kpreempt_disabled());
+#ifdef DIAGNOSTIC
+ vmx_vmptrst(&oldpa);
+ KASSERT(oldpa == cpudata->vmcs_pa);
+#endif
+ KASSERT(cpudata->vmcs_refcnt == 1);
+ cpudata->vmcs_refcnt--;
+
vmx_vmclear(&cpudata->vmcs_pa);
kpreempt_enable();
}
@@ -1721,11 +1777,12 @@ vmx_vcpu_run(struct nvmm_machine *mach,
uint64_t intstate;
uint64_t machgen;
int hcpu, s, ret;
- bool launched = false;
+ bool launched;
vmx_vmcs_enter(vcpu);
ci = curcpu();
hcpu = cpu_number();
+ launched = cpudata->vmcs_launched;
vmx_gtlb_catchup(vcpu, hcpu);
vmx_htlb_catchup(vcpu, hcpu);
@@ -1860,6 +1917,8 @@ vmx_vcpu_run(struct nvmm_machine *mach,
}
}
+ cpudata->vmcs_launched = launched;
+
vmx_vcpu_guest_misc_leave(vcpu);
vmx_vcpu_guest_dbregs_leave(vcpu);
@@ -2515,7 +2574,7 @@ vmx_vcpu_destroy(struct nvmm_machine *ma
vmx_vmcs_enter(vcpu);
vmx_asid_free(vcpu);
- vmx_vmcs_leave(vcpu);
+ vmx_vmcs_destroy(vcpu);
kcpuset_destroy(cpudata->htlb_want_flush);
