From: Fred Griffoul <[email protected]>
Introduce a gfn_to_pfn_cache to optimize L1 MSR bitmap access by
replacing map/unmap operations. This optimization reduces overhead
during L2 VM-entry where nested_vmx_prepare_msr_bitmap() merges L1's
MSR intercepts with L0's requirements.
The current implementation maps and unmaps the page on every bitmap
merge, which is a significant performance impact with unmanaged guest
memory (where the map is a memremap/memunmap cycle).
The cache is initialized when entering VMX operation and deactivated
when VMX operation ends. Readers run under kvm->gpc_srcu, following
the check/activate/retry pattern; KVM only ever reads the L1 bitmap,
so the cache is marked never-dirty.
This exports the core pfncache API to kvm-intel.ko, which is its
first modular user.
[dwmw2: Port from the rwlock-protected pfncache to the SRCU reader
protocol, mark the cache never-dirty, rebase over the
kvm_vcpu_map_local_readonly CLASS() conversion.]
Signed-off-by: Fred Griffoul <[email protected]>
Co-developed-by: David Woodhouse <[email protected]>
Signed-off-by: David Woodhouse <[email protected]>
Assisted-by: Claude:claude-mythos-5
---
arch/x86/kvm/vmx/nested.c | 47 +++++++++++++++++++++++++++++++++++----
arch/x86/kvm/vmx/vmx.h | 2 ++
virt/kvm/pfncache.c | 5 +++++
3 files changed, 50 insertions(+), 4 deletions(-)
diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c
index 151873407abd..5fe7e5d1f72d 100644
--- a/arch/x86/kvm/vmx/nested.c
+++ b/arch/x86/kvm/vmx/nested.c
@@ -320,6 +320,38 @@ static void vmx_switch_vmcs(struct kvm_vcpu *vcpu, struct
loaded_vmcs *vmcs)
kvm_reset_dirty_registers(vcpu);
}
+/*
+ * Map a single page of L1 guest memory at @gpa and enter an SRCU read-side
+ * critical section protecting the mapping. Returns the SRCU index (>= 0) to
+ * pass to nested_gpc_unlock(), or a negative error code. While locked, the
+ * cache's khva and pfn are guaranteed to remain valid: an invalidation
+ * cannot complete until the reader drains.
+ */
+static int nested_gpc_lock(struct gfn_to_pfn_cache *gpc, gpa_t gpa)
+{
+ int idx, err;
+
+ if (!PAGE_ALIGNED(gpa))
+ return -EINVAL;
+retry:
+ idx = srcu_read_lock_atomic(&gpc->kvm->gpc_srcu);
+ if (!kvm_gpc_check(gpc, PAGE_SIZE) || gpc->gpa != gpa) {
+ srcu_read_unlock_atomic(&gpc->kvm->gpc_srcu, idx);
+ err = kvm_gpc_activate(gpc, gpa, PAGE_SIZE);
+ if (err)
+ return err;
+
+ goto retry;
+ }
+
+ return idx;
+}
+
+static void nested_gpc_unlock(struct gfn_to_pfn_cache *gpc, int idx)
+{
+ srcu_read_unlock_atomic(&gpc->kvm->gpc_srcu, idx);
+}
+
static void nested_put_vmcs12_pages(struct kvm_vcpu *vcpu)
{
struct vcpu_vmx *vmx = to_vmx(vcpu);
@@ -370,6 +402,8 @@ static void free_nested(struct kvm_vcpu *vcpu)
kfree(vmx->nested.cached_shadow_vmcs12);
vmx->nested.cached_shadow_vmcs12 = NULL;
+ kvm_gpc_deactivate(&vmx->nested.msr_bitmap_cache);
+
nested_put_vmcs12_pages(vcpu);
kvm_mmu_free_roots(vcpu->kvm, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL);
@@ -734,7 +768,7 @@ static inline bool nested_vmx_prepare_msr_bitmap(struct
kvm_vcpu *vcpu,
struct vmcs12 *vmcs12)
{
struct vcpu_vmx *vmx = to_vmx(vcpu);
- int msr;
+ int msr, idx;
unsigned long *msr_bitmap_l1;
unsigned long *msr_bitmap_l0 = vmx->nested.vmcs02.msr_bitmap;
@@ -759,11 +793,11 @@ static inline bool nested_vmx_prepare_msr_bitmap(struct
kvm_vcpu *vcpu,
return true;
}
- CLASS(kvm_vcpu_map_local_readonly, m)(vcpu,
gpa_to_gfn(vmcs12->msr_bitmap));
- if (m.ret)
+ idx = nested_gpc_lock(&vmx->nested.msr_bitmap_cache,
vmcs12->msr_bitmap);
+ if (idx < 0)
return false;
- msr_bitmap_l1 = (unsigned long *)m.map.hva;
+ msr_bitmap_l1 = (unsigned long *)vmx->nested.msr_bitmap_cache.khva;
/*
* To keep the control flow simple, pay eight 8-byte writes (sixteen
@@ -843,6 +877,8 @@ static inline bool nested_vmx_prepare_msr_bitmap(struct
kvm_vcpu *vcpu,
nested_vmx_merge_pmu_msr_bitmaps(vcpu, msr_bitmap_l1, msr_bitmap_l0);
+ nested_gpc_unlock(&vmx->nested.msr_bitmap_cache, idx);
+
vmx->nested.force_msr_bitmap_recalc = false;
return true;
@@ -5443,6 +5479,9 @@ static int enter_vmx_operation(struct kvm_vcpu *vcpu)
vmx->nested.vpid02 = allocate_vpid();
+ /* KVM only ever reads the L1 MSR bitmap, so never mark it dirty. */
+ __kvm_gpc_init(&vmx->nested.msr_bitmap_cache, vcpu->kvm, true, NULL, 0);
+
/*
* Clear last_vpid to ensure that the VPID is flushed on the first
* nested VM-Enter. Otherwise, stale TLB entries from a previous life of
diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h
index dc8517f15bc4..d5aa5aa83c95 100644
--- a/arch/x86/kvm/vmx/vmx.h
+++ b/arch/x86/kvm/vmx/vmx.h
@@ -146,6 +146,8 @@ struct nested_vmx {
* Guest pages referred to in the vmcs02 with host-physical
* pointers, so we must keep them pinned while L2 runs.
*/
+ struct gfn_to_pfn_cache msr_bitmap_cache;
+
struct kvm_host_map apic_access_page_map;
struct kvm_host_map virtual_apic_map;
struct kvm_host_map pi_desc_map;
diff --git a/virt/kvm/pfncache.c b/virt/kvm/pfncache.c
index 29397ac02668..c634058a73bf 100644
--- a/virt/kvm/pfncache.c
+++ b/virt/kvm/pfncache.c
@@ -241,6 +241,7 @@ bool kvm_gpc_check(struct gfn_to_pfn_cache *gpc, unsigned
long len)
return true;
}
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_gpc_check);
/*
* A mutator's xchg of the state word to zero may have consumed a live
@@ -758,6 +759,7 @@ int kvm_gpc_refresh(struct gfn_to_pfn_cache *gpc, unsigned
long len)
return __kvm_gpc_refresh(gpc, gpc->gpa, uhva);
}
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_gpc_refresh);
void __kvm_gpc_init(struct gfn_to_pfn_cache *gpc, struct kvm *kvm,
bool never_dirty, struct kvm_vcpu *vcpu, u32 vcpu_req)
@@ -774,6 +776,7 @@ void __kvm_gpc_init(struct gfn_to_pfn_cache *gpc, struct
kvm *kvm,
gpc->vcpu = vcpu;
gpc->vcpu_req = vcpu_req;
}
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(__kvm_gpc_init);
static int __kvm_gpc_activate(struct gfn_to_pfn_cache *gpc, gpa_t gpa,
unsigned long uhva,
unsigned long len)
@@ -816,6 +819,7 @@ int kvm_gpc_activate(struct gfn_to_pfn_cache *gpc, gpa_t
gpa, unsigned long len)
return __kvm_gpc_activate(gpc, gpa, KVM_HVA_ERR_BAD, len);
}
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_gpc_activate);
int kvm_gpc_activate_hva(struct gfn_to_pfn_cache *gpc, unsigned long uhva,
unsigned long len)
{
@@ -875,3 +879,4 @@ void kvm_gpc_deactivate(struct gfn_to_pfn_cache *gpc)
gpc_unmap(old_pfn, old_khva);
}
}
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_gpc_deactivate);
--
2.55.0
