From: Sean Christopherson <[email protected]> Stub in global variable to enable in-place guest_memfd private<=>shared memory conversion, which will eventually be exposed to userspace via a module param, and wire up the __kvm_mem_is_private() static call to the guest_memfd version when in-place conversion is enabled, i.e. when gmem is the sole authority on private vs. shared memory.
Cc: Fuad Tabba <[email protected]> Cc: Xiaoyao Li <[email protected]> Signed-off-by: Sean Christopherson <[email protected]> Co-developed-by: Ackerley Tng <[email protected]> Signed-off-by: Ackerley Tng <[email protected]> --- include/linux/kvm_host.h | 6 ++++++ virt/kvm/guest_memfd.c | 26 ++++++++++++++++++++++++++ virt/kvm/kvm_main.c | 12 +++++++++++- 3 files changed, 43 insertions(+), 1 deletion(-) diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 65fbce46b63f4..9477ecebbbced 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -2580,6 +2580,8 @@ static inline bool kvm_vm_mem_is_private(struct kvm *kvm, gfn_t gfn) #endif /* CONFIG_KVM_VM_MEMORY_ATTRIBUTES */ #ifdef kvm_arch_has_private_mem +extern bool gmem_in_place_conversion; + typedef bool (kvm_mem_is_private_t)(struct kvm *kvm, gfn_t gfn); DECLARE_STATIC_CALL(__kvm_mem_is_private, kvm_mem_is_private_t); @@ -2588,6 +2590,8 @@ static inline bool kvm_mem_is_private(struct kvm *kvm, gfn_t gfn) return static_call(__kvm_mem_is_private)(kvm, gfn); } #else +#define gmem_in_place_conversion false + static inline bool kvm_mem_is_private(struct kvm *kvm, gfn_t gfn) { return false; @@ -2595,6 +2599,8 @@ static inline bool kvm_mem_is_private(struct kvm *kvm, gfn_t gfn) #endif /* kvm_arch_has_private_mem */ #ifdef CONFIG_KVM_GUEST_MEMFD +bool kvm_gmem_is_private(struct kvm *kvm, gfn_t gfn); + int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot, gfn_t gfn, kvm_pfn_t *pfn, struct page **page, int *max_order); diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 1f1ae61b4878b..75979c885e030 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -475,6 +475,32 @@ static int kvm_gmem_mmap(struct file *file, struct vm_area_struct *vma) return 0; } +bool kvm_gmem_is_private(struct kvm *kvm, gfn_t gfn) +{ + struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn); + struct inode *inode; + + if (!slot) + return 0; + + CLASS(gmem_get_file, file)(slot); + if (!file) + return 0; + + inode = file_inode(file); + + /* + * Rely on the maple tree's internal RCU lock to ensure a stable result. + * This result can become stale as soon as the lock is dropped, so the + * caller _must_ protect consumption of private vs. shared either by + * holding guest_memfd's invalidate lock for the entire duration, or by + * checking mmu_invalidate_retry_gfn() under mmu_lock to serialize + * against concurrent attribute updates. + */ + return kvm_gmem_is_private_mem(inode, kvm_gmem_get_index(slot, gfn)); +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_gmem_is_private); + static struct file_operations kvm_gmem_fops = { .mmap = kvm_gmem_mmap, .open = generic_file_open, diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index a01d336eab18b..0a5a1bee8de0f 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -102,6 +102,10 @@ EXPORT_SYMBOL_FOR_KVM_INTERNAL(halt_poll_ns_shrink); static bool __ro_after_init allow_unsafe_mappings; module_param(allow_unsafe_mappings, bool, 0444); +#ifdef kvm_arch_has_private_mem +bool __ro_after_init gmem_in_place_conversion = false; +#endif + /* * Ordering of locks: * @@ -2423,6 +2427,9 @@ static int kvm_vm_ioctl_clear_dirty_log(struct kvm *kvm, static u64 kvm_supported_vm_mem_attributes(struct kvm *kvm) { #ifdef kvm_arch_has_private_mem + if (gmem_in_place_conversion) + return 0; + if (!kvm || kvm_arch_has_private_mem(kvm)) return KVM_MEMORY_ATTRIBUTE_PRIVATE; #endif @@ -2634,8 +2641,11 @@ EXPORT_STATIC_CALL_GPL(__kvm_mem_is_private); static void kvm_init_memory_attributes(void) { + if (gmem_in_place_conversion) + static_call_update(__kvm_mem_is_private, kvm_gmem_is_private); #ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES - static_call_update(__kvm_mem_is_private, kvm_vm_mem_is_private); + else + static_call_update(__kvm_mem_is_private, kvm_vm_mem_is_private); #endif } #else -- 2.55.0.654.g21b8a5bc05-goog
