perf report shows heavy overhead from down/up of slots_lock. Attempted to remove slots_lock by having vcpus stop on a synchronization point, but this introduced further complexity (a vcpu can be scheduled out before reaching the synchronization point, and can sched back in at points which are slots_lock protected, etc).
This patch changes vcpu_enter_guest to conditionally release/acquire slots_lock in case a vcpu state bit is set. vmexit performance improves by 5-10% on UP guest. Signed-off-by: Marcelo Tosatti <[email protected]> Index: kvm-requests/arch/x86/kvm/vmx.c =================================================================== --- kvm-requests.orig/arch/x86/kvm/vmx.c +++ kvm-requests/arch/x86/kvm/vmx.c @@ -2169,7 +2169,7 @@ static int alloc_apic_access_page(struct struct kvm_userspace_memory_region kvm_userspace_mem; int r = 0; - down_write(&kvm->slots_lock); + kvm_grab_global_lock(kvm); if (kvm->arch.apic_access_page) goto out; kvm_userspace_mem.slot = APIC_ACCESS_PAGE_PRIVATE_MEMSLOT; @@ -2191,7 +2191,7 @@ static int alloc_identity_pagetable(stru struct kvm_userspace_memory_region kvm_userspace_mem; int r = 0; - down_write(&kvm->slots_lock); + kvm_grab_global_lock(kvm); if (kvm->arch.ept_identity_pagetable) goto out; kvm_userspace_mem.slot = IDENTITY_PAGETABLE_PRIVATE_MEMSLOT; Index: kvm-requests/arch/x86/kvm/x86.c =================================================================== --- kvm-requests.orig/arch/x86/kvm/x86.c +++ kvm-requests/arch/x86/kvm/x86.c @@ -1926,7 +1926,7 @@ static int kvm_vm_ioctl_set_nr_mmu_pages if (kvm_nr_mmu_pages < KVM_MIN_ALLOC_MMU_PAGES) return -EINVAL; - down_write(&kvm->slots_lock); + kvm_grab_global_lock(kvm); spin_lock(&kvm->mmu_lock); kvm_mmu_change_mmu_pages(kvm, kvm_nr_mmu_pages); @@ -1982,7 +1982,7 @@ static int kvm_vm_ioctl_set_memory_alias < alias->target_phys_addr) goto out; - down_write(&kvm->slots_lock); + kvm_grab_global_lock(kvm); spin_lock(&kvm->mmu_lock); p = &kvm->arch.aliases[alias->slot]; @@ -2137,7 +2137,7 @@ int kvm_vm_ioctl_get_dirty_log(struct kv struct kvm_memory_slot *memslot; int is_dirty = 0; - down_write(&kvm->slots_lock); + kvm_grab_global_lock(kvm); r = kvm_get_dirty_log(kvm, log, &is_dirty); if (r) @@ -2253,7 +2253,7 @@ long kvm_arch_vm_ioctl(struct file *filp sizeof(struct kvm_pit_config))) goto out; create_pit: - down_write(&kvm->slots_lock); + kvm_grab_global_lock(kvm); r = -EEXIST; if (kvm->arch.vpit) goto create_pit_unlock; @@ -3548,7 +3548,7 @@ static void inject_pending_event(struct static int vcpu_enter_guest(struct kvm_vcpu *vcpu) { - int r; + int r, dropped_slots_lock = 0; bool req_int_win = !irqchip_in_kernel(vcpu->kvm) && vcpu->run->request_interrupt_window; @@ -3616,7 +3616,10 @@ static int vcpu_enter_guest(struct kvm_v kvm_lapic_sync_to_vapic(vcpu); } - up_read(&vcpu->kvm->slots_lock); + if (unlikely(test_bit(KVM_VCPU_DROP_LOCK, &vcpu->vcpu_state))) { + dropped_slots_lock = 1; + up_read(&vcpu->kvm->slots_lock); + } kvm_guest_enter(); @@ -3668,8 +3671,8 @@ static int vcpu_enter_guest(struct kvm_v preempt_enable(); - down_read(&vcpu->kvm->slots_lock); - + if (dropped_slots_lock) + down_read(&vcpu->kvm->slots_lock); /* * Profile KVM exit RIPs: */ Index: kvm-requests/include/linux/kvm_host.h =================================================================== --- kvm-requests.orig/include/linux/kvm_host.h +++ kvm-requests/include/linux/kvm_host.h @@ -44,6 +44,7 @@ #define KVM_VCPU_GUEST_MODE 0 #define KVM_VCPU_KICKED 1 +#define KVM_VCPU_DROP_LOCK 2 struct kvm; struct kvm_vcpu; @@ -408,6 +409,7 @@ void kvm_unregister_irq_ack_notifier(str struct kvm_irq_ack_notifier *kian); int kvm_request_irq_source_id(struct kvm *kvm); void kvm_free_irq_source_id(struct kvm *kvm, int irq_source_id); +void kvm_grab_global_lock(struct kvm *kvm); /* For vcpu->arch.iommu_flags */ #define KVM_IOMMU_CACHE_COHERENCY 0x1 Index: kvm-requests/virt/kvm/coalesced_mmio.c =================================================================== --- kvm-requests.orig/virt/kvm/coalesced_mmio.c +++ kvm-requests/virt/kvm/coalesced_mmio.c @@ -117,7 +117,7 @@ int kvm_vm_ioctl_register_coalesced_mmio if (dev == NULL) return -EINVAL; - down_write(&kvm->slots_lock); + kvm_grab_global_lock(kvm); if (dev->nb_zones >= KVM_COALESCED_MMIO_ZONE_MAX) { up_write(&kvm->slots_lock); return -ENOBUFS; @@ -140,7 +140,7 @@ int kvm_vm_ioctl_unregister_coalesced_mm if (dev == NULL) return -EINVAL; - down_write(&kvm->slots_lock); + kvm_grab_global_lock(kvm); i = dev->nb_zones; while(i) { Index: kvm-requests/virt/kvm/eventfd.c =================================================================== --- kvm-requests.orig/virt/kvm/eventfd.c +++ kvm-requests/virt/kvm/eventfd.c @@ -498,7 +498,7 @@ kvm_assign_ioeventfd(struct kvm *kvm, st else p->wildcard = true; - down_write(&kvm->slots_lock); + kvm_grab_global_lock(kvm); /* Verify that there isnt a match already */ if (ioeventfd_check_collision(kvm, p)) { @@ -541,7 +541,7 @@ kvm_deassign_ioeventfd(struct kvm *kvm, if (IS_ERR(eventfd)) return PTR_ERR(eventfd); - down_write(&kvm->slots_lock); + kvm_grab_global_lock(kvm); list_for_each_entry_safe(p, tmp, &kvm->ioeventfds, list) { bool wildcard = !(args->flags & KVM_IOEVENTFD_FLAG_DATAMATCH); Index: kvm-requests/virt/kvm/kvm_main.c =================================================================== --- kvm-requests.orig/virt/kvm/kvm_main.c +++ kvm-requests/virt/kvm/kvm_main.c @@ -787,6 +787,22 @@ void kvm_reload_remote_mmus(struct kvm * kvm_vcpus_request(kvm, KVM_REQ_MMU_RELOAD); } +void kvm_grab_global_lock(struct kvm *kvm) +{ + int i; + struct kvm_vcpu *vcpu; + + kvm_for_each_vcpu(i, vcpu, kvm) { + set_bit(KVM_VCPU_DROP_LOCK, &vcpu->vcpu_state); + barrier(); + kvm_vcpu_ipi(vcpu); + } + down_write(&kvm->slots_lock); + kvm_for_each_vcpu(i, vcpu, kvm) + clear_bit(KVM_VCPU_DROP_LOCK, &vcpu->vcpu_state); +} +EXPORT_SYMBOL_GPL(kvm_grab_global_lock); + int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id) { struct page *page; @@ -1286,7 +1302,7 @@ int kvm_set_memory_region(struct kvm *kv { int r; - down_write(&kvm->slots_lock); + kvm_grab_global_lock(kvm); r = __kvm_set_memory_region(kvm, mem, user_alloc); up_write(&kvm->slots_lock); return r; @@ -2556,7 +2572,7 @@ int kvm_io_bus_register_dev(struct kvm * { int ret; - down_write(&kvm->slots_lock); + kvm_grab_global_lock(kvm); ret = __kvm_io_bus_register_dev(bus, dev); up_write(&kvm->slots_lock); @@ -2579,7 +2595,7 @@ void kvm_io_bus_unregister_dev(struct kv struct kvm_io_bus *bus, struct kvm_io_device *dev) { - down_write(&kvm->slots_lock); + kvm_grab_global_lock(kvm); __kvm_io_bus_unregister_dev(bus, dev); up_write(&kvm->slots_lock); } -- To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
