From: Paolo Bonzini <[email protected]> Each plane will have its own set of VCPUs, so move the vcpu_array to the plane structure.
Signed-off-by: Paolo Bonzini <[email protected]> Co-developed-by: Joerg Roedel <[email protected]> Signed-off-by: Joerg Roedel <[email protected]> --- include/linux/kvm_host.h | 30 +++++++++++++++++++++--------- virt/kvm/kvm_main.c | 33 +++++++++++++++++++++++++-------- 2 files changed, 46 insertions(+), 17 deletions(-) diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 5be4c9f118b4..5a72f73a2f31 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -324,12 +324,14 @@ struct kvm_mmio_fragment { struct kvm_vcpu { struct kvm *kvm; + struct kvm_plane *plane; + #ifdef CONFIG_PREEMPT_NOTIFIERS struct preempt_notifier preempt_notifier; #endif int cpu; int vcpu_id; /* id given by userspace at creation */ - int vcpu_idx; /* index into kvm->vcpu_array */ + int vcpu_idx; /* index into kvm->planes[]->vcpu_array */ int ____srcu_idx; /* Don't use this directly. You've been warned. */ #ifdef CONFIG_PROVE_RCU int srcu_depth; @@ -770,6 +772,9 @@ struct kvm_memslots { struct kvm_plane { struct kvm *kvm; unsigned level; + + /* Per-Plane VCPU array */ + struct xarray vcpu_array; }; struct kvm { @@ -795,7 +800,6 @@ struct kvm { struct kvm_memslots __memslots[KVM_MAX_NR_ADDRESS_SPACES][2]; /* The current active memslot set for each address space */ struct kvm_memslots __rcu *memslots[KVM_MAX_NR_ADDRESS_SPACES]; - struct xarray vcpu_array; /* * Protected by slots_lock, but can be read outside if an * incorrect answer is acceptable. @@ -996,9 +1000,9 @@ static inline struct kvm_io_bus *kvm_get_bus(struct kvm *kvm, enum kvm_bus idx) lockdep_is_held(&kvm->slots_lock)); } -static inline struct kvm_vcpu *kvm_get_vcpu(struct kvm *kvm, int i) +static inline struct kvm_vcpu *plane_get_vcpu(struct kvm_plane *plane, int i) { - int num_vcpus = atomic_read(&kvm->online_vcpus); + int num_vcpus = atomic_read(&plane->kvm->online_vcpus); /* * Explicitly verify the target vCPU is online, as the anti-speculation @@ -1012,13 +1016,21 @@ static inline struct kvm_vcpu *kvm_get_vcpu(struct kvm *kvm, int i) /* Pairs with smp_wmb() in kvm_vm_ioctl_create_vcpu. */ smp_rmb(); - return xa_load(&kvm->vcpu_array, i); + return xa_load(&plane->vcpu_array, i); } -#define kvm_for_each_vcpu(idx, vcpup, kvm) \ - if (atomic_read(&kvm->online_vcpus)) \ - xa_for_each_range(&kvm->vcpu_array, idx, vcpup, 0, \ - (atomic_read(&kvm->online_vcpus) - 1)) +static inline struct kvm_vcpu *kvm_get_vcpu(struct kvm *kvm, int i) +{ + return plane_get_vcpu(kvm->planes[0], i); +} + +#define plane_for_each_vcpu(idx, vcpup, plane) \ + if (atomic_read(&plane->kvm->online_vcpus)) \ + xa_for_each_range(&plane->vcpu_array, idx, vcpup, 0, \ + (atomic_read(&plane->kvm->online_vcpus) - 1)) + +#define kvm_for_each_vcpu(idx, vcpup, kvm) \ + plane_for_each_vcpu(idx, vcpup, kvm->planes[0]) static inline struct kvm_vcpu *kvm_get_vcpu_by_id(struct kvm *kvm, int id) { diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index a68469c6d12e..668645dd3945 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -443,6 +443,7 @@ static void kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id) mutex_init(&vcpu->mutex); vcpu->cpu = -1; vcpu->kvm = kvm; + vcpu->plane = kvm->planes[0]; vcpu->vcpu_id = id; vcpu->pid = NULL; rwlock_init(&vcpu->pid_lock); @@ -479,14 +480,14 @@ static void kvm_vcpu_destroy(struct kvm_vcpu *vcpu) kmem_cache_free(kvm_vcpu_cache, vcpu); } -void kvm_destroy_vcpus(struct kvm *kvm) +static void plane_destroy_vcpus(struct kvm_plane *plane) { unsigned long i; struct kvm_vcpu *vcpu; - kvm_for_each_vcpu(i, vcpu, kvm) { + plane_for_each_vcpu(i, vcpu, plane) { kvm_vcpu_destroy(vcpu); - xa_erase(&kvm->vcpu_array, i); + xa_erase(&plane->vcpu_array, i); /* * Assert that the vCPU isn't visible in any way, to ensure KVM @@ -494,7 +495,22 @@ void kvm_destroy_vcpus(struct kvm *kvm) * in VM-wide request, e.g. to flush remote TLBs when tearing * down MMUs, or to mark the VM dead if a KVM_BUG_ON() fires. */ - WARN_ON_ONCE(xa_load(&kvm->vcpu_array, i) || kvm_get_vcpu(kvm, i)); + WARN_ON_ONCE(xa_load(&plane->vcpu_array, i) || plane_get_vcpu(plane, i)); + } + +} + +void kvm_destroy_vcpus(struct kvm *kvm) +{ + unsigned lvl; + + for (lvl = KVM_MAX_PLANES; lvl > 0; lvl--) { + struct kvm_plane *plane = kvm->planes[lvl - 1]; + + if (plane == NULL) + continue; + + plane_destroy_vcpus(plane); } atomic_set(&kvm->online_vcpus, 0); @@ -1105,6 +1121,8 @@ static struct kvm_plane *kvm_create_plane(struct kvm *kvm, unsigned plane_level) plane->kvm = kvm; plane->level = plane_level; + xa_init(&plane->vcpu_array); + kvm->planes[plane_level] = plane; return plane; @@ -1146,7 +1164,6 @@ static struct kvm *kvm_create_vm(unsigned long type, const char *fdname) mutex_init(&kvm->slots_arch_lock); spin_lock_init(&kvm->mn_invalidate_lock); rcuwait_init(&kvm->mn_memslots_update_rcuwait); - xa_init(&kvm->vcpu_array); #ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES xa_init(&kvm->mem_attr_array); #endif @@ -4039,7 +4056,7 @@ void kvm_vcpu_on_spin(struct kvm_vcpu *me, bool yield_to_kernel_mode) if (idx == me->vcpu_idx) continue; - vcpu = xa_load(&kvm->vcpu_array, idx); + vcpu = xa_load(&kvm->planes[0]->vcpu_array, idx); if (!READ_ONCE(vcpu->ready)) continue; if (kvm_vcpu_is_blocking(vcpu) && !vcpu_dy_runnable(vcpu)) @@ -4258,7 +4275,7 @@ static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, unsigned long id) } vcpu->vcpu_idx = atomic_read(&kvm->online_vcpus); - r = xa_insert(&kvm->vcpu_array, vcpu->vcpu_idx, vcpu, GFP_KERNEL_ACCOUNT); + r = xa_insert(&kvm->planes[0]->vcpu_array, vcpu->vcpu_idx, vcpu, GFP_KERNEL_ACCOUNT); WARN_ON_ONCE(r == -EBUSY); if (r) goto unlock_vcpu_destroy; @@ -4293,7 +4310,7 @@ static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, unsigned long id) kvm_put_xa_erase: mutex_unlock(&vcpu->mutex); kvm_put_kvm_no_destroy(kvm); - xa_erase(&kvm->vcpu_array, vcpu->vcpu_idx); + xa_erase(&kvm->planes[0]->vcpu_array, vcpu->vcpu_idx); unlock_vcpu_destroy: mutex_unlock(&kvm->lock); kvm_dirty_ring_free(&vcpu->dirty_ring); -- 2.53.0
